diff --git a/apps/gittensory-ui/public/openapi.json b/apps/gittensory-ui/public/openapi.json index cb340760cf..709c2fcd5a 100644 --- a/apps/gittensory-ui/public/openapi.json +++ b/apps/gittensory-ui/public/openapi.json @@ -15216,6 +15216,26 @@ } ] } + }, + "/v1/orb/ingest": { + "post": { + "responses": { + "200": { + "description": "Batch accepted; returns { accepted: number }" + }, + "400": { + "description": "Malformed JSON or invalid payload shape" + } + }, + "security": [ + { + "GittensoryBearer": [] + }, + { + "GittensorySessionCookie": [] + } + ] + } } }, "servers": [ diff --git a/migrations/0058_orb_signals.sql b/migrations/0058_orb_signals.sql new file mode 100644 index 0000000000..669235c3f0 --- /dev/null +++ b/migrations/0058_orb_signals.sql @@ -0,0 +1,16 @@ +-- Gittensory Orb (#1219): central collector store. Receives anonymized outcome signal batches +-- from self-hosted instances running exportOrbBatch. repo_hash and pr_hash are HMAC-anonymized +-- by the sender — no repo names, owner identifiers, or PR content is stored here. +CREATE TABLE IF NOT EXISTS orb_signals ( + id INTEGER PRIMARY KEY, + instance_id TEXT NOT NULL, + repo_hash TEXT NOT NULL, + pr_hash TEXT NOT NULL, + outcome TEXT NOT NULL CHECK (outcome IN ('merged', 'closed')), + gate_verdict TEXT, + time_to_close_ms INTEGER, + sent_at TEXT, + received_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE (instance_id, pr_hash) +); +CREATE INDEX IF NOT EXISTS orb_signals_instance ON orb_signals (instance_id, received_at); diff --git a/src/api/routes.ts b/src/api/routes.ts index 2bf8a04e17..64d54426ab 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -122,6 +122,7 @@ import { type GittensoryMentionCommandName, } from "../github/commands"; import { handleGitHubWebhook } from "../github/webhook"; +import { handleOrbIngest } from "../orb/ingest"; import { handleMcpRequest } from "../mcp/server"; import { buildOpenApiSpec } from "../openapi/spec"; import { generateSignalSnapshots } from "../queue/processors"; @@ -2862,6 +2863,17 @@ export function createApp() { app.post("/v1/github/webhook", handleGitHubWebhook); + // Gittensory Orb (#1219) — central collector. Receives anonymized outcome signal batches + // from self-hosted instances. No auth required: all data is HMAC-anonymized by the sender; + // dedup is enforced via UNIQUE(instance_id, pr_hash) in orb_signals. + app.post("/v1/orb/ingest", async (c) => { + const body = await c.req.text().catch(() => null); + if (!body) return c.json({ error: "invalid_request" }, 400); + const result = await handleOrbIngest(body, c.env.DB); + if ("error" in result) return c.json(result, 400); + return c.json(result, 200); + }); + // Convergence (ops / observability, flag GITTENSORY_REVIEW_OPS). Cross-repo review-OUTCOME aggregate (gate-block // ledger + recommendation/slop calibration) for an operator dashboard. Bearer-gated by the `/v1/internal/*` // middleware above (INTERNAL_JOB_TOKEN). Flag-OFF (default) → 404, so the endpoint does not exist and the @@ -4808,6 +4820,7 @@ function requiresApiToken(path: string): boolean { if (path === "/v1/drafts" || path.startsWith("/v1/drafts/")) return false; if (path.startsWith("/v1/auth/")) return false; if (path === "/v1/github/webhook") return false; + if (path === "/v1/orb/ingest") return false; if (path.startsWith("/v1/internal/")) return false; return path.startsWith("/v1/"); } diff --git a/src/openapi/spec.ts b/src/openapi/spec.ts index 36ad6d74dc..451147d957 100644 --- a/src/openapi/spec.ts +++ b/src/openapi/spec.ts @@ -664,6 +664,14 @@ export function buildOpenApiSpec() { 401: { description: "Invalid webhook signature" }, }, }); + registry.registerPath({ + method: "post", + path: "/v1/orb/ingest", + responses: { + 200: { description: "Batch accepted; returns { accepted: number }" }, + 400: { description: "Malformed JSON or invalid payload shape" }, + }, + }); registry.registerPath({ method: "get", path: "/v1/auth/github/start", diff --git a/src/orb/ingest.ts b/src/orb/ingest.ts new file mode 100644 index 0000000000..6e91ddbcfb --- /dev/null +++ b/src/orb/ingest.ts @@ -0,0 +1,81 @@ +// Gittensory Orb (#1219) — central collector receiver. +// Accepts anonymized outcome signal batches from self-hosted instances running exportOrbBatch. +// No raw repo names, owner identifiers, or PR content is accepted or stored — only HMAC-anonymized +// hashes + aggregate outcome metadata (verdict, timing). + +const MAX_BATCH = 500; +const VALID_OUTCOMES = new Set(["merged", "closed"]); + +interface OrbIngestEvent { + repo_hash: string; + pr_hash: string; + outcome: string; + gate_verdict?: string | null; + time_to_close_ms?: number | null; + created_at?: string | null; +} + +interface OrbIngestPayload { + instance_id: string; + events: OrbIngestEvent[]; +} + +export type OrbIngestResult = { accepted: number } | { error: string }; + +export async function handleOrbIngest(body: string, db: D1Database): Promise { + let payload: unknown; + try { + payload = JSON.parse(body); + } catch { + return { error: "invalid_json" }; + } + + if ( + typeof (payload as OrbIngestPayload)?.instance_id !== "string" || + !Array.isArray((payload as OrbIngestPayload)?.events) + ) { + return { error: "invalid_payload" }; + } + + const { instance_id, events } = payload as OrbIngestPayload; + if (!instance_id || events.length === 0) { + return { error: "invalid_payload" }; + } + + const batch = events.slice(0, MAX_BATCH); + let accepted = 0; + + for (const event of batch) { + if ( + typeof event.repo_hash !== "string" || !event.repo_hash || + typeof event.pr_hash !== "string" || !event.pr_hash || + !VALID_OUTCOMES.has(event.outcome) + ) { + continue; + } + + try { + const result = await db + .prepare( + `INSERT OR IGNORE INTO orb_signals + (instance_id, repo_hash, pr_hash, outcome, gate_verdict, time_to_close_ms, sent_at) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + ) + .bind( + instance_id, + event.repo_hash, + event.pr_hash, + event.outcome, + typeof event.gate_verdict === "string" ? event.gate_verdict : null, + typeof event.time_to_close_ms === "number" ? event.time_to_close_ms : null, + typeof event.created_at === "string" ? event.created_at : null, + ) + .run(); + if (result.meta.changes > 0) accepted++; + } catch { + // best-effort — skip rows that violate constraints or hit transient errors + } + } + + return { accepted }; +} diff --git a/test/integration/orb-ingest.test.ts b/test/integration/orb-ingest.test.ts new file mode 100644 index 0000000000..1a25c33a42 --- /dev/null +++ b/test/integration/orb-ingest.test.ts @@ -0,0 +1,242 @@ +import { describe, expect, it } from "vitest"; +import { createApp } from "../../src/api/routes"; +import { handleOrbIngest } from "../../src/orb/ingest"; +import { createTestEnv, TestD1Database } from "../helpers/d1"; + +// ── handleOrbIngest unit-style tests ────────────────────────────────────────── + +describe("handleOrbIngest()", () => { + function makeDb(): D1Database { + return new TestD1Database() as unknown as D1Database; + } + + function makePayload(overrides: Record = {}): string { + return JSON.stringify({ + instance_id: "abc123def456abc0", + events: [ + { + repo_hash: "a1b2c3d4e5f6a1b2c3d4e5f6", + pr_hash: "f6e5d4c3b2a1f6e5d4c3b2a1", + outcome: "merged", + gate_verdict: "approve", + time_to_close_ms: 3600000, + created_at: "2024-01-01T00:00:00Z", + }, + ], + ...overrides, + }); + } + + it("accepts a valid batch and returns accepted count", async () => { + const db = makeDb(); + const result = await handleOrbIngest(makePayload(), db); + expect(result).toEqual({ accepted: 1 }); + }); + + it("returns invalid_json when body is not valid JSON (covers JSON.parse catch branch)", async () => { + const db = makeDb(); + expect(await handleOrbIngest("{not json}", db)).toEqual({ error: "invalid_json" }); + }); + + it("returns invalid_payload when instance_id is not a string", async () => { + const db = makeDb(); + expect(await handleOrbIngest(JSON.stringify({ instance_id: 123, events: [] }), db)).toEqual({ error: "invalid_payload" }); + }); + + it("returns invalid_payload when events is not an array (covers !Array.isArray branch)", async () => { + const db = makeDb(); + expect(await handleOrbIngest(JSON.stringify({ instance_id: "abc", events: "bad" }), db)).toEqual({ error: "invalid_payload" }); + }); + + it("returns invalid_payload when instance_id is an empty string (covers !instance_id branch)", async () => { + const db = makeDb(); + expect(await handleOrbIngest(JSON.stringify({ instance_id: "", events: [{ repo_hash: "a", pr_hash: "b", outcome: "merged" }] }), db)).toEqual({ error: "invalid_payload" }); + }); + + it("returns invalid_payload when events array is empty (covers events.length === 0 branch)", async () => { + const db = makeDb(); + expect(await handleOrbIngest(JSON.stringify({ instance_id: "abc", events: [] }), db)).toEqual({ error: "invalid_payload" }); + }); + + it("skips events with a non-string repo_hash (covers typeof repo_hash !== string branch)", async () => { + const db = makeDb(); + const result = await handleOrbIngest( + JSON.stringify({ instance_id: "inst1", events: [{ repo_hash: 99, pr_hash: "hash", outcome: "merged" }] }), + db, + ); + expect(result).toEqual({ accepted: 0 }); + }); + + it("skips events with an empty repo_hash (covers !repo_hash branch)", async () => { + const db = makeDb(); + const result = await handleOrbIngest( + JSON.stringify({ instance_id: "inst1", events: [{ repo_hash: "", pr_hash: "hash", outcome: "merged" }] }), + db, + ); + expect(result).toEqual({ accepted: 0 }); + }); + + it("skips events with a non-string pr_hash (covers typeof pr_hash !== string branch)", async () => { + const db = makeDb(); + const result = await handleOrbIngest( + JSON.stringify({ instance_id: "inst1", events: [{ repo_hash: "rhash", pr_hash: null, outcome: "merged" }] }), + db, + ); + expect(result).toEqual({ accepted: 0 }); + }); + + it("skips events with an empty pr_hash (covers !pr_hash branch)", async () => { + const db = makeDb(); + const result = await handleOrbIngest( + JSON.stringify({ instance_id: "inst1", events: [{ repo_hash: "rhash", pr_hash: "", outcome: "merged" }] }), + db, + ); + expect(result).toEqual({ accepted: 0 }); + }); + + it("skips events with an invalid outcome (covers !VALID_OUTCOMES.has branch)", async () => { + const db = makeDb(); + const result = await handleOrbIngest( + JSON.stringify({ instance_id: "inst1", events: [{ repo_hash: "rh", pr_hash: "ph", outcome: "opened" }] }), + db, + ); + expect(result).toEqual({ accepted: 0 }); + }); + + it("stores null gate_verdict when field is absent (covers typeof gate_verdict !== string branch)", async () => { + const db = makeDb(); + await handleOrbIngest( + JSON.stringify({ instance_id: "inst1", events: [{ repo_hash: "rh1", pr_hash: "ph1", outcome: "closed" }] }), + db, + ); + const row = await (db as unknown as TestD1Database).prepare("SELECT gate_verdict FROM orb_signals WHERE pr_hash='ph1'").first<{ gate_verdict: string | null }>(); + expect(row?.gate_verdict).toBeNull(); + }); + + it("stores gate_verdict string when field is present (covers typeof gate_verdict === string branch)", async () => { + const db = makeDb(); + await handleOrbIngest( + JSON.stringify({ instance_id: "inst1", events: [{ repo_hash: "rh2", pr_hash: "ph2", outcome: "merged", gate_verdict: "approve" }] }), + db, + ); + const row = await (db as unknown as TestD1Database).prepare("SELECT gate_verdict FROM orb_signals WHERE pr_hash='ph2'").first<{ gate_verdict: string | null }>(); + expect(row?.gate_verdict).toBe("approve"); + }); + + it("stores null time_to_close_ms when field is absent (covers typeof time_to_close_ms !== number branch)", async () => { + const db = makeDb(); + await handleOrbIngest( + JSON.stringify({ instance_id: "inst1", events: [{ repo_hash: "rh3", pr_hash: "ph3", outcome: "closed" }] }), + db, + ); + const row = await (db as unknown as TestD1Database).prepare("SELECT time_to_close_ms FROM orb_signals WHERE pr_hash='ph3'").first<{ time_to_close_ms: number | null }>(); + expect(row?.time_to_close_ms).toBeNull(); + }); + + it("stores time_to_close_ms when field is a number (covers typeof time_to_close_ms === number branch)", async () => { + const db = makeDb(); + await handleOrbIngest( + JSON.stringify({ instance_id: "inst1", events: [{ repo_hash: "rh4", pr_hash: "ph4", outcome: "merged", time_to_close_ms: 7200000 }] }), + db, + ); + const row = await (db as unknown as TestD1Database).prepare("SELECT time_to_close_ms FROM orb_signals WHERE pr_hash='ph4'").first<{ time_to_close_ms: number | null }>(); + expect(row?.time_to_close_ms).toBe(7200000); + }); + + it("stores null sent_at when created_at is absent (covers typeof created_at !== string branch)", async () => { + const db = makeDb(); + await handleOrbIngest( + JSON.stringify({ instance_id: "inst1", events: [{ repo_hash: "rh5", pr_hash: "ph5", outcome: "merged" }] }), + db, + ); + const row = await (db as unknown as TestD1Database).prepare("SELECT sent_at FROM orb_signals WHERE pr_hash='ph5'").first<{ sent_at: string | null }>(); + expect(row?.sent_at).toBeNull(); + }); + + it("stores sent_at when created_at is a string (covers typeof created_at === string branch)", async () => { + const db = makeDb(); + await handleOrbIngest( + JSON.stringify({ instance_id: "inst1", events: [{ repo_hash: "rh6", pr_hash: "ph6", outcome: "merged", created_at: "2024-06-01T12:00:00Z" }] }), + db, + ); + const row = await (db as unknown as TestD1Database).prepare("SELECT sent_at FROM orb_signals WHERE pr_hash='ph6'").first<{ sent_at: string | null }>(); + expect(row?.sent_at).toBe("2024-06-01T12:00:00Z"); + }); + + it("deduplicates via INSERT OR IGNORE — second insert is not counted (covers result.meta.changes === 0 branch)", async () => { + const db = makeDb(); + const body = JSON.stringify({ instance_id: "inst1", events: [{ repo_hash: "rh7", pr_hash: "ph7", outcome: "merged" }] }); + expect(await handleOrbIngest(body, db)).toEqual({ accepted: 1 }); + expect(await handleOrbIngest(body, db)).toEqual({ accepted: 0 }); // duplicate ignored + }); + + it("counts both accepted and skipped events in the same batch", async () => { + const db = makeDb(); + const result = await handleOrbIngest( + JSON.stringify({ + instance_id: "inst1", + events: [ + { repo_hash: "rh8", pr_hash: "ph8", outcome: "merged" }, + { repo_hash: "", pr_hash: "ph9", outcome: "merged" }, // invalid — skipped + { repo_hash: "rh10", pr_hash: "ph10", outcome: "invalid" }, // invalid outcome — skipped + ], + }), + db, + ); + expect(result).toEqual({ accepted: 1 }); + }); + + it("caps batch at 500 events (MAX_BATCH) — extra events not inserted", async () => { + const db = makeDb(); + const events = Array.from({ length: 501 }, (_, i) => ({ + repo_hash: `rh${i}`, + pr_hash: `ph${i}`, + outcome: "merged" as const, + })); + const result = await handleOrbIngest(JSON.stringify({ instance_id: "inst-batch", events }), db); + expect(result).toEqual({ accepted: 500 }); + }); + + it("does not throw when the DB throws on insert (covers inner catch branch)", async () => { + const brokenDb = { + prepare: () => ({ bind: () => ({ run: () => Promise.reject(new Error("disk full")) }) }), + } as unknown as D1Database; + const result = await handleOrbIngest( + JSON.stringify({ instance_id: "inst1", events: [{ repo_hash: "rh", pr_hash: "ph", outcome: "merged" }] }), + brokenDb, + ); + expect(result).toEqual({ accepted: 0 }); + }); +}); + +// ── Route integration tests (covers routes.ts new lines) ────────────────────── + +describe("POST /v1/orb/ingest route", () => { + const app = createApp(); + + it("returns 200 with accepted count for a valid batch", async () => { + const env = createTestEnv(); + const body = JSON.stringify({ + instance_id: "abc123def456abc0", + events: [{ repo_hash: "rhash1234567890123456", pr_hash: "phash1234567890123456", outcome: "merged" }], + }); + const res = await app.request("/v1/orb/ingest", { method: "POST", headers: { "content-type": "application/json" }, body }, env); + expect(res.status).toBe(200); + const json = await res.json() as { accepted: number }; + expect(json.accepted).toBe(1); + }); + + it("returns 400 for invalid JSON (covers error-in-result branch)", async () => { + const env = createTestEnv(); + const res = await app.request("/v1/orb/ingest", { method: "POST", headers: { "content-type": "application/json" }, body: "{bad" }, env); + expect(res.status).toBe(400); + const json = await res.json() as { error: string }; + expect(json.error).toBe("invalid_json"); + }); + + it("returns 400 for an empty body (covers !body branch in route)", async () => { + const env = createTestEnv(); + const res = await app.request("/v1/orb/ingest", { method: "POST", body: "" }, env); + expect(res.status).toBe(400); + }); +});