Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions packages/loopover-engine/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@
"types": "./dist/index.d.ts",
"default": "./dist/index.js"
},
"./parse-pull-request-target-key": {
"types": "./dist/parse-pull-request-target-key.d.ts",
"default": "./dist/parse-pull-request-target-key.js"
},
"./scoring/model": {
"types": "./dist/scoring/model.d.ts",
"default": "./dist/scoring/model.js"
Expand Down
5 changes: 4 additions & 1 deletion src/db/repositories.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
import { parsePullRequestTargetKey } from "@loopover/engine";
// Subpath import, not the engine barrel: this file needs exactly ONE tiny parser, and the barrel
// (dist/index.js re-exporting calibration/advisory/policy modules) measured ~420ms of cold import
// under vitest — a tax paid by every test file that transitively touches repositories (#test-import-cost).
import { parsePullRequestTargetKey } from "@loopover/engine/parse-pull-request-target-key";
import { and, asc, desc, eq, gte, inArray, isNotNull, lt, not, or, sql, type SQL } from "drizzle-orm";
import { getDb } from "./client";
import {
Expand Down
15 changes: 12 additions & 3 deletions src/github/client.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import { Octokit } from "@octokit/core";
import { isGlobalAgentFrozen, recordAuditEvent } from "../db/repositories";
import { isGlobalAgentPause, resolveAgentActionMode, type AgentActionMode } from "../settings/agent-execution";
import { incr } from "../selfhost/metrics";
import type { RepositorySettings } from "../types";
Expand Down Expand Up @@ -32,8 +31,10 @@ export const GITHUB_RESPONSE_CACHE_REPLAY_HEADER = "x-loopover-cache";
/** The single source of truth for the product's outbound User-Agent, used by every raw-`fetch`/`timeoutFetch`
* call across `src/` that identifies itself generically (as opposed to a service-specific variant like the
* self-host or content-lane User-Agent). Consolidates what had drifted into ~16 independently hardcoded
* copies of the same literal. */
export const PRODUCT_USER_AGENT = "loopover/0.1";
* copies of the same literal. Defined in ./user-agent (a leaf module) so constant-only importers don't pay
* this file's import graph (#test-import-cost); re-exported here so existing importers are unchanged. */
import { PRODUCT_USER_AGENT } from "./user-agent";
export { PRODUCT_USER_AGENT };

/** The single shared GitHub REST header-builder for every raw-`fetch`/`timeoutFetch` call in `src/` (Octokit
* calls set their own headers internally and don't need this). Consolidates four independent, drifted
Expand Down Expand Up @@ -640,6 +641,12 @@ const WRITE_METHODS = new Set(["POST", "PATCH", "PUT", "DELETE"]);
* per-write hot path.
*/
export async function resolveRepoActionMode(env: Env, settings: Pick<RepositorySettings, "agentPaused" | "agentDryRun"> | null | undefined): Promise<AgentActionMode> {
// Lazy import (#test-import-cost): db/repositories is this file's only heavy dependency, needed by just
// this function and the suppressed-write audit hook below — a static import re-created the client ↔
// repositories cycle (~1.1s cold import under vitest) for every importer of this module. Module-cached
// after the first call, so the per-call cost is a resolved-promise tick; same idiom as
// processors.ts's own `await import("../github/pr-command-request")`.
const { isGlobalAgentFrozen } = await import("../db/repositories");
return resolveAgentActionMode({
globalPaused: isGlobalAgentPause(env) || (await isGlobalAgentFrozen(env)),
agentPaused: settings?.agentPaused,
Expand Down Expand Up @@ -701,6 +708,8 @@ export function makeInstallationOctokit(env: Env, token: string, mode: AgentActi
const method = options.method.toUpperCase();
if (!WRITE_METHODS.has(method)) return request(options); // reads + create-vs-update probes always run
const url = options.url;
// Same lazy-import reasoning as resolveRepoActionMode above (#test-import-cost).
const { recordAuditEvent } = await import("../db/repositories");
await recordAuditEvent(env, {
eventType: "github.write.suppressed",
actor: "loopover",
Expand Down
5 changes: 5 additions & 0 deletions src/github/user-agent.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
// The product User-Agent, in its own leaf module (#test-import-cost): several modules (notably
// src/gittensor/api.ts, itself imported by db/repositories) need ONLY this constant, and importing it
// from ./client dragged the whole client ↔ repositories dependency cycle (~1.1s of cold import under
// vitest) into every one of their importers. ./client re-exports it, so existing importers are unchanged.
export const PRODUCT_USER_AGENT = "loopover/0.1";
5 changes: 4 additions & 1 deletion src/gittensor/api.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import type { ContributorRepoStatRecord } from "../types";
import { PRODUCT_USER_AGENT } from "../github/client";
// The leaf module, NOT ../github/client: this file needs only the constant, and the client import
// dragged the client ↔ repositories cycle (~1.1s cold import under vitest) into db/repositories'
// graph — the single most-imported path in the test suite (#test-import-cost).
import { PRODUCT_USER_AGENT } from "../github/user-agent";
import { errorMessage } from "../utils/json";

const GITTENSOR_API_BASE = "https://api.gittensor.io";
Expand Down
10 changes: 5 additions & 5 deletions src/review/rag-index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ import {
filePriority,
getStoredChunkMeta,
isIndexablePath,
MAX_CHUNKS_PER_REPO,
maxChunksPerRepo,
MAX_FILE_BYTES,
ragNamespace,
type RagChunk,
Expand Down Expand Up @@ -209,8 +209,8 @@ async function upsertChunksCapped(env: Env, project: string, repo: string, chunk
const infra = createReviewAdapters(env);
let stored = alreadyStored;
let upserted = 0;
for (let i = 0; i < chunks.length && stored < MAX_CHUNKS_PER_REPO; i += UPSERT_BATCH) {
const remaining = MAX_CHUNKS_PER_REPO - stored;
for (let i = 0; i < chunks.length && stored < maxChunksPerRepo(); i += UPSERT_BATCH) {
const remaining = maxChunksPerRepo() - stored;
const batch = chunks.slice(i, i + Math.min(UPSERT_BATCH, remaining));
if (batch.length === 0) break;
const n = await upsertChunks(infra, project, repo, batch, blobSha);
Expand Down Expand Up @@ -324,7 +324,7 @@ export async function indexRepo(
skipped += 1;
continue; // unchanged since the last full index — skip the fetch/chunk/embed entirely
}
if (stored >= MAX_CHUNKS_PER_REPO && (!known || known.count <= 0)) {
if (stored >= maxChunksPerRepo() && (!known || known.count <= 0)) {
capped = true;
break;
}
Expand Down Expand Up @@ -395,7 +395,7 @@ export async function reindexChangedPaths(
let filesIndexed = 0;
let capped = false;
for (const path of indexable) {
if (stored >= MAX_CHUNKS_PER_REPO) {
if (stored >= maxChunksPerRepo()) {
capped = true;
break;
}
Expand Down
12 changes: 12 additions & 0 deletions src/review/rag.ts
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,18 @@ const CHUNK_OVERLAP = 1500;
* recurring one per cron cycle (#4365's blob-SHA skip-cache). Self-host only: this is not a Cloudflare
* free-tier constraint on this deployment, but the name/comment history predates self-host. */
export const MAX_CHUNKS_PER_REPO = 4000;
// Test-only override (#test-hotspots): the cap tests exist to pin capping BEHAVIOR, not the number
// 4000 — building 4,000 real chunk rows per cap test made rag-index.test.ts one of the suite's
// slowest files (~7s per cap test). Same `...ForTest` hook convention as
// clearInstallationTokenCacheForTest / clearGitHubResponseCacheForTest; production call sites read
// maxChunksPerRepo() and never touch the override.
let maxChunksPerRepoOverride: number | null = null;
export function maxChunksPerRepo(): number {
return maxChunksPerRepoOverride ?? MAX_CHUNKS_PER_REPO;
}
export function setMaxChunksPerRepoForTest(value: number | null): void {
maxChunksPerRepoOverride = value;
}
const EMBED_BATCH = 96; // Workers AI caps embedding input at 100 items/call; kept as a conservative general
// bound — other embed providers (Ollama/vLLM/etc via the self-host adapter) may not share this exact cap.
const MAX_CONTEXT_CHARS = 14000; // bound the injected block (mirrors diff/knowledge budgets)
Expand Down
20 changes: 18 additions & 2 deletions test/unit/rag-index.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,15 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { indexRepo, reindexChangedPaths } from "../../src/review/rag-index";
import { MAX_CHUNKS_PER_REPO, MAX_FILE_BYTES, RAG_DIMENSIONS, ragNamespace } from "../../src/review/rag";
import { MAX_FILE_BYTES, RAG_DIMENSIONS, maxChunksPerRepo, ragNamespace, setMaxChunksPerRepoForTest } from "../../src/review/rag";

// #test-hotspots: the cap tests pin capping BEHAVIOR, not the production constant (4000) — building
// 4,000 real chunk rows per cap test made this file one of the suite's slowest (~7s per cap test).
// The whole file runs with a small cap via setMaxChunksPerRepoForTest: cap tests hit it at 24 rows,
// and no other fixture in this file indexes anywhere near 24 files, so their semantics are unchanged.
// Keeps the production constant's NAME so every existing test body reads exactly as before.
const MAX_CHUNKS_PER_REPO = 24;
beforeEach(() => setMaxChunksPerRepoForTest(MAX_CHUNKS_PER_REPO));
afterEach(() => setMaxChunksPerRepoForTest(null));
import { processJob, splitRepoForRag } from "../../src/queue/processors";
import { upsertRepositoryFromGitHub } from "../../src/db/repositories";
import { upsertRepoFocusManifest } from "../../src/signals/focus-manifest-loader";
Expand Down Expand Up @@ -96,6 +105,13 @@ async function pathsFor(env: Env, project: string, repo: string): Promise<string
return [...new Set((rows.results ?? []).map((r) => r.path))];
}

describe("maxChunksPerRepo test override", () => {
it("falls back to the production cap (4000) when no override is armed", () => {
setMaxChunksPerRepoForTest(null);
expect(maxChunksPerRepo()).toBe(4000);
});
});

describe("rag-index migration: repo_chunks exists in the test D1", () => {
it("the 0051 migration created repo_chunks (insert + read round-trips)", async () => {
const db = new TestD1Database() as unknown as D1Database;
Expand Down