diff --git a/docs/WIKI_BENCH.md b/docs/WIKI_BENCH.md new file mode 100644 index 000000000..d4fbdd24d --- /dev/null +++ b/docs/WIKI_BENCH.md @@ -0,0 +1,117 @@ +# wiki-bench v0 + +Retrieval quality benchmark over Tobi Lütke’s public-appearances wiki +(`tobi/wiki`). Ground-truth labels come from `meta/registry.json` and page +YAML frontmatter only — **no LLM-as-judge**. + +## Privacy: fixture only in this repo + +**This public repository ships the fixture (queries + expected paths/ids) +only.** It does **not** vendor wiki page bodies. + +| Artifact | Path | In public repo? | +|----------|------|-----------------| +| Fixture (queries + expected paths) | `src/bench/fixtures/wiki-v0.json` | Yes | +| Fixture schema test | `test/wiki-bench-fixture.test.ts` | Yes | +| Optional BM25 floors (env-gated) | `test/wiki-bench-bm25.test.ts` | Yes (skips without corpus) | +| Local BM25 runner (temp index) | `scripts/run-wiki-bench-local.mjs` | Yes (requires local corpus) | +| Wiki markdown corpus | private `tobi/wiki` (or a local checkout) | **No — never commit** | + +Do **not** add `test/wiki-bench-docs/**` or any wiki `.md` bodies to this repo. +CI must stay green without access to the private corpus. + +Expected file paths in the fixture are **wiki-relative**: +`concepts|sources|entities|syntheses/...md` (collection rooted at the wiki +markdown directory, typically `wiki/` inside `tobi/wiki`). + +## Validate the fixture (no corpus required) + +```bash +npx vitest run test/wiki-bench-fixture.test.ts +# or +bun test test/wiki-bench-fixture.test.ts +``` + +This checks version/collection, query shape, allowed `type` values +(`exact|semantic|topical|cross-domain|alias`), unique ids, and that +`expected_files` are safe wiki-relative `.md` paths. + +## Index a local wiki checkout + +Point QMD at your local clone of the private wiki. Markdown usually lives +under `wiki/`: + +```bash +# If markdown is under /wiki/{concepts,sources,entities,...}: +qmd collection add /path/to/wiki/wiki --name wiki-bench + +# If you already keep a bare wiki-root directory of those folders: +qmd collection add /path/to/wiki-root --name wiki-bench +``` + +Then update/embed as usual for your install (`qmd update`, etc.). + +## Run the full multi-backend bench + +`qmd bench` is wired in `src/cli/qmd.ts` and implemented by +`src/bench/bench.ts`. It loads a fixture JSON and scores bm25 / vector / +hybrid / full against an already-indexed collection: + +```bash +qmd bench src/bench/fixtures/wiki-v0.json -c wiki-bench +qmd bench src/bench/fixtures/wiki-v0.json -c wiki-bench --json +``` + +Usage (from CLI): `qmd bench [--json] [-c collection]`. + +## Local BM25 runner (private corpus) + +`scripts/run-wiki-bench-local.mjs` indexes a **temporary** collection named +`wiki-bench` (under `$TMPDIR`; never writes into the git tree), runs BM25 via +`runBenchmark`, prints summary metrics, and exits non-zero if quality floors +fail. Same env vars / floors as the skip-gated vitest suite: + +```bash +QMD_WIKI_PATH=~/src/wiki node scripts/run-wiki-bench-local.mjs +# or: +QMD_WIKI_BENCH_DOCS=/path/to/wiki/wiki node scripts/run-wiki-bench-local.mjs +``` + +- `QMD_WIKI_PATH` — checkout of private `tobi/wiki`; docs root is + `$QMD_WIKI_PATH/wiki` +- `QMD_WIKI_BENCH_DOCS` — directory already laid out as + `concepts|sources|entities/...` + +Exits with a clear error if neither env var is set or the path is missing. +Does **not** copy or commit wiki markdown into this repository. + +## Optional BM25 quality floors (local corpus) + +`test/wiki-bench-bm25.test.ts` indexes a **local** corpus into a temp DB and +asserts floors on `exact` + `alias` queries. It **skips** unless one of these +env vars points at an existing directory: + +- `QMD_WIKI_BENCH_DOCS` — preferred; directory that contains + `concepts/`, `sources/`, etc. +- `QMD_WIKI_PATH` — wiki repo root or wiki markdown root; if a nested `wiki/` + subdirectory contains those folders, that nested path is used. + +```bash +export QMD_WIKI_BENCH_DOCS=/path/to/wiki/wiki +npx vitest run test/wiki-bench-bm25.test.ts +``` + +Without the env var (default CI), the suite skips and stays green. + +Asserted floors (measured baseline when corpus is present): + +- exact: mean recall@3 ≥ 0.85 +- alias: mean recall@3 ≥ 0.80 +- exact+alias: mean MRR ≥ 0.70 + +## Notes + +- Eval/benchmark scope only — this does not change search ranking code. +- Labels were derived from registry titles + frontmatter (`title`, `sources`, + entity names), never from an LLM judge. +- Never commit private wiki `.md` content into `tobi/qmd`. diff --git a/scripts/run-wiki-bench-local.mjs b/scripts/run-wiki-bench-local.mjs new file mode 100755 index 000000000..db4a72e2f --- /dev/null +++ b/scripts/run-wiki-bench-local.mjs @@ -0,0 +1,57 @@ +#!/usr/bin/env node +/** + * wiki-bench v0 — private/local BM25 runner. + * + * Requires a local corpus via env (never vendors wiki bodies into the repo): + * QMD_WIKI_PATH — checkout of tobi/wiki; uses $QMD_WIKI_PATH/wiki + * QMD_WIKI_BENCH_DOCS — markdown root with concepts|sources|entities/... + * + * Builds a temporary QMD index/collection named `wiki-bench`, runs BM25 against + * src/bench/fixtures/wiki-v0.json via runBenchmark, prints metrics, and exits + * non-zero if exact R@3 < 0.85, alias R@3 < 0.80, or exact+alias MRR < 0.70. + * + * Usage: + * QMD_WIKI_PATH=~/src/wiki node scripts/run-wiki-bench-local.mjs + */ +import { spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = fileURLToPath(new URL("..", import.meta.url)); +const helper = join(root, "scripts", "wiki-bench-local.ts"); + +function resolveRunner() { + const tsx = join(root, "node_modules", ".bin", "tsx"); + if (existsSync(tsx)) { + return { command: tsx, args: [helper], label: "tsx" }; + } + const bun = spawnSync("bun", ["--version"], { encoding: "utf8" }); + if (bun.status === 0) { + return { command: "bun", args: [helper], label: "bun" }; + } + console.error( + "wiki-bench-local: need local TypeScript runner (node_modules/.bin/tsx or bun).", + ); + process.exit(1); +} + +if (!existsSync(helper)) { + console.error(`wiki-bench-local: missing helper ${helper}`); + process.exit(1); +} + +const runner = resolveRunner(); +const result = spawnSync(runner.command, runner.args, { + cwd: root, + stdio: "inherit", + env: process.env, + shell: process.platform === "win32", +}); + +if (result.error) { + console.error(`wiki-bench-local: failed to launch ${runner.label}:`, result.error.message); + process.exit(1); +} + +process.exit(result.status ?? 1); diff --git a/scripts/wiki-bench-local.ts b/scripts/wiki-bench-local.ts new file mode 100755 index 000000000..0830f725f --- /dev/null +++ b/scripts/wiki-bench-local.ts @@ -0,0 +1,211 @@ +/** + * wiki-bench v0 — private/local BM25 runner body. + * + * Invoked by scripts/run-wiki-bench-local.mjs. Builds a temporary QMD index + * (collection `wiki-bench`) outside the git tree, runs BM25 via runBenchmark, + * prints metrics, and exits non-zero if quality floors fail. + * + * Never vendors wiki page bodies into the repo. + */ + +import { + existsSync, + mkdtempSync, + rmSync, + statSync, +} from "node:fs"; +import { join, resolve } from "node:path"; +import { tmpdir } from "node:os"; +import { fileURLToPath } from "node:url"; +import { createStore } from "../src/index.js"; +import { runBenchmark } from "../src/bench/bench.js"; +import type { BenchmarkResult, QueryResult } from "../src/bench/types.js"; + +const COLLECTION = "wiki-bench"; +const FLOOR_EXACT_RECALL_AT_3 = 0.85; +const FLOOR_ALIAS_RECALL_AT_3 = 0.80; +const FLOOR_COMBINED_MRR = 0.70; + +const root = fileURLToPath(new URL("..", import.meta.url)); +const FIXTURE_PATH = join(root, "src", "bench", "fixtures", "wiki-v0.json"); + +function die(message: string, code = 1): never { + console.error(`wiki-bench-local: ${message}`); + process.exit(code); +} + +function looksLikeWikiDocsRoot(dir: string): boolean { + return existsSync(join(dir, "concepts")) || existsSync(join(dir, "sources")); +} + +/** Resolve corpus root from env. Never looks under test/wiki-bench-docs. */ +function resolveWikiDocsDir(): string { + const benchDocs = process.env.QMD_WIKI_BENCH_DOCS?.trim(); + if (benchDocs) { + const abs = resolve(benchDocs); + if (!existsSync(abs) || !statSync(abs).isDirectory()) { + die(`QMD_WIKI_BENCH_DOCS is set but not a directory: ${abs}`); + } + if (!looksLikeWikiDocsRoot(abs)) { + die( + `QMD_WIKI_BENCH_DOCS must contain wiki-relative folders (concepts|sources|...): ${abs}`, + ); + } + return abs; + } + + const wikiPath = process.env.QMD_WIKI_PATH?.trim(); + if (wikiPath) { + const abs = resolve(wikiPath); + if (!existsSync(abs) || !statSync(abs).isDirectory()) { + die(`QMD_WIKI_PATH is set but not a directory: ${abs}`); + } + // Spec: use $QMD_WIKI_PATH/wiki (Obsidian wiki/ folder) as the docs root. + const nested = join(abs, "wiki"); + if (!existsSync(nested) || !statSync(nested).isDirectory()) { + die( + `QMD_WIKI_PATH set but missing Obsidian wiki/ folder at: ${nested}\n` + + ` (expected layout: $QMD_WIKI_PATH/wiki/{concepts,sources,entities,...})`, + ); + } + if (!looksLikeWikiDocsRoot(nested)) { + die( + `QMD_WIKI_PATH/wiki must contain wiki-relative folders (concepts|sources|...): ${nested}`, + ); + } + return nested; + } + + die( + "Set QMD_WIKI_PATH (checkout of tobi/wiki) or QMD_WIKI_BENCH_DOCS (markdown root with concepts|sources|...).\n" + + " Example: QMD_WIKI_PATH=~/src/wiki node scripts/run-wiki-bench-local.mjs\n" + + "Private wiki page bodies must never be committed into this repo.", + ); +} + +function mean(values: number[]): number { + if (values.length === 0) return 0; + return values.reduce((a, b) => a + b, 0) / values.length; +} + +function bm25Metrics(results: QueryResult[], type: string): { r3: number; mrr: number; n: number } { + const subset = results.filter((r) => r.type === type); + const scores = subset + .map((r) => r.backends.bm25) + .filter((b): b is NonNullable => !!b); + return { + r3: mean(scores.map((s) => s.recall_at_3)), + mrr: mean(scores.map((s) => s.mrr)), + n: scores.length, + }; +} + +function printFloorSummary(result: BenchmarkResult): { + exactR3: number; + aliasR3: number; + combinedMrr: number; +} { + const exact = bm25Metrics(result.results, "exact"); + const alias = bm25Metrics(result.results, "alias"); + const combinedScores = result.results + .filter((r) => r.type === "exact" || r.type === "alias") + .map((r) => r.backends.bm25) + .filter((b): b is NonNullable => !!b); + const combinedMrr = mean(combinedScores.map((s) => s.mrr)); + + console.log("\n[wiki-bench-local] BM25 quality floors (exact + alias):"); + console.log( + ` exact mean recall@3 = ${exact.r3.toFixed(4)} (n=${exact.n}; floor ${FLOOR_EXACT_RECALL_AT_3})`, + ); + console.log( + ` alias mean recall@3 = ${alias.r3.toFixed(4)} (n=${alias.n}; floor ${FLOOR_ALIAS_RECALL_AT_3})`, + ); + console.log( + ` exact+alias mean MRR = ${combinedMrr.toFixed(4)} (n=${combinedScores.length}; floor ${FLOOR_COMBINED_MRR})`, + ); + + return { exactR3: exact.r3, aliasR3: alias.r3, combinedMrr }; +} + +async function main(): Promise { + if (!existsSync(FIXTURE_PATH)) { + die(`fixture not found: ${FIXTURE_PATH}`); + } + + const docsDir = resolveWikiDocsDir(); + const tempDir = mkdtempSync(join(tmpdir(), "qmd-wiki-bench-local-")); + const dbPath = join(tempDir, "wiki-bench.sqlite"); + + console.log(`[wiki-bench-local] docs: ${docsDir}`); + console.log(`[wiki-bench-local] temp index: ${dbPath}`); + console.log(`[wiki-bench-local] fixture: ${FIXTURE_PATH}`); + + let exitCode = 0; + try { + const store = await createStore({ + dbPath, + config: { + collections: { + [COLLECTION]: { path: docsDir, pattern: "**/*.md" }, + }, + }, + }); + + const updateResult = await store.update({ collections: [COLLECTION] }); + await store.close(); + + const indexed = + updateResult.indexed + updateResult.updated + updateResult.unchanged; + if (indexed <= 0) { + die(`no markdown documents indexed from ${docsDir}`); + } + console.log( + `[wiki-bench-local] indexed collection '${COLLECTION}': ` + + `${updateResult.indexed} new, ${updateResult.updated} updated, ` + + `${updateResult.unchanged} unchanged (${indexed} total)`, + ); + + const benchResult = await runBenchmark(FIXTURE_PATH, { + backends: ["bm25"], + collection: COLLECTION, + dbPath, + }); + + const { exactR3, aliasR3, combinedMrr } = printFloorSummary(benchResult); + + const failures: string[] = []; + if (exactR3 < FLOOR_EXACT_RECALL_AT_3) { + failures.push( + `exact R@3 ${exactR3.toFixed(4)} < ${FLOOR_EXACT_RECALL_AT_3}`, + ); + } + if (aliasR3 < FLOOR_ALIAS_RECALL_AT_3) { + failures.push( + `alias R@3 ${aliasR3.toFixed(4)} < ${FLOOR_ALIAS_RECALL_AT_3}`, + ); + } + if (combinedMrr < FLOOR_COMBINED_MRR) { + failures.push( + `exact+alias MRR ${combinedMrr.toFixed(4)} < ${FLOOR_COMBINED_MRR}`, + ); + } + + if (failures.length > 0) { + console.error("\nwiki-bench-local: FAILED quality floors:"); + for (const f of failures) console.error(` - ${f}`); + exitCode = 1; + } else { + console.log("\nwiki-bench-local: PASSED quality floors."); + } + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + + process.exit(exitCode); +} + +main().catch((err) => { + console.error("wiki-bench-local: unexpected error"); + console.error(err); + process.exit(1); +}); diff --git a/src/bench/fixtures/wiki-v0.json b/src/bench/fixtures/wiki-v0.json new file mode 100644 index 000000000..d6398f0cc --- /dev/null +++ b/src/bench/fixtures/wiki-v0.json @@ -0,0 +1,365 @@ +{ + "description": "wiki-bench v0: retrieval over Tobi Lütke public-appearances wiki. Ground truth from meta/registry.json + page YAML frontmatter only (no LLM judge). Collection rooted at wiki/ so paths are concepts|sources|entities/*.md.", + "version": 1, + "collection": "wiki-bench", + "queries": [ + { + "id": "src-kp41", + "query": "Knowledge Project Episode 41 Trust Battery", + "type": "exact", + "description": "Exact show + episode title → source page (registry title)", + "expected_files": ["sources/2018-knowledge-project.md"], + "expected_in_top_k": 1 + }, + { + "id": "src-hibt", + "query": "How I Built This Guy Raz Shopify", + "type": "exact", + "description": "Show + host → 2019 NPR biography source", + "expected_files": ["sources/2019-how-i-built-this.md"], + "expected_in_top_k": 1 + }, + { + "id": "src-kara", + "query": "Kara Swisher Startupfest 2016 Rise of Shopify", + "type": "exact", + "description": "Host + event + year → source", + "expected_files": ["sources/2016-kara-swisher.md"], + "expected_in_top_k": 1 + }, + { + "id": "src-mos", + "query": "Masters of Scale Reid Hoffman be a platform", + "type": "exact", + "description": "Show + host → 2020 Masters of Scale source", + "expected_files": ["sources/2020-masters-of-scale-reid-hoffman.md"], + "expected_in_top_k": 1 + }, + { + "id": "src-motley-2017", + "query": "Motley Fool Tom Gardner 2017 trust battery", + "type": "exact", + "description": "Show + host + year → source that first names trust battery", + "expected_files": ["sources/2017-motley-fool-tom-gardner.md"], + "expected_in_top_k": 1 + }, + { + "id": "src-leo", + "query": "Leo Laporte Call for Help Liquid 2006", + "type": "exact", + "description": "Early TV interview → source", + "expected_files": ["sources/2006-leo-laporte.md"], + "expected_in_top_k": 1 + }, + { + "id": "src-mixergy", + "query": "Mixergy Andrew Warner Shopify profitable", + "type": "exact", + "description": "Show + host → 2010 Mixergy source", + "expected_files": ["sources/2010-mixergy.md"], + "expected_in_top_k": 1 + }, + { + "id": "src-twist", + "query": "This Week in Startups Jason Calacanis Ottawa", + "type": "exact", + "description": "Show + host → 2013 TWiST source", + "expected_files": ["sources/2013-this-week-in-startups.md"], + "expected_in_top_k": 1 + }, + { + "id": "src-balaji", + "query": "Network State Balaji country-sized economy", + "type": "exact", + "description": "Show + host + phrase → 2023 source", + "expected_files": ["sources/2023-network-state-balaji.md"], + "expected_in_top_k": 1 + }, + { + "id": "src-iltb173", + "query": "Invest Like the Best 173 Patrick O'Shaughnessy", + "type": "exact", + "description": "Show + episode number → source", + "expected_files": ["sources/2020-invest-like-the-best-173.md"], + "expected_in_top_k": 1 + }, + { + "id": "concept-trust-battery", + "query": "trust battery", + "type": "exact", + "description": "Concept title exact match", + "expected_files": ["concepts/trust-battery.md"], + "expected_in_top_k": 1 + }, + { + "id": "concept-arming-rebels", + "query": "arming the rebels", + "type": "exact", + "description": "Concept title exact match", + "expected_files": ["concepts/arming-the-rebels.md"], + "expected_in_top_k": 1 + }, + { + "id": "concept-antifragility", + "query": "antifragility Taleb", + "type": "exact", + "description": "Concept + author cue from tldr", + "expected_files": ["concepts/antifragility.md"], + "expected_in_top_k": 1 + }, + { + "id": "concept-liquid", + "query": "Liquid template language Shopify", + "type": "exact", + "description": "Concept title + tldr keywords", + "expected_files": ["concepts/liquid.md"], + "expected_in_top_k": 1 + }, + { + "id": "concept-tobi-test", + "query": "Tobi test", + "type": "exact", + "description": "Concept title exact match", + "expected_files": ["concepts/tobi-test.md"], + "expected_in_top_k": 1 + }, + { + "id": "concept-pendulum", + "query": "pendulum model management", + "type": "exact", + "description": "Concept title + domain word", + "expected_files": ["concepts/pendulum-model.md"], + "expected_in_top_k": 1 + }, + { + "id": "concept-hundred-year", + "query": "hundred year vision", + "type": "exact", + "description": "Concept title exact match", + "expected_files": ["concepts/hundred-year-vision.md"], + "expected_in_top_k": 1 + }, + { + "id": "concept-apprenticeship", + "query": "apprenticeship German craft", + "type": "exact", + "description": "Concept + tldr keywords", + "expected_files": ["concepts/apprenticeship.md"], + "expected_in_top_k": 1 + }, + { + "id": "concept-autonomy", + "query": "autonomy NUMMI rip cord", + "type": "exact", + "description": "Concept + distinctive exhibit from page body/frontmatter framing", + "expected_files": ["concepts/autonomy.md"], + "expected_in_top_k": 1 + }, + { + "id": "concept-platform-shift", + "query": "AI as platform shift", + "type": "exact", + "description": "Concept title phrasing", + "expected_files": ["concepts/ai-as-platform-shift.md"], + "expected_in_top_k": 1 + }, + { + "id": "sem-trust-paraphrase", + "query": "interpersonal trust that charges with follow-through and buys autonomy", + "type": "semantic", + "description": "Paraphrase of trust-battery tldr (no title words)", + "expected_files": ["concepts/trust-battery.md"], + "expected_in_top_k": 3 + }, + { + "id": "sem-rebels-paraphrase", + "query": "give independent merchants platform tools to compete with Amazon", + "type": "semantic", + "description": "Paraphrase of arming-the-rebels tldr", + "expected_files": ["concepts/arming-the-rebels.md"], + "expected_in_top_k": 3 + }, + { + "id": "sem-tobi-test-paraphrase", + "query": "randomly turning off production services to train for failure", + "type": "semantic", + "description": "Paraphrase of tobi-test definition (chaos drill)", + "expected_files": ["concepts/tobi-test.md"], + "expected_in_top_k": 3 + }, + { + "id": "sem-calm-progress", + "query": "counter-philosophy to hype and calm progress at scale", + "type": "semantic", + "description": "Paraphrase tied to calm-progress / KP 152 framing", + "expected_files": ["concepts/calm-progress.md"], + "expected_in_top_k": 3 + }, + { + "id": "cit-trust-battery-intro", + "query": "which interview first named the trust battery metaphor", + "type": "topical", + "description": "Citation grounding: concept frontmatter sources include 2017 Motley Fool (first naming) and 2018 KP", + "expected_files": [ + "sources/2017-motley-fool-tom-gardner.md", + "sources/2018-knowledge-project.md" + ], + "expected_in_top_k": 2 + }, + { + "id": "cit-liquid-debut", + "query": "interview where Liquid template engine was first demoed", + "type": "topical", + "description": "Citation: liquid frontmatter sources → 2006 Leo Laporte", + "expected_files": ["sources/2006-leo-laporte.md"], + "expected_in_top_k": 1 + }, + { + "id": "cit-tobi-test-named", + "query": "podcast episode where the Tobi test chaos drill was named", + "type": "topical", + "description": "Citation: tobi-test frontmatter → 2018 Knowledge Project", + "expected_files": ["sources/2018-knowledge-project.md"], + "expected_in_top_k": 1 + }, + { + "id": "cit-apprenticeship", + "query": "appearance discussing Siemens apprenticeship and German craft education", + "type": "topical", + "description": "Citation: apprenticeship frontmatter → How I Built This", + "expected_files": ["sources/2019-how-i-built-this.md"], + "expected_in_top_k": 1 + }, + { + "id": "cit-arming-rebels", + "query": "podcast where Tobi said Shopify arms the rebels against Amazon", + "type": "topical", + "description": "Citation: arming-the-rebels frontmatter → Escape Velocity", + "expected_files": ["sources/2019-escape-velocity-dan-martell.md"], + "expected_in_top_k": 1 + }, + { + "id": "cit-autonomy-nummi", + "query": "talk where Tobi told the NUMMI Toyota autonomy rip cord story", + "type": "topical", + "description": "Citation: autonomy frontmatter → Business of Software 2011", + "expected_files": ["sources/2011-business-of-software.md"], + "expected_in_top_k": 1 + }, + { + "id": "cit-calm-progress", + "query": "2022 Farnam Street return interview about calm progress", + "type": "topical", + "description": "Citation: calm-progress frontmatter → KP 152", + "expected_files": ["sources/2022-knowledge-project-152.md"], + "expected_in_top_k": 1 + }, + { + "id": "cit-country-economy", + "query": "conversation with Balaji about Shopify as a country-sized economy", + "type": "topical", + "description": "Citation: country-sized-economy frontmatter → Network State", + "expected_files": ["sources/2023-network-state-balaji.md"], + "expected_in_top_k": 1 + }, + { + "id": "cit-pendulum", + "query": "Motley Fool interview introducing the pendulum management metaphor", + "type": "topical", + "description": "Citation: pendulum-model frontmatter → 2017 Motley Fool", + "expected_files": ["sources/2017-motley-fool-tom-gardner.md"], + "expected_in_top_k": 1 + }, + { + "id": "cit-antifragile", + "query": "appearances discussing Taleb antifragile and thriving on change", + "type": "topical", + "description": "Citation: antifragility frontmatter sources", + "expected_files": [ + "sources/2017-motley-fool-tom-gardner.md", + "sources/2016-kara-swisher.md" + ], + "expected_in_top_k": 2 + }, + { + "id": "alias-guy-raz", + "query": "Guy Raz", + "type": "alias", + "description": "Person entity by common name", + "expected_files": ["entities/p-guy-raz.md"], + "expected_in_top_k": 1 + }, + { + "id": "alias-shane-parrish", + "query": "Shane Parrish", + "type": "alias", + "description": "Person entity by common name", + "expected_files": ["entities/p-shane-parrish.md"], + "expected_in_top_k": 1 + }, + { + "id": "alias-hibt-show", + "query": "How I Built This", + "type": "alias", + "description": "Show entity title", + "expected_files": ["entities/s-how-i-built-this.md"], + "expected_in_top_k": 1 + }, + { + "id": "alias-knowledge-project", + "query": "The Knowledge Project Farnam Street", + "type": "alias", + "description": "Show entity + alternate name", + "expected_files": ["entities/s-the-knowledge-project.md"], + "expected_in_top_k": 1 + }, + { + "id": "alias-kara", + "query": "Kara Swisher", + "type": "alias", + "description": "Person entity", + "expected_files": ["entities/p-kara-swisher.md"], + "expected_in_top_k": 1 + }, + { + "id": "alias-reid", + "query": "Reid Hoffman", + "type": "alias", + "description": "Person entity", + "expected_files": ["entities/p-reid-hoffman.md"], + "expected_in_top_k": 1 + }, + { + "id": "alias-amazon", + "query": "Amazon aggregator", + "type": "alias", + "description": "Org entity + tldr cue", + "expected_files": ["entities/o-amazon.md"], + "expected_in_top_k": 1 + }, + { + "id": "alias-calacanis", + "query": "Jason Calacanis", + "type": "alias", + "description": "Person entity", + "expected_files": ["entities/p-jason-calacanis.md"], + "expected_in_top_k": 1 + }, + { + "id": "alias-shopify", + "query": "Shopify", + "type": "alias", + "description": "Primary org entity", + "expected_files": ["entities/shopify.md"], + "expected_in_top_k": 1 + }, + { + "id": "alias-ferriss", + "query": "Tim Ferriss", + "type": "alias", + "description": "Person entity", + "expected_files": ["entities/p-tim-ferriss.md"], + "expected_in_top_k": 1 + } + ] +} diff --git a/test/wiki-bench-bm25.test.ts b/test/wiki-bench-bm25.test.ts new file mode 100644 index 000000000..a8ebc2200 --- /dev/null +++ b/test/wiki-bench-bm25.test.ts @@ -0,0 +1,222 @@ +/** + * wiki-bench v0 — optional BM25/FTS quality floors. + * + * Skips unless QMD_WIKI_BENCH_DOCS or QMD_WIKI_PATH points at an existing + * corpus directory (the wiki markdown root containing concepts|sources|entities). + * Default CI has neither set, so this suite is a no-op and stays green. + * + * Does NOT vendor wiki page bodies into this repo. + * + * Measured BM25 baseline (lex/FTS only, when corpus is present): + * exact mean recall@3: 1.0000 (n=20) + * alias mean recall@3: 0.9000 (n=10) + * exact+alias mean MRR: 0.8811 (n=30) + * Floors: 0.85 / 0.80 / 0.70 + */ + +import { describe, test, expect, beforeAll, afterAll } from "vitest"; +import { + mkdtempSync, + rmSync, + readFileSync, + readdirSync, + statSync, + existsSync, +} from "fs"; +import { join, dirname, relative } from "path"; +import { tmpdir } from "os"; +import { createHash } from "crypto"; +import { fileURLToPath } from "url"; +import type { Database } from "../src/db.js"; +import { + createStore, + searchFTS, + insertDocument, + insertContent, +} from "../src/store"; +import { scoreResults } from "../src/bench/score.js"; +import type { BenchmarkFixture, BenchmarkQuery } from "../src/bench/types.js"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const FIXTURE_PATH = join(__dirname, "..", "src", "bench", "fixtures", "wiki-v0.json"); +const COLLECTION = "wiki-bench"; + +const FLOOR_EXACT_RECALL_AT_3 = 0.85; +const FLOOR_ALIAS_RECALL_AT_3 = 0.80; +const FLOOR_COMBINED_MRR = 0.70; + +/** Resolve local corpus dir; never looks under test/wiki-bench-docs in-repo. */ +function resolveWikiDocsDir(): string | null { + const candidates = [ + process.env.QMD_WIKI_BENCH_DOCS, + process.env.QMD_WIKI_PATH, + ].filter((v): v is string => typeof v === "string" && v.length > 0); + + for (const c of candidates) { + if (!existsSync(c) || !statSync(c).isDirectory()) continue; + if (existsSync(join(c, "concepts")) || existsSync(join(c, "sources"))) { + return c; + } + const nested = join(c, "wiki"); + if ( + existsSync(nested) && + (existsSync(join(nested, "concepts")) || existsSync(join(nested, "sources"))) + ) { + return nested; + } + } + return null; +} + +const DOCS_DIR = resolveWikiDocsDir(); + +function walkMarkdownFiles(dir: string, base: string = dir): string[] { + const out: string[] = []; + for (const ent of readdirSync(dir, { withFileTypes: true })) { + const full = join(dir, ent.name); + if (ent.isDirectory()) { + out.push(...walkMarkdownFiles(full, base)); + } else if (ent.isFile() && ent.name.endsWith(".md")) { + out.push(relative(base, full)); + } + } + return out.sort(); +} + +function extractTitle(content: string, fallback: string): string { + const fm = content.match(/^---\r?\n([\s\S]*?)\r?\n---/); + if (fm) { + const m = fm[1]!.match(/^title:\s*(.+)$/m); + if (m) { + const raw = m[1]!.trim(); + const quoted = raw.match(/^["'](.*)["']$/); + if (quoted) return quoted[1]!.trim() || fallback; + return raw || fallback; + } + } + const heading = content.match(/^#\s+(.+)$/m); + if (heading?.[1]) return heading[1]!.trim(); + return fallback; +} + +function mean(values: number[]): number { + if (values.length === 0) return 0; + return values.reduce((a, b) => a + b, 0) / values.length; +} + +type Scored = { + id: string; + type: string; + recall_at_3: number; + mrr: number; + top_files: string[]; +}; + +function scoreQuery(db: Database, q: BenchmarkQuery): Scored { + const results = searchFTS(db, q.query, 10, COLLECTION); + const files = results.map((r) => r.filepath); + const scores = scoreResults(files, q.expected_files, Math.max(q.expected_in_top_k, 3)); + return { + id: q.id, + type: q.type, + recall_at_3: scores.recall_at_3, + mrr: scores.mrr, + top_files: files.slice(0, 3), + }; +} + +describe.skipIf(!DOCS_DIR)("wiki-bench BM25 (FTS, local corpus)", () => { + let store: ReturnType; + let db: Database; + let tempDir: string; + let exactScores: Scored[]; + let aliasScores: Scored[]; + let indexedCount: number; + + beforeAll(() => { + if (!DOCS_DIR) return; + + tempDir = mkdtempSync(join(tmpdir(), "qmd-wiki-bench-")); + process.env.INDEX_PATH = join(tempDir, "wiki-bench.sqlite"); + + store = createStore(); + db = store.db; + + const files = walkMarkdownFiles(DOCS_DIR); + expect(files.length).toBeGreaterThan(0); + + for (const rel of files) { + const full = join(DOCS_DIR, rel); + expect(statSync(full).size).toBeGreaterThan(0); + const content = readFileSync(full, "utf-8"); + const title = extractTitle(content, rel); + const hash = createHash("sha256").update(content).digest("hex").slice(0, 12); + const now = new Date().toISOString(); + insertContent(db, hash, content, now); + insertDocument(db, COLLECTION, rel, title, hash, now, now); + } + indexedCount = files.length; + + const fixture = JSON.parse(readFileSync(FIXTURE_PATH, "utf-8")) as BenchmarkFixture; + const exactQueries = fixture.queries.filter((q) => q.type === "exact"); + const aliasQueries = fixture.queries.filter((q) => q.type === "alias"); + expect(exactQueries.length).toBeGreaterThan(0); + expect(aliasQueries.length).toBeGreaterThan(0); + + exactScores = exactQueries.map((q) => scoreQuery(db, q)); + aliasScores = aliasQueries.map((q) => scoreQuery(db, q)); + + const exactR3 = mean(exactScores.map((s) => s.recall_at_3)); + const aliasR3 = mean(aliasScores.map((s) => s.recall_at_3)); + const combinedMrr = mean([...exactScores, ...aliasScores].map((s) => s.mrr)); + + console.log("\n[wiki-bench] measured BM25 metrics (local corpus):"); + console.log(` docs = ${indexedCount} from ${DOCS_DIR}`); + console.log(` exact mean recall@3 = ${exactR3.toFixed(4)} (n=${exactScores.length})`); + console.log(` alias mean recall@3 = ${aliasR3.toFixed(4)} (n=${aliasScores.length})`); + console.log(` exact+alias mean MRR = ${combinedMrr.toFixed(4)}`); + }); + + afterAll(() => { + store?.close(); + if (tempDir) rmSync(tempDir, { recursive: true, force: true }); + }); + + test("indexes non-empty wiki pages from local corpus", () => { + const count = ( + db + .prepare(`SELECT COUNT(*) as n FROM documents WHERE collection = ? AND active = 1`) + .get(COLLECTION) as { n: number } + ).n; + expect(count).toBe(indexedCount); + expect(count).toBeGreaterThan(0); + }); + + test("exact: mean recall_at_3 meets floor", () => { + const exactR3 = mean(exactScores.map((s) => s.recall_at_3)); + expect(exactR3).toBeGreaterThanOrEqual(FLOOR_EXACT_RECALL_AT_3); + }); + + test("alias: mean recall_at_3 meets floor", () => { + const aliasR3 = mean(aliasScores.map((s) => s.recall_at_3)); + expect(aliasR3).toBeGreaterThanOrEqual(FLOOR_ALIAS_RECALL_AT_3); + }); + + test("exact+alias: mean MRR meets floor", () => { + const combinedMrr = mean([...exactScores, ...aliasScores].map((s) => s.mrr)); + expect(combinedMrr).toBeGreaterThanOrEqual(FLOOR_COMBINED_MRR); + }); +}); + +// Explicit always-on guard so CI reports a passing skip reason rather than an empty file. +describe("wiki-bench BM25 gate", () => { + test("skips quality floors unless local corpus env is set", () => { + if (!DOCS_DIR) { + expect( + process.env.QMD_WIKI_BENCH_DOCS || process.env.QMD_WIKI_PATH || "", + ).toBe(""); + } else { + expect(existsSync(DOCS_DIR)).toBe(true); + } + }); +}); diff --git a/test/wiki-bench-fixture.test.ts b/test/wiki-bench-fixture.test.ts new file mode 100644 index 000000000..e116b03a2 --- /dev/null +++ b/test/wiki-bench-fixture.test.ts @@ -0,0 +1,84 @@ +/** + * wiki-bench v0 — fixture schema validation (no wiki corpus required). + * + * Asserts that src/bench/fixtures/wiki-v0.json is well-formed for the + * benchmark harness. Does not read or require any wiki page bodies. + */ + +import { describe, test, expect } from "vitest"; +import { readFileSync } from "fs"; +import { join, dirname } from "path"; +import { fileURLToPath } from "url"; +import type { BenchmarkFixture, BenchmarkQuery } from "../src/bench/types.js"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const FIXTURE_PATH = join(__dirname, "..", "src", "bench", "fixtures", "wiki-v0.json"); + +const ALLOWED_TYPES = new Set([ + "exact", + "semantic", + "topical", + "cross-domain", + "alias", +] as const); + +/** Wiki-relative paths only: concepts|sources|entities|syntheses/...md */ +const WIKI_REL_PATH = + /^(concepts|sources|entities|syntheses)\/(?:[\w.-]+\/)*[\w.-]+\.md$/; + +describe("wiki-bench fixture (schema only)", () => { + const raw = readFileSync(FIXTURE_PATH, "utf-8"); + const fixture = JSON.parse(raw) as BenchmarkFixture; + + test("has version, collection, and non-empty queries", () => { + expect(typeof fixture.version).toBe("number"); + expect(fixture.version).toBeGreaterThanOrEqual(1); + expect(typeof fixture.collection).toBe("string"); + expect(fixture.collection!.length).toBeGreaterThan(0); + expect(Array.isArray(fixture.queries)).toBe(true); + expect(fixture.queries.length).toBeGreaterThan(0); + }); + + test("has at least 40 queries", () => { + expect(fixture.queries.length).toBeGreaterThanOrEqual(40); + }); + + test("each query has required fields and allowed type", () => { + for (const q of fixture.queries) { + expect(typeof q.id).toBe("string"); + expect(q.id.length).toBeGreaterThan(0); + expect(typeof q.query).toBe("string"); + expect(q.query.length).toBeGreaterThan(0); + expect(ALLOWED_TYPES.has(q.type as BenchmarkQuery["type"])).toBe(true); + expect(typeof q.description).toBe("string"); + expect(q.description.length).toBeGreaterThan(0); + expect(Array.isArray(q.expected_files)).toBe(true); + expect(q.expected_files.length).toBeGreaterThan(0); + expect(typeof q.expected_in_top_k).toBe("number"); + expect(q.expected_in_top_k).toBeGreaterThanOrEqual(1); + } + }); + + test("query ids are unique", () => { + const ids = fixture.queries.map((q) => q.id); + expect(new Set(ids).size).toBe(ids.length); + }); + + test("expected_files are wiki-relative .md paths (no abs, no ..)", () => { + for (const q of fixture.queries) { + for (const file of q.expected_files) { + expect(typeof file).toBe("string"); + expect(file.endsWith(".md")).toBe(true); + expect(file.startsWith("/")).toBe(false); + expect(file.includes("..")).toBe(false); + expect(file).toMatch(WIKI_REL_PATH); + } + } + }); + + test("does not vendor or require wiki corpus under test/", () => { + // Schema-only suite: fixture JSON is the only input. + expect(FIXTURE_PATH.endsWith("wiki-v0.json")).toBe(true); + expect(fixture.queries.length).toBeGreaterThan(0); + }); +});