perf(search): reuse unchanged index source entries

docs/guide/knowledge.md

@@ -57,6 +57,8 @@ teamai recall feedback --negative <docId>
 
 Local feedback affects ranking on the next index rebuild (for example, `teamai pull`), even when report uploads are disabled or fail. Rebuilds combine confirmed team votes with undelivered local votes. If the votes file is busy, that rebuild uses confirmed votes only.
 
+Index rebuilds reuse unchanged files' parsed tokens. Changes, deletions, source selection and votes still take effect on every build. `~/.teamai/debug.log` reports how many files were reused, rebuilt or dropped. Older indexes and a different Node/ICU version trigger a full parse once.
+
 When the log has runs, `teamai stats` lists the last 10 sessions that ran recall.
 
 ### Enabling / Disabling Recall

docs/guide/zh-CN/knowledge.md

@@ -57,6 +57,8 @@ teamai recall feedback --negative <docId>
 
 本地反馈在下次重建索引（例如 `teamai pull`）时影响排序，即使上报被禁用或失败也生效。重建时合并已确认的团队票数和本地未送达的票；若投票文件正被占用，本次重建只使用已确认票数。
 
+重建索引会复用未变文件的解析和分词结果；文件修改、删除、来源选择和票数仍在每次构建时生效。`~/.teamai/debug.log` 会记录复用、重建和移除的文件数。旧索引或不同的 Node/ICU 版本会触发一次完整解析。
+
 `teamai stats` 在有记录时，会列出最近 10 个跑过 recall 的会话。
 
 ### 开启 / 关闭 Recall

src/__tests__/e2e/incremental-index.test.ts

@@ -0,0 +1,73 @@
+import { describe, expect, it } from 'vitest';
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { execFileSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import YAML from 'yaml';
+
+const cli = fileURLToPath(new URL('../../../dist/index.js', import.meta.url));
+
+describe('incremental index through pull (real CLI)', () => {
+  it('reuses sources on the unchanged-revision path and refreshes an edited doc', async () => {
+    const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'teamai-incremental-cli-'));
+    const home = path.join(dir, 'home');
+    const seed = path.join(dir, 'seed');
+    const repo = path.join(home, '.teamai', 'team-repo');
+    const remote = path.join(dir, 'team.git');
+    const env = { ...process.env, HOME: home, USERPROFILE: home,
+      GIT_AUTHOR_NAME: 'Test', GIT_AUTHOR_EMAIL: 'test@example.invalid',
+      GIT_COMMITTER_NAME: 'Test', GIT_COMMITTER_EMAIL: 'test@example.invalid',
+      TEAMAI_TEST_NO_DETACH: '1', FORCE_COLOR: '0' };
+    const git = (args: string[], cwd: string) => execFileSync('git', args, { cwd, env, stdio: 'pipe' });
+    const run = (args: string[]) => execFileSync(process.execPath, [cli, ...args], { cwd: home, env, encoding: 'utf8', timeout: 30_000 });
+    try {
+      await fs.mkdir(path.join(seed, 'docs'), { recursive: true });
+      await fs.mkdir(path.join(seed, 'learnings'), { recursive: true });
+      await fs.writeFile(path.join(seed, 'learnings', 'stable.md'), '# Stable learning\n\nUnchanged knowledge.\n');
+      await fs.mkdir(path.dirname(repo), { recursive: true });
+      await fs.writeFile(path.join(seed, 'teamai.yaml'), YAML.stringify({ team: 'incremental', repo: remote, provider: 'git', usageReport: false }));
+      await fs.writeFile(path.join(seed, 'docs', 'alpha.md'), 'Alpha original.');
+      git(['init', '-qb', 'main'], seed);
+      git(['add', '.'], seed); git(['commit', '-qm', 'fixture'], seed);
+      git(['clone', '-q', '--bare', seed, remote], dir);
+      git(['clone', '-q', remote, repo], dir);
+      await fs.writeFile(path.join(home, '.teamai', 'config.yaml'), YAML.stringify({
+        repo: { localPath: repo, remote }, username: 'fixture', scope: 'user',
+        enabledAgents: [], codexTrustEnabled: false, usageReportEnabled: false, updatePolicy: 'auto',
+      }));
+      run(['pull']);
+      const indexFile = path.join(home, '.teamai', 'search-index.json');
+      const first = JSON.parse(await fs.readFile(indexFile, 'utf8'));
+      expect(first.entries.find((e: { filename: string }) => e.filename === 'alpha.md').tokens).toContain('original');
+      await fs.writeFile(path.join(home, '.teamai', 'debug.log'), '');
+      const probe = path.join(dir, 'read-probe.mjs');
+      const reads = path.join(dir, 'index-reads.jsonl');
+      await fs.writeFile(probe, `import fs from 'node:fs';
+import promises from 'node:fs/promises';
+import { syncBuiltinESMExports } from 'node:module';
+const original = promises.readFile;
+promises.readFile = function(file, ...args) {
+ if (String(file).includes('/learnings/') && new Error().stack.includes('entryFromMdFile')) fs.appendFileSync(${JSON.stringify(reads)}, JSON.stringify(String(file)) + '\\n');
+ return original.call(this, file, ...args);
+};
+syncBuiltinESMExports();
+`);
+      for (let attempt = 0; attempt < 3; attempt++) {
+        execFileSync(process.execPath, ['--import', probe, cli, 'pull'], { cwd: home, env, encoding: 'utf8', timeout: 30000 });
+      }
+      expect(await fs.readFile(reads, 'utf8').catch(() => '')).toBe('');
+      expect(await fs.readFile(path.join(home, '.teamai', 'debug.log'), 'utf8')).toMatch(/Search index files: \d+ reused, 0 rebuilt, 0 dropped/);
+
+      await fs.writeFile(path.join(seed, 'docs', 'alpha.md'), 'Alpha revised.');
+      await fs.writeFile(path.join(seed, 'learnings', 'stable.md'), '# Stable learning\n\nRevised knowledge.\n');
+      git(['add', '.'], seed); git(['commit', '-qm', 'edit'], seed); git(['push', '-q', remote, 'main'], seed);
+      execFileSync(process.execPath, ['--import', probe, cli, 'pull'], { cwd: home, env, encoding: 'utf8', timeout: 30000 });
+      expect((await fs.readFile(reads, 'utf8')).trim().length).toBeGreaterThan(0);
+      const after = JSON.parse(await fs.readFile(indexFile, 'utf8'));
+      expect(after.entries.find((e: { filename: string }) => e.filename === 'alpha.md').tokens).toContain('revised');
+    } finally {
+      await fs.rm(dir, { recursive: true, force: true });
+    }
+  });
+});

src/__tests__/learnings-mirror.test.ts

@@ -25,6 +25,28 @@ describe('mirrorLearnings', () => {
     await fse.remove(tmpDir);
   });
 
+  it('keeps unchanged root and namespaced learning timestamps stable', async () => {
+    await mirrorLearnings(sourceDir, destinationDir, ['alpha']);
+    const files = ['shared-a.md', 'alpha/keep.md'];
+    for (const file of files) await fse.utimes(path.join(destinationDir, file), new Date(1000000), new Date(1000000));
+    await mirrorLearnings(sourceDir, destinationDir, ['alpha']);
+    for (const file of files) expect((await fse.stat(path.join(destinationDir, file))).mtimeMs).toBe(1000000);
+    await fse.writeFile(path.join(sourceDir, 'alpha/keep.md'), '# updated');
+    await mirrorLearnings(sourceDir, destinationDir, ['alpha']);
+    expect(await fse.readFile(path.join(destinationDir, 'alpha/keep.md'), 'utf8')).toBe('# updated');
+  });
+
+  it.skipIf(process.platform === 'win32')('propagates executable permissions even when mirrored bytes are unchanged', async () => {
+    const source = path.join(sourceDir, 'alpha/helper.sh');
+    const destination = path.join(destinationDir, 'alpha/helper.sh');
+    await fse.outputFile(source, '#!/bin/sh\necho helper\n');
+    await fse.chmod(source, 0o644);
+    await mirrorLearnings(sourceDir, destinationDir, ['alpha']);
+    await fse.chmod(source, 0o755);
+    await mirrorLearnings(sourceDir, destinationDir, ['alpha']);
+    expect((await fse.stat(destination)).mode & 0o777).toBe(0o755);
+  });
+
   it('keeps what another source root provides instead of deleting it', async () => {
     // The mirror deletes anything its source does not have. With learnings split
     // across roots (#485) a single-source mirror would wipe the other root's

src/__tests__/search-index-incremental.test.ts

@@ -0,0 +1,127 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+
+vi.mock('node:fs/promises', async (original) => {
+  const actual = await original<typeof import('node:fs/promises')>();
+  return { ...actual, readFile: vi.fn(actual.readFile) };
+});
+import { readFile } from 'node:fs/promises';
+import { buildIndex, indexInMemory, loadIndex } from '../utils/search-index.js';
+
+describe('incremental search index (#1239)', () => {
+  let dir: string;
+  let learningsDir: string;
+  let indexPath: string;
+  const sourceReads = () => vi.mocked(readFile).mock.calls
+    .map(([file]) => String(file)).filter((file) => file.endsWith('.md'));
+
+  beforeEach(async () => {
+    dir = await fs.mkdtemp(path.join(os.tmpdir(), 'teamai-index-incremental-'));
+    learningsDir = path.join(dir, 'learnings');
+    indexPath = path.join(dir, 'index.json');
+    await fs.mkdir(learningsDir);
+    await fs.writeFile(path.join(learningsDir, 'a.md'), '---\ntitle: Alpha\n---\nOriginal token.');
+    await fs.writeFile(path.join(learningsDir, 'b.md'), '---\ntitle: Beta\n---\nOther token.');
+    vi.mocked(readFile).mockClear();
+  });
+  afterEach(async () => { await fs.rm(dir, { recursive: true, force: true }); });
+
+  it('reads no unchanged source, and rebuilds just an edited file', async () => {
+    await buildIndex({ learningsDir, indexPath });
+    vi.mocked(readFile).mockClear();
+    await buildIndex({ learningsDir, indexPath });
+    expect(sourceReads()).toEqual([]);
+
+    await fs.writeFile(path.join(learningsDir, 'a.md'), '---\ntitle: Alpha\n---\nChanged token.');
+    vi.mocked(readFile).mockClear();
+    await buildIndex({ learningsDir, indexPath });
+    expect(sourceReads()).toEqual([path.join(learningsDir, 'a.md')]);
+    expect((await loadIndex(indexPath))?.entries[0].tokens).toContain('changed');
+  });
+
+  it('detects same-size edits with restored mtime and removes deleted entries', async () => {
+    await buildIndex({ learningsDir, indexPath });
+    const file = path.join(learningsDir, 'a.md');
+    const before = await fs.stat(file);
+    await fs.writeFile(file, '---\ntitle: Alpha\n---\nModified token.');
+    await fs.utimes(file, before.atime, before.mtime);
+    await fs.unlink(path.join(learningsDir, 'b.md'));
+    await buildIndex({ learningsDir, indexPath });
+    const index = (await loadIndex(indexPath))!;
+    expect(index.entries).toHaveLength(1);
+    expect(index.entries[0].tokens).toContain('modified');
+    expect(index.entries[0].tokens).not.toContain('original');
+  });
+
+  it('reapplies votes and produces the same entries and frequencies as a full build', async () => {
+    const votesDir = path.join(dir, 'votes');
+    await fs.mkdir(votesDir);
+    await fs.writeFile(path.join(votesDir, 'alice.yaml'), 'votes:\n  a:\n    at: 2026-10-23\n');
+    await buildIndex({ learningsDir, votesDir, indexPath });
+    await fs.unlink(path.join(votesDir, 'alice.yaml'));
+    const reused = await indexInMemory({ learningsDir, votesDir, indexPath });
+    const full = await indexInMemory({ learningsDir, votesDir, indexPath: path.join(dir, 'absent.json') });
+    expect(reused.entries).toEqual(full.entries);
+    expect(reused.df).toEqual(full.df);
+    expect(reused.dfByDomain).toEqual(full.dfByDomain);
+    expect(reused.entries[0].votes).toBe(0);
+  });
+
+  it('does not reuse a different source or leak an inactive namespace', async () => {
+    const privateDir = path.join(learningsDir, 'private');
+    await fs.mkdir(privateDir);
+    await fs.writeFile(path.join(privateDir, 'secret.md'), 'Private data.');
+    await buildIndex({ learningsDir, learningsNamespaces: ['private'], indexPath });
+    const other = path.join(dir, 'queue');
+    await fs.mkdir(other);
+    await fs.writeFile(path.join(other, 'a.md'), 'Queued edit.');
+    await buildIndex({ learningsDirs: [other, learningsDir], indexPath });
+    const index = (await loadIndex(indexPath))!;
+    expect(index.entries.map((entry) => entry.filename)).toEqual(['a.md', 'b.md']);
+    expect(index.entries[0].tokens).toContain('queued');
+    expect(JSON.stringify(index)).not.toContain('Private data');
+    expect(JSON.stringify(index)).not.toContain('secret.md');
+  });
+
+  it('reuses copied bytes without tokenizing again and discards incompatible caches', async () => {
+    await buildIndex({ learningsDir, indexPath });
+    const before = (await loadIndex(indexPath))!;
+    const file = path.join(learningsDir, 'a.md');
+    const bytes = await fs.readFile(file);
+    await fs.writeFile(file, bytes);
+    await buildIndex({ learningsDir, indexPath });
+    expect((await loadIndex(indexPath))?.entries).toEqual(before.entries);
+
+    const invalid = (await loadIndex(indexPath))!;
+    invalid.fileCache!.derivation = 'obsolete';
+    await fs.writeFile(indexPath, JSON.stringify(invalid));
+    vi.mocked(readFile).mockClear();
+    await buildIndex({ learningsDir, indexPath });
+    expect(sourceReads()).toHaveLength(2);
+    expect((await loadIndex(indexPath))?.entries).toEqual(before.entries);
+  });
+
+  it('keeps every category and recomputes ids and selection independently of cached tokens', async () => {
+    const docsDir = path.join(dir, 'docs');
+    const rulesDir = path.join(dir, 'rules');
+    const skillsDir = path.join(dir, 'skills');
+    await fs.mkdir(docsDir);
+    await fs.mkdir(rulesDir);
+    await fs.mkdir(path.join(skillsDir, 'demo'), { recursive: true });
+    await fs.writeFile(path.join(docsDir, 'guide.md'), 'Guide words.');
+    await fs.writeFile(path.join(rulesDir, 'rule.md'), 'Rule words.');
+    await fs.writeFile(path.join(skillsDir, 'demo', 'SKILL.md'), 'Skill words.');
+    const opts = { learningsDir, docsDir, rulesDir, skillsDir, indexPath };
+    await buildIndex(opts);
+    vi.mocked(readFile).mockClear();
+    const cached = await indexInMemory(opts);
+    expect(sourceReads()).toEqual([]);
+    const full = await indexInMemory({ ...opts, indexPath: path.join(dir, 'full.json') });
+    expect(cached.entries).toEqual(full.entries);
+    expect(cached.dfByDomain).toEqual(full.dfByDomain);
+    const narrowed = await indexInMemory({ ...opts, docFiles: [], ruleFiles: [], skills: { kind: 'dirs', dirs: [] } });
+    expect(narrowed.entries.map((entry) => entry.type)).toEqual(['learnings', 'learnings']);
+  });
+});

src/types.ts

@@ -2304,8 +2304,21 @@ export interface SearchIndexEntry {
 /** Schema version of the on-disk search-index.json (bump on breaking change). */
 export const SEARCH_INDEX_VERSION = 7;
 
+/** Per-file derivation only; votes and selection are recomputed on every build. */
+export interface SearchIndexFileCache {
+  derivation: string;
+  files: Array<{
+    key: string;
+    fingerprint: string;
+    hash: string;
+    /** Selected entries refer to `SearchIndex.entries` instead of duplicating tokens. */
+    entry: number | SearchIndexEntry | null;
+  }>;
+}
+
 /** Shape of the search-index.json file. */
 export interface SearchIndex {
+  fileCache?: SearchIndexFileCache;
   /** Schema version. Phase 1 introduces v2 (multi-category index). */
   version?: number;
   /** ISO timestamp of when the index was built */

src/utils/learnings-mirror.ts

@@ -17,6 +17,17 @@ async function isDirectory(dir: string): Promise<boolean> {
   return await pathExists(dir) && (await fse.stat(dir)).isDirectory();
 }
 
+/** Preserve the cache fingerprint when a pull has not changed its bytes. */
+async function copyIfChanged(source: string, destination: string): Promise<void> {
+  if (await pathExists(destination)) {
+    const [incoming, existing, sourceStat, destinationStat] = await Promise.all([
+      fse.readFile(source), fse.readFile(destination), fse.stat(source), fse.stat(destination),
+    ]);
+    if (incoming.equals(existing) && (sourceStat.mode & 0o777) === (destinationStat.mode & 0o777)) return;
+  }
+  await fse.copy(source, destination, { overwrite: true });
+}
+
 /**
  * Reconcile the machine-local learnings cache with the selected repo content.
  *
@@ -108,11 +119,11 @@ export async function mirrorLearnings(
   }
 
   for (const [file, source] of sourceRootFiles) {
-    await fse.copy(source, path.join(destinationDir, file), { overwrite: true });
+    await copyIfChanged(source, path.join(destinationDir, file));
   }
   for (const [namespace, files] of sourceNamespaceFiles) {
     for (const [file, source] of files) {
-      await fse.copy(source, path.join(destinationDir, namespace, file), { overwrite: true });
+      await copyIfChanged(source, path.join(destinationDir, namespace, file));
     }
   }
 }