diff --git a/CHANGELOG.md b/CHANGELOG.md index 88a2176e7..1c9f5369c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,9 @@ - **#997**: Added the `bm hook` harness front door (SPEC-55). Lifecycle verbs (`session-start`, `pre-compact`) move the plugin hook logic into the package behind per-harness stdin adapters, with the session brief fenced as reference - data. Opt-in envelope capture (`captureEvents: true`, fail-closed) records - redacted lifecycle events into a local inbox WAL, projected deterministically - by `bm hook flush`; `bm hook status` shows the surface. `bm hook install` / + data. Default-on envelope capture records bounded lifecycle metadata into a + local inbox WAL, and `bm hook flush` archives it locally; `bm hook status` + shows the surface. `captureEvents: false` disables capture. `bm hook install` / `bm hook remove` wire the hooks into user-level harness config for standalone users with ownership-tagged, surgical merging. The Claude Code and Codex plugin hooks are now zero-logic PEP 723 uv scripts (`uv run --script`) that @@ -17,6 +17,13 @@ release tooling, and `BM_BIN` overrides the uv-managed environment for development. +### Maintenance + +- Removed the hook-specific redaction subsystem, its `detect-secrets` + dependency, and the retired lifecycle projector compatibility module. +- Kept Milvus as a first-party optional vector backend while removing the + unused Python entry-point registry for separately packaged vector adapters. + ## v0.22.1 (2026-06-12) Follow-up patch to v0.22.0. Fixes project and default-project resolution on diff --git a/docs/semantic-search.md b/docs/semantic-search.md index 413fc2207..7a2efeddd 100644 --- a/docs/semantic-search.md +++ b/docs/semantic-search.md @@ -106,7 +106,7 @@ All settings are fields on `BasicMemoryConfig` and can be set via environment va | Config Field | Env Var | Default | Description | |---|---|---|---| | `semantic_search_enabled` | `BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED` | Auto (`true` when semantic deps are available) | Enable semantic search. Required before vector/hybrid modes work. | -| `semantic_vector_index` | `BASIC_MEMORY_SEMANTIC_VECTOR_INDEX` | `"pgvector"` | Postgres vector storage adapter. `"pgvector"` is built in, `"milvus"` is available through the `basic-memory[milvus]` extra, and other names resolve through the `basic_memory.semantic_vector_indexes` Python entry-point group. SQLite always uses its built-in `sqlite-vec` adapter. | +| `semantic_vector_index` | `BASIC_MEMORY_SEMANTIC_VECTOR_INDEX` | `"pgvector"` | Postgres vector storage adapter: `"pgvector"` or first-party `"milvus"` through the `basic-memory[milvus]` extra. SQLite always uses its built-in `sqlite-vec` adapter. | | `milvus_uri` | `BASIC_MEMORY_MILVUS_URI` | Unset | Milvus, Milvus Lite, or Zilliz Cloud connection URI. Required when `semantic_vector_index="milvus"`. | | `milvus_token` | `BASIC_MEMORY_MILVUS_TOKEN` | Unset | Optional Milvus or Zilliz Cloud authentication token. | | `milvus_timeout_seconds` | `BASIC_MEMORY_MILVUS_TIMEOUT_SECONDS` | `30.0` | Finite per-operation timeout for Milvus and Zilliz client calls. Increase it for unusually slow deployments. | @@ -473,7 +473,7 @@ bm reindex -p my-project - **Dimension change**: After changing `semantic_embedding_dimensions` - **LiteLLM role change**: After changing `semantic_embedding_document_input_type` or `semantic_embedding_query_input_type` - **Literal prefix change**: After changing `semantic_embedding_document_prefix` or `semantic_embedding_query_prefix` -- **Vector index change**: After completing the external-adapter cleanup procedure below and changing `semantic_vector_index` +- **Vector index change**: After completing the vector-store cleanup procedure below and changing `semantic_vector_index` The reindex command shows progress with embedded/skipped/error counts: @@ -538,7 +538,7 @@ The sqlite-vec extension is loaded per-connection. Vector tables are created laz The Alembic migration creates the dimension-independent chunks table. The embeddings table and HNSW index are deferred to runtime because they depend on the configured vector dimensions. -## Milvus and Pluggable Vector Indexes +## Milvus Vector Index Postgres deployments can replace pgvector storage and nearest-neighbour lookup without replacing Basic Memory's SQL repositories or embedding providers. Milvus is the first-party @@ -572,76 +572,12 @@ an existing collection uses another dimension or an incompatible schema, Basic M it and fails initialization instead of deleting shared vectors during a rolling deployment. Coordinate the exact collection migration, then run `bm reindex --embeddings`. -Other provider names still resolve through the extension entry-point contract. A configured -extension that is missing, duplicated, invalid, or returns an incompatible adapter fails -explicitly at startup. Basic Memory does not silently fall back to pgvector, because doing so -would split vectors across stores while appearing healthy. - SQLite remains automatic in this version: local SQLite databases always select `sqlite-vec`, even if `semantic_vector_index` is set. The selector controls Postgres-backed runtimes only. -### Extension Package Contract - -A separately distributed package registers one factory under the -`basic_memory.semantic_vector_indexes` entry-point group: - -```toml -[project.entry-points."basic_memory.semantic_vector_indexes"] -qdrant = "acme_basic_memory_qdrant:create_index" -``` - -The factory receives an explicit scope and the validated Basic Memory configuration: - -```python -from basic_memory.config import BasicMemoryConfig -from basic_memory.repository.semantic_vector_index import ( - SemanticVectorIndex, - VectorIndexScope, -) - - -def create_index( - *, - scope: VectorIndexScope, - app_config: BasicMemoryConfig, -) -> SemanticVectorIndex: - ... -``` - -`VectorIndexScope` contains a stable, credential-free database namespace, project ID, embedding -identity, and vector dimensions. Extensions must isolate physical storage by -`scope.storage_key`, which contains only the stable database namespace and project ID. -`embedding_identity` and `dimensions` describe the current vector schema for validation and -initialization; adapters must not use those mutable fields to create a second unreachable -project collection when the embedding configuration changes. Extensions own their client -lifecycle, credentials, collection/index creation, vector persistence, and nearest-neighbour -implementation. - -The returned `SemanticVectorIndex` has five asynchronous operations: - -- `initialize()` validates or creates backend storage. -- `upsert(records)` idempotently writes vectors by `(entity_id, chunk_key)` for each record's - `source_hash` generation. -- `delete(records)` removes stable keys only for each record's `source_hash` generation; stale or - missing records are successful no-ops. -- `delete_entity(entity_id)` removes all vectors for one entity in the scope. -- `search(query, limit)` returns stable keys with normalized cosine similarity in `[0, 1]`. - -The adapter never receives a SQLAlchemy session and never calls the embedding provider. Basic -Memory owns chunking and embedding, while the extension owns vector storage and lookup. -The built-in pgvector and sqlite-vec adapters additionally remove each pending SQL manifest row in -the same database transaction as its vector so no newer generation can enter between those steps. -Each `VectorRecord` and `VectorDeletion` carries the SHA-256 source generation that produced its -value. Basic Memory holds the matching SQL manifest locked across extension adapter I/O and the -ready-state transition or deletion (`FOR UPDATE` on Postgres and a conditional write lock on -SQLite), so an older overlapping sync cannot overwrite or remove a newer generation under the same -stable adapter key. - -Adapters may also implement the separate `SemanticVectorIndexReconciler` capability. After a -vector reindex, Basic Memory passes it the complete set of current ready keys so the adapter can -delete scoped external orphans. Keeping reconciliation separate preserves the narrow required -storage protocol while allowing external stores to reclaim records left by interrupted deletes or -ready-state commits. +Milvus implements Basic Memory's internal `SemanticVectorIndex` storage boundary. Basic Memory +continues to own chunking, embeddings, and the SQL manifest; Milvus owns only vector persistence, +nearest-neighbour lookup, and scoped orphan cleanup. ### SQL Manifest and Failure Recovery @@ -649,17 +585,17 @@ ready-state commits. store. Each row records the selected `vector_index`, embedding identity, stable chunk key, and an `embedding_status` of `pending` or `ready`. -Writes and deletes commit `pending` before calling the adapter. A successful adapter operation then +Writes and deletes commit `pending` before calling the vector index. A successful operation then makes the manifest row ready or removes it. Vector writes are generation checked inside built-in -adapter transactions; extension writes retain the manifest lock across adapter I/O. If the external +SQL adapter transactions; Milvus writes retain the manifest lock across client I/O. If the external operation fails, the pending row is not searchable and the next sync safely retries the idempotent operation. Adapter search results are hydrated only through current, ready manifest rows, so stale or orphaned external matches fail closed. -Basic Memory deliberately refuses to mutate manifest rows owned by a different external adapter. -Before switching `semantic_vector_index`, keep the old adapter configured and use that extension's -project-scope administrative cleanup to remove the old vectors. Remove the corresponding +Basic Memory deliberately refuses to mutate manifest rows owned by a different vector index. +Before switching `semantic_vector_index`, keep the old index configured and remove its +project-scoped vectors. Remove the corresponding `search_vector_chunks` manifest rows only after the external cleanup succeeds. Then switch the configured adapter and run `bm reindex --embeddings` to populate the new store. If configuration -was switched too early, restore the old adapter first; the ownership check will continue to fail +was switched too early, restore the old index first; the ownership check will continue to fail closed until cleanup is completed. diff --git a/plugins/claude-code/CHANGELOG.md b/plugins/claude-code/CHANGELOG.md index 558453aa2..0f34522cc 100644 --- a/plugins/claude-code/CHANGELOG.md +++ b/plugins/claude-code/CHANGELOG.md @@ -98,10 +98,10 @@ Memory's durable graph**, rather than a memory layer of its own. See by release tooling) and the script invokes `basic-memory hook --harness claude` in-process with the hook JSON on stdin. `BM_BIN` overrides the uv-managed environment for development. - The brief/checkpoint logic lives in the released package; opt-in - `captureEvents: true` additionally records redacted event envelopes to a - local inbox. uv is the required prerequisite; the first run fetches from - PyPI, later runs use uv's cache. + The brief/checkpoint logic lives in the released package; lifecycle envelopes + containing bounded metadata are captured by default and can be disabled with + `captureEvents: false`. uv is the required prerequisite; the first run fetches + from PyPI, later runs use uv's cache. - **SessionStart hook now nudges toward `/basic-memory:bm-setup` on first run** — when no `basicMemory` config block is present in either settings file. The nudge survives a failed/empty task query (so a brand-new user with no project yet still @@ -109,6 +109,9 @@ Memory's durable graph**, rather than a memory layer of its own. See ### Removed (clean break) +- Hook-specific secret scanning and the `redactKeys` / `redactPaths` settings. + Lifecycle envelopes now stay small by construction instead of carrying a + general-purpose redaction subsystem. - The six bundled skills (`placement`, `knowledge-capture`, `knowledge-organize`, `continue-conversation`, `research`, `edit-note`). Equivalent, framework-agnostic workflows live in the top-level [`skills/`](../../skills) package diff --git a/plugins/claude-code/README.md b/plugins/claude-code/README.md index b8022825c..a9e2db4f6 100644 --- a/plugins/claude-code/README.md +++ b/plugins/claude-code/README.md @@ -91,11 +91,11 @@ local checkout. Two disclosures: - **Network fetch on first run.** uv downloads `basic-memory` from PyPI at a pinned minimum version (bumped by release tooling); later runs use uv's cache. -- **Event capture is opt-in and off by default.** Setting `captureEvents: true` - (the JSON boolean — strings never enable it) records redacted lifecycle-event - envelopes to a local inbox under your Basic Memory home. Inspect with - `basic-memory hook status`, archive locally with `basic-memory hook flush`. - The lifecycle trace never becomes a graph note. +- **Event capture is on by default.** Setting `captureEvents: false` disables + lifecycle-event envelopes. Other values fail closed. Envelopes contain bounded + lifecycle metadata and live in a local inbox under your Basic Memory home. + Inspect with `basic-memory hook status`, archive locally with + `basic-memory hook flush`. The lifecycle trace never becomes a graph note. Every failure path exits 0 — the hooks stay invisible rather than disrupt a session. @@ -151,9 +151,7 @@ settings (or select it via `/config`). | `preCompactCapture` | `extractive` | How checkpoints are produced | | `sessionProfile` | `general` | `coding` makes checkpoints schema-backed `coding_session` notes with required Git identity | | `repository` | _(none)_ | User-confirmed stable repository identifier (`owner/name`); required for the `coding` profile | -| `captureEvents` | `false` | Opt-in: record redacted lifecycle-event envelopes to the local inbox (see `basic-memory hook status` / `flush`). Only the JSON boolean `true` enables it. | -| `redactKeys` | `[]` | Additional payload keys to redact before an event enters the local inbox | -| `redactPaths` | `[]` | Additional paths to redact from working-directory and path-bearing capture content | +| `captureEvents` | `true` | Record bounded lifecycle-event envelopes to the local inbox (see `basic-memory hook status` / `flush`). Set the JSON boolean `false` to disable it; other values fail closed. | The plugin seeds schemas for notes the Claude integration writes directly: `decision`, `task`, and the session type relevant to the selected profile. A diff --git a/plugins/claude-code/hooks/pre_compact.py b/plugins/claude-code/hooks/pre_compact.py index a09018aee..bdb002fd4 100755 --- a/plugins/claude-code/hooks/pre_compact.py +++ b/plugins/claude-code/hooks/pre_compact.py @@ -4,7 +4,7 @@ # dependencies = ["basic-memory>=0.22.1"] # /// """PreCompact hook — the entire hook. All logic (settings resolution, the -extractive checkpoint note, opt-in envelope capture) lives in the released +extractive checkpoint note, lifecycle-envelope capture) lives in the released basic-memory package behind `basic-memory hook pre-compact`; this script is only the launcher, and uv resolves the dependency floor declared above. scripts/update_versions.py bumps that floor at release time. diff --git a/plugins/claude-code/hooks/session_start.py b/plugins/claude-code/hooks/session_start.py index 6642899c9..2dc789223 100755 --- a/plugins/claude-code/hooks/session_start.py +++ b/plugins/claude-code/hooks/session_start.py @@ -4,7 +4,7 @@ # dependencies = ["basic-memory>=0.22.1"] # /// """SessionStart hook — the entire hook. All logic (settings resolution, the -context brief, opt-in envelope capture) lives in the released basic-memory +context brief, lifecycle-envelope capture) lives in the released basic-memory package behind `basic-memory hook session-start`; this script is only the launcher, and uv resolves the dependency floor declared above. scripts/update_versions.py bumps that floor at release time. diff --git a/plugins/claude-code/settings.example.json b/plugins/claude-code/settings.example.json index bfc3dfb9a..34ac768f4 100644 --- a/plugins/claude-code/settings.example.json +++ b/plugins/claude-code/settings.example.json @@ -9,9 +9,7 @@ "recallPrompt": "You have Basic Memory available for this project. Before answering recall questions (\"what did we decide\", \"where did we leave off\"), search the graph first — prefer structured filters (search_notes with type/status). When the user makes a material decision, capture it as a note with type: decision. Cite permalinks when referencing prior work.", "preCompactCapture": "extractive", "sessionProfile": "general", - "captureEvents": false, - "redactKeys": [], - "redactPaths": [], + "captureEvents": true, "placementConventions": null, "teamProjects": { "my-team/notes": { "promoteFolder": "shared" } diff --git a/plugins/claude-code/skills/bm-setup/SKILL.md b/plugins/claude-code/skills/bm-setup/SKILL.md index 5bc3c57e4..dcdd86f93 100644 --- a/plugins/claude-code/skills/bm-setup/SKILL.md +++ b/plugins/claude-code/skills/bm-setup/SKILL.md @@ -111,15 +111,12 @@ Ask only what you can't infer. Cover: task schemas, so I can find them precisely later — okay?" (See "Seed the schemas" below.) -6. **Lifecycle-event capture.** "Should I also keep a local, redacted trail of - SessionStart and PreCompact events for diagnostics?" Default to **off**. - Explain that the normal session brief and PreCompact checkpoint work either - way; enabling this adds envelopes to a local inbox until `bm hook flush` - archives them locally. It never creates knowledge-graph notes. Only the JSON - boolean `true` enables capture. - - If enabled, optionally ask for repo-specific `redactKeys` (additional payload - keys) and `redactPaths` (working directories or path-bearing content). The - built-in redaction floor still applies when these lists are empty. +6. **Lifecycle-event capture.** "The plugin keeps a local trail of SessionStart + and PreCompact events for diagnostics by default. Keep that on?" Default to + **on**. Explain that the normal session brief and PreCompact checkpoint work + either way; capture adds bounded lifecycle metadata to a local inbox until + `bm hook flush` archives it locally. It never creates knowledge-graph notes. + Persist the JSON boolean `false` only when the user opts out. - Capture stays local and personal. It never writes directly to team projects. 7. **How active should I be? (output style)** "Want me to proactively capture — @@ -210,9 +207,7 @@ Build the `basicMemory` block from the interview: "preCompactCapture": "extractive", "sessionProfile": "coding", "repository": "owner/name", - "captureEvents": false, - "redactKeys": [], - "redactPaths": [], + "captureEvents": true, "placementConventions": "", "teamProjects": {} }, @@ -229,9 +224,7 @@ Only include `outputStyle` if the user opted in. Ask whether this is a **team default** (write/merge into `.claude/settings.json`, suggest committing it) or **personal** (`.claude/settings.local.json`). **Merge** into any existing file — read it, add/replace only the keys above, preserve everything else. Use compact, -valid JSON. Always persist `captureEvents` as a JSON boolean. Empty `redactKeys` -and `redactPaths` lists may be omitted; when present, they must be JSON arrays of -strings. +valid JSON. Always persist `captureEvents` as a JSON boolean. Writing the `basicMemory` block is also what stops the SessionStart hook's first-run nudge — the config's presence is the signal that setup has run. @@ -264,8 +257,8 @@ ref before closing — don't let the next session's brief come up empty. Confirm what you did in a few lines: the project mapping, the session profile (and confirmed repository for a coding setup), which schemas were seeded vs. already present, whether placement was learned or suggested, the smoke-test -result, whether lifecycle-event capture is enabled, any extra redaction controls, -the shared hook inbox/flush state, and whether the output style is on. +result, whether lifecycle-event capture is enabled, the shared hook inbox/flush +state, and whether the output style is on. Then handle activation based on the output style: - **Output style enabled** → it's fixed at session start, so the full capture diff --git a/plugins/claude-code/skills/bm-status/SKILL.md b/plugins/claude-code/skills/bm-status/SKILL.md index 64ad9a736..fdf4b5c95 100644 --- a/plugins/claude-code/skills/bm-status/SKILL.md +++ b/plugins/claude-code/skills/bm-status/SKILL.md @@ -26,7 +26,7 @@ This is a quick diagnostic — gather the facts and lay them out; don't over-inv (default `sessions`), `rememberFolder` (default `bm-remember`), `preCompactCapture` mode (default `extractive`), `sessionProfile` (default `general`), `repository` (coding profile only), `captureEvents` (default - `false`), `redactKeys`, and `redactPaths`. + `true`). - From the **root** settings object (not `basicMemory`): whether `outputStyle` is `basic-memory` — i.e. whether the capture reflexes are on. @@ -73,8 +73,6 @@ you couldn't determine, rather than failing the whole report): - Session profile: - Repository: - Event capture: -- Redact keys: -- Redact paths: - Shared hook inbox: - Shared pending envelopes: - Shared archived envelopes: diff --git a/pyproject.toml b/pyproject.toml index 34e03061b..42b6b4392 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,6 @@ dependencies = [ # asyncpg engine-dispose race ("IndexError: pop from an empty deque") that # crashes the Postgres backend cannot fire under it. Not available on Windows. "uvloop>=0.21.0; sys_platform != 'win32'", - "detect-secrets>=1.5", # Cross-platform advisory file lock (already resolved transitively via # huggingface-hub; declared directly since the hook flush path imports it). # Serializes concurrent `bm hook flush` runs so a stale sweep can't overwrite diff --git a/scripts/validate_claude_plugin.py b/scripts/validate_claude_plugin.py index b25d93650..e8fe10059 100644 --- a/scripts/validate_claude_plugin.py +++ b/scripts/validate_claude_plugin.py @@ -41,8 +41,7 @@ REQUIRED_SKILL_TEXT: dict[str, tuple[str, ...]] = { "bm-setup": ( "captureEvents", - "redactKeys", - "redactPaths", + '"captureEvents": true', "sessionProfile", "coding-session.md", "hook status --harness claude", @@ -203,6 +202,14 @@ def validate_claude_plugin(plugin_dir: Path) -> None: if required_text not in readme: raise SystemExit(f"README.md: missing schema ownership text {required_text!r}") + settings = read_json(plugin_dir / "settings.example.json") + basic_memory_settings = settings.get("basicMemory", {}) + if basic_memory_settings.get("captureEvents") is not True: + raise SystemExit("settings.example.json: captureEvents must default to true") + for retired_key in ("redactKeys", "redactPaths"): + if retired_key in basic_memory_settings: + raise SystemExit(f"settings.example.json: retired key {retired_key!r}") + print(f"validated Claude Code plugin in {plugin_dir}") diff --git a/src/basic_memory/cli/commands/hook.py b/src/basic_memory/cli/commands/hook.py index 31f10a21c..5f7c379bc 100644 --- a/src/basic_memory/cli/commands/hook.py +++ b/src/basic_memory/cli/commands/hook.py @@ -13,8 +13,8 @@ existing Codex configuration has not been reinstalled yet. - Codex checkpoint prompting defaults on. An explicit JSON boolean ``false`` disables it; malformed values and malformed config fail closed. - - Codex event capture defaults on. An explicit JSON boolean ``false`` turns - it off, while malformed values and malformed config fail closed. + - Lifecycle-event capture defaults on for both harnesses. An explicit JSON + boolean ``false`` turns it off, while malformed values fail closed. - Graph-derived brief content is fenced and labeled as reference data, not instructions — the prompt-injection boundary. @@ -54,8 +54,7 @@ from basic_memory.hooks.adapters import NormalizedHookEvent, for_harness # Envelope event names, duplicated as literals would invite drift; the -# envelope module itself is imported lazily (it pulls detect-secrets) inside -# the capture path (#886: keep CLI import time lean). +# envelope module itself is imported lazily to keep CLI import time lean. SESSION_STARTED = "session_started" COMPACTION_IMMINENT = "compaction_imminent" @@ -76,8 +75,8 @@ class Harness(str, Enum): # Cap how many shared projects we read per session — bounds latency and output. MAX_SHARED = 6 CODING_SESSION_PROFILE = "coding" +DEFAULT_CAPTURE_EVENTS = True CODEX_DEFAULT_CHECKPOINT_ON_COMPACT = True -CODEX_DEFAULT_CAPTURE_EVENTS = True CODEX_CHECKPOINT_PROMPT = ( "Basic Memory checkpoint required after compaction. Use the " "`codex:bm-checkpoint` skill now to write one deliberate, durable handoff " @@ -206,13 +205,20 @@ def _read_stdin_payload() -> dict: # --- Harness settings resolution (ported from the plugin hook scripts) --- -def _read_claude_block(path: Path) -> dict | None: +def _read_claude_block(path: Path) -> tuple[dict | None, bool]: + """Read one Claude settings block and preserve malformed-file presence.""" try: data = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + return None, False except (OSError, json.JSONDecodeError): - return None - block = data.get("basicMemory") if isinstance(data, dict) else None - return block if isinstance(block, dict) else None + return None, True + if not isinstance(data, dict): + return None, True + if "basicMemory" not in data: + return None, False + block = data["basicMemory"] + return (block if isinstance(block, dict) else None), True def _claude_project_dir(directory: Path) -> Path: @@ -238,9 +244,12 @@ def load_claude_settings(directory: Path) -> tuple[dict, bool]: nearest project ``.claude/settings.json`` and ``.claude/settings.local.json``. A single user-level block can cover every project; any project can still pin its own mapping, which wins. ``found`` reports whether any file - declared a block — the first-run sentinel for the setup nudge. + declared a block or was malformed — the first-run sentinel for the setup + nudge. Any malformed source fails closed for capture for the whole + evaluation so a later source cannot rebuild routing from incomplete + settings. """ - merged: dict = {} + merged: dict = {"captureEvents": DEFAULT_CAPTURE_EVENTS} found = False home = Path.home() sources: list[tuple[Path, tuple[str, ...]]] = [(home, ("settings.json",))] @@ -249,10 +258,16 @@ def load_claude_settings(directory: Path) -> tuple[dict, bool]: sources.append((project, ("settings.json", "settings.local.json"))) for base, names in sources: for name in names: - block = _read_claude_block(base / ".claude" / name) - if block is not None: - found = True - merged.update(block) + block, present = _read_claude_block(base / ".claude" / name) + if not present: + continue + found = True + if block is None: + # Trigger: a configured source exists but cannot be trusted. + # Why: its unreadable value may be an explicit capture opt-out. + # Outcome: discard every route and disable capture for this event. + return {"captureEvents": False}, True + merged.update(block) return merged, found @@ -316,12 +331,11 @@ def load_codex_settings(directory: Path) -> tuple[dict, bool]: checkpoint prompting are enabled when omitted. The default folder is namespaced by the Git repository directory. Any malformed source counts as configured and fails closed for the whole evaluation so a later source - cannot rebuild routing from incomplete settings. Existing redaction lists - continue to accumulate only for local lifecycle-envelope capture. + cannot rebuild routing from incomplete settings. """ defaults: dict = { "checkpointOnCompact": CODEX_DEFAULT_CHECKPOINT_ON_COMPACT, - "captureEvents": CODEX_DEFAULT_CAPTURE_EVENTS, + "captureEvents": DEFAULT_CAPTURE_EVENTS, "captureFolder": _codex_default_capture_folder(directory), } merged = dict(defaults) @@ -348,18 +362,7 @@ def load_codex_settings(directory: Path) -> tuple[dict, bool]: "checkpointOnCompact": False, "captureEvents": False, }, True - cumulative_redactions: dict[str, list[str]] = {} - for key in ("redactKeys", "redactPaths"): - values = [*_string_list(merged.get(key)), *_string_list(block.get(key))] - if values: - cumulative_redactions[key] = list(dict.fromkeys(values)) merged.update(block) - merged.update(cumulative_redactions) - - # This legacy option imposed an extra model-authored checkpoint scan. - # Ignore it so existing config cannot restore that gate. Redaction lists - # remain available only to the separate lifecycle-envelope capture path. - merged.pop("checkpointPrivacyReview", None) return merged, found @@ -370,13 +373,6 @@ def load_harness_settings(harness: Harness, directory: Path) -> tuple[dict, bool return load_codex_settings(directory) -def _string_list(value: Any) -> list[str]: - """Guard config JSON types: only a list of strings passes through.""" - if not isinstance(value, list): - return [] - return [item for item in value if isinstance(item, str)] - - def _shared_project_refs(cfg: dict, primary_project: str) -> tuple[list[str], bool]: """Resolve the shared/team read set: secondaryProjects + teamProjects keys. @@ -407,11 +403,10 @@ def _mapping_dir(project_dir: Optional[Path], event_cwd: str) -> Path: return Path.cwd() -# --- Envelope capture (opt-in, fail-closed gate) --- +# --- Envelope capture --- def _capture_envelope( - profile: HarnessProfile, event: NormalizedHookEvent, envelope_event: str, cfg: dict, @@ -421,16 +416,13 @@ def _capture_envelope( """Capture one lifecycle event into the inbox WAL when enabled. Trigger: ``captureEvents`` is the JSON boolean ``true`` — strict identity, - never truthiness. Why: a privacy gate must fail closed; a hand-edited - string like "false" (truthy in Python) must not enable recording. - Outcome: envelope built, floor-redacted, appended; failures are best-effort - (stderr) so the brief/checkpoint still runs. + never truthiness. Why: a hand-edited string like "false" must not enable + recording. Outcome: append bounded lifecycle metadata; failures remain + best-effort so the brief or checkpoint still runs. """ if cfg.get("captureEvents") is not True: return try: - # Deferred: the envelope module pulls detect-secrets; loading it on - # every CLI start would slow all commands (#886). from basic_memory.hooks.envelope import create_envelope from basic_memory.hooks.inbox import write_envelope @@ -451,8 +443,6 @@ def _capture_envelope( project_hint=str(cfg.get("primaryProject") or "").strip(), turn_id=event.turn_id, payload=payload, - extra_redact_keys=_string_list(cfg.get("redactKeys")), - extra_redact_paths=_string_list(cfg.get("redactPaths")), ) write_envelope(envelope) except Exception as exc: @@ -907,7 +897,6 @@ def _checkpoint_note( primary: str, working_directory: str, coding_context: CodingContext | None, - extra_redact_paths: list[str], ) -> tuple[str, str, dict[str, Any]]: """Build the pre-compaction checkpoint note (title, body, frontmatter). @@ -919,22 +908,7 @@ def _checkpoint_note( a hand-built frontmatter block and, via fail-open, silently drop the checkpoint. ``type`` is supplied to write_note separately (``note_type``). """ - # Transcript text is lifted verbatim into the graph (title, summary, and - # observations), so it must pass the same secret floor as inbox payloads - # (#997: redact obvious secrets before writing artifacts). Redact once at - # extraction — every downstream use draws from the redacted strings. - # Deferred import: redaction pulls detect-secrets, too heavy for CLI start (#886). - from basic_memory.hooks.redaction import Redactor - - # One ruleset for the whole checkpoint: every turn and the cwd share the - # same deny rules, so compile the patterns once rather than per string. - redactor = Redactor.build(extra_redact_paths=extra_redact_paths) - - user_messages = [redactor.redact_text(text) for role, text in conversation if role == "user"] - # cwd is a user path too: a session under a configured redactPaths (or a - # default deny dir) must not leak the raw path into the note frontmatter or - # body. Redact once so both draw from the scrubbed string. - safe_cwd = redactor.redact_text(working_directory) + user_messages = [text for role, text in conversation if role == "user"] opening = user_messages[0] recent_user = user_messages[-3:] @@ -951,7 +925,7 @@ def _checkpoint_note( "started": iso, "ended": iso, "project": primary, - "cwd": safe_cwd, + "cwd": working_directory, } if event.session_id: metadata[profile.session_id_key] = event.session_id @@ -963,25 +937,20 @@ def _checkpoint_note( metadata["model"] = event.model metadata["capture"] = "extractive" - safe_coding_context: dict[str, str] | None = None + checkpoint_coding_context: dict[str, str] | None = None if coding_context is not None: # Trigger: coding sessions require repo_root == cwd to be comparable identity. # Why: git emits repo_root with forward slashes on every platform, while the # event cwd arrives in native form — on Windows that's C:\Users vs C:/Users. # Outcome: store the coding note's cwd in the same POSIX form as repo_root. - # (Redaction ran first; the [redacted-path] sentinel has no separators and - # passes through as_posix unchanged.) - metadata["cwd"] = Path(safe_cwd).as_posix() - safe_coding_context = { - "repository": redactor.redact_text(coding_context.repository), - "repo_root": redactor.redact_text(coding_context.repo_root), - "branch": redactor.redact_text(coding_context.branch), - # A commit SHA is public repository identity, but its hex shape trips - # high-entropy secret detection. Preserve it so coding checkpoints - # remain queryable by the exact revision they describe. + metadata["cwd"] = Path(working_directory).as_posix() + checkpoint_coding_context = { + "repository": coding_context.repository, + "repo_root": coding_context.repo_root, + "branch": coding_context.branch, "git_sha": coding_context.git_sha, } - metadata.update(safe_coding_context) + metadata.update(checkpoint_coding_context) if coding_context.pull_request is not None: pull_request = coding_context.pull_request metadata.update( @@ -989,11 +958,11 @@ def _checkpoint_note( # PR numbers are identifiers, not quantities. Keeping them as strings # also makes exact metadata queries portable across SQLite and Postgres. "pull_request_number": str(pull_request.number), - "pull_request_title": redactor.redact_text(pull_request.title), - "pull_request_url": redactor.redact_text(pull_request.url), + "pull_request_title": pull_request.title, + "pull_request_url": pull_request.url, "pull_request_state": pull_request.state, - "pull_request_base": redactor.redact_text(pull_request.base_branch), - "pull_request_head": redactor.redact_text(pull_request.head_branch), + "pull_request_base": pull_request.base_branch, + "pull_request_head": pull_request.head_branch, } ) @@ -1006,19 +975,19 @@ def _checkpoint_note( "resume._", "", "## Summary", - f"Working in `{safe_cwd}`.", + f"Working in `{working_directory}`.", f"- Opening request: {_clip(opening, 300)}", "", "## Recent thread", *[f"- {_clip(message, 200)}" for message in recent_user], ] - if safe_coding_context is not None: + if checkpoint_coding_context is not None: body += [ "", "## Repository", - f"- Repository: `{safe_coding_context['repository']}`", - f"- Branch: `{safe_coding_context['branch']}`", - f"- Git SHA: `{safe_coding_context['git_sha']}`", + f"- Repository: `{checkpoint_coding_context['repository']}`", + f"- Branch: `{checkpoint_coding_context['branch']}`", + f"- Git SHA: `{checkpoint_coding_context['git_sha']}`", ] if coding_context is not None and coding_context.pull_request is not None: body.append( @@ -1065,7 +1034,7 @@ def _session_start(harness: Harness, project_dir: Optional[Path]) -> None: cfg, configured = load_harness_settings(harness, mapping_dir) capture_folder = str(cfg.get("captureFolder") or profile.default_capture_folder).strip() - _capture_envelope(profile, event, SESSION_STARTED, cfg, mapping_dir, capture_folder) + _capture_envelope(event, SESSION_STARTED, cfg, mapping_dir, capture_folder) primary = str(cfg.get("primaryProject") or "").strip() checkpoint_prompt = ( @@ -1092,7 +1061,7 @@ def _pre_compact(harness: Harness, project_dir: Optional[Path]) -> None: # Capture before the checkpoint gates: capture is dumb, and an unmapped or # transcript-less session is still trace worth keeping in the WAL. - _capture_envelope(profile, event, COMPACTION_IMMINENT, cfg, mapping_dir, capture_folder) + _capture_envelope(event, COMPACTION_IMMINENT, cfg, mapping_dir, capture_folder) primary = str(cfg.get("primaryProject") or "").strip() # Trigger: no project pinned. Why: a checkpoint must land somewhere @@ -1125,7 +1094,6 @@ def _pre_compact(harness: Harness, project_dir: Optional[Path]) -> None: primary, working_directory, coding_context, - _string_list(cfg.get("redactPaths")), ) # Deferred import (#886); same internal write path as `bm tool write-note`. @@ -1200,7 +1168,6 @@ def flush( ), ) -> None: """Archive pending lifecycle envelopes locally; never write graph notes.""" - # Deferred: the archive sweep pulls the envelope stack (detect-secrets) (#886). from basic_memory.hooks.archive import flush as run_flush result = run_with_cleanup(run_flush(older_than_days=older_than_days)) diff --git a/src/basic_memory/config_models.py b/src/basic_memory/config_models.py index 28c10dfd4..e165362ab 100644 --- a/src/basic_memory/config_models.py +++ b/src/basic_memory/config_models.py @@ -243,15 +243,13 @@ def __init__(self, **data: Any) -> None: ... default_factory=_default_semantic_search_enabled, description="Enable semantic search (vector/hybrid retrieval). Works on both SQLite and Postgres backends. Requires semantic dependencies (included by default).", ) - semantic_vector_index: str = Field( + semantic_vector_index: Literal["pgvector", "milvus"] = Field( default="pgvector", description=( - "Semantic vector index backend for Postgres deployments. 'pgvector' is built in, " - "'milvus' is available through the optional basic-memory[milvus] extra, and other " - "names resolve through the basic_memory.semantic_vector_indexes entry-point group. " - "SQLite continues to use sqlite-vec." + "Semantic vector index backend for Postgres deployments. Use 'pgvector' or " + "'milvus' through the optional basic-memory[milvus] extra. SQLite continues " + "to use sqlite-vec." ), - min_length=1, ) milvus_uri: str | None = Field( default=None, diff --git a/src/basic_memory/hooks/__init__.py b/src/basic_memory/hooks/__init__.py index 458c4a775..a252358de 100644 --- a/src/basic_memory/hooks/__init__.py +++ b/src/basic_memory/hooks/__init__.py @@ -2,17 +2,15 @@ Agent harnesses (Claude Code, Codex) fire lifecycle hooks; this package is the producer side of the harness WAL. Capture is dumb: hook stdin is normalized by -a per-harness adapter, wrapped in a redacted producer envelope, and appended to +a per-harness adapter, wrapped in a bounded producer envelope, and appended to the local inbox. ``bm hook flush`` retires valid trace into a local audit archive; durable knowledge is written separately by an active agent or explicit workflow. Modules: - ``_uuid7`` time-ordered event ids (inbox filenames sort chronologically) - ``envelope`` the SPEC-55 producer envelope contract - - ``redaction`` Stage-1 deterministic redaction floor (always on) - ``inbox`` append-only WAL under the Basic Memory home dir - ``adapters`` per-harness hook stdin normalization - ``archive`` idempotent local audit-archive sweep - - ``projector`` compatibility imports for the retired graph projector - ``project_ref`` project-name / project-id routing helpers """ diff --git a/src/basic_memory/hooks/_uuid7.py b/src/basic_memory/hooks/_uuid7.py index 50af5378b..59e7c1809 100644 --- a/src/basic_memory/hooks/_uuid7.py +++ b/src/basic_memory/hooks/_uuid7.py @@ -5,7 +5,7 @@ ``uuid.uuid7()`` when the floor rises. The 48-bit millisecond timestamp prefix means UUIDv7 strings sort -lexicographically into chronological order — the projector processes +lexicographically into chronological order — the archive sweep processes ``sorted(glob)`` with no mtime/stat dependence. """ diff --git a/src/basic_memory/hooks/adapters/__init__.py b/src/basic_memory/hooks/adapters/__init__.py index d4ecc5637..276696302 100644 --- a/src/basic_memory/hooks/adapters/__init__.py +++ b/src/basic_memory/hooks/adapters/__init__.py @@ -1,7 +1,7 @@ """Per-harness hook stdin adapters. Each harness speaks its own hook JSON dialect; an adapter normalizes it into -``NormalizedHookEvent`` so everything downstream (envelope, projector, CLI) is +``NormalizedHookEvent`` so everything downstream (envelope, archive, CLI) is harness-agnostic. Adding a harness means adding one small module here plus its recorded fixtures — nothing else changes. """ diff --git a/src/basic_memory/hooks/envelope.py b/src/basic_memory/hooks/envelope.py index a06bd16fb..fe1fd3edd 100644 --- a/src/basic_memory/hooks/envelope.py +++ b/src/basic_memory/hooks/envelope.py @@ -7,7 +7,7 @@ Envelopes are trace, not memory: they remain ``promotion_status: raw`` and are archived locally rather than promoted into the graph. The idempotency key is -computed from metadata only, so redaction never changes identity. +computed from metadata only, so bounded payload changes never change identity. """ from __future__ import annotations @@ -18,7 +18,6 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator from basic_memory.hooks._uuid7 import uuid7 -from basic_memory.hooks.redaction import Redactor ENVELOPE_VERSION = 1 @@ -74,7 +73,7 @@ class Envelope(BaseModel): actor: str = ACTOR_RUNTIME # "runtime" | "user" | routine name caused_by: str | None = None # id of the triggering event, when known promotion_status: str = PROMOTION_RAW - payload: dict = Field(default_factory=dict) # redacted summary only + payload: dict = Field(default_factory=dict) # bounded lifecycle metadata only @field_validator("envelope_version") @classmethod @@ -120,31 +119,18 @@ def create_envelope( actor: str = ACTOR_RUNTIME, caused_by: str | None = None, payload: dict | None = None, - extra_redact_keys: list[str] | None = None, - extra_redact_paths: list[str] | None = None, ) -> Envelope: """Factory: build a producer envelope from normalized hook inputs. Keyword-only to prevent positional-order mistakes when callers construct - envelopes from heterogeneous payload shapes. Both the payload and the ``cwd`` - pass through the Stage-1 redaction floor here, at the factory — no envelope - built through this path can carry unredacted payload values or a denied - workspace path into the inbox. ``project_hint`` is a project name, not a - path, so it is left intact as capture-time diagnostic context. + envelopes from heterogeneous payload shapes. Callers keep payloads bounded + to lifecycle metadata; the factory validates the stable envelope contract + before it enters the inbox. """ if event not in V0_EVENTS: raise ValueError(f"Unknown event {event!r}; v0 supports: {sorted(V0_EVENTS)}") resolved_ts = ts or datetime.now(timezone.utc).isoformat(timespec="seconds") - # One ruleset for both the payload and the cwd: they share the same deny - # rules, so compiling once avoids re-expanding paths and recompiling patterns. - redactor = Redactor.build( - extra_redact_keys=extra_redact_keys, extra_redact_paths=extra_redact_paths - ) - safe_payload = redactor.redact_payload(payload or {}) - # cwd is a user path: a session under a configured redactPaths (or a default - # deny dir) must not persist the raw path in the inbox WAL. - safe_cwd = redactor.redact_text(cwd) return Envelope( id=str(uuid7()), @@ -153,12 +139,12 @@ def create_envelope( source_session_id=session_id, source_turn_id=turn_id, ts=resolved_ts, - cwd=safe_cwd, + cwd=cwd, project_hint=project_hint, actor=actor, caused_by=caused_by, idempotency_key=idempotency_key(source, session_id, event, resolved_ts), - payload=safe_payload, + payload=dict(payload or {}), ) diff --git a/src/basic_memory/hooks/projector.py b/src/basic_memory/hooks/projector.py deleted file mode 100644 index b50da25e0..000000000 --- a/src/basic_memory/hooks/projector.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Compatibility imports for the retired lifecycle-event projector. - -Lifecycle trace is no longer promoted into graph notes. New code should import -the local archive sweep from :mod:`basic_memory.hooks.archive` and project-ref -routing from :mod:`basic_memory.hooks.project_ref`. -""" - -from basic_memory.hooks.archive import FlushResult, flush -from basic_memory.hooks.project_ref import UUID_RE, split_project_ref - -__all__ = ["FlushResult", "UUID_RE", "flush", "split_project_ref"] diff --git a/src/basic_memory/hooks/redaction.py b/src/basic_memory/hooks/redaction.py deleted file mode 100644 index 309569173..000000000 --- a/src/basic_memory/hooks/redaction.py +++ /dev/null @@ -1,370 +0,0 @@ -"""Stage-1 deterministic redaction floor for captured hook payloads (SPEC-55). - -Everything that enters the inbox passes through this floor at capture time. -It combines two layers: - - 1. ``detect-secrets`` (Yelp) scanning over every payload string — known token - formats (AWS ``AKIA…``, GitHub ``ghp_…``, JWTs, private-key blocks, …) plus - an entropy threshold on long opaque strings. - 2. The recursive deny-key / deny-path / env-pair / truncation rules carried - over from the #1064 salvage branch, hardened for Windows separators. - -Dependency decision (2026-07-15): ``detect-secrets`` is a core dependency, not -an extra. Its tree is light (pyyaml — already core — plus requests), and the -floor must be unconditionally present on the capture hot path: an optional -extra would make redaction availability configuration-dependent, violating the -"Stage 1 · always on" contract. The Stage-2 model scrub (phase 2) is what ships -behind ``basic-memory[redaction]``. - -The public surface is the :class:`Redactor` value object: build a ruleset once -(``Redactor.build(...)``) and reuse it across many payloads/strings — redacting -each turn of a transcript must not recompile deny patterns or re-expand paths. -The module-level :func:`redact_payload` / :func:`redact_text` are one-shot -conveniences that build a throwaway redactor for a single value. - -Contract: redaction is pure (never mutates its input) and idempotent -(``redact_payload(redact_payload(p)) == redact_payload(p)``) — the projector -may re-apply it freely. -""" - -from __future__ import annotations - -import os -import re -from dataclasses import dataclass -from typing import Any - -from detect_secrets.core.plugins.util import get_mapping_from_secret_type_to_class -from detect_secrets.core.scan import scan_line -from detect_secrets.plugins.high_entropy_strings import HighEntropyStringsPlugin -from detect_secrets.settings import default_settings - -REDACTED = "[REDACTED]" -REDACTED_PATH = "[REDACTED_PATH]" - -# Maximum length for any single payload string before truncation. -MAX_PAYLOAD_VALUE_LEN = 500 -TRUNCATION_MARKER = "…[truncated]" - -# Keys whose values look like secrets, matched case-insensitively against -# payload dict keys as full word segments (delimited by _ or . or string -# boundaries). This catches API_KEY, AUTH_TOKEN, DB_PASSWORD but not -# "safe_key" or "monkey". Users extend the list via extra_redact_keys. -DEFAULT_REDACT_KEY_PATTERNS = ( - re.compile(r"(?i)(?:^|[_.])(?:SECRET|TOKEN|PASSWORD|CREDENTIAL|AUTH)(?:[_.]|$)"), - re.compile(r"(?i)(?:^|[_.])(?:API[_.]KEY|ACCESS[_.]KEY|PRIVATE[_.]KEY)(?:[_.]|$)"), -) - -# Values that look like environment secrets: KEY=. -SECRET_VALUE_RE = re.compile(r"^[A-Za-z0-9_]+=.{20,}$") - -# Sensitive home directories, in the ``~/`` shell form users actually type. -_SENSITIVE_HOME_DIRS = ("~/.ssh/", "~/.aws/", "~/.gnupg/") - -# detect-secrets entropy plugins, keyed by the secret type they emit. Rebuilt -# per redaction call inside a ``default_settings()`` context, so this shape is -# passed down the traversal rather than stored on the ruleset. -type EntropyPlugins = dict[str, HighEntropyStringsPlugin] - - -# --- Path helpers --- - - -def _normalize_path(path: str) -> str: - """Compare paths with forward slashes only. - - ``os.path.expanduser("~/.ssh/")`` yields mixed separators on Windows - (``C:\\Users\\x/.ssh/``) while native payload values use backslashes, so an - un-normalized ``startswith`` never matches there. - """ - return path.replace("\\", "/") - - -def _expand_deny_paths(paths: tuple[str, ...]) -> tuple[str, ...]: - """Normalize deny-path prefixes into both matchable forms. - - Both forms are denied for each prefix: the expanded absolute path (payload - values — hook cwd especially — usually carry it resolved) and the literal - ``~/`` prefix (prose, config, and transcript excerpts commonly write - ``~/.ssh/id_rsa`` unexpanded — the expanded pattern alone would let that - survive, and vice versa). dict.fromkeys dedupes while preserving order in - case expanduser is a no-op (HOME unset, or an already-absolute path). - """ - expanded = (_normalize_path(os.path.expanduser(prefix)) for prefix in paths) - literal = (_normalize_path(prefix) for prefix in paths) - return tuple(dict.fromkeys((*expanded, *literal))) - - -def _default_redact_paths() -> tuple[str, ...]: - # Resolved per call, not at import: tests (and long-lived processes) may - # repoint HOME, and a stale import-time expansion would silently miss. - return _expand_deny_paths(_SENSITIVE_HOME_DIRS) - - -# --- detect-secrets scanning --- - - -def _entropy_plugins() -> EntropyPlugins: - """Instantiate the entropy plugins with their default limits, keyed by secret type.""" - return { - cls.secret_type: cls() - for cls in get_mapping_from_secret_type_to_class().values() - if issubclass(cls, HighEntropyStringsPlugin) - } - - -def _detected_secret_values(line: str, entropy_plugins: EntropyPlugins) -> list[str] | None: - """Return secret substrings detect-secrets found in ``line``. - - Returns None when a detection cannot be localized to a substring — the - caller must then redact the whole line. - - Constraint: ``scan_line`` runs entropy plugins in eager mode, which - deliberately skips their entropy limit so ad-hoc scans can show "why" - values. That surfaces every token as a candidate, so the limit is re-applied - here — otherwise ordinary prose would be redacted wholesale. - """ - values: list[str] = [] - for secret in scan_line(line): - value = secret.secret_value - if value is None: # pragma: no cover - no default plugin emits valueless secrets - return None - entropy_plugin = entropy_plugins.get(secret.type) - if entropy_plugin is not None and ( - entropy_plugin.calculate_shannon_entropy(value) <= entropy_plugin.entropy_limit - ): - continue - values.append(value) - return values - - -def _scrub_secrets(value: str, entropy_plugins: EntropyPlugins) -> str: - # detect-secrets plugins are line-oriented; scan each line so a secret in a - # multi-line payload value is caught just like a single-line one. - scrubbed_lines: list[str] = [] - for line in value.split("\n"): - found = _detected_secret_values(line, entropy_plugins) - if found is None: # pragma: no cover - see _detected_secret_values - scrubbed_lines.append(REDACTED) - continue - # Longest-first replacement: a detector may report both a full token and - # a prefix of it; replacing the prefix first would break the full match. - for secret_value in sorted(set(found), key=len, reverse=True): - line = line.replace(secret_value, REDACTED) - scrubbed_lines.append(line) - return "\n".join(scrubbed_lines) - - -def _truncate(value: str) -> str: - if len(value) <= MAX_PAYLOAD_VALUE_LEN: - return value - # Idempotence: a value truncated by a previous pass is MAX + marker long; - # truncating it again would chew the marker into the payload text. - if value.endswith(TRUNCATION_MARKER) and ( - len(value) <= MAX_PAYLOAD_VALUE_LEN + len(TRUNCATION_MARKER) - ): - return value - return value[:MAX_PAYLOAD_VALUE_LEN] + TRUNCATION_MARKER - - -# --- Deny paths --- - - -@dataclass(frozen=True, slots=True) -class DenyPath: - """A denied directory as both a normalized ``root`` and its prose matcher. - - Deny paths are stored with forward slashes and a trailing separator. The - ``root`` (trailing slash stripped) drives the whole-value check, which - tolerates spaces the substring ``\\S*`` tail would truncate; ``pattern`` - matches a denied path token embedded in free text. - """ - - root: str - pattern: re.Pattern[str] - case_insensitive: bool - - @classmethod - def compile(cls, prefix: str, *, case_insensitive: bool) -> DenyPath | None: - """Compile a normalized deny-path prefix, or ``None`` to skip it. - - A bare ``"/"`` (or empty) prefix would redact every path, so it is - skipped. The prose matcher matches the denied directory **root itself** - (``~/.ssh``) as well as any descendant (``~/.ssh/id_rsa``): a - negative-lookahead boundary ``(?![A-Za-z0-9_-])`` rejects only a bare - alphanumeric/underscore/hyphen continuation — so a sibling like - ``/srv/clientsbackup`` (for ``/srv/clients/``) can't match — while a - separator, whitespace, end, or punctuation ends the token (prose puts a - root right before ``,`` or ``.``). The optional ``[/\\]\\S*`` tail - consumes a descendant up to the next whitespace; a path embedded in - prose whose directory contains a space is therefore truncated at that - space (the whole-value check below covers the real capture channel). - - Each ``/`` matches either separator so native Windows backslash values - match a forward-slash deny path. On Windows the filesystem is - case-insensitive, so the pattern is compiled case-insensitively there - (``C:\\Users\\Alice\\.ssh`` == ``c:\\users\\alice\\.ssh``); POSIX stays - case-sensitive (``/home/Alice`` and ``/home/alice`` are distinct). - """ - root = prefix.rstrip("/") - if not root: - return None - escaped = re.escape(root).replace("/", r"[/\\]") - flags = re.IGNORECASE if case_insensitive else 0 - pattern = re.compile(escaped + r"(?![A-Za-z0-9_-])(?:[/\\]\S*)?", flags) - return cls(root=root, pattern=pattern, case_insensitive=case_insensitive) - - def matches_whole(self, normalized_value: str) -> bool: - """Whether ``normalized_value`` is, in full, this directory or a descendant. - - Path-prefix logic on the whole value, so a spaced path (a cwd like - ``/srv/clients/acme corp/repo``) is caught — the ``pattern`` tail would - stop at the first space and leak the rest. Case-folded when the ruleset - is case-insensitive to match the Windows filesystem. - """ - candidate = normalized_value.casefold() if self.case_insensitive else normalized_value - target = self.root.casefold() if self.case_insensitive else self.root - return candidate == target or candidate.startswith(target + "/") - - -# --- The redactor --- - - -@dataclass(frozen=True, slots=True) -class Redactor: - """A compiled Stage-1 redaction ruleset, reusable across many values. - - Build once (:meth:`build`) and reuse: the deny-key and deny-path patterns - are compiled up front so redacting each turn of a transcript does not - recompile the ruleset or re-expand paths. Redaction is pure (never mutates - its input) and idempotent. - """ - - deny_key_patterns: tuple[re.Pattern[str], ...] - deny_paths: tuple[DenyPath, ...] - - @classmethod - def build( - cls, - *, - extra_redact_keys: list[str] | None = None, - extra_redact_paths: list[str] | None = None, - ) -> Redactor: - """Compile the default ruleset, extended with caller-supplied deny rules. - - ``extra_redact_paths`` are expanded like the built-in defaults: a - configured ``~/clients/secret`` must match the absolute cwd - ``/home/alice/clients/...`` a hook actually captures. - """ - key_patterns = list(DEFAULT_REDACT_KEY_PATTERNS) - if extra_redact_keys: - key_patterns.extend( - re.compile(re.escape(pattern), re.IGNORECASE) for pattern in extra_redact_keys - ) - - prefixes = _default_redact_paths() - if extra_redact_paths: - prefixes = prefixes + _expand_deny_paths(tuple(extra_redact_paths)) - - # Read os.name live (not at import): tests repoint it and the same - # interpreter serves one platform for its whole life, so build-time is - # the right, cheap place to settle case sensitivity for the ruleset. - case_insensitive = os.name == "nt" - deny_paths = tuple( - path - for prefix in prefixes - if (path := DenyPath.compile(prefix, case_insensitive=case_insensitive)) is not None - ) - return cls(deny_key_patterns=tuple(key_patterns), deny_paths=deny_paths) - - def redact_payload(self, payload: dict) -> dict: - """Return a copy of ``payload`` with secrets, denied paths, and oversized - values replaced by markers, recursively over nested dicts and lists. - - Nothing downstream (inbox, projector, artifacts) sees unredacted values. - """ - # One settings context per payload: detect-secrets reads plugin/filter - # configuration from process-global settings, and the context both pins - # the default configuration and restores whatever was active before. - with default_settings(): - return self._redact_dict(payload, _entropy_plugins()) - - def redact_text(self, value: str) -> str: - """Return ``value`` with secrets and denied paths replaced by markers. - - Key-based denial has no meaning for free text; this runs the per-string - floor (secret/entropy scanning + path denial) that payload strings get. - """ - with default_settings(): - return self._redact_str(value, _entropy_plugins()) - - # --- traversal --- - - def _redact_str(self, value: str, entropy_plugins: EntropyPlugins) -> str: - if SECRET_VALUE_RE.match(value): - return REDACTED - # A value that is wholly a denied path (or a descendant) collapses to the - # marker via path-prefix logic, so spaces in the path don't leak. - normalized = _normalize_path(value) - if any(path.matches_whole(normalized) for path in self.deny_paths): - return REDACTED_PATH - # Otherwise replace any denied-path token embedded in prose (checkpoint - # excerpts, #997) in place, then run secret/entropy scanning + truncation - # on the remainder. - for path in self.deny_paths: - value = path.pattern.sub(REDACTED_PATH, value) - return _truncate(_scrub_secrets(value, entropy_plugins)) - - def _redact_value(self, value: Any, entropy_plugins: EntropyPlugins) -> Any: - """Redact a payload value of any JSON-compatible shape. - - Payloads arrive from hook JSON, so nested dicts and lists are normal — a - secret one level down must be caught just like a top-level one. - """ - if isinstance(value, str): - return self._redact_str(value, entropy_plugins) - if isinstance(value, dict): - return self._redact_dict(value, entropy_plugins) - if isinstance(value, (list, tuple)): - return [self._redact_value(item, entropy_plugins) for item in value] - return value - - def _redact_dict(self, payload: dict, entropy_plugins: EntropyPlugins) -> dict: - result: dict = {} - for key, value in payload.items(): - # A denied key redacts the whole value, however deeply nested — - # partial redaction inside a secret-named subtree is not worth the risk. - if any(pattern.search(str(key)) for pattern in self.deny_key_patterns): - result[key] = REDACTED - continue - result[key] = self._redact_value(value, entropy_plugins) - return result - - -# --- One-shot convenience wrappers --- - - -def redact_payload( - payload: dict, - extra_redact_keys: list[str] | None = None, - extra_redact_paths: list[str] | None = None, -) -> dict: - """Redact a single payload with a throwaway ruleset. - - Reuse a :class:`Redactor` instead when redacting many values (e.g. every - turn of a transcript) so the ruleset is compiled once. - """ - redactor = Redactor.build( - extra_redact_keys=extra_redact_keys, extra_redact_paths=extra_redact_paths - ) - return redactor.redact_payload(payload) - - -def redact_text(value: str, extra_redact_paths: list[str] | None = None) -> str: - """Redact a single free-text string with a throwaway ruleset. - - The pre-compaction checkpoint lifts transcript excerpts straight into the - graph, so that text must pass the same secret floor as inbox payloads - (issue #997). Reuse a :class:`Redactor` when scrubbing many strings. - """ - return Redactor.build(extra_redact_paths=extra_redact_paths).redact_text(value) diff --git a/src/basic_memory/repository/semantic_vector_index.py b/src/basic_memory/repository/semantic_vector_index.py index a68880344..288e9cfc6 100644 --- a/src/basic_memory/repository/semantic_vector_index.py +++ b/src/basic_memory/repository/semantic_vector_index.py @@ -7,9 +7,6 @@ from typing import Protocol, runtime_checkable -SEMANTIC_VECTOR_INDEX_ENTRY_POINT_GROUP = "basic_memory.semantic_vector_indexes" - - @dataclass(frozen=True, slots=True) class VectorIndexScope: """Stable project storage identity plus the current embedding schema.""" diff --git a/src/basic_memory/repository/semantic_vector_index_factory.py b/src/basic_memory/repository/semantic_vector_index_factory.py index fb967d0f5..d9bd91869 100644 --- a/src/basic_memory/repository/semantic_vector_index_factory.py +++ b/src/basic_memory/repository/semantic_vector_index_factory.py @@ -1,11 +1,10 @@ -"""Composition-root factory for built-in and extension vector indexes.""" +"""Composition-root factory for supported semantic vector indexes.""" from __future__ import annotations import hashlib import re -from importlib.metadata import entry_points -from typing import Protocol +from typing import Literal, assert_never from sqlalchemy.engine import make_url from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker @@ -17,34 +16,21 @@ ) from basic_memory.repository.semantic_errors import ( SemanticDependenciesMissingError, - SemanticVectorIndexExtensionError, ) from basic_memory.repository.semantic_vector_index import ( - SEMANTIC_VECTOR_INDEX_ENTRY_POINT_GROUP, SemanticVectorIndex, VectorIndexScope, ) -class SemanticVectorIndexFactory(Protocol): - """Factory signature exposed to separately installed extension packages.""" - - def __call__( - self, - *, - scope: VectorIndexScope, - app_config: BasicMemoryConfig, - ) -> SemanticVectorIndex: ... - - def resolve_semantic_vector_index_name( app_config: BasicMemoryConfig, database_backend: DatabaseBackend, -) -> str: +) -> Literal["sqlite-vec", "pgvector", "milvus"]: """Resolve the effective index while preserving sqlite-vec for local SQLite.""" if database_backend == DatabaseBackend.SQLITE: return "sqlite-vec" - return app_config.semantic_vector_index.strip().lower() + return app_config.semantic_vector_index def semantic_embedding_identity(provider: EmbeddingProvider) -> str: @@ -101,28 +87,6 @@ def build_vector_index_scope( ) -def _load_extension_factory(name: str) -> SemanticVectorIndexFactory: - matches = list(entry_points(group=SEMANTIC_VECTOR_INDEX_ENTRY_POINT_GROUP, name=name)) - if not matches: - raise SemanticVectorIndexExtensionError( - f"Semantic vector index '{name}' is configured but no extension is installed. " - f"Install a package that provides the '{name}' entry point in " - f"'{SEMANTIC_VECTOR_INDEX_ENTRY_POINT_GROUP}'." - ) - if len(matches) > 1: - providers = ", ".join(sorted(entry_point.value for entry_point in matches)) - raise SemanticVectorIndexExtensionError( - f"Multiple semantic vector index extensions provide '{name}': {providers}." - ) - - loaded = matches[0].load() - if not callable(loaded): - raise SemanticVectorIndexExtensionError( - f"Semantic vector index entry point '{name}' must load a callable factory." - ) - return loaded - - def _create_milvus_index( scope: VectorIndexScope, app_config: BasicMemoryConfig, @@ -150,7 +114,7 @@ def create_semantic_vector_index( database_backend: DatabaseBackend, embedding_provider: EmbeddingProvider, ) -> tuple[str, SemanticVectorIndex]: - """Create the selected first-party adapter or load one external extension.""" + """Create the vector adapter selected by the validated application config.""" name = resolve_semantic_vector_index_name(app_config, database_backend) scope = build_vector_index_scope(app_config, embedding_provider, project_id) @@ -165,14 +129,4 @@ def create_semantic_vector_index( if name == "milvus": return name, _create_milvus_index(scope, app_config) - factory = _load_extension_factory(name) - index = factory(scope=scope, app_config=app_config) - if not isinstance(index, SemanticVectorIndex): - raise SemanticVectorIndexExtensionError( - f"Semantic vector index extension '{name}' returned an incompatible adapter." - ) - if index.scope != scope: - raise SemanticVectorIndexExtensionError( - f"Semantic vector index extension '{name}' returned an adapter for the wrong scope." - ) - return name, index + assert_never(name) diff --git a/tests/cli/test_hook_command.py b/tests/cli/test_hook_command.py index e6e9d5d21..fa5abe7de 100644 --- a/tests/cli/test_hook_command.py +++ b/tests/cli/test_hook_command.py @@ -636,8 +636,8 @@ def test_codex_compact_checkpoint_prompt_survives_unreachable_memory( # --- session-start / pre-compact: envelope capture gate --- -def test_capture_events_true_boolean_writes_envelope(bm_home: Path, claude_project: Path) -> None: - _write_claude_settings(claude_project, {"primaryProject": "demo", "captureEvents": True}) +def test_claude_capture_events_defaults_on(bm_home: Path, claude_project: Path) -> None: + _write_claude_settings(claude_project, {"primaryProject": "demo"}) with patch( "basic_memory.mcp.tools.search_notes", new_callable=AsyncMock, return_value=SEARCH_EMPTY ): @@ -660,13 +660,11 @@ def test_capture_events_true_boolean_writes_envelope(bm_home: Path, claude_proje assert envelope["payload"]["capture_folder"] == "sessions" -def test_codex_capture_events_defaults_on_and_preserves_legacy_redaction( - bm_home: Path, tmp_path: Path -) -> None: +def test_codex_capture_events_defaults_on(bm_home: Path, tmp_path: Path) -> None: project = tmp_path / "codex-proj" (project / ".codex").mkdir(parents=True) (project / ".codex" / "basic-memory.json").write_text( - json.dumps({"basicMemory": {"primaryProject": "demo", "redactKeys": ["trigger"]}}), + json.dumps({"basicMemory": {"primaryProject": "demo"}}), encoding="utf-8", ) with patch( @@ -683,16 +681,35 @@ def test_codex_capture_events_defaults_on_and_preserves_legacy_redaction( assert len(envelopes) == 1 assert envelopes[0]["source"] == "codex" assert envelopes[0]["payload"]["capture_folder"] == "codex" - assert envelopes[0]["payload"]["trigger"] == "[REDACTED]" + assert envelopes[0]["payload"]["trigger"] == "startup" + + +def test_claude_capture_events_can_be_disabled(bm_home: Path, claude_project: Path) -> None: + _write_claude_settings( + claude_project, + {"primaryProject": "demo", "captureEvents": False}, + ) + with patch( + "basic_memory.mcp.tools.search_notes", + new_callable=AsyncMock, + return_value=SEARCH_EMPTY, + ): + result = runner.invoke( + cli_app, + ["hook", "session-start", "--project-dir", str(claude_project)], + input=_payload(claude_project), + ) + + assert result.exit_code == 0 + assert not (bm_home / "inbox").exists() @pytest.mark.parametrize("gate_value", ["true", "false", 1, "yes", {"on": True}]) def test_capture_events_fails_closed_on_non_boolean( bm_home: Path, claude_project: Path, gate_value ) -> None: - # A privacy gate must fail closed: only the JSON boolean true enables - # capture. A hand-edited string like "false" is truthy in Python and must - # never switch recording on. + # Only the JSON boolean true enables capture. A hand-edited string like + # "false" is truthy in Python and must not switch recording on. _write_claude_settings(claude_project, {"primaryProject": "demo", "captureEvents": gate_value}) with patch( "basic_memory.mcp.tools.search_notes", new_callable=AsyncMock, return_value=SEARCH_EMPTY @@ -765,11 +782,9 @@ def test_pre_compact_writes_checkpoint_note( assert "tool noise" not in content -def test_pre_compact_redacts_secrets_in_checkpoint( +def test_pre_compact_preserves_transcript_text( bm_home: Path, claude_project: Path, tmp_path: Path ) -> None: - """Regression: transcript excerpts pass the secret floor before landing in - the checkpoint note or its title (#997).""" lines = [ { "message": {"role": "user", "content": "deploy with AKIAIOSFODNN7EXAMPLE please"}, @@ -789,37 +804,8 @@ def test_pre_compact_redacts_secrets_in_checkpoint( assert result.exit_code == 0 assert mock_write.await_args is not None kwargs = mock_write.await_args.kwargs - assert "AKIAIOSFODNN7EXAMPLE" not in kwargs["content"] - assert "AKIAIOSFODNN7EXAMPLE" not in kwargs["title"] - - -def test_pre_compact_redacts_cwd_under_denied_path( - bm_home: Path, claude_project: Path, tmp_path: Path -) -> None: - """Regression: a session under a configured redactPaths dir must not leak the - raw cwd into the checkpoint frontmatter or body (#997).""" - _write_claude_settings( - claude_project, - {"primaryProject": "demo", "redactPaths": ["/srv/clients/"]}, - ) - transcript = _transcript(tmp_path) - mock_write = AsyncMock(return_value={"action": "created"}) - with patch("basic_memory.mcp.tools.write_note", mock_write): - result = runner.invoke( - cli_app, - ["hook", "pre-compact", "--project-dir", str(claude_project)], - input=_payload( - "/srv/clients/acme/repo", - transcript_path=str(transcript), - trigger="auto", - ), - ) - - assert result.exit_code == 0 - assert mock_write.await_args is not None - kwargs = mock_write.await_args.kwargs - assert "/srv/clients/acme/repo" not in kwargs["content"] # body - assert kwargs["metadata"]["cwd"] == "[REDACTED_PATH]" # frontmatter + assert "AKIAIOSFODNN7EXAMPLE" in kwargs["content"] + assert "AKIAIOSFODNN7EXAMPLE" in kwargs["title"] def test_pre_compact_checkpoint_handles_yaml_special_cwd( @@ -1199,7 +1185,7 @@ def test_status_defaults_when_nothing_configured( assert "last flush: never" in result.stdout assert "primary project: (not set)" in result.stdout assert "checkpoint on compact: off" in result.stdout - assert "capture events: off" in result.stdout + assert "capture events: on" in result.stdout assert "uv: (not found)" in result.stdout @@ -1869,18 +1855,18 @@ def test_claude_settings_precedence_and_local_overrides(tmp_path: Path) -> None: assert merged["recallTimeframe"] == "9d" # user-level survives unless overridden -def test_claude_settings_ignore_malformed_files(tmp_path: Path) -> None: +def test_claude_settings_malformed_file_fails_closed(tmp_path: Path) -> None: project = tmp_path / "proj" (project / ".claude").mkdir(parents=True) (project / ".claude" / "settings.json").write_text("{broken", encoding="utf-8") merged, found = hook_module.load_claude_settings(project) - assert merged == {} - assert found is False + assert merged == {"captureEvents": False} + assert found is True -def test_claude_settings_non_dict_block_is_ignored(tmp_path: Path) -> None: +def test_claude_settings_non_dict_block_fails_closed(tmp_path: Path) -> None: project = tmp_path / "proj" (project / ".claude").mkdir(parents=True) (project / ".claude" / "settings.json").write_text( @@ -1889,8 +1875,32 @@ def test_claude_settings_non_dict_block_is_ignored(tmp_path: Path) -> None: merged, found = hook_module.load_claude_settings(project) - assert merged == {} - assert found is False + assert merged == {"captureEvents": False} + assert found is True + + +def test_claude_settings_malformed_project_invalidates_user_fallback(tmp_path: Path) -> None: + home = Path.home() + (home / ".claude").mkdir(parents=True, exist_ok=True) + (home / ".claude" / "settings.json").write_text( + json.dumps( + { + "basicMemory": { + "primaryProject": "user-wide", + "captureEvents": True, + } + } + ), + encoding="utf-8", + ) + project = tmp_path / "proj" + (project / ".claude").mkdir(parents=True) + (project / ".claude" / "settings.json").write_text("{broken", encoding="utf-8") + + merged, found = hook_module.load_claude_settings(project) + + assert merged == {"captureEvents": False} + assert found is True def test_codex_settings_broken_file_counts_as_configured(tmp_path: Path) -> None: @@ -2020,8 +2030,6 @@ def test_codex_settings_merge_user_then_project_with_checkout_folder(tmp_path: P "primaryProject": "user-wide", "recallTimeframe": "9d", "captureEvents": True, - "redactKeys": ["token", "shared-secret"], - "redactPaths": ["/shared/private"], } } ), @@ -2035,8 +2043,6 @@ def test_codex_settings_merge_user_then_project_with_checkout_folder(tmp_path: P { "basicMemory": { "primaryProject": "project-level", - "redactKeys": ["token", "repo-secret"], - "redactPaths": [], } } ), @@ -2051,8 +2057,6 @@ def test_codex_settings_merge_user_then_project_with_checkout_folder(tmp_path: P assert merged["checkpointOnCompact"] is True assert merged["captureEvents"] is True assert merged["captureFolder"] == "codex/widgets" - assert merged["redactKeys"] == ["token", "shared-secret", "repo-secret"] - assert merged["redactPaths"] == ["/shared/private"] def test_codex_project_settings_override_user_capture_defaults(tmp_path: Path) -> None: @@ -2102,36 +2106,6 @@ def test_codex_project_settings_can_disable_user_checkpoint_setting(tmp_path: Pa assert merged["checkpointOnCompact"] is False -def test_codex_settings_ignore_legacy_checkpoint_privacy_review(tmp_path: Path) -> None: - project = tmp_path / "proj" - (project / ".codex").mkdir(parents=True) - (project / ".codex" / "basic-memory.json").write_text( - json.dumps( - { - "basicMemory": { - "checkpointPrivacyReview": True, - "redactKeys": ["token"], - "redactPaths": ["/private"], - } - } - ), - encoding="utf-8", - ) - - merged, found = hook_module.load_codex_settings(project) - - assert found is True - assert "checkpointPrivacyReview" not in merged - assert merged["redactKeys"] == ["token"] - assert merged["redactPaths"] == ["/private"] - - -def test_string_list_guards_config_types() -> None: - assert hook_module._string_list(None) == [] - assert hook_module._string_list("not-a-list") == [] - assert hook_module._string_list(["ok", 3, "fine"]) == ["ok", "fine"] - - def test_mapping_dir_fallback_order(tmp_path: Path) -> None: explicit = tmp_path / "explicit" assert hook_module._mapping_dir(explicit, "/payload/cwd") == explicit diff --git a/tests/hooks/test_projector.py b/tests/hooks/test_archive.py similarity index 100% rename from tests/hooks/test_projector.py rename to tests/hooks/test_archive.py index 08d2e1930..6d96de524 100644 --- a/tests/hooks/test_projector.py +++ b/tests/hooks/test_archive.py @@ -5,8 +5,8 @@ from unittest.mock import AsyncMock, patch from basic_memory.hooks import inbox -from basic_memory.hooks.envelope import SESSION_STARTED, create_envelope from basic_memory.hooks.archive import flush +from basic_memory.hooks.envelope import SESSION_STARTED, create_envelope from basic_memory.hooks.project_ref import split_project_ref diff --git a/tests/hooks/test_envelope.py b/tests/hooks/test_envelope.py index 2fcf06311..2e552325c 100644 --- a/tests/hooks/test_envelope.py +++ b/tests/hooks/test_envelope.py @@ -67,22 +67,10 @@ def test_create_envelope_defaults_ts_to_now() -> None: assert "T" in envelope.ts -def test_create_envelope_redacts_payload_recursively() -> None: +def test_create_envelope_preserves_bounded_payload() -> None: envelope = _envelope(payload={"nested": {"password": "p" * 30}, "note": "safe"}) - assert envelope.payload["nested"]["password"] == "[REDACTED]" - assert envelope.payload["note"] == "safe" - - -def test_create_envelope_redacts_cwd_under_denied_path() -> None: - # cwd is a user path; a session under a configured redactPaths dir must not - # persist the raw path into the inbox WAL. - envelope = _envelope( - cwd="/srv/clients/acme/repo", - extra_redact_paths=["/srv/clients/"], - ) - - assert envelope.cwd == "[REDACTED_PATH]" + assert envelope.payload == {"nested": {"password": "p" * 30}, "note": "safe"} def test_create_envelope_keeps_ordinary_cwd() -> None: @@ -113,7 +101,6 @@ def test_idempotency_key_differs_across_minutes_and_inputs() -> None: def test_idempotency_key_is_metadata_only() -> None: - # Redaction changes the payload, never the identity. plain = _envelope(payload={"note": "hello"}) secret = _envelope(payload={"password": "x" * 30}) diff --git a/tests/hooks/test_redaction.py b/tests/hooks/test_redaction.py deleted file mode 100644 index fc24aa3cc..000000000 --- a/tests/hooks/test_redaction.py +++ /dev/null @@ -1,388 +0,0 @@ -"""Unit tests for the Stage-1 deterministic redaction floor.""" - -import copy -from pathlib import Path - -from basic_memory.hooks.redaction import ( - MAX_PAYLOAD_VALUE_LEN, - REDACTED, - REDACTED_PATH, - TRUNCATION_MARKER, - DenyPath, - Redactor, - redact_payload, - redact_text, -) - -# --- Redactor value object --- - - -def test_redactor_is_reusable_across_values() -> None: - # The whole point of the value object: compile the ruleset once, apply it to - # many payloads/strings (every turn of a transcript) without rebuilding. - redactor = Redactor.build(extra_redact_paths=["/srv/clients/"]) - - assert redactor.redact_text("/srv/clients/a/b") == REDACTED_PATH - assert redactor.redact_text("ordinary prose") == "ordinary prose" - assert redactor.redact_payload({"cwd": "/srv/clients/x"})["cwd"] == REDACTED_PATH - # Reuse is pure: a later call is unaffected by an earlier one. - assert redactor.redact_text("/srv/clients/a/b") == REDACTED_PATH - - -def test_denypath_compile_skips_bare_root() -> None: - # A bare "/" or empty prefix would redact every path, so it compiles to None - # and is dropped from the ruleset. - assert DenyPath.compile("/", case_insensitive=False) is None - assert DenyPath.compile("", case_insensitive=False) is None - assert DenyPath.compile("/srv/secrets/", case_insensitive=False) is not None - - -# --- Deny-key rules (recursive) --- - - -def test_redacts_nested_dict_secrets_by_key() -> None: - payload = {"config": {"api_key": "sk-" + "a" * 30, "region": "us-east-1"}} - - redacted = redact_payload(payload) - - assert redacted["config"]["api_key"] == REDACTED - assert redacted["config"]["region"] == "us-east-1" - - -def test_redacts_secrets_inside_lists() -> None: - payload = { - "env_dump": ["PATH=/usr/bin", "AWS_SECRET_ACCESS_KEY=" + "s" * 30], - "steps": [{"auth_token": "t" * 30}, {"note": "safe"}], - } - - redacted = redact_payload(payload) - - assert redacted["env_dump"][0] == "PATH=/usr/bin" - assert redacted["env_dump"][1] == REDACTED - assert redacted["steps"][0]["auth_token"] == REDACTED - assert redacted["steps"][1]["note"] == "safe" - - -def test_denied_key_redacts_whole_subtree() -> None: - payload = {"auth": {"user": "alice", "nested": {"deep": "value"}}} - - assert redact_payload(payload)["auth"] == REDACTED - - -def test_benign_key_names_pass_through() -> None: - payload = {"safe_key": "value", "monkey": "value"} - - assert redact_payload(payload) == payload - - -def test_extra_keys_apply_at_depth() -> None: - payload = {"outer": {"internal_id": "abc"}} - - redacted = redact_payload(payload, extra_redact_keys=["internal_id"]) - - assert redacted["outer"]["internal_id"] == REDACTED - - -def test_non_string_scalars_pass_through() -> None: - payload = {"count": 3, "ratio": 0.5, "flag": True, "nothing": None} - - assert redact_payload(payload) == payload - - -def test_tuples_normalize_to_lists() -> None: - assert redact_payload({"steps": ("a", "b")})["steps"] == ["a", "b"] - - -# --- Deny-path rules --- - - -def test_deny_paths_apply_at_depth() -> None: - home_ssh = str(Path("~/.ssh/id_rsa").expanduser()) - payload = {"files": [{"path": home_ssh, "preview": "y" * 600}]} - - redacted = redact_payload(payload) - - entry = redacted["files"][0] - assert entry["path"] == REDACTED_PATH - assert entry["preview"].endswith(TRUNCATION_MARKER) - assert len(entry["preview"]) < 600 - - -def test_deny_paths_redact_embedded_substring_in_value() -> None: - # A payload string value can embed a secret path mid-text; only the path - # token is replaced, the rest of the value is preserved. - home_ssh = str(Path("~/.ssh/id_rsa").expanduser()) - redacted = redact_payload({"note": f"copied {home_ssh} to backup"}) - - assert home_ssh not in redacted["note"] - assert redacted["note"] == f"copied {REDACTED_PATH} to backup" - - -def test_deny_paths_redact_whole_value_path_with_spaces() -> None: - # A whole-value path with spaces (a client/project dir) must redact fully — - # the substring pass's \S* tail would stop at the first space and leak the - # rest. Path-prefix logic on the whole value handles it. - redacted = redact_payload( - {"cwd": "/srv/clients/acme corp/secret.txt"}, extra_redact_paths=["/srv/clients/"] - ) - assert redacted["cwd"] == REDACTED_PATH - - -def test_redact_text_redacts_whole_value_path_with_spaces() -> None: - assert ( - redact_text("/srv/clients/acme corp/repo", extra_redact_paths=["/srv/clients/"]) - == REDACTED_PATH - ) - - -def test_redact_text_preserves_prose_after_denied_path() -> None: - # The embedded-prose descendant stops at whitespace, so a denied path - # followed by prose redacts the path and keeps the following words. - result = redact_text( - "read /srv/clients/foo then continue", - extra_redact_paths=["/srv/clients/"], - ) - assert result == f"read {REDACTED_PATH} then continue" - - -def test_redact_text_truncates_embedded_spaced_path_at_whitespace() -> None: - # A denied path embedded in prose whose directory contains a space is - # truncated at that space: the sensitive root and its leading component are - # redacted, but a spaced tail can survive. This residual is intentional — - # distinguishing a spaced path from "path then prose" is ambiguous, and - # heuristics that consumed across the space either swallowed connecting prose - # or broke on Windows drive letters. Whole-value path values (the real - # capture channel) are redacted in full by the path-prefix check; see - # test_redact_text_redacts_whole_value_path_with_spaces. - result = redact_text( - "please inspect /srv/clients/acme corp/secret.txt now", - extra_redact_paths=["/srv/clients/"], - ) - assert "/srv/clients/acme" not in result - assert result == f"please inspect {REDACTED_PATH} corp/secret.txt now" - - -def test_deny_paths_match_across_windows_separators() -> None: - # Windows payload values carry backslashes while deny paths are usually - # written with forward slashes; both sides normalize before comparison. - payload = {"path": "C:\\Users\\dev\\vault\\key.txt"} - - redacted = redact_payload(payload, extra_redact_paths=["C:/Users/dev/vault/"]) - - assert redacted["path"] == REDACTED_PATH - - -def test_extra_deny_paths_accept_backslash_prefixes() -> None: - payload = {"path": "C:/Users/dev/vault/key.txt"} - - redacted = redact_payload(payload, extra_redact_paths=["C:\\Users\\dev\\vault\\"]) - - assert redacted["path"] == REDACTED_PATH - - -# --- Env-pair and truncation rules --- - - -def test_env_style_pairs_redact_wholesale() -> None: - assert redact_payload({"line": "MY_TOKEN_VALUE=" + "v" * 25})["line"] == REDACTED - - -def test_truncation_caps_long_values() -> None: - redacted = redact_payload({"long": "z" * (MAX_PAYLOAD_VALUE_LEN + 100)}) - - assert redacted["long"] == "z" * MAX_PAYLOAD_VALUE_LEN + TRUNCATION_MARKER - - -# --- detect-secrets hits --- - - -def test_detect_secrets_redacts_aws_key_in_prose() -> None: - payload = {"opening": "use key AKIAIOSFODNN7EXAMPLE for the deploy"} - - redacted = redact_payload(payload) - - assert "AKIAIOSFODNN7EXAMPLE" not in redacted["opening"] - assert REDACTED in redacted["opening"] - assert "for the deploy" in redacted["opening"] - - -def test_detect_secrets_redacts_github_token() -> None: - token = "ghp_abcdefghijklmnopqrstuvwxyz0123456789" - redacted = redact_payload({"opening": f"push with {token} now"}) - - assert token not in redacted["opening"] - - -def test_detect_secrets_redacts_keyword_assignments() -> None: - redacted = redact_payload({"opening": "password = 'hunter2-super-secret-value'"}) - - assert "hunter2-super-secret-value" not in redacted["opening"] - - -def test_detect_secrets_redacts_private_key_blocks_multiline() -> None: - value = "context\n-----BEGIN RSA PRIVATE KEY-----\nplain trailing line" - redacted = redact_payload({"dump": value}) - - assert "BEGIN RSA PRIVATE KEY" not in redacted["dump"] - assert "plain trailing line" in redacted["dump"] - - -def test_detect_secrets_redacts_high_entropy_strings() -> None: - opaque = "Zm9vYmFyYmF6cXV4cXV1eDEyMzQ1Njc4OTBhYmNkZWY=" - redacted = redact_payload({"opening": f"blob {opaque} end"}) - - assert opaque not in redacted["opening"] - assert "end" in redacted["opening"] - - -def test_ordinary_prose_survives_entropy_scan() -> None: - prose = "totally benign sentence about coffee brewing methods" - - assert redact_payload({"opening": prose})["opening"] == prose - - -# --- Purity and idempotence --- - - -def test_redaction_is_pure() -> None: - payload = {"config": {"api_key": "sk-" + "a" * 30}, "items": ["MY_TOKEN=" + "v" * 25]} - snapshot = copy.deepcopy(payload) - - redact_payload(payload) - - assert payload == snapshot - - -def test_redaction_is_idempotent() -> None: - payload = { - "config": {"api_key": "sk-" + "a" * 30}, - "opening": "use key AKIAIOSFODNN7EXAMPLE plus password = 'hunter2-super-secret-value'", - "long": "z" * (MAX_PAYLOAD_VALUE_LEN + 100), - "env": "MY_TOKEN_VALUE=" + "v" * 25, - "nested": [{"path": "C:\\Users\\dev\\vault\\key.txt"}], - } - - once = redact_payload(payload, extra_redact_paths=["C:/Users/dev/vault/"]) - twice = redact_payload(once, extra_redact_paths=["C:/Users/dev/vault/"]) - - assert once == twice - - -# --- redact_text: single free-text string (checkpoint excerpts, #997) --- - - -def test_redact_text_scrubs_secret_embedded_in_prose() -> None: - scrubbed = redact_text("deploy with AKIAIOSFODNN7EXAMPLE now") - - assert "AKIAIOSFODNN7EXAMPLE" not in scrubbed - assert "deploy with" in scrubbed - - -def test_redact_text_leaves_ordinary_prose_intact() -> None: - assert redact_text("Fix the login bug in the auth handler") == ( - "Fix the login bug in the auth handler" - ) - - -def test_redact_text_redacts_denied_path() -> None: - home_ssh = str(Path("~/.ssh/id_rsa").expanduser()) - assert redact_text(home_ssh) == REDACTED_PATH - - -def test_redact_text_honors_extra_deny_paths() -> None: - assert redact_text("/srv/secrets/prod.env", extra_redact_paths=["/srv/secrets/"]) == ( - REDACTED_PATH - ) - - -def test_redact_text_redacts_exact_denied_directory_root() -> None: - # The denied directory itself (no trailing separator, no child) must redact, - # not only its descendants. - assert redact_text("/srv/clients", extra_redact_paths=["/srv/clients/"]) == REDACTED_PATH - home_ssh = str(Path("~/.ssh").expanduser()) - assert redact_text(home_ssh) == REDACTED_PATH - - -def test_redact_text_leaves_sibling_of_denied_directory_intact() -> None: - # A sibling sharing the prefix chars must not match (bounded root). - assert redact_text("/srv/clientsbackup", extra_redact_paths=["/srv/clients/"]) == ( - "/srv/clientsbackup" - ) - - -def test_redact_text_ignores_bare_root_deny_path() -> None: - # A "/" (or empty) deny path would otherwise redact every path; it's skipped. - assert redact_text("/opt/app/main.py", extra_redact_paths=["/"]) == "/opt/app/main.py" - - -def test_deny_paths_match_case_insensitively_on_windows(monkeypatch) -> None: - # Windows paths are case-insensitive: a different drive/user casing than the - # configured deny path is the same directory and must still redact. - monkeypatch.setattr("basic_memory.hooks.redaction.os.name", "nt") - result = redact_text( - "open C:/Users/ALICE/vault/key.txt", extra_redact_paths=["c:/users/alice/vault"] - ) - assert result == f"open {REDACTED_PATH}" - - -def test_deny_paths_stay_case_sensitive_on_posix(monkeypatch) -> None: - # POSIX: different casing is a different directory, so it must NOT redact. - # Pin os.name so the assertion holds when the suite runs on a Windows host - # (where deny paths are matched case-insensitively) — the companion - # test_deny_paths_match_case_insensitively_on_windows pins "nt" the same way. - monkeypatch.setattr("basic_memory.hooks.redaction.os.name", "posix") - result = redact_text("open /home/ALICE/vault/key.txt", extra_redact_paths=["/home/alice/vault"]) - assert REDACTED_PATH not in result - - -def test_redact_text_expands_user_tilde_deny_path() -> None: - # A user-configured redactPaths entry in ~/ form must match the absolute cwd - # the hook actually captures (expanded like the built-in defaults). - absolute = str(Path("~/clients/secret/repo").expanduser()) - scrubbed = redact_text(f"working in {absolute}", extra_redact_paths=["~/clients/secret"]) - - assert absolute not in scrubbed - assert scrubbed == f"working in {REDACTED_PATH}" - - -def test_redact_text_redacts_denied_root_before_punctuation() -> None: - # Prose puts punctuation right after a path; the root must still redact. - home_ssh = str(Path("~/.ssh").expanduser()) - assert redact_text(f"key at {home_ssh}, done") == f"key at {REDACTED_PATH}, done" - assert redact_text("/srv/clients.", extra_redact_paths=["/srv/clients/"]) == f"{REDACTED_PATH}." - - -def test_redact_text_redacts_denied_path_embedded_in_prose() -> None: - # A checkpoint excerpt may reference a secret path mid-sentence; the whole - # path token is replaced in place while the surrounding prose survives. - home_ssh = str(Path("~/.ssh/id_rsa").expanduser()) - scrubbed = redact_text(f"please read {home_ssh} then continue") - - assert home_ssh not in scrubbed - assert scrubbed == f"please read {REDACTED_PATH} then continue" - - -def test_redact_text_redacts_multiple_embedded_paths() -> None: - ssh = str(Path("~/.ssh/id_rsa").expanduser()) - aws = str(Path("~/.aws/credentials").expanduser()) - scrubbed = redact_text(f"compare {ssh} and {aws} carefully") - - assert ssh not in scrubbed - assert aws not in scrubbed - assert scrubbed == f"compare {REDACTED_PATH} and {REDACTED_PATH} carefully" - - -def test_redact_text_redacts_unexpanded_tilde_home_path() -> None: - # Prose commonly names the shell form (~/.ssh/id_rsa) rather than the - # expanded absolute path; the literal ~/ prefix must be denied too. - scrubbed = redact_text("please read ~/.ssh/id_rsa then continue") - - assert "~/.ssh/id_rsa" not in scrubbed - assert scrubbed == f"please read {REDACTED_PATH} then continue" - - -def test_redact_payload_redacts_unexpanded_tilde_home_path() -> None: - redacted = redact_payload({"note": "key at ~/.aws/credentials please"}) - - assert "~/.aws/credentials" not in redacted["note"] - assert redacted["note"] == f"key at {REDACTED_PATH} please" diff --git a/tests/repository/test_semantic_vector_index.py b/tests/repository/test_semantic_vector_index.py index 4b95c62fa..96ec7b4f8 100644 --- a/tests/repository/test_semantic_vector_index.py +++ b/tests/repository/test_semantic_vector_index.py @@ -1,14 +1,14 @@ -"""Contract and discovery tests for pluggable semantic vector indexes.""" +"""Contract and composition tests for semantic vector indexes.""" from __future__ import annotations import builtins from collections.abc import Sequence -from dataclasses import dataclass from typing import Any from unittest.mock import MagicMock import pytest +from pydantic import ValidationError from basic_memory.config import BasicMemoryConfig, DatabaseBackend from basic_memory.repository.embedding_provider import EmbeddingProvider @@ -16,10 +16,8 @@ from basic_memory.repository.search_repository import create_search_repository from basic_memory.repository.semantic_errors import ( SemanticDependenciesMissingError, - SemanticVectorIndexExtensionError, ) from basic_memory.repository.semantic_vector_index import ( - SEMANTIC_VECTOR_INDEX_ENTRY_POINT_GROUP, SemanticVectorIndex, VectorDeletion, VectorIndexScope, @@ -79,22 +77,13 @@ async def search( return [] -@dataclass(frozen=True) -class StubEntryPoint: - value: str - loaded: object - - def load(self) -> object: - return self.loaded - - def _postgres_config(**overrides: object) -> BasicMemoryConfig: values: dict[str, object] = { "env": "test", "database_backend": DatabaseBackend.POSTGRES, "database_url": "postgresql+asyncpg://user:secret@db.example.test:5432/memory", "semantic_search_enabled": True, - "semantic_vector_index": "test-extension", + "semantic_vector_index": "pgvector", } values.update(overrides) return BasicMemoryConfig(**values) @@ -125,15 +114,17 @@ def test_vector_contract_values_and_dimension_validation() -> None: def test_selector_defaults_to_pgvector_and_sqlite_remains_automatic() -> None: default_config = BasicMemoryConfig(env="test") - extension_config = _postgres_config() + postgres_config = _postgres_config() assert default_config.semantic_vector_index == "pgvector" assert ( resolve_semantic_vector_index_name(default_config, DatabaseBackend.POSTGRES) == "pgvector" ) assert ( - resolve_semantic_vector_index_name(extension_config, DatabaseBackend.SQLITE) == "sqlite-vec" + resolve_semantic_vector_index_name(postgres_config, DatabaseBackend.SQLITE) == "sqlite-vec" ) + with pytest.raises(ValidationError, match="semantic_vector_index"): + _postgres_config(semantic_vector_index="test-extension") def test_scope_is_stable_credential_free_and_project_isolated() -> None: @@ -196,25 +187,6 @@ def test_scope_is_stable_credential_free_and_project_isolated() -> None: assert first.storage_key == rotated_password.storage_key -def test_missing_configured_extension_fails_without_fallback(monkeypatch) -> None: - monkeypatch.setattr( - "basic_memory.repository.semantic_vector_index_factory.entry_points", - lambda **_kwargs: (), - ) - - with pytest.raises( - SemanticVectorIndexExtensionError, - match="configured but no extension is installed", - ): - create_semantic_vector_index( - session_maker=MagicMock(), - project_id=7, - app_config=_postgres_config(), - database_backend=DatabaseBackend.POSTGRES, - embedding_provider=StubEmbeddingProvider(), - ) - - def test_milvus_without_optional_dependencies_reports_install_extra(monkeypatch) -> None: config = _postgres_config( semantic_vector_index="milvus", @@ -248,110 +220,6 @@ def import_without_pymilvus( ) -def test_extension_factory_receives_explicit_scope_and_config(monkeypatch) -> None: - captured: dict[str, object] = {} - - def factory(*, scope: VectorIndexScope, app_config: BasicMemoryConfig) -> StubVectorIndex: - captured.update(scope=scope, app_config=app_config) - return StubVectorIndex(scope) - - monkeypatch.setattr( - "basic_memory.repository.semantic_vector_index_factory.entry_points", - lambda **kwargs: ( - ( - StubEntryPoint( - value="test_extension:create_index", - loaded=factory, - ), - ) - if kwargs - == { - "group": SEMANTIC_VECTOR_INDEX_ENTRY_POINT_GROUP, - "name": "test-extension", - } - else () - ), - ) - config = _postgres_config() - - name, index = create_semantic_vector_index( - session_maker=MagicMock(), - project_id=7, - app_config=config, - database_backend=DatabaseBackend.POSTGRES, - embedding_provider=StubEmbeddingProvider(), - ) - - assert name == "test-extension" - assert isinstance(index, StubVectorIndex) - assert captured["app_config"] is config - assert captured["scope"] == index.scope - - -@pytest.mark.parametrize( - ("entry_points", "message"), - [ - ( - ( - StubEntryPoint("first:create", lambda **_kwargs: None), - StubEntryPoint("second:create", lambda **_kwargs: None), - ), - "Multiple semantic vector index extensions", - ), - ((StubEntryPoint("invalid:value", object()),), "must load a callable factory"), - ( - (StubEntryPoint("incompatible:create", lambda **_kwargs: object()),), - "returned an incompatible adapter", - ), - ], -) -def test_invalid_extension_registration_fails_explicitly( - monkeypatch, - entry_points: tuple[StubEntryPoint, ...], - message: str, -) -> None: - monkeypatch.setattr( - "basic_memory.repository.semantic_vector_index_factory.entry_points", - lambda **_kwargs: entry_points, - ) - - with pytest.raises(SemanticVectorIndexExtensionError, match=message): - create_semantic_vector_index( - session_maker=MagicMock(), - project_id=7, - app_config=_postgres_config(), - database_backend=DatabaseBackend.POSTGRES, - embedding_provider=StubEmbeddingProvider(), - ) - - -def test_extension_cannot_replace_the_required_scope(monkeypatch) -> None: - wrong_scope = VectorIndexScope( - namespace="other-installation", - project_id=999, - embedding_identity="other-model", - dimensions=3, - ) - monkeypatch.setattr( - "basic_memory.repository.semantic_vector_index_factory.entry_points", - lambda **_kwargs: ( - StubEntryPoint( - "wrong-scope:create", - lambda **_factory_kwargs: StubVectorIndex(wrong_scope), - ), - ), - ) - - with pytest.raises(SemanticVectorIndexExtensionError, match="wrong scope"): - create_semantic_vector_index( - session_maker=MagicMock(), - project_id=7, - app_config=_postgres_config(), - database_backend=DatabaseBackend.POSTGRES, - embedding_provider=StubEmbeddingProvider(), - ) - - def test_search_repository_composition_root_injects_selected_adapter(monkeypatch) -> None: provider = StubEmbeddingProvider() scope = build_vector_index_scope(_postgres_config(), provider, project_id=7) diff --git a/uv.lock b/uv.lock index 266a4f85a..e90379b0f 100644 --- a/uv.lock +++ b/uv.lock @@ -288,7 +288,6 @@ dependencies = [ { name = "anyio" }, { name = "asyncpg" }, { name = "dateparser" }, - { name = "detect-secrets" }, { name = "fastapi", extra = ["standard"] }, { name = "fastembed" }, { name = "fastmcp" }, @@ -365,7 +364,6 @@ requires-dist = [ { name = "anyio", specifier = ">=4.10.0" }, { name = "asyncpg", specifier = ">=0.30.0" }, { name = "dateparser", specifier = ">=1.2.0" }, - { name = "detect-secrets", specifier = ">=1.5" }, { name = "fastapi", extras = ["standard"], specifier = ">=0.136.1" }, { name = "fastembed", specifier = ">=0.7.4" }, { name = "fastmcp", specifier = ">=3.3.1,<4" }, @@ -830,19 +828,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cc/34/8cc73273414405086c58852916e4031812a6a30fe04c057e37ad99397b7f/detect_installer-0.1.0-py3-none-any.whl", hash = "sha256:034fb20fd665c36e6ba52b8821525ea07fb4f7f938cac459df889fb33801528a", size = 4539, upload-time = "2026-02-23T10:40:23.807Z" }, ] -[[package]] -name = "detect-secrets" -version = "1.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyyaml" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/69/67/382a863fff94eae5a0cf05542179169a1c49a4c8784a9480621e2066ca7d/detect_secrets-1.5.0.tar.gz", hash = "sha256:6bb46dcc553c10df51475641bb30fd69d25645cc12339e46c824c1e0c388898a", size = 97351, upload-time = "2024-05-06T17:46:19.721Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4e/5e/4f5fe4b89fde1dc3ed0eb51bd4ce4c0bca406246673d370ea2ad0c58d747/detect_secrets-1.5.0-py3-none-any.whl", hash = "sha256:e24e7b9b5a35048c313e983f76c4bd09dad89f045ff059e354f9943bf45aa060", size = 120341, upload-time = "2024-05-06T17:46:16.628Z" }, -] - [[package]] name = "distro" version = "1.9.0"