diff --git a/docs/research/git-repository-root-binding.md b/docs/research/git-repository-root-binding.md new file mode 100644 index 0000000000..32b81cf717 --- /dev/null +++ b/docs/research/git-repository-root-binding.md @@ -0,0 +1,126 @@ +# Repository-local Git reads for research publication + +The shared source and publication tools now remove inherited Git repository, +index, object-directory and command-line configuration overrides from their +**Git subprocesses**. Git discovers the repository from the supplied working +directory, including an ordinary linked worktree's `.git` file. Existing +stored-object checks still disable replacement refs. Parent-process and +package-verifier environments are unchanged. + +## Reproduced failure + +At canonical base `81e9bd69e4a5b00ba9c713a1620e36d576d5522e`, two real +repositories suffice: + +1. The intended repository has one checker committed at HEAD. Modify its + working bytes without committing them. +2. A second repository has those modified bytes committed. Set inherited + `GIT_DIR` to its Git directory (also reproduced with `GIT_WORK_TREE`). +3. Call the publication checker with the **intended** repository as `root`. + +Without the environment override, the checker rejects the dirty bytes. With +it, Git resolves the foreign HEAD while Python reads the intended working +files. Their equality produces an incorrect publication success. Export also +resolves and exports the foreign HEAD. This is observable repository +misselection, not a hypothetical hostile verifier or a compressed-byte change. + +The regression additionally constructs foreign live source pins in the +intended checkout. Those pins previously validated against the foreign object +store. After the fix they cannot pass as sources of the intended repository. +The original eight-method suite failed with five assertion failures and four +errors at the unchanged base; all eight pass after the repair. Subtests account +for multiple failures within a method. + +## Policy and callers + +`research_package.git_source_env()` returns a copy, removing: + +- `GIT_DIR`, `GIT_WORK_TREE`, `GIT_COMMON_DIR`, `GIT_INDEX_FILE`; +- `GIT_OBJECT_DIRECTORY`, `GIT_ALTERNATE_OBJECT_DIRECTORIES`, `GIT_NAMESPACE`, + `GIT_CEILING_DIRECTORIES`; +- `GIT_CONFIG_PARAMETERS`, `GIT_CONFIG_COUNT`, and the + `GIT_CONFIG_KEY_` / `GIT_CONFIG_VALUE_` families. + +It retains the existing `GIT_NO_REPLACE_OBJECTS=1` policy. It does not rewrite +repository configuration, objects, refs, the caller's environment, or the +separate environment used to launch a package's verifier. Normal repository +configuration and on-disk linked-worktree/alternate-object topology remain +Git's responsibility. This is not a sandbox for an untrusted Git executable, +repository configuration, or concurrent repository modification. + +The shared helper reaches: + +| Consumer | Covered operations | +| --- | --- | +| Package scaffolding and catalogue lookup | Base resolution, pinned catalogue and input reads | +| Metadata validation | HEAD spelling, exact pinned blobs and working-byte comparison | +| Replay planning | HEAD/index inventories, sparse manifest reads and changed paths | +| Publication preflight | Commit resolution, file inventory, committed bytes | +| Package export | Committed manifest, inventory and payload objects | +| Source-history preflight | Committed source metadata and ancestry; its existing graft/shallow guards remain | +| Overlap local coverage | HEAD and note-tree inventory; its no-lazy-fetch setting remains | + +The legacy line reader and the NUL reader/fallback use the same policy. +`find_overlap.py` imports the helper from its adjacent `research_package.py`, +including direct invocation under `-I`. Standalone copies of that tool need +both files; the helper has standard-library dependencies only. Existing CLI +fixtures now copy that closure. + +Package-local workflows that intentionally use an external Git object store +inside their verifier remain unchanged. The shared publication/source APIs +use the repository provided as their root; inherited shell selection is no +longer an alternate way to override that argument. + +## Validation + +Environment: macOS/Darwin 27.0 arm64, CPython 3.12.7, Git 2.44.0. The eight new +real-Git methods cover dirty-publication rejection, foreign-pin rejection, +exported payload/commit identity, source history, all blob readers, scaffolding, +HEAD inventory, planning, overlap coverage, a real linked worktree, other +inherited selection channels, and unchanged parent/verifier environments. +They observe results before disposable fixture cleanup. + +```sh +python3.12 -W error -B -m unittest discover -s tests \ + -p 'test_research_git_root_binding.py' -v +python3.12 -O -W error -B -m unittest discover -s tests \ + -p 'test_research_git_root_binding.py' -v +``` + +Targeted compatibility passes 143 methods in each mode: `test_research_git*`, +`test_research_source_history.py`, `test_research_commit_bytes.py`, +`test_research_export*`, `test_research_overlap*`, +`test_research_sparse_planning.py` and `test_research_plan_explanations.py`. +The normal run was split into 132 plus 11 methods; the optimized run was one +143-method run. No platform skips occurred on this host. Existing replacement +ref, binary/newline object, source-history, export, sparse-index and standalone +CLI checks remain intact. No scientific package replay or full archive suite +was run. Native Windows execution remains untested for this change. + +The scoped committed source closure can run the new regression and standalone +overlap tests from a directory without `.git`; their temporary real-Git +repositories still require a Git executable. This is a Git-free **source +checkout**, not a claim of Git-absent tests. The registered research-package +exporter is not applicable to this shared-tooling change, which has no +`research//VERIFY.json`; use the exact committed files and record their +hashes. The final handoff records the actual archive and cold-run evidence. + +## Open next questions + +Native Windows can run the same real-Git controls without a POSIX-only fixture; +record that platform separately. General isolation of user/system Git config, +untrusted executables, concurrent writers, and package-specific external-Git +contracts is outside this repair. No further construction or mathematical +replay follows from a publication transport fix. + +## Credit + +Prepared by JoyboyBrian with substantial OpenAI Codex assistance; Apache-2.0. +The environment policy is adapted from the existing package-local +`research/t74-2530-source-partner-audit/audit_common.py` (introduced at +`feb727ba471b5dc3f057f867804aa70ae58b2e71`); that prior isolation technique is +not new here. Retains Douglas Colkitt's shared runner/scaffold, Tekalign's +exporter, Nick Petalas's PR3304 source-history checks, and roger-sato's +stored-object policy at `1a885a3546cd68995be448e2ca91f28cc23ec155`. +The new contribution is the shared explicit-root failure reproduction, +consistent caller repair and compatibility evidence. diff --git a/scripts/check_research_commit.py b/scripts/check_research_commit.py index 08a956131b..7ab20e1487 100644 --- a/scripts/check_research_commit.py +++ b/scripts/check_research_commit.py @@ -17,6 +17,7 @@ canonical_relative_path, git, git_blobs, + git_source_env, relative_file, require, source_pin, @@ -29,7 +30,7 @@ def history_git(root, *args): - env = os.environ.copy() + env = git_source_env() env['GIT_GRAFT_FILE'] = os.devnull env.pop('GIT_SHALLOW_FILE', None) return subprocess.run(['git', '--no-replace-objects', *args], cwd=root, env=env, diff --git a/scripts/find_overlap.py b/scripts/find_overlap.py index c49a9a77a2..c8155a90b0 100644 --- a/scripts/find_overlap.py +++ b/scripts/find_overlap.py @@ -18,6 +18,13 @@ import tempfile import time +# Retain direct script execution under -I; the shared helper uses stdlib only. +sys.path.insert(0, str(Path(__file__).resolve().parent)) +try: + from research_package import git_source_env +finally: + sys.path.pop(0) + ROOT = Path(__file__).resolve().parents[1] REPO = "CrocSwap/integer-mult-bounds" @@ -332,7 +339,7 @@ def git(*args): return subprocess.run( ["git", "--no-replace-objects", *args], cwd=root, check=True, capture_output=True, encoding="utf-8", errors="surrogateescape", timeout=15, - env={**os.environ, "GIT_NO_LAZY_FETCH": "1"}, + env={**git_source_env(), "GIT_NO_LAZY_FETCH": "1"}, ).stdout # Pin the tree once; only metadata is read, never omitted note blobs. diff --git a/scripts/research_package.py b/scripts/research_package.py index 4b72915c91..2c70e7f033 100644 --- a/scripts/research_package.py +++ b/scripts/research_package.py @@ -66,11 +66,30 @@ def relative_file(root, name): return result +# Adapted from the package-local policy in +# research/t74-2530-source-partner-audit/audit_common.py (Apache-2.0). +# Keep repository configuration and linked-worktree .git discovery intact; +# inherited shell overrides must not choose a different repository or index. +_GIT_REDIRECT_VARS = frozenset(( + "GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY", + "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_NAMESPACE", "GIT_CEILING_DIRECTORIES", + "GIT_COMMON_DIR", "GIT_CONFIG_PARAMETERS", "GIT_CONFIG_COUNT", +)) +_GIT_CONFIG_PREFIXES = ("GIT_CONFIG_KEY_", "GIT_CONFIG_VALUE_") + + def git_source_env(): - # Replacement refs can pair an original object ID with different bytes. - # Pins and HEAD inventories must describe the stored objects even when the - # caller has configured a replacement namespace or an inherited override. - return {**os.environ, "GIT_NO_REPLACE_OBJECTS": "1"} + """Use cwd's repository and stored objects, without changing the parent. + + Git can ignore cwd when inherited selection variables point elsewhere. + Config-injection variables can also redirect worktree/index operations. + This policy is only for Git metadata/source subprocesses, not verifiers. + """ + env = {name: value for name, value in os.environ.items() + if name not in _GIT_REDIRECT_VARS and not name.startswith(_GIT_CONFIG_PREFIXES)} + # Replacement refs can report an original ID while returning other bytes. + env["GIT_NO_REPLACE_OBJECTS"] = "1" + return env def git(root, *args): diff --git a/tests/test_research_git_root_binding.py b/tests/test_research_git_root_binding.py new file mode 100644 index 0000000000..2366ac7e7a --- /dev/null +++ b/tests/test_research_git_root_binding.py @@ -0,0 +1,180 @@ +"""Real repository boundaries for shared publication and source readers. + +JoyboyBrian, with substantial OpenAI Codex assistance. Apache-2.0. +Credit the package-local Git environment policy in the PR2530 source-partner + audit and the shared stored-object, history and export regression authors. +""" +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tarfile +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'scripts')) +import check_research_commit as publication +import export_research_package as exporter +import find_overlap as overlap +import new_research_package as scaffold +import research_package as package +sys.path.pop(0) + + +class GitRootBindingTests(unittest.TestCase): + package = 'research/fixture' + + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='git-root-binding-') + self.addCleanup(temporary.cleanup) + self.parent = Path(temporary.name) + # Fixture setup must not itself be redirected by the donor's shell. + self.env = {k: v for k, v in os.environ.items() if not k.startswith('GIT_')} + self.env.update(GIT_AUTHOR_NAME='JoyboyBrian', + GIT_AUTHOR_EMAIL='122925040+JoyboyBrian@users.noreply.github.com', + GIT_COMMITTER_NAME='JoyboyBrian', + GIT_COMMITTER_EMAIL='122925040+JoyboyBrian@users.noreply.github.com') + self.environment = patch.dict(os.environ, self.env, clear=True) + self.environment.start() + self.addCleanup(self.environment.stop) + self.root = self.make_repo('intended', b'intended source\n') + self.foreign = self.make_repo('foreign', b'foreign source\n') + self.head = self.git(self.root, 'rev-parse', 'HEAD').decode().strip() + self.foreign_head = self.git(self.foreign, 'rev-parse', 'HEAD').decode().strip() + self.redirect = {'GIT_DIR': str(self.foreign / '.git'), + 'GIT_WORK_TREE': str(self.foreign)} + + def git(self, root, *args): + return subprocess.run(['git', *args], cwd=root, env=self.env, + capture_output=True, check=True, timeout=15).stdout + + def commit(self, root): + self.git(root, 'add', '.') + self.git(root, '-c', 'core.hooksPath=/dev/null', '-c', 'commit.gpgsign=false', + '-c', 'maintenance.auto=false', '-c', 'gc.auto=0', 'commit', '-qm', 'fixture') + + def make_repo(self, name, payload): + root = self.parent / name + root.mkdir() + self.git(root, 'init', '-q') + self.git(root, 'config', 'core.autocrlf', 'false') + files = {'input.txt': payload, 'certificates/selected-result.json': b'{}', + 'docs/research/agent-task-catalog.json': json.dumps({ + 'priority_revision': 7, 'tracks': [{'tasks': [ + {'id': 'fixture', 'question': 'Bind the requested repository'}]}]}).encode()} + for path, raw in files.items(): + target = root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(raw) + self.commit(root) + target = scaffold.create(root, 'fixture', 'fixture', 'Fixture', ['input.txt']) + task = json.loads((target / 'TASK.json').read_bytes()) + task.update(bottleneck_link='Source binding', stop_rule='Bounded fixture', + outcome='completed', evidence='Real repository boundary fixture', + ai_assistance='OpenAI Codex', open_questions=[], + no_open_questions_reason='Only publication transport is checked') + (target / 'TASK.json').write_text(json.dumps(task)) + manifest = json.loads((target / 'VERIFY.json').read_bytes()) + manifest['status'] = 'ready' + manifest['commands'][0]['checks'] = 'Source binding fixture' + (target / 'VERIFY.json').write_text(json.dumps(manifest)) + self.commit(root) + return root + + def test_dirty_publication_cannot_pass_against_foreign_head(self): + # The intended dirty tree exactly matches the foreign publication. This + # formerly passed, instead of comparing with the intended HEAD. + shutil.copytree(self.foreign / self.package, self.root / self.package, dirs_exist_ok=True) + shutil.copyfile(self.foreign / 'input.txt', self.root / 'input.txt') + with self.assertRaisesRegex(ValueError, 'working/commit bytes differ'): + publication.check(self.root, self.package) + for redirect in ({'GIT_DIR': self.redirect['GIT_DIR']}, self.redirect): + with self.subTest(redirect=redirect), patch.dict(os.environ, redirect): + with self.assertRaisesRegex(ValueError, 'working/commit bytes differ'): + publication.check(self.root, self.package) + + def test_export_and_history_stay_in_requested_repository(self): + with patch.dict(os.environ, self.redirect): + result = publication.check(self.root, self.package, check_source_history=True) + self.assertEqual(result['commit'], self.head) + receipt = exporter.export(self.root, self.package, 'HEAD', self.parent / 'bundle.tar') + self.assertEqual(receipt['commit'], self.head) + with tarfile.open(self.parent / 'bundle.tar') as archive: + self.assertEqual(archive.extractfile('input.txt').read(), b'intended source\n') + self.assertEqual(exporter.check_bundle(self.parent / 'bundle.tar'), receipt) + self.assertEqual(publication.history_git(self.root, 'rev-parse', 'HEAD').stdout.strip(), + self.head.encode()) + + def test_foreign_live_pins_do_not_validate(self): + shutil.copytree(self.foreign / self.package, self.root / self.package, dirs_exist_ok=True) + shutil.copyfile(self.foreign / 'input.txt', self.root / 'input.txt') + with patch.dict(os.environ, self.redirect): + with self.assertRaises((ValueError, subprocess.CalledProcessError)): + package.validate(self.root, self.package) + + def test_every_reader_and_scaffold_uses_local_objects(self): + name = 'input.txt' + oid = self.git(self.root, 'rev-parse', 'HEAD:' + name).decode().strip() + expected = [(oid, b'intended source\n')] + with patch.dict(os.environ, self.redirect): + self.assertEqual(list(package.git_source_rows(self.root, 'HEAD', [name])), expected) + self.assertEqual(list(package.git_blobs(self.root, ['HEAD:' + name])), expected) + self.assertEqual(list(package._git_blob_batch(self.root, ['HEAD:' + name], False)), expected) + self.assertEqual(package.git(self.root, 'show', 'HEAD:' + name), expected[0][1]) + package.validate(self.root, self.package) + created = scaffold.create(self.root, 'second', 'fixture', 'Fixture', [name]) + self.assertEqual(json.loads((created / 'SOURCES.json').read_bytes())['base_commit'], self.head) + + def test_head_inventory_planning_and_overlap_cannot_use_foreign_notes(self): + path = self.root / 'research/local-only/README.md' + path.parent.mkdir() + path.write_text('local note') + self.commit(self.root) + head = self.git(self.root, 'rev-parse', 'HEAD').decode().strip() + with patch.dict(os.environ, self.redirect): + self.assertIn('research/local-only/README.md', package.tracked_input_paths(self.root)) + self.assertIn(self.package, package.plan(self.root, ['input.txt'])) + coverage, warnings = overlap.local_coverage(self.root, overlap.local_records(self.root)) + self.assertEqual(coverage['head'], head) + self.assertTrue(coverage['complete_head_inventory']) + self.assertEqual(warnings, []) + + def test_redirecting_environment_does_not_break_linked_worktree(self): + linked = self.parent / 'linked' + self.git(self.root, 'worktree', 'add', '--detach', str(linked), self.head) + self.assertTrue((linked / '.git').is_file()) + with patch.dict(os.environ, self.redirect): + self.assertEqual(publication.check(linked, self.package, check_source_history=True)['commit'], self.head) + package.validate(linked, self.package) + self.assertEqual(package.git(linked, 'show', 'HEAD:input.txt'), b'intended source\n') + + def test_other_repository_selection_channels_are_not_inherited(self): + variants = [ + {'GIT_COMMON_DIR': str(self.foreign / '.git')}, + {'GIT_OBJECT_DIRECTORY': str(self.foreign / '.git/objects')}, + {'GIT_INDEX_FILE': str(self.parent / 'absent-index')}, + {'GIT_CONFIG_COUNT': '1', 'GIT_CONFIG_KEY_0': 'core.bare', 'GIT_CONFIG_VALUE_0': 'true'}, + {'GIT_CONFIG_PARAMETERS': "'core.bare'='true'"}, + ] + for env in variants: + with self.subTest(env=env), patch.dict(os.environ, env): + self.assertEqual(publication.check(self.root, self.package, check_source_history=True)['commit'], self.head) + + def test_parent_and_verifier_environments_are_unchanged(self): + with patch.dict(os.environ, self.redirect): + before = dict(os.environ) + package.git(self.root, 'show', 'HEAD:input.txt') + self.assertEqual(dict(os.environ), before) + # Repository isolation is not applied to package command processes. + output = self.parent / 'child.json' + command = 'import os,json,pathlib;pathlib.Path(' + repr(str(output)) + ').write_text(json.dumps({k:os.environ[k] for k in ("GIT_DIR","GIT_WORK_TREE")}))' + package.run_command([sys.executable, '-I', '-B', '-c', command], self.root, 5) + self.assertEqual(json.loads(output.read_text()), self.redirect) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_research_overlap_local.py b/tests/test_research_overlap_local.py index 0d932c1c39..60de842a37 100644 --- a/tests/test_research_overlap_local.py +++ b/tests/test_research_overlap_local.py @@ -30,6 +30,7 @@ def setUp(self): self.omitted = "research/omitted κ\npackage" for path, body in { "scripts/find_overlap.py": (ROOT / "scripts/find_overlap.py").read_text(encoding="utf-8"), + "scripts/research_package.py": (ROOT / "scripts/research_package.py").read_text(encoding="utf-8"), "research/kept/README.md": "needle local note", "research/kept/TASK.json": '{"scope": "needle local task"}', f"{self.omitted}/README.md": "only-omitted needle", diff --git a/tests/test_research_overlap_terms.py b/tests/test_research_overlap_terms.py index 5acfa6c27e..ed2f4bde8d 100644 --- a/tests/test_research_overlap_terms.py +++ b/tests/test_research_overlap_terms.py @@ -24,6 +24,8 @@ def report(self, card, records, *args): script = root / "scripts/find_overlap.py" script.parent.mkdir(parents=True) script.write_bytes((ROOT / "scripts/find_overlap.py").read_bytes()) + (script.parent / "research_package.py").write_bytes( + (ROOT / "scripts/research_package.py").read_bytes()) catalog = root / "docs/research/agent-task-catalog.json" catalog.parent.mkdir(parents=True) catalog.write_text(json.dumps({"tracks": [{"tasks": [card]}]}), encoding="utf-8")