Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions tests/factories.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

Import with `from factories import ...` (tests/ is on sys.path under pytest; see conftest.py).
"""
import contextlib
import datetime
import hashlib
import json
Expand All @@ -26,6 +27,7 @@
from tracekit.core import now_ts # noqa: E402
from tracekit.daemon import Signer, load_config # noqa: E402
from tracekit.ledger import read_records # noqa: E402
from tracekit.storage import file as storage_file # noqa: E402


def ev(typ, data, run="r1", agent="main", source="hook"):
Expand Down Expand Up @@ -361,6 +363,34 @@ def persisted_queue(signer):
return signer.log.storage.witness_queue()


# the modules whose syncs a signer on the file store makes: its logs, its new files, and its state and key files
SIGNER_SYNCS = {"tracekit.storage.file", "tracekit.signer.service", "tracekit.deploy.files"}


@contextlib.contextmanager
def signer_without_fsync():
"""The signer's syncs (SIGNER_SYNCS) skipped in this process while it's open, for a test of what a signer decides
and records, not of its durability, which has tests of its own (test_storage_file's test_durability_modes and
test_a_failed_background_sync_names_the_records_not_known_durable, storage_contract's
test_torn_tail_is_set_aside_and_reported): fsync is most of such a test's time on windows-latest (#722). Yields the
modules of the syncs made anywhere else, which still run: a sync the signer moves to another module fails the test
instead of silently making it slow again."""
elsewhere, real = [], {name: getattr(os, name) for name in ("fsync", "fdatasync") if hasattr(os, name)}

def stub(name):
def sync(fd):
caller = sys._getframe(1).f_globals.get("__name__")
if caller not in SIGNER_SYNCS:
elsewhere.append(caller)
real[name](fd)
return sync
with contextlib.ExitStack() as stack:
stack.enter_context(mock.patch.object(storage_file, "_sync", lambda fd, full: None)) # macOS's is an fcntl
for name in real:
stack.enter_context(mock.patch.object(os, name, stub(name)))
yield elsewhere


def process_pool():
"""Worker processes, one per CPU this process may use (at most 61: Windows takes no more), spawned: a test process
has threads, which a fork would copy mid-lock."""
Expand Down
17 changes: 12 additions & 5 deletions tests/test_bounded.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

import test_signer_service as ts
import test_ts_verifier as tv
from factories import process_pool
from factories import process_pool, signer_without_fsync
from test_verify_v2_property import zipped

from tracekit import merkle
Expand Down Expand Up @@ -371,14 +371,21 @@ def explore(self, actions, need, prefix=()):
starts = range(0, len(actions) ** APPROVALS_DEPTH, RUNS_PER_SIGNER)
with process_pool() as pool:
parts = list(pool.map(self.part, [actions] * len(starts), [prefix] * len(starts), starts))
self.assertEqual([w for wrong, _ in parts for w in wrong][:20], [])
self.assertEqual([w for wrong, _, _ in parts for w in wrong][:20], [])
self.assertEqual(sorted({m for _, _, elsewhere in parts for m in elsewhere}), []) # every sync was skipped
if APPROVALS_DEPTH >= 5: # the shortest sequence to each outcome takes five steps
self.assertEqual(set().union(*(reached for _, reached in parts)), need)
self.assertEqual(set().union(*(reached for _, reached, _ in parts)), need)

@staticmethod
def part(actions, prefix, start):
"""(in a worker process) Sequences start.. of the space, RUNS_PER_SIGNER of them, on a signer of their own:
(what was wrong, the outcomes reached)."""
"""(in a worker process) Sequences start.. of the space, RUNS_PER_SIGNER of them, on a signer of their own,
without its fsync: (what was wrong, the outcomes reached, the modules of the syncs made outside the signer)."""
with signer_without_fsync() as elsewhere:
wrong, reached = ApprovalStateSpace._part(actions, prefix, start)
return wrong, reached, elsewhere

@staticmethod
def _part(actions, prefix, start):
d, expect, wrong, reached = tempfile.mkdtemp(), {}, [], set() # lean: left behind when a part raises
s = svc.SignerService(d, policy=ts.PAY_ASKS, limits=LIMITS)
sequences = itertools.islice(itertools.product(actions, repeat=APPROVALS_DEPTH), start, start + RUNS_PER_SIGNER)
Expand Down
5 changes: 4 additions & 1 deletion tests/test_review_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from unittest import mock

import test_findings_v2 as fv
from factories import signer_without_fsync

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, ROOT)
Expand Down Expand Up @@ -507,7 +508,9 @@ def answer(body): # flags a run whose edit names a fault target, citing that c
return message({**FLAG, "run_seqs": bad} if bad else OK)
_, url = serve(self, answer)
out_path = os.path.join(d, "e20_reviewer.json")
out = e20.model_main({"E20_REVIEW_URL": f"{url}/v1/messages", "E20_REVIEW_MODEL": "gpt-5"}, out_path)
with signer_without_fsync() as elsewhere: # the model arm's scoring, not its signer's durability
out = e20.model_main({"E20_REVIEW_URL": f"{url}/v1/messages", "E20_REVIEW_MODEL": "gpt-5"}, out_path)
self.assertEqual(sorted(set(elsewhere)), []) # every sync was the signer's, skipped
with open(out_path) as f:
self.assertEqual(json.load(f)["families"], {"agent": "anthropic", "reviewer": "openai"})
self.assertLessEqual({"tuned", "development", "development_v2", "held_out"}, set(out)) # scored apart
Expand Down
Loading