From 16b5cbc847f5bd825ac9e9cf8dc07ebb7ec9940d Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Fri, 13 Nov 2026 13:35:24 +0530 Subject: [PATCH 1/2] tests: skip the signer's fsync in the approval state space's workers (#722) fsync was most of ApprovalStateSpace's time on windows-latest: 689 s as on main, 50 s with fsync skipped in the workers (52 s and 36 s on ubuntu-latest), with the same sequences and outcomes. The class checks the approval state machine; durability has its own tests, which the comment names. Each part skips the syncs of the signer's modules (its logs, its new files, and its state and key files) for its own run only, a patch held while the part runs in its worker process. A sync from any other module still runs and is counted, and the test expects none, so a sync the signer moves elsewhere fails it instead of making it slow again. Signed-off-by: Khushi Jain --- tests/test_bounded.py | 45 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/tests/test_bounded.py b/tests/test_bounded.py index f18788828..33aebd1fb 100644 --- a/tests/test_bounded.py +++ b/tests/test_bounded.py @@ -5,14 +5,17 @@ the first).""" import base64 import collections +import contextlib import importlib.util import itertools import json import os import shutil +import sys import tempfile import unittest import zipfile +from unittest import mock import test_signer_service as ts import test_ts_verifier as tv @@ -27,6 +30,7 @@ from tracekit.signer.quotas import Limits from tracekit.signer.rpc_schema import RPCError from tracekit.signer.service import STRICTNESS, _stricter +from tracekit.storage import file as storage_file from tracekit.verify import v2 ENGINES = ("re2", "regex") if importlib.util.find_spec("re2") else ("regex",) @@ -351,6 +355,32 @@ class EveryStructuralMutationOfARunInParts(EveryStructuralMutation): # that does fails the test). Past ~200, the signer's scans of its live approvals cost more than the starts they save RUNS_PER_SIGNER = 200 LIMITS = Limits(open_runs=2 * RUNS_PER_SIGNER, pending_approvals=2 * RUNS_PER_SIGNER, events_per_s=1e9, burst=10 ** 9) +# the modules whose syncs a signer on the file store makes: its logs, its new files, and its state and key files +SIGNER_SYNCS = {"tracekit.storage.file", "tracekit.signer.service", "tracekit.deploy.files"} + + +@contextlib.contextmanager +def signer_without_fsync(): + """The signer's syncs (SIGNER_SYNCS) skipped in this process while it's open: the state space checks the approval + path, and durability has tests of its own (test_storage_file's test_durability_modes and + test_a_failed_background_sync_names_the_records_not_known_durable, storage_contract's + test_torn_tail_is_set_aside_and_reported), while fsync made the space ~14x slower on windows-latest (#722). Yields + the modules of the syncs made anywhere else, which still run: a sync the signer moves to another module fails the + test instead of silently making it slow again.""" + elsewhere, real = [], {name: getattr(os, name) for name in ("fsync", "fdatasync") if hasattr(os, name)} + + def stub(name): + def sync(fd): + caller = sys._getframe(1).f_globals.get("__name__") + if caller not in SIGNER_SYNCS: + elsewhere.append(caller) + real[name](fd) + return sync + with contextlib.ExitStack() as stack: + stack.enter_context(mock.patch.object(storage_file, "_sync", lambda fd, full: None)) # macOS's is an fcntl + for name in real: + stack.enter_context(mock.patch.object(os, name, stub(name))) + yield elsewhere class ApprovalStateSpace(unittest.TestCase): @@ -371,14 +401,21 @@ def explore(self, actions, need, prefix=()): starts = range(0, len(actions) ** APPROVALS_DEPTH, RUNS_PER_SIGNER) with process_pool() as pool: parts = list(pool.map(self.part, [actions] * len(starts), [prefix] * len(starts), starts)) - self.assertEqual([w for wrong, _ in parts for w in wrong][:20], []) + self.assertEqual([w for wrong, _, _ in parts for w in wrong][:20], []) + self.assertEqual(sorted({m for _, _, elsewhere in parts for m in elsewhere}), []) # every sync was skipped if APPROVALS_DEPTH >= 5: # the shortest sequence to each outcome takes five steps - self.assertEqual(set().union(*(reached for _, reached in parts)), need) + self.assertEqual(set().union(*(reached for _, reached, _ in parts)), need) @staticmethod def part(actions, prefix, start): - """(in a worker process) Sequences start.. of the space, RUNS_PER_SIGNER of them, on a signer of their own: - (what was wrong, the outcomes reached).""" + """(in a worker process) Sequences start.. of the space, RUNS_PER_SIGNER of them, on a signer of their own, + without its fsync: (what was wrong, the outcomes reached, the modules of the syncs made outside the signer).""" + with signer_without_fsync() as elsewhere: + wrong, reached = ApprovalStateSpace._part(actions, prefix, start) + return wrong, reached, elsewhere + + @staticmethod + def _part(actions, prefix, start): d, expect, wrong, reached = tempfile.mkdtemp(), {}, [], set() # lean: left behind when a part raises s = svc.SignerService(d, policy=ts.PAY_ASKS, limits=LIMITS) sequences = itertools.islice(itertools.product(actions, repeat=APPROVALS_DEPTH), start, start + RUNS_PER_SIGNER) From dc7f2a1ea12b48bf0d17e8f19d363fd4a10f0361 Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Fri, 13 Nov 2026 14:36:14 +0530 Subject: [PATCH 2/2] tests: skip the signer's fsync in the e20 model arm test (#722) test_e20_model_arm_scores_both_sets_with_a_fake_model records every scored session in e20's in-process signer, and its syncs were most of the test's time on windows-latest. signer_without_fsync moves from test_bounded.py to tests/factories.py, unchanged but for its docstring, and the test holds it around model_main, with the same guard: a sync from outside the signer's modules still runs and fails the test. Signed-off-by: Khushi Jain --- tests/factories.py | 30 ++++++++++++++++++++++++++++++ tests/test_bounded.py | 32 +------------------------------- tests/test_review_model.py | 5 ++++- 3 files changed, 35 insertions(+), 32 deletions(-) diff --git a/tests/factories.py b/tests/factories.py index 8b452f306..46debe832 100644 --- a/tests/factories.py +++ b/tests/factories.py @@ -2,6 +2,7 @@ Import with `from factories import ...` (tests/ is on sys.path under pytest; see conftest.py). """ +import contextlib import datetime import hashlib import json @@ -26,6 +27,7 @@ from tracekit.core import now_ts # noqa: E402 from tracekit.daemon import Signer, load_config # noqa: E402 from tracekit.ledger import read_records # noqa: E402 +from tracekit.storage import file as storage_file # noqa: E402 def ev(typ, data, run="r1", agent="main", source="hook"): @@ -361,6 +363,34 @@ def persisted_queue(signer): return signer.log.storage.witness_queue() +# the modules whose syncs a signer on the file store makes: its logs, its new files, and its state and key files +SIGNER_SYNCS = {"tracekit.storage.file", "tracekit.signer.service", "tracekit.deploy.files"} + + +@contextlib.contextmanager +def signer_without_fsync(): + """The signer's syncs (SIGNER_SYNCS) skipped in this process while it's open, for a test of what a signer decides + and records, not of its durability, which has tests of its own (test_storage_file's test_durability_modes and + test_a_failed_background_sync_names_the_records_not_known_durable, storage_contract's + test_torn_tail_is_set_aside_and_reported): fsync is most of such a test's time on windows-latest (#722). Yields the + modules of the syncs made anywhere else, which still run: a sync the signer moves to another module fails the test + instead of silently making it slow again.""" + elsewhere, real = [], {name: getattr(os, name) for name in ("fsync", "fdatasync") if hasattr(os, name)} + + def stub(name): + def sync(fd): + caller = sys._getframe(1).f_globals.get("__name__") + if caller not in SIGNER_SYNCS: + elsewhere.append(caller) + real[name](fd) + return sync + with contextlib.ExitStack() as stack: + stack.enter_context(mock.patch.object(storage_file, "_sync", lambda fd, full: None)) # macOS's is an fcntl + for name in real: + stack.enter_context(mock.patch.object(os, name, stub(name))) + yield elsewhere + + def process_pool(): """Worker processes, one per CPU this process may use (at most 61: Windows takes no more), spawned: a test process has threads, which a fork would copy mid-lock.""" diff --git a/tests/test_bounded.py b/tests/test_bounded.py index 33aebd1fb..71c045bb9 100644 --- a/tests/test_bounded.py +++ b/tests/test_bounded.py @@ -5,21 +5,18 @@ the first).""" import base64 import collections -import contextlib import importlib.util import itertools import json import os import shutil -import sys import tempfile import unittest import zipfile -from unittest import mock import test_signer_service as ts import test_ts_verifier as tv -from factories import process_pool +from factories import process_pool, signer_without_fsync from test_verify_v2_property import zipped from tracekit import merkle @@ -30,7 +27,6 @@ from tracekit.signer.quotas import Limits from tracekit.signer.rpc_schema import RPCError from tracekit.signer.service import STRICTNESS, _stricter -from tracekit.storage import file as storage_file from tracekit.verify import v2 ENGINES = ("re2", "regex") if importlib.util.find_spec("re2") else ("regex",) @@ -355,32 +351,6 @@ class EveryStructuralMutationOfARunInParts(EveryStructuralMutation): # that does fails the test). Past ~200, the signer's scans of its live approvals cost more than the starts they save RUNS_PER_SIGNER = 200 LIMITS = Limits(open_runs=2 * RUNS_PER_SIGNER, pending_approvals=2 * RUNS_PER_SIGNER, events_per_s=1e9, burst=10 ** 9) -# the modules whose syncs a signer on the file store makes: its logs, its new files, and its state and key files -SIGNER_SYNCS = {"tracekit.storage.file", "tracekit.signer.service", "tracekit.deploy.files"} - - -@contextlib.contextmanager -def signer_without_fsync(): - """The signer's syncs (SIGNER_SYNCS) skipped in this process while it's open: the state space checks the approval - path, and durability has tests of its own (test_storage_file's test_durability_modes and - test_a_failed_background_sync_names_the_records_not_known_durable, storage_contract's - test_torn_tail_is_set_aside_and_reported), while fsync made the space ~14x slower on windows-latest (#722). Yields - the modules of the syncs made anywhere else, which still run: a sync the signer moves to another module fails the - test instead of silently making it slow again.""" - elsewhere, real = [], {name: getattr(os, name) for name in ("fsync", "fdatasync") if hasattr(os, name)} - - def stub(name): - def sync(fd): - caller = sys._getframe(1).f_globals.get("__name__") - if caller not in SIGNER_SYNCS: - elsewhere.append(caller) - real[name](fd) - return sync - with contextlib.ExitStack() as stack: - stack.enter_context(mock.patch.object(storage_file, "_sync", lambda fd, full: None)) # macOS's is an fcntl - for name in real: - stack.enter_context(mock.patch.object(os, name, stub(name))) - yield elsewhere class ApprovalStateSpace(unittest.TestCase): diff --git a/tests/test_review_model.py b/tests/test_review_model.py index 1c4c79dec..ed99801c4 100644 --- a/tests/test_review_model.py +++ b/tests/test_review_model.py @@ -17,6 +17,7 @@ from unittest import mock import test_findings_v2 as fv +from factories import signer_without_fsync ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, ROOT) @@ -507,7 +508,9 @@ def answer(body): # flags a run whose edit names a fault target, citing that c return message({**FLAG, "run_seqs": bad} if bad else OK) _, url = serve(self, answer) out_path = os.path.join(d, "e20_reviewer.json") - out = e20.model_main({"E20_REVIEW_URL": f"{url}/v1/messages", "E20_REVIEW_MODEL": "gpt-5"}, out_path) + with signer_without_fsync() as elsewhere: # the model arm's scoring, not its signer's durability + out = e20.model_main({"E20_REVIEW_URL": f"{url}/v1/messages", "E20_REVIEW_MODEL": "gpt-5"}, out_path) + self.assertEqual(sorted(set(elsewhere)), []) # every sync was the signer's, skipped with open(out_path) as f: self.assertEqual(json.load(f)["families"], {"agent": "anthropic", "reviewer": "openai"}) self.assertLessEqual({"tuned", "development", "development_v2", "held_out"}, set(out)) # scored apart