diff --git a/.github/workflows/frontier1094.yml b/.github/workflows/frontier1094.yml index 7d2a8d6f56..9b02d3341a 100644 --- a/.github/workflows/frontier1094.yml +++ b/.github/workflows/frontier1094.yml @@ -10,6 +10,65 @@ on: - 'research/gauge44-slots-t16/**' - 'scripts/verify_frontier1094.py' - '.github/workflows/frontier1094.yml' + # Selected-record inputs checked by scripts/check_selected_ci_inputs.py + - 'research/complex-sw4-frame8/**' + - 'research/e8-bit-p111-tuned/**' + - 'research/e8-bit-carrier-skeleton/**' + - 'research/direct-delivery504-global/**' + - 'research/gauge44-frames-x/**' + - 'research/complex-t21-owners/**' + - 'research/complex-t21-owner-bound/**' + - 'research/complex-t37-owners/**' + - 'research/gauge44-leaver/**' + - 'research/community-frontier1094-audit/**' + - 'certificates/checkpoints/frontier1042.json' + - 'certificates/replay-coverage/pr741-sw4-v1.json' + - 'certificates/selected-result.json' + - 'research/community-frontier1042-audit/**' + - 'scripts/verify_frontier1042.py' + - 'scripts/verify_frontier741.py' + - 'scripts/verify_frontier741_bindings.py' + - 'scripts/verify_frontier741_bridge_coverage.py' + - 'scripts/verify_frontier799.py' + - 'scripts/verify_frontier799_bindings.py' + - 'scripts/verify_replay_coverage.py' + - 'scripts/verify_selected_result.py' + # Live-checkout files the replay hash-asserts through UPSTREAM-PINS / INPUTS main + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/split-centres/CANONICAL-WORD.json.gz' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/split-centres/CLIFFORD-RECIPES.json' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/split-centres/CLIFFORD-REVIEW.json' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/split-centres/INDEPENDENT-LOCAL-REPLAY.json' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/split-centres/INDEPENDENT-REPLAY.json' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/split-centres/INTEGRATION.json' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/split-centres/LOCAL-CHECKS.json' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/split-centres/LOCAL-PROGRAMS.json.gz' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/split-centres/RESIDUAL-ADAPTERS.json' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/split-centres/SCALAR-FRAME-AUDIT.json' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/split-centres/WRAPPER-CHECKS.json' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/work/complex-nine-native/DATA-PHASES.json' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/controller-input/split-centres/CANONICAL-WORD.json.gz' + - 'research/complex-k492-owner-repair/vendor/direct_delivery/inputs/hamming/PACKET-PROGRAMS.json.gz' + - 'research/complex-k492-owner-repair/vendor/direct_delivery/inputs/hamming/PROTOTYPE.json' + - 'research/complex-k492-owner-repair/vendor/joint_prefix/code/build_joint_prefix.py' + - 'research/complex-k492-owner-repair/vendor/joint_prefix/data/JOINT-PREFIX.json' + - 'research/complex-k492-owner-repair/vendor/multicold/code/build_cold_kernel.py' + - 'research/complex-k492-owner-repair/vendor/multicold/code/run_portable.py' + - 'research/complex-k492-owner-repair/vendor/multicold/data/EIGHT-COLD.json' + - 'research/complex-k492-owner-repair/vendor/multicold/engines/base_two_moment.py' + - 'research/complex-k492-owner-repair/vendor/multicold/engines/moment.py' + - 'research/complex-k492-owner-repair/vendor/multicold/engines/outer.py' + - 'research/complex-k492-owner-repair/vendor/multicold/inputs/PR543-EXPECTED.json' + - 'research/complex-k492-owner-repair/vendor/multicold/vendor/pr558/code/phases_full.py' + - 'research/complex-k492-owner-repair/vendor/multicold/vendor/pr558/vendor/pr552/complex-global/vendor531/code/prepare_supplier_base.py' + - 'research/complex-k492-owner-repair/vendor/multicold/vendor/pr558/vendor/pr552/complex-global/vendor531/evidence/audit/PREFIX-AND-INVOICE.json' + - 'research/complex-k492-owner-repair/vendor/multicold/vendor/pr558/vendor/pr552/complex-global/vendor531/vendor/clifford/clifford_exact.py' + - 'research/complex-k492-owner-repair/vendor/multicold/vendor/pr558/vendor/pr552/complex-global/vendor535/inputs/ROUND20-PARENT-TEMPLATE.json.gz' + - 'research/complex-k492-owner-repair/vendor/multicold/vendor/pr558/vendor/pr552/complex-local/inputs/phase-baseline/CANONICAL-WORD.json.gz' + - 'research/complex-k492-owner-repair/vendor/multicold/vendor/pr558/vendor/pr552/complex-local/static44/gk/gk_common.py' + - 'research/complex-k492-owner-repair/vendor/multicold/vendor/pr558/vendor/pr552/complex-local/static44/gk/gk_word.py' + - 'research/complex-k492-owner-repair/vendor/multicold/vendor/pr558/vendor/pr552/complex-local/static44/inputs/RAW504.json.gz' + - 'research/structural-gain/frontier-composition/global-replay/base/split-centres/FULL-WORD.json.gz' + - 'research/structural-gain/frontier-composition/multicold-eight-word/FULL-WORD.json.gz' pull_request: paths: - 'research/complex-t16-rare-pattern-exchanges/**' @@ -18,6 +77,65 @@ on: - 'research/gauge44-slots-t16/**' - 'scripts/verify_frontier1094.py' - '.github/workflows/frontier1094.yml' + # Selected-record inputs checked by scripts/check_selected_ci_inputs.py + - 'research/complex-sw4-frame8/**' + - 'research/e8-bit-p111-tuned/**' + - 'research/e8-bit-carrier-skeleton/**' + - 'research/direct-delivery504-global/**' + - 'research/gauge44-frames-x/**' + - 'research/complex-t21-owners/**' + - 'research/complex-t21-owner-bound/**' + - 'research/complex-t37-owners/**' + - 'research/gauge44-leaver/**' + - 'research/community-frontier1094-audit/**' + - 'certificates/checkpoints/frontier1042.json' + - 'certificates/replay-coverage/pr741-sw4-v1.json' + - 'certificates/selected-result.json' + - 'research/community-frontier1042-audit/**' + - 'scripts/verify_frontier1042.py' + - 'scripts/verify_frontier741.py' + - 'scripts/verify_frontier741_bindings.py' + - 'scripts/verify_frontier741_bridge_coverage.py' + - 'scripts/verify_frontier799.py' + - 'scripts/verify_frontier799_bindings.py' + - 'scripts/verify_replay_coverage.py' + - 'scripts/verify_selected_result.py' + # Live-checkout files the replay hash-asserts through UPSTREAM-PINS / INPUTS main + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/split-centres/CANONICAL-WORD.json.gz' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/split-centres/CLIFFORD-RECIPES.json' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/split-centres/CLIFFORD-REVIEW.json' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/split-centres/INDEPENDENT-LOCAL-REPLAY.json' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/split-centres/INDEPENDENT-REPLAY.json' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/split-centres/INTEGRATION.json' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/split-centres/LOCAL-CHECKS.json' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/split-centres/LOCAL-PROGRAMS.json.gz' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/split-centres/RESIDUAL-ADAPTERS.json' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/split-centres/SCALAR-FRAME-AUDIT.json' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/split-centres/WRAPPER-CHECKS.json' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/base/work/complex-nine-native/DATA-PHASES.json' + - 'research/community-k492-audit/pr622/replay/baseline558/complex-global/controller-input/split-centres/CANONICAL-WORD.json.gz' + - 'research/complex-k492-owner-repair/vendor/direct_delivery/inputs/hamming/PACKET-PROGRAMS.json.gz' + - 'research/complex-k492-owner-repair/vendor/direct_delivery/inputs/hamming/PROTOTYPE.json' + - 'research/complex-k492-owner-repair/vendor/joint_prefix/code/build_joint_prefix.py' + - 'research/complex-k492-owner-repair/vendor/joint_prefix/data/JOINT-PREFIX.json' + - 'research/complex-k492-owner-repair/vendor/multicold/code/build_cold_kernel.py' + - 'research/complex-k492-owner-repair/vendor/multicold/code/run_portable.py' + - 'research/complex-k492-owner-repair/vendor/multicold/data/EIGHT-COLD.json' + - 'research/complex-k492-owner-repair/vendor/multicold/engines/base_two_moment.py' + - 'research/complex-k492-owner-repair/vendor/multicold/engines/moment.py' + - 'research/complex-k492-owner-repair/vendor/multicold/engines/outer.py' + - 'research/complex-k492-owner-repair/vendor/multicold/inputs/PR543-EXPECTED.json' + - 'research/complex-k492-owner-repair/vendor/multicold/vendor/pr558/code/phases_full.py' + - 'research/complex-k492-owner-repair/vendor/multicold/vendor/pr558/vendor/pr552/complex-global/vendor531/code/prepare_supplier_base.py' + - 'research/complex-k492-owner-repair/vendor/multicold/vendor/pr558/vendor/pr552/complex-global/vendor531/evidence/audit/PREFIX-AND-INVOICE.json' + - 'research/complex-k492-owner-repair/vendor/multicold/vendor/pr558/vendor/pr552/complex-global/vendor531/vendor/clifford/clifford_exact.py' + - 'research/complex-k492-owner-repair/vendor/multicold/vendor/pr558/vendor/pr552/complex-global/vendor535/inputs/ROUND20-PARENT-TEMPLATE.json.gz' + - 'research/complex-k492-owner-repair/vendor/multicold/vendor/pr558/vendor/pr552/complex-local/inputs/phase-baseline/CANONICAL-WORD.json.gz' + - 'research/complex-k492-owner-repair/vendor/multicold/vendor/pr558/vendor/pr552/complex-local/static44/gk/gk_common.py' + - 'research/complex-k492-owner-repair/vendor/multicold/vendor/pr558/vendor/pr552/complex-local/static44/gk/gk_word.py' + - 'research/complex-k492-owner-repair/vendor/multicold/vendor/pr558/vendor/pr552/complex-local/static44/inputs/RAW504.json.gz' + - 'research/structural-gain/frontier-composition/global-replay/base/split-centres/FULL-WORD.json.gz' + - 'research/structural-gain/frontier-composition/multicold-eight-word/FULL-WORD.json.gz' permissions: contents: read jobs: diff --git a/scripts/check_selected_ci_inputs.py b/scripts/check_selected_ci_inputs.py new file mode 100644 index 0000000000..a32ab26bca --- /dev/null +++ b/scripts/check_selected_ci_inputs.py @@ -0,0 +1,366 @@ +#!/usr/bin/env python3 +"""Fast guard for live-checkout inputs of the selected cold replay. + +The selected `reproduce` command passes `--main ROOT` to package stages that +hash-assert repository files outside the selected package roots, through each +root's data/UPSTREAM-PINS.json and construction/INPUTS.json `files.main`. Its +first stage also checks the checkpoint records the driver names. The per-PR +check of the selected record (verify_selected_result.py) covers neither, so a +changed byte could otherwise surface only inside the cold replay. + +Hard checks (standard library only, about a second): +1. every such live pin matches the checkout, with no conflicting pins, and any + in-root pin equals the selected source inventory; +2. every selected-result-format record named by the driver's static script + closure passes verify_selected_result.check. + +Advisory report (enforced only with --require-triggers): the one workflow whose +run commands invoke the driver, or its make target, must trigger on push and +pull_request for every live pin, every inventoried root file, the driver's +script closure and the repository files that closure names, including the +named records' certificates, manifests and inventories. Trigger +forms this reader cannot evaluate exactly are reported, never assumed. + +A trigger and drift guard, not a replay or a mathematical check. +Sagar Thakrar with substantial Anthropic Claude assistance. Apache-2.0. +""" +import argparse +import ast +from functools import lru_cache +from hashlib import sha256 +import importlib.util +import json +from pathlib import Path, PurePosixPath +import re +import sys + +ROOT = Path(__file__).resolve().parents[1] +PIN_FILES = ('data/UPSTREAM-PINS.json', 'construction/INPUTS.json') +EVENTS = ('push', 'pull_request') +RECORD_KEYS = {'kappa', 'certificate', 'certificate_sha256', 'source_manifest', 'proof_supplements'} + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def read(path): + return json.loads(Path(path).read_bytes()) + + +def relative(name): + p = PurePosixPath(name) if type(name) is str else None + require(p is not None and name and not p.is_absolute() and '..' not in p.parts and + p.as_posix() == name and '\\' not in name, 'Invalid repository path: ' + repr(name)) + return name + + +def inside(name, roots): + return any(name == r or name.startswith(r + '/') for r in roots) + + +def live_pins(root=ROOT): + """Return (record, manifest, pins) for pinned files outside the selected roots.""" + record = read(root/'certificates/selected-result.json') + manifest = read(root/relative(record['source_manifest'])) + roots = [relative(r) for r in manifest['package_roots']] + pins = {} + for package in roots: + for name in PIN_FILES: + path = root/package/name + if not path.is_file(): + continue + value = read(path) + if name.endswith('INPUTS.json'): + value = value['files']['main'] + require(type(value) is dict, 'Pin map is not an object: ' + package + '/' + name) + for rel, digest in value.items(): + relative(rel) + require(type(digest) is str and re.fullmatch('[0-9a-f]{64}', digest), 'Invalid pin: ' + rel) + if inside(rel, roots): + require(manifest['files'].get(rel) == digest, + 'Live pin differs from the selected source inventory: ' + rel) + continue + require(pins.setdefault(rel, digest) == digest, 'Conflicting live pins: ' + rel) + return record, manifest, pins + + +def drift(pins, root=ROOT): + bad = [] + for rel, digest in sorted(pins.items()): + path = root/rel + if not path.is_file() or sha256(path.read_bytes()).hexdigest() != digest: + bad.append(rel) + return bad + + +def driver(record, root=ROOT): + """Return (make target, driver script) for the record's reproduce command.""" + words = record['reproduce'].split() + require(len(words) == 2 and words[0] == 'make', 'Unsupported reproduce command') + lines = (root/'Makefile').read_text().splitlines() + starts = [i for i, line in enumerate(lines) if line.startswith(words[1] + ':')] + require(len(starts) == 1, 'Reproduce target not unique in Makefile') + scripts = [] + for line in lines[starts[0]+1:]: + if not line.startswith('\t'): + break + scripts += re.findall(r'scripts/[\w.-]+\.py', line) + require(len(set(scripts)) == 1, 'Reproduce target must run exactly one driver script') + return words[1], scripts[0] + + +def closure(script, root=ROOT): + """Local script imports and existing repository files named by string literals.""" + seen, files, todo = set(), set(), [script] + while todo: + name = todo.pop() + if name in seen: + continue + seen.add(name) + for node in ast.walk(ast.parse((root/name).read_bytes())): + modules = [] + if isinstance(node, ast.Import): + modules = [a.name for a in node.names] + elif isinstance(node, ast.ImportFrom) and node.module and not node.level: + modules = [node.module] + for module in modules: + local = 'scripts/' + module.split('.')[0] + '.py' + if (root/local).is_file(): + todo.append(local) + if isinstance(node, ast.Constant) and type(node.value) is str and '/' in node.value: + value = node.value.rstrip('/') + if not value.startswith('/') and '..' not in value.split('/') and (root/value).is_file(): + files.add(value) + if value.startswith('scripts/') and value.endswith('.py'): + todo.append(value) + return sorted(seen), sorted(files - seen) + + +def named_records(named, root=ROOT): + """Closure-named JSON files in selected-result format (checkpoint records).""" + out = [] + for name in named: + if name.startswith('certificates/') and name.endswith('.json'): + value = read(root/name) + if type(value) is dict and RECORD_KEYS <= set(value): + out.append(name) + return out + + +def check_records(names, root=ROOT): + """Run the repository's own selected-record check on each named record.""" + spec = importlib.util.spec_from_file_location('ci_inputs_selected_result', root/'scripts/verify_selected_result.py') + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + failures, files = {}, set() + for name in names: + record = read(root/name) + try: + module.check(record, root) + except (ValueError, KeyError, OSError) as error: + failures[name] = str(error) + # Proof supplements and reviews are hard-checked above on every PR; the + # replay trigger needs the certificate, manifest and inventory. + manifest = read(root/record['source_manifest']) + files |= {record['certificate'], record['source_manifest'], *manifest['files']} + return failures, files + + +def strip_comment(line): + return '' if line.lstrip().startswith('#') else line.split(' #')[0].rstrip() + + +def indent(line): + return len(line) - len(line.lstrip()) + + +def key(line): + return line.strip().split(':', 1)[0].strip().strip('\'"') + + +def event_paths(text, event): + """Read `on.` path filters from block-style workflows. + + Returns None if the event is absent, [] if it has no path filter and the + list of patterns otherwise. Any other filter (paths-ignore, branches without + main, types, tags, ...) raises, because its effect is not evaluated here. + """ + lines = [strip_comment(line) for line in text.splitlines()] + top = [i for i, line in enumerate(lines) if indent(line) == 0 and line and key(line) == 'on' + and line.strip().endswith(':')] + require(len(top) == 1, 'Workflow needs one block-style on: key') + body = [] + for line in lines[top[0]+1:]: + if line and indent(line) == 0: + break + body.append(line) + heads = [i for i, line in enumerate(body) if line and key(line) == event] + if not heads: + return None + require(len(heads) == 1 and body[heads[0]].strip().endswith(':'), 'Unsupported ' + event + ' form') + event_indent, section = indent(body[heads[0]]), [] + for line in body[heads[0]+1:]: + if line and indent(line) <= event_indent: + break + section.append(line) + entries = [i for i, line in enumerate(section) if line and not line.lstrip().startswith('-')] + child = min((indent(section[i]) for i in entries), default=None) + filters = {} + for i in entries: + require(indent(section[i]) == child and ':' in section[i], 'Unsupported ' + event + ' layout') + name, rest = key(section[i]), section[i].split(':', 1)[1].strip() + require(name not in filters, 'Repeated ' + event + ' key: ' + name) + if rest: + require(rest.startswith('[') and rest.endswith(']'), 'Unsupported ' + event + '.' + name) + filters[name] = [x.strip().strip('\'"') for x in rest[1:-1].split(',') if x.strip()] + continue + items = [] + for line in section[i+1:]: + if not line: + continue + if not line.lstrip().startswith('-'): + break + items.append(line.strip()[1:].strip().strip('\'"')) + filters[name] = items + unsupported = set(filters) - {'paths', 'branches'} + require(not unsupported, 'Unsupported ' + event + ' filter: ' + ', '.join(sorted(unsupported))) + if 'branches' in filters: + require('main' in filters['branches'], event + ' branch filter excludes main') + return filters.get('paths', []) + + +@lru_cache(maxsize=None) +def glob_regex(pattern): + out, i = [], 0 + while i < len(pattern): + if pattern.startswith('**/', i): + out.append('(?:.*/)?') + i += 3 + elif pattern.startswith('**', i): + out.append('.*') + i += 2 + elif pattern[i] == '*': + out.append('[^/]*') + i += 1 + else: + # GitHub gives ? and + quantifier meanings; fail closed rather than emulate them. + require(pattern[i] not in '?+[]{}', 'Unsupported glob syntax: ' + pattern) + out.append(re.escape(pattern[i])) + i += 1 + return re.compile('^' + ''.join(out) + '$') + + +def triggers(patterns, name): + """GitHub semantics: patterns apply in order; a later !pattern excludes.""" + if patterns is None: + return False + if not patterns: + return True + matched = False + for pattern in patterns: + negate = pattern.startswith('!') + if glob_regex(pattern[1:] if negate else pattern).match(name): + matched = not negate + return matched + + +def run_commands(text): + """Shell commands of `run:` steps, including block scalars.""" + lines, out, i = text.splitlines(), [], 0 + while i < len(lines): + line = strip_comment(lines[i]) + item = line.strip() + if item.startswith('- '): + item = item[2:].lstrip() + if item.startswith('run:'): + rest = item[4:].strip() + if rest[:1] in ('|', '>'): + base = indent(line) + i += 1 + while i < len(lines) and (not lines[i].strip() or indent(lines[i]) > base): + out.append(lines[i].strip()) + i += 1 + continue + out.append(rest) + i += 1 + return out + + +def replay_workflow(target, script, root=ROOT): + found = [] + for path in sorted((root/'.github/workflows').glob('*.y*ml')): + commands = run_commands(path.read_text()) + if any(script in c or re.search(r'\bmake\b.*\b' + re.escape(target) + r'\b', c) for c in commands): + found.append(path) + require(len(found) == 1, 'Expected one workflow running ' + script + ', found ' + + str([p.name for p in found])) + return found[0] + + +def required_paths(record, manifest, pins, scripts, named, record_files, workflow): + return sorted(set(pins) | set(manifest['files']) | set(scripts) | set(named) | set(record_files) | + {record['source_manifest'], record['certificate'], workflow}) + + +def coverage(root, record, manifest, pins, target, script, scripts, named, record_files): + workflow = replay_workflow(target, script, root) + required = required_paths(record, manifest, pins, scripts, named, record_files, + workflow.relative_to(root).as_posix()) + text = workflow.read_text() + uncovered = {} + for event in EVENTS: + patterns = event_paths(text, event) + uncovered[event] = [p for p in required if not triggers(patterns, p)] + return workflow.relative_to(root).as_posix(), len(required), uncovered + + +def check(root=ROOT): + record, manifest, pins = live_pins(root) + result = dict(live_pins=len(pins), pin_drift=drift(pins, root), driver=None, driver_closure=[], + named_files=[], named_records=[], record_failures={}, workflow=None, + required_paths=None, uncovered=None, coverage_error=None) + try: + target, script = driver(record, root) + scripts, named = closure(script, root) + except ValueError as error: + # A reproduce command this guard cannot resolve is reported, not blocking. + result['coverage_error'] = 'Driver not resolved: ' + str(error) + return result + records = named_records(named, root) + failures, record_files = check_records(records, root) + result.update(driver=script, driver_closure=scripts, named_files=named, named_records=records, + record_failures=failures) + try: + workflow, count, uncovered = coverage(root, record, manifest, pins, target, script, + scripts, named, record_files) + result.update(workflow=workflow, required_paths=count, uncovered=uncovered) + except ValueError as error: + result['coverage_error'] = str(error) + return result + + +def complete(result): + return result['coverage_error'] is None and not any(result['uncovered'].values()) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--root', type=Path, default=ROOT) + parser.add_argument('--require-triggers', action='store_true', + help='also fail unless the replay workflow triggers on every required path') + args = parser.parse_args(argv) + result = check(args.root.resolve()) + ok = not result['pin_drift'] and not result['record_failures'] + if args.require_triggers: + ok = ok and complete(result) + print(json.dumps(result, indent=1)) + if not complete(result): + print('ADVISORY: replay workflow triggers are incomplete or not evaluable') + print('PASS_SELECTED_CI_INPUTS' if ok else 'FAIL_SELECTED_CI_INPUTS') + return 0 if ok else 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tests/test_selected_ci_inputs.py b/tests/test_selected_ci_inputs.py new file mode 100644 index 0000000000..533963269c --- /dev/null +++ b/tests/test_selected_ci_inputs.py @@ -0,0 +1,325 @@ +"""Live-checkout inputs of the selected replay must stay pinned; its workflow should trigger on them.""" +from contextlib import redirect_stdout +from hashlib import sha256 +import importlib.util +import io +import json +from pathlib import Path +import shutil +import tempfile +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location('selected_ci_inputs', ROOT / 'scripts/check_selected_ci_inputs.py') +guard = importlib.util.module_from_spec(spec) +spec.loader.exec_module(guard) +RECORD = json.loads((ROOT / 'certificates/selected-result.json').read_bytes()) +SELECTED_1094 = RECORD['reproduce'] == 'make frontier1094-verify' + + +def quiet_main(argv): + with redirect_stdout(io.StringIO()): + return guard.main(argv) + + +class CurrentTree(unittest.TestCase): + def test_hard_checks_pass(self): + result = guard.check(ROOT) + self.assertEqual(result['pin_drift'], []) + self.assertEqual(result['record_failures'], {}) + self.assertEqual(quiet_main(['--root', str(ROOT)]), 0) + + def test_live_pins_are_outside_the_fast_inventory(self): + record, manifest, pins = guard.live_pins(ROOT) + for rel in pins: + self.assertNotIn(rel, manifest['files']) + self.assertNotIn(rel, record['proof_supplements']) + + @unittest.skipUnless(SELECTED_1094, 'expectations below describe the #1094 selection') + def test_frontier1094_shape(self): + _, _, pins = guard.live_pins(ROOT) + self.assertEqual(len(pins), 35) + # One pin each that only INPUTS.json main or only UPSTREAM-PINS.json supplies. + self.assertIn('research/complex-k492-owner-repair/vendor/joint_prefix/code/build_joint_prefix.py', pins) + self.assertIn('research/community-k492-audit/pr622/replay/baseline558/complex-global/base/' + 'split-centres/CLIFFORD-RECIPES.json', pins) + result = guard.check(ROOT) + self.assertEqual(result['driver_closure'], [ + 'scripts/verify_frontier1042.py', 'scripts/verify_frontier1094.py', 'scripts/verify_frontier741.py', + 'scripts/verify_frontier741_bindings.py', 'scripts/verify_frontier741_bridge_coverage.py', + 'scripts/verify_frontier799.py', 'scripts/verify_frontier799_bindings.py', + 'scripts/verify_replay_coverage.py', 'scripts/verify_selected_result.py']) + self.assertIn('certificates/checkpoints/frontier1042.json', result['named_records']) + self.assertEqual(result['workflow'], '.github/workflows/frontier1094.yml') + + @unittest.skipUnless(SELECTED_1094, 'expectations below describe the #1094 selection') + def test_frontier1094_required_paths(self): + record, manifest, pins = guard.live_pins(ROOT) + target, script = guard.driver(record, ROOT) + scripts, named = guard.closure(script, ROOT) + records = guard.named_records(named, ROOT) + _, record_files = guard.check_records(records, ROOT) + required = set(guard.required_paths(record, manifest, pins, scripts, named, record_files, + '.github/workflows/frontier1094.yml')) + for path in ('research/complex-t16-rare-pattern-exchanges/README.md', # #1094-only root inventory + 'certificates/checkpoints/frontier1042.json', # named by the driver closure + 'research/community-frontier1042-audit/ASSEMBLY.json', # checkpoint certificate + 'research/community-frontier1042-audit/SELECTED-SOURCE.json', + 'scripts/verify_replay_coverage.py', *pins): + self.assertIn(path, required) + + @unittest.skipUnless(SELECTED_1094, 'trigger coverage is enforced for the #1094 workflow only') + def test_frontier1094_workflow_triggers_complete(self): + result = guard.check(ROOT) + self.assertIsNone(result['coverage_error']) + self.assertEqual(result['uncovered'], dict(push=[], pull_request=[])) + self.assertEqual(quiet_main(['--root', str(ROOT), '--require-triggers']), 0) + + +class ExitCodes(unittest.TestCase): + def outcome(self, argv=(), **changes): + result = dict(pin_drift=[], record_failures={}, coverage_error=None, + uncovered=dict(push=[], pull_request=[])) + result.update(changes) + with mock.patch.object(guard, 'check', return_value=result): + return quiet_main(['--root', str(ROOT), *argv]) + + def test_hard_failures(self): + self.assertEqual(self.outcome(), 0) + self.assertEqual(self.outcome(pin_drift=['x']), 1) + self.assertEqual(self.outcome(record_failures={'r': 'changed'}), 1) + + def test_triggers_are_advisory_unless_required(self): + gaps = dict(uncovered=dict(push=[], pull_request=['x'])) + self.assertEqual(self.outcome(**gaps), 0) + self.assertEqual(self.outcome(['--require-triggers'], **gaps), 1) + error = dict(coverage_error='Unsupported', uncovered=None) + self.assertEqual(self.outcome(**error), 0) + self.assertEqual(self.outcome(['--require-triggers'], **error), 1) + + +class Skeleton(unittest.TestCase): + """A minimal copy holding the files the pin and trigger checks read, derived from the record.""" + + @classmethod + def setUpClass(cls): + record, manifest, pins = guard.live_pins(ROOT) + if not pins: + raise unittest.SkipTest('the selected record has no live-checkout pins') + target, script = guard.driver(record, ROOT) + scripts, named = guard.closure(script, ROOT) + cls.workflow = guard.replay_workflow(target, script, ROOT).relative_to(ROOT).as_posix() + cls.manifest_path = record['source_manifest'] + cls.target, cls.script = target, script + cls.tmp = tempfile.TemporaryDirectory() + cls.root = Path(cls.tmp.name) + names = {'certificates/selected-result.json', cls.manifest_path, 'Makefile', cls.workflow, + *pins, *scripts, *named} + names |= {r + '/' + n for r in manifest['package_roots'] for n in guard.PIN_FILES + if (ROOT / r / n).is_file()} + for name in names: + target_path = cls.root / name + target_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(ROOT / name, target_path) + cls.pins = sorted(pins) + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def edit(self, name, change): + path = self.root / name + original = path.read_bytes() + path.write_bytes(change(original)) + self.addCleanup(path.write_bytes, original) + + def uncovered_pins(self): + text = (self.root / self.workflow).read_text() + return {event: [p for p in self.pins if not guard.triggers(guard.event_paths(text, event), p)] + for event in guard.EVENTS} + + def append_to_pull_request_paths(self, line): + def change(b): + lines = b.decode().split('\n') + head = next(i for i, x in enumerate(lines) if x.strip() == 'pull_request:') + items = [i for i in range(head + 1, len(lines)) if lines[i].lstrip().startswith('- ')] + last = max(i for i in items if all(lines[j].strip() == '' or lines[j].lstrip().startswith(('- ', '#')) + or guard.key(lines[j]) == 'paths' for j in range(head + 1, i))) + prefix = lines[last][:guard.indent(lines[last])] + return '\n'.join(lines[:last + 1] + [prefix + line] + lines[last + 1:]).encode() + self.edit(self.workflow, change) + + def test_skeleton_matches_repository(self): + self.assertEqual(guard.live_pins(self.root)[2], guard.live_pins(ROOT)[2]) + if SELECTED_1094: + self.assertEqual(self.uncovered_pins(), dict(push=[], pull_request=[])) + + def test_every_single_byte_pin_change_is_rejected(self): + _, _, pins = guard.live_pins(self.root) + for rel in self.pins: + with self.subTest(pin=rel): + path = self.root / rel + original = path.read_bytes() + path.write_bytes(original + b'\n') + try: + self.assertEqual(guard.drift(pins, self.root), [rel]) + finally: + path.write_bytes(original) + self.assertEqual(guard.drift(pins, self.root), []) + + def test_missing_pinned_file_is_rejected(self): + rel = self.pins[0] + path = self.root / rel + original = path.read_bytes() + path.unlink() + self.addCleanup(path.write_bytes, original) + self.assertEqual(guard.drift(guard.live_pins(self.root)[2], self.root), [rel]) + + @unittest.skipUnless(SELECTED_1094, 'baseline coverage holds for the #1094 workflow') + def test_later_negated_pull_request_pattern_is_reported(self): + rel = self.pins[0] + self.append_to_pull_request_paths("- '!" + rel + "'") + self.assertEqual(self.uncovered_pins(), dict(push=[], pull_request=[rel])) + + def test_unsupported_trigger_filters_fail_closed(self): + for extra in ("paths-ignore:\n - 'research/**'", 'types: [closed]', + "branches: [release]", "tags:\n - 'v*'"): + with self.subTest(extra=extra): + original = (self.root / self.workflow).read_text() + text = original.replace(' pull_request:\n', ' pull_request:\n ' + extra + '\n', 1) + self.assertNotEqual(text, original) + with self.assertRaises(ValueError): + guard.event_paths(text, 'pull_request') + + def test_quoted_and_branch_filtered_keys_are_read(self): + text = (self.root / self.workflow).read_text() + quoted = text.replace(' pull_request:\n paths:\n', " pull_request:\n 'paths':\n", 1) + self.assertNotEqual(quoted, text) + self.assertEqual(guard.event_paths(quoted, 'pull_request'), guard.event_paths(text, 'pull_request')) + self.assertEqual(guard.event_paths(text, 'push'), guard.event_paths(text, 'pull_request')) + + def test_conflicting_live_pins_are_rejected(self): + rel = self.pins[0] + manifest = guard.read(self.root / self.manifest_path) + roots = [r for r in manifest['package_roots'] if (self.root / r / 'data/UPSTREAM-PINS.json').is_file() + and rel in guard.read(self.root / r / 'data/UPSTREAM-PINS.json')] + if len(roots) < 2: + self.skipTest('no live pin shared by two roots') + def conflict(b): + value = json.loads(b) + value[rel] = '0' * 64 + return json.dumps(value).encode() + self.edit(roots[-1] + '/data/UPSTREAM-PINS.json', conflict) + with self.assertRaisesRegex(ValueError, 'Conflicting live pins'): + guard.live_pins(self.root) + + def test_pin_paths_must_stay_in_the_repository(self): + manifest = guard.read(self.root / self.manifest_path) + owner = next(r for r in manifest['package_roots'] if (self.root / r / 'data/UPSTREAM-PINS.json').is_file()) + for bad in ('../outside.json', '/etc/hosts', 'research/./x.json'): + with self.subTest(path=bad): + def escape(b, bad=bad): + value = json.loads(b) + value[bad] = '2' * 64 + return json.dumps(value).encode() + path = self.root / owner / 'data/UPSTREAM-PINS.json' + original = path.read_bytes() + path.write_bytes(escape(original)) + try: + with self.assertRaisesRegex(ValueError, 'Invalid repository path'): + guard.live_pins(self.root) + finally: + path.write_bytes(original) + + def test_inside_root_pin_must_match_inventory(self): + manifest = guard.read(self.root / self.manifest_path) + owner = next(r for r in manifest['package_roots'] if (self.root / r / 'data/UPSTREAM-PINS.json').is_file()) + member = next(f for f in manifest['files'] if f.startswith(owner + '/')) + def wrong(b): + value = json.loads(b) + value[member] = '1' * 64 + return json.dumps(value).encode() + self.edit(owner + '/data/UPSTREAM-PINS.json', wrong) + with self.assertRaisesRegex(ValueError, 'selected source inventory'): + guard.live_pins(self.root) + + def test_replay_workflow_detection(self): + other = self.root / '.github/workflows/other.yml' + self.addCleanup(lambda: other.unlink() if other.exists() else None) + other.write_text('# mentions ' + self.script + ' and make ' + self.target + '\nname: other\n' + 'on:\n push:\njobs:\n j:\n steps:\n - name: run ' + self.script + '\n' + ' run: echo unrelated\n') + self.assertEqual(guard.replay_workflow(self.target, self.script, self.root).name, Path(self.workflow).name) + for command in (' - run: python -B ' + self.script + ' --work x\n', + ' - run: |\n make ' + self.target + '\n'): + with self.subTest(command=command): + other.write_text('name: other\non:\n push:\njobs:\n j:\n steps:\n' + command) + with self.assertRaisesRegex(ValueError, 'Expected one workflow'): + guard.replay_workflow(self.target, self.script, self.root) + + +class Records(unittest.TestCase): + def test_changed_checkpoint_certificate_is_reported(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + (root / 'scripts').mkdir() + shutil.copyfile(ROOT / 'scripts/verify_selected_result.py', root / 'scripts/verify_selected_result.py') + package = root / 'supplier' + package.mkdir() + (package / 'proof.md').write_text('frozen proof\n') + digest = sha256((package / 'proof.md').read_bytes()).hexdigest() + cert = root / 'certificate.json' + cert.write_text('{"kappa":"1/1000"}') + (root / 'manifest.json').write_text(json.dumps({'format': 'repository-packages-sha256-v1', + 'package_roots': ['supplier'], + 'files': {'supplier/proof.md': digest}})) + record = dict(kappa='1/1000', decimal='0.001', certificate='certificate.json', + certificate_sha256=sha256(cert.read_bytes()).hexdigest(), + source_manifest='manifest.json', proof='supplier/proof.md', + review='supplier/proof.md', validation='certificate.json', + proof_supplements={'supplier/proof.md': digest}) + (root / 'certificates').mkdir() + (root / 'certificates/checkpoint.json').write_text(json.dumps(record)) + self.assertEqual(guard.named_records(['certificates/checkpoint.json'], root), ['certificates/checkpoint.json']) + failures, files = guard.check_records(['certificates/checkpoint.json'], root) + self.assertEqual(failures, {}) + self.assertEqual(files, {'certificate.json', 'manifest.json', 'supplier/proof.md'}) + cert.write_text('{"kappa":"1/1000"} ') + failures, _ = guard.check_records(['certificates/checkpoint.json'], root) + self.assertIn('certificates/checkpoint.json', failures) + + +class Globs(unittest.TestCase): + def test_github_path_filter_semantics(self): + cases = [ + (['research/a/**'], 'research/a/b/c.json', True), + (['research/a/*'], 'research/a/b/c.json', False), + (['research/**/c.json'], 'research/c.json', True), + (['research/**'], 'scripts/x.py', False), + (['research/**', '!research/a/**'], 'research/a/x', False), + (['!research/a/**', 'research/**'], 'research/a/x', True), + ([], 'anything', True), + (None, 'anything', False), + ] + for patterns, name, expected in cases: + with self.subTest(patterns=patterns, name=name): + self.assertEqual(guard.triggers(patterns, name), expected) + + def test_unsupported_syntax_fails_closed(self): + for pattern in ('research/[ab]/**', 'scripts/verify_?.py', 'research/CORE-23+.json', 'a/{b,c}'): + with self.subTest(pattern=pattern), self.assertRaises(ValueError): + guard.triggers([pattern], 'research/a/x') + with self.assertRaises(ValueError): + guard.event_paths("on: [push]\n", 'push') + + def test_event_reader(self): + text = ("name: x\non:\n workflow_dispatch:\n push:\n branches: [main]\n paths:\n" + " - 'a/**'\n # note\n - \"b.py\"\n pull_request:\njobs:\n j:\n") + self.assertEqual(guard.event_paths(text, 'push'), ['a/**', 'b.py']) + self.assertEqual(guard.event_paths(text, 'pull_request'), []) + self.assertIsNone(guard.event_paths(text, 'schedule')) + + +if __name__ == '__main__': + unittest.main()