diff --git a/.github/workflows/corpus.yml b/.github/workflows/corpus.yml index 9e5bb31..0aa0346 100644 --- a/.github/workflows/corpus.yml +++ b/.github/workflows/corpus.yml @@ -27,3 +27,19 @@ jobs: run: cargo test --release --locked -p zerosyntax-analysis --test corpus -- --ignored --nocapture - name: Corpus incremental equivalence run: cargo test --release --locked -p zerosyntax-analysis --test incremental corpus_edit_sequences_match_full_parse -- --ignored --nocapture + - name: Corpus performance benchmark + run: cargo bench --locked -p zerosyntax-analysis --bench analyze -- corpus + - name: Real-server corpus latency + run: | + cargo build --locked --release -p zerosyntax-server + python3 crates/server/tests/typing_latency.py \ + target/release/zerosyntax-lsp \ + corpus/GeneralsGamePatch2/GeneralsZH/Data/INI/ParticleSystem.ini \ + --workspace-root corpus/GeneralsGamePatch2/GeneralsZH/Data/INI \ + > "$RUNNER_TEMP/server-corpus.json" + - name: Check absolute performance ceilings + if: always() + run: | + python3 scripts/check-performance.py \ + --head "$RUNNER_TEMP/server-corpus.json" \ + --summary "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml new file mode 100644 index 0000000..eb3f825 --- /dev/null +++ b/.github/workflows/performance.yml @@ -0,0 +1,64 @@ +name: Performance + +on: + pull_request: + branches: [dev, prod] + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + +jobs: + performance: + runs-on: ubuntu-latest + steps: + - name: Check out PR head tools + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + - name: Preserve head measurement tools + run: | + cp crates/server/tests/e2e.py "$RUNNER_TEMP/e2e.py" + cp crates/server/tests/typing_latency.py "$RUNNER_TEMP/typing_latency.py" + - uses: Swatinem/rust-cache@v2 + + - name: Check out base + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.base.sha }} + fetch-depth: 0 + - name: Benchmark base + run: | + cargo bench --locked -p zerosyntax-syntax --bench parse -- 50000 --save-baseline pr-base + cargo bench --locked -p zerosyntax-analysis --bench analyze -- 50000 --save-baseline pr-base + - name: Measure base server + run: | + cargo build --locked --release -p zerosyntax-server + python3 "$RUNNER_TEMP/typing_latency.py" target/release/zerosyntax-lsp > "$RUNNER_TEMP/server-base.json" + + - name: Check out head + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + clean: false + - name: Benchmark head + run: | + cargo bench --locked -p zerosyntax-syntax --bench parse -- 50000 --baseline pr-base + cargo bench --locked -p zerosyntax-analysis --bench analyze -- 50000 --baseline pr-base + - name: Measure head server + run: | + cargo build --locked --release -p zerosyntax-server + python3 "$RUNNER_TEMP/typing_latency.py" target/release/zerosyntax-lsp > "$RUNNER_TEMP/server-head.json" + + - name: Check performance + if: always() + run: | + python3 scripts/check-performance.py \ + --criterion target/criterion \ + --base "$RUNNER_TEMP/server-base.json" \ + --head "$RUNNER_TEMP/server-head.json" \ + --summary "$GITHUB_STEP_SUMMARY" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4cc11cb..f11ee81 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,6 +27,22 @@ cargo test --release -p zerosyntax-analysis --test corpus -- --ignored --nocaptu On Windows, use `pwsh scripts/fetch-corpus.ps1`. +## Performance + +Run the existing synthetic benchmarks and real-server driver before changing a +hot path: + +```sh +cargo bench -p zerosyntax-syntax --bench parse +cargo bench -p zerosyntax-analysis --bench analyze +cargo build --release -p zerosyntax-server +python crates/server/tests/typing_latency.py target/release/zerosyntax-lsp +``` + +Pull requests warn when a probe regresses by 20% and fail at 50%; loose absolute +latency ceilings catch emergency-level slowdowns. Profile a failing probe before +weakening its threshold. + ## Pull requests - Keep changes focused. Split unrelated fixes into separate PRs. diff --git a/crates/analysis/benches/analyze.rs b/crates/analysis/benches/analyze.rs index cd21860..7276a95 100644 --- a/crates/analysis/benches/analyze.rs +++ b/crates/analysis/benches/analyze.rs @@ -10,7 +10,9 @@ use std::hint::black_box; use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use zerosyntax_analysis::diagnostics::DiagnosticsCache; -use zerosyntax_analysis::{diagnostics, index, semantic, Analyzer}; +use zerosyntax_analysis::{ + completion, diagnostics, index, semantic, Analyzer, Span, WorkspaceIndex, +}; use zerosyntax_syntax::{Edit, Strategy}; /// Same shape as the syntax-crate generator, but kept schema-conformant so the @@ -98,6 +100,69 @@ fn bench_analyze(c: &mut Criterion) { &parse, |b, parse| b.iter(|| black_box(index::definitions_in(&analyzer, parse, "bench.ini"))), ); + if lines == 50_000 { + group.bench_with_input( + BenchmarkId::new("index_refresh", lines), + &parse, + |b, parse| { + b.iter(|| { + let file = "bench.ini"; + let mut index = WorkspaceIndex::new(); + index.set_file(file, index::definitions_in(&analyzer, parse, file)); + index.set_file_refs(file, index::references_in(&analyzer, parse)); + index.set_file_tags(file, index::module_tags_in(&analyzer, parse)); + index.set_file_object_models( + file, + index::object_models_in(&analyzer, parse), + ); + index.set_file_object_parents(file, index::object_parents_in(parse)); + black_box(index) + }) + }, + ); + + let mut completion_src = src.clone(); + completion_src.push_str("Weapon CompletionBench\n ProjectileObject = \nEnd\n"); + let completion_offset = (completion_src.len() - "\nEnd\n".len()) as u32; + let completion_parse = analyzer.parse(&completion_src); + let mut completion_index = WorkspaceIndex::new(); + completion_index.set_file( + "bench.ini", + index::definitions_in(&analyzer, &completion_parse, "bench.ini"), + ); + assert!(completion::complete( + &analyzer, + &completion_parse, + completion_offset, + Some(&completion_index), + Some("bench.ini"), + ) + .iter() + .any(|item| item.label == "GenBenchTank1")); + group.bench_function(BenchmarkId::new("completion_reference", lines), |b| { + b.iter(|| { + black_box(completion::complete( + &analyzer, + &completion_parse, + completion_offset, + Some(&completion_index), + Some("bench.ini"), + )) + }) + }); + + let start = (src.len() / 2) as u32; + let viewport = Span::new(start, (start + 2_000).min(src.len() as u32)); + group.bench_with_input( + BenchmarkId::new("semantic_tokens_range", lines), + &parse, + |b, parse| { + b.iter(|| { + black_box(semantic::semantic_tokens_range(&analyzer, parse, viewport)) + }) + }, + ); + } // The full keystroke path as the server ran it pre-Phase-3: // full parse + full diagnose. group.bench_with_input(BenchmarkId::new("keystroke", lines), &src, |b, src| { diff --git a/crates/server/tests/typing_latency.py b/crates/server/tests/typing_latency.py index 78cddaa..c990e7e 100644 --- a/crates/server/tests/typing_latency.py +++ b/crates/server/tests/typing_latency.py @@ -1,15 +1,18 @@ #!/usr/bin/env python3 -"""Measure typing responsiveness through the real LSP binary. +"""Measure user-visible latency through the release LSP binary. -Usage: python typing_latency.py +Usage: typing_latency.py EXE [LARGE_INI] [--workspace-root DIR] """ +import argparse import json +import math import pathlib import queue import statistics import subprocess import sys +import tempfile import threading import time @@ -17,18 +20,128 @@ from e2e import frame, reader +def synthetic_document(target_lines=50_000): + out = ["Weapon CompletionProbe\n ProjectileObject = \nEnd\n\n"] + lines = 4 + i = 0 + while lines < target_lines: + if i % 2 == 0: + out.append( + f"Weapon PerfWeapon{i}\n" + " PrimaryDamage = 40.0\n" + " PrimaryDamageRadius = 5.0\n" + " SecondaryDamage = 10.0\n" + " SecondaryDamageRadius = 10.0\n" + " AttackRange = 150.0\n" + " MinimumAttackRange = 10.0\n" + " DamageType = ARMOR_PIERCING\n" + " DeathType = EXPLODED\n" + " WeaponSpeed = 600.0\n" + f" ProjectileObject = PerfObject{i + 1}\n" + " FireSound = NoSound\n" + " ScatterRadius = 2.5\n" + " AcceptableAimDelta = 5.0\n" + " RadiusDamageAngle = 180.0\n" + "End\n\n" + ) + lines += 17 + else: + out.append( + f"Object PerfObject{i}\n" + " Side = America\n" + " BuildCost = 900\n" + " BuildTime = 10.0\n" + " VisionRange = 150.0\n" + " KindOf = VEHICLE SELECTABLE\n" + " Draw = W3DModelDraw ModuleTag_Draw\n" + " ConditionState NONE\n" + f" Animation = PerfObject{i}.Idle\n" + " End\n" + " ConditionState REALLYDAMAGED\n" + f" Animation = PerfObject{i}.IdleDamaged\n" + " End\n" + " End\n" + " Body = ActiveBody ModuleTag_Body\n" + " MaxHealth = 300.0\n" + " InitialHealth = 300.0\n" + " End\n" + " Behavior = ArmorUpgrade ModuleTag_Armor\n" + " TriggeredBy = None\n" + " End\n" + "End\n\n" + ) + lines += 23 + i += 1 + return "".join(out) + + +def synthetic_workspace(root, file_count=200, target_lines=50_000): + lines_per_file = math.ceil(target_lines / file_count) + total_lines = 0 + serial = 0 + for file_number in range(file_count): + chunks = [] + lines = 0 + while lines < lines_per_file: + chunks.append( + f"Object WorkspaceObject{serial}\n" + " Side = America\n" + "End\n\n" + f"Weapon WorkspaceWeapon{serial}\n" + " PrimaryDamage = 10.0\n" + f" ProjectileObject = WorkspaceObject{serial}\n" + "End\n\n" + ) + lines += 9 + serial += 1 + (root / f"workspace-{file_number:03}.ini").write_text( + "".join(chunks), encoding="utf-8" + ) + total_lines += lines + return file_count, total_lines + + +def percentile_95(values): + return sorted(values)[math.ceil(len(values) * 0.95) - 1] + + def main() -> int: - if len(sys.argv) != 3: - print(f"usage: {sys.argv[0]} ", file=sys.stderr) - return 2 + parser = argparse.ArgumentParser() + parser.add_argument("exe") + parser.add_argument("large_ini", nargs="?") + parser.add_argument("--workspace-root", type=pathlib.Path) + args = parser.parse_args() + + temporary = None + if args.large_ini: + path = pathlib.Path(args.large_ini).resolve() + text = path.read_text(encoding="utf-8", errors="replace") + source = "file" + else: + temporary = tempfile.TemporaryDirectory(prefix="zerosyntax-performance-") + temporary_root = pathlib.Path(temporary.name) + path = temporary_root / "editing.ini" + text = synthetic_document() + path.write_text(text, encoding="utf-8") + source = "synthetic" + + workspace_root = args.workspace_root.resolve() if args.workspace_root else None + workspace_files = workspace_lines = 0 + if source == "synthetic" and workspace_root is None: + workspace_root = pathlib.Path(temporary.name) / "workspace" + workspace_root.mkdir() + workspace_files, workspace_lines = synthetic_workspace(workspace_root) + elif workspace_root: + workspace_files = sum( + 1 for item in workspace_root.rglob("*") if item.is_file() and item.suffix.lower() == ".ini" + ) - exe, filename = sys.argv[1:] - path = pathlib.Path(filename).resolve() - text = path.read_text(encoding="utf-8", errors="replace") lines = text.splitlines() uri = path.as_uri() + exe = pathlib.Path(args.exe) + executable = str(exe.resolve()) if exe.exists() else args.exe proc = subprocess.Popen( - [exe], + [executable], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, @@ -49,100 +162,173 @@ def receive(timeout=30.0): if message.get("method") == "textDocument/publishDiagnostics": params = message["params"] diagnostics.append( - (time.perf_counter(), params.get("version"), len(params["diagnostics"])) + ( + time.perf_counter(), + params["uri"], + params.get("version"), + len(params["diagnostics"]), + ) ) return message - def wait_id(request_id): + def wait_for(predicate, what): try: while True: message = receive() - if message.get("id") == request_id: + if predicate(message): return message except queue.Empty as error: - raise RuntimeError(f"timed out waiting for response {request_id}") from error + raise RuntimeError(f"timed out waiting for {what}") from error + + def wait_id(request_id): + return wait_for(lambda message: message.get("id") == request_id, f"response {request_id}") def wait_diagnostics(version): try: while True: - match = next((item for item in diagnostics if item[1] == version), None) + match = next( + (item for item in diagnostics if item[1] == uri and item[2] == version), + None, + ) if match: return match receive() except queue.Empty as error: - seen = [item[1] for item in diagnostics] + seen = [item[2] for item in diagnostics if item[1] == uri] raise RuntimeError( f"timed out waiting for diagnostics {version}; saw {seen}" ) from error - send({ - "jsonrpc": "2.0", - "id": 1, - "method": "initialize", - "params": { - "processId": None, - "rootUri": None, - "capabilities": {"general": {"positionEncodings": ["utf-8"]}}, - }, - }) + root_uri = workspace_root.as_uri() if workspace_root else None + send( + { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "processId": None, + "rootUri": root_uri, + "capabilities": {"general": {"positionEncodings": ["utf-8"]}}, + }, + } + ) wait_id(1) + initialized = time.perf_counter() send({"jsonrpc": "2.0", "method": "initialized", "params": {}}) + wait_for( + lambda message: message.get("method") == "window/logMessage" + and "language server ready" in message.get("params", {}).get("message", ""), + "language server ready notification", + ) + workspace_ready_ms = (time.perf_counter() - initialized) * 1000 + opened = time.perf_counter() - send({ - "jsonrpc": "2.0", - "method": "textDocument/didOpen", - "params": { - "textDocument": { - "uri": uri, - "languageId": "generals-ini", - "version": 1, - "text": text, - } - }, - }) + send( + { + "jsonrpc": "2.0", + "method": "textDocument/didOpen", + "params": { + "textDocument": { + "uri": uri, + "languageId": "generals-ini", + "version": 1, + "text": text, + } + }, + } + ) first_diag = wait_diagnostics(1) line_number, line = next( (i, line) for i, line in enumerate(lines) - if "=" in line and not line.lstrip().startswith(";") and line.split("=", 1)[1].strip() + if "=" in line + and not line.lstrip().startswith(";") + and line.split("=", 1)[1].strip() ) value_column = line.index("=") + 1 while line[value_column].isspace(): value_column += 1 original = line[value_column] - replacement = "X" if original != "X" else "Y" + replacement = "9" if original != "9" else "8" position = {"line": line_number, "character": value_column} end = {"line": line_number, "character": value_column + 1} - completion_position = {"line": line_number, "character": len(line)} + completion_line = next( + ( + i + for i, candidate in enumerate(lines) + if candidate.strip() == "ProjectileObject =" + ), + line_number, + ) + completion_position = { + "line": completion_line, + "character": len(lines[completion_line]), + } version = 1 request_id = 10 + current = original + + def content_change(character): + return { + "range": {"start": position, "end": end}, + "text": character, + } - def change(character): - nonlocal version + def change(): + nonlocal version, current + current = replacement if current == original else original version += 1 - send({ - "jsonrpc": "2.0", - "method": "textDocument/didChange", - "params": { - "textDocument": {"uri": uri, "version": version}, - "contentChanges": [{ - "range": {"start": position, "end": end}, - "text": character, - }], - }, - }) + send( + { + "jsonrpc": "2.0", + "method": "textDocument/didChange", + "params": { + "textDocument": {"uri": uri, "version": version}, + "contentChanges": [content_change(current)], + }, + } + ) - def complete(): + def bulk_change(count): + nonlocal version, current + changes = [] + for _ in range(count): + current = replacement if current == original else original + changes.append(content_change(current)) + version += 1 + send( + { + "jsonrpc": "2.0", + "method": "textDocument/didChange", + "params": { + "textDocument": {"uri": uri, "version": version}, + "contentChanges": changes, + }, + } + ) + + def request(method, params): nonlocal request_id request_id += 1 - send({ - "jsonrpc": "2.0", - "id": request_id, - "method": "textDocument/completion", - "params": {"textDocument": {"uri": uri}, "position": completion_position}, - }) - wait_id(request_id) + send( + { + "jsonrpc": "2.0", + "id": request_id, + "method": method, + "params": params, + } + ) + return wait_id(request_id) + + def complete(): + request( + "textDocument/completion", + { + "textDocument": {"uri": uri}, + "position": completion_position, + }, + ) idle = [] for _ in range(20): @@ -150,37 +336,69 @@ def complete(): complete() idle.append((time.perf_counter() - started) * 1000) - results = [] - current = original + semantic = [] + for _ in range(5): + started = time.perf_counter() + request( + "textDocument/semanticTokens/full", + {"textDocument": {"uri": uri}}, + ) + semantic.append((time.perf_counter() - started) * 1000) + + completion_metrics = {} + diagnostic_versions = [] + names = { + 1: "single_edit_to_completion", + 4: "four_edit_burst_to_completion", + 8: "eight_edit_burst_to_completion", + 16: "sixteen_edit_burst_to_completion", + } for burst in (1, 4, 8, 16): started = time.perf_counter() for _ in range(burst): - current = replacement if current == original else original - change(current) + change() latest = version complete() - completion_ms = (time.perf_counter() - started) * 1000 - latest_diag = wait_diagnostics(latest) - results.append({ - "edits": burst, - "completion_ms": round(completion_ms, 1), - "diagnostics_ms": round((latest_diag[0] - started) * 1000, 1), - "published_versions": [ - item[1] for item in diagnostics if started <= item[0] <= latest_diag[0] - ], - }) + completion_metrics[names[burst]] = (time.perf_counter() - started) * 1000 + wait_diagnostics(latest) + diagnostic_versions.append(latest) + started = time.perf_counter() + bulk_change(16) + latest = version + complete() + bulk_ms = (time.perf_counter() - started) * 1000 + latest_diag = wait_diagnostics(latest) + latest_diagnostics_ms = (latest_diag[0] - started) * 1000 + diagnostic_versions.append(latest) + + metrics = { + "workspace_ready": workspace_ready_ms, + "open_to_diagnostics": (first_diag[0] - opened) * 1000, + "idle_completion_median": statistics.median(idle), + "idle_completion_p95": percentile_95(idle), + "full_semantic_tokens": statistics.median(semantic), + **completion_metrics, + "sixteen_change_bulk_notification": bulk_ms, + "latest_diagnostics_after_editing": latest_diagnostics_ms, + } report = { - "file": str(path), - "file_mib": round(len(text.encode("utf-8")) / 1024 / 1024, 2), - "lines": len(lines), - "diagnostics": first_diag[2], - "open_to_diagnostics_ms": round((first_diag[0] - opened) * 1000, 1), - "idle_completion_median_ms": round(statistics.median(idle), 2), - "idle_completion_p95_ms": round(sorted(idle)[-2], 2), - "bursts": results, + "workload": { + "source": source, + "file": str(path), + "file_mib": round(len(text.encode("utf-8")) / 1024 / 1024, 2), + "lines": len(lines), + "workspace_root": str(workspace_root) if workspace_root else None, + "workspace_files": workspace_files, + "workspace_lines": workspace_lines or None, + "initial_diagnostics": first_diag[3], + "separate_edit_notifications": [1, 4, 8, 16], + "bulk_notification_changes": 16, + "diagnostic_versions": diagnostic_versions, + }, + "metrics_ms": {name: round(value, 4) for name, value in metrics.items()}, } - print(json.dumps(report, indent=2)) + print(json.dumps(report, indent=2, sort_keys=True)) send({"jsonrpc": "2.0", "id": 99, "method": "shutdown", "params": None}) wait_id(99) @@ -189,12 +407,14 @@ def complete(): proc.wait(timeout=5) except subprocess.TimeoutExpired: proc.kill() + if temporary: + temporary.cleanup() return 0 if __name__ == "__main__": try: sys.exit(main()) - except (AssertionError, RuntimeError, queue.Empty) as error: + except (AssertionError, OSError, RuntimeError, queue.Empty) as error: print(f"typing benchmark failed: {error}", file=sys.stderr) sys.exit(1) diff --git a/scripts/check-performance.py b/scripts/check-performance.py new file mode 100644 index 0000000..810a26f --- /dev/null +++ b/scripts/check-performance.py @@ -0,0 +1,469 @@ +#!/usr/bin/env python3 +"""Compare Criterion and real-server performance results. + +Exit 0 for pass/warnings, 1 for a performance failure, and 2 for bad input. +""" + +import argparse +import json +import math +import pathlib +import sys +import tempfile + + +REQUIRED_SERVER_METRICS = ( + "workspace_ready", + "open_to_diagnostics", + "idle_completion_median", + "idle_completion_p95", + "full_semantic_tokens", + "single_edit_to_completion", + "four_edit_burst_to_completion", + "eight_edit_burst_to_completion", + "sixteen_edit_burst_to_completion", + "sixteen_change_bulk_notification", + "latest_diagnostics_after_editing", +) + +ABSOLUTE_CEILINGS = { + "workspace_ready": (3_000, 5_000), + "open_to_diagnostics": (1_000, 2_000), + "idle_completion_p95": (50, 100), + "full_semantic_tokens": (500, 1_000), + "single_edit_to_completion": (50, 100), + "eight_edit_burst_to_completion": (250, 500), + "sixteen_edit_burst_to_completion": (500, 1_000), + "sixteen_change_bulk_notification": (500, 1_000), + "latest_diagnostics_after_editing": (1_000, 2_000), +} + +LEVEL = {"info": 0, "pass": 0, "warn": 1, "fail": 2} + + +class DataError(Exception): + pass + + +def load_json(path): + try: + with pathlib.Path(path).open(encoding="utf-8") as source: + return json.load(source) + except (OSError, json.JSONDecodeError) as error: + raise DataError(f"cannot read {path}: {error}") from error + + +def number(value, description, *, positive=False): + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise DataError(f"{description} must be a number") + value = float(value) + if not math.isfinite(value): + raise DataError(f"{description} must be finite") + if positive and value <= 0: + raise DataError(f"{description} must be positive") + return value + + +def estimate(path, relative=False): + data = load_json(path) + try: + mean = data["mean"] + point = number(mean["point_estimate"], f"{path} mean.point_estimate") + interval = mean["confidence_interval"] + lower = number(interval["lower_bound"], f"{path} lower_bound") + upper = number(interval["upper_bound"], f"{path} upper_bound") + except (KeyError, TypeError) as error: + raise DataError(f"malformed Criterion estimate {path}") from error + if not relative and point <= 0: + raise DataError(f"{path} mean.point_estimate must be positive") + return point, lower, upper + + +def baseline_estimate(benchmark): + preferred = (benchmark / "pr-base" / "estimates.json", benchmark / "base" / "estimates.json") + for path in preferred: + if path.is_file(): + return path + candidates = [ + path + for path in benchmark.glob("*/estimates.json") + if path.parent.name not in {"new", "change", "report"} + ] + return candidates[0] if len(candidates) == 1 else None + + +def worse(current, candidate): + return candidate if LEVEL[candidate] > LEVEL[current] else current + + +def criterion_rows(root): + root = pathlib.Path(root) + if not root.is_dir(): + raise DataError(f"Criterion directory is missing: {root}") + rows = [] + compared = set() + artifacts = list(root.rglob("estimates.json")) + if not artifacts: + raise DataError(f"no Criterion estimates found below {root}") + + for change_path in sorted(root.rglob("change/estimates.json")): + benchmark = change_path.parent.parent + name = benchmark.relative_to(root).as_posix() + base_path = baseline_estimate(benchmark) + head_path = benchmark / "new" / "estimates.json" + if base_path is None or not head_path.is_file(): + raise DataError(f"missing base/head estimates for Criterion benchmark {name}") + base = estimate(base_path)[0] + head = estimate(head_path)[0] + point, lower, upper = estimate(change_path, relative=True) + status = "fail" if lower >= 0.50 else "warn" if lower >= 0.20 else "pass" + rows.append( + { + "probe": f"Criterion: {name}", + "base": format_ns(base), + "head": format_ns(head), + "change": f"{point:+.1%}", + "confidence": f"[{lower:+.1%}, {upper:+.1%}]", + "budget": "95% CI lower bound: warn +20%, fail +50%", + "status": status, + } + ) + compared.add(name) + + benchmark_dirs = { + path.parent.parent for path in root.rglob("new/estimates.json") + } | { + path.parent.parent + for path in root.rglob("*/estimates.json") + if path.parent.name not in {"new", "change", "report"} + } + for benchmark in sorted(benchmark_dirs): + name = benchmark.relative_to(root).as_posix() + if name in compared: + continue + base_path = baseline_estimate(benchmark) + head_path = benchmark / "new" / "estimates.json" + if base_path and not head_path.is_file(): + detail = "removed benchmark" + elif not base_path and head_path.is_file(): + detail = "new benchmark" + else: + detail = "benchmark not compared" + rows.append( + { + "probe": f"Criterion: {name}", + "base": format_ns(estimate(base_path)[0]) if base_path else "—", + "head": format_ns(estimate(head_path)[0]) if head_path.is_file() else "—", + "change": "—", + "confidence": "—", + "budget": detail, + "status": "info", + } + ) + return rows + + +def server_metrics(path): + data = load_json(path) + try: + metrics = data["metrics_ms"] + except (KeyError, TypeError) as error: + raise DataError(f"{path} has no metrics_ms object") from error + if not isinstance(metrics, dict): + raise DataError(f"{path} metrics_ms must be an object") + parsed = { + name: number(value, f"{path} metrics_ms.{name}", positive=True) + for name, value in metrics.items() + } + missing = [name for name in REQUIRED_SERVER_METRICS if name not in parsed] + if missing: + raise DataError(f"{path} is missing server metrics: {', '.join(missing)}") + return parsed + + +def server_rows(base_path, head_path): + head = server_metrics(head_path) + base = server_metrics(base_path) if base_path else {} + rows = [] + for name in sorted(set(base) | set(head)): + if name not in head: + rows.append(info_row(f"Server: {name}", f"{base[name]:.2f} ms", "—", "removed metric")) + continue + if name not in base: + status = absolute_status(name, head[name]) + rows.append( + { + "probe": f"Server: {name}", + "base": "—", + "head": f"{head[name]:.2f} ms", + "change": "—", + "confidence": "—", + "budget": budget_text(name), + "status": status, + } + ) + continue + + delta = head[name] - base[name] + change = delta / base[name] + relative = ( + "fail" + if change >= 0.50 and delta >= 25 + else "warn" + if change >= 0.20 and delta >= 10 + else "pass" + ) + status = worse(relative, absolute_status(name, head[name])) + rows.append( + { + "probe": f"Server: {name}", + "base": f"{base[name]:.2f} ms", + "head": f"{head[name]:.2f} ms", + "change": f"{change:+.1%} ({delta:+.2f} ms)", + "confidence": "direct", + "budget": budget_text(name), + "status": status, + } + ) + return rows + + +def absolute_status(name, value): + ceiling = ABSOLUTE_CEILINGS.get(name) + if not ceiling: + return "pass" + warn, fail = ceiling + return "fail" if value >= fail else "warn" if value >= warn else "pass" + + +def budget_text(name): + relative = "relative: warn +20%/+10 ms, fail +50%/+25 ms" + if name not in ABSOLUTE_CEILINGS: + return relative + warn, fail = ABSOLUTE_CEILINGS[name] + return f"{relative}; absolute: warn {warn:g} ms, fail {fail:g} ms" + + +def info_row(probe, base, head, detail): + return { + "probe": probe, + "base": base, + "head": head, + "change": "—", + "confidence": "—", + "budget": detail, + "status": "info", + } + + +def format_ns(value): + if value >= 1_000_000: + return f"{value / 1_000_000:.2f} ms" + if value >= 1_000: + return f"{value / 1_000:.2f} µs" + return f"{value:.2f} ns" + + +def markdown(rows): + lines = [ + "## Performance", + "", + "| Probe | Base | Head | Change | 95% CI | Budget | Result |", + "|---|---:|---:|---:|---:|---|---|", + ] + labels = {"pass": "pass", "warn": "⚠️ warning", "fail": "❌ failure", "info": "info"} + for row in rows: + cells = [ + row["probe"], + row["base"], + row["head"], + row["change"], + row["confidence"], + row["budget"], + labels[row["status"]], + ] + lines.append("| " + " | ".join(str(cell).replace("|", "\\|") for cell in cells) + " |") + return "\n".join(lines) + "\n" + + +def annotate(rows): + for row in rows: + if row["status"] not in {"warn", "fail"}: + continue + kind = "warning" if row["status"] == "warn" else "error" + message = ( + f"{row['probe']}: base {row['base']}, head {row['head']}, " + f"change {row['change']}; budget {row['budget']}" + ) + message = message.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + print(f"::{kind} title=Performance::{message}") + + +def check(criterion, base, head, emit=True): + if not head: + raise DataError("--head is required") + rows = [] + if criterion: + rows.extend(criterion_rows(criterion)) + rows.extend(server_rows(base, head)) + if emit: + annotate(rows) + return (1 if any(row["status"] == "fail" for row in rows) else 0), rows + + +def write_summary(path, content): + destination = pathlib.Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + with destination.open("a", encoding="utf-8") as output: + output.write(content) + + +def fixture_server(path, **changes): + values = { + "workspace_ready": 100, + "open_to_diagnostics": 100, + "idle_completion_median": 5, + "idle_completion_p95": 10, + "full_semantic_tokens": 20, + "single_edit_to_completion": 10, + "four_edit_burst_to_completion": 20, + "eight_edit_burst_to_completion": 30, + "sixteen_edit_burst_to_completion": 40, + "sixteen_change_bulk_notification": 40, + "latest_diagnostics_after_editing": 300, + } + values.update(changes) + path.write_text(json.dumps({"metrics_ms": values}), encoding="utf-8") + + +def fixture_criterion(root, *, point=0.0, lower=-0.01, upper=0.01, baseline=True): + benchmark = root / "analysis" / "probe" / "50000" + (benchmark / "new").mkdir(parents=True) + absolute = { + "mean": { + "point_estimate": 110_000 if point else 100_000, + "confidence_interval": {"lower_bound": 99_000, "upper_bound": 111_000}, + } + } + (benchmark / "new" / "estimates.json").write_text(json.dumps(absolute), encoding="utf-8") + if baseline: + (benchmark / "pr-base").mkdir() + (benchmark / "pr-base" / "estimates.json").write_text( + json.dumps( + { + "mean": { + "point_estimate": 100_000, + "confidence_interval": {"lower_bound": 99_000, "upper_bound": 101_000}, + } + } + ), + encoding="utf-8", + ) + (benchmark / "change").mkdir() + (benchmark / "change" / "estimates.json").write_text( + json.dumps( + { + "mean": { + "point_estimate": point, + "confidence_interval": {"lower_bound": lower, "upper_bound": upper}, + } + } + ), + encoding="utf-8", + ) + + +def self_test(): + with tempfile.TemporaryDirectory(prefix="check-performance-") as directory: + root = pathlib.Path(directory) + + def run(name, *, criterion_args=None, base_changes=None, head_changes=None): + case = root / name + case.mkdir() + base = case / "base.json" + head = case / "head.json" + fixture_server(base, **(base_changes or {})) + fixture_server(head, **(head_changes or {})) + criterion = None + if criterion_args is not None: + criterion = case / "criterion" + fixture_criterion(criterion, **criterion_args) + return check(criterion, base, head, emit=False) + + assert run("pass", criterion_args={})[0] == 0 + result, rows = run( + "warning", + criterion_args={"point": 0.30, "lower": 0.25, "upper": 0.35}, + ) + assert result == 0 and any(row["status"] == "warn" for row in rows) + assert ( + run( + "blocking", + criterion_args={"point": 0.60, "lower": 0.50, "upper": 0.70}, + )[0] + == 1 + ) + assert run("absolute", head_changes={"workspace_ready": 5_001})[0] == 1 + assert ( + run( + "tiny-noise", + base_changes={"idle_completion_median": 0.1}, + head_changes={"idle_completion_median": 0.2}, + )[0] + == 0 + ) + result, rows = run("new-benchmark", criterion_args={"baseline": False}) + assert result == 0 and any(row["budget"] == "new benchmark" for row in rows) + + malformed_criterion = root / "malformed-criterion" + malformed_criterion.mkdir() + base = malformed_criterion / "base.json" + head = malformed_criterion / "head.json" + fixture_server(base) + fixture_server(head) + change = malformed_criterion / "criterion" / "probe" / "change" + change.mkdir(parents=True) + (change / "estimates.json").write_text("{", encoding="utf-8") + try: + check(malformed_criterion / "criterion", base, head, emit=False) + raise AssertionError("malformed Criterion JSON passed") + except DataError: + pass + + malformed_server = root / "malformed-server.json" + malformed_server.write_text("{", encoding="utf-8") + try: + check(None, None, malformed_server, emit=False) + raise AssertionError("malformed server JSON passed") + except DataError: + pass + + print("check-performance self-test: 8 cases passed") + return 0 + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--criterion", type=pathlib.Path) + parser.add_argument("--base", type=pathlib.Path) + parser.add_argument("--head", type=pathlib.Path) + parser.add_argument("--summary", type=pathlib.Path) + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args() + if args.self_test: + return self_test() + if not args.summary: + parser.error("--summary is required") + try: + result, rows = check(args.criterion, args.base, args.head) + write_summary(args.summary, markdown(rows)) + return result + except DataError as error: + message = f"performance results are malformed or incomplete: {error}" + print(f"::error title=Performance::{message}") + write_summary(args.summary, f"## Performance\n\n❌ {message}\n") + return 2 + + +if __name__ == "__main__": + sys.exit(main())