From 9e5b50809f09569654850a570cca8864820fa9da Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Mon, 3 Aug 2026 00:30:45 -0600 Subject: [PATCH 1/3] test(ci): measure native binding Rust coverage --- .github/workflows/test.yml | 3 + Makefile | 25 +- .../tests/composite_transaction.test.mjs | 133 +++++++++ .../tests/composite_transaction.py | 141 ++++++++++ docs/development/testing.md | 28 +- docs/engineering/TESTING.md | 18 ++ scripts/check-coverage-rust.sh | 47 +--- scripts/ci/test-rust-coverage-ledger.py | 136 +++++++++ scripts/coverage-rust.sh | 217 ++++++++++----- scripts/coverage_rust_ledger.py | 263 ++++++++++++++++++ 10 files changed, 885 insertions(+), 126 deletions(-) create mode 100755 scripts/ci/test-rust-coverage-ledger.py create mode 100755 scripts/coverage_rust_ledger.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8580bf3c6..f192f3f1e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -80,6 +80,9 @@ jobs: - name: Test Rust non-Cypher surface inventory run: python3 scripts/ci/test-non-cypher-surface-gate.py + - name: Test Rust coverage ledger failure modes + run: python3 scripts/ci/test-rust-coverage-ledger.py + - name: Test Python and Node non-Cypher parity policy run: python3 scripts/ci/check-binding-parity-policy.py diff --git a/Makefile b/Makefile index 60ed7fef5..861de685f 100644 --- a/Makefile +++ b/Makefile @@ -67,6 +67,8 @@ test-tck: ## Run TCK compliance tests via Rust BDD runner # Multi-surface coverage thresholds (#742 §2). Override per surface as needed. COVERAGE_FAIL_UNDER_RUST ?= 85 +COVERAGE_FAIL_UNDER_RUST_PYTHON_ADAPTER ?= 80 +COVERAGE_FAIL_UNDER_RUST_NODE_ADAPTER ?= 80 COVERAGE_FAIL_UNDER_PYTHON ?= 85 COVERAGE_FAIL_UNDER_NODE ?= 85 # Back-compat alias used by coverage-python. @@ -88,7 +90,7 @@ coverage: ## Rust + Python + Node coverage with per-surface thresholds @echo "━━━ Node JS surface coverage ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @$(MAKE) coverage-node @echo "━━━ Coverage complete ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - @echo "Rust: build/coverage-rust/ (lcov + html + summary)" + @echo "Rust: build/coverage-rust/ (ledger + merged lcov + core HTML + summary)" @echo "Python: coverage.xml + htmlcov/" @echo "Node: crates/graphforge-bindings-node/coverage/" @echo "✅ All surfaces collected; thresholds enforced per surface" @@ -96,7 +98,7 @@ coverage: ## Rust + Python + Node coverage with per-surface thresholds coverage-python: ## Run unit tests with Python wrapper coverage (requires maturin develop) @$(MAKE) _ensure-graphforge uv run coverage erase - uv run pytest tests/unit \ + uv run pytest tests/unit crates/graphforge-bindings-py/tests/cli_entrypoint.py \ -n $${PYTEST_WORKERS:-4} \ --cov=$(PYTHON_COVERAGE_SRC) --cov-branch \ --cov-report=term-missing \ @@ -137,10 +139,13 @@ check-coverage-python: ## Validate Python wrapper coverage (≥85% default) (echo "❌ Python wrapper coverage below $(COVERAGE_FAIL_UNDER_PYTHON)%" && exit 1) @echo "✅ Python wrapper coverage meets threshold" -check-coverage-rust: ## Validate Rust line coverage from llvm-cov summary (≥85% default) - @test -f build/coverage-rust/summary.txt || \ - (echo "❌ Missing build/coverage-rust/summary.txt — run make coverage-rust first"; exit 1) - @COVERAGE_FAIL_UNDER_RUST=$(COVERAGE_FAIL_UNDER_RUST) bash scripts/check-coverage-rust.sh +check-coverage-rust: ## Validate core Rust (≥85%) and each native adapter (≥80%) + @test -f build/coverage-rust/ledger.json || \ + (echo "❌ Missing build/coverage-rust/ledger.json — run make coverage-rust first"; exit 1) + @COVERAGE_FAIL_UNDER_RUST=$(COVERAGE_FAIL_UNDER_RUST) \ + COVERAGE_FAIL_UNDER_RUST_PYTHON_ADAPTER=$(COVERAGE_FAIL_UNDER_RUST_PYTHON_ADAPTER) \ + COVERAGE_FAIL_UNDER_RUST_NODE_ADAPTER=$(COVERAGE_FAIL_UNDER_RUST_NODE_ADAPTER) \ + bash scripts/check-coverage-rust.sh coverage-strict: ## Strict 90% coverage check for new features @echo "Checking strict coverage (90%)..." @@ -205,6 +210,7 @@ pre-push-fast: ## Run fast checks only — format, lint, type, security, docstr pre-push: ## Run local policy checks plus multi-surface coverage thresholds @$(MAKE) pre-push-fast + @uv run --no-sync python scripts/ci/test-rust-coverage-ledger.py @echo "━━━ Coverage + thresholds (Rust + Python + Node) ━━━━━━━━━━━━━━━━━━━━━━━" @$(MAKE) coverage @echo "━━━ Public API BDD mutation sentinels ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @@ -242,8 +248,11 @@ cargo-test: ## Run all Rust workspace tests # cargo install cargo-llvm-cov # rustup component add llvm-tools-preview # Prefer an isolated CARGO_TARGET_DIR when other builds are running (AGENTS.md). -coverage-rust: ## Workspace Rust coverage via cargo-llvm-cov (term + lcov + html under build/coverage-rust) - @COVERAGE_FAIL_UNDER_RUST=$(COVERAGE_FAIL_UNDER_RUST) bash scripts/coverage-rust.sh +coverage-rust: ## Core + same-SHA Python/Node adapter Rust coverage ledger + @COVERAGE_FAIL_UNDER_RUST=$(COVERAGE_FAIL_UNDER_RUST) \ + COVERAGE_FAIL_UNDER_RUST_PYTHON_ADAPTER=$(COVERAGE_FAIL_UNDER_RUST_PYTHON_ADAPTER) \ + COVERAGE_FAIL_UNDER_RUST_NODE_ADAPTER=$(COVERAGE_FAIL_UNDER_RUST_NODE_ADAPTER) \ + bash scripts/coverage-rust.sh @$(MAKE) check-coverage-rust bench-traversal: ## Run the #767 traversal scaling benchmark (release, manual; see benchmarks/traversal_scaling.md) diff --git a/crates/graphforge-bindings-node/tests/composite_transaction.test.mjs b/crates/graphforge-bindings-node/tests/composite_transaction.test.mjs index 7c7b4b9af..f312c7342 100644 --- a/crates/graphforge-bindings-node/tests/composite_transaction.test.mjs +++ b/crates/graphforge-bindings-node/tests/composite_transaction.test.mjs @@ -264,3 +264,136 @@ test("Node exposes one composite publish entrypoint with no inference helpers", assert.equal(typeof forge.publishCompositeKnowledgeTransaction, "undefined"); assert.equal(typeof forge.inferCompositeParticipants, "undefined"); }); + +test("every explicit composite participant family crosses the Node adapter", async () => { + const project = mkdtempSync( + join(tmpdir(), "gf-composite-participants-node-"), + ); + try { + const forge = new GraphForge(project); + await enableCapabilities(forge); + const operationUuid = "018f0f4e-7b8c-7000-8000-00000000e001"; + const provenanceUuid = "018f0f4e-7b8c-7000-8000-00000000e002"; + const assertionUuid = "018f0f4e-7b8c-7000-8000-00000000e003"; + const confidenceUuid = "018f0f4e-7b8c-7000-8000-00000000e004"; + const reasoningUuid = "018f0f4e-7b8c-7000-8000-00000000e005"; + const groupUuid = "018f0f4e-7b8c-7000-8000-00000000e006"; + const before = projectDigest(project); + assert.throws( + () => + forge.publishCompositeTransaction({ + contractVersion: 1, + operationUuid, + graphMutations: [ + { + kind: "create_node", + nodeUuid: "018f0f4e-7b8c-7000-8000-00000000e010", + label: "Person", + properties: { name: "Grace" }, + }, + ], + knowledge: { + confidenceAssessments: [ + { + confidenceUuid, + assertionUuid, + policy: "conservative_min", + value: 0.5, + provenanceUuid, + recordedAtMicros: RECORDED_AT, + }, + ], + confidenceInputs: [ + { + confidenceUuid, + inputConfidenceUuid: "018f0f4e-7b8c-7000-8000-00000000e007", + inputValue: 0.5, + ordinal: 0, + }, + ], + evidence: [ + { + evidenceUuid: "018f0f4e-7b8c-7000-8000-00000000e008", + assertionUuid, + sourceUuid: "018f0f4e-7b8c-7000-8000-00000000e009", + sourceKind: "document", + role: "supports", + weight: 0.8, + provenanceUuid, + recordedAtMicros: RECORDED_AT, + }, + ], + reasoning: [ + { + reasoningUuid, + assertionUuid, + kind: "methodological_note", + contentFormat: "text/plain", + content: Buffer.from("explicit participant conversion"), + provenanceUuid, + recordedAtMicros: RECORDED_AT, + }, + ], + assertionSupersessions: [ + { + supersessionUuid: "018f0f4e-7b8c-7000-8000-00000000e00a", + priorAssertionUuid: assertionUuid, + replacementAssertionUuid: + "018f0f4e-7b8c-7000-8000-00000000e00b", + statusEventUuid: "018f0f4e-7b8c-7000-8000-00000000e00c", + reasoningUuid, + provenanceUuid, + recordedAtMicros: RECORDED_AT, + }, + ], + hypothesisGroups: [ + { + groupUuid, + questionKey: "which hypothesis", + provenanceUuid, + recordedAtMicros: RECORDED_AT, + }, + ], + hypothesisMembership: [ + { + membershipEventUuid: "018f0f4e-7b8c-7000-8000-00000000e00d", + operationUuid, + groupUuid, + assertionUuid, + action: "added", + reasoningUuid, + provenanceUuid, + recordedAtMicros: RECORDED_AT, + }, + ], + hypothesisSelection: [ + { + selectionEventUuid: "018f0f4e-7b8c-7000-8000-00000000e00e", + operationUuid, + groupUuid, + selectedAssertionUuid: assertionUuid, + reasoningUuid, + provenanceUuid, + recordedAtMicros: RECORDED_AT, + }, + ], + assertionValidity: [ + { + validityEventUuid: "018f0f4e-7b8c-7000-8000-00000000e00f", + assertionUuid, + validFromMicros: 1, + validToMicros: 2, + reasoningUuid, + provenanceUuid, + recordedAtMicros: RECORDED_AT, + }, + ], + }, + }), + (error) => error?.code === "GF_NOT_FOUND", + ); + assert.equal(projectDigest(project), before); + } finally { + rmSync(project, { recursive: true, force: true }); + } +}); diff --git a/crates/graphforge-bindings-py/tests/composite_transaction.py b/crates/graphforge-bindings-py/tests/composite_transaction.py index 54f40c755..390be22b1 100644 --- a/crates/graphforge-bindings-py/tests/composite_transaction.py +++ b/crates/graphforge-bindings-py/tests/composite_transaction.py @@ -247,6 +247,17 @@ def check_composite_transaction(project: Path) -> None: ) assert project_digest(project) == before + try: + forge.publish_composite_transaction( + operation_uuid="018f0f4e-7b8c-7000-8000-00000000e020", + graph_mutations=[{"kind": "unknown_mutation"}], + ) + except TypeError as exc: + assert "unknown composite graph mutation kind" in str(exc) + else: + raise SystemExit("expected an unknown composite mutation TypeError") + assert project_digest(project) == before + # Invalid identity (nil operation UUID) stays in Rust validation. expect_code( "GF_VALIDATION", @@ -358,10 +369,140 @@ def check_strict_ontology_composite(project: Path) -> None: assert project_digest(project) == before +def check_all_explicit_participant_conversions(project: Path) -> None: + """Every documented participant family crosses the thin adapter.""" + project.mkdir() + forge = g.GraphForge(str(project)) + enable_capabilities(forge) + operation = "018f0f4e-7b8c-7000-8000-00000000e001" + provenance = "018f0f4e-7b8c-7000-8000-00000000e002" + assertion = "018f0f4e-7b8c-7000-8000-00000000e003" + confidence = "018f0f4e-7b8c-7000-8000-00000000e004" + reasoning = "018f0f4e-7b8c-7000-8000-00000000e005" + group = "018f0f4e-7b8c-7000-8000-00000000e006" + before = project_digest(project) + + # These rows are structurally valid at the language boundary but deliberately + # reference an absent provenance event. Rust must reject the entire request + # without publishing the graph mutation. + expect_code( + "GF_NOT_FOUND", + lambda: forge.publish_composite_transaction( + operation_uuid=operation, + graph_mutations=[ + { + "kind": "create_node", + "node_uuid": "018f0f4e-7b8c-7000-8000-00000000e010", + "label": "Person", + "properties": {"name": "Grace"}, + } + ], + knowledge={ + "confidence_assessments": [ + { + "confidence_uuid": confidence, + "assertion_uuid": assertion, + "policy": "conservative_min", + "value": 0.5, + "provenance_uuid": provenance, + "recorded_at_micros": RECORDED_AT, + } + ], + "confidence_inputs": [ + { + "confidence_uuid": confidence, + "input_confidence_uuid": "018f0f4e-7b8c-7000-8000-00000000e007", + "input_value": 0.5, + "ordinal": 0, + } + ], + "evidence": [ + { + "evidence_uuid": "018f0f4e-7b8c-7000-8000-00000000e008", + "assertion_uuid": assertion, + "source_uuid": "018f0f4e-7b8c-7000-8000-00000000e009", + "source_kind": "document", + "role": "supports", + "weight": 0.8, + "provenance_uuid": provenance, + "recorded_at_micros": RECORDED_AT, + } + ], + "reasoning": [ + { + "reasoning_uuid": reasoning, + "assertion_uuid": assertion, + "kind": "methodological_note", + "content_format": "text/plain", + "content": b"explicit participant conversion", + "provenance_uuid": provenance, + "recorded_at_micros": RECORDED_AT, + } + ], + "assertion_supersessions": [ + { + "supersession_uuid": "018f0f4e-7b8c-7000-8000-00000000e00a", + "prior_assertion_uuid": assertion, + "replacement_assertion_uuid": "018f0f4e-7b8c-7000-8000-00000000e00b", + "status_event_uuid": "018f0f4e-7b8c-7000-8000-00000000e00c", + "reasoning_uuid": reasoning, + "provenance_uuid": provenance, + "recorded_at_micros": RECORDED_AT, + } + ], + "hypothesis_groups": [ + { + "group_uuid": group, + "question_key": "which hypothesis", + "provenance_uuid": provenance, + "recorded_at_micros": RECORDED_AT, + } + ], + "hypothesis_membership": [ + { + "membership_event_uuid": "018f0f4e-7b8c-7000-8000-00000000e00d", + "operation_uuid": operation, + "group_uuid": group, + "assertion_uuid": assertion, + "action": "added", + "reasoning_uuid": reasoning, + "provenance_uuid": provenance, + "recorded_at_micros": RECORDED_AT, + } + ], + "hypothesis_selection": [ + { + "selection_event_uuid": "018f0f4e-7b8c-7000-8000-00000000e00e", + "operation_uuid": operation, + "group_uuid": group, + "selected_assertion_uuid": assertion, + "reasoning_uuid": reasoning, + "provenance_uuid": provenance, + "recorded_at_micros": RECORDED_AT, + } + ], + "assertion_validity": [ + { + "validity_event_uuid": "018f0f4e-7b8c-7000-8000-00000000e00f", + "assertion_uuid": assertion, + "valid_from": 1, + "valid_to": 2, + "reasoning_uuid": reasoning, + "provenance_uuid": provenance, + "recorded_at_micros": RECORDED_AT, + } + ], + }, + ), + ) + assert project_digest(project) == before + + if __name__ == "__main__": check_no_inference_helpers() with tempfile.TemporaryDirectory(prefix="gf-composite-py-") as directory: check_composite_transaction(Path(directory)) + check_all_explicit_participant_conversions(Path(directory) / "participants") onto_root = Path(directory) / "strict" onto_root.mkdir() project = onto_root / "project" diff --git a/docs/development/testing.md b/docs/development/testing.md index 7b455e0f9..1511e8315 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -197,9 +197,9 @@ make test-unit make test-integration make test-tck -# With coverage (Rust + Python + Node; ≥85% lines per surface) +# With coverage (core Rust ≥85%; Rust adapters ≥80%; wrappers ≥85%) make coverage # all surfaces + thresholds -make coverage-rust # cargo llvm-cov → build/coverage-rust/ +make coverage-rust # core + same-SHA native adapter ledger make coverage-python # pytest-cov on graphforge-bindings-py/python/graphforge make coverage-node # c8 on @curatelabs/graphforge lib/ (needs *.node) make coverage-report # open Python HTML report @@ -229,17 +229,31 @@ def tmp_db(tmp_path): ### Coverage Requirements -Local `make coverage` instruments three surfaces independently (pytest does **not** -cover Rust; Codecov CI is separate/out of scope for this gate): +Local `make coverage` instruments the shipped surfaces independently. The Rust +run builds PyO3 and napi-rs once with LLVM instrumentation and executes the real +Python and Node native acceptance suites against those exact artifacts. The +command defaults to an isolated target under `build/coverage-rust/`; set +`CARGO_TARGET_DIR` to choose a different isolated location. | Surface | Tool | Scope | Threshold | |---------|------|-------|-----------| -| Rust | `cargo llvm-cov` | workspace crates via `cargo test --workspace` | ≥85% lines | +| Core Rust | `cargo llvm-cov` | workspace Rust excluding binding adapters | ≥85% lines | +| Python adapter Rust | `cargo llvm-cov` + native acceptance | `graphforge-bindings-py` Rust | ≥80% lines | +| Node adapter Rust | `cargo llvm-cov` + native acceptance | `graphforge-bindings-node` Rust | ≥80% lines | | Python | `pytest-cov` | `crates/graphforge-bindings-py/python/graphforge` | ≥85% lines; ≥90% patch on changed wrapper files | | Node | `c8` | hand-written `@curatelabs/graphforge` `lib/**/*.mjs` | ≥85% lines | -Override floors with `COVERAGE_FAIL_UNDER_RUST`, `COVERAGE_FAIL_UNDER_PYTHON`, -and `COVERAGE_FAIL_UNDER_NODE`. +`build/coverage-rust/ledger.json` is the machine-readable source of truth. It +records the exact source SHA, toolchain, artifact hashes, profile counts, and +separate core, adapter, and merged-workspace totals. `summary.txt` is the human +view and `lcov.info` is the merged report. A high workspace average never +substitutes for a missing profile or a failed adapter floor; stale, empty, +wrong-artifact, and wrong-SHA evidence fails closed. + +Override floors with `COVERAGE_FAIL_UNDER_RUST`, +`COVERAGE_FAIL_UNDER_RUST_PYTHON_ADAPTER`, +`COVERAGE_FAIL_UNDER_RUST_NODE_ADAPTER`, `COVERAGE_FAIL_UNDER_PYTHON`, and +`COVERAGE_FAIL_UNDER_NODE`. ### Required Checks (all PRs) diff --git a/docs/engineering/TESTING.md b/docs/engineering/TESTING.md index 5943cd5f7..fdf330c75 100644 --- a/docs/engineering/TESTING.md +++ b/docs/engineering/TESTING.md @@ -165,6 +165,24 @@ weakened assertions (`AGENTS.md`). ## Behavior coverage +### Rust coverage evidence + +`make coverage-rust` measures four explicit totals: core Rust, Python adapter +Rust, Node adapter Rust, and their merged workspace. The adapter totals come +from the functional native acceptance suites—not placeholder binding tests—and +therefore include persistence/reopen, structured lifecycle errors, parity, and +no-fallback behavior executed through the instrumented PyO3 and napi-rs +artifacts. + +The run uses an isolated `CARGO_TARGET_DIR` (defaulting under its output tree), +builds each native artifact once, and verifies that the loaded artifact hash +matches the measured object. +`build/coverage-rust/ledger.json` also binds the evidence to `HEAD` and the LLVM +toolchain. Missing, empty, stale, wrong-artifact, or wrong-SHA adapter profiles +fail before totals are accepted. Core has an 85% floor and each Rust binding +adapter has an independent 80% floor; the merged workspace percentage cannot +average away a failed surface. + | Experience / Requirement | Scenario (Given/When/Then) | Test / evidence | | ------------------------ | -------------------------- | ---- | | FR-1 Cypher → Arrow | Given a graph, when `execute` runs, then Arrow rows match | Workspace/`graphforge-api` query tests; TCK corpus | diff --git a/scripts/check-coverage-rust.sh b/scripts/check-coverage-rust.sh index c7e891636..b4fc55682 100755 --- a/scripts/check-coverage-rust.sh +++ b/scripts/check-coverage-rust.sh @@ -1,42 +1,13 @@ #!/usr/bin/env bash -# Parse cargo-llvm-cov term summary and enforce a line-coverage floor. +# Revalidate the same-SHA per-surface Rust coverage ledger. set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -SUMMARY="${COVERAGE_RUST_SUMMARY:-$ROOT/build/coverage-rust/summary.txt}" -MIN="${COVERAGE_FAIL_UNDER_RUST:-85}" - -if [[ ! -f "${SUMMARY}" ]]; then - echo "❌ Missing Rust coverage summary at ${SUMMARY}" - exit 1 -fi - -# Match the TOTAL row from llvm-cov's text table, e.g.: -# TOTAL 1234 100 200 50 800 100 85.00% -line_pct="$( - awk ' - /^TOTAL/ { - for (i = NF; i >= 1; i--) { - if ($i ~ /%$/) { - gsub(/%/, "", $i) - print $i - exit - } - } - } - ' "${SUMMARY}" -)" - -if [[ -z "${line_pct}" ]]; then - echo "❌ Could not parse TOTAL line coverage from ${SUMMARY}" - exit 1 -fi - -echo "Rust lines: ${line_pct}%" -awk -v pct="${line_pct}" -v min="${MIN}" 'BEGIN { - if (pct + 0 < min + 0) { - printf("❌ Rust coverage below %s%% (got %s%%)\n", min, pct) - exit 1 - } - printf("✅ Rust coverage meets %s%% threshold\n", min) -}' +LEDGER="${COVERAGE_RUST_LEDGER:-$ROOT/build/coverage-rust/ledger.json}" + +python3 "$ROOT/scripts/coverage_rust_ledger.py" \ + --root "$ROOT" \ + --ledger "$LEDGER" \ + --core-floor "${COVERAGE_FAIL_UNDER_RUST:-85}" \ + --python-floor "${COVERAGE_FAIL_UNDER_RUST_PYTHON_ADAPTER:-80}" \ + --node-floor "${COVERAGE_FAIL_UNDER_RUST_NODE_ADAPTER:-80}" diff --git a/scripts/ci/test-rust-coverage-ledger.py b/scripts/ci/test-rust-coverage-ledger.py new file mode 100755 index 000000000..6ca8595a0 --- /dev/null +++ b/scripts/ci/test-rust-coverage-ledger.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Deterministic mutation sentinels for the Rust coverage ledger.""" + +from __future__ import annotations + +import importlib.util +import json +import os +from pathlib import Path +import tempfile +import time +from types import SimpleNamespace + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location( + "coverage_rust_ledger", ROOT / "scripts" / "coverage_rust_ledger.py" +) +assert SPEC is not None and SPEC.loader is not None +ledger_module = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(ledger_module) + + +def expect_error(fragment: str, call) -> None: + try: + call() + except ledger_module.LedgerError as error: + assert fragment in str(error), (fragment, str(error)) + else: + raise AssertionError(f"expected LedgerError containing {fragment!r}") + + +def main() -> None: + head = ledger_module.git_head(ROOT) + with tempfile.TemporaryDirectory(prefix="gf-rust-coverage-ledger-") as directory: + temp = Path(directory) + artifact = temp / "adapter.so" + runtime = temp / "loaded.so" + profile = temp / "adapter.profraw" + artifact.write_bytes(b"same instrumented artifact") + runtime.write_bytes(artifact.read_bytes()) + time.sleep(0.01) + profile.write_bytes(b"non-empty profile") + artifact_hash = ledger_module.sha256(artifact) + evidence = { + "source_sha": head, + "rustc": "rustc test", + "cargo_llvm_cov": "cargo-llvm-cov test", + "surfaces": { + name: { + "artifact": str(artifact), + "runtime_artifact": str(runtime), + "artifact_sha256": artifact_hash, + "profiles": [str(profile)], + } + for name in ("python_adapter", "node_adapter") + }, + } + + ledger_module.validate_evidence(evidence, ROOT, head) + + mutated = json.loads(json.dumps(evidence)) + mutated["source_sha"] = "0" * 40 + expect_error("source SHA", lambda: ledger_module.validate_evidence(mutated, ROOT, head)) + + mutated = json.loads(json.dumps(evidence)) + mutated["surfaces"].pop("node_adapter") + expect_error( + "missing node_adapter", lambda: ledger_module.validate_evidence(mutated, ROOT, head) + ) + + empty = temp / "empty.profraw" + empty.touch() + mutated = json.loads(json.dumps(evidence)) + mutated["surfaces"]["python_adapter"]["profiles"] = [str(empty)] + expect_error( + "missing or empty", lambda: ledger_module.validate_evidence(mutated, ROOT, head) + ) + + stale = temp / "stale.profraw" + stale.write_bytes(b"old profile") + old = artifact.stat().st_mtime - 10 + os.utime(stale, (old, old)) + mutated = json.loads(json.dumps(evidence)) + mutated["surfaces"]["node_adapter"]["profiles"] = [str(stale)] + expect_error("predates", lambda: ledger_module.validate_evidence(mutated, ROOT, head)) + + mutated = json.loads(json.dumps(evidence)) + mutated["surfaces"]["python_adapter"]["artifact_sha256"] = "bad" + expect_error("artifact hash", lambda: ledger_module.validate_evidence(mutated, ROOT, head)) + + lcov = temp / "valid.lcov" + lcov.write_text( + f"SF:{ROOT / 'crates/graphforge-api/src/lib.rs'}\nDA:1,1\nend_of_record\n", + encoding="utf-8", + ) + assert ledger_module.parse_lcov(lcov, ROOT) + malformed = temp / "malformed.lcov" + malformed.write_text("SF:no-lines.rs\nend_of_record\n", encoding="utf-8") + expect_error("no executable lines", lambda: ledger_module.parse_lcov(malformed, ROOT)) + + args = SimpleNamespace( + root=ROOT, + core_floor=85.0, + python_floor=80.0, + node_floor=80.0, + ) + valid_ledger = { + "schema_version": 1, + "source_sha": head, + "surfaces": { + name: {"measured_lines": 100, "line_percent": 90.0} + for name in ("core", "python_adapter", "node_adapter") + }, + } + ledger_module.validate_floors(valid_ledger, args) + + mutated = json.loads(json.dumps(valid_ledger)) + mutated["surfaces"]["python_adapter"]["line_percent"] = 79.99 + expect_error( + "python_adapter Rust coverage below", + lambda: ledger_module.validate_floors(mutated, args), + ) + + mutated = json.loads(json.dumps(valid_ledger)) + mutated["surfaces"]["node_adapter"]["measured_lines"] = 0 + expect_error("malformed node_adapter", lambda: ledger_module.validate_floors(mutated, args)) + + mutated = json.loads(json.dumps(valid_ledger)) + mutated["schema_version"] = 99 + expect_error("unsupported or missing", lambda: ledger_module.validate_floors(mutated, args)) + + print("rust coverage ledger mutation sentinels: ok") + + +if __name__ == "__main__": + main() diff --git a/scripts/coverage-rust.sh b/scripts/coverage-rust.sh index 214a0b736..115c7d343 100755 --- a/scripts/coverage-rust.sh +++ b/scripts/coverage-rust.sh @@ -1,98 +1,169 @@ #!/usr/bin/env bash -# Run workspace Rust tests under cargo-llvm-cov and emit term + lcov + HTML. -# Prerequisites: -# cargo install cargo-llvm-cov -# rustup component add llvm-tools-preview -# -# Outputs (gitignored via build/): -# build/coverage-rust/lcov.info -# build/coverage-rust/html/index.html -# build/coverage-rust/summary.txt +# Measure core Rust plus the Rust code executed by real Python and Node acceptance. set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT" OUT_DIR="${COVERAGE_RUST_DIR:-build/coverage-rust}" -LCOV_PATH="${OUT_DIR}/lcov.info" -# cargo-llvm-cov appends /html under --output-dir -HTML_DIR="${OUT_DIR}/html" -SUMMARY_PATH="${OUT_DIR}/summary.txt" -# Default: whole workspace. Override for focused runs, e.g. COVERAGE_RUST_ARGS='-p graphforge-api' +mkdir -p "$OUT_DIR" +OUT_DIR="$(cd "$OUT_DIR" && pwd)" +PROFILE_DIR="$OUT_DIR/profiles" +CORE_LCOV="$OUT_DIR/core.lcov.info" +PYTHON_LCOV="$OUT_DIR/python-adapter.lcov.info" +NODE_LCOV="$OUT_DIR/node-adapter.lcov.info" +WORKSPACE_LCOV="$OUT_DIR/lcov.info" +LEDGER="$OUT_DIR/ledger.json" +EVIDENCE="$OUT_DIR/evidence.json" +SUMMARY="$OUT_DIR/summary.txt" COVERAGE_RUST_ARGS="${COVERAGE_RUST_ARGS:---workspace}" +PYTHON_FLOOR="${COVERAGE_FAIL_UNDER_RUST_PYTHON_ADAPTER:-80}" +NODE_FLOOR="${COVERAGE_FAIL_UNDER_RUST_NODE_ADAPTER:-80}" +CORE_FLOOR="${COVERAGE_FAIL_UNDER_RUST:-85}" -if ! command -v cargo-llvm-cov >/dev/null 2>&1 && ! cargo llvm-cov --version >/dev/null 2>&1; then - echo "❌ cargo-llvm-cov not found." - echo " Install with: cargo install cargo-llvm-cov" - echo " Also require: rustup component add llvm-tools-preview" +if ! cargo llvm-cov --version >/dev/null 2>&1; then + echo "Rust coverage evidence error: cargo-llvm-cov is not installed" >&2 exit 1 fi - if ! rustup component list --installed 2>/dev/null | grep -Eq '^llvm-tools'; then - echo "❌ rustup llvm-tools component not installed." - echo " Install with: rustup component add llvm-tools-preview" + echo "Rust coverage evidence error: rustup llvm-tools is not installed" >&2 exit 1 fi +export CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-$OUT_DIR/native-target}" +CORE_TARGET_DIR="$CARGO_TARGET_DIR/llvm-cov-target" -mkdir -p "${OUT_DIR}" -rm -rf "${HTML_DIR}" "${LCOV_PATH}" "${SUMMARY_PATH}" - -echo "━━━ Rust coverage (cargo llvm-cov ${COVERAGE_RUST_ARGS}) ━━━━━━━━━━━━━━━" -echo " CARGO_TARGET_DIR=${CARGO_TARGET_DIR:-}" -echo " output: ${OUT_DIR}/" -echo +rm -rf "$PROFILE_DIR" "$OUT_DIR/core-html" +rm -f "$CORE_LCOV" "$PYTHON_LCOV" "$NODE_LCOV" "$WORKSPACE_LCOV" \ + "$LEDGER" "$EVIDENCE" "$SUMMARY" +mkdir -p "$PROFILE_DIR" +echo "━━━ Core Rust coverage ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # shellcheck disable=SC2086 cargo llvm-cov clean ${COVERAGE_RUST_ARGS} - -# Instrument + run once; generate reports from the collected data (lcov/html are mutually exclusive flags). # shellcheck disable=SC2086 cargo llvm-cov ${COVERAGE_RUST_ARGS} --no-report - -echo -echo "━━━ Generating reports ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -cargo llvm-cov report 2>&1 | tee "${SUMMARY_PATH}" -cargo llvm-cov report --lcov --output-path "${LCOV_PATH}" -# --output-dir receives the parent; llvm-cov writes /html/index.html -cargo llvm-cov report --html --output-dir "${OUT_DIR}" - -if [[ ! -s "${LCOV_PATH}" ]]; then - echo "❌ Rust coverage produced empty or missing lcov at ${LCOV_PATH}" - exit 1 -fi - -if ! grep -q '^SF:' "${LCOV_PATH}"; then - echo "❌ Rust coverage lcov has no source file records (SF:) at ${LCOV_PATH}" +cargo llvm-cov report --lcov --output-path "$CORE_LCOV" +cargo llvm-cov report --html --output-dir "$OUT_DIR/core-html" + +# Export the exact instrumentation environment used by cargo-llvm-cov. Native +# artifacts are built once, sequentially, and then reused by all acceptance. +eval "$(cargo llvm-cov show-env --sh)" + +echo "━━━ Instrumented native artifacts ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +uv run maturin develop --release -m crates/graphforge-bindings-py/Cargo.toml +pnpm --filter @curatelabs/graphforge exec napi build --platform --release + +PYTHON_OBJECT="$(find "$CARGO_TARGET_DIR" -type f \( -name 'libgraphforge_bindings_py.so' -o -name 'libgraphforge_bindings_py.dylib' -o -name 'graphforge_bindings_py.dll' \) | head -1)" +NODE_OBJECT="$(find "$CARGO_TARGET_DIR" -type f \( -name 'libgraphforge_bindings_node.so' -o -name 'libgraphforge_bindings_node.dylib' -o -name 'graphforge_bindings_node.dll' \) | head -1)" +PYTHON_RUNTIME="$(uv run --no-sync python -c 'from graphforge import _graphforge_rs; print(_graphforge_rs.__file__)')" +NODE_RUNTIME="$(find crates/graphforge-bindings-node -maxdepth 1 -type f -name 'graphforge.*.node' | head -1)" +for artifact in "$PYTHON_OBJECT" "$NODE_OBJECT" "$PYTHON_RUNTIME" "$NODE_RUNTIME"; do + if [[ -z "$artifact" || ! -s "$artifact" ]]; then + echo "Rust coverage evidence error: expected instrumented native artifact is missing" >&2 + exit 1 + fi +done + +hash_file() { + python3 -c 'import hashlib, pathlib, sys; print(hashlib.sha256(pathlib.Path(sys.argv[1]).read_bytes()).hexdigest())' "$1" +} +python_hash="$(hash_file "$PYTHON_OBJECT")" +node_hash="$(hash_file "$NODE_OBJECT")" +if [[ "$(hash_file "$PYTHON_RUNTIME")" != "$python_hash" ]]; then + echo "Rust coverage evidence error: Python did not import the measured artifact" >&2 exit 1 fi - -if [[ ! -f "${HTML_DIR}/index.html" ]]; then - echo "❌ Rust coverage HTML report missing at ${HTML_DIR}/index.html" - exit 1 -fi - -if [[ ! -s "${SUMMARY_PATH}" ]]; then - echo "❌ Rust coverage term summary is empty at ${SUMMARY_PATH}" +if [[ "$(hash_file "$NODE_RUNTIME")" != "$node_hash" ]]; then + echo "Rust coverage evidence error: Node did not load the measured artifact" >&2 exit 1 fi -if ! grep -q '^TOTAL' "${SUMMARY_PATH}"; then - echo "❌ Rust coverage term summary has no TOTAL row at ${SUMMARY_PATH}" - exit 1 -fi - -lines=$(grep -c '^SF:' "${LCOV_PATH}" || true) -bytes=$(wc -c < "${LCOV_PATH}" | tr -d ' ') -echo -echo "✅ Rust coverage report ready" -echo " lcov: ${LCOV_PATH} (${bytes} bytes, ${lines} source files)" -echo " html: ${HTML_DIR}/index.html" -echo " summary: ${SUMMARY_PATH}" - -# Optional fail-under (coordinator aggregation). Empty/0 skips the check here; -# `make check-coverage-rust` always enforces COVERAGE_FAIL_UNDER_RUST. -if [[ -n "${COVERAGE_FAIL_UNDER_RUST:-}" && "${COVERAGE_FAIL_UNDER_RUST}" != "0" ]]; then - COVERAGE_RUST_SUMMARY="${SUMMARY_PATH}" \ - COVERAGE_FAIL_UNDER_RUST="${COVERAGE_FAIL_UNDER_RUST}" \ - bash "${ROOT}/scripts/check-coverage-rust.sh" -fi +echo "━━━ Python native acceptance ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +export LLVM_PROFILE_FILE="$PROFILE_DIR/python-%p-%10m.profraw" +for test_file in crates/graphforge-bindings-py/tests/*.py; do + CARGO_TARGET_DIR="$CORE_TARGET_DIR" uv run --no-sync python "$test_file" +done +uv run --no-sync pytest tests/unit tests/integration tests/features \ + --ignore=tests/unit/test_publish_dry_run.py \ + -n "${PYTEST_WORKERS:-4}" --tb=short + +echo "━━━ Node native acceptance ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +export LLVM_PROFILE_FILE="$PROFILE_DIR/node-%p-%10m.profraw" +pnpm --filter @curatelabs/graphforge test:smoke +pnpm --filter @curatelabs/graphforge test +( + cd tests/features/node + node node_modules/@cucumber/cucumber/bin/cucumber.js \ + "../api/**/*.feature" \ + --require-module ts-node/register \ + --require "step_definitions/**/*.ts" \ + --tags "not @excluded-api-bdd and not @excluded-node-api-bdd" \ + --format summary +) + +TOOLS_DIR="$(rustc --print sysroot)/lib/rustlib/$(rustc -vV | sed -n 's/^host: //p')/bin" +"$TOOLS_DIR/llvm-profdata" merge -sparse "$PROFILE_DIR"/python-*.profraw -o "$OUT_DIR/python.profdata" +"$TOOLS_DIR/llvm-profdata" merge -sparse "$PROFILE_DIR"/node-*.profraw -o "$OUT_DIR/node.profdata" +"$TOOLS_DIR/llvm-cov" export "$PYTHON_OBJECT" \ + -instr-profile="$OUT_DIR/python.profdata" -format=lcov > "$PYTHON_LCOV" +"$TOOLS_DIR/llvm-cov" export "$NODE_OBJECT" \ + -instr-profile="$OUT_DIR/node.profdata" -format=lcov > "$NODE_LCOV" + +export GF_COVERAGE_ROOT="$ROOT" +export GF_COVERAGE_SOURCE_SHA="$(git rev-parse HEAD)" +export GF_COVERAGE_RUSTC="$(rustc --version)" +export GF_COVERAGE_LLVM_COV="$(cargo llvm-cov --version)" +export GF_COVERAGE_PYTHON_OBJECT="$PYTHON_OBJECT" +export GF_COVERAGE_PYTHON_RUNTIME="$PYTHON_RUNTIME" +export GF_COVERAGE_PYTHON_HASH="$python_hash" +export GF_COVERAGE_NODE_OBJECT="$NODE_OBJECT" +export GF_COVERAGE_NODE_RUNTIME="$NODE_RUNTIME" +export GF_COVERAGE_NODE_HASH="$node_hash" +export GF_COVERAGE_PROFILE_DIR="$PROFILE_DIR" +export GF_COVERAGE_EVIDENCE="$EVIDENCE" +python3 - <<'PY' +import glob +import json +import os +from pathlib import Path + +profiles = Path(os.environ["GF_COVERAGE_PROFILE_DIR"]) +data = { + "source_sha": os.environ["GF_COVERAGE_SOURCE_SHA"], + "rustc": os.environ["GF_COVERAGE_RUSTC"], + "cargo_llvm_cov": os.environ["GF_COVERAGE_LLVM_COV"], + "surfaces": { + "python_adapter": { + "artifact": os.environ["GF_COVERAGE_PYTHON_OBJECT"], + "runtime_artifact": os.environ["GF_COVERAGE_PYTHON_RUNTIME"], + "artifact_sha256": os.environ["GF_COVERAGE_PYTHON_HASH"], + "profiles": sorted(glob.glob(str(profiles / "python-*.profraw"))), + }, + "node_adapter": { + "artifact": os.environ["GF_COVERAGE_NODE_OBJECT"], + "runtime_artifact": os.environ["GF_COVERAGE_NODE_RUNTIME"], + "artifact_sha256": os.environ["GF_COVERAGE_NODE_HASH"], + "profiles": sorted(glob.glob(str(profiles / "node-*.profraw"))), + }, + }, +} +Path(os.environ["GF_COVERAGE_EVIDENCE"]).write_text( + json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8" +) +PY + +echo "━━━ Rust coverage ledger ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +python3 scripts/coverage_rust_ledger.py \ + --root "$ROOT" \ + --ledger "$LEDGER" \ + --build \ + --core-lcov "$CORE_LCOV" \ + --python-lcov "$PYTHON_LCOV" \ + --node-lcov "$NODE_LCOV" \ + --workspace-lcov "$WORKSPACE_LCOV" \ + --evidence "$EVIDENCE" \ + --core-floor "$CORE_FLOOR" \ + --python-floor "$PYTHON_FLOOR" \ + --node-floor "$NODE_FLOOR" | tee "$SUMMARY" + +echo "✅ Rust coverage ledger ready: $LEDGER" diff --git a/scripts/coverage_rust_ledger.py b/scripts/coverage_rust_ledger.py new file mode 100755 index 000000000..9a074000f --- /dev/null +++ b/scripts/coverage_rust_ledger.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +"""Build and validate GraphForge's Rust per-surface coverage ledger.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +import sys +from typing import Any + +BINDING_PATHS = { + "python_adapter": "crates/graphforge-bindings-py/", + "node_adapter": "crates/graphforge-bindings-node/", +} + + +class LedgerError(RuntimeError): + """Coverage evidence is incomplete or inconsistent.""" + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def git_head(root: Path) -> str: + return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() + + +def normalize_source(raw: str, root: Path) -> str | None: + path = Path(raw) + try: + return path.resolve().relative_to(root.resolve()).as_posix() + except ValueError: + marker = "/crates/" + normalized = path.as_posix() + if marker in normalized: + return f"crates/{normalized.split(marker, 1)[1]}" + return None + + +def parse_lcov(path: Path, root: Path) -> dict[str, dict[int, int]]: + if not path.is_file() or path.stat().st_size == 0: + raise LedgerError(f"missing or empty lcov report: {path}") + records: dict[str, dict[int, int]] = {} + source: str | None = None + saw_source = False + for raw_line in path.read_text(encoding="utf-8").splitlines(): + if raw_line.startswith("SF:"): + saw_source = True + source = normalize_source(raw_line[3:], root) + if source is not None: + records.setdefault(source, {}) + elif raw_line.startswith("DA:"): + if not saw_source: + raise LedgerError(f"DA record precedes SF record in {path}") + if source is None: + continue + fields = raw_line[3:].split(",") + if len(fields) < 2: + raise LedgerError(f"malformed DA record in {path}: {raw_line}") + try: + line, hits = int(fields[0]), int(fields[1]) + except ValueError as error: + raise LedgerError(f"malformed DA record in {path}: {raw_line}") from error + records[source][line] = max(records[source].get(line, 0), hits) + if not records or not any(lines for lines in records.values()): + raise LedgerError(f"lcov report has no executable lines: {path}") + return records + + +def merge_records(*reports: dict[str, dict[int, int]]) -> dict[str, dict[int, int]]: + merged: dict[str, dict[int, int]] = {} + for report in reports: + for source, lines in report.items(): + destination = merged.setdefault(source, {}) + for line, hits in lines.items(): + destination[line] = max(destination.get(line, 0), hits) + return merged + + +def select_surface(records: dict[str, dict[int, int]], surface: str) -> dict[str, dict[int, int]]: + if surface == "core": + return { + path: lines + for path, lines in records.items() + if not any(path.startswith(prefix) for prefix in BINDING_PATHS.values()) + } + prefix = BINDING_PATHS[surface] + return {path: lines for path, lines in records.items() if path.startswith(prefix)} + + +def totals(records: dict[str, dict[int, int]]) -> dict[str, int | float]: + measured = sum(len(lines) for lines in records.values()) + covered = sum(sum(1 for hits in lines.values() if hits > 0) for lines in records.values()) + if measured == 0: + raise LedgerError("coverage surface has zero executable lines") + return { + "covered_lines": covered, + "measured_lines": measured, + "line_percent": round(covered * 100 / measured, 2), + "source_files": len(records), + } + + +def write_lcov(records: dict[str, dict[int, int]], path: Path, root: Path) -> None: + output: list[str] = [] + for source in sorted(records): + output.append(f"SF:{root / source}") + for line, hits in sorted(records[source].items()): + output.append(f"DA:{line},{hits}") + output.extend(("end_of_record", "")) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(output), encoding="utf-8") + + +def validate_evidence(evidence: dict[str, Any], root: Path, expected_sha: str) -> None: + if evidence.get("source_sha") != expected_sha: + raise LedgerError( + "coverage evidence source SHA does not match HEAD: " + f"expected {expected_sha}, got {evidence.get('source_sha')!r}" + ) + if not evidence.get("rustc") or not evidence.get("cargo_llvm_cov"): + raise LedgerError("coverage evidence is missing toolchain identity") + for surface in ("python_adapter", "node_adapter"): + item = evidence.get("surfaces", {}).get(surface) + if not isinstance(item, dict): + raise LedgerError(f"missing {surface} evidence") + artifact = root / str(item.get("artifact", "")) + runtime_artifact = root / str(item.get("runtime_artifact", "")) + expected_hash = item.get("artifact_sha256") + if not artifact.is_file() or not runtime_artifact.is_file(): + raise LedgerError(f"{surface} artifact evidence is missing") + actual_hash = sha256(artifact) + if not expected_hash or actual_hash != expected_hash: + raise LedgerError(f"{surface} artifact hash is stale or incorrect") + if sha256(runtime_artifact) != actual_hash: + raise LedgerError(f"{surface} tests did not load the measured artifact") + profiles = [root / str(value) for value in item.get("profiles", [])] + if not profiles: + raise LedgerError(f"{surface} contributed no runtime profiles") + for profile in profiles: + if not profile.is_file() or profile.stat().st_size == 0: + raise LedgerError(f"{surface} profile is missing or empty: {profile}") + if profile.stat().st_mtime_ns <= artifact.stat().st_mtime_ns: + raise LedgerError(f"{surface} profile predates its instrumented artifact") + + +def build_ledger(args: argparse.Namespace) -> dict[str, Any]: + root = args.root.resolve() + expected_sha = git_head(root) + evidence = json.loads(args.evidence.read_text(encoding="utf-8")) + validate_evidence(evidence, root, expected_sha) + + core_report = parse_lcov(args.core_lcov, root) + python_report = parse_lcov(args.python_lcov, root) + node_report = parse_lcov(args.node_lcov, root) + merged = merge_records(core_report, python_report, node_report) + if args.workspace_lcov is not None: + write_lcov(merged, args.workspace_lcov, root) + surfaces = { + "core": totals(select_surface(core_report, "core")), + "python_adapter": totals(select_surface(python_report, "python_adapter")), + "node_adapter": totals(select_surface(node_report, "node_adapter")), + "workspace": totals(merged), + } + return { + "schema_version": 1, + "source_sha": expected_sha, + "toolchain": { + "rustc": evidence["rustc"], + "cargo_llvm_cov": evidence["cargo_llvm_cov"], + }, + "artifacts": { + name: { + "sha256": evidence["surfaces"][name]["artifact_sha256"], + "profile_count": len(evidence["surfaces"][name]["profiles"]), + } + for name in ("python_adapter", "node_adapter") + }, + "surfaces": surfaces, + } + + +def validate_floors(ledger: dict[str, Any], args: argparse.Namespace) -> None: + expected_sha = git_head(args.root.resolve()) + if ledger.get("schema_version") != 1: + raise LedgerError("unsupported or missing Rust coverage ledger schema") + if ledger.get("source_sha") != expected_sha: + raise LedgerError("Rust coverage ledger is stale for the current HEAD") + floors = { + "core": args.core_floor, + "python_adapter": args.python_floor, + "node_adapter": args.node_floor, + } + for surface, floor in floors.items(): + data = ledger.get("surfaces", {}).get(surface) + if not isinstance(data, dict): + raise LedgerError(f"Rust coverage ledger is missing {surface}") + measured = data.get("measured_lines") + percent = data.get("line_percent") + if not isinstance(measured, int) or measured <= 0 or not isinstance(percent, (int, float)): + raise LedgerError(f"Rust coverage ledger has malformed {surface} totals") + if float(percent) < floor: + raise LedgerError( + f"{surface} Rust coverage below {floor:.2f}% (got {float(percent):.2f}%)" + ) + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser() + result.add_argument("--root", type=Path, required=True) + result.add_argument("--ledger", type=Path, required=True) + result.add_argument("--core-floor", type=float, default=85.0) + result.add_argument("--python-floor", type=float, default=80.0) + result.add_argument("--node-floor", type=float, default=80.0) + result.add_argument("--build", action="store_true") + result.add_argument("--core-lcov", type=Path) + result.add_argument("--python-lcov", type=Path) + result.add_argument("--node-lcov", type=Path) + result.add_argument("--evidence", type=Path) + result.add_argument("--workspace-lcov", type=Path) + return result + + +def main() -> int: + args = parser().parse_args() + try: + if args.build: + required = (args.core_lcov, args.python_lcov, args.node_lcov, args.evidence) + if any(value is None for value in required): + raise LedgerError("ledger build requires all lcov reports and evidence") + ledger = build_ledger(args) + args.ledger.parent.mkdir(parents=True, exist_ok=True) + args.ledger.write_text( + json.dumps(ledger, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + else: + if not args.ledger.is_file(): + raise LedgerError(f"missing Rust coverage ledger: {args.ledger}") + ledger = json.loads(args.ledger.read_text(encoding="utf-8")) + validate_floors(ledger, args) + except (LedgerError, json.JSONDecodeError) as error: + print(f"Rust coverage evidence error: {error}", file=sys.stderr) + return 1 + + for name in ("core", "python_adapter", "node_adapter", "workspace"): + data = ledger["surfaces"][name] + print( + f"{name}: {data['covered_lines']}/{data['measured_lines']} " + f"lines ({data['line_percent']:.2f}%)" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From e9cbd8b0dabeee7eda4da67edb25bc645075bc52 Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Mon, 3 Aug 2026 04:55:30 -0600 Subject: [PATCH 2/3] test(ci): harden native coverage evidence --- scripts/ci/test-rust-coverage-ledger.py | 27 +++++++++++---- scripts/coverage-rust.sh | 44 ++++++++++++++++++++----- scripts/coverage_rust_ledger.py | 28 ++++++++++------ 3 files changed, 73 insertions(+), 26 deletions(-) diff --git a/scripts/ci/test-rust-coverage-ledger.py b/scripts/ci/test-rust-coverage-ledger.py index 6ca8595a0..15845a5e6 100755 --- a/scripts/ci/test-rust-coverage-ledger.py +++ b/scripts/ci/test-rust-coverage-ledger.py @@ -8,14 +8,14 @@ import os from pathlib import Path import tempfile -import time from types import SimpleNamespace ROOT = Path(__file__).resolve().parents[2] SPEC = importlib.util.spec_from_file_location( "coverage_rust_ledger", ROOT / "scripts" / "coverage_rust_ledger.py" ) -assert SPEC is not None and SPEC.loader is not None +if SPEC is None or SPEC.loader is None: + raise RuntimeError("failed to load the Rust coverage ledger module") ledger_module = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(ledger_module) @@ -24,7 +24,8 @@ def expect_error(fragment: str, call) -> None: try: call() except ledger_module.LedgerError as error: - assert fragment in str(error), (fragment, str(error)) + if fragment not in str(error): + raise AssertionError(f"expected {fragment!r} in {str(error)!r}") from error else: raise AssertionError(f"expected LedgerError containing {fragment!r}") @@ -38,8 +39,9 @@ def main() -> None: profile = temp / "adapter.profraw" artifact.write_bytes(b"same instrumented artifact") runtime.write_bytes(artifact.read_bytes()) - time.sleep(0.01) profile.write_bytes(b"non-empty profile") + artifact_mtime = artifact.stat().st_mtime + os.utime(profile, (artifact_mtime + 10, artifact_mtime + 10)) artifact_hash = ledger_module.sha256(artifact) evidence = { "source_sha": head, @@ -93,7 +95,10 @@ def main() -> None: f"SF:{ROOT / 'crates/graphforge-api/src/lib.rs'}\nDA:1,1\nend_of_record\n", encoding="utf-8", ) - assert ledger_module.parse_lcov(lcov, ROOT) + if not ledger_module.parse_lcov(lcov, ROOT): + raise AssertionError("valid LCOV fixture produced no records") + if ledger_module.normalize_source("/tmp/dependency/crates/foreign/src/lib.rs", ROOT): + raise AssertionError("out-of-tree crate path entered workspace coverage") malformed = temp / "malformed.lcov" malformed.write_text("SF:no-lines.rs\nend_of_record\n", encoding="utf-8") expect_error("no executable lines", lambda: ledger_module.parse_lcov(malformed, ROOT)) @@ -108,8 +113,8 @@ def main() -> None: "schema_version": 1, "source_sha": head, "surfaces": { - name: {"measured_lines": 100, "line_percent": 90.0} - for name in ("core", "python_adapter", "node_adapter") + name: {"covered_lines": 90, "measured_lines": 100, "line_percent": 90.0} + for name in ("core", "python_adapter", "node_adapter", "workspace") }, } ledger_module.validate_floors(valid_ledger, args) @@ -129,6 +134,14 @@ def main() -> None: mutated["schema_version"] = 99 expect_error("unsupported or missing", lambda: ledger_module.validate_floors(mutated, args)) + mutated = json.loads(json.dumps(valid_ledger)) + mutated["surfaces"].pop("workspace") + expect_error("missing workspace", lambda: ledger_module.validate_floors(mutated, args)) + + mutated = json.loads(json.dumps(valid_ledger)) + mutated["surfaces"]["core"].pop("covered_lines") + expect_error("malformed core", lambda: ledger_module.validate_floors(mutated, args)) + print("rust coverage ledger mutation sentinels: ok") diff --git a/scripts/coverage-rust.sh b/scripts/coverage-rust.sh index 115c7d343..89a137665 100755 --- a/scripts/coverage-rust.sh +++ b/scripts/coverage-rust.sh @@ -34,7 +34,7 @@ CORE_TARGET_DIR="$CARGO_TARGET_DIR/llvm-cov-target" rm -rf "$PROFILE_DIR" "$OUT_DIR/core-html" rm -f "$CORE_LCOV" "$PYTHON_LCOV" "$NODE_LCOV" "$WORKSPACE_LCOV" \ - "$LEDGER" "$EVIDENCE" "$SUMMARY" + "$LEDGER" "$EVIDENCE" "$SUMMARY" "$OUT_DIR/python.profdata" "$OUT_DIR/node.profdata" mkdir -p "$PROFILE_DIR" echo "━━━ Core Rust coverage ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @@ -53,10 +53,25 @@ echo "━━━ Instrumented native artifacts ━━━━━━━━━━━ uv run maturin develop --release -m crates/graphforge-bindings-py/Cargo.toml pnpm --filter @curatelabs/graphforge exec napi build --platform --release -PYTHON_OBJECT="$(find "$CARGO_TARGET_DIR" -type f \( -name 'libgraphforge_bindings_py.so' -o -name 'libgraphforge_bindings_py.dylib' -o -name 'graphforge_bindings_py.dll' \) | head -1)" -NODE_OBJECT="$(find "$CARGO_TARGET_DIR" -type f \( -name 'libgraphforge_bindings_node.so' -o -name 'libgraphforge_bindings_node.dylib' -o -name 'graphforge_bindings_node.dll' \) | head -1)" +find_one() { + local description="$1" + shift + local matches=() + while IFS= read -r match; do + matches+=("$match") + done < <(find "$@" -print | sort) + if [[ "${#matches[@]}" -ne 1 ]]; then + echo "Rust coverage evidence error: expected exactly one $description, got ${#matches[@]}" >&2 + printf '%s\n' "${matches[@]}" >&2 + exit 1 + fi + printf '%s\n' "${matches[0]}" +} + +PYTHON_OBJECT="$(find_one 'Python adapter object' "$CARGO_TARGET_DIR" -type f ! -path '*/deps/*' \( -name 'libgraphforge_bindings_py.so' -o -name 'libgraphforge_bindings_py.dylib' -o -name 'graphforge_bindings_py.dll' \))" +NODE_OBJECT="$(find_one 'Node adapter object' "$CARGO_TARGET_DIR" -type f ! -path '*/deps/*' \( -name 'libgraphforge_bindings_node.so' -o -name 'libgraphforge_bindings_node.dylib' -o -name 'graphforge_bindings_node.dll' \))" PYTHON_RUNTIME="$(uv run --no-sync python -c 'from graphforge import _graphforge_rs; print(_graphforge_rs.__file__)')" -NODE_RUNTIME="$(find crates/graphforge-bindings-node -maxdepth 1 -type f -name 'graphforge.*.node' | head -1)" +NODE_RUNTIME="$(find_one 'Node runtime addon' crates/graphforge-bindings-node -maxdepth 1 -type f -name 'graphforge.*.node')" for artifact in "$PYTHON_OBJECT" "$NODE_OBJECT" "$PYTHON_RUNTIME" "$NODE_RUNTIME"; do if [[ -z "$artifact" || ! -s "$artifact" ]]; then echo "Rust coverage evidence error: expected instrumented native artifact is missing" >&2 @@ -83,6 +98,8 @@ export LLVM_PROFILE_FILE="$PROFILE_DIR/python-%p-%10m.profraw" for test_file in crates/graphforge-bindings-py/tests/*.py; do CARGO_TARGET_DIR="$CORE_TARGET_DIR" uv run --no-sync python "$test_file" done +# The publication dry-run invokes pnpm publish hooks that replace the measured +# addon; #365 tracks separating that integration check from adapter acceptance. uv run --no-sync pytest tests/unit tests/integration tests/features \ --ignore=tests/unit/test_publish_dry_run.py \ -n "${PYTEST_WORKERS:-4}" --tb=short @@ -102,17 +119,26 @@ pnpm --filter @curatelabs/graphforge test ) TOOLS_DIR="$(rustc --print sysroot)/lib/rustlib/$(rustc -vV | sed -n 's/^host: //p')/bin" -"$TOOLS_DIR/llvm-profdata" merge -sparse "$PROFILE_DIR"/python-*.profraw -o "$OUT_DIR/python.profdata" -"$TOOLS_DIR/llvm-profdata" merge -sparse "$PROFILE_DIR"/node-*.profraw -o "$OUT_DIR/node.profdata" +shopt -s nullglob +python_profiles=("$PROFILE_DIR"/python-*.profraw) +node_profiles=("$PROFILE_DIR"/node-*.profraw) +shopt -u nullglob +if [[ "${#python_profiles[@]}" -eq 0 || "${#node_profiles[@]}" -eq 0 ]]; then + echo "Rust coverage evidence error: an acceptance suite produced no instrumented profiles" >&2 + exit 1 +fi +"$TOOLS_DIR/llvm-profdata" merge -sparse "${python_profiles[@]}" -o "$OUT_DIR/python.profdata" +"$TOOLS_DIR/llvm-profdata" merge -sparse "${node_profiles[@]}" -o "$OUT_DIR/node.profdata" "$TOOLS_DIR/llvm-cov" export "$PYTHON_OBJECT" \ -instr-profile="$OUT_DIR/python.profdata" -format=lcov > "$PYTHON_LCOV" "$TOOLS_DIR/llvm-cov" export "$NODE_OBJECT" \ -instr-profile="$OUT_DIR/node.profdata" -format=lcov > "$NODE_LCOV" export GF_COVERAGE_ROOT="$ROOT" -export GF_COVERAGE_SOURCE_SHA="$(git rev-parse HEAD)" -export GF_COVERAGE_RUSTC="$(rustc --version)" -export GF_COVERAGE_LLVM_COV="$(cargo llvm-cov --version)" +GF_COVERAGE_SOURCE_SHA="$(git rev-parse HEAD)" +GF_COVERAGE_RUSTC="$(rustc --version)" +GF_COVERAGE_LLVM_COV="$(cargo llvm-cov --version)" +export GF_COVERAGE_SOURCE_SHA GF_COVERAGE_RUSTC GF_COVERAGE_LLVM_COV export GF_COVERAGE_PYTHON_OBJECT="$PYTHON_OBJECT" export GF_COVERAGE_PYTHON_RUNTIME="$PYTHON_RUNTIME" export GF_COVERAGE_PYTHON_HASH="$python_hash" diff --git a/scripts/coverage_rust_ledger.py b/scripts/coverage_rust_ledger.py index 9a074000f..e9b109b05 100755 --- a/scripts/coverage_rust_ledger.py +++ b/scripts/coverage_rust_ledger.py @@ -38,10 +38,6 @@ def normalize_source(raw: str, root: Path) -> str | None: try: return path.resolve().relative_to(root.resolve()).as_posix() except ValueError: - marker = "/crates/" - normalized = path.as_posix() - if marker in normalized: - return f"crates/{normalized.split(marker, 1)[1]}" return None @@ -194,20 +190,29 @@ def validate_floors(ledger: dict[str, Any], args: argparse.Namespace) -> None: raise LedgerError("unsupported or missing Rust coverage ledger schema") if ledger.get("source_sha") != expected_sha: raise LedgerError("Rust coverage ledger is stale for the current HEAD") - floors = { + floors: dict[str, float | None] = { "core": args.core_floor, "python_adapter": args.python_floor, "node_adapter": args.node_floor, + "workspace": None, } for surface, floor in floors.items(): data = ledger.get("surfaces", {}).get(surface) if not isinstance(data, dict): raise LedgerError(f"Rust coverage ledger is missing {surface}") + covered = data.get("covered_lines") measured = data.get("measured_lines") percent = data.get("line_percent") - if not isinstance(measured, int) or measured <= 0 or not isinstance(percent, (int, float)): + if ( + not isinstance(covered, int) + or covered < 0 + or not isinstance(measured, int) + or measured <= 0 + or covered > measured + or not isinstance(percent, (int, float)) + ): raise LedgerError(f"Rust coverage ledger has malformed {surface} totals") - if float(percent) < floor: + if floor is not None and float(percent) < floor: raise LedgerError( f"{surface} Rust coverage below {floor:.2f}% (got {float(percent):.2f}%)" ) @@ -246,12 +251,15 @@ def main() -> int: raise LedgerError(f"missing Rust coverage ledger: {args.ledger}") ledger = json.loads(args.ledger.read_text(encoding="utf-8")) validate_floors(ledger, args) - except (LedgerError, json.JSONDecodeError) as error: + report = [ + (name, ledger["surfaces"][name]) + for name in ("core", "python_adapter", "node_adapter", "workspace") + ] + except (LedgerError, json.JSONDecodeError, KeyError, TypeError) as error: print(f"Rust coverage evidence error: {error}", file=sys.stderr) return 1 - for name in ("core", "python_adapter", "node_adapter", "workspace"): - data = ledger["surfaces"][name] + for name, data in report: print( f"{name}: {data['covered_lines']}/{data['measured_lines']} " f"lines ({data['line_percent']:.2f}%)" From 0033e11168e14d7e409d9f08a4800d4f30edaff8 Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Mon, 3 Aug 2026 05:13:44 -0600 Subject: [PATCH 3/3] test(ci): fail fast on coverage setup errors --- scripts/ci/test-rust-coverage-ledger.py | 1 + scripts/coverage-rust.sh | 3 ++- scripts/coverage_rust_ledger.py | 7 ++++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/ci/test-rust-coverage-ledger.py b/scripts/ci/test-rust-coverage-ledger.py index 15845a5e6..f982ce308 100755 --- a/scripts/ci/test-rust-coverage-ledger.py +++ b/scripts/ci/test-rust-coverage-ledger.py @@ -34,6 +34,7 @@ def main() -> None: head = ledger_module.git_head(ROOT) with tempfile.TemporaryDirectory(prefix="gf-rust-coverage-ledger-") as directory: temp = Path(directory) + expect_error("failed to resolve git HEAD", lambda: ledger_module.git_head(temp)) artifact = temp / "adapter.so" runtime = temp / "loaded.so" profile = temp / "adapter.profraw" diff --git a/scripts/coverage-rust.sh b/scripts/coverage-rust.sh index 89a137665..d769b99c2 100755 --- a/scripts/coverage-rust.sh +++ b/scripts/coverage-rust.sh @@ -47,7 +47,8 @@ cargo llvm-cov report --html --output-dir "$OUT_DIR/core-html" # Export the exact instrumentation environment used by cargo-llvm-cov. Native # artifacts are built once, sequentially, and then reused by all acceptance. -eval "$(cargo llvm-cov show-env --sh)" +COVERAGE_ENV="$(cargo llvm-cov show-env --sh)" +eval "$COVERAGE_ENV" echo "━━━ Instrumented native artifacts ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" uv run maturin develop --release -m crates/graphforge-bindings-py/Cargo.toml diff --git a/scripts/coverage_rust_ledger.py b/scripts/coverage_rust_ledger.py index e9b109b05..1b3395954 100755 --- a/scripts/coverage_rust_ledger.py +++ b/scripts/coverage_rust_ledger.py @@ -30,7 +30,12 @@ def sha256(path: Path) -> str: def git_head(root: Path) -> str: - return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() + try: + return subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=root, text=True, stderr=subprocess.PIPE + ).strip() + except (subprocess.CalledProcessError, OSError) as error: + raise LedgerError("failed to resolve git HEAD") from error def normalize_source(raw: str, root: Path) -> str | None: