diff --git a/packages/runtime/src/artifact-gates.ts b/packages/runtime/src/artifact-gates.ts index 74375a507..af503b095 100644 --- a/packages/runtime/src/artifact-gates.ts +++ b/packages/runtime/src/artifact-gates.ts @@ -2853,6 +2853,7 @@ function verifyFinalReportPropertyReferences( catalog.value, implementation.value, readCampaignFuzzerBackends(layout), + report, reportPath ) ); @@ -3081,6 +3082,7 @@ function reportPropertyJoinDiagnostics( catalog: PropertiesArtifact, implementation: ImplementedPropertiesArtifact, fuzzerBackendsByFinding: ReadonlyMap, + report: Record, reportPath: string ): RuntimeDiagnostic[] { const catalogById = new Map(catalog.properties.map((property) => [property.id, property])); @@ -3150,8 +3152,9 @@ function reportPropertyJoinDiagnostics( `${reportPath}#$.property_provenance[${entryIndex}].test_paths` ); - const expectedBackends = - typeof entry.finding_id === "string" ? (fuzzerBackendsByFinding.get(entry.finding_id) ?? []) : []; + const expectedBackends = verifiedReportFindingAliases(entry, report).flatMap( + (findingId) => fuzzerBackendsByFinding.get(findingId) ?? [] + ); const actualBackends = Array.isArray(entry.fuzzer_backends) ? stringArray(entry.fuzzer_backends) : typeof entry.fuzzer_backend === "string" @@ -3168,6 +3171,44 @@ function reportPropertyJoinDiagnostics( return diagnostics; } +function verifiedReportFindingAliases(entry: Record, report: Record): string[] { + const findingId = typeof entry.finding_id === "string" ? entry.finding_id : undefined; + if (findingId === undefined) { + return []; + } + const verifiedAliases = new Set([findingId]); + const outcomes = [report.issues, report.non_production_outcomes].flatMap((entries) => + Array.isArray(entries) ? entries.filter(isRecord) : [] + ); + for (const outcome of outcomes) { + if ( + outcome.id !== findingId || + !isRecord(outcome.lifecycle) || + !Array.isArray(outcome.lifecycle.source_artifacts) + ) { + continue; + } + const lifecycleFindingIds = new Set( + outcome.lifecycle.source_artifacts.flatMap((source) => + isRecord(source) && + campaignLogicalNodeIds.some((nodeId) => nodeId === source.node_id) && + source.relationship === "primary" && + typeof source.finding_id === "string" + ? [source.finding_id] + : [] + ) + ); + for (const alias of stringArray([outcome.finding_id, outcome.upstream_id, outcome.source_finding_id])) { + if (lifecycleFindingIds.has(alias)) { + verifiedAliases.add(alias); + } + } + } + return [...new Set(stringArray([entry.finding_id, entry.upstream_id, entry.source_finding_id]))].filter((alias) => + verifiedAliases.has(alias) + ); +} + function readCampaignFuzzerBackends(layout: RunLayout): ReadonlyMap { const campaigns: PropertyCampaignArtifact[] = []; const findings: Array> = []; diff --git a/packages/runtime/test/artifact-gates.test.ts b/packages/runtime/test/artifact-gates.test.ts index fa6a69e7b..8832f62d0 100644 --- a/packages/runtime/test/artifact-gates.test.ts +++ b/packages/runtime/test/artifact-gates.test.ts @@ -4026,6 +4026,116 @@ test("final report gate joins the default recon-only campaign backend", () => { assert.ok(mismatch.diagnostics.some((diagnostic) => diagnostic.code === "PROPERTY_REPORT_FUZZER_BACKEND_MISMATCH")); }); +test("final report gate joins renumbered findings through their source finding id", () => { + const layout = createRunLayout({ projectRoot: tempProject(), runId: "run-renumbered-report" }); + const node = { ...plannedNode(["report.json"]), id: "final-report", logical_id: "final-report" }; + writeArtifact( + layout, + "property-specification-fanin", + "properties.json", + JSON.stringify({ + schema_version: "ultrafuzz.properties.v1", + properties: [ + { + id: "property-1", + description: "Balances remain conserved", + category: "accounting", + priority: "high", + sources: [{ source_node_id: "property-specification-recon", source_property_id: "recon-1" }] + } + ] + }) + ); + writeArtifact( + layout, + "stateful-invariant-implement-properties", + "implemented-properties.json", + JSON.stringify({ + schema_version: "ultrafuzz.implemented-properties.v1", + properties: [ + { + property_id: "property-1", + status: "implemented", + implementation_paths: ["tests/recon/Properties.sol"], + test_paths: ["tests/recon/CryticToFoundry.sol"] + } + ] + }) + ); + for (const [artifactName, backend, failureId] of [ + ["recon-fuzzer-results.json", "recon", "failure-1"], + ["medusa-results.json", "medusa", "failure-2"] + ] as const) { + writeArtifact( + layout, + "stateful-invariant-campaign", + artifactName, + JSON.stringify({ + schema_version: "ultrafuzz.property-campaign.v1", + fuzzer_backend: backend, + failures: [{ id: failureId, status: "reproduced", property_ids: ["property-1"] }] + }) + ); + } + + const report = (sourceFindingId: string, fuzzerBackends: string[]) => + JSON.stringify({ + schema_version: "1.0", + run_metadata: {}, + issues: [], + non_production_outcomes: [ + { + id: "NP-01", + finding_id: "failure-1", + lifecycle: { + source_artifacts: [ + { + node_id: "stateful-invariant-campaign", + finding_id: "failure-1", + relationship: "primary" + } + ] + } + }, + { + id: "NP-02", + finding_id: "failure-2", + lifecycle: { + source_artifacts: [ + { + node_id: "stateful-invariant-campaign", + finding_id: "failure-2", + relationship: "primary" + } + ] + } + } + ], + property_provenance: [ + { + finding_id: "NP-01", + source_finding_id: sourceFindingId, + title: "Renumbered non-production outcome", + property_ids: ["property-1"], + sources: [{ source_node_id: "property-specification-recon", source_property_id: "recon-1" }], + implementation_paths: ["tests/recon/Properties.sol"], + test_paths: ["tests/recon/CryticToFoundry.sol"], + fuzzer_backends: fuzzerBackends + } + ] + }); + + writeArtifact(layout, node.id, "report.json", report("failure-1", ["recon"])); + assert.equal(verifyRequiredArtifactsForAttempt(layout, node, node.id).ok, true); + + // NP-02 owns failure-2. NP-01 cannot borrow that unrelated finding's Medusa + // provenance merely by placing its ID in source_finding_id. + writeArtifact(layout, node.id, "report.json", report("failure-2", ["medusa"])); + const mismatch = verifyRequiredArtifactsForAttempt(layout, node, node.id); + assert.equal(mismatch.ok, false); + assert.ok(mismatch.diagnostics.some((diagnostic) => diagnostic.code === "PROPERTY_REPORT_FUZZER_BACKEND_MISMATCH")); +}); + test("final report gate rejects dangling property references while allowing historical provenance", () => { const historicalLayout = createRunLayout({ projectRoot: tempProject(), runId: "run-historical-report" }); const node = {