feat: add strict producer-visible JSON contracts

.github/workflows/ci.yml

@@ -71,7 +71,6 @@ jobs:
       ULTRAFUZZ_CODEX_REAL_DRY_RUN: "0"
       ULTRAFUZZ_RELEASE_DASHBOARD: "0"
       ULTRAFUZZ_RELEASE_REFERENCES: "0"
-
     steps:
       - name: Check out repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

.github/workflows/eval-benchmark-recovery.yml

@@ -72,7 +72,7 @@ jobs:
           plan_count=0
           plan_mode=
           plan_artifact_name=
-          for mode in smoke full threat-model; do
+          for mode in smoke full; do
             candidate_name="modal-benchmark-plan-$mode-$SOURCE_RUN_ID-$SOURCE_RUN_ATTEMPT"
             candidate_count="$(jq --arg name "$candidate_name" \
               '[.artifacts[] | select(.name == $name)] | length' \

.github/workflows/eval-history-publication.yml

@@ -31,11 +31,21 @@ jobs:
           ref: main
           persist-credentials: false
 
+      - name: Set up pnpm
+        uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
+        with:
+          version: 11.1.1
+
       - name: Set up Node.js
         uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
           node-version: 24
 
+      - name: Build trusted qualification dependency
+        run: |
+          pnpm install --frozen-lockfile
+          pnpm --filter @ultrafuzz/artifacts... build
+
       - name: Qualify the exact completed producer attempt
         id: qualify
         env:
@@ -149,6 +159,8 @@ jobs:
             "$BENCHMARK_MODE" \
             "$PUBLICATION_GENERATION" \
             "$PUBLICATION_PLAN"
+          publication_rows="$RUNNER_TEMP/eval-history-publication-rows.tsv"
+          node scripts/ci/prepare-eval-history-publication.mjs plan-rows "$PUBLICATION_PLAN" > "$publication_rows"
           mkdir "$PUBLICATION_UNPACKED"
           while IFS=$'\t' read -r bundle_path unpack_path eval_run_id benchmark lane model_slug; do
             metadata="$(node packages/modal/dist/cli.js unpack-public \
@@ -166,8 +178,7 @@ jobs:
                .model_slug == $model_slug and
                .candidate_commit == $candidate_commit' \
               <<< "$metadata" > /dev/null
-          done < <(jq -r '.pairs[] | [.bundle_path, .unpack_path, .eval_run_id, .benchmark, .lane, .model_slug] | @tsv' \
-            "$PUBLICATION_PLAN")
+          done < "$publication_rows"
 
       - name: Create the narrowly scoped eval-history publisher token
         id: publisher-token

.gitignore

@@ -24,3 +24,4 @@ smithers.db*
 
 # Agent session artifacts written by package-level test runs
 packages/*/.ultrafuzz/
+packages/dashboard/frontend/.generated/

.prettierignore

@@ -11,6 +11,8 @@ pnpm-lock.yaml
 
 packages/*/dist
 packages/*/dist-test
+packages/dashboard/frontend/.generated
+packages/config/test/fixtures/resolved-config.valid.json
 
 prompts/**/*.md
 docs/reference/audit-profiles.md

.ultrafuzz/evals/bug-finding.yml

@@ -4,7 +4,7 @@
 # binding and credentials env-var names live in ultrafuzz.toml under [eval].
 # Ground-truth files resolve under [eval].ground_truth_root, which MUST live
 # outside the repository.
-schema_version: ultrafuzz.eval.v1
+schema_version: ultrafuzz.eval.v2
 suite: bug-finding-regression
 
 model_profiles:
@@ -34,7 +34,6 @@ run:
   max_parallel_runs: 8
 
 metrics:
-  primary: [precision, recall, f1_score]
   recall_threshold: 0.7
 
 recovery_equivalence:
@@ -48,5 +47,5 @@ reporting: # policy only — provider selection/credentials live in ultrafuzz.to
   experiment_prefix: bug-finding
   artifacts:
     mode: manifest-only # forced default when target.sensitivity == private; opt in to "upload"
-    include: ["report.md", "report.json", "findings.normalized.json"]
+    include: ["report.md", "report.json"]
     max_file_bytes: 5000000

.ultrafuzz/prompts/_templates/output-contract/boundary-recipes.mdx

@@ -1 +1 @@
-`boundary-recipes.md` should be a markdown index of reproduction recipes. `boundary-recipes.json` must be a JSON object with `schema_version: "1.0"` and a `recipes` array. Each recipe should include `id`, `title`, `path`, and any known `finding_ids`, `property_ids`, or `summary`. Use an empty `recipes` array if no recipes were produced.
+For every output declared with `Contract: ultrafuzz/boundary-recipes@1`, read the exact pinned schema named by `Validate against` in the central Ultrafuzz Output Contract and run the exact `Validation command` rendered beside that output. The declared path is authoritative; do not create a differently named structured artifact. That schema alone owns the JSON version, field names, types, required and optional members, and empty form. When a human-readable boundary-recipe companion is declared, keep it and the structured artifact semantically aligned with the same source-backed boundary recipes; this contextual correspondence is checked separately from JSON Schema validation.

.ultrafuzz/prompts/_templates/output-contract/findings.mdx

@@ -1 +1 @@
-`findings.json` must be a JSON array. Use `[]` when there are no findings. Each finding object must include non-empty string fields `id`, `title`, `status`, `severity_guess`, `confidence`, and `summary`. Every finding's `severity_guess` must be exactly `High`, `Medium`, or `Low`, including findings that are or may become non-production records. `schema_version` is optional; omit it, or set it to `"1.0"` or `"ultrafuzz.finding.v1"`. `status` must be a non-empty lifecycle string. Prefer canonical values such as `candidate`, `needs-review`, `duplicate`, `false-positive`, `confirmed`, `fixed`, or `wont-fix` when they fit. If you include `triage_classification`, use one of `true-positive`, `false-positive`, `undetermined`, `incomplete-spec`, `harness-defect`, `repair-candidate`, `spec-gated`, or `defensive-hardening`. Use `severity_guess`; do not use `severity` as a replacement. If you include `evidence`, each entry may be a non-empty string reference or an object with optional non-empty string `kind` and `path` fields plus any other useful metadata. Object `path` fields must be safe relative file paths without anchors or line selectors. Put a section anchor in `fragment`, one source span in positive integer `line` and optional `end_line`, and disjoint spans in `line_ranges`, an array of at least two objects such as `[{"line":105,"end_line":107},{"line":154,"end_line":185}]`; every `end_line` must be no smaller than its `line`. Never emit a one-entry `line_ranges`, and never combine `line_ranges` with `line` or `end_line`. Keep independent explanatory prose in `detail`; never embed selectors into `path` or replace prose with them.
+For every output declared with `Contract: ultrafuzz/findings@2`, read the exact pinned schema named by `Validate against` in the central Ultrafuzz Output Contract and run the exact `Validation command` rendered beside that output. The declared path is authoritative; do not create a differently named findings artifact. That schema alone owns the JSON version, field names, types, enums, required members, and empty form. Producer lanes make only a preliminary severity estimate; later severity review owns final severity. Cite evidence at its real source location, keep base paths free of embedded selectors, and keep independent explanation separate from verbatim or source-location data. Those ownership and evidence semantics remain required in addition to schema validation.

.ultrafuzz/prompts/_templates/output-contract/generated-tests.mdx

@@ -1,2 +1,9 @@
-Write generated test source files under `{{strategy_attempt_test_dir}}` and mirror the files under `{{generated_tests_dir}}`.
-`generated-tests.json` must be a JSON object with `schema_version: "1.0"`, `run_id`, `node_id`, and `generated_tests`. Use `generated_tests` as the only file list. Each `generated_tests` entry must include `path` using the `generated-tests/<file>` prefix plus `language`, `framework`, and `description` when known. Use an empty `generated_tests` array if no tests were produced.
+## Generated-test Bundle Instructions
+
+This node is a generated-test producer. Author runnable generated test source files under {{strategy_attempt_test_dir}}, mirror every declared bundle file under {{generated_tests_dir}}, and write the manifest to exactly {{generated_tests_manifest_path}}.
+
+Read the exact pinned schema named by `Validate against` in the central Ultrafuzz Output Contract and run the exact `Validation command` rendered beside it. That schema alone owns the manifest's JSON version, field names, types, required and optional members, and empty form.
+
+Bind `run_id` exactly to {{run_id}} and `node_id` exactly to {{logical_node_id}}. The node identity is the logical producer shown here, not a concrete attempt, loop, or model-fanout identity. Bind the bundle framework to the one checked-in native framework used by the whole atomic bundle. Never mix frameworks in one bundle or assign a separate framework to an individual companion. These run, producer, and repository-context bindings are checked beyond JSON Schema.
+
+Classify only independently runnable tests and reproducers as generated tests. Classify imported helper libraries, mocks, fixtures, deployment scripts, text data fixtures, and other non-runnable dependencies as support files. Every declared companion, including data fixtures, must be strict UTF-8 text rather than binary data and must live beneath the artifact bundle, never at a workspace path. When no runnable test was produced, use the empty bundle defined by the schema while still binding its framework to the checked-in native framework. Before returning, validate after every declared companion is present, singly linked, regular, non-empty, and neither a symlink nor reachable through one.

.ultrafuzz/prompts/_templates/output-contract/output-contract.mdx

@@ -4,10 +4,18 @@ Before finishing, you MUST ensure every artifact below exists and satisfies its
 
 {{artifact_contracts}}
 
-{{schema_guidance}}Artifact writing is part of the task, not optional final-response metadata. Create parent directories before writing, write each artifact to the exact absolute path shown above, and then verify each path exists, is non-empty unless its contract explicitly allows an empty form, and matches its JSON or Markdown contract. Do not write required artifacts only under the repository root, a relative working directory, or a different similarly named path.
+{{schema_guidance}}Artifact writing is part of the task, not optional final-response metadata. Create parent directories before writing and write each artifact to the exact absolute path shown above. Also verify every path exists, is non-empty unless its contract explicitly allows an empty form, and matches its JSON or Markdown contract. Do not write required artifacts only under the repository root, a relative working directory, or a different similarly named path.
 
-After completing the work and writing every required artifact, your final response MUST contain ONLY one raw, valid JSON object in exactly this form:
+For every schema-backed JSON artifact:
 
-{"summary":"A concise description of the completed work and written artifacts."}
+1. Write the artifact in its canonical schema.
+2. After your final write and before returning the node's final response, run every exact `Validation command` shown above.
+3. If a command exits 1, correct that artifact yourself and rerun the same command during this session.
+4. If you change an artifact after it passes validation, rerun its command.
+5. Never edit a supplied schema. Its bytes are pinned and the command accepts it only when they match the trusted digest.
+6. Exit 2 is a setup or tool failure. Do not edit the schema or claim validation succeeded.
+7. Finish only after every displayed validation command exits 0.
+8. The validation command never modifies the artifact.
+9. Host semantic and context verification still runs after you finish.
 
-The `summary` field is REQUIRED and MUST be a JSON string. Do NOT include Markdown fences, introductory text, explanations, or any other content before or after the JSON object. Do NOT omit the JSON object even when the work succeeds.
+After completing the work and writing every required artifact, return only the structured task result requested by the runtime. The runtime's output schema alone owns that response's keys, types, and envelope. Put a concise description of the completed work and written artifacts in its schema-defined summary value. Do not add Markdown fences, introductory text, explanations, or any content outside the structured result, and do not omit the result when the work succeeds.

.ultrafuzz/prompts/properties/0kn0t-lens.md

@@ -30,11 +30,12 @@ the catalog, and set every property priority to `high`, `medium`, or `low`.
 The JSON catalog is the machine-readable source of truth.
 
 Populate `reference_expectations` only from exact identifiers present in the
-supplied pinned-reference artifacts. Preserve one
-identifier per named expectation and carry the supplied identifier unchanged;
-when the supplied inputs contain no named expectation, leave the field absent.
+supplied pinned-reference artifacts. Preserve one identifier per named
+expectation and carry the supplied identifier unchanged. When the supplied
+inputs contain no named expectation, do not invent expectation IDs; use the
+pinned property-lens schema's no-expectation representation.
 
-When a pinned-reference node declares a catalog with the `ultrafuzz/reference-expectations@1` contract, read its declared artifact and validate it with `{{schema_path}}/reference-expectations.schema.json` before copying identifiers.
+When a pinned-reference node declares a catalog with the `ultrafuzz/reference-expectations@2` contract, read its declared artifact and validate it with `{{schema_path}}/reference-expectations.schema.json` before copying identifiers.
 
 ## Target-derived invariant extraction
 
@@ -73,13 +74,13 @@ Keep the Markdown table readable and identical in ids, descriptions,
 categories, and priorities. Do not put findings in either property artifact.
 
 Findings discipline: property candidates are planning material, not campaign
-findings. Do not copy the property table into `findings.json`; write `[]`
-there unless you independently identify a concrete target vulnerability with
-specific evidence.
+findings. Do not copy the property table into `findings.json`; use only the
+empty form defined by its exact pinned schema unless you independently identify
+a concrete target vulnerability with specific evidence.
 
 Artifact finalization: after writing the property table, immediately write the
-required findings JSON to `{{output_findings_path}}`; use `[]` when there is no
-concrete vulnerability. Do not use Bash to validate the property catalog or
+required findings JSON to `{{output_findings_path}}`; use its schema-defined
+empty form when there is no concrete vulnerability. Do not use Bash to validate the property catalog or
 findings with `cat`, `grep`, `tr`, `wc`, pipes, redirection, or command chains.
 If you inspect the artifact after writing it, use the Read tool or one simple
 allowlisted command, then finish.

.ultrafuzz/prompts/properties/aviggiano-lens.md

@@ -30,11 +30,12 @@ the catalog, and set every property priority to `high`, `medium`, or `low`.
 The JSON catalog is the machine-readable source of truth.
 
 Populate `reference_expectations` only from exact identifiers present in the
-supplied pinned-reference artifacts. Preserve one
-identifier per named expectation and carry the supplied identifier unchanged;
-when the supplied inputs contain no named expectation, leave the field absent.
+supplied pinned-reference artifacts. Preserve one identifier per named
+expectation and carry the supplied identifier unchanged. When the supplied
+inputs contain no named expectation, do not invent expectation IDs; use the
+pinned property-lens schema's no-expectation representation.
 
-When a pinned-reference node declares a catalog with the `ultrafuzz/reference-expectations@1` contract, read its declared artifact and validate it with `{{schema_path}}/reference-expectations.schema.json` before copying identifiers.
+When a pinned-reference node declares a catalog with the `ultrafuzz/reference-expectations@2` contract, read its declared artifact and validate it with `{{schema_path}}/reference-expectations.schema.json` before copying identifiers.
 
 ## Target-derived invariant extraction
 
@@ -73,13 +74,13 @@ Keep the Markdown table readable and identical in ids, descriptions,
 categories, and priorities. Do not put findings in either property artifact.
 
 Findings discipline: property candidates are planning material, not campaign
-findings. Do not copy the property table into `findings.json`; write `[]`
-there unless you independently identify a concrete target vulnerability with
-specific evidence.
+findings. Do not copy the property table into `findings.json`; use only the
+empty form defined by its exact pinned schema unless you independently identify
+a concrete target vulnerability with specific evidence.
 
 Artifact finalization: after writing the property table, immediately write the
-required findings JSON to `{{output_findings_path}}`; use `[]` when there is no
-concrete vulnerability. Do not use Bash to validate the property catalog or
+required findings JSON to `{{output_findings_path}}`; use its schema-defined
+empty form when there is no concrete vulnerability. Do not use Bash to validate the property catalog or
 findings with `cat`, `grep`, `tr`, `wc`, pipes, redirection, or command chains.
 If you inspect the artifact after writing it, use the Read tool or one simple
 allowlisted command, then finish.

.ultrafuzz/prompts/properties/certora-thinking-lens.md

@@ -36,11 +36,12 @@ the catalog, and set every property priority to `high`, `medium`, or `low`.
 The JSON catalog is the machine-readable source of truth.
 
 Populate `reference_expectations` only from exact identifiers present in the
-supplied pinned-reference artifacts. Preserve one
-identifier per named expectation and carry the supplied identifier unchanged;
-when the supplied inputs contain no named expectation, leave the field absent.
+supplied pinned-reference artifacts. Preserve one identifier per named
+expectation and carry the supplied identifier unchanged. When the supplied
+inputs contain no named expectation, do not invent expectation IDs; use the
+pinned property-lens schema's no-expectation representation.
 
-When a pinned-reference node declares a catalog with the `ultrafuzz/reference-expectations@1` contract, read its declared artifact and validate it with `{{schema_path}}/reference-expectations.schema.json` before copying identifiers.
+When a pinned-reference node declares a catalog with the `ultrafuzz/reference-expectations@2` contract, read its declared artifact and validate it with `{{schema_path}}/reference-expectations.schema.json` before copying identifiers.
 
 ## Target-derived invariant extraction
 
@@ -79,13 +80,13 @@ Keep the Markdown table readable and identical in ids, descriptions,
 categories, and priorities. Do not put findings in either property artifact.
 
 Findings discipline: property candidates are planning material, not campaign
-findings. Do not copy the property table into `findings.json`; write `[]`
-there unless you independently identify a concrete target vulnerability with
-specific evidence.
+findings. Do not copy the property table into `findings.json`; use only the
+empty form defined by its exact pinned schema unless you independently identify
+a concrete target vulnerability with specific evidence.
 
 Artifact finalization: after writing the property table, immediately write the
-required findings JSON to `{{output_findings_path}}`; use `[]` when there is no
-concrete vulnerability. Do not use Bash to validate the property catalog or
+required findings JSON to `{{output_findings_path}}`; use its schema-defined
+empty form when there is no concrete vulnerability. Do not use Bash to validate the property catalog or
 findings with `cat`, `grep`, `tr`, `wc`, pipes, redirection, or command chains.
 If you inspect the artifact after writing it, use the Read tool or one simple
 allowlisted command, then finish.