v0.1.0 release (open source candidate)

.github/workflows/ci.yml

@@ -14,11 +14,9 @@ concurrency:
 jobs:
   release-gates:
     runs-on: ubuntu-latest
-    env:
-      ULTRAFUZZ_CODEX_REAL_DRY_RUN: "0"
-      ULTRAFUZZ_RELEASE_DASHBOARD: "0"
-      ULTRAFUZZ_RELEASE_REFERENCES: "0"
-    timeout-minutes: 30
+    # A clean local run is ~21 minutes after removing the duplicated benchmark
+    # suites. Keep enough margin for dependency setup and hosted-runner variance.
+    timeout-minutes: 60
 
     steps:
       - name: Check out repository
@@ -52,12 +50,15 @@ jobs:
       - name: Build
         run: pnpm -w build
 
-      - name: Validate benchmark manifests and history charts
-        run: pnpm -w benchmark:check
-
       - name: Validate release
         run: pnpm -w validate:release
 
+      # validate:release already runs the config, runtime, and eval suites used by
+      # benchmark:check. The built CLI is all that is needed for the remaining
+      # byte-for-byte history/chart check, so do not run the long suites twice.
+      - name: Validate benchmark history charts
+        run: pnpm -w benchmark:history:check
+
       - name: Upload release validation report
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

.github/workflows/eval-benchmarks.yml

@@ -10,16 +10,24 @@ on:
       - docs/assets/eval-history/**
   workflow_dispatch:
     inputs:
-      benchmark_mode:
-        description: Lane to dispatch. smoke runs one runner over the three smoke targets; full runs the four-provider EVMBench matrix
+      benchmark_lane:
+        description: >-
+          Which lane to dispatch. `smoke` runs one selected runner over the three smoke targets;
+          `full` is the EVMBench model matrix; `threat-model` is the v0.1.0 release gate from #183:
+          the pinned three-target Ultrafuzz-bench cohort through the PRODUCTION topology, so the
+          threat-model, goal-plan and dynamic goal-fanout nodes actually run. The threat-model lane
+          ignores every model input below and uses the checked-in benchmarks/lanes.json profile
+          (OpenAI gpt-5.6-luna / high), and it is never published to longitudinal history. It costs
+          real money: confirm an owner and a compute budget before dispatching.
         required: true
         default: full
         type: choice
         options:
-          - full
           - smoke
+          - full
+          - threat-model
       smoke_provider:
-        description: smoke mode only, runner provider
+        description: smoke lane only, runner provider
         required: true
         default: deepseek
         type: choice
@@ -29,12 +37,12 @@ on:
           - deepseek
           - kimi
       smoke_model:
-        description: smoke mode only, runner model
+        description: smoke lane only, runner model
         required: true
         default: deepseek-v4-flash
         type: string
       smoke_reasoning:
-        description: smoke mode only, runner reasoning effort
+        description: smoke lane only, runner reasoning effort
         required: true
         default: max
         type: string
@@ -80,15 +88,15 @@ on:
         type: string
 
 env:
-  BENCHMARK_MODE: ${{ github.event_name == 'workflow_dispatch' && (inputs.benchmark_mode || 'full') || 'smoke' }}
+  BENCHMARK_MODE: ${{ github.event_name == 'workflow_dispatch' && inputs.benchmark_lane || 'smoke' }}
   BENCHMARK_CANDIDATE: ${{ github.event_name == 'push' && github.event.after || github.sha }}
 
 permissions:
   actions: read
   contents: read
 
 concurrency:
-  group: modal-eval-${{ github.event_name == 'workflow_dispatch' && (inputs.benchmark_mode || 'full') || 'smoke' }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || github.ref }}
+  group: modal-eval-${{ github.event_name == 'workflow_dispatch' && inputs.benchmark_lane || 'smoke' }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || github.ref }}
   cancel-in-progress: ${{ github.event_name == 'push' }}
 
 jobs:
@@ -109,6 +117,11 @@ jobs:
           fetch-depth: 0
           persist-credentials: false
 
+      # Keep the selected lane visible in the job metadata for operators. The
+      # publication qualifier independently derives it from exact-run artifacts.
+      - name: Benchmark lane ${{ github.event_name == 'workflow_dispatch' && inputs.benchmark_lane || 'smoke' }}
+        run: echo "Dispatching the $BENCHMARK_MODE benchmark lane"
+
       - name: Set up pnpm
         uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
         with:
@@ -130,18 +143,18 @@ jobs:
           MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
           MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          ANTHROPIC_API_KEY: ${{ github.event_name == 'workflow_dispatch' && secrets.ANTHROPIC_API_KEY || '' }}
-          DEEPSEEK_API_KEY: ${{ github.event_name == 'workflow_dispatch' && secrets.DEEPSEEK_API_KEY || '' }}
-          KIMI_API_KEY: ${{ github.event_name == 'workflow_dispatch' && secrets.KIMI_API_KEY || github.event_name == 'workflow_dispatch' && secrets.MOONSHOT_API_KEY || '' }}
-          KIMI_BASE_URL: ${{ github.event_name == 'workflow_dispatch' && secrets.KIMI_BASE_URL || github.event_name == 'workflow_dispatch' && vars.KIMI_BASE_URL || '' }}
-          BENCHMARK_SMOKE_PROVIDER: ${{ github.event_name == 'workflow_dispatch' && (inputs.smoke_provider || 'openai') || 'openai' }}
+          ANTHROPIC_API_KEY: ${{ (inputs.benchmark_lane == 'full' || inputs.benchmark_lane == 'smoke') && secrets.ANTHROPIC_API_KEY || '' }}
+          DEEPSEEK_API_KEY: ${{ (inputs.benchmark_lane == 'full' || inputs.benchmark_lane == 'smoke') && secrets.DEEPSEEK_API_KEY || '' }}
+          KIMI_API_KEY: ${{ (inputs.benchmark_lane == 'full' || inputs.benchmark_lane == 'smoke') && (secrets.KIMI_API_KEY || secrets.MOONSHOT_API_KEY) || '' }}
+          KIMI_BASE_URL: ${{ (inputs.benchmark_lane == 'full' || inputs.benchmark_lane == 'smoke') && (secrets.KIMI_BASE_URL || vars.KIMI_BASE_URL) || '' }}
+          BENCHMARK_SMOKE_PROVIDER: ${{ inputs.benchmark_lane == 'smoke' && (inputs.smoke_provider || 'openai') || 'openai' }}
         run: |
           # The judge profile is always gpt-5.6-sol on CodexAgent, so OPENAI_API_KEY is
           # required for every lane regardless of which runner provider is selected.
           required=(MODAL_TOKEN_ID MODAL_TOKEN_SECRET OPENAI_API_KEY)
           if [ "$BENCHMARK_MODE" = full ]; then
             required+=(ANTHROPIC_API_KEY KIMI_API_KEY DEEPSEEK_API_KEY)
-          elif [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then
+          elif [ "$BENCHMARK_MODE" = smoke ] && [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then
             case "$BENCHMARK_SMOKE_PROVIDER" in
               anthropic) required+=(ANTHROPIC_API_KEY) ;;
               deepseek) required+=(DEEPSEEK_API_KEY) ;;
@@ -165,16 +178,20 @@ jobs:
           BENCHMARK_KIMI_REASONING: ${{ inputs.kimi_reasoning || 'max' }}
           BENCHMARK_DEEPSEEK_MODEL: ${{ inputs.deepseek_model || 'deepseek-v4-pro' }}
           BENCHMARK_DEEPSEEK_REASONING: ${{ inputs.deepseek_reasoning || 'max' }}
-          BENCHMARK_SMOKE_PROVIDER: ${{ github.event_name == 'workflow_dispatch' && (inputs.smoke_provider || 'openai') || 'openai' }}
-          BENCHMARK_SMOKE_MODEL: ${{ github.event_name == 'workflow_dispatch' && inputs.smoke_model || '' }}
-          BENCHMARK_SMOKE_REASONING: ${{ github.event_name == 'workflow_dispatch' && inputs.smoke_reasoning || '' }}
+          BENCHMARK_SMOKE_PROVIDER: ${{ inputs.benchmark_lane == 'smoke' && (inputs.smoke_provider || 'openai') || 'openai' }}
+          BENCHMARK_SMOKE_MODEL: ${{ inputs.benchmark_lane == 'smoke' && inputs.smoke_model || '' }}
+          BENCHMARK_SMOKE_REASONING: ${{ inputs.benchmark_lane == 'smoke' && inputs.smoke_reasoning || '' }}
         run: |
           candidate="$(git rev-parse HEAD)"
           generation="${{ github.run_id }}-${{ github.run_attempt }}"
-          if [ "$BENCHMARK_MODE" = smoke ] && [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then
-            # Dispatched single-runner smoke lane. prepare-modal-benchmarks.mjs accepts a
-            # one-entry smoke matrix from any provider, so the runner is not pinned to the
-            # checked-in lane profile; the lane's targets, loops and judge are unchanged.
+          # The release gate has a fixed model identity (#183). Leaving the matrix
+          # unset makes prepare-modal-benchmarks.mjs fall back to the lane's own
+          # checked-in profile, so no dispatch-form value can retune the gate.
+          if [ "$BENCHMARK_MODE" = threat-model ]; then
+            BENCHMARK_MODELS_JSON=""
+          elif [ "$BENCHMARK_MODE" = smoke ] && [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then
+            # A dispatched smoke run uses the requested single runner. Its targets,
+            # loops, topology transform, and judge remain pinned by the smoke lane.
             BENCHMARK_MODELS_JSON="$(jq -cn \
               --arg provider "$BENCHMARK_SMOKE_PROVIDER" \
               --arg model "$BENCHMARK_SMOKE_MODEL" \
@@ -242,28 +259,31 @@ jobs:
           MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
           MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          ANTHROPIC_API_KEY: ${{ github.event_name == 'workflow_dispatch' && secrets.ANTHROPIC_API_KEY || '' }}
-          DEEPSEEK_API_KEY: ${{ github.event_name == 'workflow_dispatch' && secrets.DEEPSEEK_API_KEY || '' }}
-          KIMI_API_KEY: ${{ github.event_name == 'workflow_dispatch' && secrets.KIMI_API_KEY || github.event_name == 'workflow_dispatch' && secrets.MOONSHOT_API_KEY || '' }}
-          KIMI_BASE_URL: ${{ github.event_name == 'workflow_dispatch' && secrets.KIMI_BASE_URL || github.event_name == 'workflow_dispatch' && vars.KIMI_BASE_URL || '' }}
+          ANTHROPIC_API_KEY: ${{ (inputs.benchmark_lane == 'full' || inputs.benchmark_lane == 'smoke') && secrets.ANTHROPIC_API_KEY || '' }}
+          DEEPSEEK_API_KEY: ${{ (inputs.benchmark_lane == 'full' || inputs.benchmark_lane == 'smoke') && secrets.DEEPSEEK_API_KEY || '' }}
+          KIMI_API_KEY: ${{ (inputs.benchmark_lane == 'full' || inputs.benchmark_lane == 'smoke') && (secrets.KIMI_API_KEY || secrets.MOONSHOT_API_KEY) || '' }}
+          KIMI_BASE_URL: ${{ (inputs.benchmark_lane == 'full' || inputs.benchmark_lane == 'smoke') && (secrets.KIMI_BASE_URL || vars.KIMI_BASE_URL) || '' }}
         run: |
           : > "$BENCHMARK_CONTROL/launch-attempts.jsonl"
           while IFS=$'\t' read -r config state pair; do
             launch_outcome=succeeded
+            launch_category=launched
             if ! timeout --signal=TERM --kill-after=30s 20m \
               node packages/modal/dist/cli.js launch \
               --config "$BENCHMARK_CONTROL/$config" \
               --state "$BENCHMARK_CONTROL/$state" \
               --mode fresh \
               --repo-root .; then
               launch_outcome=failed
+              launch_category=initial-launch-failed
               echo "Initial detached launch failed for $pair; continuing with the remaining pairs" >&2
             fi
             jq -cn \
               --arg pair "$pair" \
               --arg state "$state" \
               --arg outcome "$launch_outcome" \
-              '{pair: $pair, state_path: $state, outcome: $outcome}' \
+              --arg category "$launch_category" \
+              '{pair: $pair, state_path: $state, outcome: $outcome, category: $category}' \
               >> "$BENCHMARK_CONTROL/launch-attempts.jsonl"
           done < <(jq -r '.pairs[] | [.config_path, .state_path, .pair] | @tsv' "$BENCHMARK_CONTROL/manifest.json")
 
@@ -374,10 +394,10 @@ jobs:
           MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
           MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          ANTHROPIC_API_KEY: ${{ github.event_name == 'workflow_dispatch' && secrets.ANTHROPIC_API_KEY || '' }}
-          DEEPSEEK_API_KEY: ${{ github.event_name == 'workflow_dispatch' && secrets.DEEPSEEK_API_KEY || '' }}
-          KIMI_API_KEY: ${{ github.event_name == 'workflow_dispatch' && secrets.KIMI_API_KEY || github.event_name == 'workflow_dispatch' && secrets.MOONSHOT_API_KEY || '' }}
-          KIMI_BASE_URL: ${{ github.event_name == 'workflow_dispatch' && secrets.KIMI_BASE_URL || github.event_name == 'workflow_dispatch' && vars.KIMI_BASE_URL || '' }}
+          ANTHROPIC_API_KEY: ${{ (inputs.benchmark_lane == 'full' || inputs.benchmark_lane == 'smoke') && secrets.ANTHROPIC_API_KEY || '' }}
+          DEEPSEEK_API_KEY: ${{ (inputs.benchmark_lane == 'full' || inputs.benchmark_lane == 'smoke') && secrets.DEEPSEEK_API_KEY || '' }}
+          KIMI_API_KEY: ${{ (inputs.benchmark_lane == 'full' || inputs.benchmark_lane == 'smoke') && (secrets.KIMI_API_KEY || secrets.MOONSHOT_API_KEY) || '' }}
+          KIMI_BASE_URL: ${{ (inputs.benchmark_lane == 'full' || inputs.benchmark_lane == 'smoke') && (secrets.KIMI_BASE_URL || vars.KIMI_BASE_URL) || '' }}
         run: |
           control_timeout_seconds="$(jq -er \
             '.control_timeout_seconds | select(type == "number" and . == floor and . >= 300)' \
@@ -506,14 +526,14 @@ jobs:
                   all_terminal=false
                   ;;
                 transient-operational-failure)
-                  all_terminal=false
                   if [ ! -f "$BENCHMARK_CONTROL/$config" ]; then
                     jq -n \
                       --arg pair "$pair" \
                       --arg category "launch-config-missing" \
                       '{pair: $pair, terminal_status: "failed", category: $category}' \
                       > "$outcome_file"
                   else
+                    all_terminal=false
                     resume_exit=0
                     timeout --signal=TERM --kill-after=30s 5m \
                     node packages/modal/dist/cli.js launch \
@@ -578,10 +598,10 @@ jobs:
           MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
           MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          ANTHROPIC_API_KEY: ${{ github.event_name == 'workflow_dispatch' && secrets.ANTHROPIC_API_KEY || '' }}
-          DEEPSEEK_API_KEY: ${{ github.event_name == 'workflow_dispatch' && secrets.DEEPSEEK_API_KEY || '' }}
-          KIMI_API_KEY: ${{ github.event_name == 'workflow_dispatch' && secrets.KIMI_API_KEY || github.event_name == 'workflow_dispatch' && secrets.MOONSHOT_API_KEY || '' }}
-          KIMI_BASE_URL: ${{ github.event_name == 'workflow_dispatch' && secrets.KIMI_BASE_URL || github.event_name == 'workflow_dispatch' && vars.KIMI_BASE_URL || '' }}
+          ANTHROPIC_API_KEY: ${{ (inputs.benchmark_lane == 'full' || inputs.benchmark_lane == 'smoke') && secrets.ANTHROPIC_API_KEY || '' }}
+          DEEPSEEK_API_KEY: ${{ (inputs.benchmark_lane == 'full' || inputs.benchmark_lane == 'smoke') && secrets.DEEPSEEK_API_KEY || '' }}
+          KIMI_API_KEY: ${{ (inputs.benchmark_lane == 'full' || inputs.benchmark_lane == 'smoke') && (secrets.KIMI_API_KEY || secrets.MOONSHOT_API_KEY) || '' }}
+          KIMI_BASE_URL: ${{ (inputs.benchmark_lane == 'full' || inputs.benchmark_lane == 'smoke') && (secrets.KIMI_BASE_URL || vars.KIMI_BASE_URL) || '' }}
         run: |
           mkdir -p "$BENCHMARK_RESULTS"
           collect_pair() {
@@ -674,6 +694,13 @@ jobs:
           overwrite: true
           retention-days: 30
 
+      - name: Validate threat-model structural gate
+        if: always() && env.BENCHMARK_MODE == 'threat-model'
+        run: |
+          node scripts/ci/validate-threat-model-benchmark-gate.mjs \
+            "$BENCHMARK_CONTROL/manifest.json" \
+            "$BENCHMARK_RESULTS"
+
       - name: Fail an incomplete matrix after preserving artifacts
         if: always()
         env:

.gitignore

@@ -16,6 +16,8 @@
 !.ultrafuzz/evals/
 !.ultrafuzz/evals/*.yml
 .ultrafuzz/evals/runs/
+packages/*/.ultrafuzz/
+packages/prompts/.pack-test-*/
 node_modules/
 dist/
 dist-test/