feat(gdpval): harden multistage ELO orchestration

resources_servers/gdpval/configs/gdpval_minimax_multistage_judge.yaml

@@ -0,0 +1,193 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# GDPVal overlay: MULTI-STAGE ELO comparison mode judged by a LOCAL, SELF-HOSTED
+# MiniMax-M3. You serve MiniMax-M3 yourself (vendor `vllm/vllm-openai:minimax-m3`
+# container); gym does NOT spawn it — gym talks to your OpenAI-compatible /v1
+# endpoint. This is the self-hosted judge overlay (see
+# gdpval_minimax_selfhosted_judge.yaml) PLUS comparison mode with the reference
+# models + ELO anchors the multi-stage driver fits against.
+#
+# ── STEP 1: serve MiniMax-M3 yourself (see "Local multimodal judge
+#            (MiniMax-M3)" in benchmarks/gdpval/README.md), then:
+#   export MINIMAX_BASE_URL=http://$HEAD_IP:8000/v1   # must include /v1
+#   export MINIMAX_MODEL=minimax-m3                   # match --served-model-name
+#   curl -s "$MINIMAX_BASE_URL/models"                # confirm it's up
+#
+# ── STEP 2: run multi-stage ELO through the benchmark's driver. `--benchmark
+#            gdpval` loads the multistage orchestrator + policy model + dataset;
+#            this overlay swaps in comparison mode + the MiniMax judge; the
+#            ++multistage.* flags opt into staged ELO:
+#
+#   export PERSIST_DELIVERABLES_DIR=/abs/path/to/eval_deliverables_cache
+#   gym eval run \
+#     --model-type vllm_model --benchmark gdpval --split benchmark \
+#     --config resources_servers/gdpval/configs/gdpval_minimax_multistage_judge.yaml \
+#     --output results/gdpval_minimax_multistage.jsonl \
+#     ++multistage.enabled=true \
+#     ++multistage.stages='[{num_tasks: 5}, {num_tasks: 15, num_models: 4}]' \
+#     ++num_repeats=2
+#
+# This example is a small, smoke-sized shape: stage 1 samples 5 tasks, each judged
+# against one reference drawn from ALL the references below, for a coarse ELO;
+# stage 2 judges 15 tasks against the 4 anchors nearest the stage-1 estimate for
+# the final fit. Grow num_tasks for a reportable run (see
+# benchmarks/gdpval/README.md). ++num_repeats=2 gives 2 independent rollouts per
+# task in EVERY stage, pooled into each stage's BT fit (there is NO per-stage
+# repeats key — one inside a stage dict is silently ignored). The headline
+# comparison/eval_elo is the last stage's fit (per-stage values under
+# comparison/stage_<k>/*). Add ++multistage.seed=<int> for reproducible task
+# sampling / reference selection.
+#
+# Stage 1 here is strict: every planned row must be judged before stage 2 is
+# planned. A timeout or other retryable failure stops the run after stage 1
+# (exit 0, no comparison/eval_elo); a row that can no longer be retried fails
+# the run. To accept stage 1 with a bounded number of missing rows, give it a
+# partial_completion policy (see benchmarks/gdpval/README.md).
+#
+# To resume, rerun the same command with --resume, adding RERUN_INCOMPLETE=true
+# to reuse finished deliverables and cached judgements. The resume journal is
+# discarded (renamed *.stale.<ns>) when the dataset, stage shape, seed,
+# num_repeats, references or any model/agent/judge setting other than
+# endpoints, API keys, headers, RERUN_INCOMPLETE and agent concurrency changes.
+#
+# NOTE ON MEDIA: MiniMax-M3 reads images + video but has NO audio tower, so audio
+# deliverables are dropped with a warning (everything else is still graded).
+
+# --- Repoint the judge proxy at your self-hosted MiniMax-M3 endpoint ---------
+gdpval_judge_model:
+  responses_api_models:
+    openai_model:
+      entrypoint: app.py
+      # Your MiniMax-M3 server's OpenAI-compatible endpoint. Must include /v1.
+      openai_base_url: ${oc.env:MINIMAX_BASE_URL,http://127.0.0.1:8000/v1}
+      openai_api_key: ${oc.env:MINIMAX_API_KEY,sk-local}
+      # Must match the server's --served-model-name (defaults to the HF id).
+      openai_model: ${oc.env:MINIMAX_MODEL,MiniMaxAI/MiniMax-M3}
+      # Ride out overload 502/503s instead of failing after 3 quick retries.
+      max_http_attempts: 1800
+      openai_default_headers: {}
+
+# --- Comparison mode + references + single MiniMax-M3 judge -------------------
+gdpval_resources_server:
+  resources_servers:
+    gdpval:
+      verified: false
+      reward_mode: comparison
+      # Reference models the anchored Bradley-Terry MLE ELO is fit against. The
+      # multi-stage driver picks the `num_models` nearest anchors per stage.
+      #
+      # ANCHOR VINTAGE: Artificial Analysis GDPval-AA v2 live board, read
+      # 2026-09-09 (nine anchors), 2026-09-11 (deepseek_v4_flash_0731,
+      # inkling_small, qwen35_122b) and 2026-09-17 (the six under the 2026-09-17
+      # note below). Anchors are held FIXED in the fit, so a stale anchor shifts
+      # every candidate score by roughly the anchor error; re-read them from the
+      # current board before a reportable run.
+      # Each anchor is one exact AA row: deepseek_v4_pro is the plain "DeepSeek
+      # V4 Pro (Reasoning, Max Effort)" row (1223), and deepseek_v4_pro_0813_max
+      # is the "DeepSeek V4 Pro 0813" row (1493). Use deliverables from the exact
+      # row variant (release and effort) whose elo is listed, and check the
+      # variant, not just the family name, when refreshing.
+      # Leave out references whose deliverables are incomplete: missing
+      # deliverables inflate the fit. Keep the anchors spread across the board's
+      # range: a candidate rated above the top anchor is fitted by extrapolation
+      # past the ceiling.
+      # The reference deliverables are NOT shipped with Gym: you must provide
+      # each reference model's per-task `task_<task_id>/` deliverables yourself
+      # (the layout the Stirrup agent persists), produced by the exact model
+      # variant whose `elo` is listed. /gdpval/refs is only a placeholder.
+      # Each deliverables_dir resolves to $GDPVAL_REFS_ROOT/<id>, defaulting to
+      # /gdpval/refs/<id>. Three ways to point at your copies:
+      #   - refs share a parent: export GDPVAL_REFS_ROOT=/abs/path/to/refs
+      #   - refs are scattered (e.g. per-run output dirs, which share no parent):
+      #     edit these paths directly in your local checkout, or leave the file
+      #     as-is and override them from `env.yaml` in the repo root — it is
+      #     gitignored and merges over this overlay, so nothing site-specific
+      #     gets committed and no extra flags are needed per run:
+      #
+      #       gdpval_resources_server:
+      #         resources_servers:
+      #           gdpval:
+      #             reference_models:
+      #               kimi_k26: {deliverables_dir: /abs/path/to/deliverables_cache}
+      #
+      #   - one-off: ++...gdpval.reference_models.<id>.deliverables_dir=/abs/path
+      reference_models:
+        deepseek_v4_flash_0731:
+          deliverables_dir: ${oc.env:GDPVAL_REFS_ROOT,/gdpval/refs}/deepseek_v4_flash_0731
+          elo: 1442
+        deepseek_v4_pro:
+          deliverables_dir: ${oc.env:GDPVAL_REFS_ROOT,/gdpval/refs}/deepseek_v4_pro
+          elo: 1223
+        inkling_small:
+          deliverables_dir: ${oc.env:GDPVAL_REFS_ROOT,/gdpval/refs}/inkling_small
+          elo: 1191
+        glm51_fp8:
+          deliverables_dir: ${oc.env:GDPVAL_REFS_ROOT,/gdpval/refs}/glm51_fp8
+          elo: 1181
+        kimi_k26:
+          deliverables_dir: ${oc.env:GDPVAL_REFS_ROOT,/gdpval/refs}/kimi_k26
+          elo: 1096
+        nemotron3_ultra:
+          deliverables_dir: ${oc.env:GDPVAL_REFS_ROOT,/gdpval/refs}/nemotron3_ultra
+          elo: 1091
+        qwen36_35b:
+          deliverables_dir: ${oc.env:GDPVAL_REFS_ROOT,/gdpval/refs}/qwen36_35b
+          elo: 992
+        qwen35_122b:
+          deliverables_dir: ${oc.env:GDPVAL_REFS_ROOT,/gdpval/refs}/qwen35_122b
+          elo: 925
+        qwen35_397b:
+          deliverables_dir: ${oc.env:GDPVAL_REFS_ROOT,/gdpval/refs}/qwen35_397b
+          elo: 905
+        gptoss_120b:
+          deliverables_dir: ${oc.env:GDPVAL_REFS_ROOT,/gdpval/refs}/gptoss_120b
+          elo: 745
+        gemma4_26b:
+          deliverables_dir: ${oc.env:GDPVAL_REFS_ROOT,/gdpval/refs}/gemma4_26b
+          elo: 693
+        qwen3_30b_thinking:
+          deliverables_dir: ${oc.env:GDPVAL_REFS_ROOT,/gdpval/refs}/qwen3_30b_thinking
+          elo: 274
+        # 2026-09-17 additions. Effort is part of the AA row, so the key
+        # carries it where AA scores efforts separately.
+        kimi_k3_max:
+          deliverables_dir: ${oc.env:GDPVAL_REFS_ROOT,/gdpval/refs}/kimi_k3_max
+          elo: 1548
+        qwen38_27b_medium:
+          deliverables_dir: ${oc.env:GDPVAL_REFS_ROOT,/gdpval/refs}/qwen38_27b_medium
+          elo: 1430
+        gpt55_high:
+          deliverables_dir: ${oc.env:GDPVAL_REFS_ROOT,/gdpval/refs}/gpt55_high
+          elo: 1372
+        claude_sonnet5_high:
+          deliverables_dir: ${oc.env:GDPVAL_REFS_ROOT,/gdpval/refs}/claude_sonnet5_high
+          elo: 1308
+        minimax_m3:
+          deliverables_dir: ${oc.env:GDPVAL_REFS_ROOT,/gdpval/refs}/minimax_m3
+          elo: 1304
+        # 0813 release at max effort, matching AA's only 0813 row (1493). Not a
+        # duplicate of deepseek_v4_pro (1223), which is the earlier release.
+        deepseek_v4_pro_0813_max:
+          deliverables_dir: ${oc.env:GDPVAL_REFS_ROOT,/gdpval/refs}/deepseek_v4_pro_0813_max
+          elo: 1493
+      # Pairwise trials per (task x reference x repeat). 4 debiases position with
+      # swap/no-swap; lower to 2 for a faster smoke test.
+      num_comparison_trials: 4
+      # Single self-hosted MiniMax-M3 judge (no frontier panel).
+      judge_model_server:
+        type: responses_api_models
+        name: gdpval_judge_model
+      judge_responses_create_params_overrides:
+        model: ${oc.env:MINIMAX_MODEL,MiniMaxAI/MiniMax-M3}
+      judge_panel: null
+      # A VLM can't decode an application/pdf data URL, so rasterize PDFs/Office
+      # docs to page images + extracted text.
+      judge_media_mode: images_and_text
+      judge_pdf_render_dpi: 144
+      judge_pdf_max_pages: 30
+      judge_pdf_include_text: true
+      # MiniMax-M3 reads video (native video_url blocks) but has no audio tower.
+      judge_handles_video: true
+      judge_handles_audio: false
+      on_missing_av_judge: warn

resources_servers/gdpval/multistage_elo.py

@@ -20,8 +20,9 @@
    ``responses_api_agents.stirrup_agent.task_distribution``); ``T`` is
    configurable per stage and **defaults to the full task set**,
 2. Includes a set of ``M`` reference models and assigns **each task a single
-   reference** drawn uniformly (equal weight) from that set — so a task is
-   judged against one reference, not all ``M``,
+   reference** from that set — an independent uniform draw by default, with an
+   opt-in seeded balanced assignment — so a task is judged against one reference,
+   not all ``M``,
 3. Fits an anchored Bradley-Terry MLE ELO from that stage's win/loss/tie
    battles pooled per reference (reusing ``comparison.calculate_mle_elo``), and
 4. Uses that estimate to choose the ``M`` references for the next stage.
@@ -47,7 +48,7 @@
 import random
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Dict, List, Mapping, Optional, Sequence
+from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple
 
 from resources_servers.gdpval.comparison import calculate_mle_elo
 
@@ -58,6 +59,45 @@
 PerReferenceTotals = Dict[str, Dict[str, float]]
 
 
+@dataclass
+class PartialStagePolicy:
+    """Opt-in policy for accepting incomplete non-final calibration stages.
+
+    ``min_success_fraction`` gates successful rollout coverage across the whole
+    stage. ``min_per_reference_success_fraction`` and
+    ``min_successful_rows_per_reference`` apply independently to every selected
+    reference model, so an anchor cannot disappear entirely from a partial fit.
+    ``waivable_failure_classes`` lists the persisted failure classes that may be
+    newly waived; it defaults to task timeouts only. Rows already resolved by the standard terminal/max-attempt rules
+    remain eligible, but every omission still counts against the configured
+    coverage floors. Drained/no-persist and missing rows keep the stage open.
+
+    ``tolerate_unresolved`` relaxes *only* the failure-class test: an unresolved
+    row that came back with any persisted failure class is accepted, subject to
+    the coverage floors. It never waives a row that never returned, that was
+    drained/no-persist, or that carries no failure class at all -- their absence
+    is the resume signal that lets the next allocation re-dispatch them intact,
+    so those keep the stage open as they do without the flag. Set it on a
+    calibration stage whose omissions are structural -- a missing reference
+    deliverable does not appear on retry unless the reference tree is repaired,
+    so holding the stage open for it only costs further launches until the row
+    reaches ``NEMO_GYM_MAX_ROLLOUT_ATTEMPTS``.
+
+    Enabling it accepts more than structural gaps: *every* persisted class
+    becomes waivable, including the infrastructure-wide ones
+    (``agent_request_failed``, ``agent_run_error``, ...). A whole agent-server
+    outage mid-stage is therefore indistinguishable from scattered structural
+    omissions, and the coverage floors above are the only remaining guard. Set
+    them deliberately tight before turning this on.
+    """
+
+    min_success_fraction: float = 1.0
+    min_per_reference_success_fraction: float = 1.0
+    min_successful_rows_per_reference: int = 1
+    waivable_failure_classes: Tuple[str, ...] = ("timeout_exceeded",)
+    tolerate_unresolved: bool = False
+
+
 @dataclass
 class StageSpec:
     """Configuration for a single stage.
@@ -69,12 +109,15 @@ class StageSpec:
     available references" (used for the first, broad stage). Each task is judged
     against **one** reference sampled uniformly from the included set. ``seed``
     makes this stage's task sampling and per-task reference assignment
-    reproducible.
+    reproducible. ``partial_completion`` is an explicit, non-final-stage-only
+    escape hatch for accepting a calibrated ELO with bounded missing evidence;
+    it is disabled by default.
     """
 
     num_tasks: Optional[int] = None
     num_models: Optional[int] = None
     seed: Optional[int] = None
+    partial_completion: Optional[PartialStagePolicy] = None
 
 
 # ---------------------------------------------------------------------------
@@ -146,21 +189,33 @@ def assign_task_references(
     reference_ids: Sequence[str],
     *,
     rng: random.Random,
+    balanced: bool = False,
 ) -> Dict[str, str]:
-    """Assign each task a single reference model, sampled with equal probability.
+    """Assign each task a single reference model.
 
     Rather than judging every task against every included reference, each task is
-    compared against **one** reference drawn uniformly (each included reference
-    weighted equally) from ``reference_ids``. Returns a ``{task_id: ref_id}`` map;
-    empty when no references are included.
+    compared against **one** reference from ``reference_ids``. By default each
+    assignment is an independent uniform draw. With ``balanced=True``, assignment
+    slots are shuffled while ensuring per-reference task counts differ by at most
+    one. Returns a ``{task_id: ref_id}`` map; empty when no references are
+    included.
 
     Deterministic given *rng*, so a stage's assignment replays identically on
     resume (it is also recorded in the stage journal).
     """
     refs = list(reference_ids)
     if not refs:
         return {}
-    return {str(tid): rng.choice(refs) for tid in task_ids}
+    if not balanced:
+        return {str(tid): rng.choice(refs) for tid in task_ids}
+
+    tasks = [str(tid) for tid in task_ids]
+    full_cycles, remainder = divmod(len(tasks), len(refs))
+    slots = refs * full_cycles
+    if remainder:
+        slots.extend(rng.sample(refs, remainder))
+    rng.shuffle(slots)
+    return dict(zip(tasks, slots))
 
 
 # ---------------------------------------------------------------------------