[GDPVal] Fix judge input loading, panel routing, and Stage 1 loss handling

benchmarks/gdpval/config.yaml

@@ -32,9 +32,7 @@
 # otherwise behaves exactly like the standard single-pass collection.
 rollout_collection_driver: resources_servers.gdpval.multistage_orchestrator:run_rollout_collection
 
-# Judge model — proxy to the NVIDIA inference API. A single proxy server fronts
-# every frontier judge (Gemini, GPT, Claude); the panel members below select
-# among them by model id, so one server is enough.
+# Preserve the configurable single-judge endpoint for existing overlays.
 gdpval_judge_model:
   responses_api_models:
     openai_model:
@@ -43,6 +41,36 @@ gdpval_judge_model:
       openai_api_key: ${oc.env:JUDGE_API_KEY,dummy}
       openai_model: ${oc.env:JUDGE_MODEL_NAME,gcp/google/gemini-3.1-pro-preview}
 
+# Each OpenAI proxy fixes the upstream model ID, so panel members use separate
+# proxies even when they share the NVIDIA inference API endpoint.
+gdpval_gpt55_judge_model:
+  responses_api_models:
+    openai_model:
+      entrypoint: app.py
+      openai_base_url: ${oc.env:JUDGE_BASE_URL,https://inference-api.nvidia.com/v1}
+      openai_api_key: ${oc.env:JUDGE_API_KEY,dummy}
+      openai_model: ${oc.env:JUDGE_GPT_MODEL,openai/openai/gpt-5.5}
+
+gdpval_gemini31_judge_model:
+  responses_api_models:
+    openai_model:
+      entrypoint: app.py
+      openai_base_url: ${oc.env:JUDGE_BASE_URL,https://inference-api.nvidia.com/v1}
+      openai_api_key: ${oc.env:JUDGE_GEMINI_API_KEY,${oc.env:JUDGE_API_KEY,dummy}}
+      openai_model: ${oc.env:JUDGE_GEMINI_MODEL,gcp/google/gemini-3.1-pro-preview}
+      extra_body: {reasoning_effort: high}
+      max_concurrent_requests: ${oc.env:GDPVAL_GEMINI_MAX_CONCURRENT_REQUESTS,2}
+
+gdpval_claude48_judge_model:
+  responses_api_models:
+    openai_model:
+      entrypoint: app.py
+      openai_base_url: ${oc.env:JUDGE_BASE_URL,https://inference-api.nvidia.com/v1}
+      openai_api_key: ${oc.env:JUDGE_API_KEY,dummy}
+      openai_model: ${oc.env:JUDGE_CLAUDE_MODEL,aws/anthropic/bedrock-claude-opus-4-8}
+      # Keep the gateway timeout aligned with the 900-second judge request bound.
+      extra_body: {thinking: {type: adaptive}, output_config: {effort: high}, timeout: 900, request_timeout: 900}
+
 # GDPVal resources server (rubric scoring by default; switch to comparison via override).
 gdpval_resources_server:
   resources_servers:
@@ -62,34 +90,53 @@ gdpval_resources_server:
       num_comparison_trials: 4
       preconvert_office_to_pdf: true
       preconvert_max_concurrent: 1
+      judge_pdf_max_pages: 1000
       judge_model_server:
         type: responses_api_models
         name: gdpval_judge_model
       judge_responses_create_params_overrides: {}
       # Multi-judge panel (applies to every judge mode — rubric text/visual/
       # structured and pairwise comparison, including multi-stage ELO): a panel
       # of frontier judges, one sampled per comparison/scoring call. This is the
-      # default. All three route through the single gdpval_judge_model proxy and
-      # differ only by model id + reasoning knobs; set a per-member model_server
-      # to point at distinct endpoints. To fall back to a single judge, override
+      # default. Each member routes through its own fixed-model proxy.
+      # To fall back to a single judge, override
       # ++gdpval_resources_server.resources_servers.gdpval.judge_panel=null
       # (the lone judge then comes from judge_model_server above).
       judge_panel:
         - name: gpt-5.5
           # Served via NVIDIA inference on the OpenAI /v1/chat/completions route.
           model: ${oc.env:JUDGE_GPT_MODEL,openai/openai/gpt-5.5}
+          model_server: {type: responses_api_models, name: gdpval_gpt55_judge_model}
+          media_mode: images_and_text
           create_params_overrides: { reasoning_effort: medium }
+          raster_dpi_tiers: [144, 120, 108, 96, 90, 84, 81, 80, 72]
+          max_total_image_base64_bytes: 12000000
+          max_serialized_request_bytes: 30000000
         - name: gemini-3.1-pro
           model: ${oc.env:JUDGE_GEMINI_MODEL,gcp/google/gemini-3.1-pro-preview}
+          model_server: {type: responses_api_models, name: gdpval_gemini31_judge_model}
+          media_mode: native_pdf_overflow_images
           create_params_overrides: { extra_body: { reasoning_effort: high } }
           # Reads audio AND video natively — tasks with those files route here.
           handles_audio: true
           handles_video: true
+          max_video_files: 10
+          max_native_pdf_pages: 1000
+          max_native_pdf_bytes: 50000000
+          max_native_pdf_bytes_per_document: 50000000
+          max_serialized_request_bytes: 495000000
         - name: claude-opus-4.8
           # Served via NVIDIA inference (Bedrock). The proxy exposes it on the
           # Anthropic /v1/messages route; extra_body.thinking enables reasoning.
           model: ${oc.env:JUDGE_CLAUDE_MODEL,aws/anthropic/bedrock-claude-opus-4-8}
-          create_params_overrides: { extra_body: { thinking: { type: enabled } } }
+          model_server: {type: responses_api_models, name: gdpval_claude48_judge_model}
+          media_mode: native_pdf
+          create_params_overrides: {temperature: null, extra_body: {thinking: {type: adaptive}, output_config: {effort: high}}}
+          max_native_pdf_pages: 100
+          max_native_pdf_documents: 5
+          max_native_pdf_bytes: 25165824
+          max_image_base64_bytes: 5242880
+          max_serialized_request_bytes: 440401920
       # Seed for reproducible per-call judge sampling (null = seed only on task
       # identity, still stable per task).
       judge_sampling_seed: ${oc.env:JUDGE_SAMPLING_SEED,null}

resources_servers/gdpval/app.py

@@ -184,6 +184,9 @@ class JudgePanelMember(BaseModel):
     # max_native_pdf_bytes eligibility ceiling, overflow mode rasterizes only
     # documents above this lossless representation threshold.
     max_native_pdf_bytes_per_document: Optional[int] = None
+    max_image_base64_bytes: Optional[int] = Field(default=None, gt=0)
+    max_total_image_base64_bytes: Optional[int] = Field(default=None, gt=0)
+    max_video_files: Optional[int] = Field(default=None, ge=0)
     # Tried in order for images_and_text. The first lossless projection below
     # max_serialized_request_bytes wins; otherwise this member is excluded.
     raster_dpi_tiers: Tuple[int, ...] = ()
@@ -242,6 +245,11 @@ class GDPValResourcesServerConfig(BaseResourcesServerConfig):
     # selected from fewer votes than the configured scientific contract.
     strict_comparison_trials: bool = False
 
+    # Stage 1 only: explicit imported task IDs with no finish marker may count
+    # as audited losses. Missing paths outside this list remain failures.
+    count_eval_missing_as_loss: bool = False
+    missing_eval_task_ids: List[str] = Field(default_factory=list)
+
     # ELO assigned to the (legacy single) reference model in pairwise mode.
     # Ignored when ``reference_models`` is set (each carries its own ``elo``).
     reference_elo: float = _DEFAULT_REFERENCE_ELO
@@ -269,6 +277,12 @@ class GDPValResourcesServerConfig(BaseResourcesServerConfig):
     judge_pdf_include_text: bool = True
     # Exact request-wide image cap for raster and PDF-overflow transports.
     judge_max_images_per_request: int = 450
+    # Include nested task inputs while leaving submission directories shallow.
+    judge_reference_files_recursive: bool = False
+    # Use the eval tree's prepared copies of the original benchmark inputs,
+    # persisted from host downloads separately from model-authored submissions.
+    # Enable only after validating those inputs during preparation.
+    judge_reference_files_from_eval: bool = False
     # Whether the (single) local judge natively reads audio / video, tracked
     # SEPARATELY because MiniMax-M3 — the reference self-hosted judge — reads video
     # but NOT audio (its config has an image + video tower but no audio config). So
@@ -501,6 +515,9 @@ def _url(server: ModelServerRef) -> str:
                     max_native_pdf_documents=member.max_native_pdf_documents,
                     max_native_pdf_bytes=member.max_native_pdf_bytes,
                     max_native_pdf_bytes_per_document=member.max_native_pdf_bytes_per_document,
+                    max_image_base64_bytes=member.max_image_base64_bytes,
+                    max_total_image_base64_bytes=member.max_total_image_base64_bytes,
+                    max_video_files=member.max_video_files,
                     raster_dpi_tiers=tuple(member.raster_dpi_tiers),
                     max_serialized_request_bytes=member.max_serialized_request_bytes,
                 )
@@ -789,6 +806,44 @@ async def _verify_comparison(self, body: GDPValVerifyRequest) -> GDPValVerifyRes
 
         if eval_task_dir is None or not task_attempted(str(eval_task_dir)):
             print(f"[gdpval] eval deliverable missing for task {body.task_id}", flush=True)
+            if (
+                self.config.count_eval_missing_as_loss
+                and body.stage_index == 1
+                and body.task_id in self.config.missing_eval_task_ids
+            ):
+                per_reference = {
+                    ref_id: {
+                        "wins": 0,
+                        "losses": self.config.num_comparison_trials * len(dirs),
+                        "ties": 0,
+                        "reference_elo": self._references[ref_id].elo,
+                        "ref_repeat_count": len(dirs),
+                    }
+                    for ref_id, dirs in ref_dirs_by_id.items()
+                }
+                total_losses = sum(counts["losses"] for counts in per_reference.values())
+                return GDPValVerifyResponse(
+                    **body.model_dump(),
+                    reward=0.0,
+                    verify_mode="comparison",
+                    judge_response={
+                        "manual_imputation": "eval_missing_as_loss",
+                        "per_reference": per_reference,
+                        "total_wins": 0,
+                        "total_losses": total_losses,
+                        "total_ties": 0,
+                        "total_judged": total_losses,
+                        "total_invalid": 0,
+                        "ref_errors": {},
+                    },
+                    win=False,
+                    loss=True,
+                    tie=False,
+                    total_wins=0,
+                    total_losses=total_losses,
+                    total_ties=0,
+                    per_reference=per_reference,
+                )
             if self.config.strict_comparison_trials:
                 raise RuntimeError(f"strict comparison trial contract failed for task {body.task_id}: eval_missing")
             # Terminal for the same reason as reference_missing above: a
@@ -846,6 +901,9 @@ def _client_for(judge: ResolvedJudge) -> Any:
                 max_native_pdf_documents=rj.max_native_pdf_documents,
                 max_native_pdf_bytes=rj.max_native_pdf_bytes,
                 max_native_pdf_bytes_per_document=rj.max_native_pdf_bytes_per_document,
+                max_image_base64_bytes=rj.max_image_base64_bytes,
+                max_total_image_base64_bytes=rj.max_total_image_base64_bytes,
+                max_video_files=rj.max_video_files,
                 raster_dpi_tiers=rj.raster_dpi_tiers,
                 max_serialized_request_bytes=rj.max_serialized_request_bytes,
             )
@@ -911,6 +969,9 @@ async def _section(path: Optional[Path], mode: str, render_dpi: int) -> List[dic
                     include_text=self.config.judge_pdf_include_text,
                     audio_capable=audio_capable,
                     video_capable=video_capable,
+                    recursive=bool(
+                        self.config.judge_reference_files_recursive and path and path.name == "reference_files"
+                    ),
                 )
             return section_cache[key]
 
@@ -952,7 +1013,8 @@ def _native_pdf_stats(*sections: List[dict]) -> Dict[str, int]:
                 ref_wins = ref_losses = ref_ties = 0
                 ref_judged_repeats = 0
                 for ref_dir in dirs:
-                    refs_subdir = ref_dir / "reference_files"
+                    refs_root = eval_task_dir if self.config.judge_reference_files_from_eval else ref_dir
+                    refs_subdir = refs_root / "reference_files"
                     attempted_matchups += 1
                     # Seed per (task, ref_id, ref_repeat) so judge sampling is
                     # reproducible and each reference subset draws independently —
@@ -1459,6 +1521,20 @@ def _fit_mle(ref_totals: Dict[str, List[Any]]) -> tuple[Optional[float], Optiona
                     extra[f"{prefix}/normalized_elo"] = stage_norm
                 extra[f"{prefix}/num_references"] = stage_nref
                 extra[f"{prefix}/num_tasks"] = len({vr.get("task_id") for vr in stage_responses})
+                imputed, judged_rows = [], []
+                for vr in stage_responses:
+                    judge_response = vr.get("judge_response")
+                    if (
+                        isinstance(judge_response, dict)
+                        and judge_response.get("manual_imputation") == "eval_missing_as_loss"
+                    ):
+                        imputed.append(vr)
+                    elif sum(_votes(vr)) > 0:
+                        judged_rows.append(vr)
+                extra[f"{prefix}/judged_tasks"] = len({vr.get("task_id") for vr in judged_rows})
+                extra[f"{prefix}/judged_votes"] = sum(sum(_votes(vr)) for vr in judged_rows)
+                extra[f"{prefix}/imputed_loss_tasks"] = len({vr.get("task_id") for vr in imputed})
+                extra[f"{prefix}/imputed_loss_votes"] = sum(_votes(vr)[1] for vr in imputed)
 
             headline_stage_index: Optional[int] = None
             headline: Optional[tuple[Optional[float], Optional[float], int]] = None