feat(launcher): add fast benchmark profiles and named resumable runs

benchmarks/ifbench/config_fast.yaml

@@ -0,0 +1,15 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+config_paths:
+- benchmarks/ifbench/config.yaml
+
+ifbench_benchmark_simple_agent:
+  responses_api_agents:
+    simple_agent:
+      datasets:
+      - name: ifbench
+        type: benchmark
+        jsonl_fpath: benchmarks/ifbench/data/ifbench_benchmark.jsonl
+        prompt_config: benchmarks/ifbench/prompts/default.yaml
+        prepare_script: benchmarks/ifbench/prepare.py
+        num_repeats: 1

benchmarks/launcher/eval.sh

@@ -12,7 +12,9 @@ export VLLM_CONTAINER=${VLLM_CONTAINER:-registry-1.docker.io#vllm/vllm-openai:v0
 checkpoint=""
 profile=""
 smoke=false
-usage() { echo "Usage: eval.sh --profile PATH --checkpoint PATH [--smoke]"; }
+run_name=""
+restarts=1
+usage() { echo "Usage: eval.sh --profile PATH --checkpoint PATH [--smoke] [--name NAME [--restarts N]]"; }
 while (( $# )); do
     case "$1" in
         --profile)
@@ -25,6 +27,8 @@ while (( $# )); do
             shift 2
             ;;
         --smoke) smoke=true; shift ;;
+        --name) run_name=${2:?--name requires a run name}; shift 2 ;;
+        --restarts) restarts=${2:?--restarts requires a count}; shift 2 ;;
         --help|-h) usage; exit 0 ;;
         *) usage >&2; echo "Unknown argument: $1" >&2; exit 2 ;;
     esac
@@ -71,6 +75,14 @@ gym_run_args=(
     ++policy_model_name=$MODEL_NAME
     ++upload_rollouts=false
 )
+# Named run: fixed directory $RUNS_DIR/NAME shared by all restarts; each hop resumes from the cached rollouts
+# (Gym skips resume when no cache exists yet, so the first hop is a normal run).
+if [[ -n $run_name ]]; then
+    export RUN_NAME=$run_name
+    gym_run_args+=(++resume_from_cache=true)
+elif (( restarts > 1 )); then
+    echo '--restarts requires --name' >&2; exit 2
+fi
 printf -v GYM_RUN_ARGS '%q ' "${gym_run_args[@]}"
 export GYM_RUN_ARGS
 
@@ -89,10 +101,14 @@ if [[ $smoke == true ]]; then
 fi
 
 printf 'Profile: %s\nCheckpoint: %s\n' "$profile" "$checkpoint"
+# With --restarts N, N identical jobs are queued under one job name with --dependency=singleton:
+# they run one after another, each resuming the same run dir, and exit immediately once results exist.
+for ((hop = 1; hop <= restarts; hop++)); do
 bash "$launcher_dir/sbatch.sh" \
     --config responses_api_models/vllm_model/configs/vllm_model.yaml \
     --config "$config" \
     ++limit="$limit" ++num_repeats="$repeats" \
     ++num_samples_in_parallel="$concurrency" \
     ++observability_enabled="${OBSERVABILITY_ENABLED:-true}" \
     "${BENCHMARK_EXTRA_ARGS[@]}"
+done

benchmarks/launcher/lighting/fast_common.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+# Shared settings for the fast profiles: capped generation on the fast vLLM profile.
+VLLM_CONFIG="$(dirname -- "${BASH_SOURCE[0]}")/vllm_profile_fast.sh"
+BENCHMARK_EXTRA_ARGS=(
+  "++policy_model.responses_api_models.vllm_model.sampling_overrides.max_tokens=49152"
+  "++policy_model.responses_api_models.vllm_model.sampling_overrides.thinking_token_budget=32768"
+  # a failed agent /run (sandbox infra error -> HTTP 500) becomes a sidecar failure row instead of aborting the run
+  "++route_failures_to_sidecar=true"
+)

benchmarks/launcher/lighting/ifbench_fast.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+source "$(dirname -- "${BASH_SOURCE[0]}")/fast_common.sh"
+BENCHMARK=ifbench_fast
+BENCHMARK_CONFIG=benchmarks/ifbench/config_fast.yaml
+BENCHMARK_CONCURRENCY=512
+BENCHMARK_EXTRA_ARGS+=(++overwrite_metrics_conflicts=true)

benchmarks/launcher/lighting/swe_multilingual_fast.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+source "$(dirname -- "${BASH_SOURCE[0]}")/fast_common.sh"
+BENCHMARK=swe_multilingual_fast
+BENCHMARK_CONFIG=benchmarks/swebench/multilingual/opencode_fast.yaml
+BENCHMARK_CONCURRENCY=256

benchmarks/launcher/lighting/swe_verified_fast.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+source "$(dirname -- "${BASH_SOURCE[0]}")/fast_common.sh"
+BENCHMARK=swe_verified_fast
+BENCHMARK_CONFIG=benchmarks/swebench/verified/opencode_fast.yaml
+BENCHMARK_CONCURRENCY=256

benchmarks/launcher/lighting/terminal_bench_2_1_fast.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+source "$(dirname -- "${BASH_SOURCE[0]}")/fast_common.sh"
+BENCHMARK=tb21fast
+BENCHMARK_CONFIG=benchmarks/terminal_bench_2_1/terminus_2_fast.yaml
+BENCHMARK_CONCURRENCY=512

benchmarks/launcher/lighting/terminal_bench_2_1_opencode_fast.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+source "$(dirname -- "${BASH_SOURCE[0]}")/fast_common.sh"
+BENCHMARK=tb21opencodefast
+BENCHMARK_CONFIG=benchmarks/terminal_bench_2_1/opencode_fast.yaml
+BENCHMARK_CONCURRENCY=512

benchmarks/launcher/lighting/terminal_bench_2_1_smoke4.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+source "$(dirname -- "${BASH_SOURCE[0]}")/fast_common.sh"
+BENCHMARK=tb21smoke4
+BENCHMARK_CONFIG=benchmarks/terminal_bench_2_1/terminus_2_smoke4.yaml
+BENCHMARK_CONCURRENCY=512

benchmarks/launcher/lighting/vllm_profile_fast.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+# Lightning 3.5 BF16, TP1, native MTP from the served checkpoint.
+MODEL_NAME=lightning35-bf16
+# Pin to the nightly the Sep 20 TB 2.1 runs used (0.29.1rc1.dev422+gd05da62e9).
+# Later nightlies renamed --enable-mamba-fine-grained-prefix-cache.
+export VLLM_CONTAINER="registry-1.docker.io#vllm/vllm-openai:nightly-d05da62e9ccdf8e342b15bf6785d83224cc165af"
+export ROUTER_BALANCE_ABS_THRESHOLD=40
+export ROUTER_BALANCE_REL_THRESHOLD=2
+# The checkpoint declares 256K; explicitly allow the 1M serving limit.
+export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
+VLLM_COMMON_ARGS=(
+    --trust-remote-code
+    --disable-uvicorn-access-log
+    --gpu-memory-utilization 0.9
+    --tensor-parallel-size 1
+    --data-parallel-size 1
+    --distributed-executor-backend mp
+    --api-server-count 16
+    --renderer-num-workers 16
+    --enable-auto-tool-choice
+    --tool-call-parser qwen3_coder
+    --reasoning-parser nemotron_v3
+    --enable-chunked-prefill
+    --enable-prefix-caching
+    --max-model-len 1048576
+    --max-num-batched-tokens 32768
+    --max-num-seqs 128
+    --kv-cache-dtype fp8
+    --no-disable-hybrid-kv-cache-manager
+    --no-async-scheduling
+    --block-size 128
+    --mamba-cache-mode align
+    --enable-mamba-fine-grained-prefix-cache
+    --prefix-match-unit 16
+    --mamba-ssm-cache-dtype float32
+    --model-loader-extra-config '{"enable_multithread_load": true, "num_threads": 16}'
+)
+
+VLLM_COMMON_ARGS+=(
+    --reasoning-config '{"reasoning_start_str": "<think>", "reasoning_end_str": "</think>"}'
+    --speculative-config '{"method":"mtp","num_speculative_tokens":4}'
+)

benchmarks/launcher/sbatch.sh

@@ -31,7 +31,7 @@ source "$VLLM_CONFIG"
 host=$(hostname)
 IFS=, read -r -a gpus <<< "${CUDA_VISIBLE_DEVICES:-0,1,2,3}"
 (( ${#gpus[@]} >= REPLICAS_PER_NODE )) || { echo "Not enough visible GPUs for replicas" >&2; exit 1; }
-logs="$RUN_DIR/inference"
+logs="$RUN_DIR/inference${RUN_NAME:+-$SLURM_JOB_ID}"
 mkdir -p "$logs"
 pids=()
 cleanup() {
@@ -101,6 +101,8 @@ if [[ -d /mnt/gym-dev ]]; then
         --exclude=cache --exclude=.cache --exclude=logs --exclude=results \
         --exclude=runs --exclude=wandb --exclude='swe_*_setup' \
         -cf - . | tar -C /opt/nemo-gym -xf -
+    # setup.cfg uses egg_base=cache; the exclude above drops that dir and the editable install fails without it
+    mkdir -p /opt/nemo-gym/cache
 fi
 
 source /opt/nemo_gym_venv/bin/activate
@@ -145,8 +147,13 @@ EVAL
 # 4) Slurm job — run inference, router, and Gym; stop all when any exits.
 export batch_command=$(cat <<'BATCH'
 set -euo pipefail
-export RUN_DIR="$RUNS_DIR/$SLURM_JOB_ID-$BENCHMARK"
-mkdir -p "$RUN_DIR/inference"
+export RUN_DIR="${RUN_NAME:+$RUNS_DIR/$RUN_NAME}"
+: "${RUN_DIR:=$RUNS_DIR/$SLURM_JOB_ID-$BENCHMARK}"
+if [[ -f "$RUN_DIR/rollouts_aggregate_metrics.json" ]]; then
+    echo "Run $RUN_DIR already has final metrics; nothing to do."
+    exit 0
+fi
+mkdir -p "$RUN_DIR/inference${RUN_NAME:+-$SLURM_JOB_ID}"
 nodes=($(scontrol show hostnames "$SLURM_JOB_NODELIST"))
 export ALL_NODES="${nodes[*]}" ROUTER_NODE="${nodes[0]}"
 container_args=(
@@ -171,7 +178,7 @@ srun --overlap --exact --nodes=1 --ntasks=1 --gpus=0 \
     --cpus-per-task=8 --nodelist="$ROUTER_NODE" \
     --container-image="$ROUTER_CONTAINER" --container-workdir=/ \
     --no-container-mount-home --no-container-entrypoint bash -c "$router_command" \
-    > "$RUN_DIR/inference/router.log" 2>&1 &
+    > "$RUN_DIR/inference${RUN_NAME:+-$SLURM_JOB_ID}/router.log" 2>&1 &
 pids+=("$!")
 
 srun --overlap --exact --nodes=1 --ntasks=1 --gpus=0 \
@@ -193,24 +200,38 @@ BATCH
 # Allocate four GPUs per node, independently of replica count.
 # Hold briefly so the output directory exists before Slurm opens its log.
 NUM_NODES=${NUM_NODES:-1}
+if [[ -n ${RUN_NAME:-} ]]; then
+    # Named run: one shared directory, one Slurm job name, jobs serialized with singleton.
+    run_dir="$RUNS_DIR/$RUN_NAME"
+    mkdir -p "$run_dir"
+    naming=(--job-name="gym-$RUN_NAME-$USER" --dependency=singleton --output="$run_dir/slurm-%j.log")
+else
+    naming=(--job-name="gym-$EXPERIMENT_NAME-$USER" --output="$RUNS_DIR/%j-$BENCHMARK/slurm.log")
+fi
 # Evaluation can leave GPUs idle while task sandboxes run tools and verifiers.
 # Match the exemption to this allocation's four-hour wall time.
 default_slurm_comment='{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"240","reason":"benchmarking","description":"Benchmark evaluation with idle periods during sandbox tools and verification"}}'
 job=$(sbatch --hold --parsable --nodes="$NUM_NODES" --ntasks-per-node=1 --gpus-per-node=4 \
     --exclusive --segment="$NUM_NODES" --time=04:00:00 \
     --comment="${SLURM_COMMENT:-$default_slurm_comment}" \
-    --job-name="gym-$EXPERIMENT_NAME-$USER" --output="$RUNS_DIR/%j-$BENCHMARK/slurm.log" \
+    "${naming[@]}" \
     --wrap 'exec bash -c "$batch_command"')
 job=${job%%;*}
-run_dir="$RUNS_DIR/$job-$BENCHMARK"
-mkdir -p "$run_dir" || { scancel "$job"; exit 1; }
+if [[ -z ${RUN_NAME:-} ]]; then
+    run_dir="$RUNS_DIR/$job-$BENCHMARK"
+    mkdir -p "$run_dir" || { scancel "$job"; exit 1; }
+fi
 printf 'Submitted eval job %s\nLogs and results: %s\n' "$job" "$run_dir"
 
 # 5) Sandbox cleanup — also runs if the GPU job fails or is cancelled.
 export CLEANUP_RUN_ID="$job"
 export cleanup_command=$(cat <<'CLEANUP'
 set -euo pipefail
-exec /opt/nemo_gym_venv/bin/python /opt/nemo-gym/nemo_gym/sandbox/providers/opensandbox/cleanup_sandboxes.py \
+# The base image may predate cleanup_sandboxes.py; prefer the mounted dev checkout when present.
+script=/opt/nemo-gym/nemo_gym/sandbox/providers/opensandbox/cleanup_sandboxes.py
+[[ -f /mnt/gym-dev/nemo_gym/sandbox/providers/opensandbox/cleanup_sandboxes.py ]] \
+    && script=/mnt/gym-dev/nemo_gym/sandbox/providers/opensandbox/cleanup_sandboxes.py
+exec /opt/nemo_gym_venv/bin/python "$script" \
     --domain "$OPENSANDBOX_DOMAIN" --api-key "$OPENSANDBOX_API_KEY" \
     --run-id "$CLEANUP_RUN_ID" --user "$NEMO_GYM_USER" --reap
 CLEANUP
@@ -220,7 +241,7 @@ if ! sbatch --parsable --dependency="afterany:$job" \
     --partition=cpu --qos=cpu-normal --gres=none --gpus-per-node=0 \
     --nodes=1 --ntasks=1 --cpus-per-task=2 --mem=4G --time=00:30:00 \
     --job-name="gym-cleanup-$job" --output="$run_dir/cleanup.log" \
-    --wrap 'exec srun --container-image="$GYM_CONTAINER" --container-workdir=/opt/nemo-gym --no-container-mount-home --no-container-entrypoint bash -c "$cleanup_command"'; then
+    --wrap 'exec srun --container-image="$GYM_CONTAINER" ${GYM_DEV_CHECKOUT:+--container-mounts=$GYM_DEV_CHECKOUT:/mnt/gym-dev:ro} --container-workdir=/opt/nemo-gym --no-container-mount-home --no-container-entrypoint bash -c "$cleanup_command"'; then
     scancel "$job"
     echo "Cancelled held eval job $job: sandbox cleanup could not be scheduled." >&2
     exit 1

benchmarks/swebench/multilingual/opencode_fast.yaml

@@ -0,0 +1,15 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+config_paths:
+- benchmarks/swebench/multilingual/opencode.yaml
+
+swebench_multilingual_opencode_sandboxed_agent:
+  responses_api_agents:
+    opencode_sandboxed_agent:
+      sandbox_timeout: 3600
+      datasets:
+        - name: swebench_multilingual
+          type: benchmark
+          jsonl_fpath: benchmarks/swebench/data/swebench_multilingual_benchmark.jsonl
+          prepare_script: benchmarks/swebench/multilingual/prepare.py
+          num_repeats: 1

benchmarks/swebench/verified/opencode_fast.yaml

@@ -0,0 +1,15 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+config_paths:
+- benchmarks/swebench/verified/opencode.yaml
+
+swebench_verified_opencode_sandboxed_agent:
+  responses_api_agents:
+    opencode_sandboxed_agent:
+      sandbox_timeout: 3600
+      datasets:
+        - name: swebench_verified
+          type: benchmark
+          jsonl_fpath: benchmarks/swebench/data/swebench_verified_benchmark.jsonl
+          prepare_script: benchmarks/swebench/verified/prepare.py
+          num_repeats: 1

benchmarks/terminal_bench_2_1/opencode_fast.yaml

@@ -0,0 +1,15 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+config_paths:
+- benchmarks/terminal_bench_2_1/opencode.yaml
+
+terminal_bench_2_1_opencode_sandboxed_agent:
+  responses_api_agents:
+    opencode_sandboxed_agent:
+      sandbox_timeout: 3600
+      datasets:
+        - name: terminal_bench_2_1
+          type: benchmark
+          jsonl_fpath: benchmarks/terminal_bench_2_1/data/benchmark.jsonl
+          prepare_script: benchmarks/terminal_bench_2_1/prepare.py
+          num_repeats: 1

benchmarks/terminal_bench_2_1/terminus_2_fast.yaml

@@ -0,0 +1,11 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+config_paths:
+- benchmarks/terminal_bench_2_1/terminus_2.yaml
+
+terminal_bench_2_1_terminus_2_sandboxed_agent:
+  responses_api_agents:
+    terminus_2_sandboxed_agent:
+      max_turns: 250
+      sandbox_timeout: 3600
+      model_output_limit: 49152

benchmarks/terminal_bench_2_1/terminus_2_smoke4.yaml

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+config_paths:
+- benchmarks/terminal_bench_2_1/terminus_2_fast.yaml
+
+terminal_bench_2_1_terminus_2_sandboxed_agent:
+  responses_api_agents:
+    terminus_2_sandboxed_agent:
+      datasets:
+        - name: terminal_bench_2_1
+          type: benchmark
+          jsonl_fpath: benchmarks/terminal_bench_2_1/data/smoke4_distinct.jsonl
+          prepare_script: benchmarks/terminal_bench_2_1/prepare.py
+          num_repeats: 1