From d7ab56514ad903d8d81424bc170507a78a76cf9f Mon Sep 17 00:00:00 2001 From: Serge Panev Date: Mon, 27 Jul 2026 17:54:42 -0700 Subject: [PATCH 01/14] [GDPVal] Add Slurm orchestration scripts for multi-allocation rollouts Signed-off-by: Serge Panev --- .gitignore | 1 + benchmarks/gdpval/datasets/example.env | 54 +++ benchmarks/gdpval/slurm/README.md | 93 ++++ .../gdpval/slurm/bootstrap_gdpval_runtime.sh | 168 +++++++ benchmarks/gdpval/slurm/cluster.env.example | 64 +++ .../gdpval/slurm/monitor_gdpval_pipeline.sh | 157 ++++++ benchmarks/gdpval/slurm/rpm2cpio_stdin.sh | 14 + .../gdpval/slurm/run_gdpval_client.sbatch | 119 +++++ .../slurm/run_gdpval_continuation_gate.sbatch | 335 +++++++++++++ .../run_gdpval_continuation_monitor.sbatch | 36 ++ .../slurm/run_parallel_bf16-glm52-vllm.sh | 453 ++++++++++++++++++ .../slurm/run_parallel_fp8-glm52-vllm.sh | 449 +++++++++++++++++ .../gdpval/slurm/smoke_apptainer_worker.sh | 44 ++ .../slurm/submit_gdpval_continuation.sh | 188 ++++++++ .../gdpval/slurm/submit_gdpval_glm52.sh | 405 ++++++++++++++++ .../slurm/validate_gdpval_resume_state.py | 169 +++++++ .../gdpval/slurm/wait_for_slurm_account.sh | 198 ++++++++ 17 files changed, 2947 insertions(+) create mode 100644 benchmarks/gdpval/datasets/example.env create mode 100644 benchmarks/gdpval/slurm/README.md create mode 100755 benchmarks/gdpval/slurm/bootstrap_gdpval_runtime.sh create mode 100644 benchmarks/gdpval/slurm/cluster.env.example create mode 100644 benchmarks/gdpval/slurm/monitor_gdpval_pipeline.sh create mode 100755 benchmarks/gdpval/slurm/rpm2cpio_stdin.sh create mode 100755 benchmarks/gdpval/slurm/run_gdpval_client.sbatch create mode 100755 benchmarks/gdpval/slurm/run_gdpval_continuation_gate.sbatch create mode 100755 benchmarks/gdpval/slurm/run_gdpval_continuation_monitor.sbatch create mode 100755 benchmarks/gdpval/slurm/run_parallel_bf16-glm52-vllm.sh create mode 100755 benchmarks/gdpval/slurm/run_parallel_fp8-glm52-vllm.sh create mode 100755 benchmarks/gdpval/slurm/smoke_apptainer_worker.sh create mode 100755 benchmarks/gdpval/slurm/submit_gdpval_continuation.sh create mode 100755 benchmarks/gdpval/slurm/submit_gdpval_glm52.sh create mode 100644 benchmarks/gdpval/slurm/validate_gdpval_resume_state.py create mode 100755 benchmarks/gdpval/slurm/wait_for_slurm_account.sh diff --git a/.gitignore b/.gitignore index 21bedd7777c..41db61806d0 100644 --- a/.gitignore +++ b/.gitignore @@ -236,3 +236,4 @@ env.yaml responses_api_agents/cvdp_agent/deps/ resources_servers/cvdp/data/gym_agentic_code_generation_no_commercial.jsonl resources_servers/cvdp/data/gym_cvdp_nonagentic_code_generation_no_commercial.jsonl +benchmarks/gdpval/slurm/cluster.env diff --git a/benchmarks/gdpval/datasets/example.env b/benchmarks/gdpval/datasets/example.env new file mode 100644 index 00000000000..76664b35175 --- /dev/null +++ b/benchmarks/gdpval/datasets/example.env @@ -0,0 +1,54 @@ +# GDPVal dataset profile — template. +# +# One profile per batch. Adding a batch means adding a profile, not editing the +# orchestration scripts. Select one with GDPVAL_ENV_FILE: +# +# GDPVAL_ENV_FILE=/path/to/my_batch.env ./run_gdpval_rollouts.sh preflight +# +# Keep real profiles outside the repo when they point at non-public inputs: +# the paths below are read from the cluster filesystem, not shipped with Gym. + +# Human-readable batch id. Used in log lines and run roots. +export GDPVAL_DATASET_NAME="my-batch-2026-01-01" + +# Absolute path to the batch JSONL, readable from every compute node. +export GDPVAL_INPUT_JSONL="/path/to/my_batch.jsonl" + +# Row count and content hash. Both are checked before a run starts and again on +# resume, so a batch cannot silently change underneath a multi-allocation run. +export GDPVAL_EXPECTED_COUNT="200" +export GDPVAL_EXPECTED_SOURCE_SHA256="" + +# Anchored regex every task_id must match. Catches a batch loaded in the wrong +# format long before the rollouts are wasted. +export GDPVAL_TASK_ID_PATTERN='task_[0-9a-f]{32}$' + +# "local" — reference_file_urls are filesystem paths, read directly at runtime +# "https" — reference_file_urls are URLs to be fetched +export GDPVAL_REFERENCE_MODE="local" + +# Require the input JSONL to be mode 0600. Set for confidential batches. +export GDPVAL_REQUIRE_PRIVATE_FILES="false" + +# Optional JSON of per-task reference-file repairs, for batches whose reference +# URLs need patching. Empty when the batch needs none. +export GDPVAL_REFERENCE_OVERRIDES="" + +# Run root, so artifacts from different batches never mix. +export GDPVAL_RUN_TAG="my-model_my-batch_2026-01-01" + +# Served model name. Dataset-scoped so endpoint validation and the recorded +# response model cannot be confused with a different batch. +export GDPVAL_MODEL_BASE_NAME="my-model-my-batch" + +# Agent turn budget. 250 is the GDPVal config default. Per-turn generation is +# unaffected by this. +export GDPVAL_AGENT_MAX_TURNS="250" + +# Set after `preflight` produces a launch input, so later actions can prove the +# staged file has not drifted: +# export GDPVAL_EXPECTED_LAUNCH_SHA256="" + +# Any task_id from the batch, used by the `smoke` and `run` actions to validate +# the pipeline end-to-end before committing an allocation. `full` skips it. +export GDPVAL_SMOKE_TASK_ID="" diff --git a/benchmarks/gdpval/slurm/README.md b/benchmarks/gdpval/slurm/README.md new file mode 100644 index 00000000000..b1b575c174d --- /dev/null +++ b/benchmarks/gdpval/slurm/README.md @@ -0,0 +1,93 @@ +# GDPVal rollout orchestration on Slurm + +Reference scripts for collecting GDPVal rollouts on a Slurm cluster with a +self-hosted vLLM endpoint, across more wall-clock time than one allocation +gives you. + +They exist because a GDPVal rollout batch does not fit in a single job. Tasks +run for tens of minutes each, allocations are time-boxed, and a run that is +merely *restarted* after an allocation expires silently loses or duplicates +work. These scripts keep the model server and the rollout client in separate +jobs, hand off between allocations, and validate that a resume is safe before +it is allowed to proceed. + +Nothing here is required to run GDPVal — `gym eval run` is. Reach for these +when a batch is too large for one allocation. + +## Setup + +```bash +cd benchmarks/gdpval/slurm +cp cluster.env.example cluster.env +$EDITOR cluster.env # account, model path, container image +set -a; . ./cluster.env; set +a +``` + +Three settings are required and deliberately have no default — +`GDPVAL_SLURM_ACCOUNT`, `GDPVAL_MODEL_PATH`, `GDPVAL_VLLM_IMAGE`. A default +that looks plausible but points at the wrong model or account fails hours +later as bad numbers instead of immediately as an error. + +## Running a batch + +```bash +# 1. one-time: build an isolated checkout + runtime so an in-flight run is +# never disturbed by edits to your working checkout +./bootstrap_gdpval_runtime.sh + +# 2. submit the model server and wait for /v1/models to serve the expected name +./submit_gdpval_glm52.sh + +# 3. submit exactly one rollout client against that endpoint +./monitor_gdpval_pipeline.sh + +# 4. when the allocation is nearly up, chain the next one +./submit_gdpval_continuation.sh +``` + +`run_parallel_{fp8,bf16}-glm52-vllm.sh` are the underlying multi-instance vLLM +launchers; `submit_gdpval_glm52.sh` drives the fp8 one by default. Run either +directly with `--help` for its own flags. + +## Dataset profiles + +Per-dataset settings (input JSONL, expected row count, task-id pattern, model +name, concurrency) live in `benchmarks/gdpval/datasets/*.env` (see `example.env`) and are selected +with `GDPVAL_ENV_FILE`. Adding a new batch means adding a profile, not editing +these scripts. + +## Resuming safely + +`validate_gdpval_resume_state.py` is the gate. It checks that a partial run's +state is internally consistent — no duplicate or missing task indices, launch +and source files unchanged since the run started — before another allocation is +allowed to continue it. + +Editing the launcher or validator mid-run changes their hashes and the +continuation gate will refuse to resume. That is deliberate: silently resuming +against changed code produces a batch that is two experiments stitched +together. + +## What each script does + +| Script | Purpose | +|---|---| +| `bootstrap_gdpval_runtime.sh` | Build an isolated checkout + venv so a running batch is insulated from edits | +| `submit_gdpval_glm52.sh` | Submit the vLLM server, wait for readiness, validate the served model name | +| `monitor_gdpval_pipeline.sh` | Wait for the endpoint, then submit exactly one rollout client | +| `run_gdpval_client.sbatch` | The rollout client job itself (CPU-only) | +| `submit_gdpval_continuation.sh` | Chain the next allocation once the current one is nearly spent | +| `run_gdpval_continuation_gate.sbatch` | Validate resume state, then release the next hop | +| `run_gdpval_continuation_monitor.sbatch` | Passive watcher for the continuation chain | +| `validate_gdpval_resume_state.py` | Standalone resume-safety check | +| `run_parallel_fp8-glm52-vllm.sh` | Multi-instance vLLM launcher (fp8) | +| `run_parallel_bf16-glm52-vllm.sh` | Multi-instance vLLM launcher (bf16) | +| `wait_for_slurm_account.sh` | Poll until a Slurm account/QOS association exists; never submits or cancels | +| `smoke_apptainer_worker.sh` | Verify Apptainer works on a compute node before committing an allocation | +| `rpm2cpio_stdin.sh` | Shim: BusyBox `rpm2cpio` rejects `-` for stdin, which the unprivileged Apptainer installer needs | + +## Portability + +Written against Slurm with pyxis/enroot or Apptainer, and tested on one +cluster. Job shapes (partition names, QOS, GPUs per node) are configurable but +the defaults reflect that cluster; expect to adjust `cluster.env` for yours. diff --git a/benchmarks/gdpval/slurm/bootstrap_gdpval_runtime.sh b/benchmarks/gdpval/slurm/bootstrap_gdpval_runtime.sh new file mode 100755 index 00000000000..55883dff779 --- /dev/null +++ b/benchmarks/gdpval/slurm/bootstrap_gdpval_runtime.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Build isolated, reproducible component environments for the AfterQuery +# GDPVal rollout without modifying the concurrent gym-lj checkout. + +umask 077 + +CHECKOUT="${GDPVAL_SLURM_CHECKOUT:-${GDPVAL_SLURM_STAGING_ROOT:-${SCRATCH:-$HOME}/gdpval-rollouts}/checkout}" +BOOT_ROOT="${GDPVAL_BOOT_ROOT:-${GDPVAL_SLURM_STAGING_ROOT:-${SCRATCH:-$HOME}/gdpval-rollouts}/bootstrap}" +REFERENCE="${GDPVAL_REFERENCE_CHECKOUT:-}" +UV_BIN="${GDPVAL_UV_BIN:-${CHECKOUT}/.venv/bin/uv}" +PYTHON_BIN="${GDPVAL_BOOTSTRAP_PYTHON:-/cm/local/apps/python3/bin/python3.12}" +CONSTRAINTS_DIR="${BOOT_ROOT}/constraints" +VENV_ROOT="${BOOT_ROOT}/venvs" +UV_CACHE_DIR="${BOOT_ROOT}/uv-cache" +READY_MARKER="${BOOT_ROOT}/READY" + +COMPONENTS=( + responses_api_agents/stirrup_agent + responses_api_models/vllm_model + responses_api_models/openai_model + resources_servers/gdpval +) + +usage() { + cat <<'EOF' +Usage: bootstrap_gdpval_runtime.sh ACTION + +Actions: + bootstrap Snapshot known-good constraints and build four isolated venvs. + validate Verify imports and the recorded READY marker. +EOF +} + +preflight() { + [[ -x "${UV_BIN}" ]] || { echo "ERROR: uv not executable: ${UV_BIN}" >&2; return 1; } + [[ -x "${PYTHON_BIN}" ]] || { echo "ERROR: Python not executable: ${PYTHON_BIN}" >&2; return 1; } + [[ -f "${CHECKOUT}/pyproject.toml" ]] || { echo "ERROR: invalid checkout: ${CHECKOUT}" >&2; return 1; } + local component + for component in "${COMPONENTS[@]}"; do + [[ -d "${CHECKOUT}/${component}" ]] || { echo "ERROR: missing checkout component: ${component}" >&2; return 1; } + [[ -x "${REFERENCE}/${component}/.venv/bin/python" ]] || { + echo "ERROR: missing reference component venv: ${REFERENCE}/${component}/.venv" >&2 + return 1 + } + done +} + +constraint_path() { + local component="$1" + printf '%s/%s.txt' "${CONSTRAINTS_DIR}" "${component//\//__}" +} + +target_python() { + local component="$1" + printf '%s/%s/.venv/bin/python' "${VENV_ROOT}" "${component}" +} + +snapshot_constraints() { + local component constraints + for component in "${COMPONENTS[@]}"; do + constraints="$(constraint_path "${component}")" + "${UV_BIN}" pip freeze \ + --exclude-editable \ + --python "${REFERENCE}/${component}/.venv/bin/python" \ + >"${constraints}" + [[ -s "${constraints}" ]] || { echo "ERROR: empty constraints for ${component}" >&2; return 1; } + done +} + +create_component_venv() { + local component="$1" + local target="${VENV_ROOT}/${component}/.venv" + mkdir -p "$(dirname "${target}")" + UV_CACHE_DIR="${UV_CACHE_DIR}" "${UV_BIN}" venv \ + --seed \ + --allow-existing \ + --python "${PYTHON_BIN}" \ + "${target}" +} + +install_requirements_component() { + local component="$1" + local python constraints + python="$(target_python "${component}")" + constraints="$(constraint_path "${component}")" + create_component_venv "${component}" + ( + cd "${CHECKOUT}/${component}" + UV_CACHE_DIR="${UV_CACHE_DIR}" "${UV_BIN}" pip install \ + --python "${python}" \ + --constraints "${constraints}" \ + -r requirements.txt \ + 'ray[default]==2.55.1' \ + 'openai==2.7.2' + ) +} + +install_vllm_proxy() { + local component="responses_api_models/vllm_model" + local python constraints + python="$(target_python "${component}")" + constraints="$(constraint_path "${component}")" + create_component_venv "${component}" + ( + cd "${CHECKOUT}/${component}" + UV_CACHE_DIR="${UV_CACHE_DIR}" "${UV_BIN}" pip install \ + --python "${python}" \ + --constraints "${constraints}" \ + -e . \ + 'ray[default]==2.55.1' \ + 'openai==2.7.2' + ) +} + +write_ready_marker() { + local tmp="${READY_MARKER}.tmp.$$" + { + echo "created_at_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "checkout_revision=$(git -C "${CHECKOUT}" rev-parse HEAD)" + echo "uv_version=$("${UV_BIN}" --version)" + echo "python_version=$("${PYTHON_BIN}" --version 2>&1)" + echo "venv_root=${VENV_ROOT}" + echo "uv_cache_dir=${UV_CACHE_DIR}" + sha256sum "${CONSTRAINTS_DIR}"/*.txt + } >"${tmp}" + chmod 600 "${tmp}" + mv "${tmp}" "${READY_MARKER}" +} + +validate_runtime() { + [[ -f "${READY_MARKER}" ]] || { echo "ERROR: runtime marker not found: ${READY_MARKER}" >&2; return 1; } + local agent_python proxy_python judge_python resource_python + agent_python="$(target_python responses_api_agents/stirrup_agent)" + proxy_python="$(target_python responses_api_models/vllm_model)" + judge_python="$(target_python responses_api_models/openai_model)" + resource_python="$(target_python resources_servers/gdpval)" + + "${agent_python}" -c 'import nemo_gym, ray, stirrup, transformers' + "${proxy_python}" -c 'import nemo_gym, openai, ray' + "${judge_python}" -c 'import nemo_gym, openai, ray' + "${resource_python}" -c 'import nemo_gym, openai, ray' + "${agent_python}" -c \ + 'import importlib.metadata as m; print("stirrup=" + m.version("stirrup")); print("transformers=" + m.version("transformers")); print("ray=" + m.version("ray")); print("openai=" + m.version("openai"))' + echo "Runtime validation passed: ${READY_MARKER}" +} + +bootstrap() { + preflight + mkdir -p "${CONSTRAINTS_DIR}" "${VENV_ROOT}" "${UV_CACHE_DIR}" + export UV_PYTHON_DOWNLOADS=never + export UV_LINK_MODE=copy + snapshot_constraints + install_requirements_component responses_api_agents/stirrup_agent + install_vllm_proxy + install_requirements_component responses_api_models/openai_model + install_requirements_component resources_servers/gdpval + write_ready_marker + validate_runtime +} + +case "${1:-validate}" in + bootstrap) bootstrap ;; + validate) preflight; validate_runtime ;; + -h|--help|help) usage ;; + *) usage >&2; exit 2 ;; +esac diff --git a/benchmarks/gdpval/slurm/cluster.env.example b/benchmarks/gdpval/slurm/cluster.env.example new file mode 100644 index 00000000000..7fb32ca3605 --- /dev/null +++ b/benchmarks/gdpval/slurm/cluster.env.example @@ -0,0 +1,64 @@ +# GDPVal Slurm orchestration — site configuration. +# +# cp cluster.env.example cluster.env # cluster.env is gitignored +# $EDITOR cluster.env +# set -a; . ./cluster.env; set +a +# +# Nothing here has a site-specific default. A plausible-looking default would +# silently point a run at the wrong model, image or account, and the failure +# shows up hours later as bad numbers rather than as an error. + +# --------------------------------------------------------------------------- +# Required +# --------------------------------------------------------------------------- + +# Slurm account for GPU (server) jobs. +GDPVAL_SLURM_ACCOUNT= + +# Model weights, readable from every compute node. +GDPVAL_MODEL_PATH= + +# vLLM container image (.sqsh for pyxis/enroot, or a squashfs Apptainer image). +GDPVAL_VLLM_IMAGE= + +# --------------------------------------------------------------------------- +# Paths — defaults are portable; override to put state on a shared filesystem +# --------------------------------------------------------------------------- + +# Rollout state, logs and results. Must be shared across nodes. +# Default: ${SCRATCH:-$HOME}/gdpval-rollouts +#GDPVAL_SLURM_STAGING_ROOT= + +# Isolated checkout + bootstrapped runtime built by bootstrap_gdpval_runtime.sh. +# Default: /checkout and /bootstrap +#GDPVAL_SLURM_CHECKOUT= +#GDPVAL_BOOT_ROOT= + +# Existing checkout to seed the bootstrap from, instead of a fresh clone. +#GDPVAL_REFERENCE_CHECKOUT= + +# HF/pip/torch cache root. Default: ${SCRATCH:-$HOME}/gdpval-cache +#CACHE_ROOT= + +# --------------------------------------------------------------------------- +# Slurm shape +# --------------------------------------------------------------------------- + +#GDPVAL_SLURM_PARTITION=batch +#GDPVAL_SLURM_QOS=normal +#GDPVAL_SERVER_TIME_LIMIT=04:00:00 + +# CPU-side client job. Leave unset to reuse the GPU account/partition. +#GDPVAL_CLIENT_SLURM_ACCOUNT= +#GDPVAL_CLIENT_SLURM_PARTITION=cpu +#GDPVAL_CLIENT_SLURM_QOS=cpu-normal + +# Slurm reservation, if your site uses one. Unset means no --reservation flag. +#GDPVAL_SLURM_RESERVATION= + +# --------------------------------------------------------------------------- +# Apptainer path (only if you run the container-based worker) +# --------------------------------------------------------------------------- + +#GDPVAL_APPTAINER_BIN=apptainer +#GDPVAL_CONTAINER_PATH= diff --git a/benchmarks/gdpval/slurm/monitor_gdpval_pipeline.sh b/benchmarks/gdpval/slurm/monitor_gdpval_pipeline.sh new file mode 100644 index 00000000000..8a4ed8802e4 --- /dev/null +++ b/benchmarks/gdpval/slurm/monitor_gdpval_pipeline.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Wait for the staged GLM endpoint, then submit exactly one rollout client. +# This runs on the Slurm login node; the client itself runs on cpu/cpu-normal. + +umask 077 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +STAGING_ROOT="${GDPVAL_SLURM_STAGING_ROOT:-${SCRATCH:-$HOME}/gdpval-rollouts}" +# The monitor drives ` wait`, which validates /v1/models against the +# dataset's expected model name, so it must see the dataset profile itself -- +# forwarding it to the client alone is not enough. +if [[ -n "${GDPVAL_ENV_FILE:-}" ]]; then + if [[ ! -r "${GDPVAL_ENV_FILE}" ]]; then + echo "ERROR: GDPVAL_ENV_FILE is not readable: ${GDPVAL_ENV_FILE}" >&2 + exit 2 + fi + set -a + # shellcheck disable=SC1090 + source "${GDPVAL_ENV_FILE}" + set +a +fi +PRIVATE_ENV="${GDPVAL_PRIVATE_ENV_FILE:-${STAGING_ROOT}/afterquery.private.env}" +ENDPOINT_ENV="${GDPVAL_ENDPOINT_ENV_FILE:-${STAGING_ROOT}/endpoint.current.env}" +CLIENT_SCRIPT="${SCRIPT_DIR}/run_gdpval_client.sbatch" +SERVER_WRAPPER="${SCRIPT_DIR}/submit_gdpval_glm52.sh" +CLIENT_STATE="${GDPVAL_CLIENT_STATE_FILE:-${STAGING_ROOT}/client.current}" +LOCK_DIR="${GDPVAL_PIPELINE_LOCK_DIR:-${STAGING_ROOT}/.monitor_gdpval_pipeline.lock}" +ACTION="${GDPVAL_ACTION:-run}" +TAVILY_WAIT_TIMEOUT="${GDPVAL_TAVILY_WAIT_TIMEOUT:-1800}" +POLL_INTERVAL="${GDPVAL_PIPELINE_POLL_INTERVAL:-15}" +LOCK_HELD=0 +SERVER_WAIT_PID="" + +cleanup() { + if [[ -n "${SERVER_WAIT_PID}" ]] && kill -0 "${SERVER_WAIT_PID}" 2>/dev/null; then + kill "${SERVER_WAIT_PID}" 2>/dev/null || true + wait "${SERVER_WAIT_PID}" 2>/dev/null || true + fi + if [[ "${LOCK_HELD}" == "1" ]]; then + rmdir "${LOCK_DIR}" 2>/dev/null || true + fi +} + +handle_signal() { + local signal="$1" + cleanup + trap - EXIT + if [[ "${signal}" == "INT" ]]; then + exit 130 + fi + exit 143 +} + +trap cleanup EXIT +trap 'handle_signal INT' INT +trap 'handle_signal TERM' TERM + +case "${ACTION}" in + run|full|resume) ;; + *) echo "ERROR: GDPVAL_ACTION must be run, full, or resume (got ${ACTION})" >&2; exit 2 ;; +esac +for value in "${TAVILY_WAIT_TIMEOUT}" "${POLL_INTERVAL}"; do + [[ "${value}" =~ ^[1-9][0-9]*$ ]] || { echo "ERROR: wait settings must be positive integers" >&2; exit 2; } +done + +mkdir -p "${STAGING_ROOT}/client_logs" +chmod 700 "${STAGING_ROOT}/client_logs" +if ! mkdir "${LOCK_DIR}" 2>/dev/null; then + echo "ERROR: another pipeline monitor owns ${LOCK_DIR}" >&2 + exit 2 +fi +LOCK_HELD=1 + +[[ -x "${SERVER_WRAPPER}" ]] || { echo "ERROR: missing server wrapper: ${SERVER_WRAPPER}" >&2; exit 1; } +[[ -r "${CLIENT_SCRIPT}" ]] || { echo "ERROR: missing client script: ${CLIENT_SCRIPT}" >&2; exit 1; } +[[ -f "${PRIVATE_ENV}" ]] || { echo "ERROR: missing private environment: ${PRIVATE_ENV}" >&2; exit 1; } + +"${SERVER_WRAPPER}" wait & +SERVER_WAIT_PID=$! +wait "${SERVER_WAIT_PID}" +SERVER_WAIT_PID="" +[[ -f "${ENDPOINT_ENV}" ]] || { echo "ERROR: endpoint wait returned without ${ENDPOINT_ENV}" >&2; exit 1; } + +deadline=$((SECONDS + TAVILY_WAIT_TIMEOUT)) +while true; do + if grep -q '^export GDPVAL_ALLOW_NO_TAVILY=1$' "${PRIVATE_ENV}"; then + echo "Private environment explicitly enables offline Tavily mode" + break + fi + if grep -Eq '^export TAVILY_API_KEY=.+$' "${PRIVATE_ENV}" \ + && ! grep -q 'REPLACE_WITH_PRIVATE_TAVILY_KEY' "${PRIVATE_ENV}"; then + echo "Private Tavily configuration is present" + break + fi + if (( SECONDS >= deadline )); then + echo "ERROR: timed out waiting for Tavily configuration in ${PRIVATE_ENV}" >&2 + exit 1 + fi + echo "Endpoint is ready; waiting for private Tavily configuration" + sleep "${POLL_INTERVAL}" +done + +if [[ -e "${CLIENT_STATE}" ]]; then + prior_job="$(grep '^job_id=' "${CLIENT_STATE}" | cut -d= -f2 || true)" + prior_state="" + if [[ "${prior_job}" =~ ^[0-9]+$ ]]; then + # squeue exits nonzero once a job leaves the queue ("Invalid job id + # specified"), which under `set -euo pipefail` aborted this monitor before + # it could archive a stale client state. Treat a purged job as "no live + # client", but still fail loudly when squeue itself is unusable: a + # controller outage must not be misread as "safe to submit another client". + if ! squeue_out="$(squeue -h -j "${prior_job}" -o '%T' 2>&1)"; then + if ! grep -qi 'invalid job id' <<<"${squeue_out}"; then + echo "ERROR: squeue failed for prior client job ${prior_job}: ${squeue_out}" >&2 + exit 1 + fi + squeue_out="" + fi + prior_state="$(head -1 <<<"${squeue_out}")" + fi + if [[ -n "${prior_state}" ]]; then + echo "ERROR: rollout client job ${prior_job} is already ${prior_state}" >&2 + exit 1 + fi + stale_state="${CLIENT_STATE}.stale.$(date -u +%Y%m%dT%H%M%SZ).$$" + mv "${CLIENT_STATE}" "${stale_state}" + chmod 600 "${stale_state}" +fi + +# sbatch --export is an allowlist: GDPVAL_PRIVATE_ENV_FILE selects which private env +# (and therefore which dataset profile) the client sources, so a Mercor run does +# not silently inherit AfterQuery's. +submission="$( + sbatch --parsable \ + --export="USER=${USER},HOME=${HOME},LOGNAME=${LOGNAME:-${USER}},PATH=${PATH},LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-},SLURM_CONF=${SLURM_CONF:-},GDPVAL_ACTION=${ACTION},GDPVAL_PRIVATE_ENV_FILE=${PRIVATE_ENV},GDPVAL_ENV_FILE=${GDPVAL_ENV_FILE:-}" \ + "${CLIENT_SCRIPT}" +)" +job_id="${submission%%;*}" +if [[ ! "${job_id}" =~ ^[0-9]+$ ]]; then + echo "ERROR: could not parse rollout client job ID from sbatch" >&2 + exit 1 +fi + +tmp_state="${CLIENT_STATE}.tmp.$$" +{ + echo "job_id=${job_id}" + echo "action=${ACTION}" + echo "server_endpoint_env=${ENDPOINT_ENV}" + echo "submitted_at_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" +} >"${tmp_state}" +chmod 600 "${tmp_state}" +mv "${tmp_state}" "${CLIENT_STATE}" + +echo "Submitted rollout client job ${job_id} with action ${ACTION}" +echo "Client state: ${CLIENT_STATE}" diff --git a/benchmarks/gdpval/slurm/rpm2cpio_stdin.sh b/benchmarks/gdpval/slurm/rpm2cpio_stdin.sh new file mode 100755 index 00000000000..6d321465da2 --- /dev/null +++ b/benchmarks/gdpval/slurm/rpm2cpio_stdin.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +# BusyBox rpm2cpio does not accept '-' for stdin, while Apptainer's +# unprivileged installer streams RPMs through that interface. + +if [[ "${1:-}" != "-" ]]; then + exec /usr/bin/busybox rpm2cpio "$@" +fi + +temporary_rpm="$(mktemp "${TMPDIR:-/tmp}/rpm2cpio.XXXXXX.rpm")" +trap 'rm -f "${temporary_rpm}"' EXIT INT TERM +/bin/cat >"${temporary_rpm}" +/usr/bin/busybox rpm2cpio "${temporary_rpm}" diff --git a/benchmarks/gdpval/slurm/run_gdpval_client.sbatch b/benchmarks/gdpval/slurm/run_gdpval_client.sbatch new file mode 100755 index 00000000000..eef9ed5ecc5 --- /dev/null +++ b/benchmarks/gdpval/slurm/run_gdpval_client.sbatch @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +#SBATCH --job-name=gdpval-glm52-client +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=64 +#SBATCH --mem=180G +#SBATCH --time=03:45:00 + +set -euo pipefail +umask 077 + +STAGING_ROOT="${GDPVAL_SLURM_STAGING_ROOT:-${SCRATCH:-$HOME}/gdpval-rollouts}" +CHECKOUT="${GDPVAL_SLURM_CHECKOUT:-${STAGING_ROOT}/checkout}" +PRIVATE_ENV="${GDPVAL_PRIVATE_ENV_FILE:-${STAGING_ROOT}/afterquery.private.env}" +ENDPOINT_ENV="${GDPVAL_ENDPOINT_ENV_FILE:-${STAGING_ROOT}/endpoint.current.env}" +BOOTSTRAP="${CHECKOUT}/benchmarks/gdpval/slurm/bootstrap_gdpval_runtime.sh" +RUNNER="${CHECKOUT}/benchmarks/gdpval/run_gdpval_rollouts.sh" +ACTION="${GDPVAL_ACTION:-run}" + +validate_action() { + case "$1" in + run|full|resume) ;; + *) + echo "ERROR: GDPVAL_ACTION must be run, full, or resume (got $1)" >&2 + return 1 + ;; + esac +} + +# Fail invalid sbatch --export values before touching private launch state. +validate_action "${ACTION}" + +require_private_file() { + local path="$1" + if [[ ! -f "${path}" ]]; then + echo "ERROR: required private environment file not found: ${path}" >&2 + return 1 + fi + local mode + mode="$(stat -c '%a' "${path}")" + if [[ "${mode}" != "600" && "${mode}" != "400" ]]; then + echo "ERROR: private environment file must have mode 600 or 400: ${path} (${mode})" >&2 + return 1 + fi +} + +require_private_file "${PRIVATE_ENV}" +require_private_file "${ENDPOINT_ENV}" +[[ -x "${BOOTSTRAP}" ]] || { echo "ERROR: bootstrap script not executable: ${BOOTSTRAP}" >&2; exit 1; } +[[ -x "${RUNNER}" ]] || { echo "ERROR: rollout runner not executable: ${RUNNER}" >&2; exit 1; } + +set -a +# The private file supplies Tavily and run settings. The validated endpoint file +# is loaded second so stale placeholder endpoint values cannot win. +# shellcheck disable=SC1090 +source "${PRIVATE_ENV}" +# shellcheck disable=SC1090 +source "${ENDPOINT_ENV}" +set +a +unset GDPVAL_ENV_FILE + +ACTION="${GDPVAL_ACTION:-run}" +validate_action "${ACTION}" + +: "${POLICY_MODEL_NAME:?validated endpoint model is unset}" +: "${POLICY_BASE_URL:?validated endpoint URL is unset}" +: "${POLICY_API_KEY:?validated endpoint API key is unset}" +: "${GDPVAL_SERVER_JOB_ID:?validated endpoint server job is unset}" +: "${GDPVAL_SERVER_ROOT:?validated endpoint server root is unset}" +: "${GDPVAL_ENDPOINT_CREATED_AT_UTC:?validated endpoint creation time is unset}" +: "${GDPVAL_APPTAINER_BIN:?GDPVAL_APPTAINER_BIN must name the private Apptainer executable}" +if [[ "${GDPVAL_ALLOW_NO_TAVILY:-0}" != "1" ]]; then + : "${TAVILY_API_KEY:?TAVILY_API_KEY must be set in the private environment file}" + if [[ "${TAVILY_API_KEY}" == REPLACE_WITH_* ]]; then + echo "ERROR: TAVILY_API_KEY still contains the template placeholder" >&2 + exit 1 + fi +fi +if [[ "${GDPVAL_APPTAINER_BIN}" != /* ]]; then + echo "ERROR: GDPVAL_APPTAINER_BIN must be absolute (got ${GDPVAL_APPTAINER_BIN})" >&2 + exit 1 +fi +if [[ ! -x "${GDPVAL_APPTAINER_BIN}" ]]; then + echo "ERROR: GDPVAL_APPTAINER_BIN is not executable: ${GDPVAL_APPTAINER_BIN}" >&2 + exit 1 +fi +if [[ ! "${GDPVAL_SERVER_JOB_ID}" =~ ^[0-9]+$ ]]; then + echo "ERROR: validated endpoint has an invalid server job ID: ${GDPVAL_SERVER_JOB_ID}" >&2 + exit 1 +fi +case "${GDPVAL_SERVER_ROOT}" in + "${STAGING_ROOT}/model_servers/"*) ;; + *) + echo "ERROR: validated endpoint server root is outside the staging tree: ${GDPVAL_SERVER_ROOT}" >&2 + exit 1 + ;; +esac +if [[ ! -d "${GDPVAL_SERVER_ROOT}" ]]; then + echo "ERROR: validated endpoint server root does not exist: ${GDPVAL_SERVER_ROOT}" >&2 + exit 1 +fi +command -v squeue >/dev/null +# squeue exits nonzero once the server job leaves the queue, which under +# `set -euo pipefail` killed this script before the explicit check below could +# report why. Failing closed here is correct: any unreadable state is treated as +# "not RUNNING" so the clear error is printed instead of a silent bash abort. +SERVER_STATE="$(squeue -h -j "${GDPVAL_SERVER_JOB_ID}" -o '%T' 2>/dev/null | head -1 || true)" +if [[ "${SERVER_STATE}" != "RUNNING" ]]; then + echo "ERROR: endpoint server job ${GDPVAL_SERVER_JOB_ID} is not RUNNING (state ${SERVER_STATE:-UNKNOWN})" >&2 + exit 1 +fi +export PATH="$(dirname "${GDPVAL_APPTAINER_BIN}"):${PATH}" +# Ray on a Lustre path is slow to start and can exceed the 107-byte AF_UNIX +# socket path limit; /tmp is node-local. +export RAY_TMPDIR="${RAY_TMPDIR:-/tmp}" + +cd "${CHECKOUT}" +"${BOOTSTRAP}" validate +"${RUNNER}" "${ACTION}" diff --git a/benchmarks/gdpval/slurm/run_gdpval_continuation_gate.sbatch b/benchmarks/gdpval/slurm/run_gdpval_continuation_gate.sbatch new file mode 100755 index 00000000000..c35a483e7c4 --- /dev/null +++ b/benchmarks/gdpval/slurm/run_gdpval_continuation_gate.sbatch @@ -0,0 +1,335 @@ +#!/usr/bin/env bash +#SBATCH --job-name=gdpval-cont-gate +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=2 +#SBATCH --mem=4G +#SBATCH --time=00:20:00 + +set -euo pipefail +umask 077 + +STAGING_ROOT="${GDPVAL_SLURM_STAGING_ROOT:-${SCRATCH:-$HOME}/gdpval-rollouts}" +CHECKOUT="${GDPVAL_SLURM_CHECKOUT:-${STAGING_ROOT}/checkout}" +RUN_ROOT="${GDPVAL_RUN_ROOT:-${STAGING_ROOT}/GLM-52-fp8_afterquery-subset100_20260721}" +PARENT_SERVER_JOB_ID="${GDPVAL_PARENT_SERVER_JOB_ID:?GDPVAL_PARENT_SERVER_JOB_ID is required}" +PARENT_CLIENT_JOB_ID="${GDPVAL_PARENT_CLIENT_JOB_ID:?GDPVAL_PARENT_CLIENT_JOB_ID is required}" +CONTINUATION_STATE="${GDPVAL_CONTINUATION_STATE_FILE:-${STAGING_ROOT}/continuation.current}" +SERVER_STATE="${GDPVAL_SERVER_STATE_FILE:-${STAGING_ROOT}/model_server.current}" +CLIENT_STATE="${GDPVAL_CLIENT_STATE_FILE:-${STAGING_ROOT}/client.current}" +SUBMITTER="${CHECKOUT}/benchmarks/gdpval/slurm/submit_gdpval_glm52.sh" +MONITOR_BATCH="${CHECKOUT}/benchmarks/gdpval/slurm/run_gdpval_continuation_monitor.sbatch" +VALIDATOR="${CHECKOUT}/benchmarks/gdpval/validate_gdpval_batch.py" +RESUME_VALIDATOR="${CHECKOUT}/benchmarks/gdpval/slurm/validate_gdpval_resume_state.py" +PYTHON_BIN="${CHECKOUT}/.venv/bin/python" +LAUNCH_INPUT="${RUN_ROOT}/input/afterquery_gdpval_subset_100_20260721.launch.jsonl" +REFERENCE_OVERRIDES="${CHECKOUT}/benchmarks/gdpval/data/afterquery_gdpval_subset_100_20260721.reference_overrides.json" +RESULTS_DIR="${RUN_ROOT}/results" +FULL_OUTPUT="${RESULTS_DIR}/rollouts.jsonl" +FULL_MATERIALIZED="${RESULTS_DIR}/rollouts_materialized_inputs.jsonl" +FULL_FAILURES="${RESULTS_DIR}/rollouts_failures.jsonl" +DELIVERABLES_DIR="${RUN_ROOT}/deliverables" +SMOKE_TASK_ID="${GDPVAL_SMOKE_TASK_ID:-AQ-09135}" +SMOKE_OUTPUT="${RESULTS_DIR}/smoke_${SMOKE_TASK_ID}.jsonl" +SMOKE_MATERIALIZED="${RESULTS_DIR}/smoke_${SMOKE_TASK_ID}_materialized_inputs.jsonl" +SMOKE_FAILURES="${RESULTS_DIR}/smoke_${SMOKE_TASK_ID}_failures.jsonl" +SMOKE_DELIVERABLES="${RUN_ROOT}/smoke_deliverables" +EXPECTED_MODEL_NAME="${GDPVAL_EXPECTED_MODEL_NAME:-GLM-52-bf16-afterquery-20260721-0}" +EXPECTED_LAUNCH_SHA256="${GDPVAL_EXPECTED_LAUNCH_SHA256:-9e8d2d182a142e44f60fabb0dff1eaf9f7bd9f7e5c8d23fc2dc5879373d04bad}" +LOCK_DIR="${GDPVAL_CONTINUATION_LOCK_DIR:-${STAGING_ROOT}/.gdpval_continuation_gate.lock}" +LOCK_HELD=0 + +cleanup() { + if [[ "${LOCK_HELD}" == "1" ]]; then + rmdir "${LOCK_DIR}" 2>/dev/null || true + fi +} +trap cleanup EXIT + +require_file() { + local path="$1" + [[ -f "${path}" ]] || { + echo "ERROR: required file not found: ${path}" >&2 + return 1 + } +} + +job_state() { + local job_id="$1" + local state + state="$(squeue -h -j "${job_id}" -o '%T' | head -1)" + if [[ -z "${state}" ]]; then + state="$(sacct -n -X -j "${job_id}" -o State | awk 'NF {print $1; exit}')" + fi + printf '%s' "${state:-UNKNOWN}" +} + +require_terminal_job() { + local job_id="$1" + local label="$2" + local state + state="$(job_state "${job_id}")" + case "${state}" in + COMPLETED*|FAILED*|CANCELLED*|TIMEOUT*|NODE_FAIL*|OUT_OF_MEMORY*|PREEMPTED*|BOOT_FAIL*|DEADLINE*|REVOKED*) + echo "${label} job ${job_id} is terminal: ${state}" + ;; + *) + echo "ERROR: ${label} job ${job_id} is not terminal: ${state}" >&2 + return 1 + ;; + esac +} + +state_value() { + local path="$1" + local wanted="$2" + awk -F= -v wanted="${wanted}" '$1 == wanted {sub(/^[^=]*=/, ""); print; exit}' "${path}" +} + +write_continuation_state() { + local status="$1" + local action="${2:-}" + local new_server_job_id="${3:-}" + local monitor_job_id="${4:-}" + local tmp="${CONTINUATION_STATE}.tmp.$$" + { + echo "status=${status}" + echo "parent_server_job_id=${PARENT_SERVER_JOB_ID}" + echo "parent_client_job_id=${PARENT_CLIENT_JOB_ID}" + echo "gate_job_id=${SLURM_JOB_ID:-manual}" + [[ -n "${action}" ]] && echo "action=${action}" + [[ -n "${new_server_job_id}" ]] && echo "new_server_job_id=${new_server_job_id}" + [[ -n "${monitor_job_id}" ]] && echo "monitor_job_id=${monitor_job_id}" + echo "updated_at_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + } >"${tmp}" + chmod 600 "${tmp}" + mv "${tmp}" "${CONTINUATION_STATE}" +} + +line_count() { + local path="$1" + if [[ -f "${path}" ]]; then + wc -l <"${path}" | tr -d ' ' + else + printf '0' + fi +} + +validate_full() { + "${PYTHON_BIN}" "${VALIDATOR}" \ + --input "${LAUNCH_INPUT}" \ + --expected-count 100 \ + --expected-sha256 "${EXPECTED_LAUNCH_SHA256}" \ + --reference-overrides "${REFERENCE_OVERRIDES}" \ + --rollouts "${FULL_OUTPUT}" \ + --deliverables-dir "${DELIVERABLES_DIR}" \ + --expected-response-model "${EXPECTED_MODEL_NAME}" +} + +validate_smoke() { + "${PYTHON_BIN}" "${VALIDATOR}" \ + --input "${LAUNCH_INPUT}" \ + --expected-count 100 \ + --expected-sha256 "${EXPECTED_LAUNCH_SHA256}" \ + --reference-overrides "${REFERENCE_OVERRIDES}" \ + --smoke-task-id "${SMOKE_TASK_ID}" \ + --rollouts "${SMOKE_OUTPUT}" \ + --deliverables-dir "${SMOKE_DELIVERABLES}" \ + --require-deliverable \ + --expected-response-model "${EXPECTED_MODEL_NAME}" +} + +validate_partial_full_state() { + "${PYTHON_BIN}" "${RESUME_VALIDATOR}" \ + --launch-input "${LAUNCH_INPUT}" \ + --rollouts "${FULL_OUTPUT}" \ + --materialized-inputs "${FULL_MATERIALIZED}" \ + --failures "${FULL_FAILURES}" \ + --run-config "${RUN_ROOT}/run_config.sha256" \ + --expected-launch-sha256 "${EXPECTED_LAUNCH_SHA256}" \ + --expected-model "${EXPECTED_MODEL_NAME}" \ + --deliverables-dir "${DELIVERABLES_DIR}" +} + +directory_has_entries() { + local path="$1" + [[ -d "${path}" && -n "$(find "${path}" -mindepth 1 -maxdepth 1 -print -quit)" ]] +} + +archive_empty_lock() { + local path="$1" + local archive_dir="$2" + if [[ ! -e "${path}" ]]; then + return + fi + if [[ ! -d "${path}" ]] || directory_has_entries "${path}"; then + echo "ERROR: refusing to archive nonempty or non-directory lock: ${path}" >&2 + return 1 + fi + mv "${path}" "${archive_dir}/$(basename "${path}")" + echo "Archived stale lock: ${path}" +} + +archive_unfinished_smoke() { + local timestamp archive_dir + timestamp="$(date -u +%Y%m%dT%H%M%SZ)" + archive_dir="${RUN_ROOT}/continuation_archive/${timestamp}_after_${PARENT_SERVER_JOB_ID}" + mkdir -p "${archive_dir}/results" "${archive_dir}/logs" + chmod 700 "${RUN_ROOT}/continuation_archive" "${archive_dir}" "${archive_dir}/results" "${archive_dir}/logs" + + local unexpected + unexpected="$( + find "${RESULTS_DIR}" -mindepth 1 -maxdepth 1 \ + ! -name "smoke_${SMOKE_TASK_ID}.jsonl" \ + ! -name "smoke_${SMOKE_TASK_ID}_materialized_inputs.jsonl" \ + ! -name "smoke_${SMOKE_TASK_ID}_failures.jsonl" \ + -print -quit + )" + if [[ -n "${unexpected}" ]]; then + echo "ERROR: unfinished smoke has unexpected result state: ${unexpected}" >&2 + return 1 + fi + + local path + for path in "${SMOKE_OUTPUT}" "${SMOKE_MATERIALIZED}" "${SMOKE_FAILURES}"; do + if [[ -e "${path}" ]]; then + mv "${path}" "${archive_dir}/results/" + fi + done + if [[ -d "${SMOKE_DELIVERABLES}" ]]; then + mv "${SMOKE_DELIVERABLES}" "${archive_dir}/smoke_deliverables" + fi + mkdir -p "${SMOKE_DELIVERABLES}" + chmod 700 "${SMOKE_DELIVERABLES}" + + if [[ -f "${RUN_ROOT}/logs/gym_env_start.log" ]]; then + mv "${RUN_ROOT}/logs/gym_env_start.log" "${archive_dir}/logs/" + fi + if [[ -f "${RUN_ROOT}/run_manifest.txt" ]]; then + mv "${RUN_ROOT}/run_manifest.txt" "${archive_dir}/" + fi + archive_empty_lock "${RUN_ROOT}/.launch.lock" "${archive_dir}" + echo "Archived unfinished smoke state: ${archive_dir}" +} + +archive_server_state() { + require_file "${SERVER_STATE}" + local captured_job_id terminal_state + captured_job_id="$(state_value "${SERVER_STATE}" job_id)" + if [[ "${captured_job_id}" != "${PARENT_SERVER_JOB_ID}" ]]; then + echo "ERROR: current server state points to ${captured_job_id}, expected ${PARENT_SERVER_JOB_ID}" >&2 + return 1 + fi + terminal_state="$(job_state "${PARENT_SERVER_JOB_ID}")" + local archived="${SERVER_STATE}.terminal.${PARENT_SERVER_JOB_ID}.$(date -u +%Y%m%dT%H%M%SZ)" + mv "${SERVER_STATE}" "${archived}" + chmod 600 "${archived}" + echo "Archived terminal server state (${terminal_state}): ${archived}" +} + +verify_captured_state_identity() { + require_file "${SERVER_STATE}" + require_file "${CLIENT_STATE}" + require_file "${CONTINUATION_STATE}" + local current_server current_client captured_gate captured_server captured_client + current_server="$(state_value "${SERVER_STATE}" job_id)" + current_client="$(state_value "${CLIENT_STATE}" job_id)" + captured_gate="$(state_value "${CONTINUATION_STATE}" gate_job_id)" + captured_server="$(state_value "${CONTINUATION_STATE}" parent_server_job_id)" + captured_client="$(state_value "${CONTINUATION_STATE}" parent_client_job_id)" + if [[ "${current_server}" != "${PARENT_SERVER_JOB_ID}" || "${captured_server}" != "${PARENT_SERVER_JOB_ID}" ]]; then + echo "ERROR: captured server identity changed; refusing continuation" >&2 + return 1 + fi + if [[ "${current_client}" != "${PARENT_CLIENT_JOB_ID}" || "${captured_client}" != "${PARENT_CLIENT_JOB_ID}" ]]; then + echo "ERROR: captured client identity changed; refusing continuation" >&2 + return 1 + fi + if [[ -n "${SLURM_JOB_ID:-}" && "${captured_gate}" != "${SLURM_JOB_ID}" ]]; then + echo "ERROR: continuation state belongs to gate ${captured_gate}, not ${SLURM_JOB_ID}" >&2 + return 1 + fi +} + +for path in "${SUBMITTER}" "${MONITOR_BATCH}" "${VALIDATOR}" "${RESUME_VALIDATOR}" "${PYTHON_BIN}" "${LAUNCH_INPUT}" "${REFERENCE_OVERRIDES}"; do + require_file "${path}" +done +for command_name in sbatch squeue sacct awk sha256sum; do + command -v "${command_name}" >/dev/null +done +for job_id in "${PARENT_SERVER_JOB_ID}" "${PARENT_CLIENT_JOB_ID}"; do + [[ "${job_id}" =~ ^[0-9]+$ ]] || { + echo "ERROR: invalid parent job ID: ${job_id}" >&2 + exit 2 + } +done + +mkdir -p "${STAGING_ROOT}/continuation_logs" +chmod 700 "${STAGING_ROOT}/continuation_logs" +if ! mkdir "${LOCK_DIR}" 2>/dev/null; then + echo "ERROR: another continuation gate owns ${LOCK_DIR}" >&2 + exit 2 +fi +LOCK_HELD=1 + +require_terminal_job "${PARENT_SERVER_JOB_ID}" "server" +require_terminal_job "${PARENT_CLIENT_JOB_ID}" "client" +verify_captured_state_identity + +if [[ "$(line_count "${FULL_OUTPUT}")" == "100" ]] && validate_full; then + echo "All 100 rollouts are valid; no continuation allocation is needed" + write_continuation_state complete_no_continuation + exit 0 +fi + +ACTION="" +if [[ -f "${FULL_OUTPUT}" || -f "${FULL_MATERIALIZED}" ]]; then + if [[ ! -f "${FULL_OUTPUT}" || ! -f "${FULL_MATERIALIZED}" ]]; then + echo "ERROR: full continuation requires both ${FULL_OUTPUT} and ${FULL_MATERIALIZED}" >&2 + exit 1 + fi + validate_partial_full_state + ACTION=resume + echo "Detected partial full collection: $(line_count "${FULL_OUTPUT}")/100 rows; scheduling resume" +elif [[ "$(line_count "${SMOKE_OUTPUT}")" == "1" ]] && validate_smoke; then + ACTION=resume + echo "Smoke is valid and full collection has not started; scheduling full collection" +else + if directory_has_entries "${DELIVERABLES_DIR}"; then + echo "ERROR: unfinished smoke has an unexpected nonempty full deliverables cache: ${DELIVERABLES_DIR}" >&2 + exit 1 + fi + archive_unfinished_smoke + ACTION=run + echo "Smoke did not finish; scheduling one clean smoke retry" +fi + +mkdir -p "${RUN_ROOT}/continuation_archive" +chmod 700 "${RUN_ROOT}/continuation_archive" +archive_empty_lock "${RUN_ROOT}/.launch.lock" "${RUN_ROOT}/continuation_archive" +archive_empty_lock "${STAGING_ROOT}/.monitor_gdpval_pipeline.lock" "${RUN_ROOT}/continuation_archive" +archive_server_state + +GDPVAL_SLURM_STAGING_ROOT="${STAGING_ROOT}" "${SUBMITTER}" submit +NEW_SERVER_JOB_ID="$(state_value "${SERVER_STATE}" job_id)" +[[ "${NEW_SERVER_JOB_ID}" =~ ^[0-9]+$ ]] || { + echo "ERROR: submitter wrote an invalid new server job ID: ${NEW_SERVER_JOB_ID}" >&2 + exit 1 +} +write_continuation_state server_queued "${ACTION}" "${NEW_SERVER_JOB_ID}" + +MONITOR_SUBMISSION="$( + sbatch --parsable \ + --dependency="after:${NEW_SERVER_JOB_ID}+1" \ + --export="USER=${USER},HOME=${HOME},LOGNAME=${LOGNAME:-${USER}},PATH=${PATH},LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-},SLURM_CONF=${SLURM_CONF:-},GDPVAL_ACTION=${ACTION},GDPVAL_SLURM_STAGING_ROOT=${STAGING_ROOT}" \ + "${MONITOR_BATCH}" +)" +MONITOR_JOB_ID="${MONITOR_SUBMISSION%%;*}" +[[ "${MONITOR_JOB_ID}" =~ ^[0-9]+$ ]] || { + echo "ERROR: could not parse continuation monitor job ID" >&2 + exit 1 +} + +write_continuation_state continuation_queued "${ACTION}" "${NEW_SERVER_JOB_ID}" "${MONITOR_JOB_ID}" +echo "Queued continuation server ${NEW_SERVER_JOB_ID}" +echo "Queued endpoint monitor ${MONITOR_JOB_ID} with action ${ACTION}" diff --git a/benchmarks/gdpval/slurm/run_gdpval_continuation_monitor.sbatch b/benchmarks/gdpval/slurm/run_gdpval_continuation_monitor.sbatch new file mode 100755 index 00000000000..2b001d9b4b9 --- /dev/null +++ b/benchmarks/gdpval/slurm/run_gdpval_continuation_monitor.sbatch @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +#SBATCH --job-name=gdpval-cont-monitor +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=1 +#SBATCH --mem=2G +#SBATCH --time=01:00:00 + +set -euo pipefail +umask 077 + +STAGING_ROOT="${GDPVAL_SLURM_STAGING_ROOT:-${SCRATCH:-$HOME}/gdpval-rollouts}" +CHECKOUT="${GDPVAL_SLURM_CHECKOUT:-${STAGING_ROOT}/checkout}" +MONITOR="${CHECKOUT}/benchmarks/gdpval/slurm/monitor_gdpval_pipeline.sh" +ACTION="${GDPVAL_ACTION:-resume}" + +case "${ACTION}" in + run|full|resume) ;; + *) + echo "ERROR: GDPVAL_ACTION must be run, full, or resume (got ${ACTION})" >&2 + exit 2 + ;; +esac + +[[ -x "${MONITOR}" ]] || { + echo "ERROR: pipeline monitor is not executable: ${MONITOR}" >&2 + exit 1 +} + +mkdir -p "${STAGING_ROOT}/continuation_logs" +chmod 700 "${STAGING_ROOT}/continuation_logs" + +export GDPVAL_ACTION="${ACTION}" +export GDPVAL_SLURM_STAGING_ROOT="${STAGING_ROOT}" +export GDPVAL_SERVER_READY_TIMEOUT="${GDPVAL_SERVER_READY_TIMEOUT:-3300}" +exec "${MONITOR}" diff --git a/benchmarks/gdpval/slurm/run_parallel_bf16-glm52-vllm.sh b/benchmarks/gdpval/slurm/run_parallel_bf16-glm52-vllm.sh new file mode 100755 index 00000000000..0c8a112698d --- /dev/null +++ b/benchmarks/gdpval/slurm/run_parallel_bf16-glm52-vllm.sh @@ -0,0 +1,453 @@ +#!/bin/bash +#SBATCH --time=04:00:00 +#SBATCH --nodes=16 +#SBATCH --ntasks=16 +#SBATCH --ntasks-per-node=1 +#SBATCH --gres=gpu:4 +#SBATCH --switches=1 +#SBATCH --output=%j_%x.out +#SBATCH --error=%j_%x.err + +set -euo pipefail + +OUTPUT_DIR="" +LUSTRE_DIR=/lustre + +# Keep the upstream flag names for compatibility with the existing launcher stack. +VLLM_IMAGE="${VLLM_IMAGE:-}" +MODEL_PATH="${MODEL_PATH:-}" +MODEL_NAME="${MODEL_NAME:-GLM-52-bf16-parallel}" +NUM_INSTANCES="" +INSTANCE_IDX="" + +# BF16 GLM 5.2 uses the GLM 5.1 bf16 resource shape: one endpoint spans +# 16 nodes with TP=4 and Ray data parallelism across all 16 replicas. +NUM_NODES=16 +# Slurm settings mirror run_parallel_glm52-vllm.sh. +# Set GDPVAL_SLURM_ACCOUNT (or --account) to your Slurm account. +# export QOS="${QOS-}" +# export RESERVATION="${RESERVATION-}" +ACCOUNT="${GDPVAL_SLURM_ACCOUNT:-}" +PARTITION=batch +export QOS="${QOS-normal}" +export RESERVATION="${RESERVATION-}" +TIME_LIMIT=04:00:00 + +BASE_RAY_PORT=6379 +BASE_VLLM_PORT=10240 + +TENSOR_PARALLEL_SIZE=4 +PIPELINE_PARALLEL_SIZE=1 +DATA_PARALLEL_SIZE=16 +DATA_PARALLEL_SIZE_LOCAL=1 +GPU_MEMORY_UTILIZATION=0.85 +API_SERVER_COUNT=1 +# Bumped from 202752 -> 262144 so the served context covers the rollout +# token budget (MAX_TOKENS=262144 in test_parallel_rollout_run.sh); the +# old 202752 was below that budget and could truncate/reject long requests. +# Not pushed to GLM-5.2's native 1M window: KV cache here is bf16 (no +# --kv-cache-dtype fp8), so 1M would collapse per-replica concurrency. +MAX_MODEL_LEN=262144 + +CACHE_ROOT="${CACHE_ROOT:-${SCRATCH:-$HOME}/gdpval-cache}" +HF_HOME="${HF_HOME:-${CACHE_ROOT}/huggingface}" +VLLM_CACHE_ROOT="${VLLM_CACHE_ROOT:-${CACHE_ROOT}/vllm}" + +usage() { + echo "Usage: $0 --num-instances --output-dir [OPTIONS]" + echo "" + echo "Required outside Slurm:" + echo " --num-instances Number of endpoint instances to launch" + echo " --output-dir Output directory for logs and server_info" + echo "" + echo "Optional overrides:" + echo " --vllm-image vLLM Ray container image (default: ${VLLM_IMAGE})" + echo " --model-path Model checkpoint path (default: ${MODEL_PATH})" + echo " --model-name Base served model name (default: ${MODEL_NAME})" + echo " --num-nodes Nodes per GLM 5.2 bf16 server job (default: ${NUM_NODES})" + echo " --account Slurm account for self-submitted jobs (default: ${ACCOUNT})" + echo " --partition Slurm partition for self-submitted jobs (default: ${PARTITION})" + echo " --qos Slurm QoS for self-submitted jobs (default: ${QOS:-})" + echo " --reservation Slurm reservation for self-submitted jobs (default: ${RESERVATION:-})" + echo " --time-limit