From 00b266c556a6d41fa2c7ad13dfa1700b94f7a4df Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Mon, 7 Sep 2026 21:42:15 -0400 Subject: [PATCH 1/2] =?UTF-8?q?OUTERLOOP=5FAUTO=5FUPDATE:=20the=20deploy?= =?UTF-8?q?=20step's=20update=20policy=20=E2=80=94=20off=20by=20default,?= =?UTF-8?q?=20release,=20or=20main?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The deploy step used to pull main whenever a PAT file was present. That is the right behaviour for the kernel's own developers and the wrong one for an adopter, who installs a release and upgrades when they choose. The policy is now explicit, read from .env before the fetch so a change needs no chain restart: - off (default): the checkout stays as it is. - release: fetch the public repo's release tags and move to the newest by version order (dev/rc pre-releases count). No credential needed. - main: follow every merge. The lab deployment sets this. The environment rollback logic is unchanged: whatever moved the checkout, the venv is synced to the commit that is checked out or the deploy goes back to the pair that matched. init writes the default for Slurm deployments; the .env trust check now also works with BSD stat. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 7 ++ docs/design/architecture.md | 2 +- docs/install.md | 11 +++ docs/roadmap.md | 7 +- scripts/README.md | 2 +- scripts/tick_chain.sbatch | 11 +-- scripts/tick_deploy.sh | 171 +++++++++++++++++++++++++----------- src/outerloop/init.py | 3 + tests/test_init.py | 8 ++ tests/test_tick_resident.py | 143 ++++++++++++++++++++++++++++++ 10 files changed, 305 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0958529c..6c0675a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ Versions follow [SemVer](https://semver.org). ### Added +- `OUTERLOOP_AUTO_UPDATE` (`off` by default, `release`, `main`) sets what the + deploy step does to a checkout-based deployment: nothing, move to the newest + release tag, or follow every merge. Adopters run what they installed until + they choose to upgrade; following `main` is for the kernel's own developers. + `init` writes the default for Slurm deployments. - `docs/design/eval-cache.md`: why every eval downloads its dependencies today, and the two changes that stop it — a kernel-warmed seed cache each job copies, and a per-target image named in the contract (`environment.container`). @@ -80,6 +85,8 @@ Versions follow [SemVer](https://semver.org). ### Changed +- The deploy step no longer pulls `main` on its own: a deployment that relied + on the PAT-gated pull sets `OUTERLOOP_AUTO_UPDATE=main` in its `.env`. - Launches always queue. The two admission schemes tried this cycle — held launches released under `OUTERLOOP_MAX_LAUNCH_GPUS`, then refusing a sleep's launches while the account waited on a cap — are gone: the kernel keeps no diff --git a/docs/design/architecture.md b/docs/design/architecture.md index ef252d41..0cfb196f 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -233,7 +233,7 @@ outage notice, not a round — because silence only visible in the Actions tab is not a defensible way to miss a verification. **Every flight has a snapshot.** Submitted jobs do not run from the -shared checkout — the deploy step resets it at every tick, so a queued +shared checkout — the deploy step may move it at any tick, so a queued job could have its code swapped mid-flight. Each submission gets a detached worktree of the checkout as it was at submit time (`flights/`, beside the checkout); the job runs exactly the tree that submitted it, diff --git a/docs/install.md b/docs/install.md index d6abc0c7..be5a4bb5 100644 --- a/docs/install.md +++ b/docs/install.md @@ -241,6 +241,17 @@ that can reach GitHub and your LLM provider works. you start it again. On a headless machine start it under `nohup` or in a tmux window, or as a user service. +**Updates.** A Slurm deployment runs from a checkout, and +`OUTERLOOP_AUTO_UPDATE` in `.env` says whether the deploy step moves it. `off`, +the default, leaves the checkout alone: you upgrade when you choose, and the +loop keeps running the code you validated. `release` moves it to the newest +release tag at the next cadence (pre-releases included; the repo is public, so +no credential is needed). `main` follows every merge and is meant for the +kernel's own developers. Whatever moves the checkout, the environment is +synced to the commit that is checked out, or the deploy rolls back to the pair +that matched. The local loop runs the installed package: upgrade it with +`pip install -U outerloop-science` and start it again. + Experiments run wherever your `compute` backend says. Slurm is the first backend; the interface is small (submit a job, poll for completion), so a CI runner, a cloud backend, or a hardware rig plugs in the same way. diff --git a/docs/roadmap.md b/docs/roadmap.md index 73ab42ab..65c8a8c1 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -86,10 +86,9 @@ standing instruction they supersede — a resumed agent honors stale constraints `--dependency=singleton`, absolute `--begin` cadence grid, sbatch retry/backoff; successors submitted FIRST so nothing below can break the chain -- [x] Deploy shim (same script): pull main with the bot PAT → `uv sync - --locked` → exec tick; every step best-effort (bad merges crash ticks, - never the chain). Bot PAT on Torch + repo in the token's selection - still owed by the maintainer before live deploy +- [x] Deploy shim (same script): move the checkout per `OUTERLOOP_AUTO_UPDATE` + (`off` by default, `release`, `main`) → `uv sync --locked` → exec tick; + every step best-effort (bad merges crash ticks, never the chain) - [x] Pause sentinel, heartbeat at tick start (now with disk preflight payload), stale-lease reaping (TTL + tombstone CAS) - [x] `compute`: sbatch submit / sacct status / cancel behind an injectable diff --git a/scripts/README.md b/scripts/README.md index 5053a734..ff206659 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -8,7 +8,7 @@ runs the local loop elsewhere, reading placement from `~/.config/outerloop/.env` | --- | --- | | `setup_branch_protection.sh` | Apply/refresh branch protection on `main`. Solo-phase default: 0 approvals, checks required, admins enforced. Re-run with `1` once a second code owner joins. | | `tick_chain.sbatch` | The Slurm tick entry: per-cadence chain (deploy, one tick, schedule the next) or, with `OUTERLOOP_RESIDENT=1`, the resident loop (`docs/design/resident-tick.md`). | -| `tick_deploy.sh` | The deploy step both modes source: stale-lock sweep, pull `main`, sync deps, read the operator's `.env` knobs, install backends. | +| `tick_deploy.sh` | The deploy step both modes source: stale-lock sweep, move the checkout per `OUTERLOOP_AUTO_UPDATE` (`off` by default; `release`, `main`), sync deps, read the operator's `.env` knobs, install backends. | | `tick_resident.sh` | The resident loop: deploy → one tick under a timeout → sleep to the slot, one `afterany:self` successor, pause exits clean, shim changes resubmit the successor. | | `requeue_moved_successors.sh` | Cancel pending same-name jobs that are no longer on the requested partition. The tick chain runs this before adding successors. | | `sweep_git_locks.sh` | Remove stale `.git/*.lock` files from a checkout (older than N minutes, no live git process); the chain runs it before each deploy. | diff --git a/scripts/tick_chain.sbatch b/scripts/tick_chain.sbatch index cf577be7..5810542b 100755 --- a/scripts/tick_chain.sbatch +++ b/scripts/tick_chain.sbatch @@ -3,8 +3,9 @@ # # Order matters and is the fail-safety: # 1. top up successors FIRST — a crash anywhere below never breaks the chain -# 2. pull main + sync (deploy-at-tick-start; failures logged, tick continues -# on the previous code — a bad merge crashes ticks, never the chain) +# 2. update the checkout per OUTERLOOP_AUTO_UPDATE + sync (deploy-at-tick- +# start; failures logged, tick continues on the previous code — a bad +# merge crashes ticks, never the chain) # 3. exec the tick # # Deploy/config knobs come from the environment (set once via @@ -14,8 +15,8 @@ # OUTERLOOP_ACCOUNT / OUTERLOOP_PARTITION slurm placement (optional; # unset uses Slurm's default association/partition) # OUTERLOOP_CADENCE_MIN tick cadence, default 30 -# OUTERLOOP_PAT_FILE bot PAT for the deploy pull (optional; no file -# means run whatever code is already deployed) +# OUTERLOOP_PAT_FILE bot PAT; a deploy fetch authenticates with it when +# present (optional — the kernel repo is public) # OUTERLOOP_RESIDENT=1 run as the RESIDENT tick (design/resident-tick.md): # one long-lived job that loops deploy -> tick -> # sleep, with a single afterany:self successor. @@ -135,7 +136,7 @@ if [ -w "$LOG_DIR" ]; then fi echo "=== tick $(date -Is) on $(hostname -s) job=${SLURM_JOB_ID:-none}" -# --- 2. deploy: pull main, sync deps, read the operator's knobs (best-effort; +# --- 2. deploy: update the checkout, sync deps, read the operator's knobs (best-effort; # scripts/tick_deploy.sh, sourced so its exports reach the tick) --- . "$OUTERLOOP_HOME/scripts/tick_deploy.sh" diff --git a/scripts/tick_deploy.sh b/scripts/tick_deploy.sh index 4517b4c5..291741ef 100755 --- a/scripts/tick_deploy.sh +++ b/scripts/tick_deploy.sh @@ -1,31 +1,124 @@ #!/usr/bin/env bash # The chain's deploy step, SOURCED by tick_chain.sbatch (it exports the # operator's config knobs into the tick's environment): sweep stale git locks, -# pull main with the bot PAT, sync deps, read the allowlisted .env knobs, -# install the non-claude backends. Every step is best-effort — a bad merge -# crashes the tick, never the chain. Shared by the per-cadence chain (once per -# job) and the resident loop (once per iteration). +# move the checkout per the operator's update policy, sync deps, read the +# allowlisted .env knobs, install the non-claude backends. Every step is +# best-effort — a bad merge crashes the tick, never the chain. Shared by the +# per-cadence chain (once per job) and the resident loop (once per iteration). # # Expects: OUTERLOOP_HOME, OUTERLOOP_ROOT; optional OUTERLOOP_PAT_FILE. -# --- 2. deploy: pull main with the bot PAT, sync deps (best-effort) --- +# --- the operator's .env: ~/.config/outerloop (new) or ~/.config/autoresearch +# (pre-rename). We do NOT source it: sourcing would execute it and let it set +# ANY variable (the chain's own HOME/ROOT/PATH/cadence, arbitrary code). +# Instead single allowlisted keys are read from it, and only when the file is +# ours and not group/world-writable (a writable one could still inject a +# malicious VALUE, e.g. a bad codex binary path). +ENV_FILE="$HOME/.config/outerloop/.env" +[ -r "$ENV_FILE" ] || ENV_FILE="$HOME/.config/autoresearch/.env" +ENV_TRUSTED="" +if [ -r "$ENV_FILE" ]; then + # GNU stat first, BSD stat second (a developer's Mac runs this too) + perms=$(stat -c "%a" "$ENV_FILE" 2>/dev/null || stat -f "%Lp" "$ENV_FILE" 2>/dev/null || echo 777) + owner=$(stat -c "%u" "$ENV_FILE" 2>/dev/null || stat -f "%u" "$ENV_FILE" 2>/dev/null || echo -1) + if [ "$owner" = "$(id -u)" ] && [ $((8#$perms & 8#022)) -eq 0 ]; then + ENV_TRUSTED=1 + else + echo "deploy: refusing to read $ENV_FILE (owner=$owner perms=$perms;" \ + "needs to be yours and not group/world-writable)" + fi +fi +# env_line: the last assignment of key $_k in the trusted .env, into $_line. +# Either spelling, the canonical key first: a pre-rename AUTORESEARCH_ twin +# counts only when the OUTERLOOP_ key is absent, whatever the order. Empty +# when the key is absent or the file is not trusted. +env_line() { + _line="" + [ -n "$ENV_TRUSTED" ] || return 0 + _line=$(grep -E "^${_k}=" "$ENV_FILE" 2>/dev/null | tail -1) + [ -n "$_line" ] || _line=$(grep -E "^AUTORESEARCH_${_k#OUTERLOOP_}=" "$ENV_FILE" 2>/dev/null | tail -1) +} +# env_value: the value of $_line into $_v — the CR of a CRLF-edited file and +# one pair of surrounding quotes stripped +env_value() { + _v=${_line#*=} + _v=${_v%$'\r'} + _v=${_v#[\"\']}; _v=${_v%[\"\']} +} + +# --- 2. deploy: move the checkout per the update policy, sync deps --- +# OUTERLOOP_AUTO_UPDATE, the .env line first, then the environment: +# off (default) run the checkout as it is; the operator upgrades by hand +# release move to the newest release tag (vX.Y.Z; dev/rc pre-releases count) +# main follow every merge — for the kernel's own developers +_k=OUTERLOOP_AUTO_UPDATE; env_line +if [ -n "$_line" ]; then + env_value; POLICY="${_v:-off}" +else + POLICY="${OUTERLOOP_AUTO_UPDATE:-${AUTORESEARCH_AUTO_UPDATE:-off}}" +fi +case "$POLICY" in + off|release|main) ;; + *) echo "deploy: OUTERLOOP_AUTO_UPDATE=$POLICY is not one of off, release, main; not updating" + POLICY=off ;; +esac +# newest_release: of the tags on stdin, the newest release in version order — +# at one X.Y.Z a final release beats rc beats beta beats alpha beats dev; +# anything not shaped like a release tag is ignored +newest_release() { + sed -E -n \ + -e 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)$/\1 \2 \3 4 0 &/p' \ + -e 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)rc([0-9]+)$/\1 \2 \3 3 \4 &/p' \ + -e 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)b([0-9]+)$/\1 \2 \3 2 \4 &/p' \ + -e 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)a([0-9]+)$/\1 \2 \3 1 \4 &/p' \ + -e 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)\.dev([0-9]+)$/\1 \2 \3 0 \4 &/p' \ + | sort -k1,1n -k2,2n -k3,3n -k4,4n -k5,5n | tail -1 | awk '{print $6}' +} # A tick killed mid-fetch leaves .git/*.lock files that make every later # deploy fail and the chain run stale code; sweep locks older than a few # minutes (a live git op holds one for seconds) before touching the checkout. bash "$OUTERLOOP_HOME/scripts/sweep_git_locks.sh" "$OUTERLOOP_HOME" 10 || true -# The PAT never appears in argv (argv is world-readable via /proc on shared -# nodes): git asks for it through GIT_ASKPASS instead. -if [ -n "${OUTERLOOP_PAT_FILE:-}" ] && [ -r "$OUTERLOOP_PAT_FILE" ]; then - ASKPASS=$(mktemp 2>/dev/null || echo "") && [ -n "$ASKPASS" ] && chmod 700 "$ASKPASS" - printf '#!/bin/sh\ncat "%s"\n' "$OUTERLOOP_PAT_FILE" > "$ASKPASS" +DEPLOY_PREV="" +if [ "$POLICY" != off ]; then + # The repo is public, so the fetch needs no credential; with a bot PAT the + # fetch authenticates, and the PAT never appears in argv (argv is + # world-readable via /proc on shared nodes): git asks through GIT_ASKPASS. + KERNEL="https://github.com/outerloop-science/outerloop.git" + ASKPASS="" + if [ -n "${OUTERLOOP_PAT_FILE:-}" ] && [ -r "$OUTERLOOP_PAT_FILE" ]; then + ASKPASS=$(mktemp 2>/dev/null || echo "") + if [ -n "$ASKPASS" ]; then + chmod 700 "$ASKPASS" + printf '#!/bin/sh\ncat "%s"\n' "$OUTERLOOP_PAT_FILE" > "$ASKPASS" + KERNEL="https://x-access-token@github.com/outerloop-science/outerloop.git" + fi + fi + fetch() { + env ${ASKPASS:+GIT_ASKPASS="$ASKPASS"} GIT_TERMINAL_PROMPT=0 \ + git -C "$OUTERLOOP_HOME" fetch --quiet "$KERNEL" "$@" + } DEPLOY_PREV=$(git -C "$OUTERLOOP_HOME" rev-parse HEAD 2>/dev/null || echo "") - if GIT_ASKPASS="$ASKPASS" GIT_TERMINAL_PROMPT=0 git -C "$OUTERLOOP_HOME" fetch --quiet \ - "https://x-access-token@github.com/outerloop-science/outerloop.git" main; then - git -C "$OUTERLOOP_HOME" reset --hard --quiet FETCH_HEAD || echo "deploy: reset failed" + if [ "$POLICY" = main ]; then + if fetch main; then + git -C "$OUTERLOOP_HOME" reset --hard --quiet FETCH_HEAD || echo "deploy: reset failed" + else + echo "deploy: fetch failed; running previous code" + fi + elif fetch 'refs/tags/v*:refs/tags/v*'; then + TAG=$(git -C "$OUTERLOOP_HOME" tag -l 'v*' | newest_release) + if [ -z "$TAG" ]; then + echo "deploy: no release tag found; running previous code" + elif [ "$(git -C "$OUTERLOOP_HOME" rev-parse "refs/tags/$TAG^{commit}" 2>/dev/null)" != "$DEPLOY_PREV" ]; then + if git -C "$OUTERLOOP_HOME" reset --hard --quiet "refs/tags/$TAG"; then + echo "deploy: at release $TAG" + else + echo "deploy: reset to $TAG failed" + fi + fi else echo "deploy: fetch failed; running previous code" fi - rm -f "$ASKPASS" + [ -n "$ASKPASS" ] && rm -f "$ASKPASS" fi # Host-side caches must never land in $HOME: home quotas are tiny on many # clusters and invisible until EDQUOT (verified on Torch — a full home took @@ -76,25 +169,14 @@ if ! (cd "$OUTERLOOP_HOME" && uv sync --locked --quiet); then fi export OUTERLOOP_DEPLOY_BROKEN -# --- config knobs: read the config-driven AUTHOR knobs from the operator .env so +# --- config knobs: the config-driven AUTHOR knobs from the operator .env, so # live config changes need no chain restart. These are where # OUTERLOOP_AUTHOR_BACKEND/_MODEL and the per-backend key files live; the # config-driven climb/followup default from them and the tick preflights them. -# -# We do NOT source .env: sourcing would execute it and let it set ANY variable -# (the chain's own HOME/ROOT/PATH/cadence, arbitrary code). Instead we extract -# only an ALLOWLIST of author knobs — so .env is structurally per-tick author -# config and can never hijack the chain's identity or scheduling. And only from a -# file that is ours and not group/world-writable (a writable one could still -# inject a malicious VALUE, e.g. a bad codex binary path). -# the operator's config dir: ~/.config/outerloop (new) or ~/.config/autoresearch (pre-rename) -ENV_FILE="$HOME/.config/outerloop/.env" -[ -r "$ENV_FILE" ] || ENV_FILE="$HOME/.config/autoresearch/.env" -if [ -r "$ENV_FILE" ]; then - perms=$(stat -c "%a" "$ENV_FILE" 2>/dev/null || echo 777) - owner=$(stat -c "%u" "$ENV_FILE" 2>/dev/null || echo -1) - if [ "$owner" = "$(id -u)" ] && [ $((8#$perms & 8#022)) -eq 0 ]; then - for _k in OUTERLOOP_AUTHOR_BACKEND OUTERLOOP_AUTHOR_MODEL \ +# Only this ALLOWLIST is read, so .env is structurally per-tick author config +# and can never hijack the chain's identity or scheduling. +if [ -n "$ENV_TRUSTED" ]; then + for _k in OUTERLOOP_AUTHOR_BACKEND OUTERLOOP_AUTHOR_MODEL \ OUTERLOOP_CLAUDE_BIN OUTERLOOP_CODEX_BIN OUTERLOOP_CODEX_KEY_FILE \ OUTERLOOP_CLAUDE_KEY_FILE OUTERLOOP_HARNESS_KEY_FILE \ OUTERLOOP_VERTEX_PROJECT OUTERLOOP_VERTEX_REGION \ @@ -107,25 +189,16 @@ if [ -r "$ENV_FILE" ]; then OUTERLOOP_PANEL_CODEX_KEY_FILE \ OUTERLOOP_PANEL_HERMES_KEY_FILE \ REVIEW_HERMES_REPO REVIEW_HERMES_PROVIDER; do - # either spelling, the canonical key first: a pre-rename AUTORESEARCH_ - # twin counts only when the OUTERLOOP_ key is absent, whatever the order - _line=$(grep -E "^${_k}=" "$ENV_FILE" 2>/dev/null | tail -1) - [ -n "$_line" ] || _line=$(grep -E "^AUTORESEARCH_${_k#OUTERLOOP_}=" "$ENV_FILE" 2>/dev/null | tail -1) - # PRESENCE-based, not value-based: a key set to "" in .env is a - # live OFF-SWITCH (OUTERLOOP_PANEL="" disables the panel, - # VERTEX_PROJECT="" reverts to API-key billing) and must override - # an inherited chain value; an ABSENT key changes nothing. - if [ -n "$_line" ]; then - _v=${_line#*=} - _v=${_v%$'\r'} # CRLF-edited file - _v=${_v#[\"\']}; _v=${_v%[\"\']} # optional surrounding quote - export "$_k=$_v" - fi - done - else - echo "deploy: refusing to read $ENV_FILE (owner=$owner perms=$perms;" \ - "needs to be yours and not group/world-writable)" - fi + env_line + # PRESENCE-based, not value-based: a key set to "" in .env is a + # live OFF-SWITCH (OUTERLOOP_PANEL="" disables the panel, + # VERTEX_PROJECT="" reverts to API-key billing) and must override + # an inherited chain value; an ABSENT key changes nothing. + if [ -n "$_line" ]; then + env_value + export "$_k=$_v" + fi + done fi # The codex author binary is a host prerequisite; install it (idempotent, fast diff --git a/src/outerloop/init.py b/src/outerloop/init.py index b6058fe4..8aef10a4 100644 --- a/src/outerloop/init.py +++ b/src/outerloop/init.py @@ -69,6 +69,9 @@ def render_env( lines.append(f"OUTERLOOP_ACCOUNT={a.account}") if a.partition: # optional: unset lets Slurm pick its default partition lines.append(f"OUTERLOOP_PARTITION={a.partition}") + # the checkout stays put until the operator upgrades; `release` follows + # the release tags, `main` every merge (docs/install.md) + lines.append("OUTERLOOP_AUTO_UPDATE=off") if a.image: lines.append(f"OUTERLOOP_IMAGE={a.image}") elif a.uncontained: diff --git a/tests/test_init.py b/tests/test_init.py index d3806014..d715f786 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -159,6 +159,14 @@ def test_render_env_records_the_image_when_set() -> None: a = InitAnswers(compute="local", target="o/r", image="/img/agent-py312.sif") assert "OUTERLOOP_IMAGE=/img/agent-py312.sif" in render_env(a, "/p") assert "OUTERLOOP_IMAGE" not in render_env(InitAnswers(compute="local", target="o/r"), "/p") + + +def test_a_slurm_env_carries_the_default_update_policy() -> None: + """The checkout-based deployment gets `off` written out so the knob is + visible where it is set; the local loop runs the installed package.""" + slurm = render_env(InitAnswers(compute="slurm", target="o/r", root="/r"), "") + assert "\nOUTERLOOP_AUTO_UPDATE=off\n" in slurm + assert "AUTO_UPDATE" not in render_env(InitAnswers(compute="local", target="o/r"), "") # --no-image writes the off-switch, so an image already on disk is not picked up off = InitAnswers(compute="local", target="o/r", uncontained=True) assert "\nOUTERLOOP_IMAGE=\n" in render_env(off, "/p") diff --git a/tests/test_tick_resident.py b/tests/test_tick_resident.py index 29ae4da6..370014dc 100644 --- a/tests/test_tick_resident.py +++ b/tests/test_tick_resident.py @@ -347,6 +347,7 @@ def test_a_failed_sync_rolls_back_when_dependencies_changed(tmp_path: Path) -> N "OUTERLOOP_HOME": str(home), "OUTERLOOP_ROOT": str(root), "OUTERLOOP_PAT_FILE": str(pat), + "OUTERLOOP_AUTO_UPDATE": "main", "HOME": str(tmp_path), } proc = subprocess.run( @@ -403,6 +404,7 @@ def test_a_failed_rollback_marks_the_deploy_broken_so_no_tick_runs(tmp_path: Pat "OUTERLOOP_HOME": str(home), "OUTERLOOP_ROOT": str(root), "OUTERLOOP_PAT_FILE": str(pat), + "OUTERLOOP_AUTO_UPDATE": "main", "HOME": str(tmp_path), } proc = subprocess.run( @@ -463,6 +465,7 @@ def test_a_failed_sync_with_unchanged_lock_keeps_ticking_on_the_current_env(tmp_ "OUTERLOOP_HOME": str(home), "OUTERLOOP_ROOT": str(root), "OUTERLOOP_PAT_FILE": str(pat), + "OUTERLOOP_AUTO_UPDATE": "main", "HOME": str(tmp_path), } proc = subprocess.run( @@ -480,3 +483,143 @@ def test_a_failed_sync_with_unchanged_lock_keeps_ticking_on_the_current_env(tmp_ assert "uv.lock is unchanged; running new code on the current environment" in proc.stdout assert "BROKEN=\n" in proc.stdout assert "reset --hard --quiet OLD" not in log.read_text() # no rollback: HEAD kept + + +def _deploy( + tmp_path: Path, + *, + tags: str = "", + at_tag: bool = False, + env_file: str | None = None, + **env_extra: str, +) -> tuple[subprocess.CompletedProcess[str], str]: + """Source tick_deploy.sh alone. A shim git logs its argv and answers the + release questions (the tags it has; whether HEAD is already at the chosen + tag); uv sync succeeds. Returns the process and git's argv log.""" + home = tmp_path / "home" + (home / "scripts").mkdir(parents=True) + for name in ("tick_deploy.sh", "sweep_git_locks.sh"): + (home / "scripts" / name).write_text((ROOT / "scripts" / name).read_text()) + bindir = tmp_path / "bin" + bindir.mkdir() + log = tmp_path / "gitlog" + log.touch() + tag_commit = "OLD" if at_tag else "TAGGED" + (bindir / "git").write_text( + f"""#!/bin/sh +echo "$@" >> "{log}" +case "$*" in + *"tag -l v*"*) for t in {tags}; do echo "$t"; done ;; + *"rev-parse refs/tags/"*) echo {tag_commit} ;; + *"rev-parse HEAD"*) echo OLD ;; +esac +exit 0 +""" + ) + (bindir / "uv").write_text("#!/bin/sh\nexit 0\n") + for p in bindir.iterdir(): + os.chmod(p, 0o755) + if env_file is not None: + cfg = tmp_path / ".config" / "outerloop" + cfg.mkdir(parents=True) + (cfg / ".env").write_text(env_file) + os.chmod(cfg / ".env", 0o600) + (tmp_path / "pat").write_text("ghp_secret\n") + env = { + **os.environ, + "PATH": f"{bindir}:{os.environ['PATH']}", + "OUTERLOOP_HOME": str(home), + "OUTERLOOP_ROOT": str(tmp_path / "root"), + "HOME": str(tmp_path), + } + for k in ("OUTERLOOP_AUTO_UPDATE", "AUTORESEARCH_AUTO_UPDATE", "OUTERLOOP_PAT_FILE"): + env.pop(k, None) + env.update(env_extra) + proc = subprocess.run( + [ + "bash", + "-c", + f'. "{home}/scripts/tick_deploy.sh"; echo "BROKEN=$OUTERLOOP_DEPLOY_BROKEN"', + ], + env=env, + capture_output=True, + text=True, + timeout=60, + ) + return proc, log.read_text() + + +def test_the_default_update_policy_leaves_the_checkout_alone(tmp_path: Path) -> None: + """No OUTERLOOP_AUTO_UPDATE: the deploy fetches nothing, PAT or not — an + adopter runs the code they installed until they choose to upgrade.""" + proc, gitlog = _deploy(tmp_path, OUTERLOOP_PAT_FILE=str(tmp_path / "pat")) + assert proc.returncode == 0, proc.stderr + assert "fetch" not in gitlog and "reset" not in gitlog + assert "BROKEN=\n" in proc.stdout + + +def test_release_policy_moves_to_the_newest_release_tag(tmp_path: Path) -> None: + """`release` fetches the public repo's tags without a credential and resets + to the newest by version order: at one X.Y.Z a final release beats rc + beats alpha beats dev, and dev10 beats dev2; tags that are not releases + are ignored.""" + proc, gitlog = _deploy( + tmp_path, + tags="v0.1.0.dev2 v0.1.0 v0.1.0rc1 v0.2.0.dev1 v0.1.1 vlatest v1 v0.2.0a1 v0.1.0.dev10", + OUTERLOOP_AUTO_UPDATE="release", + ) + assert proc.returncode == 0, proc.stderr + fetches = [ln for ln in gitlog.splitlines() if " fetch " in ln] + assert fetches == [ + f"-C {tmp_path / 'home'} fetch --quiet" + " https://github.com/outerloop-science/outerloop.git refs/tags/v*:refs/tags/v*" + ] + assert "reset --hard --quiet refs/tags/v0.2.0a1" in gitlog + assert "deploy: at release v0.2.0a1" in proc.stdout + + +def test_release_policy_at_the_current_release_stays_put(tmp_path: Path) -> None: + proc, gitlog = _deploy( + tmp_path, tags="v0.1.0 v0.1.1", at_tag=True, OUTERLOOP_AUTO_UPDATE="release" + ) + assert proc.returncode == 0, proc.stderr + assert "reset" not in gitlog + assert "deploy: at release" not in proc.stdout + + +def test_release_policy_without_a_release_tag_runs_the_previous_code(tmp_path: Path) -> None: + proc, gitlog = _deploy(tmp_path, tags="vlatest", OUTERLOOP_AUTO_UPDATE="release") + assert "reset" not in gitlog + assert "no release tag found" in proc.stdout + + +def test_main_policy_follows_main_and_authenticates_with_the_pat(tmp_path: Path) -> None: + proc, gitlog = _deploy( + tmp_path, OUTERLOOP_AUTO_UPDATE="main", OUTERLOOP_PAT_FILE=str(tmp_path / "pat") + ) + assert proc.returncode == 0, proc.stderr + authed = "fetch --quiet https://x-access-token@github.com/outerloop-science/outerloop.git main" + assert authed in gitlog + assert "reset --hard --quiet FETCH_HEAD" in gitlog + assert "ghp_secret" not in gitlog # the PAT rides GIT_ASKPASS, never argv + + +def test_the_env_file_policy_wins_over_the_environment(tmp_path: Path) -> None: + """The .env line is read before the fetch, in either spelling; an explicit + `off` there switches off an inherited `main`.""" + proc, gitlog = _deploy( + tmp_path, env_file="OUTERLOOP_AUTO_UPDATE=off\n", OUTERLOOP_AUTO_UPDATE="main" + ) + assert proc.returncode == 0, proc.stderr + assert "fetch" not in gitlog + proc, gitlog = _deploy( + tmp_path / "b", tags="v0.1.0", env_file="AUTORESEARCH_AUTO_UPDATE=release\n" + ) + assert "refs/tags/v*:refs/tags/v*" in gitlog + assert "reset --hard --quiet refs/tags/v0.1.0" in gitlog + + +def test_an_unknown_policy_is_off_and_says_so(tmp_path: Path) -> None: + proc, gitlog = _deploy(tmp_path, OUTERLOOP_AUTO_UPDATE="nightly") + assert "fetch" not in gitlog + assert "OUTERLOOP_AUTO_UPDATE=nightly is not one of off, release, main" in proc.stdout From af75951b4af9f53d6f66b5e4b110b870a4128e01 Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Mon, 7 Sep 2026 21:57:04 -0400 Subject: [PATCH 2/2] Auto-update review round 1: the rollback baseline is the recorded synced commit; release tags come from the public remote - A successful sync records the installed commit beside the environment (.venv/.synced-head), and a failed sync rolls back to that commit rather than to "HEAD before the fetch". A checkout the operator moved by hand under the off policy now gets the same code-and-environment guarantee. Without a record the checkout stays, as before. - The release policy asks the public repo for its tags (ls-remote), picks the newest, fetches only that tag with a forced update and resets to it; the checkout's own tags are never consulted, so a stale or local v-tag cannot be chosen. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 4 +- docs/install.md | 6 +-- scripts/tick_deploy.sh | 41 ++++++++++++------ tests/test_tick_resident.py | 84 +++++++++++++++++++++++++++++++------ 4 files changed, 105 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52c04a63..534fb562 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,9 @@ Versions follow [SemVer](https://semver.org). deploy step does to a checkout-based deployment: nothing, move to the newest release tag, or follow every merge. Adopters run what they installed until they choose to upgrade; following `main` is for the kernel's own developers. - `init` writes the default for Slurm deployments. + `init` writes the default for Slurm deployments. A successful sync records + the installed commit, so a checkout moved by hand rolls back the same way a + policy move does when its sync fails. - A seed cache per target (docs/design/eval-cache.md): the tick warms one uv cache from the target's lockfile on the tick host — wheels only, `--no-build`, never the target's code — and every eval and launch job copies it into its diff --git a/docs/install.md b/docs/install.md index be5a4bb5..1624c66b 100644 --- a/docs/install.md +++ b/docs/install.md @@ -247,9 +247,9 @@ the default, leaves the checkout alone: you upgrade when you choose, and the loop keeps running the code you validated. `release` moves it to the newest release tag at the next cadence (pre-releases included; the repo is public, so no credential is needed). `main` follows every merge and is meant for the -kernel's own developers. Whatever moves the checkout, the environment is -synced to the commit that is checked out, or the deploy rolls back to the pair -that matched. The local loop runs the installed package: upgrade it with +kernel's own developers. Whatever moves the checkout, you or the policy, the +environment is synced to the commit that is checked out, or the deploy rolls +back to the last commit whose environment was installed. The local loop runs the installed package: upgrade it with `pip install -U outerloop-science` and start it again. Experiments run wherever your `compute` backend says. Slurm is the first diff --git a/scripts/tick_deploy.sh b/scripts/tick_deploy.sh index 291741ef..ad5415be 100755 --- a/scripts/tick_deploy.sh +++ b/scripts/tick_deploy.sh @@ -78,7 +78,15 @@ newest_release() { # deploy fail and the chain run stale code; sweep locks older than a few # minutes (a live git op holds one for seconds) before touching the checkout. bash "$OUTERLOOP_HOME/scripts/sweep_git_locks.sh" "$OUTERLOOP_HOME" 10 || true -DEPLOY_PREV="" +# DEPLOY_PREV is the commit whose environment is installed: a successful sync +# records it beside the environment, so a checkout moved by hand (the `off` +# policy) still rolls back to it when its sync fails. Before the first record +# it is the checkout as found, which is right when this step is what moves it. +VENV="${UV_PROJECT_ENVIRONMENT:-$OUTERLOOP_HOME/.venv}" +record_synced() { [ -d "$VENV" ] && printf '%s\n' "$1" > "$VENV/.synced-head" 2>/dev/null || true; } +HEAD_BEFORE=$(git -C "$OUTERLOOP_HOME" rev-parse HEAD 2>/dev/null || echo "") +DEPLOY_PREV=$(cat "$VENV/.synced-head" 2>/dev/null || echo "") +[ -n "$DEPLOY_PREV" ] || DEPLOY_PREV="$HEAD_BEFORE" if [ "$POLICY" != off ]; then # The repo is public, so the fetch needs no credential; with a bot PAT the # fetch authenticates, and the PAT never appears in argv (argv is @@ -93,22 +101,25 @@ if [ "$POLICY" != off ]; then KERNEL="https://x-access-token@github.com/outerloop-science/outerloop.git" fi fi - fetch() { - env ${ASKPASS:+GIT_ASKPASS="$ASKPASS"} GIT_TERMINAL_PROMPT=0 \ - git -C "$OUTERLOOP_HOME" fetch --quiet "$KERNEL" "$@" + kgit() { + env ${ASKPASS:+GIT_ASKPASS="$ASKPASS"} GIT_TERMINAL_PROMPT=0 git -C "$OUTERLOOP_HOME" "$@" } - DEPLOY_PREV=$(git -C "$OUTERLOOP_HOME" rev-parse HEAD 2>/dev/null || echo "") if [ "$POLICY" = main ]; then - if fetch main; then + if kgit fetch --quiet "$KERNEL" main; then git -C "$OUTERLOOP_HOME" reset --hard --quiet FETCH_HEAD || echo "deploy: reset failed" else echo "deploy: fetch failed; running previous code" fi - elif fetch 'refs/tags/v*:refs/tags/v*'; then - TAG=$(git -C "$OUTERLOOP_HOME" tag -l 'v*' | newest_release) + # the newest release among the tags the public repo has NOW; a stale or + # local v-tag in this checkout is never chosen, and the chosen tag is + # force-updated to the public one + elif TAGS=$(kgit ls-remote --tags --refs "$KERNEL" 'v*'); then + TAG=$(printf '%s\n' "$TAGS" | sed -n 's|.*refs/tags/||p' | newest_release) if [ -z "$TAG" ]; then echo "deploy: no release tag found; running previous code" - elif [ "$(git -C "$OUTERLOOP_HOME" rev-parse "refs/tags/$TAG^{commit}" 2>/dev/null)" != "$DEPLOY_PREV" ]; then + elif ! kgit fetch --quiet "$KERNEL" "+refs/tags/$TAG:refs/tags/$TAG"; then + echo "deploy: fetch failed; running previous code" + elif [ "$(git -C "$OUTERLOOP_HOME" rev-parse "refs/tags/$TAG^{commit}" 2>/dev/null)" != "$HEAD_BEFORE" ]; then if git -C "$OUTERLOOP_HOME" reset --hard --quiet "refs/tags/$TAG"; then echo "deploy: at release $TAG" else @@ -116,7 +127,7 @@ if [ "$POLICY" != off ]; then fi fi else - echo "deploy: fetch failed; running previous code" + echo "deploy: tag lookup failed; running previous code" fi [ -n "$ASKPASS" ] && rm -f "$ASKPASS" fi @@ -137,11 +148,13 @@ mkdir -p "$UV_CACHE_DIR" "$APPTAINER_CACHEDIR" || true # iteration: the checkout and the installed environment do not match and # could not be made to (the rollback itself failed). Cleared on every deploy. OUTERLOOP_DEPLOY_BROKEN="" -if ! (cd "$OUTERLOOP_HOME" && uv sync --locked --quiet); then +if (cd "$OUTERLOOP_HOME" && uv sync --locked --quiet); then + record_synced "$(git -C "$OUTERLOOP_HOME" rev-parse HEAD 2>/dev/null || echo "")" +else NEW_HEAD=$(git -C "$OUTERLOOP_HOME" rev-parse HEAD 2>/dev/null || echo "") if [ -z "${DEPLOY_PREV:-}" ] || [ "$NEW_HEAD" = "$DEPLOY_PREV" ]; then - # nothing was fetched: the environment is whatever the last good - # deploy installed, and the checkout still matches it + # the checkout is at the commit whose environment is installed (or + # neither is known): nothing to go back to, keep running it echo "deploy: uv sync failed; environment unchanged" elif [ -n "$NEW_HEAD" ] \ && [ "$(git -C "$OUTERLOOP_HOME" rev-parse "$DEPLOY_PREV":uv.lock 2>/dev/null)" \ @@ -152,6 +165,7 @@ if ! (cd "$OUTERLOOP_HOME" && uv sync --locked --quiet); then # not touch uv.lock — and it keeps the tick alive (2026-09-03: the # scratch quota filled and every tick died here for two hours). echo "deploy: uv sync failed but uv.lock is unchanged; running new code on the current environment" + record_synced "$NEW_HEAD" else # dependencies changed and could not be installed; the partial sync # may have altered the environment, so go back to the previous commit @@ -161,6 +175,7 @@ if ! (cd "$OUTERLOOP_HOME" && uv sync --locked --quiet); then if git -C "$OUTERLOOP_HOME" reset --hard --quiet "$DEPLOY_PREV" \ && (cd "$OUTERLOOP_HOME" && uv sync --locked --quiet); then echo "deploy: uv sync failed; back on $DEPLOY_PREV with its environment" + record_synced "$DEPLOY_PREV" else echo "deploy: uv sync failed and the environment could not be made consistent; tick skipped until a deploy succeeds" OUTERLOOP_DEPLOY_BROKEN=1 diff --git a/tests/test_tick_resident.py b/tests/test_tick_resident.py index 370014dc..823d0bc1 100644 --- a/tests/test_tick_resident.py +++ b/tests/test_tick_resident.py @@ -490,16 +490,26 @@ def _deploy( *, tags: str = "", at_tag: bool = False, + head: str = "OLD", + synced: str | None = None, + sync_fails_once: bool = False, env_file: str | None = None, **env_extra: str, ) -> tuple[subprocess.CompletedProcess[str], str]: """Source tick_deploy.sh alone. A shim git logs its argv and answers the - release questions (the tags it has; whether HEAD is already at the chosen - tag); uv sync succeeds. Returns the process and git's argv log.""" + questions the script asks: the public repo's tags (ls-remote), whether + HEAD is already at the chosen tag, HEAD itself (`head`, OLD after a + rollback), and lockfiles that DIFFER between OLD and HEAD. `synced` plants + the recorded synced commit beside a (bare) .venv; `sync_fails_once` makes + the first `uv sync` fail with a quota error. Returns the process and git's + argv log.""" home = tmp_path / "home" (home / "scripts").mkdir(parents=True) for name in ("tick_deploy.sh", "sweep_git_locks.sh"): (home / "scripts" / name).write_text((ROOT / "scripts" / name).read_text()) + if synced is not None: + (home / ".venv").mkdir() + (home / ".venv" / ".synced-head").write_text(synced + "\n") bindir = tmp_path / "bin" bindir.mkdir() log = tmp_path / "gitlog" @@ -509,14 +519,26 @@ def _deploy( f"""#!/bin/sh echo "$@" >> "{log}" case "$*" in - *"tag -l v*"*) for t in {tags}; do echo "$t"; done ;; + *"ls-remote"*) for t in {tags}; do printf 'abc\trefs/tags/%s\n' "$t"; done ;; *"rev-parse refs/tags/"*) echo {tag_commit} ;; - *"rev-parse HEAD"*) echo OLD ;; + *"rev-parse OLD:uv.lock"*) echo lockOLD ;; + *"rev-parse HEAD:uv.lock"*) echo lockNEW ;; + *"rev-parse HEAD"*) if [ -f "{tmp_path}/rolledback" ]; then echo OLD; else echo {head}; fi ;; + *"reset --hard --quiet OLD"*) touch "{tmp_path}/rolledback" ;; esac exit 0 """ ) - (bindir / "uv").write_text("#!/bin/sh\nexit 0\n") + syncs = tmp_path / "syncs" + fail = ( + f'[ "$(wc -l < "{syncs}" | tr -d " ")" -eq 1 ] && ' + '{ echo "error: Disk quota exceeded" >&2; exit 2; }' + if sync_fails_once + else "true" + ) + (bindir / "uv").write_text( + f'#!/bin/sh\ncase "$1" in sync) echo x >> "{syncs}"; {fail} ;; esac\nexit 0\n' + ) for p in bindir.iterdir(): os.chmod(p, 0o755) if env_file is not None: @@ -532,7 +554,12 @@ def _deploy( "OUTERLOOP_ROOT": str(tmp_path / "root"), "HOME": str(tmp_path), } - for k in ("OUTERLOOP_AUTO_UPDATE", "AUTORESEARCH_AUTO_UPDATE", "OUTERLOOP_PAT_FILE"): + for k in ( + "OUTERLOOP_AUTO_UPDATE", + "AUTORESEARCH_AUTO_UPDATE", + "OUTERLOOP_PAT_FILE", + "UV_PROJECT_ENVIRONMENT", + ): env.pop(k, None) env.update(env_extra) proc = subprocess.run( @@ -559,21 +586,23 @@ def test_the_default_update_policy_leaves_the_checkout_alone(tmp_path: Path) -> def test_release_policy_moves_to_the_newest_release_tag(tmp_path: Path) -> None: - """`release` fetches the public repo's tags without a credential and resets - to the newest by version order: at one X.Y.Z a final release beats rc - beats alpha beats dev, and dev10 beats dev2; tags that are not releases - are ignored.""" + """`release` asks the public repo for its tags without a credential, picks + the newest by version order (at one X.Y.Z a final release beats rc beats + alpha beats dev, and dev10 beats dev2; tags that are not releases are + ignored), fetches only that tag and resets to it. Local tags never count.""" proc, gitlog = _deploy( tmp_path, tags="v0.1.0.dev2 v0.1.0 v0.1.0rc1 v0.2.0.dev1 v0.1.1 vlatest v1 v0.2.0a1 v0.1.0.dev10", OUTERLOOP_AUTO_UPDATE="release", ) assert proc.returncode == 0, proc.stderr + public = "https://github.com/outerloop-science/outerloop.git" + assert f"ls-remote --tags --refs {public} v*" in gitlog fetches = [ln for ln in gitlog.splitlines() if " fetch " in ln] assert fetches == [ - f"-C {tmp_path / 'home'} fetch --quiet" - " https://github.com/outerloop-science/outerloop.git refs/tags/v*:refs/tags/v*" + f"-C {tmp_path / 'home'} fetch --quiet {public} +refs/tags/v0.2.0a1:refs/tags/v0.2.0a1" ] + assert "tag -l" not in gitlog # the checkout's own tags are never consulted assert "reset --hard --quiet refs/tags/v0.2.0a1" in gitlog assert "deploy: at release v0.2.0a1" in proc.stdout @@ -615,7 +644,7 @@ def test_the_env_file_policy_wins_over_the_environment(tmp_path: Path) -> None: proc, gitlog = _deploy( tmp_path / "b", tags="v0.1.0", env_file="AUTORESEARCH_AUTO_UPDATE=release\n" ) - assert "refs/tags/v*:refs/tags/v*" in gitlog + assert "ls-remote --tags --refs" in gitlog assert "reset --hard --quiet refs/tags/v0.1.0" in gitlog @@ -623,3 +652,32 @@ def test_an_unknown_policy_is_off_and_says_so(tmp_path: Path) -> None: proc, gitlog = _deploy(tmp_path, OUTERLOOP_AUTO_UPDATE="nightly") assert "fetch" not in gitlog assert "OUTERLOOP_AUTO_UPDATE=nightly is not one of off, release, main" in proc.stdout + + +def test_off_policy_rolls_a_hand_moved_checkout_back_when_its_sync_fails(tmp_path: Path) -> None: + """The operator pulled by hand (HEAD is NEW), the recorded synced commit is + OLD and their lockfiles differ: when NEW's sync fails the checkout goes + back to OLD and OLD's environment is reinstalled — the same guarantee the + policies get, without any fetch.""" + proc, gitlog = _deploy(tmp_path, head="NEW", synced="OLD", sync_fails_once=True) + assert proc.returncode == 0, proc.stderr + assert "fetch" not in gitlog + assert "reset --hard --quiet OLD" in gitlog + assert "back on OLD with its environment" in proc.stdout + assert "BROKEN=\n" in proc.stdout + assert (tmp_path / "home" / ".venv" / ".synced-head").read_text() == "OLD\n" + + +def test_a_successful_sync_records_the_installed_commit(tmp_path: Path) -> None: + proc, gitlog = _deploy(tmp_path, head="NEW", synced="OLD") + assert proc.returncode == 0, proc.stderr + assert "reset" not in gitlog + assert (tmp_path / "home" / ".venv" / ".synced-head").read_text() == "NEW\n" + + +def test_without_a_record_a_failed_sync_keeps_the_checkout(tmp_path: Path) -> None: + """Nothing recorded and nothing moved by this step: there is no known-good + commit to return to, so the checkout stays and the failure is logged.""" + proc, gitlog = _deploy(tmp_path, head="NEW", sync_fails_once=True) + assert "reset" not in gitlog + assert "uv sync failed; environment unchanged" in proc.stdout