chore(ci): bound CI's apt steps and run them again when they stall

.github/workflows/pages.yml

@@ -21,7 +21,7 @@ jobs:
         with:
           bun-version: 1.4.2
       # poster frames and scaled stills (scripts/build-site.ts); the runner image has no ffmpeg
-      - run: sudo apt-get update -qq && sudo apt-get install -y -qq --no-install-recommends ffmpeg
+      - run: sudo bash scripts/ci-bounded-retry.sh 600 bash -c 'apt-get update -qq && apt-get install -y -qq --no-install-recommends ffmpeg'
       # the demo is a Vite build of the client
       - run: bun install --frozen-lockfile
       - run: bun scripts/build-site.ts

.github/workflows/remote-bundles.yml

@@ -95,7 +95,7 @@ jobs:
       - uses: actions/setup-node@v7
         with:
           node-version: '22'
-      - run: sudo apt-get update && sudo apt-get install -y openssh-server && sudo mkdir -p /run/sshd
+      - run: sudo bash scripts/ci-bounded-retry.sh 300 bash -c 'apt-get update && apt-get install -y openssh-server' && sudo mkdir -p /run/sshd
       - run: bun install --frozen-lockfile && bun run build && bun run scripts/build-remote-bundle.ts
       - run: bun run scripts/ssh-integration.ts
         env:

scripts/ci-bounded-retry.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# CI only: apt on the runners has sat on an Ubuntu mirror until the job's own timeout, past
+# apt's own timers (a stall they do not see). Each attempt gets a wall-clock limit; one that
+# fails or runs past it is ended with everything it started, and the command runs again.
+# usage: ci-bounded-retry.sh <seconds per attempt> <command...>
+set -uo pipefail
+limit=$1
+shift
+for attempt in 1 2 3; do
+  timeout --kill-after=15 "$limit" "$@" && exit 0
+  echo "ci-bounded-retry: attempt $attempt of 3 failed or ran past ${limit}s: $*" >&2
+done
+exit 1

scripts/ci-browser.sh

@@ -3,7 +3,9 @@
 set -euo pipefail
 # CI's runner image has none of Chromium's system libraries; a PC is not asked for sudo
 if [ -n "${CI:-}" ]; then
-  bun node_modules/playwright-core/cli.js install --with-deps chromium
+  # as root, so a timed-out attempt's apt can be killed (an unprivileged timeout cannot SIGKILL root's apt)
+  sudo bash scripts/ci-bounded-retry.sh 300 "$(command -v bun)" node_modules/playwright-core/cli.js install-deps chromium
+  bun node_modules/playwright-core/cli.js install chromium
 else
   bun node_modules/playwright-core/cli.js install chromium
 fi