Create the /usr/local/bin/copilot entry in the agent chroot when it is missing

.github/workflows/build.yml

@@ -76,3 +76,4 @@ jobs:
         run: |
           bash tests/setup-iptables-port-spec.test.sh
           bash tests/cli-proxy-probe-classify.test.sh
+          bash tests/usr-local-bin-shim.test.sh

containers/agent/entrypoint.sh

@@ -726,6 +726,210 @@ copy_dind_runner_binary() {
   fi
 }
 
+resolve_chroot_binary_path() {
+  # Resolve a binary name to a chroot-absolute path by searching the same
+  # directories the chroot PATH will contain: $GITHUB_PATH entries (written by
+  # setup-* actions, including tool-cache activations), AWF_HOST_PATH, the
+  # AWF-staged binary directories, and the standard system directories.
+  # Prints the chroot path on success; returns 1 when the binary is not found.
+  local name="$1"
+  local search_path=""
+  local entry=""
+
+  if [ -n "${GITHUB_PATH:-}" ] && [ -f "/host${GITHUB_PATH}" ]; then
+    while IFS= read -r entry; do
+      entry="${entry%$'\r'}"  # strip trailing CR (Windows-style CRLF files)
+      [ -z "${entry}" ] && continue
+      search_path="${search_path}${entry}:"
+    done < "/host${GITHUB_PATH}"
+  fi
+  if [ -n "${AWF_HOST_PATH:-}" ]; then
+    search_path="${search_path}${AWF_HOST_PATH}:"
+  fi
+  search_path="${search_path}/tmp/awf-runner-bin:/tmp/awf-lib:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
+  local dir=""
+  local IFS=':'
+  for dir in ${search_path}; do
+    case "${dir}" in
+      /*) ;;
+      *) continue ;;
+    esac
+    if [ -f "/host${dir}/${name}" ] && [ -x "/host${dir}/${name}" ]; then
+      printf '%s\n' "${dir}/${name}"
+      return 0
+    fi
+  done
+  return 1
+}
+
+prepare_usr_local_bin_overlay() {
+  # Stage a writable replacement for the chroot's /usr/local/bin.
+  #
+  # The host /usr tree is bind-mounted read-only at /host/usr, so a missing
+  # /usr/local/bin/<tool> entry cannot simply be created. Instead we bind-mount
+  # the original directory to a second location that stays reachable inside the
+  # chroot (a unique /tmp/awf-usr-local-bin-orig-XXXXXX directory, read-only) and
+  # build a symlink farm in /tmp/awf-usr-local-bin-XXXXXX that mirrors every
+  # existing entry. The farm is later bind-mounted over /host/usr/local/bin so
+  # both the original binaries and the AWF-created shims resolve.
+  #
+  # Sets USR_LOCAL_BIN_OVERLAY_DIR, USR_LOCAL_BIN_ORIG_DIR and
+  # USR_LOCAL_BIN_OVERLAY_READY=1 on success. No host filesystem content is
+  # modified.
+  if [ "${USR_LOCAL_BIN_OVERLAY_READY}" = "1" ]; then
+    return 0
+  fi
+  if [ ! -d /host/usr/local/bin ]; then
+    return 1
+  fi
+  # Unique staging directories per container run: a fixed name could be reused
+  # by a concurrent AWF container or left stale by a previous run on the same
+  # host /tmp, which would shadow /usr/local/bin with outdated symlinks.
+  local overlay_host_dir="" orig_host_dir=""
+  overlay_host_dir="$(mktemp -d /host/tmp/awf-usr-local-bin-XXXXXX 2>/dev/null || true)"
+  orig_host_dir="$(mktemp -d /host/tmp/awf-usr-local-bin-orig-XXXXXX 2>/dev/null || true)"
+  if [ -z "${overlay_host_dir}" ] || [ -z "${orig_host_dir}" ]; then
+    rmdir "${overlay_host_dir:-/nonexistent}" "${orig_host_dir:-/nonexistent}" 2>/dev/null || true
+    return 1
+  fi
+  USR_LOCAL_BIN_OVERLAY_DIR="${overlay_host_dir#/host}"
+  USR_LOCAL_BIN_ORIG_DIR="${orig_host_dir#/host}"
+  if ! mount --bind /host/usr/local/bin "/host${USR_LOCAL_BIN_ORIG_DIR}" 2>/dev/null; then
+    rmdir "${overlay_host_dir}" "${orig_host_dir}" 2>/dev/null || true
+    USR_LOCAL_BIN_OVERLAY_DIR=""
+    USR_LOCAL_BIN_ORIG_DIR=""
+    return 1
+  fi
+  # Install the teardown here — from this point on there is a bind mount and two
+  # staging directories that must be removed however the container exits.
+  trap cleanup_usr_local_bin_overlay EXIT
+  USR_LOCAL_BIN_OVERLAY_READY=1
+  mount -o remount,ro,bind "/host${USR_LOCAL_BIN_ORIG_DIR}" 2>/dev/null || \
+    echo "[entrypoint][WARN] Could not remount ${USR_LOCAL_BIN_ORIG_DIR} read-only"
+
+  populate_usr_local_bin_farm
+  chown 0:0 "/host${USR_LOCAL_BIN_OVERLAY_DIR}" 2>/dev/null || true
+  chmod 0755 "/host${USR_LOCAL_BIN_OVERLAY_DIR}" 2>/dev/null || true
+  return 0
+}
+
+populate_usr_local_bin_farm() {
+  # Mirror every entry of the real /usr/local/bin into the staging farm as a
+  # symlink. Uses USR_LOCAL_BIN_OVERLAY_DIR / USR_LOCAL_BIN_ORIG_DIR.
+  local entry="" base="" real="" target=""
+  # Iterate the original directory (the overlay is not mounted yet) so relative
+  # symlinks resolve against their real location rather than the staging path.
+  # The dotfile globs are required as well: * skips hidden entries, which would
+  # otherwise disappear from /usr/local/bin once the overlay is activated.
+  for entry in /host/usr/local/bin/* /host/usr/local/bin/.[!.]* /host/usr/local/bin/..?*; do
+    [ -e "${entry}" ] || continue
+    base="$(basename "${entry}")"
+    real="$(readlink -f "${entry}" 2>/dev/null || true)"
+    [ -z "${real}" ] && continue
+    case "${real}" in
+      /host/usr/local/bin/*)
+        # Regular file living directly in /usr/local/bin — point at the
+        # read-only copy of the original directory.
+        target="${USR_LOCAL_BIN_ORIG_DIR}/${real#/host/usr/local/bin/}"
+        ;;
+      /host/*)
+        # Symlink into another host location — use the resolved chroot path so
+        # relative symlinks do not break when re-rooted.
+        target="${real#/host}"
+        ;;
+      *) continue ;;
+    esac
+    ln -sfn "${target}" "/host${USR_LOCAL_BIN_OVERLAY_DIR}/${base}" 2>/dev/null || true
+  done
+}
+
+cleanup_usr_local_bin_overlay() {
+  # Tear down the /usr/local/bin overlay. Installed as an EXIT trap in the
+  # container's root shell: the chroot shell exec's capsh, so its own EXIT trap
+  # never fires, and the unmounts need CAP_SYS_ADMIN, which is dropped before the
+  # user command runs.
+  #
+  # Order matters: the farm has to be unmounted from /usr/local/bin before its
+  # symlinks are deleted, and the mirrored original directory has to be unmounted
+  # before its (now empty) mountpoint can be removed. Only symlinks are deleted —
+  # the mirrored directory is a read-only bind mount of the real /usr/local/bin
+  # and must never have its contents removed.
+  [ "${USR_LOCAL_BIN_OVERLAY_READY:-0}" = "1" ] || return 0
+  if [ "${USR_LOCAL_BIN_OVERLAY_MOUNTED:-0}" = "1" ]; then
+    umount /host/usr/local/bin 2>/dev/null || \
+      echo "[entrypoint][WARN] Could not unmount the /usr/local/bin overlay"
+    USR_LOCAL_BIN_OVERLAY_MOUNTED=0
+  fi
+  find "/host${USR_LOCAL_BIN_OVERLAY_DIR}" -maxdepth 1 -type l -delete 2>/dev/null || true
+  umount "/host${USR_LOCAL_BIN_ORIG_DIR}" 2>/dev/null || \
+    echo "[entrypoint][WARN] Could not unmount ${USR_LOCAL_BIN_ORIG_DIR}"
+  rmdir "/host${USR_LOCAL_BIN_OVERLAY_DIR}" "/host${USR_LOCAL_BIN_ORIG_DIR}" 2>/dev/null || true
+  USR_LOCAL_BIN_OVERLAY_READY=0
+}
+
+ensure_usr_local_bin_shims() {
+  # Some agentic harnesses spawn their CLI through a hardcoded absolute path
+  # (e.g. gh-aw's Copilot engine uses /usr/local/bin/copilot) instead of a PATH
+  # lookup. Their installers can skip creating that file — a Copilot CLI
+  # tool-cache hit only exports the cache directory to PATH/GITHUB_PATH — which
+  # makes the harness fail immediately with `spawn /usr/local/bin/copilot ENOENT`.
+  #
+  # AWF_ENSURE_USR_LOCAL_BIN holds a comma-separated list of binary names that
+  # must be reachable at /usr/local/bin/<name> inside the chroot. For each
+  # missing name we resolve the real binary via PATH/GITHUB_PATH and create the
+  # expected symlink, so AWF no longer depends on upstream tool-cache behavior.
+  # The staging directories live in /tmp and are root-owned (0755), so the
+  # unprivileged chroot user cannot inject binaries into them. They are unmounted
+  # and removed by cleanup_usr_local_bin_overlay() on container exit.
+  USR_LOCAL_BIN_OVERLAY_DIR=""
+  USR_LOCAL_BIN_ORIG_DIR=""
+  USR_LOCAL_BIN_OVERLAY_READY=0
+  USR_LOCAL_BIN_OVERLAY_MOUNTED=0
+
+  [ -z "${AWF_ENSURE_USR_LOCAL_BIN:-}" ] && return 0
+
+  local name="" resolved="" created=0
+  local names=()
+  IFS=',' read -r -a names <<< "${AWF_ENSURE_USR_LOCAL_BIN}"
+  for name in "${names[@]}"; do
+    [ -z "${name}" ] && continue
+    if [[ ! "${name}" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]*$ ]]; then
+      echo "[entrypoint][WARN] Ignoring invalid AWF_ENSURE_USR_LOCAL_BIN entry: ${name}"
+      continue
+    fi
+    if [ -e "/host/usr/local/bin/${name}" ]; then
+      continue
+    fi
+    resolved="$(resolve_chroot_binary_path "${name}" || true)"
+    if [ -z "${resolved}" ]; then
+      echo "[entrypoint][WARN] /usr/local/bin/${name} is missing and '${name}' could not be resolved from PATH/GITHUB_PATH"
+      continue
+    fi
+    echo "[entrypoint] /usr/local/bin/${name} is missing; resolved '${name}' at ${resolved}"
+    if ln -sfn "${resolved}" "/host/usr/local/bin/${name}" 2>/dev/null; then
+      echo "[entrypoint] Created /usr/local/bin/${name} -> ${resolved}"
+    elif prepare_usr_local_bin_overlay && \
+         ln -sfn "${resolved}" "/host${USR_LOCAL_BIN_OVERLAY_DIR}/${name}" 2>/dev/null; then
+      created=1
+      echo "[entrypoint] Staged /usr/local/bin/${name} -> ${resolved} (read-only /usr overlay)"
+    else
+      echo "[entrypoint][WARN] Could not create /usr/local/bin/${name}; harnesses using that hardcoded path may fail"
+    fi
+  done
+
+  if [ "${created}" = "1" ]; then
+    if mount --bind "/host${USR_LOCAL_BIN_OVERLAY_DIR}" /host/usr/local/bin 2>/dev/null; then
+      USR_LOCAL_BIN_OVERLAY_MOUNTED=1
+      mount -o remount,ro,bind /host/usr/local/bin 2>/dev/null || \
+        echo "[entrypoint][WARN] Could not remount /host/usr/local/bin read-only"
+      echo "[entrypoint] Activated /usr/local/bin overlay with AWF-created shims"
+    else
+      echo "[entrypoint][WARN] Could not activate /usr/local/bin overlay; hardcoded-path spawns may fail"
+    fi
+  fi
+}
+
 copy_awf_ca_cert() {
   # Copy AWF CA certificate to chroot-accessible path for ssl-bump TLS trust.
   # NODE_EXTRA_CA_CERTS points to /usr/local/share/ca-certificates/awf-ca.crt which
@@ -1264,6 +1468,7 @@ run_chroot_command() {
   copy_preload_libs
   copy_agent_helper_scripts
   copy_dind_runner_binary
+  ensure_usr_local_bin_shims
   copy_awf_ca_cert
   copy_system_ca_bundle
   setup_chroot_etc
@@ -1325,6 +1530,10 @@ run_chroot_command() {
   if [ -n "${ONE_SHOT_TOKEN_LIB}" ] || [ -n "${AWF_CA_CHROOT}" ] || [ -n "${SYSTEM_CA_CHROOT}" ] || [ -n "${CHROOT_KEY_HELPER}" ] || [ -n "${STAGED_RUNNER_BINARY_CHROOT}" ]; then
     CLEANUP_CMD="${CLEANUP_CMD}; rm -rf /tmp/awf-lib 2>/dev/null || true"
   fi
+  # NOTE: the /usr/local/bin overlay is torn down by cleanup_usr_local_bin_overlay(),
+  # which is installed as an EXIT trap in the container's root shell — the chroot
+  # shell exec's capsh below, so its EXIT trap never fires, and the bind mounts
+  # must be unmounted with the capabilities the chroot user no longer has.
 
   # Transfer ownership of gh-aw config directories to the chroot user.
   # On self-hosted runners these directories are created by the host-side

docs/awf-config-spec.md

@@ -406,7 +406,7 @@ passthrough. A conforming implementation MUST NOT inherit them from the host:
 | System | `PATH`, `PWD`, `OLDPWD`, `SHLVL`, `_`, `SUDO_COMMAND`, `SUDO_USER`, `SUDO_UID`, `SUDO_GID` |
 | Proxy | `HTTP_PROXY`, `HTTPS_PROXY`, `http_proxy`, `https_proxy`, `NO_PROXY`, `no_proxy`, `ALL_PROXY`, `all_proxy`, `FTP_PROXY`, `ftp_proxy` |
 | Actions runtime credentials | `ACTIONS_RUNTIME_TOKEN`, `ACTIONS_RESULTS_URL`, `ACTIONS_ID_TOKEN_REQUEST_URL`, `ACTIONS_ID_TOKEN_REQUEST_TOKEN` |
-| AWF internal controls | `AWF_PREFLIGHT_BINARY`, `AWF_GEMINI_ENABLED` |
+| AWF internal controls | `AWF_PREFLIGHT_BINARY`, `AWF_ENSURE_USR_LOCAL_BIN`, `AWF_GEMINI_ENABLED` |
 
 > **Note:** Host proxy variables are read for upstream proxy auto-detection
 > (see `--upstream-proxy`) but MUST NOT propagate into the agent container.

docs/troubleshooting.md

@@ -328,7 +328,18 @@ agent container mounts the host filesystem:
 - Commands that run *inside* the AWF sandbox still cannot create this symlink,
   because `/usr` is mounted read-only there.
 
-**Workarounds:**
+**Automatic mitigation (Copilot CLI):** For Copilot runs, AWF's agent
+entrypoint now checks `/usr/local/bin/copilot` at container start. When the
+entry is missing, it resolves `copilot` through the chroot `PATH` and the
+`$GITHUB_PATH` entries (which is where a warm Copilot CLI tool-cache is
+activated) and creates the expected symlink before the command runs. Because
+host `/usr` is read-only, AWF stages the symlink in a root-owned directory and
+bind-mounts it over the chroot's `/usr/local/bin`, preserving every existing
+entry; the host filesystem is never modified. The behavior is driven by the
+internal `AWF_ENSURE_USR_LOCAL_BIN` environment variable (comma-separated
+binary names) and is a no-op when the path already exists.
+
+**Workarounds (other tools):**
 
 1. **Host-side symlink before invoking `awf`** — if you control the step
    immediately before AWF starts the sandbox, create the missing binary on

src/services/agent-environment-runtime.test.ts

@@ -113,6 +113,36 @@ describe('agent environment: runtime', () => {
     expect(environment.AWF_PREFLIGHT_BINARY).toBeUndefined();
   });
 
+  it('should set AWF_ENSURE_USR_LOCAL_BIN=copilot when running Copilot CLI command', () => {
+    const result = generateDockerCompose(
+      { ...mockConfig, agentCommand: 'copilot --version' },
+      mockNetworkConfig,
+    );
+    const environment = result.services.agent.environment as Record<string, string>;
+
+    expect(environment.AWF_ENSURE_USR_LOCAL_BIN).toBe('copilot');
+  });
+
+  it('should set AWF_ENSURE_USR_LOCAL_BIN=copilot when copilotGithubToken is present', () => {
+    const result = generateDockerCompose(
+      { ...mockConfig, agentCommand: './my-copilot-wrapper.sh', copilotGithubToken: 'ghu_test_token' },
+      mockNetworkConfig,
+    );
+    const environment = result.services.agent.environment as Record<string, string>;
+
+    expect(environment.AWF_ENSURE_USR_LOCAL_BIN).toBe('copilot');
+  });
+
+  it('should not set AWF_ENSURE_USR_LOCAL_BIN for non-Copilot commands', () => {
+    const result = generateDockerCompose(
+      { ...mockConfig, agentCommand: 'echo test' },
+      mockNetworkConfig,
+    );
+    const environment = result.services.agent.environment as Record<string, string>;
+
+    expect(environment.AWF_ENSURE_USR_LOCAL_BIN).toBeUndefined();
+  });
+
   it('should set AWF_STAGED_RUNNER_BINARY_NAME in /tmp docker-host-path-prefix mode', () => {
     const stagePrefix = fs.mkdtempSync(path.join('/tmp', 'gh-aw-'));
     try {

src/services/agent-environment/excluded-vars.ts

@@ -18,6 +18,7 @@ export function buildExclusionSet(config: WrapperConfig): Set<string> {
     'ACTIONS_ID_TOKEN_REQUEST_TOKEN',
     ...PROXY_ENV_VARS,
     'AWF_PREFLIGHT_BINARY',
+    'AWF_ENSURE_USR_LOCAL_BIN',
     'AWF_STAGED_RUNNER_BINARY_NAME',
     'AWF_GEMINI_ENABLED',
     'MCP_GATEWAY_HOST_DOMAIN',

src/services/agent-environment/tool-specific-environment.ts

@@ -46,6 +46,15 @@ export function buildToolEnvironment(params: ToolEnvironmentParams): void {
     environment.AWF_PREFLIGHT_BINARY = 'codex';
   }
 
+  // gh-aw's Copilot engine spawns the CLI through the hardcoded path
+  // /usr/local/bin/copilot. Its installer skips creating that wrapper on a
+  // tool-cache hit (it only exports the cache dir to PATH/GITHUB_PATH), so the
+  // spawn fails with ENOENT. Tell the entrypoint to create the expected
+  // /usr/local/bin/copilot entry inside the chroot when it is missing.
+  if (config.copilotGithubToken || hasCopilotProviderApiKey || isCopilotCommand) {
+    environment.AWF_ENSURE_USR_LOCAL_BIN = 'copilot';
+  }
+
   // Claude Code uses Bun with JavaScriptCore (JSC). Under gVisor's userspace
   // kernel, JSC's JIT compiler triggers SIGSEGV/SIGABRT crashes. Setting
   // BUN_JSC_useJIT=0 forces Bun into interpreter-only mode, which is slower

tests/entrypoint-phase-functions.test.sh

@@ -31,6 +31,11 @@ required_functions=(
   copy_preload_libs
   copy_agent_helper_scripts
   copy_dind_runner_binary
+  resolve_chroot_binary_path
+  prepare_usr_local_bin_overlay
+  populate_usr_local_bin_farm
+  cleanup_usr_local_bin_overlay
+  ensure_usr_local_bin_shims
   copy_awf_ca_cert
   copy_system_ca_bundle
   check_chroot_prereqs
@@ -117,6 +122,7 @@ chroot_helpers=(
   'copy_preload_libs'
   'copy_agent_helper_scripts'
   'copy_dind_runner_binary'
+  'ensure_usr_local_bin_shims'
   'copy_awf_ca_cert'
   'copy_system_ca_bundle'
   'setup_chroot_etc'