fix(server): sweep stale worktree trash dirs via rm -rf subprocess

apps/server/src/vcs/GitVcsDriverCore.test.ts

@@ -635,6 +635,41 @@ it.effect("backs off failed upstream refreshes across linked worktrees", () =>
   ).pipe(Effect.provide(ServerConfigLayer.pipe(Layer.provideMerge(NodeServices.layer)))),
 );
 
+// Runs with the live clock (outside the layered suite) because the trash
+// delete happens in a detached fiber doing real subprocess/filesystem work.
+it.live("force-removal sweeps stale trash directories left by earlier removals", () =>
+  Effect.gen(function* () {
+    const cwd = yield* makeTmpDir();
+    const { initialBranch } = yield* initRepoWithCommit(cwd);
+    const pathService = yield* Path.Path;
+    const worktreesRoot = yield* makeTmpDir("git-worktrees-");
+    const worktreePath = pathService.join(worktreesRoot, "feature-worktree");
+    const driver = yield* GitVcsDriver.GitVcsDriver;
+    const fileSystem = yield* FileSystem.FileSystem;
+
+    yield* driver.createWorktree({
+      cwd,
+      path: worktreePath,
+      refName: initialBranch,
+      newRefName: "feature/sweep-trash",
+    });
+    const staleTrashPath = pathService.join(worktreesRoot, ".feature-worktree.removing-deadbeef");
+    yield* fileSystem.makeDirectory(staleTrashPath, { recursive: true });
+    yield* writeTextFile(staleTrashPath, "leftover.txt", "stale");
+
+    yield* driver.removeWorktree({ cwd, path: worktreePath, force: true });
+    assert.equal(yield* fileSystem.exists(worktreePath), false);
+
+    // The delete runs in a detached fiber; wait for it to clear both the
+    // renamed-aside worktree and the stale trash directory.
+    yield* Effect.gen(function* () {
+      while ((yield* fileSystem.readDirectory(worktreesRoot)).length > 0) {
+        yield* Effect.sleep("50 millis");
+      }
+    }).pipe(Effect.timeout("10 seconds"));
+  }).pipe(Effect.provide(TestLayer)),
+);
+
 it.layer(TestLayer)("GitVcsDriver core integration", (it) => {
   describe("process environment", () => {
     it.effect("preserves the caller locale for general Git subprocesses", () =>

apps/server/src/vcs/GitVcsDriverCore.ts

@@ -27,6 +27,7 @@ import {
   type VcsRef,
 } from "@t3tools/contracts";
 import { dedupeRemoteBranchesWithLocalMatches, normalizeGitRemoteUrl } from "@t3tools/shared/git";
+import { HostProcessPlatform } from "@t3tools/shared/hostProcess";
 import { compactTraceAttributes } from "@t3tools/shared/observability";
 import { decodeJsonResult } from "@t3tools/shared/schemaJson";
 import { gitCommandDuration, gitCommandsTotal, withMetrics } from "../observability/Metrics.ts";
@@ -39,6 +40,12 @@ import {
 import { ServerConfig } from "../config.ts";
 
 const DEFAULT_TIMEOUT_MS = 30_000;
+// Matches the scratch names produced by removeWorktree's rename-aside step.
+const WORKTREE_TRASH_PATTERN = /^\..+\.removing-[0-9a-f]{8}$/;
+
+class WorktreeTrashDeleteError extends Data.TaggedError("WorktreeTrashDeleteError")<{
+  readonly detail: string;
+}> {}
 const DEFAULT_MAX_OUTPUT_BYTES = 1_000_000;
 const OUTPUT_TRUNCATED_MARKER = "\n\n[truncated]";
 const PREPARED_COMMIT_PATCH_MAX_OUTPUT_BYTES = 49_000;
@@ -2896,6 +2903,58 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
       input.branch,
     ]);
 
+  // Deleting via a child `rm -rf` keeps a multi-gigabyte tree walk off the
+  // server's libuv threadpool (shared with WebSocket compression and the rest
+  // of its file I/O), and unlike `fs.rm` it keeps going past files it cannot
+  // unlink (macOS App Management denies unlinking inside .app bundles) instead
+  // of aborting the whole walk.
+  const deleteWorktreeTrashDirectory = Effect.fn("deleteWorktreeTrashDirectory")(function* (
+    targetPath: string,
+  ) {
+    const platform = yield* HostProcessPlatform;
+    if (platform === "win32") {
+      return yield* fileSystem.remove(targetPath, { recursive: true, force: true });
+    }
+    const { exitCode, stderr } = yield* Effect.scoped(
+      Effect.gen(function* () {
+        const child = yield* commandSpawner.spawn(
+          ChildProcess.make("rm", ["-rf", "--", targetPath], {
+            stdin: "ignore",
+            stdout: "ignore",
+          }),
+        );
+        const [stderrText, code] = yield* Effect.all(
+          [Stream.mkString(Stream.decodeText(child.stderr)), child.exitCode],
+          { concurrency: "unbounded" },
+        );
+        return { exitCode: code, stderr: stderrText };
+      }),
+    );
+    if (exitCode !== 0) {
+      return yield* new WorktreeTrashDeleteError({
+        detail: `rm exited with code ${exitCode}: ${stderr.trim() || "no stderr output"}`,
+      });
+    }
+  });
+
+  const sweepWorktreeTrash = Effect.fn("sweepWorktreeTrash")(function* (parentDir: string) {
+    const entries = yield* fileSystem
+      .readDirectory(parentDir)
+      .pipe(Effect.orElseSucceed((): ReadonlyArray<string> => []));
+    for (const entry of entries) {
+      if (!WORKTREE_TRASH_PATTERN.test(entry)) continue;
+      const target = path.join(parentDir, entry);
+      yield* deleteWorktreeTrashDirectory(target).pipe(
+        Effect.catch((error) =>
+          Effect.logWarning(
+            `GitVcsDriver.removeWorktree: could not delete worktree trash at ${target}`,
+            error,
+          ),
+        ),
+      );
+    }
+  });
+
   const removeWorktree: GitVcsDriver.GitVcsDriver["Service"]["removeWorktree"] = Effect.fn(
     "removeWorktree",
   )(function* (input) {
@@ -2974,14 +3033,13 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
     const worktreeDirExists = yield* fileSystem
       .exists(matched.onDiskPath)
       .pipe(Effect.orElseSucceed(() => false));
-    let trashPath: string | null = null;
     if (worktreeDirExists) {
       const suffix = yield* crypto.randomUUIDv4.pipe(
         Effect.mapError((cause) =>
           failure("Could not generate a scratch name for worktree removal.", cause),
         ),
       );
-      trashPath = path.join(
+      const trashPath = path.join(
         path.dirname(matched.onDiskPath),
         `.${path.basename(matched.onDiskPath)}.removing-${suffix.slice(0, 8)}`,
       );
@@ -2999,11 +3057,12 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
       fallbackErrorDetail: "git worktree prune failed",
     });
 
-    if (trashPath !== null) {
-      yield* fileSystem
-        .remove(trashPath, { recursive: true, force: true })
-        .pipe(Effect.ignoreCause({ log: true }), Effect.forkDetach);
-    }
+    // The sweep picks up the directory renamed aside above along with any
+    // trash left behind by earlier removals whose deletes failed.
+    yield* sweepWorktreeTrash(path.dirname(matched.onDiskPath)).pipe(
+      Effect.ignoreCause({ log: true }),
+      Effect.forkDetach,
+    );
   });
 
   const renameBranch: GitVcsDriver.GitVcsDriver["Service"]["renameBranch"] = Effect.fn(