diff --git a/.agents/skills/test-t3-app/references/sqlite-fixtures.md b/.agents/skills/test-t3-app/references/sqlite-fixtures.md index 8eca543cf979..42959281e61f 100644 --- a/.agents/skills/test-t3-app/references/sqlite-fixtures.md +++ b/.agents/skills/test-t3-app/references/sqlite-fixtures.md @@ -4,7 +4,7 @@ Load this reference only when inspecting or seeding local T3 state directly. ## Select the correct database -When `--base-dir` or `--home-dir` is explicit, runtime state lives under `/userdata` and the database path is `/userdata/state.sqlite`. The `/dev` state directory is only the fallback for an implicit development home, preventing an ordinary `vp run dev` from touching production state. +When `--base-dir` or `--home-dir` is explicit, runtime state lives under `/userdata`. Hot projections and runtime state are stored in `state.sqlite`; cold archive manifests and compressed chunks are stored in `archive.sqlite`. The `/dev` state directory is only the fallback for an implicit development home, preventing an ordinary `vp run dev` from touching production state. Start the target runtime once before seeding so all migrations have run. Use an isolated base directory. Stop the server before writes to avoid racing application state or an active projection. @@ -18,6 +18,15 @@ node apps/server/scripts/t3-sqlite-state.ts query \ --sql "SELECT name FROM sqlite_schema WHERE type = 'table' ORDER BY name" ``` +The helper targets `state.sqlite` by default. Select the cold archive database explicitly for read-only inspection: + +```bash +node apps/server/scripts/t3-sqlite-state.ts query \ + --base-dir \ + --database archive \ + --sql "SELECT t.thread_id, t.archive_version, COUNT(c.chunk_index) AS chunk_count FROM archive_threads AS t LEFT JOIN archive_thread_chunks AS c ON c.thread_id = t.thread_id GROUP BY t.thread_id, t.archive_version ORDER BY t.thread_id" +``` + Inspect current columns before writing a fixture: ```bash @@ -36,6 +45,8 @@ node apps/server/scripts/t3-sqlite-state.ts exec \ Use one statement per invocation for both `query` and `exec`; the helper wraps writes in a transaction and prints the backup path after a successful mutation. Use a single insert with multiple value rows when a fixture needs several records. +Cold archive manifests and chunks are authoritative recovery data. Prefer `--database archive` for queries only. Exercise archive and restore behavior through application commands and APIs instead of constructing or changing archive bundles directly; only use archive writes for a deliberately isolated corruption or compatibility fixture. + ## Seed projection data carefully The web UI primarily reads these projection tables: diff --git a/apps/mobile/src/features/archive/ArchivedThreadsRouteScreen.tsx b/apps/mobile/src/features/archive/ArchivedThreadsRouteScreen.tsx index 9ad4790faab1..eb0b62e97b95 100644 --- a/apps/mobile/src/features/archive/ArchivedThreadsRouteScreen.tsx +++ b/apps/mobile/src/features/archive/ArchivedThreadsRouteScreen.tsx @@ -2,7 +2,7 @@ import type { EnvironmentId } from "@t3tools/contracts"; import * as Arr from "effect/Array"; import * as Order from "effect/Order"; import { useFocusEffect } from "@react-navigation/native"; -import { useCallback, useMemo, useState } from "react"; +import { useCallback, useMemo, useRef, useState } from "react"; import { useSavedRemoteConnections } from "../../state/use-remote-environment-registry"; import { useArchivedThreadListActions } from "../home/useThreadListActions"; @@ -15,12 +15,18 @@ import { refreshArchivedThreadsForEnvironment, useArchivedThreadSnapshots, } from "./useArchivedThreadSnapshots"; +import type { EnvironmentThreadShell } from "@t3tools/client-runtime/state/shell"; +import { scopedThreadKey } from "../../lib/scopedEntities"; export function ArchivedThreadsRouteScreen() { const { savedConnectionsById } = useSavedRemoteConnections(); const [searchQuery, setSearchQuery] = useState(""); const [selectedEnvironmentId, setSelectedEnvironmentId] = useState(null); const [sortOrder, setSortOrder] = useState("newest"); + const [unarchivingThreadKeys, setUnarchivingThreadKeys] = useState>( + () => new Set(), + ); + const unarchivingThreadKeysRef = useRef(new Set()); const environments = useMemo>( () => Arr.sort( @@ -65,6 +71,25 @@ export function ArchivedThreadsRouteScreen() { ); const { unarchiveThread, confirmDeleteThread } = useArchivedThreadListActions(refreshChangedEnvironment); + const handleUnarchiveThread = useCallback( + async (thread: EnvironmentThreadShell) => { + const threadKey = scopedThreadKey(thread.environmentId, thread.id); + if (unarchivingThreadKeysRef.current.has(threadKey)) return; + unarchivingThreadKeysRef.current.add(threadKey); + setUnarchivingThreadKeys((current) => new Set(current).add(threadKey)); + try { + await unarchiveThread(thread); + } finally { + unarchivingThreadKeysRef.current.delete(threadKey); + setUnarchivingThreadKeys((current) => { + const next = new Set(current); + next.delete(threadKey); + return next; + }); + } + }, + [unarchiveThread], + ); useFocusEffect( useCallback(() => { @@ -83,10 +108,11 @@ export function ArchivedThreadsRouteScreen() { onRefresh={refresh} onSearchQueryChange={setSearchQuery} onSortOrderChange={setSortOrder} - onUnarchiveThread={unarchiveThread} + onUnarchiveThread={(thread) => void handleUnarchiveThread(thread)} searchQuery={searchQuery} selectedEnvironmentId={selectedEnvironmentId} sortOrder={sortOrder} + unarchivingThreadKeys={unarchivingThreadKeys} /> ); } diff --git a/apps/mobile/src/features/archive/ArchivedThreadsScreen.tsx b/apps/mobile/src/features/archive/ArchivedThreadsScreen.tsx index 374d17ac4762..962adb840aff 100644 --- a/apps/mobile/src/features/archive/ArchivedThreadsScreen.tsx +++ b/apps/mobile/src/features/archive/ArchivedThreadsScreen.tsx @@ -27,11 +27,15 @@ import { AppText as Text } from "../../components/AppText"; import { EmptyState } from "../../components/EmptyState"; import { EnvironmentMachineSymbol } from "../../components/EnvironmentMachineSymbol"; import { ProjectFavicon } from "../../components/ProjectFavicon"; -import { relativeTime } from "../../lib/time"; +import { scopedThreadKey } from "../../lib/scopedEntities"; import { useUniwindTheme } from "../../lib/useUniwindTheme"; import { useServerConfigs } from "../../state/entities"; import { ThreadSwipeable } from "../home/thread-swipe-actions"; -import type { ArchivedThreadGroup, ArchivedThreadSortOrder } from "./archivedThreadList"; +import { + formatArchivedThreadRelativeTime, + type ArchivedThreadGroup, + type ArchivedThreadSortOrder, +} from "./archivedThreadList"; import { SettingsScreenContent } from "../settings/components/SettingsScreen"; export interface ArchivedThreadsHeaderEnvironment { @@ -182,6 +186,7 @@ function ArchivedThreadRow(props: { readonly environmentLabel: string | null; readonly isFirst: boolean; readonly isLast: boolean; + readonly isUnarchiving: boolean; readonly onDelete: () => void; readonly onSwipeableClose: (methods: SwipeableMethods) => void; readonly onSwipeableWillOpen: (methods: SwipeableMethods) => void; @@ -191,10 +196,13 @@ function ArchivedThreadRow(props: { }) { const { width: windowWidth } = useWindowDimensions(); const cardColor = useUniwindTheme()["--color-card"]; - const timestamp = relativeTime(props.thread.archivedAt ?? props.thread.updatedAt); + const timestamp = formatArchivedThreadRelativeTime( + props.thread.archivedAt ?? props.thread.updatedAt, + ); const subtitle = [props.environmentLabel, props.thread.branch].filter((part): part is string => Boolean(part), ); + const onDelete = props.isUnarchiving ? () => undefined : props.onDelete; return ( undefined : props.onUnarchive, }} simultaneousWith={props.simultaneousSwipeGesture} threadTitle={props.thread.title} @@ -227,12 +236,16 @@ function ArchivedThreadRow(props: { className={`flex-row items-center gap-3 bg-card px-4 py-3 ${props.isLast ? "" : "border-b border-separator"}`} > - + {props.isUnarchiving ? ( + + ) : ( + + )} @@ -243,9 +256,11 @@ function ArchivedThreadRow(props: { > {props.thread.title} - - {timestamp} - + {timestamp ? ( + + {timestamp} + + ) : null} {subtitle.length > 0 ? ( @@ -298,6 +313,7 @@ export function ArchivedThreadsScreen(props: { readonly onSearchQueryChange: (query: string) => void; readonly onSortOrderChange: (sortOrder: ArchivedThreadSortOrder) => void; readonly onUnarchiveThread: (thread: EnvironmentThreadShell) => void; + readonly unarchivingThreadKeys: ReadonlySet; }) { const { onDeleteThread, onUnarchiveThread } = props; const openSwipeableRef = useRef(null); @@ -369,6 +385,9 @@ export function ArchivedThreadsScreen(props: { environmentLabel={item.environmentLabel} isFirst={item.isFirst} isLast={item.isLast} + isUnarchiving={props.unarchivingThreadKeys.has( + scopedThreadKey(item.thread.environmentId, item.thread.id), + )} onDelete={() => onDeleteThread(item.thread)} onSwipeableClose={handleSwipeableClose} onSwipeableWillOpen={handleSwipeableWillOpen} @@ -384,6 +403,7 @@ export function ArchivedThreadsScreen(props: { handleSwipeableWillOpen, onDeleteThread, onUnarchiveThread, + props.unarchivingThreadKeys, ], ); const listEmptyComponent = useMemo(() => { diff --git a/apps/mobile/src/features/archive/archivedThreadList.test.ts b/apps/mobile/src/features/archive/archivedThreadList.test.ts index 474bd4481ca8..268f564501d5 100644 --- a/apps/mobile/src/features/archive/archivedThreadList.test.ts +++ b/apps/mobile/src/features/archive/archivedThreadList.test.ts @@ -3,7 +3,7 @@ import type { OrchestrationProjectShell, OrchestrationThreadShell } from "@t3too import { EnvironmentId, ProjectId, ProviderInstanceId, ThreadId } from "@t3tools/contracts"; import { describe, expect, it } from "vite-plus/test"; -import { buildArchivedThreadGroups } from "./archivedThreadList"; +import { buildArchivedThreadGroups, formatArchivedThreadRelativeTime } from "./archivedThreadList"; const environmentId = EnvironmentId.make("environment-1"); @@ -144,4 +144,78 @@ describe("buildArchivedThreadGroups", () => { expect(result).toEqual([]); }); + + it("sorts invalid archive timestamps after valid timestamps in either direction", () => { + const project = makeProject({ id: ProjectId.make("project-1"), title: "T3 Code" }); + const invalid = makeThread({ + archivedAt: "not-a-timestamp", + id: ThreadId.make("thread-invalid"), + projectId: project.id, + title: "Invalid", + }); + const anotherInvalid = makeThread({ + archivedAt: "2026-02-30T00:00:00.000Z", + id: ThreadId.make("thread-another-invalid"), + projectId: project.id, + title: "Another invalid", + }); + const older = makeThread({ + id: ThreadId.make("thread-older"), + projectId: project.id, + title: "Older", + }); + const newer = makeThread({ + archivedAt: "2026-06-03T00:00:00.000Z", + id: ThreadId.make("thread-newer"), + projectId: project.id, + title: "Newer", + }); + const snapshots = [makeSnapshot([project], [invalid, anotherInvalid, older, newer])]; + + const newest = buildArchivedThreadGroups({ + snapshots, + environmentLabels: {}, + environmentId: null, + searchQuery: "", + sortOrder: "newest", + }); + const oldest = buildArchivedThreadGroups({ + snapshots, + environmentLabels: {}, + environmentId: null, + searchQuery: "", + sortOrder: "oldest", + }); + + expect(newest[0]?.threads.map((thread) => thread.id)).toEqual([ + "thread-newer", + "thread-older", + "thread-another-invalid", + "thread-invalid", + ]); + expect(oldest[0]?.threads.map((thread) => thread.id)).toEqual([ + "thread-older", + "thread-newer", + "thread-another-invalid", + "thread-invalid", + ]); + }); +}); + +describe("formatArchivedThreadRelativeTime", () => { + it.each(["not-a-timestamp", "2026-02-30T00:00:00.000Z", "2025-02-29T12:00:00+02:00"])( + "omits invalid archive timestamp %s", + (timestamp) => { + expect(formatArchivedThreadRelativeTime(timestamp)).toBeNull(); + }, + ); + + it.each([ + "2024-02-29T12:00:00+02:00", + "2026-06-01T23:30:00-05:00", + "0099-01-01T00:00:00Z", + "+010000-01-01T00:00:00Z", + ])("keeps the label for valid archive timestamp %s", (timestamp) => { + expect(formatArchivedThreadRelativeTime(timestamp)).not.toBeNull(); + }); }); diff --git a/apps/mobile/src/features/archive/archivedThreadList.ts b/apps/mobile/src/features/archive/archivedThreadList.ts index 6146bba20447..11cb3e194bae 100644 --- a/apps/mobile/src/features/archive/archivedThreadList.ts +++ b/apps/mobile/src/features/archive/archivedThreadList.ts @@ -6,10 +6,12 @@ import { type EnvironmentThreadShell, } from "@t3tools/client-runtime/state/shell"; import type { EnvironmentId } from "@t3tools/contracts"; +import { parseTimestamp as parseZonedTimestamp } from "@t3tools/shared/dateTime"; import * as Arr from "effect/Array"; import * as Order from "effect/Order"; import { scopedProjectKey } from "../../lib/scopedEntities"; +import { relativeTime } from "../../lib/time"; export type ArchivedThreadSortOrder = "newest" | "oldest"; @@ -19,15 +21,33 @@ export interface ArchivedThreadGroup { readonly threads: ReadonlyArray; } -function archiveTimestamp(thread: EnvironmentThreadShell): number { - const timestamp = Date.parse(thread.archivedAt ?? thread.updatedAt); - return Number.isNaN(timestamp) ? 0 : timestamp; +function parseTimestamp(input: string): number | null { + const timestamp = parseZonedTimestamp(input); + return Number.isNaN(timestamp) ? null : timestamp; +} + +function archiveTimestamp(thread: EnvironmentThreadShell): number | null { + return parseTimestamp(thread.archivedAt ?? thread.updatedAt); +} + +export function formatArchivedThreadRelativeTime(input: string): string | null { + return parseTimestamp(input) === null ? null : relativeTime(input); } function matchesQuery(value: string | null, query: string): boolean { return value?.toLocaleLowerCase().includes(query) ?? false; } +function makeArchiveTimestampOrder(sortOrder: ArchivedThreadSortOrder): Order.Order { + const timestampOrder = sortOrder === "newest" ? Order.flip(Order.Number) : Order.Number; + return Order.make((left, right) => { + if (left === null && right === null) return 0; + if (left === null) return 1; + if (right === null) return -1; + return timestampOrder(left, right); + }); +} + export function buildArchivedThreadGroups(input: { readonly snapshots: ReadonlyArray; readonly environmentLabels: Readonly>; @@ -72,14 +92,18 @@ export function buildArchivedThreadGroups(input: { continue; } - const timestampOrder = input.sortOrder === "newest" ? Order.flip(Order.Number) : Order.Number; + const timestampOrder = makeArchiveTimestampOrder(input.sortOrder); groups.push({ key: scopedProjectKey(project.environmentId, project.id), project, threads: Arr.sort( matchingThreads, Order.mapInput( - Order.Struct({ timestamp: timestampOrder, title: Order.String, id: Order.String }), + Order.Struct({ + timestamp: timestampOrder, + title: Order.String, + id: Order.String, + }), (thread: EnvironmentThreadShell) => ({ timestamp: archiveTimestamp(thread), title: thread.title, @@ -91,13 +115,13 @@ export function buildArchivedThreadGroups(input: { } } - const timestampOrder = input.sortOrder === "newest" ? Order.flip(Order.Number) : Order.Number; + const timestampOrder = makeArchiveTimestampOrder(input.sortOrder); return Arr.sort( groups, Order.mapInput( Order.Struct({ timestamp: timestampOrder, title: Order.String, key: Order.String }), (group: ArchivedThreadGroup) => ({ - timestamp: group.threads[0] ? archiveTimestamp(group.threads[0]) : 0, + timestamp: group.threads[0] ? archiveTimestamp(group.threads[0]) : null, title: group.project.title, key: group.key, }), diff --git a/apps/mobile/src/features/home/useThreadListActions.ts b/apps/mobile/src/features/home/useThreadListActions.ts index 4a43facc6172..0511da571978 100644 --- a/apps/mobile/src/features/home/useThreadListActions.ts +++ b/apps/mobile/src/features/home/useThreadListActions.ts @@ -753,7 +753,7 @@ export function useThreadListActions(): { export function useArchivedThreadListActions( onCompleted: (thread: EnvironmentThreadShell) => void, ): { - readonly unarchiveThread: (thread: EnvironmentThreadShell) => void; + readonly unarchiveThread: (thread: EnvironmentThreadShell) => Promise; readonly confirmDeleteThread: (thread: EnvironmentThreadShell) => void; } { const handleCompleted = useCallback( @@ -764,8 +764,8 @@ export function useArchivedThreadListActions( ); const executeAction = useThreadActionExecutor(handleCompleted); const unarchiveThread = useCallback( - (thread: EnvironmentThreadShell) => { - void executeAction("unarchive", thread); + async (thread: EnvironmentThreadShell) => { + await executeAction("unarchive", thread); }, [executeAction], ); diff --git a/apps/mobile/src/lib/storage.test.ts b/apps/mobile/src/lib/storage.test.ts index 78a165c5ff1d..159025f71e9b 100644 --- a/apps/mobile/src/lib/storage.test.ts +++ b/apps/mobile/src/lib/storage.test.ts @@ -43,7 +43,7 @@ const mocks = vi.hoisted(() => { ), getFirstAsync: vi.fn((sql: string) => { if (sql.includes("PRAGMA user_version")) { - return Promise.resolve({ user_version: 1 }); + return Promise.resolve({ user_version: 2 }); } if (loadPreferencesFails) { return Promise.reject(new Error("database unavailable")); diff --git a/apps/mobile/src/persistence/mobile-database.test.ts b/apps/mobile/src/persistence/mobile-database.test.ts index 56c3e31657ec..a15a1c18c0f8 100644 --- a/apps/mobile/src/persistence/mobile-database.test.ts +++ b/apps/mobile/src/persistence/mobile-database.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; -import { vi } from "vite-plus/test"; +import { beforeEach, vi } from "vite-plus/test"; const openDatabaseAsync = vi.hoisted(() => vi.fn()); @@ -9,6 +9,10 @@ vi.mock("expo-sqlite", () => ({ openDatabaseAsync })); import { decodeLegacyCacheRecord, make } from "./mobile-database"; describe("mobile database legacy cache migration", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + it.effect("keeps acquisition failures typed on database operations", () => Effect.scoped( Effect.gen(function* () { @@ -68,4 +72,31 @@ describe("mobile database legacy cache migration", () => { ), ).toBeNull(); }); + + it.effect("clears persisted thread details when upgrading from schema version 1", () => + Effect.scoped( + Effect.gen(function* () { + const transactionExecAsync = vi.fn(() => Promise.resolve()); + const transactionRunAsync = vi.fn(() => Promise.resolve()); + const database = { + closeAsync: vi.fn(() => Promise.resolve()), + execAsync: vi.fn(() => Promise.resolve()), + getFirstAsync: vi.fn(() => Promise.resolve({ user_version: 1 })), + withExclusiveTransactionAsync: vi.fn((run: (transaction: unknown) => Promise) => + run({ execAsync: transactionExecAsync, runAsync: transactionRunAsync }), + ), + }; + openDatabaseAsync.mockResolvedValueOnce(database); + + yield* make; + + expect(transactionRunAsync).toHaveBeenCalledOnce(); + expect(transactionRunAsync).toHaveBeenCalledWith( + "DELETE FROM client_cache WHERE kind = ?", + "thread", + ); + expect(transactionExecAsync).toHaveBeenCalledWith("PRAGMA user_version = 2;"); + }), + ), + ); }); diff --git a/apps/mobile/src/persistence/mobile-database.ts b/apps/mobile/src/persistence/mobile-database.ts index 7c7b6bc5dedf..fcf0897fbc4a 100644 --- a/apps/mobile/src/persistence/mobile-database.ts +++ b/apps/mobile/src/persistence/mobile-database.ts @@ -7,7 +7,9 @@ import * as Schema from "effect/Schema"; import type { SQLiteDatabase } from "expo-sqlite"; const DATABASE_NAME = "t3code-client.db"; -const DATABASE_SCHEMA_VERSION = 1; +const LEGACY_FILE_CACHE_DATABASE_SCHEMA_VERSION = 1; +const ARCHIVED_THREAD_CACHE_EVICTION_DATABASE_SCHEMA_VERSION = 2; +const DATABASE_SCHEMA_VERSION = ARCHIVED_THREAD_CACHE_EVICTION_DATABASE_SCHEMA_VERSION; const LEGACY_CACHE_DIRECTORIES = [ "connection-shell-snapshots", "shell-snapshots", @@ -277,11 +279,18 @@ const makeAvailable = Effect.gen(function* () { ); `); }); - if ((schema?.user_version ?? 0) < DATABASE_SCHEMA_VERSION) { - const migrated = await migrateLegacyFileCaches(database); - if (migrated) { - await database.execAsync(`PRAGMA user_version = ${DATABASE_SCHEMA_VERSION};`); - } + const previousSchemaVersion = schema?.user_version ?? 0; + const legacyCachesMigrated = + previousSchemaVersion < LEGACY_FILE_CACHE_DATABASE_SCHEMA_VERSION + ? await migrateLegacyFileCaches(database) + : true; + if (legacyCachesMigrated && previousSchemaVersion < DATABASE_SCHEMA_VERSION) { + await database.withExclusiveTransactionAsync(async (transaction) => { + if (previousSchemaVersion < ARCHIVED_THREAD_CACHE_EVICTION_DATABASE_SCHEMA_VERSION) { + await transaction.runAsync("DELETE FROM client_cache WHERE kind = ?", "thread"); + } + await transaction.execAsync(`PRAGMA user_version = ${DATABASE_SCHEMA_VERSION};`); + }); } }, catch: databaseError("migrate"), diff --git a/apps/server/integration/OrchestrationEngineHarness.integration.ts b/apps/server/integration/OrchestrationEngineHarness.integration.ts index 5e3b4c6e4afe..863ac5411fd3 100644 --- a/apps/server/integration/OrchestrationEngineHarness.integration.ts +++ b/apps/server/integration/OrchestrationEngineHarness.integration.ts @@ -54,6 +54,7 @@ import { OrchestrationEngineLive } from "../src/orchestration/Layers/Orchestrati import { OrchestrationProjectionPipelineLive } from "../src/orchestration/Layers/ProjectionPipeline.ts"; import { OrchestrationProjectionSnapshotQueryLive } from "../src/orchestration/Layers/ProjectionSnapshotQuery.ts"; import * as ThreadBackgroundLiveness from "../src/orchestration/ThreadBackgroundLiveness.ts"; +import * as ThreadColdStorage from "../src/orchestration/ThreadColdStorage.ts"; import * as ThreadPlanProgress from "../src/orchestration/ThreadPlanProgress.ts"; import { RuntimeReceiptBusTest } from "../src/orchestration/Layers/RuntimeReceiptBus.ts"; import { OrchestrationReactorLive } from "../src/orchestration/Layers/OrchestrationReactor.ts"; @@ -269,6 +270,7 @@ export const makeOrchestrationIntegrationHarness = ( const persistenceLayer = makeSqlitePersistenceLive(dbPath); const orchestrationLayer = OrchestrationEngineLive.pipe( + Layer.provide(ThreadColdStorage.noOpLayer), Layer.provide(OrchestrationProjectionPipelineLive), Layer.provide(OrchestrationEventStoreLive), Layer.provide(OrchestrationCommandReceiptRepositoryLive), diff --git a/apps/server/integration/orphanedProviderSessionStartup.integration.test.ts b/apps/server/integration/orphanedProviderSessionStartup.integration.test.ts index b483a3f99f8e..691178dbb0df 100644 --- a/apps/server/integration/orphanedProviderSessionStartup.integration.test.ts +++ b/apps/server/integration/orphanedProviderSessionStartup.integration.test.ts @@ -32,6 +32,7 @@ import { OrchestrationLayerLive } from "../src/orchestration/runtimeLayer.ts"; import * as OrchestrationEngine from "../src/orchestration/Services/OrchestrationEngine.ts"; import * as OrchestrationReactor from "../src/orchestration/Services/OrchestrationReactor.ts"; import * as ProjectionSnapshotQuery from "../src/orchestration/Services/ProjectionSnapshotQuery.ts"; +import * as ThreadColdStorage from "../src/orchestration/ThreadColdStorage.ts"; import { makeSqlitePersistenceLive } from "../src/persistence/Layers/Sqlite.ts"; import * as ProviderSessionRuntime from "../src/persistence/ProviderSessionRuntime.ts"; import * as ExternalLauncher from "../src/process/externalLauncher.ts"; @@ -59,6 +60,7 @@ const stoppedBindingResumeCursor = { const makePersistedRuntimeLayer = (dbPath: string) => { const persistence = makeSqlitePersistenceLive(dbPath); const orchestration = OrchestrationLayerLive.pipe( + Layer.provide(ThreadColdStorage.noOpLayer), Layer.provideMerge(RepositoryIdentityResolver.layer), Layer.provideMerge(persistence), ); diff --git a/apps/server/scripts/t3-sqlite-state.test.ts b/apps/server/scripts/t3-sqlite-state.test.ts index fa5471bfa8c4..3ecda3ddc892 100644 --- a/apps/server/scripts/t3-sqlite-state.test.ts +++ b/apps/server/scripts/t3-sqlite-state.test.ts @@ -5,23 +5,26 @@ import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as ServerConfig from "../src/config.ts"; +import { runSqliteState, type SqliteStateDatabase } from "./t3-sqlite-state.ts"; import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; -import { runSqliteState } from "./t3-sqlite-state.ts"; import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import { symlinksSupported } from "@t3tools/shared/testing/symlinks"; const createFixtureDatabase = Effect.fn("createSqliteStateFixtureDatabase")(function* ( baseDir: string, + database: SqliteStateDatabase = "state", ) { const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const stateDir = path.join(baseDir, "userdata"); - const databasePath = path.join(stateDir, "state.sqlite"); - yield* fs.makeDirectory(stateDir, { recursive: true }); + const derivedPaths = yield* ServerConfig.deriveServerPaths(baseDir, undefined, { + baseDirIsExplicit: true, + }); + const databasePath = database === "archive" ? derivedPaths.archiveDbPath : derivedPaths.dbPath; + yield* fs.makeDirectory(derivedPaths.stateDir, { recursive: true }); yield* Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; yield* sql`CREATE TABLE fixtures (id INTEGER PRIMARY KEY, label TEXT NOT NULL)`; - yield* sql`INSERT INTO fixtures (id, label) VALUES (1, 'existing')`; + yield* sql`INSERT INTO fixtures (id, label) VALUES (1, ${database})`; }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: databasePath }))); }); @@ -70,7 +73,30 @@ it.layer(NodeServices.layer)("t3-sqlite-state", (it) => { assert.equal(result.operation, "query"); if (result.operation === "query") { - assert.deepStrictEqual(result.rows, [{ id: 1, label: "existing" }]); + assert.deepStrictEqual(result.rows, [{ id: 1, label: "state" }]); + } + }), + ); + + it.effect("selects the cold archive database through shared server paths", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-sqlite-archive-query-" }); + yield* createFixtureDatabase(baseDir); + yield* createFixtureDatabase(baseDir, "archive"); + + const result = yield* runSqliteState({ + operation: "query", + baseDir, + database: "archive", + sql: "SELECT id, label FROM fixtures", + }); + + assert.equal(result.database, path.join(baseDir, "userdata", "archive.sqlite")); + assert.equal(result.operation, "query"); + if (result.operation === "query") { + assert.deepStrictEqual(result.rows, [{ id: 1, label: "archive" }]); } }), ); diff --git a/apps/server/scripts/t3-sqlite-state.ts b/apps/server/scripts/t3-sqlite-state.ts index 5e274588671f..0489c5b10b5c 100644 --- a/apps/server/scripts/t3-sqlite-state.ts +++ b/apps/server/scripts/t3-sqlite-state.ts @@ -14,10 +14,13 @@ import * as Schema from "effect/Schema"; import * as SqlClient from "effect/unstable/sql/SqlClient"; import { Argument, Command, Flag } from "effect/unstable/cli"; +import * as ServerConfig from "../src/config.ts"; import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; export const SqliteStateOperation = Schema.Literals(["query", "exec"]); export type SqliteStateOperation = typeof SqliteStateOperation.Type; +export const SqliteStateDatabase = Schema.Literals(["state", "archive"]); +export type SqliteStateDatabase = typeof SqliteStateDatabase.Type; export class SqliteStateMultipleSqlSourcesError extends Schema.TaggedError()( "SqliteStateMultipleSqlSourcesError", @@ -119,6 +122,7 @@ type RawSqliteRow = Readonly>; export interface RunSqliteStateInput { readonly operation: SqliteStateOperation; readonly baseDir: string; + readonly database?: SqliteStateDatabase | undefined; readonly sql?: string | undefined; readonly file?: string | undefined; } @@ -182,7 +186,11 @@ export const runSqliteState = Effect.fn("runSqliteState")(function* ( const path = yield* Path.Path; const baseDir = path.resolve(input.baseDir); const sharedHome = path.resolve(options.sharedHome ?? path.join(NodeOS.homedir(), ".t3")); - const databasePath = path.join(baseDir, "userdata", "state.sqlite"); + const derivedPaths = yield* ServerConfig.deriveServerPaths(baseDir, undefined, { + baseDirIsExplicit: true, + }); + const databasePath = + input.database === "archive" ? derivedPaths.archiveDbPath : derivedPaths.dbPath; const source = yield* resolveSqlSource(input.sql, input.file); if (!(yield* fs.exists(databasePath))) { @@ -252,7 +260,11 @@ const t3SqliteStateCommand = Command.make( Argument.withDescription("Run a read-only query or a backed-up fixture mutation."), ), baseDir: Flag.String("base-dir").pipe( - Flag.withDescription("Explicit T3 base directory containing userdata/state.sqlite."), + Flag.withDescription("Explicit T3 base directory containing runtime SQLite databases."), + ), + database: Flag.Literals("database", SqliteStateDatabase.literals).pipe( + Flag.withDescription("Runtime database to inspect or seed (default: state)."), + Flag.withDefault("state"), ), sql: Flag.String("sql").pipe( Flag.optional, @@ -263,10 +275,11 @@ const t3SqliteStateCommand = Command.make( Flag.withDescription("Path to a SQL source file."), ), }, - ({ operation, baseDir, sql, file }) => + ({ operation, baseDir, database, sql, file }) => runSqliteState({ operation, baseDir, + database, sql: Option.getOrUndefined(sql), file: Option.getOrUndefined(file), }).pipe(Effect.flatMap(encodeSqliteStateResult), Effect.flatMap(Console.log)), diff --git a/apps/server/src/bin.test.ts b/apps/server/src/bin.test.ts index aecf82eeac67..04c6d822e968 100644 --- a/apps/server/src/bin.test.ts +++ b/apps/server/src/bin.test.ts @@ -39,6 +39,7 @@ import * as ServerConfig from "./config.ts"; import * as ServerEnvironment from "./environment/ServerEnvironment.ts"; import * as ProjectionSnapshotQuery from "./orchestration/Services/ProjectionSnapshotQuery.ts"; import * as OrchestrationEngine from "./orchestration/Services/OrchestrationEngine.ts"; +import * as ThreadColdStorage from "./orchestration/ThreadColdStorage.ts"; import { OrchestrationLayerLive } from "./orchestration/runtimeLayer.ts"; import { orchestrationHttpApiLayer } from "./orchestration/http.ts"; import * as ProjectCloneTracker from "./project/ProjectCloneTracker.ts"; @@ -129,6 +130,7 @@ const makeCliTestServerConfig = (baseDir: string) => const makeProjectPersistenceLayer = (config: ServerConfig.ServerConfig["Service"]) => Layer.mergeAll( OrchestrationLayerLive.pipe( + Layer.provide(ThreadColdStorage.noOpLayer), Layer.provideMerge(RepositoryIdentityResolver.layer), Layer.provideMerge(SqlitePersistenceLayerLive), ), diff --git a/apps/server/src/cli/project.ts b/apps/server/src/cli/project.ts index cef4dc879073..b5cb09fd27e8 100644 --- a/apps/server/src/cli/project.ts +++ b/apps/server/src/cli/project.ts @@ -27,6 +27,7 @@ import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; import * as ServerConfig from "../config.ts"; import * as OrchestrationEngine from "../orchestration/Services/OrchestrationEngine.ts"; import * as ProjectionSnapshotQuery from "../orchestration/Services/ProjectionSnapshotQuery.ts"; +import * as ThreadColdStorage from "../orchestration/ThreadColdStorage.ts"; import { OrchestrationLayerLive } from "../orchestration/runtimeLayer.ts"; import { layerConfig as SqlitePersistenceLayerLive } from "../persistence/Layers/Sqlite.ts"; import * as RepositoryIdentityResolver from "../project/RepositoryIdentityResolver.ts"; @@ -200,6 +201,7 @@ const projectCommandUuid = Crypto.Crypto.pipe( const ProjectCliRuntimeLive = Layer.mergeAll( WorkspacePaths.layer, OrchestrationLayerLive.pipe( + Layer.provide(ThreadColdStorage.noOpLayer), Layer.provideMerge(RepositoryIdentityResolver.layer), Layer.provideMerge(SqlitePersistenceLayerLive), ), diff --git a/apps/server/src/config.ts b/apps/server/src/config.ts index d8dad5ae4d24..b70b0488615b 100644 --- a/apps/server/src/config.ts +++ b/apps/server/src/config.ts @@ -34,6 +34,7 @@ export type StartupPresentation = typeof StartupPresentation.Type; export interface ServerDerivedPaths { readonly stateDir: string; readonly dbPath: string; + readonly archiveDbPath: string; readonly keybindingsConfigPath: string; readonly settingsPath: string; /** Palettes this machine publishes for clients to follow, one file per theme. */ @@ -138,6 +139,7 @@ export const deriveServerPaths = Effect.fn(function* ( devUrl !== undefined && !options.baseDirIsExplicit ? "dev" : "userdata", ); const dbPath = join(stateDir, "state.sqlite"); + const archiveDbPath = join(stateDir, "archive.sqlite"); const attachmentsDir = join(stateDir, "attachments"); const logsDir = join(stateDir, "logs"); const providerLogsDir = join(logsDir, "provider"); @@ -145,6 +147,7 @@ export const deriveServerPaths = Effect.fn(function* ( return { stateDir, dbPath, + archiveDbPath, keybindingsConfigPath: join(stateDir, "keybindings.json"), settingsPath: join(stateDir, "settings.json"), environmentThemesDir: join(stateDir, "themes"), diff --git a/apps/server/src/orchestration/Layers/CheckpointReactor.test.ts b/apps/server/src/orchestration/Layers/CheckpointReactor.test.ts index 1d0b3da1bdbc..126f61765561 100644 --- a/apps/server/src/orchestration/Layers/CheckpointReactor.test.ts +++ b/apps/server/src/orchestration/Layers/CheckpointReactor.test.ts @@ -46,6 +46,7 @@ import { OrchestrationEngineLive } from "./OrchestrationEngine.ts"; import { OrchestrationProjectionPipelineLive } from "./ProjectionPipeline.ts"; import { OrchestrationProjectionSnapshotQueryLive } from "./ProjectionSnapshotQuery.ts"; import * as ThreadBackgroundLiveness from "../ThreadBackgroundLiveness.ts"; +import * as ThreadColdStorage from "../ThreadColdStorage.ts"; import * as ThreadPlanProgress from "../ThreadPlanProgress.ts"; import { RuntimeReceiptBusTest } from "./RuntimeReceiptBus.ts"; import * as RuntimeReceiptBus from "../Services/RuntimeReceiptBus.ts"; @@ -323,6 +324,7 @@ describe("CheckpointReactor", () => { options?.providerName ?? ProviderDriverKind.make("codex"), ); const orchestrationLayer = OrchestrationEngineLive.pipe( + Layer.provide(ThreadColdStorage.noOpLayer), Layer.provide(OrchestrationProjectionSnapshotQueryLive), Layer.provide(ThreadBackgroundLiveness.layer), Layer.provide(ThreadPlanProgress.layer), diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts index 9ccf78ca1744..a2e9250dbaed 100644 --- a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts +++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts @@ -20,12 +20,16 @@ import { import * as NodeServices from "@effect/platform-node/NodeServices"; import { it as effectIt } from "@effect/vitest"; import * as Effect from "effect/Effect"; +import * as Deferred from "effect/Deferred"; +import * as Fiber from "effect/Fiber"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as ManagedRuntime from "effect/ManagedRuntime"; import * as Metric from "effect/Metric"; import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; import { TestClock } from "effect/testing"; import { describe, expect, it, vi } from "vite-plus/test"; @@ -54,15 +58,21 @@ import { } from "../Services/ProjectionPipeline.ts"; import { ProjectionSnapshotQuery } from "../Services/ProjectionSnapshotQuery.ts"; import { ServerConfig } from "../../config.ts"; +import * as ThreadColdStorage from "../ThreadColdStorage.ts"; const asProjectId = (value: string): ProjectId => ProjectId.make(value); const asMessageId = (value: string): MessageId => MessageId.make(value); const asTurnId = (value: string): TurnId => TurnId.make(value); const asCheckpointRef = (value: string): CheckpointRef => CheckpointRef.make(value); +const OrchestrationEngineNoColdStorageLive = OrchestrationEngineLive.pipe( + Layer.provide(ThreadColdStorage.noOpLayer), +); + function makeOrchestrationLayer( databasePath?: string, repositoryIdentityResolver?: RepositoryIdentityResolver.RepositoryIdentityResolver["Service"], + useColdStorage = false, ) { const persistence = databasePath ? makeSqlitePersistenceLive(databasePath) @@ -72,6 +82,7 @@ function makeOrchestrationLayer( }); return Layer.mergeAll( OrchestrationEngineLive.pipe( + Layer.provideMerge(useColdStorage ? ThreadColdStorage.layer : ThreadColdStorage.noOpLayer), Layer.provide(OrchestrationProjectionSnapshotQueryLive), Layer.provide(OrchestrationProjectionPipelineLive), ), @@ -89,7 +100,7 @@ function makeOrchestrationLayer( ) : RepositoryIdentityResolver.layer, ), - Layer.provide(persistence), + Layer.provideMerge(persistence), Layer.provideMerge(ServerConfigLayer), Layer.provideMerge(NodeServices.layer), ); @@ -130,6 +141,203 @@ const hasMetricSnapshot = ( ); describe("OrchestrationEngine", () => { + effectIt.effect("leaves deleted archives cold when unarchive precedes permanent cleanup", () => + Effect.gen(function* () { + const engine = yield* OrchestrationEngineService; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const sql = yield* SqlClient.SqlClient; + const fs = yield* FileSystem.FileSystem; + const config = yield* ServerConfig; + const threadId = ThreadId.make("deleted-archive-restore"); + const projectId = ProjectId.make("deleted-archive-project"); + const modelSelection = { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5-codex", + }; + yield* engine.dispatch({ + type: "project.create", + commandId: CommandId.make("deleted-archive-project"), + projectId, + title: "Archive deletion", + workspaceRoot: "/tmp/deleted-archive-project", + defaultModelSelection: modelSelection, + createdAt: now(), + }); + yield* engine.dispatch({ + type: "thread.create", + commandId: CommandId.make("deleted-archive-create"), + threadId, + projectId, + title: "Deleted archive", + modelSelection, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "full-access", + branch: null, + worktreePath: null, + createdAt: now(), + }); + const attachmentId = `${threadId}-00000000-0000-4000-8000-000000000001`; + const attachmentPath = NodePath.join(config.attachmentsDir, `${attachmentId}.png`); + yield* fs.makeDirectory(config.attachmentsDir, { recursive: true }); + yield* fs.writeFileString(attachmentPath, "deleted attachment"); + yield* engine.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make("deleted-archive-message"), + threadId, + message: { + messageId: MessageId.make("deleted-archive-message"), + role: "user", + text: "Deleted history", + attachments: [ + { + type: "image", + id: attachmentId, + name: "image.png", + mimeType: "image/png", + sizeBytes: 18, + }, + ], + }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "full-access", + createdAt: now(), + }); + yield* engine.dispatch({ + type: "thread.archive", + commandId: CommandId.make("deleted-archive-archive"), + threadId, + }); + yield* storage.archiveThread(threadId); + expect(yield* fs.exists(attachmentPath)).toBe(false); + yield* engine.dispatch({ + type: "thread.delete", + commandId: CommandId.make("deleted-archive-delete"), + threadId, + }); + + // Drive cleanup explicitly so unarchive reaches the real engine while deletion is pending. + expect(yield* storage.listPendingDeleteThreadIds).toContain(threadId); + yield* engine.dispatch({ + type: "thread.unarchive", + commandId: CommandId.make("deleted-archive-unarchive"), + threadId, + }); + expect(yield* fs.exists(attachmentPath)).toBe(false); + expect( + yield* sql`SELECT message_id FROM projection_thread_messages WHERE thread_id = ${threadId}`, + ).toEqual([]); + expect( + yield* sql`SELECT status FROM thread_archive_manifests WHERE thread_id = ${threadId}`, + ).toEqual([{ status: "cold" }]); + expect(yield* storage.listPendingDeleteThreadIds).toContain(threadId); + yield* storage.deleteThread(threadId); + expect( + yield* sql`SELECT thread_id FROM cold_archive.archive_threads WHERE thread_id = ${threadId}`, + ).toEqual([]); + expect(yield* storage.listPendingDeleteThreadIds).not.toContain(threadId); + }).pipe(Effect.provide(makeOrchestrationLayer(undefined, undefined, true))), + ); + + effectIt.effect( + "keeps recreation unpublished and retryable until deletion cleanup completes", + () => + Effect.gen(function* () { + const engine = yield* OrchestrationEngineService; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const sql = yield* SqlClient.SqlClient; + const threadId = ThreadId.make("cleanup-recreation"); + const projectId = ProjectId.make("cleanup-recreation-project"); + const create = { + type: "thread.create", + commandId: CommandId.make("cleanup-first-create"), + threadId, + projectId, + title: "Recreated thread", + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5-codex" }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "full-access", + branch: null, + worktreePath: null, + createdAt: now(), + } as const; + yield* engine.dispatch({ + type: "project.create", + commandId: CommandId.make("cleanup-project-create"), + projectId, + title: "Cleanup project", + workspaceRoot: "/tmp/cleanup-project", + defaultModelSelection: create.modelSelection, + createdAt: now(), + }); + yield* engine.dispatch(create); + yield* engine.dispatch({ + type: "thread.delete", + commandId: CommandId.make("cleanup-delete"), + threadId, + }); + const recreate = { ...create, commandId: CommandId.make("cleanup-recreate") }; + const sequenceBefore = yield* engine.latestSequence; + const assertRetryable = Effect.gen(function* () { + const error = yield* Effect.flip(engine.dispatch(recreate)); + expect(error.message).toContain("still being cleaned up; retry shortly"); + expect(yield* engine.latestSequence).toBe(sequenceBefore); + expect( + yield* sql`SELECT 1 FROM orchestration_command_receipts WHERE command_id = ${recreate.commandId}`, + ).toEqual([]); + }); + + // The soft-deleted shell protects the interval before a worker inserts its marker. + yield* assertRetryable; + const quiescing = yield* Deferred.make(); + const finishQuiescing = yield* Deferred.make(); + const replacementObserved = yield* Deferred.make(); + let replacementResources = false; + let quiesceCalls = 0; + const subscription = yield* engine.subscribeDomainEvents; + yield* subscription.pipe( + Stream.runForEach((event) => + event.type === "thread.created" && event.commandId === recreate.commandId + ? Effect.sync(() => { + replacementResources = true; + }).pipe(Effect.andThen(Deferred.succeed(replacementObserved, undefined))) + : Effect.void, + ), + Effect.forkChild, + ); + const quiesce = Effect.gen(function* () { + quiesceCalls += 1; + yield* Deferred.succeed(quiescing, undefined); + yield* Deferred.await(finishQuiescing); + replacementResources = false; + }); + const deleting = yield* Effect.forkChild(storage.deleteThread(threadId, quiesce)); + yield* Deferred.await(quiescing); + yield* assertRetryable; + expect(replacementResources).toBe(false); + yield* Deferred.succeed(finishQuiescing, undefined); + yield* Fiber.join(deleting); + + // A failed cleanup after shell removal still blocks reuse through its durable marker. + yield* sql`INSERT INTO thread_cleanup_queue (thread_id, reason, created_at) VALUES (${threadId}, 'deleted', CURRENT_TIMESTAMP)`; + const failure = yield* Effect.flip( + storage.deleteThread(threadId, Effect.fail("writer close failed")), + ); + expect(failure.operation).toBe("delete"); + yield* assertRetryable; + yield* storage.deleteThread(threadId); + yield* engine.dispatch(recreate); + yield* Deferred.await(replacementObserved); + expect(replacementResources).toBe(true); + // Another client reacted to publication; an obsolete job must not stop its resources. + yield* storage.deleteThread(threadId, quiesce); + expect(replacementResources).toBe(true); + expect(quiesceCalls).toBe(1); + expect( + yield* sql`SELECT deleted_at FROM projection_threads WHERE thread_id = ${threadId}`, + ).toEqual([{ deleted_at: null }]); + }).pipe(Effect.provide(makeOrchestrationLayer(undefined, undefined, true))), + ); + it.each(["running", "stopped"] as const)( "sends async answers with a %s session and rejects old duplicate replies", async (status) => { @@ -416,7 +624,7 @@ describe("OrchestrationEngine", () => { }; let fullSnapshotReadCount = 0; - const layer = OrchestrationEngineLive.pipe( + const layer = OrchestrationEngineNoColdStorageLive.pipe( Layer.provide( Layer.succeed(ProjectionSnapshotQuery, { getUserInputActivity: () => Effect.die("unused"), @@ -1511,7 +1719,7 @@ describe("OrchestrationEngine", () => { }); const runtime = ManagedRuntime.make( - OrchestrationEngineLive.pipe( + OrchestrationEngineNoColdStorageLive.pipe( Layer.provide(OrchestrationProjectionSnapshotQueryLive), Layer.provide(ThreadBackgroundLiveness.layer), Layer.provide(ThreadPlanProgress.layer), @@ -1595,6 +1803,237 @@ describe("OrchestrationEngine", () => { await runtime.dispose(); }); + effectIt.effect( + "records invalid unarchive rejections while keeping restore failures retryable", + () => { + type StoredEvent = + ReturnType extends Effect.Effect + ? A + : never; + const events: StoredEvent[] = []; + const rolledBackThreadIds: ThreadId[] = []; + const finishedThreadIds: ThreadId[] = []; + const restoreThreadIds: ThreadId[] = []; + let nextSequence = 1; + let restoreOnUnarchive = false; + let failNextRestore = false; + let failNextFinish = false; + let partialRestoreActive = false; + + const flakyStore: OrchestrationEventStoreShape = { + append(event) { + if (event.commandId === CommandId.make("cmd-cold-unarchive-fail")) { + return Effect.fail( + new PersistenceSqlError({ + operation: "test.append", + detail: "unarchive append failed", + }), + ); + } + const savedEvent = { ...event, sequence: nextSequence } as StoredEvent; + nextSequence += 1; + events.push(savedEvent); + return Effect.succeed(savedEvent); + }, + readFromSequence(sequenceExclusive) { + return Stream.fromIterable(events.filter((event) => event.sequence > sequenceExclusive)); + }, + readAll() { + return Stream.fromIterable(events); + }, + hasEventAfter: () => Effect.succeed(false), + readAggregateRange: () => Stream.die("unused aggregate replay"), + getAggregateReplayStats: () => Effect.die("unused aggregate replay stats"), + }; + const coldStorage: ThreadColdStorage.ThreadColdStorage["Service"] = { + archiveThread: () => Effect.void, + restoreTree: (threadId) => { + restoreThreadIds.push(threadId); + if (failNextRestore) { + failNextRestore = false; + partialRestoreActive = true; + return Effect.fail( + new ThreadColdStorage.ThreadColdStorageError({ + operation: "restore", + threadId, + cause: new Error("temporary restore failure"), + }), + ); + } + return Effect.succeed(restoreOnUnarchive); + }, + rollbackRestoreTree: (threadId) => + Effect.sync(() => { + partialRestoreActive = false; + rolledBackThreadIds.push(threadId); + }), + finishRestoreTree: (threadId) => + Effect.gen(function* () { + yield* Effect.sync(() => { + finishedThreadIds.push(threadId); + }); + if (failNextFinish) { + failNextFinish = false; + return yield* Effect.die("finish restore failed"); + } + }), + deleteThread: () => Effect.void, + removeProviderLogs: () => Effect.void, + compactLegacyStorage: Effect.void, + listPendingArchiveThreadIds: Effect.succeed([]), + listPendingDeleteThreadIds: Effect.succeed([]), + }; + const ServerConfigLayer = ServerConfig.layerTest(process.cwd(), { + prefix: "t3-orchestration-engine-test-", + }); + const testLayer = OrchestrationEngineLive.pipe( + Layer.provide(OrchestrationProjectionSnapshotQueryLive), + Layer.provide(OrchestrationProjectionPipelineLive), + Layer.provide(ThreadBackgroundLiveness.layer), + Layer.provide(ThreadPlanProgress.layer), + Layer.provide(Layer.succeed(OrchestrationEventStore, flakyStore)), + Layer.provide(Layer.succeed(ThreadColdStorage.ThreadColdStorage, coldStorage)), + Layer.provide(OrchestrationCommandReceiptRepositoryLive), + Layer.provide(RepositoryIdentityResolver.layer), + Layer.provide(SqlitePersistenceMemory), + Layer.provideMerge(ServerConfigLayer), + Layer.provideMerge(NodeServices.layer), + ); + return Effect.gen(function* () { + const engine = yield* OrchestrationEngineService; + const createdAt = now(); + + yield* engine.dispatch({ + type: "project.create", + commandId: CommandId.make("cmd-cold-project-create"), + projectId: asProjectId("project-cold-rollback"), + title: "Cold rollback project", + workspaceRoot: "/tmp/project-cold-rollback", + defaultModelSelection: { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5-codex", + }, + createdAt, + }); + yield* engine.dispatch({ + type: "thread.create", + commandId: CommandId.make("cmd-cold-thread-create"), + threadId: ThreadId.make("thread-cold-rollback"), + projectId: asProjectId("project-cold-rollback"), + title: "Cold rollback thread", + modelSelection: { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5-codex", + }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "full-access", + branch: null, + worktreePath: null, + createdAt, + }); + for (const [threadId, expectedMessage] of [ + ["thread-cold-rollback", "is not archived"], + ["thread-cold-absent", "does not exist"], + ] as const) { + const command = { + type: "thread.unarchive", + commandId: CommandId.make(`cmd-invalid-unarchive-${threadId}`), + threadId: ThreadId.make(threadId), + } as const; + const rejection = yield* Effect.flip(engine.dispatch(command)); + expect(rejection.message).toContain(expectedMessage); + const retry = yield* Effect.flip(engine.dispatch(command)); + expect(retry._tag).toBe("OrchestrationCommandPreviouslyRejectedError"); + } + expect(restoreThreadIds).toEqual([]); + expect(rolledBackThreadIds).toEqual([]); + expect(finishedThreadIds).toEqual([]); + + yield* engine.dispatch({ + type: "thread.archive", + commandId: CommandId.make("cmd-cold-thread-archive"), + threadId: ThreadId.make("thread-cold-rollback"), + }); + const rejectedBeforeArchive = yield* Effect.flip( + engine.dispatch({ + type: "thread.unarchive", + commandId: CommandId.make("cmd-invalid-unarchive-thread-cold-rollback"), + threadId: ThreadId.make("thread-cold-rollback"), + }), + ); + expect(rejectedBeforeArchive._tag).toBe("OrchestrationCommandPreviouslyRejectedError"); + expect(restoreThreadIds).toEqual([]); + + const missingArchiveFailure = yield* Effect.flip( + engine.dispatch({ + type: "thread.unarchive", + commandId: CommandId.make("cmd-cold-unarchive-missing"), + threadId: ThreadId.make("thread-cold-rollback"), + }), + ); + expect(missingArchiveFailure.message).toContain( + "Failed to restore the archived conversation", + ); + expect(events.at(-1)?.type).toBe("thread.archived"); + expect(rolledBackThreadIds).toEqual([ThreadId.make("thread-cold-rollback")]); + rolledBackThreadIds.length = 0; + + restoreOnUnarchive = true; + failNextRestore = true; + const retryableRestoreCommand = { + type: "thread.unarchive", + commandId: CommandId.make("cmd-cold-unarchive-retryable-restore"), + threadId: ThreadId.make("thread-cold-rollback"), + } as const; + const retryableRestoreFailure = yield* Effect.flip( + engine.dispatch(retryableRestoreCommand), + ); + expect(retryableRestoreFailure.message).toContain( + "Failed to restore the archived conversation", + ); + expect(partialRestoreActive).toBe(false); + expect(rolledBackThreadIds).toEqual([ThreadId.make("thread-cold-rollback")]); + rolledBackThreadIds.length = 0; + const retriedRestoreResult = yield* engine.dispatch(retryableRestoreCommand); + expect(retriedRestoreResult.sequence).toBeGreaterThan(0); + expect(events.at(-1)?.type).toBe("thread.unarchived"); + yield* engine.dispatch({ + type: "thread.archive", + commandId: CommandId.make("cmd-cold-thread-rearchive-after-restore-retry"), + threadId: ThreadId.make("thread-cold-rollback"), + }); + + const unarchiveFailure = yield* Effect.flip( + engine.dispatch({ + type: "thread.unarchive", + commandId: CommandId.make("cmd-cold-unarchive-fail"), + threadId: ThreadId.make("thread-cold-rollback"), + }), + ); + expect(unarchiveFailure.message).toContain("unarchive append failed"); + expect(rolledBackThreadIds).toEqual([ThreadId.make("thread-cold-rollback")]); + expect(finishedThreadIds).toEqual([ThreadId.make("thread-cold-rollback")]); + + failNextFinish = true; + const acceptedCommand = { + type: "thread.unarchive", + commandId: CommandId.make("cmd-cold-unarchive-accepted"), + threadId: ThreadId.make("thread-cold-rollback"), + } as const; + const acceptedResult = yield* engine.dispatch(acceptedCommand); + const retriedResult = yield* engine.dispatch(acceptedCommand); + + expect(retriedResult).toEqual(acceptedResult); + expect(finishedThreadIds).toEqual([ + ThreadId.make("thread-cold-rollback"), + ThreadId.make("thread-cold-rollback"), + ThreadId.make("thread-cold-rollback"), + ]); + expect(events.filter((event) => event.type === "thread.unarchived")).toHaveLength(2); + }).pipe(Effect.provide(testLayer)); + }, + ); + it("rolls back all events for a multi-event command when projection fails mid-dispatch", async () => { let shouldFailRequestedProjection = true; const flakyProjectionPipeline: OrchestrationProjectionPipelineShape = { @@ -1619,7 +2058,7 @@ describe("OrchestrationEngine", () => { }; const runtime = ManagedRuntime.make( - OrchestrationEngineLive.pipe( + OrchestrationEngineNoColdStorageLive.pipe( Layer.provide(OrchestrationProjectionSnapshotQueryLive), Layer.provide(ThreadBackgroundLiveness.layer), Layer.provide(ThreadPlanProgress.layer), @@ -1768,7 +2207,7 @@ describe("OrchestrationEngine", () => { }; const runtime = ManagedRuntime.make( - OrchestrationEngineLive.pipe( + OrchestrationEngineNoColdStorageLive.pipe( Layer.provide(OrchestrationProjectionSnapshotQueryLive), Layer.provide(ThreadBackgroundLiveness.layer), Layer.provide(ThreadPlanProgress.layer), diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.ts index 9136d080c1c3..0f2055ae1b22 100644 --- a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts +++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.ts @@ -49,6 +49,7 @@ import { OrchestrationEngineService, type OrchestrationEngineShape, } from "../Services/OrchestrationEngine.ts"; +import * as ThreadColdStorage from "../ThreadColdStorage.ts"; const isOrchestrationCommandPreviouslyRejectedError = Schema.is( OrchestrationCommandPreviouslyRejectedError, ); @@ -87,6 +88,7 @@ const makeOrchestrationEngine = Effect.gen(function* () { const commandReceiptRepository = yield* OrchestrationCommandReceiptRepository; const projectionPipeline = yield* OrchestrationProjectionPipeline; const projectionSnapshotQuery = yield* ProjectionSnapshotQuery; + const threadColdStorage = yield* ThreadColdStorage.ThreadColdStorage; const threadBackgroundLiveness = yield* ThreadBackgroundLivenessService; const crypto = yield* Crypto.Crypto; @@ -96,6 +98,16 @@ const makeOrchestrationEngine = Effect.gen(function* () { const commandQueue = yield* Queue.unbounded(); const eventPubSub = yield* PubSub.unbounded(); + const finishRestoredUnarchiveTree = (threadId: ThreadId) => + threadColdStorage.finishRestoreTree(threadId).pipe( + Effect.catchCause((cause) => + Effect.logWarning("failed to finalize restored archive bundle", { + threadId, + cause: Cause.pretty(cause), + }), + ), + ); + const projectEventsOntoReadModel = ( baseReadModel: OrchestrationReadModel, events: ReadonlyArray, @@ -111,6 +123,10 @@ const makeOrchestrationEngine = Effect.gen(function* () { const processEnvelope = (envelope: CommandEnvelope): Effect.Effect => { const dispatchStartSequence = commandReadModel.snapshotSequence; let processingStartedAtMs = 0; + let restoredUnarchiveThreadId: ThreadId | null = null; + let restoredUnarchiveCommitted = false; + let unarchiveRestoreFailed = false; + let creationCleanupPending = false; const aggregateRef = commandToAggregateRef(envelope.command); const baseMetricAttributes = { commandType: envelope.command.type, @@ -141,6 +157,8 @@ const makeOrchestrationEngine = Effect.gen(function* () { "orchestration.aggregate_id": aggregateRef.aggregateId, }); + const unarchiveThreadId = + envelope.command.type === "thread.unarchive" ? envelope.command.threadId : null; const existingReceipt = yield* commandReceiptRepository.getByCommandId({ commandId: envelope.command.commandId, }); @@ -161,6 +179,13 @@ const makeOrchestrationEngine = Effect.gen(function* () { }); } if (existingReceipt.value.status === "accepted") { + if ( + unarchiveThreadId !== null && + existingReceipt.value.aggregateKind === "thread" && + existingReceipt.value.aggregateId === unarchiveThreadId + ) { + yield* finishRestoredUnarchiveTree(unarchiveThreadId); + } return { sequence: existingReceipt.value.resultSequence, }; @@ -171,6 +196,37 @@ const makeOrchestrationEngine = Effect.gen(function* () { }); } + const unarchiveTarget = + unarchiveThreadId === null + ? undefined + : commandReadModel.threads.find((thread) => thread.id === unarchiveThreadId); + if ( + unarchiveTarget !== undefined && + unarchiveTarget.archivedAt !== null && + unarchiveTarget.deletedAt === null + ) { + restoredUnarchiveThreadId = unarchiveTarget.id; + const restored = yield* threadColdStorage.restoreTree(unarchiveTarget.id).pipe( + Effect.mapError((cause) => { + unarchiveRestoreFailed = true; + return new OrchestrationCommandInvariantError({ + commandType: envelope.command.type, + detail: "Failed to restore the archived conversation.", + cause, + }); + }), + ); + if (restored) { + commandReadModel = yield* projectionSnapshotQuery.getCommandReadModel(); + } else { + unarchiveRestoreFailed = true; + return yield* new OrchestrationCommandInvariantError({ + commandType: envelope.command.type, + detail: "Failed to restore the archived conversation.", + }); + } + } + if ( envelope.command.type === "thread.auto-settle" && (yield* eventStore.hasEventAfter({ @@ -270,9 +326,26 @@ const makeOrchestrationEngine = Effect.gen(function* () { ...planned, metadata: { ...planned.metadata, origin: envelope.origin }, })); - const committedCommand = yield* sql + const commitCommand = sql .withTransaction( Effect.gen(function* () { + if (envelope.command.type === "thread.create") { + const pendingDeletion = yield* sql` + SELECT 1 FROM projection_threads + WHERE thread_id = ${envelope.command.threadId} AND deleted_at IS NOT NULL + UNION ALL + SELECT 1 FROM thread_cleanup_queue + WHERE thread_id = ${envelope.command.threadId} + LIMIT 1 + `; + if (pendingDeletion.length > 0) { + creationCleanupPending = true; + return yield* new OrchestrationCommandInvariantError({ + commandType: envelope.command.type, + detail: "Thread id is still being cleaned up; retry shortly.", + }); + } + } const committedEvents: OrchestrationEvent[] = []; const attachmentCleanups: Effect.Effect[] = []; let nextCommandReadModel = commandReadModel; @@ -319,7 +392,20 @@ const makeOrchestrationEngine = Effect.gen(function* () { ), ); + const committedCommand = yield* Effect.uninterruptibleMask(() => + Effect.gen(function* () { + // The transaction helper restores interruptibility after COMMIT, so + // keep the entire command masked until ownership has transferred. + const result = yield* commitCommand; + restoredUnarchiveCommitted = restoredUnarchiveThreadId !== null; + return result; + }), + ); + commandReadModel = committedCommand.nextCommandReadModel; + if (restoredUnarchiveThreadId !== null) { + yield* finishRestoredUnarchiveTree(restoredUnarchiveThreadId); + } for (const cleanup of committedCommand.attachmentCleanups) { yield* cleanup; } @@ -371,6 +457,36 @@ const makeOrchestrationEngine = Effect.gen(function* () { return; } + if (restoredUnarchiveThreadId !== null && !restoredUnarchiveCommitted) { + const rolledBack = yield* threadColdStorage + .rollbackRestoreTree(restoredUnarchiveThreadId) + .pipe( + Effect.as(true), + Effect.catchCause((cause) => + Effect.logWarning("failed to roll back restored archive bundle", { + threadId: restoredUnarchiveThreadId, + cause: Cause.pretty(cause), + }).pipe(Effect.as(false)), + ), + ); + if (rolledBack) { + const refreshedReadModel = yield* projectionSnapshotQuery.getCommandReadModel().pipe( + Effect.catchCause((cause) => + Effect.logWarning( + "failed to refresh orchestration read model after archive rollback", + { + threadId: restoredUnarchiveThreadId, + cause: Cause.pretty(cause), + }, + ).pipe(Effect.as(null)), + ), + ); + if (refreshedReadModel !== null) { + commandReadModel = refreshedReadModel; + } + } + } + const error = Cause.squash(exit.cause) as OrchestrationDispatchError; if ( !isOrchestrationCommandPreviouslyRejectedError(error) && @@ -389,7 +505,11 @@ const makeOrchestrationEngine = Effect.gen(function* () { ), ); - if (isOrchestrationCommandRejection(error)) { + if ( + isOrchestrationCommandRejection(error) && + !unarchiveRestoreFailed && + !creationCleanupPending + ) { yield* commandReceiptRepository .upsert({ commandId: envelope.command.commandId, diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts index e8c7154f38e7..ac350bac92b6 100644 --- a/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts +++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts @@ -42,6 +42,7 @@ import { import { OrchestrationProjectionSnapshotQueryLive } from "./ProjectionSnapshotQuery.ts"; import { ProjectionSnapshotQuery } from "../Services/ProjectionSnapshotQuery.ts"; import * as ThreadBackgroundLiveness from "../ThreadBackgroundLiveness.ts"; +import * as ThreadColdStorage from "../ThreadColdStorage.ts"; import * as ThreadPlanProgress from "../ThreadPlanProgress.ts"; import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts"; import { OrchestrationProjectionPipeline } from "../Services/ProjectionPipeline.ts"; @@ -4416,6 +4417,7 @@ it.effect("restores pending turn-start metadata across projection pipeline resta const engineLayer = it.layer( OrchestrationEngineLive.pipe( + Layer.provide(ThreadColdStorage.noOpLayer), Layer.provideMerge(OrchestrationProjectionSnapshotQueryLive), Layer.provide(ThreadBackgroundLiveness.layer), Layer.provide(ThreadPlanProgress.layer), diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts index 87f9bd03d46c..9212975572c2 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts @@ -61,6 +61,7 @@ import { OrchestrationEngineLive } from "./OrchestrationEngine.ts"; import { OrchestrationProjectionPipelineLive } from "./ProjectionPipeline.ts"; import { OrchestrationProjectionSnapshotQueryLive } from "./ProjectionSnapshotQuery.ts"; import * as ThreadBackgroundLiveness from "../ThreadBackgroundLiveness.ts"; +import * as ThreadColdStorage from "../ThreadColdStorage.ts"; import * as ThreadPlanProgress from "../ThreadPlanProgress.ts"; import { providerErrorLabelFromInstanceHint, @@ -404,6 +405,7 @@ describe("ProviderCommandReactor", () => { }; const orchestrationLayer = OrchestrationEngineLive.pipe( + Layer.provide(ThreadColdStorage.noOpLayer), Layer.provide(OrchestrationProjectionSnapshotQueryLive), Layer.provide(ThreadBackgroundLiveness.layer), Layer.provide(ThreadPlanProgress.layer), diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts index 4c6e3703ba82..028079516bd0 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts @@ -57,6 +57,7 @@ import { OrchestrationEngineLive } from "./OrchestrationEngine.ts"; import { OrchestrationProjectionPipelineLive } from "./ProjectionPipeline.ts"; import { OrchestrationProjectionSnapshotQueryLive } from "./ProjectionSnapshotQuery.ts"; import * as ThreadBackgroundLiveness from "../ThreadBackgroundLiveness.ts"; +import * as ThreadColdStorage from "../ThreadColdStorage.ts"; import * as ThreadPlanProgress from "../ThreadPlanProgress.ts"; import { ProviderRuntimeIngestionLive, @@ -282,6 +283,7 @@ describe("ProviderRuntimeIngestion", () => { const provider = createProviderServiceHarness(); const sqlCounter = makeSqlStatementCounter(); const orchestrationLayer = OrchestrationEngineLive.pipe( + Layer.provide(ThreadColdStorage.noOpLayer), Layer.provide(OrchestrationProjectionSnapshotQueryLive), Layer.provide(OrchestrationProjectionPipelineLive), Layer.provide(OrchestrationEventStoreLive), diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts index f83f1dd1b9fa..84995b0b741a 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts @@ -1,36 +1,196 @@ import { CommandId, CorrelationId, + DEFAULT_PROVIDER_INTERACTION_MODE, EventId, - type OrchestrationEvent, + PreviewSessionLookupError, + ProjectId, + ProviderDriverKind, + ProviderInstanceId, ThreadId, + type OrchestrationEvent, } from "@t3tools/contracts"; -import { it as effectIt } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it as effectIt } from "@effect/vitest"; import * as Cause from "effect/Cause"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as PubSub from "effect/PubSub"; import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as TestClock from "effect/testing/TestClock"; import { describe, expect, it } from "vite-plus/test"; +import * as ServerConfig from "../../config.ts"; +import { SqlitePersistenceMemory } from "../../persistence/Layers/Sqlite.ts"; import { - ProviderService, - type ProviderServiceShape, -} from "../../provider/Services/ProviderService.ts"; + NoOpProviderEventLoggers, + ProviderEventLoggers, +} from "../../provider/Layers/ProviderEventLoggers.ts"; +import { ProviderValidationError } from "../../provider/Errors.ts"; +import { ProviderService } from "../../provider/Services/ProviderService.ts"; +import { ProviderSessionDirectory } from "../../provider/Services/ProviderSessionDirectory.ts"; +import { ProjectionThreadSessionRepository } from "../../persistence/Services/ProjectionThreadSessions.ts"; +import { PreviewManager } from "../../preview/Manager.ts"; import * as TerminalManager from "../../terminal/Manager.ts"; -import { - OrchestrationEngineService, - type OrchestrationEngineShape, -} from "../Services/OrchestrationEngine.ts"; +import { ServerActivation } from "../../serverActivation.ts"; +import { decideOrchestrationCommand } from "../decider.ts"; +import { createEmptyReadModel, projectEvent } from "../projector.ts"; +import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts"; +import * as ThreadColdStorage from "../ThreadColdStorage.ts"; import { ThreadDeletionReactor } from "../Services/ThreadDeletionReactor.ts"; +import * as ThreadBackgroundLiveness from "../ThreadBackgroundLiveness.ts"; +import { makeTestOrchestrationEngine } from "../testUtils/orchestrationEngine.ts"; import { + enqueueLifecycleJobOnce, logCleanupCauseUnlessInterrupted, + THREAD_LIFECYCLE_RETRY_DELAY, ThreadDeletionReactorLive, } from "./ThreadDeletionReactor.ts"; +const encodeUnknownJsonString = Schema.encodeUnknownSync(Schema.fromJsonString(Schema.Unknown)); + +const archivedEvent = (threadId: ThreadId): OrchestrationEvent => ({ + sequence: 1, + eventId: EventId.make(`event-archive-${threadId}`), + aggregateKind: "thread", + aggregateId: threadId, + type: "thread.archived", + occurredAt: "2026-07-20T00:00:00.000Z", + commandId: CommandId.make(`command-archive-${threadId}`), + causationEventId: null, + correlationId: CommandId.make(`command-archive-${threadId}`), + metadata: {}, + payload: { + threadId, + archivedAt: "2026-07-20T00:00:00.000Z", + updatedAt: "2026-07-20T00:00:00.000Z", + }, +}); + +const deletedEvent = (threadId: ThreadId): OrchestrationEvent => ({ + sequence: 1, + eventId: EventId.make(`event-delete-${threadId}`), + aggregateKind: "thread", + aggregateId: threadId, + type: "thread.deleted", + occurredAt: "2026-07-20T00:00:00.000Z", + commandId: CommandId.make(`command-delete-${threadId}`), + causationEventId: null, + correlationId: CommandId.make(`command-delete-${threadId}`), + metadata: {}, + payload: { + threadId, + deletedAt: "2026-07-20T00:00:00.000Z", + }, +}); + +const unarchivedEvent = (threadId: ThreadId): OrchestrationEvent => ({ + sequence: 2, + eventId: EventId.make(`event-unarchive-${threadId}`), + aggregateKind: "thread", + aggregateId: threadId, + type: "thread.unarchived", + occurredAt: "2026-07-20T00:00:01.000Z", + commandId: CommandId.make(`command-unarchive-${threadId}`), + causationEventId: null, + correlationId: CommandId.make(`command-unarchive-${threadId}`), + metadata: {}, + payload: { + threadId, + updatedAt: "2026-07-20T00:00:01.000Z", + }, +}); + +function testReactorLayer(input: { + readonly eventStream: Stream.Stream; + readonly stopSession: ProviderService["Service"]["stopSession"]; + readonly getBinding: ProviderSessionDirectory["Service"]["getBinding"]; + readonly getProjectedSession: ProjectionThreadSessionRepository["Service"]["getByThreadId"]; + readonly archiveThread: ThreadColdStorage.ThreadColdStorage["Service"]["archiveThread"]; + readonly deleteThread?: ThreadColdStorage.ThreadColdStorage["Service"]["deleteThread"]; + readonly closePreview?: PreviewManager["Service"]["close"]; + readonly closeTerminal?: TerminalManager.TerminalManager["Service"]["close"]; + readonly pendingArchives?: ReadonlyArray; + readonly runArchiveQuiesce?: boolean; + readonly backgroundLiveness?: ThreadBackgroundLiveness.ThreadBackgroundLivenessService["Service"]; + readonly latestSequence?: Effect.Effect; +}) { + return ThreadDeletionReactorLive.pipe( + Layer.provide( + Layer.succeed( + OrchestrationEngineService, + makeTestOrchestrationEngine({ + streamDomainEvents: input.eventStream, + latestSequence: input.latestSequence ?? Effect.succeed(0), + }), + ), + ), + Layer.provide(Layer.mock(ProviderService)({ stopSession: input.stopSession })), + Layer.provide( + Layer.mock(ProviderSessionDirectory)({ + getBinding: input.getBinding, + }), + ), + Layer.provide( + Layer.mock(ProjectionThreadSessionRepository)({ + getByThreadId: input.getProjectedSession, + }), + ), + Layer.provide( + Layer.mock(TerminalManager.TerminalManager)({ + close: input.closeTerminal ?? (() => Effect.void), + }), + ), + Layer.provide( + Layer.mock(PreviewManager)({ + close: input.closePreview ?? (() => Effect.void), + }), + ), + Layer.provide(Layer.succeed(ProviderEventLoggers, NoOpProviderEventLoggers)), + Layer.provide( + Layer.succeed( + ThreadBackgroundLiveness.ThreadBackgroundLivenessService, + input.backgroundLiveness ?? ThreadBackgroundLiveness.make(), + ), + ), + Layer.provide( + Layer.mock(ThreadColdStorage.ThreadColdStorage)({ + archiveThread: (threadId, quiesce) => + (input.runArchiveQuiesce === false ? Effect.void : (quiesce ?? Effect.void)).pipe( + Effect.mapError( + (cause) => + new ThreadColdStorage.ThreadColdStorageError({ + operation: "archive", + threadId, + cause, + }), + ), + Effect.andThen(input.archiveThread(threadId)), + ), + deleteThread: (threadId, quiesce = Effect.void) => + quiesce.pipe( + Effect.mapError((cause) => + ThreadColdStorage.ThreadColdStorageError.normalize("delete", threadId, cause), + ), + Effect.andThen(input.deleteThread?.(threadId) ?? Effect.void), + ), + compactLegacyStorage: Effect.void, + listPendingArchiveThreadIds: Effect.succeed(input.pendingArchives ?? []), + listPendingDeleteThreadIds: Effect.succeed([]), + }), + ), + ); +} + describe("logCleanupCauseUnlessInterrupted", () => { const threadId = ThreadId.make("thread-deletion-reactor-test"); @@ -62,6 +222,793 @@ describe("logCleanupCauseUnlessInterrupted", () => { }); }); +effectIt.effect("does not delete while provider quiescence is failing", () => + Effect.gen(function* () { + const events = yield* PubSub.unbounded(); + const subscription = yield* PubSub.subscribe(events); + const threadId = ThreadId.make("thread-delete-provider-still-active"); + const stopAttempted = yield* Deferred.make(); + const deleteCalls = yield* Ref.make(0); + const layer = testReactorLayer({ + eventStream: Stream.fromSubscription(subscription), + stopSession: () => + Deferred.succeed(stopAttempted, undefined).pipe( + Effect.andThen( + Effect.fail( + new ProviderValidationError({ + operation: "ProviderService.stopSession", + issue: "provider is still active", + }), + ), + ), + ), + getBinding: () => + Effect.succeed( + Option.some({ + threadId, + provider: ProviderDriverKind.make("codex"), + providerInstanceId: ProviderInstanceId.make("codex"), + }), + ), + getProjectedSession: () => Effect.succeed(Option.none()), + archiveThread: () => Effect.void, + deleteThread: () => Ref.update(deleteCalls, (count) => count + 1), + }); + + yield* Effect.gen(function* () { + const reactor = yield* ThreadDeletionReactor; + yield* reactor.start(); + yield* PubSub.publish(events, deletedEvent(threadId)); + yield* Deferred.await(stopAttempted); + yield* reactor.drainThrough(1); + + expect(yield* Ref.get(deleteCalls)).toBe(0); + }).pipe(Effect.provide(layer)); + }), +); + +effectIt.effect("deletes a settled thread when its provider binding is already absent", () => + Effect.gen(function* () { + const events = yield* PubSub.unbounded(); + const subscription = yield* PubSub.subscribe(events); + const threadId = ThreadId.make("thread-delete-missing-binding"); + const deleted = yield* Deferred.make(); + const stopCalls = yield* Ref.make(0); + const layer = testReactorLayer({ + eventStream: Stream.fromSubscription(subscription), + stopSession: () => Ref.update(stopCalls, (count) => count + 1), + getBinding: () => Effect.succeed(Option.none()), + getProjectedSession: () => Effect.succeed(Option.none()), + archiveThread: () => Effect.void, + deleteThread: () => Deferred.succeed(deleted, undefined).pipe(Effect.asVoid), + }); + + yield* Effect.gen(function* () { + const reactor = yield* ThreadDeletionReactor; + yield* reactor.start(); + yield* PubSub.publish(events, deletedEvent(threadId)); + yield* Deferred.await(deleted); + yield* reactor.drainThrough(1); + + expect(yield* Ref.get(stopCalls)).toBe(0); + }).pipe(Effect.provide(layer)); + }), +); + +effectIt.effect("releases a lifecycle job reservation when enqueueing is interrupted", () => + Effect.gen(function* () { + const scheduledJobs = new Set(); + const enqueueCalls = yield* Ref.make(0); + + const interrupted = yield* Effect.exit( + enqueueLifecycleJobOnce(scheduledJobs, "archive:thread-interrupted", Effect.interrupt), + ); + expect(Exit.isFailure(interrupted)).toBe(true); + expect(scheduledJobs.has("archive:thread-interrupted")).toBe(false); + + yield* enqueueLifecycleJobOnce( + scheduledJobs, + "archive:thread-interrupted", + Ref.update(enqueueCalls, (count) => count + 1), + ); + expect(yield* Ref.get(enqueueCalls)).toBe(1); + }), +); + +effectIt.effect("archives a settled thread when its provider binding is already absent", () => + Effect.gen(function* () { + const events = yield* PubSub.unbounded(); + const subscription = yield* PubSub.subscribe(events); + const threadId = ThreadId.make("thread-archive-missing-binding"); + const archived = yield* Deferred.make(); + const stopCalls = yield* Ref.make(0); + const layer = testReactorLayer({ + eventStream: Stream.fromSubscription(subscription), + stopSession: () => Ref.update(stopCalls, (count) => count + 1), + getBinding: () => Effect.succeed(Option.none()), + getProjectedSession: () => Effect.succeed(Option.none()), + archiveThread: () => Deferred.succeed(archived, undefined).pipe(Effect.asVoid), + }); + + yield* Effect.gen(function* () { + const reactor = yield* ThreadDeletionReactor; + yield* reactor.start(); + yield* PubSub.publish(events, archivedEvent(threadId)); + yield* Deferred.await(archived); + yield* reactor.drainThrough(1); + + expect(yield* Ref.get(stopCalls)).toBe(0); + }).pipe(Effect.provide(layer)); + }), +); + +effectIt.effect("clears background liveness after archive provider quiescence", () => + Effect.gen(function* () { + const events = yield* PubSub.unbounded(); + const subscription = yield* PubSub.subscribe(events); + const threadId = ThreadId.make("thread-archive-background-liveness"); + const archived = yield* Deferred.make(); + const backgroundLiveness = ThreadBackgroundLiveness.make(); + backgroundLiveness.recordTaskLiveness({ + threadId, + taskId: "task-background-liveness", + taskType: "spawn_agent", + status: "running", + kind: "started", + }); + const layer = testReactorLayer({ + eventStream: Stream.fromSubscription(subscription), + stopSession: () => Effect.void, + getBinding: () => Effect.succeed(Option.none()), + getProjectedSession: () => Effect.succeed(Option.none()), + archiveThread: () => Deferred.succeed(archived, undefined).pipe(Effect.asVoid), + backgroundLiveness, + }); + + expect(backgroundLiveness.getThreadBackgroundLiveness(threadId)).toBe("working"); + yield* Effect.gen(function* () { + const reactor = yield* ThreadDeletionReactor; + yield* reactor.start(); + yield* PubSub.publish(events, archivedEvent(threadId)); + yield* Deferred.await(archived); + yield* reactor.drainThrough(1); + }).pipe(Effect.provide(layer)); + + expect(backgroundLiveness.getThreadBackgroundLiveness(threadId)).toBeNull(); + }), +); + +effectIt.effect("closes previews when observing an archive event", () => + Effect.gen(function* () { + const events = yield* PubSub.unbounded(); + const subscription = yield* PubSub.subscribe(events); + const threadId = ThreadId.make("thread-archive-preview"); + const archived = yield* Deferred.make(); + const previewCloseCalls = yield* Ref.make>([]); + const layer = testReactorLayer({ + eventStream: Stream.fromSubscription(subscription), + stopSession: () => Effect.void, + getBinding: () => Effect.succeed(Option.none()), + getProjectedSession: () => Effect.succeed(Option.none()), + closePreview: ({ threadId: closedThreadId }) => + Ref.update(previewCloseCalls, (calls) => [...calls, closedThreadId]).pipe( + Effect.andThen( + Effect.fail( + new PreviewSessionLookupError({ + threadId: closedThreadId, + tabId: "missing-preview", + }), + ), + ), + ), + archiveThread: () => Deferred.succeed(archived, undefined).pipe(Effect.asVoid), + }); + + yield* Effect.gen(function* () { + const reactor = yield* ThreadDeletionReactor; + yield* reactor.start(); + yield* PubSub.publish(events, archivedEvent(threadId)); + yield* Deferred.await(archived); + yield* reactor.drainThrough(1); + + expect(yield* Ref.get(previewCloseCalls)).toEqual([threadId]); + }).pipe(Effect.provide(layer)); + }), +); + +effectIt.effect("leaves stale archive cleanup to cold-storage eligibility", () => + Effect.gen(function* () { + const events = yield* PubSub.unbounded(); + const subscription = yield* PubSub.subscribe(events); + const threadId = ThreadId.make("thread-stale-archive-cleanup"); + const archived = yield* Deferred.make(); + const stopCalls = yield* Ref.make(0); + const terminalCloseCalls = yield* Ref.make(0); + const backgroundLiveness = ThreadBackgroundLiveness.make(); + backgroundLiveness.recordTaskLiveness({ + threadId, + taskId: "task-restored-background-liveness", + taskType: "spawn_agent", + status: "running", + kind: "started", + }); + const layer = testReactorLayer({ + eventStream: Stream.fromSubscription(subscription), + stopSession: () => Ref.update(stopCalls, (count) => count + 1), + getBinding: () => + Effect.succeed( + Option.some({ + threadId, + provider: ProviderDriverKind.make("codex"), + providerInstanceId: ProviderInstanceId.make("codex"), + }), + ), + getProjectedSession: () => Effect.succeed(Option.none()), + closeTerminal: () => Ref.update(terminalCloseCalls, (count) => count + 1), + archiveThread: () => Deferred.succeed(archived, undefined).pipe(Effect.asVoid), + runArchiveQuiesce: false, + backgroundLiveness, + }); + + yield* Effect.gen(function* () { + const reactor = yield* ThreadDeletionReactor; + yield* reactor.start(); + yield* PubSub.publish(events, archivedEvent(threadId)); + yield* Deferred.await(archived); + yield* reactor.drainThrough(1); + + expect(yield* Ref.get(stopCalls)).toBe(0); + expect(yield* Ref.get(terminalCloseCalls)).toBe(0); + expect(backgroundLiveness.getThreadBackgroundLiveness(threadId)).toBe("working"); + }).pipe(Effect.provide(layer)); + }), +); + +effectIt.effect("closes a restored preview at unarchive instead of from a stale archive job", () => + Effect.gen(function* () { + const events = yield* PubSub.unbounded(); + const subscription = yield* PubSub.subscribe(events); + const threadId = ThreadId.make("thread-restored-before-archive-job"); + const staleArchiveChecked = yield* Deferred.make(); + const restoredPreviewClosed = yield* Deferred.make(); + const previewCloseCalls = yield* Ref.make(0); + const layer = testReactorLayer({ + eventStream: Stream.fromSubscription(subscription), + stopSession: () => Effect.void, + getBinding: () => Effect.succeed(Option.none()), + getProjectedSession: () => Effect.succeed(Option.none()), + closePreview: () => + Ref.update(previewCloseCalls, (count) => count + 1).pipe( + Effect.andThen(Deferred.succeed(restoredPreviewClosed, undefined)), + Effect.asVoid, + ), + archiveThread: () => Deferred.succeed(staleArchiveChecked, undefined).pipe(Effect.asVoid), + // Models cold storage rejecting a queued archive after its locked + // eligibility recheck because the thread has already been restored. + runArchiveQuiesce: false, + }); + + yield* Effect.gen(function* () { + const reactor = yield* ThreadDeletionReactor; + yield* reactor.start(); + yield* PubSub.publish(events, archivedEvent(threadId)); + yield* Deferred.await(staleArchiveChecked); + yield* reactor.drainThrough(1); + + expect(yield* Ref.get(previewCloseCalls)).toBe(0); + + yield* PubSub.publish(events, unarchivedEvent(threadId)); + yield* Deferred.await(restoredPreviewClosed); + expect(yield* Ref.get(previewCloseCalls)).toBe(1); + }).pipe(Effect.provide(layer)); + }), +); + +effectIt.effect( + "keeps archive cleanup fail-closed without a binding for an active projection", + () => + Effect.gen(function* () { + const events = yield* PubSub.unbounded(); + const subscription = yield* PubSub.subscribe(events); + const threadId = ThreadId.make("thread-archive-active-missing-binding"); + const stopAttempted = yield* Deferred.make(); + const archiveCalls = yield* Ref.make(0); + const layer = testReactorLayer({ + eventStream: Stream.fromSubscription(subscription), + stopSession: () => + Deferred.succeed(stopAttempted, undefined).pipe( + Effect.andThen( + Effect.fail( + new ProviderValidationError({ + operation: "ProviderService.stopSession", + issue: "missing provider binding", + }), + ), + ), + ), + getBinding: () => Effect.succeed(Option.none()), + getProjectedSession: () => + Effect.succeed( + Option.some({ + threadId, + status: "running", + providerName: "codex", + providerInstanceId: ProviderInstanceId.make("codex"), + runtimeMode: "full-access", + activeTurnId: null, + lastError: null, + updatedAt: "2026-07-20T00:00:00.000Z", + }), + ), + archiveThread: () => Ref.update(archiveCalls, (count) => count + 1), + }); + + yield* Effect.gen(function* () { + const reactor = yield* ThreadDeletionReactor; + yield* reactor.start(); + yield* PubSub.publish(events, archivedEvent(threadId)); + yield* Deferred.await(stopAttempted); + yield* reactor.drainThrough(1); + + expect(yield* Ref.get(archiveCalls)).toBe(0); + }).pipe(Effect.provide(layer)); + }), +); + +effectIt.effect("retries a failed durable archive job after a delay", () => + Effect.gen(function* () { + const threadId = ThreadId.make("thread-archive-retry"); + const firstAttempt = yield* Deferred.make(); + const secondAttempt = yield* Deferred.make(); + const archiveAttempts = yield* Ref.make(0); + const layer = testReactorLayer({ + eventStream: Stream.empty, + stopSession: () => Effect.void, + getBinding: () => Effect.succeed(Option.none()), + getProjectedSession: () => Effect.succeed(Option.none()), + pendingArchives: [threadId], + archiveThread: () => + Ref.updateAndGet(archiveAttempts, (count) => count + 1).pipe( + Effect.flatMap((attempt) => + attempt === 1 + ? Deferred.succeed(firstAttempt, undefined).pipe( + Effect.andThen( + Effect.fail( + new ThreadColdStorage.ThreadColdStorageError({ + operation: "archive", + threadId, + cause: new Error("temporary archive failure"), + }), + ), + ), + ) + : Deferred.succeed(secondAttempt, undefined).pipe(Effect.asVoid), + ), + ), + }); + + yield* Effect.gen(function* () { + const reactor = yield* ThreadDeletionReactor; + yield* reactor.start(); + yield* Deferred.await(firstAttempt); + yield* reactor.drainThrough(0); + yield* Effect.yieldNow; + yield* TestClock.adjust(THREAD_LIFECYCLE_RETRY_DELAY); + yield* Deferred.await(secondAttempt); + yield* reactor.drainThrough(0); + + expect(yield* Ref.get(archiveAttempts)).toBe(2); + }).pipe(Effect.provide(layer)); + }), +); + +effectIt.effect("coalesces concurrent lifecycle failures into one delayed rescan", () => + Effect.gen(function* () { + const firstThreadId = ThreadId.make("thread-archive-retry-first"); + const secondThreadId = ThreadId.make("thread-archive-retry-second"); + const retryCompleted = yield* Deferred.make(); + const archiveAttempts = yield* Ref.make(0); + const layer = testReactorLayer({ + eventStream: Stream.empty, + stopSession: () => Effect.void, + getBinding: () => Effect.succeed(Option.none()), + getProjectedSession: () => Effect.succeed(Option.none()), + pendingArchives: [firstThreadId, secondThreadId], + archiveThread: (threadId) => + Ref.updateAndGet(archiveAttempts, (count) => count + 1).pipe( + Effect.flatMap((attempt) => { + if (attempt <= 2) { + return Effect.fail( + new ThreadColdStorage.ThreadColdStorageError({ + operation: "archive", + threadId, + cause: new Error("temporary archive failure"), + }), + ); + } + return attempt === 4 + ? Deferred.succeed(retryCompleted, undefined).pipe(Effect.asVoid) + : Effect.void; + }), + ), + }); + + yield* Effect.gen(function* () { + const reactor = yield* ThreadDeletionReactor; + yield* reactor.start(); + yield* reactor.drainThrough(0); + expect(yield* Ref.get(archiveAttempts)).toBe(2); + + yield* TestClock.adjust(THREAD_LIFECYCLE_RETRY_DELAY); + yield* Deferred.await(retryCompleted); + yield* reactor.drainThrough(0); + expect(yield* Ref.get(archiveAttempts)).toBe(4); + + yield* TestClock.adjust(THREAD_LIFECYCLE_RETRY_DELAY); + yield* Effect.yieldNow; + yield* reactor.drainThrough(0); + expect(yield* Ref.get(archiveAttempts)).toBe(4); + }).pipe(Effect.provide(layer)); + }), +); + +effectIt.effect("unarchive events reclaim bundles left by failed finalization", () => + Effect.gen(function* () { + const events = yield* PubSub.unbounded(); + const subscription = yield* PubSub.subscribe(events); + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const threadId = ThreadId.make("thread-unarchive-finalize-retry"); + const reactorLayer = testReactorLayer({ + eventStream: Stream.fromSubscription(subscription), + stopSession: () => Effect.die("Finalization must not stop the provider"), + getBinding: () => Effect.succeed(Option.none()), + getProjectedSession: () => Effect.succeed(Option.none()), + closeTerminal: () => Effect.die("Finalization must not close the terminal"), + runArchiveQuiesce: false, + archiveThread: (id) => + storage.archiveThread(id, Effect.die("Active-shell cleanup must not quiesce")), + }); + + yield* Effect.gen(function* () { + const reactor = yield* ThreadDeletionReactor; + yield* reactor.start(); + yield* sql` + INSERT INTO projection_threads ( + thread_id, project_id, title, model_selection_json, runtime_mode, + interaction_mode, created_at, updated_at, archived_at + ) VALUES ( + ${threadId}, 'project-finalize', 'Finalize after unarchive', + '{"instanceId":"codex","model":"gpt-5.5","options":[]}', + 'full-access', 'default', '2026-07-20T00:00:00.000Z', + '2026-07-20T00:00:00.000Z', '2026-07-20T00:00:00.000Z' + ) + `; + yield* storage.archiveThread(threadId); + assert.isTrue(yield* storage.restoreTree(threadId)); + yield* sql`UPDATE projection_threads SET archived_at = NULL WHERE thread_id = ${threadId}`; + yield* sql.unsafe(` + CREATE TEMP TRIGGER fail_finalization BEFORE DELETE ON thread_archive_manifests + BEGIN SELECT RAISE(ABORT, 'temporary finalization failure'); END + `); + yield* Effect.flip(storage.finishRestoreTree(threadId)).pipe( + Effect.ensuring(sql.unsafe("DROP TRIGGER temp.fail_finalization").pipe(Effect.orDie)), + ); + yield* PubSub.publish(events, unarchivedEvent(threadId)); + yield* reactor.drainThrough(2); + assert.deepStrictEqual( + yield* sql`SELECT archived_at FROM projection_threads WHERE thread_id = ${threadId}`, + [{ archived_at: null }], + ); + assert.deepStrictEqual( + yield* sql`SELECT thread_id FROM thread_archive_manifests WHERE thread_id = ${threadId}`, + [], + ); + assert.deepStrictEqual( + yield* sql`SELECT thread_id FROM cold_archive.archive_threads WHERE thread_id = ${threadId}`, + [], + ); + }).pipe(Effect.provide(reactorLayer)); + }).pipe( + Effect.provide( + ThreadColdStorage.layer.pipe( + Layer.provideMerge(SqlitePersistenceMemory), + Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "t3-finalize-retry-" })), + Layer.provideMerge(NodeServices.layer), + ), + ), + ), +); + +effectIt.effect("keeps startup archive data intact until server activation", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const config = yield* ServerConfig.ServerConfig; + const activation = yield* Deferred.make(); + const cleanupStarted = yield* Deferred.make(); + const threadId = ThreadId.make("archive-awaiting-activation"); + const attachmentId = `${threadId}-00000000-0000-4000-8000-000000000001`; + const attachmentPath = path.join(config.attachmentsDir, `${attachmentId}.png`); + yield* fs.makeDirectory(config.attachmentsDir, { recursive: true }); + yield* fs.writeFileString(attachmentPath, "attachment retained across a failed trial"); + yield* sql`INSERT INTO projection_threads ( + thread_id, project_id, title, model_selection_json, runtime_mode, + interaction_mode, created_at, updated_at, archived_at + ) VALUES (${threadId}, 'project-activation', 'Pending archive', + '{"instanceId":"codex","model":"gpt-5.5","options":[]}', 'full-access', 'default', + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`; + yield* sql`INSERT INTO projection_thread_messages ( + message_id, thread_id, role, text, attachments_json, is_streaming, created_at, updated_at + ) VALUES ('activation-message', ${threadId}, 'user', 'hot history', + ${encodeUnknownJsonString([{ id: attachmentId }])}, 0, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`; + + const reactorLayer = ThreadDeletionReactorLive.pipe( + Layer.provide( + Layer.succeed( + OrchestrationEngineService, + makeTestOrchestrationEngine({ + streamDomainEvents: Stream.never, + }), + ), + ), + Layer.provide(Layer.mock(ProviderService)({ stopSession: () => Effect.void })), + Layer.provide(Layer.mock(ProviderSessionDirectory)({ getBinding: () => Effect.succeedNone })), + Layer.provide( + Layer.mock(ProjectionThreadSessionRepository)({ getByThreadId: () => Effect.succeedNone }), + ), + Layer.provide(Layer.mock(PreviewManager)({ close: () => Effect.void })), + Layer.provide( + Layer.mock(TerminalManager.TerminalManager)({ + close: () => Deferred.succeed(cleanupStarted, undefined).pipe(Effect.asVoid), + }), + ), + Layer.provide(Layer.succeed(ProviderEventLoggers, NoOpProviderEventLoggers)), + Layer.provide(ThreadBackgroundLiveness.layer), + Layer.provide(Layer.succeed(ThreadColdStorage.ThreadColdStorage, storage)), + ); + yield* Effect.gen(function* () { + const reactor = yield* ThreadDeletionReactor; + yield* reactor.start(); + yield* reactor.drainThrough(0); + assert.deepStrictEqual( + yield* sql`SELECT text FROM projection_thread_messages WHERE thread_id = ${threadId}`, + [{ text: "hot history" }], + ); + assert.strictEqual( + yield* fs.readFileString(attachmentPath), + "attachment retained across a failed trial", + ); + assert.deepStrictEqual( + yield* sql`SELECT thread_id FROM cold_archive.archive_threads WHERE thread_id = ${threadId}`, + [], + ); + + yield* Deferred.succeed(activation, undefined); + yield* Deferred.await(cleanupStarted); + yield* reactor.drainThrough(0); + assert.deepStrictEqual( + yield* sql`SELECT text FROM projection_thread_messages WHERE thread_id = ${threadId}`, + [], + ); + assert.isFalse(yield* fs.exists(attachmentPath)); + assert.deepStrictEqual( + yield* sql`SELECT status FROM thread_archive_manifests WHERE thread_id = ${threadId}`, + [{ status: "cold" }], + ); + assert.isTrue(yield* storage.restoreTree(threadId)); + assert.strictEqual( + yield* fs.readFileString(attachmentPath), + "attachment retained across a failed trial", + ); + }).pipe( + Effect.provide(reactorLayer), + Effect.provideService(ServerActivation, Deferred.await(activation)), + ); + }).pipe( + Effect.provide( + ThreadColdStorage.layer.pipe( + Layer.provideMerge(SqlitePersistenceMemory), + Layer.provideMerge( + ServerConfig.layerTest(process.cwd(), { prefix: "t3-archive-activation-" }), + ), + Layer.provideMerge(NodeServices.layer), + ), + ), + ), +); + +effectIt.effect("force-deleting a project removes an already-cold archived thread", () => + Effect.gen(function* () { + const events = yield* PubSub.unbounded(); + const eventSubscription = yield* PubSub.subscribe(events); + const now = "2026-07-20T00:00:00.000Z"; + const projectId = ProjectId.make("project-force-delete-cold"); + const threadId = ThreadId.make("thread-force-delete-cold"); + const commandId = CommandId.make("command-force-delete-cold"); + const terminalCloseStarted = yield* Deferred.make(); + const previewCloseCalls = yield* Ref.make>([]); + + const orchestrationEngineLayer = Layer.succeed( + OrchestrationEngineService, + makeTestOrchestrationEngine({ + streamDomainEvents: Stream.fromSubscription(eventSubscription), + }), + ); + const providerLayer = Layer.mock(ProviderService)({ + stopSession: () => Effect.void, + }); + const terminalLayer = Layer.mock(TerminalManager.TerminalManager)({ + close: () => Deferred.succeed(terminalCloseStarted, undefined).pipe(Effect.asVoid), + }); + const previewLayer = Layer.mock(PreviewManager)({ + close: ({ threadId: closedThreadId }) => + Ref.update(previewCloseCalls, (calls) => [...calls, closedThreadId]), + }); + const loggerLayer = Layer.succeed(ProviderEventLoggers, NoOpProviderEventLoggers); + const coldStorageLayer = ThreadColdStorage.layer.pipe( + Layer.provideMerge(SqlitePersistenceMemory), + Layer.provideMerge( + ServerConfig.layerTest(process.cwd(), { prefix: "t3-force-delete-cold-" }), + ), + Layer.provideMerge(NodeServices.layer), + ); + const runtimeLayer = ThreadDeletionReactorLive.pipe( + Layer.provide(orchestrationEngineLayer), + Layer.provide(providerLayer), + Layer.provide( + Layer.mock(ProviderSessionDirectory)({ + getBinding: () => Effect.succeed(Option.none()), + }), + ), + Layer.provide( + Layer.mock(ProjectionThreadSessionRepository)({ + getByThreadId: () => Effect.succeed(Option.none()), + }), + ), + Layer.provide(terminalLayer), + Layer.provide(previewLayer), + Layer.provide(loggerLayer), + Layer.provide(ThreadBackgroundLiveness.layer), + Layer.provideMerge(coldStorageLayer), + ); + + yield* Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const reactor = yield* ThreadDeletionReactor; + + yield* sql` + INSERT INTO projection_threads ( + thread_id, project_id, title, model_selection_json, runtime_mode, + interaction_mode, created_at, updated_at, archived_at + ) VALUES ( + ${threadId}, ${projectId}, 'Cold forced-delete thread', + '{"instanceId":"codex","model":"gpt-5.5","options":[]}', + 'full-access', 'default', ${now}, ${now}, ${now} + ) + `; + yield* sql` + INSERT INTO projection_thread_messages ( + message_id, thread_id, turn_id, role, text, attachments_json, + is_streaming, created_at, updated_at + ) VALUES ( + 'message-force-delete-cold', ${threadId}, NULL, 'user', + 'delete this cold content', '[]', 0, ${now}, ${now} + ) + `; + yield* storage.archiveThread(threadId); + const coldShells = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count FROM projection_threads WHERE thread_id = ${threadId} + `; + const coldManifests = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count FROM thread_archive_manifests WHERE thread_id = ${threadId} + `; + const coldChunks = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count FROM cold_archive.archive_thread_chunks WHERE thread_id = ${threadId} + `; + assert.deepStrictEqual(coldShells, [{ count: 1 }]); + assert.deepStrictEqual(coldManifests, [{ count: 1 }]); + assert.isAbove(coldChunks[0]?.count ?? 0, 0); + + let readModel = createEmptyReadModel(now); + readModel = yield* projectEvent(readModel, { + sequence: 1, + eventId: EventId.make("event-force-delete-project-created"), + aggregateKind: "project", + aggregateId: projectId, + type: "project.created", + occurredAt: now, + commandId: CommandId.make("command-force-delete-project-created"), + causationEventId: null, + correlationId: CommandId.make("command-force-delete-project-created"), + metadata: {}, + payload: { + projectId, + title: "Force Delete Cold", + workspaceRoot: "/tmp/project-force-delete-cold", + defaultModelSelection: null, + scripts: [], + createdAt: now, + updatedAt: now, + }, + }); + readModel = yield* projectEvent(readModel, { + sequence: 2, + eventId: EventId.make("event-force-delete-thread-created"), + aggregateKind: "thread", + aggregateId: threadId, + type: "thread.created", + occurredAt: now, + commandId: CommandId.make("command-force-delete-thread-created"), + causationEventId: null, + correlationId: CommandId.make("command-force-delete-thread-created"), + metadata: {}, + payload: { + threadId, + projectId, + title: "Cold forced-delete thread", + modelSelection: { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5.5", + }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "full-access", + branch: null, + worktreePath: null, + createdAt: now, + updatedAt: now, + }, + }); + readModel = yield* projectEvent(readModel, { + sequence: 3, + eventId: EventId.make("event-force-delete-thread-archived"), + aggregateKind: "thread", + aggregateId: threadId, + type: "thread.archived", + occurredAt: now, + commandId: CommandId.make("command-force-delete-thread-archived"), + causationEventId: null, + correlationId: CommandId.make("command-force-delete-thread-archived"), + metadata: {}, + payload: { threadId, archivedAt: now, updatedAt: now }, + }); + + const planned = yield* decideOrchestrationCommand({ + command: { type: "project.delete", commandId, projectId, force: true }, + readModel, + }); + const plannedEvents = Array.isArray(planned) ? planned : [planned]; + const deletedEvent = plannedEvents.find( + (event): event is Extract<(typeof plannedEvents)[number], { type: "thread.deleted" }> => + event.type === "thread.deleted" && event.payload.threadId === threadId, + ); + assert.isDefined(deletedEvent); + + yield* sql`UPDATE projection_threads SET deleted_at = ${now} WHERE thread_id = ${threadId}`; + yield* reactor.start(); + yield* PubSub.publish(events, { ...deletedEvent, sequence: 4 }); + yield* Deferred.await(terminalCloseStarted); + yield* reactor.drainThrough(4); + expect(yield* Ref.get(previewCloseCalls)).toEqual([threadId]); + + const hotRows = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count FROM projection_threads WHERE thread_id = ${threadId} + `; + const manifests = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count FROM thread_archive_manifests WHERE thread_id = ${threadId} + `; + const chunks = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count FROM cold_archive.archive_thread_chunks WHERE thread_id = ${threadId} + `; + assert.deepStrictEqual(hotRows, [{ count: 0 }]); + assert.deepStrictEqual(manifests, [{ count: 0 }]); + assert.deepStrictEqual(chunks, [{ count: 0 }]); + }).pipe(Effect.provide(runtimeLayer)); + }), +); describe("ThreadDeletionReactor drain", () => { const now = "2026-01-01T00:00:00.000Z"; const threadId = ThreadId.make("thread-deletion-reactor-drain"); @@ -79,6 +1026,94 @@ describe("ThreadDeletionReactor drain", () => { payload: { threadId, deletedAt: now }, }); + effectIt.effect("does not block new threads behind a bulk archive backlog", () => + Effect.gen(function* () { + const archiveStarted = yield* Deferred.make(); + const releaseArchive = yield* Deferred.make(); + const archivedThreadIds = Array.from({ length: 650 }, (_, index) => + ThreadId.make(`bulk-archive-${index}`), + ); + const completed: Array = []; + const layer = testReactorLayer({ + eventStream: Stream.never, + stopSession: () => Effect.void, + getBinding: () => Effect.succeed(Option.none()), + getProjectedSession: () => Effect.succeed(Option.none()), + pendingArchives: archivedThreadIds, + archiveThread: (id) => + Effect.gen(function* () { + yield* Deferred.succeed(archiveStarted, undefined); + yield* Deferred.await(releaseArchive); + completed.push(id); + }), + }); + + yield* Effect.scoped( + Effect.gen(function* () { + const reactor = yield* ThreadDeletionReactor; + yield* reactor.start(); + yield* Deferred.await(archiveStarted); + yield* reactor.drainThrough(0, ThreadId.make("unrelated-new-thread")); + expect(completed).toEqual([]); + yield* Deferred.succeed(releaseArchive, undefined); + yield* reactor.drainThrough(0); + expect(completed).toEqual(archivedThreadIds); + }), + ).pipe(Effect.provide(layer)); + }), + ); + + effectIt.effect("waits for the reused thread's cleanup without draining later archives", () => + Effect.gen(function* () { + const archiveStarted = yield* Deferred.make(); + const releaseArchive = yield* Deferred.make(); + const unrelatedArchiveStarted = yield* Deferred.make(); + const releaseUnrelatedArchive = yield* Deferred.make(); + const unrelatedThreadId = ThreadId.make("unrelated-archive"); + const completed: Array = []; + const layer = testReactorLayer({ + eventStream: Stream.never, + stopSession: () => Effect.void, + getBinding: () => Effect.succeed(Option.none()), + getProjectedSession: () => Effect.succeed(Option.none()), + pendingArchives: [threadId, unrelatedThreadId], + archiveThread: (id) => + Effect.gen(function* () { + yield* Deferred.succeed( + id === threadId ? archiveStarted : unrelatedArchiveStarted, + undefined, + ); + yield* Deferred.await(id === threadId ? releaseArchive : releaseUnrelatedArchive); + completed.push(id); + }), + }); + + yield* Effect.scoped( + Effect.gen(function* () { + const reactor = yield* ThreadDeletionReactor; + yield* reactor.start(); + yield* Deferred.await(archiveStarted); + const drained = yield* reactor.drainThrough(0, threadId).pipe( + Effect.tap(() => + Effect.sync(() => { + expect(completed).toContain(threadId); + }), + ), + Effect.forkChild, + ); + yield* Effect.yieldNow; + yield* Deferred.succeed(releaseArchive, undefined); + yield* Deferred.await(unrelatedArchiveStarted); + yield* Fiber.join(drained); + expect(completed).toEqual([threadId]); + yield* Deferred.succeed(releaseUnrelatedArchive, undefined); + yield* reactor.drainThrough(0); + expect(completed).toEqual([threadId, unrelatedThreadId]); + }), + ).pipe(Effect.provide(layer)); + }), + ); + effectIt.effect("waits for a published deletion the subscriber has not consumed yet", () => Effect.gen(function* () { const stops: Array = []; @@ -87,16 +1122,15 @@ describe("ThreadDeletionReactor drain", () => { // subscriber has not received it yet: the stream releases it on demand. const releaseSecondEvent = yield* Deferred.make(); const latestSequence = yield* Ref.make(0); - const engine = { - latestSequence: Ref.get(latestSequence), - streamDomainEvents: Stream.concat( - Stream.make(deletedEvent(1)), - Stream.fromEffect(Deferred.await(releaseSecondEvent)).pipe( - Stream.map(() => deletedEvent(2)), - ), + const eventStream = Stream.concat( + Stream.make(deletedEvent(1)), + Stream.fromEffect(Deferred.await(releaseSecondEvent)).pipe( + Stream.map(() => deletedEvent(2)), ), - } as unknown as OrchestrationEngineShape; - const providerService = { + ); + const layer = testReactorLayer({ + eventStream, + latestSequence: Ref.get(latestSequence), stopSession: () => Effect.gen(function* () { stops.push(stops.length + 1); @@ -104,15 +1138,18 @@ describe("ThreadDeletionReactor drain", () => { yield* Deferred.succeed(firstCleanupDone, undefined); } }), - } as unknown as ProviderServiceShape; - const terminalManager = { - close: () => Effect.void, - } as unknown as TerminalManager.TerminalManager["Service"]; - const layer = ThreadDeletionReactorLive.pipe( - Layer.provide(Layer.succeed(ProviderService, providerService)), - Layer.provide(Layer.succeed(TerminalManager.TerminalManager, terminalManager)), - Layer.provide(Layer.succeed(OrchestrationEngineService, engine)), - ); + getBinding: () => + Effect.succeed( + Option.some({ + threadId, + provider: ProviderDriverKind.make("codex"), + providerInstanceId: ProviderInstanceId.make("codex"), + }), + ), + getProjectedSession: () => Effect.succeed(Option.none()), + archiveThread: () => Effect.void, + deleteThread: () => Effect.void, + }); yield* Effect.scoped( Effect.gen(function* () { @@ -123,7 +1160,7 @@ describe("ThreadDeletionReactor drain", () => { // Sequence 1 is fully cleaned and the worker queue is idle. Sequence // 2 is committed and published but still in flight to the subscriber. yield* Ref.set(latestSequence, 2); - const drained = yield* Effect.forkChild(reactor.drainThrough(2)); + const drained = yield* Effect.forkChild(reactor.drainThrough(2, threadId)); yield* Effect.yieldNow; yield* Effect.yieldNow; expect(stops).toEqual([1]); diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts index 092ca1b1d471..3204c5b2695f 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts @@ -2,20 +2,59 @@ import type { OrchestrationEvent } from "@t3tools/contracts"; import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; import * as Cause from "effect/Cause"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; import * as SubscriptionRef from "effect/SubscriptionRef"; +import { ProviderEventLoggers } from "../../provider/Layers/ProviderEventLoggers.ts"; import { ProviderService } from "../../provider/Services/ProviderService.ts"; +import { ProviderSessionDirectory } from "../../provider/Services/ProviderSessionDirectory.ts"; +import { ProjectionThreadSessionRepository } from "../../persistence/Services/ProjectionThreadSessions.ts"; +import { PreviewManager } from "../../preview/Manager.ts"; import * as TerminalManager from "../../terminal/Manager.ts"; import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts"; +import * as ThreadColdStorage from "../ThreadColdStorage.ts"; import { ThreadDeletionReactor, type ThreadDeletionReactorShape, } from "../Services/ThreadDeletionReactor.ts"; -import { forkParked } from "../../serverActivation.ts"; +import { forkParked, ServerActivation } from "../../serverActivation.ts"; +import { ThreadBackgroundLivenessService } from "../ThreadBackgroundLiveness.ts"; type ThreadDeletedEvent = Extract; +type ThreadArchivedEvent = Extract; +type ThreadLifecycleJob = + | { readonly type: "archive"; readonly threadId: ThreadArchivedEvent["payload"]["threadId"] } + | { readonly type: "delete"; readonly threadId: ThreadDeletedEvent["payload"]["threadId"] } + | { readonly type: "compact-legacy-storage" }; + +export const THREAD_LIFECYCLE_RETRY_DELAY = "30 seconds"; + +function lifecycleJobKey(job: ThreadLifecycleJob): string { + return job.type === "compact-legacy-storage" ? job.type : `${job.type}:${job.threadId}`; +} + +export const enqueueLifecycleJobOnce = ( + scheduledJobs: Set, + key: string, + enqueue: Effect.Effect, +) => + Effect.suspend(() => { + if (scheduledJobs.has(key)) return Effect.void; + scheduledJobs.add(key); + return enqueue.pipe( + Effect.onExit((exit) => + Exit.isSuccess(exit) + ? Effect.void + : Effect.sync(() => { + scheduledJobs.delete(key); + }), + ), + ); + }); export const logCleanupCauseUnlessInterrupted = ({ effect, @@ -25,64 +64,168 @@ export const logCleanupCauseUnlessInterrupted = ({ readonly effect: Effect.Effect; readonly message: string; readonly threadId: ThreadDeletedEvent["payload"]["threadId"]; -}): Effect.Effect => +}): Effect.Effect => effect.pipe( - Effect.catchCauseIf( - (cause) => !Cause.hasInterruptsOnly(cause), - (cause) => - Effect.logDebug(message, { - threadId, - cause: Cause.pretty(cause), - }), - ), + Effect.catchCause((cause) => { + if (Cause.hasInterruptsOnly(cause)) { + return Effect.interrupt; + } + return Effect.logDebug(message, { + threadId, + cause: Cause.pretty(cause), + }); + }), ); const make = Effect.gen(function* () { const orchestrationEngine = yield* OrchestrationEngineService; const providerService = yield* ProviderService; + const providerSessionDirectory = yield* ProviderSessionDirectory; + const projectionThreadSessions = yield* ProjectionThreadSessionRepository; + const previewManager = yield* PreviewManager; const terminalManager = yield* TerminalManager.TerminalManager; + const threadColdStorage = yield* ThreadColdStorage.ThreadColdStorage; + const threadBackgroundLiveness = yield* ThreadBackgroundLivenessService; + const providerEventLoggers = yield* ProviderEventLoggers; + const retryRequested = yield* Queue.sliding(1); - const stopProviderSession = (threadId: ThreadDeletedEvent["payload"]["threadId"]) => + const closeThreadPreviews = (threadId: ThreadDeletedEvent["payload"]["threadId"]) => logCleanupCauseUnlessInterrupted({ - effect: providerService.stopSession({ threadId }), - message: "thread deletion cleanup skipped provider session stop", + effect: previewManager.close({ threadId }), + message: "thread lifecycle cleanup skipped preview close", threadId, }); - const closeThreadTerminals = (threadId: ThreadDeletedEvent["payload"]["threadId"]) => - logCleanupCauseUnlessInterrupted({ - effect: terminalManager.close({ threadId, deleteHistory: true }), - message: "thread deletion cleanup skipped terminal close", + const closeProviderLogWritersRequired = (threadId: ThreadDeletedEvent["payload"]["threadId"]) => + Effect.all( + [providerEventLoggers.native, providerEventLoggers.canonical].flatMap((logger) => + logger?.closeThread ? [logger.closeThread(threadId)] : [], + ), + { discard: true }, + ); + + const stopProviderSessionRequired = Effect.fn("stopProviderSessionRequired")(function* ( + threadId: ThreadArchivedEvent["payload"]["threadId"], + ) { + const binding = yield* providerSessionDirectory.getBinding(threadId); + if (Option.isSome(binding)) { + yield* providerService.stopSession({ threadId }); + return; + } + + const projectedSession = yield* projectionThreadSessions.getByThreadId({ threadId }); + const status = Option.isSome(projectedSession) ? projectedSession.value.status : null; + if (status === "starting" || status === "running") { + // The projection still permits an active writer. Keep failing closed and + // let durable lifecycle recovery retry after the provider binding appears. + yield* providerService.stopSession({ threadId }); + return; + } + + yield* Effect.logDebug("lifecycle cleanup found no provider binding for a settled thread", { threadId, + projectedSessionStatus: status, }); + }); - const processThreadDeleted = Effect.fn("processThreadDeleted")(function* ( - event: ThreadDeletedEvent, + const processLifecycleJob = Effect.fn("processThreadLifecycleJob")(function* ( + job: ThreadLifecycleJob, ) { - const { threadId } = event.payload; - yield* stopProviderSession(threadId); - yield* closeThreadTerminals(threadId); + if (job.type === "compact-legacy-storage") { + yield* threadColdStorage.compactLegacyStorage; + return; + } + const { threadId } = job; + if (job.type === "archive") { + // Archiving must not snapshot or delete hot rows while any active writer + // can still mutate them. Cold storage runs this cleanup while holding the + // same tree lock used by unarchive and only after rechecking eligibility, + // so a stale archive job cannot stop a newly active thread. + yield* threadColdStorage.archiveThread( + threadId, + Effect.gen(function* () { + yield* closeThreadPreviews(threadId); + yield* stopProviderSessionRequired(threadId); + // Provider runtime ingestion normally clears this process-local state + // when it observes session.exited. Archive cleanup can also reach a + // settled legacy/subagent shell with no provider binding, so clear it + // explicitly once the provider quiescence boundary has succeeded. + threadBackgroundLiveness.clearThreadLiveness(threadId); + yield* terminalManager.close({ threadId, deleteHistory: true }); + yield* closeProviderLogWritersRequired(threadId); + }), + ); + return; + } + // Deletion is destructive, so every writer-quiescence boundary must fail + // closed. processLifecycleJobSafely records the failure and requests a + // durable rescan instead of deleting while a provider, terminal, or log + // writer may still be active. + yield* threadColdStorage.deleteThread( + threadId, + Effect.gen(function* () { + yield* stopProviderSessionRequired(threadId); + yield* terminalManager.close({ threadId, deleteHistory: true }); + yield* closeProviderLogWritersRequired(threadId); + }), + ); }); - const processThreadDeletedSafely = (event: ThreadDeletedEvent) => - processThreadDeleted(event).pipe( + const processLifecycleJobSafely = (job: ThreadLifecycleJob) => + processLifecycleJob(job).pipe( Effect.catchCauseIf( (cause) => !Cause.hasInterruptsOnly(cause), (cause) => - Effect.logWarning("thread deletion reactor failed to process event", { - eventType: event.type, - threadId: event.payload.threadId, + Effect.logWarning("thread lifecycle reactor failed to process job", { + lifecycleAction: job.type, + ...(job.type === "compact-legacy-storage" ? {} : { threadId: job.threadId }), cause: Cause.pretty(cause), - }), + }).pipe(Effect.andThen(Queue.offer(retryRequested, undefined)), Effect.asVoid), ), ); - const worker = yield* makeDrainableWorker(processThreadDeletedSafely); + const scheduledJobs = new Set(); + const completedJobs = yield* SubscriptionRef.make(0); + const worker = yield* makeDrainableWorker((job: ThreadLifecycleJob) => + processLifecycleJobSafely(job).pipe( + Effect.ensuring( + Effect.sync(() => { + scheduledJobs.delete(lifecycleJobKey(job)); + }).pipe(Effect.andThen(SubscriptionRef.update(completedJobs, (count) => count + 1))), + ), + ), + ); + + const enqueueLifecycleJob = (job: ThreadLifecycleJob) => + enqueueLifecycleJobOnce(scheduledJobs, lifecycleJobKey(job), worker.enqueue(job)); - // Highest event sequence the subscriber has handed to the worker. Waiting - // through a successful thread.created sequence covers every deletion that - // was ahead of that create in the engine queue; the worker drain then covers - // the in-flight cleanup. + const enqueuePendingLifecycleJobs = Effect.fn("enqueuePendingThreadLifecycleJobs")(function* () { + const pendingJobs = yield* Effect.all([ + threadColdStorage.listPendingDeleteThreadIds, + threadColdStorage.listPendingArchiveThreadIds, + ]).pipe( + Effect.catchCause((cause) => + Effect.logWarning("failed to read pending thread storage migrations", { + cause: Cause.pretty(cause), + }).pipe(Effect.andThen(Queue.offer(retryRequested, undefined)), Effect.as(null)), + ), + ); + if (pendingJobs === null) return; + const [pendingDeletes, pendingArchives] = pendingJobs; + yield* Effect.forEach( + pendingDeletes, + (threadId) => enqueueLifecycleJob({ type: "delete", threadId }), + { discard: true }, + ); + yield* Effect.forEach( + pendingArchives, + (threadId) => enqueueLifecycleJob({ type: "archive", threadId }), + { discard: true }, + ); + }); + + // The creation sequence fences all prior lifecycle events before callers + // check whether this thread still has queued or in-flight cleanup. const seenSequence = yield* SubscriptionRef.make(0); const noteSeen = (sequence: number) => SubscriptionRef.update(seenSequence, (seen) => Math.max(seen, sequence)); @@ -95,22 +238,73 @@ const make = Effect.gen(function* () { // start the watermark at the current head instead of zero. Stream.onStart(orchestrationEngine.latestSequence.pipe(Effect.flatMap(noteSeen))), ), - (event) => - (event.type === "thread.deleted" ? worker.enqueue(event) : Effect.void).pipe( - Effect.andThen(noteSeen(event.sequence)), - ), + (event) => { + let handleEvent: Effect.Effect; + if (event.type === "thread.deleted") { + handleEvent = closeThreadPreviews(event.payload.threadId).pipe( + Effect.andThen( + enqueueLifecycleJob({ type: "delete", threadId: event.payload.threadId }), + ), + ); + } else if (event.type === "thread.archived") { + handleEvent = enqueueLifecycleJob({ + type: "archive", + threadId: event.payload.threadId, + }); + } else if (event.type === "thread.unarchived") { + // A fast archive -> unarchive can make the queued archive job fail + // its locked eligibility recheck before archive quiescence runs. + // Close the pre-archive preview lease at the authoritative restore + // transition as well, without letting a stale archive job close a + // preview for an already-active thread later. + handleEvent = closeThreadPreviews(event.payload.threadId).pipe( + // A committed unarchive can leave a restored bundle after a + // finalization failure. The same worker retries that cleanup. + Effect.andThen( + enqueueLifecycleJob({ type: "archive", threadId: event.payload.threadId }), + ), + ); + } else { + handleEvent = Effect.void; + } + return handleEvent.pipe(Effect.andThen(noteSeen(event.sequence))); + }, ), ); + + yield* Queue.take(retryRequested).pipe( + Effect.andThen(Effect.sleep(THREAD_LIFECYCLE_RETRY_DELAY)), + Effect.andThen(Queue.poll(retryRequested)), + Effect.andThen(enqueuePendingLifecycleJobs()), + Effect.andThen(enqueueLifecycleJob({ type: "compact-legacy-storage" })), + Effect.forever, + forkParked, + ); + + const recover = enqueuePendingLifecycleJobs().pipe( + Effect.andThen(enqueueLifecycleJob({ type: "compact-legacy-storage" })), + ); + const activation = yield* ServerActivation; + yield* activation === undefined ? recover : forkParked(recover); }); const drainThrough: ThreadDeletionReactorShape["drainThrough"] = Effect.fn( "ThreadDeletionReactor.drainThrough", - )(function* (target) { + )(function* (target, threadId) { yield* SubscriptionRef.changes(seenSequence).pipe( Stream.filter((seen) => seen >= target), Stream.runHead, ); - yield* worker.drain; + if (threadId === undefined) { + yield* worker.drain; + return; + } + yield* SubscriptionRef.changes(completedJobs).pipe( + Stream.filter( + () => !scheduledJobs.has(`archive:${threadId}`) && !scheduledJobs.has(`delete:${threadId}`), + ), + Stream.runHead, + ); }); return { diff --git a/apps/server/src/orchestration/Services/ThreadDeletionReactor.ts b/apps/server/src/orchestration/Services/ThreadDeletionReactor.ts index cdbb70919a8e..cd690a0ac382 100644 --- a/apps/server/src/orchestration/Services/ThreadDeletionReactor.ts +++ b/apps/server/src/orchestration/Services/ThreadDeletionReactor.ts @@ -1,12 +1,13 @@ /** * ThreadDeletionReactor - Thread deletion cleanup reactor service interface. * - * Owns background workers that react to thread deletion domain events and - * perform best-effort runtime cleanup for provider sessions and terminals. + * Owns background workers that react to thread archive/delete domain events + * and perform runtime cleanup plus durable cold-storage lifecycle work. * * @module ThreadDeletionReactor */ import * as Context from "effect/Context"; +import type { ThreadId } from "@t3tools/contracts"; import type * as Effect from "effect/Effect"; import type * as Scope from "effect/Scope"; @@ -15,7 +16,7 @@ import type * as Scope from "effect/Scope"; */ export interface ThreadDeletionReactorShape { /** - * Start reacting to thread.deleted orchestration domain events. + * Start reacting to thread lifecycle orchestration domain events. * * The returned effect must be run in a scope so all worker fibers can be * finalized on shutdown. @@ -23,12 +24,12 @@ export interface ThreadDeletionReactorShape { readonly start: () => Effect.Effect; /** - * Resolves once every thread.deleted at or before the supplied event - * sequence has been handed to the worker and the worker is empty and idle. - * A successful thread.create sequence is the fence callers use before the - * new incarnation can own runtime resources. + * Waits until the subscriber reaches the supplied event sequence, then + * waits for pending cleanup for the supplied thread, or all cleanup when + * no thread is supplied. Creation fences only the reused thread id so + * unrelated archives cannot delay its new runtime resources. */ - readonly drainThrough: (sequence: number) => Effect.Effect; + readonly drainThrough: (sequence: number, threadId?: ThreadId) => Effect.Effect; } /** diff --git a/apps/server/src/orchestration/ThreadColdStorage.test.ts b/apps/server/src/orchestration/ThreadColdStorage.test.ts new file mode 100644 index 000000000000..de81c51b2ccf --- /dev/null +++ b/apps/server/src/orchestration/ThreadColdStorage.test.ts @@ -0,0 +1,1721 @@ +import { assert, it } from "@effect/vitest"; +import { vi } from "vite-plus/test"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { ThreadId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +import * as ServerConfig from "../config.ts"; +import { + makeSqlitePersistenceLive, + SqlitePersistenceMemory, +} from "../persistence/Layers/Sqlite.ts"; +import * as ThreadColdStorage from "./ThreadColdStorage.ts"; + +const encodeUnknownJsonString = Schema.encodeUnknownSync(Schema.fromJsonString(Schema.Unknown)); + +const insertArchivedThread = Effect.fn("insertArchivedThreadTestFixture")(function* ( + threadId: ThreadId, + title: string, +) { + const sql = yield* SqlClient.SqlClient; + yield* sql` + INSERT INTO projection_threads ( + thread_id, project_id, title, model_selection_json, runtime_mode, + interaction_mode, created_at, updated_at, archived_at + ) VALUES ( + ${threadId}, 'project-1', ${title}, + '{"instanceId":"codex","model":"gpt-5.5","options":[]}', + 'full-access', 'default', '2026-07-01T00:00:00.000Z', + '2026-07-02T00:00:00.000Z', '2026-07-02T00:00:00.000Z' + ) + `; +}); + +const layer = it.layer( + ThreadColdStorage.layer.pipe( + Layer.provideMerge(SqlitePersistenceMemory), + Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "t3-cold-storage-" })), + Layer.provideMerge(NodeServices.layer), + ), +); + +it.effect("reclaims an active thread's restored bundle after storage restarts", () => + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const threadId = ThreadId.make("thread-restart-restored-finalization"); + const persistentLayer = ThreadColdStorage.layer.pipe( + Layer.provideMerge(makeSqlitePersistenceLive(config.dbPath)), + ); + yield* Effect.scoped( + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + yield* insertArchivedThread(threadId, "Restored before restart"); + yield* storage.archiveThread(threadId); + assert.isTrue(yield* storage.restoreTree(threadId)); + yield* sql`UPDATE projection_threads SET archived_at = NULL WHERE thread_id = ${threadId}`; + // The process ends after committing unarchive and before finalization. + }).pipe(Effect.provide(persistentLayer)), + ); + yield* Effect.scoped( + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + assert.include(yield* storage.listPendingArchiveThreadIds, threadId); + yield* storage.archiveThread(threadId, Effect.die("Must not quiesce after restart")); + assert.deepStrictEqual( + yield* sql`SELECT archived_at FROM projection_threads WHERE thread_id = ${threadId}`, + [{ archived_at: null }], + ); + assert.deepStrictEqual( + yield* sql`SELECT thread_id FROM thread_archive_manifests WHERE thread_id = ${threadId}`, + [], + ); + assert.deepStrictEqual( + yield* sql`SELECT thread_id FROM cold_archive.archive_threads WHERE thread_id = ${threadId}`, + [], + ); + }).pipe(Effect.provide(persistentLayer)), + ); + }).pipe( + Effect.provide( + ServerConfig.layerTest(process.cwd(), { prefix: "t3-finalize-restart-" }).pipe( + Layer.provideMerge(NodeServices.layer), + ), + ), + ), +); + +layer("ThreadColdStorage", (it) => { + it.effect("preserves recreated resources while retrying stale bundle cleanup", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const fs = yield* FileSystem.FileSystem; + const config = yield* ServerConfig.ServerConfig; + const path = yield* Path.Path; + const threadId = ThreadId.make("recreated-stale-bundle"); + yield* insertArchivedThread(threadId, "Previous incarnation"); + yield* storage.archiveThread(threadId); + yield* sql`UPDATE projection_threads SET archived_at = NULL WHERE thread_id = ${threadId}`; + yield* sql`INSERT INTO thread_cleanup_queue (thread_id, reason, created_at) VALUES (${threadId}, 'deleted', CURRENT_TIMESTAMP)`; + yield* sql`INSERT INTO projection_thread_messages (message_id, thread_id, role, text, attachments_json, is_streaming, created_at, updated_at) + VALUES ('recreated-live-message', ${threadId}, 'user', 'replacement data', '[]', 0, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`; + const attachment = path.join( + config.attachmentsDir, + `${threadId}-00000000-0000-4000-8000-000000000001.png`, + ); + yield* fs.makeDirectory(config.attachmentsDir, { recursive: true }); + yield* fs.writeFileString(attachment, "replacement file"); + yield* sql`CREATE TEMP TRIGGER fail_stale_bundle BEFORE DELETE ON cold_archive.archive_threads + BEGIN SELECT RAISE(FAIL, 'cold cleanup failed'); END`; + const quiesce = Effect.die("Obsolete deletion must not stop replacement resources"); + yield* Effect.flip(storage.deleteThread(threadId, quiesce)).pipe( + Effect.ensuring(sql.unsafe("DROP TRIGGER temp.fail_stale_bundle").pipe(Effect.orDie)), + ); + assert.include(yield* storage.listPendingDeleteThreadIds, threadId); + yield* storage.deleteThread(threadId, quiesce); + assert.deepStrictEqual( + yield* sql`SELECT deleted_at FROM projection_threads WHERE thread_id = ${threadId}`, + [{ deleted_at: null }], + ); + assert.deepStrictEqual( + yield* sql`SELECT text FROM projection_thread_messages WHERE thread_id = ${threadId}`, + [{ text: "replacement data" }], + ); + assert.strictEqual(yield* fs.readFileString(attachment), "replacement file"); + assert.deepStrictEqual( + yield* sql`SELECT 1 FROM thread_archive_manifests WHERE thread_id = ${threadId}`, + [], + ); + assert.deepStrictEqual( + yield* sql`SELECT 1 FROM cold_archive.archive_threads WHERE thread_id = ${threadId}`, + [], + ); + assert.deepStrictEqual( + yield* sql`SELECT 1 FROM thread_cleanup_queue WHERE thread_id = ${threadId}`, + [], + ); + }), + ); + + it.effect("retains a live archive and ignores obsolete deletion without durable markers", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const threadId = ThreadId.make("recreated-matching-archive"); + yield* insertArchivedThread(threadId, "Current incarnation"); + yield* storage.archiveThread(threadId); + yield* sql`INSERT INTO thread_cleanup_queue (thread_id, reason, created_at) VALUES (${threadId}, 'deleted', CURRENT_TIMESTAMP)`; + const quiesce = Effect.die("Obsolete deletion must not quiesce"); + yield* storage.deleteThread(threadId, quiesce); + assert.deepStrictEqual( + yield* sql`SELECT status FROM thread_archive_manifests WHERE thread_id = ${threadId}`, + [{ status: "cold" }], + ); + assert.deepStrictEqual( + yield* sql`SELECT 1 FROM thread_cleanup_queue WHERE thread_id = ${threadId}`, + [], + ); + const absentId = ThreadId.make("obsolete-no-markers"); + yield* storage.deleteThread(absentId, quiesce); + assert.deepStrictEqual( + yield* sql`SELECT 1 FROM thread_cleanup_queue WHERE thread_id = ${absentId}`, + [], + ); + }), + ); + + it.effect( + "uses indexed event pages and preserves other aggregates through archive and delete", + () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const threadId = ThreadId.make("thread-indexed-archive"); + yield* insertArchivedThread(threadId, "Indexed archive"); + yield* sql` + WITH RECURSIVE versions(n) AS ( + SELECT 1 UNION ALL SELECT n + 1 FROM versions WHERE n < 501 + ) + INSERT INTO orchestration_events ( + event_id, aggregate_kind, stream_id, stream_version, event_type, + occurred_at, actor_kind, payload_json, metadata_json + ) + SELECT 'indexed-thread-' || n, 'thread', ${threadId}, n, 'thread.created', + '2026-07-01T00:00:00.000Z', 'system', '{}', '{}' FROM versions + `; + yield* sql` + INSERT INTO orchestration_events ( + event_id, aggregate_kind, stream_id, stream_version, event_type, + occurred_at, actor_kind, payload_json, metadata_json + ) VALUES ( + 'indexed-project', 'project', ${threadId}, 1, 'project.created', + '2026-07-01T00:00:00.000Z', 'system', '{}', '{}' + ) + `; + const originalEvents = yield* sql` + SELECT * FROM orchestration_events WHERE stream_id = ${threadId} ORDER BY sequence + `; + const eventQueries: Parameters[] = []; + const unsafe = sql.unsafe.bind(sql); + const spy = vi + .spyOn(sql, "unsafe") + .mockImplementation((...args: Parameters) => { + if (args[0].includes("FROM orchestration_events")) eventQueries.push(args); + return unsafe(...args); + }); + yield* Effect.gen(function* () { + yield* storage.archiveThread(threadId); + assert.deepStrictEqual( + yield* sql`SELECT event_id FROM orchestration_events WHERE stream_id = ${threadId}`, + [{ event_id: "indexed-project" }], + ); + assert.isTrue(yield* storage.restoreTree(threadId)); + assert.deepStrictEqual( + yield* sql`SELECT * FROM orchestration_events WHERE stream_id = ${threadId} ORDER BY sequence`, + originalEvents, + ); + yield* sql`UPDATE projection_threads SET deleted_at = CURRENT_TIMESTAMP WHERE thread_id = ${threadId}`; + yield* storage.deleteThread(threadId); + assert.deepStrictEqual( + yield* sql`SELECT event_id FROM orchestration_events WHERE stream_id = ${threadId}`, + [{ event_id: "indexed-project" }], + ); + }).pipe(Effect.ensuring(Effect.sync(() => spy.mockRestore()))); + + // Inspect the production queries, without timing thresholds or a duplicate query fixture. + assert.isAtLeast(eventQueries.filter(([query]) => query.startsWith("SELECT")).length, 3); + assert.lengthOf( + eventQueries.filter(([query]) => query.startsWith("DELETE")), + 2, + ); + for (const [query, params] of eventQueries) { + const plan = yield* sql.unsafe<{ detail: string }>(`EXPLAIN QUERY PLAN ${query}`, params); + assert.isTrue( + plan.some((row) => + /USING INDEX idx_orch_events_stream_sequence \(aggregate_kind=\? AND stream_id=\?/.test( + row.detail, + ), + ), + plan.map((row) => row.detail).join("\n"), + ); + } + yield* sql`DELETE FROM orchestration_events WHERE event_id = 'indexed-project'`; + }), + ); + + for (const operation of ["finishRestoreTree", "rollbackRestoreTree"] as const) { + for (const failureAt of [1, 2]) { + for (const cold of [false, true]) { + it.effect( + `releases ${cold ? "cold" : "hot"} restore claim after ${operation} root lookup ${failureAt} fails`, + () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const suffix = `${operation}-${failureAt}-${cold}`; + const threadId = ThreadId.make(`thread-claim-${suffix}`); + const otherId = ThreadId.make(`thread-other-claim-${suffix}`); + yield* insertArchivedThread(threadId, "Failed claim release"); + yield* insertArchivedThread(otherId, "Unrelated restore claim"); + if (cold) yield* storage.archiveThread(threadId); + assert.isTrue(yield* storage.restoreTree(threadId)); + assert.isTrue(yield* storage.restoreTree(otherId)); + yield* sql`UPDATE projection_threads SET archived_at = NULL WHERE thread_id = ${otherId}`; + if (operation === "finishRestoreTree") { + yield* sql`UPDATE projection_threads SET archived_at = NULL WHERE thread_id = ${threadId}`; + } + const unsafe = sql.unsafe.bind(sql); + let rootLookups = 0; + const spy = vi + .spyOn(sql, "unsafe") + .mockImplementation((query, params) => + unsafe( + query.includes("SELECT COALESCE(") && ++rootLookups === failureAt + ? "SELECT * FROM missing_restore_lookup_fixture" + : query, + params, + ), + ); + yield* Effect.flip(storage[operation](threadId)).pipe( + Effect.ensuring(Effect.sync(() => spy.mockRestore())), + ); + assert.strictEqual(rootLookups, failureAt); + assert.include(yield* storage.listPendingArchiveThreadIds, threadId); + yield* storage.archiveThread( + threadId, + operation === "finishRestoreTree" + ? Effect.die("Must not quiesce active thread") + : Effect.void, + ); + assert.deepStrictEqual( + yield* sql`SELECT status FROM thread_archive_manifests WHERE thread_id = ${threadId}`, + operation === "finishRestoreTree" ? [] : [{ status: "cold" }], + ); + yield* storage.archiveThread(otherId, Effect.die("Must preserve unrelated claim")); + assert.deepStrictEqual( + yield* sql`SELECT status FROM thread_archive_manifests WHERE thread_id = ${otherId}`, + [{ status: "restored" }], + ); + yield* storage.finishRestoreTree(otherId); + }), + ); + } + } + } + + it.effect("commits complete bundle metadata before deleting hot rows", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const threadId = ThreadId.make("thread-durable-bundle-before-delete"); + yield* insertArchivedThread(threadId, "Interrupted archive handoff"); + yield* sql` + INSERT INTO projection_thread_messages ( + message_id, thread_id, turn_id, role, text, attachments_json, + is_streaming, created_at, updated_at + ) VALUES ( + 'message-durable-bundle', ${threadId}, NULL, 'user', 'survive interruption', '[]', + 0, '2026-07-01T00:00:00.000Z', '2026-07-01T00:00:00.000Z' + ) + `; + yield* sql.unsafe(` + CREATE TEMP TRIGGER block_archive_hot_delete + BEFORE DELETE ON projection_thread_messages + BEGIN SELECT RAISE(ABORT, 'interrupted hot-row deletion'); END + `); + yield* Effect.flip(storage.archiveThread(threadId)).pipe( + Effect.ensuring( + sql.unsafe("DROP TRIGGER temp.block_archive_hot_delete").pipe(Effect.orDie), + ), + ); + assert.deepStrictEqual( + yield* sql`SELECT text FROM projection_thread_messages WHERE thread_id = ${threadId}`, + [{ text: "survive interruption" }], + ); + assert.deepStrictEqual( + yield* sql`SELECT thread_id FROM cold_archive.archive_threads WHERE thread_id = ${threadId}`, + [{ thread_id: threadId }], + ); + assert.deepStrictEqual( + yield* sql`SELECT status FROM thread_archive_manifests WHERE thread_id = ${threadId}`, + [{ status: "archiving" }], + ); + yield* storage.archiveThread(threadId); + assert.isTrue(yield* storage.restoreTree(threadId)); + assert.deepStrictEqual( + yield* sql`SELECT text FROM projection_thread_messages WHERE thread_id = ${threadId}`, + [{ text: "survive interruption" }], + ); + yield* sql`UPDATE projection_threads SET deleted_at = CURRENT_TIMESTAMP WHERE thread_id = ${threadId}`; + yield* storage.deleteThread(threadId); + }), + ); + + it.effect("rediscovers failed finalization without quiescing an active thread", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const threadId = ThreadId.make("thread-retry-restored-finalization"); + yield* insertArchivedThread(threadId, "Retry finalization"); + yield* sql` + INSERT INTO projection_thread_messages ( + message_id, thread_id, turn_id, role, text, attachments_json, + is_streaming, created_at, updated_at + ) VALUES ( + 'message-retry-finalization', ${threadId}, NULL, 'user', 'keep active data', '[]', + 0, '2026-07-01T00:00:00.000Z', '2026-07-01T00:00:00.000Z' + ) + `; + yield* storage.archiveThread(threadId); + assert.isTrue(yield* storage.restoreTree(threadId)); + yield* sql`UPDATE projection_threads SET archived_at = NULL WHERE thread_id = ${threadId}`; + const quiesce = Effect.die("Finalization must not stop an active thread"); + yield* storage.archiveThread(threadId, quiesce); + assert.deepStrictEqual( + yield* sql`SELECT status FROM thread_archive_manifests WHERE thread_id = ${threadId}`, + [{ status: "restored" }], + ); + yield* sql.unsafe(` + CREATE TEMP TRIGGER block_bundle_finalization + BEFORE DELETE ON thread_archive_manifests + BEGIN SELECT RAISE(ABORT, 'temporary finalization failure'); END + `); + yield* Effect.flip(storage.finishRestoreTree(threadId)).pipe( + Effect.ensuring( + sql.unsafe("DROP TRIGGER temp.block_bundle_finalization").pipe(Effect.orDie), + ), + ); + assert.include(yield* storage.listPendingArchiveThreadIds, threadId); + assert.deepStrictEqual( + yield* sql`SELECT thread_id FROM cold_archive.archive_thread_chunks WHERE thread_id = ${threadId}`, + [], + ); + assert.deepStrictEqual( + yield* sql`SELECT thread_id FROM cold_archive.archive_threads WHERE thread_id = ${threadId}`, + [], + ); + yield* storage.archiveThread(threadId, quiesce); + assert.deepStrictEqual( + yield* sql`SELECT text FROM projection_thread_messages WHERE thread_id = ${threadId}`, + [{ text: "keep active data" }], + ); + assert.deepStrictEqual( + yield* sql`SELECT thread_id FROM thread_archive_manifests WHERE thread_id = ${threadId}`, + [], + ); + assert.deepStrictEqual( + yield* sql`SELECT thread_id FROM cold_archive.archive_threads WHERE thread_id = ${threadId}`, + [], + ); + assert.deepStrictEqual( + yield* sql`SELECT thread_id FROM cold_archive.archive_thread_chunks WHERE thread_id = ${threadId}`, + [], + ); + }), + ); + + it.effect("keeps active-shell cleanup discoverable until cold deletes have committed", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const threadId = ThreadId.make("thread-active-cleanup-order"); + yield* insertArchivedThread(threadId, "Retry active cleanup"); + yield* storage.archiveThread(threadId); + assert.isTrue(yield* storage.restoreTree(threadId)); + yield* sql`UPDATE projection_threads SET archived_at = NULL WHERE thread_id = ${threadId}`; + yield* sql.unsafe(` + CREATE TEMP TRIGGER fail_cold_delete BEFORE DELETE ON cold_archive.archive_threads + BEGIN SELECT RAISE(ABORT, 'cold cleanup failure'); END + `); + yield* Effect.flip(storage.finishRestoreTree(threadId)).pipe( + Effect.ensuring(sql.unsafe("DROP TRIGGER temp.fail_cold_delete").pipe(Effect.orDie)), + ); + yield* sql.unsafe(` + CREATE TEMP TRIGGER fail_active_manifest BEFORE DELETE ON thread_archive_manifests + BEGIN SELECT RAISE(ABORT, 'manifest cleanup failure'); END + `); + yield* Effect.flip(storage.archiveThread(threadId, Effect.die("Must not quiesce"))).pipe( + Effect.ensuring(sql.unsafe("DROP TRIGGER temp.fail_active_manifest").pipe(Effect.orDie)), + ); + assert.include(yield* storage.listPendingArchiveThreadIds, threadId); + assert.deepStrictEqual( + yield* sql`SELECT thread_id FROM cold_archive.archive_threads WHERE thread_id = ${threadId}`, + [], + ); + yield* storage.archiveThread(threadId, Effect.die("Must not quiesce")); + assert.deepStrictEqual( + yield* sql`SELECT thread_id FROM thread_archive_manifests WHERE thread_id = ${threadId}`, + [], + ); + }), + ); + + it.effect("commits cold cleanup before removing an aborted archive manifest", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const threadId = ThreadId.make("thread-aborted-archive-order"); + yield* insertArchivedThread(threadId, "Archive became stale"); + yield* sql.unsafe(` + CREATE TEMP TRIGGER activate_before_archive_delete + AFTER INSERT ON cold_archive.archive_threads + BEGIN UPDATE projection_threads SET archived_at = NULL WHERE thread_id = NEW.thread_id; END + `); + yield* sql.unsafe(` + CREATE TEMP TRIGGER fail_aborted_manifest BEFORE DELETE ON thread_archive_manifests + BEGIN SELECT RAISE(ABORT, 'manifest cleanup failure'); END + `); + yield* Effect.flip(storage.archiveThread(threadId)).pipe( + Effect.ensuring( + Effect.gen(function* () { + yield* sql.unsafe("DROP TRIGGER temp.activate_before_archive_delete"); + yield* sql.unsafe("DROP TRIGGER temp.fail_aborted_manifest"); + }).pipe(Effect.orDie), + ), + ); + assert.include(yield* storage.listPendingArchiveThreadIds, threadId); + assert.deepStrictEqual( + yield* sql`SELECT thread_id FROM cold_archive.archive_threads WHERE thread_id = ${threadId}`, + [], + ); + assert.deepStrictEqual( + yield* sql`SELECT archived_at FROM projection_threads WHERE thread_id = ${threadId}`, + [{ archived_at: null }], + ); + yield* storage.archiveThread(threadId, Effect.die("Must not quiesce")); + assert.deepStrictEqual( + yield* sql`SELECT thread_id FROM thread_archive_manifests WHERE thread_id = ${threadId}`, + [], + ); + }), + ); + + it.effect("waits for pending archive and delete work before legacy compaction", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const archiveThreadId = ThreadId.make("thread-compact-pending-archive"); + const deleteThreadId = ThreadId.make("thread-compact-pending-delete"); + + yield* insertArchivedThread(archiveThreadId, "Pending archive before compaction"); + yield* insertArchivedThread(deleteThreadId, "Pending delete before compaction"); + yield* sql` + UPDATE projection_threads + SET deleted_at = '2026-07-03T00:00:00.000Z' + WHERE thread_id = ${deleteThreadId} + `; + + yield* storage.compactLegacyStorage; + const deferredMaintenance = yield* sql<{ readonly status: string }>` + SELECT status FROM thread_storage_maintenance + WHERE task = 'compact-legacy-thread-storage' + `; + assert.deepStrictEqual(deferredMaintenance, [{ status: "pending" }]); + + yield* storage.archiveThread(archiveThreadId); + yield* storage.deleteThread(deleteThreadId); + yield* storage.compactLegacyStorage; + const completedMaintenance = yield* sql<{ readonly status: string }>` + SELECT status FROM thread_storage_maintenance + WHERE task = 'compact-legacy-thread-storage' + `; + assert.deepStrictEqual(completedMaintenance, [{ status: "complete" }]); + }), + ); + + it.effect("retains archived PR relations until permanent deletion", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const threadId = ThreadId.make("thread-pr-relations"); + yield* insertArchivedThread(threadId, "PR-linked thread"); + yield* sql` + INSERT INTO projection_thread_pull_requests ( + thread_id, host, repository, number, url, source, linked_at + ) VALUES ( + ${threadId}, 'github.com', 'example/repo', 12, + 'https://github.com/example/repo/pull/12', 'manual', '2026-07-01T00:00:00.000Z' + ) + `; + const links = sql`SELECT number FROM projection_thread_pull_requests WHERE thread_id = ${threadId}`; + yield* storage.archiveThread(threadId); + assert.deepStrictEqual(yield* links, [{ number: 12 }]); + assert.isTrue(yield* storage.restoreTree(threadId)); + assert.deepStrictEqual(yield* links, [{ number: 12 }]); + yield* storage.finishRestoreTree(threadId); + yield* sql`UPDATE projection_threads SET deleted_at = CURRENT_TIMESTAMP WHERE thread_id = ${threadId}`; + yield* storage.deleteThread(threadId); + assert.deepStrictEqual(yield* links, []); + }), + ); + + it.effect("normalizes typed quiesce failures at the archive boundary", () => + Effect.gen(function* () { + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const threadId = ThreadId.make("thread-quiesce-failure"); + const quiesceFailure = { _tag: "QuiesceFailure" } as const; + + yield* insertArchivedThread(threadId, "Quiesce failure thread"); + + const archiveFailure = yield* Effect.flip( + storage.archiveThread(threadId, Effect.fail(quiesceFailure)), + ); + + assert.strictEqual(archiveFailure.operation, "archive"); + assert.strictEqual(archiveFailure.threadId, threadId); + assert.strictEqual(archiveFailure.cause, quiesceFailure); + + const normalizedFailure = new ThreadColdStorage.ThreadColdStorageError({ + operation: "archive", + threadId, + cause: quiesceFailure, + }); + const repeatedFailure = yield* Effect.flip( + storage.archiveThread(threadId, Effect.fail(normalizedFailure)), + ); + assert.strictEqual(repeatedFailure, normalizedFailure); + + const differentlyAttributedFailure = new ThreadColdStorage.ThreadColdStorageError({ + operation: "restore", + threadId, + cause: quiesceFailure, + }); + const attributedFailure = yield* Effect.flip( + storage.archiveThread(threadId, Effect.fail(differentlyAttributedFailure)), + ); + assert.strictEqual(attributedFailure.operation, "archive"); + assert.strictEqual(attributedFailure.threadId, threadId); + assert.strictEqual(attributedFailure.cause, differentlyAttributedFailure); + }), + ); + + it.effect("archives post-quiescence state while a running thread awaits user input", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const threadId = ThreadId.make("thread-running-pending-user-input"); + + yield* insertArchivedThread(threadId, "Running thread awaiting user input"); + yield* sql` + UPDATE projection_threads + SET pending_user_input_count = 1 + WHERE thread_id = ${threadId} + `; + yield* sql` + INSERT INTO projection_thread_sessions ( + thread_id, status, provider_name, provider_session_id, + provider_thread_id, active_turn_id, last_error, updated_at, + runtime_mode, provider_instance_id + ) VALUES ( + ${threadId}, 'running', 'codex', 'session-pending-user-input', + 'provider-thread-pending-user-input', 'turn-pending-user-input', NULL, + '2026-07-02T00:00:00.000Z', 'full-access', 'codex' + ) + `; + yield* sql` + INSERT INTO projection_thread_activities ( + activity_id, thread_id, turn_id, tone, kind, summary, + payload_json, created_at, sequence + ) VALUES ( + 'activity-user-input-requested', ${threadId}, 'turn-pending-user-input', + 'info', 'user-input.requested', 'Waiting for user input', + '{"requestId":"request-pending-user-input"}', + '2026-07-02T00:00:00.000Z', 1 + ) + `; + + yield* storage.archiveThread( + threadId, + Effect.gen(function* () { + yield* sql` + UPDATE projection_thread_sessions + SET status = 'stopped', active_turn_id = NULL, + updated_at = '2026-07-02T00:01:00.000Z' + WHERE thread_id = ${threadId} + `; + yield* sql` + UPDATE projection_threads + SET pending_user_input_count = 0 + WHERE thread_id = ${threadId} + `; + yield* sql` + INSERT INTO projection_thread_activities ( + activity_id, thread_id, turn_id, tone, kind, summary, + payload_json, created_at, sequence + ) VALUES ( + 'activity-user-input-resolved', ${threadId}, 'turn-pending-user-input', + 'info', 'user-input.resolved', 'User input request interrupted', + '{"requestId":"request-pending-user-input"}', + '2026-07-02T00:01:00.000Z', 2 + ) + `; + }), + ); + + const coldActivities = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count + FROM projection_thread_activities + WHERE thread_id = ${threadId} + `; + const coldSessions = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count + FROM projection_thread_sessions + WHERE thread_id = ${threadId} + `; + const postArchiveShells = yield* sql<{ readonly pendingUserInputCount: number }>` + SELECT pending_user_input_count AS "pendingUserInputCount" + FROM projection_threads + WHERE thread_id = ${threadId} + `; + assert.deepStrictEqual(coldActivities, [{ count: 0 }]); + assert.deepStrictEqual(coldSessions, [{ count: 0 }]); + // The lightweight shell deliberately stays hot while child projections + // move cold, so verify its post-quiescence value before any restore. + assert.deepStrictEqual(postArchiveShells, [{ pendingUserInputCount: 0 }]); + + assert.isTrue(yield* storage.restoreTree(threadId)); + const restoredActivities = yield* sql<{ + readonly kind: string; + readonly payloadJson: string; + readonly createdAt: string; + readonly sequence: number; + }>` + SELECT kind, payload_json AS "payloadJson", created_at AS "createdAt", sequence + FROM projection_thread_activities + WHERE thread_id = ${threadId} + ORDER BY sequence ASC + `; + const restoredSessions = yield* sql<{ + readonly status: string; + readonly activeTurnId: string | null; + readonly updatedAt: string; + }>` + SELECT status, active_turn_id AS "activeTurnId", updated_at AS "updatedAt" + FROM projection_thread_sessions + WHERE thread_id = ${threadId} + `; + assert.deepStrictEqual(restoredActivities, [ + { + kind: "user-input.requested", + payloadJson: '{"requestId":"request-pending-user-input"}', + createdAt: "2026-07-02T00:00:00.000Z", + sequence: 1, + }, + { + kind: "user-input.resolved", + payloadJson: '{"requestId":"request-pending-user-input"}', + createdAt: "2026-07-02T00:01:00.000Z", + sequence: 2, + }, + ]); + assert.deepStrictEqual(restoredSessions, [ + { + status: "stopped", + activeTurnId: null, + updatedAt: "2026-07-02T00:01:00.000Z", + }, + ]); + }), + ); + + it.effect("discovers archived shells before a lifecycle manifest exists", () => + Effect.gen(function* () { + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const threadId = ThreadId.make("thread-archive-queue-fallback"); + + yield* insertArchivedThread(threadId, "Archive queue fallback thread"); + + assert.deepInclude(yield* storage.listPendingArchiveThreadIds, threadId); + }), + ); + + it.effect("discovers deleted shells before a cleanup queue entry exists", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const threadId = ThreadId.make("thread-delete-queue-fallback"); + + yield* insertArchivedThread(threadId, "Delete queue fallback thread"); + yield* sql` + UPDATE projection_threads + SET deleted_at = '2026-07-03T00:00:00.000Z' + WHERE thread_id = ${threadId} + `; + + assert.deepInclude(yield* storage.listPendingDeleteThreadIds, threadId); + }), + ); + + it.effect("reserves hot archived rows while an unarchive command is pending", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const threadId = ThreadId.make("thread-unarchive-hot-reservation"); + + yield* insertArchivedThread(threadId, "Pending hot unarchive"); + yield* sql` + INSERT INTO projection_thread_messages ( + message_id, thread_id, turn_id, role, text, attachments_json, + is_streaming, created_at, updated_at + ) VALUES ( + 'message-unarchive-hot-reservation', ${threadId}, NULL, 'user', + 'keep hot until unarchive commits', '[]', 0, + '2026-07-01T00:00:00.000Z', '2026-07-01T00:00:00.000Z' + ) + `; + yield* sql` + INSERT INTO thread_archive_manifests ( + thread_id, root_thread_id, status, archive_version, + archived_at, updated_at, error + ) VALUES ( + ${threadId}, ${threadId}, 'pending', 0, + '2026-06-01T00:00:00.000Z', CURRENT_TIMESTAMP, NULL + ) + `; + + assert.isTrue(yield* storage.restoreTree(threadId)); + assert.deepInclude(yield* storage.listPendingArchiveThreadIds, threadId); + let quiesceCalls = 0; + yield* storage.archiveThread( + threadId, + Effect.sync(() => { + quiesceCalls += 1; + }), + ); + + const messages = yield* sql<{ readonly text: string }>` + SELECT text FROM projection_thread_messages WHERE thread_id = ${threadId} + `; + const manifest = yield* sql<{ + readonly rootThreadId: string; + readonly status: string; + readonly archiveVersion: number; + readonly archivedAt: string; + }>` + SELECT + root_thread_id AS "rootThreadId", + status, + archive_version AS "archiveVersion", + archived_at AS "archivedAt" + FROM thread_archive_manifests + WHERE thread_id = ${threadId} + `; + assert.strictEqual(quiesceCalls, 0); + assert.deepStrictEqual(messages, [{ text: "keep hot until unarchive commits" }]); + assert.deepStrictEqual(manifest, [ + { + rootThreadId: threadId, + status: "restored", + archiveVersion: 1, + archivedAt: "2026-07-02T00:00:00.000Z", + }, + ]); + + yield* storage.finishRestoreTree(threadId); + const remainingManifest = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count FROM thread_archive_manifests WHERE thread_id = ${threadId} + `; + assert.deepStrictEqual(remainingManifest, [{ count: 0 }]); + }), + ); + + it.effect("reports unknown archive chunk kinds with structured validation detail", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const threadId = ThreadId.make("thread-unknown-archive-chunk-kind"); + + yield* insertArchivedThread(threadId, "Unknown archive chunk kind"); + yield* sql` + INSERT INTO projection_thread_messages ( + message_id, thread_id, turn_id, role, text, attachments_json, + is_streaming, created_at, updated_at + ) VALUES ( + 'message-unknown-archive-chunk-kind', ${threadId}, NULL, 'user', + 'corrupt this chunk kind', '[]', 0, + '2026-07-01T00:00:00.000Z', '2026-07-01T00:00:00.000Z' + ) + `; + + yield* storage.archiveThread(threadId); + yield* sql` + UPDATE cold_archive.archive_thread_chunks + SET kind = 'future:unsupported' + WHERE thread_id = ${threadId} + `; + + const failure = yield* Effect.flip(storage.restoreTree(threadId)); + assert.deepInclude(failure.cause, { + _tag: "ArchiveChunkKindValidationError", + chunkKind: "future:unsupported", + }); + }), + ); + + it.effect("re-archives an orphaned restore reservation discovered after restart", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const threadId = ThreadId.make("thread-orphaned-restore-reservation"); + + yield* insertArchivedThread(threadId, "Orphaned restore reservation"); + yield* sql` + INSERT INTO projection_thread_messages ( + message_id, thread_id, turn_id, role, text, attachments_json, + is_streaming, created_at, updated_at + ) VALUES ( + 'message-orphaned-restore-reservation', ${threadId}, NULL, 'user', + 'move cold during startup recovery', '[]', 0, + '2026-07-01T00:00:00.000Z', '2026-07-01T00:00:00.000Z' + ) + `; + yield* sql` + INSERT INTO thread_archive_manifests ( + thread_id, root_thread_id, status, archive_version, + archived_at, updated_at, error + ) VALUES ( + ${threadId}, ${threadId}, 'restored', 1, + '2026-07-02T00:00:00.000Z', CURRENT_TIMESTAMP, NULL + ) + `; + + assert.deepInclude(yield* storage.listPendingArchiveThreadIds, threadId); + yield* storage.archiveThread(threadId); + + const messages = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count + FROM projection_thread_messages + WHERE thread_id = ${threadId} + `; + const manifest = yield* sql<{ readonly status: string }>` + SELECT status + FROM thread_archive_manifests + WHERE thread_id = ${threadId} + `; + assert.deepStrictEqual(messages, [{ count: 0 }]); + assert.deepStrictEqual(manifest, [{ status: "cold" }]); + }), + ); + + it.effect("re-archives hot rows after restored-bundle finalization fails", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const threadId = ThreadId.make("thread-rearchive-stale-restored"); + + yield* insertArchivedThread(threadId, "Re-archive stale restore"); + yield* sql` + INSERT INTO projection_thread_messages ( + message_id, thread_id, turn_id, role, text, attachments_json, + is_streaming, created_at, updated_at + ) VALUES ( + 'message-rearchive-stale-restored', ${threadId}, NULL, 'user', + 'move cold after the next archive', '[]', 0, + '2026-07-01T00:00:00.000Z', '2026-07-01T00:00:00.000Z' + ) + `; + + assert.isTrue(yield* storage.restoreTree(threadId)); + yield* sql` + UPDATE projection_threads + SET archived_at = NULL, updated_at = '2026-07-03T00:00:00.000Z' + WHERE thread_id = ${threadId} + `; + // Simulate finishRestoreTree failing after the unarchive transaction, + // then the user archiving the same thread again. + yield* sql` + UPDATE projection_threads + SET archived_at = '2026-07-04T00:00:00.000Z', + updated_at = '2026-07-04T00:00:00.000Z' + WHERE thread_id = ${threadId} + `; + + yield* storage.archiveThread(threadId); + + const messages = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count + FROM projection_thread_messages + WHERE thread_id = ${threadId} + `; + const manifest = yield* sql<{ + readonly status: string; + readonly archivedAt: string; + }>` + SELECT status, archived_at AS "archivedAt" + FROM thread_archive_manifests + WHERE thread_id = ${threadId} + `; + assert.deepStrictEqual(messages, [{ count: 0 }]); + assert.deepStrictEqual(manifest, [ + { status: "cold", archivedAt: "2026-07-04T00:00:00.000Z" }, + ]); + }), + ); + + it.effect("does not replay a restored bundle over active thread data", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const threadId = ThreadId.make("thread-stale-restored-bundle"); + const attachmentName = + "thread-stale-restored-bundle-00000000-0000-4000-8000-000000000001.txt"; + const attachmentPath = path.join(config.attachmentsDir, attachmentName); + + yield* insertArchivedThread(threadId, "Stale restored bundle"); + yield* sql.unsafe( + `INSERT INTO projection_thread_messages ( + message_id, thread_id, turn_id, role, text, attachments_json, + is_streaming, created_at, updated_at + ) VALUES (?, ?, NULL, 'user', 'cold value', ?, 0, ?, ?)`, + [ + "message-stale-restored-bundle", + threadId, + encodeUnknownJsonString([ + { + type: "text", + id: attachmentName.slice(0, -4), + name: "content.txt", + mimeType: "text/plain", + }, + ]), + "2026-07-01T00:00:00.000Z", + "2026-07-01T00:00:00.000Z", + ], + ); + yield* fs.writeFileString(attachmentPath, "cold attachment"); + + yield* storage.archiveThread(threadId); + assert.isTrue(yield* storage.restoreTree(threadId)); + + yield* sql` + UPDATE projection_threads SET archived_at = NULL WHERE thread_id = ${threadId} + `; + yield* sql` + UPDATE projection_thread_messages + SET text = 'active value' + WHERE thread_id = ${threadId} + `; + yield* fs.writeFileString(attachmentPath, "active attachment"); + + assert.isTrue(yield* storage.restoreTree(threadId)); + + const messages = yield* sql<{ readonly text: string }>` + SELECT text FROM projection_thread_messages WHERE thread_id = ${threadId} + `; + assert.deepStrictEqual(messages, [{ text: "active value" }]); + assert.strictEqual(yield* fs.readFileString(attachmentPath), "active attachment"); + }), + ); + + it.effect("keeps command receipts hot while conversation data is cold", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const threadId = ThreadId.make("thread-hot-command-receipt"); + + yield* insertArchivedThread(threadId, "Hot command receipt"); + yield* sql` + INSERT INTO orchestration_command_receipts ( + command_id, aggregate_kind, aggregate_id, accepted_at, + result_sequence, status, error + ) VALUES ( + 'command-hot-receipt', 'thread', ${threadId}, + '2026-07-02T00:00:00.000Z', 42, 'accepted', NULL + ) + `; + + yield* storage.archiveThread(threadId); + + const hotReceipts = yield* sql<{ readonly commandId: string }>` + SELECT command_id AS "commandId" + FROM orchestration_command_receipts + WHERE aggregate_id = ${threadId} + `; + const receiptChunks = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count + FROM cold_archive.archive_thread_chunks + WHERE thread_id = ${threadId} + AND kind = 'table:orchestration_command_receipts' + `; + assert.deepStrictEqual(hotReceipts, [{ commandId: "command-hot-receipt" }]); + assert.deepStrictEqual(receiptChunks, [{ count: 0 }]); + + yield* sql`UPDATE projection_threads SET deleted_at = CURRENT_TIMESTAMP WHERE thread_id = ${threadId}`; + yield* storage.deleteThread(threadId); + const deletedReceipts = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count + FROM orchestration_command_receipts + WHERE aggregate_id = ${threadId} + `; + assert.deepStrictEqual(deletedReceipts, [{ count: 0 }]); + }), + ); + + it.effect("compresses conversation data, destroys logs, restores content, and hard-deletes", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const threadId = ThreadId.make("thread-cold"); + const attachmentName = "thread-cold-00000000-0000-4000-8000-000000000001.webp"; + + yield* sql` + INSERT INTO projection_threads ( + thread_id, project_id, title, model_selection_json, runtime_mode, + interaction_mode, created_at, updated_at, archived_at, pinned_at + ) VALUES ( + ${threadId}, 'project-1', 'Cold thread', + '{"instanceId":"codex","model":"gpt-5.5","options":[]}', + 'full-access', 'default', '2026-07-01T00:00:00.000Z', + '2026-07-02T00:00:00.000Z', '2026-07-02T00:00:00.000Z', + '2026-07-01T12:00:00.000Z' + ) + `; + yield* sql` + INSERT INTO projection_thread_messages ( + message_id, thread_id, turn_id, role, text, attachments_json, + is_streaming, created_at, updated_at + ) VALUES ( + 'message-1', ${threadId}, NULL, 'user', 'keep this conversation', + '[{"type":"image","id":"thread-cold-00000000-0000-4000-8000-000000000001","name":"image.webp","mimeType":"image/webp"}]', + 0, '2026-07-01T00:00:00.000Z', '2026-07-01T00:00:00.000Z' + ) + `; + yield* sql` + INSERT INTO orchestration_events ( + event_id, aggregate_kind, stream_id, stream_version, event_type, + occurred_at, command_id, causation_event_id, correlation_id, + actor_kind, payload_json, metadata_json + ) VALUES ( + 'event-1', 'thread', ${threadId}, 1, 'thread.created', + '2026-07-01T00:00:00.000Z', 'command-1', NULL, 'command-1', + 'system', '{}', '{}' + ) + `; + + const attachmentPath = path.join(config.attachmentsDir, attachmentName); + const providerLogPath = path.join(config.providerLogsDir, "events.thread-cold.log"); + const rotatedProviderLogPath = `${providerLogPath}.1`; + const similarlyPrefixedProviderLogPath = path.join( + config.providerLogsDir, + "events.thread-cold-extra.log", + ); + yield* fs.writeFileString(attachmentPath, "image bytes"); + yield* fs.writeFileString(providerLogPath, "diagnostic"); + yield* fs.writeFileString(rotatedProviderLogPath, "old diagnostic"); + yield* fs.writeFileString(similarlyPrefixedProviderLogPath, "other thread diagnostic"); + + yield* storage.archiveThread(threadId); + + const archivedMessageCount = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count FROM projection_thread_messages WHERE thread_id = ${threadId} + `; + const archivedEventCount = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count FROM orchestration_events WHERE stream_id = ${threadId} + `; + const shell = yield* sql<{ readonly pinnedAt: string | null }>` + SELECT pinned_at AS "pinnedAt" FROM projection_threads WHERE thread_id = ${threadId} + `; + const manifest = yield* sql<{ readonly status: string; readonly compressedBytes: number }>` + SELECT status, compressed_bytes AS "compressedBytes" + FROM thread_archive_manifests WHERE thread_id = ${threadId} + `; + assert.strictEqual(archivedMessageCount[0]?.count, 0); + assert.strictEqual(archivedEventCount[0]?.count, 0); + assert.deepStrictEqual(shell, [{ pinnedAt: "2026-07-01T12:00:00.000Z" }]); + assert.strictEqual(manifest[0]?.status, "cold"); + assert.isAbove(manifest[0]?.compressedBytes ?? 0, 0); + assert.isFalse(yield* fs.exists(attachmentPath)); + assert.isFalse(yield* fs.exists(providerLogPath)); + assert.isFalse(yield* fs.exists(rotatedProviderLogPath)); + assert.isTrue(yield* fs.exists(similarlyPrefixedProviderLogPath)); + + assert.isTrue(yield* storage.restoreTree(threadId)); + const restoredMessages = yield* sql<{ readonly text: string }>` + SELECT text FROM projection_thread_messages WHERE thread_id = ${threadId} + `; + const restoredShell = yield* sql<{ readonly pinnedAt: string | null }>` + SELECT pinned_at AS "pinnedAt" FROM projection_threads WHERE thread_id = ${threadId} + `; + assert.deepStrictEqual(restoredMessages, [{ text: "keep this conversation" }]); + assert.deepStrictEqual(restoredShell, [{ pinnedAt: "2026-07-01T12:00:00.000Z" }]); + assert.strictEqual(yield* fs.readFileString(attachmentPath), "image bytes"); + assert.isFalse(yield* fs.exists(providerLogPath)); + + // A queued archive job can run after restore but before the unarchive + // command commits. It must not undo a restore owned by that command. + yield* storage.archiveThread(threadId); + const stillRestoredMessages = yield* sql<{ readonly text: string }>` + SELECT text FROM projection_thread_messages WHERE thread_id = ${threadId} + `; + const stillRestoredManifest = yield* sql<{ readonly status: string }>` + SELECT status FROM thread_archive_manifests WHERE thread_id = ${threadId} + `; + assert.deepStrictEqual(stillRestoredMessages, [{ text: "keep this conversation" }]); + assert.deepStrictEqual(stillRestoredManifest, [{ status: "restored" }]); + + yield* storage.rollbackRestoreTree(threadId); + const rolledBackMessages = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count FROM projection_thread_messages WHERE thread_id = ${threadId} + `; + const rolledBackManifest = yield* sql<{ readonly status: string }>` + SELECT status FROM thread_archive_manifests WHERE thread_id = ${threadId} + `; + assert.deepStrictEqual(rolledBackMessages, [{ count: 0 }]); + assert.deepStrictEqual(rolledBackManifest, [{ status: "cold" }]); + + assert.isTrue(yield* storage.restoreTree(threadId)); + assert.strictEqual(yield* fs.readFileString(attachmentPath), "image bytes"); + + yield* storage.finishRestoreTree(threadId); + const remainingManifestCount = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count FROM thread_archive_manifests WHERE thread_id = ${threadId} + `; + assert.strictEqual(remainingManifestCount[0]?.count, 0); + + yield* sql`UPDATE projection_threads SET deleted_at = CURRENT_TIMESTAMP WHERE thread_id = ${threadId}`; + yield* storage.deleteThread(threadId); + const remainingShellCount = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count FROM projection_threads WHERE thread_id = ${threadId} + `; + const remainingMessageCount = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count FROM projection_thread_messages WHERE thread_id = ${threadId} + `; + assert.strictEqual(remainingShellCount[0]?.count, 0); + assert.strictEqual(remainingMessageCount[0]?.count, 0); + assert.isFalse(yield* fs.exists(attachmentPath)); + + yield* storage.compactLegacyStorage; + const maintenance = yield* sql<{ readonly status: string }>` + SELECT status FROM thread_storage_maintenance + WHERE task = 'compact-legacy-thread-storage' + `; + assert.deepStrictEqual(maintenance, [{ status: "complete" }]); + }), + ); + + it.effect( + "retains cold data when an unsafe attachment entry blocks restore, then retries after repair", + () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const threadId = ThreadId.make("thread-traversal"); + const attachmentName = "thread-traversal-00000000-0000-4000-8000-000000000001.png"; + + yield* insertArchivedThread(threadId, "Traversal thread"); + yield* sql.unsafe( + `INSERT INTO projection_thread_messages ( + message_id, thread_id, turn_id, role, text, attachments_json, + is_streaming, created_at, updated_at + ) VALUES (?, ?, NULL, 'user', 'validate attachment name', ?, 0, ?, ?)`, + [ + "message-traversal", + threadId, + encodeUnknownJsonString([ + { + type: "image", + id: attachmentName.slice(0, -4), + name: "image.png", + mimeType: "image/png", + }, + ]), + "2026-07-01T00:00:00.000Z", + "2026-07-01T00:00:00.000Z", + ], + ); + yield* fs.writeFileString(path.join(config.attachmentsDir, attachmentName), "image bytes"); + yield* storage.archiveThread(threadId); + const escapedPath = path.join(config.attachmentsDir, "..", "thread-traversal-escape"); + yield* sql` + UPDATE cold_archive.archive_thread_chunks + SET kind = 'attachment:../thread-traversal-escape' + WHERE thread_id = ${threadId} AND kind LIKE 'attachment:%' + `; + + const failure = yield* Effect.flip(storage.restoreTree(threadId)); + assert.strictEqual(failure.operation, "restore"); + const manifest = yield* sql<{ readonly status: string }>` + SELECT status FROM thread_archive_manifests WHERE thread_id = ${threadId} + `; + assert.deepStrictEqual(manifest, [{ status: "cold" }]); + assert.deepStrictEqual( + yield* sql`SELECT message_id FROM projection_thread_messages WHERE thread_id = ${threadId}`, + [], + ); + assert.isAbove( + (yield* sql<{ + count: number; + }>`SELECT COUNT(*) AS count FROM cold_archive.archive_thread_chunks WHERE thread_id = ${threadId}`)[0] + ?.count ?? 0, + 0, + ); + assert.isFalse(yield* fs.exists(escapedPath)); + yield* sql` + UPDATE cold_archive.archive_thread_chunks + SET kind = ${`attachment:${attachmentName}`} + WHERE thread_id = ${threadId} AND kind LIKE 'attachment:%' + `; + assert.isTrue(yield* storage.restoreTree(threadId)); + assert.strictEqual( + yield* fs.readFileString(path.join(config.attachmentsDir, attachmentName)), + "image bytes", + ); + assert.deepStrictEqual( + yield* sql`SELECT text FROM projection_thread_messages WHERE thread_id = ${threadId}`, + [{ text: "validate attachment name" }], + ); + }), + ); + + it.effect("keeps cold SQL data authoritative when attachment restore fails", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const threadId = ThreadId.make("thread-attachment-restore-failure"); + const attachmentName = + "thread-attachment-restore-failure-00000000-0000-4000-8000-000000000001.png"; + + yield* insertArchivedThread(threadId, "Attachment restore failure"); + yield* sql` + INSERT INTO projection_thread_messages ( + message_id, thread_id, turn_id, role, text, attachments_json, + is_streaming, created_at, updated_at + ) VALUES ( + 'message-attachment-restore-failure', ${threadId}, NULL, 'user', 'keep me cold', + '[{"type":"image","id":"thread-attachment-restore-failure-00000000-0000-4000-8000-000000000001","name":"image.png","mimeType":"image/png"}]', + 0, '2026-07-01T00:00:00.000Z', '2026-07-01T00:00:00.000Z' + ) + `; + yield* fs.writeFileString(path.join(config.attachmentsDir, attachmentName), "image bytes"); + yield* storage.archiveThread(threadId); + + const blockedTarget = path.join(config.attachmentsDir, "blocked-restore.bin"); + yield* fs.makeDirectory(blockedTarget); + yield* fs.writeFileString(path.join(blockedTarget, "keep"), "prevent replacement"); + yield* sql` + UPDATE cold_archive.archive_thread_chunks + SET kind = 'attachment:blocked-restore.bin' + WHERE thread_id = ${threadId} AND kind LIKE 'attachment:%' + `; + + const failure = yield* Effect.flip(storage.restoreTree(threadId)); + assert.strictEqual(failure.operation, "restore"); + const messages = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count FROM projection_thread_messages WHERE thread_id = ${threadId} + `; + const manifest = yield* sql<{ readonly status: string }>` + SELECT status FROM thread_archive_manifests WHERE thread_id = ${threadId} + `; + const chunks = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count FROM cold_archive.archive_thread_chunks WHERE thread_id = ${threadId} + `; + assert.deepStrictEqual(messages, [{ count: 0 }]); + assert.deepStrictEqual(manifest, [{ status: "cold" }]); + assert.isAbove(chunks[0]?.count ?? 0, 0); + }), + ); + + it.effect("round-trips binary SQL values", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const threadId = ThreadId.make("thread-binary"); + const diffBytes = new Uint8Array([0, 1, 2, 127, 128, 255]); + + yield* insertArchivedThread(threadId, "Binary thread"); + yield* sql.unsafe( + `INSERT INTO checkpoint_diff_blobs + (thread_id, from_turn_count, to_turn_count, diff, created_at) + VALUES (?, 0, 1, ?, '2026-07-01T00:00:00.000Z')`, + [threadId, diffBytes], + ); + + yield* storage.archiveThread(threadId); + assert.isTrue(yield* storage.restoreTree(threadId)); + const restored = (yield* sql.unsafe( + `SELECT diff FROM checkpoint_diff_blobs WHERE thread_id = ?`, + [threadId], + )) as ReadonlyArray<{ readonly diff: Uint8Array }>; + assert.deepStrictEqual(restored[0]?.diff, diffBytes); + }), + ); + + it.effect("retries archive cleanup without rebuilding deleted hot data", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const threadId = ThreadId.make("thread-cleanup-retry"); + const providerLogPath = path.join(config.providerLogsDir, "events.thread-cleanup-retry.log"); + + yield* insertArchivedThread(threadId, "Cleanup retry thread"); + yield* sql` + INSERT INTO projection_thread_messages ( + message_id, thread_id, turn_id, role, text, attachments_json, + is_streaming, created_at, updated_at + ) VALUES ( + 'message-cleanup-retry', ${threadId}, NULL, 'user', 'preserve across cleanup retry', + '[]', 0, '2026-07-01T00:00:00.000Z', '2026-07-01T00:00:00.000Z' + ) + `; + yield* fs.makeDirectory(providerLogPath); + yield* fs.writeFileString(path.join(providerLogPath, "keep"), "force cleanup failure"); + + const archiveFailure = yield* Effect.flip(storage.archiveThread(threadId)); + assert.strictEqual(archiveFailure.operation, "archive"); + const pendingManifest = yield* sql<{ readonly status: string }>` + SELECT status FROM thread_archive_manifests WHERE thread_id = ${threadId} + `; + assert.deepStrictEqual(pendingManifest, [{ status: "cleanup_pending" }]); + assert.deepInclude(yield* storage.listPendingArchiveThreadIds, threadId); + + yield* fs.remove(providerLogPath, { recursive: true }); + yield* storage.archiveThread(threadId); + const coldManifest = yield* sql<{ readonly status: string }>` + SELECT status FROM thread_archive_manifests WHERE thread_id = ${threadId} + `; + assert.deepStrictEqual(coldManifest, [{ status: "cold" }]); + + assert.isTrue(yield* storage.restoreTree(threadId)); + const restoredMessages = yield* sql<{ readonly text: string }>` + SELECT text FROM projection_thread_messages WHERE thread_id = ${threadId} + `; + assert.deepStrictEqual(restoredMessages, [{ text: "preserve across cleanup retry" }]); + }), + ); + + it.effect("finishes cleanup-pending archives after their shell is removed", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const threadId = ThreadId.make("thread-cleanup-missing-shell"); + const providerLogPath = path.join( + config.providerLogsDir, + "events.thread-cleanup-missing-shell.log", + ); + + yield* insertArchivedThread(threadId, "Cleanup missing shell thread"); + yield* fs.makeDirectory(providerLogPath); + yield* fs.writeFileString(path.join(providerLogPath, "keep"), "force cleanup failure"); + + const archiveFailure = yield* Effect.flip(storage.archiveThread(threadId)); + assert.strictEqual(archiveFailure.operation, "archive"); + yield* sql`DELETE FROM projection_threads WHERE thread_id = ${threadId}`; + yield* fs.remove(providerLogPath, { recursive: true }); + + yield* storage.archiveThread(threadId); + const manifest = yield* sql<{ readonly status: string }>` + SELECT status FROM thread_archive_manifests WHERE thread_id = ${threadId} + `; + assert.deepStrictEqual(manifest, [{ status: "cold" }]); + assert.notDeepInclude(yield* storage.listPendingArchiveThreadIds, threadId); + }), + ); + + it.effect("archives only attachments owned by colliding thread segments", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const archivedThreadId = ThreadId.make("Thread.Foo"); + const liveThreadId = ThreadId.make("thread foo"); + const archivedAttachmentId = "thread-foo-00000000-0000-4000-8000-000000000001"; + const liveAttachmentId = "thread-foo-00000000-0000-4000-8000-000000000002"; + const archivedAttachmentName = `${archivedAttachmentId}.png`; + const liveAttachmentName = `${liveAttachmentId}.png`; + + yield* insertArchivedThread(archivedThreadId, "Archived colliding thread"); + yield* insertArchivedThread(liveThreadId, "Live colliding thread"); + yield* sql` + UPDATE projection_threads SET archived_at = NULL WHERE thread_id = ${liveThreadId} + `; + yield* sql.unsafe( + `INSERT INTO projection_thread_messages ( + message_id, thread_id, turn_id, role, text, attachments_json, + is_streaming, created_at, updated_at + ) VALUES (?, ?, NULL, 'user', ?, ?, 0, ?, ?)`, + [ + "message-archived-collision", + archivedThreadId, + "archive only my attachment", + encodeUnknownJsonString([ + { + type: "image", + id: archivedAttachmentId, + name: "archived.png", + mimeType: "image/png", + }, + ]), + "2026-07-01T00:00:00.000Z", + "2026-07-01T00:00:00.000Z", + ], + ); + yield* sql.unsafe( + `INSERT INTO projection_thread_messages ( + message_id, thread_id, turn_id, role, text, attachments_json, + is_streaming, created_at, updated_at + ) VALUES (?, ?, NULL, 'user', ?, ?, 0, ?, ?)`, + [ + "message-live-collision", + liveThreadId, + "keep my attachment live", + encodeUnknownJsonString([ + { + type: "image", + id: liveAttachmentId, + name: "live.png", + mimeType: "image/png", + }, + ]), + "2026-07-01T00:00:00.000Z", + "2026-07-01T00:00:00.000Z", + ], + ); + const archivedAttachmentPath = path.join(config.attachmentsDir, archivedAttachmentName); + const liveAttachmentPath = path.join(config.attachmentsDir, liveAttachmentName); + yield* fs.writeFileString(archivedAttachmentPath, "archived image"); + yield* fs.writeFileString(liveAttachmentPath, "live image"); + + yield* storage.archiveThread(archivedThreadId); + + assert.isFalse(yield* fs.exists(archivedAttachmentPath)); + assert.strictEqual(yield* fs.readFileString(liveAttachmentPath), "live image"); + const archivedChunks = yield* sql<{ readonly kind: string }>` + SELECT kind FROM cold_archive.archive_thread_chunks + WHERE thread_id = ${archivedThreadId} AND kind LIKE 'attachment:%' + `; + assert.deepStrictEqual(archivedChunks, [{ kind: `attachment:${archivedAttachmentName}` }]); + }), + ); + + it.effect("restores cleanup-pending bundles before unarchiving", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const threadId = ThreadId.make("thread-cleanup-pending-restore"); + + yield* insertArchivedThread(threadId, "Cleanup-pending restore thread"); + yield* sql` + INSERT INTO projection_thread_messages ( + message_id, thread_id, turn_id, role, text, attachments_json, + is_streaming, created_at, updated_at + ) VALUES ( + 'message-cleanup-pending-restore', ${threadId}, NULL, 'user', + 'restore while cleanup is pending', '[]', 0, + '2026-07-01T00:00:00.000Z', '2026-07-01T00:00:00.000Z' + ) + `; + + yield* storage.archiveThread(threadId); + yield* sql` + UPDATE thread_archive_manifests + SET status = 'cleanup_pending' + WHERE thread_id = ${threadId} + `; + + assert.isTrue(yield* storage.restoreTree(threadId)); + const restoredMessages = yield* sql<{ readonly text: string }>` + SELECT text FROM projection_thread_messages WHERE thread_id = ${threadId} + `; + const manifest = yield* sql<{ readonly status: string }>` + SELECT status FROM thread_archive_manifests WHERE thread_id = ${threadId} + `; + assert.deepStrictEqual(restoredMessages, [{ text: "restore while cleanup is pending" }]); + assert.deepStrictEqual(manifest, [{ status: "restored" }]); + }), + ); + + it.effect("abandons an incomplete archive after the shell is unarchived", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const threadId = ThreadId.make("thread-unarchived-before-archive"); + + yield* insertArchivedThread(threadId, "Unarchived before archive thread"); + yield* sql` + INSERT INTO projection_thread_messages ( + message_id, thread_id, turn_id, role, text, attachments_json, + is_streaming, created_at, updated_at + ) VALUES ( + 'message-unarchived-before-archive', ${threadId}, NULL, 'user', + 'keep active data hot', '[]', 0, + '2026-07-01T00:00:00.000Z', '2026-07-01T00:00:00.000Z' + ) + `; + yield* sql` + INSERT INTO thread_archive_manifests ( + thread_id, root_thread_id, status, archive_version, archived_at, updated_at + ) VALUES ( + ${threadId}, ${threadId}, 'pending', 1, + '2026-07-02T00:00:00.000Z', CURRENT_TIMESTAMP + ) + `; + yield* sql` + UPDATE projection_threads SET archived_at = NULL WHERE thread_id = ${threadId} + `; + + yield* sql` + INSERT INTO cold_archive.archive_threads (thread_id, root_thread_id, archive_version, archived_at, original_bytes, compressed_bytes, created_at) + VALUES (${threadId}, ${threadId}, 1, '2026-07-02T00:00:00.000Z', 1, 1, CURRENT_TIMESTAMP) + `; + yield* sql` + INSERT INTO cold_archive.archive_thread_chunks (thread_id, chunk_index, kind, row_count, data) + VALUES (${threadId}, 0, 'table:projection_thread_messages', 0, ${new Uint8Array([1])}) + `; + const unsafe = sql.unsafe.bind(sql); + const spy = vi + .spyOn(sql, "unsafe") + .mockImplementation((query, params) => + unsafe( + query.includes("DELETE FROM thread_archive_manifests") + ? "SELECT * FROM missing_aborted_archive_cleanup_fixture" + : query, + params, + ), + ); + yield* Effect.flip(storage.archiveThread(threadId)).pipe( + Effect.ensuring(Effect.sync(() => spy.mockRestore())), + ); + assert.deepStrictEqual( + yield* sql`SELECT status FROM thread_archive_manifests WHERE thread_id = ${threadId}`, + [{ status: "pending" }], + ); + assert.deepStrictEqual( + yield* sql`SELECT thread_id FROM cold_archive.archive_threads WHERE thread_id = ${threadId}`, + [], + ); + assert.deepStrictEqual( + yield* sql`SELECT chunk_index FROM cold_archive.archive_thread_chunks WHERE thread_id = ${threadId}`, + [], + ); + assert.include(yield* storage.listPendingArchiveThreadIds, threadId); + yield* storage.archiveThread(threadId); + const messages = yield* sql<{ readonly text: string }>` + SELECT text FROM projection_thread_messages WHERE thread_id = ${threadId} + `; + const manifests = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count FROM thread_archive_manifests WHERE thread_id = ${threadId} + `; + const chunks = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count + FROM cold_archive.archive_thread_chunks + WHERE thread_id = ${threadId} + `; + assert.deepStrictEqual(messages, [{ text: "keep active data hot" }]); + assert.deepStrictEqual(manifests, [{ count: 0 }]); + assert.deepStrictEqual(chunks, [{ count: 0 }]); + }), + ); + + it.effect("retries attachment cleanup after directory I/O errors", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const threadId = ThreadId.make("thread-attachment-directory-error"); + + yield* insertArchivedThread(threadId, "Attachment directory error thread"); + yield* fs.remove(config.attachmentsDir, { recursive: true }); + yield* fs.writeFileString(config.attachmentsDir, "not a directory"); + + const archiveFailure = yield* Effect.flip(storage.archiveThread(threadId)); + assert.strictEqual(archiveFailure.operation, "archive"); + const shell = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count FROM projection_threads WHERE thread_id = ${threadId} + `; + const manifests = yield* sql<{ readonly status: string }>` + SELECT status FROM thread_archive_manifests WHERE thread_id = ${threadId} + `; + assert.deepStrictEqual(shell, [{ count: 1 }]); + assert.deepStrictEqual(manifests, [{ status: "archiving" }]); + + yield* fs.remove(config.attachmentsDir); + yield* fs.makeDirectory(config.attachmentsDir); + yield* storage.archiveThread(threadId); + const completedManifest = yield* sql<{ readonly status: string }>` + SELECT status FROM thread_archive_manifests WHERE thread_id = ${threadId} + `; + assert.deepStrictEqual(completedManifest, [{ status: "cold" }]); + }), + ); + + it.effect("keeps the delete cleanup queue entry until external cleanup succeeds", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* ThreadColdStorage.ThreadColdStorage; + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const threadId = ThreadId.make("thread-delete-retry"); + const attachmentName = "thread-delete-retry-00000000-0000-4000-8000-000000000001.png"; + const attachmentPath = path.join(config.attachmentsDir, attachmentName); + + yield* insertArchivedThread(threadId, "Delete retry thread"); + yield* sql.unsafe( + `INSERT INTO projection_thread_messages ( + message_id, thread_id, turn_id, role, text, attachments_json, + is_streaming, created_at, updated_at + ) VALUES (?, ?, NULL, 'user', 'delete me', ?, 0, ?, ?)`, + [ + "message-delete-retry", + threadId, + encodeUnknownJsonString([ + { + type: "image", + id: attachmentName.slice(0, -4), + name: "delete.png", + mimeType: "image/png", + }, + ]), + "2026-07-01T00:00:00.000Z", + "2026-07-01T00:00:00.000Z", + ], + ); + yield* fs.makeDirectory(attachmentPath); + yield* fs.writeFileString(path.join(attachmentPath, "keep"), "force cleanup failure"); + + yield* sql`UPDATE projection_threads SET deleted_at = CURRENT_TIMESTAMP WHERE thread_id = ${threadId}`; + const deleteFailure = yield* Effect.flip(storage.deleteThread(threadId)); + assert.strictEqual(deleteFailure.operation, "delete"); + const pendingCleanup = yield* sql<{ readonly reason: string }>` + SELECT reason FROM thread_cleanup_queue WHERE thread_id = ${threadId} + `; + assert.deepStrictEqual(pendingCleanup, [{ reason: "deleted" }]); + + yield* fs.remove(attachmentPath, { recursive: true }); + yield* sql`UPDATE projection_threads SET deleted_at = CURRENT_TIMESTAMP WHERE thread_id = ${threadId}`; + yield* storage.deleteThread(threadId); + const completedCleanup = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count FROM thread_cleanup_queue WHERE thread_id = ${threadId} + `; + assert.deepStrictEqual(completedCleanup, [{ count: 0 }]); + }), + ); +}); diff --git a/apps/server/src/orchestration/ThreadColdStorage.ts b/apps/server/src/orchestration/ThreadColdStorage.ts new file mode 100644 index 000000000000..b4ee6da7fc69 --- /dev/null +++ b/apps/server/src/orchestration/ThreadColdStorage.ts @@ -0,0 +1,1180 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeUtil from "node:util"; +import * as NodeZlib from "node:zlib"; + +import { ThreadId } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; +import * as SynchronizedRef from "effect/SynchronizedRef"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +import { + parseAttachmentIdFromRelativePath, + toSafeThreadAttachmentSegment, +} from "../attachmentStore.ts"; +import { ServerConfig } from "../config.ts"; + +const gzipAsync = NodeUtil.promisify(NodeZlib.gzip); +const gunzipAsync = NodeUtil.promisify(NodeZlib.gunzip); +const ARCHIVE_SCHEMA = "cold_archive"; +const ARCHIVE_VERSION = 1; +const ROW_CHUNK_SIZE = 250; +const RESTORE_CHUNK_PAGE_SIZE = 32; +const BINARY_VALUE_KEY = "__t3_archive_binary_base64"; +const UnknownFromJsonString = Schema.fromJsonString(Schema.Unknown); +const encodeUnknownJsonString = Schema.encodeUnknownEffect(UnknownFromJsonString); +const decodeUnknownJsonString = Schema.decodeUnknownEffect(UnknownFromJsonString); + +export const ThreadColdStorageOperation = Schema.Literals([ + "archive", + "restore", + "rollback-restore", + "finish-restore", + "delete", + "remove-provider-logs", + "compact-legacy-storage", + "list-pending-archives", + "list-pending-deletes", +]); +export type ThreadColdStorageOperation = typeof ThreadColdStorageOperation.Type; + +export class ThreadColdStorageError extends Schema.TaggedError()( + "ThreadColdStorageError", + { + operation: ThreadColdStorageOperation, + threadId: Schema.optional(ThreadId), + cause: Schema.Defect(), + }, +) { + override get message(): string { + const context = this.threadId === undefined ? "" : ` for '${this.threadId}'`; + return `Cold thread storage failed during ${this.operation}${context}.`; + } + + static normalize( + operation: ThreadColdStorageOperation, + threadId: ThreadId | undefined, + cause: unknown, + ): ThreadColdStorageError { + const isThreadColdStorageError = Schema.is(ThreadColdStorageError); + return isThreadColdStorageError(cause) && + cause.operation === operation && + cause.threadId === threadId + ? cause + : new ThreadColdStorageError({ operation, threadId, cause }); + } +} + +export class ThreadColdStorage extends Context.Service< + ThreadColdStorage, + { + readonly archiveThread: ( + threadId: ThreadId, + quiesce?: Effect.Effect, + ) => Effect.Effect; + readonly restoreTree: (threadId: ThreadId) => Effect.Effect; + readonly rollbackRestoreTree: ( + threadId: ThreadId, + ) => Effect.Effect; + readonly finishRestoreTree: (threadId: ThreadId) => Effect.Effect; + readonly deleteThread: ( + threadId: ThreadId, + quiesce?: Effect.Effect, + ) => Effect.Effect; + readonly removeProviderLogs: ( + threadId: ThreadId, + ) => Effect.Effect; + readonly compactLegacyStorage: Effect.Effect; + readonly listPendingArchiveThreadIds: Effect.Effect< + ReadonlyArray, + ThreadColdStorageError + >; + readonly listPendingDeleteThreadIds: Effect.Effect< + ReadonlyArray, + ThreadColdStorageError + >; + } +>()("t3/orchestration/ThreadColdStorage") {} + +const noOp = ThreadColdStorage.of({ + archiveThread: () => Effect.void, + restoreTree: () => Effect.succeed(true), + rollbackRestoreTree: () => Effect.void, + finishRestoreTree: () => Effect.void, + deleteThread: (threadId, quiesce = Effect.void) => + quiesce.pipe( + Effect.mapError((cause) => ThreadColdStorageError.normalize("delete", threadId, cause)), + ), + removeProviderLogs: () => Effect.void, + compactLegacyStorage: Effect.void, + listPendingArchiveThreadIds: Effect.succeed([]), + listPendingDeleteThreadIds: Effect.succeed([]), +}); + +export const noOpLayer = Layer.succeed(ThreadColdStorage, noOp); + +type SqlRow = Record; +type ThreadLockEntry = { + readonly semaphore: Semaphore.Semaphore; + readonly users: number; +}; +type AcquiredThreadLock = { + readonly rootThreadId: ThreadId; + readonly semaphore: Semaphore.Semaphore; +}; + +class ArchiveCodecError extends Schema.TaggedError()("ArchiveCodecError", { + operation: Schema.Literals(["encode", "decode", "compress", "decompress"]), + cause: Schema.Defect(), +}) { + override get message(): string { + return `Archive ${this.operation} failed`; + } +} + +const isArchiveCodecError = Schema.is(ArchiveCodecError); +class ArchiveTableValidationError extends Schema.TaggedError()( + "ArchiveTableValidationError", + { + table: Schema.String, + }, +) { + override get message(): string { + return `Archive chunk targets unknown table '${this.table}'`; + } +} + +class ArchiveChunkKindValidationError extends Schema.TaggedError()( + "ArchiveChunkKindValidationError", + { + chunkKind: Schema.String, + }, +) { + override get message(): string { + return `Archive contains unknown chunk kind '${this.chunkKind}'`; + } +} + +class ArchiveAttachmentEntryValidationError extends Schema.TaggedError()( + "ArchiveAttachmentEntryValidationError", + { entry: Schema.String }, +) { + override get message(): string { + return `Archive contains unsafe attachment entry '${this.entry}'; cold bundle retained for recovery`; + } +} + +const ARCHIVED_THREAD_TABLES = [ + // Match the event index's aggregate prefix for both paging and deletion. + ["orchestration_events", "aggregate_kind = 'thread' AND stream_id = ?"], + ["checkpoint_diff_blobs", "thread_id = ?"], + ["provider_session_runtime", "thread_id = ?"], + ["projection_thread_messages", "thread_id = ?"], + ["projection_thread_activities", "thread_id = ?"], + ["projection_thread_sessions", "thread_id = ?"], + ["projection_turns", "thread_id = ?"], + ["projection_pending_approvals", "thread_id = ?"], + ["projection_thread_proposed_plans", "thread_id = ?"], +] as const; + +const THREAD_TABLES = [ + ...ARCHIVED_THREAD_TABLES, + // Archived PR relations remain queryable alongside their thread shells. + ["projection_thread_pull_requests", "thread_id = ?"] as const, + ["orchestration_command_receipts", "aggregate_id = ?"] as const, +] as const; + +function encodeRows(rows: ReadonlyArray): Effect.Effect { + return encodeUnknownJsonString( + rows.map((row) => + Object.fromEntries( + Object.entries(row).map(([column, value]) => [ + column, + value instanceof Uint8Array + ? { [BINARY_VALUE_KEY]: Buffer.from(value).toString("base64") } + : value, + ]), + ), + ), + ).pipe( + Effect.map((encoded) => Buffer.from(encoded, "utf8")), + Effect.mapError((cause) => new ArchiveCodecError({ operation: "encode", cause })), + ); +} + +function decodeRows(data: Uint8Array): Effect.Effect, ArchiveCodecError> { + return decodeUnknownJsonString(Buffer.from(data).toString("utf8")).pipe( + Effect.flatMap((decoded) => + Effect.try({ + try: () => { + if (!Array.isArray(decoded)) { + throw new TypeError("Archived table chunk must contain an array of rows"); + } + return decoded.map((row): SqlRow => { + if (row === null || typeof row !== "object" || Array.isArray(row)) { + throw new TypeError("Archived table chunk contains an invalid row"); + } + return Object.fromEntries( + Object.entries(row).map(([column, value]) => { + if ( + value !== null && + typeof value === "object" && + !Array.isArray(value) && + Object.keys(value).length === 1 && + typeof (value as Record)[BINARY_VALUE_KEY] === "string" + ) { + return [ + column, + new Uint8Array( + Buffer.from( + (value as Record)[BINARY_VALUE_KEY] as string, + "base64", + ), + ), + ]; + } + return [column, value]; + }), + ); + }); + }, + catch: (cause) => new ArchiveCodecError({ operation: "decode", cause }), + }), + ), + Effect.mapError((cause) => + isArchiveCodecError(cause) ? cause : new ArchiveCodecError({ operation: "decode", cause }), + ), + ); +} + +const THREAD_TABLE_NAMES = new Set(THREAD_TABLES.map(([table]) => table)); + +function quoteIdentifier(identifier: string): string { + return `"${identifier.replaceAll('"', '""')}"`; +} + +function isSafeAttachmentEntry(entry: string): boolean { + return ( + entry.length > 0 && + entry !== "." && + entry !== ".." && + !entry.includes("/") && + !entry.includes("\\") && + !entry.includes("\0") + ); +} + +const compress = (data: Uint8Array) => + Effect.tryPromise({ + try: () => gzipAsync(data), + catch: (cause) => new ArchiveCodecError({ operation: "compress", cause }), + }).pipe(Effect.map((value) => new Uint8Array(value))); + +const decompress = (data: Uint8Array) => + Effect.tryPromise({ + try: () => gunzipAsync(data), + catch: (cause) => new ArchiveCodecError({ operation: "decompress", cause }), + }).pipe(Effect.map((value) => new Uint8Array(value))); + +const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const config = yield* ServerConfig; + // One server owns lifecycle work for this storage set. Unlike auxiliary CLI + // writes or multiple clients, a second lifecycle owner cannot share these + // process-local claims: it would mistake a live restore for crash residue. + const threadLocksRef = yield* SynchronizedRef.make(new Map()); + const activeRestoreRoots = new Set(); + const restoreClaims = new Map(); + + const claimRestore = (threadId: ThreadId, rootThreadId: ThreadId) => { + restoreClaims.set(threadId, { rootThreadId: String(rootThreadId) }); + activeRestoreRoots.add(String(rootThreadId)); + }; + + yield* sql.unsafe(`ATTACH DATABASE ? AS ${ARCHIVE_SCHEMA}`, [config.archiveDbPath]); + yield* sql.unsafe(`PRAGMA ${ARCHIVE_SCHEMA}.auto_vacuum = INCREMENTAL`); + yield* sql.unsafe(` + CREATE TABLE IF NOT EXISTS ${ARCHIVE_SCHEMA}.archive_threads ( + thread_id TEXT PRIMARY KEY, + root_thread_id TEXT NOT NULL, + archive_version INTEGER NOT NULL, + archived_at TEXT NOT NULL, + original_bytes INTEGER NOT NULL, + compressed_bytes INTEGER NOT NULL, + created_at TEXT NOT NULL + ) + `); + yield* sql.unsafe(` + CREATE TABLE IF NOT EXISTS ${ARCHIVE_SCHEMA}.archive_thread_chunks ( + thread_id TEXT NOT NULL, + chunk_index INTEGER NOT NULL, + kind TEXT NOT NULL, + row_count INTEGER NOT NULL, + data BLOB NOT NULL, + PRIMARY KEY (thread_id, chunk_index) + ) + `); + yield* sql.unsafe(` + CREATE INDEX IF NOT EXISTS ${ARCHIVE_SCHEMA}.idx_archive_thread_chunks_thread + ON archive_thread_chunks(thread_id, chunk_index) + `); + + const attachmentEntriesForThread = Effect.fn("attachmentEntriesForThread")(function* ( + threadId: string, + ) { + const attachmentIds = new Set(); + const attachmentRows = (yield* sql.unsafe( + `SELECT attachments_json + FROM projection_thread_messages + WHERE thread_id = ? AND attachments_json IS NOT NULL`, + [threadId], + )) as ReadonlyArray; + for (const row of attachmentRows) { + const attachments = yield* decodeUnknownJsonString(String(row.attachments_json)); + if (!Array.isArray(attachments)) continue; + for (const attachment of attachments) { + if (attachment === null || typeof attachment !== "object" || Array.isArray(attachment)) { + continue; + } + const id = (attachment as Record).id; + if (typeof id === "string" && id.length > 0) { + attachmentIds.add(id); + } + } + } + + const archivedEntries = new Set( + ( + (yield* sql.unsafe( + `SELECT kind + FROM ${ARCHIVE_SCHEMA}.archive_thread_chunks + WHERE thread_id = ? AND kind LIKE 'attachment:%'`, + [threadId], + )) as ReadonlyArray + ).flatMap((row) => { + const entry = String(row.kind).slice("attachment:".length); + return isSafeAttachmentEntry(entry) ? [entry] : []; + }), + ); + const entries = yield* fs + .readDirectory(config.attachmentsDir, { recursive: false }) + .pipe( + Effect.catch((error) => + error.reason._tag === "NotFound" ? Effect.succeed([] as string[]) : Effect.fail(error), + ), + ); + return entries.filter((entry) => { + if (archivedEntries.has(entry)) return true; + const attachmentId = parseAttachmentIdFromRelativePath(entry); + return attachmentId !== null && attachmentIds.has(attachmentId); + }); + }); + + const removeAttachments = Effect.fn("removeThreadAttachments")(function* (threadId: string) { + const entries = yield* attachmentEntriesForThread(threadId); + yield* Effect.forEach( + entries, + (entry) => fs.remove(path.join(config.attachmentsDir, entry), { force: true }), + { concurrency: 4, discard: true }, + ); + }); + + const removeProviderLogsImpl = Effect.fn("removeProviderLogs")(function* (threadId: string) { + const segment = toSafeThreadAttachmentSegment(threadId); + if (!segment) return; + const configuredBaseName = path.basename(config.providerEventLogPath); + const configuredExtension = path.extname(configuredBaseName); + const providerLogPrefix = `${ + configuredExtension.length > 0 + ? configuredBaseName.slice(0, -configuredExtension.length) + : configuredBaseName + }.`; + const baseName = `${providerLogPrefix}${segment}.log`; + const entries = yield* fs + .readDirectory(config.providerLogsDir, { recursive: false }) + .pipe( + Effect.catch((error) => + error.reason._tag === "NotFound" ? Effect.succeed([] as string[]) : Effect.fail(error), + ), + ); + yield* Effect.forEach( + entries.filter((entry) => { + if (entry === baseName) return true; + if (!entry.startsWith(`${baseName}.`)) return false; + const rotation = entry.slice(baseName.length + 1); + return ( + rotation.length > 0 && + [...rotation].every((character) => character >= "0" && character <= "9") + ); + }), + (entry) => fs.remove(path.join(config.providerLogsDir, entry), { force: true }), + { concurrency: 4, discard: true }, + ); + }); + + const reclaimFreePages = Effect.fn("reclaimThreadStorageFreePages")(function* () { + yield* sql.unsafe("PRAGMA main.incremental_vacuum(2048)"); + yield* sql.unsafe(`PRAGMA ${ARCHIVE_SCHEMA}.incremental_vacuum(2048)`); + }); + + const completeArchiveCleanup = Effect.fn("completeThreadArchiveCleanup")(function* ( + threadId: ThreadId, + ) { + yield* removeAttachments(threadId); + yield* removeProviderLogsImpl(threadId); + yield* reclaimFreePages(); + yield* sql.unsafe( + `UPDATE thread_archive_manifests + SET status = 'cold', updated_at = CURRENT_TIMESTAMP, error = NULL + WHERE thread_id = ? AND status = 'cleanup_pending'`, + [threadId], + ); + }); + + const insertChunk = Effect.fn("insertArchiveChunk")(function* (input: { + readonly threadId: string; + readonly chunkIndex: number; + readonly kind: string; + readonly rowCount: number; + readonly data: Uint8Array; + }) { + yield* sql.unsafe( + `INSERT INTO ${ARCHIVE_SCHEMA}.archive_thread_chunks + (thread_id, chunk_index, kind, row_count, data) + VALUES (?, ?, ?, ?, ?)`, + [input.threadId, input.chunkIndex, input.kind, input.rowCount, input.data], + ); + }); + + const discardIncompleteArchive = Effect.fn("discardIncompleteThreadArchive")(function* ( + threadId: ThreadId, + ) { + // Commit cold cleanup before removing its recovery marker in the main WAL file. + yield* sql.unsafe(`DELETE FROM ${ARCHIVE_SCHEMA}.archive_thread_chunks WHERE thread_id = ?`, [ + threadId, + ]); + yield* sql.unsafe(`DELETE FROM ${ARCHIVE_SCHEMA}.archive_threads WHERE thread_id = ?`, [ + threadId, + ]); + yield* sql.unsafe( + `DELETE FROM thread_archive_manifests + WHERE thread_id = ? AND status IN ('pending', 'archiving')`, + [threadId], + ); + }); + + const deleteRestoredBundle = Effect.fn("deleteRestoredArchiveBundle")(function* ( + threadId: string, + ) { + // Commit cold cleanup before dropping its main-database recovery marker. + // A transaction spanning attached WAL files cannot preserve that ordering. + yield* sql.unsafe(`DELETE FROM ${ARCHIVE_SCHEMA}.archive_thread_chunks WHERE thread_id = ?`, [ + threadId, + ]); + yield* sql.unsafe(`DELETE FROM ${ARCHIVE_SCHEMA}.archive_threads WHERE thread_id = ?`, [ + threadId, + ]); + yield* sql.unsafe(`DELETE FROM thread_archive_manifests WHERE thread_id = ?`, [threadId]); + }); + + const archiveImpl = Effect.fn("archiveThreadImpl")(function* ( + threadId: ThreadId, + allowRestored: boolean, + quiesce: Effect.Effect, + ) { + const manifestRows = (yield* sql.unsafe( + `SELECT root_thread_id, archived_at, status + FROM thread_archive_manifests + WHERE thread_id = ?`, + [threadId], + )) as ReadonlyArray; + const threadRows = (yield* sql.unsafe( + `SELECT thread_id AS root_thread_id, archived_at + FROM projection_threads + WHERE thread_id = ? AND deleted_at IS NULL AND archived_at IS NOT NULL`, + [threadId], + )) as ReadonlyArray; + const manifest = manifestRows[0]; + const source = manifest ?? threadRows[0]; + if (!source) return; + if (source.status === "cold") return; + if (source.status === "cleanup_pending") { + yield* completeArchiveCleanup(threadId); + return; + } + const rootThreadId = String(source.root_thread_id ?? threadId); + if (threadRows.length === 0) { + if (source.status === "restored" && !activeRestoreRoots.has(rootThreadId)) { + const activeShell = yield* sql.unsafe( + `SELECT 1 FROM projection_threads + WHERE thread_id = ? AND archived_at IS NULL AND deleted_at IS NULL`, + [threadId], + ); + if (activeShell.length > 0) { + // Finalization never quiesces or archives the already-active thread. + yield* deleteRestoredBundle(threadId); + yield* reclaimFreePages(); + } + } + if (source.status === "pending" || source.status === "archiving") { + yield* discardIncompleteArchive(threadId); + } + return; + } + // A restored manifest reserves this archive epoch for an in-flight + // unarchive only while this process owns that restore. A reservation left + // behind by a crashed process is durable recovery work, while a later + // archive has a new shell timestamp and may replace a reservation that + // finishRestoreTree failed to remove after commit. + if ( + source.status === "restored" && + !allowRestored && + activeRestoreRoots.has(rootThreadId) && + String(source.archived_at) === String(threadRows[0]?.archived_at) + ) { + return; + } + yield* quiesce; + const archivedAt = String( + threadRows[0]?.archived_at ?? source.archived_at ?? DateTime.formatIso(yield* DateTime.now), + ); + + yield* sql.unsafe( + `INSERT INTO thread_archive_manifests + (thread_id, root_thread_id, status, archive_version, archived_at, updated_at, error) + VALUES (?, ?, 'archiving', ?, ?, CURRENT_TIMESTAMP, NULL) + ON CONFLICT(thread_id) DO UPDATE SET + root_thread_id = excluded.root_thread_id, + status = 'archiving', + archive_version = excluded.archive_version, + archived_at = excluded.archived_at, + updated_at = CURRENT_TIMESTAMP, + error = NULL`, + [threadId, rootThreadId, ARCHIVE_VERSION, archivedAt], + ); + yield* sql.unsafe(`DELETE FROM ${ARCHIVE_SCHEMA}.archive_thread_chunks WHERE thread_id = ?`, [ + threadId, + ]); + yield* sql.unsafe(`DELETE FROM ${ARCHIVE_SCHEMA}.archive_threads WHERE thread_id = ?`, [ + threadId, + ]); + + let chunkIndex = 0; + let originalBytes = 0; + let compressedBytes = 0; + for (const [table, predicate] of ARCHIVED_THREAD_TABLES) { + let lastRowId = 0; + while (true) { + const rows = (yield* sql.unsafe( + `SELECT rowid AS __archive_rowid, * + FROM ${table} + WHERE ${predicate} AND rowid > ? + ORDER BY rowid ASC + LIMIT ${ROW_CHUNK_SIZE}`, + [threadId, lastRowId], + )) as ReadonlyArray; + if (rows.length === 0) break; + const normalizedRows = rows.map((row) => { + const { __archive_rowid, ...stored } = row; + lastRowId = Number(__archive_rowid); + return stored; + }); + const encoded = yield* encodeRows(normalizedRows); + const compressed = yield* compress(encoded); + yield* insertChunk({ + threadId, + chunkIndex, + kind: `table:${table}`, + rowCount: normalizedRows.length, + data: compressed, + }); + chunkIndex += 1; + originalBytes += encoded.byteLength; + compressedBytes += compressed.byteLength; + } + } + + const attachmentEntries = yield* attachmentEntriesForThread(threadId); + for (const entry of attachmentEntries) { + const bytes = yield* fs.readFile(path.join(config.attachmentsDir, entry)); + const compressed = yield* compress(bytes); + yield* insertChunk({ + threadId, + chunkIndex, + kind: `attachment:${entry}`, + rowCount: 1, + data: compressed, + }); + chunkIndex += 1; + originalBytes += bytes.byteLength; + compressedBytes += compressed.byteLength; + } + + yield* sql.unsafe( + `INSERT INTO ${ARCHIVE_SCHEMA}.archive_threads + (thread_id, root_thread_id, archive_version, archived_at, original_bytes, + compressed_bytes, created_at) + VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)`, + [threadId, rootThreadId, ARCHIVE_VERSION, archivedAt, originalBytes, compressedBytes], + ); + + // WAL does not commit attached files atomically. Persist the complete bundle + // first so the destructive success path writes only main. Retries replace it + // while the original hot rows are still available. + const archivedAtDestructiveBoundary = yield* sql.withTransaction( + Effect.gen(function* () { + const archivedShell = (yield* sql.unsafe( + `SELECT 1 AS present + FROM projection_threads + WHERE thread_id = ? AND deleted_at IS NULL AND archived_at IS NOT NULL + LIMIT 1`, + [threadId], + )) as ReadonlyArray; + if (archivedShell.length === 0) { + return false; + } + for (const [table, predicate] of [...ARCHIVED_THREAD_TABLES].toReversed()) { + yield* sql.unsafe(`DELETE FROM ${table} WHERE ${predicate}`, [threadId]); + } + yield* sql.unsafe( + `UPDATE thread_archive_manifests + SET status = 'cleanup_pending', original_bytes = ?, compressed_bytes = ?, + updated_at = CURRENT_TIMESTAMP, error = NULL + WHERE thread_id = ?`, + [originalBytes, compressedBytes, threadId], + ); + return true; + }), + ); + + if (!archivedAtDestructiveBoundary) { + yield* sql.unsafe(`DELETE FROM ${ARCHIVE_SCHEMA}.archive_thread_chunks WHERE thread_id = ?`, [ + threadId, + ]); + yield* sql.unsafe(`DELETE FROM ${ARCHIVE_SCHEMA}.archive_threads WHERE thread_id = ?`, [ + threadId, + ]); + yield* sql.unsafe( + `DELETE FROM thread_archive_manifests WHERE thread_id = ? AND status = 'archiving'`, + [threadId], + ); + return; + } + + yield* completeArchiveCleanup(threadId); + }); + + const insertRows = Effect.fn("restoreArchiveRows")(function* ( + table: string, + rows: ReadonlyArray, + ) { + if (!THREAD_TABLE_NAMES.has(table)) { + return yield* new ArchiveTableValidationError({ table }); + } + const tableInfo = (yield* sql.unsafe( + `PRAGMA main.table_info(${quoteIdentifier(table)})`, + )) as ReadonlyArray; + const currentColumns = new Set( + tableInfo.flatMap((column) => + typeof column.name === "string" && column.name.length > 0 ? [column.name] : [], + ), + ); + for (const row of rows) { + // Archived rows can outlive schema migrations. Ignore columns that no + // longer exist and let newly-added columns use their database defaults. + const columns = Object.keys(row).filter((column) => currentColumns.has(column)); + if (columns.length === 0) continue; + const placeholders = columns.map(() => "?").join(", "); + yield* sql.unsafe( + `INSERT OR REPLACE INTO ${quoteIdentifier(table)} (${columns.map(quoteIdentifier).join(", ")}) VALUES (${placeholders})`, + columns.map((column) => row[column]), + ); + } + }); + + const restoreThread = Effect.fn("restoreArchivedThread")(function* (threadId: ThreadId) { + const bundleRows = (yield* sql.unsafe( + `SELECT 1 AS present FROM ${ARCHIVE_SCHEMA}.archive_threads WHERE thread_id = ? LIMIT 1`, + [threadId], + )) as ReadonlyArray; + if (bundleRows.length === 0) return false; + + const invalidKinds = (yield* sql.unsafe( + `SELECT kind FROM ${ARCHIVE_SCHEMA}.archive_thread_chunks + WHERE thread_id = ? AND kind NOT LIKE 'table:%' AND kind NOT LIKE 'attachment:%' + LIMIT 1`, + [threadId], + )) as ReadonlyArray; + if (invalidKinds.length > 0) { + return yield* new ArchiveChunkKindValidationError({ + chunkKind: String(invalidKinds[0]?.kind), + }); + } + + // Restore files before changing SQL state. A failed or interrupted file + // write leaves the cold bundle authoritative and can be retried safely. + let attachmentChunkIndex = -1; + while (true) { + const attachmentChunks = (yield* sql.unsafe( + `SELECT chunk_index, kind, data + FROM ${ARCHIVE_SCHEMA}.archive_thread_chunks + WHERE thread_id = ? AND chunk_index > ? AND kind LIKE 'attachment:%' + ORDER BY chunk_index ASC + LIMIT ${RESTORE_CHUNK_PAGE_SIZE}`, + [threadId, attachmentChunkIndex], + )) as ReadonlyArray; + if (attachmentChunks.length === 0) break; + for (const chunk of attachmentChunks) { + attachmentChunkIndex = Number(chunk.chunk_index); + const entry = String(chunk.kind).slice("attachment:".length); + if (!isSafeAttachmentEntry(entry)) { + return yield* new ArchiveAttachmentEntryValidationError({ entry }); + } + const data = yield* decompress(chunk.data as Uint8Array); + const targetPath = path.join(config.attachmentsDir, entry); + const temporaryPath = `${targetPath}.t3-restore`; + yield* fs + .writeFile(temporaryPath, data) + .pipe( + Effect.andThen(fs.remove(targetPath, { force: true })), + Effect.andThen(fs.rename(temporaryPath, targetPath)), + Effect.ensuring(fs.remove(temporaryPath, { force: true }).pipe(Effect.ignore)), + ); + } + } + + yield* sql.withTransaction( + Effect.gen(function* () { + let tableChunkIndex = -1; + while (true) { + const tableChunks = (yield* sql.unsafe( + `SELECT chunk_index, kind, data + FROM ${ARCHIVE_SCHEMA}.archive_thread_chunks + WHERE thread_id = ? AND chunk_index > ? AND kind LIKE 'table:%' + ORDER BY chunk_index ASC + LIMIT ${RESTORE_CHUNK_PAGE_SIZE}`, + [threadId, tableChunkIndex], + )) as ReadonlyArray; + if (tableChunks.length === 0) break; + for (const chunk of tableChunks) { + tableChunkIndex = Number(chunk.chunk_index); + const kind = String(chunk.kind); + const data = yield* decompress(chunk.data as Uint8Array); + yield* insertRows(kind.slice("table:".length), yield* decodeRows(data)); + } + } + yield* sql.unsafe( + `UPDATE thread_archive_manifests + SET status = 'restored', updated_at = CURRENT_TIMESTAMP, error = NULL + WHERE thread_id = ?`, + [threadId], + ); + }), + ); + + return true; + }); + + const resolveTreeRoot = Effect.fn("resolveArchiveTreeRoot")(function* (threadId: ThreadId) { + const rows = (yield* sql.unsafe( + `SELECT COALESCE( + (SELECT root_thread_id FROM thread_archive_manifests WHERE thread_id = ?), + (SELECT thread_id FROM projection_threads WHERE thread_id = ?), + ? + ) AS root_thread_id`, + [threadId, threadId, threadId], + )) as ReadonlyArray; + return ThreadId.make(String(rows[0]?.root_thread_id ?? threadId)); + }); + + const restoreTreeImpl = Effect.fn("restoreArchiveTreeImpl")(function* (threadId: ThreadId) { + const rootThreadId = yield* resolveTreeRoot(threadId); + const rows = (yield* sql.unsafe( + `SELECT thread_id, status + FROM thread_archive_manifests + WHERE root_thread_id = ? AND status IN ('cleanup_pending', 'cold', 'restored') + ORDER BY CASE WHEN thread_id = ? THEN 1 ELSE 0 END, thread_id ASC`, + [rootThreadId, rootThreadId], + )) as ReadonlyArray; + let restored = false; + for (const row of rows) { + if (row.status === "restored") continue; + restored = (yield* restoreThread(ThreadId.make(String(row.thread_id)))) || restored; + } + if (restored || rows.some((row) => row.status === "restored")) { + claimRestore(threadId, rootThreadId); + return true; + } + if (rows.length > 0) { + return false; + } + + // The lifecycle worker may not have started archiving this shell yet. Mark + // its still-hot rows as owned by the unarchive command before releasing the + // tree lock, so a queued archive job cannot delete them before that command + // commits. A failed command rolls this reservation back through archiveImpl; + // a successful command removes it through finishRestoreTreeImpl. + const reserved = yield* sql.withTransaction( + Effect.gen(function* () { + const archivedShell = (yield* sql.unsafe( + `SELECT archived_at + FROM projection_threads + WHERE thread_id = ? AND deleted_at IS NULL AND archived_at IS NOT NULL + LIMIT 1`, + [threadId], + )) as ReadonlyArray; + const source = archivedShell[0]; + if (!source) return false; + + yield* sql.unsafe( + `INSERT INTO thread_archive_manifests + (thread_id, root_thread_id, status, archive_version, archived_at, updated_at, error) + VALUES (?, ?, 'restored', ?, ?, CURRENT_TIMESTAMP, NULL) + ON CONFLICT(thread_id) DO UPDATE SET + root_thread_id = excluded.root_thread_id, + status = 'restored', + archive_version = excluded.archive_version, + archived_at = excluded.archived_at, + updated_at = CURRENT_TIMESTAMP, + error = NULL + WHERE thread_archive_manifests.status IN ('pending', 'archiving')`, + [threadId, rootThreadId, ARCHIVE_VERSION, String(source.archived_at)], + ); + return true; + }), + ); + if (reserved) { + claimRestore(threadId, rootThreadId); + } + return reserved; + }); + + const rollbackRestoreTreeImpl = Effect.fn("rollbackRestoreArchiveTreeImpl")(function* ( + threadId: ThreadId, + ) { + const rootThreadId = yield* resolveTreeRoot(threadId); + yield* Effect.gen(function* () { + const rows = (yield* sql.unsafe( + `SELECT thread_id + FROM thread_archive_manifests + WHERE root_thread_id = ? AND status = 'restored' + ORDER BY CASE WHEN thread_id = ? THEN 1 ELSE 0 END, thread_id ASC`, + [rootThreadId, rootThreadId], + )) as ReadonlyArray; + yield* Effect.forEach( + rows, + (row) => archiveImpl(ThreadId.make(String(row.thread_id)), true, Effect.void), + { + discard: true, + }, + ); + }); + }); + + const finishRestoreTreeImpl = Effect.fn("finishRestoreArchiveTreeImpl")(function* ( + threadId: ThreadId, + ) { + const rootThreadId = yield* resolveTreeRoot(threadId); + yield* Effect.gen(function* () { + const rows = (yield* sql.unsafe( + `SELECT thread_id + FROM thread_archive_manifests + WHERE root_thread_id = ? AND status = 'restored'`, + [rootThreadId], + )) as ReadonlyArray; + for (const row of rows) { + yield* deleteRestoredBundle(String(row.thread_id)); + } + yield* sql.unsafe(`PRAGMA ${ARCHIVE_SCHEMA}.incremental_vacuum(2048)`); + }); + }); + + const skipObsoleteDelete = Effect.fn("skipObsoleteThreadDelete")(function* (threadId: ThreadId) { + const shells = yield* sql<{ deleted_at: string | null; archived_at: string | null }>` + SELECT deleted_at, archived_at FROM projection_threads WHERE thread_id = ${threadId} + `; + const shell = shells[0]; + if (shell?.deleted_at === null) { + const matchingArchive = yield* sql` + SELECT 1 FROM thread_archive_manifests + WHERE thread_id = ${threadId} AND archived_at IS ${shell.archived_at} + `; + if (matchingArchive.length === 0) { + yield* deleteRestoredBundle(threadId); + } + yield* sql`DELETE FROM thread_cleanup_queue WHERE thread_id = ${threadId}`; + return true; + } + if (shell !== undefined) return false; + const pending = yield* sql`SELECT 1 FROM thread_cleanup_queue WHERE thread_id = ${threadId}`; + return pending.length === 0; + }); + + const deleteImpl = Effect.fn("deleteThreadPermanentlyImpl")(function* ( + threadId: ThreadId, + quiesce: Effect.Effect, + ) { + if (yield* skipObsoleteDelete(threadId)) return; + yield* sql.unsafe( + `INSERT OR IGNORE INTO thread_cleanup_queue (thread_id, reason, created_at) + VALUES (?, 'deleted', CURRENT_TIMESTAMP)`, + [threadId], + ); + yield* quiesce; + // Keep the hot rows or cold chunks available until external cleanup has + // succeeded so an interrupted delete can recover exact attachment owners. + yield* removeAttachments(threadId); + yield* removeProviderLogsImpl(threadId); + const recreated = yield* sql.withTransaction( + Effect.gen(function* () { + const liveShell = yield* sql` + SELECT 1 FROM projection_threads WHERE thread_id = ${threadId} AND deleted_at IS NULL + `; + if (liveShell.length > 0) return true; + yield* sql.unsafe( + `UPDATE projection_thread_proposed_plans + SET implementation_thread_id = NULL + WHERE implementation_thread_id = ?`, + [threadId], + ); + yield* sql.unsafe( + `UPDATE projection_turns + SET source_proposed_plan_thread_id = NULL, source_proposed_plan_id = NULL + WHERE source_proposed_plan_thread_id = ?`, + [threadId], + ); + for (const [table, predicate] of [...THREAD_TABLES].toReversed()) { + yield* sql.unsafe(`DELETE FROM ${table} WHERE ${predicate}`, [threadId]); + } + yield* sql.unsafe(`DELETE FROM projection_threads WHERE thread_id = ?`, [threadId]); + yield* sql.unsafe( + `DELETE FROM ${ARCHIVE_SCHEMA}.archive_thread_chunks WHERE thread_id = ?`, + [threadId], + ); + yield* sql.unsafe(`DELETE FROM ${ARCHIVE_SCHEMA}.archive_threads WHERE thread_id = ?`, [ + threadId, + ]); + yield* sql.unsafe(`DELETE FROM thread_archive_manifests WHERE thread_id = ?`, [threadId]); + return false; + }), + ); + if (recreated) { + yield* skipObsoleteDelete(threadId); + return; + } + yield* reclaimFreePages(); + yield* sql.unsafe(`DELETE FROM thread_cleanup_queue WHERE thread_id = ?`, [threadId]); + }); + + const listIds = (query: string, operation: ThreadColdStorageOperation) => + sql.unsafe(query).pipe( + Effect.map((rows) => + (rows as ReadonlyArray).map((row) => ThreadId.make(String(row.thread_id))), + ), + Effect.mapError((cause) => ThreadColdStorageError.normalize(operation, undefined, cause)), + ); + + const pendingArchiveThreadIds = listIds( + `SELECT thread_id + FROM ( + SELECT thread_id, archived_at + FROM thread_archive_manifests + WHERE status IN ('pending', 'archiving', 'cleanup_pending') + UNION ALL + SELECT thread_archive_manifests.thread_id, thread_archive_manifests.archived_at + FROM thread_archive_manifests + INNER JOIN projection_threads + ON projection_threads.thread_id = thread_archive_manifests.thread_id + AND projection_threads.deleted_at IS NULL + AND (projection_threads.archived_at = thread_archive_manifests.archived_at + OR projection_threads.archived_at IS NULL) + WHERE thread_archive_manifests.status = 'restored' + UNION ALL + SELECT projection_threads.thread_id, projection_threads.archived_at + FROM projection_threads + WHERE projection_threads.archived_at IS NOT NULL + AND projection_threads.deleted_at IS NULL + AND NOT EXISTS ( + SELECT 1 FROM thread_archive_manifests + WHERE thread_archive_manifests.thread_id = projection_threads.thread_id + ) + ) + ORDER BY archived_at ASC, thread_id ASC`, + "list-pending-archives", + ); + + const pendingDeleteThreadIds = listIds( + `SELECT thread_id + FROM ( + SELECT thread_id, created_at + FROM thread_cleanup_queue + WHERE reason = 'deleted' + UNION ALL + SELECT thread_id, deleted_at AS created_at + FROM projection_threads + WHERE deleted_at IS NOT NULL + ) + GROUP BY thread_id + ORDER BY MIN(created_at) ASC, thread_id ASC`, + "list-pending-deletes", + ); + + const compactLegacyStorageImpl = Effect.fn("compactLegacyThreadStorage")(function* () { + const rows = (yield* sql.unsafe( + `SELECT status FROM thread_storage_maintenance + WHERE task = 'compact-legacy-thread-storage'`, + )) as ReadonlyArray; + if (rows[0]?.status === "complete") return; + + const [pendingArchives, pendingDeletes] = yield* Effect.all([ + pendingArchiveThreadIds, + pendingDeleteThreadIds, + ]); + if (pendingArchives.length > 0 || pendingDeletes.length > 0) return; + + yield* sql.unsafe( + `UPDATE thread_storage_maintenance + SET status = 'running', updated_at = CURRENT_TIMESTAMP, error = NULL + WHERE task = 'compact-legacy-thread-storage'`, + ); + yield* sql.unsafe("PRAGMA wal_checkpoint(TRUNCATE)"); + yield* sql.unsafe("PRAGMA main.auto_vacuum = INCREMENTAL"); + yield* sql.unsafe("VACUUM main"); + yield* reclaimFreePages(); + yield* sql.unsafe( + `UPDATE thread_storage_maintenance + SET status = 'complete', updated_at = CURRENT_TIMESTAMP, error = NULL + WHERE task = 'compact-legacy-thread-storage'`, + ); + }); + + const getTreeSemaphore = (threadId: ThreadId) => + Effect.flatMap(resolveTreeRoot(threadId), (rootThreadId) => + SynchronizedRef.modifyEffect(threadLocksRef, (current) => { + const existing = current.get(rootThreadId); + if (existing) { + const next = new Map(current); + next.set(rootThreadId, { ...existing, users: existing.users + 1 }); + return Effect.succeed([ + { rootThreadId, semaphore: existing.semaphore } satisfies AcquiredThreadLock, + next, + ] as const); + } + return Semaphore.make(1).pipe( + Effect.map((semaphore) => { + const next = new Map(current); + next.set(rootThreadId, { semaphore, users: 1 }); + return [{ rootThreadId, semaphore } satisfies AcquiredThreadLock, next] as const; + }), + ); + }), + ); + + const releaseTreeSemaphore = (acquired: AcquiredThreadLock) => + SynchronizedRef.update(threadLocksRef, (current) => { + const existing = current.get(acquired.rootThreadId); + if (!existing || existing.semaphore !== acquired.semaphore) return current; + const next = new Map(current); + if (existing.users === 1) { + next.delete(acquired.rootThreadId); + } else { + next.set(acquired.rootThreadId, { ...existing, users: existing.users - 1 }); + } + return next; + }); + + const wrap = ( + operation: ThreadColdStorageOperation, + threadId: ThreadId, + effect: Effect.Effect, + ) => + Effect.acquireUseRelease( + getTreeSemaphore(threadId), + ({ semaphore }) => semaphore.withPermit(effect), + releaseTreeSemaphore, + ).pipe( + Effect.mapError((cause) => ThreadColdStorageError.normalize(operation, threadId, cause)), + ); + + // Release the recorded claim even if root lookup fails before the tree lock + // is acquired. An older operation must not release a replacement claim. + const releaseRestoreClaim = (threadId: ThreadId, effect: Effect.Effect) => + Effect.suspend(() => { + const claim = restoreClaims.get(threadId); + return effect.pipe( + Effect.ensuring( + Effect.sync(() => { + if (!claim || restoreClaims.get(threadId) !== claim) return; + restoreClaims.delete(threadId); + if ( + ![...restoreClaims.values()].some( + (other) => other.rootThreadId === claim.rootThreadId, + ) + ) { + activeRestoreRoots.delete(claim.rootThreadId); + } + }), + ), + ); + }); + + return { + archiveThread: (threadId: ThreadId, quiesce: Effect.Effect = Effect.void) => { + const operation = "archive"; + return wrap( + operation, + threadId, + archiveImpl( + threadId, + false, + quiesce.pipe( + Effect.mapError((cause) => + ThreadColdStorageError.normalize(operation, threadId, cause), + ), + ), + ), + ); + }, + restoreTree: (threadId) => wrap("restore", threadId, restoreTreeImpl(threadId)), + rollbackRestoreTree: (threadId) => + releaseRestoreClaim( + threadId, + wrap("rollback-restore", threadId, rollbackRestoreTreeImpl(threadId)), + ), + finishRestoreTree: (threadId) => + releaseRestoreClaim( + threadId, + wrap("finish-restore", threadId, finishRestoreTreeImpl(threadId)), + ), + deleteThread: (threadId: ThreadId, quiesce: Effect.Effect = Effect.void) => + wrap( + "delete", + threadId, + deleteImpl( + threadId, + quiesce.pipe( + Effect.mapError((cause) => ThreadColdStorageError.normalize("delete", threadId, cause)), + ), + ), + ), + removeProviderLogs: (threadId) => + wrap("remove-provider-logs", threadId, removeProviderLogsImpl(threadId)), + compactLegacyStorage: compactLegacyStorageImpl().pipe( + Effect.mapError((cause) => + ThreadColdStorageError.normalize("compact-legacy-storage", undefined, cause), + ), + ), + listPendingArchiveThreadIds: pendingArchiveThreadIds, + listPendingDeleteThreadIds: pendingDeleteThreadIds, + } satisfies ThreadColdStorage["Service"]; +}); + +export const layer = Layer.effect(ThreadColdStorage, make); diff --git a/apps/server/src/orchestration/decider.delete.test.ts b/apps/server/src/orchestration/decider.delete.test.ts index fea36b5717fe..32ada9085406 100644 --- a/apps/server/src/orchestration/decider.delete.test.ts +++ b/apps/server/src/orchestration/decider.delete.test.ts @@ -6,6 +6,7 @@ import { ThreadId, type OrchestrationCommand, type OrchestrationEvent, + type OrchestrationReadModel, ProviderInstanceId, } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; @@ -102,6 +103,27 @@ const seedReadModel = Effect.gen(function* () { }); }); +function archiveThread(readModel: OrchestrationReadModel, threadId: ThreadId, index: number) { + const archivedAt = `2026-01-01T00:0${index}:00.000Z`; + return projectEvent(readModel, { + sequence: readModel.snapshotSequence + 1, + eventId: asEventId(`evt-thread-archive-${index}`), + aggregateKind: "thread", + aggregateId: threadId, + type: "thread.archived", + occurredAt: archivedAt, + commandId: asCommandId(`cmd-thread-archive-${index}`), + causationEventId: null, + correlationId: asCommandId(`cmd-thread-archive-${index}`), + metadata: {}, + payload: { + threadId, + archivedAt, + updatedAt: archivedAt, + }, + }); +} + type PlannedEvent = Omit; function normalizeDeleteEvent(event: PlannedEvent | ReadonlyArray) { @@ -154,6 +176,80 @@ it.layer(NodeServices.layer)("decider deletion flows", (it) => { }), ); + it.effect("rejects deleteArchivedThreads when the project still has a live thread", () => + Effect.gen(function* () { + const readModel = yield* seedReadModel; + const withArchivedThread = yield* archiveThread(readModel, asThreadId("thread-delete-1"), 1); + expect( + withArchivedThread.threads.find((thread) => thread.id === "thread-delete-2")?.archivedAt, + ).toBeNull(); + + const error = yield* Effect.flip( + decideOrchestrationCommand({ + command: { + type: "project.delete", + commandId: asCommandId("cmd-project-delete-archived-only-mixed"), + projectId: asProjectId("project-delete"), + deleteArchivedThreads: true, + }, + readModel: withArchivedThread, + }), + ); + + expect(error._tag).toBe("OrchestrationCommandInvariantError"); + expect(error.message).toContain("cannot be deleted without force=true"); + }), + ); + + it.effect("rejects deleting archived threads without explicit opt-in", () => + Effect.gen(function* () { + let readModel = yield* seedReadModel; + for (const [index, threadId] of ["thread-delete-1", "thread-delete-2"].entries()) { + readModel = yield* archiveThread(readModel, asThreadId(threadId), index + 1); + } + + const error = yield* Effect.flip( + decideOrchestrationCommand({ + command: { + type: "project.delete", + commandId: asCommandId("cmd-project-delete-archived-no-opt-in"), + projectId: asProjectId("project-delete"), + }, + readModel, + }), + ); + + expect(error._tag).toBe("OrchestrationCommandInvariantError"); + expect(error.message).toContain("cannot be deleted without force=true"); + }), + ); + + it.effect("deletes a project containing only archived threads without force", () => + Effect.gen(function* () { + let readModel = yield* seedReadModel; + for (const [index, threadId] of ["thread-delete-1", "thread-delete-2"].entries()) { + readModel = yield* archiveThread(readModel, asThreadId(threadId), index + 1); + } + + const result = yield* decideOrchestrationCommand({ + command: { + type: "project.delete", + commandId: asCommandId("cmd-project-delete-archived-only"), + projectId: asProjectId("project-delete"), + deleteArchivedThreads: true, + }, + readModel, + }); + const events = Array.isArray(result) ? result : [result]; + + expect(events.map((event) => event.type)).toEqual([ + "thread.deleted", + "thread.deleted", + "project.deleted", + ]); + }), + ); + it.effect("reuses thread.delete semantics when force-deleting a non-empty project", () => Effect.gen(function* () { const readModel = yield* seedReadModel; diff --git a/apps/server/src/orchestration/decider.ts b/apps/server/src/orchestration/decider.ts index 21df39a162e6..d8f7757e7408 100644 --- a/apps/server/src/orchestration/decider.ts +++ b/apps/server/src/orchestration/decider.ts @@ -330,20 +330,23 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" command, projectId: command.projectId, }); - const activeThreads = listThreadsByProjectId(readModel, command.projectId).filter( + const undeletedThreads = listThreadsByProjectId(readModel, command.projectId).filter( (thread) => thread.deletedAt === null, ); - if (activeThreads.length > 0 && command.force !== true) { + const hasLiveThreads = undeletedThreads.some((thread) => thread.archivedAt === null); + const canDeleteNonEmptyProject = + command.force === true || (command.deleteArchivedThreads === true && !hasLiveThreads); + if (undeletedThreads.length > 0 && !canDeleteNonEmptyProject) { return yield* new OrchestrationCommandInvariantError({ commandType: command.type, detail: `Project '${command.projectId}' is not empty and cannot be deleted without force=true.`, }); } - if (activeThreads.length > 0) { + if (undeletedThreads.length > 0) { return yield* decideCommandSequence({ readModel, commands: [ - ...activeThreads.map( + ...undeletedThreads.map( (thread): Extract => ({ type: "thread.delete", commandId: command.commandId, diff --git a/apps/server/src/orchestration/testUtils/orchestrationEngine.ts b/apps/server/src/orchestration/testUtils/orchestrationEngine.ts new file mode 100644 index 000000000000..9d25d97899d4 --- /dev/null +++ b/apps/server/src/orchestration/testUtils/orchestrationEngine.ts @@ -0,0 +1,27 @@ +import * as Effect from "effect/Effect"; +import * as Stream from "effect/Stream"; + +import { + OrchestrationEngineService, + type OrchestrationEngineShape, +} from "../Services/OrchestrationEngine.ts"; + +/** + * Supply inert defaults for tests outside the orchestration engine module. + * Event-stream overrides also drive the scoped subscription by default. + */ +export function makeTestOrchestrationEngine( + overrides: Partial = {}, +): OrchestrationEngineShape { + const streamDomainEvents = overrides.streamDomainEvents ?? Stream.empty; + return OrchestrationEngineService.of({ + readEvents: () => Stream.empty, + readThreadEvents: () => Stream.empty, + getThreadReplayStats: () => Effect.die("unused thread replay stats"), + dispatch: () => Effect.succeed({ sequence: 0 }), + streamDomainEvents, + subscribeDomainEvents: Effect.succeed(streamDomainEvents), + latestSequence: Effect.succeed(0), + ...overrides, + }); +} diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index c837ae3f4c50..1b0339ff4e0a 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -66,6 +66,7 @@ import Migration0051 from "./Migrations/051_ProjectionThreadMessageContext.ts"; import Migration0052 from "./Migrations/052_ProjectionThreadTitleState.ts"; import Migration0053 from "./Migrations/053_PullRequestFilesViewed.ts"; import Migration0054 from "./Migrations/054_ProjectionThreadsAutoSettleDisabledAt.ts"; +import Migration0055 from "./Migrations/055_ThreadStorageLifecycle.ts"; /** * Migration loader with all migrations defined inline. @@ -132,6 +133,7 @@ const migrationEntries = [ [52, "ProjectionThreadTitleState", Migration0052], [53, "PullRequestFilesViewed", Migration0053], [54, "ProjectionThreadsAutoSettleDisabledAt", Migration0054], + [55, "ThreadStorageLifecycle", Migration0055], ] as const; export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); diff --git a/apps/server/src/persistence/Migrations/055_ThreadStorageLifecycle.test.ts b/apps/server/src/persistence/Migrations/055_ThreadStorageLifecycle.test.ts new file mode 100644 index 000000000000..6c1b208993bc --- /dev/null +++ b/apps/server/src/persistence/Migrations/055_ThreadStorageLifecycle.test.ts @@ -0,0 +1,69 @@ +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +import { runMigrations } from "../Migrations.ts"; +import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; + +const layer = it.layer(Layer.mergeAll(NodeSqliteClient.layer({ filename: ":memory:" }))); + +layer("055 thread storage lifecycle", (it) => { + it.effect("creates the final schema and queues existing threads", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* runMigrations({ toMigrationInclusive: 54 }); + + const insertThread = (input: { + readonly threadId: string; + readonly archivedAt: string | null; + readonly deletedAt: string | null; + }) => sql` + INSERT INTO projection_threads ( + thread_id, project_id, title, model_selection_json, runtime_mode, + interaction_mode, created_at, updated_at, archived_at, deleted_at + ) VALUES ( + ${input.threadId}, 'project-1', ${input.threadId}, + '{"instanceId":"codex","model":"gpt-5.5","options":[]}', + 'full-access', 'default', '2026-07-01T00:00:00.000Z', + '2026-07-01T00:00:00.000Z', ${input.archivedAt}, ${input.deletedAt} + ) + `; + + yield* insertThread({ + threadId: "archived-thread", + archivedAt: "2026-07-02T00:00:00.000Z", + deletedAt: null, + }); + yield* insertThread({ + threadId: "deleted-thread", + archivedAt: null, + deletedAt: "2026-07-03T00:00:00.000Z", + }); + + const executed = yield* runMigrations(); + assert.deepStrictEqual(executed, [[55, "ThreadStorageLifecycle"]]); + + const manifests = yield* sql<{ + readonly threadId: string; + readonly rootThreadId: string; + readonly status: string; + }>` + SELECT thread_id AS "threadId", root_thread_id AS "rootThreadId", status + FROM thread_archive_manifests + `; + assert.deepStrictEqual(manifests, [ + { + threadId: "archived-thread", + rootThreadId: "archived-thread", + status: "pending", + }, + ]); + + const cleanup = yield* sql<{ readonly threadId: string; readonly reason: string }>` + SELECT thread_id AS "threadId", reason FROM thread_cleanup_queue + `; + assert.deepStrictEqual(cleanup, [{ threadId: "deleted-thread", reason: "deleted" }]); + }), + ); +}); diff --git a/apps/server/src/persistence/Migrations/055_ThreadStorageLifecycle.ts b/apps/server/src/persistence/Migrations/055_ThreadStorageLifecycle.ts new file mode 100644 index 000000000000..1b971b3a72ae --- /dev/null +++ b/apps/server/src/persistence/Migrations/055_ThreadStorageLifecycle.ts @@ -0,0 +1,81 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +/** + * Creates the complete cold-storage lifecycle schema and queues existing + * archived and deleted threads for background processing. + */ +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + yield* sql` + CREATE TABLE thread_archive_manifests ( + thread_id TEXT PRIMARY KEY, + root_thread_id TEXT NOT NULL, + status TEXT NOT NULL, + archive_version INTEGER NOT NULL DEFAULT 1, + archived_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + original_bytes INTEGER NOT NULL DEFAULT 0, + compressed_bytes INTEGER NOT NULL DEFAULT 0, + error TEXT + ) + `; + + yield* sql` + CREATE INDEX idx_thread_archive_manifests_root_status + ON thread_archive_manifests(root_thread_id, status, archived_at, thread_id) + `; + + yield* sql` + CREATE TABLE thread_storage_maintenance ( + task TEXT PRIMARY KEY, + status TEXT NOT NULL, + updated_at TEXT NOT NULL, + error TEXT + ) + `; + + yield* sql` + INSERT INTO thread_storage_maintenance (task, status, updated_at) + VALUES ('compact-legacy-thread-storage', 'pending', CURRENT_TIMESTAMP) + `; + + yield* sql` + CREATE TABLE thread_cleanup_queue ( + thread_id TEXT PRIMARY KEY, + reason TEXT NOT NULL, + created_at TEXT NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0, + last_error TEXT + ) + `; + + yield* sql` + INSERT INTO thread_archive_manifests ( + thread_id, + root_thread_id, + status, + archive_version, + archived_at, + updated_at + ) + SELECT + thread_id, + thread_id, + 'pending', + 1, + archived_at, + CURRENT_TIMESTAMP + FROM projection_threads + WHERE archived_at IS NOT NULL + AND deleted_at IS NULL + `; + + yield* sql` + INSERT INTO thread_cleanup_queue (thread_id, reason, created_at) + SELECT thread_id, 'deleted', CURRENT_TIMESTAMP + FROM projection_threads + WHERE deleted_at IS NOT NULL + `; +}); diff --git a/apps/server/src/project/AgentSessionImporter.test.ts b/apps/server/src/project/AgentSessionImporter.test.ts index 2438edca8b1b..cc3340288974 100644 --- a/apps/server/src/project/AgentSessionImporter.test.ts +++ b/apps/server/src/project/AgentSessionImporter.test.ts @@ -1,3 +1,4 @@ +import * as ThreadColdStorage from "../orchestration/ThreadColdStorage.ts"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { describe, expect, it, vi } from "@effect/vitest"; import { @@ -563,6 +564,7 @@ const integrationRuntimeRepository = ProviderSessionRuntime.layer.pipe( ); const integrationLayer = Layer.mergeAll( OrchestrationEngineLive.pipe( + Layer.provide(ThreadColdStorage.noOpLayer), Layer.provide(OrchestrationProjectionSnapshotQueryLive), Layer.provide(OrchestrationProjectionPipelineLive), ), diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.test.ts b/apps/server/src/provider/Layers/ClaudeAdapter.test.ts index a91d82cef9e0..966657dcfb1b 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.test.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.test.ts @@ -3401,6 +3401,121 @@ describe("ClaudeAdapterLive", () => { ); }); + it.effect( + "stopSession settles the first Claude turn before its archived cursor is resumed", + () => { + const harness = makeHarness(); + const resumedHarness = makeHarness(); + let persistedSessionId: string | undefined; + let persistedPrompt: string | undefined; + let closeCallsAtInterrupt: number | undefined; + let promptReadError: unknown; + return Effect.gen(function* () { + const archivedCursor = yield* Effect.gen(function* () { + const adapter = yield* ClaudeAdapter; + const session = yield* adapter.startSession({ + threadId: THREAD_ID, + provider: ProviderDriverKind.make("claudeAgent"), + runtimeMode: "full-access", + }); + yield* adapter.sendTurn({ threadId: THREAD_ID, input: "first prompt", attachments: [] }); + // ProviderService saves this cursor before archive stops the session. + const cursor = (yield* adapter.listSessions())[0]?.resumeCursor; + assert.ok(cursor); + const input = harness.getLastCreateQueryInput(); + harness.query.interrupt = async () => { + closeCallsAtInterrupt = harness.query.closeCalls; + persistedSessionId = input?.options.sessionId; + try { + persistedPrompt = await readFirstPromptText(input); + } catch (error) { + promptReadError = error; + } + harness.query.emit({ + type: "result", + subtype: "error_during_execution", + is_error: false, + errors: ["Error: Request was aborted."], + session_id: persistedSessionId, + uuid: "result-archive-interrupted", + } as unknown as SDKMessage); + }; + const completed = yield* adapter.streamEvents.pipe( + Stream.filter((event) => event.type === "turn.completed"), + Stream.take(1), + Stream.runCollect, + Effect.forkChild, + ); + + yield* adapter.stopSession(session.threadId); + + assert.equal(closeCallsAtInterrupt, 0); + assert.equal(promptReadError, undefined); + assert.equal(persistedPrompt, "first prompt"); + assert.equal(harness.query.closeCalls, 1); + const [event] = yield* Fiber.join(completed); + assert.equal(event?.type, "turn.completed"); + if (event?.type === "turn.completed") { + assert.equal(event.payload.state, "interrupted"); + // The SDK result, rather than synthetic shutdown, completed the turn. + assert.notEqual(event.payload.errorMessage, "Session stopped."); + } + assert.equal(yield* adapter.hasSession(THREAD_ID), false); + return cursor; + }).pipe(Effect.provide(harness.layer)); + + yield* Effect.gen(function* () { + const adapter = yield* ClaudeAdapter; + yield* adapter.startSession({ + threadId: THREAD_ID, + provider: ProviderDriverKind.make("claudeAgent"), + runtimeMode: "full-access", + resumeCursor: archivedCursor, + }); + yield* adapter.sendTurn({ + threadId: THREAD_ID, + input: "after unarchive", + attachments: [], + }); + const input = resumedHarness.getLastCreateQueryInput(); + assert.ok(persistedSessionId); + assert.equal(input?.options.resume, persistedSessionId); + assert.equal(yield* Effect.promise(() => readFirstPromptText(input)), "after unarchive"); + }).pipe(Effect.provide(resumedHarness.layer)); + }).pipe(Effect.provideService(Random.Random, makeDeterministicRandomService())); + }, + ); + + it.effect("stopSession bounds archive shutdown when Claude never aborts the turn", () => { + const harness = makeHarness(); + return Effect.gen(function* () { + const adapter = yield* ClaudeAdapter; + yield* adapter.startSession({ + threadId: THREAD_ID, + provider: ProviderDriverKind.make("claudeAgent"), + runtimeMode: "full-access", + }); + yield* adapter.sendTurn({ threadId: THREAD_ID, input: "first prompt", attachments: [] }); + const interrupted = Promise.withResolvers(); + harness.query.interrupt = () => { + interrupted.resolve(); + return new Promise(() => {}); + }; + + const stop = yield* adapter.stopSession(THREAD_ID).pipe(Effect.forkChild); + yield* Effect.promise(() => interrupted.promise); + assert.equal(harness.query.closeCalls, 0); + yield* TestClock.adjust("3 seconds"); + yield* Fiber.join(stop); + + assert.equal(harness.query.closeCalls, 1); + assert.equal(yield* adapter.hasSession(THREAD_ID), false); + }).pipe( + Effect.provideService(Random.Random, makeDeterministicRandomService()), + Effect.provide(harness.layer), + ); + }); + it.effect("keeps the session available when process close fails", () => { const harness = makeHarness(); return Effect.gen(function* () { diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.ts b/apps/server/src/provider/Layers/ClaudeAdapter.ts index e4b4bea1d02f..d18c883b385f 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.ts @@ -5617,6 +5617,9 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( const stopSession: ClaudeAdapterShape["stopSession"] = Effect.fn("stopSession")( function* (threadId) { const context = yield* requireSession(threadId); + // Cold archive retains the resume cursor, so let Claude save the active + // prompt before closing just as an explicit turn interrupt does. + yield* settleInterruptedTurn(context); yield* stopSessionInternal(context, { emitExitEvent: true, }); diff --git a/apps/server/src/provider/Layers/EventNdjsonLogger.test.ts b/apps/server/src/provider/Layers/EventNdjsonLogger.test.ts index e87372e03381..9cfbf3a34f5d 100644 --- a/apps/server/src/provider/Layers/EventNdjsonLogger.test.ts +++ b/apps/server/src/provider/Layers/EventNdjsonLogger.test.ts @@ -242,6 +242,40 @@ describe("EventNdjsonLogger", () => { }), ); + it.effect("flushes one thread before destructive cleanup without closing the store", () => + Effect.gen(function* () { + const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); + const basePath = NodePath.join(tempDir, "events.log"); + const archivedPath = ownedLogPath(basePath, "thread-archived"); + const activePath = ownedLogPath(basePath, "thread-active"); + + try { + const store = yield* makeEventNdjsonLogStore(basePath, { batchWindowMs: 1_000 }); + const native = store.logger("native"); + const canonical = store.logger("canonical"); + + yield* native.write({ id: "before-archive" }, ThreadId.make("thread-archived")); + assert.equal(NodeFS.existsSync(archivedPath), false); + + assert.exists(native.closeThread); + if (!native.closeThread) return; + yield* native.closeThread(ThreadId.make("thread-archived")); + assert.equal(NodeFS.existsSync(archivedPath), true); + + yield* canonical.write( + { type: "item.completed", id: "still-active" }, + ThreadId.make("thread-active"), + ); + yield* store.close(); + + assert.include(NodeFS.readFileSync(archivedPath, "utf8"), '"id":"before-archive"'); + assert.include(NodeFS.readFileSync(activePath, "utf8"), '"id":"still-active"'); + } finally { + NodeFS.rmSync(tempDir, { recursive: true, force: true }); + } + }), + ); + it.effect("flushes an active batch without a permanent polling loop", () => Effect.gen(function* () { const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); diff --git a/apps/server/src/provider/Layers/EventNdjsonLogger.ts b/apps/server/src/provider/Layers/EventNdjsonLogger.ts index dda83fe4a007..08192e9c1150 100644 --- a/apps/server/src/provider/Layers/EventNdjsonLogger.ts +++ b/apps/server/src/provider/Layers/EventNdjsonLogger.ts @@ -66,12 +66,14 @@ export type EventNdjsonStream = "native" | "canonical" | "orchestration"; export interface EventNdjsonLogger { readonly filePath: string; readonly write: (event: unknown, threadId: ThreadId | null) => Effect.Effect; + readonly closeThread?: (threadId: ThreadId) => Effect.Effect; readonly close: () => Effect.Effect; } export interface EventNdjsonLogStore { readonly filePath: string; readonly logger: (stream: EventNdjsonStream) => EventNdjsonLogger; + readonly closeThread: (threadId: ThreadId) => Effect.Effect; readonly close: () => Effect.Effect; } @@ -734,6 +736,17 @@ export const makeEventNdjsonLogStore = Effect.fnUntraced(function* ( yield* Scope.close(timerScope, Exit.void); }); + const closeThread = Effect.fnUntraced(function* (threadId: ThreadId) { + const threadSegment = resolveThreadSegment(threadId); + yield* flush(false, false); + yield* SynchronizedRef.update(stateRef, (state) => { + if (!state.sinks.has(threadSegment)) return state; + const sinks = new Map(state.sinks); + sinks.delete(threadSegment); + return { ...state, sinks }; + }); + }); + const loggerViews = new Map(); const logger = (stream: EventNdjsonStream): EventNdjsonLogger => { const existing = loggerViews.get(stream); @@ -781,12 +794,17 @@ export const makeEventNdjsonLogStore = Effect.fnUntraced(function* ( } }); - const view = { filePath, write, close: () => Effect.void } satisfies EventNdjsonLogger; + const view = { + filePath, + write, + closeThread, + close: () => Effect.void, + } satisfies EventNdjsonLogger; loggerViews.set(stream, view); return view; }; - return { filePath, logger, close } satisfies EventNdjsonLogStore; + return { filePath, logger, closeThread, close } satisfies EventNdjsonLogStore; }); export const makeEventNdjsonLogger = Effect.fnUntraced(function* ( diff --git a/apps/server/src/provider/acp/AcpNativeLogging.test.ts b/apps/server/src/provider/acp/AcpNativeLogging.test.ts index 84926fbe1d61..3ad63405f764 100644 --- a/apps/server/src/provider/acp/AcpNativeLogging.test.ts +++ b/apps/server/src/provider/acp/AcpNativeLogging.test.ts @@ -21,6 +21,7 @@ nodeServicesIt("ACP native logging", (it) => { const nativeEventLogger: EventNdjsonLogger = { filePath: "/tmp/provider-native.ndjson", write: (event) => Effect.sync(() => void records.push(event)), + closeThread: () => Effect.void, close: () => Effect.void, }; const makeLogger = yield* makeAcpNativeLoggerFactory(); diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 221941aa7833..58613ecdff4b 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -9632,6 +9632,65 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(NodeHttpServer.layerTest), TestClock.withLive), ); + it.effect("buffers archived shell removals published while replay catches up", () => + Effect.gen(function* () { + const liveEvents = yield* PubSub.unbounded(); + const archivedEvent = { + sequence: 2, + eventId: EventId.make("event-archived"), + aggregateKind: "thread", + aggregateId: defaultThreadId, + occurredAt: "2026-01-01T00:00:01.000Z", + commandId: CommandId.make("command-archive"), + causationEventId: null, + correlationId: CommandId.make("command-archive"), + metadata: {}, + type: "thread.archived", + payload: { + threadId: defaultThreadId, + archivedAt: "2026-01-01T00:00:01.000Z", + updatedAt: "2026-01-01T00:00:01.000Z", + }, + } satisfies Extract; + + yield* buildAppUnderTest({ + layers: { + orchestrationEngine: { + streamDomainEvents: Stream.fromPubSub(liveEvents), + latestSequence: Effect.succeed(2), + readEvents: () => + Stream.unwrap( + Effect.gen(function* () { + yield* Effect.sleep("25 millis"); + yield* PubSub.publish(liveEvents, archivedEvent); + return Stream.empty; + }), + ), + }, + }, + }); + + const wsUrl = yield* getWsServerUrl("/ws"); + const items = yield* Effect.scoped( + withWsRpcClient(wsUrl, (client) => + client[ORCHESTRATION_WS_METHODS.subscribeShell]({ + // Represents the authoritative shell snapshot already loaded over HTTP. + afterSequence: 1, + requestCompletionMarker: true, + }).pipe(Stream.take(2), Stream.runCollect), + ), + ).pipe(Effect.timeout("2 seconds")); + + const item = items[0]; + if (item?.kind !== "thread-removed") { + assert.fail(`Expected thread-removed, received ${item?.kind ?? "no item"}`); + } + assert.equal(item.sequence, 2); + assert.equal(item.threadId, defaultThreadId); + assert.deepEqual(items[1], { kind: "synchronized" }); + }).pipe(Effect.provide(NodeHttpServer.layerTest), TestClock.withLive), + ); + it.effect("buffers thread events published while the initial snapshot loads", () => Effect.gen(function* () { const thread = makeDefaultOrchestrationReadModel().threads[0]!; @@ -12902,26 +12961,27 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); - it.effect("drains deletion cleanup through the re-created thread event", () => + it.effect("waits for same-id deletion before direct and bootstrap creation dispatch", () => Effect.gen(function* () { // A draft retry reuses the thread id its failed bootstrap deleted. The // deletion reactor stops sessions and closes terminals by that id, so - // both thread.create paths use the created event as a fence, then drain - // cleanup before handing the new incarnation to resource-owning work. + // both paths must wait before a replacement can publish its creation. const trace: Array = []; - const drainRequested = yield* Deferred.make(); - const cleanupDone = yield* Deferred.make(); + let drainRequested = yield* Deferred.make(); + let cleanupDone = yield* Deferred.make(); yield* buildAppUnderTest({ layers: { threadDeletionReactor: { - drainThrough: (sequence) => + drainThrough: (sequence, cleanupThreadId) => Effect.gen(function* () { + assert.strictEqual(cleanupThreadId, ThreadId.make("thread-retry-after-delete")); trace.push(`drain:${sequence}`); yield* Deferred.succeed(drainRequested, undefined); yield* Deferred.await(cleanupDone); }), }, orchestrationEngine: { + latestSequence: Effect.succeed(41), dispatch: (command) => Effect.sync(() => { trace.push(command.type); @@ -12955,17 +13015,18 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }), ); yield* Deferred.await(drainRequested); - assert.deepEqual(trace, ["thread.create", "drain:1"]); + assert.deepEqual(trace, ["drain:41"]); yield* Deferred.succeed(cleanupDone, undefined); yield* Fiber.join(directCreate); }), ), ); - assert.deepEqual(trace, ["thread.create", "drain:1"]); + assert.deepEqual(trace, ["drain:41", "thread.create"]); - // Cleanup is already released; the bootstrap path must still drain - // between creating the thread and starting its turn. + // A second connection's bootstrap retry waits on its own unfinished cleanup. trace.length = 0; + drainRequested = yield* Deferred.make(); + cleanupDone = yield* Deferred.make(); yield* Effect.scoped( withWsRpcClient(wsUrl, (client) => Effect.gen(function* () { @@ -12999,13 +13060,16 @@ it.layer(NodeServices.layer)("server router seam", (it) => { createdAt, }), ); + yield* Deferred.await(drainRequested); + assert.deepEqual(trace, ["drain:41"]); + yield* Deferred.succeed(cleanupDone, undefined); yield* Fiber.join(bootstrapCreate); }), ), ); assert.deepEqual(trace, [ + "drain:41", "thread.create", - "drain:1", "thread.message.user.append", "thread.turn.start", ]); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 43bba34bd8c5..88456ee01fc3 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -90,6 +90,7 @@ import { ProviderRuntimeIngestionLive } from "./orchestration/Layers/ProviderRun import { ProviderCommandReactorLive } from "./orchestration/Layers/ProviderCommandReactor.ts"; import { CheckpointReactorLive } from "./orchestration/Layers/CheckpointReactor.ts"; import { ThreadDeletionReactorLive } from "./orchestration/Layers/ThreadDeletionReactor.ts"; +import * as ThreadColdStorage from "./orchestration/ThreadColdStorage.ts"; import * as ThreadSettlementReactor from "./orchestration/ThreadSettlementReactor.ts"; import * as StorageCleanup from "./storageCleanup.ts"; import * as PullRequestSyncReactor from "./orchestration/PullRequestSyncReactor.ts"; @@ -468,12 +469,16 @@ const CloudManagedEndpointRuntimeLive = Layer.mergeAll( ), ); +const OrchestrationLayerWithColdStorageLive = OrchestrationLayerLive.pipe( + Layer.provideMerge(ThreadColdStorage.layer), +); + const ProviderRuntimeLayerLive = ProviderSessionReaperLive.pipe( // Subscribes to `account.rate-limits.updated` so usage bars track live // telemetry instead of waiting for the next status probe. Layer.provideMerge(ProviderUsageLimitsIngestionLive), Layer.provideMerge(ProviderLayerLive), - Layer.provideMerge(OrchestrationLayerLive), + Layer.provideMerge(OrchestrationLayerWithColdStorageLive), ); const ProviderInstallationRefreshLive = Layer.effectDiscard( diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index bcc19be00da5..107160b06dae 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -1425,7 +1425,11 @@ const makeWsRpcLayer = ( } if (bootstrap?.createThread) { - const created = yield* dispatchFromClient({ + yield* threadDeletionReactor.drainThrough( + yield* orchestrationEngine.latestSequence, + command.threadId, + ); + yield* dispatchFromClient({ type: "thread.create", commandId: yield* serverCommandId("bootstrap-thread-create"), threadId: command.threadId, @@ -1438,12 +1442,7 @@ const makeWsRpcLayer = ( worktreePath: bootstrap.createThread.worktreePath, createdAt: bootstrap.createThread.createdAt, }); - // The successful create is a fence in the engine command queue: - // every delete for the prior incarnation committed before it. - // Drain through that event before setup or turn start can own - // terminals and provider sessions under the reused thread id. createdThread = true; - yield* threadDeletionReactor.drainThrough(created.sequence); // Persist the send now rather than with the turn: the thread is // real from here on, so any client (or a reload) sees the message // while the worktree is still being prepared. The turn start @@ -1920,15 +1919,17 @@ const makeWsRpcLayer = ( const dispatchEffect = normalizedCommand.type === "thread.turn.start" && normalizedCommand.bootstrap ? dispatchBootstrapTurnStart(normalizedCommand) - : dispatchFromClient(normalizedCommand).pipe( - Effect.tap(({ sequence }) => - // Returning from thread.create is the handoff point at which - // clients may start resources for the new incarnation. Use - // its event sequence as the exact deletion-cleanup fence. - normalizedCommand.type === "thread.create" - ? threadDeletionReactor.drainThrough(sequence) - : Effect.void, - ), + : Effect.gen(function* () { + if (normalizedCommand.type === "thread.create") { + // Wait outside the command worker; durable cleanup failures + // still reach the engine's retryable creation gate. + yield* threadDeletionReactor.drainThrough( + yield* orchestrationEngine.latestSequence, + normalizedCommand.threadId, + ); + } + return yield* dispatchFromClient(normalizedCommand); + }).pipe( Effect.mapError((cause) => toDispatchCommandError(cause, "Failed to dispatch orchestration command"), ), diff --git a/apps/web/src/AppRoot.tsx b/apps/web/src/AppRoot.tsx index 857125c9fdaf..2cf740be7122 100644 --- a/apps/web/src/AppRoot.tsx +++ b/apps/web/src/AppRoot.tsx @@ -1,6 +1,7 @@ import { RouterProvider } from "@tanstack/react-router"; import { ElectronBrowserHost } from "./browser/ElectronBrowserHost"; +import { ComposerDraftArchiveObserver } from "./composerDraftArchiveObserver"; import { PreviewAutomationHosts } from "./components/preview/PreviewAutomationHosts"; import { QuitHoldOverlay } from "./components/QuitHoldOverlay"; import { AppAtomRegistryProvider } from "./rpc/atomRegistry"; @@ -14,6 +15,7 @@ import type { AppRouter } from "./router"; export function AppRoot({ router }: { readonly router: AppRouter }) { return ( + diff --git a/apps/web/src/authoritativeThreadLifecycle.test.ts b/apps/web/src/authoritativeThreadLifecycle.test.ts new file mode 100644 index 000000000000..4b133f5fca44 --- /dev/null +++ b/apps/web/src/authoritativeThreadLifecycle.test.ts @@ -0,0 +1,105 @@ +import { scopeThreadRef } from "@t3tools/client-runtime/environment"; +import { EnvironmentId, ThreadId } from "@t3tools/contracts"; +import { describe, expect, it } from "vite-plus/test"; + +import { reconcileAuthoritativeThreadRefs } from "./authoritativeThreadLifecycle"; + +const environmentOne = EnvironmentId.make("env-1"); +const environmentTwo = EnvironmentId.make("env-2"); +const activeRef = scopeThreadRef(environmentOne, ThreadId.make("thread-active")); +const archivedRef = scopeThreadRef(environmentOne, ThreadId.make("thread-archived")); +const deletedRef = scopeThreadRef(environmentTwo, ThreadId.make("thread-deleted")); + +describe("reconcileAuthoritativeThreadRefs", () => { + it("finds removed thread state only in live environment shells", () => { + expect( + reconcileAuthoritativeThreadRefs({ + previousActiveThreadRefs: [activeRef, archivedRef, deletedRef], + activeThreadRefs: [activeRef], + catalogEnvironmentIds: new Set([environmentOne, environmentTwo]), + liveEnvironmentIds: new Set([environmentOne]), + }), + ).toEqual({ + removedThreadRefs: [archivedRef], + nextActiveThreadRefs: [deletedRef, activeRef], + }); + }); + + it("retains the prior baseline until shell synchronization completes", () => { + const synchronizing = reconcileAuthoritativeThreadRefs({ + previousActiveThreadRefs: [activeRef, archivedRef], + activeThreadRefs: [activeRef], + catalogEnvironmentIds: new Set([environmentOne]), + liveEnvironmentIds: new Set(), + }); + expect(synchronizing).toEqual({ + removedThreadRefs: [], + nextActiveThreadRefs: [activeRef, archivedRef], + }); + + expect( + reconcileAuthoritativeThreadRefs({ + previousActiveThreadRefs: synchronizing.nextActiveThreadRefs, + activeThreadRefs: [activeRef], + catalogEnvironmentIds: new Set([environmentOne]), + liveEnvironmentIds: new Set([environmentOne]), + }), + ).toEqual({ + removedThreadRefs: [archivedRef], + nextActiveThreadRefs: [activeRef], + }); + }); + + it("removes retained thread state when an environment leaves the catalog", () => { + expect( + reconcileAuthoritativeThreadRefs({ + previousActiveThreadRefs: [activeRef, deletedRef], + activeThreadRefs: [activeRef], + catalogEnvironmentIds: new Set([environmentOne]), + liveEnvironmentIds: new Set([environmentOne]), + }), + ).toEqual({ + removedThreadRefs: [deletedRef], + nextActiveThreadRefs: [activeRef], + }); + }); + + it("retains the prior baseline while the environment catalog is unavailable", () => { + expect( + reconcileAuthoritativeThreadRefs({ + previousActiveThreadRefs: [activeRef, deletedRef], + activeThreadRefs: [], + catalogEnvironmentIds: null, + liveEnvironmentIds: new Set(), + }), + ).toEqual({ + removedThreadRefs: [], + nextActiveThreadRefs: [activeRef, deletedRef], + }); + }); + + it("refreshes live baselines while the environment catalog is unavailable", () => { + expect( + reconcileAuthoritativeThreadRefs({ + previousActiveThreadRefs: [archivedRef], + activeThreadRefs: [activeRef], + catalogEnvironmentIds: null, + liveEnvironmentIds: new Set([environmentOne]), + }), + ).toEqual({ + removedThreadRefs: [archivedRef], + nextActiveThreadRefs: [activeRef], + }); + }); + + it("does not report unchanged reordered thread references", () => { + expect( + reconcileAuthoritativeThreadRefs({ + previousActiveThreadRefs: [activeRef, archivedRef], + activeThreadRefs: [{ ...archivedRef }, { ...activeRef }], + catalogEnvironmentIds: new Set([environmentOne]), + liveEnvironmentIds: new Set([environmentOne]), + }).removedThreadRefs, + ).toEqual([]); + }); +}); diff --git a/apps/web/src/authoritativeThreadLifecycle.ts b/apps/web/src/authoritativeThreadLifecycle.ts new file mode 100644 index 000000000000..3bd60fdabb88 --- /dev/null +++ b/apps/web/src/authoritativeThreadLifecycle.ts @@ -0,0 +1,51 @@ +import { scopedThreadKey } from "@t3tools/client-runtime/environment"; +import type { EnvironmentId, ScopedThreadRef } from "@t3tools/contracts"; + +interface ReconcileAuthoritativeThreadRefsInput { + readonly previousActiveThreadRefs: readonly ScopedThreadRef[]; + readonly activeThreadRefs: readonly ScopedThreadRef[]; + readonly catalogEnvironmentIds: ReadonlySet | null; + readonly liveEnvironmentIds: ReadonlySet; +} + +interface ReconcileAuthoritativeThreadRefsResult { + readonly removedThreadRefs: readonly ScopedThreadRef[]; + readonly nextActiveThreadRefs: readonly ScopedThreadRef[]; +} + +/** + * Retains each environment's prior thread baseline until its shell becomes + * authoritative, then reports active threads that disappeared. Catalog + * removal is authoritative once catalog hydration completes too. + */ +export function reconcileAuthoritativeThreadRefs( + input: ReconcileAuthoritativeThreadRefsInput, +): ReconcileAuthoritativeThreadRefsResult { + const activeThreadKeys = new Set(input.activeThreadRefs.map(scopedThreadKey)); + const baselineEnvironmentIds = new Set( + input.previousActiveThreadRefs.map((threadRef) => threadRef.environmentId), + ); + const removedThreadRefs = input.previousActiveThreadRefs.filter( + (threadRef) => + (input.catalogEnvironmentIds !== null && + !input.catalogEnvironmentIds.has(threadRef.environmentId)) || + (input.liveEnvironmentIds.has(threadRef.environmentId) && + !activeThreadKeys.has(scopedThreadKey(threadRef))), + ); + const nextActiveThreadRefs = [ + ...input.previousActiveThreadRefs.filter( + (threadRef) => + (input.catalogEnvironmentIds === null || + input.catalogEnvironmentIds.has(threadRef.environmentId)) && + !input.liveEnvironmentIds.has(threadRef.environmentId), + ), + ...input.activeThreadRefs.filter( + (threadRef) => + (input.catalogEnvironmentIds === null || + input.catalogEnvironmentIds.has(threadRef.environmentId)) && + (input.liveEnvironmentIds.has(threadRef.environmentId) || + !baselineEnvironmentIds.has(threadRef.environmentId)), + ), + ]; + return { removedThreadRefs, nextActiveThreadRefs }; +} diff --git a/apps/web/src/browser/ElectronBrowserHost.tsx b/apps/web/src/browser/ElectronBrowserHost.tsx index de7e23603298..c29ed3aee67e 100644 --- a/apps/web/src/browser/ElectronBrowserHost.tsx +++ b/apps/web/src/browser/ElectronBrowserHost.tsx @@ -12,10 +12,12 @@ import { readPreviewAnnotationTheme } from "./annotationTheme"; import { useBrowserPointerStore } from "./browserPointerStore"; import { HostedBrowserWebview } from "./HostedBrowserWebview"; import { previewRuntimeTabId } from "./previewRuntimeTabId"; +import { usePreviewThreadLifecycleCleanup } from "./usePreviewThreadLifecycleCleanup"; export function ElectronBrowserHost() { const { resolvedTheme } = useTheme(); const previewByThreadKey = useActivePreviewSessions(); + usePreviewThreadLifecycleCleanup(); const sessions = useMemo( () => Object.entries(previewByThreadKey).flatMap(([threadKey, previewState]) => { diff --git a/apps/web/src/browser/usePreviewThreadLifecycleCleanup.ts b/apps/web/src/browser/usePreviewThreadLifecycleCleanup.ts new file mode 100644 index 000000000000..091a9dd32f28 --- /dev/null +++ b/apps/web/src/browser/usePreviewThreadLifecycleCleanup.ts @@ -0,0 +1,29 @@ +import { useEffect, useRef } from "react"; + +import { reconcileAuthoritativeThreadRefs } from "~/authoritativeThreadLifecycle"; +import { usePreviewMiniPlayerStore } from "~/previewMiniPlayerStore"; +import { removePreviewThread } from "~/previewStateStore"; +import { useLiveEnvironmentIds, useThreadRefs } from "~/state/entities"; +import { useEnvironments } from "~/state/environments"; + +export function usePreviewThreadLifecycleCleanup(): void { + const activeThreadRefs = useThreadRefs(); + const liveEnvironmentIds = useLiveEnvironmentIds(); + const { environmentIds: catalogEnvironmentIds, isReady: environmentCatalogReady } = + useEnvironments(); + const previousActiveThreadRefs = useRef(activeThreadRefs); + + useEffect(() => { + const reconciliation = reconcileAuthoritativeThreadRefs({ + previousActiveThreadRefs: previousActiveThreadRefs.current, + activeThreadRefs, + catalogEnvironmentIds: environmentCatalogReady ? catalogEnvironmentIds : null, + liveEnvironmentIds, + }); + previousActiveThreadRefs.current = reconciliation.nextActiveThreadRefs; + for (const threadRef of reconciliation.removedThreadRefs) { + removePreviewThread(threadRef); + usePreviewMiniPlayerStore.getState().removeThread(threadRef); + } + }, [activeThreadRefs, catalogEnvironmentIds, environmentCatalogReady, liveEnvironmentIds]); +} diff --git a/apps/web/src/components/CommandPalette.logic.ts b/apps/web/src/components/CommandPalette.logic.ts index b75ce84d1469..4134847cddd2 100644 --- a/apps/web/src/components/CommandPalette.logic.ts +++ b/apps/web/src/components/CommandPalette.logic.ts @@ -208,7 +208,7 @@ export function buildCommandPaletteProjectMetadata(input: { return { searchTerms, environmentLabels: [...environmentLabels] }; } -export function buildProjectActionItems(input: { +export interface BuildProjectActionItemsInput { projects: ReadonlyArray; valuePrefix: string; icon: (project: CommandPaletteProject) => ReactNode; @@ -216,7 +216,11 @@ export function buildProjectActionItems(input: { searchTerms?: (project: CommandPaletteProject) => ReadonlyArray; renderDescription?: (project: CommandPaletteProject) => ReactNode; shortcutCommand?: KeybindingCommand; -}): CommandPaletteActionItem[] { +} + +export function buildProjectActionItems( + input: BuildProjectActionItemsInput, +): CommandPaletteActionItem[] { return input.projects.map((project) => ({ kind: "action", value: `${input.valuePrefix}:${project.environmentId}:${project.id}`, @@ -254,7 +258,9 @@ export type BuildThreadActionItemsThread = Pick< latestUserMessageAt?: string | null; }; -export function buildThreadActionItems(input: { +export interface BuildThreadActionItemsInput< + TThread extends BuildThreadActionItemsThread = BuildThreadActionItemsThread, +> { threads: ReadonlyArray; activeThreadId?: Thread["id"]; projectTitleById: ReadonlyMap; @@ -269,7 +275,11 @@ export function buildThreadActionItems CommandPaletteThreadContentMatch | undefined; runThread: (thread: Pick) => Promise; limit?: number; -}): CommandPaletteActionItem[] { +} + +export function buildThreadActionItems( + input: BuildThreadActionItemsInput, +): CommandPaletteActionItem[] { const sortedThreads = sortThreads( input.threads.filter((thread) => thread.archivedAt === null), input.sortOrder, diff --git a/apps/web/src/components/CommandPalette.merged-seam.test.tsx b/apps/web/src/components/CommandPalette.merged-seam.test.tsx new file mode 100644 index 000000000000..687d30376fa6 --- /dev/null +++ b/apps/web/src/components/CommandPalette.merged-seam.test.tsx @@ -0,0 +1,123 @@ +import { EnvironmentId, ProjectId, ProviderInstanceId, ThreadId } from "@t3tools/contracts"; +import { renderToStaticMarkup } from "react-dom/server"; +import { describe, expect, it } from "vite-plus/test"; + +import type { Project, Thread } from "../types"; +import { + buildLocationAwareProjectActionItems, + buildVisibleThreadActionItems, +} from "./CommandPalette.thread-project-items"; + +const environmentId = EnvironmentId.make("environment-local"); +const projectId = ProjectId.make("project-local"); + +function makeThread(id: string, title: string): Thread { + return { + id: ThreadId.make(id), + environmentId, + projectId, + title, + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5" }, + runtimeMode: "full-access", + interactionMode: "default", + session: null, + messages: [], + proposedPlans: [], + createdAt: "2026-08-18T12:00:00.000Z", + archivedAt: null, + settledOverride: null, + settledAt: null, + deletedAt: null, + updatedAt: "2026-08-18T12:00:00.000Z", + latestTurn: null, + pullRequests: [], + branch: null, + worktreePath: null, + checkpoints: [], + activities: [], + }; +} + +describe("CommandPalette merged archive and project-location seam", () => { + it("hides optimistic archives while retaining location description and search metadata", () => { + const visibleThread = makeThread("thread-visible", "Visible thread"); + const optimisticArchive = { + ...makeThread("thread-optimistic", "Optimistically archived thread"), + archivedAt: "2026-08-18T12:00:00.000Z", + }; + const threadItems = buildVisibleThreadActionItems({ + threads: [visibleThread, optimisticArchive], + projectTitleById: new Map([[projectId, "Local project"]]), + sortOrder: "updated_at", + icon: null, + runThread: async () => undefined, + }); + + const project: Project = { + id: projectId, + environmentId, + title: "Local project", + workspaceRoot: "E:\\Projects\\local-project", + repositoryIdentity: null, + defaultModelSelection: { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5", + }, + createdAt: "2026-08-18T12:00:00.000Z", + updatedAt: "2026-08-18T12:00:00.000Z", + scripts: [], + }; + const location = { kind: "local", label: "Local", machine: "desktop" } as const; + const [projectItem] = buildLocationAwareProjectActionItems({ + projects: [{ ...project, displayName: `Group: ${project.title}` }], + valuePrefix: "new-thread-in", + icon: () => null, + getLocation: () => location, + runProject: async () => undefined, + }); + + expect(threadItems.map((item) => item.title)).toEqual(["Visible thread"]); + expect(projectItem?.searchTerms).toEqual([ + "Group: Local project", + "Local project", + "E:\\Projects\\local-project", + "Local", + ]); + const descriptionMarkup = renderToStaticMarkup(projectItem?.description); + expect(descriptionMarkup).toContain(">Local"); + expect(descriptionMarkup).toContain(">·"); + expect(descriptionMarkup).toContain(">E:\\Projects\\local-project"); + }); + + it("keeps the rendered remote fallback searchable", () => { + const project: Project = { + id: projectId, + environmentId, + title: "Remote project", + workspaceRoot: "/workspace/remote-project", + repositoryIdentity: null, + defaultModelSelection: { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5", + }, + createdAt: "2026-08-18T12:00:00.000Z", + updatedAt: "2026-08-18T12:00:00.000Z", + scripts: [], + }; + const [projectItem] = buildLocationAwareProjectActionItems({ + projects: [{ ...project, displayName: `Group: ${project.title}` }], + valuePrefix: "new-thread-in", + icon: () => null, + getLocation: () => undefined, + runProject: async () => undefined, + }); + + expect(projectItem?.searchTerms).toEqual([ + "Group: Remote project", + "Remote project", + "/workspace/remote-project", + "Remote", + ]); + expect(renderToStaticMarkup(projectItem?.description)).toContain(">Remote"); + }); +}); diff --git a/apps/web/src/components/CommandPalette.thread-project-items.tsx b/apps/web/src/components/CommandPalette.thread-project-items.tsx new file mode 100644 index 000000000000..dc7cea8103e4 --- /dev/null +++ b/apps/web/src/components/CommandPalette.thread-project-items.tsx @@ -0,0 +1,53 @@ +import type { Project } from "../types"; +import { + buildProjectActionItems, + buildThreadActionItems, + type BuildProjectActionItemsInput, + type BuildThreadActionItemsInput, + type BuildThreadActionItemsThread, + type CommandPaletteActionItem, +} from "./CommandPalette.logic"; +import { filterVisibleSidebarThreads } from "./Sidebar.logic"; +import { + ProjectCommandSubtitle, + projectCommandLocationSearchTerms, + type ProjectCommandLocation, +} from "./ThreadCommandSubtitle"; + +export function buildLocationAwareProjectActionItems( + input: Omit & { + readonly getLocation: (project: Project) => ProjectCommandLocation | undefined; + readonly searchTerms?: (project: Project) => ReadonlyArray; + }, +): CommandPaletteActionItem[] { + const { getLocation, searchTerms, ...actionInput } = input; + + return buildProjectActionItems({ + ...actionInput, + searchTerms: (project) => [ + ...(searchTerms?.(project) ?? []), + ...projectCommandLocationSearchTerms( + getLocation(project) ?? { kind: "remote", label: "Remote", machine: "server" }, + ), + ], + renderDescription: (project) => ( + + ), + }); +} + +export function buildVisibleThreadActionItems( + input: Omit, "threads"> & { + readonly threads: ReadonlyArray; + }, +): CommandPaletteActionItem[] { + const { threads, ...actionInput } = input; + + return buildThreadActionItems({ + ...actionInput, + threads: filterVisibleSidebarThreads(threads), + }); +} diff --git a/apps/web/src/components/CommandPalette.tsx b/apps/web/src/components/CommandPalette.tsx index 95f2df5928ae..268cf522f3db 100644 --- a/apps/web/src/components/CommandPalette.tsx +++ b/apps/web/src/components/CommandPalette.tsx @@ -1,5 +1,7 @@ "use client"; +import { EnvironmentMachineIcon } from "./EnvironmentMachineIcon"; + import { threadPullRequestLinkMode } from "@t3tools/client-runtime/thread-pull-request-compatibility"; import { visibleThreadPullRequests } from "@t3tools/shared/threadPullRequests"; @@ -155,7 +157,6 @@ import { buildCommandPaletteProjectMetadata, buildProjectActionItems, buildRootGroups, - buildThreadActionItems, buildLinkedThreadActionItems, enumerateCommandPaletteItems, type CommandPaletteActionItem, @@ -171,24 +172,31 @@ import { reduceCommandPaletteUiState, type SearchOverlayMode, } from "./CommandPalette.logic"; -import { orderItemsByPreferredIds, sortLogicalProjectsForSidebar } from "./Sidebar.logic"; +import { + filterVisibleSidebarThreads, + orderItemsByPreferredIds, + sortLogicalProjectsForSidebar, +} from "./Sidebar.logic"; import { resolveEnvironmentOptionLabel } from "./BranchToolbar.logic"; import { CommandPaletteContent } from "./CommandPaletteContent"; import { CommandPaletteResults } from "./CommandPaletteResults"; import { AzureDevOpsIcon, BitbucketIcon, GitHubIcon, GitLabIcon, ForgejoIcon } from "./Icons"; -import { EnvironmentMachineIcon } from "./EnvironmentMachineIcon"; import { Checkbox } from "./ui/checkbox"; import { ProjectFavicon } from "./ProjectFavicon"; import { ProjectFilePicker } from "./files/ProjectFilePicker"; import { openLinkPullRequestDialog } from "./pullRequest/LinkPullRequestDialog"; import { ProjectContentSearchDialog } from "./search/ProjectContentSearchDialog"; import { toggleThemeEditorForTheme } from "./settings/themeEditorStore"; -import { searchSettings, SETTINGS_SECTION_LABELS } from "./settings/settingsSearch"; import { COMMAND_PALETTE_META_ICON_CLASS, CommandPaletteMetaDot, ThreadCommandSubtitle, } from "./ThreadCommandSubtitle"; +import { searchSettings, SETTINGS_SECTION_LABELS } from "./settings/settingsSearch"; +import { + buildLocationAwareProjectActionItems, + buildVisibleThreadActionItems, +} from "./CommandPalette.thread-project-items"; import { ThreadRowLeadingStatus, ThreadRowTrailingStatus } from "./ThreadStatusIndicators"; import { primaryServerKeybindingsAtom, primaryServerProvidersAtom } from "../state/server"; import { deriveProviderInstanceEntries, type ProviderInstanceEntry } from "../providerInstances"; @@ -797,6 +805,7 @@ function OpenCommandPaletteDialog(props: { }, [activeThreadReferenceCopyTarget]); const projectOrder = useUiStateStore((store) => store.projectOrder); const threads = useThreadShells(); + const visibleThreads = useMemo(() => filterVisibleSidebarThreads(threads), [threads]); const keybindings = useAtomValue(primaryServerKeybindingsAtom); const { theme, @@ -952,10 +961,10 @@ function OpenCommandPaletteDialog(props: { () => sortLogicalProjectsForSidebar( unsortedProjectGroups, - threads, + visibleThreads, clientSettings.sidebarProjectSortOrder, ), - [clientSettings.sidebarProjectSortOrder, threads, unsortedProjectGroups], + [clientSettings.sidebarProjectSortOrder, unsortedProjectGroups, visibleThreads], ); const contextualProjectRef = useMemo( () => @@ -1233,15 +1242,13 @@ function OpenCommandPaletteDialog(props: { : null; const latestThread = groupedProjectKeys ? (sortThreads( - threads.filter( - (thread) => - thread.archivedAt === null && - groupedProjectKeys.has(`${thread.environmentId}:${thread.projectId}`), + visibleThreads.filter((thread) => + groupedProjectKeys.has(`${thread.environmentId}:${thread.projectId}`), ), clientSettings.sidebarThreadSortOrder, )[0] ?? null) : getLatestThreadForProject( - threads.filter((thread) => thread.environmentId === project.environmentId), + visibleThreads.filter((thread) => thread.environmentId === project.environmentId), project.id, clientSettings.sidebarThreadSortOrder, ); @@ -1262,7 +1269,7 @@ function OpenCommandPaletteDialog(props: { handleNewThread, navigate, projectGroupByTargetKey, - threads, + visibleThreads, ], ); @@ -1314,7 +1321,7 @@ function OpenCommandPaletteDialog(props: { const projectThreadItems = useMemo( () => enumerateCommandPaletteItems([ - ...buildProjectActionItems({ + ...buildLocationAwareProjectActionItems({ // The no-project home shows once, as the "No project" item below. projects: pickerProjects.filter( (project) => !isScratchProject(project, scratchWorkspaceRootFor(project.environmentId)), @@ -1322,36 +1329,11 @@ function OpenCommandPaletteDialog(props: { valuePrefix: "new-thread-in", searchTerms: (project) => { const group = projectGroupByTargetKey.get(`${project.environmentId}:${project.id}`); - const location = projectEnvironmentLocationById.get(project.environmentId); - return [ - ...(group?.memberProjects.flatMap((member) => [member.title, member.workspaceRoot]) ?? - []), - ...(location ? [location.label] : []), - ]; - }, - renderDescription: (project) => { - const location = projectEnvironmentLocationById.get(project.environmentId) ?? { - kind: "remote", - label: "Remote", - machine: "server" as const, - }; return ( - - - {location.kind === "remote" ? ( - - ) : null} - {location.label} - - - {project.workspaceRoot} - + group?.memberProjects.flatMap((member) => [member.title, member.workspaceRoot]) ?? [] ); }, + getLocation: (project) => projectEnvironmentLocationById.get(project.environmentId), icon: projectFavicon, runProject: async (project) => { const group = projectGroupByTargetKey.get(`${project.environmentId}:${project.id}`); @@ -1397,7 +1379,7 @@ function OpenCommandPaletteDialog(props: { const allThreadItems = useMemo( () => - buildThreadActionItems({ + buildVisibleThreadActionItems({ threads, ...(activeThreadId ? { activeThreadId } : {}), projectTitleById, @@ -2402,7 +2384,7 @@ function OpenCommandPaletteDialog(props: { ); if (existing) { const latestThread = getLatestThreadForProject( - threads.filter((thread) => thread.environmentId === existing.environmentId), + visibleThreads.filter((thread) => thread.environmentId === existing.environmentId), existing.id, clientSettings.sidebarThreadSortOrder, ); @@ -2484,7 +2466,7 @@ function OpenCommandPaletteDialog(props: { providers, setOpen, clientSettings.sidebarThreadSortOrder, - threads, + visibleThreads, ], ); diff --git a/apps/web/src/components/LegacySidebar.tsx b/apps/web/src/components/LegacySidebar.tsx index 1991dc502283..20618cd07780 100644 --- a/apps/web/src/components/LegacySidebar.tsx +++ b/apps/web/src/components/LegacySidebar.tsx @@ -177,6 +177,7 @@ import { } from "../threadSelectionStore"; import { isCommandPaletteOpen, openCommandPalette } from "../commandPaletteBus"; import { + filterVisibleSidebarThreads, archiveSelectedThreadEntries, buildMultiSelectThreadContextMenuItems, deleteSelectedThreadEntries, @@ -1342,7 +1343,7 @@ const SidebarProjectItem = memo(function SidebarProjectItem(props: SidebarProjec }); }; const visibleProjectThreads = sortThreads( - projectThreads.filter((thread) => thread.archivedAt === null), + filterVisibleSidebarThreads(projectThreads), threadSortOrder, ); const projectStatus = resolveProjectStatusIndicator( @@ -3409,7 +3410,7 @@ export default function LegacySidebar() { }, []); const visibleThreads = useMemo( - () => sidebarThreads.filter((thread) => thread.archivedAt === null), + () => filterVisibleSidebarThreads(sidebarThreads), [sidebarThreads], ); const sortedProjects = useMemo(() => { @@ -3448,9 +3449,7 @@ export default function LegacySidebar() { () => sortedProjects.flatMap((project) => { const projectThreads = sortThreads( - (threadsByProjectKey.get(project.projectKey) ?? []).filter( - (thread) => thread.archivedAt === null, - ), + filterVisibleSidebarThreads(threadsByProjectKey.get(project.projectKey) ?? []), sidebarThreadSortOrder, ); const projectExpanded = resolveProjectExpanded( diff --git a/apps/web/src/components/Sidebar.logic.test.ts b/apps/web/src/components/Sidebar.logic.test.ts index a4a606bb46c7..353a4a9886a0 100644 --- a/apps/web/src/components/Sidebar.logic.test.ts +++ b/apps/web/src/components/Sidebar.logic.test.ts @@ -11,6 +11,7 @@ import { buildMultiSelectThreadContextMenuItems, createThreadJumpHintVisibilityController, deleteSelectedThreadEntries, + filterVisibleSidebarThreads, filterSidebarProjectScopeItems, getSidebarThreadIdsToPrewarm, resolveAdjacentThreadId, @@ -60,7 +61,6 @@ import { ProviderInstanceId, ThreadId, } from "@t3tools/contracts"; - import { DEFAULT_INTERACTION_MODE, DEFAULT_RUNTIME_MODE, @@ -2024,6 +2024,26 @@ function makeThread(overrides: Partial = {}): Thread { }; } +describe("filterVisibleSidebarThreads", () => { + it("excludes archived shells and optimistically archived threads", () => { + const visibleThread = makeThread({ id: ThreadId.make("thread-visible") }); + const optimisticThread = makeThread({ + id: ThreadId.make("thread-optimistic"), + archivedAt: "2026-03-09T10:05:00.000Z", + }); + const archivedThread = makeThread({ + id: ThreadId.make("thread-archived"), + archivedAt: "2026-03-09T10:05:00.000Z", + }); + + expect( + filterVisibleSidebarThreads([visibleThread, optimisticThread, archivedThread]).map( + (thread) => thread.id, + ), + ).toEqual([visibleThread.id]); + }); +}); + describe("getFallbackThreadIdAfterDelete", () => { it("returns the top remaining thread in the deleted thread's project sidebar order", () => { const fallbackThreadId = getFallbackThreadIdAfterDelete({ @@ -2361,6 +2381,45 @@ describe("sortScopedProjectsForSidebar", () => { "Archived-only project", ]); }); + + it("does not use optimistically archived threads as project activity", () => { + const visibleProjectId = ProjectId.make("project-visible"); + const optimisticProjectId = ProjectId.make("project-optimistic"); + const optimisticThread = makeThread({ + id: ThreadId.make("thread-optimistic"), + projectId: optimisticProjectId, + archivedAt: "2026-03-09T10:05:00.000Z", + updatedAt: "2026-03-09T10:10:00.000Z", + }); + const sorted = sortScopedProjectsForSidebar( + [ + makeProject({ + id: visibleProjectId, + title: "Visible project", + updatedAt: "2026-03-09T10:01:00.000Z", + }), + makeProject({ + id: optimisticProjectId, + title: "Optimistic-only project", + updatedAt: "2026-03-09T10:00:00.000Z", + }), + ], + [ + makeThread({ + id: ThreadId.make("thread-visible"), + projectId: visibleProjectId, + updatedAt: "2026-03-09T10:02:00.000Z", + }), + optimisticThread, + ], + "updated_at", + ); + + expect(sorted.map((project) => project.title)).toEqual([ + "Visible project", + "Optimistic-only project", + ]); + }); }); describe("sortLogicalProjectsForSidebar", () => { @@ -2398,6 +2457,52 @@ describe("sortLogicalProjectsForSidebar", () => { ), ).toEqual(["logical-newer", "logical-older"]); }); + + it("does not use optimistically archived threads as logical project activity", () => { + const visibleProjectId = ProjectId.make("project-visible"); + const optimisticProjectId = ProjectId.make("project-optimistic"); + const projects = [ + { + ...makeProject({ + id: visibleProjectId, + title: "Visible project", + updatedAt: "2026-03-09T10:01:00.000Z", + }), + projectKey: "logical-visible", + memberProjectRefs: [{ environmentId: localEnvironmentId, projectId: visibleProjectId }], + }, + { + ...makeProject({ + id: optimisticProjectId, + title: "Optimistic-only project", + updatedAt: "2026-03-09T10:00:00.000Z", + }), + projectKey: "logical-optimistic", + memberProjectRefs: [{ environmentId: localEnvironmentId, projectId: optimisticProjectId }], + }, + ]; + const optimisticThread = makeThread({ + id: ThreadId.make("thread-optimistic"), + projectId: optimisticProjectId, + archivedAt: "2026-03-09T10:05:00.000Z", + updatedAt: "2026-03-09T10:10:00.000Z", + }); + + expect( + sortLogicalProjectsForSidebar( + projects, + [ + makeThread({ + id: ThreadId.make("thread-visible"), + projectId: visibleProjectId, + updatedAt: "2026-03-09T10:02:00.000Z", + }), + optimisticThread, + ], + "updated_at", + ).map((project) => project.projectKey), + ).toEqual(["logical-visible", "logical-optimistic"]); + }); }); describe("resolveSidebarDropVerb", () => { diff --git a/apps/web/src/components/Sidebar.logic.ts b/apps/web/src/components/Sidebar.logic.ts index a63b728cd4fa..9ccbc689103d 100644 --- a/apps/web/src/components/Sidebar.logic.ts +++ b/apps/web/src/components/Sidebar.logic.ts @@ -6,7 +6,7 @@ import { type AtomCommandResult, } from "@t3tools/client-runtime/state/runtime"; import { threadSearchMatchKey } from "@t3tools/client-runtime/state/thread-search"; -import type { ContextMenuItem, EnvironmentId, ThreadId } from "@t3tools/contracts"; +import type { ContextMenuItem, EnvironmentId, ProjectId, ThreadId } from "@t3tools/contracts"; import type { SidebarProjectSortOrder, SidebarThreadSortOrder } from "@t3tools/contracts/settings"; import type { AsyncResult } from "effect/unstable/reactivity"; import { planPinnedReorder } from "@t3tools/client-runtime/state/thread-sort"; @@ -421,8 +421,9 @@ type ScopedSidebarProject = SidebarProject & { }; type ScopedSidebarThread = ThreadSortInput & { - environmentId: string; - projectId: string; + id: ThreadId; + environmentId: EnvironmentId; + projectId: ProjectId; archivedAt: string | null; }; @@ -1146,6 +1147,12 @@ export function resolveProjectStatusIndicator( return highestPriorityStatus; } +export function filterVisibleSidebarThreads>( + threads: readonly T[], +): T[] { + return threads.filter((thread) => thread.archivedAt === null); +} + export function getFallbackThreadIdAfterDelete< T extends Pick & ThreadSortInput, >(input: { @@ -1256,7 +1263,9 @@ export function sortLogicalProjectsForSidebar< ); const threadsByProjectKey = new Map(); for (const thread of threads) { - if (thread.archivedAt !== null) continue; + if (thread.archivedAt !== null) { + continue; + } const projectKey = groupKeyByProjectRef.get(`${thread.environmentId}\0${thread.projectId}`); if (!projectKey) continue; const existing = threadsByProjectKey.get(projectKey); diff --git a/apps/web/src/components/Sidebar.tsx b/apps/web/src/components/Sidebar.tsx index 29849561efe2..4e83dc55b1e5 100644 --- a/apps/web/src/components/Sidebar.tsx +++ b/apps/web/src/components/Sidebar.tsx @@ -158,6 +158,7 @@ import { buildThreadActionMenuItems } from "./threadActionMenu.logic"; import { animateSidebarLayoutChanges, applySidebarThreadDrop, + filterVisibleSidebarThreads, buildBulkTitleRegenerationContextMenuItem, buildBulkUnpinContextMenuItem, deleteSelectedThreadEntries, @@ -2330,8 +2331,6 @@ export default function Sidebar() { [routeDraftThread, routeTarget], ); const routeThreadKey = routeThreadRef ? scopedThreadKey(routeThreadRef) : null; - const routeTargetRef = useRef(routeTarget); - routeTargetRef.current = routeTarget; // Post-settle navigation validates against the CURRENT route, not the one // captured when the settle started: if the user navigated elsewhere while // the command was in flight, completing it must not yank them away. @@ -2598,6 +2597,11 @@ export default function Sidebar() { override holds until all of them appear in canonical state. */ readonly assignedKeys: ReadonlyMap; } | null>(null); + // Settled threads stay in the live shell stream (settled ≠ archived), so + // the partition works directly off live shells: no archived-snapshot + // merging. Archived threads remain hidden here — including while a local + // archive command is still in flight — because archive keeps its original + // "remove from sidebar" meaning. const { pinnedThreads, draggableThreadKeys, @@ -2614,11 +2618,10 @@ export default function Sidebar() { // memo exactly at the next wake boundary. void snoozeWakeTick; const preciseNow = new Date().toISOString(); - const visible = threads.filter( + const visible = filterVisibleSidebarThreads(threads).filter( (thread) => - thread.archivedAt === null && - (scopedProjectKeys === null || - scopedProjectKeys.has(`${thread.environmentId}:${thread.projectId}`)), + scopedProjectKeys === null || + scopedProjectKeys.has(`${thread.environmentId}:${thread.projectId}`), ); observeInboxReturns(workingShelfEnabled ? threads : null); const pinned: EnvironmentThreadShell[] = []; diff --git a/apps/web/src/components/ThreadCommandSubtitle.tsx b/apps/web/src/components/ThreadCommandSubtitle.tsx index 58c28609dcef..f90669aa1418 100644 --- a/apps/web/src/components/ThreadCommandSubtitle.tsx +++ b/apps/web/src/components/ThreadCommandSubtitle.tsx @@ -1,5 +1,6 @@ -import type { ProviderDriverKind } from "@t3tools/contracts"; +import type { EnvironmentMachineKind, ProviderDriverKind } from "@t3tools/contracts"; import { FolderGit2Icon, FolderIcon, GitBranchIcon } from "lucide-react"; +import { EnvironmentMachineIcon } from "./EnvironmentMachineIcon"; import { ProjectFavicon, type ProjectFaviconProject } from "./ProjectFavicon"; import { ProviderInstanceIcon } from "./chat/ProviderInstanceIcon"; import { cn } from "~/lib/utils"; @@ -25,6 +26,40 @@ export function CommandPaletteMetaDot() { return ·; } +export interface ProjectCommandLocation { + readonly kind: "local" | "remote"; + readonly label: string; + readonly machine: EnvironmentMachineKind; +} + +export function projectCommandLocationSearchTerms( + location: ProjectCommandLocation | undefined, +): ReadonlyArray { + return location ? [location.label] : []; +} + +export function ProjectCommandSubtitle(props: { + readonly location: ProjectCommandLocation; + readonly workspaceRoot: string; +}) { + return ( + + + {props.location.kind === "remote" ? ( + + ) : null} + {props.location.label} + + + {props.workspaceRoot} + + ); +} + function WorkspaceIcon(props: { variant: ThreadCommandSubtitleVariant; isWorktree: boolean }) { if (props.isWorktree) { return ; diff --git a/apps/web/src/components/settings/ArchivedThreadsPanel.environment.test.tsx b/apps/web/src/components/settings/ArchivedThreadsPanel.environment.test.tsx new file mode 100644 index 000000000000..94bb117ca8c3 --- /dev/null +++ b/apps/web/src/components/settings/ArchivedThreadsPanel.environment.test.tsx @@ -0,0 +1,148 @@ +import type { ArchivedSnapshotEntry } from "@t3tools/client-runtime/state/threads"; +import { + EnvironmentId, + ProjectId, + ProviderInstanceId, + ThreadId, + type OrchestrationShellSnapshot, +} from "@t3tools/contracts"; +import type { ResolvedSettingsScope } from "./settingsScope"; +import type { ReactElement } from "react"; +import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +import { visitElements } from "../../test/reactElementTree"; +import { reactHookHarness as hooks } from "../../test/reactHookHarness"; + +const archiveState = vi.hoisted(() => ({ + scope: { kind: "all", label: "All", members: [], environmentIds: [] } as ResolvedSettingsScope, + error: null as string | null, + snapshots: [] as ReadonlyArray, +})); + +const projectState = vi.hoisted(() => ({ + projects: [] as ReadonlyArray>, +})); + +vi.mock("react", async (importOriginal) => { + const actual = await importOriginal(); + const { reactHookHarness } = await import("../../test/reactHookHarness"); + return { + ...actual, + useCallback: reactHookHarness.useCallback, + useMemo: reactHookHarness.useMemo, + useRef: reactHookHarness.useRef, + useState: reactHookHarness.useState, + }; +}); + +vi.mock("react/compiler-runtime", async () => { + const { reactHookHarness } = await import("../../test/reactHookHarness"); + return { c: reactHookHarness.useMemoCache }; +}); + +vi.mock("./SettingsScopeContext", () => ({ + useSettingsScope: () => ({ scope: archiveState.scope }), +})); + +vi.mock("../../lib/archivedThreadsState", () => ({ + useArchivedThreadSnapshots: () => ({ + snapshots: archiveState.snapshots, + error: archiveState.error, + isLoading: false, + refresh: vi.fn(), + }), +})); + +vi.mock("../../hooks/useThreadActions", () => ({ + useThreadActions: () => ({ + unarchiveThread: vi.fn(), + confirmAndDeleteThread: vi.fn(), + }), +})); + +import { ArchivedThreadsPanel } from "./ArchivedThreadsPanel"; + +const environmentId = EnvironmentId.make("partial-archive-environment"); +const projectId = ProjectId.make("partial-archive-project"); + +function archivedSnapshot(): OrchestrationShellSnapshot { + return { + snapshotSequence: 1, + projects: [ + { + id: projectId, + title: "Loaded project", + workspaceRoot: "C:\\loaded-project", + defaultModelSelection: null, + faviconPath: null, + scripts: [], + createdAt: "2026-08-20T00:00:00.000Z", + updatedAt: "2026-08-20T00:00:00.000Z", + }, + ], + threads: [ + { + id: ThreadId.make("partial-archive-thread"), + projectId, + title: "Loaded archived thread", + modelSelection: { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5.5", + options: [], + }, + runtimeMode: "full-access", + interactionMode: "default", + pullRequests: [], + branch: null, + worktreePath: null, + latestTurn: null, + createdAt: "2026-08-19T00:00:00.000Z", + updatedAt: "2026-08-20T00:00:00.000Z", + archivedAt: "2026-08-20T00:00:00.000Z", + settledOverride: null, + settledAt: null, + session: null, + latestUserMessageAt: null, + hasPendingApprovals: false, + hasPendingUserInput: false, + hasActionableProposedPlan: false, + }, + ], + updatedAt: "2026-08-20T00:00:00.000Z", + }; +} + +function renderPanel(): ReactElement> { + hooks.beginRender(); + return ArchivedThreadsPanel() as ReactElement>; +} + +describe("ArchivedThreadsPanel environment loading", () => { + beforeEach(() => { + hooks.reset(); + archiveState.scope = { kind: "all", label: "All", members: [], environmentIds: [] }; + archiveState.error = null; + archiveState.snapshots = []; + projectState.projects = []; + }); + + it("renders a partial-load error alongside successful archived groups", () => { + archiveState.error = "Failed to load archived threads."; + archiveState.snapshots = [{ environmentId, snapshot: archivedSnapshot() }]; + projectState.projects = [{ environmentId }]; + + const panel = renderPanel(); + expect( + visitElements( + panel, + (element) => element.props.title === "Could not load all archived threads", + ), + ).not.toBeNull(); + expect( + visitElements(panel, (element) => element.props.title === "Loaded project"), + ).not.toBeNull(); + expect( + visitElements(panel, (element) => element.props.title === "Loaded archived thread"), + ).not.toBeNull(); + }); +}); diff --git a/apps/web/src/components/settings/ArchivedThreadsPanel.logic.ts b/apps/web/src/components/settings/ArchivedThreadsPanel.logic.ts new file mode 100644 index 000000000000..b65d73469d31 --- /dev/null +++ b/apps/web/src/components/settings/ArchivedThreadsPanel.logic.ts @@ -0,0 +1,14 @@ +import type { ContextMenuItem } from "@t3tools/contracts"; + +export type ArchivedThreadContextMenuAction = "unarchive" | "delete"; + +export const archivedThreadContextMenuItems = [ + { id: "unarchive", label: "Unarchive", icon: "archive" }, + { + id: "delete", + label: "Delete", + destructive: true, + icon: "trash", + separatorBefore: true, + }, +] as const satisfies readonly ContextMenuItem[]; diff --git a/apps/web/src/components/settings/ArchivedThreadsPanel.tsx b/apps/web/src/components/settings/ArchivedThreadsPanel.tsx new file mode 100644 index 000000000000..e705c5641e08 --- /dev/null +++ b/apps/web/src/components/settings/ArchivedThreadsPanel.tsx @@ -0,0 +1,270 @@ +import { Spinner } from "../ui/spinner"; +import { scopedThreadKey, scopeThreadRef } from "@t3tools/client-runtime/environment"; +import { + isAtomCommandInterrupted, + settlePromise, + squashAtomCommandFailure, +} from "@t3tools/client-runtime/state/runtime"; +import { toSortableTimestamp } from "@t3tools/client-runtime/state/thread-sort"; +import type { ScopedThreadRef } from "@t3tools/contracts"; +import { ArchiveIcon, ArchiveX } from "lucide-react"; +import { useCallback, useMemo, useRef, useState } from "react"; + +import { useThreadActions } from "../../hooks/useThreadActions"; +import { useArchivedThreadSnapshots } from "../../lib/archivedThreadsState"; +import { readLocalApi } from "../../localApi"; +import { useSettingsScope } from "./SettingsScopeContext"; +import { formatRelativeTimeLabel } from "../../timestampFormat"; +import { ProjectFavicon } from "../ProjectFavicon"; +import { Button } from "../ui/button"; +import { stackedThreadToast, toastManager } from "../ui/toast"; +import { archivedThreadContextMenuItems } from "./ArchivedThreadsPanel.logic"; +import { SettingsPageContainer, SettingsRow, SettingsSection } from "./settingsLayout"; +import { searchableSetting } from "./settingsSearch"; + +export function ArchivedThreadsPanel() { + const { scope } = useSettingsScope(); + const { unarchiveThread, confirmAndDeleteThread } = useThreadActions(); + const [unarchivingThreadKeys, setUnarchivingThreadKeys] = useState>( + () => new Set(), + ); + const unarchivingThreadKeysRef = useRef(new Set()); + const { + snapshots: archivedSnapshots, + error: archiveError, + isLoading: isLoadingArchive, + refresh: refreshArchivedThreads, + } = useArchivedThreadSnapshots(scope.environmentIds); + + const archivedGroups = useMemo(() => { + const selectedProjectKeys = + scope.kind === "project" || scope.kind === "checkout" + ? new Set(scope.members.map((member) => `${member.environmentId}:${member.id}`)) + : null; + const projectsByEnvironmentAndId = new Map( + archivedSnapshots.flatMap(({ environmentId, snapshot }) => + snapshot.projects + .filter( + (project) => + selectedProjectKeys === null || + selectedProjectKeys.has(`${environmentId}:${project.id}`), + ) + .map( + (project) => + [ + `${environmentId}:${project.id}`, + { + ...project, + environmentId, + }, + ] as const, + ), + ), + ); + const threads = archivedSnapshots.flatMap(({ environmentId, snapshot }) => + snapshot.threads.map((thread) => ({ + ...thread, + environmentId, + })), + ); + + const archivedProjects = Array.from(projectsByEnvironmentAndId.values()); + const groups: Array<{ + readonly project: (typeof archivedProjects)[number]; + readonly threads: Array<(typeof threads)[number]>; + }> = []; + for (const project of archivedProjects) { + const projectThreads: Array<(typeof threads)[number]> = []; + for (const thread of threads) { + if (thread.projectId === project.id && thread.environmentId === project.environmentId) { + projectThreads.push(thread); + } + } + if (projectThreads.length > 0) { + groups.push({ + project, + threads: projectThreads.toSorted((left, right) => { + const leftKey = toSortableTimestamp(left.archivedAt ?? left.createdAt); + const rightKey = toSortableTimestamp(right.archivedAt ?? right.createdAt); + return ( + (rightKey ?? Number.NEGATIVE_INFINITY) - (leftKey ?? Number.NEGATIVE_INFINITY) || + right.id.localeCompare(left.id) + ); + }), + }); + } + } + return groups; + }, [archivedSnapshots, scope]); + + const handleUnarchiveThread = useCallback( + async (threadRef: ScopedThreadRef) => { + const threadKey = scopedThreadKey(threadRef); + if (unarchivingThreadKeysRef.current.has(threadKey)) return; + unarchivingThreadKeysRef.current.add(threadKey); + setUnarchivingThreadKeys((current) => new Set(current).add(threadKey)); + try { + const result = await unarchiveThread(threadRef); + if (result._tag === "Success") { + refreshArchivedThreads(); + } else if (!isAtomCommandInterrupted(result)) { + const error = squashAtomCommandFailure(result); + toastManager.add( + stackedThreadToast({ + type: "error", + title: "Failed to unarchive thread", + description: error instanceof Error ? error.message : "An error occurred.", + }), + ); + } + } finally { + unarchivingThreadKeysRef.current.delete(threadKey); + setUnarchivingThreadKeys((current) => { + const next = new Set(current); + next.delete(threadKey); + return next; + }); + } + }, + [refreshArchivedThreads, unarchiveThread], + ); + + const handleArchivedThreadContextMenu = useCallback( + async (threadRef: ScopedThreadRef, position: { x: number; y: number }) => { + const api = readLocalApi(); + if (!api) return; + const clicked = await api.contextMenu.show(archivedThreadContextMenuItems, position); + + if (clicked === "unarchive") { + await handleUnarchiveThread(threadRef); + return; + } + + if (clicked === "delete") { + const result = await confirmAndDeleteThread(threadRef); + if (result._tag === "Success") { + refreshArchivedThreads(); + } else if (!isAtomCommandInterrupted(result)) { + const error = squashAtomCommandFailure(result); + toastManager.add( + stackedThreadToast({ + type: "error", + title: "Failed to delete thread", + description: error instanceof Error ? error.message : "An error occurred.", + }), + ); + } + } + }, + [confirmAndDeleteThread, handleUnarchiveThread, refreshArchivedThreads], + ); + + return ( + + {archivedGroups.length > 0 && archiveError ? ( + + + + ) : null} + {archivedGroups.length === 0 ? ( + + + {isLoadingArchive ? ( + + ) : ( + + )} + {isLoadingArchive + ? "Loading archived threads" + : archiveError + ? "Could not load archived threads" + : "No archived threads"} + + } + description={ + isLoadingArchive + ? "Checking connected environments." + : (archiveError ?? "Archived threads will appear here.") + } + /> + + ) : ( + archivedGroups.map(({ project, threads: projectThreads }, index) => ( + } + > + {projectThreads.map((thread) => { + const threadRef = scopeThreadRef(thread.environmentId, thread.id); + const isUnarchiving = unarchivingThreadKeys.has(scopedThreadKey(threadRef)); + return ( + { + event.preventDefault(); + if (isUnarchiving) return; + void (async () => { + const result = await settlePromise(() => + handleArchivedThreadContextMenu( + scopeThreadRef(thread.environmentId, thread.id), + { + x: event.clientX, + y: event.clientY, + }, + ), + ); + if (result._tag === "Failure") { + const error = squashAtomCommandFailure(result); + toastManager.add( + stackedThreadToast({ + type: "error", + title: "Archived thread action failed", + description: + error instanceof Error ? error.message : "An error occurred.", + }), + ); + } + })(); + }} + title={thread.title} + description={ + <> + Archived {formatRelativeTimeLabel(thread.archivedAt ?? thread.createdAt)} + {" \u00b7 Created "} + {formatRelativeTimeLabel(thread.createdAt)} + + } + control={ + + } + /> + ); + })} + + )) + )} + + ); +} diff --git a/apps/web/src/components/settings/ProjectSettingsPanel.logic.test.ts b/apps/web/src/components/settings/ProjectSettingsPanel.logic.test.ts index 8a72b3510ceb..fb444801016f 100644 --- a/apps/web/src/components/settings/ProjectSettingsPanel.logic.test.ts +++ b/apps/web/src/components/settings/ProjectSettingsPanel.logic.test.ts @@ -1,6 +1,80 @@ import { describe, expect, it } from "vite-plus/test"; -import { projectGroupTitleNeedsUpdate } from "./ProjectSettingsPanel.logic"; +import { + buildArchivedProjectRemovalPlans, + getArchivedProjectRemovalWarning, + projectGroupTitleNeedsUpdate, +} from "./ProjectSettingsPanel.logic"; + +describe("archived project removal", () => { + it("derives archived-bundle command scope from each project member's threads", () => { + const members = [ + { environmentId: "environment-live", id: "grouped-project" }, + { environmentId: "environment-archived", id: "grouped-project" }, + ]; + const projectThreads = [ + { + environmentId: "environment-live", + projectId: "grouped-project", + id: "thread-live", + }, + ]; + + expect( + buildArchivedProjectRemovalPlans(members, projectThreads).map((plan) => ({ + environmentId: plan.member.environmentId, + threadIds: plan.memberThreads.map((thread) => thread.id), + options: plan.commandOptions, + })), + ).toEqual([ + { + environmentId: "environment-live", + threadIds: ["thread-live"], + options: { force: true }, + }, + { + environmentId: "environment-archived", + threadIds: [], + options: { deleteArchivedThreads: true }, + }, + ]); + }); + + it("describes archived deletion for standalone and grouped removals", () => { + expect( + getArchivedProjectRemovalWarning({ + memberCount: 1, + hasLiveThreads: true, + }), + ).toBe( + "This permanently clears conversation history for those threads and any archived conversations in this project.", + ); + expect( + getArchivedProjectRemovalWarning({ + memberCount: 1, + hasLiveThreads: false, + }), + ).toBe( + "If this project has archived conversations, their history will also be permanently deleted.", + ); + expect( + getArchivedProjectRemovalWarning({ + memberCount: 2, + hasLiveThreads: true, + }), + ).toBe( + "This permanently clears conversation history for those threads and any archived conversations in these projects.", + ); + expect( + getArchivedProjectRemovalWarning({ + memberCount: 2, + hasLiveThreads: false, + }), + ).toBe( + "If these projects have archived conversations, their history will also be permanently deleted.", + ); + }); +}); describe("projectGroupTitleNeedsUpdate", () => { it("updates divergent member titles even when the next title is the derived group label", () => { diff --git a/apps/web/src/components/settings/ProjectSettingsPanel.logic.ts b/apps/web/src/components/settings/ProjectSettingsPanel.logic.ts index 17ff824099fb..bbfab35b9f24 100644 --- a/apps/web/src/components/settings/ProjectSettingsPanel.logic.ts +++ b/apps/web/src/components/settings/ProjectSettingsPanel.logic.ts @@ -1,3 +1,51 @@ +export function getArchivedProjectRemovalWarning(input: { + memberCount: number; + hasLiveThreads: boolean; +}): string { + const projectLabel = input.memberCount === 1 ? "this project" : "these projects"; + if (input.hasLiveThreads) { + return `This permanently clears conversation history for those threads and any archived conversations in ${projectLabel}.`; + } + const verb = input.memberCount === 1 ? "has" : "have"; + return `If ${projectLabel} ${verb} archived conversations, their history will also be permanently deleted.`; +} + +export function resolveArchivedProjectRemovalCommandOptions( + hasLiveThreads: boolean, +): { readonly force: true } | { readonly deleteArchivedThreads: true } { + if (hasLiveThreads) { + return { force: true }; + } + + // Archived shells are intentionally absent from the client's live thread + // list. Preserve the server's live-thread precondition while opting this + // project member's cold bundles into removal. + return { deleteArchivedThreads: true }; +} + +export function buildArchivedProjectRemovalPlans< + TMember extends { readonly environmentId: string; readonly id: string }, + TThread extends { readonly environmentId: string; readonly projectId: string }, +>( + members: readonly TMember[], + projectThreads: readonly TThread[], +): { + readonly member: TMember; + readonly memberThreads: TThread[]; + readonly commandOptions: { readonly force: true } | { readonly deleteArchivedThreads: true }; +}[] { + return members.map((member) => { + const memberThreads = projectThreads.filter( + (thread) => thread.environmentId === member.environmentId && thread.projectId === member.id, + ); + return { + member, + memberThreads, + commandOptions: resolveArchivedProjectRemovalCommandOptions(memberThreads.length > 0), + }; + }); +} + export function projectGroupTitleNeedsUpdate( memberTitles: ReadonlyArray, nextTitle: string, diff --git a/apps/web/src/components/settings/ProjectSettingsPanel.tsx b/apps/web/src/components/settings/ProjectSettingsPanel.tsx index 8e5edec0f485..f590d1ea23fd 100644 --- a/apps/web/src/components/settings/ProjectSettingsPanel.tsx +++ b/apps/web/src/components/settings/ProjectSettingsPanel.tsx @@ -41,7 +41,11 @@ import { } from "./ProjectFaviconPickerDialog"; import { ProjectActionsSettings } from "./ProjectActionsSettings"; import { ProjectDefaultsSettings } from "./ProjectDefaultsSettings"; -import { projectGroupTitleNeedsUpdate } from "./ProjectSettingsPanel.logic"; +import { + buildArchivedProjectRemovalPlans, + getArchivedProjectRemovalWarning, + projectGroupTitleNeedsUpdate, +} from "./ProjectSettingsPanel.logic"; import { useSettingsProjectGroups } from "./useSettingsProjectGroups"; const ProjectIconPickerDialog = lazy(() => @@ -304,6 +308,7 @@ function ProjectDetail({ const projectThreads = threads.filter((thread) => memberKeys.has(`${thread.environmentId}:${thread.projectId}`), ); + const memberRemovalPlans = buildArchivedProjectRemovalPlans(members, projectThreads); const isWholeGroup = members.length === group.memberProjects.length; const targetKind = hasOtherMembers || !isWholeGroup ? "checkout" : "project"; const singleMember = members.length === 1 ? members[0]! : null; @@ -322,11 +327,10 @@ function ProjectDetail({ : []), ] : [`This removes ${members.length} grouped project entries.`]), - ...(projectThreads.length > 0 - ? [ - "This permanently clears conversation history for those threads and any archived threads.", - ] - : ["This permanently clears any archived conversation history."]), + getArchivedProjectRemovalWarning({ + memberCount: members.length, + hasLiveThreads: projectThreads.length > 0, + }), isWholeGroup && !hasOtherMembers ? "This removes only the project entries, not the files on disk." : "Other entries in this grouped project are unaffected.", @@ -338,17 +342,13 @@ function ProjectDetail({ if (confirmed._tag === "Failure" || !confirmed.value) return; const draftStore = useComposerDraftStore.getState(); - for (const member of members) { - const memberThreads = projectThreads.filter( - (thread) => - thread.environmentId === member.environmentId && thread.projectId === member.id, - ); + for (const { member, memberThreads, commandOptions } of memberRemovalPlans) { const result = mapAtomCommandResult( await deleteProject({ environmentId: member.environmentId, input: { projectId: member.id, - force: true, + ...commandOptions, }, }), () => undefined, diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsx index d44f64356b52..936b53178828 100644 --- a/apps/web/src/components/settings/SettingsPanels.tsx +++ b/apps/web/src/components/settings/SettingsPanels.tsx @@ -1,7 +1,15 @@ +import { + AlertDialog, + AlertDialogClose, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogPopup, + AlertDialogTitle, +} from "../ui/alert-dialog"; import { SettingsGroup } from "./SettingsGroup"; -import { Spinner } from "~/components/ui/spinner"; import { NotificationSettings } from "./NotificationSettings"; -import { ArchiveIcon, ArchiveX, CheckIcon, ChevronRightIcon, SettingsIcon } from "lucide-react"; +import { CheckIcon, ChevronRightIcon, SettingsIcon } from "lucide-react"; import { Link, useNavigate } from "@tanstack/react-router"; import type { CSSProperties, ReactNode } from "react"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; @@ -10,15 +18,8 @@ import { type DesktopUpdateChannel, ProviderDriverKind, type ProviderInstanceId, - type ScopedThreadRef, type SidebarProjectGroupingMode, } from "@t3tools/contracts"; -import { scopeThreadRef } from "@t3tools/client-runtime/environment"; -import { - isAtomCommandInterrupted, - settlePromise, - squashAtomCommandFailure, -} from "@t3tools/client-runtime/state/runtime"; import { DEFAULT_ENVIRONMENT_IDENTIFICATION_MODE, DEFAULT_UNIFIED_SETTINGS, @@ -81,7 +82,6 @@ import { import { useScopedModelDisabledReason } from "./useScopedModelAvailability"; import { useSettingsScope } from "./SettingsScopeContext"; import { ProjectDefaultsSettings } from "./ProjectDefaultsSettings"; -import { useThreadActions } from "../../hooks/useThreadActions"; import { useDesktopUpdateState } from "../../state/desktopUpdate"; import { getCustomModelOptionsByInstance, @@ -95,17 +95,6 @@ import { import { ensureLocalApi, readLocalApi } from "../../localApi"; import { isMacPlatform } from "../../lib/utils"; import { EMPTY_SERVER_PROVIDERS } from "../../state/server"; -import { useArchivedThreadSnapshots } from "../../lib/archivedThreadsState"; -import { formatRelativeTimeLabel } from "../../timestampFormat"; -import { - AlertDialog, - AlertDialogClose, - AlertDialogDescription, - AlertDialogFooter, - AlertDialogHeader, - AlertDialogPopup, - AlertDialogTitle, -} from "../ui/alert-dialog"; import { Button } from "../ui/button"; import { Collapsible, CollapsiblePanel, CollapsibleTrigger } from "../ui/collapsible"; import { @@ -170,7 +159,6 @@ import { useSettingsSearchTargetId, } from "./settingsLayout"; import { searchableSetting } from "./settingsSearch"; -import { ProjectFavicon } from "../ProjectFavicon"; import { PanelAnimationsPreview } from "./PanelAnimationsPreview"; const ENVIRONMENT_IDENTIFICATION_LABELS: Record = { @@ -3311,227 +3299,3 @@ export function GeneralSettingsPanel() { ); } - -export function ArchivedThreadsPanel() { - const { scope } = useSettingsScope(); - const { unarchiveThread, confirmAndDeleteThread } = useThreadActions(); - const { - snapshots: archivedSnapshots, - error: archiveError, - isLoading: isLoadingArchive, - refresh: refreshArchivedThreads, - } = useArchivedThreadSnapshots(scope.environmentIds); - - const archivedGroups = useMemo(() => { - const selectedProjectKeys = - scope.kind === "project" || scope.kind === "checkout" - ? new Set(scope.members.map((member) => `${member.environmentId}:${member.id}`)) - : null; - const projectsByEnvironmentAndId = new Map( - archivedSnapshots.flatMap(({ environmentId, snapshot }) => - snapshot.projects - .filter( - (project) => - selectedProjectKeys === null || - selectedProjectKeys.has(`${environmentId}:${project.id}`), - ) - .map( - (project) => [`${environmentId}:${project.id}`, { ...project, environmentId }] as const, - ), - ), - ); - const threads = archivedSnapshots.flatMap(({ environmentId, snapshot }) => - snapshot.threads.map((thread) => ({ - ...thread, - environmentId, - })), - ); - - const archivedProjects = Array.from(projectsByEnvironmentAndId.values()); - const groups: Array<{ - readonly project: (typeof archivedProjects)[number]; - readonly threads: Array<(typeof threads)[number]>; - }> = []; - for (const project of archivedProjects) { - const projectThreads: Array<(typeof threads)[number]> = []; - for (const thread of threads) { - if (thread.projectId === project.id && thread.environmentId === project.environmentId) { - projectThreads.push(thread); - } - } - if (projectThreads.length > 0) { - groups.push({ - project, - threads: projectThreads.toSorted((left, right) => { - const leftKey = left.archivedAt ?? left.createdAt; - const rightKey = right.archivedAt ?? right.createdAt; - return rightKey.localeCompare(leftKey) || right.id.localeCompare(left.id); - }), - }); - } - } - return groups; - }, [archivedSnapshots, scope]); - - const handleArchivedThreadContextMenu = useCallback( - async (threadRef: ScopedThreadRef, position: { x: number; y: number }) => { - const api = readLocalApi(); - if (!api) return; - const clicked = await api.contextMenu.show( - [ - { id: "unarchive", label: "Unarchive" }, - { id: "delete", label: "Delete", destructive: true }, - ], - position, - ); - - if (clicked === "unarchive") { - const result = await unarchiveThread(threadRef); - if (result._tag === "Success") { - refreshArchivedThreads(); - } else if (!isAtomCommandInterrupted(result)) { - const error = squashAtomCommandFailure(result); - toastManager.add( - stackedThreadToast({ - type: "error", - title: "Failed to unarchive thread", - description: error instanceof Error ? error.message : "An error occurred.", - }), - ); - } - return; - } - - if (clicked === "delete") { - const result = await confirmAndDeleteThread(threadRef); - if (result._tag === "Success") { - refreshArchivedThreads(); - } else if (!isAtomCommandInterrupted(result)) { - const error = squashAtomCommandFailure(result); - toastManager.add( - stackedThreadToast({ - type: "error", - title: "Failed to delete thread", - description: error instanceof Error ? error.message : "An error occurred.", - }), - ); - } - } - }, - [confirmAndDeleteThread, refreshArchivedThreads, unarchiveThread], - ); - - return ( - - {archivedGroups.length === 0 ? ( - - - {isLoadingArchive ? ( - - ) : ( - - )} - {isLoadingArchive - ? "Loading archived threads" - : archiveError - ? "Could not load archived threads" - : "No archived threads"} - - } - description={ - isLoadingArchive - ? "Checking connected environments." - : (archiveError ?? "Archived threads will appear here.") - } - /> - - ) : ( - archivedGroups.map(({ project, threads: projectThreads }, index) => ( - } - > - {projectThreads.map((thread) => ( - { - event.preventDefault(); - void (async () => { - const result = await settlePromise(() => - handleArchivedThreadContextMenu( - scopeThreadRef(thread.environmentId, thread.id), - { - x: event.clientX, - y: event.clientY, - }, - ), - ); - if (result._tag === "Failure") { - const error = squashAtomCommandFailure(result); - toastManager.add( - stackedThreadToast({ - type: "error", - title: "Archived thread action failed", - description: - error instanceof Error ? error.message : "An error occurred.", - }), - ); - } - })(); - }} - title={thread.title} - description={ - <> - Archived {formatRelativeTimeLabel(thread.archivedAt ?? thread.createdAt)} - {" \u00b7 Created "} - {formatRelativeTimeLabel(thread.createdAt)} - - } - control={ - - } - /> - ))} - - )) - )} - - ); -} diff --git a/apps/web/src/composerDraftArchiveObserver.test.ts b/apps/web/src/composerDraftArchiveObserver.test.ts new file mode 100644 index 000000000000..1c1f1d82068c --- /dev/null +++ b/apps/web/src/composerDraftArchiveObserver.test.ts @@ -0,0 +1,89 @@ +import { scopeThreadRef } from "@t3tools/client-runtime/environment"; +import { EnvironmentId, ThreadId } from "@t3tools/contracts"; +import { describe, expect, it } from "vite-plus/test"; + +import { reconcileAuthoritativeThreadRefs } from "./authoritativeThreadLifecycle"; + +const environmentId = EnvironmentId.make("environment-one"); +const threadRef = scopeThreadRef(environmentId, ThreadId.make("thread-one")); + +function reconcile( + previousActiveThreadRefs: ReadonlyArray, + activeThreadRefs: ReadonlyArray, +) { + return reconcileAuthoritativeThreadRefs({ + previousActiveThreadRefs, + activeThreadRefs, + catalogEnvironmentIds: new Set([environmentId]), + liveEnvironmentIds: new Set([environmentId]), + }); +} + +describe("composer draft archive observation", () => { + it("releases a thread after an authoritative remote archive", () => { + const afterArchive = reconcile([threadRef], []); + + expect(afterArchive.removedThreadRefs).toEqual([threadRef]); + }); + + it("keeps uploads when shell data is unavailable", () => { + const unavailable = reconcileAuthoritativeThreadRefs({ + previousActiveThreadRefs: [threadRef], + activeThreadRefs: [], + catalogEnvironmentIds: new Set([environmentId]), + liveEnvironmentIds: new Set(), + }); + + expect(unavailable.removedThreadRefs).toEqual([]); + expect(unavailable.nextActiveThreadRefs).toEqual([threadRef]); + }); + + it("waits through an authorization session handoff before cleaning an archived draft", () => { + const synchronizingReplacement = reconcileAuthoritativeThreadRefs({ + previousActiveThreadRefs: [threadRef], + activeThreadRefs: [], + catalogEnvironmentIds: new Set([environmentId]), + liveEnvironmentIds: new Set(), + }); + + expect(synchronizingReplacement.removedThreadRefs).toEqual([]); + expect(synchronizingReplacement.nextActiveThreadRefs).toEqual([threadRef]); + + const authoritativeReplacement = reconcileAuthoritativeThreadRefs({ + previousActiveThreadRefs: synchronizingReplacement.nextActiveThreadRefs, + activeThreadRefs: [], + catalogEnvironmentIds: new Set([environmentId]), + liveEnvironmentIds: new Set([environmentId]), + }); + + expect(authoritativeReplacement.removedThreadRefs).toEqual([threadRef]); + expect(authoritativeReplacement.nextActiveThreadRefs).toEqual([]); + }); + + it("releases a cached thread archived before its shell becomes live", () => { + const cached = reconcileAuthoritativeThreadRefs({ + previousActiveThreadRefs: [], + activeThreadRefs: [threadRef], + catalogEnvironmentIds: new Set([environmentId]), + liveEnvironmentIds: new Set(), + }); + + expect(cached.removedThreadRefs).toEqual([]); + expect(cached.nextActiveThreadRefs).toEqual([threadRef]); + + const liveAfterArchive = reconcileAuthoritativeThreadRefs({ + previousActiveThreadRefs: cached.nextActiveThreadRefs, + activeThreadRefs: [], + catalogEnvironmentIds: new Set([environmentId]), + liveEnvironmentIds: new Set([environmentId]), + }); + + expect(liveAfterArchive.removedThreadRefs).toEqual([threadRef]); + }); + + it("does not release uploads when an archived thread is unarchived", () => { + const unarchived = reconcile([], [threadRef]); + + expect(unarchived.removedThreadRefs).toEqual([]); + }); +}); diff --git a/apps/web/src/composerDraftArchiveObserver.tsx b/apps/web/src/composerDraftArchiveObserver.tsx new file mode 100644 index 000000000000..ef22f1430671 --- /dev/null +++ b/apps/web/src/composerDraftArchiveObserver.tsx @@ -0,0 +1,29 @@ +import { useEffect, useRef } from "react"; + +import { reconcileAuthoritativeThreadRefs } from "./authoritativeThreadLifecycle"; +import { releaseComposerDraftUploads } from "./lib/composerDraftUploads"; +import { useLiveEnvironmentIds, useThreadRefs } from "./state/entities"; +import { useEnvironments } from "./state/environments"; + +export function ComposerDraftArchiveObserver() { + const activeThreadRefs = useThreadRefs(); + const liveEnvironmentIds = useLiveEnvironmentIds(); + const { environmentIds: catalogEnvironmentIds, isReady: environmentCatalogReady } = + useEnvironments(); + const previousActiveThreadRefs = useRef(activeThreadRefs); + + useEffect(() => { + const reconciliation = reconcileAuthoritativeThreadRefs({ + previousActiveThreadRefs: previousActiveThreadRefs.current, + activeThreadRefs, + catalogEnvironmentIds: environmentCatalogReady ? catalogEnvironmentIds : null, + liveEnvironmentIds, + }); + previousActiveThreadRefs.current = reconciliation.nextActiveThreadRefs; + for (const threadRef of reconciliation.removedThreadRefs) { + releaseComposerDraftUploads(threadRef); + } + }, [activeThreadRefs, catalogEnvironmentIds, environmentCatalogReady, liveEnvironmentIds]); + + return null; +} diff --git a/apps/web/src/connection/storage.test.ts b/apps/web/src/connection/storage.test.ts index 206220768e9a..a249009b00a9 100644 --- a/apps/web/src/connection/storage.test.ts +++ b/apps/web/src/connection/storage.test.ts @@ -16,6 +16,7 @@ import { makeBrowserGitHubRoutingPermissions, makeCatalogBackend, makeCatalogStore, + upgradeConnectionDatabase, } from "./storage"; const emptyCatalog = { @@ -198,3 +199,57 @@ describe("browser GitHub routing permissions", () => { }).pipe(Effect.scoped), ); }); + +describe("upgradeConnectionDatabase", () => { + it("does not clear the empty thread store when creating a new database", () => { + const stores = new Set(); + const clear = vi.fn(); + const database = { + objectStoreNames: { contains: (name: string) => stores.has(name) }, + createObjectStore: vi.fn((name: string) => stores.add(name)), + } as unknown as IDBDatabase; + const transaction = { + objectStore: vi.fn(() => ({ clear })), + } as unknown as IDBTransaction; + + upgradeConnectionDatabase(database, transaction, 0); + + expect(stores).toEqual(new Set(["catalog", "shell", "thread", "server-config", "vcs-refs"])); + expect(transaction.objectStore).not.toHaveBeenCalled(); + expect(clear).not.toHaveBeenCalled(); + }); + + it("clears existing thread snapshots when upgrading to archive-time cache eviction", () => { + const stores = new Set(["catalog", "shell", "thread", "server-config", "vcs-refs"]); + const clear = vi.fn(); + const database = { + objectStoreNames: { contains: (name: string) => stores.has(name) }, + createObjectStore: vi.fn((name: string) => stores.add(name)), + } as unknown as IDBDatabase; + const transaction = { + objectStore: vi.fn(() => ({ clear })), + } as unknown as IDBTransaction; + + upgradeConnectionDatabase(database, transaction, 4); + + expect(transaction.objectStore).toHaveBeenCalledWith("thread"); + expect(clear).toHaveBeenCalledOnce(); + }); + + it("does not clear thread snapshots after the migration has already run", () => { + const stores = new Set(["catalog", "shell", "thread", "server-config", "vcs-refs"]); + const clear = vi.fn(); + const database = { + objectStoreNames: { contains: (name: string) => stores.has(name) }, + createObjectStore: vi.fn((name: string) => stores.add(name)), + } as unknown as IDBDatabase; + const transaction = { + objectStore: vi.fn(() => ({ clear })), + } as unknown as IDBTransaction; + + upgradeConnectionDatabase(database, transaction, 5); + + expect(transaction.objectStore).not.toHaveBeenCalled(); + expect(clear).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/connection/storage.ts b/apps/web/src/connection/storage.ts index ec2e4ea5175c..fe3c83fe26e7 100644 --- a/apps/web/src/connection/storage.ts +++ b/apps/web/src/connection/storage.ts @@ -45,7 +45,7 @@ import * as Stream from "effect/Stream"; import { projectFaviconCache } from "../assets/projectFaviconCache"; const DATABASE_NAME = "t3code:connection-runtime"; -const DATABASE_VERSION = 4; +const DATABASE_VERSION = 5; const CATALOG_STORE_NAME = "catalog"; const SHELL_STORE_NAME = "shell"; const THREAD_STORE_NAME = "thread"; @@ -53,6 +53,7 @@ const SERVER_CONFIG_STORE_NAME = "server-config"; const VCS_REFS_STORE_NAME = "vcs-refs"; const CATALOG_KEY = "document"; const SHELL_SNAPSHOT_CACHE_SCHEMA_VERSION = 1; +const ARCHIVED_THREAD_CACHE_EVICTION_DATABASE_VERSION = 5; const StoredShellSnapshot = Schema.Struct({ schemaVersion: Schema.Literal(SHELL_SNAPSHOT_CACHE_SCHEMA_VERSION), @@ -133,6 +134,32 @@ function persistenceError( }); } +export function upgradeConnectionDatabase( + database: IDBDatabase, + transaction: IDBTransaction | null, + oldVersion: number, +) { + if (!database.objectStoreNames.contains(CATALOG_STORE_NAME)) { + database.createObjectStore(CATALOG_STORE_NAME); + } + if (!database.objectStoreNames.contains(SHELL_STORE_NAME)) { + database.createObjectStore(SHELL_STORE_NAME); + } + if (!database.objectStoreNames.contains(THREAD_STORE_NAME)) { + database.createObjectStore(THREAD_STORE_NAME); + } + if (!database.objectStoreNames.contains(SERVER_CONFIG_STORE_NAME)) { + database.createObjectStore(SERVER_CONFIG_STORE_NAME); + } + if (!database.objectStoreNames.contains(VCS_REFS_STORE_NAME)) { + database.createObjectStore(VCS_REFS_STORE_NAME); + } + + if (oldVersion > 0 && oldVersion < ARCHIVED_THREAD_CACHE_EVICTION_DATABASE_VERSION) { + transaction?.objectStore(THREAD_STORE_NAME).clear(); + } +} + const openDatabase = Effect.fn("web.connectionStorage.openDatabase")(function* () { return yield* Effect.callback((resume) => { if (typeof indexedDB === "undefined") { @@ -142,22 +169,8 @@ const openDatabase = Effect.fn("web.connectionStorage.openDatabase")(function* ( return; } const request = indexedDB.open(DATABASE_NAME, DATABASE_VERSION); - request.addEventListener("upgradeneeded", () => { - if (!request.result.objectStoreNames.contains(CATALOG_STORE_NAME)) { - request.result.createObjectStore(CATALOG_STORE_NAME); - } - if (!request.result.objectStoreNames.contains(SHELL_STORE_NAME)) { - request.result.createObjectStore(SHELL_STORE_NAME); - } - if (!request.result.objectStoreNames.contains(THREAD_STORE_NAME)) { - request.result.createObjectStore(THREAD_STORE_NAME); - } - if (!request.result.objectStoreNames.contains(SERVER_CONFIG_STORE_NAME)) { - request.result.createObjectStore(SERVER_CONFIG_STORE_NAME); - } - if (!request.result.objectStoreNames.contains(VCS_REFS_STORE_NAME)) { - request.result.createObjectStore(VCS_REFS_STORE_NAME); - } + request.addEventListener("upgradeneeded", (event) => { + upgradeConnectionDatabase(request.result, request.transaction, event.oldVersion); }); request.addEventListener("error", () => { resume(Effect.fail(catalogError("open", request.error ?? "Unknown IndexedDB error"))); diff --git a/apps/web/src/contextMenuFallback.test.ts b/apps/web/src/contextMenuFallback.test.ts index ddc5da05f806..f285f616a672 100644 --- a/apps/web/src/contextMenuFallback.test.ts +++ b/apps/web/src/contextMenuFallback.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; +import { archivedThreadContextMenuItems } from "./components/settings/ArchivedThreadsPanel.logic"; import { dismissContextMenu, showContextMenuFallback } from "./contextMenuFallback"; type FakeListener = (event: FakeDomEvent) => void; @@ -157,6 +158,10 @@ class FakeDocument { return new FakeElement(tagName); } + createElementNS(_namespace: string, tagName: string) { + return new FakeElement(tagName); + } + addEventListener(type: string, listener: FakeListener) { const existing = this.listeners.get(type) ?? []; existing.push(listener); @@ -219,6 +224,25 @@ afterEach(() => { }); describe("showContextMenuFallback", () => { + it("renders and dispatches the archived-thread actions", async () => { + expect(archivedThreadContextMenuItems.map(({ id }) => id)).toEqual(["unarchive", "delete"]); + + const selectionPromise = showContextMenuFallback(archivedThreadContextMenuItems); + const unarchiveButton = findButton("Unarchive"); + const deleteButton = findButton("Delete"); + const separators = (document as unknown as FakeDocument) + .querySelectorAll("div") + .filter((element) => element.dataset.contextMenuSeparator === "true"); + + expect(unarchiveButton?.querySelectorAll("svg")[0]?.children).toHaveLength(3); + expect(deleteButton?.querySelectorAll("svg")[0]?.children).toHaveLength(5); + expect(deleteButton?.style.color).toBe("var(--destructive-foreground)"); + expect(separators).toHaveLength(1); + + deleteButton?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + await expect(selectionPromise).resolves.toBe("delete"); + }); + it("renders one separator between menu sections", async () => { const selectionPromise = showContextMenuFallback([ { id: "rename", label: "Rename" }, diff --git a/apps/web/src/hooks/useThreadActions.test.ts b/apps/web/src/hooks/useThreadActions.test.ts index b042f19893da..236fecb23939 100644 --- a/apps/web/src/hooks/useThreadActions.test.ts +++ b/apps/web/src/hooks/useThreadActions.test.ts @@ -1,14 +1,24 @@ -import { EnvironmentId, ThreadId } from "@t3tools/contracts"; -import { afterEach, describe, expect, it, vi } from "vite-plus/test"; +import { EnvironmentId, ProjectId, ThreadId } from "@t3tools/contracts"; +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; -import { - navigateAfterThreadDeletion, - requestThreadUnpinConfirmation, - ThreadArchiveBlockedError, -} from "./useThreadActions"; +const mocks = vi.hoisted(() => ({ + archiveMutation: vi.fn(), + clearComposerDraftForThread: vi.fn(), + clearProjectDraftThreadById: vi.fn(), + clearTerminalUiState: vi.fn(), + handleNewThread: vi.fn(), + markThreadVisited: vi.fn(), + readThreadShell: vi.fn(), + refreshArchivedThreads: vi.fn(), + releaseComposerDraftUploads: vi.fn(), +})); +import { navigateAfterThreadDeletion } from "./useThreadActions"; import { toastManager } from "../components/ui/toast"; describe("navigateAfterThreadDeletion", () => { + beforeEach(() => vi.clearAllMocks()); afterEach(() => vi.restoreAllMocks()); it("reports a rejected navigation without failing the completed deletion", async () => { @@ -36,6 +46,130 @@ describe("navigateAfterThreadDeletion", () => { }); }); +vi.mock("@tanstack/react-router", () => ({ + useRouter: () => ({ + navigate: vi.fn(), + state: { matches: [{ params: {} }] }, + }), +})); + +vi.mock("../components/Sidebar.logic", () => ({ + getFallbackThreadIdAfterDelete: vi.fn(() => null), + pinOrderKeyBetween: vi.fn(() => null), +})); + +vi.mock("../composerDraftStore", () => ({ + useComposerDraftStore: ( + selector: (state: { + clearDraftThread: typeof mocks.clearComposerDraftForThread; + clearProjectDraftThreadById: typeof mocks.clearProjectDraftThreadById; + }) => T, + ) => + selector({ + clearDraftThread: mocks.clearComposerDraftForThread, + clearProjectDraftThreadById: mocks.clearProjectDraftThreadById, + }), +})); + +vi.mock("../state/terminal", () => ({ terminalEnvironment: { close: {} } })); +vi.mock("../state/threads", () => ({ + threadEnvironment: { + archive: {}, + delete: {}, + pin: {}, + reorderPin: {}, + settle: {}, + snooze: {}, + stopSession: {}, + unarchive: {}, + unpin: {}, + unsettle: {}, + unsnooze: {}, + }, +})); +vi.mock("../state/vcs", () => ({ + vcsEnvironment: { refreshStatus: {}, removeWorktree: {} }, +})); +vi.mock("./useHandleNewThread", () => ({ + useNewThreadHandler: () => mocks.handleNewThread, +})); +vi.mock("../lib/archivedThreadsState", () => ({ + refreshArchivedThreadsForEnvironment: mocks.refreshArchivedThreads, +})); +vi.mock("../lib/composerDraftUploads", () => ({ + releaseComposerDraftUploads: mocks.releaseComposerDraftUploads, +})); +vi.mock("../localApi", () => ({ readLocalApi: vi.fn(() => null) })); +vi.mock("../state/entities", () => ({ + readEnvironmentSupportsPinning: vi.fn(() => false), + readEnvironmentSupportsPinReorder: vi.fn(() => false), + readEnvironmentSupportsSettlement: vi.fn(() => false), + readEnvironmentSupportsSnooze: vi.fn(() => false), + readEnvironmentThreadRefs: vi.fn(() => []), + readProject: vi.fn(() => null), + readThreadShell: mocks.readThreadShell, + readThreadShells: vi.fn(() => []), +})); +vi.mock("../terminalUiStateStore", () => ({ + useTerminalUiStateStore: ( + selector: (state: { clearTerminalUiState: typeof mocks.clearTerminalUiState }) => T, + ) => selector({ clearTerminalUiState: mocks.clearTerminalUiState }), +})); +vi.mock("../uiStateStore", () => ({ + useUiStateStore: ( + selector: (state: { markThreadVisited: typeof mocks.markThreadVisited }) => T, + ) => selector({ markThreadVisited: mocks.markThreadVisited }), +})); +vi.mock("../worktreeCleanup", () => ({ + formatWorktreePathForDisplay: vi.fn((path: string) => path), + getOrphanedWorktreePathForThread: vi.fn(() => null), +})); +vi.mock("../components/ui/toast", () => ({ + stackedThreadToast: vi.fn((input: unknown) => input), + toastManager: { add: vi.fn() }, +})); +vi.mock("./useSettings", () => ({ + useClientSettings: ( + selector: (settings: { + confirmThreadDelete: boolean; + sidebarThreadSortOrder: "updated_at"; + }) => T, + ) => selector({ confirmThreadDelete: false, sidebarThreadSortOrder: "updated_at" }), +})); +vi.mock("../state/use-atom-command", () => ({ + useAtomCommand: vi.fn(() => mocks.archiveMutation), +})); +vi.mock("@t3tools/client-runtime/state/thread-settled", () => ({ + canSettle: vi.fn(() => true), + canSnooze: vi.fn(() => true), + threadWokeAt: vi.fn(() => null), +})); + +import { + requestThreadUnpinConfirmation, + ThreadArchiveBlockedError, + useThreadActions, +} from "./useThreadActions"; + +const threadRef = { + environmentId: EnvironmentId.make("environment-1"), + threadId: ThreadId.make("thread-1"), +}; + +function renderThreadActions(): ReturnType { + let actions: ReturnType | undefined; + + function Probe() { + actions = useThreadActions(); + return null; + } + + renderToStaticMarkup(createElement(Probe)); + if (actions === undefined) { + throw new Error("Thread actions did not render."); + } + return actions; +} describe("ThreadArchiveBlockedError", () => { it("keeps the blocked thread context with the fixed message", () => { const error = new ThreadArchiveBlockedError({ @@ -51,6 +185,37 @@ describe("ThreadArchiveBlockedError", () => { }); }); +describe("archive draft uploads", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.readThreadShell.mockReturnValue({ + environmentId: threadRef.environmentId, + id: threadRef.threadId, + projectId: ProjectId.make("project-1"), + session: null, + }); + }); + + it("releases pending draft uploads after archive succeeds", async () => { + mocks.archiveMutation.mockResolvedValue({ _tag: "Success", value: undefined }); + + const result = await renderThreadActions().archiveThread(threadRef); + + expect(result._tag).toBe("Success"); + expect(mocks.releaseComposerDraftUploads).toHaveBeenCalledOnce(); + expect(mocks.releaseComposerDraftUploads).toHaveBeenCalledWith(threadRef); + }); + + it("keeps pending draft uploads when archive fails", async () => { + mocks.archiveMutation.mockResolvedValue({ _tag: "Failure", cause: new Error("offline") }); + + const result = await renderThreadActions().archiveThread(threadRef); + + expect(result._tag).toBe("Failure"); + expect(mocks.releaseComposerDraftUploads).not.toHaveBeenCalled(); + }); +}); + describe("requestThreadUnpinConfirmation", () => { it("skips the dialog when confirmation is disabled", async () => { let callCount = 0; diff --git a/apps/web/src/hooks/useThreadActions.ts b/apps/web/src/hooks/useThreadActions.ts index 2dae445041bb..f16b71a29371 100644 --- a/apps/web/src/hooks/useThreadActions.ts +++ b/apps/web/src/hooks/useThreadActions.ts @@ -327,6 +327,11 @@ export function useThreadActions() { markThreadVisited(scopedThreadKey(threadRef), wokeAt); } refreshArchivedThreadsForEnvironment(threadRef.environmentId); + // The local composer draft intentionally survives archive, but its + // pending server uploads are transient and never enter the cold bundle. + // Release them now; opening the restored draft starts fresh uploads from + // its local image data. + releaseComposerDraftUploads(threadRef); opts.onArchived?.(); showThreadUndoNotice({ action: "Archived", diff --git a/apps/web/src/lib/composerDraftUploads.test.ts b/apps/web/src/lib/composerDraftUploads.test.ts new file mode 100644 index 000000000000..1cf313ce3457 --- /dev/null +++ b/apps/web/src/lib/composerDraftUploads.test.ts @@ -0,0 +1,52 @@ +import { EnvironmentId, ThreadId } from "@t3tools/contracts"; +import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +const mocks = vi.hoisted(() => ({ + getComposerDraft: vi.fn(), + releaseDraftAttachments: vi.fn(), +})); + +vi.mock("../composerDraftStore", () => ({ + useComposerDraftStore: { + getState: () => ({ + getComposerDraft: mocks.getComposerDraft, + }), + }, +})); + +vi.mock("./attachmentUploadQueue", () => ({ + releaseDraftAttachments: mocks.releaseDraftAttachments, +})); + +import { releaseComposerDraftUploads } from "./composerDraftUploads"; + +describe("releaseComposerDraftUploads", () => { + beforeEach(() => { + mocks.getComposerDraft.mockReset(); + mocks.releaseDraftAttachments.mockReset(); + }); + + it("releases every pending draft attachment upload", () => { + const images = [{ id: "image-1" }, { id: "image-2" }]; + const files = [{ id: "file-1" }]; + mocks.getComposerDraft.mockReturnValue({ images, files }); + + releaseComposerDraftUploads({ + environmentId: EnvironmentId.make("environment-1"), + threadId: ThreadId.make("thread-1"), + }); + + expect(mocks.releaseDraftAttachments).toHaveBeenCalledWith([...images, ...files]); + }); + + it("does nothing when the thread has no composer draft", () => { + mocks.getComposerDraft.mockReturnValue(null); + + releaseComposerDraftUploads({ + environmentId: EnvironmentId.make("environment-1"), + threadId: ThreadId.make("thread-1"), + }); + + expect(mocks.releaseDraftAttachments).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/previewMiniPlayerStore.test.ts b/apps/web/src/previewMiniPlayerStore.test.ts index e4eb40c8b08a..7e19b6932607 100644 --- a/apps/web/src/previewMiniPlayerStore.test.ts +++ b/apps/web/src/previewMiniPlayerStore.test.ts @@ -95,4 +95,18 @@ describe("previewMiniPlayerStore", () => { selectThreadPreviewMiniPlayer(usePreviewMiniPlayerStore.getState().byThreadKey, refA), ).toBe(floating); }); + + it("removes only the targeted thread lifecycle state", () => { + usePreviewMiniPlayerStore.getState().open(refA, tabA); + usePreviewMiniPlayerStore.getState().open(refB, tabB); + + usePreviewMiniPlayerStore.getState().removeThread(refA); + + expect( + selectThreadPreviewMiniPlayer(usePreviewMiniPlayerStore.getState().byThreadKey, refA), + ).toBeNull(); + expect( + selectThreadPreviewMiniPlayer(usePreviewMiniPlayerStore.getState().byThreadKey, refB), + ).toMatchObject({ source: tabB }); + }); }); diff --git a/apps/web/src/previewStateStore.ts b/apps/web/src/previewStateStore.ts index 1e7ec3706618..c2789cda2341 100644 --- a/apps/web/src/previewStateStore.ts +++ b/apps/web/src/previewStateStore.ts @@ -476,3 +476,7 @@ export const __testing = { EMPTY_THREAD_PREVIEW_STATE, RECENT_URL_LIMIT: PREVIEW_RECENT_URL_LIMIT, }; + +export function removePreviewThread(ref: ScopedThreadRef): void { + updateThreadPreviewState(ref, () => EMPTY_THREAD_PREVIEW_STATE); +} diff --git a/apps/web/src/routes/settings.archived.tsx b/apps/web/src/routes/settings.archived.tsx index 3ad690afc027..b411f6440af7 100644 --- a/apps/web/src/routes/settings.archived.tsx +++ b/apps/web/src/routes/settings.archived.tsx @@ -1,6 +1,6 @@ import { createFileRoute } from "@tanstack/react-router"; -import { ArchivedThreadsPanel } from "../components/settings/SettingsPanels"; +import { ArchivedThreadsPanel } from "../components/settings/ArchivedThreadsPanel"; export const Route = createFileRoute("/settings/archived")({ component: ArchivedThreadsPanel, diff --git a/apps/web/src/state/entities.ts b/apps/web/src/state/entities.ts index af977d567f2b..99d917117500 100644 --- a/apps/web/src/state/entities.ts +++ b/apps/web/src/state/entities.ts @@ -18,6 +18,7 @@ import { environmentServerConfigsAtom } from "./server"; import { allEnvironmentProjectSnapshotsReadyAtom, allEnvironmentShellsBootstrappedAtom, + liveEnvironmentIdsAtom, } from "./shell"; import { environmentThreadDetails, environmentThreadShells } from "./threads"; @@ -86,6 +87,10 @@ export function useAllEnvironmentProjectSnapshotsReady(): boolean { return useAtomValue(allEnvironmentProjectSnapshotsReadyAtom); } +export function useLiveEnvironmentIds(): ReadonlySet { + return useAtomValue(liveEnvironmentIdsAtom); +} + export function useThreadShellsForProjectRefs( refs: ReadonlyArray, ): ReadonlyArray { diff --git a/apps/web/src/state/environments.ts b/apps/web/src/state/environments.ts index f085075fdd7c..af95219df395 100644 --- a/apps/web/src/state/environments.ts +++ b/apps/web/src/state/environments.ts @@ -38,6 +38,7 @@ export function useEnvironments() { const catalog = useAtomValue(environmentCatalog.catalogValueAtom); const networkStatus = useAtomValue(environmentCatalog.networkStatusValueAtom); const presentationById = useAtomValue(environmentPresentations.presentationsAtom); + const environmentIds = useMemo(() => new Set(catalog.entries.keys()), [catalog.entries]); const environments = useMemo( () => @@ -50,6 +51,7 @@ export function useEnvironments() { return { isReady: catalog.isReady, networkStatus, + environmentIds, environments, presentationById, }; diff --git a/apps/web/src/state/shell.ts b/apps/web/src/state/shell.ts index e8f61209ae64..d9f5dca98f5e 100644 --- a/apps/web/src/state/shell.ts +++ b/apps/web/src/state/shell.ts @@ -24,6 +24,24 @@ export const shellEnvironment = createShellEnvironmentAtoms(connectionAtomRuntim export const environmentShell = createEnvironmentShellAtoms(connectionAtomRuntime); export const environmentSnapshotAtom = createEnvironmentSnapshotAtom(environmentShell.stateAtom); +let previousLiveEnvironmentIds: ReadonlySet = new Set(); +export const liveEnvironmentIdsAtom = Atom.make((get) => { + const next = new Set(); + for (const environmentId of get(environmentCatalog.catalogValueAtom).entries.keys()) { + if (get(environmentShell.stateValueAtom(environmentId)).status === "live") { + next.add(environmentId); + } + } + if ( + previousLiveEnvironmentIds.size === next.size && + [...previousLiveEnvironmentIds].every((environmentId) => next.has(environmentId)) + ) { + return previousLiveEnvironmentIds; + } + previousLiveEnvironmentIds = next; + return previousLiveEnvironmentIds; +}).pipe(Atom.withLabel("web-live-environment-ids")); + export const allEnvironmentShellsBootstrappedAtom = Atom.make((get) => { const catalog = AsyncResult.value(get(environmentCatalog.catalogAtom)); if (Option.isNone(catalog)) { diff --git a/docs/user/background-service.md b/docs/user/background-service.md index 9964fba54f74..1b9a52593601 100644 --- a/docs/user/background-service.md +++ b/docs/user/background-service.md @@ -35,6 +35,8 @@ maintainers' test train: its builds can be broken and are never offered as updates, so the installer and `t3 update` ask for confirmation before installing one. +Versions without compressed archive storage cannot read archived conversations once a newer version has compressed them, including conversations archived before you updated. Unarchive any conversations you need before downgrading; changing the version alone does not restore archived history. + `t3 uninstall` removes the background service, the `t3` launcher, and the downloaded versions after showing you the list and asking once. Your projects, threads, and settings under `~/.t3/userdata` are kept. Pass `--yes` from a diff --git a/packages/client-runtime/src/operations/commands.test.ts b/packages/client-runtime/src/operations/commands.test.ts index dde2d3f2fab9..d0bd422e99ad 100644 --- a/packages/client-runtime/src/operations/commands.test.ts +++ b/packages/client-runtime/src/operations/commands.test.ts @@ -19,8 +19,8 @@ import { type PreparedConnection, } from "../connection/model.ts"; import * as EnvironmentSupervisor from "../connection/supervisor.ts"; -import * as RpcSession from "../rpc/session.ts"; import type { WsRpcProtocolClient } from "../rpc/protocol.ts"; +import { makeTestRpcSession } from "../rpc/testUtils/rpcSession.ts"; import { archiveThread, createProject, @@ -56,14 +56,7 @@ const makeSupervisor = Effect.fn("TestEnvironmentCommands.makeSupervisor")(funct return { sequence: dispatched.length }; }), } as unknown as WsRpcProtocolClient; - const session: RpcSession.RpcSession = { - client, - initialConfig: Effect.never, - subscribeServerConfig: (input) => client.subscribeServerConfig(input), - ready: Effect.void, - probe: Effect.void, - closed: Effect.never, - }; + const session = makeTestRpcSession(client); return EnvironmentSupervisor.EnvironmentSupervisor.of({ target: TARGET, state: yield* SubscriptionRef.make(AVAILABLE_CONNECTION_STATE), diff --git a/packages/client-runtime/src/rpc/testUtils/rpcSession.ts b/packages/client-runtime/src/rpc/testUtils/rpcSession.ts new file mode 100644 index 000000000000..a3c97d0b967e --- /dev/null +++ b/packages/client-runtime/src/rpc/testUtils/rpcSession.ts @@ -0,0 +1,27 @@ +import * as Effect from "effect/Effect"; + +import type { WsRpcProtocolClient } from "../protocol.ts"; +import type { RpcSession } from "../session.ts"; + +export interface TestRpcSessionOptions { + readonly initialConfig?: RpcSession["initialConfig"]; +} + +/** + * Keep protocol-only tests insulated from additions to RpcSession. + * + * Tests that exercise connection setup should use the real session factory. + */ +export function makeTestRpcSession( + client: WsRpcProtocolClient, + options: TestRpcSessionOptions = {}, +): RpcSession { + return { + client, + initialConfig: options.initialConfig ?? Effect.never, + subscribeServerConfig: (input) => client.subscribeServerConfig(input), + ready: Effect.void, + probe: Effect.void, + closed: Effect.never, + }; +} diff --git a/packages/client-runtime/src/state/shell-sync.test.ts b/packages/client-runtime/src/state/shell-sync.test.ts index 9483d86de0ff..29d8b89bef7c 100644 --- a/packages/client-runtime/src/state/shell-sync.test.ts +++ b/packages/client-runtime/src/state/shell-sync.test.ts @@ -1,6 +1,7 @@ import { EnvironmentId, ORCHESTRATION_WS_METHODS, + ThreadId, type OrchestrationShellSnapshot, type OrchestrationShellStreamItem, } from "@t3tools/contracts"; @@ -24,6 +25,12 @@ import * as Persistence from "../platform/persistence.ts"; import * as RpcSession from "../rpc/session.ts"; import type { WsRpcProtocolClient } from "../rpc/protocol.ts"; import { makeEnvironmentShellState, ShellSnapshotLoader } from "./shell.ts"; +import { + cachedThreadGeneration, + evictCachedThread, + persistCachedThread, + retainCachedThread, +} from "./threadCache.ts"; const TARGET = new PrimaryConnectionTarget({ environmentId: EnvironmentId.make("environment-1"), @@ -240,34 +247,36 @@ describe("environment shell synchronization", () => { }).pipe(Effect.scoped), ); - it.effect("requests a full socket snapshot when the HTTP refresh fails", () => + it.effect("revalidates an HTTP shell snapshot with the authoritative socket snapshot", () => Effect.gen(function* () { const cachedSnapshot: OrchestrationShellSnapshot = { snapshotSequence: 5, projects: [], - threads: [{ id: "cached-thread" } as never], + threads: [{ id: "stale-thread" } as never], updatedAt: "2026-06-06T00:00:00.000Z", }; - const resetSnapshot: OrchestrationShellSnapshot = { + const httpSnapshot: OrchestrationShellSnapshot = { ...cachedSnapshot, - snapshotSequence: 9_999, - threads: [], + snapshotSequence: 9, + threads: [{ id: "archived-after-http-snapshot" } as never], updatedAt: "2026-06-07T00:00:00.000Z", }; const events = yield* Queue.unbounded(); - const wakeups = yield* Queue.unbounded(); - const subscribeInputs = yield* Queue.unbounded<{ + const capturedInput = yield* SubscriptionRef.make<{ readonly afterSequence?: number; readonly requestCompletionMarker?: boolean; - }>(); - const loaderCalls = yield* Ref.make(0); + } | null>(null); + const loaderCalls = yield* SubscriptionRef.make(0); + const removedThreads = yield* Ref.make([]); + const savedThreads = yield* Ref.make([]); + const addedThreadId = ThreadId.make("archived-after-http-snapshot"); const client = { [ORCHESTRATION_WS_METHODS.subscribeShell]: (input: { readonly afterSequence?: number; readonly requestCompletionMarker?: boolean; }) => Stream.unwrap( - Queue.offer(subscribeInputs, input).pipe(Effect.as(Stream.fromQueue(events))), + SubscriptionRef.set(capturedInput, input).pipe(Effect.as(Stream.fromQueue(events))), ), } as unknown as WsRpcProtocolClient; const supervisorState = yield* SubscriptionRef.make(AVAILABLE_CONNECTION_STATE); @@ -286,10 +295,12 @@ describe("environment shell synchronization", () => { const cache = Persistence.EnvironmentCacheStore.of({ loadShell: () => Effect.succeedSome(cachedSnapshot), saveShell: () => Effect.void, - loadThread: () => Effect.succeedNone, - saveThread: () => Effect.void, - removeThread: () => Effect.void, - loadServerConfig: () => Effect.succeedNone, + loadThread: () => Effect.succeed(Option.none()), + saveThread: (_environmentId, snapshot) => + Ref.update(savedThreads, (threadIds) => [...threadIds, snapshot.thread.id]), + removeThread: (_environmentId, threadId) => + Ref.update(removedThreads, (threadIds) => [...threadIds, threadId]), + loadServerConfig: () => Effect.succeed(Option.none()), saveServerConfig: () => Effect.void, loadVcsRefs: () => Effect.succeedNone, saveVcsRefs: () => Effect.void, @@ -297,58 +308,327 @@ describe("environment shell synchronization", () => { clearVcsRefs: () => Effect.void, clear: () => Effect.void, }); + yield* retainCachedThread(cache, TARGET.environmentId, addedThreadId); + const staleGeneration = cachedThreadGeneration(cache, TARGET.environmentId, addedThreadId); + yield* evictCachedThread(cache, TARGET.environmentId, addedThreadId); + yield* Ref.set(removedThreads, []); const snapshotLoader = ShellSnapshotLoader.of({ - load: () => Ref.update(loaderCalls, (count) => count + 1).pipe(Effect.as(Option.none())), + load: () => + SubscriptionRef.update(loaderCalls, (count) => count + 1).pipe( + Effect.as(Option.some(httpSnapshot)), + ), }); const shellState = yield* makeEnvironmentShellState().pipe( Effect.provideService(EnvironmentSupervisor.EnvironmentSupervisor, supervisor), Effect.provideService(Persistence.EnvironmentCacheStore, cache), Effect.provideService(ShellSnapshotLoader, snapshotLoader), - Effect.provideService( - ConnectionWakeups.ConnectionWakeups, - ConnectionWakeups.ConnectionWakeups.of({ changes: Stream.fromQueue(wakeups) }), - ), ); - const subscribeInput = yield* Queue.take(subscribeInputs); - expect(subscribeInput.afterSequence).toBeUndefined(); - expect(subscribeInput.requestCompletionMarker).toBe(true); - expect(yield* Ref.get(loaderCalls)).toBe(1); + // Wait until the authoritative socket snapshot subscription is established. + yield* SubscriptionRef.changes(capturedInput).pipe( + Stream.filter((value) => value !== null), + Stream.runHead, + ); + + expect(yield* SubscriptionRef.get(capturedInput)).toEqual({ + requestCompletionMarker: true, + }); + expect(yield* SubscriptionRef.get(loaderCalls)).toBe(1); const synchronizing = yield* SubscriptionRef.get(shellState); expect(synchronizing.status).toBe("synchronizing"); - expect(Option.getOrThrow(synchronizing.snapshot)).toEqual(cachedSnapshot); + expect(Option.getOrThrow(synchronizing.snapshot)).toEqual(httpSnapshot); + expect(yield* Ref.get(removedThreads)).toEqual(["stale-thread"]); + yield* persistCachedThread( + cache, + TARGET.environmentId, + { snapshotSequence: 9, thread: { id: addedThreadId } as never }, + staleGeneration, + ); + expect(yield* Ref.get(savedThreads)).toEqual([]); + yield* persistCachedThread( + cache, + TARGET.environmentId, + { snapshotSequence: 9, thread: { id: addedThreadId } as never }, + cachedThreadGeneration(cache, TARGET.environmentId, addedThreadId), + ); + expect(yield* Ref.get(savedThreads)).toEqual([addedThreadId]); - yield* Queue.offer(events, { kind: "snapshot", snapshot: resetSnapshot }); + yield* Queue.offer(events, { + kind: "snapshot", + snapshot: { + ...httpSnapshot, + snapshotSequence: 10, + threads: [], + updatedAt: "2026-06-07T00:00:01.000Z", + }, + }); yield* Queue.offer(events, { kind: "synchronized" }); yield* SubscriptionRef.changes(shellState).pipe( Stream.filter((value) => value.status === "live"), Stream.runHead, ); + expect(Option.getOrThrow((yield* SubscriptionRef.get(shellState)).snapshot).threads).toEqual( + [], + ); + expect(yield* Ref.get(removedThreads)).toEqual([ + "stale-thread", + "archived-after-http-snapshot", + ]); + }), + ); - const live = yield* SubscriptionRef.get(shellState); - expect(Option.getOrThrow(live.snapshot)).toEqual(resetSnapshot); - expect(yield* Ref.get(loaderCalls)).toBe(1); + it.effect( + "preserves the shell across an authorization session handoff until the replacement snapshot archives it", + () => + Effect.gen(function* () { + const threadId = ThreadId.make("archived-during-authorization-refresh"); + const activeSnapshot: OrchestrationShellSnapshot = { + snapshotSequence: 1, + projects: [], + threads: [{ id: threadId } as never], + updatedAt: "2026-06-06T00:00:00.000Z", + }; + const firstEvents = yield* Queue.unbounded(); + const replacementEvents = yield* Queue.unbounded(); + const subscriptionCount = yield* Ref.make(0); + const removedThreads = yield* Ref.make([]); + const savedThreads = yield* Ref.make([]); + const makeClient = (events: Queue.Queue) => + ({ + [ORCHESTRATION_WS_METHODS.subscribeShell]: () => + Stream.unwrap( + Ref.update(subscriptionCount, (count) => count + 1).pipe( + Effect.as(Stream.fromQueue(events)), + ), + ), + }) as unknown as WsRpcProtocolClient; + const firstSession = session(makeClient(firstEvents)); + const replacementSession = session(makeClient(replacementEvents)); + const activeSession = yield* SubscriptionRef.make>( + Option.some(firstSession), + ); + const supervisor = EnvironmentSupervisor.EnvironmentSupervisor.of({ + target: TARGET, + state: yield* SubscriptionRef.make(AVAILABLE_CONNECTION_STATE), + session: activeSession, + prepared: yield* SubscriptionRef.make(Option.some(PREPARED)), + connect: Effect.void, + disconnect: Effect.void, + retryNow: Effect.void, + } satisfies EnvironmentSupervisor.EnvironmentSupervisor["Service"]); + const cache = Persistence.EnvironmentCacheStore.of({ + loadShell: () => Effect.succeed(Option.some(activeSnapshot)), + saveShell: () => Effect.void, + loadThread: () => Effect.succeed(Option.none()), + saveThread: (_environmentId, snapshot) => + Ref.update(savedThreads, (threadIds) => [...threadIds, snapshot.thread.id]), + removeThread: (_environmentId, removedThreadId) => + Ref.update(removedThreads, (threadIds) => [...threadIds, removedThreadId]), + loadServerConfig: () => Effect.succeed(Option.none()), + saveServerConfig: () => Effect.void, + loadVcsRefs: () => Effect.succeed(Option.none()), + saveVcsRefs: () => Effect.void, + removeVcsRefs: () => Effect.void, + clearVcsRefs: () => Effect.void, + clear: () => Effect.void, + }); + yield* retainCachedThread(cache, TARGET.environmentId, threadId); + const generationBeforeArchive = cachedThreadGeneration( + cache, + TARGET.environmentId, + threadId, + ); + const loaderCalls = yield* Ref.make(0); + const shellState = yield* makeEnvironmentShellState().pipe( + Effect.provideService(EnvironmentSupervisor.EnvironmentSupervisor, supervisor), + Effect.provideService(Persistence.EnvironmentCacheStore, cache), + Effect.provideService( + ShellSnapshotLoader, + ShellSnapshotLoader.of({ + load: () => + Ref.updateAndGet(loaderCalls, (count) => count + 1).pipe( + Effect.map((count) => + count === 1 ? Option.some(activeSnapshot) : Option.none(), + ), + ), + }), + ), + ); - yield* Queue.offer(wakeups, "application-active"); - const resumedInput = yield* Queue.take(subscribeInputs); - expect(resumedInput.afterSequence).toBe(resetSnapshot.snapshotSequence); - expect(resumedInput.requestCompletionMarker).toBe(true); - expect(yield* Ref.get(loaderCalls)).toBe(1); + for (let attempt = 0; attempt < 100; attempt += 1) { + if ((yield* Ref.get(subscriptionCount)) === 1) break; + yield* Effect.yieldNow; + } + yield* Queue.offer(firstEvents, { kind: "snapshot", snapshot: activeSnapshot }); + yield* Queue.offer(firstEvents, { kind: "synchronized" }); + yield* SubscriptionRef.changes(shellState).pipe( + Stream.filter((value) => value.status === "live"), + Stream.runHead, + ); + + // The supervisor publishes a replacement session only after its new + // authorization is ready. Shell synchronization keeps the previous + // authoritative baseline until that session supplies its own snapshot. + yield* SubscriptionRef.set(activeSession, Option.some(replacementSession)); + for (let attempt = 0; attempt < 100; attempt += 1) { + if ((yield* Ref.get(subscriptionCount)) === 2) break; + yield* Effect.yieldNow; + } + expect(yield* Ref.get(loaderCalls)).toBe(2); + expect((yield* SubscriptionRef.get(shellState)).status).toBe("synchronizing"); + expect( + Option.getOrThrow((yield* SubscriptionRef.get(shellState)).snapshot).threads, + ).toEqual(activeSnapshot.threads); + expect(yield* Ref.get(removedThreads)).toEqual([]); + + const archivedSnapshot: OrchestrationShellSnapshot = { + ...activeSnapshot, + snapshotSequence: 2, + threads: [], + updatedAt: "2026-06-06T00:00:01.000Z", + }; + yield* Queue.offer(replacementEvents, { + kind: "snapshot", + snapshot: archivedSnapshot, + }); + yield* Queue.offer(replacementEvents, { kind: "synchronized" }); + yield* SubscriptionRef.changes(shellState).pipe( + Stream.filter( + (value) => + value.status === "live" && + Option.isSome(value.snapshot) && + value.snapshot.value.snapshotSequence === 2, + ), + Stream.runHead, + ); + + expect(yield* Ref.get(removedThreads)).toEqual([threadId]); + yield* persistCachedThread( + cache, + TARGET.environmentId, + { snapshotSequence: 1, thread: { id: threadId } as never }, + generationBeforeArchive, + ); + expect(yield* Ref.get(savedThreads)).toEqual([]); + }), + ); + + it.effect("retries failed detail eviction while the thread remains absent", () => + Effect.gen(function* () { + const staleThreadId = ThreadId.make("stale-thread"); + const cachedSnapshot: OrchestrationShellSnapshot = { + snapshotSequence: 1, + projects: [], + threads: [{ id: staleThreadId } as never], + updatedAt: "2026-06-06T00:00:00.000Z", + }; + const httpSnapshot: OrchestrationShellSnapshot = { + ...cachedSnapshot, + snapshotSequence: 2, + threads: [], + updatedAt: "2026-06-06T00:00:01.000Z", + }; + const events = yield* Queue.unbounded(); + const capturedInput = yield* SubscriptionRef.make(false); + const evictionAttempts = yield* Ref.make(0); + const client = { + [ORCHESTRATION_WS_METHODS.subscribeShell]: () => + Stream.unwrap( + SubscriptionRef.set(capturedInput, true).pipe(Effect.as(Stream.fromQueue(events))), + ), + } as unknown as WsRpcProtocolClient; + const supervisor = EnvironmentSupervisor.EnvironmentSupervisor.of({ + target: TARGET, + state: yield* SubscriptionRef.make(AVAILABLE_CONNECTION_STATE), + session: yield* SubscriptionRef.make(Option.some(session(client))), + prepared: yield* SubscriptionRef.make(Option.some(PREPARED)), + connect: Effect.void, + disconnect: Effect.void, + retryNow: Effect.void, + } satisfies EnvironmentSupervisor.EnvironmentSupervisor["Service"]); + const cache = Persistence.EnvironmentCacheStore.of({ + loadShell: () => Effect.succeed(Option.some(cachedSnapshot)), + saveShell: () => Effect.void, + loadThread: () => Effect.succeed(Option.none()), + saveThread: () => Effect.void, + removeThread: () => + Ref.updateAndGet(evictionAttempts, (attempts) => attempts + 1).pipe( + Effect.flatMap((attempts) => + attempts === 1 + ? Effect.fail( + new Persistence.ConnectionPersistenceError({ + operation: "remove-thread", + message: "temporary failure", + }), + ) + : Effect.void, + ), + ), + loadServerConfig: () => Effect.succeed(Option.none()), + saveServerConfig: () => Effect.void, + loadVcsRefs: () => Effect.succeed(Option.none()), + saveVcsRefs: () => Effect.void, + removeVcsRefs: () => Effect.void, + clearVcsRefs: () => Effect.void, + clear: () => Effect.void, + }); + const shellState = yield* makeEnvironmentShellState().pipe( + Effect.provideService(EnvironmentSupervisor.EnvironmentSupervisor, supervisor), + Effect.provideService(Persistence.EnvironmentCacheStore, cache), + Effect.provideService( + ShellSnapshotLoader, + ShellSnapshotLoader.of({ + load: () => Effect.succeed(Option.some(httpSnapshot)), + }), + ), + ); + + yield* SubscriptionRef.changes(capturedInput).pipe( + Stream.filter((subscribed) => subscribed), + Stream.runHead, + ); + expect(yield* Ref.get(evictionAttempts)).toBe(1); + + yield* Queue.offer(events, { + kind: "snapshot", + snapshot: { + ...httpSnapshot, + snapshotSequence: 3, + updatedAt: "2026-06-06T00:00:02.000Z", + }, + }); + yield* SubscriptionRef.changes(shellState).pipe( + Stream.filter( + (value) => Option.isSome(value.snapshot) && value.snapshot.value.snapshotSequence === 3, + ), + Stream.runHead, + ); + + expect(yield* Ref.get(evictionAttempts)).toBe(2); }), ); - it.effect("resubscribes from the in-memory shell cursor when the app becomes active", () => + it.effect("revalidates the shell without trusting a foreground cursor", () => Effect.gen(function* () { const events = yield* Queue.unbounded(); const wakeups = yield* Queue.unbounded(); const loaderCalls = yield* Ref.make(0); - const capturedAfterSequences = yield* Ref.make>([]); + const subscriptionCount = yield* Ref.make(0); + const subscribeInputs = yield* Ref.make< + ReadonlyArray<{ + readonly afterSequence?: number; + readonly requestCompletionMarker?: boolean; + }> + >([]); const client = { - [ORCHESTRATION_WS_METHODS.subscribeShell]: (input: { readonly afterSequence?: number }) => + [ORCHESTRATION_WS_METHODS.subscribeShell]: (input: { + readonly afterSequence?: number; + readonly requestCompletionMarker?: boolean; + }) => Stream.unwrap( - Ref.update(capturedAfterSequences, (captured) => [ - ...captured, - input.afterSequence, + Effect.all([ + Ref.update(subscriptionCount, (count) => count + 1), + Ref.update(subscribeInputs, (inputs) => [...inputs, input]), ]).pipe(Effect.as(Stream.fromQueue(events))), ), } as unknown as WsRpcProtocolClient; @@ -381,7 +661,9 @@ describe("environment shell synchronization", () => { load: () => Ref.updateAndGet(loaderCalls, (count) => count + 1).pipe( Effect.map((count) => - Option.some({ ...LIVE_SHELL_SNAPSHOT, snapshotSequence: count * 10 }), + count === 1 + ? Option.some({ ...LIVE_SHELL_SNAPSHOT, snapshotSequence: count * 10 }) + : Option.none(), ), ), }); @@ -395,60 +677,68 @@ describe("environment shell synchronization", () => { ), ); - // A new session starts from an authoritative HTTP snapshot. - for (let attempt = 0; attempt < 100; attempt += 1) { - if ((yield* Ref.get(capturedAfterSequences)).length >= 1) break; - yield* Effect.yieldNow; - } - expect(yield* Ref.get(capturedAfterSequences)).toEqual([10]); - yield* Queue.offer(events, { kind: "synchronized" }); yield* SubscriptionRef.changes(shellState).pipe( - Stream.filter((value) => value.status === "live"), + Stream.filter( + (value) => + value.status === "synchronizing" && + Option.isSome(value.snapshot) && + value.snapshot.value.snapshotSequence === 10, + ), Stream.runHead, ); - - // A newer snapshot arrives on the stream and advances the cursor. - yield* Queue.offer(events, { - kind: "snapshot", - snapshot: { ...LIVE_SHELL_SNAPSHOT, snapshotSequence: 40 }, - }); + yield* Queue.offer(events, { kind: "synchronized" }); yield* SubscriptionRef.changes(shellState).pipe( - Stream.filter( - (value) => Option.isSome(value.snapshot) && value.snapshot.value.snapshotSequence === 40, - ), + Stream.filter((value) => value.status === "live"), Stream.runHead, ); + expect(yield* Ref.get(subscribeInputs)).toEqual([{ requestCompletionMarker: true }]); yield* Queue.offer(wakeups, "application-active"); for (let attempt = 0; attempt < 100; attempt += 1) { - if ((yield* Ref.get(capturedAfterSequences)).length >= 2) break; + if ((yield* Ref.get(subscriptionCount)) >= 2) break; yield* Effect.yieldNow; } - expect(yield* Ref.get(capturedAfterSequences)).toEqual([10, 40]); - yield* Queue.offer(events, { kind: "synchronized" }); + + expect(yield* Ref.get(loaderCalls)).toBe(1); + expect(yield* Ref.get(subscriptionCount)).toBe(2); + expect(yield* Ref.get(subscribeInputs)).toEqual([ + { requestCompletionMarker: true }, + { requestCompletionMarker: true }, + ]); yield* Queue.offer(wakeups, "application-active-probe"); for (let attempt = 0; attempt < 100; attempt += 1) { - if ((yield* Ref.get(capturedAfterSequences)).length >= 3) break; + if ((yield* Ref.get(subscriptionCount)) >= 3) break; yield* Effect.yieldNow; } - expect(yield* Ref.get(capturedAfterSequences)).toEqual([10, 40, 40]); + expect(yield* Ref.get(loaderCalls)).toBe(1); + expect(yield* Ref.get(subscriptionCount)).toBe(3); + expect(yield* Ref.get(subscribeInputs)).toEqual([ + { requestCompletionMarker: true }, + { requestCompletionMarker: true }, + { requestCompletionMarker: true }, + ]); yield* Queue.offer(wakeups, "application-active-reconnect"); for (let attempt = 0; attempt < 10; attempt += 1) { yield* Effect.yieldNow; } - expect((yield* Ref.get(capturedAfterSequences)).length).toBe(3); expect(yield* Ref.get(loaderCalls)).toBe(1); + expect(yield* Ref.get(subscriptionCount)).toBe(3); - // Replacing the session performs another authoritative refresh. + // A new RPC session attempts an authoritative HTTP refresh. If that + // refresh fails, it still requests a full socket-owned snapshot rather + // than trusting the previous session's cursor. yield* SubscriptionRef.set(activeSession, Option.some(session(client))); for (let attempt = 0; attempt < 100; attempt += 1) { - if ((yield* Ref.get(capturedAfterSequences)).length >= 4) break; + if ((yield* Ref.get(subscriptionCount)) >= 4) break; yield* Effect.yieldNow; } - expect(yield* Ref.get(capturedAfterSequences)).toEqual([10, 40, 40, 20]); expect(yield* Ref.get(loaderCalls)).toBe(2); + expect(yield* Ref.get(subscriptionCount)).toBe(4); + expect((yield* Ref.get(subscribeInputs)).at(-1)).toEqual({ + requestCompletionMarker: true, + }); }), ); }); diff --git a/packages/client-runtime/src/state/shell.ts b/packages/client-runtime/src/state/shell.ts index b7f39b509462..910c06951426 100644 --- a/packages/client-runtime/src/state/shell.ts +++ b/packages/client-runtime/src/state/shell.ts @@ -4,6 +4,7 @@ import { type OrchestrationShellSnapshot, type OrchestrationShellStreamItem, type ServerConfig, + type ThreadId, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; import * as Effect from "effect/Effect"; @@ -25,6 +26,7 @@ import type { RpcSession } from "../rpc/session.ts"; import { ShellSnapshotLoader } from "./shellSnapshotHttp.ts"; import { applyShellStreamEvent } from "./shellReducer.ts"; import { type EnvironmentCatalogState, enabledEnvironmentIds } from "./connections.ts"; +import { evictCachedThread, reviveCachedThread } from "./threadCache.ts"; import { followStreamInEnvironment } from "./runtime.ts"; export type EnvironmentShellStatus = "empty" | "cached" | "synchronizing" | "live"; @@ -72,6 +74,7 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make") error: Option.none(), }); const awaitingCompletion = yield* Ref.make(false); + const pendingThreadEvictions = yield* Ref.make>(new Set()); const lastAuthoritativeSession = yield* Ref.make(null); const activeSubscriptionSession = yield* Ref.make(null); const persistence = yield* Queue.sliding(1); @@ -172,7 +175,54 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make") }; } yield* Ref.set(awaitingCompletion, waiting); - if (next === initial) return; + if (next === initial || Option.isNone(next.snapshot)) return; + const nextSnapshot = next.snapshot.value; + const nextThreadIds = new Set(nextSnapshot.threads.map((thread) => thread.id)); + const currentThreadIds = Option.match(initial.snapshot, { + onNone: () => new Set(), + onSome: (snapshot) => new Set(snapshot.threads.map((thread) => thread.id)), + }); + const removedThreadIds = Option.match(initial.snapshot, { + onNone: () => [] as ReadonlyArray, + onSome: (snapshot) => + snapshot.threads + .filter((thread) => !nextThreadIds.has(thread.id)) + .map((thread) => thread.id), + }); + const addedThreadIds = Option.match(initial.snapshot, { + onNone: () => [] as ReadonlyArray, + onSome: () => + nextSnapshot.threads + .filter((thread) => !currentThreadIds.has(thread.id)) + .map((thread) => thread.id), + }); + + const evictionCandidates = new Set([ + ...(yield* Ref.get(pendingThreadEvictions)), + ...removedThreadIds, + ]); + for (const threadId of nextThreadIds) { + evictionCandidates.delete(threadId); + } + // Advance cache tombstones before publishing the new shell so detail + // observers cannot enqueue an obsolete write in the transition window. + // Keep failed disk removals pending while the shell still omits the thread; + // a later snapshot or event can then retry instead of stranding stale data. + const evictionResults = yield* Effect.forEach(evictionCandidates, (threadId) => + evictCachedThread(cache, environmentId, threadId).pipe( + Effect.map((removed) => [threadId, removed] as const), + ), + ); + yield* Ref.set( + pendingThreadEvictions, + new Set(evictionResults.filter(([, removed]) => !removed).map(([threadId]) => threadId)), + ); + yield* Effect.forEach( + addedThreadIds, + (threadId) => reviveCachedThread(cache, environmentId, threadId), + { discard: true }, + ); + yield* SubscriptionRef.set(state, next); if (receivedSnapshot) { const session = yield* Ref.get(activeSubscriptionSession); @@ -205,12 +255,17 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make") yield* setSynchronizing; // Foreground resubscriptions on the same live session can resume from - // the in-memory cursor. A new session reloads the authoritative HTTP - // snapshot so a valid cursor cannot preserve incomplete cached data. + // a socket-owned snapshot. A new session reloads the authoritative + // HTTP snapshot first so a valid cursor cannot preserve incomplete + // cached data. const hasAuthoritativeSnapshot = (yield* Ref.get(lastAuthoritativeSession)) === session; let canResume = hasAuthoritativeSnapshot; let current = yield* SubscriptionRef.get(state); - if (!hasAuthoritativeSnapshot || Option.isNone(current.snapshot)) { + if ( + !hasAuthoritativeSnapshot || + Option.isNone(current.snapshot) || + !supportsCompletionMarker + ) { const prepared = yield* SubscriptionRef.get(supervisor.prepared).pipe( Effect.flatMap( Option.match({ @@ -247,9 +302,16 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make") error: Option.none(), })); } + // A cold archive can compact the event that removed a thread after an + // HTTP snapshot was read or while the app was backgrounded. Servers + // with the synchronized handoff therefore revalidate using a full + // socket-owned snapshot instead of resuming from a potentially stale + // cursor. Older servers retain the HTTP-refresh cursor path above. + if (supportsCompletionMarker) { + return { requestCompletionMarker: true as const }; + } return { afterSequence: current.snapshot.value.snapshotSequence, - ...(supportsCompletionMarker ? { requestCompletionMarker: true as const } : {}), }; }), { diff --git a/packages/client-runtime/src/state/shellSnapshotHttp.ts b/packages/client-runtime/src/state/shellSnapshotHttp.ts index fe5a3d5fcec1..71af5befee13 100644 --- a/packages/client-runtime/src/state/shellSnapshotHttp.ts +++ b/packages/client-runtime/src/state/shellSnapshotHttp.ts @@ -48,7 +48,8 @@ export const fetchEnvironmentShellSnapshot = Effect.fn( * Loads the environment shell snapshot over HTTP, returning `Option.none()` when * it cannot be loaded (so the caller falls back to the socket-embedded snapshot). * Decouples the shell state machine from the underlying HTTP + DPoP details and - * keeps them out of test contexts. + * keeps them out of test contexts. Callers refresh this snapshot for every + * WebSocket session; the persisted cache is only an immediate rendering source. */ export class ShellSnapshotLoader extends Context.Service< ShellSnapshotLoader, diff --git a/packages/client-runtime/src/state/threadCache.ts b/packages/client-runtime/src/state/threadCache.ts new file mode 100644 index 000000000000..1e0689bd77d6 --- /dev/null +++ b/packages/client-runtime/src/state/threadCache.ts @@ -0,0 +1,237 @@ +import type { EnvironmentId, ThreadId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Semaphore from "effect/Semaphore"; + +import { safeErrorLogAttributes } from "../errors/safeLog.ts"; +import { EnvironmentCacheStore } from "../platform/persistence.ts"; + +interface ThreadCacheState { + generation: number; + evicted: boolean; + retainers: number; + operations: number; + readonly evictionListeners: Set<() => void>; + readonly lock: Semaphore.Semaphore; +} + +type EnvironmentThreadCacheStates = Map; +type CacheStates = Map; + +const cacheStates = new WeakMap(); + +function existingThreadCacheState( + cache: EnvironmentCacheStore["Service"], + environmentId: EnvironmentId, + threadId: ThreadId, +): ThreadCacheState | undefined { + return cacheStates.get(cache)?.get(environmentId)?.get(threadId); +} + +function threadCacheState( + cache: EnvironmentCacheStore["Service"], + environmentId: EnvironmentId, + threadId: ThreadId, +): ThreadCacheState { + let environmentEntries = cacheStates.get(cache); + if (environmentEntries === undefined) { + environmentEntries = new Map(); + cacheStates.set(cache, environmentEntries); + } + + let threadEntries = environmentEntries.get(environmentId); + if (threadEntries === undefined) { + threadEntries = new Map(); + environmentEntries.set(environmentId, threadEntries); + } + + const existing = threadEntries.get(threadId); + if (existing !== undefined) { + return existing; + } + + const created: ThreadCacheState = { + generation: 0, + evicted: false, + retainers: 0, + operations: 0, + evictionListeners: new Set(), + lock: Semaphore.makeUnsafe(1), + }; + threadEntries.set(threadId, created); + return created; +} + +function pruneThreadCacheState( + cache: EnvironmentCacheStore["Service"], + environmentId: EnvironmentId, + threadId: ThreadId, + state: ThreadCacheState, +): void { + if (state.retainers > 0 || state.operations > 0) return; + + const environmentEntries = cacheStates.get(cache); + const threadEntries = environmentEntries?.get(environmentId); + if (threadEntries?.get(threadId) !== state) return; + + threadEntries.delete(threadId); + if (threadEntries.size === 0) { + environmentEntries?.delete(environmentId); + } + if (environmentEntries?.size === 0) { + cacheStates.delete(cache); + } +} + +function withThreadCacheState( + cache: EnvironmentCacheStore["Service"], + environmentId: EnvironmentId, + threadId: ThreadId, + use: (state: ThreadCacheState) => Effect.Effect, +): Effect.Effect { + return Effect.acquireUseRelease( + Effect.sync(() => { + const state = threadCacheState(cache, environmentId, threadId); + state.operations += 1; + return state; + }), + use, + (state) => + Effect.sync(() => { + state.operations -= 1; + pruneThreadCacheState(cache, environmentId, threadId, state); + }), + ); +} + +// Warm snapshots retain this boundary after the RPC scope closes. Their +// disposer releases both the listener and the otherwise collectible tombstone. +export function retainCachedThreadUnsafe( + cache: EnvironmentCacheStore["Service"], + environmentId: EnvironmentId, + threadId: ThreadId, + onEviction?: () => void, +): () => void { + const state = threadCacheState(cache, environmentId, threadId); + state.retainers += 1; + if (onEviction) state.evictionListeners.add(onEviction); + let released = false; + return () => { + if (released) return; + released = true; + if (onEviction) state.evictionListeners.delete(onEviction); + state.retainers -= 1; + pruneThreadCacheState(cache, environmentId, threadId, state); + }; +} + +export const retainCachedThread = Effect.fn("EnvironmentThreadCache.retain")(function* ( + cache: EnvironmentCacheStore["Service"], + environmentId: EnvironmentId, + threadId: ThreadId, +) { + yield* Effect.acquireRelease( + Effect.sync(() => retainCachedThreadUnsafe(cache, environmentId, threadId)), + (release) => Effect.sync(release), + ); +}); + +export function cachedThreadGeneration( + cache: EnvironmentCacheStore["Service"], + environmentId: EnvironmentId, + threadId: ThreadId, +): number { + return existingThreadCacheState(cache, environmentId, threadId)?.generation ?? 0; +} + +export function isCachedThreadEvicted( + cache: EnvironmentCacheStore["Service"], + environmentId: EnvironmentId, + threadId: ThreadId, +): boolean { + return existingThreadCacheState(cache, environmentId, threadId)?.evicted === true; +} + +export const persistCachedThread = Effect.fn("EnvironmentThreadCache.persist")(function* ( + cache: EnvironmentCacheStore["Service"], + environmentId: EnvironmentId, + snapshot: Parameters[1], + generation: number, +) { + const threadId = snapshot.thread.id; + return yield* withThreadCacheState(cache, environmentId, threadId, (state) => + // Keep persistence under the same permit as eviction. Otherwise an older + // write can pass its generation check and finish after cache removal. + state.lock.withPermit( + Effect.gen(function* () { + if (state.evicted || state.generation !== generation) { + return false; + } + return yield* cache.saveThread(environmentId, snapshot).pipe( + Effect.as(true), + Effect.catch((error) => + Effect.logWarning("Could not persist the thread cache.").pipe( + Effect.annotateLogs({ + environmentId, + threadId, + ...safeErrorLogAttributes(error), + }), + Effect.as(false), + ), + ), + ); + }), + ), + ); +}); + +export const evictCachedThread = Effect.fn("EnvironmentThreadCache.evict")(function* ( + cache: EnvironmentCacheStore["Service"], + environmentId: EnvironmentId, + threadId: ThreadId, + isCurrent: () => boolean = () => true, +) { + return yield* withThreadCacheState(cache, environmentId, threadId, (state) => + state.lock.withPermit( + Effect.gen(function* () { + if (!isCurrent()) return false; + state.generation += 1; + state.evicted = true; + for (const onEviction of state.evictionListeners) onEviction(); + return yield* cache.removeThread(environmentId, threadId).pipe( + Effect.as(true), + Effect.catch((error) => + Effect.logWarning("Could not evict cached thread detail.").pipe( + Effect.annotateLogs({ + environmentId, + threadId, + ...safeErrorLogAttributes(error), + }), + Effect.as(false), + ), + ), + ); + }), + ), + ); +}); + +export const reviveCachedThread = Effect.fn("EnvironmentThreadCache.revive")(function* ( + cache: EnvironmentCacheStore["Service"], + environmentId: EnvironmentId, + threadId: ThreadId, + isCurrent: () => boolean = () => true, +) { + yield* withThreadCacheState(cache, environmentId, threadId, (state) => + state.lock.withPermit( + Effect.sync(() => { + if (!isCurrent()) return; + if (state.evicted) { + // Invalidate writes that captured the eviction generation while the + // tombstone was active before making the cache writable again. + state.generation += 1; + state.evicted = false; + } + }), + ), + ); +}); diff --git a/packages/client-runtime/src/state/threadCommands.archive.test.ts b/packages/client-runtime/src/state/threadCommands.archive.test.ts new file mode 100644 index 000000000000..78eb003a52d6 --- /dev/null +++ b/packages/client-runtime/src/state/threadCommands.archive.test.ts @@ -0,0 +1,145 @@ +import { + CommandId, + EnvironmentId, + ORCHESTRATION_WS_METHODS, + ThreadId, + type ClientOrchestrationCommand, +} from "@t3tools/contracts"; +import { describe, expect, it } from "@effect/vitest"; +import * as Crypto from "effect/Crypto"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Ref from "effect/Ref"; +import * as SubscriptionRef from "effect/SubscriptionRef"; + +import { + AVAILABLE_CONNECTION_STATE, + PrimaryConnectionTarget, + type PreparedConnection, +} from "../connection/model.ts"; +import * as EnvironmentSupervisor from "../connection/supervisor.ts"; +import * as Persistence from "../platform/persistence.ts"; +import type { WsRpcProtocolClient } from "../rpc/protocol.ts"; +import { makeTestRpcSession } from "../rpc/testUtils/rpcSession.ts"; +import { archiveThreadAndEvictCache } from "./threadCommands.ts"; + +const TEST_CRYPTO_LAYER = Layer.succeed( + Crypto.Crypto, + Crypto.make({ + randomBytes: (size) => new Uint8Array(size), + digest: (_algorithm, data) => Effect.succeed(data), + }), +); + +const ENVIRONMENT_ID = EnvironmentId.make("environment-1"); +const THREAD_ID = ThreadId.make("thread-1"); +const TARGET = new PrimaryConnectionTarget({ + environmentId: ENVIRONMENT_ID, + label: "Test environment", + httpBaseUrl: "https://environment.example.test", + wsBaseUrl: "wss://environment.example.test", +}); + +const makeSupervisor = Effect.fn("TestThreadCommands.makeSupervisor")(function* ( + client: WsRpcProtocolClient, +) { + const session = makeTestRpcSession(client); + return EnvironmentSupervisor.EnvironmentSupervisor.of({ + target: TARGET, + state: yield* SubscriptionRef.make(AVAILABLE_CONNECTION_STATE), + session: yield* SubscriptionRef.make(Option.some(session)), + prepared: yield* SubscriptionRef.make(Option.none()), + connect: Effect.void, + disconnect: Effect.void, + retryNow: Effect.void, + } satisfies EnvironmentSupervisor.EnvironmentSupervisor["Service"]); +}); + +function makeCache( + removeThread: Persistence.EnvironmentCacheStore["Service"]["removeThread"], +): Persistence.EnvironmentCacheStore["Service"] { + return Persistence.EnvironmentCacheStore.of({ + loadShell: () => Effect.succeed(Option.none()), + saveShell: () => Effect.void, + loadThread: () => Effect.succeed(Option.none()), + saveThread: () => Effect.void, + removeThread, + loadServerConfig: () => Effect.succeed(Option.none()), + saveServerConfig: () => Effect.void, + loadVcsRefs: () => Effect.succeed(Option.none()), + saveVcsRefs: () => Effect.void, + removeVcsRefs: () => Effect.void, + clearVcsRefs: () => Effect.void, + clear: () => Effect.void, + }); +} + +describe("thread commands", () => { + it.effect("evicts cached detail after the archive acknowledgement", () => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const acknowledged = yield* Deferred.make(); + const dispatched = yield* Ref.make([]); + const removals = yield* Ref.make>([]); + const client = { + [ORCHESTRATION_WS_METHODS.dispatchCommand]: (command: ClientOrchestrationCommand) => + Ref.update(dispatched, (commands) => [...commands, command]).pipe( + Effect.andThen(Deferred.succeed(started, undefined)), + Effect.andThen(Deferred.await(acknowledged)), + Effect.as({ sequence: 1 }), + ), + } as unknown as WsRpcProtocolClient; + const supervisor = yield* makeSupervisor(client); + const cache = makeCache((environmentId, threadId) => + Ref.update(removals, (entries) => [...entries, [environmentId, threadId] as const]), + ); + + const archive = yield* archiveThreadAndEvictCache({ + commandId: CommandId.make("archive-command"), + threadId: THREAD_ID, + }).pipe( + Effect.provideService(EnvironmentSupervisor.EnvironmentSupervisor, supervisor), + Effect.provideService(Persistence.EnvironmentCacheStore, cache), + Effect.forkChild({ startImmediately: true }), + ); + + yield* Deferred.await(started); + expect(yield* Ref.get(removals)).toEqual([]); + + yield* Deferred.succeed(acknowledged, undefined); + expect(yield* Fiber.join(archive)).toEqual({ sequence: 1 }); + expect(yield* Ref.get(removals)).toEqual([[ENVIRONMENT_ID, THREAD_ID]]); + }).pipe(Effect.provide(TEST_CRYPTO_LAYER)), + ); + + it.effect("keeps a successful archive successful when cache eviction fails", () => + Effect.gen(function* () { + const client = { + [ORCHESTRATION_WS_METHODS.dispatchCommand]: (_command: ClientOrchestrationCommand) => + Effect.succeed({ sequence: 1 }), + } as unknown as WsRpcProtocolClient; + const supervisor = yield* makeSupervisor(client); + const cache = makeCache(() => + Effect.fail( + new Persistence.ConnectionPersistenceError({ + operation: "remove-thread", + message: "IndexedDB unavailable", + }), + ), + ); + + const result = yield* archiveThreadAndEvictCache({ + commandId: CommandId.make("archive-command"), + threadId: THREAD_ID, + }).pipe( + Effect.provideService(EnvironmentSupervisor.EnvironmentSupervisor, supervisor), + Effect.provideService(Persistence.EnvironmentCacheStore, cache), + ); + + expect(result).toEqual({ sequence: 1 }); + }).pipe(Effect.provide(TEST_CRYPTO_LAYER)), + ); +}); diff --git a/packages/client-runtime/src/state/threadCommands.test.ts b/packages/client-runtime/src/state/threadCommands.test.ts index dc70ac5548b1..d52682ac2c33 100644 --- a/packages/client-runtime/src/state/threadCommands.test.ts +++ b/packages/client-runtime/src/state/threadCommands.test.ts @@ -20,6 +20,7 @@ import { Atom, AtomRegistry } from "effect/unstable/reactivity"; import { EnvironmentRegistry } from "../connection/registry.ts"; import { EnvironmentSupervisor } from "../connection/supervisor.ts"; +import { EnvironmentCacheStore } from "../platform/persistence.ts"; import type { RpcSession } from "../rpc/session.ts"; import { createThreadEnvironmentAtoms } from "./threadCommands.ts"; @@ -78,6 +79,20 @@ const makeHarness = Effect.fn("TestThreadCommands.makeHarness")(function* () { } as EnvironmentSupervisor["Service"]); const runtime = Atom.runtime( Layer.mergeAll( + Layer.succeed(EnvironmentCacheStore, { + loadShell: () => Effect.succeed(Option.none()), + saveShell: () => Effect.void, + loadThread: () => Effect.succeed(Option.none()), + saveThread: () => Effect.void, + removeThread: () => Effect.void, + loadServerConfig: () => Effect.succeed(Option.none()), + saveServerConfig: () => Effect.void, + loadVcsRefs: () => Effect.succeed(Option.none()), + saveVcsRefs: () => Effect.void, + removeVcsRefs: () => Effect.void, + clearVcsRefs: () => Effect.void, + clear: () => Effect.void, + }), Layer.succeed(EnvironmentRegistry, { run: (_environmentId, effect) => Effect.provideService(effect, EnvironmentSupervisor, supervisor), @@ -102,6 +117,7 @@ const makeHarness = Effect.fn("TestThreadCommands.makeHarness")(function* () { describe("remote thread lifecycle commands", () => { const actions = [ + ["archive", {}, { archivedAt: expect.any(String) }], ["settle", {}, { settledOverride: "settled", pinnedAt: null, snoozedUntil: null }], ["unsettle", { reason: "user" }, { settledOverride: "active", settledAt: null }], [ @@ -160,6 +176,81 @@ describe("remote thread lifecycle commands", () => { ); } + for (const shellFirst of [false, true]) { + it.effect( + `reconciles archive with a restored shell (${shellFirst ? "shell first" : "reply first"})`, + () => + Effect.gen(function* () { + const h = yield* makeHarness(); + const result = h.commands.archive.run(h.registry, { + environmentId: ENVIRONMENT_ID, + input: { threadId: THREAD_ID }, + }); + const request = yield* Queue.take(h.requests); + expect(h.registry.get(h.visibleAtom)?.threads[0]?.archivedAt).not.toBeNull(); + expect(h.registry.get(h.commands.snapshotAtom(EnvironmentId.make("other")))).toBe( + SNAPSHOT, + ); + const restored = { ...SNAPSHOT, snapshotSequence: 4 }; + if (shellFirst) h.registry.set(h.snapshotAtom(ENVIRONMENT_ID), restored); + yield* Deferred.succeed(request.reply, { sequence: 3 }); + expect((yield* Effect.promise(() => result))._tag).toBe("Success"); + if (!shellFirst) { + h.registry.set(h.snapshotAtom(ENVIRONMENT_ID), { ...SNAPSHOT, snapshotSequence: 2 }); + expect(h.registry.get(h.visibleAtom)?.threads[0]?.archivedAt).not.toBeNull(); + h.registry.set(h.snapshotAtom(ENVIRONMENT_ID), { + ...SNAPSHOT, + snapshotSequence: 3, + threads: [], + }); + expect(h.registry.get(h.visibleAtom)?.threads).toEqual([]); + h.registry.set(h.snapshotAtom(ENVIRONMENT_ID), restored); + } + expect(h.registry.get(h.visibleAtom)).toBe(restored); + }), + ); + } + + it.effect("keeps a second archive pending when an earlier archive fails", () => + Effect.gen(function* () { + const h = yield* makeHarness(); + const target = { environmentId: ENVIRONMENT_ID, input: { threadId: THREAD_ID } }; + const firstResult = h.commands.archive.run(h.registry, target); + const first = yield* Queue.take(h.requests); + const secondResult = h.commands.archive.run(h.registry, target); + yield* Deferred.fail(first.reply, new Error("Archive rejected")); + yield* Effect.promise(() => firstResult); + expect(h.registry.get(h.visibleAtom)?.threads[0]?.archivedAt).not.toBeNull(); + const second = yield* Queue.take(h.requests); + yield* Deferred.interrupt(second.reply); + expect((yield* Effect.promise(() => secondResult))._tag).toBe("Failure"); + expect(h.registry.get(h.visibleAtom)).toBe(SNAPSHOT); + }), + ); + + it.effect("keeps an accepted archive hidden when a queued unarchive fails", () => + Effect.gen(function* () { + const h = yield* makeHarness(); + const target = { environmentId: ENVIRONMENT_ID, input: { threadId: THREAD_ID } }; + const archived = h.commands.archive.run(h.registry, target); + const first = yield* Queue.take(h.requests); + const restored = h.commands.unarchive.run(h.registry, target); + expect(h.registry.get(h.visibleAtom)?.threads[0]?.archivedAt).toBeNull(); + yield* Deferred.succeed(first.reply, { sequence: 3 }); + yield* Effect.promise(() => archived); + const second = yield* Queue.take(h.requests); + yield* Deferred.fail(second.reply, new Error("Restore rejected")); + yield* Effect.promise(() => restored); + expect(h.registry.get(h.visibleAtom)?.threads[0]?.archivedAt).not.toBeNull(); + h.registry.set(h.snapshotAtom(ENVIRONMENT_ID), { + ...SNAPSHOT, + snapshotSequence: 3, + threads: [], + }); + expect(h.registry.get(h.visibleAtom)?.threads).toEqual([]); + }), + ); + it.effect("keeps the preview after acknowledgement until the matching shell update arrives", () => Effect.gen(function* () { const h = yield* makeHarness(); diff --git a/packages/client-runtime/src/state/threadCommands.ts b/packages/client-runtime/src/state/threadCommands.ts index bb70356aab6a..6327ab67923a 100644 --- a/packages/client-runtime/src/state/threadCommands.ts +++ b/packages/client-runtime/src/state/threadCommands.ts @@ -1,4 +1,5 @@ import * as Crypto from "effect/Crypto"; +import * as Effect from "effect/Effect"; import { Atom } from "effect/unstable/reactivity"; import { WS_METHODS, @@ -9,11 +10,14 @@ import { import { createOptimisticThreadLifecycle } from "./threadLifecycle.ts"; import { canSnooze } from "./threadSettled.ts"; +import { EnvironmentSupervisor } from "../connection/supervisor.ts"; +import { EnvironmentCacheStore } from "../platform/persistence.ts"; import { createAtomCommandScheduler, createEnvironmentCommand, createEnvironmentRpcCommand, } from "./runtime.ts"; +import { evictCachedThread } from "./threadCache.ts"; import { type ArchiveThreadInput, type CreateThreadInput, @@ -96,8 +100,24 @@ export type { UpdateThreadMetadataInput, } from "../operations/commands.ts"; +export const archiveThreadAndEvictCache = Effect.fn( + "EnvironmentCommands.archiveThreadAndEvictCache", +)(function* (input: ArchiveThreadInput) { + return yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const result = yield* restore(archiveThread(input)); + const supervisor = yield* EnvironmentSupervisor; + const cache = yield* EnvironmentCacheStore; + // The shell/detail event paths also evict. This acknowledgement-side + // eviction closes the route-teardown race when those events arrive late. + yield* evictCachedThread(cache, supervisor.target.environmentId, input.threadId); + return result; + }), + ); +}); + export function createThreadEnvironmentAtoms( - runtime: Atom.AtomRuntime, + runtime: Atom.AtomRuntime, snapshotAtom: (environmentId: EnvironmentId) => Atom.Atom, ) { const scheduler = createAtomCommandScheduler(); @@ -121,7 +141,7 @@ export function createThreadEnvironmentAtoms( }), archive: createEnvironmentCommand(runtime, { label: "environment-data:commands:thread:archive", - execute: (input: ArchiveThreadInput) => archiveThread(input), + execute: (input: ArchiveThreadInput) => archiveThreadAndEvictCache(input), scheduler, concurrency, }), @@ -268,6 +288,14 @@ export function createThreadEnvironmentAtoms( return { ...commands, snapshotAtom: optimistic.snapshotAtom, + archive: optimistic.wrap(commands.archive, (thread, _input, now) => ({ + ...thread, + archivedAt: thread.archivedAt ?? now, + })), + unarchive: optimistic.wrap(commands.unarchive, (thread) => ({ + ...thread, + archivedAt: null, + })), settle: optimistic.wrap(commands.settle, (thread, _input, now, accepted) => !accepted && (!canSnooze(thread, { now }) || diff --git a/packages/client-runtime/src/state/threads-atoms.test.ts b/packages/client-runtime/src/state/threads-atoms.test.ts index 34e6857041d3..52f85c78ea6d 100644 --- a/packages/client-runtime/src/state/threads-atoms.test.ts +++ b/packages/client-runtime/src/state/threads-atoms.test.ts @@ -36,10 +36,11 @@ import { } from "../connection/model.ts"; import { EnvironmentSupervisor } from "../connection/supervisor.ts"; import { ConnectionWakeups, type ConnectionWakeup } from "../connection/wakeups.ts"; -import { EnvironmentCacheStore } from "../platform/persistence.ts"; +import { ConnectionPersistenceError, EnvironmentCacheStore } from "../platform/persistence.ts"; import type { WsRpcProtocolClient } from "../rpc/protocol.ts"; import type { RpcSession } from "../rpc/session.ts"; import { createEnvironmentThreadDetailAtoms } from "./threadDetail.ts"; +import { cachedThreadGeneration, evictCachedThread, reviveCachedThread } from "./threadCache.ts"; import { THREAD_SNAPSHOT_IDLE_TTL_MS } from "./threadRetention.ts"; import type { ThreadSnapshotWindow } from "./threadSnapshotHttp.ts"; import { @@ -96,6 +97,8 @@ const makeHarness = Effect.fn("TestThreadAtoms.makeHarness")(function* (options? readonly connected?: boolean; readonly httpNone?: boolean; readonly initialLoad?: Effect.Effect>; + readonly diskLoad?: ReturnType; + readonly removeThread?: ReturnType; readonly stream?: Stream.Stream; }) { const clock = yield* Clock.Clock; @@ -115,6 +118,27 @@ const makeHarness = Effect.fn("TestThreadAtoms.makeHarness")(function* (options? let diskLoads = 0; let opened = 0; let active = 0; + const savedSnapshots: OrchestrationThreadDetailSnapshot[] = []; + const cache = EnvironmentCacheStore.of({ + loadShell: () => Effect.succeed(Option.none()), + saveShell: () => Effect.void, + loadThread: () => + Effect.sync(() => { + diskLoads += 1; + }).pipe(Effect.andThen(options?.diskLoad ?? Effect.succeed(Option.none()))), + saveThread: (_environmentId, value) => + Effect.sync(() => { + savedSnapshots.push(value); + }), + removeThread: () => options?.removeThread ?? Effect.void, + loadServerConfig: () => Effect.succeed(Option.none()), + saveServerConfig: () => Effect.void, + loadVcsRefs: () => Effect.succeed(Option.none()), + saveVcsRefs: () => Effect.void, + removeVcsRefs: () => Effect.void, + clearVcsRefs: () => Effect.void, + clear: () => Effect.void, + }); const client = { [ORCHESTRATION_WS_METHODS.subscribeThread]: (input: { readonly afterSequence?: number }) => Stream.unwrap( @@ -197,27 +221,7 @@ const makeHarness = Effect.fn("TestThreadAtoms.makeHarness")(function* (options? Layer.succeed(Clock.Clock, clock), Layer.succeed(ConnectionWakeups, { changes: Stream.fromQueue(wakeups) }), Layer.succeed(EnvironmentRegistry, environmentRegistry), - Layer.succeed( - EnvironmentCacheStore, - EnvironmentCacheStore.of({ - loadShell: () => Effect.succeedNone, - saveShell: () => Effect.void, - loadThread: () => - Effect.sync(() => { - diskLoads += 1; - return Option.none(); - }), - saveThread: () => Effect.void, - removeThread: () => Effect.void, - loadServerConfig: () => Effect.succeedNone, - saveServerConfig: () => Effect.void, - loadVcsRefs: () => Effect.succeedNone, - saveVcsRefs: () => Effect.void, - removeVcsRefs: () => Effect.void, - clearVcsRefs: () => Effect.void, - clear: () => Effect.void, - }), - ), + Layer.succeed(EnvironmentCacheStore, cache), Layer.succeed( ThreadSnapshotLoader, ThreadSnapshotLoader.of({ @@ -257,6 +261,8 @@ const makeHarness = Effect.fn("TestThreadAtoms.makeHarness")(function* (options? return { runtime, + cache, + savedSnapshots, supervisor, registry, makeRegistry, @@ -683,6 +689,219 @@ describe("createEnvironmentThreadStateAtoms", () => { }), ); + it.effect("does not reload an archived disk snapshot when removal fails", () => + Effect.gen(function* () { + const h = yield* makeHarness({ + httpNone: true, + diskLoad: Effect.succeed(Option.some(SNAPSHOT)), + removeThread: Effect.fail( + new ConnectionPersistenceError({ + operation: "remove-thread", + message: "Test removal failure", + }), + ), + }); + const unmount = h.registry.mount(h.stateAtom); + const first = yield* Queue.take(h.subscriptions); + yield* Queue.offer(first.events, { kind: "synchronized" }); + yield* observeState(h.registry, h.stateAtom, (state) => state.status === "live"); + unmount(); + yield* Deferred.await(first.closed); + expect(yield* evictCachedThread(h.cache, TARGET.environmentId, THREAD_ID)).toBe(false); + const remount = h.registry.mount(h.stateAtom); + expect(h.registry.get(h.stateAtom).data).toEqual(Option.none()); + const next = yield* Queue.take(h.subscriptions); + expect(next.afterSequence).toBeUndefined(); + expect(h.counts().diskLoads).toBe(1); + expect(h.registry.get(h.stateAtom).data).toEqual(Option.none()); + remount(); + yield* Deferred.await(next.closed); + expect(h.savedSnapshots).toHaveLength(0); + }), + ); + + it.effect.each([false, true])("discards a disk load crossing eviction (revived: %s)", (revived) => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const disk = yield* Deferred.make>(); + const h = yield* makeHarness({ + httpNone: true, + diskLoad: Deferred.succeed(started, undefined).pipe(Effect.andThen(Deferred.await(disk))), + }); + const unmount = h.registry.mount(h.stateAtom); + yield* Deferred.await(started); + yield* evictCachedThread(h.cache, TARGET.environmentId, THREAD_ID); + if (revived) yield* reviveCachedThread(h.cache, TARGET.environmentId, THREAD_ID); + yield* Deferred.succeed(disk, Option.some(SNAPSHOT)); + const first = yield* Queue.take(h.subscriptions); + expect(first.afterSequence).toBeUndefined(); + expect(h.registry.get(h.stateAtom).data).toEqual(Option.none()); + unmount(); + yield* Deferred.await(first.closed); + expect(h.savedSnapshots).toHaveLength(0); + }), + ); + + it.effect("does not republish a live body after eviction and a later stream update", () => + Effect.gen(function* () { + let available = Option.some(SNAPSHOT); + const h = yield* makeHarness({ initialLoad: Effect.sync(() => available) }); + const unmount = h.registry.mount(h.stateAtom); + const first = yield* Queue.take(h.subscriptions); + yield* Queue.offer(first.events, { kind: "synchronized" }); + yield* observeState(h.registry, h.stateAtom, (state) => state.status === "live"); + yield* evictCachedThread(h.cache, TARGET.environmentId, THREAD_ID); + const writes = h.savedSnapshots.length; + available = Option.none(); + yield* Queue.offer(first.events, { + kind: "event", + event: { + type: "thread.message-sent", + sequence: 8, + eventId: EventId.make("after-eviction"), + aggregateKind: "thread", + aggregateId: THREAD_ID, + occurredAt: THREAD.createdAt, + commandId: null, + causationEventId: null, + correlationId: null, + metadata: {}, + payload: { + threadId: THREAD_ID, + messageId: MessageId.make("after-eviction"), + role: "assistant", + text: "Late body", + turnId: null, + streaming: false, + createdAt: THREAD.createdAt, + updatedAt: THREAD.createdAt, + }, + }, + }); + yield* observeState(h.registry, h.stateAtom, (state) => + Option.exists(state.data, (thread) => + thread.messages.some((message) => message.text === "Late body"), + ), + ); + unmount(); + yield* Deferred.await(first.closed); + const remount = h.registry.mount(h.stateAtom); + expect(h.registry.get(h.stateAtom).data).toEqual(Option.none()); + const next = yield* Queue.take(h.subscriptions); + expect(next.afterSequence).toBeUndefined(); + yield* TestClock.adjust("1 second"); + expect(h.savedSnapshots).toHaveLength(writes); + expect(h.registry.get(h.stateAtom).data).toEqual(Option.none()); + remount(); + yield* Deferred.await(next.closed); + }), + ); + + it.effect.each([false, true])( + "retains a deleted tombstone through failed eviction (shell first: %s)", + (shellFirst) => + Effect.gen(function* () { + let removals = 0; + const h = yield* makeHarness({ + diskLoad: Effect.succeed(Option.some(SNAPSHOT)), + removeThread: Effect.sync(() => { + removals += 1; + }).pipe( + Effect.andThen( + Effect.fail( + new ConnectionPersistenceError({ + operation: "remove-thread", + message: "Test removal failure", + }), + ), + ), + ), + }); + const unmount = h.registry.mount(h.stateAtom); + const first = yield* Queue.take(h.subscriptions); + yield* Queue.offer(first.events, { kind: "synchronized" }); + yield* observeState(h.registry, h.stateAtom, (state) => state.status === "live"); + if (shellFirst) yield* evictCachedThread(h.cache, TARGET.environmentId, THREAD_ID); + yield* Queue.offer(first.events, { + kind: "event", + event: { + type: "thread.deleted", + sequence: 8, + eventId: EventId.make("retained-deletion"), + aggregateKind: "thread", + aggregateId: THREAD_ID, + occurredAt: THREAD.createdAt, + commandId: null, + causationEventId: null, + correlationId: null, + metadata: {}, + payload: { threadId: THREAD_ID, deletedAt: THREAD.createdAt }, + }, + }); + yield* observeState(h.registry, h.stateAtom, (state) => state.status === "deleted"); + unmount(); + yield* Deferred.await(first.closed); + expect(removals).toBe(shellFirst ? 2 : 1); + yield* evictCachedThread(h.cache, TARGET.environmentId, THREAD_ID); + const before = h.counts(); + const remount = h.registry.mount(h.stateAtom); + expect(h.registry.get(h.stateAtom).status).toBe("deleted"); + expect(h.registry.get(h.stateAtom).data).toEqual(Option.none()); + yield* TestClock.adjust("0 millis"); + expect(h.counts().diskLoads).toBe(before.diskLoads); + expect(h.counts().httpLoads).toBe(before.httpLoads); + remount(); + }), + ); + + it.effect("invalidates a warm snapshot when archived after the detail subscription closes", () => + Effect.gen(function* () { + let available = Option.some(SNAPSHOT); + const h = yield* makeHarness({ initialLoad: Effect.sync(() => available) }); + const unmount = h.registry.mount(h.stateAtom); + const first = yield* Queue.take(h.subscriptions); + yield* Queue.offer(first.events, { kind: "synchronized" }); + yield* observeState(h.registry, h.stateAtom, (state) => state.status === "live"); + unmount(); + yield* Deferred.await(first.closed); + const writesBeforeArchive = h.savedSnapshots.length; + + available = Option.none(); + yield* evictCachedThread(h.cache, TARGET.environmentId, THREAD_ID); + // The warm atom owns the generation even with no live detail scope. + expect(cachedThreadGeneration(h.cache, TARGET.environmentId, THREAD_ID)).toBeGreaterThan(0); + const remount = h.registry.mount(h.stateAtom); + expect(h.registry.get(h.stateAtom).data).toEqual(Option.none()); + const archived = yield* Queue.take(h.subscriptions); + expect(archived.afterSequence).toBeUndefined(); + yield* Queue.offer(archived.events, { kind: "synchronized" }); + yield* TestClock.adjust("1 second"); + expect(h.registry.get(h.stateAtom).data).toEqual(Option.none()); + expect(h.savedSnapshots).toHaveLength(writesBeforeArchive); + + // Repeated shell eviction can arrive while the replacement subscription + // is live; an authoritative restore must still make its cache writable. + yield* evictCachedThread(h.cache, TARGET.environmentId, THREAD_ID); + + const restored = { snapshotSequence: 10, thread: { ...THREAD, title: "Restored thread" } }; + yield* Queue.offer(archived.events, { kind: "snapshot", snapshot: restored }); + yield* Queue.offer(archived.events, { kind: "synchronized" }); + yield* observeState(h.registry, h.stateAtom, (state) => state.status === "live"); + remount(); + yield* Deferred.await(archived.closed); + expect(h.savedSnapshots.at(-1)).toEqual(restored); + const returnAgain = h.registry.mount(h.stateAtom); + expect(currentThread(h.registry, h.stateAtom).title).toBe("Restored thread"); + const resumed = yield* Queue.take(h.subscriptions); + expect(resumed.afterSequence).toBe(10); + returnAgain(); + yield* Deferred.await(resumed.closed); + h.registry.dispose(); + yield* TestClock.adjust("0 millis"); + expect(cachedThreadGeneration(h.cache, TARGET.environmentId, THREAD_ID)).toBe(0); + }), + ); + it.effect("keeps warm data and resumes a completed cursor without loading another snapshot", () => Effect.gen(function* () { const h = yield* makeHarness(); @@ -853,8 +1072,11 @@ describe("createEnvironmentThreadStateAtoms", () => { const first = yield* Queue.take(h.subscriptions); unmount(); yield* Deferred.await(first.closed); + yield* evictCachedThread(h.cache, TARGET.environmentId, THREAD_ID); + expect(cachedThreadGeneration(h.cache, TARGET.environmentId, THREAD_ID)).toBeGreaterThan(0); yield* Effect.yieldNow; yield* Effect.promise(() => vi.advanceTimersByTimeAsync(THREAD_SNAPSHOT_IDLE_TTL_MS + 1)); + expect(cachedThreadGeneration(h.cache, TARGET.environmentId, THREAD_ID)).toBe(0); const remount = h.registry.mount(h.stateAtom); const next = yield* Queue.take(h.subscriptions); expect(h.counts()).toEqual({ httpLoads: 2, diskLoads: 2, opened: 2, active: 1 }); diff --git a/packages/client-runtime/src/state/threads-pagination.test.ts b/packages/client-runtime/src/state/threads-pagination.test.ts index 4dbc3329994e..daf123156de5 100644 --- a/packages/client-runtime/src/state/threads-pagination.test.ts +++ b/packages/client-runtime/src/state/threads-pagination.test.ts @@ -30,6 +30,7 @@ import { import * as EnvironmentSupervisor from "../connection/supervisor.ts"; import * as Persistence from "../platform/persistence.ts"; import * as RpcSession from "../rpc/session.ts"; +import { makeTestRpcSession } from "../rpc/testUtils/rpcSession.ts"; import type { ThreadSnapshotWindow } from "./threadSnapshotHttp.ts"; import { INITIAL_THREAD_USER_TURN_LIMIT, @@ -154,17 +155,12 @@ const makeHarness = Effect.fn("TestThreadPagination.makeHarness")(function* (opt [ORCHESTRATION_WS_METHODS.subscribeThread]: (input: Record) => Stream.unwrap(Ref.set(lastSubscribeInput, input).pipe(Effect.as(Stream.fromQueue(inputs)))), } as unknown as WsRpcProtocolClient; - const session: RpcSession.RpcSession = { - client, + const session = makeTestRpcSession(client, { initialConfig: Effect.succeed({ threadSnapshotPagination: options?.paginationCapability !== false, reasoningMessages: options?.reasoningCapability === true, } as never), - subscribeServerConfig: (input) => client.subscribeServerConfig(input), - ready: Effect.void, - probe: Effect.void, - closed: Effect.never, - }; + }); const supervisorSession = yield* SubscriptionRef.make>( Option.some(session), ); diff --git a/packages/client-runtime/src/state/threads-sync.test.ts b/packages/client-runtime/src/state/threads-sync.test.ts index 848357ec2bef..a11856716a14 100644 --- a/packages/client-runtime/src/state/threads-sync.test.ts +++ b/packages/client-runtime/src/state/threads-sync.test.ts @@ -40,6 +40,13 @@ import { ThreadSnapshotLoader, type EnvironmentThreadState, } from "./threads.ts"; +import { + cachedThreadGeneration, + evictCachedThread, + isCachedThreadEvicted, + persistCachedThread, + reviveCachedThread, +} from "./threadCache.ts"; const TARGET = new PrimaryConnectionTarget({ environmentId: EnvironmentId.make("environment-1"), @@ -137,6 +144,8 @@ function awaitThreadState( const makeHarness = Effect.fn("TestEnvironmentThreads.makeHarness")(function* (options?: { readonly cached?: OrchestrationThread; + readonly cache?: Persistence.EnvironmentCacheStore["Service"]; + readonly removeThread?: Persistence.EnvironmentCacheStore["Service"]["removeThread"]; readonly httpSnapshot?: Option.Option; readonly completionMarker?: boolean; readonly resumeCache?: NonNullable[1]>; @@ -215,33 +224,37 @@ const makeHarness = Effect.fn("TestEnvironmentThreads.makeHarness")(function* (o disconnect: Effect.void, retryNow: Ref.update(retryCount, (count) => count + 1), } satisfies EnvironmentSupervisor.EnvironmentSupervisor["Service"]); - const cache = Persistence.EnvironmentCacheStore.of({ - loadShell: () => Effect.succeedNone, - saveShell: () => Effect.void, - loadThread: (_environmentId, threadId) => - options?.loadCached ?? - Effect.succeed( - threadId === THREAD_ID && options?.cached !== undefined - ? Option.some({ - snapshotSequence: CACHED_SNAPSHOT_SEQUENCE, - thread: options.cached, - }) - : Option.none(), - ), - saveThread: (environmentId, thread) => - Ref.update(savedThreads, (current) => [...current, thread]).pipe( - Effect.andThen(options?.saveThread?.(environmentId, thread) ?? Effect.void), - ), - removeThread: (_environmentId, threadId) => - Ref.update(removedThreads, (current) => [...current, threadId]), - loadServerConfig: () => Effect.succeedNone, - saveServerConfig: () => Effect.void, - loadVcsRefs: () => Effect.succeedNone, - saveVcsRefs: () => Effect.void, - removeVcsRefs: () => Effect.void, - clearVcsRefs: () => Effect.void, - clear: () => Effect.void, - }); + const cache = + options?.cache ?? + Persistence.EnvironmentCacheStore.of({ + loadShell: () => Effect.succeed(Option.none()), + saveShell: () => Effect.void, + loadThread: (_environmentId, threadId) => + options?.loadCached ?? + Effect.succeed( + threadId === THREAD_ID && options?.cached !== undefined + ? Option.some({ + snapshotSequence: CACHED_SNAPSHOT_SEQUENCE, + thread: options.cached, + }) + : Option.none(), + ), + saveThread: (environmentId, thread) => + Ref.update(savedThreads, (current) => [...current, thread]).pipe( + Effect.andThen(options?.saveThread?.(environmentId, thread) ?? Effect.void), + ), + removeThread: (_environmentId, threadId) => + Ref.update(removedThreads, (current) => [...current, threadId]).pipe( + Effect.andThen(options?.removeThread?.(_environmentId, threadId) ?? Effect.void), + ), + loadServerConfig: () => Effect.succeed(Option.none()), + saveServerConfig: () => Effect.void, + loadVcsRefs: () => Effect.succeed(Option.none()), + saveVcsRefs: () => Effect.void, + removeVcsRefs: () => Effect.void, + clearVcsRefs: () => Effect.void, + clear: () => Effect.void, + }); const threadState = yield* makeEnvironmentThreadState(THREAD_ID, options?.resumeCache).pipe( Effect.provideService(EnvironmentSupervisor.EnvironmentSupervisor, supervisor), Effect.provideService(Persistence.EnvironmentCacheStore, cache), @@ -276,6 +289,7 @@ const makeHarness = Effect.fn("TestEnvironmentThreads.makeHarness")(function* (o supervisorSession, savedThreads, removedThreads, + cache, wakeups, replaceSession: SubscriptionRef.set( supervisorSession, @@ -372,6 +386,47 @@ const deleted = (): OrchestrationThreadStreamItem => ({ }, }); +const archived = (sequence = 3): OrchestrationThreadStreamItem => ({ + kind: "event", + event: { + eventId: EventId.make("event-archived"), + sequence, + occurredAt: "2026-04-01T02:00:00.000Z", + commandId: null, + causationEventId: null, + correlationId: null, + metadata: {}, + aggregateKind: "thread", + aggregateId: THREAD_ID, + type: "thread.archived", + payload: { + threadId: THREAD_ID, + archivedAt: "2026-04-01T02:00:00.000Z", + updatedAt: "2026-04-01T02:00:00.000Z", + }, + }, +}); + +const unarchived = (sequence = 4): OrchestrationThreadStreamItem => ({ + kind: "event", + event: { + eventId: EventId.make("event-unarchived"), + sequence, + occurredAt: "2026-04-01T03:00:00.000Z", + commandId: null, + causationEventId: null, + correlationId: null, + metadata: {}, + aggregateKind: "thread", + aggregateId: THREAD_ID, + type: "thread.unarchived", + payload: { + threadId: THREAD_ID, + updatedAt: "2026-04-01T03:00:00.000Z", + }, + }, +}); + describe("EnvironmentThreads", () => { for (const source of ["disk", "HTTP"] as const) { it.effect(`does not rewrite an unchanged ${source} snapshot on navigation or warm return`, () => @@ -613,6 +668,101 @@ describe("EnvironmentThreads", () => { }), ); + for (const delivery of ["event", "snapshot"] as const) { + it.effect(`preserves successor persistence after an obsolete archive ${delivery}`, () => + Effect.gen(function* () { + const resumeCache: NonNullable[1]> = { + snapshot: undefined, + owner: undefined, + }; + const persisted = yield* Deferred.make(); + const old = yield* makeHarness({ + cached: BASE_THREAD, + resumeCache, + saveThread: (_environmentId, saved) => + saved.thread.title === "Successor still persists" + ? Deferred.succeed(persisted, undefined).pipe(Effect.asVoid) + : Effect.void, + }); + yield* awaitThreadState(old.observed, (value) => value.status === "live"); + const successor = yield* makeHarness({ resumeCache, cache: old.cache }); + yield* Queue.offer(successor.inputs, titleUpdated("Successor title", 9)); + yield* awaitThreadState( + successor.observed, + (value) => Option.getOrNull(value.data)?.title === "Successor title", + ); + const update = archived(); + if (update.kind !== "event") return yield* Effect.die("Expected an event"); + yield* Queue.offer( + old.inputs, + delivery === "event" + ? { ...update, event: { ...update.event, sequence: 8 } } + : snapshot({ ...BASE_THREAD, archivedAt: "2026-04-01T02:00:00.000Z" }), + ); + yield* awaitThreadState( + old.observed, + (value) => Option.getOrNull(value.data)?.archivedAt != null, + ); + // A later item confirms the prior archive finished applying. + yield* Queue.offer(old.inputs, titleUpdated("Old scope processed archive", 10)); + yield* awaitThreadState( + old.observed, + (value) => Option.getOrNull(value.data)?.title === "Old scope processed archive", + ); + expect(resumeCache.snapshot?.sequence).toBe(9); + expect(Option.getOrThrow(resumeCache.snapshot!.state.data).title).toBe("Successor title"); + expect(yield* Ref.get(old.removedThreads)).toEqual([]); + expect(isCachedThreadEvicted(old.cache, TARGET.environmentId, THREAD_ID)).toBe(false); + + yield* Queue.offer(successor.inputs, titleUpdated("Successor still persists", 11)); + yield* awaitThreadState( + successor.observed, + (value) => Option.getOrNull(value.data)?.title === "Successor still persists", + ); + yield* TestClock.adjust("500 millis"); + yield* Deferred.await(persisted); + }), + ); + + it.effect(`preserves the successor tombstone after an obsolete unarchive ${delivery}`, () => + Effect.gen(function* () { + const resumeCache: NonNullable[1]> = { + snapshot: undefined, + owner: undefined, + }; + const old = yield* makeHarness({ cached: BASE_THREAD, resumeCache }); + yield* awaitThreadState(old.observed, (value) => value.status === "live"); + const successor = yield* makeHarness({ resumeCache, cache: old.cache }); + yield* awaitThreadState(successor.observed, (value) => value.status === "live"); + yield* evictCachedThread(old.cache, TARGET.environmentId, THREAD_ID); + const generation = cachedThreadGeneration(old.cache, TARGET.environmentId, THREAD_ID); + const update = unarchived(); + if (update.kind !== "event") return yield* Effect.die("Expected an event"); + yield* Queue.offer( + old.inputs, + delivery === "event" + ? { ...update, event: { ...update.event, sequence: 8 } } + : snapshot({ ...BASE_THREAD, title: "Obsolete active snapshot" }), + ); + yield* awaitThreadState(old.observed, (value) => { + const thread = Option.getOrNull(value.data); + return delivery === "event" + ? thread?.updatedAt === "2026-04-01T03:00:00.000Z" + : thread?.title === "Obsolete active snapshot"; + }); + yield* Queue.offer(old.inputs, titleUpdated("Old scope processed unarchive", 10)); + yield* awaitThreadState( + old.observed, + (value) => Option.getOrNull(value.data)?.title === "Old scope processed unarchive", + ); + expect(isCachedThreadEvicted(old.cache, TARGET.environmentId, THREAD_ID)).toBe(true); + expect(cachedThreadGeneration(old.cache, TARGET.environmentId, THREAD_ID)).toBe(generation); + expect(resumeCache.invalidated).toBe(true); + expect(resumeCache.snapshot).toBeUndefined(); + }), + ); + } + it.effect("retains a deletion instead of restoring the old disk snapshot", () => Effect.gen(function* () { const resumeCache: NonNullable[1]> = { @@ -778,6 +928,336 @@ describe("EnvironmentThreads", () => { }), ); + it.effect("removes cached data when the thread is archived and does not persist it again", () => + Effect.gen(function* () { + const savedThreads = yield* Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeHarness({ cached: BASE_THREAD }); + yield* Queue.offer(harness.inputs, snapshot(BASE_THREAD)); + yield* Queue.offer(harness.inputs, archived()); + + const state = yield* awaitThreadState( + harness.observed, + (value) => Option.isSome(value.data) && value.data.value.archivedAt !== null, + ); + yield* TestClock.adjust("500 millis"); + yield* Effect.yieldNow; + + expect(Option.getOrThrow(state.data).archivedAt).toBe("2026-04-01T02:00:00.000Z"); + expect(yield* Ref.get(harness.removedThreads)).toEqual([THREAD_ID]); + expect(yield* Ref.get(harness.savedThreads)).toEqual([]); + return harness.savedThreads; + }), + ); + + expect(yield* Ref.get(savedThreads)).toEqual([]); + }), + ); + + it.effect("retries a missing archived thread so it can observe unarchive", () => + Effect.gen(function* () { + const harness = yield* makeHarness({ cached: BASE_THREAD }); + yield* Queue.offer(harness.inputs, snapshot(BASE_THREAD)); + yield* Queue.offer(harness.inputs, archived()); + yield* awaitThreadState( + harness.observed, + (value) => Option.isSome(value.data) && value.data.value.archivedAt !== null, + ); + + yield* Queue.offer(harness.inputs, new Error("thread was moved to cold storage")); + yield* awaitThreadState(harness.observed, (value) => Option.isSome(value.error)); + yield* TestClock.adjust("250 millis"); + for (let attempt = 0; attempt < 10; attempt += 1) { + yield* Effect.yieldNow; + } + + expect(yield* Ref.get(harness.subscriptionCount)).toBe(2); + yield* Queue.offer(harness.inputs, unarchived()); + const state = yield* awaitThreadState( + harness.observed, + (value) => Option.isSome(value.data) && value.data.value.archivedAt === null, + ); + expect(Option.getOrThrow(state.data).archivedAt).toBeNull(); + }), + ); + + it.effect("removes cached data when an archived thread arrives in a snapshot", () => + Effect.gen(function* () { + const archivedThread = { + ...BASE_THREAD, + archivedAt: "2026-04-01T02:00:00.000Z", + }; + const harness = yield* makeHarness({ cached: BASE_THREAD }); + + yield* Queue.offer(harness.inputs, snapshot(archivedThread)); + const state = yield* awaitThreadState( + harness.observed, + (value) => Option.isSome(value.data) && value.data.value.archivedAt !== null, + ); + yield* TestClock.adjust("500 millis"); + yield* Effect.yieldNow; + + expect(Option.getOrThrow(state.data).archivedAt).toBe("2026-04-01T02:00:00.000Z"); + expect(yield* Ref.get(harness.removedThreads)).toEqual([THREAD_ID]); + expect(yield* Ref.get(harness.savedThreads)).toEqual([]); + }), + ); + + it.effect("does not restore an out-of-band cache eviction from queued or teardown writes", () => + Effect.gen(function* () { + const savedThreads = yield* Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeHarness({ cached: BASE_THREAD }); + yield* Queue.offer( + harness.inputs, + titleUpdated("Stale pending title", CACHED_SNAPSHOT_SEQUENCE + 1), + ); + yield* awaitThreadState( + harness.observed, + (value) => + Option.isSome(value.data) && value.data.value.title === "Stale pending title", + ); + + yield* evictCachedThread(harness.cache, TARGET.environmentId, THREAD_ID); + yield* TestClock.adjust("500 millis"); + yield* Effect.yieldNow; + + expect(yield* Ref.get(harness.removedThreads)).toEqual([THREAD_ID]); + expect(yield* Ref.get(harness.savedThreads)).toEqual([]); + return harness.savedThreads; + }), + ); + + expect(yield* Ref.get(savedThreads)).toEqual([]); + }), + ); + + for (const operation of ["evict", "revive"] as const) { + it.effect(`rechecks ownership when a queued cache ${operation} acquires its permit`, () => + Effect.gen(function* () { + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const block = Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Deferred.await(release)), + ); + const harness = yield* makeHarness({ + cached: BASE_THREAD, + ...(operation === "evict" ? { saveThread: () => block } : { removeThread: () => block }), + }); + yield* awaitThreadState(harness.observed, (value) => value.status === "live"); + const holding = yield* ( + operation === "evict" + ? persistCachedThread( + harness.cache, + TARGET.environmentId, + { snapshotSequence: 7, thread: BASE_THREAD }, + cachedThreadGeneration(harness.cache, TARGET.environmentId, THREAD_ID), + ) + : evictCachedThread(harness.cache, TARGET.environmentId, THREAD_ID) + ).pipe(Effect.forkChild); + yield* Deferred.await(entered); + let ownsCache = true; + const queued = yield* ( + operation === "evict" + ? evictCachedThread(harness.cache, TARGET.environmentId, THREAD_ID, () => ownsCache) + : reviveCachedThread(harness.cache, TARGET.environmentId, THREAD_ID, () => ownsCache) + ).pipe(Effect.forkChild({ startImmediately: true })); + const generation = cachedThreadGeneration(harness.cache, TARGET.environmentId, THREAD_ID); + ownsCache = false; + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(holding); + yield* Fiber.join(queued); + expect(cachedThreadGeneration(harness.cache, TARGET.environmentId, THREAD_ID)).toBe( + generation, + ); + expect(isCachedThreadEvicted(harness.cache, TARGET.environmentId, THREAD_ID)).toBe( + operation === "revive", + ); + expect(yield* Ref.get(harness.removedThreads)).toEqual( + operation === "revive" ? [THREAD_ID] : [], + ); + }), + ); + } + + it.effect("retries a failed archive eviction for the current cache owner", () => + Effect.gen(function* () { + let attempts = 0; + const removed = yield* Deferred.make(); + const harness = yield* makeHarness({ + cached: BASE_THREAD, + resumeCache: { snapshot: undefined, owner: undefined }, + removeThread: () => + Effect.suspend(() => { + attempts += 1; + return attempts === 1 + ? Effect.fail( + new Persistence.ConnectionPersistenceError({ + operation: "remove-thread", + message: "Temporary cache removal failure", + }), + ) + : Deferred.succeed(removed, undefined).pipe(Effect.asVoid); + }), + }); + yield* awaitThreadState(harness.observed, (value) => value.status === "live"); + yield* Queue.offer( + harness.inputs, + snapshot({ ...BASE_THREAD, archivedAt: "2026-04-01T02:00:00.000Z" }), + ); + yield* Queue.offer(harness.inputs, titleUpdated("First eviction processed", 9)); + yield* awaitThreadState( + harness.observed, + (value) => Option.getOrNull(value.data)?.title === "First eviction processed", + ); + expect(attempts).toBe(1); + yield* Queue.offer( + harness.inputs, + snapshot({ ...BASE_THREAD, archivedAt: "2026-04-01T02:00:00.000Z" }), + ); + yield* Deferred.await(removed); + expect(attempts).toBe(2); + }), + ); + + it.effect("keeps an active cache write valid when revival is already satisfied", () => + Effect.gen(function* () { + const harness = yield* makeHarness({ cached: BASE_THREAD }); + const generation = cachedThreadGeneration(harness.cache, TARGET.environmentId, THREAD_ID); + + yield* reviveCachedThread(harness.cache, TARGET.environmentId, THREAD_ID); + yield* persistCachedThread( + harness.cache, + TARGET.environmentId, + { snapshotSequence: CACHED_SNAPSHOT_SEQUENCE, thread: BASE_THREAD }, + generation, + ); + + expect(yield* Ref.get(harness.savedThreads)).toEqual([ + { snapshotSequence: CACHED_SNAPSHOT_SEQUENCE, thread: BASE_THREAD }, + ]); + }), + ); + + it.effect("rejects a pre-eviction write after cache revival", () => + Effect.gen(function* () { + const harness = yield* makeHarness({ cached: BASE_THREAD }); + const staleGeneration = cachedThreadGeneration( + harness.cache, + TARGET.environmentId, + THREAD_ID, + ); + + yield* evictCachedThread(harness.cache, TARGET.environmentId, THREAD_ID); + yield* reviveCachedThread(harness.cache, TARGET.environmentId, THREAD_ID); + yield* persistCachedThread( + harness.cache, + TARGET.environmentId, + { snapshotSequence: CACHED_SNAPSHOT_SEQUENCE, thread: BASE_THREAD }, + staleGeneration, + ); + expect(yield* Ref.get(harness.savedThreads)).toEqual([]); + + const revivedGeneration = cachedThreadGeneration( + harness.cache, + TARGET.environmentId, + THREAD_ID, + ); + yield* persistCachedThread( + harness.cache, + TARGET.environmentId, + { snapshotSequence: CACHED_SNAPSHOT_SEQUENCE, thread: BASE_THREAD }, + revivedGeneration, + ); + expect(yield* Ref.get(harness.savedThreads)).toEqual([ + { snapshotSequence: CACHED_SNAPSHOT_SEQUENCE, thread: BASE_THREAD }, + ]); + }), + ); + + it.effect("rejects a write captured while the cache is evicted after revival", () => + Effect.gen(function* () { + const harness = yield* makeHarness({ cached: BASE_THREAD }); + + yield* evictCachedThread(harness.cache, TARGET.environmentId, THREAD_ID); + const evictedGeneration = cachedThreadGeneration( + harness.cache, + TARGET.environmentId, + THREAD_ID, + ); + + yield* reviveCachedThread(harness.cache, TARGET.environmentId, THREAD_ID); + yield* persistCachedThread( + harness.cache, + TARGET.environmentId, + { snapshotSequence: CACHED_SNAPSHOT_SEQUENCE, thread: BASE_THREAD }, + evictedGeneration, + ); + + expect(yield* Ref.get(harness.savedThreads)).toEqual([]); + }), + ); + + it.effect("persists thread detail again after an authoritative unarchive event", () => + Effect.gen(function* () { + const harness = yield* makeHarness({ cached: BASE_THREAD }); + yield* Queue.offer(harness.inputs, snapshot(BASE_THREAD)); + yield* Queue.offer(harness.inputs, archived()); + yield* awaitThreadState( + harness.observed, + (value) => Option.isSome(value.data) && value.data.value.archivedAt !== null, + ); + + yield* Queue.offer(harness.inputs, unarchived()); + yield* awaitThreadState( + harness.observed, + (value) => Option.isSome(value.data) && value.data.value.archivedAt === null, + ); + yield* TestClock.adjust("500 millis"); + yield* Effect.yieldNow; + + expect((yield* Ref.get(harness.savedThreads)).at(-1)?.thread.archivedAt).toBeNull(); + }), + ); + + it.effect("persists thread detail again after an authoritative active snapshot", () => + Effect.gen(function* () { + const harness = yield* makeHarness({ cached: BASE_THREAD }); + yield* Queue.offer( + harness.inputs, + snapshot({ + ...BASE_THREAD, + archivedAt: "2026-04-01T02:00:00.000Z", + }), + ); + yield* awaitThreadState( + harness.observed, + (value) => Option.isSome(value.data) && value.data.value.archivedAt !== null, + ); + + yield* Queue.offer( + harness.inputs, + snapshot({ + ...BASE_THREAD, + title: "Restored from snapshot", + updatedAt: "2026-04-01T03:00:00.000Z", + }), + ); + yield* awaitThreadState( + harness.observed, + (value) => Option.isSome(value.data) && value.data.value.title === "Restored from snapshot", + ); + yield* TestClock.adjust("500 millis"); + yield* Effect.yieldNow; + + expect(yield* Ref.get(harness.removedThreads)).toEqual([THREAD_ID]); + expect((yield* Ref.get(harness.savedThreads)).at(-1)?.thread).toMatchObject({ + title: "Restored from snapshot", + archivedAt: null, + }); + }), + ); + it.effect("does not resurrect a deleted thread when the app returns to the foreground", () => Effect.gen(function* () { const harness = yield* makeHarness({ @@ -1035,6 +1515,63 @@ describe("EnvironmentThreads", () => { }), ); + it.effect("evicts cached data without repersisting a body archived in a batch", () => + Effect.gen(function* () { + const resumeCache: NonNullable[1]> = { + snapshot: undefined, + owner: undefined, + }; + const harness = yield* makeHarness({ cached: BASE_THREAD, resumeCache }); + yield* awaitThreadState(harness.observed, (value) => value.status === "live"); + yield* Queue.offerAll(harness.inputs, [ + titleUpdated("Settled before archive", CACHED_SNAPSHOT_SEQUENCE + 1), + archived(CACHED_SNAPSHOT_SEQUENCE + 2), + ]); + yield* awaitThreadState( + harness.observed, + (value) => Option.getOrNull(value.data)?.archivedAt != null, + ); + yield* TestClock.adjust("500 millis"); + yield* Effect.yieldNow; + expect(yield* Ref.get(harness.removedThreads)).toEqual([THREAD_ID]); + expect(yield* Ref.get(harness.savedThreads)).toEqual([]); + expect(isCachedThreadEvicted(harness.cache, TARGET.environmentId, THREAD_ID)).toBe(true); + expect(resumeCache.snapshot).toBeUndefined(); + }), + ); + + it.effect("revives and persists cached data after an unarchive in a batch", () => + Effect.gen(function* () { + const resumeCache: NonNullable[1]> = { + snapshot: undefined, + owner: undefined, + }; + const harness = yield* makeHarness({ cached: BASE_THREAD, resumeCache }); + yield* awaitThreadState(harness.observed, (value) => value.status === "live"); + yield* Queue.offer(harness.inputs, archived(CACHED_SNAPSHOT_SEQUENCE + 1)); + yield* awaitThreadState( + harness.observed, + (value) => Option.getOrNull(value.data)?.archivedAt != null, + ); + yield* Queue.offerAll(harness.inputs, [ + unarchived(CACHED_SNAPSHOT_SEQUENCE + 2), + titleUpdated("Restored in batch", CACHED_SNAPSHOT_SEQUENCE + 3), + ]); + yield* awaitThreadState( + harness.observed, + (value) => Option.getOrNull(value.data)?.title === "Restored in batch", + ); + yield* TestClock.adjust("500 millis"); + yield* Effect.yieldNow; + expect(isCachedThreadEvicted(harness.cache, TARGET.environmentId, THREAD_ID)).toBe(false); + expect((yield* Ref.get(harness.savedThreads)).at(-1)).toMatchObject({ + snapshotSequence: CACHED_SNAPSHOT_SEQUENCE + 3, + thread: { title: "Restored in batch", archivedAt: null }, + }); + expect(resumeCache.invalidated).toBe(false); + }), + ); + it.effect( "persists a turn that settles mid-batch when the next turn starts in the same batch", () => diff --git a/packages/client-runtime/src/state/threads.ts b/packages/client-runtime/src/state/threads.ts index e0be27d35358..73bb26fdf250 100644 --- a/packages/client-runtime/src/state/threads.ts +++ b/packages/client-runtime/src/state/threads.ts @@ -25,6 +25,15 @@ import * as ConnectionWakeups from "../connection/wakeups.ts"; import { EnvironmentCacheStore } from "../platform/persistence.ts"; import { subscribeDynamic } from "../rpc/client.ts"; import { ThreadSnapshotLoader, type ThreadSnapshotWindow } from "./threadSnapshotHttp.ts"; +import { + cachedThreadGeneration, + evictCachedThread, + isCachedThreadEvicted, + persistCachedThread, + retainCachedThread, + retainCachedThreadUnsafe, + reviveCachedThread, +} from "./threadCache.ts"; import { parseThreadKey, threadKey } from "./entities.ts"; import { applyThreadDetailEvent } from "./threadReducer.ts"; import { THREAD_SNAPSHOT_IDLE_TTL_MS } from "./threadRetention.ts"; @@ -135,7 +144,7 @@ export function isThreadSessionRunning(session: OrchestrationThread["session"]): } function shouldPersistThread(thread: OrchestrationThread): boolean { - return !isThreadSessionRunning(thread.session); + return thread.archivedAt === null && !isThreadSessionRunning(thread.session); } interface ThreadResumeSnapshot { @@ -147,6 +156,11 @@ interface ThreadResumeSnapshot { interface ThreadResumeCache { snapshot: ThreadResumeSnapshot | undefined; owner: object | undefined; + invalidated?: boolean; + eviction?: { + readonly cache: EnvironmentCacheStore["Service"]; + readonly release: () => void; + }; } function matchesThreadSnapshot( @@ -191,11 +205,30 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make const snapshotLoader = yield* ThreadSnapshotLoader; const wakeups = yield* Effect.serviceOption(ConnectionWakeups.ConnectionWakeups); const environmentId = supervisor.target.environmentId; + yield* retainCachedThread(cache, environmentId, threadId); + if (resumeCache && resumeCache.eviction?.cache !== cache) { + resumeCache.eviction?.release(); + resumeCache.eviction = { + cache, + release: retainCachedThreadUnsafe(cache, environmentId, threadId, () => { + // Keep a body-free deletion tombstone so a failed disk removal cannot + // make a deleted thread reappear on the next visit. + if (resumeCache.snapshot?.state.status !== "deleted") resumeCache.snapshot = undefined; + // An evicted subscription must not republish its committed snapshot + // during an in-flight update or its persistence finalizer. + resumeCache.invalidated = true; + }), + }; + } const retained = resumeCache?.snapshot; const owner = {}; - if (resumeCache) resumeCache.owner = owner; - const cached = - retained === undefined + if (resumeCache) { + resumeCache.owner = owner; + resumeCache.invalidated = isCachedThreadEvicted(cache, environmentId, threadId); + } + const loadGeneration = cachedThreadGeneration(cache, environmentId, threadId); + const loaded = + retained === undefined && !isCachedThreadEvicted(cache, environmentId, threadId) ? yield* cache.loadThread(environmentId, threadId).pipe( Effect.catch((error) => Effect.logWarning("Could not load cached thread.").pipe( @@ -209,6 +242,13 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make ), ) : Option.none(); + // Removal can fail, and a read can finish after archive (or archive/restore). + // The retained generation protects admission as well as later cache writes. + const cached = + !isCachedThreadEvicted(cache, environmentId, threadId) && + cachedThreadGeneration(cache, environmentId, threadId) === loadGeneration + ? loaded + : Option.none(); const cachedThread = Option.map(cached, (snapshot) => snapshot.thread); const initialState: EnvironmentThreadState = retained ? cachedThreadState(retained.state) @@ -232,7 +272,7 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make sequence: initialSequence, persisted: retained?.persisted ?? Option.isSome(cached), }; - if (resumeCache?.owner === owner) resumeCache.snapshot = committed; + if (resumeCache?.owner === owner && !resumeCache.invalidated) resumeCache.snapshot = committed; const awaitingCompletion = yield* Ref.make(false); // Bumped whenever loaded history may have been rewritten out from under an // in-flight older-page fetch (snapshot replacement, revert, deletion). A @@ -259,7 +299,7 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make Option.getOrUndefined(current.page), ), }; - if (resumeCache?.owner === owner) resumeCache.snapshot = committed; + if (resumeCache?.owner === owner && !resumeCache.invalidated) resumeCache.snapshot = committed; }); // Whether the connected server accepts windowed reads; set per subscription // from the session config. Gates loadOlderTurns so a reconnect to a @@ -273,43 +313,32 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make readonly snapshot: OrchestrationThreadDetailSnapshot; readonly epoch: number; } | null>(null); - const persistence = yield* Queue.sliding(1); + const persistence = yield* Queue.sliding<{ + readonly generation: number; + readonly snapshot: OrchestrationThreadDetailSnapshot; + }>(1); - const persist = Effect.fn("EnvironmentThreadState.persist")(function* ( - snapshot: OrchestrationThreadDetailSnapshot, - ) { - if (resumeCache !== undefined && resumeCache.owner !== owner) return; + const persist = Effect.fn("EnvironmentThreadState.persist")(function* (pending: { + readonly generation: number; + readonly snapshot: OrchestrationThreadDetailSnapshot; + }) { + if (resumeCache !== undefined && (resumeCache.owner !== owner || resumeCache.invalidated)) + return; + const snapshot = pending.snapshot; if ( committed.persisted && matchesThreadSnapshot(committed, snapshot.thread, snapshot.snapshotSequence, snapshot.page) ) return; - yield* cache.saveThread(environmentId, snapshot).pipe( - Effect.tap(() => - Effect.sync(() => { - if ( - !matchesThreadSnapshot( - committed, - snapshot.thread, - snapshot.snapshotSequence, - snapshot.page, - ) - ) - return; - committed = { ...committed, persisted: true }; - if (resumeCache?.owner === owner) resumeCache.snapshot = committed; - }), - ), - Effect.catch((error) => - Effect.logWarning("Could not persist the thread cache.").pipe( - Effect.annotateLogs({ - environmentId, - threadId, - error: error.message, - }), - ), - ), - ); + const saved = yield* persistCachedThread(cache, environmentId, snapshot, pending.generation); + if ( + saved && + matchesThreadSnapshot(committed, snapshot.thread, snapshot.snapshotSequence, snapshot.page) + ) { + committed = { ...committed, persisted: true }; + if (resumeCache?.owner === owner && !resumeCache.invalidated) + resumeCache.snapshot = committed; + } }); yield* Stream.fromQueue(persistence).pipe( @@ -318,6 +347,27 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make Effect.forkScoped, ); + const removeCachedThread = Effect.fn("EnvironmentThreadState.removeCachedThread")(function* () { + yield* evictCachedThread( + cache, + environmentId, + threadId, + () => resumeCache === undefined || resumeCache.owner === owner, + ); + }); + + const reviveOwnedCachedThread = Effect.fn("EnvironmentThreadState.reviveOwnedCachedThread")( + function* () { + yield* reviveCachedThread( + cache, + environmentId, + threadId, + () => resumeCache === undefined || resumeCache.owner === owner, + ); + if (resumeCache?.owner === owner) resumeCache.invalidated = false; + }, + ); + const setConnecting = SubscriptionRef.update(state, (current) => current.status === "deleted" || Option.isSome(current.error) ? current @@ -366,21 +416,24 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make function* (thread: OrchestrationThread, snapshotSequence: number) { const currentPage = yield* SubscriptionRef.get(state).pipe(Effect.map((value) => value.page)); yield* Queue.offer(persistence, { - snapshotSequence, - thread, - // Persist the window boundary with the window's content so a cache - // restore can keep paging from where the loaded history ends. - ...Option.match(currentPage, { - onNone: () => ({}), - onSome: (value) => - ({ - page: { - beforeCursor: value.beforeCursor, - hasMore: value.hasMore, - snapshotSequence, - }, - }) as const, - }), + generation: cachedThreadGeneration(cache, environmentId, threadId), + snapshot: { + snapshotSequence, + thread, + // Persist the window boundary with the window's content so a cache + // restore can keep paging from where the loaded history ends. + ...Option.match(currentPage, { + onNone: () => ({}), + onSome: (value) => + ({ + page: { + beforeCursor: value.beforeCursor, + hasMore: value.hasMore, + snapshotSequence, + }, + }) as const, + }), + }, }); }, ); @@ -422,18 +475,10 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make page: Option.none(), }); yield* remember; - if (resumeCache !== undefined && resumeCache.owner !== owner) return; - yield* cache.removeThread(environmentId, threadId).pipe( - Effect.catch((error) => - Effect.logWarning("Could not remove the cached thread.").pipe( - Effect.annotateLogs({ - environmentId, - threadId, - error: error.message, - }), - ), - ), - ); + // A shell eviction may precede this detail event. Its invalidation blocks + // bodies, but the current owner's terminal, body-free state is safe to keep. + yield* removeCachedThread(); + if (resumeCache?.owner === owner) resumeCache.snapshot = committed; }); // Body of applyItem, running under applyLock. @@ -460,6 +505,11 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make // from the replacement snapshot's cursor. yield* Ref.set(pendingOlderPage, null); yield* SubscriptionRef.set(lastSequence, item.snapshot.snapshotSequence); + if (item.snapshot.thread.archivedAt !== null) { + yield* removeCachedThread(); + } else { + yield* reviveOwnedCachedThread(); + } yield* setThread(item.snapshot.thread, pageStateFromSnapshot(item.snapshot.page)); return; } @@ -488,7 +538,13 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make } const result = applyThreadDetailEvent(current.data.value, item.event); if (result.kind === "updated") { + if (item.event.type === "thread.unarchived") { + yield* reviveOwnedCachedThread(); + } yield* setThread(result.thread, "keep"); + if (item.event.type === "thread.archived") { + yield* removeCachedThread(); + } } else if (result.kind === "deleted") { yield* setDeleted(); } @@ -540,11 +596,15 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make if ( Option.isNone(current.data) || (yield* Ref.get(pendingOlderPage)) !== null || + // Lifecycle events need the per-item cache eviction and revival effects. items.some( (item) => item.kind === "snapshot" || (item.kind === "event" && - (item.event.type === "thread.reverted" || item.event.type === "thread.deleted")), + (item.event.type === "thread.reverted" || + item.event.type === "thread.deleted" || + item.event.type === "thread.archived" || + item.event.type === "thread.unarchived")), ) ) { for (const item of items) { @@ -632,9 +692,12 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make if (merged !== null && shouldPersistThread(merged)) { const snapshotSequence = yield* SubscriptionRef.get(lastSequence); yield* Queue.offer(persistence, { - snapshotSequence, - thread: merged, - ...(snapshot.page === undefined ? {} : { page: { ...snapshot.page, snapshotSequence } }), + generation: cachedThreadGeneration(cache, environmentId, threadId), + snapshot: { + snapshotSequence, + thread: merged, + ...(snapshot.page === undefined ? {} : { page: { ...snapshot.page, snapshotSequence } }), + }, }); } yield* remember; @@ -850,6 +913,10 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make { onDefect: () => setStreamError("Could not synchronize the thread."), onExpectedFailure: (cause) => setStreamError(formatThreadError(cause)), + // A retained route can stay mounted while another surface unarchives + // the thread. Keep retrying after cold storage makes the detail + // temporarily unavailable so this subscription can observe the later + // unarchive without requiring a wakeup or a replacement session. retryExpectedFailureAfter: "250 millis", resubscribe: foregroundResubscriptions, }, @@ -886,19 +953,22 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make onSome: (thread) => shouldPersistThread(thread) ? persist({ - snapshotSequence, - thread, - ...Option.match(current.page, { - onNone: () => ({}), - onSome: (page) => - ({ - page: { - beforeCursor: page.beforeCursor, - hasMore: page.hasMore, - snapshotSequence, - }, - }) as const, - }), + generation: cachedThreadGeneration(cache, environmentId, threadId), + snapshot: { + snapshotSequence, + thread, + ...Option.match(current.page, { + onNone: () => ({}), + onSome: (page) => + ({ + page: { + beforeCursor: page.beforeCursor, + hasMore: page.hasMore, + snapshotSequence, + }, + }) as const, + }), + }, }) : Effect.void, }); @@ -930,10 +1000,11 @@ export function createEnvironmentThreadStateAtoms( // Cache definitions must outlive collectible live-atom definitions. The // registry retains these nodes without retaining environment or RPC scopes. const resumeFamily = Atom.family((key: string) => - Atom.make((): ThreadResumeCache => ({ - snapshot: undefined, - owner: undefined, - })).pipe( + Atom.make((get): ThreadResumeCache => { + const resume: ThreadResumeCache = { snapshot: undefined, owner: undefined }; + get.addFinalizer(() => resume.eviction?.release()); + return resume; + }).pipe( Atom.setIdleTTL(THREAD_SNAPSHOT_IDLE_TTL_MS), Atom.withLabel(`environment-thread-resume:${key}`), ), diff --git a/packages/contracts/src/orchestration.test.ts b/packages/contracts/src/orchestration.test.ts index 7c7bade4502e..6f583e5c5035 100644 --- a/packages/contracts/src/orchestration.test.ts +++ b/packages/contracts/src/orchestration.test.ts @@ -200,6 +200,21 @@ it.effect("decodes project.create with createWorkspaceRootIfMissing enabled", () }), ); +it.effect("decodes project.delete without an archived-thread opt-in", () => + Effect.gen(function* () { + const parsed = yield* decodeOrchestrationCommand({ + type: "project.delete", + commandId: "cmd-project-delete", + projectId: "project-delete", + }); + + if (parsed.type !== "project.delete") { + assert.fail(`Expected project.delete, received ${parsed.type}`); + } + assert.strictEqual(parsed.deleteArchivedThreads, undefined); + }), +); + it.effect("decodes historical project.created payloads with a default provider", () => Effect.gen(function* () { const parsed = yield* decodeProjectCreatedPayload({ diff --git a/packages/contracts/src/orchestration.ts b/packages/contracts/src/orchestration.ts index 1ae704d3f666..b3a7f7a45db0 100644 --- a/packages/contracts/src/orchestration.ts +++ b/packages/contracts/src/orchestration.ts @@ -1114,6 +1114,7 @@ const ProjectDeleteCommand = Schema.Struct({ commandId: CommandId, projectId: ProjectId, force: Schema.optional(Schema.Boolean), + deleteArchivedThreads: Schema.optional(Schema.Boolean), }); const ThreadCreateCommand = Schema.Struct({ diff --git a/packages/shared/src/dateTime.ts b/packages/shared/src/dateTime.ts index 544dd2d0d73b..888d05ebeecf 100644 --- a/packages/shared/src/dateTime.ts +++ b/packages/shared/src/dateTime.ts @@ -11,7 +11,8 @@ const isZonedIsoDateTime = Schema.is( ), ); -function parseTimestamp(value: string): number { +/** Parse a zoned ISO timestamp, rejecting malformed or impossible calendar dates. */ +export function parseTimestamp(value: string): number { if (!isZonedIsoDateTime(value)) return Number.NaN; // Engines can normalize invalid calendar dates instead of rejecting them.