diff --git a/apps/mobile/src/features/settings/settings-scoped-server.test.ts b/apps/mobile/src/features/settings/settings-scoped-server.test.ts index 4c8d8d67b55c..e2a93344b9c8 100644 --- a/apps/mobile/src/features/settings/settings-scoped-server.test.ts +++ b/apps/mobile/src/features/settings/settings-scoped-server.test.ts @@ -80,7 +80,7 @@ describe("mobile project settings scope", () => { ...DEFAULT_SERVER_SETTINGS, agentToolCapabilities: ["quality-records"], projectSettingsOverrides: { - [firstProject]: { agentToolCapabilities: [], enableMemoryAutoRecall: true }, + [firstProject]: { agentToolCapabilities: ["automation"], enableMemoryAutoRecall: true }, [secondProject]: { defaultAutoPull: true }, }, }; @@ -97,7 +97,7 @@ describe("mobile project settings scope", () => { patch: { projectSettingsOverrides: { [firstProject]: { - agentToolCapabilities: ["memory"], + agentToolCapabilities: ["memory", "automation"], enableMemoryAutoRecall: true, }, [secondProject]: { @@ -122,7 +122,7 @@ describe("mobile project settings scope", () => { }), environment(secondId, { ...DEFAULT_SERVER_SETTINGS, - agentToolCapabilities: [], + agentToolCapabilities: ["automation"], }), ], null, @@ -131,7 +131,7 @@ describe("mobile project settings scope", () => { planMobileAgentToolCapability(targets, false, "memory", false).map( (write) => write.patch.agentToolCapabilities, ), - ).toEqual([["quality-records"], []]); + ).toEqual([["quality-records"], ["automation"]]); }); it("edits each checkout's own override without changing either environment default", () => { const firstSettings: ServerSettings = { diff --git a/apps/server/integration/orphanedProviderSessionStartup.integration.test.ts b/apps/server/integration/orphanedProviderSessionStartup.integration.test.ts index b483a3f99f8e..c001660b09fa 100644 --- a/apps/server/integration/orphanedProviderSessionStartup.integration.test.ts +++ b/apps/server/integration/orphanedProviderSessionStartup.integration.test.ts @@ -41,6 +41,9 @@ import * as ProviderSessionDirectory from "../src/provider/Services/ProviderSess import * as ProviderSessionReaper from "../src/provider/Services/ProviderSessionReaper.ts"; import * as RepositoryIdentityResolver from "../src/project/RepositoryIdentityResolver.ts"; import * as ServerLifecycleEvents from "../src/serverLifecycleEvents.ts"; +import { parityStartupDependenciesLayer } from "../src/testUtils/parityDependencies.ts"; +import * as BackgroundPolicy from "../src/background/BackgroundPolicy.ts"; +import * as HostPowerMonitor from "../src/background/HostPowerMonitor.ts"; import * as ServerRuntimeStartup from "../src/serverRuntimeStartup.ts"; import * as ServerSettings from "../src/serverSettings.ts"; import * as AnalyticsService from "../src/telemetry/AnalyticsService.ts"; @@ -259,7 +262,13 @@ it.effect( const secondRuntime = makePersistedRuntimeLayer(config.dbPath); const startupLayer = ServerRuntimeStartup.layer.pipe( + Layer.provideMerge(parityStartupDependenciesLayer), Layer.provideMerge(secondRuntime), + Layer.provideMerge( + BackgroundPolicy.layer.pipe( + Layer.provide(Layer.effect(HostPowerMonitor.HostPowerMonitor, HostPowerMonitor.make())), + ), + ), Layer.provideMerge(startupDependencies), ); diff --git a/apps/server/src/auth/RpcAuthorization.test.ts b/apps/server/src/auth/RpcAuthorization.test.ts index 6b0ace72f7ca..3fd2fc38e9aa 100644 --- a/apps/server/src/auth/RpcAuthorization.test.ts +++ b/apps/server/src/auth/RpcAuthorization.test.ts @@ -25,6 +25,12 @@ describe("RPC authorization scopes", () => { WS_METHODS.memoryRelated, WS_METHODS.qualityRead, WS_METHODS.qualitySubscribeChanges, + WS_METHODS.scheduledList, + WS_METHODS.scheduledGet, + WS_METHODS.ambientGet, + WS_METHODS.backgroundJobList, + WS_METHODS.backgroundJobOutput, + WS_METHODS.backgroundJobWait, ]) expect(requiredScopeForRpcMethod(method)).toBe(AuthOrchestrationReadScope); for (const method of [ @@ -33,6 +39,11 @@ describe("RPC authorization scopes", () => { WS_METHODS.memoryRemember, WS_METHODS.memoryForget, WS_METHODS.qualityUpdate, + WS_METHODS.scheduledCreate, + WS_METHODS.scheduledCancel, + WS_METHODS.backgroundJobStart, + WS_METHODS.backgroundJobCancel, + WS_METHODS.backgroundJobSubscribe, ]) expect(requiredScopeForRpcMethod(method)).toBe(AuthOrchestrationOperateScope); }); @@ -41,6 +52,7 @@ describe("RPC authorization scopes", () => { for (const method of [ WS_METHODS.unattendedGrantCreate, WS_METHODS.unattendedGrantRevoke, + WS_METHODS.ambientConfigure, WS_METHODS.memoryGlobalRead, WS_METHODS.memoryGlobalWrite, ]) diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts index a777adc683a5..479d842a1c57 100644 --- a/apps/server/src/auth/RpcAuthorization.ts +++ b/apps/server/src/auth/RpcAuthorization.ts @@ -28,9 +28,24 @@ export const RPC_REQUIRED_SCOPES = { [WS_METHODS.coordinationMailboxRead]: AuthOrchestrationReadScope, [WS_METHODS.coordinationMailboxWrite]: AuthOrchestrationOperateScope, [WS_METHODS.qualitySubscribeChanges]: AuthOrchestrationReadScope, + [WS_METHODS.ambientConfigure]: AuthAccessWriteScope, + [WS_METHODS.ambientGet]: AuthOrchestrationReadScope, + [WS_METHODS.ambientStop]: AuthOrchestrationOperateScope, + [WS_METHODS.scheduledCreate]: AuthOrchestrationOperateScope, + [WS_METHODS.scheduledList]: AuthOrchestrationReadScope, + [WS_METHODS.scheduledGet]: AuthOrchestrationReadScope, + [WS_METHODS.scheduledCancel]: AuthOrchestrationOperateScope, [WS_METHODS.unattendedGrantCreate]: AuthAccessWriteScope, [WS_METHODS.unattendedGrantList]: AuthOrchestrationReadScope, [WS_METHODS.unattendedGrantRevoke]: AuthAccessWriteScope, + [WS_METHODS.backgroundJobStart]: AuthOrchestrationOperateScope, + [WS_METHODS.backgroundJobList]: AuthOrchestrationReadScope, + [WS_METHODS.backgroundJobGet]: AuthOrchestrationReadScope, + [WS_METHODS.backgroundJobOutput]: AuthOrchestrationReadScope, + [WS_METHODS.backgroundJobWait]: AuthOrchestrationReadScope, + [WS_METHODS.backgroundJobCancel]: AuthOrchestrationOperateScope, + [WS_METHODS.backgroundJobSubscribe]: AuthOrchestrationOperateScope, + [WS_METHODS.backgroundJobCleanup]: AuthOrchestrationOperateScope, [WS_METHODS.memoryRemember]: AuthOrchestrationOperateScope, [WS_METHODS.memoryRecall]: AuthOrchestrationReadScope, [WS_METHODS.memorySearch]: AuthOrchestrationReadScope, diff --git a/apps/server/src/background/BackgroundJobAuthority.ts b/apps/server/src/background/BackgroundJobAuthority.ts new file mode 100644 index 000000000000..5a0aff790848 --- /dev/null +++ b/apps/server/src/background/BackgroundJobAuthority.ts @@ -0,0 +1,138 @@ +import { CommandId, EventId, type ThreadId } from "@t3tools/contracts"; +import { BackgroundJobError } from "../../../../packages/contracts/src/backgroundJobs.ts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as Option from "effect/Option"; +import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; +import * as ProjectionSnapshotQuery from "../orchestration/Services/ProjectionSnapshotQuery.ts"; +import * as OrchestrationEngine from "../orchestration/Services/OrchestrationEngine.ts"; +import * as UnattendedGrants from "../orchestration/UnattendedGrants.ts"; +import * as ScheduledWork from "../orchestration/ScheduledWork.ts"; +import * as McpInvocationContext from "../mcp/McpInvocationContext.ts"; +import { BackgroundJobAuthority } from "./BackgroundJobs.ts"; + +const make = Effect.gen(function* () { + const snapshots = yield* ProjectionSnapshotQuery.ProjectionSnapshotQuery; + const paths = yield* WorkspacePaths.WorkspacePaths; + const engine = yield* OrchestrationEngine.OrchestrationEngineService; + const grants = yield* UnattendedGrants.UnattendedGrants; + const scheduled = yield* ScheduledWork.ScheduledWork; + const capabilities = yield* McpInvocationContext.makeThreadMcpCapabilities; + const wrap = (effect: Effect.Effect) => + effect.pipe( + Effect.mapError((cause) => + Schema.is(BackgroundJobError)(cause) + ? cause + : new BackgroundJobError({ + code: "internal", + detail: "Host-job authority operation failed.", + cause, + }), + ), + ); + const findGrant = (caller: ThreadId) => + wrap( + Effect.gen(function* () { + const invocation = yield* Effect.serviceOption(McpInvocationContext.McpInvocationContext); + const persisted = yield* snapshots.getThreadActivationAuthority(caller); + const retained = + (Option.isSome(invocation) ? invocation.value.unattendedAuthority : undefined) ?? + Option.getOrUndefined(persisted); + const available = yield* grants.list({ callerThreadId: caller }); + const grant = available.find( + (entry) => + !entry.revoked && + entry.hostJobs && + (!retained || + (entry.id === retained.grantId && + entry.revision === retained.grantRevision && + retained.ownerThreadId === caller && + retained.mcpCapabilityCeiling.includes("background-jobs"))), + ); + if (!grant) + return yield* new BackgroundJobError({ + code: "forbidden", + detail: "Explicit client host-job consent is required.", + }); + return grant; + }), + ); + const authorize = (caller: ThreadId) => + wrap( + Effect.gen(function* () { + const state = yield* snapshots.getWorkerState(caller); + if (Option.isNone(state)) + return yield* new BackgroundJobError({ + code: "not-found", + detail: "Host-job owner is unavailable.", + }); + if (state.value.thread.worker?.stopRequestedAt != null) + return yield* new BackgroundJobError({ + code: "forbidden", + detail: "Stopped workers cannot execute host jobs.", + }); + const access = yield* capabilities(caller); + if (!access?.has("background-jobs")) + return yield* new BackgroundJobError({ + code: "forbidden", + detail: "Host-job capability is disabled for this project or worker.", + }); + const grant = yield* findGrant(caller); + if (grant.projectId !== state.value.thread.projectId) + return yield* new BackgroundJobError({ + code: "forbidden", + detail: "Host-job consent belongs to another project.", + }); + const project = yield* snapshots.getProjectShellById(state.value.thread.projectId); + if (Option.isNone(project)) + return yield* new BackgroundJobError({ + code: "not-found", + detail: "Host-job project is unavailable.", + }); + const cwd = yield* paths.normalizeWorkspaceRoot( + state.value.thread.worktreePath ?? project.value.workspaceRoot, + ); + return { projectId: project.value.id, cwd }; + }), + ); + return BackgroundJobAuthority.of({ + authorize, + notify: (record, subscription) => + wrap( + Effect.gen(function* () { + const key = `job-terminal:${record.id}:${subscription.callerThreadId}`; + if (subscription.notify) + yield* engine.dispatch({ + type: "thread.activity.append", + commandId: CommandId.make(key), + threadId: subscription.callerThreadId, + activity: { + id: EventId.make(key), + tone: record.state === "completed" ? "info" : "error", + kind: "background-job.terminal", + summary: `Background job ${record.id}: ${record.state}`, + payload: { jobId: record.id, state: record.state, exitCode: record.exitCode }, + turnId: null, + createdAt: record.updatedAt, + }, + createdAt: record.updatedAt, + }); + if (subscription.wake) { + yield* authorize(subscription.callerThreadId); + const grant = yield* findGrant(subscription.callerThreadId); + yield* scheduled.create({ + id: key, + callerThreadId: subscription.callerThreadId, + target: { type: "resume", threadId: subscription.callerThreadId }, + prompt: `Background job ${record.id} finished with state ${record.state}. Retrieve its bounded output if needed.`, + delayMs: 0, + onBusy: "wait", + grantId: grant.id, + }); + } + }), + ), + }); +}); +export const layer = Layer.effect(BackgroundJobAuthority, make); diff --git a/apps/server/src/background/BackgroundJobs.test.ts b/apps/server/src/background/BackgroundJobs.test.ts new file mode 100644 index 000000000000..5c17bda8e51b --- /dev/null +++ b/apps/server/src/background/BackgroundJobs.test.ts @@ -0,0 +1,254 @@ +import * as Deferred from "effect/Deferred"; +import * as Stream from "effect/Stream"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; +import { BackgroundJobError } from "@t3tools/contracts"; +import { expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; +import { ProjectId, ThreadId } from "@t3tools/contracts"; +import * as BackgroundJobs from "./BackgroundJobs.ts"; +import migrate from "../persistence/Migrations/061_BackgroundJobs.ts"; + +const owner = ThreadId.make("job-owner"); +const authority = Layer.succeed(BackgroundJobs.BackgroundJobAuthority, { + authorize: () => Effect.succeed({ projectId: ProjectId.make("project"), cwd: process.cwd() }), + notify: () => Effect.void, +}); +const dependencies = Layer.mergeAll( + NodeSqliteClient.layer({ filename: ":memory:" }), + NodeServices.layer, + authority, +); +const live = BackgroundJobs.layer.pipe(Layer.provideMerge(dependencies)); +const input = (id: string, source: string) => ({ + id, + callerThreadId: owner, + command: process.execPath, + args: ["-e", source], + timeoutMs: 10_000, + maxOutputBytes: 1024, +}); + +it.live("streams real stdout/stderr/progress, cursor slices and bounded truncation", () => + Effect.scoped( + Effect.gen(function* () { + yield* migrate; + const jobs = yield* BackgroundJobs.BackgroundJobs; + yield* jobs.start( + input( + "short", + 'process.stdout.write("hello\\nT3_PROGRESS {\\"value\\":0.5,\\"label\\":\\"half\\"}\\n"); process.stderr.write("warning\\n")', + ), + ); + const result = yield* jobs.wait({ callerThreadId: owner, id: "short", timeoutMs: 10_000 }); + expect(result.timedOut).toBe(false); + expect(result.job.state).toBe("completed"); + expect(result.job.progress).toEqual({ value: 0.5, label: "half" }); + const output = yield* jobs.output({ + callerThreadId: owner, + id: "short", + cursor: 0, + limitBytes: 65_536, + }); + expect( + output.chunks + .filter((chunk) => chunk.stream === "stdout") + .map((chunk) => chunk.text) + .join(""), + ).toContain("hello"); + expect( + output.chunks + .filter((chunk) => chunk.stream === "stderr") + .map((chunk) => chunk.text) + .join(""), + ).toContain("warning"); + const first = yield* jobs.output({ + callerThreadId: owner, + id: "short", + cursor: 0, + limitBytes: 2, + }); + expect(first.nextCursor).toBe(2); + yield* jobs.start(input("large", 'process.stdout.write("x".repeat(4096))')); + const large = yield* jobs.wait({ callerThreadId: owner, id: "large", timeoutMs: 10_000 }); + expect(large.job.outputBytes).toBe(1024); + expect(large.job.truncated).toBe(true); + }).pipe(Effect.provide(live)), + ), +); + +it.live("cancels only its captured process, rejects foreign owners and refuses live cleanup", () => + Effect.scoped( + Effect.gen(function* () { + yield* migrate; + const jobs = yield* BackgroundJobs.BackgroundJobs; + yield* jobs.start( + input("owned", 'process.stdout.write("ready\\n"); setInterval(() => {}, 1000)'), + ); + const foreign = yield* jobs + .get({ callerThreadId: ThreadId.make("foreign"), id: "owned" }) + .pipe(Effect.result); + expect(foreign._tag).toBe("Failure"); + const cleanup = yield* jobs + .cleanup({ callerThreadId: owner, id: "owned" }) + .pipe(Effect.result); + expect(cleanup._tag).toBe("Failure"); + yield* jobs.cancel({ callerThreadId: owner, id: "owned" }); + const result = yield* jobs.wait({ callerThreadId: owner, id: "owned", timeoutMs: 10_000 }); + expect(result.timedOut).toBe(false); + expect(result.job.state).toBe("cancelled"); + }).pipe(Effect.provide(live)), + ), +); + +it.live("rejects job retries with changed timeout or output bounds", () => + Effect.scoped( + Effect.gen(function* () { + yield* migrate; + const jobs = yield* BackgroundJobs.BackgroundJobs; + const request = input("retry-job", "process.stdout.write('done')"); + yield* jobs.start(request); + yield* jobs.wait({ callerThreadId: owner, id: request.id, timeoutMs: 10_000 }); + expect((yield* jobs.start(request)).id).toBe(request.id); + expect((yield* jobs.start({ ...request, timeoutMs: 2000 }).pipe(Effect.result))._tag).toBe( + "Failure", + ); + expect( + (yield* jobs.start({ ...request, maxOutputBytes: 2048 }).pipe(Effect.result))._tag, + ).toBe("Failure"); + }).pipe(Effect.provide(live)), + ), +); + +it.live("persists denied terminal wake suppression and recovery remains ready", () => + Effect.scoped( + Effect.gen(function* () { + yield* migrate; + const jobs = yield* BackgroundJobs.BackgroundJobs; + yield* jobs.start(input("revoked-notification", "process.stdout.write('done')")); + yield* jobs.wait({ callerThreadId: owner, id: "revoked-notification", timeoutMs: 10_000 }); + yield* jobs.subscribe({ + callerThreadId: owner, + id: "revoked-notification", + notify: true, + wake: true, + }); + yield* jobs.reconcile; + yield* jobs.reconcile; + const sql = yield* SqlClient.SqlClient; + const rows = yield* sql<{ + delivered: number; + suppressed_reason: string; + }>`SELECT delivered,suppressed_reason FROM background_job_notifications WHERE job_id = 'revoked-notification'`; + expect(rows[0]?.delivered).toBe(2); + expect(rows[0]?.suppressed_reason).toContain("revoked"); + }).pipe( + Effect.provide( + BackgroundJobs.layer.pipe( + Layer.provideMerge( + Layer.mergeAll( + NodeSqliteClient.layer({ filename: ":memory:" }), + NodeServices.layer, + Layer.succeed(BackgroundJobs.BackgroundJobAuthority, { + authorize: () => + Effect.succeed({ projectId: ProjectId.make("project"), cwd: process.cwd() }), + notify: () => + Effect.fail( + new BackgroundJobError({ code: "forbidden", detail: "Grant revoked." }), + ), + }), + ), + ), + ), + ), + ), + ), +); + +it.live( + "scope shutdown kills its SIGTERM-ignoring captured child before persisting interruption", + () => + Effect.scoped( + Effect.gen(function* () { + yield* migrate; + const sql = yield* SqlClient.SqlClient; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const ready = yield* Deferred.make(); + let launches = 0; + const capturedSpawner = { + ...spawner, + spawn: (command: Parameters[0]) => + spawner.spawn(command).pipe( + Effect.map((handle) => { + launches++; + return { + ...handle, + stdout: handle.stdout.pipe(Stream.tap(() => Deferred.succeed(ready, undefined))), + }; + }), + ), + }; + yield* Effect.scoped( + Effect.gen(function* () { + const jobs = yield* BackgroundJobs.BackgroundJobs; + yield* jobs.start( + input( + "shutdown-owned", + "process.on('SIGTERM', () => {}); process.stdout.write('ready'); setInterval(() => {}, 1000)", + ), + ); + yield* Deferred.await(ready); + }).pipe( + Effect.provide(BackgroundJobs.layer), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, capturedSpawner), + ), + ); + const rows = yield* sql<{ + document_json: string; + }>`SELECT document_json FROM background_jobs WHERE id = 'shutdown-owned'`; + expect(JSON.parse(rows[0]!.document_json).state).toBe("interrupted"); + yield* Effect.scoped( + Effect.gen(function* () { + const jobs = yield* BackgroundJobs.BackgroundJobs; + yield* jobs.reconcile; + }).pipe( + Effect.provide(BackgroundJobs.layer), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, capturedSpawner), + ), + ); + expect(launches).toBe(1); + }).pipe(Effect.provide(dependencies)), + ), +); + +it.live( + "retains partial output on timeout and does not rerun a persisted interrupted process", + () => + Effect.scoped( + Effect.gen(function* () { + yield* migrate; + const jobs = yield* BackgroundJobs.BackgroundJobs; + yield* jobs.start({ + ...input("timeout", 'process.stdout.write("partial\\n"); setInterval(() => {}, 1000)'), + timeoutMs: 1000, + }); + const result = yield* jobs.wait({ + callerThreadId: owner, + id: "timeout", + timeoutMs: 10_000, + }); + expect(result.job.state).toBe("failed"); + expect(result.job.reason).toContain("timeout"); + const output = yield* jobs.output({ + callerThreadId: owner, + id: "timeout", + cursor: 0, + limitBytes: 65_536, + }); + expect(output.chunks.map((chunk) => chunk.text).join("")).toContain("partial"); + }).pipe(Effect.provide(live)), + ), +); diff --git a/apps/server/src/background/BackgroundJobs.ts b/apps/server/src/background/BackgroundJobs.ts new file mode 100644 index 000000000000..30954e2e708e --- /dev/null +++ b/apps/server/src/background/BackgroundJobs.ts @@ -0,0 +1,550 @@ +import { type ProjectId, type ThreadId } from "@t3tools/contracts"; +import { + BackgroundJobStartInput, + BackgroundJobReadInput, + BackgroundJobOutputInput, + BackgroundJobWaitInput, + BackgroundJobRecord, + BackgroundJobError, + type BackgroundJobOutput, +} from "../../../../packages/contracts/src/backgroundJobs.ts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Clock from "effect/Clock"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; +import * as PubSub from "effect/PubSub"; +import * as Stream from "effect/Stream"; +import * as Option from "effect/Option"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as ChildProcess from "effect/unstable/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; + +/** Resolves the authenticated owner's workspace and explicit host-job permission. + * Notification delivery must deduplicate by job ID and subscriber thread ID. */ +export class BackgroundJobAuthority extends Context.Service< + BackgroundJobAuthority, + { + readonly authorize: ( + caller: ThreadId, + ) => Effect.Effect<{ projectId: ProjectId; cwd: string }, BackgroundJobError>; + readonly notify: ( + record: BackgroundJobRecord, + subscription: { callerThreadId: ThreadId; notify: boolean; wake: boolean }, + ) => Effect.Effect; + } +>()("t3/background/BackgroundJobAuthority") {} +export class BackgroundJobs extends Context.Service< + BackgroundJobs, + { + readonly start: ( + input: BackgroundJobStartInput, + ) => Effect.Effect; + readonly list: (input: { + callerThreadId: ThreadId; + }) => Effect.Effect, BackgroundJobError>; + readonly get: ( + input: BackgroundJobReadInput, + ) => Effect.Effect; + readonly output: ( + input: BackgroundJobOutputInput, + ) => Effect.Effect; + readonly cancel: ( + input: BackgroundJobReadInput, + ) => Effect.Effect; + readonly wait: ( + input: BackgroundJobWaitInput, + ) => Effect.Effect<{ timedOut: boolean; job: BackgroundJobRecord }, BackgroundJobError>; + readonly subscribe: ( + input: BackgroundJobReadInput & { notify: boolean; wake: boolean }, + ) => Effect.Effect; + readonly cleanup: (input: BackgroundJobReadInput) => Effect.Effect; + readonly reconcile: Effect.Effect; + } +>()("t3/background/BackgroundJobs") {} +const terminal = (state: BackgroundJobRecord["state"]) => + ["completed", "failed", "cancelled", "interrupted"].includes(state); +const failure = (code: BackgroundJobError["code"], detail: string, cause?: unknown) => + new BackgroundJobError({ code, detail, ...(cause === undefined ? {} : { cause }) }); +const decodeBackgroundJobRecord = Schema.decodeUnknownEffect(BackgroundJobRecord); +const decodeBackgroundJobReadInput = Schema.decodeUnknownEffect(BackgroundJobReadInput); +const decodeBackgroundJobStartInput = Schema.decodeUnknownEffect(BackgroundJobStartInput); +const decodeBackgroundJobOutputInput = Schema.decodeUnknownEffect(BackgroundJobOutputInput); +const decodeBackgroundJobWaitInput = Schema.decodeUnknownEffect(BackgroundJobWaitInput); + +const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const authority = yield* BackgroundJobAuthority; + const scope = yield* Scope.Scope; + const lock = yield* Semaphore.make(1); + const changes = yield* PubSub.sliding(64); + const outputChunks = new Map(); + const decoders = new Map>(); + const progressLines = new Map(); + const handles = new Map(); + const wrap = (effect: Effect.Effect) => + effect.pipe( + Effect.mapError((cause) => + Schema.is(BackgroundJobError)(cause) + ? cause + : failure("internal", "Background job operation failed.", cause), + ), + ); + const read = (id: string) => + wrap( + Effect.gen(function* () { + const rows = yield* sql<{ + document_json: string; + }>`SELECT document_json FROM background_jobs WHERE id = ${id}`; + if (!rows[0]) return null; + return yield* decodeBackgroundJobRecord(JSON.parse(rows[0].document_json)); + }), + ); + const save = (record: BackgroundJobRecord) => + wrap( + Effect.gen(function* () { + yield* sql`UPDATE background_jobs SET state = ${record.state}, document_json = ${JSON.stringify(record)} WHERE id = ${record.id}`; + yield* PubSub.publish(changes, record.id); + }), + ); + const update = (record: BackgroundJobRecord, change: Partial) => + Effect.gen(function* () { + const next = { + ...record, + ...change, + updatedAt: new Date(yield* Clock.currentTimeMillis).toISOString(), + }; + yield* save(next); + return next; + }); + const get = (raw: BackgroundJobReadInput) => + Effect.gen(function* () { + const input = yield* decodeBackgroundJobReadInput(raw).pipe( + Effect.mapError(() => failure("invalid", "Invalid job identifier.")), + ); + const job = yield* read(input.id); + if (!job) return yield* failure("not-found", "Background job not found."); + if (job.ownerThreadId !== input.callerThreadId) + return yield* failure("forbidden", "Job belongs to another thread."); + return job; + }); + const deliver = (record: BackgroundJobRecord) => + wrap( + Effect.gen(function* () { + if (!terminal(record.state)) return; + const rows = yield* sql<{ + subscriber_thread_id: string; + notify: number; + wake: number; + }>`SELECT subscriber_thread_id, notify, wake FROM background_job_notifications WHERE job_id = ${record.id} AND delivered = 0`; + for (const row of rows) { + const delivery = yield* authority + .notify(record, { + callerThreadId: Schema.decodeUnknownSync(Schema.String)( + row.subscriber_thread_id, + ) as ThreadId, + notify: row.notify === 1, + wake: row.wake === 1, + }) + .pipe(Effect.result); + if (delivery._tag === "Failure") { + if (delivery.failure.code === "forbidden" || delivery.failure.code === "not-found") { + yield* sql`UPDATE background_job_notifications SET delivered = 2, suppressed_reason = ${delivery.failure.detail} WHERE job_id = ${record.id} AND subscriber_thread_id = ${row.subscriber_thread_id}`; + continue; + } + yield* Effect.logWarning("Background notification delivery deferred", { + jobId: record.id, + error: delivery.failure.detail, + }); + continue; + } + yield* sql`UPDATE background_job_notifications SET delivered = 1 WHERE job_id = ${record.id} AND subscriber_thread_id = ${row.subscriber_thread_id}`; + } + }), + ); + const append = (id: string, stream: "stdout" | "stderr", bytes: Uint8Array) => + lock.withPermit( + wrap( + Effect.gen(function* () { + const job = yield* read(id); + if (!job) return; + const chunkCount = outputChunks.get(id) ?? 0; + const remaining = + chunkCount >= 4096 ? 0 : Math.max(0, job.maxOutputBytes - job.outputBytes); + const decoderKey = `${id}:${stream}`; + const decoder = decoders.get(decoderKey) ?? new TextDecoder(); + decoders.set(decoderKey, decoder); + const decoded = decoder.decode(bytes, { stream: true }); + let text = decoded; + if (new TextEncoder().encode(text).byteLength > remaining) { + let low = 0; + let high = text.length; + while (low < high) { + const middle = Math.ceil((low + high) / 2); + if (new TextEncoder().encode(text.slice(0, middle)).byteLength <= remaining) + low = middle; + else high = middle - 1; + } + text = text.slice(0, low); + if (text.length > 0 && /[\uD800-\uDBFF]$/.test(text)) text = text.slice(0, -1); + } + // Output remains ordinary text. A bounded, valid line may additionally update progress. + let progress = job.progress; + if (stream === "stdout") { + const pending = (progressLines.get(id) ?? "") + decoded; + const lines = pending.split("\n"); + const unfinished = lines.pop() ?? ""; + progressLines.set(id, unfinished.length <= 8192 ? unfinished : ""); + for (const line of lines) { + if (line.length > 8192 || !line.startsWith("T3_PROGRESS ")) continue; + try { + const parsed: unknown = JSON.parse(line.slice(12)); + progress = Schema.decodeUnknownSync(BackgroundJobRecord.fields.progress)(parsed); + } catch { + /* Malformed progress is retained as ordinary stdout. */ + } + } + } + const size = new TextEncoder().encode(text).byteLength; + if (size > 0) { + yield* sql`INSERT INTO background_job_output (job_id,cursor,stream,text) VALUES (${id},${job.outputBytes},${stream},${text})`; + outputChunks.set(id, chunkCount + 1); + } + yield* update(job, { + outputBytes: job.outputBytes + size, + truncated: job.truncated || bytes.byteLength > remaining, + progress, + }); + }), + ), + ); + const run = (input: BackgroundJobStartInput, job: BackgroundJobRecord) => + Effect.scoped( + Effect.gen(function* () { + const handle = yield* lock.withPermit( + Effect.gen(function* () { + const latest = yield* read(job.id); + if (!latest || terminal(latest.state) || latest.state === "cancelling") return null; + yield* authority.authorize(job.ownerThreadId); + const captured = yield* wrap( + spawner.spawn( + ChildProcess.make(input.command, input.args, { + cwd: job.cwd, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + killSignal: "SIGKILL", + forceKillAfter: 1000, + }), + ), + ); + handles.set(job.id, captured); + yield* update(latest, { state: "running" }); + return captured; + }), + ); + if (!handle) return; + yield* Effect.addFinalizer(() => + Effect.gen(function* () { + // Keep the captured handle until native termination is acknowledged. + yield* handle.kill({ killSignal: "SIGKILL" }).pipe(Effect.catch(() => Effect.void)); + yield* handle.exitCode.pipe(Effect.catch(() => Effect.void)); + handles.delete(job.id); + outputChunks.delete(job.id); + decoders.delete(`${job.id}:stdout`); + decoders.delete(`${job.id}:stderr`); + progressLines.delete(job.id); + yield* lock + .withPermit( + Effect.gen(function* () { + const latest = yield* read(job.id); + if (latest && !terminal(latest.state)) + yield* update(latest, { + state: "interrupted", + reason: "Host supervisor stopped. Job was not rerun.", + }); + }), + ) + .pipe(Effect.catch(() => Effect.void)); + }), + ); + + const result = yield* Effect.all( + [ + handle.stdout.pipe(Stream.runForEach((chunk) => append(job.id, "stdout", chunk))), + handle.stderr.pipe(Stream.runForEach((chunk) => append(job.id, "stderr", chunk))), + handle.exitCode, + ], + { concurrency: "unbounded" }, + ).pipe(Effect.timeoutOption(input.timeoutMs), Effect.result); + if (result._tag === "Failure" || Option.isNone(result.success)) { + yield* handle.kill({ killSignal: "SIGKILL" }).pipe(Effect.catch(() => Effect.void)); + yield* handle.exitCode.pipe(Effect.catch(() => Effect.void)); + } + const completed = yield* lock.withPermit( + Effect.gen(function* () { + const latest = yield* read(job.id); + if (!latest) return null; + const timedOut = result._tag === "Success" && Option.isNone(result.success); + const exitCode = + result._tag === "Success" && Option.isSome(result.success) + ? result.success.value[2] + : null; + return yield* update(latest, { + state: + latest.state === "cancelling" + ? "cancelled" + : timedOut || result._tag === "Failure" || exitCode !== 0 + ? "failed" + : "completed", + exitCode, + reason: + latest.state === "cancelling" + ? "Cancelled by owner." + : timedOut + ? "Process timeout. Partial output retained." + : result._tag === "Failure" + ? "Process stream or execution failed." + : null, + }); + }), + ); + if (completed) yield* deliver(completed); + }), + ).pipe( + Effect.catch((error) => + lock.withPermit( + Effect.gen(function* () { + const latest = yield* read(job.id); + if (latest && !terminal(latest.state)) { + const failed = yield* update(latest, { state: "failed", reason: error.detail }); + yield* deliver(failed); + } + }), + ), + ), + ); + const start = (raw: BackgroundJobStartInput) => + lock.withPermit( + Effect.gen(function* () { + const input = yield* decodeBackgroundJobStartInput(raw).pipe( + Effect.mapError(() => failure("invalid", "Invalid bounded process input.")), + ); + const permission = yield* authority.authorize(input.callerThreadId); + const requestJson = JSON.stringify({ + command: input.command, + args: input.args, + timeoutMs: input.timeoutMs, + maxOutputBytes: input.maxOutputBytes, + }); + const existing = yield* read(input.id); + if (existing) { + const requests = yield* wrap( + sql<{ + request_json: string; + }>`SELECT request_json FROM background_jobs WHERE id = ${input.id}`, + ); + if (requests[0]?.request_json !== requestJson) + return yield* failure( + "conflict", + "Job identifier already has different execution bounds.", + ); + if ( + existing.ownerThreadId !== input.callerThreadId || + existing.command !== input.command || + JSON.stringify(existing.args) !== JSON.stringify(input.args) + ) + return yield* failure("conflict", "Job identifier already has a different request."); + return existing; + } + const retained = yield* wrap( + sql<{ count: number }>`SELECT COUNT(*) AS count FROM background_jobs`, + ); + if ((retained[0]?.count ?? 0) >= 256) + return yield* failure( + "running", + "Retained job limit reached. Clean up terminal jobs before starting another.", + ); + const active = yield* wrap( + sql<{ + count: number; + }>`SELECT COUNT(*) AS count FROM background_jobs WHERE state IN ('pending','running','cancelling')`, + ); + if ((active[0]?.count ?? 0) >= 16) + return yield* failure("running", "Host job concurrency limit reached."); + const now = new Date(yield* Clock.currentTimeMillis).toISOString(); + const job: BackgroundJobRecord = { + id: input.id, + ownerThreadId: input.callerThreadId, + projectId: permission.projectId, + command: input.command, + args: input.args, + cwd: permission.cwd, + state: "pending", + createdAt: now, + updatedAt: now, + exitCode: null, + reason: null, + maxOutputBytes: input.maxOutputBytes, + outputBytes: 0, + truncated: false, + progress: null, + }; + yield* wrap( + sql`INSERT INTO background_jobs(id,owner_thread_id,state,request_json,document_json) VALUES(${job.id},${job.ownerThreadId},${job.state},${requestJson},${JSON.stringify(job)})`, + ); + yield* run(input, job).pipe(Effect.forkIn(scope)); + return job; + }), + ); + const list = (input: { callerThreadId: ThreadId }) => + wrap( + Effect.gen(function* () { + const rows = yield* sql<{ + document_json: string; + }>`SELECT document_json FROM background_jobs WHERE owner_thread_id = ${input.callerThreadId} ORDER BY id LIMIT 200`; + return yield* Effect.forEach(rows, (row) => + decodeBackgroundJobRecord(JSON.parse(row.document_json)), + ); + }), + ); + const output = (raw: BackgroundJobOutputInput) => + wrap( + Effect.gen(function* () { + const input = yield* decodeBackgroundJobOutputInput(raw); + const job = yield* get(input); + if (input.cursor > job.outputBytes) + return yield* failure("invalid", "Output cursor exceeds retained output."); + const rows = yield* sql<{ + cursor: number; + stream: "stdout" | "stderr"; + text: string; + }>`SELECT cursor,stream,text FROM background_job_output WHERE job_id = ${input.id} ORDER BY cursor`; + const chunks: Array = []; + let remaining = input.limitBytes; + let nextCursor = input.cursor; + for (const row of rows) { + const bytes = new TextEncoder().encode(row.text); + if (row.cursor + bytes.length <= input.cursor) continue; + const start = Math.max(0, input.cursor - row.cursor); + const piece = bytes.subarray(start, start + remaining); + if (piece.length === 0) break; + chunks.push({ + cursor: row.cursor + start, + stream: row.stream, + text: new TextDecoder().decode(piece), + }); + nextCursor = row.cursor + start + piece.length; + remaining -= piece.length; + if (remaining === 0) break; + } + return { chunks, nextCursor, truncated: job.truncated }; + }), + ); + const cancel = (input: BackgroundJobReadInput) => + Effect.gen(function* () { + const job = yield* lock.withPermit( + Effect.gen(function* () { + const current = yield* get(input); + if (terminal(current.state)) return current; + return yield* update(current, { state: "cancelling" }); + }), + ); + if (!terminal(job.state)) { + const handle = handles.get(job.id); + if (handle) yield* wrap(handle.kill({ killSignal: "SIGKILL" })); + else + return yield* lock.withPermit( + update(job, { state: "cancelled", reason: "Cancelled before process activation." }), + ); + } + return yield* get(input); + }); + const wait = (raw: BackgroundJobWaitInput) => + Effect.scoped( + Effect.gen(function* () { + const input = yield* decodeBackgroundJobWaitInput(raw).pipe( + Effect.mapError(() => failure("invalid", "Invalid wait input.")), + ); + const subscription = yield* PubSub.subscribe(changes); + const initial = yield* get(input); + if (terminal(initial.state)) return { timedOut: false, job: initial }; + const settled = yield* Stream.fromSubscription(subscription).pipe( + Stream.mapEffect(() => get(input)), + Stream.filter((job) => terminal(job.state)), + Stream.runHead, + Effect.timeoutOption(input.timeoutMs), + ); + if (Option.isSome(settled) && Option.isSome(settled.value)) + return { timedOut: false, job: settled.value.value }; + return { timedOut: true, job: yield* get(input) }; + }), + ); + const subscribe = (input: BackgroundJobReadInput & { notify: boolean; wake: boolean }) => + lock.withPermit( + wrap( + Effect.gen(function* () { + const job = yield* get(input); + if (input.wake) yield* authority.authorize(input.callerThreadId); + yield* sql`INSERT INTO background_job_notifications(job_id,subscriber_thread_id,notify,wake) VALUES(${job.id},${input.callerThreadId},${input.notify ? 1 : 0},${input.wake ? 1 : 0}) ON CONFLICT(job_id,subscriber_thread_id) DO UPDATE SET notify=excluded.notify,wake=excluded.wake`; + yield* deliver(job); + return job; + }), + ), + ); + const cleanup = (input: BackgroundJobReadInput) => + lock.withPermit( + wrap( + Effect.gen(function* () { + const job = yield* get(input); + if (!terminal(job.state) || handles.has(job.id)) + return yield* failure("running", "Cannot clean up a live job."); + yield* sql.withTransaction( + Effect.gen(function* () { + yield* sql`DELETE FROM background_job_output WHERE job_id = ${job.id}`; + yield* sql`DELETE FROM background_job_notifications WHERE job_id = ${job.id}`; + yield* sql`DELETE FROM background_jobs WHERE id = ${job.id}`; + }), + ); + }), + ), + ); + const reconcile = lock.withPermit( + wrap( + Effect.gen(function* () { + const rows = yield* sql<{ + document_json: string; + }>`SELECT document_json FROM background_jobs WHERE state IN ('pending','running','cancelling')`; + for (const row of rows) { + const job = yield* decodeBackgroundJobRecord(JSON.parse(row.document_json)); + if (handles.has(job.id)) continue; + const interrupted = yield* update(job, { + state: "interrupted", + reason: "Host restarted. Captured process cannot be reattached and was not rerun.", + }); + yield* deliver(interrupted); + } + const pendingNotifications = yield* sql<{ + document_json: string; + }>`SELECT DISTINCT jobs.document_json FROM background_jobs jobs JOIN background_job_notifications subscriptions ON subscriptions.job_id = jobs.id WHERE subscriptions.delivered = 0 AND jobs.state IN ('completed','failed','cancelled','interrupted')`; + for (const row of pendingNotifications) + yield* deliver(yield* decodeBackgroundJobRecord(JSON.parse(row.document_json))); + }), + ), + ); + return BackgroundJobs.of({ + start, + list, + get, + output, + cancel, + wait, + subscribe, + cleanup, + reconcile, + }); +}); +export const layer = Layer.effect(BackgroundJobs, make); diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index 230b78b26ad4..9387dbc02f14 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -40,6 +40,8 @@ import { MemoryToolkit } from "./toolkits/memory/tools.ts"; import { MemoryToolkitHandlersLive } from "./toolkits/memory/handlers.ts"; import { QualityRecordsToolkit } from "./toolkits/qualityRecords/tools.ts"; import { QualityRecordsToolkitHandlersLive } from "./toolkits/qualityRecords/handlers.ts"; +import { AutomationToolkit } from "./toolkits/automation/tools.ts"; +import { AutomationToolkitHandlersLive } from "./toolkits/automation/handlers.ts"; import { DeviceScreenshotToolkitHandlersLive, DeviceStandardToolkitHandlersLive, @@ -686,5 +688,6 @@ export const layer = Layer.mergeAll( McpServer.toolkit(CoordinationToolkit).pipe(Layer.provide(CoordinationToolkitHandlersLive)), McpServer.toolkit(MemoryToolkit).pipe(Layer.provide(MemoryToolkitHandlersLive)), McpServer.toolkit(QualityRecordsToolkit).pipe(Layer.provide(QualityRecordsToolkitHandlersLive)), + McpServer.toolkit(AutomationToolkit).pipe(Layer.provide(AutomationToolkitHandlersLive)), DeviceToolkitRegistrationLive, ).pipe(Layer.provideMerge(McpTransportLive)); diff --git a/apps/server/src/mcp/ParityToolkitRegistration.test.ts b/apps/server/src/mcp/ParityToolkitRegistration.test.ts index 2b1553c95704..f844ba2dafe8 100644 --- a/apps/server/src/mcp/ParityToolkitRegistration.test.ts +++ b/apps/server/src/mcp/ParityToolkitRegistration.test.ts @@ -7,6 +7,7 @@ import * as McpSchema from "effect/unstable/ai/McpSchema"; import * as Tool from "effect/unstable/ai/Tool"; import { CoordinationToolkit } from "./toolkits/coordination/tools.ts"; import { WorkersToolkit } from "./toolkits/workers/tools.ts"; +import { AutomationToolkit } from "./toolkits/automation/tools.ts"; import { MemoryToolkit } from "./toolkits/memory/tools.ts"; import { QualityRecordsToolkit } from "./toolkits/qualityRecords/tools.ts"; @@ -28,6 +29,21 @@ const registrationHandlers = Layer.mergeAll( workers_stop: uncalled, workers_wait: uncalled, }), + AutomationToolkit.toLayer({ + schedule_create: uncalled, + schedule_list: uncalled, + schedule_get: uncalled, + schedule_cancel: uncalled, + unattended_grants_list: uncalled, + background_job_start: uncalled, + background_job_list: uncalled, + background_job_get: uncalled, + background_job_output: uncalled, + background_job_cancel: uncalled, + background_job_wait: uncalled, + background_job_subscribe: uncalled, + background_job_cleanup: uncalled, + }), MemoryToolkit.toLayer({ memory_remember: uncalled, memory_recall: uncalled, @@ -47,10 +63,11 @@ it.effect("all added toolkits register through the MCP server", () => Effect.gen(function* () { yield* McpServer.registerToolkit(CoordinationToolkit); yield* McpServer.registerToolkit(WorkersToolkit); + yield* McpServer.registerToolkit(AutomationToolkit); yield* McpServer.registerToolkit(MemoryToolkit); yield* McpServer.registerToolkit(QualityRecordsToolkit); const server = yield* McpServer.McpServer; - expect(server.tools.length).toBe(19); + expect(server.tools.length).toBe(32); for (const { tool } of server.tools) expect(tool.inputSchema.type, tool.name).toBe("object"); }).pipe(Effect.provide(registrationHandlers), Effect.provide(McpServer.McpServer.layer)), ), @@ -61,6 +78,7 @@ it.effect("all added toolkits expose MCP-compatible object parameter schemas", ( for (const toolkit of [ CoordinationToolkit, WorkersToolkit, + AutomationToolkit, MemoryToolkit, QualityRecordsToolkit, ]) { diff --git a/apps/server/src/mcp/toolkits/automation/handlers.test.ts b/apps/server/src/mcp/toolkits/automation/handlers.test.ts new file mode 100644 index 000000000000..b8e46b970740 --- /dev/null +++ b/apps/server/src/mcp/toolkits/automation/handlers.test.ts @@ -0,0 +1,95 @@ +import { expect, it } from "@effect/vitest"; +import { EnvironmentId, ProjectId, ProviderInstanceId, ThreadId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Stream from "effect/Stream"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; +import * as BackgroundJobs from "../../../background/BackgroundJobs.ts"; +import * as ScheduledWork from "../../../orchestration/ScheduledWork.ts"; +import * as UnattendedGrants from "../../../orchestration/UnattendedGrants.ts"; +import * as McpInvocationContext from "../../McpInvocationContext.ts"; +import migrate from "../../../persistence/Migrations/061_BackgroundJobs.ts"; +import { AutomationToolkit } from "./tools.ts"; +import { AutomationToolkitHandlersLive } from "./handlers.ts"; + +const owner = ThreadId.make("mcp-job-owner"); +const invocation = ( + capabilities: ReadonlyArray, +): McpInvocationContext.McpInvocationScope => ({ + environmentId: EnvironmentId.make("test"), + threadId: owner, + providerSessionId: "test-session", + providerInstanceId: ProviderInstanceId.make("codex"), + capabilities: new Set(capabilities), + issuedAt: 1, +}); +const base = Layer.mergeAll( + NodeSqliteClient.layer({ filename: ":memory:" }), + NodeServices.layer, + Layer.succeed(BackgroundJobs.BackgroundJobAuthority, { + authorize: () => Effect.succeed({ projectId: ProjectId.make("project"), cwd: process.cwd() }), + notify: () => Effect.void, + }), + Layer.mock(ScheduledWork.ScheduledWork)({}), + Layer.mock(UnattendedGrants.UnattendedGrants)({}), +); +const live = BackgroundJobs.layer.pipe(Layer.provideMerge(base)); + +it.live( + "authenticated MCP executes a real scoped host process and caller JSON cannot spoof its owner", + () => + Effect.scoped( + Effect.gen(function* () { + yield* migrate; + const toolkit = yield* AutomationToolkit.pipe( + Effect.provide(AutomationToolkitHandlersLive), + ); + const maliciousInput = { + id: "mcp-job", + callerThreadId: "spoofed", + command: process.execPath, + args: ["-e", 'process.stdout.write("MCP real output")'], + timeoutMs: 10_000, + maxOutputBytes: 1024, + }; + yield* toolkit + .handle("background_job_start", maliciousInput) + .pipe(Stream.unwrap, Stream.runDrain); + const jobs = yield* BackgroundJobs.BackgroundJobs; + const result = yield* jobs.wait({ + callerThreadId: owner, + id: "mcp-job", + timeoutMs: 10_000, + }); + expect(result.job.ownerThreadId).toBe(owner); + expect(result.job.state).toBe("completed"); + const output = yield* jobs.output({ + callerThreadId: owner, + id: "mcp-job", + cursor: 0, + limitBytes: 1024, + }); + expect(output.chunks.map((chunk) => chunk.text).join("")).toBe("MCP real output"); + const denied = yield* toolkit + .handle("background_job_list", {}) + .pipe( + Stream.unwrap, + Stream.runDrain, + Effect.provideService( + McpInvocationContext.McpInvocationContext, + invocation(["automation"]), + ), + Effect.result, + ); + expect(denied._tag).toBe("Failure"); + expect(Object.keys(AutomationToolkit.tools)).not.toContain("unattended_grants_create"); + }).pipe( + Effect.provide(live), + Effect.provideService( + McpInvocationContext.McpInvocationContext, + invocation(["background-jobs"]), + ), + ), + ), +); diff --git a/apps/server/src/mcp/toolkits/automation/handlers.ts b/apps/server/src/mcp/toolkits/automation/handlers.ts new file mode 100644 index 000000000000..5e92688b2881 --- /dev/null +++ b/apps/server/src/mcp/toolkits/automation/handlers.ts @@ -0,0 +1,70 @@ +import * as Effect from "effect/Effect"; +import * as ScheduledWork from "../../../orchestration/ScheduledWork.ts"; +import * as BackgroundJobs from "../../../background/BackgroundJobs.ts"; +import * as UnattendedGrants from "../../../orchestration/UnattendedGrants.ts"; +import * as McpInvocationContext from "../../McpInvocationContext.ts"; +import { AutomationToolkit } from "./tools.ts"; + +const make = Effect.gen(function* () { + const scheduled = yield* ScheduledWork.ScheduledWork; + const jobs = yield* BackgroundJobs.BackgroundJobs; + const grants = yield* UnattendedGrants.UnattendedGrants; + return AutomationToolkit.of({ + schedule_create: Effect.fn("AutomationToolkit.create")(function* (input) { + const caller = yield* McpInvocationContext.requireMcpCapability("automation"); + return yield* scheduled.create( + { ...input, callerThreadId: caller.threadId }, + { mcpCapabilityCeiling: [...caller.capabilities] }, + ); + }), + schedule_list: Effect.fn("AutomationToolkit.list")(function* () { + const caller = yield* McpInvocationContext.requireMcpCapability("automation"); + return yield* scheduled.list({ callerThreadId: caller.threadId }); + }), + schedule_get: Effect.fn("AutomationToolkit.get")(function* (input) { + const caller = yield* McpInvocationContext.requireMcpCapability("automation"); + return yield* scheduled.get({ ...input, callerThreadId: caller.threadId }); + }), + schedule_cancel: Effect.fn("AutomationToolkit.cancel")(function* (input) { + const caller = yield* McpInvocationContext.requireMcpCapability("automation"); + return yield* scheduled.cancel({ ...input, callerThreadId: caller.threadId }); + }), + unattended_grants_list: Effect.fn("AutomationToolkit.grants")(function* () { + const caller = yield* McpInvocationContext.requireMcpCapability("automation"); + return yield* grants.list({ callerThreadId: caller.threadId }); + }), + background_job_start: Effect.fn("AutomationToolkit.jobStart")(function* (input) { + const caller = yield* McpInvocationContext.requireMcpCapability("background-jobs"); + return yield* jobs.start({ ...input, callerThreadId: caller.threadId }); + }), + background_job_list: Effect.fn("AutomationToolkit.jobList")(function* () { + const caller = yield* McpInvocationContext.requireMcpCapability("background-jobs"); + return yield* jobs.list({ callerThreadId: caller.threadId }); + }), + background_job_get: Effect.fn("AutomationToolkit.jobGet")(function* (input) { + const caller = yield* McpInvocationContext.requireMcpCapability("background-jobs"); + return yield* jobs.get({ ...input, callerThreadId: caller.threadId }); + }), + background_job_output: Effect.fn("AutomationToolkit.jobOutput")(function* (input) { + const caller = yield* McpInvocationContext.requireMcpCapability("background-jobs"); + return yield* jobs.output({ ...input, callerThreadId: caller.threadId }); + }), + background_job_cancel: Effect.fn("AutomationToolkit.jobCancel")(function* (input) { + const caller = yield* McpInvocationContext.requireMcpCapability("background-jobs"); + return yield* jobs.cancel({ ...input, callerThreadId: caller.threadId }); + }), + background_job_wait: Effect.fn("AutomationToolkit.jobWait")(function* (input) { + const caller = yield* McpInvocationContext.requireMcpCapability("background-jobs"); + return yield* jobs.wait({ ...input, callerThreadId: caller.threadId }); + }), + background_job_subscribe: Effect.fn("AutomationToolkit.jobSubscribe")(function* (input) { + const caller = yield* McpInvocationContext.requireMcpCapability("background-jobs"); + return yield* jobs.subscribe({ ...input, callerThreadId: caller.threadId }); + }), + background_job_cleanup: Effect.fn("AutomationToolkit.jobCleanup")(function* (input) { + const caller = yield* McpInvocationContext.requireMcpCapability("background-jobs"); + return yield* jobs.cleanup({ ...input, callerThreadId: caller.threadId }); + }), + }); +}); +export const AutomationToolkitHandlersLive = AutomationToolkit.toLayer(make); diff --git a/apps/server/src/mcp/toolkits/automation/tools.ts b/apps/server/src/mcp/toolkits/automation/tools.ts new file mode 100644 index 000000000000..959ab87afc24 --- /dev/null +++ b/apps/server/src/mcp/toolkits/automation/tools.ts @@ -0,0 +1,139 @@ +import { McpCapabilityUnavailableError } from "@t3tools/contracts"; +import { + ScheduledWorkCreateInput, + ScheduledWorkReadInput, + ScheduledWorkRecord, + ScheduledWorkError, +} from "../../../../../..//packages/contracts/src/scheduledWork.ts"; +import { + BackgroundJobStartInput, + BackgroundJobReadInput, + BackgroundJobOutputInput, + BackgroundJobWaitInput, + BackgroundJobRecord, + BackgroundJobOutput, + BackgroundJobError, +} from "../../../../../..//packages/contracts/src/backgroundJobs.ts"; +import { UnattendedGrant } from "../../../../../..//packages/contracts/src/unattendedGrants.ts"; +import * as Schema from "effect/Schema"; +import * as Tool from "effect/unstable/ai/Tool"; +import * as Toolkit from "effect/unstable/ai/Toolkit"; +import * as ScheduledWork from "../../../orchestration/ScheduledWork.ts"; +import * as BackgroundJobs from "../../../background/BackgroundJobs.ts"; +import * as UnattendedGrants from "../../../orchestration/UnattendedGrants.ts"; +import * as McpInvocationContext from "../../McpInvocationContext.ts"; + +const { callerThreadId: _scheduleCaller, ...scheduleFields } = ScheduledWorkCreateInput.fields; +const { callerThreadId: _scheduleReadCaller, ...scheduleReadFields } = + ScheduledWorkReadInput.fields; +const { callerThreadId: _jobCaller, ...jobFields } = BackgroundJobStartInput.fields; +const { callerThreadId: _jobReadCaller, ...jobReadFields } = BackgroundJobReadInput.fields; +const { callerThreadId: _outputCaller, ...outputFields } = BackgroundJobOutputInput.fields; +const { callerThreadId: _waitCaller, ...waitFields } = BackgroundJobWaitInput.fields; +const scheduleDependencies = [ + McpInvocationContext.McpInvocationContext, + ScheduledWork.ScheduledWork, +]; +const jobDependencies = [McpInvocationContext.McpInvocationContext, BackgroundJobs.BackgroundJobs]; +const scheduleFailure = Schema.Union([McpCapabilityUnavailableError, ScheduledWorkError]); +const jobFailure = Schema.Union([McpCapabilityUnavailableError, BackgroundJobError]); + +export const AutomationToolkit = Toolkit.make( + Tool.make("schedule_create", { + description: + "Persist a one-shot resume or owned spawn. Requires an existing explicit human unattended grant. Cannot mint consent. Specify exactly one dueAt or delayMs, reuse id on retries. Acceptance is not completion.", + parameters: Schema.Struct(scheduleFields), + success: ScheduledWorkRecord, + failure: scheduleFailure, + dependencies: scheduleDependencies, + }), + Tool.make("schedule_list", { + description: "List this thread's durable scheduled activations and truthful execution states.", + success: Schema.Array(ScheduledWorkRecord), + failure: scheduleFailure, + dependencies: scheduleDependencies, + }), + Tool.make("schedule_get", { + description: "Inspect one owned schedule.", + parameters: Schema.Struct(scheduleReadFields), + success: ScheduledWorkRecord, + failure: scheduleFailure, + dependencies: scheduleDependencies, + }), + Tool.make("schedule_cancel", { + description: + "Cancel an owned schedule, targeting only its exact activation. Stopping remains pending until acknowledged.", + parameters: Schema.Struct(scheduleReadFields), + success: ScheduledWorkRecord, + failure: scheduleFailure, + dependencies: scheduleDependencies, + }), + Tool.make("unattended_grants_list", { + description: + "Inspect existing explicit human consent and effective ceilings. This tool cannot create or elevate a grant.", + success: Schema.Array(UnattendedGrant), + failure: scheduleFailure, + dependencies: [McpInvocationContext.McpInvocationContext, UnattendedGrants.UnattendedGrants], + }), + Tool.make("background_job_start", { + description: + "Execute a noninteractive host process in this thread's resolved project workspace, with explicit host-job consent. Streams bounded stdout/stderr, parses T3_PROGRESS JSON, and kills only its captured process on timeout/cancel. Never returns native provider background status. Shell execution requires explicitly choosing a shell executable and its arguments.", + parameters: Schema.Struct(jobFields), + success: BackgroundJobRecord, + failure: jobFailure, + dependencies: jobDependencies, + }), + Tool.make("background_job_list", { + description: "List scoped host jobs, including terminal jobs retained across host restart.", + success: Schema.Array(BackgroundJobRecord), + failure: jobFailure, + dependencies: jobDependencies, + }), + Tool.make("background_job_get", { + description: "Inspect this thread's captured-process job.", + parameters: Schema.Struct(jobReadFields), + success: BackgroundJobRecord, + failure: jobFailure, + dependencies: jobDependencies, + }), + Tool.make("background_job_output", { + description: + "Read bounded stdout/stderr using a byte cursor. Truncation is explicit, output is retained in host SQLite until cleanup.", + parameters: Schema.Struct(outputFields), + success: BackgroundJobOutput, + failure: jobFailure, + dependencies: jobDependencies, + }), + Tool.make("background_job_cancel", { + description: + "Cancel only this job's captured process. Cancellation does not use PID/name matching.", + parameters: Schema.Struct(jobReadFields), + success: BackgroundJobRecord, + failure: jobFailure, + dependencies: jobDependencies, + }), + Tool.make("background_job_wait", { + description: + "Event-driven bounded wait for a terminal process state, subscribe-before-snapshot.", + parameters: Schema.Struct(waitFields), + success: Schema.Struct({ timedOut: Schema.Boolean, job: BackgroundJobRecord }), + failure: jobFailure, + dependencies: jobDependencies, + }), + Tool.make("background_job_subscribe", { + description: + "Persist terminal notify/wake preferences. Wake uses granted idle scheduling, not an injection into live foreground work.", + parameters: Schema.Struct({ ...jobReadFields, notify: Schema.Boolean, wake: Schema.Boolean }), + success: BackgroundJobRecord, + failure: jobFailure, + dependencies: jobDependencies, + }), + Tool.make("background_job_cleanup", { + description: + "Delete only an owned terminal job and its retained bounded output. Live jobs reject cleanup.", + parameters: Schema.Struct(jobReadFields), + success: Schema.Void, + failure: jobFailure, + dependencies: jobDependencies, + }), +); diff --git a/apps/server/src/orchestration/AmbientWork.test.ts b/apps/server/src/orchestration/AmbientWork.test.ts new file mode 100644 index 000000000000..5c9761fc4302 --- /dev/null +++ b/apps/server/src/orchestration/AmbientWork.test.ts @@ -0,0 +1,153 @@ +import { expect, it } from "@effect/vitest"; +import { ProjectId, ThreadId, type BackgroundPolicySnapshot } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as DateTime from "effect/DateTime"; +import * as Option from "effect/Option"; +import * as Stream from "effect/Stream"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; +import { TestClock } from "effect/testing"; +import * as AmbientWork from "./AmbientWork.ts"; +import * as ScheduledWork from "./ScheduledWork.ts"; +import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; +import { ProjectionSnapshotQuery } from "./Services/ProjectionSnapshotQuery.ts"; +import { OrchestrationEngineService } from "./Services/OrchestrationEngine.ts"; +import migrate from "../persistence/Migrations/060_ScheduledWork.ts"; + +const owner = ThreadId.make("ambient-owner"); +const project = ProjectId.make("ambient-project"); +const config = { + callerThreadId: owner, + enabled: true, + grantId: "human-consent", + prompt: "Inspect recent work without changing permissions.", + idleDelayMs: 60_000, + minimumCycleMs: 60_000, + maxCyclesPerDay: 2, + timezone: "UTC", + allowedHourStart: 0, + allowedHourEnd: 24, + allowUnknownQuota: true, +}; +function harness() { + let snapshot: BackgroundPolicySnapshot = { + hostPower: { + source: "node-macos-native", + idle: "true", + idleSeconds: 3600, + locked: "false", + suspended: false, + onBattery: "false", + lowPowerMode: "false", + thermalState: "nominal", + stale: false, + updatedAt: DateTime.makeUnsafe(0), + }, + leases: [], + activeForegroundLeaseCount: 0, + activeScopeKeys: [], + shouldRunOpportunisticWork: false, + updatedAt: DateTime.makeUnsafe(0), + }; + const activation = Layer.succeed(ScheduledWork.ScheduledWorkActivation, { + authorize: () => + Effect.succeed({ + projectId: project, + grantRevision: 1, + ceiling: { runtimeMode: "approval-required", mcpCapabilities: ["workers"] }, + }), + validate: () => Effect.void, + dispatch: () => Effect.succeed({ sequence: 1, threadId: owner }), + reconcile: () => Effect.succeed(null), + cancel: () => Effect.succeed("cancelled"), + }); + const engine = Layer.mock(OrchestrationEngineService)({ + subscribeDomainEvents: Effect.succeed(Stream.empty), + }); + const query = Layer.succeed(ProjectionSnapshotQuery, { + getWorkerState: () => + Effect.succeed( + Option.some({ thread: { id: owner, projectId: project }, pendingMessageId: null }), + ), + getShellSnapshot: () => Effect.succeed({ threads: [] }), + } as unknown as ProjectionSnapshotQuery["Service"]); + const policy = Layer.succeed(BackgroundPolicy.BackgroundPolicy, { + snapshot: Effect.sync(() => snapshot), + } as BackgroundPolicy.BackgroundPolicy["Service"]); + const base = Layer.mergeAll( + NodeSqliteClient.layer({ filename: ":memory:" }), + activation, + engine, + query, + policy, + ); + const scheduler = ScheduledWork.layer.pipe(Layer.provideMerge(base)); + return { + layer: AmbientWork.layer.pipe(Layer.provideMerge(scheduler)), + setForeground: (count: number) => { + snapshot = { ...snapshot, activeForegroundLeaseCount: count }; + }, + }; +} + +it.effect( + "is absent by default, waits for idle, persists cycles, never duplicates an active cycle and disables pending work", + () => { + const test = harness(); + return Effect.gen(function* () { + yield* migrate; + const ambient = yield* AmbientWork.AmbientWork; + const scheduler = yield* ScheduledWork.ScheduledWork; + expect(yield* ambient.get({ callerThreadId: owner })).toBe(null); + yield* ambient.configure(config, { source: "client" }); + yield* ambient.drain; + expect((yield* scheduler.list({ callerThreadId: owner })).length).toBe(0); + yield* TestClock.adjust(60_000); + yield* ambient.drain; + const first = yield* scheduler.list({ callerThreadId: owner }); + expect(first.length).toBe(1); + expect(first[0]?.target.type).toBe("ambient"); + yield* ambient.drain; + expect((yield* scheduler.list({ callerThreadId: owner })).length).toBe(1); + expect((yield* ambient.get({ callerThreadId: owner }))?.cycles).toBe(1); + yield* ambient.stop({ callerThreadId: owner }); + expect((yield* scheduler.get({ callerThreadId: owner, id: first[0]!.id })).state).toBe( + "cancelled", + ); + yield* TestClock.adjust(86_400_000); + yield* ambient.drain; + expect((yield* scheduler.list({ callerThreadId: owner })).length).toBe(1); + }).pipe(Effect.provide(test.layer)); + }, +); + +it.effect( + "foreground interaction resets the idle boundary, unknown quota blocks without explicit opt-in", + () => { + const test = harness(); + return Effect.gen(function* () { + yield* migrate; + const ambient = yield* AmbientWork.AmbientWork; + const scheduler = yield* ScheduledWork.ScheduledWork; + yield* ambient.configure({ ...config, allowUnknownQuota: false }, { source: "client" }); + yield* TestClock.adjust(60_000); + yield* ambient.drain; + expect((yield* ambient.get({ callerThreadId: owner }))?.reason).toContain("quota"); + yield* ambient.configure(config, { source: "client" }); + test.setForeground(1); + yield* ambient.drain; + test.setForeground(0); + yield* TestClock.adjust(59_999); + yield* ambient.drain; + expect((yield* scheduler.list({ callerThreadId: owner })).length).toBe(0); + yield* TestClock.adjust(1); + yield* ambient.drain; + expect((yield* scheduler.list({ callerThreadId: owner })).length).toBe(1); + const sql = yield* SqlClient.SqlClient; + expect( + (yield* sql<{ count: number }>`SELECT count(*) AS count FROM ambient_work`)[0]?.count, + ).toBe(1); + }).pipe(Effect.provide(test.layer)); + }, +); diff --git a/apps/server/src/orchestration/AmbientWork.ts b/apps/server/src/orchestration/AmbientWork.ts new file mode 100644 index 000000000000..371d20323fd0 --- /dev/null +++ b/apps/server/src/orchestration/AmbientWork.ts @@ -0,0 +1,288 @@ +import { type ThreadId } from "@t3tools/contracts"; +import { + AmbientWorkConfigureInput, + AmbientWorkRecord, + ScheduledWorkError, +} from "../../../../packages/contracts/src/scheduledWork.ts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Clock from "effect/Clock"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; +import * as Queue from "effect/Queue"; +import * as Stream from "effect/Stream"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as Option from "effect/Option"; +import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; +import * as ScheduledWork from "./ScheduledWork.ts"; +import * as ProjectionSnapshotQuery from "./Services/ProjectionSnapshotQuery.ts"; +import * as OrchestrationEngine from "./Services/OrchestrationEngine.ts"; + +export class AmbientWork extends Context.Service< + AmbientWork, + { + readonly configure: ( + input: AmbientWorkConfigureInput, + approval: { source: "client" }, + ) => Effect.Effect; + readonly get: (input: { + callerThreadId: ThreadId; + }) => Effect.Effect; + readonly stop: (input: { + callerThreadId: ThreadId; + }) => Effect.Effect; + readonly drain: Effect.Effect; + readonly start: Effect.Effect; + } +>()("t3/orchestration/AmbientWork") {} +const fail = (code: ScheduledWorkError["code"], detail: string) => + new ScheduledWorkError({ code, detail }); +const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const policy = yield* BackgroundPolicy.BackgroundPolicy; + const scheduled = yield* ScheduledWork.ScheduledWork; + const activation = yield* ScheduledWork.ScheduledWorkActivation; + const snapshots = yield* ProjectionSnapshotQuery.ProjectionSnapshotQuery; + const engine = yield* OrchestrationEngine.OrchestrationEngineService; + const lock = yield* Semaphore.make(1); + const wake = yield* Queue.sliding(1); + let started = false; + const wrap = (effect: Effect.Effect) => + effect.pipe( + Effect.mapError((cause) => + Schema.is(ScheduledWorkError)(cause) + ? cause + : new ScheduledWorkError({ + code: "internal", + detail: "Ambient work operation failed.", + cause, + }), + ), + ); + const save = (record: AmbientWorkRecord) => + wrap( + sql`INSERT INTO ambient_work(owner_thread_id,document_json) VALUES(${record.config.callerThreadId},${JSON.stringify(record)}) ON CONFLICT(owner_thread_id) DO UPDATE SET document_json=excluded.document_json`, + ).pipe(Effect.asVoid); + const get = (input: { callerThreadId: ThreadId }) => + wrap( + Effect.gen(function* () { + const rows = yield* sql<{ + document_json: string; + }>`SELECT document_json FROM ambient_work WHERE owner_thread_id = ${input.callerThreadId}`; + return rows[0] + ? yield* Schema.decodeUnknownEffect(AmbientWorkRecord)(JSON.parse(rows[0].document_json)) + : null; + }), + ); + const stop = (input: { callerThreadId: ThreadId }) => + lock.withPermit( + Effect.gen(function* () { + const previous = yield* get(input); + if (!previous) return null; + const next = { + ...previous, + config: { ...previous.config, enabled: false }, + reason: "Disabled by owner.", + }; + yield* save(next); + if (next.activeScheduleId) + yield* scheduled.cancel({ + callerThreadId: input.callerThreadId, + id: next.activeScheduleId, + }); + yield* Queue.offer(wake, undefined); + return next; + }), + ); + const configure = (raw: AmbientWorkConfigureInput, approval: { source: "client" }) => + lock.withPermit( + Effect.gen(function* () { + if (approval?.source !== "client") + return yield* fail("forbidden", "Ambient admission requires explicit client consent."); + const config = yield* Schema.decodeUnknownEffect(AmbientWorkConfigureInput)(raw).pipe( + Effect.mapError(() => fail("invalid", "Invalid ambient configuration.")), + ); + yield* Effect.try({ + try: () => + new Intl.DateTimeFormat("en-CA", { timeZone: config.timezone }).format(new Date()), + catch: () => fail("invalid", "Unknown ambient timezone."), + }); + if (config.allowedHourStart >= config.allowedHourEnd) + return yield* fail("invalid", "Allowed hours must be one nonempty daytime interval."); + if (config.enabled) yield* activation.authorize(config.callerThreadId, config.grantId); + const previous = yield* get(config); + const next: AmbientWorkRecord = previous + ? { ...previous, config } + : { + config, + lastInteractionAt: yield* Clock.currentTimeMillis, + lastCycleAt: null, + day: "", + cycles: 0, + activeScheduleId: null, + reason: "Waiting for idle admission.", + }; + yield* save(next); + if (!config.enabled && next.activeScheduleId) + yield* scheduled.cancel({ + callerThreadId: config.callerThreadId, + id: next.activeScheduleId, + }); + yield* Queue.offer(wake, undefined); + return next; + }), + ); + const drain = lock.withPermit( + wrap( + Effect.gen(function* () { + const rows = yield* sql<{ + document_json: string; + }>`SELECT document_json FROM ambient_work ORDER BY owner_thread_id`; + const host = yield* policy.snapshot; + const now = yield* Clock.currentTimeMillis; + for (const row of rows) { + let record = yield* Schema.decodeUnknownEffect(AmbientWorkRecord)( + JSON.parse(row.document_json), + ); + const config = record.config; + if (!config.enabled) continue; + const date = new Date(now); + const parts = new Intl.DateTimeFormat("en-CA", { + timeZone: config.timezone, + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + hourCycle: "h23", + }).formatToParts(date); + const value = (type: Intl.DateTimeFormatPartTypes) => + parts.find((part) => part.type === type)?.value ?? ""; + const day = `${value("year")}-${value("month")}-${value("day")}`; + if (record.day !== day) record = { ...record, day, cycles: 0 }; + if (host.activeForegroundLeaseCount > 0) record = { ...record, lastInteractionAt: now }; + let reason: string | null = null; + if (record.activeScheduleId) { + const previous = yield* scheduled.get({ + callerThreadId: config.callerThreadId, + id: record.activeScheduleId, + }); + if (!["completed", "failed", "interrupted", "cancelled"].includes(previous.state)) + reason = "An ambient cycle is already active."; + else record = { ...record, activeScheduleId: null }; + } + if (!reason && host.activeForegroundLeaseCount > 0) + reason = "Foreground interaction is active."; + if ( + !reason && + (host.hostPower.stale || + host.hostPower.source === "unknown" || + host.hostPower.idle === "unknown") + ) + reason = "Host idle telemetry is unavailable."; + if ( + !reason && + (host.hostPower.suspended || + host.hostPower.lowPowerMode === "true" || + ["serious", "critical"].includes(host.hostPower.thermalState)) + ) + reason = "Host power state does not permit ambient work."; + if ( + !reason && + (host.hostPower.idle !== "true" || + (host.hostPower.idleSeconds ?? 0) * 1000 < config.idleDelayMs || + now - record.lastInteractionAt < config.idleDelayMs) + ) + reason = "Waiting for idle delay."; + if (!reason && !config.allowUnknownQuota) + reason = + "Provider quota telemetry is unavailable. Explicitly allow unknown quota to proceed."; + const hour = Number(value("hour")); + if (!reason && (hour < config.allowedHourStart || hour >= config.allowedHourEnd)) + reason = "Outside configured hours."; + if (!reason && record.cycles >= config.maxCyclesPerDay) + reason = "Daily cycle limit reached."; + if ( + !reason && + record.lastCycleAt !== null && + now - record.lastCycleAt < config.minimumCycleMs + ) + reason = "Waiting for cycle interval."; + const owner = yield* snapshots.getWorkerState(config.callerThreadId); + if (!reason && Option.isNone(owner)) reason = "Ambient owner was removed."; + if (!reason && Option.isSome(owner)) { + const shell = yield* snapshots.getShellSnapshot(); + if ( + shell.threads.some( + (thread) => + thread.projectId === owner.value.thread.projectId && + (thread.latestTurn?.state === "running" || + thread.session?.status === "starting" || + thread.session?.activeTurnId != null || + thread.hasPendingApprovals || + thread.hasPendingUserInput || + thread.backgroundLiveness != null), + ) + ) + reason = "Project execution or user input is active."; + } + const grant = yield* activation + .authorize(config.callerThreadId, config.grantId) + .pipe(Effect.result); + if (!reason && grant._tag === "Failure") reason = grant.failure.detail; + if (reason) { + yield* save({ ...record, reason }); + continue; + } + const id = `ambient:${config.callerThreadId}:${day}:${record.cycles + 1}`; + // Deterministic cycle IDs close crash-after-create-before-cadence-save window. + yield* scheduled.create({ + id, + callerThreadId: config.callerThreadId, + target: { type: "ambient", threadId: config.callerThreadId }, + prompt: config.prompt, + delayMs: 0, + onBusy: "wait", + grantId: config.grantId, + }); + yield* save({ + ...record, + activeScheduleId: id, + lastCycleAt: now, + cycles: record.cycles + 1, + reason: null, + }); + } + }), + ), + ); + const start = Effect.gen(function* () { + if (started) return; + started = true; + yield* Effect.addFinalizer(() => + Effect.sync(() => { + started = false; + }), + ); + const events = yield* engine.subscribeDomainEvents; + const changes = yield* policy.subscribe; + yield* events.pipe( + Stream.runForEach(() => Queue.offer(wake, undefined)), + Effect.forkScoped, + ); + yield* changes.changes.pipe( + Stream.runForEach(() => Queue.offer(wake, undefined)), + Effect.forkScoped, + ); + yield* drain; + yield* Effect.gen(function* () { + while (true) { + yield* Effect.raceFirst(Queue.take(wake), Effect.sleep(60_000)); + yield* drain.pipe(Effect.catch((error) => Effect.logError(error))); + } + }).pipe(Effect.forkScoped); + }); + return AmbientWork.of({ configure, get, stop, drain, start }); +}); +export const layer = Layer.effect(AmbientWork, make); diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.ts index 1f5aa1f328f5..d0c382b54f3d 100644 --- a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts +++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.ts @@ -477,10 +477,11 @@ const makeOrchestrationEngine = Effect.gen(function* () { }) : undefined; if ( - (currentCommand.type === "thread.turn.interrupt" || + (currentCommand.type === "thread.turn.start" && currentCommand.expectedIdle) || + ((currentCommand.type === "thread.turn.interrupt" || currentCommand.type === "thread.session.stop") && - (currentCommand.expectedMessageId !== undefined || - currentCommand.expectedTurnId !== undefined) + (currentCommand.expectedMessageId !== undefined || + currentCommand.expectedTurnId !== undefined)) ) { const guarded = yield* projectionSnapshotQuery.getWorkerState(currentCommand.threadId); if (Option.isNone(guarded)) @@ -489,26 +490,43 @@ const makeOrchestrationEngine = Effect.gen(function* () { detail: "Guarded target is unavailable.", }); const { thread, pendingMessageId } = guarded.value; - const currentRequests = - currentCommand.expectedMessageId === undefined - ? [] - : yield* sql<{ - pending_message_id: string; - }>`SELECT pending_message_id FROM projection_turns WHERE thread_id = ${thread.id} AND turn_id = ${thread.latestTurn?.turnId ?? ""} LIMIT 1`; - // A queued activation supersedes the terminal turn's message identity. - const messageMatches = - currentCommand.expectedMessageId === undefined || - (pendingMessageId !== null - ? pendingMessageId === currentCommand.expectedMessageId - : currentRequests[0]?.pending_message_id === currentCommand.expectedMessageId); - const turnMatches = - currentCommand.expectedTurnId === undefined || - thread.latestTurn?.turnId === currentCommand.expectedTurnId; - if (!messageMatches || !turnMatches) - return yield* new CoordinationError({ - code: "conflict", - detail: "Guarded cancellation no longer targets the current assignment.", - }); + if (currentCommand.type === "thread.turn.start") { + const busy = + pendingMessageId !== null || + thread.latestTurn?.state === "running" || + thread.session?.status === "starting" || + thread.session?.activeTurnId != null || + thread.hasPendingApprovals || + thread.hasPendingUserInput || + thread.backgroundLiveness != null || + threadBackgroundLiveness.getThreadBackgroundLiveness(thread.id) !== null; + if (busy) + return yield* new CoordinationError({ + code: "busy", + detail: "Guarded start requires an idle target.", + }); + } else { + const currentRequests = + currentCommand.expectedMessageId === undefined + ? [] + : yield* sql<{ + pending_message_id: string; + }>`SELECT pending_message_id FROM projection_turns WHERE thread_id = ${thread.id} AND turn_id = ${thread.latestTurn?.turnId ?? ""} LIMIT 1`; + // A queued activation supersedes the terminal turn's message identity. + const messageMatches = + currentCommand.expectedMessageId === undefined || + (pendingMessageId !== null + ? pendingMessageId === currentCommand.expectedMessageId + : currentRequests[0]?.pending_message_id === currentCommand.expectedMessageId); + const turnMatches = + currentCommand.expectedTurnId === undefined || + thread.latestTurn?.turnId === currentCommand.expectedTurnId; + if (!messageMatches || !turnMatches) + return yield* new CoordinationError({ + code: "conflict", + detail: "Guarded cancellation no longer targets the current assignment.", + }); + } } const coordinationPlan = currentCommand.type === "coordination.plan.write" || diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts index 275b1de966d1..b4c8e768909a 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts @@ -691,6 +691,7 @@ describe("ProviderCommandReactor", () => { message: { messageId, role: "user", text: "Scheduled work", attachments: [] }, runtimeMode: "approval-required", interactionMode: "default", + expectedIdle: true, createdAt: now, }); yield* harness.engine.dispatch({ @@ -724,6 +725,7 @@ describe("ProviderCommandReactor", () => { message: { messageId, role: "user", text: "Scheduled work", attachments: [] }, runtimeMode: "approval-required", interactionMode: "default", + expectedIdle: true, createdAt: now, }); yield* harness.engine.dispatch({ diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts index 598708cc33d6..245fd125e6b1 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts @@ -767,7 +767,8 @@ const make = Effect.gen(function* () { Effect.gen(function* () { if ( options?.activationMessageId && - (yield* activationFence.cancelled(threadId, options.activationMessageId)) + ((yield* activationFence.cancelled(threadId, options.activationMessageId)) || + !(yield* activationFence.authorized(threadId, options.activationMessageId))) ) return yield* Effect.interrupt; return yield* providerService @@ -1269,7 +1270,11 @@ const make = Effect.gen(function* () { receivedEvent.commandId !== null ? resumedTurnStarts.get(receivedEvent.commandId) : undefined; const event = resumed ? { ...receivedEvent, payload: resumed.event.payload } : receivedEvent; const key = turnStartKeyForEvent(event); - if (yield* activationFence.cancelled(event.payload.threadId, event.payload.messageId)) return; + if ( + (yield* activationFence.cancelled(event.payload.threadId, event.payload.messageId)) || + !(yield* activationFence.authorized(event.payload.threadId, event.payload.messageId)) + ) + return; if (yield* hasHandledTurnStartRecently(key)) { return; } @@ -1581,7 +1586,10 @@ const make = Effect.gen(function* () { const send = withActivationOrigin( event, Effect.gen(function* () { - if (yield* activationFence.cancelled(event.payload.threadId, event.payload.messageId)) + if ( + (yield* activationFence.cancelled(event.payload.threadId, event.payload.messageId)) || + !(yield* activationFence.authorized(event.payload.threadId, event.payload.messageId)) + ) return; const current = yield* resolveThreadShell(event.payload.threadId); if ( diff --git a/apps/server/src/orchestration/OwnedWorkers.ts b/apps/server/src/orchestration/OwnedWorkers.ts index 45d63c4c405a..d534751cb541 100644 --- a/apps/server/src/orchestration/OwnedWorkers.ts +++ b/apps/server/src/orchestration/OwnedWorkers.ts @@ -2,6 +2,7 @@ import { type ThreadUnattendedAuthority, isWorkerRuntimeModeAllowed, CoordinationError, + type RuntimeMode, ThreadId, WorkerOperationError, WorkerSpawnInput, @@ -40,6 +41,7 @@ import * as ProjectionSnapshotQuery from "./Services/ProjectionSnapshotQuery.ts" export interface WorkerSpawnAuthority { readonly unattendedAuthority?: ThreadUnattendedAuthority; + readonly runtimeModeCeiling?: RuntimeMode; readonly mcpCapabilityCeiling?: ReadonlyArray< NonNullable["mcpCapabilityCeiling"][number] >; @@ -275,11 +277,12 @@ const make = Effect.gen(function* () { snapshots.getThreadActivationAuthority(input.callerThreadId), ); const unattendedAuthority = authority?.unattendedAuthority ?? Option.getOrUndefined(inherited); + const requestedMode = authority?.runtimeModeCeiling ?? caller.runtimeMode; const runtimeModeCeiling = unattendedAuthority - ? isWorkerRuntimeModeAllowed(caller.runtimeMode, unattendedAuthority.runtimeModeCeiling) - ? caller.runtimeMode + ? isWorkerRuntimeModeAllowed(requestedMode, unattendedAuthority.runtimeModeCeiling) + ? requestedMode : unattendedAuthority.runtimeModeCeiling - : undefined; + : authority?.runtimeModeCeiling; let mcpCapabilityCeiling = [...((yield* threadCapabilities(input.callerThreadId)) ?? [])]; if (unattendedAuthority) mcpCapabilityCeiling = mcpCapabilityCeiling.filter((capability) => diff --git a/apps/server/src/orchestration/ScheduledWork.test.ts b/apps/server/src/orchestration/ScheduledWork.test.ts new file mode 100644 index 000000000000..284570744699 --- /dev/null +++ b/apps/server/src/orchestration/ScheduledWork.test.ts @@ -0,0 +1,674 @@ +import { + captureNativeUnattendedActivation, + validateNativeUnattendedAuthority, +} from "./nativeUnattendedAuthority.ts"; +import { + CommandId, + MessageId, + ProjectId, + ProviderInstanceId, + ThreadId, + type ScheduledWorkCreateInput, +} from "@t3tools/contracts"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as Stream from "effect/Stream"; +import * as Option from "effect/Option"; +import * as FileSystem from "effect/FileSystem"; +import { TestClock } from "effect/testing"; +import { ServerConfig } from "../config.ts"; +import { OrchestrationCommandReceiptRepositoryLive } from "../persistence/Layers/OrchestrationCommandReceipts.ts"; +import { OrchestrationEventStoreLive } from "../persistence/Layers/OrchestrationEventStore.ts"; +import { + SqlitePersistenceMemory, + makeSqlitePersistenceLive, +} from "../persistence/Layers/Sqlite.ts"; +import * as RepositoryIdentityResolver from "../project/RepositoryIdentityResolver.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import { OrchestrationEngineLive } from "./Layers/OrchestrationEngine.ts"; +import { OrchestrationProjectionPipelineLive } from "./Layers/ProjectionPipeline.ts"; +import { OrchestrationProjectionSnapshotQueryLive } from "./Layers/ProjectionSnapshotQuery.ts"; +import * as OwnedWorkers from "./OwnedWorkers.ts"; +import * as ScheduledWork from "./ScheduledWork.ts"; +import * as Activation from "./ScheduledWorkActivation.ts"; +import * as UnattendedGrants from "./UnattendedGrants.ts"; +import { OrchestrationEngineService } from "./Services/OrchestrationEngine.ts"; +import { ProjectionSnapshotQuery } from "./Services/ProjectionSnapshotQuery.ts"; +import * as ThreadBackgroundLiveness from "./ThreadBackgroundLiveness.ts"; +import * as ThreadPlanProgress from "./ThreadPlanProgress.ts"; + +import { EnvironmentId } from "@t3tools/contracts"; +import * as McpInvocationContext from "../mcp/McpInvocationContext.ts"; +import * as BackgroundJobs from "../background/BackgroundJobs.ts"; +import * as JobAuthority from "../background/BackgroundJobAuthority.ts"; +import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; +import { AutomationToolkit } from "../mcp/toolkits/automation/tools.ts"; +import { AutomationToolkitHandlersLive } from "../mcp/toolkits/automation/handlers.ts"; + +const NOW = "2026-10-03T00:00:00.000Z"; +const ROOT = ThreadId.make("schedule-root"); +const PROJECT = ProjectId.make("schedule-project"); +const MODEL = { instanceId: ProviderInstanceId.make("codex"), model: "test" }; +function testLayer(databasePath?: string) { + const core = Layer.mergeAll( + OrchestrationEngineLive.pipe( + Layer.provide(OrchestrationProjectionSnapshotQueryLive), + Layer.provide(OrchestrationProjectionPipelineLive), + ), + OrchestrationProjectionSnapshotQueryLive, + ).pipe( + Layer.provideMerge(ThreadBackgroundLiveness.layer), + Layer.provide(ThreadPlanProgress.layer), + Layer.provide(OrchestrationEventStoreLive), + Layer.provideMerge(OrchestrationCommandReceiptRepositoryLive), + Layer.provide( + Layer.succeed(RepositoryIdentityResolver.RepositoryIdentityResolver, { + resolve: () => Effect.succeed(null), + }), + ), + Layer.provideMerge( + databasePath ? makeSqlitePersistenceLive(databasePath) : SqlitePersistenceMemory, + ), + Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "t3-scheduler-test-" })), + Layer.provideMerge(NodeServices.layer), + Layer.provideMerge( + ServerSettings.layerTest({ + enableAgentBrowserAccess: true, + agentToolCapabilities: ["automation", "background-jobs"], + }), + ), + ); + const authority = Layer.mergeAll(OwnedWorkers.layer, UnattendedGrants.layer).pipe( + Layer.provideMerge(core), + ); + const activation = Activation.layer.pipe(Layer.provideMerge(authority)); + return ScheduledWork.layer.pipe(Layer.provideMerge(activation)); +} +const setup = Effect.gen(function* () { + const engine = yield* OrchestrationEngineService; + yield* engine.dispatch({ + type: "project.create", + commandId: CommandId.make("project"), + projectId: PROJECT, + title: "Scheduler", + workspaceRoot: process.cwd(), + createdAt: NOW, + }); + yield* engine.dispatch({ + type: "thread.create", + commandId: CommandId.make("root"), + threadId: ROOT, + projectId: PROJECT, + title: "Root", + modelSelection: MODEL, + runtimeMode: "approval-required", + interactionMode: "default", + branch: "main", + worktreePath: null, + createdAt: NOW, + }); + const grants = yield* UnattendedGrants.UnattendedGrants; + yield* grants.create( + { + id: "consent", + callerThreadId: ROOT, + ceiling: { runtimeMode: "approval-required", mcpCapabilities: ["workers"] }, + hostJobs: false, + }, + { source: "client" }, + ); +}); +const request = (id: string): ScheduledWorkCreateInput => ({ + id, + callerThreadId: ROOT, + target: { type: "resume", threadId: ROOT }, + prompt: "Scheduled test", + delayMs: 1000, + onBusy: "wait", + grantId: "consent", +}); +const foreground = (id: string) => + Effect.gen(function* () { + const engine = yield* OrchestrationEngineService; + yield* engine.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make(id), + threadId: ROOT, + message: { messageId: MessageId.make(id), role: "user", text: "Foreground", attachments: [] }, + runtimeMode: "approval-required", + interactionMode: "default", + createdAt: NOW, + }); + }); + +it.effect( + "retained unattended MCP grant cannot borrow broader scheduling or host-job consent", + () => + Effect.scoped( + Effect.gen(function* () { + yield* setup; + const engine = yield* OrchestrationEngineService; + const grants = yield* UnattendedGrants.UnattendedGrants; + yield* engine.dispatch({ + type: "thread.runtime-mode.set", + commandId: CommandId.make("nested-full-shell"), + threadId: ROOT, + runtimeMode: "full-access", + createdAt: NOW, + }); + yield* grants.create( + { + id: "G", + callerThreadId: ROOT, + ceiling: { + runtimeMode: "approval-required", + mcpCapabilities: ["workers", "automation", "background-jobs"], + }, + hostJobs: false, + }, + { source: "client" }, + ); + yield* grants.create( + { + id: "H", + callerThreadId: ROOT, + ceiling: { + runtimeMode: "full-access", + mcpCapabilities: ["workers", "automation", "background-jobs"], + }, + hostJobs: true, + }, + { source: "client" }, + ); + const context: McpInvocationContext.McpInvocationScope = { + environmentId: EnvironmentId.make("test"), + threadId: ROOT, + providerSessionId: "retained-G", + providerInstanceId: MODEL.instanceId, + capabilities: new Set(["workers", "automation", "background-jobs"]), + issuedAt: 1, + unattendedAuthority: { + grantId: "G", + grantRevision: 1, + ownerThreadId: ROOT, + runtimeModeCeiling: "approval-required", + mcpCapabilityCeiling: ["workers", "automation", "background-jobs"], + }, + }; + yield* Effect.gen(function* () { + const toolkit = yield* AutomationToolkit.pipe( + Effect.provide(AutomationToolkitHandlersLive), + ); + const nested = { ...request("cross-grant"), grantId: "H" }; + expect( + (yield* toolkit + .handle("schedule_create", nested) + .pipe(Stream.unwrap, Stream.runDrain, Effect.result))._tag, + ).toBe("Failure"); + yield* toolkit + .handle("schedule_create", { ...request("same-grant"), grantId: "G" }) + .pipe(Stream.unwrap, Stream.runDrain); + const schedules = yield* ScheduledWork.ScheduledWork; + expect( + (yield* schedules.get({ callerThreadId: ROOT, id: "same-grant" })).ceiling.runtimeMode, + ).toBe("approval-required"); + expect( + (yield* toolkit + .handle("background_job_start", { + id: "cross-host", + command: process.execPath, + args: ["-e", "process.exit(0)"], + timeoutMs: 1000, + maxOutputBytes: 1024, + }) + .pipe(Stream.unwrap, Stream.runDrain, Effect.result))._tag, + ).toBe("Failure"); + yield* grants.revoke({ callerThreadId: ROOT, id: "G" }, { source: "client" }); + expect( + (yield* toolkit + .handle("schedule_create", { ...request("revoked-nested"), grantId: "G" }) + .pipe(Stream.unwrap, Stream.runDrain, Effect.result))._tag, + ).toBe("Failure"); + }).pipe(Effect.provideService(McpInvocationContext.McpInvocationContext, context)); + }).pipe( + Effect.provide( + BackgroundJobs.layer.pipe( + Layer.provideMerge(JobAuthority.layer.pipe(Layer.provideMerge(WorkspacePaths.layer))), + Layer.provideMerge(testLayer()), + ), + ), + ), + ), +); + +it.effect( + "automatically dispatches once at TestClock due time, keeps deterministic IDs and recovers accepted receipt", + () => + Effect.scoped( + Effect.gen(function* () { + yield* setup; + const scheduled = yield* ScheduledWork.ScheduledWork; + const engine = yield* OrchestrationEngineService; + const sql = yield* SqlClient.SqlClient; + const created = yield* scheduled.create(request("once")); + expect(created.dueAt).toBe("1970-01-01T00:00:01.000Z"); + const events = yield* engine.subscribeDomainEvents; + yield* scheduled.start; + yield* TestClock.adjust(999); + expect((yield* scheduled.get({ callerThreadId: ROOT, id: "once" })).state).toBe("queued"); + yield* TestClock.adjust(1); + yield* events.pipe( + Stream.filter((event) => event.type === "thread.turn-start-requested"), + Stream.runHead, + ); + yield* scheduled.drainDue; + const accepted = yield* scheduled.get({ callerThreadId: ROOT, id: "once" }); + expect(accepted.state).toBe("accepted"); + expect(accepted.commandId).toBe("scheduled:once"); + const crashed = { + ...accepted, + state: "dispatching", + acceptedSequence: null, + executionThreadId: null, + }; + yield* sql`UPDATE scheduled_work SET state = 'dispatching', document_json = ${JSON.stringify(crashed)} WHERE id = 'once'`; + yield* scheduled.reconcile; + expect((yield* scheduled.get({ callerThreadId: ROOT, id: "once" })).acceptedSequence).toBe( + accepted.acceptedSequence, + ); + const messages = yield* sql<{ + count: number; + }>`SELECT count(*) AS count FROM projection_thread_messages WHERE message_id = ${accepted.messageId}`; + expect(messages[0]?.count).toBe(1); + }).pipe(Effect.provide(testLayer())), + ), +); + +it.effect( + "foreground admission wins atomically, busy waits without rejected receipt and exact cancellation cannot stop later work", + () => + Effect.scoped( + Effect.gen(function* () { + yield* setup; + const scheduled = yield* ScheduledWork.ScheduledWork; + const engine = yield* OrchestrationEngineService; + const snapshots = yield* ProjectionSnapshotQuery; + yield* scheduled.create({ ...request("busy"), delayMs: 0 }); + yield* foreground("human-first"); + yield* scheduled.drainDue; + expect((yield* scheduled.get({ callerThreadId: ROOT, id: "busy" })).state).toBe("blocked"); + const sql = yield* SqlClient.SqlClient; + const receipts = yield* sql<{ + count: number; + }>`SELECT count(*) AS count FROM orchestration_command_receipts WHERE command_id = 'scheduled:busy'`; + expect(receipts[0]?.count).toBe(0); + yield* scheduled.cancel({ callerThreadId: ROOT, id: "busy" }); + const loaded = yield* snapshots.getWorkerState(ROOT); + expect(Option.isSome(loaded) && loaded.value.pendingMessageId).toBe("human-first"); + // Wrong activation cannot cancel the foreground message, even in serialized dispatch. + const denied = yield* engine + .dispatch({ + type: "thread.turn.interrupt", + commandId: CommandId.make("wrong-cancel"), + threadId: ROOT, + expectedMessageId: MessageId.make("scheduled:busy"), + createdAt: NOW, + }) + .pipe(Effect.result); + expect(denied._tag).toBe("Failure"); + const after = yield* snapshots.getWorkerState(ROOT); + expect(Option.isSome(after) && after.value.pendingMessageId).toBe("human-first"); + }).pipe(Effect.provide(testLayer())), + ), +); + +it.effect( + "revoked consent, expired deadlines, foreign ownership and busy-fail are visible terminal or blocked states", + () => + Effect.scoped( + Effect.gen(function* () { + yield* setup; + const scheduled = yield* ScheduledWork.ScheduledWork; + const grants = yield* UnattendedGrants.UnattendedGrants; + yield* scheduled.create({ + ...request("expired"), + latestStartAt: "1970-01-01T00:00:02.000Z", + }); + yield* TestClock.adjust(3000); + yield* scheduled.drainDue; + expect((yield* scheduled.get({ callerThreadId: ROOT, id: "expired" })).state).toBe( + "failed", + ); + yield* scheduled.create({ ...request("revoked"), delayMs: 0 }); + yield* grants.revoke({ callerThreadId: ROOT, id: "consent" }, { source: "client" }); + yield* scheduled.drainDue; + expect((yield* scheduled.get({ callerThreadId: ROOT, id: "revoked" })).reason).toContain( + "revoked", + ); + const foreign = yield* scheduled + .get({ callerThreadId: ThreadId.make("foreign"), id: "revoked" }) + .pipe(Effect.result); + expect(foreign._tag).toBe("Failure"); + }).pipe(Effect.provide(testLayer())), + ), +); + +it.effect( + "scheduled spawn lowers inherited runtime mode and retries preserve worker identity", + () => + Effect.scoped( + Effect.gen(function* () { + yield* setup; + const engine = yield* OrchestrationEngineService; + const scheduled = yield* ScheduledWork.ScheduledWork; + const snapshots = yield* ProjectionSnapshotQuery; + yield* engine.dispatch({ + type: "thread.runtime-mode.set", + commandId: CommandId.make("full-parent"), + threadId: ROOT, + runtimeMode: "full-access", + createdAt: NOW, + }); + const created = yield* scheduled.create({ + ...request("spawn-ceiling"), + target: { type: "spawn", label: "bounded", modelSelection: MODEL }, + delayMs: 0, + }); + expect(created.ceiling.runtimeMode).toBe("approval-required"); + yield* scheduled.drainDue; + const accepted = yield* scheduled.get({ callerThreadId: ROOT, id: "spawn-ceiling" }); + expect(accepted.state).toBe("accepted"); + const worker = yield* snapshots.getWorkerState(accepted.executionThreadId!); + expect(Option.isSome(worker) && worker.value.thread.runtimeMode).toBe("approval-required"); + expect(Option.isSome(worker) && worker.value.pendingMessageId).toBe(created.messageId); + const origin = yield* captureNativeUnattendedActivation( + yield* SqlClient.SqlClient, + accepted.executionThreadId!, + ); + expect(origin?.authority.grantId).toBe("consent"); + const workers = yield* OwnedWorkers.OwnedWorkers; + const child = yield* workers.spawn({ + commandId: CommandId.make("grant-child"), + callerThreadId: accepted.executionThreadId!, + label: "child", + prompt: "bounded child", + modelSelection: MODEL, + }); + const childOrigin = yield* captureNativeUnattendedActivation( + yield* SqlClient.SqlClient, + child.workerThreadId, + ); + expect(childOrigin?.authority.grantId).toBe("consent"); + yield* engine.dispatch({ + type: "thread.turn.interrupt", + commandId: CommandId.make("child-settle"), + threadId: child.workerThreadId, + expectedMessageId: childOrigin!.messageId, + createdAt: NOW, + }); + yield* workers.send({ + commandId: CommandId.make("child-follow-up"), + callerThreadId: accepted.executionThreadId!, + workerThreadId: child.workerThreadId, + text: "still bounded", + }); + expect( + (yield* captureNativeUnattendedActivation( + yield* SqlClient.SqlClient, + child.workerThreadId, + ))?.authority.grantId, + ).toBe("consent"); + const grants = yield* UnattendedGrants.UnattendedGrants; + yield* grants.revoke({ callerThreadId: ROOT, id: "consent" }, { source: "client" }); + expect( + yield* validateNativeUnattendedAuthority( + yield* SqlClient.SqlClient, + child.workerThreadId, + "approval-required", + childOrigin, + ), + ).toBe(false); + yield* scheduled.drainDue; + expect( + (yield* scheduled.get({ callerThreadId: ROOT, id: "spawn-ceiling" })).executionThreadId, + ).toBe(accepted.executionThreadId); + }).pipe(Effect.provide(testLayer())), + ), +); + +it.effect("spawn busy-wait retries the same command after owned capacity is released", () => + Effect.scoped( + Effect.gen(function* () { + yield* setup; + const workers = yield* OwnedWorkers.OwnedWorkers; + const schedules = yield* ScheduledWork.ScheduledWork; + const sql = yield* SqlClient.SqlClient; + const occupied = []; + for (let i = 0; i < 4; i++) + occupied.push( + yield* workers.spawn({ + commandId: CommandId.make(`capacity:${i}`), + callerThreadId: ROOT, + label: `occupied ${i}`, + prompt: "wait", + modelSelection: MODEL, + }), + ); + yield* schedules.create({ + ...request("capacity-retry"), + target: { type: "spawn", label: "retry", modelSelection: MODEL }, + delayMs: 0, + }); + yield* schedules.drainDue; + expect((yield* schedules.get({ callerThreadId: ROOT, id: "capacity-retry" })).state).toBe( + "blocked", + ); + expect( + (yield* sql`SELECT command_id FROM orchestration_command_receipts WHERE command_id = 'scheduled:capacity-retry'`) + .length, + ).toBe(0); + yield* workers.stop({ + commandId: CommandId.make("release-capacity"), + callerThreadId: ROOT, + workerThreadId: occupied[0]!.workerThreadId, + }); + yield* schedules.drainDue; + expect((yield* schedules.get({ callerThreadId: ROOT, id: "capacity-retry" })).state).toBe( + "accepted", + ); + }).pipe(Effect.provide(testLayer())), + ), +); + +for (const primaryState of ["completed", "error", "interrupted"] as const) { + it.effect(`primary ${primaryState} remains cancellable until native background quiesces`, () => + Effect.scoped( + Effect.gen(function* () { + yield* setup; + const schedules = yield* ScheduledWork.ScheduledWork; + const sql = yield* SqlClient.SqlClient; + const liveness = yield* ThreadBackgroundLiveness.ThreadBackgroundLivenessService; + const id = `live-${primaryState}`; + yield* schedules.create({ ...request(id), delayMs: 0 }); + yield* schedules.drainDue; + const accepted = yield* schedules.get({ callerThreadId: ROOT, id }); + yield* sql`UPDATE projection_turns SET state = ${primaryState}, turn_id = ${`native:${id}`} WHERE pending_message_id = ${accepted.messageId}`; + yield* sql`UPDATE projection_threads SET latest_turn_id = ${`native:${id}`} WHERE thread_id = ${ROOT}`; + liveness.recordTaskLiveness({ + threadId: ROOT, + taskId: "owned-background", + taskType: "agent", + status: "running", + kind: "started", + }); + yield* schedules.reconcile; + expect((yield* schedules.get({ callerThreadId: ROOT, id })).state).toBe("running"); + yield* schedules.cancel({ callerThreadId: ROOT, id }); + expect( + (yield* sql`SELECT message_id FROM projection_turn_cancellations WHERE message_id = ${accepted.messageId}`) + .length, + ).toBe(1); + liveness.clearThreadLiveness(ROOT); + yield* schedules.reconcile; + expect(["completed", "failed", "cancelled"]).toContain( + (yield* schedules.get({ callerThreadId: ROOT, id })).state, + ); + }).pipe(Effect.provide(testLayer())), + ), + ); +} + +it.effect("exact retries preserve relative due time and reject changed delay or deadline", () => + Effect.scoped( + Effect.gen(function* () { + yield* setup; + const schedules = yield* ScheduledWork.ScheduledWork; + const original = yield* schedules.create(request("retry-bounds")); + yield* TestClock.adjust(50); + expect((yield* schedules.create(request("retry-bounds"))).dueAt).toBe(original.dueAt); + expect( + (yield* schedules.create({ ...request("retry-bounds"), delayMs: 2000 }).pipe(Effect.result)) + ._tag, + ).toBe("Failure"); + expect( + (yield* schedules + .create({ ...request("retry-bounds"), latestStartAt: "1970-01-01T00:00:03.000Z" }) + .pipe(Effect.result))._tag, + ).toBe("Failure"); + }).pipe(Effect.provide(testLayer())), + ), +); + +it.effect( + "persisted activation authority limits native mode, observes revocation and clears only on explicit normal start", + () => + Effect.scoped( + Effect.gen(function* () { + yield* setup; + const schedules = yield* ScheduledWork.ScheduledWork; + const snapshots = yield* ProjectionSnapshotQuery; + const sql = yield* SqlClient.SqlClient; + yield* schedules.create({ ...request("native-consent"), delayMs: 0 }); + yield* schedules.drainDue; + const current = yield* snapshots.getThreadActivationAuthority(ROOT); + expect(Option.isSome(current) && current.value.mcpCapabilityCeiling).toEqual(["workers"]); + expect(yield* validateNativeUnattendedAuthority(sql, ROOT, "approval-required")).toBe(true); + expect(yield* validateNativeUnattendedAuthority(sql, ROOT, "full-access")).toBe(false); + expect(yield* validateNativeUnattendedAuthority(sql, ROOT, "auto")).toBe(false); + expect(yield* validateNativeUnattendedAuthority(sql, ROOT, "auto-accept-edits")).toBe( + false, + ); + const captured = yield* captureNativeUnattendedActivation(sql, ROOT); + const cancelReceipt = yield* (yield* OrchestrationEngineService).dispatch({ + type: "thread.turn.interrupt", + commandId: CommandId.make("authority-cancel"), + threadId: ROOT, + expectedMessageId: captured!.messageId, + createdAt: NOW, + }); + const grants = yield* UnattendedGrants.UnattendedGrants; + yield* grants.revoke({ callerThreadId: ROOT, id: "consent" }, { source: "client" }); + expect(yield* validateNativeUnattendedAuthority(sql, ROOT, "approval-required")).toBe( + false, + ); + yield* foreground("explicit-normal-after-revoke"); + expect(Option.isNone(yield* snapshots.getThreadActivationAuthority(ROOT))).toBe(true); + const staleAck = yield* (yield* OrchestrationEngineService) + .dispatch({ + type: "thread.session.set", + commandId: CommandId.make("stale-stop-ack"), + threadId: ROOT, + expectedMessageId: captured!.messageId, + expectedActivationSequence: cancelReceipt.sequence, + session: { + threadId: ROOT, + status: "stopped", + providerName: "codex", + runtimeMode: "approval-required", + activeTurnId: null, + lastError: null, + updatedAt: NOW, + }, + createdAt: NOW, + }) + .pipe(Effect.result); + expect(staleAck._tag).toBe("Failure"); + const sequenceOnlyAck = yield* (yield* OrchestrationEngineService) + .dispatch({ + type: "thread.session.set", + commandId: CommandId.make("stale-turn-only-stop-ack"), + threadId: ROOT, + expectedActivationSequence: cancelReceipt.sequence, + session: { + threadId: ROOT, + status: "stopped", + providerName: "codex", + runtimeMode: "approval-required", + activeTurnId: null, + lastError: null, + updatedAt: NOW, + }, + createdAt: NOW, + }) + .pipe(Effect.result); + expect(sequenceOnlyAck._tag).toBe("Failure"); + expect( + yield* validateNativeUnattendedAuthority(sql, ROOT, "approval-required", captured), + ).toBe(false); + expect(yield* validateNativeUnattendedAuthority(sql, ROOT, "approval-required")).toBe(true); + }).pipe(Effect.provide(testLayer())), + ), +); + +it.effect( + "cancel before native activation settles from durable tombstone with no bound session", + () => + Effect.scoped( + Effect.gen(function* () { + yield* setup; + const schedules = yield* ScheduledWork.ScheduledWork; + yield* schedules.create({ ...request("cancel-no-session"), delayMs: 0 }); + yield* schedules.drainDue; + yield* schedules.cancel({ callerThreadId: ROOT, id: "cancel-no-session" }); + yield* schedules.reconcile; + expect( + (yield* schedules.get({ callerThreadId: ROOT, id: "cancel-no-session" })).state, + ).toBe("cancelled"); + }).pipe(Effect.provide(testLayer())), + ), +); + +it.effect("reopens real SQLite and dispatches persisted one-shot only once", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-scheduler-reopen-" }); + const filename = `${directory}/state.sqlite`; + yield* Effect.gen(function* () { + yield* setup; + const service = yield* ScheduledWork.ScheduledWork; + yield* service.create({ ...request("restart"), delayMs: 0 }); + }).pipe(Effect.provide(testLayer(filename))); + yield* Effect.gen(function* () { + const service = yield* ScheduledWork.ScheduledWork; + yield* service.drainDue; + expect((yield* service.get({ callerThreadId: ROOT, id: "restart" })).state).toBe( + "accepted", + ); + }).pipe(Effect.provide(testLayer(filename))); + yield* Effect.gen(function* () { + const service = yield* ScheduledWork.ScheduledWork; + yield* service.reconcile; + yield* service.drainDue; + const sql = yield* SqlClient.SqlClient; + const rows = yield* sql<{ + count: number; + }>`SELECT count(*) AS count FROM projection_thread_messages WHERE message_id = 'scheduled:restart'`; + expect(rows[0]?.count).toBe(1); + }).pipe(Effect.provide(testLayer(filename))); + }).pipe(Effect.provide(NodeServices.layer)), + ), +); diff --git a/apps/server/src/orchestration/ScheduledWork.ts b/apps/server/src/orchestration/ScheduledWork.ts new file mode 100644 index 000000000000..cead82f564de --- /dev/null +++ b/apps/server/src/orchestration/ScheduledWork.ts @@ -0,0 +1,390 @@ +import { CommandId, MessageId, type ProjectId, type ThreadId } from "@t3tools/contracts"; +import { + ScheduledWorkCreateInput, + ScheduledWorkReadInput, + ScheduledWorkListInput, + ScheduledWorkRecord, + ScheduledWorkError, + type UnattendedCeiling, +} from "../../../../packages/contracts/src/scheduledWork.ts"; +import * as Context from "effect/Context"; +import * as Option from "effect/Option"; +import * as McpInvocationContext from "../mcp/McpInvocationContext.ts"; +import { isWorkerRuntimeModeAllowed } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Clock from "effect/Clock"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; +import * as Queue from "effect/Queue"; +import * as Stream from "effect/Stream"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as OrchestrationEngine from "./Services/OrchestrationEngine.ts"; + +/** Implemented at the engine boundary, where idle and exact cancellation guards + * are serialized with foreground commands. Authority is never caller JSON. */ +export class ScheduledWorkActivation extends Context.Service< + ScheduledWorkActivation, + { + readonly authorize: ( + caller: ThreadId, + grantId: string, + ) => Effect.Effect< + { projectId: ProjectId; ceiling: UnattendedCeiling; grantRevision: number }, + ScheduledWorkError + >; + readonly validate: (record: ScheduledWorkRecord) => Effect.Effect; + readonly dispatch: ( + record: ScheduledWorkRecord, + ) => Effect.Effect<{ sequence: number; threadId: ThreadId }, ScheduledWorkError>; + readonly reconcile: (record: ScheduledWorkRecord) => Effect.Effect< + { + state: ScheduledWorkRecord["state"]; + sequence: number | null; + threadId: ThreadId | null; + reason: string | null; + } | null, + ScheduledWorkError + >; + readonly cancel: ( + record: ScheduledWorkRecord, + ) => Effect.Effect<"stopping" | "cancelled", ScheduledWorkError>; + } +>()("t3/orchestration/ScheduledWorkActivation") {} + +export class ScheduledWork extends Context.Service< + ScheduledWork, + { + readonly create: ( + input: ScheduledWorkCreateInput, + authority?: { readonly mcpCapabilityCeiling: UnattendedCeiling["mcpCapabilities"] }, + ) => Effect.Effect; + readonly list: ( + input: ScheduledWorkListInput, + ) => Effect.Effect, ScheduledWorkError>; + readonly get: ( + input: ScheduledWorkReadInput, + ) => Effect.Effect; + readonly cancel: ( + input: ScheduledWorkReadInput, + ) => Effect.Effect; + readonly drainDue: Effect.Effect; + readonly reconcile: Effect.Effect; + readonly start: Effect.Effect; + } +>()("t3/orchestration/ScheduledWork") {} + +const terminal = (state: ScheduledWorkRecord["state"]) => + ["completed", "failed", "interrupted", "cancelled"].includes(state); +const failure = (code: ScheduledWorkError["code"], detail: string, cause?: unknown) => + new ScheduledWorkError({ code, detail, ...(cause === undefined ? {} : { cause }) }); +const decodeScheduledWorkRecord = Schema.decodeUnknownEffect(ScheduledWorkRecord); +const decodeScheduledWorkReadInput = Schema.decodeUnknownEffect(ScheduledWorkReadInput); +const decodeScheduledWorkListInput = Schema.decodeUnknownEffect(ScheduledWorkListInput); +const decodeScheduledWorkCreateInput = Schema.decodeUnknownEffect(ScheduledWorkCreateInput); + +const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const activation = yield* ScheduledWorkActivation; + const engine = yield* OrchestrationEngine.OrchestrationEngineService; + const lock = yield* Semaphore.make(1); + const wake = yield* Queue.sliding(1); + let started = false; + const wrap = (effect: Effect.Effect) => + effect.pipe( + Effect.mapError((cause) => + Schema.is(ScheduledWorkError)(cause) + ? cause + : failure("internal", "Scheduled work persistence failed.", cause), + ), + ); + const read = (id: string) => + wrap( + Effect.gen(function* () { + const rows = yield* sql<{ + document_json: string; + }>`SELECT document_json FROM scheduled_work WHERE id = ${id}`; + if (!rows[0]) return null; + return yield* decodeScheduledWorkRecord(JSON.parse(rows[0].document_json)); + }), + ); + const save = (record: ScheduledWorkRecord) => + wrap( + sql`UPDATE scheduled_work SET state = ${record.state}, due_at = ${record.dueAt}, document_json = ${JSON.stringify(record)} WHERE id = ${record.id}`, + ).pipe(Effect.asVoid); + const update = (record: ScheduledWorkRecord, change: Partial) => + Effect.gen(function* () { + const next = { + ...record, + ...change, + updatedAt: new Date(yield* Clock.currentTimeMillis).toISOString(), + }; + yield* save(next); + return next; + }); + const get = (input: ScheduledWorkReadInput) => + Effect.gen(function* () { + const decoded = yield* decodeScheduledWorkReadInput(input).pipe( + Effect.mapError(() => failure("invalid", "Invalid scheduled work identifier.")), + ); + const record = yield* read(decoded.id); + if (!record) return yield* failure("not-found", "Scheduled work not found."); + if (record.ownerThreadId !== decoded.callerThreadId) + return yield* failure("forbidden", "Scheduled work belongs to another thread."); + return record; + }); + const list = (input: ScheduledWorkListInput) => + wrap( + Effect.gen(function* () { + const decoded = yield* decodeScheduledWorkListInput(input); + const rows = yield* sql<{ + document_json: string; + }>`SELECT document_json FROM scheduled_work WHERE owner_thread_id = ${decoded.callerThreadId} ORDER BY due_at, id LIMIT 200`; + return yield* Effect.forEach(rows, (row) => + decodeScheduledWorkRecord(JSON.parse(row.document_json)), + ); + }), + ); + const create = ( + raw: ScheduledWorkCreateInput, + callerAuthority?: { readonly mcpCapabilityCeiling: UnattendedCeiling["mcpCapabilities"] }, + ) => + lock.withPermit( + Effect.gen(function* () { + const input = yield* decodeScheduledWorkCreateInput(raw).pipe( + Effect.mapError(() => failure("invalid", "Invalid bounded schedule input.")), + ); + if ((input.dueAt === undefined) === (input.delayMs === undefined)) + return yield* failure("invalid", "Specify exactly one dueAt or delayMs."); + const invocation = yield* Effect.serviceOption(McpInvocationContext.McpInvocationContext); + const retained = Option.isSome(invocation) + ? invocation.value.unattendedAuthority + : undefined; + if ( + retained && + (input.grantId !== retained.grantId || input.callerThreadId !== retained.ownerThreadId) + ) + return yield* failure( + "forbidden", + "Nested schedules must retain their credential-bound unattended grant.", + ); + const authority = yield* activation.authorize(input.callerThreadId, input.grantId); + if (retained && retained.grantRevision !== authority.grantRevision) + return yield* failure( + "forbidden", + "Credential-bound unattended grant was revoked or changed.", + ); + const requestJson = JSON.stringify({ + target: input.target, + prompt: input.prompt, + dueAt: input.dueAt ?? null, + delayMs: input.delayMs ?? null, + latestStartAt: input.latestStartAt ?? null, + onBusy: input.onBusy, + grantId: input.grantId, + }); + const existing = yield* read(input.id); + if (existing) { + const requests = yield* wrap( + sql<{ + request_json: string; + }>`SELECT request_json FROM scheduled_work WHERE id = ${input.id}`, + ); + if (requests[0]?.request_json !== requestJson) + return yield* failure( + "conflict", + "Schedule identifier already has different timing or execution parameters.", + ); + if ( + existing.ownerThreadId !== input.callerThreadId || + existing.prompt !== input.prompt || + JSON.stringify(existing.target) !== JSON.stringify(input.target) || + existing.grantId !== input.grantId || + existing.onBusy !== input.onBusy || + (input.dueAt !== undefined && existing.dueAt !== input.dueAt) + ) + return yield* failure( + "conflict", + "Schedule identifier already has a different request.", + ); + return existing; + } + const now = yield* Clock.currentTimeMillis; + const dueAt = input.dueAt ?? new Date(now + input.delayMs!).toISOString(); + if (input.latestStartAt && Date.parse(input.latestStartAt) < Date.parse(dueAt)) + return yield* failure("invalid", "Latest start precedes due time."); + const record: ScheduledWorkRecord = { + id: input.id, + ownerThreadId: input.callerThreadId, + projectId: authority.projectId, + target: input.target, + prompt: input.prompt, + dueAt, + latestStartAt: input.latestStartAt ?? null, + onBusy: input.onBusy, + grantId: input.grantId, + grantRevision: authority.grantRevision, + ceiling: { + ...authority.ceiling, + runtimeMode: + retained && + !isWorkerRuntimeModeAllowed( + authority.ceiling.runtimeMode, + retained.runtimeModeCeiling, + ) + ? retained.runtimeModeCeiling + : authority.ceiling.runtimeMode, + mcpCapabilities: authority.ceiling.mcpCapabilities.filter( + (capability) => + (!retained || retained.mcpCapabilityCeiling.includes(capability)) && + (callerAuthority === undefined || + callerAuthority.mcpCapabilityCeiling.includes(capability)), + ), + }, + commandId: CommandId.make(`scheduled:${input.id}`), + messageId: MessageId.make( + input.target.type === "spawn" + ? `worker-message:scheduled:${input.id}` + : `scheduled:${input.id}`, + ), + state: "queued", + reason: null, + cancelRequested: false, + acceptedSequence: null, + executionThreadId: null, + createdAt: new Date(now).toISOString(), + updatedAt: new Date(now).toISOString(), + }; + yield* activation.validate(record); + yield* wrap( + sql`INSERT INTO scheduled_work (id, owner_thread_id, due_at, state, request_json, document_json) VALUES (${record.id}, ${record.ownerThreadId}, ${record.dueAt}, ${record.state}, ${requestJson}, ${JSON.stringify(record)})`, + ); + yield* Queue.offer(wake, undefined); + return record; + }), + ); + const reconcileOne = (record: ScheduledWorkRecord) => + Effect.gen(function* () { + const receipt = yield* activation.reconcile(record); + if (receipt) + return yield* update(record, { + state: receipt.state, + acceptedSequence: receipt.sequence, + executionThreadId: receipt.threadId, + reason: receipt.reason, + }); + return record; + }); + const allActive = wrap( + Effect.gen(function* () { + const rows = yield* sql<{ + document_json: string; + }>`SELECT document_json FROM scheduled_work WHERE state NOT IN ('completed','failed','interrupted','cancelled') ORDER BY due_at,id`; + return yield* Effect.forEach(rows, (row) => + decodeScheduledWorkRecord(JSON.parse(row.document_json)), + ); + }), + ); + const process = (original: ScheduledWorkRecord) => + Effect.gen(function* () { + let record = original; + if (["dispatching", "accepted", "running", "stopping"].includes(record.state)) + record = yield* reconcileOne(record); + if (terminal(record.state)) return; + if (record.cancelRequested) { + const state = + record.acceptedSequence !== null ? yield* activation.cancel(record) : "cancelled"; + yield* update(record, { state }); + return; + } + if (!["queued", "blocked", "dispatching"].includes(record.state)) return; + const now = yield* Clock.currentTimeMillis; + if (Date.parse(record.dueAt) > now) return; + if (record.latestStartAt && Date.parse(record.latestStartAt) < now) { + yield* update(record, { state: "failed", reason: "Latest start time expired." }); + return; + } + const validated = yield* activation.validate(record).pipe(Effect.result); + if (validated._tag === "Failure") { + yield* update(record, { state: "blocked", reason: validated.failure.detail }); + return; + } + record = yield* update(record, { state: "dispatching", reason: null }); + // Persisted deterministic IDs recover accepted-before-outbox-update crashes. + const dispatched = yield* activation.dispatch(record).pipe(Effect.result); + if (dispatched._tag === "Failure") { + const error = dispatched.failure; + yield* update(record, { + state: error.code === "busy" && record.onBusy === "wait" ? "blocked" : "failed", + reason: error.detail, + }); + return; + } + yield* update(record, { + state: "accepted", + acceptedSequence: dispatched.success.sequence, + executionThreadId: dispatched.success.threadId, + }); + }); + const drainDue = lock.withPermit( + Effect.gen(function* () { + for (const record of yield* allActive) yield* process(record); + }), + ); + const reconcile = lock.withPermit( + Effect.gen(function* () { + for (const record of yield* allActive) + if (["dispatching", "accepted", "running", "stopping"].includes(record.state)) + yield* reconcileOne(record); + }), + ); + const cancel = (input: ScheduledWorkReadInput) => + lock.withPermit( + Effect.gen(function* () { + let record = yield* get(input); + if (terminal(record.state)) return record; + record = yield* update(record, { cancelRequested: true }); + // Reconcile before cancel: acceptance can precede the durable outbox update. + if (record.state === "dispatching") record = yield* reconcileOne(record); + const state = + record.acceptedSequence !== null ? yield* activation.cancel(record) : "cancelled"; + const result = yield* update(record, { state }); + yield* Queue.offer(wake, undefined); + return result; + }), + ); + const start = Effect.gen(function* () { + if (started) return; + started = true; + yield* Effect.addFinalizer(() => + Effect.sync(() => { + started = false; + }), + ); + const events = yield* engine.subscribeDomainEvents; + yield* events.pipe( + Stream.runForEach(() => Queue.offer(wake, undefined)), + Effect.forkScoped, + ); + yield* reconcile; + yield* drainDue; + yield* Effect.gen(function* () { + while (true) { + const rows = yield* wrap( + sql<{ + due_at: string; + }>`SELECT due_at FROM scheduled_work WHERE state = 'queued' ORDER BY due_at LIMIT 1`, + ); + const delay = rows[0] + ? Math.max(0, Date.parse(rows[0].due_at) - (yield* Clock.currentTimeMillis)) + : null; + if (delay === null) yield* Queue.take(wake); + else yield* Effect.raceFirst(Queue.take(wake), Effect.sleep(delay)); + yield* drainDue.pipe(Effect.catch((error) => Effect.logError(error))); + } + }).pipe(Effect.forkScoped); + }); + return ScheduledWork.of({ create, list, get, cancel, drainDue, reconcile, start }); +}); +export const layer = Layer.effect(ScheduledWork, make); diff --git a/apps/server/src/orchestration/ScheduledWorkActivation.ts b/apps/server/src/orchestration/ScheduledWorkActivation.ts new file mode 100644 index 000000000000..034fe8df85b9 --- /dev/null +++ b/apps/server/src/orchestration/ScheduledWorkActivation.ts @@ -0,0 +1,316 @@ +import { + CommandId, + CoordinationError, + WorkerOperationError, + type ThreadId, +} from "@t3tools/contracts"; +import { + ScheduledWorkError, + type ScheduledWorkRecord, +} from "../../../../packages/contracts/src/scheduledWork.ts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as Option from "effect/Option"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as ProjectionSnapshotQuery from "./Services/ProjectionSnapshotQuery.ts"; +import * as OrchestrationEngine from "./Services/OrchestrationEngine.ts"; +import * as Receipts from "../persistence/Services/OrchestrationCommandReceipts.ts"; +import * as OwnedWorkers from "./OwnedWorkers.ts"; +import * as UnattendedGrants from "./UnattendedGrants.ts"; +import * as McpInvocationContext from "../mcp/McpInvocationContext.ts"; +import { ScheduledWorkActivation } from "./ScheduledWork.ts"; + +const fail = (code: ScheduledWorkError["code"], detail: string) => + new ScheduledWorkError({ code, detail }); +const make = Effect.gen(function* () { + const engine = yield* OrchestrationEngine.OrchestrationEngineService; + const sql = yield* SqlClient.SqlClient; + const snapshots = yield* ProjectionSnapshotQuery.ProjectionSnapshotQuery; + const receipts = yield* Receipts.OrchestrationCommandReceiptRepository; + const grants = yield* UnattendedGrants.UnattendedGrants; + const workers = yield* OwnedWorkers.OwnedWorkers; + const capabilities = yield* McpInvocationContext.makeThreadMcpCapabilities; + const wrap = (effect: Effect.Effect) => + effect.pipe( + Effect.mapError((cause) => + Schema.is(ScheduledWorkError)(cause) + ? cause + : Schema.is(WorkerOperationError)(cause) && + (cause.code === "busy" || cause.code === "limit") + ? fail("busy", cause.detail) + : Schema.is(CoordinationError)(cause) && cause.code === "busy" + ? fail("busy", cause.detail) + : new ScheduledWorkError({ + code: "internal", + detail: "Scheduled activation failed.", + cause, + }), + ), + ); + const read = (id: ThreadId) => + wrap( + Effect.gen(function* () { + const state = yield* snapshots.getWorkerState(id); + if (Option.isNone(state)) + return yield* fail("not-found", "Scheduled target thread is unavailable."); + const project = yield* snapshots.getProjectShellById(state.value.thread.projectId); + if (Option.isNone(project)) + return yield* fail("not-found", "Scheduled project is unavailable."); + return state.value; + }), + ); + const authorize = (caller: ThreadId, grantId: string) => + wrap( + Effect.gen(function* () { + const grant = yield* grants.get({ callerThreadId: caller, id: grantId }); + if (grant.revoked) + return yield* fail("forbidden", "Explicit unattended grant was revoked."); + const state = yield* read(caller); + if ( + grant.projectId !== state.thread.projectId || + state.thread.worker?.stopRequestedAt != null + ) + return yield* fail("forbidden", "Grant owner is no longer eligible."); + const current = { + runtimeMode: state.thread.runtimeMode, + mcpCapabilities: [...((yield* capabilities(caller)) ?? [])], + }; + return { + projectId: grant.projectId, + grantRevision: grant.revision, + ceiling: UnattendedGrants.intersectCeiling(grant.ceiling, current), + }; + }), + ); + const validate = (record: ScheduledWorkRecord) => + wrap( + Effect.gen(function* () { + const authority = yield* authorize(record.ownerThreadId, record.grantId); + if ( + authority.grantRevision !== record.grantRevision || + authority.projectId !== record.projectId + ) + return yield* fail("forbidden", "Scheduled grant revision changed."); + if (record.prompt.trimStart().startsWith("/")) + return yield* fail("invalid", "Provider-native commands cannot run unattended."); + if (record.target.type === "ambient") { + const ambient = yield* sql<{ + document_json: string; + }>`SELECT document_json FROM ambient_work WHERE owner_thread_id = ${record.ownerThreadId}`; + const configured: unknown = ambient[0] ? JSON.parse(ambient[0].document_json) : null; + const enabled = Schema.decodeUnknownOption( + Schema.Struct({ + config: Schema.Struct({ enabled: Schema.Boolean, grantId: Schema.String }), + }), + )(configured); + if ( + Option.isNone(enabled) || + !enabled.value.config.enabled || + enabled.value.config.grantId !== record.grantId + ) + return yield* fail( + "forbidden", + "Ambient work is disabled or lacks its configured grant.", + ); + } + if (record.target.type === "spawn") { + if (!authority.ceiling.mcpCapabilities.includes("workers")) + return yield* fail("forbidden", "Worker spawning is outside the current grant."); + return; + } + let current = yield* read(record.target.threadId); + if (current.thread.projectId !== record.projectId) + return yield* fail("forbidden", "Target is outside the grant project."); + const seen = new Set(); + while (current.thread.id !== record.ownerThreadId) { + if (!current.thread.worker || seen.has(current.thread.id) || seen.size >= 2) + return yield* fail("forbidden", "Scheduled target is not an owned descendant."); + seen.add(current.thread.id); + current = yield* read(current.thread.worker.ownerThreadId); + } + }), + ); + const dispatch = (record: ScheduledWorkRecord) => + wrap( + Effect.gen(function* () { + yield* validate(record); + const authority = yield* authorize(record.ownerThreadId, record.grantId); + const ceiling = UnattendedGrants.intersectCeiling(record.ceiling, authority.ceiling); + const unattendedAuthority = { + grantId: record.grantId, + grantRevision: record.grantRevision, + ownerThreadId: record.ownerThreadId, + runtimeModeCeiling: ceiling.runtimeMode, + mcpCapabilityCeiling: [...ceiling.mcpCapabilities], + }; + if (record.target.type === "spawn") { + const result = yield* workers.spawn( + { + commandId: record.commandId, + callerThreadId: record.ownerThreadId, + label: record.target.label, + prompt: record.prompt, + modelSelection: record.target.modelSelection, + }, + { + mcpCapabilityCeiling: [...ceiling.mcpCapabilities], + runtimeModeCeiling: ceiling.runtimeMode, + unattendedAuthority, + }, + ); + return { sequence: result.sequence, threadId: result.workerThreadId }; + } + const state = yield* read(record.target.threadId); + const receipt = yield* engine.dispatch({ + type: "thread.turn.start", + commandId: record.commandId, + threadId: record.target.threadId, + expectedIdle: true, + unattendedAuthority: { + grantId: record.grantId, + grantRevision: record.grantRevision, + ownerThreadId: record.ownerThreadId, + runtimeModeCeiling: ceiling.runtimeMode, + mcpCapabilityCeiling: [...ceiling.mcpCapabilities], + }, + message: { + messageId: record.messageId, + role: "user", + text: record.prompt, + attachments: [], + }, + runtimeMode: ceiling.runtimeMode, + interactionMode: state.thread.interactionMode, + createdAt: record.createdAt, + }); + return { sequence: receipt.sequence, threadId: record.target.threadId }; + }), + ); + const reconcile = (record: ScheduledWorkRecord) => + wrap( + Effect.gen(function* () { + const receipt = yield* receipts.getByCommandId({ commandId: record.commandId }); + if (Option.isNone(receipt)) return null; + if (receipt.value.status !== "accepted") + return { + state: "failed" as const, + sequence: receipt.value.resultSequence, + threadId: null, + reason: receipt.value.error ?? "Activation was rejected.", + }; + const threadId = receipt.value.aggregateId as ThreadId; + const loaded = yield* snapshots.getWorkerState(threadId); + if (Option.isNone(loaded)) + return { + state: "interrupted" as const, + sequence: receipt.value.resultSequence, + threadId, + reason: "Accepted target was removed.", + }; + const turns = yield* sql<{ + state: string; + turn_id: string | null; + }>`SELECT state,turn_id FROM projection_turns WHERE thread_id = ${threadId} AND pending_message_id = ${record.messageId} ORDER BY row_id DESC LIMIT 1`; + const turn = turns[0]; + const barriers = + yield* sql`SELECT message_id FROM projection_turn_cancellations WHERE thread_id = ${threadId} AND message_id = ${record.messageId}`; + const currentActivations = yield* sql<{ + message_id: string; + }>`SELECT message_id FROM projection_thread_activation_authorities WHERE thread_id = ${threadId}`; + const isCurrentActivation = currentActivations[0]?.message_id === record.messageId; + const nativeLive = + isCurrentActivation && + (loaded.value.thread.session?.status === "starting" || + loaded.value.thread.session?.activeTurnId != null || + loaded.value.thread.latestTurn?.state === "running" || + loaded.value.thread.hasPendingApprovals || + loaded.value.thread.hasPendingUserInput || + loaded.value.thread.backgroundLiveness != null); + if (!isCurrentActivation && (!turn || turn.state === "pending" || turn.state === "running")) + return { + state: record.cancelRequested ? ("cancelled" as const) : ("interrupted" as const), + sequence: receipt.value.resultSequence, + threadId, + reason: "Activation was superseded by a later accepted turn.", + }; + if (record.cancelRequested && barriers.length > 0 && !turn && !nativeLive) + return { + state: "cancelled" as const, + sequence: receipt.value.resultSequence, + threadId, + reason: null, + }; + if ( + nativeLive && + ((turn && ["completed", "error", "interrupted"].includes(turn.state)) || + ["error", "interrupted", "stopped"].includes(loaded.value.thread.session?.status ?? "")) + ) + return { + state: record.cancelRequested ? ("stopping" as const) : ("running" as const), + sequence: receipt.value.resultSequence, + threadId, + reason: null, + }; + const state: ScheduledWorkRecord["state"] = + turn?.state === "pending" + ? record.cancelRequested + ? "stopping" + : "accepted" + : turn?.state === "running" + ? record.cancelRequested + ? "stopping" + : "running" + : turn?.state === "completed" + ? nativeLive + ? "running" + : "completed" + : turn?.state === "error" || loaded.value.thread.session?.status === "error" + ? "failed" + : turn?.state === "interrupted" || + loaded.value.thread.session?.status === "interrupted" || + loaded.value.thread.session?.status === "stopped" + ? record.cancelRequested + ? "cancelled" + : "interrupted" + : "accepted"; + return { + state, + sequence: receipt.value.resultSequence, + threadId, + reason: state === "failed" ? "Provider execution failed after acceptance." : null, + }; + }), + ); + const cancel = (record: ScheduledWorkRecord) => + wrap( + Effect.gen(function* () { + const resolved = yield* reconcile(record); + const threadId = resolved?.threadId ?? record.executionThreadId; + if (!threadId) return "cancelled" as const; + if ( + resolved && + ["completed", "failed", "interrupted", "cancelled"].includes(resolved.state) + ) + return "cancelled" as const; + yield* read(threadId); + const activationTurns = yield* sql<{ + turn_id: string | null; + }>`SELECT turn_id FROM projection_turns WHERE thread_id = ${threadId} AND pending_message_id = ${record.messageId} ORDER BY row_id DESC LIMIT 1`; + // Interrupt and stop are guarded against this exact accepted activation. + yield* engine.dispatch({ + type: "thread.turn.interrupt", + commandId: CommandId.make(`scheduled-cancel:${record.id}`), + threadId, + expectedMessageId: record.messageId, + ...(activationTurns[0]?.turn_id + ? { expectedTurnId: activationTurns[0].turn_id as import("@t3tools/contracts").TurnId } + : {}), + createdAt: record.updatedAt, + }); + return "stopping" as const; + }), + ); + return ScheduledWorkActivation.of({ authorize, validate, dispatch, reconcile, cancel }); +}); +export const layer = Layer.effect(ScheduledWorkActivation, make); diff --git a/apps/server/src/orchestration/activationCancellationFence.ts b/apps/server/src/orchestration/activationCancellationFence.ts index 1a44e9c20742..79c3d481e2c3 100644 --- a/apps/server/src/orchestration/activationCancellationFence.ts +++ b/apps/server/src/orchestration/activationCancellationFence.ts @@ -21,6 +21,38 @@ export const makeActivationCancellationFence = Effect.gen(function* () { SELECT event_sequence FROM projection_turn_cancellations WHERE thread_id = ${threadId} AND message_id = ${messageId} `.pipe(Effect.map((rows) => rows.length > 0)); + const authorized = (threadId: ThreadId, messageId: MessageId) => + Effect.gen(function* () { + const schedules = yield* sql<{ + grant_id: string; + grant_revision: number; + owner_thread_id: string; + project_id: string; + cancel_requested: number; + }>` + SELECT json_extract(document_json, '$.grantId') AS grant_id, + json_extract(document_json, '$.grantRevision') AS grant_revision, + owner_thread_id, json_extract(document_json, '$.projectId') AS project_id, + json_extract(document_json, '$.cancelRequested') AS cancel_requested + FROM scheduled_work WHERE json_extract(document_json, '$.messageId') = ${messageId} + AND (json_extract(document_json, '$.target.threadId') = ${threadId} + OR json_extract(document_json, '$.executionThreadId') = ${threadId} + OR json_extract(document_json, '$.target.type') = 'spawn') + `; + if (schedules.length === 0) return true; + const schedule = schedules[0]!; + if (schedule.cancel_requested) return false; + const activations = yield* sql<{ + message_id: string; + }>`SELECT message_id FROM projection_thread_activation_authorities WHERE thread_id = ${threadId}`; + if (activations[0]?.message_id !== messageId) return false; + const grants = yield* sql` + SELECT grant_id FROM unattended_grants WHERE grant_id = ${schedule.grant_id} + AND revision = ${schedule.grant_revision} AND revoked = 0 + AND owner_thread_id = ${schedule.owner_thread_id} AND project_id = ${schedule.project_id} + `; + return grants.length > 0; + }); const canCancel = (input: { threadId: ThreadId; sequence: number; @@ -55,5 +87,5 @@ export const makeActivationCancellationFence = Effect.gen(function* () { `; return activation.length > 0; }); - return { cancelled, canCancel }; + return { cancelled, authorized, canCancel }; }); diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index 71dfd80914d5..9b770df1bece 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -71,6 +71,8 @@ import Migration0056 from "./Migrations/056_UnattendedAuthority.ts"; import Migration0057 from "./Migrations/057_CoordinationPlans.ts"; import Migration0058 from "./Migrations/058_Memory.ts"; import Migration0059 from "./Migrations/059_QualityRecords.ts"; +import Migration0060 from "./Migrations/060_ScheduledWork.ts"; +import Migration0061 from "./Migrations/061_BackgroundJobs.ts"; /** * Migration loader with all migrations defined inline. @@ -142,6 +144,8 @@ const migrationEntries = [ [57, "CoordinationPlans", Migration0057], [58, "Memory", Migration0058], [59, "QualityRecords", Migration0059], + [60, "ScheduledWork", Migration0060], + [61, "BackgroundJobs", Migration0061], ] as const; export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); diff --git a/apps/server/src/persistence/Migrations/060_ScheduledWork.test.ts b/apps/server/src/persistence/Migrations/060_ScheduledWork.test.ts new file mode 100644 index 000000000000..ca6e62898943 --- /dev/null +++ b/apps/server/src/persistence/Migrations/060_ScheduledWork.test.ts @@ -0,0 +1,37 @@ +import { expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; +import scheduled from "./060_ScheduledWork.ts"; +import jobs from "./061_BackgroundJobs.ts"; + +it.layer(NodeSqliteClient.layer({ filename: ":memory:" }))("automation migrations", (it) => { + it.effect("are repeatable and retain durable schedule, output and notification records", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* scheduled; + yield* jobs; + yield* sql`INSERT INTO scheduled_work(id,owner_thread_id,due_at,state,document_json) VALUES('schedule','owner','2026-10-03T00:00:00.000Z','queued','{}')`; + yield* sql`INSERT INTO background_jobs(id,owner_thread_id,state,document_json) VALUES('job','owner','completed','{}')`; + yield* sql`INSERT INTO background_job_output(job_id,cursor,stream,text) VALUES('job',0,'stderr','retained')`; + yield* sql`INSERT INTO background_job_notifications(job_id,subscriber_thread_id,notify,wake) VALUES('job','owner',1,0)`; + yield* scheduled; + yield* jobs; + expect( + (yield* sql<{ + text: string; + }>`SELECT text FROM background_job_output WHERE job_id = 'job'`)[0]?.text, + ).toBe("retained"); + expect( + (yield* sql<{ count: number }>`SELECT count(*) AS count FROM scheduled_work`)[0]?.count, + ).toBe(1); + const indexes = yield* sql<{ name: string }>`PRAGMA index_list(scheduled_work)`; + expect(indexes.some((row) => row.name === "idx_scheduled_work_due")).toBe(true); + const invalid = + yield* sql`INSERT INTO background_job_output(job_id,cursor,stream,text) VALUES('job',1,'bogus','bad')`.pipe( + Effect.result, + ); + expect(invalid._tag).toBe("Failure"); + }), + ); +}); diff --git a/apps/server/src/persistence/Migrations/060_ScheduledWork.ts b/apps/server/src/persistence/Migrations/060_ScheduledWork.ts new file mode 100644 index 000000000000..6d95d3d1675c --- /dev/null +++ b/apps/server/src/persistence/Migrations/060_ScheduledWork.ts @@ -0,0 +1,19 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`CREATE TABLE IF NOT EXISTS scheduled_work ( + id TEXT PRIMARY KEY NOT NULL, owner_thread_id TEXT NOT NULL, + due_at TEXT NOT NULL, state TEXT NOT NULL, + request_json TEXT NOT NULL DEFAULT '{}' CHECK(json_valid(request_json)), + document_json TEXT NOT NULL CHECK(json_valid(document_json)) + )`; + yield* sql`CREATE INDEX IF NOT EXISTS idx_scheduled_work_message ON scheduled_work(json_extract(document_json, '$.messageId'))`; + yield* sql`CREATE INDEX IF NOT EXISTS idx_scheduled_work_due ON scheduled_work(state, due_at, id)`; + yield* sql`CREATE INDEX IF NOT EXISTS idx_scheduled_work_owner ON scheduled_work(owner_thread_id, id)`; + yield* sql`CREATE TABLE IF NOT EXISTS ambient_work ( + owner_thread_id TEXT PRIMARY KEY NOT NULL, + document_json TEXT NOT NULL CHECK(json_valid(document_json)) + )`; +}); diff --git a/apps/server/src/persistence/Migrations/061_BackgroundJobs.ts b/apps/server/src/persistence/Migrations/061_BackgroundJobs.ts new file mode 100644 index 000000000000..975bfd6e6d5c --- /dev/null +++ b/apps/server/src/persistence/Migrations/061_BackgroundJobs.ts @@ -0,0 +1,24 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`CREATE TABLE IF NOT EXISTS background_jobs ( + id TEXT PRIMARY KEY NOT NULL, owner_thread_id TEXT NOT NULL, + state TEXT NOT NULL, request_json TEXT NOT NULL DEFAULT '{}' CHECK(json_valid(request_json)), + document_json TEXT NOT NULL CHECK(json_valid(document_json)) + )`; + yield* sql`CREATE INDEX IF NOT EXISTS idx_background_jobs_owner ON background_jobs(owner_thread_id, id)`; + yield* sql`CREATE TABLE IF NOT EXISTS background_job_output ( + job_id TEXT NOT NULL REFERENCES background_jobs(id) ON DELETE CASCADE, + cursor INTEGER NOT NULL, stream TEXT NOT NULL CHECK(stream IN ('stdout','stderr')), + text TEXT NOT NULL, PRIMARY KEY(job_id, cursor) + )`; + yield* sql`CREATE TABLE IF NOT EXISTS background_job_notifications ( + job_id TEXT NOT NULL REFERENCES background_jobs(id) ON DELETE CASCADE, + subscriber_thread_id TEXT NOT NULL, notify INTEGER NOT NULL CHECK(notify IN (0,1)), + wake INTEGER NOT NULL CHECK(wake IN (0,1)), delivered INTEGER NOT NULL DEFAULT 0, + suppressed_reason TEXT, + PRIMARY KEY(job_id, subscriber_thread_id) + )`; +}); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 1a2f9fd93791..b7eb23a5ec41 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -166,7 +166,12 @@ import * as OwnedWorkers from "./orchestration/OwnedWorkers.ts"; import * as SharedWorkspaceActivity from "./workspace/SharedWorkspaceActivity.ts"; import * as Memory from "./memory/Memory.ts"; import * as QualityRecords from "./orchestration/QualityRecords.ts"; +import * as ScheduledWork from "./orchestration/ScheduledWork.ts"; +import * as ScheduledWorkActivation from "./orchestration/ScheduledWorkActivation.ts"; +import * as AmbientWork from "./orchestration/AmbientWork.ts"; import * as UnattendedGrants from "./orchestration/UnattendedGrants.ts"; +import * as BackgroundJobs from "./background/BackgroundJobs.ts"; +import * as BackgroundJobAuthority from "./background/BackgroundJobAuthority.ts"; import * as ProjectionSnapshotQuery from "./orchestration/Services/ProjectionSnapshotQuery.ts"; import { clearPersistedServerRuntimeState, @@ -477,6 +482,10 @@ const ProviderRuntimeLayerLive = ProviderSessionReaperLive.pipe( // Subscribes to `account.rate-limits.updated` so usage bars track live // telemetry instead of waiting for the next status probe. Layer.provideMerge(ProviderUsageLimitsIngestionLive), + Layer.provideMerge(AmbientWork.layer), + Layer.provideMerge(BackgroundJobs.layer.pipe(Layer.provide(BackgroundJobAuthority.layer))), + Layer.provideMerge(ScheduledWork.layer), + Layer.provideMerge(ScheduledWorkActivation.layer), Layer.provideMerge(UnattendedGrants.layer), Layer.provideMerge(OwnedWorkers.layer), Layer.provideMerge(Layer.mergeAll(Memory.layer, QualityRecords.layer)), diff --git a/apps/server/src/serverRuntimeStartup.ts b/apps/server/src/serverRuntimeStartup.ts index a7aac17e24c5..df2a9394b9cb 100644 --- a/apps/server/src/serverRuntimeStartup.ts +++ b/apps/server/src/serverRuntimeStartup.ts @@ -34,6 +34,9 @@ import * as Scope from "effect/Scope"; import * as ServerConfig from "./config.ts"; import * as CoordinationReactor from "./orchestration/CoordinationReactor.ts"; +import * as ScheduledWork from "./orchestration/ScheduledWork.ts"; +import * as AmbientWork from "./orchestration/AmbientWork.ts"; +import * as BackgroundJobs from "./background/BackgroundJobs.ts"; import { flushCompileCache } from "./compileCache.ts"; import * as Keybindings from "./keybindings.ts"; import * as ExternalLauncher from "./process/externalLauncher.ts"; @@ -961,6 +964,9 @@ export const make = (options?: StartupOptions) => const projectionSnapshotQuery = yield* ProjectionSnapshotQuery.ProjectionSnapshotQuery; const providerSessionDirectory = yield* ProviderSessionDirectory.ProviderSessionDirectory; const coordinationReactor = yield* CoordinationReactor.CoordinationReactor; + const scheduledWork = yield* ScheduledWork.ScheduledWork; + const ambientWork = yield* AmbientWork.AmbientWork; + const backgroundJobs = yield* BackgroundJobs.BackgroundJobs; const crypto = yield* Crypto.Crypto; const launcher = yield* ServiceLauncherClient.ServiceLauncherClient; @@ -1024,6 +1030,15 @@ export const make = (options?: StartupOptions) => yield* runStartupPhase("workers.pending-starts.reconcile", reconcileWorkerPendingStarts); yield* runStartupPhase("provider-sessions.reconcile", reconcileProviderSessions); yield* runStartupPhase("coordination.recover", coordinationReactor.recover); + yield* runStartupPhase("background-jobs.reconcile", backgroundJobs.reconcile); + yield* runStartupPhase( + "scheduled-work.start", + scheduledWork.start.pipe(Scope.provide(reactorScope)), + ); + yield* runStartupPhase( + "ambient-work.start", + ambientWork.start.pipe(Scope.provide(reactorScope)), + ); yield* runStartupPhase( "coordination.start", coordinationReactor.start.pipe(Scope.provide(reactorScope)), diff --git a/apps/server/src/testUtils/parityDependencies.ts b/apps/server/src/testUtils/parityDependencies.ts index a37cbf3c0a14..17b39dd02c57 100644 --- a/apps/server/src/testUtils/parityDependencies.ts +++ b/apps/server/src/testUtils/parityDependencies.ts @@ -1,4 +1,9 @@ import * as Layer from "effect/Layer"; +import * as AmbientWork from "../orchestration/AmbientWork.ts"; +import * as BackgroundJobs from "../background/BackgroundJobs.ts"; +import * as BackgroundJobAuthority from "../background/BackgroundJobAuthority.ts"; +import * as ScheduledWork from "../orchestration/ScheduledWork.ts"; +import * as ScheduledWorkActivation from "../orchestration/ScheduledWorkActivation.ts"; import * as UnattendedGrants from "../orchestration/UnattendedGrants.ts"; import * as OwnedWorkers from "../orchestration/OwnedWorkers.ts"; import * as CoordinationPlans from "../orchestration/CoordinationPlans.ts"; @@ -13,9 +18,20 @@ import { OrchestrationCommandReceiptRepositoryLive } from "../persistence/Layers // Supply the caller's isolated persistence, engine, query, settings and provider registry // outside this graph so every domain service captures the same test dependencies. -export const parityDependenciesLayer = UnattendedGrants.layer.pipe( +export const parityStartupDependenciesLayer = Layer.mergeAll( + AmbientWork.layer, + BackgroundJobs.layer.pipe(Layer.provide(BackgroundJobAuthority.layer)), +).pipe( + Layer.provideMerge(ScheduledWork.layer), + Layer.provideMerge(ScheduledWorkActivation.layer), + Layer.provideMerge(UnattendedGrants.layer), Layer.provideMerge(OwnedWorkers.layer), Layer.provideMerge(OrchestrationCommandReceiptRepositoryLive), + Layer.provide(WorkspacePaths.layer), + Layer.provide(ProcessRunner.layer), +); + +export const parityDependenciesLayer = parityStartupDependenciesLayer.pipe( Layer.provideMerge(GlobalMemory.layer), Layer.provideMerge( Layer.mergeAll( @@ -26,5 +42,4 @@ export const parityDependenciesLayer = UnattendedGrants.layer.pipe( ), ), Layer.provide(WorkspacePaths.layer), - Layer.provide(ProcessRunner.layer), ); diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index f0b0abc876f9..ed3b58df5282 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -108,10 +108,13 @@ import { } from "./orchestration/Normalizer.ts"; import * as OrchestrationEngine from "./orchestration/Services/OrchestrationEngine.ts"; import * as OwnedWorkers from "./orchestration/OwnedWorkers.ts"; +import * as AmbientWork from "./orchestration/AmbientWork.ts"; import * as GlobalMemory from "./memory/GlobalMemory.ts"; import * as Memory from "./memory/Memory.ts"; import * as QualityRecords from "./orchestration/QualityRecords.ts"; +import * as ScheduledWork from "./orchestration/ScheduledWork.ts"; import * as UnattendedGrants from "./orchestration/UnattendedGrants.ts"; +import * as BackgroundJobs from "./background/BackgroundJobs.ts"; import * as CoordinationPlans from "./orchestration/CoordinationPlans.ts"; import * as ProjectionSnapshotQuery from "./orchestration/Services/ProjectionSnapshotQuery.ts"; import { ThreadDeletionReactor } from "./orchestration/Services/ThreadDeletionReactor.ts"; @@ -537,18 +540,37 @@ const makeWsRpcLayer = ( const makeParityToolsRpcLayer = () => ParityToolsRpcGroup.toLayer( Effect.gen(function* () { + const ambientWork = yield* AmbientWork.AmbientWork; const memory = yield* Memory.MemoryService; const qualityRecords = yield* QualityRecords.QualityRecords; + const scheduledWork = yield* ScheduledWork.ScheduledWork; const unattendedGrants = yield* UnattendedGrants.UnattendedGrants; + const backgroundJobs = yield* BackgroundJobs.BackgroundJobs; const coordinationPlans = yield* CoordinationPlans.CoordinationPlans; return ParityToolsRpcGroup.of({ [WS_METHODS.qualitySubscribeChanges]: (input) => qualityRecords.subscribe(input, { threadId: input.threadId, source: "user-reported" }), + [WS_METHODS.ambientConfigure]: (input) => + ambientWork.configure(input, { source: "client" }), + [WS_METHODS.ambientGet]: (input) => ambientWork.get(input), + [WS_METHODS.ambientStop]: (input) => ambientWork.stop(input), + [WS_METHODS.scheduledCreate]: (input) => scheduledWork.create(input), + [WS_METHODS.scheduledList]: (input) => scheduledWork.list(input), + [WS_METHODS.scheduledGet]: (input) => scheduledWork.get(input), + [WS_METHODS.scheduledCancel]: (input) => scheduledWork.cancel(input), [WS_METHODS.unattendedGrantCreate]: (input) => unattendedGrants.create(input, { source: "client" }), [WS_METHODS.unattendedGrantList]: (input) => unattendedGrants.list(input), [WS_METHODS.unattendedGrantRevoke]: (input) => unattendedGrants.revoke(input, { source: "client" }), + [WS_METHODS.backgroundJobStart]: (input) => backgroundJobs.start(input), + [WS_METHODS.backgroundJobList]: (input) => backgroundJobs.list(input), + [WS_METHODS.backgroundJobGet]: (input) => backgroundJobs.get(input), + [WS_METHODS.backgroundJobOutput]: (input) => backgroundJobs.output(input), + [WS_METHODS.backgroundJobWait]: (input) => backgroundJobs.wait(input), + [WS_METHODS.backgroundJobCancel]: (input) => backgroundJobs.cancel(input), + [WS_METHODS.backgroundJobSubscribe]: (input) => backgroundJobs.subscribe(input), + [WS_METHODS.backgroundJobCleanup]: (input) => backgroundJobs.cleanup(input), [WS_METHODS.memoryRemember]: ({ projectId, input }) => memory.remember(input, { projectId, allowGlobal: false }), [WS_METHODS.memoryRecall]: ({ projectId, input }) => @@ -4210,10 +4232,13 @@ export const websocketRpcRouteLayer = Layer.unwrap( }); const pullRequests = yield* PullRequestService.PullRequestService; const ownedWorkers = yield* OwnedWorkers.OwnedWorkers; + const ambientWork = yield* AmbientWork.AmbientWork; const globalMemory = yield* GlobalMemory.GlobalMemory; const memory = yield* Memory.MemoryService; const qualityRecords = yield* QualityRecords.QualityRecords; + const scheduledWork = yield* ScheduledWork.ScheduledWork; const unattendedGrants = yield* UnattendedGrants.UnattendedGrants; + const backgroundJobs = yield* BackgroundJobs.BackgroundJobs; const coordinationPlans = yield* CoordinationPlans.CoordinationPlans; const sql = yield* SqlClient.SqlClient; return HttpRouter.add( @@ -4260,9 +4285,12 @@ export const websocketRpcRouteLayer = Layer.unwrap( // mutation invalidates the HTTP diff cache that every client reads from. Layer.provide(Layer.succeed(PullRequestService.PullRequestService, pullRequests)), Layer.provide(Layer.succeed(OwnedWorkers.OwnedWorkers, ownedWorkers)), + Layer.provide(Layer.succeed(AmbientWork.AmbientWork, ambientWork)), Layer.provide(Layer.succeed(Memory.MemoryService, memory)), Layer.provide(Layer.succeed(QualityRecords.QualityRecords, qualityRecords)), + Layer.provide(Layer.succeed(ScheduledWork.ScheduledWork, scheduledWork)), Layer.provide(Layer.succeed(UnattendedGrants.UnattendedGrants, unattendedGrants)), + Layer.provide(Layer.succeed(BackgroundJobs.BackgroundJobs, backgroundJobs)), ) .pipe( Layer.provide(Layer.succeed(GlobalMemory.GlobalMemory, globalMemory)), diff --git a/apps/web/src/components/settings/scopedSettings.test.ts b/apps/web/src/components/settings/scopedSettings.test.ts index 276b2791b545..a9233d489a86 100644 --- a/apps/web/src/components/settings/scopedSettings.test.ts +++ b/apps/web/src/components/settings/scopedSettings.test.ts @@ -110,25 +110,25 @@ describe("optional agent tool scope", () => { it("fans out environment toggles without replacing unrelated target flags", () => { const selected = [ environment("Laptop", { settings: { agentToolCapabilities: ["quality-records"] } }), - environment("Server", { settings: { agentToolCapabilities: [] } }), + environment("Server", { settings: { agentToolCapabilities: ["automation"] } }), ]; const writes = planScopedAgentToolCapability(all, selected, "memory", true).serverWrites; expect(writes.map((write) => write.patch.agentToolCapabilities)).toEqual([ ["memory", "quality-records"], - ["memory"], + ["memory", "automation"], ]); expect( planScopedAgentToolCapability(all, selected, "quality-records", false).serverWrites.map( (write) => write.patch.agentToolCapabilities, ), - ).toEqual([[], []]); + ).toEqual([[], ["automation"]]); }); it("uses each project's effective flags and reset restores its own environment", () => { const selected = [ environment("Laptop", { settings: { agentToolCapabilities: ["quality-records"] } }), environment("Server", { settings: { - agentToolCapabilities: ["memory"], + agentToolCapabilities: ["automation"], projectSettingsOverrides: { [projectId]: { agentToolCapabilities: ["quality-records"], @@ -165,7 +165,7 @@ describe("optional agent tool scope", () => { resolveScopedSettingsTargets(project, [ { ...selected[1]!, serverConfig: { ...selected[1]!.serverConfig!, settings: restored } }, ])[0]?.settings.agentToolCapabilities, - ).toEqual(["memory"]); + ).toEqual(["automation"]); }); }); const checkout = resolveSettingsScope( diff --git a/docs/user/project-settings.md b/docs/user/project-settings.md index 049c66f62a58..c7209282db05 100644 --- a/docs/user/project-settings.md +++ b/docs/user/project-settings.md @@ -78,7 +78,7 @@ saved memories. **Allow global memory** is environment-wide and lets authorized clients manage memories shared across projects. It does not grant agents global access or enable automatic global recall. -Unattended work requires a separate, +Scheduling, ambient work, and background commands require a separate, administrator-issued grant for the owning thread. Tool access alone is not that grant. Revoking a grant prevents further work under it; it cannot be replaced by an agent choosing a more permissive grant. diff --git a/packages/client-runtime/src/operations/agentToolSettings.test.ts b/packages/client-runtime/src/operations/agentToolSettings.test.ts index 01686dea1d92..a1e070ab5856 100644 --- a/packages/client-runtime/src/operations/agentToolSettings.test.ts +++ b/packages/client-runtime/src/operations/agentToolSettings.test.ts @@ -8,14 +8,14 @@ import { describe("optional agent tool settings", () => { it("preserves each target's unrelated flags when enabling or disabling one tool", () => { const first = ["quality-records", "memory"] as const; - const second = ["quality-records"] as const; + const second = ["automation"] as const; expect(setOptionalAgentToolCapability(first, "memory", false)).toEqual(["quality-records"]); expect(setOptionalAgentToolCapability(second, "memory", true)).toEqual([ "memory", - "quality-records", + "automation", ]); expect(first).toEqual(["quality-records", "memory"]); - expect(second).toEqual(["quality-records"]); + expect(second).toEqual(["automation"]); }); it("is idempotent, deduplicated and bounded by the capability catalog", () => { expect(setOptionalAgentToolCapability(["memory", "memory"], "memory", true)).toEqual([ @@ -23,7 +23,7 @@ describe("optional agent tool settings", () => { ]); expect( setOptionalAgentToolCapability(OPTIONAL_AGENT_TOOL_CAPABILITIES, "memory", true), - ).toHaveLength(2); + ).toHaveLength(4); expect(setOptionalAgentToolCapability([], "memory", false)).toEqual([]); }); it("disabling any capability never broadens access", () => { @@ -33,7 +33,7 @@ describe("optional agent tool settings", () => { capability, false, ); - expect(narrowed).toHaveLength(1); + expect(narrowed).toHaveLength(3); expect(narrowed).not.toContain(capability); expect(narrowed.every((entry) => OPTIONAL_AGENT_TOOL_CAPABILITIES.includes(entry))).toBe( true, diff --git a/packages/client-runtime/src/operations/agentToolSettings.ts b/packages/client-runtime/src/operations/agentToolSettings.ts index 401da429a24a..add120229784 100644 --- a/packages/client-runtime/src/operations/agentToolSettings.ts +++ b/packages/client-runtime/src/operations/agentToolSettings.ts @@ -17,6 +17,14 @@ export const OPTIONAL_AGENT_TOOL_DETAILS: Record< label: "Quality records", description: "Record task progress and verification evidence.", }, + automation: { + label: "Automation", + description: "Schedule agent work and manage unattended tasks.", + }, + "background-jobs": { + label: "Background jobs", + description: "Start and manage background commands.", + }, }; /** Change one flag using each target's effective value, never a representative's array. */ diff --git a/packages/client-runtime/src/operations/parity.ts b/packages/client-runtime/src/operations/parity.ts index 3e2619f89b4a..c2d071a462cb 100644 --- a/packages/client-runtime/src/operations/parity.ts +++ b/packages/client-runtime/src/operations/parity.ts @@ -38,9 +38,30 @@ export const parityOperations = { subscribeChanges: (input: EnvironmentRpcInput) => subscribe(WS_METHODS.qualitySubscribeChanges, input), }, + scheduled: { + create: unary(WS_METHODS.scheduledCreate), + list: unary(WS_METHODS.scheduledList), + get: unary(WS_METHODS.scheduledGet), + cancel: unary(WS_METHODS.scheduledCancel), + }, unattendedGrants: { create: unary(WS_METHODS.unattendedGrantCreate), list: unary(WS_METHODS.unattendedGrantList), revoke: unary(WS_METHODS.unattendedGrantRevoke), }, + ambient: { + configure: unary(WS_METHODS.ambientConfigure), + get: unary(WS_METHODS.ambientGet), + stop: unary(WS_METHODS.ambientStop), + }, + backgroundJobs: { + start: unary(WS_METHODS.backgroundJobStart), + list: unary(WS_METHODS.backgroundJobList), + get: unary(WS_METHODS.backgroundJobGet), + output: unary(WS_METHODS.backgroundJobOutput), + cancel: unary(WS_METHODS.backgroundJobCancel), + wait: unary(WS_METHODS.backgroundJobWait), + subscribe: unary(WS_METHODS.backgroundJobSubscribe), + cleanup: unary(WS_METHODS.backgroundJobCleanup), + }, }; diff --git a/packages/contracts/src/backgroundJobs.ts b/packages/contracts/src/backgroundJobs.ts new file mode 100644 index 000000000000..c663384a4f44 --- /dev/null +++ b/packages/contracts/src/backgroundJobs.ts @@ -0,0 +1,81 @@ +import * as Schema from "effect/Schema"; +import { IsoDateTime, ProjectId, ThreadId, TrimmedNonEmptyString } from "./baseSchemas.ts"; + +export const BackgroundJobId = TrimmedNonEmptyString.check(Schema.isMaxLength(160)); +export const BackgroundJobStartInput = Schema.Struct({ + id: BackgroundJobId, + callerThreadId: ThreadId, + command: TrimmedNonEmptyString.check(Schema.isMaxLength(4096)), + args: Schema.Array(Schema.String.check(Schema.isMaxLength(8192))).check(Schema.isMaxLength(128)), + timeoutMs: Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 86_400_000 })), + maxOutputBytes: Schema.Int.check(Schema.isBetween({ minimum: 1024, maximum: 1_048_576 })), +}); +export type BackgroundJobStartInput = typeof BackgroundJobStartInput.Type; +export const BackgroundJobReadInput = Schema.Struct({ + callerThreadId: ThreadId, + id: BackgroundJobId, +}); +export type BackgroundJobReadInput = typeof BackgroundJobReadInput.Type; +export const BackgroundJobOutputInput = Schema.Struct({ + ...BackgroundJobReadInput.fields, + cursor: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + limitBytes: Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 65_536 })), +}); +export type BackgroundJobOutputInput = typeof BackgroundJobOutputInput.Type; +export const BackgroundJobWaitInput = Schema.Struct({ + ...BackgroundJobReadInput.fields, + timeoutMs: Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 600_000 })), +}); +export type BackgroundJobWaitInput = typeof BackgroundJobWaitInput.Type; +export const BackgroundJobState = Schema.Literals([ + "pending", + "running", + "cancelling", + "completed", + "failed", + "cancelled", + "interrupted", +]); +export const BackgroundJobRecord = Schema.Struct({ + id: BackgroundJobId, + ownerThreadId: ThreadId, + projectId: ProjectId, + command: Schema.String, + args: Schema.Array(Schema.String), + cwd: Schema.String, + state: BackgroundJobState, + createdAt: IsoDateTime, + updatedAt: IsoDateTime, + exitCode: Schema.NullOr(Schema.Int), + reason: Schema.NullOr(Schema.String), + maxOutputBytes: Schema.Int, + outputBytes: Schema.Int, + truncated: Schema.Boolean, + progress: Schema.NullOr( + Schema.Struct({ + value: Schema.Number.check(Schema.isBetween({ minimum: 0, maximum: 1 })), + label: Schema.String.check(Schema.isMaxLength(500)), + }), + ), +}); +export type BackgroundJobRecord = typeof BackgroundJobRecord.Type; +export const BackgroundJobOutput = Schema.Struct({ + chunks: Schema.Array( + Schema.Struct({ + cursor: Schema.Int, + stream: Schema.Literals(["stdout", "stderr"]), + text: Schema.String, + }), + ), + nextCursor: Schema.Int, + truncated: Schema.Boolean, +}); +export type BackgroundJobOutput = typeof BackgroundJobOutput.Type; +export class BackgroundJobError extends Schema.TaggedError()( + "BackgroundJobError", + { + code: Schema.Literals(["invalid", "forbidden", "not-found", "conflict", "running", "internal"]), + detail: Schema.String, + cause: Schema.optional(Schema.Defect()), + }, +) {} diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index bfb599dca6f8..19e86705e3e5 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -34,6 +34,7 @@ export * from "./memory.ts"; export * from "./qualityRecords.ts"; export * from "./scheduledWork.ts"; export * from "./unattendedGrants.ts"; +export * from "./backgroundJobs.ts"; export * from "./t3ProjectFile.ts"; export * from "./editor.ts"; export * from "./project.ts"; diff --git a/packages/contracts/src/orchestration.ts b/packages/contracts/src/orchestration.ts index 4df866ae9459..554344538d7d 100644 --- a/packages/contracts/src/orchestration.ts +++ b/packages/contracts/src/orchestration.ts @@ -139,7 +139,12 @@ export const RuntimeMode = Schema.Literals([ ]); export type RuntimeMode = typeof RuntimeMode.Type; export const DEFAULT_RUNTIME_MODE: RuntimeMode = "full-access"; -export const OPTIONAL_AGENT_TOOL_CAPABILITIES = ["memory", "quality-records"] as const; +export const OPTIONAL_AGENT_TOOL_CAPABILITIES = [ + "memory", + "quality-records", + "automation", + "background-jobs", +] as const; export const OptionalAgentToolCapability = Schema.Literals(OPTIONAL_AGENT_TOOL_CAPABILITIES); export type OptionalAgentToolCapability = typeof OptionalAgentToolCapability.Type; export const WorkerMcpCapability = Schema.Literals([ @@ -1364,6 +1369,7 @@ export type ThreadTurnStartBootstrap = typeof ThreadTurnStartBootstrap.Type; export const ThreadTurnStartCommand = Schema.Struct({ unattendedAuthority: Schema.optional(ThreadUnattendedAuthority), + expectedIdle: Schema.optional(Schema.Literal(true)), type: Schema.Literal("thread.turn.start"), commandId: CommandId, threadId: ThreadId, diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index c183862fbd92..a8928b04175f 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -12,6 +12,7 @@ import * as Coordination from "./coordination.ts"; import * as Memory from "./memory.ts"; import * as Quality from "./qualityRecords.ts"; import * as Scheduled from "./scheduledWork.ts"; +import * as Jobs from "./backgroundJobs.ts"; import * as Grants from "./unattendedGrants.ts"; import { ProjectId } from "./baseSchemas.ts"; import { @@ -309,9 +310,24 @@ export const WS_METHODS = { memoryGlobalRead: "memoryGlobal.read", memoryGlobalWrite: "memoryGlobal.write", qualitySubscribeChanges: "quality.subscribeChanges", + ambientConfigure: "ambient.configure", + ambientGet: "ambient.get", + ambientStop: "ambient.stop", + scheduledCreate: "scheduled.create", + scheduledList: "scheduled.list", + scheduledGet: "scheduled.get", + scheduledCancel: "scheduled.cancel", unattendedGrantCreate: "unattendedGrants.create", unattendedGrantList: "unattendedGrants.list", unattendedGrantRevoke: "unattendedGrants.revoke", + backgroundJobStart: "backgroundJobs.start", + backgroundJobList: "backgroundJobs.list", + backgroundJobGet: "backgroundJobs.get", + backgroundJobOutput: "backgroundJobs.output", + backgroundJobCancel: "backgroundJobs.cancel", + backgroundJobWait: "backgroundJobs.wait", + backgroundJobSubscribe: "backgroundJobs.subscribe", + backgroundJobCleanup: "backgroundJobs.cleanup", memoryRemember: "memory.remember", memoryRecall: "memory.recall", memorySearch: "memory.search", @@ -1557,6 +1573,41 @@ export const ParityToolsRpcGroup = RpcGroup.make( stream: true, error: Schema.Union([Quality.QualityRecordsError, EnvironmentAuthorizationError]), }), + Rpc.make(WS_METHODS.ambientConfigure, { + payload: Scheduled.AmbientWorkConfigureInput, + success: Scheduled.AmbientWorkRecord, + error: Schema.Union([Scheduled.ScheduledWorkError, EnvironmentAuthorizationError]), + }), + Rpc.make(WS_METHODS.ambientGet, { + payload: Scheduled.ScheduledWorkListInput, + success: Schema.NullOr(Scheduled.AmbientWorkRecord), + error: Schema.Union([Scheduled.ScheduledWorkError, EnvironmentAuthorizationError]), + }), + Rpc.make(WS_METHODS.ambientStop, { + payload: Scheduled.ScheduledWorkListInput, + success: Schema.NullOr(Scheduled.AmbientWorkRecord), + error: Schema.Union([Scheduled.ScheduledWorkError, EnvironmentAuthorizationError]), + }), + Rpc.make(WS_METHODS.scheduledCreate, { + payload: Scheduled.ScheduledWorkCreateInput, + success: Scheduled.ScheduledWorkRecord, + error: Schema.Union([Scheduled.ScheduledWorkError, EnvironmentAuthorizationError]), + }), + Rpc.make(WS_METHODS.scheduledList, { + payload: Scheduled.ScheduledWorkListInput, + success: Schema.Array(Scheduled.ScheduledWorkRecord), + error: Schema.Union([Scheduled.ScheduledWorkError, EnvironmentAuthorizationError]), + }), + Rpc.make(WS_METHODS.scheduledGet, { + payload: Scheduled.ScheduledWorkReadInput, + success: Scheduled.ScheduledWorkRecord, + error: Schema.Union([Scheduled.ScheduledWorkError, EnvironmentAuthorizationError]), + }), + Rpc.make(WS_METHODS.scheduledCancel, { + payload: Scheduled.ScheduledWorkReadInput, + success: Scheduled.ScheduledWorkRecord, + error: Schema.Union([Scheduled.ScheduledWorkError, EnvironmentAuthorizationError]), + }), Rpc.make(WS_METHODS.unattendedGrantCreate, { payload: Grants.UnattendedGrantCreateInput, success: Grants.UnattendedGrant, @@ -1572,6 +1623,50 @@ export const ParityToolsRpcGroup = RpcGroup.make( success: Grants.UnattendedGrant, error: Schema.Union([Scheduled.ScheduledWorkError, EnvironmentAuthorizationError]), }), + Rpc.make(WS_METHODS.backgroundJobStart, { + payload: Jobs.BackgroundJobStartInput, + success: Jobs.BackgroundJobRecord, + error: Schema.Union([Jobs.BackgroundJobError, EnvironmentAuthorizationError]), + }), + Rpc.make(WS_METHODS.backgroundJobList, { + payload: Scheduled.ScheduledWorkListInput, + success: Schema.Array(Jobs.BackgroundJobRecord), + error: Schema.Union([Jobs.BackgroundJobError, EnvironmentAuthorizationError]), + }), + Rpc.make(WS_METHODS.backgroundJobGet, { + payload: Jobs.BackgroundJobReadInput, + success: Jobs.BackgroundJobRecord, + error: Schema.Union([Jobs.BackgroundJobError, EnvironmentAuthorizationError]), + }), + Rpc.make(WS_METHODS.backgroundJobOutput, { + payload: Jobs.BackgroundJobOutputInput, + success: Jobs.BackgroundJobOutput, + error: Schema.Union([Jobs.BackgroundJobError, EnvironmentAuthorizationError]), + }), + Rpc.make(WS_METHODS.backgroundJobCancel, { + payload: Jobs.BackgroundJobReadInput, + success: Jobs.BackgroundJobRecord, + error: Schema.Union([Jobs.BackgroundJobError, EnvironmentAuthorizationError]), + }), + Rpc.make(WS_METHODS.backgroundJobWait, { + payload: Jobs.BackgroundJobWaitInput, + success: Schema.Struct({ timedOut: Schema.Boolean, job: Jobs.BackgroundJobRecord }), + error: Schema.Union([Jobs.BackgroundJobError, EnvironmentAuthorizationError]), + }), + Rpc.make(WS_METHODS.backgroundJobSubscribe, { + payload: Schema.Struct({ + ...Jobs.BackgroundJobReadInput.fields, + notify: Schema.Boolean, + wake: Schema.Boolean, + }), + success: Jobs.BackgroundJobRecord, + error: Schema.Union([Jobs.BackgroundJobError, EnvironmentAuthorizationError]), + }), + Rpc.make(WS_METHODS.backgroundJobCleanup, { + payload: Jobs.BackgroundJobReadInput, + success: Schema.Void, + error: Schema.Union([Jobs.BackgroundJobError, EnvironmentAuthorizationError]), + }), Rpc.make(WS_METHODS.memoryRemember, { payload: Schema.Struct({ projectId: ProjectId, input: Memory.MemoryRememberInput }), success: Memory.MemoryMutationResult, diff --git a/packages/contracts/src/scheduledWork.ts b/packages/contracts/src/scheduledWork.ts index 5e7ee4579dbb..dbb7b5e6b458 100644 --- a/packages/contracts/src/scheduledWork.ts +++ b/packages/contracts/src/scheduledWork.ts @@ -1,14 +1,86 @@ import * as Schema from "effect/Schema"; -import { ThreadId } from "./baseSchemas.ts"; -import { RuntimeMode, WorkerMcpCapability } from "./orchestration.ts"; +import { + CommandId, + IsoDateTime, + MessageId, + ProjectId, + ThreadId, + TrimmedNonEmptyString, +} from "./baseSchemas.ts"; +import { ModelSelection, RuntimeMode, WorkerMcpCapability } from "./orchestration.ts"; +export const ScheduledWorkId = TrimmedNonEmptyString.check(Schema.isMaxLength(160)); +export type ScheduledWorkId = typeof ScheduledWorkId.Type; export const UnattendedCeiling = Schema.Struct({ runtimeMode: RuntimeMode, mcpCapabilities: Schema.Array(WorkerMcpCapability).check(Schema.isMaxLength(32)), }); export type UnattendedCeiling = typeof UnattendedCeiling.Type; +export const ScheduledWorkTarget = Schema.Union([ + Schema.Struct({ type: Schema.Literal("resume"), threadId: ThreadId }), + Schema.Struct({ + type: Schema.Literal("spawn"), + label: TrimmedNonEmptyString.check(Schema.isMaxLength(120)), + modelSelection: ModelSelection, + }), + Schema.Struct({ type: Schema.Literal("ambient"), threadId: ThreadId }), +]); +export const ScheduledWorkCreateInput = Schema.Struct({ + id: ScheduledWorkId, + callerThreadId: ThreadId, + target: ScheduledWorkTarget, + prompt: TrimmedNonEmptyString.check(Schema.isMaxLength(64_000)), + dueAt: Schema.optional(IsoDateTime), + delayMs: Schema.optional( + Schema.Int.check(Schema.isBetween({ minimum: 0, maximum: 31_536_000_000 })), + ), + latestStartAt: Schema.optional(IsoDateTime), + onBusy: Schema.Literals(["wait", "fail"]), + grantId: TrimmedNonEmptyString.check(Schema.isMaxLength(160)), +}); +export type ScheduledWorkCreateInput = typeof ScheduledWorkCreateInput.Type; +export const ScheduledWorkReadInput = Schema.Struct({ + callerThreadId: ThreadId, + id: ScheduledWorkId, +}); +export type ScheduledWorkReadInput = typeof ScheduledWorkReadInput.Type; export const ScheduledWorkListInput = Schema.Struct({ callerThreadId: ThreadId }); export type ScheduledWorkListInput = typeof ScheduledWorkListInput.Type; +export const ScheduledWorkState = Schema.Literals([ + "queued", + "dispatching", + "blocked", + "accepted", + "running", + "stopping", + "completed", + "failed", + "interrupted", + "cancelled", +]); +export const ScheduledWorkRecord = Schema.Struct({ + id: ScheduledWorkId, + ownerThreadId: ThreadId, + projectId: ProjectId, + target: ScheduledWorkTarget, + prompt: Schema.String, + dueAt: IsoDateTime, + latestStartAt: Schema.NullOr(IsoDateTime), + onBusy: Schema.Literals(["wait", "fail"]), + grantId: Schema.String, + grantRevision: Schema.Int, + ceiling: UnattendedCeiling, + commandId: CommandId, + messageId: MessageId, + state: ScheduledWorkState, + reason: Schema.NullOr(Schema.String), + cancelRequested: Schema.Boolean, + acceptedSequence: Schema.NullOr(Schema.Int), + executionThreadId: Schema.NullOr(ThreadId), + createdAt: IsoDateTime, + updatedAt: IsoDateTime, +}); +export type ScheduledWorkRecord = typeof ScheduledWorkRecord.Type; export class ScheduledWorkError extends Schema.TaggedError()( "ScheduledWorkError", { @@ -17,3 +89,28 @@ export class ScheduledWorkError extends Schema.TaggedError() cause: Schema.optional(Schema.Defect()), }, ) {} + +export const AmbientWorkConfigureInput = Schema.Struct({ + callerThreadId: ThreadId, + enabled: Schema.Boolean, + grantId: Schema.String, + prompt: TrimmedNonEmptyString.check(Schema.isMaxLength(16_000)), + idleDelayMs: Schema.Int.check(Schema.isBetween({ minimum: 60_000, maximum: 86_400_000 })), + minimumCycleMs: Schema.Int.check(Schema.isBetween({ minimum: 60_000, maximum: 86_400_000 })), + maxCyclesPerDay: Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 24 })), + timezone: Schema.String.check(Schema.isMaxLength(120)), + allowedHourStart: Schema.Int.check(Schema.isBetween({ minimum: 0, maximum: 23 })), + allowedHourEnd: Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 24 })), + allowUnknownQuota: Schema.Boolean, +}); +export type AmbientWorkConfigureInput = typeof AmbientWorkConfigureInput.Type; +export const AmbientWorkRecord = Schema.Struct({ + config: AmbientWorkConfigureInput, + lastInteractionAt: Schema.Number, + lastCycleAt: Schema.NullOr(Schema.Number), + day: Schema.String, + cycles: Schema.Int, + activeScheduleId: Schema.NullOr(Schema.String), + reason: Schema.NullOr(Schema.String), +}); +export type AmbientWorkRecord = typeof AmbientWorkRecord.Type; diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 73d914496d17..d93fc9b35c1e 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -1067,7 +1067,7 @@ export const ProjectSettingsOverrides = Schema.Struct({ enableAgentBrowserAccess: Schema.optionalKey(Schema.Boolean), enableAgentDeviceAccess: Schema.optionalKey(Schema.Boolean), agentToolCapabilities: Schema.optionalKey( - Schema.Array(OptionalAgentToolCapability).check(Schema.isMaxLength(2)), + Schema.Array(OptionalAgentToolCapability).check(Schema.isMaxLength(4)), ), enableMemoryAutoRecall: Schema.optionalKey(Schema.Boolean), textGenerationModelSelection: Schema.optionalKey(ModelSelection), @@ -1144,7 +1144,7 @@ export const ServerSettings = Schema.Struct({ */ enableAgentBrowserAccess: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), agentToolCapabilities: Schema.Array(OptionalAgentToolCapability) - .check(Schema.isMaxLength(2)) + .check(Schema.isMaxLength(4)) .pipe(Schema.withDecodingDefault(Effect.succeed([]))), enableMemoryAutoRecall: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), projectAgentBrowserAccessOverrides: Schema.Record(ProjectId, Schema.Boolean).pipe( @@ -1513,7 +1513,7 @@ export const ServerSettingsPatch = Schema.Struct({ continueThreadsAfterServerUpdate: Schema.optionalKey(Schema.Boolean), enableAgentBrowserAccess: Schema.optionalKey(Schema.Boolean), agentToolCapabilities: Schema.optionalKey( - Schema.Array(OptionalAgentToolCapability).check(Schema.isMaxLength(2)), + Schema.Array(OptionalAgentToolCapability).check(Schema.isMaxLength(4)), ), enableMemoryAutoRecall: Schema.optionalKey(Schema.Boolean), projectAgentBrowserAccessOverrides: Schema.optionalKey(