From 1d77e427008ac950f3575d6c51a89f6879b4dfb2 Mon Sep 17 00:00:00 2001 From: Nicholas Tindle Date: Sat, 10 Oct 2026 00:58:11 -0500 Subject: [PATCH 1/2] fix(self-host): stop tracing idle scheduler and provisioning polls Self-host checks for due scheduled runs and for queued provisioning jobs every second, and traced every check: a schedule.dispatch span with its storage and SQL spans, plus a root sql.execute span. An instance with no traffic exported about 800 spans a minute into the bundled Motel store, which reaches its 1 GiB bound within hours and then evicts on every export, writing to disk constantly. A check that finds nothing now records nothing. Each scheduled operation a check finds runs in its own schedule.dispatch trace, so the background work under it keeps its dispatcher root, and each provisioning job is traced as before. Local runs the same scheduler and stops tracing its idle checks too. Co-Authored-By: Claude Opus 5.5 --- .changeset/untraced-idle-polls.md | 11 +++++++++++ .../server/src/implementation/provisioning.ts | 2 ++ packages/sdk/src/implementation/schedule-worker.ts | 13 ++++++++++--- 3 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 .changeset/untraced-idle-polls.md diff --git a/.changeset/untraced-idle-polls.md b/.changeset/untraced-idle-polls.md new file mode 100644 index 0000000000..8a03940249 --- /dev/null +++ b/.changeset/untraced-idle-polls.md @@ -0,0 +1,11 @@ +--- +"@executor-js/sdk": patch +"@executor-js/hosted-server": patch +--- + +The scheduler's check for due runs, every second on self-host and local, and +self-host's check for queued provisioning jobs, every second, no longer record +traces when they find nothing to do. Each scheduled operation a check finds is +still traced, in its own `schedule.dispatch` trace, and so is each provisioning +job. An idle self-host instance no longer fills the bundled Motel store with +polling spans. diff --git a/apps/hosted/server/src/implementation/provisioning.ts b/apps/hosted/server/src/implementation/provisioning.ts index e2e94b8345..377d9d6e81 100644 --- a/apps/hosted/server/src/implementation/provisioning.ts +++ b/apps/hosted/server/src/implementation/provisioning.ts @@ -210,6 +210,8 @@ export const drainProvisioning = (services: ProvisioningServices) => Effect.flatMap( Schema.decodeUnknownEffect(Schema.Array(Schema.Struct({ id: Schema.String }))), ), + // Self-host polls every second. The poll records nothing; the jobs it finds are traced. + Effect.withTracerEnabled(false), ); for (const job of jobs) yield* provision(job.id, services).pipe( diff --git a/packages/sdk/src/implementation/schedule-worker.ts b/packages/sdk/src/implementation/schedule-worker.ts index d768a15c93..ead6a165ec 100644 --- a/packages/sdk/src/implementation/schedule-worker.ts +++ b/packages/sdk/src/implementation/schedule-worker.ts @@ -23,11 +23,18 @@ export const startScheduleWorker = ( runner: config.runner, maxCandidates: config.concurrency, authorize, - execute: (operation) => pool.withPermitsIfAvailable(1)(operation).pipe(Effect.asVoid), + // Each operation a pass finds is one trace, rooted in a dispatch as a Cloud + // coordinator's pass is. + execute: (operation) => + pool + .withPermitsIfAvailable(1)( + operation.pipe(Effect.withSpan("schedule.dispatch"), Effect.withTracerEnabled(true)), + ) + .pipe(Effect.asVoid), }) .pipe( - // Each pass is one trace, as a Cloud coordinator's dispatch is. - Effect.withSpan("schedule.dispatch"), + // A pass runs every second; one that finds nothing to run records nothing. + Effect.withTracerEnabled(false), Effect.catch(() => Effect.logError("Scheduled dispatch failed")), Effect.forkIn(scope), Effect.asVoid, From 1af069e540da94df00f19a5dec9223027c9ee020 Mon Sep 17 00:00:00 2001 From: Nicholas Tindle Date: Sat, 10 Oct 2026 01:17:47 -0500 Subject: [PATCH 2/2] fix(self-host): stop tracing idle event delivery polls Self-host and local look for due event deliveries every two seconds, and traced every pass: an events.dispatch span with its delivery, subscription and retention statements, about 15 spans each. On an instance with no traffic that is about 450 spans a minute in the bundled Motel store, the largest idle source since the event worker arrived. A pass that finds nothing to deliver now records nothing. Each delivery attempt and each subscription a pass expires is still traced, in its own trace. Cloud's coordinator calls the same delivery code with tracing on and is unchanged. Co-Authored-By: Claude Opus 5.5 --- .changeset/untraced-event-poll.md | 8 ++++++++ packages/sdk/src/implementation/event-worker.ts | 3 ++- packages/sdk/src/implementation/events.ts | 12 ++++++++++-- 3 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 .changeset/untraced-event-poll.md diff --git a/.changeset/untraced-event-poll.md b/.changeset/untraced-event-poll.md new file mode 100644 index 0000000000..10ada67374 --- /dev/null +++ b/.changeset/untraced-event-poll.md @@ -0,0 +1,8 @@ +--- +"@executor-js/sdk": patch +--- + +Self-host and local check for due event deliveries every two seconds. That +check no longer records traces when it finds nothing to deliver; each delivery +attempt and each subscription it expires is still traced. An idle self-host +instance no longer fills the bundled Motel store with polling spans. diff --git a/packages/sdk/src/implementation/event-worker.ts b/packages/sdk/src/implementation/event-worker.ts index 12fb1d95a0..e4009f441a 100644 --- a/packages/sdk/src/implementation/event-worker.ts +++ b/packages/sdk/src/implementation/event-worker.ts @@ -12,7 +12,8 @@ export const deliverEvents = (executor: Executor) => yield* Effect.flatten(ScheduleHostReady); yield* executor.events.deliver({ maxDeliveries: batch }).pipe( Effect.repeat({ while: (more) => more }), - Effect.withSpan("events.dispatch"), + // A pass runs every two seconds; one that finds nothing to deliver records nothing. + Effect.withTracerEnabled(false), Effect.catch(() => Effect.logError("Event delivery failed")), Effect.repeat(Schedule.spaced("2 seconds")), ); diff --git a/packages/sdk/src/implementation/events.ts b/packages/sdk/src/implementation/events.ts index fde0f19d89..e3361d5e1e 100644 --- a/packages/sdk/src/implementation/events.ts +++ b/packages/sdk/src/implementation/events.ts @@ -808,7 +808,11 @@ export const makeEvents = (input: { limit: 100, }), ); - for (const row of lapsed) yield* stopWith(EventSubscriptionId.make(row.id), "expired"); + // A host may poll without tracing; the work a pass finds is traced all the same. + for (const row of lapsed) + yield* stopWith(EventSubscriptionId.make(row.id), "expired").pipe( + Effect.withTracerEnabled(true), + ); const due = yield* query(() => db.findMany("eventDeliveries", { where: (b) => @@ -825,7 +829,11 @@ export const makeEvents = (input: { due, (row) => claim(row.id).pipe( - Effect.flatMap((claimed) => (claimed === null ? Effect.void : attempt(claimed))), + Effect.flatMap((claimed) => + claimed === null + ? Effect.void + : attempt(claimed).pipe(Effect.withTracerEnabled(true)), + ), Effect.catch(() => Effect.logWarning("Event delivery attempt failed")), ), { concurrency: 8, discard: true },