From 1ebb06aa67ac8cccff68503565b7897cfeaa238b Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 6 Oct 2026 03:52:00 +0000 Subject: [PATCH 1/2] fix(workflows): retry ses-throttled emails instead of dropping them The SES throttle reschedule did not re-attach the email queue parameters, so the retry resumed the Hog VM and the email was never sent. Re-attach them, the same way as the per-workflow and team cap reschedules. Co-Authored-By: Claude Opus 5.5 (1M context) --- nodejs/src/cdp/services/messaging/email.service.test.ts | 3 +++ nodejs/src/cdp/services/messaging/email.service.ts | 3 +++ 2 files changed, 6 insertions(+) diff --git a/nodejs/src/cdp/services/messaging/email.service.test.ts b/nodejs/src/cdp/services/messaging/email.service.test.ts index a50885b752b7..8be5f4a8085e 100644 --- a/nodejs/src/cdp/services/messaging/email.service.test.ts +++ b/nodejs/src/cdp/services/messaging/email.service.test.ts @@ -420,6 +420,9 @@ describe('EmailService', () => { // than the upper bound + scheduler overhead. expect(scheduledMs).toBeGreaterThanOrEqual(before + 400) expect(scheduledMs).toBeLessThan(before + 2000) + // Without the email payload the rescheduled dequeue resumes the Hog VM instead of + // retrying the send, so the throttled email is dropped rather than delayed. + expect(result.invocation.queueParameters).toEqual(invocation.queueParameters) // No business metric emitted on throttle — the eventual retry // will produce email_sent. expect(result.metrics ?? []).toEqual([]) diff --git a/nodejs/src/cdp/services/messaging/email.service.ts b/nodejs/src/cdp/services/messaging/email.service.ts index 8c906c3f8657..e936437ca005 100644 --- a/nodejs/src/cdp/services/messaging/email.service.ts +++ b/nodejs/src/cdp/services/messaging/email.service.ts @@ -627,6 +627,9 @@ export class EmailService { // catches the cases where SES disagrees with our estimate. throttled = true result.finished = false + // Re-attach the email payload for the same reason as the pacing delays above: + // without it the rescheduled dequeue resumes the Hog VM and drops the send. + result.invocation.queueParameters = params result.invocation.queueScheduledAt = DateTime.utc().plus({ milliseconds: error.retryAfterMs }) addLog('warn', `SES rate-limited (${error.errorCode}); rescheduling email in ${error.retryAfterMs}ms`) } else { From 9668491696973119592cfe494afaa279f87222f1 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 6 Oct 2026 04:29:51 +0000 Subject: [PATCH 2/2] chore(workflows): drop comments on the ses throttle retry Co-Authored-By: Claude Opus 5.5 (1M context) --- nodejs/src/cdp/services/messaging/email.service.test.ts | 2 -- nodejs/src/cdp/services/messaging/email.service.ts | 2 -- 2 files changed, 4 deletions(-) diff --git a/nodejs/src/cdp/services/messaging/email.service.test.ts b/nodejs/src/cdp/services/messaging/email.service.test.ts index 8be5f4a8085e..adf73135e11a 100644 --- a/nodejs/src/cdp/services/messaging/email.service.test.ts +++ b/nodejs/src/cdp/services/messaging/email.service.test.ts @@ -420,8 +420,6 @@ describe('EmailService', () => { // than the upper bound + scheduler overhead. expect(scheduledMs).toBeGreaterThanOrEqual(before + 400) expect(scheduledMs).toBeLessThan(before + 2000) - // Without the email payload the rescheduled dequeue resumes the Hog VM instead of - // retrying the send, so the throttled email is dropped rather than delayed. expect(result.invocation.queueParameters).toEqual(invocation.queueParameters) // No business metric emitted on throttle — the eventual retry // will produce email_sent. diff --git a/nodejs/src/cdp/services/messaging/email.service.ts b/nodejs/src/cdp/services/messaging/email.service.ts index e936437ca005..f1c3571e49f6 100644 --- a/nodejs/src/cdp/services/messaging/email.service.ts +++ b/nodejs/src/cdp/services/messaging/email.service.ts @@ -627,8 +627,6 @@ export class EmailService { // catches the cases where SES disagrees with our estimate. throttled = true result.finished = false - // Re-attach the email payload for the same reason as the pacing delays above: - // without it the rescheduled dequeue resumes the Hog VM and drops the send. result.invocation.queueParameters = params result.invocation.queueScheduledAt = DateTime.utc().plus({ milliseconds: error.retryAfterMs }) addLog('warn', `SES rate-limited (${error.errorCode}); rescheduling email in ${error.retryAfterMs}ms`)