From 1ebb06aa67ac8cccff68503565b7897cfeaa238b Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 6 Oct 2026 03:52:00 +0000 Subject: [PATCH 1/6] fix(workflows): retry ses-throttled emails instead of dropping them The SES throttle reschedule did not re-attach the email queue parameters, so the retry resumed the Hog VM and the email was never sent. Re-attach them, the same way as the per-workflow and team cap reschedules. Co-Authored-By: Claude Opus 5.5 (1M context) --- nodejs/src/cdp/services/messaging/email.service.test.ts | 3 +++ nodejs/src/cdp/services/messaging/email.service.ts | 3 +++ 2 files changed, 6 insertions(+) diff --git a/nodejs/src/cdp/services/messaging/email.service.test.ts b/nodejs/src/cdp/services/messaging/email.service.test.ts index a50885b752b7..8be5f4a8085e 100644 --- a/nodejs/src/cdp/services/messaging/email.service.test.ts +++ b/nodejs/src/cdp/services/messaging/email.service.test.ts @@ -420,6 +420,9 @@ describe('EmailService', () => { // than the upper bound + scheduler overhead. expect(scheduledMs).toBeGreaterThanOrEqual(before + 400) expect(scheduledMs).toBeLessThan(before + 2000) + // Without the email payload the rescheduled dequeue resumes the Hog VM instead of + // retrying the send, so the throttled email is dropped rather than delayed. + expect(result.invocation.queueParameters).toEqual(invocation.queueParameters) // No business metric emitted on throttle — the eventual retry // will produce email_sent. expect(result.metrics ?? []).toEqual([]) diff --git a/nodejs/src/cdp/services/messaging/email.service.ts b/nodejs/src/cdp/services/messaging/email.service.ts index 8c906c3f8657..e936437ca005 100644 --- a/nodejs/src/cdp/services/messaging/email.service.ts +++ b/nodejs/src/cdp/services/messaging/email.service.ts @@ -627,6 +627,9 @@ export class EmailService { // catches the cases where SES disagrees with our estimate. throttled = true result.finished = false + // Re-attach the email payload for the same reason as the pacing delays above: + // without it the rescheduled dequeue resumes the Hog VM and drops the send. + result.invocation.queueParameters = params result.invocation.queueScheduledAt = DateTime.utc().plus({ milliseconds: error.retryAfterMs }) addLog('warn', `SES rate-limited (${error.errorCode}); rescheduling email in ${error.retryAfterMs}ms`) } else { From 9668491696973119592cfe494afaa279f87222f1 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 6 Oct 2026 04:29:51 +0000 Subject: [PATCH 2/6] chore(workflows): drop comments on the ses throttle retry Co-Authored-By: Claude Opus 5.5 (1M context) --- nodejs/src/cdp/services/messaging/email.service.test.ts | 2 -- nodejs/src/cdp/services/messaging/email.service.ts | 2 -- 2 files changed, 4 deletions(-) diff --git a/nodejs/src/cdp/services/messaging/email.service.test.ts b/nodejs/src/cdp/services/messaging/email.service.test.ts index 8be5f4a8085e..adf73135e11a 100644 --- a/nodejs/src/cdp/services/messaging/email.service.test.ts +++ b/nodejs/src/cdp/services/messaging/email.service.test.ts @@ -420,8 +420,6 @@ describe('EmailService', () => { // than the upper bound + scheduler overhead. expect(scheduledMs).toBeGreaterThanOrEqual(before + 400) expect(scheduledMs).toBeLessThan(before + 2000) - // Without the email payload the rescheduled dequeue resumes the Hog VM instead of - // retrying the send, so the throttled email is dropped rather than delayed. expect(result.invocation.queueParameters).toEqual(invocation.queueParameters) // No business metric emitted on throttle — the eventual retry // will produce email_sent. diff --git a/nodejs/src/cdp/services/messaging/email.service.ts b/nodejs/src/cdp/services/messaging/email.service.ts index e936437ca005..f1c3571e49f6 100644 --- a/nodejs/src/cdp/services/messaging/email.service.ts +++ b/nodejs/src/cdp/services/messaging/email.service.ts @@ -627,8 +627,6 @@ export class EmailService { // catches the cases where SES disagrees with our estimate. throttled = true result.finished = false - // Re-attach the email payload for the same reason as the pacing delays above: - // without it the rescheduled dequeue resumes the Hog VM and drops the send. result.invocation.queueParameters = params result.invocation.queueScheduledAt = DateTime.utc().plus({ milliseconds: error.retryAfterMs }) addLog('warn', `SES rate-limited (${error.errorCode}); rescheduling email in ${error.retryAfterMs}ms`) From 70e5d09b5c75d552fe0822a847e999a2fb6f2a36 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 6 Oct 2026 05:56:29 +0000 Subject: [PATCH 3/6] chore(workflows): send test email through a local ses fake --- .github/workflows/ci-nodejs.yml | 10 +- docker-compose.base.yml | 15 ++ docker-compose.dev.yml | 7 + .../engineering/developing-locally.md | 4 + .../messaging/email.service.ses.test.ts | 223 ++++++++++++++++++ nodejs/tests/helpers/ses.ts | 119 ++++++++++ 6 files changed, 376 insertions(+), 2 deletions(-) create mode 100644 nodejs/src/cdp/services/messaging/email.service.ses.test.ts create mode 100644 nodejs/tests/helpers/ses.ts diff --git a/.github/workflows/ci-nodejs.yml b/.github/workflows/ci-nodejs.yml index 596e3947233e..d82b293c59c7 100644 --- a/.github/workflows/ci-nodejs.yml +++ b/.github/workflows/ci-nodejs.yml @@ -542,13 +542,19 @@ jobs: install-command: pnpm --filter=@posthog/nodejs... install --frozen-lockfile token: ${{ steps.setup-gh-token.outputs.token || github.token }} - - name: Wait for Clickhouse, Redis, Kafka, DynamoDB, SeaweedFS + - name: Wait for Clickhouse, Redis, Kafka, DynamoDB, SeaweedFS, SES # seaweedfs backs the blob-store integration tests; the profile-scoped # stack launch above never starts it (it sits in the replay/agents # profiles), so start it explicitly here — no profiles overlay in play. # objectstorage starts by default but is listed so --wait blocks on its # bootstrap sentinel: it returns InvalidAccessKeyId until that finishes. - run: docker compose -f docker-compose.dev.yml up kafka redis7 valkey-cluster clickhouse maildev dynamodb objectstorage seaweedfs -d --wait + run: | + services=(kafka redis7 valkey-cluster clickhouse maildev dynamodb objectstorage seaweedfs) + # The workflow also runs on unrebased branches whose compose file has no SES service. + if docker compose -f docker-compose.dev.yml config --services | grep -qx ses; then + services+=(ses) + fi + docker compose -f docker-compose.dev.yml up "${services[@]}" -d --wait - name: Create Kafka topics run: docker compose -f docker-compose.dev.yml up kafka-init diff --git a/docker-compose.base.yml b/docker-compose.base.yml index 91390aa113e1..3f0bf7eb04e2 100644 --- a/docker-compose.base.yml +++ b/docker-compose.base.yml @@ -331,6 +331,21 @@ services: profiles: ['batch-exports'] command: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --location /data + ses: + image: ghcr.io/domdomegg/aws-ses-v2-local:2.10.2@sha256:1c0db32e27b52d7ac7c16048e074ffc176c9aa7f117e63babd0d911c5ef8330f + restart: on-failure + healthcheck: + test: + [ + 'CMD', + 'node', + '-e', + "fetch('http://127.0.0.1:8005/health-check').then(r => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))", + ] + interval: 3s + timeout: 1s + retries: 10 + maildev: image: maildev/maildev:2.0.5 restart: on-failure diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 32ca54beca2f..349998aac0bb 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -640,6 +640,13 @@ services: ports: - '10000:10000' + ses: + extends: + file: docker-compose.base.yml + service: ses + ports: + - '127.0.0.1:4566:8005' + maildev: extends: file: docker-compose.base.yml diff --git a/docs/published/handbook/engineering/developing-locally.md b/docs/published/handbook/engineering/developing-locally.md index f0ad66188c60..b29e831840f4 100644 --- a/docs/published/handbook/engineering/developing-locally.md +++ b/docs/published/handbook/engineering/developing-locally.md @@ -465,6 +465,10 @@ This allows you to easily confirm that emails are being sent and formatted corre Emails sent via SMTP are stored in HTML files in `posthog/templates/*/*.html`. They use Django Template Language (DTL). +Workflow SES tests use the pinned `aws-ses-v2-local` compose service at `http://127.0.0.1:4566`, which also exposes its inbox at `/store`. Start just that service with `docker compose -f docker-compose.dev.yml up ses -d --wait` and run `hogli test nodejs/src/cdp/services/messaging/email.service.ses.test.ts`. + +The `LocalSes` test helper gives each test its own HTTP proxy for request capture and persistent SESv2 errors. It reads only the message IDs that proxy delivered, without clearing the shared inbox. Tests use invented AWS credentials and distinguish SDK HTTP attempts from retries of the workflow invocation. The inbox schema and delivery assertions check the fake's contract when its image changes; the fake does not validate AWS tenant or configuration-set provisioning. + ## Extra: Integrating with slack You can connect to a real slack workspace in your local development setup by adding the required slack environment variables to your `.env` file. diff --git a/nodejs/src/cdp/services/messaging/email.service.ses.test.ts b/nodejs/src/cdp/services/messaging/email.service.ses.test.ts new file mode 100644 index 000000000000..ffef3839d92e --- /dev/null +++ b/nodejs/src/cdp/services/messaging/email.service.ses.test.ts @@ -0,0 +1,223 @@ +import { createExampleInvocation, insertIntegration } from '~/cdp/_tests/fixtures' +import { CyclotronInvocationQueueParametersEmailType } from '~/cdp/schema/cyclotron' +import { CyclotronJobInvocationHogFunction } from '~/cdp/types' +import { closeHub, createHub } from '~/common/utils/db/hub' +import { LocalSes } from '~/tests/helpers/ses' +import { createTestTeamFixture } from '~/tests/helpers/sql' + +import { Hub } from '../../../types' +import { RecipientsManagerService } from '../managers/recipients-manager.service' +import { TeamWorkflowsConfigService } from '../managers/team-workflows-config.service' +import { EmailSuppressionService, emailSuppressionConfigFromEnv } from './email-suppression.service' +import { EmailService } from './email.service' +import { EmailTrackingCodeSigner } from './helpers/tracking-code' + +describe('EmailService with local SES', () => { + let hub: Hub + let service: EmailService + let invocation: CyclotronJobInvocationHogFunction + let params: CyclotronInvocationQueueParametersEmailType + let ses: LocalSes + const originalEnv = { ...process.env } + + const createService = (endpoint: string): EmailService => + new EmailService( + { + sesAccessKeyId: 'local-ses-test', + sesSecretAccessKey: 'local-ses-test', + sesRegion: 'us-east-1', + sesEndpoint: endpoint, + sesTrackedConfigurationSet: 'local-tracked', + sesUntrackedConfigurationSet: 'local-untracked', + }, + hub.integrationManager, + new TeamWorkflowsConfigService(hub.postgres, hub.pubSub), + hub.ENCRYPTION_SALT_KEYS, + hub.SITE_URL, + new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL), + new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()), + new RecipientsManagerService(hub.postgres) + ) + + beforeEach(async () => { + process.env.AWS_ACCESS_KEY_ID = 'local-ses-test' + process.env.AWS_SECRET_ACCESS_KEY = 'local-ses-test' + delete process.env.AWS_SESSION_TOKEN + process.env.AWS_MAX_ATTEMPTS = '1' + ses = new LocalSes() + await ses.start() + hub = await createHub({ SITE_URL: 'http://localhost:8000' }) + const { team } = await createTestTeamFixture(hub.postgres) + const integration = await insertIntegration(hub.postgres, team.id, { + kind: 'email', + config: { + email: 'sender@example.com', + name: 'Example Sender', + domain: 'example.com', + verified: true, + provider: 'ses', + }, + }) + service = createService(ses.endpoint) + invocation = createExampleInvocation({ team_id: team.id, metadata: { tracking_enabled: false } }) + params = { + type: 'email', + from: { integrationId: integration.id }, + to: { email: `${invocation.id}@example.com`, name: 'Example Recipient' }, + cc: 'copy@example.com', + bcc: 'hidden@example.com', + replyTo: 'reply@example.com', + subject: 'Local SES delivery', + text: 'A plain text message.', + html: '

An HTML message.

', + } + invocation.queueParameters = params + }) + + afterEach(async () => { + service?.sesV2Client?.destroy() + await ses?.stop() + if (hub) { + await closeHub(hub) + } + process.env = { ...originalEnv } + }) + + it('delivers the email through the real SDK with bodies, addresses and custom headers', async () => { + const result = await service.executeSendEmail(invocation) + + expect(result).toMatchObject({ finished: true }) + expect(result.error).toBeUndefined() + expect(result.metrics).toEqual( + expect.arrayContaining([expect.objectContaining({ metric_name: 'email_sent', count: 1 })]) + ) + expect(ses.requests).toHaveLength(1) + expect(ses.requests[0]).toMatchObject({ + TenantName: `team-${invocation.teamId}`, + ConfigurationSetName: 'local-untracked', + FeedbackForwardingEmailAddress: 'sender@example.com', + EmailTags: [{ Name: 'ph_id', Value: expect.any(String) }], + }) + const emails = await ses.getEmails() + expect(emails).toHaveLength(1) + expect(emails[0]).toMatchObject({ + messageId: expect.any(String), + from: '"Example Sender" ', + destination: { + to: [`"Example Recipient" <${params.to.email}>`], + cc: ['copy@example.com'], + bcc: ['hidden@example.com'], + }, + replyTo: ['reply@example.com'], + subject: 'Local SES delivery', + body: { text: 'A plain text message.', html: '

An HTML message.

' }, + headers: expect.arrayContaining([ + { name: 'Auto-Submitted', value: 'auto-generated' }, + { name: 'List-Unsubscribe-Post', value: 'List-Unsubscribe=One-Click' }, + { name: 'List-Unsubscribe', value: expect.stringContaining('http') }, + { name: 'X-PostHog-Tracking-Code', value: expect.any(String) }, + ]), + }) + }) + + it.each([1, 3])('reschedules after %i SDK attempt(s) and sends its email payload on retry', async (attempts) => { + process.env.AWS_MAX_ATTEMPTS = String(attempts) + service.sesV2Client?.destroy() + service = createService(ses.endpoint) + ses.setError('TooManyRequestsException') + invocation.queuePriority = 1 + + const throttled = await service.executeSendEmail(invocation) + + expect(throttled).toMatchObject({ finished: false, metrics: [] }) + expect(throttled.error).toBeUndefined() + expect(throttled.invocation.queueParameters).toEqual(params) + expect(throttled.invocation.queuePriority).toBe(1) + expect(throttled.invocation.queueScheduledAt?.isValid).toBe(true) + expect(ses.requests).toHaveLength(attempts) + expect(await ses.getEmails()).toEqual([]) + + ses.setError(undefined) + const retry = await service.executeSendEmail(throttled.invocation) + + expect(retry).toMatchObject({ finished: true }) + expect(retry.error).toBeUndefined() + expect(retry.metrics).toEqual( + expect.arrayContaining([expect.objectContaining({ metric_name: 'email_sent', count: 1 })]) + ) + expect(ses.requests).toHaveLength(attempts + 1) + expect(ses.requests[attempts]).toMatchObject({ + FromEmailAddress: '"Example Sender" ', + Destination: { + ToAddresses: [`"Example Recipient" <${params.to.email}>`], + CcAddresses: ['copy@example.com'], + BccAddresses: ['hidden@example.com'], + }, + Content: { + Simple: { + Subject: { Data: 'Local SES delivery', Charset: 'UTF-8' }, + Body: { + Text: { Data: 'A plain text message.', Charset: 'UTF-8' }, + Html: { Data: '

An HTML message.

', Charset: 'UTF-8' }, + }, + }, + }, + }) + expect(await ses.getEmails()).toEqual([ + expect.objectContaining({ subject: 'Local SES delivery', body: { text: params.text, html: params.html } }), + ]) + }) + + it.each(['LimitExceededException', 'SendingPausedException'] as const)( + 'fails an HTTP %s without rescheduling or delivering mail', + async (error) => { + ses.setError(error) + + const result = await service.executeSendEmail(invocation) + + expect(result).toMatchObject({ + finished: true, + error: `Failed to send email via SES: Local SES injected ${error}`, + metrics: [expect.objectContaining({ metric_name: 'email_failed', count: 1 })], + }) + expect(result.invocation.queueParameters).toBeUndefined() + expect(result.invocation.queueScheduledAt).toBeUndefined() + expect(ses.requests).toHaveLength(1) + expect(await ses.getEmails()).toEqual([]) + } + ) + + it('isolates concurrent proxy faults and inbox reads for identical recipients', async () => { + const otherSes = new LocalSes() + await otherSes.start() + const otherService = createService(otherSes.endpoint) + ses.setError('TooManyRequestsException') + + try { + const [throttled, delivered] = await Promise.all([ + service.executeSendEmail(invocation), + otherService.executeSendEmail(invocation), + ]) + expect(throttled).toMatchObject({ finished: false, metrics: [] }) + expect(delivered).toMatchObject({ finished: true }) + expect(delivered.error).toBeUndefined() + expect(await ses.getEmails()).toEqual([]) + const otherEmails = await otherSes.getEmails() + expect(otherEmails).toHaveLength(1) + + ses.setError(undefined) + const retry = await service.executeSendEmail(throttled.invocation) + expect(retry.error).toBeUndefined() + const emails = await ses.getEmails() + expect(emails).toHaveLength(1) + expect(emails[0].destination).toEqual(otherEmails[0].destination) + expect(emails[0].messageId).not.toBe(otherEmails[0].messageId) + expect(await otherSes.getEmails()).toEqual(otherEmails) + expect(ses.requests).toHaveLength(2) + expect(otherSes.requests).toHaveLength(1) + } finally { + otherService.sesV2Client?.destroy() + await otherSes.stop() + } + }) +}) diff --git a/nodejs/tests/helpers/ses.ts b/nodejs/tests/helpers/ses.ts new file mode 100644 index 000000000000..f640a72da43e --- /dev/null +++ b/nodejs/tests/helpers/ses.ts @@ -0,0 +1,119 @@ +import { SendEmailCommandInput } from '@aws-sdk/client-sesv2' +import { IncomingMessage, Server, ServerResponse, createServer } from 'node:http' +import { AddressInfo } from 'node:net' +import { z } from 'zod' + +import { parseJSON } from '~/common/utils/json-parse' + +const fetchLocalSes = globalThis.fetch + +const storedEmailSchema = z.object({ + messageId: z.string().min(1), + from: z.string(), + destination: z.object({ to: z.array(z.string()), cc: z.array(z.string()), bcc: z.array(z.string()) }), + replyTo: z.array(z.string()), + subject: z.string(), + body: z.object({ text: z.string().optional(), html: z.string().optional() }), + headers: z.array(z.object({ name: z.string(), value: z.string() })), +}) + +export type LocalSesEmail = z.infer +export type LocalSesError = 'TooManyRequestsException' | 'LimitExceededException' | 'SendingPausedException' + +export class LocalSes { + readonly requests: SendEmailCommandInput[] = [] + private readonly messageIds = new Set() + private readonly server: Server + private error: LocalSesError | undefined + + constructor(private readonly fakeEndpoint = 'http://127.0.0.1:4566') { + this.server = createServer((request, response) => { + void this.handleRequest(request, response).catch((error: Error) => { + response.writeHead(502, { 'content-type': 'application/json' }) + response.end(JSON.stringify({ message: error.message })) + }) + }) + } + + get endpoint(): string { + return `http://127.0.0.1:${(this.server.address() as AddressInfo).port}` + } + + async start(): Promise { + await new Promise((resolve, reject) => { + this.server.once('error', reject) + this.server.listen(0, '127.0.0.1', () => { + this.server.removeListener('error', reject) + resolve() + }) + }) + } + + async stop(): Promise { + await new Promise((resolve, reject) => { + this.server.close((error) => (error ? reject(error) : resolve())) + }) + } + + setError(error: LocalSesError | undefined): void { + this.error = error + } + + async getEmails(): Promise { + const response = await fetchLocalSes(`${this.fakeEndpoint}/store`, { signal: AbortSignal.timeout(10_000) }) + if (!response.ok) { + throw new Error(`Local SES inbox returned HTTP ${response.status}`) + } + const store = z.object({ emails: z.array(storedEmailSchema) }).safeParse(await response.json()) + if (!store.success) { + throw new Error(`Local SES /store contract changed: ${store.error.message}`) + } + return store.data.emails.filter((email) => this.messageIds.has(email.messageId)) + } + + private async handleRequest(request: IncomingMessage, response: ServerResponse): Promise { + if (request.method !== 'POST' || request.url !== '/v2/email/outbound-emails') { + response.writeHead(404) + response.end() + return + } + const chunks: Buffer[] = [] + for await (const chunk of request) { + chunks.push(Buffer.from(chunk)) + } + const body = Buffer.concat(chunks).toString('utf8') + this.requests.push(parseJSON(body) as SendEmailCommandInput) + + if (this.error) { + response.writeHead(this.error === 'TooManyRequestsException' ? 429 : 400, { + 'content-type': 'application/json', + 'x-amzn-errortype': this.error, + }) + response.end(JSON.stringify({ message: `Local SES injected ${this.error}` })) + return + } + + const headers: Record = {} + for (const [name, value] of Object.entries(request.headers)) { + if (typeof value === 'string' && name !== 'host' && name !== 'connection') { + headers[name] = value + } + } + const upstream = await fetchLocalSes(`${this.fakeEndpoint}/v2/email/outbound-emails`, { + method: 'POST', + headers, + body, + signal: AbortSignal.timeout(10_000), + }) + const responseBody = await upstream.text() + if (upstream.ok) { + const result = z.object({ MessageId: z.string().min(1) }).safeParse(parseJSON(responseBody)) + if (!result.success) { + throw new Error(`Local SES SendEmail contract changed: ${result.error.message}`) + } + this.messageIds.add(result.data.MessageId) + } + response.writeHead(upstream.status, { 'content-type': 'application/json' }) + response.end(responseBody) + } +} From 748ef2d7c1ee8c6fc5bb081a64dd5abbb1c5a6bb Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 6 Oct 2026 06:13:15 +0000 Subject: [PATCH 4/6] chore(workflows): isolate local ses tests and opt in service --- .github/workflows/ci-nodejs.yml | 4 ++- docker-compose.dev.yml | 1 + .../engineering/developing-locally.md | 2 +- .../messaging/email.service.ses.test.ts | 20 +++++++++++++ nodejs/tests/helpers/ses.ts | 29 +++++++++++++++---- 5 files changed, 49 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-nodejs.yml b/.github/workflows/ci-nodejs.yml index d82b293c59c7..58eb6d86c636 100644 --- a/.github/workflows/ci-nodejs.yml +++ b/.github/workflows/ci-nodejs.yml @@ -156,6 +156,8 @@ jobs: # Anything here invalidates jest's view of the module graph, or changes # the native addons every suite links, so no narrowed run can be trusted. node_full: + - 'docker-compose.base.yml' + - 'docker-compose.dev.yml' # readFileSync, so jest cannot reach the parity test through --findRelatedTests. - 'posthog/cdp/filter_globals.json' - 'common/hogvm/typescript/src/**' @@ -551,7 +553,7 @@ jobs: run: | services=(kafka redis7 valkey-cluster clickhouse maildev dynamodb objectstorage seaweedfs) # The workflow also runs on unrebased branches whose compose file has no SES service. - if docker compose -f docker-compose.dev.yml config --services | grep -qx ses; then + if docker compose -f docker-compose.dev.yml --profile ses config --services | grep -qx ses; then services+=(ses) fi docker compose -f docker-compose.dev.yml up "${services[@]}" -d --wait diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 349998aac0bb..2579e6f7a415 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -641,6 +641,7 @@ services: - '10000:10000' ses: + profiles: [ses] extends: file: docker-compose.base.yml service: ses diff --git a/docs/published/handbook/engineering/developing-locally.md b/docs/published/handbook/engineering/developing-locally.md index b29e831840f4..77c7b93c09a1 100644 --- a/docs/published/handbook/engineering/developing-locally.md +++ b/docs/published/handbook/engineering/developing-locally.md @@ -465,7 +465,7 @@ This allows you to easily confirm that emails are being sent and formatted corre Emails sent via SMTP are stored in HTML files in `posthog/templates/*/*.html`. They use Django Template Language (DTL). -Workflow SES tests use the pinned `aws-ses-v2-local` compose service at `http://127.0.0.1:4566`, which also exposes its inbox at `/store`. Start just that service with `docker compose -f docker-compose.dev.yml up ses -d --wait` and run `hogli test nodejs/src/cdp/services/messaging/email.service.ses.test.ts`. +Workflow SES tests use the pinned `aws-ses-v2-local` compose service at `http://127.0.0.1:4566`, which also exposes its inbox at `/store`. With the rest of the dev stack running, start this optional service with `docker compose -f docker-compose.dev.yml up ses -d --wait` and run `hogli test nodejs/src/cdp/services/messaging/email.service.ses.test.ts`. The `LocalSes` test helper gives each test its own HTTP proxy for request capture and persistent SESv2 errors. It reads only the message IDs that proxy delivered, without clearing the shared inbox. Tests use invented AWS credentials and distinguish SDK HTTP attempts from retries of the workflow invocation. The inbox schema and delivery assertions check the fake's contract when its image changes; the fake does not validate AWS tenant or configuration-set provisioning. diff --git a/nodejs/src/cdp/services/messaging/email.service.ses.test.ts b/nodejs/src/cdp/services/messaging/email.service.ses.test.ts index ffef3839d92e..309a60b20675 100644 --- a/nodejs/src/cdp/services/messaging/email.service.ses.test.ts +++ b/nodejs/src/cdp/services/messaging/email.service.ses.test.ts @@ -1,3 +1,5 @@ +import { SESv2Client, SendEmailCommand } from '@aws-sdk/client-sesv2' + import { createExampleInvocation, insertIntegration } from '~/cdp/_tests/fixtures' import { CyclotronInvocationQueueParametersEmailType } from '~/cdp/schema/cyclotron' import { CyclotronJobInvocationHogFunction } from '~/cdp/types' @@ -43,6 +45,7 @@ describe('EmailService with local SES', () => { process.env.AWS_ACCESS_KEY_ID = 'local-ses-test' process.env.AWS_SECRET_ACCESS_KEY = 'local-ses-test' delete process.env.AWS_SESSION_TOKEN + delete process.env.AWS_PROFILE process.env.AWS_MAX_ATTEMPTS = '1' ses = new LocalSes() await ses.start() @@ -188,6 +191,23 @@ describe('EmailService with local SES', () => { ) it('isolates concurrent proxy faults and inbox reads for identical recipients', async () => { + const unrelatedClient = new SESv2Client({ region: 'us-east-1', endpoint: 'http://127.0.0.1:4566' }) + try { + await unrelatedClient.send( + new SendEmailCommand({ + FromEmailAddress: 'sender@example.com', + Destination: { ToAddresses: [params.to.email] }, + Content: { + Simple: { + Subject: { Data: 'Unrelated email without headers' }, + Body: { Text: { Data: 'Another inbox writer.' } }, + }, + }, + }) + ) + } finally { + unrelatedClient.destroy() + } const otherSes = new LocalSes() await otherSes.start() const otherService = createService(otherSes.endpoint) diff --git a/nodejs/tests/helpers/ses.ts b/nodejs/tests/helpers/ses.ts index f640a72da43e..1dcf7f748d41 100644 --- a/nodejs/tests/helpers/ses.ts +++ b/nodejs/tests/helpers/ses.ts @@ -59,16 +59,36 @@ export class LocalSes { this.error = error } + private async fetchFake(path: string, options: RequestInit = {}): Promise { + try { + return await fetchLocalSes(`${this.fakeEndpoint}${path}`, { + ...options, + signal: AbortSignal.timeout(10_000), + }) + } catch (error) { + const cause = error instanceof Error && error.cause instanceof Error ? error.cause.message : String(error) + throw new Error(`Local SES fake unreachable at ${this.fakeEndpoint}: ${cause}`) + } + } + async getEmails(): Promise { - const response = await fetchLocalSes(`${this.fakeEndpoint}/store`, { signal: AbortSignal.timeout(10_000) }) + const response = await this.fetchFake('/store') if (!response.ok) { throw new Error(`Local SES inbox returned HTTP ${response.status}`) } - const store = z.object({ emails: z.array(storedEmailSchema) }).safeParse(await response.json()) + const store = z + .object({ emails: z.array(z.object({ messageId: z.string().min(1) }).passthrough()) }) + .safeParse(await response.json()) if (!store.success) { throw new Error(`Local SES /store contract changed: ${store.error.message}`) } - return store.data.emails.filter((email) => this.messageIds.has(email.messageId)) + const emails = z + .array(storedEmailSchema) + .safeParse(store.data.emails.filter((email) => this.messageIds.has(email.messageId))) + if (!emails.success) { + throw new Error(`Local SES /store contract changed: ${emails.error.message}`) + } + return emails.data } private async handleRequest(request: IncomingMessage, response: ServerResponse): Promise { @@ -99,11 +119,10 @@ export class LocalSes { headers[name] = value } } - const upstream = await fetchLocalSes(`${this.fakeEndpoint}/v2/email/outbound-emails`, { + const upstream = await this.fetchFake('/v2/email/outbound-emails', { method: 'POST', headers, body, - signal: AbortSignal.timeout(10_000), }) const responseBody = await upstream.text() if (upstream.ok) { From a9a434bfd31d3f1c868f1be02416ca9012114b1f Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 6 Oct 2026 06:19:13 +0000 Subject: [PATCH 5/6] chore(workflows): preserve native ses connection errors --- nodejs/tests/helpers/ses.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nodejs/tests/helpers/ses.ts b/nodejs/tests/helpers/ses.ts index 1dcf7f748d41..31f3bb7b70b8 100644 --- a/nodejs/tests/helpers/ses.ts +++ b/nodejs/tests/helpers/ses.ts @@ -1,6 +1,7 @@ import { SendEmailCommandInput } from '@aws-sdk/client-sesv2' import { IncomingMessage, Server, ServerResponse, createServer } from 'node:http' import { AddressInfo } from 'node:net' +import { isNativeError } from 'node:util/types' import { z } from 'zod' import { parseJSON } from '~/common/utils/json-parse' @@ -66,7 +67,7 @@ export class LocalSes { signal: AbortSignal.timeout(10_000), }) } catch (error) { - const cause = error instanceof Error && error.cause instanceof Error ? error.cause.message : String(error) + const cause = isNativeError(error) && isNativeError(error.cause) ? error.cause.message : String(error) throw new Error(`Local SES fake unreachable at ${this.fakeEndpoint}: ${cause}`) } } From 902b9fdeb2c0126cda76d26c7409b41514c38208 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 12 Oct 2026 03:19:16 +0000 Subject: [PATCH 6/6] chore(workflows): answer ses sends in the test proxy LocalSes now returns a MessageId and records delivered emails itself, so the tests no longer need the aws-ses-v2-local compose service. This drops the service, its CI wait step and node_full trigger, the docs paragraph, and the shared-inbox isolation test. Port 4566 stays dead locally, which keeps an unconfigured send off real AWS. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci-nodejs.yml | 12 +- docker-compose.base.yml | 15 --- docker-compose.dev.yml | 8 -- .../engineering/developing-locally.md | 4 - .../messaging/email.service.ses.test.ts | 53 -------- nodejs/tests/helpers/ses.ts | 121 +++++++----------- 6 files changed, 48 insertions(+), 165 deletions(-) diff --git a/.github/workflows/ci-nodejs.yml b/.github/workflows/ci-nodejs.yml index 58eb6d86c636..596e3947233e 100644 --- a/.github/workflows/ci-nodejs.yml +++ b/.github/workflows/ci-nodejs.yml @@ -156,8 +156,6 @@ jobs: # Anything here invalidates jest's view of the module graph, or changes # the native addons every suite links, so no narrowed run can be trusted. node_full: - - 'docker-compose.base.yml' - - 'docker-compose.dev.yml' # readFileSync, so jest cannot reach the parity test through --findRelatedTests. - 'posthog/cdp/filter_globals.json' - 'common/hogvm/typescript/src/**' @@ -544,19 +542,13 @@ jobs: install-command: pnpm --filter=@posthog/nodejs... install --frozen-lockfile token: ${{ steps.setup-gh-token.outputs.token || github.token }} - - name: Wait for Clickhouse, Redis, Kafka, DynamoDB, SeaweedFS, SES + - name: Wait for Clickhouse, Redis, Kafka, DynamoDB, SeaweedFS # seaweedfs backs the blob-store integration tests; the profile-scoped # stack launch above never starts it (it sits in the replay/agents # profiles), so start it explicitly here — no profiles overlay in play. # objectstorage starts by default but is listed so --wait blocks on its # bootstrap sentinel: it returns InvalidAccessKeyId until that finishes. - run: | - services=(kafka redis7 valkey-cluster clickhouse maildev dynamodb objectstorage seaweedfs) - # The workflow also runs on unrebased branches whose compose file has no SES service. - if docker compose -f docker-compose.dev.yml --profile ses config --services | grep -qx ses; then - services+=(ses) - fi - docker compose -f docker-compose.dev.yml up "${services[@]}" -d --wait + run: docker compose -f docker-compose.dev.yml up kafka redis7 valkey-cluster clickhouse maildev dynamodb objectstorage seaweedfs -d --wait - name: Create Kafka topics run: docker compose -f docker-compose.dev.yml up kafka-init diff --git a/docker-compose.base.yml b/docker-compose.base.yml index 3f0bf7eb04e2..91390aa113e1 100644 --- a/docker-compose.base.yml +++ b/docker-compose.base.yml @@ -331,21 +331,6 @@ services: profiles: ['batch-exports'] command: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --location /data - ses: - image: ghcr.io/domdomegg/aws-ses-v2-local:2.10.2@sha256:1c0db32e27b52d7ac7c16048e074ffc176c9aa7f117e63babd0d911c5ef8330f - restart: on-failure - healthcheck: - test: - [ - 'CMD', - 'node', - '-e', - "fetch('http://127.0.0.1:8005/health-check').then(r => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))", - ] - interval: 3s - timeout: 1s - retries: 10 - maildev: image: maildev/maildev:2.0.5 restart: on-failure diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 2579e6f7a415..32ca54beca2f 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -640,14 +640,6 @@ services: ports: - '10000:10000' - ses: - profiles: [ses] - extends: - file: docker-compose.base.yml - service: ses - ports: - - '127.0.0.1:4566:8005' - maildev: extends: file: docker-compose.base.yml diff --git a/docs/published/handbook/engineering/developing-locally.md b/docs/published/handbook/engineering/developing-locally.md index 77c7b93c09a1..f0ad66188c60 100644 --- a/docs/published/handbook/engineering/developing-locally.md +++ b/docs/published/handbook/engineering/developing-locally.md @@ -465,10 +465,6 @@ This allows you to easily confirm that emails are being sent and formatted corre Emails sent via SMTP are stored in HTML files in `posthog/templates/*/*.html`. They use Django Template Language (DTL). -Workflow SES tests use the pinned `aws-ses-v2-local` compose service at `http://127.0.0.1:4566`, which also exposes its inbox at `/store`. With the rest of the dev stack running, start this optional service with `docker compose -f docker-compose.dev.yml up ses -d --wait` and run `hogli test nodejs/src/cdp/services/messaging/email.service.ses.test.ts`. - -The `LocalSes` test helper gives each test its own HTTP proxy for request capture and persistent SESv2 errors. It reads only the message IDs that proxy delivered, without clearing the shared inbox. Tests use invented AWS credentials and distinguish SDK HTTP attempts from retries of the workflow invocation. The inbox schema and delivery assertions check the fake's contract when its image changes; the fake does not validate AWS tenant or configuration-set provisioning. - ## Extra: Integrating with slack You can connect to a real slack workspace in your local development setup by adding the required slack environment variables to your `.env` file. diff --git a/nodejs/src/cdp/services/messaging/email.service.ses.test.ts b/nodejs/src/cdp/services/messaging/email.service.ses.test.ts index 309a60b20675..05d1b6295b81 100644 --- a/nodejs/src/cdp/services/messaging/email.service.ses.test.ts +++ b/nodejs/src/cdp/services/messaging/email.service.ses.test.ts @@ -1,5 +1,3 @@ -import { SESv2Client, SendEmailCommand } from '@aws-sdk/client-sesv2' - import { createExampleInvocation, insertIntegration } from '~/cdp/_tests/fixtures' import { CyclotronInvocationQueueParametersEmailType } from '~/cdp/schema/cyclotron' import { CyclotronJobInvocationHogFunction } from '~/cdp/types' @@ -189,55 +187,4 @@ describe('EmailService with local SES', () => { expect(await ses.getEmails()).toEqual([]) } ) - - it('isolates concurrent proxy faults and inbox reads for identical recipients', async () => { - const unrelatedClient = new SESv2Client({ region: 'us-east-1', endpoint: 'http://127.0.0.1:4566' }) - try { - await unrelatedClient.send( - new SendEmailCommand({ - FromEmailAddress: 'sender@example.com', - Destination: { ToAddresses: [params.to.email] }, - Content: { - Simple: { - Subject: { Data: 'Unrelated email without headers' }, - Body: { Text: { Data: 'Another inbox writer.' } }, - }, - }, - }) - ) - } finally { - unrelatedClient.destroy() - } - const otherSes = new LocalSes() - await otherSes.start() - const otherService = createService(otherSes.endpoint) - ses.setError('TooManyRequestsException') - - try { - const [throttled, delivered] = await Promise.all([ - service.executeSendEmail(invocation), - otherService.executeSendEmail(invocation), - ]) - expect(throttled).toMatchObject({ finished: false, metrics: [] }) - expect(delivered).toMatchObject({ finished: true }) - expect(delivered.error).toBeUndefined() - expect(await ses.getEmails()).toEqual([]) - const otherEmails = await otherSes.getEmails() - expect(otherEmails).toHaveLength(1) - - ses.setError(undefined) - const retry = await service.executeSendEmail(throttled.invocation) - expect(retry.error).toBeUndefined() - const emails = await ses.getEmails() - expect(emails).toHaveLength(1) - expect(emails[0].destination).toEqual(otherEmails[0].destination) - expect(emails[0].messageId).not.toBe(otherEmails[0].messageId) - expect(await otherSes.getEmails()).toEqual(otherEmails) - expect(ses.requests).toHaveLength(2) - expect(otherSes.requests).toHaveLength(1) - } finally { - otherService.sesV2Client?.destroy() - await otherSes.stop() - } - }) }) diff --git a/nodejs/tests/helpers/ses.ts b/nodejs/tests/helpers/ses.ts index 31f3bb7b70b8..100359592914 100644 --- a/nodejs/tests/helpers/ses.ts +++ b/nodejs/tests/helpers/ses.ts @@ -1,36 +1,31 @@ import { SendEmailCommandInput } from '@aws-sdk/client-sesv2' +import { randomUUID } from 'node:crypto' import { IncomingMessage, Server, ServerResponse, createServer } from 'node:http' import { AddressInfo } from 'node:net' -import { isNativeError } from 'node:util/types' -import { z } from 'zod' import { parseJSON } from '~/common/utils/json-parse' -const fetchLocalSes = globalThis.fetch - -const storedEmailSchema = z.object({ - messageId: z.string().min(1), - from: z.string(), - destination: z.object({ to: z.array(z.string()), cc: z.array(z.string()), bcc: z.array(z.string()) }), - replyTo: z.array(z.string()), - subject: z.string(), - body: z.object({ text: z.string().optional(), html: z.string().optional() }), - headers: z.array(z.object({ name: z.string(), value: z.string() })), -}) - -export type LocalSesEmail = z.infer +export type LocalSesEmail = { + messageId: string + from: string | undefined + destination: { to: string[]; cc: string[]; bcc: string[] } + replyTo: string[] + subject: string | undefined + body: { text?: string; html?: string } + headers: { name: string | undefined; value: string | undefined }[] +} export type LocalSesError = 'TooManyRequestsException' | 'LimitExceededException' | 'SendingPausedException' export class LocalSes { readonly requests: SendEmailCommandInput[] = [] - private readonly messageIds = new Set() + private readonly deliveredEmails: LocalSesEmail[] = [] private readonly server: Server private error: LocalSesError | undefined - constructor(private readonly fakeEndpoint = 'http://127.0.0.1:4566') { + constructor() { this.server = createServer((request, response) => { void this.handleRequest(request, response).catch((error: Error) => { - response.writeHead(502, { 'content-type': 'application/json' }) + response.writeHead(500, { 'content-type': 'application/json' }) response.end(JSON.stringify({ message: error.message })) }) }) @@ -60,36 +55,8 @@ export class LocalSes { this.error = error } - private async fetchFake(path: string, options: RequestInit = {}): Promise { - try { - return await fetchLocalSes(`${this.fakeEndpoint}${path}`, { - ...options, - signal: AbortSignal.timeout(10_000), - }) - } catch (error) { - const cause = isNativeError(error) && isNativeError(error.cause) ? error.cause.message : String(error) - throw new Error(`Local SES fake unreachable at ${this.fakeEndpoint}: ${cause}`) - } - } - - async getEmails(): Promise { - const response = await this.fetchFake('/store') - if (!response.ok) { - throw new Error(`Local SES inbox returned HTTP ${response.status}`) - } - const store = z - .object({ emails: z.array(z.object({ messageId: z.string().min(1) }).passthrough()) }) - .safeParse(await response.json()) - if (!store.success) { - throw new Error(`Local SES /store contract changed: ${store.error.message}`) - } - const emails = z - .array(storedEmailSchema) - .safeParse(store.data.emails.filter((email) => this.messageIds.has(email.messageId))) - if (!emails.success) { - throw new Error(`Local SES /store contract changed: ${emails.error.message}`) - } - return emails.data + getEmails(): Promise { + return Promise.resolve([...this.deliveredEmails]) } private async handleRequest(request: IncomingMessage, response: ServerResponse): Promise { @@ -98,12 +65,8 @@ export class LocalSes { response.end() return } - const chunks: Buffer[] = [] - for await (const chunk of request) { - chunks.push(Buffer.from(chunk)) - } - const body = Buffer.concat(chunks).toString('utf8') - this.requests.push(parseJSON(body) as SendEmailCommandInput) + const input = parseJSON(await readBody(request)) as SendEmailCommandInput + this.requests.push(input) if (this.error) { response.writeHead(this.error === 'TooManyRequestsException' ? 429 : 400, { @@ -114,26 +77,34 @@ export class LocalSes { return } - const headers: Record = {} - for (const [name, value] of Object.entries(request.headers)) { - if (typeof value === 'string' && name !== 'host' && name !== 'connection') { - headers[name] = value - } - } - const upstream = await this.fetchFake('/v2/email/outbound-emails', { - method: 'POST', - headers, - body, - }) - const responseBody = await upstream.text() - if (upstream.ok) { - const result = z.object({ MessageId: z.string().min(1) }).safeParse(parseJSON(responseBody)) - if (!result.success) { - throw new Error(`Local SES SendEmail contract changed: ${result.error.message}`) - } - this.messageIds.add(result.data.MessageId) - } - response.writeHead(upstream.status, { 'content-type': 'application/json' }) - response.end(responseBody) + const messageId = randomUUID() + this.deliveredEmails.push(toEmail(messageId, input)) + response.writeHead(200, { 'content-type': 'application/json' }) + response.end(JSON.stringify({ MessageId: messageId })) + } +} + +async function readBody(request: IncomingMessage): Promise { + const chunks: Buffer[] = [] + for await (const chunk of request) { + chunks.push(Buffer.from(chunk)) + } + return Buffer.concat(chunks).toString('utf8') +} + +function toEmail(messageId: string, input: SendEmailCommandInput): LocalSesEmail { + const simple = input.Content?.Simple + return { + messageId, + from: input.FromEmailAddress, + destination: { + to: input.Destination?.ToAddresses ?? [], + cc: input.Destination?.CcAddresses ?? [], + bcc: input.Destination?.BccAddresses ?? [], + }, + replyTo: input.ReplyToAddresses ?? [], + subject: simple?.Subject?.Data, + body: { text: simple?.Body?.Text?.Data, html: simple?.Body?.Html?.Data }, + headers: (simple?.Headers ?? []).map(({ Name, Value }) => ({ name: Name, value: Value })), } }