diff --git a/nodejs/src/cdp/services/messaging/email.service.ses.test.ts b/nodejs/src/cdp/services/messaging/email.service.ses.test.ts new file mode 100644 index 000000000000..05d1b6295b81 --- /dev/null +++ b/nodejs/src/cdp/services/messaging/email.service.ses.test.ts @@ -0,0 +1,190 @@ +import { createExampleInvocation, insertIntegration } from '~/cdp/_tests/fixtures' +import { CyclotronInvocationQueueParametersEmailType } from '~/cdp/schema/cyclotron' +import { CyclotronJobInvocationHogFunction } from '~/cdp/types' +import { closeHub, createHub } from '~/common/utils/db/hub' +import { LocalSes } from '~/tests/helpers/ses' +import { createTestTeamFixture } from '~/tests/helpers/sql' + +import { Hub } from '../../../types' +import { RecipientsManagerService } from '../managers/recipients-manager.service' +import { TeamWorkflowsConfigService } from '../managers/team-workflows-config.service' +import { EmailSuppressionService, emailSuppressionConfigFromEnv } from './email-suppression.service' +import { EmailService } from './email.service' +import { EmailTrackingCodeSigner } from './helpers/tracking-code' + +describe('EmailService with local SES', () => { + let hub: Hub + let service: EmailService + let invocation: CyclotronJobInvocationHogFunction + let params: CyclotronInvocationQueueParametersEmailType + let ses: LocalSes + const originalEnv = { ...process.env } + + const createService = (endpoint: string): EmailService => + new EmailService( + { + sesAccessKeyId: 'local-ses-test', + sesSecretAccessKey: 'local-ses-test', + sesRegion: 'us-east-1', + sesEndpoint: endpoint, + sesTrackedConfigurationSet: 'local-tracked', + sesUntrackedConfigurationSet: 'local-untracked', + }, + hub.integrationManager, + new TeamWorkflowsConfigService(hub.postgres, hub.pubSub), + hub.ENCRYPTION_SALT_KEYS, + hub.SITE_URL, + new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL), + new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()), + new RecipientsManagerService(hub.postgres) + ) + + beforeEach(async () => { + process.env.AWS_ACCESS_KEY_ID = 'local-ses-test' + process.env.AWS_SECRET_ACCESS_KEY = 'local-ses-test' + delete process.env.AWS_SESSION_TOKEN + delete process.env.AWS_PROFILE + process.env.AWS_MAX_ATTEMPTS = '1' + ses = new LocalSes() + await ses.start() + hub = await createHub({ SITE_URL: 'http://localhost:8000' }) + const { team } = await createTestTeamFixture(hub.postgres) + const integration = await insertIntegration(hub.postgres, team.id, { + kind: 'email', + config: { + email: 'sender@example.com', + name: 'Example Sender', + domain: 'example.com', + verified: true, + provider: 'ses', + }, + }) + service = createService(ses.endpoint) + invocation = createExampleInvocation({ team_id: team.id, metadata: { tracking_enabled: false } }) + params = { + type: 'email', + from: { integrationId: integration.id }, + to: { email: `${invocation.id}@example.com`, name: 'Example Recipient' }, + cc: 'copy@example.com', + bcc: 'hidden@example.com', + replyTo: 'reply@example.com', + subject: 'Local SES delivery', + text: 'A plain text message.', + html: '

An HTML message.

', + } + invocation.queueParameters = params + }) + + afterEach(async () => { + service?.sesV2Client?.destroy() + await ses?.stop() + if (hub) { + await closeHub(hub) + } + process.env = { ...originalEnv } + }) + + it('delivers the email through the real SDK with bodies, addresses and custom headers', async () => { + const result = await service.executeSendEmail(invocation) + + expect(result).toMatchObject({ finished: true }) + expect(result.error).toBeUndefined() + expect(result.metrics).toEqual( + expect.arrayContaining([expect.objectContaining({ metric_name: 'email_sent', count: 1 })]) + ) + expect(ses.requests).toHaveLength(1) + expect(ses.requests[0]).toMatchObject({ + TenantName: `team-${invocation.teamId}`, + ConfigurationSetName: 'local-untracked', + FeedbackForwardingEmailAddress: 'sender@example.com', + EmailTags: [{ Name: 'ph_id', Value: expect.any(String) }], + }) + const emails = await ses.getEmails() + expect(emails).toHaveLength(1) + expect(emails[0]).toMatchObject({ + messageId: expect.any(String), + from: '"Example Sender" ', + destination: { + to: [`"Example Recipient" <${params.to.email}>`], + cc: ['copy@example.com'], + bcc: ['hidden@example.com'], + }, + replyTo: ['reply@example.com'], + subject: 'Local SES delivery', + body: { text: 'A plain text message.', html: '

An HTML message.

' }, + headers: expect.arrayContaining([ + { name: 'Auto-Submitted', value: 'auto-generated' }, + { name: 'List-Unsubscribe-Post', value: 'List-Unsubscribe=One-Click' }, + { name: 'List-Unsubscribe', value: expect.stringContaining('http') }, + { name: 'X-PostHog-Tracking-Code', value: expect.any(String) }, + ]), + }) + }) + + it.each([1, 3])('reschedules after %i SDK attempt(s) and sends its email payload on retry', async (attempts) => { + process.env.AWS_MAX_ATTEMPTS = String(attempts) + service.sesV2Client?.destroy() + service = createService(ses.endpoint) + ses.setError('TooManyRequestsException') + invocation.queuePriority = 1 + + const throttled = await service.executeSendEmail(invocation) + + expect(throttled).toMatchObject({ finished: false, metrics: [] }) + expect(throttled.error).toBeUndefined() + expect(throttled.invocation.queueParameters).toEqual(params) + expect(throttled.invocation.queuePriority).toBe(1) + expect(throttled.invocation.queueScheduledAt?.isValid).toBe(true) + expect(ses.requests).toHaveLength(attempts) + expect(await ses.getEmails()).toEqual([]) + + ses.setError(undefined) + const retry = await service.executeSendEmail(throttled.invocation) + + expect(retry).toMatchObject({ finished: true }) + expect(retry.error).toBeUndefined() + expect(retry.metrics).toEqual( + expect.arrayContaining([expect.objectContaining({ metric_name: 'email_sent', count: 1 })]) + ) + expect(ses.requests).toHaveLength(attempts + 1) + expect(ses.requests[attempts]).toMatchObject({ + FromEmailAddress: '"Example Sender" ', + Destination: { + ToAddresses: [`"Example Recipient" <${params.to.email}>`], + CcAddresses: ['copy@example.com'], + BccAddresses: ['hidden@example.com'], + }, + Content: { + Simple: { + Subject: { Data: 'Local SES delivery', Charset: 'UTF-8' }, + Body: { + Text: { Data: 'A plain text message.', Charset: 'UTF-8' }, + Html: { Data: '

An HTML message.

', Charset: 'UTF-8' }, + }, + }, + }, + }) + expect(await ses.getEmails()).toEqual([ + expect.objectContaining({ subject: 'Local SES delivery', body: { text: params.text, html: params.html } }), + ]) + }) + + it.each(['LimitExceededException', 'SendingPausedException'] as const)( + 'fails an HTTP %s without rescheduling or delivering mail', + async (error) => { + ses.setError(error) + + const result = await service.executeSendEmail(invocation) + + expect(result).toMatchObject({ + finished: true, + error: `Failed to send email via SES: Local SES injected ${error}`, + metrics: [expect.objectContaining({ metric_name: 'email_failed', count: 1 })], + }) + expect(result.invocation.queueParameters).toBeUndefined() + expect(result.invocation.queueScheduledAt).toBeUndefined() + expect(ses.requests).toHaveLength(1) + expect(await ses.getEmails()).toEqual([]) + } + ) +}) diff --git a/nodejs/src/cdp/services/messaging/email.service.test.ts b/nodejs/src/cdp/services/messaging/email.service.test.ts index a50885b752b7..adf73135e11a 100644 --- a/nodejs/src/cdp/services/messaging/email.service.test.ts +++ b/nodejs/src/cdp/services/messaging/email.service.test.ts @@ -420,6 +420,7 @@ describe('EmailService', () => { // than the upper bound + scheduler overhead. expect(scheduledMs).toBeGreaterThanOrEqual(before + 400) expect(scheduledMs).toBeLessThan(before + 2000) + expect(result.invocation.queueParameters).toEqual(invocation.queueParameters) // No business metric emitted on throttle — the eventual retry // will produce email_sent. expect(result.metrics ?? []).toEqual([]) diff --git a/nodejs/src/cdp/services/messaging/email.service.ts b/nodejs/src/cdp/services/messaging/email.service.ts index 8c906c3f8657..f1c3571e49f6 100644 --- a/nodejs/src/cdp/services/messaging/email.service.ts +++ b/nodejs/src/cdp/services/messaging/email.service.ts @@ -627,6 +627,7 @@ export class EmailService { // catches the cases where SES disagrees with our estimate. throttled = true result.finished = false + result.invocation.queueParameters = params result.invocation.queueScheduledAt = DateTime.utc().plus({ milliseconds: error.retryAfterMs }) addLog('warn', `SES rate-limited (${error.errorCode}); rescheduling email in ${error.retryAfterMs}ms`) } else { diff --git a/nodejs/tests/helpers/ses.ts b/nodejs/tests/helpers/ses.ts new file mode 100644 index 000000000000..100359592914 --- /dev/null +++ b/nodejs/tests/helpers/ses.ts @@ -0,0 +1,110 @@ +import { SendEmailCommandInput } from '@aws-sdk/client-sesv2' +import { randomUUID } from 'node:crypto' +import { IncomingMessage, Server, ServerResponse, createServer } from 'node:http' +import { AddressInfo } from 'node:net' + +import { parseJSON } from '~/common/utils/json-parse' + +export type LocalSesEmail = { + messageId: string + from: string | undefined + destination: { to: string[]; cc: string[]; bcc: string[] } + replyTo: string[] + subject: string | undefined + body: { text?: string; html?: string } + headers: { name: string | undefined; value: string | undefined }[] +} +export type LocalSesError = 'TooManyRequestsException' | 'LimitExceededException' | 'SendingPausedException' + +export class LocalSes { + readonly requests: SendEmailCommandInput[] = [] + private readonly deliveredEmails: LocalSesEmail[] = [] + private readonly server: Server + private error: LocalSesError | undefined + + constructor() { + this.server = createServer((request, response) => { + void this.handleRequest(request, response).catch((error: Error) => { + response.writeHead(500, { 'content-type': 'application/json' }) + response.end(JSON.stringify({ message: error.message })) + }) + }) + } + + get endpoint(): string { + return `http://127.0.0.1:${(this.server.address() as AddressInfo).port}` + } + + async start(): Promise { + await new Promise((resolve, reject) => { + this.server.once('error', reject) + this.server.listen(0, '127.0.0.1', () => { + this.server.removeListener('error', reject) + resolve() + }) + }) + } + + async stop(): Promise { + await new Promise((resolve, reject) => { + this.server.close((error) => (error ? reject(error) : resolve())) + }) + } + + setError(error: LocalSesError | undefined): void { + this.error = error + } + + getEmails(): Promise { + return Promise.resolve([...this.deliveredEmails]) + } + + private async handleRequest(request: IncomingMessage, response: ServerResponse): Promise { + if (request.method !== 'POST' || request.url !== '/v2/email/outbound-emails') { + response.writeHead(404) + response.end() + return + } + const input = parseJSON(await readBody(request)) as SendEmailCommandInput + this.requests.push(input) + + if (this.error) { + response.writeHead(this.error === 'TooManyRequestsException' ? 429 : 400, { + 'content-type': 'application/json', + 'x-amzn-errortype': this.error, + }) + response.end(JSON.stringify({ message: `Local SES injected ${this.error}` })) + return + } + + const messageId = randomUUID() + this.deliveredEmails.push(toEmail(messageId, input)) + response.writeHead(200, { 'content-type': 'application/json' }) + response.end(JSON.stringify({ MessageId: messageId })) + } +} + +async function readBody(request: IncomingMessage): Promise { + const chunks: Buffer[] = [] + for await (const chunk of request) { + chunks.push(Buffer.from(chunk)) + } + return Buffer.concat(chunks).toString('utf8') +} + +function toEmail(messageId: string, input: SendEmailCommandInput): LocalSesEmail { + const simple = input.Content?.Simple + return { + messageId, + from: input.FromEmailAddress, + destination: { + to: input.Destination?.ToAddresses ?? [], + cc: input.Destination?.CcAddresses ?? [], + bcc: input.Destination?.BccAddresses ?? [], + }, + replyTo: input.ReplyToAddresses ?? [], + subject: simple?.Subject?.Data, + body: { text: simple?.Body?.Text?.Data, html: simple?.Body?.Html?.Data }, + headers: (simple?.Headers ?? []).map(({ Name, Value }) => ({ name: Name, value: Value })), + } +}