Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
190 changes: 190 additions & 0 deletions nodejs/src/cdp/services/messaging/email.service.ses.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
import { createExampleInvocation, insertIntegration } from '~/cdp/_tests/fixtures'
import { CyclotronInvocationQueueParametersEmailType } from '~/cdp/schema/cyclotron'
import { CyclotronJobInvocationHogFunction } from '~/cdp/types'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { LocalSes } from '~/tests/helpers/ses'
import { createTestTeamFixture } from '~/tests/helpers/sql'

import { Hub } from '../../../types'
import { RecipientsManagerService } from '../managers/recipients-manager.service'
import { TeamWorkflowsConfigService } from '../managers/team-workflows-config.service'
import { EmailSuppressionService, emailSuppressionConfigFromEnv } from './email-suppression.service'
import { EmailService } from './email.service'
import { EmailTrackingCodeSigner } from './helpers/tracking-code'

describe('EmailService with local SES', () => {
let hub: Hub
let service: EmailService
let invocation: CyclotronJobInvocationHogFunction
let params: CyclotronInvocationQueueParametersEmailType
let ses: LocalSes
const originalEnv = { ...process.env }

const createService = (endpoint: string): EmailService =>
new EmailService(
{
sesAccessKeyId: 'local-ses-test',
sesSecretAccessKey: 'local-ses-test',
sesRegion: 'us-east-1',
sesEndpoint: endpoint,
sesTrackedConfigurationSet: 'local-tracked',
sesUntrackedConfigurationSet: 'local-untracked',
},
hub.integrationManager,
new TeamWorkflowsConfigService(hub.postgres, hub.pubSub),
hub.ENCRYPTION_SALT_KEYS,
hub.SITE_URL,
new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),
new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),
new RecipientsManagerService(hub.postgres)
)

beforeEach(async () => {
process.env.AWS_ACCESS_KEY_ID = 'local-ses-test'
process.env.AWS_SECRET_ACCESS_KEY = 'local-ses-test'
delete process.env.AWS_SESSION_TOKEN
delete process.env.AWS_PROFILE
process.env.AWS_MAX_ATTEMPTS = '1'
ses = new LocalSes()
await ses.start()
hub = await createHub({ SITE_URL: 'http://localhost:8000' })
const { team } = await createTestTeamFixture(hub.postgres)
const integration = await insertIntegration(hub.postgres, team.id, {
kind: 'email',
config: {
email: 'sender@example.com',
name: 'Example Sender',
domain: 'example.com',
verified: true,
provider: 'ses',
},
})
service = createService(ses.endpoint)
invocation = createExampleInvocation({ team_id: team.id, metadata: { tracking_enabled: false } })
params = {
type: 'email',
from: { integrationId: integration.id },
to: { email: `${invocation.id}@example.com`, name: 'Example Recipient' },
cc: 'copy@example.com',
bcc: 'hidden@example.com',
replyTo: 'reply@example.com',
subject: 'Local SES delivery',
text: 'A plain text message.',
html: '<p>An HTML message.</p>',
}
invocation.queueParameters = params
})

afterEach(async () => {
service?.sesV2Client?.destroy()
await ses?.stop()
if (hub) {
await closeHub(hub)
}
process.env = { ...originalEnv }
})

it('delivers the email through the real SDK with bodies, addresses and custom headers', async () => {
const result = await service.executeSendEmail(invocation)

expect(result).toMatchObject({ finished: true })
expect(result.error).toBeUndefined()
expect(result.metrics).toEqual(
expect.arrayContaining([expect.objectContaining({ metric_name: 'email_sent', count: 1 })])
)
expect(ses.requests).toHaveLength(1)
expect(ses.requests[0]).toMatchObject({
TenantName: `team-${invocation.teamId}`,
ConfigurationSetName: 'local-untracked',
FeedbackForwardingEmailAddress: 'sender@example.com',
EmailTags: [{ Name: 'ph_id', Value: expect.any(String) }],
})
const emails = await ses.getEmails()
expect(emails).toHaveLength(1)
expect(emails[0]).toMatchObject({
messageId: expect.any(String),
from: '"Example Sender" <sender@example.com>',
destination: {
to: [`"Example Recipient" <${params.to.email}>`],
cc: ['copy@example.com'],
bcc: ['hidden@example.com'],
},
replyTo: ['reply@example.com'],
subject: 'Local SES delivery',
body: { text: 'A plain text message.', html: '<p>An HTML message.</p>' },
headers: expect.arrayContaining([
{ name: 'Auto-Submitted', value: 'auto-generated' },
{ name: 'List-Unsubscribe-Post', value: 'List-Unsubscribe=One-Click' },
{ name: 'List-Unsubscribe', value: expect.stringContaining('http') },
{ name: 'X-PostHog-Tracking-Code', value: expect.any(String) },
]),
})
})

it.each([1, 3])('reschedules after %i SDK attempt(s) and sends its email payload on retry', async (attempts) => {
process.env.AWS_MAX_ATTEMPTS = String(attempts)
service.sesV2Client?.destroy()
service = createService(ses.endpoint)
ses.setError('TooManyRequestsException')
invocation.queuePriority = 1

const throttled = await service.executeSendEmail(invocation)

expect(throttled).toMatchObject({ finished: false, metrics: [] })
expect(throttled.error).toBeUndefined()
expect(throttled.invocation.queueParameters).toEqual(params)
expect(throttled.invocation.queuePriority).toBe(1)
expect(throttled.invocation.queueScheduledAt?.isValid).toBe(true)
expect(ses.requests).toHaveLength(attempts)
expect(await ses.getEmails()).toEqual([])

ses.setError(undefined)
const retry = await service.executeSendEmail(throttled.invocation)

expect(retry).toMatchObject({ finished: true })
expect(retry.error).toBeUndefined()
expect(retry.metrics).toEqual(
expect.arrayContaining([expect.objectContaining({ metric_name: 'email_sent', count: 1 })])
)
expect(ses.requests).toHaveLength(attempts + 1)
expect(ses.requests[attempts]).toMatchObject({
FromEmailAddress: '"Example Sender" <sender@example.com>',
Destination: {
ToAddresses: [`"Example Recipient" <${params.to.email}>`],
CcAddresses: ['copy@example.com'],
BccAddresses: ['hidden@example.com'],
},
Content: {
Simple: {
Subject: { Data: 'Local SES delivery', Charset: 'UTF-8' },
Body: {
Text: { Data: 'A plain text message.', Charset: 'UTF-8' },
Html: { Data: '<p>An HTML message.</p>', Charset: 'UTF-8' },
},
},
},
})
expect(await ses.getEmails()).toEqual([
expect.objectContaining({ subject: 'Local SES delivery', body: { text: params.text, html: params.html } }),
])
})

it.each(['LimitExceededException', 'SendingPausedException'] as const)(
'fails an HTTP %s without rescheduling or delivering mail',
async (error) => {
ses.setError(error)

const result = await service.executeSendEmail(invocation)

expect(result).toMatchObject({
finished: true,
error: `Failed to send email via SES: Local SES injected ${error}`,
metrics: [expect.objectContaining({ metric_name: 'email_failed', count: 1 })],
})
expect(result.invocation.queueParameters).toBeUndefined()
expect(result.invocation.queueScheduledAt).toBeUndefined()
expect(ses.requests).toHaveLength(1)
expect(await ses.getEmails()).toEqual([])
}
)
})
1 change: 1 addition & 0 deletions nodejs/src/cdp/services/messaging/email.service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -420,6 +420,7 @@ describe('EmailService', () => {
// than the upper bound + scheduler overhead.
expect(scheduledMs).toBeGreaterThanOrEqual(before + 400)
expect(scheduledMs).toBeLessThan(before + 2000)
expect(result.invocation.queueParameters).toEqual(invocation.queueParameters)
// No business metric emitted on throttle — the eventual retry
// will produce email_sent.
expect(result.metrics ?? []).toEqual([])
Expand Down
1 change: 1 addition & 0 deletions nodejs/src/cdp/services/messaging/email.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -627,6 +627,7 @@ export class EmailService {
// catches the cases where SES disagrees with our estimate.
throttled = true
result.finished = false
result.invocation.queueParameters = params
result.invocation.queueScheduledAt = DateTime.utc().plus({ milliseconds: error.retryAfterMs })
addLog('warn', `SES rate-limited (${error.errorCode}); rescheduling email in ${error.retryAfterMs}ms`)
} else {
Expand Down
110 changes: 110 additions & 0 deletions nodejs/tests/helpers/ses.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
import { SendEmailCommandInput } from '@aws-sdk/client-sesv2'
import { randomUUID } from 'node:crypto'
import { IncomingMessage, Server, ServerResponse, createServer } from 'node:http'
import { AddressInfo } from 'node:net'

import { parseJSON } from '~/common/utils/json-parse'

export type LocalSesEmail = {
messageId: string
from: string | undefined
destination: { to: string[]; cc: string[]; bcc: string[] }
replyTo: string[]
subject: string | undefined
body: { text?: string; html?: string }
headers: { name: string | undefined; value: string | undefined }[]
}
export type LocalSesError = 'TooManyRequestsException' | 'LimitExceededException' | 'SendingPausedException'

export class LocalSes {
readonly requests: SendEmailCommandInput[] = []
private readonly deliveredEmails: LocalSesEmail[] = []
private readonly server: Server
private error: LocalSesError | undefined

constructor() {
this.server = createServer((request, response) => {
void this.handleRequest(request, response).catch((error: Error) => {
response.writeHead(500, { 'content-type': 'application/json' })
response.end(JSON.stringify({ message: error.message }))
})
})
}

get endpoint(): string {
return `http://127.0.0.1:${(this.server.address() as AddressInfo).port}`
}

async start(): Promise<void> {
await new Promise<void>((resolve, reject) => {
this.server.once('error', reject)
this.server.listen(0, '127.0.0.1', () => {
this.server.removeListener('error', reject)
resolve()
})
})
}

async stop(): Promise<void> {
await new Promise<void>((resolve, reject) => {
this.server.close((error) => (error ? reject(error) : resolve()))
})
}

setError(error: LocalSesError | undefined): void {
this.error = error
}

getEmails(): Promise<LocalSesEmail[]> {
return Promise.resolve([...this.deliveredEmails])
}

private async handleRequest(request: IncomingMessage, response: ServerResponse): Promise<void> {
if (request.method !== 'POST' || request.url !== '/v2/email/outbound-emails') {
response.writeHead(404)
response.end()
return
}
const input = parseJSON(await readBody(request)) as SendEmailCommandInput
this.requests.push(input)

if (this.error) {
response.writeHead(this.error === 'TooManyRequestsException' ? 429 : 400, {
'content-type': 'application/json',
'x-amzn-errortype': this.error,
})
response.end(JSON.stringify({ message: `Local SES injected ${this.error}` }))
return
}

const messageId = randomUUID()
this.deliveredEmails.push(toEmail(messageId, input))
response.writeHead(200, { 'content-type': 'application/json' })
response.end(JSON.stringify({ MessageId: messageId }))
}
}

async function readBody(request: IncomingMessage): Promise<string> {
const chunks: Buffer[] = []
for await (const chunk of request) {
chunks.push(Buffer.from(chunk))
}
return Buffer.concat(chunks).toString('utf8')
}

function toEmail(messageId: string, input: SendEmailCommandInput): LocalSesEmail {
const simple = input.Content?.Simple
return {
messageId,
from: input.FromEmailAddress,
destination: {
to: input.Destination?.ToAddresses ?? [],
cc: input.Destination?.CcAddresses ?? [],
bcc: input.Destination?.BccAddresses ?? [],
},
replyTo: input.ReplyToAddresses ?? [],
subject: simple?.Subject?.Data,
body: { text: simple?.Body?.Text?.Data, html: simple?.Body?.Html?.Data },
headers: (simple?.Headers ?? []).map(({ Name, Value }) => ({ name: Name, value: Value })),
}
}