From 71b7661aa7fcd38ace7d5a947310a725a3083716 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Fri, 2 Oct 2026 04:44:37 +0000 Subject: [PATCH 01/13] chore(workflows): add stateful ses, dns and domain connect fakes for email domain tests FakeDnsZone answers dnspython at the network edge, so the real resolver, zone_for_name and CNAME chaining run unchanged. FakeSes serves the SES, SESv2 and STS clients through boto3.client and flips each identity part to Success only when the fake zone holds its records. FakeDomainConnectProvider serves settings over HTTP, checks the apply URL signature, renders our real template and publishes its records. Every fake reads one FaultInjector keyed by operation. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../workflows/backend/test/fakes/fake_dns.py | 214 +++++++++++++++++ .../backend/test/fakes/fake_domain_connect.py | 178 ++++++++++++++ .../workflows/backend/test/fakes/fake_ses.py | 225 ++++++++++++++++++ .../workflows/backend/test/fakes/faults.py | 64 +++++ .../workflows/backend/test/fakes/world.py | 46 ++++ 5 files changed, 727 insertions(+) create mode 100644 products/workflows/backend/test/fakes/fake_dns.py create mode 100644 products/workflows/backend/test/fakes/fake_domain_connect.py create mode 100644 products/workflows/backend/test/fakes/fake_ses.py create mode 100644 products/workflows/backend/test/fakes/faults.py create mode 100644 products/workflows/backend/test/fakes/world.py diff --git a/products/workflows/backend/test/fakes/fake_dns.py b/products/workflows/backend/test/fakes/fake_dns.py new file mode 100644 index 000000000000..828f58558e24 --- /dev/null +++ b/products/workflows/backend/test/fakes/fake_dns.py @@ -0,0 +1,214 @@ +import time +import threading +from collections.abc import Iterator +from contextlib import ExitStack, contextmanager +from typing import Any, Literal + +from unittest.mock import patch + +import dns.name +import dns.query +import dns.rcode +import dns.rdata +import dns.rrset +import dns.message +import dns.resolver +import dns.exception +import dns.rdatatype +import dns.rdataclass +from dns.rdtypes.ANY.CNAME import CNAME +from dns.rdtypes.ANY.TXT import TXT + +from posthog.dataclasses import frozen + +from products.workflows.backend.test.fakes.faults import DnsFault, FaultInjector + +PUBLIC_RESOLVER_ADDRESS = "8.8.8.8" +UNANSWERED_QUERY_SECONDS = 2.0 +MAX_CNAME_CHAIN = 8 +TTL = 300 + +RecordKey = tuple[dns.name.Name, dns.rdatatype.RdataType] +Lookup = Literal["zone", "public"] + + +def dns_operation(name: str, rdtype: str, *, at: Lookup | None = None) -> str: + """Fault key for one lookup. `at` narrows it to the zone's own nameservers or to the public resolver.""" + lookup = f"{_relative_text(_absolute(name))}/{rdtype.upper()}" + return f"dns:{at}:{lookup}" if at else f"dns:{lookup}" + + +@frozen +class _Zone: + apex: dns.name.Name + nameserver_addresses: tuple[str, ...] + + +@frozen +class _Resolution: + rcode: dns.rcode.Rcode + answer: tuple[dns.rrset.RRset, ...] = () + authority: tuple[dns.rrset.RRset, ...] = () + + +class _ResolverClock: + """Simulated time for dnspython, so an unanswered query uses up the resolver's lifetime without a real wait. + Each thread keeps its own offset because the code under test runs lookups in parallel threads.""" + + def __init__(self) -> None: + self._offsets = threading.local() + + def time(self) -> float: + return time.time() + self._offset() + + def sleep(self, seconds: float) -> None: + self._offsets.value = self._offset() + seconds + + def __getattr__(self, name: str) -> Any: + return getattr(time, name) + + def _offset(self) -> float: + return getattr(self._offsets, "value", 0.0) + + +class FakeDnsZone: + """The DNS seen through dnspython: zones with their own nameservers, and a public resolver that follows + CNAME chains across zones. It answers at the network edge (`dns.query`), so the real `Resolver`, + `zone_for_name` and answer parsing run unchanged.""" + + def __init__(self, faults: FaultInjector) -> None: + self._faults = faults + self._zones: list[_Zone] = [] + self._records: dict[RecordKey, list[dns.rdata.Rdata]] = {} + self._clock = _ResolverClock() + self.queried_addresses: set[str] = set() + + @contextmanager + def installed(self) -> Iterator["FakeDnsZone"]: + with ExitStack() as stack: + stack.enter_context(patch("dns.query.udp", self._answer_query)) + stack.enter_context(patch("dns.query.tcp", self._answer_query)) + stack.enter_context(patch.object(dns.resolver.Resolver, "read_resolv_conf", _use_public_resolver)) + stack.enter_context(patch.object(dns.resolver, "default_resolver", None)) + stack.enter_context(patch.object(dns.resolver, "time", self._clock)) + yield self + + def add_zone(self, apex: str, *, nameservers: dict[str, str] | None = None) -> None: + """Adds a zone served by `nameservers` (hostname to address). Without nameservers the zone only + answers through the public resolver.""" + nameservers = nameservers or {} + self._zones.append(_Zone(apex=_absolute(apex), nameserver_addresses=tuple(nameservers.values()))) + self.publish(apex, "SOA", f"ns.{apex}. hostmaster.{apex}. 1 7200 3600 1209600 300") + for hostname, address in nameservers.items(): + self.publish(apex, "NS", f"{hostname}.") + if self._zone_containing(_absolute(hostname)) is None: + self.add_zone(hostname.split(".", 1)[1]) + self.publish(hostname, "A", address) + + def publish(self, name: str, rdtype: str, *values: str) -> None: + records = self._records.setdefault((_absolute(name), dns.rdatatype.from_text(rdtype)), []) + records.extend( + dns.rdata.from_text(dns.rdataclass.IN, rdtype, value, origin=dns.name.root, relativize=False) + for value in values + ) + + def publish_txt(self, name: str, *values: str) -> None: + self.publish(name, "TXT", *(f'"{value}"' for value in values)) + + def unpublish(self, name: str, rdtype: str) -> None: + self._records.pop((_absolute(name), dns.rdatatype.from_text(rdtype)), None) + + def answers(self, name: str, rdtype: str) -> list[dns.rdata.Rdata]: + """What a public resolver returns for the lookup, following CNAMEs. Empty when nothing answers.""" + rdtype_value = dns.rdatatype.from_text(rdtype) + resolution = self._resolve(_absolute(name), rdtype_value, within=None) + return [rdata for rrset in resolution.answer if rrset.rdtype == rdtype_value for rdata in rrset] + + def txt_values(self, name: str) -> list[str]: + return [_txt_value(rdata) for rdata in self.answers(name, "TXT") if isinstance(rdata, TXT)] + + def cname_target(self, name: str) -> str | None: + targets = [rdata.target for rdata in self.answers(name, "CNAME") if isinstance(rdata, CNAME)] + return _relative_text(targets[0]) if targets else None + + def _answer_query( + self, query: dns.message.Message, where: str, timeout: float | None = None, *args: Any, **kwargs: Any + ) -> dns.message.Message: + self.queried_addresses.add(where) + question = query.question[0] + lookup = self._lookup_at(where) + if lookup is None: + self._time_out(timeout) + name, rdtype = _relative_text(question.name), dns.rdatatype.to_text(question.rdtype) + fault = self._faults.fault_for( + dns_operation(name, rdtype, at="public" if lookup == "public" else "zone"), dns_operation(name, rdtype) + ) + if fault == DnsFault.TIMEOUT: + self._time_out(timeout) + + response = dns.message.make_response(query) + if fault in (DnsFault.SERVFAIL, DnsFault.NXDOMAIN): + response.set_rcode(dns.rcode.SERVFAIL if fault == DnsFault.SERVFAIL else dns.rcode.NXDOMAIN) + else: + resolution = self._resolve(question.name, question.rdtype, within=None if lookup == "public" else lookup) + response.set_rcode(resolution.rcode) + response.answer.extend(resolution.answer) + response.authority.extend(resolution.authority) + return dns.message.from_wire(response.to_wire()) + + def _lookup_at(self, address: str) -> _Zone | Literal["public"] | None: + if address == PUBLIC_RESOLVER_ADDRESS: + return "public" + return next((zone for zone in self._zones if address in zone.nameserver_addresses), None) + + def _time_out(self, timeout: float | None) -> None: + self._clock.sleep(timeout or UNANSWERED_QUERY_SECONDS) + raise dns.exception.Timeout(timeout=timeout) + + def _resolve(self, name: dns.name.Name, rdtype: dns.rdatatype.RdataType, *, within: _Zone | None) -> _Resolution: + chain: list[dns.rrset.RRset] = [] + for _ in range(MAX_CNAME_CHAIN): + zone = self._zone_containing(name) + if zone is None or (within is not None and zone != within): + if chain: + return _Resolution(rcode=dns.rcode.NOERROR, answer=tuple(chain)) + return _Resolution(rcode=dns.rcode.REFUSED if within else dns.rcode.NXDOMAIN) + if records := self._records.get((name, rdtype)): + return _Resolution(rcode=dns.rcode.NOERROR, answer=(*chain, _rrset(name, records))) + alias = self._records.get((name, dns.rdatatype.CNAME)) + if not alias or rdtype == dns.rdatatype.CNAME or not isinstance(alias[0], CNAME): + rcode = dns.rcode.NOERROR if self._name_exists(name) else dns.rcode.NXDOMAIN + return _Resolution(rcode=rcode, answer=tuple(chain), authority=(self._soa(zone),)) + chain.append(_rrset(name, alias)) + name = alias[0].target + return _Resolution(rcode=dns.rcode.SERVFAIL) + + def _zone_containing(self, name: dns.name.Name) -> _Zone | None: + candidates = [zone for zone in self._zones if name.is_subdomain(zone.apex)] + return max(candidates, key=lambda zone: len(zone.apex), default=None) + + def _name_exists(self, name: dns.name.Name) -> bool: + return any(record_name.is_subdomain(name) for record_name, _ in self._records) + + def _soa(self, zone: _Zone) -> dns.rrset.RRset: + return _rrset(zone.apex, self._records[(zone.apex, dns.rdatatype.SOA)]) + + +def _use_public_resolver(resolver: dns.resolver.Resolver, _resolv_conf: object) -> None: + resolver.nameservers = [PUBLIC_RESOLVER_ADDRESS] + + +def _rrset(name: dns.name.Name, records: list[dns.rdata.Rdata]) -> dns.rrset.RRset: + return dns.rrset.from_rdata_list(name, TTL, records) + + +def _absolute(name: str) -> dns.name.Name: + return dns.name.from_text(name.lower(), origin=dns.name.root) + + +def _relative_text(name: dns.name.Name) -> str: + return name.to_text(omit_final_dot=True) + + +def _txt_value(rdata: TXT) -> str: + return b"".join(rdata.strings).decode() diff --git a/products/workflows/backend/test/fakes/fake_domain_connect.py b/products/workflows/backend/test/fakes/fake_domain_connect.py new file mode 100644 index 000000000000..58daf00ec150 --- /dev/null +++ b/products/workflows/backend/test/fakes/fake_domain_connect.py @@ -0,0 +1,178 @@ +import re +import json +import base64 +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Any +from urllib.parse import parse_qs, urlsplit + +from django.conf import settings + +import requests +import responses +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import padding +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey, RSAPublicKey + +from products.workflows.backend.test.fakes.fake_dns import FakeDnsZone +from products.workflows.backend.test.fakes.faults import FaultInjector, HttpFault + +TEMPLATE_DIR = ( + Path(__file__).resolve().parents[5] / "frontend" / "src" / "lib" / "components" / "DomainConnect" / "templates" +) +CLOUDFLARE_ENDPOINT = "api.cloudflare.com/client/v4/dns/domainconnect" +CLOUDFLARE_SYNC_UX = "https://dash.cloudflare.com/domainconnect" +SETTINGS_OPERATION = "http.get:settings" +_TEMPLATE_VARIABLE = re.compile(r"%(\w+)%") +_SETTINGS_PATH = re.compile(r"/v2/(?P[^/]+)/settings$") +_APPLY_PATH = re.compile(r"/v2/domainTemplates/providers/(?P[^/]+)/services/(?P[^/]+)/apply$") + + +class DomainConnectRejected(Exception): + pass + + +class FakeDomainConnectProvider: + """A Domain Connect DNS provider. Discovery reads the `_domainconnect` TXT record from `FakeDnsZone`, settings + are served over HTTP, and `apply` checks the signature, renders our real template and publishes its records.""" + + def __init__( + self, + dns: FakeDnsZone, + faults: FaultInjector, + *, + endpoint: str = CLOUDFLARE_ENDPOINT, + url_sync_ux: str = CLOUDFLARE_SYNC_UX, + ) -> None: + self._dns = dns + self._faults = faults + self.endpoint = endpoint + self.url_sync_ux = url_sync_ux + self.hosted_domains: set[str] = set() + + @contextmanager + def installed(self) -> Iterator["FakeDomainConnectProvider"]: + with responses.RequestsMock(assert_all_requests_are_fired=False) as http: + http.add_callback( + responses.GET, re.compile(rf"https://{re.escape(self.endpoint)}/v2/[^/]+/settings"), self._settings + ) + yield self + + def host(self, root_domain: str) -> None: + self.hosted_domains.add(root_domain) + self._dns.publish_txt(f"_domainconnect.{root_domain}", self.endpoint) + + def apply(self, url: str) -> str | None: + """Approves the apply URL the way a person does at the provider, and returns where it redirects.""" + parts = urlsplit(url) + if not url.startswith(self.url_sync_ux): + raise DomainConnectRejected(f"{url} is not this provider's sync URL") + self._check_signature(parts.query) + path = _APPLY_PATH.search(parts.path) + if path is None: + raise DomainConnectRejected(f"Unknown apply path {parts.path}") + params = {key: values[0] for key, values in parse_qs(parts.query).items()} + if params["domain"] not in self.hosted_domains: + raise DomainConnectRejected(f"{params['domain']} is not hosted here") + template = _load_template(path["provider"], path["service"]) + redirect_uri = params.get("redirect_uri") + if redirect_uri and urlsplit(redirect_uri).hostname != template["syncRedirectDomain"]: + raise DomainConnectRejected(f"{redirect_uri} is outside the template's redirect domain") + selected_groups = set(params["groupId"].split(",")) if "groupId" in params else None + for record in template["records"]: + if selected_groups is None or record.get("groupId") in selected_groups: + self._publish(_render(record, params), domain=params["domain"], host=params.get("host", "")) + return redirect_uri + + def _settings(self, request: requests.PreparedRequest) -> tuple[int, dict[str, str], str]: + settings_path = _SETTINGS_PATH.search(urlsplit(request.url or "").path) + domain = settings_path["domain"] if settings_path else "" + match self._faults.fault_for(SETTINGS_OPERATION): + case HttpFault.TIMEOUT: + raise requests.exceptions.ConnectTimeout(f"Injected timeout for {request.url}") + case HttpFault.SERVER_ERROR: + return 500, {}, "Internal Server Error" + case HttpFault.RATE_LIMITED: + return 429, {"Retry-After": "30"}, "Too Many Requests" + case HttpFault.MALFORMED_JSON: + return 200, {"Content-Type": "application/json"}, "{not json" + case HttpFault.MISSING_URL_SYNC_UX: + return 200, {"Content-Type": "application/json"}, json.dumps({"providerName": "Cloudflare"}) + if domain not in self.hosted_domains: + return 404, {}, "" + body = {"providerId": "cloudflare.com", "providerName": "Cloudflare", "urlSyncUX": self.url_sync_ux} + return 200, {"Content-Type": "application/json"}, json.dumps(body) + + def _check_signature(self, query: str) -> None: + signed_query, separator, signature_query = query.partition("&sig=") + if not separator: + raise DomainConnectRejected("The apply URL is not signed") + signature = parse_qs(f"sig={signature_query}") + if signature["key"][0] != settings.DOMAIN_CONNECT_KEY_ID: + raise DomainConnectRejected(f"Unknown signing key {signature['key'][0]}") + try: + _public_key().verify( + base64.b64decode(signature["sig"][0]), signed_query.encode(), padding.PKCS1v15(), hashes.SHA256() + ) + except InvalidSignature as error: + raise DomainConnectRejected("The apply URL signature does not match") from error + + def _publish(self, record: dict[str, Any], *, domain: str, host: str) -> None: + name = _record_name(record["host"], domain=domain, host=host) + match record["type"]: + case "TXT": + self._dns.publish_txt(name, record["data"]) + case "CNAME": + self._dns.unpublish(name, "CNAME") + self._dns.publish(name, "CNAME", f"{record['pointsTo']}.") + case "MX": + self._dns.publish(name, "MX", f"{record['priority']} {record['pointsTo']}.") + case "SPFM": + self._merge_spf_rules(name, record["spfRules"]) + case unknown: + raise DomainConnectRejected(f"Unsupported record type {unknown}") + + def _merge_spf_rules(self, name: str, rules: str) -> None: + existing = next((value for value in self._dns.txt_values(name) if value.startswith("v=spf1")), None) + if existing is None: + self._dns.publish_txt(name, f"v=spf1 {rules} ~all") + return + terms = existing.split() + merged = [terms[0], *(rule for rule in rules.split() if rule not in terms), *terms[1:]] + others = [value for value in self._dns.txt_values(name) if value != existing] + self._dns.unpublish(name, "TXT") + self._dns.publish_txt(name, *others, " ".join(merged)) + + +def _record_name(record_host: str, *, domain: str, host: str) -> str: + labels = [label for label in (record_host if record_host != "@" else "", host) if label] + return ".".join([*labels, domain]) + + +def _render(record: dict[str, Any], variables: dict[str, str]) -> dict[str, Any]: + def substitute(value: Any) -> Any: + if not isinstance(value, str): + return value + missing = set(_TEMPLATE_VARIABLE.findall(value)) - variables.keys() + if missing: + raise DomainConnectRejected(f"The apply URL is missing template variables {sorted(missing)}") + return _TEMPLATE_VARIABLE.sub(lambda match: variables[match[1]], value) + + return {key: substitute(value) for key, value in record.items()} + + +def _load_template(provider_id: str, service_id: str) -> dict[str, Any]: + path = TEMPLATE_DIR / f"{provider_id}.{service_id}.json" + if not path.exists(): + raise DomainConnectRejected(f"No template {provider_id}/{service_id}") + return json.loads(path.read_text()) + + +def _public_key() -> RSAPublicKey: + if not settings.DOMAIN_CONNECT_PRIVATE_KEY: + raise DomainConnectRejected("PostHog has no published signing key") + private_key = serialization.load_pem_private_key(settings.DOMAIN_CONNECT_PRIVATE_KEY.encode(), password=None) + assert isinstance(private_key, RSAPrivateKey) + return private_key.public_key() diff --git a/products/workflows/backend/test/fakes/fake_ses.py b/products/workflows/backend/test/fakes/fake_ses.py new file mode 100644 index 000000000000..647ba9fc46e4 --- /dev/null +++ b/products/workflows/backend/test/fakes/fake_ses.py @@ -0,0 +1,225 @@ +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from typing import Any + +from unittest.mock import patch + +from botocore.exceptions import ClientError +from dns.rdtypes.ANY.MX import MX + +from posthog.dataclasses import frozen + +from products.workflows.backend.test.fakes.fake_dns import FakeDnsZone +from products.workflows.backend.test.fakes.faults import AwsFault, FaultInjector + +ACCOUNT_ID = "123456789012" +SUCCESS = "Success" +PENDING = "Pending" + + +@frozen(frozen=False) +class SesIdentity: + domain: str + verification_token: str | None + dkim_tokens: tuple[str, ...] = () + mail_from_domain: str | None = None + forced_verification_status: str | None = None + forced_dkim_status: str | None = None + checks_paused: bool = False + + +class FakeSes: + """SES v1, SESv2 tenants and STS behind `boto3.client`. Each read checks `FakeDnsZone` the way SES polls + DNS, so a part flips to Success only once its records are published.""" + + def __init__(self, dns: FakeDnsZone, faults: FaultInjector, *, region: str = "us-east-1") -> None: + self._dns = dns + self._faults = faults + self.region = region + self.identities: dict[str, SesIdentity] = {} + self.tenant_resources: dict[str, set[str]] = {} + self.writes: list[str] = [] + + @contextmanager + def installed(self) -> Iterator["FakeSes"]: + with patch("boto3.client", self._client): + yield self + + def add_identity(self, domain: str, **fields: Any) -> SesIdentity: + identity = SesIdentity(domain=domain, **{"verification_token": f"token-{domain}", **fields}) + self.identities[domain] = identity + return identity + + def associate(self, domain: str, tenant: str) -> None: + self.tenant_resources.setdefault(tenant, set()).add(self.identity_arn(domain)) + + def transfer_identity(self, domain: str, tenant: str) -> None: + for resources in self.tenant_resources.values(): + resources.discard(self.identity_arn(domain)) + self.associate(domain, tenant) + + def tenants_of(self, domain: str) -> set[str]: + return {tenant for tenant, resources in self.tenant_resources.items() if self.identity_arn(domain) in resources} + + def identity_arn(self, domain: str) -> str: + return f"arn:aws:ses:{self.region}:{ACCOUNT_ID}:identity/{domain}" + + def verification_status(self, identity: SesIdentity) -> str: + if identity.checks_paused: + return PENDING + if identity.forced_verification_status: + return identity.forced_verification_status + published = identity.verification_token in self._dns.txt_values(f"_amazonses.{identity.domain}") + return SUCCESS if published else PENDING + + def dkim_status(self, identity: SesIdentity) -> str: + if identity.checks_paused: + return PENDING + if identity.forced_dkim_status: + return identity.forced_dkim_status + if not identity.dkim_tokens: + return "NotStarted" + published = all( + self._dns.cname_target(f"{token}._domainkey.{identity.domain}") == f"{token}.dkim.amazonses.com" + for token in identity.dkim_tokens + ) + return SUCCESS if published else PENDING + + def mail_from_status(self, identity: SesIdentity) -> str: + assert identity.mail_from_domain is not None + if identity.checks_paused: + return PENDING + expected_exchange = f"feedback-smtp.{self.region}.amazonses.com." + exchanges = [ + rdata.exchange.to_text() + for rdata in self._dns.answers(identity.mail_from_domain, "MX") + if isinstance(rdata, MX) + ] + # SES rejects a MAIL FROM domain that has any MX record besides its own. + return SUCCESS if exchanges == [expected_exchange] else PENDING + + def _client(self, service_name: str, **_config: Any) -> "_FakeAwsClient": + handlers: dict[str, dict[str, Callable[..., Any]]] = { + "ses": { + "verify_domain_identity": self._verify_domain_identity, + "verify_domain_dkim": self._verify_domain_dkim, + "set_identity_mail_from_domain": self._set_identity_mail_from_domain, + "get_identity_verification_attributes": self._get_identity_verification_attributes, + "get_identity_dkim_attributes": self._get_identity_dkim_attributes, + "get_identity_mail_from_domain_attributes": self._get_identity_mail_from_domain_attributes, + }, + "sesv2": { + "create_tenant": self._create_tenant, + "create_tenant_resource_association": self._create_tenant_resource_association, + "list_resource_tenants": self._list_resource_tenants, + }, + "sts": {"get_caller_identity": lambda: {"Account": ACCOUNT_ID}}, + } + if service_name not in handlers: + raise ValueError(f"FakeSes does not serve the {service_name} client") + return _FakeAwsClient(service_name, handlers[service_name], self._faults) + + def _verify_domain_identity(self, *, Domain: str) -> dict[str, Any]: + self.writes.append("verify_domain_identity") + identity = self.identities.get(Domain) or self.add_identity(Domain) + return {"VerificationToken": identity.verification_token} + + def _verify_domain_dkim(self, *, Domain: str) -> dict[str, Any]: + self.writes.append("verify_domain_dkim") + identity = self.identities.get(Domain) or self.add_identity(Domain) + if not identity.dkim_tokens: + identity.dkim_tokens = tuple(f"dk{index}{Domain.replace('.', '')}" for index in range(1, 4)) + return {"DkimTokens": list(identity.dkim_tokens)} + + def _set_identity_mail_from_domain(self, *, Identity: str, MailFromDomain: str, **_options: Any) -> dict: + self.writes.append("set_identity_mail_from_domain") + self._existing(Identity, "SetIdentityMailFromDomain").mail_from_domain = MailFromDomain + return {} + + def _get_identity_verification_attributes(self, *, Identities: list[str]) -> dict[str, Any]: + return { + "VerificationAttributes": { + identity.domain: { + "VerificationStatus": self.verification_status(identity), + **({"VerificationToken": identity.verification_token} if identity.verification_token else {}), + } + for identity in self._known(Identities) + } + } + + def _get_identity_dkim_attributes(self, *, Identities: list[str]) -> dict[str, Any]: + return { + "DkimAttributes": { + identity.domain: { + "DkimEnabled": bool(identity.dkim_tokens), + "DkimVerificationStatus": self.dkim_status(identity), + **({"DkimTokens": list(identity.dkim_tokens)} if identity.dkim_tokens else {}), + } + for identity in self._known(Identities) + } + } + + def _get_identity_mail_from_domain_attributes(self, *, Identities: list[str]) -> dict[str, Any]: + return { + "MailFromDomainAttributes": { + identity.domain: { + "MailFromDomain": identity.mail_from_domain, + "MailFromDomainStatus": self.mail_from_status(identity), + "BehaviorOnMXFailure": "UseDefaultValue", + } + for identity in self._known(Identities) + if identity.mail_from_domain + } + } + + def _create_tenant(self, *, TenantName: str, **_options: Any) -> dict: + self.writes.append("create_tenant") + if TenantName in self.tenant_resources: + raise ClientError({"Error": {"Code": "AlreadyExistsException"}}, "CreateTenant") + self.tenant_resources[TenantName] = set() + return {"TenantName": TenantName} + + def _create_tenant_resource_association(self, *, TenantName: str, ResourceArn: str) -> dict: + self.writes.append("create_tenant_resource_association") + resources = self.tenant_resources.setdefault(TenantName, set()) + if ResourceArn in resources: + raise ClientError({"Error": {"Code": "AlreadyExistsException"}}, "CreateTenantResourceAssociation") + resources.add(ResourceArn) + return {} + + def _list_resource_tenants(self, *, ResourceArn: str) -> dict[str, Any]: + return { + "ResourceTenants": [ + {"TenantName": tenant, "ResourceArn": ResourceArn} + for tenant, resources in sorted(self.tenant_resources.items()) + if ResourceArn in resources + ] + } + + def _known(self, domains: list[str]) -> list[SesIdentity]: + return [self.identities[domain] for domain in domains if domain in self.identities] + + def _existing(self, domain: str, operation: str) -> SesIdentity: + if domain not in self.identities: + raise ClientError({"Error": {"Code": "InvalidParameterValue"}}, operation) + return self.identities[domain] + + +class _FakeAwsClient: + def __init__(self, service: str, handlers: dict[str, Callable[..., Any]], faults: FaultInjector) -> None: + self._service = service + self._handlers = handlers + self._faults = faults + + def __getattr__(self, operation: str) -> Callable[..., Any]: + if operation not in self._handlers: + raise AttributeError(f"FakeSes {self._service} client has no {operation}") + handler = self._handlers[operation] + + def call(**kwargs: Any) -> Any: + fault = self._faults.fault_for(f"{self._service}.{operation}") + if isinstance(fault, AwsFault): + raise fault.error(operation) + return handler(**kwargs) + + return call diff --git a/products/workflows/backend/test/fakes/faults.py b/products/workflows/backend/test/fakes/faults.py new file mode 100644 index 000000000000..613db7503f50 --- /dev/null +++ b/products/workflows/backend/test/fakes/faults.py @@ -0,0 +1,64 @@ +from enum import StrEnum + +from botocore.exceptions import ClientError, EndpointConnectionError, ReadTimeoutError + +AWS_ENDPOINT = "https://email.us-east-1.amazonaws.com" + + +class AwsFault(StrEnum): + THROTTLING = "throttling" + ACCESS_DENIED = "access_denied" + INVALID_PARAMETER = "invalid_parameter" + ENDPOINT_UNREACHABLE = "endpoint_unreachable" + READ_TIMEOUT = "read_timeout" + + def error(self, operation: str) -> Exception: + match self: + case AwsFault.THROTTLING: + return _client_error("Throttling", operation) + case AwsFault.ACCESS_DENIED: + return _client_error("AccessDenied", operation) + case AwsFault.INVALID_PARAMETER: + return _client_error("InvalidParameterValue", operation) + case AwsFault.ENDPOINT_UNREACHABLE: + return EndpointConnectionError(endpoint_url=AWS_ENDPOINT) + case AwsFault.READ_TIMEOUT: + return ReadTimeoutError(endpoint_url=AWS_ENDPOINT) + + +class DnsFault(StrEnum): + TIMEOUT = "timeout" + SERVFAIL = "servfail" + NXDOMAIN = "nxdomain" + + +class HttpFault(StrEnum): + SERVER_ERROR = "server_error" + RATE_LIMITED = "rate_limited" + TIMEOUT = "timeout" + MALFORMED_JSON = "malformed_json" + MISSING_URL_SYNC_UX = "missing_url_sync_ux" + + +Fault = AwsFault | DnsFault | HttpFault + + +class FaultInjector: + """Faults keyed by operation, such as `ses.get_identity_dkim_attributes`, `dns:zone:_dmarc.example.com/TXT` + or `http.get:settings`. A fault holds until it is cleared, so retries inside a client see it too.""" + + def __init__(self) -> None: + self._faults: dict[str, Fault] = {} + + def inject(self, operation: str, fault: Fault) -> None: + self._faults[operation] = fault + + def clear(self) -> None: + self._faults.clear() + + def fault_for(self, *operations: str) -> Fault | None: + return next((self._faults[operation] for operation in operations if operation in self._faults), None) + + +def _client_error(code: str, operation: str) -> ClientError: + return ClientError({"Error": {"Code": code, "Message": f"Injected {code}"}}, operation) diff --git a/products/workflows/backend/test/fakes/world.py b/products/workflows/backend/test/fakes/world.py new file mode 100644 index 000000000000..207995b9c0c9 --- /dev/null +++ b/products/workflows/backend/test/fakes/world.py @@ -0,0 +1,46 @@ +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from functools import cache +from typing import Any + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa + +from products.workflows.backend.test.fakes.fake_dns import FakeDnsZone +from products.workflows.backend.test.fakes.fake_domain_connect import FakeDomainConnectProvider +from products.workflows.backend.test.fakes.fake_ses import FakeSes +from products.workflows.backend.test.fakes.faults import FaultInjector + + +class EmailDomainWorld: + """Everything outside PostHog that the email domain setup talks to, sharing one DNS and one fault injector.""" + + def __init__(self) -> None: + self.faults = FaultInjector() + self.dns = FakeDnsZone(self.faults) + self.ses = FakeSes(self.dns, self.faults) + self.domain_connect = FakeDomainConnectProvider(self.dns, self.faults) + + @contextmanager + def installed(self) -> Iterator["EmailDomainWorld"]: + with self.dns.installed(), self.ses.installed(), self.domain_connect.installed(): + yield self + + def publish_record(self, record: Mapping[str, Any]) -> None: + """Publishes one record from the status API the way a person copies it into their DNS host.""" + hostname, value = record["hostname"], record["value"] + match record["type"]: + case "TXT": + self.dns.publish_txt(hostname, value) + case "CNAME": + self.dns.publish(hostname, "CNAME", f"{value}.") + case "MX": + self.dns.publish(hostname, "MX", f"{record['priority']} {value}.") + + +@cache +def domain_connect_signing_key_pem() -> str: + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + return key.private_bytes( + serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption() + ).decode() From a3a9518906d559a328a73481fa66aa140b972f68 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Fri, 2 Oct 2026 04:44:47 +0000 Subject: [PATCH 02/13] feat(workflows): add read-only email domain status endpoint Add GET integrations/{id}/email/status. It reads SES identity attributes and the zone's own nameservers without changing SES, and returns the overall status, three setup steps and every DNS record with pending, found or verified. Records the zone's nameservers do not confirm, such as a DMARC CNAME into another zone, are looked up again through a public resolver. Responses are cached for 10 seconds; refresh=true skips the cache. When SES reports the domain verified, the senders on that domain are marked verified once and an "email domain verified" event reports the setup method (manual or agent). The verify endpoint shares the same write. The endpoint stays honest through provider and DNS failures: - An SES or STS read error is a typed 503 with the code email_provider_unavailable instead of an unhandled 500. Errors other than throttling go to error tracking. - A check where a DNS lookup got no answer neither verifies the domain nor is cached, so the next poll decides once DNS answers again. - A failed worker reload after the verified write no longer fails the request. Workers refresh integrations on their own schedule. - The SES MAIL FROM status counts only when SES holds this sender's MAIL FROM domain, and the MAIL FROM SPF record shows verified only when it is published. - DNS checks stay off internal addresses, and the AWS account id is looked up once instead of on every read. Scenarios drive the endpoint end to end against the SES and DNS fakes. The status tool is enabled for MCP behind workflows-email-domain-agent-setup. Co-Authored-By: Claude Opus 5.5 (1M context) --- posthog/api/integration.py | 70 +++- .../integration_email_status_serializers.py | 72 ++++ posthog/models/integration/email.py | 98 ++++- .../frontend/generated/api.schemas.ts | 171 ++++++++ .../integrations/frontend/generated/api.ts | 38 ++ products/integrations/mcp/tools.yaml | 16 + products/workflows/backend/facade/api.py | 15 + .../workflows/backend/facade/contracts.py | 40 ++ products/workflows/backend/facade/enums.py | 42 ++ .../workflows/backend/providers/__init__.py | 4 +- .../workflows/backend/providers/maildev.py | 20 +- products/workflows/backend/providers/ses.py | 321 ++++++++++----- .../backend/services/email_domain_dns.py | 152 +++++++ .../backend/services/email_domain_status.py | 111 +++++ .../test/email_domain_api_test_case.py | 89 ++++ .../backend/test/test_email_domain_status.py | 54 +++ .../test/test_email_domain_status_api.py | 381 ++++++++++++++++++ .../test/test_email_integration_ses.py | 15 +- .../backend/test/test_ses_provider.py | 1 + .../schema/generated-tool-definitions.json | 15 + services/mcp/schema/tool-definitions-all.json | 15 + services/mcp/src/api/generated.ts | 171 ++++++++ .../mcp/src/generated/integrations/api.ts | 24 +- .../mcp/src/tools/generated/integrations.ts | 28 ++ 24 files changed, 1827 insertions(+), 136 deletions(-) create mode 100644 posthog/api/integration_email_status_serializers.py create mode 100644 products/workflows/backend/services/email_domain_dns.py create mode 100644 products/workflows/backend/services/email_domain_status.py create mode 100644 products/workflows/backend/test/email_domain_api_test_case.py create mode 100644 products/workflows/backend/test/test_email_domain_status.py create mode 100644 products/workflows/backend/test/test_email_domain_status_api.py diff --git a/posthog/api/integration.py b/posthog/api/integration.py index 67af063ef3dc..6a920481c21b 100644 --- a/posthog/api/integration.py +++ b/posthog/api/integration.py @@ -20,7 +20,7 @@ from django_filters.rest_framework import DjangoFilterBackend from django_redis.cache import RedisCache from django_redis.exceptions import ConnectionInterrupted -from drf_spectacular.utils import extend_schema, extend_schema_field, extend_schema_serializer +from drf_spectacular.utils import OpenApiResponse, extend_schema, extend_schema_field, extend_schema_serializer from prometheus_client import Counter from redis.exceptions import RedisError from rest_framework import mixins, serializers, status, viewsets @@ -45,10 +45,15 @@ github_app_install_url, is_valid_github_installation_id, ) +from posthog.api.integration_email_status_serializers import ( + EmailDomainStatusQuerySerializer, + EmailDomainStatusSerializer, +) +from posthog.api.mixins import ValidatedRequest, validated_request from posthog.api.routing import TeamAndOrgViewSetMixin from posthog.api.shared import UserBasicSerializer from posthog.api.utils import action -from posthog.auth import SessionAuthentication +from posthog.auth import OAuthAccessTokenAuthentication, PersonalAPIKeyAuthentication, SessionAuthentication from posthog.domain_connect import discover_domain_connect, extract_root_domain_and_host, get_available_providers from posthog.egress.github.transport import GitHubRateLimitError from posthog.event_usage import report_user_action @@ -129,6 +134,7 @@ from products.tasks.backend.facade.api import get_in_progress_runs_for_github_integration from products.tasks.backend.facade.contracts import InProgressGithubRunsDTO from products.workflows.backend.facade.api import get_active_workflows_using_integration +from products.workflows.backend.facade.contracts import EmailProviderUnavailableError logger = structlog.get_logger(__name__) @@ -145,6 +151,13 @@ GITHUB_REPOSITORY_NAME_RE = re.compile(r"[A-Za-z0-9_.\-]+") +class EmailProviderUnavailable(APIException): + # A stable code lets the setup page keep polling through an outage instead of treating it as a broken sender. + status_code = status.HTTP_503_SERVICE_UNAVAILABLE + default_code = "email_provider_unavailable" + default_detail = "Couldn't reach the email provider to check this domain. Try again in a moment." + + class SlackIntegrationInactiveError(APIException): # Actionable 4xx (reconnect Slack) rather than a raw 500, so the channel picker can render # guidance inline instead of an unhelpful "server error" toast on every retry. @@ -1476,6 +1489,7 @@ class IntegrationViewSet( "anthropic_managed_agents", "anthropic_managed_agent_environments", "anthropic_managed_agent_vaults", + "email_status", ] scope_object_write_actions = [ "create", @@ -2633,10 +2647,58 @@ def gitlab_members(self, request: Request, *args: Any, **kwargs: Any) -> Respons @action(methods=["POST"], detail=True, url_path="email/verify") def email_verify(self, request, **kwargs): - email = EmailIntegration(self.get_object()) - verification_result = email.verify() + instance = self.get_object() + email = EmailIntegration(instance) + verification_result = email.verify( + on_domain_verified=lambda: self._report_email_domain_verified(request, instance) + ) return Response(verification_result) + @validated_request( + query_serializer=EmailDomainStatusQuerySerializer, + responses={ + 200: EmailDomainStatusSerializer, + 503: OpenApiResponse( + description="The email provider could not be reached. The error code is email_provider_unavailable. " + "Nothing changed, so poll again." + ), + }, + summary="Get email sending domain status", + description="Read the setup progress of an email sender's domain from SES and public DNS without " + "changing either. Poll this until verified is true. Marks the domain's senders as verified once SES " + "verifies the domain.", + ) + @action(methods=["GET"], detail=True, url_path="email/status") + def email_status(self, request: ValidatedRequest, **kwargs: Any) -> Response: + instance = self.get_object() + if instance.kind != "email": + raise ValidationError("email/status is only supported for email integrations") + try: + report = EmailIntegration(instance).status( + refresh=request.validated_query_data["refresh"], + on_domain_verified=lambda: self._report_email_domain_verified(request, instance), + ) + except EmailProviderUnavailableError as error: + raise EmailProviderUnavailable() from error + return Response(EmailDomainStatusSerializer(report).data) + + def _report_email_domain_verified(self, request: Request, instance: Integration) -> None: + report_user_action( + request.user, + "email domain verified", + { + "method": self._email_domain_setup_method(request, instance), + "seconds_since_created": int((timezone.now() - instance.created_at).total_seconds()), + }, + team=self.team, + ) + + @staticmethod + def _email_domain_setup_method(request: Request, instance: Integration) -> str: + if isinstance(request.successful_authenticator, PersonalAPIKeyAuthentication | OAuthAccessTokenAuthentication): + return "agent" + return instance.config.get("setup_method", "manual") + @extend_schema(responses={200: IntegrationSerializer}) @action(methods=["PATCH"], detail=True, url_path="email") def email_update(self, request, **kwargs) -> Response: diff --git a/posthog/api/integration_email_status_serializers.py b/posthog/api/integration_email_status_serializers.py new file mode 100644 index 000000000000..3844191f2793 --- /dev/null +++ b/posthog/api/integration_email_status_serializers.py @@ -0,0 +1,72 @@ +from rest_framework import serializers + +from products.workflows.backend.facade.enums import ( + EmailDomainSetupRecordKind, + EmailDomainSetupRecordStatus, + EmailDomainSetupRecordType, + EmailDomainSetupStatus, + EmailDomainSetupStepKey, + EmailDomainSetupStepState, +) + + +class EmailDomainStatusQuerySerializer(serializers.Serializer): + refresh = serializers.BooleanField( + default=False, + help_text="Skip the 10 second status cache and read SES and DNS again. Use it after changing DNS records.", + ) + + +class EmailDomainStatusStepSerializer(serializers.Serializer): + key = serializers.ChoiceField( + choices=EmailDomainSetupStepKey.choices, + help_text="The setup step: domain_added (SES knows the domain), records_found (every DNS record is " + "published), verified (SES verified the domain and the sender can send).", + ) + state = serializers.ChoiceField(choices=EmailDomainSetupStepState.choices, help_text="Progress of this step.") + + +class EmailDomainStatusRecordSerializer(serializers.Serializer): + kind = serializers.ChoiceField( + choices=EmailDomainSetupRecordKind.choices, + help_text="What the record does: verification proves domain ownership, dkim signs email, spf and " + "mail_from_spf list SES as an allowed sender, mail_from_mx routes bounces, dmarc sets the policy for " + "unauthenticated mail.", + ) + hostname = serializers.CharField(help_text="Full DNS name to create the record at, including the domain.") + type = serializers.ChoiceField(choices=EmailDomainSetupRecordType.choices, help_text="DNS record type.") + value = serializers.CharField( + help_text="Value to publish. For an SPF record, add the include to an existing SPF record instead of " + "creating a second one. For DMARC, any existing v=DMARC1 record counts." + ) + priority = serializers.IntegerField(allow_null=True, help_text="MX priority. Null for other record types.") + status = serializers.ChoiceField( + choices=EmailDomainSetupRecordStatus.choices, + help_text="pending: not found in DNS yet. found: published in DNS, SES has not confirmed it yet. " + "verified: SES confirmed the record, or for DMARC a v=DMARC1 record exists.", + ) + + +class EmailDomainStatusSerializer(serializers.Serializer): + status = serializers.ChoiceField( + source="check.status", + choices=EmailDomainSetupStatus.choices, + help_text="Overall setup status. records_found means every record is published and SES has not " + "confirmed yet. temporary_failure means SES could not see the records, usually because DNS changes " + "have not spread yet. failed means SES gave up after 72 hours; start verification again.", + ) + verified = serializers.BooleanField( + help_text="Whether the sender can send email. Set once SES verifies the domain." + ) + checked_at = serializers.DateTimeField( + help_text="When SES and DNS were read. Responses are cached for 10 seconds unless refresh is set." + ) + steps = EmailDomainStatusStepSerializer( + source="check.steps", many=True, help_text="The three setup steps, in order." + ) + records = EmailDomainStatusRecordSerializer( + source="check.records", + many=True, + help_text="DNS records to publish at the domain's DNS host, with the status of each. Empty when " + "status is not_started.", + ) diff --git a/posthog/models/integration/email.py b/posthog/models/integration/email.py index 269f0df8b7f3..955e8ae3c4cc 100644 --- a/posthog/models/integration/email.py +++ b/posthog/models/integration/email.py @@ -1,15 +1,21 @@ """Native email-sending integration (SES / maildev) and its cleanup signal.""" +from collections.abc import Callable +from datetime import datetime from typing import TYPE_CHECKING, Any from django.conf import settings +from django.core.cache import cache from django.db import models, transaction from django.dispatch import receiver +from django.utils import timezone from disposable_email_domains import blocklist as disposable_email_domains_list from free_email_domains import whitelist as free_email_domains_list from rest_framework.exceptions import ValidationError +from posthog.dataclasses import frozen +from posthog.exceptions_capture import capture_exception from posthog.models.team.team import Team from posthog.models.user import User from posthog.plugins.plugin_server_api import reload_integrations_on_workers @@ -17,7 +23,16 @@ from . import model if TYPE_CHECKING: - from products.workflows.backend.facade.contracts import EmailDomainVerification + from products.workflows.backend.facade.contracts import EmailDomainCheck, EmailDomainVerification + +EMAIL_DOMAIN_STATUS_CACHE_SECONDS = 10 + + +@frozen +class EmailDomainStatusReport: + check: "EmailDomainCheck" + verified: bool + checked_at: datetime class EmailIntegration: @@ -122,7 +137,7 @@ def update_native_integration(self, config: dict, team_id: int) -> model.Integra return self.integration - def verify(self) -> "EmailDomainVerification": + def verify(self, *, on_domain_verified: Callable[[], None] | None = None) -> "EmailDomainVerification": domain = self.integration.config.get("domain") provider = self.integration.config.get("provider", "ses") mail_from_subdomain = self.integration.config.get("mail_from_subdomain", "feedback") @@ -150,22 +165,77 @@ def verify(self) -> "EmailDomainVerification": raise ValueError(f"Invalid provider: {provider}") if verification_result.get("status") == "success": - # We can validate all other integrations with the same domain and provider - all_integrations_for_domain = model.Integration.objects.filter( - team_id=self.integration.team_id, - kind="email", - config__domain=domain, - config__provider=provider, + self._mark_domain_verified(on_domain_verified) + + return verification_result + + def status(self, *, refresh: bool, on_domain_verified: Callable[[], None] | None = None) -> EmailDomainStatusReport: + cache_key = f"email_domain_status:{self.integration.id}" + if not refresh and (cached := cache.get(cache_key)) is not None: + return cached + + check = self._check_domain() + if check.is_verified and not self.integration.config.get("verified"): + self._mark_domain_verified(on_domain_verified) + + report = EmailDomainStatusReport( + check=check, verified=bool(self.integration.config.get("verified")), checked_at=timezone.now() + ) + if check.every_dns_lookup_answered: + cache.set(cache_key, report, EMAIL_DOMAIN_STATUS_CACHE_SECONDS) + return report + + def _check_domain(self) -> "EmailDomainCheck": + domain = self.integration.config["domain"] + mail_from_subdomain = self.integration.config.get("mail_from_subdomain", "feedback") + provider = self.integration.config.get("provider", "ses") + + if provider == "ses": + from products.workflows.backend.facade.api import ( + get_ses_email_domain_status, # noqa: PLC0415 — keeps the workflows facade off the model import path ) - for integration in all_integrations_for_domain: - integration.config["verified"] = True - integration.save() - reload_integrations_on_workers( - self.integration.team_id, [integration.id for integration in all_integrations_for_domain] + return get_ses_email_domain_status( + domain, mail_from_subdomain=mail_from_subdomain, team_id=self.integration.team_id + ) + if provider == "maildev": + from products.workflows.backend.facade.api import ( + get_maildev_email_domain_status, # noqa: PLC0415 — keeps the workflows facade off the model import path ) - return verification_result + return get_maildev_email_domain_status(domain, mail_from_subdomain=mail_from_subdomain) + raise ValueError(f"Invalid provider: {provider}") + + def _mark_domain_verified(self, on_domain_verified: Callable[[], None] | None) -> None: + with transaction.atomic(): + senders = list( + model.Integration.objects.select_for_update() + .filter( + team_id=self.integration.team_id, + kind="email", + config__domain=self.integration.config.get("domain"), + config__provider=self.integration.config.get("provider", "ses"), + ) + .order_by("id") + ) + unverified_senders = [sender for sender in senders if not sender.config.get("verified")] + for sender in unverified_senders: + sender.config["verified"] = True + sender.save() + + self.integration.config["verified"] = True + # A sender added to a domain that is already verified joins it, so only the first verification counts. + if on_domain_verified and len(unverified_senders) == len(senders): + on_domain_verified() + self._reload_senders_on_workers(senders) + + # Workers also refresh integrations on their own schedule, so a failed reload only delays sending. + # Failing the request instead would hide a verification that is already saved. + def _reload_senders_on_workers(self, senders: list[model.Integration]) -> None: + try: + reload_integrations_on_workers(self.integration.team_id, [sender.id for sender in senders]) + except Exception as error: + capture_exception(error, {"team_id": self.integration.team_id}) @receiver(models.signals.post_delete, sender=model.Integration) diff --git a/products/integrations/frontend/generated/api.schemas.ts b/products/integrations/frontend/generated/api.schemas.ts index 30eda29b193d..0a496540c65e 100644 --- a/products/integrations/frontend/generated/api.schemas.ts +++ b/products/integrations/frontend/generated/api.schemas.ts @@ -348,6 +348,170 @@ export interface PatchedIntegrationConfigApi { readonly installation_status?: InstallationStatusEnumApi | null } +/** + * * `not_started` - Not started + * * `pending` - Waiting for DNS records + * * `records_found` - Records found, waiting for verification + * * `verified` - Verified + * * `temporary_failure` - Temporary failure + * * `failed` - Failed + */ +export type EmailDomainSetupStatusEnumApi = + (typeof EmailDomainSetupStatusEnumApi)[keyof typeof EmailDomainSetupStatusEnumApi] + +export const EmailDomainSetupStatusEnumApi = { + NotStarted: 'not_started', + Pending: 'pending', + RecordsFound: 'records_found', + Verified: 'verified', + TemporaryFailure: 'temporary_failure', + Failed: 'failed', +} as const + +/** + * * `domain_added` - Domain added + * * `records_found` - DNS records found + * * `verified` - Verified + */ +export type EmailDomainSetupStepKeyEnumApi = + (typeof EmailDomainSetupStepKeyEnumApi)[keyof typeof EmailDomainSetupStepKeyEnumApi] + +export const EmailDomainSetupStepKeyEnumApi = { + DomainAdded: 'domain_added', + RecordsFound: 'records_found', + Verified: 'verified', +} as const + +/** + * * `done` - Done + * * `pending` - Pending + * * `failed` - Failed + */ +export type EmailDomainSetupStepStateEnumApi = + (typeof EmailDomainSetupStepStateEnumApi)[keyof typeof EmailDomainSetupStepStateEnumApi] + +export const EmailDomainSetupStepStateEnumApi = { + Done: 'done', + Pending: 'pending', + Failed: 'failed', +} as const + +export interface EmailDomainStatusStepApi { + /** The setup step: domain_added (SES knows the domain), records_found (every DNS record is published), verified (SES verified the domain and the sender can send). + * + * * `domain_added` - Domain added + * * `records_found` - DNS records found + * * `verified` - Verified */ + key: EmailDomainSetupStepKeyEnumApi + /** Progress of this step. + * + * * `done` - Done + * * `pending` - Pending + * * `failed` - Failed */ + state: EmailDomainSetupStepStateEnumApi +} + +/** + * * `verification` - Domain ownership + * * `dkim` - DKIM signing + * * `spf` - SPF + * * `mail_from_mx` - MAIL FROM MX + * * `mail_from_spf` - MAIL FROM SPF + * * `dmarc` - DMARC + */ +export type EmailDomainSetupRecordKindEnumApi = + (typeof EmailDomainSetupRecordKindEnumApi)[keyof typeof EmailDomainSetupRecordKindEnumApi] + +export const EmailDomainSetupRecordKindEnumApi = { + Verification: 'verification', + Dkim: 'dkim', + Spf: 'spf', + MailFromMx: 'mail_from_mx', + MailFromSpf: 'mail_from_spf', + Dmarc: 'dmarc', +} as const + +/** + * * `TXT` - TXT + * * `CNAME` - CNAME + * * `MX` - MX + */ +export type EmailDomainSetupRecordTypeEnumApi = + (typeof EmailDomainSetupRecordTypeEnumApi)[keyof typeof EmailDomainSetupRecordTypeEnumApi] + +export const EmailDomainSetupRecordTypeEnumApi = { + Txt: 'TXT', + Cname: 'CNAME', + Mx: 'MX', +} as const + +/** + * * `pending` - Not found yet + * * `found` - Found in DNS + * * `verified` - Verified + */ +export type EmailDomainSetupRecordStatusEnumApi = + (typeof EmailDomainSetupRecordStatusEnumApi)[keyof typeof EmailDomainSetupRecordStatusEnumApi] + +export const EmailDomainSetupRecordStatusEnumApi = { + Pending: 'pending', + Found: 'found', + Verified: 'verified', +} as const + +export interface EmailDomainStatusRecordApi { + /** What the record does: verification proves domain ownership, dkim signs email, spf and mail_from_spf list SES as an allowed sender, mail_from_mx routes bounces, dmarc sets the policy for unauthenticated mail. + * + * * `verification` - Domain ownership + * * `dkim` - DKIM signing + * * `spf` - SPF + * * `mail_from_mx` - MAIL FROM MX + * * `mail_from_spf` - MAIL FROM SPF + * * `dmarc` - DMARC */ + kind: EmailDomainSetupRecordKindEnumApi + /** Full DNS name to create the record at, including the domain. */ + hostname: string + /** DNS record type. + * + * * `TXT` - TXT + * * `CNAME` - CNAME + * * `MX` - MX */ + type: EmailDomainSetupRecordTypeEnumApi + /** Value to publish. For an SPF record, add the include to an existing SPF record instead of creating a second one. For DMARC, any existing v=DMARC1 record counts. */ + value: string + /** + * MX priority. Null for other record types. + * @nullable + */ + priority: number | null + /** pending: not found in DNS yet. found: published in DNS, SES has not confirmed it yet. verified: SES confirmed the record, or for DMARC a v=DMARC1 record exists. + * + * * `pending` - Not found yet + * * `found` - Found in DNS + * * `verified` - Verified */ + status: EmailDomainSetupRecordStatusEnumApi +} + +export interface EmailDomainStatusApi { + /** Overall setup status. records_found means every record is published and SES has not confirmed yet. temporary_failure means SES could not see the records, usually because DNS changes have not spread yet. failed means SES gave up after 72 hours; start verification again. + * + * * `not_started` - Not started + * * `pending` - Waiting for DNS records + * * `records_found` - Records found, waiting for verification + * * `verified` - Verified + * * `temporary_failure` - Temporary failure + * * `failed` - Failed */ + status: EmailDomainSetupStatusEnumApi + /** Whether the sender can send email. Set once SES verifies the domain. */ + verified: boolean + /** When SES and DNS were read. Responses are cached for 10 seconds unless refresh is set. */ + checked_at: string + /** The three setup steps, in order. */ + steps: EmailDomainStatusStepApi[] + /** DNS records to publish at the domain's DNS host, with the status of each. Empty when status is not_started. */ + records: EmailDomainStatusRecordApi[] +} + export interface IntegrationAssigneeApi { /** Provider user identifier to pass as error tracking config.assignee: a Linear user ID, a GitHub login, a GitLab user ID, or a Jira account ID. */ id: string @@ -904,6 +1068,13 @@ export type IntegrationsChannelsRetrieveParams = { search?: string } +export type IntegrationsEmailStatusRetrieveParams = { + /** + * Skip the 10 second status cache and read SES and DNS again. Use it after changing DNS records. + */ + refresh?: boolean +} + export type IntegrationsGithubAssigneesRetrieveParams = { /** * Repository name, or owner/name, whose assignable users to list. diff --git a/products/integrations/frontend/generated/api.ts b/products/integrations/frontend/generated/api.ts index 54bdf2e0adc4..6bb7e502d3af 100644 --- a/products/integrations/frontend/generated/api.ts +++ b/products/integrations/frontend/generated/api.ts @@ -9,6 +9,7 @@ import { apiMutator } from '../../../../frontend/src/lib/api-orval-mutator' * OpenAPI spec version: 1.0.0 */ import type { + EmailDomainStatusApi, GitHubAvailableInstallationsResponseApi, GitHubBranchesResponseApi, GitHubLinkExistingRequestApi, @@ -23,6 +24,7 @@ import type { IntegrationAssigneesResponseApi, IntegrationConfigApi, IntegrationsChannelsRetrieveParams, + IntegrationsEmailStatusRetrieveParams, IntegrationsGithubAssigneesRetrieveParams, IntegrationsGithubBranchesRetrieveParams, IntegrationsGithubReposRetrieveParams, @@ -397,6 +399,42 @@ export const integrationsEmailPartialUpdate = async ( }) } +export const getIntegrationsEmailStatusRetrieveUrl = ( + projectId: string, + id: number, + params?: IntegrationsEmailStatusRetrieveParams +) => { + const normalizedParams = new URLSearchParams() + + Object.entries(params || {}).forEach(([key, value]) => { + if (value !== undefined) { + normalizedParams.append(key, value === null ? 'null' : String(value)) + } + }) + + const stringifiedParams = normalizedParams.toString() + + return stringifiedParams.length > 0 + ? `/api/projects/${projectId}/integrations/${id}/email/status/?${stringifiedParams}` + : `/api/projects/${projectId}/integrations/${id}/email/status/` +} + +/** + * Read the setup progress of an email sender's domain from SES and public DNS without changing either. Poll this until verified is true. Marks the domain's senders as verified once SES verifies the domain. + * @summary Get email sending domain status + */ +export const integrationsEmailStatusRetrieve = async ( + projectId: string, + id: number, + params?: IntegrationsEmailStatusRetrieveParams, + options?: RequestInit +): Promise => { + return apiMutator(getIntegrationsEmailStatusRetrieveUrl(projectId, id, params), { + ...options, + method: 'GET', + }) +} + export const getIntegrationsEmailVerifyCreateUrl = (projectId: string, id: number) => { return `/api/projects/${projectId}/integrations/${id}/email/verify/` } diff --git a/products/integrations/mcp/tools.yaml b/products/integrations/mcp/tools.yaml index d92d6daae08f..96ef7538e2c4 100644 --- a/products/integrations/mcp/tools.yaml +++ b/products/integrations/mcp/tools.yaml @@ -59,6 +59,22 @@ tools: to find a channel_id when wiring up Slack delivery for an alert — pass the channel id into the cdp-functions-create inputs.channel.value field. Requires the Slack integration's id; use integrations-list (filter by kind=slack) to find it. + integrations-email-status-retrieve: + operation: integrations_email_status_retrieve + enabled: true + scopes: + - integration:read + annotations: + readOnly: true + destructive: false + idempotent: true + feature_flag: workflows-email-domain-agent-setup + title: Get email sending domain status + description: > + Get the setup progress of an email sender's domain by integration ID. Returns the overall status, the + three setup steps, and every DNS record to publish with its current status (pending, found, verified). + Poll it after adding DNS records until verified is true, waiting at least 10 seconds between calls. + Pass refresh=true right after changing DNS to skip the 10 second cache. It does not change DNS or SES. integrations-github-repos-retrieve: operation: integrations_github_repos_retrieve enabled: true diff --git a/products/workflows/backend/facade/api.py b/products/workflows/backend/facade/api.py index c088dbd58675..e3c943c8b3e3 100644 --- a/products/workflows/backend/facade/api.py +++ b/products/workflows/backend/facade/api.py @@ -9,6 +9,7 @@ from products.access_control.backend.facade.user_access_control import UserAccessControl from products.workflows.backend.facade.contracts import ( + EmailDomainCheck, EmailDomainDnsRecord, EmailDomainVerification, RecentWorkflow, @@ -296,6 +297,20 @@ def verify_ses_email_domain(domain: str, *, mail_from_subdomain: str, team_id: i return providers.SESProvider().verify_email_domain(domain, mail_from_subdomain=mail_from_subdomain, team_id=team_id) +def get_ses_email_domain_status(domain: str, *, mail_from_subdomain: str, team_id: int) -> EmailDomainCheck: + from products.workflows.backend import providers # noqa: PLC0415 + + return providers.SESProvider().get_email_domain_status( + domain, mail_from_subdomain=mail_from_subdomain, team_id=team_id + ) + + +def get_maildev_email_domain_status(domain: str, *, mail_from_subdomain: str) -> EmailDomainCheck: + from products.workflows.backend import providers # noqa: PLC0415 + + return providers.maildev_email_domain_status(domain, mail_from_subdomain=mail_from_subdomain) + + def delete_ses_identity(identity: str) -> None: from products.workflows.backend import providers # noqa: PLC0415 diff --git a/products/workflows/backend/facade/contracts.py b/products/workflows/backend/facade/contracts.py index 708eb3bcb6cc..659678938540 100644 --- a/products/workflows/backend/facade/contracts.py +++ b/products/workflows/backend/facade/contracts.py @@ -7,6 +7,12 @@ from posthog.dataclasses import frozen from products.workflows.backend.facade.enums import ( + EmailDomainSetupRecordKind, + EmailDomainSetupRecordStatus, + EmailDomainSetupRecordType, + EmailDomainSetupStatus, + EmailDomainSetupStepKey, + EmailDomainSetupStepState, HogFlowBatchJobState, HogFlowScheduleStatus, HogFlowTemplateExitCondition, @@ -313,6 +319,40 @@ class EmailDomainVerification(TypedDict): dnsRecords: list[EmailDomainDnsRecord] +@frozen +class EmailDomainStatusRecord: + kind: EmailDomainSetupRecordKind + hostname: str + type: EmailDomainSetupRecordType + value: str + priority: int | None = None + status: EmailDomainSetupRecordStatus = EmailDomainSetupRecordStatus.PENDING + + +@frozen +class EmailDomainStatusStep: + key: EmailDomainSetupStepKey + state: EmailDomainSetupStepState + + +@frozen +class EmailDomainCheck: + """What the email provider and public DNS report for a sending domain, read without changing either.""" + + status: EmailDomainSetupStatus + steps: tuple[EmailDomainStatusStep, ...] + records: tuple[EmailDomainStatusRecord, ...] + every_dns_lookup_answered: bool = True + + @property + def is_verified(self) -> bool: + return self.status == EmailDomainSetupStatus.VERIFIED + + +class EmailProviderUnavailableError(Exception): + """The email provider could not be reached or refused the read, so the domain's state is unknown.""" + + class TwilioPhoneNumber(TypedDict): """The fields callers read. The Twilio payload has more keys.""" diff --git a/products/workflows/backend/facade/enums.py b/products/workflows/backend/facade/enums.py index a31a0b1f5b2f..c75d6d38b4db 100644 --- a/products/workflows/backend/facade/enums.py +++ b/products/workflows/backend/facade/enums.py @@ -54,3 +54,45 @@ class WorkflowProposalStatus(LabeledStrEnum): APPROVED = "approved", "Approved" REJECTED = "rejected", "Rejected" APPLIED = "applied", "Applied" + + +class EmailDomainSetupStatus(LabeledStrEnum): + NOT_STARTED = "not_started", "Not started" + PENDING = "pending", "Waiting for DNS records" + RECORDS_FOUND = "records_found", "Records found, waiting for verification" + VERIFIED = "verified", "Verified" + TEMPORARY_FAILURE = "temporary_failure", "Temporary failure" + FAILED = "failed", "Failed" + + +class EmailDomainSetupRecordKind(LabeledStrEnum): + VERIFICATION = "verification", "Domain ownership" + DKIM = "dkim", "DKIM signing" + SPF = "spf", "SPF" + MAIL_FROM_MX = "mail_from_mx", "MAIL FROM MX" + MAIL_FROM_SPF = "mail_from_spf", "MAIL FROM SPF" + DMARC = "dmarc", "DMARC" + + +class EmailDomainSetupRecordType(LabeledStrEnum): + TXT = "TXT", "TXT" + CNAME = "CNAME", "CNAME" + MX = "MX", "MX" + + +class EmailDomainSetupRecordStatus(LabeledStrEnum): + PENDING = "pending", "Not found yet" + FOUND = "found", "Found in DNS" + VERIFIED = "verified", "Verified" + + +class EmailDomainSetupStepKey(LabeledStrEnum): + DOMAIN_ADDED = "domain_added", "Domain added" + RECORDS_FOUND = "records_found", "DNS records found" + VERIFIED = "verified", "Verified" + + +class EmailDomainSetupStepState(LabeledStrEnum): + DONE = "done", "Done" + PENDING = "pending", "Pending" + FAILED = "failed", "Failed" diff --git a/products/workflows/backend/providers/__init__.py b/products/workflows/backend/providers/__init__.py index 4d32cdad2c54..7aec5e3bc60d 100644 --- a/products/workflows/backend/providers/__init__.py +++ b/products/workflows/backend/providers/__init__.py @@ -1,5 +1,5 @@ -from .maildev import MAILDEV_MOCK_DNS_RECORDS +from .maildev import MAILDEV_MOCK_DNS_RECORDS, maildev_email_domain_status from .ses import SESProvider from .twilio import TwilioProvider -__all__ = ["TwilioProvider", "SESProvider", "MAILDEV_MOCK_DNS_RECORDS"] +__all__ = ["TwilioProvider", "SESProvider", "MAILDEV_MOCK_DNS_RECORDS", "maildev_email_domain_status"] diff --git a/products/workflows/backend/providers/maildev.py b/products/workflows/backend/providers/maildev.py index 2a8025809456..27f63fc697f9 100644 --- a/products/workflows/backend/providers/maildev.py +++ b/products/workflows/backend/providers/maildev.py @@ -1,4 +1,8 @@ -from products.workflows.backend.facade.contracts import EmailDomainDnsRecord +from dataclasses import replace + +from products.workflows.backend.facade.contracts import EmailDomainCheck, EmailDomainDnsRecord +from products.workflows.backend.facade.enums import EmailDomainSetupRecordStatus, EmailDomainSetupStatus +from products.workflows.backend.services.email_domain_status import build_email_domain_check, build_email_domain_records MAILDEV_MOCK_DNS_RECORDS: list[EmailDomainDnsRecord] = [ # Mock DNS records for email domain setup when using local maildev @@ -53,3 +57,17 @@ "status": "success", }, ] + + +def maildev_email_domain_status(domain: str, *, mail_from_subdomain: str) -> EmailDomainCheck: + records = build_email_domain_records( + domain=domain, + mail_from_subdomain=mail_from_subdomain, + verification_token="mock-verification-token", + dkim_tokens=["mock1", "mock2", "mock3"], + ses_region="us-east-1", + ) + return build_email_domain_check( + provider_status=EmailDomainSetupStatus.VERIFIED, + records=[replace(record, status=EmailDomainSetupRecordStatus.VERIFIED) for record in records], + ) diff --git a/products/workflows/backend/providers/ses.py b/products/workflows/backend/providers/ses.py index 2b6a4371c9de..740b08537225 100644 --- a/products/workflows/backend/providers/ses.py +++ b/products/workflows/backend/providers/ses.py @@ -2,13 +2,15 @@ import logging from collections import defaultdict from collections.abc import Iterable, Iterator, Mapping, Sequence +from dataclasses import replace from datetime import UTC, datetime, timedelta from functools import cached_property from itertools import batched from time import monotonic -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Literal, cast from django.conf import settings +from django.core.cache import cache import boto3 import dns.name @@ -18,12 +20,23 @@ from rest_framework import exceptions from posthog.dataclasses import frozen +from posthog.exceptions_capture import capture_exception from products.workflows.backend.facade.contracts import ( + EmailDomainCheck, EmailDomainDnsRecord, + EmailDomainStatusRecord, EmailDomainVerification, EmailDomainVerificationStatus, + EmailProviderUnavailableError, ) +from products.workflows.backend.facade.enums import ( + EmailDomainSetupRecordKind, + EmailDomainSetupRecordStatus, + EmailDomainSetupStatus, +) +from products.workflows.backend.services.email_domain_dns import find_published_records +from products.workflows.backend.services.email_domain_status import build_email_domain_check, build_email_domain_records if TYPE_CHECKING: from types_boto3_ses.client import SESClient @@ -32,6 +45,9 @@ logger = logging.getLogger(__name__) +AWS_ACCOUNT_ID_CACHE_KEY = "workflows:ses:aws_account_id" +AWS_ACCOUNT_ID_CACHE_SECONDS = 24 * 60 * 60 + # DELIVERY_COMPLAINT is the complaint denominator rather than SEND, because AWS defines it as # deliveries excluding recipients at ISPs it has no feedback-loop agreement with. # TRANSIENT_BOUNCE is queried because without it a provider that defers rather than rejects reads @@ -276,6 +292,107 @@ def remainder(metric: str) -> dict[str, int]: ) +_LegacyRecordPurpose = Literal["verification", "dkim", "mail_from", "dmarc"] +_LegacyRecordType = Literal["TXT", "CNAME", "MX"] + +_LEGACY_RECORD_PURPOSES: dict[EmailDomainSetupRecordKind, _LegacyRecordPurpose] = { + EmailDomainSetupRecordKind.VERIFICATION: "verification", + EmailDomainSetupRecordKind.SPF: "verification", + EmailDomainSetupRecordKind.DKIM: "dkim", + EmailDomainSetupRecordKind.MAIL_FROM_MX: "mail_from", + EmailDomainSetupRecordKind.MAIL_FROM_SPF: "mail_from", + EmailDomainSetupRecordKind.DMARC: "dmarc", +} + + +def _legacy_dns_record(record: EmailDomainStatusRecord) -> EmailDomainDnsRecord: + legacy: EmailDomainDnsRecord = { + "type": _LEGACY_RECORD_PURPOSES[record.kind], + "recordType": cast(_LegacyRecordType, record.type.value), + "recordHostname": "@" if record.kind == EmailDomainSetupRecordKind.SPF else record.hostname, + "recordValue": record.value, + "status": "pending", + } + if record.priority is not None: + legacy["priority"] = record.priority + return legacy + + +_UNKNOWN = "Unknown" + + +@frozen +class _SESIdentityState: + """The SES v1 identity attributes. A status is "Unknown" when SES has no attribute for the domain.""" + + verification_status: str + verification_token: str | None + dkim_status: str + dkim_tokens: tuple[str, ...] + mail_from_status: str + mail_from_domain: str | None + + @classmethod + def unknown(cls) -> "_SESIdentityState": + return cls( + verification_status=_UNKNOWN, + verification_token=None, + dkim_status=_UNKNOWN, + dkim_tokens=(), + mail_from_status=_UNKNOWN, + mail_from_domain=None, + ) + + # SES keeps one MAIL FROM domain per identity, and senders on the same domain can ask for + # different ones. The status SES reports belongs to whichever was set last. + def for_mail_from_domain(self, mail_from_domain: str) -> "_SESIdentityState": + if self.mail_from_domain == mail_from_domain: + return self + return replace(self, mail_from_status=_UNKNOWN) + + @property + def statuses(self) -> frozenset[str]: + return frozenset((self.verification_status, self.dkim_status, self.mail_from_status)) + + def is_ready(self, *, dmarc_published: bool) -> bool: + return dmarc_published and all(status == "Success" for status in self.statuses) + + def setup_status(self, *, is_ready: bool) -> EmailDomainSetupStatus: + if is_ready: + return EmailDomainSetupStatus.VERIFIED + if "Failed" in self.statuses: + return EmailDomainSetupStatus.FAILED + if "TemporaryFailure" in self.statuses: + return EmailDomainSetupStatus.TEMPORARY_FAILURE + return EmailDomainSetupStatus.PENDING + + def record_status(self, record: EmailDomainStatusRecord, *, published: bool) -> EmailDomainSetupRecordStatus: + if self._is_verified(record.kind, published=published): + return EmailDomainSetupRecordStatus.VERIFIED + return EmailDomainSetupRecordStatus.FOUND if published else EmailDomainSetupRecordStatus.PENDING + + def _is_verified(self, kind: EmailDomainSetupRecordKind, *, published: bool) -> bool: + match kind: + case EmailDomainSetupRecordKind.VERIFICATION: + return self.verification_status == "Success" + case EmailDomainSetupRecordKind.DKIM: + return self.dkim_status == "Success" + case EmailDomainSetupRecordKind.MAIL_FROM_MX: + return self.mail_from_status == "Success" + # SES checks only the MX record of the MAIL FROM domain, so its success says nothing about the SPF record. + case EmailDomainSetupRecordKind.MAIL_FROM_SPF: + return published and self.mail_from_status == "Success" + case EmailDomainSetupRecordKind.DMARC: + return published + case _: + return False + + +# Throttling is SES pacing a busy account. The next poll retries it, so reporting it adds only noise. +def _is_throttling(error: Exception) -> bool: + return isinstance(error, ClientError) and error.response["Error"]["Code"] in ("Throttling", "ThrottlingException") + + class SESProvider: ses_client: "SESClient" ses_v2_client: "SESV2Client" @@ -440,7 +557,11 @@ def _finding_scope(resource_arn: str) -> str: @cached_property def _aws_account_id(self) -> str: - return self.sts_client.get_caller_identity()["Account"] + account_id: str | None = cache.get(AWS_ACCOUNT_ID_CACHE_KEY) + if account_id is None: + account_id = self.sts_client.get_caller_identity()["Account"] + cache.set(AWS_ACCOUNT_ID_CACHE_KEY, account_id, AWS_ACCOUNT_ID_CACHE_SECONDS) + return account_id def _identity_arn(self, domain: str) -> str: return f"arn:aws:ses:{settings.SES_REGION}:{self._aws_account_id}:identity/{domain}" @@ -516,8 +637,6 @@ def verify_email_domain(self, domain: str, mail_from_subdomain: str, team_id: in if not re.match(DOMAIN_REGEX, domain): raise exceptions.ValidationError("Please enter a valid domain or subdomain name.") - dns_records: list[EmailDomainDnsRecord] = [] - # Start/ensure domain verification (TXT at _amazonses.domain) --- verification_token: str | None = None try: @@ -528,17 +647,6 @@ def verify_email_domain(self, domain: str, mail_from_subdomain: str, team_id: in if e.response["Error"]["Code"] not in ("InvalidParameterValue",): raise - if verification_token: - dns_records.append( - { - "type": "verification", - "recordType": "TXT", - "recordHostname": f"_amazonses.{domain}", - "recordValue": verification_token, - "status": "pending", - } - ) - # Start/ensure DKIM (three CNAMEs) --- dkim_tokens: list[str] = [] try: @@ -548,27 +656,6 @@ def verify_email_domain(self, domain: str, mail_from_subdomain: str, team_id: in if e.response["Error"]["Code"] not in ("InvalidParameterValue",): raise - for t in dkim_tokens: - dns_records.append( - { - "type": "dkim", - "recordType": "CNAME", - "recordHostname": f"{t}._domainkey.{domain}", - "recordValue": f"{t}.dkim.amazonses.com", - "status": "pending", - } - ) - - dns_records.append( - { - "type": "verification", - "recordType": "TXT", - "recordHostname": "@", - "recordValue": "v=spf1 include:amazonses.com ~all", - "status": "pending", - } - ) - # Start/ensure MAIL FROM setup (MX + TXT) --- try: self.ses_client.set_identity_mail_from_domain( @@ -580,67 +667,23 @@ def verify_email_domain(self, domain: str, mail_from_subdomain: str, team_id: in if e.response["Error"]["Code"] not in ("InvalidParameterValue",): raise - ses_region = getattr(settings, "SES_REGION", "us-east-1") - - dns_records.append( - { - "type": "mail_from", - "recordType": "MX", - "recordHostname": f"{mail_from_subdomain}.{domain}", - "recordValue": f"feedback-smtp.{ses_region}.amazonses.com", - "priority": 10, - "status": "pending", - } - ) - dns_records.append( - { - "type": "mail_from", - "recordType": "TXT", - "recordHostname": f"{mail_from_subdomain}.{domain}", - "recordValue": "v=spf1 include:amazonses.com ~all", - "status": "pending", - } - ) - - # DMARC — AWS SES has no method to check its presence, so we do a direct DNS - # lookup further below and include the result in the overall status. - dns_records.append( - { - "type": "dmarc", - "recordType": "TXT", - "recordHostname": f"_dmarc.{domain}", - "recordValue": "v=DMARC1; p=none;", - "status": "pending", - } - ) - - # Current verification / DKIM statuses to compute overall status & per-record statuses --- - verification_status: str = "Unknown" - try: - id_attrs = self.ses_client.get_identity_verification_attributes(Identities=[domain]) - id_for_domain = id_attrs["VerificationAttributes"].get(domain) - if id_for_domain is not None: - verification_status = id_for_domain["VerificationStatus"] - except ClientError: - pass + # DMARC is in the list too: AWS SES has no method to check its presence, so we do a direct + # DNS lookup further below and include the result in the overall status. + dns_records = [ + _legacy_dns_record(record) + for record in build_email_domain_records( + domain=domain, + mail_from_subdomain=mail_from_subdomain, + verification_token=verification_token, + dkim_tokens=dkim_tokens, + ses_region=self._ses_region, + ) + ] - dkim_status: str = "Unknown" - try: - dkim_attrs = self.ses_client.get_identity_dkim_attributes(Identities=[domain]) - dkim_for_domain = dkim_attrs["DkimAttributes"].get(domain) - if dkim_for_domain is not None: - dkim_status = dkim_for_domain["DkimVerificationStatus"] - except ClientError: - pass - - mail_from_status: str = "Unknown" - try: - mail_from_attrs = self.ses_client.get_identity_mail_from_domain_attributes(Identities=[domain]) - mail_from_for_domain = mail_from_attrs["MailFromDomainAttributes"].get(domain) - if mail_from_for_domain is not None: - mail_from_status = mail_from_for_domain["MailFromDomainStatus"] - except ClientError: - pass + identity = self._read_identity_state_or_unknown(domain) + verification_status = identity.verification_status + dkim_status = identity.dkim_status + mail_from_status = identity.mail_from_status # DMARC: check via direct DNS lookup since AWS SES doesn't track it dmarc_status = "Pending" @@ -660,26 +703,17 @@ def verify_email_domain(self, domain: str, mail_from_subdomain: str, team_id: in except Exception: logger.exception("Unexpected error during DMARC lookup for %s", domain) - all_statuses = [verification_status, dkim_status, mail_from_status] - # Normalize overall status overall: EmailDomainVerificationStatus - if ( - verification_status == "Success" - and dkim_status == "Success" - and mail_from_status == "Success" - and dmarc_status == "Success" - ): + if identity.is_ready(dmarc_published=dmarc_status == "Success"): overall = "success" - elif "Failed" in all_statuses: + elif "Failed" in identity.statuses: overall = "failed" else: overall = "pending" - if overall == "success": - expected_tenant = self._tenant_name_for_team(team_id) - if expected_tenant not in self._list_identity_tenants(domain): - overall = "pending" + if overall == "success" and not self._is_associated_with_team(domain, team_id): + overall = "pending" # Upgrade per-record statuses if SES reports success # - Domain verification TXT is considered verified when VerificationStatus == Success @@ -729,6 +763,75 @@ def verify_email_domain(self, domain: str, mail_from_subdomain: str, team_id: in "dnsRecords": dns_records, } + def get_email_domain_status(self, domain: str, mail_from_subdomain: str, team_id: int) -> EmailDomainCheck: + try: + return self._check_email_domain(domain, mail_from_subdomain=mail_from_subdomain, team_id=team_id) + except (ClientError, BotoCoreError) as error: + if not _is_throttling(error): + capture_exception(error, {"domain": domain, "team_id": team_id}) + raise EmailProviderUnavailableError(f"Could not read the SES state of {domain}") from error + + def _check_email_domain(self, domain: str, *, mail_from_subdomain: str, team_id: int) -> EmailDomainCheck: + identity = self._read_identity_state(domain).for_mail_from_domain(f"{mail_from_subdomain}.{domain}") + if identity.verification_token is None: + return build_email_domain_check(provider_status=EmailDomainSetupStatus.NOT_STARTED, records=()) + + records = build_email_domain_records( + domain=domain, + mail_from_subdomain=mail_from_subdomain, + verification_token=identity.verification_token, + dkim_tokens=identity.dkim_tokens, + ses_region=self._ses_region, + ) + published = find_published_records(domain, records) + dmarc_published = any(record.kind == EmailDomainSetupRecordKind.DMARC for record in published.records) + is_ready = ( + identity.is_ready(dmarc_published=dmarc_published) + and published.every_lookup_answered + and self._is_associated_with_team(domain, team_id) + ) + return build_email_domain_check( + provider_status=identity.setup_status(is_ready=is_ready), + records=[ + replace(record, status=identity.record_status(record, published=record in published.records)) + for record in records + ], + every_dns_lookup_answered=published.every_lookup_answered, + ) + + @property + def _ses_region(self) -> str: + return getattr(settings, "SES_REGION", "us-east-1") + + def _is_associated_with_team(self, domain: str, team_id: int) -> bool: + return self._tenant_name_for_team(team_id) in self._list_identity_tenants(domain) + + def _read_identity_state_or_unknown(self, domain: str) -> "_SESIdentityState": + try: + return self._read_identity_state(domain) + except ClientError: + logger.warning("Could not read the SES identity attributes for %s", domain, exc_info=True) + return _SESIdentityState.unknown() + + def _read_identity_state(self, domain: str) -> "_SESIdentityState": + id_for_domain = self.ses_client.get_identity_verification_attributes(Identities=[domain])[ + "VerificationAttributes" + ].get(domain) + dkim_for_domain = self.ses_client.get_identity_dkim_attributes(Identities=[domain])["DkimAttributes"].get( + domain + ) + mail_from_for_domain = self.ses_client.get_identity_mail_from_domain_attributes(Identities=[domain])[ + "MailFromDomainAttributes" + ].get(domain) + return _SESIdentityState( + verification_status=id_for_domain["VerificationStatus"] if id_for_domain else _UNKNOWN, + verification_token=id_for_domain.get("VerificationToken") if id_for_domain else None, + dkim_status=dkim_for_domain["DkimVerificationStatus"] if dkim_for_domain else _UNKNOWN, + dkim_tokens=tuple(dkim_for_domain.get("DkimTokens", [])) if dkim_for_domain else (), + mail_from_status=mail_from_for_domain["MailFromDomainStatus"] if mail_from_for_domain else _UNKNOWN, + mail_from_domain=mail_from_for_domain.get("MailFromDomain") if mail_from_for_domain else None, + ) + def update_mail_from_subdomain(self, domain: str, mail_from_subdomain: str): """ Update the MAIL FROM subdomain for a given identity diff --git a/products/workflows/backend/services/email_domain_dns.py b/products/workflows/backend/services/email_domain_dns.py new file mode 100644 index 000000000000..0334e59db99a --- /dev/null +++ b/products/workflows/backend/services/email_domain_dns.py @@ -0,0 +1,152 @@ +import logging +import ipaddress +from collections.abc import Sequence +from concurrent.futures import ThreadPoolExecutor + +import dns.rdata +import dns.resolver +import dns.exception +from dns.rdtypes.ANY.CNAME import CNAME +from dns.rdtypes.ANY.MX import MX +from dns.rdtypes.ANY.NS import NS +from dns.rdtypes.ANY.TXT import TXT +from dns.rdtypes.IN.A import A + +from posthog.dataclasses import frozen + +from products.workflows.backend.facade.contracts import EmailDomainStatusRecord +from products.workflows.backend.facade.enums import ( + EmailDomainSetupRecordKind as Kind, + EmailDomainSetupRecordType as RecordType, +) + +logger = logging.getLogger(__name__) + +LOOKUP_TIMEOUT_SECONDS = 3 +MAX_AUTHORITATIVE_NAMESERVERS = 2 +ABSENT_ERRORS = (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) + + +@frozen +class _Lookup: + name: str + type: RecordType + + +@frozen +class PublishedRecords: + records: frozenset[EmailDomainStatusRecord] + unanswered: frozenset[EmailDomainStatusRecord] = frozenset() + + @property + def every_lookup_answered(self) -> bool: + return not self.unanswered + + +# The zone's own nameservers answer first because public resolvers cache a negative answer for the +# zone's SOA minimum TTL, which hides a record the customer just added for up to an hour. A public +# resolver then checks what they could not confirm, because they cannot follow a CNAME to another +# zone, such as a DMARC record hosted by a DMARC reporting service. +def find_published_records(domain: str, records: Sequence[EmailDomainStatusRecord]) -> PublishedRecords: + authoritative = _authoritative_resolver(domain) + from_zone = _published_with(authoritative, records) if authoritative else PublishedRecords(records=frozenset()) + from_public = _published_with(_system_resolver(), [record for record in records if record not in from_zone.records]) + return PublishedRecords(records=from_zone.records | from_public.records, unanswered=from_public.unanswered) + + +def _published_with(resolver: dns.resolver.Resolver, records: Sequence[EmailDomainStatusRecord]) -> PublishedRecords: + if not records: + return PublishedRecords(records=frozenset()) + lookups = {_lookup_for(record) for record in records} + with ThreadPoolExecutor(max_workers=len(lookups)) as pool: + answers = dict(zip(lookups, pool.map(lambda lookup: _resolve(resolver, lookup), lookups))) + return PublishedRecords( + records=frozenset(record for record in records if _is_published(record, answers[_lookup_for(record)] or [])), + unanswered=frozenset(record for record in records if answers[_lookup_for(record)] is None), + ) + + +def _lookup_for(record: EmailDomainStatusRecord) -> _Lookup: + return _Lookup(name=record.hostname, type=record.type) + + +def _system_resolver() -> dns.resolver.Resolver: + resolver = dns.resolver.Resolver() + resolver.lifetime = LOOKUP_TIMEOUT_SECONDS + return resolver + + +def _authoritative_resolver(domain: str) -> dns.resolver.Resolver | None: + system = _system_resolver() + try: + zone = dns.resolver.zone_for_name(domain, resolver=system, lifetime=LOOKUP_TIMEOUT_SECONDS) + nameservers = [rdata.target for rdata in system.resolve(zone, "NS") if isinstance(rdata, NS)] + addresses = [ + rdata.address + for nameserver in nameservers[:MAX_AUTHORITATIVE_NAMESERVERS] + for rdata in system.resolve(nameserver, "A") + if isinstance(rdata, A) and _is_public_address(rdata.address) + ] + except dns.exception.DNSException: + return None + if not addresses: + return None + authoritative = dns.resolver.Resolver(configure=False) + authoritative.nameservers = addresses + authoritative.lifetime = LOOKUP_TIMEOUT_SECONDS + return authoritative + + +# The customer controls the zone's NS records, so they could point our queries at internal addresses. +def _is_public_address(address: str) -> bool: + ip = ipaddress.ip_address(address) + return ip.is_global and not ip.is_multicast + + +def _resolve(resolver: dns.resolver.Resolver, lookup: _Lookup) -> list[str] | None: + try: + answers = resolver.resolve(lookup.name, lookup.type.value) + except ABSENT_ERRORS: + return [] + except dns.exception.DNSException: + logger.info("No DNS answer for %s %s", lookup.type.value, lookup.name, exc_info=True) + return None + return [answer for rdata in answers if (answer := _normalized_answer(rdata)) is not None] + + +def _normalized_answer(rdata: dns.rdata.Rdata) -> str | None: + if isinstance(rdata, TXT): + return b"".join(rdata.strings).decode("utf-8", errors="replace").strip().strip('"') + if isinstance(rdata, CNAME): + return _normalized_hostname(rdata.target.to_text()) + if isinstance(rdata, MX): + return f"{rdata.preference} {_normalized_hostname(rdata.exchange.to_text())}" + return None + + +def _normalized_hostname(hostname: str) -> str: + return hostname.lower().rstrip(".") + + +def _is_published(record: EmailDomainStatusRecord, answers: list[str]) -> bool: + match record.kind, record.type: + case Kind.DMARC, _: + return any(answer.lower().startswith("v=dmarc1") for answer in answers) + case Kind.SPF | Kind.MAIL_FROM_SPF, _: + return any(_spf_includes(answer, record.value) for answer in answers) + case _, RecordType.CNAME: + return _normalized_hostname(record.value) in answers + case _, RecordType.MX: + return f"{record.priority} {_normalized_hostname(record.value)}" in answers + case _: + return record.value in answers + + +# A domain can publish only one SPF record, so SES's mechanism usually sits next to another +# sender's. The record counts as published when it carries every include the expected one does. +def _spf_includes(published: str, expected: str) -> bool: + published_terms = published.lower().split() + if not published_terms or published_terms[0] != "v=spf1": + return False + expected_includes = {term for term in expected.lower().split() if term.startswith("include:")} + return expected_includes.issubset(published_terms) diff --git a/products/workflows/backend/services/email_domain_status.py b/products/workflows/backend/services/email_domain_status.py new file mode 100644 index 000000000000..c6a590392036 --- /dev/null +++ b/products/workflows/backend/services/email_domain_status.py @@ -0,0 +1,111 @@ +from collections.abc import Sequence + +from products.workflows.backend.facade.contracts import EmailDomainCheck, EmailDomainStatusRecord, EmailDomainStatusStep +from products.workflows.backend.facade.enums import ( + EmailDomainSetupRecordKind as Kind, + EmailDomainSetupRecordStatus as RecordStatus, + EmailDomainSetupRecordType as RecordType, + EmailDomainSetupStatus as Status, + EmailDomainSetupStepKey as StepKey, + EmailDomainSetupStepState as StepState, +) + +SES_SPF_VALUE = "v=spf1 include:amazonses.com ~all" +DEFAULT_DMARC_VALUE = "v=DMARC1; p=none;" + + +def build_email_domain_records( + *, + domain: str, + mail_from_subdomain: str, + verification_token: str | None, + dkim_tokens: Sequence[str], + ses_region: str, +) -> tuple[EmailDomainStatusRecord, ...]: + mail_from_domain = f"{mail_from_subdomain}.{domain}" + verification = ( + [ + EmailDomainStatusRecord( + kind=Kind.VERIFICATION, hostname=f"_amazonses.{domain}", type=RecordType.TXT, value=verification_token + ) + ] + if verification_token + else [] + ) + dkim = [ + EmailDomainStatusRecord( + kind=Kind.DKIM, + hostname=f"{token}._domainkey.{domain}", + type=RecordType.CNAME, + value=f"{token}.dkim.amazonses.com", + ) + for token in dkim_tokens + ] + return ( + *verification, + *dkim, + EmailDomainStatusRecord(kind=Kind.SPF, hostname=domain, type=RecordType.TXT, value=SES_SPF_VALUE), + EmailDomainStatusRecord( + kind=Kind.MAIL_FROM_MX, + hostname=mail_from_domain, + type=RecordType.MX, + value=f"feedback-smtp.{ses_region}.amazonses.com", + priority=10, + ), + EmailDomainStatusRecord( + kind=Kind.MAIL_FROM_SPF, hostname=mail_from_domain, type=RecordType.TXT, value=SES_SPF_VALUE + ), + EmailDomainStatusRecord( + kind=Kind.DMARC, hostname=f"_dmarc.{domain}", type=RecordType.TXT, value=DEFAULT_DMARC_VALUE + ), + ) + + +def build_email_domain_check( + *, provider_status: Status, records: Sequence[EmailDomainStatusRecord], every_dns_lookup_answered: bool = True +) -> EmailDomainCheck: + status = _overall_status(provider_status, records) + return EmailDomainCheck( + status=status, + steps=_steps(status, records), + records=tuple(records), + every_dns_lookup_answered=every_dns_lookup_answered, + ) + + +def _overall_status(provider_status: Status, records: Sequence[EmailDomainStatusRecord]) -> Status: + if provider_status == Status.PENDING and _all_records_published(records): + return Status.RECORDS_FOUND + return provider_status + + +def _all_records_published(records: Sequence[EmailDomainStatusRecord]) -> bool: + return bool(records) and all(record.status != RecordStatus.PENDING for record in records) + + +def _steps(status: Status, records: Sequence[EmailDomainStatusRecord]) -> tuple[EmailDomainStatusStep, ...]: + return ( + EmailDomainStatusStep(key=StepKey.DOMAIN_ADDED, state=_domain_added_state(status)), + EmailDomainStatusStep(key=StepKey.RECORDS_FOUND, state=_records_found_state(status, records)), + EmailDomainStatusStep(key=StepKey.VERIFIED, state=_verified_state(status)), + ) + + +def _domain_added_state(status: Status) -> StepState: + return StepState.PENDING if status == Status.NOT_STARTED else StepState.DONE + + +def _records_found_state(status: Status, records: Sequence[EmailDomainStatusRecord]) -> StepState: + if status == Status.VERIFIED or _all_records_published(records): + return StepState.DONE + if status in (Status.FAILED, Status.TEMPORARY_FAILURE): + return StepState.FAILED + return StepState.PENDING + + +def _verified_state(status: Status) -> StepState: + if status == Status.VERIFIED: + return StepState.DONE + if status == Status.FAILED: + return StepState.FAILED + return StepState.PENDING diff --git a/products/workflows/backend/test/email_domain_api_test_case.py b/products/workflows/backend/test/email_domain_api_test_case.py new file mode 100644 index 000000000000..d6a8437a3e0c --- /dev/null +++ b/products/workflows/backend/test/email_domain_api_test_case.py @@ -0,0 +1,89 @@ +from typing import Any + +from posthog.test.base import APIBaseTest +from unittest.mock import patch + +from django.core.cache import cache +from django.test import override_settings + +from rest_framework import status + +from posthog.models.integration import Integration +from posthog.models.organization import OrganizationMembership +from posthog.models.team import Team + +from products.workflows.backend.test.fakes.world import EmailDomainWorld + +DOMAIN = "mail.example.com" +ROOT_DOMAIN = "example.com" +ZONE_NAMESERVERS = {"ns1.example-dns.net": "34.120.0.1", "ns2.example-dns.net": "34.120.0.2"} +ALL_KINDS = ("verification", "dkim", "spf", "mail_from_mx", "mail_from_spf", "dmarc") + + +@override_settings(SES_REGION="us-east-1", SES_TENANT_CONFIGURATION_SETS=[]) +class EmailDomainApiTestCase(APIBaseTest): + zone_nameservers: dict[str, str] = ZONE_NAMESERVERS + + def setUp(self) -> None: + super().setUp() + cache.clear() + self.organization_membership.level = OrganizationMembership.Level.ADMIN + self.organization_membership.save() + self.world = self.enterContext(EmailDomainWorld().installed()) + self.world.dns.add_zone(ROOT_DOMAIN, nameservers=self.zone_nameservers) + self.reload_workers = self.enterContext( + patch("posthog.models.integration.email.reload_integrations_on_workers") + ) + self.report_user_action = self.enterContext(patch("posthog.api.integration.report_user_action")) + + def create_sender(self, email: str, *, team: Team | None = None) -> int: + response = self.client.post( + f"/api/projects/{(team or self.team).id}/integrations", + {"kind": "email", "config": {"email": email, "name": "Example", "provider": "ses"}}, + format="json", + ) + assert response.status_code == status.HTTP_201_CREATED, response.json() + return response.json()["id"] + + def get_status(self, integration_id: int, *, refresh: bool = True) -> Any: + return self.client.get( + f"/api/projects/{self.team.id}/integrations/{integration_id}/email/status", + {"refresh": str(refresh).lower()}, + ) + + def status_body(self, integration_id: int, *, refresh: bool = True) -> dict[str, Any]: + response = self.get_status(integration_id, refresh=refresh) + assert response.status_code == status.HTTP_200_OK, response.json() + return response.json() + + def records_to_publish(self, integration_id: int, *, domain: str = DOMAIN) -> list[dict[str, Any]]: + identity = self.world.ses.identities[domain] + identity.checks_paused = True + records = self.status_body(integration_id)["records"] + identity.checks_paused = False + return records + + def publish(self, records: list[dict[str, Any]], *kinds: str) -> None: + for record in records: + if record["kind"] in kinds: + self.world.publish_record(record) + + def is_verified(self, integration_id: int) -> bool: + return Integration.objects.get(id=integration_id).config["verified"] + + def verified_events(self) -> list[Any]: + return [call for call in self.report_user_action.call_args_list if call.args[1] == "email domain verified"] + + def verified_event_methods(self) -> list[str]: + return [call.args[2]["method"] for call in self.verified_events()] + + +def record_statuses(body: dict[str, Any]) -> dict[str, list[str]]: + statuses: dict[str, list[str]] = {} + for record in body["records"]: + statuses.setdefault(record["kind"], []).append(record["status"]) + return statuses + + +def step_states(body: dict[str, Any]) -> list[str]: + return [step["state"] for step in body["steps"]] diff --git a/products/workflows/backend/test/test_email_domain_status.py b/products/workflows/backend/test/test_email_domain_status.py new file mode 100644 index 000000000000..f39afcf01558 --- /dev/null +++ b/products/workflows/backend/test/test_email_domain_status.py @@ -0,0 +1,54 @@ +from django.test import SimpleTestCase + +from parameterized import parameterized + +from products.workflows.backend.facade.contracts import EmailDomainStatusRecord +from products.workflows.backend.facade.enums import ( + EmailDomainSetupRecordKind as Kind, + EmailDomainSetupRecordType as RecordType, +) +from products.workflows.backend.services.email_domain_dns import find_published_records +from products.workflows.backend.test.fakes.world import EmailDomainWorld + +DOMAIN = "mail.example.com" + + +def _record(kind: Kind, record_type: RecordType, hostname: str, value: str, priority: int | None = None): + return EmailDomainStatusRecord(kind=kind, hostname=hostname, type=record_type, value=value, priority=priority) + + +SPF = _record(Kind.SPF, RecordType.TXT, DOMAIN, "v=spf1 include:amazonses.com ~all") +VERIFICATION = _record(Kind.VERIFICATION, RecordType.TXT, f"_amazonses.{DOMAIN}", "tok") +DKIM = _record(Kind.DKIM, RecordType.CNAME, f"dk1._domainkey.{DOMAIN}", "dk1.dkim.amazonses.com") +MAIL_FROM_MX = _record( + Kind.MAIL_FROM_MX, RecordType.MX, f"feedback.{DOMAIN}", "feedback-smtp.us-east-1.amazonses.com", priority=10 +) +DMARC = _record(Kind.DMARC, RecordType.TXT, f"_dmarc.{DOMAIN}", "v=DMARC1; p=none;") + + +class TestFindPublishedRecords(SimpleTestCase): + @parameterized.expand( + [ + ("txt_split_into_chunks", VERIFICATION, '"to" "k"', True), + ("txt_other_value", VERIFICATION, '"other"', False), + ( + "spf_merged_with_another_sender", + SPF, + '"v=spf1 include:_spf.google.com include:amazonses.com -all"', + True, + ), + ("spf_without_ses_include", SPF, '"v=spf1 include:_spf.google.com ~all"', False), + ("cname_in_upper_case", DKIM, "DK1.DKIM.AMAZONSES.COM.", True), + ("mx_with_other_priority", MAIL_FROM_MX, "20 feedback-smtp.us-east-1.amazonses.com.", False), + ("existing_strict_dmarc", DMARC, '"v=DMARC1; p=reject; rua=mailto:dmarc@example.com"', True), + ] + ) + def test_matches_published_value( + self, _name: str, record: EmailDomainStatusRecord, published: str, expected: bool + ) -> None: + world = EmailDomainWorld() + world.dns.add_zone("example.com", nameservers={"ns1.example-dns.net": "34.120.0.1"}) + world.dns.publish(record.hostname, record.type.value, published) + + with world.installed(): + assert (record in find_published_records(DOMAIN, [record]).records) is expected diff --git a/products/workflows/backend/test/test_email_domain_status_api.py b/products/workflows/backend/test/test_email_domain_status_api.py new file mode 100644 index 000000000000..4fb62213172e --- /dev/null +++ b/products/workflows/backend/test/test_email_domain_status_api.py @@ -0,0 +1,381 @@ +from collections.abc import Callable +from itertools import product + +from unittest.mock import ANY, patch + +from django.core.cache import cache + +from parameterized import parameterized +from rest_framework import status + +from posthog.models.integration import Integration +from posthog.models.organization import Organization +from posthog.models.personal_api_key import PersonalAPIKey +from posthog.models.team import Team +from posthog.models.utils import hash_key_value + +from products.workflows.backend.test.email_domain_api_test_case import ( + ALL_KINDS, + DOMAIN, + ROOT_DOMAIN, + EmailDomainApiTestCase, + record_statuses, + step_states, +) +from products.workflows.backend.test.fakes.fake_dns import dns_operation +from products.workflows.backend.test.fakes.faults import AwsFault, DnsFault +from products.workflows.backend.test.fakes.world import EmailDomainWorld + +SES_STATUS_READS = ( + "ses.get_identity_verification_attributes", + "ses.get_identity_dkim_attributes", + "ses.get_identity_mail_from_domain_attributes", + "sts.get_caller_identity", + "sesv2.list_resource_tenants", +) + + +class TestEmailDomainVerification(EmailDomainApiTestCase): + def test_senders_become_verified_as_the_records_are_published_and_the_event_fires_once(self) -> None: + sender = self.create_sender(f"hello@{DOMAIN}") + sibling = self.create_sender(f"news@{DOMAIN}") + self.world.ses.identities[DOMAIN].checks_paused = True + + body = self.status_body(sender) + assert (body["status"], body["verified"], step_states(body)) == ( + "pending", + False, + ["done", "pending", "pending"], + ) + assert {status for statuses in record_statuses(body).values() for status in statuses} == {"pending"} + records = body["records"] + + for kind in ALL_KINDS[:-1]: + self.publish(records, kind) + body = self.status_body(sender) + assert (body["status"], set(record_statuses(body)[kind])) == ("pending", {"found"}), kind + + self.publish(records, "dmarc") + body = self.status_body(sender) + assert (body["status"], step_states(body)) == ("records_found", ["done", "done", "pending"]) + assert not self.is_verified(sender) + assert self.verified_events() == [] + + self.world.ses.identities[DOMAIN].checks_paused = False + body = self.status_body(sender) + assert (body["status"], body["verified"], step_states(body)) == ("verified", True, ["done", "done", "done"]) + assert record_statuses(body) == { + "verification": ["verified"], + "dkim": ["verified"] * 3, + "spf": ["found"], + "mail_from_mx": ["verified"], + "mail_from_spf": ["verified"], + "dmarc": ["verified"], + } + assert self.is_verified(sender) and self.is_verified(sibling) + assert self.verified_events() == [ + ((ANY, "email domain verified", {"method": "manual", "seconds_since_created": ANY}), {"team": ANY}) + ] + self.reload_workers.assert_called_once_with(self.team.id, sorted([sender, sibling])) + assert self.world.ses.writes.count("verify_domain_identity") == 2 + + late_sender = self.create_sender(f"billing@{DOMAIN}") + assert self.status_body(late_sender)["verified"] is True + assert self.is_verified(late_sender) + assert len(self.verified_events()) == 1 + + def test_status_reads_ses_without_writing_to_it(self) -> None: + sender = self.create_sender(f"hello@{DOMAIN}") + self.publish(self.records_to_publish(sender), *ALL_KINDS) + writes_after_create = list(self.world.ses.writes) + + assert self.status_body(sender)["status"] == "verified" + + assert self.world.ses.writes == writes_after_create + + def test_status_is_cached_until_refresh(self) -> None: + sender = self.create_sender(f"hello@{DOMAIN}") + records = self.records_to_publish(sender) + assert self.status_body(sender, refresh=False)["status"] == "pending" + + self.publish(records, *ALL_KINDS) + + assert self.status_body(sender, refresh=False)["status"] == "pending" + assert self.status_body(sender, refresh=True)["status"] == "verified" + + def test_verification_through_a_personal_api_key_is_reported_as_agent_setup(self) -> None: + sender = self.create_sender(f"hello@{DOMAIN}") + self.publish(self.records_to_publish(sender), *ALL_KINDS) + PersonalAPIKey.objects.create( + label="Agent", user=self.user, secure_value=hash_key_value("agent_key"), scopes=["integration:read"] + ) + self.client.logout() + self.client.credentials(HTTP_AUTHORIZATION="Bearer agent_key") + + assert self.status_body(sender)["verified"] is True + + assert self.verified_event_methods() == ["agent"] + + def test_failed_worker_reload_keeps_the_verified_answer_and_is_not_retried(self) -> None: + sender = self.create_sender(f"hello@{DOMAIN}") + self.publish(self.records_to_publish(sender), *ALL_KINDS) + self.reload_workers.side_effect = ConnectionError("Redis is down") + + with patch("posthog.models.integration.email.capture_exception") as capture_exception: + assert self.status_body(sender)["verified"] is True + capture_exception.assert_called_once() + + self.reload_workers.side_effect = None + assert self.status_body(sender)["verified"] is True + assert self.is_verified(sender) + assert self.reload_workers.call_count == 1 + assert len(self.verified_events()) == 1 + + +class TestEmailDomainPartialSesState(EmailDomainApiTestCase): + @parameterized.expand( + [ + ( + "verification_token_missing", + lambda world: setattr(world.ses.identities[DOMAIN], "verification_token", None), + "not_started", + {}, + ), + ( + "dkim_never_enabled", + lambda world: setattr(world.ses.identities[DOMAIN], "dkim_tokens", ()), + "records_found", + { + "verification": ["verified"], + "spf": ["found"], + "mail_from_mx": ["verified"], + "mail_from_spf": ["verified"], + "dmarc": ["verified"], + }, + ), + ( + "only_two_of_three_dkim_records_published", + lambda world: world.dns.unpublish( + f"{world.ses.identities[DOMAIN].dkim_tokens[2]}._domainkey.{DOMAIN}", "CNAME" + ), + "pending", + {"dkim": ["found", "found", "pending"]}, + ), + ( + "mail_from_never_set", + lambda world: setattr(world.ses.identities[DOMAIN], "mail_from_domain", None), + "records_found", + {"mail_from_mx": ["found"], "mail_from_spf": ["found"]}, + ), + ( + "mail_from_set_for_another_senders_subdomain", + lambda world: setattr(world.ses.identities[DOMAIN], "mail_from_domain", f"bounce.{DOMAIN}"), + "records_found", + {"mail_from_mx": ["found"], "mail_from_spf": ["found"]}, + ), + ( + "mail_from_mx_published_without_its_spf_record", + lambda world: world.dns.unpublish(f"feedback.{DOMAIN}", "TXT"), + "verified", + {"mail_from_mx": ["verified"], "mail_from_spf": ["pending"]}, + ), + ( + "second_mx_record_on_the_bounce_subdomain", + lambda world: world.dns.publish(f"feedback.{DOMAIN}", "MX", "20 mx.example.net."), + "records_found", + {"mail_from_mx": ["found"], "mail_from_spf": ["found"]}, + ), + ( + "identity_owned_by_another_teams_tenant", + lambda world: world.ses.transfer_identity(DOMAIN, "team-999999"), + "records_found", + {"verification": ["verified"], "dkim": ["verified"] * 3}, + ), + ( + "dkim_failed", + lambda world: setattr(world.ses.identities[DOMAIN], "forced_dkim_status", "Failed"), + "failed", + {"dkim": ["found"] * 3}, + ), + ( + "verification_temporary_failure", + lambda world: setattr(world.ses.identities[DOMAIN], "forced_verification_status", "TemporaryFailure"), + "temporary_failure", + {"verification": ["found"]}, + ), + ] + ) + def test_reports_partial_ses_state_without_verifying( + self, + _name: str, + arrange: Callable[[EmailDomainWorld], object], + expected_status: str, + expected_records: dict[str, list[str]], + ) -> None: + sender = self.create_sender(f"hello@{DOMAIN}") + self.publish(self.records_to_publish(sender), *ALL_KINDS) + arrange(self.world) + + body = self.status_body(sender) + + assert body["status"] == expected_status + assert {kind: statuses for kind, statuses in record_statuses(body).items() if kind in expected_records} == ( + expected_records + ) + if expected_status != "verified": + assert (body["verified"], self.is_verified(sender), self.verified_events()) == (False, False, []) + + +class TestEmailDomainTenantScoping(EmailDomainApiTestCase): + def test_verifying_a_domain_leaves_other_teams_senders_on_it_untouched(self) -> None: + other_team = Team.objects.create(organization=self.organization, name="Other project") + sender = self.create_sender(f"hello@{DOMAIN}") + other_teams_sender = self.create_sender(f"hello@{DOMAIN}", team=other_team) + self.publish(self.records_to_publish(sender), *ALL_KINDS) + + assert self.status_body(sender)["verified"] is True + + assert self.is_verified(sender) + assert not self.is_verified(other_teams_sender) + self.reload_workers.assert_called_once_with(self.team.id, [sender]) + + def test_a_sender_of_another_organization_is_not_found(self) -> None: + other_team = Team.objects.create(organization=Organization.objects.create(name="Other"), name="Other") + foreign_sender = Integration.objects.create( + team=other_team, + kind="email", + integration_id=f"hello@{DOMAIN}", + config={"email": f"hello@{DOMAIN}", "domain": DOMAIN, "provider": "ses", "verified": False}, + ) + + response = self.get_status(foreign_sender.id) + + assert response.status_code == status.HTTP_404_NOT_FOUND + assert not self.is_verified(foreign_sender.id) + + +class TestEmailDomainDnsLookups(EmailDomainApiTestCase): + def test_dmarc_record_hosted_in_another_zone_through_a_cname_counts(self) -> None: + self.world.dns.add_zone("dmarc-reports.example.net") + sender = self.create_sender(f"hello@{DOMAIN}") + self.publish(self.records_to_publish(sender), *ALL_KINDS[:-1]) + self.world.dns.publish(f"_dmarc.{DOMAIN}", "CNAME", f"{DOMAIN}._dmarc.dmarc-reports.example.net.") + self.world.dns.publish_txt(f"{DOMAIN}._dmarc.dmarc-reports.example.net", "v=DMARC1; p=reject") + + body = self.status_body(sender) + + assert (body["status"], record_statuses(body)["dmarc"]) == ("verified", ["verified"]) + + @parameterized.expand([(DnsFault.TIMEOUT,), (DnsFault.SERVFAIL,)]) + def test_public_resolver_confirms_a_record_the_zone_nameservers_cannot_answer(self, fault: DnsFault) -> None: + sender = self.create_sender(f"hello@{DOMAIN}") + self.publish(self.records_to_publish(sender), *ALL_KINDS) + self.world.faults.inject(dns_operation(f"_dmarc.{DOMAIN}", "TXT", at="zone"), fault) + + assert self.status_body(sender)["status"] == "verified" + + def test_zone_nameservers_confirm_a_record_the_public_resolver_has_not_seen_yet(self) -> None: + sender = self.create_sender(f"hello@{DOMAIN}") + self.publish(self.records_to_publish(sender), *ALL_KINDS) + self.world.faults.inject(dns_operation(f"_dmarc.{DOMAIN}", "TXT", at="public"), DnsFault.NXDOMAIN) + + assert self.status_body(sender)["status"] == "verified" + + @parameterized.expand( + [ + ("zone_apex_lookup", ROOT_DOMAIN, "SOA"), + ("nameserver_lookup", ROOT_DOMAIN, "NS"), + ("nameserver_address_lookup", "ns1.example-dns.net", "A"), + ] + ) + def test_failing_zone_nameserver_discovery_falls_back_to_the_public_resolver( + self, _name: str, name: str, rdtype: str + ) -> None: + sender = self.create_sender(f"hello@{DOMAIN}") + self.publish(self.records_to_publish(sender), *ALL_KINDS) + self.world.faults.inject(dns_operation(name, rdtype), DnsFault.SERVFAIL) + + assert self.status_body(sender)["status"] == "verified" + + def test_txt_records_split_into_several_strings_count(self) -> None: + sender = self.create_sender(f"hello@{DOMAIN}") + records = self.records_to_publish(sender) + self.publish(records, "verification", "dkim", "mail_from_mx") + self.world.dns.publish(DOMAIN, "TXT", '"v=spf1 include:" "amazonses.com ~all"') + self.world.dns.publish(f"feedback.{DOMAIN}", "TXT", '"v=spf1 " "include:amazonses.com ~all"') + self.world.dns.publish(f"_dmarc.{DOMAIN}", "TXT", '"v=DMARC1; " "p=none;"') + + body = self.status_body(sender) + + assert body["status"] == "verified" + assert record_statuses(body)["spf"] == ["found"] + + def test_never_queries_a_zone_nameserver_at_an_internal_address(self) -> None: + self.world.dns.add_zone("internal.example.org", nameservers={"ns.internal.example.org": "10.0.0.53"}) + sender = self.create_sender("hello@internal.example.org") + self.publish(self.records_to_publish(sender, domain="internal.example.org"), *ALL_KINDS) + + assert self.status_body(sender)["status"] == "verified" + assert "10.0.0.53" not in self.world.dns.queried_addresses + + +class TestEmailDomainStatusChaos(EmailDomainApiTestCase): + def setUp(self) -> None: + super().setUp() + self.capture_exception = self.enterContext(patch("products.workflows.backend.providers.ses.capture_exception")) + self.sender = self.create_sender(f"hello@{DOMAIN}") + self.publish(self.records_to_publish(self.sender), *ALL_KINDS) + cache.clear() + + @parameterized.expand( + product(SES_STATUS_READS, list(AwsFault)), name_func=lambda f, _n, p: f"{f.__name__}_{p.args[0]}_{p.args[1]}" + ) + def test_aws_failure_answers_503_and_the_next_poll_recovers(self, operation: str, fault: AwsFault) -> None: + self.world.faults.inject(operation, fault) + + response = self.get_status(self.sender, refresh=False) + + assert response.status_code == status.HTTP_503_SERVICE_UNAVAILABLE + assert response.json() == { + "type": "server_error", + "code": "email_provider_unavailable", + "detail": "Couldn't reach the email provider to check this domain. Try again in a moment.", + "attr": None, + } + assert (self.is_verified(self.sender), self.verified_events()) == (False, []) + assert self.capture_exception.called is (fault != AwsFault.THROTTLING) + + self.world.faults.clear() + assert self.status_body(self.sender, refresh=False)["verified"] is True + assert self.verified_event_methods() == ["manual"] + + @parameterized.expand( + product( + [ + ("verification", f"_amazonses.{DOMAIN}", "TXT", "verified"), + ("dkim", None, "CNAME", "verified"), + ("spf", DOMAIN, "TXT", "found"), + ("mail_from_mx", f"feedback.{DOMAIN}", "MX", "verified"), + ("mail_from_spf", f"feedback.{DOMAIN}", "TXT", "verified"), + ("dmarc", f"_dmarc.{DOMAIN}", "TXT", "verified"), + ], + [DnsFault.TIMEOUT, DnsFault.SERVFAIL], + ), + name_func=lambda f, _n, p: f"{f.__name__}_{p.args[0][0]}_{p.args[1]}", + ) + def test_unanswered_dns_lookup_holds_verification_until_it_answers( + self, lookup: tuple[str, str | None, str, str], fault: DnsFault + ) -> None: + kind, hostname, rdtype, status_once_answered = lookup + dkim_token = self.world.ses.identities[DOMAIN].dkim_tokens[0] + self.world.faults.inject(dns_operation(hostname or f"{dkim_token}._domainkey.{DOMAIN}", rdtype), fault) + + body = self.status_body(self.sender, refresh=False) + + assert body["status"] in ("pending", "records_found") + assert (body["verified"], self.is_verified(self.sender), self.verified_events()) == (False, False, []) + + self.world.faults.clear() + body = self.status_body(self.sender, refresh=False) + assert (body["status"], record_statuses(body)[kind][0]) == ("verified", status_once_answered) + assert self.verified_event_methods() == ["manual"] diff --git a/products/workflows/backend/test/test_email_integration_ses.py b/products/workflows/backend/test/test_email_integration_ses.py index ad8f8c6380c4..11d2c9cb1a3b 100644 --- a/products/workflows/backend/test/test_email_integration_ses.py +++ b/products/workflows/backend/test/test_email_integration_ses.py @@ -2,12 +2,15 @@ from posthog.test.base import BaseTest from unittest.mock import MagicMock, patch +from django.core.cache import cache + from posthog.api.integration import IntegrationViewSet from posthog.models.integration import EmailIntegration, Integration from posthog.models.organization import Organization from posthog.models.team.team import Team from products.workflows.backend.providers import SESProvider +from products.workflows.backend.providers.ses import AWS_ACCOUNT_ID_CACHE_KEY class TestEmailIntegrationCrossTenantStaleVerification(BaseTest): @@ -159,12 +162,14 @@ def test_takeover_after_owner_deletes_integration_is_blocked(self, mock_create_e integration_b.refresh_from_db() assert integration_b.config.get("verified") is False - def test_aws_account_id_is_cached_per_provider(self): - provider = self._build_ses_provider() - provider.sts_client.get_caller_identity.reset_mock() + def test_aws_account_id_is_looked_up_once_across_providers(self): + cache.delete(AWS_ACCOUNT_ID_CACHE_KEY) + providers = [self._build_ses_provider(), self._build_ses_provider()] + for provider in providers: + provider.sts_client.get_caller_identity.reset_mock() - for _ in range(5): + for provider in providers: provider._identity_arn("partner.com") provider._identity_arn("other.com") - assert provider.sts_client.get_caller_identity.call_count == 1 + assert sum(provider.sts_client.get_caller_identity.call_count for provider in providers) == 1 diff --git a/products/workflows/backend/test/test_ses_provider.py b/products/workflows/backend/test/test_ses_provider.py index af87522ef152..2d0308cc4834 100644 --- a/products/workflows/backend/test/test_ses_provider.py +++ b/products/workflows/backend/test/test_ses_provider.py @@ -123,6 +123,7 @@ def fail_config_set_associations(TenantName: str, ResourceArn: str) -> dict: @patch("products.workflows.backend.providers.ses.boto3.client") def test_create_email_domain_invalid_domain(self, mock_boto_client): + mock_boto_client.return_value.get_caller_identity.return_value = {"Account": "123456789012"} with override_settings( SES_ACCESS_KEY_ID="test_access_key", SES_SECRET_ACCESS_KEY="test_secret_key", SES_REGION="us-east-1" ): diff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.json index 4165130e0ed1..5b11e0345e40 100644 --- a/services/mcp/schema/generated-tool-definitions.json +++ b/services/mcp/schema/generated-tool-definitions.json @@ -7296,6 +7296,21 @@ "readOnlyHint": true } }, + "integrations-email-status-retrieve": { + "description": "Get the setup progress of an email sender's domain by integration ID. Returns the overall status, the three setup steps, and every DNS record to publish with its current status (pending, found, verified). Poll it after adding DNS records until verified is true, waiting at least 10 seconds between calls. Pass refresh=true right after changing DNS to skip the 10 second cache. It does not change DNS or SES.", + "category": "Integrations", + "feature": "integrations", + "summary": "Get email sending domain status", + "title": "Get email sending domain status", + "required_scopes": ["integration:read"], + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": true, + "readOnlyHint": true + }, + "feature_flag": "workflows-email-domain-agent-setup" + }, "integrations-github-repos-retrieve": { "description": "List the repositories accessible to a GitHub integration (by integration ID). Serves the cached repository list from the GitHub App installation. Supports `search` (case-insensitive name match), `limit` (default 100, max 500), and `offset` query parameters. Returns repository id, name, and full_name (owner/repo), a has_more flag, a total count, and next_offset. When has_more is true, call again with `offset` set to next_offset to get the next page. Set `compact` to false only when you need the extra repository details. Use this to validate that a repository is accessible to the integration before referencing it (e.g. when creating a GitHub data warehouse source).", "category": "Integrations", diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.json index f1e5dc287c31..0a474902e5c5 100644 --- a/services/mcp/schema/tool-definitions-all.json +++ b/services/mcp/schema/tool-definitions-all.json @@ -7507,6 +7507,21 @@ "readOnlyHint": true } }, + "integrations-email-status-retrieve": { + "description": "Get the setup progress of an email sender's domain by integration ID. Returns the overall status, the three setup steps, and every DNS record to publish with its current status (pending, found, verified). Poll it after adding DNS records until verified is true, waiting at least 10 seconds between calls. Pass refresh=true right after changing DNS to skip the 10 second cache. It does not change DNS or SES.", + "category": "Integrations", + "feature": "integrations", + "summary": "Get email sending domain status", + "title": "Get email sending domain status", + "required_scopes": ["integration:read"], + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": true, + "readOnlyHint": true + }, + "feature_flag": "workflows-email-domain-agent-setup" + }, "integrations-github-repos-retrieve": { "description": "List the repositories accessible to a GitHub integration (by integration ID). Serves the cached repository list from the GitHub App installation. Supports `search` (case-insensitive name match), `limit` (default 100, max 500), and `offset` query parameters. Returns repository id, name, and full_name (owner/repo), a has_more flag, a total count, and next_offset. When has_more is true, call again with `offset` set to next_offset to get the next page. Set `compact` to false only when you need the extra repository details. Use this to validate that a repository is accessible to the integration before referencing it (e.g. when creating a GitHub data warehouse source).", "category": "Integrations", diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts index 8a53e4d4e41f..f2ed4c5693b7 100644 --- a/services/mcp/src/api/generated.ts +++ b/services/mcp/src/api/generated.ts @@ -37775,6 +37775,170 @@ export namespace Schemas { name: string; } + /** + * * `verification` - Domain ownership + * * `dkim` - DKIM signing + * * `spf` - SPF + * * `mail_from_mx` - MAIL FROM MX + * * `mail_from_spf` - MAIL FROM SPF + * * `dmarc` - DMARC + */ + export type EmailDomainSetupRecordKindEnum = typeof EmailDomainSetupRecordKindEnum[keyof typeof EmailDomainSetupRecordKindEnum]; + + + export const EmailDomainSetupRecordKindEnum = { + Verification: 'verification', + Dkim: 'dkim', + Spf: 'spf', + MailFromMx: 'mail_from_mx', + MailFromSpf: 'mail_from_spf', + Dmarc: 'dmarc', + } as const; + + /** + * * `pending` - Not found yet + * * `found` - Found in DNS + * * `verified` - Verified + */ + export type EmailDomainSetupRecordStatusEnum = typeof EmailDomainSetupRecordStatusEnum[keyof typeof EmailDomainSetupRecordStatusEnum]; + + + export const EmailDomainSetupRecordStatusEnum = { + Pending: 'pending', + Found: 'found', + Verified: 'verified', + } as const; + + /** + * * `TXT` - TXT + * * `CNAME` - CNAME + * * `MX` - MX + */ + export type EmailDomainSetupRecordTypeEnum = typeof EmailDomainSetupRecordTypeEnum[keyof typeof EmailDomainSetupRecordTypeEnum]; + + + export const EmailDomainSetupRecordTypeEnum = { + Txt: 'TXT', + Cname: 'CNAME', + Mx: 'MX', + } as const; + + /** + * * `not_started` - Not started + * * `pending` - Waiting for DNS records + * * `records_found` - Records found, waiting for verification + * * `verified` - Verified + * * `temporary_failure` - Temporary failure + * * `failed` - Failed + */ + export type EmailDomainSetupStatusEnum = typeof EmailDomainSetupStatusEnum[keyof typeof EmailDomainSetupStatusEnum]; + + + export const EmailDomainSetupStatusEnum = { + NotStarted: 'not_started', + Pending: 'pending', + RecordsFound: 'records_found', + Verified: 'verified', + TemporaryFailure: 'temporary_failure', + Failed: 'failed', + } as const; + + /** + * * `domain_added` - Domain added + * * `records_found` - DNS records found + * * `verified` - Verified + */ + export type EmailDomainSetupStepKeyEnum = typeof EmailDomainSetupStepKeyEnum[keyof typeof EmailDomainSetupStepKeyEnum]; + + + export const EmailDomainSetupStepKeyEnum = { + DomainAdded: 'domain_added', + RecordsFound: 'records_found', + Verified: 'verified', + } as const; + + /** + * * `done` - Done + * * `pending` - Pending + * * `failed` - Failed + */ + export type EmailDomainSetupStepStateEnum = typeof EmailDomainSetupStepStateEnum[keyof typeof EmailDomainSetupStepStateEnum]; + + + export const EmailDomainSetupStepStateEnum = { + Done: 'done', + Pending: 'pending', + Failed: 'failed', + } as const; + + export interface EmailDomainStatusStep { + /** The setup step: domain_added (SES knows the domain), records_found (every DNS record is published), verified (SES verified the domain and the sender can send). + * + * * `domain_added` - Domain added + * * `records_found` - DNS records found + * * `verified` - Verified */ + key: EmailDomainSetupStepKeyEnum; + /** Progress of this step. + * + * * `done` - Done + * * `pending` - Pending + * * `failed` - Failed */ + state: EmailDomainSetupStepStateEnum; + } + + export interface EmailDomainStatusRecord { + /** What the record does: verification proves domain ownership, dkim signs email, spf and mail_from_spf list SES as an allowed sender, mail_from_mx routes bounces, dmarc sets the policy for unauthenticated mail. + * + * * `verification` - Domain ownership + * * `dkim` - DKIM signing + * * `spf` - SPF + * * `mail_from_mx` - MAIL FROM MX + * * `mail_from_spf` - MAIL FROM SPF + * * `dmarc` - DMARC */ + kind: EmailDomainSetupRecordKindEnum; + /** Full DNS name to create the record at, including the domain. */ + hostname: string; + /** DNS record type. + * + * * `TXT` - TXT + * * `CNAME` - CNAME + * * `MX` - MX */ + type: EmailDomainSetupRecordTypeEnum; + /** Value to publish. For an SPF record, add the include to an existing SPF record instead of creating a second one. For DMARC, any existing v=DMARC1 record counts. */ + value: string; + /** + * MX priority. Null for other record types. + * @nullable + */ + priority: number | null; + /** pending: not found in DNS yet. found: published in DNS, SES has not confirmed it yet. verified: SES confirmed the record, or for DMARC a v=DMARC1 record exists. + * + * * `pending` - Not found yet + * * `found` - Found in DNS + * * `verified` - Verified */ + status: EmailDomainSetupRecordStatusEnum; + } + + export interface EmailDomainStatus { + /** Overall setup status. records_found means every record is published and SES has not confirmed yet. temporary_failure means SES could not see the records, usually because DNS changes have not spread yet. failed means SES gave up after 72 hours; start verification again. + * + * * `not_started` - Not started + * * `pending` - Waiting for DNS records + * * `records_found` - Records found, waiting for verification + * * `verified` - Verified + * * `temporary_failure` - Temporary failure + * * `failed` - Failed */ + status: EmailDomainSetupStatusEnum; + /** Whether the sender can send email. Set once SES verifies the domain. */ + verified: boolean; + /** When SES and DNS were read. Responses are cached for 10 seconds unless refresh is set. */ + checked_at: string; + /** The three setup steps, in order. */ + steps: EmailDomainStatusStep[]; + /** DNS records to publish at the domain's DNS host, with the status of each. Empty when status is not_started. */ + records: EmailDomainStatusRecord[]; + } + /** * How much workflow email this project may send, and how much of that it has used. */ @@ -121944,6 +122108,13 @@ export namespace Schemas { search?: string; }; + export type IntegrationsEmailStatusRetrieveParams = { + /** + * Skip the 10 second status cache and read SES and DNS again. Use it after changing DNS records. + */ + refresh?: boolean; + }; + export type IntegrationsGithubAssigneesRetrieveParams = { /** * Repository name, or owner/name, whose assignable users to list. diff --git a/services/mcp/src/generated/integrations/api.ts b/services/mcp/src/generated/integrations/api.ts index 89932c5bc6dc..7dd020058b69 100644 --- a/services/mcp/src/generated/integrations/api.ts +++ b/services/mcp/src/generated/integrations/api.ts @@ -3,7 +3,7 @@ * MCP service uses these Zod schemas for generated tool handlers. * To regenerate: hogli build:openapi * - * PostHog API - MCP 9 enabled ops + * PostHog API - MCP 10 enabled ops * OpenAPI spec version: 1.0.0 */ import * as zod from 'zod' @@ -145,6 +145,28 @@ export const IntegrationsChannelsRetrieveQueryParams = () => zod.object({ .describe('Optional case-insensitive channel name or ID search query.'), }) +/** + * Read the setup progress of an email sender's domain from SES and public DNS without changing either. Poll this until verified is true. Marks the domain's senders as verified once SES verifies the domain. + * @summary Get email sending domain status + */ +export const IntegrationsEmailStatusRetrieveParams = () => zod.object({ + id: zod.number().describe('A unique integer value identifying this integration.'), + project_id: zod + .string() + .describe( + "Project ID of the project you're trying to access. To find the ID of the project, make a call to \/api\/projects\/." + ), +}) + +export const integrationsEmailStatusRetrieveQueryRefreshDefault = false + +export const IntegrationsEmailStatusRetrieveQueryParams = () => zod.object({ + refresh: zod + .boolean() + .default(integrationsEmailStatusRetrieveQueryRefreshDefault) + .describe('Skip the 10 second status cache and read SES and DNS again. Use it after changing DNS records.'), +}) + export const IntegrationsGithubReposRetrieveParams = () => zod.object({ id: zod.number().describe('A unique integer value identifying this integration.'), project_id: zod diff --git a/services/mcp/src/tools/generated/integrations.ts b/services/mcp/src/tools/generated/integrations.ts index eaeb379c6f1d..2076e3062d22 100644 --- a/services/mcp/src/tools/generated/integrations.ts +++ b/services/mcp/src/tools/generated/integrations.ts @@ -81,6 +81,33 @@ const integrationsChannelsRetrieve = (): ToolBase< }, }) +const IntegrationsEmailStatusRetrieveSchema = () => { + const IntegrationsEmailStatusRetrieveParams = orvalSchemas.IntegrationsEmailStatusRetrieveParams() + const IntegrationsEmailStatusRetrieveQueryParams = orvalSchemas.IntegrationsEmailStatusRetrieveQueryParams() + return IntegrationsEmailStatusRetrieveParams.omit({ project_id: true }).extend( + IntegrationsEmailStatusRetrieveQueryParams.shape + ) +} + +const integrationsEmailStatusRetrieve = (): ToolBase< + ReturnType, + Schemas.EmailDomainStatus +> => ({ + name: 'integrations-email-status-retrieve', + schema: IntegrationsEmailStatusRetrieveSchema(), + handler: async (context: Context, params: z.infer>) => { + const projectId = await context.stateManager.getProjectId() + const result = await context.api.request({ + method: 'GET', + path: `/api/projects/${encodeURIComponent(String(projectId))}/integrations/${encodeURIComponent(String(params.id))}/email/status/`, + query: { + refresh: params.refresh, + }, + }) + return result + }, +}) + const IntegrationsGithubReposRetrieveSchema = () => { const IntegrationsGithubReposRetrieveParams = orvalSchemas.IntegrationsGithubReposRetrieveParams() const IntegrationsGithubReposRetrieveQueryParams = orvalSchemas.IntegrationsGithubReposRetrieveQueryParams() @@ -256,6 +283,7 @@ export const GENERATED_TOOLS: Record ToolBase> = { 'integration-delete': integrationDelete, 'integration-get': integrationGet, 'integrations-channels-retrieve': integrationsChannelsRetrieve, + 'integrations-email-status-retrieve': integrationsEmailStatusRetrieve, 'integrations-github-repos-retrieve': integrationsGithubReposRetrieve, 'integrations-jira-projects-retrieve': integrationsJiraProjectsRetrieve, 'integrations-linear-teams-retrieve': integrationsLinearTeamsRetrieve, From 0f4c648d9376f55c630749b692efcfd4eb3ebee4 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 7 Oct 2026 08:26:21 +0000 Subject: [PATCH 03/13] fix(workflows): gate new email domain status capability --- posthog/api/integration.py | 17 ++++++++++++++++- .../backend/test/email_domain_api_test_case.py | 6 ++++++ .../test/test_email_domain_status_api.py | 17 ++++++++++++++--- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/posthog/api/integration.py b/posthog/api/integration.py index 6a920481c21b..c0091b100802 100644 --- a/posthog/api/integration.py +++ b/posthog/api/integration.py @@ -24,7 +24,7 @@ from prometheus_client import Counter from redis.exceptions import RedisError from rest_framework import mixins, serializers, status, viewsets -from rest_framework.exceptions import APIException, PermissionDenied, Throttled, ValidationError +from rest_framework.exceptions import APIException, NotFound, PermissionDenied, Throttled, ValidationError from rest_framework.permissions import BasePermission, IsAuthenticated from rest_framework.request import Request from rest_framework.response import Response @@ -123,6 +123,7 @@ TeamMemberStrictManagementPermission, TimeSensitiveActionPermission, ) +from posthog.ph_client import feature_enabled_or_false from posthog.rate_limit import GitHubRepositoryRefreshThrottle from posthog.tasks.email import send_integration_access_request from posthog.utils import absolute_uri, is_relative_url @@ -2670,6 +2671,20 @@ def email_verify(self, request, **kwargs): ) @action(methods=["GET"], detail=True, url_path="email/status") def email_status(self, request: ValidatedRequest, **kwargs: Any) -> Response: + if not any( + feature_enabled_or_false( + flag, + str(cast(User, request.user).distinct_id), + groups={"organization": str(self.team.organization_id), "project": str(self.team.uuid)}, + group_properties={ + "organization": {"id": str(self.team.organization_id)}, + "project": {"id": str(self.team.uuid)}, + }, + send_feature_flag_events=False, + ) + for flag in ("workflows-email-domain-wizard", "workflows-email-domain-agent-setup") + ): + raise NotFound() instance = self.get_object() if instance.kind != "email": raise ValidationError("email/status is only supported for email integrations") diff --git a/products/workflows/backend/test/email_domain_api_test_case.py b/products/workflows/backend/test/email_domain_api_test_case.py index d6a8437a3e0c..4711ead5592f 100644 --- a/products/workflows/backend/test/email_domain_api_test_case.py +++ b/products/workflows/backend/test/email_domain_api_test_case.py @@ -35,6 +35,12 @@ def setUp(self) -> None: patch("posthog.models.integration.email.reload_integrations_on_workers") ) self.report_user_action = self.enterContext(patch("posthog.api.integration.report_user_action")) + self.feature_flags = self.enterContext( + patch( + "posthoganalytics.feature_enabled", + side_effect=lambda key, *args, **kwargs: key == "workflows-email-domain-wizard", + ) + ) def create_sender(self, email: str, *, team: Team | None = None) -> int: response = self.client.post( diff --git a/products/workflows/backend/test/test_email_domain_status_api.py b/products/workflows/backend/test/test_email_domain_status_api.py index 4fb62213172e..dee807d798c5 100644 --- a/products/workflows/backend/test/test_email_domain_status_api.py +++ b/products/workflows/backend/test/test_email_domain_status_api.py @@ -84,12 +84,23 @@ def test_senders_become_verified_as_the_records_are_published_and_the_event_fire assert self.is_verified(late_sender) assert len(self.verified_events()) == 1 - def test_status_reads_ses_without_writing_to_it(self) -> None: + @parameterized.expand( + [(False, False, 404), (True, False, 200), (False, True, 200), (True, True, 200), (None, None, 404)] + ) + def test_status_reads_ses_without_writing_to_it( + self, wizard_enabled: bool | None, agent_enabled: bool | None, expected_status: int + ) -> None: sender = self.create_sender(f"hello@{DOMAIN}") self.publish(self.records_to_publish(sender), *ALL_KINDS) writes_after_create = list(self.world.ses.writes) - - assert self.status_body(sender)["status"] == "verified" + self.feature_flags.side_effect = lambda key, *args, **kwargs: { + "workflows-email-domain-wizard": wizard_enabled, + "workflows-email-domain-agent-setup": agent_enabled, + }.get(key, False) + response = self.get_status(sender) + assert response.status_code == expected_status, response.json() + if expected_status == 200: + assert response.json()["status"] == "verified" assert self.world.ses.writes == writes_after_create From 8e7a51b1cdaa8dea1e6cf7c5ab31479677582800 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Fri, 2 Oct 2026 04:57:31 +0000 Subject: [PATCH 04/13] feat(workflows): detect the dns host and existing email tools for one-click email setup domain-connect/check now returns the DNS host detected from the root domain's nameservers, with a link to its DNS settings, and the email tools that already send for the domain, detected from SPF and DKIM records. Its response gets a typed schema. The Domain Connect apply-url call records setup_method auto on the email integration and reports the "email domain verified" event when the domain verifies during that call. The sender update records setup_method manual and skips the SES MAIL FROM write when the bounce subdomain is unchanged. Lookups that got no answer are retried instead of cached: - A DNS timeout or SERVFAIL on the _domainconnect record, and a timeout, rate limit, 5xx or malformed answer from the provider settings, no longer hide one-click setup for an hour. Only a definitive answer is cached. - Existing email tool detection no longer caches an empty list for an hour after a lookup got no answer. Scenarios prove one-click email setup end to end: the check endpoint finds the provider and DNS host, the apply URL is signed, the fake provider renders our real template, publishes its records, and status polls reach verified with the auto setup method. They cover a person returning without approval, the sender update, and the provider failures. They replace the tests that patched generate_apply_url, the verify call and the private discovery helpers. The case for an existing DMARC record is skipped until Co-Authored-By: Claude Opus 5.5 (1M context) --- posthog/api/integration.py | 29 +- .../integration_email_status_serializers.py | 31 ++ posthog/dns_hosts.py | 82 +++++ posthog/domain_connect.py | 71 +++- posthog/models/integration/email.py | 29 +- .../database_free_test_classes_baseline.txt | 1 - posthog/test/test_dns_hosts.py | 28 ++ posthog/test/test_domain_connect.py | 216 ------------ .../frontend/generated/api.schemas.ts | 30 ++ .../integrations/frontend/generated/api.ts | 5 +- products/workflows/backend/facade/api.py | 7 + .../backend/services/existing_email_tools.py | 91 +++++ .../test/test_email_domain_connect_api.py | 332 ++++++++++++++++++ .../backend/test/test_existing_email_tools.py | 38 ++ services/mcp/src/api/generated.ts | 30 ++ 15 files changed, 769 insertions(+), 251 deletions(-) create mode 100644 posthog/dns_hosts.py create mode 100644 posthog/test/test_dns_hosts.py create mode 100644 products/workflows/backend/services/existing_email_tools.py create mode 100644 products/workflows/backend/test/test_email_domain_connect_api.py create mode 100644 products/workflows/backend/test/test_existing_email_tools.py diff --git a/posthog/api/integration.py b/posthog/api/integration.py index c0091b100802..abf93c436d04 100644 --- a/posthog/api/integration.py +++ b/posthog/api/integration.py @@ -46,6 +46,8 @@ is_valid_github_installation_id, ) from posthog.api.integration_email_status_serializers import ( + EmailDomainConnectCheckSerializer, + EmailDomainDnsHostSerializer, EmailDomainStatusQuerySerializer, EmailDomainStatusSerializer, ) @@ -54,6 +56,7 @@ from posthog.api.shared import UserBasicSerializer from posthog.api.utils import action from posthog.auth import OAuthAccessTokenAuthentication, PersonalAPIKeyAuthentication, SessionAuthentication +from posthog.dns_hosts import lookup_dns_host from posthog.domain_connect import discover_domain_connect, extract_root_domain_and_host, get_available_providers from posthog.egress.github.transport import GitHubRateLimitError from posthog.event_usage import report_user_action @@ -134,7 +137,7 @@ from products.slack_app.backend.services.slack_auth import SLACK_AUTH_FAILURE_CODES from products.tasks.backend.facade.api import get_in_progress_runs_for_github_integration from products.tasks.backend.facade.contracts import InProgressGithubRunsDTO -from products.workflows.backend.facade.api import get_active_workflows_using_integration +from products.workflows.backend.facade.api import get_active_workflows_using_integration, lookup_existing_email_tools from products.workflows.backend.facade.contracts import EmailProviderUnavailableError logger = structlog.get_logger(__name__) @@ -322,6 +325,12 @@ class NativeEmailIntegrationSerializer(serializers.Serializer): name = serializers.CharField() provider = serializers.ChoiceField(choices=["ses", "maildev"] if settings.DEBUG else ["ses"]) mail_from_subdomain = serializers.CharField(required=False, allow_blank=True) + setup_method = serializers.ChoiceField( + choices=["auto", "manual"], + required=False, + help_text="How the DNS records are being published: `auto` through Domain Connect, `manual` by hand. " + "Reported on the verification event.", + ) def validate_email(self, value: str) -> str: return value.lower() @@ -2729,6 +2738,7 @@ def email_update(self, request, **kwargs) -> Response: return Response(IntegrationSerializer(email.integration).data) # nosemgrep: api-path-underscore -- shipped public API path, a rename breaks clients + @extend_schema(responses={200: EmailDomainConnectCheckSerializer}) @action(methods=["GET"], detail=False, url_path="domain-connect/check") def domain_connect_check(self, request: Request, *args: Any, **kwargs: Any) -> Response: domain = request.query_params.get("domain", "") @@ -2738,11 +2748,14 @@ def domain_connect_check(self, request: Request, *args: Any, **kwargs: Any) -> R # Extract root domain so subdomains (e.g. ph.example.com) resolve correctly domain_parts = extract_root_domain_and_host(domain) result = discover_domain_connect(domain_parts.root_domain) + dns_host = lookup_dns_host(domain_parts.root_domain) return Response( { "supported": result is not None, "provider_name": result["provider_name"] if result else None, "available_providers": get_available_providers() if result is None else [], + "dns_host": EmailDomainDnsHostSerializer(dns_host).data if dns_host else None, + "existing_email_tools": lookup_existing_email_tools(domain_parts.root_domain), } ) @@ -2775,7 +2788,11 @@ def domain_connect_apply_url(self, request: Request, *args: Any, **kwargs: Any) if not integration_id: raise ValidationError("integration_id is required for email context") try: - resolved = resolve_email_context(integration_id, self.team_id) + resolved = resolve_email_context( + integration_id, + self.team_id, + on_domain_verified=lambda instance: self._report_email_domain_verified(request, instance), + ) except ValueError as e: capture_exception(e, {"integration_id": integration_id, "team_id": self.team_id, "context": context}) raise ValidationError( @@ -2831,8 +2848,16 @@ def domain_connect_apply_url(self, request: Request, *args: Any, **kwargs: Any) ) raise ValidationError("Error generating apply URL. Please try again later or contact support.") + if context == "email" and integration_id: + self._record_email_setup_method(integration_id, "auto") return Response({"url": url}) + def _record_email_setup_method(self, integration_id: int, setup_method: str) -> None: + with transaction.atomic(): + instance = Integration.objects.select_for_update().get(id=integration_id, team_id=self.team_id) + instance.config["setup_method"] = setup_method + instance.save(update_fields=["config"]) + # Defined last: a method named `list` shadows the builtin for the annotations of every method # declared after it in this class body. def list(self, request: Request, *args: Any, **kwargs: Any) -> Response: diff --git a/posthog/api/integration_email_status_serializers.py b/posthog/api/integration_email_status_serializers.py index 3844191f2793..9e556dd3ae4d 100644 --- a/posthog/api/integration_email_status_serializers.py +++ b/posthog/api/integration_email_status_serializers.py @@ -70,3 +70,34 @@ class EmailDomainStatusSerializer(serializers.Serializer): help_text="DNS records to publish at the domain's DNS host, with the status of each. Empty when " "status is not_started.", ) + + +class EmailDomainDnsHostSerializer(serializers.Serializer): + name = serializers.CharField(help_text="Name of the company that hosts the domain's DNS, such as Cloudflare.") + dns_settings_url = serializers.URLField(help_text="Link to the DNS settings page at that host.") + + +class EmailDomainConnectProviderSerializer(serializers.Serializer): + endpoint = serializers.CharField(help_text="Domain Connect endpoint of the provider.") + name = serializers.CharField(help_text="Display name of the provider.") + + +class EmailDomainConnectCheckSerializer(serializers.Serializer): + supported = serializers.BooleanField( + help_text="Whether the domain's DNS provider supports automatic setup through Domain Connect." + ) + provider_name = serializers.CharField( + allow_null=True, help_text="Domain Connect provider that hosts the domain. Null when not supported." + ) + available_providers = EmailDomainConnectProviderSerializer( + many=True, + help_text="Providers the user can pick manually when automatic detection fails. Empty when supported.", + ) + dns_host = EmailDomainDnsHostSerializer( + allow_null=True, help_text="The detected DNS host of the root domain, from its nameservers. Null when unknown." + ) + existing_email_tools = serializers.ListField( + child=serializers.CharField(), + help_text="Email tools that already send for the root domain, detected from SPF and DKIM records. Best " + "effort, empty when none are found.", + ) diff --git a/posthog/dns_hosts.py b/posthog/dns_hosts.py new file mode 100644 index 000000000000..4df81799e29f --- /dev/null +++ b/posthog/dns_hosts.py @@ -0,0 +1,82 @@ +import logging + +from django.core.cache import cache + +import dns.resolver +import dns.exception +from dns.rdtypes.ANY.NS import NS + +from posthog.dataclasses import frozen + +logger = logging.getLogger(__name__) + +LOOKUP_TIMEOUT_SECONDS = 3 +CACHE_SECONDS = 60 * 60 + + +@frozen +class DnsHost: + name: str + dns_settings_url: str + + +ROUTE_53 = DnsHost(name="Route 53", dns_settings_url="https://console.aws.amazon.com/route53/v2/hostedzones") +GODADDY = DnsHost(name="GoDaddy", dns_settings_url="https://dcc.godaddy.com/control/portfolio") +NAMECHEAP = DnsHost(name="Namecheap", dns_settings_url="https://ap.www.namecheap.com/domains/list/") +SQUARESPACE = DnsHost(name="Squarespace", dns_settings_url="https://account.squarespace.com/domains") +HOSTINGER = DnsHost(name="Hostinger", dns_settings_url="https://hpanel.hostinger.com/domains") +IONOS = DnsHost(name="IONOS", dns_settings_url="https://my.ionos.com/domains") +DIGITALOCEAN = DnsHost(name="DigitalOcean", dns_settings_url="https://cloud.digitalocean.com/networking/domains") +CLOUDFLARE = DnsHost(name="Cloudflare", dns_settings_url="https://dash.cloudflare.com/?to=/:account/:zone/dns/records") +VERCEL = DnsHost(name="Vercel", dns_settings_url="https://vercel.com/dashboard/domains") + +# Each marker is matched against "." + the nameserver hostname, so a leading dot anchors it to a +# label boundary. Route 53 nameservers look like ns-1234.awsdns-56.org, so its marker has no dot. +_DNS_HOSTS_BY_NAMESERVER_MARKER: tuple[tuple[str, DnsHost], ...] = ( + (".awsdns-", ROUTE_53), + (".domaincontrol.com", GODADDY), + (".registrar-servers.com", NAMECHEAP), + (".squarespacedns.com", SQUARESPACE), + (".dns-parking.com", HOSTINGER), + (".hostinger.", HOSTINGER), + (".ui-dns.", IONOS), + (".digitalocean.com", DIGITALOCEAN), + (".ns.cloudflare.com", CLOUDFLARE), + (".vercel-dns.com", VERCEL), +) + + +def detect_dns_host(nameservers: list[str]) -> DnsHost | None: + for nameserver in nameservers: + anchored = "." + nameserver.lower().rstrip(".") + for marker, host in _DNS_HOSTS_BY_NAMESERVER_MARKER: + if marker in anchored: + return host + return None + + +def lookup_dns_host(root_domain: str) -> DnsHost | None: + cache_key = f"dns_host:{root_domain}" + cached = cache.get(cache_key) + if cached is not None: + return cached or None + + nameservers = _nameservers(root_domain) + if nameservers is None: + return None + host = detect_dns_host(nameservers) + cache.set(cache_key, host or False, CACHE_SECONDS) + return host + + +def _nameservers(root_domain: str) -> list[str] | None: + resolver = dns.resolver.Resolver() + resolver.lifetime = LOOKUP_TIMEOUT_SECONDS + try: + answers = resolver.resolve(root_domain, "NS") + except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): + return [] + except dns.exception.DNSException: + logger.info("Could not resolve nameservers for %s", root_domain, exc_info=True) + return None + return [rdata.target.to_text() for rdata in answers if isinstance(rdata, NS)] diff --git a/posthog/domain_connect.py b/posthog/domain_connect.py index 5a18c7390b17..1f04b87ce59f 100644 --- a/posthog/domain_connect.py +++ b/posthog/domain_connect.py @@ -9,6 +9,8 @@ import base64 import logging +from collections.abc import Callable +from typing import TYPE_CHECKING from urllib.parse import urlencode from django.conf import settings @@ -17,6 +19,7 @@ import requests import tldextract import dns.resolver +import dns.exception from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import padding from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey @@ -26,6 +29,9 @@ from posthog.dataclasses import frozen +if TYPE_CHECKING: + from posthog.models.integration import Integration + logger = logging.getLogger(__name__) # Maps _domainconnect TXT record values (provider endpoints) to display names. @@ -56,6 +62,13 @@ class DomainConnectSigningKeyMissing(Exception): pass +TRANSIENT_HTTP_STATUSES = frozenset({408, 429}) + + +class _ProviderUnreachable(Exception): + """The discovery record or the provider settings could not be read this time, so the answer is unknown.""" + + def discover_domain_connect(domain: str) -> dict | None: """Discover whether a domain's DNS provider supports Domain Connect. @@ -72,14 +85,16 @@ def discover_domain_connect(domain: str) -> dict | None: # We cache both hits (dict) and misses (False) to avoid repeated lookups return cached if cached is not False else None - endpoint = _lookup_domain_connect_endpoint(domain) - if not endpoint or endpoint not in DOMAIN_CONNECT_PROVIDERS: - cache.set(cache_key, False, 60 * 60) + try: + endpoint = _lookup_domain_connect_endpoint(domain) + if not endpoint or endpoint not in DOMAIN_CONNECT_PROVIDERS: + cache.set(cache_key, False, 60 * 60) + return None + provider_settings = _fetch_provider_settings(endpoint, domain) + except _ProviderUnreachable: return None provider_name = DOMAIN_CONNECT_PROVIDERS[endpoint] - - provider_settings = _fetch_provider_settings(endpoint, domain) if not provider_settings: cache.set(cache_key, False, 60 * 60) return None @@ -234,7 +249,12 @@ class DomainConnectContext: EMAIL_TEMPLATE_GROUPS_WITHOUT_DMARC: tuple[str, ...] = ("verification", "dkim", "spf", "mailfrom") -def resolve_email_context(integration_id: int, team_id: int) -> DomainConnectContext: +def resolve_email_context( + integration_id: int, + team_id: int, + *, + on_domain_verified: Callable[["Integration"], None] | None = None, +) -> DomainConnectContext: """Resolve Domain Connect parameters for an email integration. Triggers SES verification to get current tokens, then extracts the @@ -247,7 +267,9 @@ def resolve_email_context(integration_id: int, team_id: int) -> DomainConnectCon raise ValueError("Integration must be of kind 'email'") email_integration = EmailIntegration(instance) - verification_result = email_integration.verify() + verification_result = email_integration.verify( + on_domain_verified=(lambda: on_domain_verified(instance)) if on_domain_verified else None + ) dns_records = verification_result.get("dnsRecords", []) domain_parts = extract_root_domain_and_host(instance.config.get("domain", "")) @@ -351,7 +373,10 @@ def generate_apply_url( ) # 3. Fetch provider settings to get urlSyncUX (cached by _fetch_provider_settings) - provider_settings = _fetch_provider_settings(endpoint, domain) + try: + provider_settings = _fetch_provider_settings(endpoint, domain) + except _ProviderUnreachable as error: + raise ValueError(f"Could not reach provider: {endpoint}") from error if not provider_settings: raise ValueError(f"Could not fetch settings from provider: {endpoint}") @@ -395,8 +420,10 @@ def _lookup_domain_connect_endpoint(domain: str) -> str | None: txt_value = _txt_value(rdata) if txt_value: return txt_value - except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.NoNameservers, dns.resolver.Timeout): + except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): pass + except dns.exception.DNSException as error: + raise _ProviderUnreachable(f"No DNS answer for _domainconnect.{domain}") from error except Exception: logger.exception("Unexpected error during Domain Connect DNS lookup for %s", domain) return None @@ -416,14 +443,24 @@ def _fetch_provider_settings(endpoint: str, domain: str) -> dict | None: url = f"https://{endpoint}/v2/{domain}/settings" try: resp = requests.get(url, timeout=10) - resp.raise_for_status() - data = resp.json() - if "urlSyncUX" not in data: - cache.set(cache_key, False, 60 * 60) - return None - cache.set(cache_key, data, 60 * 60) - return data - except (requests.RequestException, ValueError): + except requests.RequestException as error: logger.warning("Failed to fetch Domain Connect settings from %s", url) + raise _ProviderUnreachable(f"Could not reach {url}") from error + # A 4xx means the provider does not serve this domain. A timeout, a rate limit, a 5xx or a garbled + # body says nothing about it. + if resp.status_code in TRANSIENT_HTTP_STATUSES or resp.status_code >= 500: + logger.warning("Domain Connect settings at %s answered %s", url, resp.status_code) + raise _ProviderUnreachable(f"{url} answered {resp.status_code}") + if not resp.ok: + cache.set(cache_key, False, 60 * 60) + return None + try: + data = resp.json() + except ValueError as error: + logger.warning("Domain Connect settings at %s are not JSON", url) + raise _ProviderUnreachable(f"{url} answered malformed JSON") from error + if not isinstance(data, dict) or "urlSyncUX" not in data: cache.set(cache_key, False, 60 * 60) return None + cache.set(cache_key, data, 60 * 60) + return data diff --git a/posthog/models/integration/email.py b/posthog/models/integration/email.py index 955e8ae3c4cc..21e86038645a 100644 --- a/posthog/models/integration/email.py +++ b/posthog/models/integration/email.py @@ -109,31 +109,34 @@ def create_native_integration( def update_native_integration(self, config: dict, team_id: int) -> model.Integration: provider = self.integration.config.get("provider") domain = self.integration.config.get("domain") - # Only name and mail_from_subdomain can be updated + # Only name, mail_from_subdomain and setup_method can be updated name: str = config.get("name", self.integration.config.get("name")) mail_from_subdomain: str = config.get( "mail_from_subdomain", self.integration.config.get("mail_from_subdomain", "feedback") ) - # Update domain in the appropriate provider + mail_from_changed = mail_from_subdomain != self.integration.config.get("mail_from_subdomain", "feedback") + if provider == "ses": - from products.workflows.backend.facade.api import ( - update_ses_mail_from_subdomain, # noqa: PLC0415 — keeps the workflows facade off the model import path - ) + if mail_from_changed: + from products.workflows.backend.facade.api import ( + update_ses_mail_from_subdomain, # noqa: PLC0415 — keeps the workflows facade off the model import path + ) - update_ses_mail_from_subdomain(domain, mail_from_subdomain=mail_from_subdomain) + update_ses_mail_from_subdomain(domain, mail_from_subdomain=mail_from_subdomain) elif provider == "maildev" and settings.DEBUG: pass else: raise ValueError(f"Invalid provider: must be 'ses'") - self.integration.config.update( - { - "name": name, - "mail_from_subdomain": mail_from_subdomain, - } - ) - self.integration.save() + changes: dict[str, str] = {"name": name, "mail_from_subdomain": mail_from_subdomain} + if setup_method := config.get("setup_method"): + changes["setup_method"] = setup_method + # The row is reread under a lock so a concurrent verification write is not overwritten. + with transaction.atomic(): + self.integration = model.Integration.objects.select_for_update().get(id=self.integration.id) + self.integration.config.update(changes) + self.integration.save(update_fields=["config"]) return self.integration diff --git a/posthog/test/repo_invariants/database_free_test_classes_baseline.txt b/posthog/test/repo_invariants/database_free_test_classes_baseline.txt index 9a56bbcc4d84..9e086162a8e1 100644 --- a/posthog/test/repo_invariants/database_free_test_classes_baseline.txt +++ b/posthog/test/repo_invariants/database_free_test_classes_baseline.txt @@ -204,7 +204,6 @@ posthog/test/test_cache_utils.py::TestCacheUtils posthog/test/test_check_product_migrations.py::TestCheckProductMigrations posthog/test/test_dbrouter.py::TestReplicaRouter posthog/test/test_domain_connect.py::TestBuildSyncApplyUrl -posthog/test/test_domain_connect.py::TestDiscoverDomainConnect posthog/test/test_domain_connect.py::TestExtractRootDomainAndHost posthog/test/test_domain_connect.py::TestGenerateApplyUrl posthog/test/test_domain_connect.py::TestGetAvailableProviders diff --git a/posthog/test/test_dns_hosts.py b/posthog/test/test_dns_hosts.py new file mode 100644 index 000000000000..af514dca37dc --- /dev/null +++ b/posthog/test/test_dns_hosts.py @@ -0,0 +1,28 @@ +from django.test import SimpleTestCase + +from parameterized import parameterized + +from posthog.dns_hosts import detect_dns_host + + +class TestDetectDnsHost(SimpleTestCase): + @parameterized.expand( + [ + (["ns-1234.awsdns-56.org.", "ns-789.awsdns-12.co.uk."], "Route 53"), + (["ns51.domaincontrol.com"], "GoDaddy"), + (["dns1.registrar-servers.com"], "Namecheap"), + (["ns-cloud-a1.squarespacedns.com"], "Squarespace"), + (["ns1.dns-parking.com"], "Hostinger"), + (["ns1.hostinger.com"], "Hostinger"), + (["ns1045.ui-dns.de"], "IONOS"), + (["ns1.digitalocean.com"], "DigitalOcean"), + (["ADA.NS.CLOUDFLARE.COM."], "Cloudflare"), + (["ns1.vercel-dns.com"], "Vercel"), + (["ns1.mydigitalocean.com.evil.example"], None), + (["ns1.example.com"], None), + ([], None), + ] + ) + def test_names_host_from_nameservers(self, nameservers: list[str], expected: str | None) -> None: + host = detect_dns_host(nameservers) + assert (host.name if host else None) == expected diff --git a/posthog/test/test_domain_connect.py b/posthog/test/test_domain_connect.py index ae726bdeba9e..95228af4e6f3 100644 --- a/posthog/test/test_domain_connect.py +++ b/posthog/test/test_domain_connect.py @@ -7,9 +7,6 @@ from posthog.test.base import BaseTest from unittest.mock import MagicMock, patch -from django.core.cache import cache - -import dns.resolver from cryptography.hazmat.primitives import hashes as crypto_hashes from cryptography.hazmat.primitives.asymmetric import ( padding as asym_padding, @@ -19,15 +16,11 @@ from posthog.domain_connect import ( DOMAIN_CONNECT_PROVIDERS, - EMAIL_TEMPLATE_GROUPS_WITHOUT_DMARC, - DomainConnectSigningKeyMissing, build_sync_apply_url, - discover_domain_connect, extract_root_domain_and_host, generate_apply_url, get_available_providers, get_service_id_for_region, - resolve_email_context, resolve_proxy_context, sign_query_string, ) @@ -187,54 +180,6 @@ def test_sign_and_verify(self) -> None: ) -class TestDiscoverDomainConnect(BaseTest): - def setUp(self) -> None: - super().setUp() - cache.clear() - - @patch("posthog.domain_connect._fetch_provider_settings") - @patch("posthog.domain_connect._lookup_domain_connect_endpoint") - def test_supported_provider(self, mock_lookup: MagicMock, mock_settings: MagicMock) -> None: - mock_lookup.return_value = "api.cloudflare.com/client/v4/dns/domainconnect" - mock_settings.return_value = {"urlSyncUX": "https://dash.cloudflare.com/domainconnect"} - - with patch.dict(DOMAIN_CONNECT_PROVIDERS, {"api.cloudflare.com/client/v4/dns/domainconnect": "Cloudflare"}): - result = discover_domain_connect("example.com") - - self.assertIsNotNone(result) - - if result: - self.assertEqual(result["provider_name"], "Cloudflare") - self.assertEqual(result["url_sync_ux"], "https://dash.cloudflare.com/domainconnect") - - @patch("posthog.domain_connect._lookup_domain_connect_endpoint") - def test_unsupported_provider(self, mock_lookup: MagicMock) -> None: - mock_lookup.return_value = "unknown.provider.example" - - result = discover_domain_connect("example.com") - - self.assertIsNone(result) - - @patch("posthog.domain_connect._lookup_domain_connect_endpoint") - def test_no_txt_record(self, mock_lookup: MagicMock) -> None: - mock_lookup.return_value = None - - result = discover_domain_connect("example.com") - - self.assertIsNone(result) - - @patch("posthog.domain_connect._fetch_provider_settings") - @patch("posthog.domain_connect._lookup_domain_connect_endpoint") - def test_provider_settings_unavailable(self, mock_lookup: MagicMock, mock_settings: MagicMock) -> None: - mock_lookup.return_value = "api.cloudflare.com/client/v4/dns/domainconnect" - mock_settings.return_value = None - - with patch.dict(DOMAIN_CONNECT_PROVIDERS, {"api.cloudflare.com/client/v4/dns/domainconnect": "Cloudflare"}): - result = discover_domain_connect("example.com") - - self.assertIsNone(result) - - class TestGetAvailableProviders(BaseTest): def test_returns_all_providers(self) -> None: with patch.dict( @@ -266,53 +211,6 @@ def test_rejects_unknown_provider_endpoint(self) -> None: provider_endpoint="evil.internal.service", ) - @patch("posthog.domain_connect._fetch_provider_settings") - @patch("posthog.domain_connect.get_signing_key") - def test_allows_known_provider_endpoint(self, mock_key: MagicMock, mock_settings: MagicMock) -> None: - mock_key.return_value = rsa.generate_private_key(public_exponent=65537, key_size=2048) - mock_settings.return_value = {"urlSyncUX": "https://dash.cloudflare.com/domainconnect"} - - url = generate_apply_url( - domain="example.com", - service_id="reverse-proxy-us", - variables={"target": "abc.proxy.posthog.com"}, - provider_endpoint="api.cloudflare.com/client/v4/dns/domainconnect", - ) - - self.assertIn("domain=example.com", url) - self.assertIn("sig=", url) - - @patch("posthog.domain_connect.get_signing_key") - def test_rejects_signing_required_provider_without_key(self, mock_key: MagicMock) -> None: - mock_key.return_value = None - - with self.assertRaises(DomainConnectSigningKeyMissing): - generate_apply_url( - domain="example.com", - service_id="reverse-proxy-us", - variables={"target": "abc.proxy.posthog.com"}, - provider_endpoint="api.cloudflare.com/client/v4/dns/domainconnect", - ) - - @patch("posthog.domain_connect.discover_domain_connect") - @patch("posthog.domain_connect.get_signing_key") - def test_rejects_discovered_signing_required_provider_without_key( - self, mock_key: MagicMock, mock_discover: MagicMock - ) -> None: - mock_key.return_value = None - mock_discover.return_value = { - "provider_name": "Cloudflare", - "endpoint": "api.cloudflare.com/client/v4/dns/domainconnect", - "url_sync_ux": "https://dash.cloudflare.com/domainconnect", - } - - with self.assertRaises(DomainConnectSigningKeyMissing): - generate_apply_url( - domain="example.com", - service_id="reverse-proxy-us", - variables={"target": "abc.proxy.posthog.com"}, - ) - class TestTemplateResolverAlignment(BaseTest): """Ensure backend resolvers produce variables that exactly match the template placeholders.""" @@ -344,117 +242,3 @@ def test_proxy_resolver_variables_match_template( self.assertEqual(resolved.host, "ph") if template.get("hostRequired"): self.assertTrue(resolved.host, "hostRequired template but resolver returned empty host") - - @parameterized.expand( - [ - ( - "subdomain sender without a dmarc record", - "posthog.com.email-verification-us.json", - "US", - "news.example.com", - "news", - dns.resolver.NXDOMAIN(), - True, - ), - ( - "root sender without a dmarc record", - "posthog.com.email-verification-eu.json", - "EU", - "example.com", - "", - dns.resolver.NoAnswer(), - True, - ), - ( - "sender with an existing dmarc record", - "posthog.com.email-verification-us.json", - "US", - "news.example.com", - "news", - [MagicMock(strings=[b"v=DMARC1; p=reject;"])], - False, - ), - ( - "sender whose dmarc lookup times out", - "posthog.com.email-verification-eu.json", - "EU", - "example.com", - "", - dns.resolver.Timeout(), - False, - ), - ( - "sender whose dmarc record is not utf-8", - "posthog.com.email-verification-us.json", - "US", - "news.example.com", - "news", - [MagicMock(strings=[b"\xff"])], - False, - ), - ] - ) - @patch("posthog.models.integration.EmailIntegration") - @patch("posthog.models.integration.Integration") - def test_email_resolver_variables_match_template( - self, - _name: str, - template_file: str, - region: str, - sender_domain: str, - expected_host: str, - dmarc_lookup: Exception | list[MagicMock], - applies_dmarc: bool, - mock_integration_cls: MagicMock, - mock_email_cls: MagicMock, - ) -> None: - template = _load_template(template_file) - expected_vars = _extract_template_variables(template) - - mock_instance = MagicMock() - mock_instance.kind = "email" - mock_instance.config = {"domain": sender_domain, "mail_from_subdomain": "feedback"} - mock_integration_cls.objects.get.return_value = mock_instance - - mock_email = MagicMock() - mock_email.verify.return_value = { - "dnsRecords": [ - { - "type": "verification", - "recordType": "TXT", - "recordHostname": f"_amazonses.{sender_domain}", - "recordValue": "verify-token-123", - }, - {"type": "dkim", "recordHostname": f"aaa._domainkey.{sender_domain}"}, - {"type": "dkim", "recordHostname": f"bbb._domainkey.{sender_domain}"}, - {"type": "dkim", "recordHostname": f"ccc._domainkey.{sender_domain}"}, - ] - } - mock_email_cls.return_value = mock_email - - with ( - self.settings(CLOUD_DEPLOYMENT=region, SES_REGION="us-east-1"), - patch("posthog.domain_connect.dns.resolver.resolve") as mock_resolve, - ): - if isinstance(dmarc_lookup, Exception): - mock_resolve.side_effect = dmarc_lookup - else: - mock_resolve.return_value = dmarc_lookup - resolved = resolve_email_context(1, 1) - - mock_resolve.assert_called_once_with(f"_dmarc.{sender_domain}", "TXT", lifetime=5) - - self.assertEqual(set(resolved.variables.keys()), expected_vars) - self.assertEqual(resolved.service_id, template["serviceId"]) - self.assertEqual(resolved.root_domain, "example.com") - self.assertEqual(resolved.host, expected_host) - self.assertEqual(resolved.variables["verifyToken"], "verify-token-123") - self.assertEqual(resolved.variables["dkim1"], "aaa") - self.assertEqual(resolved.variables["mailFromSub"], "feedback") - - template_groups = {record["groupId"] for record in template["records"]} - if applies_dmarc: - self.assertEqual(resolved.group_ids, ()) - else: - self.assertEqual(resolved.group_ids, EMAIL_TEMPLATE_GROUPS_WITHOUT_DMARC) - self.assertEqual(template_groups - set(resolved.group_ids), {"dmarc"}) diff --git a/products/integrations/frontend/generated/api.schemas.ts b/products/integrations/frontend/generated/api.schemas.ts index 0a496540c65e..c083ccd33cb9 100644 --- a/products/integrations/frontend/generated/api.schemas.ts +++ b/products/integrations/frontend/generated/api.schemas.ts @@ -645,6 +645,36 @@ export interface SlackUsersResponseApi { has_more?: boolean } +export interface EmailDomainConnectProviderApi { + /** Domain Connect endpoint of the provider. */ + endpoint: string + /** Display name of the provider. */ + name: string +} + +export interface EmailDomainDnsHostApi { + /** Name of the company that hosts the domain's DNS, such as Cloudflare. */ + name: string + /** Link to the DNS settings page at that host. */ + dns_settings_url: string +} + +export interface EmailDomainConnectCheckApi { + /** Whether the domain's DNS provider supports automatic setup through Domain Connect. */ + supported: boolean + /** + * Domain Connect provider that hosts the domain. Null when not supported. + * @nullable + */ + provider_name: string | null + /** Providers the user can pick manually when automatic detection fails. Empty when supported. */ + available_providers: EmailDomainConnectProviderApi[] + /** The detected DNS host of the root domain, from its nameservers. Null when unknown. */ + dns_host: EmailDomainDnsHostApi | null + /** Email tools that already send for the root domain, detected from SPF and DKIM records. Best effort, empty when none are found. */ + existing_email_tools: string[] +} + /** * * `ok` - Ok * * `not_connected` - Not Connected diff --git a/products/integrations/frontend/generated/api.ts b/products/integrations/frontend/generated/api.ts index 6bb7e502d3af..2625fda24b5d 100644 --- a/products/integrations/frontend/generated/api.ts +++ b/products/integrations/frontend/generated/api.ts @@ -9,6 +9,7 @@ import { apiMutator } from '../../../../frontend/src/lib/api-orval-mutator' * OpenAPI spec version: 1.0.0 */ import type { + EmailDomainConnectCheckApi, EmailDomainStatusApi, GitHubAvailableInstallationsResponseApi, GitHubBranchesResponseApi, @@ -880,8 +881,8 @@ export const getIntegrationsDomainConnectCheckRetrieveUrl = (projectId: string) export const integrationsDomainConnectCheckRetrieve = async ( projectId: string, options?: RequestInit -): Promise => { - return apiMutator(getIntegrationsDomainConnectCheckRetrieveUrl(projectId), { +): Promise => { + return apiMutator(getIntegrationsDomainConnectCheckRetrieveUrl(projectId), { ...options, method: 'GET', }) diff --git a/products/workflows/backend/facade/api.py b/products/workflows/backend/facade/api.py index e3c943c8b3e3..cf5d388f66ee 100644 --- a/products/workflows/backend/facade/api.py +++ b/products/workflows/backend/facade/api.py @@ -29,6 +29,9 @@ suspend_email_sending, unsuspend_email_sending, ) +from products.workflows.backend.services.existing_email_tools import ( + lookup_existing_email_tools as _lookup_existing_email_tools, +) from products.workflows.backend.services.integration_usage import get_active_hog_flows_using_integration from products.workflows.backend.services.template_input_usage import ( filter_hog_flow_references_by_access_level, @@ -311,6 +314,10 @@ def get_maildev_email_domain_status(domain: str, *, mail_from_subdomain: str) -> return providers.maildev_email_domain_status(domain, mail_from_subdomain=mail_from_subdomain) +def lookup_existing_email_tools(root_domain: str) -> list[str]: + return _lookup_existing_email_tools(root_domain) + + def delete_ses_identity(identity: str) -> None: from products.workflows.backend import providers # noqa: PLC0415 diff --git a/products/workflows/backend/services/existing_email_tools.py b/products/workflows/backend/services/existing_email_tools.py new file mode 100644 index 000000000000..7ed7d22ea258 --- /dev/null +++ b/products/workflows/backend/services/existing_email_tools.py @@ -0,0 +1,91 @@ +from concurrent.futures import ThreadPoolExecutor + +from django.core.cache import cache + +import dns.resolver +import dns.exception +from dns.rdtypes.ANY.CNAME import CNAME +from dns.rdtypes.ANY.TXT import TXT + +LOOKUP_TIMEOUT_SECONDS = 3 +CACHE_SECONDS = 60 * 60 + +# Matched as substrings of SPF include targets and DKIM CNAME targets. Customer.io sends through +# Mailgun, so its domains often show up as Mailgun here. +_TOOLS_BY_TARGET_MARKER: tuple[tuple[str, str], ...] = ( + ("customeriomail.com", "Customer.io"), + ("mcsv.net", "Mailchimp"), + ("mandrillapp.com", "Mailchimp"), + ("sendgrid.net", "SendGrid"), + ("mailgun.org", "Mailgun"), + ("mtasv.net", "Postmark"), + ("hubspotemail.net", "HubSpot"), + ("klaviyo", "Klaviyo"), + ("brevo.com", "Brevo"), + ("sendinblue.com", "Brevo"), +) +_DKIM_CNAME_SELECTORS = ("k1", "s1", "kl", "brevo1") +_RESEND_DKIM_SELECTOR = "resend" + + +def detect_existing_email_tools(*, spf_records: list[str], dkim_targets: list[str], has_resend_dkim: bool) -> list[str]: + targets = [*_spf_include_targets(spf_records), *(target.lower() for target in dkim_targets)] + tools = [tool for marker, tool in _TOOLS_BY_TARGET_MARKER if any(marker in target for target in targets)] + if has_resend_dkim: + tools.append("Resend") + return list(dict.fromkeys(tools)) + + +def lookup_existing_email_tools(root_domain: str) -> list[str]: + cache_key = f"existing_email_tools:{root_domain}" + cached = cache.get(cache_key) + if cached is not None: + return cached + + resolver = dns.resolver.Resolver() + resolver.lifetime = LOOKUP_TIMEOUT_SECONDS + lookups = { + "spf": (root_domain, "TXT"), + _RESEND_DKIM_SELECTOR: (f"{_RESEND_DKIM_SELECTOR}._domainkey.{root_domain}", "TXT"), + **{selector: (f"{selector}._domainkey.{root_domain}", "CNAME") for selector in _DKIM_CNAME_SELECTORS}, + } + with ThreadPoolExecutor(max_workers=len(lookups)) as pool: + answers = dict(zip(lookups, pool.map(lambda lookup: _resolve(resolver, *lookup), lookups.values()))) + tools = detect_existing_email_tools( + spf_records=_txt_values(answers["spf"]), + dkim_targets=[target for selector in _DKIM_CNAME_SELECTORS for target in _cname_targets(answers[selector])], + has_resend_dkim=bool(answers[_RESEND_DKIM_SELECTOR]), + ) + # A lookup without an answer says nothing about the domain, so the next check asks again. + if all(answer is not None for answer in answers.values()): + cache.set(cache_key, tools, CACHE_SECONDS) + return tools + + +def _spf_include_targets(txt_records: list[str]) -> list[str]: + return [ + term.removeprefix("include:") + for record in txt_records + if record.lower().startswith("v=spf1") + for term in record.lower().split() + if term.startswith("include:") + ] + + +def _txt_values(answers: list[object] | None) -> list[str]: + return [ + b"".join(rdata.strings).decode("utf-8", errors="replace") for rdata in answers or [] if isinstance(rdata, TXT) + ] + + +def _cname_targets(answers: list[object] | None) -> list[str]: + return [rdata.target.to_text() for rdata in answers or [] if isinstance(rdata, CNAME)] + + +def _resolve(resolver: dns.resolver.Resolver, name: str, record_type: str) -> list[object] | None: + try: + return list(resolver.resolve(name, record_type)) + except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): + return [] + except dns.exception.DNSException: + return None diff --git a/products/workflows/backend/test/test_email_domain_connect_api.py b/products/workflows/backend/test/test_email_domain_connect_api.py new file mode 100644 index 000000000000..07c629917a39 --- /dev/null +++ b/products/workflows/backend/test/test_email_domain_connect_api.py @@ -0,0 +1,332 @@ +from collections.abc import Callable +from typing import Any + +from posthog.test.base import APIBaseTest + +from django.core.cache import cache +from django.test import override_settings + +from dns.rdtypes.ANY.TXT import TXT +from parameterized import parameterized +from rest_framework import status + +from posthog.models.integration import EmailIntegration, Integration + +from products.workflows.backend.test.email_domain_api_test_case import ( + ALL_KINDS, + DOMAIN, + ROOT_DOMAIN, + EmailDomainApiTestCase, +) +from products.workflows.backend.test.fakes.fake_dns import dns_operation +from products.workflows.backend.test.fakes.fake_domain_connect import SETTINGS_OPERATION +from products.workflows.backend.test.fakes.faults import DnsFault, HttpFault +from products.workflows.backend.test.fakes.world import EmailDomainWorld, domain_connect_signing_key_pem + +CLOUDFLARE_NAMESERVERS = {"ana.ns.cloudflare.com": "172.64.32.1", "bob.ns.cloudflare.com": "108.162.192.1"} +CLOUDFLARE_HOST = { + "name": "Cloudflare", + "dns_settings_url": "https://dash.cloudflare.com/?to=/:account/:zone/dns/records", +} +ROUTE_53_HOST = {"name": "Route 53", "dns_settings_url": "https://console.aws.amazon.com/route53/v2/hostedzones"} +APPLY_URL_FAILED = "Error generating apply URL. Please try again later or contact support." +SIGNING_UNAVAILABLE = ( + "Automatic DNS configuration is temporarily unavailable for this provider. " + "Please configure your DNS records manually." +) +DISCOVERY_RECORD = f"_domainconnect.{ROOT_DOMAIN}" +DMARC_LOOKUP = dns_operation(f"_dmarc.{DOMAIN}", "TXT") + + +@override_settings(DOMAIN_CONNECT_PRIVATE_KEY=domain_connect_signing_key_pem(), CLOUD_DEPLOYMENT="US") +class EmailDomainConnectTestCase(EmailDomainApiTestCase): + zone_nameservers = CLOUDFLARE_NAMESERVERS + + def setUp(self) -> None: + super().setUp() + self.world.domain_connect.host(ROOT_DOMAIN) + + def check(self, domain: str = DOMAIN) -> dict[str, Any]: + response = self.client.get( + f"/api/projects/{self.team.id}/integrations/domain-connect/check", {"domain": domain} + ) + assert response.status_code == status.HTTP_200_OK, response.json() + return response.json() + + def apply_url(self, sender: int, *, redirect_host: str = "us.posthog.com") -> Any: + return self.client.post( + f"/api/projects/{self.team.id}/integrations/domain-connect/apply-url", + { + "context": "email", + "integration_id": sender, + "redirect_uri": f"https://{redirect_host}/workflows/channels/email/{sender}?domain_connect=email", + }, + format="json", + ) + + def config(self, sender: int) -> dict[str, Any]: + return Integration.objects.get(id=sender).config + + +class TestOneClickEmailDomainSetup(EmailDomainConnectTestCase): + @parameterized.expand( + [ + ("us_sender_on_a_subdomain", "US", DOMAIN, "us.posthog.com"), + ("eu_sender_on_the_root_domain", "EU", ROOT_DOMAIN, "eu.posthog.com"), + ] + ) + def test_approved_setup_publishes_the_template_records_and_verifies_the_sender( + self, _name: str, region: str, domain: str, redirect_host: str + ) -> None: + with self.settings(CLOUD_DEPLOYMENT=region): + assert self.check(domain) == { + "supported": True, + "provider_name": "Cloudflare", + "available_providers": [], + "dns_host": CLOUDFLARE_HOST, + "existing_email_tools": [], + } + sender = self.create_sender(f"hello@{domain}") + + response = self.apply_url(sender, redirect_host=redirect_host) + assert response.status_code == status.HTTP_200_OK, response.json() + redirect_uri = self.world.domain_connect.apply(response.json()["url"]) + + assert redirect_uri == f"https://{redirect_host}/workflows/channels/email/{sender}?domain_connect=email" + body = self.status_body(sender) + assert (body["status"], body["verified"]) == ("verified", True) + assert self.config(sender)["setup_method"] == "auto" + assert self.verified_event_methods() == ["auto"] + + def test_approved_setup_keeps_an_existing_dmarc_record(self) -> None: + existing_dmarc = "v=DMARC1; p=reject; rua=mailto:dmarc@example.com" + self.world.dns.publish_txt(f"_dmarc.{DOMAIN}", existing_dmarc) + sender = self.create_sender(f"hello@{DOMAIN}") + + self.world.domain_connect.apply(self.apply_url(sender).json()["url"]) + + assert self.world.dns.txt_values(f"_dmarc.{DOMAIN}") == [existing_dmarc] + assert self.status_body(sender)["status"] == "verified" + + @parameterized.expand( + [ + ("dmarc_lookup_times_out", lambda world: world.faults.inject(DMARC_LOOKUP, DnsFault.TIMEOUT)), + ("dmarc_lookup_fails", lambda world: world.faults.inject(DMARC_LOOKUP, DnsFault.SERVFAIL)), + ("dmarc_record_is_not_utf8", lambda world: world.dns.publish(f"_dmarc.{DOMAIN}", "TXT", '"\\255"')), + ] + ) + def test_approved_setup_leaves_dmarc_alone_when_an_existing_record_cannot_be_ruled_out( + self, _name: str, arrange: Callable[[EmailDomainWorld], None] + ) -> None: + sender = self.create_sender(f"hello@{DOMAIN}") + arrange(self.world) + + self.world.domain_connect.apply(self.apply_url(sender).json()["url"]) + self.world.faults.clear() + + published_dmarc = [ + b"".join(rdata.strings) + for rdata in self.world.dns.answers(f"_dmarc.{DOMAIN}", "TXT") + if isinstance(rdata, TXT) + ] + assert b"v=DMARC1; p=none;" not in published_dmarc + assert self.world.dns.txt_values(f"_amazonses.{DOMAIN}") != [] + + def test_returning_without_approving_publishes_nothing_and_manual_setup_still_verifies(self) -> None: + sender = self.create_sender(f"hello@{DOMAIN}") + assert self.apply_url(sender).status_code == status.HTTP_200_OK + + body = self.status_body(sender) + assert (body["status"], self.world.dns.txt_values(f"_amazonses.{DOMAIN}")) == ("pending", []) + + response = self.client.patch( + f"/api/projects/{self.team.id}/integrations/{sender}/email", + {"config": {**self.config(sender), "setup_method": "manual"}}, + format="json", + ) + assert response.status_code == status.HTTP_200_OK, response.json() + self.publish(body["records"], *ALL_KINDS) + + assert self.status_body(sender)["status"] == "verified" + assert self.verified_event_methods() == ["manual"] + + def test_asking_for_setup_on_an_already_published_domain_verifies_it_at_once(self) -> None: + sender = self.create_sender(f"hello@{DOMAIN}") + self.publish(self.records_to_publish(sender), *ALL_KINDS) + + assert self.apply_url(sender).status_code == status.HTTP_200_OK + + assert self.config(sender)["verified"] is True + assert len(self.verified_events()) == 1 + assert self.status_body(sender)["verified"] is True + assert len(self.verified_events()) == 1 + + @parameterized.expand([("same_bounce_subdomain", "feedback", 0), ("new_bounce_subdomain", "bounce", 1)]) + def test_sender_update_stores_the_setup_method_and_writes_ses_only_for_a_new_bounce_subdomain( + self, _name: str, mail_from_subdomain: str, expected_ses_writes: int + ) -> None: + sender = self.create_sender(f"hello@{DOMAIN}") + writes_before = self.world.ses.writes.count("set_identity_mail_from_domain") + + response = self.client.patch( + f"/api/projects/{self.team.id}/integrations/{sender}/email", + {"config": {**self.config(sender), "mail_from_subdomain": mail_from_subdomain, "setup_method": "manual"}}, + format="json", + ) + + assert response.status_code == status.HTTP_200_OK, response.json() + assert self.config(sender)["setup_method"] == "manual" + assert self.world.ses.writes.count("set_identity_mail_from_domain") - writes_before == expected_ses_writes + assert self.world.ses.identities[DOMAIN].mail_from_domain == f"{mail_from_subdomain}.{DOMAIN}" + + def test_sender_update_keeps_a_verification_written_after_the_sender_was_loaded(self) -> None: + sender = self.create_sender(f"hello@{DOMAIN}") + loaded_before_verification = Integration.objects.get(id=sender) + Integration.objects.filter(id=sender).update(config={**self.config(sender), "verified": True}) + + EmailIntegration(loaded_before_verification).update_native_integration( + {"name": "Renamed", "setup_method": "manual"}, self.team.id + ) + + assert self.config(sender)["verified"] is True + assert self.config(sender)["name"] == "Renamed" + assert self.config(sender)["setup_method"] == "manual" + + +class TestDomainConnectChaos(EmailDomainConnectTestCase): + @parameterized.expand( + [ + ("discovery_lookup_times_out", dns_operation(DISCOVERY_RECORD, "TXT"), DnsFault.TIMEOUT), + ("discovery_lookup_fails", dns_operation(DISCOVERY_RECORD, "TXT"), DnsFault.SERVFAIL), + ("settings_answer_a_server_error", SETTINGS_OPERATION, HttpFault.SERVER_ERROR), + ("settings_are_rate_limited", SETTINGS_OPERATION, HttpFault.RATE_LIMITED), + ("settings_time_out", SETTINGS_OPERATION, HttpFault.TIMEOUT), + ("settings_answer_malformed_json", SETTINGS_OPERATION, HttpFault.MALFORMED_JSON), + ] + ) + def test_transient_provider_failure_falls_back_to_manual_setup_and_recovers( + self, _name: str, operation: str, fault: DnsFault | HttpFault + ) -> None: + sender = self.create_sender(f"hello@{DOMAIN}") + config_before = self.config(sender) + self.world.faults.inject(operation, fault) + + assert self.check()["supported"] is False + response = self.apply_url(sender) + + assert (response.status_code, response.json()["detail"]) == (status.HTTP_400_BAD_REQUEST, APPLY_URL_FAILED) + assert self.config(sender) == config_before + + self.world.faults.clear() + assert self.check()["supported"] is True + assert self.apply_url(sender).status_code == status.HTTP_200_OK + assert self.config(sender)["setup_method"] == "auto" + + @parameterized.expand( + [ + ( + "no_discovery_record", + lambda test: test.world.dns.unpublish(DISCOVERY_RECORD, "TXT"), + False, + APPLY_URL_FAILED, + ), + ( + "discovery_record_does_not_exist", + lambda test: test.world.faults.inject(dns_operation(DISCOVERY_RECORD, "TXT"), DnsFault.NXDOMAIN), + False, + APPLY_URL_FAILED, + ), + ( + "unsupported_provider", + lambda test: ( + test.world.dns.unpublish(DISCOVERY_RECORD, "TXT") + or test.world.dns.publish_txt(DISCOVERY_RECORD, "domainconnect.unsupported-dns.example") + ), + False, + APPLY_URL_FAILED, + ), + ( + "settings_without_a_sync_url", + lambda test: test.world.faults.inject(SETTINGS_OPERATION, HttpFault.MISSING_URL_SYNC_UX), + False, + APPLY_URL_FAILED, + ), + ( + "signing_key_missing", + lambda test: test.enterContext(override_settings(DOMAIN_CONNECT_PRIVATE_KEY=None)), + True, + SIGNING_UNAVAILABLE, + ), + ] + ) + def test_unusable_provider_answers_400_and_leaves_the_sender_unchanged( + self, _name: str, arrange: Callable[[EmailDomainConnectTestCase], object], supported: bool, detail: str + ) -> None: + sender = self.create_sender(f"hello@{DOMAIN}") + config_before = self.config(sender) + arrange(self) + + check = self.check() + response = self.apply_url(sender) + + assert (check["supported"], bool(check["available_providers"])) == (supported, not supported) + assert (response.status_code, response.json()["detail"]) == (status.HTTP_400_BAD_REQUEST, detail) + assert self.config(sender) == config_before + + +@override_settings(SES_REGION="us-east-1") +class TestDomainConnectCheckDetectsTheDnsHost(APIBaseTest): + def setUp(self) -> None: + super().setUp() + cache.clear() + self.world = self.enterContext(EmailDomainWorld().installed()) + + def check(self, domain: str) -> dict[str, Any]: + response = self.client.get( + f"/api/projects/{self.team.id}/integrations/domain-connect/check", {"domain": domain} + ) + assert response.status_code == status.HTTP_200_OK, response.json() + return response.json() + + @parameterized.expand( + [ + ("cloudflare", CLOUDFLARE_NAMESERVERS, CLOUDFLARE_HOST), + ("route_53", {"ns-1536.awsdns-00.co.uk": "205.251.198.0"}, ROUTE_53_HOST), + ("unknown_host", {"ns1.example-dns.net": "34.120.0.1"}, None), + ] + ) + def test_names_the_dns_host_from_the_root_domains_nameservers( + self, _name: str, nameservers: dict[str, str], expected_host: dict[str, str] | None + ) -> None: + self.world.dns.add_zone(ROOT_DOMAIN, nameservers=nameservers) + + assert self.check(DOMAIN)["dns_host"] == expected_host + + def test_lists_the_email_tools_already_sending_for_the_root_domain_and_caches_them(self) -> None: + self.world.dns.add_zone(ROOT_DOMAIN, nameservers=CLOUDFLARE_NAMESERVERS) + self.world.dns.publish_txt(ROOT_DOMAIN, "v=spf1 include:sendgrid.net include:_spf.google.com ~all") + self.world.dns.publish(f"k1._domainkey.{ROOT_DOMAIN}", "CNAME", "dkim.mcsv.net.") + self.world.dns.publish_txt(f"resend._domainkey.{ROOT_DOMAIN}", "p=MIGfMA0GCSqGSIb3DQEB") + + assert self.check(DOMAIN)["existing_email_tools"] == ["Mailchimp", "SendGrid", "Resend"] + + self.world.dns.unpublish(ROOT_DOMAIN, "TXT") + assert self.check(DOMAIN)["existing_email_tools"] == ["Mailchimp", "SendGrid", "Resend"] + + @parameterized.expand( + [ + ("nameserver_lookup", ROOT_DOMAIN, "NS", "dns_host", CLOUDFLARE_HOST), + ("spf_lookup", ROOT_DOMAIN, "TXT", "existing_email_tools", ["SendGrid"]), + ] + ) + def test_failed_lookup_answers_empty_and_is_retried_on_the_next_check( + self, _name: str, name: str, rdtype: str, field: str, recovered: Any + ) -> None: + self.world.dns.add_zone(ROOT_DOMAIN, nameservers=CLOUDFLARE_NAMESERVERS) + self.world.dns.publish_txt(ROOT_DOMAIN, "v=spf1 include:sendgrid.net ~all") + self.world.faults.inject(dns_operation(name, rdtype), DnsFault.TIMEOUT) + + assert not self.check(DOMAIN)[field] + + self.world.faults.clear() + assert self.check(DOMAIN)[field] == recovered diff --git a/products/workflows/backend/test/test_existing_email_tools.py b/products/workflows/backend/test/test_existing_email_tools.py new file mode 100644 index 000000000000..d489995ee1c4 --- /dev/null +++ b/products/workflows/backend/test/test_existing_email_tools.py @@ -0,0 +1,38 @@ +from django.test import SimpleTestCase + +from parameterized import parameterized + +from products.workflows.backend.services.existing_email_tools import detect_existing_email_tools + + +class TestDetectExistingEmailTools(SimpleTestCase): + @parameterized.expand( + [ + ( + "spf_includes", + ["v=spf1 include:servers.mcsv.net include:sendgrid.net ~all"], + [], + False, + ["Mailchimp", "SendGrid"], + ), + ("not_spf", ["google-site-verification=include:sendgrid.net"], [], False, []), + ("dkim_cname", [], ["s1.domainkey.u123.wl.sendgrid.net."], False, ["SendGrid"]), + ("resend_dkim", [], [], True, ["Resend"]), + ( + "deduplicated", + ["v=spf1 include:spf.brevo.com ~all"], + ["b1.example-com.dkim.brevo.com."], + False, + ["Brevo"], + ), + ] + ) + def test_names_tools_from_dns( + self, _name: str, spf_records: list[str], dkim_targets: list[str], has_resend_dkim: bool, expected: list[str] + ) -> None: + assert ( + detect_existing_email_tools( + spf_records=spf_records, dkim_targets=dkim_targets, has_resend_dkim=has_resend_dkim + ) + == expected + ) diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts index f2ed4c5693b7..603863141aee 100644 --- a/services/mcp/src/api/generated.ts +++ b/services/mcp/src/api/generated.ts @@ -37775,6 +37775,36 @@ export namespace Schemas { name: string; } + export interface EmailDomainConnectProvider { + /** Domain Connect endpoint of the provider. */ + endpoint: string; + /** Display name of the provider. */ + name: string; + } + + export interface EmailDomainDnsHost { + /** Name of the company that hosts the domain's DNS, such as Cloudflare. */ + name: string; + /** Link to the DNS settings page at that host. */ + dns_settings_url: string; + } + + export interface EmailDomainConnectCheck { + /** Whether the domain's DNS provider supports automatic setup through Domain Connect. */ + supported: boolean; + /** + * Domain Connect provider that hosts the domain. Null when not supported. + * @nullable + */ + provider_name: string | null; + /** Providers the user can pick manually when automatic detection fails. Empty when supported. */ + available_providers: EmailDomainConnectProvider[]; + /** The detected DNS host of the root domain, from its nameservers. Null when unknown. */ + dns_host: EmailDomainDnsHost | null; + /** Email tools that already send for the root domain, detected from SPF and DKIM records. Best effort, empty when none are found. */ + existing_email_tools: string[]; + } + /** * * `verification` - Domain ownership * * `dkim` - DKIM signing From 9e438347c5afbb9d2848f493df580550034fbe64 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 7 Oct 2026 08:57:23 +0000 Subject: [PATCH 05/13] chore(workflows): preserve shipped domain connect route exemption --- posthog/api/integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/posthog/api/integration.py b/posthog/api/integration.py index abf93c436d04..d6ab424ebf09 100644 --- a/posthog/api/integration.py +++ b/posthog/api/integration.py @@ -2737,8 +2737,8 @@ def email_update(self, request, **kwargs) -> Response: return Response(IntegrationSerializer(email.integration).data) - # nosemgrep: api-path-underscore -- shipped public API path, a rename breaks clients @extend_schema(responses={200: EmailDomainConnectCheckSerializer}) + # nosemgrep: api-path-underscore -- shipped public API path, a rename breaks clients @action(methods=["GET"], detail=False, url_path="domain-connect/check") def domain_connect_check(self, request: Request, *args: Any, **kwargs: Any) -> Response: domain = request.query_params.get("domain", "") From 1346a74fd002b6c2eb4eee10aec288a8a4f4a370 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:20:48 +0000 Subject: [PATCH 06/13] fix(workflows): name email domain setup method choices --- posthog/api/integration.py | 3 ++- products/workflows/backend/facade/enums.py | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/posthog/api/integration.py b/posthog/api/integration.py index d6ab424ebf09..4b6e3dda23ee 100644 --- a/posthog/api/integration.py +++ b/posthog/api/integration.py @@ -139,6 +139,7 @@ from products.tasks.backend.facade.contracts import InProgressGithubRunsDTO from products.workflows.backend.facade.api import get_active_workflows_using_integration, lookup_existing_email_tools from products.workflows.backend.facade.contracts import EmailProviderUnavailableError +from products.workflows.backend.facade.enums import EmailDomainSetupMethod logger = structlog.get_logger(__name__) @@ -326,7 +327,7 @@ class NativeEmailIntegrationSerializer(serializers.Serializer): provider = serializers.ChoiceField(choices=["ses", "maildev"] if settings.DEBUG else ["ses"]) mail_from_subdomain = serializers.CharField(required=False, allow_blank=True) setup_method = serializers.ChoiceField( - choices=["auto", "manual"], + choices=EmailDomainSetupMethod.choices, required=False, help_text="How the DNS records are being published: `auto` through Domain Connect, `manual` by hand. " "Reported on the verification event.", diff --git a/products/workflows/backend/facade/enums.py b/products/workflows/backend/facade/enums.py index c75d6d38b4db..fee899d0559d 100644 --- a/products/workflows/backend/facade/enums.py +++ b/products/workflows/backend/facade/enums.py @@ -56,6 +56,11 @@ class WorkflowProposalStatus(LabeledStrEnum): APPLIED = "applied", "Applied" +class EmailDomainSetupMethod(LabeledStrEnum): + AUTO = "auto", "Domain Connect" + MANUAL = "manual", "Manual DNS records" + + class EmailDomainSetupStatus(LabeledStrEnum): NOT_STARTED = "not_started", "Not started" PENDING = "pending", "Waiting for DNS records" From 87e0470d7852192d26cfa8fd4d58898a30327919 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Fri, 2 Oct 2026 05:09:05 +0000 Subject: [PATCH 07/13] chore(workflows): add the email domain wizard flags and type the domain connect check Adds the workflows-email-domain-wizard and workflows-email-domain-agent-setup flags. The Domain Connect check client and logic read the generated EmailDomainConnectCheckApi type instead of a hand-written copy, so they see the new dns_host and existing_email_tools keys. Co-Authored-By: Claude Opus 5.5 (1M context) --- frontend/src/lib/api.ts | 11 ++--------- .../components/DomainConnect/domainConnectLogic.ts | 12 +++++------- frontend/src/lib/constants.tsx | 2 ++ 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 5b32879c3c8e..914b61d5c0a4 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -38,7 +38,6 @@ import { DashboardFilter, DatabaseSerializedFieldType, DataWarehouseManagedViewsetKind, - DomainConnectProviderName, EndpointLastExecutionTimesRequest, EndpointRequest, EndpointRunRequest, @@ -219,6 +218,7 @@ import type { SymbolSetOrder } from 'products/error_tracking/frontend/scenes/Err import type { ErrorTrackingRecommendation } from 'products/error_tracking/frontend/scenes/ErrorTrackingScene/tabs/recommendations/types' import type { CopyFlagsResponseApi } from 'products/feature_flags/frontend/generated/api.schemas' import type { + EmailDomainConnectCheckApi, GitHubBranchesResponseApi, GitHubReposResponseApi, } from 'products/integrations/frontend/generated/api.schemas' @@ -6080,14 +6080,7 @@ const api = { ): Promise { return await new ApiRequest().integrationEmail(integrationId).update({ data }) }, - async domainConnectCheck(domain: string): Promise<{ - supported: boolean - provider_name: DomainConnectProviderName | null - available_providers: { - endpoint: string - name: DomainConnectProviderName - }[] - }> { + async domainConnectCheck(domain: string): Promise { return await new ApiRequest() .integrationsDomainConnectCheck() .withQueryString(`domain=${encodeURIComponent(domain)}`) diff --git a/frontend/src/lib/components/DomainConnect/domainConnectLogic.ts b/frontend/src/lib/components/DomainConnect/domainConnectLogic.ts index a2a9f2caa801..1862a3e86bdf 100644 --- a/frontend/src/lib/components/DomainConnect/domainConnectLogic.ts +++ b/frontend/src/lib/components/DomainConnect/domainConnectLogic.ts @@ -7,16 +7,14 @@ import { lemonToast } from 'lib/lemon-ui/LemonToast' import { DomainConnectProviderName } from '~/queries/schema/schema-general' +import type { EmailDomainConnectCheckApi } from 'products/integrations/frontend/generated/api.schemas' + export interface DomainConnectProvider { endpoint: string name: DomainConnectProviderName } -export interface DomainConnectInfo { - supported: boolean - provider_name: DomainConnectProviderName | null - available_providers: DomainConnectProvider[] -} +export type DomainConnectInfo = EmailDomainConnectCheckApi export interface DomainConnectLogicProps { /** Unique key to prevent state sharing between instances (not `key` because React strips it) */ @@ -115,13 +113,13 @@ export const domainConnectLogic = kea([ providerName: [ (s) => [s.domainConnectInfo], (info: DomainConnectInfo | null): DomainConnectProviderName | null => { - return info?.provider_name ?? null + return (info?.provider_name as DomainConnectProviderName | null) ?? null }, ], availableProviders: [ (s) => [s.domainConnectInfo], (info: DomainConnectInfo | null): DomainConnectProvider[] => { - return info?.available_providers ?? [] + return (info?.available_providers as DomainConnectProvider[] | undefined) ?? [] }, ], hasAnyOption: [ diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx index 5e5b415ec165..91c0ec611c2e 100644 --- a/frontend/src/lib/constants.tsx +++ b/frontend/src/lib/constants.tsx @@ -620,6 +620,8 @@ export const FEATURE_FLAGS = { WORKFLOW_RUN_SCOUT_ACTION: 'workflow-run-scout-action', // owner: #team-workflows WORKFLOWS_AI_FIRST_NEW: 'workflows-ai-first-new', // owner: @mayteio #team-workflows WORKFLOWS_DELAY_UNTIL_DATE: 'workflows-delay-until-date', // owner: @dmarchuk #team-workflows + WORKFLOWS_EMAIL_DOMAIN_AGENT_SETUP: 'workflows-email-domain-agent-setup', // owner: @silthus #team-workflows + WORKFLOWS_EMAIL_DOMAIN_WIZARD: 'workflows-email-domain-wizard', // owner: @silthus #team-workflows WORKFLOWS_EMAIL_REPUTATION: 'workflows-email-reputation', // owner: #team-workflows WORKFLOWS_EMAIL_SENDER_ROTATION: 'workflows-email-sender-rotation', // owner: @arthurdedeus #team-workflows WORKFLOWS_INTERNAL_EVENT_FILTERS: 'workflows-internal-event-filters', // owner: @haven #team-workflows From a4cc25cad6015245d493155b7d8890cfd6a4b4c5 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Fri, 2 Oct 2026 05:09:16 +0000 Subject: [PATCH 08/13] feat(workflows): add the email sending domain logics and test them against a fake backend The sending domain page state lives in six logics keyed by the integration id, each with one job: the sender record (load, name, setup method, test email), the status and its polling schedule, host detection and the Domain Connect round trip, the manual records path, the agent hand-off prompt, and a composer that derives the phase, the verification steps and the breadcrumbs from the others. The domain step logic infers likely sending domains from the project's event hosts. Domain parsing, domain inference, the agent prompt, the test email workflow and the page URL are pure modules. The status logic polls the read-only email status endpoint while the tab is visible, keeps polling through a 500 or an email_provider_unavailable 503, and stops on 400, 403 and 404. Both manual paths record setup_method manual on the integration. Tests talk to the HTTP boundary: a stateful fake backend typed with the generated API types holds senders, SES state and published DNS records, and a test advances it from pending to records_found to verified or injects a failure per endpoint. The polling schedule and the phase decisions are proven against it. The scene key and the urls.workflowsEmailDomain helper are registered here because the logics use them. The page itself, its route and the steps come in the next layer. Co-Authored-By: Claude Opus 5.5 (1M context) --- frontend/src/products.tsx | 1 + frontend/src/scenes/sceneTypes.ts | 1 + .../__mocks__/fakeEmailDomainBackend.ts | 490 ++++++++++++++++++ .../Channels/EmailDomain/buildSetupPrompt.ts | 79 +++ .../EmailDomain/buildTestEmailWorkflow.ts | 74 +++ .../frontend/Channels/EmailDomain/dnsHosts.ts | 11 + .../Channels/EmailDomain/domainInput.ts | 72 +++ .../EmailDomain/emailDomainAgentLogic.ts | 185 +++++++ .../emailDomainAutoConfigureLogic.ts | 248 +++++++++ .../EmailDomain/emailDomainLogic.test.ts | 75 +++ .../Channels/EmailDomain/emailDomainLogic.ts | 288 ++++++++++ .../EmailDomain/emailDomainLogicProps.ts | 6 + .../emailDomainManualSetupLogic.ts | 111 ++++ .../EmailDomain/emailDomainPageUrl.ts | 4 + .../EmailDomain/emailDomainSenderLogic.ts | 307 +++++++++++ .../emailDomainStatusLogic.test.ts | 172 ++++++ .../EmailDomain/emailDomainStatusLogic.ts | 315 +++++++++++ .../emailDomainWizardLogic.test.ts | 69 +++ .../EmailDomain/emailDomainWizardLogic.ts | 354 +++++++++++++ .../Channels/EmailDomain/inferDomains.ts | 75 +++ products/workflows/manifest.tsx | 1 + 21 files changed, 2938 insertions(+) create mode 100644 products/workflows/frontend/Channels/EmailDomain/__mocks__/fakeEmailDomainBackend.ts create mode 100644 products/workflows/frontend/Channels/EmailDomain/buildSetupPrompt.ts create mode 100644 products/workflows/frontend/Channels/EmailDomain/buildTestEmailWorkflow.ts create mode 100644 products/workflows/frontend/Channels/EmailDomain/dnsHosts.ts create mode 100644 products/workflows/frontend/Channels/EmailDomain/domainInput.ts create mode 100644 products/workflows/frontend/Channels/EmailDomain/emailDomainAgentLogic.ts create mode 100644 products/workflows/frontend/Channels/EmailDomain/emailDomainAutoConfigureLogic.ts create mode 100644 products/workflows/frontend/Channels/EmailDomain/emailDomainLogic.test.ts create mode 100644 products/workflows/frontend/Channels/EmailDomain/emailDomainLogic.ts create mode 100644 products/workflows/frontend/Channels/EmailDomain/emailDomainLogicProps.ts create mode 100644 products/workflows/frontend/Channels/EmailDomain/emailDomainManualSetupLogic.ts create mode 100644 products/workflows/frontend/Channels/EmailDomain/emailDomainPageUrl.ts create mode 100644 products/workflows/frontend/Channels/EmailDomain/emailDomainSenderLogic.ts create mode 100644 products/workflows/frontend/Channels/EmailDomain/emailDomainStatusLogic.test.ts create mode 100644 products/workflows/frontend/Channels/EmailDomain/emailDomainStatusLogic.ts create mode 100644 products/workflows/frontend/Channels/EmailDomain/emailDomainWizardLogic.test.ts create mode 100644 products/workflows/frontend/Channels/EmailDomain/emailDomainWizardLogic.ts create mode 100644 products/workflows/frontend/Channels/EmailDomain/inferDomains.ts diff --git a/frontend/src/products.tsx b/frontend/src/products.tsx index 1e7c0bbb84d0..92f1b4c736f8 100644 --- a/frontend/src/products.tsx +++ b/frontend/src/products.tsx @@ -1842,6 +1842,7 @@ export const productUrls = { workflows: (tab?: WorkflowsSceneTab): string => `/workflows${tab ? `/${tab}` : ''}`, workflow: (id: string, tab: string): string => `/workflows/${id}/${tab}`, workflowNew: (): string => '/workflows/new/workflow', + workflowsEmailDomain: (id: string | number): string => `/workflows/channels/email/${id}`, workflowsLibraryMessage: (id: string): string => `/workflows/library/messages/${id}`, workflowsLibraryTemplate: (id?: string): string => `/workflows/library/templates/${id}`, workflowsLibraryTemplateNew: (): string => '/workflows/library/templates/new', diff --git a/frontend/src/scenes/sceneTypes.ts b/frontend/src/scenes/sceneTypes.ts index 2cb33c91c6dc..ccf540fb17c2 100644 --- a/frontend/src/scenes/sceneTypes.ts +++ b/frontend/src/scenes/sceneTypes.ts @@ -219,6 +219,7 @@ export enum Scene { Broadcasts = 'Broadcasts', Workflows = 'Workflows', Broadcast = 'Broadcast', + WorkflowsEmailDomain = 'WorkflowsEmailDomain', Wizard = 'Wizard', EarlyAccessFeature = 'EarlyAccessFeature', EndpointsScene = 'EndpointsScene', diff --git a/products/workflows/frontend/Channels/EmailDomain/__mocks__/fakeEmailDomainBackend.ts b/products/workflows/frontend/Channels/EmailDomain/__mocks__/fakeEmailDomainBackend.ts new file mode 100644 index 000000000000..bc099fec4c10 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/__mocks__/fakeEmailDomainBackend.ts @@ -0,0 +1,490 @@ +import { MockResolverInfo, Mocks } from '~/mocks/utils' +import { EmailSenderDomainStatus } from '~/types' + +import type { + EmailDomainConnectCheckApi, + EmailDomainSetupRecordKindEnumApi, + EmailDomainStatusApi, + EmailDomainStatusRecordApi, + IntegrationConfigApi, + PaginatedIntegrationConfigListApi, +} from 'products/integrations/frontend/generated/api.schemas' + +import { HogflowTestResult } from '../../../Workflows/hogflows/steps/types' +import { EmailSenderConfig, EmailSetupMethod } from '../emailDomainSenderLogic' + +export type FakeDnsHost = 'cloudflare' | 'route53' | 'unknown' + +/** What SES knows about the domain's identity. */ +export type FakeSesState = 'none' | 'pending' | 'temporary_failure' | 'failed' | 'verified' + +export type FakeEndpoint = + | 'integration_list' + | 'integration_get' + | 'integration_create' + | 'email_status' + | 'email_verify' + | 'email_update' + | 'domain_connect_check' + | 'domain_connect_apply_url' + | 'event_hosts' + | 'test_email' + +export interface FakeFailure { + status: number + body?: Record +} + +export interface FakeRequest { + endpoint: FakeEndpoint + method: string + url: string + body: unknown +} + +type RecordKind = EmailDomainSetupRecordKindEnumApi + +/** A record the domain needs, before DNS or SES say anything about it. */ +export type FakeRecord = Omit + +/** Which of a domain's records to touch: a list of kinds, or a predicate over the record and its position. */ +export type FakeRecordSelection = RecordKind[] | ((record: FakeRecord, index: number) => boolean) + +type RecordTemplate = FakeRecord +type FakeResponse = [status: number, body?: unknown] | object + +interface FakeDomain { + domain: string + mailFromSubdomain: string + ses: FakeSesState + published: Set + preexisting: Set + confirmed: Set +} + +interface Hold { + promise: Promise + release: () => void +} + +const CHECKED_AT = '2026-01-01T00:00:00Z' +const FIRST_SENDER_ID = 42 +const VERIFICATION_TOKEN = 'pSM7OoC2Jl6dNKGRmkc4KZpesve3QP3cUUT2FGZZ1PU=' +const DKIM_SELECTORS = [ + 'v4fyczifqlyachgcw4jqrsdcbbblc236', + 'fxaevqyhjczvbnhst7funhfphvbo4hv6', + 'xzipw445tsam4wdwe5y33mgkutbsyqaj', +] +const SPF_VALUE = 'v=spf1 include:amazonses.com ~all' + +const DNS_HOSTS: Record = { + cloudflare: { + supported: true, + provider_name: 'Cloudflare', + available_providers: [], + dns_host: { + name: 'Cloudflare', + dns_settings_url: 'https://dash.cloudflare.com/?to=/:account/:zone/dns/records', + }, + existing_email_tools: [], + }, + route53: { + supported: false, + provider_name: null, + available_providers: [], + dns_host: { name: 'Route 53', dns_settings_url: 'https://console.aws.amazon.com/route53/v2/hostedzones' }, + existing_email_tools: ['Customer.io'], + }, + unknown: { + supported: false, + provider_name: null, + available_providers: [], + dns_host: null, + existing_email_tools: [], + }, +} + +const FAKE_USER: IntegrationConfigApi['created_by'] = { + id: 1, + uuid: '00000000-0000-0000-0000-000000000001', + distinct_id: 'fake-user', + first_name: 'Jane', + email: 'jane@example.com', + hedgehog_config: null, +} + +const recordKey = (record: Pick): string => + `${record.type} ${record.hostname}` + +const recordTemplates = (domain: string, mailFromSubdomain: string): RecordTemplate[] => [ + { kind: 'verification', type: 'TXT', hostname: `_amazonses.${domain}`, value: VERIFICATION_TOKEN, priority: null }, + ...DKIM_SELECTORS.map( + (selector): RecordTemplate => ({ + kind: 'dkim', + type: 'CNAME', + hostname: `${selector}._domainkey.${domain}`, + value: `${selector}.dkim.amazonses.com`, + priority: null, + }) + ), + { kind: 'spf', type: 'TXT', hostname: domain, value: SPF_VALUE, priority: null }, + { + kind: 'mail_from_mx', + type: 'MX', + hostname: `${mailFromSubdomain}.${domain}`, + value: 'feedback-smtp.us-east-1.amazonses.com', + priority: 10, + }, + { + kind: 'mail_from_spf', + type: 'TXT', + hostname: `${mailFromSubdomain}.${domain}`, + value: SPF_VALUE, + priority: null, + }, + { kind: 'dmarc', type: 'TXT', hostname: `_dmarc.${domain}`, value: 'v=DMARC1; p=none;', priority: null }, +] + +const domainOfEmail = (email: string): string => email.split('@')[1] ?? '' + +const readBody = async (request: Request): Promise => { + try { + return await request.clone().json() + } catch { + return null + } +} + +const createHold = (): Hold => { + let release = (): void => {} + const promise = new Promise((resolve) => { + release = resolve + }) + return { promise, release } +} + +/** + * The email sending domain backend as the wizard sees it over HTTP: senders, their SES identity and the DNS + * records published for each domain. A test or story seeds it, registers `mocks()` with MSW and advances the + * domain with `publishRecords` and `verifyDomain`. Every response is typed with the generated API types. + */ +export class FakeEmailDomainBackend { + readonly requests: FakeRequest[] = [] + eventHosts: [host: string, count: number][] = [] + + private senders = new Map() + private domains = new Map() + private dnsHosts = new Map() + private failures = new Map() + private oneOffFailures = new Map() + private holds = new Map() + private nextSenderId = FIRST_SENDER_ID + + addSender(config: Pick & Partial): IntegrationConfigApi { + const domain = config.domain ?? domainOfEmail(config.email) + const sender: IntegrationConfigApi = { + id: this.nextSenderId++, + kind: 'email', + config: { + provider: 'ses', + name: 'Acme', + mail_from_subdomain: 'feedback', + verified: false, + ...config, + domain, + } satisfies EmailSenderConfig, + created_at: CHECKED_AT, + created_by: FAKE_USER, + errors: '', + display_name: config.email, + files_write_requestable: false, + installation_shared: null, + installation_status: null, + } + this.senders.set(sender.id, sender) + if (!this.domains.has(domain)) { + this.domains.set(domain, { + domain, + mailFromSubdomain: config.mail_from_subdomain ?? 'feedback', + ses: 'pending', + published: new Set(), + preexisting: new Set(), + confirmed: new Set(), + }) + } + return sender + } + + sender(id: number): IntegrationConfigApi | undefined { + return this.senders.get(id) + } + + senderConfig(id: number): EmailSenderConfig | undefined { + return this.senders.get(id)?.config as EmailSenderConfig | undefined + } + + /** Which company answers for the domain's nameservers. Matches subdomains of the given domain too. */ + hostDnsAt(domain: string, host: FakeDnsHost): void { + this.dnsHosts.set(domain, host) + } + + /** SES has no identity for the domain yet, so the status has no records to hand out. */ + withoutSesIdentity(domain: string): void { + this.domainState(domain).ses = 'none' + } + + /** Records that existed at the DNS host before setup started, such as a DMARC policy. */ + preexistingRecords(domain: string, kinds: RecordKind[]): void { + kinds.forEach((kind) => this.domainState(domain).preexisting.add(kind)) + } + + /** The person (or Domain Connect) published records at the DNS host. All of them when nothing is selected. */ + publishRecords(domain: string, selection?: FakeRecordSelection): void { + const state = this.domainState(domain) + const selected = + typeof selection === 'function' + ? selection + : (record: FakeRecord): boolean => !selection || selection.includes(record.kind) + recordTemplates(domain, state.mailFromSubdomain) + .filter(selected) + .forEach((record) => state.published.add(recordKey(record))) + } + + /** SES confirmed these parts of the identity, while the domain as a whole is not verified yet. */ + sesConfirms(domain: string, kinds: RecordKind[]): void { + this.publishRecords(domain, kinds) + kinds.forEach((kind) => this.domainState(domain).confirmed.add(kind)) + } + + /** SES confirmed the records and marked every sender on the domain as able to send. */ + verifyDomain(domain: string): void { + this.publishRecords(domain) + this.domainState(domain).ses = 'verified' + this.senders.forEach((sender) => { + const config = sender.config as EmailSenderConfig + if (config.domain === domain) { + sender.config = { ...config, verified: true } + } + }) + } + + sesReports(domain: string, state: Exclude): void { + this.domainState(domain).ses = state + } + + /** What the DNS host does once the person approves the Domain Connect template. */ + approveDomainConnect(domain: string): void { + this.publishRecords(domain) + } + + fail(endpoint: FakeEndpoint, failure: FakeFailure): void { + this.failures.set(endpoint, failure) + } + + failOnce(endpoint: FakeEndpoint, failure: FakeFailure): void { + this.oneOffFailures.set(endpoint, failure) + } + + recover(endpoint: FakeEndpoint): void { + this.failures.delete(endpoint) + this.oneOffFailures.delete(endpoint) + } + + /** Keeps every response from the endpoint pending until the returned function is called. */ + hold(endpoint: FakeEndpoint): () => void { + const hold = createHold() + this.holds.set(endpoint, hold) + return () => { + this.holds.delete(endpoint) + hold.release() + } + } + + requestsTo(endpoint: FakeEndpoint): FakeRequest[] { + return this.requests.filter((request) => request.endpoint === endpoint) + } + + statusOf(domain: string): EmailDomainStatusApi { + const state = this.domainState(domain) + const records = + state.ses === 'none' + ? [] + : recordTemplates(domain, state.mailFromSubdomain).map((record) => ({ + ...record, + status: this.recordStatus(state, record), + })) + const allFound = records.length > 0 && records.every((record) => record.status !== 'pending') + const status = this.overallStatus(state, allFound) + return { + status, + verified: state.ses === 'verified', + checked_at: CHECKED_AT, + steps: [ + { key: 'domain_added', state: 'done' }, + { key: 'records_found', state: allFound ? 'done' : 'pending' }, + { + key: 'verified', + state: state.ses === 'verified' ? 'done' : state.ses === 'failed' ? 'failed' : 'pending', + }, + ], + records, + } + } + + detectionOf(domain: string): EmailDomainConnectCheckApi { + const host = [...this.dnsHosts.entries()].find( + ([registered]) => domain === registered || domain.endsWith(`.${registered}`) + )?.[1] + return DNS_HOSTS[host ?? 'unknown'] + } + + mocks(): Mocks { + return { + get: { + '/api/environments/:team_id/integrations/': this.serve('integration_list', () => this.listSenders()), + '/api/environments/:team_id/integrations/:id/': this.serve('integration_get', ({ params }) => + this.withSender(Number(params.id), (sender) => sender) + ), + '/api/environments/:team_id/integrations/:id/email/status/': this.serve('email_status', ({ params }) => + this.withSender(Number(params.id), (sender) => + this.statusOf((sender.config as EmailSenderConfig).domain) + ) + ), + '/api/environments/:team_id/integrations/domain-connect/check/': this.serve( + 'domain_connect_check', + ({ request }) => this.detectionOf(new URL(request.url).searchParams.get('domain') ?? '') + ), + }, + post: { + '/api/environments/:team_id/integrations/': this.serve('integration_create', (_, body) => + this.createSender(body as { config: EmailSenderConfig }) + ), + '/api/environments/:team_id/integrations/:id/email/verify/': this.serve('email_verify', ({ params }) => + this.withSender(Number(params.id), (sender) => this.restartVerification(sender)) + ), + '/api/environments/:team_id/integrations/domain-connect/apply-url/': this.serve( + 'domain_connect_apply_url', + (_, body) => this.applyUrl(body as { integration_id: number; redirect_uri: string }) + ), + '/api/environments/:team_id/query/': this.serve('event_hosts', () => ({ results: this.eventHosts })), + '/api/environments/:team_id/hog_flows/:id/invocations/': this.serve( + 'test_email', + (): HogflowTestResult => ({ status: 'success', nextActionId: null, logs: [] }) + ), + }, + patch: { + '/api/environments/:team_id/integrations/:id/email/': this.serve('email_update', ({ params }, body) => + this.withSender(Number(params.id), (sender) => + this.updateSender(sender, (body as { config: Partial }).config) + ) + ), + }, + } + } + + private serve( + endpoint: FakeEndpoint, + respond: (info: MockResolverInfo, body: unknown) => FakeResponse + ): (info: MockResolverInfo) => Promise { + return async (info) => { + const body = await readBody(info.request) + this.requests.push({ endpoint, method: info.request.method, url: info.request.url, body }) + await this.holds.get(endpoint)?.promise + const failure = this.failureFor(endpoint) + if (failure) { + return [failure.status, failure.body ?? { detail: 'The fake backend refused this request' }] + } + return respond(info, body) + } + } + + private failureFor(endpoint: FakeEndpoint): FakeFailure | undefined { + const oneOff = this.oneOffFailures.get(endpoint) + if (oneOff) { + this.oneOffFailures.delete(endpoint) + return oneOff + } + return this.failures.get(endpoint) + } + + private domainState(domain: string): FakeDomain { + const state = this.domains.get(domain) + if (!state) { + throw new Error(`The fake backend has no sender for ${domain}. Add one with addSender first.`) + } + return state + } + + private withSender(id: number, respond: (sender: IntegrationConfigApi) => FakeResponse): FakeResponse { + const sender = this.senders.get(id) + return sender ? respond(sender) : [404, { detail: 'Not found.' }] + } + + private recordStatus(state: FakeDomain, record: RecordTemplate): EmailDomainStatusRecordApi['status'] { + if (state.ses === 'verified' || state.confirmed.has(record.kind)) { + return 'verified' + } + const published = state.published.has(recordKey(record)) || state.preexisting.has(record.kind) + if (!published) { + return 'pending' + } + return record.kind === 'dmarc' ? 'verified' : 'found' + } + + private overallStatus(state: FakeDomain, allFound: boolean): EmailDomainStatusApi['status'] { + switch (state.ses) { + case 'none': + return 'not_started' + case 'verified': + return 'verified' + case 'failed': + return 'failed' + case 'temporary_failure': + return 'temporary_failure' + case 'pending': + return allFound ? 'records_found' : 'pending' + } + } + + private listSenders(): PaginatedIntegrationConfigListApi { + const results = [...this.senders.values()] + return { count: results.length, next: null, previous: null, results } + } + + private createSender(body: { config: EmailSenderConfig }): IntegrationConfigApi { + return this.addSender(body.config) + } + + private updateSender(sender: IntegrationConfigApi, config: Partial): IntegrationConfigApi { + const current = sender.config as EmailSenderConfig + sender.config = { + ...current, + name: config.name ?? current.name, + mail_from_subdomain: config.mail_from_subdomain ?? current.mail_from_subdomain, + setup_method: config.setup_method ?? current.setup_method, + } satisfies EmailSenderConfig + return sender + } + + private restartVerification(sender: IntegrationConfigApi): EmailSenderDomainStatus { + const state = this.domainState((sender.config as EmailSenderConfig).domain) + if (state.ses !== 'verified') { + state.ses = 'pending' + } + return { status: 'pending', dnsRecords: [] } + } + + private applyUrl(body: { integration_id: number; redirect_uri: string }): FakeResponse { + const sender = this.senders.get(body.integration_id) + if (!sender) { + return [404, { detail: 'Not found.' }] + } + const method: EmailSetupMethod = 'auto' + this.updateSender(sender, { setup_method: method }) + const domain = (sender.config as EmailSenderConfig).domain + const url = new URL('https://dns.example.com/domain-connect/apply') + url.searchParams.set('domain', domain) + url.searchParams.set('redirect_uri', body.redirect_uri) + return { url: url.toString() } + } +} diff --git a/products/workflows/frontend/Channels/EmailDomain/buildSetupPrompt.ts b/products/workflows/frontend/Channels/EmailDomain/buildSetupPrompt.ts new file mode 100644 index 000000000000..3cda0ff35c94 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/buildSetupPrompt.ts @@ -0,0 +1,79 @@ +import type { + EmailDomainDnsHostApi, + EmailDomainStatusRecordApi, +} from 'products/integrations/frontend/generated/api.schemas' + +export interface SetupPromptInput { + domain: string + mailFromSubdomain: string + projectId: number + integrationId: number + pageUrl: string + dnsHost: EmailDomainDnsHostApi | null + supportsDomainConnect: boolean + records: EmailDomainStatusRecordApi[] + agentUsesBrowser: boolean +} + +const formatRecord = (record: EmailDomainStatusRecordApi): string => { + const priority = record.priority != null ? ` (priority ${record.priority})` : '' + return `- ${record.type} ${record.hostname} -> ${record.value}${priority}` +} + +const buildMcpPrompt = ({ + domain, + mailFromSubdomain, + projectId, + integrationId, + dnsHost, + supportsDomainConnect, + records, +}: SetupPromptInput): string => { + const hostLine = dnsHost + ? `- DNS host detected from the nameservers: ${dnsHost.name}${supportsDomainConnect ? ' (supports Domain Connect)' : ''}` + : '- DNS host: unknown. Ask me where the DNS for this domain is managed before you change anything.' + return [ + `Get the email sending domain ${domain} verified for PostHog Workflows. If you have the setting-up-an-email-domain skill, follow it.`, + '', + 'Facts', + `- PostHog project id: ${projectId}`, + `- Email sender (integration) id: ${integrationId}. It already exists. Do not create it again with integrations-create; re-creating it marks it unverified.`, + `- Domain: ${domain}`, + `- MAIL FROM subdomain: ${mailFromSubdomain}.${domain}. It applies to every sender on this domain. Never pick a new one; changing it needs new DNS records and breaks the other senders until they are published.`, + hostLine, + '', + 'Steps', + `1. Read the current records and status with integrations-email-status-retrieve (project ${projectId}, integration ${integrationId}). If the PostHog MCP is not available, use the records listed below.`, + '2. If the host supports Domain Connect, call integrations-domain-connect-check-retrieve, then integrations-domain-connect-apply-url-create to get an approval URL. Show me that URL and wait for me to approve it in my browser. Do not open it yourself.', + '3. Otherwise add the records at the DNS host yourself. Paste names and values exactly. If the host appends the domain to the name field, enter only the part before the domain. Always publish both SPF TXT records, merging include:amazonses.com into an existing SPF record at the same name instead of adding a second one. Never add a second _dmarc record; keep an existing one.', + '4. Call integrations-email-verify-create once. Then poll integrations-email-status-retrieve every 30 seconds until the status is "verified". Stop after 30 minutes and tell me which records are still missing.', + "5. Only change the sender's display name or MAIL FROM subdomain with integrations-email-partial-update if I ask. Send the sender's current email unchanged.", + '', + 'Records', + ...records.map(formatRecord), + ].join('\n') +} + +const buildBrowserPrompt = ({ domain, dnsHost, records, pageUrl }: SetupPromptInput): string => { + const hostLine = dnsHost + ? `The DNS for this domain is managed at ${dnsHost.name}. Open ${dnsHost.dns_settings_url} in the browser. I am already signed in.` + : 'Ask me where the DNS for this domain is managed, then open that provider in the browser. I am already signed in.' + return [ + `Use the browser to add the DNS records below for ${domain}, then verify the domain in PostHog.`, + '', + hostLine, + '', + 'Rules', + `- Never add a second _dmarc record. If a TXT record already exists at _dmarc.${domain}, keep it and skip ours.`, + '- Do not delete or edit any existing record except to merge include:amazonses.com into an SPF record that already exists at the same name.', + '- If the provider appends the domain to the name field, enter only the part before the domain.', + '- Show me the list of records you added before you leave the DNS page.', + `- Then open ${pageUrl} and click "Check again". Wait until the page says the domain is ready to send. Tell me if any record is still missing after 10 minutes.`, + '', + 'Records', + ...records.map(formatRecord), + ].join('\n') +} + +export const buildSetupPrompt = (input: SetupPromptInput): string => + input.agentUsesBrowser ? buildBrowserPrompt(input) : buildMcpPrompt(input) diff --git a/products/workflows/frontend/Channels/EmailDomain/buildTestEmailWorkflow.ts b/products/workflows/frontend/Channels/EmailDomain/buildTestEmailWorkflow.ts new file mode 100644 index 000000000000..66656a609c28 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/buildTestEmailWorkflow.ts @@ -0,0 +1,74 @@ +import { createExampleEvent } from '../../Workflows/hogflows/testEventFactory' +import type { HogFlow, HogFlowAction, HogFlowEdge } from '../../Workflows/hogflows/types' +import { EXIT_NODE_ID, NEW_WORKFLOW, TRIGGER_NODE_ID } from '../../Workflows/workflowLogic' + +const TEST_EMAIL_ACTION_ID = 'send_test_email' + +export interface TestEmailWorkflowInput { + integrationId: number + teamId: number | null + domain: string + recipient: string +} + +export interface TestEmailInvocation { + configuration: HogFlow + globals: ReturnType + mock_async_functions: false + current_action_id: string +} + +const testEmailAction = (integrationId: number, domain: string, recipient: string): HogFlowAction => { + const text = `Your sending domain ${domain} is set up. Emails from PostHog Workflows will arrive like this one.` + return { + id: TEST_EMAIL_ACTION_ID, + type: 'function_email', + name: 'Send test email', + description: '', + created_at: 0, + updated_at: 0, + config: { + template_id: 'template-email', + inputs: { + email: { + value: { + design: null, + subject: `Test email from ${domain}`, + html: `

${text}

`, + text, + from: { integrationId }, + to: { email: recipient, name: '' }, + cc: '', + bcc: '', + }, + templating: 'liquid', + }, + }, + }, + } +} + +export const buildTestEmailInvocation = ({ + integrationId, + teamId, + domain, + recipient, +}: TestEmailWorkflowInput): TestEmailInvocation => { + const edges: HogFlowEdge[] = [ + { from: TRIGGER_NODE_ID, to: TEST_EMAIL_ACTION_ID, type: 'continue' }, + { from: TEST_EMAIL_ACTION_ID, to: EXIT_NODE_ID, type: 'continue' }, + ] + const configuration: HogFlow = { + ...NEW_WORKFLOW, + team_id: teamId ?? NEW_WORKFLOW.team_id, + name: 'Test email', + actions: [NEW_WORKFLOW.actions[0], testEmailAction(integrationId, domain, recipient), NEW_WORKFLOW.actions[1]], + edges, + } + return { + configuration, + globals: createExampleEvent(teamId ?? undefined, configuration.name, '$pageview', recipient), + mock_async_functions: false, + current_action_id: TEST_EMAIL_ACTION_ID, + } +} diff --git a/products/workflows/frontend/Channels/EmailDomain/dnsHosts.ts b/products/workflows/frontend/Channels/EmailDomain/dnsHosts.ts new file mode 100644 index 000000000000..cc3a770f73d2 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/dnsHosts.ts @@ -0,0 +1,11 @@ +const DNS_HOST_GOTCHAS: Record = { + Cloudflare: 'Keep the proxy off (grey cloud) on the CNAME records. Proxied CNAMEs do not work for DKIM.', + 'Route 53': 'Route 53 asks for the full record name. Paste the name as shown, including your domain.', + Namecheap: + 'Namecheap adds your domain to the Host field. Paste only the part before your domain, so "feedback" and not "feedback.acme.com".', +} + +const GENERIC_GOTCHA = + 'If the name field already ends with your domain, paste only the part before it. Otherwise paste the full name.' + +export const dnsHostGotcha = (hostName: string | null): string => DNS_HOST_GOTCHAS[hostName ?? ''] ?? GENERIC_GOTCHA diff --git a/products/workflows/frontend/Channels/EmailDomain/domainInput.ts b/products/workflows/frontend/Channels/EmailDomain/domainInput.ts new file mode 100644 index 000000000000..e417b3e9bad0 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/domainInput.ts @@ -0,0 +1,72 @@ +const FREE_MAILBOX_DOMAINS = new Set([ + 'gmail.com', + 'googlemail.com', + 'outlook.com', + 'hotmail.com', + 'live.com', + 'yahoo.com', + 'icloud.com', + 'me.com', + 'proton.me', + 'protonmail.com', + 'web.de', +]) + +const DOMAIN_PATTERN = /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)*\.[a-z]{2,}$/ + +export interface NormalizedDomainInput { + domain: string + localPart: string | null +} + +export const normalizeDomainInput = (raw: string): NormalizedDomainInput => { + let value = raw.trim().toLowerCase() + let localPart: string | null = null + if (value.includes('@')) { + const [local, host] = value.split('@') + localPart = local || null + value = host ?? '' + } + value = value.replace(/^[a-z]+:\/\//, '') + value = value.split('/')[0].split('?')[0].split(':')[0] + value = value.replace(/^www\./, '') + return { domain: value, localPart } +} + +export const isFreeMailboxDomain = (domain: string): boolean => + FREE_MAILBOX_DOMAINS.has(domain) || domain.startsWith('gmx.') + +export const isValidDomain = (domain: string): boolean => DOMAIN_PATTERN.test(domain) + +export const rootDomainOf = (hostname: string): string => hostname.split('.').filter(Boolean).slice(-2).join('.') + +export const nameFromDomain = (domain: string): string => { + const label = domain.split('.').filter(Boolean).slice(-2)[0] ?? domain + return label ? label.charAt(0).toUpperCase() + label.slice(1) : '' +} + +export const hostnameOfUrl = (url: string): string | null => { + try { + const hostname = new URL(url.includes('://') ? url : `https://${url}`).hostname.toLowerCase() + return hostname.replace(/^www\./, '') + } catch { + return null + } +} + +export type DomainProblem = 'free_mailbox' | 'invalid' + +export const domainProblemOf = (domain: string): DomainProblem | null => { + if (!domain) { + return null + } + if (isFreeMailboxDomain(domain)) { + return 'free_mailbox' + } + if (!isValidDomain(domain)) { + return 'invalid' + } + return null +} + +export const asSubdomainLabel = (value: string): string => value.toLowerCase().replace(/[^a-z0-9-]/g, '') diff --git a/products/workflows/frontend/Channels/EmailDomain/emailDomainAgentLogic.ts b/products/workflows/frontend/Channels/EmailDomain/emailDomainAgentLogic.ts new file mode 100644 index 000000000000..359dfad5baeb --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/emailDomainAgentLogic.ts @@ -0,0 +1,185 @@ +import { MakeLogicType, actions, connect, kea, key, listeners, path, props, reducers, selectors } from 'kea' +import posthog from 'posthog-js' + +import { FEATURE_FLAGS } from 'lib/constants' +import { lemonToast } from 'lib/lemon-ui/LemonToast' +import { featureFlagLogic } from 'lib/logic/featureFlagLogic' +import type { FeatureFlagsSet } from 'lib/logic/featureFlagLogic' +import { copyToClipboard } from 'lib/utils/copyToClipboard' + +import type { + EmailDomainDnsHostApi, + EmailDomainStatusRecordApi, +} from 'products/integrations/frontend/generated/api.schemas' + +import { buildSetupPrompt } from './buildSetupPrompt' +import { emailDomainAutoConfigureLogic } from './emailDomainAutoConfigureLogic' +import { EmailDomainLogicProps } from './emailDomainLogicProps' +import { emailDomainSenderLogic } from './emailDomainSenderLogic' +import { emailDomainStatusLogic } from './emailDomainStatusLogic' + +// Generated by kea-typegen. Update if you're an agent, ignore if you're human. +export interface emailDomainAgentLogicValues { + dnsHost: EmailDomainDnsHostApi | null // emailDomainAutoConfigureLogic + pageUrl: string | null // emailDomainAutoConfigureLogic + supportsDomainConnect: boolean // emailDomainAutoConfigureLogic + currentTeamId: number | null // emailDomainSenderLogic + domain: string | null // emailDomainSenderLogic + integrationId: number | null // emailDomainSenderLogic + mailFromSubdomain: string // emailDomainSenderLogic + records: EmailDomainStatusRecordApi[] // emailDomainStatusLogic + featureFlags: FeatureFlagsSet // featureFlagLogic + agentModalOpen: boolean + agentSetupEnabled: boolean + agentUsesBrowser: boolean + claudeUrl: string + setupPrompt: string +} + +// Generated by kea-typegen. Update if you're an agent, ignore if you're human. +export interface emailDomainAgentLogicActions { + closeAgentModal: () => { + value: true + } + copyPrompt: () => { + value: true + } + openAgentModal: () => { + value: true + } + promptOpenedInClaude: () => { + value: true + } + setAgentUsesBrowser: (agentUsesBrowser: boolean) => { + agentUsesBrowser: boolean + } +} + +// Generated by kea-typegen. Update if you're an agent, ignore if you're human. +export interface emailDomainAgentLogicMeta { + key: string + __keaTypeGenInternalSelectorTypes: { + agentSetupEnabled: (featureFlags: FeatureFlagsSet) => boolean + setupPrompt: ( + domain: string | null, + mailFromSubdomain: string, + currentTeamId: number | null, + integrationId: any, + pageUrl: string | null, + dnsHost: EmailDomainDnsHostApi | null, + supportsDomainConnect: boolean, + records: EmailDomainStatusRecordApi[], + agentUsesBrowser: boolean + ) => string + claudeUrl: (setupPrompt: string) => string + } +} + +export type emailDomainAgentLogicType = MakeLogicType< + emailDomainAgentLogicValues, + emailDomainAgentLogicActions, + EmailDomainLogicProps, + emailDomainAgentLogicMeta +> + +/** Hands the DNS setup to a coding agent: builds the prompt for the sender and offers it to copy or open in Claude. */ +export const emailDomainAgentLogic = kea([ + path((key) => ['products', 'workflows', 'frontend', 'Channels', 'EmailDomain', 'emailDomainAgentLogic', key]), + props({} as EmailDomainLogicProps), + key(({ id }) => id), + connect((props: EmailDomainLogicProps) => ({ + values: [ + featureFlagLogic, + ['featureFlags'], + emailDomainSenderLogic(props), + ['currentTeamId', 'domain', 'integrationId', 'mailFromSubdomain'], + emailDomainStatusLogic(props), + ['records'], + emailDomainAutoConfigureLogic(props), + ['dnsHost', 'pageUrl', 'supportsDomainConnect'], + ], + })), + actions({ + openAgentModal: true, + closeAgentModal: true, + setAgentUsesBrowser: (agentUsesBrowser: boolean) => ({ agentUsesBrowser }), + copyPrompt: true, + promptOpenedInClaude: true, + }), + reducers({ + agentModalOpen: [ + false, + { + openAgentModal: () => true, + closeAgentModal: () => false, + }, + ], + agentUsesBrowser: [ + false, + { + setAgentUsesBrowser: (_, { agentUsesBrowser }) => agentUsesBrowser, + }, + ], + }), + selectors({ + agentSetupEnabled: [ + (s) => [s.featureFlags], + (featureFlags: FeatureFlagsSet): boolean => + !!featureFlags[FEATURE_FLAGS.WORKFLOWS_EMAIL_DOMAIN_AGENT_SETUP], + ], + setupPrompt: [ + (s) => [ + s.domain, + s.mailFromSubdomain, + s.currentTeamId, + s.integrationId, + s.pageUrl, + s.dnsHost, + s.supportsDomainConnect, + s.records, + s.agentUsesBrowser, + ], + ( + domain: string | null, + mailFromSubdomain: string, + currentTeamId: number | null, + integrationId: number | null, + pageUrl: string | null, + dnsHost: EmailDomainDnsHostApi | null, + supportsDomainConnect: boolean, + records: EmailDomainStatusRecordApi[], + agentUsesBrowser: boolean + ): string => + domain && integrationId && currentTeamId && pageUrl + ? buildSetupPrompt({ + domain, + mailFromSubdomain, + projectId: currentTeamId, + integrationId, + pageUrl, + dnsHost, + supportsDomainConnect, + records, + agentUsesBrowser, + }) + : '', + ], + claudeUrl: [ + (s) => [s.setupPrompt], + (setupPrompt: string): string => `https://claude.ai/new?q=${encodeURIComponent(setupPrompt)}`, + ], + }), + listeners(({ values }) => ({ + copyPrompt: async () => { + if (await copyToClipboard(values.setupPrompt, 'prompt', { silent: true })) { + // pinned: analytics event name + posthog.capture('email domain setup prompt copied', { browser: values.agentUsesBrowser }) + lemonToast.success('Prompt copied. Paste it into your agent.') + } + }, + promptOpenedInClaude: () => { + // pinned: analytics event name + posthog.capture('email domain setup prompt opened in claude', { browser: values.agentUsesBrowser }) + }, + })), +]) diff --git a/products/workflows/frontend/Channels/EmailDomain/emailDomainAutoConfigureLogic.ts b/products/workflows/frontend/Channels/EmailDomain/emailDomainAutoConfigureLogic.ts new file mode 100644 index 000000000000..49f8fe2b0804 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/emailDomainAutoConfigureLogic.ts @@ -0,0 +1,248 @@ +import { MakeLogicType, actions, afterMount, connect, kea, key, listeners, path, props, reducers, selectors } from 'kea' +import { loaders } from 'kea-loaders' +import { router } from 'kea-router' +import posthog from 'posthog-js' + +import api, { ApiError } from 'lib/api' +import { lemonToast } from 'lib/lemon-ui/LemonToast' +import { teamLogic } from 'scenes/teamLogic' + +import type { + EmailDomainConnectCheckApi, + EmailDomainDnsHostApi, +} from 'products/integrations/frontend/generated/api.schemas' + +import { EmailDomainLogicProps } from './emailDomainLogicProps' +import { emailDomainPageUrl } from './emailDomainPageUrl' +import { emailDomainSenderLogic } from './emailDomainSenderLogic' +import { emailDomainStatusLogic } from './emailDomainStatusLogic' + +// pinned: URL search param, the Domain Connect redirect_uri carries it back to this page +export const DOMAIN_CONNECT_RETURN_PARAM = 'domain_connect' + +const UNKNOWN_HOST: EmailDomainConnectCheckApi = { + supported: false, + provider_name: null, + available_providers: [], + dns_host: null, + existing_email_tools: [], +} + +const openBlankTab = (): Window | null => window.open('', '_blank') + +const showInTab = (tab: Window | null, url: string): void => { + if (!tab) { + window.location.assign(url) + return + } + tab.opener = null + tab.location.href = url +} + +const returnedFromDomainConnect = (): boolean => { + const { location, searchParams } = router.values + if (searchParams[DOMAIN_CONNECT_RETURN_PARAM] !== 'email') { + return false + } + const { [DOMAIN_CONNECT_RETURN_PARAM]: _, ...otherParams } = searchParams + router.actions.replace(location.pathname, otherParams, location.hash) + return true +} + +// Generated by kea-typegen. Update if you're an agent, ignore if you're human. +export interface emailDomainAutoConfigureLogicValues { + domain: string | null // emailDomainSenderLogic + integrationId: number | null // emailDomainSenderLogic + currentTeamId: number | null // teamLogic + autoConfigureLoading: boolean + detection: EmailDomainConnectCheckApi | null + detectionLoading: boolean + dnsHost: EmailDomainDnsHostApi | null + hostName: string | null + pageUrl: string | null + returnedFromAutoConfigure: boolean + supportsDomainConnect: boolean +} + +// Generated by kea-typegen. Update if you're an agent, ignore if you're human. +export interface emailDomainAutoConfigureLogicActions { + loadSenderSuccess: ( + sender: null | import('products/integrations/frontend/generated/api.schemas').IntegrationConfigApi, + payload?: any + ) => { + payload?: any + sender: null | import('products/integrations/frontend/generated/api.schemas').IntegrationConfigApi + } // emailDomainSenderLogic + recheck: () => { + value: true + } // emailDomainStatusLogic + loadDetection: () => any + loadDetectionFailure: ( + error: string, + errorObject?: any + ) => { + error: string + errorObject?: any + } + loadDetectionSuccess: ( + detection: EmailDomainConnectCheckApi | null, + payload?: any + ) => { + detection: EmailDomainConnectCheckApi | null + payload?: any + } + openAutoConfigure: () => { + value: true + } + returnFromAutoConfigure: () => { + value: true + } + setAutoConfigureLoading: (loading: boolean) => { + loading: boolean + } +} + +// Generated by kea-typegen. Update if you're an agent, ignore if you're human. +export interface emailDomainAutoConfigureLogicMeta { + key: string + __keaTypeGenInternalSelectorTypes: { + dnsHost: (detection: EmailDomainConnectCheckApi | null) => EmailDomainDnsHostApi | null + supportsDomainConnect: (detection: EmailDomainConnectCheckApi | null) => boolean + hostName: (detection: EmailDomainConnectCheckApi | null, dnsHost: EmailDomainDnsHostApi | null) => string | null + pageUrl: (currentTeamId: number | null, integrationId: any) => string | null + } +} + +export type emailDomainAutoConfigureLogicType = MakeLogicType< + emailDomainAutoConfigureLogicValues, + emailDomainAutoConfigureLogicActions, + EmailDomainLogicProps, + emailDomainAutoConfigureLogicMeta +> + +/** Finds who hosts the domain's DNS and, when that host supports Domain Connect, sends the person there to add the records in one click. */ +export const emailDomainAutoConfigureLogic = kea([ + path((key) => [ + 'products', + 'workflows', + 'frontend', + 'Channels', + 'EmailDomain', + 'emailDomainAutoConfigureLogic', + key, + ]), + props({} as EmailDomainLogicProps), + key(({ id }) => id), + connect((props: EmailDomainLogicProps) => ({ + values: [teamLogic, ['currentTeamId'], emailDomainSenderLogic(props), ['domain', 'integrationId']], + actions: [emailDomainSenderLogic(props), ['loadSenderSuccess'], emailDomainStatusLogic(props), ['recheck']], + })), + actions({ + openAutoConfigure: true, + setAutoConfigureLoading: (loading: boolean) => ({ loading }), + returnFromAutoConfigure: true, + }), + loaders(({ values }) => ({ + detection: [ + null as EmailDomainConnectCheckApi | null, + { + loadDetection: async (): Promise => { + if (!values.domain) { + return null + } + try { + // nosemgrep: prefer-codegen-api-namespaced-integrations -- the generated integrationsDomainConnectCheckRetrieve has no domain query parameter + return await api.integrations.domainConnectCheck(values.domain) + } catch { + return UNKNOWN_HOST + } + }, + }, + ], + })), + reducers({ + autoConfigureLoading: [ + false, + { + setAutoConfigureLoading: (_, { loading }) => loading, + }, + ], + returnedFromAutoConfigure: [ + false, + { + returnFromAutoConfigure: () => true, + }, + ], + }), + selectors({ + dnsHost: [ + (s) => [s.detection], + (detection: EmailDomainConnectCheckApi | null): EmailDomainDnsHostApi | null => detection?.dns_host ?? null, + ], + supportsDomainConnect: [ + (s) => [s.detection], + (detection: EmailDomainConnectCheckApi | null): boolean => + detection?.supported === true && !!detection.provider_name, + ], + hostName: [ + (s) => [s.detection, s.dnsHost], + (detection: EmailDomainConnectCheckApi | null, dnsHost: EmailDomainDnsHostApi | null): string | null => + dnsHost?.name ?? detection?.provider_name ?? null, + ], + pageUrl: [ + (s) => [s.currentTeamId, s.integrationId], + (currentTeamId: number | null, integrationId: number | null): string | null => + currentTeamId && integrationId ? emailDomainPageUrl(currentTeamId, integrationId) : null, + ], + }), + listeners(({ actions, values, cache }) => ({ + loadSenderSuccess: ({ sender }) => { + if (sender && !values.detection) { + actions.loadDetection() + } + }, + loadDetectionSuccess: () => { + if (values.returnedFromAutoConfigure && !cache.returnReported) { + cache.returnReported = true + // pinned: analytics event name + posthog.capture('email domain auto configure returned', { provider: values.hostName }) + } + }, + returnFromAutoConfigure: () => { + actions.recheck() + }, + openAutoConfigure: async () => { + const { integrationId, pageUrl } = values + if (!integrationId || !pageUrl) { + return + } + // pinned: analytics event name + posthog.capture('email domain auto configure clicked', { provider: values.hostName }) + // Opened before the request so the click's user activation still covers it; a later open is a popup. + const tab = openBlankTab() + actions.setAutoConfigureLoading(true) + try { + // nosemgrep: prefer-codegen-api-namespaced-integrations -- the generated integrationsDomainConnectApplyUrlCreate returns void, and this reads the apply URL + const { url } = await api.integrations.domainConnectApplyUrl({ + context: 'email', + integration_id: integrationId, + redirect_uri: `${pageUrl}?${DOMAIN_CONNECT_RETURN_PARAM}=email`, + }) + showInTab(tab, url) + } catch (error) { + tab?.close() + const detail = error instanceof ApiError ? error.detail : null + lemonToast.error( + detail || `Could not open ${values.hostName ?? 'your DNS host'}. Add the settings yourself instead.` + ) + } finally { + actions.setAutoConfigureLoading(false) + } + }, + })), + afterMount(({ actions, values }) => { + if (values.integrationId !== null && returnedFromDomainConnect()) { + actions.returnFromAutoConfigure() + } + }), +]) diff --git a/products/workflows/frontend/Channels/EmailDomain/emailDomainLogic.test.ts b/products/workflows/frontend/Channels/EmailDomain/emailDomainLogic.test.ts new file mode 100644 index 000000000000..011edf980974 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/emailDomainLogic.test.ts @@ -0,0 +1,75 @@ +import { resumeKeaLoadersErrors, silenceKeaLoadersErrors } from '~/initKea' +import { useMocks } from '~/mocks/jest' +import { initKeaTests } from '~/test/init' + +import type { EmailDomainSetupRecordKindEnumApi } from 'products/integrations/frontend/generated/api.schemas' + +import { FakeEmailDomainBackend } from './__mocks__/fakeEmailDomainBackend' +import { emailDomainLogic } from './emailDomainLogic' + +const DOMAIN = 'mail.acme.com' + +describe('emailDomainLogic', () => { + let backend: FakeEmailDomainBackend + let logic: ReturnType + + beforeEach(() => { + jest.useFakeTimers() + backend = new FakeEmailDomainBackend() + backend.hostDnsAt('acme.com', 'cloudflare') + backend.addSender({ email: `hello@${DOMAIN}` }) + useMocks(backend.mocks()) + initKeaTests() + }) + + afterEach(() => { + logic?.unmount() + jest.useRealTimers() + }) + + const mountAndLoad = async (id = '42'): Promise => { + logic = emailDomainLogic({ id }) + logic.mount() + await jest.advanceTimersByTimeAsync(0) + } + + it('shows the domain step for a new sender', async () => { + await mountAndLoad('new') + expect(logic.values.phase).toBe('domain') + expect(backend.requestsTo('integration_get')).toHaveLength(0) + expect(backend.requestsTo('email_status')).toHaveLength(0) + }) + + it.each<[string, EmailDomainSetupRecordKindEnumApi[], string]>([ + ['an existing DMARC record', ['dmarc'], 'settings'], + ['an existing apex SPF record', ['spf', 'dmarc'], 'settings'], + ['a DKIM record', ['dkim', 'dmarc'], 'verifying'], + ['the verification record', ['verification'], 'verifying'], + ])('shows the %s phase when the first check finds %s', async (_, kinds, phase) => { + backend.preexistingRecords(DOMAIN, kinds) + await mountAndLoad() + expect(logic.values.phase).toBe(phase) + }) + + it('shows the ready phase once the domain is verified', async () => { + backend.verifyDomain(DOMAIN) + await mountAndLoad() + expect(logic.values.phase).toBe('ready') + expect(logic.values.verificationSteps.map((step) => step.state)).toEqual(['done', 'done', 'done']) + }) + + it('reloads only what failed when the person tries again', async () => { + silenceKeaLoadersErrors() + backend.failOnce('integration_get', { status: 500, body: { detail: 'Could not reach the email provider' } }) + await mountAndLoad() + resumeKeaLoadersErrors() + expect(logic.values.phase).toBe('unavailable') + const statusChecks = backend.requestsTo('email_status').length + + logic.actions.retry() + await jest.advanceTimersByTimeAsync(0) + expect(logic.values.phase).toBe('settings') + expect(backend.requestsTo('integration_get')).toHaveLength(2) + expect(backend.requestsTo('email_status')).toHaveLength(statusChecks) + }) +}) diff --git a/products/workflows/frontend/Channels/EmailDomain/emailDomainLogic.ts b/products/workflows/frontend/Channels/EmailDomain/emailDomainLogic.ts new file mode 100644 index 000000000000..d6ed82b1e9d2 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/emailDomainLogic.ts @@ -0,0 +1,288 @@ +import { MakeLogicType, actions, connect, kea, key, listeners, path, props, selectors } from 'kea' + +import { Scene } from 'scenes/sceneTypes' +import { urls } from 'scenes/urls' + +import { Breadcrumb } from '~/types' + +import type { + EmailDomainStatusApi, + EmailDomainStatusRecordApi, + IntegrationConfigApi, +} from 'products/integrations/frontend/generated/api.schemas' + +import { emailDomainAutoConfigureLogic } from './emailDomainAutoConfigureLogic' +import { EmailDomainLogicProps } from './emailDomainLogicProps' +import { emailDomainManualSetupLogic } from './emailDomainManualSetupLogic' +import { emailDomainSenderLogic } from './emailDomainSenderLogic' +import { emailDomainStatusLogic } from './emailDomainStatusLogic' + +export type EmailDomainPhase = 'domain' | 'loading' | 'unavailable' | 'settings' | 'verifying' | 'ready' + +export interface VerificationStep { + key: 'domain_added' | 'records_found' | 'verified' + label: string + detail: string + state: 'done' | 'active' | 'todo' | 'stuck' +} + +interface PhaseInput { + integrationId: number | null + sender: IntegrationConfigApi | null + senderFailed: boolean + status: EmailDomainStatusApi | null + statusUnavailable: boolean + recordsAdded: boolean + returnedFromAutoConfigure: boolean + setupEvidenceFound: boolean +} + +const STATUSES_IN_PROGRESS: EmailDomainStatusApi['status'][] = ['records_found', 'temporary_failure', 'failed'] + +const phaseOf = ({ + integrationId, + sender, + senderFailed, + status, + statusUnavailable, + recordsAdded, + returnedFromAutoConfigure, + setupEvidenceFound, +}: PhaseInput): EmailDomainPhase => { + if (integrationId === null) { + return 'domain' + } + if (senderFailed || (!status && statusUnavailable)) { + return 'unavailable' + } + if (!sender || !status) { + return 'loading' + } + if (status.verified) { + return 'ready' + } + const inProgress = recordsAdded || returnedFromAutoConfigure || STATUSES_IN_PROGRESS.includes(status.status) + return inProgress || setupEvidenceFound ? 'verifying' : 'settings' +} + +interface StepsInput { + status: EmailDomainStatusApi | null + phase: EmailDomainPhase + domain: string | null + records: EmailDomainStatusRecordApi[] + foundCount: number + pollingStopped: boolean +} + +const verificationStepsOf = ({ + status, + phase, + domain, + records, + foundCount, + pollingStopped, +}: StepsInput): VerificationStep[] => { + const verified = phase === 'ready' + const verifying = phase === 'verifying' + const allFound = records.length > 0 && foundCount === records.length + const sesFailed = status?.status === 'failed' + const stuck = pollingStopped || status?.status === 'temporary_failure' + const foundState: VerificationStep['state'] = verified + ? 'done' + : stuck + ? 'stuck' + : allFound + ? 'done' + : verifying + ? 'active' + : 'todo' + const verifiedState: VerificationStep['state'] = verified + ? 'done' + : sesFailed + ? 'stuck' + : allFound + ? 'active' + : 'todo' + const foundDetail = + verifying || verified + ? `${foundCount} of ${records.length} found` + : records.length + ? `${records.length} settings to add` + : 'Nothing to add yet' + const verifiedDetail = verified + ? 'Inboxes trust your emails' + : sesFailed + ? 'Verification stopped. Restart it below.' + : allFound + ? 'Records look right, waiting for confirmation' + : 'Takes about a minute after the settings are found' + return [ + { key: 'domain_added', label: 'Domain added', detail: domain ?? '', state: 'done' }, + { key: 'records_found', label: 'Settings found at your DNS host', detail: foundDetail, state: foundState }, + { key: 'verified', label: 'Ready to send', detail: verifiedDetail, state: verifiedState }, + ] +} + +// Generated by kea-typegen. Update if you're an agent, ignore if you're human. +export interface emailDomainLogicValues { + returnedFromAutoConfigure: boolean // emailDomainAutoConfigureLogic + recordsAdded: boolean // emailDomainManualSetupLogic + domain: string | null // emailDomainSenderLogic + integrationId: number | null // emailDomainSenderLogic + sender: IntegrationConfigApi | null // emailDomainSenderLogic + senderFailed: boolean // emailDomainSenderLogic + senderLoading: boolean // emailDomainSenderLogic + foundCount: number // emailDomainStatusLogic + pollingStopped: boolean // emailDomainStatusLogic + records: EmailDomainStatusRecordApi[] // emailDomainStatusLogic + setupEvidenceFound: boolean // emailDomainStatusLogic + status: EmailDomainStatusApi | null // emailDomainStatusLogic + statusLoading: boolean // emailDomainStatusLogic + statusUnavailable: boolean // emailDomainStatusLogic + breadcrumbs: Breadcrumb[] + phase: EmailDomainPhase + verificationSteps: VerificationStep[] +} + +// Generated by kea-typegen. Update if you're an agent, ignore if you're human. +export interface emailDomainLogicActions { + loadSender: () => any // emailDomainSenderLogic + recheck: () => { + value: true + } // emailDomainStatusLogic + retry: () => { + value: true + } +} + +// Generated by kea-typegen. Update if you're an agent, ignore if you're human. +export interface emailDomainLogicMeta { + key: string + __keaTypeGenInternalSelectorTypes: { + phase: ( + integrationId: any, + sender: IntegrationConfigApi | null, + senderFailed: boolean, + status: EmailDomainStatusApi | null, + statusUnavailable: boolean, + recordsAdded: boolean, + returnedFromAutoConfigure: boolean, + setupEvidenceFound: boolean + ) => EmailDomainPhase + verificationSteps: ( + status: EmailDomainStatusApi | null, + phase: EmailDomainPhase, + domain: string | null, + records: EmailDomainStatusRecordApi[], + foundCount: number, + pollingStopped: boolean + ) => VerificationStep[] + breadcrumbs: (domain: string | null, integrationId: any) => Breadcrumb[] + } +} + +export type emailDomainLogicType = MakeLogicType< + emailDomainLogicValues, + emailDomainLogicActions, + EmailDomainLogicProps, + emailDomainLogicMeta +> + +/** The sending domain page: composes the step to show from the sender, its status and the setup path the person took. */ +export const emailDomainLogic = kea([ + path((key) => ['products', 'workflows', 'frontend', 'Channels', 'EmailDomain', 'emailDomainLogic', key]), + props({} as EmailDomainLogicProps), + key(({ id }) => id), + connect((props: EmailDomainLogicProps) => ({ + values: [ + emailDomainSenderLogic(props), + ['domain', 'integrationId', 'sender', 'senderFailed', 'senderLoading'], + emailDomainStatusLogic(props), + [ + 'foundCount', + 'pollingStopped', + 'records', + 'setupEvidenceFound', + 'status', + 'statusUnavailable', + 'statusLoading', + ], + emailDomainManualSetupLogic(props), + ['recordsAdded'], + emailDomainAutoConfigureLogic(props), + ['returnedFromAutoConfigure'], + ], + actions: [emailDomainSenderLogic(props), ['loadSender'], emailDomainStatusLogic(props), ['recheck']], + })), + actions({ + retry: true, + }), + selectors({ + phase: [ + (s) => [ + s.integrationId, + s.sender, + s.senderFailed, + s.status, + s.statusUnavailable, + s.recordsAdded, + s.returnedFromAutoConfigure, + s.setupEvidenceFound, + ], + ( + integrationId: number | null, + sender: IntegrationConfigApi | null, + senderFailed: boolean, + status: EmailDomainStatusApi | null, + statusUnavailable: boolean, + recordsAdded: boolean, + returnedFromAutoConfigure: boolean, + setupEvidenceFound: boolean + ): EmailDomainPhase => + phaseOf({ + integrationId, + sender, + senderFailed, + status, + statusUnavailable, + recordsAdded, + returnedFromAutoConfigure, + setupEvidenceFound, + }), + ], + verificationSteps: [ + (s) => [s.status, s.phase, s.domain, s.records, s.foundCount, s.pollingStopped], + ( + status: EmailDomainStatusApi | null, + phase: EmailDomainPhase, + domain: string | null, + records: EmailDomainStatusRecordApi[], + foundCount: number, + pollingStopped: boolean + ): VerificationStep[] => + verificationStepsOf({ status, phase, domain, records, foundCount, pollingStopped }), + ], + breadcrumbs: [ + (s) => [s.domain, s.integrationId], + (domain: string | null, integrationId: number | null): Breadcrumb[] => [ + { key: Scene.Workflows, name: 'Workflows', path: urls.workflows(), iconType: 'workflows' }, + { key: 'channels', name: 'Channels', path: urls.workflows('channels'), iconType: 'workflows' }, + { + key: [Scene.WorkflowsEmailDomain, integrationId ?? 'new'], + name: domain ?? 'New sending domain', + iconType: 'workflows', + }, + ], + ], + }), + listeners(({ actions, values }) => ({ + retry: () => { + if (values.senderFailed) { + actions.loadSender() + } + if (values.statusUnavailable) { + actions.recheck() + } + }, + })), +]) diff --git a/products/workflows/frontend/Channels/EmailDomain/emailDomainLogicProps.ts b/products/workflows/frontend/Channels/EmailDomain/emailDomainLogicProps.ts new file mode 100644 index 000000000000..8ba249b12d70 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/emailDomainLogicProps.ts @@ -0,0 +1,6 @@ +export interface EmailDomainLogicProps { + /** The integration id, or `new` for the domain step. */ + id: string +} + +export const integrationIdOf = (id: string): number | null => (/^\d+$/.test(id) ? Number(id) : null) diff --git a/products/workflows/frontend/Channels/EmailDomain/emailDomainManualSetupLogic.ts b/products/workflows/frontend/Channels/EmailDomain/emailDomainManualSetupLogic.ts new file mode 100644 index 000000000000..227377c0f7f7 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/emailDomainManualSetupLogic.ts @@ -0,0 +1,111 @@ +import { MakeLogicType, actions, connect, kea, key, listeners, path, props, reducers } from 'kea' +import posthog from 'posthog-js' + +import { emailDomainAutoConfigureLogic } from './emailDomainAutoConfigureLogic' +import { EmailDomainLogicProps } from './emailDomainLogicProps' +import { EmailSetupMethod, emailDomainSenderLogic } from './emailDomainSenderLogic' +import { emailDomainStatusLogic } from './emailDomainStatusLogic' + +// Generated by kea-typegen. Update if you're an agent, ignore if you're human. +export interface emailDomainManualSetupLogicValues { + domain: string | null // emailDomainSenderLogic + manualSetupRecorded: boolean + recordsAdded: boolean + recordsRevealed: boolean +} + +// Generated by kea-typegen. Update if you're an agent, ignore if you're human. +export interface emailDomainManualSetupLogicActions { + openAutoConfigure: () => { + value: true + } // emailDomainAutoConfigureLogic + recordSetupMethod: (method: EmailSetupMethod) => { + method: EmailSetupMethod + } // emailDomainSenderLogic + recheck: () => { + value: true + } // emailDomainStatusLogic + markRecordsAdded: () => { + value: true + } + recordManualSetup: () => { + value: true + } + revealRecords: () => { + value: true + } +} + +// Generated by kea-typegen. Update if you're an agent, ignore if you're human. +export interface emailDomainManualSetupLogicMeta { + key: string +} + +export type emailDomainManualSetupLogicType = MakeLogicType< + emailDomainManualSetupLogicValues, + emailDomainManualSetupLogicActions, + EmailDomainLogicProps, + emailDomainManualSetupLogicMeta +> + +/** The path where the person adds the DNS records by hand: reveals them, marks them added and records that method once. */ +export const emailDomainManualSetupLogic = kea([ + path((key) => ['products', 'workflows', 'frontend', 'Channels', 'EmailDomain', 'emailDomainManualSetupLogic', key]), + props({} as EmailDomainLogicProps), + key(({ id }) => id), + connect((props: EmailDomainLogicProps) => ({ + values: [emailDomainSenderLogic(props), ['domain']], + actions: [ + emailDomainSenderLogic(props), + ['recordSetupMethod'], + emailDomainStatusLogic(props), + ['recheck'], + emailDomainAutoConfigureLogic(props), + ['openAutoConfigure'], + ], + })), + actions({ + revealRecords: true, + markRecordsAdded: true, + recordManualSetup: true, + }), + reducers({ + recordsRevealed: [ + false, + { + revealRecords: () => true, + markRecordsAdded: () => true, + }, + ], + recordsAdded: [ + false, + { + markRecordsAdded: () => true, + }, + ], + // Auto-configure writes `auto` on the backend, so a later manual step must be recorded again. + manualSetupRecorded: [ + false, + { + recordSetupMethod: (_, { method }) => method === 'manual', + openAutoConfigure: () => false, + }, + ], + }), + listeners(({ actions, values }) => ({ + revealRecords: () => { + actions.recordManualSetup() + }, + markRecordsAdded: () => { + // pinned: analytics event name + posthog.capture('email domain records added', { domain: values.domain }) + actions.recheck() + actions.recordManualSetup() + }, + recordManualSetup: () => { + if (!values.manualSetupRecorded) { + actions.recordSetupMethod('manual') + } + }, + })), +]) diff --git a/products/workflows/frontend/Channels/EmailDomain/emailDomainPageUrl.ts b/products/workflows/frontend/Channels/EmailDomain/emailDomainPageUrl.ts new file mode 100644 index 000000000000..54b503485e0a --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/emailDomainPageUrl.ts @@ -0,0 +1,4 @@ +import { urls } from 'scenes/urls' + +export const emailDomainPageUrl = (teamId: number, integrationId: number): string => + `${window.location.origin}/project/${teamId}${urls.workflowsEmailDomain(integrationId)}` diff --git a/products/workflows/frontend/Channels/EmailDomain/emailDomainSenderLogic.ts b/products/workflows/frontend/Channels/EmailDomain/emailDomainSenderLogic.ts new file mode 100644 index 000000000000..103cdf6168fd --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/emailDomainSenderLogic.ts @@ -0,0 +1,307 @@ +import { MakeLogicType, actions, afterMount, connect, kea, key, listeners, path, props, reducers, selectors } from 'kea' +import { loaders } from 'kea-loaders' + +import api from 'lib/api' +import { integrationsLogic } from 'lib/integrations/integrationsLogic' +import { lemonToast } from 'lib/lemon-ui/LemonToast' +import { teamLogic } from 'scenes/teamLogic' +import { userLogic } from 'scenes/userLogic' + +import { UserType } from '~/types' + +import { integrationsEmailPartialUpdate, integrationsRetrieve } from 'products/integrations/frontend/generated/api' +import type { IntegrationConfigApi } from 'products/integrations/frontend/generated/api.schemas' + +import { HogflowTestResult } from '../../Workflows/hogflows/steps/types' +import { buildTestEmailInvocation } from './buildTestEmailWorkflow' +import { EmailDomainLogicProps, integrationIdOf } from './emailDomainLogicProps' + +export type EmailSetupMethod = 'auto' | 'manual' + +export interface EmailSenderConfig { + domain: string + email: string + name: string + provider: string + mail_from_subdomain?: string + verified?: boolean + setup_method?: EmailSetupMethod +} + +const DEFAULT_MAIL_FROM_SUBDOMAIN = 'feedback' + +const configOf = (sender: IntegrationConfigApi | null): EmailSenderConfig | null => + (sender?.config as EmailSenderConfig | undefined) ?? null + +// Generated by kea-typegen. Update if you're an agent, ignore if you're human. +export interface emailDomainSenderLogicValues { + currentTeamId: number | null // teamLogic + user: UserType | null // userLogic + domain: string | null + fromAddress: string | null + integrationId: number | null + mailFromSubdomain: string + sender: IntegrationConfigApi | null + senderConfig: EmailSenderConfig | null + senderFailed: boolean + senderLoading: boolean + senderName: string + senderNameChanged: boolean + senderSaving: boolean + testEmailSent: boolean + testSendResult: HogflowTestResult | null + testSendResultLoading: boolean +} + +// Generated by kea-typegen. Update if you're an agent, ignore if you're human. +export interface emailDomainSenderLogicActions { + loadIntegrations: (_?: any) => any // integrationsLogic + loadSender: () => any + loadSenderFailure: ( + error: string, + errorObject?: any + ) => { + error: string + errorObject?: any + } + loadSenderSuccess: ( + sender: IntegrationConfigApi | null, + payload?: any + ) => { + sender: IntegrationConfigApi | null + payload?: any + } + recordSetupMethod: (method: EmailSetupMethod) => { + method: EmailSetupMethod + } + recordSetupMethodFailure: ( + error: string, + errorObject?: any + ) => { + error: string + errorObject?: any + } + recordSetupMethodSuccess: ( + sender: IntegrationConfigApi | null, + payload?: { + method: EmailSetupMethod + } + ) => { + sender: IntegrationConfigApi | null + payload?: { + method: EmailSetupMethod + } + } + saveSender: () => { + value: true + } + saveSenderFailed: () => { + value: true + } + sendTestEmail: () => any + sendTestEmailFailure: ( + error: string, + errorObject?: any + ) => { + error: string + errorObject?: any + } + sendTestEmailSuccess: ( + testSendResult: HogflowTestResult | null, + payload?: any + ) => { + testSendResult: HogflowTestResult | null + payload?: any + } + senderSaved: (sender: IntegrationConfigApi) => { + sender: IntegrationConfigApi + } + setSenderName: (name: string) => { + name: string + } +} + +// Generated by kea-typegen. Update if you're an agent, ignore if you're human. +export interface emailDomainSenderLogicMeta { + key: string + __keaTypeGenInternalSelectorTypes: { + integrationId: (arg: string) => number | null + senderConfig: (sender: IntegrationConfigApi | null) => EmailSenderConfig | null + domain: (senderConfig: EmailSenderConfig | null) => string | null + fromAddress: (senderConfig: EmailSenderConfig | null) => string | null + mailFromSubdomain: (senderConfig: EmailSenderConfig | null) => string + senderNameChanged: (senderName: string, senderConfig: EmailSenderConfig | null) => boolean + } +} + +export type emailDomainSenderLogicType = MakeLogicType< + emailDomainSenderLogicValues, + emailDomainSenderLogicActions, + EmailDomainLogicProps, + emailDomainSenderLogicMeta +> + +/** The email sender behind the page: loads it, edits its name, records how its DNS was set up, and sends a test email. */ +export const emailDomainSenderLogic = kea([ + path((key) => ['products', 'workflows', 'frontend', 'Channels', 'EmailDomain', 'emailDomainSenderLogic', key]), + props({} as EmailDomainLogicProps), + key(({ id }) => id), + connect(() => ({ + values: [teamLogic, ['currentTeamId'], userLogic, ['user']], + actions: [integrationsLogic, ['loadIntegrations']], + })), + actions({ + setSenderName: (name: string) => ({ name }), + saveSender: true, + senderSaved: (sender: IntegrationConfigApi) => ({ sender }), + saveSenderFailed: true, + recordSetupMethod: (method: EmailSetupMethod) => ({ method }), + }), + loaders(({ values }) => ({ + sender: [ + null as IntegrationConfigApi | null, + { + loadSender: async (): Promise => { + const { integrationId, currentTeamId } = values + if (!integrationId || !currentTeamId) { + return null + } + return await integrationsRetrieve(String(currentTeamId), integrationId) + }, + recordSetupMethod: async ({ method }): Promise => { + const { integrationId, currentTeamId, senderConfig, sender } = values + if (!integrationId || !currentTeamId || !senderConfig) { + return sender + } + try { + return await integrationsEmailPartialUpdate(String(currentTeamId), integrationId, { + kind: 'email', + config: { ...senderConfig, setup_method: method }, + }) + } catch { + return sender + } + }, + }, + ], + testSendResult: [ + null as HogflowTestResult | null, + { + sendTestEmail: async (): Promise => { + const { integrationId, currentTeamId, user, domain } = values + const recipient = user?.email + if (!integrationId || !domain || !recipient) { + return null + } + try { + // nosemgrep: prefer-codegen-api-namespaced-workflows -- the generated hogFlowsInvocationsCreate returns void, and this reads the test result + return await api.hogFlows.createTestInvocation( + 'new', + buildTestEmailInvocation({ integrationId, teamId: currentTeamId, domain, recipient }) + ) + } catch { + lemonToast.error('Could not send the test email. Try again in a moment.') + return null + } + }, + }, + ], + })), + reducers({ + sender: { + senderSaved: (_, { sender }) => sender, + }, + senderSaving: [ + false, + { + saveSender: () => true, + senderSaved: () => false, + saveSenderFailed: () => false, + }, + ], + senderFailed: [ + false, + { + loadSender: () => false, + loadSenderFailure: () => true, + }, + ], + senderName: [ + '', + { + setSenderName: (_, { name }) => name, + loadSenderSuccess: (state, { sender }) => configOf(sender)?.name ?? state, + senderSaved: (state, { sender }) => configOf(sender)?.name ?? state, + }, + ], + testEmailSent: [ + false, + { + sendTestEmailSuccess: (state, { testSendResult }) => testSendResult?.status === 'success' || state, + }, + ], + }), + selectors({ + integrationId: [ + () => [(_, props: EmailDomainLogicProps) => props.id], + (id: string): number | null => integrationIdOf(id), + ], + senderConfig: [ + (s) => [s.sender], + (sender: IntegrationConfigApi | null): EmailSenderConfig | null => configOf(sender), + ], + domain: [ + (s) => [s.senderConfig], + (senderConfig: EmailSenderConfig | null): string | null => senderConfig?.domain ?? null, + ], + fromAddress: [ + (s) => [s.senderConfig], + (senderConfig: EmailSenderConfig | null): string | null => senderConfig?.email ?? null, + ], + mailFromSubdomain: [ + (s) => [s.senderConfig], + (senderConfig: EmailSenderConfig | null): string => + senderConfig?.mail_from_subdomain || DEFAULT_MAIL_FROM_SUBDOMAIN, + ], + senderNameChanged: [ + (s) => [s.senderName, s.senderConfig], + (senderName: string, senderConfig: EmailSenderConfig | null): boolean => + senderName !== (senderConfig?.name ?? ''), + ], + }), + listeners(({ actions, values }) => ({ + saveSender: async () => { + const { integrationId, currentTeamId, senderConfig } = values + if (!integrationId || !currentTeamId || !senderConfig) { + actions.saveSenderFailed() + return + } + try { + const saved = await integrationsEmailPartialUpdate(String(currentTeamId), integrationId, { + kind: 'email', + config: { ...senderConfig, name: values.senderName }, + }) + actions.senderSaved(saved) + } catch { + actions.saveSenderFailed() + lemonToast.error('Could not save the sender name. Try again in a moment.') + } + }, + senderSaved: () => { + actions.loadIntegrations() + lemonToast.success('Sender name saved') + }, + sendTestEmailSuccess: ({ testSendResult }) => { + if (testSendResult?.status === 'success') { + lemonToast.success(`Test email sent to ${values.user?.email}. Check your inbox.`) + } else if (testSendResult) { + lemonToast.error('Could not send the test email. Try again in a moment.') + } + }, + })), + afterMount(({ actions, values }) => { + if (values.integrationId !== null) { + actions.loadSender() + } + }), +]) diff --git a/products/workflows/frontend/Channels/EmailDomain/emailDomainStatusLogic.test.ts b/products/workflows/frontend/Channels/EmailDomain/emailDomainStatusLogic.test.ts new file mode 100644 index 000000000000..fc43e491526a --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/emailDomainStatusLogic.test.ts @@ -0,0 +1,172 @@ +import { useMocks } from '~/mocks/jest' +import { initKeaTests } from '~/test/init' + +import { FakeEmailDomainBackend } from './__mocks__/fakeEmailDomainBackend' +import { emailDomainStatusLogic } from './emailDomainStatusLogic' + +const DOMAIN = 'mail.acme.com' +const MINUTE = 60_000 + +const setDocumentHidden = (hidden: boolean): void => { + Object.defineProperty(document, 'hidden', { configurable: true, get: () => hidden }) + document.dispatchEvent(new Event('visibilitychange')) +} + +describe('emailDomainStatusLogic', () => { + let backend: FakeEmailDomainBackend + let logic: ReturnType + + const statusChecks = (): number => backend.requestsTo('email_status').length + + beforeEach(() => { + jest.useFakeTimers() + backend = new FakeEmailDomainBackend() + backend.hostDnsAt('acme.com', 'cloudflare') + backend.addSender({ email: `hello@${DOMAIN}` }) + useMocks(backend.mocks()) + initKeaTests() + }) + + afterEach(() => { + logic?.unmount() + setDocumentHidden(false) + jest.useRealTimers() + }) + + const mountAndLoad = async (): Promise => { + logic = emailDomainStatusLogic({ id: '42' }) + logic.mount() + await jest.advanceTimersByTimeAsync(0) + } + + it('polls every 10 seconds for 5 minutes, then every 30 seconds, and stops after 30 minutes', async () => { + await mountAndLoad() + expect(statusChecks()).toBe(1) + + await jest.advanceTimersByTimeAsync(10_000) + expect(statusChecks()).toBe(2) + + await jest.advanceTimersByTimeAsync(5 * MINUTE - 10_000) + expect(statusChecks()).toBe(31) + + await jest.advanceTimersByTimeAsync(10_000) + expect(statusChecks()).toBe(31) + await jest.advanceTimersByTimeAsync(20_000) + expect(statusChecks()).toBe(32) + + await jest.advanceTimersByTimeAsync(25 * MINUTE) + expect(statusChecks()).toBe(81) + expect(logic.values.pollingStopped).toBe(true) + + await jest.advanceTimersByTimeAsync(5 * MINUTE) + expect(statusChecks()).toBe(81) + }) + + it('pauses polling while the tab is hidden and resumes when it is shown', async () => { + await mountAndLoad() + expect(statusChecks()).toBe(1) + + setDocumentHidden(true) + await jest.advanceTimersByTimeAsync(MINUTE) + expect(statusChecks()).toBe(1) + + setDocumentHidden(false) + await jest.advanceTimersByTimeAsync(10_000) + expect(statusChecks()).toBe(2) + }) + + it('restarts the schedule with an uncached check when the person checks again after polling stopped', async () => { + await mountAndLoad() + await jest.advanceTimersByTimeAsync(31 * MINUTE) + expect(logic.values.pollingStopped).toBe(true) + const checksBefore = statusChecks() + + logic.actions.checkAgain() + await jest.advanceTimersByTimeAsync(0) + expect(backend.requestsTo('email_status').at(-1)?.url).toContain('refresh=true') + expect(logic.values.pollingStopped).toBe(false) + + await jest.advanceTimersByTimeAsync(10_000) + expect(statusChecks()).toBe(checksBefore + 2) + }) + + it.each([ + [500, true], + [400, false], + [403, false], + [404, false], + ])('marks the status failed when a check answers %s and keeps polling: %s', async (httpStatus, keepsPolling) => { + backend.fail('email_status', { status: httpStatus, body: { detail: 'SES read failed' } }) + await mountAndLoad() + expect(logic.values.statusUnavailable).toBe(true) + expect(logic.values.status).toBeNull() + + await jest.advanceTimersByTimeAsync(10_000) + expect(statusChecks()).toBe(keepsPolling ? 2 : 1) + + backend.recover('email_status') + logic.actions.recheck() + await jest.advanceTimersByTimeAsync(0) + expect(logic.values.statusUnavailable).toBe(false) + expect(logic.values.status?.status).toBe('pending') + }) + + it('keeps the last status and the progress, and keeps polling, when a later check fails', async () => { + backend.publishRecords(DOMAIN) + await mountAndLoad() + expect(logic.values.status?.status).toBe('records_found') + + backend.fail('email_status', { status: 503, body: { code: 'ses_unavailable', detail: 'SES is down' } }) + await jest.advanceTimersByTimeAsync(10_000) + expect(logic.values.statusUnavailable).toBe(false) + expect(logic.values.status?.status).toBe('records_found') + expect(logic.values.foundCount).toBe(logic.values.records.length) + + backend.recover('email_status') + backend.verifyDomain(DOMAIN) + await jest.advanceTimersByTimeAsync(10_000) + expect(statusChecks()).toBe(3) + expect(logic.values.status?.verified).toBe(true) + }) + + it('stops polling once the domain is verified and refreshes the channel list', async () => { + backend.verifyDomain(DOMAIN) + await mountAndLoad() + expect(logic.values.status?.verified).toBe(true) + expect(backend.requestsTo('integration_list').length).toBeGreaterThan(0) + + await jest.advanceTimersByTimeAsync(MINUTE) + expect(statusChecks()).toBe(1) + }) + + it('rechecks right after verification restarts, and stays usable when the restart fails', async () => { + backend.sesReports(DOMAIN, 'failed') + await mountAndLoad() + expect(logic.values.status?.status).toBe('failed') + + backend.failOnce('email_verify', { status: 500, body: { detail: 'SES is down' } }) + logic.actions.restartVerification() + await jest.advanceTimersByTimeAsync(0) + expect(logic.values.restartResultLoading).toBe(false) + expect(logic.values.status?.status).toBe('failed') + + logic.actions.restartVerification() + await jest.advanceTimersByTimeAsync(0) + expect(backend.requestsTo('email_verify')).toHaveLength(2) + expect(logic.values.status?.status).toBe('pending') + }) + + it('keeps checking after a restart when the first check finds the provider unavailable', async () => { + backend.sesReports(DOMAIN, 'failed') + await mountAndLoad() + expect(logic.values.status?.status).toBe('failed') + + backend.failOnce('email_status', { status: 503, body: { code: 'email_provider_unavailable' } }) + logic.actions.restartVerification() + await jest.advanceTimersByTimeAsync(0) + expect(logic.values.status?.status).toBe('failed') + + await jest.advanceTimersByTimeAsync(10_000) + expect(logic.values.status?.status).toBe('pending') + }) +}) diff --git a/products/workflows/frontend/Channels/EmailDomain/emailDomainStatusLogic.ts b/products/workflows/frontend/Channels/EmailDomain/emailDomainStatusLogic.ts new file mode 100644 index 000000000000..804ee3b5a771 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/emailDomainStatusLogic.ts @@ -0,0 +1,315 @@ +import { MakeLogicType, actions, afterMount, connect, kea, key, listeners, path, props, reducers, selectors } from 'kea' +import { loaders } from 'kea-loaders' +import posthog from 'posthog-js' + +import api, { ApiError } from 'lib/api' +import { integrationsLogic } from 'lib/integrations/integrationsLogic' +import { lemonToast } from 'lib/lemon-ui/LemonToast' +import { teamLogic } from 'scenes/teamLogic' + +import { integrationsEmailStatusRetrieve } from 'products/integrations/frontend/generated/api' +import type { + EmailDomainSetupRecordKindEnumApi, + EmailDomainStatusApi, + EmailDomainStatusRecordApi, +} from 'products/integrations/frontend/generated/api.schemas' + +import { EmailDomainLogicProps, integrationIdOf } from './emailDomainLogicProps' +import { emailDomainSenderLogic } from './emailDomainSenderLogic' + +export const POLL_FAST_MS = 10_000 +export const POLL_FAST_WINDOW_MS = 5 * 60_000 +export const POLL_SLOW_MS = 30_000 +export const POLL_MAX_MS = 30 * 60_000 + +const STATUS_ERRORS_THAT_END_POLLING = [400, 403, 404] +// A DMARC or apex SPF record often exists before the person adds anything, so finding one proves nothing about setup. +const RECORD_KINDS_THAT_PREDATE_SETUP: EmailDomainSetupRecordKindEnumApi[] = ['dmarc', 'spf'] + +const isRecordFound = (record: EmailDomainStatusRecordApi): boolean => record.status !== 'pending' +const isSetupEvidence = (record: EmailDomainStatusRecordApi): boolean => + isRecordFound(record) && !RECORD_KINDS_THAT_PREDATE_SETUP.includes(record.kind) + +// Generated by kea-typegen. Update if you're an agent, ignore if you're human. +export interface emailDomainStatusLogicValues { + domain: string | null // emailDomainSenderLogic + currentTeamId: number | null // teamLogic + foundCount: number + integrationId: number | null + lastCheckFailed: boolean + missingRecords: EmailDomainStatusRecordApi[] + pollingStartedAt: number | null + pollingStopped: boolean + records: EmailDomainStatusRecordApi[] + restartResult: boolean | null + restartResultLoading: boolean + setupEvidenceFound: boolean + status: EmailDomainStatusApi | null + statusLoading: boolean + statusUnavailable: boolean +} + +// Generated by kea-typegen. Update if you're an agent, ignore if you're human. +export interface emailDomainStatusLogicActions { + loadIntegrations: (_?: any) => any // integrationsLogic + checkAgain: () => { + value: true + } + loadStatus: (refresh?: boolean) => { + refresh: boolean + } + loadStatusFailure: ( + error: string, + errorObject?: any + ) => { + error: string + errorObject?: any + } + loadStatusSuccess: ( + status: EmailDomainStatusApi | null, + payload?: { + refresh: boolean + } + ) => { + status: EmailDomainStatusApi | null + payload?: { + refresh: boolean + } + } + recheck: () => { + value: true + } + restartVerification: () => any + restartVerificationFailure: ( + error: string, + errorObject?: any + ) => { + error: string + errorObject?: any + } + restartVerificationSuccess: ( + restartResult: boolean | null, + payload?: any + ) => { + restartResult: boolean | null + payload?: any + } + schedulePoll: () => { + value: true + } + startPolling: () => { + value: true + } + statusCheckFailed: (httpStatus: number | null) => { + httpStatus: number | null + } + stopPolling: () => { + value: true + } +} + +// Generated by kea-typegen. Update if you're an agent, ignore if you're human. +export interface emailDomainStatusLogicMeta { + key: string + __keaTypeGenInternalSelectorTypes: { + integrationId: (arg: string) => number | null + records: (status: EmailDomainStatusApi | null) => EmailDomainStatusRecordApi[] + foundCount: (records: EmailDomainStatusRecordApi[]) => number + setupEvidenceFound: (records: EmailDomainStatusRecordApi[]) => boolean + missingRecords: ( + records: EmailDomainStatusRecordApi[], + pollingStopped: boolean, + status: EmailDomainStatusApi | null + ) => EmailDomainStatusRecordApi[] + } +} + +export type emailDomainStatusLogicType = MakeLogicType< + emailDomainStatusLogicValues, + emailDomainStatusLogicActions, + EmailDomainLogicProps, + emailDomainStatusLogicMeta +> + +/** Reads the domain's setup status and keeps polling it while the page is open and the domain is not verified. */ +export const emailDomainStatusLogic = kea([ + path((key) => ['products', 'workflows', 'frontend', 'Channels', 'EmailDomain', 'emailDomainStatusLogic', key]), + props({} as EmailDomainLogicProps), + key(({ id }) => id), + connect((props: EmailDomainLogicProps) => ({ + values: [teamLogic, ['currentTeamId'], emailDomainSenderLogic(props), ['domain']], + actions: [integrationsLogic, ['loadIntegrations']], + })), + actions({ + loadStatus: (refresh: boolean = false) => ({ refresh }), + statusCheckFailed: (httpStatus: number | null) => ({ httpStatus }), + startPolling: true, + schedulePoll: true, + stopPolling: true, + recheck: true, + checkAgain: true, + }), + loaders(({ values, actions }) => ({ + status: [ + null as EmailDomainStatusApi | null, + { + loadStatus: async ({ refresh }): Promise => { + const { integrationId, currentTeamId } = values + if (!integrationId || !currentTeamId) { + return null + } + try { + return await integrationsEmailStatusRetrieve(String(currentTeamId), integrationId, { refresh }) + } catch (error) { + actions.statusCheckFailed(error instanceof ApiError ? (error.status ?? null) : null) + return values.status + } + }, + }, + ], + restartResult: [ + null as boolean | null, + { + restartVerification: async (): Promise => { + if (!values.integrationId) { + return null + } + try { + // nosemgrep: prefer-codegen-api-namespaced-integrations -- the generated integrationsEmailVerifyCreate returns void and demands a config body the endpoint ignores + await api.integrations.verifyEmail(values.integrationId) + return true + } catch { + lemonToast.error('Could not restart verification. Try again in a moment.') + return false + } + }, + }, + ], + })), + reducers({ + lastCheckFailed: [ + false, + { + loadStatus: () => false, + statusCheckFailed: () => true, + }, + ], + statusUnavailable: [ + false, + { + statusCheckFailed: () => true, + loadStatusSuccess: (state, { status }) => (status ? false : state), + }, + ], + pollingStartedAt: [ + null as number | null, + { + startPolling: () => Date.now(), + recheck: () => Date.now(), + }, + ], + pollingStopped: [ + false, + { + stopPolling: () => true, + recheck: () => false, + }, + ], + }), + selectors({ + integrationId: [ + () => [(_, props: EmailDomainLogicProps) => props.id], + (id: string): number | null => integrationIdOf(id), + ], + records: [ + (s) => [s.status], + (status: EmailDomainStatusApi | null): EmailDomainStatusRecordApi[] => status?.records ?? [], + ], + foundCount: [ + (s) => [s.records], + (records: EmailDomainStatusRecordApi[]): number => records.filter(isRecordFound).length, + ], + setupEvidenceFound: [ + (s) => [s.records], + (records: EmailDomainStatusRecordApi[]): boolean => records.some(isSetupEvidence), + ], + missingRecords: [ + (s) => [s.records, s.pollingStopped, s.status], + ( + records: EmailDomainStatusRecordApi[], + pollingStopped: boolean, + status: EmailDomainStatusApi | null + ): EmailDomainStatusRecordApi[] => + pollingStopped || status?.status === 'temporary_failure' + ? records.filter((record) => !isRecordFound(record)) + : [], + ], + }), + listeners(({ actions, values, cache }) => ({ + loadStatusSuccess: ({ status }) => { + if (values.integrationId === null) { + return + } + // A failed check keeps the last status, which must not end polling as if it were fresh. + if (values.lastCheckFailed) { + if (values.pollingStartedAt === null) { + actions.startPolling() + } + actions.schedulePoll() + return + } + if (status?.verified || status?.status === 'failed') { + cache.disposables.dispose('poll') + if (status.verified) { + actions.loadIntegrations() + } + return + } + if (values.pollingStartedAt === null) { + actions.startPolling() + } + actions.schedulePoll() + }, + schedulePoll: () => { + if (values.pollingStopped) { + return + } + const elapsed = Date.now() - (values.pollingStartedAt ?? Date.now()) + if (elapsed >= POLL_MAX_MS) { + actions.stopPolling() + return + } + const delay = elapsed < POLL_FAST_WINDOW_MS ? POLL_FAST_MS : POLL_SLOW_MS + cache.disposables.add(() => { + const timer = window.setTimeout(() => actions.loadStatus(), delay) + return () => clearTimeout(timer) + }, 'poll') + }, + stopPolling: () => { + cache.disposables.dispose('poll') + }, + statusCheckFailed: ({ httpStatus }) => { + if (httpStatus && STATUS_ERRORS_THAT_END_POLLING.includes(httpStatus)) { + actions.stopPolling() + } + }, + recheck: () => { + actions.loadStatus(true) + }, + checkAgain: () => { + // pinned: analytics event name + posthog.capture('email domain check again clicked', { domain: values.domain }) + actions.recheck() + }, + restartVerificationSuccess: ({ restartResult }) => { + if (restartResult) { + actions.recheck() + } + }, + })), + afterMount(({ actions, values }) => { + if (values.integrationId !== null) { + actions.loadStatus() + } + }), +]) diff --git a/products/workflows/frontend/Channels/EmailDomain/emailDomainWizardLogic.test.ts b/products/workflows/frontend/Channels/EmailDomain/emailDomainWizardLogic.test.ts new file mode 100644 index 000000000000..758d6f804074 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/emailDomainWizardLogic.test.ts @@ -0,0 +1,69 @@ +import { useMocks } from '~/mocks/jest' +import { initKeaTests } from '~/test/init' + +import { FakeEmailDomainBackend } from './__mocks__/fakeEmailDomainBackend' +import { emailDomainWizardLogic } from './emailDomainWizardLogic' + +describe('emailDomainWizardLogic', () => { + beforeEach(() => { + useMocks(new FakeEmailDomainBackend().mocks()) + initKeaTests() + }) + + const mountWizard = (): ReturnType => { + const logic = emailDomainWizardLogic({ logicKey: 'test', onCreated: () => {} }) + logic.mount() + return logic + } + + it.each([ + ['acme.com', 'acme.com', 'hello'], + [' ACME.com ', 'acme.com', 'hello'], + ['https://www.acme.com/pricing?x=1', 'acme.com', 'hello'], + ['app.acme.com', 'app.acme.com', 'hello'], + ['jane@acme.com', 'acme.com', 'jane'], + ['Support@WWW.Acme.com', 'acme.com', 'support'], + ])('turns the input %s into the domain %s and the address %s@', (input, domain, localPart) => { + const logic = mountWizard() + + logic.actions.setCustomDomainInput(input) + logic.actions.useCustomDomain() + + expect(logic.values.selectedDomain).toBe(domain) + expect(logic.values.localPart).toBe(localPart) + expect(logic.values.fromAddress).toBe(`${localPart}@mail.${domain}`) + }) + + it.each([ + ['jane@gmail.com', 'free_mailbox'], + ['gmx.de', 'free_mailbox'], + ['not a domain', 'invalid'], + ['acme', 'invalid'], + ['127.0.0.1', 'invalid'], + ['http://10.0.0.5:3000', 'invalid'], + ['acme.123', 'invalid'], + ])('refuses %s as %s and keeps the domain unselected', (input, problem) => { + const logic = mountWizard() + + logic.actions.setCustomDomainInput(input) + logic.actions.useCustomDomain() + + expect(logic.values.customDomain.problem).toBe(problem) + expect(logic.values.selectedDomain).toBeNull() + }) + + it.each([ + ['mail', 'news!', 'news'], + ['send', 'Send', 'send'], + ['', 'My Sub', 'mysub'], + ])('keeps the sending subdomain to dns label characters: %s then %s becomes %s', (first, typed, expected) => { + const logic = mountWizard() + logic.actions.selectDomain('acme.com') + + logic.actions.setSendPrefix(first) + logic.actions.setSendPrefix(typed) + + expect(logic.values.sendPrefix).toBe(expected) + expect(logic.values.sendingDomain).toBe(`${expected}.acme.com`) + }) +}) diff --git a/products/workflows/frontend/Channels/EmailDomain/emailDomainWizardLogic.ts b/products/workflows/frontend/Channels/EmailDomain/emailDomainWizardLogic.ts new file mode 100644 index 000000000000..7e519c899ef7 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/emailDomainWizardLogic.ts @@ -0,0 +1,354 @@ +import { MakeLogicType, actions, afterMount, connect, kea, key, listeners, path, props, reducers, selectors } from 'kea' +import { loaders } from 'kea-loaders' +import posthog from 'posthog-js' + +import api, { ApiError } from 'lib/api' +import { integrationsLogic } from 'lib/integrations/integrationsLogic' +import { Scene } from 'scenes/sceneTypes' +import { teamLogic } from 'scenes/teamLogic' +import { userLogic } from 'scenes/userLogic' + +import { ProductKey } from '~/queries/schema/schema-general' +import { hogql } from '~/queries/utils' +import { IntegrationType, TeamPublicType, TeamType, UserType } from '~/types' + +import { integrationsCreate } from 'products/integrations/frontend/generated/api' +import type { IntegrationConfigApi } from 'products/integrations/frontend/generated/api.schemas' + +import { DomainProblem, asSubdomainLabel, domainProblemOf, nameFromDomain, normalizeDomainInput } from './domainInput' +import { EventHost, InferredDomain, inferDomains } from './inferDomains' + +export interface EmailDomainWizardLogicProps { + logicKey: string + onCreated: (sender: IntegrationConfigApi) => void +} + +export interface CustomDomain { + domain: string + localPart: string | null + problem: DomainProblem | null +} + +export const DEFAULT_SEND_PREFIX: string = 'mail' +export const DEFAULT_LOCAL_PART: string = 'hello' +export const DEFAULT_BOUNCE_PREFIX: string = 'feedback' + +const EVENT_HOSTS_QUERY = hogql`SELECT properties.$host AS host, count() AS events + FROM events + WHERE timestamp >= now() - INTERVAL 30 DAY + AND properties.$host IS NOT NULL + AND properties.$host != '' + GROUP BY host + ORDER BY events DESC + LIMIT 5` + +// Generated by kea-typegen. Update if you're an agent, ignore if you're human. +export interface emailDomainWizardLogicValues { + integrations: IntegrationType[] | null // integrationsLogic + currentTeam: TeamPublicType | TeamType | null // teamLogic + currentTeamId: number | null // teamLogic + user: UserType | null // userLogic + advancedOpen: boolean + bouncePrefix: string + createdSender: IntegrationConfigApi | null + createdSenderLoading: boolean + customDomain: CustomDomain + customDomainInput: string + customPrefixOpen: boolean + eventHosts: EventHost[] + eventHostsLoading: boolean + existingSender: IntegrationType | null + fromAddress: string | null + inferredDomains: InferredDomain[] + localPart: string + selectedDomain: string | null + selectedIsInferred: boolean + sendPrefix: string + sendingDomain: string | null + submitError: string | null +} + +// Generated by kea-typegen. Update if you're an agent, ignore if you're human. +export interface emailDomainWizardLogicActions { + loadIntegrations: (_?: any) => any // integrationsLogic + clearDomain: () => { + value: true + } + createSender: () => any + createSenderFailure: ( + error: string, + errorObject?: any + ) => { + error: string + errorObject?: any + } + createSenderSuccess: ( + createdSender: IntegrationConfigApi | null, + payload?: any + ) => { + createdSender: IntegrationConfigApi | null + payload?: any + } + loadEventHosts: () => any + loadEventHostsFailure: ( + error: string, + errorObject?: any + ) => { + error: string + errorObject?: any + } + loadEventHostsSuccess: ( + eventHosts: EventHost[], + payload?: any + ) => { + eventHosts: EventHost[] + payload?: any + } + selectDomain: (domain: string) => { + domain: string + } + setAdvancedOpen: (open: boolean) => { + open: boolean + } + setBouncePrefix: (prefix: string) => { + prefix: string + } + setCustomDomainInput: (value: string) => { + value: string + } + setCustomPrefixOpen: (open: boolean) => { + open: boolean + } + setLocalPart: (localPart: string) => { + localPart: string + } + setSendPrefix: (prefix: string) => { + prefix: string + } + setSubmitError: (error: string | null) => { + error: string | null + } + useCustomDomain: () => { + value: true + } +} + +// Generated by kea-typegen. Update if you're an agent, ignore if you're human. +export interface emailDomainWizardLogicMeta { + key: string + __keaTypeGenInternalSelectorTypes: { + inferredDomains: ( + currentTeam: TeamPublicType | TeamType | null, + user: UserType | null, + eventHosts: EventHost[] + ) => InferredDomain[] + customDomain: (customDomainInput: string) => CustomDomain + selectedIsInferred: (selectedDomain: string | null, inferredDomains: InferredDomain[]) => boolean + sendingDomain: (selectedDomain: string | null, sendPrefix: 'mail') => string | null + fromAddress: (sendingDomain: string | null, localPart: 'hello') => string | null + existingSender: (integrations: IntegrationType[] | null, sendingDomain: string | null) => IntegrationType | null + } +} + +export type emailDomainWizardLogicType = MakeLogicType< + emailDomainWizardLogicValues, + emailDomainWizardLogicActions, + EmailDomainWizardLogicProps, + emailDomainWizardLogicMeta +> + +/** The domain step: picks or types the domain, shapes the first sender address and creates the sender. */ +export const emailDomainWizardLogic = kea([ + path((key) => ['products', 'workflows', 'frontend', 'Channels', 'EmailDomain', 'emailDomainWizardLogic', key]), + props({} as EmailDomainWizardLogicProps), + key(({ logicKey }) => logicKey), + connect(() => ({ + values: [teamLogic, ['currentTeam', 'currentTeamId'], userLogic, ['user'], integrationsLogic, ['integrations']], + actions: [integrationsLogic, ['loadIntegrations']], + })), + actions({ + selectDomain: (domain: string) => ({ domain }), + clearDomain: true, + setCustomDomainInput: (value: string) => ({ value }), + useCustomDomain: true, + setSendPrefix: (prefix: string) => ({ prefix }), + setCustomPrefixOpen: (open: boolean) => ({ open }), + setBouncePrefix: (prefix: string) => ({ prefix }), + setAdvancedOpen: (open: boolean) => ({ open }), + setLocalPart: (localPart: string) => ({ localPart }), + setSubmitError: (error: string | null) => ({ error }), + }), + loaders(({ values, actions }) => ({ + eventHosts: [ + [] as EventHost[], + { + loadEventHosts: async (): Promise => { + try { + const response = await api.queryHogQL(EVENT_HOSTS_QUERY, { + scene: Scene.WorkflowsEmailDomain, + productKey: ProductKey.WORKFLOWS, + name: 'email_domain_event_hosts', + }) + return (response.results ?? []).map(([host, count]) => ({ + host: String(host), + count: Number(count), + })) + } catch { + return [] + } + }, + }, + ], + createdSender: [ + null as IntegrationConfigApi | null, + { + createSender: async (): Promise => { + const { selectedDomain, sendingDomain, currentTeamId, localPart, bouncePrefix } = values + if (!selectedDomain || !sendingDomain || !currentTeamId) { + return null + } + actions.setSubmitError(null) + try { + return await integrationsCreate(String(currentTeamId), { + kind: 'email', + config: { + email: `${localPart || DEFAULT_LOCAL_PART}@${sendingDomain}`, + name: nameFromDomain(selectedDomain), + provider: 'ses', + mail_from_subdomain: bouncePrefix || DEFAULT_BOUNCE_PREFIX, + }, + }) + } catch (error) { + const detail = error instanceof ApiError ? error.detail : null + actions.setSubmitError(detail || 'Could not create the sender. Try again in a moment.') + throw error + } + }, + }, + ], + })), + reducers({ + selectedDomain: [ + null as string | null, + { + selectDomain: (_, { domain }) => domain, + clearDomain: () => null, + }, + ], + customDomainInput: [ + '', + { + setCustomDomainInput: (_, { value }) => value, + selectDomain: () => '', + }, + ], + localPart: [ + DEFAULT_LOCAL_PART, + { + setLocalPart: (_, { localPart }) => localPart, + }, + ], + sendPrefix: [ + DEFAULT_SEND_PREFIX, + { + setSendPrefix: (_, { prefix }) => asSubdomainLabel(prefix), + }, + ], + customPrefixOpen: [ + false, + { + setCustomPrefixOpen: (_, { open }) => open, + }, + ], + bouncePrefix: [ + DEFAULT_BOUNCE_PREFIX, + { + setBouncePrefix: (_, { prefix }) => asSubdomainLabel(prefix), + }, + ], + advancedOpen: [ + false, + { + setAdvancedOpen: (_, { open }) => open, + }, + ], + submitError: [ + null as string | null, + { + setSubmitError: (_, { error }) => error, + selectDomain: () => null, + clearDomain: () => null, + }, + ], + }), + selectors({ + inferredDomains: [ + (s) => [s.currentTeam, s.user, s.eventHosts], + ( + currentTeam: TeamPublicType | TeamType | null, + user: UserType | null, + eventHosts: EventHost[] + ): InferredDomain[] => inferDomains(currentTeam?.app_urls ?? [], user?.email ?? null, eventHosts), + ], + customDomain: [ + (s) => [s.customDomainInput], + (customDomainInput: string): CustomDomain => { + const { domain, localPart } = normalizeDomainInput(customDomainInput) + return { domain, localPart, problem: domainProblemOf(domain) } + }, + ], + selectedIsInferred: [ + (s) => [s.selectedDomain, s.inferredDomains], + (selectedDomain: string | null, inferredDomains: InferredDomain[]): boolean => + inferredDomains.some((candidate) => candidate.domain === selectedDomain), + ], + sendingDomain: [ + (s) => [s.selectedDomain, s.sendPrefix], + (selectedDomain: string | null, sendPrefix: string): string | null => + selectedDomain ? (sendPrefix ? `${sendPrefix}.${selectedDomain}` : selectedDomain) : null, + ], + fromAddress: [ + (s) => [s.sendingDomain, s.localPart], + (sendingDomain: string | null, localPart: string): string | null => + sendingDomain ? `${localPart || DEFAULT_LOCAL_PART}@${sendingDomain}` : null, + ], + existingSender: [ + (s) => [s.integrations, s.sendingDomain], + (integrations: IntegrationType[] | null, sendingDomain: string | null): IntegrationType | null => + integrations?.find( + (integration) => integration.kind === 'email' && integration.config?.domain === sendingDomain + ) ?? null, + ], + }), + listeners(({ actions, values, props }) => ({ + useCustomDomain: () => { + const { domain, localPart, problem } = values.customDomain + if (!domain || problem) { + return + } + if (localPart) { + actions.setLocalPart(localPart) + } + actions.selectDomain(domain) + }, + createSenderSuccess: async ({ createdSender }) => { + if (!createdSender) { + return + } + actions.loadIntegrations() + props.onCreated(createdSender) + const domain = values.sendingDomain ?? '' + // nosemgrep: prefer-codegen-api-namespaced-integrations -- the generated integrationsDomainConnectCheckRetrieve has no domain query parameter + const detection = await api.integrations.domainConnectCheck(domain).catch(() => null) + // pinned: analytics event name + posthog.capture('email domain submitted', { + detected_provider: detection?.dns_host?.name ?? detection?.provider_name ?? null, + has_domain_connect: detection?.supported ?? false, + existing_senders: detection?.existing_email_tools ?? [], + }) + }, + })), + afterMount(({ actions }) => { + actions.loadEventHosts() + }), +]) diff --git a/products/workflows/frontend/Channels/EmailDomain/inferDomains.ts b/products/workflows/frontend/Channels/EmailDomain/inferDomains.ts new file mode 100644 index 000000000000..f976d0146ebb --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/inferDomains.ts @@ -0,0 +1,75 @@ +import { humanFriendlyNumber } from 'lib/utils/numbers' + +import { hostnameOfUrl, isFreeMailboxDomain, isValidDomain, normalizeDomainInput, rootDomainOf } from './domainInput' + +export interface EventHost { + host: string + count: number +} + +export interface InferredDomain { + domain: string + reasons: string[] + recommended: boolean +} + +const isUsableHost = (domain: string | null): domain is string => + !!domain && isValidDomain(domain) && domain !== 'localhost' + +class DomainCandidates { + private reasons = new Map() + private rootWeight = new Map() + + has(domain: string): boolean { + return this.reasons.has(domain) + } + + add(domain: string, reason: string, weight: number): void { + this.reasons.set(domain, [...(this.reasons.get(domain) ?? []), reason]) + const root = rootDomainOf(domain) + this.rootWeight.set(root, (this.rootWeight.get(root) ?? 0) + weight) + } + + private recommendedDomain(): string | undefined { + const topRoot = [...this.rootWeight.entries()].sort((a, b) => b[1] - a[1])[0]?.[0] + const sameRoot = [...this.reasons.keys()].filter((domain) => rootDomainOf(domain) === topRoot) + return sameRoot.includes(topRoot) ? topRoot : sameRoot[0] + } + + ranked(): InferredDomain[] { + const recommended = this.recommendedDomain() + return [...this.reasons.entries()] + .map(([domain, reasons]) => ({ domain, reasons, recommended: domain === recommended })) + .sort((a, b) => Number(b.recommended) - Number(a.recommended)) + } +} + +/** Domains the project probably sends from: where its events come from, the login email and the project URLs. */ +export const inferDomains = ( + appUrls: string[], + userEmail: string | null, + eventHosts: EventHost[] +): InferredDomain[] => { + const candidates = new DomainCandidates() + for (const { host, count } of eventHosts) { + const domain = hostnameOfUrl(host) + if (isUsableHost(domain)) { + candidates.add( + domain, + `${humanFriendlyNumber(count)} events came from ${domain} in the last 30 days`, + count + ) + } + } + const emailDomain = userEmail ? normalizeDomainInput(userEmail).domain : '' + if (isUsableHost(emailDomain) && !isFreeMailboxDomain(emailDomain)) { + candidates.add(emailDomain, `Your login is ${userEmail}`, 1) + } + for (const url of appUrls) { + const domain = hostnameOfUrl(url) + if (isUsableHost(domain) && !candidates.has(domain)) { + candidates.add(domain, 'One of your project URLs', 1) + } + } + return candidates.ranked() +} diff --git a/products/workflows/manifest.tsx b/products/workflows/manifest.tsx index ba99ee32aab5..f2d585e2c0aa 100644 --- a/products/workflows/manifest.tsx +++ b/products/workflows/manifest.tsx @@ -70,6 +70,7 @@ export const manifest: ProductManifest = { workflows: (tab?: WorkflowsSceneTab): string => `/workflows${tab ? `/${tab}` : ''}`, workflow: (id: string, tab: string): string => `/workflows/${id}/${tab}`, workflowNew: (): string => '/workflows/new/workflow', + workflowsEmailDomain: (id: string | number): string => `/workflows/channels/email/${id}`, workflowsLibraryMessage: (id: string): string => `/workflows/library/messages/${id}`, workflowsLibraryTemplate: (id?: string): string => `/workflows/library/templates/${id}`, workflowsLibraryTemplateNew: (): string => '/workflows/library/templates/new', From 931a0f9d4b77915c5b3dd413bbeac9404c355fe6 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:02:39 +0000 Subject: [PATCH 09/13] fix(workflows): keep polling deadline when the tab resumes --- .../emailDomainStatusLogic.test.ts | 28 +++++++++++-------- .../EmailDomain/emailDomainStatusLogic.ts | 7 ++++- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/products/workflows/frontend/Channels/EmailDomain/emailDomainStatusLogic.test.ts b/products/workflows/frontend/Channels/EmailDomain/emailDomainStatusLogic.test.ts index fc43e491526a..8654f9c4ed75 100644 --- a/products/workflows/frontend/Channels/EmailDomain/emailDomainStatusLogic.test.ts +++ b/products/workflows/frontend/Channels/EmailDomain/emailDomainStatusLogic.test.ts @@ -62,18 +62,22 @@ describe('emailDomainStatusLogic', () => { expect(statusChecks()).toBe(81) }) - it('pauses polling while the tab is hidden and resumes when it is shown', async () => { - await mountAndLoad() - expect(statusChecks()).toBe(1) - - setDocumentHidden(true) - await jest.advanceTimersByTimeAsync(MINUTE) - expect(statusChecks()).toBe(1) - - setDocumentHidden(false) - await jest.advanceTimersByTimeAsync(10_000) - expect(statusChecks()).toBe(2) - }) + it.each([MINUTE, 31 * MINUTE])( + 'pauses polling for %s milliseconds hidden and resumes within the original deadline', + async (hiddenDuration) => { + await mountAndLoad() + expect(statusChecks()).toBe(1) + + setDocumentHidden(true) + await jest.advanceTimersByTimeAsync(hiddenDuration) + expect(statusChecks()).toBe(1) + + setDocumentHidden(false) + await jest.advanceTimersByTimeAsync(10_000) + expect(statusChecks()).toBe(hiddenDuration < 30 * MINUTE ? 2 : 1) + expect(logic.values.pollingStopped).toBe(hiddenDuration >= 30 * MINUTE) + } + ) it('restarts the schedule with an uncached check when the person checks again after polling stopped', async () => { await mountAndLoad() diff --git a/products/workflows/frontend/Channels/EmailDomain/emailDomainStatusLogic.ts b/products/workflows/frontend/Channels/EmailDomain/emailDomainStatusLogic.ts index 804ee3b5a771..fcf3237596df 100644 --- a/products/workflows/frontend/Channels/EmailDomain/emailDomainStatusLogic.ts +++ b/products/workflows/frontend/Channels/EmailDomain/emailDomainStatusLogic.ts @@ -279,8 +279,13 @@ export const emailDomainStatusLogic = kea([ actions.stopPolling() return } - const delay = elapsed < POLL_FAST_WINDOW_MS ? POLL_FAST_MS : POLL_SLOW_MS cache.disposables.add(() => { + const elapsed = Date.now() - (values.pollingStartedAt ?? Date.now()) + if (elapsed >= POLL_MAX_MS) { + actions.stopPolling() + return () => undefined + } + const delay = elapsed < POLL_FAST_WINDOW_MS ? POLL_FAST_MS : POLL_SLOW_MS const timer = window.setTimeout(() => actions.loadStatus(), delay) return () => clearTimeout(timer) }, 'poll') From 9042ab6d912afcd41ce85eeaf8321debf8db1c66 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:22:16 +0000 Subject: [PATCH 10/13] fix(workflows): type supported email sender providers --- .../frontend/Channels/EmailDomain/emailDomainSenderLogic.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/products/workflows/frontend/Channels/EmailDomain/emailDomainSenderLogic.ts b/products/workflows/frontend/Channels/EmailDomain/emailDomainSenderLogic.ts index 103cdf6168fd..6cca1c5ed137 100644 --- a/products/workflows/frontend/Channels/EmailDomain/emailDomainSenderLogic.ts +++ b/products/workflows/frontend/Channels/EmailDomain/emailDomainSenderLogic.ts @@ -22,7 +22,7 @@ export interface EmailSenderConfig { domain: string email: string name: string - provider: string + provider: 'ses' | 'maildev' mail_from_subdomain?: string verified?: boolean setup_method?: EmailSetupMethod From b049529874168fbe933de4f4292c0c3d8f431e87 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:40:53 +0000 Subject: [PATCH 11/13] fix(workflows): send only config when updating email senders --- .../frontend/Channels/EmailDomain/emailDomainSenderLogic.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/products/workflows/frontend/Channels/EmailDomain/emailDomainSenderLogic.ts b/products/workflows/frontend/Channels/EmailDomain/emailDomainSenderLogic.ts index 6cca1c5ed137..a7e10e10adf5 100644 --- a/products/workflows/frontend/Channels/EmailDomain/emailDomainSenderLogic.ts +++ b/products/workflows/frontend/Channels/EmailDomain/emailDomainSenderLogic.ts @@ -175,7 +175,6 @@ export const emailDomainSenderLogic = kea([ } try { return await integrationsEmailPartialUpdate(String(currentTeamId), integrationId, { - kind: 'email', config: { ...senderConfig, setup_method: method }, }) } catch { @@ -278,7 +277,6 @@ export const emailDomainSenderLogic = kea([ } try { const saved = await integrationsEmailPartialUpdate(String(currentTeamId), integrationId, { - kind: 'email', config: { ...senderConfig, name: values.senderName }, }) actions.senderSaved(saved) From 14f4b1782c01c26a25dac0c039fd6fa87e54848d Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Fri, 2 Oct 2026 05:12:50 +0000 Subject: [PATCH 12/13] feat(workflows): add the email sending domain wizard page Adds a flag-gated page at /workflows/channels/email/:id that walks through picking a domain, adding the DNS settings at the detected host, and watching verification until the domain is ready to send. The page handles the Domain Connect return, offers an agent hand-off prompt behind a second flag, and shows an unavailable state with a Try again button when the sender or the status cannot be loaded. Every step and component reads from the logic that owns its state, bound once per integration id by BindEmailDomainLogics. The record rows tell people to merge include:amazonses.com into an SPF record that already exists and to keep an existing DMARC record. The Domain Connect return reads the kea router instead of window.location, so the same code path runs in the app and under the memory history in tests. The stories render the same fake backend the tests use, staged per state, so a story and a test cannot drift apart. Rendered flow tests drive the real scene outside in: the manual path from a typed domain to a verified first sender with a renamed sender and a test email; the one-click path through Cloudflare and the return with domain_connect=email; and recovery from a failed create, a failed sender load, a status outage after the records were found, a failed host detection, a failed restart and a return without approval. Co-Authored-By: Claude Opus 5.5 (1M context) --- frontend/src/productScenes.tsx | 1 + frontend/src/products.tsx | 20 +- frontend/src/scenes/sceneTypes.ts | 1 + .../EmailDomain/BindEmailDomainLogics.tsx | 31 ++ .../EmailDomain/EmailDomain.stories.tsx | 164 ++++++++++ .../EmailDomain/EmailDomainScene.test.tsx | 294 ++++++++++++++++++ .../Channels/EmailDomain/EmailDomainScene.tsx | 32 ++ .../Channels/EmailDomain/EmailDomainSetup.tsx | 57 ++++ .../components/AgentHandoffModal.tsx | 92 ++++++ .../components/AgentSetupButton.tsx | 24 ++ .../components/AnotherDomainInput.tsx | 47 +++ .../EmailDomain/components/BigAction.tsx | 9 + .../EmailDomain/components/DnsHostLink.tsx | 27 ++ .../EmailDomain/components/DnsRecordRows.tsx | 131 ++++++++ .../EmailDomain/components/DomainCard.tsx | 47 +++ .../components/FirstSenderCard.tsx | 91 ++++++ .../components/NextWorkflowCard.tsx | 45 +++ .../components/SendingAddressCard.tsx | 114 +++++++ .../EmailDomain/components/StepHeading.tsx | 23 ++ .../EmailDomain/components/StepIndicator.tsx | 46 +++ .../components/VerificationStepsStrip.tsx | 42 +++ .../EmailDomain/emailDomainRoutes.test.ts | 46 +++ .../frontend/Channels/EmailDomain/hoggies.ts | 19 ++ .../Channels/EmailDomain/steps/DomainStep.tsx | 104 +++++++ .../Channels/EmailDomain/steps/ReadyStep.tsx | 50 +++ .../EmailDomain/steps/SettingsStep.tsx | 179 +++++++++++ .../EmailDomain/steps/UnavailableStep.tsx | 32 ++ .../EmailDomain/steps/VerifyingStep.tsx | 140 +++++++++ products/workflows/manifest.tsx | 7 + 29 files changed, 1913 insertions(+), 2 deletions(-) create mode 100644 products/workflows/frontend/Channels/EmailDomain/BindEmailDomainLogics.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/EmailDomain.stories.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/EmailDomainScene.test.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/EmailDomainScene.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/EmailDomainSetup.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/components/AgentHandoffModal.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/components/AgentSetupButton.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/components/AnotherDomainInput.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/components/BigAction.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/components/DnsHostLink.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/components/DnsRecordRows.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/components/DomainCard.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/components/FirstSenderCard.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/components/NextWorkflowCard.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/components/SendingAddressCard.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/components/StepHeading.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/components/StepIndicator.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/components/VerificationStepsStrip.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/emailDomainRoutes.test.ts create mode 100644 products/workflows/frontend/Channels/EmailDomain/hoggies.ts create mode 100644 products/workflows/frontend/Channels/EmailDomain/steps/DomainStep.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/steps/ReadyStep.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/steps/SettingsStep.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/steps/UnavailableStep.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/steps/VerifyingStep.tsx diff --git a/frontend/src/productScenes.tsx b/frontend/src/productScenes.tsx index e9adf6118e71..df3c667bc7d1 100644 --- a/frontend/src/productScenes.tsx +++ b/frontend/src/productScenes.tsx @@ -207,4 +207,5 @@ export const productScenes: Record Promise> = { WorkflowsLibraryTemplate: () => import('../../products/workflows/frontend/TemplateLibrary/MessageTemplate'), Broadcasts: () => import('../../products/workflows/frontend/Broadcasts/BroadcastsScene'), Broadcast: () => import('../../products/workflows/frontend/Broadcasts/BroadcastScene'), + WorkflowsEmailDomain: () => import('../../products/workflows/frontend/Channels/EmailDomain/EmailDomainScene'), } diff --git a/frontend/src/products.tsx b/frontend/src/products.tsx index 92f1b4c736f8..1297f0dfb7e7 100644 --- a/frontend/src/products.tsx +++ b/frontend/src/products.tsx @@ -315,6 +315,7 @@ export const productRoutes: Record = { '/wizard/runs': ['WizardRuns', 'wizardRuns'], '/workflows': ['Workflows', 'workflows'], '/workflows/:tab': ['Workflows', 'workflows'], + '/workflows/channels/email/:id': ['WorkflowsEmailDomain', 'workflowsEmailDomain'], '/workflows/:id/:tab': ['Workflow', 'workflowTab'], '/workflows/library/templates/:id': ['WorkflowsLibraryTemplate', 'workflowsLibraryTemplate'], '/workflows/library/templates/new': ['WorkflowsLibraryTemplate', 'workflowsLibraryTemplate'], @@ -1217,6 +1218,7 @@ export const productConfiguration: Record = { projectBased: true, description: 'Send a one-time or scheduled email to a group of people', }, + WorkflowsEmailDomain: { name: 'Workflows', iconType: 'workflows', projectBased: true }, } /** This const is auto-generated, as is the whole file */ @@ -2257,7 +2259,14 @@ export const getTreeItemsProducts = (): FileSystemImport[] => [ iconType: 'broadcasts', iconColor: ['var(--color-product-broadcasts-light)', 'var(--color-product-broadcasts-dark)'], sceneKey: 'Broadcasts', - sceneKeys: ['Workflows', 'Workflow', 'WorkflowsLibraryTemplate', 'Broadcasts', 'Broadcast'], + sceneKeys: [ + 'Workflows', + 'Workflow', + 'WorkflowsLibraryTemplate', + 'Broadcasts', + 'Broadcast', + 'WorkflowsEmailDomain', + ], }, { path: 'Business intelligence', @@ -3017,7 +3026,14 @@ export const getTreeItemsProducts = (): FileSystemImport[] => [ iconType: 'workflows', iconColor: ['var(--color-product-workflows-light)'] as FileSystemIconColor, sceneKey: 'Workflows', - sceneKeys: ['Workflows', 'Workflow', 'WorkflowsLibraryTemplate', 'Broadcasts', 'Broadcast'], + sceneKeys: [ + 'Workflows', + 'Workflow', + 'WorkflowsLibraryTemplate', + 'Broadcasts', + 'Broadcast', + 'WorkflowsEmailDomain', + ], }, ] diff --git a/frontend/src/scenes/sceneTypes.ts b/frontend/src/scenes/sceneTypes.ts index ccf540fb17c2..b231f1cf0759 100644 --- a/frontend/src/scenes/sceneTypes.ts +++ b/frontend/src/scenes/sceneTypes.ts @@ -453,6 +453,7 @@ export const sceneToAccessControlResourceType: Partial< [Scene.Broadcasts]: AccessControlResourceType.Workflow, [Scene.Workflows]: AccessControlResourceType.Workflow, [Scene.Broadcast]: AccessControlResourceType.Workflow, + [Scene.WorkflowsEmailDomain]: AccessControlResourceType.Workflow, // Product Tours [Scene.ProductTour]: AccessControlResourceType.ProductTour, diff --git a/products/workflows/frontend/Channels/EmailDomain/BindEmailDomainLogics.tsx b/products/workflows/frontend/Channels/EmailDomain/BindEmailDomainLogics.tsx new file mode 100644 index 000000000000..d5d9de0d5b08 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/BindEmailDomainLogics.tsx @@ -0,0 +1,31 @@ +import { BindLogic } from 'kea' +import { ReactNode } from 'react' + +import { emailDomainAgentLogic } from './emailDomainAgentLogic' +import { emailDomainAutoConfigureLogic } from './emailDomainAutoConfigureLogic' +import { emailDomainLogic } from './emailDomainLogic' +import { EmailDomainLogicProps } from './emailDomainLogicProps' +import { emailDomainManualSetupLogic } from './emailDomainManualSetupLogic' +import { emailDomainSenderLogic } from './emailDomainSenderLogic' +import { emailDomainStatusLogic } from './emailDomainStatusLogic' + +const LOGICS = [ + emailDomainSenderLogic, + emailDomainStatusLogic, + emailDomainAutoConfigureLogic, + emailDomainManualSetupLogic, + emailDomainAgentLogic, + emailDomainLogic, +] + +/** Binds every logic of the sending domain page to one integration id, so the steps below can use them without props. */ +export function BindEmailDomainLogics({ id, children }: EmailDomainLogicProps & { children: ReactNode }): JSX.Element { + return LOGICS.reduceRight( + (inner, logic) => ( + + {inner} + + ), + <>{children} + ) +} diff --git a/products/workflows/frontend/Channels/EmailDomain/EmailDomain.stories.tsx b/products/workflows/frontend/Channels/EmailDomain/EmailDomain.stories.tsx new file mode 100644 index 000000000000..619327c8bc89 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/EmailDomain.stories.tsx @@ -0,0 +1,164 @@ +import { Meta, StoryObj } from '@storybook/react' +import { useEffect, useMemo } from 'react' + +import { FEATURE_FLAGS } from 'lib/constants' + +import { useStorybookMocks } from '~/mocks/browser' + +import { FakeDnsHost, FakeEmailDomainBackend, FakeRecord } from './__mocks__/fakeEmailDomainBackend' +import { emailDomainAgentLogic } from './emailDomainAgentLogic' +import { emailDomainManualSetupLogic } from './emailDomainManualSetupLogic' +import { EmailDomainScene } from './EmailDomainScene' +import { emailDomainStatusLogic } from './emailDomainStatusLogic' + +const DOMAIN = 'mail.acme.com' +const EMAIL = `hello@${DOMAIN}` + +const halfPublished = (record: FakeRecord, index: number): boolean => + ['verification', 'spf'].includes(record.kind) || (record.kind === 'dkim' && index < 3) +const allButOneDkimPublished = (_: FakeRecord, index: number): boolean => index !== 2 + +type Stage = (backend: FakeEmailDomainBackend, id: string) => void + +interface DomainPageArgs { + host: FakeDnsHost + seed?: (backend: FakeEmailDomainBackend) => void + stage?: Stage +} + +const seededBackend = (host: FakeDnsHost, seed?: DomainPageArgs['seed']): FakeEmailDomainBackend => { + const backend = new FakeEmailDomainBackend() + backend.hostDnsAt('acme.com', host) + backend.addSender({ email: EMAIL }) + seed?.(backend) + return backend +} + +function Staged({ backend, id, stage }: { backend: FakeEmailDomainBackend; id: string; stage?: Stage }): null { + useEffect(() => { + stage?.(backend, id) + }, [backend, id, stage]) + return null +} + +function DomainPage({ host, seed, stage }: DomainPageArgs): JSX.Element { + const backend = useMemo(() => seededBackend(host, seed), [host, seed]) + const id = String(backend.sender(42)?.id) + useStorybookMocks(backend.mocks()) + return ( + <> + + + + ) +} + +function UnavailableDomainPage(): JSX.Element { + const backend = useMemo(() => { + const seeded = seededBackend('unknown') + seeded.fail('integration_get', { status: 500, body: { detail: 'Could not reach the email provider' } }) + return seeded + }, []) + useStorybookMocks(backend.mocks()) + return +} + +function NewDomainPage(): JSX.Element { + const backend = useMemo(() => { + const empty = new FakeEmailDomainBackend() + empty.eventHosts = [ + ['https://acme.com', 1_240_000], + ['https://app.acme.com', 860_000], + ] + return empty + }, []) + useStorybookMocks(backend.mocks()) + return +} + +const meta: Meta = { + title: 'Scenes-App/Workflows/Email domain setup', + parameters: { + layout: 'fullscreen', + featureFlags: [FEATURE_FLAGS.WORKFLOWS_EMAIL_DOMAIN_WIZARD], + testOptions: { waitForLoadersToDisappear: true }, + }, + render: (args) => , +} +export default meta + +type Story = StoryObj + +export const DomainStep: Story = { render: () => } + +export const SettingsNotStarted: Story = { + args: { host: 'cloudflare', seed: (backend) => backend.withoutSesIdentity(DOMAIN) }, +} + +export const SettingsCloudflare: Story = { args: { host: 'cloudflare' } } + +export const SettingsCloudflareExistingDmarc: Story = { + args: { host: 'cloudflare', seed: (backend) => backend.preexistingRecords(DOMAIN, ['dmarc']) }, +} + +export const SettingsCloudflareRecordsRevealed: Story = { + args: { + host: 'cloudflare', + stage: (_, id) => emailDomainManualSetupLogic.findMounted({ id })?.actions.revealRecords(), + }, +} + +export const SettingsRoute53: Story = { args: { host: 'route53' } } + +export const SettingsUnknownHost: Story = { args: { host: 'unknown' } } + +export const Verifying: Story = { + args: { host: 'route53', seed: (backend) => backend.publishRecords(DOMAIN, halfPublished) }, +} + +export const VerifyingRecordsFound: Story = { + args: { host: 'cloudflare', seed: (backend) => backend.publishRecords(DOMAIN) }, +} + +export const VerifyingPollingStopped: Story = { + args: { + host: 'route53', + seed: (backend) => backend.publishRecords(DOMAIN, halfPublished), + stage: (_, id) => emailDomainStatusLogic.findMounted({ id })?.actions.stopPolling(), + }, +} + +export const VerifyingTemporaryFailure: Story = { + args: { + host: 'unknown', + seed: (backend) => { + backend.sesConfirms(DOMAIN, ['verification', 'dkim']) + backend.publishRecords(DOMAIN, ['spf', 'dmarc']) + backend.sesReports(DOMAIN, 'temporary_failure') + }, + }, +} + +export const VerifyingFailed: Story = { + args: { + host: 'route53', + seed: (backend) => { + backend.publishRecords(DOMAIN, allButOneDkimPublished) + backend.sesReports(DOMAIN, 'failed') + }, + }, +} + +export const Ready: Story = { args: { host: 'cloudflare', seed: (backend) => backend.verifyDomain(DOMAIN) } } + +export const Unavailable: Story = { render: () => } + +export const AgentHandoff: Story = { + args: { + host: 'route53', + stage: (_, id) => emailDomainAgentLogic.findMounted({ id })?.actions.openAgentModal(), + }, + parameters: { + featureFlags: [FEATURE_FLAGS.WORKFLOWS_EMAIL_DOMAIN_WIZARD, FEATURE_FLAGS.WORKFLOWS_EMAIL_DOMAIN_AGENT_SETUP], + }, +} diff --git a/products/workflows/frontend/Channels/EmailDomain/EmailDomainScene.test.tsx b/products/workflows/frontend/Channels/EmailDomain/EmailDomainScene.test.tsx new file mode 100644 index 000000000000..a1a2b9503ee1 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/EmailDomainScene.test.tsx @@ -0,0 +1,294 @@ +import '@testing-library/jest-dom' + +import { act, cleanup, fireEvent, render, screen } from '@testing-library/react' +import { Provider, useValues } from 'kea' +import { router } from 'kea-router' + +import { FEATURE_FLAGS } from 'lib/constants' +import { featureFlagLogic } from 'lib/logic/featureFlagLogic' +import { urls } from 'scenes/urls' + +import { resumeKeaLoadersErrors, silenceKeaLoadersErrors } from '~/initKea' +import { useMocks } from '~/mocks/jest' +import { initKeaTests } from '~/test/init' + +import { FakeEmailDomainBackend } from './__mocks__/fakeEmailDomainBackend' +import { EmailDomainScene } from './EmailDomainScene' + +const DOMAIN = 'mail.acme.com' +const SENDER_ID = 42 +const POLL_MS = 10_000 + +/** Renders the scene for whichever sender the URL names, the way the scene router does in the app. */ +function RoutedScene(): JSX.Element { + const { location } = useValues(router) + const id = location.pathname.match(/\/workflows\/channels\/email\/([^/]+)/)?.[1] ?? 'new' + return +} + +const element = (attr: string): HTMLElement => { + const found = document.querySelector(`[data-attr="${attr}"]`) + if (!found) { + throw new Error(`No element with data-attr ${attr} on the page`) + } + return found +} + +const click = (attr: string): void => { + fireEvent.click(element(attr)) +} + +const typeInto = (id: string, value: string): void => { + fireEvent.change(document.getElementById(id) as HTMLInputElement, { target: { value } }) +} + +const flush = async (ms = 0): Promise => { + await act(async () => { + await jest.advanceTimersByTimeAsync(ms) + }) +} + +const expectStep = (title: string | RegExp): void => { + expect(screen.getByText(title)).toBeInTheDocument() +} + +const expectProgress = (text: string): void => { + expect(screen.getAllByText(text).length).toBeGreaterThan(0) +} + +describe('EmailDomainScene', () => { + let backend: FakeEmailDomainBackend + + const open = async (path: string): Promise => { + router.actions.push(path) + render( + + + + ) + await flush() + } + + const seedSender = (): void => { + backend.addSender({ email: `hello@${DOMAIN}` }) + } + + beforeEach(() => { + jest.useFakeTimers() + backend = new FakeEmailDomainBackend() + backend.hostDnsAt('acme.com', 'cloudflare') + useMocks(backend.mocks()) + initKeaTests() + featureFlagLogic.mount() + featureFlagLogic.actions.setFeatureFlags([FEATURE_FLAGS.WORKFLOWS_EMAIL_DOMAIN_WIZARD], { + [FEATURE_FLAGS.WORKFLOWS_EMAIL_DOMAIN_WIZARD]: true, + }) + }) + + afterEach(() => { + cleanup() + jest.useRealTimers() + jest.restoreAllMocks() + }) + + it('takes a typed domain through the manual path to a verified first sender', async () => { + await open(urls.workflowsEmailDomain('new')) + expectStep('Which domain should your emails come from?') + + typeInto('email-domain-another-domain', 'acme.com') + click('email-domain-use-custom-domain') + expect(screen.getByText('hello@mail.acme.com')).toBeInTheDocument() + + click('email-domain-continue') + await flush() + expect(backend.requestsTo('integration_create')[0].body).toEqual({ + kind: 'email', + config: { email: 'hello@mail.acme.com', name: 'Acme', provider: 'ses', mail_from_subdomain: 'feedback' }, + }) + expect(router.values.location.pathname).toMatch(new RegExp(`${urls.workflowsEmailDomain(SENDER_ID)}$`)) + expectStep('Add 8 settings at Cloudflare') + + click('email-domain-reveal-records') + await flush() + expect(screen.getByText('Proves you own the domain')).toBeInTheDocument() + expect(backend.requestsTo('email_update').map((request) => request.body)).toEqual([ + { kind: 'email', config: expect.objectContaining({ setup_method: 'manual' }) }, + ]) + expect(backend.senderConfig(SENDER_ID)?.setup_method).toBe('manual') + + backend.publishRecords(DOMAIN) + await flush(POLL_MS) + expectStep('Checking your settings…') + expectProgress('8 of 8 found') + + backend.verifyDomain(DOMAIN) + await flush(POLL_MS) + expectStep(/You can send from/) + + typeInto('email-domain-sender-name', 'Acme Team') + click('email-domain-save-sender-name') + await flush() + expect(backend.senderConfig(SENDER_ID)?.name).toBe('Acme Team') + expect(document.querySelector('[data-attr="email-domain-save-sender-name"]')).toBeNull() + + click('email-domain-send-test-email') + await flush() + const invocation = backend.requestsTo('test_email')[0].body as any + const emailInput = invocation.configuration.actions[1].config.inputs.email.value + expect(emailInput.from).toEqual({ integrationId: SENDER_ID }) + expect(emailInput.to.email).toBe(invocation.globals.person.properties.email) + expect(screen.getByText('Send another test')).toBeInTheDocument() + }) + + it('sends the person to Cloudflare and verifies after they return', async () => { + seedSender() + const tab = { location: { href: '' }, opener: {}, close: jest.fn() } + jest.spyOn(window, 'open').mockReturnValue(tab as unknown as Window) + await open(urls.workflowsEmailDomain(SENDER_ID)) + expectStep('Add 8 settings at Cloudflare') + + click('email-domain-auto-configure') + await flush() + const returnUrl = `http://localhost/project/997${urls.workflowsEmailDomain(SENDER_ID)}?domain_connect=email` + expect(backend.requestsTo('domain_connect_apply_url')[0].body).toEqual({ + context: 'email', + integration_id: SENDER_ID, + redirect_uri: returnUrl, + }) + expect(tab.location.href).toContain(`redirect_uri=${encodeURIComponent(returnUrl)}`) + expect(backend.senderConfig(SENDER_ID)?.setup_method).toBe('auto') + + backend.approveDomainConnect(DOMAIN) + cleanup() + await open(`${urls.workflowsEmailDomain(SENDER_ID)}?domain_connect=email`) + expect(router.values.location.search).toBe('') + expect(backend.requestsTo('email_status').at(-1)?.url).toContain('refresh=true') + expectStep('Checking your settings…') + expectProgress('8 of 8 found') + + backend.verifyDomain(DOMAIN) + await flush(POLL_MS) + expectStep(/You can send from/) + }) + + it('stays on the settings when the return from Cloudflare brought no records, and still finds them later', async () => { + seedSender() + await open(`${urls.workflowsEmailDomain(SENDER_ID)}?domain_connect=email`) + expectStep('Checking your settings…') + expectProgress('0 of 8 found') + expect(backend.senderConfig(SENDER_ID)?.verified).toBe(false) + + backend.publishRecords(DOMAIN) + await flush(POLL_MS) + expectProgress('8 of 8 found') + backend.verifyDomain(DOMAIN) + await flush(POLL_MS) + expectStep(/You can send from/) + }) + + it('refuses a second submit while creating, shows the error, and lets the person try again', async () => { + const release = backend.hold('integration_create') + backend.failOnce('integration_create', { + status: 400, + body: { detail: 'This domain is already used by another organization' }, + }) + await open(urls.workflowsEmailDomain('new')) + typeInto('email-domain-another-domain', 'acme.com') + click('email-domain-use-custom-domain') + + click('email-domain-continue') + click('email-domain-continue') + await flush() + expect(backend.requestsTo('integration_create')).toHaveLength(1) + + silenceKeaLoadersErrors() + release() + await flush() + resumeKeaLoadersErrors() + expect(screen.getByText('This domain is already used by another organization')).toBeInTheDocument() + expect(router.values.location.pathname).toMatch(/\/new$/) + + click('email-domain-continue') + await flush() + expect(backend.requestsTo('integration_create')).toHaveLength(2) + expectStep('Add 8 settings at Cloudflare') + }) + + it('keeps the edited sender name on screen when the save fails, and saves it on the next try', async () => { + seedSender() + backend.verifyDomain(DOMAIN) + backend.failOnce('email_update', { status: 500, body: { detail: 'Could not reach the email provider' } }) + await open(urls.workflowsEmailDomain(SENDER_ID)) + expectStep(/You can send from/) + + typeInto('email-domain-sender-name', 'Acme Team') + click('email-domain-save-sender-name') + await flush() + expect(document.getElementById('email-domain-sender-name')).toHaveValue('Acme Team') + expect(backend.senderConfig(SENDER_ID)?.name).toBe('Acme') + + click('email-domain-save-sender-name') + await flush() + expect(backend.senderConfig(SENDER_ID)?.name).toBe('Acme Team') + expect(document.querySelector('[data-attr="email-domain-save-sender-name"]')).toBeNull() + }) + + it('offers try again when the sender does not load, then shows the settings', async () => { + seedSender() + backend.failOnce('integration_get', { status: 500, body: { detail: 'Could not reach the email provider' } }) + silenceKeaLoadersErrors() + await open(urls.workflowsEmailDomain(SENDER_ID)) + resumeKeaLoadersErrors() + expectStep('We could not load your sending domain') + + click('email-domain-retry') + await flush() + expectStep('Add 8 settings at Cloudflare') + expect(backend.requestsTo('integration_get')).toHaveLength(2) + }) + + it('keeps the progress on screen when a status check fails after the records were found', async () => { + seedSender() + backend.publishRecords(DOMAIN) + await open(urls.workflowsEmailDomain(SENDER_ID)) + expectProgress('8 of 8 found') + + backend.fail('email_status', { status: 503, body: { code: 'ses_unavailable', detail: 'SES is down' } }) + await flush(POLL_MS) + expectStep('Checking your settings…') + expectProgress('8 of 8 found') + expect(document.querySelector('[data-attr="email-domain-retry"]')).toBeNull() + + backend.recover('email_status') + backend.verifyDomain(DOMAIN) + await flush(POLL_MS) + expectStep(/You can send from/) + }) + + it('falls back to the generic manual setup when host detection fails', async () => { + seedSender() + backend.fail('domain_connect_check', { status: 500, body: { detail: 'DNS lookup timed out' } }) + await open(urls.workflowsEmailDomain(SENDER_ID)) + expectStep('Add 8 settings at your DNS host') + expect(document.querySelector('[data-attr="email-domain-auto-configure"]')).toBeNull() + expect(element('email-domain-records-added')).toBeInTheDocument() + }) + + it('lets the person restart a stopped verification again after the first restart fails', async () => { + seedSender() + backend.sesReports(DOMAIN, 'failed') + backend.failOnce('email_verify', { status: 500, body: { detail: 'SES is down' } }) + await open(urls.workflowsEmailDomain(SENDER_ID)) + expectStep('Verification stopped') + + click('email-domain-restart-verification') + await flush() + expectStep('Verification stopped') + expect(element('email-domain-restart-verification')).not.toHaveAttribute('aria-disabled', 'true') + + click('email-domain-restart-verification') + await flush() + expect(backend.requestsTo('email_verify')).toHaveLength(2) + expectStep('Add 8 settings at Cloudflare') + }) +}) diff --git a/products/workflows/frontend/Channels/EmailDomain/EmailDomainScene.tsx b/products/workflows/frontend/Channels/EmailDomain/EmailDomainScene.tsx new file mode 100644 index 000000000000..6f9a07f1d83e --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/EmailDomainScene.tsx @@ -0,0 +1,32 @@ +import { useValues } from 'kea' + +import { NotFound } from 'lib/components/NotFound' +import { FEATURE_FLAGS } from 'lib/constants' +import { featureFlagLogic } from 'lib/logic/featureFlagLogic' +import { SceneExport } from 'scenes/sceneTypes' + +import { ProductKey } from '~/queries/schema/schema-general' + +import { BindEmailDomainLogics } from './BindEmailDomainLogics' +import { emailDomainLogic } from './emailDomainLogic' +import { EmailDomainLogicProps } from './emailDomainLogicProps' +import { EmailDomainSetup } from './EmailDomainSetup' + +export const scene: SceneExport = { + component: EmailDomainScene, + logic: emailDomainLogic, + paramsToProps: ({ params: { id } }): EmailDomainLogicProps => ({ id: id || 'new' }), + productKey: ProductKey.WORKFLOWS, +} + +export function EmailDomainScene({ id }: EmailDomainLogicProps): JSX.Element { + const { featureFlags } = useValues(featureFlagLogic) + if (!featureFlags[FEATURE_FLAGS.WORKFLOWS_EMAIL_DOMAIN_WIZARD]) { + return + } + return ( + + + + ) +} diff --git a/products/workflows/frontend/Channels/EmailDomain/EmailDomainSetup.tsx b/products/workflows/frontend/Channels/EmailDomain/EmailDomainSetup.tsx new file mode 100644 index 000000000000..a29a9afd22b7 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/EmailDomainSetup.tsx @@ -0,0 +1,57 @@ +import { useValues } from 'kea' +import { router } from 'kea-router' + +import { LemonSkeleton } from '@posthog/lemon-ui' + +import { urls } from 'scenes/urls' + +import { AgentHandoffModal } from './components/AgentHandoffModal' +import { StepIndicator } from './components/StepIndicator' +import { emailDomainLogic } from './emailDomainLogic' +import { DomainStep } from './steps/DomainStep' +import { ReadyStep } from './steps/ReadyStep' +import { SettingsStep } from './steps/SettingsStep' +import { UnavailableStep } from './steps/UnavailableStep' +import { VerifyingStep } from './steps/VerifyingStep' + +function CurrentStep(): JSX.Element { + const { phase } = useValues(emailDomainLogic) + switch (phase) { + case 'domain': + return ( + router.actions.replace(urls.workflowsEmailDomain(integration.id))} + /> + ) + case 'loading': + return ( +
+ + + +
+ ) + case 'unavailable': + return + case 'settings': + return + case 'verifying': + return + case 'ready': + return + } +} + +export function EmailDomainSetup(): JSX.Element { + const { phase } = useValues(emailDomainLogic) + return ( +
+
+ + +
+ +
+ ) +} diff --git a/products/workflows/frontend/Channels/EmailDomain/components/AgentHandoffModal.tsx b/products/workflows/frontend/Channels/EmailDomain/components/AgentHandoffModal.tsx new file mode 100644 index 000000000000..bcc42a21d6d9 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/components/AgentHandoffModal.tsx @@ -0,0 +1,92 @@ +import { useActions, useValues } from 'kea' + +import { IconCopy, IconExternal } from '@posthog/icons' +import { LemonButton, LemonCheckbox, LemonModal } from '@posthog/lemon-ui' + +import { emailDomainAgentLogic } from '../emailDomainAgentLogic' +import { emailDomainAutoConfigureLogic } from '../emailDomainAutoConfigureLogic' +import { emailDomainStatusLogic } from '../emailDomainStatusLogic' +import { HedgehogRobot } from '../hoggies' + +function AgentSteps({ + host, + recordCount, + usesBrowser, +}: { + host: string + recordCount: number + usesBrowser: boolean +}): JSX.Element { + return ( +
    + {usesBrowser ? ( + <> +
  1. Opens {host} in a browser you are signed in to
    2. +
  2. Adds the {recordCount} records without touching anything else
    3. +
  3. Comes back here and clicks Check again until you are ready to send
    4. + + ) : ( + <> +
  4. Reads the records through the PostHog MCP tools
    5. +
  5. Adds them at {host} through its API, or hands you an approval link
    6. +
  6. Polls PostHog until you are ready to send
    7. + + )} +
+ ) +} + +export function AgentHandoffModal(): JSX.Element { + const { agentModalOpen, agentUsesBrowser, setupPrompt, claudeUrl } = useValues(emailDomainAgentLogic) + const { closeAgentModal, setAgentUsesBrowser, copyPrompt, promptOpenedInClaude } = useActions(emailDomainAgentLogic) + const { hostName } = useValues(emailDomainAutoConfigureLogic) + const { records } = useValues(emailDomainStatusLogic) + const host = hostName ?? 'your DNS host' + + return ( + + } + onClick={copyPrompt} + data-attr="email-domain-copy-prompt" + > + Copy prompt + + } + to={claudeUrl} + targetBlank + onClick={promptOpenedInClaude} + data-attr="email-domain-open-in-claude" + > + Open in Claude + + + } + > +
+
+ + +
+ + 
+                    {setupPrompt}
+
+
+ + ) +} diff --git a/products/workflows/frontend/Channels/EmailDomain/components/AgentSetupButton.tsx b/products/workflows/frontend/Channels/EmailDomain/components/AgentSetupButton.tsx new file mode 100644 index 000000000000..0ba3cbe27c00 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/components/AgentSetupButton.tsx @@ -0,0 +1,24 @@ +import { useActions, useValues } from 'kea' + +import { IconSparkles } from '@posthog/icons' +import { LemonButton } from '@posthog/lemon-ui' + +import { emailDomainAgentLogic } from '../emailDomainAgentLogic' + +export function AgentSetupButton(): JSX.Element | null { + const { agentSetupEnabled } = useValues(emailDomainAgentLogic) + const { openAgentModal } = useActions(emailDomainAgentLogic) + if (!agentSetupEnabled) { + return null + } + return ( + } + onClick={openAgentModal} + data-attr="email-domain-agent-setup" + > + Let an agent do it + + ) +} diff --git a/products/workflows/frontend/Channels/EmailDomain/components/AnotherDomainInput.tsx b/products/workflows/frontend/Channels/EmailDomain/components/AnotherDomainInput.tsx new file mode 100644 index 000000000000..60d3fd9d65af --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/components/AnotherDomainInput.tsx @@ -0,0 +1,47 @@ +import { useActions, useValues } from 'kea' + +import { IconArrowRight } from '@posthog/icons' +import { LemonButton, LemonInput } from '@posthog/lemon-ui' + +import { DomainProblem } from '../domainInput' +import { EmailDomainWizardLogicProps, emailDomainWizardLogic } from '../emailDomainWizardLogic' + +const PROBLEM_COPY: Record = { + free_mailbox: 'That is a free mailbox provider. Use a domain you own, like yourcompany.com.', + invalid: 'That does not look like a domain. Try something like yourcompany.com.', +} + +export function AnotherDomainInput(props: EmailDomainWizardLogicProps): JSX.Element { + const logic = emailDomainWizardLogic(props) + const { customDomainInput, customDomain } = useValues(logic) + const { setCustomDomainInput, useCustomDomain } = useActions(logic) + const canUse = Boolean(customDomain.domain) && !customDomain.problem + return ( +
+ +
+ + } + data-attr="email-domain-use-custom-domain" + > + Use + +
+ {customDomain.problem && {PROBLEM_COPY[customDomain.problem]}} +
+ ) +} diff --git a/products/workflows/frontend/Channels/EmailDomain/components/BigAction.tsx b/products/workflows/frontend/Channels/EmailDomain/components/BigAction.tsx new file mode 100644 index 000000000000..d9cfba8a1ed4 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/components/BigAction.tsx @@ -0,0 +1,9 @@ +import { LemonButton, LemonButtonProps } from '@posthog/lemon-ui' + +export function BigAction({ children, ...props }: LemonButtonProps): JSX.Element { + return ( + + {children} + + ) +} diff --git a/products/workflows/frontend/Channels/EmailDomain/components/DnsHostLink.tsx b/products/workflows/frontend/Channels/EmailDomain/components/DnsHostLink.tsx new file mode 100644 index 000000000000..72689b66c08c --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/components/DnsHostLink.tsx @@ -0,0 +1,27 @@ +import { useValues } from 'kea' + +import { IconExternal } from '@posthog/icons' +import { LemonButton } from '@posthog/lemon-ui' + +import { emailDomainAutoConfigureLogic } from '../emailDomainAutoConfigureLogic' + +export function DnsHostLink({ primary }: { primary?: boolean }): JSX.Element | null { + const { dnsHost } = useValues(emailDomainAutoConfigureLogic) + if (!dnsHost) { + return null + } + return ( + } + data-attr="email-domain-open-dns-host" + > + Open {dnsHost.name} + + ) +} diff --git a/products/workflows/frontend/Channels/EmailDomain/components/DnsRecordRows.tsx b/products/workflows/frontend/Channels/EmailDomain/components/DnsRecordRows.tsx new file mode 100644 index 000000000000..625d6014b168 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/components/DnsRecordRows.tsx @@ -0,0 +1,131 @@ +import clsx from 'clsx' +import posthog from 'posthog-js' + +import { IconCheck, IconCopy, IconHourglass, IconWarning } from '@posthog/icons' +import { LemonButton, LemonTag, lemonToast } from '@posthog/lemon-ui' + +import { copyToClipboard } from 'lib/utils/copyToClipboard' + +import type { + EmailDomainSetupRecordKindEnumApi, + EmailDomainStatusRecordApi, +} from 'products/integrations/frontend/generated/api.schemas' + +export const RECORD_KIND_LABEL: Record = { + verification: 'Proves you own the domain', + dkim: 'Signs your emails (DKIM)', + spf: 'Lists who may send for you (SPF)', + mail_from_mx: 'Routes bounces back to you (MAIL FROM)', + mail_from_spf: 'Lists who may send bounces (MAIL FROM SPF)', + dmarc: 'Tells inboxes what to do with fakes (DMARC)', +} + +const RECORD_KIND_HINT: Partial> = { + spf: 'Already have an SPF record at this name? Add include:amazonses.com to it instead of adding a second one.', + mail_from_spf: + 'Already have an SPF record at this name? Add include:amazonses.com to it instead of adding a second one.', + dmarc: 'Already have a DMARC record? Keep yours and skip this one.', +} + +type RowStatus = EmailDomainStatusRecordApi['status'] | 'missing' + +const ROW_STATUS: Record = { + pending: { label: 'Not found yet', icon: , className: 'text-secondary' }, + found: { label: 'Found', icon: , className: 'text-success' }, + verified: { label: 'Verified', icon: , className: 'text-success' }, + missing: { label: 'Missing', icon: , className: 'text-danger' }, +} + +function CopyField({ + label, + value, + record, +}: { + label: string + value: string + record: EmailDomainStatusRecordApi +}): JSX.Element { + const copy = async (): Promise => { + if (await copyToClipboard(value, label, { silent: true })) { + // pinned: analytics event name + posthog.capture('email domain records copied', { kind: record.kind, field: label.toLowerCase() }) + lemonToast.success(`Copied the ${label.toLowerCase()}`) + } + } + return ( +
+ {label} +
+ + {value} + + } + tooltip={`Copy ${label.toLowerCase()}`} + onClick={() => void copy()} + data-attr="email-domain-copy-record" + /> +
+
+ ) +} + +export function DnsRecordRows({ + records, + missing, + showStatus, +}: { + records: EmailDomainStatusRecordApi[] + missing?: EmailDomainStatusRecordApi[] + showStatus: boolean +}): JSX.Element { + const rowStatus = (record: EmailDomainStatusRecordApi): RowStatus => + missing?.includes(record) ? 'missing' : record.status + const ordered = missing?.length ? [...missing, ...records.filter((record) => !missing.includes(record))] : records + return ( +
    + {ordered.map((record, index) => { + const status = ROW_STATUS[rowStatus(record)] + const isMissing = rowStatus(record) === 'missing' + const hint = isMissing || record.status === 'pending' ? RECORD_KIND_HINT[record.kind] : undefined + return ( +
  1. +
    + + {record.type} + + {RECORD_KIND_LABEL[record.kind]} + {showStatus && ( + + {status.icon} + {status.label} + + )} +
    +
    + + +
    + {hint &&

    {hint}

    } +
    2. + ) + })} +
+ ) +} diff --git a/products/workflows/frontend/Channels/EmailDomain/components/DomainCard.tsx b/products/workflows/frontend/Channels/EmailDomain/components/DomainCard.tsx new file mode 100644 index 000000000000..8d2ec92187d2 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/components/DomainCard.tsx @@ -0,0 +1,47 @@ +import clsx from 'clsx' + +import { IconCheck } from '@posthog/icons' +import { LemonTag } from '@posthog/lemon-ui' + +import { InferredDomain } from '../inferDomains' + +export function DomainCard({ + domain, + reasons, + recommended, + selected, + onSelect, +}: InferredDomain & { selected: boolean; onSelect: () => void }): JSX.Element { + return ( + + ) +} diff --git a/products/workflows/frontend/Channels/EmailDomain/components/FirstSenderCard.tsx b/products/workflows/frontend/Channels/EmailDomain/components/FirstSenderCard.tsx new file mode 100644 index 000000000000..6023c8837b47 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/components/FirstSenderCard.tsx @@ -0,0 +1,91 @@ +import { useActions, useValues } from 'kea' + +import { IconCheck, IconSend } from '@posthog/icons' +import { LemonButton, LemonInput } from '@posthog/lemon-ui' + +import { emailDomainSenderLogic } from '../emailDomainSenderLogic' + +function SenderNameField(): JSX.Element { + const { senderName, senderNameChanged, senderSaving } = useValues(emailDomainSenderLogic) + const { setSenderName, saveSender } = useActions(emailDomainSenderLogic) + const canSave = senderNameChanged && Boolean(senderName.trim()) && !senderSaving + return ( +
+ +
+ + {senderNameChanged && ( + + Save + + )} +
+
+ ) +} + +function TestEmailButton(): JSX.Element { + const { testSendResultLoading, testEmailSent, user } = useValues(emailDomainSenderLogic) + const { sendTestEmail } = useActions(emailDomainSenderLogic) + return ( + : } + loading={testSendResultLoading} + disabledReason={testSendResultLoading ? 'Sending' : !user?.email ? 'Your account has no email' : undefined} + onClick={sendTestEmail} + tooltip={user?.email ? `Sends to ${user.email}` : undefined} + data-attr="email-domain-send-test-email" + > + {testEmailSent ? 'Send another test' : 'Send me a test email'} + + ) +} + +export function FirstSenderCard(): JSX.Element { + const { senderName, fromAddress } = useValues(emailDomainSenderLogic) + return ( +
+
+

Your first sender

+

The name and address people see in their inbox.

+
+
+ +
+ + +
+
+
+ + {senderName || 'Your name'}{' '} + <{fromAddress}> + + +
+
+ ) +} diff --git a/products/workflows/frontend/Channels/EmailDomain/components/NextWorkflowCard.tsx b/products/workflows/frontend/Channels/EmailDomain/components/NextWorkflowCard.tsx new file mode 100644 index 000000000000..1341b50ba1b8 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/components/NextWorkflowCard.tsx @@ -0,0 +1,45 @@ +import { useValues } from 'kea' +import { router } from 'kea-router' + +import { IconPlus } from '@posthog/icons' +import { Link } from '@posthog/lemon-ui' + +import { urls } from 'scenes/urls' + +import { emailDomainSenderLogic } from '../emailDomainSenderLogic' +import { HedgehogMailbox } from '../hoggies' +import { BigAction } from './BigAction' + +// The id of the "Welcome email sequence" template shipped in products/workflows/backend/templates. +const WELCOME_EMAIL_TEMPLATE_ID = '019b6f44-f9a3-0000-c4a7-b8050d25d690' + +export function NextWorkflowCard(): JSX.Element { + const { domain, senderName, fromAddress } = useValues(emailDomainSenderLogic) + return ( +
+
+ +
+ Next +

Send your first email from {domain}

+

+ Start with a welcome email that goes out when someone signs up. We open the Welcome email + sequence template so you can pick {senderName}{' '} + <{fromAddress}> as the sender. +

+
+
+ } + onClick={() => router.actions.push(urls.workflowNew(), { templateId: WELCOME_EMAIL_TEMPLATE_ID })} + data-attr="email-domain-create-welcome-workflow" + > + Create the welcome email workflow + +
+ Pick another template + Go to workflows +
+
+ ) +} diff --git a/products/workflows/frontend/Channels/EmailDomain/components/SendingAddressCard.tsx b/products/workflows/frontend/Channels/EmailDomain/components/SendingAddressCard.tsx new file mode 100644 index 000000000000..012e3362fa85 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/components/SendingAddressCard.tsx @@ -0,0 +1,114 @@ +import { useActions, useValues } from 'kea' + +import { LemonButton, LemonInput, LemonSegmentedButton } from '@posthog/lemon-ui' + +import { EmailDomainWizardLogicProps, emailDomainWizardLogic } from '../emailDomainWizardLogic' + +interface SendPrefixOption { + prefix: string + label: string + why: string +} + +const SEND_PREFIX_OPTIONS: SendPrefixOption[] = [ + { prefix: 'mail', label: 'mail.', why: 'Keeps your main domain safe if a campaign ever lands in spam.' }, + { prefix: 'send', label: 'send.', why: 'Same protection, different name.' }, + { + prefix: '', + label: 'No subdomain', + why: 'Send straight from the main domain. One bad campaign can hurt your login and support emails too.', + }, +] + +const CUSTOM_PREFIX = '__custom__' +const CUSTOM_PREFIX_PLACEHOLDER = 'news' +const CUSTOM_PREFIX_WHY = 'Any name works, as long as nothing else uses it yet.' + +function SendPrefixPicker(props: EmailDomainWizardLogicProps): JSX.Element { + const logic = emailDomainWizardLogic(props) + const { selectedDomain, sendPrefix, customPrefixOpen } = useValues(logic) + const { setSendPrefix, setCustomPrefixOpen } = useActions(logic) + const preset = SEND_PREFIX_OPTIONS.find((option) => option.prefix === sendPrefix) + const custom = customPrefixOpen || !preset + const pick = (value: string): void => { + setCustomPrefixOpen(value === CUSTOM_PREFIX) + setSendPrefix(value === CUSTOM_PREFIX ? CUSTOM_PREFIX_PLACEHOLDER : value) + } + return ( +
+ Sending subdomain + ({ value: option.prefix, label: option.label })), + { value: CUSTOM_PREFIX, label: 'Custom' }, + ]} + /> + {custom && ( +
+ + .{selectedDomain} +
+ )} + {custom ? CUSTOM_PREFIX_WHY : preset.why} +
+ ) +} + +function BouncePrefixField(props: EmailDomainWizardLogicProps): JSX.Element { + const logic = emailDomainWizardLogic(props) + const { bouncePrefix, advancedOpen, sendingDomain } = useValues(logic) + const { setBouncePrefix, setAdvancedOpen } = useActions(logic) + if (!advancedOpen) { + return ( + setAdvancedOpen(true)} + data-attr="email-domain-advanced" + > + Advanced + + ) + } + return ( +
+ +
+ + .{sendingDomain} +
+ Where bounced emails return to. Almost nobody changes this. +
+ ) +} + +export function SendingAddressCard(props: EmailDomainWizardLogicProps): JSX.Element { + const { fromAddress } = useValues(emailDomainWizardLogic(props)) + return ( +
+
+ Emails will come from + {fromAddress} +
+ + +
+ ) +} diff --git a/products/workflows/frontend/Channels/EmailDomain/components/StepHeading.tsx b/products/workflows/frontend/Channels/EmailDomain/components/StepHeading.tsx new file mode 100644 index 000000000000..e3cfcdd4ace4 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/components/StepHeading.tsx @@ -0,0 +1,23 @@ +import { ComponentType, ReactNode } from 'react' + +import { Spinner } from '@posthog/lemon-ui' + +interface StepHeadingProps { + Hoggie: ComponentType<{ className?: string }> + title: ReactNode + lead?: ReactNode + busy?: boolean +} + +export function StepHeading({ Hoggie, title, lead, busy }: StepHeadingProps): JSX.Element { + return ( +
+ +

+ {busy && } + {title} +

+ {lead &&

{lead}

} +
+ ) +} diff --git a/products/workflows/frontend/Channels/EmailDomain/components/StepIndicator.tsx b/products/workflows/frontend/Channels/EmailDomain/components/StepIndicator.tsx new file mode 100644 index 000000000000..c46549367a7d --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/components/StepIndicator.tsx @@ -0,0 +1,46 @@ +import clsx from 'clsx' + +import { IconCheck } from '@posthog/icons' + +import { EmailDomainPhase } from '../emailDomainLogic' + +const STEPS: { label: string; phases: EmailDomainPhase[] }[] = [ + { label: 'Domain', phases: ['domain'] }, + { label: 'Settings', phases: ['loading', 'unavailable', 'settings', 'verifying'] }, + { label: 'Ready', phases: ['ready'] }, +] + +export function StepIndicator({ phase }: { phase: EmailDomainPhase }): JSX.Element { + const currentIndex = STEPS.findIndex((step) => step.phases.includes(phase)) + return ( +
    + {STEPS.map((step, index) => { + const done = index < currentIndex + const current = index === currentIndex + return ( +
  1. + {index > 0 && ( + + )} + + {done ? : index + 1} + + + {step.label} + +
    2. + ) + })} +
+ ) +} diff --git a/products/workflows/frontend/Channels/EmailDomain/components/VerificationStepsStrip.tsx b/products/workflows/frontend/Channels/EmailDomain/components/VerificationStepsStrip.tsx new file mode 100644 index 000000000000..e15525cba1bd --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/components/VerificationStepsStrip.tsx @@ -0,0 +1,42 @@ +import clsx from 'clsx' + +import { IconCheck, IconWarning } from '@posthog/icons' +import { Spinner } from '@posthog/lemon-ui' + +import { VerificationStep } from '../emailDomainLogic' + +const STATE_ICON: Record = { + done: , + active: , + todo: , + stuck: , +} + +export function VerificationStepsStrip({ steps }: { steps: VerificationStep[] }): JSX.Element { + return ( +
    + {steps.map((step) => ( +
  1. + + {STATE_ICON[step.state]} + + + + {step.label} + + {step.detail} + +
    2. + ))} +
+ ) +} diff --git a/products/workflows/frontend/Channels/EmailDomain/emailDomainRoutes.test.ts b/products/workflows/frontend/Channels/EmailDomain/emailDomainRoutes.test.ts new file mode 100644 index 000000000000..37173ebb4b67 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/emailDomainRoutes.test.ts @@ -0,0 +1,46 @@ +import { kea, path } from 'kea' +import { router, urlToAction } from 'kea-router' + +import { productRoutes } from '~/products' +import { initKeaTests } from '~/test/init' + +describe('email domain routes', () => { + // The sending domain page sits under /workflows/channels/email, next to the '/workflows/:id/:tab' + // workflow route. kea-router takes the first match, so the page must never resolve as a workflow. + let resolvedScene: string | null = null + + const routeLogic = kea([ + path(['products', 'workflows', 'frontend', 'Channels', 'EmailDomain', 'emailDomainRoutesTest']), + urlToAction(() => + Object.fromEntries( + Object.entries(productRoutes) + .filter(([route]) => route.startsWith('/workflows')) + .map(([route, [scene]]) => [ + route, + () => { + resolvedScene = scene + }, + ]) + ) + ), + ]) + + beforeEach(() => { + resolvedScene = null + initKeaTests(false) + routeLogic.mount() + }) + + afterEach(() => routeLogic.unmount()) + + it.each([ + ['/workflows/channels/email/new', 'WorkflowsEmailDomain'], + ['/workflows/channels/email/42', 'WorkflowsEmailDomain'], + ['/workflows/channels', 'Workflows'], + ['/workflows/01a05e5e-38c2-0000-b81a-21fe48581914/workflow', 'Workflow'], + ])('opens %s in the %s scene', (url, scene) => { + router.actions.push(url) + + expect(resolvedScene).toBe(scene) + }) +}) diff --git a/products/workflows/frontend/Channels/EmailDomain/hoggies.ts b/products/workflows/frontend/Channels/EmailDomain/hoggies.ts new file mode 100644 index 000000000000..7f6f700429f6 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/hoggies.ts @@ -0,0 +1,19 @@ +import * as cowboyLassoPng from '@posthog/brand/hoggies/png/cowboy-lasso' +import * as hourglassPng from '@posthog/brand/hoggies/png/hourglass' +import * as magnifyingGlassPng from '@posthog/brand/hoggies/png/magnifying-glass' +import * as mailboxPng from '@posthog/brand/hoggies/png/mailbox' +import * as panicPng from '@posthog/brand/hoggies/png/panic' +import * as robotPng from '@posthog/brand/hoggies/png/robot' +import * as rocketPng from '@posthog/brand/hoggies/png/rocket' +import * as wizardPng from '@posthog/brand/hoggies/png/wizard-1' + +import { pngHoggie } from 'lib/brand/hoggies' + +export const HedgehogCowboyLasso = pngHoggie(cowboyLassoPng) +export const HedgehogHourglass = pngHoggie(hourglassPng) +export const HedgehogMagnifyingGlass = pngHoggie(magnifyingGlassPng) +export const HedgehogMailbox = pngHoggie(mailboxPng) +export const HedgehogPanic = pngHoggie(panicPng) +export const HedgehogRobot = pngHoggie(robotPng) +export const HedgehogRocket = pngHoggie(rocketPng) +export const HedgehogWizard = pngHoggie(wizardPng) diff --git a/products/workflows/frontend/Channels/EmailDomain/steps/DomainStep.tsx b/products/workflows/frontend/Channels/EmailDomain/steps/DomainStep.tsx new file mode 100644 index 000000000000..408762d07fc0 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/steps/DomainStep.tsx @@ -0,0 +1,104 @@ +import { useActions, useValues } from 'kea' + +import { IconArrowRight } from '@posthog/icons' +import { LemonButton, LemonSkeleton, Link } from '@posthog/lemon-ui' + +import { urls } from 'scenes/urls' + +import { AnotherDomainInput } from '../components/AnotherDomainInput' +import { BigAction } from '../components/BigAction' +import { DomainCard } from '../components/DomainCard' +import { SendingAddressCard } from '../components/SendingAddressCard' +import { StepHeading } from '../components/StepHeading' +import { EmailDomainWizardLogicProps, emailDomainWizardLogic } from '../emailDomainWizardLogic' +import { HedgehogMailbox } from '../hoggies' + +function DomainChoices(props: EmailDomainWizardLogicProps): JSX.Element { + const logic = emailDomainWizardLogic(props) + const { inferredDomains, eventHostsLoading, selectedDomain, selectedIsInferred } = useValues(logic) + const { selectDomain, clearDomain } = useActions(logic) + return ( +
+ {eventHostsLoading && inferredDomains.length === 0 && } + {inferredDomains.map((candidate) => ( + selectDomain(candidate.domain)} + /> + ))} + {selectedDomain && !selectedIsInferred && ( + + )} +
+ ) +} + +function CreateSender(props: EmailDomainWizardLogicProps): JSX.Element { + const logic = emailDomainWizardLogic(props) + const { existingSender, submitError, createdSenderLoading, bouncePrefix } = useValues(logic) + const { clearDomain, createSender } = useActions(logic) + const disabledReason = createdSenderLoading + ? 'Creating the sender' + : existingSender + ? 'A sender for this domain already exists' + : !bouncePrefix + ? 'Type a bounce subdomain' + : undefined + return ( + <> + {existingSender && ( +

+ A sender for this domain already exists in this project.{' '} + Open it to add settings or check its + status. +

+ )} + {submitError &&

{submitError}

} +
+ } + loading={createdSenderLoading} + disabledReason={disabledReason} + data-attr="email-domain-continue" + > + Continue + + + Another domain + +
+ + ) +} + +export function DomainStep(props: EmailDomainWizardLogicProps): JSX.Element { + const { inferredDomains, eventHostsLoading, selectedDomain } = useValues(emailDomainWizardLogic(props)) + const lead = + eventHostsLoading || inferredDomains.length > 0 + ? 'We looked at your project and found these. Pick one or type your own.' + : 'Type the domain your emails should come from.' + + return ( +
+ + + {selectedDomain ? ( + <> + + + + ) : ( + + )} +
+ ) +} diff --git a/products/workflows/frontend/Channels/EmailDomain/steps/ReadyStep.tsx b/products/workflows/frontend/Channels/EmailDomain/steps/ReadyStep.tsx new file mode 100644 index 000000000000..39a5b77b895e --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/steps/ReadyStep.tsx @@ -0,0 +1,50 @@ +import { useValues } from 'kea' + +import { IconCheck } from '@posthog/icons' +import { LemonCollapse } from '@posthog/lemon-ui' + +import { DnsRecordRows } from '../components/DnsRecordRows' +import { FirstSenderCard } from '../components/FirstSenderCard' +import { NextWorkflowCard } from '../components/NextWorkflowCard' +import { StepHeading } from '../components/StepHeading' +import { emailDomainAutoConfigureLogic } from '../emailDomainAutoConfigureLogic' +import { emailDomainSenderLogic } from '../emailDomainSenderLogic' +import { emailDomainStatusLogic } from '../emailDomainStatusLogic' +import { HedgehogRocket } from '../hoggies' + +export function ReadyStep(): JSX.Element { + const { domain } = useValues(emailDomainSenderLogic) + const { records } = useValues(emailDomainStatusLogic) + const { hostName } = useValues(emailDomainAutoConfigureLogic) + return ( +
+
+ + You can send from {domain} + + } + lead={`All ${records.length} settings are in place. Inboxes now know these emails really come from you.`} + /> +
+ + + + {records.length} settings in place at {hostName ?? 'your DNS host'} + + ), + content: , + }, + ]} + /> + +
+ ) +} diff --git a/products/workflows/frontend/Channels/EmailDomain/steps/SettingsStep.tsx b/products/workflows/frontend/Channels/EmailDomain/steps/SettingsStep.tsx new file mode 100644 index 000000000000..11ee86152917 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/steps/SettingsStep.tsx @@ -0,0 +1,179 @@ +import { useActions, useValues } from 'kea' + +import { IconRefresh } from '@posthog/icons' +import { LemonButton } from '@posthog/lemon-ui' + +import { AgentSetupButton } from '../components/AgentSetupButton' +import { BigAction } from '../components/BigAction' +import { DnsHostLink } from '../components/DnsHostLink' +import { DnsRecordRows } from '../components/DnsRecordRows' +import { StepHeading } from '../components/StepHeading' +import { VerificationStepsStrip } from '../components/VerificationStepsStrip' +import { dnsHostGotcha } from '../dnsHosts' +import { emailDomainAutoConfigureLogic } from '../emailDomainAutoConfigureLogic' +import { emailDomainLogic } from '../emailDomainLogic' +import { emailDomainManualSetupLogic } from '../emailDomainManualSetupLogic' +import { emailDomainSenderLogic } from '../emailDomainSenderLogic' +import { emailDomainStatusLogic } from '../emailDomainStatusLogic' +import { HedgehogMagnifyingGlass, HedgehogWizard } from '../hoggies' + +const RECORDS_EXPLAINER = + 'Each setting is a DNS record. Add them exactly as shown, name and value, and keep the rest of your DNS as it is.' + +function Gotcha(): JSX.Element { + const { hostName } = useValues(emailDomainAutoConfigureLogic) + return

Heads up: {dnsHostGotcha(hostName)}

+} + +function AddedThemButton({ primary }: { primary?: boolean }): JSX.Element { + const { statusLoading } = useValues(emailDomainStatusLogic) + const { markRecordsAdded } = useActions(emailDomainManualSetupLogic) + const Button = primary ? BigAction : LemonButton + return ( + + ) +} + +function AutoConfigureChoice(): JSX.Element { + const { hostName, autoConfigureLoading } = useValues(emailDomainAutoConfigureLogic) + const { openAutoConfigure } = useActions(emailDomainAutoConfigureLogic) + const { recordsRevealed } = useValues(emailDomainManualSetupLogic) + const { revealRecords } = useActions(emailDomainManualSetupLogic) + const { records } = useValues(emailDomainStatusLogic) + return ( +
+
+ + Add them to {hostName} for me + + You approve it once in {hostName}. Nothing else changes. +
+ {!recordsRevealed && ( + + I'll add them myself + + )} + +
+
+ {recordsRevealed && ( +
+

{RECORDS_EXPLAINER}

+ + +
+ + +
+
+ )} +
+ ) +} + +function ManualChoice(): JSX.Element { + const { dnsHost } = useValues(emailDomainAutoConfigureLogic) + const { records } = useValues(emailDomainStatusLogic) + return ( +
+
+ {dnsHost ? ( + <> + + + + ) : ( + + )} + +
+ +

{RECORDS_EXPLAINER}

+ +
+ ) +} + +function NoRecordsYet(): JSX.Element { + const { restartResultLoading } = useValues(emailDomainStatusLogic) + const { restartVerification } = useActions(emailDomainStatusLogic) + return ( +
+

+ The email provider has not handed out your settings yet. Start verification to get them. +

+ } + onClick={restartVerification} + loading={restartResultLoading} + disabledReason={restartResultLoading ? 'Starting' : undefined} + data-attr="email-domain-start-verification" + > + Start verification + +
+ ) +} + +function FindingHost({ domain, count }: { domain: string | null; count: number }): JSX.Element { + return ( +
+ + + {count ? `Add ${count} settings` : 'Get your settings'} + +
+ ) +} + +export function SettingsStep(): JSX.Element { + const { domain } = useValues(emailDomainSenderLogic) + const { detectionLoading, hostName, supportsDomainConnect } = useValues(emailDomainAutoConfigureLogic) + const { records } = useValues(emailDomainStatusLogic) + const { verificationSteps } = useValues(emailDomainLogic) + const count = records.length + const host = hostName ?? 'your DNS host' + + if (detectionLoading) { + return + } + + const lead = supportsDomainConnect + ? `${hostName} runs ${domain}'s DNS and lets PostHog add settings for you.` + : hostName + ? `${hostName} runs ${domain}'s DNS. Add these there, then come back.` + : `We could not tell who runs ${domain}'s DNS. Open your DNS provider's settings and add these.` + + return ( +
+ + {count === 0 ? : supportsDomainConnect ? : } +
+ What happens next + +
+
+ ) +} diff --git a/products/workflows/frontend/Channels/EmailDomain/steps/UnavailableStep.tsx b/products/workflows/frontend/Channels/EmailDomain/steps/UnavailableStep.tsx new file mode 100644 index 000000000000..f0bdd20814b8 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/steps/UnavailableStep.tsx @@ -0,0 +1,32 @@ +import { useActions, useValues } from 'kea' + +import { IconRefresh } from '@posthog/icons' + +import { BigAction } from '../components/BigAction' +import { StepHeading } from '../components/StepHeading' +import { emailDomainLogic } from '../emailDomainLogic' +import { HedgehogPanic } from '../hoggies' + +export function UnavailableStep(): JSX.Element { + const { senderLoading, statusLoading } = useValues(emailDomainLogic) + const { retry } = useActions(emailDomainLogic) + const retrying = senderLoading || statusLoading + return ( +
+ + } + onClick={retry} + loading={retrying} + disabledReason={retrying ? 'Loading' : undefined} + data-attr="email-domain-retry" + > + Try again + +
+ ) +} diff --git a/products/workflows/frontend/Channels/EmailDomain/steps/VerifyingStep.tsx b/products/workflows/frontend/Channels/EmailDomain/steps/VerifyingStep.tsx new file mode 100644 index 000000000000..fe9185c98f34 --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/steps/VerifyingStep.tsx @@ -0,0 +1,140 @@ +import { useActions, useValues } from 'kea' + +import { IconRefresh } from '@posthog/icons' +import { LemonBanner } from '@posthog/lemon-ui' + +import { LemonProgress } from 'lib/lemon-ui/LemonProgress' + +import { AgentSetupButton } from '../components/AgentSetupButton' +import { BigAction } from '../components/BigAction' +import { DnsHostLink } from '../components/DnsHostLink' +import { DnsRecordRows, RECORD_KIND_LABEL } from '../components/DnsRecordRows' +import { StepHeading } from '../components/StepHeading' +import { VerificationStepsStrip } from '../components/VerificationStepsStrip' +import { emailDomainAutoConfigureLogic } from '../emailDomainAutoConfigureLogic' +import { emailDomainLogic } from '../emailDomainLogic' +import { emailDomainManualSetupLogic } from '../emailDomainManualSetupLogic' +import { emailDomainStatusLogic } from '../emailDomainStatusLogic' +import { HedgehogHourglass, HedgehogPanic } from '../hoggies' + +function Heading(): JSX.Element { + const { status, pollingStopped, missingRecords, foundCount, records } = useValues(emailDomainStatusLogic) + const { hostName } = useValues(emailDomainAutoConfigureLogic) + const { recordsAdded } = useValues(emailDomainManualSetupLogic) + const host = hostName ?? 'your DNS host' + if (status?.status === 'failed') { + return ( + + ) + } + if (pollingStopped) { + const missing = missingRecords.length + return ( + + ) + } + return ( + + ) +} + +function SecondaryActions(): JSX.Element { + return ( +
+ + +
+ ) +} + +function Actions(): JSX.Element { + const { status, pollingStopped, statusLoading, restartResultLoading } = useValues(emailDomainStatusLogic) + const { checkAgain, restartVerification } = useActions(emailDomainStatusLogic) + if (status?.status === 'failed') { + return ( +
+ } + onClick={restartVerification} + loading={restartResultLoading} + disabledReason={restartResultLoading ? 'Restarting' : undefined} + data-attr="email-domain-restart-verification" + > + Restart verification + + +
+ ) + } + if (pollingStopped) { + return ( +
+ + Still waiting on DNS. We stopped checking automatically, so check again once you have added the + missing settings. + + } + onClick={checkAgain} + loading={statusLoading} + disabledReason={statusLoading ? 'Checking' : undefined} + data-attr="email-domain-check-again" + > + Check again + + +
+ ) + } + return +} + +export function VerifyingStep(): JSX.Element { + const { status, pollingStopped, missingRecords, foundCount, records } = useValues(emailDomainStatusLogic) + const { verificationSteps } = useValues(emailDomainLogic) + const stuck = pollingStopped || status?.status === 'failed' + const temporaryFailure = status?.status === 'temporary_failure' + return ( +
+ +
+ + + {foundCount} of {records.length} found + +
+ + {temporaryFailure && ( + + The email provider could not find these settings:{' '} + {missingRecords.map((record) => RECORD_KIND_LABEL[record.kind]).join(', ') || 'some of them'}. Check + them at your DNS host. We keep checking. + + )} + + +
+ ) +} diff --git a/products/workflows/manifest.tsx b/products/workflows/manifest.tsx index f2d585e2c0aa..ef3d3211811b 100644 --- a/products/workflows/manifest.tsx +++ b/products/workflows/manifest.tsx @@ -43,11 +43,18 @@ export const manifest: ProductManifest = { projectBased: true, description: 'Send a one-time or scheduled email to a group of people', }, + WorkflowsEmailDomain: { + import: () => import('./frontend/Channels/EmailDomain/EmailDomainScene'), + name: 'Workflows', + iconType: 'workflows', + projectBased: true, + }, }, routes: { // URL: [Scene, SceneKey] '/workflows': ['Workflows', 'workflows'], '/workflows/:tab': ['Workflows', 'workflows'], + '/workflows/channels/email/:id': ['WorkflowsEmailDomain', 'workflowsEmailDomain'], '/workflows/:id/:tab': ['Workflow', 'workflowTab'], '/workflows/library/templates/:id': ['WorkflowsLibraryTemplate', 'workflowsLibraryTemplate'], '/workflows/library/templates/new': ['WorkflowsLibraryTemplate', 'workflowsLibraryTemplate'], From 2f974741e62d2cc09f9d59df2fb73ba0c0d445ed Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Fri, 2 Oct 2026 05:14:52 +0000 Subject: [PATCH 13/13] feat(workflows): open the email domain wizard from channels, configure and broadcasts With workflows-email-domain-wizard on, the empty channels page, the new channel menu and configure on an unverified sender open the sending domain page, and the broadcast sender modal shows the wizard inline. The modal reports the new sender to the broadcast right after creating it and stays open for the DNS settings. With the flag off, every entry point keeps the old email sender modal. Rendered tests open the setup from each entry point with the flag off and on. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../IntegrationEmailDomainView.tsx | 22 ++- .../Broadcasts/steps/BroadcastContentStep.tsx | 19 ++- .../EmailDomain/EmailDomainSetupModal.tsx | 97 ++++++++++++ .../emailDomainEntryPoints.test.tsx | 140 ++++++++++++++++++ .../EmailDomain/useStartEmailDomainSetup.ts | 20 +++ .../Channels/EmailSetup/EmailSetupModal.tsx | 15 +- .../frontend/Channels/MessageChannels.tsx | 4 +- .../frontend/MessagingTabActions.tsx | 4 +- 8 files changed, 310 insertions(+), 11 deletions(-) create mode 100644 products/workflows/frontend/Channels/EmailDomain/EmailDomainSetupModal.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/emailDomainEntryPoints.test.tsx create mode 100644 products/workflows/frontend/Channels/EmailDomain/useStartEmailDomainSetup.ts diff --git a/frontend/src/lib/integrations/IntegrationEmailDomainView.tsx b/frontend/src/lib/integrations/IntegrationEmailDomainView.tsx index f4fef5bc1777..77c4eef678db 100644 --- a/frontend/src/lib/integrations/IntegrationEmailDomainView.tsx +++ b/frontend/src/lib/integrations/IntegrationEmailDomainView.tsx @@ -1,9 +1,15 @@ -import { useActions } from 'kea' +import { useActions, useValues } from 'kea' +import { router } from 'kea-router' +import posthog from 'posthog-js' import { useState } from 'react' import { IconCollapse, IconExpand, IconGear, IconLetter, IconTrash } from '@posthog/icons' import { LemonButton, LemonTag, Tooltip } from '@posthog/lemon-ui' +import { FEATURE_FLAGS } from 'lib/constants' +import { featureFlagLogic } from 'lib/logic/featureFlagLogic' +import { urls } from 'scenes/urls' + import { EmailIntegrationDomainGroupedType, IntegrationType } from '~/types' import { ChannelType } from 'products/workflows/frontend/Channels/MessageChannels' @@ -29,7 +35,17 @@ export function IntegrationEmailDomainView({ integration: EmailIntegrationDomainGroupedType }): JSX.Element { const { openSetupModal, deleteIntegration } = useActions(integrationsLogic) + const { featureFlags } = useValues(featureFlagLogic) const { domain, integrations } = integration + const configure = (sender: IntegrationType): void => { + if (featureFlags[FEATURE_FLAGS.WORKFLOWS_EMAIL_DOMAIN_WIZARD]) { + // pinned: analytics event name + posthog.capture('email domain setup started', { entry: 'configure' }) + router.actions.push(urls.workflowsEmailDomain(sender.id)) + return + } + openSetupModal(sender, sender.kind as ChannelType) + } const verified = integrations.every(isVerified) const verificationRequired = integrations.some(isVerificationRequired) const [isExpanded, setIsExpanded] = useState(false) @@ -81,9 +97,7 @@ export function IntegrationEmailDomainView({ { - openSetupModal(integration, integration.kind as ChannelType) - }} + onClick={() => configure(integration)} icon={} > Configure diff --git a/products/workflows/frontend/Broadcasts/steps/BroadcastContentStep.tsx b/products/workflows/frontend/Broadcasts/steps/BroadcastContentStep.tsx index cb95137ddb59..764cb7589812 100644 --- a/products/workflows/frontend/Broadcasts/steps/BroadcastContentStep.tsx +++ b/products/workflows/frontend/Broadcasts/steps/BroadcastContentStep.tsx @@ -3,7 +3,9 @@ import { useState } from 'react' import { LemonBanner, LemonButton, LemonSwitch } from '@posthog/lemon-ui' +import { FEATURE_FLAGS } from 'lib/constants' import { integrationsLogic } from 'lib/integrations/integrationsLogic' +import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { EmailTemplater, TemplatePickerModal } from 'scenes/hog-functions/email-templater/EmailTemplater' import type { EmailFieldErrors, EmailTemplate } from 'scenes/hog-functions/email-templater/types' @@ -26,6 +28,7 @@ export function BroadcastContentStep(): JSX.Element { const { setEmail, setEmailSettings } = useActions(broadcastWizardLogic) const { integrations, integrationsLoading } = useValues(integrationsLogic) const { loadIntegrations } = useActions(integrationsLogic) + const { featureFlags } = useValues(featureFlagLogic) const [templatePickerOpen, setTemplatePickerOpen] = useState(false) // null: closed. 'new': set up a sender. An integration: finish verifying that one. const [senderSetup, setSenderSetup] = useState<'new' | IntegrationType | null>(null) @@ -46,9 +49,7 @@ export function BroadcastContentStep(): JSX.Element { } : email - // Closing the modal after Continue also keeps the sender it created or verified. - const closeSenderSetup = (integrationId?: number): void => { - setSenderSetup(null) + const selectSender = (integrationId?: number): void => { if (integrationId) { loadIntegrations() setEmail({ @@ -62,6 +63,15 @@ export function BroadcastContentStep(): JSX.Element { }) } } + // Closing the modal after Continue also keeps the sender it created or verified. + const closeSenderSetup = (integrationId?: number): void => { + setSenderSetup(null) + selectSender(integrationId) + } + // The domain wizard reports the sender right after creating it and stays open for the DNS settings. + const completeSenderSetup = featureFlags[FEATURE_FLAGS.WORKFLOWS_EMAIL_DOMAIN_WIZARD] + ? selectSender + : closeSenderSetup const errors = stepValidationErrors.content const fieldErrors: EmailFieldErrors = { @@ -115,8 +125,9 @@ export function BroadcastContentStep(): JSX.Element { {senderSetup ? ( ) : null} void + onClose: (savedIntegrationId?: number) => void +} + +function ModalStep(): JSX.Element { + const { phase } = useValues(emailDomainLogic) + switch (phase) { + case 'unavailable': + return + case 'settings': + return + case 'verifying': + return + case 'ready': + return + default: + return + } +} + +function ModalSettings({ integrationId }: { integrationId: number }): JSX.Element { + return ( + +
+ + } + > + Open full page + + +
+ + ) +} + +export function EmailDomainSetupModal({ + integration, + entry, + onComplete, + onClose, +}: EmailDomainSetupModalProps): JSX.Element { + const [integrationId, setIntegrationId] = useState(integration?.id ?? null) + + useEffect(() => { + // pinned: analytics event name + posthog.capture('email domain setup started', { entry }) + }, [entry]) + + return ( + onClose(integrationId ?? undefined)} + > + {integrationId ? ( + + ) : ( +
+ { + setIntegrationId(created.id) + onComplete(created.id) + }} + /> +
+ )} + + ) +} diff --git a/products/workflows/frontend/Channels/EmailDomain/emailDomainEntryPoints.test.tsx b/products/workflows/frontend/Channels/EmailDomain/emailDomainEntryPoints.test.tsx new file mode 100644 index 000000000000..9b354973c55d --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/emailDomainEntryPoints.test.tsx @@ -0,0 +1,140 @@ +import '@testing-library/jest-dom' + +import { act, cleanup, fireEvent, render, screen } from '@testing-library/react' +import { Provider } from 'kea' +import { router } from 'kea-router' + +import { FEATURE_FLAGS } from 'lib/constants' +import { featureFlagLogic } from 'lib/logic/featureFlagLogic' +import { urls } from 'scenes/urls' + +import { useMocks } from '~/mocks/jest' +import { initKeaTests } from '~/test/init' + +import { EmailSetupModal } from '../EmailSetup/EmailSetupModal' +import { MessageChannels } from '../MessageChannels' +import { FakeEmailDomainBackend } from './__mocks__/fakeEmailDomainBackend' + +const DOMAIN = 'mail.acme.com' +const SENDER_ID = 42 +const LEGACY_MODAL_TITLE = 'Configure email sender' + +const element = (attr: string): HTMLElement => { + const found = document.querySelector(`[data-attr="${attr}"]`) + if (!found) { + throw new Error(`No element with data-attr ${attr} on the page`) + } + return found +} + +const flush = async (): Promise => { + await act(async () => { + await jest.advanceTimersByTimeAsync(0) + }) +} + +const renderInApp = async (ui: JSX.Element): Promise => { + render({ui}) + await flush() +} + +type EntryExpectation = (pagePath: string) => void + +const expectOldModal: EntryExpectation = () => { + expect(screen.getByText(LEGACY_MODAL_TITLE)).toBeInTheDocument() + expect(router.values.location.pathname).toMatch(/\/workflows\/channels$/) +} + +const expectSendingDomainPage: EntryExpectation = (pagePath) => { + expect(router.values.location.pathname).toMatch(new RegExp(`${pagePath}$`)) + expect(screen.queryByText(LEGACY_MODAL_TITLE)).not.toBeInTheDocument() +} + +const expectOldBroadcastModal = (): void => { + expect(screen.getByText(LEGACY_MODAL_TITLE)).toBeInTheDocument() +} + +const expectWizardInBroadcastModal = (): void => { + expect(screen.getByText('Set up a sending domain')).toBeInTheDocument() + expect(screen.getByText('Which domain should your emails come from?')).toBeInTheDocument() +} + +describe('email domain setup entry points', () => { + let backend: FakeEmailDomainBackend + + const setWizardFlag = (enabled: boolean): void => { + featureFlagLogic.actions.setFeatureFlags(enabled ? [FEATURE_FLAGS.WORKFLOWS_EMAIL_DOMAIN_WIZARD] : [], { + [FEATURE_FLAGS.WORKFLOWS_EMAIL_DOMAIN_WIZARD]: enabled, + }) + } + + beforeEach(() => { + jest.useFakeTimers() + backend = new FakeEmailDomainBackend() + backend.hostDnsAt('acme.com', 'cloudflare') + useMocks(backend.mocks()) + initKeaTests() + featureFlagLogic.mount() + router.actions.push(urls.workflows('channels')) + }) + + afterEach(() => { + cleanup() + jest.useRealTimers() + }) + + describe.each([ + ['off', false, expectOldModal, expectOldBroadcastModal], + ['on', true, expectSendingDomainPage, expectWizardInBroadcastModal], + ])('with the wizard flag %s', (_, wizardEnabled, expectEntry, expectBroadcastModal) => { + beforeEach(() => setWizardFlag(wizardEnabled)) + + it('starts from the empty channels page', async () => { + await renderInApp() + expect(screen.queryByText(LEGACY_MODAL_TITLE)).not.toBeInTheDocument() + fireEvent.click(element('create-channel-integration')) + await flush() + expectEntry(urls.workflowsEmailDomain('new')) + }) + + it('starts from configure on an unverified sender', async () => { + backend.addSender({ email: `hello@${DOMAIN}` }) + await renderInApp() + fireEvent.click(screen.getByText(DOMAIN)) + expect(screen.getByText('Unverified')).toBeInTheDocument() + fireEvent.click(screen.getByText('Configure')) + await flush() + expectEntry(urls.workflowsEmailDomain(SENDER_ID)) + }) + + it('starts from the broadcast sender modal', async () => { + const onComplete = jest.fn() + await renderInApp() + expectBroadcastModal() + expect(onComplete).not.toHaveBeenCalled() + }) + }) + + it('reports the new sender to the broadcast right after creating it and stays open for the settings', async () => { + setWizardFlag(true) + const onComplete = jest.fn() + const onClose = jest.fn() + await renderInApp() + + fireEvent.change(document.getElementById('email-domain-another-domain') as HTMLInputElement, { + target: { value: 'acme.com' }, + }) + fireEvent.click(element('email-domain-use-custom-domain')) + fireEvent.click(element('email-domain-continue')) + await flush() + + expect(onComplete).toHaveBeenCalledWith(SENDER_ID) + expect(onClose).not.toHaveBeenCalled() + expect(screen.getByText('Verify your sending domain')).toBeInTheDocument() + expect(screen.getByText('Add 8 settings at Cloudflare')).toBeInTheDocument() + expect(screen.getByText('Open full page').closest('a')).toHaveAttribute( + 'href', + expect.stringContaining(urls.workflowsEmailDomain(SENDER_ID)) + ) + }) +}) diff --git a/products/workflows/frontend/Channels/EmailDomain/useStartEmailDomainSetup.ts b/products/workflows/frontend/Channels/EmailDomain/useStartEmailDomainSetup.ts new file mode 100644 index 000000000000..5088f81ce28f --- /dev/null +++ b/products/workflows/frontend/Channels/EmailDomain/useStartEmailDomainSetup.ts @@ -0,0 +1,20 @@ +import { useValues } from 'kea' +import { router } from 'kea-router' +import posthog from 'posthog-js' + +import { FEATURE_FLAGS } from 'lib/constants' +import { featureFlagLogic } from 'lib/logic/featureFlagLogic' +import { urls } from 'scenes/urls' + +/** Opens the sending domain page when the wizard flag is on. Returns null when the flag is off, so the caller keeps its modal. */ +export function useStartEmailDomainSetup(entry: 'channels' | 'broadcast'): (() => void) | null { + const { featureFlags } = useValues(featureFlagLogic) + if (!featureFlags[FEATURE_FLAGS.WORKFLOWS_EMAIL_DOMAIN_WIZARD]) { + return null + } + return () => { + // pinned: analytics event name + posthog.capture('email domain setup started', { entry }) + router.actions.push(urls.workflowsEmailDomain('new')) + } +} diff --git a/products/workflows/frontend/Channels/EmailSetup/EmailSetupModal.tsx b/products/workflows/frontend/Channels/EmailSetup/EmailSetupModal.tsx index b0f59a820fb2..44aa50429829 100644 --- a/products/workflows/frontend/Channels/EmailSetup/EmailSetupModal.tsx +++ b/products/workflows/frontend/Channels/EmailSetup/EmailSetupModal.tsx @@ -15,11 +15,24 @@ import { import { DomainConnectBanner } from 'lib/components/DomainConnect' import { FlaggedFeature } from 'lib/components/FlaggedFeature' +import { FEATURE_FLAGS } from 'lib/constants' import { LemonField } from 'lib/lemon-ui/LemonField' +import { featureFlagLogic } from 'lib/logic/featureFlagLogic' +import { EmailDomainSetupModal, EmailDomainSetupModalProps } from '../EmailDomain/EmailDomainSetupModal' import { DnsRecord, EmailSetupModalLogicProps, emailSetupModalLogic } from './emailSetupModalLogic' -export const EmailSetupModal = (props: EmailSetupModalLogicProps): JSX.Element => { +export type EmailSetupModalProps = EmailSetupModalLogicProps & { entry?: EmailDomainSetupModalProps['entry'] } + +export const EmailSetupModal = ({ entry = 'channels', ...props }: EmailSetupModalProps): JSX.Element => { + const { featureFlags } = useValues(featureFlagLogic) + if (featureFlags[FEATURE_FLAGS.WORKFLOWS_EMAIL_DOMAIN_WIZARD]) { + return + } + return +} + +const LegacyEmailSetupModal = (props: EmailSetupModalLogicProps): JSX.Element => { const logic = emailSetupModalLogic(props) const { savedIntegration, verificationLoading, isEmailSenderSubmitting, dnsRecords, domain, isDomainVerified } = useValues(logic) diff --git a/products/workflows/frontend/Channels/MessageChannels.tsx b/products/workflows/frontend/Channels/MessageChannels.tsx index 3836e47d18dd..dd801a7e4c05 100644 --- a/products/workflows/frontend/Channels/MessageChannels.tsx +++ b/products/workflows/frontend/Channels/MessageChannels.tsx @@ -11,6 +11,7 @@ import { IntegrationsList } from 'lib/integrations/IntegrationsList' import { integrationsLogic } from 'lib/integrations/integrationsLogic' import { ChannelSetupModal } from './ChannelSetupModal' +import { useStartEmailDomainSetup } from './EmailDomain/useStartEmailDomainSetup' const HedgehogReporter = pngHoggie(reporterPng) @@ -21,6 +22,7 @@ export function MessageChannels(): JSX.Element { const { setupModalOpen, integrations, integrationsLoading, setupModalType, selectedIntegration } = useValues(integrationsLogic) const { openSetupModal, closeSetupModal, markTaskAsCompleted } = useActions(integrationsLogic) + const startEmailDomainSetup = useStartEmailDomainSetup('channels') const allWorkflowIntegrations = integrations?.filter((integration) => MESSAGING_CHANNEL_TYPES.includes(integration.kind as ChannelType)) ?? [] @@ -53,7 +55,7 @@ export function MessageChannels(): JSX.Element { thingName="channel integration" description="Set up messaging channels to automatically send emails, SMS, or Slack notifications triggered by user actions and events." docsURL="https://posthog.com/docs/workflows/configure-channels" - action={() => openSetupModal(undefined, 'email')} + action={startEmailDomainSetup ?? (() => openSetupModal(undefined, 'email'))} customHog={HedgehogReporter} isEmpty /> diff --git a/products/workflows/frontend/MessagingTabActions.tsx b/products/workflows/frontend/MessagingTabActions.tsx index 9281c6087427..01e3f1940d9c 100644 --- a/products/workflows/frontend/MessagingTabActions.tsx +++ b/products/workflows/frontend/MessagingTabActions.tsx @@ -13,6 +13,7 @@ import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { AccessControlLevel, AccessControlResourceType } from '~/types' +import { useStartEmailDomainSetup } from './Channels/EmailDomain/useStartEmailDomainSetup' import type { MessagingNavTabKey } from './messagingTabs' import { optOutCategoriesLogic } from './OptOuts/optOutCategoriesLogic' import { newTemplateAgentLogic } from './TemplateLibrary/newTemplateAgentLogic' @@ -56,6 +57,7 @@ function NewTemplateButton(): JSX.Element { function NewChannelButton({ channelsUrl }: { channelsUrl: string }): JSX.Element { const { featureFlags } = useValues(featureFlagLogic) const { openSetupModal } = useActions(integrationsLogic) + const startEmailDomainSetup = useStartEmailDomainSetup('channels') const newChannelRestrictedReason = useRestrictedArea({ scope: RestrictionScope.Project, minimumAccessLevel: TeamMembershipLevel.Admin, @@ -68,7 +70,7 @@ function NewChannelButton({ channelsUrl }: { channelsUrl: string }): JSX.Element Email ), - onClick: () => openSetupModal(undefined, 'email'), + onClick: startEmailDomainSetup ?? (() => openSetupModal(undefined, 'email')), }, {