From 93895bc8d787759a352d4dd10005841cac334e86 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 7 Oct 2026 08:23:39 +0000 Subject: [PATCH 1/3] feat(workflows): enable gated agent email domain setup --- posthog/api/integration.py | 189 +++++---- posthog/api/integration_domain_serializers.py | 204 ++++++++++ posthog/api/test/test_integration.py | 346 ++++++++++++++++- .../test_integration_domain_serializers.py | 57 +++ posthog/domain_connect.py | 4 +- posthog/models/integration/email.py | 4 + .../frontend/generated/api.schemas.ts | 209 +++++++++- .../integrations/frontend/generated/api.ts | 67 +++- .../frontend/generated/api.zod.ts | 246 +++--------- products/integrations/mcp/tools.yaml | 117 ++++++ .../eval_harness/harness/services.py | 9 +- .../test/test_sandboxed_harness_services.py | 1 + products/workflows/evals/__init__.py | 0 .../evals/eval_email_domain_setup.py | 340 ++++++++++++++++ products/workflows/evals/scorers.py | 282 ++++++++++++++ products/workflows/evals/seeders.py | 172 +++++++++ .../evals/simulated_email_domains.py | 363 ++++++++++++++++++ products/workflows/evals/test/__init__.py | 0 products/workflows/evals/test/test_scorers.py | 40 ++ .../test/test_simulated_email_domains.py | 119 ++++++ products/workflows/package.json | 2 +- .../setting-up-an-email-domain/SKILL.md | 111 ++++++ .../references/publishing-records.md | 27 ++ .../schema/generated-tool-definitions.json | 75 ++++ services/mcp/schema/tool-definitions-all.json | 75 ++++ services/mcp/schema/tool-inputs.json | 27 ++ services/mcp/scripts/generate-tools.ts | 5 +- services/mcp/src/api/generated.ts | 219 +++++++++-- .../mcp/src/generated/integrations/api.ts | 114 +++++- services/mcp/src/schema/tool-inputs.ts | 10 + .../mcp/src/tools/generated/integrations.ts | 149 +++++++ .../mcp/tests/unit/generate-tools.test.ts | 17 +- ...egrations-domain-connect-apply-url.test.ts | 24 ++ .../mcp/tests/unit/tool-filtering.test.ts | 1 + 34 files changed, 3295 insertions(+), 330 deletions(-) create mode 100644 posthog/api/integration_domain_serializers.py create mode 100644 posthog/api/test/test_integration_domain_serializers.py create mode 100644 products/workflows/evals/__init__.py create mode 100644 products/workflows/evals/eval_email_domain_setup.py create mode 100644 products/workflows/evals/scorers.py create mode 100644 products/workflows/evals/seeders.py create mode 100644 products/workflows/evals/simulated_email_domains.py create mode 100644 products/workflows/evals/test/__init__.py create mode 100644 products/workflows/evals/test/test_scorers.py create mode 100644 products/workflows/evals/test/test_simulated_email_domains.py create mode 100644 products/workflows/skills/setting-up-an-email-domain/SKILL.md create mode 100644 products/workflows/skills/setting-up-an-email-domain/references/publishing-records.md create mode 100644 services/mcp/tests/unit/integrations-domain-connect-apply-url.test.ts diff --git a/posthog/api/integration.py b/posthog/api/integration.py index 67af063ef3dc..2452873cdb12 100644 --- a/posthog/api/integration.py +++ b/posthog/api/integration.py @@ -4,7 +4,7 @@ import time from collections.abc import Callable, Iterable from typing import Any, NoReturn, Protocol, cast -from urllib.parse import urlencode +from urllib.parse import urlencode, urlsplit from uuid import uuid4 from django.conf import settings @@ -15,6 +15,7 @@ from django.shortcuts import redirect from django.utils import timezone from django.utils.dateparse import parse_datetime +from django.utils.http import url_has_allowed_host_and_scheme import structlog from django_filters.rest_framework import DjangoFilterBackend @@ -24,7 +25,7 @@ from prometheus_client import Counter from redis.exceptions import RedisError from rest_framework import mixins, serializers, status, viewsets -from rest_framework.exceptions import APIException, PermissionDenied, Throttled, ValidationError +from rest_framework.exceptions import APIException, NotFound, PermissionDenied, Throttled, ValidationError from rest_framework.permissions import BasePermission, IsAuthenticated from rest_framework.request import Request from rest_framework.response import Response @@ -45,16 +46,34 @@ github_app_install_url, is_valid_github_installation_id, ) +from posthog.api.integration_domain_serializers import ( + DomainConnectApplyUrlRequestSerializer, + DomainConnectApplyUrlResponseSerializer, + DomainConnectCheckQuerySerializer, + DomainConnectCheckResponseSerializer, + EmailDomainVerificationSerializer, + EmailSenderUpdateRequestSerializer, + NativeEmailIntegrationSerializer, +) +from posthog.api.mixins import ValidatedRequest, validated_request from posthog.api.routing import TeamAndOrgViewSetMixin from posthog.api.shared import UserBasicSerializer from posthog.api.utils import action from posthog.auth import SessionAuthentication -from posthog.domain_connect import discover_domain_connect, extract_root_domain_and_host, get_available_providers +from posthog.domain_connect import ( + DomainConnectSigningKeyMissing, + discover_domain_connect, + extract_root_domain_and_host, + generate_apply_url, + get_available_providers, + resolve_email_context, + resolve_proxy_context, +) from posthog.egress.github.transport import GitHubRateLimitError from posthog.event_usage import report_user_action from posthog.exceptions_capture import capture_exception from posthog.helpers.fuzzy_search import fuzzy_filter -from posthog.models import OrganizationMembership, User +from posthog.models import OrganizationMembership, ProxyRecord, Team, User from posthog.models.github_integration_base import GitHubIntegrationBase from posthog.models.integration import ( ANTHROPIC_DEFAULT_INTEGRATION_ID_PREFIX, @@ -117,7 +136,9 @@ TeamMemberLightManagementPermission, TeamMemberStrictManagementPermission, TimeSensitiveActionPermission, + get_authenticator_scopes, ) +from posthog.ph_client import feature_enabled_or_false from posthog.rate_limit import GitHubRepositoryRefreshThrottle from posthog.tasks.email import send_integration_access_request from posthog.utils import absolute_uri, is_relative_url @@ -144,6 +165,8 @@ GITHUB_REPOSITORY_NAME_RE = re.compile(r"[A-Za-z0-9_.\-]+") +EMAIL_DOMAIN_AGENT_SETUP_FLAG = "workflows-email-domain-agent-setup" + class SlackIntegrationInactiveError(APIException): # Actionable 4xx (reconnect Slack) rather than a raw 500, so the channel picker can render @@ -303,16 +326,6 @@ def _github_disconnect_blocked_message(live_runs: InProgressGithubRunsDTO, team_ ) -class NativeEmailIntegrationSerializer(serializers.Serializer): - email = serializers.EmailField() - name = serializers.CharField() - provider = serializers.ChoiceField(choices=["ses", "maildev"] if settings.DEBUG else ["ses"]) - mail_from_subdomain = serializers.CharField(required=False, allow_blank=True) - - def validate_email(self, value: str) -> str: - return value.lower() - - class GitHubRepoSerializer(serializers.Serializer): id = serializers.IntegerField(help_text="GitHub repository numeric identifier.") name = serializers.CharField(help_text="Repository short name (without the owner prefix).") @@ -1448,6 +1461,28 @@ def has_object_permission(self, request: Request, view: APIView, obj: object) -> return True +def email_domain_agent_setup_enabled(team: Team, user: User) -> bool: + return feature_enabled_or_false( + EMAIL_DOMAIN_AGENT_SETUP_FLAG, + str(user.distinct_id), + groups={"organization": str(team.organization_id), "project": str(team.uuid)}, + group_properties={"organization": {"id": str(team.organization_id)}, "project": {"id": str(team.uuid)}}, + send_feature_flag_events=False, + ) + + +class EmailDomainAgentSetupPermission(BasePermission): + message = "This action does not support personal API key access" + token_actions = frozenset({"domain_connect_check", "domain_connect_apply_url", "email_update", "email_verify"}) + + def has_permission(self, request: Request, view: APIView) -> bool: + if getattr(view, "action", None) not in self.token_actions: + return True + if get_authenticator_scopes(request.successful_authenticator) is None: + return True + return email_domain_agent_setup_enabled(cast("IntegrationViewSet", view).team, cast(User, request.user)) + + @extend_schema(extensions={"x-product": "integrations"}) class IntegrationViewSet( TeamAndOrgViewSetMixin, @@ -1476,6 +1511,7 @@ class IntegrationViewSet( "anthropic_managed_agents", "anthropic_managed_agent_environments", "anthropic_managed_agent_vaults", + "domain_connect_check", ] scope_object_write_actions = [ "create", @@ -1489,8 +1525,15 @@ class IntegrationViewSet( "github_oauth_authorize", # Side-effecting POST (emails admins) — a read-only token must not be able to trigger it. "request_access", + "email_update", + "email_verify", + "domain_connect_apply_url", + ] + permission_classes = [ + IntegrationManagementPermission, + PersonalConnectionRecentAuthPermission, + EmailDomainAgentSetupPermission, ] - permission_classes = [IntegrationManagementPermission, PersonalConnectionRecentAuthPermission] # LimitOffsetPagination needs a total order, or Postgres can return a row on neither side of a # page boundary. Clients page this list to find one kind, so a dropped row reads as # "not configured". Order oldest-first: several clients take the first row of a kind as their @@ -2631,35 +2674,49 @@ def gitlab_members(self, request: Request, *args: Any, **kwargs: Any) -> Respons search = query_serializer.validated_data["search"] return assignees_response(lambda: GitLabIntegration(instance).list_assignees(search)) + @validated_request( + responses={200: EmailDomainVerificationSerializer}, + summary="Verify an email sender's domain", + description=( + "Ask the email provider to check the sending domain's DNS records and return each record with its status. " + "When every record is verified, all senders on the domain in this project become able to send." + ), + ) @action(methods=["POST"], detail=True, url_path="email/verify") - def email_verify(self, request, **kwargs): - email = EmailIntegration(self.get_object()) - verification_result = email.verify() + def email_verify(self, request: ValidatedRequest, **kwargs: Any) -> Response: + verification_result = self._get_email_integration().verify() return Response(verification_result) - @extend_schema(responses={200: IntegrationSerializer}) + @validated_request( + EmailSenderUpdateRequestSerializer, + responses={200: IntegrationSerializer}, + summary="Update an email sender", + ) @action(methods=["PATCH"], detail=True, url_path="email") - def email_update(self, request, **kwargs) -> Response: - instance = self.get_object() - config = request.data.get("config", {}) - - serializer = NativeEmailIntegrationSerializer(data=config) - serializer.is_valid(raise_exception=True) - - email = EmailIntegration(instance) - email.update_native_integration(serializer.validated_data, instance.team_id) - + def email_update(self, request: ValidatedRequest, **kwargs: Any) -> Response: + email = self._get_email_integration() + config = request.validated_data["config"] + if email_domain_agent_setup_enabled(self.team, cast(User, request.user)): + email.reject_address_change(config["email"]) + email.update_native_integration(config, email.integration.team_id) return Response(IntegrationSerializer(email.integration).data) + def _get_email_integration(self) -> EmailIntegration: + instance = self.get_object() + if instance.kind != "email": + raise ValidationError("This endpoint is only supported for email integrations") + return EmailIntegration(instance) + + @validated_request( + query_serializer=DomainConnectCheckQuerySerializer, + responses={200: DomainConnectCheckResponseSerializer}, + summary="Check Domain Connect support for a domain", + ) # nosemgrep: api-path-underscore -- shipped public API path, a rename breaks clients @action(methods=["GET"], detail=False, url_path="domain-connect/check") - def domain_connect_check(self, request: Request, *args: Any, **kwargs: Any) -> Response: - domain = request.query_params.get("domain", "") - if not domain: - raise ValidationError("domain query parameter is required") - + def domain_connect_check(self, request: ValidatedRequest, *args: Any, **kwargs: Any) -> Response: # Extract root domain so subdomains (e.g. ph.example.com) resolve correctly - domain_parts = extract_root_domain_and_host(domain) + domain_parts = extract_root_domain_and_host(request.validated_query_data["domain"]) result = discover_domain_connect(domain_parts.root_domain) return Response( { @@ -2669,49 +2726,44 @@ def domain_connect_check(self, request: Request, *args: Any, **kwargs: Any) -> R } ) + @validated_request( + DomainConnectApplyUrlRequestSerializer, + responses={200: DomainConnectApplyUrlResponseSerializer}, + summary="Generate a Domain Connect apply URL", + description=( + "Build the signed URL that sends a person to their DNS host to approve the records for an email " + "sending domain or a reverse proxy domain." + ), + ) # nosemgrep: api-path-underscore -- shipped public API path, a rename breaks clients @action(methods=["POST"], detail=False, url_path="domain-connect/apply-url") - def domain_connect_apply_url(self, request: Request, *args: Any, **kwargs: Any) -> Response: - """Unified endpoint for generating Domain Connect apply URLs. - - Accepts a context ("email" or "proxy") and the relevant resource ID. - The backend resolves the domain, template variables, and service ID - based on context, then builds the signed apply URL. - """ - from posthog.domain_connect import ( - DOMAIN_CONNECT_PROVIDERS, - DomainConnectSigningKeyMissing, - generate_apply_url, - resolve_email_context, - resolve_proxy_context, - ) - - context = request.data.get("context") - redirect_uri = request.data.get("redirect_uri") - provider_endpoint = request.data.get("provider_endpoint") + def domain_connect_apply_url(self, request: ValidatedRequest, *args: Any, **kwargs: Any) -> Response: + if not email_domain_agent_setup_enabled(self.team, cast(User, request.user)): + raise NotFound() - if provider_endpoint and provider_endpoint not in DOMAIN_CONNECT_PROVIDERS: - raise ValidationError("Unsupported provider endpoint") + data = request.validated_data + context = data["context"] + redirect_uri = self._redirect_back_to_posthog(data.get("redirect_uri")) + provider_endpoint = data.get("provider_endpoint") if context == "email": - integration_id = request.data.get("integration_id") - if not integration_id: - raise ValidationError("integration_id is required for email context") + integration_id = data["integration_id"] try: resolved = resolve_email_context(integration_id, self.team_id) + except Integration.DoesNotExist: + raise NotFound("No email sender with this integration_id exists in this project.") except ValueError as e: capture_exception(e, {"integration_id": integration_id, "team_id": self.team_id, "context": context}) raise ValidationError( "Validation error resolving email context. Please try again later or contact support." ) - - elif context == "proxy": - proxy_record_id = request.data.get("proxy_record_id") - if not proxy_record_id: - raise ValidationError("proxy_record_id is required for proxy context") + else: + proxy_record_id = data["proxy_record_id"] organization = self.organization try: - resolved = resolve_proxy_context(proxy_record_id, str(organization.id)) + resolved = resolve_proxy_context(str(proxy_record_id), str(organization.id)) + except ProxyRecord.DoesNotExist: + raise NotFound("No reverse proxy record with this proxy_record_id exists in this organization.") except ValueError as e: capture_exception( e, {"proxy_record_id": proxy_record_id, "organization_id": organization.id, "context": context} @@ -2719,8 +2771,6 @@ def domain_connect_apply_url(self, request: Request, *args: Any, **kwargs: Any) raise ValidationError( "Validation error resolving proxy context. Please try again later or contact support." ) - else: - raise ValidationError("context must be 'email' or 'proxy'") try: url = generate_apply_url( @@ -2756,6 +2806,13 @@ def domain_connect_apply_url(self, request: Request, *args: Any, **kwargs: Any) return Response({"url": url}) + @staticmethod + def _redirect_back_to_posthog(redirect_uri: str | None) -> str | None: + posthog_host = urlsplit(settings.SITE_URL).netloc + if redirect_uri and not url_has_allowed_host_and_scheme(redirect_uri, allowed_hosts={posthog_host}): + raise ValidationError("redirect_uri must point back to PostHog.") + return redirect_uri + # Defined last: a method named `list` shadows the builtin for the annotations of every method # declared after it in this class body. def list(self, request: Request, *args: Any, **kwargs: Any) -> Response: diff --git a/posthog/api/integration_domain_serializers.py b/posthog/api/integration_domain_serializers.py new file mode 100644 index 000000000000..5b56ad5fab84 --- /dev/null +++ b/posthog/api/integration_domain_serializers.py @@ -0,0 +1,204 @@ +from typing import Any + +from django.conf import settings +from django.db import models + +from rest_framework import serializers + +from posthog.domain_connect import DOMAIN_CONNECT_PROVIDERS + + +class NativeEmailProvider(models.TextChoices): + SES = "ses", "Amazon SES" + MAILDEV = "maildev", "Maildev (local development only)" + + +class EmailDomainRecordPurpose(models.TextChoices): + VERIFICATION = "verification", "Domain ownership or SPF" + DKIM = "dkim", "DKIM signing" + MAIL_FROM = "mail_from", "Custom MAIL FROM" + DMARC = "dmarc", "DMARC policy" + + +class EmailDomainRecordType(models.TextChoices): + TXT = "TXT", "TXT record" + CNAME = "CNAME", "CNAME record" + MX = "MX", "MX record" + + +class EmailDomainRecordStatus(models.TextChoices): + SUCCESS = "success", "Record found" + PENDING = "pending", "Record not found yet" + + +class EmailDomainStatus(models.TextChoices): + SUCCESS = "success", "Domain verified" + PENDING = "pending", "Verification pending" + FAILED = "failed", "Verification failed" + + +class DomainConnectContextKind(models.TextChoices): + EMAIL = "email", "Email sending domain" + PROXY = "proxy", "Reverse proxy domain" + + +class NativeEmailIntegrationSerializer(serializers.Serializer): + email = serializers.EmailField( + help_text=( + "Sender address, for example `hello@mail.example.com`. Its domain is the sending domain that needs " + "DNS records. Free and disposable mailbox domains such as gmail.com are rejected. " + "Cannot be changed after creation; send the current address when updating." + ) + ) + name = serializers.CharField(help_text="Sender display name recipients see in their inbox, for example `Acme`.") + provider = serializers.ChoiceField( + choices=NativeEmailProvider.choices, + help_text="Sending provider. Always `ses`. `maildev` is only accepted in local development.", + ) + mail_from_subdomain = serializers.CharField( + required=False, + allow_blank=True, + help_text=( + "Subdomain of the sending domain used as the custom MAIL FROM (bounce) domain. " + "`feedback` gives `feedback.mail.example.com`. Defaults to `feedback`. " + "Pick another value if that subdomain already has MX records." + ), + ) + + def validate_email(self, value: str) -> str: + return value.lower() + + def validate_provider(self, value: str) -> str: + if value == NativeEmailProvider.MAILDEV and not settings.DEBUG: + raise serializers.ValidationError(f'"{value}" is not a valid choice.', code="invalid_choice") + return value + + +class EmailSenderUpdateRequestSerializer(serializers.Serializer): + config = NativeEmailIntegrationSerializer( + help_text="The full sender config. Only `name` and `mail_from_subdomain` change; `email` must stay the same." + ) + + +class EmailDomainDnsRecordSerializer(serializers.Serializer): + type = serializers.ChoiceField( + choices=EmailDomainRecordPurpose.choices, + help_text=( + "What the record is for: domain ownership or the sending domain's SPF, DKIM signing, " + "the custom MAIL FROM domain, or DMARC." + ), + ) + recordType = serializers.ChoiceField(choices=EmailDomainRecordType.choices, help_text="DNS record type.") + recordHostname = serializers.CharField( + help_text=( + "Fully qualified record name, or `@` for the sending domain itself. " + "Many DNS hosts append the zone, so enter only the part before it." + ) + ) + recordValue = serializers.CharField(help_text="Exact record value to publish.") + status = serializers.ChoiceField( + choices=EmailDomainRecordStatus.choices, + help_text="`success` once the record is visible in DNS, `pending` until then.", + ) + priority = serializers.IntegerField(required=False, help_text="MX priority. Only present on MX records.") + + +class EmailDomainVerificationSerializer(serializers.Serializer): + status = serializers.ChoiceField( + choices=EmailDomainStatus.choices, + help_text=( + "`success` when every record is verified and the sender can send. `pending` while DNS is not visible " + "yet, which can take minutes and up to 72 hours. `failed` when the provider gave up on the records." + ), + ) + dnsRecords = EmailDomainDnsRecordSerializer( + many=True, help_text="Every DNS record the sending domain needs, each with its own status." + ) + + +class DomainConnectCheckQuerySerializer(serializers.Serializer): + domain = serializers.CharField( + help_text="Domain to check, for example `mail.example.com`. Subdomains resolve to their registrable domain.", + error_messages={ + "required": "domain query parameter is required", + "blank": "domain query parameter is required", + }, + ) + + +class DomainConnectProviderSerializer(serializers.Serializer): + endpoint = serializers.CharField(help_text="Provider endpoint. Pass it as `provider_endpoint` to apply-url.") + name = serializers.CharField(help_text="Provider display name, for example `Cloudflare`.") + + +class DomainConnectCheckResponseSerializer(serializers.Serializer): + supported = serializers.BooleanField( + help_text="True when the domain's DNS host supports one-click setup with Domain Connect." + ) + provider_name = serializers.CharField( + allow_null=True, help_text="Detected DNS host, for example `Cloudflare`. Null when not supported." + ) + available_providers = DomainConnectProviderSerializer( + many=True, + help_text=( + "Providers the user can pick by hand when detection fails. Empty when detection succeeded. " + "Only offer one if the user confirms their DNS is hosted there." + ), + ) + + +class DomainConnectApplyUrlRequestSerializer(serializers.Serializer): + context = serializers.ChoiceField( # type: ignore[assignment] + choices=DomainConnectContextKind.choices, + help_text="`email` to configure an email sending domain, `proxy` for a reverse proxy domain.", + error_messages={ + "required": "context must be 'email' or 'proxy'", + "invalid_choice": "context must be 'email' or 'proxy'", + }, + ) + integration_id = serializers.IntegerField( + required=False, + allow_null=True, + help_text="ID of the email integration (sender). Required when `context` is `email`.", + ) + proxy_record_id = serializers.UUIDField( + required=False, + allow_null=True, + help_text="ID of the reverse proxy record. Required when `context` is `proxy`.", + ) + redirect_uri = serializers.CharField( + required=False, + allow_null=True, + allow_blank=True, + help_text="Where the DNS host sends the user after they approve. Omit it when handing the URL to a person.", + ) + provider_endpoint = serializers.CharField( + required=False, + allow_null=True, + allow_blank=True, + help_text=( + "Provider endpoint from `available_providers` in the domain-connect check. " + "Omit it to use the provider detected from the domain's DNS." + ), + ) + + def validate_provider_endpoint(self, value: str | None) -> str | None: + if value and value not in DOMAIN_CONNECT_PROVIDERS: + raise serializers.ValidationError("Unsupported provider endpoint") + return value + + def validate(self, attrs: dict[str, Any]) -> dict[str, Any]: + if attrs["context"] == DomainConnectContextKind.EMAIL and not attrs.get("integration_id"): + raise serializers.ValidationError("integration_id is required for email context") + if attrs["context"] == DomainConnectContextKind.PROXY and not attrs.get("proxy_record_id"): + raise serializers.ValidationError("proxy_record_id is required for proxy context") + return attrs + + +class DomainConnectApplyUrlResponseSerializer(serializers.Serializer): + url = serializers.CharField( + help_text=( + "Signed Domain Connect URL. A person opens it, signs in at their DNS host and approves the records. " + "The records do not change until they approve." + ) + ) diff --git a/posthog/api/test/test_integration.py b/posthog/api/test/test_integration.py index 9062402a8ad0..dd95b98d6c2f 100644 --- a/posthog/api/test/test_integration.py +++ b/posthog/api/test/test_integration.py @@ -44,7 +44,7 @@ list_org_github_installations, ) from posthog.api.github_callback.types import FlowKind, GitHubAuthorizeState -from posthog.api.integration import IntegrationSerializer, IntegrationViewSet +from posthog.api.integration import EMAIL_DOMAIN_AGENT_SETUP_FLAG, IntegrationSerializer, IntegrationViewSet from posthog.constants import AvailableFeature from posthog.egress.github.transport import GitHubEgressBudgetExhausted from posthog.models.activity_logging.activity_log import ActivityLog, apply_activity_visibility_restrictions @@ -93,6 +93,15 @@ from products.workflows.backend.facade.contracts import EmailDomainVerification, WorkflowSummary from products.workflows.backend.facade.testing import create_workflow_for_test +EMAIL_CONFIG = {"email": "hello@mail.example.com", "domain": "mail.example.com", "provider": "ses"} + + +def patch_email_domain_agent_setup_flag(enabled: bool) -> Any: + return patch( + "posthoganalytics.feature_enabled", + side_effect=lambda key, *_args, **_kwargs: enabled and key == EMAIL_DOMAIN_AGENT_SETUP_FLAG, + ) + def _p256_public_pem() -> str: return ( @@ -1748,6 +1757,341 @@ def test_provider_lookup_actions_on_wrong_integration_return_400( assert response.status_code == status.HTTP_400_BAD_REQUEST assert expected_provider in response.json()["detail"] + @pytest.mark.parametrize( + "method,url_suffix,body,scope,level,expected_status,expected_json", + [ + ( + "get", + "domain-connect/check/?domain=mail.example.com", + None, + "integration:read", + OrganizationMembership.Level.MEMBER, + 200, + {"supported": False, "provider_name": None}, + ), + ( + "post", + "domain-connect/apply-url/", + {"context": "email"}, + "integration:read", + OrganizationMembership.Level.ADMIN, + 403, + {"detail": "API key missing required scope 'integration:write'"}, + ), + ( + "post", + "domain-connect/apply-url/", + {"context": "email"}, + "integration:write", + OrganizationMembership.Level.ADMIN, + 400, + {"detail": "integration_id is required for email context"}, + ), + ( + "post", + "domain-connect/apply-url/", + {"context": "email", "integration_id": 999999999}, + "integration:write", + OrganizationMembership.Level.ADMIN, + 404, + {"detail": "No email sender with this integration_id exists in this project."}, + ), + ( + "post", + "domain-connect/apply-url/", + {"context": "email", "integration_id": "{twilio_id}"}, + "integration:write", + OrganizationMembership.Level.ADMIN, + 404, + {"detail": "No email sender with this integration_id exists in this project."}, + ), + ( + "post", + "domain-connect/apply-url/", + {"context": "proxy", "proxy_record_id": "6f1c1a52-3b7e-4c1e-9d0a-2f4b8e6c9a10"}, + "integration:write", + OrganizationMembership.Level.ADMIN, + 404, + {"detail": "No reverse proxy record with this proxy_record_id exists in this organization."}, + ), + ( + "post", + "{email_id}/email/verify/", + None, + "integration:read", + OrganizationMembership.Level.ADMIN, + 403, + {"detail": "API key missing required scope 'integration:write'"}, + ), + ( + "post", + "{email_id}/email/verify/", + None, + "integration:write", + OrganizationMembership.Level.ADMIN, + 200, + {"status": "pending", "dnsRecords": [ANY]}, + ), + ( + "post", + "{twilio_id}/email/verify/", + None, + "integration:write", + OrganizationMembership.Level.ADMIN, + 400, + {"detail": "This endpoint is only supported for email integrations"}, + ), + ( + "patch", + "{email_id}/email/", + {"config": {}}, + "integration:read", + OrganizationMembership.Level.ADMIN, + 403, + {"detail": "API key missing required scope 'integration:write'"}, + ), + ( + "patch", + "{email_id}/email/", + { + "config": { + "email": "hello@mail.example.com", + "name": "Acme", + "mail_from_subdomain": "bounce", + "provider": "ses", + } + }, + "integration:write", + OrganizationMembership.Level.ADMIN, + 200, + {"kind": "email", "config": {**EMAIL_CONFIG, "name": "Acme", "mail_from_subdomain": "bounce"}}, + ), + ( + "patch", + "{email_id}/email/", + {"config": {"email": "team@mail.example.com", "name": "Acme", "provider": "ses"}}, + "integration:write", + OrganizationMembership.Level.ADMIN, + 400, + {"detail": "The sender address cannot change. Create a new sender for team@mail.example.com instead."}, + ), + ], + ) + @patch("products.workflows.backend.facade.api.update_ses_mail_from_subdomain") + @patch( + "products.workflows.backend.facade.api.verify_ses_email_domain", + return_value={ + "status": "pending", + "dnsRecords": [ + { + "type": "dkim", + "recordType": "CNAME", + "recordHostname": "token._domainkey.mail.example.com", + "recordValue": "token.dkim.amazonses.com", + "status": "pending", + } + ], + }, + ) + @patch("posthog.api.integration.discover_domain_connect", return_value=None) + def test_email_domain_actions_with_scoped_api_key( + self, + _mock_discover: MagicMock, + _mock_verify: MagicMock, + _mock_update: MagicMock, + method: str, + url_suffix: str, + body: dict[str, Any] | None, + scope: str, + level: OrganizationMembership.Level, + expected_status: int, + expected_json: dict[str, Any], + client: HttpClient, + ) -> None: + OrganizationMembership.objects.filter(user=self.user).update(level=level) + email_integration = Integration.objects.create( + team=self.team, + kind="email", + integration_id="hello@mail.example.com", + config=EMAIL_CONFIG, + ) + key_value = "test_key_email_domain" + PersonalAPIKey.objects.create( + label="Test Key", user=self.user, secure_value=hash_key_value(key_value), scopes=[scope] + ) + ids = {"{email_id}": email_integration.id, "{twilio_id}": self.twilio_integration.id} + path = url_suffix.format(email_id=email_integration.id, twilio_id=self.twilio_integration.id) + if body is not None: + body = {key: ids.get(value, value) if isinstance(value, str) else value for key, value in body.items()} + + with patch_email_domain_agent_setup_flag(True): + response = getattr(client, method)( + f"/api/environments/{self.team.pk}/integrations/{path}", + data=json.dumps(body) if body is not None else None, + content_type="application/json", + HTTP_AUTHORIZATION=f"Bearer {key_value}", + ) + + assert response.status_code == expected_status, response.json() + assert response.json() == {**response.json(), **expected_json} + + @pytest.mark.parametrize( + "method,url_suffix", + [ + ("get", "domain-connect/check/?domain=mail.example.com"), + ("post", "domain-connect/apply-url/"), + ("post", "{email_id}/email/verify/"), + ("patch", "{email_id}/email/"), + ], + ) + @patch("products.workflows.backend.facade.api.verify_ses_email_domain") + @patch("posthog.api.integration.discover_domain_connect", return_value=None) + def test_email_domain_actions_reject_api_keys_while_the_agent_setup_flag_is_off( + self, _mock_discover: MagicMock, _mock_verify: MagicMock, method: str, url_suffix: str, client: HttpClient + ) -> None: + OrganizationMembership.objects.filter(user=self.user).update(level=OrganizationMembership.Level.ADMIN) + email_integration = Integration.objects.create( + team=self.team, kind="email", integration_id="hello@mail.example.com", config=EMAIL_CONFIG + ) + key_value = "test_key_email_domain" + PersonalAPIKey.objects.create( + label="Test Key", user=self.user, secure_value=hash_key_value(key_value), scopes=["integration:write"] + ) + + with patch_email_domain_agent_setup_flag(False): + response = getattr(client, method)( + f"/api/environments/{self.team.pk}/integrations/{url_suffix.format(email_id=email_integration.id)}", + content_type="application/json", + HTTP_AUTHORIZATION=f"Bearer {key_value}", + ) + + assert response.status_code == status.HTTP_403_FORBIDDEN, response.json() + assert response.json()["detail"] == "This action does not support personal API key access" + + @pytest.mark.parametrize( + "flag_on,use_session,site_url,request_host,redirect_uri,expected_status", + [ + ( + True, + False, + "https://us.posthog.com", + "us.posthog.com", + "https://us.posthog.com/project/1/channels?dc=email", + 200, + ), + ( + True, + False, + "https://eu.posthog.com", + "eu.posthog.com", + "https://eu.posthog.com/project/1/channels?dc=email", + 200, + ), + ( + True, + False, + "http://posthog.internal:8010", + "posthog.internal:8010", + "http://posthog.internal:8010/channels", + 200, + ), + (True, False, "https://us.posthog.com", "us.posthog.com", "https://attacker.example/landing", 400), + (True, False, "https://us.posthog.com", "attacker.example", "https://attacker.example/landing", 400), + (True, False, "https://us.posthog.com", "us.posthog.com", "//attacker.example/landing", 400), + (True, False, "https://us.posthog.com", "us.posthog.com", "javascript:alert(1)", 400), + (True, True, "https://us.posthog.com", "us.posthog.com", "https://us.posthog.com/channels", 200), + (False, True, "https://us.posthog.com", "us.posthog.com", "https://us.posthog.com/channels", 404), + ], + ) + @patch("posthog.api.integration.generate_apply_url", return_value="https://dns.example/apply") + @patch( + "products.workflows.backend.facade.api.verify_ses_email_domain", + return_value={ + "status": "pending", + "dnsRecords": [ + { + "type": "verification", + "recordType": "TXT", + "recordHostname": "_amazonses.mail.example.com", + "recordValue": "verify-token", + }, + *( + {"type": "dkim", "recordType": "CNAME", "recordHostname": f"dkim{n}._domainkey.mail.example.com"} + for n in range(3) + ), + ], + }, + ) + def test_domain_connect_apply_url_only_redirects_back_to_posthog( + self, + _mock_verify: MagicMock, + mock_generate_apply_url: MagicMock, + flag_on: bool, + use_session: bool, + site_url: str, + request_host: str, + redirect_uri: str, + expected_status: int, + client: HttpClient, + ) -> None: + OrganizationMembership.objects.filter(user=self.user).update(level=OrganizationMembership.Level.ADMIN) + email_integration = Integration.objects.create( + team=self.team, kind="email", integration_id="hello@mail.example.com", config=EMAIL_CONFIG + ) + key_value = "test_key_email_domain" + PersonalAPIKey.objects.create( + label="Test Key", user=self.user, secure_value=hash_key_value(key_value), scopes=["integration:write"] + ) + + if use_session: + client.force_login(self.user) + auth_headers = {} if use_session else {"HTTP_AUTHORIZATION": f"Bearer {key_value}"} + + with patch_email_domain_agent_setup_flag(flag_on), override_settings(SITE_URL=site_url): + response = client.post( + f"/api/environments/{self.team.pk}/integrations/domain-connect/apply-url/", + data={"context": "email", "integration_id": email_integration.id, "redirect_uri": redirect_uri}, + content_type="application/json", + **auth_headers, + HTTP_HOST=request_host, + ) + + assert response.status_code == expected_status, response.json() + if expected_status == 200: + assert mock_generate_apply_url.call_args.kwargs["redirect_uri"] == redirect_uri + elif expected_status == 400: + assert response.json()["detail"] == "redirect_uri must point back to PostHog." + mock_generate_apply_url.assert_not_called() + else: + mock_generate_apply_url.assert_not_called() + + @pytest.mark.parametrize( + "flag_on,stored_email,sent_email", + [ + (True, "Hello@mail.example.com", "Hello@mail.example.com"), + (False, "hello@mail.example.com", "team@mail.example.com"), + ], + ) + @patch("products.workflows.backend.facade.api.update_ses_mail_from_subdomain") + def test_email_update_from_the_app_keeps_the_sender_address( + self, _mock_update: MagicMock, flag_on: bool, stored_email: str, sent_email: str, client: HttpClient + ) -> None: + OrganizationMembership.objects.filter(user=self.user).update(level=OrganizationMembership.Level.ADMIN) + email_integration = Integration.objects.create( + team=self.team, kind="email", integration_id=stored_email, config={**EMAIL_CONFIG, "email": stored_email} + ) + client.force_login(self.user) + + with patch_email_domain_agent_setup_flag(flag_on): + response = client.patch( + f"/api/environments/{self.team.pk}/integrations/{email_integration.id}/email/", + data=json.dumps({"config": {"email": sent_email, "name": "Acme", "provider": "ses"}}), + content_type="application/json", + ) + + assert response.status_code == status.HTTP_200_OK, response.json() + assert response.json()["config"]["email"] == stored_email + @patch("posthog.models.integration.github.GitHubIntegration.list_cached_repositories") def test_github_repos_with_scope_succeeds(self, mock_list_repos, client: HttpClient): mock_list_repos.return_value = ( diff --git a/posthog/api/test/test_integration_domain_serializers.py b/posthog/api/test/test_integration_domain_serializers.py new file mode 100644 index 000000000000..7736df22f0cd --- /dev/null +++ b/posthog/api/test/test_integration_domain_serializers.py @@ -0,0 +1,57 @@ +from typing import Any + +from django.test import SimpleTestCase, override_settings + +from parameterized import parameterized + +from posthog.api.integration_domain_serializers import ( + DomainConnectApplyUrlRequestSerializer, + NativeEmailIntegrationSerializer, +) + + +class TestDomainConnectApplyUrlRequestSerializer(SimpleTestCase): + @parameterized.expand( + [ + ("missing_context", {"integration_id": 1}, "context must be 'email' or 'proxy'"), + ("unknown_context", {"context": "sms", "integration_id": 1}, "context must be 'email' or 'proxy'"), + ("email_without_integration", {"context": "email"}, "integration_id is required for email context"), + ("proxy_without_record", {"context": "proxy"}, "proxy_record_id is required for proxy context"), + ( + "unknown_provider_endpoint", + {"context": "email", "integration_id": 1, "provider_endpoint": "dns.example.com"}, + "Unsupported provider endpoint", + ), + ("email_with_integration", {"context": "email", "integration_id": 1}, None), + ("proxy_with_malformed_record", {"context": "proxy", "proxy_record_id": "a1b2"}, "Must be a valid UUID."), + ( + "proxy_with_record", + {"context": "proxy", "proxy_record_id": "6f1c1a52-3b7e-4c1e-9d0a-2f4b8e6c9a10"}, + None, + ), + ("null_optionals", {"context": "email", "integration_id": 1, "provider_endpoint": None}, None), + ] + ) + def test_validation(self, _name: str, data: dict[str, Any], expected_error: str | None) -> None: + serializer = DomainConnectApplyUrlRequestSerializer(data=data) + + assert serializer.is_valid() is (expected_error is None) + if expected_error: + assert expected_error in str(serializer.errors) + + +class TestNativeEmailIntegrationSerializer(SimpleTestCase): + @parameterized.expand( + [ + ("ses_in_production", "ses", False, True), + ("maildev_in_production", "maildev", False, False), + ("maildev_in_development", "maildev", True, True), + ] + ) + def test_provider(self, _name: str, provider: str, debug: bool, expected_valid: bool) -> None: + with override_settings(DEBUG=debug): + serializer = NativeEmailIntegrationSerializer( + data={"email": "hello@mail.example.com", "name": "Acme", "provider": provider} + ) + + assert serializer.is_valid() is expected_valid diff --git a/posthog/domain_connect.py b/posthog/domain_connect.py index 5a18c7390b17..eaff7957edbc 100644 --- a/posthog/domain_connect.py +++ b/posthog/domain_connect.py @@ -242,9 +242,7 @@ def resolve_email_context(integration_id: int, team_id: int) -> DomainConnectCon """ from posthog.models.integration import EmailIntegration, Integration - instance = Integration.objects.get(id=integration_id, team_id=team_id) - if instance.kind != "email": - raise ValueError("Integration must be of kind 'email'") + instance = Integration.objects.get(id=integration_id, team_id=team_id, kind="email") email_integration = EmailIntegration(instance) verification_result = email_integration.verify() diff --git a/posthog/models/integration/email.py b/posthog/models/integration/email.py index 269f0df8b7f3..4c16fe394228 100644 --- a/posthog/models/integration/email.py +++ b/posthog/models/integration/email.py @@ -122,6 +122,10 @@ def update_native_integration(self, config: dict, team_id: int) -> model.Integra return self.integration + def reject_address_change(self, email_address: str) -> None: + if email_address.lower() != self.integration.config.get("email", "").lower(): + raise ValidationError(f"The sender address cannot change. Create a new sender for {email_address} instead.") + def verify(self) -> "EmailDomainVerification": domain = self.integration.config.get("domain") provider = self.integration.config.get("provider", "ses") diff --git a/products/integrations/frontend/generated/api.schemas.ts b/products/integrations/frontend/generated/api.schemas.ts index 30eda29b193d..a97ac911935e 100644 --- a/products/integrations/frontend/generated/api.schemas.ts +++ b/products/integrations/frontend/generated/api.schemas.ts @@ -327,25 +327,126 @@ export interface SlackChannelsResponseApi { } /** - * Standard Integration serializer. + * * `ses` - Amazon SES + * * `maildev` - Maildev (local development only) */ -export interface PatchedIntegrationConfigApi { - readonly id?: number - kind?: IntegrationKindEnumApi - config?: unknown - readonly created_at?: string - readonly created_by?: UserBasicApi - readonly errors?: string - readonly display_name?: string - /** Slack only: whether reconnecting can request the files:write scope. */ - readonly files_write_requestable?: boolean - /** - * GitHub only, null otherwise. Whether another project's GitHub integration references the same App installation. When false, disconnecting this integration also uninstalls the GitHub App from the connected account or organization and removes personal GitHub connections that share it. - * @nullable - */ - readonly installation_shared?: boolean | null - /** GitHub only, null otherwise. `unavailable` means the App was uninstalled or suspended on GitHub and PostHog can no longer mint tokens for it; `connected` otherwise. */ - readonly installation_status?: InstallationStatusEnumApi | null +export type NativeEmailProviderEnumApi = (typeof NativeEmailProviderEnumApi)[keyof typeof NativeEmailProviderEnumApi] + +export const NativeEmailProviderEnumApi = { + Ses: 'ses', + Maildev: 'maildev', +} as const + +export interface NativeEmailIntegrationApi { + /** Sender address, for example `hello@mail.example.com`. Its domain is the sending domain that needs DNS records. Free and disposable mailbox domains such as gmail.com are rejected. Cannot be changed after creation; send the current address when updating. */ + email: string + /** Sender display name recipients see in their inbox, for example `Acme`. */ + name: string + /** Sending provider. Always `ses`. `maildev` is only accepted in local development. + * + * * `ses` - Amazon SES + * * `maildev` - Maildev (local development only) */ + provider: NativeEmailProviderEnumApi + /** Subdomain of the sending domain used as the custom MAIL FROM (bounce) domain. `feedback` gives `feedback.mail.example.com`. Defaults to `feedback`. Pick another value if that subdomain already has MX records. */ + mail_from_subdomain?: string +} + +export interface PatchedEmailSenderUpdateRequestApi { + /** The full sender config. Only `name` and `mail_from_subdomain` change; `email` must stay the same. */ + config?: NativeEmailIntegrationApi +} + +/** + * * `success` - Domain verified + * * `pending` - Verification pending + * * `failed` - Verification failed + */ +export type EmailDomainStatusEnumApi = (typeof EmailDomainStatusEnumApi)[keyof typeof EmailDomainStatusEnumApi] + +export const EmailDomainStatusEnumApi = { + Success: 'success', + Pending: 'pending', + Failed: 'failed', +} as const + +/** + * * `verification` - Domain ownership or SPF + * * `dkim` - DKIM signing + * * `mail_from` - Custom MAIL FROM + * * `dmarc` - DMARC policy + */ +export type EmailDomainRecordPurposeEnumApi = + (typeof EmailDomainRecordPurposeEnumApi)[keyof typeof EmailDomainRecordPurposeEnumApi] + +export const EmailDomainRecordPurposeEnumApi = { + Verification: 'verification', + Dkim: 'dkim', + MailFrom: 'mail_from', + Dmarc: 'dmarc', +} as const + +/** + * * `TXT` - TXT record + * * `CNAME` - CNAME record + * * `MX` - MX record + */ +export type EmailDomainRecordTypeEnumApi = + (typeof EmailDomainRecordTypeEnumApi)[keyof typeof EmailDomainRecordTypeEnumApi] + +export const EmailDomainRecordTypeEnumApi = { + Txt: 'TXT', + Cname: 'CNAME', + Mx: 'MX', +} as const + +/** + * * `success` - Record found + * * `pending` - Record not found yet + */ +export type EmailDomainRecordStatusEnumApi = + (typeof EmailDomainRecordStatusEnumApi)[keyof typeof EmailDomainRecordStatusEnumApi] + +export const EmailDomainRecordStatusEnumApi = { + Success: 'success', + Pending: 'pending', +} as const + +export interface EmailDomainDnsRecordApi { + /** What the record is for: domain ownership or the sending domain's SPF, DKIM signing, the custom MAIL FROM domain, or DMARC. + * + * * `verification` - Domain ownership or SPF + * * `dkim` - DKIM signing + * * `mail_from` - Custom MAIL FROM + * * `dmarc` - DMARC policy */ + type: EmailDomainRecordPurposeEnumApi + /** DNS record type. + * + * * `TXT` - TXT record + * * `CNAME` - CNAME record + * * `MX` - MX record */ + recordType: EmailDomainRecordTypeEnumApi + /** Fully qualified record name, or `@` for the sending domain itself. Many DNS hosts append the zone, so enter only the part before it. */ + recordHostname: string + /** Exact record value to publish. */ + recordValue: string + /** `success` once the record is visible in DNS, `pending` until then. + * + * * `success` - Record found + * * `pending` - Record not found yet */ + status: EmailDomainRecordStatusEnumApi + /** MX priority. Only present on MX records. */ + priority?: number +} + +export interface EmailDomainVerificationApi { + /** `success` when every record is verified and the sender can send. `pending` while DNS is not visible yet, which can take minutes and up to 72 hours. `failed` when the provider gave up on the records. + * + * * `success` - Domain verified + * * `pending` - Verification pending + * * `failed` - Verification failed */ + status: EmailDomainStatusEnumApi + /** Every DNS record the sending domain needs, each with its own status. */ + dnsRecords: EmailDomainDnsRecordApi[] } export interface IntegrationAssigneeApi { @@ -481,6 +582,70 @@ export interface SlackUsersResponseApi { has_more?: boolean } +/** + * * `email` - Email sending domain + * * `proxy` - Reverse proxy domain + */ +export type DomainConnectContextKindEnumApi = + (typeof DomainConnectContextKindEnumApi)[keyof typeof DomainConnectContextKindEnumApi] + +export const DomainConnectContextKindEnumApi = { + Email: 'email', + Proxy: 'proxy', +} as const + +export interface DomainConnectApplyUrlRequestApi { + /** `email` to configure an email sending domain, `proxy` for a reverse proxy domain. + * + * * `email` - Email sending domain + * * `proxy` - Reverse proxy domain */ + context: DomainConnectContextKindEnumApi + /** + * ID of the email integration (sender). Required when `context` is `email`. + * @nullable + */ + integration_id?: number | null + /** + * ID of the reverse proxy record. Required when `context` is `proxy`. + * @nullable + */ + proxy_record_id?: string | null + /** + * Where the DNS host sends the user after they approve. Omit it when handing the URL to a person. + * @nullable + */ + redirect_uri?: string | null + /** + * Provider endpoint from `available_providers` in the domain-connect check. Omit it to use the provider detected from the domain's DNS. + * @nullable + */ + provider_endpoint?: string | null +} + +export interface DomainConnectApplyUrlResponseApi { + /** Signed Domain Connect URL. A person opens it, signs in at their DNS host and approves the records. The records do not change until they approve. */ + url: string +} + +export interface DomainConnectProviderApi { + /** Provider endpoint. Pass it as `provider_endpoint` to apply-url. */ + endpoint: string + /** Provider display name, for example `Cloudflare`. */ + name: string +} + +export interface DomainConnectCheckResponseApi { + /** True when the domain's DNS host supports one-click setup with Domain Connect. */ + supported: boolean + /** + * Detected DNS host, for example `Cloudflare`. Null when not supported. + * @nullable + */ + provider_name: string | null + /** Providers the user can pick by hand when detection fails. Empty when detection succeeded. Only offer one if the user confirms their DNS is hosted there. */ + available_providers: DomainConnectProviderApi[] +} + /** * * `ok` - Ok * * `not_connected` - Not Connected @@ -1035,3 +1200,11 @@ export type IntegrationsUsersRetrieveParams = { */ user_id?: string } + +export type IntegrationsDomainConnectCheckRetrieveParams = { + /** + * Domain to check, for example `mail.example.com`. Subdomains resolve to their registrable domain. + * @minLength 1 + */ + domain: string +} diff --git a/products/integrations/frontend/generated/api.ts b/products/integrations/frontend/generated/api.ts index 54bdf2e0adc4..53d22e1055a1 100644 --- a/products/integrations/frontend/generated/api.ts +++ b/products/integrations/frontend/generated/api.ts @@ -9,6 +9,10 @@ import { apiMutator } from '../../../../frontend/src/lib/api-orval-mutator' * OpenAPI spec version: 1.0.0 */ import type { + DomainConnectApplyUrlRequestApi, + DomainConnectApplyUrlResponseApi, + DomainConnectCheckResponseApi, + EmailDomainVerificationApi, GitHubAvailableInstallationsResponseApi, GitHubBranchesResponseApi, GitHubLinkExistingRequestApi, @@ -23,6 +27,7 @@ import type { IntegrationAssigneesResponseApi, IntegrationConfigApi, IntegrationsChannelsRetrieveParams, + IntegrationsDomainConnectCheckRetrieveParams, IntegrationsGithubAssigneesRetrieveParams, IntegrationsGithubBranchesRetrieveParams, IntegrationsGithubReposRetrieveParams, @@ -37,7 +42,7 @@ import type { OrganizationIntegrationApi, PaginatedIntegrationConfigListApi, PaginatedRoleExternalReferenceListApi, - PatchedIntegrationConfigApi, + PatchedEmailSenderUpdateRequestApi, PatchedOrganizationIntegrationApi, PostHogConnectionForwardApi, PostHogConnectionForwardResponseApi, @@ -383,17 +388,20 @@ export const getIntegrationsEmailPartialUpdateUrl = (projectId: string, id: numb return `/api/projects/${projectId}/integrations/${id}/email/` } +/** + * @summary Update an email sender + */ export const integrationsEmailPartialUpdate = async ( projectId: string, id: number, - patchedIntegrationConfigApi?: NonReadonly, + patchedEmailSenderUpdateRequestApi?: PatchedEmailSenderUpdateRequestApi, options?: RequestInit ): Promise => { return apiMutator(getIntegrationsEmailPartialUpdateUrl(projectId, id), { ...options, method: 'PATCH', headers: { 'Content-Type': 'application/json', ...options?.headers }, - body: JSON.stringify(patchedIntegrationConfigApi), + body: JSON.stringify(patchedEmailSenderUpdateRequestApi), }) } @@ -401,17 +409,18 @@ export const getIntegrationsEmailVerifyCreateUrl = (projectId: string, id: numbe return `/api/projects/${projectId}/integrations/${id}/email/verify/` } +/** + * Ask the email provider to check the sending domain's DNS records and return each record with its status. When every record is verified, all senders on the domain in this project become able to send. + * @summary Verify an email sender's domain + */ export const integrationsEmailVerifyCreate = async ( projectId: string, id: number, - integrationConfigApi: NonReadonly, options?: RequestInit -): Promise => { - return apiMutator(getIntegrationsEmailVerifyCreateUrl(projectId, id), { +): Promise => { + return apiMutator(getIntegrationsEmailVerifyCreateUrl(projectId, id), { ...options, method: 'POST', - headers: { 'Content-Type': 'application/json', ...options?.headers }, - body: JSON.stringify(integrationConfigApi), }) } @@ -816,34 +825,50 @@ export const getIntegrationsDomainConnectApplyUrlCreateUrl = (projectId: string) } /** - * Unified endpoint for generating Domain Connect apply URLs. - * - * Accepts a context ("email" or "proxy") and the relevant resource ID. - * The backend resolves the domain, template variables, and service ID - * based on context, then builds the signed apply URL. + * Build the signed URL that sends a person to their DNS host to approve the records for an email sending domain or a reverse proxy domain. + * @summary Generate a Domain Connect apply URL */ export const integrationsDomainConnectApplyUrlCreate = async ( projectId: string, - integrationConfigApi: NonReadonly, + domainConnectApplyUrlRequestApi: DomainConnectApplyUrlRequestApi, options?: RequestInit -): Promise => { - return apiMutator(getIntegrationsDomainConnectApplyUrlCreateUrl(projectId), { +): Promise => { + return apiMutator(getIntegrationsDomainConnectApplyUrlCreateUrl(projectId), { ...options, method: 'POST', headers: { 'Content-Type': 'application/json', ...options?.headers }, - body: JSON.stringify(integrationConfigApi), + body: JSON.stringify(domainConnectApplyUrlRequestApi), }) } -export const getIntegrationsDomainConnectCheckRetrieveUrl = (projectId: string) => { - return `/api/projects/${projectId}/integrations/domain-connect/check/` +export const getIntegrationsDomainConnectCheckRetrieveUrl = ( + projectId: string, + params: IntegrationsDomainConnectCheckRetrieveParams +) => { + const normalizedParams = new URLSearchParams() + + Object.entries(params || {}).forEach(([key, value]) => { + if (value !== undefined) { + normalizedParams.append(key, value === null ? 'null' : String(value)) + } + }) + + const stringifiedParams = normalizedParams.toString() + + return stringifiedParams.length > 0 + ? `/api/projects/${projectId}/integrations/domain-connect/check/?${stringifiedParams}` + : `/api/projects/${projectId}/integrations/domain-connect/check/` } +/** + * @summary Check Domain Connect support for a domain + */ export const integrationsDomainConnectCheckRetrieve = async ( projectId: string, + params: IntegrationsDomainConnectCheckRetrieveParams, options?: RequestInit -): Promise => { - return apiMutator(getIntegrationsDomainConnectCheckRetrieveUrl(projectId), { +): Promise => { + return apiMutator(getIntegrationsDomainConnectCheckRetrieveUrl(projectId, params), { ...options, method: 'GET', }) diff --git a/products/integrations/frontend/generated/api.zod.ts b/products/integrations/frontend/generated/api.zod.ts index a4e1ca6b826c..86a7e6b08942 100644 --- a/products/integrations/frontend/generated/api.zod.ts +++ b/products/integrations/frontend/generated/api.zod.ts @@ -120,199 +120,65 @@ export const IntegrationsCreateBody = /* @__PURE__ */ zod }) .describe('Standard Integration serializer.') -export const IntegrationsEmailPartialUpdateBody = /* @__PURE__ */ zod - .object({ - kind: zod - .enum([ - 'anthropic', - 'apple-ads', - 'apns', - 'aws-redshift', - 'aws-s3', - 'azure-blob', - 'bing-ads', - 'clickup', - 'customerio-app', - 'customerio-track', - 'customerio-webhook', - 'databricks', - 'email', - 'firebase', - 'github', - 'gitlab', - 'google-ads', - 'google-analytics', - 'google-calendar', - 'google-cloud-service-account', - 'google-cloud-storage', - 'google-pubsub', - 'google-search-console', - 'google-sheets', - 'helpscout', - 'hubspot', - 'instagram', - 'intercom', - 'jira', - 'linear', - 'linkedin-ads', - 'meta-ads', - 'pardot', - 'pinterest-ads', - 'postgresql', - 'posthog', - 'reddit-ads', - 'twitter-ads', - 'resend', - 's3-compatible', - 'salesforce', - 'slack', - 'slack-posthog-code', - 'snapchat', - 'snowflake', - 'stripe', - 'tiktok-ads', - 'twilio', - 'vercel', - 'youtube-analytics', - ]) - .optional() - .describe( - '\* `anthropic` - Anthropic\n\* `apple-ads` - Apple Ads\n\* `apns` - Apple Push\n\* `aws-redshift` - Aws Redshift\n\* `aws-s3` - Aws S3\n\* `azure-blob` - Azure Blob\n\* `bing-ads` - Bing Ads\n\* `clickup` - Clickup\n\* `customerio-app` - Customerio App\n\* `customerio-track` - Customerio Track\n\* `customerio-webhook` - Customerio Webhook\n\* `databricks` - Databricks\n\* `email` - Email\n\* `firebase` - Firebase\n\* `github` - Github\n\* `gitlab` - Gitlab\n\* `google-ads` - Google Ads\n\* `google-analytics` - Google Analytics\n\* `google-calendar` - Google Calendar\n\* `google-cloud-service-account` - Google Cloud Service Account\n\* `google-cloud-storage` - Google Cloud Storage\n\* `google-pubsub` - Google Pubsub\n\* `google-search-console` - Google Search Console\n\* `google-sheets` - Google Sheets\n\* `helpscout` - Helpscout\n\* `hubspot` - Hubspot\n\* `instagram` - Instagram\n\* `intercom` - Intercom\n\* `jira` - Jira\n\* `linear` - Linear\n\* `linkedin-ads` - Linkedin Ads\n\* `meta-ads` - Meta Ads\n\* `pardot` - Pardot\n\* `pinterest-ads` - Pinterest Ads\n\* `postgresql` - Postgresql\n\* `posthog` - Posthog\n\* `reddit-ads` - Reddit Ads\n\* `twitter-ads` - Twitter Ads\n\* `resend` - Resend\n\* `s3-compatible` - S3 Compatible\n\* `salesforce` - Salesforce\n\* `slack` - Slack\n\* `slack-posthog-code` - Slack Posthog Code\n\* `snapchat` - Snapchat\n\* `snowflake` - Snowflake\n\* `stripe` - Stripe\n\* `tiktok-ads` - Tiktok Ads\n\* `twilio` - Twilio\n\* `vercel` - Vercel\n\* `youtube-analytics` - Youtube Analytics' - ), - config: zod.unknown().optional(), - }) - .describe('Standard Integration serializer.') - -export const IntegrationsEmailVerifyCreateBody = /* @__PURE__ */ zod - .object({ - kind: zod - .enum([ - 'anthropic', - 'apple-ads', - 'apns', - 'aws-redshift', - 'aws-s3', - 'azure-blob', - 'bing-ads', - 'clickup', - 'customerio-app', - 'customerio-track', - 'customerio-webhook', - 'databricks', - 'email', - 'firebase', - 'github', - 'gitlab', - 'google-ads', - 'google-analytics', - 'google-calendar', - 'google-cloud-service-account', - 'google-cloud-storage', - 'google-pubsub', - 'google-search-console', - 'google-sheets', - 'helpscout', - 'hubspot', - 'instagram', - 'intercom', - 'jira', - 'linear', - 'linkedin-ads', - 'meta-ads', - 'pardot', - 'pinterest-ads', - 'postgresql', - 'posthog', - 'reddit-ads', - 'twitter-ads', - 'resend', - 's3-compatible', - 'salesforce', - 'slack', - 'slack-posthog-code', - 'snapchat', - 'snowflake', - 'stripe', - 'tiktok-ads', - 'twilio', - 'vercel', - 'youtube-analytics', - ]) - .describe( - '\* `anthropic` - Anthropic\n\* `apple-ads` - Apple Ads\n\* `apns` - Apple Push\n\* `aws-redshift` - Aws Redshift\n\* `aws-s3` - Aws S3\n\* `azure-blob` - Azure Blob\n\* `bing-ads` - Bing Ads\n\* `clickup` - Clickup\n\* `customerio-app` - Customerio App\n\* `customerio-track` - Customerio Track\n\* `customerio-webhook` - Customerio Webhook\n\* `databricks` - Databricks\n\* `email` - Email\n\* `firebase` - Firebase\n\* `github` - Github\n\* `gitlab` - Gitlab\n\* `google-ads` - Google Ads\n\* `google-analytics` - Google Analytics\n\* `google-calendar` - Google Calendar\n\* `google-cloud-service-account` - Google Cloud Service Account\n\* `google-cloud-storage` - Google Cloud Storage\n\* `google-pubsub` - Google Pubsub\n\* `google-search-console` - Google Search Console\n\* `google-sheets` - Google Sheets\n\* `helpscout` - Helpscout\n\* `hubspot` - Hubspot\n\* `instagram` - Instagram\n\* `intercom` - Intercom\n\* `jira` - Jira\n\* `linear` - Linear\n\* `linkedin-ads` - Linkedin Ads\n\* `meta-ads` - Meta Ads\n\* `pardot` - Pardot\n\* `pinterest-ads` - Pinterest Ads\n\* `postgresql` - Postgresql\n\* `posthog` - Posthog\n\* `reddit-ads` - Reddit Ads\n\* `twitter-ads` - Twitter Ads\n\* `resend` - Resend\n\* `s3-compatible` - S3 Compatible\n\* `salesforce` - Salesforce\n\* `slack` - Slack\n\* `slack-posthog-code` - Slack Posthog Code\n\* `snapchat` - Snapchat\n\* `snowflake` - Snowflake\n\* `stripe` - Stripe\n\* `tiktok-ads` - Tiktok Ads\n\* `twilio` - Twilio\n\* `vercel` - Vercel\n\* `youtube-analytics` - Youtube Analytics' - ), - config: zod.unknown().optional(), - }) - .describe('Standard Integration serializer.') +/** + * @summary Update an email sender + */ +export const IntegrationsEmailPartialUpdateBody = /* @__PURE__ */ zod.object({ + config: zod + .object({ + email: zod + .email() + .describe( + 'Sender address, for example `hello@mail.example.com`. Its domain is the sending domain that needs DNS records. Free and disposable mailbox domains such as gmail.com are rejected. Cannot be changed after creation; send the current address when updating.' + ), + name: zod.string().describe('Sender display name recipients see in their inbox, for example `Acme`.'), + provider: zod + .enum(['ses', 'maildev']) + .describe('\* `ses` - Amazon SES\n\* `maildev` - Maildev (local development only)') + .describe( + 'Sending provider. Always `ses`. `maildev` is only accepted in local development.\n\n\* `ses` - Amazon SES\n\* `maildev` - Maildev (local development only)' + ), + mail_from_subdomain: zod + .string() + .optional() + .describe( + 'Subdomain of the sending domain used as the custom MAIL FROM (bounce) domain. `feedback` gives `feedback.mail.example.com`. Defaults to `feedback`. Pick another value if that subdomain already has MX records.' + ), + }) + .optional() + .describe('The full sender config. Only `name` and `mail_from_subdomain` change; `email` must stay the same.'), +}) /** - * Unified endpoint for generating Domain Connect apply URLs. - * - * Accepts a context ("email" or "proxy") and the relevant resource ID. - * The backend resolves the domain, template variables, and service ID - * based on context, then builds the signed apply URL. + * Build the signed URL that sends a person to their DNS host to approve the records for an email sending domain or a reverse proxy domain. + * @summary Generate a Domain Connect apply URL */ -export const IntegrationsDomainConnectApplyUrlCreateBody = /* @__PURE__ */ zod - .object({ - kind: zod - .enum([ - 'anthropic', - 'apple-ads', - 'apns', - 'aws-redshift', - 'aws-s3', - 'azure-blob', - 'bing-ads', - 'clickup', - 'customerio-app', - 'customerio-track', - 'customerio-webhook', - 'databricks', - 'email', - 'firebase', - 'github', - 'gitlab', - 'google-ads', - 'google-analytics', - 'google-calendar', - 'google-cloud-service-account', - 'google-cloud-storage', - 'google-pubsub', - 'google-search-console', - 'google-sheets', - 'helpscout', - 'hubspot', - 'instagram', - 'intercom', - 'jira', - 'linear', - 'linkedin-ads', - 'meta-ads', - 'pardot', - 'pinterest-ads', - 'postgresql', - 'posthog', - 'reddit-ads', - 'twitter-ads', - 'resend', - 's3-compatible', - 'salesforce', - 'slack', - 'slack-posthog-code', - 'snapchat', - 'snowflake', - 'stripe', - 'tiktok-ads', - 'twilio', - 'vercel', - 'youtube-analytics', - ]) - .describe( - '\* `anthropic` - Anthropic\n\* `apple-ads` - Apple Ads\n\* `apns` - Apple Push\n\* `aws-redshift` - Aws Redshift\n\* `aws-s3` - Aws S3\n\* `azure-blob` - Azure Blob\n\* `bing-ads` - Bing Ads\n\* `clickup` - Clickup\n\* `customerio-app` - Customerio App\n\* `customerio-track` - Customerio Track\n\* `customerio-webhook` - Customerio Webhook\n\* `databricks` - Databricks\n\* `email` - Email\n\* `firebase` - Firebase\n\* `github` - Github\n\* `gitlab` - Gitlab\n\* `google-ads` - Google Ads\n\* `google-analytics` - Google Analytics\n\* `google-calendar` - Google Calendar\n\* `google-cloud-service-account` - Google Cloud Service Account\n\* `google-cloud-storage` - Google Cloud Storage\n\* `google-pubsub` - Google Pubsub\n\* `google-search-console` - Google Search Console\n\* `google-sheets` - Google Sheets\n\* `helpscout` - Helpscout\n\* `hubspot` - Hubspot\n\* `instagram` - Instagram\n\* `intercom` - Intercom\n\* `jira` - Jira\n\* `linear` - Linear\n\* `linkedin-ads` - Linkedin Ads\n\* `meta-ads` - Meta Ads\n\* `pardot` - Pardot\n\* `pinterest-ads` - Pinterest Ads\n\* `postgresql` - Postgresql\n\* `posthog` - Posthog\n\* `reddit-ads` - Reddit Ads\n\* `twitter-ads` - Twitter Ads\n\* `resend` - Resend\n\* `s3-compatible` - S3 Compatible\n\* `salesforce` - Salesforce\n\* `slack` - Slack\n\* `slack-posthog-code` - Slack Posthog Code\n\* `snapchat` - Snapchat\n\* `snowflake` - Snowflake\n\* `stripe` - Stripe\n\* `tiktok-ads` - Tiktok Ads\n\* `twilio` - Twilio\n\* `vercel` - Vercel\n\* `youtube-analytics` - Youtube Analytics' - ), - config: zod.unknown().optional(), - }) - .describe('Standard Integration serializer.') +export const IntegrationsDomainConnectApplyUrlCreateBody = /* @__PURE__ */ zod.object({ + context: zod + .enum(['email', 'proxy']) + .describe('\* `email` - Email sending domain\n\* `proxy` - Reverse proxy domain') + .describe( + '`email` to configure an email sending domain, `proxy` for a reverse proxy domain.\n\n\* `email` - Email sending domain\n\* `proxy` - Reverse proxy domain' + ), + integration_id: zod + .number() + .nullish() + .describe('ID of the email integration (sender). Required when `context` is `email`.'), + proxy_record_id: zod + .uuid() + .nullish() + .describe('ID of the reverse proxy record. Required when `context` is `proxy`.'), + redirect_uri: zod + .string() + .nullish() + .describe('Where the DNS host sends the user after they approve. Omit it when handing the URL to a person.'), + provider_endpoint: zod + .string() + .nullish() + .describe( + "Provider endpoint from `available_providers` in the domain-connect check. Omit it to use the provider detected from the domain's DNS." + ), +}) /** * Reuse a GitHub installation already linked to a sibling team in the same organization. diff --git a/products/integrations/mcp/tools.yaml b/products/integrations/mcp/tools.yaml index d92d6daae08f..382d27a142f4 100644 --- a/products/integrations/mcp/tools.yaml +++ b/products/integrations/mcp/tools.yaml @@ -59,6 +59,123 @@ tools: to find a channel_id when wiring up Slack delivery for an alert — pass the channel id into the cdp-functions-create inputs.channel.value field. Requires the Slack integration's id; use integrations-list (filter by kind=slack) to find it. + integrations-domain-connect-apply-url-create: + operation: integrations_domain_connect_apply_url_create + enabled: true + scopes: + - integration:write + annotations: + readOnly: false + destructive: false + idempotent: true + title: Get a Domain Connect URL for DNS records + description: > + Build a signed URL that adds an email sender's DNS records at the domain's DNS host in one step. Call it + with the sender's integration_id after integrations-domain-connect-check-retrieve says the domain is + supported. If the domain already has a DMARC record, the URL leaves it out and keeps the existing policy. + You cannot open this URL yourself: give it to the person you work with. They sign in at their DNS host and + approve the records there; nothing changes until they approve. Omit redirect_uri. Then call + integrations-email-verify-create until it reports success. If this tool returns an error, fall back to + adding the records from integrations-email-verify-create by hand. + exclude_params: + - context + - proxy_record_id + param_overrides: + integration_id: + required: true + inject_body: + context: email + feature_flag: workflows-email-domain-agent-setup + integrations-domain-connect-check-retrieve: + operation: integrations_domain_connect_check_retrieve + enabled: true + scopes: + - integration:read + annotations: + readOnly: true + destructive: false + idempotent: true + title: Check one-click DNS setup for a domain + description: > + Check whether a domain's DNS host supports one-click record setup with Domain Connect (for example + Cloudflare or Vercel). Pass the sending domain of an email sender, for example `mail.example.com`. When + `supported` is true, use integrations-domain-connect-apply-url-create to get a URL the person approves at + their DNS host. When it is false, add the records from integrations-email-verify-create yourself, or give + them to the person to add. + feature_flag: workflows-email-domain-agent-setup + integrations-email-create: + operation: integrations_create + enabled: true + scopes: + - integration:write + annotations: + readOnly: false + destructive: false + idempotent: false + title: Create an email sender + description: > + Create an email sender for Workflows on a custom domain, for example hello@mail.example.com. This registers + the sending domain with the email provider and returns the new integration; its `id` is the integration_id + the other email tools take. The sender cannot send until its domain is verified. Next, call + integrations-email-verify-create to get the DNS records the domain needs, then publish them: either with + integrations-domain-connect-check-retrieve and integrations-domain-connect-apply-url-create, or with your + own DNS tools. Do not call it for an address that already has a sender: re-creating it marks the sender + unverified until the next successful verify. Use integrations-email-partial-update to change an existing + sender. mail_from_subdomain applies to the whole domain: when the domain already has a sender, pass that + sender's MAIL FROM label (the subdomain of its MX record from integrations-email-verify-create), or the + existing senders stop verifying. Free mailbox domains such as gmail.com are rejected, and a domain already + used by another organization fails. + exclude_params: + - kind + inject_body: + kind: email + param_overrides: + config: + input_schema: EmailSenderConfigSchema + feature_flag: workflows-email-domain-agent-setup + integrations-email-partial-update: + operation: integrations_email_partial_update + enabled: true + scopes: + - integration:write + annotations: + readOnly: false + destructive: false + idempotent: true + title: Update an email sender + description: > + Change an email sender's display name or its MAIL FROM subdomain. Send the full config with the sender's + current `email`, which cannot change, and its current `name` unless you rename it. No read tool returns the + name: take it from the integrations-email-create response, or ask the person. mail_from_subdomain applies to + every sender on the domain and changes the MX and SPF records the domain needs, so confirm it with the + person first, then call integrations-email-verify-create and publish the new records. + param_overrides: + config: + input_schema: EmailSenderConfigSchema + required: true + feature_flag: workflows-email-domain-agent-setup + integrations-email-verify-create: + operation: integrations_email_verify_create + enabled: true + scopes: + - integration:write + annotations: + readOnly: false + destructive: false + idempotent: true + title: Verify an email sender's domain + description: > + Check an email sender's domain with the email provider and return the DNS records it needs, each with its + hostname, type, value and status. Use the records to publish DNS: through the person's DNS host with + integrations-domain-connect-apply-url-create, or with your own DNS tools. When you add them yourself, add + the records whose status is not `success`, and always publish both SPF TXT records, which verify never + checks (their status copies other records). Look up each SPF hostname first: if it already has a `v=spf1` + record, merge `include:amazonses.com` into it before its `all` term instead of adding a second SPF record, + which breaks SPF for all mail at that name. Never create a second `_dmarc` TXT record either: if the domain + already has one, keep it and skip the DMARC record. Repeat this call until `status` is `success`, waiting at + least a minute between calls. DNS usually shows up within minutes but can take up to 72 hours. A success + makes every sender on the domain in this project able to send. + feature_flag: workflows-email-domain-agent-setup integrations-github-repos-retrieve: operation: integrations_github_repos_retrieve enabled: true diff --git a/products/posthog_ai/eval_harness/harness/services.py b/products/posthog_ai/eval_harness/harness/services.py index 8ff4ffb7bc55..1a18e58e87d2 100644 --- a/products/posthog_ai/eval_harness/harness/services.py +++ b/products/posthog_ai/eval_harness/harness/services.py @@ -143,10 +143,15 @@ def start_mcp_server( # add-cell, update-cell, delete-cell, get, list-frames). It also *hides* the # legacy notebooks-create / notebooks-retrieve pair, which the two surfaces # being mutually exclusive makes unavoidable — an eval of the legacy tools - # needs its own lever, not this one. mcp-exec-skills follows the run's + # needs its own lever, not this one. workflows-email-domain-agent-setup gates + # the email sender and Domain Connect tools. mcp-exec-skills follows the run's # skill delivery mode. "FEATURE_FLAG_OVERRIDES": json.dumps( - {"revamped-py-notebooks": True, MCP_EXEC_SKILLS_FEATURE_FLAG: exec_skills_enabled} + { + "revamped-py-notebooks": True, + "workflows-email-domain-agent-setup": True, + MCP_EXEC_SKILLS_FEATURE_FLAG: exec_skills_enabled, + } ), } diff --git a/products/posthog_ai/eval_harness/test/test_sandboxed_harness_services.py b/products/posthog_ai/eval_harness/test/test_sandboxed_harness_services.py index 73300a085eed..c4661f7d9ba8 100644 --- a/products/posthog_ai/eval_harness/test/test_sandboxed_harness_services.py +++ b/products/posthog_ai/eval_harness/test/test_sandboxed_harness_services.py @@ -66,6 +66,7 @@ def test_start_mcp_server_isolates_the_selected_skill_delivery( assert start.call_args.kwargs.get("readiness_timeout", 30) == (120 if exec_skills_enabled else 30) # The skill delivery switch must not displace the overrides every eval run relies on. assert overrides["revamped-py-notebooks"] is True + assert overrides["workflows-email-domain-agent-setup"] is True if skill_archive_url is None: assert "POSTHOG_MCP_SKILLS_URL" not in env else: diff --git a/products/workflows/evals/__init__.py b/products/workflows/evals/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/products/workflows/evals/eval_email_domain_setup.py b/products/workflows/evals/eval_email_domain_setup.py new file mode 100644 index 000000000000..806968814c49 --- /dev/null +++ b/products/workflows/evals/eval_email_domain_setup.py @@ -0,0 +1,340 @@ +"""Can an agent set up a Workflows email sending domain with the setting-up-an-email-domain skill? + +The agent drives the real email integration API through MCP. SES, DNS and Domain Connect +are simulated (see ``simulated_email_domains.py``), so each case controls which records +are already published and every status follows from that. + +The cases cover the three ways records get published, then the rules that keep an +existing setup intact and the errors an agent has to relay instead of working around: + +* ``domain_connect_link`` and ``manual_records`` are the two hand-offs on a new domain. +* ``already_verified_sender`` and ``second_sender_on_shared_domain`` must reuse what exists. +* ``existing_dmarc_kept``, ``existing_spf_merged`` and ``mail_from_label_taken`` must not + break records the domain already uses. +* ``free_mailbox_domain``, ``domain_claimed_by_another_org`` and ``member_cannot_verify`` + must end with the person knowing what to do next. + +Each case runs once per run. PostHog refuses a sending domain that another organization +uses, and every trial runs in its own organization, so a second trial of a case could +not create its sender. + +To run: + flox activate -- bash -c "hogli evals eval_email_domain_setup" +""" + +from __future__ import annotations + +from products.posthog_ai.eval_harness.base import SandboxedPrivateEval +from products.posthog_ai.eval_harness.config import SandboxedEvalCase +from products.posthog_ai.eval_harness.harness.context import EvalContext +from products.workflows.evals.scorers import ( + AvoidedTool, + BoundedVerifyPolling, + FinalMessageJudge, + FinalMessageMentions, + MergedSpfRecord, + RecordsHandoffJudge, + SendersInProject, + SharedApplyUrl, +) +from products.workflows.evals.seeders import ( + CLAIMED_DOMAIN_SENDER, + DOMAIN_CONNECT_SENDER, + EXISTING_DMARC_SENDER, + EXISTING_DMARC_VALUE, + EXISTING_SPF_INCLUDE, + EXISTING_SPF_SENDER, + FREE_MAILBOX_SENDER, + MAIL_FROM_TAKEN_SENDER, + MANUAL_SENDER, + MEMBER_SENDER, + SECOND_SENDER, + SHARED_DOMAIN_MAIL_FROM_SUBDOMAIN, + SHARED_DOMAIN_SENDER, + VERIFIED_SENDER, + domain_of, + seed_claimed_domain, + seed_domain_connect_support, + seed_existing_dmarc, + seed_existing_spf, + seed_helpdesk_mx, + seed_member_without_admin, + seed_new_sender, + seed_shared_domain_sender, + seed_verified_sender, +) +from products.workflows.evals.simulated_email_domains import ( + MX_PRIORITY, + SES_SPF_VALUE, + DnsRecord, + dkim_tokens, + mail_from_mx_host, + ses_records, + simulated_email_domains, + verification_token, +) + +EMAIL_CREATE_TOOL = "integrations-email-create" + +WAITS_FOR_DOMAIN_CONNECT_APPROVAL = ( + "A user asked an assistant to set up a sending domain for email. The assistant can only prepare a " + "link, and the user has to open it and approve the DNS changes at their DNS provider. Does the " + "message below give the user that link, and make clear the sender cannot send until they approve " + "it and the records verify? A message that says the domain is already verified or ready to send " + "is a no." +) +WAITS_FOR_MANUAL_RECORDS = ( + "A user asked an assistant to set up a sending domain for email, and the user has to add DNS " + "records at their DNS host. Does the message below ask the user to add the records, and make clear " + "the sender cannot send until they are published and verified? A message that says the domain is " + "already verified or ready to send is a no." +) +READY_WITHOUT_DNS_WORK = ( + "A user asked an assistant to set up an email sender on a domain whose DNS records are already in " + "place and verified. Does the message below tell the user the sender is verified and ready to send, " + "without making sending wait on new DNS records? A reminder to keep the existing SPF records published " + "is fine." +) +FREE_MAILBOX_EXPLAINED = ( + "A user asked an assistant to set up email sending from an address at a free mailbox provider such " + "as Gmail, which the product does not allow. Does the message below say the address cannot be used " + "because it is on a free mailbox provider, and suggest an address on a domain the user owns? A " + "message that says the sender was set up is a no." +) +CLAIMED_DOMAIN_EXPLAINED = ( + "A user asked an assistant to set up email sending from a domain that another organization already " + "uses, so the request was refused. Does the message below tell the user the domain is already in " + "use elsewhere, and point them to support or to a different domain? A message that says the sender " + "was set up is a no, and so is one that only reports an unexplained error." +) +HANDED_OVER_RECORDS = ( + "A user asked an assistant for the DNS records to add so a domain can send email. Does the " + "message below give the user every record listed in , each with its type, its host name " + "(fully qualified, relative to the user's DNS zone, or `@` for the domain itself), its exact value, " + "and the priority of the MX record? Layout and wording do not matter. A missing record, a wrong " + "type, a host name that points somewhere else, or a changed or shortened value is a no." +) +HANDED_OVER_RECORDS_KEEPING_DMARC = ( + f"{HANDED_OVER_RECORDS} The domain already publishes a DMARC record, `{EXISTING_DMARC_VALUE}`, " + "which must stay as it is, so it is not in the list. A message that tells the user to add a second " + "`_dmarc` record, or to replace the existing policy, is a no. Saying the existing record stays is fine." +) +HANDED_OVER_RECORDS_MERGING_SPF = ( + f"{HANDED_OVER_RECORDS} The sending domain already has the SPF record `v=spf1 {EXISTING_SPF_INCLUDE} ~all`, " + "and a host can publish only one SPF record. The list shows the single merged record that must " + "replace it at the sending domain. A message that adds a second SPF record at the sending domain, " + "or that drops the existing include, is a no." +) +MAIL_FROM_AVOIDS_HELPDESK = ( + "A user asked an assistant to set up an email sending domain. The `feedback` subdomain of the sending " + "domain already has an MX record for the user's helpdesk tool, which must keep working. Does the message " + "below put the bounce (MAIL FROM) MX and TXT records on a different subdomain, and leave the record at " + "`feedback` as it is? A message that tells the user to add, change, or replace a record at `feedback` is a no." +) +ADMIN_NEEDED_EXPLAINED = ( + "A user asked an assistant to set up an email sending domain, but the user is a project member " + "and a step needs project admin access, so the request was refused. Does the message below tell " + "the user that a project admin has to do that step or grant access, rather than describing a " + "broken tool or an unexplained failure? A message that says the domain is verified is a no." +) + + +def _exact_values(domain: str) -> list[str]: + return [verification_token(domain), *dkim_tokens(domain), mail_from_mx_host()] + + +def _records_reference(records: list[DnsRecord]) -> str: + return "\n".join( + f"{record.record_type} {record.name} {record.value}" + + (f" (priority {MX_PRIORITY})" if record.record_type == "MX" else "") + for record in records + ) + + +def _without_dmarc(domain: str) -> list[DnsRecord]: + return [record for record in ses_records(domain) if not record.name.startswith("_dmarc.")] + + +def _with_merged_root_spf(domain: str) -> list[DnsRecord]: + merged = SES_SPF_VALUE.replace("v=spf1", f"v=spf1 {EXISTING_SPF_INCLUDE}") + return [ + DnsRecord(name=record.name, record_type="TXT", value=merged) + if record.name == domain and record.value == SES_SPF_VALUE + else record + for record in ses_records(domain) + ] + + +def _cases() -> list[SandboxedEvalCase]: + return [ + SandboxedEvalCase( + name="domain_connect_link", + prompt=f"Set up {DOMAIN_CONNECT_SENDER} as our Workflows email sender, with Hedgebox as the display name.", + setup=seed_domain_connect_support, + expected={ + "senders_in_project": {"senders": {DOMAIN_CONNECT_SENDER: {"name": "Hedgebox"}}}, + "shared_apply_url": {}, + "bounded_verify_polling": {"max_calls": 3}, + "waits_for_domain_connect_approval": {}, + }, + ), + SandboxedEvalCase( + name="manual_records", + prompt=( + f"We want to send Workflows emails from {MANUAL_SENDER} (display name Hedgebox). " + "I'll add the DNS records myself, just tell me what to add." + ), + setup=seed_new_sender(MANUAL_SENDER), + expected={ + "senders_in_project": {"senders": {MANUAL_SENDER: {"name": "Hedgebox"}}}, + "final_message_mentions": {"values": _exact_values(domain_of(MANUAL_SENDER))}, + "handed_over_records": {"records": _records_reference(ses_records(domain_of(MANUAL_SENDER)))}, + "bounded_verify_polling": {"max_calls": 3}, + "waits_for_manual_records": {}, + }, + ), + SandboxedEvalCase( + name="already_verified_sender", + prompt=f"Can you set up {VERIFIED_SENDER} so our workflows can send email from it?", + setup=seed_verified_sender, + expected={ + "senders_in_project": {"senders": {VERIFIED_SENDER: {"verified": True}}}, + "avoided_tool": {"tools": [EMAIL_CREATE_TOOL]}, + "bounded_verify_polling": {"max_calls": 3}, + "ready_without_dns_work": {}, + }, + ), + SandboxedEvalCase( + name="second_sender_on_shared_domain", + prompt=f"Add {SECOND_SENDER} as another Workflows email sender, display name Hedgebox News.", + setup=seed_shared_domain_sender, + expected={ + "senders_in_project": { + "senders": { + SHARED_DOMAIN_SENDER: { + "mail_from_subdomain": SHARED_DOMAIN_MAIL_FROM_SUBDOMAIN, + "verified": True, + }, + SECOND_SENDER: { + "mail_from_subdomain": SHARED_DOMAIN_MAIL_FROM_SUBDOMAIN, + "name": "Hedgebox News", + "verified": True, + }, + } + }, + "bounded_verify_polling": {"max_calls": 3}, + "ready_without_dns_work": {}, + }, + ), + SandboxedEvalCase( + name="existing_dmarc_kept", + prompt=( + f"Set up {EXISTING_DMARC_SENDER} for Workflows email, display name Hedgebox. " + "I manage our DNS by hand, so list the records I need to add." + ), + setup=seed_existing_dmarc, + expected={ + "senders_in_project": {"senders": {EXISTING_DMARC_SENDER: {"name": "Hedgebox"}}}, + "final_message_mentions": {"values": _exact_values(domain_of(EXISTING_DMARC_SENDER))}, + "handed_over_records_keeping_dmarc": { + "records": _records_reference(_without_dmarc(domain_of(EXISTING_DMARC_SENDER))) + }, + "waits_for_manual_records": {}, + "bounded_verify_polling": {"max_calls": 3}, + }, + ), + SandboxedEvalCase( + name="existing_spf_merged", + prompt=( + f"Set up {EXISTING_SPF_SENDER} for Workflows email, display name Hedgebox. " + f"That domain already has an SPF record, `v=spf1 {EXISTING_SPF_INCLUDE} ~all`, for our " + "Google Workspace mail. I'll update DNS myself, so give me the exact records." + ), + setup=seed_existing_spf, + expected={ + "senders_in_project": {"senders": {EXISTING_SPF_SENDER: {"name": "Hedgebox"}}}, + "merged_spf_record": {"includes": [EXISTING_SPF_INCLUDE, "include:amazonses.com"]}, + "final_message_mentions": {"values": _exact_values(domain_of(EXISTING_SPF_SENDER))}, + "handed_over_records_merging_spf": { + "records": _records_reference(_with_merged_root_spf(domain_of(EXISTING_SPF_SENDER))) + }, + "waits_for_manual_records": {}, + "bounded_verify_polling": {"max_calls": 3}, + }, + ), + SandboxedEvalCase( + name="mail_from_label_taken", + prompt=( + f"Set up {MAIL_FROM_TAKEN_SENDER} as a Workflows sender, display name Hedgebox. Heads up: " + f"feedback.{domain_of(MAIL_FROM_TAKEN_SENDER)} already has an MX record for our helpdesk " + "tool, and that has to keep working." + ), + setup=seed_helpdesk_mx, + expected={ + "senders_in_project": {"senders": {MAIL_FROM_TAKEN_SENDER: {"mail_from_subdomain_not": "feedback"}}}, + "final_message_mentions": {"values": _exact_values(domain_of(MAIL_FROM_TAKEN_SENDER))}, + "mail_from_avoids_helpdesk": {}, + "waits_for_manual_records": {}, + "bounded_verify_polling": {"max_calls": 3}, + }, + ), + SandboxedEvalCase( + name="free_mailbox_domain", + prompt=f"Set up {FREE_MAILBOX_SENDER} as the sender for our Workflows emails.", + setup=seed_new_sender(FREE_MAILBOX_SENDER), + expected={ + "senders_in_project": {"senders": {}}, + "free_mailbox_explained": {}, + }, + ), + SandboxedEvalCase( + name="domain_claimed_by_another_org", + prompt=f"Set up {CLAIMED_DOMAIN_SENDER} as our Workflows email sender, display name Hedgebox.", + setup=seed_claimed_domain, + expected={ + "senders_in_project": {"senders": {}}, + "claimed_domain_explained": {}, + }, + ), + SandboxedEvalCase( + name="member_cannot_verify", + prompt=f"Set up {MEMBER_SENDER} as our Workflows email sender, display name Hedgebox.", + setup=seed_member_without_admin, + expected={ + "senders_in_project": {"senders": {MEMBER_SENDER: {"name": "Hedgebox"}}}, + "bounded_verify_polling": {"max_calls": 2}, + "admin_needed_explained": {}, + }, + ), + ] + + +async def eval_email_domain_setup(ctx: EvalContext) -> None: + if ctx.trials > 1: + raise ValueError("eval_email_domain_setup runs one trial per case, because trials of a case share its domain") + with simulated_email_domains(): + await SandboxedPrivateEval( + experiment_name="sandboxed-email-domain-setup-cli", + cases=_cases(), + scorers=[ + SendersInProject(), + AvoidedTool(), + FinalMessageMentions(), + MergedSpfRecord(), + SharedApplyUrl(), + BoundedVerifyPolling(), + FinalMessageJudge(name="waits_for_domain_connect_approval", question=WAITS_FOR_DOMAIN_CONNECT_APPROVAL), + FinalMessageJudge(name="waits_for_manual_records", question=WAITS_FOR_MANUAL_RECORDS), + FinalMessageJudge(name="ready_without_dns_work", question=READY_WITHOUT_DNS_WORK), + FinalMessageJudge(name="free_mailbox_explained", question=FREE_MAILBOX_EXPLAINED), + FinalMessageJudge(name="claimed_domain_explained", question=CLAIMED_DOMAIN_EXPLAINED), + FinalMessageJudge(name="admin_needed_explained", question=ADMIN_NEEDED_EXPLAINED), + RecordsHandoffJudge(name="handed_over_records", question=HANDED_OVER_RECORDS), + RecordsHandoffJudge( + name="handed_over_records_keeping_dmarc", question=HANDED_OVER_RECORDS_KEEPING_DMARC + ), + RecordsHandoffJudge(name="handed_over_records_merging_spf", question=HANDED_OVER_RECORDS_MERGING_SPF), + FinalMessageJudge(name="mail_from_avoids_helpdesk", question=MAIL_FROM_AVOIDS_HELPDESK), + ], + ctx=ctx, + ) diff --git a/products/workflows/evals/scorers.py b/products/workflows/evals/scorers.py new file mode 100644 index 000000000000..2fda1fc2001e --- /dev/null +++ b/products/workflows/evals/scorers.py @@ -0,0 +1,282 @@ +"""Scorers for the email domain setup eval suite. + +They grade outcomes rather than the path the agent took, so a reordered or extended +setup flow keeps passing: which senders the project holds after the run, what the final +message hands the person, and a few rules the skill exists to enforce. + +Every scorer reads its per-case parameters from ``expected`` under its own ``_name()`` +and returns ``score=None`` when its key is absent, so one scorer list spans the suite. +""" + +from __future__ import annotations + +import re +import html +import asyncio +from typing import Any + +from posthog.models.integration import Integration + +from products.posthog_ai.eval_harness.log_parser import LogParser, ToolCall +from products.posthog_ai.eval_harness.scorers import ( + BINARY_CHOICE_SCORES, + JUDGE_MODEL, + AsyncOnlyScorerMixin, + JudgedScorer, +) +from products.posthog_ai.eval_harness.scorers.contract import Score, Scorer +from products.workflows.evals.simulated_email_domains import DOMAIN_CONNECT_SYNC_UX + +__all__ = [ + "APPLY_URL_PATH", + "AvoidedTool", + "BoundedVerifyPolling", + "FinalMessageJudge", + "FinalMessageMentions", + "MergedSpfRecord", + "RecordsHandoffJudge", + "SendersInProject", + "SharedApplyUrl", +] + +APPLY_URL_TOOL = "integrations-domain-connect-apply-url-create" +VERIFY_TOOL = "integrations-email-verify-create" +APPLY_URL_PATH = f"{DOMAIN_CONNECT_SYNC_UX}/v2/domainTemplates/providers/posthog.com/" +APPLY_URL = re.compile(re.escape(APPLY_URL_PATH) + r"[^\s\"'\\)>\]]+") +SPF_RECORD = re.compile(r"v=spf1[^\n`\"'|]*", re.IGNORECASE) +SPF_ALL_MECHANISM = re.compile(r"^[~?+-]?all$", re.IGNORECASE) + + +def _spec(expected: dict | None, scorer_name: str) -> dict | None: + spec = (expected or {}).get(scorer_name) + return spec if isinstance(spec, dict) else None + + +def _parser(output: dict | None) -> LogParser | None: + raw_log = (output or {}).get("raw_log") + if not raw_log: + return None + return LogParser.cached(raw_log, initial_prompt=(output or {}).get("prompt", "") or "") + + +def _successful(parser: LogParser, name: str) -> list[ToolCall]: + return [call for call in parser.get_tool_calls(name) if not call.is_error] + + +def _final_message(output: dict | None) -> str: + return (output or {}).get("last_message") or "" + + +def _skip(name: str, reason: str) -> Score: + return Score(name=name, score=None, metadata={"reason": reason}) + + +def _missing_log(name: str) -> Score: + return Score(name=name, score=0.0, metadata={"reason": "No raw log"}) + + +def read_senders(team_id: int) -> dict[str, dict[str, Any]]: + return { + integration.integration_id: integration.config + for integration in Integration.objects.filter(team_id=team_id, kind="email") + } + + +class SendersInProject(AsyncOnlyScorerMixin, Scorer): + """Binary: does the project hold exactly the expected senders, configured as expected? + + Reads the database, so a sender created through any path counts, and so does a wrong + address or a duplicate the agent left behind. Per sender, each rule pins a config value, + such as ``verified`` or ``mail_from_subdomain``, and a ``_not`` rule rules one out. + """ + + def _name(self) -> str: + return "senders_in_project" + + async def _run_eval_async(self, output: dict | None, expected: dict | None = None, **kwargs: Any) -> Score: + spec = _spec(expected, self._name()) + if spec is None: + return _skip(self._name(), "Not applicable to this case") + team_id = ((output or {}).get("seed") or {}).get("team_id") + if not team_id: + return Score(name=self._name(), score=0.0, metadata={"reason": "No team id in the seed"}) + + senders = await asyncio.to_thread(read_senders, team_id) + wanted: dict[str, dict[str, Any]] = spec.get("senders", {}) + problems = [ + *(f"missing {address}" for address in wanted.keys() - senders.keys()), + *(f"unexpected {address}" for address in senders.keys() - wanted.keys()), + *( + problem + for address, rules in wanted.items() + if address in senders + for problem in self._config_problems(address, senders[address], rules) + ), + ] + if problems: + return Score(name=self._name(), score=0.0, metadata={"problems": problems}) + return Score(name=self._name(), score=1.0, metadata={"senders": sorted(senders)}) + + def _config_problems(self, address: str, config: dict[str, Any], rules: dict[str, Any]) -> list[str]: + problems = [] + for rule, value in rules.items(): + key = rule.removesuffix("_not") + if rule != key and config.get(key) == value: + problems.append(f"{address} has {key}={value!r}, which this case rules out") + elif rule == key and config.get(key) != value: + problems.append(f"{address} has {key}={config.get(key)!r}, expected {value!r}") + return problems + + +class AvoidedTool(Scorer): + """Binary: did no call to any of ``expected.tools`` succeed? A refused attempt passes.""" + + def _name(self) -> str: + return "avoided_tool" + + def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs: Any) -> Score: + spec = _spec(expected, self._name()) + if spec is None: + return _skip(self._name(), "Not applicable to this case") + parser = _parser(output) + if parser is None: + return _missing_log(self._name()) + called = sorted(tool for tool in spec.get("tools", []) if _successful(parser, tool)) + return Score(name=self._name(), score=0.0 if called else 1.0, metadata={"called": called}) + + +class FinalMessageMentions(Scorer): + """Binary: does the final message contain every value in ``expected.values``? + + Record values are exact strings a person has to copy into their DNS host, so a + summary that paraphrases or truncates one hands over a record that cannot verify. + """ + + def _name(self) -> str: + return "final_message_mentions" + + def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs: Any) -> Score: + spec = _spec(expected, self._name()) + if spec is None: + return _skip(self._name(), "Not applicable to this case") + message = _final_message(output) + missing = [value for value in spec.get("values", []) if value not in message] + return Score(name=self._name(), score=0.0 if missing else 1.0, metadata={"missing": missing}) + + +class MergedSpfRecord(Scorer): + """Binary: does one SPF record in the final message carry every include in ``expected.includes``? + + A domain can publish only one SPF record, so handing over a second one breaks the mail + the existing include authorizes. Receivers stop reading at the ``all`` mechanism, so an + include after it authorizes nothing. Which host the record goes on is the judge's call. + """ + + def _name(self) -> str: + return "merged_spf_record" + + def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs: Any) -> Score: + spec = _spec(expected, self._name()) + if spec is None: + return _skip(self._name(), "Not applicable to this case") + includes = {include.lower() for include in spec.get("includes", [])} + records = SPF_RECORD.findall(_final_message(output)) + merged = [record for record in records if includes <= self._effective_mechanisms(record)] + return Score(name=self._name(), score=1.0 if merged else 0.0, metadata={"spf_records": records}) + + def _effective_mechanisms(self, record: str) -> set[str]: + mechanisms: set[str] = set() + for term in record.lower().split()[1:]: + if SPF_ALL_MECHANISM.match(term): + break + mechanisms.add(term) + return mechanisms + + +class SharedApplyUrl(Scorer): + """Binary: does the final message carry the complete URL a successful apply-url call returned? + + Only the person can approve the change at their DNS host, so a URL that stays in a + tool result helps nobody, and a shortened one fails its signature check. + """ + + def _name(self) -> str: + return "shared_apply_url" + + def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs: Any) -> Score: + if _spec(expected, self._name()) is None: + return _skip(self._name(), "Not applicable to this case") + parser = _parser(output) + if parser is None: + return _missing_log(self._name()) + returned = {url for call in _successful(parser, APPLY_URL_TOOL) for url in APPLY_URL.findall(call.output)} + if not returned: + return Score(name=self._name(), score=0.0, metadata={"reason": f"No URL from a {APPLY_URL_TOOL} call"}) + message = html.unescape(_final_message(output)) + shared = any(url in message for url in returned) + return Score(name=self._name(), score=1.0 if shared else 0.0, metadata={"url_in_message": shared}) + + +class BoundedVerifyPolling(Scorer): + """Binary: did the agent call verify at most ``expected.max_calls`` times? + + Each verify call queries DNS and the email provider. When the records still wait on the + person, or a permission error repeats, more calls cannot change the answer. + """ + + def _name(self) -> str: + return "bounded_verify_polling" + + def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs: Any) -> Score: + spec = _spec(expected, self._name()) + if spec is None: + return _skip(self._name(), "Not applicable to this case") + parser = _parser(output) + if parser is None: + return _missing_log(self._name()) + calls = len(parser.get_tool_calls(VERIFY_TOOL)) + return Score(name=self._name(), score=1.0 if calls <= spec["max_calls"] else 0.0, metadata={"calls": calls}) + + +class FinalMessageJudge(JudgedScorer): + """Judge one yes/no question about the final message. ``name`` is the ``expected`` key that opts a case in.""" + + def __init__(self, *, name: str, question: str, **kwargs: Any) -> None: + super().__init__( + name=name, + prompt_template=self._prompt_template(question), + choice_scores=BINARY_CHOICE_SCORES, + model=JUDGE_MODEL, + max_completion_tokens=256, + **kwargs, + ) + + def _prompt_template(self, question: str) -> str: + return f"{question}\n\n{{{{output.last_message}}}}\n\nAnswer `yes` or `no`." + + def _prepare(self, output: dict | None, expected: dict | None) -> dict[str, Any] | Score: + if _spec(expected, self._name()) is None: + return _skip(self._name(), "Not applicable to this case") + if not _final_message(output): + return Score(name=self._name(), score=0.0, metadata={"reason": "No final message"}) + return {"output": {"last_message": _final_message(output)}} + + +class RecordsHandoffJudge(FinalMessageJudge): + """Judge whether the final message hands over ``expected.records`` the way ``question`` asks. + + The reference list lets the judge check every record, while the exact long values stay + with ``FinalMessageMentions``, which cannot miss a changed character. + """ + + def _prompt_template(self, question: str) -> str: + return ( + f"{question}\n\n{{{{expected.records}}}}\n\n" + "{{output.last_message}}\n\nAnswer `yes` or `no`." + ) + + def _prepare(self, output: dict | None, expected: dict | None) -> dict[str, Any] | Score: + prepared = super()._prepare(output, expected) + if isinstance(prepared, Score): + return prepared + return {**prepared, "expected": {"records": (_spec(expected, self._name()) or {})["records"]}} diff --git a/products/workflows/evals/seeders.py b/products/workflows/evals/seeders.py new file mode 100644 index 000000000000..9b8c777e1c61 --- /dev/null +++ b/products/workflows/evals/seeders.py @@ -0,0 +1,172 @@ +"""Seeders for the email domain setup eval cases. + +Each case sends from its own subdomain of a reserved example domain, so the records one +case publishes never answer for another. The addresses are module constants because the +prompt, the seeder and the scorers all have to mean the same sender. + +PostHog refuses a sending domain that another organization already uses, and every case +runs in a new organization in a database that outlives the run. So each address carries +a label that is new for every run, and a domain from an earlier run never blocks this one. + +Seeders that change a policy verify it took effect before returning. A policy that did +not apply would turn its case into a hollow pass, while raising marks the case an infra +error, which the harness keeps out of score averages. +""" + +from __future__ import annotations + +import uuid +from collections.abc import Callable +from typing import Any + +from posthog.models.integration import Integration +from posthog.models.organization import OrganizationMembership +from posthog.models.team import Team +from posthog.models.user import User +from posthog.user_permissions import UserPermissions + +from products.tasks.backend.facade.agents import CustomPromptSandboxContext +from products.workflows.evals.simulated_email_domains import ( + SIMULATED_EMAIL_DOMAINS, + DnsRecord, + publish_domain_connect_support, + ses_records, +) + +__all__ = [ + "CLAIMED_DOMAIN_SENDER", + "DOMAIN_CONNECT_SENDER", + "EXISTING_DMARC_SENDER", + "EXISTING_DMARC_VALUE", + "EXISTING_SPF_INCLUDE", + "EXISTING_SPF_SENDER", + "FREE_MAILBOX_SENDER", + "MAIL_FROM_TAKEN_SENDER", + "MANUAL_SENDER", + "MEMBER_SENDER", + "SECOND_SENDER", + "SHARED_DOMAIN_MAIL_FROM_SUBDOMAIN", + "SHARED_DOMAIN_SENDER", + "VERIFIED_SENDER", + "domain_of", + "seed_claimed_domain", + "seed_domain_connect_support", + "seed_existing_dmarc", + "seed_existing_spf", + "seed_helpdesk_mx", + "seed_member_without_admin", + "seed_new_sender", + "seed_shared_domain_sender", + "seed_verified_sender", +] + +RUN_LABEL = uuid.uuid4().hex[:8] + +DOMAIN_CONNECT_SENDER = f"hello@mail.connect-{RUN_LABEL}.example.net" +MANUAL_SENDER = f"hello@mail.manual-{RUN_LABEL}.example.com" +VERIFIED_SENDER = f"hello@mail.ready-{RUN_LABEL}.example.com" +SHARED_DOMAIN_SENDER = f"ops@mail.shared-{RUN_LABEL}.example.com" +SHARED_DOMAIN_MAIL_FROM_SUBDOMAIN = "bounce" +SECOND_SENDER = f"newsletter@mail.shared-{RUN_LABEL}.example.com" +FREE_MAILBOX_SENDER = "hedgebox.team@gmail.com" +CLAIMED_DOMAIN_SENDER = f"hello@mail.claimed-{RUN_LABEL}.example.org" +EXISTING_DMARC_SENDER = f"hello@mail.dmarc-{RUN_LABEL}.example.org" +EXISTING_DMARC_VALUE = "v=DMARC1; p=quarantine; rua=mailto:dmarc@example.org" +EXISTING_SPF_SENDER = f"hello@mail.spf-{RUN_LABEL}.example.org" +EXISTING_SPF_INCLUDE = "include:_spf.google.com" +HELPDESK_MX_HOST = "mx.helpdesk.example.com" +MEMBER_SENDER = f"hello@mail.member-{RUN_LABEL}.example.com" +MAIL_FROM_TAKEN_SENDER = f"hello@mail.helpdesk-{RUN_LABEL}.example.com" + + +def domain_of(address: str) -> str: + return address.split("@", 1)[1] + + +def _seeded(context: CustomPromptSandboxContext, sender: str, **extra: Any) -> dict[str, Any]: + return {"team_id": context.team_id, "sender": sender, **extra} + + +def seed_new_sender(sender: str) -> Callable[[CustomPromptSandboxContext], dict[str, Any]]: + """A seeder for a case that starts from an empty project and needs only its team recorded.""" + + def seed(context: CustomPromptSandboxContext) -> dict[str, Any]: + return _seeded(context, sender) + + return seed + + +def _create_sender(context: CustomPromptSandboxContext, address: str, *, mail_from_subdomain: str) -> Integration: + domain = domain_of(address) + return Integration.objects.create( + team_id=context.team_id, + kind="email", + integration_id=address, + created_by_id=context.user_id, + config={ + "email": address, + "domain": domain, + "mail_from_subdomain": mail_from_subdomain, + "name": "Hedgebox", + "provider": "ses", + "verified": True, + }, + ) + + +def _publish_verified_domain(context: CustomPromptSandboxContext, domain: str, mail_from_subdomain: str) -> None: + SIMULATED_EMAIL_DOMAINS.publish(ses_records(domain, mail_from_subdomain)) + SIMULATED_EMAIL_DOMAINS.associate(context.team_id, domain) + + +def seed_domain_connect_support(context: CustomPromptSandboxContext) -> dict[str, Any]: + publish_domain_connect_support() + return _seeded(context, DOMAIN_CONNECT_SENDER) + + +def seed_verified_sender(context: CustomPromptSandboxContext) -> dict[str, Any]: + sender = _create_sender(context, VERIFIED_SENDER, mail_from_subdomain="feedback") + _publish_verified_domain(context, domain_of(VERIFIED_SENDER), "feedback") + return _seeded(context, VERIFIED_SENDER, integration_id=sender.id) + + +def seed_shared_domain_sender(context: CustomPromptSandboxContext) -> dict[str, Any]: + sender = _create_sender(context, SHARED_DOMAIN_SENDER, mail_from_subdomain=SHARED_DOMAIN_MAIL_FROM_SUBDOMAIN) + _publish_verified_domain(context, domain_of(SHARED_DOMAIN_SENDER), SHARED_DOMAIN_MAIL_FROM_SUBDOMAIN) + return _seeded(context, SHARED_DOMAIN_SENDER, integration_id=sender.id) + + +def seed_claimed_domain(context: CustomPromptSandboxContext) -> dict[str, Any]: + SIMULATED_EMAIL_DOMAINS.claim_for_other_organization(domain_of(CLAIMED_DOMAIN_SENDER)) + return _seeded(context, CLAIMED_DOMAIN_SENDER) + + +def seed_existing_dmarc(context: CustomPromptSandboxContext) -> dict[str, Any]: + domain = domain_of(EXISTING_DMARC_SENDER) + SIMULATED_EMAIL_DOMAINS.publish([DnsRecord(name=f"_dmarc.{domain}", record_type="TXT", value=EXISTING_DMARC_VALUE)]) + return _seeded(context, EXISTING_DMARC_SENDER) + + +def seed_existing_spf(context: CustomPromptSandboxContext) -> dict[str, Any]: + domain = domain_of(EXISTING_SPF_SENDER) + SIMULATED_EMAIL_DOMAINS.publish( + [DnsRecord(name=domain, record_type="TXT", value=f"v=spf1 {EXISTING_SPF_INCLUDE} ~all")] + ) + return _seeded(context, EXISTING_SPF_SENDER) + + +def seed_helpdesk_mx(context: CustomPromptSandboxContext) -> dict[str, Any]: + domain = domain_of(MAIL_FROM_TAKEN_SENDER) + SIMULATED_EMAIL_DOMAINS.publish([DnsRecord(name=f"feedback.{domain}", record_type="MX", value=HELPDESK_MX_HOST)]) + return _seeded(context, MAIL_FROM_TAKEN_SENDER) + + +def seed_member_without_admin(context: CustomPromptSandboxContext) -> dict[str, Any]: + team = Team.objects.get(id=context.team_id) + OrganizationMembership.objects.filter(user_id=context.user_id, organization_id=team.organization_id).update( + level=OrganizationMembership.Level.MEMBER + ) + user = User.objects.get(id=context.user_id) + if UserPermissions(user).team(team).effective_membership_level != OrganizationMembership.Level.MEMBER: + raise RuntimeError("The case user still has admin access to the project, so this case would test nothing") + return _seeded(context, MEMBER_SENDER) diff --git a/products/workflows/evals/simulated_email_domains.py b/products/workflows/evals/simulated_email_domains.py new file mode 100644 index 000000000000..eb91e957ee45 --- /dev/null +++ b/products/workflows/evals/simulated_email_domains.py @@ -0,0 +1,363 @@ +"""Simulated SES, DNS and Domain Connect for the email domain eval suite. + +The agent drives the real API through MCP, and the real SES provider code builds the +DNS records and statuses it reads. Only the network edges are replaced: the boto3 +clients, DNS answers for the reserved example domains, and the Domain Connect provider +settings. Each case publishes its own records, so every status the agent sees follows +from what the case seeded, the same way it follows from a real DNS zone. +""" + +from __future__ import annotations + +import uuid +import hashlib +import threading +from collections.abc import Iterable, Iterator +from contextlib import ExitStack, contextmanager +from typing import Any, cast + +from unittest.mock import patch + +from django.conf import settings +from django.core.cache.backends.locmem import LocMemCache + +import dns.rdata +import dns.resolver +import dns.rdatatype +import dns.rdataclass +from cryptography.hazmat.primitives.asymmetric import rsa + +from posthog import domain_connect +from posthog.dataclasses import frozen + +from products.workflows.backend import providers +from products.workflows.backend.facade.contracts import EmailDomainVerification +from products.workflows.backend.providers.ses import SESProvider + +__all__ = [ + "DOMAIN_CONNECT_ROOT_DOMAIN", + "DOMAIN_CONNECT_SYNC_UX", + "MX_PRIORITY", + "SES_SPF_VALUE", + "SIMULATED_EMAIL_DOMAINS", + "DnsRecord", + "SimulatedEmailDomains", + "SimulatedSESProvider", + "dkim_tokens", + "mail_from_mx_host", + "publish_domain_connect_support", + "ses_records", + "simulated_email_domains", + "verification_token", +] + +SIMULATED_ROOT_DOMAINS = ("example.com", "example.net", "example.org") +DOMAIN_CONNECT_ROOT_DOMAIN = "example.net" +DOMAIN_CONNECT_ENDPOINT = next( + endpoint + for endpoint, name in domain_connect.DOMAIN_CONNECT_PROVIDERS.items() + if name == domain_connect.DomainConnectProviderName.CLOUDFLARE +) +DOMAIN_CONNECT_SYNC_UX = "https://dash.cloudflare.com/domainconnect" +SES_SPF_VALUE = "v=spf1 include:amazonses.com ~all" +MX_PRIORITY = 10 + + +@frozen +class DnsRecord: + name: str + record_type: str + value: str + + +def _digest(purpose: str, domain: str) -> str: + return hashlib.sha256(f"{purpose}:{domain}".encode()).hexdigest() + + +def verification_token(domain: str) -> str: + return _digest("verification", domain)[:44] + + +def dkim_tokens(domain: str) -> list[str]: + return [_digest(f"dkim-{index}", domain)[:32] for index in range(3)] + + +def mail_from_mx_host() -> str: + return f"feedback-smtp.{settings.SES_REGION}.amazonses.com" + + +def _ownership_record(domain: str) -> DnsRecord: + return DnsRecord(name=f"_amazonses.{domain}", record_type="TXT", value=verification_token(domain)) + + +def _dkim_records(domain: str) -> list[DnsRecord]: + return [ + DnsRecord(name=f"{token}._domainkey.{domain}", record_type="CNAME", value=f"{token}.dkim.amazonses.com") + for token in dkim_tokens(domain) + ] + + +def _mail_from_mx_record(mail_from_domain: str) -> DnsRecord: + return DnsRecord(name=mail_from_domain, record_type="MX", value=mail_from_mx_host()) + + +def ses_records(domain: str, mail_from_subdomain: str = "feedback") -> list[DnsRecord]: + """Every record SES asks for, with the values the simulated SES issues for `domain`.""" + mail_from_domain = f"{mail_from_subdomain}.{domain}" + return [ + _ownership_record(domain), + *_dkim_records(domain), + DnsRecord(name=domain, record_type="TXT", value=SES_SPF_VALUE), + _mail_from_mx_record(mail_from_domain), + DnsRecord(name=mail_from_domain, record_type="TXT", value=SES_SPF_VALUE), + DnsRecord(name=f"_dmarc.{domain}", record_type="TXT", value="v=DMARC1; p=none;"), + ] + + +def _normalize(name: str) -> str: + return name.rstrip(".").lower() + + +def _tenant_name(team_id: int) -> str: + return f"team-{team_id}" + + +class SimulatedEmailDomains: + """The DNS zones and SES tenant ownership every simulated call reads. + + Seeders write it and the live server's request threads read it, so access is locked. + Tenant ownership is keyed by team, so a case only owns the identities its own team + created, the way SES tenants scope them. + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + self._records: dict[tuple[str, str], set[str]] = {} + self._other_organization_domains: set[str] = set() + self._team_domains: set[tuple[int, str]] = set() + self._mail_from_domains: dict[str, str] = {} + + def covers(self, name: str) -> bool: + normalized = _normalize(name) + return any(normalized == root or normalized.endswith(f".{root}") for root in SIMULATED_ROOT_DOMAINS) + + def publish(self, records: Iterable[DnsRecord]) -> None: + with self._lock: + for record in records: + self._records.setdefault((_normalize(record.name), record.record_type), set()).add(record.value) + + def answers(self, name: str, record_type: str) -> set[str]: + with self._lock: + return set(self._records.get((_normalize(name), record_type), set())) + + def claim_for_other_organization(self, domain: str) -> None: + with self._lock: + self._other_organization_domains.add(_normalize(domain)) + + def associate(self, team_id: int, domain: str) -> None: + with self._lock: + self._team_domains.add((team_id, _normalize(domain))) + + def disassociate(self, team_id: int, domain: str) -> None: + with self._lock: + self._team_domains.discard((team_id, _normalize(domain))) + + def tenants(self, domain: str, team_id: int | None) -> set[str]: + normalized = _normalize(domain) + with self._lock: + tenants = {"team-0"} if normalized in self._other_organization_domains else set() + if team_id is not None and (team_id, normalized) in self._team_domains: + tenants.add(_tenant_name(team_id)) + return tenants + + def set_mail_from_domain(self, domain: str, mail_from_domain: str) -> None: + with self._lock: + self._mail_from_domains[_normalize(domain)] = mail_from_domain + + def mail_from_domain(self, domain: str) -> str | None: + with self._lock: + return self._mail_from_domains.get(_normalize(domain)) + + def is_published(self, record: DnsRecord) -> bool: + return record.value in self.answers(record.name, record.record_type) + + +SIMULATED_EMAIL_DOMAINS = SimulatedEmailDomains() + + +def _domain_from_identity_arn(arn: str) -> str | None: + _, separator, identity = arn.partition(":identity/") + return identity if separator else None + + +class _SimulatedSes: + def __init__(self, domains: SimulatedEmailDomains) -> None: + self._domains = domains + + def verify_domain_identity(self, *, Domain: str) -> dict[str, Any]: + return {"VerificationToken": verification_token(Domain)} + + def verify_domain_dkim(self, *, Domain: str) -> dict[str, Any]: + return {"DkimTokens": dkim_tokens(Domain)} + + def set_identity_mail_from_domain(self, *, Identity: str, MailFromDomain: str, **_: Any) -> dict[str, Any]: + self._domains.set_mail_from_domain(Identity, MailFromDomain) + return {} + + def delete_identity(self, *, Identity: str) -> dict[str, Any]: + return {} + + def get_identity_verification_attributes(self, *, Identities: list[str]) -> dict[str, Any]: + return { + "VerificationAttributes": { + domain: {"VerificationStatus": self._status([_ownership_record(domain)])} for domain in Identities + } + } + + def get_identity_dkim_attributes(self, *, Identities: list[str]) -> dict[str, Any]: + return { + "DkimAttributes": { + domain: {"DkimVerificationStatus": self._status(_dkim_records(domain))} for domain in Identities + } + } + + def get_identity_mail_from_domain_attributes(self, *, Identities: list[str]) -> dict[str, Any]: + return { + "MailFromDomainAttributes": { + domain: {"MailFromDomainStatus": self._status([_mail_from_mx_record(mail_from_domain)])} + for domain in Identities + if (mail_from_domain := self._domains.mail_from_domain(domain)) + } + } + + def _status(self, required: list[DnsRecord]) -> str: + return "Success" if all(self._domains.is_published(record) for record in required) else "Pending" + + +class _SimulatedSesV2: + def __init__(self, domains: SimulatedEmailDomains) -> None: + self._domains = domains + self.team_id: int | None = None + + def list_resource_tenants(self, *, ResourceArn: str) -> dict[str, Any]: + domain = _domain_from_identity_arn(ResourceArn) or "" + return { + "ResourceTenants": [{"TenantName": name} for name in sorted(self._domains.tenants(domain, self.team_id))] + } + + def create_tenant(self, **_: Any) -> dict[str, Any]: + return {} + + def create_tenant_resource_association(self, *, TenantName: str, ResourceArn: str) -> dict[str, Any]: + domain = _domain_from_identity_arn(ResourceArn) + if domain and self.team_id is not None and TenantName == _tenant_name(self.team_id): + self._domains.associate(self.team_id, domain) + return {} + + def delete_tenant_resource_association(self, *, TenantName: str, ResourceArn: str) -> dict[str, Any]: + domain = _domain_from_identity_arn(ResourceArn) + if domain and self.team_id is not None: + self._domains.disassociate(self.team_id, domain) + return {} + + +class _SimulatedSts: + def get_caller_identity(self) -> dict[str, Any]: + return {"Account": "000000000000"} + + +class SimulatedSESProvider(SESProvider): + """The real SES provider logic on top of simulated AWS clients.""" + + def __init__(self, domains: SimulatedEmailDomains) -> None: + self._simulated_v2 = _SimulatedSesV2(domains) + self.sts_client = cast(Any, _SimulatedSts()) + self.ses_client = cast(Any, _SimulatedSes(domains)) + self.ses_v2_client = cast(Any, self._simulated_v2) + self.ses_v2_metrics_client = self.ses_v2_client + + def create_email_domain( + self, domain: str, mail_from_subdomain: str, team_id: int, org_team_ids: Iterable[int] | None = None + ) -> None: + self._simulated_v2.team_id = team_id + super().create_email_domain(domain, mail_from_subdomain, team_id, org_team_ids) + + def verify_email_domain(self, domain: str, mail_from_subdomain: str, team_id: int) -> EmailDomainVerification: + self._simulated_v2.team_id = team_id + return super().verify_email_domain(domain, mail_from_subdomain, team_id) + + +def _rdata(record_type: str, value: str) -> dns.rdata.Rdata: + text = { + "TXT": '"{}"'.format(value.replace('"', '\\"')), + "CNAME": f"{value}.", + "MX": f"{MX_PRIORITY} {value}.", + }[record_type] + return dns.rdata.from_text(dns.rdataclass.IN, dns.rdatatype.from_text(record_type), text) + + +def _simulated_answers(domains: SimulatedEmailDomains, qname: Any, rdtype: Any) -> list[dns.rdata.Rdata]: + record_type = dns.rdatatype.to_text(dns.rdatatype.RdataType.make(rdtype)) + values = domains.answers(str(qname), record_type) + if not values: + raise dns.resolver.NXDOMAIN() + return [_rdata(record_type, value) for value in sorted(values)] + + +@contextmanager +def simulated_email_domains(domains: SimulatedEmailDomains = SIMULATED_EMAIL_DOMAINS) -> Iterator[None]: + """Route SES, DNS and Domain Connect calls for the example domains through `domains`. + + Lookups for any other name reach the real resolver, so suites running alongside this + one in the same process keep working. Their Domain Connect lookups use a fresh cache + while this runs, which costs a repeated lookup and nothing else. + """ + original_resolver_method = dns.resolver.Resolver.resolve + original_resolve = dns.resolver.resolve + original_fetch_settings = domain_connect._fetch_provider_settings + original_signing_key = domain_connect.get_signing_key + eval_signing_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + + def resolver_method( + resolver: dns.resolver.Resolver, qname: Any, rdtype: Any = "A", *args: Any, **kwargs: Any + ) -> list[dns.rdata.Rdata] | dns.resolver.Answer: + if domains.covers(str(qname)): + return _simulated_answers(domains, qname, rdtype) + return original_resolver_method(resolver, qname, rdtype, *args, **kwargs) + + def resolve( + qname: Any, rdtype: Any = "A", *args: Any, **kwargs: Any + ) -> list[dns.rdata.Rdata] | dns.resolver.Answer: + if domains.covers(str(qname)): + return _simulated_answers(domains, qname, rdtype) + return original_resolve(qname, rdtype, *args, **kwargs) + + def fetch_provider_settings(endpoint: str, domain: str) -> dict | None: + if domains.covers(domain): + return {"providerName": "Cloudflare", "urlSyncUX": DOMAIN_CONNECT_SYNC_UX} + return original_fetch_settings(endpoint, domain) + + with ExitStack() as stack: + stack.enter_context(patch.object(providers, "SESProvider", lambda: SimulatedSESProvider(domains))) + stack.enter_context(patch.object(dns.resolver.Resolver, "resolve", resolver_method)) + stack.enter_context(patch.object(dns.resolver, "resolve", resolve)) + stack.enter_context(patch.object(domain_connect, "_fetch_provider_settings", fetch_provider_settings)) + stack.enter_context( + patch.object(domain_connect, "get_signing_key", lambda: original_signing_key() or eval_signing_key) + ) + # Discovery caches each domain for an hour, and LocMemCache instances with the same + # name share one store, so each simulation gets a store no earlier lookup has filled. + stack.enter_context( + patch.object(domain_connect, "cache", LocMemCache(f"email-domain-evals-{uuid.uuid4()}", {})) + ) + yield + + +def publish_domain_connect_support(domains: SimulatedEmailDomains = SIMULATED_EMAIL_DOMAINS) -> None: + domains.publish( + [ + DnsRecord( + name=f"_domainconnect.{DOMAIN_CONNECT_ROOT_DOMAIN}", record_type="TXT", value=DOMAIN_CONNECT_ENDPOINT + ) + ] + ) diff --git a/products/workflows/evals/test/__init__.py b/products/workflows/evals/test/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/products/workflows/evals/test/test_scorers.py b/products/workflows/evals/test/test_scorers.py new file mode 100644 index 000000000000..6910ce5b956f --- /dev/null +++ b/products/workflows/evals/test/test_scorers.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import pytest + +from products.workflows.evals.scorers import FinalMessageMentions, MergedSpfRecord + +EXPECTED = {"merged_spf_record": {"includes": ["include:_spf.google.com", "include:amazonses.com"]}} +TOKEN = "3f1c9a52b7e04c1e9d0a2f4b8e6c9a10" + + +@pytest.mark.parametrize( + "message,score", + [ + ("Replace it with:\n```\nv=spf1 include:_spf.google.com include:amazonses.com ~all\n```", 1.0), + ("| TXT | @ | `v=spf1 include:_spf.google.com include:amazonses.com ~all` |", 1.0), + ("Keep `v=spf1 include:_spf.google.com ~all` and add `v=spf1 include:amazonses.com ~all`", 0.0), + ("Add `v=spf1 include:amazonses.com ~all` at the root.", 0.0), + ("Use `v=spf1 include:_spf.google.com ~all include:amazonses.com`", 0.0), + ], +) +def test_merged_spf_record_needs_one_record_with_every_include(message: str, score: float) -> None: + result = MergedSpfRecord().eval({"last_message": message}, expected=EXPECTED) + + assert result.score == score + + +@pytest.mark.parametrize( + "message,score", + [ + (f"| TXT | _amazonses.mail.example.com | `{TOKEN}` |", 1.0), + (f"| TXT | _amazonses.mail.example.com | `{TOKEN.upper()}` |", 0.0), + (f"| TXT | _amazonses.mail.example.com | `{TOKEN[:16]}...` |", 0.0), + ], +) +def test_final_message_mentions_needs_each_value_exactly(message: str, score: float) -> None: + result = FinalMessageMentions().eval( + {"last_message": message}, expected={"final_message_mentions": {"values": [TOKEN]}} + ) + + assert result.score == score diff --git a/products/workflows/evals/test/test_simulated_email_domains.py b/products/workflows/evals/test/test_simulated_email_domains.py new file mode 100644 index 000000000000..1569e91edaf7 --- /dev/null +++ b/products/workflows/evals/test/test_simulated_email_domains.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import pytest +from posthog.test.base import APIBaseTest +from unittest.mock import patch + +from rest_framework.exceptions import ValidationError + +from posthog import domain_connect +from posthog.models.organization import OrganizationMembership + +from products.tasks.backend.facade.agents import CustomPromptSandboxContext +from products.workflows.backend import providers +from products.workflows.evals.scorers import APPLY_URL_PATH +from products.workflows.evals.seeders import DOMAIN_CONNECT_SENDER, seed_domain_connect_support +from products.workflows.evals.simulated_email_domains import ( + DOMAIN_CONNECT_ROOT_DOMAIN, + SimulatedEmailDomains, + publish_domain_connect_support, + ses_records, + simulated_email_domains, +) + +DOMAIN = "mail.unit.example.com" +TEAM_ID = 101 + + +def _records_by_name(records: list) -> dict[tuple[str, str], str]: + return {(record["recordHostname"], record["recordType"]): record["status"] for record in records} + + +@pytest.mark.parametrize( + "published_label,expected_status", + [ + (None, "pending"), + ("feedback", "success"), + ("bounce", "pending"), + ], +) +def test_verify_status_follows_published_records(published_label: str | None, expected_status: str) -> None: + domains = SimulatedEmailDomains() + if published_label: + domains.publish(ses_records(DOMAIN, published_label)) + domains.associate(TEAM_ID, DOMAIN) + + with simulated_email_domains(domains): + result = providers.SESProvider().verify_email_domain(DOMAIN, mail_from_subdomain="feedback", team_id=TEAM_ID) + + assert result["status"] == expected_status + issued = {(record.name if record.name != DOMAIN else "@", record.record_type) for record in ses_records(DOMAIN)} + assert set(_records_by_name(result["dnsRecords"])) == issued + + +def test_create_refuses_a_domain_another_organization_uses_but_not_another_trial() -> None: + domains = SimulatedEmailDomains() + with simulated_email_domains(domains): + providers.SESProvider().create_email_domain(DOMAIN, mail_from_subdomain="feedback", team_id=TEAM_ID) + providers.SESProvider().create_email_domain(DOMAIN, mail_from_subdomain="feedback", team_id=TEAM_ID + 1) + + domains.claim_for_other_organization(DOMAIN) + with pytest.raises(ValidationError, match="another organization"): + providers.SESProvider().create_email_domain(DOMAIN, mail_from_subdomain="feedback", team_id=TEAM_ID) + + +@pytest.mark.parametrize( + "root_domain,supported", + [ + (DOMAIN_CONNECT_ROOT_DOMAIN, True), + ("example.com", False), + ], +) +def test_domain_connect_is_supported_only_where_published(root_domain: str, supported: bool) -> None: + domains = SimulatedEmailDomains() + publish_domain_connect_support(domains) + + with simulated_email_domains(domains): + discovery = domain_connect.discover_domain_connect(root_domain) + + assert (discovery is not None) is supported + + +class TestEvalCasesThroughTheApi(APIBaseTest): + def _create_sender(self, address: str, **config: str) -> int: + response = self.client.post( + f"/api/environments/{self.team.pk}/integrations", + {"kind": "email", "config": {"email": address, "name": "Hedgebox", "provider": "ses", **config}}, + format="json", + ) + assert response.status_code == 201, response.json() + return response.json()["id"] + + def _verify(self, integration_id: int) -> str: + response = self.client.post(f"/api/environments/{self.team.pk}/integrations/{integration_id}/email/verify") + assert response.status_code == 200, response.json() + return response.json()["status"] + + def _context(self) -> CustomPromptSandboxContext: + return CustomPromptSandboxContext(team_id=self.team.pk, user_id=self.user.pk) + + def test_domain_connect_case_hands_back_an_apply_url(self) -> None: + self.organization_membership.level = OrganizationMembership.Level.ADMIN + self.organization_membership.save() + seed_domain_connect_support(self._context()) + with simulated_email_domains(), patch("posthoganalytics.feature_enabled", return_value=True): + integration_id = self._create_sender(DOMAIN_CONNECT_SENDER) + assert self._verify(integration_id) == "pending" + check = self.client.get( + f"/api/environments/{self.team.pk}/integrations/domain-connect/check", + {"domain": DOMAIN_CONNECT_SENDER.split("@")[1]}, + ) + apply_url = self.client.post( + f"/api/environments/{self.team.pk}/integrations/domain-connect/apply-url", + {"context": "email", "integration_id": integration_id}, + format="json", + ) + + assert check.json()["supported"] is True + assert apply_url.status_code == 200, apply_url.json() + assert apply_url.json()["url"].startswith(APPLY_URL_PATH) diff --git a/products/workflows/package.json b/products/workflows/package.json index e250226db894..71e637c286ad 100644 --- a/products/workflows/package.json +++ b/products/workflows/package.json @@ -1,7 +1,7 @@ { "name": "@posthog/products-workflows", "scripts": { - "backend:test": "pytest -c ../../pytest.ini --rootdir ../.. backend/ -v --tb=short" + "backend:test": "pytest -c ../../pytest.ini --rootdir ../.. backend/ evals/test/ -v --tb=short" }, "dependencies": { "@posthog/quill-charts": "workspace:*", diff --git a/products/workflows/skills/setting-up-an-email-domain/SKILL.md b/products/workflows/skills/setting-up-an-email-domain/SKILL.md new file mode 100644 index 000000000000..00c03e07847b --- /dev/null +++ b/products/workflows/skills/setting-up-an-email-domain/SKILL.md @@ -0,0 +1,111 @@ +--- +name: setting-up-an-email-domain +description: 'Set up a custom sending domain for PostHog Workflows email end to end: create the email sender, get its DNS records, publish them through Domain Connect or your own DNS tools, and verify until the domain can send. Use when asked to set up, connect, verify, or fix an email sender or sending domain for workflows, broadcasts, or campaigns, or when DNS records for SPF, DKIM, DMARC, or MAIL FROM are missing.' +--- + +# Setting up an email domain + +Workflows sends email from the customer's own domain. +A sender (`hello@mail.example.com`) cannot send until its domain is verified, and verification needs DNS records the customer publishes at their DNS host. +Your job is to get from "I want to send email" to a verified sender with as little work for the person as possible. + +## Tools + +These tools exist only when the project has the email domain agent setup feature enabled. +If `integrations-email-create` or `integrations-email-verify-create` is missing, stop and tell the person the tools are not available to them yet. +Do not fall back to `integrations-create` or other integration tools. + +| Step | Tool | +| ------------------------ | ---------------------------------------------- | +| Find existing senders | `integrations-list` with `kind=email` | +| Create the sender | `integrations-email-create` | +| Get records, check them | `integrations-email-verify-create` | +| One-click DNS support? | `integrations-domain-connect-check-retrieve` | +| One-click DNS URL | `integrations-domain-connect-apply-url-create` | +| Rename, change MAIL FROM | `integrations-email-partial-update` | + +Listing, checking and creating need project membership. +Verifying, updating and the Domain Connect URL need project admin access, so a member can create a sender but not verify it. +A 403 with "You don't have sufficient permissions in the project" means the person must ask a project admin; do not retry. + +## Workflow + +### 1. Pick the domain and the sender + +Ask which domain to send from if the person has not said. +Recommend the `mail.` subdomain, for example `mail.example.com`, or `send.example.com` if `mail.` is taken: it keeps marketing reputation and DMARC changes away from the root domain the company uses for everyday mail. +Accept the root domain if they prefer it. + +If they only give a domain, default the sender to `hello@` with the company name as the display name, and tell them both can change later. +Free mailbox domains such as gmail.com are rejected, and so is a domain another PostHog organization already uses; relay that error and suggest contacting support for the second case. + +Check `integrations-list` with `kind=email` first; each sender's `display_name` is its address. +If a sender with the exact requested address exists, skip to step 3 with its `id`. +Never create it again: re-creating an existing sender marks it unverified until the next successful verify. +If a sender with a different address exists on the same domain, call `integrations-email-verify-create` with its `id` before creating the new one. +Its MAIL FROM records sit at `