feat(signals): verify fixes after merged pull requests

posthog/egress/typesafe/README.md

@@ -0,0 +1,26 @@
+# TypeSafe egress
+
+## Identity
+
+TypeSafe meters the account behind an API key. The limiter and metrics use a short SHA-256 fingerprint of the key, never the credential itself.
+
+## Budget
+
+TypeSafe publishes 1,200 requests per minute for Jev. PostHog defaults to 1,000 requests per minute and 50,000 per hour so background verification cannot consume the provider limit or create unbounded spend.
+
+## Lanes and callers
+
+Signals verification uses the `BATCH` lane. A denied call remains an errored, retryable check and does not become a verdict.
+
+## Rate-limit headers
+
+The API documents `Retry-After` for HTTP 429 and 529 responses. The check runner does not sleep inside the coordinator tick; it records a retryable error and uses the check schedule for backoff.
+
+## Auth
+
+Callers pass `TYPESAFE_API_KEY` as a bearer token. The transport is credential-agnostic and never stores it.
+
+## Sources
+
+- [TypeSafe API](https://docs.typesafe.ai/api)
+- [Jev models and limits](https://docs.typesafe.ai/models)

posthog/egress/typesafe/__init__.py

@@ -0,0 +1,3 @@
+from posthog.egress.typesafe.transport import TypeSafeEgressBudgetExhausted, typesafe_request
+
+__all__ = ["TypeSafeEgressBudgetExhausted", "typesafe_request"]

posthog/egress/typesafe/limiter.py

@@ -0,0 +1,31 @@
+"""TypeSafe egress budget keyed by the API credential's non-reversible fingerprint."""
+
+import hashlib
+
+from posthog.egress.limiter.outbound import get_outbound_rate_limiter
+from posthog.egress.limiter.policies import Priority, per_minute_and_hourly_policy, register_policy
+
+TYPESAFE_DOMAIN = "typesafe"
+
+register_policy(
+    TYPESAFE_DOMAIN,
+    per_minute_and_hourly_policy(
+        per_minute_setting="TYPESAFE_EGRESS_PER_MINUTE_BUDGET",
+        per_minute_default=1_000,
+        hourly_setting="TYPESAFE_EGRESS_HOURLY_BUDGET",
+        hourly_default=50_000,
+    ),
+)
+
+
+def typesafe_account_id(api_key: str) -> str:
+    return hashlib.sha256(api_key.encode()).hexdigest()[:16]
+
+
+def consume_typesafe_sync(account_id: str, *, priority: Priority = Priority.BATCH, source: str = "unknown") -> bool:
+    return get_outbound_rate_limiter().consume_sync(
+        f"{TYPESAFE_DOMAIN}:account:{account_id}",
+        1,
+        priority=priority,
+        source=source,
+    )

posthog/egress/typesafe/observability.py

@@ -0,0 +1,13 @@
+from prometheus_client import Counter
+
+from posthog.egress.observability.observability import EgressMetrics, EgressObservability
+
+typesafe_egress = EgressObservability(
+    EgressMetrics(
+        request_counter=Counter(
+            "typesafe_api_requests",
+            "Number of TypeSafe API requests made through the TypeSafe egress client.",
+            labelnames=["account", "method", "endpoint", "status_code", "source"],
+        ),
+    )
+)

posthog/egress/typesafe/transport.py

@@ -0,0 +1,52 @@
+from typing import Any
+
+import requests
+
+from posthog.egress.limiter.policies import Priority
+from posthog.egress.transport.transport import EgressBudgetExhausted, EgressClient
+from posthog.egress.typesafe.limiter import consume_typesafe_sync, typesafe_account_id
+from posthog.egress.typesafe.observability import typesafe_egress
+
+
+class TypeSafeEgressBudgetExhausted(EgressBudgetExhausted):
+    pass
+
+
+class TypeSafeClient(EgressClient):
+    observability = typesafe_egress
+
+    def _standard_headers(self) -> dict[str, str]:
+        return {"Accept": "application/json", "Content-Type": "application/json"}
+
+    def _consume(self, scope: str, priority: Priority, source: str, url: str) -> bool:
+        return consume_typesafe_sync(scope, priority=priority, source=source)
+
+    def _budget_exhausted_error(self, scope: str) -> TypeSafeEgressBudgetExhausted:
+        return TypeSafeEgressBudgetExhausted("TypeSafe egress budget exhausted; retry later")
+
+
+_typesafe_client = TypeSafeClient()
+
+
+def typesafe_request(
+    method: str,
+    url: str,
+    *,
+    api_key: str,
+    source: str,
+    endpoint: str,
+    priority: Priority = Priority.BATCH,
+    timeout: float | tuple[float, float] | None = None,
+    **kwargs: Any,
+) -> requests.Response:
+    return _typesafe_client.request(
+        method,
+        url,
+        source=source,
+        headers={"Authorization": f"Bearer {api_key}"},
+        scope=typesafe_account_id(api_key),
+        priority=priority,
+        endpoint=endpoint,
+        timeout=timeout,
+        **kwargs,
+    )

posthog/settings/web.py

@@ -914,6 +914,15 @@ def static_varies_origin(headers, path, url):
 FIRECRAWL_EGRESS_PER_MINUTE_BUDGET = get_from_env("FIRECRAWL_EGRESS_PER_MINUTE_BUDGET", 60, type_cast=int)
 FIRECRAWL_EGRESS_HOURLY_BUDGET = get_from_env("FIRECRAWL_EGRESS_HOURLY_BUDGET", 1000, type_cast=int)
 
+####
+# TypeSafe Jev (Signals verification, see posthog/egress/typesafe/)
+TYPESAFE_API_KEY = get_from_env("TYPESAFE_API_KEY", "")
+TYPESAFE_BASE_URL = get_from_env("TYPESAFE_BASE_URL", "https://api.typesafe.ai")
+TYPESAFE_DEFAULT_MODEL = get_from_env("TYPESAFE_DEFAULT_MODEL", "jev-latest")
+TYPESAFE_VERIFICATION_MIN_CONFIDENCE = get_from_env("TYPESAFE_VERIFICATION_MIN_CONFIDENCE", 0.6, type_cast=float)
+TYPESAFE_EGRESS_PER_MINUTE_BUDGET = get_from_env("TYPESAFE_EGRESS_PER_MINUTE_BUDGET", 1000, type_cast=int)
+TYPESAFE_EGRESS_HOURLY_BUDGET = get_from_env("TYPESAFE_EGRESS_HOURLY_BUDGET", 50000, type_cast=int)
+
 ####
 # Feature flag billing analytics
 # Used to track feature flag requests for billing purposes.

products/signals/ARCHITECTURE.md

@@ -476,6 +476,8 @@ The activity log shows PR references as title links with repository, number, and
 | `task_run`               | `{"task_id": "...", "run_id"?: "...", "product": "...", "type": "..."}` — a task run associated with the report (see below)                                                                                                                                                                                    |
 | `note`                   | `{"note": "...", "author"?: "..."}` — free-form note (markdown allowed)                                                                                                                                                                                                                                        |
 | `check_result`           | `{"check_id": "...", "kind": "...", "title": "...", "outcome": "passed"\|"failed"\|"errored", "explanation": "...", "observed_value"?, "baseline_value"?, "threshold"?, "run_id"?}` — one run of a report check                                                                                                |
+| `verification_query`     | A bounded HogQL query, exact pre-fix snapshot, success criteria, inconclusive conditions, and MCP provenance. The latest row is the report's current verification definition.                                                                                                                                  |
+| `verification_result`    | The latest attempt in one uninterrupted verification period: the query artefact and PR ids, current result, outcome (`solved`, `not_solved`, `inconclusive`, or `errored`), Jev confidence, exact model version, and explanation.                                                                              |
 
 **Content schemas.** `artefact_schemas.py` is the canonical, pydantic-only home of every content shape, collected in `ARTEFACT_CONTENT_SCHEMAS` (one model per type; a test asserts exact coverage). Raw payloads become typed models once, at the boundaries (`parse_artefact_content`); the model helpers derive a row's type from the content model's class (`artefact_type_for`), so a type can never mismatch its content. `repo_selection` reuses the tasks product's `RepoSelectionResult` DTO directly (kept in the dependency-light leaf module `repo_selection/types.py` so importing the schema registry doesn't pull in the sandbox runtime). Reads of legacy rows stay tolerant — parse failures are skipped or degraded, never raised.
 
@@ -497,6 +499,8 @@ The schedule is the check's own clock (`next_run_at`, optional `run_interval_min
 
 `agent` is the kind for a claim no single number settles, which is most of the inbox: "did the exception stop?" needs the issue looked up and its recent events read rather than a threshold compared. Its config is `{instructions, skill_name?, probe_hints}`, all of it untrusted prompt material. `report_check_agent.py` owns the lane. A due check is dispatched as a scout run with `triggered_by="check"` — a fourth dispatch source next to `schedule` / `manual` / `workflow`, so it shares the fleet, spend, and single-flight gates in `run_gates.py` but stamps no `last_run_at` and never feeds the failure-streak breaker, whose threshold is sized on the scout's own cadence. The run note carries the check id, title, rationale, instructions, probe hints, the report, and the note whoever resolved or dismissed it typed; the prompt renders it as the run's assignment (`prompt._CHECK_NOTE_TEMPLATE`) rather than as the nudge a manual run's note is.
 
+A `verification_query` check is created automatically when a linked pull request first reaches `merged` and the report has a verification artefact. It starts one hour after merge, runs once an hour for up to 14 days, and stops early only on a sufficiently confident `solved` verdict. The current UTC window begins at the later of the merge time or one snapshot duration ago, so early attempts contain no pre-merge events and can remain inconclusive until a full comparison window exists. `not_solved`, `insufficient_evidence`, low-confidence, and provider-error outcomes keep the check active until its horizon. The latest `verification_result` is replaced when it is still the newest report artefact for the same query and pull request; intervening report activity causes the next result to append instead. Raw HogQL and both result sets use the same fail-closed query/property access gate as report metric snapshots. The result schema accepts only aggregate numeric, boolean, or null cells, so row-level identifiers, property values, URLs, messages, and free text cannot be sent to Jev. The Jev call also requires organization-level AI data processing approval at execution time.
+
 Most reports are pipeline-authored and have no scout behind them, so `skill_name` is optional: a check that names none runs on `signals-scout-inbox-validation`, the fleet's follow-up scout, which is an operational scout and so is seeded enabled on every enrolled team. When the lane genuinely cannot run — the project is not enrolled, a person paused the scout, the skill is withheld or gone — the check records an `errored` result naming the reason, so a reader finds out on the report rather than waiting out a silent horizon. A refusal that clears on its own (a run of the same scout in flight, a daily run budget, a spend gate) defers the check by an hour instead, spending no error.
 
 A dispatch is not a verdict. The row stays `active` with `dispatched_at` stamped, and only `scout-check-record-result` (`scout_harness/tools/checks.py`) closes it. A run may close only a check its own lane was dispatched for and that is still waiting on a dispatch, so the tool can answer the question the run was asked and nothing else. A run that ends without calling it leaves `dispatched_at` set, and the first tick past `AGENT_CHECK_RESULT_WINDOW` records an errored result saying so.

products/signals/VERIFICATION_QUERY_PLAN.md

@@ -0,0 +1,111 @@
+# Verification query artefacts
+
+## Goal
+
+Teach Signals report research runs to leave behind a reproducible measurement of the problem they found. After an implementation pull request merges, the report check coordinator can run the same measurement over a comparable window and ask Jev whether the result supports that the fix worked.
+
+The full evidence chain must stay inspectable:
+
+1. Research writes a `verification_query` artefact with the query, baseline result, decision criteria, and MCP provenance.
+2. A check binds that artefact to a specific pull request and soak window.
+3. The check runs the query over the current window and sends the baseline and current result to Jev.
+4. Evaluation writes a `verification_result` artefact that points to both the verification query and the pull request.
+
+## Artefact contract
+
+`verification_query` is an append-only, latest-wins status artefact. The newest valid definition is the report's canonical verification recipe, while older definitions remain in the activity history.
+
+The payload contains:
+
+- A short description of what the query measures.
+- An executable HogQL template with allowlisted UTC window placeholders.
+- A normalized baseline result with its exact window.
+- Success and inconclusive criteria.
+- Opportunity evidence, normally returned as a denominator in the same query.
+- MCP tool names and documentation references used to construct the query.
+
+Results are bounded and JSON-only. Verification queries should return aggregate evidence rather than raw event or person rows.
+
+## Research behavior
+
+The research protocol must tell the agent to prepare verification evidence while it investigates the report:
+
+- Discover event and property shapes before querying them.
+- Use the documented MCP tool schema rather than guessing arguments.
+- Prefer a single aggregate query that returns both the symptom and its opportunity denominator.
+- Use equal-length windows and explicit UTC bounds.
+- Run the exact query before proposing it.
+- Never emit a verification query from a failed, truncated, or hypothetical call.
+- Fall back to the existing manual verification note for reports that are not meaningfully queryable.
+
+The final structured turn serializes evidence already collected during research. It does not perform new research.
+
+## Trusted snapshots
+
+The model nominates the successful MCP call that produced the query and result. Before persistence, the server re-runs the read-only query with the claimed snapshot bounds and requires the columns and rows to match. An invented, stale, or failed snapshot is dropped.
+
+The pipeline rejects:
+
+- Queries without both window placeholders.
+- Queries that cannot be executed as HogQL by the report team's query engine.
+- Results over the row or column limits.
+- Results whose shape does not match the declared evidence columns.
+- Results containing anything other than aggregate numbers, booleans, or nulls.
+
+## Access control
+
+The artefact is generated by the research pipeline and is not writable through the generic artefact API.
+
+The query and result follow the same access policy as report metric queries and snapshots. A viewer without access receives the artefact metadata and decision description, with the query and snapshot values redacted.
+
+Only aggregate numeric, boolean, or null result cells may leave PostHog for Jev. Verification queries must not return identifiers, property values, URLs, names, free text, exception messages, or other row-level data.
+
+## Pull request binding and execution
+
+The research artefact is report-level because it normally predates implementation. When a linked implementation pull request merges, the system creates a `verification_query` report check that references both the source artefact and the pull request.
+
+The check:
+
+1. Starts one hour after merge and runs once an hour for up to 14 days.
+2. Runs the stored query over the available post-merge window, capped at the snapshot duration. Jev treats the shorter early windows as inconclusive when they are not yet comparable.
+3. Refuses to conclude success when the opportunity evidence is missing or too small.
+4. Sends the description, query, baseline result, current result, and decision criteria to Jev.
+5. Maps `solved` to `passed`, `not_solved` to `failed`, and `insufficient_evidence` to the retryable inconclusive path.
+6. Writes a `verification_result` associated with the exact pull request and source verification artefact.
+7. Stops early only when Jev returns a sufficiently confident `solved`; `not_solved`, inconclusive, and provider-error results continue hourly until the 14-day horizon.
+
+When the previous verification result is still the report's newest artefact and belongs to the same query and pull request, the next attempt replaces that row. If another pull request or any other report activity arrived in between, the next attempt appends a new result instead. This keeps a quiet report's work log compact without erasing the boundary created by meaningful intervening work.
+
+Low-confidence model answers are inconclusive. A deterministic metric comparison should continue to use the existing `metric_threshold` check kind instead of Jev.
+
+## Inbox presentation
+
+The report activity log shows:
+
+- `Verification query prepared` with the description and baseline window.
+- The HogQL and captured row count.
+- The later verification outcome, explanation, confidence, and model.
+
+Restricted query data remains visibly redacted rather than disappearing.
+
+## Test coverage
+
+Extend the nearest existing tests to cover:
+
+- Artefact enum and schema parity.
+- Query/result bounds and placeholder validation.
+- Research output with a verified query and with the manual fallback.
+- Latest-wins re-research behavior and task attribution.
+- API write protection and restricted-data redaction.
+- Check creation from a merged pull request.
+- Jev response parsing, confidence gating, and retryable failures.
+- Pull request and verification-query provenance on `verification_result` rows.
+- Frontend rendering for complete and redacted artefacts.
+
+Use invented fixtures that cover clear recovery, continued failure, missing opportunity, tiny baselines, schema drift, and fixes whose intended effect is a behavior change rather than complete disappearance. Do not commit production query results from the exploratory dataset.
+
+## Operational controls
+
+The TypeSafe API key, base URL, model, minimum confidence, and egress budgets are environment settings. Requests use the shared egress limiter on the batch lane. Rate-limit and provider failures become retryable errored attempts rather than verdicts.
+
+Research generation and snapshot-validation failures are logged. The latest `verification_result` for each uninterrupted verification period provides the durable Jev outcome, confidence, model, and explanation for later analysis.