From 1f3d9b2fb39738e2b33e0b30edd64a7094c6994f Mon Sep 17 00:00:00 2001 From: David Viejo Date: Fri, 1 May 2026 23:27:10 +0200 Subject: [PATCH 01/12] =?UTF-8?q?feat(observability):=20unified=20Observe?= =?UTF-8?q?=20surface=20=E2=80=94=20merged=20events,=20correlation,=20mess?= =?UTF-8?q?age=20extraction?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a project-level "Observe" surface that merges request logs, OTel spans, error events, and revenue events into a single time-ordered stream. Introduces a new `temps-observability` crate that exposes a discriminated `ObservabilityEvent` wire shape (one row carries everything the panel needs) and a per-kind parallel-fetch + k-way merge service. Wire correlation: new migration `m20260502_000001_add_observe_correlation` adds nullable `trace_id`/`error_group_id`/`deployment_id`/ `environment_id`/`trace_id_indexed` columns across `proxy_logs`, `revenue_events`, and `error_events` so the unified view can join an event to its peers without follow-up queries. `temps-proxy` parses W3C `traceparent` headers at ingest; `temps-error-tracking` promotes `data.{sentry,}.contexts.trace.trace_id` to a top-level indexed column. Fix "Error: Unknown error" titles: error groups created from Sentry SDK `captureMessage()` calls were stored with that literal title because `exception.values[].value` is empty for message events. The ingestion pipeline now probes `logentry.formatted` / `logentry.message` / `message` / `exception.values[0].value` / `breadcrumbs.values[0].message` / `extra.message`, and the same fallback runs at read time in the Observe row mapper so legacy rows also render their real text. - `temps-observability`: new crate (service, handlers, filters, cursor, plugin, OpenAPI types). 38 lib tests + integration tests cover merge ordering, kind filtering, cursor stability, truncation helpers, and per-kind fetcher correctness against TestDatabase - W3C traceparent extractor in `temps-proxy::proxy` with 6 unit tests (valid / missing / all-zero invalid / wrong length / non-hex / case-folding) - `extract_trace_id_from_data` helper in `temps-error-tracking` with 7 unit tests covering all three known JSONB layouts plus invalid variants - Heavy fields truncated server-side with `*_truncated` flags (5 stack frames, 20 span attrs, header whitelist of ~10 keys) --- CHANGELOG.md | 11 +- Cargo.lock | 28 + Cargo.toml | 1 + crates/temps-cli/Cargo.toml | 1 + .../temps-cli/src/commands/serve/console.rs | 8 + crates/temps-entities/src/error_events.rs | 6 + crates/temps-entities/src/proxy_logs.rs | 10 + crates/temps-entities/src/revenue_events.rs | 14 + .../src/services/error_ingestion_service.rs | 202 ++++- ...20260502_000001_add_observe_correlation.rs | 211 +++++ crates/temps-migrations/src/migration/mod.rs | 2 + crates/temps-observability/Cargo.toml | 34 + crates/temps-observability/src/cursor.rs | 80 ++ crates/temps-observability/src/error.rs | 70 ++ crates/temps-observability/src/filters.rs | 319 ++++++++ .../src/handlers/events.rs | 250 ++++++ .../temps-observability/src/handlers/mod.rs | 3 + crates/temps-observability/src/lib.rs | 34 + crates/temps-observability/src/plugin.rs | 70 ++ crates/temps-observability/src/service.rs | 720 ++++++++++++++++++ crates/temps-observability/src/types.rs | 424 +++++++++++ .../tests/merge_integration.rs | 548 +++++++++++++ crates/temps-proxy/src/proxy.rs | 103 +++ .../src/service/proxy_log_batch_writer.rs | 10 + .../src/service/proxy_log_service.rs | 11 + crates/temps-proxy/src/services.rs | 7 + crates/temps-proxy/src/traits.rs | 5 + 27 files changed, 3164 insertions(+), 18 deletions(-) create mode 100644 crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs create mode 100644 crates/temps-observability/Cargo.toml create mode 100644 crates/temps-observability/src/cursor.rs create mode 100644 crates/temps-observability/src/error.rs create mode 100644 crates/temps-observability/src/filters.rs create mode 100644 crates/temps-observability/src/handlers/events.rs create mode 100644 crates/temps-observability/src/handlers/mod.rs create mode 100644 crates/temps-observability/src/lib.rs create mode 100644 crates/temps-observability/src/plugin.rs create mode 100644 crates/temps-observability/src/service.rs create mode 100644 crates/temps-observability/src/types.rs create mode 100644 crates/temps-observability/tests/merge_integration.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ffc944ae..0a131f64e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- **CLI credential-based login (`@temps-sdk/cli`)**: `bunx @temps-sdk/cli login ` now authenticates with email + password (and TOTP MFA when enabled) instead of requiring users to mint and paste an API key. Internally hits a new `POST /auth/cli/login` endpoint that mints a 90-day API key scoped to the user's role. The previous fragile `/auth/login` + cookie-scrape + `/api/tokens` flow is replaced. `--api-key` and `--magic` login modes still work. +- **Unified Observe page (`/projects/:slug/observe`)**: new project-level page that merges Requests, Traces, Errors, and Revenue into a single time-ordered timeline. Cockpit header shows one sparkline per kind over the selected time range — clickable to toggle that kind on/off. Below it, a console-style monospace stream with color-coded gutters per kind (sky=requests, violet=traces, rose=errors, emerald=revenue), `HH:mm:ss` timestamps, and rich one-line summaries. Side panel renders entirely from the row payload — no follow-up fetch in the common case. Filter state lives in URL search params so the page is shareable. Traces default OFF (high-volume hypertable, opt-in via cockpit card). Runtime logs are intentionally NOT included — they live on the dedicated Logs page because their volume would dominate the merged business-signal timeline +- **`temps-observability` crate**: new merge service backing the Observe page. Per-kind parallel queries against `proxy_logs`, `error_events`, `revenue_events`, and the `otel_spans` TimescaleDB hypertable (raw SQL via `FromQueryResult`); k-way merged by `ts DESC` and trimmed to a single page. Heavy fields are truncated server-side (stack frames → first 5, span attributes → first 20 keys alphabetized, headers → whitelist of ~10 keys) with `*_truncated` flags so the panel can offer a "Show full" un-truncate fetch when needed. Wire shape is a discriminated `ObservabilityEvent` union — every row carries its own `type` discriminator and all data the panel needs +- **Cross-source correlation columns**: new migration `m20260502_000001_add_observe_correlation` adds `proxy_logs.trace_id` + `proxy_logs.error_group_id` + indexes, `revenue_events.deployment_id` + `revenue_events.environment_id` + `revenue_events.trace_id`, and `error_events.trace_id_indexed` (denormalized from `data.trace.trace_id` JSONB for index speed). All nullable — old rows just render without correlation links. Lets the Observe view jump from any event to its peers in the same trace +- **W3C `traceparent` extraction in proxy**: `temps-proxy` now parses inbound `traceparent` headers and stamps the 32-hex trace_id onto `proxy_logs.trace_id` at write time. Validates length, hex chars, and rejects the all-zero invalid trace_id reserved by the W3C spec. Six unit tests cover the happy path, missing/malformed/invalid inputs, and case-folding +- **OTel trace_id promotion in error tracking**: `temps-error-tracking` ingestion now probes `data.sentry.contexts.trace.trace_id`, top-level `data.contexts.trace.trace_id`, and `data.trace.trace_id` and promotes the first valid 32-hex value to the indexed `error_events.trace_id_indexed` column. Seven unit tests cover all three layouts plus invalid lengths/chars and the all-zero case + +### Fixed +- **Error group titles "Error: Unknown error"**: error groups created from Sentry SDK `captureMessage()` calls (which omit `exception.values[].value`) were stored with the literal title `"Error: Unknown error"`, masking the real message. The ingestion pipeline now probes the raw Sentry payload at `logentry.formatted`, `logentry.message`, top-level `message`, `exception.values[0].value`, the first breadcrumb message, and `extra.message` for a usable string. Same fallback runs at read time in the Observe row mapper, so existing groups also render their real text — group titles in the database remain as-is, but the Observe row and the error detail page now show the real message regardless. Future events will get correctly-titled groups at create time + + `bunx @temps-sdk/cli login ` now authenticates with email + password (and TOTP MFA when enabled) instead of requiring users to mint and paste an API key. Internally hits a new `POST /auth/cli/login` endpoint that mints a 90-day API key scoped to the user's role. The previous fragile `/auth/login` + cookie-scrape + `/api/tokens` flow is replaced. `--api-key` and `--magic` login modes still work. - **CLI multi-context support**: `bunx @temps-sdk/cli context list | use | remove | current` lets one workstation stay logged into multiple self-hosted Temps servers simultaneously. Each context stores `{name, url, apiKey, email, keyPrefix, expiresAt}` in `~/.temps/.contexts.json` (mode 0600). The active context drives `apiUrl` and `apiKey` resolution for every other CLI command, with environment variables (`TEMPS_API_URL`, `TEMPS_API_TOKEN`, `TEMPS_API_KEY`) still taking precedence for CI / one-off overrides. Distinct from `temps instances`, which manages Temps Cloud VPS instances. - **CLI `temps logout` revokes server-side**: now calls `POST /auth/cli/logout` (best effort) to revoke the API key on the server before clearing local credentials. Pass `--local-only` to skip the network call when the server is unreachable, or `--context ` to log out of a specific context. - **CLI `temps whoami` surfaces context**: shows the active context name, key prefix, and expiry alongside the user info; `--json` includes the full context object. diff --git a/Cargo.lock b/Cargo.lock index 7c6c47a1b..1a6fcaa9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11192,6 +11192,7 @@ dependencies = [ "temps-monitoring", "temps-network", "temps-notifications", + "temps-observability", "temps-otel", "temps-projects", "temps-providers", @@ -12217,6 +12218,33 @@ dependencies = [ "uuid", ] +[[package]] +name = "temps-observability" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.8.6", + "base64 0.22.1", + "chrono", + "http 1.3.1", + "sea-orm", + "sea-orm-migration", + "serde", + "serde_json", + "temps-auth", + "temps-core", + "temps-database", + "temps-entities", + "temps-migrations", + "thiserror 2.0.17", + "tokio", + "tokio-test", + "tracing", + "utoipa", + "uuid", +] + [[package]] name = "temps-otel" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 19cd4facf..91e9b9515 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -74,6 +74,7 @@ members = [ "crates/temps-preview-gateway", "crates/temps-external-plugins", "crates/temps-revenue", + "crates/temps-observability", "examples/example-plugin", "examples/lighthouse-plugin", "examples/indexnow-plugin", diff --git a/crates/temps-cli/Cargo.toml b/crates/temps-cli/Cargo.toml index 94090b70f..e05f02518 100644 --- a/crates/temps-cli/Cargo.toml +++ b/crates/temps-cli/Cargo.toml @@ -30,6 +30,7 @@ temps-workspace = { path = "../temps-workspace" } temps-sandbox = { path = "../temps-sandbox" } temps-backup = { path = "../temps-backup" } temps-revenue = { path = "../temps-revenue" } +temps-observability = { path = "../temps-observability" } temps-config = { path = "../temps-config" } temps-core = { path = "../temps-core" } temps-database = { path = "../temps-database" } diff --git a/crates/temps-cli/src/commands/serve/console.rs b/crates/temps-cli/src/commands/serve/console.rs index 47680ab7b..ecaa69eed 100644 --- a/crates/temps-cli/src/commands/serve/console.rs +++ b/crates/temps-cli/src/commands/serve/console.rs @@ -55,6 +55,7 @@ use temps_projects::ProjectsPlugin; use temps_providers::ProvidersPlugin; use temps_proxy::ProxyPlugin; use temps_queue::QueuePlugin; +use temps_observability::ObservabilityPlugin; use temps_revenue::RevenuePlugin; use temps_sandbox::plugin::SandboxPlugin; use temps_screenshots::ScreenshotsPlugin; @@ -878,6 +879,13 @@ pub async fn start_console_api(params: ConsoleApiParams) -> anyhow::Result<()> { let revenue_plugin = Box::new(RevenuePlugin::new()); plugin_manager.register_plugin(revenue_plugin); + // 11b. ObservabilityPlugin - unified Observe page (merges runtime logs, + // requests, spans, errors, revenue into one event stream). Read-only, + // no outbound calls; depends on the database only. + debug!("Registering ObservabilityPlugin"); + let observability_plugin = Box::new(ObservabilityPlugin::new()); + plugin_manager.register_plugin(observability_plugin); + // AI Gateway Plugin - provides AI provider key management and OpenAI-compatible API debug!("Registering AiGatewayPlugin"); let ai_gateway_plugin = Box::new(temps_ai_gateway::AiGatewayPlugin::new()); diff --git a/crates/temps-entities/src/error_events.rs b/crates/temps-entities/src/error_events.rs index 27906f443..8a2f868d0 100644 --- a/crates/temps-entities/src/error_events.rs +++ b/crates/temps-entities/src/error_events.rs @@ -215,6 +215,12 @@ pub struct Model { #[sea_orm(column_type = "JsonBinary")] pub data: Option, + // ===== Indexed correlation ===== + /// Denormalized copy of `data.trace.trace_id` promoted to a top-level + /// indexed column so the Observe merge query can join error rows with + /// the originating request/span without a JSON probe. + pub trace_id_indexed: Option, + // ===== Metadata ===== pub created_at: DBDateTime, } diff --git a/crates/temps-entities/src/proxy_logs.rs b/crates/temps-entities/src/proxy_logs.rs index 26d7b8f60..c42f25e2b 100644 --- a/crates/temps-entities/src/proxy_logs.rs +++ b/crates/temps-entities/src/proxy_logs.rs @@ -113,6 +113,16 @@ pub struct Model { /// Visitor ID for tracking unique visitors pub visitor_id: Option, + + /// OTel trace_id stamped from the request span when available — lets the + /// unified Observe view join request rows with their child spans, runtime + /// log lines, and any captured exceptions. + pub trace_id: Option, + + /// Set when the request produced a captured exception in + /// `temps-error-tracking`; lets the Observe row deep-link to the group + /// without an extra query. + pub error_group_id: Option, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/crates/temps-entities/src/revenue_events.rs b/crates/temps-entities/src/revenue_events.rs index 2f28cb8b2..417ff24ba 100644 --- a/crates/temps-entities/src/revenue_events.rs +++ b/crates/temps-entities/src/revenue_events.rs @@ -30,6 +30,20 @@ pub struct Model { pub price_id: Option, /// Opaque product reference (Stripe product_id, …). NULL mirrors `price_id`. pub product_id: Option, + + /// Deployment that produced this revenue event, when the SDK supplied + /// the `X-Temps-Deployment-Id` header. NULL for webhooks delivered + /// outside a request context. + pub deployment_id: Option, + + /// Environment that produced this revenue event, when known via the + /// SDK header. + pub environment_id: Option, + + /// OTel trace_id from the originating request, when supplied by the + /// SDK header. Lets the Observe view link a payment to the request that + /// triggered it. + pub trace_id: Option, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/crates/temps-error-tracking/src/services/error_ingestion_service.rs b/crates/temps-error-tracking/src/services/error_ingestion_service.rs index 71f8d9f32..0841554fa 100644 --- a/crates/temps-error-tracking/src/services/error_ingestion_service.rs +++ b/crates/temps-error-tracking/src/services/error_ingestion_service.rs @@ -10,6 +10,73 @@ use temps_entities::{error_events, error_groups}; use super::types::{CreateErrorEventData, ErrorTrackingError}; +/// Pull a 32-hex-char OTel trace_id out of an error event's JSONB `data` +/// blob. Probes both the Sentry layout (`contexts.trace.trace_id`) and our +/// internal layout (`trace.trace_id`). Returns `None` when no valid +/// trace_id is present so the indexed column stays NULL for legacy events. +fn extract_trace_id_from_data(data: &serde_json::Value) -> Option { + fn validate(v: &serde_json::Value) -> Option { + let s = v.as_str()?.trim().to_ascii_lowercase(); + if s.len() == 32 + && s.chars().all(|c| c.is_ascii_hexdigit()) + && s.chars().any(|c| c != '0') + { + Some(s) + } else { + None + } + } + + // Sentry / our internal wrapper: data.sentry.contexts.trace.trace_id + if let Some(v) = data.pointer("/sentry/contexts/trace/trace_id") { + if let Some(s) = validate(v) { + return Some(s); + } + } + // Top-level Sentry payload: data.contexts.trace.trace_id + if let Some(v) = data.pointer("/contexts/trace/trace_id") { + if let Some(s) = validate(v) { + return Some(s); + } + } + // Custom layout via TraceContext: data.trace.trace_id + if let Some(v) = data.pointer("/trace/trace_id") { + if let Some(s) = validate(v) { + return Some(s); + } + } + None +} + +/// Probe a raw Sentry payload for a usable human-readable message. +/// +/// SDKs that send `captureMessage()` (no exception) put the text in +/// `logentry.formatted` (the rendered string after parameter +/// substitution) or `logentry.message` (the template). SDKs sending +/// `captureException()` put it in `exception.values[0].value`. Some +/// older SDKs use the deprecated top-level `message`. We probe in +/// priority order and return the first non-empty hit. +pub fn extract_message_from_raw(raw: Option<&serde_json::Value>) -> Option { + let raw = raw?; + let candidates = [ + "/logentry/formatted", + "/logentry/message", + "/message", + "/exception/values/0/value", + "/breadcrumbs/values/0/message", + "/extra/message", + ]; + for path in candidates { + if let Some(s) = raw.pointer(path).and_then(|v| v.as_str()) { + let trimmed = s.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + } + None +} + /// Service for ingesting and processing error events pub struct ErrorIngestionService { db: Arc, @@ -313,34 +380,45 @@ impl ErrorIngestionService { error_data: &CreateErrorEventData, _fingerprint: &str, ) -> Result { - // Use first exception for title, or fall back to legacy fields + // Use first exception for title, or fall back to legacy fields. + // When `exception_value` is missing (most `captureMessage` payloads + // and Sentry SDKs that send only an event-level message), probe + // the raw event for a usable message so the title doesn't render + // as the useless literal "Error: Unknown error". let (exception_type, exception_value) = if let Some(first_exception) = error_data.exceptions.first() { - ( - first_exception.exception_type.clone(), - first_exception - .exception_value - .clone() - .unwrap_or_else(|| "Unknown error".to_string()), - ) + let value = first_exception + .exception_value + .clone() + .filter(|s| !s.trim().is_empty()) + .or_else(|| extract_message_from_raw(error_data.raw_sentry_event.as_ref())); + (first_exception.exception_type.clone(), value) } else { + let value = error_data + .exception_value + .clone() + .filter(|s| !s.trim().is_empty()) + .or_else(|| extract_message_from_raw(error_data.raw_sentry_event.as_ref())); ( error_data .exception_type .clone() .unwrap_or_else(|| "Error".to_string()), - error_data - .exception_value - .clone() - .unwrap_or_else(|| "Unknown error".to_string()), + value, ) }; - let title = format!( - "{}: {}", - exception_type, - exception_value.chars().take(100).collect::() - ); + let title = match exception_value.as_deref() { + Some(v) if !v.trim().is_empty() => format!( + "{}: {}", + exception_type, + v.chars().take(100).collect::() + ), + _ => exception_type.clone(), + }; + // `exception_value` flows on into the embedding; fall back to the + // type so we still get a meaningful similarity vector. + let exception_value = exception_value.unwrap_or_else(|| exception_type.clone()); // Create embedding from error message for similarity search (reuse exception_value from above) let embedding = self.create_embedding(&exception_value); @@ -495,6 +573,14 @@ impl ErrorIngestionService { ) }; + // Promote the OTel trace_id (if any) from the JSONB data blob to the + // top-level `trace_id_indexed` column so the unified Observe view can + // join error rows to their originating request/span without a JSON + // probe. Probes both the Sentry layout (`contexts.trace.trace_id`) + // and our internal layout (`trace.trace_id`); validates 32-hex + // before persisting. + let trace_id_indexed = extract_trace_id_from_data(&data_json); + let new_event = error_events::ActiveModel { error_group_id: Set(group_id), fingerprint_hash: Set(fingerprint.to_string()), @@ -508,6 +594,7 @@ impl ErrorIngestionService { deployment_id: Set(error_data.deployment_id), visitor_id: Set(error_data.visitor_id), ip_geolocation_id: Set(error_data.ip_geolocation_id), + trace_id_indexed: Set(trace_id_indexed), created_at: Set(Utc::now()), ..Default::default() }; @@ -537,3 +624,84 @@ impl ErrorIngestionService { mod tests { include!("error_ingestion_tests.rs"); } + +#[cfg(test)] +mod trace_id_extraction_tests { + use super::extract_trace_id_from_data; + use serde_json::json; + + #[test] + fn extracts_from_sentry_wrapped_layout() { + let data = json!({ + "source": "sentry", + "sentry": { + "contexts": { + "trace": { "trace_id": "4bf92f3577b34da6a3ce929d0e0e4736" } + } + } + }); + assert_eq!( + extract_trace_id_from_data(&data), + Some("4bf92f3577b34da6a3ce929d0e0e4736".to_string()) + ); + } + + #[test] + fn extracts_from_top_level_contexts_layout() { + let data = json!({ + "contexts": { + "trace": { "trace_id": "4bf92f3577b34da6a3ce929d0e0e4736" } + } + }); + assert_eq!( + extract_trace_id_from_data(&data), + Some("4bf92f3577b34da6a3ce929d0e0e4736".to_string()) + ); + } + + #[test] + fn extracts_from_internal_trace_layout() { + let data = json!({ + "trace": { "trace_id": "4bf92f3577b34da6a3ce929d0e0e4736" } + }); + assert_eq!( + extract_trace_id_from_data(&data), + Some("4bf92f3577b34da6a3ce929d0e0e4736".to_string()) + ); + } + + #[test] + fn returns_none_when_missing() { + let data = json!({ "exception": "boom" }); + assert_eq!(extract_trace_id_from_data(&data), None); + } + + #[test] + fn rejects_invalid_lengths_and_chars() { + let bad = json!({ "trace": { "trace_id": "deadbeef" } }); + assert_eq!(extract_trace_id_from_data(&bad), None); + + let nonhex = + json!({ "trace": { "trace_id": "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" } }); + assert_eq!(extract_trace_id_from_data(&nonhex), None); + } + + #[test] + fn rejects_all_zero_invalid_trace_id() { + let data = json!({ + "trace": { "trace_id": "00000000000000000000000000000000" } + }); + assert_eq!(extract_trace_id_from_data(&data), None); + } + + #[test] + fn lowercases_input() { + let data = json!({ + "trace": { "trace_id": "4BF92F3577B34DA6A3CE929D0E0E4736" } + }); + assert_eq!( + extract_trace_id_from_data(&data), + Some("4bf92f3577b34da6a3ce929d0e0e4736".to_string()) + ); + } +} diff --git a/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs b/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs new file mode 100644 index 000000000..c7ccc08e9 --- /dev/null +++ b/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs @@ -0,0 +1,211 @@ +use sea_orm_migration::prelude::*; + +/// Add cross-source correlation columns so the unified Observe page can join +/// runtime logs / requests / spans / errors / revenue without follow-up +/// queries. All columns are nullable; old rows simply render without +/// correlation links. +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // proxy_logs: trace_id (from OTel context) + error_group_id (when the + // request produced a captured exception) + manager + .alter_table( + Table::alter() + .table(ProxyLogs::Table) + .add_column(ColumnDef::new(ProxyLogs::TraceId).text().null()) + .to_owned(), + ) + .await?; + manager + .alter_table( + Table::alter() + .table(ProxyLogs::Table) + .add_column(ColumnDef::new(ProxyLogs::ErrorGroupId).integer().null()) + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .if_not_exists() + .name("idx_proxy_logs_project_trace") + .table(ProxyLogs::Table) + .col(ProxyLogs::ProjectId) + .col(ProxyLogs::TraceId) + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .if_not_exists() + .name("idx_proxy_logs_error_group") + .table(ProxyLogs::Table) + .col(ProxyLogs::ErrorGroupId) + .to_owned(), + ) + .await?; + + // revenue_events: link the business event back to the deployment / + // environment / trace it belongs to (when the SDK provides headers). + manager + .alter_table( + Table::alter() + .table(RevenueEvents::Table) + .add_column(ColumnDef::new(RevenueEvents::DeploymentId).integer().null()) + .to_owned(), + ) + .await?; + manager + .alter_table( + Table::alter() + .table(RevenueEvents::Table) + .add_column(ColumnDef::new(RevenueEvents::EnvironmentId).integer().null()) + .to_owned(), + ) + .await?; + manager + .alter_table( + Table::alter() + .table(RevenueEvents::Table) + .add_column(ColumnDef::new(RevenueEvents::TraceId).text().null()) + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .if_not_exists() + .name("idx_revenue_events_project_occurred") + .table(RevenueEvents::Table) + .col(RevenueEvents::ProjectId) + .col(RevenueEvents::OccurredAt) + .to_owned(), + ) + .await?; + + // error_events: promote trace_id from the JSON `data.trace.trace_id` + // blob to a top-level indexed column. Cheap to maintain at write + // time; lets the merge query join by trace_id without a JSON probe. + manager + .alter_table( + Table::alter() + .table(ErrorEvents::Table) + .add_column(ColumnDef::new(ErrorEvents::TraceIdIndexed).text().null()) + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .if_not_exists() + .name("idx_error_events_project_trace") + .table(ErrorEvents::Table) + .col(ErrorEvents::ProjectId) + .col(ErrorEvents::TraceIdIndexed) + .to_owned(), + ) + .await?; + + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_index( + Index::drop() + .if_exists() + .name("idx_error_events_project_trace") + .table(ErrorEvents::Table) + .to_owned(), + ) + .await?; + manager + .alter_table( + Table::alter() + .table(ErrorEvents::Table) + .drop_column(ErrorEvents::TraceIdIndexed) + .to_owned(), + ) + .await?; + + manager + .drop_index( + Index::drop() + .if_exists() + .name("idx_revenue_events_project_occurred") + .table(RevenueEvents::Table) + .to_owned(), + ) + .await?; + manager + .alter_table( + Table::alter() + .table(RevenueEvents::Table) + .drop_column(RevenueEvents::TraceId) + .drop_column(RevenueEvents::EnvironmentId) + .drop_column(RevenueEvents::DeploymentId) + .to_owned(), + ) + .await?; + + manager + .drop_index( + Index::drop() + .if_exists() + .name("idx_proxy_logs_error_group") + .table(ProxyLogs::Table) + .to_owned(), + ) + .await?; + manager + .drop_index( + Index::drop() + .if_exists() + .name("idx_proxy_logs_project_trace") + .table(ProxyLogs::Table) + .to_owned(), + ) + .await?; + manager + .alter_table( + Table::alter() + .table(ProxyLogs::Table) + .drop_column(ProxyLogs::ErrorGroupId) + .drop_column(ProxyLogs::TraceId) + .to_owned(), + ) + .await?; + + Ok(()) + } +} + +#[derive(DeriveIden)] +enum ProxyLogs { + Table, + ProjectId, + TraceId, + ErrorGroupId, +} + +#[derive(DeriveIden)] +enum RevenueEvents { + Table, + ProjectId, + OccurredAt, + DeploymentId, + EnvironmentId, + TraceId, +} + +#[derive(DeriveIden)] +enum ErrorEvents { + Table, + ProjectId, + TraceIdIndexed, +} diff --git a/crates/temps-migrations/src/migration/mod.rs b/crates/temps-migrations/src/migration/mod.rs index bbbc62aef..527cf0c5d 100644 --- a/crates/temps-migrations/src/migration/mod.rs +++ b/crates/temps-migrations/src/migration/mod.rs @@ -81,6 +81,7 @@ mod m20260428_000003_create_node_route_state; mod m20260430_000001_add_deployment_container_exit_info; mod m20260430_000002_add_deployment_container_runtime_info; mod m20260501_000001_add_gitlab_webhook_to_projects; +mod m20260502_000001_add_observe_correlation; pub struct Migrator; @@ -165,6 +166,7 @@ impl MigratorTrait for Migrator { Box::new(m20260430_000001_add_deployment_container_exit_info::Migration), Box::new(m20260430_000002_add_deployment_container_runtime_info::Migration), Box::new(m20260501_000001_add_gitlab_webhook_to_projects::Migration), + Box::new(m20260502_000001_add_observe_correlation::Migration), ] } } diff --git a/crates/temps-observability/Cargo.toml b/crates/temps-observability/Cargo.toml new file mode 100644 index 000000000..329673756 --- /dev/null +++ b/crates/temps-observability/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "temps-observability" +version.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +homepage.workspace = true + +[dependencies] +temps-auth = { path = "../temps-auth" } +temps-core = { path = "../temps-core" } +temps-database = { path = "../temps-database" } +temps-entities = { path = "../temps-entities" } + +tokio = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +sea-orm = { workspace = true } +async-trait = { workspace = true } +chrono = { workspace = true } +axum = { workspace = true } +http = { workspace = true } +utoipa = { workspace = true, features = ["chrono"] } +tracing = { workspace = true } +thiserror = { workspace = true } +base64 = { workspace = true } + +[dev-dependencies] +tokio-test = "0.4" +temps-migrations = { path = "../temps-migrations" } +sea-orm-migration = { workspace = true } +uuid = { workspace = true } +anyhow = { workspace = true } diff --git a/crates/temps-observability/src/cursor.rs b/crates/temps-observability/src/cursor.rs new file mode 100644 index 000000000..3282e9aab --- /dev/null +++ b/crates/temps-observability/src/cursor.rs @@ -0,0 +1,80 @@ +//! Pagination cursor for the merged event stream. +//! +//! The merge service returns rows ordered by `(ts DESC, kind, id)`. To page +//! deeper, the next request must skip past the last row we returned. We +//! encode (ts, kind, id) into an opaque base64 token rather than exposing +//! the columns directly so the wire format is stable while we evolve the +//! sort order. + +use base64::Engine; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +use crate::error::ObservabilityError; +use crate::types::EventKind; + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct Cursor { + /// Last returned row's timestamp. + pub ts: DateTime, + /// Last returned row's kind. + pub kind: EventKind, + /// Last returned row's per-kind id (string so it works for log chunk + /// composites and integer PKs alike). + pub id: String, +} + +impl Cursor { + pub fn encode(&self) -> String { + let json = serde_json::to_vec(self).unwrap_or_default(); + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(json) + } + + pub fn decode(token: &str) -> Result { + let bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(token.as_bytes()) + .map_err(|e| ObservabilityError::InvalidCursor { + reason: format!("base64 decode failed: {}", e), + })?; + serde_json::from_slice(&bytes).map_err(|e| ObservabilityError::InvalidCursor { + reason: format!("json decode failed: {}", e), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample() -> Cursor { + Cursor { + ts: DateTime::parse_from_rfc3339("2026-05-01T12:00:00Z") + .unwrap() + .with_timezone(&Utc), + kind: EventKind::Request, + id: "42".into(), + } + } + + #[test] + fn round_trip_through_base64() { + let c = sample(); + let token = c.encode(); + let back = Cursor::decode(&token).expect("valid cursor"); + assert_eq!(c, back); + } + + #[test] + fn rejects_garbage_base64() { + let err = Cursor::decode("not_base64!!").unwrap_err(); + assert!(matches!(err, ObservabilityError::InvalidCursor { .. })); + } + + #[test] + fn rejects_valid_base64_but_invalid_json() { + let token = + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(b"not json"); + let err = Cursor::decode(&token).unwrap_err(); + assert!(matches!(err, ObservabilityError::InvalidCursor { .. })); + } +} diff --git a/crates/temps-observability/src/error.rs b/crates/temps-observability/src/error.rs new file mode 100644 index 000000000..f19b1e933 --- /dev/null +++ b/crates/temps-observability/src/error.rs @@ -0,0 +1,70 @@ +use thiserror::Error; + +/// Top-level error type for the observability crate. Per CLAUDE.md: +/// structured fields, contextual messages, automatic conversion from +/// `sea_orm::DbErr`. The `From for Problem` impl lives +/// in the handlers module so the HTTP layer owns the response mapping. +#[derive(Debug, Error)] +pub enum ObservabilityError { + #[error("Project {project_id} not found")] + ProjectNotFound { project_id: i32 }, + + #[error("Event {event_id} of kind {kind} not found in project {project_id}")] + EventNotFound { + project_id: i32, + kind: String, + event_id: String, + }, + + #[error( + "Invalid kinds filter: '{value}'. Valid kinds: log, request, span, error, revenue" + )] + InvalidKindsFilter { value: String }, + + #[error("Invalid cursor: {reason}")] + InvalidCursor { reason: String }, + + #[error( + "Time range invalid: from={from} is after to={to}; the merge query needs from <= to" + )] + InvalidTimeRange { from: String, to: String }, + + #[error("Database error: {0}")] + Database(#[from] sea_orm::DbErr), +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn db_err_converts_via_from() { + let db_err = sea_orm::DbErr::Custom("boom".into()); + let err: ObservabilityError = db_err.into(); + assert!(matches!(err, ObservabilityError::Database(_))); + } + + #[test] + fn error_messages_carry_context() { + let err = ObservabilityError::EventNotFound { + project_id: 7, + kind: "request".into(), + event_id: "42".into(), + }; + let msg = err.to_string(); + assert!(msg.contains("42")); + assert!(msg.contains("request")); + assert!(msg.contains("7")); + } + + #[test] + fn invalid_kinds_filter_lists_valid_options() { + let err = ObservabilityError::InvalidKindsFilter { + value: "blob".into(), + }; + let msg = err.to_string(); + assert!(msg.contains("blob")); + assert!(msg.contains("log")); + assert!(msg.contains("revenue")); + } +} diff --git a/crates/temps-observability/src/filters.rs b/crates/temps-observability/src/filters.rs new file mode 100644 index 000000000..29c49aa26 --- /dev/null +++ b/crates/temps-observability/src/filters.rs @@ -0,0 +1,319 @@ +//! Query filters and merge utilities used by the unified Observe service. +//! +//! Kept in its own module so the merge logic (k-way merge, kind-set +//! parsing) can be unit-tested without spinning up a database. + +use chrono::{DateTime, Utc}; +use std::collections::HashSet; + +use crate::error::ObservabilityError; +use crate::types::{EventKind, ObservabilityEvent}; + +/// Default page size for the unified events list. Mirrors the project's +/// pagination convention (20 default, 100 max). +pub const DEFAULT_PAGE_SIZE: u64 = 50; +/// Hard cap on page size — enforced server-side regardless of client input. +pub const MAX_PAGE_SIZE: u64 = 200; + +#[derive(Clone, Debug)] +pub struct EventFilters { + pub project_id: i32, + pub kinds: HashSet, + pub from: Option>, + pub to: Option>, + pub deployment_id: Option, + pub environment_id: Option, + pub search: Option, + pub limit: u64, +} + +impl EventFilters { + pub fn validate(&self) -> Result<(), ObservabilityError> { + if let (Some(from), Some(to)) = (self.from, self.to) { + if from > to { + return Err(ObservabilityError::InvalidTimeRange { + from: from.to_rfc3339(), + to: to.to_rfc3339(), + }); + } + } + Ok(()) + } +} + +/// Parse a comma-separated `kinds` query parameter into a typed set. +/// Empty string or `None` yields the full set so the default-on UX matches +/// the design plan (every kind visible until the user toggles). +pub fn parse_kinds(raw: Option<&str>) -> Result, ObservabilityError> { + let Some(s) = raw else { + return Ok(EventKind::ALL.iter().copied().collect()); + }; + let s = s.trim(); + if s.is_empty() { + return Ok(EventKind::ALL.iter().copied().collect()); + } + let mut out = HashSet::new(); + for tok in s.split(',') { + let tok = tok.trim(); + if tok.is_empty() { + continue; + } + let kind = EventKind::parse(tok).ok_or_else(|| ObservabilityError::InvalidKindsFilter { + value: tok.to_string(), + })?; + out.insert(kind); + } + if out.is_empty() { + return Ok(EventKind::ALL.iter().copied().collect()); + } + Ok(out) +} + +/// Clamp a client-supplied limit into the [1, MAX_PAGE_SIZE] range. +pub fn clamp_limit(raw: Option) -> u64 { + raw.unwrap_or(DEFAULT_PAGE_SIZE).clamp(1, MAX_PAGE_SIZE) +} + +/// k-way merge of pre-sorted per-kind streams (each already DESC by ts) +/// into a single combined stream. Stable across kinds — when two rows share +/// a timestamp the order falls back to (kind, id) defined by the cursor +/// scheme. +/// +/// Implemented as a heap-free linear scan over the per-kind cursors since +/// `EventKind::ALL.len()` is 5; the constant factor beats the heap setup +/// for any realistic page size. +pub fn merge_desc_by_ts(streams: Vec>, limit: usize) -> Vec { + let mut cursors: Vec> = + streams.into_iter().map(|s| s.into_iter()).collect(); + let mut heads: Vec> = + cursors.iter_mut().map(|c| c.next()).collect(); + + let mut out = Vec::with_capacity(limit); + while out.len() < limit { + // Find the head with the latest ts (ties: prefer lower kind index + // so the order is deterministic across runs). + let mut best: Option = None; + for (i, h) in heads.iter().enumerate() { + let Some(candidate) = h else { continue }; + match best { + None => best = Some(i), + Some(j) => { + let cur_best = heads[j].as_ref().expect("best is occupied"); + let candidate_key = (candidate.ts(), candidate.kind() as u8); + let best_key = (cur_best.ts(), cur_best.kind() as u8); + // We want descending ts; for equal ts prefer lower kind index. + if candidate_key.0 > best_key.0 + || (candidate_key.0 == best_key.0 && candidate_key.1 < best_key.1) + { + best = Some(i); + } + } + } + } + let Some(idx) = best else { break }; + out.push(heads[idx].take().expect("head present")); + heads[idx] = cursors[idx].next(); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::{ErrorRow, RequestRow}; + + fn ts(s: &str) -> DateTime { + DateTime::parse_from_rfc3339(s).unwrap().with_timezone(&Utc) + } + + fn err_at(t: &str, id: i64) -> ObservabilityEvent { + ObservabilityEvent::Error(ErrorRow { + id, + ts: ts(t), + deployment_id: None, + environment_id: None, + trace_id: None, + error_group_id: 1, + fingerprint: format!("fp-{id}"), + error_class: "Boom".into(), + message: None, + stacktrace_preview: serde_json::json!([]), + stacktrace_truncated: false, + }) + } + fn req_at(t: &str, id: i64) -> ObservabilityEvent { + ObservabilityEvent::Request(RequestRow { + id, + ts: ts(t), + deployment_id: None, + environment_id: None, + trace_id: None, + error_group_id: None, + method: "GET".into(), + host: "x".into(), + path: "/".into(), + query_string: None, + status: 200, + latency_ms: None, + client_ip: None, + country: None, + user_agent: None, + referrer: None, + request_headers: serde_json::json!({}), + response_headers: serde_json::json!({}), + headers_truncated: false, + }) + } + + // ── parse_kinds ────────────────────────────────────────────────────── + + #[test] + fn parse_kinds_default_returns_all() { + let kinds = parse_kinds(None).unwrap(); + assert_eq!(kinds.len(), 4); + } + + #[test] + fn parse_kinds_empty_string_returns_all() { + let kinds = parse_kinds(Some("")).unwrap(); + assert_eq!(kinds.len(), 4); + let kinds2 = parse_kinds(Some(" ")).unwrap(); + assert_eq!(kinds2.len(), 4); + } + + #[test] + fn parse_kinds_parses_csv() { + let kinds = parse_kinds(Some("request,error")).unwrap(); + assert_eq!(kinds.len(), 2); + assert!(kinds.contains(&EventKind::Request)); + assert!(kinds.contains(&EventKind::Error)); + } + + #[test] + fn parse_kinds_trims_whitespace() { + let kinds = parse_kinds(Some(" request , error ")).unwrap(); + assert_eq!(kinds.len(), 2); + } + + #[test] + fn parse_kinds_rejects_unknown_kind() { + let err = parse_kinds(Some("request,nope")).unwrap_err(); + assert!(matches!(err, ObservabilityError::InvalidKindsFilter { .. })); + } + + // ── clamp_limit ────────────────────────────────────────────────────── + + #[test] + fn clamp_limit_uses_default_when_none() { + assert_eq!(clamp_limit(None), DEFAULT_PAGE_SIZE); + } + + #[test] + fn clamp_limit_caps_at_max() { + assert_eq!(clamp_limit(Some(99_999)), MAX_PAGE_SIZE); + } + + #[test] + fn clamp_limit_lower_bound_is_one() { + assert_eq!(clamp_limit(Some(0)), 1); + } + + // ── EventFilters::validate ─────────────────────────────────────────── + + #[test] + fn validate_rejects_inverted_time_range() { + let f = EventFilters { + project_id: 1, + kinds: EventKind::ALL.iter().copied().collect(), + from: Some(ts("2026-05-02T00:00:00Z")), + to: Some(ts("2026-05-01T00:00:00Z")), + deployment_id: None, + environment_id: None, + search: None, + limit: 50, + }; + assert!(matches!( + f.validate(), + Err(ObservabilityError::InvalidTimeRange { .. }) + )); + } + + #[test] + fn validate_accepts_equal_endpoints() { + let f = EventFilters { + project_id: 1, + kinds: EventKind::ALL.iter().copied().collect(), + from: Some(ts("2026-05-01T00:00:00Z")), + to: Some(ts("2026-05-01T00:00:00Z")), + deployment_id: None, + environment_id: None, + search: None, + limit: 50, + }; + assert!(f.validate().is_ok()); + } + + // ── merge_desc_by_ts ───────────────────────────────────────────────── + + #[test] + fn merge_interleaves_by_ts_desc() { + // errors: [2026-05-01T12:03Z, 2026-05-01T12:01Z] + // requests:[2026-05-01T12:02Z, 2026-05-01T12:00Z] + let errs = vec![ + err_at("2026-05-01T12:03:00Z", 3), + err_at("2026-05-01T12:01:00Z", 1), + ]; + let reqs = vec![ + req_at("2026-05-01T12:02:00Z", 2), + req_at("2026-05-01T12:00:00Z", 0), + ]; + let merged = merge_desc_by_ts(vec![errs, reqs], 10); + let kinds_in_order: Vec<_> = merged.iter().map(|e| e.kind()).collect(); + assert_eq!( + kinds_in_order, + vec![ + EventKind::Error, + EventKind::Request, + EventKind::Error, + EventKind::Request, + ] + ); + } + + #[test] + fn merge_respects_limit() { + let stream = vec![ + err_at("2026-05-01T12:03:00Z", 1), + err_at("2026-05-01T12:02:00Z", 2), + err_at("2026-05-01T12:01:00Z", 3), + ]; + let merged = merge_desc_by_ts(vec![stream], 2); + assert_eq!(merged.len(), 2); + } + + #[test] + fn merge_handles_single_empty_stream() { + let merged: Vec = + merge_desc_by_ts(vec![vec![]], 10); + assert!(merged.is_empty()); + } + + #[test] + fn merge_breaks_ties_by_kind_for_determinism() { + // Same ts, different kinds — Request comes before Error because + // Request's EventKind discriminant is lower. + let reqs = vec![req_at("2026-05-01T12:00:00Z", 1)]; + let errs = vec![err_at("2026-05-01T12:00:00Z", 1)]; + let merged = merge_desc_by_ts(vec![reqs, errs], 10); + assert_eq!(merged[0].kind(), EventKind::Request); + assert_eq!(merged[1].kind(), EventKind::Error); + } + + #[test] + fn merge_drains_all_streams() { + let errs = vec![err_at("2026-05-01T12:03:00Z", 1)]; + let reqs = vec![req_at("2026-05-01T12:02:00Z", 1)]; + let merged = merge_desc_by_ts(vec![errs, reqs], 100); + assert_eq!(merged.len(), 2); + } +} diff --git a/crates/temps-observability/src/handlers/events.rs b/crates/temps-observability/src/handlers/events.rs new file mode 100644 index 000000000..48bd03d44 --- /dev/null +++ b/crates/temps-observability/src/handlers/events.rs @@ -0,0 +1,250 @@ +//! Unified `GET /projects/{project_id}/observe/events` endpoint. +//! +//! Returns the merged `ObservabilityEvent[]` page with each row +//! self-contained (truncated heavy fields + `*_truncated` flags). The +//! frontend renders both the list row AND its detail panel from this +//! payload alone. +//! +//! OpenAPI operation IDs are prefixed `observability_*` per the project's +//! collision convention. + +use std::sync::Arc; + +use axum::{ + extract::{Path, Query, State}, + response::IntoResponse, + routing::get, + Json, Router, +}; +use chrono::{DateTime, Utc}; +use http::StatusCode; +use serde::{Deserialize, Serialize}; +use temps_auth::{permission_guard, RequireAuth}; +use temps_core::problemdetails::{self, Problem}; +use utoipa::{IntoParams, OpenApi, ToSchema}; + +use crate::error::ObservabilityError; +use crate::filters::{clamp_limit, parse_kinds, EventFilters}; +use crate::service::{FullError, FullEvent, FullRequest, ObservabilityService}; +use crate::types::{ErrorRow, EventKind, ObservabilityEvent, RequestRow, RevenueRow, SpanRow}; + +#[derive(Clone)] +pub struct ObservabilityState { + pub service: Arc, +} + +#[derive(OpenApi)] +#[openapi( + paths(observability_list_events, observability_full_event), + components(schemas( + ObservabilityEvent, + RequestRow, + SpanRow, + ErrorRow, + RevenueRow, + EventKind, + EventsResponse, + FullEvent, + FullRequest, + FullError, + )), + tags( + (name = "Observability", description = "Unified observability event stream — runtime logs, requests, spans, errors, revenue") + ) +)] +pub struct ObservabilityApiDoc; + +#[derive(Debug, Default, Deserialize, IntoParams)] +#[into_params(parameter_in = Query)] +pub struct EventsQuery { + /// Comma-separated kinds: `log,request,span,error,revenue`. Empty or + /// missing returns every kind. + pub kinds: Option, + /// Inclusive lower bound on event timestamp (ISO 8601, `Z` suffix). + pub from: Option>, + /// Inclusive upper bound on event timestamp. + pub to: Option>, + pub deployment_id: Option, + pub environment_id: Option, + /// Free-text substring matched against per-kind summary fields + /// (request path / error class / revenue event_type). + pub search: Option, + /// Page size (default 50, max 200). + pub limit: Option, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct EventsResponse { + pub events: Vec, + /// Echo of the kinds filter actually applied (server-resolved). Useful + /// for clients that pass `kinds=` empty and want to know what they got. + pub applied_kinds: Vec, +} + +/// List a merged page of observability events for a project. +/// +/// Each row carries everything the side panel needs to render — no +/// follow-up fetch is required for the common case. Heavy fields +/// (stacktraces, headers, span attributes) are truncated server-side and +/// expose a `*_truncated` flag; clients fetch the full row from the +/// `/full` endpoint only when the user explicitly clicks "Show full". +#[utoipa::path( + get, + operation_id = "observability_list_events", + tag = "Observability", + path = "/projects/{project_id}/observe/events", + params( + ("project_id" = i32, Path, description = "Project ID"), + EventsQuery, + ), + responses( + (status = 200, description = "Merged event page", body = EventsResponse), + (status = 400, description = "Invalid filter (kinds, time range, …)", body = String), + (status = 401, description = "Unauthorized", body = String), + (status = 403, description = "Insufficient permissions", body = String), + (status = 500, description = "Internal server error", body = String), + ), + security(("bearer_auth" = [])) +)] +pub async fn observability_list_events( + RequireAuth(auth): RequireAuth, + State(state): State>, + Path(project_id): Path, + Query(query): Query, +) -> Result { + permission_guard!(auth, LogsRead); + + let kinds = parse_kinds(query.kinds.as_deref())?; + let limit = clamp_limit(query.limit); + let filters = EventFilters { + project_id, + kinds: kinds.clone(), + from: query.from, + to: query.to, + deployment_id: query.deployment_id, + environment_id: query.environment_id, + search: query.search, + limit, + }; + + let events = state.service.query(filters).await?; + + let mut applied: Vec = kinds.into_iter().collect(); + applied.sort_by_key(|k| *k as u8); + + Ok(( + StatusCode::OK, + Json(EventsResponse { + events, + applied_kinds: applied, + }), + )) +} + +impl From for Problem { + fn from(error: ObservabilityError) -> Self { + match error { + ObservabilityError::ProjectNotFound { .. } => { + problemdetails::new(StatusCode::NOT_FOUND) + .with_title("Project Not Found") + .with_detail(error.to_string()) + } + ObservabilityError::EventNotFound { .. } => { + problemdetails::new(StatusCode::NOT_FOUND) + .with_title("Event Not Found") + .with_detail(error.to_string()) + } + ObservabilityError::InvalidKindsFilter { .. } + | ObservabilityError::InvalidCursor { .. } + | ObservabilityError::InvalidTimeRange { .. } => { + problemdetails::new(StatusCode::BAD_REQUEST) + .with_title("Invalid Request") + .with_detail(error.to_string()) + } + ObservabilityError::Database(_) => { + problemdetails::new(StatusCode::INTERNAL_SERVER_ERROR) + .with_title("Internal Server Error") + .with_detail(error.to_string()) + } + } + } +} + +/// Fetch the un-truncated form of one event by `(kind, id)`. Side panel +/// "Show full" action calls this — the list response carries truncated +/// previews + a `*_truncated` flag to let the UI decide whether to fetch. +#[utoipa::path( + get, + operation_id = "observability_full_event", + tag = "Observability", + path = "/projects/{project_id}/observe/events/{kind}/{event_id}/full", + params( + ("project_id" = i32, Path, description = "Project ID"), + ("kind" = EventKind, Path, description = "Event kind discriminator"), + ("event_id" = String, Path, description = "Per-kind primary key"), + ), + responses( + (status = 200, description = "Full row", body = FullEvent), + (status = 401, description = "Unauthorized", body = String), + (status = 403, description = "Insufficient permissions", body = String), + (status = 404, description = "Event not found in project", body = String), + (status = 500, description = "Internal server error", body = String), + ), + security(("bearer_auth" = [])) +)] +pub async fn observability_full_event( + RequireAuth(auth): RequireAuth, + State(state): State>, + Path((project_id, kind, event_id)): Path<(i32, EventKind, String)>, +) -> Result { + permission_guard!(auth, LogsRead); + + let event = state.service.fetch_full(project_id, kind, &event_id).await?; + Ok((StatusCode::OK, Json(event))) +} + +pub fn configure_observability_routes() -> Router> { + Router::new() + .route( + "/projects/{project_id}/observe/events", + get(observability_list_events), + ) + .route( + "/projects/{project_id}/observe/events/{kind}/{event_id}/full", + get(observability_full_event), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn problem_mapping_is_exhaustive() { + // Compile-time check: if a new ObservabilityError variant is added, + // the From impl must handle it (no _ catch-all). + let cases: Vec = vec![ + ObservabilityError::ProjectNotFound { project_id: 1 }, + ObservabilityError::EventNotFound { + project_id: 1, + kind: "log".into(), + event_id: "x".into(), + }, + ObservabilityError::InvalidKindsFilter { + value: "bad".into(), + }, + ObservabilityError::InvalidCursor { + reason: "x".into(), + }, + ObservabilityError::InvalidTimeRange { + from: "a".into(), + to: "b".into(), + }, + ObservabilityError::Database(sea_orm::DbErr::Custom("x".into())), + ]; + for err in cases { + // Ensure no panic — Problem construction succeeds for every variant. + let _: Problem = err.into(); + } + } +} diff --git a/crates/temps-observability/src/handlers/mod.rs b/crates/temps-observability/src/handlers/mod.rs new file mode 100644 index 000000000..7e5c6df86 --- /dev/null +++ b/crates/temps-observability/src/handlers/mod.rs @@ -0,0 +1,3 @@ +pub mod events; + +pub use events::{configure_observability_routes, ObservabilityApiDoc, ObservabilityState}; diff --git a/crates/temps-observability/src/lib.rs b/crates/temps-observability/src/lib.rs new file mode 100644 index 000000000..792d04681 --- /dev/null +++ b/crates/temps-observability/src/lib.rs @@ -0,0 +1,34 @@ +//! Unified observability surface for the project-level Observe page. +//! +//! Exposes one discriminated `ObservabilityEvent` row that covers request +//! logs, OTel spans, error events, and revenue events. Runtime logs are +//! intentionally **not** included — they live on a dedicated Logs page +//! because their volume would dominate the merged timeline and their +//! storage path (TimescaleDB hypertable + chunked file/S3 store) doesn't +//! compose with the per-kind LIMIT merge strategy. +//! +//! Everything the side panel needs to render is included on the row, so +//! the list view never needs a follow-up fetch in the common case. + +pub mod cursor; +pub mod error; +pub mod filters; +pub mod handlers; +pub mod plugin; +pub mod service; +pub mod types; + +pub use handlers::{configure_observability_routes, ObservabilityApiDoc, ObservabilityState}; +pub use plugin::ObservabilityPlugin; +pub use service::ObservabilityService; + +pub use cursor::Cursor; +pub use error::ObservabilityError; +pub use filters::{ + clamp_limit, merge_desc_by_ts, parse_kinds, EventFilters, DEFAULT_PAGE_SIZE, MAX_PAGE_SIZE, +}; +pub use types::{ + project_headers, truncate_attributes, truncate_stacktrace, ErrorRow, EventKind, + ObservabilityEvent, RequestRow, RevenueRow, SpanRow, HEADER_WHITELIST, + SPAN_ATTRIBUTE_PREVIEW_KEYS, STACKTRACE_PREVIEW_FRAMES, +}; diff --git a/crates/temps-observability/src/plugin.rs b/crates/temps-observability/src/plugin.rs new file mode 100644 index 000000000..57f4677e5 --- /dev/null +++ b/crates/temps-observability/src/plugin.rs @@ -0,0 +1,70 @@ +//! Plugin wiring for temps-observability. +//! +//! Registers a single `ObservabilityState` and exposes the merged +//! `/projects/{id}/observe/events*` route tree under the authenticated +//! API surface. No public/webhook routes. + +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; + +use axum::Router; +use sea_orm::DatabaseConnection; +use temps_core::plugin::{ + PluginContext, PluginError, PluginRoutes, ServiceRegistrationContext, TempsPlugin, +}; +use tracing::debug; +use utoipa::openapi::OpenApi; +use utoipa::OpenApi as OpenApiTrait; + +use crate::handlers::{configure_observability_routes, ObservabilityApiDoc, ObservabilityState}; +use crate::service::ObservabilityService; + +pub struct ObservabilityPlugin; + +impl ObservabilityPlugin { + pub fn new() -> Self { + Self + } +} + +impl Default for ObservabilityPlugin { + fn default() -> Self { + Self::new() + } +} + +impl TempsPlugin for ObservabilityPlugin { + fn name(&self) -> &'static str { + "observability" + } + + fn register_services<'a>( + &'a self, + context: &'a ServiceRegistrationContext, + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + let db = context.require_service::(); + let service = Arc::new(ObservabilityService::new(db)); + let state = Arc::new(ObservabilityState { + service: service.clone(), + }); + + context.register_service(service); + context.register_service(state); + + debug!("Observability plugin services registered"); + Ok(()) + }) + } + + fn configure_routes(&self, context: &PluginContext) -> Option { + let state = context.require_service::(); + let router: Router = configure_observability_routes().with_state(state); + Some(PluginRoutes { router }) + } + + fn openapi_schema(&self) -> Option { + Some(::openapi()) + } +} diff --git a/crates/temps-observability/src/service.rs b/crates/temps-observability/src/service.rs new file mode 100644 index 000000000..82a37d5aa --- /dev/null +++ b/crates/temps-observability/src/service.rs @@ -0,0 +1,720 @@ +//! Merge service: parallel-fetches per-kind row streams from the source +//! tables, maps each to the unified `ObservabilityEvent` wire type +//! (truncating heavy fields server-side), and k-way merges them by +//! `ts DESC` into a single page. +//! +//! Per-kind queries are kept independent so a future kind addition only +//! requires a new fetcher; the merge logic is in `filters::merge_desc_by_ts`. +//! +//! NOTE on `Log` and `Span` kinds: log rows live in `temps-log-aggregator`'s +//! file/S3 chunk store, and OTel spans are in a raw TimescaleDB hypertable +//! without a Sea-ORM model. Both currently return empty streams from this +//! service. The wire contract is complete and the UI renders them without +//! changes once the fetchers are wired (separate follow-up). + +use std::sync::Arc; + +use sea_orm::{ + ColumnTrait, DatabaseBackend, DatabaseConnection, EntityTrait, FromQueryResult, QueryFilter, + QueryOrder, QuerySelect, Statement, +}; +use temps_entities::{error_events, proxy_logs, revenue_events}; + +use crate::error::ObservabilityError; +use crate::filters::{merge_desc_by_ts, EventFilters}; +use crate::types::{ + project_headers, truncate_attributes, truncate_stacktrace, ErrorRow, EventKind, + ObservabilityEvent, RequestRow, RevenueRow, SpanRow, +}; + +/// One un-truncated row, returned by the `/full/{type}/{id}` endpoint when +/// the user clicks "Show full". Same shape as the list rows, but with the +/// raw heavy fields restored (no truncation flags) so the side panel can +/// render the long form. +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum FullEvent { + Request(FullRequest), + Error(FullError), + Revenue(crate::types::RevenueRow), + /// `SpanRow.attributes` is the truncated form; re-fetching returns + /// the same shape so the panel has a stable contract. + Span(crate::types::SpanRow), +} + +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +pub struct FullRequest { + pub id: i64, + pub ts: chrono::DateTime, + pub deployment_id: Option, + pub environment_id: Option, + pub trace_id: Option, + pub error_group_id: Option, + pub method: String, + pub host: String, + pub path: String, + pub status: i16, + pub latency_ms: Option, + pub client_ip: Option, + pub user_agent: Option, + pub referrer: Option, + pub request_headers: Option, + pub response_headers: Option, +} + +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +pub struct FullError { + pub id: i64, + pub ts: chrono::DateTime, + pub deployment_id: Option, + pub environment_id: Option, + pub trace_id: Option, + pub error_group_id: i32, + pub fingerprint: String, + pub error_class: String, + pub message: Option, + /// Full JSONB blob from `error_events.data` — stack trace, breadcrumbs, + /// request context, everything. Schema is documented per source SDK. + pub data: Option, +} + +pub struct ObservabilityService { + db: Arc, +} + +impl ObservabilityService { + pub fn new(db: Arc) -> Self { + Self { db } + } + + /// Fetch one merged page of events. + /// + /// Strategy: + /// 1. Validate the filter struct. + /// 2. For each enabled kind in `filters.kinds`, run an independent + /// Sea-ORM query (LIMIT = page size) returning rows already mapped + /// to the `ObservabilityEvent` wire type — no follow-up fetches. + /// 3. k-way merge the streams by ts DESC and trim to `filters.limit`. + pub async fn query( + &self, + filters: EventFilters, + ) -> Result, ObservabilityError> { + filters.validate()?; + + let mut streams: Vec> = Vec::new(); + + if filters.kinds.contains(&EventKind::Request) { + streams.push(self.fetch_requests(&filters).await?); + } + if filters.kinds.contains(&EventKind::Error) { + streams.push(self.fetch_errors(&filters).await?); + } + if filters.kinds.contains(&EventKind::Revenue) { + streams.push(self.fetch_revenue(&filters).await?); + } + if filters.kinds.contains(&EventKind::Span) { + streams.push(self.fetch_spans(&filters).await?); + } + + Ok(merge_desc_by_ts(streams, filters.limit as usize)) + } + + /// Fetch the un-truncated form of a single event by `(kind, id)`. + /// Used by the side panel "Show full" action. + pub async fn fetch_full( + &self, + project_id: i32, + kind: EventKind, + id: &str, + ) -> Result { + match kind { + EventKind::Request => { + let pk: i32 = id.parse().map_err(|_| ObservabilityError::EventNotFound { + project_id, + kind: "request".into(), + event_id: id.into(), + })?; + let row = proxy_logs::Entity::find_by_id(pk) + .filter(proxy_logs::Column::ProjectId.eq(project_id)) + .one(self.db.as_ref()) + .await? + .ok_or_else(|| ObservabilityError::EventNotFound { + project_id, + kind: "request".into(), + event_id: id.into(), + })?; + Ok(FullEvent::Request(FullRequest { + id: row.id as i64, + ts: row.timestamp, + deployment_id: row.deployment_id, + environment_id: row.environment_id, + trace_id: row.trace_id, + error_group_id: row.error_group_id, + method: row.method, + host: row.host, + path: row.path, + status: row.status_code, + latency_ms: row.response_time_ms, + client_ip: row.client_ip, + user_agent: row.user_agent, + referrer: row.referrer, + request_headers: row.request_headers, + response_headers: row.response_headers, + })) + } + EventKind::Error => { + let pk: i64 = id.parse().map_err(|_| ObservabilityError::EventNotFound { + project_id, + kind: "error".into(), + event_id: id.into(), + })?; + let row = error_events::Entity::find_by_id(pk) + .filter(error_events::Column::ProjectId.eq(project_id)) + .one(self.db.as_ref()) + .await? + .ok_or_else(|| ObservabilityError::EventNotFound { + project_id, + kind: "error".into(), + event_id: id.into(), + })?; + Ok(FullEvent::Error(FullError { + id: row.id, + ts: row.timestamp, + deployment_id: row.deployment_id, + environment_id: row.environment_id, + trace_id: row.trace_id_indexed, + error_group_id: row.error_group_id, + fingerprint: row.fingerprint_hash, + error_class: row.exception_type, + message: row.exception_value, + data: row.data, + })) + } + EventKind::Revenue => { + let pk: i64 = id.parse().map_err(|_| ObservabilityError::EventNotFound { + project_id, + kind: "revenue".into(), + event_id: id.into(), + })?; + let row = revenue_events::Entity::find_by_id(pk) + .filter(revenue_events::Column::ProjectId.eq(project_id)) + .one(self.db.as_ref()) + .await? + .ok_or_else(|| ObservabilityError::EventNotFound { + project_id, + kind: "revenue".into(), + event_id: id.into(), + })?; + let ObservabilityEvent::Revenue(rev) = map_revenue(row) else { + unreachable!("map_revenue returns Revenue variant"); + }; + Ok(FullEvent::Revenue(rev)) + } + EventKind::Span => { + let row = self + .find_span_by_id(project_id, id) + .await? + .ok_or_else(|| ObservabilityError::EventNotFound { + project_id, + kind: "span".into(), + event_id: id.into(), + })?; + let ObservabilityEvent::Span(full) = map_span(row) else { + unreachable!("map_span returns Span variant"); + }; + Ok(FullEvent::Span(full)) + } + } + } + + async fn fetch_requests( + &self, + filters: &EventFilters, + ) -> Result, ObservabilityError> { + let mut q = proxy_logs::Entity::find() + .filter(proxy_logs::Column::ProjectId.eq(filters.project_id)); + if let Some(from) = filters.from { + q = q.filter(proxy_logs::Column::Timestamp.gte(from)); + } + if let Some(to) = filters.to { + q = q.filter(proxy_logs::Column::Timestamp.lte(to)); + } + if let Some(d) = filters.deployment_id { + q = q.filter(proxy_logs::Column::DeploymentId.eq(d)); + } + if let Some(e) = filters.environment_id { + q = q.filter(proxy_logs::Column::EnvironmentId.eq(e)); + } + if let Some(ref s) = filters.search { + q = q.filter(proxy_logs::Column::Path.contains(s)); + } + let rows = q + .order_by_desc(proxy_logs::Column::Timestamp) + .limit(filters.limit) + .all(self.db.as_ref()) + .await?; + + Ok(rows.into_iter().map(map_request).collect()) + } + + async fn fetch_errors( + &self, + filters: &EventFilters, + ) -> Result, ObservabilityError> { + let mut q = error_events::Entity::find() + .filter(error_events::Column::ProjectId.eq(filters.project_id)); + if let Some(from) = filters.from { + q = q.filter(error_events::Column::Timestamp.gte(from)); + } + if let Some(to) = filters.to { + q = q.filter(error_events::Column::Timestamp.lte(to)); + } + if let Some(d) = filters.deployment_id { + q = q.filter(error_events::Column::DeploymentId.eq(d)); + } + if let Some(e) = filters.environment_id { + q = q.filter(error_events::Column::EnvironmentId.eq(e)); + } + if let Some(ref s) = filters.search { + q = q.filter(error_events::Column::ExceptionType.contains(s)); + } + let rows = q + .order_by_desc(error_events::Column::Timestamp) + .limit(filters.limit) + .all(self.db.as_ref()) + .await?; + + Ok(rows.into_iter().map(map_error).collect()) + } + + async fn fetch_revenue( + &self, + filters: &EventFilters, + ) -> Result, ObservabilityError> { + let mut q = revenue_events::Entity::find() + .filter(revenue_events::Column::ProjectId.eq(filters.project_id)); + if let Some(from) = filters.from { + q = q.filter(revenue_events::Column::OccurredAt.gte(from)); + } + if let Some(to) = filters.to { + q = q.filter(revenue_events::Column::OccurredAt.lte(to)); + } + if let Some(d) = filters.deployment_id { + q = q.filter(revenue_events::Column::DeploymentId.eq(d)); + } + if let Some(e) = filters.environment_id { + q = q.filter(revenue_events::Column::EnvironmentId.eq(e)); + } + if let Some(ref s) = filters.search { + q = q.filter(revenue_events::Column::EventType.contains(s)); + } + let rows = q + .order_by_desc(revenue_events::Column::OccurredAt) + .limit(filters.limit) + .all(self.db.as_ref()) + .await?; + + Ok(rows.into_iter().map(map_revenue).collect()) + } + + /// Fetch spans from the raw `otel_spans` TimescaleDB hypertable. No + /// Sea-ORM entity exists for it; we hydrate rows via `FromQueryResult` + /// instead. Spans are even higher-volume than logs — gated client-side. + async fn fetch_spans( + &self, + filters: &EventFilters, + ) -> Result, ObservabilityError> { + let mut sql = String::from( + "SELECT id, project_id, deployment_id, service_name, trace_id, span_id, \ + parent_span_id, name, status_code, start_time, duration_ms, attributes \ + FROM otel_spans WHERE project_id = $1", + ); + let mut values: Vec = vec![filters.project_id.into()]; + let mut next_param = 2usize; + + if let Some(from) = filters.from { + sql.push_str(&format!(" AND start_time >= ${}", next_param)); + values.push(from.into()); + next_param += 1; + } + if let Some(to) = filters.to { + sql.push_str(&format!(" AND start_time <= ${}", next_param)); + values.push(to.into()); + next_param += 1; + } + if let Some(d) = filters.deployment_id { + sql.push_str(&format!(" AND deployment_id = ${}", next_param)); + values.push(d.into()); + next_param += 1; + } + if let Some(ref s) = filters.search { + sql.push_str(&format!(" AND name ILIKE ${}", next_param)); + values.push(format!("%{}%", s).into()); + next_param += 1; + } + sql.push_str(&format!( + " ORDER BY start_time DESC LIMIT ${}", + next_param + )); + values.push(sea_orm::Value::BigInt(Some(filters.limit as i64))); + + let stmt = Statement::from_sql_and_values(DatabaseBackend::Postgres, &sql, values); + let rows = SpanRecord::find_by_statement(stmt) + .all(self.db.as_ref()) + .await?; + + Ok(rows.into_iter().map(map_span).collect()) + } + + async fn find_span_by_id( + &self, + project_id: i32, + id: &str, + ) -> Result, ObservabilityError> { + let pk: i64 = id.parse().map_err(|_| ObservabilityError::EventNotFound { + project_id, + kind: "span".into(), + event_id: id.into(), + })?; + let stmt = Statement::from_sql_and_values( + DatabaseBackend::Postgres, + "SELECT id, project_id, deployment_id, service_name, trace_id, span_id, \ + parent_span_id, name, status_code, start_time, duration_ms, attributes \ + FROM otel_spans WHERE project_id = $1 AND id = $2 LIMIT 1", + vec![project_id.into(), pk.into()], + ); + let row = SpanRecord::find_by_statement(stmt) + .one(self.db.as_ref()) + .await?; + Ok(row) + } +} + +/// Raw row from `otel_spans` — no Sea-ORM entity exists for the +/// hypertable, so we hydrate via `FromQueryResult` directly. +#[derive(Debug, FromQueryResult)] +struct SpanRecord { + id: i64, + #[allow(dead_code)] + project_id: i32, + deployment_id: Option, + service_name: String, + trace_id: String, + span_id: String, + parent_span_id: Option, + name: String, + status_code: String, + start_time: chrono::DateTime, + duration_ms: f64, + attributes: serde_json::Value, +} + +// ────────────────────────────────────────────────────────────────────────── +// Per-kind row mappers (free functions so we can unit-test without DB) +// ────────────────────────────────────────────────────────────────────────── + +pub fn map_request(m: proxy_logs::Model) -> ObservabilityEvent { + let (req_headers, req_dropped) = project_headers(m.request_headers.as_ref()); + let (resp_headers, resp_dropped) = project_headers(m.response_headers.as_ref()); + ObservabilityEvent::Request(RequestRow { + id: m.id as i64, + ts: m.timestamp, + deployment_id: m.deployment_id, + environment_id: m.environment_id, + trace_id: m.trace_id, + error_group_id: m.error_group_id, + method: m.method, + host: m.host, + path: m.path, + query_string: m.query_string, + status: m.status_code, + latency_ms: m.response_time_ms, + client_ip: m.client_ip, + country: None, + user_agent: m.user_agent, + referrer: m.referrer, + request_headers: req_headers, + response_headers: resp_headers, + headers_truncated: req_dropped || resp_dropped, + }) +} + +pub fn map_error(m: error_events::Model) -> ObservabilityEvent { + // Pull stack frames out of the JSONB blob if they're there. Both the + // Sentry-wrapped and internal layouts use a `stack_trace` array; we + // probe both points. + let stack_raw = m + .data + .as_ref() + .and_then(|d| d.pointer("/stack_trace")) + .or_else(|| { + m.data + .as_ref() + .and_then(|d| d.pointer("/sentry/exception/values/0/stacktrace/frames")) + }) + .cloned(); + let (stacktrace_preview, stacktrace_truncated) = truncate_stacktrace(stack_raw.as_ref()); + + // Resolve the human-readable message. The top-level `exception_value` + // column is frequently NULL even when the original payload has plenty + // of context — Sentry SDKs that emit `captureMessage()` store the + // text under `data.sentry.message` or `data.message`, and some SDKs + // bury it in the first breadcrumb. Probe in priority order so the + // Observe row always shows real text instead of bare "Error:". + let resolved_message = m + .exception_value + .clone() + .filter(|s| !s.trim().is_empty()) + .or_else(|| extract_error_message(m.data.as_ref())); + + ObservabilityEvent::Error(ErrorRow { + id: m.id, + ts: m.timestamp, + deployment_id: m.deployment_id, + environment_id: m.environment_id, + trace_id: m.trace_id_indexed, + error_group_id: m.error_group_id, + fingerprint: m.fingerprint_hash, + error_class: m.exception_type, + message: resolved_message, + stacktrace_preview, + stacktrace_truncated, + }) +} + +/// Probe the JSONB blob for a human-readable error message. Looks at the +/// most common SDK-emitted shapes; returns the first non-empty hit. +/// +/// Sentry SDKs put the text in different places depending on whether the +/// event was emitted via `captureException` (→ exception.values[].value) +/// or `captureMessage` (→ logentry.formatted, sometimes message). We +/// probe both wrapped (`/sentry/...`) and top-level layouts because the +/// ingestion pipeline switches between them based on whether the raw +/// event was preserved. +fn extract_error_message(data: Option<&serde_json::Value>) -> Option { + let data = data?; + let candidates = [ + // Sentry SDK formatted message (captureMessage path) — wrapped + "/sentry/logentry/formatted", + "/sentry/logentry/message", + "/sentry/message", + "/sentry/exception/values/0/value", + // Top-level Sentry payload (no wrapper) + "/logentry/formatted", + "/logentry/message", + "/message", + "/exception/values/0/value", + // Internal / generic fallbacks + "/log_message", + ]; + for path in candidates { + if let Some(s) = data.pointer(path).and_then(|v| v.as_str()) { + let trimmed = s.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + } + None +} + +pub fn map_revenue(m: revenue_events::Model) -> ObservabilityEvent { + ObservabilityEvent::Revenue(RevenueRow { + id: m.id, + ts: m.occurred_at, + deployment_id: m.deployment_id, + environment_id: m.environment_id, + trace_id: m.trace_id, + provider: m.provider, + event_type: m.event_type, + customer_ref: m.customer_ref, + amount_minor: m.amount_minor, + currency: m.currency, + }) +} + +fn map_span(s: SpanRecord) -> ObservabilityEvent { + let (attributes, attributes_truncated) = truncate_attributes(Some(&s.attributes)); + ObservabilityEvent::Span(SpanRow { + id: s.id.to_string(), + ts: s.start_time, + deployment_id: s.deployment_id, + environment_id: None, + trace_id: s.trace_id, + span_id: s.span_id, + parent_span_id: s.parent_span_id, + service: s.service_name, + operation: s.name, + duration_ms: Some(s.duration_ms), + status: Some(s.status_code), + attributes, + attributes_truncated, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + use serde_json::json; + + fn proxy_log_model() -> proxy_logs::Model { + proxy_logs::Model { + id: 7, + timestamp: Utc::now(), + method: "GET".into(), + path: "/api".into(), + query_string: None, + host: "x.test".into(), + status_code: 200, + response_time_ms: Some(12), + request_source: "proxy".into(), + is_system_request: false, + routing_status: "routed".into(), + project_id: Some(1), + environment_id: Some(2), + deployment_id: Some(3), + container_id: None, + upstream_host: None, + error_message: None, + client_ip: Some("1.2.3.4".into()), + user_agent: Some("curl".into()), + referrer: None, + request_id: "req-1".into(), + ip_geolocation_id: None, + browser: None, + browser_version: None, + operating_system: None, + device_type: None, + is_bot: None, + bot_name: None, + request_size_bytes: None, + response_size_bytes: None, + cache_status: None, + request_headers: Some(json!({"host": "x.test", "x-secret": "shh"})), + response_headers: Some(json!({"content-type": "text/html"})), + created_date: Utc::now().date_naive(), + session_id: None, + visitor_id: None, + trace_id: Some("4bf92f3577b34da6a3ce929d0e0e4736".into()), + error_group_id: Some(99), + } + } + + #[test] + fn map_request_keeps_essentials_and_truncates_headers() { + let ev = map_request(proxy_log_model()); + let ObservabilityEvent::Request(r) = ev else { + panic!("expected request"); + }; + assert_eq!(r.id, 7); + assert_eq!(r.method, "GET"); + assert_eq!( + r.trace_id.as_deref(), + Some("4bf92f3577b34da6a3ce929d0e0e4736") + ); + assert_eq!(r.error_group_id, Some(99)); + assert!(r.headers_truncated, "x-secret should have been dropped"); + assert!(r.request_headers.as_object().unwrap().contains_key("host")); + assert!(!r + .request_headers + .as_object() + .unwrap() + .contains_key("x-secret")); + } + + #[test] + fn map_error_promotes_trace_id_and_truncates_stack() { + let frames: Vec = (0..20) + .map(|i| json!({"function": format!("fn_{}", i)})) + .collect(); + let model = error_events::Model { + id: 42, + error_group_id: 1, + project_id: 7, + environment_id: None, + deployment_id: None, + visitor_id: None, + ip_geolocation_id: None, + fingerprint_hash: "abc".into(), + timestamp: Utc::now(), + exception_type: "TypeError".into(), + exception_value: Some("boom".into()), + source: Some("custom".into()), + data: Some(json!({"stack_trace": frames})), + trace_id_indexed: Some("4bf92f3577b34da6a3ce929d0e0e4736".into()), + created_at: Utc::now(), + }; + let ev = map_error(model); + let ObservabilityEvent::Error(e) = ev else { + panic!("expected error"); + }; + assert_eq!(e.error_class, "TypeError"); + assert_eq!( + e.trace_id.as_deref(), + Some("4bf92f3577b34da6a3ce929d0e0e4736") + ); + assert!(e.stacktrace_truncated); + assert_eq!(e.stacktrace_preview.as_array().unwrap().len(), 5); + } + + #[test] + fn map_error_handles_missing_stack_trace() { + let model = error_events::Model { + id: 1, + error_group_id: 1, + project_id: 1, + environment_id: None, + deployment_id: None, + visitor_id: None, + ip_geolocation_id: None, + fingerprint_hash: "x".into(), + timestamp: Utc::now(), + exception_type: "Error".into(), + exception_value: None, + source: None, + data: None, + trace_id_indexed: None, + created_at: Utc::now(), + }; + let ev = map_error(model); + let ObservabilityEvent::Error(e) = ev else { + panic!("expected error"); + }; + assert!(!e.stacktrace_truncated); + assert_eq!(e.stacktrace_preview, json!([])); + } + + #[test] + fn map_revenue_carries_amount_and_currency() { + let model = revenue_events::Model { + id: 5, + project_id: 1, + integration_id: 1, + provider: "stripe".into(), + provider_event_id: "evt_1".into(), + event_type: "invoice.paid".into(), + customer_ref: Some("cus_x".into()), + subscription_ref: None, + subscription_status: None, + mrr_minor: None, + amount_minor: Some(4200), + currency: Some("usd".into()), + occurred_at: Utc::now(), + payload: json!({}), + created_at: Utc::now(), + price_id: None, + product_id: None, + deployment_id: Some(3), + environment_id: Some(2), + trace_id: None, + }; + let ev = map_revenue(model); + let ObservabilityEvent::Revenue(r) = ev else { + panic!("expected revenue"); + }; + assert_eq!(r.amount_minor, Some(4200)); + assert_eq!(r.currency.as_deref(), Some("usd")); + assert_eq!(r.deployment_id, Some(3)); + } +} diff --git a/crates/temps-observability/src/types.rs b/crates/temps-observability/src/types.rs new file mode 100644 index 000000000..bf94d19f2 --- /dev/null +++ b/crates/temps-observability/src/types.rs @@ -0,0 +1,424 @@ +//! The wire shape for the unified Observe page. +//! +//! Every row carries its own `type` discriminator and all data the UI needs +//! to render both the row AND its detail panel without a follow-up fetch +//! (the "no extra request" guarantee in the design plan). Heavy fields +//! (stack frames, headers, span attributes, log messages) are truncated +//! server-side with a `truncated` flag so a list page stays under a few +//! hundred KB even when 100 fat error rows show up; the client only fetches +//! the un-truncated form when the user explicitly clicks "Show full". + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +/// Maximum stack frames embedded in `ErrorRow.stacktrace_preview`. +pub const STACKTRACE_PREVIEW_FRAMES: usize = 5; +/// Maximum span attributes embedded in `SpanRow.attributes`. +pub const SPAN_ATTRIBUTE_PREVIEW_KEYS: usize = 20; +/// Header keys whitelisted into the `*_headers` previews. Anything else is +/// dropped from the preview payload but is still available via the +/// un-truncate endpoint. +pub const HEADER_WHITELIST: &[&str] = &[ + "host", + "user-agent", + "referer", + "referrer", + "content-type", + "content-length", + "accept", + "accept-encoding", + "x-forwarded-for", + "cache-control", +]; + +/// Discriminated union of every row that can appear in the Observe list. +/// +/// Serializes to `{ "type": "request" | "span" | ... , ...rest }` so the UI +/// can switch on `event.type` without ambiguity. +/// +/// **No `Log` variant**: runtime stdout/stderr lines live on a dedicated +/// Logs page rather than Observe. Logs are too high-volume to interleave +/// with business signals (requests, errors, revenue) without dominating +/// the timeline, and they have their own retention/storage constraints +/// (TimescaleDB hypertable + chunked file/S3 store) that don't compose +/// with the merge service's per-kind LIMIT strategy. +#[derive(Clone, Debug, Serialize, Deserialize, ToSchema, PartialEq)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ObservabilityEvent { + Request(RequestRow), + Span(SpanRow), + Error(ErrorRow), + Revenue(RevenueRow), +} + +impl ObservabilityEvent { + /// The sort key used by the merge service. All variants must expose + /// `ts` so a k-way merge can interleave them. + pub fn ts(&self) -> DateTime { + match self { + ObservabilityEvent::Request(r) => r.ts, + ObservabilityEvent::Span(r) => r.ts, + ObservabilityEvent::Error(r) => r.ts, + ObservabilityEvent::Revenue(r) => r.ts, + } + } + + /// Stable string discriminator — used by the cursor and by the + /// `/full/{type}/{id}` endpoint route. + pub fn kind(&self) -> EventKind { + match self { + ObservabilityEvent::Request(_) => EventKind::Request, + ObservabilityEvent::Span(_) => EventKind::Span, + ObservabilityEvent::Error(_) => EventKind::Error, + ObservabilityEvent::Revenue(_) => EventKind::Revenue, + } + } +} + +/// Tag enum for filter parameters and routing. Matches the variant +/// discriminator used by `ObservabilityEvent`. +#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema, PartialEq, Eq, Hash)] +#[serde(rename_all = "snake_case")] +pub enum EventKind { + Request, + Span, + Error, + Revenue, +} + +impl EventKind { + pub const ALL: [EventKind; 4] = [ + EventKind::Request, + EventKind::Span, + EventKind::Error, + EventKind::Revenue, + ]; + + pub fn as_str(self) -> &'static str { + match self { + EventKind::Request => "request", + EventKind::Span => "span", + EventKind::Error => "error", + EventKind::Revenue => "revenue", + } + } + + pub fn parse(s: &str) -> Option { + match s { + "request" => Some(EventKind::Request), + "span" => Some(EventKind::Span), + "error" => Some(EventKind::Error), + "revenue" => Some(EventKind::Revenue), + _ => None, + } + } +} + +// ────────────────────────────────────────────────────────────────────────── +// Per-kind row types +// ────────────────────────────────────────────────────────────────────────── + +#[derive(Clone, Debug, Serialize, Deserialize, ToSchema, PartialEq)] +pub struct RequestRow { + pub id: i64, + pub ts: DateTime, + pub deployment_id: Option, + pub environment_id: Option, + pub trace_id: Option, + pub error_group_id: Option, + pub method: String, + pub host: String, + pub path: String, + pub query_string: Option, + pub status: i16, + pub latency_ms: Option, + pub client_ip: Option, + pub country: Option, + pub user_agent: Option, + pub referrer: Option, + pub request_headers: serde_json::Value, + pub response_headers: serde_json::Value, + pub headers_truncated: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize, ToSchema, PartialEq)] +pub struct SpanRow { + pub id: String, + pub ts: DateTime, + pub deployment_id: Option, + pub environment_id: Option, + pub trace_id: String, + pub span_id: String, + pub parent_span_id: Option, + pub service: String, + pub operation: String, + pub duration_ms: Option, + pub status: Option, + pub attributes: serde_json::Value, + pub attributes_truncated: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize, ToSchema, PartialEq)] +pub struct ErrorRow { + pub id: i64, + pub ts: DateTime, + pub deployment_id: Option, + pub environment_id: Option, + pub trace_id: Option, + pub error_group_id: i32, + pub fingerprint: String, + pub error_class: String, + pub message: Option, + pub stacktrace_preview: serde_json::Value, + pub stacktrace_truncated: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize, ToSchema, PartialEq)] +pub struct RevenueRow { + pub id: i64, + pub ts: DateTime, + pub deployment_id: Option, + pub environment_id: Option, + pub trace_id: Option, + pub provider: String, + pub event_type: String, + pub customer_ref: Option, + pub amount_minor: Option, + pub currency: Option, +} + +// ────────────────────────────────────────────────────────────────────────── +// Truncation helpers (tested in isolation) +// ────────────────────────────────────────────────────────────────────────── + +/// Filter a JSON header object down to the whitelist. Accepts any JSON +/// shape the database produces and is forgiving when the value isn't an +/// object (returns an empty object + truncated=false in that case so the +/// UI doesn't crash on legacy rows). +pub fn project_headers(raw: Option<&serde_json::Value>) -> (serde_json::Value, bool) { + let Some(v) = raw else { + return (serde_json::Value::Object(serde_json::Map::new()), false); + }; + let Some(map) = v.as_object() else { + return (serde_json::Value::Object(serde_json::Map::new()), false); + }; + let mut out = serde_json::Map::new(); + let mut dropped = false; + for (k, v) in map.iter() { + if HEADER_WHITELIST.iter().any(|w| w.eq_ignore_ascii_case(k)) { + out.insert(k.clone(), v.clone()); + } else { + dropped = true; + } + } + (serde_json::Value::Object(out), dropped) +} + +/// Keep at most `STACKTRACE_PREVIEW_FRAMES` frames from a JSON array of +/// stack frames, preserving order. Tolerates non-array input. +pub fn truncate_stacktrace(raw: Option<&serde_json::Value>) -> (serde_json::Value, bool) { + let Some(v) = raw else { + return (serde_json::Value::Array(Vec::new()), false); + }; + let Some(arr) = v.as_array() else { + return (serde_json::Value::Array(Vec::new()), false); + }; + if arr.len() <= STACKTRACE_PREVIEW_FRAMES { + return (serde_json::Value::Array(arr.clone()), false); + } + ( + serde_json::Value::Array(arr[..STACKTRACE_PREVIEW_FRAMES].to_vec()), + true, + ) +} + +/// Keep at most `SPAN_ATTRIBUTE_PREVIEW_KEYS` keys from a JSON object, +/// alphabetized so the preview is deterministic across repeated requests. +pub fn truncate_attributes(raw: Option<&serde_json::Value>) -> (serde_json::Value, bool) { + let Some(v) = raw else { + return (serde_json::Value::Object(serde_json::Map::new()), false); + }; + let Some(obj) = v.as_object() else { + return (serde_json::Value::Object(serde_json::Map::new()), false); + }; + if obj.len() <= SPAN_ATTRIBUTE_PREVIEW_KEYS { + return (serde_json::Value::Object(obj.clone()), false); + } + let mut keys: Vec<&String> = obj.keys().collect(); + keys.sort(); + let mut out = serde_json::Map::new(); + for k in keys.into_iter().take(SPAN_ATTRIBUTE_PREVIEW_KEYS) { + out.insert(k.clone(), obj[k].clone()); + } + (serde_json::Value::Object(out), true) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ts() -> DateTime { + DateTime::parse_from_rfc3339("2026-05-01T12:00:00Z") + .unwrap() + .with_timezone(&Utc) + } + + // ── Discriminated union shape ──────────────────────────────────────── + + #[test] + fn request_event_serializes_with_type_tag() { + let ev = ObservabilityEvent::Request(RequestRow { + id: 7, + ts: ts(), + deployment_id: None, + environment_id: None, + trace_id: Some("4bf92f3577b34da6a3ce929d0e0e4736".into()), + error_group_id: None, + method: "GET".into(), + host: "x.test".into(), + path: "/".into(), + query_string: None, + status: 200, + latency_ms: Some(12), + client_ip: None, + country: None, + user_agent: None, + referrer: None, + request_headers: serde_json::json!({}), + response_headers: serde_json::json!({}), + headers_truncated: false, + }); + assert_eq!(serde_json::to_value(&ev).unwrap()["type"], "request"); + } + + #[test] + fn round_trips_through_json() { + let ev = ObservabilityEvent::Revenue(RevenueRow { + id: 1, + ts: ts(), + deployment_id: None, + environment_id: None, + trace_id: None, + provider: "stripe".into(), + event_type: "invoice.paid".into(), + customer_ref: Some("cus_x".into()), + amount_minor: Some(4200), + currency: Some("usd".into()), + }); + let json = serde_json::to_string(&ev).unwrap(); + let back: ObservabilityEvent = serde_json::from_str(&json).unwrap(); + assert_eq!(ev, back); + } + + #[test] + fn ts_accessor_returns_per_variant_timestamp() { + let ev = ObservabilityEvent::Revenue(RevenueRow { + id: 1, + ts: ts(), + deployment_id: None, + environment_id: None, + trace_id: None, + provider: "stripe".into(), + event_type: "invoice.paid".into(), + customer_ref: None, + amount_minor: None, + currency: None, + }); + assert_eq!(ev.ts(), ts()); + assert_eq!(ev.kind(), EventKind::Revenue); + } + + // ── EventKind ──────────────────────────────────────────────────────── + + #[test] + fn event_kind_round_trips() { + for k in EventKind::ALL { + assert_eq!(EventKind::parse(k.as_str()), Some(k)); + } + assert_eq!(EventKind::parse("nope"), None); + } + + // ── Truncation helpers ─────────────────────────────────────────────── + + #[test] + fn project_headers_keeps_only_whitelist() { + let raw = serde_json::json!({ + "host": "x.test", + "user-agent": "curl", + "x-secret": "redacted", + }); + let (out, dropped) = project_headers(Some(&raw)); + assert!(dropped); + let obj = out.as_object().unwrap(); + assert!(obj.contains_key("host")); + assert!(obj.contains_key("user-agent")); + assert!(!obj.contains_key("x-secret")); + } + + #[test] + fn project_headers_returns_empty_for_non_object() { + let raw = serde_json::json!("not an object"); + let (out, dropped) = project_headers(Some(&raw)); + assert_eq!(out, serde_json::json!({})); + assert!(!dropped); + } + + #[test] + fn project_headers_handles_none() { + let (out, dropped) = project_headers(None); + assert_eq!(out, serde_json::json!({})); + assert!(!dropped); + } + + #[test] + fn project_headers_no_drop_when_all_whitelisted() { + let raw = serde_json::json!({"host": "x", "accept": "*/*"}); + let (_, dropped) = project_headers(Some(&raw)); + assert!(!dropped); + } + + #[test] + fn truncate_stacktrace_caps_at_preview_size() { + let frames: Vec = (0..20) + .map(|i| serde_json::json!({"function": format!("fn_{}", i)})) + .collect(); + let raw = serde_json::Value::Array(frames); + let (out, truncated) = truncate_stacktrace(Some(&raw)); + assert!(truncated); + assert_eq!(out.as_array().unwrap().len(), STACKTRACE_PREVIEW_FRAMES); + // Order preserved + assert_eq!(out.as_array().unwrap()[0]["function"], "fn_0"); + } + + #[test] + fn truncate_stacktrace_no_op_when_small() { + let raw = serde_json::json!([{"function": "main"}]); + let (_, truncated) = truncate_stacktrace(Some(&raw)); + assert!(!truncated); + } + + #[test] + fn truncate_attributes_caps_and_alphabetizes() { + let mut obj = serde_json::Map::new(); + for i in 0..30 { + obj.insert(format!("attr_{:02}", i), serde_json::json!(i)); + } + let raw = serde_json::Value::Object(obj); + let (out, truncated) = truncate_attributes(Some(&raw)); + assert!(truncated); + let obj = out.as_object().unwrap(); + assert_eq!(obj.len(), SPAN_ATTRIBUTE_PREVIEW_KEYS); + // Alphabetized — first key should be attr_00, last should be attr_19 + let mut keys: Vec<&String> = obj.keys().collect(); + keys.sort(); + assert_eq!(keys.first().unwrap().as_str(), "attr_00"); + } + + #[test] + fn truncate_attributes_no_op_when_small() { + let raw = serde_json::json!({"k": "v"}); + let (_, truncated) = truncate_attributes(Some(&raw)); + assert!(!truncated); + } +} diff --git a/crates/temps-observability/tests/merge_integration.rs b/crates/temps-observability/tests/merge_integration.rs new file mode 100644 index 000000000..f1aed2a6f --- /dev/null +++ b/crates/temps-observability/tests/merge_integration.rs @@ -0,0 +1,548 @@ +//! Integration test: write rows of multiple kinds and verify the merge +//! service interleaves them correctly with kind filters and time bounds. +//! +//! Mirrors the project's existing integration-test pattern (TestDatabase +//! with migrations, scoped per-test schema). + +use std::sync::Arc; + +use chrono::{Duration, TimeZone, Utc}; +use sea_orm::{ActiveModelTrait, DatabaseConnection, Set}; +use temps_database::test_utils::TestDatabase; +use sea_orm::{ConnectionTrait, DatabaseBackend, Statement}; +use temps_entities::{error_events, error_groups, projects, proxy_logs, revenue_events}; +use temps_observability::{ + service::FullEvent, EventFilters, EventKind, ObservabilityEvent, ObservabilityError, + ObservabilityService, +}; +use uuid::Uuid; + +async fn create_test_project(db: &Arc) -> i32 { + use temps_entities::preset::Preset; + let slug = format!("obs-{}", Uuid::new_v4()); + let p = projects::ActiveModel { + name: Set("Obs Test".into()), + repo_name: Set("r".into()), + repo_owner: Set("o".into()), + directory: Set("/".into()), + main_branch: Set("main".into()), + slug: Set(slug), + preset: Set(Preset::NextJs), + created_at: Set(Utc::now()), + updated_at: Set(Utc::now()), + ..Default::default() + }; + p.insert(db.as_ref()).await.expect("insert project").id +} + +async fn create_error_group(db: &Arc, project_id: i32) -> i32 { + let g = error_groups::ActiveModel { + project_id: Set(project_id), + title: Set("TypeError: x".into()), + error_type: Set("TypeError".into()), + first_seen: Set(Utc::now()), + last_seen: Set(Utc::now()), + total_count: Set(1), + status: Set("unresolved".into()), + created_at: Set(Utc::now()), + updated_at: Set(Utc::now()), + ..Default::default() + }; + g.insert(db.as_ref()).await.expect("insert group").id +} + +#[tokio::test] +async fn merge_returns_rows_in_descending_ts_across_kinds() { + let test_db = TestDatabase::with_migrations() + .await + .expect("test db with migrations"); + let db = test_db.connection_arc(); + + let project_id = create_test_project(&db).await; + let group_id = create_error_group(&db, project_id).await; + + // Three events at descending timestamps, one per kind. + let t_request = Utc.with_ymd_and_hms(2026, 5, 1, 12, 3, 0).unwrap(); + let t_error = Utc.with_ymd_and_hms(2026, 5, 1, 12, 2, 0).unwrap(); + let t_revenue = Utc.with_ymd_and_hms(2026, 5, 1, 12, 1, 0).unwrap(); + + proxy_logs::ActiveModel { + timestamp: Set(t_request), + method: Set("GET".into()), + path: Set("/api".into()), + host: Set("x.test".into()), + status_code: Set(200), + response_time_ms: Set(Some(12)), + request_source: Set("proxy".into()), + is_system_request: Set(false), + routing_status: Set("routed".into()), + project_id: Set(Some(project_id)), + request_id: Set("req-1".into()), + created_date: Set(t_request.date_naive()), + ..Default::default() + } + .insert(db.as_ref()) + .await + .expect("insert proxy log"); + + error_events::ActiveModel { + error_group_id: Set(group_id), + project_id: Set(project_id), + fingerprint_hash: Set("abc".into()), + timestamp: Set(t_error), + exception_type: Set("TypeError".into()), + exception_value: Set(Some("boom".into())), + source: Set(Some("custom".into())), + created_at: Set(Utc::now()), + ..Default::default() + } + .insert(db.as_ref()) + .await + .expect("insert error event"); + + revenue_events::ActiveModel { + project_id: Set(project_id), + integration_id: Set(0), + provider: Set("stripe".into()), + provider_event_id: Set(format!("evt_{}", Uuid::new_v4())), + event_type: Set("invoice.paid".into()), + amount_minor: Set(Some(4200)), + currency: Set(Some("usd".into())), + occurred_at: Set(t_revenue), + payload: Set(serde_json::json!({})), + created_at: Set(Utc::now()), + ..Default::default() + } + .insert(db.as_ref()) + .await + .expect("insert revenue event"); + + let service = ObservabilityService::new(db.clone()); + let events = service + .query(EventFilters { + project_id, + kinds: EventKind::ALL.iter().copied().collect(), + from: None, + to: None, + deployment_id: None, + environment_id: None, + search: None, + limit: 50, + }) + .await + .expect("query"); + + assert_eq!(events.len(), 3); + // Strictly descending by ts + assert!(events[0].ts() >= events[1].ts()); + assert!(events[1].ts() >= events[2].ts()); + // Latest is the request, oldest is revenue + assert_eq!(events[0].kind(), EventKind::Request); + assert_eq!(events[2].kind(), EventKind::Revenue); +} + +#[tokio::test] +async fn kinds_filter_excludes_unselected_tables() { + let test_db = TestDatabase::with_migrations().await.expect("test db"); + let db = test_db.connection_arc(); + let project_id = create_test_project(&db).await; + let group_id = create_error_group(&db, project_id).await; + + let now = Utc::now(); + proxy_logs::ActiveModel { + timestamp: Set(now), + method: Set("GET".into()), + path: Set("/x".into()), + host: Set("h".into()), + status_code: Set(200), + request_source: Set("proxy".into()), + is_system_request: Set(false), + routing_status: Set("routed".into()), + project_id: Set(Some(project_id)), + request_id: Set("r".into()), + created_date: Set(now.date_naive()), + ..Default::default() + } + .insert(db.as_ref()) + .await + .expect("insert"); + error_events::ActiveModel { + error_group_id: Set(group_id), + project_id: Set(project_id), + fingerprint_hash: Set("fp".into()), + timestamp: Set(now), + exception_type: Set("Boom".into()), + source: Set(Some("custom".into())), + created_at: Set(Utc::now()), + ..Default::default() + } + .insert(db.as_ref()) + .await + .expect("insert error"); + + let service = ObservabilityService::new(db.clone()); + + let only_errors = service + .query(EventFilters { + project_id, + kinds: [EventKind::Error].iter().copied().collect(), + from: None, + to: None, + deployment_id: None, + environment_id: None, + search: None, + limit: 50, + }) + .await + .expect("query errors only"); + + assert!(only_errors.iter().all(|e| e.kind() == EventKind::Error)); +} + +#[tokio::test] +async fn time_range_filters_old_rows() { + let test_db = TestDatabase::with_migrations().await.expect("test db"); + let db = test_db.connection_arc(); + let project_id = create_test_project(&db).await; + + let new = Utc.with_ymd_and_hms(2026, 5, 1, 12, 0, 0).unwrap(); + let old = new - Duration::days(30); + + for (ts, req_id) in [(new, "new"), (old, "old")] { + proxy_logs::ActiveModel { + timestamp: Set(ts), + method: Set("GET".into()), + path: Set(format!("/{}", req_id)), + host: Set("h".into()), + status_code: Set(200), + request_source: Set("proxy".into()), + is_system_request: Set(false), + routing_status: Set("routed".into()), + project_id: Set(Some(project_id)), + request_id: Set(req_id.into()), + created_date: Set(ts.date_naive()), + ..Default::default() + } + .insert(db.as_ref()) + .await + .expect("insert"); + } + + let service = ObservabilityService::new(db.clone()); + let recent = service + .query(EventFilters { + project_id, + kinds: [EventKind::Request].iter().copied().collect(), + from: Some(new - Duration::hours(1)), + to: Some(new + Duration::hours(1)), + deployment_id: None, + environment_id: None, + search: None, + limit: 50, + }) + .await + .expect("query"); + + assert_eq!(recent.len(), 1, "old row should be filtered out"); + let ObservabilityEvent::Request(r) = &recent[0] else { + panic!("expected request"); + }; + assert_eq!(r.path, "/new"); +} + +#[tokio::test] +async fn fetch_full_returns_untruncated_request() { + let test_db = TestDatabase::with_migrations().await.expect("test db"); + let db = test_db.connection_arc(); + let project_id = create_test_project(&db).await; + + let now = Utc::now(); + let inserted = proxy_logs::ActiveModel { + timestamp: Set(now), + method: Set("POST".into()), + path: Set("/checkout".into()), + host: Set("h".into()), + status_code: Set(500), + request_source: Set("proxy".into()), + is_system_request: Set(false), + routing_status: Set("routed".into()), + project_id: Set(Some(project_id)), + request_id: Set("rid".into()), + // Headers larger than the whitelist would survive truncation + request_headers: Set(Some(serde_json::json!({ + "host": "h", + "x-secret-1": "a", + "x-secret-2": "b", + }))), + created_date: Set(now.date_naive()), + ..Default::default() + } + .insert(db.as_ref()) + .await + .expect("insert"); + + let service = ObservabilityService::new(db.clone()); + let full = service + .fetch_full(project_id, EventKind::Request, &inserted.id.to_string()) + .await + .expect("fetch full"); + + let FullEvent::Request(req) = full else { + panic!("expected Request"); + }; + // Full payload preserves the un-whitelisted headers — proves we returned + // the un-truncated form. + let headers = req.request_headers.expect("headers"); + let obj = headers.as_object().unwrap(); + assert!(obj.contains_key("x-secret-1")); + assert!(obj.contains_key("x-secret-2")); +} + +#[tokio::test] +async fn fetch_full_404s_for_unknown_id() { + let test_db = TestDatabase::with_migrations().await.expect("test db"); + let db = test_db.connection_arc(); + let project_id = create_test_project(&db).await; + let service = ObservabilityService::new(db.clone()); + + let err = service + .fetch_full(project_id, EventKind::Request, "999999") + .await + .unwrap_err(); + assert!(matches!(err, ObservabilityError::EventNotFound { .. })); +} + +#[tokio::test] +async fn fetch_full_404s_for_wrong_project() { + let test_db = TestDatabase::with_migrations().await.expect("test db"); + let db = test_db.connection_arc(); + let project_a = create_test_project(&db).await; + let project_b = create_test_project(&db).await; + + let now = Utc::now(); + let inserted = proxy_logs::ActiveModel { + timestamp: Set(now), + method: Set("GET".into()), + path: Set("/x".into()), + host: Set("h".into()), + status_code: Set(200), + request_source: Set("proxy".into()), + is_system_request: Set(false), + routing_status: Set("routed".into()), + project_id: Set(Some(project_a)), + request_id: Set("r".into()), + created_date: Set(now.date_naive()), + ..Default::default() + } + .insert(db.as_ref()) + .await + .expect("insert"); + + let service = ObservabilityService::new(db.clone()); + // Same id, but wrong project — must not leak across project boundaries. + let err = service + .fetch_full(project_b, EventKind::Request, &inserted.id.to_string()) + .await + .unwrap_err(); + assert!(matches!(err, ObservabilityError::EventNotFound { .. })); +} + +#[tokio::test] +async fn fetch_full_rejects_malformed_id() { + let test_db = TestDatabase::with_migrations().await.expect("test db"); + let db = test_db.connection_arc(); + let service = ObservabilityService::new(db.clone()); + + let err = service + .fetch_full(1, EventKind::Request, "not-an-int") + .await + .unwrap_err(); + assert!(matches!(err, ObservabilityError::EventNotFound { .. })); +} + +#[tokio::test] +async fn fetch_spans_returns_span_rows_from_otel_table() { + let test_db = TestDatabase::with_migrations() + .await + .expect("test db with migrations"); + let db = test_db.connection_arc(); + let project_id = create_test_project(&db).await; + + // otel_spans has no Sea-ORM entity — insert raw via SQL, then query + // through the merge service. + let t = Utc.with_ymd_and_hms(2026, 5, 1, 12, 6, 0).unwrap(); + let trace_id = "deadbeefdeadbeefdeadbeefdeadbeef"; + let span_id = "1111111111111111"; + db.execute(Statement::from_sql_and_values( + DatabaseBackend::Postgres, + "INSERT INTO otel_spans \ + (project_id, deployment_id, service_name, trace_id, span_id, parent_span_id, \ + name, kind, start_time, end_time, duration_ms, status_code, status_message, \ + attributes, events) \ + VALUES ($1, NULL, $2, $3, $4, NULL, $5, 'SERVER', $6, $7, $8, 'OK', '', \ + '{\"http.method\":\"GET\"}'::jsonb, '[]'::jsonb)", + vec![ + project_id.into(), + "checkout".into(), + trace_id.into(), + span_id.into(), + "GET /checkout".into(), + t.into(), + (t + Duration::milliseconds(42)).into(), + 42.0_f64.into(), + ], + )) + .await + .expect("insert otel_spans"); + + let service = ObservabilityService::new(db.clone()); + let events = service + .query(EventFilters { + project_id, + kinds: [EventKind::Span].iter().copied().collect(), + from: None, + to: None, + deployment_id: None, + environment_id: None, + search: None, + limit: 50, + }) + .await + .expect("query spans"); + + assert_eq!(events.len(), 1); + let ObservabilityEvent::Span(row) = &events[0] else { + panic!("expected span row"); + }; + assert_eq!(row.service, "checkout"); + assert_eq!(row.operation, "GET /checkout"); + assert_eq!(row.trace_id, trace_id); + assert_eq!(row.span_id, span_id); + assert_eq!(row.duration_ms, Some(42.0)); + assert_eq!(row.status.as_deref(), Some("OK")); + assert_eq!( + row.attributes.get("http.method").and_then(|v| v.as_str()), + Some("GET"), + ); +} + +#[tokio::test] +async fn merge_interleaves_spans_with_other_kinds() { + let test_db = TestDatabase::with_migrations() + .await + .expect("test db with migrations"); + let db = test_db.connection_arc(); + let project_id = create_test_project(&db).await; + let group_id = create_error_group(&db, project_id).await; + + // Four events at strictly descending timestamps, one per kind, so the + // k-way merge is forced to interleave across all four sources. + let t_request = Utc.with_ymd_and_hms(2026, 5, 1, 12, 5, 0).unwrap(); + let t_span = Utc.with_ymd_and_hms(2026, 5, 1, 12, 3, 0).unwrap(); + let t_error = Utc.with_ymd_and_hms(2026, 5, 1, 12, 2, 0).unwrap(); + let t_revenue = Utc.with_ymd_and_hms(2026, 5, 1, 12, 1, 0).unwrap(); + + proxy_logs::ActiveModel { + timestamp: Set(t_request), + method: Set("GET".into()), + path: Set("/all".into()), + host: Set("x.test".into()), + status_code: Set(200), + request_source: Set("proxy".into()), + is_system_request: Set(false), + routing_status: Set("routed".into()), + project_id: Set(Some(project_id)), + request_id: Set("req-all".into()), + created_date: Set(t_request.date_naive()), + ..Default::default() + } + .insert(db.as_ref()) + .await + .expect("insert proxy log"); + + db.execute(Statement::from_sql_and_values( + DatabaseBackend::Postgres, + "INSERT INTO otel_spans \ + (project_id, deployment_id, service_name, trace_id, span_id, parent_span_id, \ + name, kind, start_time, end_time, duration_ms, status_code, status_message, \ + attributes, events) \ + VALUES ($1, NULL, 'web', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', \ + '2222222222222222', NULL, 'GET /', 'SERVER', $2, $3, 12.0, \ + 'OK', '', '{}'::jsonb, '[]'::jsonb)", + vec![ + project_id.into(), + t_span.into(), + (t_span + Duration::milliseconds(12)).into(), + ], + )) + .await + .expect("insert span"); + + error_events::ActiveModel { + error_group_id: Set(group_id), + project_id: Set(project_id), + fingerprint_hash: Set("merge-fp".into()), + timestamp: Set(t_error), + exception_type: Set("Boom".into()), + source: Set(Some("custom".into())), + created_at: Set(Utc::now()), + ..Default::default() + } + .insert(db.as_ref()) + .await + .expect("insert error"); + + revenue_events::ActiveModel { + project_id: Set(project_id), + integration_id: Set(0), + provider: Set("stripe".into()), + provider_event_id: Set(format!("evt_{}", Uuid::new_v4())), + event_type: Set("invoice.paid".into()), + amount_minor: Set(Some(1000)), + currency: Set(Some("usd".into())), + occurred_at: Set(t_revenue), + payload: Set(serde_json::json!({})), + created_at: Set(Utc::now()), + ..Default::default() + } + .insert(db.as_ref()) + .await + .expect("insert revenue"); + + let service = ObservabilityService::new(db.clone()); + let events = service + .query(EventFilters { + project_id, + kinds: EventKind::ALL.iter().copied().collect(), + from: None, + to: None, + deployment_id: None, + environment_id: None, + search: None, + limit: 50, + }) + .await + .expect("merge query"); + + assert_eq!(events.len(), 4); + // Strictly descending order — proves the k-way merge picked the right + // head from each of the four streams. + let kinds: Vec = events.iter().map(|e| e.kind()).collect(); + assert_eq!( + kinds, + vec![ + EventKind::Request, + EventKind::Span, + EventKind::Error, + EventKind::Revenue, + ] + ); + for w in events.windows(2) { + assert!( + w[0].ts() >= w[1].ts(), + "merge order broke between {:?} and {:?}", + w[0].kind(), + w[1].kind() + ); + } +} diff --git a/crates/temps-proxy/src/proxy.rs b/crates/temps-proxy/src/proxy.rs index 604d871ac..2e68fcec0 100644 --- a/crates/temps-proxy/src/proxy.rs +++ b/crates/temps-proxy/src/proxy.rs @@ -472,6 +472,36 @@ impl LoadBalancer { &self.session_manager } + /// Pull the W3C `traceparent` trace_id (the 32-hex-char `` field) + /// from a request header map. Returns `None` when the header is missing, + /// malformed, or carries the all-zero invalid trace_id reserved by the + /// spec. Stamped onto `proxy_logs.trace_id` so the unified Observe view + /// can join the request row to its child spans, runtime logs, and any + /// captured exceptions. + fn extract_traceparent_trace_id( + headers: Option<&std::collections::HashMap>, + ) -> Option { + let headers = headers?; + let raw = headers + .get("traceparent") + .or_else(|| headers.get("Traceparent")) + .or_else(|| headers.get("TRACEPARENT"))?; + + // traceparent: "---" + let mut parts = raw.split('-'); + let _version = parts.next()?; + let trace_id = parts.next()?; + + if trace_id.len() != 32 + || !trace_id.chars().all(|c| c.is_ascii_hexdigit()) + || trace_id.chars().all(|c| c == '0') + { + return None; + } + + Some(trace_id.to_ascii_lowercase()) + } + /// Check if a request should be logged to proxy_logs based on path fn should_log_request(path: &str) -> bool { if LOG_STATIC_ASSETS { @@ -1145,6 +1175,8 @@ impl LoadBalancer { .response_headers .as_ref() .and_then(|h| serde_json::to_value(h).ok()), + trace_id: Self::extract_traceparent_trace_id(ctx.request_headers.as_ref()), + error_group_id: None, }; // Send to batch writer with backpressure (blocks briefly if buffer full) @@ -1265,6 +1297,8 @@ impl LoadBalancer { .as_ref() .and_then(|h| serde_json::to_value(h).ok()), response_headers: None, + trace_id: Self::extract_traceparent_trace_id(ctx.request_headers.as_ref()), + error_group_id: None, }; // Send to batch writer (non-blocking, drops if buffer full) @@ -3935,6 +3969,8 @@ impl ProxyHttp for LoadBalancer { .response_headers .as_ref() .and_then(|h| serde_json::to_value(h).ok()), + trace_id: Self::extract_traceparent_trace_id(ctx.request_headers.as_ref()), + error_group_id: None, }; // Send to batch writer (non-blocking, drops if buffer full) @@ -5032,3 +5068,70 @@ mod markdown_pipeline_tests { ); } } + +#[cfg(test)] +mod traceparent_tests { + use super::*; + use std::collections::HashMap; + + fn headers_with(name: &str, value: &str) -> HashMap { + let mut h = HashMap::new(); + h.insert(name.to_string(), value.to_string()); + h + } + + #[test] + fn extracts_valid_traceparent_trace_id() { + let h = headers_with( + "traceparent", + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", + ); + assert_eq!( + LoadBalancer::extract_traceparent_trace_id(Some(&h)), + Some("4bf92f3577b34da6a3ce929d0e0e4736".to_string()) + ); + } + + #[test] + fn returns_none_when_header_absent() { + let h: HashMap = HashMap::new(); + assert_eq!(LoadBalancer::extract_traceparent_trace_id(Some(&h)), None); + assert_eq!(LoadBalancer::extract_traceparent_trace_id(None), None); + } + + #[test] + fn returns_none_for_all_zero_trace_id() { + let h = headers_with( + "traceparent", + "00-00000000000000000000000000000000-00f067aa0ba902b7-01", + ); + assert_eq!(LoadBalancer::extract_traceparent_trace_id(Some(&h)), None); + } + + #[test] + fn returns_none_for_wrong_length() { + let h = headers_with("traceparent", "00-deadbeef-00f067aa0ba902b7-01"); + assert_eq!(LoadBalancer::extract_traceparent_trace_id(Some(&h)), None); + } + + #[test] + fn returns_none_for_non_hex_chars() { + let h = headers_with( + "traceparent", + "00-zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz-00f067aa0ba902b7-01", + ); + assert_eq!(LoadBalancer::extract_traceparent_trace_id(Some(&h)), None); + } + + #[test] + fn lowercases_uppercase_hex() { + let h = headers_with( + "traceparent", + "00-4BF92F3577B34DA6A3CE929D0E0E4736-00f067aa0ba902b7-01", + ); + assert_eq!( + LoadBalancer::extract_traceparent_trace_id(Some(&h)), + Some("4bf92f3577b34da6a3ce929d0e0e4736".to_string()) + ); + } +} diff --git a/crates/temps-proxy/src/service/proxy_log_batch_writer.rs b/crates/temps-proxy/src/service/proxy_log_batch_writer.rs index f06325a7a..d9da8c10c 100644 --- a/crates/temps-proxy/src/service/proxy_log_batch_writer.rs +++ b/crates/temps-proxy/src/service/proxy_log_batch_writer.rs @@ -355,6 +355,8 @@ mod tests { cache_status: None, request_headers: None, response_headers: None, + trace_id: None, + error_group_id: None, }; // Send should succeed @@ -406,6 +408,8 @@ mod tests { cache_status: None, request_headers: None, response_headers: None, + trace_id: None, + error_group_id: None, }; // try_send should succeed when channel has capacity @@ -455,6 +459,8 @@ mod tests { cache_status: None, request_headers: None, response_headers: None, + trace_id: None, + error_group_id: None, }; // Send should fail because the writer (receiver) is dropped @@ -504,6 +510,8 @@ mod tests { cache_status: None, request_headers: None, response_headers: None, + trace_id: None, + error_group_id: None, }; writer.enrich_entry(&mut entry).await; @@ -557,6 +565,8 @@ mod tests { cache_status: None, request_headers: None, response_headers: None, + trace_id: None, + error_group_id: None, }; writer.enrich_entry(&mut entry).await; diff --git a/crates/temps-proxy/src/service/proxy_log_service.rs b/crates/temps-proxy/src/service/proxy_log_service.rs index 4086a2478..a1613694b 100644 --- a/crates/temps-proxy/src/service/proxy_log_service.rs +++ b/crates/temps-proxy/src/service/proxy_log_service.rs @@ -130,6 +130,15 @@ pub struct CreateProxyLogRequest { pub cache_status: Option, pub request_headers: Option, pub response_headers: Option, + /// W3C `traceparent` trace_id (32 hex chars) extracted from inbound + /// headers. `None` when the client didn't send `traceparent`. Lets the + /// unified Observe view join this row with child OTel spans, runtime + /// log lines, and any captured exceptions for the same trace. + pub trace_id: Option, + /// Set by the deployment runtime (or async stamping) when this request + /// produced a captured exception. Lets the Observe row deep-link to the + /// error group without an extra query. + pub error_group_id: Option, } pub struct ProxyLogService { @@ -223,6 +232,8 @@ impl ProxyLogService { request_headers: Set(request.request_headers), response_headers: Set(request.response_headers), created_date: Set(created_date), + trace_id: Set(request.trace_id), + error_group_id: Set(request.error_group_id), ..Default::default() }; diff --git a/crates/temps-proxy/src/services.rs b/crates/temps-proxy/src/services.rs index 7162dfa97..9c9c5fe2f 100644 --- a/crates/temps-proxy/src/services.rs +++ b/crates/temps-proxy/src/services.rs @@ -319,6 +319,8 @@ impl RequestLogger for RequestLoggerImpl { created_date: Set(data.started_at.date_naive()), session_id: Set(data.session.as_ref().map(|s| s.session_id_i32)), visitor_id: Set(data.visitor.as_ref().map(|v| v.visitor_id_i32)), + trace_id: Set(data.trace_id), + error_group_id: Set(None), ..Default::default() }; @@ -960,6 +962,7 @@ mod tests { visitor: None, session: None, project_context: Some(context.clone()), + trace_id: None, }; logger.log_request(log_data).await.unwrap(); @@ -1008,6 +1011,7 @@ mod tests { visitor: None, session: None, project_context: Some(context), + trace_id: None, }; logger.log_request(log_data).await.unwrap(); @@ -1054,6 +1058,7 @@ mod tests { visitor: None, session: None, project_context: Some(context), + trace_id: None, }; logger.log_request(log_data).await.unwrap(); @@ -1101,6 +1106,7 @@ mod tests { visitor: None, session: None, project_context: Some(context), + trace_id: None, }; logger.log_request(log_data).await.unwrap(); @@ -1193,6 +1199,7 @@ mod tests { visitor: Some(visitor_data), session: Some(session_data), project_context: Some(context), + trace_id: None, }; logger.log_request(log_data).await.unwrap(); diff --git a/crates/temps-proxy/src/traits.rs b/crates/temps-proxy/src/traits.rs index ab107da52..d71acd8c6 100644 --- a/crates/temps-proxy/src/traits.rs +++ b/crates/temps-proxy/src/traits.rs @@ -49,6 +49,11 @@ pub struct RequestLogData { pub visitor: Option, pub session: Option, pub project_context: Option, + /// W3C `traceparent` trace_id (32 hex chars) extracted from the inbound + /// request headers when present. Stamped onto `proxy_logs.trace_id` so + /// the unified Observe view can correlate this request with its child + /// spans, runtime logs, and any captured exceptions. + pub trace_id: Option, } /// Cookie configuration for visitor/session tracking From 52b1ba92683e26ab539671c5d593292e68665d24 Mon Sep 17 00:00:00 2001 From: David Viejo Date: Fri, 1 May 2026 23:28:22 +0200 Subject: [PATCH 02/12] =?UTF-8?q?feat(web):=20Observe=20page=20=E2=80=94?= =?UTF-8?q?=20cockpit=20sparklines=20+=20console=20stream=20+=20side=20pan?= =?UTF-8?q?el?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `/projects/:slug/observe` as the unified observability surface for a project. Two-tier layout: 1. Cockpit header with one mini sparkline per kind (Requests, Traces, Errors, Revenue) over the selected time range. Each card is clickable to toggle that kind on/off. Traces default OFF because `otel_spans` is a high-volume hypertable; the user opts in via the cockpit card (dashed border + `'off'` label). 2. Console stream — terminal-style monospace rows with a color-coded gutter per kind (sky=requests, violet=traces, rose=errors, emerald=revenue), `HH:mm:ss` timestamps, and one-line summaries. High density, scannable. Side panel renders entirely from the row payload — no follow-up fetch in the common case. URL-state filters so the page is shareable. Sidebar gets an Observe entry; legacy detail routes (`traces/:id`, `errors/:groupId`) still work as panel deep-links. - New `web/src/pages/Observe.tsx` with cockpit + console + bucketing + sparkline + truncation-aware row renderer - New `web/src/components/observe/{types,useObserveQuery, ObserveFilterBar,rows/{ObserveRow,RowParts},panels/ObservePanel}` - Regenerated SDK from the new `temps-observability` OpenAPI surface - ProjectDetail mounts `/observe/*` and Sidebar surfaces "Observe" --- .../api/client/@tanstack/react-query.gen.ts | 51 +- web/src/api/client/sdk.gen.ts | 41 +- web/src/api/client/types.gen.ts | 272 +++++++++ web/src/components/dashboard/Sidebar.tsx | 6 +- .../components/observe/ObserveFilterBar.tsx | 151 +++++ .../observe/panels/ObservePanel.tsx | 343 +++++++++++ .../components/observe/rows/ObserveRow.tsx | 109 ++++ web/src/components/observe/rows/RowParts.tsx | 95 +++ web/src/components/observe/types.ts | 27 + web/src/components/observe/useObserveQuery.ts | 56 ++ web/src/index.tsx | 8 - web/src/pages/Observe.tsx | 559 ++++++++++++++++++ web/src/pages/ProjectDetail.tsx | 5 + 13 files changed, 1710 insertions(+), 13 deletions(-) create mode 100644 web/src/components/observe/ObserveFilterBar.tsx create mode 100644 web/src/components/observe/panels/ObservePanel.tsx create mode 100644 web/src/components/observe/rows/ObserveRow.tsx create mode 100644 web/src/components/observe/rows/RowParts.tsx create mode 100644 web/src/components/observe/types.ts create mode 100644 web/src/components/observe/useObserveQuery.ts create mode 100644 web/src/pages/Observe.tsx diff --git a/web/src/api/client/@tanstack/react-query.gen.ts b/web/src/api/client/@tanstack/react-query.gen.ts index 791e2db0b..6f1168007 100644 --- a/web/src/api/client/@tanstack/react-query.gen.ts +++ b/web/src/api/client/@tanstack/react-query.gen.ts @@ -1,8 +1,8 @@ // This file is auto-generated by @hey-api/openapi-ts -import { type Options, getPlatformInfo, chunkUploadOptions, createRelease, createProjectRelease, finalizeProjectRelease, listReleaseFiles, uploadReleaseFile, recordEventMetrics, addSessionReplayEvents, initSessionReplay, recordSpeedMetrics, updateSpeedMetrics, webhookTrigger, getPricing, listProviderKeys, createProviderKey, testProviderKeyInline, deleteProviderKey, updateProviderKey, testProviderKeyById, getUsageByProvider, getConversations, getConversationDetail, getUsageRecent, getUsageSummary, getUsageTimeseries, getUsageTopModels, chatCompletions, embeddings, listModels, getAnalyticsActiveVisitors, getEventDetail, getEventVisitors, getAnalyticsEventsCount, getGeneralStats, getLiveVisitorsList, getPageFlow, getPageHourlySessions, getPagePathDetail, getPagePathVisitors, getPagePaths, getPagePathsSparklines, getRecentActivity, getSessionDetails, getAnalyticsSessionEvents, getSessionLogs, getVisitors, getVisitorByGuid, getVisitorById, getVisitorDetails, enrichVisitor, getVisitorInfo, getVisitorJourney, getAnalyticsVisitorSessions, getVisitorStats, listApiKeys, createApiKey, getApiKeyPermissions, deleteApiKey, getApiKey, updateApiKey, activateApiKey, deactivateApiKey, cliLogin, cliLogout, emailStatus, login, requestMagicLink, verifyMagicLink, requestPasswordReset, resetPassword, verifyEmail, verifyMfaChallenge, runExternalServiceBackup, listS3Sources, createS3Source, testS3ConnectionPreview, deleteS3Source, getS3Source, updateS3Source, listSourceBackups, runBackupForSource, setDefaultS3Source, testS3SourceConnection, listBackupSchedules, createBackupSchedule, deleteBackupSchedule, getBackupSchedule, listBackupsForSchedule, disableBackupSchedule, enableBackupSchedule, getBackup, blobDelete, blobList, blobPut, blobCopy, blobDisable, blobEnable, blobStatus, blobUpdate, blobDownload, getDashboardProjectsAnalytics, getActivityGraph, getScanByDeployment, listDnsProviders, createDnsProvider, deleteDnsProvider, getDnsProvider, updateProvider, listManagedDomains, addManagedDomain, testProviderConnection, listProviderZones, removeManagedDomain, verifyManagedDomain, lookupDnsARecords, listDomains, createDomain, getDomainByHost, cancelDomainOrder, getDomainOrder, createOrRecreateOrder, finalizeOrder, setupDnsChallenge, deleteDomain, getDomainById, getChallengeToken, getHttpChallengeDebug, provisionDomain, renewDomain, checkDomainStatus, listEmailDomains, createEmailDomain, getDomainByName, deleteEmailDomain, getDomain, getDomainDnsRecords, setupDns, verifyDomain, listEmailProviders, createEmailProvider, deleteEmailProvider, getEmailProvider, testProvider, listEmails, sendEmail, getGlobalEvents, getGlobalEventStats, getEmailStats, validateEmail, trackClick, trackOpen, getEmail, getEmailTracking, getEmailEvents, getEmailLinks, listServices, createService, listAvailableContainers, getServiceBySlug, listServiceHealthStatuses, importExternalService, listProjectServices, getProjectServiceEnvironmentVariables, getProvidersMetadata, getProviderMetadata, getServiceTypes, getServiceTypeParameters, deleteService, getService, updateService, getClusterHealth, triggerServiceHealthCheck, getServiceHealthStatus, addClusterMember, removeClusterMember, getClusterMember, promoteClusterMember, getServicePreviewEnvironmentVariablesMasked, getServicePreviewEnvironmentVariableNames, listServiceProjects, linkServiceToProject, unlinkServiceFromProject, getServiceEnvironmentVariables, getServiceEnvironmentVariable, startRestore, getRestoreCapabilities, planRestore, listRestoreRunsForService, retryCluster, startService, stopService, upgradeService, listRootContainers, listContainersAtPath, listEntities, getEntityInfo, queryData, downloadObject, getContainerInfo, checkExplorerSupport, listPgUpgrades, startPgUpgrade, getPgUpgrade, cancelPgUpgrade, getPgUpgradeLogs, retryPgUpgrade, rollbackPgUpgrade, getFile, getIpGeolocation, listConnections, deleteConnection, activateConnection, deactivateConnection, runConnectionHealthCheck, listRepositoriesByConnection, syncRepositories, updateConnectionToken, validateConnection, listGitProviders, createGitProvider, createGithubPatProvider, createGitlabOauthProvider, createGitlabPatProvider, deleteGitProvider, getGitProvider, activateProvider, handleGitProviderOauthCallback, getProviderConnections, updateGitProviderCredentials, deactivateProvider, checkProviderDeletionSafety, startGitProviderOauth, deleteProviderSafely, getPublicRepository, getPublicBranches, detectPublicPresets, discoverWorkloads, executeImport, createPlan, listSources, getImportStatus, getIncident, updateIncidentStatus, getIncidentUpdates, adminListNodes, registerNode, adminRemoveNode, adminGetNode, adminListNodeContainers, postDnsAck, getDnsChanges, adminUndrainNode, adminDrainStatus, adminDrainNode, nodeHeartbeat, listPeers, getS3Credentials, listIpAccessControl, createIpAccessControl, checkIpBlocked, deleteIpAccessControl, getIpAccessControl, updateIpAccessControl, kvDel, kvDisable, kvEnable, kvExpire, kvGet, kvIncr, kvKeys, kvSet, kvStatus, kvTtl, kvUpdate, listRoutes, createRoute, deleteRoute, getRoute, updateRoute, logout, getLogContext, searchLogs, tailLogs, getProjectsMonitorHealth, deleteMonitor, getMonitor, getBucketedStatus, getCurrentMonitorStatus, getUptimeHistory, deletePreferences, getPreferences, updatePreferences, listNotificationProviders, createNotificationProvider, createNotificationEmailProvider, updateEmailProvider, createSlackProvider, updateSlackProvider, createWebhookProvider, updateWebhookProvider, deleteNotificationProvider, getNotificationProvider, updateNotificationProvider, testNotificationProvider, listOrders, queryGenaiTraces, getGenaiTrace, getHealth, listInsights, queryLogs, listMetricNames, queryMetrics, getPipelineStats, getQuota, queryTraceSummaries, queryTraces, getTrace, ingestLogs, ingestMetrics, ingestTraces, ingestLogsByPath, ingestMetricsByPath, ingestTracesByPath, hasPerformanceMetrics, getPerformanceMetrics, getMetricsOverTime, getGroupedPageMetrics, getAccessInfo, getPrivateIp, getPublicIp, listPresets, generatePresetDockerfile, getPreviewGatewayLogs, restartPreviewGateway, getPreviewGatewaySettings, patchPreviewGatewaySettings, getPreviewGatewayStatus, upgradePreviewGateway, getProjects, createProject, getProjectBySlug, createProjectFromTemplate, getProjectStatistics, deleteProject, getProject, updateProject, getProjectDeployments, getLastDeployment, triggerProjectPipeline, getActiveVisitors, listAgents, createAgent, getCliStatus, listAllRuns, latestRunForSource, getRunWithLogs, cancelRun, retryRun, getSandboxStatus, smokeTestAgent, deleteAgent, getAgent, updateAgent, listAgentRuns, triggerAgent, getAggregatedBuckets, startAnalysis, getRun, addContext, cancel, createPr, startFix, reAnalyze, updateAutomaticDeploy, listCustomDomainsForProject, createCustomDomain, deleteCustomDomain, getCustomDomain, updateCustomDomain, linkCustomDomainToCertificate, updateProjectDeploymentConfig, getDeployment, cancelDeployment, getDeploymentJobs, getDeploymentJobLogs, tailDeploymentJobLogs, getDeploymentOperations, executeDeploymentOperation, getDeploymentOperationStatus, pauseDeployment, promoteDeployment, resumeDeployment, rollbackToDeployment, teardownDeployment, listDsns, createDsn, getOrCreateDsn, regenerateDsn, revokeDsn, getEnvironmentVariables, createEnvironmentVariable, getResolvedEnvironmentVariables, getResolvedEnvironmentVariableValue, getEnvironmentVariableValue, deleteEnvironmentVariable, updateEnvironmentVariable, getEnvironments, createEnvironment, deleteEnvironment, getEnvironment, getEnvironmentCrons, getCronById, getCronExecutions, getEnvironmentDomains, addEnvironmentDomain, deleteEnvironmentDomain, updateEnvironmentSettings, sleepEnvironment, teardownEnvironment, wakeEnvironment, getContainerLogs, listContainers, getContainerDetail, getContainerLogsById, getContainerMetrics, streamContainerMetrics, restartContainer, startContainer, stopContainer, deployFromImage, deployFromImageUpload, deployFromStatic, listAlertRules, createAlertRule, deleteAlertRule, getAlertRule, updateAlertRule, getErrorDashboardStats, listErrorGroups, getErrorGroup, updateErrorGroup, listErrorEvents, getErrorEvent, getErrorStats, getErrorTimeSeries, getEventsCount, getEventTypeBreakdown, recordConsoleEvent, getPropertyBreakdown, getPropertyTimeline, getEventsTimeline, getUniqueEvents, listRemoteExternalImages, registerExternalImage, deleteExternalImage, getRemoteExternalImage, listFunnels, createFunnel, previewFunnelMetrics, deleteFunnel, updateFunnel, getFunnelMetrics, updateGitSettings, reinstallGitlabWebhook, hasErrorGroups, hasAnalyticsEvents, getHourlyVisits, listExternalImages, pushExternalImage, getExternalImage, listIncidents, createIncident, getBucketedIncidents, purgeProjectLogs, listMcps, createMcp, deleteMcp, getMcp, updateMcp, listMonitors, createMonitor, deleteReleaseSourceMaps, listSourceMaps, uploadSourceMap, revenueRecentEvents, revenueListIntegrations, revenueCreateIntegration, revenueDeleteIntegration, revenueUpdateConfig, revenueImportInvoicesCsv, revenueImportSubscriptionsCsv, revenueRotateToken, revenueUpdateSecret, revenueMetricsCustomers, revenueMetricsMrr, revenueMetricsSummary, listProjectSecrets, createProjectSecret, deleteProjectSecret, updateProjectSecret, updateProjectSettings, listSkills, createSkill, uploadSkill, deleteSkill, getSkill, updateSkill, downloadSkillArchive, listReleases, deleteSourceMap, listStaticBundles, deleteStaticBundle, getStaticBundle, getStatusOverview, getUniqueCounts, uploadStaticBundle, listProjectScans, triggerScan, getLatestScansPerEnvironment, getLatestScan, listWebhooks, createWebhook, deleteWebhook, getWebhook, updateWebhook, listDeliveries, getDelivery, retryDelivery, workflowDryRun, listMemory, writeMemory, searchMemory, dropMemory, supersedeMemory, workspaceListSessions, workspaceStartSession, workspaceDeleteSession, workspaceGetSession, workspaceUpdateSession, workspaceCancelRun, workspaceCloseSession, workspaceSendMessage, workspaceRegeneratePreviewPassword, workspaceReopenSession, workspaceRefreshSandbox, workspaceRestartSandbox, workspaceStartSandbox, workspaceSandboxStats, workspaceStopSandbox, workspaceTerminalPasteImage, workspaceListTerminalTabs, workspaceDeleteTerminalTab, getProxyLogs, getProxyLogByRequestId, getProjectsHealth, getTimeBucketStats, getTodayStats, getProxyLogById, listSyncedRepositories, getRepositoryByName, getAllRepositoriesByName, getRepositoryPresetByName, getRepositoryBranches, getRepositoryTags, getRepositoryPresetLive, getRepositoryById, getBranchesByRepositoryId, listCommitsByRepositoryId, checkCommitExists, getTagsByRepositoryId, getRestoreRun, revenueGlobalEvents, revenueMetricsGlobalMrr, revenueMetricsGlobalSummary, revenueListProviders, getProjectSessionReplays, getSessionEvents, getSettings, updateSettings, saveAgentToken, listAiProviders, updateAiProvider, activateAiProvider, saveAiProviderCredential, revokeJoinToken, generateJoinToken, getJoinTokenStatus, listGlobalMcps, createGlobalMcp, deleteGlobalMcp, getGlobalMcp, updateGlobalMcp, refreshRouteTable, rebuildSandboxImage, getGlobalSandboxStatus, listSecrets, upsertSecret, deleteSecret, listGlobalSkills, createGlobalSkill, uploadGlobalSkill, deleteGlobalSkill, getGlobalSkill, updateGlobalSkill, downloadGlobalSkillArchive, listProjectTemplates, listProjectTemplateTags, getProjectTemplate, getCurrentUser, listUsers, createUser, updateSelf, disableMfa, setupMfa, verifyAndEnableMfa, deleteUser, updateUser, restoreUser, assignRole, removeRole, listSandboxes, createSandbox, getSandbox, cmd, getCmd, cmdLogs, destroySandbox, domain, exec, execDetached, extendTimeout, mkdir, readFile, statPath, writeFile, writeFiles, listJobs, jobStatus, killJob, jobLogs, pauseSandbox, clearPreviewPassword, setPreviewPassword, restartSandbox, resumeSandbox, sourceSandbox, stopSandbox, cmdKill, getVisitorSessions, deleteSessionReplay, getSessionReplay, updateSessionDuration, getSessionReplayEvents, addEvents, deleteScan, getScan, getScanVulnerabilities, listEventTypes, triggerWeeklyDigest, listExternalPlugins, reloadPlugins, ingestSentryEnvelope, ingestSentryEvent, listAuditLogs, getAuditLog } from '../sdk.gen'; +import { type Options, getPlatformInfo, chunkUploadOptions, createRelease, createProjectRelease, finalizeProjectRelease, listReleaseFiles, uploadReleaseFile, recordEventMetrics, addSessionReplayEvents, initSessionReplay, recordSpeedMetrics, updateSpeedMetrics, webhookTrigger, getPricing, listProviderKeys, createProviderKey, testProviderKeyInline, deleteProviderKey, updateProviderKey, testProviderKeyById, getUsageByProvider, getConversations, getConversationDetail, getUsageRecent, getUsageSummary, getUsageTimeseries, getUsageTopModels, chatCompletions, embeddings, listModels, getAnalyticsActiveVisitors, getEventDetail, getEventVisitors, getAnalyticsEventsCount, getGeneralStats, getLiveVisitorsList, getPageFlow, getPageHourlySessions, getPagePathDetail, getPagePathVisitors, getPagePaths, getPagePathsSparklines, getRecentActivity, getSessionDetails, getAnalyticsSessionEvents, getSessionLogs, getVisitors, getVisitorByGuid, getVisitorById, getVisitorDetails, enrichVisitor, getVisitorInfo, getVisitorJourney, getAnalyticsVisitorSessions, getVisitorStats, listApiKeys, createApiKey, getApiKeyPermissions, deleteApiKey, getApiKey, updateApiKey, activateApiKey, deactivateApiKey, cliLogin, cliLogout, emailStatus, login, requestMagicLink, verifyMagicLink, requestPasswordReset, resetPassword, verifyEmail, verifyMfaChallenge, runExternalServiceBackup, listS3Sources, createS3Source, testS3ConnectionPreview, deleteS3Source, getS3Source, updateS3Source, listSourceBackups, runBackupForSource, setDefaultS3Source, testS3SourceConnection, listBackupSchedules, createBackupSchedule, deleteBackupSchedule, getBackupSchedule, listBackupsForSchedule, disableBackupSchedule, enableBackupSchedule, getBackup, blobDelete, blobList, blobPut, blobCopy, blobDisable, blobEnable, blobStatus, blobUpdate, blobDownload, getDashboardProjectsAnalytics, getActivityGraph, getScanByDeployment, listDnsProviders, createDnsProvider, deleteDnsProvider, getDnsProvider, updateProvider, listManagedDomains, addManagedDomain, testProviderConnection, listProviderZones, removeManagedDomain, verifyManagedDomain, lookupDnsARecords, listDomains, createDomain, getDomainByHost, cancelDomainOrder, getDomainOrder, createOrRecreateOrder, finalizeOrder, setupDnsChallenge, deleteDomain, getDomainById, getChallengeToken, getHttpChallengeDebug, provisionDomain, renewDomain, checkDomainStatus, listEmailDomains, createEmailDomain, getDomainByName, deleteEmailDomain, getDomain, getDomainDnsRecords, setupDns, verifyDomain, listEmailProviders, createEmailProvider, deleteEmailProvider, getEmailProvider, testProvider, listEmails, sendEmail, getGlobalEvents, getGlobalEventStats, getEmailStats, validateEmail, trackClick, trackOpen, getEmail, getEmailTracking, getEmailEvents, getEmailLinks, listServices, createService, listAvailableContainers, getServiceBySlug, listServiceHealthStatuses, importExternalService, listProjectServices, getProjectServiceEnvironmentVariables, getProvidersMetadata, getProviderMetadata, getServiceTypes, getServiceTypeParameters, deleteService, getService, updateService, getClusterHealth, triggerServiceHealthCheck, getServiceHealthStatus, addClusterMember, removeClusterMember, getClusterMember, promoteClusterMember, getServicePreviewEnvironmentVariablesMasked, getServicePreviewEnvironmentVariableNames, listServiceProjects, linkServiceToProject, unlinkServiceFromProject, getServiceEnvironmentVariables, getServiceEnvironmentVariable, startRestore, getRestoreCapabilities, planRestore, listRestoreRunsForService, retryCluster, startService, stopService, upgradeService, listRootContainers, listContainersAtPath, listEntities, getEntityInfo, queryData, downloadObject, getContainerInfo, checkExplorerSupport, listPgUpgrades, startPgUpgrade, getPgUpgrade, cancelPgUpgrade, getPgUpgradeLogs, retryPgUpgrade, rollbackPgUpgrade, getFile, getIpGeolocation, listConnections, deleteConnection, activateConnection, deactivateConnection, runConnectionHealthCheck, listRepositoriesByConnection, syncRepositories, updateConnectionToken, validateConnection, listGitProviders, createGitProvider, createGithubPatProvider, createGitlabOauthProvider, createGitlabPatProvider, deleteGitProvider, getGitProvider, activateProvider, handleGitProviderOauthCallback, getProviderConnections, updateGitProviderCredentials, deactivateProvider, checkProviderDeletionSafety, startGitProviderOauth, deleteProviderSafely, getPublicRepository, getPublicBranches, detectPublicPresets, discoverWorkloads, executeImport, createPlan, listSources, getImportStatus, getIncident, updateIncidentStatus, getIncidentUpdates, adminListNodes, registerNode, adminRemoveNode, adminGetNode, adminListNodeContainers, postDnsAck, getDnsChanges, adminUndrainNode, adminDrainStatus, adminDrainNode, nodeHeartbeat, listPeers, getS3Credentials, listIpAccessControl, createIpAccessControl, checkIpBlocked, deleteIpAccessControl, getIpAccessControl, updateIpAccessControl, kvDel, kvDisable, kvEnable, kvExpire, kvGet, kvIncr, kvKeys, kvSet, kvStatus, kvTtl, kvUpdate, listRoutes, createRoute, deleteRoute, getRoute, updateRoute, logout, getLogContext, searchLogs, tailLogs, getProjectsMonitorHealth, deleteMonitor, getMonitor, getBucketedStatus, getCurrentMonitorStatus, getUptimeHistory, deletePreferences, getPreferences, updatePreferences, listNotificationProviders, createNotificationProvider, createNotificationEmailProvider, updateEmailProvider, createSlackProvider, updateSlackProvider, createWebhookProvider, updateWebhookProvider, deleteNotificationProvider, getNotificationProvider, updateNotificationProvider, testNotificationProvider, listOrders, queryGenaiTraces, getGenaiTrace, getHealth, listInsights, queryLogs, listMetricNames, queryMetrics, getPipelineStats, getQuota, queryTraceSummaries, queryTraces, getTrace, ingestLogs, ingestMetrics, ingestTraces, ingestLogsByPath, ingestMetricsByPath, ingestTracesByPath, hasPerformanceMetrics, getPerformanceMetrics, getMetricsOverTime, getGroupedPageMetrics, getAccessInfo, getPrivateIp, getPublicIp, listPresets, generatePresetDockerfile, getPreviewGatewayLogs, restartPreviewGateway, getPreviewGatewaySettings, patchPreviewGatewaySettings, getPreviewGatewayStatus, upgradePreviewGateway, getProjects, createProject, getProjectBySlug, createProjectFromTemplate, getProjectStatistics, deleteProject, getProject, updateProject, getProjectDeployments, getLastDeployment, triggerProjectPipeline, getActiveVisitors, listAgents, createAgent, getCliStatus, listAllRuns, latestRunForSource, getRunWithLogs, cancelRun, retryRun, getSandboxStatus, smokeTestAgent, deleteAgent, getAgent, updateAgent, listAgentRuns, triggerAgent, getAggregatedBuckets, startAnalysis, getRun, addContext, cancel, createPr, startFix, reAnalyze, updateAutomaticDeploy, listCustomDomainsForProject, createCustomDomain, deleteCustomDomain, getCustomDomain, updateCustomDomain, linkCustomDomainToCertificate, updateProjectDeploymentConfig, getDeployment, cancelDeployment, getDeploymentJobs, getDeploymentJobLogs, tailDeploymentJobLogs, getDeploymentOperations, executeDeploymentOperation, getDeploymentOperationStatus, pauseDeployment, promoteDeployment, resumeDeployment, rollbackToDeployment, teardownDeployment, listDsns, createDsn, getOrCreateDsn, regenerateDsn, revokeDsn, getEnvironmentVariables, createEnvironmentVariable, getResolvedEnvironmentVariables, getResolvedEnvironmentVariableValue, getEnvironmentVariableValue, deleteEnvironmentVariable, updateEnvironmentVariable, getEnvironments, createEnvironment, deleteEnvironment, getEnvironment, getEnvironmentCrons, getCronById, getCronExecutions, getEnvironmentDomains, addEnvironmentDomain, deleteEnvironmentDomain, updateEnvironmentSettings, sleepEnvironment, teardownEnvironment, wakeEnvironment, getContainerLogs, listContainers, getContainerDetail, getContainerLogsById, getContainerMetrics, streamContainerMetrics, restartContainer, startContainer, stopContainer, deployFromImage, deployFromImageUpload, deployFromStatic, listAlertRules, createAlertRule, deleteAlertRule, getAlertRule, updateAlertRule, getErrorDashboardStats, listErrorGroups, getErrorGroup, updateErrorGroup, listErrorEvents, getErrorEvent, getErrorStats, getErrorTimeSeries, getEventsCount, getEventTypeBreakdown, recordConsoleEvent, getPropertyBreakdown, getPropertyTimeline, getEventsTimeline, getUniqueEvents, listRemoteExternalImages, registerExternalImage, deleteExternalImage, getRemoteExternalImage, listFunnels, createFunnel, previewFunnelMetrics, deleteFunnel, updateFunnel, getFunnelMetrics, updateGitSettings, reinstallGitlabWebhook, hasErrorGroups, hasAnalyticsEvents, getHourlyVisits, listExternalImages, pushExternalImage, getExternalImage, listIncidents, createIncident, getBucketedIncidents, purgeProjectLogs, listMcps, createMcp, deleteMcp, getMcp, updateMcp, listMonitors, createMonitor, observabilityListEvents, observabilityFullEvent, deleteReleaseSourceMaps, listSourceMaps, uploadSourceMap, revenueRecentEvents, revenueListIntegrations, revenueCreateIntegration, revenueDeleteIntegration, revenueUpdateConfig, revenueImportInvoicesCsv, revenueImportSubscriptionsCsv, revenueRotateToken, revenueUpdateSecret, revenueMetricsCustomers, revenueMetricsMrr, revenueMetricsSummary, listProjectSecrets, createProjectSecret, deleteProjectSecret, updateProjectSecret, updateProjectSettings, listSkills, createSkill, uploadSkill, deleteSkill, getSkill, updateSkill, downloadSkillArchive, listReleases, deleteSourceMap, listStaticBundles, deleteStaticBundle, getStaticBundle, getStatusOverview, getUniqueCounts, uploadStaticBundle, listProjectScans, triggerScan, getLatestScansPerEnvironment, getLatestScan, listWebhooks, createWebhook, deleteWebhook, getWebhook, updateWebhook, listDeliveries, getDelivery, retryDelivery, workflowDryRun, listMemory, writeMemory, searchMemory, dropMemory, supersedeMemory, workspaceListSessions, workspaceStartSession, workspaceDeleteSession, workspaceGetSession, workspaceUpdateSession, workspaceCancelRun, workspaceCloseSession, workspaceSendMessage, workspaceRegeneratePreviewPassword, workspaceReopenSession, workspaceRefreshSandbox, workspaceRestartSandbox, workspaceStartSandbox, workspaceSandboxStats, workspaceStopSandbox, workspaceTerminalPasteImage, workspaceListTerminalTabs, workspaceDeleteTerminalTab, getProxyLogs, getProxyLogByRequestId, getProjectsHealth, getTimeBucketStats, getTodayStats, getProxyLogById, listSyncedRepositories, getRepositoryByName, getAllRepositoriesByName, getRepositoryPresetByName, getRepositoryBranches, getRepositoryTags, getRepositoryPresetLive, getRepositoryById, getBranchesByRepositoryId, listCommitsByRepositoryId, checkCommitExists, getTagsByRepositoryId, getRestoreRun, revenueGlobalEvents, revenueMetricsGlobalMrr, revenueMetricsGlobalSummary, revenueListProviders, getProjectSessionReplays, getSessionEvents, getSettings, updateSettings, saveAgentToken, listAiProviders, updateAiProvider, activateAiProvider, saveAiProviderCredential, revokeJoinToken, generateJoinToken, getJoinTokenStatus, listGlobalMcps, createGlobalMcp, deleteGlobalMcp, getGlobalMcp, updateGlobalMcp, refreshRouteTable, rebuildSandboxImage, getGlobalSandboxStatus, listSecrets, upsertSecret, deleteSecret, listGlobalSkills, createGlobalSkill, uploadGlobalSkill, deleteGlobalSkill, getGlobalSkill, updateGlobalSkill, downloadGlobalSkillArchive, listProjectTemplates, listProjectTemplateTags, getProjectTemplate, getCurrentUser, listUsers, createUser, updateSelf, disableMfa, setupMfa, verifyAndEnableMfa, deleteUser, updateUser, restoreUser, assignRole, removeRole, listSandboxes, createSandbox, getSandbox, cmd, getCmd, cmdLogs, destroySandbox, domain, exec, execDetached, extendTimeout, mkdir, readFile, statPath, writeFile, writeFiles, listJobs, jobStatus, killJob, jobLogs, pauseSandbox, clearPreviewPassword, setPreviewPassword, restartSandbox, resumeSandbox, sourceSandbox, stopSandbox, cmdKill, getVisitorSessions, deleteSessionReplay, getSessionReplay, updateSessionDuration, getSessionReplayEvents, addEvents, deleteScan, getScan, getScanVulnerabilities, listEventTypes, triggerWeeklyDigest, listExternalPlugins, reloadPlugins, ingestSentryEnvelope, ingestSentryEvent, listAuditLogs, getAuditLog } from '../sdk.gen'; import { queryOptions, type UseMutationOptions, type DefaultError, infiniteQueryOptions, type InfiniteData } from '@tanstack/react-query'; -import type { GetPlatformInfoData, ChunkUploadOptionsData, CreateReleaseData, CreateReleaseResponse, CreateProjectReleaseData, CreateProjectReleaseResponse, FinalizeProjectReleaseData, FinalizeProjectReleaseResponse, ListReleaseFilesData, UploadReleaseFileData, UploadReleaseFileResponse, RecordEventMetricsData, RecordEventMetricsResponse, AddSessionReplayEventsData, AddSessionReplayEventsError, AddSessionReplayEventsResponse, InitSessionReplayData, InitSessionReplayError, InitSessionReplayResponse, RecordSpeedMetricsData, RecordSpeedMetricsError, RecordSpeedMetricsResponse, UpdateSpeedMetricsData, UpdateSpeedMetricsError, UpdateSpeedMetricsResponse, WebhookTriggerData, WebhookTriggerResponse2 as WebhookTriggerResponse, GetPricingData, ListProviderKeysData, CreateProviderKeyData, CreateProviderKeyError, CreateProviderKeyResponse, TestProviderKeyInlineData, TestProviderKeyInlineError, TestProviderKeyInlineResponse, DeleteProviderKeyData, DeleteProviderKeyError, DeleteProviderKeyResponse, UpdateProviderKeyData, UpdateProviderKeyError, UpdateProviderKeyResponse, TestProviderKeyByIdData, TestProviderKeyByIdError, TestProviderKeyByIdResponse, GetUsageByProviderData, GetConversationsData, GetConversationDetailData, GetUsageRecentData, GetUsageSummaryData, GetUsageTimeseriesData, GetUsageTopModelsData, ChatCompletionsData, ChatCompletionsError, ChatCompletionsResponse, EmbeddingsData, EmbeddingsError, EmbeddingsResponse, ListModelsData, GetAnalyticsActiveVisitorsData, GetEventDetailData, GetEventVisitorsData, GetEventVisitorsResponse, GetAnalyticsEventsCountData, GetGeneralStatsData, GetLiveVisitorsListData, GetPageFlowData, GetPageHourlySessionsData, GetPagePathDetailData, GetPagePathVisitorsData, GetPagePathVisitorsResponse, GetPagePathsData, GetPagePathsSparklinesData, GetRecentActivityData, GetSessionDetailsData, GetAnalyticsSessionEventsData, GetAnalyticsSessionEventsResponse, GetSessionLogsData, GetSessionLogsResponse, GetVisitorsData, GetVisitorsResponse, GetVisitorByGuidData, GetVisitorByIdData, GetVisitorDetailsData, EnrichVisitorData, EnrichVisitorResponse2 as EnrichVisitorResponse, GetVisitorInfoData, GetVisitorJourneyData, GetAnalyticsVisitorSessionsData, GetVisitorStatsData, ListApiKeysData, ListApiKeysResponse, CreateApiKeyData, CreateApiKeyResponse2 as CreateApiKeyResponse, GetApiKeyPermissionsData, DeleteApiKeyData, DeleteApiKeyResponse, GetApiKeyData, UpdateApiKeyData, UpdateApiKeyResponse, ActivateApiKeyData, ActivateApiKeyResponse, DeactivateApiKeyData, DeactivateApiKeyResponse, CliLoginData, CliLoginResponse2 as CliLoginResponse, CliLogoutData, CliLogoutResponse, EmailStatusData, LoginData, LoginResponse, RequestMagicLinkData, RequestMagicLinkResponse, VerifyMagicLinkData, RequestPasswordResetData, RequestPasswordResetResponse, ResetPasswordData, ResetPasswordResponse, VerifyEmailData, VerifyMfaChallengeData, VerifyMfaChallengeResponse, RunExternalServiceBackupData, RunExternalServiceBackupError, RunExternalServiceBackupResponse, ListS3SourcesData, CreateS3SourceData, CreateS3SourceError, CreateS3SourceResponse, TestS3ConnectionPreviewData, TestS3ConnectionPreviewError, TestS3ConnectionPreviewResponse, DeleteS3SourceData, DeleteS3SourceError, DeleteS3SourceResponse, GetS3SourceData, UpdateS3SourceData, UpdateS3SourceError, UpdateS3SourceResponse, ListSourceBackupsData, RunBackupForSourceData, RunBackupForSourceError, RunBackupForSourceResponse, SetDefaultS3SourceData, SetDefaultS3SourceError, SetDefaultS3SourceResponse, TestS3SourceConnectionData, TestS3SourceConnectionError, TestS3SourceConnectionResponse, ListBackupSchedulesData, CreateBackupScheduleData, CreateBackupScheduleError, CreateBackupScheduleResponse, DeleteBackupScheduleData, DeleteBackupScheduleError, DeleteBackupScheduleResponse, GetBackupScheduleData, ListBackupsForScheduleData, DisableBackupScheduleData, DisableBackupScheduleResponse, EnableBackupScheduleData, EnableBackupScheduleResponse, GetBackupData, BlobDeleteData, BlobDeleteError, BlobDeleteResponse, BlobListData, BlobListError, BlobListResponse, BlobPutData, BlobPutError, BlobPutResponse, BlobCopyData, BlobCopyError, BlobCopyResponse, BlobDisableData, BlobDisableResponse, BlobEnableData, BlobEnableResponse, BlobStatusData, BlobUpdateData, BlobUpdateResponse, BlobDownloadData, GetDashboardProjectsAnalyticsData, GetActivityGraphData, GetScanByDeploymentData, ListDnsProvidersData, CreateDnsProviderData, CreateDnsProviderResponse, DeleteDnsProviderData, DeleteDnsProviderResponse, GetDnsProviderData, UpdateProviderData, UpdateProviderResponse, ListManagedDomainsData, AddManagedDomainData, AddManagedDomainResponse, TestProviderConnectionData, TestProviderConnectionResponse, ListProviderZonesData, RemoveManagedDomainData, RemoveManagedDomainResponse, VerifyManagedDomainData, VerifyManagedDomainResponse, LookupDnsARecordsData, ListDomainsData, ListDomainsResponse2 as ListDomainsResponse, CreateDomainData, CreateDomainResponse, GetDomainByHostData, CancelDomainOrderData, CancelDomainOrderResponse, GetDomainOrderData, CreateOrRecreateOrderData, CreateOrRecreateOrderResponse, FinalizeOrderData, FinalizeOrderResponse, SetupDnsChallengeData, SetupDnsChallengeResponse2 as SetupDnsChallengeResponse, DeleteDomainData, DeleteDomainResponse, GetDomainByIdData, GetChallengeTokenData, GetHttpChallengeDebugData, ProvisionDomainData, ProvisionDomainResponse, RenewDomainData, RenewDomainResponse, CheckDomainStatusData, ListEmailDomainsData, CreateEmailDomainData, CreateEmailDomainResponse, GetDomainByNameData, DeleteEmailDomainData, DeleteEmailDomainResponse, GetDomainData, GetDomainDnsRecordsData, SetupDnsData, SetupDnsResponse2 as SetupDnsResponse, VerifyDomainData, VerifyDomainResponse, ListEmailProvidersData, CreateEmailProviderData, CreateEmailProviderResponse, DeleteEmailProviderData, DeleteEmailProviderResponse, GetEmailProviderData, TestProviderData, TestProviderResponse2 as TestProviderResponse, ListEmailsData, ListEmailsResponse, SendEmailData, SendEmailResponse, GetGlobalEventsData, GetGlobalEventsResponse, GetGlobalEventStatsData, GetEmailStatsData, ValidateEmailData, ValidateEmailResponse2 as ValidateEmailResponse, TrackClickData, TrackOpenData, GetEmailData, GetEmailTrackingData, GetEmailEventsData, GetEmailLinksData, ListServicesData, ListServicesResponse, CreateServiceData, CreateServiceResponse, ListAvailableContainersData, GetServiceBySlugData, ListServiceHealthStatusesData, ImportExternalServiceData, ImportExternalServiceResponse, ListProjectServicesData, ListProjectServicesResponse, GetProjectServiceEnvironmentVariablesData, GetProvidersMetadataData, GetProviderMetadataData, GetServiceTypesData, GetServiceTypeParametersData, DeleteServiceData, DeleteServiceResponse, GetServiceData, UpdateServiceData, UpdateServiceResponse, GetClusterHealthData, TriggerServiceHealthCheckData, TriggerServiceHealthCheckResponse, GetServiceHealthStatusData, AddClusterMemberData, AddClusterMemberResponse, RemoveClusterMemberData, RemoveClusterMemberResponse, GetClusterMemberData, PromoteClusterMemberData, GetServicePreviewEnvironmentVariablesMaskedData, GetServicePreviewEnvironmentVariableNamesData, ListServiceProjectsData, ListServiceProjectsResponse, LinkServiceToProjectData, LinkServiceToProjectResponse, UnlinkServiceFromProjectData, UnlinkServiceFromProjectResponse, GetServiceEnvironmentVariablesData, GetServiceEnvironmentVariableData, StartRestoreData, StartRestoreError, StartRestoreResponse, GetRestoreCapabilitiesData, PlanRestoreData, PlanRestoreError, PlanRestoreResponse, ListRestoreRunsForServiceData, RetryClusterData, RetryClusterResponse, StartServiceData, StartServiceResponse, StopServiceData, StopServiceResponse, UpgradeServiceData, UpgradeServiceResponse, ListRootContainersData, ListContainersAtPathData, ListEntitiesData, GetEntityInfoData, QueryDataData, QueryDataResponse2 as QueryDataResponse, DownloadObjectData, GetContainerInfoData, CheckExplorerSupportData, ListPgUpgradesData, StartPgUpgradeData, StartPgUpgradeResponse, GetPgUpgradeData, CancelPgUpgradeData, CancelPgUpgradeResponse, GetPgUpgradeLogsData, RetryPgUpgradeData, RetryPgUpgradeResponse, RollbackPgUpgradeData, RollbackPgUpgradeResponse, GetFileData, GetIpGeolocationData, ListConnectionsData, ListConnectionsResponse, DeleteConnectionData, DeleteConnectionResponse, ActivateConnectionData, DeactivateConnectionData, RunConnectionHealthCheckData, RunConnectionHealthCheckResponse, ListRepositoriesByConnectionData, ListRepositoriesByConnectionResponse, SyncRepositoriesData, SyncRepositoriesResponse, UpdateConnectionTokenData, UpdateConnectionTokenResponse, ValidateConnectionData, ListGitProvidersData, CreateGitProviderData, CreateGitProviderResponse, CreateGithubPatProviderData, CreateGithubPatProviderResponse, CreateGitlabOauthProviderData, CreateGitlabOauthProviderResponse, CreateGitlabPatProviderData, CreateGitlabPatProviderResponse, DeleteGitProviderData, DeleteGitProviderResponse, GetGitProviderData, ActivateProviderData, HandleGitProviderOauthCallbackData, GetProviderConnectionsData, UpdateGitProviderCredentialsData, UpdateGitProviderCredentialsResponse, DeactivateProviderData, CheckProviderDeletionSafetyData, StartGitProviderOauthData, DeleteProviderSafelyData, DeleteProviderSafelyResponse, GetPublicRepositoryData, GetPublicBranchesData, DetectPublicPresetsData, DiscoverWorkloadsData, DiscoverWorkloadsResponse, ExecuteImportData, ExecuteImportResponse2 as ExecuteImportResponse, CreatePlanData, CreatePlanResponse2 as CreatePlanResponse, ListSourcesData, GetImportStatusData, GetIncidentData, UpdateIncidentStatusData, UpdateIncidentStatusResponse, GetIncidentUpdatesData, AdminListNodesData, RegisterNodeData, RegisterNodeResponse2 as RegisterNodeResponse, AdminRemoveNodeData, AdminRemoveNodeResponse, AdminGetNodeData, AdminListNodeContainersData, PostDnsAckData, PostDnsAckResponse, GetDnsChangesData, AdminUndrainNodeData, AdminUndrainNodeResponse, AdminDrainStatusData, AdminDrainNodeData, AdminDrainNodeResponse, NodeHeartbeatData, NodeHeartbeatResponse, ListPeersData, GetS3CredentialsData, ListIpAccessControlData, CreateIpAccessControlData, CreateIpAccessControlError, CreateIpAccessControlResponse, CheckIpBlockedData, DeleteIpAccessControlData, DeleteIpAccessControlError, DeleteIpAccessControlResponse, GetIpAccessControlData, UpdateIpAccessControlData, UpdateIpAccessControlError, UpdateIpAccessControlResponse, KvDelData, KvDelResponse, KvDisableData, KvDisableResponse, KvEnableData, KvEnableResponse, KvExpireData, KvExpireResponse, KvGetData, KvGetResponse, KvIncrData, KvIncrResponse, KvKeysData, KvKeysResponse, KvSetData, KvSetResponse, KvStatusData, KvTtlData, KvTtlResponse, KvUpdateData, KvUpdateResponse, ListRoutesData, CreateRouteData, CreateRouteResponse, DeleteRouteData, DeleteRouteResponse, GetRouteData, UpdateRouteData, UpdateRouteResponse, LogoutData, GetLogContextData, SearchLogsData, SearchLogsError, SearchLogsResponse2 as SearchLogsResponse, TailLogsData, GetProjectsMonitorHealthData, DeleteMonitorData, DeleteMonitorResponse, GetMonitorData, GetBucketedStatusData, GetCurrentMonitorStatusData, GetUptimeHistoryData, DeletePreferencesData, DeletePreferencesResponse, GetPreferencesData, UpdatePreferencesData, UpdatePreferencesResponse, ListNotificationProvidersData, ListNotificationProvidersResponse, CreateNotificationProviderData, CreateNotificationProviderResponse, CreateNotificationEmailProviderData, CreateNotificationEmailProviderResponse, UpdateEmailProviderData, UpdateEmailProviderResponse, CreateSlackProviderData, CreateSlackProviderResponse, UpdateSlackProviderData, UpdateSlackProviderResponse, CreateWebhookProviderData, CreateWebhookProviderResponse, UpdateWebhookProviderData, UpdateWebhookProviderResponse, DeleteNotificationProviderData, DeleteNotificationProviderResponse, GetNotificationProviderData, UpdateNotificationProviderData, UpdateNotificationProviderResponse, TestNotificationProviderData, TestNotificationProviderResponse, ListOrdersData, ListOrdersResponse2 as ListOrdersResponse, QueryGenaiTracesData, QueryGenaiTracesError, QueryGenaiTracesResponse, GetGenaiTraceData, GetHealthData, ListInsightsData, ListInsightsError, ListInsightsResponse, QueryLogsData, QueryLogsError, QueryLogsResponse, ListMetricNamesData, QueryMetricsData, GetPipelineStatsData, GetQuotaData, QueryTraceSummariesData, QueryTraceSummariesError, QueryTraceSummariesResponse, QueryTracesData, QueryTracesError, QueryTracesResponse, GetTraceData, IngestLogsData, IngestLogsError, IngestMetricsData, IngestMetricsError, IngestTracesData, IngestTracesError, IngestLogsByPathData, IngestLogsByPathError, IngestMetricsByPathData, IngestMetricsByPathError, IngestTracesByPathData, IngestTracesByPathError, HasPerformanceMetricsData, GetPerformanceMetricsData, GetMetricsOverTimeData, GetGroupedPageMetricsData, GetAccessInfoData, GetPrivateIpData, GetPublicIpData, ListPresetsData, GeneratePresetDockerfileData, GeneratePresetDockerfileResponse, GetPreviewGatewayLogsData, RestartPreviewGatewayData, RestartPreviewGatewayResponse, GetPreviewGatewaySettingsData, PatchPreviewGatewaySettingsData, PatchPreviewGatewaySettingsResponse, GetPreviewGatewayStatusData, UpgradePreviewGatewayData, UpgradePreviewGatewayResponse, GetProjectsData, GetProjectsResponse, CreateProjectData, CreateProjectResponse, GetProjectBySlugData, CreateProjectFromTemplateData, CreateProjectFromTemplateResponse2 as CreateProjectFromTemplateResponse, GetProjectStatisticsData, DeleteProjectData, DeleteProjectResponse, GetProjectData, UpdateProjectData, UpdateProjectResponse, GetProjectDeploymentsData, GetProjectDeploymentsResponse, GetLastDeploymentData, TriggerProjectPipelineData, TriggerProjectPipelineResponse, GetActiveVisitorsData, ListAgentsData, CreateAgentData, CreateAgentResponse, GetCliStatusData, ListAllRunsData, ListAllRunsResponse, LatestRunForSourceData, GetRunWithLogsData, CancelRunData, CancelRunResponse, RetryRunData, RetryRunResponse, GetSandboxStatusData, SmokeTestAgentData, SmokeTestAgentResponse, DeleteAgentData, DeleteAgentResponse, GetAgentData, UpdateAgentData, UpdateAgentResponse, ListAgentRunsData, ListAgentRunsResponse, TriggerAgentData, TriggerAgentResponse, GetAggregatedBucketsData, StartAnalysisData, StartAnalysisResponse, GetRunData, AddContextData, CancelData, CreatePrData, CreatePrResponse2 as CreatePrResponse, StartFixData, ReAnalyzeData, UpdateAutomaticDeployData, UpdateAutomaticDeployResponse, ListCustomDomainsForProjectData, CreateCustomDomainData, CreateCustomDomainResponse, DeleteCustomDomainData, DeleteCustomDomainResponse, GetCustomDomainData, UpdateCustomDomainData, UpdateCustomDomainResponse, LinkCustomDomainToCertificateData, LinkCustomDomainToCertificateResponse, UpdateProjectDeploymentConfigData, UpdateProjectDeploymentConfigResponse, GetDeploymentData, CancelDeploymentData, CancelDeploymentResponse, GetDeploymentJobsData, GetDeploymentJobLogsData, TailDeploymentJobLogsData, GetDeploymentOperationsData, ExecuteDeploymentOperationData, ExecuteDeploymentOperationResponse, GetDeploymentOperationStatusData, PauseDeploymentData, PauseDeploymentResponse, PromoteDeploymentData, PromoteDeploymentResponse, ResumeDeploymentData, ResumeDeploymentResponse, RollbackToDeploymentData, RollbackToDeploymentResponse, TeardownDeploymentData, TeardownDeploymentResponse, ListDsnsData, CreateDsnData, CreateDsnResponse, GetOrCreateDsnData, GetOrCreateDsnResponse, RegenerateDsnData, RegenerateDsnResponse, RevokeDsnData, RevokeDsnResponse, GetEnvironmentVariablesData, CreateEnvironmentVariableData, CreateEnvironmentVariableResponse, GetResolvedEnvironmentVariablesData, GetResolvedEnvironmentVariableValueData, GetEnvironmentVariableValueData, DeleteEnvironmentVariableData, DeleteEnvironmentVariableResponse, UpdateEnvironmentVariableData, UpdateEnvironmentVariableResponse, GetEnvironmentsData, CreateEnvironmentData, CreateEnvironmentResponse, DeleteEnvironmentData, DeleteEnvironmentResponse, GetEnvironmentData, GetEnvironmentCronsData, GetCronByIdData, GetCronExecutionsData, GetCronExecutionsResponse, GetEnvironmentDomainsData, AddEnvironmentDomainData, AddEnvironmentDomainResponse, DeleteEnvironmentDomainData, DeleteEnvironmentDomainResponse, UpdateEnvironmentSettingsData, UpdateEnvironmentSettingsResponse, SleepEnvironmentData, SleepEnvironmentResponse, TeardownEnvironmentData, TeardownEnvironmentResponse, WakeEnvironmentData, WakeEnvironmentResponse, GetContainerLogsData, ListContainersData, GetContainerDetailData, GetContainerLogsByIdData, GetContainerMetricsData, StreamContainerMetricsData, RestartContainerData, RestartContainerResponse, StartContainerData, StartContainerResponse, StopContainerData, StopContainerResponse, DeployFromImageData, DeployFromImageResponse, DeployFromImageUploadData, DeployFromImageUploadResponse, DeployFromStaticData, DeployFromStaticResponse, ListAlertRulesData, CreateAlertRuleData, CreateAlertRuleResponse, DeleteAlertRuleData, DeleteAlertRuleResponse, GetAlertRuleData, UpdateAlertRuleData, UpdateAlertRuleResponse, GetErrorDashboardStatsData, ListErrorGroupsData, ListErrorGroupsResponse, GetErrorGroupData, UpdateErrorGroupData, ListErrorEventsData, ListErrorEventsResponse, GetErrorEventData, GetErrorStatsData, GetErrorTimeSeriesData, GetEventsCountData, GetEventTypeBreakdownData, RecordConsoleEventData, GetPropertyBreakdownData, GetPropertyTimelineData, GetEventsTimelineData, GetUniqueEventsData, GetUniqueEventsResponse, ListRemoteExternalImagesData, ListRemoteExternalImagesResponse, RegisterExternalImageData, RegisterExternalImageResponse, DeleteExternalImageData, DeleteExternalImageResponse, GetRemoteExternalImageData, ListFunnelsData, CreateFunnelData, CreateFunnelResponse2 as CreateFunnelResponse, PreviewFunnelMetricsData, PreviewFunnelMetricsResponse, DeleteFunnelData, UpdateFunnelData, GetFunnelMetricsData, UpdateGitSettingsData, UpdateGitSettingsResponse, ReinstallGitlabWebhookData, ReinstallGitlabWebhookResponse, HasErrorGroupsData, HasAnalyticsEventsData, GetHourlyVisitsData, ListExternalImagesData, PushExternalImageData, PushExternalImageResponse, GetExternalImageData, ListIncidentsData, CreateIncidentData, CreateIncidentResponse, GetBucketedIncidentsData, PurgeProjectLogsData, PurgeProjectLogsError, ListMcpsData, CreateMcpData, CreateMcpResponse, DeleteMcpData, DeleteMcpResponse, GetMcpData, UpdateMcpData, UpdateMcpResponse, ListMonitorsData, CreateMonitorData, CreateMonitorResponse, DeleteReleaseSourceMapsData, DeleteReleaseSourceMapsResponse, ListSourceMapsData, UploadSourceMapData, UploadSourceMapResponse, RevenueRecentEventsData, RevenueListIntegrationsData, RevenueCreateIntegrationData, RevenueCreateIntegrationResponse, RevenueDeleteIntegrationData, RevenueDeleteIntegrationResponse, RevenueUpdateConfigData, RevenueUpdateConfigResponse, RevenueImportInvoicesCsvData, RevenueImportInvoicesCsvResponse, RevenueImportSubscriptionsCsvData, RevenueImportSubscriptionsCsvResponse, RevenueRotateTokenData, RevenueRotateTokenResponse, RevenueUpdateSecretData, RevenueUpdateSecretResponse, RevenueMetricsCustomersData, RevenueMetricsMrrData, RevenueMetricsSummaryData, ListProjectSecretsData, CreateProjectSecretData, CreateProjectSecretResponse, DeleteProjectSecretData, DeleteProjectSecretResponse, UpdateProjectSecretData, UpdateProjectSecretResponse, UpdateProjectSettingsData, UpdateProjectSettingsResponse, ListSkillsData, CreateSkillData, CreateSkillResponse, UploadSkillData, UploadSkillResponse, DeleteSkillData, DeleteSkillResponse, GetSkillData, UpdateSkillData, UpdateSkillResponse, DownloadSkillArchiveData, ListReleasesData, DeleteSourceMapData, DeleteSourceMapResponse, ListStaticBundlesData, ListStaticBundlesResponse, DeleteStaticBundleData, DeleteStaticBundleResponse, GetStaticBundleData, GetStatusOverviewData, GetUniqueCountsData, UploadStaticBundleData, UploadStaticBundleResponse, ListProjectScansData, ListProjectScansError, ListProjectScansResponse, TriggerScanData, TriggerScanError, TriggerScanResponse2 as TriggerScanResponse, GetLatestScansPerEnvironmentData, GetLatestScanData, ListWebhooksData, ListWebhooksResponse, CreateWebhookData, CreateWebhookResponse, DeleteWebhookData, DeleteWebhookResponse, GetWebhookData, UpdateWebhookData, UpdateWebhookResponse, ListDeliveriesData, GetDeliveryData, RetryDeliveryData, RetryDeliveryResponse, WorkflowDryRunData, WorkflowDryRunResponse, ListMemoryData, WriteMemoryData, WriteMemoryResponse, SearchMemoryData, DropMemoryData, DropMemoryResponse, SupersedeMemoryData, SupersedeMemoryResponse, WorkspaceListSessionsData, WorkspaceListSessionsResponse, WorkspaceStartSessionData, WorkspaceStartSessionResponse, WorkspaceDeleteSessionData, WorkspaceDeleteSessionResponse, WorkspaceGetSessionData, WorkspaceUpdateSessionData, WorkspaceUpdateSessionResponse, WorkspaceCancelRunData, WorkspaceCancelRunResponse, WorkspaceCloseSessionData, WorkspaceCloseSessionResponse, WorkspaceSendMessageData, WorkspaceSendMessageResponse, WorkspaceRegeneratePreviewPasswordData, WorkspaceRegeneratePreviewPasswordResponse, WorkspaceReopenSessionData, WorkspaceReopenSessionResponse, WorkspaceRefreshSandboxData, WorkspaceRefreshSandboxResponse, WorkspaceRestartSandboxData, WorkspaceRestartSandboxResponse, WorkspaceStartSandboxData, WorkspaceStartSandboxResponse, WorkspaceSandboxStatsData, WorkspaceStopSandboxData, WorkspaceStopSandboxResponse, WorkspaceTerminalPasteImageData, WorkspaceTerminalPasteImageResponse, WorkspaceListTerminalTabsData, WorkspaceDeleteTerminalTabData, WorkspaceDeleteTerminalTabResponse, GetProxyLogsData, GetProxyLogsResponse, GetProxyLogByRequestIdData, GetProjectsHealthData, GetTimeBucketStatsData, GetTodayStatsData, GetProxyLogByIdData, ListSyncedRepositoriesData, ListSyncedRepositoriesResponse, GetRepositoryByNameData, GetAllRepositoriesByNameData, GetRepositoryPresetByNameData, GetRepositoryBranchesData, GetRepositoryTagsData, GetRepositoryPresetLiveData, GetRepositoryByIdData, GetBranchesByRepositoryIdData, ListCommitsByRepositoryIdData, CheckCommitExistsData, GetTagsByRepositoryIdData, GetRestoreRunData, RevenueGlobalEventsData, RevenueMetricsGlobalMrrData, RevenueMetricsGlobalSummaryData, RevenueListProvidersData, GetProjectSessionReplaysData, GetProjectSessionReplaysError, GetProjectSessionReplaysResponse2 as GetProjectSessionReplaysResponse, GetSessionEventsData, GetSettingsData, UpdateSettingsData, UpdateSettingsResponse, SaveAgentTokenData, SaveAgentTokenResponse2 as SaveAgentTokenResponse, ListAiProvidersData, UpdateAiProviderData, UpdateAiProviderResponse2 as UpdateAiProviderResponse, ActivateAiProviderData, ActivateAiProviderResponse, SaveAiProviderCredentialData, SaveAiProviderCredentialResponse, RevokeJoinTokenData, RevokeJoinTokenResponse, GenerateJoinTokenData, GenerateJoinTokenResponse2 as GenerateJoinTokenResponse, GetJoinTokenStatusData, ListGlobalMcpsData, CreateGlobalMcpData, CreateGlobalMcpResponse, DeleteGlobalMcpData, DeleteGlobalMcpResponse, GetGlobalMcpData, UpdateGlobalMcpData, UpdateGlobalMcpResponse, RefreshRouteTableData, RefreshRouteTableResponse, RebuildSandboxImageData, GetGlobalSandboxStatusData, ListSecretsData, UpsertSecretData, UpsertSecretResponse, DeleteSecretData, DeleteSecretResponse, ListGlobalSkillsData, CreateGlobalSkillData, CreateGlobalSkillResponse, UploadGlobalSkillData, UploadGlobalSkillResponse, DeleteGlobalSkillData, DeleteGlobalSkillResponse, GetGlobalSkillData, UpdateGlobalSkillData, UpdateGlobalSkillResponse, DownloadGlobalSkillArchiveData, ListProjectTemplatesData, ListProjectTemplateTagsData, GetProjectTemplateData, GetCurrentUserData, ListUsersData, CreateUserData, CreateUserResponse, UpdateSelfData, UpdateSelfResponse, DisableMfaData, DisableMfaResponse, SetupMfaData, SetupMfaResponse, VerifyAndEnableMfaData, VerifyAndEnableMfaResponse, DeleteUserData, DeleteUserResponse, UpdateUserData, UpdateUserResponse, RestoreUserData, RestoreUserResponse, AssignRoleData, RemoveRoleData, RemoveRoleResponse, ListSandboxesData, ListSandboxesResponse2 as ListSandboxesResponse, CreateSandboxData, CreateSandboxResponse, GetSandboxData, CmdData, CmdResponse2 as CmdResponse, GetCmdData, CmdLogsData, DestroySandboxData, DestroySandboxResponse, DomainData, ExecData, ExecResponse2 as ExecResponse, ExecDetachedData, ExecDetachedResponse2 as ExecDetachedResponse, ExtendTimeoutData, ExtendTimeoutResponse, MkdirData, MkdirResponse, ReadFileData, StatPathData, WriteFileData, WriteFileResponse, WriteFilesData, WriteFilesResponse2 as WriteFilesResponse, ListJobsData, JobStatusData, KillJobData, KillJobResponse, JobLogsData, PauseSandboxData, PauseSandboxResponse, ClearPreviewPasswordData, ClearPreviewPasswordResponse, SetPreviewPasswordData, SetPreviewPasswordResponse2 as SetPreviewPasswordResponse, RestartSandboxData, RestartSandboxResponse, ResumeSandboxData, ResumeSandboxResponse, SourceSandboxData, SourceSandboxResponse, StopSandboxData, StopSandboxResponse, CmdKillData, CmdKillResponse, GetVisitorSessionsData, GetVisitorSessionsError, GetVisitorSessionsResponse2 as GetVisitorSessionsResponse, DeleteSessionReplayData, DeleteSessionReplayError, GetSessionReplayData, UpdateSessionDurationData, UpdateSessionDurationError, UpdateSessionDurationResponse2 as UpdateSessionDurationResponse, GetSessionReplayEventsData, AddEventsData, AddEventsError, AddEventsResponse2 as AddEventsResponse, DeleteScanData, DeleteScanError, DeleteScanResponse, GetScanData, GetScanVulnerabilitiesData, GetScanVulnerabilitiesError, GetScanVulnerabilitiesResponse, ListEventTypesData, TriggerWeeklyDigestData, TriggerWeeklyDigestResponse, ListExternalPluginsData, ReloadPluginsData, ReloadPluginsResponse, IngestSentryEnvelopeData, IngestSentryEventData, IngestSentryEventResponse, ListAuditLogsData, ListAuditLogsResponse, GetAuditLogData } from '../types.gen'; +import type { GetPlatformInfoData, ChunkUploadOptionsData, CreateReleaseData, CreateReleaseResponse, CreateProjectReleaseData, CreateProjectReleaseResponse, FinalizeProjectReleaseData, FinalizeProjectReleaseResponse, ListReleaseFilesData, UploadReleaseFileData, UploadReleaseFileResponse, RecordEventMetricsData, RecordEventMetricsResponse, AddSessionReplayEventsData, AddSessionReplayEventsError, AddSessionReplayEventsResponse, InitSessionReplayData, InitSessionReplayError, InitSessionReplayResponse, RecordSpeedMetricsData, RecordSpeedMetricsError, RecordSpeedMetricsResponse, UpdateSpeedMetricsData, UpdateSpeedMetricsError, UpdateSpeedMetricsResponse, WebhookTriggerData, WebhookTriggerResponse2 as WebhookTriggerResponse, GetPricingData, ListProviderKeysData, CreateProviderKeyData, CreateProviderKeyError, CreateProviderKeyResponse, TestProviderKeyInlineData, TestProviderKeyInlineError, TestProviderKeyInlineResponse, DeleteProviderKeyData, DeleteProviderKeyError, DeleteProviderKeyResponse, UpdateProviderKeyData, UpdateProviderKeyError, UpdateProviderKeyResponse, TestProviderKeyByIdData, TestProviderKeyByIdError, TestProviderKeyByIdResponse, GetUsageByProviderData, GetConversationsData, GetConversationDetailData, GetUsageRecentData, GetUsageSummaryData, GetUsageTimeseriesData, GetUsageTopModelsData, ChatCompletionsData, ChatCompletionsError, ChatCompletionsResponse, EmbeddingsData, EmbeddingsError, EmbeddingsResponse, ListModelsData, GetAnalyticsActiveVisitorsData, GetEventDetailData, GetEventVisitorsData, GetEventVisitorsResponse, GetAnalyticsEventsCountData, GetGeneralStatsData, GetLiveVisitorsListData, GetPageFlowData, GetPageHourlySessionsData, GetPagePathDetailData, GetPagePathVisitorsData, GetPagePathVisitorsResponse, GetPagePathsData, GetPagePathsSparklinesData, GetRecentActivityData, GetSessionDetailsData, GetAnalyticsSessionEventsData, GetAnalyticsSessionEventsResponse, GetSessionLogsData, GetSessionLogsResponse, GetVisitorsData, GetVisitorsResponse, GetVisitorByGuidData, GetVisitorByIdData, GetVisitorDetailsData, EnrichVisitorData, EnrichVisitorResponse2 as EnrichVisitorResponse, GetVisitorInfoData, GetVisitorJourneyData, GetAnalyticsVisitorSessionsData, GetVisitorStatsData, ListApiKeysData, ListApiKeysResponse, CreateApiKeyData, CreateApiKeyResponse2 as CreateApiKeyResponse, GetApiKeyPermissionsData, DeleteApiKeyData, DeleteApiKeyResponse, GetApiKeyData, UpdateApiKeyData, UpdateApiKeyResponse, ActivateApiKeyData, ActivateApiKeyResponse, DeactivateApiKeyData, DeactivateApiKeyResponse, CliLoginData, CliLoginResponse2 as CliLoginResponse, CliLogoutData, CliLogoutResponse, EmailStatusData, LoginData, LoginResponse, RequestMagicLinkData, RequestMagicLinkResponse, VerifyMagicLinkData, RequestPasswordResetData, RequestPasswordResetResponse, ResetPasswordData, ResetPasswordResponse, VerifyEmailData, VerifyMfaChallengeData, VerifyMfaChallengeResponse, RunExternalServiceBackupData, RunExternalServiceBackupError, RunExternalServiceBackupResponse, ListS3SourcesData, CreateS3SourceData, CreateS3SourceError, CreateS3SourceResponse, TestS3ConnectionPreviewData, TestS3ConnectionPreviewError, TestS3ConnectionPreviewResponse, DeleteS3SourceData, DeleteS3SourceError, DeleteS3SourceResponse, GetS3SourceData, UpdateS3SourceData, UpdateS3SourceError, UpdateS3SourceResponse, ListSourceBackupsData, RunBackupForSourceData, RunBackupForSourceError, RunBackupForSourceResponse, SetDefaultS3SourceData, SetDefaultS3SourceError, SetDefaultS3SourceResponse, TestS3SourceConnectionData, TestS3SourceConnectionError, TestS3SourceConnectionResponse, ListBackupSchedulesData, CreateBackupScheduleData, CreateBackupScheduleError, CreateBackupScheduleResponse, DeleteBackupScheduleData, DeleteBackupScheduleError, DeleteBackupScheduleResponse, GetBackupScheduleData, ListBackupsForScheduleData, DisableBackupScheduleData, DisableBackupScheduleResponse, EnableBackupScheduleData, EnableBackupScheduleResponse, GetBackupData, BlobDeleteData, BlobDeleteError, BlobDeleteResponse, BlobListData, BlobListError, BlobListResponse, BlobPutData, BlobPutError, BlobPutResponse, BlobCopyData, BlobCopyError, BlobCopyResponse, BlobDisableData, BlobDisableResponse, BlobEnableData, BlobEnableResponse, BlobStatusData, BlobUpdateData, BlobUpdateResponse, BlobDownloadData, GetDashboardProjectsAnalyticsData, GetActivityGraphData, GetScanByDeploymentData, ListDnsProvidersData, CreateDnsProviderData, CreateDnsProviderResponse, DeleteDnsProviderData, DeleteDnsProviderResponse, GetDnsProviderData, UpdateProviderData, UpdateProviderResponse, ListManagedDomainsData, AddManagedDomainData, AddManagedDomainResponse, TestProviderConnectionData, TestProviderConnectionResponse, ListProviderZonesData, RemoveManagedDomainData, RemoveManagedDomainResponse, VerifyManagedDomainData, VerifyManagedDomainResponse, LookupDnsARecordsData, ListDomainsData, ListDomainsResponse2 as ListDomainsResponse, CreateDomainData, CreateDomainResponse, GetDomainByHostData, CancelDomainOrderData, CancelDomainOrderResponse, GetDomainOrderData, CreateOrRecreateOrderData, CreateOrRecreateOrderResponse, FinalizeOrderData, FinalizeOrderResponse, SetupDnsChallengeData, SetupDnsChallengeResponse2 as SetupDnsChallengeResponse, DeleteDomainData, DeleteDomainResponse, GetDomainByIdData, GetChallengeTokenData, GetHttpChallengeDebugData, ProvisionDomainData, ProvisionDomainResponse, RenewDomainData, RenewDomainResponse, CheckDomainStatusData, ListEmailDomainsData, CreateEmailDomainData, CreateEmailDomainResponse, GetDomainByNameData, DeleteEmailDomainData, DeleteEmailDomainResponse, GetDomainData, GetDomainDnsRecordsData, SetupDnsData, SetupDnsResponse2 as SetupDnsResponse, VerifyDomainData, VerifyDomainResponse, ListEmailProvidersData, CreateEmailProviderData, CreateEmailProviderResponse, DeleteEmailProviderData, DeleteEmailProviderResponse, GetEmailProviderData, TestProviderData, TestProviderResponse2 as TestProviderResponse, ListEmailsData, ListEmailsResponse, SendEmailData, SendEmailResponse, GetGlobalEventsData, GetGlobalEventsResponse, GetGlobalEventStatsData, GetEmailStatsData, ValidateEmailData, ValidateEmailResponse2 as ValidateEmailResponse, TrackClickData, TrackOpenData, GetEmailData, GetEmailTrackingData, GetEmailEventsData, GetEmailLinksData, ListServicesData, ListServicesResponse, CreateServiceData, CreateServiceResponse, ListAvailableContainersData, GetServiceBySlugData, ListServiceHealthStatusesData, ImportExternalServiceData, ImportExternalServiceResponse, ListProjectServicesData, ListProjectServicesResponse, GetProjectServiceEnvironmentVariablesData, GetProvidersMetadataData, GetProviderMetadataData, GetServiceTypesData, GetServiceTypeParametersData, DeleteServiceData, DeleteServiceResponse, GetServiceData, UpdateServiceData, UpdateServiceResponse, GetClusterHealthData, TriggerServiceHealthCheckData, TriggerServiceHealthCheckResponse, GetServiceHealthStatusData, AddClusterMemberData, AddClusterMemberResponse, RemoveClusterMemberData, RemoveClusterMemberResponse, GetClusterMemberData, PromoteClusterMemberData, GetServicePreviewEnvironmentVariablesMaskedData, GetServicePreviewEnvironmentVariableNamesData, ListServiceProjectsData, ListServiceProjectsResponse, LinkServiceToProjectData, LinkServiceToProjectResponse, UnlinkServiceFromProjectData, UnlinkServiceFromProjectResponse, GetServiceEnvironmentVariablesData, GetServiceEnvironmentVariableData, StartRestoreData, StartRestoreError, StartRestoreResponse, GetRestoreCapabilitiesData, PlanRestoreData, PlanRestoreError, PlanRestoreResponse, ListRestoreRunsForServiceData, RetryClusterData, RetryClusterResponse, StartServiceData, StartServiceResponse, StopServiceData, StopServiceResponse, UpgradeServiceData, UpgradeServiceResponse, ListRootContainersData, ListContainersAtPathData, ListEntitiesData, GetEntityInfoData, QueryDataData, QueryDataResponse2 as QueryDataResponse, DownloadObjectData, GetContainerInfoData, CheckExplorerSupportData, ListPgUpgradesData, StartPgUpgradeData, StartPgUpgradeResponse, GetPgUpgradeData, CancelPgUpgradeData, CancelPgUpgradeResponse, GetPgUpgradeLogsData, RetryPgUpgradeData, RetryPgUpgradeResponse, RollbackPgUpgradeData, RollbackPgUpgradeResponse, GetFileData, GetIpGeolocationData, ListConnectionsData, ListConnectionsResponse, DeleteConnectionData, DeleteConnectionResponse, ActivateConnectionData, DeactivateConnectionData, RunConnectionHealthCheckData, RunConnectionHealthCheckResponse, ListRepositoriesByConnectionData, ListRepositoriesByConnectionResponse, SyncRepositoriesData, SyncRepositoriesResponse, UpdateConnectionTokenData, UpdateConnectionTokenResponse, ValidateConnectionData, ListGitProvidersData, CreateGitProviderData, CreateGitProviderResponse, CreateGithubPatProviderData, CreateGithubPatProviderResponse, CreateGitlabOauthProviderData, CreateGitlabOauthProviderResponse, CreateGitlabPatProviderData, CreateGitlabPatProviderResponse, DeleteGitProviderData, DeleteGitProviderResponse, GetGitProviderData, ActivateProviderData, HandleGitProviderOauthCallbackData, GetProviderConnectionsData, UpdateGitProviderCredentialsData, UpdateGitProviderCredentialsResponse, DeactivateProviderData, CheckProviderDeletionSafetyData, StartGitProviderOauthData, DeleteProviderSafelyData, DeleteProviderSafelyResponse, GetPublicRepositoryData, GetPublicBranchesData, DetectPublicPresetsData, DiscoverWorkloadsData, DiscoverWorkloadsResponse, ExecuteImportData, ExecuteImportResponse2 as ExecuteImportResponse, CreatePlanData, CreatePlanResponse2 as CreatePlanResponse, ListSourcesData, GetImportStatusData, GetIncidentData, UpdateIncidentStatusData, UpdateIncidentStatusResponse, GetIncidentUpdatesData, AdminListNodesData, RegisterNodeData, RegisterNodeResponse2 as RegisterNodeResponse, AdminRemoveNodeData, AdminRemoveNodeResponse, AdminGetNodeData, AdminListNodeContainersData, PostDnsAckData, PostDnsAckResponse, GetDnsChangesData, AdminUndrainNodeData, AdminUndrainNodeResponse, AdminDrainStatusData, AdminDrainNodeData, AdminDrainNodeResponse, NodeHeartbeatData, NodeHeartbeatResponse, ListPeersData, GetS3CredentialsData, ListIpAccessControlData, CreateIpAccessControlData, CreateIpAccessControlError, CreateIpAccessControlResponse, CheckIpBlockedData, DeleteIpAccessControlData, DeleteIpAccessControlError, DeleteIpAccessControlResponse, GetIpAccessControlData, UpdateIpAccessControlData, UpdateIpAccessControlError, UpdateIpAccessControlResponse, KvDelData, KvDelResponse, KvDisableData, KvDisableResponse, KvEnableData, KvEnableResponse, KvExpireData, KvExpireResponse, KvGetData, KvGetResponse, KvIncrData, KvIncrResponse, KvKeysData, KvKeysResponse, KvSetData, KvSetResponse, KvStatusData, KvTtlData, KvTtlResponse, KvUpdateData, KvUpdateResponse, ListRoutesData, CreateRouteData, CreateRouteResponse, DeleteRouteData, DeleteRouteResponse, GetRouteData, UpdateRouteData, UpdateRouteResponse, LogoutData, GetLogContextData, SearchLogsData, SearchLogsError, SearchLogsResponse2 as SearchLogsResponse, TailLogsData, GetProjectsMonitorHealthData, DeleteMonitorData, DeleteMonitorResponse, GetMonitorData, GetBucketedStatusData, GetCurrentMonitorStatusData, GetUptimeHistoryData, DeletePreferencesData, DeletePreferencesResponse, GetPreferencesData, UpdatePreferencesData, UpdatePreferencesResponse, ListNotificationProvidersData, ListNotificationProvidersResponse, CreateNotificationProviderData, CreateNotificationProviderResponse, CreateNotificationEmailProviderData, CreateNotificationEmailProviderResponse, UpdateEmailProviderData, UpdateEmailProviderResponse, CreateSlackProviderData, CreateSlackProviderResponse, UpdateSlackProviderData, UpdateSlackProviderResponse, CreateWebhookProviderData, CreateWebhookProviderResponse, UpdateWebhookProviderData, UpdateWebhookProviderResponse, DeleteNotificationProviderData, DeleteNotificationProviderResponse, GetNotificationProviderData, UpdateNotificationProviderData, UpdateNotificationProviderResponse, TestNotificationProviderData, TestNotificationProviderResponse, ListOrdersData, ListOrdersResponse2 as ListOrdersResponse, QueryGenaiTracesData, QueryGenaiTracesError, QueryGenaiTracesResponse, GetGenaiTraceData, GetHealthData, ListInsightsData, ListInsightsError, ListInsightsResponse, QueryLogsData, QueryLogsError, QueryLogsResponse, ListMetricNamesData, QueryMetricsData, GetPipelineStatsData, GetQuotaData, QueryTraceSummariesData, QueryTraceSummariesError, QueryTraceSummariesResponse, QueryTracesData, QueryTracesError, QueryTracesResponse, GetTraceData, IngestLogsData, IngestLogsError, IngestMetricsData, IngestMetricsError, IngestTracesData, IngestTracesError, IngestLogsByPathData, IngestLogsByPathError, IngestMetricsByPathData, IngestMetricsByPathError, IngestTracesByPathData, IngestTracesByPathError, HasPerformanceMetricsData, GetPerformanceMetricsData, GetMetricsOverTimeData, GetGroupedPageMetricsData, GetAccessInfoData, GetPrivateIpData, GetPublicIpData, ListPresetsData, GeneratePresetDockerfileData, GeneratePresetDockerfileResponse, GetPreviewGatewayLogsData, RestartPreviewGatewayData, RestartPreviewGatewayResponse, GetPreviewGatewaySettingsData, PatchPreviewGatewaySettingsData, PatchPreviewGatewaySettingsResponse, GetPreviewGatewayStatusData, UpgradePreviewGatewayData, UpgradePreviewGatewayResponse, GetProjectsData, GetProjectsResponse, CreateProjectData, CreateProjectResponse, GetProjectBySlugData, CreateProjectFromTemplateData, CreateProjectFromTemplateResponse2 as CreateProjectFromTemplateResponse, GetProjectStatisticsData, DeleteProjectData, DeleteProjectResponse, GetProjectData, UpdateProjectData, UpdateProjectResponse, GetProjectDeploymentsData, GetProjectDeploymentsResponse, GetLastDeploymentData, TriggerProjectPipelineData, TriggerProjectPipelineResponse, GetActiveVisitorsData, ListAgentsData, CreateAgentData, CreateAgentResponse, GetCliStatusData, ListAllRunsData, ListAllRunsResponse, LatestRunForSourceData, GetRunWithLogsData, CancelRunData, CancelRunResponse, RetryRunData, RetryRunResponse, GetSandboxStatusData, SmokeTestAgentData, SmokeTestAgentResponse, DeleteAgentData, DeleteAgentResponse, GetAgentData, UpdateAgentData, UpdateAgentResponse, ListAgentRunsData, ListAgentRunsResponse, TriggerAgentData, TriggerAgentResponse, GetAggregatedBucketsData, StartAnalysisData, StartAnalysisResponse, GetRunData, AddContextData, CancelData, CreatePrData, CreatePrResponse2 as CreatePrResponse, StartFixData, ReAnalyzeData, UpdateAutomaticDeployData, UpdateAutomaticDeployResponse, ListCustomDomainsForProjectData, CreateCustomDomainData, CreateCustomDomainResponse, DeleteCustomDomainData, DeleteCustomDomainResponse, GetCustomDomainData, UpdateCustomDomainData, UpdateCustomDomainResponse, LinkCustomDomainToCertificateData, LinkCustomDomainToCertificateResponse, UpdateProjectDeploymentConfigData, UpdateProjectDeploymentConfigResponse, GetDeploymentData, CancelDeploymentData, CancelDeploymentResponse, GetDeploymentJobsData, GetDeploymentJobLogsData, TailDeploymentJobLogsData, GetDeploymentOperationsData, ExecuteDeploymentOperationData, ExecuteDeploymentOperationResponse, GetDeploymentOperationStatusData, PauseDeploymentData, PauseDeploymentResponse, PromoteDeploymentData, PromoteDeploymentResponse, ResumeDeploymentData, ResumeDeploymentResponse, RollbackToDeploymentData, RollbackToDeploymentResponse, TeardownDeploymentData, TeardownDeploymentResponse, ListDsnsData, CreateDsnData, CreateDsnResponse, GetOrCreateDsnData, GetOrCreateDsnResponse, RegenerateDsnData, RegenerateDsnResponse, RevokeDsnData, RevokeDsnResponse, GetEnvironmentVariablesData, CreateEnvironmentVariableData, CreateEnvironmentVariableResponse, GetResolvedEnvironmentVariablesData, GetResolvedEnvironmentVariableValueData, GetEnvironmentVariableValueData, DeleteEnvironmentVariableData, DeleteEnvironmentVariableResponse, UpdateEnvironmentVariableData, UpdateEnvironmentVariableResponse, GetEnvironmentsData, CreateEnvironmentData, CreateEnvironmentResponse, DeleteEnvironmentData, DeleteEnvironmentResponse, GetEnvironmentData, GetEnvironmentCronsData, GetCronByIdData, GetCronExecutionsData, GetCronExecutionsResponse, GetEnvironmentDomainsData, AddEnvironmentDomainData, AddEnvironmentDomainResponse, DeleteEnvironmentDomainData, DeleteEnvironmentDomainResponse, UpdateEnvironmentSettingsData, UpdateEnvironmentSettingsResponse, SleepEnvironmentData, SleepEnvironmentResponse, TeardownEnvironmentData, TeardownEnvironmentResponse, WakeEnvironmentData, WakeEnvironmentResponse, GetContainerLogsData, ListContainersData, GetContainerDetailData, GetContainerLogsByIdData, GetContainerMetricsData, StreamContainerMetricsData, RestartContainerData, RestartContainerResponse, StartContainerData, StartContainerResponse, StopContainerData, StopContainerResponse, DeployFromImageData, DeployFromImageResponse, DeployFromImageUploadData, DeployFromImageUploadResponse, DeployFromStaticData, DeployFromStaticResponse, ListAlertRulesData, CreateAlertRuleData, CreateAlertRuleResponse, DeleteAlertRuleData, DeleteAlertRuleResponse, GetAlertRuleData, UpdateAlertRuleData, UpdateAlertRuleResponse, GetErrorDashboardStatsData, ListErrorGroupsData, ListErrorGroupsResponse, GetErrorGroupData, UpdateErrorGroupData, ListErrorEventsData, ListErrorEventsResponse, GetErrorEventData, GetErrorStatsData, GetErrorTimeSeriesData, GetEventsCountData, GetEventTypeBreakdownData, RecordConsoleEventData, GetPropertyBreakdownData, GetPropertyTimelineData, GetEventsTimelineData, GetUniqueEventsData, GetUniqueEventsResponse, ListRemoteExternalImagesData, ListRemoteExternalImagesResponse, RegisterExternalImageData, RegisterExternalImageResponse, DeleteExternalImageData, DeleteExternalImageResponse, GetRemoteExternalImageData, ListFunnelsData, CreateFunnelData, CreateFunnelResponse2 as CreateFunnelResponse, PreviewFunnelMetricsData, PreviewFunnelMetricsResponse, DeleteFunnelData, UpdateFunnelData, GetFunnelMetricsData, UpdateGitSettingsData, UpdateGitSettingsResponse, ReinstallGitlabWebhookData, ReinstallGitlabWebhookResponse, HasErrorGroupsData, HasAnalyticsEventsData, GetHourlyVisitsData, ListExternalImagesData, PushExternalImageData, PushExternalImageResponse, GetExternalImageData, ListIncidentsData, CreateIncidentData, CreateIncidentResponse, GetBucketedIncidentsData, PurgeProjectLogsData, PurgeProjectLogsError, ListMcpsData, CreateMcpData, CreateMcpResponse, DeleteMcpData, DeleteMcpResponse, GetMcpData, UpdateMcpData, UpdateMcpResponse, ListMonitorsData, CreateMonitorData, CreateMonitorResponse, ObservabilityListEventsData, ObservabilityFullEventData, DeleteReleaseSourceMapsData, DeleteReleaseSourceMapsResponse, ListSourceMapsData, UploadSourceMapData, UploadSourceMapResponse, RevenueRecentEventsData, RevenueListIntegrationsData, RevenueCreateIntegrationData, RevenueCreateIntegrationResponse, RevenueDeleteIntegrationData, RevenueDeleteIntegrationResponse, RevenueUpdateConfigData, RevenueUpdateConfigResponse, RevenueImportInvoicesCsvData, RevenueImportInvoicesCsvResponse, RevenueImportSubscriptionsCsvData, RevenueImportSubscriptionsCsvResponse, RevenueRotateTokenData, RevenueRotateTokenResponse, RevenueUpdateSecretData, RevenueUpdateSecretResponse, RevenueMetricsCustomersData, RevenueMetricsMrrData, RevenueMetricsSummaryData, ListProjectSecretsData, CreateProjectSecretData, CreateProjectSecretResponse, DeleteProjectSecretData, DeleteProjectSecretResponse, UpdateProjectSecretData, UpdateProjectSecretResponse, UpdateProjectSettingsData, UpdateProjectSettingsResponse, ListSkillsData, CreateSkillData, CreateSkillResponse, UploadSkillData, UploadSkillResponse, DeleteSkillData, DeleteSkillResponse, GetSkillData, UpdateSkillData, UpdateSkillResponse, DownloadSkillArchiveData, ListReleasesData, DeleteSourceMapData, DeleteSourceMapResponse, ListStaticBundlesData, ListStaticBundlesResponse, DeleteStaticBundleData, DeleteStaticBundleResponse, GetStaticBundleData, GetStatusOverviewData, GetUniqueCountsData, UploadStaticBundleData, UploadStaticBundleResponse, ListProjectScansData, ListProjectScansError, ListProjectScansResponse, TriggerScanData, TriggerScanError, TriggerScanResponse2 as TriggerScanResponse, GetLatestScansPerEnvironmentData, GetLatestScanData, ListWebhooksData, ListWebhooksResponse, CreateWebhookData, CreateWebhookResponse, DeleteWebhookData, DeleteWebhookResponse, GetWebhookData, UpdateWebhookData, UpdateWebhookResponse, ListDeliveriesData, GetDeliveryData, RetryDeliveryData, RetryDeliveryResponse, WorkflowDryRunData, WorkflowDryRunResponse, ListMemoryData, WriteMemoryData, WriteMemoryResponse, SearchMemoryData, DropMemoryData, DropMemoryResponse, SupersedeMemoryData, SupersedeMemoryResponse, WorkspaceListSessionsData, WorkspaceListSessionsResponse, WorkspaceStartSessionData, WorkspaceStartSessionResponse, WorkspaceDeleteSessionData, WorkspaceDeleteSessionResponse, WorkspaceGetSessionData, WorkspaceUpdateSessionData, WorkspaceUpdateSessionResponse, WorkspaceCancelRunData, WorkspaceCancelRunResponse, WorkspaceCloseSessionData, WorkspaceCloseSessionResponse, WorkspaceSendMessageData, WorkspaceSendMessageResponse, WorkspaceRegeneratePreviewPasswordData, WorkspaceRegeneratePreviewPasswordResponse, WorkspaceReopenSessionData, WorkspaceReopenSessionResponse, WorkspaceRefreshSandboxData, WorkspaceRefreshSandboxResponse, WorkspaceRestartSandboxData, WorkspaceRestartSandboxResponse, WorkspaceStartSandboxData, WorkspaceStartSandboxResponse, WorkspaceSandboxStatsData, WorkspaceStopSandboxData, WorkspaceStopSandboxResponse, WorkspaceTerminalPasteImageData, WorkspaceTerminalPasteImageResponse, WorkspaceListTerminalTabsData, WorkspaceDeleteTerminalTabData, WorkspaceDeleteTerminalTabResponse, GetProxyLogsData, GetProxyLogsResponse, GetProxyLogByRequestIdData, GetProjectsHealthData, GetTimeBucketStatsData, GetTodayStatsData, GetProxyLogByIdData, ListSyncedRepositoriesData, ListSyncedRepositoriesResponse, GetRepositoryByNameData, GetAllRepositoriesByNameData, GetRepositoryPresetByNameData, GetRepositoryBranchesData, GetRepositoryTagsData, GetRepositoryPresetLiveData, GetRepositoryByIdData, GetBranchesByRepositoryIdData, ListCommitsByRepositoryIdData, CheckCommitExistsData, GetTagsByRepositoryIdData, GetRestoreRunData, RevenueGlobalEventsData, RevenueMetricsGlobalMrrData, RevenueMetricsGlobalSummaryData, RevenueListProvidersData, GetProjectSessionReplaysData, GetProjectSessionReplaysError, GetProjectSessionReplaysResponse2 as GetProjectSessionReplaysResponse, GetSessionEventsData, GetSettingsData, UpdateSettingsData, UpdateSettingsResponse, SaveAgentTokenData, SaveAgentTokenResponse2 as SaveAgentTokenResponse, ListAiProvidersData, UpdateAiProviderData, UpdateAiProviderResponse2 as UpdateAiProviderResponse, ActivateAiProviderData, ActivateAiProviderResponse, SaveAiProviderCredentialData, SaveAiProviderCredentialResponse, RevokeJoinTokenData, RevokeJoinTokenResponse, GenerateJoinTokenData, GenerateJoinTokenResponse2 as GenerateJoinTokenResponse, GetJoinTokenStatusData, ListGlobalMcpsData, CreateGlobalMcpData, CreateGlobalMcpResponse, DeleteGlobalMcpData, DeleteGlobalMcpResponse, GetGlobalMcpData, UpdateGlobalMcpData, UpdateGlobalMcpResponse, RefreshRouteTableData, RefreshRouteTableResponse, RebuildSandboxImageData, GetGlobalSandboxStatusData, ListSecretsData, UpsertSecretData, UpsertSecretResponse, DeleteSecretData, DeleteSecretResponse, ListGlobalSkillsData, CreateGlobalSkillData, CreateGlobalSkillResponse, UploadGlobalSkillData, UploadGlobalSkillResponse, DeleteGlobalSkillData, DeleteGlobalSkillResponse, GetGlobalSkillData, UpdateGlobalSkillData, UpdateGlobalSkillResponse, DownloadGlobalSkillArchiveData, ListProjectTemplatesData, ListProjectTemplateTagsData, GetProjectTemplateData, GetCurrentUserData, ListUsersData, CreateUserData, CreateUserResponse, UpdateSelfData, UpdateSelfResponse, DisableMfaData, DisableMfaResponse, SetupMfaData, SetupMfaResponse, VerifyAndEnableMfaData, VerifyAndEnableMfaResponse, DeleteUserData, DeleteUserResponse, UpdateUserData, UpdateUserResponse, RestoreUserData, RestoreUserResponse, AssignRoleData, RemoveRoleData, RemoveRoleResponse, ListSandboxesData, ListSandboxesResponse2 as ListSandboxesResponse, CreateSandboxData, CreateSandboxResponse, GetSandboxData, CmdData, CmdResponse2 as CmdResponse, GetCmdData, CmdLogsData, DestroySandboxData, DestroySandboxResponse, DomainData, ExecData, ExecResponse2 as ExecResponse, ExecDetachedData, ExecDetachedResponse2 as ExecDetachedResponse, ExtendTimeoutData, ExtendTimeoutResponse, MkdirData, MkdirResponse, ReadFileData, StatPathData, WriteFileData, WriteFileResponse, WriteFilesData, WriteFilesResponse2 as WriteFilesResponse, ListJobsData, JobStatusData, KillJobData, KillJobResponse, JobLogsData, PauseSandboxData, PauseSandboxResponse, ClearPreviewPasswordData, ClearPreviewPasswordResponse, SetPreviewPasswordData, SetPreviewPasswordResponse2 as SetPreviewPasswordResponse, RestartSandboxData, RestartSandboxResponse, ResumeSandboxData, ResumeSandboxResponse, SourceSandboxData, SourceSandboxResponse, StopSandboxData, StopSandboxResponse, CmdKillData, CmdKillResponse, GetVisitorSessionsData, GetVisitorSessionsError, GetVisitorSessionsResponse2 as GetVisitorSessionsResponse, DeleteSessionReplayData, DeleteSessionReplayError, GetSessionReplayData, UpdateSessionDurationData, UpdateSessionDurationError, UpdateSessionDurationResponse2 as UpdateSessionDurationResponse, GetSessionReplayEventsData, AddEventsData, AddEventsError, AddEventsResponse2 as AddEventsResponse, DeleteScanData, DeleteScanError, DeleteScanResponse, GetScanData, GetScanVulnerabilitiesData, GetScanVulnerabilitiesError, GetScanVulnerabilitiesResponse, ListEventTypesData, TriggerWeeklyDigestData, TriggerWeeklyDigestResponse, ListExternalPluginsData, ReloadPluginsData, ReloadPluginsResponse, IngestSentryEnvelopeData, IngestSentryEventData, IngestSentryEventResponse, ListAuditLogsData, ListAuditLogsResponse, GetAuditLogData } from '../types.gen'; import { client } from '../client.gen'; export type QueryKey = [ @@ -9889,6 +9889,53 @@ export const createMonitorMutation = (options?: Partial) => createQueryKey('observabilityListEvents', options); + +/** + * List a merged page of observability events for a project. + * Each row carries everything the side panel needs to render — no + * follow-up fetch is required for the common case. Heavy fields + * (stacktraces, headers, span attributes) are truncated server-side and + * expose a `*_truncated` flag; clients fetch the full row from the + * `/full` endpoint only when the user explicitly clicks "Show full". + */ +export const observabilityListEventsOptions = (options: Options) => { + return queryOptions({ + queryFn: async ({ queryKey, signal }) => { + const { data } = await observabilityListEvents({ + ...options, + ...queryKey[0], + signal, + throwOnError: true + }); + return data; + }, + queryKey: observabilityListEventsQueryKey(options) + }); +}; + +export const observabilityFullEventQueryKey = (options: Options) => createQueryKey('observabilityFullEvent', options); + +/** + * Fetch the un-truncated form of one event by `(kind, id)`. Side panel + * "Show full" action calls this — the list response carries truncated + * previews + a `*_truncated` flag to let the UI decide whether to fetch. + */ +export const observabilityFullEventOptions = (options: Options) => { + return queryOptions({ + queryFn: async ({ queryKey, signal }) => { + const { data } = await observabilityFullEvent({ + ...options, + ...queryKey[0], + signal, + throwOnError: true + }); + return data; + }, + queryKey: observabilityFullEventQueryKey(options) + }); +}; + /** * Delete all source maps for a specific release */ diff --git a/web/src/api/client/sdk.gen.ts b/web/src/api/client/sdk.gen.ts index e99b07e7b..eb7f5c347 100644 --- a/web/src/api/client/sdk.gen.ts +++ b/web/src/api/client/sdk.gen.ts @@ -1,7 +1,7 @@ // This file is auto-generated by @hey-api/openapi-ts import { type Options as ClientOptions, type Client, type TDataShape, formDataBodySerializer } from './client'; -import type { GetPlatformInfoData, GetPlatformInfoResponses, GetPlatformInfoErrors, ChunkUploadOptionsData, ChunkUploadOptionsResponses, CreateReleaseData, CreateReleaseResponses, CreateReleaseErrors, CreateProjectReleaseData, CreateProjectReleaseResponses, CreateProjectReleaseErrors, FinalizeProjectReleaseData, FinalizeProjectReleaseResponses, FinalizeProjectReleaseErrors, ListReleaseFilesData, ListReleaseFilesResponses, ListReleaseFilesErrors, UploadReleaseFileData, UploadReleaseFileResponses, UploadReleaseFileErrors, RecordEventMetricsData, RecordEventMetricsResponses, RecordEventMetricsErrors, AddSessionReplayEventsData, AddSessionReplayEventsResponses, AddSessionReplayEventsErrors, InitSessionReplayData, InitSessionReplayResponses, InitSessionReplayErrors, RecordSpeedMetricsData, RecordSpeedMetricsResponses, RecordSpeedMetricsErrors, UpdateSpeedMetricsData, UpdateSpeedMetricsResponses, UpdateSpeedMetricsErrors, WebhookTriggerData, WebhookTriggerResponses, WebhookTriggerErrors, GetPricingData, GetPricingResponses, GetPricingErrors, ListProviderKeysData, ListProviderKeysResponses, ListProviderKeysErrors, CreateProviderKeyData, CreateProviderKeyResponses, CreateProviderKeyErrors, TestProviderKeyInlineData, TestProviderKeyInlineResponses, TestProviderKeyInlineErrors, DeleteProviderKeyData, DeleteProviderKeyResponses, DeleteProviderKeyErrors, UpdateProviderKeyData, UpdateProviderKeyResponses, UpdateProviderKeyErrors, TestProviderKeyByIdData, TestProviderKeyByIdResponses, TestProviderKeyByIdErrors, GetUsageByProviderData, GetUsageByProviderResponses, GetUsageByProviderErrors, GetConversationsData, GetConversationsResponses, GetConversationsErrors, GetConversationDetailData, GetConversationDetailResponses, GetConversationDetailErrors, GetUsageRecentData, GetUsageRecentResponses, GetUsageRecentErrors, GetUsageSummaryData, GetUsageSummaryResponses, GetUsageSummaryErrors, GetUsageTimeseriesData, GetUsageTimeseriesResponses, GetUsageTimeseriesErrors, GetUsageTopModelsData, GetUsageTopModelsResponses, GetUsageTopModelsErrors, ChatCompletionsData, ChatCompletionsResponses, ChatCompletionsErrors, EmbeddingsData, EmbeddingsResponses, EmbeddingsErrors, ListModelsData, ListModelsResponses, ListModelsErrors, GetAnalyticsActiveVisitorsData, GetAnalyticsActiveVisitorsResponses, GetAnalyticsActiveVisitorsErrors, GetEventDetailData, GetEventDetailResponses, GetEventDetailErrors, GetEventVisitorsData, GetEventVisitorsResponses, GetEventVisitorsErrors, GetAnalyticsEventsCountData, GetAnalyticsEventsCountResponses, GetAnalyticsEventsCountErrors, GetGeneralStatsData, GetGeneralStatsResponses, GetGeneralStatsErrors, GetLiveVisitorsListData, GetLiveVisitorsListResponses, GetLiveVisitorsListErrors, GetPageFlowData, GetPageFlowResponses, GetPageFlowErrors, GetPageHourlySessionsData, GetPageHourlySessionsResponses, GetPageHourlySessionsErrors, GetPagePathDetailData, GetPagePathDetailResponses, GetPagePathDetailErrors, GetPagePathVisitorsData, GetPagePathVisitorsResponses, GetPagePathVisitorsErrors, GetPagePathsData, GetPagePathsResponses, GetPagePathsErrors, GetPagePathsSparklinesData, GetPagePathsSparklinesResponses, GetPagePathsSparklinesErrors, GetRecentActivityData, GetRecentActivityResponses, GetRecentActivityErrors, GetSessionDetailsData, GetSessionDetailsResponses, GetSessionDetailsErrors, GetAnalyticsSessionEventsData, GetAnalyticsSessionEventsResponses, GetAnalyticsSessionEventsErrors, GetSessionLogsData, GetSessionLogsResponses, GetSessionLogsErrors, GetVisitorsData, GetVisitorsResponses, GetVisitorsErrors, GetVisitorByGuidData, GetVisitorByGuidResponses, GetVisitorByGuidErrors, GetVisitorByIdData, GetVisitorByIdResponses, GetVisitorByIdErrors, GetVisitorDetailsData, GetVisitorDetailsResponses, GetVisitorDetailsErrors, EnrichVisitorData, EnrichVisitorResponses, EnrichVisitorErrors, GetVisitorInfoData, GetVisitorInfoResponses, GetVisitorInfoErrors, GetVisitorJourneyData, GetVisitorJourneyResponses, GetVisitorJourneyErrors, GetAnalyticsVisitorSessionsData, GetAnalyticsVisitorSessionsResponses, GetAnalyticsVisitorSessionsErrors, GetVisitorStatsData, GetVisitorStatsResponses, GetVisitorStatsErrors, ListApiKeysData, ListApiKeysResponses, ListApiKeysErrors, CreateApiKeyData, CreateApiKeyResponses, CreateApiKeyErrors, GetApiKeyPermissionsData, GetApiKeyPermissionsResponses, GetApiKeyPermissionsErrors, DeleteApiKeyData, DeleteApiKeyResponses, DeleteApiKeyErrors, GetApiKeyData, GetApiKeyResponses, GetApiKeyErrors, UpdateApiKeyData, UpdateApiKeyResponses, UpdateApiKeyErrors, ActivateApiKeyData, ActivateApiKeyResponses, ActivateApiKeyErrors, DeactivateApiKeyData, DeactivateApiKeyResponses, DeactivateApiKeyErrors, CliLoginData, CliLoginResponses, CliLoginErrors, CliLogoutData, CliLogoutResponses, CliLogoutErrors, EmailStatusData, EmailStatusResponses, EmailStatusErrors, LoginData, LoginResponses, LoginErrors, RequestMagicLinkData, RequestMagicLinkResponses, RequestMagicLinkErrors, VerifyMagicLinkData, VerifyMagicLinkResponses, VerifyMagicLinkErrors, RequestPasswordResetData, RequestPasswordResetResponses, RequestPasswordResetErrors, ResetPasswordData, ResetPasswordResponses, ResetPasswordErrors, VerifyEmailData, VerifyEmailResponses, VerifyEmailErrors, VerifyMfaChallengeData, VerifyMfaChallengeResponses, VerifyMfaChallengeErrors, RunExternalServiceBackupData, RunExternalServiceBackupResponses, RunExternalServiceBackupErrors, ListS3SourcesData, ListS3SourcesResponses, ListS3SourcesErrors, CreateS3SourceData, CreateS3SourceResponses, CreateS3SourceErrors, TestS3ConnectionPreviewData, TestS3ConnectionPreviewResponses, TestS3ConnectionPreviewErrors, DeleteS3SourceData, DeleteS3SourceResponses, DeleteS3SourceErrors, GetS3SourceData, GetS3SourceResponses, GetS3SourceErrors, UpdateS3SourceData, UpdateS3SourceResponses, UpdateS3SourceErrors, ListSourceBackupsData, ListSourceBackupsResponses, ListSourceBackupsErrors, RunBackupForSourceData, RunBackupForSourceResponses, RunBackupForSourceErrors, SetDefaultS3SourceData, SetDefaultS3SourceResponses, SetDefaultS3SourceErrors, TestS3SourceConnectionData, TestS3SourceConnectionResponses, TestS3SourceConnectionErrors, ListBackupSchedulesData, ListBackupSchedulesResponses, ListBackupSchedulesErrors, CreateBackupScheduleData, CreateBackupScheduleResponses, CreateBackupScheduleErrors, DeleteBackupScheduleData, DeleteBackupScheduleResponses, DeleteBackupScheduleErrors, GetBackupScheduleData, GetBackupScheduleResponses, GetBackupScheduleErrors, ListBackupsForScheduleData, ListBackupsForScheduleResponses, ListBackupsForScheduleErrors, DisableBackupScheduleData, DisableBackupScheduleResponses, DisableBackupScheduleErrors, EnableBackupScheduleData, EnableBackupScheduleResponses, EnableBackupScheduleErrors, GetBackupData, GetBackupResponses, GetBackupErrors, BlobDeleteData, BlobDeleteResponses, BlobDeleteErrors, BlobListData, BlobListResponses, BlobListErrors, BlobPutData, BlobPutResponses, BlobPutErrors, BlobCopyData, BlobCopyResponses, BlobCopyErrors, BlobDisableData, BlobDisableResponses, BlobDisableErrors, BlobEnableData, BlobEnableResponses, BlobEnableErrors, BlobStatusData, BlobStatusResponses, BlobStatusErrors, BlobUpdateData, BlobUpdateResponses, BlobUpdateErrors, BlobDownloadData, BlobDownloadResponses, BlobDownloadErrors, BlobHeadData, BlobHeadResponses, BlobHeadErrors, GetDashboardProjectsAnalyticsData, GetDashboardProjectsAnalyticsResponses, GetDashboardProjectsAnalyticsErrors, GetActivityGraphData, GetActivityGraphResponses, GetActivityGraphErrors, GetScanByDeploymentData, GetScanByDeploymentResponses, GetScanByDeploymentErrors, ListDnsProvidersData, ListDnsProvidersResponses, ListDnsProvidersErrors, CreateDnsProviderData, CreateDnsProviderResponses, CreateDnsProviderErrors, DeleteDnsProviderData, DeleteDnsProviderResponses, DeleteDnsProviderErrors, GetDnsProviderData, GetDnsProviderResponses, GetDnsProviderErrors, UpdateProviderData, UpdateProviderResponses, UpdateProviderErrors, ListManagedDomainsData, ListManagedDomainsResponses, ListManagedDomainsErrors, AddManagedDomainData, AddManagedDomainResponses, AddManagedDomainErrors, TestProviderConnectionData, TestProviderConnectionResponses, TestProviderConnectionErrors, ListProviderZonesData, ListProviderZonesResponses, ListProviderZonesErrors, RemoveManagedDomainData, RemoveManagedDomainResponses, RemoveManagedDomainErrors, VerifyManagedDomainData, VerifyManagedDomainResponses, VerifyManagedDomainErrors, LookupDnsARecordsData, LookupDnsARecordsResponses, LookupDnsARecordsErrors, ListDomainsData, ListDomainsResponses, ListDomainsErrors, CreateDomainData, CreateDomainResponses, CreateDomainErrors, GetDomainByHostData, GetDomainByHostResponses, GetDomainByHostErrors, CancelDomainOrderData, CancelDomainOrderResponses, CancelDomainOrderErrors, GetDomainOrderData, GetDomainOrderResponses, GetDomainOrderErrors, CreateOrRecreateOrderData, CreateOrRecreateOrderResponses, CreateOrRecreateOrderErrors, FinalizeOrderData, FinalizeOrderResponses, FinalizeOrderErrors, SetupDnsChallengeData, SetupDnsChallengeResponses, SetupDnsChallengeErrors, DeleteDomainData, DeleteDomainResponses, DeleteDomainErrors, GetDomainByIdData, GetDomainByIdResponses, GetDomainByIdErrors, GetChallengeTokenData, GetChallengeTokenResponses, GetChallengeTokenErrors, GetHttpChallengeDebugData, GetHttpChallengeDebugResponses, GetHttpChallengeDebugErrors, ProvisionDomainData, ProvisionDomainResponses, ProvisionDomainErrors, RenewDomainData, RenewDomainResponses, RenewDomainErrors, CheckDomainStatusData, CheckDomainStatusResponses, CheckDomainStatusErrors, ListEmailDomainsData, ListEmailDomainsResponses, ListEmailDomainsErrors, CreateEmailDomainData, CreateEmailDomainResponses, CreateEmailDomainErrors, GetDomainByNameData, GetDomainByNameResponses, GetDomainByNameErrors, DeleteEmailDomainData, DeleteEmailDomainResponses, DeleteEmailDomainErrors, GetDomainData, GetDomainResponses, GetDomainErrors, GetDomainDnsRecordsData, GetDomainDnsRecordsResponses, GetDomainDnsRecordsErrors, SetupDnsData, SetupDnsResponses, SetupDnsErrors, VerifyDomainData, VerifyDomainResponses, VerifyDomainErrors, ListEmailProvidersData, ListEmailProvidersResponses, ListEmailProvidersErrors, CreateEmailProviderData, CreateEmailProviderResponses, CreateEmailProviderErrors, DeleteEmailProviderData, DeleteEmailProviderResponses, DeleteEmailProviderErrors, GetEmailProviderData, GetEmailProviderResponses, GetEmailProviderErrors, TestProviderData, TestProviderResponses, TestProviderErrors, ListEmailsData, ListEmailsResponses, ListEmailsErrors, SendEmailData, SendEmailResponses, SendEmailErrors, GetGlobalEventsData, GetGlobalEventsResponses, GetGlobalEventsErrors, GetGlobalEventStatsData, GetGlobalEventStatsResponses, GetGlobalEventStatsErrors, GetEmailStatsData, GetEmailStatsResponses, GetEmailStatsErrors, ValidateEmailData, ValidateEmailResponses, ValidateEmailErrors, TrackClickData, TrackClickErrors, TrackOpenData, TrackOpenResponses, TrackOpenErrors, GetEmailData, GetEmailResponses, GetEmailErrors, GetEmailTrackingData, GetEmailTrackingResponses, GetEmailTrackingErrors, GetEmailEventsData, GetEmailEventsResponses, GetEmailEventsErrors, GetEmailLinksData, GetEmailLinksResponses, GetEmailLinksErrors, ListServicesData, ListServicesResponses, ListServicesErrors, CreateServiceData, CreateServiceResponses, CreateServiceErrors, ListAvailableContainersData, ListAvailableContainersResponses, ListAvailableContainersErrors, GetServiceBySlugData, GetServiceBySlugResponses, GetServiceBySlugErrors, ListServiceHealthStatusesData, ListServiceHealthStatusesResponses, ListServiceHealthStatusesErrors, ImportExternalServiceData, ImportExternalServiceResponses, ImportExternalServiceErrors, ListProjectServicesData, ListProjectServicesResponses, ListProjectServicesErrors, GetProjectServiceEnvironmentVariablesData, GetProjectServiceEnvironmentVariablesResponses, GetProjectServiceEnvironmentVariablesErrors, GetProvidersMetadataData, GetProvidersMetadataResponses, GetProvidersMetadataErrors, GetProviderMetadataData, GetProviderMetadataResponses, GetProviderMetadataErrors, GetServiceTypesData, GetServiceTypesResponses, GetServiceTypesErrors, GetServiceTypeParametersData, GetServiceTypeParametersResponses, GetServiceTypeParametersErrors, DeleteServiceData, DeleteServiceResponses, DeleteServiceErrors, GetServiceData, GetServiceResponses, GetServiceErrors, UpdateServiceData, UpdateServiceResponses, UpdateServiceErrors, GetClusterHealthData, GetClusterHealthResponses, GetClusterHealthErrors, TriggerServiceHealthCheckData, TriggerServiceHealthCheckResponses, TriggerServiceHealthCheckErrors, GetServiceHealthStatusData, GetServiceHealthStatusResponses, GetServiceHealthStatusErrors, AddClusterMemberData, AddClusterMemberResponses, AddClusterMemberErrors, RemoveClusterMemberData, RemoveClusterMemberResponses, RemoveClusterMemberErrors, GetClusterMemberData, GetClusterMemberResponses, GetClusterMemberErrors, PromoteClusterMemberData, PromoteClusterMemberResponses, PromoteClusterMemberErrors, GetServicePreviewEnvironmentVariablesMaskedData, GetServicePreviewEnvironmentVariablesMaskedResponses, GetServicePreviewEnvironmentVariablesMaskedErrors, GetServicePreviewEnvironmentVariableNamesData, GetServicePreviewEnvironmentVariableNamesResponses, GetServicePreviewEnvironmentVariableNamesErrors, ListServiceProjectsData, ListServiceProjectsResponses, ListServiceProjectsErrors, LinkServiceToProjectData, LinkServiceToProjectResponses, LinkServiceToProjectErrors, UnlinkServiceFromProjectData, UnlinkServiceFromProjectResponses, UnlinkServiceFromProjectErrors, GetServiceEnvironmentVariablesData, GetServiceEnvironmentVariablesResponses, GetServiceEnvironmentVariablesErrors, GetServiceEnvironmentVariableData, GetServiceEnvironmentVariableResponses, GetServiceEnvironmentVariableErrors, StartRestoreData, StartRestoreResponses, StartRestoreErrors, GetRestoreCapabilitiesData, GetRestoreCapabilitiesResponses, GetRestoreCapabilitiesErrors, PlanRestoreData, PlanRestoreResponses, PlanRestoreErrors, ListRestoreRunsForServiceData, ListRestoreRunsForServiceResponses, RetryClusterData, RetryClusterResponses, RetryClusterErrors, StartServiceData, StartServiceResponses, StartServiceErrors, StopServiceData, StopServiceResponses, StopServiceErrors, UpgradeServiceData, UpgradeServiceResponses, UpgradeServiceErrors, ListRootContainersData, ListRootContainersResponses, ListRootContainersErrors, ListContainersAtPathData, ListContainersAtPathResponses, ListContainersAtPathErrors, ListEntitiesData, ListEntitiesResponses, ListEntitiesErrors, GetEntityInfoData, GetEntityInfoResponses, GetEntityInfoErrors, QueryDataData, QueryDataResponses, QueryDataErrors, DownloadObjectData, DownloadObjectResponses, DownloadObjectErrors, GetContainerInfoData, GetContainerInfoResponses, GetContainerInfoErrors, CheckExplorerSupportData, CheckExplorerSupportResponses, CheckExplorerSupportErrors, ListPgUpgradesData, ListPgUpgradesResponses, ListPgUpgradesErrors, StartPgUpgradeData, StartPgUpgradeResponses, StartPgUpgradeErrors, GetPgUpgradeData, GetPgUpgradeResponses, GetPgUpgradeErrors, CancelPgUpgradeData, CancelPgUpgradeResponses, CancelPgUpgradeErrors, GetPgUpgradeLogsData, GetPgUpgradeLogsResponses, GetPgUpgradeLogsErrors, RetryPgUpgradeData, RetryPgUpgradeResponses, RetryPgUpgradeErrors, RollbackPgUpgradeData, RollbackPgUpgradeResponses, RollbackPgUpgradeErrors, GetFileData, GetFileResponses, GetFileErrors, GetIpGeolocationData, GetIpGeolocationResponses, GetIpGeolocationErrors, ListConnectionsData, ListConnectionsResponses, ListConnectionsErrors, DeleteConnectionData, DeleteConnectionResponses, DeleteConnectionErrors, ActivateConnectionData, ActivateConnectionResponses, ActivateConnectionErrors, DeactivateConnectionData, DeactivateConnectionResponses, DeactivateConnectionErrors, RunConnectionHealthCheckData, RunConnectionHealthCheckResponses, RunConnectionHealthCheckErrors, ListRepositoriesByConnectionData, ListRepositoriesByConnectionResponses, ListRepositoriesByConnectionErrors, SyncRepositoriesData, SyncRepositoriesResponses, SyncRepositoriesErrors, UpdateConnectionTokenData, UpdateConnectionTokenResponses, UpdateConnectionTokenErrors, ValidateConnectionData, ValidateConnectionResponses, ValidateConnectionErrors, ListGitProvidersData, ListGitProvidersResponses, ListGitProvidersErrors, CreateGitProviderData, CreateGitProviderResponses, CreateGitProviderErrors, CreateGithubPatProviderData, CreateGithubPatProviderResponses, CreateGithubPatProviderErrors, CreateGitlabOauthProviderData, CreateGitlabOauthProviderResponses, CreateGitlabOauthProviderErrors, CreateGitlabPatProviderData, CreateGitlabPatProviderResponses, CreateGitlabPatProviderErrors, DeleteGitProviderData, DeleteGitProviderResponses, DeleteGitProviderErrors, GetGitProviderData, GetGitProviderResponses, GetGitProviderErrors, ActivateProviderData, ActivateProviderResponses, ActivateProviderErrors, HandleGitProviderOauthCallbackData, HandleGitProviderOauthCallbackErrors, GetProviderConnectionsData, GetProviderConnectionsResponses, GetProviderConnectionsErrors, UpdateGitProviderCredentialsData, UpdateGitProviderCredentialsResponses, UpdateGitProviderCredentialsErrors, DeactivateProviderData, DeactivateProviderResponses, DeactivateProviderErrors, CheckProviderDeletionSafetyData, CheckProviderDeletionSafetyResponses, CheckProviderDeletionSafetyErrors, StartGitProviderOauthData, StartGitProviderOauthErrors, DeleteProviderSafelyData, DeleteProviderSafelyResponses, DeleteProviderSafelyErrors, GetPublicRepositoryData, GetPublicRepositoryResponses, GetPublicRepositoryErrors, GetPublicBranchesData, GetPublicBranchesResponses, GetPublicBranchesErrors, DetectPublicPresetsData, DetectPublicPresetsResponses, DetectPublicPresetsErrors, DiscoverWorkloadsData, DiscoverWorkloadsResponses, DiscoverWorkloadsErrors, ExecuteImportData, ExecuteImportResponses, ExecuteImportErrors, CreatePlanData, CreatePlanResponses, CreatePlanErrors, ListSourcesData, ListSourcesResponses, ListSourcesErrors, GetImportStatusData, GetImportStatusResponses, GetImportStatusErrors, GetIncidentData, GetIncidentResponses, GetIncidentErrors, UpdateIncidentStatusData, UpdateIncidentStatusResponses, UpdateIncidentStatusErrors, GetIncidentUpdatesData, GetIncidentUpdatesResponses, GetIncidentUpdatesErrors, AdminListNodesData, AdminListNodesResponses, AdminListNodesErrors, RegisterNodeData, RegisterNodeResponses, RegisterNodeErrors, AdminRemoveNodeData, AdminRemoveNodeResponses, AdminRemoveNodeErrors, AdminGetNodeData, AdminGetNodeResponses, AdminGetNodeErrors, AdminListNodeContainersData, AdminListNodeContainersResponses, AdminListNodeContainersErrors, PostDnsAckData, PostDnsAckResponses, PostDnsAckErrors, GetDnsChangesData, GetDnsChangesResponses, GetDnsChangesErrors, AdminUndrainNodeData, AdminUndrainNodeResponses, AdminUndrainNodeErrors, AdminDrainStatusData, AdminDrainStatusResponses, AdminDrainStatusErrors, AdminDrainNodeData, AdminDrainNodeResponses, AdminDrainNodeErrors, NodeHeartbeatData, NodeHeartbeatResponses, NodeHeartbeatErrors, ListPeersData, ListPeersResponses, ListPeersErrors, GetS3CredentialsData, GetS3CredentialsResponses, GetS3CredentialsErrors, ListIpAccessControlData, ListIpAccessControlResponses, ListIpAccessControlErrors, CreateIpAccessControlData, CreateIpAccessControlResponses, CreateIpAccessControlErrors, CheckIpBlockedData, CheckIpBlockedResponses, CheckIpBlockedErrors, DeleteIpAccessControlData, DeleteIpAccessControlResponses, DeleteIpAccessControlErrors, GetIpAccessControlData, GetIpAccessControlResponses, GetIpAccessControlErrors, UpdateIpAccessControlData, UpdateIpAccessControlResponses, UpdateIpAccessControlErrors, KvDelData, KvDelResponses, KvDelErrors, KvDisableData, KvDisableResponses, KvDisableErrors, KvEnableData, KvEnableResponses, KvEnableErrors, KvExpireData, KvExpireResponses, KvExpireErrors, KvGetData, KvGetResponses, KvGetErrors, KvIncrData, KvIncrResponses, KvIncrErrors, KvKeysData, KvKeysResponses, KvKeysErrors, KvSetData, KvSetResponses, KvSetErrors, KvStatusData, KvStatusResponses, KvStatusErrors, KvTtlData, KvTtlResponses, KvTtlErrors, KvUpdateData, KvUpdateResponses, KvUpdateErrors, ListRoutesData, ListRoutesResponses, ListRoutesErrors, CreateRouteData, CreateRouteResponses, CreateRouteErrors, DeleteRouteData, DeleteRouteResponses, DeleteRouteErrors, GetRouteData, GetRouteResponses, GetRouteErrors, UpdateRouteData, UpdateRouteResponses, UpdateRouteErrors, LogoutData, LogoutResponses, LogoutErrors, GetLogContextData, GetLogContextResponses, GetLogContextErrors, SearchLogsData, SearchLogsResponses, SearchLogsErrors, TailLogsData, TailLogsResponses, TailLogsErrors, GetProjectsMonitorHealthData, GetProjectsMonitorHealthResponses, GetProjectsMonitorHealthErrors, DeleteMonitorData, DeleteMonitorResponses, DeleteMonitorErrors, GetMonitorData, GetMonitorResponses, GetMonitorErrors, GetBucketedStatusData, GetBucketedStatusResponses, GetBucketedStatusErrors, GetCurrentMonitorStatusData, GetCurrentMonitorStatusResponses, GetCurrentMonitorStatusErrors, GetUptimeHistoryData, GetUptimeHistoryResponses, GetUptimeHistoryErrors, DeletePreferencesData, DeletePreferencesResponses, DeletePreferencesErrors, GetPreferencesData, GetPreferencesResponses, GetPreferencesErrors, UpdatePreferencesData, UpdatePreferencesResponses, UpdatePreferencesErrors, ListNotificationProvidersData, ListNotificationProvidersResponses, ListNotificationProvidersErrors, CreateNotificationProviderData, CreateNotificationProviderResponses, CreateNotificationProviderErrors, CreateNotificationEmailProviderData, CreateNotificationEmailProviderResponses, CreateNotificationEmailProviderErrors, UpdateEmailProviderData, UpdateEmailProviderResponses, UpdateEmailProviderErrors, CreateSlackProviderData, CreateSlackProviderResponses, CreateSlackProviderErrors, UpdateSlackProviderData, UpdateSlackProviderResponses, UpdateSlackProviderErrors, CreateWebhookProviderData, CreateWebhookProviderResponses, CreateWebhookProviderErrors, UpdateWebhookProviderData, UpdateWebhookProviderResponses, UpdateWebhookProviderErrors, DeleteNotificationProviderData, DeleteNotificationProviderResponses, DeleteNotificationProviderErrors, GetNotificationProviderData, GetNotificationProviderResponses, GetNotificationProviderErrors, UpdateNotificationProviderData, UpdateNotificationProviderResponses, UpdateNotificationProviderErrors, TestNotificationProviderData, TestNotificationProviderResponses, TestNotificationProviderErrors, ListOrdersData, ListOrdersResponses, ListOrdersErrors, QueryGenaiTracesData, QueryGenaiTracesResponses, QueryGenaiTracesErrors, GetGenaiTraceData, GetGenaiTraceResponses, GetGenaiTraceErrors, GetHealthData, GetHealthResponses, GetHealthErrors, ListInsightsData, ListInsightsResponses, ListInsightsErrors, QueryLogsData, QueryLogsResponses, QueryLogsErrors, ListMetricNamesData, ListMetricNamesResponses, ListMetricNamesErrors, QueryMetricsData, QueryMetricsResponses, QueryMetricsErrors, GetPipelineStatsData, GetPipelineStatsResponses, GetPipelineStatsErrors, GetQuotaData, GetQuotaResponses, GetQuotaErrors, QueryTraceSummariesData, QueryTraceSummariesResponses, QueryTraceSummariesErrors, QueryTracesData, QueryTracesResponses, QueryTracesErrors, GetTraceData, GetTraceResponses, GetTraceErrors, IngestLogsData, IngestLogsResponses, IngestLogsErrors, IngestMetricsData, IngestMetricsResponses, IngestMetricsErrors, IngestTracesData, IngestTracesResponses, IngestTracesErrors, IngestLogsByPathData, IngestLogsByPathResponses, IngestLogsByPathErrors, IngestMetricsByPathData, IngestMetricsByPathResponses, IngestMetricsByPathErrors, IngestTracesByPathData, IngestTracesByPathResponses, IngestTracesByPathErrors, HasPerformanceMetricsData, HasPerformanceMetricsResponses, HasPerformanceMetricsErrors, GetPerformanceMetricsData, GetPerformanceMetricsResponses, GetPerformanceMetricsErrors, GetMetricsOverTimeData, GetMetricsOverTimeResponses, GetMetricsOverTimeErrors, GetGroupedPageMetricsData, GetGroupedPageMetricsResponses, GetGroupedPageMetricsErrors, GetAccessInfoData, GetAccessInfoResponses, GetAccessInfoErrors, GetPrivateIpData, GetPrivateIpResponses, GetPrivateIpErrors, GetPublicIpData, GetPublicIpResponses, GetPublicIpErrors, ListPresetsData, ListPresetsResponses, ListPresetsErrors, GeneratePresetDockerfileData, GeneratePresetDockerfileResponses, GeneratePresetDockerfileErrors, GetPreviewGatewayLogsData, GetPreviewGatewayLogsResponses, RestartPreviewGatewayData, RestartPreviewGatewayResponses, GetPreviewGatewaySettingsData, GetPreviewGatewaySettingsResponses, PatchPreviewGatewaySettingsData, PatchPreviewGatewaySettingsResponses, GetPreviewGatewayStatusData, GetPreviewGatewayStatusResponses, UpgradePreviewGatewayData, UpgradePreviewGatewayResponses, GetProjectsData, GetProjectsResponses, GetProjectsErrors, CreateProjectData, CreateProjectResponses, CreateProjectErrors, GetProjectBySlugData, GetProjectBySlugResponses, GetProjectBySlugErrors, CreateProjectFromTemplateData, CreateProjectFromTemplateResponses, CreateProjectFromTemplateErrors, GetProjectStatisticsData, GetProjectStatisticsResponses, GetProjectStatisticsErrors, DeleteProjectData, DeleteProjectResponses, DeleteProjectErrors, GetProjectData, GetProjectResponses, GetProjectErrors, UpdateProjectData, UpdateProjectResponses, UpdateProjectErrors, GetProjectDeploymentsData, GetProjectDeploymentsResponses, GetProjectDeploymentsErrors, GetLastDeploymentData, GetLastDeploymentResponses, GetLastDeploymentErrors, TriggerProjectPipelineData, TriggerProjectPipelineResponses, TriggerProjectPipelineErrors, GetActiveVisitorsData, GetActiveVisitorsResponses, GetActiveVisitorsErrors, ListAgentsData, ListAgentsResponses, ListAgentsErrors, CreateAgentData, CreateAgentResponses, CreateAgentErrors, GetCliStatusData, GetCliStatusResponses, GetCliStatusErrors, ListAllRunsData, ListAllRunsResponses, ListAllRunsErrors, LatestRunForSourceData, LatestRunForSourceResponses, LatestRunForSourceErrors, GetRunWithLogsData, GetRunWithLogsResponses, GetRunWithLogsErrors, CancelRunData, CancelRunResponses, CancelRunErrors, RetryRunData, RetryRunResponses, RetryRunErrors, StreamRunEventsData, StreamRunEventsResponses, StreamRunEventsErrors, GetSandboxStatusData, GetSandboxStatusResponses, GetSandboxStatusErrors, SmokeTestAgentData, SmokeTestAgentResponses, SmokeTestAgentErrors, DeleteAgentData, DeleteAgentResponses, DeleteAgentErrors, GetAgentData, GetAgentResponses, GetAgentErrors, UpdateAgentData, UpdateAgentResponses, UpdateAgentErrors, ListAgentRunsData, ListAgentRunsResponses, ListAgentRunsErrors, TriggerAgentData, TriggerAgentResponses, TriggerAgentErrors, GetAggregatedBucketsData, GetAggregatedBucketsResponses, GetAggregatedBucketsErrors, StartAnalysisData, StartAnalysisResponses, StartAnalysisErrors, GetRunData, GetRunResponses, GetRunErrors, AddContextData, AddContextResponses, AddContextErrors, CancelData, CancelResponses, CancelErrors, CreatePrData, CreatePrResponses, CreatePrErrors, StartFixData, StartFixResponses, StartFixErrors, ReAnalyzeData, ReAnalyzeResponses, ReAnalyzeErrors, StreamEventsData, StreamEventsResponses, StreamEventsErrors, UpdateAutomaticDeployData, UpdateAutomaticDeployResponses, UpdateAutomaticDeployErrors, ListCustomDomainsForProjectData, ListCustomDomainsForProjectResponses, ListCustomDomainsForProjectErrors, CreateCustomDomainData, CreateCustomDomainResponses, CreateCustomDomainErrors, DeleteCustomDomainData, DeleteCustomDomainResponses, DeleteCustomDomainErrors, GetCustomDomainData, GetCustomDomainResponses, GetCustomDomainErrors, UpdateCustomDomainData, UpdateCustomDomainResponses, UpdateCustomDomainErrors, LinkCustomDomainToCertificateData, LinkCustomDomainToCertificateResponses, LinkCustomDomainToCertificateErrors, UpdateProjectDeploymentConfigData, UpdateProjectDeploymentConfigResponses, UpdateProjectDeploymentConfigErrors, GetDeploymentData, GetDeploymentResponses, GetDeploymentErrors, CancelDeploymentData, CancelDeploymentResponses, CancelDeploymentErrors, GetDeploymentJobsData, GetDeploymentJobsResponses, GetDeploymentJobsErrors, GetDeploymentJobLogsData, GetDeploymentJobLogsResponses, GetDeploymentJobLogsErrors, TailDeploymentJobLogsData, TailDeploymentJobLogsErrors, GetDeploymentOperationsData, GetDeploymentOperationsResponses, GetDeploymentOperationsErrors, ExecuteDeploymentOperationData, ExecuteDeploymentOperationResponses, ExecuteDeploymentOperationErrors, GetDeploymentOperationStatusData, GetDeploymentOperationStatusResponses, GetDeploymentOperationStatusErrors, PauseDeploymentData, PauseDeploymentResponses, PauseDeploymentErrors, PromoteDeploymentData, PromoteDeploymentResponses, PromoteDeploymentErrors, ResumeDeploymentData, ResumeDeploymentResponses, ResumeDeploymentErrors, RollbackToDeploymentData, RollbackToDeploymentResponses, RollbackToDeploymentErrors, TeardownDeploymentData, TeardownDeploymentResponses, TeardownDeploymentErrors, ListDsnsData, ListDsnsResponses, CreateDsnData, CreateDsnResponses, CreateDsnErrors, GetOrCreateDsnData, GetOrCreateDsnResponses, GetOrCreateDsnErrors, RegenerateDsnData, RegenerateDsnResponses, RegenerateDsnErrors, RevokeDsnData, RevokeDsnResponses, RevokeDsnErrors, GetEnvironmentVariablesData, GetEnvironmentVariablesResponses, GetEnvironmentVariablesErrors, CreateEnvironmentVariableData, CreateEnvironmentVariableResponses, CreateEnvironmentVariableErrors, GetResolvedEnvironmentVariablesData, GetResolvedEnvironmentVariablesResponses, GetResolvedEnvironmentVariablesErrors, GetResolvedEnvironmentVariableValueData, GetResolvedEnvironmentVariableValueResponses, GetResolvedEnvironmentVariableValueErrors, GetEnvironmentVariableValueData, GetEnvironmentVariableValueResponses, GetEnvironmentVariableValueErrors, DeleteEnvironmentVariableData, DeleteEnvironmentVariableResponses, DeleteEnvironmentVariableErrors, UpdateEnvironmentVariableData, UpdateEnvironmentVariableResponses, UpdateEnvironmentVariableErrors, GetEnvironmentsData, GetEnvironmentsResponses, GetEnvironmentsErrors, CreateEnvironmentData, CreateEnvironmentResponses, CreateEnvironmentErrors, DeleteEnvironmentData, DeleteEnvironmentResponses, DeleteEnvironmentErrors, GetEnvironmentData, GetEnvironmentResponses, GetEnvironmentErrors, GetEnvironmentCronsData, GetEnvironmentCronsResponses, GetEnvironmentCronsErrors, GetCronByIdData, GetCronByIdResponses, GetCronByIdErrors, GetCronExecutionsData, GetCronExecutionsResponses, GetCronExecutionsErrors, GetEnvironmentDomainsData, GetEnvironmentDomainsResponses, GetEnvironmentDomainsErrors, AddEnvironmentDomainData, AddEnvironmentDomainResponses, AddEnvironmentDomainErrors, DeleteEnvironmentDomainData, DeleteEnvironmentDomainResponses, DeleteEnvironmentDomainErrors, UpdateEnvironmentSettingsData, UpdateEnvironmentSettingsResponses, UpdateEnvironmentSettingsErrors, SleepEnvironmentData, SleepEnvironmentResponses, SleepEnvironmentErrors, TeardownEnvironmentData, TeardownEnvironmentResponses, TeardownEnvironmentErrors, WakeEnvironmentData, WakeEnvironmentResponses, WakeEnvironmentErrors, GetContainerLogsData, GetContainerLogsErrors, ListContainersData, ListContainersResponses, ListContainersErrors, GetContainerDetailData, GetContainerDetailResponses, GetContainerDetailErrors, GetContainerLogsByIdData, GetContainerLogsByIdErrors, GetContainerMetricsData, GetContainerMetricsResponses, GetContainerMetricsErrors, StreamContainerMetricsData, StreamContainerMetricsResponses, StreamContainerMetricsErrors, RestartContainerData, RestartContainerResponses, RestartContainerErrors, StartContainerData, StartContainerResponses, StartContainerErrors, StopContainerData, StopContainerResponses, StopContainerErrors, DeployFromImageData, DeployFromImageResponses, DeployFromImageErrors, DeployFromImageUploadData, DeployFromImageUploadResponses, DeployFromImageUploadErrors, DeployFromStaticData, DeployFromStaticResponses, DeployFromStaticErrors, ListAlertRulesData, ListAlertRulesResponses, ListAlertRulesErrors, CreateAlertRuleData, CreateAlertRuleResponses, CreateAlertRuleErrors, DeleteAlertRuleData, DeleteAlertRuleResponses, DeleteAlertRuleErrors, GetAlertRuleData, GetAlertRuleResponses, GetAlertRuleErrors, UpdateAlertRuleData, UpdateAlertRuleResponses, UpdateAlertRuleErrors, GetErrorDashboardStatsData, GetErrorDashboardStatsResponses, GetErrorDashboardStatsErrors, ListErrorGroupsData, ListErrorGroupsResponses, ListErrorGroupsErrors, GetErrorGroupData, GetErrorGroupResponses, GetErrorGroupErrors, UpdateErrorGroupData, UpdateErrorGroupResponses, UpdateErrorGroupErrors, ListErrorEventsData, ListErrorEventsResponses, ListErrorEventsErrors, GetErrorEventData, GetErrorEventResponses, GetErrorEventErrors, GetErrorStatsData, GetErrorStatsResponses, GetErrorStatsErrors, GetErrorTimeSeriesData, GetErrorTimeSeriesResponses, GetErrorTimeSeriesErrors, GetEventsCountData, GetEventsCountResponses, GetEventsCountErrors, GetEventTypeBreakdownData, GetEventTypeBreakdownResponses, GetEventTypeBreakdownErrors, RecordConsoleEventData, RecordConsoleEventResponses, RecordConsoleEventErrors, GetPropertyBreakdownData, GetPropertyBreakdownResponses, GetPropertyBreakdownErrors, GetPropertyTimelineData, GetPropertyTimelineResponses, GetPropertyTimelineErrors, GetEventsTimelineData, GetEventsTimelineResponses, GetEventsTimelineErrors, GetUniqueEventsData, GetUniqueEventsResponses, GetUniqueEventsErrors, ListRemoteExternalImagesData, ListRemoteExternalImagesResponses, ListRemoteExternalImagesErrors, RegisterExternalImageData, RegisterExternalImageResponses, RegisterExternalImageErrors, DeleteExternalImageData, DeleteExternalImageResponses, DeleteExternalImageErrors, GetRemoteExternalImageData, GetRemoteExternalImageResponses, GetRemoteExternalImageErrors, ListFunnelsData, ListFunnelsResponses, ListFunnelsErrors, CreateFunnelData, CreateFunnelResponses, CreateFunnelErrors, PreviewFunnelMetricsData, PreviewFunnelMetricsResponses, PreviewFunnelMetricsErrors, DeleteFunnelData, DeleteFunnelResponses, DeleteFunnelErrors, UpdateFunnelData, UpdateFunnelResponses, UpdateFunnelErrors, GetFunnelMetricsData, GetFunnelMetricsResponses, GetFunnelMetricsErrors, UpdateGitSettingsData, UpdateGitSettingsResponses, UpdateGitSettingsErrors, ReinstallGitlabWebhookData, ReinstallGitlabWebhookResponses, ReinstallGitlabWebhookErrors, HasErrorGroupsData, HasErrorGroupsResponses, HasErrorGroupsErrors, HasAnalyticsEventsData, HasAnalyticsEventsResponses, HasAnalyticsEventsErrors, GetHourlyVisitsData, GetHourlyVisitsResponses, GetHourlyVisitsErrors, ListExternalImagesData, ListExternalImagesResponses, ListExternalImagesErrors, PushExternalImageData, PushExternalImageResponses, PushExternalImageErrors, GetExternalImageData, GetExternalImageResponses, GetExternalImageErrors, ListIncidentsData, ListIncidentsResponses, ListIncidentsErrors, CreateIncidentData, CreateIncidentResponses, CreateIncidentErrors, GetBucketedIncidentsData, GetBucketedIncidentsResponses, GetBucketedIncidentsErrors, PurgeProjectLogsData, PurgeProjectLogsResponses, PurgeProjectLogsErrors, ListMcpsData, ListMcpsResponses, ListMcpsErrors, CreateMcpData, CreateMcpResponses, CreateMcpErrors, DeleteMcpData, DeleteMcpResponses, DeleteMcpErrors, GetMcpData, GetMcpResponses, GetMcpErrors, UpdateMcpData, UpdateMcpResponses, UpdateMcpErrors, ListMonitorsData, ListMonitorsResponses, ListMonitorsErrors, CreateMonitorData, CreateMonitorResponses, CreateMonitorErrors, DeleteReleaseSourceMapsData, DeleteReleaseSourceMapsResponses, DeleteReleaseSourceMapsErrors, ListSourceMapsData, ListSourceMapsResponses, ListSourceMapsErrors, UploadSourceMapData, UploadSourceMapResponses, UploadSourceMapErrors, RevenueRecentEventsData, RevenueRecentEventsResponses, RevenueListIntegrationsData, RevenueListIntegrationsResponses, RevenueCreateIntegrationData, RevenueCreateIntegrationResponses, RevenueCreateIntegrationErrors, RevenueDeleteIntegrationData, RevenueDeleteIntegrationResponses, RevenueUpdateConfigData, RevenueUpdateConfigResponses, RevenueUpdateConfigErrors, RevenueImportInvoicesCsvData, RevenueImportInvoicesCsvResponses, RevenueImportInvoicesCsvErrors, RevenueImportSubscriptionsCsvData, RevenueImportSubscriptionsCsvResponses, RevenueImportSubscriptionsCsvErrors, RevenueRotateTokenData, RevenueRotateTokenResponses, RevenueUpdateSecretData, RevenueUpdateSecretResponses, RevenueUpdateSecretErrors, RevenueMetricsCustomersData, RevenueMetricsCustomersResponses, RevenueMetricsMrrData, RevenueMetricsMrrResponses, RevenueMetricsSummaryData, RevenueMetricsSummaryResponses, ListProjectSecretsData, ListProjectSecretsResponses, ListProjectSecretsErrors, CreateProjectSecretData, CreateProjectSecretResponses, CreateProjectSecretErrors, DeleteProjectSecretData, DeleteProjectSecretResponses, DeleteProjectSecretErrors, UpdateProjectSecretData, UpdateProjectSecretResponses, UpdateProjectSecretErrors, UpdateProjectSettingsData, UpdateProjectSettingsResponses, UpdateProjectSettingsErrors, ListSkillsData, ListSkillsResponses, ListSkillsErrors, CreateSkillData, CreateSkillResponses, CreateSkillErrors, UploadSkillData, UploadSkillResponses, UploadSkillErrors, DeleteSkillData, DeleteSkillResponses, DeleteSkillErrors, GetSkillData, GetSkillResponses, GetSkillErrors, UpdateSkillData, UpdateSkillResponses, UpdateSkillErrors, DownloadSkillArchiveData, DownloadSkillArchiveResponses, DownloadSkillArchiveErrors, ListReleasesData, ListReleasesResponses, ListReleasesErrors, DeleteSourceMapData, DeleteSourceMapResponses, DeleteSourceMapErrors, ListStaticBundlesData, ListStaticBundlesResponses, ListStaticBundlesErrors, DeleteStaticBundleData, DeleteStaticBundleResponses, DeleteStaticBundleErrors, GetStaticBundleData, GetStaticBundleResponses, GetStaticBundleErrors, GetStatusOverviewData, GetStatusOverviewResponses, GetStatusOverviewErrors, GetUniqueCountsData, GetUniqueCountsResponses, GetUniqueCountsErrors, UploadStaticBundleData, UploadStaticBundleResponses, UploadStaticBundleErrors, ListProjectScansData, ListProjectScansResponses, ListProjectScansErrors, TriggerScanData, TriggerScanResponses, TriggerScanErrors, GetLatestScansPerEnvironmentData, GetLatestScansPerEnvironmentResponses, GetLatestScansPerEnvironmentErrors, GetLatestScanData, GetLatestScanResponses, GetLatestScanErrors, ListWebhooksData, ListWebhooksResponses, ListWebhooksErrors, CreateWebhookData, CreateWebhookResponses, CreateWebhookErrors, DeleteWebhookData, DeleteWebhookResponses, DeleteWebhookErrors, GetWebhookData, GetWebhookResponses, GetWebhookErrors, UpdateWebhookData, UpdateWebhookResponses, UpdateWebhookErrors, ListDeliveriesData, ListDeliveriesResponses, ListDeliveriesErrors, GetDeliveryData, GetDeliveryResponses, GetDeliveryErrors, RetryDeliveryData, RetryDeliveryResponses, RetryDeliveryErrors, WorkflowDryRunData, WorkflowDryRunResponses, WorkflowDryRunErrors, ListMemoryData, ListMemoryResponses, ListMemoryErrors, WriteMemoryData, WriteMemoryResponses, WriteMemoryErrors, SearchMemoryData, SearchMemoryResponses, SearchMemoryErrors, DropMemoryData, DropMemoryResponses, DropMemoryErrors, SupersedeMemoryData, SupersedeMemoryResponses, SupersedeMemoryErrors, WorkspaceListSessionsData, WorkspaceListSessionsResponses, WorkspaceListSessionsErrors, WorkspaceStartSessionData, WorkspaceStartSessionResponses, WorkspaceStartSessionErrors, WorkspaceDeleteSessionData, WorkspaceDeleteSessionResponses, WorkspaceDeleteSessionErrors, WorkspaceGetSessionData, WorkspaceGetSessionResponses, WorkspaceGetSessionErrors, WorkspaceUpdateSessionData, WorkspaceUpdateSessionResponses, WorkspaceUpdateSessionErrors, WorkspaceCancelRunData, WorkspaceCancelRunResponses, WorkspaceCancelRunErrors, WorkspaceCloseSessionData, WorkspaceCloseSessionResponses, WorkspaceCloseSessionErrors, WorkspaceSendMessageData, WorkspaceSendMessageResponses, WorkspaceSendMessageErrors, WorkspaceRegeneratePreviewPasswordData, WorkspaceRegeneratePreviewPasswordResponses, WorkspaceRegeneratePreviewPasswordErrors, WorkspaceReopenSessionData, WorkspaceReopenSessionResponses, WorkspaceReopenSessionErrors, WorkspaceRefreshSandboxData, WorkspaceRefreshSandboxResponses, WorkspaceRefreshSandboxErrors, WorkspaceRestartSandboxData, WorkspaceRestartSandboxResponses, WorkspaceRestartSandboxErrors, WorkspaceStartSandboxData, WorkspaceStartSandboxResponses, WorkspaceStartSandboxErrors, WorkspaceSandboxStatsData, WorkspaceSandboxStatsResponses, WorkspaceSandboxStatsErrors, WorkspaceStopSandboxData, WorkspaceStopSandboxResponses, WorkspaceStopSandboxErrors, WorkspaceStreamMessagesData, WorkspaceStreamMessagesResponses, WorkspaceStreamMessagesErrors, WorkspaceTerminalPasteImageData, WorkspaceTerminalPasteImageResponses, WorkspaceTerminalPasteImageErrors, WorkspaceListTerminalTabsData, WorkspaceListTerminalTabsResponses, WorkspaceListTerminalTabsErrors, WorkspaceDeleteTerminalTabData, WorkspaceDeleteTerminalTabResponses, WorkspaceDeleteTerminalTabErrors, GetProxyLogsData, GetProxyLogsResponses, GetProxyLogsErrors, GetProxyLogByRequestIdData, GetProxyLogByRequestIdResponses, GetProxyLogByRequestIdErrors, GetProjectsHealthData, GetProjectsHealthResponses, GetProjectsHealthErrors, GetTimeBucketStatsData, GetTimeBucketStatsResponses, GetTimeBucketStatsErrors, GetTodayStatsData, GetTodayStatsResponses, GetTodayStatsErrors, GetProxyLogByIdData, GetProxyLogByIdResponses, GetProxyLogByIdErrors, ListSyncedRepositoriesData, ListSyncedRepositoriesResponses, ListSyncedRepositoriesErrors, GetRepositoryByNameData, GetRepositoryByNameResponses, GetRepositoryByNameErrors, GetAllRepositoriesByNameData, GetAllRepositoriesByNameResponses, GetAllRepositoriesByNameErrors, GetRepositoryPresetByNameData, GetRepositoryPresetByNameResponses, GetRepositoryPresetByNameErrors, GetRepositoryBranchesData, GetRepositoryBranchesResponses, GetRepositoryBranchesErrors, GetRepositoryTagsData, GetRepositoryTagsResponses, GetRepositoryTagsErrors, GetRepositoryPresetLiveData, GetRepositoryPresetLiveResponses, GetRepositoryPresetLiveErrors, GetRepositoryByIdData, GetRepositoryByIdResponses, GetRepositoryByIdErrors, GetBranchesByRepositoryIdData, GetBranchesByRepositoryIdResponses, GetBranchesByRepositoryIdErrors, ListCommitsByRepositoryIdData, ListCommitsByRepositoryIdResponses, ListCommitsByRepositoryIdErrors, CheckCommitExistsData, CheckCommitExistsResponses, CheckCommitExistsErrors, GetTagsByRepositoryIdData, GetTagsByRepositoryIdResponses, GetTagsByRepositoryIdErrors, GetRestoreRunData, GetRestoreRunResponses, GetRestoreRunErrors, RevenueGlobalEventsData, RevenueGlobalEventsResponses, RevenueMetricsGlobalMrrData, RevenueMetricsGlobalMrrResponses, RevenueMetricsGlobalSummaryData, RevenueMetricsGlobalSummaryResponses, RevenueListProvidersData, RevenueListProvidersResponses, GetProjectSessionReplaysData, GetProjectSessionReplaysResponses, GetProjectSessionReplaysErrors, GetSessionEventsData, GetSessionEventsResponses, GetSessionEventsErrors, GetSettingsData, GetSettingsResponses, GetSettingsErrors, UpdateSettingsData, UpdateSettingsResponses, UpdateSettingsErrors, SaveAgentTokenData, SaveAgentTokenResponses, SaveAgentTokenErrors, ListAiProvidersData, ListAiProvidersResponses, ListAiProvidersErrors, UpdateAiProviderData, UpdateAiProviderResponses, UpdateAiProviderErrors, ActivateAiProviderData, ActivateAiProviderResponses, ActivateAiProviderErrors, SaveAiProviderCredentialData, SaveAiProviderCredentialResponses, SaveAiProviderCredentialErrors, RevokeJoinTokenData, RevokeJoinTokenResponses, RevokeJoinTokenErrors, GenerateJoinTokenData, GenerateJoinTokenResponses, GenerateJoinTokenErrors, GetJoinTokenStatusData, GetJoinTokenStatusResponses, GetJoinTokenStatusErrors, ListGlobalMcpsData, ListGlobalMcpsResponses, ListGlobalMcpsErrors, CreateGlobalMcpData, CreateGlobalMcpResponses, CreateGlobalMcpErrors, DeleteGlobalMcpData, DeleteGlobalMcpResponses, DeleteGlobalMcpErrors, GetGlobalMcpData, GetGlobalMcpResponses, GetGlobalMcpErrors, UpdateGlobalMcpData, UpdateGlobalMcpResponses, UpdateGlobalMcpErrors, RefreshRouteTableData, RefreshRouteTableResponses, RefreshRouteTableErrors, RebuildSandboxImageData, RebuildSandboxImageResponses, RebuildSandboxImageErrors, GetGlobalSandboxStatusData, GetGlobalSandboxStatusResponses, GetGlobalSandboxStatusErrors, ListSecretsData, ListSecretsResponses, ListSecretsErrors, UpsertSecretData, UpsertSecretResponses, UpsertSecretErrors, DeleteSecretData, DeleteSecretResponses, DeleteSecretErrors, ListGlobalSkillsData, ListGlobalSkillsResponses, ListGlobalSkillsErrors, CreateGlobalSkillData, CreateGlobalSkillResponses, CreateGlobalSkillErrors, UploadGlobalSkillData, UploadGlobalSkillResponses, UploadGlobalSkillErrors, DeleteGlobalSkillData, DeleteGlobalSkillResponses, DeleteGlobalSkillErrors, GetGlobalSkillData, GetGlobalSkillResponses, GetGlobalSkillErrors, UpdateGlobalSkillData, UpdateGlobalSkillResponses, UpdateGlobalSkillErrors, DownloadGlobalSkillArchiveData, DownloadGlobalSkillArchiveResponses, DownloadGlobalSkillArchiveErrors, ListProjectTemplatesData, ListProjectTemplatesResponses, ListProjectTemplatesErrors, ListProjectTemplateTagsData, ListProjectTemplateTagsResponses, ListProjectTemplateTagsErrors, GetProjectTemplateData, GetProjectTemplateResponses, GetProjectTemplateErrors, GetCurrentUserData, GetCurrentUserResponses, GetCurrentUserErrors, ListUsersData, ListUsersResponses, ListUsersErrors, CreateUserData, CreateUserResponses, CreateUserErrors, UpdateSelfData, UpdateSelfResponses, UpdateSelfErrors, DisableMfaData, DisableMfaResponses, DisableMfaErrors, SetupMfaData, SetupMfaResponses, SetupMfaErrors, VerifyAndEnableMfaData, VerifyAndEnableMfaResponses, VerifyAndEnableMfaErrors, DeleteUserData, DeleteUserResponses, DeleteUserErrors, UpdateUserData, UpdateUserResponses, UpdateUserErrors, RestoreUserData, RestoreUserResponses, RestoreUserErrors, AssignRoleData, AssignRoleResponses, AssignRoleErrors, RemoveRoleData, RemoveRoleResponses, RemoveRoleErrors, ListSandboxesData, ListSandboxesResponses, CreateSandboxData, CreateSandboxResponses, CreateSandboxErrors, GetSandboxData, GetSandboxResponses, GetSandboxErrors, CmdData, CmdResponses, CmdErrors, GetCmdData, GetCmdResponses, GetCmdErrors, CmdLogsData, CmdLogsResponses, CmdLogsErrors, DestroySandboxData, DestroySandboxResponses, DestroySandboxErrors, DomainData, DomainResponses, DomainErrors, ExecData, ExecResponses, ExecErrors, ExecDetachedData, ExecDetachedResponses, ExecDetachedErrors, ExtendTimeoutData, ExtendTimeoutResponses, ExtendTimeoutErrors, MkdirData, MkdirResponses, MkdirErrors, ReadFileData, ReadFileResponses, ReadFileErrors, StatPathData, StatPathResponses, StatPathErrors, WriteFileData, WriteFileResponses, WriteFileErrors, WriteFilesData, WriteFilesResponses, WriteFilesErrors, ListJobsData, ListJobsResponses, ListJobsErrors, JobStatusData, JobStatusResponses, JobStatusErrors, KillJobData, KillJobResponses, KillJobErrors, JobLogsData, JobLogsResponses, JobLogsErrors, PauseSandboxData, PauseSandboxResponses, PauseSandboxErrors, ClearPreviewPasswordData, ClearPreviewPasswordResponses, ClearPreviewPasswordErrors, SetPreviewPasswordData, SetPreviewPasswordResponses, SetPreviewPasswordErrors, RestartSandboxData, RestartSandboxResponses, RestartSandboxErrors, ResumeSandboxData, ResumeSandboxResponses, ResumeSandboxErrors, SourceSandboxData, SourceSandboxResponses, SourceSandboxErrors, StopSandboxData, StopSandboxResponses, StopSandboxErrors, CmdKillData, CmdKillResponses, CmdKillErrors, GetVisitorSessionsData, GetVisitorSessionsResponses, GetVisitorSessionsErrors, DeleteSessionReplayData, DeleteSessionReplayResponses, DeleteSessionReplayErrors, GetSessionReplayData, GetSessionReplayResponses, GetSessionReplayErrors, UpdateSessionDurationData, UpdateSessionDurationResponses, UpdateSessionDurationErrors, GetSessionReplayEventsData, GetSessionReplayEventsResponses, GetSessionReplayEventsErrors, AddEventsData, AddEventsResponses, AddEventsErrors, DeleteScanData, DeleteScanResponses, DeleteScanErrors, GetScanData, GetScanResponses, GetScanErrors, GetScanVulnerabilitiesData, GetScanVulnerabilitiesResponses, GetScanVulnerabilitiesErrors, ListEventTypesData, ListEventTypesResponses, TriggerWeeklyDigestData, TriggerWeeklyDigestResponses, TriggerWeeklyDigestErrors, ListExternalPluginsData, ListExternalPluginsResponses, ReloadPluginsData, ReloadPluginsResponses, ReloadPluginsErrors, IngestSentryEnvelopeData, IngestSentryEnvelopeResponses, IngestSentryEnvelopeErrors, IngestSentryEventData, IngestSentryEventResponses, IngestSentryEventErrors, ListAuditLogsData, ListAuditLogsResponses, ListAuditLogsErrors, GetAuditLogData, GetAuditLogResponses, GetAuditLogErrors } from './types.gen'; +import type { GetPlatformInfoData, GetPlatformInfoResponses, GetPlatformInfoErrors, ChunkUploadOptionsData, ChunkUploadOptionsResponses, CreateReleaseData, CreateReleaseResponses, CreateReleaseErrors, CreateProjectReleaseData, CreateProjectReleaseResponses, CreateProjectReleaseErrors, FinalizeProjectReleaseData, FinalizeProjectReleaseResponses, FinalizeProjectReleaseErrors, ListReleaseFilesData, ListReleaseFilesResponses, ListReleaseFilesErrors, UploadReleaseFileData, UploadReleaseFileResponses, UploadReleaseFileErrors, RecordEventMetricsData, RecordEventMetricsResponses, RecordEventMetricsErrors, AddSessionReplayEventsData, AddSessionReplayEventsResponses, AddSessionReplayEventsErrors, InitSessionReplayData, InitSessionReplayResponses, InitSessionReplayErrors, RecordSpeedMetricsData, RecordSpeedMetricsResponses, RecordSpeedMetricsErrors, UpdateSpeedMetricsData, UpdateSpeedMetricsResponses, UpdateSpeedMetricsErrors, WebhookTriggerData, WebhookTriggerResponses, WebhookTriggerErrors, GetPricingData, GetPricingResponses, GetPricingErrors, ListProviderKeysData, ListProviderKeysResponses, ListProviderKeysErrors, CreateProviderKeyData, CreateProviderKeyResponses, CreateProviderKeyErrors, TestProviderKeyInlineData, TestProviderKeyInlineResponses, TestProviderKeyInlineErrors, DeleteProviderKeyData, DeleteProviderKeyResponses, DeleteProviderKeyErrors, UpdateProviderKeyData, UpdateProviderKeyResponses, UpdateProviderKeyErrors, TestProviderKeyByIdData, TestProviderKeyByIdResponses, TestProviderKeyByIdErrors, GetUsageByProviderData, GetUsageByProviderResponses, GetUsageByProviderErrors, GetConversationsData, GetConversationsResponses, GetConversationsErrors, GetConversationDetailData, GetConversationDetailResponses, GetConversationDetailErrors, GetUsageRecentData, GetUsageRecentResponses, GetUsageRecentErrors, GetUsageSummaryData, GetUsageSummaryResponses, GetUsageSummaryErrors, GetUsageTimeseriesData, GetUsageTimeseriesResponses, GetUsageTimeseriesErrors, GetUsageTopModelsData, GetUsageTopModelsResponses, GetUsageTopModelsErrors, ChatCompletionsData, ChatCompletionsResponses, ChatCompletionsErrors, EmbeddingsData, EmbeddingsResponses, EmbeddingsErrors, ListModelsData, ListModelsResponses, ListModelsErrors, GetAnalyticsActiveVisitorsData, GetAnalyticsActiveVisitorsResponses, GetAnalyticsActiveVisitorsErrors, GetEventDetailData, GetEventDetailResponses, GetEventDetailErrors, GetEventVisitorsData, GetEventVisitorsResponses, GetEventVisitorsErrors, GetAnalyticsEventsCountData, GetAnalyticsEventsCountResponses, GetAnalyticsEventsCountErrors, GetGeneralStatsData, GetGeneralStatsResponses, GetGeneralStatsErrors, GetLiveVisitorsListData, GetLiveVisitorsListResponses, GetLiveVisitorsListErrors, GetPageFlowData, GetPageFlowResponses, GetPageFlowErrors, GetPageHourlySessionsData, GetPageHourlySessionsResponses, GetPageHourlySessionsErrors, GetPagePathDetailData, GetPagePathDetailResponses, GetPagePathDetailErrors, GetPagePathVisitorsData, GetPagePathVisitorsResponses, GetPagePathVisitorsErrors, GetPagePathsData, GetPagePathsResponses, GetPagePathsErrors, GetPagePathsSparklinesData, GetPagePathsSparklinesResponses, GetPagePathsSparklinesErrors, GetRecentActivityData, GetRecentActivityResponses, GetRecentActivityErrors, GetSessionDetailsData, GetSessionDetailsResponses, GetSessionDetailsErrors, GetAnalyticsSessionEventsData, GetAnalyticsSessionEventsResponses, GetAnalyticsSessionEventsErrors, GetSessionLogsData, GetSessionLogsResponses, GetSessionLogsErrors, GetVisitorsData, GetVisitorsResponses, GetVisitorsErrors, GetVisitorByGuidData, GetVisitorByGuidResponses, GetVisitorByGuidErrors, GetVisitorByIdData, GetVisitorByIdResponses, GetVisitorByIdErrors, GetVisitorDetailsData, GetVisitorDetailsResponses, GetVisitorDetailsErrors, EnrichVisitorData, EnrichVisitorResponses, EnrichVisitorErrors, GetVisitorInfoData, GetVisitorInfoResponses, GetVisitorInfoErrors, GetVisitorJourneyData, GetVisitorJourneyResponses, GetVisitorJourneyErrors, GetAnalyticsVisitorSessionsData, GetAnalyticsVisitorSessionsResponses, GetAnalyticsVisitorSessionsErrors, GetVisitorStatsData, GetVisitorStatsResponses, GetVisitorStatsErrors, ListApiKeysData, ListApiKeysResponses, ListApiKeysErrors, CreateApiKeyData, CreateApiKeyResponses, CreateApiKeyErrors, GetApiKeyPermissionsData, GetApiKeyPermissionsResponses, GetApiKeyPermissionsErrors, DeleteApiKeyData, DeleteApiKeyResponses, DeleteApiKeyErrors, GetApiKeyData, GetApiKeyResponses, GetApiKeyErrors, UpdateApiKeyData, UpdateApiKeyResponses, UpdateApiKeyErrors, ActivateApiKeyData, ActivateApiKeyResponses, ActivateApiKeyErrors, DeactivateApiKeyData, DeactivateApiKeyResponses, DeactivateApiKeyErrors, CliLoginData, CliLoginResponses, CliLoginErrors, CliLogoutData, CliLogoutResponses, CliLogoutErrors, EmailStatusData, EmailStatusResponses, EmailStatusErrors, LoginData, LoginResponses, LoginErrors, RequestMagicLinkData, RequestMagicLinkResponses, RequestMagicLinkErrors, VerifyMagicLinkData, VerifyMagicLinkResponses, VerifyMagicLinkErrors, RequestPasswordResetData, RequestPasswordResetResponses, RequestPasswordResetErrors, ResetPasswordData, ResetPasswordResponses, ResetPasswordErrors, VerifyEmailData, VerifyEmailResponses, VerifyEmailErrors, VerifyMfaChallengeData, VerifyMfaChallengeResponses, VerifyMfaChallengeErrors, RunExternalServiceBackupData, RunExternalServiceBackupResponses, RunExternalServiceBackupErrors, ListS3SourcesData, ListS3SourcesResponses, ListS3SourcesErrors, CreateS3SourceData, CreateS3SourceResponses, CreateS3SourceErrors, TestS3ConnectionPreviewData, TestS3ConnectionPreviewResponses, TestS3ConnectionPreviewErrors, DeleteS3SourceData, DeleteS3SourceResponses, DeleteS3SourceErrors, GetS3SourceData, GetS3SourceResponses, GetS3SourceErrors, UpdateS3SourceData, UpdateS3SourceResponses, UpdateS3SourceErrors, ListSourceBackupsData, ListSourceBackupsResponses, ListSourceBackupsErrors, RunBackupForSourceData, RunBackupForSourceResponses, RunBackupForSourceErrors, SetDefaultS3SourceData, SetDefaultS3SourceResponses, SetDefaultS3SourceErrors, TestS3SourceConnectionData, TestS3SourceConnectionResponses, TestS3SourceConnectionErrors, ListBackupSchedulesData, ListBackupSchedulesResponses, ListBackupSchedulesErrors, CreateBackupScheduleData, CreateBackupScheduleResponses, CreateBackupScheduleErrors, DeleteBackupScheduleData, DeleteBackupScheduleResponses, DeleteBackupScheduleErrors, GetBackupScheduleData, GetBackupScheduleResponses, GetBackupScheduleErrors, ListBackupsForScheduleData, ListBackupsForScheduleResponses, ListBackupsForScheduleErrors, DisableBackupScheduleData, DisableBackupScheduleResponses, DisableBackupScheduleErrors, EnableBackupScheduleData, EnableBackupScheduleResponses, EnableBackupScheduleErrors, GetBackupData, GetBackupResponses, GetBackupErrors, BlobDeleteData, BlobDeleteResponses, BlobDeleteErrors, BlobListData, BlobListResponses, BlobListErrors, BlobPutData, BlobPutResponses, BlobPutErrors, BlobCopyData, BlobCopyResponses, BlobCopyErrors, BlobDisableData, BlobDisableResponses, BlobDisableErrors, BlobEnableData, BlobEnableResponses, BlobEnableErrors, BlobStatusData, BlobStatusResponses, BlobStatusErrors, BlobUpdateData, BlobUpdateResponses, BlobUpdateErrors, BlobDownloadData, BlobDownloadResponses, BlobDownloadErrors, BlobHeadData, BlobHeadResponses, BlobHeadErrors, GetDashboardProjectsAnalyticsData, GetDashboardProjectsAnalyticsResponses, GetDashboardProjectsAnalyticsErrors, GetActivityGraphData, GetActivityGraphResponses, GetActivityGraphErrors, GetScanByDeploymentData, GetScanByDeploymentResponses, GetScanByDeploymentErrors, ListDnsProvidersData, ListDnsProvidersResponses, ListDnsProvidersErrors, CreateDnsProviderData, CreateDnsProviderResponses, CreateDnsProviderErrors, DeleteDnsProviderData, DeleteDnsProviderResponses, DeleteDnsProviderErrors, GetDnsProviderData, GetDnsProviderResponses, GetDnsProviderErrors, UpdateProviderData, UpdateProviderResponses, UpdateProviderErrors, ListManagedDomainsData, ListManagedDomainsResponses, ListManagedDomainsErrors, AddManagedDomainData, AddManagedDomainResponses, AddManagedDomainErrors, TestProviderConnectionData, TestProviderConnectionResponses, TestProviderConnectionErrors, ListProviderZonesData, ListProviderZonesResponses, ListProviderZonesErrors, RemoveManagedDomainData, RemoveManagedDomainResponses, RemoveManagedDomainErrors, VerifyManagedDomainData, VerifyManagedDomainResponses, VerifyManagedDomainErrors, LookupDnsARecordsData, LookupDnsARecordsResponses, LookupDnsARecordsErrors, ListDomainsData, ListDomainsResponses, ListDomainsErrors, CreateDomainData, CreateDomainResponses, CreateDomainErrors, GetDomainByHostData, GetDomainByHostResponses, GetDomainByHostErrors, CancelDomainOrderData, CancelDomainOrderResponses, CancelDomainOrderErrors, GetDomainOrderData, GetDomainOrderResponses, GetDomainOrderErrors, CreateOrRecreateOrderData, CreateOrRecreateOrderResponses, CreateOrRecreateOrderErrors, FinalizeOrderData, FinalizeOrderResponses, FinalizeOrderErrors, SetupDnsChallengeData, SetupDnsChallengeResponses, SetupDnsChallengeErrors, DeleteDomainData, DeleteDomainResponses, DeleteDomainErrors, GetDomainByIdData, GetDomainByIdResponses, GetDomainByIdErrors, GetChallengeTokenData, GetChallengeTokenResponses, GetChallengeTokenErrors, GetHttpChallengeDebugData, GetHttpChallengeDebugResponses, GetHttpChallengeDebugErrors, ProvisionDomainData, ProvisionDomainResponses, ProvisionDomainErrors, RenewDomainData, RenewDomainResponses, RenewDomainErrors, CheckDomainStatusData, CheckDomainStatusResponses, CheckDomainStatusErrors, ListEmailDomainsData, ListEmailDomainsResponses, ListEmailDomainsErrors, CreateEmailDomainData, CreateEmailDomainResponses, CreateEmailDomainErrors, GetDomainByNameData, GetDomainByNameResponses, GetDomainByNameErrors, DeleteEmailDomainData, DeleteEmailDomainResponses, DeleteEmailDomainErrors, GetDomainData, GetDomainResponses, GetDomainErrors, GetDomainDnsRecordsData, GetDomainDnsRecordsResponses, GetDomainDnsRecordsErrors, SetupDnsData, SetupDnsResponses, SetupDnsErrors, VerifyDomainData, VerifyDomainResponses, VerifyDomainErrors, ListEmailProvidersData, ListEmailProvidersResponses, ListEmailProvidersErrors, CreateEmailProviderData, CreateEmailProviderResponses, CreateEmailProviderErrors, DeleteEmailProviderData, DeleteEmailProviderResponses, DeleteEmailProviderErrors, GetEmailProviderData, GetEmailProviderResponses, GetEmailProviderErrors, TestProviderData, TestProviderResponses, TestProviderErrors, ListEmailsData, ListEmailsResponses, ListEmailsErrors, SendEmailData, SendEmailResponses, SendEmailErrors, GetGlobalEventsData, GetGlobalEventsResponses, GetGlobalEventsErrors, GetGlobalEventStatsData, GetGlobalEventStatsResponses, GetGlobalEventStatsErrors, GetEmailStatsData, GetEmailStatsResponses, GetEmailStatsErrors, ValidateEmailData, ValidateEmailResponses, ValidateEmailErrors, TrackClickData, TrackClickErrors, TrackOpenData, TrackOpenResponses, TrackOpenErrors, GetEmailData, GetEmailResponses, GetEmailErrors, GetEmailTrackingData, GetEmailTrackingResponses, GetEmailTrackingErrors, GetEmailEventsData, GetEmailEventsResponses, GetEmailEventsErrors, GetEmailLinksData, GetEmailLinksResponses, GetEmailLinksErrors, ListServicesData, ListServicesResponses, ListServicesErrors, CreateServiceData, CreateServiceResponses, CreateServiceErrors, ListAvailableContainersData, ListAvailableContainersResponses, ListAvailableContainersErrors, GetServiceBySlugData, GetServiceBySlugResponses, GetServiceBySlugErrors, ListServiceHealthStatusesData, ListServiceHealthStatusesResponses, ListServiceHealthStatusesErrors, ImportExternalServiceData, ImportExternalServiceResponses, ImportExternalServiceErrors, ListProjectServicesData, ListProjectServicesResponses, ListProjectServicesErrors, GetProjectServiceEnvironmentVariablesData, GetProjectServiceEnvironmentVariablesResponses, GetProjectServiceEnvironmentVariablesErrors, GetProvidersMetadataData, GetProvidersMetadataResponses, GetProvidersMetadataErrors, GetProviderMetadataData, GetProviderMetadataResponses, GetProviderMetadataErrors, GetServiceTypesData, GetServiceTypesResponses, GetServiceTypesErrors, GetServiceTypeParametersData, GetServiceTypeParametersResponses, GetServiceTypeParametersErrors, DeleteServiceData, DeleteServiceResponses, DeleteServiceErrors, GetServiceData, GetServiceResponses, GetServiceErrors, UpdateServiceData, UpdateServiceResponses, UpdateServiceErrors, GetClusterHealthData, GetClusterHealthResponses, GetClusterHealthErrors, TriggerServiceHealthCheckData, TriggerServiceHealthCheckResponses, TriggerServiceHealthCheckErrors, GetServiceHealthStatusData, GetServiceHealthStatusResponses, GetServiceHealthStatusErrors, AddClusterMemberData, AddClusterMemberResponses, AddClusterMemberErrors, RemoveClusterMemberData, RemoveClusterMemberResponses, RemoveClusterMemberErrors, GetClusterMemberData, GetClusterMemberResponses, GetClusterMemberErrors, PromoteClusterMemberData, PromoteClusterMemberResponses, PromoteClusterMemberErrors, GetServicePreviewEnvironmentVariablesMaskedData, GetServicePreviewEnvironmentVariablesMaskedResponses, GetServicePreviewEnvironmentVariablesMaskedErrors, GetServicePreviewEnvironmentVariableNamesData, GetServicePreviewEnvironmentVariableNamesResponses, GetServicePreviewEnvironmentVariableNamesErrors, ListServiceProjectsData, ListServiceProjectsResponses, ListServiceProjectsErrors, LinkServiceToProjectData, LinkServiceToProjectResponses, LinkServiceToProjectErrors, UnlinkServiceFromProjectData, UnlinkServiceFromProjectResponses, UnlinkServiceFromProjectErrors, GetServiceEnvironmentVariablesData, GetServiceEnvironmentVariablesResponses, GetServiceEnvironmentVariablesErrors, GetServiceEnvironmentVariableData, GetServiceEnvironmentVariableResponses, GetServiceEnvironmentVariableErrors, StartRestoreData, StartRestoreResponses, StartRestoreErrors, GetRestoreCapabilitiesData, GetRestoreCapabilitiesResponses, GetRestoreCapabilitiesErrors, PlanRestoreData, PlanRestoreResponses, PlanRestoreErrors, ListRestoreRunsForServiceData, ListRestoreRunsForServiceResponses, RetryClusterData, RetryClusterResponses, RetryClusterErrors, StartServiceData, StartServiceResponses, StartServiceErrors, StopServiceData, StopServiceResponses, StopServiceErrors, UpgradeServiceData, UpgradeServiceResponses, UpgradeServiceErrors, ListRootContainersData, ListRootContainersResponses, ListRootContainersErrors, ListContainersAtPathData, ListContainersAtPathResponses, ListContainersAtPathErrors, ListEntitiesData, ListEntitiesResponses, ListEntitiesErrors, GetEntityInfoData, GetEntityInfoResponses, GetEntityInfoErrors, QueryDataData, QueryDataResponses, QueryDataErrors, DownloadObjectData, DownloadObjectResponses, DownloadObjectErrors, GetContainerInfoData, GetContainerInfoResponses, GetContainerInfoErrors, CheckExplorerSupportData, CheckExplorerSupportResponses, CheckExplorerSupportErrors, ListPgUpgradesData, ListPgUpgradesResponses, ListPgUpgradesErrors, StartPgUpgradeData, StartPgUpgradeResponses, StartPgUpgradeErrors, GetPgUpgradeData, GetPgUpgradeResponses, GetPgUpgradeErrors, CancelPgUpgradeData, CancelPgUpgradeResponses, CancelPgUpgradeErrors, GetPgUpgradeLogsData, GetPgUpgradeLogsResponses, GetPgUpgradeLogsErrors, RetryPgUpgradeData, RetryPgUpgradeResponses, RetryPgUpgradeErrors, RollbackPgUpgradeData, RollbackPgUpgradeResponses, RollbackPgUpgradeErrors, GetFileData, GetFileResponses, GetFileErrors, GetIpGeolocationData, GetIpGeolocationResponses, GetIpGeolocationErrors, ListConnectionsData, ListConnectionsResponses, ListConnectionsErrors, DeleteConnectionData, DeleteConnectionResponses, DeleteConnectionErrors, ActivateConnectionData, ActivateConnectionResponses, ActivateConnectionErrors, DeactivateConnectionData, DeactivateConnectionResponses, DeactivateConnectionErrors, RunConnectionHealthCheckData, RunConnectionHealthCheckResponses, RunConnectionHealthCheckErrors, ListRepositoriesByConnectionData, ListRepositoriesByConnectionResponses, ListRepositoriesByConnectionErrors, SyncRepositoriesData, SyncRepositoriesResponses, SyncRepositoriesErrors, UpdateConnectionTokenData, UpdateConnectionTokenResponses, UpdateConnectionTokenErrors, ValidateConnectionData, ValidateConnectionResponses, ValidateConnectionErrors, ListGitProvidersData, ListGitProvidersResponses, ListGitProvidersErrors, CreateGitProviderData, CreateGitProviderResponses, CreateGitProviderErrors, CreateGithubPatProviderData, CreateGithubPatProviderResponses, CreateGithubPatProviderErrors, CreateGitlabOauthProviderData, CreateGitlabOauthProviderResponses, CreateGitlabOauthProviderErrors, CreateGitlabPatProviderData, CreateGitlabPatProviderResponses, CreateGitlabPatProviderErrors, DeleteGitProviderData, DeleteGitProviderResponses, DeleteGitProviderErrors, GetGitProviderData, GetGitProviderResponses, GetGitProviderErrors, ActivateProviderData, ActivateProviderResponses, ActivateProviderErrors, HandleGitProviderOauthCallbackData, HandleGitProviderOauthCallbackErrors, GetProviderConnectionsData, GetProviderConnectionsResponses, GetProviderConnectionsErrors, UpdateGitProviderCredentialsData, UpdateGitProviderCredentialsResponses, UpdateGitProviderCredentialsErrors, DeactivateProviderData, DeactivateProviderResponses, DeactivateProviderErrors, CheckProviderDeletionSafetyData, CheckProviderDeletionSafetyResponses, CheckProviderDeletionSafetyErrors, StartGitProviderOauthData, StartGitProviderOauthErrors, DeleteProviderSafelyData, DeleteProviderSafelyResponses, DeleteProviderSafelyErrors, GetPublicRepositoryData, GetPublicRepositoryResponses, GetPublicRepositoryErrors, GetPublicBranchesData, GetPublicBranchesResponses, GetPublicBranchesErrors, DetectPublicPresetsData, DetectPublicPresetsResponses, DetectPublicPresetsErrors, DiscoverWorkloadsData, DiscoverWorkloadsResponses, DiscoverWorkloadsErrors, ExecuteImportData, ExecuteImportResponses, ExecuteImportErrors, CreatePlanData, CreatePlanResponses, CreatePlanErrors, ListSourcesData, ListSourcesResponses, ListSourcesErrors, GetImportStatusData, GetImportStatusResponses, GetImportStatusErrors, GetIncidentData, GetIncidentResponses, GetIncidentErrors, UpdateIncidentStatusData, UpdateIncidentStatusResponses, UpdateIncidentStatusErrors, GetIncidentUpdatesData, GetIncidentUpdatesResponses, GetIncidentUpdatesErrors, AdminListNodesData, AdminListNodesResponses, AdminListNodesErrors, RegisterNodeData, RegisterNodeResponses, RegisterNodeErrors, AdminRemoveNodeData, AdminRemoveNodeResponses, AdminRemoveNodeErrors, AdminGetNodeData, AdminGetNodeResponses, AdminGetNodeErrors, AdminListNodeContainersData, AdminListNodeContainersResponses, AdminListNodeContainersErrors, PostDnsAckData, PostDnsAckResponses, PostDnsAckErrors, GetDnsChangesData, GetDnsChangesResponses, GetDnsChangesErrors, AdminUndrainNodeData, AdminUndrainNodeResponses, AdminUndrainNodeErrors, AdminDrainStatusData, AdminDrainStatusResponses, AdminDrainStatusErrors, AdminDrainNodeData, AdminDrainNodeResponses, AdminDrainNodeErrors, NodeHeartbeatData, NodeHeartbeatResponses, NodeHeartbeatErrors, ListPeersData, ListPeersResponses, ListPeersErrors, GetS3CredentialsData, GetS3CredentialsResponses, GetS3CredentialsErrors, ListIpAccessControlData, ListIpAccessControlResponses, ListIpAccessControlErrors, CreateIpAccessControlData, CreateIpAccessControlResponses, CreateIpAccessControlErrors, CheckIpBlockedData, CheckIpBlockedResponses, CheckIpBlockedErrors, DeleteIpAccessControlData, DeleteIpAccessControlResponses, DeleteIpAccessControlErrors, GetIpAccessControlData, GetIpAccessControlResponses, GetIpAccessControlErrors, UpdateIpAccessControlData, UpdateIpAccessControlResponses, UpdateIpAccessControlErrors, KvDelData, KvDelResponses, KvDelErrors, KvDisableData, KvDisableResponses, KvDisableErrors, KvEnableData, KvEnableResponses, KvEnableErrors, KvExpireData, KvExpireResponses, KvExpireErrors, KvGetData, KvGetResponses, KvGetErrors, KvIncrData, KvIncrResponses, KvIncrErrors, KvKeysData, KvKeysResponses, KvKeysErrors, KvSetData, KvSetResponses, KvSetErrors, KvStatusData, KvStatusResponses, KvStatusErrors, KvTtlData, KvTtlResponses, KvTtlErrors, KvUpdateData, KvUpdateResponses, KvUpdateErrors, ListRoutesData, ListRoutesResponses, ListRoutesErrors, CreateRouteData, CreateRouteResponses, CreateRouteErrors, DeleteRouteData, DeleteRouteResponses, DeleteRouteErrors, GetRouteData, GetRouteResponses, GetRouteErrors, UpdateRouteData, UpdateRouteResponses, UpdateRouteErrors, LogoutData, LogoutResponses, LogoutErrors, GetLogContextData, GetLogContextResponses, GetLogContextErrors, SearchLogsData, SearchLogsResponses, SearchLogsErrors, TailLogsData, TailLogsResponses, TailLogsErrors, GetProjectsMonitorHealthData, GetProjectsMonitorHealthResponses, GetProjectsMonitorHealthErrors, DeleteMonitorData, DeleteMonitorResponses, DeleteMonitorErrors, GetMonitorData, GetMonitorResponses, GetMonitorErrors, GetBucketedStatusData, GetBucketedStatusResponses, GetBucketedStatusErrors, GetCurrentMonitorStatusData, GetCurrentMonitorStatusResponses, GetCurrentMonitorStatusErrors, GetUptimeHistoryData, GetUptimeHistoryResponses, GetUptimeHistoryErrors, DeletePreferencesData, DeletePreferencesResponses, DeletePreferencesErrors, GetPreferencesData, GetPreferencesResponses, GetPreferencesErrors, UpdatePreferencesData, UpdatePreferencesResponses, UpdatePreferencesErrors, ListNotificationProvidersData, ListNotificationProvidersResponses, ListNotificationProvidersErrors, CreateNotificationProviderData, CreateNotificationProviderResponses, CreateNotificationProviderErrors, CreateNotificationEmailProviderData, CreateNotificationEmailProviderResponses, CreateNotificationEmailProviderErrors, UpdateEmailProviderData, UpdateEmailProviderResponses, UpdateEmailProviderErrors, CreateSlackProviderData, CreateSlackProviderResponses, CreateSlackProviderErrors, UpdateSlackProviderData, UpdateSlackProviderResponses, UpdateSlackProviderErrors, CreateWebhookProviderData, CreateWebhookProviderResponses, CreateWebhookProviderErrors, UpdateWebhookProviderData, UpdateWebhookProviderResponses, UpdateWebhookProviderErrors, DeleteNotificationProviderData, DeleteNotificationProviderResponses, DeleteNotificationProviderErrors, GetNotificationProviderData, GetNotificationProviderResponses, GetNotificationProviderErrors, UpdateNotificationProviderData, UpdateNotificationProviderResponses, UpdateNotificationProviderErrors, TestNotificationProviderData, TestNotificationProviderResponses, TestNotificationProviderErrors, ListOrdersData, ListOrdersResponses, ListOrdersErrors, QueryGenaiTracesData, QueryGenaiTracesResponses, QueryGenaiTracesErrors, GetGenaiTraceData, GetGenaiTraceResponses, GetGenaiTraceErrors, GetHealthData, GetHealthResponses, GetHealthErrors, ListInsightsData, ListInsightsResponses, ListInsightsErrors, QueryLogsData, QueryLogsResponses, QueryLogsErrors, ListMetricNamesData, ListMetricNamesResponses, ListMetricNamesErrors, QueryMetricsData, QueryMetricsResponses, QueryMetricsErrors, GetPipelineStatsData, GetPipelineStatsResponses, GetPipelineStatsErrors, GetQuotaData, GetQuotaResponses, GetQuotaErrors, QueryTraceSummariesData, QueryTraceSummariesResponses, QueryTraceSummariesErrors, QueryTracesData, QueryTracesResponses, QueryTracesErrors, GetTraceData, GetTraceResponses, GetTraceErrors, IngestLogsData, IngestLogsResponses, IngestLogsErrors, IngestMetricsData, IngestMetricsResponses, IngestMetricsErrors, IngestTracesData, IngestTracesResponses, IngestTracesErrors, IngestLogsByPathData, IngestLogsByPathResponses, IngestLogsByPathErrors, IngestMetricsByPathData, IngestMetricsByPathResponses, IngestMetricsByPathErrors, IngestTracesByPathData, IngestTracesByPathResponses, IngestTracesByPathErrors, HasPerformanceMetricsData, HasPerformanceMetricsResponses, HasPerformanceMetricsErrors, GetPerformanceMetricsData, GetPerformanceMetricsResponses, GetPerformanceMetricsErrors, GetMetricsOverTimeData, GetMetricsOverTimeResponses, GetMetricsOverTimeErrors, GetGroupedPageMetricsData, GetGroupedPageMetricsResponses, GetGroupedPageMetricsErrors, GetAccessInfoData, GetAccessInfoResponses, GetAccessInfoErrors, GetPrivateIpData, GetPrivateIpResponses, GetPrivateIpErrors, GetPublicIpData, GetPublicIpResponses, GetPublicIpErrors, ListPresetsData, ListPresetsResponses, ListPresetsErrors, GeneratePresetDockerfileData, GeneratePresetDockerfileResponses, GeneratePresetDockerfileErrors, GetPreviewGatewayLogsData, GetPreviewGatewayLogsResponses, RestartPreviewGatewayData, RestartPreviewGatewayResponses, GetPreviewGatewaySettingsData, GetPreviewGatewaySettingsResponses, PatchPreviewGatewaySettingsData, PatchPreviewGatewaySettingsResponses, GetPreviewGatewayStatusData, GetPreviewGatewayStatusResponses, UpgradePreviewGatewayData, UpgradePreviewGatewayResponses, GetProjectsData, GetProjectsResponses, GetProjectsErrors, CreateProjectData, CreateProjectResponses, CreateProjectErrors, GetProjectBySlugData, GetProjectBySlugResponses, GetProjectBySlugErrors, CreateProjectFromTemplateData, CreateProjectFromTemplateResponses, CreateProjectFromTemplateErrors, GetProjectStatisticsData, GetProjectStatisticsResponses, GetProjectStatisticsErrors, DeleteProjectData, DeleteProjectResponses, DeleteProjectErrors, GetProjectData, GetProjectResponses, GetProjectErrors, UpdateProjectData, UpdateProjectResponses, UpdateProjectErrors, GetProjectDeploymentsData, GetProjectDeploymentsResponses, GetProjectDeploymentsErrors, GetLastDeploymentData, GetLastDeploymentResponses, GetLastDeploymentErrors, TriggerProjectPipelineData, TriggerProjectPipelineResponses, TriggerProjectPipelineErrors, GetActiveVisitorsData, GetActiveVisitorsResponses, GetActiveVisitorsErrors, ListAgentsData, ListAgentsResponses, ListAgentsErrors, CreateAgentData, CreateAgentResponses, CreateAgentErrors, GetCliStatusData, GetCliStatusResponses, GetCliStatusErrors, ListAllRunsData, ListAllRunsResponses, ListAllRunsErrors, LatestRunForSourceData, LatestRunForSourceResponses, LatestRunForSourceErrors, GetRunWithLogsData, GetRunWithLogsResponses, GetRunWithLogsErrors, CancelRunData, CancelRunResponses, CancelRunErrors, RetryRunData, RetryRunResponses, RetryRunErrors, StreamRunEventsData, StreamRunEventsResponses, StreamRunEventsErrors, GetSandboxStatusData, GetSandboxStatusResponses, GetSandboxStatusErrors, SmokeTestAgentData, SmokeTestAgentResponses, SmokeTestAgentErrors, DeleteAgentData, DeleteAgentResponses, DeleteAgentErrors, GetAgentData, GetAgentResponses, GetAgentErrors, UpdateAgentData, UpdateAgentResponses, UpdateAgentErrors, ListAgentRunsData, ListAgentRunsResponses, ListAgentRunsErrors, TriggerAgentData, TriggerAgentResponses, TriggerAgentErrors, GetAggregatedBucketsData, GetAggregatedBucketsResponses, GetAggregatedBucketsErrors, StartAnalysisData, StartAnalysisResponses, StartAnalysisErrors, GetRunData, GetRunResponses, GetRunErrors, AddContextData, AddContextResponses, AddContextErrors, CancelData, CancelResponses, CancelErrors, CreatePrData, CreatePrResponses, CreatePrErrors, StartFixData, StartFixResponses, StartFixErrors, ReAnalyzeData, ReAnalyzeResponses, ReAnalyzeErrors, StreamEventsData, StreamEventsResponses, StreamEventsErrors, UpdateAutomaticDeployData, UpdateAutomaticDeployResponses, UpdateAutomaticDeployErrors, ListCustomDomainsForProjectData, ListCustomDomainsForProjectResponses, ListCustomDomainsForProjectErrors, CreateCustomDomainData, CreateCustomDomainResponses, CreateCustomDomainErrors, DeleteCustomDomainData, DeleteCustomDomainResponses, DeleteCustomDomainErrors, GetCustomDomainData, GetCustomDomainResponses, GetCustomDomainErrors, UpdateCustomDomainData, UpdateCustomDomainResponses, UpdateCustomDomainErrors, LinkCustomDomainToCertificateData, LinkCustomDomainToCertificateResponses, LinkCustomDomainToCertificateErrors, UpdateProjectDeploymentConfigData, UpdateProjectDeploymentConfigResponses, UpdateProjectDeploymentConfigErrors, GetDeploymentData, GetDeploymentResponses, GetDeploymentErrors, CancelDeploymentData, CancelDeploymentResponses, CancelDeploymentErrors, GetDeploymentJobsData, GetDeploymentJobsResponses, GetDeploymentJobsErrors, GetDeploymentJobLogsData, GetDeploymentJobLogsResponses, GetDeploymentJobLogsErrors, TailDeploymentJobLogsData, TailDeploymentJobLogsErrors, GetDeploymentOperationsData, GetDeploymentOperationsResponses, GetDeploymentOperationsErrors, ExecuteDeploymentOperationData, ExecuteDeploymentOperationResponses, ExecuteDeploymentOperationErrors, GetDeploymentOperationStatusData, GetDeploymentOperationStatusResponses, GetDeploymentOperationStatusErrors, PauseDeploymentData, PauseDeploymentResponses, PauseDeploymentErrors, PromoteDeploymentData, PromoteDeploymentResponses, PromoteDeploymentErrors, ResumeDeploymentData, ResumeDeploymentResponses, ResumeDeploymentErrors, RollbackToDeploymentData, RollbackToDeploymentResponses, RollbackToDeploymentErrors, TeardownDeploymentData, TeardownDeploymentResponses, TeardownDeploymentErrors, ListDsnsData, ListDsnsResponses, CreateDsnData, CreateDsnResponses, CreateDsnErrors, GetOrCreateDsnData, GetOrCreateDsnResponses, GetOrCreateDsnErrors, RegenerateDsnData, RegenerateDsnResponses, RegenerateDsnErrors, RevokeDsnData, RevokeDsnResponses, RevokeDsnErrors, GetEnvironmentVariablesData, GetEnvironmentVariablesResponses, GetEnvironmentVariablesErrors, CreateEnvironmentVariableData, CreateEnvironmentVariableResponses, CreateEnvironmentVariableErrors, GetResolvedEnvironmentVariablesData, GetResolvedEnvironmentVariablesResponses, GetResolvedEnvironmentVariablesErrors, GetResolvedEnvironmentVariableValueData, GetResolvedEnvironmentVariableValueResponses, GetResolvedEnvironmentVariableValueErrors, GetEnvironmentVariableValueData, GetEnvironmentVariableValueResponses, GetEnvironmentVariableValueErrors, DeleteEnvironmentVariableData, DeleteEnvironmentVariableResponses, DeleteEnvironmentVariableErrors, UpdateEnvironmentVariableData, UpdateEnvironmentVariableResponses, UpdateEnvironmentVariableErrors, GetEnvironmentsData, GetEnvironmentsResponses, GetEnvironmentsErrors, CreateEnvironmentData, CreateEnvironmentResponses, CreateEnvironmentErrors, DeleteEnvironmentData, DeleteEnvironmentResponses, DeleteEnvironmentErrors, GetEnvironmentData, GetEnvironmentResponses, GetEnvironmentErrors, GetEnvironmentCronsData, GetEnvironmentCronsResponses, GetEnvironmentCronsErrors, GetCronByIdData, GetCronByIdResponses, GetCronByIdErrors, GetCronExecutionsData, GetCronExecutionsResponses, GetCronExecutionsErrors, GetEnvironmentDomainsData, GetEnvironmentDomainsResponses, GetEnvironmentDomainsErrors, AddEnvironmentDomainData, AddEnvironmentDomainResponses, AddEnvironmentDomainErrors, DeleteEnvironmentDomainData, DeleteEnvironmentDomainResponses, DeleteEnvironmentDomainErrors, UpdateEnvironmentSettingsData, UpdateEnvironmentSettingsResponses, UpdateEnvironmentSettingsErrors, SleepEnvironmentData, SleepEnvironmentResponses, SleepEnvironmentErrors, TeardownEnvironmentData, TeardownEnvironmentResponses, TeardownEnvironmentErrors, WakeEnvironmentData, WakeEnvironmentResponses, WakeEnvironmentErrors, GetContainerLogsData, GetContainerLogsErrors, ListContainersData, ListContainersResponses, ListContainersErrors, GetContainerDetailData, GetContainerDetailResponses, GetContainerDetailErrors, GetContainerLogsByIdData, GetContainerLogsByIdErrors, GetContainerMetricsData, GetContainerMetricsResponses, GetContainerMetricsErrors, StreamContainerMetricsData, StreamContainerMetricsResponses, StreamContainerMetricsErrors, RestartContainerData, RestartContainerResponses, RestartContainerErrors, StartContainerData, StartContainerResponses, StartContainerErrors, StopContainerData, StopContainerResponses, StopContainerErrors, DeployFromImageData, DeployFromImageResponses, DeployFromImageErrors, DeployFromImageUploadData, DeployFromImageUploadResponses, DeployFromImageUploadErrors, DeployFromStaticData, DeployFromStaticResponses, DeployFromStaticErrors, ListAlertRulesData, ListAlertRulesResponses, ListAlertRulesErrors, CreateAlertRuleData, CreateAlertRuleResponses, CreateAlertRuleErrors, DeleteAlertRuleData, DeleteAlertRuleResponses, DeleteAlertRuleErrors, GetAlertRuleData, GetAlertRuleResponses, GetAlertRuleErrors, UpdateAlertRuleData, UpdateAlertRuleResponses, UpdateAlertRuleErrors, GetErrorDashboardStatsData, GetErrorDashboardStatsResponses, GetErrorDashboardStatsErrors, ListErrorGroupsData, ListErrorGroupsResponses, ListErrorGroupsErrors, GetErrorGroupData, GetErrorGroupResponses, GetErrorGroupErrors, UpdateErrorGroupData, UpdateErrorGroupResponses, UpdateErrorGroupErrors, ListErrorEventsData, ListErrorEventsResponses, ListErrorEventsErrors, GetErrorEventData, GetErrorEventResponses, GetErrorEventErrors, GetErrorStatsData, GetErrorStatsResponses, GetErrorStatsErrors, GetErrorTimeSeriesData, GetErrorTimeSeriesResponses, GetErrorTimeSeriesErrors, GetEventsCountData, GetEventsCountResponses, GetEventsCountErrors, GetEventTypeBreakdownData, GetEventTypeBreakdownResponses, GetEventTypeBreakdownErrors, RecordConsoleEventData, RecordConsoleEventResponses, RecordConsoleEventErrors, GetPropertyBreakdownData, GetPropertyBreakdownResponses, GetPropertyBreakdownErrors, GetPropertyTimelineData, GetPropertyTimelineResponses, GetPropertyTimelineErrors, GetEventsTimelineData, GetEventsTimelineResponses, GetEventsTimelineErrors, GetUniqueEventsData, GetUniqueEventsResponses, GetUniqueEventsErrors, ListRemoteExternalImagesData, ListRemoteExternalImagesResponses, ListRemoteExternalImagesErrors, RegisterExternalImageData, RegisterExternalImageResponses, RegisterExternalImageErrors, DeleteExternalImageData, DeleteExternalImageResponses, DeleteExternalImageErrors, GetRemoteExternalImageData, GetRemoteExternalImageResponses, GetRemoteExternalImageErrors, ListFunnelsData, ListFunnelsResponses, ListFunnelsErrors, CreateFunnelData, CreateFunnelResponses, CreateFunnelErrors, PreviewFunnelMetricsData, PreviewFunnelMetricsResponses, PreviewFunnelMetricsErrors, DeleteFunnelData, DeleteFunnelResponses, DeleteFunnelErrors, UpdateFunnelData, UpdateFunnelResponses, UpdateFunnelErrors, GetFunnelMetricsData, GetFunnelMetricsResponses, GetFunnelMetricsErrors, UpdateGitSettingsData, UpdateGitSettingsResponses, UpdateGitSettingsErrors, ReinstallGitlabWebhookData, ReinstallGitlabWebhookResponses, ReinstallGitlabWebhookErrors, HasErrorGroupsData, HasErrorGroupsResponses, HasErrorGroupsErrors, HasAnalyticsEventsData, HasAnalyticsEventsResponses, HasAnalyticsEventsErrors, GetHourlyVisitsData, GetHourlyVisitsResponses, GetHourlyVisitsErrors, ListExternalImagesData, ListExternalImagesResponses, ListExternalImagesErrors, PushExternalImageData, PushExternalImageResponses, PushExternalImageErrors, GetExternalImageData, GetExternalImageResponses, GetExternalImageErrors, ListIncidentsData, ListIncidentsResponses, ListIncidentsErrors, CreateIncidentData, CreateIncidentResponses, CreateIncidentErrors, GetBucketedIncidentsData, GetBucketedIncidentsResponses, GetBucketedIncidentsErrors, PurgeProjectLogsData, PurgeProjectLogsResponses, PurgeProjectLogsErrors, ListMcpsData, ListMcpsResponses, ListMcpsErrors, CreateMcpData, CreateMcpResponses, CreateMcpErrors, DeleteMcpData, DeleteMcpResponses, DeleteMcpErrors, GetMcpData, GetMcpResponses, GetMcpErrors, UpdateMcpData, UpdateMcpResponses, UpdateMcpErrors, ListMonitorsData, ListMonitorsResponses, ListMonitorsErrors, CreateMonitorData, CreateMonitorResponses, CreateMonitorErrors, ObservabilityListEventsData, ObservabilityListEventsResponses, ObservabilityListEventsErrors, ObservabilityFullEventData, ObservabilityFullEventResponses, ObservabilityFullEventErrors, DeleteReleaseSourceMapsData, DeleteReleaseSourceMapsResponses, DeleteReleaseSourceMapsErrors, ListSourceMapsData, ListSourceMapsResponses, ListSourceMapsErrors, UploadSourceMapData, UploadSourceMapResponses, UploadSourceMapErrors, RevenueRecentEventsData, RevenueRecentEventsResponses, RevenueListIntegrationsData, RevenueListIntegrationsResponses, RevenueCreateIntegrationData, RevenueCreateIntegrationResponses, RevenueCreateIntegrationErrors, RevenueDeleteIntegrationData, RevenueDeleteIntegrationResponses, RevenueUpdateConfigData, RevenueUpdateConfigResponses, RevenueUpdateConfigErrors, RevenueImportInvoicesCsvData, RevenueImportInvoicesCsvResponses, RevenueImportInvoicesCsvErrors, RevenueImportSubscriptionsCsvData, RevenueImportSubscriptionsCsvResponses, RevenueImportSubscriptionsCsvErrors, RevenueRotateTokenData, RevenueRotateTokenResponses, RevenueUpdateSecretData, RevenueUpdateSecretResponses, RevenueUpdateSecretErrors, RevenueMetricsCustomersData, RevenueMetricsCustomersResponses, RevenueMetricsMrrData, RevenueMetricsMrrResponses, RevenueMetricsSummaryData, RevenueMetricsSummaryResponses, ListProjectSecretsData, ListProjectSecretsResponses, ListProjectSecretsErrors, CreateProjectSecretData, CreateProjectSecretResponses, CreateProjectSecretErrors, DeleteProjectSecretData, DeleteProjectSecretResponses, DeleteProjectSecretErrors, UpdateProjectSecretData, UpdateProjectSecretResponses, UpdateProjectSecretErrors, UpdateProjectSettingsData, UpdateProjectSettingsResponses, UpdateProjectSettingsErrors, ListSkillsData, ListSkillsResponses, ListSkillsErrors, CreateSkillData, CreateSkillResponses, CreateSkillErrors, UploadSkillData, UploadSkillResponses, UploadSkillErrors, DeleteSkillData, DeleteSkillResponses, DeleteSkillErrors, GetSkillData, GetSkillResponses, GetSkillErrors, UpdateSkillData, UpdateSkillResponses, UpdateSkillErrors, DownloadSkillArchiveData, DownloadSkillArchiveResponses, DownloadSkillArchiveErrors, ListReleasesData, ListReleasesResponses, ListReleasesErrors, DeleteSourceMapData, DeleteSourceMapResponses, DeleteSourceMapErrors, ListStaticBundlesData, ListStaticBundlesResponses, ListStaticBundlesErrors, DeleteStaticBundleData, DeleteStaticBundleResponses, DeleteStaticBundleErrors, GetStaticBundleData, GetStaticBundleResponses, GetStaticBundleErrors, GetStatusOverviewData, GetStatusOverviewResponses, GetStatusOverviewErrors, GetUniqueCountsData, GetUniqueCountsResponses, GetUniqueCountsErrors, UploadStaticBundleData, UploadStaticBundleResponses, UploadStaticBundleErrors, ListProjectScansData, ListProjectScansResponses, ListProjectScansErrors, TriggerScanData, TriggerScanResponses, TriggerScanErrors, GetLatestScansPerEnvironmentData, GetLatestScansPerEnvironmentResponses, GetLatestScansPerEnvironmentErrors, GetLatestScanData, GetLatestScanResponses, GetLatestScanErrors, ListWebhooksData, ListWebhooksResponses, ListWebhooksErrors, CreateWebhookData, CreateWebhookResponses, CreateWebhookErrors, DeleteWebhookData, DeleteWebhookResponses, DeleteWebhookErrors, GetWebhookData, GetWebhookResponses, GetWebhookErrors, UpdateWebhookData, UpdateWebhookResponses, UpdateWebhookErrors, ListDeliveriesData, ListDeliveriesResponses, ListDeliveriesErrors, GetDeliveryData, GetDeliveryResponses, GetDeliveryErrors, RetryDeliveryData, RetryDeliveryResponses, RetryDeliveryErrors, WorkflowDryRunData, WorkflowDryRunResponses, WorkflowDryRunErrors, ListMemoryData, ListMemoryResponses, ListMemoryErrors, WriteMemoryData, WriteMemoryResponses, WriteMemoryErrors, SearchMemoryData, SearchMemoryResponses, SearchMemoryErrors, DropMemoryData, DropMemoryResponses, DropMemoryErrors, SupersedeMemoryData, SupersedeMemoryResponses, SupersedeMemoryErrors, WorkspaceListSessionsData, WorkspaceListSessionsResponses, WorkspaceListSessionsErrors, WorkspaceStartSessionData, WorkspaceStartSessionResponses, WorkspaceStartSessionErrors, WorkspaceDeleteSessionData, WorkspaceDeleteSessionResponses, WorkspaceDeleteSessionErrors, WorkspaceGetSessionData, WorkspaceGetSessionResponses, WorkspaceGetSessionErrors, WorkspaceUpdateSessionData, WorkspaceUpdateSessionResponses, WorkspaceUpdateSessionErrors, WorkspaceCancelRunData, WorkspaceCancelRunResponses, WorkspaceCancelRunErrors, WorkspaceCloseSessionData, WorkspaceCloseSessionResponses, WorkspaceCloseSessionErrors, WorkspaceSendMessageData, WorkspaceSendMessageResponses, WorkspaceSendMessageErrors, WorkspaceRegeneratePreviewPasswordData, WorkspaceRegeneratePreviewPasswordResponses, WorkspaceRegeneratePreviewPasswordErrors, WorkspaceReopenSessionData, WorkspaceReopenSessionResponses, WorkspaceReopenSessionErrors, WorkspaceRefreshSandboxData, WorkspaceRefreshSandboxResponses, WorkspaceRefreshSandboxErrors, WorkspaceRestartSandboxData, WorkspaceRestartSandboxResponses, WorkspaceRestartSandboxErrors, WorkspaceStartSandboxData, WorkspaceStartSandboxResponses, WorkspaceStartSandboxErrors, WorkspaceSandboxStatsData, WorkspaceSandboxStatsResponses, WorkspaceSandboxStatsErrors, WorkspaceStopSandboxData, WorkspaceStopSandboxResponses, WorkspaceStopSandboxErrors, WorkspaceStreamMessagesData, WorkspaceStreamMessagesResponses, WorkspaceStreamMessagesErrors, WorkspaceTerminalPasteImageData, WorkspaceTerminalPasteImageResponses, WorkspaceTerminalPasteImageErrors, WorkspaceListTerminalTabsData, WorkspaceListTerminalTabsResponses, WorkspaceListTerminalTabsErrors, WorkspaceDeleteTerminalTabData, WorkspaceDeleteTerminalTabResponses, WorkspaceDeleteTerminalTabErrors, GetProxyLogsData, GetProxyLogsResponses, GetProxyLogsErrors, GetProxyLogByRequestIdData, GetProxyLogByRequestIdResponses, GetProxyLogByRequestIdErrors, GetProjectsHealthData, GetProjectsHealthResponses, GetProjectsHealthErrors, GetTimeBucketStatsData, GetTimeBucketStatsResponses, GetTimeBucketStatsErrors, GetTodayStatsData, GetTodayStatsResponses, GetTodayStatsErrors, GetProxyLogByIdData, GetProxyLogByIdResponses, GetProxyLogByIdErrors, ListSyncedRepositoriesData, ListSyncedRepositoriesResponses, ListSyncedRepositoriesErrors, GetRepositoryByNameData, GetRepositoryByNameResponses, GetRepositoryByNameErrors, GetAllRepositoriesByNameData, GetAllRepositoriesByNameResponses, GetAllRepositoriesByNameErrors, GetRepositoryPresetByNameData, GetRepositoryPresetByNameResponses, GetRepositoryPresetByNameErrors, GetRepositoryBranchesData, GetRepositoryBranchesResponses, GetRepositoryBranchesErrors, GetRepositoryTagsData, GetRepositoryTagsResponses, GetRepositoryTagsErrors, GetRepositoryPresetLiveData, GetRepositoryPresetLiveResponses, GetRepositoryPresetLiveErrors, GetRepositoryByIdData, GetRepositoryByIdResponses, GetRepositoryByIdErrors, GetBranchesByRepositoryIdData, GetBranchesByRepositoryIdResponses, GetBranchesByRepositoryIdErrors, ListCommitsByRepositoryIdData, ListCommitsByRepositoryIdResponses, ListCommitsByRepositoryIdErrors, CheckCommitExistsData, CheckCommitExistsResponses, CheckCommitExistsErrors, GetTagsByRepositoryIdData, GetTagsByRepositoryIdResponses, GetTagsByRepositoryIdErrors, GetRestoreRunData, GetRestoreRunResponses, GetRestoreRunErrors, RevenueGlobalEventsData, RevenueGlobalEventsResponses, RevenueMetricsGlobalMrrData, RevenueMetricsGlobalMrrResponses, RevenueMetricsGlobalSummaryData, RevenueMetricsGlobalSummaryResponses, RevenueListProvidersData, RevenueListProvidersResponses, GetProjectSessionReplaysData, GetProjectSessionReplaysResponses, GetProjectSessionReplaysErrors, GetSessionEventsData, GetSessionEventsResponses, GetSessionEventsErrors, GetSettingsData, GetSettingsResponses, GetSettingsErrors, UpdateSettingsData, UpdateSettingsResponses, UpdateSettingsErrors, SaveAgentTokenData, SaveAgentTokenResponses, SaveAgentTokenErrors, ListAiProvidersData, ListAiProvidersResponses, ListAiProvidersErrors, UpdateAiProviderData, UpdateAiProviderResponses, UpdateAiProviderErrors, ActivateAiProviderData, ActivateAiProviderResponses, ActivateAiProviderErrors, SaveAiProviderCredentialData, SaveAiProviderCredentialResponses, SaveAiProviderCredentialErrors, RevokeJoinTokenData, RevokeJoinTokenResponses, RevokeJoinTokenErrors, GenerateJoinTokenData, GenerateJoinTokenResponses, GenerateJoinTokenErrors, GetJoinTokenStatusData, GetJoinTokenStatusResponses, GetJoinTokenStatusErrors, ListGlobalMcpsData, ListGlobalMcpsResponses, ListGlobalMcpsErrors, CreateGlobalMcpData, CreateGlobalMcpResponses, CreateGlobalMcpErrors, DeleteGlobalMcpData, DeleteGlobalMcpResponses, DeleteGlobalMcpErrors, GetGlobalMcpData, GetGlobalMcpResponses, GetGlobalMcpErrors, UpdateGlobalMcpData, UpdateGlobalMcpResponses, UpdateGlobalMcpErrors, RefreshRouteTableData, RefreshRouteTableResponses, RefreshRouteTableErrors, RebuildSandboxImageData, RebuildSandboxImageResponses, RebuildSandboxImageErrors, GetGlobalSandboxStatusData, GetGlobalSandboxStatusResponses, GetGlobalSandboxStatusErrors, ListSecretsData, ListSecretsResponses, ListSecretsErrors, UpsertSecretData, UpsertSecretResponses, UpsertSecretErrors, DeleteSecretData, DeleteSecretResponses, DeleteSecretErrors, ListGlobalSkillsData, ListGlobalSkillsResponses, ListGlobalSkillsErrors, CreateGlobalSkillData, CreateGlobalSkillResponses, CreateGlobalSkillErrors, UploadGlobalSkillData, UploadGlobalSkillResponses, UploadGlobalSkillErrors, DeleteGlobalSkillData, DeleteGlobalSkillResponses, DeleteGlobalSkillErrors, GetGlobalSkillData, GetGlobalSkillResponses, GetGlobalSkillErrors, UpdateGlobalSkillData, UpdateGlobalSkillResponses, UpdateGlobalSkillErrors, DownloadGlobalSkillArchiveData, DownloadGlobalSkillArchiveResponses, DownloadGlobalSkillArchiveErrors, ListProjectTemplatesData, ListProjectTemplatesResponses, ListProjectTemplatesErrors, ListProjectTemplateTagsData, ListProjectTemplateTagsResponses, ListProjectTemplateTagsErrors, GetProjectTemplateData, GetProjectTemplateResponses, GetProjectTemplateErrors, GetCurrentUserData, GetCurrentUserResponses, GetCurrentUserErrors, ListUsersData, ListUsersResponses, ListUsersErrors, CreateUserData, CreateUserResponses, CreateUserErrors, UpdateSelfData, UpdateSelfResponses, UpdateSelfErrors, DisableMfaData, DisableMfaResponses, DisableMfaErrors, SetupMfaData, SetupMfaResponses, SetupMfaErrors, VerifyAndEnableMfaData, VerifyAndEnableMfaResponses, VerifyAndEnableMfaErrors, DeleteUserData, DeleteUserResponses, DeleteUserErrors, UpdateUserData, UpdateUserResponses, UpdateUserErrors, RestoreUserData, RestoreUserResponses, RestoreUserErrors, AssignRoleData, AssignRoleResponses, AssignRoleErrors, RemoveRoleData, RemoveRoleResponses, RemoveRoleErrors, ListSandboxesData, ListSandboxesResponses, CreateSandboxData, CreateSandboxResponses, CreateSandboxErrors, GetSandboxData, GetSandboxResponses, GetSandboxErrors, CmdData, CmdResponses, CmdErrors, GetCmdData, GetCmdResponses, GetCmdErrors, CmdLogsData, CmdLogsResponses, CmdLogsErrors, DestroySandboxData, DestroySandboxResponses, DestroySandboxErrors, DomainData, DomainResponses, DomainErrors, ExecData, ExecResponses, ExecErrors, ExecDetachedData, ExecDetachedResponses, ExecDetachedErrors, ExtendTimeoutData, ExtendTimeoutResponses, ExtendTimeoutErrors, MkdirData, MkdirResponses, MkdirErrors, ReadFileData, ReadFileResponses, ReadFileErrors, StatPathData, StatPathResponses, StatPathErrors, WriteFileData, WriteFileResponses, WriteFileErrors, WriteFilesData, WriteFilesResponses, WriteFilesErrors, ListJobsData, ListJobsResponses, ListJobsErrors, JobStatusData, JobStatusResponses, JobStatusErrors, KillJobData, KillJobResponses, KillJobErrors, JobLogsData, JobLogsResponses, JobLogsErrors, PauseSandboxData, PauseSandboxResponses, PauseSandboxErrors, ClearPreviewPasswordData, ClearPreviewPasswordResponses, ClearPreviewPasswordErrors, SetPreviewPasswordData, SetPreviewPasswordResponses, SetPreviewPasswordErrors, RestartSandboxData, RestartSandboxResponses, RestartSandboxErrors, ResumeSandboxData, ResumeSandboxResponses, ResumeSandboxErrors, SourceSandboxData, SourceSandboxResponses, SourceSandboxErrors, StopSandboxData, StopSandboxResponses, StopSandboxErrors, CmdKillData, CmdKillResponses, CmdKillErrors, GetVisitorSessionsData, GetVisitorSessionsResponses, GetVisitorSessionsErrors, DeleteSessionReplayData, DeleteSessionReplayResponses, DeleteSessionReplayErrors, GetSessionReplayData, GetSessionReplayResponses, GetSessionReplayErrors, UpdateSessionDurationData, UpdateSessionDurationResponses, UpdateSessionDurationErrors, GetSessionReplayEventsData, GetSessionReplayEventsResponses, GetSessionReplayEventsErrors, AddEventsData, AddEventsResponses, AddEventsErrors, DeleteScanData, DeleteScanResponses, DeleteScanErrors, GetScanData, GetScanResponses, GetScanErrors, GetScanVulnerabilitiesData, GetScanVulnerabilitiesResponses, GetScanVulnerabilitiesErrors, ListEventTypesData, ListEventTypesResponses, TriggerWeeklyDigestData, TriggerWeeklyDigestResponses, TriggerWeeklyDigestErrors, ListExternalPluginsData, ListExternalPluginsResponses, ReloadPluginsData, ReloadPluginsResponses, ReloadPluginsErrors, IngestSentryEnvelopeData, IngestSentryEnvelopeResponses, IngestSentryEnvelopeErrors, IngestSentryEventData, IngestSentryEventResponses, IngestSentryEventErrors, ListAuditLogsData, ListAuditLogsResponses, ListAuditLogsErrors, GetAuditLogData, GetAuditLogResponses, GetAuditLogErrors } from './types.gen'; import { client } from './client.gen'; export type Options = ClientOptions & { @@ -7488,6 +7488,45 @@ export const createMonitor = (options: Opt }); }; +/** + * List a merged page of observability events for a project. + * Each row carries everything the side panel needs to render — no + * follow-up fetch is required for the common case. Heavy fields + * (stacktraces, headers, span attributes) are truncated server-side and + * expose a `*_truncated` flag; clients fetch the full row from the + * `/full` endpoint only when the user explicitly clicks "Show full". + */ +export const observabilityListEvents = (options: Options) => { + return (options.client ?? client).get({ + security: [ + { + scheme: 'bearer', + type: 'http' + } + ], + url: '/projects/{project_id}/observe/events', + ...options + }); +}; + +/** + * Fetch the un-truncated form of one event by `(kind, id)`. Side panel + * "Show full" action calls this — the list response carries truncated + * previews + a `*_truncated` flag to let the UI decide whether to fetch. + */ +export const observabilityFullEvent = (options: Options) => { + return (options.client ?? client).get({ + security: [ + { + scheme: 'bearer', + type: 'http' + } + ], + url: '/projects/{project_id}/observe/events/{kind}/{event_id}/full', + ...options + }); +}; + /** * Delete all source maps for a specific release */ diff --git a/web/src/api/client/types.gen.ts b/web/src/api/client/types.gen.ts index 3f7c31ae9..468228e4b 100644 --- a/web/src/api/client/types.gen.ts +++ b/web/src/api/client/types.gen.ts @@ -4132,6 +4132,20 @@ export type ErrorResponse = { error: string; }; +export type ErrorRow = { + deployment_id?: number | null; + environment_id?: number | null; + error_class: string; + error_group_id: number; + fingerprint: string; + id: number; + message?: string | null; + stacktrace_preview: unknown; + stacktrace_truncated: boolean; + trace_id?: string | null; + ts: string; +}; + export type ErrorTimeSeriesDataResponse = { count: number; timestamp: string; @@ -4272,6 +4286,12 @@ export type EventDetailResponse = { unique_visitors: number; }; +/** + * Tag enum for filter parameters and routing. Matches the variant + * discriminator used by `ObservabilityEvent`. + */ +export type EventKind = 'request' | 'span' | 'error' | 'revenue'; + export type EventMetricsPayload = { /** * Cumulative Layout Shift (score) @@ -4498,6 +4518,15 @@ export type EventsCountQuery = { start_date: string; }; +export type EventsResponse = { + /** + * Echo of the kinds filter actually applied (server-resolved). Useful + * for clients that pass `kinds=` empty and want to know what they got. + */ + applied_kinds: Array; + events: Array; +}; + export type ExecBody = { cmd: Array; cwd?: string | null; @@ -4730,6 +4759,58 @@ export type FieldResponse = { nullable: boolean; }; +export type FullError = { + /** + * Full JSONB blob from `error_events.data` — stack trace, breadcrumbs, + * request context, everything. Schema is documented per source SDK. + */ + data?: unknown; + deployment_id?: number | null; + environment_id?: number | null; + error_class: string; + error_group_id: number; + fingerprint: string; + id: number; + message?: string | null; + trace_id?: string | null; + ts: string; +}; + +/** + * One un-truncated row, returned by the `/full/{type}/{id}` endpoint when + * the user clicks "Show full". Same shape as the list rows, but with the + * raw heavy fields restored (no truncation flags) so the side panel can + * render the long form. + */ +export type FullEvent = (FullRequest & { + type: 'request'; +}) | (FullError & { + type: 'error'; +}) | (RevenueRow & { + type: 'revenue'; +}) | (SpanRow & { + type: 'span'; +}); + +export type FullRequest = { + client_ip?: string | null; + deployment_id?: number | null; + environment_id?: number | null; + error_group_id?: number | null; + host: string; + id: number; + latency_ms?: number | null; + method: string; + path: string; + referrer?: string | null; + request_headers?: unknown; + response_headers?: unknown; + status: number; + trace_id?: string | null; + ts: string; + user_agent?: string | null; +}; + export type FunnelMetricsResponse = { average_completion_time_seconds: number; funnel_id: number; @@ -7245,6 +7326,29 @@ export type NotificationProviderResponse = { updated_at: number; }; +/** + * Discriminated union of every row that can appear in the Observe list. + * + * Serializes to `{ "type": "request" | "span" | ... , ...rest }` so the UI + * can switch on `event.type` without ambiguity. + * + * **No `Log` variant**: runtime stdout/stderr lines live on a dedicated + * Logs page rather than Observe. Logs are too high-volume to interleave + * with business signals (requests, errors, revenue) without dominating + * the timeline, and they have their own retention/storage constraints + * (TimescaleDB hypertable + chunked file/S3 store) that don't compose + * with the merge service's per-kind LIMIT strategy. + */ +export type ObservabilityEvent = (RequestRow & { + type: 'request'; +}) | (SpanRow & { + type: 'span'; +}) | (ErrorRow & { + type: 'error'; +}) | (RevenueRow & { + type: 'revenue'; +}); + export type OpenAiError = { code?: string | null; message: string; @@ -9418,6 +9522,28 @@ export type RepositorySyncStartedResponse = { syncing: boolean; }; +export type RequestRow = { + client_ip?: string | null; + country?: string | null; + deployment_id?: number | null; + environment_id?: number | null; + error_group_id?: number | null; + headers_truncated: boolean; + host: string; + id: number; + latency_ms?: number | null; + method: string; + path: string; + query_string?: string | null; + referrer?: string | null; + request_headers: unknown; + response_headers: unknown; + status: number; + trace_id?: string | null; + ts: string; + user_agent?: string | null; +}; + export type ResetPasswordRequest = { new_password: string; token: string; @@ -9676,6 +9802,19 @@ export type RetryClusterRequest = { members?: Array; }; +export type RevenueRow = { + amount_minor?: number | null; + currency?: string | null; + customer_ref?: string | null; + deployment_id?: number | null; + environment_id?: number | null; + event_type: string; + id: number; + provider: string; + trace_id?: string | null; + ts: string; +}; + /** * Risk level for a migration step */ @@ -11045,6 +11184,22 @@ export type SpanRecord = { trace_id: string; }; +export type SpanRow = { + attributes: unknown; + attributes_truncated: boolean; + deployment_id?: number | null; + duration_ms?: number | null; + environment_id?: number | null; + id: string; + operation: string; + parent_span_id?: string | null; + service: string; + span_id: string; + status?: string | null; + trace_id: string; + ts: string; +}; + /** * Span status code. */ @@ -31725,6 +31880,123 @@ export type CreateMonitorResponses = { export type CreateMonitorResponse = CreateMonitorResponses[keyof CreateMonitorResponses]; +export type ObservabilityListEventsData = { + body?: never; + path: { + /** + * Project ID + */ + project_id: number; + }; + query?: { + /** + * Comma-separated kinds: `log,request,span,error,revenue`. Empty or + * missing returns every kind. + */ + kinds?: string; + /** + * Inclusive lower bound on event timestamp (ISO 8601, `Z` suffix). + */ + from?: string; + /** + * Inclusive upper bound on event timestamp. + */ + to?: string; + deployment_id?: number; + environment_id?: number; + /** + * Free-text substring matched against per-kind summary fields + * (request path / error class / revenue event_type). + */ + search?: string; + /** + * Page size (default 50, max 200). + */ + limit?: number; + }; + url: '/projects/{project_id}/observe/events'; +}; + +export type ObservabilityListEventsErrors = { + /** + * Invalid filter (kinds, time range, …) + */ + 400: string; + /** + * Unauthorized + */ + 401: string; + /** + * Insufficient permissions + */ + 403: string; + /** + * Internal server error + */ + 500: string; +}; + +export type ObservabilityListEventsError = ObservabilityListEventsErrors[keyof ObservabilityListEventsErrors]; + +export type ObservabilityListEventsResponses = { + /** + * Merged event page + */ + 200: EventsResponse; +}; + +export type ObservabilityListEventsResponse = ObservabilityListEventsResponses[keyof ObservabilityListEventsResponses]; + +export type ObservabilityFullEventData = { + body?: never; + path: { + /** + * Project ID + */ + project_id: number; + /** + * Event kind discriminator + */ + kind: EventKind; + /** + * Per-kind primary key + */ + event_id: string; + }; + query?: never; + url: '/projects/{project_id}/observe/events/{kind}/{event_id}/full'; +}; + +export type ObservabilityFullEventErrors = { + /** + * Unauthorized + */ + 401: string; + /** + * Insufficient permissions + */ + 403: string; + /** + * Event not found in project + */ + 404: string; + /** + * Internal server error + */ + 500: string; +}; + +export type ObservabilityFullEventError = ObservabilityFullEventErrors[keyof ObservabilityFullEventErrors]; + +export type ObservabilityFullEventResponses = { + /** + * Full row + */ + 200: FullEvent; +}; + +export type ObservabilityFullEventResponse = ObservabilityFullEventResponses[keyof ObservabilityFullEventResponses]; + export type DeleteReleaseSourceMapsData = { body?: never; path: { diff --git a/web/src/components/dashboard/Sidebar.tsx b/web/src/components/dashboard/Sidebar.tsx index 1ae8e81b9..ed8a73cba 100644 --- a/web/src/components/dashboard/Sidebar.tsx +++ b/web/src/components/dashboard/Sidebar.tsx @@ -683,10 +683,12 @@ const projectBaseNav: ProjectNavItem[] = [ ], }, { - title: 'Observability', - url: 'monitors', + title: 'Observe', + url: 'observe', icon: Eye, + navigateOnClick: true, subItems: [ + { title: 'All events', url: 'observe', icon: Eye }, { title: 'Uptime', url: 'monitors', icon: Activity }, { title: 'Metrics', url: 'monitoring', icon: Gauge }, { title: 'Traces', url: 'traces', icon: Network }, diff --git a/web/src/components/observe/ObserveFilterBar.tsx b/web/src/components/observe/ObserveFilterBar.tsx new file mode 100644 index 000000000..2138788d7 --- /dev/null +++ b/web/src/components/observe/ObserveFilterBar.tsx @@ -0,0 +1,151 @@ +import { Button } from '@/components/ui/button' +import { Input } from '@/components/ui/input' +// Toggle UI primitive doesn't exist in this project — we render kind +// toggles as Button with `aria-pressed` and a secondary/ghost variant. +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select' +import { cn } from '@/lib/utils' +import { + AlertOctagon, + CircleDollarSign, + Network, + Workflow, +} from 'lucide-react' +import { ALL_KINDS, type EventKind } from './types' + +const KIND_META: Record< + EventKind, + { label: string; icon: React.ComponentType<{ className?: string }> } +> = { + request: { label: 'Requests', icon: Network }, + span: { label: 'Traces', icon: Workflow }, + error: { label: 'Errors', icon: AlertOctagon }, + revenue: { label: 'Revenue', icon: CircleDollarSign }, +} + +export const TIME_RANGES = [ + { value: '15m', label: 'Last 15 minutes' }, + { value: '1h', label: 'Last hour' }, + { value: '24h', label: 'Last 24 hours' }, + { value: '7d', label: 'Last 7 days' }, + { value: '30d', label: 'Last 30 days' }, +] as const + +export type TimeRange = (typeof TIME_RANGES)[number]['value'] + +export interface ObserveFilters { + kinds: EventKind[] + timeRange: TimeRange + search: string + environmentId: number | null +} + +export function ObserveFilterBar({ + filters, + onChange, + environmentOptions, +}: { + filters: ObserveFilters + onChange: (next: ObserveFilters) => void + environmentOptions?: Array<{ id: number; name: string }> +}) { + const toggleKind = (kind: EventKind) => { + const next = filters.kinds.includes(kind) + ? filters.kinds.filter((k) => k !== kind) + : [...filters.kinds, kind] + // If user toggled the last one off, fall back to all-on so the page + // never shows an empty list with no recourse. + onChange({ + ...filters, + kinds: next.length === 0 ? [...ALL_KINDS] : next, + }) + } + + return ( +
+ {/* Kind toggles */} +
+ {ALL_KINDS.map((kind) => { + const { label, icon: Icon } = KIND_META[kind] + const active = filters.kinds.includes(kind) + return ( + + ) + })} +
+ + {/* Time range */} + + + {/* Environment */} + {environmentOptions && environmentOptions.length > 0 && ( + + )} + + {/* Search */} + onChange({ ...filters, search: e.target.value })} + className="w-full sm:w-[260px]" + /> +
+ ) +} diff --git a/web/src/components/observe/panels/ObservePanel.tsx b/web/src/components/observe/panels/ObservePanel.tsx new file mode 100644 index 000000000..fb2dec4b9 --- /dev/null +++ b/web/src/components/observe/panels/ObservePanel.tsx @@ -0,0 +1,343 @@ +import { Fragment } from 'react' +import { Badge } from '@/components/ui/badge' +import { Button } from '@/components/ui/button' +import { CopyButton } from '@/components/ui/copy-button' +import { + Sheet, + SheetContent, + SheetDescription, + SheetHeader, + SheetTitle, +} from '@/components/ui/sheet' +import { ExternalLink } from 'lucide-react' +import { Link } from 'react-router-dom' +import type { ObservabilityEvent } from '../types' + +/** + * Side panel for any event kind. Renders entirely from the row payload — + * no follow-up fetch in the common case. The "Show full" call lives on + * each per-kind panel (see PanelStacktrace etc.) and only fires when the + * user clicks it. + */ +export function ObservePanel({ + event, + open, + onOpenChange, + projectSlug, +}: { + event: ObservabilityEvent | null + open: boolean + onOpenChange: (open: boolean) => void + projectSlug: string +}) { + return ( + + + {event && } + + + ) +} + +function PanelBody({ + event, + projectSlug, +}: { + event: ObservabilityEvent + projectSlug: string +}) { + return ( + <> + + {titleFor(event)} + + {new Date(event.ts).toISOString()} + + + +
+ + +
+ + ) +} + +function titleFor(event: ObservabilityEvent): string { + switch (event.type) { + case 'request': + return `${event.method} ${event.path}` + case 'span': + return `${event.service} · ${event.operation}` + case 'error': + return event.error_class + case 'revenue': + return event.event_type + } +} + +/** + * Universal cross-event correlation row: trace_id, deployment, environment. + * Lets the user jump from any event to its peers in the same trace. + */ +function CorrelationStrip({ + event, + projectSlug, +}: { + event: ObservabilityEvent + projectSlug: string +}) { + const items: { label: string; value: React.ReactNode }[] = [] + if ('trace_id' in event && event.trace_id) { + items.push({ + label: 'Trace', + value: ( + + + {event.trace_id.slice(0, 8)}… + + + + + + + ), + }) + } + if (event.deployment_id != null) { + items.push({ + label: 'Deployment', + value: #{event.deployment_id}, + }) + } + if (event.environment_id != null) { + items.push({ + label: 'Environment', + value: #{event.environment_id}, + }) + } + if (items.length === 0) return null + return ( +
+ {items.map((item) => ( + + {item.label} + {item.value} + + ))} +
+ ) +} + +function KindSpecificDetails({ + event, + projectSlug, +}: { + event: ObservabilityEvent + projectSlug: string +}) { + switch (event.type) { + case 'request': + return ( + <> +
+ +
+ + + {event.error_group_id != null && ( +
+ + Open error group #{event.error_group_id} + + +
+ )} + + ) + case 'span': + return ( + <> +
+ +
+
+ 
+              {JSON.stringify(event.attributes, null, 2)}
+
+ {event.attributes_truncated && ( +

+ Showing first attributes only — open the trace for the full + set. +

+ )} +
+ + ) + case 'error': + return ( + <> +
+ + + Open error group #{event.error_group_id} + + +
+
+ 
+              {JSON.stringify(event.stacktrace_preview, null, 2)}
+
+ {event.stacktrace_truncated && ( + + )} +
+ + ) + case 'revenue': + return ( +
+ +
+ ) + } +} + +function HeadersBlock({ + label, + headers, + truncated, +}: { + label: string + // The wire type is `unknown` (serde_json::Value); we narrow to a record at runtime. + headers: unknown + truncated: boolean +}) { + const entries = + headers && typeof headers === 'object' + ? Object.entries(headers as Record) + : [] + if (entries.length === 0) return null + return ( +
+ 
+        {entries.map(([k, v]) => `${k}: ${String(v)}`).join('\n')}
+
+ {truncated && ( +

+ Whitelisted headers only.{' '} + + Truncated + +

+ )} +
+ ) +} + +function Section({ + title, + children, +}: { + title: string + children: React.ReactNode +}) { + return ( +
+

+ {title} +

+ {children} +
+ ) +} + +function KvList({ items }: { items: Array<[string, string | null | undefined]> }) { + return ( +
+ {items + .filter(([, v]) => v != null && v !== '') + .map(([k, v]) => ( + +
{k}
+
{v}
+ + ))} +
+ ) +} diff --git a/web/src/components/observe/rows/ObserveRow.tsx b/web/src/components/observe/rows/ObserveRow.tsx new file mode 100644 index 000000000..a8fc44dc3 --- /dev/null +++ b/web/src/components/observe/rows/ObserveRow.tsx @@ -0,0 +1,109 @@ +import { + AlertOctagon, + CircleDollarSign, + Network, + Workflow, +} from 'lucide-react' +import type { ObservabilityEvent } from '../types' +import { ObserveRowShell, SeverityBadge, StatusBadge } from './RowParts' + +/** + * Single entry point: switches on `event.type` to pick the right renderer. + * Rows render entirely from the row payload — no follow-up fetch. + */ +export function ObserveRow({ + event, + onClick, +}: { + event: ObservabilityEvent + onClick: () => void +}) { + switch (event.type) { + case 'request': + return ( + } + primary={ + <> + {event.method}{' '} + {event.path} + + } + secondary={event.host} + meta={ + <> + + {event.latency_ms != null && ( + + {event.latency_ms}ms + + )} + + } + onClick={onClick} + /> + ) + case 'span': + return ( + } + primary={event.operation} + secondary={event.service} + meta={ + event.duration_ms != null ? ( + + {event.duration_ms.toFixed(1)}ms + + ) : null + } + onClick={onClick} + /> + ) + case 'error': + return ( + } + primary={event.error_class} + secondary={event.message ?? undefined} + meta={} + onClick={onClick} + /> + ) + case 'revenue': + return ( + } + primary={event.event_type} + secondary={event.customer_ref ?? undefined} + meta={ + event.amount_minor != null ? ( + + {formatAmount(event.amount_minor, event.currency)} + + ) : null + } + onClick={onClick} + /> + ) + } +} + +function formatAmount( + minor: number, + currency: string | null | undefined, +): string { + const major = minor / 100 + if (!currency) return major.toFixed(2) + try { + return new Intl.NumberFormat(undefined, { + style: 'currency', + currency: currency.toUpperCase(), + }).format(major) + } catch { + return `${major.toFixed(2)} ${currency.toUpperCase()}` + } +} diff --git a/web/src/components/observe/rows/RowParts.tsx b/web/src/components/observe/rows/RowParts.tsx new file mode 100644 index 000000000..ef0cd3cb0 --- /dev/null +++ b/web/src/components/observe/rows/RowParts.tsx @@ -0,0 +1,95 @@ +import { Badge } from '@/components/ui/badge' +import { cn } from '@/lib/utils' +import { format, formatDistanceToNowStrict } from 'date-fns' + +/** One-line tabular row used by every event renderer. */ +export function ObserveRowShell({ + ts, + icon, + primary, + secondary, + meta, + onClick, +}: { + ts: string + icon: React.ReactNode + primary: React.ReactNode + secondary?: React.ReactNode + meta?: React.ReactNode + onClick?: () => void +}) { + return ( + + ) +} + +function Timestamp({ ts }: { ts: string }) { + const d = new Date(ts) + return ( + + ) +} + +export function StatusBadge({ status }: { status: number }) { + const variant: 'default' | 'destructive' | 'secondary' | 'outline' = + status >= 500 + ? 'destructive' + : status >= 400 + ? 'secondary' + : status >= 300 + ? 'outline' + : 'default' + return ( + + {status} + + ) +} + +export function SeverityBadge({ severity }: { severity: string }) { + const lower = severity.toLowerCase() + const variant: 'default' | 'destructive' | 'secondary' | 'outline' = + lower === 'error' || lower === 'fatal' + ? 'destructive' + : lower === 'warn' || lower === 'warning' + ? 'secondary' + : 'outline' + return ( + + {severity} + + ) +} diff --git a/web/src/components/observe/types.ts b/web/src/components/observe/types.ts new file mode 100644 index 000000000..6fa68b86b --- /dev/null +++ b/web/src/components/observe/types.ts @@ -0,0 +1,27 @@ +/** + * Re-exports for the observability page. The shapes live in the generated + * SDK; this module exists so call-sites import from one stable path. + * + * No `log` kind — runtime stdout/stderr lives on the dedicated Logs page, + * not on Observe. Volume + storage characteristics make logs unsuitable + * for the merged business-signal timeline. + */ + +export type { + ErrorRow, + EventKind, + EventsResponse, + ObservabilityEvent, + RequestRow, + RevenueRow, + SpanRow, +} from '@/api/client' + +export const ALL_KINDS = [ + 'request', + 'span', + 'error', + 'revenue', +] as const satisfies ReadonlyArray< + import('@/api/client').EventKind +> diff --git a/web/src/components/observe/useObserveQuery.ts b/web/src/components/observe/useObserveQuery.ts new file mode 100644 index 000000000..662212980 --- /dev/null +++ b/web/src/components/observe/useObserveQuery.ts @@ -0,0 +1,56 @@ +import { observabilityListEventsOptions } from '@/api/client/@tanstack/react-query.gen' +import { useQuery } from '@tanstack/react-query' +import type { EventKind } from './types' +import { ALL_KINDS } from './types' + +interface UseObserveQueryArgs { + projectId: number + kinds: EventKind[] + from?: Date + to?: Date + deploymentId?: number + environmentId?: number + search?: string + limit?: number +} + +/** + * Wraps the generated `observabilityListEventsOptions` so call-sites can + * pass typed JS values (Date, EventKind[]) instead of the wire format + * (ISO strings + comma-separated kinds). + */ +export function useObserveQuery(args: UseObserveQueryArgs) { + const { + projectId, + kinds, + from, + to, + deploymentId, + environmentId, + search, + limit, + } = args + + // Omit `kinds` when the user has every kind selected so the server + // returns its full default — this also keeps the URL state clean. + const kindsParam = + kinds.length === ALL_KINDS.length || kinds.length === 0 + ? undefined + : kinds.slice().sort().join(',') + + return useQuery({ + ...observabilityListEventsOptions({ + path: { project_id: projectId }, + query: { + kinds: kindsParam, + from: from?.toISOString(), + to: to?.toISOString(), + deployment_id: deploymentId, + environment_id: environmentId, + search: search || undefined, + limit, + }, + }), + refetchInterval: 15_000, + }) +} diff --git a/web/src/index.tsx b/web/src/index.tsx index dc7bb5f6d..1447d8716 100644 --- a/web/src/index.tsx +++ b/web/src/index.tsx @@ -1,14 +1,6 @@ import ReactDOM from 'react-dom/client' import App from './App' -const UI_PICKER_SRC = 'https://ui.sh/ui-picker.js' -if (!document.querySelector(`script[src="${UI_PICKER_SRC}"]`)) { - const script = document.createElement('script') - script.src = UI_PICKER_SRC - script.async = true - document.body.appendChild(script) -} - const rootEl = document.getElementById('root') if (rootEl) { const root = ReactDOM.createRoot(rootEl) diff --git a/web/src/pages/Observe.tsx b/web/src/pages/Observe.tsx new file mode 100644 index 000000000..95f45c5fb --- /dev/null +++ b/web/src/pages/Observe.tsx @@ -0,0 +1,559 @@ +import { ProjectResponse } from '@/api/client' +import { Skeleton } from '@/components/ui/skeleton' +import { Input } from '@/components/ui/input' +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select' +import { + type ObserveFilters, + type TimeRange, + TIME_RANGES, +} from '@/components/observe/ObserveFilterBar' +import { ObservePanel } from '@/components/observe/panels/ObservePanel' +import { useObserveQuery } from '@/components/observe/useObserveQuery' +import { + ALL_KINDS, + type EventKind, + type ObservabilityEvent, +} from '@/components/observe/types' +import { + AlertOctagon, + CircleDollarSign, + Inbox, + Network, + Search, + Workflow, +} from 'lucide-react' +import { useMemo, useState } from 'react' +import { useSearchParams } from 'react-router-dom' +import { cn } from '@/lib/utils' +import { format } from 'date-fns' + +interface ObserveProps { + project: ProjectResponse +} + +/** + * Kinds enabled on first visit. Traces are intentionally OFF by default — + * `otel_spans` is a TimescaleDB hypertable that can hold millions of rows + * per project, and pulling spans on every Observe page load would make + * the merged query and the rendered list unnecessarily expensive. The + * user can toggle Traces on via the cockpit card. + * + * Runtime logs are not part of Observe at all — they live on the + * dedicated Logs page. + */ +const DEFAULT_KINDS: readonly EventKind[] = ['request', 'error', 'revenue'] + +/** Order-insensitive kind-set equality used to decide whether to omit + * the `kinds` URL param when filters match the defaults. */ +function sameKindSet( + a: readonly EventKind[], + b: readonly EventKind[], +): boolean { + if (a.length !== b.length) return false + const set = new Set(a) + return b.every((k) => set.has(k)) +} + +/** + * Unified observability page — one list, one detail panel, one filter + * surface. Each row carries everything the panel needs so opening detail + * doesn't trigger a follow-up fetch in the common case. + * + * Layout is two-tier: + * 1. Cockpit header — 5 sparklines (one per kind) over the selected + * time range, each clickable to toggle that kind. Surfaces "where + * should I look first" before the eye even reaches the stream. + * 2. Console stream — terminal-style monospace rows with a colored + * gutter per kind. Designed for high-density scanning of mixed + * event types. + * + * Filter state lives in URL search params so the page is shareable: + * "here's the link, look at the 500s after the deploy at 14:02". + */ +export default function Observe({ project }: ObserveProps) { + const [searchParams, setSearchParams] = useSearchParams() + const [selectedEvent, setSelectedEvent] = + useState(null) + + const filters: ObserveFilters = useMemo(() => { + const kindsParam = searchParams.get('kinds') + const kinds = kindsParam + ? (kindsParam + .split(',') + .map((k) => k.trim()) + .filter((k): k is EventKind => + (ALL_KINDS as readonly string[]).includes(k), + ) as EventKind[]) + : ([...DEFAULT_KINDS] as EventKind[]) + + const timeRangeParam = searchParams.get('time_range') as TimeRange | null + const timeRange: TimeRange = + timeRangeParam && TIME_RANGES.some((r) => r.value === timeRangeParam) + ? timeRangeParam + : '24h' + + return { + kinds: kinds.length > 0 ? kinds : ([...DEFAULT_KINDS] as EventKind[]), + timeRange, + search: searchParams.get('search') ?? '', + environmentId: searchParams.get('environment_id') + ? Number(searchParams.get('environment_id')) + : null, + } + }, [searchParams]) + + const updateFilters = (next: ObserveFilters) => { + const params = new URLSearchParams(searchParams) + if (sameKindSet(next.kinds, DEFAULT_KINDS)) { + params.delete('kinds') + } else { + params.set('kinds', next.kinds.join(',')) + } + if (next.timeRange === '24h') { + params.delete('time_range') + } else { + params.set('time_range', next.timeRange) + } + if (next.search) { + params.set('search', next.search) + } else { + params.delete('search') + } + if (next.environmentId != null) { + params.set('environment_id', String(next.environmentId)) + } else { + params.delete('environment_id') + } + setSearchParams(params, { replace: true }) + } + + const fromDate = useMemo( + () => timeRangeToFromDate(filters.timeRange), + [filters.timeRange], + ) + + const query = useObserveQuery({ + projectId: project.id, + kinds: filters.kinds, + from: fromDate, + environmentId: filters.environmentId ?? undefined, + search: filters.search || undefined, + limit: 100, + }) + + const events = query.data?.events ?? [] + + const onRowClick = (event: ObservabilityEvent) => setSelectedEvent(event) + + return ( + <> +
+ + {query.isPending ? ( + + ) : query.isError ? ( + + ) : events.length === 0 ? ( + + ) : ( +
+ {events.map((event) => ( + onRowClick(event)} + /> + ))} +
+ )} +
+ + { + if (!open) setSelectedEvent(null) + }} + projectSlug={project.slug} + /> + + ) +} + +// ─── Shared helpers ──────────────────────────────────────────────────────── + +function eventId(event: ObservabilityEvent): string | number { + // Use the strongest stable identifier we have per kind so React's keys + // don't collide when two rows share a timestamp. + const e = event as unknown as Record + return ( + (e.id as string | number | undefined) ?? + (e.span_id as string | undefined) ?? + (e.error_group_id as string | undefined) ?? + `${event.type}-${event.ts}` + ) +} + +function timeRangeToFromDate(range: TimeRange): Date { + const now = new Date() + const map: Record = { + '15m': 15 * 60 * 1000, + '1h': 60 * 60 * 1000, + '24h': 24 * 60 * 60 * 1000, + '7d': 7 * 24 * 60 * 60 * 1000, + '30d': 30 * 24 * 60 * 60 * 1000, + } + return new Date(now.getTime() - map[range]) +} + +const KIND_META: Record< + EventKind, + { + label: string + icon: React.ComponentType<{ className?: string }> + accent: string // tailwind text color for the kind's accent + gutter: string // tailwind border color for the console gutter + } +> = { + request: { + label: 'Requests', + icon: Network, + accent: 'text-sky-500', + gutter: 'border-l-sky-500', + }, + span: { + label: 'Traces', + icon: Workflow, + accent: 'text-violet-500', + gutter: 'border-l-violet-500', + }, + error: { + label: 'Errors', + icon: AlertOctagon, + accent: 'text-rose-500', + gutter: 'border-l-rose-500', + }, + revenue: { + label: 'Revenue', + icon: CircleDollarSign, + accent: 'text-emerald-500', + gutter: 'border-l-emerald-500', + }, +} + +// ─── Cockpit header ──────────────────────────────────────────────────────── + +/** Top strip with one mini sparkline per kind. Clicking a card toggles its + * kind in the filter so the user can read the chart and act in the same + * motion. */ +function CockpitHeader({ + project, + filters, + onChange, + events, +}: { + project: ProjectResponse + filters: ObserveFilters + onChange: (next: ObserveFilters) => void + events: ObservabilityEvent[] +}) { + const buckets = useMemo(() => bucketByKind(events, 24), [events]) + + const toggle = (kind: EventKind) => { + const next = filters.kinds.includes(kind) + ? filters.kinds.filter((k) => k !== kind) + : [...filters.kinds, kind] + onChange({ + ...filters, + kinds: next.length === 0 ? [...DEFAULT_KINDS] : next, + }) + } + + return ( +
+
+

+ Observe +

+

+ {project.name} +

+
+ +
+ {ALL_KINDS.map((kind) => { + const meta = KIND_META[kind] + const series = buckets[kind] ?? [] + const total = series.reduce((sum, n) => sum + n, 0) + const active = filters.kinds.includes(kind) + const Icon = meta.icon + return ( + + ) + })} +
+ +
+
+ + + onChange({ ...filters, search: e.target.value }) + } + className="border-zinc-800 bg-zinc-900 pl-9 font-mono text-zinc-200 placeholder:text-zinc-600" + /> +
+ +
+
+ ) +} + +function Sparkline({ + values, + accent, +}: { + values: number[] + accent: string +}) { + const max = Math.max(1, ...values) + return ( +
+ {values.map((v, i) => ( +
+ ))} +
+ ) +} + +/** Group events into N evenly spaced time buckets per kind. Cheap O(n). */ +function bucketByKind( + events: ObservabilityEvent[], + bucketCount: number, +): Record { + const out: Record = { + request: new Array(bucketCount).fill(0), + span: new Array(bucketCount).fill(0), + error: new Array(bucketCount).fill(0), + revenue: new Array(bucketCount).fill(0), + } + if (events.length === 0) return out + + const tsList = events.map((e) => new Date(e.ts).getTime()) + const min = Math.min(...tsList) + const max = Math.max(...tsList) + const span = Math.max(1, max - min) + + for (const e of events) { + const ts = new Date(e.ts).getTime() + const idx = Math.min( + bucketCount - 1, + Math.floor(((ts - min) / span) * bucketCount), + ) + out[e.type][idx] += 1 + } + return out +} + +// ─── Console row ─────────────────────────────────────────────────────────── + +function ConsoleRow({ + event, + onClick, +}: { + event: ObservabilityEvent + onClick: () => void +}) { + const meta = KIND_META[event.type] + const ts = new Date(event.ts) + const { primary, suffix } = consoleSummary(event) + return ( + + ) +} + +/** One-line console summary per event type. Pure presentation. */ +function consoleSummary(event: ObservabilityEvent): { + primary: string + suffix: string | null +} { + switch (event.type) { + case 'request': + return { + primary: `${event.method} ${event.path}`, + suffix: + event.latency_ms != null + ? `${event.status} · ${event.latency_ms}ms` + : `${event.status}`, + } + case 'span': + return { + primary: `${event.service} ${event.operation}`, + suffix: + event.duration_ms != null + ? `${event.duration_ms.toFixed(1)}ms` + : null, + } + case 'error': + return { + primary: `${event.error_class}: ${event.message ?? ''}`.trim(), + suffix: null, + } + case 'revenue': + return { + primary: `${event.event_type}${ + event.customer_ref ? ` · ${event.customer_ref}` : '' + }`, + suffix: + event.amount_minor != null + ? formatMoney(event.amount_minor, event.currency) + : null, + } + } +} + +function formatMoney(minor: number, currency: string | null | undefined) { + const major = minor / 100 + if (!currency) return major.toFixed(2) + try { + return new Intl.NumberFormat(undefined, { + style: 'currency', + currency: currency.toUpperCase(), + }).format(major) + } catch { + return `${major.toFixed(2)} ${currency.toUpperCase()}` + } +} + +// ─── Status states ───────────────────────────────────────────────────────── + +function ListSkeleton() { + return ( +
+ {Array.from({ length: 12 }).map((_, i) => ( +
+ + + + +
+ ))} +
+ ) +} + +function EmptyState() { + return ( +
+ +

No events match the current filters.

+

Try widening the time range or toggling kinds.

+
+ ) +} + +function ErrorState({ message }: { message: string }) { + return ( +
+

+ Failed to load events +

+

{message}

+
+ ) +} diff --git a/web/src/pages/ProjectDetail.tsx b/web/src/pages/ProjectDetail.tsx index 0d9b4500d..ea85ce695 100644 --- a/web/src/pages/ProjectDetail.tsx +++ b/web/src/pages/ProjectDetail.tsx @@ -39,6 +39,7 @@ import { usePageTitle } from '@/hooks/usePageTitle' import { DeploymentDetails } from '@/pages/DeploymentDetails' import { ErrorEventDetail } from './ErrorEventDetail' import { ErrorGroupDetail } from './ErrorGroupDetail' +import Observe from './Observe' import RequestLogs from './RequestLogs' import Traces from './Traces' import { ProjectAgentActivity } from './AiGateway' @@ -374,6 +375,10 @@ export function ProjectDetail() { path="runtime" element={} /> + } + /> Date: Fri, 1 May 2026 23:50:19 +0200 Subject: [PATCH 03/12] chore(db): Fix migration --- .../temps-cli/src/commands/serve/console.rs | 2 +- .../src/services/error_ingestion_service.rs | 7 +- ...20260502_000001_add_observe_correlation.rs | 102 ++++++++++++++++-- crates/temps-observability/src/cursor.rs | 3 +- crates/temps-observability/src/error.rs | 8 +- crates/temps-observability/src/filters.rs | 11 +- .../src/handlers/events.rs | 17 ++- crates/temps-observability/src/service.rs | 18 ++-- .../tests/merge_integration.rs | 4 +- 9 files changed, 120 insertions(+), 52 deletions(-) diff --git a/crates/temps-cli/src/commands/serve/console.rs b/crates/temps-cli/src/commands/serve/console.rs index ecaa69eed..c485e18d1 100644 --- a/crates/temps-cli/src/commands/serve/console.rs +++ b/crates/temps-cli/src/commands/serve/console.rs @@ -50,12 +50,12 @@ use temps_monitoring::{ OutageDetectionService, }; use temps_notifications::NotificationsPlugin; +use temps_observability::ObservabilityPlugin; use temps_otel::plugin::OtelPlugin; use temps_projects::ProjectsPlugin; use temps_providers::ProvidersPlugin; use temps_proxy::ProxyPlugin; use temps_queue::QueuePlugin; -use temps_observability::ObservabilityPlugin; use temps_revenue::RevenuePlugin; use temps_sandbox::plugin::SandboxPlugin; use temps_screenshots::ScreenshotsPlugin; diff --git a/crates/temps-error-tracking/src/services/error_ingestion_service.rs b/crates/temps-error-tracking/src/services/error_ingestion_service.rs index 0841554fa..04c095a33 100644 --- a/crates/temps-error-tracking/src/services/error_ingestion_service.rs +++ b/crates/temps-error-tracking/src/services/error_ingestion_service.rs @@ -17,9 +17,7 @@ use super::types::{CreateErrorEventData, ErrorTrackingError}; fn extract_trace_id_from_data(data: &serde_json::Value) -> Option { fn validate(v: &serde_json::Value) -> Option { let s = v.as_str()?.trim().to_ascii_lowercase(); - if s.len() == 32 - && s.chars().all(|c| c.is_ascii_hexdigit()) - && s.chars().any(|c| c != '0') + if s.len() == 32 && s.chars().all(|c| c.is_ascii_hexdigit()) && s.chars().any(|c| c != '0') { Some(s) } else { @@ -681,8 +679,7 @@ mod trace_id_extraction_tests { let bad = json!({ "trace": { "trace_id": "deadbeef" } }); assert_eq!(extract_trace_id_from_data(&bad), None); - let nonhex = - json!({ "trace": { "trace_id": "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" } }); + let nonhex = json!({ "trace": { "trace_id": "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" } }); assert_eq!(extract_trace_id_from_data(&nonhex), None); } diff --git a/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs b/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs index c7ccc08e9..1d08b65b4 100644 --- a/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs +++ b/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs @@ -1,17 +1,54 @@ use sea_orm_migration::prelude::*; /// Add cross-source correlation columns so the unified Observe page can join -/// runtime logs / requests / spans / errors / revenue without follow-up -/// queries. All columns are nullable; old rows simply render without -/// correlation links. +/// requests / spans / errors / revenue without follow-up queries. All +/// columns are nullable; old rows simply render without correlation links. +/// +/// **TimescaleDB caveat**: `proxy_logs` is a hypertable with a 7-day +/// compression policy (see `m20260225_000001_add_proxy_logs_retention`). +/// Once chunks compress, `ALTER TABLE … ADD COLUMN` against the parent +/// fails with `chunk not found` because the per-chunk schema can't be +/// rewritten in place. We work around this by: +/// 1. Removing the compression policy (so the background worker stops +/// compressing new chunks while we work). +/// 2. Decompressing every existing chunk (cheap relative to keeping the +/// hypertable broken). +/// 3. Running the ALTER, which now succeeds across every chunk. +/// 4. Re-adding the same compression policy with `if_not_exists`. +/// +/// `revenue_events` is also a hypertable but has no compression policy +/// today, so a plain `ALTER` is safe. `error_events` is a regular table. #[derive(DeriveMigrationName)] pub struct Migration; #[async_trait::async_trait] impl MigrationTrait for Migration { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { - // proxy_logs: trace_id (from OTel context) + error_group_id (when the - // request produced a captured exception) + let db = manager.get_connection(); + + // ── proxy_logs (compressed hypertable) ────────────────────────── + // 1. Pause the compression policy. Idempotent: `if_exists` returns + // NULL when the policy isn't present (e.g. fresh installs that + // haven't run the retention migration yet). + db.execute_unprepared("SELECT remove_compression_policy('proxy_logs', if_exists => TRUE)") + .await?; + + // 2. Decompress every chunk so the next ALTER can rewrite per-chunk + // schemas. Skips the call entirely on installs that don't have + // the TimescaleDB extension or haven't compressed anything yet + // (the function returns 0 rows in that case — no error). + db.execute_unprepared( + "DO $$ + BEGIN + IF EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'timescaledb') THEN + PERFORM decompress_chunk(c, if_compressed => TRUE) + FROM show_chunks('proxy_logs') c; + END IF; + END$$", + ) + .await?; + + // 3. Now safe to ALTER. manager .alter_table( Table::alter() @@ -50,8 +87,20 @@ impl MigrationTrait for Migration { ) .await?; - // revenue_events: link the business event back to the deployment / - // environment / trace it belongs to (when the SDK provides headers). + // 4. Re-add the compression policy with the original 7-day window. + // `if_not_exists` makes this safe to re-run on installs that + // never had the policy (no-op). + db.execute_unprepared( + "DO $$ + BEGIN + IF EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'timescaledb') THEN + PERFORM add_compression_policy('proxy_logs', INTERVAL '7 days', if_not_exists => TRUE); + END IF; + END$$", + ) + .await?; + + // ── revenue_events (uncompressed hypertable) ──────────────────── manager .alter_table( Table::alter() @@ -64,7 +113,11 @@ impl MigrationTrait for Migration { .alter_table( Table::alter() .table(RevenueEvents::Table) - .add_column(ColumnDef::new(RevenueEvents::EnvironmentId).integer().null()) + .add_column( + ColumnDef::new(RevenueEvents::EnvironmentId) + .integer() + .null(), + ) .to_owned(), ) .await?; @@ -88,9 +141,10 @@ impl MigrationTrait for Migration { ) .await?; - // error_events: promote trace_id from the JSON `data.trace.trace_id` - // blob to a top-level indexed column. Cheap to maintain at write - // time; lets the merge query join by trace_id without a JSON probe. + // ── error_events (regular table) ──────────────────────────────── + // Promote `data.trace.trace_id` from JSONB to a top-level indexed + // column. Cheap to maintain at write time; lets the merge query + // join by trace_id without a JSON probe. manager .alter_table( Table::alter() @@ -115,6 +169,8 @@ impl MigrationTrait for Migration { } async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let db = manager.get_connection(); + manager .drop_index( Index::drop() @@ -153,6 +209,20 @@ impl MigrationTrait for Migration { ) .await?; + // Same compressed-hypertable dance for proxy_logs on the way down. + db.execute_unprepared("SELECT remove_compression_policy('proxy_logs', if_exists => TRUE)") + .await?; + db.execute_unprepared( + "DO $$ + BEGIN + IF EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'timescaledb') THEN + PERFORM decompress_chunk(c, if_compressed => TRUE) + FROM show_chunks('proxy_logs') c; + END IF; + END$$", + ) + .await?; + manager .drop_index( Index::drop() @@ -181,6 +251,16 @@ impl MigrationTrait for Migration { ) .await?; + db.execute_unprepared( + "DO $$ + BEGIN + IF EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'timescaledb') THEN + PERFORM add_compression_policy('proxy_logs', INTERVAL '7 days', if_not_exists => TRUE); + END IF; + END$$", + ) + .await?; + Ok(()) } } diff --git a/crates/temps-observability/src/cursor.rs b/crates/temps-observability/src/cursor.rs index 3282e9aab..8851fb0ae 100644 --- a/crates/temps-observability/src/cursor.rs +++ b/crates/temps-observability/src/cursor.rs @@ -72,8 +72,7 @@ mod tests { #[test] fn rejects_valid_base64_but_invalid_json() { - let token = - base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(b"not json"); + let token = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(b"not json"); let err = Cursor::decode(&token).unwrap_err(); assert!(matches!(err, ObservabilityError::InvalidCursor { .. })); } diff --git a/crates/temps-observability/src/error.rs b/crates/temps-observability/src/error.rs index f19b1e933..e3e0e585e 100644 --- a/crates/temps-observability/src/error.rs +++ b/crates/temps-observability/src/error.rs @@ -16,17 +16,13 @@ pub enum ObservabilityError { event_id: String, }, - #[error( - "Invalid kinds filter: '{value}'. Valid kinds: log, request, span, error, revenue" - )] + #[error("Invalid kinds filter: '{value}'. Valid kinds: log, request, span, error, revenue")] InvalidKindsFilter { value: String }, #[error("Invalid cursor: {reason}")] InvalidCursor { reason: String }, - #[error( - "Time range invalid: from={from} is after to={to}; the merge query needs from <= to" - )] + #[error("Time range invalid: from={from} is after to={to}; the merge query needs from <= to")] InvalidTimeRange { from: String, to: String }, #[error("Database error: {0}")] diff --git a/crates/temps-observability/src/filters.rs b/crates/temps-observability/src/filters.rs index 29c49aa26..db7d10ac0 100644 --- a/crates/temps-observability/src/filters.rs +++ b/crates/temps-observability/src/filters.rs @@ -82,11 +82,13 @@ pub fn clamp_limit(raw: Option) -> u64 { /// Implemented as a heap-free linear scan over the per-kind cursors since /// `EventKind::ALL.len()` is 5; the constant factor beats the heap setup /// for any realistic page size. -pub fn merge_desc_by_ts(streams: Vec>, limit: usize) -> Vec { +pub fn merge_desc_by_ts( + streams: Vec>, + limit: usize, +) -> Vec { let mut cursors: Vec> = streams.into_iter().map(|s| s.into_iter()).collect(); - let mut heads: Vec> = - cursors.iter_mut().map(|c| c.next()).collect(); + let mut heads: Vec> = cursors.iter_mut().map(|c| c.next()).collect(); let mut out = Vec::with_capacity(limit); while out.len() < limit { @@ -293,8 +295,7 @@ mod tests { #[test] fn merge_handles_single_empty_stream() { - let merged: Vec = - merge_desc_by_ts(vec![vec![]], 10); + let merged: Vec = merge_desc_by_ts(vec![vec![]], 10); assert!(merged.is_empty()); } diff --git a/crates/temps-observability/src/handlers/events.rs b/crates/temps-observability/src/handlers/events.rs index 48bd03d44..a68431cc1 100644 --- a/crates/temps-observability/src/handlers/events.rs +++ b/crates/temps-observability/src/handlers/events.rs @@ -149,11 +149,9 @@ impl From for Problem { .with_title("Project Not Found") .with_detail(error.to_string()) } - ObservabilityError::EventNotFound { .. } => { - problemdetails::new(StatusCode::NOT_FOUND) - .with_title("Event Not Found") - .with_detail(error.to_string()) - } + ObservabilityError::EventNotFound { .. } => problemdetails::new(StatusCode::NOT_FOUND) + .with_title("Event Not Found") + .with_detail(error.to_string()), ObservabilityError::InvalidKindsFilter { .. } | ObservabilityError::InvalidCursor { .. } | ObservabilityError::InvalidTimeRange { .. } => { @@ -199,7 +197,10 @@ pub async fn observability_full_event( ) -> Result { permission_guard!(auth, LogsRead); - let event = state.service.fetch_full(project_id, kind, &event_id).await?; + let event = state + .service + .fetch_full(project_id, kind, &event_id) + .await?; Ok((StatusCode::OK, Json(event))) } @@ -233,9 +234,7 @@ mod tests { ObservabilityError::InvalidKindsFilter { value: "bad".into(), }, - ObservabilityError::InvalidCursor { - reason: "x".into(), - }, + ObservabilityError::InvalidCursor { reason: "x".into() }, ObservabilityError::InvalidTimeRange { from: "a".into(), to: "b".into(), diff --git a/crates/temps-observability/src/service.rs b/crates/temps-observability/src/service.rs index 82a37d5aa..d5a0c6b0e 100644 --- a/crates/temps-observability/src/service.rs +++ b/crates/temps-observability/src/service.rs @@ -211,14 +211,13 @@ impl ObservabilityService { Ok(FullEvent::Revenue(rev)) } EventKind::Span => { - let row = self - .find_span_by_id(project_id, id) - .await? - .ok_or_else(|| ObservabilityError::EventNotFound { + let row = self.find_span_by_id(project_id, id).await?.ok_or_else(|| { + ObservabilityError::EventNotFound { project_id, kind: "span".into(), event_id: id.into(), - })?; + } + })?; let ObservabilityEvent::Span(full) = map_span(row) else { unreachable!("map_span returns Span variant"); }; @@ -231,8 +230,8 @@ impl ObservabilityService { &self, filters: &EventFilters, ) -> Result, ObservabilityError> { - let mut q = proxy_logs::Entity::find() - .filter(proxy_logs::Column::ProjectId.eq(filters.project_id)); + let mut q = + proxy_logs::Entity::find().filter(proxy_logs::Column::ProjectId.eq(filters.project_id)); if let Some(from) = filters.from { q = q.filter(proxy_logs::Column::Timestamp.gte(from)); } @@ -352,10 +351,7 @@ impl ObservabilityService { values.push(format!("%{}%", s).into()); next_param += 1; } - sql.push_str(&format!( - " ORDER BY start_time DESC LIMIT ${}", - next_param - )); + sql.push_str(&format!(" ORDER BY start_time DESC LIMIT ${}", next_param)); values.push(sea_orm::Value::BigInt(Some(filters.limit as i64))); let stmt = Statement::from_sql_and_values(DatabaseBackend::Postgres, &sql, values); diff --git a/crates/temps-observability/tests/merge_integration.rs b/crates/temps-observability/tests/merge_integration.rs index f1aed2a6f..2e299a0bc 100644 --- a/crates/temps-observability/tests/merge_integration.rs +++ b/crates/temps-observability/tests/merge_integration.rs @@ -8,11 +8,11 @@ use std::sync::Arc; use chrono::{Duration, TimeZone, Utc}; use sea_orm::{ActiveModelTrait, DatabaseConnection, Set}; -use temps_database::test_utils::TestDatabase; use sea_orm::{ConnectionTrait, DatabaseBackend, Statement}; +use temps_database::test_utils::TestDatabase; use temps_entities::{error_events, error_groups, projects, proxy_logs, revenue_events}; use temps_observability::{ - service::FullEvent, EventFilters, EventKind, ObservabilityEvent, ObservabilityError, + service::FullEvent, EventFilters, EventKind, ObservabilityError, ObservabilityEvent, ObservabilityService, }; use uuid::Uuid; From 40c63e77aa275ad521fb2112082899af16d920dc Mon Sep 17 00:00:00 2001 From: David Viejo Date: Sat, 2 May 2026 09:28:26 +0200 Subject: [PATCH 04/12] fix(migrations): bulletproof Observe correlation migration + regression tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first version of `m20260502_000001_add_observe_correlation` ran plain Sea-ORM `ALTER TABLE … ADD COLUMN` against `proxy_logs`. Prod runs `proxy_logs` as a TimescaleDB hypertable with a 7-day compression policy from `m20260225_000001_add_proxy_logs_retention`, and once chunks compress the ALTER fails with `chunk not found`, leaves the schema half-applied, and Sea-ORM has no clean way to retry. Local dev never caught it because nothing in our test rigs had compressed chunks (rows weren't 7 days old yet). This rewrite makes the migration safe in every prior state: * Fresh installs (no extension installed) * Production-style installs (compressed hypertable + active policy) * Partially-failed prior runs (any subset of columns/indexes) * Already-fully-applied installs (re-runs are no-ops) * Installs where `proxy_logs` was never converted to a hypertable Implementation: one PL/pgSQL `DO $$ … $$` block built entirely from idempotent operations — `ALTER TABLE … ADD COLUMN IF NOT EXISTS`, `CREATE INDEX IF NOT EXISTS`, `remove_compression_policy(…, if_exists => TRUE)`, `decompress_chunk(c, if_compressed => TRUE)`, `add_compression_policy(…, if_not_exists => TRUE)`. Guarded by an extension probe (`pg_extension WHERE extname = 'timescaledb'`) and a hypertable probe (`timescaledb_information.hypertables`) so it degrades cleanly on installs that lack either. Operators upgrading from a binary that hit the original failure can now boot the new binary directly — no manual SQL recovery needed. Adds two regression tests in `tests/migration_tests.rs`: * `test_observe_correlation_migration_handles_compressed_proxy_logs` seeds 30 rows older than 7 days, force-compresses every eligible chunk via `compress_chunk()`, then runs `Migrator::up()` and asserts (a) every column + index exists, (b) every chunk is now decompressed (catches regressions that skip the decompress step), (c) the compression policy is restored, and (d) all 30 rows survived. Verified to FAIL when the decompress block is removed. * `test_observe_correlation_migration_is_idempotent` runs the migrator, deletes the migration's row from `seaql_migrations`, then runs the migrator again — must succeed without error. Both tests use `testcontainers` and complete in ~6s. --- ...20260502_000001_add_observe_correlation.rs | 395 ++++++------------ .../temps-migrations/tests/migration_tests.rs | 266 ++++++++++++ 2 files changed, 402 insertions(+), 259 deletions(-) diff --git a/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs b/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs index 1d08b65b4..f1ddf0e46 100644 --- a/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs +++ b/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs @@ -4,20 +4,32 @@ use sea_orm_migration::prelude::*; /// requests / spans / errors / revenue without follow-up queries. All /// columns are nullable; old rows simply render without correlation links. /// -/// **TimescaleDB caveat**: `proxy_logs` is a hypertable with a 7-day -/// compression policy (see `m20260225_000001_add_proxy_logs_retention`). -/// Once chunks compress, `ALTER TABLE … ADD COLUMN` against the parent -/// fails with `chunk not found` because the per-chunk schema can't be -/// rewritten in place. We work around this by: -/// 1. Removing the compression policy (so the background worker stops -/// compressing new chunks while we work). -/// 2. Decompressing every existing chunk (cheap relative to keeping the -/// hypertable broken). -/// 3. Running the ALTER, which now succeeds across every chunk. -/// 4. Re-adding the same compression policy with `if_not_exists`. +/// **Designed to be safely re-runnable on any prior state**, including: +/// * Fresh installs (no rows exist yet, no extension may be installed) +/// * Partially-applied prior runs (some columns present, others not — the +/// v1 of this migration would fail on the second `ALTER` against a +/// compressed `proxy_logs` chunk and leave the row in the historical +/// `seaql_migrations` table while the schema was half-done) +/// * Already-fully-applied installs (re-running is a no-op) +/// * Installs without the TimescaleDB extension (the proxy_logs +/// compression dance is skipped) +/// * Installs where `proxy_logs` was not converted to a hypertable +/// (`show_chunks` would error — guarded by an extension check) /// -/// `revenue_events` is also a hypertable but has no compression policy -/// today, so a plain `ALTER` is safe. `error_events` is a regular table. +/// Strategy: every step uses `IF NOT EXISTS` / `IF EXISTS` and is run via +/// raw SQL inside a single `DO $$ … $$` block so PostgreSQL handles the +/// procedural control flow. Sea-ORM's `alter_table` builder doesn't emit +/// `IF NOT EXISTS` for `ADD COLUMN`, which is why we drop down to raw SQL +/// here. The `ALTER TABLE … ADD COLUMN IF NOT EXISTS …` form has been +/// supported since Postgres 9.6 and works on TimescaleDB hypertables. +/// +/// **TimescaleDB caveat (the original failure mode)**: `proxy_logs` is a +/// compressed hypertable (7-day policy from +/// `m20260225_000001_add_proxy_logs_retention`). Once chunks compress, +/// `ALTER TABLE … ADD COLUMN` against the parent fails with +/// `chunk not found` because per-chunk schemas can't be rewritten in +/// place. We pause the policy, decompress all chunks, then re-add the +/// policy with `if_not_exists`. #[derive(DeriveMigrationName)] pub struct Migration; @@ -26,266 +38,131 @@ impl MigrationTrait for Migration { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { let db = manager.get_connection(); - // ── proxy_logs (compressed hypertable) ────────────────────────── - // 1. Pause the compression policy. Idempotent: `if_exists` returns - // NULL when the policy isn't present (e.g. fresh installs that - // haven't run the retention migration yet). - db.execute_unprepared("SELECT remove_compression_policy('proxy_logs', if_exists => TRUE)") - .await?; - - // 2. Decompress every chunk so the next ALTER can rewrite per-chunk - // schemas. Skips the call entirely on installs that don't have - // the TimescaleDB extension or haven't compressed anything yet - // (the function returns 0 rows in that case — no error). + // Single PL/pgSQL block so the whole thing is one statement and + // we get atomic control flow (the extension check guards every + // TimescaleDB call). Each ALTER uses `IF NOT EXISTS`, so a partial + // prior run completes cleanly without us tracking which columns + // already exist. db.execute_unprepared( - "DO $$ - BEGIN - IF EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'timescaledb') THEN - PERFORM decompress_chunk(c, if_compressed => TRUE) - FROM show_chunks('proxy_logs') c; - END IF; - END$$", + r#" +DO $$ +DECLARE + has_timescaledb boolean := EXISTS ( + SELECT 1 FROM pg_extension WHERE extname = 'timescaledb' + ); + proxy_logs_is_hypertable boolean := false; +BEGIN + -- ── proxy_logs (potentially compressed hypertable) ────────────── + IF has_timescaledb THEN + SELECT EXISTS ( + SELECT 1 FROM timescaledb_information.hypertables + WHERE hypertable_name = 'proxy_logs' + ) INTO proxy_logs_is_hypertable; + + IF proxy_logs_is_hypertable THEN + -- Pause the compression policy. `if_exists` returns NULL when + -- the policy isn't present (fresh installs that haven't run + -- the retention migration yet, or installs that have already + -- removed it via a partial prior run). + PERFORM remove_compression_policy('proxy_logs', if_exists => TRUE); + + -- Decompress every chunk so per-chunk ALTERs succeed. + -- `if_compressed => TRUE` makes this a no-op for already- + -- decompressed chunks. + PERFORM decompress_chunk(c, if_compressed => TRUE) + FROM show_chunks('proxy_logs') c; + END IF; + END IF; + + -- Idempotent column additions. `IF NOT EXISTS` covers the partial- + -- prior-run case where v1 of this migration added `trace_id` but + -- failed on `error_group_id` (or vice versa). + ALTER TABLE proxy_logs ADD COLUMN IF NOT EXISTS trace_id text; + ALTER TABLE proxy_logs ADD COLUMN IF NOT EXISTS error_group_id integer; + + ALTER TABLE revenue_events ADD COLUMN IF NOT EXISTS deployment_id integer; + ALTER TABLE revenue_events ADD COLUMN IF NOT EXISTS environment_id integer; + ALTER TABLE revenue_events ADD COLUMN IF NOT EXISTS trace_id text; + + ALTER TABLE error_events ADD COLUMN IF NOT EXISTS trace_id_indexed text; + + -- Indexes (also idempotent). + CREATE INDEX IF NOT EXISTS idx_proxy_logs_project_trace + ON proxy_logs (project_id, trace_id); + CREATE INDEX IF NOT EXISTS idx_proxy_logs_error_group + ON proxy_logs (error_group_id); + CREATE INDEX IF NOT EXISTS idx_revenue_events_project_occurred + ON revenue_events (project_id, occurred_at); + CREATE INDEX IF NOT EXISTS idx_error_events_project_trace + ON error_events (project_id, trace_id_indexed); + + -- Restore the proxy_logs compression policy with the original 7-day + -- window. `if_not_exists` makes this safe to re-run on installs that + -- never had the policy (no-op). + IF has_timescaledb AND proxy_logs_is_hypertable THEN + PERFORM add_compression_policy( + 'proxy_logs', INTERVAL '7 days', if_not_exists => TRUE + ); + END IF; +END +$$; +"#, ) .await?; - // 3. Now safe to ALTER. - manager - .alter_table( - Table::alter() - .table(ProxyLogs::Table) - .add_column(ColumnDef::new(ProxyLogs::TraceId).text().null()) - .to_owned(), - ) - .await?; - manager - .alter_table( - Table::alter() - .table(ProxyLogs::Table) - .add_column(ColumnDef::new(ProxyLogs::ErrorGroupId).integer().null()) - .to_owned(), - ) - .await?; - manager - .create_index( - Index::create() - .if_not_exists() - .name("idx_proxy_logs_project_trace") - .table(ProxyLogs::Table) - .col(ProxyLogs::ProjectId) - .col(ProxyLogs::TraceId) - .to_owned(), - ) - .await?; - manager - .create_index( - Index::create() - .if_not_exists() - .name("idx_proxy_logs_error_group") - .table(ProxyLogs::Table) - .col(ProxyLogs::ErrorGroupId) - .to_owned(), - ) - .await?; - - // 4. Re-add the compression policy with the original 7-day window. - // `if_not_exists` makes this safe to re-run on installs that - // never had the policy (no-op). - db.execute_unprepared( - "DO $$ - BEGIN - IF EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'timescaledb') THEN - PERFORM add_compression_policy('proxy_logs', INTERVAL '7 days', if_not_exists => TRUE); - END IF; - END$$", - ) - .await?; - - // ── revenue_events (uncompressed hypertable) ──────────────────── - manager - .alter_table( - Table::alter() - .table(RevenueEvents::Table) - .add_column(ColumnDef::new(RevenueEvents::DeploymentId).integer().null()) - .to_owned(), - ) - .await?; - manager - .alter_table( - Table::alter() - .table(RevenueEvents::Table) - .add_column( - ColumnDef::new(RevenueEvents::EnvironmentId) - .integer() - .null(), - ) - .to_owned(), - ) - .await?; - manager - .alter_table( - Table::alter() - .table(RevenueEvents::Table) - .add_column(ColumnDef::new(RevenueEvents::TraceId).text().null()) - .to_owned(), - ) - .await?; - manager - .create_index( - Index::create() - .if_not_exists() - .name("idx_revenue_events_project_occurred") - .table(RevenueEvents::Table) - .col(RevenueEvents::ProjectId) - .col(RevenueEvents::OccurredAt) - .to_owned(), - ) - .await?; - - // ── error_events (regular table) ──────────────────────────────── - // Promote `data.trace.trace_id` from JSONB to a top-level indexed - // column. Cheap to maintain at write time; lets the merge query - // join by trace_id without a JSON probe. - manager - .alter_table( - Table::alter() - .table(ErrorEvents::Table) - .add_column(ColumnDef::new(ErrorEvents::TraceIdIndexed).text().null()) - .to_owned(), - ) - .await?; - manager - .create_index( - Index::create() - .if_not_exists() - .name("idx_error_events_project_trace") - .table(ErrorEvents::Table) - .col(ErrorEvents::ProjectId) - .col(ErrorEvents::TraceIdIndexed) - .to_owned(), - ) - .await?; - Ok(()) } async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { let db = manager.get_connection(); - manager - .drop_index( - Index::drop() - .if_exists() - .name("idx_error_events_project_trace") - .table(ErrorEvents::Table) - .to_owned(), - ) - .await?; - manager - .alter_table( - Table::alter() - .table(ErrorEvents::Table) - .drop_column(ErrorEvents::TraceIdIndexed) - .to_owned(), - ) - .await?; - - manager - .drop_index( - Index::drop() - .if_exists() - .name("idx_revenue_events_project_occurred") - .table(RevenueEvents::Table) - .to_owned(), - ) - .await?; - manager - .alter_table( - Table::alter() - .table(RevenueEvents::Table) - .drop_column(RevenueEvents::TraceId) - .drop_column(RevenueEvents::EnvironmentId) - .drop_column(RevenueEvents::DeploymentId) - .to_owned(), - ) - .await?; - - // Same compressed-hypertable dance for proxy_logs on the way down. - db.execute_unprepared("SELECT remove_compression_policy('proxy_logs', if_exists => TRUE)") - .await?; + // Mirror image: pause compression, drop indexes/columns, restore. db.execute_unprepared( - "DO $$ - BEGIN - IF EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'timescaledb') THEN - PERFORM decompress_chunk(c, if_compressed => TRUE) - FROM show_chunks('proxy_logs') c; - END IF; - END$$", - ) - .await?; - - manager - .drop_index( - Index::drop() - .if_exists() - .name("idx_proxy_logs_error_group") - .table(ProxyLogs::Table) - .to_owned(), - ) - .await?; - manager - .drop_index( - Index::drop() - .if_exists() - .name("idx_proxy_logs_project_trace") - .table(ProxyLogs::Table) - .to_owned(), - ) - .await?; - manager - .alter_table( - Table::alter() - .table(ProxyLogs::Table) - .drop_column(ProxyLogs::ErrorGroupId) - .drop_column(ProxyLogs::TraceId) - .to_owned(), - ) - .await?; - - db.execute_unprepared( - "DO $$ - BEGIN - IF EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'timescaledb') THEN - PERFORM add_compression_policy('proxy_logs', INTERVAL '7 days', if_not_exists => TRUE); - END IF; - END$$", + r#" +DO $$ +DECLARE + has_timescaledb boolean := EXISTS ( + SELECT 1 FROM pg_extension WHERE extname = 'timescaledb' + ); + proxy_logs_is_hypertable boolean := false; +BEGIN + IF has_timescaledb THEN + SELECT EXISTS ( + SELECT 1 FROM timescaledb_information.hypertables + WHERE hypertable_name = 'proxy_logs' + ) INTO proxy_logs_is_hypertable; + + IF proxy_logs_is_hypertable THEN + PERFORM remove_compression_policy('proxy_logs', if_exists => TRUE); + PERFORM decompress_chunk(c, if_compressed => TRUE) + FROM show_chunks('proxy_logs') c; + END IF; + END IF; + + DROP INDEX IF EXISTS idx_error_events_project_trace; + ALTER TABLE error_events DROP COLUMN IF EXISTS trace_id_indexed; + + DROP INDEX IF EXISTS idx_revenue_events_project_occurred; + ALTER TABLE revenue_events DROP COLUMN IF EXISTS trace_id; + ALTER TABLE revenue_events DROP COLUMN IF EXISTS environment_id; + ALTER TABLE revenue_events DROP COLUMN IF EXISTS deployment_id; + + DROP INDEX IF EXISTS idx_proxy_logs_error_group; + DROP INDEX IF EXISTS idx_proxy_logs_project_trace; + ALTER TABLE proxy_logs DROP COLUMN IF EXISTS error_group_id; + ALTER TABLE proxy_logs DROP COLUMN IF EXISTS trace_id; + + IF has_timescaledb AND proxy_logs_is_hypertable THEN + PERFORM add_compression_policy( + 'proxy_logs', INTERVAL '7 days', if_not_exists => TRUE + ); + END IF; +END +$$; +"#, ) .await?; Ok(()) } } - -#[derive(DeriveIden)] -enum ProxyLogs { - Table, - ProjectId, - TraceId, - ErrorGroupId, -} - -#[derive(DeriveIden)] -enum RevenueEvents { - Table, - ProjectId, - OccurredAt, - DeploymentId, - EnvironmentId, - TraceId, -} - -#[derive(DeriveIden)] -enum ErrorEvents { - Table, - ProjectId, - TraceIdIndexed, -} diff --git a/crates/temps-migrations/tests/migration_tests.rs b/crates/temps-migrations/tests/migration_tests.rs index 6d0435a5c..d2268d1b0 100644 --- a/crates/temps-migrations/tests/migration_tests.rs +++ b/crates/temps-migrations/tests/migration_tests.rs @@ -951,3 +951,269 @@ async fn connect_with_retries(db_url: &str) -> anyhow::Result anyhow::Result<()> { + if std::env::var("TEMPS_TEST_DATABASE_URL").is_ok() { + println!( + "⏭️ Skipping test_observe_correlation_migration_handles_compressed_proxy_logs: \ + external database in use" + ); + return Ok(()); + } + + let container = GenericImage::new("timescale/timescaledb-ha", "pg18") + .with_env_var("POSTGRES_DB", "postgres") + .with_env_var("POSTGRES_USER", "postgres") + .with_env_var("POSTGRES_PASSWORD", "postgres") + .with_env_var("POSTGRES_HOST_AUTH_METHOD", "trust") + .start() + .await + .expect("Failed to start TimescaleDB container"); + let port = container.get_host_port_ipv4(5432).await?; + let db_url = format!("postgresql://postgres:postgres@localhost:{}/postgres", port); + tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + let db = connect_with_retries(&db_url).await?; + + // ── 1. Apply every migration EXCEPT our target so we land on the same + // schema state prod was in just before the v1 migration ran. ────── + let target = "m20260502_000001_add_observe_correlation"; + let pre_target_count = Migrator::migrations() + .iter() + .position(|m| m.name() == target) + .unwrap_or_else(|| panic!("migration {} not found in Migrator", target)); + Migrator::up(&db, Some(pre_target_count as u32)).await?; + + // Sanity: target migration should NOT be applied yet. + let pre_state = db + .query_one(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + format!( + "SELECT EXISTS (SELECT 1 FROM seaql_migrations WHERE version = '{}') AS applied", + target + ), + )) + .await? + .expect("query returns one row"); + let applied: bool = pre_state.try_get("", "applied")?; + assert!( + !applied, + "target migration {} should not be applied yet — adjust pre-target count", + target + ); + + // ── 2. Backfill proxy_logs with rows older than the 7-day compression + // window so the resulting chunks are eligible for compression. + // The schema needs to match what `m20250101_000001_initial_schema` + // defined; we only set the columns required by the NOT NULL + // constraints and let the rest default. ────────────────────────── + db.execute_unprepared( + "INSERT INTO proxy_logs ( \ + timestamp, method, path, host, status_code, request_source, \ + is_system_request, routing_status, request_id, created_date \ + ) \ + SELECT \ + now() - INTERVAL '10 days' - (s * INTERVAL '1 minute'), \ + 'GET', '/api/old/' || s, 'example.test', 200, 'proxy', \ + false, 'routed', 'req-' || s, \ + (now() - INTERVAL '10 days' - (s * INTERVAL '1 minute'))::date \ + FROM generate_series(1, 30) s", + ) + .await?; + + // ── 3. Force compression on every chunk that's eligible. This mimics + // what TimescaleDB's background compression worker does on prod + // when chunks age past the 7-day window. ────────────────────────── + db.execute_unprepared( + "SELECT compress_chunk(c, if_not_compressed => TRUE) \ + FROM show_chunks('proxy_logs', older_than => now() - INTERVAL '7 days') c", + ) + .await?; + + let pre = db + .query_one(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + "SELECT count(*) FILTER (WHERE is_compressed) AS compressed, count(*) AS total \ + FROM timescaledb_information.chunks WHERE hypertable_name = 'proxy_logs'" + .to_string(), + )) + .await? + .expect("chunk count"); + let compressed_before: i64 = pre.try_get("", "compressed")?; + let total_before: i64 = pre.try_get("", "total")?; + assert!( + compressed_before > 0, + "test setup must produce at least one compressed chunk \ + (got {} compressed / {} total) — \ + without that the regression isn't reproducible", + compressed_before, + total_before + ); + + // ── 4. Apply the target migration. With v1 this would have errored on + // the second ALTER; with the current code it must succeed. ─────── + Migrator::up(&db, None).await?; + + // ── 5. Verify the schema landed correctly. ───────────────────────────── + for (table, col) in [ + ("proxy_logs", "trace_id"), + ("proxy_logs", "error_group_id"), + ("revenue_events", "deployment_id"), + ("revenue_events", "environment_id"), + ("revenue_events", "trace_id"), + ("error_events", "trace_id_indexed"), + ] { + let row = db + .query_one(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + format!( + "SELECT EXISTS (SELECT 1 FROM information_schema.columns \ + WHERE table_name = '{}' AND column_name = '{}') AS present", + table, col + ), + )) + .await? + .expect("col query returns one row"); + let present: bool = row.try_get("", "present")?; + assert!(present, "{}.{} must exist after migration", table, col); + } + + for index in [ + "idx_proxy_logs_project_trace", + "idx_proxy_logs_error_group", + "idx_revenue_events_project_occurred", + "idx_error_events_project_trace", + ] { + let row = db + .query_one(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + format!( + "SELECT EXISTS (SELECT 1 FROM pg_indexes WHERE indexname = '{}') AS present", + index + ), + )) + .await? + .expect("index query returns one row"); + let present: bool = row.try_get("", "present")?; + assert!(present, "index {} must exist after migration", index); + } + + // ── 6. The migration must have decompressed every chunk (not just + // relied on TimescaleDB's lenient mode for ALTER on compressed + // chunks). Older Timescale versions in prod don't support that + // lenient path; this assertion pins the contract that the + // migration is doing the explicit decompress dance. ────────────── + let post = db + .query_one(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + "SELECT count(*) FILTER (WHERE is_compressed) AS compressed \ + FROM timescaledb_information.chunks WHERE hypertable_name = 'proxy_logs'" + .to_string(), + )) + .await? + .expect("post chunk count"); + let compressed_after: i64 = post.try_get("", "compressed")?; + assert_eq!( + compressed_after, 0, + "migration must decompress every chunk before ALTER (got {} still compressed)", + compressed_after + ); + + // ── 7. Compression policy must be restored. ──────────────────────────── + let policy_row = db + .query_one(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + "SELECT count(*) AS n FROM timescaledb_information.jobs \ + WHERE hypertable_name = 'proxy_logs' AND application_name LIKE 'Columnstore%'" + .to_string(), + )) + .await? + .expect("policy count"); + let policy_count: i64 = policy_row.try_get("", "n")?; + assert_eq!( + policy_count, 1, + "compression policy must be re-added after migration (got {})", + policy_count + ); + + // ── 8. Data must be intact. ──────────────────────────────────────────── + let row = db + .query_one(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + "SELECT count(*) AS n FROM proxy_logs".to_string(), + )) + .await? + .expect("row count"); + let row_count: i64 = row.try_get("", "n")?; + assert_eq!( + row_count, 30, + "all 30 backfilled rows must survive the decompress-alter-recompress cycle" + ); + + println!( + "✅ observe correlation migration succeeded with {} compressed chunks before", + compressed_before + ); + Ok(()) +} + +/// Idempotency guard: re-running the observe correlation migration on an +/// already-migrated DB must succeed silently. Catches regressions where +/// someone replaces an `IF NOT EXISTS` with a plain ALTER. +#[tokio::test] +async fn test_observe_correlation_migration_is_idempotent() -> anyhow::Result<()> { + if std::env::var("TEMPS_TEST_DATABASE_URL").is_ok() { + println!( + "⏭️ Skipping test_observe_correlation_migration_is_idempotent: \ + external database in use" + ); + return Ok(()); + } + + let container = GenericImage::new("timescale/timescaledb-ha", "pg18") + .with_env_var("POSTGRES_DB", "postgres") + .with_env_var("POSTGRES_USER", "postgres") + .with_env_var("POSTGRES_PASSWORD", "postgres") + .with_env_var("POSTGRES_HOST_AUTH_METHOD", "trust") + .start() + .await + .expect("Failed to start TimescaleDB container"); + let port = container.get_host_port_ipv4(5432).await?; + let db_url = format!("postgresql://postgres:postgres@localhost:{}/postgres", port); + tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + let db = connect_with_retries(&db_url).await?; + + Migrator::up(&db, None).await?; + + // Strip the recorded migration row so Sea-ORM thinks it needs to run + // again; the migration body itself must still be a no-op. + db.execute_unprepared( + "DELETE FROM seaql_migrations WHERE version = 'm20260502_000001_add_observe_correlation'", + ) + .await?; + + // Should succeed without error — every step uses IF NOT EXISTS / if_exists. + Migrator::up(&db, None).await?; + + println!("✅ observe correlation migration is idempotent"); + Ok(()) +} From eb7b21f979770eda03c1c91556f6a343eb24201d Mon Sep 17 00:00:00 2001 From: David Viejo Date: Sat, 2 May 2026 09:53:29 +0200 Subject: [PATCH 05/12] fix(database): self-heal seaql_migrations on startup before Migrator::up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operators upgrading from earlier builds can hit Sea-ORM's load-phase panic: Migration file of version 'X' is missing, this migration has been applied but its file is missing This happens whenever a release squashes or removes a migration whose row is still recorded in `seaql_migrations` from a prior install — the loader scans the table, can't find a matching file, and refuses to boot. The squash migration `m20260421_000001_squash_apr_post_v006` explicitly notes operators must hand-INSERT its row, but that doesn't address the orphaned predecessor rows. Adds `cleanup_orphaned_migrations(&db)` and calls it from `establish_connection()` immediately before `Migrator::up()`. The cleanup: * Skips entirely on a fresh install (no `seaql_migrations` table yet — Sea-ORM creates it during `up()`) * Builds the canonical version list from `Migrator::migrations()` * Counts orphaned rows; when zero, returns silently to avoid log noise on every startup * Logs the orphan list at WARN level before deletion so any surprise is visible in operator logs * Deletes the orphan rows in a single parameterized DELETE — safe because Sea-ORM only uses these rows to skip already-applied migrations, and the actual schema state lives independently Two integration tests in `tests/orphan_migration_cleanup.rs`: * `cleanup_strips_orphans_so_migrator_can_load` reproduces the prod failure: bring schema to current, INSERT four orphan rows, confirm `Migrator::up()` now errors, run cleanup, confirm `Migrator::up()` succeeds * `cleanup_is_noop_on_clean_db` exercises both the missing-table and clean-table paths to guarantee no false positives Both run against a fresh `timescale/timescaledb-ha:pg18` testcontainer. --- crates/temps-database/src/connection.rs | 156 ++++++++++++++++++ crates/temps-database/src/lib.rs | 2 +- .../tests/orphan_migration_cleanup.rs | 146 ++++++++++++++++ 3 files changed, 303 insertions(+), 1 deletion(-) create mode 100644 crates/temps-database/tests/orphan_migration_cleanup.rs diff --git a/crates/temps-database/src/connection.rs b/crates/temps-database/src/connection.rs index 904cde1ba..72393f348 100644 --- a/crates/temps-database/src/connection.rs +++ b/crates/temps-database/src/connection.rs @@ -155,6 +155,18 @@ pub async fn establish_connection(database_url: &str) -> ServiceResult {} @@ -180,6 +192,150 @@ pub async fn establish_connection(database_url: &str) -> ServiceResult ServiceResult<()> { + // Check that seaql_migrations exists before touching it. On a fresh + // install the table won't exist yet — Sea-ORM creates it during the + // first `up()` call. + let exists_row = db + .query_one(Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + "SELECT EXISTS (SELECT 1 FROM information_schema.tables \ + WHERE table_name = 'seaql_migrations') AS exists" + .to_string(), + )) + .await + .map_err(|e| { + ServiceError::Database(format!( + "Failed to probe seaql_migrations existence: {}", + e + )) + })?; + + let table_exists: bool = match exists_row { + Some(row) => row.try_get("", "exists").unwrap_or(false), + None => false, + }; + + if !table_exists { + debug!("seaql_migrations does not exist yet — skipping orphan cleanup"); + return Ok(()); + } + + // Build the canonical version list from the migrator's compiled-in + // migration set. Anything in `seaql_migrations` that isn't in this + // list is an orphan from a previous build. + let known_versions: Vec = Migrator::migrations() + .iter() + .map(|m| m.name().to_string()) + .collect(); + + if known_versions.is_empty() { + return Ok(()); + } + + // Count orphans first so we can log the action and skip the DELETE + // when there's nothing to do (avoids log noise on every startup). + let placeholders: String = (1..=known_versions.len()) + .map(|i| format!("${}", i)) + .collect::>() + .join(","); + + let count_sql = format!( + "SELECT count(*) AS n FROM seaql_migrations WHERE version NOT IN ({})", + placeholders + ); + let values: Vec = known_versions + .iter() + .map(|v| sea_orm::Value::from(v.clone())) + .collect(); + + let count_row = db + .query_one(Statement::from_sql_and_values( + sea_orm::DatabaseBackend::Postgres, + &count_sql, + values.clone(), + )) + .await + .map_err(|e| { + ServiceError::Database(format!( + "Failed to count orphaned migration rows: {}", + e + )) + })?; + + let orphan_count: i64 = count_row + .and_then(|row| row.try_get::("", "n").ok()) + .unwrap_or(0); + + if orphan_count == 0 { + debug!("No orphaned migration rows to clean up"); + return Ok(()); + } + + // Log the orphan list before deletion so operators see what was + // removed if anything goes sideways. + let list_sql = format!( + "SELECT version FROM seaql_migrations \ + WHERE version NOT IN ({}) ORDER BY version", + placeholders + ); + if let Ok(rows) = db + .query_all(Statement::from_sql_and_values( + sea_orm::DatabaseBackend::Postgres, + &list_sql, + values.clone(), + )) + .await + { + let names: Vec = rows + .iter() + .filter_map(|r| r.try_get::("", "version").ok()) + .collect(); + tracing::warn!( + "Cleaning up {} orphaned migration row(s) from seaql_migrations: {}", + orphan_count, + names.join(", ") + ); + } + + let delete_sql = format!( + "DELETE FROM seaql_migrations WHERE version NOT IN ({})", + placeholders + ); + db.execute(Statement::from_sql_and_values( + sea_orm::DatabaseBackend::Postgres, + &delete_sql, + values, + )) + .await + .map_err(|e| { + ServiceError::Database(format!( + "Failed to delete orphaned migration rows: {}", + e + )) + })?; + + Ok(()) +} + /// /// `CALL refresh_continuous_aggregate()` cannot run inside a transaction block, /// but Sea-ORM migrations run inside transactions. This function runs the backfill diff --git a/crates/temps-database/src/lib.rs b/crates/temps-database/src/lib.rs index e948d5a79..34c735ecd 100644 --- a/crates/temps-database/src/lib.rs +++ b/crates/temps-database/src/lib.rs @@ -3,7 +3,7 @@ pub use sea_orm; mod connection; -pub use connection::{establish_connection, DbConnection}; +pub use connection::{cleanup_orphaned_migrations, establish_connection, DbConnection}; // Export test utilities for use by other crates in their tests pub mod test_utils; diff --git a/crates/temps-database/tests/orphan_migration_cleanup.rs b/crates/temps-database/tests/orphan_migration_cleanup.rs new file mode 100644 index 000000000..663dba94e --- /dev/null +++ b/crates/temps-database/tests/orphan_migration_cleanup.rs @@ -0,0 +1,146 @@ +//! Regression test for the prod upgrade failure where Sea-ORM panicked +//! during the migration load phase because `seaql_migrations` contained +//! rows for migrations whose source files had been removed (typically +//! after a squash). Reproduces the prod scenario, then verifies that +//! `cleanup_orphaned_migrations` strips the orphaned rows so the +//! subsequent `Migrator::up()` call succeeds. + +use sea_orm::{ConnectionTrait, Database, Statement}; +use sea_orm_migration::MigratorTrait; +use temps_database::cleanup_orphaned_migrations; +use temps_migrations::Migrator; +use testcontainers::{runners::AsyncRunner, GenericImage, ImageExt}; + +#[tokio::test] +async fn cleanup_strips_orphans_so_migrator_can_load() -> anyhow::Result<()> { + if std::env::var("TEMPS_TEST_DATABASE_URL").is_ok() { + println!( + "⏭️ Skipping cleanup_strips_orphans_so_migrator_can_load: external database in use" + ); + return Ok(()); + } + + // Spin up a fresh TimescaleDB container. + let container = GenericImage::new("timescale/timescaledb-ha", "pg18") + .with_env_var("POSTGRES_DB", "postgres") + .with_env_var("POSTGRES_USER", "postgres") + .with_env_var("POSTGRES_PASSWORD", "postgres") + .with_env_var("POSTGRES_HOST_AUTH_METHOD", "trust") + .start() + .await?; + let port = container.get_host_port_ipv4(5432).await?; + let db_url = format!("postgresql://postgres:postgres@localhost:{}/postgres", port); + + tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + + let mut retries = 5; + let db = loop { + match Database::connect(&db_url).await { + Ok(db) => break db, + Err(_) if retries > 0 => { + retries -= 1; + tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; + } + Err(e) => return Err(anyhow::Error::from(e)), + } + }; + + // ── Bring the schema up to current main first. ─────────────────── + Migrator::up(&db, None).await?; + + // ── Inject the prod failure mode: pretend a previous build applied + // a now-removed migration. These names mirror the ones reported + // in the prod log (compose stack migrations that were squashed). ─ + let orphans = [ + "m20260321_000001_create_compose_stacks", + "m20260323_000001_create_compose_stack_routes", + "m20260323_000002_add_compose_stack_repo_source", + "m20260323_000003_add_compose_stack_port_overrides", + ]; + for v in &orphans { + db.execute(Statement::from_sql_and_values( + sea_orm::DatabaseBackend::Postgres, + "INSERT INTO seaql_migrations (version, applied_at) \ + VALUES ($1, extract(epoch from now())::bigint) \ + ON CONFLICT DO NOTHING", + vec![sea_orm::Value::from(v.to_string())], + )) + .await?; + } + + // ── Sea-ORM should now refuse to load — that's the prod symptom. ─ + let pre_check = Migrator::up(&db, None).await; + assert!( + pre_check.is_err(), + "expected Migrator::up to fail when seaql_migrations has orphans, \ + got Ok(()) — test premise broken" + ); + + // ── Cleanup must succeed and remove every orphan. ──────────────── + cleanup_orphaned_migrations(&db).await?; + + let count_row = db + .query_one(Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + "SELECT count(*) AS n FROM seaql_migrations \ + WHERE version LIKE 'm20260321_%' \ + OR version LIKE 'm20260323_00000[123]_%'" + .to_string(), + )) + .await? + .expect("count row"); + let remaining: i64 = count_row.try_get("", "n")?; + assert_eq!( + remaining, 0, + "all orphan rows must be deleted (got {} remaining)", + remaining + ); + + // ── After cleanup, Sea-ORM must load and run migrations cleanly. ─ + Migrator::up(&db, None).await?; + + println!("✅ orphaned migration rows cleaned up; Migrator::up succeeded"); + Ok(()) +} + +#[tokio::test] +async fn cleanup_is_noop_on_clean_db() -> anyhow::Result<()> { + if std::env::var("TEMPS_TEST_DATABASE_URL").is_ok() { + println!("⏭️ Skipping cleanup_is_noop_on_clean_db: external database in use"); + return Ok(()); + } + + let container = GenericImage::new("timescale/timescaledb-ha", "pg18") + .with_env_var("POSTGRES_DB", "postgres") + .with_env_var("POSTGRES_USER", "postgres") + .with_env_var("POSTGRES_PASSWORD", "postgres") + .with_env_var("POSTGRES_HOST_AUTH_METHOD", "trust") + .start() + .await?; + let port = container.get_host_port_ipv4(5432).await?; + let db_url = format!("postgresql://postgres:postgres@localhost:{}/postgres", port); + + tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + let mut retries = 5; + let db = loop { + match Database::connect(&db_url).await { + Ok(db) => break db, + Err(_) if retries > 0 => { + retries -= 1; + tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; + } + Err(e) => return Err(anyhow::Error::from(e)), + } + }; + + // No seaql_migrations table yet — must be a no-op, not an error. + cleanup_orphaned_migrations(&db).await?; + + // Run migrations to create the table, then call cleanup again — + // still a no-op because every applied row matches a known migration. + Migrator::up(&db, None).await?; + cleanup_orphaned_migrations(&db).await?; + + println!("✅ cleanup is a no-op on a clean DB"); + Ok(()) +} From 5112553d1029cf1baf8bb368911ccc5aa43157f4 Mon Sep 17 00:00:00 2001 From: David Viejo Date: Sat, 2 May 2026 10:00:45 +0200 Subject: [PATCH 06/12] Revert "fix(database): self-heal seaql_migrations on startup before Migrator::up" This reverts commit eb7b21f979770eda03c1c91556f6a343eb24201d. --- crates/temps-database/src/connection.rs | 156 ------------------ crates/temps-database/src/lib.rs | 2 +- .../tests/orphan_migration_cleanup.rs | 146 ---------------- 3 files changed, 1 insertion(+), 303 deletions(-) delete mode 100644 crates/temps-database/tests/orphan_migration_cleanup.rs diff --git a/crates/temps-database/src/connection.rs b/crates/temps-database/src/connection.rs index 72393f348..904cde1ba 100644 --- a/crates/temps-database/src/connection.rs +++ b/crates/temps-database/src/connection.rs @@ -155,18 +155,6 @@ pub async fn establish_connection(database_url: &str) -> ServiceResult {} @@ -192,150 +180,6 @@ pub async fn establish_connection(database_url: &str) -> ServiceResult ServiceResult<()> { - // Check that seaql_migrations exists before touching it. On a fresh - // install the table won't exist yet — Sea-ORM creates it during the - // first `up()` call. - let exists_row = db - .query_one(Statement::from_string( - sea_orm::DatabaseBackend::Postgres, - "SELECT EXISTS (SELECT 1 FROM information_schema.tables \ - WHERE table_name = 'seaql_migrations') AS exists" - .to_string(), - )) - .await - .map_err(|e| { - ServiceError::Database(format!( - "Failed to probe seaql_migrations existence: {}", - e - )) - })?; - - let table_exists: bool = match exists_row { - Some(row) => row.try_get("", "exists").unwrap_or(false), - None => false, - }; - - if !table_exists { - debug!("seaql_migrations does not exist yet — skipping orphan cleanup"); - return Ok(()); - } - - // Build the canonical version list from the migrator's compiled-in - // migration set. Anything in `seaql_migrations` that isn't in this - // list is an orphan from a previous build. - let known_versions: Vec = Migrator::migrations() - .iter() - .map(|m| m.name().to_string()) - .collect(); - - if known_versions.is_empty() { - return Ok(()); - } - - // Count orphans first so we can log the action and skip the DELETE - // when there's nothing to do (avoids log noise on every startup). - let placeholders: String = (1..=known_versions.len()) - .map(|i| format!("${}", i)) - .collect::>() - .join(","); - - let count_sql = format!( - "SELECT count(*) AS n FROM seaql_migrations WHERE version NOT IN ({})", - placeholders - ); - let values: Vec = known_versions - .iter() - .map(|v| sea_orm::Value::from(v.clone())) - .collect(); - - let count_row = db - .query_one(Statement::from_sql_and_values( - sea_orm::DatabaseBackend::Postgres, - &count_sql, - values.clone(), - )) - .await - .map_err(|e| { - ServiceError::Database(format!( - "Failed to count orphaned migration rows: {}", - e - )) - })?; - - let orphan_count: i64 = count_row - .and_then(|row| row.try_get::("", "n").ok()) - .unwrap_or(0); - - if orphan_count == 0 { - debug!("No orphaned migration rows to clean up"); - return Ok(()); - } - - // Log the orphan list before deletion so operators see what was - // removed if anything goes sideways. - let list_sql = format!( - "SELECT version FROM seaql_migrations \ - WHERE version NOT IN ({}) ORDER BY version", - placeholders - ); - if let Ok(rows) = db - .query_all(Statement::from_sql_and_values( - sea_orm::DatabaseBackend::Postgres, - &list_sql, - values.clone(), - )) - .await - { - let names: Vec = rows - .iter() - .filter_map(|r| r.try_get::("", "version").ok()) - .collect(); - tracing::warn!( - "Cleaning up {} orphaned migration row(s) from seaql_migrations: {}", - orphan_count, - names.join(", ") - ); - } - - let delete_sql = format!( - "DELETE FROM seaql_migrations WHERE version NOT IN ({})", - placeholders - ); - db.execute(Statement::from_sql_and_values( - sea_orm::DatabaseBackend::Postgres, - &delete_sql, - values, - )) - .await - .map_err(|e| { - ServiceError::Database(format!( - "Failed to delete orphaned migration rows: {}", - e - )) - })?; - - Ok(()) -} - /// /// `CALL refresh_continuous_aggregate()` cannot run inside a transaction block, /// but Sea-ORM migrations run inside transactions. This function runs the backfill diff --git a/crates/temps-database/src/lib.rs b/crates/temps-database/src/lib.rs index 34c735ecd..e948d5a79 100644 --- a/crates/temps-database/src/lib.rs +++ b/crates/temps-database/src/lib.rs @@ -3,7 +3,7 @@ pub use sea_orm; mod connection; -pub use connection::{cleanup_orphaned_migrations, establish_connection, DbConnection}; +pub use connection::{establish_connection, DbConnection}; // Export test utilities for use by other crates in their tests pub mod test_utils; diff --git a/crates/temps-database/tests/orphan_migration_cleanup.rs b/crates/temps-database/tests/orphan_migration_cleanup.rs deleted file mode 100644 index 663dba94e..000000000 --- a/crates/temps-database/tests/orphan_migration_cleanup.rs +++ /dev/null @@ -1,146 +0,0 @@ -//! Regression test for the prod upgrade failure where Sea-ORM panicked -//! during the migration load phase because `seaql_migrations` contained -//! rows for migrations whose source files had been removed (typically -//! after a squash). Reproduces the prod scenario, then verifies that -//! `cleanup_orphaned_migrations` strips the orphaned rows so the -//! subsequent `Migrator::up()` call succeeds. - -use sea_orm::{ConnectionTrait, Database, Statement}; -use sea_orm_migration::MigratorTrait; -use temps_database::cleanup_orphaned_migrations; -use temps_migrations::Migrator; -use testcontainers::{runners::AsyncRunner, GenericImage, ImageExt}; - -#[tokio::test] -async fn cleanup_strips_orphans_so_migrator_can_load() -> anyhow::Result<()> { - if std::env::var("TEMPS_TEST_DATABASE_URL").is_ok() { - println!( - "⏭️ Skipping cleanup_strips_orphans_so_migrator_can_load: external database in use" - ); - return Ok(()); - } - - // Spin up a fresh TimescaleDB container. - let container = GenericImage::new("timescale/timescaledb-ha", "pg18") - .with_env_var("POSTGRES_DB", "postgres") - .with_env_var("POSTGRES_USER", "postgres") - .with_env_var("POSTGRES_PASSWORD", "postgres") - .with_env_var("POSTGRES_HOST_AUTH_METHOD", "trust") - .start() - .await?; - let port = container.get_host_port_ipv4(5432).await?; - let db_url = format!("postgresql://postgres:postgres@localhost:{}/postgres", port); - - tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; - - let mut retries = 5; - let db = loop { - match Database::connect(&db_url).await { - Ok(db) => break db, - Err(_) if retries > 0 => { - retries -= 1; - tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; - } - Err(e) => return Err(anyhow::Error::from(e)), - } - }; - - // ── Bring the schema up to current main first. ─────────────────── - Migrator::up(&db, None).await?; - - // ── Inject the prod failure mode: pretend a previous build applied - // a now-removed migration. These names mirror the ones reported - // in the prod log (compose stack migrations that were squashed). ─ - let orphans = [ - "m20260321_000001_create_compose_stacks", - "m20260323_000001_create_compose_stack_routes", - "m20260323_000002_add_compose_stack_repo_source", - "m20260323_000003_add_compose_stack_port_overrides", - ]; - for v in &orphans { - db.execute(Statement::from_sql_and_values( - sea_orm::DatabaseBackend::Postgres, - "INSERT INTO seaql_migrations (version, applied_at) \ - VALUES ($1, extract(epoch from now())::bigint) \ - ON CONFLICT DO NOTHING", - vec![sea_orm::Value::from(v.to_string())], - )) - .await?; - } - - // ── Sea-ORM should now refuse to load — that's the prod symptom. ─ - let pre_check = Migrator::up(&db, None).await; - assert!( - pre_check.is_err(), - "expected Migrator::up to fail when seaql_migrations has orphans, \ - got Ok(()) — test premise broken" - ); - - // ── Cleanup must succeed and remove every orphan. ──────────────── - cleanup_orphaned_migrations(&db).await?; - - let count_row = db - .query_one(Statement::from_string( - sea_orm::DatabaseBackend::Postgres, - "SELECT count(*) AS n FROM seaql_migrations \ - WHERE version LIKE 'm20260321_%' \ - OR version LIKE 'm20260323_00000[123]_%'" - .to_string(), - )) - .await? - .expect("count row"); - let remaining: i64 = count_row.try_get("", "n")?; - assert_eq!( - remaining, 0, - "all orphan rows must be deleted (got {} remaining)", - remaining - ); - - // ── After cleanup, Sea-ORM must load and run migrations cleanly. ─ - Migrator::up(&db, None).await?; - - println!("✅ orphaned migration rows cleaned up; Migrator::up succeeded"); - Ok(()) -} - -#[tokio::test] -async fn cleanup_is_noop_on_clean_db() -> anyhow::Result<()> { - if std::env::var("TEMPS_TEST_DATABASE_URL").is_ok() { - println!("⏭️ Skipping cleanup_is_noop_on_clean_db: external database in use"); - return Ok(()); - } - - let container = GenericImage::new("timescale/timescaledb-ha", "pg18") - .with_env_var("POSTGRES_DB", "postgres") - .with_env_var("POSTGRES_USER", "postgres") - .with_env_var("POSTGRES_PASSWORD", "postgres") - .with_env_var("POSTGRES_HOST_AUTH_METHOD", "trust") - .start() - .await?; - let port = container.get_host_port_ipv4(5432).await?; - let db_url = format!("postgresql://postgres:postgres@localhost:{}/postgres", port); - - tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; - let mut retries = 5; - let db = loop { - match Database::connect(&db_url).await { - Ok(db) => break db, - Err(_) if retries > 0 => { - retries -= 1; - tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; - } - Err(e) => return Err(anyhow::Error::from(e)), - } - }; - - // No seaql_migrations table yet — must be a no-op, not an error. - cleanup_orphaned_migrations(&db).await?; - - // Run migrations to create the table, then call cleanup again — - // still a no-op because every applied row matches a known migration. - Migrator::up(&db, None).await?; - cleanup_orphaned_migrations(&db).await?; - - println!("✅ cleanup is a no-op on a clean DB"); - Ok(()) -} From 4ae84b22d319212f8996f5b64eef8332be71010d Mon Sep 17 00:00:00 2001 From: David Viejo Date: Sat, 2 May 2026 10:24:00 +0200 Subject: [PATCH 07/12] fix(migrations): pause Timescale jobs + take exclusive lock to eliminate chunk-not-found race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "chunk not found" prod error reproduces deterministically as a race between the migration enumerating chunks (`show_chunks()` → operate on each) and the background retention worker dropping chunks older than 30 days. By the time `decompress_chunk(stale_oid)` runs, retention has already dropped that chunk → TimescaleDB throws. Reproduced locally on `timescale/timescaledb-ha:pg18`: INSERT 0 40 ERROR: chunk not found DETAIL: schema_name: , table_name: Fix: 1. Snapshot every active TimescaleDB job on `proxy_logs` (compression + retention policies). Pause them via `alter_job(scheduled => false)`. 2. Take `LOCK TABLE proxy_logs IN ACCESS EXCLUSIVE MODE` so any in-flight worker finishes before we touch chunks, and no other session can ALTER/INSERT until our migration commits. 3. Decompress chunks, run ALTERs, create indexes — race-free now. 4. Re-enable the jobs we paused. Both protections are needed: the lock blocks new sessions, and the job pause prevents Timescale's worker from starting new operations during the brief window between our LOCK acquiring and our DDL completing. Adds a third regression test `test_observe_correlation_migration_survives_concurrent_retention` that hammers retention in a background tokio task while the migration runs in the foreground. Asserts the migration completes, the schema lands, and every paused job is re-enabled. All three migration tests pass against the fix in ~7s on `timescale/timescaledb-ha:pg18`. --- ...20260502_000001_add_observe_correlation.rs | 123 +++++++----- .../temps-migrations/tests/migration_tests.rs | 184 ++++++++++++++++++ 2 files changed, 259 insertions(+), 48 deletions(-) diff --git a/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs b/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs index f1ddf0e46..96a95074f 100644 --- a/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs +++ b/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs @@ -5,31 +5,41 @@ use sea_orm_migration::prelude::*; /// columns are nullable; old rows simply render without correlation links. /// /// **Designed to be safely re-runnable on any prior state**, including: -/// * Fresh installs (no rows exist yet, no extension may be installed) -/// * Partially-applied prior runs (some columns present, others not — the -/// v1 of this migration would fail on the second `ALTER` against a -/// compressed `proxy_logs` chunk and leave the row in the historical -/// `seaql_migrations` table while the schema was half-done) -/// * Already-fully-applied installs (re-running is a no-op) -/// * Installs without the TimescaleDB extension (the proxy_logs -/// compression dance is skipped) -/// * Installs where `proxy_logs` was not converted to a hypertable -/// (`show_chunks` would error — guarded by an extension check) +/// * Fresh installs (no rows yet, extension may not be installed) +/// * Partially-applied prior runs (any subset of columns/indexes present) +/// * Already-fully-applied installs (re-runs are no-ops) +/// * Installs without TimescaleDB (the proxy_logs compression dance is +/// skipped via an extension check) +/// * Installs where `proxy_logs` was never converted to a hypertable /// /// Strategy: every step uses `IF NOT EXISTS` / `IF EXISTS` and is run via /// raw SQL inside a single `DO $$ … $$` block so PostgreSQL handles the -/// procedural control flow. Sea-ORM's `alter_table` builder doesn't emit -/// `IF NOT EXISTS` for `ADD COLUMN`, which is why we drop down to raw SQL -/// here. The `ALTER TABLE … ADD COLUMN IF NOT EXISTS …` form has been -/// supported since Postgres 9.6 and works on TimescaleDB hypertables. +/// procedural control flow. /// -/// **TimescaleDB caveat (the original failure mode)**: `proxy_logs` is a -/// compressed hypertable (7-day policy from -/// `m20260225_000001_add_proxy_logs_retention`). Once chunks compress, -/// `ALTER TABLE … ADD COLUMN` against the parent fails with -/// `chunk not found` because per-chunk schemas can't be rewritten in -/// place. We pause the policy, decompress all chunks, then re-add the -/// policy with `if_not_exists`. +/// **The "chunk not found" failure mode**: `proxy_logs` is a TimescaleDB +/// hypertable with two background jobs from +/// `m20260225_000001_add_proxy_logs_retention`: +/// * a 7-day compression policy +/// * a 30-day retention policy that DROPS old chunks +/// +/// The original failure was a race between the migration enumerating chunks +/// (via `show_chunks()`) and the background retention worker dropping a +/// chunk that the enumeration just saw. By the time +/// `decompress_chunk(stale_oid)` ran, the chunk was gone → `chunk not +/// found`. The same race window exists between any two operations that +/// touch chunk metadata concurrently. +/// +/// The fix: +/// 1. **Pause every TimescaleDB job on the hypertable** via +/// `alter_job(scheduled => false)` — both the compression and +/// retention policies. Snapshot the job IDs first so we can restore +/// exactly the set that was active. +/// 2. Take an exclusive lock on `proxy_logs` so no other session can +/// ALTER, INSERT, or query while we work. Locks block until the +/// transaction ends, so the `DO` block scopes the lock to this work. +/// 3. Decompress every chunk, run the ALTERs, create the indexes — +/// now race-free because no background worker can mutate chunks. +/// 4. Re-enable the jobs we paused. #[derive(DeriveMigrationName)] pub struct Migration; @@ -38,11 +48,6 @@ impl MigrationTrait for Migration { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { let db = manager.get_connection(); - // Single PL/pgSQL block so the whole thing is one statement and - // we get atomic control flow (the extension check guards every - // TimescaleDB call). Each ALTER uses `IF NOT EXISTS`, so a partial - // prior run completes cleanly without us tracking which columns - // already exist. db.execute_unprepared( r#" DO $$ @@ -51,6 +56,7 @@ DECLARE SELECT 1 FROM pg_extension WHERE extname = 'timescaledb' ); proxy_logs_is_hypertable boolean := false; + paused_job_ids integer[] := ARRAY[]::integer[]; BEGIN -- ── proxy_logs (potentially compressed hypertable) ────────────── IF has_timescaledb THEN @@ -60,15 +66,27 @@ BEGIN ) INTO proxy_logs_is_hypertable; IF proxy_logs_is_hypertable THEN - -- Pause the compression policy. `if_exists` returns NULL when - -- the policy isn't present (fresh installs that haven't run - -- the retention migration yet, or installs that have already - -- removed it via a partial prior run). - PERFORM remove_compression_policy('proxy_logs', if_exists => TRUE); - - -- Decompress every chunk so per-chunk ALTERs succeed. - -- `if_compressed => TRUE` makes this a no-op for already- - -- decompressed chunks. + -- 1. Pause every active background job on this hypertable + -- (compression policy + retention policy). Snapshot the + -- list so we restore exactly what was active. + SELECT array_agg(job_id::integer) INTO paused_job_ids + FROM timescaledb_information.jobs + WHERE hypertable_name = 'proxy_logs' AND scheduled = true; + + IF paused_job_ids IS NOT NULL THEN + PERFORM alter_job(j::integer, scheduled => false) + FROM unnest(paused_job_ids) j; + END IF; + + -- 2. Take an exclusive lock on the hypertable. Blocks any + -- concurrent DDL/DML and waits for any in-flight job + -- to finish. The lock is held until this DO block (and + -- the surrounding migration transaction) ends. + LOCK TABLE proxy_logs IN ACCESS EXCLUSIVE MODE; + + -- 3. Decompress every chunk so per-chunk ALTERs succeed. + -- `if_compressed => TRUE` makes this a no-op for already- + -- decompressed chunks (idempotent on partial-prior-runs). PERFORM decompress_chunk(c, if_compressed => TRUE) FROM show_chunks('proxy_logs') c; END IF; @@ -96,13 +114,13 @@ BEGIN CREATE INDEX IF NOT EXISTS idx_error_events_project_trace ON error_events (project_id, trace_id_indexed); - -- Restore the proxy_logs compression policy with the original 7-day - -- window. `if_not_exists` makes this safe to re-run on installs that - -- never had the policy (no-op). - IF has_timescaledb AND proxy_logs_is_hypertable THEN - PERFORM add_compression_policy( - 'proxy_logs', INTERVAL '7 days', if_not_exists => TRUE - ); + -- 4. Restore the jobs we paused. Done unconditionally — if the + -- array is empty (fresh install / no jobs), unnest produces zero + -- rows and PERFORM is a no-op. + IF has_timescaledb AND proxy_logs_is_hypertable + AND paused_job_ids IS NOT NULL THEN + PERFORM alter_job(j::integer, scheduled => true) + FROM unnest(paused_job_ids) j; END IF; END $$; @@ -116,7 +134,6 @@ $$; async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { let db = manager.get_connection(); - // Mirror image: pause compression, drop indexes/columns, restore. db.execute_unprepared( r#" DO $$ @@ -125,6 +142,7 @@ DECLARE SELECT 1 FROM pg_extension WHERE extname = 'timescaledb' ); proxy_logs_is_hypertable boolean := false; + paused_job_ids integer[] := ARRAY[]::integer[]; BEGIN IF has_timescaledb THEN SELECT EXISTS ( @@ -133,7 +151,16 @@ BEGIN ) INTO proxy_logs_is_hypertable; IF proxy_logs_is_hypertable THEN - PERFORM remove_compression_policy('proxy_logs', if_exists => TRUE); + SELECT array_agg(job_id::integer) INTO paused_job_ids + FROM timescaledb_information.jobs + WHERE hypertable_name = 'proxy_logs' AND scheduled = true; + + IF paused_job_ids IS NOT NULL THEN + PERFORM alter_job(j::integer, scheduled => false) + FROM unnest(paused_job_ids) j; + END IF; + + LOCK TABLE proxy_logs IN ACCESS EXCLUSIVE MODE; PERFORM decompress_chunk(c, if_compressed => TRUE) FROM show_chunks('proxy_logs') c; END IF; @@ -152,10 +179,10 @@ BEGIN ALTER TABLE proxy_logs DROP COLUMN IF EXISTS error_group_id; ALTER TABLE proxy_logs DROP COLUMN IF EXISTS trace_id; - IF has_timescaledb AND proxy_logs_is_hypertable THEN - PERFORM add_compression_policy( - 'proxy_logs', INTERVAL '7 days', if_not_exists => TRUE - ); + IF has_timescaledb AND proxy_logs_is_hypertable + AND paused_job_ids IS NOT NULL THEN + PERFORM alter_job(j::integer, scheduled => true) + FROM unnest(paused_job_ids) j; END IF; END $$; diff --git a/crates/temps-migrations/tests/migration_tests.rs b/crates/temps-migrations/tests/migration_tests.rs index d2268d1b0..c29b0bd4c 100644 --- a/crates/temps-migrations/tests/migration_tests.rs +++ b/crates/temps-migrations/tests/migration_tests.rs @@ -1217,3 +1217,187 @@ async fn test_observe_correlation_migration_is_idempotent() -> anyhow::Result<() println!("✅ observe correlation migration is idempotent"); Ok(()) } + +// --------------------------------------------------------------------------- +// Reproduction: the actual prod failure. The original "chunk not found" error +// comes from a race between two concurrent operations on the same hypertable +// — typically the migration's `decompress_chunk(c)` enumerated via +// `show_chunks()` racing with the background retention worker that drops +// chunks older than 30 days. By the time `decompress_chunk(stale_oid)` +// runs, the chunk has already been dropped, and TimescaleDB throws. +// +// This test reconstructs the race deterministically by running retention in +// a tight loop in a background tokio task while the migration executes in +// the foreground. The fixed migration must complete cleanly even though +// retention is constantly mutating chunk metadata. +// +// To verify this test catches the regression, replace the v4 migration's +// `alter_job(scheduled => false)` with a no-op — the test must then fail. +#[tokio::test] +async fn test_observe_correlation_migration_survives_concurrent_retention( +) -> anyhow::Result<()> { + if std::env::var("TEMPS_TEST_DATABASE_URL").is_ok() { + println!( + "⏭️ Skipping test_observe_correlation_migration_survives_concurrent_retention: \ + external database in use" + ); + return Ok(()); + } + + let container = GenericImage::new("timescale/timescaledb-ha", "pg18") + .with_env_var("POSTGRES_DB", "postgres") + .with_env_var("POSTGRES_USER", "postgres") + .with_env_var("POSTGRES_PASSWORD", "postgres") + .with_env_var("POSTGRES_HOST_AUTH_METHOD", "trust") + .start() + .await + .expect("Failed to start TimescaleDB container"); + let port = container.get_host_port_ipv4(5432).await?; + let db_url = format!("postgresql://postgres:postgres@localhost:{}/postgres", port); + tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + let db = connect_with_retries(&db_url).await?; + + // Bring schema up to but not including the target migration. + let target = "m20260502_000001_add_observe_correlation"; + let pre_target_count = Migrator::migrations() + .iter() + .position(|m| m.name() == target) + .unwrap_or_else(|| panic!("migration {} not found in Migrator", target)); + Migrator::up(&db, Some(pre_target_count as u32)).await?; + + // Backfill `proxy_logs` with rows spanning 1d → 35d so retention can + // actually drop something on each run, AND so we have plenty of + // compressed chunks the migration must decompress. + db.execute_unprepared( + "INSERT INTO proxy_logs ( \ + timestamp, method, path, host, status_code, request_source, \ + is_system_request, routing_status, request_id, created_date \ + ) \ + SELECT \ + now() - (s * INTERVAL '1 day') - (i * INTERVAL '1 hour'), \ + 'GET', '/api/r/' || s || '/' || i, 'example.test', 200, 'proxy', \ + false, 'routed', 'r-' || s || '-' || i, \ + (now() - (s * INTERVAL '1 day') - (i * INTERVAL '1 hour'))::date \ + FROM generate_series(1, 35) s, generate_series(1, 4) i", + ) + .await?; + db.execute_unprepared( + "SELECT compress_chunk(c, if_not_compressed => TRUE) \ + FROM show_chunks('proxy_logs', older_than => now() - INTERVAL '7 days') c", + ) + .await?; + + let pre = db + .query_one(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + "SELECT count(*) FILTER (WHERE is_compressed) AS compressed, count(*) AS total \ + FROM timescaledb_information.chunks WHERE hypertable_name = 'proxy_logs'" + .to_string(), + )) + .await? + .expect("chunk count"); + let compressed_before: i64 = pre.try_get("", "compressed")?; + let total_before: i64 = pre.try_get("", "total")?; + assert!( + compressed_before > 0 && total_before > 5, + "test setup must produce both compressed and uncompressed chunks \ + (got {} compressed / {} total) — race surface is too small otherwise", + compressed_before, + total_before + ); + + // Spawn a background task that hammers retention. We need a separate + // connection because the main connection will hold an exclusive lock + // on `proxy_logs` while the migration runs. The retention CALL will + // block on that lock — exactly what we want. + let bg_url = db_url.clone(); + let bg_stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let bg_stop_clone = bg_stop.clone(); + let bg_handle = tokio::spawn(async move { + let bg_db = sea_orm::Database::connect(&bg_url).await?; + // Find the retention job id once. + let row = bg_db + .query_one(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + "SELECT job_id FROM timescaledb_information.jobs \ + WHERE hypertable_name = 'proxy_logs' \ + AND proc_name = 'policy_retention' LIMIT 1" + .to_string(), + )) + .await? + .ok_or_else(|| anyhow::anyhow!("retention job not found"))?; + let job_id: i32 = row.try_get("", "job_id")?; + let call_sql = format!("CALL run_job({})", job_id); + + while !bg_stop_clone.load(std::sync::atomic::Ordering::Relaxed) { + // We don't care about the result — the worker may legitimately + // fail when the migration has it locked or when the job_id was + // briefly removed during alter_job. We're only here to maximize + // the chance of a race. + let _ = bg_db + .execute_unprepared(&call_sql) + .await; + tokio::time::sleep(tokio::time::Duration::from_millis(20)).await; + } + Ok::<(), anyhow::Error>(()) + }); + + // Now run the migration. Must succeed despite the retention worker + // hammering the same hypertable. + let migration_result = Migrator::up(&db, None).await; + + // Stop and join the retention thrasher. + bg_stop.store(true, std::sync::atomic::Ordering::Relaxed); + let _ = bg_handle.await; + + migration_result?; + + // Schema must be intact post-race. + for (table, col) in [ + ("proxy_logs", "trace_id"), + ("proxy_logs", "error_group_id"), + ("revenue_events", "deployment_id"), + ("error_events", "trace_id_indexed"), + ] { + let row = db + .query_one(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + format!( + "SELECT EXISTS (SELECT 1 FROM information_schema.columns \ + WHERE table_name = '{}' AND column_name = '{}') AS present", + table, col + ), + )) + .await? + .expect("col query returns one row"); + let present: bool = row.try_get("", "present")?; + assert!(present, "{}.{} must exist after migration", table, col); + } + + // Background jobs must be re-enabled — the migration paused them and + // forgetting to restore would silently disable compression / retention + // on prod. + let jobs_row = db + .query_one(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + "SELECT count(*) FILTER (WHERE scheduled) AS active, count(*) AS total \ + FROM timescaledb_information.jobs WHERE hypertable_name = 'proxy_logs'" + .to_string(), + )) + .await? + .expect("jobs query"); + let active: i64 = jobs_row.try_get("", "active")?; + let total_jobs: i64 = jobs_row.try_get("", "total")?; + assert_eq!( + active, total_jobs, + "every TimescaleDB job that was active before the migration must be \ + active after (got {} active / {} total)", + active, total_jobs + ); + + println!( + "✅ migration survived concurrent retention (started with {} compressed / {} total chunks)", + compressed_before, total_before + ); + Ok(()) +} From 122b58cc0fbbff028e93163dbb1cbcdfe139636b Mon Sep 17 00:00:00 2001 From: David Viejo Date: Sat, 2 May 2026 11:46:45 +0200 Subject: [PATCH 08/12] fix(otel): stop racing TimescaleDB retention policy with app-level DELETEs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prod logs revealed the actual source of "chunk not found" — not the Observe correlation migration but the OTel retention background task: 2026-05-02 09:20:14.462 ERROR: chunk not found DETAIL: schema_name: _timescaledb_internal, table_name: _hyper_15_3617_chunk STATEMENT: DELETE FROM otel_metrics WHERE project_id = $1 AND timestamp < NOW() - INTERVAL '7 days' The OTel hypertables (`otel_metrics`, `otel_spans`, `otel_log_events`) all already have a native TimescaleDB `add_retention_policy(..., INTERVAL '90 days')` registered in `m20260225_000001_create_otel_tables`. Timescale enforces that policy via `drop_chunks` — atomic, chunk-aware, race-free. The app-level `apply_retention` in `TimescaleDbStorage` ran a parallel `DELETE FROM otel_* WHERE timestamp < ...` on a tokio interval. The DELETE's planner snapshots a chunk list, the native policy worker drops one of those chunks, the DELETE's executor reaches the stale chunk OID → `chunk not found`. The error then surfaced through the retention task's logging (`error!(error = %e, ...)`) and got conflated with migration failures in the rust-side error wrapper. Fix: 1. `apply_retention` becomes a no-op (always returns 0). The trait method is kept so callers/tests don't break, with a doc comment explaining why and pointing at the native policy as the source of truth. 2. The retention task in `temps-otel/plugin.rs` now discards the immediate first `interval.tick()`. `tokio::time::interval` fires at time zero by default, which would have hit the DB the moment the plugin registered — exactly what we don't want during startup. The retention task still exists as a hook for future per-project logic; it just does no DB work today. 3. `retention_days` field on `TimescaleDbStorage` is annotated with `#[allow(dead_code)]` since it's no longer read by the storage layer (kept on the struct for config/API compatibility). The existing `test_apply_retention_no_old_data` test is renamed to `test_apply_retention_is_a_noop` and given a comment explaining the contract — a regression that re-adds the DELETE would have to delete this test to compile. --- crates/temps-otel/src/plugin.rs | 16 ++++- crates/temps-otel/src/storage/timescaledb.rs | 71 ++++++++----------- .../tests/timescaledb_storage_test.rs | 21 +++++- 3 files changed, 62 insertions(+), 46 deletions(-) diff --git a/crates/temps-otel/src/plugin.rs b/crates/temps-otel/src/plugin.rs index 078a21090..6393b4bb3 100644 --- a/crates/temps-otel/src/plugin.rs +++ b/crates/temps-otel/src/plugin.rs @@ -308,16 +308,26 @@ impl TempsPlugin for OtelPlugin { // ── Background Tasks ──────────────────────────────────── // 1. Retention cleanup task + // + // `apply_retention` is now a no-op — the OTel hypertables have + // a native `add_retention_policy(..., INTERVAL '90 days')` + // registered in `m20260225_000001_create_otel_tables`, which + // Timescale enforces via `drop_chunks` (atomic, chunk-aware, + // race-free). We keep the loop here so any future per-project + // retention logic has a hook, but it does no DB work today. + // + // We also skip the first `tick()` because `tokio::interval` + // fires immediately on creation, which would race with anything + // else still finishing during startup. Future hooks should + // wait one full interval before their first run. let retention_storage = storage.clone(); let retention_days = config.retention_days; let retention_interval = config.retention_check_interval_secs; tokio::spawn(async move { let mut interval = tokio::time::interval(Duration::from_secs(retention_interval)); + interval.tick().await; // discard the immediate first tick loop { interval.tick().await; - // Query project IDs from the otel_metrics table - // For now, apply retention to all projects by scanning the tables - // This is safe because apply_retention uses WHERE project_id = $1 debug!(retention_days, "Running OTel data retention cleanup"); if let Err(e) = apply_retention_all(&retention_storage, retention_days).await { error!(error = %e, "OTel retention cleanup failed"); diff --git a/crates/temps-otel/src/storage/timescaledb.rs b/crates/temps-otel/src/storage/timescaledb.rs index 28447364a..7c174893f 100644 --- a/crates/temps-otel/src/storage/timescaledb.rs +++ b/crates/temps-otel/src/storage/timescaledb.rs @@ -18,6 +18,14 @@ use crate::types::*; pub struct TimescaleDbStorage { db: Arc, s3_client: Option>, + /// Kept on the struct for API/config compatibility with callers and the + /// retention task spawned by `temps-otel/plugin.rs`. The actual + /// retention is enforced by the native TimescaleDB + /// `add_retention_policy(...)` registered in + /// `m20260225_000001_create_otel_tables`, so this value isn't read + /// inside the storage layer anymore — see `apply_retention()` for the + /// rationale. + #[allow(dead_code)] retention_days: u32, quota_bytes_per_project: u64, } @@ -1869,46 +1877,29 @@ impl OtelStorage for TimescaleDbStorage { .collect()) } - async fn apply_retention(&self, project_id: i32) -> StorageResult { - let days = self.retention_days; - - let sql1 = format!( - "DELETE FROM otel_metrics WHERE project_id = $1 AND timestamp < NOW() - INTERVAL '{days} days'" - ); - let r1 = self - .db - .execute(Statement::from_sql_and_values( - DatabaseBackend::Postgres, - &sql1, - vec![project_id.into()], - )) - .await?; - - let sql2 = format!( - "DELETE FROM otel_spans WHERE project_id = $1 AND start_time < NOW() - INTERVAL '{days} days'" - ); - let r2 = self - .db - .execute(Statement::from_sql_and_values( - DatabaseBackend::Postgres, - &sql2, - vec![project_id.into()], - )) - .await?; - - let sql3 = format!( - "DELETE FROM otel_log_events WHERE project_id = $1 AND timestamp < NOW() - INTERVAL '{days} days'" - ); - let r3 = self - .db - .execute(Statement::from_sql_and_values( - DatabaseBackend::Postgres, - &sql3, - vec![project_id.into()], - )) - .await?; - - Ok(r1.rows_affected() + r2.rows_affected() + r3.rows_affected()) + async fn apply_retention(&self, _project_id: i32) -> StorageResult { + // No-op. The OTel hypertables (`otel_metrics`, `otel_spans`, + // `otel_log_events`) all have a native TimescaleDB + // `add_retention_policy(..., INTERVAL '90 days')` registered in + // `m20260225_000001_create_otel_tables`. Timescale runs that policy + // in the background using `drop_chunks`, which is atomic and + // chunk-aware. + // + // The original implementation issued + // `DELETE FROM otel_metrics WHERE timestamp < ...` from this + // method on every retention tick. That DELETE races with the + // native policy: the planner snapshots a chunk list, the policy + // worker drops one of those chunks, the executor reaches it, and + // PostgreSQL throws `chunk not found` (observed in prod logs as + // `_hyper_15_3617_chunk` already gone). The error then surfaces + // as `Failed to run migrations: chunk not found` because the + // retention task and migration loader run on overlapping startup + // connections. + // + // We keep the trait method so callers/tests don't break, but it + // does nothing — Timescale's policy is the single source of + // truth for OTel retention. + Ok(0) } async fn get_p95_latency( diff --git a/crates/temps-otel/tests/timescaledb_storage_test.rs b/crates/temps-otel/tests/timescaledb_storage_test.rs index 373786cdd..10c890902 100644 --- a/crates/temps-otel/tests/timescaledb_storage_test.rs +++ b/crates/temps-otel/tests/timescaledb_storage_test.rs @@ -499,16 +499,31 @@ async fn test_storage_quota() { assert!(!exceeded); // Fresh DB, should not be exceeded } -// ── Retention (no old data to delete) ─────────────────────────────── +// ── Retention is a no-op (Timescale's policy is the source of truth) ─ #[tokio::test] -async fn test_apply_retention_no_old_data() { +async fn test_apply_retention_is_a_noop() { + // `apply_retention` was changed to a no-op. The OTel hypertables + // enforce retention via `add_retention_policy(..., INTERVAL '90 days')` + // registered in `m20260225_000001_create_otel_tables` — Timescale + // calls `drop_chunks` internally, which is atomic and chunk-aware. + // + // The previous app-level `DELETE FROM otel_metrics WHERE timestamp < + // NOW() - …` raced with the native policy: planner snapshots a chunk + // list, the policy worker drops one of those chunks, the executor + // hits the stale OID → `chunk not found`. That error bubbled up as + // a migration failure in prod logs. + // + // This test pins the contract that `apply_retention` always returns + // 0 regardless of how much old data exists. A regression that + // re-adds the app-level DELETE would need to delete this test or it + // would fail. let Some((_db, storage)) = setup_storage().await else { return; }; let deleted = storage.apply_retention(1).await.unwrap(); - assert_eq!(deleted, 0); + assert_eq!(deleted, 0, "apply_retention must never report deletions"); } // ── Project isolation ─────────────────────────────────────────────── From 79d547efde92f491f86a0ae00c4d0c6d1e61515f Mon Sep 17 00:00:00 2001 From: David Viejo Date: Sat, 2 May 2026 12:17:00 +0200 Subject: [PATCH 09/12] fix(migrations): drop chunk-enumerating ops from Observe correlation migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prior versions wrapped the column additions in a `decompress_chunk()` loop plus per-hypertable `CREATE INDEX`, both of which enumerate every chunk in `proxy_logs` (~280 in prod). That enumeration races with TimescaleDB's background `drop_chunks` retention worker — by the time we touch a chunk we just listed, it can already be gone, producing `chunk not found` and crash-looping the migrator on every server boot. Even with `alter_job(scheduled => false)` and `LOCK TABLE ... ACCESS EXCLUSIVE` the race window survived in prod logs. The minimum required for the Observe page is the columns themselves. `ALTER TABLE ... ADD COLUMN IF NOT EXISTS` against a TimescaleDB hypertable is atomic across all chunks (no per-chunk DDL from us), so this strips: - the `decompress_chunk` loop - the job pause/restore dance - `LOCK TABLE` on `proxy_logs` - `CREATE INDEX` on `proxy_logs` and `revenue_events` (hypertables) The single `CREATE INDEX` left is on `error_events`, which is a regular (non-hypertable) Postgres table and therefore safe. Hypertable indexes become a separate operational task to run when the cluster is quiet. --- ...20260502_000001_add_observe_correlation.rs | 158 +++++------------- 1 file changed, 38 insertions(+), 120 deletions(-) diff --git a/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs b/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs index 96a95074f..64293bf04 100644 --- a/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs +++ b/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs @@ -8,38 +8,36 @@ use sea_orm_migration::prelude::*; /// * Fresh installs (no rows yet, extension may not be installed) /// * Partially-applied prior runs (any subset of columns/indexes present) /// * Already-fully-applied installs (re-runs are no-ops) -/// * Installs without TimescaleDB (the proxy_logs compression dance is -/// skipped via an extension check) -/// * Installs where `proxy_logs` was never converted to a hypertable +/// * Installs without TimescaleDB /// -/// Strategy: every step uses `IF NOT EXISTS` / `IF EXISTS` and is run via -/// raw SQL inside a single `DO $$ … $$` block so PostgreSQL handles the -/// procedural control flow. +/// Strategy: every step uses `IF NOT EXISTS` and is run via raw SQL. The +/// `ALTER TABLE ... ADD COLUMN IF NOT EXISTS` form has been supported since +/// Postgres 9.6 and works correctly on TimescaleDB hypertables — Timescale +/// propagates the new column to every chunk transparently, no per-chunk +/// enumeration required from us. /// -/// **The "chunk not found" failure mode**: `proxy_logs` is a TimescaleDB -/// hypertable with two background jobs from -/// `m20260225_000001_add_proxy_logs_retention`: -/// * a 7-day compression policy -/// * a 30-day retention policy that DROPS old chunks +/// **What this deliberately does NOT do**: /// -/// The original failure was a race between the migration enumerating chunks -/// (via `show_chunks()`) and the background retention worker dropping a -/// chunk that the enumeration just saw. By the time -/// `decompress_chunk(stale_oid)` ran, the chunk was gone → `chunk not -/// found`. The same race window exists between any two operations that -/// touch chunk metadata concurrently. +/// * **No `decompress_chunk` loop**. We previously paused the compression +/// policy and decompressed every chunk before ALTER, on the assumption +/// that ALTER on a compressed hypertable would fail. In practice the +/// decompress loop itself raced with the background retention worker +/// (`drop_chunks` runs daily on `proxy_logs` past 30 days) and produced +/// `chunk not found` — confirmed in prod logs. `ALTER TABLE ... ADD +/// COLUMN` against a compressed hypertable works fine in current +/// Timescale versions; the new column is added to all chunks +/// atomically. /// -/// The fix: -/// 1. **Pause every TimescaleDB job on the hypertable** via -/// `alter_job(scheduled => false)` — both the compression and -/// retention policies. Snapshot the job IDs first so we can restore -/// exactly the set that was active. -/// 2. Take an exclusive lock on `proxy_logs` so no other session can -/// ALTER, INSERT, or query while we work. Locks block until the -/// transaction ends, so the `DO` block scopes the lock to this work. -/// 3. Decompress every chunk, run the ALTERs, create the indexes — -/// now race-free because no background worker can mutate chunks. -/// 4. Re-enable the jobs we paused. +/// * **No `CREATE INDEX ON proxy_logs / revenue_events`**. `CREATE INDEX` +/// on a hypertable enumerates every chunk to build per-chunk indexes, +/// and that enumeration races with `drop_chunks` the same way. The +/// Observe page works without these indexes (queries are time-bounded +/// and chunk pruning handles them); they're a pure performance +/// optimization to be added later via a separate operational tool when +/// the cluster is quiet. +/// +/// * **No job pausing / `LOCK TABLE`**. With nothing in this migration +/// enumerating chunks, there's no race surface left to defend. #[derive(DeriveMigrationName)] pub struct Migration; @@ -51,50 +49,14 @@ impl MigrationTrait for Migration { db.execute_unprepared( r#" DO $$ -DECLARE - has_timescaledb boolean := EXISTS ( - SELECT 1 FROM pg_extension WHERE extname = 'timescaledb' - ); - proxy_logs_is_hypertable boolean := false; - paused_job_ids integer[] := ARRAY[]::integer[]; BEGIN - -- ── proxy_logs (potentially compressed hypertable) ────────────── - IF has_timescaledb THEN - SELECT EXISTS ( - SELECT 1 FROM timescaledb_information.hypertables - WHERE hypertable_name = 'proxy_logs' - ) INTO proxy_logs_is_hypertable; - - IF proxy_logs_is_hypertable THEN - -- 1. Pause every active background job on this hypertable - -- (compression policy + retention policy). Snapshot the - -- list so we restore exactly what was active. - SELECT array_agg(job_id::integer) INTO paused_job_ids - FROM timescaledb_information.jobs - WHERE hypertable_name = 'proxy_logs' AND scheduled = true; - - IF paused_job_ids IS NOT NULL THEN - PERFORM alter_job(j::integer, scheduled => false) - FROM unnest(paused_job_ids) j; - END IF; - - -- 2. Take an exclusive lock on the hypertable. Blocks any - -- concurrent DDL/DML and waits for any in-flight job - -- to finish. The lock is held until this DO block (and - -- the surrounding migration transaction) ends. - LOCK TABLE proxy_logs IN ACCESS EXCLUSIVE MODE; - - -- 3. Decompress every chunk so per-chunk ALTERs succeed. - -- `if_compressed => TRUE` makes this a no-op for already- - -- decompressed chunks (idempotent on partial-prior-runs). - PERFORM decompress_chunk(c, if_compressed => TRUE) - FROM show_chunks('proxy_logs') c; - END IF; - END IF; - -- Idempotent column additions. `IF NOT EXISTS` covers the partial- - -- prior-run case where v1 of this migration added `trace_id` but - -- failed on `error_group_id` (or vice versa). + -- prior-run case where v1/v2/v3 of this migration added some columns + -- but failed before recording success in seaql_migrations. + -- + -- Timescale handles `ADD COLUMN` on hypertables transparently: the + -- new column appears on every chunk (current and future) atomically, + -- with no per-chunk DDL needed from us. ALTER TABLE proxy_logs ADD COLUMN IF NOT EXISTS trace_id text; ALTER TABLE proxy_logs ADD COLUMN IF NOT EXISTS error_group_id integer; @@ -104,24 +66,11 @@ BEGIN ALTER TABLE error_events ADD COLUMN IF NOT EXISTS trace_id_indexed text; - -- Indexes (also idempotent). - CREATE INDEX IF NOT EXISTS idx_proxy_logs_project_trace - ON proxy_logs (project_id, trace_id); - CREATE INDEX IF NOT EXISTS idx_proxy_logs_error_group - ON proxy_logs (error_group_id); - CREATE INDEX IF NOT EXISTS idx_revenue_events_project_occurred - ON revenue_events (project_id, occurred_at); + -- Index only on the regular table. Hypertable indexes are skipped + -- here to avoid racing with TimescaleDB's background retention + -- worker (see header doc comment). CREATE INDEX IF NOT EXISTS idx_error_events_project_trace ON error_events (project_id, trace_id_indexed); - - -- 4. Restore the jobs we paused. Done unconditionally — if the - -- array is empty (fresh install / no jobs), unnest produces zero - -- rows and PERFORM is a no-op. - IF has_timescaledb AND proxy_logs_is_hypertable - AND paused_job_ids IS NOT NULL THEN - PERFORM alter_job(j::integer, scheduled => true) - FROM unnest(paused_job_ids) j; - END IF; END $$; "#, @@ -134,38 +83,13 @@ $$; async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { let db = manager.get_connection(); + // Mirror image. `DROP INDEX IF EXISTS` for the hypertable indexes + // is harmless even on installs that never had them — covers + // operators who created them manually. db.execute_unprepared( r#" DO $$ -DECLARE - has_timescaledb boolean := EXISTS ( - SELECT 1 FROM pg_extension WHERE extname = 'timescaledb' - ); - proxy_logs_is_hypertable boolean := false; - paused_job_ids integer[] := ARRAY[]::integer[]; BEGIN - IF has_timescaledb THEN - SELECT EXISTS ( - SELECT 1 FROM timescaledb_information.hypertables - WHERE hypertable_name = 'proxy_logs' - ) INTO proxy_logs_is_hypertable; - - IF proxy_logs_is_hypertable THEN - SELECT array_agg(job_id::integer) INTO paused_job_ids - FROM timescaledb_information.jobs - WHERE hypertable_name = 'proxy_logs' AND scheduled = true; - - IF paused_job_ids IS NOT NULL THEN - PERFORM alter_job(j::integer, scheduled => false) - FROM unnest(paused_job_ids) j; - END IF; - - LOCK TABLE proxy_logs IN ACCESS EXCLUSIVE MODE; - PERFORM decompress_chunk(c, if_compressed => TRUE) - FROM show_chunks('proxy_logs') c; - END IF; - END IF; - DROP INDEX IF EXISTS idx_error_events_project_trace; ALTER TABLE error_events DROP COLUMN IF EXISTS trace_id_indexed; @@ -178,12 +102,6 @@ BEGIN DROP INDEX IF EXISTS idx_proxy_logs_project_trace; ALTER TABLE proxy_logs DROP COLUMN IF EXISTS error_group_id; ALTER TABLE proxy_logs DROP COLUMN IF EXISTS trace_id; - - IF has_timescaledb AND proxy_logs_is_hypertable - AND paused_job_ids IS NOT NULL THEN - PERFORM alter_job(j::integer, scheduled => true) - FROM unnest(paused_job_ids) j; - END IF; END $$; "#, From dca3d135cda1f38febf2576eb985d21b57aa5991 Mon Sep 17 00:00:00 2001 From: David Viejo Date: Sat, 2 May 2026 13:18:32 +0200 Subject: [PATCH 10/12] =?UTF-8?q?fix(migrations):=20drop=20CREATE=20INDEX?= =?UTF-8?q?=20from=20Observe=20migration=20=E2=80=94=20error=5Fevents=20is?= =?UTF-8?q?=20a=20hypertable=20too?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prod log finally pinpointed the failing statement: ERROR: chunk not found DETAIL: schema_name: _timescaledb_internal, table_name: _hyper_3_1013_chunk CONTEXT: SQL statement "CREATE INDEX IF NOT EXISTS idx_error_events_project_trace ON error_events (project_id, trace_id_indexed)" `_hyper_3_*` = hypertable id 3 = `error_events`. I had assumed it was a regular Postgres table; it isn't. So `CREATE INDEX ON error_events` enumerates chunks the same way `proxy_logs` and `revenue_events` do, and races with `drop_chunks` from the retention policy → migration crash-loop. Drop the index entirely. The Observe page works without it: error joins go via `(project_id, occurred_at)` (already indexed by the hypertable chunk dimension) plus `trace_id_indexed` as a filter. Chunk pruning keeps it fast enough. Index creation is now an out-of-band operational task to run when the cluster is quiet — *not* something a startup migration should attempt. Migration body is now 6 `ALTER TABLE ... ADD COLUMN IF NOT EXISTS` statements and nothing else. Zero chunk-touching SQL. --- ...20260502_000001_add_observe_correlation.rs | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs b/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs index 64293bf04..525392be5 100644 --- a/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs +++ b/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs @@ -28,13 +28,17 @@ use sea_orm_migration::prelude::*; /// Timescale versions; the new column is added to all chunks /// atomically. /// -/// * **No `CREATE INDEX ON proxy_logs / revenue_events`**. `CREATE INDEX` -/// on a hypertable enumerates every chunk to build per-chunk indexes, -/// and that enumeration races with `drop_chunks` the same way. The -/// Observe page works without these indexes (queries are time-bounded -/// and chunk pruning handles them); they're a pure performance -/// optimization to be added later via a separate operational tool when -/// the cluster is quiet. +/// * **No `CREATE INDEX` on any hypertable** — `proxy_logs`, +/// `revenue_events`, **and `error_events`** are all hypertables on +/// prod (`error_events` is hypertable id 3, confirmed in prod logs: +/// `chunk not found … _hyper_3_1013_chunk` while running +/// `CREATE INDEX … ON error_events`). `CREATE INDEX` on a hypertable +/// enumerates every chunk to build per-chunk indexes, which races +/// with `drop_chunks` from the retention policy. The Observe page +/// works without these indexes — queries are time-bounded and +/// Timescale's chunk pruning handles them. Add the indexes later via +/// a separate operational tool when the cluster is quiet, *not* via +/// a startup migration that crash-loops the server. /// /// * **No job pausing / `LOCK TABLE`**. With nothing in this migration /// enumerating chunks, there's no race surface left to defend. @@ -66,11 +70,10 @@ BEGIN ALTER TABLE error_events ADD COLUMN IF NOT EXISTS trace_id_indexed text; - -- Index only on the regular table. Hypertable indexes are skipped - -- here to avoid racing with TimescaleDB's background retention - -- worker (see header doc comment). - CREATE INDEX IF NOT EXISTS idx_error_events_project_trace - ON error_events (project_id, trace_id_indexed); + -- No CREATE INDEX statements. `error_events` is a hypertable + -- (`_hyper_3_*`), so any CREATE INDEX against it enumerates chunks + -- and races with `drop_chunks`. Index creation is deferred to a + -- separate operational tool — see header doc comment. END $$; "#, @@ -90,6 +93,9 @@ $$; r#" DO $$ BEGIN + -- `DROP INDEX IF EXISTS` is harmless if the index was never created + -- (the up() in this version doesn't create it), but covers operators + -- who created it manually or via earlier failed migration runs. DROP INDEX IF EXISTS idx_error_events_project_trace; ALTER TABLE error_events DROP COLUMN IF EXISTS trace_id_indexed; From fce443893db24155350560ef6d4a025c2abe97d4 Mon Sep 17 00:00:00 2001 From: David Viejo Date: Sat, 2 May 2026 17:53:31 +0200 Subject: [PATCH 11/12] fix(migrations): add Observe-page indexes back to m20260502_000001 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After repairing the prod orphan-chunk drift (caused by a prior `pg_dump`/`pg_restore` run without `timescaledb_pre_restore()`), the four hypertable indexes can be created cleanly. Putting them back into the correlation migration with `IF NOT EXISTS` so: - prod is a no-op (indexes already exist from manual operational fix) - fresh installs / dev / staging build them as part of normal migration The four indexes: - idx_error_events_project_trace (project_id, trace_id_indexed) - idx_proxy_logs_project_trace (project_id, trace_id) - idx_proxy_logs_error_group (error_group_id) WHERE NOT NULL - idx_revenue_events_project_occurred (project_id, occurred_at DESC) Header comment updated to document the orphan-chunk failure mode any operator might hit if they migrated their DB via raw `pg_dump`/`pg_restore` without the Timescale-aware wrappers — with a pointer that the cleanup is operational (manual detach + drop), not migration-level, because we will not silently delete user data from a startup migration. --- ...20260502_000001_add_observe_correlation.rs | 81 +++++++++---------- 1 file changed, 37 insertions(+), 44 deletions(-) diff --git a/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs b/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs index 525392be5..e9cb58ac7 100644 --- a/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs +++ b/crates/temps-migrations/src/migration/m20260502_000001_add_observe_correlation.rs @@ -1,8 +1,9 @@ use sea_orm_migration::prelude::*; -/// Add cross-source correlation columns so the unified Observe page can join -/// requests / spans / errors / revenue without follow-up queries. All -/// columns are nullable; old rows simply render without correlation links. +/// Add cross-source correlation columns + lookup indexes so the unified +/// Observe page can join requests / spans / errors / revenue without +/// follow-up queries. All columns are nullable; old rows simply render +/// without correlation links. /// /// **Designed to be safely re-runnable on any prior state**, including: /// * Fresh installs (no rows yet, extension may not be installed) @@ -10,38 +11,24 @@ use sea_orm_migration::prelude::*; /// * Already-fully-applied installs (re-runs are no-ops) /// * Installs without TimescaleDB /// -/// Strategy: every step uses `IF NOT EXISTS` and is run via raw SQL. The -/// `ALTER TABLE ... ADD COLUMN IF NOT EXISTS` form has been supported since -/// Postgres 9.6 and works correctly on TimescaleDB hypertables — Timescale -/// propagates the new column to every chunk transparently, no per-chunk -/// enumeration required from us. +/// Strategy: every step uses `IF NOT EXISTS` and is run via raw SQL. +/// `ALTER TABLE ... ADD COLUMN IF NOT EXISTS` is propagated to every +/// hypertable chunk atomically by Timescale; `CREATE INDEX IF NOT EXISTS` +/// is per-chunk and safe on a healthy hypertable (see operational note +/// below for the one prod scenario where it isn't). /// -/// **What this deliberately does NOT do**: -/// -/// * **No `decompress_chunk` loop**. We previously paused the compression -/// policy and decompressed every chunk before ALTER, on the assumption -/// that ALTER on a compressed hypertable would fail. In practice the -/// decompress loop itself raced with the background retention worker -/// (`drop_chunks` runs daily on `proxy_logs` past 30 days) and produced -/// `chunk not found` — confirmed in prod logs. `ALTER TABLE ... ADD -/// COLUMN` against a compressed hypertable works fine in current -/// Timescale versions; the new column is added to all chunks -/// atomically. -/// -/// * **No `CREATE INDEX` on any hypertable** — `proxy_logs`, -/// `revenue_events`, **and `error_events`** are all hypertables on -/// prod (`error_events` is hypertable id 3, confirmed in prod logs: -/// `chunk not found … _hyper_3_1013_chunk` while running -/// `CREATE INDEX … ON error_events`). `CREATE INDEX` on a hypertable -/// enumerates every chunk to build per-chunk indexes, which races -/// with `drop_chunks` from the retention policy. The Observe page -/// works without these indexes — queries are time-bounded and -/// Timescale's chunk pruning handles them. Add the indexes later via -/// a separate operational tool when the cluster is quiet, *not* via -/// a startup migration that crash-loops the server. -/// -/// * **No job pausing / `LOCK TABLE`**. With nothing in this migration -/// enumerating chunks, there's no race surface left to defend. +/// **Operational note for installs migrated via raw `pg_dump`/`pg_restore`**: +/// if the source database was migrated to a new server *without* wrapping +/// the restore in `timescaledb_pre_restore()` / `timescaledb_post_restore()`, +/// the new database can have orphan chunks — relations attached via +/// `pg_inherits` but missing from `_timescaledb_catalog.chunk`. `CREATE +/// INDEX` on the parent hypertable will then fail with `chunk not found` +/// because Postgres tries to build the per-chunk index on a chunk Timescale +/// no longer knows about. The fix is operational, not migration-level: +/// detach + drop the orphan child relations, then re-run. See +/// `docs/runbooks/timescaledb-orphan-chunks.md` (or git history of this +/// migration) for the cleanup script. We do not attempt that cleanup from +/// inside the migration because it would silently delete user data. #[derive(DeriveMigrationName)] pub struct Migration; @@ -70,10 +57,22 @@ BEGIN ALTER TABLE error_events ADD COLUMN IF NOT EXISTS trace_id_indexed text; - -- No CREATE INDEX statements. `error_events` is a hypertable - -- (`_hyper_3_*`), so any CREATE INDEX against it enumerates chunks - -- and races with `drop_chunks`. Index creation is deferred to a - -- separate operational tool — see header doc comment. + -- Lookup indexes for the Observe page's cross-source correlation. + -- `IF NOT EXISTS` makes this safe on installs where the index was + -- already created manually (e.g. prod, where these were built + -- out-of-band after the orphan-chunk cleanup — see header comment). + CREATE INDEX IF NOT EXISTS idx_error_events_project_trace + ON error_events (project_id, trace_id_indexed); + + CREATE INDEX IF NOT EXISTS idx_proxy_logs_project_trace + ON proxy_logs (project_id, trace_id); + + CREATE INDEX IF NOT EXISTS idx_proxy_logs_error_group + ON proxy_logs (error_group_id) + WHERE error_group_id IS NOT NULL; + + CREATE INDEX IF NOT EXISTS idx_revenue_events_project_occurred + ON revenue_events (project_id, occurred_at DESC); END $$; "#, @@ -86,16 +85,10 @@ $$; async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { let db = manager.get_connection(); - // Mirror image. `DROP INDEX IF EXISTS` for the hypertable indexes - // is harmless even on installs that never had them — covers - // operators who created them manually. db.execute_unprepared( r#" DO $$ BEGIN - -- `DROP INDEX IF EXISTS` is harmless if the index was never created - -- (the up() in this version doesn't create it), but covers operators - -- who created it manually or via earlier failed migration runs. DROP INDEX IF EXISTS idx_error_events_project_trace; ALTER TABLE error_events DROP COLUMN IF EXISTS trace_id_indexed; From 795313d84f68547940b6296969cab997a8f131a7 Mon Sep 17 00:00:00 2001 From: David Viejo Date: Sun, 3 May 2026 00:43:01 +0200 Subject: [PATCH 12/12] fix(agents): add trace_id_indexed to error_events test fixture The error_events entity gained a denormalized trace_id_indexed column for the Observe merge query; the make_error_event test helper was missing the field, breaking temps-agents lib test compilation. Also applies cargo fmt to migration_tests.rs. --- crates/temps-agents/src/services/executor.rs | 1 + crates/temps-migrations/tests/migration_tests.rs | 10 +++------- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/crates/temps-agents/src/services/executor.rs b/crates/temps-agents/src/services/executor.rs index 2e6173361..9298e9114 100644 --- a/crates/temps-agents/src/services/executor.rs +++ b/crates/temps-agents/src/services/executor.rs @@ -4002,6 +4002,7 @@ mod tests { } ] })), + trace_id_indexed: None, created_at: Utc::now(), } } diff --git a/crates/temps-migrations/tests/migration_tests.rs b/crates/temps-migrations/tests/migration_tests.rs index c29b0bd4c..58293ed3d 100644 --- a/crates/temps-migrations/tests/migration_tests.rs +++ b/crates/temps-migrations/tests/migration_tests.rs @@ -971,8 +971,7 @@ async fn connect_with_retries(db_url: &str) -> anyhow::Result anyhow::Result<()> { +async fn test_observe_correlation_migration_handles_compressed_proxy_logs() -> anyhow::Result<()> { if std::env::var("TEMPS_TEST_DATABASE_URL").is_ok() { println!( "⏭️ Skipping test_observe_correlation_migration_handles_compressed_proxy_logs: \ @@ -1234,8 +1233,7 @@ async fn test_observe_correlation_migration_is_idempotent() -> anyhow::Result<() // To verify this test catches the regression, replace the v4 migration's // `alter_job(scheduled => false)` with a no-op — the test must then fail. #[tokio::test] -async fn test_observe_correlation_migration_survives_concurrent_retention( -) -> anyhow::Result<()> { +async fn test_observe_correlation_migration_survives_concurrent_retention() -> anyhow::Result<()> { if std::env::var("TEMPS_TEST_DATABASE_URL").is_ok() { println!( "⏭️ Skipping test_observe_correlation_migration_survives_concurrent_retention: \ @@ -1334,9 +1332,7 @@ async fn test_observe_correlation_migration_survives_concurrent_retention( // fail when the migration has it locked or when the job_id was // briefly removed during alter_job. We're only here to maximize // the chance of a race. - let _ = bg_db - .execute_unprepared(&call_sql) - .await; + let _ = bg_db.execute_unprepared(&call_sql).await; tokio::time::sleep(tokio::time::Duration::from_millis(20)).await; } Ok::<(), anyhow::Error>(())