diff --git a/.changeset/metrics-trace-exemplars.md b/.changeset/metrics-trace-exemplars.md new file mode 100644 index 0000000000..ff2c9dd326 --- /dev/null +++ b/.changeset/metrics-trace-exemplars.md @@ -0,0 +1,7 @@ +--- +'@posthog/core': minor +'@posthog/types': minor +'posthog-node': minor +--- + +Attach trace exemplars to metrics recorded inside sampled spans. diff --git a/packages/browser/terser-mangled-names.json b/packages/browser/terser-mangled-names.json index 66e931351b..17360dec22 100644 --- a/packages/browser/terser-mangled-names.json +++ b/packages/browser/terser-mangled-names.json @@ -12,6 +12,7 @@ "_activeMatchingSurveyConditionsUnsubscribe", "_activeReloads", "_activeTour", + "_activeTraceContext", "_addActionHook", "_addBufferedExceptionSteps", "_addDomEventHandlers", diff --git a/packages/core/src/metrics/index.spec.ts b/packages/core/src/metrics/index.spec.ts index feabff47b3..8eb1ce5983 100644 --- a/packages/core/src/metrics/index.spec.ts +++ b/packages/core/src/metrics/index.spec.ts @@ -203,6 +203,81 @@ describe('PostHogMetrics', () => { }) }) + describe('exemplars', () => { + const TRACE_ID = '4bf92f3577b34da6a3ce929d0e0e4736' + const SPAN_ID = '00f067aa0ba902b7' + const withActiveSpan = (traceparent: string | null): any => + createMockInstance({ getActiveSpan: () => ({ traceparent: () => traceparent }) }) + + const dataPoints = (instance: any): any[] => + sentMetrics(instance).map((m) => (m.sum ?? m.gauge ?? m.histogram)!.dataPoints[0]) + + it('stamps every metric type with the last sample recorded inside a sampled span', async () => { + const instance = withActiveSpan(`00-${TRACE_ID}-${SPAN_ID}-01`) + const metrics = createMetrics(undefined, instance) + metrics.count('c', 1) + metrics.count('c', 4) + metrics.gauge('g', 7) + metrics.histogram('h', 12) + await metrics.flush() + + const points = dataPoints(instance) + expect(points.map((p) => p.exemplars)).toEqual( + [4, 7, 12].map((value) => [ + { + filteredAttributes: [], + timeUnixNano: expect.stringMatching(/^\d+$/), + asDouble: value, + traceId: TRACE_ID, + spanId: SPAN_ID, + }, + ]) + ) + }) + + it.each([ + ['no getActiveSpan on the host', createMockInstance], + ['no active span', () => createMockInstance({ getActiveSpan: () => null })], + ['an unsampled span', () => withActiveSpan(`00-${TRACE_ID}-${SPAN_ID}-00`)], + ['a span with no traceparent', () => withActiveSpan(null)], + [ + 'a throwing host', + () => + createMockInstance({ + getActiveSpan: () => { + throw new Error('boom') + }, + }), + ], + ])('records the sample without an exemplar for %s', async (_, makeInstance) => { + const instance = makeInstance() + const metrics = createMetrics(undefined, instance) + metrics.count('c', 2) + await metrics.flush() + + const [point] = dataPoints(instance) + expect(point.asDouble).toBe(2) + expect(point).not.toHaveProperty('exemplars') + }) + + it('keeps the exemplar of a window merged back after a failed flush', async () => { + const instance = withActiveSpan(`00-${TRACE_ID}-${SPAN_ID}-01`) + instance._sendMetricsBatch + .mockResolvedValueOnce({ kind: 'retry-later', error: new Error('503') }) + .mockResolvedValue({ kind: 'ok' }) + const metrics = createMetrics(undefined, instance) + metrics.count('c', 1) + await metrics.flush() + instance.getActiveSpan = () => null + metrics.count('c', 2) + await metrics.flush() + + const point = dataPoints(instance)[1] + expect(point.asDouble).toBe(3) + expect(point.exemplars[0]).toMatchObject({ traceId: TRACE_ID, spanId: SPAN_ID, asDouble: 1 }) + }) + }) + describe('resource attributes', () => { it('attaches service.name, sdk identity, and user resource attributes', async () => { const metrics = createMetrics({ diff --git a/packages/core/src/metrics/index.ts b/packages/core/src/metrics/index.ts index 8043ccf054..408b072529 100644 --- a/packages/core/src/metrics/index.ts +++ b/packages/core/src/metrics/index.ts @@ -3,6 +3,7 @@ import type { MetricAttributes, MetricSample, MetricType, + OtlpExemplar, OtlpHistogramDataPoint, OtlpMetric, OtlpMetricsPayload, @@ -14,6 +15,7 @@ import { FlushTimer } from '../utils/flush-timer' import { RetryAfterWindow } from '../utils/retry-after' import { MAX_FLUSH_BACKOFF_MS, NO_JITTER, backoffDelayMs, drawJitter } from '../utils/backoff' import { toOtlpKeyValueList } from '../utils/otlp-any-value' +import { TRACE_FLAGS_SAMPLED, parseTraceparent } from '../traces/traceparent' import { DEFAULT_HISTOGRAM_BOUNDS, bucketIndexFor, @@ -26,6 +28,13 @@ import type { MetricsHost, ResolvedPostHogMetricsConfig } from './types' const OTLP_TEMPORALITY_DELTA = 1 +interface ExemplarState { + traceId: string + spanId: string + value: number + timeMs: number +} + interface HistogramState { count: number sum: number @@ -47,6 +56,8 @@ interface SeriesState { total?: number last?: number hist?: HistogramState + /** The last sample recorded inside a sampled span this window. */ + exemplar?: ExemplarState } /** @@ -62,6 +73,10 @@ interface SeriesState { * Deliberately unlike logs, no per-user context (distinct ID, session ID) is * attached: every attribute value creates a new series, and per-user series * are the canonical metrics-cardinality explosion. + * + * A sample recorded inside a sampled span stamps its series with an exemplar + * (trace ID, span ID, value). One per series per window, last sample wins, so a + * metric point links to one representative trace at no cardinality cost. */ export class PostHogMetrics { private _series = new Map() @@ -224,9 +239,29 @@ export class PostHogMetrics { } this._fold(state, filtered.value) + const traceContext = this._activeTraceContext() + if (traceContext) { + state.exemplar = { ...traceContext, value: filtered.value, timeMs: Date.now() } + } this._armFlushTimer() } + /** + * The sampled span a sample is recorded in. An unsampled span is never + * exported, so an exemplar pointing at it would link to a missing trace. + */ + private _activeTraceContext(): { traceId: string; spanId: string } | undefined { + try { + const parsed = parseTraceparent(this._instance.getActiveSpan?.()?.traceparent()) + if (parsed && parsed.flags === TRACE_FLAGS_SAMPLED) { + return { traceId: parsed.traceId, spanId: parsed.spanId } + } + } catch { + // A throwing host never costs the sample itself. + } + return undefined + } + /** * Cardinality gate for adding a series to the live window — warns once per * window when the cap is hit. Applied on capture and on merge-back alike. @@ -413,6 +448,7 @@ export class PostHogMetrics { const attributes = toOtlpKeyValueList(state.attributes ?? {}, this._logger) const startNano = msToUnixNano(state.windowStartMs) + const exemplars = toOtlpExemplars(state.exemplar) if (state.type === 'count') { const dp: OtlpNumberDataPoint = { @@ -420,6 +456,7 @@ export class PostHogMetrics { startTimeUnixNano: startNano, timeUnixNano: nowNano, asDouble: state.total ?? 0, + ...exemplars, } metric.sum!.dataPoints.push(dp) } else if (state.type === 'gauge') { @@ -427,6 +464,7 @@ export class PostHogMetrics { attributes, timeUnixNano: nowNano, asDouble: state.last ?? 0, + ...exemplars, } metric.gauge!.dataPoints.push(dp) } else if (state.hist) { @@ -440,6 +478,7 @@ export class PostHogMetrics { max: state.hist.max, bucketCounts: state.hist.bucketCounts, explicitBounds: DEFAULT_HISTOGRAM_BOUNDS, + ...exemplars, } metric.histogram!.dataPoints.push(dp) } @@ -461,6 +500,8 @@ export class PostHogMetrics { continue } current.windowStartMs = Math.min(current.windowStartMs, old.windowStartMs) + // The live window's exemplar is newer; the old one only fills a gap. + current.exemplar = current.exemplar ?? old.exemplar switch (current.type) { case 'count': current.total = (current.total ?? 0) + (old.total ?? 0) @@ -488,6 +529,25 @@ export class PostHogMetrics { } } +// Spread into the data point, so a point without an exemplar keeps its exact +// pre-exemplar wire shape. +function toOtlpExemplars(exemplar: ExemplarState | undefined): { exemplars?: OtlpExemplar[] } { + if (!exemplar) { + return {} + } + return { + exemplars: [ + { + filteredAttributes: [], + timeUnixNano: msToUnixNano(exemplar.timeMs), + asDouble: exemplar.value, + traceId: exemplar.traceId, + spanId: exemplar.spanId, + }, + ], + } +} + export { buildOtlpMetricsPayload, buildMetricsResourceAttributes, DEFAULT_HISTOGRAM_BOUNDS } from './metrics-utils' export { resolveMetricsConfig } from './config' export type { MetricsHost, PostHogMetricsConfig, ResolvedPostHogMetricsConfig, SendMetricsBatchOutcome } from './types' diff --git a/packages/core/src/metrics/types.ts b/packages/core/src/metrics/types.ts index d5bbc28668..921a8f61fb 100644 --- a/packages/core/src/metrics/types.ts +++ b/packages/core/src/metrics/types.ts @@ -13,9 +13,10 @@ export type { OtlpHistogramDataPoint, OtlpMetric, OtlpMetricsPayload, + OtlpExemplar, } from '@posthog/types' -import type { BeforeSendMetricFn, MetricAttributeValue, OtlpMetricsPayload } from '@posthog/types' +import type { BeforeSendMetricFn, MetricAttributeValue, OtlpMetricsPayload, Span } from '@posthog/types' /** Same tagged outcome shape as `SendLogsBatchOutcome` — one policy for both signals. */ export type SendMetricsBatchOutcome = @@ -43,6 +44,11 @@ export interface MetricsHost { _sendMetricsBatch(payload: OtlpMetricsPayload): Promise getLibraryId(): string getLibraryVersion(): string + /** + * The span active where a sample is recorded. A sampled one stamps the + * sample's series with an exemplar. Hosts without tracing leave it out. + */ + getActiveSpan?(): Span | null } /** diff --git a/packages/core/src/traces/ids.ts b/packages/core/src/traces/ids.ts index 52a445bf5a..3ded9dd016 100644 --- a/packages/core/src/traces/ids.ts +++ b/packages/core/src/traces/ids.ts @@ -9,8 +9,10 @@ const SPAN_ID_BYTES = 8 const TRACE_ID_HEX = TRACE_ID_BYTES * 2 const SPAN_ID_HEX = SPAN_ID_BYTES * 2 -const INVALID_TRACE_ID = '0'.repeat(TRACE_ID_HEX) -const INVALID_SPAN_ID = '0'.repeat(SPAN_ID_HEX) +// Spelled out rather than `'0'.repeat(...)`: this runs at module load and the +// browser bundle, which reaches it through metrics, still loads in IE11. +const INVALID_SPAN_ID = '0000000000000000' +const INVALID_TRACE_ID = INVALID_SPAN_ID + INVALID_SPAN_ID const HEX_RE = /^[0-9a-f]+$/ diff --git a/packages/node/src/__tests__/metrics.spec.ts b/packages/node/src/__tests__/metrics.spec.ts index a60c06d056..8c31c2e2a9 100644 --- a/packages/node/src/__tests__/metrics.spec.ts +++ b/packages/node/src/__tests__/metrics.spec.ts @@ -63,6 +63,21 @@ describe('PostHog Node.js metrics', () => { expect(metrics['job.duration'].histogram.dataPoints[0].count).toBe(1) }) + it('links a sample recorded inside withSpan to that span through an exemplar', async () => { + const traced = new PostHog('TEST_API_KEY', { ...options, traces: { serviceName: 'test-service' } }) + let traceparent: string | null | undefined + traced.withSpan('job', () => { + traced.metrics.histogram('job.duration', 42, { unit: 'ms' }) + traceparent = traced.getActiveSpan()?.traceparent() + }) + await traced.metrics.flush() + + const [, traceId, spanId] = traceparent!.split('-') + const point = metricsByName(lastMetricsBody())['job.duration'].histogram.dataPoints[0] + expect(point.exemplars).toEqual([expect.objectContaining({ traceId, spanId, asDouble: 42 })]) + await traced.shutdown() + }) + it('merges the window back and resends on network failure', async () => { mockedFetch.mockRejectedValueOnce(new Error('connection refused')) diff --git a/packages/types/src/capture-metric.ts b/packages/types/src/capture-metric.ts index 6431c90379..58419dec2f 100644 --- a/packages/types/src/capture-metric.ts +++ b/packages/types/src/capture-metric.ts @@ -81,12 +81,22 @@ export interface Metrics { // OTLP wire format types (ExportMetricsServiceRequest, JSON encoding) // ============================================================================ +/** A sampled measurement linked to the span it was recorded in. IDs are lowercase hex. */ +export interface OtlpExemplar { + filteredAttributes: OtlpKeyValue[] + timeUnixNano: string + asDouble: number + traceId: string + spanId: string +} + export interface OtlpNumberDataPoint { attributes: OtlpKeyValue[] /** Unix nanos as a decimal string (uint64 doesn't fit in JS Number). */ timeUnixNano: string startTimeUnixNano?: string asDouble: number + exemplars?: OtlpExemplar[] } export interface OtlpHistogramDataPoint { @@ -100,6 +110,7 @@ export interface OtlpHistogramDataPoint { max: number bucketCounts: number[] explicitBounds: number[] + exemplars?: OtlpExemplar[] } /** AGGREGATION_TEMPORALITY_DELTA — each data point covers one flush window. */ diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 5127f149bd..dd2c5095cd 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -136,6 +136,7 @@ export type { BeforeSendMetricFn, Metrics, OtlpNumberDataPoint, + OtlpExemplar, OtlpHistogramDataPoint, OtlpMetric, OtlpMetricsPayload,