Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changeset/metrics-trace-exemplars.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
'@posthog/core': minor
'@posthog/types': minor
'posthog-node': minor
---

Attach trace exemplars to metrics recorded inside sampled spans.
1 change: 1 addition & 0 deletions packages/browser/terser-mangled-names.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
"_activeMatchingSurveyConditionsUnsubscribe",
"_activeReloads",
"_activeTour",
"_activeTraceContext",
"_addActionHook",
"_addBufferedExceptionSteps",
"_addDomEventHandlers",
Expand Down
75 changes: 75 additions & 0 deletions packages/core/src/metrics/index.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,81 @@ describe('PostHogMetrics', () => {
})
})

describe('exemplars', () => {
const TRACE_ID = '4bf92f3577b34da6a3ce929d0e0e4736'
const SPAN_ID = '00f067aa0ba902b7'
const withActiveSpan = (traceparent: string | null): any =>
createMockInstance({ getActiveSpan: () => ({ traceparent: () => traceparent }) })

const dataPoints = (instance: any): any[] =>
sentMetrics(instance).map((m) => (m.sum ?? m.gauge ?? m.histogram)!.dataPoints[0])

it('stamps every metric type with the last sample recorded inside a sampled span', async () => {
const instance = withActiveSpan(`00-${TRACE_ID}-${SPAN_ID}-01`)
const metrics = createMetrics(undefined, instance)
metrics.count('c', 1)
metrics.count('c', 4)
metrics.gauge('g', 7)
metrics.histogram('h', 12)
await metrics.flush()

const points = dataPoints(instance)
expect(points.map((p) => p.exemplars)).toEqual(
[4, 7, 12].map((value) => [
{
filteredAttributes: [],
timeUnixNano: expect.stringMatching(/^\d+$/),
asDouble: value,
traceId: TRACE_ID,
spanId: SPAN_ID,
},
])
)
})

it.each([
['no getActiveSpan on the host', createMockInstance],
['no active span', () => createMockInstance({ getActiveSpan: () => null })],
['an unsampled span', () => withActiveSpan(`00-${TRACE_ID}-${SPAN_ID}-00`)],
['a span with no traceparent', () => withActiveSpan(null)],
[
'a throwing host',
() =>
createMockInstance({
getActiveSpan: () => {
throw new Error('boom')
},
}),
],
])('records the sample without an exemplar for %s', async (_, makeInstance) => {
const instance = makeInstance()
const metrics = createMetrics(undefined, instance)
metrics.count('c', 2)
await metrics.flush()

const [point] = dataPoints(instance)
expect(point.asDouble).toBe(2)
expect(point).not.toHaveProperty('exemplars')
})

it('keeps the exemplar of a window merged back after a failed flush', async () => {
const instance = withActiveSpan(`00-${TRACE_ID}-${SPAN_ID}-01`)
instance._sendMetricsBatch
.mockResolvedValueOnce({ kind: 'retry-later', error: new Error('503') })
.mockResolvedValue({ kind: 'ok' })
const metrics = createMetrics(undefined, instance)
metrics.count('c', 1)
await metrics.flush()
instance.getActiveSpan = () => null
metrics.count('c', 2)
await metrics.flush()

const point = dataPoints(instance)[1]
expect(point.asDouble).toBe(3)
expect(point.exemplars[0]).toMatchObject({ traceId: TRACE_ID, spanId: SPAN_ID, asDouble: 1 })
})
})

describe('resource attributes', () => {
it('attaches service.name, sdk identity, and user resource attributes', async () => {
const metrics = createMetrics({
Expand Down
60 changes: 60 additions & 0 deletions packages/core/src/metrics/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import type {
MetricAttributes,
MetricSample,
MetricType,
OtlpExemplar,
OtlpHistogramDataPoint,
OtlpMetric,
OtlpMetricsPayload,
Expand All @@ -14,6 +15,7 @@ import { FlushTimer } from '../utils/flush-timer'
import { RetryAfterWindow } from '../utils/retry-after'
import { MAX_FLUSH_BACKOFF_MS, NO_JITTER, backoffDelayMs, drawJitter } from '../utils/backoff'
import { toOtlpKeyValueList } from '../utils/otlp-any-value'
import { TRACE_FLAGS_SAMPLED, parseTraceparent } from '../traces/traceparent'
import {
DEFAULT_HISTOGRAM_BOUNDS,
bucketIndexFor,
Expand All @@ -26,6 +28,13 @@ import type { MetricsHost, ResolvedPostHogMetricsConfig } from './types'

const OTLP_TEMPORALITY_DELTA = 1

interface ExemplarState {
traceId: string
spanId: string
value: number
timeMs: number
}

interface HistogramState {
count: number
sum: number
Expand All @@ -47,6 +56,8 @@ interface SeriesState {
total?: number
last?: number
hist?: HistogramState
/** The last sample recorded inside a sampled span this window. */
exemplar?: ExemplarState
}

/**
Expand All @@ -62,6 +73,10 @@ interface SeriesState {
* Deliberately unlike logs, no per-user context (distinct ID, session ID) is
* attached: every attribute value creates a new series, and per-user series
* are the canonical metrics-cardinality explosion.
*
* A sample recorded inside a sampled span stamps its series with an exemplar
* (trace ID, span ID, value). One per series per window, last sample wins, so a
* metric point links to one representative trace at no cardinality cost.
*/
export class PostHogMetrics {
private _series = new Map<string, SeriesState>()
Expand Down Expand Up @@ -224,9 +239,29 @@ export class PostHogMetrics {
}

this._fold(state, filtered.value)
const traceContext = this._activeTraceContext()
if (traceContext) {
state.exemplar = { ...traceContext, value: filtered.value, timeMs: Date.now() }
}
this._armFlushTimer()
}

/**
* The sampled span a sample is recorded in. An unsampled span is never
* exported, so an exemplar pointing at it would link to a missing trace.
*/
private _activeTraceContext(): { traceId: string; spanId: string } | undefined {
try {
const parsed = parseTraceparent(this._instance.getActiveSpan?.()?.traceparent())
if (parsed && parsed.flags === TRACE_FLAGS_SAMPLED) {
return { traceId: parsed.traceId, spanId: parsed.spanId }
}
} catch {
// A throwing host never costs the sample itself.
}
return undefined
}

/**
* Cardinality gate for adding a series to the live window — warns once per
* window when the cap is hit. Applied on capture and on merge-back alike.
Expand Down Expand Up @@ -413,20 +448,23 @@ export class PostHogMetrics {

const attributes = toOtlpKeyValueList(state.attributes ?? {}, this._logger)
const startNano = msToUnixNano(state.windowStartMs)
const exemplars = toOtlpExemplars(state.exemplar)

if (state.type === 'count') {
const dp: OtlpNumberDataPoint = {
attributes,
startTimeUnixNano: startNano,
timeUnixNano: nowNano,
asDouble: state.total ?? 0,
...exemplars,
}
metric.sum!.dataPoints.push(dp)
} else if (state.type === 'gauge') {
const dp: OtlpNumberDataPoint = {
attributes,
timeUnixNano: nowNano,
asDouble: state.last ?? 0,
...exemplars,
}
metric.gauge!.dataPoints.push(dp)
} else if (state.hist) {
Expand All @@ -440,6 +478,7 @@ export class PostHogMetrics {
max: state.hist.max,
bucketCounts: state.hist.bucketCounts,
explicitBounds: DEFAULT_HISTOGRAM_BOUNDS,
...exemplars,
}
metric.histogram!.dataPoints.push(dp)
}
Expand All @@ -461,6 +500,8 @@ export class PostHogMetrics {
continue
}
current.windowStartMs = Math.min(current.windowStartMs, old.windowStartMs)
// The live window's exemplar is newer; the old one only fills a gap.
current.exemplar = current.exemplar ?? old.exemplar
switch (current.type) {
case 'count':
current.total = (current.total ?? 0) + (old.total ?? 0)
Expand Down Expand Up @@ -488,6 +529,25 @@ export class PostHogMetrics {
}
}

// Spread into the data point, so a point without an exemplar keeps its exact
// pre-exemplar wire shape.
function toOtlpExemplars(exemplar: ExemplarState | undefined): { exemplars?: OtlpExemplar[] } {
if (!exemplar) {
return {}
}
return {
exemplars: [
{
filteredAttributes: [],
timeUnixNano: msToUnixNano(exemplar.timeMs),
asDouble: exemplar.value,
traceId: exemplar.traceId,
spanId: exemplar.spanId,
},
],
}
}

export { buildOtlpMetricsPayload, buildMetricsResourceAttributes, DEFAULT_HISTOGRAM_BOUNDS } from './metrics-utils'
export { resolveMetricsConfig } from './config'
export type { MetricsHost, PostHogMetricsConfig, ResolvedPostHogMetricsConfig, SendMetricsBatchOutcome } from './types'
8 changes: 7 additions & 1 deletion packages/core/src/metrics/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,10 @@ export type {
OtlpHistogramDataPoint,
OtlpMetric,
OtlpMetricsPayload,
OtlpExemplar,
} from '@posthog/types'

import type { BeforeSendMetricFn, MetricAttributeValue, OtlpMetricsPayload } from '@posthog/types'
import type { BeforeSendMetricFn, MetricAttributeValue, OtlpMetricsPayload, Span } from '@posthog/types'

/** Same tagged outcome shape as `SendLogsBatchOutcome` — one policy for both signals. */
export type SendMetricsBatchOutcome =
Expand Down Expand Up @@ -43,6 +44,11 @@ export interface MetricsHost {
_sendMetricsBatch(payload: OtlpMetricsPayload): Promise<SendMetricsBatchOutcome>
getLibraryId(): string
getLibraryVersion(): string
/**
* The span active where a sample is recorded. A sampled one stamps the
* sample's series with an exemplar. Hosts without tracing leave it out.
*/
getActiveSpan?(): Span | null
}

/**
Expand Down
6 changes: 4 additions & 2 deletions packages/core/src/traces/ids.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,10 @@ const SPAN_ID_BYTES = 8
const TRACE_ID_HEX = TRACE_ID_BYTES * 2
const SPAN_ID_HEX = SPAN_ID_BYTES * 2

const INVALID_TRACE_ID = '0'.repeat(TRACE_ID_HEX)
const INVALID_SPAN_ID = '0'.repeat(SPAN_ID_HEX)
// Spelled out rather than `'0'.repeat(...)`: this runs at module load and the
// browser bundle, which reaches it through metrics, still loads in IE11.
const INVALID_SPAN_ID = '0000000000000000'
const INVALID_TRACE_ID = INVALID_SPAN_ID + INVALID_SPAN_ID

const HEX_RE = /^[0-9a-f]+$/

Expand Down
15 changes: 15 additions & 0 deletions packages/node/src/__tests__/metrics.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,21 @@ describe('PostHog Node.js metrics', () => {
expect(metrics['job.duration'].histogram.dataPoints[0].count).toBe(1)
})

it('links a sample recorded inside withSpan to that span through an exemplar', async () => {
const traced = new PostHog('TEST_API_KEY', { ...options, traces: { serviceName: 'test-service' } })
let traceparent: string | null | undefined
traced.withSpan('job', () => {
traced.metrics.histogram('job.duration', 42, { unit: 'ms' })
traceparent = traced.getActiveSpan()?.traceparent()
})
await traced.metrics.flush()

const [, traceId, spanId] = traceparent!.split('-')
const point = metricsByName(lastMetricsBody())['job.duration'].histogram.dataPoints[0]
expect(point.exemplars).toEqual([expect.objectContaining({ traceId, spanId, asDouble: 42 })])
await traced.shutdown()
})

it('merges the window back and resends on network failure', async () => {
mockedFetch.mockRejectedValueOnce(new Error('connection refused'))

Expand Down
11 changes: 11 additions & 0 deletions packages/types/src/capture-metric.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,12 +81,22 @@ export interface Metrics {
// OTLP wire format types (ExportMetricsServiceRequest, JSON encoding)
// ============================================================================

/** A sampled measurement linked to the span it was recorded in. IDs are lowercase hex. */
export interface OtlpExemplar {
filteredAttributes: OtlpKeyValue[]
timeUnixNano: string
asDouble: number
traceId: string
spanId: string
}

export interface OtlpNumberDataPoint {
attributes: OtlpKeyValue[]
/** Unix nanos as a decimal string (uint64 doesn't fit in JS Number). */
timeUnixNano: string
startTimeUnixNano?: string
asDouble: number
exemplars?: OtlpExemplar[]
}

export interface OtlpHistogramDataPoint {
Expand All @@ -100,6 +110,7 @@ export interface OtlpHistogramDataPoint {
max: number
bucketCounts: number[]
explicitBounds: number[]
exemplars?: OtlpExemplar[]
}

/** AGGREGATION_TEMPORALITY_DELTA — each data point covers one flush window. */
Expand Down
1 change: 1 addition & 0 deletions packages/types/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ export type {
BeforeSendMetricFn,
Metrics,
OtlpNumberDataPoint,
OtlpExemplar,
OtlpHistogramDataPoint,
OtlpMetric,
OtlpMetricsPayload,
Expand Down
Loading