diff --git a/apps/gittensory-ui/src/lib/mcp-package.ts b/apps/gittensory-ui/src/lib/mcp-package.ts index 749d8fe434..684d56af4e 100644 --- a/apps/gittensory-ui/src/lib/mcp-package.ts +++ b/apps/gittensory-ui/src/lib/mcp-package.ts @@ -6,7 +6,7 @@ export const MCP_PACKAGE_NAME = "@jsonbored/gittensory-mcp"; export const MCP_PACKAGE_ENCODED_NAME = "@jsonbored%2fgittensory-mcp"; export const MCP_PACKAGE_REGISTRY_URL = `https://registry.npmjs.org/${MCP_PACKAGE_ENCODED_NAME}`; export const MCP_PACKAGE_NPM_URL = `https://www.npmjs.com/package/${MCP_PACKAGE_NAME}`; -export const MCP_PACKAGE_KNOWN_LATEST_VERSION = "0.3.0"; +export const MCP_PACKAGE_KNOWN_LATEST_VERSION = "0.4.0"; export const MCP_MINIMUM_SUPPORTED_VERSION = "0.2.0"; export type NpmPackageMetadata = { diff --git a/migrations/0017_product_usage_role_retention_rollups.sql b/migrations/0017_product_usage_role_retention_rollups.sql new file mode 100644 index 0000000000..a6fc4fc42d --- /dev/null +++ b/migrations/0017_product_usage_role_retention_rollups.sql @@ -0,0 +1,11 @@ +ALTER TABLE product_usage_daily_rollups + ADD COLUMN roles_json TEXT NOT NULL DEFAULT '[]'; + +ALTER TABLE product_usage_daily_rollups + ADD COLUMN activation_by_role_json TEXT NOT NULL DEFAULT '[]'; + +ALTER TABLE product_usage_daily_rollups + ADD COLUMN activation_by_surface_json TEXT NOT NULL DEFAULT '[]'; + +ALTER TABLE product_usage_daily_rollups + ADD COLUMN retention_json TEXT NOT NULL DEFAULT '[]'; diff --git a/src/db/repositories.ts b/src/db/repositories.ts index 239f4c5cb1..94807c0592 100644 --- a/src/db/repositories.ts +++ b/src/db/repositories.ts @@ -90,11 +90,18 @@ import type { ProductUsageDailyRollupRecord, ProductUsageDailyRollupStatus, ProductUsageEventRecord, + ProductUsageRetentionRollup, ProductUsageRollupRunResult, ProductUsageRollupStatus, ProductUsageOutcome, + ProductUsageRole, + ProductUsageRoleActivationFunnel, + ProductUsageRoleDimensionCount, + ProductUsageRoleRetention, ProductUsageSummary, ProductUsageSurface, + ProductUsageSurfaceActivationFunnel, + ProductUsageSurfaceRetention, PullRequestFileRecord, PullRequestDetailSyncStateRecord, PullRequestRecord, @@ -3091,6 +3098,10 @@ function toProductUsageDailyRollupRecord(row: typeof productUsageDailyRollups.$i byTool: parseJson>(row.toolsJson, []), byRouteClass: parseJson>(row.routeClassesJson, []), activation: parseJson(row.activationJson, emptyProductUsageActivationFunnel()), + byRole: parseJson(row.rolesJson, []), + activationByRole: parseJson(row.activationByRoleJson, []), + activationBySurface: parseJson(row.activationBySurfaceJson, []), + retention: parseJson(row.retentionJson, []), generatedAt: row.generatedAt, updatedAt: row.updatedAt, }; @@ -3183,12 +3194,25 @@ async function upsertProductUsageDailyRollup(env: Env, day: string, generatedAt: .limit(PRODUCT_USAGE_ROLLUP_EVENT_SCAN_LIMIT + 1); const capped = rows.length > PRODUCT_USAGE_ROLLUP_EVENT_SCAN_LIMIT || sourceEventCount > PRODUCT_USAGE_ROLLUP_EVENT_SCAN_LIMIT; const events = rows.slice(0, PRODUCT_USAGE_ROLLUP_EVENT_SCAN_LIMIT).map(toProductUsageEventRecord); + const retentionStartIso = `${addProductUsageUtcDays(day, -PRODUCT_USAGE_RETENTION_MAX_WINDOW_DAYS)}T00:00:00.000Z`; + const retentionWhere = and(gte(productUsageEvents.occurredAt, retentionStartIso), sql`${productUsageEvents.occurredAt} < ${startIso}`, sql`${productUsageEvents.actorHash} is not null`); + const [retentionSourceRow] = await db.select({ count: sql`count(*)` }).from(productUsageEvents).where(retentionWhere); + const retentionRows = await db + .select() + .from(productUsageEvents) + .where(retentionWhere) + .orderBy(desc(productUsageEvents.occurredAt)) + .limit(PRODUCT_USAGE_RETENTION_EVENT_SCAN_LIMIT + 1); + const retentionCapped = retentionRows.length > PRODUCT_USAGE_RETENTION_EVENT_SCAN_LIMIT || Number(retentionSourceRow?.count ?? 0) > PRODUCT_USAGE_RETENTION_EVENT_SCAN_LIMIT; + const retentionEvents = retentionRows.slice(0, PRODUCT_USAGE_RETENTION_EVENT_SCAN_LIMIT).map(toProductUsageEventRecord); const record = buildProductUsageDailyRollupRecord({ day, generatedAt, sourceEventCount, capped, events, + retentionEvents, + retentionCapped, }); await db .insert(productUsageDailyRollups) @@ -3211,6 +3235,10 @@ async function upsertProductUsageDailyRollup(env: Env, day: string, generatedAt: toolsJson: jsonString(record.byTool), routeClassesJson: jsonString(record.byRouteClass), activationJson: jsonString(record.activation), + rolesJson: jsonString(record.byRole), + activationByRoleJson: jsonString(record.activationByRole), + activationBySurfaceJson: jsonString(record.activationBySurface), + retentionJson: jsonString(record.retention), generatedAt: record.generatedAt, updatedAt: record.updatedAt, }) @@ -3234,6 +3262,10 @@ async function upsertProductUsageDailyRollup(env: Env, day: string, generatedAt: toolsJson: jsonString(record.byTool), routeClassesJson: jsonString(record.byRouteClass), activationJson: jsonString(record.activation), + rolesJson: jsonString(record.byRole), + activationByRoleJson: jsonString(record.activationByRole), + activationBySurfaceJson: jsonString(record.activationBySurface), + retentionJson: jsonString(record.retention), generatedAt: record.generatedAt, updatedAt: record.updatedAt, }, @@ -3247,27 +3279,16 @@ function buildProductUsageDailyRollupRecord(args: { sourceEventCount: number; capped: boolean; events: ProductUsageEventRecord[]; + retentionEvents: ProductUsageEventRecord[]; + retentionCapped: boolean; }): ProductUsageDailyRollupRecord { const today = productUsageDayFromIso(args.generatedAt); const actorHashes = new Set(args.events.map((event) => event.actorHash).filter(isNonEmptyString)); const sessionHashes = new Set(args.events.map((event) => event.sessionHash).filter(isNonEmptyString)); const repoNames = new Set(args.events.map((event) => event.repoFullName).filter(isNonEmptyString)); - const loginActors = productUsageActorSet(args.events, (event) => event.eventName === "auth_session_created"); - const doctorPassActors = productUsageActorSet(args.events, isProductUsageDoctorPassEvent); - const firstUsefulActionActors = productUsageActorSet(args.events, isProductUsageUsefulActionEvent); - const githubInstalledRepos = productUsageRepoSet(args.events, (event) => event.eventName === "github_installation_created"); - const githubFirstCommandRepos = productUsageRepoSet(args.events, isProductUsageGitHubCommandEvent); - const githubUsefulMaintainerRepos = productUsageRepoSet(args.events, isProductUsageUsefulMaintainerEvent); - const activation: ProductUsageActivationFunnel = { - loginActors: loginActors.size, - doctorPassActors: doctorPassActors.size, - firstUsefulActionActors: firstUsefulActionActors.size, - fullyActivatedActors: intersectionCount(loginActors, doctorPassActors, firstUsefulActionActors), - githubInstalledRepos: githubInstalledRepos.size, - githubFirstCommandRepos: githubFirstCommandRepos.size, - githubUsefulMaintainerRepos: githubUsefulMaintainerRepos.size, - githubActivatedRepos: intersectionCount(githubInstalledRepos, githubFirstCommandRepos, githubUsefulMaintainerRepos), - }; + const roleBuckets = productUsageRoleBuckets(args.events); + const surfaceBuckets = productUsageSurfaceBuckets(args.events); + const activation = buildProductUsageActivationFunnel(args.events); return { day: args.day, status: args.capped ? "incomplete" : args.day === today ? "partial" : "complete", @@ -3287,11 +3308,220 @@ function buildProductUsageDailyRollupRecord(args: { byTool: countProductUsageDimensions(args.events.map((event) => productUsageMetadataString(event, "toolName"))), byRouteClass: countProductUsageDimensions(args.events.map((event) => productUsageRouteClass(event.route))), activation, + byRole: roleBuckets.map(({ role, events }) => ({ + role, + count: events.length, + activeActors: new Set(events.map((event) => event.actorHash).filter(isNonEmptyString)).size, + activeRepos: new Set(events.map((event) => event.repoFullName).filter(isNonEmptyString)).size, + })), + activationByRole: roleBuckets.map(({ role, events }) => ({ role, ...buildProductUsageActivationFunnel(events) })), + activationBySurface: surfaceBuckets.map(({ surface, events }) => ({ surface, ...buildProductUsageActivationFunnel(events) })), + retention: buildProductUsageRetentionRollups(args.day, args.events, args.retentionEvents, args.retentionCapped), generatedAt: args.generatedAt, updatedAt: args.generatedAt, }; } +function buildProductUsageActivationFunnel(events: ProductUsageEventRecord[]): ProductUsageActivationFunnel { + const loginActors = productUsageActorSet(events, (event) => event.eventName === "auth_session_created"); + const doctorPassActors = productUsageActorSet(events, isProductUsageDoctorPassEvent); + const firstUsefulActionActors = productUsageActorSet(events, isProductUsageUsefulActionEvent); + const githubInstalledRepos = productUsageRepoSet(events, (event) => event.eventName === "github_installation_created"); + const githubFirstCommandRepos = productUsageRepoSet(events, isProductUsageGitHubCommandEvent); + const githubUsefulMaintainerRepos = productUsageRepoSet(events, isProductUsageUsefulMaintainerEvent); + return { + loginActors: loginActors.size, + doctorPassActors: doctorPassActors.size, + firstUsefulActionActors: firstUsefulActionActors.size, + fullyActivatedActors: intersectionCount(loginActors, doctorPassActors, firstUsefulActionActors), + githubInstalledRepos: githubInstalledRepos.size, + githubFirstCommandRepos: githubFirstCommandRepos.size, + githubUsefulMaintainerRepos: githubUsefulMaintainerRepos.size, + githubActivatedRepos: intersectionCount(githubInstalledRepos, githubFirstCommandRepos, githubUsefulMaintainerRepos), + }; +} + +function productUsageRoleBuckets(events: ProductUsageEventRecord[]): Array<{ role: ProductUsageRole; events: ProductUsageEventRecord[] }> { + const buckets = new Map(); + const actorRoles = productUsageRolesByActor(events); + for (const event of events) { + for (const role of productUsageRolesForEvent(event, actorRoles)) { + const bucket = buckets.get(role); + if (bucket) bucket.push(event); + else buckets.set(role, [event]); + } + } + return [...buckets.entries()] + .map(([role, bucketEvents]) => ({ role, events: bucketEvents })) + .sort((a, b) => b.events.length - a.events.length || productUsageRoleSortValue(a.role) - productUsageRoleSortValue(b.role)); +} + +function productUsageSurfaceBuckets(events: ProductUsageEventRecord[]): Array<{ surface: ProductUsageSurface; events: ProductUsageEventRecord[] }> { + const buckets = new Map(); + for (const event of events) { + const surface = normalizeProductUsageSurface(event.surface); + const bucket = buckets.get(surface); + if (bucket) bucket.push(event); + else buckets.set(surface, [event]); + } + return [...buckets.entries()] + .map(([surface, bucketEvents]) => ({ surface, events: bucketEvents })) + .sort((a, b) => b.events.length - a.events.length || a.surface.localeCompare(b.surface)); +} + +function buildProductUsageRetentionRollups(day: string, currentEvents: ProductUsageEventRecord[], previousEvents: ProductUsageEventRecord[], capped: boolean): ProductUsageRetentionRollup[] { + return PRODUCT_USAGE_RETENTION_WINDOWS.map(({ window, days }) => { + const previousStartIso = `${addProductUsageUtcDays(day, -days)}T00:00:00.000Z`; + const windowPreviousEvents = previousEvents.filter((event) => event.occurredAt >= previousStartIso); + const currentActors = productUsageActorHashes(currentEvents); + const previousActors = productUsageActorHashes(windowPreviousEvents); + const retainedActors = intersectionCount(currentActors, previousActors); + return { + window, + capped, + activeActors: currentActors.size, + retainedActors, + retentionRate: productUsageRetentionRate(retainedActors, currentActors.size), + byRole: productUsageRetentionByRole(currentEvents, windowPreviousEvents), + bySurface: productUsageRetentionBySurface(currentEvents, windowPreviousEvents), + }; + }); +} + +function productUsageRetentionByRole(currentEvents: ProductUsageEventRecord[], previousEvents: ProductUsageEventRecord[]): ProductUsageRoleRetention[] { + const previousActorRoles = productUsageRolesByActor(previousEvents); + return productUsageRoleBuckets(currentEvents).map(({ role, events }) => { + const currentActors = productUsageActorHashes(events); + const previousActors = productUsageActorHashes(previousEvents.filter((event) => productUsageRolesForEvent(event, previousActorRoles).includes(role))); + const retainedActors = intersectionCount(currentActors, previousActors); + return { + role, + activeActors: currentActors.size, + retainedActors, + retentionRate: productUsageRetentionRate(retainedActors, currentActors.size), + }; + }); +} + +function productUsageRetentionBySurface(currentEvents: ProductUsageEventRecord[], previousEvents: ProductUsageEventRecord[]): ProductUsageSurfaceRetention[] { + return productUsageSurfaceBuckets(currentEvents).map(({ surface, events }) => { + const currentActors = productUsageActorHashes(events); + const previousActors = productUsageActorHashes(previousEvents.filter((event) => normalizeProductUsageSurface(event.surface) === surface)); + const retainedActors = intersectionCount(currentActors, previousActors); + return { + surface, + activeActors: currentActors.size, + retainedActors, + retentionRate: productUsageRetentionRate(retainedActors, currentActors.size), + }; + }); +} + +function productUsageActorHashes(events: ProductUsageEventRecord[]): Set { + return new Set(events.map((event) => event.actorHash).filter(isNonEmptyString)); +} + +function productUsageRetentionRate(retainedActors: number, activeActors: number): number { + return activeActors > 0 ? Number((retainedActors / activeActors).toFixed(4)) : 0; +} + +function productUsageRolesByActor(events: ProductUsageEventRecord[]): Map { + const rolesByActor = new Map>(); + for (const event of events) { + if (!event.actorHash) continue; + const roles = productUsageBaseRolesForEvent(event).filter((role) => role !== "unknown"); + if (roles.length === 0) continue; + const bucket = rolesByActor.get(event.actorHash) ?? new Set(); + for (const role of roles) bucket.add(role); + rolesByActor.set(event.actorHash, bucket); + } + return new Map( + [...rolesByActor.entries()].map(([actorHash, roles]) => [ + actorHash, + [...roles].sort((a, b) => productUsageRoleSortValue(a) - productUsageRoleSortValue(b)), + ]), + ); +} + +function productUsageRolesForEvent(event: ProductUsageEventRecord, actorRoles: Map = new Map()): ProductUsageRole[] { + const baseRoles = productUsageBaseRolesForEvent(event); + if (baseRoles.length === 1 && baseRoles[0] === "unknown" && event.actorHash) return actorRoles.get(event.actorHash) ?? baseRoles; + return baseRoles; +} + +function productUsageBaseRolesForEvent(event: ProductUsageEventRecord): ProductUsageRole[] { + const roles = new Set(); + addProductUsageRolesFromValue(roles, event.metadata.role); + addProductUsageRolesFromValue(roles, event.metadata.roles); + addProductUsageRolesFromValue(roles, event.metadata.audience); + addProductUsageRolesFromValue(roles, event.metadata.actorRole); + addProductUsageRolesFromValue(roles, event.metadata.actorKind); + if (roles.size > 0) return [...roles].sort((a, b) => productUsageRoleSortValue(a) - productUsageRoleSortValue(b)); + + if (event.eventName === "github_installation_created") return ["owner"]; + if (event.eventName === "extension_session_created" || event.eventName === "pull_context_viewed") return ["maintainer"]; + if (event.surface === "mcp") return ["miner"]; + if ( + event.eventName === "local_branch_analysis_completed" || + event.eventName === "agent_run_started" || + event.eventName === "agent_plan_next_work_completed" || + event.eventName === "agent_preflight_branch_completed" || + event.eventName === "agent_pr_packet_completed" || + event.eventName === "agent_blockers_completed" + ) { + return ["miner"]; + } + if (event.eventName === "pr_public_surface_published") return ["contributor"]; + return ["unknown"]; +} + +function addProductUsageRolesFromValue(roles: Set, value: JsonValue | undefined): void { + if (Array.isArray(value)) { + for (const entry of value) addProductUsageRolesFromValue(roles, entry); + return; + } + if (typeof value !== "string") return; + const role = normalizeProductUsageRole(value); + if (role) roles.add(role); +} + +function normalizeProductUsageRole(value: string): ProductUsageRole | null { + switch (value.trim().toLowerCase().replace(/[\s-]+/g, "_")) { + case "miner": + case "miners": + return "miner"; + case "maintainer": + case "maintainers": + case "reviewer": + return "maintainer"; + case "owner": + case "owners": + case "repo_owner": + case "repo_owners": + case "repository_owner": + case "repository_owners": + return "owner"; + case "operator": + case "operators": + return "operator"; + case "author": + case "contributor": + case "contributors": + case "outside_contributor": + case "outside_contributors": + return "contributor"; + case "none": + case "unknown": + return "unknown"; + default: + return null; + } +} + +function productUsageRoleSortValue(role: ProductUsageRole): number { + return PRODUCT_USAGE_ROLE_ORDER.indexOf(role); +} + function productUsageActorSet(events: ProductUsageEventRecord[], predicate: (event: ProductUsageEventRecord) => boolean): Set { return new Set(events.filter(predicate).map((event) => event.actorHash).filter(isNonEmptyString)); } @@ -3410,8 +3640,15 @@ const PRODUCT_USAGE_METADATA_MAX_ARRAY_ITEMS = 20; const PRODUCT_USAGE_METADATA_MAX_KEY_CHARS = 64; const PRODUCT_USAGE_METADATA_MAX_STRING_CHARS = 200; const PRODUCT_USAGE_ROLLUP_EVENT_SCAN_LIMIT = 5000; +const PRODUCT_USAGE_RETENTION_EVENT_SCAN_LIMIT = 5000; +const PRODUCT_USAGE_RETENTION_MAX_WINDOW_DAYS = 30; +const PRODUCT_USAGE_RETENTION_WINDOWS: Array<{ window: ProductUsageRetentionRollup["window"]; days: number }> = [ + { window: "previous_7_days", days: 7 }, + { window: "previous_30_days", days: 30 }, +]; const MCP_COMPATIBILITY_ADOPTION_SCAN_LIMIT = 5000; const PRODUCT_USAGE_ACTOR_REDACTION_MAX_CHARS = 256; +const PRODUCT_USAGE_ROLE_ORDER: ProductUsageRole[] = ["miner", "maintainer", "owner", "operator", "contributor", "unknown"]; const PRODUCT_USAGE_USEFUL_ACTION_EVENTS = new Set([ "command_previewed", "pull_context_viewed", diff --git a/src/db/schema.ts b/src/db/schema.ts index 45d37fe943..3b041dce63 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -842,6 +842,10 @@ export const productUsageDailyRollups = sqliteTable( toolsJson: text("tools_json").notNull().default("[]"), routeClassesJson: text("route_classes_json").notNull().default("[]"), activationJson: text("activation_json").notNull().default("{}"), + rolesJson: text("roles_json").notNull().default("[]"), + activationByRoleJson: text("activation_by_role_json").notNull().default("[]"), + activationBySurfaceJson: text("activation_by_surface_json").notNull().default("[]"), + retentionJson: text("retention_json").notNull().default("[]"), generatedAt: text("generated_at").notNull().default("CURRENT_TIMESTAMP"), updatedAt: text("updated_at").notNull().default("CURRENT_TIMESTAMP"), }, diff --git a/src/types.ts b/src/types.ts index ac516d8e78..eb3e3f3379 100644 --- a/src/types.ts +++ b/src/types.ts @@ -970,6 +970,8 @@ export type ProductUsageSurface = "api" | "mcp" | "github_app" | "control_panel" export type ProductUsageOutcome = "success" | "denied" | "error" | "queued" | "completed" | "skipped"; +export type ProductUsageRole = "miner" | "maintainer" | "owner" | "operator" | "contributor" | "unknown"; + export type ProductUsageEventRecord = { id: string; surface: ProductUsageSurface; @@ -1020,6 +1022,13 @@ export type ProductUsageDimensionCount = { count: number; }; +export type ProductUsageRoleDimensionCount = { + role: ProductUsageRole; + count: number; + activeActors: number; + activeRepos: number; +}; + export type ProductUsageActivationFunnel = { loginActors: number; doctorPassActors: number; @@ -1031,6 +1040,37 @@ export type ProductUsageActivationFunnel = { githubActivatedRepos: number; }; +export type ProductUsageRoleActivationFunnel = ProductUsageActivationFunnel & { + role: ProductUsageRole; +}; + +export type ProductUsageSurfaceActivationFunnel = ProductUsageActivationFunnel & { + surface: ProductUsageSurface; +}; + +export type ProductUsageRetentionWindow = "previous_7_days" | "previous_30_days"; + +export type ProductUsageRetentionDimension = { + activeActors: number; + retainedActors: number; + retentionRate: number; +}; + +export type ProductUsageRoleRetention = ProductUsageRetentionDimension & { + role: ProductUsageRole; +}; + +export type ProductUsageSurfaceRetention = ProductUsageRetentionDimension & { + surface: ProductUsageSurface; +}; + +export type ProductUsageRetentionRollup = ProductUsageRetentionDimension & { + window: ProductUsageRetentionWindow; + capped: boolean; + byRole: ProductUsageRoleRetention[]; + bySurface: ProductUsageSurfaceRetention[]; +}; + export type ProductUsageDailyRollupRecord = { day: string; status: ProductUsageDailyRollupStatus; @@ -1050,6 +1090,10 @@ export type ProductUsageDailyRollupRecord = { byTool: ProductUsageDimensionCount[]; byRouteClass: ProductUsageDimensionCount[]; activation: ProductUsageActivationFunnel; + byRole: ProductUsageRoleDimensionCount[]; + activationByRole: ProductUsageRoleActivationFunnel[]; + activationBySurface: ProductUsageSurfaceActivationFunnel[]; + retention: ProductUsageRetentionRollup[]; generatedAt: string; updatedAt: string; }; diff --git a/test/integration/api.test.ts b/test/integration/api.test.ts index 0d787cef5c..c292f1b4f9 100644 --- a/test/integration/api.test.ts +++ b/test/integration/api.test.ts @@ -2399,10 +2399,47 @@ describe("api routes", () => { const dailyRollups = await app.request("/v1/app/analytics/daily-rollups?limit=3", { headers: apiHeaders(env) }, env); expect(dailyRollups.status).toBe(200); - await expect(dailyRollups.json()).resolves.toMatchObject({ + const dailyRollupsBody = (await dailyRollups.json()) as { + status: { status: string; latestRollupDay?: string }; + rollups: Array<{ + day: string; + activation: Record; + byRole: Array<{ role: string; count: number; activeActors: number; activeRepos: number }>; + activationByRole: Array>; + activationBySurface: Array>; + retention: Array<{ + window: string; + activeActors: number; + retainedActors: number; + retentionRate: number; + capped: boolean; + byRole: Array>; + bySurface: Array>; + }>; + }>; + }; + expect(dailyRollupsBody).toMatchObject({ status: expect.objectContaining({ status: "partial", latestRollupDay: "2026-05-28" }), rollups: [expect.objectContaining({ day: "2026-05-28", activation: expect.any(Object) })], }); + const [dailyRollup] = dailyRollupsBody.rollups; + expect(dailyRollup).toBeDefined(); + if (!dailyRollup) throw new Error("expected daily usage rollup"); + expect(dailyRollup.byRole).toEqual(expect.arrayContaining([expect.objectContaining({ role: "miner", count: expect.any(Number), activeActors: expect.any(Number), activeRepos: expect.any(Number) })])); + expect(dailyRollup.activationByRole).toEqual(expect.arrayContaining([expect.objectContaining({ role: "miner", doctorPassActors: expect.any(Number), firstUsefulActionActors: expect.any(Number) })])); + expect(dailyRollup.activationBySurface).toEqual( + expect.arrayContaining([ + expect.objectContaining({ surface: "mcp", doctorPassActors: expect.any(Number) }), + expect.objectContaining({ surface: "browser_extension", firstUsefulActionActors: expect.any(Number) }), + ]), + ); + expect(dailyRollup.retention).toEqual( + expect.arrayContaining([ + expect.objectContaining({ window: "previous_7_days", activeActors: expect.any(Number), retainedActors: expect.any(Number), retentionRate: expect.any(Number), capped: false, byRole: expect.any(Array), bySurface: expect.any(Array) }), + expect.objectContaining({ window: "previous_30_days", activeActors: expect.any(Number), retainedActors: expect.any(Number), retentionRate: expect.any(Number), capped: false, byRole: expect.any(Array), bySurface: expect.any(Array) }), + ]), + ); + expect(JSON.stringify(dailyRollupsBody)).not.toMatch(/oktofeesh1|operator@example.com|mcp-user|old-mcp-user|current-mcp-user|mcp-session|old-mcp-session|current-mcp-session|gittensory_session|\/Users|github_pat|ghp_|private-repo|wallet|hotkey|raw trust/i); const fallbackLimitRollups = await app.request("/v1/app/analytics/daily-rollups?limit=invalid", { headers: apiHeaders(env) }, env); expect(fallbackLimitRollups.status).toBe(200); await expect(fallbackLimitRollups.json()).resolves.toMatchObject({ @@ -2421,7 +2458,7 @@ describe("api routes", () => { const usageOperatorBody = (await usageOperator.json()) as { metrics: Array<{ label: string; value: string }>; usageSummary: { totalEvents: number }; - usageRollups: Array<{ day: string }>; + usageRollups: Array<{ day: string; byRole: unknown[]; activationBySurface: unknown[]; retention: unknown[] }>; usageRollupStatus: { status: string }; mcpCompatibilityAdoption: { totalEvents: number; @@ -2449,7 +2486,7 @@ describe("api routes", () => { ]), ); expect(usageOperatorBody.usageSummary.totalEvents).toBe(productUsageEvents.length); - expect(usageOperatorBody.usageRollups).toEqual([expect.objectContaining({ day: "2026-05-28" })]); + expect(usageOperatorBody.usageRollups).toEqual([expect.objectContaining({ day: "2026-05-28", byRole: expect.any(Array), activationBySurface: expect.any(Array), retention: expect.any(Array) })]); expect(usageOperatorBody.usageRollupStatus.status).toBe("partial"); expect(usageOperatorBody.mcpCompatibilityAdoption).toMatchObject({ totalEvents: 4, diff --git a/test/unit/product-usage.test.ts b/test/unit/product-usage.test.ts index 098f8862e9..0167d17d68 100644 --- a/test/unit/product-usage.test.ts +++ b/test/unit/product-usage.test.ts @@ -431,6 +431,274 @@ describe("product usage events", () => { await expect(getProductUsageRollupStatus(env, { nowIso: "2026-05-31T00:40:00.000Z" })).resolves.toMatchObject({ status: "ready", warnings: [] }); }); + it("builds empty role and retention rollups for days without product usage", async () => { + const env = createTestEnv({ PRODUCT_USAGE_HASH_SALT: "fixed-test-salt" }); + const result = await rollupProductUsageDaily(env, { day: "2026-06-01", nowIso: "2026-06-02T00:00:00.000Z" }); + + expect(result.rollups[0]).toMatchObject({ + day: "2026-06-01", + status: "complete", + totalEvents: 0, + activeActors: 0, + byRole: [], + activationByRole: [], + activationBySurface: [], + retention: [ + { window: "previous_7_days", capped: false, activeActors: 0, retainedActors: 0, retentionRate: 0, byRole: [], bySurface: [] }, + { window: "previous_30_days", capped: false, activeActors: 0, retainedActors: 0, retentionRate: 0, byRole: [], bySurface: [] }, + ], + }); + }); + + it("recomputes single-role retention rollups when late product usage makes a day stale", async () => { + const env = createTestEnv({ PRODUCT_USAGE_HASH_SALT: "fixed-test-salt" }); + const day = "2026-06-12"; + await recordProductUsageEvent(env, { + surface: "api", + eventName: "agent_pr_packet_completed", + actor: "single-miner", + outcome: "success", + metadata: { role: "miner" }, + occurredAt: "2026-06-05T12:00:00.000Z", + }); + await rollupProductUsageDaily(env, { day: "2026-06-05", nowIso: "2026-06-06T00:00:00.000Z" }); + await recordProductUsageEvent(env, { + surface: "api", + eventName: "agent_pr_packet_completed", + actor: "single-miner", + outcome: "success", + metadata: { role: "miner" }, + occurredAt: `${day}T01:00:00.000Z`, + }); + + const firstRun = await rollupProductUsageDaily(env, { day, nowIso: "2026-06-13T00:00:00.000Z" }); + expect(firstRun.rollups[0]).toMatchObject({ + byRole: [{ role: "miner", count: 1, activeActors: 1, activeRepos: 0 }], + activationByRole: [expect.objectContaining({ role: "miner", firstUsefulActionActors: 1 })], + activationBySurface: [expect.objectContaining({ surface: "api", firstUsefulActionActors: 1 })], + retention: expect.arrayContaining([ + expect.objectContaining({ + window: "previous_7_days", + activeActors: 1, + retainedActors: 1, + byRole: [{ role: "miner", activeActors: 1, retainedActors: 1, retentionRate: 1 }], + }), + ]), + }); + + await recordProductUsageEvent(env, { + surface: "github_app", + eventName: "github_installation_created", + actor: "late-owner", + repoFullName: "JSONbored/gittensory", + outcome: "completed", + occurredAt: `${day}T23:00:00.000Z`, + }); + await expect(getProductUsageRollupStatus(env, { nowIso: "2026-06-13T00:10:00.000Z" })).resolves.toMatchObject({ status: "stale", staleDays: [day] }); + + const rerun = await rollupProductUsageDaily(env, { day, nowIso: "2026-06-13T00:20:00.000Z" }); + expect(rerun.rollups[0]).toMatchObject({ + totalEvents: 2, + activeActors: 2, + byRole: expect.arrayContaining([ + { role: "miner", count: 1, activeActors: 1, activeRepos: 0 }, + { role: "owner", count: 1, activeActors: 1, activeRepos: 1 }, + ]), + retention: expect.arrayContaining([ + expect.objectContaining({ + window: "previous_7_days", + activeActors: 2, + retainedActors: 1, + retentionRate: 0.5, + byRole: expect.arrayContaining([ + { role: "miner", activeActors: 1, retainedActors: 1, retentionRate: 1 }, + { role: "owner", activeActors: 1, retainedActors: 0, retentionRate: 0 }, + ]), + }), + ]), + }); + expect(JSON.stringify(rerun.rollups[0])).not.toMatch(/single-miner|late-owner|fixed-test-salt/i); + }); + + it("normalizes explicit role metadata variants into aggregate buckets", async () => { + const env = createTestEnv({ PRODUCT_USAGE_HASH_SALT: "fixed-test-salt" }); + const day = "2026-06-14"; + await recordProductUsageEvent(env, { + surface: "control_panel", + eventName: "command_previewed", + actor: "multi-role-actor", + outcome: "success", + metadata: { + roles: [ + "miners", + "maintainers", + "owner", + "owners", + "repo-owner", + "repo owners", + "repository-owner", + "repository owners", + "operator", + "operators", + "author", + "contributors", + "outside contributor", + "outside-contributors", + "unknown", + ], + }, + occurredAt: `${day}T01:00:00.000Z`, + }); + await recordProductUsageEvent(env, { + surface: "control_panel", + eventName: "command_previewed", + actor: "reviewer-actor", + outcome: "success", + metadata: { actorKind: "reviewer" }, + occurredAt: `${day}T02:00:00.000Z`, + }); + await recordProductUsageEvent(env, { + surface: "control_panel", + eventName: "command_previewed", + actor: "none-actor", + outcome: "success", + metadata: { audience: "none" }, + occurredAt: `${day}T03:00:00.000Z`, + }); + await recordProductUsageEvent(env, { + surface: "control_panel", + eventName: "command_previewed", + actor: "invalid-role-actor", + outcome: "success", + metadata: { role: "not-a-product-role" }, + occurredAt: `${day}T04:00:00.000Z`, + }); + await recordProductUsageEvent(env, { + surface: "api", + eventName: "pr_public_surface_published", + actor: "public-surface-actor", + outcome: "success", + occurredAt: `${day}T05:00:00.000Z`, + }); + + const result = await rollupProductUsageDaily(env, { day, nowIso: "2026-06-15T00:00:00.000Z" }); + + expect(result.rollups[0]?.byRole).toEqual( + expect.arrayContaining([ + { role: "owner", count: 1, activeActors: 1, activeRepos: 0 }, + { role: "operator", count: 1, activeActors: 1, activeRepos: 0 }, + { role: "miner", count: 1, activeActors: 1, activeRepos: 0 }, + { role: "contributor", count: 2, activeActors: 2, activeRepos: 0 }, + { role: "maintainer", count: 2, activeActors: 2, activeRepos: 0 }, + { role: "unknown", count: 3, activeActors: 3, activeRepos: 0 }, + ]), + ); + expect(JSON.stringify(result.rollups[0])).not.toMatch(/multi-role-actor|reviewer-actor|none-actor|invalid-role-actor|public-surface-actor|fixed-test-salt/i); + }); + + it("builds role activation and coarse retention rollups without exposing actors", async () => { + const env = createTestEnv({ PRODUCT_USAGE_HASH_SALT: "fixed-test-salt" }); + const day = "2026-06-10"; + await recordProductUsageEvent(env, { + surface: "mcp", + eventName: "mcp_request", + actor: "miner-retained", + outcome: "success", + metadata: { role: "miner" }, + occurredAt: "2026-06-04T12:00:00.000Z", + }); + await recordProductUsageEvent(env, { + surface: "github_app", + eventName: "agent_command_replied", + actor: "maintainer-retained", + outcome: "completed", + metadata: { command: "blockers", actorKind: "maintainer" }, + occurredAt: "2026-06-06T12:00:00.000Z", + }); + await recordProductUsageEvent(env, { + surface: "control_panel", + eventName: "auth_session_created", + actor: "miner-retained", + outcome: "success", + metadata: { role: "miner" }, + occurredAt: `${day}T01:00:00.000Z`, + }); + await recordProductUsageEvent(env, { + surface: "mcp", + eventName: "mcp_request", + actor: "miner-retained", + outcome: "success", + metadata: { role: "miner" }, + occurredAt: `${day}T01:05:00.000Z`, + }); + await recordProductUsageEvent(env, { + surface: "api", + eventName: "agent_pr_packet_completed", + actor: "miner-retained", + outcome: "success", + metadata: { role: "miner" }, + occurredAt: `${day}T01:10:00.000Z`, + }); + await recordProductUsageEvent(env, { + surface: "github_app", + eventName: "agent_command_replied", + actor: "maintainer-retained", + repoFullName: "JSONbored/gittensory", + outcome: "completed", + metadata: { command: "blockers", actorKind: "maintainer" }, + occurredAt: `${day}T02:00:00.000Z`, + }); + await recordProductUsageEvent(env, { + surface: "github_app", + eventName: "github_installation_created", + actor: "new-owner", + repoFullName: "JSONbored/gittensory", + outcome: "completed", + occurredAt: `${day}T03:00:00.000Z`, + }); + + const result = await rollupProductUsageDaily(env, { day, nowIso: "2026-06-11T00:00:00.000Z" }); + + expect(result.rollups[0]).toMatchObject({ + day, + totalEvents: 5, + activeActors: 3, + byRole: expect.arrayContaining([ + { role: "miner", count: 3, activeActors: 1, activeRepos: 0 }, + { role: "maintainer", count: 1, activeActors: 1, activeRepos: 1 }, + { role: "owner", count: 1, activeActors: 1, activeRepos: 1 }, + ]), + activationByRole: expect.arrayContaining([ + expect.objectContaining({ role: "miner", loginActors: 1, doctorPassActors: 1, firstUsefulActionActors: 1, fullyActivatedActors: 1 }), + expect.objectContaining({ role: "maintainer", githubUsefulMaintainerRepos: 1 }), + expect.objectContaining({ role: "owner", githubInstalledRepos: 1 }), + ]), + activationBySurface: expect.arrayContaining([ + expect.objectContaining({ surface: "mcp", doctorPassActors: 1 }), + expect.objectContaining({ surface: "github_app", githubInstalledRepos: 1, githubUsefulMaintainerRepos: 1 }), + ]), + retention: expect.arrayContaining([ + expect.objectContaining({ + window: "previous_7_days", + activeActors: 3, + retainedActors: 2, + retentionRate: 0.6667, + capped: false, + byRole: expect.arrayContaining([ + { role: "miner", activeActors: 1, retainedActors: 1, retentionRate: 1 }, + { role: "maintainer", activeActors: 1, retainedActors: 1, retentionRate: 1 }, + { role: "owner", activeActors: 1, retainedActors: 0, retentionRate: 0 }, + ]), + bySurface: expect.arrayContaining([ + { surface: "mcp", activeActors: 1, retainedActors: 1, retentionRate: 1 }, + { surface: "github_app", activeActors: 2, retainedActors: 1, retentionRate: 0.5 }, + ]), + }), + ]), + }); + expect(JSON.stringify(result.rollups[0])).not.toMatch(/miner-retained|maintainer-retained|new-owner|fixed-test-salt/i); + }); + it("classifies rollup route classes and rejects failed activation signals", async () => { const env = createTestEnv({ PRODUCT_USAGE_HASH_SALT: "fixed-test-salt" }); const day = "2026-05-27"; @@ -510,6 +778,10 @@ describe("product usage events", () => { const invalidDay = await rollupProductUsageDaily(env, { day: "not-a-day", nowIso: "2026-05-27T12:00:00.000Z" }); expect(invalidDay.rollups[0]?.day).toBe("2026-05-27"); + const invalidGeneratedAt = await rollupProductUsageDaily(env, { day: "not-a-day", nowIso: "not-an-iso" }); + expect(invalidGeneratedAt.requestedDays[0]).toBe(invalidGeneratedAt.rollups[0]?.day); + expect(invalidGeneratedAt.rollups[0]?.day).toMatch(/^\d{4}-\d{2}-\d{2}$/); + await env.DB.prepare( "insert into product_usage_daily_rollups (day, status, total_events, active_actors, active_sessions, active_repos, source_event_count, max_event_capacity, first_event_at, last_event_at, surfaces_json, outcomes_json, events_json, repos_json, commands_json, tools_json, route_classes_json, activation_json, generated_at, updated_at) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ) @@ -584,4 +856,72 @@ describe("product usage events", () => { incompleteDays: [day], }); }); + + it("marks retention windows capped when previous usage exceeds the retention scan cap", async () => { + const env = createTestEnv({ PRODUCT_USAGE_HASH_SALT: "fixed-test-salt" }); + const day = "2026-06-20"; + const previousDay = "2026-06-10"; + const previousStartMs = Date.parse(`${previousDay}T00:00:00.000Z`); + await env.DB.batch( + Array.from({ length: 5001 }, (_, index) => + env.DB.prepare( + "insert into product_usage_events (id, surface, event_name, route, actor_hash, session_hash, repo_full_name, target_key, outcome, latency_ms, client_name, client_version, metadata_json, occurred_at) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ).bind( + `retention-cap-event-${index}`, + "mcp", + "mcp_request", + "/mcp", + index === 5000 ? "retained-actor-hash" : `previous-actor-${index}`, + null, + null, + null, + "success", + null, + null, + null, + JSON.stringify({ role: "miner" }), + new Date(previousStartMs + index * 1000).toISOString(), + ), + ), + ); + await env.DB.prepare( + "insert into product_usage_events (id, surface, event_name, route, actor_hash, session_hash, repo_full_name, target_key, outcome, latency_ms, client_name, client_version, metadata_json, occurred_at) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .bind( + "retention-cap-current-event", + "mcp", + "mcp_request", + "/mcp", + "retained-actor-hash", + null, + null, + null, + "success", + null, + null, + null, + JSON.stringify({ role: "miner" }), + `${day}T01:00:00.000Z`, + ) + .run(); + + const result = await rollupProductUsageDaily(env, { day, nowIso: "2026-06-21T00:00:00.000Z" }); + + expect(result.rollups[0]).toMatchObject({ + day, + status: "complete", + totalEvents: 1, + retention: expect.arrayContaining([ + expect.objectContaining({ + window: "previous_30_days", + capped: true, + activeActors: 1, + retainedActors: 1, + retentionRate: 1, + byRole: [{ role: "miner", activeActors: 1, retainedActors: 1, retentionRate: 1 }], + bySurface: [{ surface: "mcp", activeActors: 1, retainedActors: 1, retentionRate: 1 }], + }), + ]), + }); + }); }); diff --git a/test/unit/weekly-value-report.test.ts b/test/unit/weekly-value-report.test.ts index 310b56dded..7bc48f6a93 100644 --- a/test/unit/weekly-value-report.test.ts +++ b/test/unit/weekly-value-report.test.ts @@ -367,6 +367,10 @@ function rollup( githubUsefulMaintainerRepos: 1, githubActivatedRepos: 1, }, + byRole: [], + activationByRole: [], + activationBySurface: [], + retention: [], generatedAt: `${day}T23:59:00.000Z`, updatedAt: `${day}T23:59:00.000Z`, };