fix(devex): read depot usage one day at a time in the weekly digest

.github/scripts/eng-analytics-weekly-digest.mjs

@@ -35,6 +35,10 @@ const DEPOT_CI_USD_PER_VCPU_MINUTE = 0.003
 const DAY_MS = 24 * 60 * 60 * 1000
 const WEEK_MS = 7 * DAY_MS
 
+// Keep this, plus one day's retries, inside the job's timeout-minutes.
+const DEPOT_TIME_BUDGET_MS = 20 * 60 * 1000
+const STARTED_AT = performance.now()
+
 const PR_DURATION_BUDGET_SECONDS = 20 * 60
 // One slow run can move the p95 of a workflow with few runs in the week by minutes.
 const MIN_PERCENTILE_RUNS = 100
@@ -112,28 +116,36 @@ function isThisRepository(name) {
     return typeof name === 'string' && name.toLowerCase() === GITHUB_REPOSITORY.toLowerCase()
 }
 
+// GetUsage runs out of processing time on a week of our Depot CI usage and answers 500 or 429, so
+// every read asks for one UTC day at a time. Past the time budget, the digest posts without spend
+// rather than letting the job limit stop it before it posts anything. The contract range includes
+// both weeks, so a day is requested once per run and reused after that.
+const dayReads = new Map()
+
 async function spend(startAt, endAt) {
-    const usage = await withRetry(() => depotUsageSpend(startAt, endAt))
-    return { ...usage, total: usage.runners + usage.depotCi + usage.containerBuilds }
+    const sum = { runners: 0, repoRunners: 0, depotCi: 0, repoDepotCi: 0, containerBuilds: 0 }
+    for (const [from, to] of utcDays(startAt, endAt)) {
+        const range = `${from.toISOString()}/${to.toISOString()}`
+        if (!dayReads.has(range)) {
+            if (performance.now() - STARTED_AT > DEPOT_TIME_BUDGET_MS) {
+                throw new Error('Depot GetUsage reads ran past their time budget')
+            }
+            dayReads.set(range, withRetry(() => depotUsageSpend(from, to)))
+        }
+        const day = await dayReads.get(range)
+        for (const key of Object.keys(sum)) {
+            sum[key] += day[key]
+        }
+    }
+    return { ...sum, total: sum.runners + sum.depotCi + sum.containerBuilds }
 }
 
-function calendarMonths(startAt, endAt) {
-    const ranges = []
+function* utcDays(startAt, endAt) {
     for (let from = startAt; from < endAt; ) {
-        const nextMonth = new Date(Date.UTC(from.getUTCFullYear(), from.getUTCMonth() + 1, 1))
-        const to = nextMonth < endAt ? nextMonth : endAt
-        ranges.push([from, to])
+        const to = new Date(Math.min(from.getTime() + DAY_MS, endAt.getTime()))
+        yield [from, to]
         from = to
     }
-    return ranges
-}
-
-async function contractSpend(startAt, endAt) {
-    let used = 0
-    for (const [from, to] of calendarMonths(startAt, endAt)) {
-        used += (await spend(from, to)).total
-    }
-    return used
 }
 
 function contractDates(weekEnd) {
@@ -174,7 +186,11 @@ async function readSpend(weekStart, weekEnd) {
     if (!priorWeek) {
         return null
     }
-    const used = contract && (await contractSpend(contract.start, weekEnd).catch(unavailable('Contract spend')))
+    const used =
+        contract &&
+        (await spend(contract.start, weekEnd)
+            .then((usage) => usage.total)
+            .catch(unavailable('Contract spend')))
     return { lastWeek, priorWeek, contract: used == null ? null : { ...contract, used } }
 }
 

.github/scripts/eng-analytics-weekly-digest.test.mjs

@@ -26,7 +26,8 @@ const WEEK = '2026-09-28T00:00:00.000Z'
 const PRIOR_WEEK = '2026-09-21T00:00:00.000Z'
 const CONTRACT = '2026-01-01T00:00:00.000Z'
 // Runners $600 ($400 in this repository), builds $400, Depot CI $1,200 ($900 here): $2,200.
-// The week before: runners $200, builds $200, Depot CI $300: $700. The contract so far: $50,000.
+// The week before: runners $200, builds $200, Depot CI $300: $700. The contract so far, which
+// includes both weeks: $50,000. Every other day has no usage.
 const DEPOT_USAGE = {
     [WEEK]: {
         githubActionsJobs: [
@@ -47,7 +48,7 @@ const DEPOT_USAGE = {
         containerBuild: [{ minutesBilled: 5_000 }],
     },
     [CONTRACT]: {
-        githubActionsJobs: [{ repo: 'PostHog/posthog', total: { minutesBilled: 10_000_000 } }],
+        githubActionsJobs: [{ repo: 'PostHog/posthog', total: { minutesBilled: 9_275_000 } }],
         depotCi: [{ repo: 'PostHog/posthog', workflowPath: 'ci-backend.yml', vcpuMinutesBilled: 2_000_000 }],
         containerBuild: [{ minutesBilled: 100_000 }],
     },
@@ -70,8 +71,14 @@ const SLOW_WORKFLOWS_NOTE = [
 ].join('\n')
 let importCount = 0
 
-async function runDigest(t, { env = {}, depotUsage = DEPOT_USAGE, workflowHealth = WORKFLOW_HEALTH } = {}) {
+async function runDigest(
+    t,
+    { env = {}, depotUsage = DEPOT_USAGE, workflowHealth = WORKFLOW_HEALTH, performanceNow } = {}
+) {
     t.mock.timers.enable({ apis: ['Date', 'setTimeout'], now: new Date('2026-10-05T13:00:00Z') })
+    if (performanceNow) {
+        t.mock.method(performance, 'now', performanceNow)
+    }
     const settings = { ...ENV, ...env }
     const previousEnv = Object.fromEntries(Object.keys(settings).map((key) => [key, process.env[key]]))
     Object.assign(process.env, settings)
@@ -144,21 +151,24 @@ describe('weekly engineering analytics digest', () => {
         ['', {}],
         [
             ' after a transient Depot failure',
-            { depotUsage: { ...DEPOT_USAGE, [WEEK]: [new Response('', { status: 503 }), DEPOT_USAGE[WEEK]] } },
+            {
+                depotUsage: {
+                    ...DEPOT_USAGE,
+                    [WEEK]: [new Response('', { status: 503 }), DEPOT_USAGE[WEEK]],
+                },
+            },
         ],
     ]) {
         it(`reports Depot spend for complete UTC weeks and the contract${name}`, async (t) => {
             const { blocks, requests, maxDepotInFlight } = await runDigest(t, override)
             assert.equal(maxDepotInFlight, 1)
-            const contractRanges = requests
+            const depotRanges = requests
                 .filter((request) => request.url.startsWith('https://api.depot.dev/'))
                 .map((request) => JSON.parse(request.body))
-                .slice(-10)
-            assert.deepEqual(
-                contractRanges.map((range) => range.startAt.slice(0, 10)),
-                Array.from({ length: 10 }, (_, i) => `2026-${String(i + 1).padStart(2, '0')}-01`)
-            )
-            assert.equal(contractRanges.at(-1).endAt, '2026-10-05T00:00:00.000Z')
+            assert.ok(depotRanges.every((range) => Date.parse(range.endAt) - Date.parse(range.startAt) === 86_400_000))
+            const contractRanges = depotRanges.slice(depotRanges.findIndex((range) => range.startAt === CONTRACT))
+            assert.equal(contractRanges.length, 263)
+            assert.equal(contractRanges.at(-1).endAt, PRIOR_WEEK)
             const overviewUrl = new URL(requests.find((request) => request.url.includes('/repo_overview/')).url)
             assert.equal(overviewUrl.searchParams.get('date_from'), WEEK)
             assert.equal(overviewUrl.searchParams.get('date_to'), '2026-10-05T00:00:00.000Z')
@@ -211,12 +221,21 @@ describe('weekly engineering analytics digest', () => {
         })
     }
 
-    for (const [name, allowance, expected] of [
-        ['a used-up contract', '50000', /The contract is used up\.$/],
-        ['a contract that lasts', '1000000', /it lasts until the contract ends on 2027-01-01\.$/],
+    for (const [name, env, expected] of [
+        ['a used-up contract', { DEPOT_CONTRACT_USD: '50000' }, /The contract is used up\.$/],
+        [
+            'a contract that lasts',
+            { DEPOT_CONTRACT_USD: '1000000' },
+            /it lasts until the contract ends on 2027-01-01\.$/,
+        ],
+        [
+            'a contract that starts inside the prior week',
+            { DEPOT_CONTRACT_START: '2026-09-24' },
+            /^Depot contract: \$2,200 of \$60,000 used through 2026-10-04/,
+        ],
     ]) {
         it(`describes ${name}`, async (t) => {
-            const { blocks } = await runDigest(t, { env: { DEPOT_CONTRACT_USD: allowance } })
+            const { blocks } = await runDigest(t, { env })
             assert.match(blocks[2].text.text, expected)
         })
     }
@@ -229,6 +248,11 @@ describe('weekly engineering analytics digest', () => {
                 depotUsage: { ...DEPOT_USAGE, [CONTRACT]: new Response('confidential-response', { status: 404 }) },
             },
         ],
+        [
+            'the contract reads run past their time budget',
+            // The import reads the clock once, then each of the 14 days of the two weeks reads it once.
+            { performanceNow: ((calls) => () => (++calls > 15 ? 21 * 60 * 1000 : 0))(0) },
+        ],
     ]) {
         it(`posts spend without the contract sentence when ${name}`, async (t) => {
             const { blocks } = await runDigest(t, override)

.github/workflows/eng-analytics-weekly-digest.yml

@@ -32,9 +32,9 @@ permissions:
 jobs:
     digest:
         runs-on: ubuntu-24.04
-        # The endpoint calls run in parallel. The slowest takes about 8.5 minutes: 3 attempts of 150s
-        # with 30s between them.
-        timeout-minutes: 12
+        # The script stops reading Depot usage after DEPOT_TIME_BUDGET_MS (20 minutes) and posts without
+        # spend. This limit leaves room past that for one day's retries and the Slack post.
+        timeout-minutes: 30
         # Never run from forks (no secret access anyway), only the canonical repo.
         # Manual dispatch is master-only so branch-controlled code can't run with the secrets.
         if: >-