diff --git a/.github/workflows/agent-rate-limits-tests.yml b/.github/workflows/agent-rate-limits-tests.yml index 4780d8d6d..778e5117e 100644 --- a/.github/workflows/agent-rate-limits-tests.yml +++ b/.github/workflows/agent-rate-limits-tests.yml @@ -8,6 +8,7 @@ # - tests/test_agent_rate_limits_config.bats # - tests/test_agent_rate_limit.bats # - tests/test_agent_rate_limit_token_budget.bats +# - tests/test_agent_rate_limit_weekly_glide.bats # - .github/workflows/agent-rate-limits-tests.yml # # Gate: bats validates that standards/agent-rate-limits.json parses as JSON, all @@ -21,7 +22,11 @@ # open/half-open transitions, and malformed-input degradation that allows) # against a stubbed `gh`; and that the org-wide token-budget breaker (#641) # defers over the config threshold, blocks on a fresh 429, and fails safe to -# allow on telemetry error — against a mocked telemetry adapter, no live network. +# allow on telemetry error; and that the weekly glide-path breaker (#994) defers +# over a time-varying threshold derived from the payload's resets_at (with +# partial days rounded up), ignores the per-model weekly_scoped limit, and fails +# safe to allow on telemetry error — against a mocked telemetry adapter, no live +# network. # ShellCheck of the lib is covered by the existing scripts/**/*.sh job in ci.yml. # # Context: ADR docs/initiatives/agent-rate-limits-adr.md (#637). #638 delivers the @@ -39,6 +44,7 @@ on: - 'tests/test_agent_rate_limits_config.bats' - 'tests/test_agent_rate_limit.bats' - 'tests/test_agent_rate_limit_token_budget.bats' + - 'tests/test_agent_rate_limit_weekly_glide.bats' - '.github/workflows/agent-rate-limits-tests.yml' push: branches: @@ -49,6 +55,7 @@ on: - 'tests/test_agent_rate_limits_config.bats' - 'tests/test_agent_rate_limit.bats' - 'tests/test_agent_rate_limit_token_budget.bats' + - 'tests/test_agent_rate_limit_weekly_glide.bats' - '.github/workflows/agent-rate-limits-tests.yml' permissions: @@ -82,4 +89,5 @@ jobs: bats --print-output-on-failure \ tests/test_agent_rate_limits_config.bats \ tests/test_agent_rate_limit.bats \ - tests/test_agent_rate_limit_token_budget.bats + tests/test_agent_rate_limit_token_budget.bats \ + tests/test_agent_rate_limit_weekly_glide.bats diff --git a/scripts/lib/agent-rate-limit.sh b/scripts/lib/agent-rate-limit.sh index 045e3e678..457501358 100644 --- a/scripts/lib/agent-rate-limit.sh +++ b/scripts/lib/agent-rate-limit.sh @@ -681,6 +681,65 @@ arl_token_budget_decision() { return 1 } +# --------------------------------------------------------------------------- +# arl_token_transport_decision — PURE-ish. The +# telemetry-TRANSPORT-level fail-safe shared by BOTH the session and weekly-glide +# breakers, so there is exactly one 429 / non-200 convention (post-ADR #1). Echoes +# exactly one token on stdout: +# defer — a fresh, unexpired 429 with a positive retry-after (the one hard, +# first-hand cap signal that BLOCKS; ADR §7). A stale 429 whose +# retry_until (or observed_at + retry_after) has already passed is +# treated as expired degraded telemetry and allows instead. +# allow — any other non-200 / unavailable telemetry (fail-safe with warning). +# proceed — a usable 200 body; the caller inspects `.body` for the window. +# Logs the human-readable reason to stderr. Returns 0 always. +# --------------------------------------------------------------------------- +arl_token_transport_decision() { + local envelope="$1" now status retry_after + now="$(arl_sanitize_int "${2:-}")" + status="$(arl_sanitize_int "$(jq -r '.status? // 0' <<<"$envelope" 2>/dev/null || printf '0')")" + + # Fresh 429 with a positive retry-after: the one telemetry response that + # blocks (ADR §7). An observation timestamp (observed_at) or absolute deadline + # (retry_until) in the envelope guards against a stale file deferring forever: + # if the deadline has already passed, treat it as expired degraded telemetry. + if [ "$status" -eq 429 ]; then + retry_after="$(arl_sanitize_int "$(jq -r '.retry_after? // 0' <<<"$envelope" 2>/dev/null || printf '0')")" + if [ "$retry_after" -gt 0 ]; then + local retry_deadline observed_at + retry_deadline="$(arl_sanitize_int "$(jq -r '.retry_until? // 0' <<<"$envelope" 2>/dev/null || printf '0')")" + if [ "$retry_deadline" -eq 0 ]; then + observed_at="$(arl_sanitize_int "$(jq -r '.observed_at? // 0' <<<"$envelope" 2>/dev/null || printf '0')")" + if [ "$observed_at" -gt 0 ]; then + retry_deadline=$(( observed_at + retry_after )) + fi + fi + if [ "$retry_deadline" -eq 0 ]; then + arl_log "warning: token-budget 429 has no deadline anchor (no retry_until/observed_at) — treating as stale, allowing dispatch (degraded)" + printf 'allow' + return 0 + fi + if [ "$now" -ge "$retry_deadline" ]; then + arl_log "warning: token-budget 429 retry window expired (deadline=${retry_deadline}, now=${now}) — allowing dispatch (degraded)" + printf 'allow' + return 0 + fi + arl_log "token-budget: fresh 429 with retry-after=${retry_after}s — deferring (hard cap signal)" + printf 'defer' + return 0 + fi + fi + + # Any other non-200 fails safe: allow-with-warning. + if [ "$status" -ne 200 ]; then + arl_log "warning: token-budget telemetry unavailable (status=${status}) — allowing dispatch (degraded)" + printf 'allow' + return 0 + fi + + printf 'proceed' +} + # --------------------------------------------------------------------------- # arl_token_budget_gate [window] — orchestrate the token-budget breaker for # (default: session). Reads the config threshold, consults the @@ -714,64 +773,322 @@ arl_token_budget_gate() { return 0 fi - local now envelope status retry_after body + local now envelope transport body now="$(arl_now)" - envelope="$(arl_token_fetch_envelope)" - status="$(arl_sanitize_int "$(jq -r '.status? // 0' <<<"$envelope" 2>/dev/null || printf '0')")" + envelope="${2:-$(arl_token_fetch_envelope)}" + + # Shared transport-level fail-safe (429 / non-200) — one convention for both + # breakers. + transport="$(arl_token_transport_decision "$envelope" "$now")" + case "$transport" in + defer) printf 'decision=defer\n'; return 1 ;; + allow) printf 'decision=allow\n'; return 0 ;; + esac - # Fresh 429 with a positive retry-after: the one telemetry response that - # blocks (ADR §7) — a hard, first-hand signal the cap is already hit. - # An observation timestamp (observed_at) or absolute deadline (retry_until) - # in the envelope guards against a stale file deferring forever: if the - # deadline has already passed, treat the 429 as expired degraded telemetry - # and allow with a warning. - if [ "$status" -eq 429 ]; then - retry_after="$(arl_sanitize_int "$(jq -r '.retry_after? // 0' <<<"$envelope" 2>/dev/null || printf '0')")" - if [ "$retry_after" -gt 0 ]; then - local retry_deadline observed_at - # Prefer explicit retry_until; fall back to observed_at + retry_after. - retry_deadline="$(arl_sanitize_int "$(jq -r '.retry_until? // 0' <<<"$envelope" 2>/dev/null || printf '0')")" - if [ "$retry_deadline" -eq 0 ]; then - observed_at="$(arl_sanitize_int "$(jq -r '.observed_at? // 0' <<<"$envelope" 2>/dev/null || printf '0')")" - if [ "$observed_at" -gt 0 ]; then - retry_deadline=$(( observed_at + retry_after )) - fi - fi - if [ "$retry_deadline" -gt 0 ] && [ "$now" -ge "$retry_deadline" ]; then - arl_log "warning: token-budget 429 retry window expired (deadline=${retry_deadline}, now=${now}) — allowing dispatch (degraded)" - printf 'decision=allow\n' - return 0 - fi - arl_log "token-budget: fresh 429 with retry-after=${retry_after}s — deferring (hard cap signal)" - printf 'decision=defer\n' - return 1 - fi + body="$(jq -c '.body? // {}' <<<"$envelope" 2>/dev/null || printf '{}')" + + if [ "$(arl_token_window_active "$body" "$window")" = "false" ]; then + arl_log "token-budget: window '${window}' is not active — allowing (not binding)" + printf 'decision=allow\n' + return 0 fi - # Any other non-200 fails safe: allow-with-warning. - if [ "$status" -ne 200 ]; then - arl_log "warning: token-budget telemetry unavailable (status=${status}) — allowing dispatch (degraded)" + local percent + percent="$(arl_token_extract_percent "$body" "$window")" + if [ -z "$percent" ]; then + arl_log "warning: token-budget telemetry has no '${window}' window entry — allowing dispatch (degraded)" printf 'decision=allow\n' return 0 fi + arl_token_budget_decision "$percent" "$threshold" "true" +} + +# =========================================================================== +# Weekly-budget GLIDE-PATH breaker (#994, Phase 6 of epic #636; ADR §7) +# --------------------------------------------------------------------------- +# The orthogonal, TIME-VARYING control on the account-wide `weekly_all` (7-day, +# FIXED) window that the static session breaker cannot express: pause new dispatch +# when +# weekly_all.percent >= clamp(ceiling_pct - reserve_pct_per_day * days_until_reset, +# floor_pct, ceiling_pct) +# so the fleet reserves ~reserve_pct_per_day% of the weekly budget per remaining +# day and cannot burn the whole week in the first two days. Every number is read +# from config (org_wide.token_budget.limits.weekly_all) — no threshold, slope, or +# weekday is hardcoded (AC #1). +# +# What makes this breaker DIFFERENT from the session breaker (keep this explicit): +# it CAN close before `resets_at` — but never because utilization fell. Within a +# fixed window percent is monotonically non-decreasing; the glide-path threshold +# RISES as the reset approaches, so each poll re-evaluates statelessly and the +# breaker may close because the THRESHOLD moved past a static percent, not because +# usage dipped. There is deliberately NO percentage resume mark (conflating this +# with the session breaker's "resets_at only" rule in either direction is the bug +# to avoid). Because each poll is a fresh, stateless re-evaluation, no persistent +# trip state is stored and a downward percentage dip (which cannot occur in a +# fixed window) is structurally unable to un-trip it. +# +# Reuses Phase 5's telemetry adapter (one call serves both windows), the shared +# transport-level fail-safe (arl_token_transport_decision), the percent extractor, +# the is_active check, the pause-worthy scope guard, and the pure decision core. +# =========================================================================== + +# --------------------------------------------------------------------------- +# arl_token_glide_int — echo a numeric weekly_all glide from config, +# or empty when absent/non-integer (the caller degrades to allow — a threshold +# must never be hardcoded, AC #1). +# --------------------------------------------------------------------------- +arl_token_glide_int() { + local key="$1" config value + config="$(arl_config_path)" + value="$(jq -er --arg k "$key" \ + '(.org_wide.token_budget.limits.weekly_all[$k])? // empty' "$config" 2>/dev/null || printf '')" + if [[ "$value" =~ ^[0-9]+$ ]]; then + printf '%s' "$value" + fi +} + +arl_token_glide_reserve() { arl_token_glide_int reserve_pct_per_day; } +arl_token_glide_floor() { arl_token_glide_int floor_pct; } +arl_token_glide_ceiling() { arl_token_glide_int ceiling_pct; } + +# --------------------------------------------------------------------------- +# arl_token_glide_enabled — echo "true"/"false" for the weekly_all.enabled config +# arm. Missing/false degrades to "false" (inert): the glide breaker stays off +# until a maintainer arms it in config, independent of the AGENT_TOKEN_BUDGET_ENABLED +# env flag that arms integration (staged rollout, AC #6/#7). +# --------------------------------------------------------------------------- +arl_token_glide_enabled() { + local config value + config="$(arl_config_path)" + value="" + if [ -n "$config" ]; then + value="$(jq -r '(.org_wide.token_budget.limits.weekly_all.enabled)? // "false"' "$config" 2>/dev/null || printf 'false')" + fi + case "$value" in + true) printf 'true' ;; + *) printf 'false' ;; + esac +} + +# --------------------------------------------------------------------------- +# arl_token_iso_to_epoch — echo the epoch seconds for an ISO-8601 +# timestamp (the telemetry `resets_at`), or empty when unparseable. Kept separate +# from the pure day arithmetic so parsing (date-dependent) and the round-up rule +# (pure, unit-tested) do not entangle. +# --------------------------------------------------------------------------- +arl_token_iso_to_epoch() { + local iso="${1:-}" epoch + [ -z "$iso" ] && return 0 + # Require an explicit timezone offset (Z or ±HH:MM) so date -d does not + # silently interpret a timezone-less value as local time. + if [[ ! "$iso" =~ (Z|[+-][0-9]{2}:[0-9]{2})$ ]]; then + return 0 + fi + epoch="$(date -d "$iso" +%s 2>/dev/null || printf '')" + [[ "$epoch" =~ ^[0-9]+$ ]] || return 0 + # Round-trip: if date normalised an invalid calendar date (e.g. Feb 30 → Mar 2) + # the reconstructed YYYY-MM-DD will not match the input prefix — reject it. + local reconstructed_date + reconstructed_date="$(date -u -d "@$epoch" +%Y-%m-%d 2>/dev/null || printf '')" + [ "$reconstructed_date" = "${iso:0:10}" ] || return 0 + printf '%s' "$epoch" +} + +# --------------------------------------------------------------------------- +# arl_token_days_until_reset — PURE. Echo the whole +# days remaining until the reset, rounding a PARTIAL day UP (AC #2): the day +# before reset evaluates as 1 day left, not 0, so the reserve is held for any +# fraction of a day that remains. At or after the reset instant echoes 0. +# --------------------------------------------------------------------------- +arl_token_days_until_reset() { + local reset now diff + reset="$(arl_sanitize_int "${1:-}")" + now="$(arl_sanitize_int "${2:-}")" + diff=$(( reset - now )) + if [ "$diff" -le 0 ]; then + printf '0' + return 0 + fi + # Ceiling division by 86400 (integer): (diff + 86399) / 86400. + printf '%s' "$(( (diff + 86399) / 86400 ))" +} + +# --------------------------------------------------------------------------- +# arl_token_glide_threshold +# — PURE. Echo the glide-path pause +# threshold: clamp(ceiling - reserve*days, floor, ceiling). The threshold RISES +# toward the ceiling as the reset approaches (fewer days left) and is held at the +# floor far from the reset (the "don't spend the whole week on day one" guard). +# --------------------------------------------------------------------------- +arl_token_glide_threshold() { + local days reserve floor ceiling raw + days="$(arl_sanitize_int "${1:-}")" + reserve="$(arl_sanitize_int "${2:-}")" + floor="$(arl_sanitize_int "${3:-}")" + ceiling="$(arl_sanitize_int "${4:-}")" + raw=$(( ceiling - reserve * days )) + if [ "$raw" -gt "$ceiling" ]; then raw="$ceiling"; fi + if [ "$raw" -lt "$floor" ]; then raw="$floor"; fi + printf '%s' "$raw" +} + +# --------------------------------------------------------------------------- +# arl_token_extract_resets_at — PURE. Echo the ISO `resets_at` +# for , preferring the ADR §4.1 limits[] entry (matched on kind) and +# falling back to the flattened key (weekly_all -> seven_day). Empty when absent +# (the caller degrades to allow — days_until_reset cannot be computed). +# --------------------------------------------------------------------------- +arl_token_extract_resets_at() { + local body="$1" window="$2" flat value + case "$window" in + session) flat="five_hour" ;; + weekly_all) flat="seven_day" ;; + *) flat="" ;; + esac + value="$(jq -r --arg k "$window" --arg f "$flat" ' + ( [ .limits[]? | select(.kind == $k) | .resets_at ] | .[0] ) as $from_limits + | ( if $from_limits != null then $from_limits + elif ($f != "" and (.[$f].resets_at? != null)) then .[$f].resets_at + else null end ) + | if . == null then empty else . end + ' <<<"$body" 2>/dev/null || printf '')" + if [ -n "$value" ]; then + printf '%s' "$value" + fi +} + +# --------------------------------------------------------------------------- +# arl_token_glide_trip_reason — echo a +# compact, MACHINE-READABLE trip reason (AC #4): which window, the observed +# percent, the computed threshold, the days until reset, and the reset timestamp +# — so the reason is a parseable object, not prose buried in a log line. +# --------------------------------------------------------------------------- +arl_token_glide_trip_reason() { + local percent threshold days resets_at + percent="$(arl_sanitize_int "${1:-}")" + threshold="$(arl_sanitize_int "${2:-}")" + days="$(arl_sanitize_int "${3:-}")" + resets_at="${4:-}" + jq -cn \ + --arg w "weekly_all" \ + --argjson p "$percent" \ + --argjson t "$threshold" \ + --argjson d "$days" \ + --arg r "$resets_at" \ + '{window: $w, observed_percent: $p, threshold_pct: $t, days_until_reset: $d, resets_at: $r}' +} + +# --------------------------------------------------------------------------- +# arl_token_weekly_glide_gate — orchestrate the weekly glide-path breaker for the +# account-wide `weekly_all` window. Reads the glide config, consults the shared +# telemetry adapter seam, derives days_until_reset from the payload's `resets_at`, +# computes the time-varying threshold, and emits decision=allow/defer. +# +# Decision order: +# 1. Config weekly_all.enabled != true -> allow (inert; config arm) +# 2. weekly_all not pause-worthy -> allow (scope guard) +# 3. Incomplete glide config (reserve/floor/ceil) -> allow (degraded) +# 4. Fresh 429 + retry-after -> defer (hard cap signal) +# 5. Any other non-200 / unavailable telemetry -> allow-with-warning +# 6. weekly_all explicitly inactive -> allow (not binding) +# 7. Missing weekly_all percent in a 200 body -> allow-with-warning +# 8. Missing / unparseable resets_at -> allow-with-warning +# 9. percent >= glide threshold -> defer, else allow +# +# Returns 0 on allow, 1 on defer. Guard-only: reads telemetry, never mutates. +# --------------------------------------------------------------------------- +arl_token_weekly_glide_gate() { + local window="weekly_all" + + # 1. Config-level arm — inert until a maintainer enables it in config, even + # when the integration env flag is on (staged dry-run rollout, AC #7). + if [ "$(arl_token_glide_enabled)" != "true" ]; then + arl_log "token-budget weekly-glide: not enabled in config (weekly_all.enabled) — allowing (inert)" + printf 'decision=allow\n' + return 0 + fi + + # 2. Scope guard — only the account-wide window is pause-worthy (ADR §2.5). + if [ "$(arl_token_window_pause_worthy "$window")" != "true" ]; then + arl_log "token-budget weekly-glide: window '${window}' is not pause-worthy — allowing (scope guard)" + printf 'decision=allow\n' + return 0 + fi + + # 3. The schedule must be fully expressible from config; a missing part + # degrades to allow rather than hardcoding a fallback (AC #1). + local reserve floor ceiling + reserve="$(arl_token_glide_reserve)" + floor="$(arl_token_glide_floor)" + ceiling="$(arl_token_glide_ceiling)" + if [ -z "$reserve" ] || [ -z "$floor" ] || [ -z "$ceiling" ]; then + arl_log "token-budget weekly-glide: incomplete glide config (reserve/floor/ceiling) — allowing (degraded)" + printf 'decision=allow\n' + return 0 + fi + + local now envelope transport body + now="$(arl_now)" + envelope="${1:-$(arl_token_fetch_envelope)}" + + # 4/5. Shared transport-level fail-safe (429 / non-200) — one convention. + transport="$(arl_token_transport_decision "$envelope" "$now")" + case "$transport" in + defer) printf 'decision=defer\n'; return 1 ;; + allow) printf 'decision=allow\n'; return 0 ;; + esac + body="$(jq -c '.body? // {}' <<<"$envelope" 2>/dev/null || printf '{}')" + # 6. An inactive window is not binding. if [ "$(arl_token_window_active "$body" "$window")" = "false" ]; then - arl_log "token-budget: window '${window}' is not active — allowing (not binding)" + arl_log "token-budget weekly-glide: window '${window}' is not active — allowing (not binding)" printf 'decision=allow\n' return 0 fi + # 7. Missing percent -> cannot evaluate -> allow-with-warning (AC #5). local percent percent="$(arl_token_extract_percent "$body" "$window")" if [ -z "$percent" ]; then - arl_log "warning: token-budget telemetry has no '${window}' window entry — allowing dispatch (degraded)" + arl_log "warning: token-budget weekly-glide telemetry has no '${window}' window entry — allowing dispatch (degraded)" printf 'decision=allow\n' return 0 fi - arl_token_budget_decision "$percent" "$threshold" "true" + # 8. days_until_reset derives from the authoritative resets_at, never a + # hardcoded weekday (AC #2). A missing/unparseable value fails safe. + local resets_at reset_epoch + resets_at="$(arl_token_extract_resets_at "$body" "$window")" + if [ -z "$resets_at" ]; then + arl_log "warning: token-budget weekly-glide telemetry has no resets_at for '${window}' — allowing dispatch (degraded)" + printf 'decision=allow\n' + return 0 + fi + reset_epoch="$(arl_token_iso_to_epoch "$resets_at")" + if [ -z "$reset_epoch" ]; then + arl_log "warning: token-budget weekly-glide could not parse resets_at='${resets_at}' — allowing dispatch (degraded)" + printf 'decision=allow\n' + return 0 + fi + + # 9. Compute the time-varying threshold and evaluate (reusing the pure core). + local days threshold decision_out + days="$(arl_token_days_until_reset "$reset_epoch" "$now")" + threshold="$(arl_token_glide_threshold "$days" "$reserve" "$floor" "$ceiling")" + arl_log "token-budget weekly-glide: ${days} day(s) until reset (${resets_at}) -> threshold ${threshold}% vs observed ${percent}%" + + # `|| true`: the pure core returns non-zero on defer; the decision rides in + # stdout — neutralize the exit so a `set -e` caller is not aborted here. + decision_out="$(arl_token_budget_decision "$percent" "$threshold" "true")" || true + if [ "$decision_out" = "decision=defer" ]; then + arl_log "token-budget weekly-glide TRIP: $(arl_token_glide_trip_reason "$percent" "$threshold" "$days" "$resets_at")" + printf 'decision=defer\n' + return 1 + fi + printf 'decision=allow\n' + return 0 } # --------------------------------------------------------------------------- @@ -874,13 +1191,26 @@ arl_admission_gate() { # defer returns non-zero — neutralize it so a `set -e` caller is not # aborted before the decision is emitted. if [ "${AGENT_TOKEN_BUDGET_ENABLED:-false}" = "true" ]; then - local token_decision - token_decision="$(arl_token_budget_gate session)" || true + local token_decision glide_decision shared_envelope + shared_envelope="$(arl_token_fetch_envelope)" + token_decision="$(arl_token_budget_gate session "$shared_envelope")" || true if [ "$token_decision" = "decision=defer" ]; then arl_log "token-budget escalation: $(arl_token_breaker_marker session) — add to tracking issue/PR body, remove when cleared" arl_finish "$agent_type" "defer" "org-wide token-budget breaker is open (5-hour Claude session window at/over threshold)" return $? fi + + # 2.6 Weekly glide-path breaker (Phase 6, #994). Orthogonal to the 5-hour + # session breaker: a time-varying threshold on the FIXED 7-day weekly_all + # window. Shares the same env arm; the config weekly_all.enabled flag keeps + # it inert until sign-off even when this env flag is on. `|| true`: the + # decision rides in captured stdout and a defer returns non-zero. + glide_decision="$(arl_token_weekly_glide_gate "$shared_envelope")" || true + if [ "$glide_decision" = "decision=defer" ]; then + arl_log "token-budget escalation: $(arl_token_breaker_marker weekly_all) — add to tracking issue/PR body, remove when cleared" + arl_finish "$agent_type" "defer" "org-wide token-budget breaker is open (7-day Claude weekly window over glide-path threshold)" + return $? + fi fi local now state diff --git a/standards/agent-rate-limits.json b/standards/agent-rate-limits.json index 3471de888..5558fa1a0 100644 --- a/standards/agent-rate-limits.json +++ b/standards/agent-rate-limits.json @@ -65,8 +65,11 @@ "kind": "weekly_all", "window_days": 7, "pause_worthy": true, + "enabled": false, "reserve_pct_per_day": 2, - "_note": "PROPOSAL (ADR §7). Glide-path breaker: pause when percent >= 100 - (reserve_pct_per_day * days_until_reset). reserve_pct_per_day defaults to 2 (INERT, pending sign-off). The threshold RISES as the reset approaches, so the breaker can close before resets_at because the threshold moved — never because utilization fell. days_until_reset derives from the authoritative resets_at (ADR §4.1), not a hardcoded weekday." + "floor_pct": 86, + "ceiling_pct": 100, + "_note": "PROPOSAL (ADR §7; Phase 6 #994). Glide-path breaker: pause when percent >= clamp(ceiling_pct - (reserve_pct_per_day * days_until_reset), floor_pct, ceiling_pct). With reserve_pct_per_day=2, ceiling_pct=100, floor_pct=86 the schedule is 100% on reset day .. 86% just after a reset — the whole schedule is expressible from these keys, nothing is hardcoded. INERT: enabled=false keeps the breaker off (a maintainer arms it in config), independent of the AGENT_TOKEN_BUDGET_ENABLED integration flag. The threshold RISES as the reset approaches, so the breaker can close before resets_at because the threshold moved past a static (monotonic) percent — never because utilization fell; there is deliberately no percentage resume mark. days_until_reset derives from the authoritative resets_at (ADR §4.1) with partial days rounded UP, not a hardcoded weekday. floor_pct is the 'don't spend the whole week on day one' guard; ceiling_pct is the reset-day cap where the budget is minutes from refilling." }, "weekly_scoped": { "kind": "weekly_scoped", diff --git a/standards/agent-rate-limits.md b/standards/agent-rate-limits.md index bf63ebbaf..208076955 100644 --- a/standards/agent-rate-limits.md +++ b/standards/agent-rate-limits.md @@ -143,7 +143,8 @@ per-agent-type controls: - The adapter reads both the `session` and `weekly_all` windows from one envelope, so the Phase 6 weekly glide-path breaker ([#994](https://github.com/petry-projects/.github/issues/994)) extends it - rather than refactoring. + rather than refactoring. Both breakers share one transport-level fail-safe + (`arl_token_transport_decision`) so there is a single 429 / non-200 convention. - When the breaker trips, the pause is surfaced through the same human-clearable primitives as the rest of the library: `arl_token_breaker_marker ` (a deduped HTML marker) plus `arl_breaker_label` (`needs-human-review`). A human @@ -151,6 +152,30 @@ per-agent-type controls: encodes the Claude-priority-on-recovery order (Claude-backed agents ahead of `initiative-driver`) from the `claude_priority` flag. +- `arl_token_weekly_glide_gate` orchestrates the **7-day glide-path** breaker on + the account-wide `weekly_all` window (Phase 6, + [#994](https://github.com/petry-projects/.github/issues/994)). Rather than a + static threshold it evaluates a **time-varying** one read entirely from config — + `clamp(ceiling_pct − reserve_pct_per_day × days_until_reset, floor_pct, + ceiling_pct)` — where `days_until_reset` derives from the telemetry payload's + authoritative `resets_at` (`arl_token_iso_to_epoch` + + `arl_token_days_until_reset`, partial days rounded **up**), never a hardcoded + weekday. The schedule is expressible purely from the `weekly_all` config keys + (`enabled`, `reserve_pct_per_day`, `floor_pct`, `ceiling_pct`); no number is + hardcoded. On a trip the reason is **machine-readable** + (`arl_token_glide_trip_reason` → `{window, observed_percent, threshold_pct, + days_until_reset, resets_at}`), surfaced through the same + `arl_token_breaker_marker weekly_all` / `arl_breaker_label` primitives. + + **This breaker can close before `resets_at`** — but never because utilization + fell. Within a fixed window percent is monotonically non-decreasing; the + glide-path threshold **rises** as the reset approaches, so each poll + re-evaluates statelessly and the breaker may close because the *threshold moved + past a static percent*, not because usage dipped. This is the single most + misread part of the design: it is distinct from the `session` breaker's + "closes only at `resets_at`" rule, yet there is still no percentage resume mark + for either breaker. + ### 4.2 Rollout — canary / dry-run first, then promotion (AC #5) Activation is **staged and reversible**, and never overrides a pause a human set @@ -176,6 +201,15 @@ deliberately (ADR §7, `.github-private#1525`): canary is clean. Because the threshold is read from this config at run time, tuning it afterward is a config edit (§7.1), not a redeploy. +**The weekly glide-path breaker carries a second, config-level arm.** In addition +to the shared `AGENT_TOKEN_BUDGET_ENABLED` env flag, `arl_token_weekly_glide_gate` +is gated by the `weekly_all.enabled` config flag (**off by default**). This lets +the `session` breaker be enabled while the weekly glide stays inert, so the +glide's own dry-run week — logging the decision it *would* have made against live +telemetry for a full weekly window before it can defer anything (AC #7) — is a +config toggle independent of the session breaker's rollout. With `enabled: false` +the gate reads no telemetry and cannot change any decision. + **Degraded / disabled-behind-flag mode (AC #6).** If the private telemetry companion is not yet wired (no seam configured) the breaker **allows with a warning** rather than blocking — the story ships complete regardless of the diff --git a/tests/test_agent_rate_limit_token_budget.bats b/tests/test_agent_rate_limit_token_budget.bats index c1778c821..6dd0b7855 100644 --- a/tests/test_agent_rate_limit_token_budget.bats +++ b/tests/test_agent_rate_limit_token_budget.bats @@ -319,9 +319,10 @@ envelope_limits() { [[ "$output" == *"decision=defer"* ]] } -@test "gate: a fresh 429 with retry-after BLOCKS (hard cap signal, ADR §7)" { +@test "gate: a fresh 429 with a future deadline BLOCKS (hard cap signal, ADR §7)" { write_token_config 90 true - write_telemetry '{"status":429,"retry_after":120}' + local future_deadline; future_deadline=$(( $(date +%s) + 300 )) + write_telemetry "{\"status\":429,\"retry_after\":120,\"retry_until\":${future_deadline}}" run bash -c "source '$LIB'; arl_token_budget_gate session" [ "$status" -eq 1 ] [[ "$output" == *"decision=defer"* ]] @@ -531,6 +532,14 @@ envelope_limits() { [[ "$output" == *"decision=defer"* ]] } +@test "gate: a 429 with no deadline anchor (no retry_until/observed_at) allows, not defers (stale cached response)" { + write_token_config 90 true + write_telemetry '{"status":429,"retry_after":3600}' + run bash -c "source '$LIB'; arl_token_budget_gate session" + [ "$status" -eq 0 ] + [[ "$output" == *"decision=allow"* ]] +} + # -------------------------------------------------------------------------- # Fail-open: empty/error from arl_token_budget_gate must not defer (gemini: line 862) # -------------------------------------------------------------------------- diff --git a/tests/test_agent_rate_limit_weekly_glide.bats b/tests/test_agent_rate_limit_weekly_glide.bats new file mode 100644 index 000000000..2e943f92c --- /dev/null +++ b/tests/test_agent_rate_limit_weekly_glide.bats @@ -0,0 +1,648 @@ +#!/usr/bin/env bats +# Tests for the org-wide weekly-budget GLIDE-PATH circuit breaker added to +# scripts/lib/agent-rate-limit.sh — the time-varying breaker that pauses new +# agentic dispatch when the shared, FIXED 7-day Claude subscription +# (`weekly_all`) window crosses a threshold that RISES as the weekly reset +# approaches (#994, Phase 6 of epic #636). +# +# Context (ADR docs/initiatives/agent-rate-limits-adr.md, #637; post-ADR +# clarification in the issue): +# - Extends the Phase-5 (#641) telemetry adapter, which already serves BOTH the +# `session` (5-hour) and `weekly_all` (7-day) windows from one call — so no +# unit test touches the network (mocked adapter seam only). +# - The threshold is `clamp(ceiling_pct - reserve_pct_per_day * days_until_reset, +# floor_pct, ceiling_pct)`. With reserve=2, ceiling=100, floor=86 the schedule +# is 100% (reset day) .. 86% (just reset). No number is hardcoded (AC #1); the +# schedule is expressible purely as config. +# - `days_until_reset` derives from the telemetry payload's authoritative +# `resets_at`, never a hardcoded weekday. Partial days round UP so the day +# before reset is 1 day left (98%), not 0 (AC #2). +# - Keys on the account-wide `weekly_all` window; a `weekly_scoped` (per-model) +# limit at critical/is_active must NOT trip a fleet pause (AC #3). +# - Fail-safe direction (ADR §7): on telemetry error (non-200, malformed body, +# missing window/resets_at entry) the breaker ALLOWS dispatch with a warning; +# a fresh 429 + retry-after BLOCKS (a hard first-hand cap signal) (AC #5). +# - The breaker can CLOSE before `resets_at` — but only because the rising +# threshold moved past a static (monotonic) percent, never because usage fell. +# There is deliberately no percentage resume mark. +# - Integration into arl_admission_gate ships INERT behind +# AGENT_TOKEN_BUDGET_ENABLED (env arm) AND the config `weekly_all.enabled` +# flag (canary / dry-run rollout, AC #6/#7). + +bats_require_minimum_version 1.5.0 + +LIB="$(cd "$BATS_TEST_DIRNAME/.." && pwd)/scripts/lib/agent-rate-limit.sh" +GH_STUB_SRC="$(cd "$BATS_TEST_DIRNAME/.." && pwd)/test/scripts/compliance-remediate/stubs/gh" + +setup() { + TMP="$(mktemp -d "$BATS_TEST_TMPDIR/stub.XXXXXX")" + export TMP + + mkdir -p "$TMP/bin" + cp "$GH_STUB_SRC" "$TMP/bin/gh" + chmod +x "$TMP/bin/gh" + PATH="$TMP/bin:$PATH" + export PATH + export GH_STUB_LOG="$TMP/gh.log" + : >"$GH_STUB_LOG" +} + +teardown() { + if [ -n "${TMP:-}" ] && [ -d "$TMP" ]; then + rm -rf "$TMP" + fi +} + +# Write an agent-rate-limits config carrying a weekly_all glide block. Args: +# $1 enabled (default true), $2 reserve (2), $3 floor (86), $4 ceiling (100), +# $5 session threshold (90). +write_glide_config() { + local enabled="${1:-true}" reserve="${2:-2}" floor="${3:-86}" ceiling="${4:-100}" session="${5:-90}" + export AGENT_RATE_LIMITS_CONFIG="$TMP/agent-rate-limits.json" + jq -n \ + --argjson enabled "$enabled" \ + --argjson reserve "$reserve" \ + --argjson floor "$floor" \ + --argjson ceiling "$ceiling" \ + --argjson session "$session" \ + '{ + status: "provisional", + _schema_version: 1, + agent_types: { + "dev-lead": { + max_concurrent_runs: 3, + max_runtime_minutes: 30, + cooldown_minutes: 5, + daily_run_budget: 50, + circuit_breaker: { consecutive_failure_threshold: 3, backoff_minutes: 30 } + } + }, + org_wide: { + token_budget: { + claude_priority: true, + limits: { + session: { kind: "session", window_hours: 5, pause_worthy: true, pause_threshold_pct: $session }, + weekly_all: { kind: "weekly_all", window_days: 7, pause_worthy: true, enabled: $enabled, reserve_pct_per_day: $reserve, floor_pct: $floor, ceiling_pct: $ceiling }, + weekly_scoped: { kind: "weekly_scoped", pause_worthy: false } + } + } + }, + exempt_actors: ["dependabot[bot]", "@petry-projects/org-leads"], + exempt_labels: ["security"] + }' >"$AGENT_RATE_LIMITS_CONFIG" +} + +write_telemetry() { + export AGENT_TOKEN_BUDGET_TELEMETRY_FILE="$TMP/telemetry.json" + printf '%s' "$1" >"$AGENT_TOKEN_BUDGET_TELEMETRY_FILE" +} + +# A 200 envelope whose weekly_all limit carries the given percent + resets_at, +# and a benign under-threshold session so only the glide path can trip. +envelope_weekly() { + local weekly_pct="$1" resets_at="$2" session_pct="${3:-10}" + jq -nc --argjson w "$weekly_pct" --arg r "$resets_at" --argjson s "$session_pct" '{ + status: 200, + body: { + limits: [ + { kind: "session", percent: $s, severity: "normal", resets_at: "2099-01-01T00:00:00Z", is_active: true }, + { kind: "weekly_all", percent: $w, severity: "normal", resets_at: $r, is_active: true }, + { kind: "weekly_scoped", percent: 100, severity: "critical", resets_at: $r, is_active: true } + ] + } + }' +} + +# Epoch for an ISO timestamp (GNU date). +iso_epoch() { date -d "$1" +%s; } + +# -------------------------------------------------------------------------- +# Source-time contract — the new functions are defined, sourcing is inert. +# -------------------------------------------------------------------------- +@test "sourcing defines the weekly-glide functions and is side-effect-free" { + run bash -c "set -euo pipefail; source '$LIB'; \ + for f in arl_token_glide_enabled arl_token_glide_reserve arl_token_glide_floor \ + arl_token_glide_ceiling arl_token_days_until_reset arl_token_glide_threshold \ + arl_token_iso_to_epoch arl_token_extract_resets_at arl_token_weekly_glide_gate \ + arl_token_glide_trip_reason; do \ + declare -F \"\$f\" >/dev/null || { echo \"missing \$f\"; exit 1; }; done; echo ok" + [ "$status" -eq 0 ] + [[ "$output" == *"ok"* ]] +} + +# -------------------------------------------------------------------------- +# Config accessors (AC #1) — every glide value read from config, never hardcoded. +# -------------------------------------------------------------------------- +@test "glide config accessors read reserve/floor/ceiling/enabled from config" { + write_glide_config true 2 86 100 90 + run bash -c "source '$LIB'; arl_token_glide_reserve" + [ "$output" = "2" ] + run bash -c "source '$LIB'; arl_token_glide_floor" + [ "$output" = "86" ] + run bash -c "source '$LIB'; arl_token_glide_ceiling" + [ "$output" = "100" ] + run bash -c "source '$LIB'; arl_token_glide_enabled" + [ "$output" = "true" ] +} + +@test "glide accessors honor non-default configured values (not hardcoded)" { + write_glide_config false 3 80 99 90 + run bash -c "source '$LIB'; arl_token_glide_reserve" + [ "$output" = "3" ] + run bash -c "source '$LIB'; arl_token_glide_floor" + [ "$output" = "80" ] + run bash -c "source '$LIB'; arl_token_glide_ceiling" + [ "$output" = "99" ] + run bash -c "source '$LIB'; arl_token_glide_enabled" + [ "$output" = "false" ] +} + +@test "arl_token_glide_enabled defaults to false when the key is absent" { + export AGENT_RATE_LIMITS_CONFIG="$TMP/agent-rate-limits.json" + jq -n '{org_wide:{token_budget:{limits:{weekly_all:{kind:"weekly_all",pause_worthy:true}}}}}' >"$AGENT_RATE_LIMITS_CONFIG" + run bash -c "source '$LIB'; arl_token_glide_enabled" + [ "$output" = "false" ] +} + +# -------------------------------------------------------------------------- +# days_until_reset — round-up semantics (AC #2). PURE: . +# -------------------------------------------------------------------------- +@test "days_until_reset is 0 at the reset instant and after it has passed" { + run bash -c "source '$LIB'; arl_token_days_until_reset 1000000 1000000" + [ "$output" = "0" ] + run bash -c "source '$LIB'; arl_token_days_until_reset 1000000 1000001" + [ "$output" = "0" ] +} + +@test "days_until_reset is exactly N for N whole days remaining (0..7)" { + local reset=1000000000 + for n in 0 1 2 3 4 5 6 7; do + local now=$(( reset - n * 86400 )) + run bash -c "source '$LIB'; arl_token_days_until_reset $reset $now" + [ "$output" = "$n" ] + done +} + +@test "days_until_reset rounds a PARTIAL day UP (12h left => 1 day, not 0)" { + local reset=1000000000 + local now=$(( reset - 43200 )) # 12 hours before reset + run bash -c "source '$LIB'; arl_token_days_until_reset $reset $now" + [ "$output" = "1" ] +} + +@test "days_until_reset rounds up just over a whole day to the next day" { + local reset=1000000000 + local now=$(( reset - 86401 )) # 24h + 1s before reset + run bash -c "source '$LIB'; arl_token_days_until_reset $reset $now" + [ "$output" = "2" ] +} + +# -------------------------------------------------------------------------- +# glide_threshold — pure clamp core (AC #1). . +# -------------------------------------------------------------------------- +@test "glide_threshold matches the issue schedule at each day-offset 0..7" { + local -a expected=(100 98 96 94 92 90 88 86) + for n in 0 1 2 3 4 5 6 7; do + run bash -c "source '$LIB'; arl_token_glide_threshold $n 2 86 100" + [ "$output" = "${expected[$n]}" ] + done +} + +@test "glide_threshold clamps to floor beyond the schedule (days=8 => floor 86)" { + run bash -c "source '$LIB'; arl_token_glide_threshold 8 2 86 100" + [ "$output" = "86" ] + run bash -c "source '$LIB'; arl_token_glide_threshold 100 2 86 100" + [ "$output" = "86" ] +} + +@test "glide_threshold clamps to ceiling on reset day (days=0 => ceiling)" { + run bash -c "source '$LIB'; arl_token_glide_threshold 0 2 86 95" + [ "$output" = "95" ] +} + +@test "glide_threshold honors a non-default reserve slope (config-driven)" { + # reserve=3: day 2 => 100 - 6 = 94 + run bash -c "source '$LIB'; arl_token_glide_threshold 2 3 80 100" + [ "$output" = "94" ] +} + +# -------------------------------------------------------------------------- +# ISO parsing + resets_at extraction adapters. +# -------------------------------------------------------------------------- +@test "arl_token_iso_to_epoch parses an ISO-8601 timestamp" { + local want; want="$(iso_epoch '2026-09-08T15:00:00Z')" + run bash -c "source '$LIB'; arl_token_iso_to_epoch '2026-09-08T15:00:00Z'" + [ "$output" = "$want" ] +} + +@test "arl_token_iso_to_epoch echoes empty for a malformed timestamp" { + run bash -c "source '$LIB'; arl_token_iso_to_epoch 'not a date'" + [ "$status" -eq 0 ] + [ -z "$output" ] +} + +@test "arl_token_iso_to_epoch echoes empty for a timezone-less timestamp (rejects local-time ambiguity)" { + run bash -c "source '$LIB'; arl_token_iso_to_epoch '2026-09-08T15:00:00'" + [ "$status" -eq 0 ] + [ -z "$output" ] +} + +@test "arl_token_iso_to_epoch echoes empty for an invalid calendar date (e.g. Feb 30) that date -d would normalise" { + run bash -c "source '$LIB'; arl_token_iso_to_epoch '2026-02-30T00:00:00Z'" + [ "$status" -eq 0 ] + [ -z "$output" ] +} + +@test "arl_token_iso_to_epoch accepts a +HH:MM offset timestamp" { + local want; want="$(date -d '2026-09-08T17:00:00+02:00' +%s 2>/dev/null || printf '')" + [ -z "$want" ] && skip "date -d does not support +HH:MM on this platform" + run bash -c "source '$LIB'; arl_token_iso_to_epoch '2026-09-08T17:00:00+02:00'" + [ "$output" = "$want" ] +} + +@test "arl_token_extract_resets_at reads the weekly_all limits[] entry" { + body='{"limits":[{"kind":"weekly_all","percent":40,"resets_at":"2026-09-02T15:59:59Z"}]}' + run bash -c "source '$LIB'; arl_token_extract_resets_at '$body' weekly_all" + [ "$output" = "2026-09-02T15:59:59Z" ] +} + +@test "arl_token_extract_resets_at falls back to the flattened seven_day key" { + body='{"seven_day":{"percent":40,"resets_at":"2026-09-02T15:59:59Z"}}' + run bash -c "source '$LIB'; arl_token_extract_resets_at '$body' weekly_all" + [ "$output" = "2026-09-02T15:59:59Z" ] +} + +@test "arl_token_extract_resets_at echoes empty when absent" { + body='{"limits":[{"kind":"weekly_all","percent":40}]}' + run bash -c "source '$LIB'; arl_token_extract_resets_at '$body' weekly_all" + [ -z "$output" ] +} + +# -------------------------------------------------------------------------- +# Orchestrator arl_token_weekly_glide_gate — boundaries (AC #1/#2) +# -------------------------------------------------------------------------- +@test "gate: weekly percent exactly at the glide threshold defers (>= boundary)" { + write_glide_config + local reset_iso="2026-09-08T15:00:00Z" reset now + reset="$(iso_epoch "$reset_iso")"; now=$(( reset - 4 * 86400 )) # 4 days => threshold 92 + export SOURCE_NOW="$now" + write_telemetry "$(envelope_weekly 92 "$reset_iso")" + run bash -c "source '$LIB'; arl_token_weekly_glide_gate" + [ "$status" -eq 1 ] + [[ "$output" == *"decision=defer"* ]] +} + +@test "gate: one point under the glide threshold allows (boundary)" { + write_glide_config + local reset_iso="2026-09-08T15:00:00Z" reset now + reset="$(iso_epoch "$reset_iso")"; now=$(( reset - 4 * 86400 )) # 4 days => threshold 92 + export SOURCE_NOW="$now" + write_telemetry "$(envelope_weekly 91 "$reset_iso")" + run bash -c "source '$LIB'; arl_token_weekly_glide_gate" + [ "$status" -eq 0 ] + [[ "$output" == *"decision=allow"* ]] +} + +@test "gate: 86% with 7 days left (just reset) defers at the floor" { + write_glide_config + local reset_iso="2026-09-08T15:00:00Z" reset now + reset="$(iso_epoch "$reset_iso")"; now=$(( reset - 7 * 86400 )) # 7 days => threshold 86 + export SOURCE_NOW="$now" + write_telemetry "$(envelope_weekly 86 "$reset_iso")" + run bash -c "source '$LIB'; arl_token_weekly_glide_gate" + [ "$status" -eq 1 ] + [[ "$output" == *"decision=defer"* ]] +} + +@test "gate: 85% with 5 days left defers (the 2026-08-20 motivating probe)" { + # Thursday: 5 days left => threshold 90. 85% is UNDER 90 so it does NOT trip; + # the story's motivating observation was one point under. Assert the near-miss. + write_glide_config + local reset_iso="2026-09-08T15:00:00Z" reset now + reset="$(iso_epoch "$reset_iso")"; now=$(( reset - 5 * 86400 )) # 5 days => threshold 90 + export SOURCE_NOW="$now" + write_telemetry "$(envelope_weekly 85 "$reset_iso")" + run bash -c "source '$LIB'; arl_token_weekly_glide_gate" + [ "$status" -eq 0 ] + [[ "$output" == *"decision=allow"* ]] +} + +@test "gate: partial-day rounding tightens the threshold (12h left => 1 day => 98)" { + write_glide_config + local reset_iso="2026-09-08T15:00:00Z" reset now + reset="$(iso_epoch "$reset_iso")"; now=$(( reset - 43200 )) # 12h => rounds up to 1 day => 98 + export SOURCE_NOW="$now" + # 97 is under 98 -> allow; if rounding were DOWN to 0 the threshold would be 100 + # and 97 would still allow, so also assert 98 trips below. + write_telemetry "$(envelope_weekly 97 "$reset_iso")" + run bash -c "source '$LIB'; arl_token_weekly_glide_gate" + [ "$status" -eq 0 ] + [[ "$output" == *"decision=allow"* ]] + write_telemetry "$(envelope_weekly 98 "$reset_iso")" + run bash -c "source '$LIB'; arl_token_weekly_glide_gate" + [ "$status" -eq 1 ] + [[ "$output" == *"decision=defer"* ]] +} + +# -------------------------------------------------------------------------- +# The differentiator (post-ADR #3): the breaker CLOSES before resets_at because +# the RISING threshold passed a static (monotonic) percent — never because usage +# fell. Same observed percent, two day-offsets, opposite decisions. +# -------------------------------------------------------------------------- +@test "gate: a fixed percent trips farther from reset and closes nearer to it (threshold rose, not usage fell)" { + write_glide_config + local reset_iso="2026-09-08T15:00:00Z" reset + reset="$(iso_epoch "$reset_iso")" + + # 5 days left => threshold 90. Observed 91% (monotonic) -> trips. + export SOURCE_NOW=$(( reset - 5 * 86400 )) + write_telemetry "$(envelope_weekly 91 "$reset_iso")" + run bash -c "source '$LIB'; arl_token_weekly_glide_gate" + [ "$status" -eq 1 ] + [[ "$output" == *"decision=defer"* ]] + + # 4 days left => threshold 92. The SAME 91% is now under the risen threshold -> + # closes. Utilization did not fall; the threshold moved past it. + export SOURCE_NOW=$(( reset - 4 * 86400 )) + write_telemetry "$(envelope_weekly 91 "$reset_iso")" + run bash -c "source '$LIB'; arl_token_weekly_glide_gate" + [ "$status" -eq 0 ] + [[ "$output" == *"decision=allow"* ]] +} + +# -------------------------------------------------------------------------- +# Scope guard (AC #3) — weekly_scoped must never trip a fleet pause. +# -------------------------------------------------------------------------- +@test "gate: a weekly_scoped limit at critical/100% does NOT trip when weekly_all is under threshold" { + write_glide_config + local reset_iso="2026-09-08T15:00:00Z" reset now + reset="$(iso_epoch "$reset_iso")"; now=$(( reset - 5 * 86400 )) # threshold 90 + export SOURCE_NOW="$now" + # weekly_all at 40 (well under 90); weekly_scoped at 100 critical is_active. + write_telemetry "$(envelope_weekly 40 "$reset_iso")" + run bash -c "source '$LIB'; arl_token_weekly_glide_gate" + [ "$status" -eq 0 ] + [[ "$output" == *"decision=allow"* ]] +} + +@test "gate: weekly_scoped window itself is never pause-worthy (scope guard)" { + write_glide_config + run bash -c "source '$LIB'; arl_token_window_pause_worthy weekly_scoped" + [ "$output" = "false" ] +} + +# -------------------------------------------------------------------------- +# Config-level inert arm (AC #6/#7) +# -------------------------------------------------------------------------- +@test "gate: config enabled=false keeps the glide breaker inert (allow) even over threshold" { + write_glide_config false + local reset_iso="2026-09-08T15:00:00Z" reset now + reset="$(iso_epoch "$reset_iso")"; now=$(( reset - 5 * 86400 )) + export SOURCE_NOW="$now" + write_telemetry "$(envelope_weekly 99 "$reset_iso")" + run bash -c "source '$LIB'; arl_token_weekly_glide_gate" + [ "$status" -eq 0 ] + [[ "$output" == *"decision=allow"* ]] +} + +# -------------------------------------------------------------------------- +# Fail-safe degradation (AC #5) +# -------------------------------------------------------------------------- +@test "gate: a fresh 429 with a future deadline BLOCKS (hard cap signal)" { + write_glide_config + local future_deadline; future_deadline=$(( $(date +%s) + 300 )) + write_telemetry "{\"status\":429,\"retry_after\":120,\"retry_until\":${future_deadline}}" + run bash -c "source '$LIB'; arl_token_weekly_glide_gate" + [ "$status" -eq 1 ] + [[ "$output" == *"decision=defer"* ]] + [[ "$output" == *"429"* ]] +} + +@test "gate: a non-200 telemetry status fails safe to allow-with-warning" { + write_glide_config + write_telemetry '{"status":503}' + run bash -c "source '$LIB'; arl_token_weekly_glide_gate" + [ "$status" -eq 0 ] + [[ "$output" == *"decision=allow"* ]] +} + +@test "gate: an unavailable telemetry source (status 0) fails safe to allow" { + write_glide_config + write_telemetry '{"status":0}' + run bash -c "source '$LIB'; arl_token_weekly_glide_gate" + [ "$status" -eq 0 ] + [[ "$output" == *"decision=allow"* ]] +} + +@test "gate: no telemetry source configured fails safe to allow (degraded, AC #6)" { + write_glide_config + run bash -c "unset AGENT_TOKEN_BUDGET_TELEMETRY_FILE AGENT_TOKEN_BUDGET_TELEMETRY_CMD; source '$LIB'; arl_token_weekly_glide_gate" + [ "$status" -eq 0 ] + [[ "$output" == *"decision=allow"* ]] +} + +@test "gate: a missing weekly_all entry in an otherwise-200 body fails safe to allow" { + write_glide_config + write_telemetry '{"status":200,"body":{"limits":[{"kind":"session","percent":50}]}}' + run bash -c "source '$LIB'; arl_token_weekly_glide_gate" + [ "$status" -eq 0 ] + [[ "$output" == *"decision=allow"* ]] +} + +@test "gate: a weekly_all entry with no resets_at fails safe to allow (cannot compute days)" { + write_glide_config + write_telemetry '{"status":200,"body":{"limits":[{"kind":"weekly_all","percent":99,"is_active":true}]}}' + run bash -c "source '$LIB'; arl_token_weekly_glide_gate" + [ "$status" -eq 0 ] + [[ "$output" == *"decision=allow"* ]] +} + +@test "gate: weekly_all flagged is_active=false does not defer even over threshold" { + write_glide_config + local reset_iso="2026-09-08T15:00:00Z" reset now + reset="$(iso_epoch "$reset_iso")"; now=$(( reset - 5 * 86400 )) + export SOURCE_NOW="$now" + write_telemetry "$(jq -nc --arg r "$reset_iso" '{status:200,body:{limits:[{kind:"weekly_all",percent:99,resets_at:$r,is_active:false}]}}')" + run bash -c "source '$LIB'; arl_token_weekly_glide_gate" + [ "$status" -eq 0 ] + [[ "$output" == *"decision=allow"* ]] +} + +@test "gate: is set -euo pipefail-safe on the defer path" { + write_glide_config + local reset_iso="2026-09-08T15:00:00Z" reset now + reset="$(iso_epoch "$reset_iso")"; now=$(( reset - 4 * 86400 )) + export SOURCE_NOW="$now" + write_telemetry "$(envelope_weekly 99 "$reset_iso")" + run bash -c "set -euo pipefail; source '$LIB'; arl_token_weekly_glide_gate" + [ "$status" -eq 1 ] + [[ "$output" == *"decision=defer"* ]] +} + +# -------------------------------------------------------------------------- +# Machine-readable trip reason (AC #4) +# -------------------------------------------------------------------------- +@test "arl_token_glide_trip_reason emits structured JSON with window/percent/threshold/days/resets_at" { + run bash -c "source '$LIB'; arl_token_glide_trip_reason 95 92 4 '2026-09-08T15:00:00Z'" + [ "$status" -eq 0 ] + run bash -c "source '$LIB'; arl_token_glide_trip_reason 95 92 4 '2026-09-08T15:00:00Z' | jq -er '.window'" + [ "$output" = "weekly_all" ] + run bash -c "source '$LIB'; arl_token_glide_trip_reason 95 92 4 '2026-09-08T15:00:00Z' | jq -er '.observed_percent'" + [ "$output" = "95" ] + run bash -c "source '$LIB'; arl_token_glide_trip_reason 95 92 4 '2026-09-08T15:00:00Z' | jq -er '.threshold_pct'" + [ "$output" = "92" ] + run bash -c "source '$LIB'; arl_token_glide_trip_reason 95 92 4 '2026-09-08T15:00:00Z' | jq -er '.days_until_reset'" + [ "$output" = "4" ] + run bash -c "source '$LIB'; arl_token_glide_trip_reason 95 92 4 '2026-09-08T15:00:00Z' | jq -er '.resets_at'" + [ "$output" = "2026-09-08T15:00:00Z" ] +} + +@test "gate: a defer logs the machine-readable trip reason (AC #4)" { + write_glide_config + local reset_iso="2026-09-08T15:00:00Z" reset now + reset="$(iso_epoch "$reset_iso")"; now=$(( reset - 4 * 86400 )) + export SOURCE_NOW="$now" + write_telemetry "$(envelope_weekly 95 "$reset_iso")" + run bash -c "source '$LIB'; arl_token_weekly_glide_gate" 2>&1 + [ "$status" -eq 1 ] + [[ "$output" == *"weekly_all"* ]] + [[ "$output" == *"threshold_pct"* ]] + [[ "$output" == *"resets_at"* ]] +} + +# -------------------------------------------------------------------------- +# Human-clearable marker / label surfacing (AC #4) — reuse the shared idiom. +# -------------------------------------------------------------------------- +@test "the weekly-glide breaker reuses the shared window marker + human-clearable label" { + run bash -c "source '$LIB'; arl_token_breaker_marker weekly_all" + [ "$status" -eq 0 ] + [[ "$output" == *""* ]] + run bash -c "source '$LIB'; arl_breaker_label" + [ "$output" = "needs-human-review" ] +} + +# -------------------------------------------------------------------------- +# Integration into arl_admission_gate (AC #4/#6/#7) — INERT behind the flag. +# -------------------------------------------------------------------------- +@test "admission gate: weekly-glide is INERT when AGENT_TOKEN_BUDGET_ENABLED is unset" { + write_glide_config + local reset_iso="2026-09-08T15:00:00Z" reset now + reset="$(iso_epoch "$reset_iso")"; now=$(( reset - 4 * 86400 )) + export SOURCE_NOW="$now" + write_telemetry "$(envelope_weekly 99 "$reset_iso")" + export AGENT_RATE_LIMITS_STATE="$TMP/state.json" + printf '%s' '{"dev-lead":{"last_run_epoch":0,"daily_count":0,"consecutive_failures":0,"breaker_opened_epoch":0}}' >"$AGENT_RATE_LIMITS_STATE" + export GH_STUB_STDOUT='[{"status":"in_progress"}]' + run bash -c "source '$LIB'; arl_admission_gate dev-lead donpetry-bot" + [ "$status" -eq 0 ] + [[ "$output" == *"decision=allow"* ]] +} + +@test "admission gate: defers when the weekly glide is armed (env + config) and tripped" { + write_glide_config + local reset_iso="2026-09-08T15:00:00Z" reset now + reset="$(iso_epoch "$reset_iso")"; now=$(( reset - 4 * 86400 )) + export SOURCE_NOW="$now" + write_telemetry "$(envelope_weekly 99 "$reset_iso")" + export AGENT_RATE_LIMITS_STATE="$TMP/state.json" + printf '%s' '{"dev-lead":{"last_run_epoch":0,"daily_count":0,"consecutive_failures":0,"breaker_opened_epoch":0}}' >"$AGENT_RATE_LIMITS_STATE" + export GH_STUB_STDOUT='[{"status":"in_progress"}]' + run bash -c "source '$LIB'; AGENT_TOKEN_BUDGET_ENABLED=true arl_admission_gate dev-lead donpetry-bot" + [ "$status" -eq 1 ] + [[ "$output" == *"decision=defer"* ]] + [[ "$output" == *"token"* ]] +} + +@test "admission gate: config enabled=false keeps the glide inert even with the env flag on" { + write_glide_config false + local reset_iso="2026-09-08T15:00:00Z" reset now + reset="$(iso_epoch "$reset_iso")"; now=$(( reset - 4 * 86400 )) + export SOURCE_NOW="$now" + write_telemetry "$(envelope_weekly 99 "$reset_iso")" + export AGENT_RATE_LIMITS_STATE="$TMP/state.json" + printf '%s' '{"dev-lead":{"last_run_epoch":0,"daily_count":0,"consecutive_failures":0,"breaker_opened_epoch":0}}' >"$AGENT_RATE_LIMITS_STATE" + export GH_STUB_STDOUT='[{"status":"in_progress"}]' + run bash -c "source '$LIB'; AGENT_TOKEN_BUDGET_ENABLED=true arl_admission_gate dev-lead donpetry-bot" + [ "$status" -eq 0 ] + [[ "$output" == *"decision=allow"* ]] +} + +@test "admission gate: an exempt actor is allowed even when the weekly glide is tripped" { + write_glide_config + local reset_iso="2026-09-08T15:00:00Z" reset now + reset="$(iso_epoch "$reset_iso")"; now=$(( reset - 4 * 86400 )) + export SOURCE_NOW="$now" + write_telemetry "$(envelope_weekly 99 "$reset_iso")" + run bash -c "source '$LIB'; AGENT_TOKEN_BUDGET_ENABLED=true arl_admission_gate dev-lead 'dependabot[bot]'" + [ "$status" -eq 0 ] + [[ "$output" == *"decision=allow"* ]] + [[ "$output" == *"exempt"* ]] +} + +@test "admission gate: weekly-glide defer emits the escalation marker to stderr" { + write_glide_config + local reset_iso="2026-09-08T15:00:00Z" reset now + reset="$(iso_epoch "$reset_iso")"; now=$(( reset - 4 * 86400 )) + export SOURCE_NOW="$now" + write_telemetry "$(envelope_weekly 99 "$reset_iso")" + export AGENT_RATE_LIMITS_STATE="$TMP/state.json" + printf '%s' '{"dev-lead":{"last_run_epoch":0,"daily_count":0,"consecutive_failures":0,"breaker_opened_epoch":0}}' >"$AGENT_RATE_LIMITS_STATE" + export GH_STUB_STDOUT='[{"status":"in_progress"}]' + run bash -c "source '$LIB'; AGENT_TOKEN_BUDGET_ENABLED=true arl_admission_gate dev-lead donpetry-bot" 2>&1 + [ "$status" -eq 1 ] + [[ "$output" == *"