From 4a36c7fdd71efdc86fa3ea5aa9351264d9915ad0 Mon Sep 17 00:00:00 2001 From: "Jakub A. W" Date: Sun, 26 Jul 2026 22:32:41 +0200 Subject: [PATCH 1/2] test(e2e): cover label-scoped budgets in the release curl matrix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds S197-S204 for budgets scoped to a request label rather than a user_path subtree: admin CRUD and validation negatives, tagging-header enforcement with verbatim case-sensitive matching, multi-label charging, managed-key label charging, a request matched by both a user_path and a label budget, reset-one, PostgreSQL/MongoDB parity, and auth gating. S200 runs on the auth-enabled gateway rather than the main SQLite one: creating a managed key on a gateway with no GOMODEL_MASTER_KEY switches every endpoint to require bearer auth, and managed keys have no delete endpoint, so registering one there would lock the gateway down for every later scenario. Two scenarios promised more than they checked, so the assertions were brought up to the prose: S199 now checks the block names the exhausted label and *not* the healthy one, and S201 checks the user-path budget that matched the same request was charged and still has room — which is what makes it evidence that the label budget alone caused the block. Co-Authored-By: Claude Opus 5 (1M context) --- tests/e2e/release-e2e-scenarios.md | 398 ++++++++++++++++++++++++++++- 1 file changed, 397 insertions(+), 1 deletion(-) diff --git a/tests/e2e/release-e2e-scenarios.md b/tests/e2e/release-e2e-scenarios.md index 11b895d72..ca45e6e89 100644 --- a/tests/e2e/release-e2e-scenarios.md +++ b/tests/e2e/release-e2e-scenarios.md @@ -1,6 +1,6 @@ # Release E2E Curl Matrix -This file contains 196 end-to-end curl scenarios for release validation. +This file contains 204 end-to-end curl scenarios for release validation. These scenarios are prepared for execution across these local gateways: - `http://localhost:18080` - SQLite-backed main test gateway @@ -118,6 +118,15 @@ Stateful note: PostgreSQL/MongoDB parity, admin-auth gating); each registers `$QA_SUFFIX`-scoped provider names against unreachable base URLs and deletes them, so they are self-contained and rerunnable in any order +- `S197`-`S204` exercise budgets scoped to a request label rather than a + `user_path` subtree (admin CRUD and validation negatives, tagging-header + label enforcement with verbatim/case-sensitive matching, multi-label + charging in one lookup, managed-API-key label charging, a request matched by + both a `user_path` and a `label` budget at once, reset-one, PostgreSQL/MongoDB + parity, and auth gating); each is self-contained and rerunnable in any + order. `S200` deliberately registers its managed key on the auth-enabled + gateway rather than the no-master-key main SQLite gateway — see the note on + that scenario for why - For stateful partial reruns, prefer a contiguous range that includes the prerequisite setup scenarios, or rerun with the same `--qa-suffix` and `--keep-artifacts` @@ -4778,3 +4787,390 @@ curl -fsS -H "$ADMIN_AUTH_HEADER" "$AUTH_BASE_URL/admin/provider-credentials/typ curl -fsS -H "$ADMIN_AUTH_HEADER" "$AUTH_BASE_URL/admin/provider-credentials" \ | jq -e 'type == "array" and any(.[]?; .managed == true)' >/dev/null ``` + +## 27. Budgets scoped to labels + +These scenarios exercise scoping a budget to a request label (`scope: "label"`) +rather than a `user_path` subtree. Each scenario uses `$QA_SUFFIX`-scoped +labels and tagging headers, and restores an empty tagging rule set and deletes +its own budgets, so they are self-contained and rerunnable in any order. + +### S197 Label budget admin CRUD and validation negatives + +A label-scoped budget round-trips through the admin API with no `user_path` +field in its response, and the request is rejected when the scope/subject +combination is invalid. + +```bash +QA_LBL="QaBudgetLabel-$QA_SUFFIX" +HEADERS_FILE=$(mktemp "$QA_RUN_DIR/s197.headers.XXXXXX") +BODY_FILE=$(mktemp "$QA_RUN_DIR/s197.body.XXXXXX") + +curl -fsS -X PUT "$BASE_URL/admin/budgets" \ + -H 'Content-Type: application/json' \ + -d "{\"scope\":\"label\",\"subject\":\"$QA_LBL\",\"budget_key\":{\"period\":\"daily\"},\"amount\":25.5}" \ + | jq -e --arg l "$QA_LBL" ' + any(.budgets[]?; .scope == "label" and .subject == $l and .amount == 25.5 and .source == "manual" and (has("user_path") | not)) + ' >/dev/null + +curl -sS -D "$HEADERS_FILE" -o "$BODY_FILE" -X PUT "$BASE_URL/admin/budgets" \ + -H 'Content-Type: application/json' \ + -d "{\"scope\":\"label\",\"subject\":\"$QA_LBL\",\"user_path\":\"/qa-conflict\",\"budget_key\":{\"period\":\"daily\"},\"amount\":5}" +grep -Eiq '^HTTP/.* 400 ' "$HEADERS_FILE" +jq -e '.error.message | test("user_path")' "$BODY_FILE" >/dev/null + +curl -sS -D "$HEADERS_FILE" -o "$BODY_FILE" -X PUT "$BASE_URL/admin/budgets" \ + -H 'Content-Type: application/json' \ + -d '{"scope":"label","budget_key":{"period":"daily"},"amount":5}' +grep -Eiq '^HTTP/.* 400 ' "$HEADERS_FILE" +jq -e '.error.message | test("subject is required")' "$BODY_FILE" >/dev/null + +curl -sS -D "$HEADERS_FILE" -o "$BODY_FILE" -X PUT "$BASE_URL/admin/budgets" \ + -H 'Content-Type: application/json' \ + -d "{\"scope\":\"bogus\",\"subject\":\"$QA_LBL\",\"budget_key\":{\"period\":\"daily\"},\"amount\":5}" +grep -Eiq '^HTTP/.* 400 ' "$HEADERS_FILE" +jq -e '.error.message | test("scope must be one of")' "$BODY_FILE" >/dev/null + +curl -fsS -X DELETE "$BASE_URL/admin/budgets" \ + -H 'Content-Type: application/json' \ + -d "{\"scope\":\"label\",\"subject\":\"$QA_LBL\",\"budget_key\":{\"period\":\"daily\"}}" \ + | jq -e --arg l "$QA_LBL" 'all(.budgets[]?; .subject != $l)' >/dev/null + +STATUS=$(curl -sS -o "$BODY_FILE" -w '%{http_code}' -X DELETE "$BASE_URL/admin/budgets" \ + -H 'Content-Type: application/json' \ + -d "{\"scope\":\"label\",\"subject\":\"qa-budget-label-absent-$QA_SUFFIX\",\"budget_key\":{\"period\":\"daily\"}}") +[ "$STATUS" = "404" ] +jq -e '.error.code == "budget_not_found"' "$BODY_FILE" >/dev/null +``` + +### S198 Label budget blocks a request once exhausted, matched verbatim + +A tagging rule extracts a label header onto the request; a tiny label budget +lets the first request through and records spend, then blocks a second request +carrying the same label with a `budget_exceeded` 429. A third request carrying +a different-case spelling of the label is not blocked, since label matching +never folds case. + +```bash +TAG_HDR="X-Qa-Budget-Label-$QA_SUFFIX" +QA_LBL="QaBudgetLabel2-$QA_SUFFIX" +curl -fsS -X PUT "$BASE_URL/admin/tagging/settings" \ + -H 'Content-Type: application/json' -d "{\"headers\":[{\"header\":\"$TAG_HDR\"}]}" >/dev/null +curl -fsS -X PUT "$BASE_URL/admin/budgets" \ + -H 'Content-Type: application/json' \ + -d "{\"scope\":\"label\",\"subject\":\"$QA_LBL\",\"budget_key\":{\"period\":\"daily\"},\"amount\":$QA_BUDGET_AMOUNT}" >/dev/null + +REQ1="qa-budget-label-$QA_SUFFIX-1" +HEADERS_FILE=$(mktemp "$QA_RUN_DIR/s198.headers.XXXXXX") +BODY_FILE=$(mktemp "$QA_RUN_DIR/s198.body.XXXXXX") +curl -fsS -o "$BODY_FILE" -X POST "$BASE_URL/v1/chat/completions" -H 'Content-Type: application/json' \ + -H "X-Request-ID: $REQ1" -H "$TAG_HDR: $QA_LBL" \ + -d '{"model":"gpt-4.1-nano","messages":[{"role":"user","content":"Reply exactly QA_BUDGET_LABEL_OK"}],"max_tokens":20,"temperature":0}' +assert_chat_response_contains "$BODY_FILE" "" "QA_BUDGET_LABEL_OK" + +USAGE_FILE="$QA_RUN_DIR/s198.usage.json" +for _ in $(seq 1 15); do + curl -fsS "$BASE_URL/admin/budgets" > "$BODY_FILE" + if jq -e --arg l "$QA_LBL" 'any(.budgets[]?; .scope == "label" and .subject == $l and .spent > 0)' "$BODY_FILE" >/dev/null; then + break + fi + sleep 1 +done +jq -e --arg l "$QA_LBL" ' + any(.budgets[]?; .scope == "label" and .subject == $l and .spent > 0 and .has_usage == true and .remaining < 0 and .usage_ratio > 1) +' "$BODY_FILE" >/dev/null + +REQ2="qa-budget-label-$QA_SUFFIX-2" +STATUS=$(curl -sS -D "$HEADERS_FILE" -o "$BODY_FILE" -w '%{http_code}' -X POST "$BASE_URL/v1/chat/completions" -H 'Content-Type: application/json' \ + -H "X-Request-ID: $REQ2" -H "$TAG_HDR: $QA_LBL" \ + -d '{"model":"gpt-4.1-nano","messages":[{"role":"user","content":"Reply exactly QA_BUDGET_LABEL_BLOCK"}],"max_tokens":20,"temperature":0}') +[ "$STATUS" = "429" ] +grep -Eiq '^Retry-After: *[0-9]+' "$HEADERS_FILE" +jq -e --arg l "$QA_LBL" '.error.type == "rate_limit_error" and .error.code == "budget_exceeded" and (.error.message | test("label " + $l))' "$BODY_FILE" >/dev/null + +REQ3="qa-budget-label-$QA_SUFFIX-3" +STATUS=$(curl -sS -o "$BODY_FILE" -w '%{http_code}' -X POST "$BASE_URL/v1/chat/completions" -H 'Content-Type: application/json' \ + -H "X-Request-ID: $REQ3" -H "$TAG_HDR: $(printf '%s' "$QA_LBL" | tr '[:upper:]' '[:lower:]')" \ + -d '{"model":"gpt-4.1-nano","messages":[{"role":"user","content":"Reply exactly QA_BUDGET_LABEL_CASE_OK"}],"max_tokens":20,"temperature":0}') +[ "$STATUS" = "200" ] + +curl -fsS -X PUT "$BASE_URL/admin/tagging/settings" -H 'Content-Type: application/json' -d '{"headers":[]}' >/dev/null +curl -fsS -X DELETE "$BASE_URL/admin/budgets" -H 'Content-Type: application/json' \ + -d "{\"scope\":\"label\",\"subject\":\"$QA_LBL\",\"budget_key\":{\"period\":\"daily\"}}" >/dev/null +``` + +### S199 A request carrying several labels is charged against every matching budget + +Two label budgets exist; a single request carrying both labels records spend +on both in one lookup, and a later request that would breach only one of them +is still blocked, naming the exhausted label rather than the healthy one. + +```bash +TAG_HDR_A="X-Qa-Budget-Multi-A-$QA_SUFFIX" +TAG_HDR_B="X-Qa-Budget-Multi-B-$QA_SUFFIX" +LBL_A="QaBudgetMultiA-$QA_SUFFIX" +LBL_B="QaBudgetMultiB-$QA_SUFFIX" +curl -fsS -X PUT "$BASE_URL/admin/tagging/settings" -H 'Content-Type: application/json' \ + -d "{\"headers\":[{\"header\":\"$TAG_HDR_A\"},{\"header\":\"$TAG_HDR_B\"}]}" >/dev/null +curl -fsS -X PUT "$BASE_URL/admin/budgets" -H 'Content-Type: application/json' \ + -d "{\"scope\":\"label\",\"subject\":\"$LBL_A\",\"budget_key\":{\"period\":\"daily\"},\"amount\":$QA_BUDGET_AMOUNT}" >/dev/null +curl -fsS -X PUT "$BASE_URL/admin/budgets" -H 'Content-Type: application/json' \ + -d "{\"scope\":\"label\",\"subject\":\"$LBL_B\",\"budget_key\":{\"period\":\"daily\"},\"amount\":1000}" >/dev/null + +REQ1="qa-budget-multi-$QA_SUFFIX-1" +BODY_FILE=$(mktemp "$QA_RUN_DIR/s199.body.XXXXXX") +curl -fsS -o "$BODY_FILE" -X POST "$BASE_URL/v1/chat/completions" -H 'Content-Type: application/json' \ + -H "X-Request-ID: $REQ1" -H "$TAG_HDR_A: $LBL_A" -H "$TAG_HDR_B: $LBL_B" \ + -d '{"model":"gpt-4.1-nano","messages":[{"role":"user","content":"Reply exactly QA_BUDGET_MULTI_OK"}],"max_tokens":20,"temperature":0}' +assert_chat_response_contains "$BODY_FILE" "" "QA_BUDGET_MULTI_OK" + +for _ in $(seq 1 15); do + curl -fsS "$BASE_URL/admin/budgets" > "$BODY_FILE" + if jq -e --arg a "$LBL_A" --arg b "$LBL_B" ' + (any(.budgets[]?; .subject == $a and .spent > 0)) and (any(.budgets[]?; .subject == $b and .spent > 0)) + ' "$BODY_FILE" >/dev/null; then + break + fi + sleep 1 +done +jq -e --arg a "$LBL_A" --arg b "$LBL_B" ' + (any(.budgets[]?; .subject == $a and .spent > 0 and .has_usage == true)) and + (any(.budgets[]?; .subject == $b and .spent > 0 and .has_usage == true)) +' "$BODY_FILE" >/dev/null + +REQ2="qa-budget-multi-$QA_SUFFIX-2" +STATUS=$(curl -sS -o "$BODY_FILE" -w '%{http_code}' -X POST "$BASE_URL/v1/chat/completions" -H 'Content-Type: application/json' \ + -H "X-Request-ID: $REQ2" -H "$TAG_HDR_A: $LBL_A" -H "$TAG_HDR_B: $LBL_B" \ + -d '{"model":"gpt-4.1-nano","messages":[{"role":"user","content":"Reply exactly QA_BUDGET_MULTI_BLOCK"}],"max_tokens":20,"temperature":0}') +[ "$STATUS" = "429" ] +jq -e --arg a "$LBL_A" --arg b "$LBL_B" ' + (.error.message | test("label " + $a)) and (.error.message | test("label " + $b) | not) +' "$BODY_FILE" >/dev/null + +curl -fsS -X PUT "$BASE_URL/admin/tagging/settings" -H 'Content-Type: application/json' -d '{"headers":[]}' >/dev/null +curl -fsS -X DELETE "$BASE_URL/admin/budgets" -H 'Content-Type: application/json' \ + -d "{\"scope\":\"label\",\"subject\":\"$LBL_A\",\"budget_key\":{\"period\":\"daily\"}}" >/dev/null +curl -fsS -X DELETE "$BASE_URL/admin/budgets" -H 'Content-Type: application/json' \ + -d "{\"scope\":\"label\",\"subject\":\"$LBL_B\",\"budget_key\":{\"period\":\"daily\"}}" >/dev/null +``` + +### S200 A managed API key's labels charge a label budget without any tagging header + +Labels attached to the managed key that authenticated the request are merged +into the request's labels the same way header-extracted labels are, so a +label budget matches even when no tagging rule is configured. This runs on the +auth-enabled gateway (which already requires a master key for every request) +rather than the main SQLite gateway: on a gateway with no `GOMODEL_MASTER_KEY`, +creating a managed key switches every endpoint, including `/v1/*`, to require +bearer auth from then on, and managed keys have no delete endpoint (only +`deactivate`, which does not undo the switch since it counts stored keys, not +active ones) — so registering one on the open gateway would leave it +permanently locked down for every later scenario. The auth-enabled gateway +also has the exact response cache on, so both chat replies are suffixed with +`$QA_BUDGET_SUFFIX`: a fixed reply string would be served from a prior run's +cached response on a rerun, bypassing usage tracking entirely and making the +budget never show spend (response cache hits return before budget/usage +enforcement by design). + +```bash +KEYNAME="qa-budget-key-$QA_SUFFIX" +KEYLABEL="QaBudgetKeyLabel-$QA_SUFFIX" +KEY_JSON=$(curl -fsS -H "$ADMIN_AUTH_HEADER" -X POST "$AUTH_BASE_URL/admin/auth-keys" -H 'Content-Type: application/json' \ + -d "{\"name\":\"$KEYNAME\",\"labels\":[\"$KEYLABEL\"]}") +KEY_VALUE=$(echo "$KEY_JSON" | jq -r '.value') +KEY_ID=$(echo "$KEY_JSON" | jq -r '.id') + +curl -fsS -H "$ADMIN_AUTH_HEADER" -X PUT "$AUTH_BASE_URL/admin/budgets" -H 'Content-Type: application/json' \ + -d "{\"scope\":\"label\",\"subject\":\"$KEYLABEL\",\"budget_key\":{\"period\":\"daily\"},\"amount\":$QA_BUDGET_AMOUNT}" >/dev/null + +REQ1="qa-budget-key-$QA_SUFFIX-1" +BODY_FILE=$(mktemp "$QA_RUN_DIR/s200.body.XXXXXX") +curl -fsS -o "$BODY_FILE" -X POST "$AUTH_BASE_URL/v1/chat/completions" \ + -H 'Content-Type: application/json' -H "Authorization: Bearer $KEY_VALUE" \ + -H "X-Request-ID: $REQ1" \ + -d "{\"model\":\"gpt-4.1-nano\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply exactly QA_BUDGET_KEYLABEL_OK_$QA_BUDGET_SUFFIX\"}],\"max_tokens\":20,\"temperature\":0}" +assert_chat_response_contains "$BODY_FILE" "" "QA_BUDGET_KEYLABEL_OK_$QA_BUDGET_SUFFIX" + +for _ in $(seq 1 15); do + curl -fsS -H "$ADMIN_AUTH_HEADER" "$AUTH_BASE_URL/admin/budgets" > "$BODY_FILE" + if jq -e --arg l "$KEYLABEL" 'any(.budgets[]?; .subject == $l and .spent > 0)' "$BODY_FILE" >/dev/null; then + break + fi + sleep 1 +done +jq -e --arg l "$KEYLABEL" 'any(.budgets[]?; .subject == $l and .spent > 0 and .has_usage == true)' "$BODY_FILE" >/dev/null + +REQ2="qa-budget-key-$QA_SUFFIX-2" +STATUS=$(curl -sS -o "$BODY_FILE" -w '%{http_code}' -X POST "$AUTH_BASE_URL/v1/chat/completions" \ + -H 'Content-Type: application/json' -H "Authorization: Bearer $KEY_VALUE" \ + -H "X-Request-ID: $REQ2" \ + -d "{\"model\":\"gpt-4.1-nano\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply exactly QA_BUDGET_KEYLABEL_BLOCK_$QA_BUDGET_SUFFIX\"}],\"max_tokens\":20,\"temperature\":0}") +[ "$STATUS" = "429" ] + +curl -fsS -H "$ADMIN_AUTH_HEADER" -X DELETE "$AUTH_BASE_URL/admin/budgets" -H 'Content-Type: application/json' \ + -d "{\"scope\":\"label\",\"subject\":\"$KEYLABEL\",\"budget_key\":{\"period\":\"daily\"}}" >/dev/null +curl -fsS -H "$ADMIN_AUTH_HEADER" -X POST "$AUTH_BASE_URL/admin/auth-keys/$KEY_ID/deactivate" >/dev/null +``` + +### S201 A request matching both a user_path and a label budget is blocked by whichever is exhausted + +One generous `user_path` budget and one tiny `label` budget both match the +same request; once the label budget is exhausted, the request is blocked even +though the user-path budget still has ample room, and the error names the +exhausted label rather than the path. + +```bash +MIX_PATH="/qa-budget-mix-$QA_SUFFIX" +MIX_LABEL="QaBudgetMixLabel-$QA_SUFFIX" +MIX_HDR="X-Qa-Budget-Mix-$QA_SUFFIX" +curl -fsS -X PUT "$BASE_URL/admin/tagging/settings" -H 'Content-Type: application/json' \ + -d "{\"headers\":[{\"header\":\"$MIX_HDR\"}]}" >/dev/null +curl -fsS -X PUT "$BASE_URL/admin/budgets" -H 'Content-Type: application/json' \ + -d "{\"scope\":\"user_path\",\"subject\":\"$MIX_PATH\",\"budget_key\":{\"period\":\"daily\"},\"amount\":1000}" >/dev/null +curl -fsS -X PUT "$BASE_URL/admin/budgets" -H 'Content-Type: application/json' \ + -d "{\"scope\":\"label\",\"subject\":\"$MIX_LABEL\",\"budget_key\":{\"period\":\"daily\"},\"amount\":$QA_BUDGET_AMOUNT}" >/dev/null + +REQ1="qa-budget-mix-$QA_SUFFIX-1" +BODY_FILE=$(mktemp "$QA_RUN_DIR/s201.body.XXXXXX") +curl -fsS -o "$BODY_FILE" -X POST "$BASE_URL/v1/chat/completions" -H 'Content-Type: application/json' \ + -H "X-Request-ID: $REQ1" -H "X-GoModel-User-Path: $MIX_PATH" -H "$MIX_HDR: $MIX_LABEL" \ + -d '{"model":"gpt-4.1-nano","messages":[{"role":"user","content":"Reply exactly QA_BUDGET_MIX_OK"}],"max_tokens":20,"temperature":0}' +assert_chat_response_contains "$BODY_FILE" "" "QA_BUDGET_MIX_OK" + +for _ in $(seq 1 15); do + curl -fsS "$BASE_URL/admin/budgets" > "$BODY_FILE" + if jq -e --arg l "$MIX_LABEL" 'any(.budgets[]?; .subject == $l and .spent > 0)' "$BODY_FILE" >/dev/null; then + break + fi + sleep 1 +done + +REQ2="qa-budget-mix-$QA_SUFFIX-2" +STATUS=$(curl -sS -o "$BODY_FILE" -w '%{http_code}' -X POST "$BASE_URL/v1/chat/completions" -H 'Content-Type: application/json' \ + -H "X-Request-ID: $REQ2" -H "X-GoModel-User-Path: $MIX_PATH" -H "$MIX_HDR: $MIX_LABEL" \ + -d '{"model":"gpt-4.1-nano","messages":[{"role":"user","content":"Reply exactly QA_BUDGET_MIX_BLOCK"}],"max_tokens":20,"temperature":0}') +[ "$STATUS" = "429" ] +jq -e --arg l "$MIX_LABEL" --arg p "$MIX_PATH" ' + (.error.message | test("label " + $l)) and (.error.message | test($p) | not) +' "$BODY_FILE" >/dev/null + +# The user-path budget was charged by the same request and still has room, so +# the block above came from the label budget alone. +curl -fsS "$BASE_URL/admin/budgets" > "$BODY_FILE" +jq -e --arg p "$MIX_PATH" ' + any(.budgets[]?; .user_path == $p and .spent > 0 and .remaining > 0 and .usage_ratio < 1) +' "$BODY_FILE" >/dev/null + +curl -fsS -X PUT "$BASE_URL/admin/tagging/settings" -H 'Content-Type: application/json' -d '{"headers":[]}' >/dev/null +curl -fsS -X DELETE "$BASE_URL/admin/budgets" -H 'Content-Type: application/json' \ + -d "{\"scope\":\"user_path\",\"subject\":\"$MIX_PATH\",\"budget_key\":{\"period\":\"daily\"}}" >/dev/null +curl -fsS -X DELETE "$BASE_URL/admin/budgets" -H 'Content-Type: application/json' \ + -d "{\"scope\":\"label\",\"subject\":\"$MIX_LABEL\",\"budget_key\":{\"period\":\"daily\"}}" >/dev/null +``` + +### S202 Reset-one clears spend for a label budget and lets the next request through + +```bash +QA_LBL="QaBudgetReset-$QA_SUFFIX" +TAG_HDR="X-Qa-Budget-Reset-$QA_SUFFIX" +curl -fsS -X PUT "$BASE_URL/admin/tagging/settings" -H 'Content-Type: application/json' \ + -d "{\"headers\":[{\"header\":\"$TAG_HDR\"}]}" >/dev/null +curl -fsS -X PUT "$BASE_URL/admin/budgets" -H 'Content-Type: application/json' \ + -d "{\"scope\":\"label\",\"subject\":\"$QA_LBL\",\"budget_key\":{\"period\":\"daily\"},\"amount\":$QA_BUDGET_AMOUNT}" >/dev/null + +REQ1="qa-budget-reset-$QA_SUFFIX-1" +BODY_FILE=$(mktemp "$QA_RUN_DIR/s202.body.XXXXXX") +curl -fsS -o "$BODY_FILE" -X POST "$BASE_URL/v1/chat/completions" -H 'Content-Type: application/json' \ + -H "X-Request-ID: $REQ1" -H "$TAG_HDR: $QA_LBL" \ + -d '{"model":"gpt-4.1-nano","messages":[{"role":"user","content":"Reply exactly QA_BUDGET_RESET_OK"}],"max_tokens":20,"temperature":0}' +assert_chat_response_contains "$BODY_FILE" "" "QA_BUDGET_RESET_OK" + +for _ in $(seq 1 15); do + curl -fsS "$BASE_URL/admin/budgets" > "$BODY_FILE" + if jq -e --arg l "$QA_LBL" 'any(.budgets[]?; .subject == $l and .spent > 0)' "$BODY_FILE" >/dev/null; then + break + fi + sleep 1 +done + +curl -fsS -X POST "$BASE_URL/admin/budgets/reset-one" -H 'Content-Type: application/json' \ + -d "{\"scope\":\"label\",\"subject\":\"$QA_LBL\",\"period\":\"daily\"}" \ + | jq -e --arg l "$QA_LBL" 'any(.budgets[]?; .subject == $l and .spent == 0 and .has_usage == false)' >/dev/null + +REQ2="qa-budget-reset-$QA_SUFFIX-2" +STATUS=$(curl -sS -o "$BODY_FILE" -w '%{http_code}' -X POST "$BASE_URL/v1/chat/completions" -H 'Content-Type: application/json' \ + -H "X-Request-ID: $REQ2" -H "$TAG_HDR: $QA_LBL" \ + -d '{"model":"gpt-4.1-nano","messages":[{"role":"user","content":"Reply exactly QA_BUDGET_RESET_OK2"}],"max_tokens":20,"temperature":0}') +[ "$STATUS" = "200" ] + +curl -fsS -X PUT "$BASE_URL/admin/tagging/settings" -H 'Content-Type: application/json' -d '{"headers":[]}' >/dev/null +curl -fsS -X DELETE "$BASE_URL/admin/budgets" -H 'Content-Type: application/json' \ + -d "{\"scope\":\"label\",\"subject\":\"$QA_LBL\",\"budget_key\":{\"period\":\"daily\"}}" >/dev/null +``` + +### S203 Label budget enforcement on PostgreSQL and MongoDB + +The same tagging-header-driven label budget enforcement flow runs against the +PostgreSQL- and MongoDB-backed gateways. + +```bash +for TARGET in "$PG_BASE_URL|pg" "$MONGO_BASE_URL|mongo"; do + URL="${TARGET%%|*}" + TAG="${TARGET##*|}" + LBL="QaBudget${TAG}Label-$QA_SUFFIX" + HDR="X-Qa-Budget-$TAG-$QA_SUFFIX" + curl -fsS -X PUT "$URL/admin/tagging/settings" -H 'Content-Type: application/json' \ + -d "{\"headers\":[{\"header\":\"$HDR\"}]}" >/dev/null + curl -fsS -X PUT "$URL/admin/budgets" -H 'Content-Type: application/json' \ + -d "{\"scope\":\"label\",\"subject\":\"$LBL\",\"budget_key\":{\"period\":\"daily\"},\"amount\":$QA_BUDGET_AMOUNT}" >/dev/null + + REQ="qa-budget-$TAG-$QA_SUFFIX" + BODY_FILE=$(mktemp "$QA_RUN_DIR/s203.$TAG.body.XXXXXX") + curl -fsS -o "$BODY_FILE" -X POST "$URL/v1/chat/completions" -H 'Content-Type: application/json' \ + -H "X-Request-ID: $REQ-1" -H "$HDR: $LBL" \ + -d '{"model":"gpt-4.1-nano","messages":[{"role":"user","content":"Reply exactly QA_BUDGET_BACKEND_OK"}],"max_tokens":20,"temperature":0}' + assert_chat_response_contains "$BODY_FILE" "" "QA_BUDGET_BACKEND_OK" + + for _ in $(seq 1 15); do + curl -fsS "$URL/admin/budgets" > "$BODY_FILE" + if jq -e --arg l "$LBL" 'any(.budgets[]?; .subject == $l and .spent > 0)' "$BODY_FILE" >/dev/null; then + break + fi + sleep 1 + done + jq -e --arg l "$LBL" 'any(.budgets[]?; .subject == $l and .spent > 0 and .has_usage == true)' "$BODY_FILE" >/dev/null + + STATUS=$(curl -sS -o "$BODY_FILE" -w '%{http_code}' -X POST "$URL/v1/chat/completions" -H 'Content-Type: application/json' \ + -H "X-Request-ID: $REQ-2" -H "$HDR: $LBL" \ + -d '{"model":"gpt-4.1-nano","messages":[{"role":"user","content":"Reply exactly QA_BUDGET_BACKEND_BLOCK"}],"max_tokens":20,"temperature":0}') + [ "$STATUS" = "429" ] + + curl -fsS -X PUT "$URL/admin/tagging/settings" -H 'Content-Type: application/json' -d '{"headers":[]}' >/dev/null + curl -fsS -X DELETE "$URL/admin/budgets" -H 'Content-Type: application/json' \ + -d "{\"scope\":\"label\",\"subject\":\"$LBL\",\"budget_key\":{\"period\":\"daily\"}}" >/dev/null +done +``` + +### S204 Budgets admin API requires auth on the auth-enabled gateway, for label scope too + +```bash +STATUS=$(curl -sS -o /dev/null -w '%{http_code}' "$AUTH_BASE_URL/admin/budgets") +[ "$STATUS" = "401" ] + +STATUS=$(curl -sS -o /dev/null -w '%{http_code}' -H "$ADMIN_AUTH_HEADER" "$AUTH_BASE_URL/admin/budgets") +[ "$STATUS" = "200" ] + +QA_LBL="QaBudgetAuth-$QA_SUFFIX" +STATUS=$(curl -sS -o /dev/null -w '%{http_code}' -X PUT "$AUTH_BASE_URL/admin/budgets" \ + -H 'Content-Type: application/json' \ + -d "{\"scope\":\"label\",\"subject\":\"$QA_LBL\",\"budget_key\":{\"period\":\"daily\"},\"amount\":5}") +[ "$STATUS" = "401" ] + +curl -fsS -H "$ADMIN_AUTH_HEADER" -X PUT "$AUTH_BASE_URL/admin/budgets" \ + -H 'Content-Type: application/json' \ + -d "{\"scope\":\"label\",\"subject\":\"$QA_LBL\",\"budget_key\":{\"period\":\"daily\"},\"amount\":5}" \ + | jq -e --arg l "$QA_LBL" 'any(.budgets[]?; .subject == $l)' >/dev/null +curl -fsS -H "$ADMIN_AUTH_HEADER" -X DELETE "$AUTH_BASE_URL/admin/budgets" \ + -H 'Content-Type: application/json' \ + -d "{\"scope\":\"label\",\"subject\":\"$QA_LBL\",\"budget_key\":{\"period\":\"daily\"}}" >/dev/null +``` From de832b9ceba0e865da46f74a40cb8f2aa1a2334d Mon Sep 17 00:00:00 2001 From: "Jakub A. W" Date: Sun, 26 Jul 2026 22:57:04 +0200 Subject: [PATCH 2/2] test(e2e): make label budget scenarios survive a failed run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups. Budget spend is a SUM over usage rows, so deleting and recreating a budget does not forget what an earlier run charged against the same label. A scenario that exits part-way on a failed assertion skips its own cleanup, and the same-suffix rerun this file recommends for partial reruns would then meet an already-exhausted tiny budget and 429 on its first request — failing for a reason unrelated to what it tests, until someone cleared the state by hand. reset_release_budget resets a budget right after creation, so that first request behaves the same on the first run and the fifth. Confirmed by leaving an exhausted budget behind and rerunning S198 with the same suffix: exit 1 without the reset, exit 0 with it, same leftover state both times. The section preamble also claimed every scenario used tagging headers. S197 and S204 are admin-only and send no model traffic, and S200 relies on managed-key labels precisely because no tagging rule is configured, so it now says which scenarios do what. Co-Authored-By: Claude Opus 5 (1M context) --- tests/e2e/release-e2e-scenarios.md | 49 ++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/tests/e2e/release-e2e-scenarios.md b/tests/e2e/release-e2e-scenarios.md index ca45e6e89..ba7a0ddac 100644 --- a/tests/e2e/release-e2e-scenarios.md +++ b/tests/e2e/release-e2e-scenarios.md @@ -465,6 +465,32 @@ run_release_budget_enforcement() { all(.budgets[]?; .user_path != $user_path) ' "$budget_json_file" >/dev/null } + +# Start a freshly created budget from zero spend. +# +# Spend is a SUM over usage rows, so deleting and recreating a budget does not +# forget what an earlier run charged against the same subject. A scenario that +# exits part-way on a failed assertion skips its own cleanup, and the rerun this +# file recommends — same --qa-suffix, so the same labels — would then meet an +# already-exhausted tiny budget and get 429 on its first request, failing for a +# reason that has nothing to do with what it tests. Resetting after creation +# makes that first request behave the same on the first run and the fifth. +reset_release_budget() { + local base_url="$1" + local scope="$2" + local subject="$3" + local auth_header="${4:-}" + + if [ -n "$auth_header" ]; then + curl -fsS -H "$auth_header" -X POST "$base_url/admin/budgets/reset-one" \ + -H 'Content-Type: application/json' \ + -d "{\"scope\":\"$scope\",\"subject\":\"$subject\",\"period\":\"daily\"}" >/dev/null + else + curl -fsS -X POST "$base_url/admin/budgets/reset-one" \ + -H 'Content-Type: application/json' \ + -d "{\"scope\":\"$scope\",\"subject\":\"$subject\",\"period\":\"daily\"}" >/dev/null + fi +} ``` ## Auth-enabled runtime environment @@ -4791,9 +4817,18 @@ curl -fsS -H "$ADMIN_AUTH_HEADER" "$AUTH_BASE_URL/admin/provider-credentials" \ ## 27. Budgets scoped to labels These scenarios exercise scoping a budget to a request label (`scope: "label"`) -rather than a `user_path` subtree. Each scenario uses `$QA_SUFFIX`-scoped -labels and tagging headers, and restores an empty tagging rule set and deletes -its own budgets, so they are self-contained and rerunnable in any order. +rather than a `user_path` subtree. Every label is `$QA_SUFFIX`-scoped and every +scenario deletes the budgets it created. How a label reaches the request varies: +S198, S199, S201 and S203 configure a tagging header and restore an empty rule +set afterwards; S200 uses the labels on the managed key that authenticated the +request and needs no tagging rule at all; S197 and S204 are admin-only and send +no model traffic. + +Each scenario that depends on starting from zero spend calls +`reset_release_budget` right after creating its budget, so a rerun following a +part-way failure — which skips the failed scenario's cleanup — does not inherit +the previous run's spend. Together that makes them self-contained and rerunnable +in any order. ### S197 Label budget admin CRUD and validation negatives @@ -4859,6 +4894,7 @@ curl -fsS -X PUT "$BASE_URL/admin/tagging/settings" \ curl -fsS -X PUT "$BASE_URL/admin/budgets" \ -H 'Content-Type: application/json' \ -d "{\"scope\":\"label\",\"subject\":\"$QA_LBL\",\"budget_key\":{\"period\":\"daily\"},\"amount\":$QA_BUDGET_AMOUNT}" >/dev/null +reset_release_budget "$BASE_URL" label "$QA_LBL" REQ1="qa-budget-label-$QA_SUFFIX-1" HEADERS_FILE=$(mktemp "$QA_RUN_DIR/s198.headers.XXXXXX") @@ -4916,6 +4952,8 @@ curl -fsS -X PUT "$BASE_URL/admin/budgets" -H 'Content-Type: application/json' \ -d "{\"scope\":\"label\",\"subject\":\"$LBL_A\",\"budget_key\":{\"period\":\"daily\"},\"amount\":$QA_BUDGET_AMOUNT}" >/dev/null curl -fsS -X PUT "$BASE_URL/admin/budgets" -H 'Content-Type: application/json' \ -d "{\"scope\":\"label\",\"subject\":\"$LBL_B\",\"budget_key\":{\"period\":\"daily\"},\"amount\":1000}" >/dev/null +reset_release_budget "$BASE_URL" label "$LBL_A" +reset_release_budget "$BASE_URL" label "$LBL_B" REQ1="qa-budget-multi-$QA_SUFFIX-1" BODY_FILE=$(mktemp "$QA_RUN_DIR/s199.body.XXXXXX") @@ -4982,6 +5020,7 @@ KEY_ID=$(echo "$KEY_JSON" | jq -r '.id') curl -fsS -H "$ADMIN_AUTH_HEADER" -X PUT "$AUTH_BASE_URL/admin/budgets" -H 'Content-Type: application/json' \ -d "{\"scope\":\"label\",\"subject\":\"$KEYLABEL\",\"budget_key\":{\"period\":\"daily\"},\"amount\":$QA_BUDGET_AMOUNT}" >/dev/null +reset_release_budget "$AUTH_BASE_URL" label "$KEYLABEL" "$ADMIN_AUTH_HEADER" REQ1="qa-budget-key-$QA_SUFFIX-1" BODY_FILE=$(mktemp "$QA_RUN_DIR/s200.body.XXXXXX") @@ -5029,6 +5068,8 @@ curl -fsS -X PUT "$BASE_URL/admin/budgets" -H 'Content-Type: application/json' \ -d "{\"scope\":\"user_path\",\"subject\":\"$MIX_PATH\",\"budget_key\":{\"period\":\"daily\"},\"amount\":1000}" >/dev/null curl -fsS -X PUT "$BASE_URL/admin/budgets" -H 'Content-Type: application/json' \ -d "{\"scope\":\"label\",\"subject\":\"$MIX_LABEL\",\"budget_key\":{\"period\":\"daily\"},\"amount\":$QA_BUDGET_AMOUNT}" >/dev/null +reset_release_budget "$BASE_URL" user_path "$MIX_PATH" +reset_release_budget "$BASE_URL" label "$MIX_LABEL" REQ1="qa-budget-mix-$QA_SUFFIX-1" BODY_FILE=$(mktemp "$QA_RUN_DIR/s201.body.XXXXXX") @@ -5077,6 +5118,7 @@ curl -fsS -X PUT "$BASE_URL/admin/tagging/settings" -H 'Content-Type: applicatio -d "{\"headers\":[{\"header\":\"$TAG_HDR\"}]}" >/dev/null curl -fsS -X PUT "$BASE_URL/admin/budgets" -H 'Content-Type: application/json' \ -d "{\"scope\":\"label\",\"subject\":\"$QA_LBL\",\"budget_key\":{\"period\":\"daily\"},\"amount\":$QA_BUDGET_AMOUNT}" >/dev/null +reset_release_budget "$BASE_URL" label "$QA_LBL" REQ1="qa-budget-reset-$QA_SUFFIX-1" BODY_FILE=$(mktemp "$QA_RUN_DIR/s202.body.XXXXXX") @@ -5123,6 +5165,7 @@ for TARGET in "$PG_BASE_URL|pg" "$MONGO_BASE_URL|mongo"; do -d "{\"headers\":[{\"header\":\"$HDR\"}]}" >/dev/null curl -fsS -X PUT "$URL/admin/budgets" -H 'Content-Type: application/json' \ -d "{\"scope\":\"label\",\"subject\":\"$LBL\",\"budget_key\":{\"period\":\"daily\"},\"amount\":$QA_BUDGET_AMOUNT}" >/dev/null + reset_release_budget "$URL" label "$LBL" REQ="qa-budget-$TAG-$QA_SUFFIX" BODY_FILE=$(mktemp "$QA_RUN_DIR/s203.$TAG.body.XXXXXX")