From fc778171c6d0fc07c12067dbb992d9a58fda2223 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=EC=9A=B0=EC=B2=A0?= Date: Tue, 20 Oct 2026 13:49:18 +0900 Subject: [PATCH 1/5] fix(server): reclaim raised inbound body capacity across sessions --- .../fr/reference/inbound-body-admission.md | 86 +- .../ja/reference/inbound-body-admission.md | 26 +- .../ko/reference/inbound-body-admission.md | 24 +- .../docs/reference/inbound-body-admission.md | 71 +- .../ru/reference/inbound-body-admission.md | 79 +- .../tr/reference/inbound-body-admission.md | 93 +- .../zh-cn/reference/inbound-body-admission.md | 26 +- .../zh-tw/reference/inbound-body-admission.md | 26 +- .../test-layout/layout-server-inbound.json | 4 + scripts/test-layout/layout.json | 2 +- src/server/inbound-body-admission.ts | 32 +- src/server/inbound-body-budget.ts | 115 +++ src/server/request-decompress.ts | 6 +- structure/transports/byte-accounting.md | 24 +- .../messages-request-id-endpoint.test.ts | 15 +- tests/fixtures/test-layout-expected.json | 2 + tests/helpers/inbound-body-budget.ts | 41 + .../chat-request-id-endpoint.test.ts | 12 +- tests/server/inbound-body-admission.test.ts | 811 ++++++++++++++++++ tests/server/inbound-body-budget.test.ts | 247 ++++++ tests/server/server-request-body-size.test.ts | 484 +---------- 21 files changed, 1595 insertions(+), 631 deletions(-) create mode 100644 scripts/test-layout/layout-server-inbound.json create mode 100644 src/server/inbound-body-budget.ts create mode 100644 tests/helpers/inbound-body-budget.ts create mode 100644 tests/server/inbound-body-admission.test.ts create mode 100644 tests/server/inbound-body-budget.test.ts diff --git a/docs-site/src/content/docs/fr/reference/inbound-body-admission.md b/docs-site/src/content/docs/fr/reference/inbound-body-admission.md index 6ca406cd940..028d71c0b79 100644 --- a/docs-site/src/content/docs/fr/reference/inbound-body-admission.md +++ b/docs-site/src/content/docs/fr/reference/inbound-body-admission.md @@ -11,12 +11,34 @@ valeur voulue. ## Augmenter la limite modifie la simultanéité -Quand la limite résolue dépasse 256 MiB, chaque requête HTTP concernée réserve toute sa marge -configurée dans un budget d'admission partagé de 512 MiB pour le processus entier. Au plus une -requête de ce type peut donc s'exécuter à la fois, même si plusieurs écouteurs partagent le -processus. Les petits corps réservent eux aussi toute la marge ; ni un petit Content-Length ni -la compression ne contournent l'admission. La réservation demeure jusqu'à la fin du flux de -réponse, son échec ou la fin de l'annulation, et pas seulement jusqu'à l'analyse du JSON entrant. +Quand la limite effective dépasse 256 MiB, chaque requête HTTP concernée doit réserver toute +sa marge par requête **avant toute lecture du corps**. Le budget d'admission partagé par le +processus passe de 512 MiB à **2 GiB**, y compris lorsque plusieurs écouteurs partagent ce +processus. Ce budget est distinct de `appOwnedMemoryBudgetMb`. + +Avant la fin de l'analyse JSON avec contrôle de taille, ni un petit `Content-Length` ni un petit +corps compressé sur le réseau ne réduisent la réservation. L'admission ne considère pas ces +valeurs comme une preuve fiable de la taille décodée ou normalisée du JSON. + +Après une analyse JSON réussie dans les limites de taille, le lecteur réduit la réservation +au maximum des octets observés sur le réseau, des octets décodés et de la mesure existante du +JSON normalisé lorsqu'un traducteur fournit un budget pour cette mesure. La réservation ne +peut que diminuer et reste plafonnée à la marge initiale par requête ; une mesure du JSON +normalisé supérieure à cette marge ne l'augmente jamais. Les contrôles UTF-8 et de JSON +resérialisé sont conservés ; la longueur d'entrée ne remplace pas la taille potentiellement +plus grande du JSON normalisé. + +Avec une limite par requête de 512 MiB, le budget peut admettre jusqu'à **4 envois réservant +chacun 512 MiB** avant la mesure et la réduction des réservations. Ce n'est pas une limite fixe +de 4 requêtes après l'analyse : par exemple, **8 très petites requêtes peuvent diffuser leur +réponse simultanément** lorsque les réservations réduites laissent de la place aux admissions +suivantes, sous réserve du budget partagé et des autres limites. + +La réservation réduite reste détenue pendant le travail asynchrone et l'attente du fournisseur, +jusqu'à la fin du flux de réponse (EOF), une erreur ou la fin de l'annulation. L'analyse seule +ne la libère pas. Si le client se déconnecte, elle reste détenue jusqu'à la fin de l'annulation +côté producteur. Elle est libérée exactement une fois, même si erreur, abandon et annulation +se produisent en concurrence. Les points de terminaison POST concernés sont `/v1/responses`, `/v1/responses/compact`, `/v1/chat/completions`, `/v1/messages`, `/v1/messages/count_tokens`, @@ -26,33 +48,49 @@ existantes par corps. Les traductions directes internes et les appels combo ne r une deuxième marge HTTP. Les limites de gestion, d'audio, d'historique de contexte et de WebSocket restent inchangées. -Un réglage absent ou égal à zéro, ou une limite résolue inférieure ou égale à 256 MiB, +Un réglage absent ou égal à zéro, ou une limite effective inférieure ou égale à 256 MiB, n'active pas ce contrôle supplémentaire de simultanéité. Les limites existantes sur le nombre de requêtes et les autres ressources continuent de s'appliquer. ## Refus temporaire ou corps trop volumineux -Si aucune marge n'est temporairement disponible, le serveur renvoie HTTP **503**, -`Retry-After: 1` et le code d'erreur `server_busy`, avant l'analyse du protocole ou l'envoi au -fournisseur. Les clients Messages et de comptage de jetons reçoivent une erreur au format -Anthropic de type `overloaded_error` ; les clients compatibles OpenAI reçoivent -`server_error`. Réessayez après la fin de la requête en cours, en respectant l'en-tête de -nouvelle tentative et la temporisation du client. Augmenter encore la limite du corps ne -résout pas l'indisponibilité de la marge. +Si la marge complète est temporairement indisponible, la requête peut rejoindre une **file +FIFO de 32 requêtes en attente au maximum**, avec une **attente limitée à 120 secondes**. +Les corps restent non lus pendant l'attente. Une réduction après mesure ou une libération +finale permet d'admettre les requêtes dans l'ordre de la file. Une requête abandonnée pendant +l'attente est retirée de la file et son corps non lu est annulé sans être consommé. + +Une file pleine ou une attente expirée entraîne HTTP **503**, `Retry-After: 1` et le code +d'erreur `server_busy`, avant l'analyse du corps ou l'envoi au fournisseur. Les corps non lus +sont annulés lors du refus. Les clients Messages et de comptage de jetons reçoivent une erreur +au format Anthropic de type `overloaded_error` ; les clients compatibles OpenAI reçoivent +`server_error`. Les contrôles d'authentification et d'origine précèdent l'admission et la mise +en file. + +Respectez l'en-tête de nouvelle tentative et la temporisation du client. L'attente bornée et +les réservations mesurées réduisent les refus de capacité évitables, sans éliminer toutes les +réponses 503. Augmenter encore la limite par requête ne résout pas un épuisement durable de +la capacité. Un corps dépassant sa limite par requête suit toujours le traitement HTTP **413** existant. Une taille déclarée excessive emprunte cette voie plutôt que de devenir un refus pour serveur -occupé. Une déconnexion conserve le comportement d'annulation existant. Les contrôles -d'authentification et d'origine précèdent ce contrôle de simultanéité. +occupé. ## Ce que mesure le budget -512 MiB est la somme des marges admises par requête, **pas une garantie que la mémoire du -processus restera sous 512 MiB**. Le décodage, les chaînes, les graphes d'objets, les copies -de requêtes et les autres états de l'application consomment davantage de mémoire. Les mesures -existantes des octets UTF-8 et du JSON resérialisé sont conservées ; la longueur de l'entrée -en octets ne remplace pas la taille potentiellement plus grande du JSON normalisé. +Le budget de **2 GiB** compte des unités d'admission : les marges complètes avant l'analyse, +puis les réservations mesurées. Il **ne borne ni la RSS du processus ni le tas des objets +analysés** et ne garantit pas une consommation mémoire inférieure à 2 GiB. Le décodage, les +chaînes, les graphes d'objets, les copies de requêtes et les autres états de l'application +consomment davantage de mémoire. + +Le passage du budget partagé de 512 MiB à 2 GiB quadruple la réservation totale possible. +Les déploiements qui choisissent une limite supérieure à 256 MiB peuvent désormais admettre +4 envois réservant chacun 512 MiB au lieu d'un seul : leur pic potentiel de consommation +mémoire augmente. Les objets analysés et les copies peuvent porter la mémoire réelle au-delà +du budget d'admission. Les limites par défaut ou inférieures n'activent pas ce contrôle et +conservent leur comportement existant. -Pour des petites requêtes parallèles, conservez si possible la limite par défaut. Augmentez-la -volontairement pour un parcours avec un historique volumineux, en acceptant la restriction de -simultanéité supplémentaire. +Conservez si possible la limite par requête par défaut. Augmentez-la volontairement si un +parcours avec un historique volumineux l'exige, en tenant compte du pic mémoire potentiellement +plus élevé et de l'attente bornée. diff --git a/docs-site/src/content/docs/ja/reference/inbound-body-admission.md b/docs-site/src/content/docs/ja/reference/inbound-body-admission.md index c458b118b09..5a693da0259 100644 --- a/docs-site/src/content/docs/ja/reference/inbound-body-admission.md +++ b/docs-site/src/content/docs/ja/reference/inbound-body-admission.md @@ -7,20 +7,34 @@ description: リクエスト本文の上限引き上げが、同時 HTTP リク ## 上限を引き上げると同時実行数が変わる -解決された上限が 256 MiB を超える場合、対象の各 HTTP リクエストは、プロセス全体で共有する 512 MiB の受け入れ予算から、設定した許容量の全量を予約します。複数のリスナーが同じプロセスを共有する場合も、同時に実行できる対象リクエストは最大 1 件です。小さな本文でも許容量全体を予約し、小さい Content-Length や圧縮によって受け入れ制御を回避することはできません。予約は JSON の解析直後ではなく、レスポンスストリームが終了、失敗、またはキャンセル処理を完了するまで続きます。 +適用される上限が 256 MiB を超える場合、対象の各 HTTP リクエストは、**本文を読み取る前に**リクエストごとの許容量全体を予約する必要があります。プロセス全体で共有する受け入れ予算は、従来の 512 MiB から **2 GiB** に増えました。同じプロセス内の複数のリスナーもこの予算を共有します。`appOwnedMemoryBudgetMb` とは別の予算です。 + +サイズを制限した JSON 解析が完了するまでは、`Content-Length` や圧縮された転送本文が小さくても予約量を減らしません。これらの値を、デコード後や正規化後の JSON サイズの根拠として信頼することはありません。 + +サイズ制限内で JSON 解析に成功すると、読み取り処理は、実測した転送バイト数とデコード後のバイト数のうち大きい値まで予約量を減らします。変換処理が正規化 JSON の測定用予算を渡した場合は、既存の正規化 JSON の測定値も含めた最大値を使います。予約量は最初のリクエスト許容量を上限として減ることしかなく、正規化後の測定値がその上限を超えても増えません。既存の UTF-8 と JSON 再シリアル化の検査は維持され、入力バイト長を、より大きくなり得る正規化 JSON サイズの代わりには使用しません。 + +リクエストごとの上限が 512 MiB なら、予約量を測定して減らす前でも、**512 MiB 全体を予約するアップロードを最大 4 件**受け入れられます。解析後の同時実行数が 4 件に固定されるわけではありません。例えば、先に解析したリクエストの予約量が減って次の受け入れに余裕ができれば、共有予算と他の制限の範囲内で、**非常に小さいリクエスト 8 件が同時にレスポンスをストリーミング**できます。 + +減量後の予約は、非同期処理やプロバイダーの応答待ちの間も保持され、レスポンスストリームの EOF、エラー、またはキャンセル処理の完了時に解放されます。解析だけでは解放されません。クライアントが切断しても、生成側のキャンセル処理が完了するまで保持されます。エラー、中断、キャンセルが競合しても、解放は正確に 1 回だけ行われます。 対象となる POST エンドポイントは `/v1/responses`、`/v1/responses/compact`、`/v1/chat/completions`、`/v1/messages`、`/v1/messages/count_tokens`、`/v1/images/generations`、`/v1/images/edits`、`/v1/alpha/search` です。画像、検索、トークン計数には、従来の設定可能な本文ごとの上限も適用されます。内部の直接変換やコンボ呼び出しは、HTTP の許容量を二重に予約しません。管理、音声、コンテキスト履歴、WebSocket の制限は変わりません。 -設定を省略またはゼロにした場合、あるいは解決された上限が 256 MiB 以下の場合、この追加の同時実行ゲートは有効になりません。既存のリクエスト件数やその他のリソース制限は引き続き適用されます。 +設定を省略またはゼロにした場合、あるいは適用される上限が 256 MiB 以下の場合、この追加の同時実行ゲートは有効になりません。既存のリクエスト件数やその他のリソース制限は引き続き適用されます。 ## 一時的な拒否と大きすぎる本文の違い -許容量が一時的に使えない場合、プロトコル解析やプロバイダーへの送信前に、HTTP **503**、`Retry-After: 1`、エラーコード `server_busy` が返されます。Messages とトークン計数のクライアントには、型が `overloaded_error` の Anthropic 形式のエラーが返されます。OpenAI 互換クライアントには `server_error` が返されます。リトライヘッダーとクライアント側のバックオフに従い、現在のリクエストが完了してから再試行してください。本文の上限をさらに引き上げても、予算不足は解消しません。 +許容量全体を一時的に予約できない場合、リクエストは **最大 32 件の FIFO 待機キュー**に入り、**最大 120 秒**待機できます。待機中は本文を読み取りません。測定後の予約量削減や最終解放で空きができると、待機順に受け入れます。待機中に中断されたリクエストはキューから除去され、未読の本文は消費せずキャンセルされます。 -本文がリクエスト単位の上限を超えた場合は、従来どおり HTTP **413** になります。宣言されたサイズが上限超過なら、混雑による拒否には置き換わりません。切断時のキャンセル動作も従来どおりです。認証とオリジンの確認は、この同時実行ゲートより先に行われます。 +キューが満杯、または待機時間が満了すると、本文の解析やプロバイダーへの送信前に HTTP **503**、`Retry-After: 1`、エラーコード `server_busy` が返されます。拒否時には未読の本文がキャンセルされます。Messages とトークン計数のクライアントには、型が `overloaded_error` の Anthropic 形式のエラーが、OpenAI 互換クライアントには `server_error` が返されます。認証とオリジンの確認は、受け入れやキューへの追加より先に行われます。 + +再試行時は、リトライヘッダーとクライアント側のバックオフに従ってください。上限のある待機と実測に基づく予約量の削減は、回避可能な容量不足による拒否を減らしますが、すべての 503 をなくすものではありません。リクエスト単位の上限をさらに引き上げても、継続的な容量不足は解決しません。 + +本文がリクエスト単位の上限を超えた場合は、従来どおり HTTP **413** になります。宣言されたサイズが上限超過なら、混雑による拒否には置き換わりません。 ## 予算が測るもの -512 MiB は受け入れたリクエストごとの許容量の合計であり、**プロセスのメモリ使用量が 512 MiB 未満に収まる保証ではありません**。デコード、文字列、オブジェクトグラフ、リクエストのコピー、その他のアプリケーション状態には追加のメモリが必要です。既存の UTF-8 と JSON の再シリアル化による測定は維持されます。入力のバイト長を、より大きくなり得る正規化後の JSON サイズの代わりには使用しません。 +**2 GiB** は受け入れを計上する予算で、解析前は許容量全体、解析後は実測に基づく予約量を合算します。**プロセス RSS や解析済みオブジェクトのヒープの上限ではなく**、メモリ使用量が 2 GiB 未満に収まる保証もありません。デコード、文字列、オブジェクトグラフ、リクエストのコピー、その他のアプリケーション状態には追加のメモリが必要です。 + +共有予算が 512 MiB から 2 GiB になり、予約できる合計量は 4 倍になりました。256 MiB を超える上限を選んだ環境では、512 MiB 全体を予約するアップロードを従来の 1 件から最大 4 件まで受け入れられるため、メモリ使用量のピークが増える可能性があります。解析済みオブジェクトやコピーによって、実際のメモリ使用量が受け入れ予算を超えることもあります。デフォルト以下の上限ではこのゲートは有効にならず、既存の動作を維持します。 -小さなリクエストを並列実行する場合は、可能な限りデフォルトの上限を維持してください。長い履歴を扱うために引き上げる際は、追加の同時実行制限を理解したうえで設定してください。 +可能な限り、リクエスト単位のデフォルト上限を維持してください。長い履歴を扱うために引き上げる場合は、メモリ使用量のピークが大きくなる可能性と待機の上限を考慮してください。 diff --git a/docs-site/src/content/docs/ko/reference/inbound-body-admission.md b/docs-site/src/content/docs/ko/reference/inbound-body-admission.md index ec9d5f5f4a2..1e3431e69a5 100644 --- a/docs-site/src/content/docs/ko/reference/inbound-body-admission.md +++ b/docs-site/src/content/docs/ko/reference/inbound-body-admission.md @@ -7,7 +7,15 @@ description: 요청 본문 한도를 높였을 때 동시 HTTP 요청, 재시도 ## 한도를 높이면 동시 처리량이 달라집니다 -적용된 한도가 256 MiB를 넘으면 해당 HTTP 요청마다 설정된 허용량 전체를 프로세스 전체가 공유하는 512 MiB 수용 예산에서 예약합니다. 따라서 여러 리스너가 한 프로세스를 공유하더라도 한 번에 요청 하나만 실행할 수 있습니다. 본문이 작아도 허용량 전체를 예약하며, 작은 Content-Length 값이나 압축으로 수용 검사를 우회할 수 없습니다. 예약은 요청 JSON 파싱 직후가 아니라 응답 스트림이 끝나거나 실패하거나 취소가 마무리될 때까지 유지됩니다. +적용된 한도가 256 MiB를 넘으면 해당 HTTP 요청은 **본문을 읽기 전에** 적용된 요청별 허용량 전체를 예약해야 합니다. 예약에 쓰는 공유 예산은 기존 512 MiB에서 **2 GiB**로 늘었습니다. 같은 프로세스의 여러 리스너도 이 예산을 함께 사용하며, `appOwnedMemoryBudgetMb`와는 별개입니다. + +크기를 제한하면서 JSON 파싱을 완료하기 전까지는 `Content-Length`가 작거나 압축된 전송 본문이 작아도 예약량을 줄이지 않습니다. 이 값들만으로 디코딩 후 크기나 정규화된 JSON 크기를 판단하지 않습니다. + +제한된 크기 안에서 JSON 파싱에 성공하면 리더는 실제 전송 바이트 수와 디코딩 후 바이트 수 중 큰 값으로 예약량을 줄입니다. 변환기가 정규화된 JSON 측정에 쓸 예산을 제공한 경우에는 기존 정규화 JSON 측정값까지 포함해 가장 큰 값을 사용합니다. 예약량은 처음 예약한 허용량 범위 안에서 줄어들기만 하며, 정규화 측정값이 그보다 커도 늘어나지 않습니다. 기존 UTF-8 및 JSON 재직렬화 검사는 유지되며, 입력 바이트 길이로 더 클 수 있는 정규화 JSON 크기를 대신하지 않습니다. + +요청별 한도가 512 MiB라면 측정으로 예약량을 줄이기 전에도 **512 MiB 전체를 예약하는 업로드를 최대 4건** 수용할 수 있습니다. 파싱 후까지 동시 요청 수를 4건으로 고정하지는 않습니다. 예를 들어 먼저 파싱된 요청의 예약량이 줄어 다음 요청을 수용할 여유가 생기면, 공유 예산과 다른 제한 안에서 **아주 작은 요청 8건도 동시에 응답을 스트리밍**할 수 있습니다. + +줄어든 예약량은 비동기 작업과 프로바이더 응답 대기 중에도 유지되며, 응답 스트림의 끝(EOF), 오류 또는 취소 완료 시점에 해제됩니다. 파싱만 끝났다고 해제하지 않습니다. 클라이언트 연결이 끊겨도 응답 생산자의 취소 처리가 끝날 때까지 유지합니다. 오류·요청 중단·취소가 겹치더라도 해제는 정확히 한 번만 일어납니다. 적용 대상 POST 엔드포인트는 `/v1/responses`, `/v1/responses/compact`, `/v1/chat/completions`, `/v1/messages`, `/v1/messages/count_tokens`, `/v1/images/generations`, `/v1/images/edits`, `/v1/alpha/search`입니다. 이미지, 검색, 토큰 계산에는 기존의 설정 가능한 본문별 한도가 계속 적용됩니다. 내부 직접 변환 호출과 콤보 호출은 HTTP 허용량을 두 번 예약하지 않습니다. 관리 API, 오디오, 컨텍스트 기록, WebSocket의 한도는 그대로입니다. @@ -15,12 +23,18 @@ description: 요청 본문 한도를 높였을 때 동시 HTTP 요청, 재시도 ## 일시적인 거부와 크기 초과 본문의 차이 -허용량을 일시적으로 사용할 수 없으면 프로토콜 파싱이나 프로바이더 호출 전에 HTTP **503**, `Retry-After: 1`, 오류 코드 `server_busy`를 반환합니다. Messages 및 토큰 계산 클라이언트에는 유형이 `overloaded_error`인 Anthropic 형식 오류를, OpenAI 호환 클라이언트에는 `server_error`를 반환합니다. 재시도 헤더와 클라이언트의 백오프를 지키면서 현재 요청이 끝난 뒤 다시 시도하세요. 본문 한도를 더 높여도 허용량 부족은 해결되지 않습니다. +허용량 전체를 바로 예약할 수 없으면 **최대 32건의 요청이 FIFO 대기열**에서 **최대 120초** 기다릴 수 있습니다. 대기 중에는 본문을 읽지 않습니다. 측정 후 예약량이 줄거나 요청이 끝나 예산이 반환되면 대기 순서대로 요청을 수용합니다. 대기 중 요청이 중단되면 대기열에서 제거하고, 읽지 않은 본문을 소비하지 않고 취소합니다. -요청별 한도보다 큰 본문은 기존 HTTP **413** 처리 경로를 따릅니다. 선언된 크기가 한도를 넘는 경우에도 일시적 거부가 아니라 그 경로를 사용합니다. 연결이 끊기면 기존 취소 동작을 유지합니다. 인증과 출처 검사는 이 동시 처리 게이트보다 먼저 실행됩니다. +대기열이 가득 찼거나 대기 시간이 만료되면 본문 파싱이나 프로바이더 호출 전에 HTTP **503**, `Retry-After: 1`, 오류 코드 `server_busy`를 반환하고, 읽지 않은 본문을 취소합니다. Messages 및 토큰 계산 클라이언트에는 유형이 `overloaded_error`인 Anthropic 형식 오류를, OpenAI 호환 클라이언트에는 `server_error`를 반환합니다. 인증과 출처 검사는 예약 및 대기열 진입보다 먼저 실행됩니다. + +재시도할 때는 재시도 헤더와 클라이언트의 백오프를 지키세요. 대기열과 파싱 후 예약량 조정은 불필요한 용량 부족 거부를 줄이지만 모든 503을 없애지는 않습니다. 요청별 본문 한도를 더 높여도 지속적인 용량 부족은 해결되지 않습니다. + +요청별 한도보다 큰 본문은 기존 HTTP **413** 처리 경로를 따릅니다. 선언된 크기가 한도를 넘는 경우에도 일시적 거부가 아니라 그 경로를 사용합니다. ## 예산이 측정하는 것 -512 MiB는 수용된 요청별 허용량의 합계이며, **프로세스 메모리가 512 MiB 아래로 유지된다는 보장은 아닙니다**. 디코딩, 문자열, 객체 그래프, 요청 사본, 기타 애플리케이션 상태에 추가 메모리가 듭니다. 기존 UTF-8 및 다시 직렬화한 JSON의 측정 방식은 유지됩니다. 입력 바이트 길이로 더 클 수 있는 정규화된 JSON 크기를 대신하지 않습니다. +**2 GiB**는 요청을 수용하기 위한 계산 예산입니다. 파싱 전에는 허용량 전체를, 파싱 후에는 측정된 예약량을 합산합니다. **프로세스 RSS나 파싱된 객체의 힙 크기 상한이 아니며**, 실제 메모리가 2 GiB 아래로 유지된다는 보장도 아닙니다. 디코딩, 문자열, 객체 그래프, 요청 사본, 기타 애플리케이션 상태에 추가 메모리가 듭니다. + +공유 예산을 512 MiB에서 2 GiB로 늘리면서 예약 가능한 총량도 4배가 됐습니다. 한도를 256 MiB보다 높여 사용하는 환경에서는 512 MiB 전체를 예약하는 업로드를 종전 1건 대신 최대 4건 수용할 수 있어, 순간 최대 메모리 사용량이 더 커질 수 있습니다. 파싱된 객체와 사본 때문에 실제 메모리 사용량이 수용 예산을 넘을 수도 있습니다. 기본 한도와 그 이하의 한도에는 이 추가 게이트가 적용되지 않으며 기존 동작을 유지합니다. -작은 요청을 병렬로 처리한다면 가능한 한 기본 한도를 유지하세요. 긴 기록이 필요한 워크플로에서는 추가 동시 처리 제한을 감수하고 한도를 높이세요. +가능한 한 요청별 기본 한도를 유지하세요. 긴 기록이 필요한 워크플로에서 한도를 높일 때는 커질 수 있는 순간 최대 메모리 사용량과 대기열의 한도도 함께 고려하세요. diff --git a/docs-site/src/content/docs/reference/inbound-body-admission.md b/docs-site/src/content/docs/reference/inbound-body-admission.md index b09e56d50fa..8c89b428e5d 100644 --- a/docs-site/src/content/docs/reference/inbound-body-admission.md +++ b/docs-site/src/content/docs/reference/inbound-body-admission.md @@ -10,14 +10,31 @@ changing the limit so the listener and readers use the intended limit together. ## Raising the limit changes concurrency -When the resolved limit is greater than 256 MiB, each covered HTTP request reserves its entire -configured allowance from a shared, process-wide 512 MiB admission budget. This means at most one -such request runs at a time, including when multiple listeners share a process. Small bodies also -reserve the full allowance; neither a small Content-Length nor compression bypasses admission. -The reservation remains until the response stream ends, fails, or finishes cancellation, rather -than ending as soon as the request JSON has been parsed. -A client disconnect releases the reservation after producer cancellation settles; limits above -256 MiB admit one covered request at a time regardless of `appOwnedMemoryBudgetMb`. +When the resolved limit is greater than 256 MiB, each covered HTTP request must reserve its full +resolved per-request allowance **before any body is read**. These reservations share a +process-wide **2 GiB** admission budget, increased from 512 MiB, including across listeners in +the same process. This budget is separate from `appOwnedMemoryBudgetMb`. + +Before bounded JSON parsing completes, a small `Content-Length` or a small compressed wire body +does not reduce the reservation. Admission does not trust either as evidence of the decoded or +normalized JSON size. + +After a successful bounded JSON parse, the reader reduces the reservation to the largest of +the observed wire bytes, decoded bytes, and the existing normalized-JSON measurement when a +translator supplies a budget for that measurement. The reservation only shrinks, accounting for +measured bytes up to the original allowance; a larger normalized measurement never increases it. +Input byte length does not replace the potentially larger normalized JSON size; the existing +UTF-8 and reserialized-JSON checks remain. + +With a 512 MiB per-request limit, the shared budget can admit up to **four full 512 MiB uploads** +before their reservations are measured and reduced. This is not a fixed four-request limit +after parsing: for example, **eight tiny requests can stream concurrently** once their reduced +reservations leave room for subsequent admissions, subject to the shared budget and other limits. + +The reduced reservation stays held across asynchronous work and provider waits, through response +stream EOF, error, or settled cancellation. Parsing alone does not release it. On a client +disconnect, it remains held until producer cancellation settles. Release happens exactly once, +including when errors, aborts, and cancellation race. The covered POST endpoints are `/v1/responses`, `/v1/responses/compact`, `/v1/chat/completions`, `/v1/messages`, `/v1/messages/count_tokens`, @@ -31,22 +48,36 @@ concurrency gate. Existing request-count and other resource limits still apply. ## Temporary refusal versus an oversized body -A temporarily unavailable allowance produces HTTP **503**, `Retry-After: 1`, and error code -`server_busy`, before protocol parsing or provider dispatch. Messages and token-counting clients -receive an Anthropic-shaped error with type `overloaded_error`; OpenAI-compatible clients receive -`server_error`. Retry after the current request completes, honoring the retry header and client -backoff. Increasing the body limit further does not resolve a busy allowance. +If the full allowance is temporarily unavailable, the request can enter a bounded **FIFO queue of +up to 32 waiting requests**, with a **120-second wait limit**. Waiting bodies remain unread. +Capacity freed by a measured reduction or final release admits waiting requests in FIFO order. +An aborted waiter is removed and its unread body is cancelled without consuming it. + +A full queue or an expired wait produces HTTP **503**, `Retry-After: 1`, and error code +`server_busy`, before body parsing or provider dispatch. Unread bodies are cancelled on refusal. +Messages and token-counting clients receive an Anthropic-shaped error with type +`overloaded_error`; OpenAI-compatible clients receive `server_error`. Authentication and origin +checks run before admission and queueing. + +Honor the retry header and client backoff when retrying. Bounded waiting and measured reservations +reduce avoidable capacity refusals, but do not eliminate all 503 responses. Raising the +per-request limit further does not fix sustained capacity exhaustion. A body larger than its per-request limit still follows the existing HTTP **413** handling. A declared oversize keeps that path instead of becoming a busy refusal. -Authentication and origin checks run before this concurrency gate. ## What the budget measures -512 MiB is a sum of admitted per-request allowances, **not a guarantee that process memory stays -below 512 MiB**. Decoding, strings, object graphs, request copies, and other application state use -additional memory. The existing UTF-8 and reserialized-JSON measurements are preserved; input -byte length is not substituted for the potentially larger normalized JSON size. +The **2 GiB** budget counts admission units: full allowances before parsing, then measured +reservations. It is **not a bound on process RSS or parsed-object heap**, and does not guarantee +memory usage below 2 GiB. Decoding, strings, object graphs, request copies, and other application +state use additional memory. + +Increasing the shared budget from 512 MiB to 2 GiB permits four times the total reservation. +Deployments that opt into limits above 256 MiB can now admit four full 512 MiB uploads instead +of one, so their potential peak memory use increases; parsed objects and copies can push actual +memory beyond the admission budget. Default and smaller limits do not activate this gate and +keep their existing behavior. -For parallel small requests, keep the default limit where possible. Raise it deliberately for a -large-history workflow, accepting the additional concurrency restriction. +Keep the default per-request limit where possible. Raise it deliberately when a large-history +workflow needs it, accounting for the larger potential memory peak and bounded waiting. diff --git a/docs-site/src/content/docs/ru/reference/inbound-body-admission.md b/docs-site/src/content/docs/ru/reference/inbound-body-admission.md index 1c739d0711c..e5397ed8e12 100644 --- a/docs-site/src/content/docs/ru/reference/inbound-body-admission.md +++ b/docs-site/src/content/docs/ru/reference/inbound-body-admission.md @@ -12,11 +12,33 @@ description: Как увеличение лимита тела запроса в ## Увеличение лимита меняет параллельность Если итоговый лимит превышает 256 МиБ, каждый охватываемый HTTP-запрос резервирует -весь настроенный объём из общего для процесса бюджета допуска в 512 МиБ. Поэтому -одновременно выполняется не более одного такого запроса, даже если процесс обслуживает -несколько слушателей. Небольшое тело тоже резервирует весь объём: ни малый Content-Length, -ни сжатие не обходят допуск. Резерв сохраняется до завершения потока ответа, ошибки -или окончания отмены, а не только до разбора JSON запроса. +весь лимит отдельного запроса **до чтения любых данных тела**. Общий для процесса бюджет +допуска увеличен с 512 МиБ до **2 ГиБ**; несколько слушателей в одном процессе также +делят этот бюджет. Он учитывается отдельно от `appOwnedMemoryBudgetMb`. + +До завершения разбора JSON с контролем размера ни малый `Content-Length`, ни малый размер +сжатого тела при передаче не уменьшают резерв. Допуск не считает эти значения надёжным +подтверждением размера декодированного или нормализованного JSON. + +После успешного разбора JSON в пределах ограничения обработчик чтения уменьшает резерв +до максимума из фактически полученных байтов, декодированных байтов и существующего +измерения нормализованного JSON, если преобразователь передал бюджет для этого измерения. +Существующие проверки UTF-8 и повторно сериализованного JSON сохраняются: длина входных +данных не заменяет потенциально больший размер нормализованного JSON. Резерв может +только уменьшаться и не превышает исходный лимит запроса; даже большее измерение +нормализованного JSON никогда не увеличивает резерв. + +При лимите 512 МиБ на запрос бюджет допускает до **4 загрузок, резервирующих по полных +512 МиБ**, ещё до измерения и уменьшения резервов. После разбора это не фиксированный +предел в 4 запроса: например, **8 очень маленьких запросов могут одновременно передавать +потоковые ответы**, когда уменьшение резервов позволяет допустить следующие запросы, +в пределах общего бюджета и других ограничений. + +Уменьшенный резерв сохраняется во время асинхронной работы и ожидания провайдера, +до конца потока ответа (EOF), ошибки или завершения отмены. Сам разбор его не освобождает. +При отключении клиента резерв сохраняется до завершения отмены на стороне производителя +ответа. Освобождение происходит ровно один раз, даже если ошибка, прерывание и отмена +возникают одновременно. Охватываются POST-эндпоинты `/v1/responses`, `/v1/responses/compact`, `/v1/chat/completions`, `/v1/messages`, `/v1/messages/count_tokens`, @@ -31,26 +53,41 @@ description: Как увеличение лимита тела запроса в ## Временный отказ и слишком большое тело -Временная нехватка допуска даёт HTTP **503**, `Retry-After: 1` и код ошибки -`server_busy` ещё до разбора протокола и отправки провайдеру. Клиенты Messages -и подсчёта токенов получают ошибку в формате Anthropic с типом `overloaded_error`; -OpenAI-совместимые клиенты — `server_error`. Повторите запрос после завершения -текущего, соблюдая заголовок повтора и задержку клиента. Дальнейшее увеличение -лимита тела не устраняет занятый допуск. +Если весь лимит временно нельзя зарезервировать, запрос может попасть в **очередь FIFO +не более чем из 32 ожидающих запросов**, с **ожиданием не более 120 секунд**. Во время +ожидания тела остаются непрочитанными. Уменьшение резерва после измерения или его +окончательное освобождение позволяет допускать запросы в порядке очереди. Прерванный +ожидающий запрос удаляется из очереди, а его непрочитанное тело отменяется без чтения. + +Полная очередь или истечение ожидания дают HTTP **503**, `Retry-After: 1` и код ошибки +`server_busy` до разбора тела и отправки провайдеру. При отказе непрочитанные тела отменяются. +Клиенты Messages и подсчёта токенов получают ошибку в формате Anthropic с типом +`overloaded_error`; OpenAI-совместимые клиенты — `server_error`. Проверки аутентификации +и источника выполняются до допуска и постановки в очередь. + +При повторе соблюдайте заголовок повтора и задержку клиента. Ограниченное ожидание и +измеренные резервы сокращают предотвратимые отказы из-за нехватки ёмкости, но не устраняют +все ответы 503. Дальнейшее увеличение лимита отдельного запроса не решает постоянное +исчерпание ёмкости. Тело, превышающее лимит отдельного запроса, по-прежнему получает HTTP **413**. Если превышение заявлено заранее, применяется именно этот путь, а не отказ из-за -занятости. При отключении сохраняется прежнее поведение отмены. Проверки -аутентификации и источника выполняются до барьера параллельности. +занятости. ## Что измеряет бюджет -512 МиБ — это сумма допущенных лимитов отдельных запросов, **а не гарантия того, -что память процесса останется ниже 512 МиБ**. Декодирование, строки, графы объектов, -копии запросов и другое состояние приложения занимают дополнительную память. -Существующие измерения UTF-8 и повторно сериализованного JSON сохраняются: длина -входных данных в байтах не подменяет потенциально больший размер нормализованного JSON. +Бюджет **2 ГиБ** считает единицы допуска: полные лимиты до разбора, затем измеренные +резервы. Это **не предел RSS процесса или кучи разобранных объектов** и не гарантия +потребления памяти ниже 2 ГиБ. Декодирование, строки, графы объектов, копии запросов +и другое состояние приложения занимают дополнительную память. + +Увеличение общего бюджета с 512 МиБ до 2 ГиБ разрешает вчетверо больший суммарный резерв. +Развёртывания, в которых выбран лимит выше 256 МиБ, теперь допускают до 4 загрузок +с резервом по полных 512 МиБ вместо одной, поэтому потенциальный пик потребления памяти +повышается. Разобранные объекты и копии могут увеличить фактическую память сверх бюджета +допуска. Лимит по умолчанию и меньшие значения не включают этот барьер и сохраняют +прежнее поведение. -Для параллельных небольших запросов по возможности оставляйте лимит по умолчанию. -Увеличивайте его осознанно для сценариев с большой историей, учитывая дополнительное -ограничение параллельности. +По возможности оставляйте лимит отдельного запроса по умолчанию. Увеличивайте его осознанно, +если этого требует сценарий с большой историей, учитывая больший потенциальный пик памяти +и ограниченное ожидание. diff --git a/docs-site/src/content/docs/tr/reference/inbound-body-admission.md b/docs-site/src/content/docs/tr/reference/inbound-body-admission.md index 573e8947957..c061800a904 100644 --- a/docs-site/src/content/docs/tr/reference/inbound-body-admission.md +++ b/docs-site/src/content/docs/tr/reference/inbound-body-admission.md @@ -11,13 +11,39 @@ okuyucuların aynı sınırı kullanması için proxy'yi yeniden başlatın. ## Sınırı artırmak eşzamanlılığı değiştirir -Çözümlenen sınır 256 MiB'ı aştığında kapsanan her HTTP isteği, yapılandırılmış -alanının tamamını süreç genelinde paylaşılan 512 MiB kabul bütçesinden ayırır. -Bu, birden çok dinleyici aynı süreci paylaşsa bile aynı anda en fazla bir böyle -isteğin çalışacağı anlamına gelir. Küçük gövdeler de alanın tamamını ayırır; -küçük bir Content-Length veya sıkıştırma kabulü atlatmaz. Rezervasyon, JSON -ayrıştırılır ayrıştırılmaz değil; yanıt akışı bitene, hata verene veya iptal -tamamlanana kadar sürer. +Etkin sınır 256 MiB'ı aştığında kapsanan her HTTP isteği, **gövdeden herhangi +bir veri okunmadan önce** istek başına etkin payın tamamını ayırmalıdır. Süreç +genelinde paylaşılan kabul bütçesi 512 MiB'dan **2 GiB**'a çıkarılmıştır; aynı +süreçteki birden çok dinleyici de bu bütçeyi paylaşır. Bu bütçe +`appOwnedMemoryBudgetMb` ayarından ayrıdır. + +Boyut denetimli JSON ayrıştırması tamamlanmadan önce küçük bir `Content-Length` +veya küçük bir sıkıştırılmış aktarım gövdesi rezervasyonu azaltmaz. Kabul +denetimi bu değerleri çözülmüş ya da normalleştirilmiş JSON boyutunun güvenilir +kanıtı olarak kullanmaz. + +Boyut sınırları içinde başarılı JSON ayrıştırmasından sonra okuyucu, +rezervasyonu aktarımda gözlenen bayt sayısı, çözülmüş bayt sayısı ve çevirici +ölçüm için bir bütçe sağladığında mevcut normalleştirilmiş JSON ölçümünün en +büyüğüne indirir. Mevcut UTF-8 ve yeniden serileştirilmiş JSON denetimleri +korunur; girdi bayt uzunluğu daha büyük olabilen normalleştirilmiş JSON +boyutunun yerine geçmez. Rezervasyon yalnızca azalır ve başlangıçta ayrılan +istek başına payı aşmaz; daha büyük bir normalleştirilmiş JSON ölçümü bile +rezervasyonu artırmaz. + +İstek başına sınır 512 MiB olduğunda bütçe, rezervasyonlar ölçülüp azaltılmadan +önce **her biri tam 512 MiB ayıran en fazla 4 yüklemeyi** kabul edebilir. +Ayrıştırma sonrasında eşzamanlılık 4 istekle sabitlenmez: örneğin azalan +rezervasyonlar sonraki kabullere yer açtığında, paylaşılan bütçe ve diğer +sınırlar izin verdiği sürece **8 çok küçük istek aynı anda yanıt akışı +iletebilir**. + +Azaltılan rezervasyon, asenkron çalışma ve sağlayıcı beklemeleri boyunca, +yanıt akışının sonuna (EOF), hataya veya iptal işleminin tamamlanmasına kadar +tutulur. Yalnızca ayrıştırmanın bitmesi onu serbest bırakmaz. İstemci +bağlantısı kesilirse üretici tarafındaki iptal tamamlanana kadar tutulur. +Hata, durdurma ve iptal aynı anda gerçekleşse bile tam olarak bir kez serbest +bırakılır. Kapsanan POST uç noktaları `/v1/responses`, `/v1/responses/compact`, `/v1/chat/completions`, `/v1/messages`, `/v1/messages/count_tokens`, @@ -26,31 +52,50 @@ Görseller, arama ve belirteç sayımı mevcut yapılandırılabilir gövde baş sınırlarını korur. İç doğrudan çeviri ve kombo çağrıları ikinci bir HTTP alanı ayırmaz. Yönetim, ses, bağlam geçmişi ve WebSocket sınırları değişmez. -Atlanan/sıfır ayarı veya 256 MiB ya da altında çözümlenen sınır, bu ek +Atlanan/sıfır ayarı veya 256 MiB ya da altındaki etkin sınır, bu ek eşzamanlılık kapısını etkinleştirmez. Mevcut istek sayısı ve diğer kaynak sınırları yine uygulanır. ## Geçici ret ile büyük gövde arasındaki fark -Geçici olarak kullanılamayan alan, protokol ayrıştırması veya sağlayıcıya -göndermeden önce HTTP **503**, `Retry-After: 1` ve `server_busy` hata kodunu -üretir. Messages ve belirteç sayan istemciler `overloaded_error` türünde -Anthropic biçimli hata; OpenAI uyumlu istemciler `server_error` alır. Yeniden -deneme başlığına ve istemci geri çekilmesine uyarak mevcut istek tamamlandıktan -sonra yeniden deneyin. Gövde sınırını daha da yükseltmek dolu alanı çözmez. +Payın tamamı geçici olarak ayrılamıyorsa istek, **en fazla 32 bekleyen istek +tutan FIFO kuyruğuna** girebilir; **bekleme sınırı 120 saniyedir**. Bekleme +sırasında gövdeler okunmaz. Ölçümle rezervasyon azalması veya son serbest +bırakma yer açtığında istekler kuyruk sırasıyla kabul edilir. Beklerken +durdurulan istek kuyruktan çıkarılır ve okunmamış gövdesi tüketilmeden iptal +edilir. + +Kuyruk doluysa veya bekleme süresi dolarsa gövde ayrıştırılmadan ya da +sağlayıcıya gönderilmeden önce HTTP **503**, `Retry-After: 1` ve `server_busy` +hata kodu döner. Ret sırasında okunmamış gövdeler iptal edilir. Messages ve +belirteç sayan istemciler `overloaded_error` türünde Anthropic biçimli hata; +OpenAI uyumlu istemciler `server_error` alır. Kimlik doğrulaması ve kaynak +denetimleri kabulden ve kuyruğa girişten önce çalışır. + +Yeniden deneme başlığına ve istemci geri çekilmesine uyun. Sınırlı bekleme ve +ölçülen rezervasyonlar önlenebilir kapasite retlerini azaltır, ancak tüm 503 +yanıtlarını ortadan kaldırmaz. İstek başına sınırı daha da yükseltmek sürekli +kapasite tükenmesini çözmez. İstek başına sınırından büyük gövde mevcut HTTP **413** işlemesini izler. -Bildirilen büyük boyut, meşgul ret yerine bu yolu korur. Bağlantı kopması -mevcut iptal davranışını korur. Kimlik doğrulaması ve kaynak denetimleri bu -eşzamanlılık kapısından önce çalışır. +Bildirilen büyük boyut, meşgul ret yerine bu yolu korur. ## Bütçe neyi ölçer? -512 MiB, kabul edilen istek başına alanların toplamıdır; **süreç belleğinin -512 MiB altında kalacağının garantisi değildir**. Çözümleme, dizeler, nesne -grafları, istek kopyaları ve diğer uygulama durumu ek bellek kullanır. Mevcut -UTF-8 ve yeniden serileştirilmiş JSON ölçümleri korunur; girdi bayt uzunluğu, -daha büyük olabilen normalleştirilmiş JSON boyutunun yerine geçmez. +**2 GiB** bütçesi kabul birimlerini sayar: ayrıştırma öncesinde tam payları, +sonrasında ölçülen rezervasyonları. **Süreç RSS'sinin veya ayrıştırılmış nesne +yığınının sınırı değildir** ve bellek kullanımının 2 GiB altında kalacağını +garanti etmez. Çözümleme, dizeler, nesne grafikleri, istek kopyaları ve diğer +uygulama durumu ek bellek kullanır. + +Paylaşılan bütçenin 512 MiB'dan 2 GiB'a çıkarılması toplam rezervasyon payını +dört katına çıkarır. Sınırı 256 MiB'ın üzerine çıkarmayı seçen kurulumlar, +her biri tam 512 MiB ayıran bir yükleme yerine artık en fazla 4 yüklemeyi kabul +edebilir; dolayısıyla olası en yüksek bellek kullanımı artar. Ayrıştırılmış +nesneler ve kopyalar gerçek bellek kullanımını kabul bütçesinin üzerine +çıkarabilir. Varsayılan ve daha küçük sınırlar bu kapıyı etkinleştirmez ve +mevcut davranışlarını korur. -Paralel küçük istekler için mümkünse varsayılan sınırı koruyun. Büyük geçmiş -iş akışı için artırırken ek eşzamanlılık kısıtını kabul edin. +Mümkünse istek başına varsayılan sınırı koruyun. Büyük geçmiş gerektiren bir +iş akışı için artırırken olası bellek tepe değerinin büyümesini ve sınırlı +beklemeyi hesaba katın. diff --git a/docs-site/src/content/docs/zh-cn/reference/inbound-body-admission.md b/docs-site/src/content/docs/zh-cn/reference/inbound-body-admission.md index 65727f30ea0..68735e6117e 100644 --- a/docs-site/src/content/docs/zh-cn/reference/inbound-body-admission.md +++ b/docs-site/src/content/docs/zh-cn/reference/inbound-body-admission.md @@ -7,20 +7,34 @@ description: 提高请求体上限如何影响并发 HTTP 请求、重试和内 ## 提高上限会改变并发能力 -当解析后的上限大于 256 MiB 时，每个受此规则约束的 HTTP 请求都会从进程共享的 512 MiB 准入预算中预留其完整配置额度。这意味着同时最多只能运行一个此类请求，即使多个监听器共用同一进程也是如此。较小的请求体也会预留全部额度；较小的 Content-Length 或压缩都不能绕过准入。预留会持续到响应流结束、失败或取消完成，而非请求 JSON 解析完毕时立即释放。 +当最终生效的上限大于 256 MiB 时，每个受此规则约束的 HTTP 请求都必须在**读取任何请求体数据之前**预留其完整的单请求额度。进程共享的准入预算已从 512 MiB 增至 **2 GiB**；同一进程内的多个监听器也共享此预算。该预算与 `appOwnedMemoryBudgetMb` 分开计算。 + +在完成有大小限制的 JSON 解析前，较小的 `Content-Length` 或较小的压缩传输体都不会减少预留量。准入控制不会将它们视为解码后或规范化 JSON 大小的可靠依据。 + +在大小限制内成功解析 JSON 后，读取器会将预留量降至实际传输字节数与解码后字节数中的较大值。如果转换器提供了用于测量规范化 JSON 的预算，还会将现有的规范化 JSON 测量值纳入比较，取最大值。预留量只会减少，且始终不超过最初的单请求额度；即使规范化测量值更大，也不会增加预留量。现有的 UTF-8 与 JSON 重新序列化检查保持不变；输入字节长度不会替代可能更大的规范化 JSON 大小。 + +单请求上限为 512 MiB 时，在测量并减少预留量之前，共享预算最多可接纳 **4 个各自预留完整 512 MiB 的上传请求**。这不意味着解析后也固定只能并发 4 个请求。例如，先解析的请求减少预留量、为后续准入腾出空间后，在共享预算和其他限制允许的范围内，**8 个很小的请求也可以同时流式返回响应**。 + +减少后的预留量在异步处理及等待提供商响应期间仍会保留，直到响应流 EOF、发生错误或取消处理完成。仅完成解析不会释放预留量。客户端断开连接时，也要等待响应生成端的取消处理完成后才释放。即使错误、中止和取消同时发生，释放也只执行一次。 受约束的 POST 端点包括 `/v1/responses`、`/v1/responses/compact`、`/v1/chat/completions`、`/v1/messages`、`/v1/messages/count_tokens`、`/v1/images/generations`、`/v1/images/edits` 和 `/v1/alpha/search`。图像、搜索和 token 计数仍遵守各自现有的可配置单请求体上限。内部直接转换和 combo 调用不会再次预留 HTTP 额度。管理、音频、上下文历史和 WebSocket 的上限不变。 -省略设置、设为零，或解析后上限不超过 256 MiB 时，不会启用这一额外的并发门槛。现有请求数量及其他资源限制仍然适用。 +省略设置、设为零，或最终生效的上限不超过 256 MiB 时，不会启用这一额外的并发门槛。现有请求数量及其他资源限制仍然适用。 ## 临时拒绝与请求体过大 -额度暂时不可用时，会在协议解析或提供商分发之前返回 HTTP **503**、`Retry-After: 1` 和错误代码 `server_busy`。Messages 与 token 计数客户端收到类型为 `overloaded_error` 的 Anthropic 格式错误；兼容 OpenAI 的客户端收到 `server_error`。请遵守重试标头和客户端退避机制，在当前请求完成后重试。进一步提高请求体上限无法解决额度繁忙问题。 +暂时无法预留完整额度时，请求可进入**最多容纳 32 个等待请求的 FIFO 队列**，**最多等待 120 秒**。等待期间不会读取请求体。测量后减少预留量或最终释放腾出空间时，按等待顺序接纳请求。等待中的请求被中止后会从队列移除，未读请求体会被取消而不会被消费。 -请求体超过单请求上限时，仍按现有逻辑返回 HTTP **413**。声明的大小已超限时，也会走这一路径，而不会变成繁忙拒绝。连接断开时仍遵循现有取消行为。认证和来源检查在并发门槛之前执行。 +队列已满或等待超时，会在请求体解析或提供商分发之前返回 HTTP **503**、`Retry-After: 1` 和错误代码 `server_busy`；拒绝时会取消未读请求体。Messages 与 token 计数客户端收到类型为 `overloaded_error` 的 Anthropic 格式错误；兼容 OpenAI 的客户端收到 `server_error`。认证和来源检查在准入及入队之前执行。 + +重试时请遵守重试标头和客户端退避机制。有界等待和按实测值减少预留量可以减少本可避免的容量拒绝，但不会消除所有 503。进一步提高单请求上限不能解决持续的容量耗尽。 + +请求体超过单请求上限时，仍按现有逻辑返回 HTTP **413**。声明的大小已超限时，也会走这一路径，而不会变成繁忙拒绝。 ## 预算衡量的内容 -512 MiB 是已准入请求各自额度之和，**并不保证进程内存低于 512 MiB**。解码、字符串、对象图、请求副本和其他应用状态还会使用额外内存。现有的 UTF-8 与重新序列化 JSON 的测量方式保持不变；不会用输入字节长度代替可能更大的规范化 JSON 大小。 +**2 GiB** 是准入计量预算：解析前累计完整额度，解析后累计实测预留量。它**不是进程 RSS 或已解析对象堆内存的上限**，也不保证内存用量低于 2 GiB。解码、字符串、对象图、请求副本和其他应用状态还会使用额外内存。 + +共享预算从 512 MiB 增至 2 GiB 后，可预留总量变为原来的 4 倍。主动将上限调高至 256 MiB 以上的部署，现在可接纳最多 4 个预留完整 512 MiB 的上传请求，而此前只能接纳 1 个，因此潜在的峰值内存用量会增加。已解析对象和副本可能使实际内存用量超过准入预算。默认及更小的上限不会启用此门槛，现有行为不变。 -对于并行的小请求，请尽可能保留默认上限。只有在需要较长历史记录的工作流中，才应有意识地提高上限，并接受额外的并发限制。 +请尽可能保留默认的单请求上限。在长历史记录工作流确有需要时再主动提高，并考虑可能增大的内存峰值及等待上限。 diff --git a/docs-site/src/content/docs/zh-tw/reference/inbound-body-admission.md b/docs-site/src/content/docs/zh-tw/reference/inbound-body-admission.md index aba73c9a415..ead7e3f0ed1 100644 --- a/docs-site/src/content/docs/zh-tw/reference/inbound-body-admission.md +++ b/docs-site/src/content/docs/zh-tw/reference/inbound-body-admission.md @@ -7,23 +7,37 @@ description: 提高請求主體上限如何影響並行 HTTP 請求、重試與 ## 提高上限會改變並行能力 -當解析後的上限超過 256 MiB，每個涵蓋的 HTTP 請求都會從程序共用的 512 MiB 准入預算中，預留完整的設定額度。因此同一時間最多只能執行一個這類請求，即使多個監聽器共用同一程序也是如此。小型主體同樣預留完整額度；較小的 Content-Length 或壓縮都不能繞過准入。預留額度會維持到回應串流結束、失敗或完成取消，而非在請求 JSON 解析完成後立即釋放。 +當實際生效的上限超過 256 MiB，每個涵蓋的 HTTP 請求都必須在**讀取任何主體資料之前**，預留完整的單次請求額度。程序共用的准入預算已從 512 MiB 增至 **2 GiB**；同一程序中的多個監聽器也共用此預算。這項預算與 `appOwnedMemoryBudgetMb` 分開計算。 + +在完成有大小限制的 JSON 解析前，即使 `Content-Length` 或壓縮後的傳輸主體很小，也不會減少預留量。准入控制不會將這些值當成解碼後或正規化 JSON 大小的可靠依據。 + +在大小限制內成功解析 JSON 後，讀取器會將預留量降至實測傳輸位元組數與解碼後位元組數中的較大值。若轉譯器提供用於測量正規化 JSON 的預算，還會將既有的正規化 JSON 測量值納入比較，採用最大值。預留量只會減少，且始終不超過最初的單次請求額度；即使正規化測量值更大，也不會增加預留量。既有的 UTF-8 與 JSON 重新序列化檢查會保留；輸入位元組長度不會取代可能更大的正規化 JSON 大小。 + +單次請求上限為 512 MiB 時，在測量並減少預留量之前，共用預算最多可接受 **4 個各自預留完整 512 MiB 的上傳請求**。解析後的並行請求數並未固定為 4 個。例如，先完成解析的請求減少預留量，讓後續請求能夠進入後，在共用預算與其他限制允許的範圍內，**8 個很小的請求也可以同時串流回傳回應**。 + +減少後的預留量會在非同步工作及等候供應商回應期間保留，直到回應串流 EOF、發生錯誤或取消處理完成才釋放。僅完成解析不會釋放預留量。用戶端中斷連線時，也會等到回應產生端的取消處理完成才釋放。即使錯誤、中止和取消同時發生，也只會釋放一次。 涵蓋的 POST 端點包括 `/v1/responses`、`/v1/responses/compact`、 `/v1/chat/completions`、`/v1/messages`、`/v1/messages/count_tokens`、 `/v1/images/generations`、`/v1/images/edits` 與 `/v1/alpha/search`。 圖片、搜尋與 token 計數仍沿用各自可設定的單一主體上限。內部直接轉譯與 combo 呼叫不會再次預留 HTTP 額度。管理、音訊、上下文歷史及 WebSocket 的上限不變。 -省略設定、設為零，或解析後的上限不超過 256 MiB，都不會啟用這道額外的並行閘門。現有的請求數及其他資源限制仍然適用。 +省略設定、設為零，或實際生效的上限不超過 256 MiB，都不會啟用這道額外的並行閘門。現有的請求數及其他資源限制仍然適用。 ## 暫時拒絕與主體過大的差別 -若額度暫時不可用，系統會在協定解析或供應商派送前回傳 HTTP **503**、`Retry-After: 1` 與錯誤碼 `server_busy`。Messages 與 token 計數用戶端會收到類 Anthropic 的錯誤，其類型為 `overloaded_error`；OpenAI 相容用戶端則會收到 `server_error`。請依重試標頭與用戶端退避機制，在目前請求完成後重試。進一步提高主體上限無法解決額度忙碌問題。 +暫時無法預留完整額度時，請求可進入**最多容納 32 個等候請求的 FIFO 佇列**，**最多等候 120 秒**。等候期間不會讀取主體。測量後減少預留量或最終釋放騰出空間時，會依等候順序接受請求。等候中的請求遭中止後，會從佇列移除，未讀取的主體會被取消，不會被消費。 -超過單次請求上限的主體仍依現有的 HTTP **413** 流程處理。已宣告的超大主體也會走這條流程，不會變成忙碌拒絕。中斷連線沿用現有的取消行為。身分驗證與 origin 檢查會先於這道並行閘門執行。 +佇列已滿或等候逾時時，系統會在主體解析或供應商派送前回傳 HTTP **503**、`Retry-After: 1` 與錯誤碼 `server_busy`；拒絕時會取消未讀取的主體。Messages 與 token 計數用戶端會收到類型為 `overloaded_error` 的 Anthropic 格式錯誤；OpenAI 相容用戶端則會收到 `server_error`。身分驗證與 origin 檢查會在准入及加入佇列之前執行。 + +重試時請遵守重試標頭與用戶端退避機制。有上限的等候與依實測值減少預留量，可以減少原本可避免的容量拒絕，但不會消除所有 503。進一步提高單次請求上限，無法解決持續的容量耗盡。 + +超過單次請求上限的主體仍依現有的 HTTP **413** 流程處理。已宣告的超大主體也會走這條流程，不會變成忙碌拒絕。 ## 預算衡量的是什麼 -512 MiB 是已准入請求的額度總和，**不保證程序記憶體用量低於 512 MiB**。解碼、字串、物件圖、請求副本與其他應用程式狀態都會使用額外記憶體。現有的 UTF-8 與重新序列化 JSON 計量會保留；輸入位元組長度不會取代可能更大的正規化 JSON 大小。 +**2 GiB** 是准入計量預算：解析前加總完整額度，解析後加總實測預留量。它**不是程序 RSS 或已解析物件堆積的上限**，也不保證記憶體用量低於 2 GiB。解碼、字串、物件圖、請求副本與其他應用程式狀態都會使用額外記憶體。 + +共用預算從 512 MiB 增至 2 GiB 後，可預留總量變為原本的 4 倍。主動將上限提高至 256 MiB 以上的部署環境，現在最多可接受 4 個預留完整 512 MiB 的上傳請求，先前則只能接受 1 個，因此潛在的尖峰記憶體用量會增加。已解析物件及副本可能讓實際記憶體用量超過准入預算。預設及更小的上限不會啟用這道閘門，既有行為不變。 -若要並行處理小型請求，盡可能維持預設上限。只有大型歷史記錄工作流程確實需要時才提高上限，並接受額外的並行限制。 +請盡可能維持預設的單次請求上限。長歷史記錄工作流程確有需要時再主動提高，並考慮可能增加的記憶體尖峰用量與等候上限。 diff --git a/scripts/test-layout/layout-server-inbound.json b/scripts/test-layout/layout-server-inbound.json new file mode 100644 index 00000000000..022ec21c211 --- /dev/null +++ b/scripts/test-layout/layout-server-inbound.json @@ -0,0 +1,4 @@ +{ + "inbound-body-admission.test.ts": "server", + "inbound-body-budget.test.ts": "server" +} diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index ba88b63756d..3d1221301b1 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1,7 +1,7 @@ { "version": 1, "root": "tests", - "explicitShards": ["layout-clients.json", "layout-codex-collision.json", "layout-agent-assertion.json", "layout-chat.json", "layout-replay.json", "layout-adapters.json", "layout-jev.json", "layout-workbuddy.json", "layout-combos.json", "layout-credit-renewal.json"], + "explicitShards": ["layout-clients.json", "layout-codex-collision.json", "layout-agent-assertion.json", "layout-chat.json", "layout-replay.json", "layout-adapters.json", "layout-jev.json", "layout-workbuddy.json", "layout-combos.json", "layout-credit-renewal.json", "layout-server-inbound.json"], "explicit": { "provider-input-modalities.test.ts": "providers", "combo-description.test.ts": "routing", "combo-description-catalog.test.ts": "codex-integration", "codex-provider-routing.test.ts": "codex-integration", "config-native-reasoning-retention.test.ts": "config", "server-native-reasoning-retention-merge.test.ts": "server", "server-native-reasoning-retention.test.ts": "server", "responses-retained-reasoning-stream-recovery.test.ts": "responses", "openai-native-hosted-search-history.test.ts": "adapters/openai", "responses-native-compact-search-history.test.ts": "responses", "config-hosted-search-projection.test.ts": "config", "gcp-credential-boundary.test.ts": "adapters/google", "gcp-sa-marker.test.ts": "adapters/google", "gcp-sa-adapter.test.ts": "adapters/google", "management-gcp-credential-input.test.ts": "server", "codex-preserve-signed-in.test.ts": "codex-integration", "cli-account-preservation.test.ts": "cli", "cli-provider-list-active.test.ts": "cli", "service-state-isolation.test.ts": "service", "compaction-history-framing.test.ts": "responses", "catalog-native-max-context.test.ts": "codex-integration", "protocol-argon-thought-boundary.test.ts": "responses", "service-log.test.ts": "service", "messages-native-empty-inline.test.ts": "claude-integration", "catalog-auto-review-override.test.ts": "codex-integration", "management-auto-review-settings.test.ts": "server", "management-auto-review-auth.test.ts": "server", "prefer-model-alias-ids.test.ts": "server", "responses-mimo-envelope.test.ts": "responses", "responses-call-id-linkage.test.ts": "responses", "bridge-terminal-refusal.test.ts": "adapters", "loop-terminal-refusal.test.ts": "images", "web-search-terminal-refusal.test.ts": "web-search", "combo-terminal-refusal.test.ts": "routing", "server-terminal-refusal-wiring.test.ts": "server", "incomplete-reason-normalization.test.ts": "responses", "advisor-retention.test.ts": "advisor", "responses-native-key-cooldown.test.ts": "responses", "responses-run-turn-key-readmission.test.ts": "responses", "qoder-spawn-readmission.test.ts": "adapters", "api-key-selection-cooldown.test.ts": "providers", "api-key-auto-pause-distribution.test.ts": "providers", "explicit-key-pause-oauth-boundary.test.ts": "providers", "server-chat-native-key-selection.test.ts": "server", "compact-key-selection.test.ts": "responses", "messages-native-key-selection.test.ts": "claude-integration", "coding-agent-windows-dispatch.test.ts": "adapters", "coding-agent-native-empty-argv.test.ts": "windows", "service-proxy-env.test.ts": "service", "openai-chat-standalone-bare-block.test.ts": "adapters/openai", "owner-registry-isolation.test.ts": "config", diff --git a/src/server/inbound-body-admission.ts b/src/server/inbound-body-admission.ts index a35e5faabfc..66e25953da9 100644 --- a/src/server/inbound-body-admission.ts +++ b/src/server/inbound-body-admission.ts @@ -1,5 +1,4 @@ import { - MAX_CONFIGURABLE_INBOUND_BODY_BYTES, MAX_DECOMPRESSED_BODY_BYTES, resolveInboundBodyLimitBytes, } from "./request-decompress"; @@ -7,10 +6,9 @@ import { withCors, type RequestPolicyView } from "./auth-cors"; import { codexCompatibleUrl } from "../codex/context-compat"; import { addFinalRequestLog, markLocalRequestLogRefusal } from "./request-log"; import type { WorkflowRefusalLog } from "./workflow-refusal"; +import { inboundBodyBudget, bindInboundBodyLease, type InboundBodyBudget } from "./inbound-body-budget"; -/** Admission units, not a bound on process RSS or the size of parsed object graphs. */ -export const MAX_CONCURRENT_INBOUND_BODY_BYTES = MAX_CONFIGURABLE_INBOUND_BODY_BYTES; -let reservedInboundBodyBytes = 0; +export { MAX_CONCURRENT_INBOUND_BODY_BYTES } from "./inbound-body-budget"; // Match the HTTP routes whose JSON readers use maxInboundBodyBytes. Keep the // current image/search/count_tokens support; management, audio, context relay, @@ -139,6 +137,7 @@ function retainUntilResponseSettles(response: Response, signal: AbortSignal, rel * EOF/error or settled cancellation. Call only at the admitted HTTP boundary, * after auth/origin checks, not from readers or internal combo/translation hops. * pathname is the same canonical pathname used by the server's dispatcher. + * The optional budget/wait override isolates lifecycle tests from the process-wide owner. */ export async function withRaisedInboundBodyAdmission( req: Request, @@ -146,6 +145,7 @@ export async function withRaisedInboundBodyAdmission( configuredLimit: number | undefined, work: () => Promise, onRefusal: (response: Response) => Response = response => response, + options?: { budget?: InboundBodyBudget; maxWaitMs?: number }, ): Promise { const maxBytes = resolveInboundBodyLimitBytes(configuredLimit); if (req.method !== "POST" || !CONFIGURABLE_JSON_BODY_ROUTES.has(pathname) @@ -159,18 +159,26 @@ export async function withRaisedInboundBodyAdmission( // protocol-specific 413 win over temporary capacity refusal, unchanged. return work(); } - if (maxBytes > MAX_CONCURRENT_INBOUND_BODY_BYTES - reservedInboundBodyBytes) { + let lease; + try { + lease = await (options?.budget ?? inboundBodyBudget).acquire(maxBytes, req.signal, options?.maxWaitMs); + } catch (error) { + cancelUnreadBody(req, error); + throw error; + } + if (!lease) { const error = new InboundBodyCapacityError(); cancelUnreadBody(req, error); return onRefusal(capacityResponse(pathname, error)); } - reservedInboundBodyBytes += maxBytes; - let released = false; - const release = (): void => { - if (released) return; - released = true; - reservedInboundBodyBytes -= maxBytes; - }; + // An immediate grant can race a disconnect before this async function resumes. + if (req.signal.aborted) { + lease.release(); + cancelUnreadBody(req, req.signal.reason); + throw req.signal.reason; + } + const unbind = bindInboundBodyLease(req, lease); + const release = (): void => { unbind(); lease.release(); }; try { return retainUntilResponseSettles(await work(), req.signal, release); } catch (error) { diff --git a/src/server/inbound-body-budget.ts b/src/server/inbound-body-budget.ts new file mode 100644 index 00000000000..7cc72d7250e --- /dev/null +++ b/src/server/inbound-body-budget.ts @@ -0,0 +1,115 @@ +/** Admission units for raised HTTP bodies, not a process RSS or parsed-heap limit. */ +export const MAX_CONCURRENT_INBOUND_BODY_BYTES = 2 * 1024 * 1024 * 1024; +export const MAX_QUEUED_INBOUND_BODIES = 32; +export const INBOUND_BODY_QUEUE_TIMEOUT_MS = 120_000; + +export interface InboundBodyLease { + /** Return unused allowance after the bounded reader measures the accepted body. */ + shrink(bytes: number): void; + /** Return the remaining charge exactly once when the HTTP owner settles. */ + release(): void; +} + +/** FIFO admission whose bounded waiters neither read uploads nor own reservations. */ +export function createInboundBodyBudget(capacity: number, maxQueued = MAX_QUEUED_INBOUND_BODIES) { + if (!Number.isSafeInteger(capacity) || capacity <= 0 + || !Number.isSafeInteger(maxQueued) || maxQueued < 0) { + throw new RangeError("Invalid inbound body budget"); + } + let reservedBytes = 0; + let activeLeases = 0; + const pending: Array<{ bytes: number; admit(): void }> = []; + /** Only the FIFO head may consume newly available capacity. */ + const drain = (): void => { + while (pending.length && pending[0]!.bytes <= capacity - reservedBytes) pending[0]!.admit(); + }; + /** Charge an allowance and give its HTTP owner an idempotent release handle. */ + const grant = (bytes: number): InboundBodyLease => { + let held = bytes; + let released = false; + reservedBytes += held; + activeLeases++; + return { + /** Never increase a reservation or accept an invalid measurement. */ + shrink(measured) { + if (released || !Number.isFinite(measured) || measured < 0) return; + const next = Math.min(held, Math.max(1, Math.ceil(measured))); + reservedBytes -= held - next; + held = next; + drain(); + }, + /** Free only this lease, including when EOF and cancellation race. */ + release() { + if (released) return; + released = true; + reservedBytes -= held; + activeLeases--; + drain(); + }, + }; + }; + return { + /** Observe counters without exposing request bodies or identities. */ + snapshot: () => ({ reservedBytes, activeLeases, queuedRequests: pending.length }), + /** Wait for an entire upload allowance; null preserves the caller's overload response. */ + acquire(bytes: number, signal: AbortSignal, waitMs = INBOUND_BODY_QUEUE_TIMEOUT_MS): Promise { + if (!Number.isSafeInteger(bytes) || bytes <= 0 || bytes > capacity) { + return Promise.reject(new RangeError("Invalid inbound body reservation")); + } + if (signal.aborted) return Promise.reject(signal.reason); + if (!pending.length && bytes <= capacity - reservedBytes) return Promise.resolve(grant(bytes)); + if (pending.length >= maxQueued || !Number.isFinite(waitMs) || waitMs <= 0) return Promise.resolve(null); + return new Promise((resolve, reject) => { + let settled = false; + let timer: ReturnType; + /** Detach every waiter resource before admitting, timing out, or rejecting it. */ + const cleanup = (): void => { + settled = true; + clearTimeout(timer); + signal.removeEventListener("abort", onAbort); + const index = pending.indexOf(entry); + if (index >= 0) pending.splice(index, 1); + }; + /** A cancelled head must not strand a smaller eligible follower. */ + const onAbort = (): void => { + if (settled) return; + cleanup(); + reject(signal.reason); + drain(); + }; + const entry = { bytes, + /** Transfer a queued caller to a charged lease once, before resolving it. */ + admit() { + if (settled) return; + cleanup(); + resolve(grant(bytes)); + }, + }; + pending.push(entry); + signal.addEventListener("abort", onAbort, { once: true }); + timer = setTimeout(() => { + if (settled) return; + cleanup(); + resolve(null); + drain(); + }, Math.min(waitMs, INBOUND_BODY_QUEUE_TIMEOUT_MS)); + if (signal.aborted) onAbort(); + }); + }, + }; +} + +export type InboundBodyBudget = ReturnType; +export const inboundBodyBudget = createInboundBodyBudget(MAX_CONCURRENT_INBOUND_BODY_BYTES); +const requestLeases = new WeakMap(); + +/** Connect only the admitted HTTP request to its bounded reader; internal reads own no lease. */ +export function bindInboundBodyLease(req: Request, lease: InboundBodyLease): () => void { + requestLeases.set(req, lease); + return () => { if (requestLeases.get(req) === lease) requestLeases.delete(req); }; +} + +/** Only an accepted, measured body can replace an HTTP owner's upload allowance. */ +export function updateInboundBodyReservation(req: Request, bytes: number): void { + requestLeases.get(req)?.shrink(bytes); +} diff --git a/src/server/request-decompress.ts b/src/server/request-decompress.ts index f2d2d71ff79..1c9a56ec127 100644 --- a/src/server/request-decompress.ts +++ b/src/server/request-decompress.ts @@ -1,6 +1,7 @@ import { Buffer } from "node:buffer"; import { gunzipSync, inflateRawSync, inflateSync, zstdDecompressSync } from "node:zlib"; import type { TranslatorBudget } from "../lib/translator-budget"; +import { updateInboundBodyReservation } from "./inbound-body-budget"; /** * Request-body decompression for the /v1/responses data plane. @@ -328,11 +329,14 @@ export async function readBoundedJsonRequestBody( // Count UTF-8 without allocating another request-sized byte array for diagnostics. releaseText = budget?.observeAcceptedRequestCopy(Buffer.byteLength(text, "utf8")); if (options && "emptyBodyFallback" in options && text.trim() === "") { + updateInboundBodyReservation(req, Math.max(raw.byteLength, decoded.byteLength)); return options.emptyBodyFallback; } const parsed = JSON.parse(text); // Keep the serialized-size contract: normalization can expand numeric literals. - budget?.observeAcceptedRequestCopy(Buffer.byteLength(JSON.stringify(parsed), "utf8")); + const parsedBytes = budget ? Buffer.byteLength(JSON.stringify(parsed), "utf8") : decoded.byteLength; + budget?.observeAcceptedRequestCopy(parsedBytes); + updateInboundBodyReservation(req, Math.max(raw.byteLength, decoded.byteLength, parsedBytes)); return parsed; } finally { releaseText?.(); diff --git a/structure/transports/byte-accounting.md b/structure/transports/byte-accounting.md index 13ae6cd51e5..350c25eb3be 100644 --- a/structure/transports/byte-accounting.md +++ b/structure/transports/byte-accounting.md @@ -95,10 +95,14 @@ still measure the uncompressed body; native codec workspace is not a counted Jav ## Raised HTTP body admission `src/server/inbound-body-admission.ts` reserves the full resolved `maxInboundBodyBytes` allowance -from a process-wide 512 MiB admission budget when the allowance exceeds the 256 MiB default. +from the process-wide 2 GiB budget in `src/server/inbound-body-budget.ts` when it exceeds the 256 MiB default. `src/server/index.ts` owns this lease in `runAdmittedHttpTurn`, after authentication and origin checks, before any request body is read. Default and smaller allowances do not consume this budget. -Missing or small Content-Length and compressed wire bodies do not reduce the reservation. +Missing or small Content-Length and compressed wire bodies do not reduce the upload reservation. +After successful bounded JSON parsing, the reader reduces it to the largest observed wire/decoded +size, including normalized JSON bytes when the existing translator measurement is enabled. +This reduction never exceeds the original allowance. The measured charge remains live; +parsing never frees the whole lease. An already-aborted request or explicitly oversized declaration keeps the existing abort/413 path. The lease covers upload, parsing, downstream awaits and response consumption. It is released on @@ -112,14 +116,20 @@ combo/translation calls share their HTTP owner's reservation rather than reservi The exact POST routes are Responses, compact, Chat Completions, Messages, count_tokens, image generations/edits and alpha search. Image/search/count_tokens retain their configured per-body limits. Management, audio, context relay and WebSocket frames retain their independent contracts. -Capacity refusal happens before protocol handlers, with HTTP 503, `Retry-After: 1`, and code +Capacity pressure queues at most 32 unread uploads in FIFO order for at most 120 seconds. +Returning unused allowance or settling a lease wakes eligible waiters; abort and timeout detach +their timers/listeners without changing another owner's reservation. A disconnect after a grant +but before its caller resumes releases it without dispatch. Full/expired queues refuse before +protocol handlers, with HTTP 503, `Retry-After: 1`, and code `server_busy`; Messages/count_tokens use the Anthropic `error`/`overloaded_error` envelope. The HTTP owner preserves receiving-listener CORS and records the refusal without reading the body. -This is an allowance budget, not a measured RSS or parsed-heap cap. All covered requests, even small -ones, serialize when configured above 256 MiB. It does not bound retained state beyond the HTTP -lifetime or change default-cap concurrency. `tests/server/server-request-body-size.test.ts` covers -lifecycle, cancellation races, protocol envelopes, and the real HTTP admission boundary. +This is an allowance budget, not a measured RSS or parsed-heap cap. Raising its ceiling from 512 MiB +to 2 GiB permits four full 512 MiB uploads and increases potential peak memory on raised-limit +deployments. Parsed small requests can overlap beyond four while their measured charges fit. +It does not bound state beyond the HTTP lifetime or change default-cap concurrency. +`tests/server/inbound-body-admission.test.ts` covers lifecycle, cancellation races, protocol +envelopes, and real HTTP streams; `tests/server/inbound-body-budget.test.ts` covers FIFO/accounting. ## Stream-buffer accounting diff --git a/tests/claude-integration/messages-request-id-endpoint.test.ts b/tests/claude-integration/messages-request-id-endpoint.test.ts index de63062a740..de09a7b36f2 100644 --- a/tests/claude-integration/messages-request-id-endpoint.test.ts +++ b/tests/claude-integration/messages-request-id-endpoint.test.ts @@ -5,11 +5,13 @@ import { join } from "node:path"; import { saveConfig } from "../../src/config"; import { startServer } from "../../src/server"; import { MAX_ACTIVE_TURNS, tryAdmitTurn } from "../../src/server/lifecycle"; -import { MAX_CONCURRENT_INBOUND_BODY_BYTES, withRaisedInboundBodyAdmission } from "../../src/server/inbound-body-admission"; +import { withRaisedInboundBodyAdmission } from "../../src/server/inbound-body-admission"; +import { MAX_CONFIGURABLE_INBOUND_BODY_BYTES } from "../../src/server/request-decompress"; import { readRecentUsageEntries } from "../../src/usage/log"; import { admitWorkflowTurn, chargeWorkflowSends, DEFAULT_WORKFLOW_BUDGET_POLICY, resetWorkflowBudgetsForTest } from "../../src/lib/workflow-budget"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; import { drainAndRemoveFixtureRoots } from "../helpers/fixture-teardown"; +import { saturateInboundBodyBudget } from "../helpers/inbound-body-budget"; import type { OcxConfig } from "../../src/types"; type Mode = "caller-forward" | "managed-native" | "translated"; @@ -230,11 +232,8 @@ for (const mode of ["caller-forward", "managed-native", "translated"] as const) } test("pre-handler body-capacity refusal receives the persisted OCX id and releases capacity", async () => { - launch("translated", "json", 200, { maxInboundBodyBytes: MAX_CONCURRENT_INBOUND_BODY_BYTES }); - const held = await withRaisedInboundBodyAdmission(new Request("http://localhost/v1/messages", { - method: "POST", body: "{}", - }), "/v1/messages", MAX_CONCURRENT_INBOUND_BODY_BYTES, - async () => new Response(new ReadableStream({}, { highWaterMark: 0 }))); + launch("translated", "json", 200, { maxInboundBodyBytes: MAX_CONFIGURABLE_INBOUND_BODY_BYTES }); + const restoreCapacity = await saturateInboundBodyBudget(); try { const response = await send("translated", false); expect(response.status).toBe(503); @@ -242,10 +241,10 @@ test("pre-handler body-capacity refusal receives the persisted OCX id and releas const row = await expectDurableId(response); expect(row.status).toBe(503); expect(row.errorCode).toBe("server_busy"); - } finally { await held.body!.cancel("fixture released"); } + } finally { await restoreCapacity(); } const admitted = await withRaisedInboundBodyAdmission(new Request("http://localhost/v1/messages", { method: "POST", body: "{}", - }), "/v1/messages", MAX_CONCURRENT_INBOUND_BODY_BYTES, async () => new Response(null, { status: 204 })); + }), "/v1/messages", MAX_CONFIGURABLE_INBOUND_BODY_BYTES, async () => new Response(null, { status: 204 })); expect(admitted.status).toBe(204); }); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 11138344887..f674d3a891e 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -794,6 +794,8 @@ "identity-subagent.test.ts": "adapters", "init-backup-cleanup.test.ts": "service", "init-eof.test.ts": "service", "initial-model-selection.test.ts": "providers", "initial-selection-write-fence.test.ts": "providers", "injection-model-api.test.ts": "codex-integration", + "inbound-body-admission.test.ts": "server", + "inbound-body-budget.test.ts": "server", "inline-think-boundaries.test.ts": "adapters/openai", "input-admission.test.ts": "server", "install-scripts.test.ts": "ci-workflows", "installed-gate-drivers.test.ts": "ci-workflows", "integrations-current-store.test.ts": "clients", "integrations-dsh-profile-patch.test.ts": "clients", "integrations-hermes-affinity.test.ts": "clients", diff --git a/tests/helpers/inbound-body-budget.ts b/tests/helpers/inbound-body-budget.ts new file mode 100644 index 00000000000..4547b647c33 --- /dev/null +++ b/tests/helpers/inbound-body-budget.ts @@ -0,0 +1,41 @@ +import assert from "node:assert/strict"; +import { + inboundBodyBudget, + MAX_CONCURRENT_INBOUND_BODY_BYTES, + MAX_QUEUED_INBOUND_BODIES, +} from "../../src/server/inbound-body-budget"; + +/** Fill admission counters and its queue without allocating request-sized bodies. */ +export async function saturateInboundBodyBudget(): Promise<() => Promise> { + const holder = await inboundBodyBudget.acquire( + MAX_CONCURRENT_INBOUND_BODY_BYTES, new AbortController().signal, 0, + ); + assert.ok(holder, "fixture requires an idle shared inbound body budget"); + const controllers: AbortController[] = []; + const waiting: Promise[] = []; + let restored = false; + const restore = async (): Promise => { + if (restored) return; + restored = true; + try { + for (const controller of controllers) controller.abort(); + await Promise.all(waiting); + } finally { holder.release(); } + }; + try { + for (let i = 0; i < MAX_QUEUED_INBOUND_BODIES; i++) { + const controller = new AbortController(); + controllers.push(controller); + waiting.push(inboundBodyBudget.acquire(1, controller.signal).catch(error => error)); + } + assert.deepEqual(inboundBodyBudget.snapshot(), { + reservedBytes: MAX_CONCURRENT_INBOUND_BODY_BYTES, + activeLeases: 1, + queuedRequests: MAX_QUEUED_INBOUND_BODIES, + }); + return restore; + } catch (error) { + await restore(); + throw error; + } +} diff --git a/tests/responses/chat-request-id-endpoint.test.ts b/tests/responses/chat-request-id-endpoint.test.ts index 04281dedd54..4b07e09b37d 100644 --- a/tests/responses/chat-request-id-endpoint.test.ts +++ b/tests/responses/chat-request-id-endpoint.test.ts @@ -5,11 +5,12 @@ import { join } from "node:path"; import { saveConfig } from "../../src/config"; import { startServer } from "../../src/server"; import { MAX_ACTIVE_TURNS, tryAdmitTurn } from "../../src/server/lifecycle"; -import { MAX_CONCURRENT_INBOUND_BODY_BYTES, withRaisedInboundBodyAdmission } from "../../src/server/inbound-body-admission"; +import { MAX_CONFIGURABLE_INBOUND_BODY_BYTES } from "../../src/server/request-decompress"; import { readRecentUsageEntries } from "../../src/usage/log"; import { admitWorkflowTurn, chargeWorkflowSends, DEFAULT_WORKFLOW_BUDGET_POLICY, resetWorkflowBudgetsForTest } from "../../src/lib/workflow-budget"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; import { drainAndRemoveFixtureRoots } from "../helpers/fixture-teardown"; +import { saturateInboundBodyBudget } from "../helpers/inbound-body-budget"; import type { OcxConfig } from "../../src/types"; let fixtureHome = ""; @@ -142,13 +143,10 @@ test("pre-handler workflow refusal has the logged history ID", async () => { }); test("pre-handler body-capacity refusal has a history ID", async () => { - launch(false, 200, { maxInboundBodyBytes: MAX_CONCURRENT_INBOUND_BODY_BYTES }); - const held = await withRaisedInboundBodyAdmission(new Request("http://localhost/v1/chat/completions", { - method: "POST", body: "{}", - }), "/v1/chat/completions", MAX_CONCURRENT_INBOUND_BODY_BYTES, - async () => new Response(new ReadableStream({}, { highWaterMark: 0 }))); + launch(false, 200, { maxInboundBodyBytes: MAX_CONFIGURABLE_INBOUND_BODY_BYTES }); + const restoreCapacity = await saturateInboundBodyBudget(); try { await durableRow(await send(), 503); } - finally { await held.body!.cancel("fixture released"); } + finally { await restoreCapacity(); } }); test("stream cancellation keeps its eventual history ID", async () => { diff --git a/tests/server/inbound-body-admission.test.ts b/tests/server/inbound-body-admission.test.ts new file mode 100644 index 00000000000..eacb75d1d79 --- /dev/null +++ b/tests/server/inbound-body-admission.test.ts @@ -0,0 +1,811 @@ +import { afterEach, beforeEach, describe, test } from "bun:test"; +import assert from "node:assert/strict"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { gzipSync } from "node:zlib"; +import { getDefaultConfig, saveConfig } from "../../src/config"; +import { createTranslatorBudget } from "../../src/lib/translator-budget"; +import { startServer } from "../../src/server"; +import { + MAX_DECOMPRESSED_BODY_BYTES, + MAX_CONFIGURABLE_INBOUND_BODY_BYTES, + readJsonRequestBody, + DecompressedBodyTooLargeError, + UnsupportedContentEncodingError, +} from "../../src/server/request-decompress"; +import { + withRaisedInboundBodyAdmission, + InboundBodyCapacityError, + CONFIGURABLE_JSON_BODY_ROUTES, + UNGATED_LOOPBACK_ROUTES, +} from "../../src/server/inbound-body-admission"; +import { createInboundBodyBudget, inboundBodyBudget } from "../../src/server/inbound-body-budget"; +import { drainAndRemoveFixtureRoots } from "../helpers/fixture-teardown"; +import { saturateInboundBodyBudget } from "../helpers/inbound-body-budget"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { repoPath } from "../helpers/repo-root"; + +let testBudget: ReturnType; +function fixtureAdmission(...args: Parameters) { + return withRaisedInboundBodyAdmission(args[0], args[1], args[2], args[3], args[4], { + budget: testBudget, maxWaitMs: 0, + }); +} + +// BEGIN raised-body admission regressions: these cases use Web streams and tiny payloads. +const RAISED_BODY_LIMIT = MAX_CONFIGURABLE_INBOUND_BODY_BYTES; +// Read the gate's own set rather than a second copy: a route added to admission +// without protocol coverage, or covered here but never admitted, must not pass. +const raisedBodyPaths = [...CONFIGURABLE_JSON_BODY_ROUTES]; +function bodyDeferred() { + let resolve!: () => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} +function admissionRequest(path = "/v1/responses", body: BodyInit = "{}", init: RequestInit = {}): Request { + return new Request(`http://localhost${path}`, { + method: "POST", body, ...init, duplex: "half", + } as RequestInit); +} +function admittedBodyWork(work: () => Promise, req = admissionRequest(), limit = RAISED_BODY_LIMIT) { + return fixtureAdmission(req, new URL(req.url).pathname, limit, work); +} +async function assertBodyCapacity(status: number) { + const response = await admittedBodyWork(async () => new Response(null, { status: 204 })); + assert.equal(response.status, status); + await response.text(); +} + +describe("raised inbound body admission lifetime", () => { + beforeEach(() => { testBudget = createInboundBodyBudget(RAISED_BODY_LIMIT); }); + afterEach(() => assert.deepEqual(testBudget.snapshot(), { + reservedBytes: 0, activeLeases: 0, queuedRequests: 0, + })); + test("keeps the parsed graph admitted while downstream work is pending", async () => { + const parsed = bodyDeferred(); + const complete = bodyDeferred(); + const req = admissionRequest("/v1/responses", '{"input":"retained fixture"}'); + const first = admittedBodyWork(async () => { + const raw = await readJsonRequestBody(req, undefined, RAISED_BODY_LIMIT) as { input: string }; + parsed.resolve(); + await complete.promise; + return Response.json({ input: raw.input }); + }, req); + try { + await parsed.promise; + assert.deepEqual(testBudget.snapshot(), { + reservedBytes: Buffer.byteLength('{"input":"retained fixture"}'), activeLeases: 1, queuedRequests: 0, + }); + await assertBodyCapacity(503); + complete.resolve(); + const response = await first; + await assertBodyCapacity(503); // Returning headers is not the end of the body lifetime. + assert.deepEqual(await response.json(), { input: "retained fixture" }); + await assertBodyCapacity(204); + } finally { + complete.resolve(); + await (await first).body?.cancel().catch(() => undefined); + } + }); + + test("reserves before the first upload finishes", async () => { + const started = bodyDeferred(); + let controller!: ReadableStreamDefaultController; + const req = admissionRequest("/v1/responses", new ReadableStream({ + start(value) { controller = value; }, + pull() { started.resolve(); }, + }, { highWaterMark: 0 })); + const first = admittedBodyWork(async () => { + await readJsonRequestBody(req, undefined, RAISED_BODY_LIMIT); + return new Response(null, { status: 204 }); + }, req); + try { + await started.promise; + await assertBodyCapacity(503); + } finally { + controller.enqueue(new TextEncoder().encode("{}")); + controller.close(); + await first; + } + await assertBodyCapacity(204); + }); + + for (const path of raisedBodyPaths) { + test(`returns retryable overload before the ${path} handler can wrap it`, async () => { + const holder = await admittedBodyWork(async () => new Response("held")); + let dispatched = false; + try { + const req = admissionRequest(path, "private-fixture-body"); + const response = await admittedBodyWork(async () => { + dispatched = true; + return Response.json({ error: "would wrap parser failures as 400" }, { status: 400 }); + }, req); + assert.equal(response.status, 503); + assert.equal(response.headers.get("retry-after"), "1"); + const payload = await response.json() as { type?: string; error: { type: string; code: string } }; + assert.equal(payload.error.code, "server_busy"); + const anthropic = path.startsWith("/v1/messages"); + assert.equal(payload.error.type, anthropic ? "overloaded_error" : "server_error"); + assert.equal(payload.type, anthropic ? "error" : undefined); + assert.equal(JSON.stringify(payload).includes("private-fixture-body"), false); + assert.equal(dispatched, false); + } finally { await holder.text(); } + await assertBodyCapacity(204); + }); + } + + for (const limit of [undefined, 0, MAX_DECOMPRESSED_BODY_BYTES, 1024 * 1024, NaN, Infinity]) { + test(`preserves ordinary concurrency for limit ${String(limit)}`, async () => { + const holder = await admittedBodyWork(async () => new Response("held")); + const original = new Response("unchanged"); + try { + const response = await fixtureAdmission(admissionRequest(), "/v1/responses", limit, async () => original); + assert.equal(response, original); + await response.text(); + } finally { await holder.text(); } + }); + } + + test("does not charge management, audio, context, or GET routes", async () => { + const holder = await admittedBodyWork(async () => new Response("held")); + try { + for (const path of ["/api/config", "/v1/audio/transcriptions", "/v1/live", "/v1/context/history", "/v1/unknown"]) { + const original = new Response(null, { status: 204 }); + assert.equal(await admittedBodyWork(async () => original, admissionRequest(path)), original); + } + const get = new Request("http://localhost/v1/responses"); + assert.equal((await admittedBodyWork(async () => new Response(null, { status: 204 }), get)).status, 204); + } finally { await holder.text(); } + }); + + // The dispatcher is the source of truth for what the loopback listener accepts. + // A new `/v1` route must be admitted or explicitly exempted; leaving it out of + // both sets is how a raised `maxInboundBodyBytes` route would skip the + // process-wide reservation while every admission test still passed. + test("every loopback /v1 route is classified for inbound body admission", async () => { + const source = await Bun.file(repoPath("src", "server", "index.ts")).text(); + const start = source.indexOf("function loopbackRouteAllowed"); + assert.notEqual(start, -1); + const end = source.indexOf("\n }", start); + assert.notEqual(end, -1); + const routes = [...source.slice(start, end).matchAll(/path === "(\/v1\/[^"]+)"/g)].map(match => match[1]); + assert.ok(routes.length >= 8, `expected the dispatcher to list /v1 routes, saw ${routes.length}`); + for (const route of routes) { + assert.ok( + CONFIGURABLE_JSON_BODY_ROUTES.has(route) || UNGATED_LOOPBACK_ROUTES.has(route), + `${route} is neither admitted nor explicitly exempt in inbound-body-admission.ts`, + ); + } + for (const route of CONFIGURABLE_JSON_BODY_ROUTES) { + assert.ok(routes.includes(route), `${route} is admitted but the dispatcher no longer serves it`); + } + }); + + test("keeps an aborted request charged until its pending work actually returns", async () => { + const abort = new AbortController(); + const started = bodyDeferred(); + const complete = bodyDeferred(); + const req = admissionRequest("/v1/responses", "{}", { signal: abort.signal }); + const first = admittedBodyWork(async () => { + await readJsonRequestBody(req, undefined, RAISED_BODY_LIMIT); + started.resolve(); + await complete.promise; + return new Response(null, { status: 499 }); + }, req); + try { + await started.promise; + abort.abort(new Error("fixture cancellation")); + await assertBodyCapacity(503); + } finally { complete.resolve(); await first; } + await assertBodyCapacity(204); + }); + + for (const rejects of [false, true]) { + test(`request abort cancels an unconsumed response and waits for settlement (rejects=${rejects})`, async () => { + const abort = new AbortController(); + const finishCancel = bodyDeferred(); + const reason = new Error("fixture request abort"); + let cancelledWith: unknown; + let cancellations = 0; + let pulls = 0; + const response = await admittedBodyWork(async () => new Response(new ReadableStream({ + pull() { pulls++; }, + async cancel(value) { + cancellations++; + cancelledWith = value; + await finishCancel.promise; + if (rejects) throw new Error("fixture producer cancellation failure"); + }, + }, { highWaterMark: 0 })), admissionRequest("/v1/responses", "{}", { signal: abort.signal })); + try { + await assertBodyCapacity(503); + abort.abort(reason); + await assertBodyCapacity(503); + assert.equal(cancellations, 1); + assert.equal(cancelledWith, reason); + assert.equal(pulls, 0); + assert.equal(response.bodyUsed, false); + finishCancel.resolve(); + await Bun.sleep(0); // Let cancellation and the admission finalizer settle. + await assertBodyCapacity(204); + } finally { + finishCancel.resolve(); + await response.body?.cancel().catch(() => undefined); + } + }); + } + + test("request abort during work cancels its eventual body before releasing capacity", async () => { + const abort = new AbortController(); + const started = bodyDeferred(); + const complete = bodyDeferred(); + const finishCancel = bodyDeferred(); + let cancellations = 0; + const first = admittedBodyWork(async () => { + started.resolve(); + await complete.promise; + return new Response(new ReadableStream({ + cancel() { cancellations++; return finishCancel.promise; }, + }, { highWaterMark: 0 })); + }, admissionRequest("/v1/responses", "{}", { signal: abort.signal })); + try { + await started.promise; + abort.abort(); + await assertBodyCapacity(503); + assert.equal(cancellations, 0); + complete.resolve(); + await first; + await assertBodyCapacity(503); + assert.equal(cancellations, 1); + finishCancel.resolve(); + await Bun.sleep(0); + await assertBodyCapacity(204); + } finally { + complete.resolve(); + finishCancel.resolve(); + await (await first).body?.cancel().catch(() => undefined); + } + }); + + for (const abortFirst of [false, true]) { + test(`request abort and explicit cancel release exactly once (abortFirst=${abortFirst})`, async () => { + const abort = new AbortController(); + const finishCancel = bodyDeferred(); + const reason = new Error("fixture request abort race"); + let cancellations = 0; + let cancelledWith: unknown; + const response = await admittedBodyWork(async () => new Response(new ReadableStream({ + cancel(value) { + cancellations++; + cancelledWith = value; + return finishCancel.promise; + }, + }, { highWaterMark: 0 })), admissionRequest("/v1/responses", "{}", { signal: abort.signal })); + let next: Response | undefined; + try { + if (abortFirst) abort.abort(reason); + const cancelled = response.body!.cancel("fixture explicit cancel").catch(error => error); + if (!abortFirst) abort.abort(reason); + await assertBodyCapacity(503); + assert.equal(cancellations, 1); + assert.equal(cancelledWith, abortFirst ? reason : "fixture explicit cancel"); + finishCancel.resolve(); + await cancelled; + await Bun.sleep(0); + await assertBodyCapacity(204); + next = await admittedBodyWork(async () => new Response("next reservation")); + assert.equal(next.status, 200); + await assertBodyCapacity(503); // A double release would admit this peer. + assert.equal(cancellations, 1); + } finally { + finishCancel.resolve(); + await response.body?.cancel().catch(() => undefined); + await next?.body?.cancel(); + } + await assertBodyCapacity(204); + }); + } + + test("preserves pre-aborted requests' existing error path", async () => { + const holder = await admittedBodyWork(async () => new Response("held")); + const abort = new AbortController(); + abort.abort(); + try { + const response = await admittedBodyWork(async () => new Response(null, { status: 499 }), + admissionRequest("/v1/responses", "{}", { signal: abort.signal })); + assert.equal(response.status, 499); + } finally { await holder.text(); } + }); + + test("does not double reserve internal body clones or translated reads", async () => { + const req = admissionRequest(); + const clone = req.clone(); + const response = await admittedBodyWork(async () => { + assert.deepEqual(await readJsonRequestBody(req, undefined, RAISED_BODY_LIMIT), {}); + assert.deepEqual(await readJsonRequestBody(clone, undefined, RAISED_BODY_LIMIT), {}); + return new Response(null, { status: 204 }); + }, req); + assert.equal(response.status, 204); + await assertBodyCapacity(204); + }); + + test("retains the original oversized-declaration refusal before capacity", async () => { + const holder = await admittedBodyWork(async () => new Response("held")); + const req = admissionRequest("/v1/responses", "{}", { headers: { "content-length": String(RAISED_BODY_LIMIT + 1) } }); + try { + await assert.rejects(admittedBodyWork(async () => { + await readJsonRequestBody(req, undefined, RAISED_BODY_LIMIT); + return new Response(null, { status: 204 }); + }, req), DecompressedBodyTooLargeError); + await assertBodyCapacity(503); // Rejecting an oversized peer cannot release the first lease. + } finally { await holder.text(); } + }); + + for (const declared of ["", "invalid", "-1", "Infinity", "0", "1"]) { + test(`an untrusted content-length ${JSON.stringify(declared)} cannot evade capacity`, async () => { + const holder = await admittedBodyWork(async () => new Response("held")); + try { + const req = admissionRequest("/v1/responses", "{}", { headers: { "content-length": declared } }); + const response = await admittedBodyWork(async () => new Response(null, { status: 204 }), req); + assert.equal(response.status, 503); + await response.text(); + } finally { await holder.text(); } + }); + } + + for (const failure of ["json", "gzip", "encoding", "downstream"]) { + test(`releases on ${failure} failure without changing the error`, async () => { + const req = admissionRequest("/v1/responses", failure === "json" ? "{" : "{}", { + headers: failure === "gzip" ? { "content-encoding": "gzip" } + : failure === "encoding" ? { "content-encoding": "not-a-codec" } : {}, + }); + const downstream = new Error("fixture downstream failure"); + await assert.rejects(admittedBodyWork(async () => { + await readJsonRequestBody(req, undefined, RAISED_BODY_LIMIT); + throw downstream; + }, req), (error: unknown) => failure === "json" ? error instanceof SyntaxError + : failure === "encoding" ? error instanceof UnsupportedContentEncodingError + : failure === "downstream" ? error === downstream : error instanceof Error); + await assertBodyCapacity(204); + }); + } + + test("reserves the full compressed upload allowance before retaining its measured wire size", async () => { + const wire = gzipSync(Buffer.from("{}")); + const req = admissionRequest("/v1/responses", wire, { headers: { "content-encoding": "gzip" } }); + const holder = await admittedBodyWork(async () => { + assert.equal(testBudget.snapshot().reservedBytes, RAISED_BODY_LIMIT); + assert.deepEqual(await readJsonRequestBody(req, undefined, RAISED_BODY_LIMIT), {}); + return new Response("held"); + }, req); + try { + assert.equal(testBudget.snapshot().reservedBytes, wire.byteLength); + await assertBodyCapacity(503); + } finally { await holder.text(); } + await assertBodyCapacity(204); + }); + + test("does not wait for cancellation of a refused tee", async () => { + const holder = await admittedBodyWork(async () => new Response("held")); + const cancelled = bodyDeferred(); + const unblockCancel = bodyDeferred(); + let reason: unknown; + const req = admissionRequest("/v1/responses", new ReadableStream({ + cancel(value) { reason = value; cancelled.resolve(); return unblockCancel.promise; }, + }, { highWaterMark: 0 })); + try { + const response = await admittedBodyWork(async () => new Response(null, { status: 204 }), req); + await cancelled.promise; + assert.equal(response.status, 503); + assert.ok(reason instanceof InboundBodyCapacityError); + await response.text(); + } finally { unblockCancel.resolve(); await holder.text(); } + }); + + test("retains a stream through EOF and preserves headers, bytes and backpressure", async () => { + let pulls = 0; + const chunk = new Uint8Array([1, 2, 3]); + const response = await admittedBodyWork(async () => new Response(new ReadableStream({ + pull(controller) { pulls++; if (pulls === 1) controller.enqueue(chunk); else controller.close(); }, + }, { highWaterMark: 0 }), { status: 201, statusText: "Fixture", headers: { "x-fixture": "preserved" } })); + assert.equal(pulls, 0); + assert.equal(response.status, 201); + assert.equal(response.statusText, "Fixture"); + assert.equal(response.headers.get("x-fixture"), "preserved"); + const reader = response.body!.getReader(); + try { + assert.equal((await reader.read()).value, chunk); // no body-sized copy + assert.equal(pulls, 1); + await assertBodyCapacity(503); + assert.equal((await reader.read()).done, true); + await assertBodyCapacity(204); + } finally { await reader.cancel(); reader.releaseLock(); } + }); + + test("releases a failed response stream and preserves its original error", async () => { + const error = new Error("fixture stream error"); + const response = await admittedBodyWork(async () => new Response(new ReadableStream({ + pull(controller) { controller.error(error); }, + }, { highWaterMark: 0 }))); + await assert.rejects(response.text(), (actual: unknown) => actual === error); + await assertBodyCapacity(204); + }); + + for (const rejects of [false, true]) { + test(`pending-read EOF never releases before cancellation settles (rejects=${rejects})`, async () => { + const reading = bodyDeferred(); + const finishCancel = bodyDeferred(); + const cancellation = new Error("fixture cancel error"); + let cancelledWith: unknown; + const response = await admittedBodyWork(async () => new Response(new ReadableStream({ + pull() { reading.resolve(); }, + async cancel(reason) { + cancelledWith = reason; + await finishCancel.promise; + if (rejects) throw cancellation; + }, + }, { highWaterMark: 0 }))); + const reader = response.body!.getReader(); + const pendingRead = reader.read(); + await reading.promise; + const cancelResult = reader.cancel("fixture stop").then(() => undefined, error => error); + try { + assert.equal((await pendingRead).done, true); + await Promise.resolve(); + await assertBodyCapacity(503); + assert.equal(cancelledWith, "fixture stop"); + } finally { finishCancel.resolve(); } + assert.equal(await cancelResult, rejects ? cancellation : undefined); + reader.releaseLock(); + await assertBodyCapacity(204); + }); + } + + test("runs the caller's refusal decoration once without dispatching work", async () => { + const holder = await admittedBodyWork(async () => new Response("held")); + let decorated = 0; + try { + const response = await fixtureAdmission(admissionRequest(), "/v1/responses", RAISED_BODY_LIMIT, + async () => { throw new Error("must not dispatch"); }, refused => { + decorated++; + refused.headers.set("x-fixture-log", "once"); + return refused; + }); + assert.equal(decorated, 1); + assert.equal(response.headers.get("x-fixture-log"), "once"); + await response.text(); + } finally { await holder.text(); } + }); + + for (const declared of [null, "", "invalid", "-1", "Infinity", "NaN", "0", "1", "2", "1.5"]) { + test(`content-length ${JSON.stringify(declared)} cannot reduce the upfront allowance`, async () => { + const holder = await testBudget.acquire(1, new AbortController().signal, 0); + assert.ok(holder); + let dispatched = false; + const req = admissionRequest("/v1/responses", "{}", { + headers: declared === null ? {} : { "content-length": declared }, + }); + try { + const response = await admittedBodyWork(async () => { + dispatched = true; + return new Response(null, { status: 204 }); + }, req); + assert.equal(response.status, 503); + await response.text(); + assert.equal(dispatched, false); + assert.deepEqual(testBudget.snapshot(), { reservedBytes: 1, activeLeases: 1, queuedRequests: 0 }); + } finally { holder.release(); } + }); + } + + for (const grant of ["immediate", "queued"] as const) { + test(`${grant} grant followed by abort before the caller resumes never dispatches`, async () => { + const holder = grant === "queued" ? await testBudget.acquire(RAISED_BODY_LIMIT, new AbortController().signal) : null; + const abort = new AbortController(); + const reason = new Error("fixture disconnect after grant"); + let dispatched = false; + let cancelledWith: unknown; + let cancellations = 0; + let pulls = 0; + const req = admissionRequest("/v1/responses", new ReadableStream({ + pull() { pulls++; }, + cancel(value) { cancellations++; cancelledWith = value; }, + }, { highWaterMark: 0 }), { signal: abort.signal }); + const pending = withRaisedInboundBodyAdmission(req, "/v1/responses", RAISED_BODY_LIMIT, async () => { + dispatched = true; + return new Response(null, { status: 204 }); + }, undefined, { budget: testBudget, maxWaitMs: 1000 }); + const outcome = pending.catch(error => error); + try { + if (grant === "queued") { + assert.equal(testBudget.snapshot().queuedRequests, 1); + holder!.release(); + } + // Do not yield: acquire has granted a lease but admission's await has not resumed. + assert.deepEqual(testBudget.snapshot(), { reservedBytes: RAISED_BODY_LIMIT, activeLeases: 1, queuedRequests: 0 }); + abort.abort(reason); + assert.equal(await outcome, reason); + assert.equal(dispatched, false); + assert.equal(pulls, 0); + assert.equal(cancellations, 1); + assert.equal(cancelledWith, reason); + assert.deepEqual(testBudget.snapshot(), { reservedBytes: 0, activeLeases: 0, queuedRequests: 0 }); + await assertBodyCapacity(204); + } finally { + abort.abort(reason); + holder?.release(); + const result = await outcome; + if (result instanceof Response) await result.body?.cancel(); + } + }); + } + + for (const end of ["abort", "timeout"] as const) { + test(`a queued HTTP upload ${end} cancels its unread body without releasing the holder`, async () => { + const holder = await admittedBodyWork(async () => new Response("held")); + const abort = new AbortController(); + const reason = new Error("fixture waiting upload left"); + let cancelledWith: unknown; + let cancellations = 0; + let dispatched = false; + let pulls = 0; + const req = admissionRequest("/v1/responses", new ReadableStream({ + pull() { pulls++; }, + cancel(value) { cancellations++; cancelledWith = value; }, + }, { highWaterMark: 0 }), { signal: abort.signal }); + const outcome = withRaisedInboundBodyAdmission(req, "/v1/responses", RAISED_BODY_LIMIT, async () => { + dispatched = true; + return new Response(null, { status: 204 }); + }, undefined, { budget: testBudget, maxWaitMs: end === "timeout" ? 20 : 1000 }).catch(error => error); + try { + assert.deepEqual(testBudget.snapshot(), { reservedBytes: RAISED_BODY_LIMIT, activeLeases: 1, queuedRequests: 1 }); + if (end === "abort") abort.abort(reason); + const result = await outcome; + if (end === "abort") assert.equal(result, reason); + else { + assert.ok(result instanceof Response); + assert.equal(result.status, 503); + assert.equal(result.headers.get("retry-after"), "1"); + await result.text(); + } + assert.equal(dispatched, false); + assert.equal(pulls, 0); + assert.equal(cancellations, 1); + if (end === "abort") assert.equal(cancelledWith, reason); + else assert.ok(cancelledWith instanceof InboundBodyCapacityError); + assert.deepEqual(testBudget.snapshot(), { reservedBytes: RAISED_BODY_LIMIT, activeLeases: 1, queuedRequests: 0 }); + } finally { + abort.abort(reason); + await outcome; + await holder.text(); + } + await assertBodyCapacity(204); + }); + } + + for (const normalized of [false, true]) { + test(`retains measured JSON bytes with translator normalization ${normalized}`, async () => { + const text = "1e20"; + const req = admissionRequest("/v1/responses", text); + const translator = normalized ? createTranslatorBudget() : undefined; + let response: Response | undefined; + try { + response = await admittedBodyWork(async () => { + assert.equal(testBudget.snapshot().reservedBytes, RAISED_BODY_LIMIT); + assert.equal(await readJsonRequestBody(req, translator, RAISED_BODY_LIMIT), 1e20); + return new Response("held"); + }, req); + const expected = normalized ? Buffer.byteLength(JSON.stringify(1e20)) : Buffer.byteLength(text); + assert.deepEqual(testBudget.snapshot(), { reservedBytes: expected, activeLeases: 1, queuedRequests: 0 }); + await response.text(); + } finally { + await response?.body?.cancel().catch(() => undefined); + translator?.dispose(); + } + }); + } + + for (const end of ["eof", "error", "cancel"] as const) { + test(`retains decoded compressed bytes until response ${end} settles exactly once`, async () => { + const text = JSON.stringify({ input: "x".repeat(10_000) }); + const wire = gzipSync(Buffer.from(text)); + assert.ok(wire.byteLength < Buffer.byteLength(text)); + const abort = new AbortController(); + const req = admissionRequest("/v1/responses", wire, { + headers: { "content-encoding": "gzip" }, signal: abort.signal, + }); + const finishCancel = bodyDeferred(); + const cancelStarted = bodyDeferred(); + const error = new Error("fixture stream failure"); + let controller!: ReadableStreamDefaultController; + let cancellations = 0; + const response = await admittedBodyWork(async () => { + assert.equal(testBudget.snapshot().reservedBytes, RAISED_BODY_LIMIT); + assert.deepEqual(await readJsonRequestBody(req, undefined, RAISED_BODY_LIMIT), JSON.parse(text)); + return new Response(new ReadableStream({ + start(value) { controller = value; }, + cancel() { cancellations++; cancelStarted.resolve(); return finishCancel.promise; }, + }, { highWaterMark: 0 })); + }, req); + let next: Response | undefined; + try { + const retained = { reservedBytes: Buffer.byteLength(text), activeLeases: 1, queuedRequests: 0 }; + assert.deepEqual(testBudget.snapshot(), retained); + if (end === "eof") { + controller.close(); + assert.equal(await response.text(), ""); + } else if (end === "error") { + controller.error(error); + await assert.rejects(response.text(), (actual: unknown) => actual === error); + } else { + const cancellation = response.body!.cancel("fixture stop"); + await cancelStarted.promise; + assert.deepEqual(testBudget.snapshot(), retained); + finishCancel.resolve(); + await cancellation; + } + assert.deepEqual(testBudget.snapshot(), { reservedBytes: 0, activeLeases: 0, queuedRequests: 0 }); + next = await admittedBodyWork(async () => new Response("next reservation")); + abort.abort(error); + await Promise.resolve(); + await assertBodyCapacity(503); // A late finalizer cannot release the next owner. + assert.equal(cancellations, end === "cancel" ? 1 : 0); + } finally { + finishCancel.resolve(); + await response.body?.cancel().catch(() => undefined); + await next?.body?.cancel(); + } + await assertBodyCapacity(204); + }); + } +}); +// END raised-body admission regressions. + +test("the public singleton admits eight tiny parsed bodies with live responses", async () => { + const empty = { reservedBytes: 0, activeLeases: 0, queuedRequests: 0 }; + assert.deepEqual(inboundBodyBudget.snapshot(), empty); + const responses: Response[] = []; + let measuredBytes = 0; + let cancellations = 0; + try { + for (let n = 0; n < 8; n++) { + const text = JSON.stringify({ request: n }); + const req = admissionRequest("/v1/responses", text); + const response = await withRaisedInboundBodyAdmission(req, "/v1/responses", RAISED_BODY_LIMIT, async () => { + assert.deepEqual(await readJsonRequestBody(req, undefined, RAISED_BODY_LIMIT), { request: n }); + return new Response(new ReadableStream({ + cancel() { cancellations++; }, + }, { highWaterMark: 0 })); + }, undefined, { maxWaitMs: 0 }); // Use the public singleton; fail promptly if full allowances stay retained. + responses.push(response); + assert.equal(response.status, 200); + assert.equal(response.bodyUsed, false); + measuredBytes += Buffer.byteLength(text); + assert.deepEqual(inboundBodyBudget.snapshot(), { + reservedBytes: measuredBytes, activeLeases: n + 1, queuedRequests: 0, + }); + } + assert.equal(responses.length, 8); + } finally { + await Promise.all(responses.map(response => response.body?.cancel().catch(() => undefined))); + } + assert.equal(cancellations, 8); + assert.deepEqual(inboundBodyBudget.snapshot(), empty); +}); + +test("eight concurrent local HTTP JSON requests retain live fixture streams without an upstream", async () => { + const budget = createInboundBodyBudget(4 * RAISED_BODY_LIMIT); + const seen = new Set(); + let cancellations = 0; + const server = Bun.serve({ + hostname: "127.0.0.1", port: 0, + fetch(req) { + return withRaisedInboundBodyAdmission(req, new URL(req.url).pathname, RAISED_BODY_LIMIT, async () => { + const parsed = await readJsonRequestBody(req, undefined, RAISED_BODY_LIMIT) as { request: number }; + seen.add(parsed.request); + return new Response(new ReadableStream({ + start(controller) { controller.enqueue(new TextEncoder().encode(`data: ${parsed.request}\n\n`)); }, + cancel() { cancellations++; }, + }, { highWaterMark: 0 }), { headers: { "content-type": "text/event-stream" } }); + }, undefined, { budget, maxWaitMs: 1000 }); + }, + }); + const controllers = Array.from({ length: 8 }, () => new AbortController()); + const readers: ReadableStreamDefaultReader[] = []; + let clients: Promise[] = []; + try { + clients = controllers.map(async (controller, n) => { + const response = await fetch(`http://127.0.0.1:${server.port}/v1/responses`, { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ request: n }), signal: controller.signal, + }); + assert.equal(response.status, 200); + assert.equal(response.headers.get("content-type"), "text/event-stream"); + const reader = response.body!.getReader(); + readers.push(reader); + const decoder = new TextDecoder(); + let event = ""; + while (!event.includes("\n\n")) { + const chunk = await reader.read(); + assert.equal(chunk.done, false); + event += decoder.decode(chunk.value, { stream: true }); + } + assert.equal(event, `data: ${n}\n\n`); + }); + await Promise.all(clients); + assert.deepEqual([...seen].sort(), [0, 1, 2, 3, 4, 5, 6, 7]); + assert.deepEqual(budget.snapshot(), { + reservedBytes: Array.from({ length: 8 }, (_, request) => Buffer.byteLength(JSON.stringify({ request }))) + .reduce((total, bytes) => total + bytes, 0), + activeLeases: 8, queuedRequests: 0, + }); + assert.equal(cancellations, 0); + } finally { + for (const controller of controllers) controller.abort(); + await Promise.allSettled(clients); + await Promise.all(readers.map(reader => reader.cancel().catch(() => undefined))); + for (const reader of readers) reader.releaseLock(); + await server.stop(true); + for (let i = 0; i < 100 && budget.snapshot().activeLeases; i++) await Bun.sleep(10); + } + assert.equal(cancellations, 8); + assert.deepEqual(budget.snapshot(), { reservedBytes: 0, activeLeases: 0, queuedRequests: 0 }); +}); + +describe("raised inbound body admission at the real HTTP boundary", () => { + let fixtureHome: string; + let previousHome: string | undefined; + let codexHome: IsolatedCodexHome; + beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + fixtureHome = mkdtempSync(join(tmpdir(), "ocx-inbound-body-admission-")); + process.env.OPENCODEX_HOME = fixtureHome; + codexHome = installIsolatedCodexHome("ocx-inbound-body-admission-codex-"); + }); + afterEach(async () => { + await drainAndRemoveFixtureRoots({ + roots: [{ path: fixtureHome }, { path: codexHome.path }], + restoreEnvironment() { + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + codexHome.restoreEnvironment(); + }, + }); + }); + + test("all configured JSON routes refuse before protocol parsing and recover after release", async () => { + saveConfig({ ...getDefaultConfig(), providers: {}, maxInboundBodyBytes: RAISED_BODY_LIMIT }); + const server = startServer(0); + let restoreCapacity: (() => Promise) | undefined; + try { + restoreCapacity = await saturateInboundBodyBudget(); + for (const path of raisedBodyPaths) { + const response = await fetch(`http://127.0.0.1:${server.port}${path}`, { + method: "POST", headers: { "content-type": "application/json" }, body: "{}", + }); + assert.equal(response.status, 503, path); + assert.equal(response.headers.get("retry-after"), "1", path); + const payload = await response.json() as { type?: string; error: { type: string; code: string } }; + assert.equal(payload.error.code, "server_busy", path); + assert.equal(payload.error.type, path.startsWith("/v1/messages") ? "overloaded_error" : "server_error", path); + } + // Admission remains after the listener's origin policy, even while busy. + const blocked = await fetch(`http://127.0.0.1:${server.port}/v1/chat/completions`, { + method: "POST", headers: { "content-type": "application/json", origin: "https://example.invalid" }, body: "{}", + }); + assert.equal(blocked.status, 403); + await blocked.text(); + await restoreCapacity(); + const recovered = await fetch(`http://127.0.0.1:${server.port}/v1/chat/completions`, { + method: "POST", headers: { "content-type": "application/json" }, body: "{}", + }); + // Invalid input reaches the original parser again; no provider is configured or contacted. + assert.equal(recovered.status, 400); + await recovered.text(); + } finally { + await restoreCapacity?.(); + await server.stop(true); + } + }, 30_000); +}); diff --git a/tests/server/inbound-body-budget.test.ts b/tests/server/inbound-body-budget.test.ts new file mode 100644 index 00000000000..d399b05e117 --- /dev/null +++ b/tests/server/inbound-body-budget.test.ts @@ -0,0 +1,247 @@ +import { describe, expect, spyOn, test } from "bun:test"; +import assert from "node:assert/strict"; +import { + createInboundBodyBudget, + INBOUND_BODY_QUEUE_TIMEOUT_MS, + MAX_CONCURRENT_INBOUND_BODY_BYTES, + MAX_QUEUED_INBOUND_BODIES, + type InboundBodyBudget, + type InboundBodyLease, +} from "../../src/server/inbound-body-budget"; + +const freshSignal = () => new AbortController().signal; +function assertEmpty(budget: InboundBodyBudget) { + assert.deepEqual(budget.snapshot(), { reservedBytes: 0, activeLeases: 0, queuedRequests: 0 }); +} + +describe("inbound body budget", () => { + test("bounds the shared allowance, waiting uploads and wait duration", () => { + expect(MAX_CONCURRENT_INBOUND_BODY_BYTES).toBe(2 * 1024 * 1024 * 1024); + expect(MAX_QUEUED_INBOUND_BODIES).toBe(32); + expect(INBOUND_BODY_QUEUE_TIMEOUT_MS).toBe(120_000); + }); + + for (const capacity of [0, -1, 0.5, NaN, Infinity, -Infinity, Number.MAX_SAFE_INTEGER + 1]) { + test(`rejects invalid capacity ${String(capacity)}`, () => { + expect(() => createInboundBodyBudget(capacity)).toThrow(RangeError); + }); + } + for (const maxQueued of [-1, 0.5, NaN, Infinity, -Infinity, Number.MAX_SAFE_INTEGER + 1]) { + test(`rejects invalid queue size ${String(maxQueued)}`, () => { + expect(() => createInboundBodyBudget(100, maxQueued)).toThrow(RangeError); + }); + } + + test("rejects invalid reservations without changing counters", async () => { + const budget = createInboundBodyBudget(100); + for (const bytes of [0, -1, 0.5, NaN, Infinity, 101, Number.MAX_SAFE_INTEGER + 1]) { + await expect(budget.acquire(bytes, freshSignal())).rejects.toBeInstanceOf(RangeError); + assertEmpty(budget); + } + }); + + test("shrinking drains waiters and releasing a shrunken lease is idempotent", async () => { + const budget = createInboundBodyBudget(100); + const first = await budget.acquire(80, freshSignal()); + assert.ok(first); + let second: InboundBodyLease | null = null; + const abort = new AbortController(); + const pending = budget.acquire(80, abort.signal, 1000); + try { + assert.deepEqual(budget.snapshot(), { reservedBytes: 80, activeLeases: 1, queuedRequests: 1 }); + first.shrink(20); + second = await pending; + assert.ok(second); + assert.deepEqual(budget.snapshot(), { reservedBytes: 100, activeLeases: 2, queuedRequests: 0 }); + first.release(); + first.release(); + first.shrink(1); + assert.deepEqual(budget.snapshot(), { reservedBytes: 80, activeLeases: 1, queuedRequests: 0 }); + second.release(); + assertEmpty(budget); + } finally { + abort.abort(); + first.release(); + (second ?? await pending.catch(() => null))?.release(); + } + }); + + test("invalid measurements and attempted growth cannot alter a lease", async () => { + const budget = createInboundBodyBudget(100); + const lease = await budget.acquire(80, freshSignal()); + assert.ok(lease); + try { + for (const measured of [NaN, -1, Infinity, -Infinity, 81, 1000]) { + lease.shrink(measured); + assert.deepEqual(budget.snapshot(), { reservedBytes: 80, activeLeases: 1, queuedRequests: 0 }); + } + lease.shrink(20); + lease.shrink(79); + assert.equal(budget.snapshot().reservedBytes, 20); + lease.release(); + lease.release(); + lease.shrink(1); + assertEmpty(budget); + } finally { lease.release(); } + }); + + test("an eligible smaller arrival cannot bypass the FIFO head", async () => { + const budget = createInboundBodyBudget(100); + const holder = await budget.acquire(90, freshSignal()); + assert.ok(holder); + const abort = new AbortController(); + const order: string[] = []; + const large = budget.acquire(80, abort.signal, 1000).then(lease => { order.push("large"); return lease; }); + const small = budget.acquire(10, abort.signal, 1000).then(lease => { order.push("small"); return lease; }); + try { + await Promise.resolve(); + assert.deepEqual(order, []); + holder.shrink(60); + assert.deepEqual(budget.snapshot(), { reservedBytes: 60, activeLeases: 1, queuedRequests: 2 }); + await Promise.resolve(); + assert.deepEqual(order, []); + holder.release(); + const leases = await Promise.all([large, small]); + assert.ok(leases.every(Boolean)); + assert.deepEqual(order, ["large", "small"]); + assert.deepEqual(budget.snapshot(), { reservedBytes: 90, activeLeases: 2, queuedRequests: 0 }); + } finally { + abort.abort(); + holder.release(); + for (const pending of [large, small]) (await pending.catch(() => null))?.release(); + } + assertEmpty(budget); + }); + + for (const removal of ["abort", "timeout"] as const) { + test(`removing a FIFO head by ${removal} drains an eligible follower`, async () => { + const budget = createInboundBodyBudget(100); + const holder = await budget.acquire(70, freshSignal()); + assert.ok(holder); + const headAbort = new AbortController(); + const followerAbort = new AbortController(); + const reason = new Error("fixture queued upload aborted"); + const head = budget.acquire(80, headAbort.signal, removal === "timeout" ? 20 : 1000).catch(error => error); + const follower = budget.acquire(30, followerAbort.signal, 1000); + try { + assert.equal(budget.snapshot().queuedRequests, 2); + if (removal === "abort") headAbort.abort(reason); + assert.equal(await head, removal === "abort" ? reason : null); + const lease = await follower; + assert.ok(lease); + assert.deepEqual(budget.snapshot(), { reservedBytes: 100, activeLeases: 2, queuedRequests: 0 }); + } finally { + headAbort.abort(); + followerAbort.abort(); + holder.release(); + (await follower.catch(() => null))?.release(); + } + assertEmpty(budget); + }); + } + + test("allows exactly 32 queued uploads and removes every aborted waiter", async () => { + const budget = createInboundBodyBudget(100); + const holder = await budget.acquire(100, freshSignal()); + assert.ok(holder); + const controllers = Array.from({ length: MAX_QUEUED_INBOUND_BODIES }, () => new AbortController()); + const waiting = controllers.map(controller => budget.acquire(100, controller.signal).catch(error => error)); + const reason = new Error("fixture queue cleared"); + try { + assert.deepEqual(budget.snapshot(), { reservedBytes: 100, activeLeases: 1, queuedRequests: 32 }); + assert.equal(await budget.acquire(1, freshSignal()), null); + assert.equal(budget.snapshot().queuedRequests, 32); + for (const controller of controllers) controller.abort(reason); + assert.deepEqual(await Promise.all(waiting), Array(32).fill(reason)); + assert.deepEqual(budget.snapshot(), { reservedBytes: 100, activeLeases: 1, queuedRequests: 0 }); + } finally { + for (const controller of controllers) controller.abort(reason); + await Promise.all(waiting); + holder.release(); + } + assertEmpty(budget); + (await budget.acquire(100, freshSignal(), 0))!.release(); + assertEmpty(budget); + }); + + test("a queue size of zero refuses busy arrivals without retaining them", async () => { + const budget = createInboundBodyBudget(100, 0); + const holder = await budget.acquire(100, freshSignal()); + assert.ok(holder); + try { + assert.equal(await budget.acquire(1, freshSignal()), null); + assert.deepEqual(budget.snapshot(), { reservedBytes: 100, activeLeases: 1, queuedRequests: 0 }); + } finally { holder.release(); } + assertEmpty(budget); + }); + + test("zero wait time never leaves a queued request", async () => { + const budget = createInboundBodyBudget(100); + const holder = await budget.acquire(100, freshSignal()); + assert.ok(holder); + try { + assert.equal(await budget.acquire(1, freshSignal(), 0), null); + assert.deepEqual(budget.snapshot(), { reservedBytes: 100, activeLeases: 1, queuedRequests: 0 }); + } finally { holder.release(); } + assertEmpty(budget); + }); + + test("a timed out waiter leaves the holder charged and permits a fresh acquisition", async () => { + const budget = createInboundBodyBudget(100); + const holder = await budget.acquire(100, freshSignal()); + assert.ok(holder); + try { + assert.equal(await budget.acquire(100, freshSignal(), 20), null); + assert.deepEqual(budget.snapshot(), { reservedBytes: 100, activeLeases: 1, queuedRequests: 0 }); + } finally { holder.release(); } + (await budget.acquire(100, freshSignal(), 0))!.release(); + assertEmpty(budget); + }); + + test("pre-aborted acquisitions preserve their reason without owning capacity", async () => { + const budget = createInboundBodyBudget(100); + const abort = new AbortController(); + const reason = new Error("fixture already aborted"); + abort.abort(reason); + await expect(budget.acquire(100, abort.signal)).rejects.toBe(reason); + assertEmpty(budget); + }); + + test("a granted waiter loses its timer and abort cannot release its lease", async () => { + const budget = createInboundBodyBudget(100); + const holder = await budget.acquire(100, freshSignal()); + assert.ok(holder); + const abort = new AbortController(); + const pending = budget.acquire(100, abort.signal, 20); + holder.release(); + const admitted = await pending; + assert.ok(admitted); + try { + abort.abort(); + await Bun.sleep(30); + assert.deepEqual(budget.snapshot(), { reservedBytes: 100, activeLeases: 1, queuedRequests: 0 }); + } finally { admitted.release(); } + assertEmpty(budget); + }); + + for (const waitMs of [undefined, 240_000]) { + test(`wait ${String(waitMs)} schedules no more than the 120 second deadline`, async () => { + const budget = createInboundBodyBudget(100); + const holder = await budget.acquire(100, freshSignal()); + assert.ok(holder); + const abort = new AbortController(); + const timer = spyOn(globalThis, "setTimeout"); + let pending: Promise | undefined; + try { + pending = budget.acquire(100, abort.signal, waitMs).catch(error => error); + expect(timer).toHaveBeenLastCalledWith(expect.any(Function), INBOUND_BODY_QUEUE_TIMEOUT_MS); + } finally { + timer.mockRestore(); + abort.abort(); + await pending; + holder.release(); + } + assertEmpty(budget); + }); + } +}); diff --git a/tests/server/server-request-body-size.test.ts b/tests/server/server-request-body-size.test.ts index cd1dfdab0db..690411ca205 100644 --- a/tests/server/server-request-body-size.test.ts +++ b/tests/server/server-request-body-size.test.ts @@ -1,17 +1,9 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; import { existsSync, mkdirSync} from "node:fs"; -import assert from "node:assert/strict"; -import { gzipSync } from "node:zlib"; import { join } from "node:path"; import { getDefaultConfig, saveConfig } from "../../src/config"; import { startServer } from "../../src/server"; -import { MAX_DECOMPRESSED_BODY_BYTES, MAX_CONFIGURABLE_INBOUND_BODY_BYTES, readJsonRequestBody, DecompressedBodyTooLargeError, UnsupportedContentEncodingError } from "../../src/server/request-decompress"; -import { - withRaisedInboundBodyAdmission, - InboundBodyCapacityError, - CONFIGURABLE_JSON_BODY_ROUTES, - UNGATED_LOOPBACK_ROUTES, -} from "../../src/server/inbound-body-admission"; +import { MAX_DECOMPRESSED_BODY_BYTES } from "../../src/server/request-decompress"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; @@ -113,477 +105,3 @@ describe("configurable listener body size (Issue #3573)", () => { } }); }); - -// BEGIN raised-body admission regressions: these cases use Web streams and tiny payloads. -const RAISED_BODY_LIMIT = MAX_CONFIGURABLE_INBOUND_BODY_BYTES; -// Read the gate's own set rather than a second copy: a route added to admission -// without protocol coverage, or covered here but never admitted, must not pass. -const raisedBodyPaths = [...CONFIGURABLE_JSON_BODY_ROUTES]; -function bodyDeferred() { - let resolve!: () => void; - const promise = new Promise(done => { resolve = done; }); - return { promise, resolve }; -} -function admissionRequest(path = "/v1/responses", body: BodyInit = "{}", init: RequestInit = {}): Request { - return new Request(`http://localhost${path}`, { - method: "POST", body, ...init, duplex: "half", - } as RequestInit); -} -function admittedBodyWork(work: () => Promise, req = admissionRequest(), limit = RAISED_BODY_LIMIT) { - return withRaisedInboundBodyAdmission(req, new URL(req.url).pathname, limit, work); -} -async function assertBodyCapacity(status: number) { - const response = await admittedBodyWork(async () => new Response(null, { status: 204 })); - assert.equal(response.status, status); - await response.text(); -} - -describe("raised inbound body admission lifetime", () => { - test("keeps the parsed graph admitted while downstream work is pending", async () => { - const parsed = bodyDeferred(); - const complete = bodyDeferred(); - const req = admissionRequest("/v1/responses", '{"input":"retained fixture"}'); - const first = admittedBodyWork(async () => { - const raw = await readJsonRequestBody(req, undefined, RAISED_BODY_LIMIT) as { input: string }; - parsed.resolve(); - await complete.promise; - return Response.json({ input: raw.input }); - }, req); - try { - await parsed.promise; - await assertBodyCapacity(503); - complete.resolve(); - const response = await first; - await assertBodyCapacity(503); // Returning headers is not the end of the body lifetime. - assert.deepEqual(await response.json(), { input: "retained fixture" }); - await assertBodyCapacity(204); - } finally { - complete.resolve(); - await (await first).body?.cancel().catch(() => undefined); - } - }); - - test("reserves before the first upload finishes", async () => { - const started = bodyDeferred(); - let controller!: ReadableStreamDefaultController; - const req = admissionRequest("/v1/responses", new ReadableStream({ - start(value) { controller = value; }, - pull() { started.resolve(); }, - }, { highWaterMark: 0 })); - const first = admittedBodyWork(async () => { - await readJsonRequestBody(req, undefined, RAISED_BODY_LIMIT); - return new Response(null, { status: 204 }); - }, req); - try { - await started.promise; - await assertBodyCapacity(503); - } finally { - controller.enqueue(new TextEncoder().encode("{}")); - controller.close(); - await first; - } - await assertBodyCapacity(204); - }); - - for (const path of raisedBodyPaths) { - test(`returns retryable overload before the ${path} handler can wrap it`, async () => { - const holder = await admittedBodyWork(async () => new Response("held")); - let dispatched = false; - try { - const req = admissionRequest(path, "private-fixture-body"); - const response = await admittedBodyWork(async () => { - dispatched = true; - return Response.json({ error: "would wrap parser failures as 400" }, { status: 400 }); - }, req); - assert.equal(response.status, 503); - assert.equal(response.headers.get("retry-after"), "1"); - const payload = await response.json() as { type?: string; error: { type: string; code: string } }; - assert.equal(payload.error.code, "server_busy"); - const anthropic = path.startsWith("/v1/messages"); - assert.equal(payload.error.type, anthropic ? "overloaded_error" : "server_error"); - assert.equal(payload.type, anthropic ? "error" : undefined); - assert.equal(JSON.stringify(payload).includes("private-fixture-body"), false); - assert.equal(dispatched, false); - } finally { await holder.text(); } - await assertBodyCapacity(204); - }); - } - - for (const limit of [undefined, 0, MAX_DECOMPRESSED_BODY_BYTES, 1024 * 1024, NaN, Infinity]) { - test(`preserves ordinary concurrency for limit ${String(limit)}`, async () => { - const holder = await admittedBodyWork(async () => new Response("held")); - const original = new Response("unchanged"); - try { - const response = await withRaisedInboundBodyAdmission(admissionRequest(), "/v1/responses", limit, async () => original); - assert.equal(response, original); - await response.text(); - } finally { await holder.text(); } - }); - } - - test("does not charge management, audio, context, or GET routes", async () => { - const holder = await admittedBodyWork(async () => new Response("held")); - try { - for (const path of ["/api/config", "/v1/audio/transcriptions", "/v1/live", "/v1/context/history", "/v1/unknown"]) { - const original = new Response(null, { status: 204 }); - assert.equal(await admittedBodyWork(async () => original, admissionRequest(path)), original); - } - const get = new Request("http://localhost/v1/responses"); - assert.equal((await admittedBodyWork(async () => new Response(null, { status: 204 }), get)).status, 204); - } finally { await holder.text(); } - }); - - // The dispatcher is the source of truth for what the loopback listener accepts. - // A new `/v1` route must be admitted or explicitly exempted; leaving it out of - // both sets is how a raised `maxInboundBodyBytes` route would skip the - // process-wide reservation while every admission test still passed. - test("every loopback /v1 route is classified for inbound body admission", async () => { - const source = await Bun.file(join(import.meta.dir, "../../src/server/index.ts")).text(); - const start = source.indexOf("function loopbackRouteAllowed"); - assert.notEqual(start, -1); - const end = source.indexOf("\n }", start); - assert.notEqual(end, -1); - const routes = [...source.slice(start, end).matchAll(/path === "(\/v1\/[^"]+)"/g)].map(match => match[1]); - assert.ok(routes.length >= 8, `expected the dispatcher to list /v1 routes, saw ${routes.length}`); - for (const route of routes) { - assert.ok( - CONFIGURABLE_JSON_BODY_ROUTES.has(route) || UNGATED_LOOPBACK_ROUTES.has(route), - `${route} is neither admitted nor explicitly exempt in inbound-body-admission.ts`, - ); - } - for (const route of CONFIGURABLE_JSON_BODY_ROUTES) { - assert.ok(routes.includes(route), `${route} is admitted but the dispatcher no longer serves it`); - } - }); - - test("keeps an aborted request charged until its pending work actually returns", async () => { - const abort = new AbortController(); - const started = bodyDeferred(); - const complete = bodyDeferred(); - const req = admissionRequest("/v1/responses", "{}", { signal: abort.signal }); - const first = admittedBodyWork(async () => { - await readJsonRequestBody(req, undefined, RAISED_BODY_LIMIT); - started.resolve(); - await complete.promise; - return new Response(null, { status: 499 }); - }, req); - try { - await started.promise; - abort.abort(new Error("fixture cancellation")); - await assertBodyCapacity(503); - } finally { complete.resolve(); await first; } - await assertBodyCapacity(204); - }); - - for (const rejects of [false, true]) { - test(`request abort cancels an unconsumed response and waits for settlement (rejects=${rejects})`, async () => { - const abort = new AbortController(); - const finishCancel = bodyDeferred(); - const reason = new Error("fixture request abort"); - let cancelledWith: unknown; - let cancellations = 0; - let pulls = 0; - const response = await admittedBodyWork(async () => new Response(new ReadableStream({ - pull() { pulls++; }, - async cancel(value) { - cancellations++; - cancelledWith = value; - await finishCancel.promise; - if (rejects) throw new Error("fixture producer cancellation failure"); - }, - }, { highWaterMark: 0 })), admissionRequest("/v1/responses", "{}", { signal: abort.signal })); - try { - await assertBodyCapacity(503); - abort.abort(reason); - await assertBodyCapacity(503); - assert.equal(cancellations, 1); - assert.equal(cancelledWith, reason); - assert.equal(pulls, 0); - assert.equal(response.bodyUsed, false); - finishCancel.resolve(); - await Bun.sleep(0); // Let cancellation and the admission finalizer settle. - await assertBodyCapacity(204); - } finally { - finishCancel.resolve(); - await response.body?.cancel().catch(() => undefined); - } - }); - } - - test("request abort during work cancels its eventual body before releasing capacity", async () => { - const abort = new AbortController(); - const started = bodyDeferred(); - const complete = bodyDeferred(); - const finishCancel = bodyDeferred(); - let cancellations = 0; - const first = admittedBodyWork(async () => { - started.resolve(); - await complete.promise; - return new Response(new ReadableStream({ - cancel() { cancellations++; return finishCancel.promise; }, - }, { highWaterMark: 0 })); - }, admissionRequest("/v1/responses", "{}", { signal: abort.signal })); - try { - await started.promise; - abort.abort(); - await assertBodyCapacity(503); - assert.equal(cancellations, 0); - complete.resolve(); - await first; - await assertBodyCapacity(503); - assert.equal(cancellations, 1); - finishCancel.resolve(); - await Bun.sleep(0); - await assertBodyCapacity(204); - } finally { - complete.resolve(); - finishCancel.resolve(); - await (await first).body?.cancel().catch(() => undefined); - } - }); - - for (const abortFirst of [false, true]) { - test(`request abort and explicit cancel release exactly once (abortFirst=${abortFirst})`, async () => { - const abort = new AbortController(); - const finishCancel = bodyDeferred(); - const reason = new Error("fixture request abort race"); - let cancellations = 0; - let cancelledWith: unknown; - const response = await admittedBodyWork(async () => new Response(new ReadableStream({ - cancel(value) { - cancellations++; - cancelledWith = value; - return finishCancel.promise; - }, - }, { highWaterMark: 0 })), admissionRequest("/v1/responses", "{}", { signal: abort.signal })); - let next: Response | undefined; - try { - if (abortFirst) abort.abort(reason); - const cancelled = response.body!.cancel("fixture explicit cancel").catch(error => error); - if (!abortFirst) abort.abort(reason); - await assertBodyCapacity(503); - assert.equal(cancellations, 1); - assert.equal(cancelledWith, abortFirst ? reason : "fixture explicit cancel"); - finishCancel.resolve(); - await cancelled; - await Bun.sleep(0); - await assertBodyCapacity(204); - next = await admittedBodyWork(async () => new Response("next reservation")); - assert.equal(next.status, 200); - await assertBodyCapacity(503); // A double release would admit this peer. - assert.equal(cancellations, 1); - } finally { - finishCancel.resolve(); - await response.body?.cancel().catch(() => undefined); - await next?.body?.cancel(); - } - await assertBodyCapacity(204); - }); - } - - test("preserves pre-aborted requests' existing error path", async () => { - const holder = await admittedBodyWork(async () => new Response("held")); - const abort = new AbortController(); - abort.abort(); - try { - const response = await admittedBodyWork(async () => new Response(null, { status: 499 }), - admissionRequest("/v1/responses", "{}", { signal: abort.signal })); - assert.equal(response.status, 499); - } finally { await holder.text(); } - }); - - test("does not double reserve internal body clones or translated reads", async () => { - const req = admissionRequest(); - const clone = req.clone(); - const response = await admittedBodyWork(async () => { - assert.deepEqual(await readJsonRequestBody(req, undefined, RAISED_BODY_LIMIT), {}); - assert.deepEqual(await readJsonRequestBody(clone, undefined, RAISED_BODY_LIMIT), {}); - return new Response(null, { status: 204 }); - }, req); - assert.equal(response.status, 204); - await assertBodyCapacity(204); - }); - - test("retains the original oversized-declaration refusal before capacity", async () => { - const holder = await admittedBodyWork(async () => new Response("held")); - const req = admissionRequest("/v1/responses", "{}", { headers: { "content-length": String(RAISED_BODY_LIMIT + 1) } }); - try { - await assert.rejects(admittedBodyWork(async () => { - await readJsonRequestBody(req, undefined, RAISED_BODY_LIMIT); - return new Response(null, { status: 204 }); - }, req), DecompressedBodyTooLargeError); - await assertBodyCapacity(503); // Rejecting an oversized peer cannot release the first lease. - } finally { await holder.text(); } - }); - - for (const declared of ["", "invalid", "-1", "Infinity", "0", "1"]) { - test(`an untrusted content-length ${JSON.stringify(declared)} cannot evade capacity`, async () => { - const holder = await admittedBodyWork(async () => new Response("held")); - try { - const req = admissionRequest("/v1/responses", "{}", { headers: { "content-length": declared } }); - const response = await admittedBodyWork(async () => new Response(null, { status: 204 }), req); - assert.equal(response.status, 503); - await response.text(); - } finally { await holder.text(); } - }); - } - - for (const failure of ["json", "gzip", "encoding", "downstream"]) { - test(`releases on ${failure} failure without changing the error`, async () => { - const req = admissionRequest("/v1/responses", failure === "json" ? "{" : "{}", { - headers: failure === "gzip" ? { "content-encoding": "gzip" } - : failure === "encoding" ? { "content-encoding": "not-a-codec" } : {}, - }); - const downstream = new Error("fixture downstream failure"); - await assert.rejects(admittedBodyWork(async () => { - await readJsonRequestBody(req, undefined, RAISED_BODY_LIMIT); - throw downstream; - }, req), (error: unknown) => failure === "json" ? error instanceof SyntaxError - : failure === "encoding" ? error instanceof UnsupportedContentEncodingError - : failure === "downstream" ? error === downstream : error instanceof Error); - await assertBodyCapacity(204); - }); - } - - test("counts compressed uploads by their allowance, not their small wire size", async () => { - const req = admissionRequest("/v1/responses", gzipSync(Buffer.from("{}")), { headers: { "content-encoding": "gzip" } }); - const holder = await admittedBodyWork(async () => { - assert.deepEqual(await readJsonRequestBody(req, undefined, RAISED_BODY_LIMIT), {}); - return new Response("held"); - }, req); - try { await assertBodyCapacity(503); } finally { await holder.text(); } - await assertBodyCapacity(204); - }); - - test("does not wait for cancellation of a refused tee", async () => { - const holder = await admittedBodyWork(async () => new Response("held")); - const cancelled = bodyDeferred(); - const unblockCancel = bodyDeferred(); - let reason: unknown; - const req = admissionRequest("/v1/responses", new ReadableStream({ - cancel(value) { reason = value; cancelled.resolve(); return unblockCancel.promise; }, - }, { highWaterMark: 0 })); - try { - const response = await admittedBodyWork(async () => new Response(null, { status: 204 }), req); - await cancelled.promise; - assert.equal(response.status, 503); - assert.ok(reason instanceof InboundBodyCapacityError); - await response.text(); - } finally { unblockCancel.resolve(); await holder.text(); } - }); - - test("retains a stream through EOF and preserves headers, bytes and backpressure", async () => { - let pulls = 0; - const chunk = new Uint8Array([1, 2, 3]); - const response = await admittedBodyWork(async () => new Response(new ReadableStream({ - pull(controller) { pulls++; if (pulls === 1) controller.enqueue(chunk); else controller.close(); }, - }, { highWaterMark: 0 }), { status: 201, statusText: "Fixture", headers: { "x-fixture": "preserved" } })); - assert.equal(pulls, 0); - assert.equal(response.status, 201); - assert.equal(response.statusText, "Fixture"); - assert.equal(response.headers.get("x-fixture"), "preserved"); - const reader = response.body!.getReader(); - try { - assert.equal((await reader.read()).value, chunk); // no body-sized copy - assert.equal(pulls, 1); - await assertBodyCapacity(503); - assert.equal((await reader.read()).done, true); - await assertBodyCapacity(204); - } finally { await reader.cancel(); reader.releaseLock(); } - }); - - test("releases a failed response stream and preserves its original error", async () => { - const error = new Error("fixture stream error"); - const response = await admittedBodyWork(async () => new Response(new ReadableStream({ - pull(controller) { controller.error(error); }, - }, { highWaterMark: 0 }))); - await assert.rejects(response.text(), (actual: unknown) => actual === error); - await assertBodyCapacity(204); - }); - - for (const rejects of [false, true]) { - test(`pending-read EOF never releases before cancellation settles (rejects=${rejects})`, async () => { - const reading = bodyDeferred(); - const finishCancel = bodyDeferred(); - const cancellation = new Error("fixture cancel error"); - let cancelledWith: unknown; - const response = await admittedBodyWork(async () => new Response(new ReadableStream({ - pull() { reading.resolve(); }, - async cancel(reason) { - cancelledWith = reason; - await finishCancel.promise; - if (rejects) throw cancellation; - }, - }, { highWaterMark: 0 }))); - const reader = response.body!.getReader(); - const pendingRead = reader.read(); - await reading.promise; - const cancelResult = reader.cancel("fixture stop").then(() => undefined, error => error); - try { - assert.equal((await pendingRead).done, true); - await Promise.resolve(); - await assertBodyCapacity(503); - assert.equal(cancelledWith, "fixture stop"); - } finally { finishCancel.resolve(); } - assert.equal(await cancelResult, rejects ? cancellation : undefined); - reader.releaseLock(); - await assertBodyCapacity(204); - }); - } - - test("runs the caller's refusal decoration once without dispatching work", async () => { - const holder = await admittedBodyWork(async () => new Response("held")); - let decorated = 0; - try { - const response = await withRaisedInboundBodyAdmission(admissionRequest(), "/v1/responses", RAISED_BODY_LIMIT, - async () => { throw new Error("must not dispatch"); }, refused => { - decorated++; - refused.headers.set("x-fixture-log", "once"); - return refused; - }); - assert.equal(decorated, 1); - assert.equal(response.headers.get("x-fixture-log"), "once"); - await response.text(); - } finally { await holder.text(); } - }); -}); -// END raised-body admission regressions. - - -describe("raised inbound body admission at the real HTTP boundary", () => { - test("all configured JSON routes refuse before protocol parsing and recover after release", async () => { - saveConfig({ ...getDefaultConfig(), providers: {}, maxInboundBodyBytes: RAISED_BODY_LIMIT }); - const server = startServer(0); - let holder: Response | undefined; - try { - holder = await admittedBodyWork(async () => new Response("held")); - for (const path of raisedBodyPaths) { - const response = await fetch(`http://127.0.0.1:${server.port}${path}`, { - method: "POST", headers: { "content-type": "application/json" }, body: "{}", - }); - assert.equal(response.status, 503, path); - assert.equal(response.headers.get("retry-after"), "1", path); - const payload = await response.json() as { type?: string; error: { type: string; code: string } }; - assert.equal(payload.error.code, "server_busy", path); - assert.equal(payload.error.type, path.startsWith("/v1/messages") ? "overloaded_error" : "server_error", path); - } - // Admission remains after the listener's origin policy, even while busy. - const blocked = await fetch(`http://127.0.0.1:${server.port}/v1/chat/completions`, { - method: "POST", headers: { "content-type": "application/json", origin: "https://example.invalid" }, body: "{}", - }); - assert.equal(blocked.status, 403); - await blocked.text(); - await holder.text(); - holder = undefined; - const recovered = await fetch(`http://127.0.0.1:${server.port}/v1/chat/completions`, { - method: "POST", headers: { "content-type": "application/json" }, body: "{}", - }); - // Invalid input reaches the original parser again; no provider is configured or contacted. - assert.equal(recovered.status, 400); - await recovered.text(); - } finally { - await holder?.body?.cancel().catch(() => undefined); - await server.stop(true); - } - }, 30_000); -}); From 8e57eb8d3fcd7bb8da597a9ad3f6eb244f423a5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=EC=9A=B0=EC=B2=A0?= Date: Tue, 20 Oct 2026 16:18:55 +0900 Subject: [PATCH 2/5] docs(ru): clarify per-upload inbound body reservation --- .../src/content/docs/ru/reference/inbound-body-admission.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-site/src/content/docs/ru/reference/inbound-body-admission.md b/docs-site/src/content/docs/ru/reference/inbound-body-admission.md index e5397ed8e12..96d63e190b3 100644 --- a/docs-site/src/content/docs/ru/reference/inbound-body-admission.md +++ b/docs-site/src/content/docs/ru/reference/inbound-body-admission.md @@ -28,8 +28,8 @@ description: Как увеличение лимита тела запроса в только уменьшаться и не превышает исходный лимит запроса; даже большее измерение нормализованного JSON никогда не увеличивает резерв. -При лимите 512 МиБ на запрос бюджет допускает до **4 загрузок, резервирующих по полных -512 МиБ**, ещё до измерения и уменьшения резервов. После разбора это не фиксированный +При лимите 512 МиБ на запрос бюджет допускает до **4 загрузок, каждая из которых резервирует полные 512 МиБ**, +ещё до измерения и уменьшения резервов. После разбора это не фиксированный предел в 4 запроса: например, **8 очень маленьких запросов могут одновременно передавать потоковые ответы**, когда уменьшение резервов позволяет допустить следующие запросы, в пределах общего бюджета и других ограничений. From f47b59cea60283d0f71a6109fe9434cf50441ecb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=EC=9A=B0=EC=B2=A0?= Date: Wed, 21 Oct 2026 09:05:39 +0900 Subject: [PATCH 3/5] test(server): document inbound admission fixtures --- tests/server/inbound-body-admission.test.ts | 5 +++++ tests/server/inbound-body-budget.test.ts | 1 + 2 files changed, 6 insertions(+) diff --git a/tests/server/inbound-body-admission.test.ts b/tests/server/inbound-body-admission.test.ts index eacb75d1d79..1f5c32a8b2f 100644 --- a/tests/server/inbound-body-admission.test.ts +++ b/tests/server/inbound-body-admission.test.ts @@ -27,6 +27,7 @@ import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/iso import { repoPath } from "../helpers/repo-root"; let testBudget: ReturnType; +/** Use the per-test budget with immediate refusal instead of the shared FIFO queue. */ function fixtureAdmission(...args: Parameters) { return withRaisedInboundBodyAdmission(args[0], args[1], args[2], args[3], args[4], { budget: testBudget, maxWaitMs: 0, @@ -38,19 +39,23 @@ const RAISED_BODY_LIMIT = MAX_CONFIGURABLE_INBOUND_BODY_BYTES; // Read the gate's own set rather than a second copy: a route added to admission // without protocol coverage, or covered here but never admitted, must not pass. const raisedBodyPaths = [...CONFIGURABLE_JSON_BODY_ROUTES]; +/** Synchronize upload and producer milestones without timer-based ordering. */ function bodyDeferred() { let resolve!: () => void; const promise = new Promise(done => { resolve = done; }); return { promise, resolve }; } +/** Build a POST fixture that supports streaming uploads and caller overrides. */ function admissionRequest(path = "/v1/responses", body: BodyInit = "{}", init: RequestInit = {}): Request { return new Request(`http://localhost${path}`, { method: "POST", body, ...init, duplex: "half", } as RequestInit); } +/** Admit a fixture handler using the request's own pathname and supplied limit. */ function admittedBodyWork(work: () => Promise, req = admissionRequest(), limit = RAISED_BODY_LIMIT) { return fixtureAdmission(req, new URL(req.url).pathname, limit, work); } +/** Probe admission and consume the response so the probe cannot retain a lease. */ async function assertBodyCapacity(status: number) { const response = await admittedBodyWork(async () => new Response(null, { status: 204 })); assert.equal(response.status, status); diff --git a/tests/server/inbound-body-budget.test.ts b/tests/server/inbound-body-budget.test.ts index d399b05e117..d5164716ae4 100644 --- a/tests/server/inbound-body-budget.test.ts +++ b/tests/server/inbound-body-budget.test.ts @@ -10,6 +10,7 @@ import { } from "../../src/server/inbound-body-budget"; const freshSignal = () => new AbortController().signal; +/** Assert that every reservation, active owner, and queued waiter was released. */ function assertEmpty(budget: InboundBodyBudget) { assert.deepEqual(budget.snapshot(), { reservedBytes: 0, activeLeases: 0, queuedRequests: 0 }); } From 1b2a9df76317ee1698da6c7b49b37f7f5f7cba6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=EC=9A=B0=EC=B2=A0?= Date: Thu, 22 Oct 2026 16:37:55 +0900 Subject: [PATCH 4/5] fix(server): make shared inbound body budget configurable --- .../fr/reference/configuration/providers.md | 2 + .../fr/reference/inbound-body-admission.md | 81 ++++++++---- .../ja/reference/configuration/providers.md | 2 + .../ja/reference/inbound-body-admission.md | 31 ++++- .../ko/reference/configuration/providers.md | 2 + .../ko/reference/inbound-body-admission.md | 31 ++++- .../docs/reference/configuration/providers.md | 3 +- .../docs/reference/inbound-body-admission.md | 73 ++++++++--- .../ru/reference/configuration/providers.md | 2 + .../ru/reference/inbound-body-admission.md | 77 +++++++---- .../tr/reference/configuration/providers.md | 2 + .../tr/reference/inbound-body-admission.md | 83 ++++++++---- .../reference/configuration/providers.md | 2 + .../zh-cn/reference/inbound-body-admission.md | 31 ++++- .../reference/configuration/providers.md | 2 + .../zh-tw/reference/inbound-body-admission.md | 31 ++++- .../test-layout/layout-server-inbound.json | 1 + src/config/schema/config-schema.ts | 7 + src/lib/inbound-body-policy.ts | 11 ++ src/server/inbound-body-admission.ts | 2 +- src/server/inbound-body-budget.ts | 19 ++- src/server/index.ts | 4 +- src/types/config.ts | 7 + structure/config.md | 2 +- structure/transports/byte-accounting.md | 15 ++- .../config-concurrent-inbound-body.test.ts | 122 ++++++++++++++++++ tests/fixtures/test-layout-expected.json | 1 + tests/helpers/inbound-body-budget.ts | 6 +- tests/server/inbound-body-admission.test.ts | 28 +++- tests/server/inbound-body-budget.test.ts | 61 +++++++++ 30 files changed, 608 insertions(+), 133 deletions(-) create mode 100644 src/lib/inbound-body-policy.ts create mode 100644 tests/config/config-concurrent-inbound-body.test.ts diff --git a/docs-site/src/content/docs/fr/reference/configuration/providers.md b/docs-site/src/content/docs/fr/reference/configuration/providers.md index a46a4f89add..e63eb4e35a0 100644 --- a/docs-site/src/content/docs/fr/reference/configuration/providers.md +++ b/docs-site/src/content/docs/fr/reference/configuration/providers.md @@ -64,6 +64,8 @@ Après une inscription ou une connexion OAuth dans l’interface, une boîte de | `codexPinnedTransientPolicy?` | `"hold" \| "detour-new-threads"` | `"hold"` | Quand un compte épinglé est dégradé, `hold` continue de l'utiliser et journalise un avertissement ; `detour-new-threads` ne déplace que les nouveaux fils. Un tour déjà envoyé n'est jamais renvoyé. | | `codexWsReuseAcrossTurns?` | `boolean` | `false` | Réutilisation optionnelle du WebSocket Codex entre les tours d'un même compte et fil. Désactivé par défaut. Activé, un socket occupé est attendu au plus 750 ms et jusqu'à deux sockets restent lors d'un changement de modèle ou de palier. | | `upstreamHostCircuitThreshold?` | `number` | `0` | Seuil facultatif du coupe-circuit pour les échecs DNS/TCP avérés avant connexion sur les requêtes Responses OpenAI natives en mode transfert et les envois compacts. `0` le désactive ; `1`–`20` ouvre, après ce nombre de requêtes logiques arrivées à leur terme, une temporisation de 30 secondes propre à l'origine du fournisseur. Tant que le circuit est ouvert, les requêtes reçoivent `503` avec `Retry-After` avant la sélection du compte ou l'envoi en amont ; après la temporisation, une requête est admise en état semi-ouvert. Les délais d'attente et les réponses HTTP ne sont jamais comptabilisés, et toute réponse HTTP ferme le circuit. Ce mécanisme s'applique uniquement au routage du pool Codex sans compte épinglé ; il reste inactif pour `codexAccountMode: "direct"` et les sélecteurs qualifiés par compte. | +| `maxInboundBodyBytes?` | `number` | `0` | Limite en octets de la taille décodée d'un corps JSON entrant par requête. Une valeur absente ou `0` conserve les 256 MiB par défaut ; la valeur configurée est bornée à 1–512 MiB. Redémarrage requis. Un dépassement produit un HTTP 413 local avec le code `inbound_body_too_large`. Une limite effective supérieure à 256 MiB active aussi le contrôle partagé `maxConcurrentInboundBodyBytes` ; relever la limite par requête ne relève pas à lui seul le budget partagé. Routes concernées et détails : [admission des corps entrants](/fr/reference/inbound-body-admission/). | +| `maxConcurrentInboundBodyBytes?` | `number` | `536870912` (512 MiB) | Budget partagé d'admission facultatif de premier niveau, en octets, actif uniquement si la valeur effective de `maxInboundBodyBytes` dépasse 256 MiB. Une valeur absente, `0` ou invalide utilise les 512 MiB par défaut, inchangés. Les entiers positifs valides sont bornés à 512 MiB–2 GiB (`536870912`–`2147483648`) lors de leur utilisation. Redémarrage requis ; tous les écouteurs d'un processus partagent un seul budget. Il s'agit d'un décompte d'admission, pas d'une préallocation mémoire, d'une limite RSS ou d'un dimensionnement automatique selon la RAM. Choisir 2 GiB exige une décision explicite de l'opérateur. Voir [l'admission des corps entrants](/fr/reference/inbound-body-admission/) pour l'exemple, les compromis de file et de cycle de vie des réponses, et la raison pour laquelle le budget par défaut ne permet qu'un seul envoi complet de 512 MiB avant l'analyse. | | `modelCacheTtlMs?` | `number` | `300000` | Fenêtre de fraîcheur pour le cache `/models` par fournisseur. | | `cacheRetention?` | `"none" \| "short" \| "long"` | Anthropic direct : `"long"` ; autres points de terminaison compatibles Anthropic : `"short"` | Politique Anthropic de mise en cache des invites : désactivée, éphémère pendant 5 minutes ou étendue à 1 heure. Sans valeur, les requêtes vers `api.anthropic.com` (clé API ou pools de comptes OAuth) utilisent un TTL d’une heure ; les passerelles et points de terminaison compatibles gardent 5 minutes. Une valeur explicite l’emporte toujours. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | désactivé | Politique facultative d'actualisation proactive OAuth et de préchauffage des comptes Codex. | diff --git a/docs-site/src/content/docs/fr/reference/inbound-body-admission.md b/docs-site/src/content/docs/fr/reference/inbound-body-admission.md index 028d71c0b79..cc3cc9efd6f 100644 --- a/docs-site/src/content/docs/fr/reference/inbound-body-admission.md +++ b/docs-site/src/content/docs/fr/reference/inbound-body-admission.md @@ -9,12 +9,36 @@ reste 256 MiB et les valeurs configurées restent comprises entre 1 MiB et 512 M proxy après avoir modifié la limite afin que l'écouteur et les lecteurs utilisent ensemble la valeur voulue. +Conserver l'ancien budget partagé par défaut de 512 MiB ne supprime pas les compromis +décrits ci-dessous concernant la file et la rétention pendant le cycle de vie des réponses. + +## Choisir explicitement le budget partagé + +`maxConcurrentInboundBodyBytes` est un réglage facultatif de **premier niveau** dans +`config.json`, exprimé en octets. Sa valeur par défaut reste **512 MiB (`536870912`)**, +comme auparavant. Une valeur absente, `0` ou invalide utilise cette valeur par défaut. +Un entier positif valide est borné à **512 MiB–2 GiB (`536870912`–`2147483648`)** lors +de son utilisation. Redémarrez le proxy après modification. Tous les écouteurs d'un même +processus partagent un seul budget, distinct de `appOwnedMemoryBudgetMb`. + +Pour choisir une limite de 512 MiB par requête et activer explicitement un budget partagé +de 2 GiB, ajoutez ces champs au premier niveau de votre `config.json`, puis redémarrez : + +```json +{ + "maxInboundBodyBytes": 536870912, + "maxConcurrentInboundBodyBytes": 2147483648 +} +``` + +Ce choix de l'opérateur augmente le pic potentiel de consommation mémoire. Le proxy ne +détecte pas automatiquement la RAM disponible et ne choisit pas le budget en fonction de celle-ci. + ## Augmenter la limite modifie la simultanéité Quand la limite effective dépasse 256 MiB, chaque requête HTTP concernée doit réserver toute -sa marge par requête **avant toute lecture du corps**. Le budget d'admission partagé par le -processus passe de 512 MiB à **2 GiB**, y compris lorsque plusieurs écouteurs partagent ce -processus. Ce budget est distinct de `appOwnedMemoryBudgetMb`. +sa marge par requête **avant toute lecture du corps**. Ces réservations partagent le budget +`maxConcurrentInboundBodyBytes` du processus, dont la valeur par défaut reste **512 MiB**. Avant la fin de l'analyse JSON avec contrôle de taille, ni un petit `Content-Length` ni un petit corps compressé sur le réseau ne réduisent la réservation. L'admission ne considère pas ces @@ -28,11 +52,18 @@ normalisé supérieure à cette marge ne l'augmente jamais. Les contrôles UTF-8 resérialisé sont conservés ; la longueur d'entrée ne remplace pas la taille potentiellement plus grande du JSON normalisé. -Avec une limite par requête de 512 MiB, le budget peut admettre jusqu'à **4 envois réservant -chacun 512 MiB** avant la mesure et la réduction des réservations. Ce n'est pas une limite fixe -de 4 requêtes après l'analyse : par exemple, **8 très petites requêtes peuvent diffuser leur -réponse simultanément** lorsque les réservations réduites laissent de la place aux admissions -suivantes, sous réserve du budget partagé et des autres limites. +Avec le **budget partagé par défaut de 512 MiB** et une **limite de 512 MiB par requête**, +**un seul envoi non encore analysé** peut réserver toute sa marge. Même si l'analyse réduit +sa réservation à **un octet**, la capacité restante est inférieure à 512 MiB. La prochaine +marge complète doit donc attendre qu'assez de capacité soit libérée. La valeur par défaut +**ne résout pas** le problème initial de simultanéité avec une limite de 512 MiB par requête. + +Définir explicitement `maxConcurrentInboundBodyBytes` à `2147483648` choisit **2 GiB** et +permet jusqu'à **4 envois réservant chacun 512 MiB** avant l'analyse et les réductions après +mesure. Ce budget de 2 GiB ne fixe pas un plafond de 4 requêtes après l'analyse : par exemple, +**8 très petites requêtes peuvent diffuser leur réponse simultanément** lorsque les réservations +réduites laissent de la place aux admissions suivantes, sous réserve du budget partagé et +des autres limites. La réservation réduite reste détenue pendant le travail asynchrone et l'attente du fournisseur, jusqu'à la fin du flux de réponse (EOF), une erreur ou la fin de l'annulation. L'analyse seule @@ -48,15 +79,18 @@ existantes par corps. Les traductions directes internes et les appels combo ne r une deuxième marge HTTP. Les limites de gestion, d'audio, d'historique de contexte et de WebSocket restent inchangées. -Un réglage absent ou égal à zéro, ou une limite effective inférieure ou égale à 256 MiB, -n'active pas ce contrôle supplémentaire de simultanéité. Les limites existantes sur le nombre -de requêtes et les autres ressources continuent de s'appliquer. +Ce contrôle supplémentaire ne s'applique que si la valeur effective de `maxInboundBodyBytes` +dépasse 256 MiB. Omettre ce réglage par requête ou le définir à `0` conserve sa valeur par +défaut de 256 MiB et laisse le contrôle inactif. Omettre ou mettre à zéro +`maxConcurrentInboundBodyBytes` choisit au contraire le budget partagé par défaut de 512 MiB, +sans désactiver un contrôle déjà actif. Les limites existantes sur le nombre de requêtes et +les autres ressources continuent de s'appliquer. ## Refus temporaire ou corps trop volumineux Si la marge complète est temporairement indisponible, la requête peut rejoindre une **file FIFO de 32 requêtes en attente au maximum**, avec une **attente limitée à 120 secondes**. -Les corps restent non lus pendant l'attente. Une réduction après mesure ou une libération +Pendant l'attente, l'application ne lit pas les corps des requêtes. Une réduction après mesure ou une libération finale permet d'admettre les requêtes dans l'ordre de la file. Une requête abandonnée pendant l'attente est retirée de la file et son corps non lu est annulé sans être consommé. @@ -78,18 +112,17 @@ occupé. ## Ce que mesure le budget -Le budget de **2 GiB** compte des unités d'admission : les marges complètes avant l'analyse, -puis les réservations mesurées. Il **ne borne ni la RSS du processus ni le tas des objets -analysés** et ne garantit pas une consommation mémoire inférieure à 2 GiB. Le décodage, les -chaînes, les graphes d'objets, les copies de requêtes et les autres états de l'application -consomment davantage de mémoire. - -Le passage du budget partagé de 512 MiB à 2 GiB quadruple la réservation totale possible. -Les déploiements qui choisissent une limite supérieure à 256 MiB peuvent désormais admettre -4 envois réservant chacun 512 MiB au lieu d'un seul : leur pic potentiel de consommation -mémoire augmente. Les objets analysés et les copies peuvent porter la mémoire réelle au-delà -du budget d'admission. Les limites par défaut ou inférieures n'activent pas ce contrôle et -conservent leur comportement existant. +Le budget configuré compte des unités d'admission : les marges complètes avant l'analyse, +puis les réservations mesurées. Il **ne préalloue ni 2 GiB ni le volume configuré** et +**ne borne ni la RSS du processus ni le tas des objets analysés**. Le décodage, les chaînes, +les graphes d'objets, les copies de requêtes et les autres états de l'application consomment +de la mémoire supplémentaire ; la consommation réelle peut donc dépasser le budget. + +Choisir 2 GiB au lieu du budget partagé par défaut de 512 MiB quadruple la réservation totale +possible. Avec une limite de 512 MiB par requête, ce choix explicite admet jusqu'à quatre envois +complets avant l'analyse au lieu d'un, ce qui augmente le pic potentiel de consommation mémoire. +Choisissez le budget selon la marge mémoire de votre déploiement. Une limite par requête +inférieure ou égale à 256 MiB laisse ce contrôle inactif, quel que soit le budget partagé. Conservez si possible la limite par requête par défaut. Augmentez-la volontairement si un parcours avec un historique volumineux l'exige, en tenant compte du pic mémoire potentiellement diff --git a/docs-site/src/content/docs/ja/reference/configuration/providers.md b/docs-site/src/content/docs/ja/reference/configuration/providers.md index fc82f872333..4ed880176bd 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ja/reference/configuration/providers.md @@ -62,6 +62,8 @@ GUI で登録または OAuth ログインが完了すると、Models ページ | `codexPinnedTransientPolicy?` | `"hold" \| "detour-new-threads"` | `"hold"` | 手動ピンが劣化したとき、`hold` は警告を出して使い続け、`detour-new-threads` は新しいスレッドだけを別アカウントへ置きます。送信済みのターンは再送しません。 | | `codexWsReuseAcrossTurns?` | `boolean` | `false` | 同一アカウントとスレッドで Codex WebSocket をターンをまたいで再利用します。既定はオフ。オンにすると忙しいソケットを最大750ms待ち、モデルやティアの変更では最大2本を残します。 | | `upstreamHostCircuitThreshold?` | `number` | `0` | ネイティブOpenAI forwardのResponses送信とcompact送信で、実証済みの接続前DNS/TCP障害に適用するオプトインのサーキットしきい値です。`0`で無効、`1`〜`20`ではその回数の終端論理リクエストが失敗するとprovider-originを30秒間遮断します。遮断中はアカウント選択やupstream送信の前に`Retry-After`付き`503`を返し、時間経過後はhalf-openリクエストを1件だけ許可します。タイムアウトとHTTP応答は数えず、HTTP応答が1件でもあれば回路を閉じます。 Codex Pool ルーティングでアカウントが固定されていない場合にのみ適用され、`codexAccountMode: "direct"` とアカウント修飾セレクターでは動作しません。 | +| `maxInboundBodyBytes?` | `number` | `0` | 受信する JSON 本文 1 件のデコード後のサイズを制限するバイト単位の設定です。省略または `0` でデフォルトの 256 MiB を維持し、設定値は 1〜512 MiB に制限されます。変更後は再起動が必要です。超過時はコード `inbound_body_too_large` のローカル HTTP 413 を返します。適用される上限が 256 MiB を超えると `maxConcurrentInboundBodyBytes` の共有ゲートも有効になり、リクエスト単位の上限だけを上げても共有予算は増えません。対象ルートと詳細は[受信リクエスト本文の受け入れ](/ja/reference/inbound-body-admission/)を参照してください。 | +| `maxConcurrentInboundBodyBytes?` | `number` | `536870912` (512 MiB) | バイト単位の任意のトップレベル共有受け入れ予算です。適用される `maxInboundBodyBytes` が 256 MiB を超える場合にのみ有効です。省略、`0`、無効な値は従来のデフォルト 512 MiB を使います。有効な正の整数は使用時に 512 MiB〜2 GiB (`536870912`〜`2147483648`) に制限されます。再起動が必要で、同一プロセス内のすべてのリスナーが 1 つの予算を共有します。受け入れの計上用であり、メモリの事前割り当て、RSS 上限、RAM に応じた自動調整ではありません。2 GiB には運用者の明示的な選択が必要です。設定例、キューとレスポンスのライフサイクルに伴う制約、デフォルトで解析前の完全な 512 MiB アップロードが 1 件だけになる理由は[受信リクエスト本文の受け入れ](/ja/reference/inbound-body-admission/)を参照してください。 | | `modelCacheTtlMs?` | `number` | `300000` |プロバイダーごとの `/models` キャッシュの鮮度ウィンドウ。 | | `cacheRetention?` | `"none" \| "short" \| "long"` | Anthropic ファーストパーティ: `"long"`、その他の Anthropic 互換エンドポイント: `"short"` | Anthropic プロンプト キャッシュ ポリシー: 無効、5 分間の一時的、または 1 時間の延長。未設定の場合、`api.anthropic.com`（API キーまたは OAuth アカウント プール）へのリクエストは 1 時間の TTL を使い、互換ゲートウェイやベンダーのエンドポイントは 5 分の TTL のままです。明示した値が常に優先されます。 | | `tokenGuardian?` | `OcxTokenGuardianConfig` |オフ |オプションのプロアクティブな OAuth 更新および Codex アカウントのウォームアップ ポリシー。 | diff --git a/docs-site/src/content/docs/ja/reference/inbound-body-admission.md b/docs-site/src/content/docs/ja/reference/inbound-body-admission.md index 5a693da0259..f2ac83d5542 100644 --- a/docs-site/src/content/docs/ja/reference/inbound-body-admission.md +++ b/docs-site/src/content/docs/ja/reference/inbound-body-admission.md @@ -5,25 +5,44 @@ description: リクエスト本文の上限引き上げが、同時 HTTP リク [プロバイダー設定リファレンス](/ja/reference/configuration/providers/)では、受信する JSON 本文 1 件のデコード後の最大サイズである `maxInboundBodyBytes` を説明しています。デフォルトは引き続き 256 MiB で、設定可能な値は 1 MiB から 512 MiB です。変更後はプロキシを再起動し、リスナーと読み取り処理の両方に意図した上限を適用してください。 +従来の共有予算のデフォルト 512 MiB を維持しても、以下に説明するキューやレスポンスのライフサイクル中の予約保持に伴う制約は残ります。 + +## 共有予算を明示的に選ぶ + +`maxConcurrentInboundBodyBytes` は、バイト単位で指定する `config.json` の任意の**トップレベル**設定です。デフォルトは従来と同じ **512 MiB (`536870912`)** です。省略、`0`、無効な値の場合はこのデフォルトを使います。有効な正の整数は、使用時に **512 MiB〜2 GiB (`536870912`〜`2147483648`)** の範囲に制限されます。設定後はプロキシを再起動してください。同一プロセス内のすべてのリスナーは 1 つの予算を共有し、`appOwnedMemoryBudgetMb` とは別に扱います。 + +リクエストごとの上限を 512 MiB にし、共有予算を明示的に 2 GiB にするには、既存の `config.json` のトップレベルに次のフィールドを追加し、再起動します。 + +```json +{ + "maxInboundBodyBytes": 536870912, + "maxConcurrentInboundBodyBytes": 2147483648 +} +``` + +これはメモリ使用量のピークが増える可能性を踏まえた運用者の選択です。プロキシは利用可能な RAM を自動検出せず、それに応じた予算も選びません。 + ## 上限を引き上げると同時実行数が変わる -適用される上限が 256 MiB を超える場合、対象の各 HTTP リクエストは、**本文を読み取る前に**リクエストごとの許容量全体を予約する必要があります。プロセス全体で共有する受け入れ予算は、従来の 512 MiB から **2 GiB** に増えました。同じプロセス内の複数のリスナーもこの予算を共有します。`appOwnedMemoryBudgetMb` とは別の予算です。 +適用される上限が 256 MiB を超える場合、対象の各 HTTP リクエストは、**本文を読み取る前に**リクエストごとの許容量全体を予約する必要があります。予約はプロセス全体の `maxConcurrentInboundBodyBytes` 予算を共有し、デフォルトは引き続き **512 MiB** です。 サイズを制限した JSON 解析が完了するまでは、`Content-Length` や圧縮された転送本文が小さくても予約量を減らしません。これらの値を、デコード後や正規化後の JSON サイズの根拠として信頼することはありません。 サイズ制限内で JSON 解析に成功すると、読み取り処理は、実測した転送バイト数とデコード後のバイト数のうち大きい値まで予約量を減らします。変換処理が正規化 JSON の測定用予算を渡した場合は、既存の正規化 JSON の測定値も含めた最大値を使います。予約量は最初のリクエスト許容量を上限として減ることしかなく、正規化後の測定値がその上限を超えても増えません。既存の UTF-8 と JSON 再シリアル化の検査は維持され、入力バイト長を、より大きくなり得る正規化 JSON サイズの代わりには使用しません。 -リクエストごとの上限が 512 MiB なら、予約量を測定して減らす前でも、**512 MiB 全体を予約するアップロードを最大 4 件**受け入れられます。解析後の同時実行数が 4 件に固定されるわけではありません。例えば、先に解析したリクエストの予約量が減って次の受け入れに余裕ができれば、共有予算と他の制限の範囲内で、**非常に小さいリクエスト 8 件が同時にレスポンスをストリーミング**できます。 +**デフォルトの共有予算 512 MiB** と**リクエストごとの上限 512 MiB** では、許容量全体を予約できる未解析のアップロードは **1 件だけ**です。解析後の予約が **1 バイト**まで減っても、残りの容量は 512 MiB 未満です。次の完全な許容量は、十分な容量が解放されるまで待つ必要があります。そのため、デフォルトではリクエストごとの上限が 512 MiB の場合の元の同時実行問題は**解決しません**。 + +`maxConcurrentInboundBodyBytes` に `2147483648` を明示すると **2 GiB** を選択し、解析と実測による予約削減の前に **512 MiB 全体を予約するアップロードを最大 4 件**受け入れられます。この 2 GiB 予算は、解析後の同時実行数を 4 件に固定しません。例えば、先に解析したリクエストの予約量が減って次の受け入れに余裕ができれば、共有予算と他の制限の範囲内で、**非常に小さいリクエスト 8 件が同時にレスポンスをストリーミング**できます。 減量後の予約は、非同期処理やプロバイダーの応答待ちの間も保持され、レスポンスストリームの EOF、エラー、またはキャンセル処理の完了時に解放されます。解析だけでは解放されません。クライアントが切断しても、生成側のキャンセル処理が完了するまで保持されます。エラー、中断、キャンセルが競合しても、解放は正確に 1 回だけ行われます。 対象となる POST エンドポイントは `/v1/responses`、`/v1/responses/compact`、`/v1/chat/completions`、`/v1/messages`、`/v1/messages/count_tokens`、`/v1/images/generations`、`/v1/images/edits`、`/v1/alpha/search` です。画像、検索、トークン計数には、従来の設定可能な本文ごとの上限も適用されます。内部の直接変換やコンボ呼び出しは、HTTP の許容量を二重に予約しません。管理、音声、コンテキスト履歴、WebSocket の制限は変わりません。 -設定を省略またはゼロにした場合、あるいは適用される上限が 256 MiB 以下の場合、この追加の同時実行ゲートは有効になりません。既存のリクエスト件数やその他のリソース制限は引き続き適用されます。 +この追加の同時実行ゲートは、適用される `maxInboundBodyBytes` が 256 MiB を超える場合にのみ有効です。このリクエスト単位の設定を省略するか `0` にすると、デフォルトの 256 MiB が使われ、ゲートは無効のままです。一方、`maxConcurrentInboundBodyBytes` を省略または `0` にすると共有予算のデフォルト 512 MiB が選ばれ、有効なゲートを無効にはしません。既存のリクエスト件数やその他のリソース制限は引き続き適用されます。 ## 一時的な拒否と大きすぎる本文の違い -許容量全体を一時的に予約できない場合、リクエストは **最大 32 件の FIFO 待機キュー**に入り、**最大 120 秒**待機できます。待機中は本文を読み取りません。測定後の予約量削減や最終解放で空きができると、待機順に受け入れます。待機中に中断されたリクエストはキューから除去され、未読の本文は消費せずキャンセルされます。 +許容量全体を一時的に予約できない場合、リクエストは **最大 32 件の FIFO 待機キュー**に入り、**最大 120 秒**待機できます。待機中はアプリケーションがリクエスト本文を読み取りません。測定後の予約量削減や最終解放で空きができると、待機順に受け入れます。待機中に中断されたリクエストはキューから除去され、未読の本文は消費せずキャンセルされます。 キューが満杯、または待機時間が満了すると、本文の解析やプロバイダーへの送信前に HTTP **503**、`Retry-After: 1`、エラーコード `server_busy` が返されます。拒否時には未読の本文がキャンセルされます。Messages とトークン計数のクライアントには、型が `overloaded_error` の Anthropic 形式のエラーが、OpenAI 互換クライアントには `server_error` が返されます。認証とオリジンの確認は、受け入れやキューへの追加より先に行われます。 @@ -33,8 +52,8 @@ description: リクエスト本文の上限引き上げが、同時 HTTP リク ## 予算が測るもの -**2 GiB** は受け入れを計上する予算で、解析前は許容量全体、解析後は実測に基づく予約量を合算します。**プロセス RSS や解析済みオブジェクトのヒープの上限ではなく**、メモリ使用量が 2 GiB 未満に収まる保証もありません。デコード、文字列、オブジェクトグラフ、リクエストのコピー、その他のアプリケーション状態には追加のメモリが必要です。 +設定した予算は受け入れを計上するもので、解析前は許容量全体、解析後は実測に基づく予約量を合算します。**2 GiB や設定容量を事前に割り当てるものではなく**、**プロセス RSS や解析済みオブジェクトのヒープの上限でもありません**。デコード、文字列、オブジェクトグラフ、リクエストのコピー、その他のアプリケーション状態には追加のメモリが必要で、実際のメモリ使用量は予算を超えることがあります。 -共有予算が 512 MiB から 2 GiB になり、予約できる合計量は 4 倍になりました。256 MiB を超える上限を選んだ環境では、512 MiB 全体を予約するアップロードを従来の 1 件から最大 4 件まで受け入れられるため、メモリ使用量のピークが増える可能性があります。解析済みオブジェクトやコピーによって、実際のメモリ使用量が受け入れ予算を超えることもあります。デフォルト以下の上限ではこのゲートは有効にならず、既存の動作を維持します。 +共有予算のデフォルト 512 MiB に代えて 2 GiB を選ぶと、予約できる合計量は 4 倍になります。リクエストごとの上限が 512 MiB なら、この明示的な選択により解析前の完全なアップロードを 1 件に代えて最大 4 件受け入れ、メモリ使用量のピークが増える可能性があります。運用環境のメモリの余裕に応じて予算を選んでください。リクエストごとの上限が 256 MiB 以下なら、共有予算の設定にかかわらず追加ゲートは無効です。 可能な限り、リクエスト単位のデフォルト上限を維持してください。長い履歴を扱うために引き上げる場合は、メモリ使用量のピークが大きくなる可能性と待機の上限を考慮してください。 diff --git a/docs-site/src/content/docs/ko/reference/configuration/providers.md b/docs-site/src/content/docs/ko/reference/configuration/providers.md index 6a0a43ef73b..af503a86dc5 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ko/reference/configuration/providers.md @@ -62,6 +62,8 @@ GUI에서 등록이나 OAuth 로그인을 마치면 Models 페이지로 이동 | `codexPinnedTransientPolicy?` | `"hold" \| "detour-new-threads"` | `"hold"` | 수동 핀이 저하되면 `hold`는 경고를 남기고 계속 사용하고, `detour-new-threads`는 새 스레드만 다른 계정에 둡니다. 이미 보낸 턴은 다시 보내지 않습니다. | | `codexWsReuseAcrossTurns?` | `boolean` | `false` | 같은 계정과 스레드의 Codex WebSocket을 턴 사이에 재사용합니다. 기본값은 꺼짐입니다. 켜면 바쁜 소켓을 최대 750ms 기다리고, 모델이나 티어가 바뀌어도 소켓을 최대 2개 유지합니다. | | `upstreamHostCircuitThreshold?` | `number` | `0` | 네이티브 OpenAI forward Responses와 compact 전송에서 입증된 연결 전 DNS/TCP 실패에 적용하는 선택적 회로 차단 임계값입니다. `0`은 비활성화하며, `1`~`20`은 이 횟수만큼 최종 논리 요청이 실패하면 provider-origin을 30초 동안 차단합니다. 차단 중에는 계정 선택이나 업스트림 전송 전에 `Retry-After`가 포함된 `503`을 반환하고, 시간이 지나면 반개방 요청 하나만 허용합니다. 타임아웃과 HTTP 응답은 집계하지 않으며, HTTP 응답이 하나라도 오면 회로를 닫습니다. Codex Pool 라우팅에서 계정이 고정되지 않은 경우에만 적용되며, `codexAccountMode: "direct"` 및 계정 한정 선택자에서는 동작하지 않습니다. | +| `maxInboundBodyBytes?` | `number` | `0` | 인바운드 JSON 본문 하나의 디코딩 후 크기를 제한하는 바이트 단위 설정입니다. 생략하거나 `0`이면 기본값 256 MiB를 유지하며, 설정값은 1~512 MiB로 제한됩니다. 변경 후 다시 시작해야 합니다. 초과 시 오류 코드가 `inbound_body_too_large`인 로컬 HTTP 413을 반환합니다. 적용된 한도가 256 MiB를 넘으면 `maxConcurrentInboundBodyBytes`의 공유 게이트도 적용되며, 요청별 한도만 높여도 공유 예산이 늘지는 않습니다. 적용 대상 경로와 자세한 내용은 [인바운드 본문 수용](/ko/reference/inbound-body-admission/)을 참고하세요. | +| `maxConcurrentInboundBodyBytes?` | `number` | `536870912` (512 MiB) | 바이트 단위의 선택적 최상위 공유 수용 예산입니다. 적용된 `maxInboundBodyBytes`가 256 MiB를 넘을 때만 사용합니다. 생략하거나 `0` 또는 잘못된 값이면 종전 기본값 512 MiB를 사용합니다. 유효한 양의 정수는 실제 사용할 때 512 MiB~2 GiB (`536870912`~`2147483648`)로 제한합니다. 변경 후 다시 시작해야 하며, 같은 프로세스의 모든 리스너가 예산 하나를 공유합니다. 수용량 계산용이며 메모리 사전 할당, RSS 상한 또는 RAM에 따른 자동 조정 기능이 아닙니다. 2 GiB는 운영자가 명시적으로 선택해야 합니다. 설정 예제, 대기열 및 응답 수명에 따른 제약, 기본값으로 파싱 전 512 MiB 전체 업로드를 1건만 수용하는 이유는 [인바운드 본문 수용](/ko/reference/inbound-body-admission/)을 참고하세요. | | `modelCacheTtlMs?` | `number` | `300000` | 공급자별 `/models` 캐시의 최신성 창입니다. | | `cacheRetention?` | `"none" \| "short" \| "long"` | Anthropic 퍼스트파티: `"long"`, 그 외 Anthropic 호환 엔드포인트: `"short"` | Anthropic 프롬프트 캐시 정책입니다. 비활성, 5분짜리 임시, 1시간짜리 확장 중 하나입니다. 설정하지 않으면 `api.anthropic.com`(API 키 또는 OAuth 계정 풀) 요청은 1시간 TTL을 쓰고, 호환 게이트웨이와 벤더 엔드포인트는 5분 TTL을 유지합니다. 명시한 값이 항상 우선합니다. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | 꺼짐 | 선택적 선제 OAuth 갱신과 Codex 계정 워밍업 정책입니다. | diff --git a/docs-site/src/content/docs/ko/reference/inbound-body-admission.md b/docs-site/src/content/docs/ko/reference/inbound-body-admission.md index 1e3431e69a5..45ccb681fd7 100644 --- a/docs-site/src/content/docs/ko/reference/inbound-body-admission.md +++ b/docs-site/src/content/docs/ko/reference/inbound-body-admission.md @@ -5,25 +5,44 @@ description: 요청 본문 한도를 높였을 때 동시 HTTP 요청, 재시도 [프로바이더 설정 레퍼런스](/ko/reference/configuration/providers/)의 `maxInboundBodyBytes`는 인바운드 JSON 본문 하나의 디코딩 후 최대 크기를 설명합니다. 기본값은 계속 256 MiB이며, 설정 가능한 값은 1 MiB에서 512 MiB 사이입니다. 한도를 바꾼 뒤에는 리스너와 리더가 같은 한도를 사용하도록 프록시를 다시 시작하세요. +종전 공유 예산 기본값인 512 MiB를 유지해도 아래에서 설명하는 대기열과 응답 수명 동안의 예약 유지에 따른 제약은 남습니다. + +## 공유 예산을 명시적으로 선택하기 + +`maxConcurrentInboundBodyBytes`는 바이트 단위로 지정하는 `config.json`의 선택적 **최상위** 설정입니다. 기본값은 종전과 같은 **512 MiB (`536870912`)**입니다. 생략하거나 `0` 또는 잘못된 값을 지정하면 이 기본값을 사용합니다. 유효한 양의 정수는 실제 사용할 때 **512 MiB~2 GiB (`536870912`~`2147483648`)** 범위로 제한합니다. 설정을 바꾼 뒤에는 프록시를 다시 시작하세요. 같은 프로세스의 모든 리스너가 예산 하나를 공유하며, `appOwnedMemoryBudgetMb`와는 별개입니다. + +요청별 한도를 512 MiB로 정하고 공유 예산을 2 GiB로 명시적으로 늘리려면, 다음 필드를 기존 `config.json`의 최상위에 추가한 뒤 다시 시작하세요. + +```json +{ + "maxInboundBodyBytes": 536870912, + "maxConcurrentInboundBodyBytes": 2147483648 +} +``` + +이 설정은 순간 최대 메모리 사용량이 커질 수 있음을 고려한 운영자의 선택입니다. 프록시는 사용 가능한 RAM을 자동으로 감지하거나 그에 맞춰 예산을 정하지 않습니다. + ## 한도를 높이면 동시 처리량이 달라집니다 -적용된 한도가 256 MiB를 넘으면 해당 HTTP 요청은 **본문을 읽기 전에** 적용된 요청별 허용량 전체를 예약해야 합니다. 예약에 쓰는 공유 예산은 기존 512 MiB에서 **2 GiB**로 늘었습니다. 같은 프로세스의 여러 리스너도 이 예산을 함께 사용하며, `appOwnedMemoryBudgetMb`와는 별개입니다. +적용된 한도가 256 MiB를 넘으면 해당 HTTP 요청은 **본문을 읽기 전에** 적용된 요청별 허용량 전체를 예약해야 합니다. 예약은 프로세스 전체의 `maxConcurrentInboundBodyBytes` 예산을 공유하며, 기본값은 계속 **512 MiB**입니다. 크기를 제한하면서 JSON 파싱을 완료하기 전까지는 `Content-Length`가 작거나 압축된 전송 본문이 작아도 예약량을 줄이지 않습니다. 이 값들만으로 디코딩 후 크기나 정규화된 JSON 크기를 판단하지 않습니다. 제한된 크기 안에서 JSON 파싱에 성공하면 리더는 실제 전송 바이트 수와 디코딩 후 바이트 수 중 큰 값으로 예약량을 줄입니다. 변환기가 정규화된 JSON 측정에 쓸 예산을 제공한 경우에는 기존 정규화 JSON 측정값까지 포함해 가장 큰 값을 사용합니다. 예약량은 처음 예약한 허용량 범위 안에서 줄어들기만 하며, 정규화 측정값이 그보다 커도 늘어나지 않습니다. 기존 UTF-8 및 JSON 재직렬화 검사는 유지되며, 입력 바이트 길이로 더 클 수 있는 정규화 JSON 크기를 대신하지 않습니다. -요청별 한도가 512 MiB라면 측정으로 예약량을 줄이기 전에도 **512 MiB 전체를 예약하는 업로드를 최대 4건** 수용할 수 있습니다. 파싱 후까지 동시 요청 수를 4건으로 고정하지는 않습니다. 예를 들어 먼저 파싱된 요청의 예약량이 줄어 다음 요청을 수용할 여유가 생기면, 공유 예산과 다른 제한 안에서 **아주 작은 요청 8건도 동시에 응답을 스트리밍**할 수 있습니다. +**기본 공유 예산 512 MiB**와 **요청별 한도 512 MiB**를 사용하면, 아직 파싱하지 않은 업로드는 **1건만** 허용량 전체를 예약할 수 있습니다. 파싱 후 예약량이 **1바이트**까지 줄어도 남는 용량은 512 MiB보다 작습니다. 따라서 충분한 용량이 반환될 때까지 다음 요청은 512 MiB 전체를 예약할 수 없습니다. 기본값만으로는 요청별 한도가 512 MiB일 때의 원래 동시 처리 문제를 **해결하지 못합니다**. + +`maxConcurrentInboundBodyBytes`를 `2147483648`로 명시하면 **2 GiB**를 선택하며, 파싱 및 측정에 따른 예약량 축소 전에도 **512 MiB 전체를 예약하는 업로드를 최대 4건** 수용할 수 있습니다. 이 2 GiB 예산이 파싱 후까지 동시 요청 수를 4건으로 고정하지는 않습니다. 예를 들어 먼저 파싱된 요청의 예약량이 줄어 다음 요청을 수용할 여유가 생기면, 공유 예산과 다른 제한 안에서 **아주 작은 요청 8건도 동시에 응답을 스트리밍**할 수 있습니다. 줄어든 예약량은 비동기 작업과 프로바이더 응답 대기 중에도 유지되며, 응답 스트림의 끝(EOF), 오류 또는 취소 완료 시점에 해제됩니다. 파싱만 끝났다고 해제하지 않습니다. 클라이언트 연결이 끊겨도 응답 생산자의 취소 처리가 끝날 때까지 유지합니다. 오류·요청 중단·취소가 겹치더라도 해제는 정확히 한 번만 일어납니다. 적용 대상 POST 엔드포인트는 `/v1/responses`, `/v1/responses/compact`, `/v1/chat/completions`, `/v1/messages`, `/v1/messages/count_tokens`, `/v1/images/generations`, `/v1/images/edits`, `/v1/alpha/search`입니다. 이미지, 검색, 토큰 계산에는 기존의 설정 가능한 본문별 한도가 계속 적용됩니다. 내부 직접 변환 호출과 콤보 호출은 HTTP 허용량을 두 번 예약하지 않습니다. 관리 API, 오디오, 컨텍스트 기록, WebSocket의 한도는 그대로입니다. -설정을 생략하거나 0으로 두거나 적용된 한도가 256 MiB 이하라면 이 추가 동시 처리 게이트는 켜지지 않습니다. 기존 요청 수 제한과 다른 자원 제한은 계속 적용됩니다. +이 추가 동시 처리 게이트는 적용된 `maxInboundBodyBytes`가 256 MiB를 넘을 때만 켜집니다. 이 요청별 설정을 생략하거나 `0`으로 두면 기본값인 256 MiB가 유지되어 게이트가 꺼져 있습니다. 반면 `maxConcurrentInboundBodyBytes`를 생략하거나 `0`으로 두면 공유 기본값인 512 MiB를 선택하며, 이미 켜진 게이트를 끄지는 않습니다. 기존 요청 수 제한과 다른 자원 제한은 계속 적용됩니다. ## 일시적인 거부와 크기 초과 본문의 차이 -허용량 전체를 바로 예약할 수 없으면 **최대 32건의 요청이 FIFO 대기열**에서 **최대 120초** 기다릴 수 있습니다. 대기 중에는 본문을 읽지 않습니다. 측정 후 예약량이 줄거나 요청이 끝나 예산이 반환되면 대기 순서대로 요청을 수용합니다. 대기 중 요청이 중단되면 대기열에서 제거하고, 읽지 않은 본문을 소비하지 않고 취소합니다. +허용량 전체를 바로 예약할 수 없으면 **최대 32건의 요청이 FIFO 대기열**에서 **최대 120초** 기다릴 수 있습니다. 대기 중에는 애플리케이션이 요청 본문을 읽지 않습니다. 측정 후 예약량이 줄거나 요청이 끝나 예산이 반환되면 대기 순서대로 요청을 수용합니다. 대기 중 요청이 중단되면 대기열에서 제거하고, 읽지 않은 본문을 소비하지 않고 취소합니다. 대기열이 가득 찼거나 대기 시간이 만료되면 본문 파싱이나 프로바이더 호출 전에 HTTP **503**, `Retry-After: 1`, 오류 코드 `server_busy`를 반환하고, 읽지 않은 본문을 취소합니다. Messages 및 토큰 계산 클라이언트에는 유형이 `overloaded_error`인 Anthropic 형식 오류를, OpenAI 호환 클라이언트에는 `server_error`를 반환합니다. 인증과 출처 검사는 예약 및 대기열 진입보다 먼저 실행됩니다. @@ -33,8 +52,8 @@ description: 요청 본문 한도를 높였을 때 동시 HTTP 요청, 재시도 ## 예산이 측정하는 것 -**2 GiB**는 요청을 수용하기 위한 계산 예산입니다. 파싱 전에는 허용량 전체를, 파싱 후에는 측정된 예약량을 합산합니다. **프로세스 RSS나 파싱된 객체의 힙 크기 상한이 아니며**, 실제 메모리가 2 GiB 아래로 유지된다는 보장도 아닙니다. 디코딩, 문자열, 객체 그래프, 요청 사본, 기타 애플리케이션 상태에 추가 메모리가 듭니다. +설정된 예산은 요청을 수용하기 위한 계산 예산입니다. 파싱 전에는 허용량 전체를, 파싱 후에는 측정된 예약량을 합산합니다. **2 GiB나 설정된 용량을 미리 할당하지 않으며**, **프로세스 RSS나 파싱된 객체의 힙 크기 상한도 아닙니다**. 디코딩, 문자열, 객체 그래프, 요청 사본, 기타 애플리케이션 상태에 추가 메모리가 들어 실제 메모리 사용량은 예산을 넘을 수 있습니다. -공유 예산을 512 MiB에서 2 GiB로 늘리면서 예약 가능한 총량도 4배가 됐습니다. 한도를 256 MiB보다 높여 사용하는 환경에서는 512 MiB 전체를 예약하는 업로드를 종전 1건 대신 최대 4건 수용할 수 있어, 순간 최대 메모리 사용량이 더 커질 수 있습니다. 파싱된 객체와 사본 때문에 실제 메모리 사용량이 수용 예산을 넘을 수도 있습니다. 기본 한도와 그 이하의 한도에는 이 추가 게이트가 적용되지 않으며 기존 동작을 유지합니다. +공유 기본값인 512 MiB 대신 2 GiB를 선택하면 예약 가능한 총량이 4배가 됩니다. 요청별 한도가 512 MiB일 때 이 명시적 선택으로 파싱 전 전체 업로드를 1건 대신 최대 4건 수용하므로, 순간 최대 메모리 사용량이 더 커질 수 있습니다. 배포 환경의 여유 메모리를 고려해 예산을 선택하세요. 요청별 한도가 256 MiB 이하라면 공유 예산 설정과 관계없이 이 추가 게이트가 꺼져 있습니다. 가능한 한 요청별 기본 한도를 유지하세요. 긴 기록이 필요한 워크플로에서 한도를 높일 때는 커질 수 있는 순간 최대 메모리 사용량과 대기열의 한도도 함께 고려하세요. diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index 808d56d2c23..74d65c45216 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -135,7 +135,8 @@ separate. Full request URLs such as `/api/v1/responses` are not provider base UR | `codexWsReuseAcrossTurns?` | `boolean` | `false` | Opt-in Codex WebSocket reuse across turns of one account and thread. When on, a busy socket is waited on for at most 750ms before another dial, up to two sockets are kept when the model or service tier changes, and turn-state headers stay on each message instead of the handshake. Authorization, account, proxy, or dial-URL changes still retire the old socket. | | `upstreamHostCircuitThreshold?` | `number` | `0` | Opt-in circuit threshold for proven pre-connection DNS/TCP failures on native OpenAI forward Responses and compact sends. `0` disables it; `1`–`20` opens a 30-second provider-origin cooldown after that many terminal logical requests. While open, requests receive `503` with `Retry-After` before account selection or upstream send; after cooldown, one half-open request is admitted. Timeouts and HTTP responses never count, and any HTTP response closes the circuit. Applies only to Codex Pool routing with no pinned account; it is inert for `codexAccountMode: "direct"` and account-qualified selectors. | | `maxUpstreamBodyBytes?` | `number` | `0` | Opt-in ceiling, in bytes, on a serialized native Responses **passthrough** body. `0` or omitted disables it — no limit is inferred for any destination. When set, a built body above the ceiling is refused locally before the send: streaming turns receive a terminal `response.failed` / `context_length_exceeded` so the client compacts instead of resending, and non-streaming turns receive a `413` naming the size, the number of embedded `input_image` items, and roughly how many megabytes of image data they represent. Checked at every build and rebuild point, including OAuth-refresh replay and alternate-account retry. Translated adapter paths are not covered. There is deliberately no default: the only measured ceiling here belongs to the WebSocket transport, which already falls back to HTTP for oversized turns, so a default would refuse requests that currently succeed. Set it when your gateway has a known request-size limit and you would rather see an actionable local error than an opaque upstream failure. | -| `maxInboundBodyBytes?` | `number` | `0` | Opt-in ceiling, in bytes, on a decompressed **inbound** data-plane request body — the mirror of `maxUpstreamBodyBytes` above. `0` or omitted keeps the built-in 256 MiB default. Raise it when a large-context session can no longer compact: Codex replays the whole history to the compaction model, so on the 922k-token opt-in window the compaction request is itself the one that crosses the limit, and the session is stuck at the only operation that would have shrunk it. Clamped to 1 MiB–512 MiB. The ceiling is not negotiable: the reader materializes the body several times over (wire bytes, decoded bytes, the decoded string, and the parsed object graph), so peak memory is a multiple of whatever is admitted, and an unbounded value would be a memory exhaustion lever. Applies to `/v1/responses`, `/v1/responses/compact`, `/v1/chat/completions`, and `/v1/messages`. The listener's accept size is fixed when the proxy binds, so a change takes effect on restart. A body above the limit is refused locally with HTTP 413 and `code: "inbound_body_too_large"`, which is deliberately distinct from the `context_length_exceeded` 413 a provider size refusal produces. | +| `maxInboundBodyBytes?` | `number` | `0` | Opt-in ceiling, in bytes, on a decompressed **inbound** data-plane request body — the mirror of `maxUpstreamBodyBytes` above. `0` or omitted keeps the built-in 256 MiB default. Raise it when a large-context session can no longer compact: Codex replays the whole history to the compaction model, so on the 922k-token opt-in window the compaction request is itself the one that crosses the limit, and the session is stuck at the only operation that would have shrunk it. Clamped to 1 MiB–512 MiB. The ceiling is not negotiable: the reader materializes the body several times over (wire bytes, decoded bytes, the decoded string, and the parsed object graph), so peak memory is a multiple of whatever is admitted, and an unbounded value would be a memory exhaustion lever. Applies to `/v1/responses`, `/v1/responses/compact`, `/v1/chat/completions`, and `/v1/messages`, plus the image, search, and token-counting routes listed in [inbound body admission](/reference/inbound-body-admission/). The listener's accept size is fixed when the proxy binds, so a change takes effect on restart. A body above the limit is refused locally with HTTP 413 and `code: "inbound_body_too_large"`, which is deliberately distinct from the `context_length_exceeded` 413 a provider size refusal produces. A resolved limit above 256 MiB also activates the shared admission gate controlled by `maxConcurrentInboundBodyBytes`; raising this per-request limit alone does not raise that budget. | +| `maxConcurrentInboundBodyBytes?` | `number` | `536870912` (512 MiB) | Optional top-level shared admission budget in bytes, active only when the resolved `maxInboundBodyBytes` exceeds 256 MiB. Omission, `0`, or an invalid value uses the unchanged 512 MiB default. Valid positive integers are clamped to 512 MiB–2 GiB (`536870912`–`2147483648`) when consumed. Restart required; all listeners in one process share one budget. This is admission accounting, not preallocated memory, an RSS cap, or automatic RAM sizing. A 2 GiB budget requires an explicit operator choice. See [inbound body admission](/reference/inbound-body-admission/) for the configuration example, queue and response-lifetime tradeoffs, and why the default still permits only one full 512 MiB upload before parsing. | | `upstreamRequestCompression?` | `"off" \| "gzip" \| "zstd"` | `"off"` | Opt-in compression of native ChatGPT Responses HTTP request bodies. Codex re-sends the whole conversation on every turn, and when that turn goes over HTTP OpenCodex otherwise forwards it to `chatgpt.com` uncompressed; on a metered or long-distance egress that upload is the dominant cost. When set, string bodies from 1 KiB to 16 MiB sent to `https://chatgpt.com/backend-api/codex/responses` and `/compact` are compressed at the final send and labelled with `Content-Encoding`. WebSocket turns, every other destination, bodies that already declare a `Content-Encoding`, and bodies outside that size window are sent unchanged, and a codec failure falls back to the uncompressed send. `zstd` is the encoding Codex CLI itself uses with this backend and is the faster choice for image-heavy turns; `gzip` (level 1) is also available. In a local measurement, code-heavy history shrank about 2–4×, while embedded images barely shrink. Compression runs synchronously on the proxy's event loop, so each eligible send briefly delays other in-flight requests — on the order of 2–3 ms per MiB for text on a recent laptop CPU, more for incompressible data with `gzip`. Byte limits such as `maxUpstreamBodyBytes` still measure the uncompressed body. Set it with `ocx config set upstreamRequestCompression zstd` (an unknown value is refused) or by editing `config.json`, where an unknown value is ignored, then restart the proxy to apply. | | `modelCacheTtlMs?` | `number` | `300000` | Freshness window for the per-provider `/models` cache. | | `cacheRetention?` | `"none" \| "short" \| "long"` | first-party Anthropic: `"long"`; other Anthropic-compatible endpoints: `"short"` | Anthropic prompt-cache policy: disabled, 5-minute ephemeral, or 1-hour extended. When unset, requests to `api.anthropic.com` (API key or the OAuth account pools) use the 1-hour TTL, because agentic turns often pause longer than five minutes and a 1-hour write (2x base input instead of 1.25x) pays for itself on the first reuse; Anthropic-compatible gateways and vendor endpoints keep the 5-minute TTL because their 1-hour support is undocumented. On native `/v1/messages` passthrough, caller breakpoints without a `ttl` get `"1h"` unless the request pins any breakpoint to `"5m"`, in which case it is forwarded unchanged. On a translated `/v1/messages` route, caller breakpoints that all ask for `ttl: "1h"` mean `"long"` and any explicit `"5m"` means `"short"`. An explicit value always wins. Changing the TTL makes each open conversation write its cache once more, since 5-minute and 1-hour entries are separate. Cost estimates price writes reported as 1-hour at the 1-hour rate. | diff --git a/docs-site/src/content/docs/reference/inbound-body-admission.md b/docs-site/src/content/docs/reference/inbound-body-admission.md index 8c89b428e5d..52533905a45 100644 --- a/docs-site/src/content/docs/reference/inbound-body-admission.md +++ b/docs-site/src/content/docs/reference/inbound-body-admission.md @@ -8,12 +8,36 @@ The [provider configuration reference](/reference/configuration/providers/) desc 256 MiB, and configured values remain bounded to 1 MiB through 512 MiB. Restart the proxy after changing the limit so the listener and readers use the intended limit together. +Preserving the previous 512 MiB shared-budget default does not remove the queueing and +response-lifetime tradeoffs described below. + +## Choose the shared budget explicitly + +`maxConcurrentInboundBodyBytes` is an optional **top-level** `config.json` setting in bytes. +It defaults to **512 MiB (`536870912`)**, preserving the previous shared budget. +Omission, `0`, or an invalid value uses that default. A valid positive integer is clamped +to **512 MiB–2 GiB (`536870912`–`2147483648`)** when consumed. Restart the proxy after +changing this setting. All listeners in one process share one budget; this setting is +separate from `appOwnedMemoryBudgetMb`. + +To choose a 512 MiB per-request limit and explicitly opt into a 2 GiB shared budget, merge +these fields into the top level of your `config.json`, then restart: + +```json +{ + "maxInboundBodyBytes": 536870912, + "maxConcurrentInboundBodyBytes": 2147483648 +} +``` + +This is an operator choice that increases the potential memory peak. The proxy does not +automatically detect available RAM or choose a budget based on it. + ## Raising the limit changes concurrency When the resolved limit is greater than 256 MiB, each covered HTTP request must reserve its full -resolved per-request allowance **before any body is read**. These reservations share a -process-wide **2 GiB** admission budget, increased from 512 MiB, including across listeners in -the same process. This budget is separate from `appOwnedMemoryBudgetMb`. +resolved per-request allowance **before any body is read**. These reservations share the +process-wide `maxConcurrentInboundBodyBytes` budget, whose default remains **512 MiB**. Before bounded JSON parsing completes, a small `Content-Length` or a small compressed wire body does not reduce the reservation. Admission does not trust either as evidence of the decoded or @@ -26,10 +50,17 @@ measured bytes up to the original allowance; a larger normalized measurement nev Input byte length does not replace the potentially larger normalized JSON size; the existing UTF-8 and reserialized-JSON checks remain. -With a 512 MiB per-request limit, the shared budget can admit up to **four full 512 MiB uploads** -before their reservations are measured and reduced. This is not a fixed four-request limit -after parsing: for example, **eight tiny requests can stream concurrently** once their reduced -reservations leave room for subsequent admissions, subject to the shared budget and other limits. +With the default **512 MiB shared budget** and a **512 MiB per-request limit**, only **one +unparsed upload** can reserve its full allowance. Even if parsing reduces its reservation to +**one byte**, the remaining capacity is less than 512 MiB, so the next full allowance must +wait until enough capacity is released. The default therefore does **not** solve the original +concurrency problem at a 512 MiB per-request limit. + +Explicitly setting `maxConcurrentInboundBodyBytes` to `2147483648` chooses **2 GiB** and permits +up to **four full 512 MiB uploads** before parsing and measured reductions. That 2 GiB budget +is not a fixed four-request limit after parsing: for example, **eight tiny requests can stream +concurrently** once their reduced reservations leave room for subsequent admissions, subject +to the shared budget and other limits. The reduced reservation stays held across asynchronous work and provider waits, through response stream EOF, error, or settled cancellation. Parsing alone does not release it. On a client @@ -43,13 +74,17 @@ Images, search, and token counting keep their existing configurable per-body lim Internal direct translation and combo calls do not reserve a second HTTP allowance. Management, audio, context-history, and WebSocket limits are unchanged. -An omitted/zero setting, or a resolved limit at or below 256 MiB, does not enable this additional -concurrency gate. Existing request-count and other resource limits still apply. +This additional concurrency gate applies only when the resolved `maxInboundBodyBytes` is +greater than 256 MiB. Omitting that per-request setting or setting it to `0` keeps its 256 MiB +default and leaves this gate inactive. Omitting or zeroing `maxConcurrentInboundBodyBytes` +instead selects the 512 MiB shared default; it does not disable an otherwise active gate. +Existing request-count and other resource limits still apply. ## Temporary refusal versus an oversized body If the full allowance is temporarily unavailable, the request can enter a bounded **FIFO queue of -up to 32 waiting requests**, with a **120-second wait limit**. Waiting bodies remain unread. +up to 32 waiting requests**, with a **120-second wait limit**. While queued, the application +does not read request bodies. Capacity freed by a measured reduction or final release admits waiting requests in FIFO order. An aborted waiter is removed and its unread body is cancelled without consuming it. @@ -68,16 +103,16 @@ A declared oversize keeps that path instead of becoming a busy refusal. ## What the budget measures -The **2 GiB** budget counts admission units: full allowances before parsing, then measured -reservations. It is **not a bound on process RSS or parsed-object heap**, and does not guarantee -memory usage below 2 GiB. Decoding, strings, object graphs, request copies, and other application -state use additional memory. +The configured budget counts admission units: full allowances before parsing, then measured +reservations. It does **not preallocate 2 GiB** or the configured amount, and it is **not a +bound on process RSS or parsed-object heap**. Decoding, strings, object graphs, request copies, +and other application state use additional memory, so actual memory use can exceed the budget. -Increasing the shared budget from 512 MiB to 2 GiB permits four times the total reservation. -Deployments that opt into limits above 256 MiB can now admit four full 512 MiB uploads instead -of one, so their potential peak memory use increases; parsed objects and copies can push actual -memory beyond the admission budget. Default and smaller limits do not activate this gate and -keep their existing behavior. +Choosing 2 GiB instead of the 512 MiB shared default permits four times the total reservation. +With a 512 MiB per-request limit, this explicit opt-in admits up to four full uploads before +parsing instead of one, increasing potential peak memory use. Choose the budget based on +your deployment's memory headroom. A per-request limit at or below 256 MiB leaves this +additional gate inactive, regardless of the shared-budget setting. Keep the default per-request limit where possible. Raise it deliberately when a large-history workflow needs it, accounting for the larger potential memory peak and bounded waiting. diff --git a/docs-site/src/content/docs/ru/reference/configuration/providers.md b/docs-site/src/content/docs/ru/reference/configuration/providers.md index bba82ea2c7b..b3cab619dab 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ru/reference/configuration/providers.md @@ -63,6 +63,8 @@ ocx models provider openrouter on | `codexPinnedTransientPolicy?` | `"hold" \| "detour-new-threads"` | `"hold"` | Если закреплённый аккаунт деградировал, `hold` продолжает его использовать и пишет предупреждение; `detour-new-threads` уводит только новые потоки. Уже отправленный ход повторно не отправляется. | | `codexWsReuseAcrossTurns?` | `boolean` | `false` | Необязательное повторное использование Codex WebSocket между ходами одного аккаунта и потока. По умолчанию выключено. Во включённом режиме занятый сокет ждут не дольше 750 мс, а при смене модели или уровня сохраняются до двух сокетов. | | `upstreamHostCircuitThreshold?` | `number` | `0` | Опциональный порог circuit breaker для доказанных DNS/TCP-сбоев до соединения в нативных OpenAI forward Responses- и compact-отправках. `0` отключает его; `1`–`20` открывает 30-секундный cooldown для provider-origin после такого числа завершившихся логических запросов. Пока circuit открыт, до выбора аккаунта и upstream-отправки возвращается `503` с `Retry-After`; после cooldown допускается один half-open запрос. Таймауты и HTTP-ответы не учитываются, а любой HTTP-ответ закрывает circuit. Применяется только к маршрутизации Codex Pool без закреплённого аккаунта; при `codexAccountMode: "direct"` и для селекторов с указанием аккаунта схема не активна. | +| `maxInboundBodyBytes?` | `number` | `0` | Лимит декодированного входящего тела JSON отдельного запроса в байтах. Отсутствие или `0` сохраняет стандартные 256 МиБ; настраиваемое значение ограничено диапазоном 1–512 МиБ. Требуется перезапуск. Превышение даёт локальный HTTP 413 с кодом `inbound_body_too_large`. При итоговом лимите выше 256 МиБ дополнительно действует общий барьер `maxConcurrentInboundBodyBytes`; увеличение лимита запроса само по себе не увеличивает общий бюджет. Охватываемые маршруты и подробности: [допуск входящих тел](/ru/reference/inbound-body-admission/). | +| `maxConcurrentInboundBodyBytes?` | `number` | `536870912` (512 МиБ) | Необязательный общий бюджет допуска верхнего уровня в байтах; действует только при итоговом `maxInboundBodyBytes` выше 256 МиБ. Отсутствие, `0` или некорректное значение сохраняет прежние 512 МиБ по умолчанию. Допустимые положительные целые при использовании ограничиваются диапазоном 512 МиБ–2 ГиБ (`536870912`–`2147483648`). Требуется перезапуск; все слушатели одного процесса делят один бюджет. Это учёт допуска, а не предварительное выделение памяти, предел RSS или автоматический подбор по RAM. Для 2 ГиБ нужен явный выбор оператора. Пример, ограничения очереди и жизненного цикла ответа, а также причина, по которой бюджет по умолчанию допускает до разбора только одну полную загрузку 512 МиБ: [допуск входящих тел](/ru/reference/inbound-body-admission/). | | `modelCacheTtlMs?` | `number` | `300000` | Окно свежести для кэша `/models` на уровне провайдера. | | `cacheRetention?` | `"none" \| "short" \| "long"` | Anthropic напрямую: `"long"`; другие Anthropic-совместимые эндпоинты: `"short"` | Политика prompt-cache Anthropic: отключено, 5-минутный ephemeral или 1-часовой extended. Если значение не задано, запросы к `api.anthropic.com` (API-ключ или пулы OAuth-аккаунтов) используют TTL 1 час; совместимые шлюзы и эндпоинты вендоров остаются на 5 минутах. Явно заданное значение всегда имеет приоритет. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | off | Необязательная политика proactive OAuth refresh и warmup'а аккаунтов Codex. | diff --git a/docs-site/src/content/docs/ru/reference/inbound-body-admission.md b/docs-site/src/content/docs/ru/reference/inbound-body-admission.md index 96d63e190b3..09dd53caad8 100644 --- a/docs-site/src/content/docs/ru/reference/inbound-body-admission.md +++ b/docs-site/src/content/docs/ru/reference/inbound-body-admission.md @@ -9,12 +9,36 @@ description: Как увеличение лимита тела запроса в После изменения лимита перезапустите прокси, чтобы слушатель и обработчики чтения одновременно перешли на нужное значение. +Сохранение прежнего общего бюджета 512 МиБ по умолчанию не устраняет описанные ниже +ограничения очереди и удержания резерва на протяжении жизненного цикла ответа. + +## Выбирайте общий бюджет явно + +`maxConcurrentInboundBodyBytes` — необязательная настройка **верхнего уровня** `config.json` +в байтах. По умолчанию используется **512 МиБ (`536870912`)**, как и прежде. +При отсутствии настройки, значении `0` или некорректном значении применяется этот бюджет. +Допустимое положительное целое при использовании ограничивается диапазоном +**512 МиБ–2 ГиБ (`536870912`–`2147483648`)**. После изменения перезапустите прокси. +Все слушатели одного процесса делят один бюджет, отдельный от `appOwnedMemoryBudgetMb`. + +Чтобы выбрать лимит 512 МиБ на запрос и явно включить общий бюджет 2 ГиБ, добавьте +эти поля на верхний уровень своего `config.json`, затем перезапустите прокси: + +```json +{ + "maxInboundBodyBytes": 536870912, + "maxConcurrentInboundBodyBytes": 2147483648 +} +``` + +Это выбор оператора, который повышает потенциальный пик потребления памяти. +Прокси не определяет доступную RAM автоматически и не подбирает по ней бюджет. + ## Увеличение лимита меняет параллельность Если итоговый лимит превышает 256 МиБ, каждый охватываемый HTTP-запрос резервирует -весь лимит отдельного запроса **до чтения любых данных тела**. Общий для процесса бюджет -допуска увеличен с 512 МиБ до **2 ГиБ**; несколько слушателей в одном процессе также -делят этот бюджет. Он учитывается отдельно от `appOwnedMemoryBudgetMb`. +весь лимит отдельного запроса **до чтения любых данных тела**. Резервы используют общий +для процесса бюджет `maxConcurrentInboundBodyBytes`, по умолчанию по-прежнему **512 МиБ**. До завершения разбора JSON с контролем размера ни малый `Content-Length`, ни малый размер сжатого тела при передаче не уменьшают резерв. Допуск не считает эти значения надёжным @@ -28,10 +52,17 @@ description: Как увеличение лимита тела запроса в только уменьшаться и не превышает исходный лимит запроса; даже большее измерение нормализованного JSON никогда не увеличивает резерв. -При лимите 512 МиБ на запрос бюджет допускает до **4 загрузок, каждая из которых резервирует полные 512 МиБ**, -ещё до измерения и уменьшения резервов. После разбора это не фиксированный -предел в 4 запроса: например, **8 очень маленьких запросов могут одновременно передавать -потоковые ответы**, когда уменьшение резервов позволяет допустить следующие запросы, +При **общем бюджете 512 МиБ по умолчанию** и **лимите 512 МиБ на запрос** весь лимит +может зарезервировать только **одна ещё не разобранная загрузка**. Даже если после разбора +её резерв уменьшится до **одного байта**, свободного места будет меньше 512 МиБ. +Следующий полный резерв ждёт освобождения достаточной ёмкости. Поэтому бюджет по умолчанию +**не решает** исходную проблему параллельности при лимите 512 МиБ на запрос. + +Явное значение `maxConcurrentInboundBodyBytes: 2147483648` выбирает **2 ГиБ** и допускает +до **4 загрузок, каждая из которых резервирует полные 512 МиБ**, ещё до разбора и уменьшения +резервов по измерениям. После разбора этот бюджет 2 ГиБ не задаёт фиксированный предел +в 4 запроса: например, **8 очень маленьких запросов могут одновременно передавать потоковые +ответы**, когда уменьшение резервов позволяет допустить следующие запросы, в пределах общего бюджета и других ограничений. Уменьшенный резерв сохраняется во время асинхронной работы и ожидания провайдера, @@ -47,15 +78,17 @@ description: Как увеличение лимита тела запроса в Внутренний прямой перевод запросов и вызовы combo не резервируют второй HTTP-допуск. Лимиты управления, аудио, истории контекста и WebSocket не меняются. -Отсутствующая или нулевая настройка, как и итоговый лимит не выше 256 МиБ, не включает -этот дополнительный барьер параллельности. Существующие ограничения числа запросов -и других ресурсов продолжают действовать. +Этот дополнительный барьер действует только при итоговом `maxInboundBodyBytes` выше 256 МиБ. +Отсутствие этой настройки отдельного запроса или значение `0` сохраняет её стандартные +256 МиБ и не включает барьер. Отсутствие или обнуление `maxConcurrentInboundBodyBytes` +выбирает общий бюджет 512 МиБ, а не отключает уже действующий барьер. +Существующие ограничения числа запросов и других ресурсов продолжают действовать. ## Временный отказ и слишком большое тело Если весь лимит временно нельзя зарезервировать, запрос может попасть в **очередь FIFO не более чем из 32 ожидающих запросов**, с **ожиданием не более 120 секунд**. Во время -ожидания тела остаются непрочитанными. Уменьшение резерва после измерения или его +ожидания приложение не читает тела запросов. Уменьшение резерва после измерения или его окончательное освобождение позволяет допускать запросы в порядке очереди. Прерванный ожидающий запрос удаляется из очереди, а его непрочитанное тело отменяется без чтения. @@ -76,17 +109,17 @@ description: Как увеличение лимита тела запроса в ## Что измеряет бюджет -Бюджет **2 ГиБ** считает единицы допуска: полные лимиты до разбора, затем измеренные -резервы. Это **не предел RSS процесса или кучи разобранных объектов** и не гарантия -потребления памяти ниже 2 ГиБ. Декодирование, строки, графы объектов, копии запросов -и другое состояние приложения занимают дополнительную память. - -Увеличение общего бюджета с 512 МиБ до 2 ГиБ разрешает вчетверо больший суммарный резерв. -Развёртывания, в которых выбран лимит выше 256 МиБ, теперь допускают до 4 загрузок -с резервом по полных 512 МиБ вместо одной, поэтому потенциальный пик потребления памяти -повышается. Разобранные объекты и копии могут увеличить фактическую память сверх бюджета -допуска. Лимит по умолчанию и меньшие значения не включают этот барьер и сохраняют -прежнее поведение. +Настроенный бюджет считает единицы допуска: полные лимиты до разбора, затем измеренные +резервы. Он **не выделяет заранее 2 ГиБ** или настроенный объём и **не ограничивает RSS +процесса или кучу разобранных объектов**. Декодирование, строки, графы объектов, копии +запросов и другое состояние приложения занимают дополнительную память, поэтому +фактическое потребление может превышать бюджет. + +Выбор 2 ГиБ вместо общего бюджета 512 МиБ по умолчанию разрешает вчетверо больший +суммарный резерв. При лимите 512 МиБ на запрос этот явный выбор допускает до четырёх +полных загрузок до разбора вместо одной, повышая потенциальный пик потребления памяти. +Выбирайте бюджет с учётом запаса памяти развёртывания. Лимит отдельного запроса +не выше 256 МиБ не включает дополнительный барьер независимо от общего бюджета. По возможности оставляйте лимит отдельного запроса по умолчанию. Увеличивайте его осознанно, если этого требует сценарий с большой историей, учитывая больший потенциальный пик памяти diff --git a/docs-site/src/content/docs/tr/reference/configuration/providers.md b/docs-site/src/content/docs/tr/reference/configuration/providers.md index 0242a1b4754..91f90523d70 100644 --- a/docs-site/src/content/docs/tr/reference/configuration/providers.md +++ b/docs-site/src/content/docs/tr/reference/configuration/providers.md @@ -64,6 +64,8 @@ Arayüzde kayıt veya OAuth girişi tamamlanınca Models sayfasını açan bir b | `codexPinnedTransientPolicy?` | `"hold" \| "detour-new-threads"` | `"hold"` | Elle sabitlenmiş hesap düştüğünde `hold` uyarı yazar ve kullanmaya devam eder; `detour-new-threads` yalnızca yeni iş parçacıklarını başka hesaba alır. Gönderilmiş bir tur yeniden gönderilmez. | | `codexWsReuseAcrossTurns?` | `boolean` | `false` | Aynı hesap ve iş parçacığında Codex WebSocket'ini turlar arasında yeniden kullanır. Varsayılan kapalıdır. Açıkken meşgul soket en fazla 750 ms beklenir; model veya katman değişince en fazla 2 soket tutulur. | | `upstreamHostCircuitThreshold?` | `number` | `0` | Yerel OpenAI iletme Responses ve sıkıştırma gönderimlerinde kanıtlanmış bağlantı öncesi DNS/TCP arızaları için isteğe bağlı devre eşiği. `0` devre dışı bırakır; `1`–`20`, bu kadar terminal mantıksal istekten sonra 30 saniyelik bir sağlayıcı-kaynak soğuma süresi açar. Açıkken istekler, hesap seçiminden veya yukarı akış gönderiminden önce `Retry-After` ile `503` alır; soğuma süresinden sonra bir yarı açık isteğe izin verilir. Zaman aşımları ve HTTP yanıtları asla sayılmaz ve herhangi bir HTTP yanıtı devreyi kapatır. Yalnızca sabitlenmiş hesabı olmayan Codex Havuz yönlendirmesi için geçerlidir; `codexAccountMode: "direct"` ve hesap nitelikli seçiciler için etkisizdir. | +| `maxInboundBodyBytes?` | `number` | `0` | Tek gelen JSON gövdesinin çözülmüş boyutunu bayt cinsinden sınırlayan istek başına ayar. Atlanırsa veya `0` ise 256 MiB varsayılanı korunur; yapılandırılan değer 1–512 MiB ile sınırlıdır. Yeniden başlatma gerekir. Aşımda `inbound_body_too_large` koduyla yerel HTTP 413 döner. Etkin sınır 256 MiB'ı aşarsa `maxConcurrentInboundBodyBytes` paylaşılan kapısı da uygulanır; yalnızca istek başına sınırı artırmak paylaşılan bütçeyi artırmaz. Kapsanan yollar ve ayrıntılar: [gelen gövde kabulü](/tr/reference/inbound-body-admission/). | +| `maxConcurrentInboundBodyBytes?` | `number` | `536870912` (512 MiB) | Bayt cinsinden isteğe bağlı üst düzey paylaşılan kabul bütçesi; yalnızca etkin `maxInboundBodyBytes` 256 MiB'ı aştığında uygulanır. Atlanan, `0` veya geçersiz değer, değişmeyen 512 MiB varsayılanını kullanır. Geçerli pozitif tam sayılar kullanılırken 512 MiB–2 GiB (`536870912`–`2147483648`) aralığına sınırlandırılır. Yeniden başlatma gerekir; bir sürecin tüm dinleyicileri tek bütçeyi paylaşır. Bu bir kabul hesabıdır; önceden ayrılan bellek, RSS sınırı veya RAM'e göre otomatik boyutlandırma değildir. 2 GiB için operatörün açık seçimi gerekir. Yapılandırma örneği, kuyruk ve yanıt yaşam döngüsü ödünleşimleri ve varsayılanın ayrıştırmadan önce neden yalnızca bir tam 512 MiB yüklemeye izin verdiği için [gelen gövde kabulüne](/tr/reference/inbound-body-admission/) bakın. | | `modelCacheTtlMs?` | `number` | `300000` | Sağlayıcı başına `/models` önbelleği için tazelik penceresi. | | `cacheRetention?` | `"none" \| "short" \| "long"` | Birinci taraf Anthropic: `"long"`; diğer Anthropic uyumlu uç noktalar: `"short"` | Anthropic istem önbelleği politikası: devre dışı, 5 dakikalık kısa ömürlü veya 1 saatlik uzatılmış. Ayarlanmadığında `api.anthropic.com` (API anahtarı veya OAuth hesap havuzları) istekleri 1 saatlik TTL kullanır; uyumlu ağ geçitleri ve sağlayıcı uç noktaları 5 dakikada kalır. Açıkça verilen değer her zaman önceliklidir. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | kapalı | İsteğe bağlı proaktif OAuth yenileme ve Codex hesabı ısınma politikası. | diff --git a/docs-site/src/content/docs/tr/reference/inbound-body-admission.md b/docs-site/src/content/docs/tr/reference/inbound-body-admission.md index c061800a904..17bbfdec6d5 100644 --- a/docs-site/src/content/docs/tr/reference/inbound-body-admission.md +++ b/docs-site/src/content/docs/tr/reference/inbound-body-admission.md @@ -9,13 +9,37 @@ ayarını açıklar. Varsayılan 256 MiB olarak kalır; yapılandırılan değer ile 512 MiB arasında sınırlandırılır. Sınırı değiştirdikten sonra dinleyici ve okuyucuların aynı sınırı kullanması için proxy'yi yeniden başlatın. +Önceki 512 MiB paylaşılan bütçe varsayılanını korumak, aşağıda açıklanan kuyruk ve +yanıtın yaşam döngüsü boyunca rezervasyonun tutulmasına ilişkin ödünleşimleri ortadan kaldırmaz. + +## Paylaşılan bütçeyi açıkça seçin + +`maxConcurrentInboundBodyBytes`, `config.json` içinde bayt cinsinden isteğe bağlı bir +**üst düzey** ayardır. Varsayılanı önceki gibi **512 MiB (`536870912`)** +olarak kalır. Ayar atlanırsa, `0` veya geçersiz bir değer verilirse bu varsayılan kullanılır. +Geçerli bir pozitif tam sayı, kullanıldığı anda **512 MiB–2 GiB (`536870912`–`2147483648`)** +aralığına sınırlandırılır. Değişiklikten sonra proxy'yi yeniden başlatın. Aynı süreçteki tüm +dinleyiciler tek bütçeyi paylaşır; bu bütçe `appOwnedMemoryBudgetMb` ayarından ayrıdır. + +İstek başına 512 MiB sınırı seçip paylaşılan bütçeyi açıkça 2 GiB yapmak için bu alanları +mevcut `config.json` dosyanızın üst düzeyine ekleyin, ardından yeniden başlatın: + +```json +{ + "maxInboundBodyBytes": 536870912, + "maxConcurrentInboundBodyBytes": 2147483648 +} +``` + +Bu, olası en yüksek bellek kullanımını artıran bir operatör seçimidir. Proxy kullanılabilir +RAM'i otomatik algılamaz veya bütçeyi buna göre seçmez. + ## Sınırı artırmak eşzamanlılığı değiştirir Etkin sınır 256 MiB'ı aştığında kapsanan her HTTP isteği, **gövdeden herhangi -bir veri okunmadan önce** istek başına etkin payın tamamını ayırmalıdır. Süreç -genelinde paylaşılan kabul bütçesi 512 MiB'dan **2 GiB**'a çıkarılmıştır; aynı -süreçteki birden çok dinleyici de bu bütçeyi paylaşır. Bu bütçe -`appOwnedMemoryBudgetMb` ayarından ayrıdır. +bir veri okunmadan önce** istek başına etkin payın tamamını ayırmalıdır. Bu rezervasyonlar +süreç genelindeki `maxConcurrentInboundBodyBytes` bütçesini paylaşır; varsayılan +**512 MiB** olarak kalır. Boyut denetimli JSON ayrıştırması tamamlanmadan önce küçük bir `Content-Length` veya küçük bir sıkıştırılmış aktarım gövdesi rezervasyonu azaltmaz. Kabul @@ -31,12 +55,17 @@ boyutunun yerine geçmez. Rezervasyon yalnızca azalır ve başlangıçta ayrıl istek başına payı aşmaz; daha büyük bir normalleştirilmiş JSON ölçümü bile rezervasyonu artırmaz. -İstek başına sınır 512 MiB olduğunda bütçe, rezervasyonlar ölçülüp azaltılmadan -önce **her biri tam 512 MiB ayıran en fazla 4 yüklemeyi** kabul edebilir. -Ayrıştırma sonrasında eşzamanlılık 4 istekle sabitlenmez: örneğin azalan -rezervasyonlar sonraki kabullere yer açtığında, paylaşılan bütçe ve diğer -sınırlar izin verdiği sürece **8 çok küçük istek aynı anda yanıt akışı -iletebilir**. +**Varsayılan 512 MiB paylaşılan bütçe** ve **istek başına 512 MiB sınırı** ile, +henüz ayrıştırılmamış **yalnızca bir yükleme** tam payını ayırabilir. Ayrıştırma rezervasyonu +**bir bayta** indirse bile kalan kapasite 512 MiB'dan azdır. Sonraki tam pay, yeterli kapasite +serbest bırakılana kadar bekler. Dolayısıyla varsayılan, istek başına 512 MiB sınırındaki +ilk eşzamanlılık sorununu **çözmez**. + +`maxConcurrentInboundBodyBytes` değerini açıkça `2147483648` yapmak **2 GiB** seçer ve +ayrıştırma ile ölçüme dayalı azaltmalardan önce **her biri tam 512 MiB ayıran en fazla +4 yüklemeye** izin verir. Bu 2 GiB bütçesi ayrıştırma sonrasında sabit 4 istek sınırı değildir: +örneğin azalan rezervasyonlar sonraki kabullere yer açtığında, paylaşılan bütçe ve diğer +sınırlar izin verdiği sürece **8 çok küçük istek aynı anda yanıt akışı iletebilir**. Azaltılan rezervasyon, asenkron çalışma ve sağlayıcı beklemeleri boyunca, yanıt akışının sonuna (EOF), hataya veya iptal işleminin tamamlanmasına kadar @@ -52,15 +81,17 @@ Görseller, arama ve belirteç sayımı mevcut yapılandırılabilir gövde baş sınırlarını korur. İç doğrudan çeviri ve kombo çağrıları ikinci bir HTTP alanı ayırmaz. Yönetim, ses, bağlam geçmişi ve WebSocket sınırları değişmez. -Atlanan/sıfır ayarı veya 256 MiB ya da altındaki etkin sınır, bu ek -eşzamanlılık kapısını etkinleştirmez. Mevcut istek sayısı ve diğer kaynak -sınırları yine uygulanır. +Bu ek eşzamanlılık kapısı yalnızca etkin `maxInboundBodyBytes` 256 MiB'ı aştığında uygulanır. +İstek başına ayarı atlamak veya `0` yapmak, 256 MiB varsayılanını korur ve kapıyı kapalı tutar. +`maxConcurrentInboundBodyBytes` ayarını atlamak veya sıfırlamak ise 512 MiB paylaşılan +varsayılanını seçer; zaten etkin olan kapıyı devre dışı bırakmaz. Mevcut istek sayısı ve +diğer kaynak sınırları yine uygulanır. ## Geçici ret ile büyük gövde arasındaki fark Payın tamamı geçici olarak ayrılamıyorsa istek, **en fazla 32 bekleyen istek tutan FIFO kuyruğuna** girebilir; **bekleme sınırı 120 saniyedir**. Bekleme -sırasında gövdeler okunmaz. Ölçümle rezervasyon azalması veya son serbest +sırasında uygulama istek gövdelerini okumaz. Ölçümle rezervasyon azalması veya son serbest bırakma yer açtığında istekler kuyruk sırasıyla kabul edilir. Beklerken durdurulan istek kuyruktan çıkarılır ve okunmamış gövdesi tüketilmeden iptal edilir. @@ -82,19 +113,17 @@ Bildirilen büyük boyut, meşgul ret yerine bu yolu korur. ## Bütçe neyi ölçer? -**2 GiB** bütçesi kabul birimlerini sayar: ayrıştırma öncesinde tam payları, -sonrasında ölçülen rezervasyonları. **Süreç RSS'sinin veya ayrıştırılmış nesne -yığınının sınırı değildir** ve bellek kullanımının 2 GiB altında kalacağını -garanti etmez. Çözümleme, dizeler, nesne grafikleri, istek kopyaları ve diğer -uygulama durumu ek bellek kullanır. - -Paylaşılan bütçenin 512 MiB'dan 2 GiB'a çıkarılması toplam rezervasyon payını -dört katına çıkarır. Sınırı 256 MiB'ın üzerine çıkarmayı seçen kurulumlar, -her biri tam 512 MiB ayıran bir yükleme yerine artık en fazla 4 yüklemeyi kabul -edebilir; dolayısıyla olası en yüksek bellek kullanımı artar. Ayrıştırılmış -nesneler ve kopyalar gerçek bellek kullanımını kabul bütçesinin üzerine -çıkarabilir. Varsayılan ve daha küçük sınırlar bu kapıyı etkinleştirmez ve -mevcut davranışlarını korur. +Yapılandırılan bütçe kabul birimlerini sayar: ayrıştırma öncesinde tam payları, +sonrasında ölçülen rezervasyonları. **2 GiB veya ayarlanan miktar önceden ayrılmaz**; +bu bütçe **süreç RSS'sinin veya ayrıştırılmış nesne yığınının sınırı değildir**. +Çözümleme, dizeler, nesne grafikleri, istek kopyaları ve diğer uygulama durumu ek bellek +kullandığından gerçek bellek kullanımı bütçeyi aşabilir. + +512 MiB paylaşılan varsayılanı yerine 2 GiB seçmek toplam rezervasyon payını dört katına +çıkarır. İstek başına 512 MiB sınırında bu açık tercih, ayrıştırma öncesinde bir yerine +en fazla dört tam yüklemeyi kabul eder ve olası en yüksek bellek kullanımını artırır. +Bütçeyi kurulumunuzun bellek payına göre seçin. İstek başına sınır 256 MiB veya altındaysa +paylaşılan bütçe ayarı ne olursa olsun bu ek kapı kapalı kalır. Mümkünse istek başına varsayılan sınırı koruyun. Büyük geçmiş gerektiren bir iş akışı için artırırken olası bellek tepe değerinin büyümesini ve sınırlı diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md index bbe40d89bbc..1cff2b9560f 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md @@ -62,6 +62,8 @@ ocx models provider openrouter on | `codexPinnedTransientPolicy?` | `"hold" \| "detour-new-threads"` | `"hold"` | 手动固定的账号降级时，`hold` 继续使用并记录警告；`detour-new-threads` 只把新线程放到其他账号。不会改派进行中的请求，也不会在已发送后重发。 | | `codexWsReuseAcrossTurns?` | `boolean` | `false` | 可选：同一账号和线程的 Codex WebSocket 跨回合复用。开启后忙套接字最多等待 750ms，模型或服务层级变化时最多保留 2 个套接字，回合状态头只出现在每条消息上。 | | `upstreamHostCircuitThreshold?` | `number` | `0` | 原生 OpenAI forward Responses 与 compact 发送的可选断路器阈值，仅统计已证明的连接前 DNS/TCP 故障。`0` 表示禁用；`1`–`20` 表示在这么多个终止逻辑请求失败后，对 provider-origin 冷却 30 秒。断路期间会在账户选择和上游发送之前返回带 `Retry-After` 的 `503`；冷却结束后只允许一个半开请求。超时和 HTTP 响应不计数，任意 HTTP 响应都会关闭断路器。 仅适用于未固定账户的 Codex Pool 路由；在 `codexAccountMode: "direct"` 或使用账户限定选择器时不会启用。 | +| `maxInboundBodyBytes?` | `number` | `0` | 单个入站 JSON 请求体解码后大小的字节上限。省略或设为 `0` 会保留 256 MiB 默认值；配置值限制在 1 至 512 MiB。修改后需要重启。超限会返回本地 HTTP 413，错误代码为 `inbound_body_too_large`。生效上限大于 256 MiB 时，还会启用 `maxConcurrentInboundBodyBytes` 共享门槛；仅提高单请求上限不会提高共享预算。受约束的路径与详情见[入站请求体准入](/zh-cn/reference/inbound-body-admission/)。 | +| `maxConcurrentInboundBodyBytes?` | `number` | `536870912` (512 MiB) | 可选的顶层共享准入预算，单位为字节，仅在生效的 `maxInboundBodyBytes` 大于 256 MiB 时启用。省略、`0` 或无效值均使用不变的 512 MiB 默认值。有效的正整数在使用时限制在 512 MiB 至 2 GiB（`536870912` 至 `2147483648`）范围内。修改后需要重启；同一进程内的所有监听器共享一个预算。这是准入计量，并非预先分配的内存、RSS 上限或按 RAM 自动调整的容量。2 GiB 必须由运营者明确选择。配置示例、队列及响应生命周期的取舍，以及默认预算为何在解析前只允许一个完整 512 MiB 上传，见[入站请求体准入](/zh-cn/reference/inbound-body-admission/)。 | | `modelCacheTtlMs?` | `number` | `300000` | 每个提供者 `/models` 缓存的新鲜度窗口。 | | `cacheRetention?` | `"none" \| "short" \| "long"` | Anthropic 官方端点：`"long"`；其他 Anthropic 兼容端点：`"short"` | Anthropic 提示缓存策略：禁用、5 分钟临时缓存，或 1 小时扩展缓存。未设置时，发往 `api.anthropic.com`（API 密钥或 OAuth 账号池）的请求使用 1 小时 TTL，兼容网关和厂商端点保持 5 分钟 TTL。显式设置的值始终优先。 | | `tokenGuardian?` | `OcxTokenGuardianConfig` | 关闭 | 可选的主动 OAuth 刷新与 Codex 账户预热策略。 | diff --git a/docs-site/src/content/docs/zh-cn/reference/inbound-body-admission.md b/docs-site/src/content/docs/zh-cn/reference/inbound-body-admission.md index 68735e6117e..ba5cb27d8bb 100644 --- a/docs-site/src/content/docs/zh-cn/reference/inbound-body-admission.md +++ b/docs-site/src/content/docs/zh-cn/reference/inbound-body-admission.md @@ -5,25 +5,44 @@ description: 提高请求体上限如何影响并发 HTTP 请求、重试和内 [提供商配置参考](/zh-cn/reference/configuration/providers/)介绍了 `maxInboundBodyBytes`，即单个入站 JSON 请求体解码后的最大大小。默认值仍为 256 MiB，可配置范围仍为 1 MiB 至 512 MiB。修改上限后请重启代理，使监听器与读取器共同使用预期上限。 +保留原有的 512 MiB 共享预算默认值，并不会消除下文所述的排队和响应生命周期内保留预留量所带来的取舍。 + +## 明确选择共享预算 + +`maxConcurrentInboundBodyBytes` 是 `config.json` 中可选的**顶层**设置，单位为字节。默认值仍为 **512 MiB（`536870912`）**，保留原有的共享预算。省略、设为 `0` 或使用无效值时，均采用此默认值。有效的正整数在使用时会被限制在 **512 MiB 至 2 GiB（`536870912` 至 `2147483648`）** 范围内。修改后需要重启代理。同一进程内的所有监听器共享一个预算；它与 `appOwnedMemoryBudgetMb` 分开计算。 + +若要选择 512 MiB 的单请求上限，并明确启用 2 GiB 的共享预算，请将以下字段合并到现有 `config.json` 的顶层，然后重启： + +```json +{ + "maxInboundBodyBytes": 536870912, + "maxConcurrentInboundBodyBytes": 2147483648 +} +``` + +这是运营者主动做出的选择，会提高潜在的峰值内存用量。代理不会自动检测可用 RAM，也不会根据 RAM 自动选择预算。 + ## 提高上限会改变并发能力 -当最终生效的上限大于 256 MiB 时，每个受此规则约束的 HTTP 请求都必须在**读取任何请求体数据之前**预留其完整的单请求额度。进程共享的准入预算已从 512 MiB 增至 **2 GiB**；同一进程内的多个监听器也共享此预算。该预算与 `appOwnedMemoryBudgetMb` 分开计算。 +当最终生效的上限大于 256 MiB 时，每个受此规则约束的 HTTP 请求都必须在**读取任何请求体数据之前**预留其完整的单请求额度。这些预留量共享进程级的 `maxConcurrentInboundBodyBytes` 预算，其默认值仍为 **512 MiB**。 在完成有大小限制的 JSON 解析前，较小的 `Content-Length` 或较小的压缩传输体都不会减少预留量。准入控制不会将它们视为解码后或规范化 JSON 大小的可靠依据。 在大小限制内成功解析 JSON 后，读取器会将预留量降至实际传输字节数与解码后字节数中的较大值。如果转换器提供了用于测量规范化 JSON 的预算，还会将现有的规范化 JSON 测量值纳入比较，取最大值。预留量只会减少，且始终不超过最初的单请求额度；即使规范化测量值更大，也不会增加预留量。现有的 UTF-8 与 JSON 重新序列化检查保持不变；输入字节长度不会替代可能更大的规范化 JSON 大小。 -单请求上限为 512 MiB 时，在测量并减少预留量之前，共享预算最多可接纳 **4 个各自预留完整 512 MiB 的上传请求**。这不意味着解析后也固定只能并发 4 个请求。例如，先解析的请求减少预留量、为后续准入腾出空间后，在共享预算和其他限制允许的范围内，**8 个很小的请求也可以同时流式返回响应**。 +使用**默认的 512 MiB 共享预算**和 **512 MiB 单请求上限**时，只能有 **1 个尚未解析的上传请求**预留完整额度。即使解析后预留量降至 **1 字节**，剩余容量也不足 512 MiB，下一个完整额度仍需等待足够容量释放。因此，默认值**不能解决**单请求上限为 512 MiB 时原有的并发问题。 + +明确将 `maxConcurrentInboundBodyBytes` 设为 `2147483648` 才会选择 **2 GiB**，可在解析及按测量值减少预留量前，接纳最多 **4 个各自预留完整 512 MiB 的上传请求**。这个 2 GiB 预算不会将解析后的并发请求数固定为 4 个。例如，先解析的请求减少预留量、为后续准入腾出空间后，在共享预算和其他限制允许的范围内，**8 个很小的请求也可以同时流式返回响应**。 减少后的预留量在异步处理及等待提供商响应期间仍会保留，直到响应流 EOF、发生错误或取消处理完成。仅完成解析不会释放预留量。客户端断开连接时，也要等待响应生成端的取消处理完成后才释放。即使错误、中止和取消同时发生，释放也只执行一次。 受约束的 POST 端点包括 `/v1/responses`、`/v1/responses/compact`、`/v1/chat/completions`、`/v1/messages`、`/v1/messages/count_tokens`、`/v1/images/generations`、`/v1/images/edits` 和 `/v1/alpha/search`。图像、搜索和 token 计数仍遵守各自现有的可配置单请求体上限。内部直接转换和 combo 调用不会再次预留 HTTP 额度。管理、音频、上下文历史和 WebSocket 的上限不变。 -省略设置、设为零，或最终生效的上限不超过 256 MiB 时，不会启用这一额外的并发门槛。现有请求数量及其他资源限制仍然适用。 +这一额外的并发门槛仅在最终生效的 `maxInboundBodyBytes` 大于 256 MiB 时启用。省略该单请求设置或设为 `0` 会保留其 256 MiB 默认值，因此不会启用门槛。省略 `maxConcurrentInboundBodyBytes` 或设为 `0` 则会选择 512 MiB 共享默认值，不会关闭已启用的门槛。现有请求数量及其他资源限制仍然适用。 ## 临时拒绝与请求体过大 -暂时无法预留完整额度时，请求可进入**最多容纳 32 个等待请求的 FIFO 队列**，**最多等待 120 秒**。等待期间不会读取请求体。测量后减少预留量或最终释放腾出空间时，按等待顺序接纳请求。等待中的请求被中止后会从队列移除，未读请求体会被取消而不会被消费。 +暂时无法预留完整额度时，请求可进入**最多容纳 32 个等待请求的 FIFO 队列**，**最多等待 120 秒**。等待期间，应用不会读取请求体。测量后减少预留量或最终释放腾出空间时，按等待顺序接纳请求。等待中的请求被中止后会从队列移除，未读请求体会被取消而不会被消费。 队列已满或等待超时，会在请求体解析或提供商分发之前返回 HTTP **503**、`Retry-After: 1` 和错误代码 `server_busy`；拒绝时会取消未读请求体。Messages 与 token 计数客户端收到类型为 `overloaded_error` 的 Anthropic 格式错误；兼容 OpenAI 的客户端收到 `server_error`。认证和来源检查在准入及入队之前执行。 @@ -33,8 +52,8 @@ description: 提高请求体上限如何影响并发 HTTP 请求、重试和内 ## 预算衡量的内容 -**2 GiB** 是准入计量预算：解析前累计完整额度，解析后累计实测预留量。它**不是进程 RSS 或已解析对象堆内存的上限**，也不保证内存用量低于 2 GiB。解码、字符串、对象图、请求副本和其他应用状态还会使用额外内存。 +配置的预算用于准入计量：解析前累计完整额度，解析后累计实测预留量。它**不会预先分配 2 GiB 或配置的容量**，也**不是进程 RSS 或已解析对象堆内存的上限**。解码、字符串、对象图、请求副本和其他应用状态还会使用额外内存，因此实际内存用量可能超过预算。 -共享预算从 512 MiB 增至 2 GiB 后，可预留总量变为原来的 4 倍。主动将上限调高至 256 MiB 以上的部署，现在可接纳最多 4 个预留完整 512 MiB 的上传请求，而此前只能接纳 1 个，因此潜在的峰值内存用量会增加。已解析对象和副本可能使实际内存用量超过准入预算。默认及更小的上限不会启用此门槛，现有行为不变。 +选择 2 GiB 而非 512 MiB 共享默认值，会使可预留总量变为 4 倍。单请求上限为 512 MiB 时，这一明确选择允许在解析前接纳最多 4 个完整上传，而非 1 个，因此潜在的峰值内存用量会增加。请根据部署环境的内存余量选择预算。单请求上限不超过 256 MiB 时，无论共享预算如何设置，都不会启用这一额外门槛。 请尽可能保留默认的单请求上限。在长历史记录工作流确有需要时再主动提高，并考虑可能增大的内存峰值及等待上限。 diff --git a/docs-site/src/content/docs/zh-tw/reference/configuration/providers.md b/docs-site/src/content/docs/zh-tw/reference/configuration/providers.md index 042eeefef15..079715c71b3 100644 --- a/docs-site/src/content/docs/zh-tw/reference/configuration/providers.md +++ b/docs-site/src/content/docs/zh-tw/reference/configuration/providers.md @@ -59,6 +59,8 @@ ocx models provider openrouter on | `codexFailureWindow?` | `boolean` | `true` | 60 秒滑動失敗率，與連續計數併用。至少 20 次且失敗率 ≥25% 時帳號降級，只讓新執行緒避開；比率 ≤10% 並持續 30 秒後恢復。`false` 只使用連續計數。 | | `codexPinnedTransientPolicy?` | `"hold" \| "detour-new-threads"` | `"hold"` | 手動釘選帳號降級時，`hold` 繼續使用並記錄警告；`detour-new-threads` 只把新執行緒放到其他帳號。不會改派進行中的請求。 | | `codexWsReuseAcrossTurns?` | `boolean` | `false` | 選用：同一帳號與執行緒的 Codex WebSocket 跨回合重用。開啟後忙碌 socket 最多等待 750ms，模型或層級變更時最多保留 2 個 socket。 | +| `maxInboundBodyBytes?` | `number` | `0` | 單一傳入 JSON 請求主體解碼後大小的位元組上限。省略或設為 `0` 會保留 256 MiB 預設值；設定值限制在 1 至 512 MiB。變更後必須重新啟動。超限會回傳本機 HTTP 413，錯誤碼為 `inbound_body_too_large`。生效上限超過 256 MiB 時，也會啟用 `maxConcurrentInboundBodyBytes` 共用閘門；僅提高單次請求上限不會提高共用預算。涵蓋的路徑與詳情請見[傳入主體准入](/zh-tw/reference/inbound-body-admission/)。 | +| `maxConcurrentInboundBodyBytes?` | `number` | `536870912` (512 MiB) | 選用的頂層共用准入預算，單位為位元組，只在生效的 `maxInboundBodyBytes` 超過 256 MiB 時啟用。省略、`0` 或無效值都會使用不變的 512 MiB 預設值。有效的正整數在使用時限制在 512 MiB 至 2 GiB（`536870912` 至 `2147483648`）範圍內。變更後必須重新啟動；同一程序內的所有監聽器共用一個預算。這是准入計量，並非預先配置的記憶體、RSS 上限或依 RAM 自動調整的容量。2 GiB 必須由操作者明確選擇。設定範例、佇列及回應生命週期的取捨，以及預設預算為何在解析前只允許一個完整 512 MiB 上傳，請見[傳入主體准入](/zh-tw/reference/inbound-body-admission/)。 | | `modelCacheTtlMs?` | `number` | `300000` | Per-供應商 `/models` 快取的新鮮度視窗。 | | `cacheRetention?` | `"none" \| "short" \| "long"` | Anthropic 官方端點：`"long"`；其他 Anthropic 相容端點：`"short"` | Anthropic prompt-cache 政策：停用、5 分鐘臨時或 1 小時延長。未設定時，送往 `api.anthropic.com`（API 金鑰或 OAuth 帳號池）的請求使用 1 小時 TTL，相容閘道與廠商端點維持 5 分鐘 TTL。明確設定的值一律優先。 | | `tokenGuardian?` | `OcxTokenGuardianConfig` | off | 可選的主動 OAuth refresh 與 Codex 帳號暖機政策。 | diff --git a/docs-site/src/content/docs/zh-tw/reference/inbound-body-admission.md b/docs-site/src/content/docs/zh-tw/reference/inbound-body-admission.md index ead7e3f0ed1..9c526b48664 100644 --- a/docs-site/src/content/docs/zh-tw/reference/inbound-body-admission.md +++ b/docs-site/src/content/docs/zh-tw/reference/inbound-body-admission.md @@ -5,15 +5,34 @@ description: 提高請求主體上限如何影響並行 HTTP 請求、重試與 [供應商設定參考](/zh-tw/reference/configuration/providers/) 說明了 `maxInboundBodyBytes`，也就是單一傳入 JSON 主體的最大解碼大小。預設值仍為 256 MiB，可設定的範圍仍是 1 MiB 至 512 MiB。變更上限後請重新啟動代理，讓監聽器和讀取器一同採用預期的上限。 +保留原有的 512 MiB 共用預算預設值，並不會消除下文所述的排隊和回應生命週期內保留預留量所帶來的取捨。 + +## 明確選擇共用預算 + +`maxConcurrentInboundBodyBytes` 是 `config.json` 中選用的**頂層**設定，單位為位元組。預設值仍為 **512 MiB（`536870912`）**，保留原有的共用預算。省略、設為 `0` 或使用無效值時，都會採用此預設值。有效的正整數在使用時會被限制在 **512 MiB 至 2 GiB（`536870912` 至 `2147483648`）** 範圍內。變更後必須重新啟動代理。同一程序內的所有監聽器共用一個預算；它與 `appOwnedMemoryBudgetMb` 分開計算。 + +若要選擇 512 MiB 的單次請求上限，並明確啟用 2 GiB 的共用預算，請將以下欄位合併至現有 `config.json` 的頂層，然後重新啟動： + +```json +{ + "maxInboundBodyBytes": 536870912, + "maxConcurrentInboundBodyBytes": 2147483648 +} +``` + +這是操作者主動做出的選擇，會提高潛在的尖峰記憶體用量。代理不會自動偵測可用 RAM，也不會根據 RAM 自動選擇預算。 + ## 提高上限會改變並行能力 -當實際生效的上限超過 256 MiB，每個涵蓋的 HTTP 請求都必須在**讀取任何主體資料之前**，預留完整的單次請求額度。程序共用的准入預算已從 512 MiB 增至 **2 GiB**；同一程序中的多個監聽器也共用此預算。這項預算與 `appOwnedMemoryBudgetMb` 分開計算。 +當實際生效的上限超過 256 MiB，每個涵蓋的 HTTP 請求都必須在**讀取任何主體資料之前**，預留完整的單次請求額度。這些預留量共用程序層級的 `maxConcurrentInboundBodyBytes` 預算，其預設值仍為 **512 MiB**。 在完成有大小限制的 JSON 解析前，即使 `Content-Length` 或壓縮後的傳輸主體很小，也不會減少預留量。准入控制不會將這些值當成解碼後或正規化 JSON 大小的可靠依據。 在大小限制內成功解析 JSON 後，讀取器會將預留量降至實測傳輸位元組數與解碼後位元組數中的較大值。若轉譯器提供用於測量正規化 JSON 的預算，還會將既有的正規化 JSON 測量值納入比較，採用最大值。預留量只會減少，且始終不超過最初的單次請求額度；即使正規化測量值更大，也不會增加預留量。既有的 UTF-8 與 JSON 重新序列化檢查會保留；輸入位元組長度不會取代可能更大的正規化 JSON 大小。 -單次請求上限為 512 MiB 時，在測量並減少預留量之前，共用預算最多可接受 **4 個各自預留完整 512 MiB 的上傳請求**。解析後的並行請求數並未固定為 4 個。例如，先完成解析的請求減少預留量，讓後續請求能夠進入後，在共用預算與其他限制允許的範圍內，**8 個很小的請求也可以同時串流回傳回應**。 +使用**預設的 512 MiB 共用預算**和 **512 MiB 單次請求上限**時，只能有 **1 個尚未解析的上傳請求**預留完整額度。即使解析後預留量降至 **1 位元組**，剩餘容量也不足 512 MiB，下一個完整額度仍須等候足夠容量釋放。因此，預設值**無法解決**單次請求上限為 512 MiB 時原有的並行問題。 + +明確將 `maxConcurrentInboundBodyBytes` 設為 `2147483648` 才會選擇 **2 GiB**，可在解析及依測量值減少預留量前，接受最多 **4 個各自預留完整 512 MiB 的上傳請求**。這個 2 GiB 預算不會將解析後的並行請求數固定為 4 個。例如，先完成解析的請求減少預留量，讓後續請求能夠進入後，在共用預算與其他限制允許的範圍內，**8 個很小的請求也可以同時串流回傳回應**。 減少後的預留量會在非同步工作及等候供應商回應期間保留，直到回應串流 EOF、發生錯誤或取消處理完成才釋放。僅完成解析不會釋放預留量。用戶端中斷連線時，也會等到回應產生端的取消處理完成才釋放。即使錯誤、中止和取消同時發生，也只會釋放一次。 @@ -22,11 +41,11 @@ description: 提高請求主體上限如何影響並行 HTTP 請求、重試與 `/v1/images/generations`、`/v1/images/edits` 與 `/v1/alpha/search`。 圖片、搜尋與 token 計數仍沿用各自可設定的單一主體上限。內部直接轉譯與 combo 呼叫不會再次預留 HTTP 額度。管理、音訊、上下文歷史及 WebSocket 的上限不變。 -省略設定、設為零，或實際生效的上限不超過 256 MiB，都不會啟用這道額外的並行閘門。現有的請求數及其他資源限制仍然適用。 +這道額外的並行閘門只在實際生效的 `maxInboundBodyBytes` 超過 256 MiB 時啟用。省略該單次請求設定或設為 `0`，會保留其 256 MiB 預設值，因此不會啟用閘門。省略 `maxConcurrentInboundBodyBytes` 或設為 `0`，則會選擇 512 MiB 共用預設值，不會關閉已啟用的閘門。現有的請求數及其他資源限制仍然適用。 ## 暫時拒絕與主體過大的差別 -暫時無法預留完整額度時，請求可進入**最多容納 32 個等候請求的 FIFO 佇列**，**最多等候 120 秒**。等候期間不會讀取主體。測量後減少預留量或最終釋放騰出空間時，會依等候順序接受請求。等候中的請求遭中止後，會從佇列移除，未讀取的主體會被取消，不會被消費。 +暫時無法預留完整額度時，請求可進入**最多容納 32 個等候請求的 FIFO 佇列**，**最多等候 120 秒**。等候期間，應用程式不會讀取請求主體。測量後減少預留量或最終釋放騰出空間時，會依等候順序接受請求。等候中的請求遭中止後，會從佇列移除，未讀取的主體會被取消，不會被消費。 佇列已滿或等候逾時時，系統會在主體解析或供應商派送前回傳 HTTP **503**、`Retry-After: 1` 與錯誤碼 `server_busy`；拒絕時會取消未讀取的主體。Messages 與 token 計數用戶端會收到類型為 `overloaded_error` 的 Anthropic 格式錯誤；OpenAI 相容用戶端則會收到 `server_error`。身分驗證與 origin 檢查會在准入及加入佇列之前執行。 @@ -36,8 +55,8 @@ description: 提高請求主體上限如何影響並行 HTTP 請求、重試與 ## 預算衡量的是什麼 -**2 GiB** 是准入計量預算：解析前加總完整額度，解析後加總實測預留量。它**不是程序 RSS 或已解析物件堆積的上限**，也不保證記憶體用量低於 2 GiB。解碼、字串、物件圖、請求副本與其他應用程式狀態都會使用額外記憶體。 +設定的預算用於准入計量：解析前加總完整額度，解析後加總實測預留量。它**不會預先配置 2 GiB 或設定的容量**，也**不是程序 RSS 或已解析物件堆積的上限**。解碼、字串、物件圖、請求副本與其他應用程式狀態都會使用額外記憶體，因此實際記憶體用量可能超過預算。 -共用預算從 512 MiB 增至 2 GiB 後，可預留總量變為原本的 4 倍。主動將上限提高至 256 MiB 以上的部署環境，現在最多可接受 4 個預留完整 512 MiB 的上傳請求，先前則只能接受 1 個，因此潛在的尖峰記憶體用量會增加。已解析物件及副本可能讓實際記憶體用量超過准入預算。預設及更小的上限不會啟用這道閘門，既有行為不變。 +選擇 2 GiB 而非 512 MiB 共用預設值，會讓可預留總量變為 4 倍。單次請求上限為 512 MiB 時，這項明確選擇允許在解析前接受最多 4 個完整上傳，而非 1 個，因此潛在的尖峰記憶體用量會增加。請根據部署環境的記憶體餘裕選擇預算。單次請求上限不超過 256 MiB 時，無論共用預算如何設定，都不會啟用這道額外閘門。 請盡可能維持預設的單次請求上限。長歷史記錄工作流程確有需要時再主動提高，並考慮可能增加的記憶體尖峰用量與等候上限。 diff --git a/scripts/test-layout/layout-server-inbound.json b/scripts/test-layout/layout-server-inbound.json index 022ec21c211..f38f6530ecd 100644 --- a/scripts/test-layout/layout-server-inbound.json +++ b/scripts/test-layout/layout-server-inbound.json @@ -1,4 +1,5 @@ { + "config-concurrent-inbound-body.test.ts": "config", "inbound-body-admission.test.ts": "server", "inbound-body-budget.test.ts": "server" } diff --git a/src/config/schema/config-schema.ts b/src/config/schema/config-schema.ts index d4a561836f3..8962ef27d87 100644 --- a/src/config/schema/config-schema.ts +++ b/src/config/schema/config-schema.ts @@ -177,6 +177,13 @@ export const configSchema = z.object({ .min(0) .optional() .catch(undefined), + // Match the per-body load/degrade contract: malformed hand edits retain providers. + // The shared resolver clamps runtime admission to 512 MiB..2 GiB even when the + // caller bypasses this schema; omission and zero retain the 512 MiB default. + maxConcurrentInboundBodyBytes: z.number().int() + .min(0) + .optional() + .catch(undefined), appOwnedMemoryBudgetMb: z.number().int() .min(MIN_APP_OWNED_MEMORY_BUDGET_MB) .max(MAX_APP_OWNED_MEMORY_BUDGET_MB) diff --git a/src/lib/inbound-body-policy.ts b/src/lib/inbound-body-policy.ts new file mode 100644 index 00000000000..2151b58609e --- /dev/null +++ b/src/lib/inbound-body-policy.ts @@ -0,0 +1,11 @@ +/** Admission accounting for raised HTTP bodies; these values do not cap RSS. */ +export const DEFAULT_CONCURRENT_INBOUND_BODY_BYTES = 512 * 1024 * 1024; +export const MAX_CONCURRENT_INBOUND_BODY_BYTES = 2 * 1024 * 1024 * 1024; + +/** Keep hand edits bounded; omission, zero and invalid values retain the old default. */ +export function resolveConcurrentInboundBodyLimitBytes(configured?: number): number { + if (typeof configured !== "number" || !Number.isSafeInteger(configured) || configured <= 0) { + return DEFAULT_CONCURRENT_INBOUND_BODY_BYTES; + } + return Math.min(MAX_CONCURRENT_INBOUND_BODY_BYTES, Math.max(DEFAULT_CONCURRENT_INBOUND_BODY_BYTES, configured)); +} diff --git a/src/server/inbound-body-admission.ts b/src/server/inbound-body-admission.ts index 66e25953da9..5dc0bc075b8 100644 --- a/src/server/inbound-body-admission.ts +++ b/src/server/inbound-body-admission.ts @@ -8,7 +8,7 @@ import { addFinalRequestLog, markLocalRequestLogRefusal } from "./request-log"; import type { WorkflowRefusalLog } from "./workflow-refusal"; import { inboundBodyBudget, bindInboundBodyLease, type InboundBodyBudget } from "./inbound-body-budget"; -export { MAX_CONCURRENT_INBOUND_BODY_BYTES } from "./inbound-body-budget"; +export { configureInboundBodyBudget, MAX_CONCURRENT_INBOUND_BODY_BYTES } from "./inbound-body-budget"; // Match the HTTP routes whose JSON readers use maxInboundBodyBytes. Keep the // current image/search/count_tokens support; management, audio, context relay, diff --git a/src/server/inbound-body-budget.ts b/src/server/inbound-body-budget.ts index 7cc72d7250e..a9e808e81ee 100644 --- a/src/server/inbound-body-budget.ts +++ b/src/server/inbound-body-budget.ts @@ -1,5 +1,5 @@ -/** Admission units for raised HTTP bodies, not a process RSS or parsed-heap limit. */ -export const MAX_CONCURRENT_INBOUND_BODY_BYTES = 2 * 1024 * 1024 * 1024; +import { DEFAULT_CONCURRENT_INBOUND_BODY_BYTES, resolveConcurrentInboundBodyLimitBytes } from "../lib/inbound-body-policy"; +export { DEFAULT_CONCURRENT_INBOUND_BODY_BYTES, MAX_CONCURRENT_INBOUND_BODY_BYTES } from "../lib/inbound-body-policy"; export const MAX_QUEUED_INBOUND_BODIES = 32; export const INBOUND_BODY_QUEUE_TIMEOUT_MS = 120_000; @@ -49,6 +49,14 @@ export function createInboundBodyBudget(capacity: number, maxQueued = MAX_QUEUED }; }; return { + /** Report the current shared allowance without confusing it with measured charges. */ + capacity: () => capacity, + /** Startup configuration changes the same owner; existing leases and FIFO waiters survive. */ + setCapacity(bytes: number): void { + if (!Number.isSafeInteger(bytes) || bytes <= 0) throw new RangeError("Invalid inbound body capacity"); + capacity = bytes; + drain(); + }, /** Observe counters without exposing request bodies or identities. */ snapshot: () => ({ reservedBytes, activeLeases, queuedRequests: pending.length }), /** Wait for an entire upload allowance; null preserves the caller's overload response. */ @@ -100,9 +108,14 @@ export function createInboundBodyBudget(capacity: number, maxQueued = MAX_QUEUED } export type InboundBodyBudget = ReturnType; -export const inboundBodyBudget = createInboundBodyBudget(MAX_CONCURRENT_INBOUND_BODY_BYTES); +export const inboundBodyBudget = createInboundBodyBudget(DEFAULT_CONCURRENT_INBOUND_BODY_BYTES); const requestLeases = new WeakMap(); +/** Adopt one startup policy for every listener without resetting live accounting. */ +export function configureInboundBodyBudget(configured?: number): void { + inboundBodyBudget.setCapacity(resolveConcurrentInboundBodyLimitBytes(configured)); +} + /** Connect only the admitted HTTP request to its bounded reader; internal reads own no lease. */ export function bindInboundBodyLease(req: Request, lease: InboundBodyLease): () => void { requestLeases.set(req, lease); diff --git a/src/server/index.ts b/src/server/index.ts index 0850f3c1761..b574593beac 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -1,7 +1,7 @@ import { remoteWorkspaceEnabled } from "../remote-control/workspace-activation"; import { scheduleDesktopCompatibilityStartup } from "./index/desktop-compatibility-startup"; import { AuxiliaryListenerBindError } from "./ports"; -import { runAdmittedBodyWork } from "./inbound-body-admission"; +import { configureInboundBodyBudget, runAdmittedBodyWork } from "./inbound-body-admission"; import { buildWarmupCompletionFrames, buildWsErrorFrame, @@ -696,7 +696,6 @@ function startServerWithSpendLedgerOwner(port: number | undefined, deps: StartSe }); server = spendLedgerLifecycle.track(Bun.serve({ ...serveOptions, port: listenPort, hostname: bindHost })); - // Both binds are one startup transaction (#1102). If the loopback bind fails after the // public one succeeded, leaving the public listener up would strand it: the CLI's port // retry would read the failure as a public-port conflict and pick a different port, @@ -738,6 +737,7 @@ function startServerWithSpendLedgerOwner(port: number | undefined, deps: StartSe throw new AuxiliaryListenerBindError("hub.managementIngress", managementIngressPort, "127.0.0.1", error); } } + configureInboundBodyBudget(config.maxConcurrentInboundBodyBytes); optionalListeners.start({ config, publicPort: server.port ?? listenPort, requestedPort: listenPort, startProxy: deps.startClaudeInterceptProxy, maxRequestBodySize: inboundBodyLimitBytes, dispatch: (req, requestServer) => serveOptions.fetch(req, requestServer) }); } catch (error) { diff --git a/src/types/config.ts b/src/types/config.ts index 276e876e81e..3dd8e289f6f 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -1323,6 +1323,13 @@ export interface OcxConfig { * server starts, so raising this takes effect on restart. */ maxInboundBodyBytes?: number; + /** + * Aggregate admission budget, in bytes, for concurrent raised-limit inbound HTTP bodies. + * Omitted or 0 = the built-in 512 MiB budget. `resolveConcurrentInboundBodyLimitBytes()` + * clamps to [512 MiB, 2 GiB]; there is no disabled or unbounded mode. Changes take effect + * on proxy restart. This accounts for admitted bodies, not total process memory. + */ + maxConcurrentInboundBodyBytes?: number; /** * Opt-in Anthropic OAuth PROACTIVE routing (#294). Default OFF. * Sticky session affinity; new sessions may pick lowest known 5h usage. diff --git a/structure/config.md b/structure/config.md index 604f1c839b7..4a4bd54666c 100644 --- a/structure/config.md +++ b/structure/config.md @@ -95,7 +95,7 @@ A schema-invalid top-level JSON value is repairable only when it is a non-array | Group | Keys | Resolution rule | | --- | --- | --- | -| Listener | `port`, `hostname` | The listener owns the port; `runtime-port.json` reports where it actually landed. | +| Listener | `port`, `hostname`, `maxConcurrentInboundBodyBytes` | The listener owns the port; `runtime-port.json` reports where it actually landed. Shared raised-body admission follows the [byte-accounting contract](transports/byte-accounting.md#raised-http-body-admission): the budget defaults to 512 MiB, clamps to 512 MiB..2 GiB, and changes on restart. Its schema and candidate recovery match `maxInboundBodyBytes`, preserving providers on malformed hand edits. | | Routing | `defaultProvider`, `providers`, per-provider `selectedModels`, `combos` | Explicit `provider/model` wins over `defaultProvider`; combo dispatch uses the selected target's existing capability ladder and does not create a second catalog authority. For Kiro OAuth, the management API validates `providers.kiro.oauthAccountFailover.strategy` (`least-loaded`) and `maxConcurrentPerAccount` (1–100) only for Kiro; the cap persists, while active lease counts remain process-local. | | Request pacing | `providers..requestPacing`, `requestPacing.models.` | Optional client-side request-start pacing supports intervals and positive-integer concurrency caps. Provider, exact-model, and uniform per-credential-reference `keys` rules compose atomically. `keySelection` defaults to `active`; opt-in `round-robin` assigns healthy pool keys per request without persisting the active choice. Project quota, TPM and daily limits are not inferred. | | Compaction and memory routing | `compactionRouting.model`, optional `compactionRouting.reasoningEffort`, optional `compactionRouting.triggers`, optional `compactionRouting.sourceModels`; `memoryModels.extract`, `memoryModels.consolidation` | Explicit Codex compaction metadata whose `compaction.trigger` is one the block names activates a request-local override; `triggers` defaults to `["manual"]`. Optional source selectors restrict the override to exact incoming models or `provider/*`. See [Responses compaction](transports/responses-failover.md#compaction-routing-overrides). Invalid hand edits disable the block with a load warning without discarding providers; candidate writes reject invalid blocks. Each optional memory phase names a nonblank model and optional declared `reasoningEffort`; absence preserves the current route, candidate writes reject malformed values, and load degrades only the invalid phase with a warning. See [memory phase routing](transports/responses-failover.md#memory-phase-routing). | diff --git a/structure/transports/byte-accounting.md b/structure/transports/byte-accounting.md index 350c25eb3be..82c328b0ecb 100644 --- a/structure/transports/byte-accounting.md +++ b/structure/transports/byte-accounting.md @@ -95,7 +95,12 @@ still measure the uncompressed body; native codec workspace is not a counted Jav ## Raised HTTP body admission `src/server/inbound-body-admission.ts` reserves the full resolved `maxInboundBodyBytes` allowance -from the process-wide 2 GiB budget in `src/server/inbound-body-budget.ts` when it exceeds the 256 MiB default. +from the process-wide budget in `src/server/inbound-body-budget.ts` when it exceeds the 256 MiB default. +`maxConcurrentInboundBodyBytes` preserves a 512 MiB default; an operator may explicitly select +512 MiB through 2 GiB. `src/lib/inbound-body-policy.ts` resolves omission, zero or invalid numbers to +512 MiB and clamps positive safe integers to that range. Startup captures the setting for all +listeners; a config edit requires restart. Configuration changes the existing owner without +discarding active leases or FIFO waiters; lowering capacity cannot erase outstanding charges. `src/server/index.ts` owns this lease in `runAdmittedHttpTurn`, after authentication and origin checks, before any request body is read. Default and smaller allowances do not consume this budget. Missing or small Content-Length and compressed wire bodies do not reduce the upload reservation. @@ -124,9 +129,11 @@ protocol handlers, with HTTP 503, `Retry-After: 1`, and code `server_busy`; Messages/count_tokens use the Anthropic `error`/`overloaded_error` envelope. The HTTP owner preserves receiving-listener CORS and records the refusal without reading the body. -This is an allowance budget, not a measured RSS or parsed-heap cap. Raising its ceiling from 512 MiB -to 2 GiB permits four full 512 MiB uploads and increases potential peak memory on raised-limit -deployments. Parsed small requests can overlap beyond four while their measured charges fit. +This is an allowance budget, not a measured RSS or parsed-heap cap. Explicitly selecting 2 GiB +permits four full 512 MiB uploads and increases potential peak memory on raised-limit deployments. +With the default 512 MiB budget, even a one-byte charge prevents another full 512 MiB allowance; +shrinking alone does not resolve that case. Parsed small requests with an explicitly larger +budget can overlap beyond four while their measured charges fit. Host RAM is not auto-detected. It does not bound state beyond the HTTP lifetime or change default-cap concurrency. `tests/server/inbound-body-admission.test.ts` covers lifecycle, cancellation races, protocol envelopes, and real HTTP streams; `tests/server/inbound-body-budget.test.ts` covers FIFO/accounting. diff --git a/tests/config/config-concurrent-inbound-body.test.ts b/tests/config/config-concurrent-inbound-body.test.ts new file mode 100644 index 00000000000..804a3c35a12 --- /dev/null +++ b/tests/config/config-concurrent-inbound-body.test.ts @@ -0,0 +1,122 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { getConfigPath, getDefaultConfig, loadConfig, saveConfig, validateConfigCandidate } from "../../src/config"; +import { configSchema } from "../../src/config/schema/config-schema"; +import { + DEFAULT_CONCURRENT_INBOUND_BODY_BYTES, + MAX_CONCURRENT_INBOUND_BODY_BYTES, + resolveConcurrentInboundBodyLimitBytes, +} from "../../src/lib/inbound-body-policy"; +import { removeTreeAfterBackgroundWork } from "../helpers/remove-tree"; + +const MiB = 1024 * 1024; +const invalid = [-1, 1.5, "unbounded", "1073741824", true, null, {}, [], Number.MAX_SAFE_INTEGER + 1]; + +/** Keep a distinct provider so fallback to defaults cannot satisfy retention checks. */ +function configWithLimit(value?: unknown) { + return { + ...getDefaultConfig(), + defaultProvider: "fixture", + providers: { fixture: { adapter: "openai-responses", baseUrl: "https://provider.invalid/v1" } }, + maxInboundBodyBytes: 512 * MiB, + maxConcurrentInboundBodyBytes: value, + }; +} + +test("omission keeps the existing 512 MiB budget without injecting a config key", () => { + expect(DEFAULT_CONCURRENT_INBOUND_BODY_BYTES).toBe(512 * MiB); + expect(MAX_CONCURRENT_INBOUND_BODY_BYTES).toBe(2 * 1024 * MiB); + expect(getDefaultConfig().maxConcurrentInboundBodyBytes).toBeUndefined(); + const parsed = configSchema.parse(configWithLimit()); + expect(parsed.maxConcurrentInboundBodyBytes).toBeUndefined(); + expect(resolveConcurrentInboundBodyLimitBytes(parsed.maxConcurrentInboundBodyBytes)).toBe(512 * MiB); +}); + +test("schema and candidates retain integers while the config resolver enforces both bounds", () => { + for (const [value, resolved] of [ + [0, 512 * MiB], [1, 512 * MiB], [512 * MiB - 1, 512 * MiB], + [512 * MiB, 512 * MiB], [768 * MiB, 768 * MiB], + [2 * 1024 * MiB, 2 * 1024 * MiB], [2 * 1024 * MiB + 1, 2 * 1024 * MiB], + [Number.MAX_SAFE_INTEGER, 2 * 1024 * MiB], + ]) { + expect(configSchema.parse(configWithLimit(value)).maxConcurrentInboundBodyBytes).toBe(value); + const candidate = validateConfigCandidate(configWithLimit(value)); + expect(candidate.ok).toBe(true); + if (!candidate.ok) throw new Error(candidate.error); + expect(candidate.config.maxConcurrentInboundBodyBytes).toBe(value); + expect(resolveConcurrentInboundBodyLimitBytes(candidate.config.maxConcurrentInboundBodyBytes)).toBe(resolved); + } +}); + +test("malformed schema and candidate values degrade like maxInboundBodyBytes", () => { + for (const value of [...invalid, NaN, Infinity, -Infinity]) { + const raw = { ...configWithLimit(value), maxInboundBodyBytes: value }; + const parsed = configSchema.parse(raw); + expect(parsed.maxConcurrentInboundBodyBytes).toBeUndefined(); + expect(parsed.maxInboundBodyBytes).toBeUndefined(); + expect(parsed.providers).toEqual(raw.providers); + const candidate = validateConfigCandidate(raw); + expect(candidate.ok).toBe(true); + if (!candidate.ok) throw new Error(candidate.error); + expect(candidate.config.maxConcurrentInboundBodyBytes).toBeUndefined(); + expect(candidate.config.providers).toEqual(raw.providers); + } +}); + +test("the resolver keeps invalid direct callers finite even without schema validation", () => { + for (const value of [undefined, 0, -1, 1.5, NaN, Infinity, -Infinity, Number.MAX_SAFE_INTEGER + 1]) { + expect(resolveConcurrentInboundBodyLimitBytes(value)).toBe(512 * MiB); + } +}); + +describe("concurrent inbound body config persistence", () => { + let home: string; + let previousHome: string | undefined; + beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + home = mkdtempSync(join(tmpdir(), "ocx-concurrent-inbound-config-")); + process.env.OPENCODEX_HOME = home; + }); + afterEach(async () => { + try { + await removeTreeAfterBackgroundWork(home); + } finally { + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + } + }); + + test("save/load preserves zero, an explicit budget and an over-ceiling request", () => { + for (const value of [0, 768 * MiB, MAX_CONCURRENT_INBOUND_BODY_BYTES + 1]) { + const candidate = validateConfigCandidate(configWithLimit(value)); + expect(candidate.ok).toBe(true); + if (!candidate.ok) throw new Error(candidate.error); + saveConfig(candidate.config); + expect(JSON.parse(readFileSync(getConfigPath(), "utf8")).maxConcurrentInboundBodyBytes).toBe(value); + const loaded = loadConfig(); + expect(loaded.maxConcurrentInboundBodyBytes).toBe(value); + expect(loaded.providers).toEqual(candidate.config.providers); + expect(loaded.maxInboundBodyBytes).toBe(512 * MiB); + expect(resolveConcurrentInboundBodyLimitBytes(loaded.maxConcurrentInboundBodyBytes)).toBe( + value === 0 ? 512 * MiB : Math.min(value, MAX_CONCURRENT_INBOUND_BODY_BYTES), + ); + } + }, 20_000); + + test("invalid hand edits keep providers and the original file instead of resetting config", () => { + for (const value of invalid) { + const raw = configWithLimit(value); + const bytes = JSON.stringify(raw); + writeFileSync(getConfigPath(), bytes, "utf8"); + const loaded = loadConfig(); + expect(loaded.maxConcurrentInboundBodyBytes).toBeUndefined(); + expect(resolveConcurrentInboundBodyLimitBytes(loaded.maxConcurrentInboundBodyBytes)).toBe(512 * MiB); + expect(loaded.defaultProvider).toBe("fixture"); + expect(loaded.providers).toEqual(raw.providers); + expect(loaded.maxInboundBodyBytes).toBe(512 * MiB); + expect(readFileSync(getConfigPath(), "utf8")).toBe(bytes); + } + }, 20_000); +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index f674d3a891e..b7c9bf2d683 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -796,6 +796,7 @@ "initial-selection-write-fence.test.ts": "providers", "injection-model-api.test.ts": "codex-integration", "inbound-body-admission.test.ts": "server", "inbound-body-budget.test.ts": "server", + "config-concurrent-inbound-body.test.ts": "config", "inline-think-boundaries.test.ts": "adapters/openai", "input-admission.test.ts": "server", "install-scripts.test.ts": "ci-workflows", "installed-gate-drivers.test.ts": "ci-workflows", "integrations-current-store.test.ts": "clients", "integrations-dsh-profile-patch.test.ts": "clients", "integrations-hermes-affinity.test.ts": "clients", diff --git a/tests/helpers/inbound-body-budget.ts b/tests/helpers/inbound-body-budget.ts index 4547b647c33..7f862fe8e8d 100644 --- a/tests/helpers/inbound-body-budget.ts +++ b/tests/helpers/inbound-body-budget.ts @@ -1,14 +1,14 @@ import assert from "node:assert/strict"; import { inboundBodyBudget, - MAX_CONCURRENT_INBOUND_BODY_BYTES, MAX_QUEUED_INBOUND_BODIES, } from "../../src/server/inbound-body-budget"; /** Fill admission counters and its queue without allocating request-sized bodies. */ export async function saturateInboundBodyBudget(): Promise<() => Promise> { + const capacity = inboundBodyBudget.capacity(); const holder = await inboundBodyBudget.acquire( - MAX_CONCURRENT_INBOUND_BODY_BYTES, new AbortController().signal, 0, + capacity, new AbortController().signal, 0, ); assert.ok(holder, "fixture requires an idle shared inbound body budget"); const controllers: AbortController[] = []; @@ -29,7 +29,7 @@ export async function saturateInboundBodyBudget(): Promise<() => Promise> waiting.push(inboundBodyBudget.acquire(1, controller.signal).catch(error => error)); } assert.deepEqual(inboundBodyBudget.snapshot(), { - reservedBytes: MAX_CONCURRENT_INBOUND_BODY_BYTES, + reservedBytes: capacity, activeLeases: 1, queuedRequests: MAX_QUEUED_INBOUND_BODIES, }); diff --git a/tests/server/inbound-body-admission.test.ts b/tests/server/inbound-body-admission.test.ts index 1f5c32a8b2f..2e42acef4ee 100644 --- a/tests/server/inbound-body-admission.test.ts +++ b/tests/server/inbound-body-admission.test.ts @@ -20,7 +20,7 @@ import { CONFIGURABLE_JSON_BODY_ROUTES, UNGATED_LOOPBACK_ROUTES, } from "../../src/server/inbound-body-admission"; -import { createInboundBodyBudget, inboundBodyBudget } from "../../src/server/inbound-body-budget"; +import { configureInboundBodyBudget, createInboundBodyBudget, inboundBodyBudget } from "../../src/server/inbound-body-budget"; import { drainAndRemoveFixtureRoots } from "../helpers/fixture-teardown"; import { saturateInboundBodyBudget } from "../helpers/inbound-body-budget"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; @@ -667,9 +667,10 @@ describe("raised inbound body admission lifetime", () => { }); // END raised-body admission regressions. -test("the public singleton admits eight tiny parsed bodies with live responses", async () => { +test("the explicitly raised public singleton admits eight tiny parsed bodies with live responses", async () => { const empty = { reservedBytes: 0, activeLeases: 0, queuedRequests: 0 }; assert.deepEqual(inboundBodyBudget.snapshot(), empty); + configureInboundBodyBudget(4 * RAISED_BODY_LIMIT); const responses: Response[] = []; let measuredBytes = 0; let cancellations = 0; @@ -694,6 +695,7 @@ test("the public singleton admits eight tiny parsed bodies with live responses", assert.equal(responses.length, 8); } finally { await Promise.all(responses.map(response => response.body?.cancel().catch(() => undefined))); + configureInboundBodyBudget(); } assert.equal(cancellations, 8); assert.deepEqual(inboundBodyBudget.snapshot(), empty); @@ -769,6 +771,7 @@ describe("raised inbound body admission at the real HTTP boundary", () => { codexHome = installIsolatedCodexHome("ocx-inbound-body-admission-codex-"); }); afterEach(async () => { + configureInboundBodyBudget(); await drainAndRemoveFixtureRoots({ roots: [{ path: fixtureHome }, { path: codexHome.path }], restoreEnvironment() { @@ -779,6 +782,27 @@ describe("raised inbound body admission at the real HTTP boundary", () => { }); }); + for (const capacity of [undefined, 0, RAISED_BODY_LIMIT, 2 * RAISED_BODY_LIMIT, 4 * RAISED_BODY_LIMIT]) { + test(`startup captures shared capacity ${String(capacity)} until the next restart`, async () => { + saveConfig({ ...getDefaultConfig(), providers: {}, maxInboundBodyBytes: RAISED_BODY_LIMIT, + maxConcurrentInboundBodyBytes: capacity }); + const server = startServer(0); + try { + const expected = capacity || RAISED_BODY_LIMIT; + assert.equal(inboundBodyBudget.capacity(), expected); + saveConfig({ ...getDefaultConfig(), providers: {}, maxInboundBodyBytes: RAISED_BODY_LIMIT, + maxConcurrentInboundBodyBytes: capacity === 4 * RAISED_BODY_LIMIT ? RAISED_BODY_LIMIT : 4 * RAISED_BODY_LIMIT }); + assert.equal(inboundBodyBudget.capacity(), expected, "a disk edit must not replace live accounting"); + const response = await fetch(`http://127.0.0.1:${server.port}/v1/chat/completions`, { + method: "POST", headers: { "content-type": "application/json" }, body: "{}", + }); + assert.equal(response.status, 400); + await response.text(); + assert.equal(inboundBodyBudget.capacity(), expected, "handling traffic must retain startup policy"); + } finally { await server.stop(true); } + }, 30_000); + } + test("all configured JSON routes refuse before protocol parsing and recover after release", async () => { saveConfig({ ...getDefaultConfig(), providers: {}, maxInboundBodyBytes: RAISED_BODY_LIMIT }); const server = startServer(0); diff --git a/tests/server/inbound-body-budget.test.ts b/tests/server/inbound-body-budget.test.ts index d5164716ae4..1dcf1cb364d 100644 --- a/tests/server/inbound-body-budget.test.ts +++ b/tests/server/inbound-body-budget.test.ts @@ -2,6 +2,9 @@ import { describe, expect, spyOn, test } from "bun:test"; import assert from "node:assert/strict"; import { createInboundBodyBudget, + configureInboundBodyBudget, + DEFAULT_CONCURRENT_INBOUND_BODY_BYTES, + inboundBodyBudget, INBOUND_BODY_QUEUE_TIMEOUT_MS, MAX_CONCURRENT_INBOUND_BODY_BYTES, MAX_QUEUED_INBOUND_BODIES, @@ -17,11 +20,69 @@ function assertEmpty(budget: InboundBodyBudget) { describe("inbound body budget", () => { test("bounds the shared allowance, waiting uploads and wait duration", () => { + expect(DEFAULT_CONCURRENT_INBOUND_BODY_BYTES).toBe(512 * 1024 * 1024); expect(MAX_CONCURRENT_INBOUND_BODY_BYTES).toBe(2 * 1024 * 1024 * 1024); expect(MAX_QUEUED_INBOUND_BODIES).toBe(32); expect(INBOUND_BODY_QUEUE_TIMEOUT_MS).toBe(120_000); }); + test("explicit startup settings preserve the same shared owner and zero restores its default", async () => { + const owner = inboundBodyBudget; + const lease = await owner.acquire(1, freshSignal(), 0); + assert.ok(lease); + try { + configureInboundBodyBudget(MAX_CONCURRENT_INBOUND_BODY_BYTES); + assert.equal(inboundBodyBudget, owner); + assert.equal(owner.capacity(), MAX_CONCURRENT_INBOUND_BODY_BYTES); + configureInboundBodyBudget(0); + assert.equal(owner.capacity(), DEFAULT_CONCURRENT_INBOUND_BODY_BYTES); + assert.deepEqual(owner.snapshot(), { reservedBytes: 1, activeLeases: 1, queuedRequests: 0 }); + } finally { lease.release(); configureInboundBodyBudget(); } + }); + + test("raising capacity wakes FIFO waiters without erasing active charges", async () => { + const budget = createInboundBodyBudget(100); + const first = await budget.acquire(100, freshSignal()); + assert.ok(first); + const pending = budget.acquire(80, freshSignal()); + budget.setCapacity(200); + const second = await pending; + assert.ok(second); + try { + assert.deepEqual(budget.snapshot(), { reservedBytes: 180, activeLeases: 2, queuedRequests: 0 }); + } finally { first.release(); second.release(); } + assertEmpty(budget); + }); + + test("lowering capacity retains outstanding charges until the FIFO head fits", async () => { + const budget = createInboundBodyBudget(100); + const first = await budget.acquire(80, freshSignal()); + assert.ok(first); + const pending = budget.acquire(60, freshSignal()); + let second: InboundBodyLease | null = null; + try { + budget.setCapacity(60); + assert.deepEqual(budget.snapshot(), { reservedBytes: 80, activeLeases: 1, queuedRequests: 1 }); + assert.equal(await budget.acquire(1, freshSignal(), 0), null); + first.shrink(20); + assert.deepEqual(budget.snapshot(), { reservedBytes: 20, activeLeases: 1, queuedRequests: 1 }); + first.release(); + second = await pending; + assert.ok(second); + assert.deepEqual(budget.snapshot(), { reservedBytes: 60, activeLeases: 1, queuedRequests: 0 }); + } finally { first.release(); (second ?? await pending)?.release(); } + assertEmpty(budget); + }); + + test("invalid capacity changes preserve the current budget", () => { + const budget = createInboundBodyBudget(100); + for (const invalid of [0, -1, 1.5, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1]) { + assert.throws(() => budget.setCapacity(invalid), RangeError); + assert.equal(budget.capacity(), 100); + } + assertEmpty(budget); + }); + for (const capacity of [0, -1, 0.5, NaN, Infinity, -Infinity, Number.MAX_SAFE_INTEGER + 1]) { test(`rejects invalid capacity ${String(capacity)}`, () => { expect(() => createInboundBodyBudget(capacity)).toThrow(RangeError); From 57900d1d8c3093d91fe3ff7061cb172fe80dd067 Mon Sep 17 00:00:00 2001 From: JUN <243035832+lidge-jun@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:35:41 +0900 Subject: [PATCH 5/5] docs(server): point the route-classification note at its current test --- src/server/inbound-body-admission.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/inbound-body-admission.ts b/src/server/inbound-body-admission.ts index 5dc0bc075b8..fded3630976 100644 --- a/src/server/inbound-body-admission.ts +++ b/src/server/inbound-body-admission.ts @@ -15,7 +15,7 @@ export { configureInboundBodyBudget, MAX_CONCURRENT_INBOUND_BODY_BYTES } from ". // and WebSocket frames have separate limits and are not part of this gate. // Exported so the contract tests classify the dispatcher's own route list // instead of copying it: a new loopback route has to land in one of these two -// sets, or `tests/server/server-request-body-size.test.ts` fails. +// sets, or `tests/server/inbound-body-admission.test.ts` fails. export const CONFIGURABLE_JSON_BODY_ROUTES: ReadonlySet = new Set([ "/v1/responses", "/v1/responses/compact",