A refused round explained itself with an HTTP header dump (#476, code half)

src/MeshWeaver.AI/ProviderFailureClassifier.cs

@@ -0,0 +1,102 @@
+using System.Net;
+
+namespace MeshWeaver.AI;
+
+/// <summary>
+/// Static, pure classification of the failures a language-model provider can end a round with —
+/// factored out of <c>ThreadExecution</c> so the decision "is this a legible, nameable condition or
+/// an unknown engineering fault?" is unit-testable without a mesh, a thread, or a live endpoint.
+///
+/// <para>Why this exists (#476): when a round dies on the provider, <c>ThreadExecution</c> used to
+/// paste <c>ex.Message</c> straight into the response cell's Text AND Summary. For the two failures
+/// the portal actually hits that message is a raw transport dump — Azure/ClientModel render
+/// <c>"Status: 429 (Too Many Requests) ErrorCode: RateLimitReached"</c> followed by the response body
+/// and the COMPLETE HTTP header block. So the thread's own record of why it failed was an
+/// unreadable, English-only blob, and a round that had been silently moved onto a substitute model
+/// (<c>AgentChatClient.ApplyStaleModelFallback</c>) reported a rate limit for a model the user never
+/// picked, with nothing saying why that model was in play.</para>
+///
+/// <para>This classifier only names the CONDITION. The prose is built at write time by the caller,
+/// from the round's own <c>AccessContext.Locale</c> — the same presentation/condition split
+/// <c>AgentChatClient.HasNoUsableModel</c> uses. The raw provider text is never discarded: it stays
+/// on the <c>LogError(ex, …)</c> that already precedes the terminal write. Classification changes
+/// what the USER reads, never what the operator can see.</para>
+/// </summary>
+public static class ProviderFailureClassifier
+{
+    /// <summary>The banner Azure.Core and System.ClientModel both put in the exception message.</summary>
+    private const string StatusMarker = "Status: ";
+
+    /// <summary>
+    /// The HTTP status a provider refused the round with, or <c>null</c> when the failure carries no
+    /// recognisable transport status (an ordinary engineering fault — a serialization bug, a tool
+    /// exception — which must keep reporting its own message verbatim).
+    ///
+    /// <para>Walks the whole inner-exception chain: the streaming pipeline wraps provider faults
+    /// (Microsoft.Extensions.AI middleware, the agent framework's invocation wrapper), so the typed
+    /// transport exception is rarely the outermost one.</para>
+    ///
+    /// <para>Three probes, in order of authority. <see cref="HttpRequestException.StatusCode"/> is
+    /// typed and covers the plain-HTTP providers (Ollama, custom gateways). The message banner covers
+    /// <c>Azure.RequestFailedException</c> and <c>System.ClientModel.ClientResultException</c> —
+    /// matched on text deliberately rather than on those types, so this file stays free of a provider
+    /// SDK dependency and keeps working for any client that renders the same conventional banner
+    /// (which is precisely why the raw text is so unreadable in the first place).</para>
+    /// </summary>
+    public static int? TryGetProviderStatus(Exception? ex)
+    {
+        for (var e = ex; e != null; e = e.InnerException)
+        {
+            if (e is HttpRequestException { StatusCode: { } code })
+                return (int)code;
+
+            if (TryParseStatusBanner(e.Message) is { } fromBanner)
+                return fromBanner;
+        }
+        return null;
+    }
+
+    /// <summary>
+    /// True when the provider refused the round because the deployment is out of quota (HTTP 429).
+    /// This is the failure #476 reports on the portal, and the one an operator must be able to read
+    /// off the thread without opening a log.
+    /// </summary>
+    public static bool IsRateLimited(Exception? ex) => TryGetProviderStatus(ex) == 429;
+
+    /// <summary>
+    /// True when the provider itself faulted (HTTP 5xx) — e.g. the managed endpoint's
+    /// <c>500 {"message":"Internal server error: 'NoneType' object has no attribute 'items'"}</c>,
+    /// the second symptom in #476. Distinct from <see cref="IsRateLimited"/> because the remedy
+    /// differs: a 5xx is the deployment misbehaving, a 429 is the deployment being over its budget.
+    /// </summary>
+    public static bool IsProviderUnavailable(Exception? ex)
+        => TryGetProviderStatus(ex) is >= 500 and < 600;
+
+    /// <summary>
+    /// Reads <c>"Status: 429 (Too Many Requests)"</c> — the conventional Azure.Core /
+    /// System.ClientModel banner — and returns the numeric status, or <c>null</c>.
+    ///
+    /// <para>Anchored on the digits IMMEDIATELY after the marker and bounded to three of them, so a
+    /// body that merely mentions a status ("Status: ok") or a prose sentence containing the word
+    /// yields nothing rather than a bogus code.</para>
+    /// </summary>
+    private static int? TryParseStatusBanner(string? message)
+    {
+        if (string.IsNullOrEmpty(message))
+            return null;
+        var at = message.IndexOf(StatusMarker, StringComparison.Ordinal);
+        if (at < 0)
+            return null;
+        var start = at + StatusMarker.Length;
+        var end = start;
+        while (end < message.Length && end - start < 3 && char.IsAsciiDigit(message[end]))
+            end++;
+        if (end == start)
+            return null;
+        // A longer digit run is not an HTTP status (a token count, an id) — reject rather than
+        // truncate it to the first three digits.
+        if (end < message.Length && char.IsAsciiDigit(message[end]))
+            return null;
+        return int.Parse(message.AsSpan(start, end - start));
+    }
+}

src/MeshWeaver.AI/ThreadExecution.cs

@@ -2572,12 +2572,60 @@ IObservable<MeshNode> UpdateThreadExecution(Func<MeshThread, MeshThread> mutate)
                             errorToolCalls = toolCallLog;
                             errorNodeChanges = nodeChangeLog;
                         }
+                        // 🪙 #476 — LEGIBLE PROVIDER FAILURE. When the round dies on the provider
+                        // rather than on our own code, `ex.Message` is a raw transport dump: Azure /
+                        // System.ClientModel render "Status: 429 (Too Many Requests) ErrorCode:
+                        // RateLimitReached" plus the response body plus the COMPLETE HTTP header
+                        // block. Pasting that into the cell is what #476 reports — an unreadable,
+                        // English-only blob standing in for the thread's account of why it failed.
+                        //
+                        // Same presentation/condition split as the NO_USABLE_MODEL branch above: the
+                        // classifier names the condition, the prose is built HERE from the round's own
+                        // AccessContext (explicit locale — never ambient CultureInfo, a round hops
+                        // schedulers), and the raw provider text stays on the LogError(ex, …) a few
+                        // lines up. Nothing is swallowed; the operator still sees everything.
+                        //
+                        // An unclassified exception (our bug, a tool fault) keeps reporting its own
+                        // message verbatim — that text is the diagnosis, and replacing it would be
+                        // the actual information loss.
+                        var servingModel = actualModel ?? effectiveModel ?? request.ModelName;
+                        var providerStatus = ProviderFailureClassifier.TryGetProviderStatus(ex);
+                        var providerMessage = providerStatus switch
+                        {
+                            429 => LocalizationCatalog.Get(
+                                "chat.modelRateLimited", userAccessContext?.Locale,
+                                servingModel ?? "(default)"),
+                            >= 500 and < 600 => LocalizationCatalog.Get(
+                                "chat.modelProviderError", userAccessContext?.Locale,
+                                servingModel ?? "(default)", providerStatus.Value),
+                            _ => null
+                        };
+                        if (providerMessage is not null)
+                        {
+                            // 🔎 The swap is SILENT on a healthy round (directive 2026-07-13 — an
+                            // unusable pin is not something the user can act on mid-thread), but a
+                            // FAILED round is exactly the case #476 opens with: the failure names a
+                            // model the user never picked, and nothing on screen says why it was in
+                            // play. Name it, once, on the terminal error only.
+                            if (!string.IsNullOrEmpty(substitutedFrom)
+                                && !string.IsNullOrEmpty(servingModel))
+                            {
+                                providerMessage += " " + LocalizationCatalog.Get(
+                                    "chat.modelSubstitutionNote", userAccessContext?.Locale,
+                                    substitutedFrom, servingModel);
+                            }
+                            logger.LogWarning(
+                                "[ThreadExec] PROVIDER_REFUSED threadPath={ThreadPath} responseId={ResponseId} "
+                                + "status={Status} serving={Serving} requested={Requested}",
+                                threadPath, responseMsgId, providerStatus,
+                                servingModel ?? "(default)", substitutedFrom ?? "(not substituted)");
+                        }
                         // A CLI harness that isn't logged in (e.g. Claude Code "Not logged in · Please
                         // run /login") surfaces as AuthRequiredException — render an actionable "/login"
                         // affordance instead of the cryptic "exit code 1" the SDK throws.
                         var errorText = (ex is AuthRequiredException authEx
                             ? errorTextBase + "\n\n" + authEx.ToMarkdown()
-                            : errorTextBase + $"\n\n*Error: {ex.Message}*").Trim();
+                            : errorTextBase + $"\n\n*Error: {providerMessage ?? ex.Message}*").Trim();
                         // 🚨 NO await on hub-touching observables in src/. Subscribe-
                         // continuation: push the error cell, then flip Idle, then notify.
                         // (Previous `.ToTask()` bridge would deadlock the action block —
@@ -2591,7 +2639,7 @@ IObservable<MeshNode> UpdateThreadExecution(Func<MeshThread, MeshThread> mutate)
                         // limit. Stamp what actually answered — and carry the requested id alongside
                         // it as RequestedModelName so the swap is visible on the failed round too.
                         var pushErrorObs = PushToResponseMessage(errorText, errorToolCalls, errorNodeChanges,
-                            request.AgentName, actualModel ?? effectiveModel ?? request.ModelName,
+                            request.AgentName, servingModel,
                             inputTokens: inputTokens, outputTokens: outputTokens,
                             totalTokens: totalTokens,
                             completedAt: DateTime.UtcNow,
@@ -2613,7 +2661,7 @@ IObservable<MeshNode> UpdateThreadExecution(Func<MeshThread, MeshThread> mutate)
                         // usage to the model that actually answered, not the one it asked for.
                         TokenUsageNodeType.RecordUsage(parentHub, threadPath,
                             AgentPickerProjection.PartitionOf(threadPath),
-                            actualModel ?? effectiveModel ?? request.ModelName, inputTokens, outputTokens, execLogger,
+                            servingModel, inputTokens, outputTokens, execLogger,
                             cacheReadTokens, cacheWriteTokens)
                         .Subscribe(
                             _ => { },
@@ -2633,9 +2681,12 @@ IObservable<MeshNode> UpdateThreadExecution(Func<MeshThread, MeshThread> mutate)
                             },
                             () =>
                             {
-                                // Summary invariant for the Error path — non-empty.
+                                // Summary invariant for the Error path — non-empty. #476: when the
+                                // provider is what refused, the fallback names the CLASSIFIED
+                                // condition, never the raw transport dump (which the LogError above
+                                // already carries in full).
                                 var errorSummary = string.IsNullOrEmpty(errorTextLocal)
-                                    ? $"Error: {ex.Message}"
+                                    ? providerMessage ?? $"Error: {ex.Message}"
                                     : errorTextLocal;
                                 // (RecordUsage for the errored round runs as an independent side
                                 // effect in the catch body ABOVE — not here, so a faulted cell

src/MeshWeaver.Documentation/Data/AI/ProviderConfiguration.md

@@ -145,6 +145,33 @@ model so the thread keeps running. Three rules make that swap honest:
   (Exhausted-fallback alone is not fatal: a deployment whose keys live in factory config is
   invisible to the credential resolver yet builds agents and runs normally.)
 
+### The credential check cannot see quota — so the refusal has to read well
+
+A usable credential means the deployment will *answer*, not that it will *serve*. A model with a
+perfectly good key can still refuse every round because it is out of quota (HTTP 429) or because the
+deployment itself is faulting (HTTP 5xx). No local check can predict that — only the provider's
+answer reveals it — so the requirement is not "never fall back onto a throttled model", it is **fail
+legibly when the provider refuses**.
+
+`ProviderFailureClassifier` names the condition from the exception chain (typed
+`HttpRequestException.StatusCode`, else the conventional `Status: NNN` banner that Azure.Core and
+System.ClientModel both render), and `ThreadExecution` builds the prose at write time off the round's
+own `AccessContext.Locale`:
+
+- **429** → `chat.modelRateLimited`, naming the model that actually served.
+- **5xx** → `chat.modelProviderError`, naming the model and the status.
+- **Substituted rounds add one sentence** (`chat.modelSubstitutionNote`) naming the requested model
+  and the one used instead. This is the only place the swap is spelled out to the user, and it earns
+  its place: the failure names a model they never picked, which is otherwise inexplicable.
+- **Anything unclassified keeps its own message verbatim** — for a tool fault or a bug in our code
+  that message *is* the diagnosis, and generic prose would erase it.
+
+The raw transport text is never discarded, only relocated: it stays on the `LogError(ex, …)` that
+precedes the terminal write, alongside a `PROVIDER_REFUSED` warning carrying the status, the serving
+model and the requested one. What changed is what the *user* reads — previously `ex.Message` went
+straight into the cell's Text and Summary, which for these failures is the status line plus the
+response body plus the complete HTTP header block.
+
 ---
 
 ## How the Model Picker Is Populated

src/MeshWeaver.Documentation/Data/WhatsNew/2026-08-08-legible-provider-refusal.md

@@ -0,0 +1,30 @@
+---
+Name: A rate-limited model now says so, instead of dumping HTTP headers into your thread
+Category: What's New
+Description: When a provider refuses a round — out of quota, or a server error — the thread now shows the reason in your language and names the model that ran, instead of pasting the raw transport error.
+Icon: Sparkle
+---
+
+# A rate-limited model now says so, instead of dumping HTTP headers into your thread
+
+Sometimes a round dies because the model deployment refuses it: it is out of quota,
+or the provider itself is faulting. Until now the thread showed you the provider's
+raw failure — the status line, the JSON body, and the complete block of HTTP
+response headers, in English, whatever your language. It was the least readable
+thing on the page, and it was the only account you had of what went wrong.
+
+A refused round now ends with a sentence: the model is out of quota (or the
+provider returned an error), in your language, naming the model that actually ran.
+The raw transport detail hasn't been thrown away — it moves to the log, where an
+operator can still see every byte of it, together with the status and both model
+names.
+
+There is a second half to this, and it is the one that used to be genuinely
+confusing. If the model you picked has no usable credentials, your round is quietly
+moved onto one that has — so the failure you were shown named a model you never
+selected, with nothing anywhere saying why. When a substituted round fails, it now
+says so in one sentence: which model you chose, and which one was used instead.
+
+Errors that aren't the provider's are untouched. If a round fails because of a bug
+or a tool, its own message still comes through word for word — that message is the
+diagnosis, and replacing it with something tidier would only hide it.

src/MeshWeaver.Messaging.Hub/Localization/strings.de.json

@@ -171,6 +171,9 @@
   "chat.noLanguageModel": "Kein Sprachmodell verfügbar. Unter Einstellungen → Sprachmodelle ein Modell hinzufügen (oder die Modelle eines Providers für diese Installation konfigurieren), um den Chat zu nutzen.",
   "chat.noUsableModel": "Kein Sprachmodell kann diese Runde bedienen: Das ausgewählte Modell „{0}“ hat keine nutzbaren Zugangsdaten, und kein anderes Modell im Katalog löst welche auf. Unter Einstellungen → Sprachmodelle einen Provider-Schlüssel hinterlegen oder ein anderes Modell wählen.",
   "chat.noUsableModelNoSelection": "Kein Sprachmodell kann diese Runde bedienen: Kein Modell im Katalog hat nutzbare Zugangsdaten. Unter Einstellungen → Sprachmodelle einen Provider-Schlüssel hinterlegen.",
+  "chat.modelRateLimited": "Das Sprachmodell „{0}“ hat kein Kontingent mehr: Der Provider hat diese Runde mit einer Ratenbegrenzung abgewiesen (HTTP 429). Bitte später erneut senden oder ein anderes Modell wählen.",
+  "chat.modelProviderError": "Das Sprachmodell „{0}“ konnte diese Runde nicht bedienen: Der Provider hat einen Fehler zurückgegeben (HTTP {1}). Bitte später erneut senden oder ein anderes Modell wählen.",
+  "chat.modelSubstitutionNote": "Diese Runde lief nicht auf dem ausgewählten Modell: „{0}“ hat keine nutzbaren Zugangsdaten, daher wurde stattdessen „{1}“ verwendet.",
   "chat.errorBoundary": "Im Chat ist ein Fehler aufgetreten — er bleibt hier isoliert, der Rest der Anwendung funktioniert weiter.",
   "chat.reloadChat": "Chat neu laden",
   "chat.sendFailed": "Nachricht konnte nicht gesendet werden: {0}",

src/MeshWeaver.Messaging.Hub/Localization/strings.en.json

@@ -171,6 +171,9 @@
   "chat.noLanguageModel": "No language model is available. Add one in Settings → Language Models (or configure a provider's models for this deployment) to start chatting.",
   "chat.noUsableModel": "No language model can serve this round: the selected model '{0}' has no usable credentials, and no other model in the catalogue resolves one. Add a provider key in Settings → Language Models, or pick a different model.",
   "chat.noUsableModelNoSelection": "No language model can serve this round: no model in the catalogue has usable credentials. Add a provider key in Settings → Language Models.",
+  "chat.modelRateLimited": "The language model '{0}' is out of quota: the provider refused this round with a rate limit (HTTP 429). Submit again later, or pick a different model.",
+  "chat.modelProviderError": "The language model '{0}' could not serve this round: the provider returned an error (HTTP {1}). Submit again later, or pick a different model.",
+  "chat.modelSubstitutionNote": "This round did not run on the model you selected: '{0}' has no usable credentials, so '{1}' was used instead.",
   "chat.errorBoundary": "The chat hit an error — contained here so the rest of the app keeps working.",
   "chat.reloadChat": "Reload chat",
   "chat.sendFailed": "Couldn't send your message: {0}",