diff --git a/README.md b/README.md index 46ae338d..51fa3ade 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ CLI / TUI / React / Windows Desktop / CI | 领域 | 当前能力 | |---|---| | Agent 运行时 | Mission/Run、可恢复 Supervisor、严格生命周期、检查点、取消、重试、预算和执行租约 | -| 模型与上下文 | Mock 与 Anthropic-compatible Provider、模型路由、资格校验、流式响应、上下文压缩、结构化记忆 | +| 模型与上下文 | Mock、Anthropic-compatible 与 OpenAI-compatible Provider、模型路由、资格校验、流式响应、上下文压缩、结构化记忆 | | 计划与协作 | Plan/Delivery、工作项、备注、最多两个核心 child、1/2/4/6 档只读 Fan-out、共享预算与取消扇出 | | 工具与权限 | Tool Gateway、JSON Schema 校验、Policy、Scope、人工审批、四档宿主权限、受控固定命令提案 | | 代码工作流 | 系统目录选择与 Workspace 导入、工作区浏览、仓库状态、提交历史、Diff 审阅、文件编辑提案、验证计划、Code Journey 与 Handoff | @@ -102,6 +102,11 @@ go run ./cmd/cyberagent tui 默认配置使用确定性的 Mock Provider，不需要 API key。接入外部模型前，请先阅读 [Provider 与模型配置](docs/usage.md#model-and-provider-commands)；凭证应进入系统凭证存储或进程环境，不能提交到仓库。 +OpenAI-compatible 连接使用独立的 `CYBERAGENT_OPENAI_API_KEY`、 +`CYBERAGENT_OPENAI_BASE_URL` 与 `CYBERAGENT_OPENAI_MODEL` 环境变量；后两项默认分别为 +`https://api.openai.com` 和 `gpt-4.1-mini`。仓库内的 `configs/models.yaml` +只是无秘密示例，不会作为运行时配置源。 + ### Windows Desktop 预览 ```powershell diff --git a/configs/models.yaml b/configs/models.yaml index 9b313aae..832abe0a 100644 --- a/configs/models.yaml +++ b/configs/models.yaml @@ -23,3 +23,23 @@ providers: mock: type: mock secrets: none + +# Documentation-only example. This repository file is not loaded as runtime +# Provider configuration and must never contain credentials or private headers. +# Configure the trusted process environment instead: +# CYBERAGENT_OPENAI_API_KEY= +# CYBERAGENT_OPENAI_BASE_URL=https://api.openai.com +# CYBERAGENT_OPENAI_MODEL=gpt-4.1-mini +# The API key may alternatively be stored under the `openai` entry in the +# Go-owned operating-system credential store. +openai_compatible_example: + provider_name: openai + type: openai_compatible + transport: openai_chat_completions + default_base_url: https://api.openai.com + default_model: gpt-4.1-mini + timeout_seconds: 60 + request_headers: [Authorization, Content-Type, Accept] + compatibility_profile: strict_chat_completions + runtime_loaded: false + secrets: environment_or_system_store_only diff --git a/docs/architecture.md b/docs/architecture.md index 71379d8c..5b59dcbf 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -429,7 +429,26 @@ The initial product remains conservative: real public-network attack automation The LLM router remains independent from orchestration. Run snapshots record the selected provider/model route without persisting API keys. Providers normalize HTTP, network, protocol, and cancellation errors into typed outcomes; only RunSupervisor decides whether a side-effect-free model request may be retried. Legacy unbound Session chat receives typed errors through Router but does not gain an implicit retry loop. -Environment adapters currently expose `mimo` and `deepseek` as separate names over the shared Anthropic-compatible transport. Each adapter reads only its own API-key/base-URL/model namespace, and the Provider object remains inside the Go control plane; credentials never enter Run configuration or event payloads. +Environment adapters expose `mimo`, `deepseek`, and `anthropic` over the shared +Anthropic-compatible transport, plus the canonical `openai` adapter over OpenAI Chat +Completions. Each adapter reads only its dedicated API-key/base-URL/model namespace. +For `openai`, that namespace is `CYBERAGENT_OPENAI_API_KEY`, +`CYBERAGENT_OPENAI_BASE_URL`, and `CYBERAGENT_OPENAI_MODEL`; its defaults are +`https://api.openai.com` and `gpt-4.1-mini`. The Provider object and secrets remain +inside the Go control plane and never enter Run configuration or event payloads. +The production Registry supplies a 60-second client timeout, and internal injected +clients are clamped to that maximum. The adapter disables redirects and emits only the +bearer authorization plus content negotiation headers required by Chat Completions; +custom headers and repository-selected compatibility modes are deliberately absent. +Repository files, including `configs/models.yaml`, are documentation rather than a +runtime Provider configuration source, so workspace content cannot inject endpoints, +headers, or credential references. + +Connectivity diagnostics and Harness qualification publish a closed, content-free +failure reason (`none`, `not_configured`, `authentication`, `network`, `rate_limit`, +`capacity`, `model_not_found`, or `protocol_incompatible`) separately from the retry +outcome. HTTP/OpenAPI, CLI, Web, and Desktop project that safe category without +returning endpoint URLs, response bodies, raw errors, prompts, tool arguments, or keys. Context is assembled from: diff --git a/docs/openapi.json b/docs/openapi.json index 610548b9..e310beed 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -466,6 +466,12 @@ "description": "Provider name", "required": true, "schema": { + "enum": [ + "anthropic", + "deepseek", + "mimo", + "openai" + ], "maxLength": 256, "minLength": 1, "pattern": "^[^/\\\\\\x00-\\x1f\\x7f]+$", @@ -14589,6 +14595,7 @@ "enum": [ "mock", "anthropic_messages", + "openai_chat_completions", "provider_contract" ], "type": "string" @@ -14648,6 +14655,19 @@ "minimum": 0, "type": "integer" }, + "failure_reason": { + "enum": [ + "none", + "not_configured", + "authentication", + "network", + "rate_limit", + "capacity", + "model_not_found", + "protocol_incompatible" + ], + "type": "string" + }, "harness": { "$ref": "#/components/schemas/ModelHarnessAvailabilityView" }, @@ -14713,6 +14733,7 @@ "model", "status", "outcome", + "failure_reason", "retryable", "network_request_attempted", "model_calls", @@ -15724,7 +15745,8 @@ "kind": { "enum": [ "local", - "anthropic_compatible" + "anthropic_compatible", + "openai_compatible" ], "type": "string" }, @@ -15768,7 +15790,8 @@ "items": { "$ref": "#/components/schemas/ProviderCredentialStatusView" }, - "maxItems": 3, + "maxItems": 4, + "minItems": 4, "type": "array" }, "protocol_version": { @@ -15833,6 +15856,12 @@ "type": "string" }, "provider": { + "enum": [ + "anthropic", + "deepseek", + "mimo", + "openai" + ], "type": "string" }, "registry_generation": { @@ -15900,6 +15929,19 @@ "minimum": 0, "type": "integer" }, + "failure_reason": { + "enum": [ + "none", + "not_configured", + "authentication", + "network", + "rate_limit", + "capacity", + "model_not_found", + "protocol_incompatible" + ], + "type": "string" + }, "model": { "type": "string" }, @@ -15952,6 +15994,7 @@ "model", "status", "outcome", + "failure_reason", "retryable", "network_request_attempted", "model_called", diff --git a/docs/usage.md b/docs/usage.md index d3355eb8..f74d2d6b 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -734,19 +734,25 @@ cyberagent provider list cyberagent provider test cyberagent provider test mimo/mimo-v2.5-pro cyberagent provider test deepseek/deepseek-v4-flash +cyberagent provider test openai/gpt-4.1-mini cyberagent provider qualify script cyberagent provider qualify mimo/mimo-v2.5-pro +cyberagent provider qualify openai/gpt-4.1-mini cyberagent model list cyberagent model set script mock/mock-code ``` -`provider test` accepts either a route name, such as `learn`, or a direct `provider/model` reference. It is an explicit online connectivity diagnostic: each invocation may send one small, content-free, tool-disabled model request with a 15-second deadline and may incur Provider charges. Passing it does not qualify streamed ToolCall, ToolResult, or strict-JSON behavior. +`provider test` accepts either a route name, such as `learn`, or a direct `provider/model` reference. It is an explicit online connectivity diagnostic: each invocation may send one small, content-free, tool-disabled model request with a 15-second deadline and may incur Provider charges. Passing it does not qualify streamed ToolCall, ToolResult, or strict-JSON behavior. Its safe `failure_reason` distinguishes `not_configured`, `authentication`, `network`, `rate_limit`, `capacity`, `model_not_found`, and `protocol_incompatible` without returning raw Provider errors. `provider qualify` accepts the same route or direct reference and performs the separate `model_harness_qualification.v1` contract. Built-in Mock returns immediately with zero model calls. An external model may receive at most two bounded requests under a 30-second deadline: one exact synthetic nonce ToolCall, then one exact strict-JSON acknowledgement after a synthetic ToolResult. Prayu never dispatches the synthetic Tool. Successful qualification is bound to the exact Provider/model/base-URL/strategy for seven days; a configuration change or expiry fails closed. The command prints status, counts, protocol strategy, and readiness only. It never prints prompts, response content, Tool arguments, API keys, endpoints, environment-variable names, or raw errors. `model set` validates and persists the route before changing the in-memory Router, so a failed SQLite write cannot create a process-only route. Route selection does not automatically qualify the model. Root, Specialist, and read-only Fan-out use one Go-owned Harness preflight immediately before every Provider call; external models that have not passed the exact qualification are rejected before normal Agent execution. -The optional `mimo`, `deepseek`, and `anthropic` Providers load credentials from the process environment first and may then use the Go-owned system credential store. Windows stores exact supported keys in Credential Manager; non-Windows has no plaintext-file fallback. Credential status and mutation never return plaintext and a change currently requires process restart. Base URLs must be absolute HTTPS URLs unless they target an exact loopback host over HTTP; embedded credentials, query strings, fragments, and redirects are rejected. API keys are bounded normalized UTF-8 without whitespace or control characters. +The optional `mimo`, `deepseek`, `anthropic`, and `openai` Providers load credentials from the process environment first and may then use the Go-owned system credential store. OpenAI-compatible configuration uses only `CYBERAGENT_OPENAI_API_KEY`, `CYBERAGENT_OPENAI_BASE_URL`, and `CYBERAGENT_OPENAI_MODEL`; the base URL and model default to `https://api.openai.com` and `gpt-4.1-mini`. Its strict Chat Completions profile uses a 60-second production HTTP deadline, clamps internal injected clients to that maximum, follows no redirects, sends only `Authorization`, `Content-Type`, and `Accept`, and accepts no caller- or repository-supplied custom headers or compatibility fields. The OS credential name is `openai`. Do not use generic `OPENAI_*` variables for this adapter. + +Windows stores exact supported keys in Credential Manager; non-Windows has no plaintext-file fallback. Credential status and mutation never return plaintext. Desktop and API control planes atomically reload a new Registry generation after a successful change; a host without the reload dependency reports `restart_required: true`. Base URLs must be absolute HTTPS URLs unless they target an exact loopback host over HTTP; embedded credentials, query strings, fragments, and redirects are rejected. API keys are bounded normalized UTF-8 without whitespace or control characters. [`../configs/models.yaml`](../configs/models.yaml) is a documentation-only, non-secret example and is not loaded at runtime; repository or Workspace content cannot configure a Provider endpoint, header, or key. + +For an opt-in live smoke, set the three `CYBERAGENT_OPENAI_*` variables in the current process, then use the exact configured model reference. With the default model, run `cyberagent provider test openai/gpt-4.1-mini` followed by `cyberagent provider qualify openai/gpt-4.1-mini`; if `CYBERAGENT_OPENAI_MODEL` has another value, replace `gpt-4.1-mini` in both commands with that exact value. These commands make real, potentially billable network calls and are never part of the default test suite. Use a local mock endpoint for automated smoke tests. Neither command prints the API key, configured endpoint, prompts, response text, Tool arguments, raw deltas, or raw errors; unset the key after the manual check. ## Durable Run Wake Intent diff --git a/internal/app/app.go b/internal/app/app.go index 5b3c5504..db095022 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -676,8 +676,12 @@ func (a *App) providerCommand(ctx context.Context, args []string) error { } switch args[0] { case "list": - for _, name := range a.router.ProviderNames() { - fmt.Fprintln(a.out, name) + if a.models == nil { + return errors.New("model registry is unavailable") + } + for _, provider := range a.models.Snapshot().Providers { + fmt.Fprintf(a.out, "%s\t%s\t%s\n", provider.Name, provider.Kind, + provider.Status) } return nil case "test": @@ -700,9 +704,10 @@ func (a *App) providerCommand(ctx context.Context, args []string) error { if err != nil { return err } - fmt.Fprintf(a.out, "protocol: %s\nprovider: %s\nmodel: %s\nstatus: %s\noutcome: %s\nretryable: %t\nnetwork_request_attempted: %t\nmodel_called: %t\ntool_called: %t\nresponse_content_returned: %t\nduration_ms: %d\n", + fmt.Fprintf(a.out, "protocol: %s\nprovider: %s\nmodel: %s\nstatus: %s\noutcome: %s\nfailure_reason: %s\nretryable: %t\nnetwork_request_attempted: %t\nmodel_called: %t\ntool_called: %t\nresponse_content_returned: %t\nduration_ms: %d\n", result.ProtocolVersion, result.Provider, result.Model, result.Status, - result.Outcome, result.Retryable, result.NetworkRequestAttempted, + result.Outcome, result.FailureReason, result.Retryable, + result.NetworkRequestAttempted, result.ModelCalled, result.ToolCalled, result.ResponseContentReturned, result.DurationMillis) return nil @@ -725,9 +730,10 @@ func (a *App) providerCommand(ctx context.Context, args []string) error { if err != nil { return err } - fmt.Fprintf(a.out, "protocol: %s\nprovider: %s\nmodel: %s\nstatus: %s\noutcome: %s\nretryable: %t\nnetwork_request_attempted: %t\nmodel_calls: %d\nsynthetic_tool_calls: %d\ntool_executed: %t\nresponse_content_returned: %t\ntransport_protocol: %s\ntool_strategy: %s\njson_strategy: %s\nqualification_status: %s\nroot_eligible: %t\nduration_ms: %d\n", + fmt.Fprintf(a.out, "protocol: %s\nprovider: %s\nmodel: %s\nstatus: %s\noutcome: %s\nfailure_reason: %s\nretryable: %t\nnetwork_request_attempted: %t\nmodel_calls: %d\nsynthetic_tool_calls: %d\ntool_executed: %t\nresponse_content_returned: %t\ntransport_protocol: %s\ntool_strategy: %s\njson_strategy: %s\nqualification_status: %s\nroot_eligible: %t\nduration_ms: %d\n", result.ProtocolVersion, result.Provider, result.Model, result.Status, - result.Outcome, result.Retryable, result.NetworkRequestAttempted, + result.Outcome, result.FailureReason, result.Retryable, + result.NetworkRequestAttempted, result.ModelCalls, result.SyntheticToolCalls, result.ToolExecuted, result.ResponseContentReturned, result.Harness.TransportProtocol, result.Harness.ToolStrategy, result.Harness.JSONStrategy, diff --git a/internal/app/provider_command_test.go b/internal/app/provider_command_test.go index a8ec5979..2fe09a09 100644 --- a/internal/app/provider_command_test.go +++ b/internal/app/provider_command_test.go @@ -50,6 +50,7 @@ func TestDeepSeekProviderRegistersFromEnvironmentAndUsesDefaults(t *testing.T) { if code != 0 || stderr != "" || !strings.Contains(output, "provider: deepseek") || !strings.Contains(output, "model: deepseek-v4-flash") || !strings.Contains(output, "status: reachable") || + !strings.Contains(output, "failure_reason: none") || !strings.Contains(output, "response_content_returned: false") { t.Fatalf("unexpected DeepSeek test output=%q stderr=%q code=%d", output, stderr, code) } @@ -66,15 +67,14 @@ func TestDeepSeekProviderRegistersFromEnvironmentAndUsesDefaults(t *testing.T) { } } -func TestDeepSeekProviderRequiresEnvironmentKey(t *testing.T) { +func TestProviderListProjectsUnconfiguredProvidersWithoutRegisteringThem(t *testing.T) { t.Setenv("DEEPSEEK_API_KEY", "") listed, stderr, code := executeTestCommand(t, "provider", "list") if code != 0 || stderr != "" { t.Fatalf("provider list failed: output=%q stderr=%q code=%d", listed, stderr, code) } - for _, name := range strings.Fields(listed) { - if name == "deepseek" { - t.Fatal("DeepSeek provider was registered without an API key") - } + if !strings.Contains(listed, "deepseek\tanthropic_compatible\tnot_configured") || + !strings.Contains(listed, "openai\topenai_compatible\tnot_configured") { + t.Fatalf("unconfigured Provider status was not projected: %q", listed) } } diff --git a/internal/application/model_control.go b/internal/application/model_control.go index 2867d8dc..65f48178 100644 --- a/internal/application/model_control.go +++ b/internal/application/model_control.go @@ -93,7 +93,7 @@ func (s *ModelControlService) Diagnose(ctx context.Context, return modelregistry.DiagnosticResult{}, apperror.New( apperror.CodeInvalidArgument, "Provider diagnostic request is invalid") } - if !snapshotContainsProviderModel(s.registry.Snapshot(), request.Provider, request.Model) { + if !snapshotContainsKnownProviderModel(s.registry.Snapshot(), request.Provider, request.Model) { return modelregistry.DiagnosticResult{}, apperror.New( apperror.CodeFailedPrecondition, "diagnostic Provider model is unavailable") } @@ -118,7 +118,7 @@ func (s *ModelControlService) QualifyHarness(ctx context.Context, return modelregistry.HarnessQualificationResult{}, apperror.New( apperror.CodeInvalidArgument, "model Harness qualification request is invalid") } - if !snapshotContainsProviderModel(s.registry.Snapshot(), request.Provider, request.Model) { + if !snapshotContainsKnownProviderModel(s.registry.Snapshot(), request.Provider, request.Model) { return modelregistry.HarnessQualificationResult{}, apperror.New( apperror.CodeFailedPrecondition, "model Harness qualification Provider model is unavailable") @@ -168,3 +168,20 @@ func snapshotContainsProviderModel(snapshot modelregistry.Snapshot, } return false } + +func snapshotContainsKnownProviderModel(snapshot modelregistry.Snapshot, + provider string, model string, +) bool { + for _, current := range snapshot.Providers { + if current.Name != provider || (current.Status != modelregistry.ProviderAvailable && + current.Status != modelregistry.ProviderNotConfigured) { + continue + } + for _, candidate := range current.Models { + if candidate == model { + return true + } + } + } + return false +} diff --git a/internal/application/model_control_test.go b/internal/application/model_control_test.go index c29e6d93..227e6eac 100644 --- a/internal/application/model_control_test.go +++ b/internal/application/model_control_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "cyberagent-workbench/internal/llm" "cyberagent-workbench/internal/modelregistry" ) @@ -58,6 +59,36 @@ func TestModelControlRequiresExplicitDiagnosticConfirmation(t *testing.T) { } } +func TestModelControlReturnsUnconfiguredProviderFacts(t *testing.T) { + service := NewModelControlService(modelregistry.New(nil), modelRouteSettings{}) + diagnostic, err := service.Diagnose(context.Background(), DiagnoseProviderRequest{ + Version: modelregistry.DiagnosticProtocolVersion, + Provider: "openai", Model: modelregistry.DefaultOpenAIModel, + ConfirmDiagnostic: true, + }) + if err != nil || diagnostic.FailureReason != llm.ProviderFailureNotConfigured || + diagnostic.ModelCalled || diagnostic.NetworkRequestAttempted { + t.Fatalf("unexpected unconfigured diagnostic: %#v err=%v", diagnostic, err) + } + qualification, err := service.QualifyHarness(context.Background(), + QualifyModelHarnessRequest{ + Version: modelregistry.HarnessQualificationProtocolVersion, + Provider: "openai", Model: modelregistry.DefaultOpenAIModel, + ConfirmQualification: true, + }) + if err != nil || qualification.FailureReason != llm.ProviderFailureNotConfigured || + qualification.ModelCalls != 0 || qualification.NetworkRequestAttempted { + t.Fatalf("unexpected unconfigured qualification: %#v err=%v", qualification, err) + } + if _, err := service.Diagnose(context.Background(), DiagnoseProviderRequest{ + Version: modelregistry.DiagnosticProtocolVersion, + Provider: "unknown", Model: modelregistry.DefaultOpenAIModel, + ConfirmDiagnostic: true, + }); err == nil { + t.Fatal("unknown Provider diagnostic was accepted") + } +} + func TestModelControlSeparatesHarnessQualificationFromConnectivity(t *testing.T) { service := NewModelControlService(modelregistry.New(nil), modelRouteSettings{}) if _, err := service.QualifyHarness(context.Background(), QualifyModelHarnessRequest{ diff --git a/internal/application/openai_tool_loop_integration_test.go b/internal/application/openai_tool_loop_integration_test.go new file mode 100644 index 00000000..2522bcb7 --- /dev/null +++ b/internal/application/openai_tool_loop_integration_test.go @@ -0,0 +1,346 @@ +package application_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "sync" + "testing" + "time" + + "cyberagent-workbench/internal/application" + "cyberagent-workbench/internal/domain" + "cyberagent-workbench/internal/llm" + "cyberagent-workbench/internal/policy" + "cyberagent-workbench/internal/store" +) + +func TestOpenAICompatibleProviderRunSupervisorToolRoundTrip(t *testing.T) { + const ( + providerName = "openai-supervisor-test" + modelName = "model" + itemTitle = "OpenAI integration item" + finalMessage = "tool transcript accepted" + ) + + var ( + mu sync.Mutex + requests []openAISupervisorWireRequest + handlerErrors []error + ) + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodPost || request.URL.Path != "/v1/chat/completions" { + http.NotFound(writer, request) + return + } + if authorization := request.Header.Get("Authorization"); authorization != "Bearer test-secret" { + http.Error(writer, "unexpected authorization header", http.StatusBadRequest) + return + } + var wire openAISupervisorWireRequest + if err := json.NewDecoder(request.Body).Decode(&wire); err != nil { + http.Error(writer, "invalid request JSON", http.StatusBadRequest) + return + } + + mu.Lock() + call := len(requests) + requests = append(requests, wire) + mu.Unlock() + + var validationErr error + switch call { + case 0: + validationErr = validateOpenAISupervisorFirstRequest(wire) + case 1: + validationErr = validateOpenAISupervisorToolResultRequest(wire, itemTitle) + default: + validationErr = fmt.Errorf("unexpected provider call %d", call+1) + } + if validationErr != nil { + mu.Lock() + handlerErrors = append(handlerErrors, validationErr) + mu.Unlock() + http.Error(writer, "wire protocol validation failed", http.StatusBadRequest) + return + } + + var events []openAISupervisorStreamEvent + if call == 0 { + finish := "tool_calls" + events = []openAISupervisorStreamEvent{ + { + Model: modelName, + Choices: []openAISupervisorStreamChoice{{ + Index: 0, + Delta: openAISupervisorStreamDelta{ + Role: "assistant", + ToolCalls: []openAISupervisorStreamToolCall{{ + Index: 0, ID: "provider-call-1", Type: "function", + Function: openAISupervisorWireFunction{ + Name: "work_item_create", + Arguments: `{"title":"OpenAI integration item","priority":"high"}`, + }, + }}, + }, + }}, + }, + {Model: modelName, Choices: []openAISupervisorStreamChoice{{ + Index: 0, FinishReason: &finish, + }}}, + {Model: modelName, Usage: &openAISupervisorWireUsage{ + PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15, + }}, + } + } else { + finish := "stop" + content := rootActionResponse(domain.RootActionContinue, finalMessage, "", "") + events = []openAISupervisorStreamEvent{ + {Model: modelName, Choices: []openAISupervisorStreamChoice{{ + Index: 0, Delta: openAISupervisorStreamDelta{Role: "assistant", Content: &content}, + }}}, + {Model: modelName, Choices: []openAISupervisorStreamChoice{{ + Index: 0, FinishReason: &finish, + }}}, + {Model: modelName, Usage: &openAISupervisorWireUsage{ + PromptTokens: 12, CompletionTokens: 6, TotalTokens: 18, + }}, + } + } + if err := writeOpenAISupervisorStream(writer, events); err != nil { + mu.Lock() + handlerErrors = append(handlerErrors, err) + mu.Unlock() + } + })) + defer server.Close() + + provider, err := llm.NewOpenAICompatibleProvider(llm.OpenAICompatibleConfig{ + Name: providerName, BaseURL: server.URL, APIKey: "test-secret", DefaultModel: modelName, + }) + if err != nil { + t.Fatal(err) + } + router := llm.NewRouter(llm.ModelRef{Provider: providerName, Model: modelName}) + router.RegisterProvider(provider) + ref := llm.ModelRef{Provider: providerName, Model: modelName} + profile, err := router.HarnessProfile(ref) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + if err := router.SetHarnessQualification(ref, llm.HarnessQualification{ + ProtocolVersion: llm.ModelHarnessProtocolVersion, + BindingDigest: profile.BindingDigest, + ToolCallsQualified: true, + ToolResultsQualified: true, + StrictJSONQualified: true, + StreamingQualified: true, + QualifiedAt: now, + ExpiresAt: now.Add(time.Hour), + }); err != nil { + t.Fatal(err) + } + + st, err := store.Open(filepath.Join(t.TempDir(), "openai-supervisor.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + run := newStartedRunForProvider(t, st, providerName, + domain.Budget{MaxTurns: 3, MaxToolCalls: 5}) + result, err := application.NewRunSupervisor(st, router, policy.NewDefaultChecker()). + Step(context.Background(), run.ID) + if err != nil { + mu.Lock() + deferredErrors := append([]error(nil), handlerErrors...) + mu.Unlock() + t.Fatalf("Supervisor OpenAI tool round trip failed: %v (mock errors: %v)", err, deferredErrors) + } + if result.Status != application.LifecycleTurnCompleted || result.ToolRounds != 1 || + result.ToolCalls != 1 || result.ModelAttempts != 2 || result.Text != finalMessage { + t.Fatalf("unexpected Supervisor result: %#v", result) + } + + mu.Lock() + captured := append([]openAISupervisorWireRequest(nil), requests...) + capturedErrors := append([]error(nil), handlerErrors...) + mu.Unlock() + if len(capturedErrors) != 0 { + t.Fatalf("mock server rejected OpenAI wire requests: %v", capturedErrors) + } + if len(captured) != 2 { + t.Fatalf("provider call count = %d, want 2", len(captured)) + } + + items, err := st.ListWorkItems(context.Background(), domain.WorkItemFilter{RunID: run.ID}) + if err != nil || len(items) != 1 || items[0].Title != itemTitle { + t.Fatalf("tool mutation was not executed exactly once: items=%#v err=%v", items, err) + } + messages, err := st.ListSessionMessages(context.Background(), run.SessionID, true) + if err != nil { + t.Fatal(err) + } + if len(messages) != 2 || messages[0].Role != "user" || messages[1].Role != "assistant" || + messages[1].Content != finalMessage { + t.Fatalf("Session did not commit exactly one user/assistant pair: %#v", messages) + } +} + +type openAISupervisorWireRequest struct { + Model string `json:"model"` + Messages []openAISupervisorWireMessage `json:"messages"` + Tools []openAISupervisorWireTool `json:"tools"` + Stream bool `json:"stream"` + StreamOptions *openAISupervisorStreamOption `json:"stream_options"` + ResponseFormat *openAISupervisorFormat `json:"response_format"` +} + +type openAISupervisorWireMessage struct { + Role string `json:"role"` + Content *string `json:"content"` + ToolCalls []openAISupervisorWireCall `json:"tool_calls"` + ToolCallID string `json:"tool_call_id"` +} + +type openAISupervisorWireTool struct { + Type string `json:"type"` + Function openAISupervisorWireFunction `json:"function"` +} + +type openAISupervisorWireCall struct { + ID string `json:"id"` + Type string `json:"type"` + Function openAISupervisorWireFunction `json:"function"` +} + +type openAISupervisorWireFunction struct { + Name string `json:"name"` + Arguments string `json:"arguments"` +} + +type openAISupervisorStreamOption struct { + IncludeUsage bool `json:"include_usage"` +} + +type openAISupervisorFormat struct { + Type string `json:"type"` +} + +type openAISupervisorWireUsage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` +} + +type openAISupervisorStreamEvent struct { + Model string `json:"model"` + Choices []openAISupervisorStreamChoice `json:"choices"` + Usage *openAISupervisorWireUsage `json:"usage,omitempty"` +} + +type openAISupervisorStreamChoice struct { + Index int `json:"index"` + Delta openAISupervisorStreamDelta `json:"delta"` + FinishReason *string `json:"finish_reason,omitempty"` +} + +type openAISupervisorStreamDelta struct { + Role string `json:"role,omitempty"` + Content *string `json:"content,omitempty"` + ToolCalls []openAISupervisorStreamToolCall `json:"tool_calls,omitempty"` +} + +type openAISupervisorStreamToolCall struct { + Index int `json:"index"` + ID string `json:"id"` + Type string `json:"type"` + Function openAISupervisorWireFunction `json:"function"` +} + +func validateOpenAISupervisorFirstRequest(request openAISupervisorWireRequest) error { + if request.Model != "model" || !request.Stream || request.StreamOptions == nil || + !request.StreamOptions.IncludeUsage || request.ResponseFormat == nil || + request.ResponseFormat.Type != "json_object" { + return errors.New("initial request omitted the qualified streaming JSON contract") + } + toolFound := false + for _, tool := range request.Tools { + if tool.Type == "function" && tool.Function.Name == "work_item_create" { + toolFound = true + } + } + if !toolFound { + return errors.New("initial request omitted the allowlisted work_item_create tool") + } + for _, message := range request.Messages { + if message.Role == "tool" || len(message.ToolCalls) != 0 { + return errors.New("initial request unexpectedly contained a prior tool transcript") + } + } + return nil +} + +func validateOpenAISupervisorToolResultRequest(request openAISupervisorWireRequest, + wantTitle string, +) error { + if request.Model != "model" || !request.Stream || request.StreamOptions == nil || + !request.StreamOptions.IncludeUsage || request.ResponseFormat == nil || + request.ResponseFormat.Type != "json_object" { + return errors.New("follow-up request omitted the qualified streaming JSON contract") + } + for index := 0; index+1 < len(request.Messages); index++ { + assistant := request.Messages[index] + toolResult := request.Messages[index+1] + if assistant.Role != "assistant" || len(assistant.ToolCalls) != 1 || toolResult.Role != "tool" { + continue + } + call := assistant.ToolCalls[0] + if call.ID == "" || call.Type != "function" || call.Function.Name != "work_item_create" || + toolResult.ToolCallID != call.ID || toolResult.Content == nil { + return errors.New("assistant tool call and role=tool result were not paired") + } + var arguments struct { + Title string `json:"title"` + Priority string `json:"priority"` + } + if err := json.Unmarshal([]byte(call.Function.Arguments), &arguments); err != nil || + arguments.Title != wantTitle || arguments.Priority != "high" { + return errors.New("assistant tool-call arguments were not preserved") + } + var result struct { + Version string `json:"version"` + Tool string `json:"tool"` + Status string `json:"status"` + Metadata map[string]string `json:"metadata"` + } + if err := json.Unmarshal([]byte(*toolResult.Content), &result); err != nil || + result.Version != "supervisor_tool_result.v1" || result.Tool != "work_item_create" || + result.Status != "completed" || result.Metadata["entity_id"] == "" { + return errors.New("role=tool message did not contain the successful tool result") + } + return nil + } + return errors.New("follow-up request did not contain adjacent assistant tool_calls and role=tool messages") +} + +func writeOpenAISupervisorStream(writer http.ResponseWriter, + events []openAISupervisorStreamEvent, +) error { + writer.Header().Set("Content-Type", "text/event-stream") + for _, event := range events { + encoded, err := json.Marshal(event) + if err != nil { + return err + } + if _, err := fmt.Fprintf(writer, "data: %s\n\n", encoded); err != nil { + return err + } + } + _, err := fmt.Fprint(writer, "data: [DONE]\n\n") + return err +} diff --git a/internal/application/provider_credential.go b/internal/application/provider_credential.go index 84b864a9..79c5fd2c 100644 --- a/internal/application/provider_credential.go +++ b/internal/application/provider_credential.go @@ -18,7 +18,7 @@ const ( ) var managedCredentialProviders = map[string]struct{}{ - "anthropic": {}, "deepseek": {}, "mimo": {}, + "anthropic": {}, "deepseek": {}, "mimo": {}, "openai": {}, } type ProviderCredentialStatus struct { diff --git a/internal/application/provider_credential_test.go b/internal/application/provider_credential_test.go index 88538b67..00123337 100644 --- a/internal/application/provider_credential_test.go +++ b/internal/application/provider_credential_test.go @@ -73,6 +73,32 @@ func TestProviderCredentialServiceReturnsStatusOnly(t *testing.T) { } } +func TestProviderCredentialServiceManagesOpenAI(t *testing.T) { + store := credential.NewMemoryStore() + service := NewProviderCredentialService(store) + secret := "temporary-openai-provider-key" + status, err := service.Change(t.Context(), ChangeProviderCredentialRequest{ + Version: credential.ProtocolVersion, Provider: "openai", + Action: ProviderCredentialSet, Secret: secret, Confirm: true, + }) + if err != nil || !status.Configured || status.PlaintextReturned { + t.Fatalf("OpenAI credential was not managed safely: %#v err=%v", status, err) + } + statuses, err := service.List(t.Context()) + if err != nil || len(statuses) != 4 { + t.Fatalf("managed credential list=%#v err=%v", statuses, err) + } + found := false + for _, current := range statuses { + if current.Provider == "openai" { + found = current.Configured && !current.PlaintextReturned + } + } + if !found { + t.Fatalf("OpenAI credential status missing: %#v", statuses) + } +} + func TestProviderCredentialServiceReloadsRegistryWithoutRestart(t *testing.T) { store := credential.NewMemoryStore() reloader := &providerCredentialReloadFake{generation: 1} diff --git a/internal/httpapi/model_availability_test.go b/internal/httpapi/model_availability_test.go index 75d32d30..4c6e4fdf 100644 --- a/internal/httpapi/model_availability_test.go +++ b/internal/httpapi/model_availability_test.go @@ -60,10 +60,10 @@ func TestModelAvailabilityIsRedactedAndDoesNotProbeProviders(t *testing.T) { t.Fatal(err) } if envelope.Data.ProtocolVersion != modelregistry.ProtocolVersion || - len(envelope.Data.Providers) != 4 || len(envelope.Data.Routes) != 5 { + len(envelope.Data.Providers) != 5 || len(envelope.Data.Routes) != 5 { t.Fatalf("unexpected model availability: %#v", envelope.Data) } - var foundMimo, foundCode bool + var foundMimo, foundOpenAI, foundCode bool for _, provider := range envelope.Data.Providers { if provider.Name == "mimo" { foundMimo = provider.Status == modelregistry.ProviderAvailable && @@ -73,6 +73,17 @@ func TestModelAvailabilityIsRedactedAndDoesNotProbeProviders(t *testing.T) { llm.HarnessQualificationRequired && !provider.Harnesses[0].RootEligible } + if provider.Name == "openai" { + foundOpenAI = provider.Kind == modelregistry.ProviderKindOpenAICompatible && + provider.Status == modelregistry.ProviderNotConfigured && + len(provider.Models) == 1 && provider.Models[0] == modelregistry.DefaultOpenAIModel && + len(provider.Harnesses) == 1 && + provider.Harnesses[0].TransportProtocol == + llm.HarnessTransportOpenAIChatCompletions && + provider.Harnesses[0].QualificationStatus == + llm.HarnessQualificationRequired && + !provider.Harnesses[0].RootEligible + } } for _, route := range envelope.Data.Routes { if route.Name == "code" { @@ -80,7 +91,7 @@ func TestModelAvailabilityIsRedactedAndDoesNotProbeProviders(t *testing.T) { route.Model == "mimo-http-test" && !route.HarnessReady } } - if !foundMimo || !foundCode { + if !foundMimo || !foundOpenAI || !foundCode { t.Fatalf("configured provider or route missing: %#v", envelope.Data) } } diff --git a/internal/httpapi/model_diff_wake_control.go b/internal/httpapi/model_diff_wake_control.go index 1bcbb4f6..50c6a848 100644 --- a/internal/httpapi/model_diff_wake_control.go +++ b/internal/httpapi/model_diff_wake_control.go @@ -474,7 +474,8 @@ func readStrictControlBody(request *http.Request, label string) ([]byte, error) func providerDiagnosticView(result modelregistry.DiagnosticResult) ProviderDiagnosticView { return ProviderDiagnosticView{ProtocolVersion: result.ProtocolVersion, Provider: result.Provider, Model: result.Model, Status: result.Status, - Outcome: result.Outcome, Retryable: result.Retryable, + Outcome: result.Outcome, FailureReason: string(result.FailureReason), + Retryable: result.Retryable, NetworkRequestAttempted: result.NetworkRequestAttempted, ModelCalled: result.ModelCalled, ToolCalled: result.ToolCalled, ResponseContentReturned: result.ResponseContentReturned, @@ -504,6 +505,7 @@ func modelHarnessQualificationView( return ModelHarnessQualificationView{ ProtocolVersion: result.ProtocolVersion, Provider: result.Provider, Model: result.Model, Status: result.Status, Outcome: result.Outcome, + FailureReason: string(result.FailureReason), Retryable: result.Retryable, NetworkRequestAttempted: result.NetworkRequestAttempted, ModelCalls: result.ModelCalls, SyntheticToolCalls: result.SyntheticToolCalls, diff --git a/internal/httpapi/model_diff_wake_control_test.go b/internal/httpapi/model_diff_wake_control_test.go index cf2442cb..2e94308a 100644 --- a/internal/httpapi/model_diff_wake_control_test.go +++ b/internal/httpapi/model_diff_wake_control_test.go @@ -83,6 +83,7 @@ func TestModelDiffAndWakeHTTPControlsRemainCapabilitySeparated(t *testing.T) { ProviderDiagnosticPath, testControlToken, "", "application/json", strings.NewReader(`{"version":"provider_diagnostic.v1","provider":"mock","model":"mock-code","confirm_diagnostic":true}`)) if diagnostic.Code != http.StatusAccepted || + !strings.Contains(diagnostic.Body.String(), `"failure_reason":"none"`) || !strings.Contains(diagnostic.Body.String(), `"response_content_returned":false`) || strings.Contains(diagnostic.Body.String(), `"response"`) || strings.Contains(diagnostic.Body.String(), `"text"`) { @@ -93,6 +94,7 @@ func TestModelDiffAndWakeHTTPControlsRemainCapabilitySeparated(t *testing.T) { strings.NewReader(`{"version":"model_harness_qualification.v1","provider":"mock","model":"mock-code","confirm_qualification":true}`)) if qualification.Code != http.StatusAccepted || !strings.Contains(qualification.Body.String(), `"status":"qualified"`) || + !strings.Contains(qualification.Body.String(), `"failure_reason":"none"`) || !strings.Contains(qualification.Body.String(), `"model_calls":0`) || !strings.Contains(qualification.Body.String(), `"tool_executed":false`) || !strings.Contains(qualification.Body.String(), `"response_content_returned":false`) || diff --git a/internal/httpapi/openapi.go b/internal/httpapi/openapi.go index 15c81d81..0c08e057 100644 --- a/internal/httpapi/openapi.go +++ b/internal/httpapi/openapi.go @@ -242,6 +242,7 @@ func openAPIOperationSpecs() []openAPIOperationSpec { "maxLength": 40, "pattern": `^[0-9a-f]{40}$`}} routeName := pathIdentityParameter("route", "Model route name") providerName := pathIdentityParameter("provider", "Provider name") + providerName.Schema["enum"] = []string{"anthropic", "deepseek", "mimo", "openai"} return []openAPIOperationSpec{ {Path: "/api/v1", OperationID: "getAPIIndex", Summary: "Inspect API resources", Description: "Returns API and application versions plus top-level resources.", Tag: "System", @@ -1419,7 +1420,8 @@ func applyOpenAPIFieldMetadata(typeName string, fieldName string, schema map[str schema["maxItems"] = application.MaxCodeHandoffReportReferences } if typeName == "ProviderCredentialListView" && fieldName == "items" { - schema["maxItems"] = 3 + schema["minItems"] = 4 + schema["maxItems"] = 4 } if typeName == "ProviderCredentialRequestView" && fieldName == "secret" { schema["writeOnly"] = true @@ -1487,19 +1489,22 @@ var openAPIFieldEnums = map[string][]string{ "ProviderDiagnosticRequestView.version": {modelregistry.DiagnosticProtocolVersion}, "ModelHarnessQualificationRequestView.version": {modelregistry.HarnessQualificationProtocolVersion}, "ModelHarnessAvailabilityView.protocol_version": {llm.ModelHarnessProtocolVersion}, - "ModelHarnessAvailabilityView.transport_protocol": {llm.HarnessTransportMock, llm.HarnessTransportAnthropicMessages, llm.HarnessTransportProviderContract}, + "ModelHarnessAvailabilityView.transport_protocol": {llm.HarnessTransportMock, llm.HarnessTransportAnthropicMessages, llm.HarnessTransportOpenAIChatCompletions, llm.HarnessTransportProviderContract}, "ModelHarnessAvailabilityView.tool_strategy": {llm.HarnessToolStrategyNative, llm.HarnessToolStrategyNone}, "ModelHarnessAvailabilityView.json_strategy": {llm.HarnessJSONStrategyNative, llm.HarnessJSONStrategyPrompt, llm.HarnessJSONStrategyNone}, "ModelHarnessAvailabilityView.qualification_status": {llm.HarnessQualificationTrusted, llm.HarnessQualificationRequired, llm.HarnessQualificationVerified}, "ModelHarnessQualificationView.protocol_version": {modelregistry.HarnessQualificationProtocolVersion}, "ModelHarnessQualificationView.status": {modelregistry.HarnessDiagnosticQualified, modelregistry.HarnessDiagnosticIncompatible, modelregistry.HarnessDiagnosticUnreachable}, "ModelHarnessQualificationView.outcome": {string(llm.OutcomeSuccess), string(llm.OutcomeRetryable), string(llm.OutcomeRateLimited), string(llm.OutcomeInvalidResponse), string(llm.OutcomeCancelled), string(llm.OutcomePermanent)}, + "ModelHarnessQualificationView.failure_reason": {string(llm.ProviderFailureNone), string(llm.ProviderFailureNotConfigured), string(llm.ProviderFailureAuthentication), string(llm.ProviderFailureNetwork), string(llm.ProviderFailureRateLimit), string(llm.ProviderFailureCapacity), string(llm.ProviderFailureModelNotFound), string(llm.ProviderFailureProtocolIncompatible)}, "ProviderDiagnosticView.protocol_version": {modelregistry.DiagnosticProtocolVersion}, "ProviderDiagnosticView.status": {modelregistry.DiagnosticReachable, modelregistry.DiagnosticUnreachable}, "ProviderDiagnosticView.outcome": {string(llm.OutcomeSuccess), string(llm.OutcomeRetryable), string(llm.OutcomeRateLimited), string(llm.OutcomeInvalidResponse), string(llm.OutcomeCancelled), string(llm.OutcomePermanent)}, + "ProviderDiagnosticView.failure_reason": {string(llm.ProviderFailureNone), string(llm.ProviderFailureNotConfigured), string(llm.ProviderFailureAuthentication), string(llm.ProviderFailureNetwork), string(llm.ProviderFailureRateLimit), string(llm.ProviderFailureCapacity), string(llm.ProviderFailureModelNotFound), string(llm.ProviderFailureProtocolIncompatible)}, "ModelRouteControlRequestView.version": {modelregistry.RouteControlProtocolVersion}, "ProviderCredentialListView.protocol_version": {credential.ProtocolVersion}, "ProviderCredentialStatusView.protocol_version": {credential.ProtocolVersion}, + "ProviderCredentialStatusView.provider": {"anthropic", "deepseek", "mimo", "openai"}, "ProviderCredentialRequestView.version": {credential.ProtocolVersion}, "ProviderCredentialRequestView.action": {string(application.ProviderCredentialSet), string(application.ProviderCredentialDelete)}, "FileEditProposalSourceView.protocol_version": {application.FileEditProposalProtocolVersion}, @@ -1601,7 +1606,7 @@ var openAPIFieldEnums = map[string][]string{ "OperatorActionItemView.kind": {string(operatoraction.KindSteeringPending), string(operatoraction.KindApprovalPending), string(operatoraction.KindFileEditReview), string(operatoraction.KindFileEditApply), string(operatoraction.KindWakeDue)}, "OperatorActionItemView.state": {"pending", "proposed", "approved", "queued"}, "OperatorActionItemView.destination": {string(operatoraction.DestinationQueue), string(operatoraction.DestinationApprovals), string(operatoraction.DestinationDiffs), string(operatoraction.DestinationWake)}, - "ProviderAvailabilityView.kind": {modelregistry.ProviderKindLocal, modelregistry.ProviderKindAnthropicCompatible}, + "ProviderAvailabilityView.kind": {modelregistry.ProviderKindLocal, modelregistry.ProviderKindAnthropicCompatible, modelregistry.ProviderKindOpenAICompatible}, "ProviderAvailabilityView.status": {modelregistry.ProviderAvailable, modelregistry.ProviderNotConfigured, modelregistry.ProviderInvalidConfiguration}, "ProviderAvailabilityView.credential_source": {"none", "environment", "system"}, "ScopeView.network_mode": {"disabled", "allowlist"}, diff --git a/internal/httpapi/openapi_test.go b/internal/httpapi/openapi_test.go index c448e813..f28a6936 100644 --- a/internal/httpapi/openapi_test.go +++ b/internal/httpapi/openapi_test.go @@ -245,6 +245,52 @@ func TestOpenAPIDocumentIsDeterministicCapabilitySeparatedAndSecretFree(t *testi "ProviderCredentialStatusView", "secret") assertOpenAPIPropertyFlag(t, document.Components.Schemas, "ProviderCredentialRequestView", "secret", "writeOnly", true) + assertOpenAPIPropertyFlag(t, document.Components.Schemas, + "ProviderCredentialListView", "items", "minItems", float64(4)) + assertOpenAPIPropertyFlag(t, document.Components.Schemas, + "ProviderCredentialListView", "items", "maxItems", float64(4)) + assertOpenAPIEnum(t, document.Components.Schemas, "ProviderCredentialStatusView", "provider", + []string{"anthropic", "deepseek", "mimo", "openai"}) + credentialProviders := []string{"anthropic", "deepseek", "mimo", "openai"} + credentialPath, credentialPathFound := document.Paths[ProviderCredentialPathTemplate] + if !credentialPathFound || credentialPath.Post == nil { + t.Fatal("Provider credential control path is missing") + } + providerParameterFound := false + for _, parameter := range credentialPath.Post.Parameters { + if parameter.Name != "provider" { + continue + } + providerParameterFound = true + raw, ok := parameter.Schema["enum"].([]any) + if !ok { + t.Fatal("Provider credential path parameter has no enum") + } + actual := make([]string, len(raw)) + for index, value := range raw { + actual[index], ok = value.(string) + if !ok { + t.Fatal("Provider credential path parameter enum is not a string list") + } + } + if !reflect.DeepEqual(actual, credentialProviders) { + t.Fatalf("Provider credential path enum=%v want=%v", actual, credentialProviders) + } + } + if !providerParameterFound { + t.Fatal("Provider credential path parameter is missing") + } + assertOpenAPIEnum(t, document.Components.Schemas, "ProviderAvailabilityView", "kind", + []string{"local", "anthropic_compatible", "openai_compatible"}) + assertOpenAPIEnum(t, document.Components.Schemas, "ModelHarnessAvailabilityView", + "transport_protocol", []string{"mock", "anthropic_messages", + "openai_chat_completions", "provider_contract"}) + failureReasons := []string{"none", "not_configured", "authentication", "network", + "rate_limit", "capacity", "model_not_found", "protocol_incompatible"} + assertOpenAPIEnum(t, document.Components.Schemas, "ProviderDiagnosticView", + "failure_reason", failureReasons) + assertOpenAPIEnum(t, document.Components.Schemas, "ModelHarnessQualificationView", + "failure_reason", failureReasons) for _, field := range []string{"path", "content", "command", "hook"} { assertOpenAPISchemaOmits(t, document.Components.Schemas, "SkillPackageInstallRequestView", field) @@ -1038,3 +1084,36 @@ func assertOpenAPIPropertyFlag(t *testing.T, schemas map[string]map[string]any, name, property, flag, value[flag], expected) } } + +func assertOpenAPIEnum(t *testing.T, schemas map[string]map[string]any, + name string, property string, expected []string, +) { + t.Helper() + schema, ok := schemas[name] + if !ok { + t.Fatalf("OpenAPI component %s is missing", name) + } + properties, ok := schema["properties"].(map[string]any) + if !ok { + t.Fatalf("OpenAPI component %s has no properties", name) + } + value, ok := properties[property].(map[string]any) + if !ok { + t.Fatalf("OpenAPI component %s property %s is missing", name, property) + } + raw, ok := value["enum"].([]any) + if !ok { + t.Fatalf("OpenAPI component %s property %s has no enum", name, property) + } + actual := make([]string, len(raw)) + for index, current := range raw { + actual[index], ok = current.(string) + if !ok { + t.Fatalf("OpenAPI component %s property %s has a non-string enum", name, property) + } + } + if !reflect.DeepEqual(actual, expected) { + t.Fatalf("OpenAPI component %s property %s enum=%v want=%v", + name, property, actual, expected) + } +} diff --git a/internal/httpapi/proposal_credential_control_test.go b/internal/httpapi/proposal_credential_control_test.go index 2b8d074b..e86f0609 100644 --- a/internal/httpapi/proposal_credential_control_test.go +++ b/internal/httpapi/proposal_credential_control_test.go @@ -154,7 +154,7 @@ func TestProviderCredentialHTTPNeverReturnsPlaintext(t *testing.T) { } var statuses ProviderCredentialListView decodeDataStatus(t, list, http.StatusOK, &statuses) - if len(statuses.Items) != 3 { - t.Fatalf("credential status count=%d want=3", len(statuses.Items)) + if len(statuses.Items) != 4 { + t.Fatalf("credential status count=%d want=4", len(statuses.Items)) } } diff --git a/internal/httpapi/views.go b/internal/httpapi/views.go index 540245e3..456d8e42 100644 --- a/internal/httpapi/views.go +++ b/internal/httpapi/views.go @@ -77,6 +77,7 @@ type ProviderDiagnosticView struct { Model string `json:"model"` Status string `json:"status"` Outcome string `json:"outcome"` + FailureReason string `json:"failure_reason"` Retryable bool `json:"retryable"` NetworkRequestAttempted bool `json:"network_request_attempted"` ModelCalled bool `json:"model_called"` @@ -91,6 +92,7 @@ type ModelHarnessQualificationView struct { Model string `json:"model"` Status string `json:"status"` Outcome string `json:"outcome"` + FailureReason string `json:"failure_reason"` Retryable bool `json:"retryable"` NetworkRequestAttempted bool `json:"network_request_attempted"` ModelCalls int `json:"model_calls"` diff --git a/internal/llm/anthropic.go b/internal/llm/anthropic.go index 4fb063de..14321f56 100644 --- a/internal/llm/anthropic.go +++ b/internal/llm/anthropic.go @@ -16,8 +16,6 @@ import ( "time" "unicode" "unicode/utf8" - - "cyberagent-workbench/internal/redact" ) const AnthropicVersion = "2023-06-01" @@ -25,6 +23,7 @@ const AnthropicVersion = "2023-06-01" const ( maxProviderBaseURLBytes = 2048 maxProviderAPIKeyBytes = 16 * 1024 + defaultProviderTimeout = 60 * time.Second ) type AnthropicCompatibleConfig struct { @@ -73,12 +72,12 @@ func NewAnthropicCompatibleProvider(config AnthropicCompatibleConfig) (*Anthropi } func providerHTTPClient(source *http.Client) *http.Client { - client := &http.Client{Timeout: 60 * time.Second} + client := &http.Client{Timeout: defaultProviderTimeout} if source != nil { copy := *source client = © - if client.Timeout <= 0 { - client.Timeout = 60 * time.Second + if client.Timeout <= 0 || client.Timeout > defaultProviderTimeout { + client.Timeout = defaultProviderTimeout } } client.CheckRedirect = func(_ *http.Request, _ []*http.Request) error { @@ -229,9 +228,12 @@ func (p *AnthropicCompatibleProvider) Chat(ctx context.Context, req ChatRequest) return &ChatResponse{ Text: text, ToolCalls: toolCalls, - Raw: raw, - Model: parsed.ModelOrDefault(model), - Provider: p.name, + // Keep the upstream response inside the adapter. Raw model payloads are + // not part of the Supervisor contract and must not cross persistence or + // activity boundaries. + Raw: nil, + Model: parsed.ModelOrDefault(model), + Provider: p.name, Usage: Usage{ InputTokens: parsed.Usage.InputTokens, OutputTokens: parsed.Usage.OutputTokens, @@ -241,21 +243,36 @@ func (p *AnthropicCompatibleProvider) Chat(ctx context.Context, req ChatRequest) } func anthropicHTTPError(provider string, statusCode int, retryAfterHeader string, raw []byte) *ProviderError { + // The response body is deliberately ignored. Provider error text may be + // persisted by the Supervisor, and a remote endpoint can echo prompts, + // tool arguments, credentials, or arbitrary private content in that body. + _ = raw kind := OutcomePermanent + reason := ProviderFailureProtocolIncompatible switch statusCode { + case http.StatusUnauthorized, http.StatusForbidden: + reason = ProviderFailureAuthentication + case http.StatusNotFound: + // A bare 404 can also mean that the configured Anthropic-compatible + // endpoint does not implement /v1/messages. Without an exact safe wire + // code, keep the conservative protocol classification. + reason = ProviderFailureProtocolIncompatible case http.StatusTooManyRequests: kind = OutcomeRateLimited + reason = ProviderFailureRateLimit + case http.StatusServiceUnavailable, 529: + kind = OutcomeRetryable + reason = ProviderFailureCapacity case http.StatusRequestTimeout, http.StatusTooEarly, http.StatusInternalServerError, - http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout, 529: + http.StatusBadGateway, http.StatusGatewayTimeout: kind = OutcomeRetryable + reason = ProviderFailureNetwork } message := fmt.Sprintf("returned HTTP %d", statusCode) - if detail := trimForError(redact.String(string(raw)), 700); detail != "" { - message += ": " + detail - } err := NewProviderError(kind, provider, message, nil) err.StatusCode = statusCode err.RetryAfter = parseRetryAfter(retryAfterHeader, time.Now()) + err.Reason = reason return err } @@ -796,11 +813,3 @@ func fallbackModelInfo(provider string, model string) []ModelInfo { Capabilities: []string{"chat", "tools"}, }} } - -func trimForError(value string, limit int) string { - value = strings.TrimSpace(value) - if len(value) <= limit { - return value - } - return value[:limit] + "..." -} diff --git a/internal/llm/anthropic_test.go b/internal/llm/anthropic_test.go index 3cd46e3f..cbb2974e 100644 --- a/internal/llm/anthropic_test.go +++ b/internal/llm/anthropic_test.go @@ -374,11 +374,18 @@ func TestAnthropicCompatibleProviderClassifiesHTTPFailures(t *testing.T) { statusCode int retryAfter string want Outcome + wantReason ProviderFailureReason }{ - {name: "rate limit", statusCode: http.StatusTooManyRequests, retryAfter: "7", want: OutcomeRateLimited}, - {name: "unavailable", statusCode: http.StatusServiceUnavailable, want: OutcomeRetryable}, - {name: "overloaded", statusCode: 529, want: OutcomeRetryable}, - {name: "auth", statusCode: http.StatusUnauthorized, want: OutcomePermanent}, + {name: "rate limit", statusCode: http.StatusTooManyRequests, retryAfter: "7", + want: OutcomeRateLimited, wantReason: ProviderFailureRateLimit}, + {name: "unavailable", statusCode: http.StatusServiceUnavailable, + want: OutcomeRetryable, wantReason: ProviderFailureCapacity}, + {name: "overloaded", statusCode: 529, + want: OutcomeRetryable, wantReason: ProviderFailureCapacity}, + {name: "auth", statusCode: http.StatusUnauthorized, + want: OutcomePermanent, wantReason: ProviderFailureAuthentication}, + {name: "ambiguous not found", statusCode: http.StatusNotFound, + want: OutcomePermanent, wantReason: ProviderFailureProtocolIncompatible}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -398,7 +405,8 @@ func TestAnthropicCompatibleProviderClassifiesHTTPFailures(t *testing.T) { } _, err = provider.Chat(context.Background(), ChatRequest{Messages: []Message{{Role: "user", Content: "hello"}}}) var providerErr *ProviderError - if !errors.As(err, &providerErr) || providerErr.Kind != test.want || providerErr.StatusCode != test.statusCode { + if !errors.As(err, &providerErr) || providerErr.Kind != test.want || + providerErr.Reason != test.wantReason || providerErr.StatusCode != test.statusCode { t.Fatalf("unexpected provider error: %#v err=%v", providerErr, err) } if test.retryAfter != "" && providerErr.RetryAfter != 7*time.Second { @@ -408,6 +416,23 @@ func TestAnthropicCompatibleProviderClassifiesHTTPFailures(t *testing.T) { } } +func TestProviderHTTPClientBoundsTimeoutAndDisablesRedirects(t *testing.T) { + source := &http.Client{Timeout: 2 * time.Minute} + client := providerHTTPClient(source) + if client == source || client.Timeout != defaultProviderTimeout || source.Timeout != 2*time.Minute { + t.Fatalf("HTTP client timeout was not safely copied and bounded: source=%s client=%s", + source.Timeout, client.Timeout) + } + request := httptest.NewRequest(http.MethodGet, "https://provider.invalid/next", nil) + if err := client.CheckRedirect(request, nil); !errors.Is(err, http.ErrUseLastResponse) { + t.Fatalf("redirect policy error = %v", err) + } + short := providerHTTPClient(&http.Client{Timeout: 5 * time.Second}) + if short.Timeout != 5*time.Second { + t.Fatalf("short caller timeout was widened: %s", short.Timeout) + } +} + func TestAnthropicCompatibleProviderRejectsMalformedAndEmptyResponses(t *testing.T) { for _, body := range []string{`not-json`, `{"model":"model","content":[]}`} { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -428,10 +453,13 @@ func TestAnthropicCompatibleProviderRejectsMalformedAndEmptyResponses(t *testing } } -func TestAnthropicHTTPErrorRedactsResponseBody(t *testing.T) { +func TestAnthropicHTTPErrorDoesNotProjectResponseBody(t *testing.T) { token := "t" + "p-" + strings.Repeat("b", 40) - err := anthropicHTTPError("mimo", http.StatusTooManyRequests, "", []byte("MIMO_API_KEY="+token)) - if strings.Contains(err.Error(), token[:12]) || !strings.Contains(err.Error(), "[REDACTED:") { + marker := "remote-marker-that-is-not-secret-shaped" + err := anthropicHTTPError("mimo", http.StatusTooManyRequests, "", + []byte("MIMO_API_KEY="+token+" prompt="+marker)) + if strings.Contains(err.Error(), token[:12]) || strings.Contains(err.Error(), marker) || + err.Reason != ProviderFailureRateLimit { t.Fatalf("HTTP error leaked response secret: %q", err.Error()) } } diff --git a/internal/llm/harness.go b/internal/llm/harness.go index a5d6826e..07e69ed0 100644 --- a/internal/llm/harness.go +++ b/internal/llm/harness.go @@ -12,9 +12,10 @@ import ( const ModelHarnessProtocolVersion = "model_harness.v1" const ( - HarnessTransportMock = "mock" - HarnessTransportAnthropicMessages = "anthropic_messages" - HarnessTransportProviderContract = "provider_contract" + HarnessTransportMock = "mock" + HarnessTransportAnthropicMessages = "anthropic_messages" + HarnessTransportOpenAIChatCompletions = "openai_chat_completions" + HarnessTransportProviderContract = "provider_contract" ) const ( @@ -83,6 +84,7 @@ func (h ModelHarness) Validate() error { } switch h.TransportProtocol { case HarnessTransportMock, HarnessTransportAnthropicMessages, + HarnessTransportOpenAIChatCompletions, HarnessTransportProviderContract: default: return errors.New("model Harness transport protocol is invalid") diff --git a/internal/llm/openai.go b/internal/llm/openai.go new file mode 100644 index 00000000..e0a09abe --- /dev/null +++ b/internal/llm/openai.go @@ -0,0 +1,988 @@ +package llm + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net" + "net/http" + "sort" + "strings" + "time" + "unicode" + "unicode/utf8" + + "cyberagent-workbench/internal/redact" +) + +const ( + defaultOpenAIModel = "gpt-4.1-mini" + maxOpenAIResponseBytes = 2 << 20 + maxOpenAIStreamLineBytes = 1 << 20 + maxOpenAIStreamEventBytes = 1 << 20 + maxOpenAIModels = 4096 + maxOpenAIModelBytes = 512 +) + +// OpenAICompatibleConfig configures an OpenAI Chat Completions compatible +// endpoint. Credentials remain in memory and are never copied into requests, +// responses, errors, or Harness metadata. +type OpenAICompatibleConfig struct { + Name string + BaseURL string + APIKey string + DefaultModel string + HTTPClient *http.Client +} + +// OpenAICompatibleProvider implements the protocol-neutral Provider contract +// using the OpenAI Chat Completions wire protocol. +type OpenAICompatibleProvider struct { + name string + baseURL string + apiKey string + defaultModel string + client *http.Client +} + +func NewOpenAICompatibleProvider(config OpenAICompatibleConfig) (*OpenAICompatibleProvider, error) { + name := strings.TrimSpace(config.Name) + if name == "" { + name = "openai_compatible" + } + baseURL, err := normalizeProviderBaseURL(config.BaseURL, name) + if err != nil { + return nil, err + } + if err := validateProviderAPIKey(config.APIKey, name); err != nil { + return nil, err + } + model := strings.TrimSpace(config.DefaultModel) + if model == "" { + model = defaultOpenAIModel + } + model, err = normalizeOpenAIModel(model) + if err != nil { + return nil, fmt.Errorf("default model for provider %s is invalid", name) + } + return &OpenAICompatibleProvider{ + name: name, baseURL: baseURL, apiKey: config.APIKey, + defaultModel: model, client: providerHTTPClient(config.HTTPClient), + }, nil +} + +func (p *OpenAICompatibleProvider) Name() string { return p.name } + +func (p *OpenAICompatibleProvider) ListModels(ctx context.Context) ([]ModelInfo, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.endpoint("/v1/models"), nil) + if err != nil { + return nil, openAILocalError(p.name, "could not create model-list request") + } + p.addHeaders(req, false) + resp, err := p.client.Do(req) + if err != nil { + return nil, openAITransportError(ctx, p.name) + } + defer resp.Body.Close() + raw, err := readOpenAIBody(resp.Body) + if err != nil { + return nil, openAIReadError(ctx, p.name, "could not read model-list response", err) + } + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return nil, openAIHTTPError(p.name, resp.StatusCode, resp.Header.Get("Retry-After"), raw) + } + if !utf8.Valid(raw) { + return nil, openAIProtocolError(p.name, "returned a non-UTF-8 model list") + } + var payload struct { + Data []struct { + ID string `json:"id"` + } `json:"data"` + Error *openAIError `json:"error,omitempty"` + } + if err := json.Unmarshal(raw, &payload); err != nil { + return nil, openAIProtocolError(p.name, "returned a malformed model list") + } + if payload.Error != nil { + return nil, openAIWireError(p.name, *payload.Error) + } + if len(payload.Data) == 0 || len(payload.Data) > maxOpenAIModels { + return nil, openAIProtocolError(p.name, "returned an invalid model list") + } + models := make([]ModelInfo, 0, len(payload.Data)) + seen := make(map[string]struct{}, len(payload.Data)) + for _, item := range payload.Data { + id, err := normalizeOpenAIModel(item.ID) + if err != nil { + return nil, openAIProtocolError(p.name, "returned an invalid model identifier") + } + if _, exists := seen[id]; exists { + continue + } + seen[id] = struct{}{} + models = append(models, ModelInfo{ + ID: id, DisplayName: id, Provider: p.name, + // The Models endpoint exposes identity and ownership, not a reliable + // per-model Chat/Tool/JSON capability contract. Those capabilities + // are established by the configured model's Harness qualification. + Capabilities: []string{}, + }) + } + if len(models) == 0 { + return nil, openAIProtocolError(p.name, "returned an empty model list") + } + return models, nil +} + +func (p *OpenAICompatibleProvider) Chat(ctx context.Context, request ChatRequest) (*ChatResponse, error) { + model, wire, err := p.prepareRequest(request, false) + if err != nil { + return nil, openAILocalError(p.name, "could not prepare request") + } + payload, err := json.Marshal(wire) + if err != nil { + return nil, openAILocalError(p.name, "could not encode request") + } + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, + p.endpoint("/v1/chat/completions"), bytes.NewReader(payload)) + if err != nil { + return nil, openAILocalError(p.name, "could not create request") + } + p.addHeaders(httpReq, false) + resp, err := p.client.Do(httpReq) + if err != nil { + return nil, openAITransportError(ctx, p.name) + } + defer resp.Body.Close() + raw, err := readOpenAIBody(resp.Body) + if err != nil { + return nil, openAIReadError(ctx, p.name, "could not read response", err) + } + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return nil, openAIHTTPError(p.name, resp.StatusCode, resp.Header.Get("Retry-After"), raw) + } + if !utf8.Valid(raw) { + return nil, openAIProtocolError(p.name, "returned non-UTF-8 JSON") + } + var parsed openAIChatResponse + if err := json.Unmarshal(raw, &parsed); err != nil { + return nil, openAIProtocolError(p.name, "returned malformed JSON") + } + if parsed.Error != nil { + return nil, openAIWireError(p.name, *parsed.Error) + } + return p.normalizeResponse(model, parsed) +} + +func (p *OpenAICompatibleProvider) StreamChat(ctx context.Context, request ChatRequest) (<-chan ChatChunk, error) { + model, wire, err := p.prepareRequest(request, true) + if err != nil { + return nil, openAILocalError(p.name, "could not prepare streaming request") + } + payload, err := json.Marshal(wire) + if err != nil { + return nil, openAILocalError(p.name, "could not encode streaming request") + } + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, + p.endpoint("/v1/chat/completions"), bytes.NewReader(payload)) + if err != nil { + return nil, openAILocalError(p.name, "could not create streaming request") + } + p.addHeaders(httpReq, true) + resp, err := p.client.Do(httpReq) + if err != nil { + return nil, openAITransportError(ctx, p.name) + } + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + defer resp.Body.Close() + raw, readErr := readOpenAIBody(resp.Body) + if readErr != nil { + return nil, openAIReadError(ctx, p.name, "could not read error response", readErr) + } + return nil, openAIHTTPError(p.name, resp.StatusCode, resp.Header.Get("Retry-After"), raw) + } + chunks := make(chan ChatChunk, 8) + go p.readStream(ctx, resp.Body, model, chunks) + return chunks, nil +} + +func (p *OpenAICompatibleProvider) SupportsTools(model string) bool { + return strings.TrimSpace(model) == p.defaultModel +} + +func (p *OpenAICompatibleProvider) SupportsVision(string) bool { return false } + +func (p *OpenAICompatibleProvider) SupportsJSONMode(model string) bool { + return strings.TrimSpace(model) == p.defaultModel +} + +func (p *OpenAICompatibleProvider) DescribeModelHarness(model string) ModelHarness { + model = strings.TrimSpace(model) + if model == "" { + model = p.defaultModel + } + return ModelHarness{ + ProtocolVersion: ModelHarnessProtocolVersion, + TransportProtocol: HarnessTransportOpenAIChatCompletions, + ToolStrategy: HarnessToolStrategyNative, JSONStrategy: HarnessJSONStrategyNative, + QualificationStatus: HarnessQualificationRequired, + BindingDigest: harnessBindingDigest(p.name, p.baseURL, model, + HarnessTransportOpenAIChatCompletions, HarnessToolStrategyNative, + HarnessJSONStrategyNative), + } +} + +func (p *OpenAICompatibleProvider) prepareRequest(request ChatRequest, stream bool) (string, openAIChatRequest, error) { + model := strings.TrimSpace(request.Model) + if model == "" { + model = p.defaultModel + } + var err error + model, err = normalizeOpenAIModel(model) + if err != nil { + return "", openAIChatRequest{}, err + } + if math.IsNaN(request.Temperature) || math.IsInf(request.Temperature, 0) || + request.Temperature < 0 || request.Temperature > 2 { + return "", openAIChatRequest{}, errors.New("temperature is outside the supported range") + } + maxTokens := request.MaxTokens + if maxTokens <= 0 { + maxTokens = 1024 + } + if maxTokens > 1_000_000 { + return "", openAIChatRequest{}, errors.New("max tokens exceeds the provider request limit") + } + wire := openAIChatRequest{Model: model, MaxTokens: maxTokens, Stream: stream} + if request.Temperature > 0 { + wire.Temperature = &request.Temperature + } + if stream { + wire.StreamOptions = &openAIStreamOptions{IncludeUsage: true} + } + if request.JSONMode { + wire.ResponseFormat = &openAIResponseFormat{Type: "json_object"} + } + for index, message := range request.Messages { + mapped, err := openAIMessages(message) + if err != nil { + return "", openAIChatRequest{}, fmt.Errorf("invalid message at index %d", index) + } + wire.Messages = append(wire.Messages, mapped...) + } + if len(wire.Messages) == 0 { + content := "Hello" + wire.Messages = append(wire.Messages, openAIMessage{Role: "user", Content: &content}) + } + if len(request.Tools) > MaxProviderToolCalls { + return "", openAIChatRequest{}, errors.New("tool specification count exceeds the provider limit") + } + for index, spec := range request.Tools { + name := strings.TrimSpace(spec.Name) + parameters := append(json.RawMessage(nil), bytes.TrimSpace(spec.Parameters)...) + if err := validateOpenAIToolName(name); err != nil || len(parameters) == 0 || + len(parameters) > MaxProviderToolPayloadSize || !utf8.Valid(parameters) || !json.Valid(parameters) { + return "", openAIChatRequest{}, fmt.Errorf("invalid tool specification at index %d", index) + } + wire.Tools = append(wire.Tools, openAITool{Type: "function", Function: openAIFunctionSpec{ + Name: name, Description: strings.TrimSpace(spec.Description), Parameters: parameters, + }}) + } + return model, wire, nil +} + +func openAIMessages(message Message) ([]openAIMessage, error) { + role := strings.ToLower(strings.TrimSpace(message.Role)) + content := strings.TrimSpace(message.Content) + if content == "" && len(message.ToolCalls) == 0 && len(message.ToolResults) == 0 { + return nil, nil + } + switch role { + case "system": + if len(message.ToolCalls) != 0 || len(message.ToolResults) != 0 || content == "" { + return nil, errors.New("system message has invalid structured content") + } + return []openAIMessage{{Role: "system", Content: &content}}, nil + case "assistant": + if len(message.ToolResults) != 0 { + return nil, errors.New("assistant message cannot contain tool results") + } + calls, err := NormalizeToolCalls(message.ToolCalls) + if err != nil { + return nil, err + } + mapped := openAIMessage{Role: "assistant"} + if content != "" { + mapped.Content = &content + } + for _, call := range calls { + mapped.ToolCalls = append(mapped.ToolCalls, openAIToolCall{ + ID: call.ID, Type: "function", Function: openAIFunctionCall{ + Name: call.Name, Arguments: string(call.Arguments), + }, + }) + } + return []openAIMessage{mapped}, nil + case "user": + if len(message.ToolCalls) != 0 { + return nil, errors.New("user message cannot contain tool calls") + } + mapped := make([]openAIMessage, 0, len(message.ToolResults)+1) + for _, result := range message.ToolResults { + normalized, err := NormalizeToolResult(result) + if err != nil { + return nil, err + } + resultContent := normalized.Content + mapped = append(mapped, openAIMessage{ + Role: "tool", Content: &resultContent, ToolCallID: normalized.ToolCallID, + }) + } + if content != "" { + mapped = append(mapped, openAIMessage{Role: "user", Content: &content}) + } + if len(mapped) == 0 { + return nil, errors.New("user message has no content") + } + return mapped, nil + default: + return nil, errors.New("message role is unsupported") + } +} + +func (p *OpenAICompatibleProvider) normalizeResponse(defaultModel string, response openAIChatResponse) (*ChatResponse, error) { + if len(response.Choices) != 1 || response.Choices[0].Index == nil || *response.Choices[0].Index != 0 { + return nil, openAIProtocolError(p.name, "returned an invalid choice list") + } + choice := response.Choices[0] + if choice.Message.Role != "" && choice.Message.Role != "assistant" { + return nil, openAIProtocolError(p.name, "returned an invalid message role") + } + text := "" + if choice.Message.Content != nil { + text = *choice.Message.Content + } + if !utf8.ValidString(text) || len(text) > MaxModelOutputBytes { + return nil, openAIProtocolError(p.name, "returned invalid response text") + } + calls, err := normalizeOpenAIToolCalls(choice.Message.ToolCalls) + if err != nil { + return nil, openAIProtocolError(p.name, "returned invalid tool calls") + } + if err := validateOpenAIFinishReason(choice.FinishReason, text != "", len(calls)); err != nil { + return nil, openAIProtocolError(p.name, "returned an incompatible finish reason") + } + if response.Usage == nil { + return nil, openAIProtocolError(p.name, "omitted token usage") + } + usage, err := normalizeOpenAIUsage(*response.Usage) + if err != nil { + return nil, openAIProtocolError(p.name, "returned invalid token usage") + } + if _, err := normalizeOpenAIModel(response.Model); err != nil { + return nil, openAIProtocolError(p.name, "returned an invalid model identity") + } + return &ChatResponse{ + Text: text, ToolCalls: calls, Usage: usage, Raw: nil, + // OpenAI may resolve a stable alias to a dated snapshot in the wire + // response. Keep the selected Router identity stable while still + // requiring a valid upstream identity above. + Model: defaultModel, Provider: p.name, + }, nil +} + +func normalizeOpenAIToolCalls(wire []openAIToolCall) ([]ToolCall, error) { + calls := make([]ToolCall, 0, len(wire)) + for _, item := range wire { + if item.Type != "" && item.Type != "function" { + return nil, errors.New("tool call type is unsupported") + } + arguments := item.Function.Arguments + if strings.TrimSpace(arguments) == "" { + arguments = `{}` + } + calls = append(calls, ToolCall{ID: item.ID, Name: item.Function.Name, + Arguments: json.RawMessage(arguments)}) + } + return NormalizeToolCalls(calls) +} + +func normalizeOpenAIUsage(wire openAIUsage) (Usage, error) { + if wire.PromptTokens == nil || wire.CompletionTokens == nil || wire.TotalTokens == nil { + return Usage{}, errors.New("token count is missing") + } + usage := Usage{InputTokens: *wire.PromptTokens, OutputTokens: *wire.CompletionTokens} + if usage.InputTokens < 0 || usage.OutputTokens < 0 || *wire.TotalTokens < 0 { + return Usage{}, errors.New("negative token count") + } + maxInt := int(^uint(0) >> 1) + if usage.InputTokens > maxInt-usage.OutputTokens { + return Usage{}, errors.New("token count overflow") + } + usage.TotalTokens = usage.InputTokens + usage.OutputTokens + if *wire.TotalTokens != usage.TotalTokens { + return Usage{}, errors.New("inconsistent total token count") + } + return usage, usage.Validate() +} + +func validateOpenAIFinishReason(reason string, hasText bool, toolCount int) error { + switch strings.TrimSpace(reason) { + case "stop": + if toolCount != 0 || !hasText { + return errors.New("stop finish did not contain text") + } + case "tool_calls": + if toolCount == 0 { + return errors.New("tool finish did not contain calls") + } + default: + return errors.New("finish reason is unsupported or incomplete") + } + return nil +} + +func normalizeOpenAIModel(value string) (string, error) { + value = strings.TrimSpace(value) + if value == "" || !utf8.ValidString(value) || len([]byte(value)) > maxOpenAIModelBytes || + strings.ContainsRune(value, 0) || redact.String(value) != value { + return "", errors.New("model identifier is invalid") + } + for _, current := range value { + if unicode.IsControl(current) || unicode.IsSpace(current) { + return "", errors.New("model identifier is invalid") + } + } + return value, nil +} + +func validateOpenAIToolName(name string) error { + _, err := NormalizeToolCall(ToolCall{ID: "spec", Name: name, Arguments: json.RawMessage(`{}`)}) + return err +} + +func (p *OpenAICompatibleProvider) endpoint(path string) string { + if strings.HasSuffix(p.baseURL, path) { + return p.baseURL + } + if strings.HasSuffix(p.baseURL, "/v1") && strings.HasPrefix(path, "/v1/") { + return p.baseURL + strings.TrimPrefix(path, "/v1") + } + return p.baseURL + path +} + +func (p *OpenAICompatibleProvider) addHeaders(request *http.Request, stream bool) { + request.Header.Set("Content-Type", "application/json") + if stream { + request.Header.Set("Accept", "text/event-stream") + } else { + request.Header.Set("Accept", "application/json") + } + request.Header.Set("Authorization", "Bearer "+p.apiKey) +} + +func readOpenAIBody(reader io.Reader) ([]byte, error) { + raw, err := io.ReadAll(io.LimitReader(reader, maxOpenAIResponseBytes+1)) + if err != nil { + return nil, err + } + if len(raw) > maxOpenAIResponseBytes { + return nil, errors.New("response exceeds size limit") + } + return raw, nil +} + +func openAILocalError(provider string, message string) *ProviderError { + err := NewProviderError(OutcomePermanent, provider, message, nil) + err.Reason = ProviderFailureProtocolIncompatible + return err +} + +func openAIProtocolError(provider string, message string) *ProviderError { + err := NewProviderError(OutcomeInvalidResponse, provider, message, nil) + err.Reason = ProviderFailureProtocolIncompatible + return err +} + +func openAITransportError(ctx context.Context, provider string) *ProviderError { + if ctx.Err() != nil { + err := NewProviderError(OutcomeCancelled, provider, "request was cancelled", nil) + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + err.Reason = ProviderFailureNetwork + } + return err + } + err := NewProviderError(OutcomeRetryable, provider, "request failed", nil) + err.Reason = ProviderFailureNetwork + return err +} + +func openAIReadError(ctx context.Context, provider string, message string, source error) *ProviderError { + if ctx.Err() != nil { + return openAITransportError(ctx, provider) + } + var network net.Error + if errors.As(source, &network) { + err := NewProviderError(OutcomeRetryable, provider, message, nil) + err.Reason = ProviderFailureNetwork + return err + } + return openAIProtocolError(provider, message) +} + +func openAIHTTPError(provider string, statusCode int, retryAfter string, raw []byte) *ProviderError { + kind := OutcomePermanent + reason := ProviderFailureProtocolIncompatible + switch statusCode { + case http.StatusUnauthorized, http.StatusForbidden: + reason = ProviderFailureAuthentication + case http.StatusTooManyRequests: + kind, reason = OutcomeRateLimited, ProviderFailureRateLimit + case http.StatusServiceUnavailable, 529: + kind, reason = OutcomeRetryable, ProviderFailureCapacity + case http.StatusRequestTimeout, http.StatusTooEarly, http.StatusInternalServerError, + http.StatusBadGateway, http.StatusGatewayTimeout: + kind, reason = OutcomeRetryable, ProviderFailureNetwork + } + var envelope struct { + Error *openAIError `json:"error"` + } + // The HTTP status is authoritative for standard status classes. Some + // compatible endpoints reuse broad error types such as server_error even + // for authentication and throttling responses. + wireReason := ProviderFailureNone + if utf8.Valid(raw) && json.Unmarshal(raw, &envelope) == nil && envelope.Error != nil { + wireReason = classifyOpenAIWireError(*envelope.Error) + } + if statusCode == http.StatusNotFound { + // A bare 404 normally means the configured Chat Completions path is + // incompatible. Only a canonical, content-free error code/type can + // prove that the requested model itself was not found. + if wireReason == ProviderFailureModelNotFound { + reason = wireReason + } + } else if reason == ProviderFailureProtocolIncompatible && wireReason != ProviderFailureNone { + reason = wireReason + switch wireReason { + case ProviderFailureRateLimit: + kind = OutcomeRateLimited + case ProviderFailureCapacity, ProviderFailureNetwork: + kind = OutcomeRetryable + default: + kind = OutcomePermanent + } + } + err := NewProviderError(kind, provider, fmt.Sprintf("returned HTTP %d", statusCode), nil) + err.StatusCode = statusCode + err.RetryAfter = parseRetryAfter(retryAfter, time.Now()) + err.Reason = reason + return err +} + +func openAIWireError(provider string, wire openAIError) *ProviderError { + reason := classifyOpenAIWireError(wire) + kind := OutcomePermanent + switch reason { + case ProviderFailureRateLimit: + kind = OutcomeRateLimited + case ProviderFailureCapacity, ProviderFailureNetwork: + kind = OutcomeRetryable + case ProviderFailureNone: + reason = ProviderFailureProtocolIncompatible + } + err := NewProviderError(kind, provider, "returned a provider error", nil) + err.Reason = reason + return err +} + +func classifyOpenAIWireError(wire openAIError) ProviderFailureReason { + code := "" + if len(wire.Code) > 0 { + _ = json.Unmarshal(wire.Code, &code) + } + // Prefer the specific code over the broader error type. Both are treated as + // untrusted wire values and must match a closed canonical allowlist; prefix + // or substring matches could turn attacker-controlled near-misses into false + // authentication, capacity, or model-not-found facts. + for _, signal := range []string{code, wire.Type} { + switch strings.ToLower(strings.TrimSpace(signal)) { + case "rate_limit", "rate_limit_error", "rate_limit_exceeded", + "insufficient_quota", "quota_exceeded": + return ProviderFailureRateLimit + case "server_error", "overloaded", "overloaded_error", + "capacity_exceeded": + return ProviderFailureCapacity + case "model_not_found", "model_not_exist", "model_not_exists": + return ProviderFailureModelNotFound + case "authentication_error", "invalid_api_key", "incorrect_api_key", + "api_key_invalid", "permission_denied", "insufficient_permissions": + return ProviderFailureAuthentication + case "request_timeout", "timeout_error": + return ProviderFailureNetwork + } + } + return ProviderFailureNone +} + +type openAIChatRequest struct { + Model string `json:"model"` + Messages []openAIMessage `json:"messages"` + Tools []openAITool `json:"tools,omitempty"` + Temperature *float64 `json:"temperature,omitempty"` + MaxTokens int `json:"max_tokens"` + ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"` + Stream bool `json:"stream,omitempty"` + StreamOptions *openAIStreamOptions `json:"stream_options,omitempty"` +} + +type openAIMessage struct { + Role string `json:"role"` + Content *string `json:"content,omitempty"` + ToolCalls []openAIToolCall `json:"tool_calls,omitempty"` + ToolCallID string `json:"tool_call_id,omitempty"` +} + +type openAITool struct { + Type string `json:"type"` + Function openAIFunctionSpec `json:"function"` +} + +type openAIFunctionSpec struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + Parameters json.RawMessage `json:"parameters"` +} + +type openAIToolCall struct { + ID string `json:"id"` + Type string `json:"type"` + Function openAIFunctionCall `json:"function"` +} + +type openAIFunctionCall struct { + Name string `json:"name"` + Arguments string `json:"arguments"` +} + +type openAIResponseFormat struct { + Type string `json:"type"` +} + +type openAIStreamOptions struct { + IncludeUsage bool `json:"include_usage"` +} + +type openAIUsage struct { + PromptTokens *int `json:"prompt_tokens"` + CompletionTokens *int `json:"completion_tokens"` + TotalTokens *int `json:"total_tokens"` +} + +type openAIError struct { + Message string `json:"message"` + Type string `json:"type"` + Code json.RawMessage `json:"code"` +} + +type openAIChatResponse struct { + Model string `json:"model"` + Choices []openAIChoice `json:"choices"` + Usage *openAIUsage `json:"usage"` + Error *openAIError `json:"error,omitempty"` +} + +type openAIChoice struct { + Index *int `json:"index"` + Message openAIMessage `json:"message"` + Delta openAIStreamDelta `json:"delta"` + FinishReason string `json:"finish_reason"` +} + +type openAIStreamDelta struct { + Role string `json:"role"` + Content *string `json:"content"` + ToolCalls []openAIStreamToolCall `json:"tool_calls"` +} + +type openAIStreamToolCall struct { + Index *int `json:"index"` + ID string `json:"id"` + Type string `json:"type"` + Function openAIFunctionCall `json:"function"` +} + +type openAIStreamResponse struct { + Model string `json:"model"` + Choices []openAIChoice `json:"choices"` + Usage *openAIUsage `json:"usage"` + Error *openAIError `json:"error,omitempty"` +} + +type openAIStreamToolState struct { + id string + name string + idSeen bool + nameSeen bool + arguments strings.Builder +} + +type openAIStreamState struct { + provider string + model string + upstreamModel string + responseModel bool + finished bool + finishReason string + usage *Usage + textBytes int + hasTextDelta bool + tools map[int]*openAIStreamToolState +} + +func (s *openAIStreamState) consume(payload []byte) (*ChatChunk, error) { + if !utf8.Valid(payload) { + return nil, openAIProtocolError(s.provider, "returned a non-UTF-8 stream event") + } + var event openAIStreamResponse + if err := json.Unmarshal(payload, &event); err != nil { + return nil, openAIProtocolError(s.provider, "returned a malformed stream event") + } + if event.Error != nil { + return nil, openAIWireError(s.provider, *event.Error) + } + if strings.TrimSpace(event.Model) != "" { + model, err := normalizeOpenAIModel(event.Model) + if err != nil { + return nil, openAIProtocolError(s.provider, "returned an invalid stream model") + } + if s.responseModel && model != s.upstreamModel { + return nil, openAIProtocolError(s.provider, "changed model identity during the stream") + } + s.upstreamModel, s.responseModel = model, true + } + if len(event.Choices) == 0 { + if event.Usage == nil || s.usage != nil || !s.finished || !s.responseModel || + strings.TrimSpace(event.Model) == "" { + return nil, openAIProtocolError(s.provider, "returned an invalid stream choice") + } + usage, err := normalizeOpenAIUsage(*event.Usage) + if err != nil { + return nil, openAIProtocolError(s.provider, "returned invalid stream usage") + } + s.usage = &usage + return nil, nil + } + if len(event.Choices) != 1 || event.Choices[0].Index == nil || *event.Choices[0].Index != 0 || s.finished { + return nil, openAIProtocolError(s.provider, "returned an invalid stream choice") + } + choice := event.Choices[0] + if choice.Delta.Role != "" && choice.Delta.Role != "assistant" { + return nil, openAIProtocolError(s.provider, "returned an invalid stream role") + } + var chunk *ChatChunk + if choice.Delta.Content != nil && *choice.Delta.Content != "" { + text := *choice.Delta.Content + if s.textBytes > MaxModelOutputBytes-len(text) { + return nil, openAIProtocolError(s.provider, "streamed text exceeds the output limit") + } + s.textBytes += len(text) + s.hasTextDelta = true + chunk = &ChatChunk{Text: text} + } + for _, delta := range choice.Delta.ToolCalls { + if err := s.consumeToolDelta(delta); err != nil { + return nil, openAIProtocolError(s.provider, "returned an invalid tool-call delta") + } + } + if choice.FinishReason != "" { + s.finished = true + s.finishReason = strings.TrimSpace(choice.FinishReason) + } + if event.Usage != nil { + // OpenAI emits a final empty-choices usage event when include_usage is + // enabled. Some compatible endpoints attach the same exact usage object + // to the choice that carries finish_reason. Accept both terminal forms, + // while rejecting early, duplicate, or structurally incomplete usage. + if s.usage != nil || !s.finished || !s.responseModel || + strings.TrimSpace(event.Model) == "" { + return nil, openAIProtocolError(s.provider, "returned usage before the final stream event") + } + usage, err := normalizeOpenAIUsage(*event.Usage) + if err != nil { + return nil, openAIProtocolError(s.provider, "returned invalid stream usage") + } + s.usage = &usage + } + return chunk, nil +} + +func (s *openAIStreamState) consumeToolDelta(delta openAIStreamToolCall) error { + if delta.Index == nil || *delta.Index < 0 || *delta.Index >= MaxProviderToolCalls { + return errors.New("tool-call index is invalid") + } + if delta.Type != "" && delta.Type != "function" { + return errors.New("tool-call type is unsupported") + } + if s.tools == nil { + s.tools = make(map[int]*openAIStreamToolState) + } + state, exists := s.tools[*delta.Index] + if !exists { + if len(s.tools) >= MaxProviderToolCalls { + return errors.New("too many tool calls") + } + state = &openAIStreamToolState{} + s.tools[*delta.Index] = state + } + if delta.ID != "" { + if state.idSeen && delta.ID != state.id { + return errors.New("tool-call id changed") + } + state.id, state.idSeen = delta.ID, true + } + if delta.Function.Name != "" { + if state.nameSeen && delta.Function.Name != state.name { + return errors.New("tool-call name changed") + } + state.name, state.nameSeen = delta.Function.Name, true + } + if !utf8.ValidString(state.id) || !utf8.ValidString(state.name) || + len([]rune(state.id)) > MaxProviderToolIdentity || len([]rune(state.name)) > MaxProviderToolIdentity { + return errors.New("tool-call identity exceeds the limit") + } + if state.arguments.Len() > MaxProviderToolPayloadSize-len(delta.Function.Arguments) { + return errors.New("tool-call arguments exceed the limit") + } + _, _ = state.arguments.WriteString(delta.Function.Arguments) + return nil +} + +func (s *openAIStreamState) finalChunk() (ChatChunk, error) { + if !s.finished { + return ChatChunk{}, openAIProtocolError(s.provider, "stream ended without a finish reason") + } + if s.usage == nil { + return ChatChunk{}, openAIProtocolError(s.provider, "stream omitted token usage") + } + if !s.responseModel { + return ChatChunk{}, openAIProtocolError(s.provider, "stream omitted its model identity") + } + indices := make([]int, 0, len(s.tools)) + for index := range s.tools { + indices = append(indices, index) + } + sort.Ints(indices) + calls := make([]ToolCall, 0, len(indices)) + for expected, index := range indices { + if index != expected { + return ChatChunk{}, openAIProtocolError(s.provider, "stream returned sparse tool-call indices") + } + state := s.tools[index] + arguments := state.arguments.String() + if strings.TrimSpace(arguments) == "" { + arguments = `{}` + } + calls = append(calls, ToolCall{ID: state.id, Name: state.name, + Arguments: json.RawMessage(arguments)}) + } + var err error + calls, err = NormalizeToolCalls(calls) + if err != nil { + return ChatChunk{}, openAIProtocolError(s.provider, "stream returned invalid tool calls") + } + if err := validateOpenAIFinishReason(s.finishReason, s.hasTextDelta, len(calls)); err != nil { + return ChatChunk{}, openAIProtocolError(s.provider, "stream returned an incompatible finish reason") + } + usage := *s.usage + return ChatChunk{Done: true, ToolCalls: calls, Usage: &usage, + Model: s.model, Provider: s.provider}, nil +} + +func (p *OpenAICompatibleProvider) readStream(ctx context.Context, body io.ReadCloser, + model string, chunks chan<- ChatChunk, +) { + defer close(chunks) + defer body.Close() + state := openAIStreamState{provider: p.name, model: model} + scanner := bufio.NewScanner(body) + scanner.Buffer(make([]byte, 64*1024), maxOpenAIStreamLineBytes) + dataLines := make([]string, 0, 1) + eventBytes := 0 + stopped := false + sendError := func(err error) bool { + return p.sendStreamChunk(ctx, chunks, ChatChunk{Err: err}) + } + flush := func() bool { + if len(dataLines) == 0 { + return true + } + payload := strings.Join(dataLines, "\n") + dataLines = dataLines[:0] + eventBytes = 0 + if strings.TrimSpace(payload) == "[DONE]" { + final, err := state.finalChunk() + if err != nil { + _ = sendError(err) + return false + } + stopped = true + return p.sendStreamChunk(ctx, chunks, final) + } + chunk, err := state.consume([]byte(payload)) + if err != nil { + _ = sendError(err) + return false + } + return chunk == nil || p.sendStreamChunk(ctx, chunks, *chunk) + } + for scanner.Scan() { + line := scanner.Text() + if line == "" { + if !flush() || stopped { + return + } + continue + } + if strings.HasPrefix(line, ":") { + continue + } + field, value, found := strings.Cut(line, ":") + if !found || field != "data" { + continue + } + value = strings.TrimPrefix(value, " ") + if !utf8.ValidString(value) || eventBytes > maxOpenAIStreamEventBytes-len(value)-1 { + _ = sendError(openAIProtocolError(p.name, "stream event exceeds its UTF-8 size limit")) + return + } + eventBytes += len(value) + 1 + dataLines = append(dataLines, value) + } + if len(dataLines) != 0 && !flush() { + return + } + if stopped || ctx.Err() != nil { + return + } + if scanner.Err() != nil { + _ = sendError(openAIReadError(ctx, p.name, "stream read failed", scanner.Err())) + return + } + _ = sendError(openAIProtocolError(p.name, "stream ended before [DONE]")) +} + +func (p *OpenAICompatibleProvider) sendStreamChunk(ctx context.Context, + chunks chan<- ChatChunk, chunk ChatChunk, +) bool { + select { + case <-ctx.Done(): + return false + case chunks <- chunk: + return true + } +} diff --git a/internal/llm/openai_test.go b/internal/llm/openai_test.go new file mode 100644 index 00000000..e6053863 --- /dev/null +++ b/internal/llm/openai_test.go @@ -0,0 +1,661 @@ +package llm + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + "unicode/utf8" +) + +func TestOpenAICompatibleProviderChatJSONAndModelList(t *testing.T) { + var captured openAIChatRequest + var auth string + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + auth = request.Header.Get("Authorization") + switch request.URL.Path { + case "/v1/models": + _, _ = writer.Write([]byte(`{"object":"list","data":[{"id":"gpt-test"},{"id":"gpt-other"}]}`)) + case "/v1/chat/completions": + if err := json.NewDecoder(request.Body).Decode(&captured); err != nil { + t.Error(err) + return + } + _, _ = writer.Write([]byte(`{ + "id":"chatcmpl-safe","model":"gpt-test", + "choices":[{"index":0,"message":{"role":"assistant","content":"{\"status\":\"ok\"}","reasoning_content":"private trace"},"finish_reason":"stop"}], + "usage":{"prompt_tokens":7,"completion_tokens":4,"total_tokens":11} + }`)) + default: + http.NotFound(writer, request) + } + })) + defer server.Close() + provider := newTestOpenAIProvider(t, server.URL) + models, err := provider.ListModels(t.Context()) + if err != nil { + t.Fatal(err) + } + if len(models) != 2 || models[0].ID != "gpt-test" || models[0].Provider != "openai" || + len(models[0].Capabilities) != 0 { + t.Fatalf("unexpected model list: %#v", models) + } + response, err := provider.Chat(t.Context(), ChatRequest{ + Messages: []Message{{Role: "system", Content: "Return JSON."}, {Role: "user", Content: "status"}}, + JSONMode: true, MaxTokens: 73, Metadata: map[string]string{"private": "must-not-cross"}, + }) + if err != nil { + t.Fatal(err) + } + if auth != "Bearer test-secret" { + t.Fatalf("authorization = %q", auth) + } + if captured.Model != "gpt-test" || captured.MaxTokens != 73 || captured.ResponseFormat == nil || + captured.ResponseFormat.Type != "json_object" || len(captured.Messages) != 2 { + t.Fatalf("unexpected request: %#v", captured) + } + encoded, _ := json.Marshal(captured) + if strings.Contains(string(encoded), "must-not-cross") { + t.Fatalf("internal metadata crossed the provider boundary: %s", encoded) + } + if response.Text != `{"status":"ok"}` || response.Model != "gpt-test" || + response.Provider != "openai" || response.Usage.TotalTokens != 11 || response.Raw != nil || + strings.Contains(response.Text, "private trace") { + t.Fatalf("unexpected response: %#v", response) + } +} + +func TestOpenAICompatibleProviderMapsToolsAndMultipleToolResults(t *testing.T) { + var captured openAIChatRequest + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if err := json.NewDecoder(request.Body).Decode(&captured); err != nil { + t.Error(err) + return + } + _, _ = writer.Write([]byte(`{ + "model":"gpt-test","choices":[{"index":0,"message":{"role":"assistant","content":null, + "tool_calls":[ + {"id":"call_c","type":"function","function":{"name":"run_command","arguments":"{\"command\":\"id\"}"}}, + {"id":"call_r","type":"function","function":{"name":"read_file","arguments":"{\"path\":\"README.md\"}"}} + ]},"finish_reason":"tool_calls"}], + "usage":{"prompt_tokens":9,"completion_tokens":5,"total_tokens":14} + }`)) + })) + defer server.Close() + provider := newTestOpenAIProvider(t, server.URL) + response, err := provider.Chat(t.Context(), ChatRequest{ + Messages: []Message{ + {Role: "user", Content: "inspect"}, + {Role: "assistant", ToolCalls: []ToolCall{ + {ID: "old_a", Name: "run_command", Arguments: json.RawMessage(`{"command":"pwd"}`)}, + {ID: "old_b", Name: "read_file", Arguments: json.RawMessage(`{"path":"go.mod"}`)}, + }}, + {Role: "user", Content: "Continue after both results.", ToolResults: []ToolResult{ + {ToolCallID: "old_a", Content: `{"stdout":"repo"}`}, + {ToolCallID: "old_b", Content: `{"content":"module"}`}, + }}, + }, + Tools: []ToolSpec{ + {Name: "run_command", Description: "Run a command", Parameters: json.RawMessage(`{"type":"object"}`)}, + {Name: "read_file", Description: "Read a file", Parameters: json.RawMessage(`{"type":"object"}`)}, + }, + }) + if err != nil { + t.Fatal(err) + } + if len(captured.Messages) != 5 { + t.Fatalf("message expansion = %#v", captured.Messages) + } + roles := []string{"user", "assistant", "tool", "tool", "user"} + for index, role := range roles { + if captured.Messages[index].Role != role { + t.Fatalf("message %d role = %q, want %q", index, captured.Messages[index].Role, role) + } + } + if captured.Messages[2].ToolCallID != "old_a" || captured.Messages[3].ToolCallID != "old_b" || + captured.Messages[4].Content == nil || *captured.Messages[4].Content != "Continue after both results." { + t.Fatalf("tool-result mapping = %#v", captured.Messages) + } + if len(captured.Tools) != 2 || captured.Tools[0].Type != "function" || + len(captured.Messages[1].ToolCalls) != 2 { + t.Fatalf("tool mapping = messages %#v tools %#v", captured.Messages, captured.Tools) + } + if len(response.ToolCalls) != 2 || response.ToolCalls[0].ID != "call_c" || + response.ToolCalls[1].Name != "read_file" || response.Text != "" { + t.Fatalf("tool response = %#v", response) + } +} + +func TestOpenAICompatibleProviderStreamsInterleavedToolCallsDeterministically(t *testing.T) { + var captured openAIChatRequest + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if err := json.NewDecoder(request.Body).Decode(&captured); err != nil { + t.Error(err) + return + } + writer.Header().Set("Content-Type", "text/event-stream") + flusher := writer.(http.Flusher) + writeSSE := func(payload string) { + _, _ = fmt.Fprintf(writer, "data: %s\n\n", payload) + flusher.Flush() + } + writeSSE(`{"id":"chunk","model":"gpt-test-2026-08-01","choices":[{"index":0,"delta":{"role":"assistant","content":null},"finish_reason":null}]}`) + writeSSE(`{"id":"chunk","model":"gpt-test-2026-08-01","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"id":"call_b","type":"function","function":{"name":"read_file","arguments":"{\"pa"}},{"index":0,"id":"call_a","type":"function","function":{"name":"run_command","arguments":"{\"com"}}]},"finish_reason":null}]}`) + writeSSE(`{"id":"chunk","model":"gpt-test-2026-08-01","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"mand\":\"id\"}"}},{"index":1,"function":{"arguments":"th\":\"README.md\"}"}}]},"finish_reason":null}]}`) + writeSSE(`{"id":"chunk","model":"gpt-test-2026-08-01","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]}`) + writeSSE(`{"id":"chunk","model":"gpt-test-2026-08-01","choices":[],"usage":{"prompt_tokens":13,"completion_tokens":6,"total_tokens":19}}`) + writeSSE(`[DONE]`) + })) + defer server.Close() + provider := newTestOpenAIProvider(t, server.URL) + chunks, err := provider.StreamChat(t.Context(), ChatRequest{ + Messages: []Message{{Role: "user", Content: "inspect"}}, + }) + if err != nil { + t.Fatal(err) + } + var all []ChatChunk + for chunk := range chunks { + if chunk.Err != nil { + t.Fatal(chunk.Err) + } + all = append(all, chunk) + } + if !captured.Stream || captured.StreamOptions == nil || !captured.StreamOptions.IncludeUsage { + t.Fatalf("stream usage option missing: %#v", captured) + } + if len(all) != 1 || !all[0].Done || all[0].Model != "gpt-test" || + all[0].Usage == nil || all[0].Usage.TotalTokens != 19 || + len(all[0].ToolCalls) != 2 { + t.Fatalf("unexpected chunks: %#v", all) + } + if first, second := all[0].ToolCalls[0], all[0].ToolCalls[1]; first.ID != "call_a" || first.Name != "run_command" || string(first.Arguments) != `{"command":"id"}` || + second.ID != "call_b" || second.Name != "read_file" || string(second.Arguments) != `{"path":"README.md"}` { + t.Fatalf("tool calls not deterministically aggregated: %#v", all[0].ToolCalls) + } +} + +func TestOpenAICompatibleProviderAcceptsUsageOnFinalChoice(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + writer.Header().Set("Content-Type", "text/event-stream") + _, _ = writer.Write([]byte("data: {\"model\":\"gpt-test-snapshot\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_a\",\"type\":\"function\",\"function\":{\"name\":\"read_file\",\"arguments\":\"{\\\"path\\\":\\\"README.md\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}],\"usage\":{\"prompt_tokens\":8,\"completion_tokens\":4,\"total_tokens\":12}}\n\n")) + _, _ = writer.Write([]byte("data: [DONE]\n\n")) + })) + defer server.Close() + provider := newTestOpenAIProvider(t, server.URL) + chunks, err := provider.StreamChat(t.Context(), ChatRequest{ + Messages: []Message{{Role: "user", Content: "inspect"}}, + }) + if err != nil { + t.Fatal(err) + } + var all []ChatChunk + for chunk := range chunks { + if chunk.Err != nil { + t.Fatal(chunk.Err) + } + all = append(all, chunk) + } + if len(all) != 1 || !all[0].Done || all[0].Usage == nil || + all[0].Usage.TotalTokens != 12 || len(all[0].ToolCalls) != 1 || + all[0].ToolCalls[0].Name != "read_file" { + t.Fatalf("unexpected final-choice usage stream: %#v", all) + } +} + +func TestOpenAICompatibleProviderAcceptsTextUsageOnFinalChoice(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + writer.Header().Set("Content-Type", "text/event-stream") + _, _ = writer.Write([]byte("data: {\"model\":\"gpt-test-snapshot\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ok\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":2,\"completion_tokens\":1,\"total_tokens\":3}}\n\n")) + _, _ = writer.Write([]byte("data: [DONE]\n\n")) + })) + defer server.Close() + provider := newTestOpenAIProvider(t, server.URL) + chunks, err := provider.StreamChat(t.Context(), ChatRequest{ + Messages: []Message{{Role: "user", Content: "hello"}}, + }) + if err != nil { + t.Fatal(err) + } + var text strings.Builder + var final *ChatChunk + for chunk := range chunks { + if chunk.Err != nil { + t.Fatal(chunk.Err) + } + text.WriteString(chunk.Text) + if chunk.Done { + copyChunk := chunk + final = ©Chunk + } + } + if text.String() != "ok" || final == nil || final.Usage == nil || + final.Usage.TotalTokens != 3 { + t.Fatalf("unexpected text final-choice usage: text=%q final=%#v", text.String(), final) + } +} + +func TestOpenAICompatibleProviderRejectsNonTerminalUsageEvents(t *testing.T) { + tests := []struct { + name string + events []string + }{ + {name: "usage before finish with empty choices", events: []string{ + `{"model":"gpt-test","choices":[],"usage":{"prompt_tokens":2,"completion_tokens":1,"total_tokens":3}}`, + }}, + {name: "duplicate usage after final choice", events: []string{ + `{"model":"gpt-test","choices":[{"index":0,"delta":{"content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":2,"completion_tokens":1,"total_tokens":3}}`, + `{"model":"gpt-test","choices":[],"usage":{"prompt_tokens":2,"completion_tokens":1,"total_tokens":3}}`, + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + writer.Header().Set("Content-Type", "text/event-stream") + for _, event := range test.events { + _, _ = fmt.Fprintf(writer, "data: %s\n\n", event) + } + _, _ = writer.Write([]byte("data: [DONE]\n\n")) + })) + defer server.Close() + provider := newTestOpenAIProvider(t, server.URL) + chunks, err := provider.StreamChat(t.Context(), ChatRequest{ + Messages: []Message{{Role: "user", Content: "hello"}}, + }) + if err != nil { + t.Fatal(err) + } + var got error + for chunk := range chunks { + if chunk.Err != nil { + got = chunk.Err + } + } + if ProviderErrorReason(got) != ProviderFailureProtocolIncompatible { + t.Fatalf("usage event error = %#v", got) + } + }) + } +} + +func TestOpenAICompatibleProviderRejectsUsageBeforeFinish(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + writer.Header().Set("Content-Type", "text/event-stream") + _, _ = writer.Write([]byte("data: {\"model\":\"gpt-test\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"partial\"},\"finish_reason\":null}],\"usage\":{\"prompt_tokens\":2,\"completion_tokens\":1,\"total_tokens\":3}}\n\n")) + _, _ = writer.Write([]byte("data: [DONE]\n\n")) + })) + defer server.Close() + provider := newTestOpenAIProvider(t, server.URL) + chunks, err := provider.StreamChat(t.Context(), ChatRequest{ + Messages: []Message{{Role: "user", Content: "hello"}}, + }) + if err != nil { + t.Fatal(err) + } + var got error + for chunk := range chunks { + if chunk.Err != nil { + got = chunk.Err + } + } + if ProviderErrorReason(got) != ProviderFailureProtocolIncompatible { + t.Fatalf("early usage error = %#v", got) + } +} + +func TestOpenAICompatibleProviderStreamsTextAcrossUTF8TransportSplits(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + writer.Header().Set("Content-Type", "text/event-stream") + flusher := writer.(http.Flusher) + payload := []byte("data: {\"model\":\"gpt-test\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"你好🙂\"},\"finish_reason\":null}]}\n\n") + for _, current := range payload { + _, _ = writer.Write([]byte{current}) + flusher.Flush() + } + _, _ = writer.Write([]byte("data: {\"model\":\"gpt-test\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n")) + _, _ = writer.Write([]byte("data: {\"model\":\"gpt-test\",\"choices\":[],\"usage\":{\"prompt_tokens\":2,\"completion_tokens\":1,\"total_tokens\":3}}\n\n")) + _, _ = writer.Write([]byte("data: [DONE]\n\n")) + flusher.Flush() + })) + defer server.Close() + provider := newTestOpenAIProvider(t, server.URL) + chunks, err := provider.StreamChat(t.Context(), ChatRequest{Messages: []Message{{Role: "user", Content: "hello"}}}) + if err != nil { + t.Fatal(err) + } + var text strings.Builder + finals := 0 + for chunk := range chunks { + if chunk.Err != nil { + t.Fatal(chunk.Err) + } + if len(chunk.ToolCalls) != 0 && !chunk.Done { + t.Fatalf("non-final tool calls: %#v", chunk) + } + text.WriteString(chunk.Text) + if chunk.Done { + finals++ + } + } + if text.String() != "你好🙂" || !utf8.ValidString(text.String()) || finals != 1 { + t.Fatalf("text=%q finals=%d", text.String(), finals) + } +} + +func TestOpenAICompatibleProviderRejectsIncompleteStreams(t *testing.T) { + tests := []struct { + name string + events []string + }{ + {name: "missing usage", events: []string{ + `{"model":"gpt-test","choices":[{"index":0,"delta":{"content":"ok"},"finish_reason":"stop"}]}`, + `[DONE]`, + }}, + {name: "truncated finish", events: []string{ + `{"model":"gpt-test","choices":[{"index":0,"delta":{"content":"partial"},"finish_reason":"length"}]}`, + `{"model":"gpt-test","choices":[],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`, + `[DONE]`, + }}, + {name: "missing done", events: []string{ + `{"model":"gpt-test","choices":[{"index":0,"delta":{"content":"ok"},"finish_reason":"stop"}]}`, + `{"model":"gpt-test","choices":[],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`, + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + writer.Header().Set("Content-Type", "text/event-stream") + for _, event := range test.events { + _, _ = fmt.Fprintf(writer, "data: %s\n\n", event) + } + })) + defer server.Close() + provider := newTestOpenAIProvider(t, server.URL) + chunks, err := provider.StreamChat(t.Context(), ChatRequest{Messages: []Message{{Role: "user", Content: "hello"}}}) + if err != nil { + t.Fatal(err) + } + var got error + for chunk := range chunks { + if chunk.Err != nil { + got = chunk.Err + } + } + if got == nil || ProviderErrorKind(got) != OutcomeInvalidResponse || + ProviderErrorReason(got) != ProviderFailureProtocolIncompatible { + t.Fatalf("error = %#v", got) + } + }) + } +} + +func TestOpenAICompatibleProviderRejectsMalformedStreamedToolCalls(t *testing.T) { + tests := []struct { + name string + events []string + }{ + {name: "sparse index", events: []string{ + `{"model":"gpt-test","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"id":"call_b","type":"function","function":{"name":"read_file","arguments":"{}"}}]},"finish_reason":"tool_calls"}]}`, + }}, + {name: "conflicting identity", events: []string{ + `{"model":"gpt-test","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_a","type":"function","function":{"name":"read_file","arguments":"{"}}]},"finish_reason":null}]}`, + `{"model":"gpt-test","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_changed","function":{"arguments":"}"}}]},"finish_reason":"tool_calls"}]}`, + }}, + {name: "invalid arguments", events: []string{ + `{"model":"gpt-test","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_a","type":"function","function":{"name":"read_file","arguments":"not-json"}}]},"finish_reason":"tool_calls"}]}`, + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + writer.Header().Set("Content-Type", "text/event-stream") + for _, event := range test.events { + _, _ = fmt.Fprintf(writer, "data: %s\n\n", event) + } + _, _ = writer.Write([]byte("data: {\"model\":\"gpt-test\",\"choices\":[],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}\n\n")) + _, _ = writer.Write([]byte("data: [DONE]\n\n")) + })) + defer server.Close() + provider := newTestOpenAIProvider(t, server.URL) + chunks, err := provider.StreamChat(t.Context(), ChatRequest{Messages: []Message{{Role: "user", Content: "hello"}}}) + if err != nil { + t.Fatal(err) + } + var got error + for chunk := range chunks { + if chunk.Err != nil { + got = chunk.Err + } + } + if got == nil || ProviderErrorReason(got) != ProviderFailureProtocolIncompatible { + t.Fatalf("error = %#v", got) + } + }) + } +} + +func TestOpenAICompatibleProviderRequiresExactUsage(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + _, _ = writer.Write([]byte(`{ + "model":"gpt-test","choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}], + "usage":{"prompt_tokens":2,"completion_tokens":1,"total_tokens":99} + }`)) + })) + defer server.Close() + provider := newTestOpenAIProvider(t, server.URL) + if _, err := provider.Chat(t.Context(), ChatRequest{Messages: []Message{{Role: "user", Content: "hello"}}}); err == nil || ProviderErrorReason(err) != ProviderFailureProtocolIncompatible { + t.Fatalf("inconsistent usage error = %#v", err) + } +} + +func TestOpenAICompatibleProviderAcceptsResolvedResponseModel(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + _, _ = writer.Write([]byte(`{ + "model":"gpt-test-2026-08-01","choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}], + "usage":{"prompt_tokens":2,"completion_tokens":1,"total_tokens":3} + }`)) + })) + defer server.Close() + provider := newTestOpenAIProvider(t, server.URL) + response, err := provider.Chat(t.Context(), ChatRequest{Messages: []Message{{ + Role: "user", Content: "hello", + }}}) + if err != nil || response.Model != "gpt-test" { + t.Fatalf("resolved response = %#v error = %#v", response, err) + } +} + +func TestOpenAICompatibleProviderHTTPFailuresAreContentFree(t *testing.T) { + secret := "sk-12345678901234567890" + tests := []struct { + status int + kind Outcome + reason ProviderFailureReason + }{ + {http.StatusUnauthorized, OutcomePermanent, ProviderFailureAuthentication}, + {http.StatusNotFound, OutcomePermanent, ProviderFailureProtocolIncompatible}, + {http.StatusTooManyRequests, OutcomeRateLimited, ProviderFailureRateLimit}, + {http.StatusServiceUnavailable, OutcomeRetryable, ProviderFailureCapacity}, + } + for _, test := range tests { + t.Run(fmt.Sprint(test.status), func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + writer.Header().Set("Retry-After", "7") + writer.WriteHeader(test.status) + _, _ = writer.Write([]byte(fmt.Sprintf(`{"error":{"message":"raw %s prompt and args","type":"server_error","code":"ignored"}}`, secret))) + })) + defer server.Close() + provider := newTestOpenAIProvider(t, server.URL) + _, err := provider.Chat(t.Context(), ChatRequest{Messages: []Message{{Role: "user", Content: "hello"}}}) + var typed *ProviderError + if !errors.As(err, &typed) { + t.Fatalf("error = %#v", err) + } + if typed.Kind != test.kind || typed.Reason != test.reason || typed.StatusCode != test.status || + typed.RetryAfter != 7*time.Second || typed.Cause != nil { + t.Fatalf("typed error = %#v", typed) + } + if strings.Contains(typed.Error(), secret) || strings.Contains(typed.Error(), "prompt") || + strings.Contains(typed.Error(), "args") || strings.Contains(typed.Error(), "server_error") { + t.Fatalf("raw response leaked through error: %q", typed.Error()) + } + }) + } +} + +func TestOpenAICompatibleProviderClassifiesCanonicalModelNotFound(t *testing.T) { + tests := []struct { + name string + typeName string + code string + want ProviderFailureReason + }{ + {name: "canonical code", typeName: "invalid_request_error", code: "model_not_found", + want: ProviderFailureModelNotFound}, + {name: "near-match code", typeName: "invalid_request_error", code: "not_model_not_found_suffix", + want: ProviderFailureProtocolIncompatible}, + {name: "near-match type", typeName: "model_not_foundish", code: "unknown", + want: ProviderFailureProtocolIncompatible}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + writer.WriteHeader(http.StatusNotFound) + _, _ = fmt.Fprintf(writer, + `{"error":{"message":"must remain private","type":%q,"code":%q}}`, + test.typeName, test.code) + })) + defer server.Close() + provider := newTestOpenAIProvider(t, server.URL) + _, err := provider.Chat(t.Context(), ChatRequest{ + Messages: []Message{{Role: "user", Content: "hello"}}, + }) + if ProviderErrorReason(err) != test.want || + strings.Contains(fmt.Sprint(err), "must remain private") { + t.Fatalf("model-not-found classification = %#v", err) + } + }) + } +} + +func TestClassifyOpenAIWireErrorRequiresExactSignals(t *testing.T) { + tests := []struct { + signal string + want ProviderFailureReason + }{ + {signal: "rate_limit_exceeded", want: ProviderFailureRateLimit}, + {signal: "overloaded_error", want: ProviderFailureCapacity}, + {signal: "model_not_exists", want: ProviderFailureModelNotFound}, + {signal: "invalid_api_key", want: ProviderFailureAuthentication}, + {signal: "request_timeout", want: ProviderFailureNetwork}, + {signal: "rate_limit_exceeded_suffix", want: ProviderFailureNone}, + {signal: "prefix_invalid_api_key", want: ProviderFailureNone}, + } + for _, test := range tests { + encoded, err := json.Marshal(test.signal) + if err != nil { + t.Fatal(err) + } + if got := classifyOpenAIWireError(openAIError{Code: encoded}); got != test.want { + t.Fatalf("signal %q classified as %q, want %q", test.signal, got, test.want) + } + } +} + +func TestOpenAICompatibleProviderDistinguishesCancellationAndDeadline(t *testing.T) { + server := httptest.NewServer(http.NotFoundHandler()) + defer server.Close() + provider := newTestOpenAIProvider(t, server.URL) + tests := []struct { + name string + context func() (context.Context, context.CancelFunc) + wantReason ProviderFailureReason + }{ + {name: "cancelled", context: func() (context.Context, context.CancelFunc) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return ctx, func() {} + }, wantReason: ProviderFailureNone}, + {name: "deadline", context: func() (context.Context, context.CancelFunc) { + return context.WithDeadline(context.Background(), time.Now().Add(-time.Second)) + }, wantReason: ProviderFailureNetwork}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx, cancel := test.context() + defer cancel() + _, err := provider.Chat(ctx, ChatRequest{ + Messages: []Message{{Role: "user", Content: "hello"}}, + }) + if ProviderErrorKind(err) != OutcomeCancelled || ProviderErrorReason(err) != test.wantReason { + t.Fatalf("transport error = %#v", err) + } + }) + } +} + +func TestOpenAICompatibleProviderRejectsStreamModelChanges(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + writer.Header().Set("Content-Type", "text/event-stream") + _, _ = writer.Write([]byte("data: {\"model\":\"gpt-test-2026-08-01\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ok\"},\"finish_reason\":null}]}\n\n")) + _, _ = writer.Write([]byte("data: {\"model\":\"gpt-test-2026-08-02\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n")) + })) + defer server.Close() + provider := newTestOpenAIProvider(t, server.URL) + chunks, err := provider.StreamChat(t.Context(), ChatRequest{Messages: []Message{{Role: "user", Content: "hello"}}}) + if err != nil { + t.Fatal(err) + } + var got error + for chunk := range chunks { + if chunk.Err != nil { + got = chunk.Err + } + } + if ProviderErrorReason(got) != ProviderFailureProtocolIncompatible { + t.Fatalf("stream model change error = %#v", got) + } +} + +func TestOpenAICompatibleProviderHarnessAndConfiguration(t *testing.T) { + server := httptest.NewServer(http.NotFoundHandler()) + defer server.Close() + provider := newTestOpenAIProvider(t, server.URL+"/v1") + profile := provider.DescribeModelHarness("") + if profile.TransportProtocol != HarnessTransportOpenAIChatCompletions || + profile.ToolStrategy != HarnessToolStrategyNative || + profile.JSONStrategy != HarnessJSONStrategyNative || + profile.QualificationStatus != HarnessQualificationRequired || profile.Validate() != nil { + t.Fatalf("invalid Harness profile: %#v", profile) + } + if !provider.SupportsTools("gpt-test") || !provider.SupportsJSONMode("gpt-test") || + provider.SupportsTools("unknown-model") || provider.SupportsJSONMode("unknown-model") { + t.Fatal("OpenAI capability checks did not stay bound to the configured model") + } + if _, err := NewOpenAICompatibleProvider(OpenAICompatibleConfig{ + BaseURL: "http://example.com", APIKey: "secret", DefaultModel: "gpt-test", + }); err == nil { + t.Fatal("accepted non-loopback HTTP endpoint") + } + if _, err := NewOpenAICompatibleProvider(OpenAICompatibleConfig{ + BaseURL: server.URL, APIKey: "secret with spaces", DefaultModel: "gpt-test", + }); err == nil { + t.Fatal("accepted unsafe API key") + } +} + +func newTestOpenAIProvider(t *testing.T, baseURL string) *OpenAICompatibleProvider { + t.Helper() + provider, err := NewOpenAICompatibleProvider(OpenAICompatibleConfig{ + Name: "openai", BaseURL: baseURL, APIKey: "test-secret", DefaultModel: "gpt-test", + }) + if err != nil { + t.Fatal(err) + } + return provider +} diff --git a/internal/llm/outcome.go b/internal/llm/outcome.go index 9c7d68b2..cf40c8fa 100644 --- a/internal/llm/outcome.go +++ b/internal/llm/outcome.go @@ -37,8 +37,37 @@ func (o Outcome) Retryable() bool { return o == OutcomeRetryable || o == OutcomeRateLimited } +// ProviderFailureReason is a stable, content-free classification for operator +// diagnostics. It is deliberately separate from Outcome: Outcome controls +// retry behavior, while the reason describes the safe failure category. +type ProviderFailureReason string + +const ( + ProviderFailureNone ProviderFailureReason = "none" + ProviderFailureNotConfigured ProviderFailureReason = "not_configured" + ProviderFailureAuthentication ProviderFailureReason = "authentication" + ProviderFailureNetwork ProviderFailureReason = "network" + ProviderFailureRateLimit ProviderFailureReason = "rate_limit" + ProviderFailureCapacity ProviderFailureReason = "capacity" + ProviderFailureModelNotFound ProviderFailureReason = "model_not_found" + ProviderFailureProtocolIncompatible ProviderFailureReason = "protocol_incompatible" +) + +func (r ProviderFailureReason) Valid() bool { + switch r { + case ProviderFailureNone, ProviderFailureNotConfigured, + ProviderFailureAuthentication, ProviderFailureNetwork, + ProviderFailureRateLimit, ProviderFailureCapacity, + ProviderFailureModelNotFound, ProviderFailureProtocolIncompatible: + return true + default: + return false + } +} + type ProviderError struct { Kind Outcome + Reason ProviderFailureReason Provider string StatusCode int RetryAfter time.Duration @@ -51,7 +80,9 @@ func NewProviderError(kind Outcome, provider string, message string, cause error kind = OutcomePermanent } return &ProviderError{ - Kind: kind, Provider: strings.TrimSpace(provider), Message: redact.String(strings.TrimSpace(message)), Cause: cause, + Kind: kind, Reason: ProviderFailureNone, + Provider: strings.TrimSpace(provider), + Message: redact.String(strings.TrimSpace(message)), Cause: cause, } } @@ -60,9 +91,6 @@ func (e *ProviderError) Error() string { return "" } message := strings.TrimSpace(e.Message) - if message == "" && e.Cause != nil { - message = redact.String(strings.TrimSpace(e.Cause.Error())) - } if message == "" { message = string(e.Kind) } @@ -92,6 +120,13 @@ func NormalizeProviderError(provider string, err error) *ProviderError { if strings.TrimSpace(copy.Provider) == "" { copy.Provider = strings.TrimSpace(provider) } + if !copy.Reason.Valid() || copy.Reason == ProviderFailureNone { + copy.Reason = defaultProviderFailureReason(copy.Kind, copy.StatusCode) + if copy.Reason == ProviderFailureNone && + errors.Is(copy.Cause, context.DeadlineExceeded) { + copy.Reason = ProviderFailureNetwork + } + } copy.Message = redact.String(strings.TrimSpace(copy.Message)) if copy.RetryAfter < 0 { copy.RetryAfter = 0 @@ -113,7 +148,17 @@ func NormalizeProviderError(provider string, err error) *ProviderError { kind = OutcomeRetryable } } - return NewProviderError(kind, provider, err.Error(), err) + message := "provider request failed" + if kind == OutcomeCancelled { + message = "provider request was cancelled" + } + providerError := NewProviderError(kind, provider, message, err) + providerError.Reason = defaultProviderFailureReason(kind, 0) + if providerError.Reason == ProviderFailureNone && + errors.Is(err, context.DeadlineExceeded) { + providerError.Reason = ProviderFailureNetwork + } + return providerError } func ProviderErrorKind(err error) Outcome { @@ -124,6 +169,43 @@ func ProviderErrorKind(err error) Outcome { return "" } +func ProviderErrorReason(err error) ProviderFailureReason { + var typed *ProviderError + if errors.As(err, &typed) && typed != nil { + if typed.Reason.Valid() && typed.Reason != ProviderFailureNone { + return typed.Reason + } + return defaultProviderFailureReason(typed.Kind, typed.StatusCode) + } + return ProviderFailureNone +} + +func defaultProviderFailureReason(kind Outcome, statusCode int) ProviderFailureReason { + switch statusCode { + case 401, 403: + return ProviderFailureAuthentication + case 404: + // A status alone cannot distinguish a missing model from a bad or + // unsupported endpoint path. Protocol adapters may upgrade this only + // after recognizing an exact, content-free upstream code or type. + return ProviderFailureProtocolIncompatible + case 429: + return ProviderFailureRateLimit + case 503, 529: + return ProviderFailureCapacity + } + switch kind { + case OutcomeRetryable: + return ProviderFailureNetwork + case OutcomeRateLimited: + return ProviderFailureRateLimit + case OutcomeInvalidResponse: + return ProviderFailureProtocolIncompatible + default: + return ProviderFailureNone + } +} + type ModelAttempt struct { Number int TransportAttempt int diff --git a/internal/llm/outcome_test.go b/internal/llm/outcome_test.go index 80c8fcdd..c674c913 100644 --- a/internal/llm/outcome_test.go +++ b/internal/llm/outcome_test.go @@ -11,19 +11,25 @@ import ( func TestNormalizeProviderErrorClassifiesFailures(t *testing.T) { tests := []struct { - name string - err error - want Outcome + name string + err error + want Outcome + wantReason ProviderFailureReason }{ - {name: "cancelled", err: context.Canceled, want: OutcomeCancelled}, - {name: "deadline", err: context.DeadlineExceeded, want: OutcomeCancelled}, - {name: "network", err: &url.Error{Op: "Post", URL: "https://provider.invalid", Err: errors.New("connection reset")}, want: OutcomeRetryable}, - {name: "permanent", err: errors.New("invalid provider configuration"), want: OutcomePermanent}, + {name: "cancelled", err: context.Canceled, want: OutcomeCancelled, + wantReason: ProviderFailureNone}, + {name: "deadline", err: context.DeadlineExceeded, want: OutcomeCancelled, + wantReason: ProviderFailureNetwork}, + {name: "network", err: &url.Error{Op: "Post", URL: "https://provider.invalid", Err: errors.New("connection reset")}, want: OutcomeRetryable, + wantReason: ProviderFailureNetwork}, + {name: "permanent", err: errors.New("invalid provider configuration"), want: OutcomePermanent, + wantReason: ProviderFailureNone}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { got := NormalizeProviderError("test", test.err) - if got.Kind != test.want || got.Provider != "test" || !errors.Is(got, test.err) { + if got.Kind != test.want || got.Reason != test.wantReason || + got.Provider != "test" || !errors.Is(got, test.err) { t.Fatalf("unexpected normalized error: %#v", got) } }) @@ -36,7 +42,9 @@ func TestProviderErrorPreservesTypedMetadataAndRedactsMessage(t *testing.T) { original.StatusCode = 429 original.RetryAfter = 3 * time.Second normalized := NormalizeProviderError("mimo", original) - if normalized.Kind != OutcomeRateLimited || normalized.Provider != "mimo" || normalized.StatusCode != 429 || normalized.RetryAfter != 3*time.Second { + if normalized.Kind != OutcomeRateLimited || normalized.Reason != ProviderFailureRateLimit || + normalized.Provider != "mimo" || normalized.StatusCode != 429 || + normalized.RetryAfter != 3*time.Second { t.Fatalf("typed metadata changed: %#v", normalized) } if strings.Contains(normalized.Error(), token[:12]) || !strings.Contains(normalized.Error(), "[REDACTED:") { @@ -44,6 +52,58 @@ func TestProviderErrorPreservesTypedMetadataAndRedactsMessage(t *testing.T) { } } +func TestProviderErrorDoesNotProjectCauseText(t *testing.T) { + marker := "remote-body-marker-that-must-remain-private" + providerErr := NewProviderError(OutcomeRetryable, "test", "", errors.New(marker)) + if strings.Contains(providerErr.Error(), marker) || !errors.Is(providerErr, providerErr.Cause) { + t.Fatalf("Provider error projected private cause text: %q", providerErr.Error()) + } + normalized := NormalizeProviderError("test", &url.Error{ + Op: "Post", URL: "https://provider.invalid", Err: errors.New(marker), + }) + if strings.Contains(normalized.Error(), marker) || normalized.Reason != ProviderFailureNetwork { + t.Fatalf("normalized Provider error projected private cause text: %#v", normalized) + } +} + +func TestProviderFailureReasonValidationAndNormalization(t *testing.T) { + for _, reason := range []ProviderFailureReason{ + ProviderFailureNone, ProviderFailureNotConfigured, + ProviderFailureAuthentication, ProviderFailureNetwork, + ProviderFailureRateLimit, ProviderFailureCapacity, + ProviderFailureModelNotFound, ProviderFailureProtocolIncompatible, + } { + if !reason.Valid() { + t.Fatalf("valid Provider failure reason rejected: %q", reason) + } + } + if ProviderFailureReason("unknown").Valid() { + t.Fatal("unknown Provider failure reason was accepted") + } + tests := []struct { + name string + error *ProviderError + reason ProviderFailureReason + }{ + {name: "authentication", error: &ProviderError{Kind: OutcomePermanent, StatusCode: 401}, reason: ProviderFailureAuthentication}, + {name: "ambiguous not found", error: &ProviderError{Kind: OutcomePermanent, StatusCode: 404}, reason: ProviderFailureProtocolIncompatible}, + {name: "explicit model not found", error: &ProviderError{Kind: OutcomePermanent, + StatusCode: 404, Reason: ProviderFailureModelNotFound}, reason: ProviderFailureModelNotFound}, + {name: "capacity", error: &ProviderError{Kind: OutcomeRetryable, StatusCode: 503}, reason: ProviderFailureCapacity}, + {name: "network", error: &ProviderError{Kind: OutcomeRetryable}, reason: ProviderFailureNetwork}, + {name: "protocol", error: &ProviderError{Kind: OutcomeInvalidResponse}, reason: ProviderFailureProtocolIncompatible}, + {name: "explicit", error: &ProviderError{Kind: OutcomeRetryable, StatusCode: 503, Reason: ProviderFailureNetwork}, reason: ProviderFailureNetwork}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + normalized := NormalizeProviderError("test", test.error) + if normalized.Reason != test.reason || ProviderErrorReason(normalized) != test.reason { + t.Fatalf("reason=%q want %q", normalized.Reason, test.reason) + } + }) + } +} + func TestNormalizeProviderErrorDowngradesInvalidTypedKind(t *testing.T) { for _, kind := range []Outcome{"unknown", OutcomeSuccess} { normalized := NormalizeProviderError("test", &ProviderError{Kind: kind, Message: "bad kind"}) @@ -53,6 +113,13 @@ func TestNormalizeProviderErrorDowngradesInvalidTypedKind(t *testing.T) { } } +func TestNewProviderErrorLeavesPermanentReasonUnclassified(t *testing.T) { + providerErr := NewProviderError(OutcomePermanent, "test", "local validation failed", nil) + if providerErr.Reason != ProviderFailureNone { + t.Fatalf("generic permanent error was overclassified: %#v", providerErr) + } +} + func TestModelAttemptValidation(t *testing.T) { base := ModelAttempt{Number: 1, MaxAttempts: 3, Provider: "test", Model: "model"} if err := base.ValidateStarted(); err != nil { diff --git a/internal/modelregistry/harness.go b/internal/modelregistry/harness.go index 6f23d2fb..8b8fb626 100644 --- a/internal/modelregistry/harness.go +++ b/internal/modelregistry/harness.go @@ -33,6 +33,7 @@ const ( maxHarnessProbeChunks = 256 maxHarnessProbeBytes = 8 * 1024 maxHarnessRecordBytes = 4 * 1024 + harnessProbeMaxTokens = 256 ) type HarnessAvailability struct { @@ -58,6 +59,7 @@ type HarnessQualificationResult struct { Model string Status string Outcome string + FailureReason llm.ProviderFailureReason Retryable bool NetworkRequestAttempted bool ModelCalls int @@ -100,8 +102,25 @@ func (r *Registry) QualifyHarness(ctx context.Context, writer RouteSettingWriter provider = strings.TrimSpace(provider) model = strings.TrimSpace(model) if !validAvailabilityIdentifier(provider, maxPublicProviderNameBytes) || - !validAvailabilityIdentifier(model, maxPublicModelNameBytes) || - !r.providerModelAvailable(provider, model) { + !validAvailabilityIdentifier(model, maxPublicModelNameBytes) { + return HarnessQualificationResult{}, + errors.New("model Harness qualification Provider model is unavailable") + } + providerStatus, fallbackHarness, known := r.providerModelStatus(provider, model) + if !known { + return HarnessQualificationResult{}, + errors.New("model Harness qualification Provider model is unavailable") + } + if providerStatus == ProviderNotConfigured { + return HarnessQualificationResult{ + ProtocolVersion: HarnessQualificationProtocolVersion, + Provider: provider, Model: model, Status: HarnessDiagnosticUnreachable, + Outcome: string(llm.OutcomePermanent), + FailureReason: llm.ProviderFailureNotConfigured, + Harness: fallbackHarness, + }, nil + } + if providerStatus != ProviderAvailable { return HarnessQualificationResult{}, errors.New("model Harness qualification Provider model is unavailable") } @@ -116,6 +135,7 @@ func (r *Registry) QualifyHarness(ctx context.Context, writer RouteSettingWriter ProtocolVersion: HarnessQualificationProtocolVersion, Provider: provider, Model: model, Status: HarnessDiagnosticIncompatible, + FailureReason: llm.ProviderFailureNone, NetworkRequestAttempted: r.providerNetworkRequired(provider), ToolExecuted: false, ResponseContentReturned: false, Harness: harnessAvailability(model, base), @@ -123,11 +143,13 @@ func (r *Registry) QualifyHarness(ctx context.Context, writer RouteSettingWriter if rootHarnessReady(base) { result.Status = HarnessDiagnosticQualified result.Outcome = string(llm.OutcomeSuccess) + result.FailureReason = llm.ProviderFailureNone return result, nil } if base.ToolStrategy != llm.HarnessToolStrategyNative || base.JSONStrategy == llm.HarnessJSONStrategyNone { result.Outcome = string(llm.OutcomeInvalidResponse) + result.FailureReason = llm.ProviderFailureProtocolIncompatible return result, nil } qualificationCtx, cancel := context.WithTimeout(ctx, HarnessQualificationTimeout) @@ -144,6 +166,7 @@ func (r *Registry) QualifyHarness(ctx context.Context, writer RouteSettingWriter if probeErr != nil { providerErr := llm.NormalizeProviderError(provider, probeErr) result.Outcome = string(providerErr.Kind) + result.FailureReason = providerErr.Reason result.Retryable = providerErr.Kind.Retryable() if providerErr.Kind != llm.OutcomeInvalidResponse { result.Status = HarnessDiagnosticUnreachable @@ -183,6 +206,7 @@ func (r *Registry) QualifyHarness(ctx context.Context, writer RouteSettingWriter } result.Status = HarnessDiagnosticQualified result.Outcome = string(llm.OutcomeSuccess) + result.FailureReason = llm.ProviderFailureNone result.Retryable = false result.Harness = harnessAvailability(model, verified) return result, nil @@ -205,7 +229,7 @@ func (r *Registry) probeHarness(ctx context.Context, ref llm.ModelRef, {Role: "system", Content: "Prayu model Harness qualification. Use only the supplied synthetic tool. Do not answer with text."}, {Role: "user", Content: "Call prayu_harness_echo exactly once with nonce " + nonce + "."}, }, - Tools: []llm.ToolSpec{tool}, MaxTokens: 96, + Tools: []llm.ToolSpec{tool}, MaxTokens: harnessProbeMaxTokens, Metadata: map[string]string{ "purpose": "model_harness_qualification", "phase": "tool_call", @@ -244,7 +268,7 @@ func (r *Registry) probeHarness(ctx context.Context, ref llm.ModelRef, ToolResults: []llm.ToolResult{{ ToolCallID: firstResponse.ToolCalls[0].ID, Content: string(resultJSON), }}}), - Tools: []llm.ToolSpec{tool}, MaxTokens: 96, + Tools: []llm.ToolSpec{tool}, MaxTokens: harnessProbeMaxTokens, JSONMode: base.JSONStrategy == llm.HarnessJSONStrategyNative, Metadata: map[string]string{ "purpose": "model_harness_qualification", @@ -292,6 +316,9 @@ func collectHarnessProbeStream(ctx context.Context, router *llm.Router, ref llm. return nil, ctx.Err() case chunk, ok := <-chunks: if !ok { + if ctx.Err() != nil { + return nil, ctx.Err() + } return nil, llm.NewProviderError(llm.OutcomeInvalidResponse, ref.Provider, "model Harness probe stream ended without a final chunk", nil) } diff --git a/internal/modelregistry/registry.go b/internal/modelregistry/registry.go index 5376826e..7636c18f 100644 --- a/internal/modelregistry/registry.go +++ b/internal/modelregistry/registry.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net/http" "os" "sort" "strings" @@ -24,6 +25,7 @@ const ( DiagnosticProtocolVersion = "provider_diagnostic.v1" RouteControlProtocolVersion = "model_route_control.v1" DiagnosticTimeout = 15 * time.Second + diagnosticMaxTokens = 128 ) const ( @@ -35,6 +37,7 @@ const ( const ( ProviderKindLocal = "local" ProviderKindAnthropicCompatible = "anthropic_compatible" + ProviderKindOpenAICompatible = "openai_compatible" ) const ( @@ -49,6 +52,9 @@ const ( DefaultDeepSeekModel = "deepseek-v4-flash" defaultAnthropicURL = "https://api.anthropic.com" DefaultAnthropicModel = "claude-3-5-sonnet-latest" + defaultOpenAIBaseURL = "https://api.openai.com" + DefaultOpenAIModel = "gpt-4.1-mini" + DefaultOpenAITimeout = 60 * time.Second ) var routeNames = []string{"code", "ctf", "learn", "review", "script"} @@ -99,6 +105,7 @@ type DiagnosticResult struct { Model string Status string Outcome string + FailureReason llm.ProviderFailureReason Retryable bool NetworkRequestAttempted bool ModelCalled bool @@ -143,6 +150,15 @@ type anthropicEnvironment struct { disableThinking bool } +type openAIEnvironment struct { + name string + apiKeyEnv string + baseURLEnv string + modelEnv string + defaultBaseURL string + defaultModel string +} + func NewFromEnvironment() *Registry { return New(os.LookupEnv) } @@ -209,6 +225,13 @@ func buildRegistry(ctx context.Context, lookup EnvironmentLookup, return nil, err } } + if err := registry.registerOpenAIEnvironment(ctx, openAIEnvironment{ + name: "openai", apiKeyEnv: "CYBERAGENT_OPENAI_API_KEY", + baseURLEnv: "CYBERAGENT_OPENAI_BASE_URL", modelEnv: "CYBERAGENT_OPENAI_MODEL", + defaultBaseURL: defaultOpenAIBaseURL, defaultModel: DefaultOpenAIModel, + }, lookup, credentials, strictCredentialReads); err != nil { + return nil, err + } sort.Slice(registry.providers, func(i, j int) bool { return registry.providers[i].Name < registry.providers[j].Name }) @@ -356,8 +379,22 @@ func (r *Registry) Diagnose(ctx context.Context, provider string, provider = strings.TrimSpace(provider) model = strings.TrimSpace(model) if !validAvailabilityIdentifier(provider, maxPublicProviderNameBytes) || - !validAvailabilityIdentifier(model, maxPublicModelNameBytes) || - !r.providerModelAvailable(provider, model) { + !validAvailabilityIdentifier(model, maxPublicModelNameBytes) { + return DiagnosticResult{}, errors.New("diagnostic Provider model is unavailable") + } + providerStatus, _, known := r.providerModelStatus(provider, model) + if !known { + return DiagnosticResult{}, errors.New("diagnostic Provider model is unavailable") + } + if providerStatus == ProviderNotConfigured { + return DiagnosticResult{ + ProtocolVersion: DiagnosticProtocolVersion, + Provider: provider, Model: model, Status: DiagnosticUnreachable, + Outcome: string(llm.OutcomePermanent), + FailureReason: llm.ProviderFailureNotConfigured, + }, nil + } + if providerStatus != ProviderAvailable { return DiagnosticResult{}, errors.New("diagnostic Provider model is unavailable") } networkRequired := r.providerNetworkRequired(provider) @@ -367,6 +404,7 @@ func (r *Registry) Diagnose(ctx context.Context, provider string, result := DiagnosticResult{ ProtocolVersion: DiagnosticProtocolVersion, Provider: provider, Model: model, Status: DiagnosticUnreachable, + FailureReason: llm.ProviderFailureNone, NetworkRequestAttempted: networkRequired, ModelCalled: true, ToolCalled: false, ResponseContentReturned: false, } @@ -375,7 +413,7 @@ func (r *Registry) Diagnose(ctx context.Context, provider string, Model: model, Messages: []llm.Message{{Role: "user", Content: "Reply with one short acknowledgement for a connectivity diagnostic."}}, - Temperature: 0, MaxTokens: 8, + Temperature: 0, MaxTokens: diagnosticMaxTokens, Metadata: map[string]string{"purpose": "connectivity_diagnostic"}, }) result.DurationMillis = time.Since(started).Milliseconds() @@ -383,20 +421,24 @@ func (r *Registry) Diagnose(ctx context.Context, provider string, result.DurationMillis = 0 } if err != nil { - outcome := llm.ProviderErrorKind(llm.NormalizeProviderError(provider, err)) + providerErr := llm.NormalizeProviderError(provider, err) + outcome := llm.ProviderErrorKind(providerErr) if !outcome.Valid() || outcome == llm.OutcomeSuccess { outcome = llm.OutcomePermanent } result.Outcome = string(outcome) + result.FailureReason = providerErr.Reason result.Retryable = outcome.Retryable() return result, nil } if response == nil || strings.TrimSpace(response.Provider) != provider { result.Outcome = string(llm.OutcomeInvalidResponse) + result.FailureReason = llm.ProviderFailureProtocolIncompatible return result, nil } result.Status = DiagnosticReachable result.Outcome = string(llm.OutcomeSuccess) + result.FailureReason = llm.ProviderFailureNone return result, nil } @@ -411,6 +453,7 @@ func (r *Registry) Snapshot() Snapshot { for index, provider := range r.providers { providers[index] = provider providers[index].Models = make([]string, len(provider.Models)) + configuredHarnesses := append([]HarnessAvailability(nil), provider.Harnesses...) providers[index].Harnesses = make([]HarnessAvailability, 0, len(provider.Models)) for modelIndex, model := range provider.Models { providers[index].Models[modelIndex] = availabilityIdentifier(model, @@ -420,6 +463,8 @@ func (r *Registry) Snapshot() Snapshot { }); err == nil { providers[index].Harnesses = append(providers[index].Harnesses, harnessAvailability(model, profile)) + } else if fallback, found := harnessForModel(configuredHarnesses, model); found { + providers[index].Harnesses = append(providers[index].Harnesses, fallback) } } } @@ -475,6 +520,25 @@ func (r *Registry) providerNetworkRequired(provider string) bool { return false } +func (r *Registry) providerModelStatus(provider string, model string) ( + string, HarnessAvailability, bool, +) { + r.mu.RLock() + defer r.mu.RUnlock() + for _, current := range r.providers { + if current.Name != provider { + continue + } + for _, candidate := range current.Models { + if candidate == model { + harness, _ := harnessForModel(current.Harnesses, model) + return current.Status, harness, true + } + } + } + return "", HarnessAvailability{}, false +} + func containsRoute(route string) bool { for _, current := range routeNames { if current == route { @@ -512,17 +576,25 @@ func (r *Registry) registerAnthropicEnvironment(ctx context.Context, config anth } } status := ProviderNotConfigured + model := environmentValue(lookup, config.modelEnv, config.defaultModel) models := []string{} + harnesses := []HarnessAvailability{} + if validAvailabilityIdentifier(model, maxPublicModelNameBytes) { + model = strings.TrimSpace(model) + models = []string{model} + harnesses = []HarnessAvailability{unqualifiedHarnessAvailability(model, + llm.HarnessTransportAnthropicMessages, llm.HarnessJSONStrategyPrompt)} + } configurationError := false if present && key != "" { baseURL := environmentValue(lookup, config.baseURLEnv, config.defaultBaseURL) - model := environmentValue(lookup, config.modelEnv, config.defaultModel) - if !validAvailabilityIdentifier(model, maxPublicModelNameBytes) { + if len(models) == 0 { status = ProviderInvalidConfiguration configurationError = true r.providers = append(r.providers, ProviderAvailability{ Name: config.name, Kind: ProviderKindAnthropicCompatible, Status: status, - Models: models, CredentialSource: credentialSource, NetworkRequired: true, + Models: models, Harnesses: harnesses, + CredentialSource: credentialSource, NetworkRequired: true, ConfigurationError: configurationError, }) return nil @@ -536,7 +608,6 @@ func (r *Registry) registerAnthropicEnvironment(ctx context.Context, config anth configurationError = true } else { status = ProviderAvailable - models = []string{strings.TrimSpace(model)} r.router.RegisterProvider(provider) r.available[config.name] = struct{}{} } @@ -546,12 +617,103 @@ func (r *Registry) registerAnthropicEnvironment(ctx context.Context, config anth } r.providers = append(r.providers, ProviderAvailability{ Name: config.name, Kind: ProviderKindAnthropicCompatible, Status: status, - Models: models, CredentialSource: credentialSource, NetworkRequired: true, + Models: models, Harnesses: harnesses, + CredentialSource: credentialSource, NetworkRequired: true, ConfigurationError: configurationError, }) return nil } +func (r *Registry) registerOpenAIEnvironment(ctx context.Context, config openAIEnvironment, + lookup EnvironmentLookup, credentials credentialLookup, strictCredentialReads bool, +) error { + key, present := lookup(config.apiKeyEnv) + credentialSource := "none" + if present { + credentialSource = "environment" + } + if !present && credentials != nil { + var err error + key, present, err = credentials(ctx, config.name) + if err != nil { + if strictCredentialReads { + return fmt.Errorf("read system credential for %s: %w", config.name, err) + } + r.providers = append(r.providers, ProviderAvailability{ + Name: config.name, Kind: ProviderKindOpenAICompatible, + Status: ProviderInvalidConfiguration, Models: []string{}, Harnesses: []HarnessAvailability{}, + CredentialSource: "system", NetworkRequired: true, ConfigurationError: true, + }) + return nil + } + if present { + credentialSource = "system" + } + } + model := environmentValue(lookup, config.modelEnv, config.defaultModel) + models := []string{} + harnesses := []HarnessAvailability{} + if validAvailabilityIdentifier(model, maxPublicModelNameBytes) { + model = strings.TrimSpace(model) + models = []string{model} + harnesses = []HarnessAvailability{unqualifiedHarnessAvailability(model, + llm.HarnessTransportOpenAIChatCompletions, llm.HarnessJSONStrategyNative)} + } + status := ProviderNotConfigured + configurationError := false + if present && key != "" { + if len(models) == 0 { + status = ProviderInvalidConfiguration + configurationError = true + } else { + provider, err := llm.NewOpenAICompatibleProvider(llm.OpenAICompatibleConfig{ + Name: config.name, + BaseURL: environmentValue(lookup, config.baseURLEnv, config.defaultBaseURL), + APIKey: key, DefaultModel: model, + HTTPClient: &http.Client{Timeout: DefaultOpenAITimeout}, + }) + if err != nil { + status = ProviderInvalidConfiguration + configurationError = true + } else { + status = ProviderAvailable + r.router.RegisterProvider(provider) + r.available[config.name] = struct{}{} + } + } + } else if present && key != strings.TrimSpace(key) { + status = ProviderInvalidConfiguration + configurationError = true + } + r.providers = append(r.providers, ProviderAvailability{ + Name: config.name, Kind: ProviderKindOpenAICompatible, Status: status, + Models: models, Harnesses: harnesses, + CredentialSource: credentialSource, NetworkRequired: true, + ConfigurationError: configurationError, + }) + return nil +} + +func unqualifiedHarnessAvailability(model string, transport string, + jsonStrategy string, +) HarnessAvailability { + return HarnessAvailability{ + ProtocolVersion: llm.ModelHarnessProtocolVersion, + Model: model, TransportProtocol: transport, + ToolStrategy: llm.HarnessToolStrategyNative, JSONStrategy: jsonStrategy, + QualificationStatus: llm.HarnessQualificationRequired, + } +} + +func harnessForModel(harnesses []HarnessAvailability, model string) (HarnessAvailability, bool) { + for _, harness := range harnesses { + if harness.Model == model { + return harness, true + } + } + return HarnessAvailability{}, false +} + func environmentValue(lookup EnvironmentLookup, name string, fallback string) string { value, found := lookup(name) if !found || strings.TrimSpace(value) == "" { diff --git a/internal/modelregistry/registry_test.go b/internal/modelregistry/registry_test.go index 39ac5a9d..48d5bd7a 100644 --- a/internal/modelregistry/registry_test.go +++ b/internal/modelregistry/registry_test.go @@ -87,7 +87,7 @@ func TestRegistryBuildsRedactedEnvironmentAvailabilityAndRoutes(t *testing.T) { t.Fatal(err) } snapshot := registry.Snapshot() - if snapshot.ProtocolVersion != ProtocolVersion || len(snapshot.Providers) != 4 || + if snapshot.ProtocolVersion != ProtocolVersion || len(snapshot.Providers) != 5 || len(snapshot.Routes) != len(routeNames) { t.Fatalf("unexpected registry snapshot: %#v", snapshot) } @@ -405,13 +405,88 @@ func TestRegistryDiagnosticReturnsOnlyBoundedConnectivityFacts(t *testing.T) { } if result.ProtocolVersion != DiagnosticProtocolVersion || result.Status != DiagnosticReachable || result.Outcome != string(llm.OutcomeSuccess) || + result.FailureReason != llm.ProviderFailureNone || result.Provider != "mock" || result.Model != "mock-fast" || result.NetworkRequestAttempted || !result.ModelCalled || result.ToolCalled || result.ResponseContentReturned || result.DurationMillis < 0 { t.Fatalf("unexpected diagnostic projection: %#v", result) } - if _, err := registry.Diagnose(context.Background(), "mimo", DefaultMimoModel); err == nil { - t.Fatal("unconfigured Provider diagnostic was accepted") + unconfigured, err := registry.Diagnose(context.Background(), "openai", DefaultOpenAIModel) + if err != nil || unconfigured.Status != DiagnosticUnreachable || + unconfigured.Outcome != string(llm.OutcomePermanent) || + unconfigured.FailureReason != llm.ProviderFailureNotConfigured || + unconfigured.NetworkRequestAttempted || unconfigured.ModelCalled || + unconfigured.ToolCalled || unconfigured.ResponseContentReturned { + t.Fatalf("unexpected unconfigured Provider diagnostic: %#v err=%v", unconfigured, err) + } + if _, err := registry.Diagnose(context.Background(), "unknown", DefaultOpenAIModel); err == nil { + t.Fatal("unknown Provider diagnostic was accepted") + } +} + +func TestRegistryRegistersOpenAIEnvironmentAndSystemCredential(t *testing.T) { + values := map[string]string{ + "CYBERAGENT_OPENAI_API_KEY": "test-openai-key-0123456789", + "CYBERAGENT_OPENAI_MODEL": "openai-test-model", + } + registry := New(func(name string) (string, bool) { + value, found := values[name] + return value, found + }) + snapshot := registry.Snapshot() + var openai ProviderAvailability + for _, provider := range snapshot.Providers { + if provider.Name == "openai" { + openai = provider + } + } + if openai.Kind != ProviderKindOpenAICompatible || openai.Status != ProviderAvailable || + openai.CredentialSource != "environment" || len(openai.Models) != 1 || + openai.Models[0] != "openai-test-model" || len(openai.Harnesses) != 1 || + openai.Harnesses[0].TransportProtocol != llm.HarnessTransportOpenAIChatCompletions || + !contains(registry.Router().ProviderNames(), "openai") || + strings.Contains(snapshotText(snapshot), values["CYBERAGENT_OPENAI_API_KEY"]) { + t.Fatalf("unexpected OpenAI registry projection: %#v", openai) + } + + registry, err := newRegistry(func(string) (string, bool) { return "", false }, + func(_ context.Context, provider string) (string, bool, error) { + if provider == "openai" { + return "system-openai-key-0123456789", true, nil + } + return "", false, nil + }) + if err != nil { + t.Fatal(err) + } + foundSystemOpenAI := false + for _, provider := range registry.Snapshot().Providers { + if provider.Name == "openai" { + foundSystemOpenAI = true + if provider.Status != ProviderAvailable || provider.CredentialSource != "system" || + !contains(registry.Router().ProviderNames(), "openai") { + t.Fatalf("system OpenAI credential did not register: %#v", provider) + } + } + } + if !foundSystemOpenAI { + t.Fatal("system OpenAI Provider was not projected") + } +} + +func TestUnconfiguredHarnessQualificationIsContentFree(t *testing.T) { + registry := New(nil) + result, err := registry.QualifyHarness(context.Background(), routeSettings{}, + "openai", DefaultOpenAIModel) + if err != nil || result.Status != HarnessDiagnosticUnreachable || + result.Outcome != string(llm.OutcomePermanent) || + result.FailureReason != llm.ProviderFailureNotConfigured || + result.NetworkRequestAttempted || result.ModelCalls != 0 || + result.SyntheticToolCalls != 0 || result.ToolExecuted || + result.ResponseContentReturned || + result.Harness.TransportProtocol != llm.HarnessTransportOpenAIChatCompletions || + result.Harness.Model != DefaultOpenAIModel { + t.Fatalf("unexpected unconfigured Harness qualification: %#v err=%v", result, err) } } @@ -419,11 +494,13 @@ func TestDeepSeekDiagnosticDisablesThinkingWithoutChangingOtherProviders(t *test var thinking struct { Type string `json:"type"` } + maxTokens := 0 server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request, ) { var body struct { - Thinking *struct { + MaxTokens int `json:"max_tokens"` + Thinking *struct { Type string `json:"type"` } `json:"thinking"` } @@ -435,6 +512,7 @@ func TestDeepSeekDiagnosticDisablesThinkingWithoutChangingOtherProviders(t *test if body.Thinking != nil { thinking.Type = body.Thinking.Type } + maxTokens = body.MaxTokens writer.Header().Set("Content-Type", "application/json") _, _ = writer.Write([]byte(`{ "id":"msg_diagnostic","type":"message","role":"assistant", @@ -456,10 +534,11 @@ func TestDeepSeekDiagnosticDisablesThinkingWithoutChangingOtherProviders(t *test if err != nil { t.Fatal(err) } - if thinking.Type != "disabled" || result.Status != DiagnosticReachable || + if thinking.Type != "disabled" || maxTokens != diagnosticMaxTokens || + result.Status != DiagnosticReachable || result.Outcome != string(llm.OutcomeSuccess) { - t.Fatalf("DeepSeek diagnostic compatibility failed: thinking=%#v result=%#v", - thinking, result) + t.Fatalf("DeepSeek diagnostic compatibility failed: thinking=%#v max_tokens=%d result=%#v", + thinking, maxTokens, result) } } @@ -480,6 +559,7 @@ func TestHarnessQualificationIsSyntheticExactAndDurable(t *testing.T) { if result.ProtocolVersion != HarnessQualificationProtocolVersion || result.Status != HarnessDiagnosticQualified || result.Outcome != string(llm.OutcomeSuccess) || + result.FailureReason != llm.ProviderFailureNone || result.ModelCalls != 2 || result.SyntheticToolCalls != 1 || result.ToolExecuted || result.ResponseContentReturned || !result.Harness.RootEligible || @@ -529,6 +609,7 @@ func TestHarnessQualificationRejectsResponseModelDrift(t *testing.T) { } if result.Status != HarnessDiagnosticIncompatible || result.Outcome != string(llm.OutcomeInvalidResponse) || + result.FailureReason != llm.ProviderFailureProtocolIncompatible || result.ModelCalls != 1 || result.Harness.RootEligible || len(settings) != 0 { t.Fatalf("response identity drift was accepted: result=%#v settings=%#v", @@ -536,6 +617,46 @@ func TestHarnessQualificationRejectsResponseModelDrift(t *testing.T) { } } +func TestCollectHarnessProbeStreamPreservesExpiredContextWhenStreamCloses(t *testing.T) { + router := llm.NewDefaultRouter() + provider := &closedProbeProvider{} + router.RegisterProvider(provider) + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second)) + defer cancel() + for attempt := 0; attempt < 32; attempt++ { + _, err := collectHarnessProbeStream(ctx, router, llm.ModelRef{ + Provider: provider.Name(), Model: "model", + }, llm.ChatRequest{}) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("closed stream attempt %d returned %#v", attempt, err) + } + } +} + +type closedProbeProvider struct{} + +func (*closedProbeProvider) Name() string { return "closed-probe" } + +func (*closedProbeProvider) ListModels(context.Context) ([]llm.ModelInfo, error) { + return []llm.ModelInfo{{ID: "model", Provider: "closed-probe"}}, nil +} + +func (*closedProbeProvider) Chat(context.Context, llm.ChatRequest) (*llm.ChatResponse, error) { + return nil, errors.New("closed probe does not support non-streaming chat") +} + +func (*closedProbeProvider) StreamChat(context.Context, + llm.ChatRequest, +) (<-chan llm.ChatChunk, error) { + chunks := make(chan llm.ChatChunk) + close(chunks) + return chunks, nil +} + +func (*closedProbeProvider) SupportsTools(string) bool { return false } +func (*closedProbeProvider) SupportsVision(string) bool { return false } +func (*closedProbeProvider) SupportsJSONMode(string) bool { return false } + type qualificationProvider struct { binding string responseModel string @@ -568,6 +689,9 @@ func (p *qualificationProvider) Chat(ctx context.Context, func (p *qualificationProvider) StreamChat(_ context.Context, request llm.ChatRequest, ) (<-chan llm.ChatChunk, error) { + if request.MaxTokens != harnessProbeMaxTokens { + return nil, errors.New("unexpected qualification token budget") + } response := &llm.ChatResponse{ Provider: p.Name(), Model: p.responseModel, Usage: llm.Usage{InputTokens: 1, OutputTokens: 1, TotalTokens: 2}, diff --git a/web/src/api/client.test.ts b/web/src/api/client.test.ts index 5e354772..9fd796f6 100644 --- a/web/src/api/client.test.ts +++ b/web/src/api/client.test.ts @@ -1072,13 +1072,14 @@ describe("CyberAgentClient", () => { harness_ready: true }; const diagnostic = { protocol_version: "provider_diagnostic.v1", provider: "mock", model: "mock-code", - status: "reachable", outcome: "success", retryable: false, + status: "reachable", outcome: "success", failure_reason: "none", retryable: false, network_request_attempted: false, model_called: true, tool_called: false, response_content_returned: false, duration_ms: 2, }; const qualification = { protocol_version: "model_harness_qualification.v1", provider: "mock", - model: "mock-code", status: "qualified", outcome: "success", retryable: false, + model: "mock-code", status: "qualified", outcome: "success", failure_reason: "none", + retryable: false, network_request_attempted: false, model_calls: 0, synthetic_tool_calls: 0, tool_executed: false, response_content_returned: false, duration_ms: 0, harness: { @@ -2108,7 +2109,7 @@ describe("CyberAgentClient", () => { }); it("keeps Provider credentials write-only and returns status metadata", async () => { - const items = ["anthropic", "deepseek", "mimo"].map((provider) => ({ + const items = ["anthropic", "deepseek", "mimo", "openai"].map((provider) => ({ protocol_version: "provider_credential.v1", provider, configured: false, store_kind: "windows_credential_manager", store_available: true, plaintext_returned: false, restart_required: false, @@ -2141,6 +2142,123 @@ describe("CyberAgentClient", () => { action: "set", secret, confirm: true }); }); + it("accepts OpenAI-compatible availability and rejects unknown qualification reasons", async () => { + const harness = { + protocol_version: "model_harness.v1", model: "gpt-4.1-mini", + transport_protocol: "openai_chat_completions", tool_strategy: "native", + json_strategy: "native", qualification_status: "qualification_required", + tool_calls_qualified: false, tool_results_qualified: false, + strict_json_qualified: false, streaming_qualified: false, root_eligible: false, + structured_json_eligible: false, qualified_at: "", expires_at: "", + }; + const availability = { + protocol_version: "model_availability.v2", generation: 2, + providers: [{ name: "openai", kind: "openai_compatible", status: "available", + models: ["gpt-4.1-mini"], harnesses: [harness], credential_source: "environment", + network_required: true, configuration_error: false }], + routes: [{ name: "code", provider: "openai", model: "gpt-4.1-mini", + available: true, harness_ready: false }], + }; + const diagnostic = { + protocol_version: "provider_diagnostic.v1", provider: "openai", + model: "gpt-4.1-mini", status: "unreachable", outcome: "permanent", + failure_reason: "authentication", retryable: false, network_request_attempted: true, + model_called: true, tool_called: false, response_content_returned: false, + duration_ms: 4, + }; + const fetchMock = vi.fn() + .mockResolvedValueOnce(new Response(JSON.stringify({ version: "api.v1", + request_id: "req-openai-models", data: availability }), { status: 200, + headers: { "Content-Type": "application/json" } })) + .mockResolvedValueOnce(new Response(JSON.stringify({ version: "api.v1", + request_id: "req-openai-diagnostic", data: diagnostic }), { status: 202, + headers: { "Content-Type": "application/json" } })) + .mockResolvedValueOnce(new Response(JSON.stringify({ version: "api.v1", + request_id: "req-openai-forged", data: { ...diagnostic, failure_reason: "raw_error" } }), + { status: 202, headers: { "Content-Type": "application/json" } })); + vi.stubGlobal("fetch", fetchMock); + const client = new CyberAgentClient("read-secret", "/api/v1", "control-secret", { + modelControlEnabled: true, + }); + await expect(client.modelAvailability()).resolves.toEqual(availability); + const request = { version: "provider_diagnostic.v1" as const, provider: "openai", + model: "gpt-4.1-mini", confirm_diagnostic: true }; + await expect(client.diagnoseProvider(request)).resolves.toEqual(diagnostic); + await expect(client.diagnoseProvider(request)).rejects.toThrow("content-free"); + }); + + it("rejects contradictory provider diagnostic semantics", async () => { + const forged = { + protocol_version: "provider_diagnostic.v1", provider: "openai", + model: "gpt-4.1-mini", status: "unreachable", outcome: "success", + failure_reason: "authentication", retryable: false, network_request_attempted: true, + model_called: true, tool_called: false, response_content_returned: false, + duration_ms: 4, + }; + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response(JSON.stringify({ + version: "api.v1", request_id: "req-openai-forged-semantics", data: forged, + }), { status: 202, headers: { "Content-Type": "application/json" } }))); + const client = new CyberAgentClient("read-secret", "/api/v1", "control-secret", { + modelControlEnabled: true, + }); + await expect(client.diagnoseProvider({ version: "provider_diagnostic.v1", provider: "openai", + model: "gpt-4.1-mini", confirm_diagnostic: true })).rejects.toThrow("content-free"); + }); + + it("rejects unknown or non-string diagnostic and qualification enums", async () => { + const diagnostic = { + protocol_version: "provider_diagnostic.v1", provider: "openai", + model: "gpt-4.1-mini", status: "unreachable", outcome: "vendor_error", + failure_reason: "authentication", retryable: false, network_request_attempted: true, + model_called: true, tool_called: false, response_content_returned: false, + duration_ms: 4, + }; + const qualification = { + protocol_version: "model_harness_qualification.v1", provider: "openai", + model: "gpt-4.1-mini", status: "unreachable", outcome: "vendor_error", + failure_reason: "authentication", retryable: false, network_request_attempted: true, + model_calls: 1, synthetic_tool_calls: 0, tool_executed: false, + response_content_returned: false, duration_ms: 4, + harness: { + protocol_version: "model_harness.v1", model: "gpt-4.1-mini", + transport_protocol: "openai_chat_completions", tool_strategy: "native", + json_strategy: "native", qualification_status: "qualification_required", + tool_calls_qualified: false, tool_results_qualified: false, + strict_json_qualified: false, streaming_qualified: false, root_eligible: false, + structured_json_eligible: false, qualified_at: "", expires_at: "", + }, + }; + const fetchMock = vi.fn() + .mockResolvedValueOnce(new Response(JSON.stringify({ version: "api.v1", + request_id: "req-unknown-diagnostic-outcome", data: diagnostic }), { status: 202, + headers: { "Content-Type": "application/json" } })) + .mockResolvedValueOnce(new Response(JSON.stringify({ version: "api.v1", + request_id: "req-unknown-qualification-outcome", data: qualification }), { status: 202, + headers: { "Content-Type": "application/json" } })) + .mockResolvedValueOnce(new Response(JSON.stringify({ version: "api.v1", + request_id: "req-array-diagnostic-outcome", + data: { ...diagnostic, outcome: ["permanent"] } }), { status: 202, + headers: { "Content-Type": "application/json" } })) + .mockResolvedValueOnce(new Response(JSON.stringify({ version: "api.v1", + request_id: "req-array-qualification-reason", + data: { ...qualification, outcome: "permanent", failure_reason: ["authentication"] } }), + { status: 202, headers: { "Content-Type": "application/json" } })); + vi.stubGlobal("fetch", fetchMock); + const client = new CyberAgentClient("read-secret", "/api/v1", "control-secret", { + modelControlEnabled: true, + }); + await expect(client.diagnoseProvider({ version: "provider_diagnostic.v1", provider: "openai", + model: "gpt-4.1-mini", confirm_diagnostic: true })).rejects.toThrow("content-free"); + await expect(client.qualifyModelHarness({ version: "model_harness_qualification.v1", + provider: "openai", model: "gpt-4.1-mini", confirm_qualification: true })) + .rejects.toThrow("qualification response is invalid"); + await expect(client.diagnoseProvider({ version: "provider_diagnostic.v1", provider: "openai", + model: "gpt-4.1-mini", confirm_diagnostic: true })).rejects.toThrow("content-free"); + await expect(client.qualifyModelHarness({ version: "model_harness_qualification.v1", + provider: "openai", model: "gpt-4.1-mini", confirm_qualification: true })) + .rejects.toThrow("qualification response is invalid"); + }); + it("creates only a pending FileEdit from an opaque Go-issued source", async () => { const source = { protocol_version: "file_edit_proposal.v1", run_id: "run-1", workspace_id: "workspace-1", path: "README.md", content: "before\n", diff --git a/web/src/api/client.ts b/web/src/api/client.ts index ba8dcc3a..237541e0 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -497,7 +497,8 @@ function parseModelAvailability(value: unknown): ModelAvailabilityView { for (const provider of value.providers) { if (!hasExactKeys(provider, ["configuration_error", "credential_source", "kind", "models", "harnesses", "name", "network_required", "status"]) || !boundedText(provider.name, 128) || - (provider.kind !== "local" && provider.kind !== "anthropic_compatible") || + !["local", "anthropic_compatible", "openai_compatible"].includes( + String(provider.kind)) || (provider.status !== "available" && provider.status !== "not_configured" && provider.status !== "invalid_configuration") || (provider.credential_source !== "none" && provider.credential_source !== "environment" && @@ -547,7 +548,7 @@ function parseModelHarnessAvailability(value: unknown): ModelHarnessAvailability "strict_json_qualified", "structured_json_eligible", "tool_calls_qualified", "tool_results_qualified", "tool_strategy", "transport_protocol"]) || value.protocol_version !== "model_harness.v1" || !boundedText(value.model, 256) || - !["mock", "anthropic_messages", "provider_contract"].includes( + !["mock", "anthropic_messages", "openai_chat_completions", "provider_contract"].includes( String(value.transport_protocol)) || !["native", "none"].includes(String(value.tool_strategy)) || !["native", "prompt", "none"].includes(String(value.json_strategy)) || @@ -921,19 +922,22 @@ function parseModelRouteControl(value: unknown, route: string, function parseModelHarnessQualification(value: unknown, request: ModelHarnessQualificationRequestView): ModelHarnessQualificationView { - if (!hasExactKeys(value, ["duration_ms", "harness", "model", "model_calls", + if (!hasExactKeys(value, ["duration_ms", "failure_reason", "harness", "model", "model_calls", "network_request_attempted", "outcome", "protocol_version", "provider", "response_content_returned", "retryable", "status", "synthetic_tool_calls", "tool_executed"]) || value.protocol_version !== "model_harness_qualification.v1" || value.provider !== request.provider || value.model !== request.model || !["qualified", "incompatible", "unreachable"].includes(String(value.status)) || - !boundedText(value.outcome, 64) || typeof value.retryable !== "boolean" || + !providerFailureReasonValid(value.failure_reason) || + !providerOutcomeValid(value.outcome) || typeof value.retryable !== "boolean" || typeof value.network_request_attempted !== "boolean" || !safeBoundedCount(value.model_calls, 2) || !safeBoundedCount(value.synthetic_tool_calls, 16) || value.tool_executed !== false || value.response_content_returned !== false || - !safeBoundedCount(value.duration_ms, 60_000)) { + !safeBoundedCount(value.duration_ms, 60_000) || + !qualificationResultSemanticsValid(value.status, value.outcome, value.failure_reason, + value.retryable, value.network_request_attempted, value.model_calls)) { throw new APIRequestError("Model Harness qualification response is invalid", "INVALID_RESPONSE", 502); } @@ -947,28 +951,79 @@ function parseModelHarnessQualification(value: unknown, } function parseProviderDiagnostic(value: unknown, request: ProviderDiagnosticRequestView): ProviderDiagnosticView { - if (!hasExactKeys(value, ["duration_ms", "model", "model_called", + if (!hasExactKeys(value, ["duration_ms", "failure_reason", "model", "model_called", "network_request_attempted", "outcome", "protocol_version", "provider", "response_content_returned", "retryable", "status", "tool_called"]) || value.protocol_version !== "provider_diagnostic.v1" || value.provider !== request.provider || value.model !== request.model || (value.status !== "reachable" && value.status !== "unreachable") || - !boundedText(value.outcome, 64) || typeof value.retryable !== "boolean" || - typeof value.network_request_attempted !== "boolean" || value.model_called !== true || + !providerFailureReasonValid(value.failure_reason) || + !providerOutcomeValid(value.outcome) || typeof value.retryable !== "boolean" || + typeof value.network_request_attempted !== "boolean" || typeof value.model_called !== "boolean" || value.tool_called !== false || value.response_content_returned !== false || - !safeBoundedCount(value.duration_ms, 60_000)) { + !safeBoundedCount(value.duration_ms, 60_000) || + !diagnosticResultSemanticsValid(value.status, value.outcome, value.failure_reason, + value.retryable, value.network_request_attempted, value.model_called ? 1 : 0)) { throw new APIRequestError("Provider diagnostic response violated its content-free contract", "INVALID_RESPONSE", 502); } return value as unknown as ProviderDiagnosticView; } +function providerFailureReasonValid(value: unknown): boolean { + return typeof value === "string" && + ["none", "not_configured", "authentication", "network", "rate_limit", "capacity", + "model_not_found", "protocol_incompatible"].includes(value); +} + +function providerOutcomeValid(value: unknown): boolean { + return typeof value === "string" && + ["success", "retryable", "rate_limited", "invalid_response", "cancelled", + "permanent"].includes(value); +} + +function qualificationResultSemanticsValid(status: unknown, outcome: unknown, reason: unknown, + retryable: unknown, networkAttempted: unknown, modelCalls: unknown): boolean { + if (status === "qualified") { + return outcome === "success" && reason === "none" && retryable === false && + (modelCalls === 0 || modelCalls === 2); + } + if (reason === "not_configured") { + return status === "unreachable" && outcome === "permanent" && retryable === false && + networkAttempted === false && modelCalls === 0; + } + if (status === "incompatible") { + return outcome === "invalid_response" && reason === "protocol_incompatible" && + retryable === false; + } + if (status !== "unreachable" || reason === "none" || outcome === "success" || + outcome === "invalid_response") { + return false; + } + return retryable === (outcome === "retryable" || outcome === "rate_limited"); +} + +function diagnosticResultSemanticsValid(status: unknown, outcome: unknown, reason: unknown, + retryable: unknown, networkAttempted: unknown, modelCalls: unknown): boolean { + if (status === "reachable") { + return outcome === "success" && reason === "none" && retryable === false && modelCalls === 1; + } + if (reason === "not_configured") { + return outcome === "permanent" && retryable === false && networkAttempted === false && + modelCalls === 0; + } + if (reason === "none" || outcome === "success" || modelCalls !== 1) { + return false; + } + return retryable === (outcome === "retryable" || outcome === "rate_limited"); +} + function parseProviderCredentialStatus(value: unknown, expectedProvider = ""): ProviderCredentialStatusView { if (!hasExactKeys(value, ["configured", "plaintext_returned", "protocol_version", "provider", "registry_generation", "registry_reloaded", "restart_required", "store_available", "store_kind"]) || value.protocol_version !== "provider_credential.v1" || - !["anthropic", "deepseek", "mimo"].includes(String(value.provider)) || + !["anthropic", "deepseek", "mimo", "openai"].includes(String(value.provider)) || (expectedProvider !== "" && value.provider !== expectedProvider) || typeof value.configured !== "boolean" || typeof value.store_available !== "boolean" || !boundedText(value.store_kind, 128) || value.plaintext_returned !== false || @@ -984,7 +1039,7 @@ function parseProviderCredentialStatus(value: unknown, function parseProviderCredentialList(value: unknown): ProviderCredentialListView { if (!hasExactKeys(value, ["items", "protocol_version"]) || value.protocol_version !== "provider_credential.v1" || !Array.isArray(value.items) || - value.items.length !== 3) { + value.items.length !== 4) { throw new APIRequestError("Provider credential list is invalid", "INVALID_RESPONSE", 502); } const items = value.items.map((item) => parseProviderCredentialStatus(item)); @@ -3734,7 +3789,8 @@ export class CyberAgentClient { async changeProviderCredential(provider: string, body: ProviderCredentialRequestView, signal?: AbortSignal): Promise { - if (!this.hasProviderCredentials || !["anthropic", "deepseek", "mimo"].includes(provider) || + if (!this.hasProviderCredentials || + !["anthropic", "deepseek", "mimo", "openai"].includes(provider) || body.version !== "provider_credential.v1" || body.confirm !== true || (body.action === "set" ? typeof body.secret !== "string" || body.secret.length < 8 : body.action !== "delete" || body.secret !== "")) { diff --git a/web/src/api/schema.d.ts b/web/src/api/schema.d.ts index f3953b7c..ade37d8d 100644 --- a/web/src/api/schema.d.ts +++ b/web/src/api/schema.d.ts @@ -3103,7 +3103,7 @@ export interface components { /** @enum {string} */ tool_strategy: "native" | "none"; /** @enum {string} */ - transport_protocol: "mock" | "anthropic_messages" | "provider_contract"; + transport_protocol: "mock" | "anthropic_messages" | "openai_chat_completions" | "provider_contract"; }; ModelHarnessQualificationRequestView: { confirm_qualification: boolean; @@ -3115,6 +3115,8 @@ export interface components { ModelHarnessQualificationView: { /** Format: int64 */ duration_ms: number; + /** @enum {string} */ + failure_reason: "none" | "not_configured" | "authentication" | "network" | "rate_limit" | "capacity" | "model_not_found" | "protocol_incompatible"; harness: components["schemas"]["ModelHarnessAvailabilityView"]; model: string; /** Format: int32 */ @@ -3383,7 +3385,7 @@ export interface components { credential_source: "none" | "environment" | "system"; harnesses: components["schemas"]["ModelHarnessAvailabilityView"][]; /** @enum {string} */ - kind: "local" | "anthropic_compatible"; + kind: "local" | "anthropic_compatible" | "openai_compatible"; models: string[]; name: string; network_required: boolean; @@ -3408,7 +3410,8 @@ export interface components { plaintext_returned: boolean; /** @enum {string} */ protocol_version: "provider_credential.v1"; - provider: string; + /** @enum {string} */ + provider: "anthropic" | "deepseek" | "mimo" | "openai"; /** Format: int64 */ registry_generation: number; registry_reloaded: boolean; @@ -3426,6 +3429,8 @@ export interface components { ProviderDiagnosticView: { /** Format: int64 */ duration_ms: number; + /** @enum {string} */ + failure_reason: "none" | "not_configured" | "authentication" | "network" | "rate_limit" | "capacity" | "model_not_found" | "protocol_incompatible"; model: string; model_called: boolean; network_request_attempted: boolean; @@ -5415,7 +5420,7 @@ export interface operations { header?: never; path: { /** @description Provider name */ - provider: string; + provider: "anthropic" | "deepseek" | "mimo" | "openai"; }; cookie?: never; }; diff --git a/web/src/api/types.ts b/web/src/api/types.ts index c2ff7dde..063c2772 100644 --- a/web/src/api/types.ts +++ b/web/src/api/types.ts @@ -34,6 +34,8 @@ export type MessageView = components["schemas"]["MessageView"]; export type ModelAvailabilityView = components["schemas"]["ModelAvailabilityView"]; export type ModelHarnessAvailabilityView = components["schemas"]["ModelHarnessAvailabilityView"]; export type ModelHarnessQualificationRequestView = components["schemas"]["ModelHarnessQualificationRequestView"]; +export type ProviderFailureReason = + components["schemas"]["ProviderDiagnosticView"]["failure_reason"]; export type ModelHarnessQualificationView = components["schemas"]["ModelHarnessQualificationView"]; export type ModelRouteControlRequestView = components["schemas"]["ModelRouteControlRequestView"]; export type OperationReceiptView = components["schemas"]["OperationReceiptView"]; diff --git a/web/src/components/model-availability-dialog.test.tsx b/web/src/components/model-availability-dialog.test.tsx index f1d8b155..2d233460 100644 --- a/web/src/components/model-availability-dialog.test.tsx +++ b/web/src/components/model-availability-dialog.test.tsx @@ -25,6 +25,13 @@ function requiredHarness(model: string) { }; } +function openAIHarness(model: string) { + return { + ...requiredHarness(model), transport_protocol: "openai_chat_completions", + json_strategy: "native", + }; +} + describe("ModelAvailabilityDialog", () => { it("renders redacted provider and route status without configuration secrets", async () => { const client = { modelAvailability: vi.fn().mockResolvedValue({ @@ -49,7 +56,7 @@ describe("ModelAvailabilityDialog", () => { const user = userEvent.setup(); const diagnoseProvider = vi.fn().mockResolvedValue({ protocol_version: "provider_diagnostic.v1", provider: "mock", model: "mock-code", - status: "reachable", outcome: "success", retryable: false, + status: "reachable", outcome: "success", failure_reason: "none", retryable: false, network_request_attempted: false, model_called: true, tool_called: false, response_content_returned: false, duration_ms: 2, }); @@ -93,7 +100,8 @@ describe("ModelAvailabilityDialog", () => { const user = userEvent.setup(); const qualifyModelHarness = vi.fn().mockResolvedValue({ protocol_version: "model_harness_qualification.v1", provider: "mimo", - model: "model-secondary", status: "qualified", outcome: "success", retryable: false, + model: "model-secondary", status: "qualified", outcome: "success", + failure_reason: "none", retryable: false, network_request_attempted: true, model_calls: 2, synthetic_tool_calls: 1, tool_executed: false, response_content_returned: false, duration_ms: 8, harness: { ...mockHarness("model-secondary"), transport_protocol: "anthropic_messages", @@ -128,10 +136,10 @@ describe("ModelAvailabilityDialog", () => { expect(await screen.findByText("2 model calls")).toBeInTheDocument(); }); - it("submits a Provider secret once and renders status without plaintext", async () => { + it("submits an OpenAI Provider secret once and renders status without plaintext", async () => { const user = userEvent.setup(); const statuses = { protocol_version: "provider_credential.v1", items: - ["anthropic", "deepseek", "mimo"].map((provider) => ({ + ["anthropic", "deepseek", "mimo", "openai"].map((provider) => ({ protocol_version: "provider_credential.v1", provider, configured: false, store_kind: "windows_credential_manager", store_available: true, plaintext_returned: false, restart_required: false, @@ -141,7 +149,7 @@ describe("ModelAvailabilityDialog", () => { const changeProviderCredential = vi.fn().mockImplementation((provider, body) => { submittedCredential = { provider, body: { ...body } }; return Promise.resolve({ - ...statuses.items[2], configured: true, registry_reloaded: true, + ...statuses.items[3], configured: true, registry_reloaded: true, registry_generation: 2, }); }); @@ -162,10 +170,10 @@ describe("ModelAvailabilityDialog", () => { ); const secret = "temporary-provider-key"; - const input = await screen.findByLabelText("mimo API credential"); + const input = await screen.findByLabelText("openai API credential"); await user.type(input, secret); - await user.click(screen.getByRole("button", { name: "Store mimo credential" })); - await waitFor(() => expect(submittedCredential).toEqual({ provider: "mimo", body: { + await user.click(screen.getByRole("button", { name: "Store openai credential" })); + await waitFor(() => expect(submittedCredential).toEqual({ provider: "openai", body: { version: "provider_credential.v1", action: "set", secret, confirm: true, } })); expect(input).toHaveValue(""); @@ -173,4 +181,86 @@ describe("ModelAvailabilityDialog", () => { expect(screen.getByText("Registry generation 2 active")).toBeInTheDocument(); expect(container.textContent).not.toContain(secret); }); + + it("renders OpenAI-compatible transport and safe connection failure reasons", async () => { + const user = userEvent.setup(); + const diagnoseProvider = vi.fn().mockResolvedValue({ + protocol_version: "provider_diagnostic.v1", provider: "openai", + model: "gpt-4.1-mini", status: "unreachable", outcome: "permanent", + failure_reason: "authentication", retryable: false, + network_request_attempted: true, model_called: true, tool_called: false, + response_content_returned: false, duration_ms: 3, + }); + const client = { + hasModelControl: true, diagnoseProvider, + modelAvailability: vi.fn().mockResolvedValue({ + protocol_version: "model_availability.v2", generation: 2, + providers: [{ name: "openai", kind: "openai_compatible", status: "available", + models: ["gpt-4.1-mini"], credential_source: "environment", + network_required: true, configuration_error: false, + harnesses: [openAIHarness("gpt-4.1-mini")] }], + routes: [{ name: "code", provider: "openai", model: "gpt-4.1-mini", + available: true, harness_ready: false }], + }), + } as unknown as CyberAgentClient; + const queryClient = new QueryClient({ defaultOptions: { + queries: { retry: false }, mutations: { retry: false }, + } }); + render( + + ); + expect(await screen.findByText("openai_chat_completions · JSON native")).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Diagnose openai/gpt-4.1-mini" })); + expect(await screen.findByText("authentication failed")).toBeInTheDocument(); + }); + + it("allows a content-free diagnostic for a known unconfigured Provider", async () => { + const user = userEvent.setup(); + const diagnoseProvider = vi.fn().mockResolvedValue({ + protocol_version: "provider_diagnostic.v1", provider: "openai", + model: "gpt-4.1-mini", status: "unreachable", outcome: "permanent", + failure_reason: "not_configured", retryable: false, + network_request_attempted: false, model_called: false, tool_called: false, + response_content_returned: false, duration_ms: 0, + }); + const qualifyModelHarness = vi.fn().mockResolvedValue({ + protocol_version: "model_harness_qualification.v1", provider: "openai", + model: "gpt-4.1-mini", status: "unreachable", outcome: "permanent", + failure_reason: "not_configured", retryable: false, + network_request_attempted: false, model_calls: 0, synthetic_tool_calls: 0, + tool_executed: false, response_content_returned: false, duration_ms: 0, + harness: openAIHarness("gpt-4.1-mini"), + }); + const client = { + hasModelControl: true, diagnoseProvider, qualifyModelHarness, + modelAvailability: vi.fn().mockResolvedValue({ + protocol_version: "model_availability.v2", generation: 2, + providers: [{ name: "openai", kind: "openai_compatible", + status: "not_configured", models: ["gpt-4.1-mini"], + credential_source: "none", network_required: true, + configuration_error: false, harnesses: [openAIHarness("gpt-4.1-mini")] }], + routes: [{ name: "code", provider: "mock", model: "mock-code", + available: true, harness_ready: true }], + }), + } as unknown as CyberAgentClient; + const queryClient = new QueryClient({ defaultOptions: { + queries: { retry: false }, mutations: { retry: false }, + } }); + render( + + ); + await user.click(await screen.findByRole("button", { + name: "Diagnose openai/gpt-4.1-mini", + })); + expect((await screen.findAllByText("not configured")).length).toBe(2); + expect(diagnoseProvider).toHaveBeenCalledWith({ version: "provider_diagnostic.v1", + provider: "openai", model: "gpt-4.1-mini", confirm_diagnostic: true }); + await user.click(screen.getByRole("button", { + name: "Qualify openai/gpt-4.1-mini Harness", + })); + await waitFor(() => expect(qualifyModelHarness).toHaveBeenCalledWith({ + version: "model_harness_qualification.v1", provider: "openai", + model: "gpt-4.1-mini", confirm_qualification: true, + })); + }); }); diff --git a/web/src/components/model-availability-dialog.tsx b/web/src/components/model-availability-dialog.tsx index 840f7ef5..eaa272ba 100644 --- a/web/src/components/model-availability-dialog.tsx +++ b/web/src/components/model-availability-dialog.tsx @@ -28,6 +28,19 @@ function ModelAvailabilitySurface({ client, open, onClose, presentation }: { presentation: "dialog" | "workspace"; }) { const { t } = useLocale(); + const failureReasonLabel = (reason: string) => { + const labels: Record = { + not_configured: ["未配置", "not configured"], + authentication: ["身份验证失败", "authentication failed"], + network: ["网络不可达", "network unreachable"], + rate_limit: ["达到速率限制", "rate limited"], + capacity: ["Provider 容量不足", "Provider capacity unavailable"], + model_not_found: ["模型不存在", "model not found"], + protocol_incompatible: ["协议不兼容", "protocol incompatible"], + }; + const label = labels[reason]; + return label ? t(label[0], label[1]) : reason; + }; const queryClient = useQueryClient(); const [selections, setSelections] = useState>({}); const [diagnostic, setDiagnostic] = useState(null); @@ -143,7 +156,9 @@ function ModelAvailabilitySurface({ client, open, onClose, presentation }: { : provider.credential_source} {harness && } - {client.hasModelControl && provider.status === "available" && + {client.hasModelControl && + (provider.status === "available" || + provider.status === "not_configured") && model && ( <>