Add Test connection to Bot settings → Model

changelog.d/409-test-connection.added.md

@@ -0,0 +1 @@
+Bot settings → Model has a **Test connection** button: one click sends one short request to that Bot's model on the same route a run uses, then shows that it works or a short reason (no or rejected API key, model not found, cannot reach the endpoint, timeout, rate limit). The request may be billed, and the row says so; keys and provider text are never shown ([#409](https://github.com/ThinkFlowLab/FlyBot/issues/409)).

core/src/client/bot-settings.js

@@ -6,6 +6,7 @@ import { isMainOf } from './sources.js'
 import { PencilIcon, ChevronRightIcon } from './icons.js'
 import { lookOf, BotMark } from './mark.js'
 import { ModelMenu, modelOf } from './model-menu.js'
+import { ModelTestRow, MODEL_TEST_CSS } from './model-test.js'
 import { FolderMenu } from './folder-menu.js'
 import { BotSchedules } from './schedules.js'
 import { BotCapabilitiesCard } from './capability-settings.js'
@@ -227,7 +228,8 @@ export function BotSettingsForm({ bot, roster, actions, run }) {
     h(Card, { title: t('Model') },
       h('div', { className: 'bt-set-row bt-bs-row' },
         h('span', { className: 'bt-set-copy' }, h('span', { className: 'bt-set-hint' }, modelHint(bot, roster))),
-        h(ModelMenu, { bot, roster, actions, run }))),
+        h(ModelMenu, { bot, roster, actions, run })),
+      bot.model ? h(ModelTestRow, { bot, modelName: modelOf(bot, roster)?.name }) : null),
     h(Card, { title: t('Working folder') },
       h('div', { className: 'bt-set-row bt-bs-row' },
         h('span', { className: 'bt-set-copy' }, h('span', { className: 'bt-bs-label' }, t('Folder'))),
@@ -275,6 +277,7 @@ label.bt-bs-row{cursor:text}
 .bt-bs-row>.bt-set-copy{flex:1 1 auto}
 .bt-bs-label{font-size:13px;line-height:18px;font-weight:500}
 .bt-bs-row .bt-model-trigger{flex:none;max-width:58%}
+${MODEL_TEST_CSS}
 .bt-bs-input{flex:0 1 220px;width:0;min-width:96px;height:30px;box-sizing:border-box;margin:-4px -6px -4px 0;padding:0 8px;border:1px solid transparent;border-radius:8px;background:none;color:var(--bt-ink);font:inherit;font-size:13px;text-align:right;outline:none;transition:background-color .12s ease,border-color .12s ease}
 .bt-bs-input:hover{background:var(--bt-hover)}
 .bt-bs-input:focus{background:var(--bt-main);border-color:var(--bt-line-2);text-align:left}

core/src/client/i18n.js

@@ -346,6 +346,25 @@ const ZH = {
   'This model is not available now. Pick another one.': '这个模型现在不可用，请换一个。',
   'Every message of this Bot runs on it.': '这个 Bot 的每条消息都用它。',
   'Applies from the next message.': '从下一条消息开始生效。',
+  'Test connection': '测试连接',
+  'Testing…': '正在测试…',
+  'Sends one short request to this model. It may be billed.': '会向这个模型发送一条很短的请求，可能产生费用。',
+  'Works. {model} answered.': '连接正常，{model} 已回复。',
+  'Works. {model} answered in {seconds} s.': '连接正常，{model} 在 {seconds} 秒内回复。',
+  'Pick a model first.': '请先选择一个模型。',
+  'This model has no API key yet. Set one up in the model settings.': '这个模型还没有 API 密钥，请在模型设置里填写。',
+  'The API key was rejected. Check it in the model settings.': 'API 密钥被拒绝，请在模型设置里检查。',
+  'The account is out of quota.': '账户额度已用完。',
+  'Too many requests right now. Try again in a moment.': '请求太频繁，请稍后再试。',
+  'The model did not answer within 30 seconds.': '模型在 30 秒内没有回复。',
+  'This model is not available now. Check its provider in the model settings.': '这个模型现在不可用，请在模型设置里检查它的服务商。',
+  'The provider does not know this model. Check the model name.': '服务商没有这个模型，请检查模型名称。',
+  'Could not reach the model service. Check the endpoint address and your network.': '连不上模型服务，请检查接口地址和网络。',
+  'The model service failed. Try again later.': '模型服务出错，请稍后再试。',
+  'The model service refused the request. Check the endpoint and the model.': '模型服务拒绝了请求，请检查接口地址和模型。',
+  'The test failed.': '测试失败。',
+  'A test is already running for this Bot.': '这个 Bot 已经在测试中。',
+  "A friend's Bot runs on its owner's model.": '朋友的 Bot 使用它主人的模型。',
   Profile: '资料',
   Label: '标签',
   'Optional. Shown under the name.': '可选，显示在名字下方。',

core/src/client/model-test.js

@@ -0,0 +1,65 @@
+// Test connection (#409): one tiny real request on the Bot's own model route. The
+// host answers with a code only, never the provider's text, which can echo a key.
+import { createElement as h, useEffect, useRef, useState } from 'react'
+import { call } from './api.js'
+import { t } from './i18n.js'
+
+const FAILURE_COPY = {
+  NO_MODEL: 'Pick a model first.',
+  NO_KEY: 'This model has no API key yet. Set one up in the model settings.',
+  AUTH: 'The API key was rejected. Check it in the model settings.',
+  QUOTA: 'The account is out of quota.',
+  RATE_LIMIT: 'Too many requests right now. Try again in a moment.',
+  TIMEOUT: 'The model did not answer within 30 seconds.',
+  NOT_SERVED: 'This model is not available now. Check its provider in the model settings.',
+  MODEL_NOT_FOUND: 'The provider does not know this model. Check the model name.',
+  TRANSPORT: 'Could not reach the model service. Check the endpoint address and your network.',
+  SERVER: 'The model service failed. Try again later.',
+  INVALID_REQUEST: 'The model service refused the request. Check the endpoint and the model.',
+  UNKNOWN: 'The test failed.',
+}
+
+// The words for one test outcome: `{ ok, text, code? }`.
+export function testOutcome(result, modelName) {
+  if (result?.ok === true) {
+    const seconds = typeof result.ms === 'number' ? Math.max(0.1, Math.round(result.ms / 100) / 10) : undefined
+    return { ok: true, text: seconds === undefined ? t('Works. {model} answered.', { model: modelName }) : t('Works. {model} answered in {seconds} s.', { model: modelName, seconds }) }
+  }
+  const code = Object.hasOwn(FAILURE_COPY, result?.code) ? result.code : 'UNKNOWN'
+  return { ok: false, code, text: t(FAILURE_COPY[code]) }
+}
+
+export const testModel = id => call('test-model', { id })
+
+// A row under the model picker. `test` is the request, swappable for tests.
+export function ModelTestRow({ bot, modelName, test = testModel }) {
+  const [state, setState] = useState({ busy: false, outcome: undefined })
+  const live = useRef(true)
+  useEffect(() => () => { live.current = false }, [])
+  // A result belongs to the model it ran on; picking another clears it.
+  useEffect(() => { setState({ busy: false, outcome: undefined }) }, [bot.id, bot.model])
+  const start = () => {
+    if (state.busy) return
+    setState({ busy: true, outcome: undefined })
+    Promise.resolve().then(() => test(bot.id)).then(
+      result => testOutcome(result, modelName ?? result?.ref ?? ''),
+      error => ({ ok: false, text: String(error?.message ?? t('The test failed.')) }),
+    ).then((outcome) => { if (live.current) setState({ busy: false, outcome }) })
+  }
+  const { busy, outcome } = state
+  return h('div', { className: 'bt-set-row bt-bs-row bt-model-test' },
+    h('span', { className: 'bt-set-copy' },
+      h('span', { className: 'bt-set-hint' }, t('Sends one short request to this model. It may be billed.')),
+      h('span', { className: 'bt-model-test-result', role: 'status', 'aria-live': 'polite', 'data-ok': outcome === undefined ? undefined : String(outcome.ok) },
+        busy ? t('Testing…') : outcome === undefined ? null : outcome.text,
+        outcome?.code ? h('code', { className: 'bt-model-test-code' }, outcome.code) : null)),
+    h('button', { type: 'button', className: 'bt-soft bt-soft-sm', disabled: busy, 'aria-busy': busy || undefined, onClick: start }, t('Test connection')))
+}
+
+export const MODEL_TEST_CSS = `
+.bt-model-test-result{font-size:13px;line-height:18px;color:var(--bt-ink-2)}
+.bt-model-test-result:empty{display:none}
+.bt-model-test-result[data-ok=true]{color:#1a7f37}
+.bt-model-test-result[data-ok=false]{color:#c21d2e}
+.bt-model-test-code{margin-left:6px;font-size:11px;color:var(--bt-ink-3)}
+`

core/src/host/authz.js

@@ -40,7 +40,7 @@ const BOT_TARGETED = new Set([
   'memory-view', 'memory-ask', 'memory-forget',
   'schedules', 'schedule-create', 'schedule-delete',
   'agent-session', 'agent-adopt', 'agent-archive', 'agent-rename', 'remove-agent',
-  'stop-member-work', 'set-main',
+  'stop-member-work', 'set-main', 'test-model',
 ])
 
 // Endpoints whose body names a room.

core/src/host/channel.js

@@ -190,6 +190,9 @@ export function install(rt) {
           return ok({ modelsKey: modelsKey() })
         case 'efforts':
           return ok(await rt.reasoningOf(String(body.ref ?? '')) ?? { efforts: [] })
+        case 'test-model':
+          await rt.load()
+          return ok(await rt.testModel(String(body.id ?? '')))
         case 'duplicate-bot':
           await rt.load()
           return ok(await rt.duplicateBot(body.id))

core/src/host/models.js

@@ -5,6 +5,28 @@ import { normalize, splitRef } from './text.js'
 
 const CATALOG_TTL_MS = 10_000
 const ROUTE_WAIT_MS = 30_000
+const TEST_TIMEOUT_MS = 30_000
+const TEST_MAX_TOKENS = 16
+const TEST_PROMPT = 'Reply with the single word OK.'
+
+// The outcome of a connection test as a code only (#409). A provider's own text can
+// echo part of a key, so none of it leaves the host: the client words each code.
+export function testFailure(failure, aborted = false) {
+  const code = typeof failure?.code === 'string' ? failure.code : ''
+  const message = String(failure?.message ?? '')
+  const status = typeof failure?.status === 'number' ? failure.status : Number(/^HTTP_(\d{3})$/.exec(code)?.[1] ?? NaN)
+  if (/\bno api key\b|\bmissing (?:an? )?api key\b|\bapi key (?:is )?(?:missing|not set|not configured)\b/i.test(message)) return 'NO_KEY'
+  if (code === 'AUTH' || code === 'INVALID_CREDENTIAL' || status === 401 || status === 403) return 'AUTH'
+  if (code === 'QUOTA' || code === 'ACCOUNT_QUOTA' || status === 402) return 'QUOTA'
+  if (code === 'RATE_LIMIT' || status === 429) return 'RATE_LIMIT'
+  if (aborted || code === 'TIMEOUT' || code === 'ABORTED') return 'TIMEOUT'
+  if (code === 'NO_ADAPTER') return 'NOT_SERVED'
+  if (status === 404 || /\bmodel\b.{0,40}\b(?:not found|does not exist|not exist|unknown|not supported)\b/i.test(message)) return 'MODEL_NOT_FOUND'
+  if (code === 'TRANSPORT') return 'TRANSPORT'
+  if (code === 'SERVER' || (status >= 500 && status < 600)) return 'SERVER'
+  if (code === 'INVALID_REQUEST' || status === 400) return 'INVALID_REQUEST'
+  return 'UNKNOWN'
+}
 
 export function install(rt) {
   const { ctx, state } = rt
@@ -202,7 +224,54 @@ export function install(rt) {
     }
   }
 
+  // Test connection (#409): one tiny real request on the route the Bot's next run takes,
+  // so a pass means that run can reach its model. It may be billed, so a Bot runs
+  // at most one test at a time and nothing retries it.
+  const testing = new Set()
+  async function testModel(botId) {
+    const bot = state.bots?.[botId]
+    if (bot === undefined) throw Object.assign(new Error('Bot not found'), { code: 'bot/not-found' })
+    if (bot.remote) throw Object.assign(new Error('A friend\'s Bot runs on its owner\'s model.'), { code: 'bot/invalid' })
+    if (testing.has(botId)) throw Object.assign(new Error('A test is already running for this Bot.'), { code: 'bot/refused' })
+    testing.add(botId)
+    const started = Date.now()
+    try {
+      const ref = await desiredModel(bot)
+      if (ref === undefined) return { ok: false, code: 'NO_MODEL' }
+      const route = splitRef(ref)
+      const reasoning = await reasoningOf(ref)
+      const efforts = reasoning?.efforts ?? []
+      // The least thinking the model offers keeps the test to a handful of tokens.
+      const effort = efforts.some(entry => entry.id === 'off') ? 'off' : efforts.at(0)?.id
+      const sessionId = rt.chatOf?.(bot.id)
+      const messages = [{ role: 'user', content: [{ type: 'text', text: TEST_PROMPT }] }]
+      rt.tagUsage?.(messages, { sessionId, kind: 'other' })
+      const signal = AbortSignal.timeout(TEST_TIMEOUT_MS)
+      let finish
+      try {
+        for await (const chunk of ctx.llm.stream({
+          provider: route.provider,
+          model: route.model,
+          ...(sessionId === undefined ? {} : { sessionId }),
+          ...(effort === undefined ? {} : { reasoningEffort: effort }),
+          messages,
+          maxTokens: TEST_MAX_TOKENS,
+          signal,
+        })) if (chunk?.type === 'finish') finish = chunk.reason
+      } catch (error) {
+        return { ok: false, ref, code: testFailure(error, signal.aborted) }
+      }
+      // A stream that ends without its finish chunk proved nothing.
+      if (finish === undefined) return { ok: false, ref, code: 'UNKNOWN' }
+      if (finish.kind === 'error' || finish.kind === 'aborted') return { ok: false, ref, code: testFailure(finish.failure, finish.kind === 'aborted') }
+      return { ok: true, ref, ms: Date.now() - started }
+    } finally {
+      testing.delete(botId)
+    }
+  }
+
   Object.assign(rt, {
+    testModel,
     catalog, cachedCatalog: () => catalogCache.value, cachedDefault: () => pickDefault(catalogCache.value),
     defaultRef, defaultModel, resolveModel, desiredModel, reasoningOf, resolveEffort, desiredEffort,
     selectSessionModel, applyModel, waitForModel, syncModels,

core/test/authz.test.js

@@ -85,8 +85,10 @@ test('owner cannot touch another owner Bot or room', async (t) => {
   assert.ok(actorA)
   assert.equal(actorA.memberId, 'owner-1')
   assert.equal(rt.authorize(actorA, 'update-bot', { id: 'a1' }).ok, true)
+  // A connection test may be billed: only the Bot's owner can start one (#409).
+  assert.equal(rt.authorize(actorA, 'test-model', { id: 'a1' }).ok, true)
 
-  for (const ep of ['update-bot', 'delete-bot', 'memory-view', 'memory-forget', 'set-flags']) {
+  for (const ep of ['update-bot', 'delete-bot', 'memory-view', 'memory-forget', 'set-flags', 'test-model']) {
     const gate = rt.authorize(actorA, ep, { id: 'b1' })
     assert.equal(gate.ok, false, ep)
     assert.equal(gate.code, 'bot/forbidden', ep)

core/test/model-test.test.js

@@ -0,0 +1,66 @@
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+import { fileURLToPath } from 'node:url'
+import { runInNewContext } from 'node:vm'
+import { build } from 'esbuild'
+
+// Test connection (#409), client side: the copy for each outcome and the row's
+// billing note. The request itself is a stub; nothing reaches a model.
+const bundle = await build({
+  entryPoints: [fileURLToPath(new URL('../src/client/model-test.js', import.meta.url))],
+  bundle: true, format: 'cjs', platform: 'node', write: false,
+  external: ['react', 'react-dom'],
+})
+const module = { exports: {} }
+runInNewContext(bundle.outputFiles[0].text, {
+  module, exports: module.exports,
+  require: (name) => {
+    if (name === 'react') {
+      return {
+        createElement: (type, props, ...children) => ({ type, props: props ?? {}, children: children.flat(Infinity) }),
+        useState: initial => [initial, () => {}],
+        useEffect: () => {},
+        useRef: initial => ({ current: initial }),
+      }
+    }
+    throw new Error(`no platform module ${name}`)
+  },
+})
+const { ModelTestRow } = module.exports
+// Copied into this realm, so deepEqual compares values, not the vm's prototypes.
+const testOutcome = (...args) => ({ ...module.exports.testOutcome(...args) })
+
+const textOf = node => (node === null || node === undefined || node === false ? '' : typeof node !== 'object' ? String(node) : node.children.map(textOf).join(' '))
+const find = (node, match) => {
+  if (node === null || typeof node !== 'object') return undefined
+  if (match(node)) return node
+  for (const child of node.children) {
+    const found = find(child, match)
+    if (found) return found
+  }
+  return undefined
+}
+
+test('a passing test names the model and how long it took', () => {
+  assert.deepEqual(testOutcome({ ok: true, ref: 'p/m', ms: 1234 }, 'Kimi K3'), { ok: true, text: 'Works. Kimi K3 answered in 1.2 s.' })
+  assert.deepEqual(testOutcome({ ok: true }, 'Kimi K3'), { ok: true, text: 'Works. Kimi K3 answered.' })
+})
+
+test('each failure code reads as a short, actionable reason, and an unknown code stays generic', () => {
+  assert.match(testOutcome({ ok: false, code: 'AUTH' }).text, /API key was rejected/)
+  assert.match(testOutcome({ ok: false, code: 'NO_KEY' }).text, /no API key yet/)
+  assert.match(testOutcome({ ok: false, code: 'TRANSPORT' }).text, /endpoint/)
+  assert.match(testOutcome({ ok: false, code: 'MODEL_NOT_FOUND' }).text, /model name/)
+  assert.match(testOutcome({ ok: false, code: 'TIMEOUT' }).text, /30 seconds/)
+  assert.match(testOutcome({ ok: false, code: 'RATE_LIMIT' }).text, /Too many requests/)
+  assert.deepEqual(testOutcome({ ok: false, code: 'sk-secret-<b>' }), { ok: false, code: 'UNKNOWN', text: 'The test failed.' })
+})
+
+test('the row offers Test connection and says the request may be billed', () => {
+  const row = ModelTestRow({ bot: { id: 'b1', model: 'p/m' }, modelName: 'M', test: async () => ({ ok: true }) })
+  assert.match(textOf(row), /may be billed/)
+  const button = find(row, node => node.type === 'button')
+  assert.equal(textOf(button).trim(), 'Test connection')
+  assert.equal(button.props.disabled, false)
+  assert.equal(find(row, node => node.props?.role === 'status')?.props['aria-live'], 'polite')
+})