From 99d0270e5f57905ec45b3b229c326cbc7e0033a1 Mon Sep 17 00:00:00 2001 From: yyh Date: Tue, 20 Oct 2026 15:57:02 +0800 Subject: [PATCH] fix(e2e): seed workspace default models explicitly --- .github/workflows/post-merge.yml | 15 +- .../agent-v2/configure-persistence.feature | 1 + e2e/features/agent-v2/support/seed.ts | 131 ++++++----- .../agent-v2/configure.steps.ts | 9 +- .../agent-v2/fixtures.steps.ts | 12 + e2e/features/support/world.ts | 2 + e2e/tests/agent-v2-seed.test.ts | 7 +- .../workspace-default-model-seed.test.ts | 212 ++++++++++++++++++ 8 files changed, 328 insertions(+), 61 deletions(-) create mode 100644 e2e/tests/workspace-default-model-seed.test.ts diff --git a/.github/workflows/post-merge.yml b/.github/workflows/post-merge.yml index 4b00687fb6ee4a..adb0aa6d58195f 100644 --- a/.github/workflows/post-merge.yml +++ b/.github/workflows/post-merge.yml @@ -1,6 +1,7 @@ name: Post-Merge Checks on: + workflow_dispatch: push: branches: ['main'] @@ -52,6 +53,12 @@ jobs: - 'api/tests/integration_tests/.env.example' - 'api/clients/agent_backend/**' - 'api/core/app/apps/agent_app/**' + - 'api/core/model_manager.py' + - 'api/core/provider_manager.py' + - 'api/controllers/console/workspace/models.py' + - 'api/controllers/console/workspace/model_providers.py' + - 'api/services/model_provider/**' + - 'api/tasks/install_default_plugins_task.py' - 'api/core/workflow/nodes/agent_v2/**' - 'api/controllers/console/agent/**' - 'api/controllers/console/app/agent*.py' @@ -70,6 +77,12 @@ jobs: - 'api/services/entities/agent_tool_inner.py' - 'api/services/errors/agent_tool_inner.py' - 'api/services/agent/**' + - 'api/services/app/agent_*.py' + - 'api/repositories/app/agent_*.py' + - 'api/extensions/application_services/agent.py' + - 'api/models/agent.py' + - 'api/models/agent_config_entities.py' + - 'api/services/entities/agent_entities.py' - 'api/core/plugin/**' - 'api/services/plugin/**' - 'api/core/tools/**' @@ -112,7 +125,7 @@ jobs: external-e2e: name: External Runtime E2E needs: check-changes - if: needs.check-changes.outputs.external-e2e-changed == 'true' + if: github.event_name == 'workflow_dispatch' || needs.check-changes.outputs.external-e2e-changed == 'true' permissions: actions: read contents: read diff --git a/e2e/features/agent-v2/configure-persistence.feature b/e2e/features/agent-v2/configure-persistence.feature index ca17f8ee67c76d..ab8bf34340e3f4 100644 --- a/e2e/features/agent-v2/configure-persistence.feature +++ b/e2e/features/agent-v2/configure-persistence.feature @@ -3,6 +3,7 @@ Feature: Agent v2 configure persistence @prepared @configure-persistence Scenario: A new Agent shows the workspace default model in Configure Given I am signed in as the default E2E admin + And the workspace default chat model is configured And an Agent v2 test agent has been created via API When I open the Agent v2 configure page Then I should see the workspace default chat model in the Agent v2 model selector diff --git a/e2e/features/agent-v2/support/seed.ts b/e2e/features/agent-v2/support/seed.ts index 25a9be453ba7e3..b5b50f32d8b510 100644 --- a/e2e/features/agent-v2/support/seed.ts +++ b/e2e/features/agent-v2/support/seed.ts @@ -4,9 +4,10 @@ import type { AgentSoulDifyToolConfig, } from '@dify/contracts/api/console/agent/types.gen' import type { KnowledgeConfig } from '@dify/contracts/api/console/datasets/types.gen' -import type { ModelType } from '@dify/contracts/api/console/workspaces/types.gen' +import type { ModelType, ParserPostDefault } from '@dify/contracts/api/console/workspaces/types.gen' import type { SeedContext, SeedResource, SeedTask } from '../../../support/seed.ts' import { readFile } from 'node:fs/promises' +import { zModelType } from '@dify/contracts/api/console/workspaces/zod.gen' import { createTestApp } from '../../../support/api/apps.ts' import { bootstrapMarketplacePlugins } from '../../../support/marketplace-plugins.ts' import { sleep } from '../../../support/process.ts' @@ -305,67 +306,87 @@ const seedAgentDecisionModel = async (context: SeedContext) => title: agentBuilderPreseededResources.agentDecisionChatModel, }) -const seedSpeechToTextModel = async (context: SeedContext) => { - const config = speechToTextModelConfig() - const title = agentBuilderPreseededResources.speechToTextModel - const modelResult = await seedModel(context, { config, title }) - if (modelResult.status === 'blocked' || modelResult.status === 'skipped') return modelResult +const seedSpeechToTextModel = async (context: SeedContext) => + seedModel(context, { + config: speechToTextModelConfig(), + title: agentBuilderPreseededResources.speechToTextModel, + }) - const model = await findModel(context.consoleClient, config) - if (!model || model.status !== activeModelStatus) - return blocked(title, `${config.provider}/${config.name} is not active after model setup.`) +const readWorkspaceDefaultModels = async (context: SeedContext) => + new Map( + await Promise.all( + zModelType.options.map(async (modelType) => { + const response = await context.consoleClient.workspaces.current.defaultModel.get({ + query: { model_type: modelType }, + }) + return [modelType, response.data] as const + }), + ), + ) - const resource = { - id: `${model.provider}/${model.name}`, - kind: 'model', - name: title, - } - const defaultModelResponse = await context.consoleClient.workspaces.current.defaultModel.get({ - query: { model_type: config.type }, - }) - const defaultModel = defaultModelResponse.data - const isExpectedDefault = - defaultModel?.model === model.name && - matchesProvider(defaultModel.provider.provider, model.provider) +const seedWorkspaceDefaultModels = (includeSpeechToText: boolean): SeedTask => ({ + id: 'workspace-default-models', + title: 'Workspace default models', + run: async (context) => { + const title = 'Workspace default models' + const chatModel = stableModelConfig() + if (chatModel.type !== 'llm') + return blocked( + title, + 'E2E_STABLE_MODEL_TYPE must be llm for the workspace default chat model.', + ) - if (isExpectedDefault) - return modelResult.status === 'updated' ? modelResult : verified(title, resource) + const configs = includeSpeechToText ? [chatModel, speechToTextModelConfig()] : [chatModel] + const defaults = await readWorkspaceDefaultModels(context) + const settings = new Map() + for (const [modelType, model] of defaults) { + if (model) + settings.set(modelType, { + model: model.model, + model_type: modelType, + provider: model.provider.provider, + }) + } - if (context.dryRun) - return skipped( - title, - `Would set ${model.provider}/${model.name} as the workspace default Speech-to-Text model.`, - ) + let needsUpdate = false + for (const config of configs) { + const model = await findModel(context.consoleClient, config) + if (!model || model.status !== activeModelStatus) { + const reason = `${config.provider}/${config.name} is not active after model setup.` + return context.dryRun ? skipped(title, reason) : blocked(title, reason) + } + + const current = settings.get(config.type) + if (current?.model !== model.name || current.provider !== model.provider) needsUpdate = true + settings.set(config.type, { + model: model.name, + model_type: config.type, + provider: model.provider, + }) + } - await context.consoleClient.workspaces.current.defaultModel.post({ - body: { - model_settings: [ - { - model: model.name, - model_type: config.type, - provider: model.provider, - }, - ], - }, - }) + if (!needsUpdate) return verified(title) + if (context.dryRun) + return skipped(title, 'Would configure the seeded workspace default models.') - const updatedDefaultModelResponse = - await context.consoleClient.workspaces.current.defaultModel.get({ - query: { model_type: config.type }, + // Saving defaults replaces every type, so retain defaults outside this seed profile. + await context.consoleClient.workspaces.current.defaultModel.post({ + body: { model_settings: [...settings.values()] }, }) - const updatedDefaultModel = updatedDefaultModelResponse.data - if ( - updatedDefaultModel?.model !== model.name || - !matchesProvider(updatedDefaultModel.provider.provider, model.provider) - ) { - return blocked( - title, - `${model.provider}/${model.name} was not selected as the workspace default Speech-to-Text model.`, - ) - } - return updated(title, resource) -} + const savedDefaults = await readWorkspaceDefaultModels(context) + for (const [modelType, setting] of settings) { + const saved = savedDefaults.get(modelType) + if (saved?.model !== setting.model || saved?.provider.provider !== setting.provider) + return blocked( + title, + `Workspace default ${modelType} was not saved as ${setting.provider}/${setting.model}.`, + ) + } + + return updated(title) + }, +}) const findBuiltinTool = async (client: SeedContext['consoleClient'], displayName: string) => { const parsed = splitToolDisplayName(displayName) @@ -849,6 +870,7 @@ const agentV2PreparedFixtureSeedTasks = (): SeedTask[] => [ const agentV2PreparedSeedTasks = (): SeedTask[] => [ ...agentV2BaseSeedTasks(), + seedWorkspaceDefaultModels(false), ...agentV2PreparedFixtureSeedTasks(), ] @@ -859,6 +881,7 @@ const agentV2ExternalRuntimeSeedTasks = (): SeedTask[] => [ title: agentBuilderPreseededResources.speechToTextModel, run: seedSpeechToTextModel, }, + seedWorkspaceDefaultModels(true), ] const agentV2PostMergeSeedTasks = (): SeedTask[] => [ diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index bce268be4f0b63..df8bb7a3b8f2df 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -299,12 +299,11 @@ Then( Then( 'I should see the workspace default chat model in the Agent v2 model selector', async function (this: DifyWorld) { - const response = await this.getConsoleClient().workspaces.current.defaultModel.get({ - query: { model_type: 'llm' }, - }) - if (!response.data) throw new Error('The workspace default chat model is not configured.') + const defaultModel = this.agentBuilder.fixtures.defaultChatModel + if (!defaultModel) + throw new Error('Resolve the workspace default chat model before creating the Agent.') - await expect(getAgentModelSelector(this.getPage())).toContainText(response.data.model, { + await expect(getAgentModelSelector(this.getPage())).toContainText(defaultModel.model, { timeout: 30_000, }) }, diff --git a/e2e/features/step-definitions/agent-v2/fixtures.steps.ts b/e2e/features/step-definitions/agent-v2/fixtures.steps.ts index 56a1b5d3b7b673..026f68e5bd0675 100644 --- a/e2e/features/step-definitions/agent-v2/fixtures.steps.ts +++ b/e2e/features/step-definitions/agent-v2/fixtures.steps.ts @@ -23,6 +23,18 @@ Given('the Agent Builder stable chat model is available', async function (this: this.agentBuilder.fixtures.stableModel = stableModel }) +Given('the workspace default chat model is configured', async function (this: DifyWorld) { + const response = await this.getConsoleClient().workspaces.current.defaultModel.get({ + query: { model_type: 'llm' }, + }) + if (!response.data) + throw new Error( + 'The workspace default chat model is not configured. Run the prepared seed first.', + ) + + this.agentBuilder.fixtures.defaultChatModel = response.data +}) + Given('the workspace default speech-to-text model is active', async function (this: DifyWorld) { const speechToTextModel = await requireAgentBuilderSpeechToTextModel( this, diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts index 664aec6883b1a5..5aebbac292b486 100644 --- a/e2e/features/support/world.ts +++ b/e2e/features/support/world.ts @@ -1,4 +1,5 @@ import type { IWorldOptions } from '@cucumber/cucumber' +import type { DefaultModelResponse } from '@dify/contracts/api/console/workspaces/types.gen' import type { APIRequestContext, Browser, BrowserContext, Download, Page } from '@playwright/test' import type { AuthSessionMetadata } from '../../fixtures/auth.ts' import type { ConsoleClient } from '../../support/api/console-client.ts' @@ -51,6 +52,7 @@ export type WorkflowPreviewFixture = { export const createAgentBuilderWorldState = () => ({ fixtures: { agentDecisionModel: undefined as AgentBuilderChatModel | undefined, + defaultChatModel: undefined as DefaultModelResponse | undefined, preseededResources: {} as Record, speechToTextModel: undefined as AgentBuilderChatModel | undefined, stableModel: undefined as AgentBuilderChatModel | undefined, diff --git a/e2e/tests/agent-v2-seed.test.ts b/e2e/tests/agent-v2-seed.test.ts index 8cadc1f9d7945f..f76c8c2e547b8a 100644 --- a/e2e/tests/agent-v2-seed.test.ts +++ b/e2e/tests/agent-v2-seed.test.ts @@ -21,13 +21,18 @@ const taskIds = (profile: string) => createAgentV2SeedTasks(profile).map((task) describe('Agent v2 seed profiles', () => { it('prepares every fixture required by prepared behavior', () => { - expect(taskIds('prepared')).toEqual([...baseTaskIds, ...preparedFixtureTaskIds]) + expect(taskIds('prepared')).toEqual([ + ...baseTaskIds, + 'workspace-default-models', + ...preparedFixtureTaskIds, + ]) }) it('prepares external runtime and prepared behavior in post-merge', () => { expect(taskIds('post-merge')).toEqual([ ...baseTaskIds, 'speech-to-text-model', + 'workspace-default-models', ...preparedFixtureTaskIds, ]) }) diff --git a/e2e/tests/workspace-default-model-seed.test.ts b/e2e/tests/workspace-default-model-seed.test.ts new file mode 100644 index 00000000000000..d602999852f78e --- /dev/null +++ b/e2e/tests/workspace-default-model-seed.test.ts @@ -0,0 +1,212 @@ +import type { + AvailableModelListResponse, + DefaultModelResponse, + ModelType, + ParserPostDefault, + ProviderModelWithStatusEntity, +} from '@dify/contracts/api/console/workspaces/types.gen' +import type { ConsoleClient } from '../support/api/console-client.ts' +import { consoleRouterContract } from '@dify/contracts/api/console/router.gen' +import { + zModelType, + zPostWorkspacesCurrentDefaultModelBody, +} from '@dify/contracts/api/console/workspaces/zod.gen' +import { createORPCClient } from '@orpc/client' +import { RequestValidationPlugin, ResponseValidationPlugin } from '@orpc/contract/plugins' +import { OpenAPILink } from '@orpc/openapi-client/fetch' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vite-plus/test' +import { createAgentV2SeedTasks } from '../features/agent-v2/support/seed.ts' + +const provider = 'langgenius/openai/openai' +const tenantId = '00000000-0000-4000-8000-000000000001' + +const defaultModel = (modelType: ModelType, model: string): DefaultModelResponse => ({ + model, + model_type: modelType, + provider: { + label: { en_US: 'OpenAI', zh_Hans: 'OpenAI' }, + provider, + supported_model_types: [modelType], + tenant_id: tenantId, + }, +}) + +const availableModel = (modelType: ModelType, model: string): ProviderModelWithStatusEntity => ({ + fetch_from: 'predefined-model', + label: { en_US: model, zh_Hans: model }, + model, + model_properties: {}, + model_type: modelType, + status: 'active', +}) + +const createSeedApi = ({ + initialDefaults = [], + models = [ + availableModel('llm', 'gpt-5-nano'), + availableModel('speech2text', 'gpt-4o-mini-transcribe'), + ], + persist = true, +}: { + initialDefaults?: DefaultModelResponse[] + models?: ProviderModelWithStatusEntity[] + persist?: boolean +} = {}) => { + const defaults = new Map(initialDefaults.map((model) => [model.model_type, model])) + const writes: ParserPostDefault[] = [] + const link = new OpenAPILink(consoleRouterContract, { + url: 'http://seed.test/console/api', + plugins: [ + new RequestValidationPlugin(consoleRouterContract), + new ResponseValidationPlugin(consoleRouterContract), + ], + fetch: async (request) => { + const url = new URL(request.url) + if (url.pathname === '/console/api/workspaces/current/default-model') { + if (request.method === 'GET') { + const modelType = zModelType.parse(url.searchParams.get('model_type')) + return Response.json({ data: defaults.get(modelType) ?? null }) + } + if (request.method === 'POST') { + const body = zPostWorkspacesCurrentDefaultModelBody.parse(await request.json()) + writes.push(body) + if (persist) { + // Match the API's replacement semantics, including deletion of omitted types. + defaults.clear() + for (const setting of body.model_settings) { + if (!setting.model || !setting.provider) continue + const model = defaultModel(setting.model_type, setting.model) + model.provider.provider = setting.provider + defaults.set(setting.model_type, model) + } + } + return Response.json({ result: 'success' }) + } + } + const modelTypesPath = '/console/api/workspaces/current/models/model-types/' + if (request.method === 'GET' && url.pathname.startsWith(modelTypesPath)) { + const modelType = zModelType.parse(url.pathname.slice(modelTypesPath.length)) + const response: AvailableModelListResponse = { + data: [ + { + label: { en_US: 'OpenAI', zh_Hans: 'OpenAI' }, + models: models.filter((model) => model.model_type === modelType), + provider, + status: 'active', + tenant_id: tenantId, + }, + ], + } + return Response.json(response) + } + throw new Error(`Unexpected seed request: ${request.method} ${url.pathname}`) + }, + }) + const consoleClient = createORPCClient(link) + const run = (profile = 'post-merge', dryRun = false) => { + const task = createAgentV2SeedTasks(profile).find( + (task) => task.id === 'workspace-default-models', + ) + if (!task) throw new Error(`No workspace default model seed in ${profile}`) + return task.run({ consoleClient, dryRun, resources: new Map() }) + } + return { defaults, run, writes } +} + +describe('workspace default model seed', () => { + beforeEach(() => { + vi.stubEnv('E2E_STABLE_MODEL_NAME', 'gpt-5-nano') + vi.stubEnv('E2E_STABLE_MODEL_PROVIDER', 'openai') + vi.stubEnv('E2E_STABLE_MODEL_TYPE', 'llm') + vi.stubEnv('E2E_SPEECH_TO_TEXT_MODEL_NAME', 'gpt-4o-mini-transcribe') + vi.stubEnv('E2E_SPEECH_TO_TEXT_MODEL_PROVIDER', 'openai') + }) + + afterEach(() => vi.unstubAllEnvs()) + + it.each(['post-merge', 'external-runtime'])( + 'saves chat and speech defaults together for %s and is idempotent', + async (profile) => { + const api = createSeedApi({ + initialDefaults: [defaultModel('text-embedding', 'text-embedding-3-small')], + }) + + expect((await api.run(profile)).status).toBe('updated') + expect(api.writes).toHaveLength(1) + expect(api.defaults.get('llm')?.model).toBe('gpt-5-nano') + expect(api.defaults.get('speech2text')?.model).toBe('gpt-4o-mini-transcribe') + expect(api.defaults.get('text-embedding')?.model).toBe('text-embedding-3-small') + expect((await api.run(profile)).status).toBe('verified') + expect(api.writes).toHaveLength(1) + }, + ) + + it('configures the prepared chat default without changing an existing speech default', async () => { + const api = createSeedApi({ + initialDefaults: [ + defaultModel('llm', 'old-chat-model'), + defaultModel('speech2text', 'whisper-1'), + ], + models: [availableModel('llm', 'gpt-5-nano')], + }) + + expect((await api.run('prepared')).status).toBe('updated') + expect(api.defaults.get('llm')?.model).toBe('gpt-5-nano') + expect(api.defaults.get('speech2text')?.model).toBe('whisper-1') + }) + + it('honors the configured stable model instead of assuming the first available model', async () => { + vi.stubEnv('E2E_STABLE_MODEL_NAME', 'gpt-5.5') + const api = createSeedApi({ + models: [availableModel('llm', 'gpt-5-nano'), availableModel('llm', 'gpt-5.5')], + }) + + expect((await api.run('prepared')).status).toBe('updated') + expect(api.defaults.get('llm')?.model).toBe('gpt-5.5') + }) + + it.each([undefined, 'no-configure'] as const)( + 'blocks missing or inactive speech models before writing any defaults (%s)', + async (status) => { + const models = [availableModel('llm', 'gpt-5-nano')] + if (status) + models.push({ ...availableModel('speech2text', 'gpt-4o-mini-transcribe'), status }) + const api = createSeedApi({ models }) + + expect((await api.run()).status).toBe('blocked') + expect(api.writes).toHaveLength(0) + }, + ) + + it('does not mutate workspace defaults during a dry run', async () => { + const api = createSeedApi() + + expect((await api.run('post-merge', true)).status).toBe('skipped') + expect(api.writes).toHaveLength(0) + expect(api.defaults.size).toBe(0) + }) + + it('fails readiness when a successful save does not persist the requested defaults', async () => { + const api = createSeedApi({ persist: false }) + + expect(await api.run()).toEqual( + expect.objectContaining({ + reason: expect.stringContaining('was not saved'), + status: 'blocked', + }), + ) + }) + + it('rejects a non-LLM stable model configuration', async () => { + vi.stubEnv('E2E_STABLE_MODEL_TYPE', 'text-embedding') + const api = createSeedApi() + + expect(await api.run('prepared')).toEqual( + expect.objectContaining({ + reason: expect.stringContaining('must be llm'), + status: 'blocked', + }), + ) + expect(api.writes).toHaveLength(0) + }) +})