diff --git a/src/lib/actions/sandbox/rebuild-messaging-stage.test.ts b/src/lib/actions/sandbox/rebuild-messaging-stage.test.ts index b5f0649476b..118f832aea3 100644 --- a/src/lib/actions/sandbox/rebuild-messaging-stage.test.ts +++ b/src/lib/actions/sandbox/rebuild-messaging-stage.test.ts @@ -30,6 +30,21 @@ const { stageMessagingManifestPlanForRebuild } = D("actions/sandbox/rebuild.js") ) => Promise; }; +const emptyStoredMessagingPlan = { + schemaVersion: 1, + sandboxName: "openclaw-sandbox", + agent: "openclaw", + workflow: "remove-channel", + channels: [], + disabledChannels: [], + credentialBindings: [], + networkPolicy: { presets: [], entries: [] }, + agentRender: [], + buildSteps: [], + stateUpdates: [], + healthChecks: [], +}; + describe("stageMessagingManifestPlanForRebuild non-messaging agent guard", () => { afterEach(() => { vi.restoreAllMocks(); @@ -61,6 +76,38 @@ describe("stageMessagingManifestPlanForRebuild non-messaging agent guard", () => expect(result).toBeNull(); }); + it("stages an explicit empty rebuild plan so token-backed channels are not rediscovered", async () => { + vi.spyOn(defs, "loadAgent").mockReturnValue({ + name: "openclaw", + messagingPlatforms: ["telegram"], + }); + const clearPlanEnvSpy = vi.spyOn(messaging.MessagingSetupApplier, "clearPlanEnv"); + const writePlanEnvSpy = vi + .spyOn(messaging.MessagingSetupApplier, "writePlanToEnv") + .mockImplementation(() => undefined); + + const messages: string[] = []; + const result = await stageMessagingManifestPlanForRebuild( + "openclaw-sandbox", + { + name: "openclaw-sandbox", + messaging: { schemaVersion: 1, plan: emptyStoredMessagingPlan }, + }, + "openclaw", + (msg) => messages.push(msg), + ); + + expect(clearPlanEnvSpy).not.toHaveBeenCalled(); + expect(writePlanEnvSpy).toHaveBeenCalledWith( + expect.objectContaining({ + workflow: "rebuild", + channels: [], + }), + ); + expect(messages).toContain("Messaging manifest rebuild plan staged: no configured channels"); + expect(result).toMatchObject({ workflow: "rebuild", channels: [] }); + }); + it("emits the empty-allowlist skip message for a known agent whose messagingPlatforms is an explicit empty allowlist", async () => { vi.spyOn(defs, "loadAgent").mockReturnValue({ name: "openclaw", diff --git a/src/lib/messaging/compiler/workflow-planner.test.ts b/src/lib/messaging/compiler/workflow-planner.test.ts index 06271511528..6d18f666c6b 100644 --- a/src/lib/messaging/compiler/workflow-planner.test.ts +++ b/src/lib/messaging/compiler/workflow-planner.test.ts @@ -8,7 +8,7 @@ import { createBuiltInRenderTemplateResolver, } from "../channels"; import { createBuiltInMessagingHookRegistry, MessagingHookRegistry } from "../hooks"; -import { createChannelManifestRegistry, type ChannelManifest } from "../manifest"; +import { type ChannelManifest, createChannelManifestRegistry } from "../manifest"; import { MessagingWorkflowPlanner } from "./workflow-planner"; const TEST_CREDENTIALS: Readonly> = { @@ -785,6 +785,47 @@ describe("MessagingWorkflowPlanner", () => { expect(removed?.agentRender.some((entry) => entry.channelId === "telegram")).toBe(false); }); + it("preserves an explicit empty plan on rebuild after the final channel is removed", async () => { + const existingPlan = await planner().buildPlan({ + sandboxName: "demo", + agent: "openclaw", + workflow: "onboard", + isInteractive: false, + configuredChannels: ["telegram"], + credentialAvailability: { TELEGRAM_BOT_TOKEN: true }, + }); + + const removed = await planner().buildChannelRemovePlanFromSandboxEntry({ + sandboxName: "demo", + agent: "openclaw", + sandboxEntry: { + name: "demo", + messaging: { schemaVersion: 1, plan: existingPlan }, + }, + channelId: "telegram", + }); + + expect(removed?.channels).toEqual([]); + + const rebuilt = await planner().buildRebuildPlanFromSandboxEntry({ + sandboxName: "demo", + agent: "openclaw", + sandboxEntry: { + name: "demo", + messaging: { schemaVersion: 1, plan: removed! }, + }, + supportedChannelIds: ["telegram"], + }); + + expect(rebuilt).toMatchObject({ + workflow: "rebuild", + channels: [], + disabledChannels: [], + credentialBindings: [], + networkPolicy: { presets: [], entries: [] }, + }); + }); + it("rebuilds from stored plan input values when config env is unavailable", async () => { const existingPlan = await withEnv( { diff --git a/src/lib/messaging/compiler/workflow-planner.ts b/src/lib/messaging/compiler/workflow-planner.ts index 768f2cfcc22..144e206519f 100644 --- a/src/lib/messaging/compiler/workflow-planner.ts +++ b/src/lib/messaging/compiler/workflow-planner.ts @@ -2,8 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 import { MessagingHookRegistry } from "../hooks"; -import { hydrateDerivedSandboxMessagingPlanFields } from "../persistence"; -import { parseSandboxMessagingPlan } from "../plan-validation"; import type { ChannelManifestRegistry, MessagingAgentId, @@ -13,6 +11,8 @@ import type { SandboxMessagingPlan, SandboxMessagingRuntimeSetupPlan, } from "../manifest"; +import { hydrateDerivedSandboxMessagingPlanFields } from "../persistence"; +import { parseSandboxMessagingPlan } from "../plan-validation"; import { planHostForward } from "./engines/host-forward-engine"; import { planRuntimeSetup } from "./engines/runtime-setup-engine"; import type { RenderTemplateReferenceResolver } from "./engines/template"; @@ -120,7 +120,7 @@ export class MessagingWorkflowPlanner { if (!existingPlan) return null; const filteredPlan = this.filterPlanChannelsToSupportedAllowlist(existingPlan, context); - if (!filteredPlan || filteredPlan.channels.length === 0) return null; + if (existingPlan.channels.length > 0 && filteredPlan.channels.length === 0) return null; return refreshDerivedPlanFields( setPlanDisabledChannels( @@ -136,7 +136,7 @@ export class MessagingWorkflowPlanner { private filterPlanChannelsToSupportedAllowlist( plan: SandboxMessagingPlan, context: Pick, - ): SandboxMessagingPlan | null { + ): SandboxMessagingPlan { if (!Array.isArray(context.supportedChannelIds)) return plan; const allowlist = new Set(context.supportedChannelIds); let filtered = plan; diff --git a/test/e2e-scenario/live/agent-turn-latency.test.ts b/test/e2e-scenario/live/agent-turn-latency.test.ts index 53e30cbdc36..d7ab7bb1d1f 100644 --- a/test/e2e-scenario/live/agent-turn-latency.test.ts +++ b/test/e2e-scenario/live/agent-turn-latency.test.ts @@ -5,6 +5,7 @@ import fs from "node:fs"; +import { containsInteger42Answer } from "../../helpers/e2e-answer-assertions.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText } from "../fixtures/clients/index.ts"; import { trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; @@ -88,9 +89,10 @@ test.skipIf(!shouldRunLiveE2EScenarios())( const openclaw = await openclawTurn(sandbox, apiKey); expect(openclaw.result.exitCode, resultText(openclaw.result)).toBe(0); assertNoOpenClawTransportErrors(resultText(openclaw.result)); - expect(extractOpenClawAgentText(openclaw.result.stdout), resultText(openclaw.result)).toMatch( - /(^|[^0-9])42([^0-9]|$)/, - ); + expect( + containsInteger42Answer(extractOpenClawAgentText(openclaw.result.stdout)), + resultText(openclaw.result), + ).toBe(true); expect(openclaw.elapsedMs).toBeLessThanOrEqual(MAX_TURN_SECONDS * 1000); results.openclaw = { elapsedMs: openclaw.elapsedMs }; @@ -152,7 +154,9 @@ test.skipIf(!shouldRunLiveE2EScenarios())( expect(hermesTurn.exitCode, resultText(hermesTurn)).toBe(0); const hermesResponse = responseBodyAndStatus(hermesTurn.stdout); expect(hermesResponse.status, resultText(hermesTurn)).toBe("200"); - expect(chatContent(hermesResponse.body)).toMatch(/(^|[^0-9])42([^0-9]|$)/); + expect(containsInteger42Answer(chatContent(hermesResponse.body)), resultText(hermesTurn)).toBe( + true, + ); expect(hermesMs).toBeLessThanOrEqual(MAX_TURN_SECONDS * 1000); results.hermes = { elapsedMs: hermesMs }; await artifacts.writeJson("turn-latency-results.json", results); diff --git a/test/e2e-scenario/live/full-e2e.test.ts b/test/e2e-scenario/live/full-e2e.test.ts index 6eefd0d818d..778124d2d2e 100644 --- a/test/e2e-scenario/live/full-e2e.test.ts +++ b/test/e2e-scenario/live/full-e2e.test.ts @@ -7,6 +7,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { containsInteger42Answer } from "../../helpers/e2e-answer-assertions.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { type HostCliClient } from "../fixtures/clients/host.ts"; import { type SandboxClient, validateSandboxName } from "../fixtures/clients/sandbox.ts"; @@ -204,7 +205,9 @@ liveTest( }, ); expect(sandboxInference.exitCode, resultText(sandboxInference)).toBe(0); - expect(sandboxInference.stdout).toMatch(/(^|[^0-9])42([^0-9]|$)/); + expect(containsInteger42Answer(sandboxInference.stdout), resultText(sandboxInference)).toBe( + true, + ); const logs = await repoNemoclaw( host, diff --git a/test/e2e-scenario/live/gpu-double-onboard.test.ts b/test/e2e-scenario/live/gpu-double-onboard.test.ts index e6091ddfc85..ace78c312dd 100644 --- a/test/e2e-scenario/live/gpu-double-onboard.test.ts +++ b/test/e2e-scenario/live/gpu-double-onboard.test.ts @@ -7,6 +7,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { containsInteger42Answer } from "../../helpers/e2e-answer-assertions.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { type HostCliClient } from "../fixtures/clients/host.ts"; import { type SandboxClient, validateSandboxName } from "../fixtures/clients/sandbox.ts"; @@ -145,7 +146,7 @@ async function expectSandboxInference42( }, ); expect(response.exitCode, resultText(response)).toBe(0); - expect(response.stdout).toMatch(/(^|[^0-9])42([^0-9]|$)/); + expect(containsInteger42Answer(response.stdout), resultText(response)).toBe(true); } liveTest( diff --git a/test/e2e-scenario/live/issue-4462-scope-upgrade-approval.test.ts b/test/e2e-scenario/live/issue-4462-scope-upgrade-approval.test.ts index 691017a74dc..7e40824968f 100644 --- a/test/e2e-scenario/live/issue-4462-scope-upgrade-approval.test.ts +++ b/test/e2e-scenario/live/issue-4462-scope-upgrade-approval.test.ts @@ -119,6 +119,13 @@ raise SystemExit(1) PY } +contains_integer_42() { + local raw compact + raw="$(cat)" + compact="$(printf '%s' "$raw" | tr -d '[:space:]')" + grep -Eq '(^|[^0-9])42([^0-9]|$)' <<<"$compact" +} + assert_agent_scopes_without_admin() { python3 - <<'PY' import json, sys @@ -159,7 +166,7 @@ if [ -z "$request_id" ]; then if printf '%s' "$state" | assert_agent_scopes_without_admin >/tmp/issue4462-approved-device.txt 2>/tmp/issue4462-approved-device.err; then echo "SCOPE_ALREADY_APPROVED=$(cat /tmp/issue4462-approved-device.txt)" elif [ "$trigger_rc" -eq 0 ] && ! grep -Eiq 'EMBEDDED FALLBACK|scope upgrade pending approval|pairing required|fallbackFrom[": ]+gateway|transport[": ]+embedded' /tmp/issue4462-trigger-agent.log \ - && grep -Eq '(^|[^0-9])42([^0-9]|$)' /tmp/issue4462-trigger-agent.log; then + && contains_integer_42 &2 exit 7 fi -if ! grep -Eq '(^|[^0-9])42([^0-9]|$)' /tmp/issue4462-final-agent.log; then +if ! contains_integer_42 &2 cat /tmp/issue4462-final-agent.log >&2 exit 8 diff --git a/test/e2e-scenario/live/launchable-smoke.test.ts b/test/e2e-scenario/live/launchable-smoke.test.ts index b9660950941..e81a30d36ca 100644 --- a/test/e2e-scenario/live/launchable-smoke.test.ts +++ b/test/e2e-scenario/live/launchable-smoke.test.ts @@ -6,8 +6,9 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { containsInteger42Answer } from "../../helpers/e2e-answer-assertions.ts"; import type { ArtifactSink } from "../fixtures/artifacts.ts"; +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; @@ -463,7 +464,7 @@ runLaunchableSmokeTest( ).toBe(0); const agentReply = parseAgentText(agent.stdout); expect( - /(^|[^0-9])42([^0-9]|$)/.test(agentReply), + containsInteger42Answer(agentReply), `expected agent reply to contain 42; rc=${agent.exitCode}; reply='${agentReply.slice(0, 200)}'; stdout='${agent.stdout.slice(0, 300)}'; stderr='${agent.stderr.slice(0, 300)}'`, ).toBe(true); diff --git a/test/e2e-scenario/live/openclaw-tui-chat-correlation.test.ts b/test/e2e-scenario/live/openclaw-tui-chat-correlation.test.ts index 9759f12c261..26034af662a 100644 --- a/test/e2e-scenario/live/openclaw-tui-chat-correlation.test.ts +++ b/test/e2e-scenario/live/openclaw-tui-chat-correlation.test.ts @@ -19,6 +19,7 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { containsReplyTokenAllowingWhitespace } from "../../helpers/e2e-answer-assertions.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { type SandboxClient, trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; @@ -160,7 +161,7 @@ function analyzeIssue2603Trace({ const finalReplyCounts = new Map(); for (const [replyToken, expectedRunId] of expectedRunByReplyToken) { for (const event of chatEvents) { - if (!event.text.includes(replyToken)) continue; + if (!containsReplyTokenAllowingWhitespace(event.text, replyToken)) continue; visibleReplyCounts.set(replyToken, (visibleReplyCounts.get(replyToken) ?? 0) + 1); if (event.state === "final") { finalReplyCounts.set(replyToken, (finalReplyCounts.get(replyToken) ?? 0) + 1); @@ -188,7 +189,7 @@ function analyzeIssue2603Trace({ .filter((event) => event.state === "final") .flatMap((event) => sentRuns - .filter((entry) => event.text.includes(entry.replyToken)) + .filter((entry) => containsReplyTokenAllowingWhitespace(event.text, entry.replyToken)) .map((entry) => entry.replyToken), ); @@ -288,8 +289,12 @@ function textFromMessage(message) { return content.map((part) => part && typeof part === "object" && typeof part.text === "string" ? part.text : "").filter(Boolean).join("\n"); } +function compactReplyTokenText(value) { + return String(value || "").replace(/\s+/g, ""); +} + function sawAllReplies(replyTokens) { - return replyTokens.every((token) => events.some((event) => event.event === "chat" && textFromMessage(event.payload?.message).includes(token))); + return replyTokens.every((token) => events.some((event) => event.event === "chat" && compactReplyTokenText(textFromMessage(event.payload?.message)).includes(compactReplyTokenText(token)))); } ws.on("message", (data) => { diff --git a/test/e2e-scenario/live/sandbox-operations.test.ts b/test/e2e-scenario/live/sandbox-operations.test.ts index acaf07de8f5..13a7e1edd5e 100644 --- a/test/e2e-scenario/live/sandbox-operations.test.ts +++ b/test/e2e-scenario/live/sandbox-operations.test.ts @@ -13,6 +13,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { containsInteger42Answer } from "../../helpers/e2e-answer-assertions.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { type SandboxClient, trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; @@ -266,7 +267,7 @@ async function assertAgentCanAnswer(host: HostCliClient, sandboxName: string): P ); const reply = parseOpenClawAgentText(result.stdout); expectExitZero(result, "openclaw agent --json"); - expect(reply, resultText(result)).toMatch(/(^|[^0-9])42([^0-9]|$)/); + expect(containsInteger42Answer(reply), resultText(result)).toBe(true); } async function assertStatusFields(host: HostCliClient, sandboxName: string): Promise { diff --git a/test/e2e/lib/openclaw-json.sh b/test/e2e/lib/openclaw-json.sh index 8f17bab69fa..74bfd814b6a 100755 --- a/test/e2e/lib/openclaw-json.sh +++ b/test/e2e/lib/openclaw-json.sh @@ -8,6 +8,12 @@ # exact envelope shape. This also tolerates wrapper output before the JSON blob # but intentionally ignores metadata fields so IDs, durations, session names, # and model/provider details cannot satisfy reply assertions. +e2e_text_contains_integer_42() { + local compact + compact="$(printf '%s' "${1:-}" | tr -d '[:space:]')" + grep -qE '(^|[^0-9])42([^0-9]|$)' <<<"$compact" +} + parse_openclaw_agent_text() { python3 -c ' import json diff --git a/test/e2e/test-agent-turn-latency-e2e.sh b/test/e2e/test-agent-turn-latency-e2e.sh index 694317f7a1d..306a7830adf 100755 --- a/test/e2e/test-agent-turn-latency-e2e.sh +++ b/test/e2e/test-agent-turn-latency-e2e.sh @@ -412,7 +412,7 @@ run_openclaw_turn() { return fi - if grep -qE '(^|[^0-9])42([^0-9]|$)' <<<"$reply"; then + if e2e_text_contains_integer_42 "$reply"; then pass "OpenClaw: real agent turn returned 42 in $(duration_s "$OPENCLAW_TURN_MS")" assert_latency_under_cap "OpenClaw" "$OPENCLAW_TURN_MS" else @@ -463,7 +463,7 @@ print(json.dumps({ return fi - if grep -qE '(^|[^0-9])42([^0-9]|$)' <<<"$content"; then + if e2e_text_contains_integer_42 "$content"; then pass "Hermes: real daemon turn returned 42 in $(duration_s "$HERMES_TURN_MS")" assert_latency_under_cap "Hermes" "$HERMES_TURN_MS" else diff --git a/test/e2e/test-full-e2e.sh b/test/e2e/test-full-e2e.sh index 6d144f13ae3..9a2aa63953a 100755 --- a/test/e2e/test-full-e2e.sh +++ b/test/e2e/test-full-e2e.sh @@ -447,7 +447,7 @@ rm -f "$ssh_config" agent_reply=$(printf '%s' "$agent_response" | parse_openclaw_agent_text 2>/dev/null) || true -if grep -qE "(^|[^0-9])42([^0-9]|$)" <<<"$agent_reply"; then +if e2e_text_contains_integer_42 "$agent_reply"; then pass "[LIVE] openclaw agent: model answered 6×7=42 through openclaw → inference.local" else fail "[LIVE] openclaw agent: expected '42' in agent reply, got: ${agent_reply:0:200}" diff --git a/test/e2e/test-issue-4462-scope-upgrade-approval.sh b/test/e2e/test-issue-4462-scope-upgrade-approval.sh index a1fc4f34416..287e1b7eb1a 100755 --- a/test/e2e/test-issue-4462-scope-upgrade-approval.sh +++ b/test/e2e/test-issue-4462-scope-upgrade-approval.sh @@ -1074,7 +1074,7 @@ openclaw agent --agent main --json --session-id "$session_id" \ last_agent_detail="agent exited ${final_rc}: ${final_output:0:500}" elif ! grep -q '^__URL_FOR_FINAL_AGENT__=ws://' <<<"$final_output"; then last_agent_detail="agent command did not preserve OPENCLAW_GATEWAY_URL: ${final_output:0:500}" - elif grep -qE '(^|[^0-9])42([^0-9]|$)' <<<"$reply"; then + elif e2e_text_contains_integer_42 "$reply"; then agent_ok=1 pass "approved openclaw agent turn answered through gateway mode" break diff --git a/test/e2e/test-launchable-smoke.sh b/test/e2e/test-launchable-smoke.sh index 73d993f227d..dcddf40b73e 100755 --- a/test/e2e/test-launchable-smoke.sh +++ b/test/e2e/test-launchable-smoke.sh @@ -539,7 +539,7 @@ rm -f "$ssh_config" "$agent_stderr_file" agent_reply=$(printf '%s' "$agent_response" | parse_openclaw_agent_text 2>/dev/null) || true -if grep -qE "(^|[^0-9])42([^0-9]|$)" <<<"$agent_reply"; then +if e2e_text_contains_integer_42 "$agent_reply"; then pass "[LIVE] openclaw agent: model answered 6×7=42 through openclaw → inference.local" else fail "[LIVE] openclaw agent: expected '42' in agent reply; rc=${agent_rc}; reply='${agent_reply:0:200}'; stdout='${agent_response:0:300}'; stderr='${agent_stderr:0:300}'" diff --git a/test/e2e/test-sandbox-operations.sh b/test/e2e/test-sandbox-operations.sh index 98d1b015379..5be9c99d277 100755 --- a/test/e2e/test-sandbox-operations.sh +++ b/test/e2e/test-sandbox-operations.sh @@ -387,7 +387,7 @@ test_sbx_02_connect_chat() { local reply reply=$(printf '%s' "$raw" | parse_openclaw_agent_text 2>/dev/null) || true - if [[ $rc -eq 0 && -n "$reply" ]] && echo "$reply" | grep -qE "(^|[^0-9])42([^0-9]|$)"; then + if [[ $rc -eq 0 && -n "$reply" ]] && e2e_text_contains_integer_42 "$reply"; then pass "TC-SBX-02: Agent computed 6×7=42 through openclaw → inference.local" else fail "TC-SBX-02: Connect & Chat" "Expected '42' in agent reply (rc=$rc); reply='${reply:0:200}'; raw output='${raw:0:200}'" diff --git a/test/helpers/e2e-answer-assertions.test.ts b/test/helpers/e2e-answer-assertions.test.ts new file mode 100644 index 00000000000..4e20cb02080 --- /dev/null +++ b/test/helpers/e2e-answer-assertions.test.ts @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + compactAnswerText, + containsInteger42Answer, + containsReplyTokenAllowingWhitespace, +} from "./e2e-answer-assertions.ts"; + +describe("E2E answer assertions", () => { + it("normalizes harmless model-inserted whitespace", () => { + expect(compactAnswerText("4\n2")).toBe("42"); + expect(containsInteger42Answer("4\n2")).toBe(true); + expect(containsInteger42Answer("The answer is 4\n2.")).toBe(true); + }); + + it("does not match unrelated integers after whitespace normalization", () => { + expect(containsInteger42Answer("142")).toBe(false); + expect(containsInteger42Answer("420")).toBe(false); + }); + + it("matches deterministic reply tokens split by streaming whitespace", () => { + expect(containsReplyTokenAllowingWhitespace("A\n2603-REPLY", "A2603-REPLY")).toBe(true); + expect(containsReplyTokenAllowingWhitespace("B 2603-REPLY", "B2603-REPLY")).toBe(true); + }); +}); diff --git a/test/helpers/e2e-answer-assertions.ts b/test/helpers/e2e-answer-assertions.ts new file mode 100644 index 00000000000..b72176d52b5 --- /dev/null +++ b/test/helpers/e2e-answer-assertions.ts @@ -0,0 +1,14 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export function compactAnswerText(text: string): string { + return text.replace(/\s+/g, ""); +} + +export function containsInteger42Answer(text: string): boolean { + return /(^|[^0-9])42([^0-9]|$)/.test(compactAnswerText(text)); +} + +export function containsReplyTokenAllowingWhitespace(text: string, replyToken: string): boolean { + return compactAnswerText(text).includes(compactAnswerText(replyToken)); +} diff --git a/test/openclaw-tui-chat-correlation.test.ts b/test/openclaw-tui-chat-correlation.test.ts index dc006296886..aca870d36d1 100644 --- a/test/openclaw-tui-chat-correlation.test.ts +++ b/test/openclaw-tui-chat-correlation.test.ts @@ -1,14 +1,16 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { execFileSync } from "node:child_process"; import type { ExecFileSyncOptionsWithStringEncoding } from "node:child_process"; +import { execFileSync } from "node:child_process"; import { randomUUID } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; +import { containsReplyTokenAllowingWhitespace } from "./helpers/e2e-answer-assertions.ts"; + const LIVE_REPRO_ENV = "NEMOCLAW_ISSUE_2603_LIVE"; const LIVE_SANDBOX_ENV = "NEMOCLAW_ISSUE_2603_SANDBOX"; const LIVE_SCRIPT_NAME = "openclaw-issue2603-chat-correlation.cjs"; @@ -150,7 +152,7 @@ function analyzeIssue2603Trace({ const finalReplyCounts = new Map(); for (const [replyToken, expectedRunId] of expectedRunByReplyToken) { for (const event of chatEvents) { - if (!event.text.includes(replyToken)) continue; + if (!containsReplyTokenAllowingWhitespace(event.text, replyToken)) continue; visibleReplyCounts.set(replyToken, (visibleReplyCounts.get(replyToken) ?? 0) + 1); if (event.state === "final") { finalReplyCounts.set(replyToken, (finalReplyCounts.get(replyToken) ?? 0) + 1); @@ -423,8 +425,12 @@ function textFromMessage(message) { return content.map((part) => part && typeof part === "object" && typeof part.text === "string" ? part.text : "").filter(Boolean).join("\n"); } +function compactReplyTokenText(value) { + return String(value || "").replace(/\s+/g, ""); +} + function sawAllReplies(replyTokens) { - return replyTokens.every((token) => events.some((event) => event.event === "chat" && textFromMessage(event.payload?.message).includes(token))); + return replyTokens.every((token) => events.some((event) => event.event === "chat" && compactReplyTokenText(textFromMessage(event.payload?.message)).includes(compactReplyTokenText(token)))); } ws.on("message", (data) => { @@ -611,6 +617,38 @@ describe("OpenClaw TUI chat correlation regression (#2603)", () => { ]); }); + it("matches deterministic reply tokens split by harmless model whitespace", () => { + const analysis = analyzeIssue2603Trace({ + sentRuns: [ + { + promptToken: "A2603", + replyToken: "A2603-REPLY", + runId: "split-reply-run", + message: "A2603: Reply exactly A2603-REPLY and nothing else.", + }, + ], + events: [ + { + event: "chat", + payload: { + runId: "split-reply-run", + state: "final", + message: { role: "assistant", content: [{ type: "text", text: "A\n2603-REPLY" }] }, + }, + }, + ], + historyMessages: [ + { + role: "user", + content: [{ type: "text", text: "A2603: Reply exactly A2603-REPLY and nothing else." }], + }, + ], + }); + + expect(analysis.missingReplies).toEqual([]); + expect(analysis.uncorrelatedReplies).toEqual([]); + }); + it("does not classify a streamed delta plus final for the same run as a duplicate reply", () => { const analysis = analyzeIssue2603Trace({ sentRuns: [