Skip to content

Commit 394929a

Browse files
t3dotggclaude
t3dotgg
and
claude
authored
test(server): CI fails when agent context grows past main (#25785)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent a9b513b commit 394929a

4 files changed

Lines changed: 192 additions & 2 deletions

File tree

‎.github/workflows/agent-context-budget.yml‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
name: Agent Context Budget
2+
3+
# Fails when a PR raises a value in agentContextBudget.json above main's. AgentContextBudget.test.ts
4+
# keeps that file equal to the measured context, so the context cannot grow by editing the file.
5+
# To accept growth on purpose, a maintainer adds the agent-context-increase label.
6+
on:
7+
pull_request:
8+
types: [opened, synchronize, reopened, labeled, unlabeled]
9+
10+
permissions:
11+
contents: read
12+
13+
concurrency:
14+
group: agent-context-budget-${{ github.event.pull_request.number }}
15+
cancel-in-progress: true
16+
17+
jobs:
18+
budget:
19+
name: Agent context budget
20+
runs-on: ubuntu-24.04
21+
timeout-minutes: 5
22+
env:
23+
BUDGET: apps/server/src/provider/agentContextBudget.json
24+
ALLOW_INCREASE: ${{ contains(github.event.pull_request.labels.*.name, 'agent-context-increase') }}
25+
steps:
26+
- name: Checkout
27+
uses: actions/checkout@v6
28+
with:
29+
sparse-checkout: apps/server/src/provider/agentContextBudget.json
30+
sparse-checkout-cone-mode: false
31+
32+
- name: Compare with main
33+
run: |
34+
git fetch --quiet --depth=1 origin main
35+
if ! git cat-file -e "FETCH_HEAD:$BUDGET" 2>/dev/null; then
36+
echo "main has no agent context budget yet."
37+
exit 0
38+
fi
39+
git show "FETCH_HEAD:$BUDGET" > "$RUNNER_TEMP/main.json"
40+
grown=$(jq -r --slurpfile main "$RUNNER_TEMP/main.json" '
41+
to_entries[]
42+
| select($main[0][.key] != null and .value > $main[0][.key])
43+
| "\(.key): \($main[0][.key]) -> \(.value) (+\(.value - $main[0][.key]))"
44+
' "$BUDGET")
45+
if [ -z "$grown" ]; then
46+
echo "Agent context is not larger than main."
47+
exit 0
48+
fi
49+
printf 'Agent context grew compared to main:\n%s\n' "$grown"
50+
if [ "$ALLOW_INCREASE" = "true" ]; then
51+
echo "Accepted by the agent-context-increase label."
52+
exit 0
53+
fi
54+
echo "::error file=$BUDGET::Agent context grew compared to main. Cut the added text, or ask a maintainer to add the agent-context-increase label."
55+
exit 1

‎apps/server/src/mcp/McpHttpServer.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1055,7 +1055,8 @@ export const layerMcpTransport = McpServer.layerHttp({
10551055
allowSessionTermination: true,
10561056
}).pipe(Layer.provide(layerMcpAuthMiddleware));
10571057

1058-
export const layer = Layer.mergeAll(
1058+
/** Every tool the T3 MCP server advertises. AgentContextBudget.test.ts measures this set. */
1059+
export const layerTools = Layer.mergeAll(
10591060
layerPreviewToolkit,
10601061
layerOrchestratorToolkit,
10611062
layerThreadToolkit,
@@ -1067,4 +1068,6 @@ export const layer = Layer.mergeAll(
10671068
layerPullRequestsToolkit,
10681069
layerDeviceToolkit,
10691070
layerHtmlToolkit,
1070-
).pipe(Layer.provideMerge(layerMcpTransport));
1071+
);
1072+
1073+
export const layer = layerTools.pipe(Layer.provideMerge(layerMcpTransport));

‎apps/server/src/provider/AgentContextBudget.test.ts‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
import * as NodeServices from "@effect/platform-node/NodeServices";
2+
import { expect, it } from "@effect/vitest";
3+
import { ProviderInstanceId } from "@t3tools/contracts";
4+
import { McpProviderSessions } from "@t3tools/provider-core/server/McpProviderSessions";
5+
import * as Effect from "effect/Effect";
6+
import * as Layer from "effect/Layer";
7+
import { McpServer } from "effect/ai";
8+
9+
import { ServerSecretStore } from "../auth/ServerSecretStore.ts";
10+
import { ServerConfig } from "../config.ts";
11+
import { DeviceService } from "../device/DeviceService.ts";
12+
import { ServerEnvironment } from "../environment/ServerEnvironment.ts";
13+
import { GitWorkflowService } from "../git/GitWorkflowService.ts";
14+
import * as McpHttpServer from "../mcp/McpHttpServer.ts";
15+
import { PreviewAutomationBroker } from "../mcp/PreviewAutomationBroker.ts";
16+
import {
17+
CLAUDE_PROVIDER,
18+
makeClaudeQueryOptions,
19+
} from "../orchestration-v2/Adapters/ClaudeAdapterV2.ts";
20+
import { OrchestratorV2 } from "../orchestration-v2/Orchestrator.ts";
21+
import { ProviderAdapterRegistryV2 } from "../orchestration-v2/ProviderAdapterRegistry.ts";
22+
import { ThreadLaunchService } from "../orchestration-v2/ThreadLaunchService.ts";
23+
import { ThreadManagementService } from "../orchestration-v2/ThreadManagementService.ts";
24+
import { ThreadSearch } from "../orchestration-v2/ThreadSearch.ts";
25+
import { PreviewManager } from "../preview/Manager.ts";
26+
import { PreviewBrowser } from "../preview/PreviewBrowser.ts";
27+
import { ManagedProjectFolders } from "../project/ManagedProjectFolders.ts";
28+
import { ProjectService } from "../project/ProjectService.ts";
29+
import { ProjectSetupScriptRunner } from "../project/ProjectSetupScriptRunner.ts";
30+
import { PullRequestReviewGuideService } from "../pullRequest/PullRequestReviewGuideService.ts";
31+
import { ScheduledTaskService } from "../scheduledTasks/ScheduledTaskService.ts";
32+
import { SecretRequests } from "../secrets/SecretRequests.ts";
33+
import { ServerSettingsService } from "../serverSettings.ts";
34+
import { SourceControlRepositoryService } from "../sourceControl/SourceControlRepositoryService.ts";
35+
import { GitVcsDriver } from "../vcs/GitVcsDriver.ts";
36+
import { VcsStatusBroadcaster } from "../vcs/VcsStatusBroadcaster.ts";
37+
import { buildCodexAdditionalContext } from "./CodexDeveloperInstructions.ts";
38+
import { DocumentText } from "./DocumentText.ts";
39+
import { ProviderRegistry } from "./ProviderRegistry.ts";
40+
import budget from "./agentContextBudget.json" with { type: "json" };
41+
42+
// Registration only stores tool definitions, so the handlers' services are never called.
43+
const toolHandlerServices = Layer.mergeAll(
44+
Layer.mock(DeviceService)({}),
45+
Layer.mock(DocumentText)({}),
46+
Layer.mock(GitVcsDriver)({}),
47+
Layer.mock(GitWorkflowService)({}),
48+
Layer.mock(ManagedProjectFolders)({ namedProjectsRoot: "" }),
49+
Layer.mock(McpProviderSessions)({}),
50+
Layer.mock(OrchestratorV2)({}),
51+
Layer.mock(PreviewAutomationBroker)({}),
52+
Layer.mock(PreviewBrowser)({}),
53+
Layer.mock(PreviewManager)({}),
54+
Layer.mock(ProjectService)({}),
55+
Layer.mock(ProjectSetupScriptRunner)({}),
56+
Layer.mock(ProviderAdapterRegistryV2)({}),
57+
Layer.mock(ProviderRegistry)({}),
58+
Layer.mock(PullRequestReviewGuideService)({}),
59+
Layer.mock(ScheduledTaskService)({}),
60+
Layer.mock(SecretRequests)({}),
61+
ServerConfig.layerTest(process.cwd(), { prefix: "t3-agent-context-budget-" }),
62+
Layer.mock(ServerEnvironment)({}),
63+
Layer.mock(ServerSecretStore)({}),
64+
Layer.mock(ServerSettingsService)({}),
65+
Layer.mock(SourceControlRepositoryService)({}),
66+
Layer.mock(ThreadLaunchService)({}),
67+
Layer.mock(ThreadManagementService)({}),
68+
Layer.mock(ThreadSearch)({}),
69+
Layer.mock(VcsStatusBroadcaster)({}),
70+
).pipe(Layer.provideMerge(NodeServices.layer));
71+
72+
const utf8Bytes = (text: string) => Buffer.byteLength(text, "utf8");
73+
74+
const claudeSystemAppend = () => {
75+
const { systemPrompt } = makeClaudeQueryOptions({
76+
modelSelection: {
77+
instanceId: ProviderInstanceId.make(CLAUDE_PROVIDER),
78+
model: "claude-opus-5-5",
79+
},
80+
nativeThreadId: "context-budget",
81+
resume: false,
82+
cwd: null,
83+
mcpServers: { "t3-code": { type: "http", url: "http://127.0.0.1/mcp" } },
84+
});
85+
return typeof systemPrompt === "object" && "append" in systemPrompt ? systemPrompt.append : "";
86+
};
87+
88+
const codexAdditionalContext = () =>
89+
Object.values(
90+
buildCodexAdditionalContext(
91+
{ model: "gpt-6.1-sol", modelName: "GPT-6.1 Sol", reasoningEffort: "high" },
92+
{ browser: true, device: true },
93+
),
94+
)
95+
.map((entry) => entry.value)
96+
.join("");
97+
98+
// CI rejects a PR that raises any value above main's agentContextBudget.json, so growth
99+
// needs the agent-context-increase label. Lower the values when the text shrinks.
100+
it.effect("agent context matches the committed budget", () =>
101+
Effect.gen(function* () {
102+
const mcp = yield* McpServer.McpServer;
103+
const measured = {
104+
claudeSystemAppendBytes: utf8Bytes(claudeSystemAppend()),
105+
codexAdditionalContextBytes: utf8Bytes(codexAdditionalContext()),
106+
// The model sees these fields. outputSchema and annotations are protocol metadata.
107+
mcpToolListJsonBytes: utf8Bytes(
108+
JSON.stringify(
109+
mcp.tools.map(({ tool }) => ({
110+
name: tool.name,
111+
description: tool.description,
112+
inputSchema: tool.inputSchema,
113+
})),
114+
),
115+
),
116+
};
117+
118+
expect(measured).toEqual(budget);
119+
}).pipe(
120+
Effect.provide(
121+
McpHttpServer.layerTools.pipe(
122+
Layer.provideMerge(McpServer.McpServer.layer),
123+
Layer.provide(toolHandlerServices),
124+
),
125+
),
126+
),
127+
);

‎apps/server/src/provider/agentContextBudget.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"claudeSystemAppendBytes": 4857,
3+
"codexAdditionalContextBytes": 5409,
4+
"mcpToolListJsonBytes": 54964
5+
}

0 commit comments

Comments
 (0)