Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 76 additions & 5 deletions api/app/clients/specs/BaseClient.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -2661,14 +2661,30 @@ describe('BaseClient', () => {
metadata: { destinationChosen: false },
};

/** A tool serves a file only once it holds it, so a record left to the sandbox needs the
* reference provisioning writes for that tool to count as its reader. */
const sandboxCsv = {
...routedCsv,
metadata: {
destinationChosen: false,
codeEnvRef: {
kind: 'user',
id: 'user-1',
storage_session_id: 'session-1',
file_id: 'sandbox-csv-file',
},
},
};

const replayCsv = async (
fileConsumers,
endpointConfig = {
defaultLLMDeliveryPath: { overrides: { 'text/csv': 'none' } },
textFallbackWithoutTools: true,
},
file = routedCsv,
) => {
getFiles.mockResolvedValueOnce([routedCsv]);
getFiles.mockResolvedValueOnce([file]);
TestClient.options.req.config = {
fileConfig: { endpoints: { [EModelEndpoint.openAI]: endpointConfig } },
};
Expand Down Expand Up @@ -2743,13 +2759,30 @@ describe('BaseClient', () => {
]);
});

test('keeps the file off the prompt when this turn can read it with code', async () => {
const message = await replayCsv({ executeCode: true, fileSearch: false });
test('keeps the file off the prompt when the sandbox running it holds the file', async () => {
const message = await replayCsv(
{ executeCode: true, fileSearch: false },
undefined,
sandboxCsv,
);

expect(TestClient.assertHistoricalAttachmentLimits).toHaveBeenCalledWith([]);
expect(TestClient.addFileContextToMessage).not.toHaveBeenCalled();
expect(message.fileContext).toBeUndefined();
});

test('replays the stored text when file search never received the file', async () => {
/* An earlier turn's upload that named no destination was filed under no tool, so the
* vector store holds nothing to search. Withholding the text for the enabled tool left
* the file unreadable on every later turn as well as the one it arrived on. */
const message = await replayCsv({ executeCode: false, fileSearch: true });
const replayed = { ...routedCsv, llmDeliveryPath: 'text' };

expect(TestClient.assertHistoricalAttachmentLimits).toHaveBeenCalledWith([replayed]);
expect(TestClient.addFileContextToMessage).toHaveBeenCalledWith(message, [replayed]);
expect(message.fileContext).toBe('region,total');
expect(routedCsv.llmDeliveryPath).toBe('none');
});
});

test('rehydrates historical file refs from owner-scoped DB rows only', async () => {
Expand Down Expand Up @@ -3520,7 +3553,7 @@ describe('BaseClient', () => {
});
});

test('keeps a tool-routed file off the prompt when this turn can read it with code', () => {
test('keeps a tool-routed file off the prompt when the sandbox running it holds the file', () => {
routeCsvToTools();
TestClient.options.agent = {
provider: EModelEndpoint.openAI,
Expand All @@ -3536,13 +3569,51 @@ describe('BaseClient', () => {
type: 'text/csv',
text: 'region,total',
llmDeliveryPath: 'none',
metadata: { destinationChosen: false },
metadata: {
destinationChosen: false,
codeEnvRef: {
kind: 'user',
id: 'user-1',
storage_session_id: 'session-1',
file_id: 'sandbox-code-csv',
},
},
};

expect(TestClient.getAttachmentDeliveryPath(file)).toBe('none');
expect(TestClient.getTextContextAttachments([file])).toEqual([]);
});

test('delivers the text when the tool that would read the file has yet to receive it', () => {
/* An upload that named no destination is filed under no tool, so the enabled tool alone
* cannot serve it and withholding the text left it readable by nothing. */
routeCsvToTools();
TestClient.options.agent = {
provider: EModelEndpoint.openAI,
fileConsumers: { executeCode: true, fileSearch: true },
};
TestClient.options.agent.deliveryRouting = resolveTurnDeliveryRouting({
agent: TestClient.options.agent,
config: TestClient.options.req?.config,
});
const file = {
file_id: 'unprovisioned-csv',
filename: 'sales.csv',
type: 'text/csv',
text: 'region,total',
llmDeliveryPath: 'none',
metadata: { destinationChosen: false },
};

expect(TestClient.getAttachmentDeliveryPath(file)).toBe('text');
/* The filter selects records by the route this turn resolves, so it returns the record as
* stored; marking the copy admission reads is `resolveTurnAttachments`. */
expect(TestClient.getTextContextAttachments([file])).toEqual([file]);
expect(TestClient.resolveTurnAttachments([file])).toEqual([
{ ...file, llmDeliveryPath: 'text' },
]);
});

test('does not fall back on an endpoint that has not enabled it', () => {
routeCsvToTools({ textFallbackWithoutTools: false });
TestClient.options.agent = {
Expand Down
13 changes: 12 additions & 1 deletion api/server/controllers/agents/client.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -6718,7 +6718,18 @@ describe('AgentClient - titleConvo', () => {
...makeUploadedFile('fallback-file', 'sales.csv', 'text/csv'),
text: 'handoff fallback content',
llmDeliveryPath: 'none',
metadata: { destinationChosen: false },
/* The primary agent runs code, and a tool serves a file only once it holds it, so the
* sandbox reference is what keeps the text out of the primary prompt while the handoff
* agent, which runs no reader at all, still receives it. */
metadata: {
destinationChosen: false,
codeEnvRef: {
kind: 'user',
id: 'user-1',
storage_session_id: 'session-1',
file_id: 'sandbox-fallback-file',
},
},
};
const { resolveTurnDeliveryRouting } = jest.requireActual('@librechat/api');
client.options.req.config.fileConfig = {
Expand Down
36 changes: 32 additions & 4 deletions packages/api/src/agents/__tests__/initialize.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4516,6 +4516,23 @@ describe('initializeAgent tool-routed text fallback', () => {
metadata: { destinationChosen: false },
}) as IMongoFile;

/** A tool serves a file only once it holds it, so these carry the evidence provisioning
* writes: vectors for file search, a sandbox pointer for code execution. */
const embeddedCsv = () => ({ ...routedCsv(), embedded: true }) as IMongoFile;
const sandboxCsv = () =>
({
...routedCsv(),
metadata: {
destinationChosen: false,
codeEnvRef: {
kind: 'user',
id: 'user_1',
storage_session_id: 'session_1',
file_id: 'sandbox_file_1',
},
},
}) as IMongoFile;

async function initializeWith({
tools,
csv,
Expand Down Expand Up @@ -4624,13 +4641,24 @@ describe('initializeAgent tool-routed text fallback', () => {
},
);

it('keeps fallback out of the prompt when file search loads successfully', async () => {
const csv = routedCsv();
it('keeps fallback out of the prompt when file search loads and holds the file', async () => {
const csv = embeddedCsv();
const { result } = await initializeWith({ tools: [EToolResources.file_search], csv });
expect(result.fileConsumers).toEqual({ executeCode: false, fileSearch: true });
expect(result.requestAttachments).toEqual([csv]);
});

it('delivers fallback text when file search runs but never received the file', async () => {
/* The plain-chat File Search toggle: the upload names no destination, so nothing files it
* under a tool resource and the vector store stays empty. Withholding the text for the
* toggle alone left the attachment readable by neither the model nor the tool. */
const csv = routedCsv();
const { result } = await initializeWith({ tools: [EToolResources.file_search], csv });
expect(result.fileConsumers).toEqual({ executeCode: false, fileSearch: true });
expect(result.requestAttachments).toEqual([{ ...csv, llmDeliveryPath: 'text' }]);
expect(csv.llmDeliveryPath).toBe('none');
});

it('falls back after file search soft-fails and admits the returned text copy', async () => {
const csv = routedCsv();
const { result } = await initializeWith({
Expand Down Expand Up @@ -4751,8 +4779,8 @@ describe('initializeAgent tool-routed text fallback', () => {
);
});

it('leaves the file to Run Code when the agent can run code', async () => {
const csv = routedCsv();
it('leaves the file to Run Code when the sandbox already holds it', async () => {
const csv = sandboxCsv();

const { result, filterFilesByEndpointRuntimeConfig } = await initializeWith({
tools: [EToolResources.execute_code],
Expand Down
32 changes: 28 additions & 4 deletions packages/api/src/agents/files/delivery.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,21 @@ const config = {
const noReader: TurnFileConsumers = { executeCode: false, fileSearch: false };
const runsCode: TurnFileConsumers = { executeCode: true, fileSearch: false };

/** A tool serves a file only once it holds it, so a record left to the sandbox needs the
* reference provisioning writes for that tool to count as its reader. */
const inSandbox = <T extends { metadata?: Record<string, unknown> }>(file: T): T => ({
...file,
metadata: {
...file.metadata,
codeEnvRef: {
kind: 'user' as const,
id: 'user_1',
storage_session_id: 'session_1',
file_id: 'sandbox_file_1',
},
},
});

describe('resolveTurnDeliveryRouting', () => {
it('routes under the endpoint an agent names before its provider', () => {
expect(resolveTurnDeliveryRouting({ agent: { provider: 'openAI' }, config }).endpoint).toBe(
Expand Down Expand Up @@ -97,12 +112,20 @@ describe('applyTurnDelivery', () => {
expect(csv.llmDeliveryPath).toBe('none');
});

it('returns the same array when a tool this turn runs can read every file', () => {
const files = [csv, pdf];
it('returns the same array when a tool this turn runs holds every file', () => {
const files = [inSandbox(csv), pdf];

expect(applyTurnDelivery(files, { agent, config, consumers: runsCode })).toBe(files);
});

it('delivers text for a file the tool that would read it has yet to receive', () => {
/* An upload that named no destination is filed under no tool, so the enabled tool alone is
* not what serves it: withholding the text on that basis left it readable by nothing. */
expect(applyTurnDelivery([csv], { agent, config, consumers: runsCode })).toEqual([
{ ...csv, llmDeliveryPath: 'text' },
]);
});

it('marks nothing where the endpoint has not enabled the fallback', () => {
const files = [csv];
const disabled = {
Expand Down Expand Up @@ -199,10 +222,11 @@ describe('applyTurnDelivery', () => {

it('removes a record this turn leaves to tools from model admission', () => {
/* Over-admitting cannot pass a limit; dropping a record the client still sends would. */
const files = [{ ...csv, llmDeliveryPath: 'text' }];
const held = inSandbox(csv);
const files = [{ ...held, llmDeliveryPath: 'text' }];

expect(applyTurnDelivery(files, { agent, config, consumers: runsCode })).toEqual([
{ ...csv, llmDeliveryPath: 'none' },
{ ...held, llmDeliveryPath: 'none' },
]);
});

Expand Down
19 changes: 18 additions & 1 deletion packages/api/src/agents/files/encode.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -184,14 +184,31 @@ describe('createRunFileMessageEncoder', () => {
},
});

const [noReader, runsCode, unknown] = await Promise.all([
/* A tool serves a file only once it holds it, so the child that runs code keeps the file off
* its prompt for the copy the sandbox has, and receives the text for the one it does not. */
const sandboxCsv: TFile = {
...csv,
metadata: {
destinationChosen: false,
codeEnvRef: {
kind: 'user',
id: 'user-1',
storage_session_id: 'session-1',
file_id: 'sandbox-input-csv',
},
},
};

const [noReader, runsCode, unprovisioned, unknown] = await Promise.all([
harness.encode([csv], 'noReader'),
harness.encode([sandboxCsv], 'runsCode'),
harness.encode([csv], 'runsCode'),
harness.encode([csv], 'unknown'),
]);

expect(JSON.stringify(noReader[0].content)).toContain('region,total');
expect(runsCode).toEqual([]);
expect(JSON.stringify(unprovisioned[0].content)).toContain('region,total');
expect(unknown).toEqual([]);
expect(harness.extractText).toHaveBeenCalledWith(
expect.objectContaining({ attachments: [{ ...csv, llmDeliveryPath: 'text' }] }),
Expand Down
11 changes: 5 additions & 6 deletions packages/api/src/agents/resources.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
FileSources,
getCodeEnvRefs,
canToolResourceConsume,
hasToolResourceProvisioning,
} from 'librechat-data-provider';
import type {
AgentToolResources,
Expand Down Expand Up @@ -441,12 +442,10 @@ const computeProvisionState = async ({
/** What the record itself shows about where it was sent. A message attachment is never
* in the agent's resources, so a reference or an embedding is the only evidence that
* the chooser picked that destination, and it is proof: nothing else creates one. */
const carriesEvidenceFor = (file: TFile, resourceType: EToolResources): boolean => {
if (resourceType === EToolResources.execute_code) {
return getCodeEnvRefs(file.metadata).length > 0;
}
return file.embedded === true || (file.metadata?.embeddedEntities?.length ?? 0) > 0;
};
/* The predicate the turn's delivery decision reads too, so a file this queue has yet to
* provision is never one the prompt withheld its text for. */
const carriesEvidenceFor = (file: TFile, resourceType: EToolResources): boolean =>
hasToolResourceProvisioning(file, resourceType);

const allowsResource = (file: TFile, resourceType: EToolResources): boolean => {
if (!cameFromChooser(file)) {
Expand Down
Loading
Loading