Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 12 additions & 12 deletions client/modules/search.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -317,15 +317,15 @@ describe("Search Module", () => {
});

describe("Cache Failure Scenarios", () => {
it("should return empty results instead of throwing when a text search fails end-to-end", async () => {
it("should rethrow when a text search fails end-to-end", async () => {
// getCachedResult/cacheResult swallow their own read/write errors, so a
// cache failure surfaces as a miss followed by a real fetch — searchText
// must still resolve gracefully instead of throwing.
// cache failure surfaces as a miss followed by a real fetch — the fetch
// failure must propagate so the caller can mark the search as failed.
mockFetch.mockRejectedValue(new Error("Cache read error"));

const results = await searchModule.searchText("test query");

expect(results).toEqual([]);
await expect(searchModule.searchText("test query")).rejects.toThrow(
"Cache read error",
);
expect(addLogEntry).toHaveBeenCalledWith(
expect.stringContaining("Text search failed"),
);
Expand All @@ -344,15 +344,15 @@ describe("Search Module", () => {
});

describe("Database Integrity Failures", () => {
it("should return empty results instead of throwing when an image search fails end-to-end", async () => {
it("should rethrow when an image search fails end-to-end", async () => {
// ensureIntegrity/cleanExpiredCache swallow their own errors, so a
// corrupted database surfaces as a miss followed by a real fetch —
// searchImages must still resolve gracefully instead of throwing.
// corrupted database surfaces as a miss followed by a real fetch — the
// fetch failure must propagate so the caller can mark the search failed.
mockFetch.mockRejectedValue(new Error("Database integrity check failed"));

const results = await searchModule.searchImages("test query");

expect(results).toEqual([]);
await expect(searchModule.searchImages("test query")).rejects.toThrow(
"Database integrity check failed",
);
expect(addLogEntry).toHaveBeenCalledWith(
expect.stringContaining("Image search failed"),
);
Expand Down
8 changes: 6 additions & 2 deletions client/modules/search.ts
Original file line number Diff line number Diff line change
Expand Up @@ -494,10 +494,12 @@ const searchService = {
},
);
} catch (error) {
// Rethrow so the caller can tell a failed search apart from one that
// genuinely has no results; an empty array means zero results only.
addLogEntry(
`Text search failed: ${error instanceof Error ? error.message : String(error)}`,
);
return [];
throw error;
}
},

Expand All @@ -522,10 +524,12 @@ const searchService = {
},
);
} catch (error) {
// Rethrow so the caller can tell a failed search apart from one that
// genuinely has no results; an empty array means zero results only.
addLogEntry(
`Image search failed: ${error instanceof Error ? error.message : String(error)}`,
);
return [];
throw error;
}
},

Expand Down
31 changes: 29 additions & 2 deletions client/modules/textGeneration.degradation.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ const harness = vi.hoisted(() => {
textGenerationState: "idle",
textSearchState: "idle",
textSearchResults: [] as unknown[],
llmTextSearchResults: [] as unknown[],
pageContents: {} as Record<string, string>,
searchRunId: "run-1",
searchPromise: Promise.resolve({}) as Promise<unknown>,
Expand Down Expand Up @@ -41,7 +42,9 @@ vi.mock("./pubSub", () => ({
updateConversationSummary: vi.fn(),
updateImageSearchResults: vi.fn(),
updateImageSearchState: vi.fn(),
updateLlmTextSearchResults: vi.fn(),
updateLlmTextSearchResults: (results: unknown[]) => {
harness.state.llmTextSearchResults = results;
},
updatePageContents: (contents: Record<string, string>) => {
harness.state.pageContents = contents;
},
Expand Down Expand Up @@ -150,6 +153,7 @@ describe("search degradation", () => {
harness.state.textGenerationState = "idle";
harness.state.textSearchState = "idle";
harness.state.textSearchResults = [];
harness.state.llmTextSearchResults = [];
harness.state.settings.enableAiResponse = false;
harness.state.settings.enableTextSearch = true;
harness.stateTransitions.length = 0;
Expand Down Expand Up @@ -179,14 +183,37 @@ describe("search degradation", () => {
expect(harness.state.textSearchState).toBe("completed");
});

it("reports the text search as failed when the keyword fallback is also empty", async () => {
it("keeps the text search completed when the keyword fallback is also empty", async () => {
vi.mocked(searchText).mockResolvedValue([]);

await searchAndRespond();
await harness.state.searchPromise;

expect(searchText).toHaveBeenCalledTimes(2);
expect(harness.state.textSearchResults).toEqual([]);
expect(harness.state.textSearchState).toBe("completed");
});

it("marks the text search as failed when the search request errors", async () => {
vi.mocked(searchText).mockRejectedValue(
new Error("HTTP error! status: 502"),
);

// A previous search left the LLM grounding channel populated; the failed
// search must clear it so the AI can't ground on stale results.
harness.state.llmTextSearchResults = [
["Stale result", "Old snippet", "https://stale.example.com"],
];

await searchAndRespond();
await harness.state.searchPromise;

// An outage is not an empty result set: the keyword fallback must not
// fire, the search ends failed so the retry UI shows up, and the stale
// grounding is dropped.
expect(searchText).toHaveBeenCalledTimes(1);
expect(harness.state.textSearchResults).toEqual([]);
expect(harness.state.llmTextSearchResults).toEqual([]);
expect(harness.state.textSearchState).toBe("failed");
});
});
Expand Down
106 changes: 58 additions & 48 deletions client/modules/textGeneration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,8 @@ export async function searchAndRespond() {

updateTextSearchResults([]);

updateLlmTextSearchResults([]);

updateImageSearchResults([]);

updateChatMessages([]);
Expand Down Expand Up @@ -425,42 +427,46 @@ async function startTextSearch(query: string) {
if (getSettings().enableTextSearch) {
updateTextSearchState("running");

let textResults = await searchText(
searchQuery,
getSettings().searchResultsLimit,
);

if (textResults.length === 0) {
const queryKeywords = await getKeywords(query, 10);
const keywordResults = await searchText(
queryKeywords.join(" "),
try {
let textResults = await searchText(
searchQuery,
getSettings().searchResultsLimit,
);
textResults = keywordResults;
}

results.textResults = textResults;
if (textResults.length === 0) {
const queryKeywords = await getKeywords(query, 10);
const keywordResults = await searchText(
queryKeywords.join(" "),
getSettings().searchResultsLimit,
);
textResults = keywordResults;
}

results.textResults = textResults;

updateTextSearchState(
results.textResults.length === 0 ? "failed" : "completed",
);
updateTextSearchResults(textResults);
updateTextSearchState("completed");
updateTextSearchResults(textResults);

const resultsForLlm = textResults.slice(0, searchResultsToConsider);
updateLlmTextSearchResults(resultsForLlm);
const resultsForLlm = textResults.slice(0, searchResultsToConsider);
updateLlmTextSearchResults(resultsForLlm);

updateSearchResults(getCurrentSearchRunId(), {
type: "text",
items: textResults.map(([title, snippet, url]) => ({
title,
url,
snippet,
})),
});
updateSearchResults(getCurrentSearchRunId(), {
type: "text",
items: textResults.map(([title, snippet, url]) => ({
title,
url,
snippet,
})),
});

// Started here but awaited last: only the AI answer waits on it, while
// image search and the history write carry on.
pageContentsRead = readPageContents(searchQuery, resultsForLlm);
// Started here but awaited last: only the AI answer waits on it, while
// image search and the history write carry on.
pageContentsRead = readPageContents(searchQuery, resultsForLlm);
} catch {
// An outage (non-200 from /search/*) is not the same as a query with no
// results; only the former leaves the search in the failed state.
updateTextSearchState("failed");
}
}

if (getSettings().enableImageSearch) {
Expand All @@ -476,25 +482,29 @@ async function startImageSearch(
searchQuery: string,
results: { textResults: TextSearchResults; imageResults: ImageSearchResults },
) {
const imageResults = await searchImages(
searchQuery,
getSettings().searchResultsLimit,
);
results.imageResults = imageResults;
updateImageSearchState(
results.imageResults.length === 0 ? "failed" : "completed",
);
updateImageSearchResults(imageResults);

updateSearchResults(getCurrentSearchRunId(), {
type: "image",
items: imageResults.map(([title, url, thumbnailUrl, sourceUrl]) => ({
title,
url,
thumbnail: thumbnailUrl,
sourceUrl,
})),
});
try {
const imageResults = await searchImages(
searchQuery,
getSettings().searchResultsLimit,
);
results.imageResults = imageResults;
updateImageSearchState("completed");
updateImageSearchResults(imageResults);

updateSearchResults(getCurrentSearchRunId(), {
type: "image",
items: imageResults.map(([title, url, thumbnailUrl, sourceUrl]) => ({
title,
url,
thumbnail: thumbnailUrl,
sourceUrl,
})),
});
} catch {
// Same distinction as the text path: an outage fails the search, an empty
// result set does not.
updateImageSearchState("failed");
}
}

function canDownloadModels(): Promise<void> {
Expand Down
2 changes: 1 addition & 1 deletion docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -291,7 +291,7 @@ INTERNAL_OPENAI_COMPATIBLE_API_MODEL="llama-3.1-70b"

MiniSearch logs internal events to an in-app log panel (see the Logs section of the menu), backed by `logEntriesPubSub` in `client/modules/logEntries.ts`. There is no separate browser-console debug flag to enable.

**Diagnosing empty search results:** if a search returns no results, the client only sees a generic failure. The actual reason is printed to the server's console (via `server/webSearchService.ts`, always-on `debug` logging), either the SearXNG engines that failed (timeouts, suspensions, rate limits, from SearXNG's `unresponsive_engines` field) or a note that all returned results were discarded during processing (missing title, snippet, or media source). Check the server logs (`docker compose logs`) when troubleshooting failed searches.
**Diagnosing empty search results:** the client distinguishes a search that returned no results (a no-results alert, state stays `completed`) from one that failed outright on an outage (a distinct failed state with a retry). The actual reason is printed to the server's console (via `server/webSearchService.ts`, always-on `debug` logging), either the SearXNG engines that failed (timeouts, suspensions, rate limits, from SearXNG's `unresponsive_engines` field) or a note that all returned results were discarded during processing (missing title, snippet, or media source). Check the server logs (`docker compose logs`) when troubleshooting failed searches.

Check effective configuration:
```typescript
Expand Down
18 changes: 7 additions & 11 deletions docs/failure-injection.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,15 @@ needed.
## Degradation Matrix

| Failure injected | Expected degradation | Where it is pinned |
|---|---|---|
| --- | --- | --- |
| SearXNG answers 500 once, then recovers | Retried with exponential backoff, results returned | `server/webSearchService.test.ts` › retry logic |
| SearXNG answers 500 on every attempt | Retry cycle exhausts (4 requests) and costs a single circuit-breaker failure | `server/webSearchService.test.ts` › graceful degradation |
| SearXNG fails five cycles in a row | Circuit opens; further searches short-circuit without calling the upstream | `server/webSearchService.test.ts` › graceful degradation |
| SearXNG recovers after the reset timeout | One healthy response closes the circuit again | `server/webSearchService.test.ts` › graceful degradation |
| SearXNG answers 200 with a non-JSON body | Empty result set, no throw | `server/webSearchService.test.ts` › graceful degradation |
| SearXNG answers 200 with a non-JSON body | `fetchSearXNG` throws and the endpoint answers HTTP 502 | `server/webSearchService.test.ts` › graceful degradation |
| SearXNG returns results that are all unusable | Empty result set, no throw | `server/webSearchService.test.ts` › graceful degradation |
| Provider down vs. genuinely zero results | Both yield `[]` (indistinguishable downstream, see Known Limitations) | `server/webSearchService.test.ts` › graceful degradation |
| Provider down vs. genuinely zero results | Provider down: `fetchSearXNG` throws, the endpoint answers HTTP 502, and the client search state becomes `failed`; genuinely zero results: HTTP 200 with `[]`, state stays `completed` and the no-results alert renders | `server/webSearchService.test.ts` and `server/searchEndpointServerHook.test.ts` › graceful degradation |
| SearXNG is down | `/search/text` and `/search/images` answer HTTP 502 with a JSON error | `server/searchEndpointServerHook.test.ts` › graceful degradation |
| Reranker is not ready | Results served in SearXNG order, HTTP 200 | `server/searchEndpointServerHook.test.ts` › graceful degradation |
| Reranking throws mid-request | Results served in SearXNG order, HTTP 200 | `server/searchEndpointServerHook.test.ts` › graceful degradation |
| Reranker is not ready on an image search | Images still served with thumbnails | `server/searchEndpointServerHook.test.ts` › graceful degradation |
Expand All @@ -25,7 +26,9 @@ needed.
| Thumbnail host never answers | Request aborted after the timeout, image dropped | `server/searchEndpointServerHook.test.ts` › graceful degradation |
| Search returns nothing to the endpoint | HTTP 200 with `[]`, not an error | `server/searchEndpointServerHook.test.ts` › graceful degradation |
| Text search returns nothing to the client | Keyword-only query retried as a fallback | `client/modules/textGeneration.degradation.test.ts` |
| Keyword fallback also returns nothing | Text search state becomes `failed` | `client/modules/textGeneration.degradation.test.ts` |
| Keyword fallback also returns nothing | Text search state stays `completed` with the no-results alert | `client/modules/textGeneration.degradation.test.ts` |
| SearXNG is down on the client | Text search state becomes `failed`, keyword fallback does not fire | `client/modules/textGeneration.degradation.test.ts` |
| Client search fails after a previous search populated the LLM channel | The grounding channel is cleared, so the AI can't ground on the previous query's results | `client/modules/textGeneration.degradation.test.ts` |
| Result page host resolves into a private range | Page skipped, no request is made | `server/pageContentService.test.ts` › fetchPageContents |
| Result page redirects into a private range | Redirect not followed, page skipped | `server/pageContentService.test.ts` › fetchPageContents |
| Result page errors, is not a document, or yields no text | That page is skipped, the others still return | `server/pageContentService.test.ts` › fetchPageContents |
Expand All @@ -41,13 +44,6 @@ needed.
| `/inference` answers 503 | Generation state becomes `failed`, nothing persisted | `client/modules/textGeneration.degradation.test.ts` |
| Generation interrupted mid-stream | Partial answer preserved, state stays `interrupted` | `client/modules/textGeneration.degradation.test.ts` |

## Known Limitations

`fetchSearXNG` catches every failure and returns `[]`, so "the provider is down"
and "there are no results for this query" reach the client as the same response.
The matrix pins that behavior rather than hiding it: changing it means changing
the endpoint contract, and the test is where that decision becomes visible.

## Adding a Row

Keep each case next to the module it covers, reusing that file's mocks and
Expand Down
30 changes: 26 additions & 4 deletions server/searchEndpointServerHook.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -443,8 +443,10 @@ describe("searchEndpointServerHook", () => {
}
});

it("serves an empty result set instead of an error when SearXNG is down", async () => {
vi.mocked(fetchSearXNG).mockResolvedValue([]);
it("answers 502 when SearXNG is down", async () => {
vi.mocked(fetchSearXNG).mockRejectedValue(
new Error("SearXNG request failed with status 503"),
);

const handler = getRegisteredHandler();
const response = createResponse();
Expand All @@ -455,8 +457,28 @@ describe("searchEndpointServerHook", () => {
vi.fn(),
);

expect(response.statusCode).toBe(200);
expect(response.end).toHaveBeenCalledWith("[]");
expect(response.statusCode).toBe(502);
expect(response.end).toHaveBeenCalledWith(
JSON.stringify({ error: "Search service unavailable" }),
);
});

it("answers 502 on image searches when SearXNG is down", async () => {
vi.mocked(fetchSearXNG).mockRejectedValue(new Error("network down"));

const handler = getRegisteredHandler();
const response = createResponse();

await handler(
createRequest("/search/images?q=cats&token=abc"),
response,
vi.fn(),
);

expect(response.statusCode).toBe(502);
expect(response.end).toHaveBeenCalledWith(
JSON.stringify({ error: "Search service unavailable" }),
);
});
});

Expand Down
12 changes: 11 additions & 1 deletion server/searchEndpointServerHook.ts
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,17 @@ export function searchEndpointServerHook<
const isTextSearch = request.url?.startsWith("/search/text");
const searchType = isTextSearch ? "text" : "images";

const searxngResults = await fetchSearXNG(query, searchType, limit);
let searxngResults: TextResult[] | ImageResult[];
try {
searxngResults = await fetchSearXNG(query, searchType, limit);
} catch {
// SearXNG is unreachable: answer non-200 so the client can tell an
// outage apart from a search that genuinely has no results.
response.statusCode = 502;
response.setHeader("Content-Type", "application/json");
response.end(JSON.stringify({ error: "Search service unavailable" }));
return;
}

if (isTextSearch) {
const results = searxngResults as TextResult[];
Expand Down
Loading