From b0740a458495aeea4c5ff852efd98069c442834b Mon Sep 17 00:00:00 2001 From: "M. Hosseyn Najafi" Date: Sun, 7 Jun 2026 18:38:47 +0200 Subject: [PATCH] Cleanup the UA detection method --- nicegui/client.py | 72 ++++++++++++++++++--------------- tests/test_markdown_response.py | 13 ++++++ 2 files changed, 52 insertions(+), 33 deletions(-) diff --git a/nicegui/client.py b/nicegui/client.py index 00a4b69bcd..e87bbbc770 100644 --- a/nicegui/client.py +++ b/nicegui/client.py @@ -7,6 +7,7 @@ import uuid from collections import defaultdict from collections.abc import Callable, Iterable +from functools import cached_property from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar, cast @@ -153,18 +154,7 @@ def __exit__(self, *_) -> None: def build_response(self, request: Request, status_code: int = 200) -> Response: """Build a FastAPI response for the client.""" - accept = request.headers.get('accept', '') - user_agent = request.headers.get('user-agent', '') - # NOTE: This simple check doesn't handle quality values (q=) or wildcards (*/*). - # It works for the real use case: agents sending exactly `Accept: text/markdown`. - explicit_markdown = 'text/markdown' in accept and 'text/html' not in accept - # UA-based fallback: clients like claude.ai's WebFetch send `Accept: */*` despite - # being agents (vs. Claude Code which sends `text/markdown, text/html, */*`). - # When the page opts in and the request looks like an agent with no explicit - # text/* preference, prefer markdown. Explicit `text/html` always wins HTML. - wildcard_accept = not accept or accept.strip() == '*/*' - ua_fallback = wildcard_accept and 'text/html' not in accept and _looks_like_agent(user_agent) - if self.page.resolve_markdown() and (explicit_markdown or ua_fallback): + if self.page.resolve_markdown() and self._did_user_request_markdown(request): parts = [] if title := self.resolve_title(): parts.append(f'# {title}') @@ -472,28 +462,44 @@ def prune_instances(cls, *, client_age_threshold: float = 60.0) -> None: except Exception: log.exception('Error while pruning clients') + @cached_property + def _agent_ua_regex(self): + """ + Known live-user / agent fetchers (verified April 2026). + + Vendor docs change frequently; refresh this list when new live-user agents are announced. + + Authoritative pages at the time of writing: + - support.claude.com (claudebot / claude-user / claude-searchbot) + - platform.openai.com/docs/bots (gptbot / oai-searchbot / chatgpt-user) + - docs.perplexity.ai (perplexitybot / perplexity-user) + - google's search central docs cover google-cloudvertexbot + - gemini-deep-research and google-agent (project mariner) are documented in their + respective product pages rather than on the common-crawlers page. + + Substring match is intentional: vendors append version + embed-product suffixes + (e.g. `claude-user (claude-code/2.1.121; ...)`). + """ + return re.compile( + r'claude-?(bot|user|searchbot)|gptbot|oai-searchbot|chatgpt-user|' + r'perplexity(bot|-user)|google-(cloudvertexbot|agent)|gemini-deep-research', + re.IGNORECASE, + ) + + def _did_user_request_markdown(self, request) -> bool: + accept = request.headers.get('accept', '') + if 'text/markdown' in accept and ('text/html' not in accept or accept.find('text/markdown') < accept.find('text/html')): + return True + if accept and '*/*' not in accept: + return False + # UA-based fallback: clients like claude.ai's WebFetch send `Accept: */*` despite + # being agents (vs. Claude Code which sends `text/markdown, text/html, */*`). + # When the page opts in and the request looks like an agent with no explicit + # text/* preference, prefer markdown. Explicit `text/html` always wins HTML. + user_agent = request.headers.get('user-agent', '') + return bool('text/html' not in accept and self._agent_ua_regex.search(user_agent)) + def _is_prefetch(request: Request) -> bool: purpose = (request.headers.get('Sec-Purpose') or request.headers.get('Purpose') or '').lower() return 'prefetch' in purpose and 'prerender' not in purpose - - -# Known live-user / agent fetchers (verified April 2026). Vendor docs change frequently; -# refresh this list when new live-user agents are announced. Authoritative pages at the time -# of writing: support.claude.com (ClaudeBot / Claude-User / Claude-SearchBot), -# platform.openai.com/docs/bots (GPTBot / OAI-SearchBot / ChatGPT-User), -# docs.perplexity.ai (PerplexityBot / Perplexity-User), Google's Search Central docs cover -# Google-CloudVertexBot; Gemini-Deep-Research and Google-Agent (Project Mariner) are -# documented in their respective product pages rather than on the common-crawlers page. -# Substring match is intentional: vendors append version + embed-product suffixes -# (e.g. `Claude-User (claude-code/2.1.121; ...)`). -_AGENT_UA_REGEX = re.compile( - r'claude-?(bot|user|searchbot)|gptbot|oai-searchbot|chatgpt-user|' - r'perplexity(bot|-user)|google-(cloudvertexbot|agent)|gemini-deep-research', - re.IGNORECASE, -) - - -def _looks_like_agent(user_agent: str) -> bool: - """Heuristic: does the User-Agent header belong to a known live-user agent fetcher?""" - return bool(_AGENT_UA_REGEX.search(user_agent)) diff --git a/tests/test_markdown_response.py b/tests/test_markdown_response.py index d0866d6b98..34536b5888 100644 --- a/tests/test_markdown_response.py +++ b/tests/test_markdown_response.py @@ -58,6 +58,19 @@ def page(): assert 'text/markdown' in response.headers['content-type'] +async def test_ua_fallback_respects_markdown_coming_before_html(user: User): + """A known-agent UA asking for `text/markdown, text/html` gets markdown because it's requested first.""" + @ui.page('/', markdown=True) + def page(): + ui.label('Hello') + + response = await user.http_client.get('/', headers={ + 'Accept': 'text/markdown, text/html', + 'User-Agent': 'GPTBot/1.1', + }) + assert 'text/markdown' in response.headers['content-type'] + + async def test_ua_fallback_respects_explicit_html_preference(user: User): """A known-agent UA asking for `text/html` still gets HTML — explicit Accept always wins.""" @ui.page('/', markdown=True)