Re-implement SEO support for website (ported to website/components/ structure)

main.py

@@ -1,16 +1,18 @@
 #!/usr/bin/env python3
 import os
 from pathlib import Path
+from xml.sax.saxutils import escape as xml_escape
 
 from fastapi import Request
+from fastapi.responses import PlainTextResponse
 from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
 from starlette.middleware.sessions import SessionMiddleware
 from starlette.responses import Response
 
 from nicegui import app, core, ui
 from nicegui.page_arguments import RouteMatch
 from website import design as d
-from website import documentation, examples_page, fly, header, imprint_privacy, main_page, rate_limits, svg
+from website import documentation, examples_page, fly, header, imprint_privacy, main_page, rate_limits, seo, svg
 from website.components import footer_section
 from website.documentation.intersection_observer import IntersectionObserver as intersection_observer
 
@@ -43,6 +45,42 @@ async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -
 documentation.build_tree()
 
 
+@app.get('/robots.txt')
+def _robots_txt() -> PlainTextResponse:
+    return PlainTextResponse(
+        f'User-agent: *\nAllow: /\n\nSitemap: {seo.SITE_URL}/sitemap.xml\n',
+        media_type='text/plain',
+    )
+
+
+@app.get('/sitemap.xml')
+def _sitemap_xml() -> Response:
+    urls = [
+        ('/', '1.0', 'weekly'),
+        ('/documentation', '0.9', 'weekly'),
+        ('/examples', '0.8', 'monthly'),
+        ('/imprint_privacy', '0.3', 'yearly'),
+    ]
+    for name in documentation.registry:
+        if name:  # skip the overview page (already added as /documentation)
+            urls.append((f'/documentation/{name}', '0.7', 'monthly'))
+    xml_urls = '\n'.join(
+        f'  <url>\n'
+        f'    <loc>{xml_escape(seo.SITE_URL + path)}</loc>\n'
+        f'    <changefreq>{freq}</changefreq>\n'
+        f'    <priority>{priority}</priority>\n'
+        f'  </url>'
+        for path, priority, freq in urls
+    )
+    xml = (
+        '<?xml version="1.0" encoding="UTF-8"?>\n'
+        '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n'
+        f'{xml_urls}\n'
+        '</urlset>\n'
+    )
+    return Response(content=xml, media_type='application/xml')
+
+
 @app.post('/dark_mode')
 async def _post_dark_mode(request: Request) -> None:
     app.storage.browser['dark_mode'] = (await request.json()).get('value')

tests/test_seo.py

@@ -0,0 +1,178 @@
+from website.seo import breadcrumb_jsonld, extract_description, page_seo_html
+
+
+def test_extract_description_plain_text():
+    text = 'This is a simple description that is long enough to pass the minimum length threshold for extraction.'
+    result = extract_description(text)
+    assert result == text
+
+
+def test_extract_description_returns_none_for_short_text():
+    assert extract_description('Too short') is None
+    assert extract_description('') is None
+
+
+def test_extract_description_strips_markdown_bold():
+    text = 'This is **bold text** that should be cleaned up properly by the extraction function.'
+    result = extract_description(text)
+    assert '**' not in result
+    assert 'bold text' in result
+
+
+def test_extract_description_strips_markdown_italic():
+    text = 'This is _italic text_ that should be cleaned up properly by the extraction function.'
+    result = extract_description(text)
+    assert result is not None
+    assert '_italic' not in result
+    assert 'italic text' in result
+
+
+def test_extract_description_strips_markdown_links():
+    text = 'Click [this link](https://example.com) to visit the site and learn about the features.'
+    result = extract_description(text)
+    assert result is not None
+    assert 'this link' in result
+    assert 'https://example.com' not in result
+    assert '[' not in result
+    assert '](' not in result
+
+
+def test_extract_description_strips_rst_links():
+    text = 'See `NiceGUI documentation <https://nicegui.io>`_ for more information about all features.'
+    result = extract_description(text)
+    assert result is not None
+    assert 'NiceGUI documentation' in result
+    assert 'https://nicegui.io' not in result
+    assert '`' not in result
+
+
+def test_extract_description_strips_backtick_code():
+    text = 'Use the `ui.button` element to create interactive buttons in your Python application.'
+    result = extract_description(text)
+    assert result is not None
+    assert 'ui.button' in result
+    assert '`' not in result
+
+
+def test_extract_description_strips_html_tags():
+    text = 'This has <b>bold HTML</b> and <a href="url">a link</a> that need to be properly cleaned.'
+    result = extract_description(text)
+    assert result is not None
+    assert '<b>' not in result
+    assert '<a ' not in result
+    assert 'bold HTML' in result
+
+
+def test_extract_description_truncates_long_text():
+    text = 'A ' * 200  # very long text
+    result = extract_description(text)
+    assert result is not None
+    assert len(result) <= 160
+    assert result.endswith('...')
+
+
+def test_extract_description_truncates_at_word_boundary():
+    text = 'word ' * 50  # 250 chars
+    result = extract_description(text)
+    assert result is not None
+    assert len(result) <= 160
+    assert result.endswith('...')
+    assert not result.endswith(' ...')  # should not have trailing space before ellipsis
+
+
+def test_extract_description_strips_param_directives():
+    text = ('This function does something useful and important for the application.\n'
+            ':param name: the name of the element\n'
+            ':type name: str')
+    result = extract_description(text)
+    assert result is not None
+    assert ':param' not in result
+    assert 'something useful' in result
+
+
+def test_extract_description_strips_return_directives():
+    text = ('This function returns a value that is useful for the calling application.\n'
+            ':return the computed value')
+    result = extract_description(text)
+    assert result is not None
+    assert ':return' not in result
+
+
+def test_extract_description_collapses_whitespace():
+    text = 'This   has    lots\n\nof    whitespace   scattered   throughout   the  entire  text  string.'
+    result = extract_description(text)
+    assert result is not None
+    assert '  ' not in result
+
+
+def test_page_seo_html_contains_meta_description():
+    result = page_seo_html(title='Test', description='A test page', path='/test')
+    assert 'name="description"' in result
+    assert 'A test page' in result
+
+
+def test_page_seo_html_tags_carry_seo_marker():
+    # SPA navigation removes previously injected SEO tags via this marker;
+    # every emitted tag must carry it so the cleanup query catches them.
+    result = page_seo_html(title='Test', description='A test page', path='/test')
+    for line in result.splitlines():
+        assert 'data-nicegui-seo' in line, f'missing SEO marker on: {line}'
+
+
+def test_page_seo_html_contains_canonical():
+    result = page_seo_html(title='Test', description='A test page', path='/test')
+    assert 'rel="canonical"' in result
+    assert 'https://nicegui.io/test' in result
+
+
+def test_page_seo_html_contains_open_graph():
+    result = page_seo_html(title='Test', description='A test page', path='/test')
+    assert 'og:title' in result
+    assert 'og:description' in result
+    assert 'og:url' in result
+    assert 'og:type' in result
+    assert 'og:site_name' in result
+
+
+def test_page_seo_html_contains_twitter_card():
+    result = page_seo_html(title='Test', description='A test page', path='/test')
+    assert 'twitter:card' in result
+    assert 'twitter:title' in result
+
+
+def test_page_seo_html_escapes_special_characters():
+    result = page_seo_html(title='Test & "Quotes"', description='A <b>bold</b> description', path='/test')
+    assert 'Test &amp; &quot;Quotes&quot;' in result
+    assert '&lt;b&gt;bold&lt;/b&gt;' in result
+
+
+def test_page_seo_html_og_type_default():
+    result = page_seo_html(title='Test', description='Desc', path='/')
+    assert 'content="website"' in result
+
+
+def test_page_seo_html_og_type_article():
+    result = page_seo_html(title='Test', description='Desc', path='/', og_type='article')
+    assert 'content="article"' in result
+
+
+def test_breadcrumb_jsonld_structure():
+    result = breadcrumb_jsonld([('Home', '/'), ('Docs', '/docs')])
+    assert 'type="application/ld+json"' in result
+    assert '</script>' in result
+    assert 'BreadcrumbList' in result
+    assert '"position":1' in result
+    assert '"position":2' in result
+    assert '"name":"Home"' in result
+    assert '"name":"Docs"' in result
+
+
+def test_breadcrumb_jsonld_includes_full_urls():
+    result = breadcrumb_jsonld([('Home', '/'), ('Docs', '/docs')])
+    assert 'https://nicegui.io/' in result
+    assert 'https://nicegui.io/docs' in result
+
+
+def test_breadcrumb_jsonld_escapes_closing_script():
+    result = breadcrumb_jsonld([('Test</script>', '/test')])
+    assert '</script><' not in result.replace('</script>', '', 1)  # only the closing tag itself

website/__init__.py

@@ -1,9 +1,14 @@
-from . import documentation, examples_page, fly, main_page, svg
+"""NiceGUI documentation website package.
 
-__all__ = [
-    'documentation',
-    'examples_page',
-    'fly',
-    'main_page',
-    'svg',
-]
+This ``__init__`` intentionally does not eagerly import submodules.
+``main.py`` imports the submodules it needs explicitly, and eagerly
+importing them here pulls in documentation modules that instantiate
+``Client`` objects at module load time (to pre-render UI snippets).
+Those ``Client`` objects schedule an ``Outbox.loop`` coroutine via
+``background_tasks.create_or_defer``; when there is no running event
+loop (e.g. during pytest collection of ``tests/test_seo.py``), the
+coroutines are deferred to ``app.on_startup``, never awaited, and
+eventually garbage-collected, producing ``RuntimeWarning`` entries
+that break the ``unraisableexception`` plugin's setup hook for the
+first test.
+"""

website/documentation/rendering.py

@@ -1,17 +1,79 @@
+import functools
+
 from nicegui import ui
 
 from ..design import section_heading, subheading
+from ..seo import DEFAULT_DESCRIPTION, TAGLINE, apply_page_seo, extract_description
 from .content import DocumentationPage
+from .content.overview import tiles
 from .custom_restructured_text import CustomRestructuredText as custom_restructured_text
 from .demo import demo
 from .reference import generate_class_doc
 
 
+@functools.cache
+def _get_tile_descriptions() -> dict[str, str]:
+    from .content import registry  # NOTE: deferred to avoid circular import
+    result: dict[str, str] = {}
+    for module, description in tiles:
+        name = module.__name__.rsplit('.', 1)[-1]
+        desc = extract_description(description)
+        if name in registry and desc is not None:
+            result[name] = desc
+    return result
+
+
+def _build_page_description(documentation: DocumentationPage) -> str:
+    tile_desc = _get_tile_descriptions().get(documentation.name)
+    if tile_desc:
+        return tile_desc
+    for part in documentation.parts:
+        if part.description and not part.link:
+            desc = extract_description(part.description)
+            if desc is not None:
+                return desc
+        if part.search_text and not part.link:
+            desc = extract_description(part.search_text)
+            if desc is not None:
+                return desc
+    if documentation.subtitle:
+        desc = extract_description(documentation.subtitle)
+        if desc is not None:
+            return desc
+    for part in documentation.parts:
+        if part.description:
+            desc = extract_description(part.description)
+            if desc is not None:
+                return desc
+    return DEFAULT_DESCRIPTION
+
+
 def render_page(documentation: DocumentationPage) -> None:
     """Render the documentation."""
     title = (documentation.title or '').replace('*', '')
+    if not title:
+        seo_title = f'NiceGUI Documentation - {TAGLINE}'
+    elif title.split()[0] == 'NiceGUI':
+        seo_title = f'{title} - {TAGLINE}'
+    else:
+        seo_title = f'{title} - NiceGUI Documentation'
     ui.page_title('NiceGUI' if not title else title if title.split()[0] == 'NiceGUI' else f'{title} | NiceGUI')
 
+    description = _build_page_description(documentation)
+    path = f'/documentation/{documentation.name}' if documentation.name else '/documentation'
+
+    breadcrumbs = [('Home', '/'), ('Documentation', '/documentation')]
+    if documentation.name:
+        if documentation.back_link is not None:
+            from .content import registry
+            parent = registry.get(documentation.back_link)
+            if parent and parent.title:
+                parent_title = parent.title.replace('*', '')
+                breadcrumbs.append((parent_title, f'/documentation/{documentation.back_link}'))
+        breadcrumbs.append((title, path))
+    apply_page_seo(title=seo_title, description=description, path=path,
+                   breadcrumbs=breadcrumbs, og_type='article')
+
     def render_content():
         first_demo_seen = False
         section_heading(documentation.subtitle or '', documentation.heading)

website/examples_page.py

@@ -3,9 +3,16 @@
 from .components.examples_section import example_card
 from .design import section_heading
 from .examples import examples
+from .seo import apply_page_seo
 
 
 def create() -> None:
+    title = 'NiceGUI Examples - Python UI Code Samples and Demos'
+    description = ('Browse in-depth NiceGUI examples including authentication, chat apps, todo lists, and more. '
+                   'See real Python GUI code with live demos.')
+    ui.page_title(title)
+    apply_page_seo(title=title, description=description, path='/examples',
+                   breadcrumbs=[('Home', '/'), ('Examples', '/examples')])
     with ui.column().classes('w-full p-8 lg:p-16 max-w-[1600px] mx-auto'):
         section_heading('In-depth examples', 'Pick your *solution*')
         with ui.grid().classes('w-full grid-cols-1 sm:grid-cols-2 xl:grid-cols-3 gap-4'):