fix(crawler): refuse crawl targets that are not publicly routable

surfsense_backend/app/proprietary/web_crawler/connector.py

@@ -43,7 +43,12 @@
 )
 from app.proprietary.web_crawler.url_policy import extract_link_records
 from app.utils.captcha import captcha_enabled, get_captcha_config
-from app.utils.crawl import BlockType, classify_block, extract_contacts
+from app.utils.crawl import (
+    BlockType,
+    classify_block,
+    extract_contacts,
+    is_publicly_routable,
+)
 from app.utils.proxy import get_proxy_url, is_pool_backed
 
 logger = logging.getLogger(__name__)
@@ -240,6 +245,18 @@ async def crawl_url(self, url: str) -> CrawlOutcome:
                     block_type=block_state["block_type"],
                 )
 
+            # ``validators.url`` only judges the spelling; it passes
+            # ``http://127.0.0.1:8000/`` and the cloud metadata endpoint alike.
+            # Resolving before any tier runs is what keeps those from being
+            # fetched from inside the backend's own network. DNS blocks, so it
+            # is offloaded the same way the browser tiers below are.
+            if not await asyncio.to_thread(is_publicly_routable, url):
+                return CrawlOutcome(
+                    status=CrawlOutcomeStatus.FAILED,
+                    error=f"Restricted URL (host is not publicly routable): {url}",
+                    block_type=block_state["block_type"],
+                )
+
             errors: list[str] = []
             # True once any tier fetched the page but extraction yielded nothing
             # (distinguishes EMPTY from FAILED, where every tier raised/was

surfsense_backend/app/utils/crawl/__init__.py

@@ -12,11 +12,13 @@
 
 from app.utils.crawl.classifier import BlockType, classify_block
 from app.utils.crawl.contacts import Contacts, extract_contacts, is_social_host
+from app.utils.crawl.net_guard import is_publicly_routable
 
 __all__ = [
     "BlockType",
     "Contacts",
     "classify_block",
     "extract_contacts",
+    "is_publicly_routable",
     "is_social_host",
 ]

surfsense_backend/app/utils/crawl/net_guard.py

@@ -0,0 +1,94 @@
+"""Destination guard — refuses crawl targets that are not publicly routable.
+
+``validators.url`` answers a syntactic question: is this a well-formed URL. It
+says nothing about *where* the URL points, so ``http://127.0.0.1:8000/`` and
+``http://169.254.169.254/latest/meta-data/`` (the cloud metadata endpoint) both
+pass it and are then fetched from inside the backend's network namespace. This
+module answers the other question — does the host resolve only to addresses the
+public internet can reach — and is the gate the crawler consults before any
+tier runs.
+
+Two details decide correctness here:
+
+* ``::ffff:127.0.0.1`` reports ``is_loopback == False``. The address flags only
+  read true on the mapped IPv4 form, so the mapping is unwrapped before the
+  address is judged; reading the flags off the IPv6 object admits loopback
+  under its v6 spelling.
+* ``is_global`` is the right primitive rather than a union of
+  ``is_private``/``is_loopback``/``is_link_local``/``is_reserved``: that union
+  admits carrier-grade NAT (``100.64.0.0/10``, RFC 6598), which is not
+  publicly routable. ``is_global`` does report ``True`` for multicast, which is
+  therefore excluded separately.
+
+Resolution failures refuse. A name that cannot be resolved is not a name that
+can be cleared, and failing open here would make an unreachable DNS server a
+way past the guard.
+
+This narrows the reachable surface; it is not a defence against DNS rebinding.
+The fetcher resolves the host again on its own, so a name that answers
+differently between this check and that fetch is out of scope for a
+resolve-then-fetch guard.
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import socket
+from urllib.parse import urlsplit
+
+_ALLOWED_SCHEMES = frozenset({"http", "https"})
+
+
+def _resolve_host(host: str) -> list[str]:
+    """Every address ``host`` resolves to, across both families."""
+    return sorted(
+        {
+            info[4][0]
+            for info in socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP)
+        }
+    )
+
+
+def _is_public_address(raw: str) -> bool:
+    """True when ``raw`` parses as an address the public internet can route to."""
+    try:
+        address = ipaddress.ip_address(raw)
+    except ValueError:
+        # Includes scoped forms such as ``fe80::1%eth0``, which are link-local
+        # anyway; anything unparseable is refused rather than guessed at.
+        return False
+    mapped = getattr(address, "ipv4_mapped", None)
+    if mapped is not None:
+        address = mapped
+    return address.is_global and not address.is_multicast
+
+
+def is_publicly_routable(url: str) -> bool:
+    """True only when every address ``url``'s host resolves to is public.
+
+    A host with both a public and a private answer is refused: the fetcher
+    resolves independently and may pick either one.
+    """
+    parts = urlsplit(url)
+    if parts.scheme not in _ALLOWED_SCHEMES:
+        return False
+    host = parts.hostname
+    if not host:
+        return False
+
+    # An address literal needs no resolver, and asking one would let a
+    # nameserver answer for a host the URL already pinned.
+    try:
+        ipaddress.ip_address(host)
+    except ValueError:
+        pass
+    else:
+        return _is_public_address(host)
+
+    try:
+        addresses = _resolve_host(host)
+    except (OSError, UnicodeError):
+        return False
+    if not addresses:
+        return False
+    return all(_is_public_address(address) for address in addresses)

surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py

@@ -12,11 +12,22 @@
     WebCrawlerConnector,
     connector as connector_module,
 )
-from app.utils.crawl import BlockType
+from app.utils.crawl import BlockType, net_guard
 
 pytestmark = pytest.mark.unit
 
 
+@pytest.fixture(autouse=True)
+def _stub_destination_resolver(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Keep the destination guard hermetic.
+
+    ``crawl_url`` now resolves the target host before running a tier. These
+    are unit tests, so the resolver answers with a fixed public address
+    instead of whatever DNS says about ``example.com`` today.
+    """
+    monkeypatch.setattr(net_guard, "_resolve_host", lambda _host: ["93.184.216.34"])
+
+
 def _result(tier: str) -> dict:
     return {
         "content": "hello world",
@@ -497,3 +508,48 @@ def test_build_result_ok_on_real_content() -> None:
     )
 
     assert block_state["block_type"] is BlockType.OK
+
+
+async def test_restricted_url_is_failed_before_any_tier(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A host that is not publicly routable never reaches a fetch tier.
+
+    ``validators.url`` passes the cloud metadata endpoint, so without the
+    destination guard this URL is fetched from inside the backend's network.
+    """
+    crawler = WebCrawlerConnector()
+    tiers: list[str] = []
+
+    async def _record_static(_url: str, *_args) -> None:
+        tiers.append("static")
+        return None
+
+    async def _record_dynamic(_url: str, *_args) -> None:
+        tiers.append("dynamic")
+        return None
+
+    async def _record_stealthy(_url: str, *_args) -> None:
+        tiers.append("stealthy")
+        return None
+
+    monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _record_static)
+    monkeypatch.setattr(crawler, "_crawl_with_dynamic", _record_dynamic)
+    monkeypatch.setattr(crawler, "_crawl_with_stealthy", _record_stealthy)
+
+    outcome = await crawler.crawl_url("http://169.254.169.254/latest/meta-data/")
+
+    assert outcome.status is CrawlOutcomeStatus.FAILED
+    assert outcome.result is None
+    assert "Restricted URL" in (outcome.error or "")
+    assert tiers == []
+
+
+async def test_restricted_url_is_reported_apart_from_a_malformed_one() -> None:
+    """The two refusals are different diagnoses and must not share a message."""
+    restricted = await WebCrawlerConnector().crawl_url("http://127.0.0.1:8000/health")
+    malformed = await WebCrawlerConnector().crawl_url("not a url")
+
+    assert "Restricted URL" in (restricted.error or "")
+    assert "Invalid URL" in (malformed.error or "")
+    assert (malformed.error or "") != (restricted.error or "")

surfsense_backend/tests/unit/utils/crawl/test_net_guard.py

@@ -0,0 +1,125 @@
+"""``is_publicly_routable`` behavior: refuse URLs whose host is not public.
+
+The literal-address cases need no resolver at all, so they exercise the real
+code end to end; only the hostname cases stub ``_resolve_host``, because a unit
+test cannot depend on what DNS answers.
+"""
+
+from __future__ import annotations
+
+import socket
+
+import pytest
+
+from app.utils.crawl import is_publicly_routable, net_guard
+
+pytestmark = pytest.mark.unit
+
+
+def test_public_literal_is_allowed() -> None:
+    assert is_publicly_routable("https://8.8.8.8/") is True
+
+
+def test_loopback_literal_is_refused() -> None:
+    assert is_publicly_routable("http://127.0.0.1:8000/health") is False
+
+
+def test_cloud_metadata_literal_is_refused() -> None:
+    """169.254.169.254 is the AWS/GCP/Azure credential endpoint."""
+    assert is_publicly_routable("http://169.254.169.254/latest/meta-data/") is False
+
+
+@pytest.mark.parametrize(
+    "url",
+    [
+        "http://10.0.0.1/",
+        "http://172.16.0.1/",
+        "http://192.168.1.1/",
+        "http://0.0.0.0/",
+    ],
+)
+def test_private_and_unspecified_literals_are_refused(url: str) -> None:
+    assert is_publicly_routable(url) is False
+
+
+def test_ipv6_loopback_literal_is_refused() -> None:
+    assert is_publicly_routable("http://[::1]:8000/") is False
+
+
+def test_ipv4_mapped_ipv6_literal_is_refused() -> None:
+    """``::ffff:127.0.0.1`` reports ``is_loopback == False``; only the mapped
+    IPv4 form does, so a guard that reads the flags off the IPv6 object alone
+    lets loopback through under its v6 spelling."""
+    assert is_publicly_routable("http://[::ffff:127.0.0.1]/") is False
+
+
+def test_carrier_grade_nat_literal_is_refused() -> None:
+    """100.64.0.0/10 (RFC 6598) is none of private/loopback/link-local/reserved
+    /multicast, so a guard built from those five flags admits it — but it is
+    carrier-internal and not publicly routable."""
+    assert is_publicly_routable("http://100.64.0.1/") is False
+
+
+def test_multicast_literal_is_refused() -> None:
+    """224.0.0.1 reports ``is_global == True``, so the routability check alone
+    is not enough to exclude it."""
+    assert is_publicly_routable("http://224.0.0.1/") is False
+
+
+@pytest.mark.parametrize(
+    "url",
+    ["file:///etc/passwd", "gopher://127.0.0.1/", "ftp://example.com/", "not a url"],
+)
+def test_non_http_scheme_is_refused(url: str) -> None:
+    assert is_publicly_routable(url) is False
+
+
+def test_hostname_resolving_to_a_private_address_is_refused(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(net_guard, "_resolve_host", lambda _host: ["10.1.2.3"])
+
+    assert is_publicly_routable("https://internal.example.com/") is False
+
+
+def test_hostname_is_refused_when_any_answer_is_private(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A host with both a public and a private answer must not be crawled: the
+    fetcher resolves independently and may pick either."""
+    monkeypatch.setattr(
+        net_guard, "_resolve_host", lambda _host: ["93.184.216.34", "192.168.0.7"]
+    )
+
+    assert is_publicly_routable("https://split-horizon.example.com/") is False
+
+
+def test_hostname_resolving_only_to_public_addresses_is_allowed(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(
+        net_guard,
+        "_resolve_host",
+        lambda _host: ["93.184.216.34", "2606:2800:220:1:248:1893:25c8:1946"],
+    )
+
+    assert is_publicly_routable("https://example.com/page") is True
+
+
+def test_unresolvable_hostname_is_refused(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Fail closed: a name we cannot resolve is not a name we can clear."""
+
+    def _boom(_host: str) -> list[str]:
+        raise socket.gaierror("Name or service not known")
+
+    monkeypatch.setattr(net_guard, "_resolve_host", _boom)
+
+    assert is_publicly_routable("https://does-not-exist.example/") is False
+
+
+def test_hostname_resolving_to_nothing_is_refused(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(net_guard, "_resolve_host", lambda _host: [])
+
+    assert is_publicly_routable("https://empty.example/") is False