Lightfuzz Overhaul (False-positive reductions, False-negative reductions, many new techniques)

README.md

@@ -387,7 +387,7 @@ For details, see [Configuration](https://www.blacklanternsecurity.com/bbot/Stabl
         - [List of Modules](https://www.blacklanternsecurity.com/bbot/Stable/modules/list_of_modules)
         - [Nuclei](https://www.blacklanternsecurity.com/bbot/Stable/modules/nuclei)
         - [Custom YARA Rules](https://www.blacklanternsecurity.com/bbot/Stable/modules/custom_yara_rules)
-        - [Lightfuzz](https://www.blacklanternsecurity.com/bbot/Stable/modules/lightfuzz)
+        - [Lightfuzz (DAST)](https://www.blacklanternsecurity.com/bbot/Stable/modules/lightfuzz)
     - **Misc**
         - [Contribution](https://www.blacklanternsecurity.com/bbot/Stable/contribution)
         - [Release History](https://www.blacklanternsecurity.com/bbot/Stable/release_history)

bbot/core/event/base.py

@@ -685,7 +685,7 @@ def parent(self, parent):
                 self.web_spider_distance = getattr(parent, "web_spider_distance", 0)
                 event_has_url = getattr(self, "parsed_url", None) is not None
                 for t in parent.tags:
-                    if t in ("affiliate",):
+                    if t in ("affiliate", "from-lightfuzz"):
                         self.add_tag(t)
                     elif t.startswith("mutation-"):
                         self.add_tag(t)
@@ -1135,10 +1135,20 @@ def sanitize_data(self, data):
 
 
 class DictEvent(BaseEvent):
+    __slots__ = ["url_extension"]
+
     def sanitize_data(self, data):
         url = data.get("url", "")
         if url:
             self.parsed_url = self.validators.validate_url_parsed(url)
+            # extract url_extension from any dict event with a URL
+            url_path = self.parsed_url.path
+            if url_path:
+                parsed_path_lower = str(url_path).lower()
+                extension = get_file_extension(parsed_path_lower)
+                if extension:
+                    self.url_extension = extension
+                    self.add_tag(f"extension-{extension}")
         return data
 
     def _data_load(self, data):
@@ -1366,7 +1376,6 @@ class URL_UNVERIFIED(DictHostEvent):
 
     __slots__ = [
         "web_spider_distance",
-        "url_extension",
         "num_redirects",
     ]
 
@@ -1567,6 +1576,7 @@ def _minimize(self):
             self._data.pop("original_value", None)
             self._data.pop("additional_params", None)
             self._data.pop("assigned_cookies", None)
+            self._data.pop("same_param_values", None)
 
     @property
     def children(self):

bbot/core/helpers/diff.py

@@ -21,6 +21,7 @@ def __init__(
         headers=None,
         cookies=None,
         timeout=10,
+        on_baseline_ready=None,
     ):
         self.parent_helper = parent_helper
         self.baseline_url = baseline_url
@@ -33,6 +34,8 @@ def __init__(
         self.headers = headers
         self.cookies = cookies
         self.timeout = 10
+        # Optional async callback fired once with baseline_1 after the baseline is established.
+        self.on_baseline_ready = on_baseline_ready
 
     @staticmethod
     def merge_dictionaries(headers1, headers2):
@@ -130,6 +133,12 @@ async def _baseline(self):
             self.baseline_ignore_headers += [x.lower() for x in dynamic_headers]
             self._baselined = True
 
+            if self.on_baseline_ready is not None:
+                try:
+                    await self.on_baseline_ready(baseline_1)
+                except Exception as e:
+                    log.debug(f"on_baseline_ready callback raised: {e}")
+
     def gen_cache_buster(self):
         return {self.parent_helper.rand_string(6): "1"}
 

bbot/core/helpers/helper.py

@@ -157,6 +157,7 @@ def http_compare(
         data=None,
         json=None,
         timeout=10,
+        on_baseline_ready=None,
     ):
         return HttpCompare(
             url,
@@ -169,6 +170,7 @@ def http_compare(
             method=method,
             data=data,
             json=json,
+            on_baseline_ready=on_baseline_ready,
         )
 
     def temp_filename(self, extension=None):

bbot/core/helpers/misc.py

@@ -2754,6 +2754,12 @@ def get_waf_strings():
     return [
         "The requested URL was rejected",
         "This content has been blocked",
+        "You don't have permission to access ",
+        "The URL you requested has been blocked",
+        "Request unsuccessful. Incapsula incident",
+        "Access Denied - Sucuri Website Firewall",
+        "Attention Required! | Cloudflare",
+        "Microsoft-Azure-Application-Gateway",
     ]
 
 

bbot/core/helpers/regexes.py

@@ -157,7 +157,14 @@
     re.DOTALL,
 )
 post_form_regex_noaction = re.compile(
-    r"<form[^>]*(?:\baction=[\"']?([^\s\"'<>]+)[\"']?)?[^>]*\bmethod=[\"']?[pP][oO][sS][tT][\"']?[^>]*>([\s\S]*?)<\/form>",
+    # Negative lookahead rejects forms that already carry action= (those are
+    # handled by post_form_regex / post_form_regex2 with the real action URL).
+    # Without it, the action group's `?` makes this regex over-eagerly match
+    # forms with action, capturing '' and emitting a phantom WEB_PARAMETER
+    # whose url falls back to event.url — racing the legitimate emission.
+    # The empty `()` keeps findall's tuple shape (form_action, form_content)
+    # consistent with the other form regexes consumed by GetForm.extract().
+    r"<form(?![^>]*\baction=)()[^>]*\bmethod=[\"']?[pP][oO][sS][tT][\"']?[^>]*>([\s\S]*?)<\/form>",
     re.DOTALL,
 )
 generic_form_regex = re.compile(
@@ -166,9 +173,15 @@
 )
 
 select_tag_regex = re.compile(
-    r"<select[^>]+?name=[\"\']?([_\-\.\w]+)[\"\']?[^>]*>(?:\s*<option[^>]*?value=[\"\']?([_\.\-\w]*)[\"\']?[^>]*>)?",
+    r"<select[^>]+?\bname=[\"\']?([_\-\.\w]+)[\"\']?[^>]*>((?:\s*<option\b[^>]*>(?:[^<]*(?:</option>)?)?\s*)*)",
     re.IGNORECASE | re.DOTALL,
 )
+option_tag_regex = re.compile(r"<option\b([^>]*)>", re.IGNORECASE)
+option_value_regex = re.compile(
+    r"""\bvalue\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>'"]+))""",
+    re.IGNORECASE,
+)
+option_selected_regex = re.compile(r"(?<![\-\w])selected(?![\-\w])", re.IGNORECASE)
 
 textarea_tag_regex = re.compile(
     r"<textarea[^>]*?\sname=[\"\']?([\-\._=+\/\w]+)[\"\']?[^>]*?\svalue=[\"\']?([:%\-\._=+\/\w]*)[\"\']?[^>]*?>"

bbot/core/helpers/web/response_event.py

@@ -0,0 +1,87 @@
+"""Build HTTP_RESPONSE event data dicts from blasthttp.Response objects.
+
+Materializes hash and cert_info eagerly, so only call when about to emit an
+HTTP_RESPONSE event (otherwise blasthttp.Response stays lazy).
+"""
+
+import re
+from urllib.parse import urlparse
+
+
+_TITLE_REGEX = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
+
+
+def response_to_event_dict(response, url_input, method="GET"):
+    """Convert a blasthttp.Response into the dict format HTTP_RESPONSE events expect.
+
+    Args:
+        response: a blasthttp.Response (native 0.5+ object with .hash, .cert_info)
+        url_input: the original scan-target identifier (host:port or full URL)
+        method: HTTP method used to fetch the response
+
+    Returns:
+        dict with keys url, input, status_code, method, path, host, raw_header,
+        header, content_type, content_length, title, body, location, hash. Adds
+        cert_info when the response carried a TLS certificate.
+    """
+    parsed = urlparse(response.url)
+    path = parsed.path or "/"
+
+    status_line = f"HTTP/1.1 {response.status} \r\n"
+    raw_header = f"{status_line}{response.raw_headers}\r\n\r\n"
+
+    header_dict = {}
+    for k, v in response.headers.items():
+        key = k.lower().replace("-", "_")
+        if key in header_dict:
+            header_dict[key] += f", {v}"
+        else:
+            header_dict[key] = v
+
+    content_type = header_dict.get("content_type", "")
+    content_length = int(header_dict.get("content_length", len(response.body_bytes)))
+    location = header_dict.get("location", "")
+
+    body = response.body
+    title = ""
+    title_match = _TITLE_REGEX.search(body)
+    if title_match:
+        title = title_match.group(1).strip()
+
+    j = {
+        "url": response.url,
+        "input": url_input,
+        "status_code": response.status,
+        "method": method,
+        "path": path,
+        "host": parsed.hostname or "",
+        "raw_header": raw_header,
+        "header": header_dict,
+        "content_type": content_type,
+        "content_length": content_length,
+        "title": title,
+        "body": body,
+        "location": location,
+        "hash": {
+            "body_md5": response.hash.body_md5,
+            "body_mmh3": response.hash.body_mmh3,
+            "body_sha256": response.hash.body_sha256,
+            "header_md5": response.hash.header_md5,
+            "header_mmh3": response.hash.header_mmh3,
+            "header_sha256": response.hash.header_sha256,
+        },
+    }
+
+    ci = response.cert_info
+    if ci is not None:
+        j["cert_info"] = {
+            "common_name": ci.common_name,
+            "sans": ci.sans,
+            "emails": ci.emails,
+            "issuer": ci.issuer,
+            "not_before": ci.not_before,
+            "not_after": ci.not_after,
+            "fingerprint_sha256": ci.fingerprint_sha256,
+        }
+
+    return j

bbot/defaults.yml

@@ -287,8 +287,23 @@ parameter_blacklist:
   - cf_clearance
   - _abck
   - bm_sz
+  - bm_sv
   - ak_bmsc
   - f5_cspm
+  # CSRF / Anti-Forgery tokens
+  - authenticity_token
+  - csrfmiddlewaretoken
+  - __RequestVerificationToken
+  - antiforgerytoken
+  - __csrf_magic
+  - _wpnonce
+  # ASP.NET session/identity cookies
+  - .ASPXANONYMOUS
+  - .ASPXAUTH
+  # PKCE (Proof Key for Code Exchange)
+  - code_verifier
+  - code_challenge
+  # Analytics
   - _ga
   - _gid
   - _gat

bbot/modules/base.py

@@ -868,13 +868,13 @@ def _event_precheck(self, event):
         if self.target_only and "target" not in event.tags:
             return False, "it did not meet target_only filter criteria"
 
-        # limit js URLs to modules that opt in to receive them
-        if (not self.accept_url_special) and event.type.startswith("URL"):
+        # limit events with special URL extensions (e.g. .js) to modules that opt in
+        if not self.accept_url_special:
             extension = getattr(event, "url_extension", "")
             if extension in self.scan.url_extension_special:
                 return (
                     False,
-                    f"it is a special URL (extension {extension}) but the module does not opt in to receive special URLs",
+                    f"it has a special URL extension ({extension}) but the module does not opt in to receive special URLs",
                 )
 
         return True, "precheck succeeded"

bbot/modules/bypass403.py

@@ -1,5 +1,6 @@
 from bbot.errors import HttpCompareError
 from bbot.modules.base import BaseModule
+from bbot.core.helpers.misc import get_waf_strings
 
 """
 Port of https://github.com/iamj0ker/bypass-403/ and https://portswigger.net/bappstore/444407b96d9c4de0adb7aed89e826122
@@ -80,6 +81,10 @@ class bypass403(BaseModule):
     meta = {"description": "Check 403 pages for common bypasses", "created_date": "2022-07-05", "author": "@liquidsec"}
     in_scope_only = True
 
+    async def setup(self):
+        self.waf_yara_rules = self.helpers.yara.compile_strings(get_waf_strings(), nocase=True)
+        return True
+
     async def do_checks(self, compare_helper, event, collapse_threshold):
         results = set()
         error_count = 0
@@ -105,10 +110,10 @@ async def do_checks(self, compare_helper, event, collapse_threshold):
 
             # In some cases WAFs will respond with a 200 code which causes a false positive
             if subject_response is not None:
-                for waf_string in self.helpers.get_waf_strings():
-                    if waf_string in subject_response.text:
-                        self.debug("Rejecting result based on presence of WAF string")
-                        return
+                waf_matches = await self.helpers.yara.match(self.waf_yara_rules, subject_response.text)
+                if waf_matches:
+                    self.debug("Rejecting result based on presence of WAF string")
+                    return
 
             if match is False:
                 if str(subject_response.status_code)[0] != "4":

bbot/modules/http.py

@@ -1,10 +1,10 @@
-import re
 from http.cookies import SimpleCookie
 from urllib.parse import urlparse
 
 import blasthttp
 
 from bbot.core.helpers.web.web import iter_batch_results
+from bbot.core.helpers.web.response_event import response_to_event_dict
 from bbot.modules.base import BaseModule
 from bbot.core.config.models import BaseModuleConfig, Field
 
@@ -100,75 +100,7 @@ def _build_headers(self):
 
     def _response_to_json(self, url_input, response):
         """Convert a blasthttp Response to a dict for HTTP_RESPONSE events."""
-        parsed = urlparse(response.url)
-        path = parsed.path or "/"
-
-        # Build raw_header string (required by HTTP_RESPONSE validation).
-        # blasthttp already builds the canonical "Name: Value\r\n..." form
-        # — reuse it instead of rebuilding.
-        status_line = f"HTTP/1.1 {response.status} \r\n"
-        raw_header = f"{status_line}{response.raw_headers}\r\n\r\n"
-
-        # Build header dict (lowercase keys, comma-joined for dupes)
-        header_dict = {}
-        for k, v in response.headers.items():
-            key = k.lower().replace("-", "_")
-            if key in header_dict:
-                header_dict[key] += f", {v}"
-            else:
-                header_dict[key] = v
-
-        content_type = header_dict.get("content_type", "")
-        content_length = int(header_dict.get("content_length", len(response.body_bytes)))
-
-        # Location header for redirects (excavate uses event.redirect_location)
-        location = header_dict.get("location", "")
-
-        # Extract title from HTML
-        title = ""
-        body = response.body
-        title_match = re.search(r"<title[^>]*>(.*?)</title>", body, re.IGNORECASE | re.DOTALL)
-        if title_match:
-            title = title_match.group(1).strip()
-
-        j = {
-            "url": response.url,
-            "input": url_input,
-            "status_code": response.status,
-            "method": "GET",
-            "path": path,
-            "host": parsed.hostname or "",
-            "raw_header": raw_header,
-            "header": header_dict,
-            "content_type": content_type,
-            "content_length": content_length,
-            "title": title,
-            "body": body,
-            "location": location,
-            "hash": {
-                "body_md5": response.hash.body_md5,
-                "body_mmh3": response.hash.body_mmh3,
-                "body_sha256": response.hash.body_sha256,
-                "header_md5": response.hash.header_md5,
-                "header_mmh3": response.hash.header_mmh3,
-                "header_sha256": response.hash.header_sha256,
-            },
-        }
-
-        # Include TLS certificate info when available (HTTPS responses)
-        ci = response.cert_info
-        if ci is not None:
-            j["cert_info"] = {
-                "common_name": ci.common_name,
-                "sans": ci.sans,
-                "emails": ci.emails,
-                "issuer": ci.issuer,
-                "not_before": ci.not_before,
-                "not_after": ci.not_after,
-                "fingerprint_sha256": ci.fingerprint_sha256,
-            }
-
-        return j
+        return response_to_event_dict(response, url_input, method="GET")
 
     async def _process_result(self, result, parent_event):
         """Emit URL + HTTP_RESPONSE events for one batch result. Returns True if status was usable."""