Convert & to & as a Characters token

This fixes a problem in LinkifyFilter when using it with the Cleaner where
the Cleaner sets up the tokenizer to not consume entities. So character
entities end up in their own Entity tokens and Linkifyfilter can't match
links that cross token boundaries. If there's a &, then LinkifyFilter
won't match across that.

This fixes that by converting & to & in the sanitizer when it's pulling out
entities and putting them in separate Entity tokens. The & Characters tokens
will get merged by BleachSanitizerFilter.__iter__ and & will get converted
back to & in the serialier.

Fixes #422

Author: willkg

bleach/sanitizer.py

@@ -395,7 +395,18 @@ def sanitize_characters(self, token):
             if part.startswith('&'):
                 entity = html5lib_shim.match_entity(part)
                 if entity is not None:
-                    new_tokens.append({'type': 'Entity', 'name': entity})
+                    if entity == 'amp':
+                        # LinkifyFilter can't match urls across token boundaries
+                        # which is problematic with & since that shows up in
+                        # querystrings all the time. This special-cases &
+                        # and converts it to a & and sticks it in as a
+                        # Characters token. It'll get merged with surrounding
+                        # tokens in the BleachSanitizerfilter.__iter__ and
+                        # escaped in the serializer.
+                        new_tokens.append({'type': 'Characters', 'data': '&'})
+                    else:
+                        new_tokens.append({'type': 'Entity', 'name': entity})
+
                     # Length of the entity plus 2--one for & at the beginning
                     # and and one for ; at the end
                     remainder = part[len(entity) + 2:]

tests/test_linkify.py

@@ -694,6 +694,10 @@ def test_only_text_is_linkified(self):
         'http://example.com?b=1&c=2',
         '<a href="http://example.com?b=1&amp;c=2">http://example.com?b=1&amp;c=2</a>'
     ),
+    (
+        'http://example.com?b=1&amp;c=2',
+        '<a href="http://example.com?b=1&amp;c=2">http://example.com?b=1&amp;c=2</a>'
+    ),
     (
         'link: https://example.com/watch#anchor',
         'link: <a href="https://example.com/watch#anchor">https://example.com/watch#anchor</a>'