• AT&T is in the s&p 500
", - u"\u2022 AT&T is in the s&p 500
" + "• AT&T is in the s&p 500
", + "\u2022 AT&T is in the s&p 500
" ) def test_apos_entity(self): self.assertSoupEquals( - u"Bob's Bar
", - u"Bob's Bar
", + "Bob's Bar
", + "Bob's Bar
", ) def test_entities_in_foreign_document_encoding(self): @@ -564,17 +564,17 @@ # characters. markup = "“Hello” -☃
" soup = self.soup(markup) --- bs4/builder/__init__.py (original) +++ bs4/builder/__init__.py (refactored) @@ -300,13 +300,13 @@ universal = self.cdata_list_attributes.get('*', []) tag_specific = self.cdata_list_attributes.get( tag_name.lower(), None) - for attr in attrs.keys(): + for attr in list(attrs.keys()): if attr in universal or (tag_specific and attr in tag_specific): # We have a "class"-type attribute whose string # value is a whitespace-separated list of # values. Split it into a list. value = attrs[attr] - if isinstance(value, basestring): + if isinstance(value, str): values = nonwhitespace_re.findall(value) else: # html5lib sometimes calls setAttributes twice @@ -496,7 +496,7 @@ """ if isinstance(message_or_exception, Exception): e = message_or_exception - message_or_exception = "%s: %s" % (e.__class__.__name__, unicode(e)) + message_or_exception = "%s: %s" % (e.__class__.__name__, str(e)) super(ParserRejectedMarkup, self).__init__(message_or_exception) # Builders are registered in reverse order of priority, so that custom --- bs4/builder/_html5lib.py (orRefactoringTool: Refactored bs4/builder/_htmlparser.py iginal) +++ bs4/builder/_html5lib.py (refactored) @@ -33,7 +33,7 @@ # Pre-0.99999999 from html5lib.treebuilders import _base as treebuilder_base new_html5lib = False -except ImportError, e: +except ImportError as e: # 0.99999999 and up from html5lib.treebuilders import base as treebuilder_base new_html5lib = True @@ -79,7 +79,7 @@ parser = html5lib.HTMLParser(tree=self.create_treebuilder) self.underlying_builder.parser = parser extra_kwargs = dict() - if not isinstance(markup, unicode): + if not isinstance(markup, str): if new_html5lib: extra_kwargs['override_encoding'] = self.user_specified_encoding else: @@ -87,13 +87,13 @@ doc = parser.parse(markup, **extra_kwargs) # Set the character encoding detected by the tokenizer. - if isinstance(markup, unicode): + if isinstance(markup, str): # We need to special-case this because html5lib sets # charEncoding to UTF-8 if it gets Unicode input. doc.original_encoding = None else: original_encoding = parser.tokenizer.stream.charEncoding[0] - if not isinstance(original_encoding, basestring): + if not isinstance(original_encoding, str): # In 0.99999999 and up, the encoding is an html5lib # Encoding object. We want to use a string for compatibility # with other tree builders. @@ -110,7 +110,7 @@ def test_fragment_to_document(self, fragment): """See `TreeBuilder`.""" - return u'%s' % fragment + return '%s' % fragment class TreeBuilderForHtml5lib(treebuilder_base.TreeBuilder): @@ -217,7 +217,7 @@ rv.append("|%s<%s>" % (' ' * indent, name)) if element.attrs: attributes = [] - for name, value in element.attrs.items(): + for name, value in list(element.attrs.items()): if isinstance(name, NamespacedAttribute): name = "%s %s" % (prefixes[name.namespace], name.name) if isinstance(value, list): @@ -272,7 +272,7 @@ def appendChild(self, node): string_child = child = None - if isinstance(node, basestring): + if isinstance(node, str): # Some other piece of code decided to pass in a string # instead of creating a TextElement object to contain the # string. @@ -289,7 +289,7 @@ child = node.element node.parent = self - if not isinstance(child, basestring) and child.parent is not None: + if not isinstance(child, str) and child.parent is not None: node.element.extract() if (string_child is not None and self.element.contents @@ -302,7 +302,7 @@ old_element.replace_with(new_element) self.soup._most_recent_element = new_element else: - if isinstance(node, basestring): + if isinstance(node, str): # Create a brand new NavigableString from this string. child = self.soup.new_string(node) @@ -340,7 +340,7 @@ self.soup.builder._replace_cdata_list_attribute_values( self.name, attributes) - for name, value in attributes.items(): + for name, value in list(attributes.items()): self.element[name] = value # The attributes may contain variables that need substitution. --- bs4/builder/_htmlparser.py (original) +++ bs4/builder/_htmlparser.py (refactored) @@ -8,11 +8,11 @@ 'HTMLParserTreeBuilder', ] -from HTMLParser import HTMLParser +from html.parser import HTMLParser try: - from HTMLParser import HTMLParseError -except ImportError, e: + from html.parser import HTMLParseError +except ImportError as e: # HTMLParseError isRefactoringTool: Refactored bs4/builder/_lxml.py removed in Python 3.5. Since it can never be # thrown in 3.5, we can just define our own class as a placeholder. class HTMLParseError(Exception): @@ -219,14 +219,14 @@ continue try: data = bytearray([real_name]).decode(encoding) - except UnicodeDecodeError, e: + except UnicodeDecodeError as e: pass if not data: try: - data = unichr(real_name) - except (ValueError, OverflowError), e: + data = chr(real_name) + except (ValueError, OverflowError) as e: pass - data = data or u"\N{REPLACEMENT CHARACTER}" + data = data or "\N{REPLACEMENT CHARACTER}" self.handle_data(data) def handle_entityref(self, name): @@ -353,7 +353,7 @@ document to Unicode and parsing it. Each strategy will be tried in turn. """ - if isinstance(markup, unicode): + if isinstance(markup, str): # Parse Unicode as-is. yield (markup, None, None, False) return @@ -376,7 +376,7 @@ try: parser.feed(markup) parser.close() - except HTMLParseError, e: + except HTMLParseError as e: warnings.warn(RuntimeWarning( "Python's built-in HTMLParser cannot parse the given document. This is not a bug in Beautiful Soup. The best solution is to install an external parser (lxml or html5lib), and use Beautiful Soup with that parser. See http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser for help.")) raise e --- bs4/builder/_lxml.py (original) +++ bs4/builder/_lxml.py (refactored) @@ -8,11 +8,11 @@ try: from collections.abc import Callable # Python 3.6 -except ImportError , e: +except ImportError as e: from collections import Callable from io import BytesIO -from StringIO import StringIO +from io import StringIO from lxml import etree from bs4.element import ( Comment, @@ -35,7 +35,7 @@ def _invert(d): "Invert a dictionary." - return dict((v,k) for k, v in d.items()) + return dict((v,k) for k, v in list(d.items())) class LXMLTreeBuilderForXML(TreeBuilder): DEFAULT_PARSER_CLASS = etree.XMLParser @@ -81,7 +81,7 @@ :param mapping: A dictionary mapping namespace prefixes to URIs. """ - for key, value in mapping.items(): + for key, value in list(mapping.items()): if key and key not in self.soup._namespaces: # Let the BeautifulSoup object know about a new namespace. # If there are multiple namespaces defined with the same @@ -169,12 +169,12 @@ else: self.processing_instruction_class = XMLProcessingInstruction - if isinstance(markup, unicode): + if isinstance(markup, str): # We were given Unicode. Maybe lxml can parse Unicode on # this system? yield markup, None, document_declared_encoding, False - if isinstance(markup, unicode): + if isinstance(markup, str): # No, apparently not. Convert the Unicode to UTF-8 and # tell lxml to parse it as UTF-8. yield (markup.encode("utf8"), "utf8", @@ -189,7 +189,7 @@ def feed(self, markup): if isinstance(markup, bytes): markup = BytesIO(markup) - elif isinstance(markup, unicode): + elif isinstance(markup, str): markup = StringIO(markup) # Call feed() at least once, even if the markup is empty, @@ -204,7 +204,7 @@ if len(data) != 0: self.parser.feed(data) self.parser.close() - except (UnicodeDecodeError, LookupError, etree.ParserError), e: + except (UnicodeDecodeError, LookupError, etree.ParserError) as e: raise ParserRejectedMarkup(e) def close(self): @@ -233,7 +233,7 @@ # Also treat the namespace mapping as RefactoringTool: No changes to bs4/tests/__init__.py RefactoringTool: No changes to bs4/tests/test_builder_registry.py RefactoringTool: No changes to bs4/tests/test_docs.py RefactoringTool: Refactored bs4/tests/test_html5lib.py RefactoringTool: Refactored bs4/tests/test_htmlparser.py a set of attributes on the # tag, so we can recreate it later. attrs = attrs.copy() - for prefix, namespace in nsmap.items(): + for prefix, namespace in list(nsmap.items()): attribute = NamespacedAttribute( "xmlns", prefix, "http://www.w3.org/2000/xmlns/") attrs[attribute] = namespace @@ -242,7 +242,7 @@ # from lxml with namespaces attached to their names, and # turn then into NamespacedAttribute objects. new_attrs = {} - for attr, value in attrs.items(): + for attr, value in list(attrs.items()): namespace, attr = self._getNsTag(attr) if namespace is None: new_attrs[attr] = value @@ -302,7 +302,7 @@ def test_fragment_to_document(self, fragment): """See `TreeBuilder`.""" - return u'\n%s' % fragment + return '\n%s' % fragment class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML): @@ -323,10 +323,10 @@ self.parser = self.parser_for(encoding) self.parser.feed(markup) self.parser.close() - except (UnicodeDecodeError, LookupError, etree.ParserError), e: + except (UnicodeDecodeError, LookupError, etree.ParserError) as e: raise ParserRejectedMarkup(e) def test_fragment_to_document(self, fragment): """See `TreeBuilder`.""" - return u'%s' % fragment + return '%s' % fragment --- bs4/tests/test_html5lib.py (original) +++ bs4/tests/test_html5lib.py (refactored) @@ -5,7 +5,7 @@ try: from bs4.builder import HTML5TreeBuilder HTML5LIB_PRESENT = True -except ImportError, e: +except ImportError as e: HTML5LIB_PRESENT = False from bs4.element import SoupStrainer from bs4.testing import ( @@ -74,14 +74,14 @@ def test_reparented_markup(self): markup = 'foo
\n' soup = self.soup(markup) - self.assertEqual(u"foo
\n", soup.body.decode()) + self.assertEqual("foo
\n", soup.body.decode()) self.assertEqual(2, len(soup.find_all('p'))) def test_reparented_markup_ends_with_whitespace(self): markup = 'foo
\n\n' soup = self.soup(markup) - self.assertEqual(u"foo
\n\n", soup.body.decode()) + self.assertEqual("foo
\n\n", soup.body.decode()) self.assertEqual(2, len(soup.find_all('p'))) def test_reparented_markup_containing_identical_whitespace_nodes(self): @@ -127,7 +127,7 @@ def test_foster_parenting(self): markup = b"""| A"""
soup = self.soup(markup)
- self.assertEqual(u"A |