0000003e
      ˆ¾‘äÇò 8à7Z”$lM?Éƒh¹ÈÆÅ      00000050
  L   ˆl–Ü4ý( ©¤ýÔâ�jã-Üeå1hß�þB gÓŒÊË‚oùÊƒh¹Éa–Ü4ý( ©¤ýÔâ�jãO\hJbÑ¿Ç      00000069
  8   ˆl–äY>”jeJe@¿q—.JbÑ¿�þY#q¬Ü¬†8åhA}à?Ì„m6ËÊ¿È <[    # encoding: utf-8
import pytest
import logging
import bs4
from bs4 import BeautifulSoup
from bs4.dammit import (
    EntitySubstitution,
    EncodingDetector,
    UnicodeDammit,
)

class TestUnicodeDammit(object):
    """Standalone tests of UnicodeDammit."""

    def test_unicode_input(self):
        markup = "I'm already Unicode! \N{SNOWMAN}"
        dammit = UnicodeDammit(markup)
        assert dammit.unicode_markup == markup

    @pytest.mark.parametrize(
        "smart_quotes_to,expect_converted",
        [(None, "\u2018\u2019\u201c\u201d"),
         ("xml", "&#x2018;&#x2019;&#x201C;&#x201D;"),
         ("html", "&lsquo;&rsquo;&ldquo;&rdquo;"),
         ("ascii", "''" + '""'),
        ]
    )
    def test_smart_quotes_to(self, smart_quotes_to, expect_converted):
        """Verify the functionality of the smart_quotes_to argument
        to the UnicodeDammit constructor."""
        markup = b"<foo>\x91\x92\x93\x94</foo>"
        converted = UnicodeDammit(
            markup, known_definite_encodings=["windows-1252"],
            smart_quotes_to=smart_quotes_to
        ).unicode_markup
        assert converted == "<foo>{}</foo>".format(expect_converted)
        
    def test_detect_utf8(self):
        utf8 = b"Sacr\xc3\xa9 bleu! \xe2\x98\x83"
        dammit = UnicodeDammit(utf8)
        assert dammit.original_encoding.lower() == 'utf-8'
        assert dammit.unicode_markup == 'Sacr\xe9 bleu! \N{SNOWMAN}'

    def test_convert_hebrew(self):
        hebrew = b"\xed\xe5\xec\xf9"
        dammit = UnicodeDammit(hebrew, ["iso-8859-8"])
        assert dammit.original_encoding.lower() == 'iso-8859-8'
        assert dammit.unicode_markup == '\u05dd\u05d5\u05dc\u05e9'

    def test_dont_see_smart_quotes_where_there_are_none(self):
        utf_8 = b"\343\202\261\343\203\274\343\202\277\343\202\244 Watch"
        dammit = UnicodeDammit(utf_8)
        assert dammit.original_encoding.lower() == 'utf-8'
        assert dammit.unicode_markup.encode("utf-8") == utf_8

    def test_ignore_inappropriate_codecs(self):
        utf8_data = "RÃ¤ksmÃ¶rgÃ¥s".encode("utf-8")
        dammit = UnicodeDammit(utf8_data, ["iso-8859-8"])
        assert dammit.original_encoding.lower() == 'utf-8'

    def test_ignore_invalid_codecs(self):
        utf8_data = "RÃ¤ksmÃ¶rgÃ¥s".encode("utf-8")
        for bad_encoding in ['.utf8', '...', 'utF---16.!']:
            dammit = UnicodeDammit(utf8_data, [bad_encoding])
            assert dammit.original_encoding.lower() == 'utf-8'

    def test_exclude_encodings(self):
        # This is UTF-8.
        utf8_data = "RÃ¤ksmÃ¶rgÃ¥s".encode("utf-8")

        # But if we exclude UTF-8 from consideration, the guess is
        # Windows-1252.
        dammit = UnicodeDammit(utf8_data, exclude_encodings=["utf-8"])
        assert dammit.original_encoding.lower() == 'windows-1252'

        # And if we exclude that, there is no valid guess at all.
        dammit = UnicodeDammit(
            utf8_data, exclude_encodings=["utf-8", "windows-1252"])
        assert dammit.original_encoding == None

class TestEncodingDetector(object):
        
    def test_encoding_detector_replaces_junk_in_encoding_name_with_replacement_character(self):
        detected = EncodingDetector(
            b'<?xml version="1.0" encoding="UTF-\xdb" ?>')
        encodings = list(detected.encodings)
        assert 'utf-\N{REPLACEMENT CHARACTER}' in encodings

    def test_detect_html5_style_meta_tag(self):

        for data in (
            b'<html><meta charset="euc-jp" /></html>',
            b"<html><meta charset='euc-jp' /></html>",
            b"<html><meta charset=euc-jp /></html>",
            b"<html><meta charset=euc-jp/></html>"):
            dammit = UnicodeDammit(data, is_html=True)
            assert "euc-jp" == dammit.original_encoding

    def test_last_ditch_entity_replacement(self):
        # This