512
  X   ˆl–Ü4ý(”Ëm
qAàCq§ÔÅ£“þBÛ‹8Ûd•Ù ¥hMìÕÿ?Å{‹„„-i[D<†ªoZƒ›Ù«Æ234_‹I|¥Žƒî4ÃVŸÂÄ  ê    ‹      U�±N1D{ÅÐ$àâüQ$
$šˆ4ù€ø|{±‘Ïkl_„øwì€BÒ­ffŸfWI‰*d
ÆD„£NN÷ž°?xîµßY—÷�Jù7óz5°FùŒÄ#7ÚKîß+q‰ÅâÖy¨V˜½¿—»í%Üöþ¦;{%„’R@bg	ãLqp²œ	1qáVÆj�éc¦`§¤c¤”á‚¥äZqžºJi ç˜ÜQí®+±×™6<èlŸð% ¥ðÆàŠÑ—@FN�øbâaöÔÑ9r*íÿû+ñæY.íV    C   ˆl–Ðz¾”d
eJe@
à�q¦ÔÅ£�þY~9nV]„$¯¾Ð?ŸÉ„N¼È_ŠI|¥�+Wé3ž¯ÄÆ 9½    # Copyright (C) 2000-2014 Bastian Kleineidam
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
"""
Handle http links.
"""

import requests

# The validity of SSL certs is ignored to be able
# the check the URL and recurse into it.
# The warning about invalid SSL certs is given to the
# user instead.
import warnings

warnings.simplefilter(
    'ignore', requests.packages.urllib3.exceptions.InsecureRequestWarning
)

from io import BytesIO
import re

from .. import (
    log,
    LOG_CHECK,
    mimeutil,
    url as urlutil,
    LinkCheckerError,
    httputil,
)
from . import internpaturl

# import warnings
from .const import WARN_HTTP_EMPTY_CONTENT, WARN_HTTP_RATE_LIMITED
from requests.sessions import REDIRECT_STATI

HTTP_SCHEMAS = ('http://', 'https://')

# match for robots meta element content attribute
nofollow_re = re.compile(r"\bnofollow\b", re.IGNORECASE)


class HttpUrl(internpaturl.InternPatternUrl):
    """
    Url link with http scheme.
    """

    def reset(self):
        """
        Initialize HTTP specific variables.
        """
        super().reset()
        # initialize check data
        # server headers
        self.headers = {}
        self.auth = None
        self.ssl_cipher = None
        self.ssl_cert = None

    def allows_robots(self, url):
        """
        Fetch and parse the robots.txt of given url. Checks if LinkChecker
        can get the requested resource content.

        @param url: the url to be requested
        @type url: string
        @return: True if access is granted, otherwise False
        @rtype: bool
        """
        return not self.aggregate.config[
            'robotstxt'
        ] or self.aggregate.robots_txt.allows_url(
            self, timeout=self.aggregate.config["timeout"]
        )

    def content_allows_robots(self):
        """
        Return False if the content of this URL forbids robots to
        search for recursive links.
        """
        if not self.is_html():
            return True

        soup = self.get_soup()
        return not soup.find("meta", attrs={"name": "robots", "content": nofollow_re})

    def add_size_info(self):
        """Get size of URL content from HTTP header."""
        if (
            self.headers
            and "Content-Length" in self.headers
            and "Transfer-Encoding" not in self.headers
        ):
            # Note that content-encoding causes size differences since
            # the content data is always decoded.
            try:
                self.size = int(self.headers["Content-Length"])
            except (ValueError, OverflowError):
                pass
        else:
            self.size = -1

    def check_connection(self):
        """
        Check a URL with HTTP protocol.
        Here is an excerpt from RFC 1945 with common response codes:
        The first digit of the Status-Code defines the class of response. The
        last two digits do not have any categorization role. There are 5
        values for the first digit:

          -