00000003
 Y    ‹      Õ–ßoÚ0ÇßûWx~Ø¡¥¨4xêÒÐ( –iš¦©2±!V;²”þõspÚ¦SÑ˜ÖVíÓù’oîîãœ„¢qoúcÒG_¦çC4ùöy8è!\ø~Øˆ¦‘qTo ýF8±6ï ¬×ëúú°®ô¦�Ø,=cµˆm�Y†ÉAX<#(L8eÎ¢Ð
›r2�Œß 5G –ÖÆÝwŒ¯DÌM1¶<ƒŒgJoJÓnƒTŒ7J·uôØBð¡]2(³…3Å6EÁË$tq·Ht¶Í[5	m‚V4ÙÅVå˜„"[ £ã.+i`–Ry]_ˆ9F4µ]üsÐÿr:°Iñ5	)J4Ÿwñ§^wt:îF˜ŒhÆC OkÎ�æ“!5eŠ‰¹àl§øÒ‹/Åíî€‘×DÜÄZäV(ù ‡XE�UjrêX[Ž ÑU•—°¿Í�MÆäì¢?šFƒ?%¬ñPÝÿÿ>L&TsiQ$4�­{UÂùLåÌä§w.*K×b‘XLª¡'•ÿÀ;W)ãºJ¼‹õ®^o«UþQV³Ñ<®A-8AA»Ó8Fèë_Êk©Ö²
àÚáI %S!9&ÞîÐPÐì4w¸-èë+äÉÆ\ù&Ã¤âì‰tZ'o
E»`I²¿•²ÿ¢ÈÕšk·(¼}Žú_yQh·­Š“ûáþÁ[é'c©u[óþºÈ,gå™áîÇïp‹]ò•;Ë0ñö9 ^¹�Š@ìêVIn0©8/ßS{^b ¼Þ…°½M†à¯´¿.Û†5    5   ˆl–Ü4ý( ©¤ýÔâ€~à›¸ÔÅ£�þB g,²<2ïóÇƒh¹ÆÄÃ      10
  G   	ˆl—äY>”*h¥2 Y¸ÛweLZ7ÿ’þFÒÍÊ2ÇÝ­�MìÕÿ?ÈÂÁÇ‚"_‹I|¥Žƒî4ÃVŸÆÅ  „    	‹      e�1Ã {^A‡]øü ”"e>�þ$Ÿ%$É–ýû@h,YÚívg ßâ ƒ!š{¶¸/i–DðI«ËT+‡­ÚZBÆùvÈ…½~hÜ.dA`_)m:í}ÛþªóIŽODNõ~¥æÙœfü[_MjÕ¬~!â¤        ˆÃÂÈ615ÁÆÅ g    ‹      Õ—]oÚ0†ïû+<_ì]/Úà©#HCkQ¦iš&dlC¬&vdhöëwB˜
hÛE+åÆŽí7>ïs¢œ8Ñ»xÜŸ}ŸÈçÙý™|ýt7ìÚ øÖíÄ³¸Z¸l¶Ú ƒ%4	!¿Øn·Ím·iÝ
fSHB–^‚N‹Ð”ARv•sì‚D‰â{RÅ†Fª'b—ì:x-ÞWx�j£…òåuP+Õ®i�È×Wà×‹ýŠtz£œ‡ÜYÔ{ëJª¸HèrónU(ûèÑÂÊ¢4Õ~g‡Ì;å¥cQHÈ†§zez4Øœ²Hg+â�èQÐÂ‹”›ÇæJ/)áièÑÃþø'ê $åÝ,â$qjÙ£û½ÑÍ¸S6â™Š€ŸÖÜ£æ–²;îÉ¬ÔK­äYñC%~Ð¿ÎoWšXyát´5ÏR@ÄCTaSŸsdý€‰;TUù·lpq”ŒÉít0šÅÃi•Ynñìî5'eî”	$ÖN‰`]±Ç­b¿7Ÿßü’=ŒÓ«$PFHƒœTþGÖæÑØ­9L&ø$¾°‰B«¢˜§Ú¨¹Ç§HÙ‰ÉC€Ž;­ÎU£ÝntZ¤ÕºÆ–œb9øò†hë‰–54žª�J)Ûu5´oÖÙB¹¹]Î½
ž²ãq�ò¤ðZð´zrîð;�%Œ²35Dô	,9Ç†,ã¿˜¨7RÆó#"×hW›kZŒC‘£ù²­¡ù5Vbƒa«¾† [^ø² s<©ÍƒÞèPPvrúõñþñ°ûcp»SxÕ¯ÁoÖ(¢0}    X   ˆl–Ðz¾”d
eJe@
à�q¦ÔÅ£�þe”«7+.†BWßhÏÊƒhµÉ_ŠI|¥�+Wé3ž¯a–Ü4ý( ©¤ýÔâ�À=q§”Å£È þ    # Copyright (C) 2006-2014 Bastian Kleineidam
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
"""
URL checking functions.
"""
import copy
import time
from . import task
from ..cache import urlqueue
from .. import parser

# Interval in which each check thread looks if it's stopped.
QUEUE_POLL_INTERVALL_SECS = 1.0


def check_urls(urlqueue, logger):
    """Check URLs without threading."""
    while not urlqueue.empty():
        url_data = urlqueue.get()
        try:
            check_url(url_data, logger)
        finally:
            urlqueue.task_done(url_data)


def check_url(url_data, logger):
    """Check a single URL with logging."""
    if url_data.has_result:
        logger.log_url(url_data.to_wire())
    else:
        cache = url_data.aggregate.result_cache
        key = url_data.cache_url
        result = cache.get_result(key)
        if result is None:
            # check
            check_start = time.time()
            try:
                url_data.check()
                do_parse = url_data.check_content()
                url_data.checktime = time.time() - check_start
                # Add result to cache
                result = url_data.to_wire()
                cache.add_result(key, result)
                for alias in url_data.aliases:
                    # redirect aliases
                    cache.add_result(alias, result)
                logger.log_url(result)
                # parse content recursively
                # XXX this could add new warnings which should be cached.
                if do_parse:
                    parser.parse_url(url_data)
            finally:
                # close/release possible open connection
                url_data.close_connection()
        else:
            # copy data from cache and adj