import urllib

from handlers.simplehandler import SimpleHandler
from routines.output import save_items

_USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36'
_TIMEOUT = 10

_DEAD_LINKS = """Сайт, указанный в теге website=* недоступен."""


class WebsiteChecker(SimpleHandler):
    def __init__(self):
        self._checked = dict()
        self._dead_links = list()

    def _check_url(self, url):
        if url in self._checked:
            return self._checked[url]

        status = False

        try:
            scheme, netloc, path, query, fragment = urllib.parse.urlsplit(url)
            if scheme == '':
                scheme, netloc, path, query, fragment = urllib.parse.urlsplit('http://' + url)

            if ':' in netloc:
                host, port = netloc.split(':')
                # support for International domains
                host = host.encode('idna').decode('utf-8')
                netloc = host + ':' + port
            else:
                netloc = netloc.encode('idna').decode('latin-1')
            path = urllib.parse.quote(path)

            p_url = urllib.parse.urlunsplit((scheme, netloc, path, query, fragment))
            req = urllib.request.Request(p_url, headers={'User-Agent': _USER_AGENT})
            urllib.request.urlopen(req, None, _TIMEOUT)
        except urllib.error.HTTPError as e:
            if e.code == 301:  # permanent redirect
                status = True
        except:
            pass
        else:
            status = True

        self._checked[url] = status
        return status

    def process(self, item):
        if 'website' in item:
            if not self._check_url(item['website']):
                self._dead_links.append((item['tag'], item['id']))

    def finish(self, output_dir):
        save_items(output_dir + 'errors/website/not_available/', self._dead_links, _DEAD_LINKS)

@@ -6,16 +6,64 @@
 t urllib
+.error%0Aimport urllib.parse%0Aimport urllib.request
 %0A%0Afrom h
@@ -333,10 +333,45 @@
 site
-=*
+, contact:website, url %D0%B8%D0%BB%D0%B8 source_ref
  %D0%BD%D0%B5%D0%B4
@@ -1740,81 +1740,376 @@
     
-if 'website' in item:%0A            if not self._check_url(item%5B'website'%5D)
+urls = %5B%5D%0A        for url_tag in 'website', 'contact:website', 'url', 'source_ref':%0A            if url_tag in item:%0A                urls.append(item%5Burl_tag%5D)%0A%0A        if urls:%0A            dead_link = False%0A            for url in urls:%0A                if not self._check_url(url):%0A                    dead_link = True%0A                    break%0A            if dead_link
 :%0A  

fix website checker; add contact:website, url, source_ref tags to checker