from copy import deepcopy
import logging
try:
    from urlparse import urljoin, parse_qsl
    from urllib import urlencode
except ImportError:
    from urllib.parse import urljoin, parse_qsl, urlencode

from django.conf import settings
from django.core.exceptions import ValidationError

from lxml import etree
import requests

from open511.utils.serialization import XML_LANG, XML_BASE
from open511.validator import Open511ValidationError
from open511_server.models import RoadEvent, ImportTaskStatus


logger = logging.getLogger(__name__)

class BaseImporter(object):

    default_language = None
    base_url = None

    def __init__(self, opts, persist_status=False):
        self.opts = opts
        self.model = RoadEvent
        self.persist_status = persist_status
        self.last_run_status = {}
        self.status = {}

    @property
    def id(self):
        if 'ID' in self.opts:
            return self.opts['ID']
        return self.opts['URL']

    def run(self):

        if self.persist_status:
            try:
                self.last_run_obj = ImportTaskStatus.objects.get(id=self.id)
            except ImportTaskStatus.DoesNotExist:
                self.last_run_obj = ImportTaskStatus.objects.create(id=self.id)
            self.last_run_status = self.last_run_obj.status_info
            self.status = deepcopy(self.last_run_status)

        created = []

        for input_document in self._logging_iterable(self.fetch(), 'fetch'):

            for o5_xml_obj in self._logging_iterable(self.convert(input_document), 'conversion'):

                for db_obj in self._logging_iterable(self.save(o5_xml_obj), 'save',
                        exceptions=(ValueError, ValidationError, Open511ValidationError)):

                    logger.debug("Imported %s %s" % (o5_xml_obj.tag, db_obj.id))
                    created.append(db_obj)

        if self.persist_status:
            self.status['objects_imported'] = len(created)
            self.status['counter'] = self.status.get('counter', 0) + 1
            self.last_run_obj.status_info = self.status
            self.last_run_obj.save()

        logger.info('Importer {} ran, created {} objects'.format(self.id, len(created)))

    def fetch(self):
        raise NotImplementedError

    def convert(self, input_document):
        raise NotImplementedError

    def save(self, xml_obj):
        save_opts = {}
        if self.default_language:
            save_opts['default_language'] = self.default_language
        if self.base_url:
            save_opts['base_url'] = self.base_url
        obj_created, obj = self.model.objects.update_or_create_from_xml(xml_obj, **save_opts)
        yield obj

    def _logging_iterable(self, iterable, step_name, exceptions=(Exception,)):
        iterator = iter(iterable)
        while True:
            try:
                yield next(iterator)
            except StopIteration:
                raise
            except exceptions as e:
                logger.exception("{} importing, during {} step: {}".format(
                    e.__class__.__name__, step_name, e))

class Open511Importer(BaseImporter):

    def _fetch_url(self, url):
        resp = requests.get(url, headers={
            'Accept': 'application/xml',
            'Open511-Version': settings.OPEN511_DEFAULT_VERSION
        })
        return etree.fromstring(resp.content)

    def fetch(self):
        next_url, _, query = self.opts['URL'].partition('?')
        query = dict(parse_qsl(query)) if query else {}
        query['status'] = 'ALL'

        if self.status.get('max_updated'):
            query['updated'] = '>=' + self.status['max_updated']

        next_url = next_url + '?' + urlencode(query)

        while next_url is not None:
            root = self._fetch_url(next_url)
            assert root.tag == 'open511'

            if root.get(XML_LANG):
                self.default_language = root.get(XML_LANG)

            if root.get(XML_BASE):
                self.base_url = root.get(XML_BASE)
            else:
                self.base_url = next_url

            self.status['max_updated'] = max(
                root.xpath('events/event/updated/text()') + [self.status.get('max_updated', '')])

            for xml_obj in root.xpath('events/event'):
                yield xml_obj

            next_link = root.xpath('pagination/link[@rel="next"]')
            if next_link:
                next_url = urljoin(next_url, next_link[0].get('href'))
            else:
                next_url = None

    def convert(self, input_document):
        yield input_document



@@ -1855,16 +1855,51 @@
 b_obj)%0A%0A
+        self.post_import(created)%0A%0A
         
@@ -1914,32 +1914,32 @@
 persist_status:%0A
-
             self
@@ -2358,32 +2358,598 @@
 plementedError%0A%0A
+    def post_import(self, imported):%0A        pass%0A%0A    def archive_existing(self, imported):%0A        if not len(imported):%0A            return%0A        if len(set(o.jurisdiction_id for o in imported)) != 1:%0A            return logger.error(%22Not archiving because events are from different jurisdictions%22)%0A        jur = imported%5B0%5D.jurisdiction%0A        updated = self.model.filter(jurisdiction=jur, active=True).exclude(%0A            id__in=%5Bo.id for o in imported%5D).update(active=False)%0A        if updated:%0A            logger.info(%22%7B%7D events archived%22.format(updated))%0A%0A
     def save(sel
@@ -4132,16 +4132,256 @@
  'ALL'%0A%0A
+        self.full_update = bool(%0A            self.opts.get('FULL_UPDATES_ONLY') or%0A            (self.opts.get('FULL_UPDATES_EVERY') and%0A                self.status.get('counter') %25 self.opts.get('FULL_UPDATES_EVERY', 1000) == 0)%0A        )%0A%0A
         
@@ -4413,16 +4413,41 @@
 pdated')
+ and not self.full_update
 :%0A      
@@ -5439,9 +5439,119 @@
 cument%0A%0A
+    def post_import(self, imported):%0A        if self.full_update:%0A            self.archive_existing(imported)%0A
 %0A

add "full update" option to Open511Importer