import logging

from elasticsearch import Elasticsearch
from elasticsearch.exceptions import NotFoundError

from scrapi import settings
from scrapi.processing.base import BaseProcessor

es = Elasticsearch(
    settings.ELASTIC_URI,
    request_timeout=settings.ELASTIC_TIMEOUT
)


logger = logging.getLogger(__name__)
logging.getLogger('urllib3').setLevel(logging.WARN)
logging.getLogger('requests').setLevel(logging.WARN)
logging.getLogger('elasticsearch').setLevel(logging.WARN)
logging.getLogger('elasticsearch.trace').setLevel(logging.WARN)


es.cluster.health(wait_for_status='yellow')


class ElasticsearchProcessor(BaseProcessor):
    NAME = 'elasticsearch'

    def process_normalized(self, raw_doc, normalized):
        data = {
            key: value for key, value in normalized.attributes.items()
            if key in settings.FRONTEND_KEYS
        }

        normalized['dateUpdated'] = self.version_dateUpdated(normalized)

        es.index(
            body=data,
            refresh=True,
            index=settings.ELASTIC_INDEX,
            doc_type=normalized['source'],
            id=normalized['id']['serviceID'],
        )

    def version_dateUpdated(self, normalized):
        try:
            old_doc = es.get_source(
                index=settings.ELASTIC_INDEX,
                doc_type=normalized['source'],
                id=normalized['id']['serviceID'],
            )
        except NotFoundError:
            # Normally I don't like exception-driven logic,
            # but this was the best way to handle missing
            # types, indices and documents together
            date = normalized['dateUpdated']
        else:
            date = old_doc['dateUpdated']

        return date


def create_index():
    body = {
        "mappings": {
            harvester: {
                "properties": {
                    "id": {
                        "properties": {
                            "doi": {
                                "type": "string",
                                "index": "not_analyzed"
                            },
                            "url": {
                                "type": "string",
                                "index": "not_analyzed"
                            },
                            "serviceID": {
                                "type": "string",
                                "index": "not_analyzed"
                            }
                        }
                    }
                }
            } for harvester in settings.MANIFESTS.keys()
        }
    }
    es.indices.create(index=settings.ELASTIC_INDEX, body=body, ignore=400)

create_index()

@@ -706,32 +706,105 @@
 c, normalized):%0A
+        normalized%5B'dateUpdated'%5D = self.version_dateUpdated(normalized)%0A
         data = %7B
@@ -935,82 +935,8 @@
  %7D%0A%0A
-        normalized%5B'dateUpdated'%5D = self.version_dateUpdated(normalized)%0A%0A
     
@@ -2044,362 +2044,1217 @@
 zed%22
-%0A                            %7D,%0A                            %22url%22: %7B%0A                                %22type%22: %22string%22,%0A                                %22index%22: %22not_analyzed%22%0A                            %7D,%0A                            %22serviceID%22: %7B%0A                                %22type%22: %22string%22,%0A                                %22index%22: %22not_analyzed%22
+,%0A                                %22fields%22: %7B%0A                                    %22analyzed%22: %7B%0A                                        %22type%22: %22string%22,%0A                                        %22index%22: %22analyzed%22%0A                                    %7D%0A                                %7D%0A                            %7D,%0A                            %22url%22: %7B%0A                                %22type%22: %22string%22,%0A                                %22index%22: %22not_analyzed%22,%0A                                %22fields%22: %7B%0A                                    %22analyzed%22: %7B%0A                                        %22type%22: %22string%22,%0A                                        %22index%22: %22analyzed%22%0A                                    %7D%0A                                %7D%0A                            %7D,%0A                            %22serviceID%22: %7B%0A                                %22type%22: %22string%22,%0A                                %22index%22: %22not_analyzed%22,%0A                                %22fields%22: %7B%0A                                    %22analyzed%22: %7B%0A                                        %22type%22: %22string%22,%0A                                        %22index%22: %22analyzed%22%0A                                    %7D%0A                                %7D
 %0A   

Add analyzed field to id mappings, so that missing aggregations can work