import os
import logging
import importlib
from datetime import datetime

from celery import Celery

import settings


app = Celery()
app.config_from_object(settings)

logger = logging.getLogger(__name__)


def import_consumer(consumer_name):
    return importlib.import_module('scrapi.consumers.{}'.format(consumer_name))


@app.task
def run_consumer(consumer_name):
    logger.info('Runing consumer "{}"'.format(consumer_name))
    # Form and start a celery chain
    chain = (consume.si(consumer_name) | begin_normalization.s(consumer_name))
    chain.apply_async()


@app.task
def begin_normalization(raw_docs, consumer_name):
    logger.info('Normalizing {} documents for consumer "{}"'
                .format(len(raw_docs), consumer_name))

    for raw in raw_docs:
        timestamp = datetime.now()

        process_raw.si(raw, timestamp).apply_async()

        chain = (normalize.si(raw, timestamp, consumer_name) | process_normalized.s())
        chain.apply_async()


@app.task
def consume(consumer_name):
    logger.info('Consumer "{}" has begun consumption'.format(consumer_name))

    consumer = import_consumer(consumer_name)
    result = consumer.consume()

    logger.info('Consumer "{}" has finished consumption'.format(consumer_name))

    return result


@app.task
def normalize(raw_doc, timestamp, consumer_name):
    consumer = import_consumer(consumer_name)
    normalized = consumer.normalize(raw_doc, timestamp)
    # Do other things here
    return normalized


@app.task
def process_raw(raw_doc, timestamp):
    pass


@app.task
def process_normalized(normalized_doc):
    pass


@app.task
def check_archive():
    pass


@app.task
def tar_archive():
    os.system('tar -czvf website/static/archive.tar.gz archive/')

@@ -93,16 +93,28 @@
 Celery%0A%0A
+from scrapi 
 import s
@@ -243,24 +243,102 @@
 umer_name):%0A
+    # TODO Make suer that consumer_name will always import the correct module%0A
     return i
@@ -1358,16 +1358,185 @@
 esult%0A%0A%0A
+@app.task%0Adef process_raw(raw_doc, timestamp):%0A    # This is where the raw_doc should be dumped to disc%0A    # And anything else that may need to happen to it%0A    pass%0A%0A%0A
 @app.tas
@@ -1693,175 +1693,578 @@
 mp)%0A
+%0A
     
-# Do other things here%0A    return normalized%0A%0A%0A@app.task%0Adef process_raw(raw_doc, timestamp):%0A    pass%0A%0A%0A@app.task%0Adef process_normalized(normalized_doc):%0A    pass
+logger.info('Document %7B%7D normalized sucessfully'.format(normalized.get('doc_id')))%0A%0A    normalized.attributes%5B'location'%5D = 'TODO'%0A    normalized.attributes%5B'isoTimestamp'%5D = str(timestamp.isoformat())%0A%0A    return normalized%0A%0A%0A@app.task%0Adef process_normalized(normalized_doc):%0A    # This is where the normalized doc should be dumped to disc%0A    # And then sent to OSF%0A    # And anything that may need to occur%0A    pass%0A%0A%0A@app.task%0Adef check_archives(reprocess):%0A    for consumer in settings.MANIFESTS.keys():%0A        check_archive(consumer, reprocess).apply_async()
 %0A%0A%0A@
@@ -2286,24 +2286,48 @@
 eck_archive(
+consumer_name, reprocess
 ):%0A    pass%0A
@@ -2317,32 +2317,40 @@
 ocess):%0A    pass
+  # TODO
 %0A%0A%0A@app.task%0Adef

Update scrapi celery tasks