import os
import logging

from scrapi import events
from scrapi import settings
from scrapi.processing.base import BaseProcessor


logger = logging.getLogger(__name__)


__all__ = []


for mod in os.listdir(os.path.dirname(__file__)):
    root, ext = os.path.splitext(mod)
    if ext == '.py' and root not in ['__init__', 'base']:
        __all__.append(root)

from . import *


def process_normalized(raw_doc, normalized, kwargs):
    ''' kwargs is a dictiorary of kwargs.
        keyed by the processor name
        Exists so that when we run check archive we
        specifiy that it's ok to overrite certain files
    '''
    for p in settings.NORMALIZED_PROCESSING:
        _normalized_event(events.STARTED, p, raw_doc)

        extras = kwargs.get(p, {})

        try:
            get_processor(p).process_normalized(raw_doc, normalized, **extras)
        except Exception as e:
            _normalized_event(events.FAILED, p, raw_doc, exception=repr(e))
            logger.error('Processor {} raised exception {}'.format(p, e))
            if settings.DEBUG:
                raise
        else:
            _normalized_event(events.COMPLETED, p, raw_doc)


def process_raw(raw_doc, kwargs):
    for p in settings.RAW_PROCESSING:
        _raw_event(events.STARTED, p, raw_doc)
        extras = kwargs.get(p, {})
        try:
            get_processor(p).process_raw(raw_doc, **extras)
        except Exception as e:
            logger.error('Processor {} raised exception {}'.format(p, e))
            _raw_event(events.FAILED, p, raw_doc, exception=repr(e))
            if settings.DEBUG:
                raise
        else:
            _raw_event(events.COMPLETED, p, raw_doc)


def get_processor(processor_name):
    for klass in BaseProcessor.__subclasses__():
        if klass.NAME == processor_name:
            return klass()

    raise NotImplementedError('No Processor {}'.format(processor_name))


def _normalized_event(status, processor, raw, **kwargs):
    __processing_event(status, processor, raw,
                       _index='normalized.{}'.format(processor), **kwargs)


def _raw_event(status, processor, raw, **kwargs):
    __processing_event(status, processor, raw,
                       _index='raw.{}'.format(processor), **kwargs)


def __processing_event(status, processor_name, raw_doc, **kwargs):
    events.dispatch(
        events.PROCESSING, status, docID=raw_doc['docID'], **kwargs)

@@ -6,23 +6,8 @@
 t os
-%0Aimport logging
 %0A%0Afr
@@ -112,48 +112,8 @@
 or%0A%0A
-%0Alogger = logging.getLogger(__name__)%0A%0A%0A
 __al
@@ -121,18 +121,16 @@
 __ = %5B%5D%0A
-%0A%0A
 for mod 
@@ -315,16 +315,242 @@
 ort *%0A%0A%0A
+def get_processor(processor_name):%0A    for klass in BaseProcessor.__subclasses__():%0A        if klass.NAME == processor_name:%0A            return klass()%0A    raise NotImplementedError('No Processor %7B%7D'.format(processor_name))%0A%0A%0A
 def proc
@@ -1032,37 +1032,32 @@
 except Exception
- as e
 :%0A            _n
@@ -1058,272 +1058,48 @@
     
-_normalized_event(events.FAILED, p, raw_doc, exception=repr(e))%0A            logger.error('Processor %7B%7D raised exception %7B%7D'.format(p, e))%0A            if settings.DEBUG:%0A                raise%0A        else:%0A            _normalized_event(events.COMPLETED, p, raw_doc)
+if settings.DEBUG:%0A                raise
 %0A%0A%0Ad
@@ -1173,55 +1173,8 @@
 NG:%0A
-        _raw_event(events.STARTED, p, raw_doc)%0A
     
@@ -1305,13 +1305,8 @@
 tion
- as e
 :%0A  
@@ -1319,989 +1319,45 @@
     
-logger.error('Processor %7B%7D raised exception %7B%7D'.format(p, e))%0A            _raw_event(events.FAILED, p, raw_doc, exception=repr(e))%0A            if settings.DEBUG:%0A                raise%0A        else:%0A            _raw_event(events.COMPLETED, p, raw_doc)%0A%0A%0Adef get_processor(processor_name):%0A    for klass in BaseProcessor.__subclasses__():%0A        if klass.NAME == processor_name:%0A            return klass()%0A%0A    raise NotImplementedError('No Processor %7B%7D'.format(processor_name))%0A%0A%0Adef _normalized_event(status, processor, raw, **kwargs):%0A    __processing_event(status, processor, raw,%0A                       _index='normalized.%7B%7D'.format(processor), **kwargs)%0A%0A%0Adef _raw_event(status, processor, raw, **kwargs):%0A    __processing_event(status, processor, raw,%0A                       _index='raw.%7B%7D'.format(processor), **kwargs)%0A%0A%0Adef __processing_event(status, processor_name, raw_doc, **kwargs):%0A    events.dispatch(%0A        events.PROCESSING, status, docID=raw_doc%5B'docID'%5D, **kwargs)
+if settings.DEBUG:%0A                raise
 %0A

Remove logging from processing