# -*- coding: utf-8 -*-
"""
    runner.py

    Copyright 2017 CodeRatchet

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0
"""
import logging
import os
import sys
from argparse import ArgumentParser

from scrapy.crawler import CrawlerProcess

from scrapytest.config import config


class Runner:
    """
    main crawler api.

    usage:
        `./run crawl [OPTIONS...]` will crawl the website and store the articles in the attached database
        `./run search [OPTIONS...]` [EXPRESSION] will search for a given article with the following search expression.
    """

    def __init__(self):

        # setup the parse and its initial values
        self._parser = ArgumentParser(prog=os.path.basename(__file__),
                                      description='Utility for parsing and searching articles on the '
                                                  '\"theguardian.com.au\" website')
        self._configure_parser()
        if len(sys.argv) < 2:
            self._parser.print_usage()
            exit(1)
        self._custom_guardian_config = {}
        self._args = {}

    def run(self):
        # Parse command line args
        parsed_args = self._parser.parse_args()
        self._args = vars(parsed_args)

        # configure the program according to user args
        self._process_arguments()

        # default function according to sub_parser is stored in the func attribute on the Namespace.
        parsed_args.func()
        self._custom_guardian_config = {}  # type: dict

    def search(self):
        """ search the database for the terms defined in the argument to --search/-s """
        # TODO: Implement
        pass

    def crawl(self):
        """ crawl through the database and either save the results to a database or text files. """
        # setup settings
        from scrapy.settings import Settings
        from scrapytest.spiders import GuardianNewsSpider

        settings = Settings()
        settings.set("USER_AGENT", config['crawler_user_agent'])
        settings.set('custom_guardian_config', self._custom_guardian_config)

        crawler = CrawlerProcess(settings=settings)
        crawler.crawl(GuardianNewsSpider)
        crawler.start()
        crawler.join()

    def _process_arguments(self):
        """ process and save the effects of command line arguments """
        # log level
        if 'log_level' in self._args:
            try:
                # convert the string to the mapped int
                uppercase_log_level = str.upper(self._args['log_level'])
                # noinspection PyProtectedMember
                logging.getLogger().setLevel(logging._nameToLevel[uppercase_log_level])
            except KeyError as key_error:
                logging.log(logging.CRITICAL,
                            "could not determine logging level from string: {}\n{}".format(self._args['log_level'],
                                                                                           str(key_error)))
                exit(1)
        if 'max_depth' in self._args:
            self._custom_guardian_config.update({'max_depth': self._args['max_depth']})
        pass

    def _configure_parser(self):
        """ Add the command line options to the argument parser """
        # All parsers have logging
        self._log_parser = ArgumentParser(add_help=False)
        self._log_parser.add_argument('-l', '--log-level', help="set level of log output", dest='log_level',
                                      action='store',
                                      type=str, default='INFO',
                                      choices=['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG'])

        self._subparsers = self._parser.add_subparsers()

        # setup search api
        self._search_parser = self._subparsers.add_parser('search', help='Search for articles in the database',
                                                          parents=[self._log_parser])  # type: ArgumentParser
        self._search_parser.set_defaults(func=self.search)

        # TODO: Implement functionality
        self._search_parser.add_argument('expression', help="search query for finding an article", nargs=1,
                                         action='store')

        # setup crawl api
        self._crawl_parser = self._subparsers.add_parser('crawl',
                                                         help='Crawl through \"theguardian.com.au\" and optionally '
                                                              'store data',
                                                         parents=[self._log_parser])  # type: ArgumentParser
        self._crawl_parser.set_defaults(func=self.crawl)
        self._crawl_parser.add_argument('--section', help="section to search under", dest='section', nargs='+',
                                        choices=config['guardian_spider']['collection_paths'])

        # TODO: Implement functionality
        self._crawl_parser.add_argument('--no-save', help="do not save results to the database",
                                        dest='save_crawl_results',
                                        action='store_false', default=True)

        # TODO: Implement functionality
        self._crawl_parser.add_argument('-p', '--print-json', help="print results in json format", action='store_true',
                                        dest='print_json')


if __name__ == '__main__':
    """ see above """
    runner = Runner()
    try:
        runner.run()
    except Exception as e:
        logging.log(logging.ERROR, str(e))

@@ -455,16 +455,51 @@
 config%0A%0A
+log = logging.getLogger(__name__)%0A%0A
 %0Aclass R
@@ -2210,24 +2210,83 @@
 er_agent'%5D)%0A
+        settings.set(%22LOG_LEVEL%22, self._args%5B'log_level'%5D)%0A
         sett
@@ -2630,308 +2630,79 @@
     
-if 'log_level' in self._args:%0A            try:%0A                # convert the string to the mapped int%0A                uppercase_log_level = str.upper(self._args%5B'log_level'%5D)%0A                # noinspection PyProtectedMember%0A                logging.getLogger().setLevel(logging._nameToLevel%5Buppercase_
+logging.basicConfig(filename=self._args%5B'log_file'%5D, level=self._args%5B'
 log_
@@ -2706,16 +2706,17 @@
 og_level
+'
 %5D)%0A     
@@ -2722,335 +2722,75 @@
     
-    
-except KeyError as key_error:%0A                logging.log(logging.CRITICAL,%0A                            %22could not determine logging level from string: %7B%7D%5Cn%7B%7D%22.format(self._args%5B'log_level'%5D,%0A                                                                                           str(key_error)))%0A                exit(1)
+self._custom_guardian_config%5B'LOG_LEVEL'%5D = self._args%5B'log_level'%5D
 %0A   
@@ -2911,29 +2911,16 @@
 epth'%5D%7D)
-%0A        pass
 %0A%0A    de
@@ -3273,46 +3273,8 @@
 re',
-%0A                                     
  typ
@@ -3278,16 +3278,22 @@
 type=str
+.upper
 , defaul
@@ -3398,16 +3398,226 @@
 DEBUG'%5D)
+%0A        self._log_parser.add_argument('-lf', '--log-file', help=%22set the file for storing log data%22, dest='log_file',%0A                                      action='store', type=str, default='./scrapytest.log')
 %0A%0A      
@@ -5487,36 +5487,14 @@
  log
-ging.log(logging.ERROR, st
+.erro
 r(e)
-)
 %0A

set LOG_LEVEL of spider from user args, general code tidying