import argparse
import configparser
from dateutil import parser

import pymongo
import requests

from indeed import IndeedClient
from pymongo import MongoClient

from feature_extraction import job_degree_strings


def _update_array_fields(model, current_values, new_field_values):
    """
    Update all array fields if they don't contain the new values
    :param model: DB Base Model
    :param current_values: Dictionary of current values for model
    :param new_field_values: Dictionary of new values that should be in arrays
    :return:
    """
    update_array_fields = {}
    for field, value in new_field_values.items():
        if value not in current_values[field]:
            update_array_fields[field] = value

    if update_array_fields:
        model.update_one({'_id': current_values['_id']}, {'$push': update_array_fields})


def scrape_indeed(database, indeed_client, job_title, locations):
    """
    Scrape job data from indeed and save it to the database
    :param database: Database to save the indeed data
    :param indeed_client: Indeed API client
    :param job_title: Job title to search for
    :param locations: Job locations to search for
    :return: 
    """
    max_indeed_limit = 25
    indeed_params = {
        'q': job_title,
        'limit': max_indeed_limit,
        'latlong': 1,
        'sort': 'date',
        'userip': '1.2.3.4',
        'useragent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2)'
    }

    for location in locations:
        result_start = 0
        newest_job = database.jobs.find_one({'search_title': job_title, 'search_location': location},
                                            sort=[('date', pymongo.DESCENDING)])
        indeed_response = indeed_client.search(**indeed_params, l=location, start=result_start)
        jobs = indeed_response['results']
        total_jobs = indeed_response['totalResults']

        while result_start < total_jobs and\
                (not newest_job or newest_job['date'] < parser.parse(jobs[0]['date']).timestamp()):
            new_jobs = []
            for job in jobs:
                found_job = database.jobs.find_one({'jobkey': job['jobkey']})
                if found_job:
                    _update_array_fields(
                        database.jobs,
                        found_job,
                        {'search_location': location, 'search_title': job_title})
                else:
                    job['search_location'] = [location]
                    job['search_title'] = [job_title]
                    job['html_posting'] = requests.get(job['url']).content
                    job['date'] = parser.parse(job['date']).timestamp()
                    new_jobs.append(job)
            if new_jobs:
                database.jobs.insert_many(new_jobs)

            result_start += indeed_params['limit']
            jobs = indeed_client.search(**indeed_params, l=location, start=result_start)['results']


def analyse(database, job_title):
    """
    Analyse job data from database and print results
    :param database: Database with the job data
    :param job_title: Job title to analyse
    :return:
    """
    city_degree_counts = {}
    for job in database.jobs.find({'search_title': job_title}):
        for city in job['search_location']:
            if city not in city_degree_counts:
                city_degree_counts[city] = {
                    'unknown': 0,
                    'undergrad': 0,
                    'ms/phd': 0,
                    'ms': 0,
                    'phd': 0}
            degree_strings = job_degree_strings(job['html_posting'])
            is_grad = degree_strings['ms'] or degree_strings['phd']
            is_undergrad = degree_strings['undergrad'] and not is_grad

            if is_undergrad:
                city_degree_counts[city]['undergrad'] += 1
            elif is_grad:
                if degree_strings['ms'] and degree_strings['phd']:
                    city_degree_counts[city]['ms/phd'] += 1
                elif degree_strings['ms']:
                    city_degree_counts[city]['ms'] += 1
                else:
                    city_degree_counts[city]['phd'] += 1
            else:
                city_degree_counts[city]['unknown'] += 1

    for city, degree_count in city_degree_counts.items():
        print('{} found {} undergrads, {} ms/phd, {} ms, {} phd, and {} unknown matches'.format(
            city,
            degree_count['undergrad'],
            degree_count['ms/phd'],
            degree_count['ms'],
            degree_count['phd'],
            degree_count['unknown']))


def run():
    config = configparser.ConfigParser()
    config.read('config.ini')
    arg_parser = argparse.ArgumentParser()
    database = MongoClient(config['DATABASE']['Host'], int(config['DATABASE']['Port']))[config['DATABASE']['Name']]

    arg_parser.add_argument('TaskType', help='Run the specified task type', choices=['analyse', 'scrape'], type=str)
    arg_parser.add_argument('JobTitle', help='Search for specific job title', type=str)
    arg_parser.add_argument('--locations', help='Location(s) to search', nargs='+', type=str)
    args = arg_parser.parse_args()

    if args.TaskType == 'analyse':
        analyse(database, args.JobTitle)
    elif args.TaskType == 'scrape':
        if not args.locations:
            arg_parser.error('TaskType "scrape" requires --locations')
        indeed_client = IndeedClient(publisher=config['INDEED']['PublisherNumber'])
        scrape_indeed(database, indeed_client, args.JobTitle, args.locations)


if __name__ == '__main__':
    run()

@@ -4244,78 +4244,232 @@
     
-for city, degree_count in city_degree_counts.items():%0A        print('%7B
+longest_city_length = len(max(city_degree_counts.keys(), key=len))%0A    for city, degree_count in city_degree_counts.items():%0A        output_string =%5C%0A            '%7B:%3C' +%5C%0A            str(longest_city_length) +%5C%0A            '
 %7D fo
@@ -4469,24 +4469,27 @@
   '%7D found %7B
+:%3E3
 %7D undergrads
@@ -4491,16 +4491,19 @@
 grads, %7B
+:%3E3
 %7D ms/phd
@@ -4509,15 +4509,21 @@
 d, %7B
+:%3E3
 %7D ms, %7B
+:%3E3
 %7D ph
@@ -4530,16 +4530,19 @@
 d, and %7B
+:%3E3
 %7D unknow
@@ -4551,16 +4551,44 @@
 matches'
+%0A        print(output_string
 .format(
@@ -4588,16 +4588,20 @@
 format(%0A
+    
         
@@ -4614,32 +4614,36 @@
 ty,%0A            
+    
 degree_count%5B'un
@@ -4657,32 +4657,36 @@
 '%5D,%0A            
+   
+ 
 degree_count%5B'ms
@@ -4697,32 +4697,36 @@
 '%5D,%0A            
+    
 degree_count%5B'ms
@@ -4733,32 +4733,36 @@
 '%5D,%0A            
+  
+  
 degree_count%5B'ph
@@ -4762,24 +4762,28 @@
 unt%5B'phd'%5D,%0A
+    
             

Clean up the formatting for the analysis output