import os
import re
import json
from datetime import datetime
from subprocess import check_output

import spicedham


def get_data_set():
    with open('responses.json', 'r') as f:
        return json.load(f)


def train():
    train_on_file('jcorpus_newest.json')
    train_on_file('jcorpus_new.json')
    train_on_file('jcorpus.json')


def train_on_file(file_name):
    with open(file_name, 'r') as f:
        data = json.load(f)
    for response in data['results']:
        spicedham.train('description', process_response(response),
                        response['spam'])


def classify():
    with open('responses.json', 'r') as f:
        responses = json.load(f)
    return {(result['id'], spicedham.classify('description',
             process_response(result)))
            for result in responses['results']}


def process_response(response):
    return filter(lambda x: x != '',
                  re.split(' \n\r.,', response['description']))


def diff_against_previous(data, current_run_name):
    commit_hash = check_output(['git', 'rev-parse', 'HEAD'])
    with open('spicedham-config.json', 'r') as f:
        config_values = f.read()
    build_set = lambda x: {(result['id'], result['spam']) for result in x}
    last_run_data, last_run = get_last_run_data()
    prev_data = build_set(last_run_data)
    newly_detected = data - prev_data
    fell_out = prev_data - data
    diff = {
        'commit_hash': commit_hash,
        'diffed_files': [last_run, current_run_name],
        'num_fell_out': len(fell_out),
        'num_newly_detected': len(newly_detected),
        'config_values': config_values,
        'fell_out': list(fell_out),
        'newly_detected': list(newly_detected),
    }
    with open('diff-' + current_run_name + '-' + last_run, 'w') as f:
        json.dump(diff, f)


def get_last_run_data():
    prev_runs = sorted(filter(lambda x: x.startswith('analyzed-data-'),
                       os.listdir('.')))
    try:
        last_run = prev_runs[0]
        with open(last_run, 'r') as f:
            last_run_data = json.load(f)
        return (last_run_data, last_run)
    except IndexError:
        return ([], '')


def output_file(data, name):
    with open(name, 'w') as f:
        json.dump(data, f)


if __name__ == '__main__':
    print 'training...'
    train()
    print 'classifying...'
    data = classify()
    print 'analyzing...'
    current_run_name = 'analyzed-data-' + str(datetime.now()) + '.json'
    diff_against_previous(data, current_run_name)
    print 'writing out the results...'
    output_file({'data': map(list, data)}, current_run_name)
    print 'done!'

@@ -572,16 +572,124 @@
 spam'%5D)%0A
+        spicedham.train('user_agent', process_response(response),%0A                        response%5B'spam'%5D)%0A
 %0A%0Adef cl
@@ -820,29 +820,19 @@
 lassify(
-'description'
+tag
 ,%0A      
@@ -911,17 +911,173 @@
 esults'%5D
-%7D
+%0A            for tag in %5B'user_agent', 'description'%5D%7D%0A%0Adef process_user_agent(user_agent):%0A    return re.sub('%5B%5EA-Za-z%5D+', '', process_response(user_agent))
 %0A%0A%0Adef p
@@ -1432,38 +1432,14 @@
 x: %7B
-(
 result
-%5B'id'%5D, result%5B'spam'%5D)
  for
@@ -2290,24 +2290,90 @@
 son.load(f)%0A
+        last_run_data = %7Btuple(x) for x in last_run_data%5B'data'%5D%7D%0A
         retu

Classify and train on user agent strings