import os
from featurize_clusters import TextFeaturizer
from load_data import load_gzip_json
from sklearn.grid_search import GridSearchCV, RandomizedSearchCV
from sklearn.metrics import classification_report
from sklearn.metrics import f1_score,fbeta_score      

def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('-labeled','-l',required=True)
    #parser.add_argument('-outfile','-o',required=True)            
    parser.add_argument('--text_features', '-t',default='Count')
    args = parser.parse_args()

    #add keys to be skipped to reduce memory footprint
    skip = frozenset([u'raw_content',u'content_type'])
    #load data
    if os.path.isdir(args.labeled):
        filelist = [os.path.join(args.labeled,f) for f in os.listdir(args.labeled) if os.path.isfile(f)]
        data = [d for d in load_gzip_json(filelist,skip)]
    elif os.path.isfile(args.labeled):
        data = [d for d in load_gzip_json([args.labeled],skip)]
    else:
        raise OSError(2, 'No such file or directory', args.labeled)
    exit()
    #TODO ER and fold generation
    #obtain clusters and generate folds
    #Cluster ids are available in the json objects
    
    #get cluster ids and build a dictionary
    #this is going to be very slow
    clusters = {}
    for d in data:
        key = d[u'cluster_id']
        if key in clusters:
            clusters[key].append(d)
        else:
            clusters[key] = [d]
    del data



    #TODO join with lattice extractinos
    

    #TODO featurize clusters
    #for fold in folds:
    folds=False
    if folds:
        #get training data
        
        #get text features
        #1. fit vectorizers to all training text
        #for each cluster:
           #2. obtain matrix for cluster
           #3. summarize
        
        #get test data
        #featurize test data

        #generate spatio temporal features

        #generate additional features

        #train classifier
        #l1
        #model = svm.LinearSVC(C=1.0, penalty='l1',dual=False,n_jobs=8)
        #l2
        #model = svm.LinearSVC(C=1.0, penalty='l2',n_jobs=8)

        #tune hyper parameter
        # run randomized search
        n_iter_search = 10
        param_grid = {'C':np.logspace(1.0, 500.0, num=n_iter_search)}
        grid_search = GridSearchCV(model, param_grid=param_grid)
        grid_search.fit(X_train, y_train)
        
        #predict test cases
        y_pred = grid_search.predict(X_test)
        
        print classification_report(y_test, y_pred)

        #get scores for evalution at end of our fold loop
        print f1_score(y_test,y_pred)

        #beta parameter determines the weight of precision in the combined score. 
        #beta < 1 lends more weight to precision, while beta > 1 favors recall (beta -> 0 considers only precision, beta -> inf only recall).
        beta = 0.5 
        print fbeta_score(y_test, y_pred, beta)

                

if __name__ == '__main__':
    main()

@@ -818,32 +818,52 @@
 f)%5D%0A        
-data = %5B
+advertisements = %7Bd%5Bu'_id'%5D:
 d for d in l
@@ -886,25 +886,25 @@
 lelist,skip)
-%5D
+%7D
 %0A    elif os
@@ -943,16 +943,39 @@
     
-data = %5B
+advertisements = %7Bd%5Bu'doc_id'%5D:
 d fo
@@ -1016,17 +1016,17 @@
 d%5D,skip)
-%5D
+%7D
 %0A    els
@@ -1100,143 +1100,8 @@
 ed)%0A
-    exit()%0A    #TODO ER and fold generation%0A    #obtain clusters and generate folds%0A    #Cluster ids are available in the json objects%0A
     
@@ -1191,17 +1191,22 @@
  cluster
-s
+_to_id
  = %7B%7D%0A  
@@ -1215,17 +1215,43 @@
 for 
+key,
 d in 
-data
+advertisements.iteritems()
 :%0A  
@@ -1256,16 +1256,18 @@
         
+c_
 key = d%5B
@@ -1292,16 +1292,18 @@
      if 
+c_
 key in c
@@ -1308,17 +1308,22 @@
  cluster
-s
+_to_id
 :%0A      
@@ -1331,26 +1331,33 @@
      cluster
-s%5B
+_to_id%5Bc_
 key%5D.append(
@@ -1356,17 +1356,19 @@
 .append(
-d
+key
 )%0A      
@@ -1398,35 +1398,154 @@
 ster
-s%5Bkey%5D = %5Bd%5D%0A    del data%0A%0A
+_to_id%5Bc_key%5D = %5Bkey%5D%0A%0A    #TODO ER and fold generation%0A    #obtain clusters and generate folds%0A    #Cluster ids are available in the json objects
 %0A%0A  
@@ -1582,10 +1582,10 @@
 acti
-n
 o
+n
 s%0A  

load data and map ids to cluster