from feature_extraction import url2mat
from collections import namedtuple
from content_extraction import FailedExtraction
from datastores import training_db, model_db
from debug import spy
from flask import g
from joblib import Memory
import logging as log
import numpy as np
import os
import sklearn.linear_model as lm
import sklearn as sk
from time import sleep
import traceback

Feedback = namedtuple("Feedback", ["feedback", "explicit"])

_memory = Memory(cachedir="score-cache", verbose=1, bytes_limit=10**9)

_model_attr_name = "_model"


def _set_model(model):
    setattr(g, _model_attr_name, model)
    model_db()["user"] = model
    model_db().sync()


def _get_model():
    model = getattr(g, _model_attr_name, None)
    if model is None:
        model = model_db().get("user", None)
        if model is None:
            model = _new_model()
        _set_model(model)
    return model


def _new_model():
    return lm.LogisticRegression(class_weight='balanced', solver="lbfgs")


@_memory.cache
def _score_entry(entry):
    try:
        X = url2mat(entry.link, entry)
        probs = _get_model().predict_proba(X=X)[:, 1]
        # implicit feedback: if not overridden we assume prediction correct
        store_feedback(
            url=entry.link, feedback=probs.mean() > 0.5, explicit=False)
        return probs
    except Exception, e:
        log.error("Failed Scoring for {link}".format(entry.link))
        raise


def score_feed(parsed_feed):
    return [_score_entry(e) for e in parsed_feed.entries]


def store_feedback(url, feedback, explicit):
    # explicit overwrites anything
    if (training_db().get(url) is None) or explicit:
        training_db()[url] = Feedback(feedback=feedback, explicit=explicit)
        training_db().sync()


def _url2mat_or_None(url):
    try:
        return url2mat(url)
    except Exception:
        return None


def learn():
    Xyw = [
        dict(X=X, y=[int(feedback)] * X.shape[0])
        for X, feedback in [(_url2mat_or_None(url), feedback)
                            for url, (feedback,
                                      _) in list(training_db().iteritems())]
        if X is not None
    ]
    X = np.concatenate([z['X'] for z in Xyw], axis=0)
    y = np.concatenate([z['y'] for z in Xyw], axis=0)
    w = np.concatenate(
        [[1.0 / z['X'].shape[0]] * z['X'].shape[0] for z in Xyw], axis=0)
    model = _new_model()
    model.fit(X=X, y=y, sample_weight=w)

    _set_model(model)

    msg = "Classifier Score: {score}".format(score=model.score(X=X, y=y))
    log.info(msg)
    return msg

@@ -1889,34 +1889,16 @@
   Xy
-w
  = %5B
-%0A        dict(X=
+(
 X, 
-y=
 %5Bint
@@ -1930,67 +1930,8 @@
 %0A   
-     for X, feedback in %5B(_url2mat_or_None(url), feedback)%0A
     
@@ -1953,17 +1953,16 @@
         
- 
 for url,
@@ -1973,16 +1973,18 @@
 edback,%0A
+  
         
@@ -2060,16 +2060,18 @@
         
+  
 if X is 
@@ -2078,21 +2078,16 @@
 not None
-%0A    
 %5D%0A    X 
@@ -2108,27 +2108,26 @@
 te(%5B
-z%5B'X'%5D for z
+X_ for X_, _
  in Xy
-w
 %5D, a
@@ -2161,27 +2161,26 @@
 te(%5B
-z%5B'y'%5D for z
+y_ for _, y_
  in Xy
-w
 %5D, a
@@ -2194,100 +2194,64 @@
     
-w = np.concatenate(%0A        %5B%5B1.0 / z%5B'X'%5D.shape%5B0%5D%5D * z%5B'X'%5D.shape%5B0%5D for z in Xyw%5D, axis=0
+    for X, feedback in %5B(_url2mat_or_None(url), feedback
 )%0A  
@@ -2299,25 +2299,8 @@
  y=y
-, sample_weight=w
 )%0A%0A 

Revert "weigh training set by inverse number of sentences"