import numpy as np
import scipy as sp
import scipy.sparse as sparse
# import pandas as pd
import itertools
# import math
# import matplotlib.pyplot as plt
from collections import deque
# import pickle

from preprocess.feature_hash import parse_row

k = 18
# Initialize weight vector
w = sparse.csc_matrix((1 << k, 1), dtype=np.float)
alpha = 0.1

# Regularization parameter
l2_reg = 0.4
intercept = 0.0
tau = 10000.


def train(samples, w):
    """Online lbfgs training"""
    batch_size = 10000
    l2_r = 0.04
    lamb_const = 1.
#     c = 0.1
    c = 1.0
    m = 10 # size of (s, y) ring buffer
    mini_batch_buffer = deque([])
    for t, row in enumerate(samples):
        mini_batch_buffer.append(parse_row(row))
        if (t + 1) % batch_size == 0 and t > 1:
            print 'batch: ' + str((t + 1) / batch_size)
            if t + 1 == batch_size:
                # For the first batch, set the intercept to line up with the CTR
                w = set_first_intercept(w, list(mini_batch_buffer))
                print w
            w_g_l = olbfgs_batch(list(mini_batch_buffer), w, l2_r, m, c, lamb_const)
            return (w_g_l[0], w_g_l[1], mini_batch_buffer, w_g_l[2], w_g_l[3])
            mini_batch_buffer = deque([])
#         y_hat = lr_predict(w, x)
#         w = update_gradient(y, y_hat, w, x)
        
#         loss += logloss(y_hat, y)
#         if t % 5000 == 0 and t > 1:
#             print('%s\tencountered: %d\tcurrent logloss: %f' % (
#                 datetime.datetime.now(), t, loss/t))
#     return w


def set_first_intercept(w, X1):
    ctr_counts = reduce(
        lambda a, b: (a[0] + b[0], a[1] + b[1]),
        map(
            lambda x: (x[0], 1),
            X1))
    ctr_ratio = float(ctr_counts[0]) / float(ctr_counts[1]) + 10**-6
    new_intercept = -np.log(ctr_ratio**-1 - 1)
    w[0, 0] = new_intercept
    return w


def set_first_intercept_spark(w, X1):
    ctr_counts = X1.map(
        lambda x: (x[0], 1),
        preservesPartitioning=True
    ).fold(
        (0, 0),
        lambda a, b: (a[0] + b[0], a[1] + b[1])
    )
    ctr_ratio = float(ctr_counts[0]) / float(ctr_counts[1]) + 10**-6
    new_intercept = -np.log(ctr_ratio**-1 - 1)
    w[0, 0] = new_intercept
    return w

@@ -1526,32 +1526,34 @@
      return w%0A%0A%0A
+# 
 def set_first_in
@@ -1560,32 +1560,34 @@
 tercept(w, X1):%0A
+# 
     ctr_counts =
@@ -1595,16 +1595,18 @@
 reduce(%0A
+# 
         
@@ -1646,16 +1646,18 @@
  b%5B1%5D),%0A
+# 
         
@@ -1657,24 +1657,26 @@
        map(%0A
+# 
             
@@ -1688,32 +1688,33 @@
 a x: (x%5B0%5D, 1),%0A
+#
             X1))
@@ -1713,13 +1713,16 @@
     
+ 
 X1))%0A
+# 
     
@@ -1778,32 +1778,34 @@
 ts%5B1%5D) + 10**-6%0A
+# 
     new_intercep
@@ -1827,36 +1827,38 @@
 _ratio**-1 - 1)%0A
+#
     
+ 
 w%5B0, 0%5D = new_in
@@ -1857,32 +1857,34 @@
 = new_intercept%0A
+# 
     return w%0A%0A%0Ad
@@ -1882,16 +1882,18 @@
 urn w%0A%0A%0A
+# 
 def set_
@@ -1922,16 +1922,18 @@
 w, X1):%0A
+# 
     ctr_
@@ -1949,24 +1949,25 @@
 X1.map(%0A
+#
         
 lambda x
@@ -1950,32 +1950,33 @@
 1.map(%0A#        
+ 
 lambda x: (x%5B0%5D,
@@ -1980,16 +1980,18 @@
 0%5D, 1),%0A
+# 
         
@@ -2017,16 +2017,18 @@
 ng=True%0A
+# 
     ).fo
@@ -2031,24 +2031,25 @@
 ).fold(%0A
+#
         
 (0, 0),%0A
@@ -2044,16 +2044,17 @@
     
+ 
 (0, 0),%0A
     
@@ -2049,24 +2049,25 @@
 (0, 0),%0A
+#
         
 lambda a
@@ -2050,32 +2050,33 @@
 0, 0),%0A#        
+ 
 lambda a, b: (a%5B
@@ -2103,14 +2103,18 @@
 1%5D)%0A
+# 
     )%0A
+# 
     
@@ -2178,20 +2178,22 @@
  10**-6%0A
+#
     
+ 
 new_inte
@@ -2227,16 +2227,18 @@
 -1 - 1)%0A
+# 
     w%5B0,
@@ -2257,16 +2257,18 @@
 tercept%0A
+# 
     retu

Move intercept methods to warm start module