# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load in 


import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

import os
print(os.listdir("../input"))

# Any results you write to the current directory are saved as output.

import sys

sys.path.insert(0, '../input/wordbatch-133/wordbatch/')
sys.path.insert(0, '../input/randomstate/randomstate/')
import wordbatch
from sklearn.metrics import mean_squared_error

from wordbatch.extractors import WordHash
from wordbatch.models import FM_FTRL
from wordbatch.data_utils import *
import threading
import pandas as pd
from sklearn.metrics import roc_auc_score
import time
import numpy as np
import gc
from contextlib import contextmanager
from sklearn import preprocessing
from sklearn.feature_extraction.text import TfidfVectorizer,CountVectorizer
import regex as re
from scipy.sparse import hstack
from nltk.corpus import stopwords
from sklearn.model_selection import KFold
stopWords = stopwords.words('russian')

@contextmanager
def timer(name):
    t0 = time.time()
    yield
    print('[{}] done in {:.0f} s'.format(name,(time.time() - t0)))

import os, psutil

start_time = time.time()


def df2csr(df, traindex, testdex, len_train):
    df.reset_index(drop=True, inplace=True)
    with timer("Adding Time Variables"):
        df["Year"] = df["activation_date"].dt.year
        df["Date of Year"] = df['activation_date'].dt.dayofyear # Day of Year
        df["Weekday"] = df['activation_date'].dt.weekday
        df["Weekd of Year"] = df['activation_date'].dt.week
        df["Day of Month"] = df['activation_date'].dt.day
        df["Quarter"] = df['activation_date'].dt.quarter
        df.drop(["activation_date","image"],axis=1,inplace=True)

    with timer("Encode Variables"):
        categorical = ["user_id","region","city","parent_category_name","category_name","item_seq_number","user_type"]
        messy_categorical = ["param_1","param_2","param_3"] # Need to find better technique for these
        # Encoder:
        lbl = preprocessing.LabelEncoder()
        for col in categorical + messy_categorical:
            df[col] = lbl.fit_transform(df[col].astype(str))
        del lbl
        gc.collect()

    with timer("Log features"):
        for fea in ['price','image_top_1']:
            df[fea]= np.log2(1 + df[fea].values).astype(int)

    with timer("splitting to train and test"):
        trn = df[:len_train].copy()
        print("Training Set shape",trn.shape)
        tst = df[len_train:].copy()
        print("Submission Set Shape: {} Rows, {} Columns".format(*tst.shape))
        del df
        gc.collect()

    with timer("Generating str_array"):
        trn_str_array = ("Y" + trn['Year'].astype(str) \
            + " DY" + trn['Date of Year'].astype(str) \
            + " WD" + trn['Weekday'].astype(str) \
            + " WDY" + trn['Weekd of Year'].astype(str) \
            + " DM" + trn['Day of Month'].astype(str) \
            + " Q" + trn['Quarter'].astype(str) \
            + " UI" + trn['user_id'].astype(str) \
            + " R" + trn['region'].astype(str) \
            + " C" + trn['city'].astype(str) \
            + " PCN" + trn['parent_category_name'].astype(str) \
            + " CN" + trn['category_name'].astype(str) \
            + " ISN" + trn['item_seq_number'].astype(str) \
            + " UT" + trn['user_type'].astype(str) \
            + " PO" + trn['param_1'].astype(str) \
            + " PT" + trn['param_2'].astype(str) \
            + " PTT" + trn['param_3'].astype(str) \
            + " T" + trn['title'].astype(str) \

            + " UXC" + trn['user_id'].astype(str)+"_"+trn['city'].astype(str) \
            + " RXC" + trn['region'].astype(str)+"_"+trn['city'].astype(str) \
            + " CXI" + trn['category_name'].astype(str)+"_"+trn['item_seq_number'].astype(str) \
            + " UXU" + trn['user_type'].astype(str)+"_"+trn['user_id'].astype(str) \
            + " PXI" + trn['parent_category_name'].astype(str)+"_"+trn['item_seq_number'].astype(str) \

            + " P" + trn['price'].astype(str) \
            + " IT" + trn['image_top_1'].astype(str)).values
        tst_str_array = ("Y" + tst['Year'].astype(str) \
            + " DY" + tst['Date of Year'].astype(str) \
            + " WD" + tst['Weekday'].astype(str) \
            + " WDY" + tst['Weekd of Year'].astype(str) \
            + " DM" + tst['Day of Month'].astype(str) \
            + " Q" + tst['Quarter'].astype(str) \
            + " UI" + tst['user_id'].astype(str) \
            + " R" + tst['region'].astype(str) \
            + " C" + tst['city'].astype(str) \
            + " PCN" + tst['parent_category_name'].astype(str) \
            + " CN" + tst['category_name'].astype(str) \
            + " ISN" + tst['item_seq_number'].astype(str) \
            + " UT" + tst['user_type'].astype(str) \
            + " PO" + tst['param_1'].astype(str) \
            + " PT" + tst['param_2'].astype(str) \
            + " PTT" + tst['param_3'].astype(str) \
            + " T" + tst['title'].astype(str) \

            + " UXC" + tst['user_id'].astype(str)+"_"+tst['city'].astype(str) \
            + " RXC" + tst['region'].astype(str)+"_"+tst['city'].astype(str) \
            + " CXI" + tst['category_name'].astype(str)+"_"+tst['item_seq_number'].astype(str) \
            + " UXU" + tst['user_type'].astype(str)+"_"+tst['user_id'].astype(str) \
            + " PXI" + tst['parent_category_name'].astype(str)+"_"+tst['item_seq_number'].astype(str) \

            + " P" + tst['price'].astype(str) \
            + " IT" + tst['image_top_1'].astype(str)).values
    del trn
    gc.collect()
    del tst
    gc.collect()
    return trn_str_array , tst_str_array

def char_analyzer(text):
    """
    This is used to split strings in small lots
    anttip saw this in an article 
    so <talk> and <talking> would have <Tal> <alk> in common
    should be similar to russian I guess
    """
    tokens = text.split()
    return [token[i: i + 3] for token in tokens for i in range(len(token) - 2)]
    
def sigmoid(x):
    return 1 / (1 + np.exp(-x))

if __name__ == '__main__':
    print("\nData Load Stage")
    training = pd.read_csv('../input/train.csv', index_col="item_id", parse_dates=["activation_date"])
    len_train = len(training)
    traindex = training.index
    testing = pd.read_csv('../input/test.csv', index_col="item_id", parse_dates=["activation_date"])
    testdex = testing.index
    labels = training['deal_probability'].values
    training.drop("deal_probability", axis=1, inplace=True)
    print('Train shape: {} Rows, {} Columns'.format(*training.shape))
    print('Test shape: {} Rows, {} Columns'.format(*testing.shape))
    # Combine Train and Test
    df_all = pd.concat([training, testing], axis=0)
    trnshape = training.shape[0]
    del training, testing
    gc.collect()
    print('\nAll Data shape: {} Rows, {} Columns'.format(*df_all.shape))

    df_all['description'] = df_all['description'].fillna("")
    df_all["text_feature"] = df_all.apply(lambda row: " ".join([str(row["param_1"]),
                                                                str(row["param_2"]), str(row["param_3"])]), axis=1)

    with timer("Tfidf on word"):
        word_vectorizer = TfidfVectorizer(
            sublinear_tf=True,
            strip_accents='unicode',
            tokenizer=lambda x: re.findall(r'[^\p{P}\W]+', x),
            analyzer='word',
            token_pattern=None,
            stop_words=stopWords,
            ngram_range=(1, 1),
            max_features=40000)
        X = word_vectorizer.fit_transform(df_all['description'])
        train_word_features = X[:trnshape]
        test_word_features = X[trnshape:]
        del (X)

    with timer("Tfidf on word"):
        word_vectorizer = TfidfVectorizer(
            sublinear_tf=True,
            strip_accents='unicode',
            tokenizer=lambda x: re.findall(r'[^\p{P}\W]+', x),
            analyzer='word',
            token_pattern=None,
            stop_words=stopWords,
            ngram_range=(1, 2),
            max_features=16000)
        X = word_vectorizer.fit_transform(df_all['title'])
        train_title_word_features = X[:trnshape]
        test_title_word_features = X[trnshape:]
        del (X)
    with timer("CountVectorizer on word"):
        word_vectorizer = CountVectorizer(
            ngram_range=(1, 1),)
        X = word_vectorizer.fit_transform(df_all['text_feature'])
        train_text_feature_word_features = X[:trnshape]
        test_text_feature_word_features = X[trnshape:]
        del (X)

    with timer("Tfidf on char n_gram"):
        char_vectorizer = TfidfVectorizer(
            sublinear_tf=True,
            strip_accents='unicode',
            tokenizer=char_analyzer,
            analyzer='word',
            ngram_range=(1, 4),
            max_features=16000)
        X = char_vectorizer.fit_transform(df_all['title'])
        train_char_features = X[:trnshape]
        test_char_features = X[trnshape:]
        del (X)

    with timer("Tfidf on char n_gram 1,1"):
        char_vectorizer = TfidfVectorizer(
            sublinear_tf=True,
            strip_accents='unicode',
            tokenizer=char_analyzer,
            analyzer='word',
            ngram_range=(1, 1),
            max_features=16000)
        X = char_vectorizer.fit_transform(df_all['description'])
        train_char6_features = X[:trnshape]
        test_char6_features = X[trnshape:]
        del (X)

    with timer("Transforming other features"):
        trn_str_arrays, tst_str_arrays = df2csr(df_all, traindex, testdex, len_train)
        del df_all
        gc.collect

        batchsize = 10000000
        D = 2 ** 10

        wb = wordbatch.WordBatch(None, extractor=(WordHash, {"ngram_range": (1, 1), "analyzer": "word",
                                                             "lowercase": False, "n_features": D,
                                                             "norm": None, "binary": True})
                                 , minibatch_size=batchsize // 80, procs=8, freeze=True, timeout=1800, verbose=0)

        trn_str_array = wb.transform(trn_str_arrays)
        del trn_str_arrays
        gc.collect()
        tst_str_array = wb.transform(tst_str_arrays)
        del tst_str_arrays
        gc.collect()
        del wb
        gc.collect()

    print("DONE")
    with timer("Stacking Features"):
        train_features = hstack(
            [
                train_char_features,
                train_word_features,
                train_title_word_features,
                train_char6_features,
                train_text_feature_word_features
            ]
        )
        del train_word_features, train_char_features,train_title_word_features,train_char6_features,train_text_feature_word_features

        train_features = hstack(
            [
                train_features,
                trn_str_array

            ]
        ).tocsr()
        del trn_str_array
        gc.collect()
        print("Fin Stack Train")
        test_features = hstack(
            [
                test_char_features,
                test_word_features,
                tst_str_array,
                test_title_word_features,
                test_char6_features,
                test_text_feature_word_features

            ]
        ).tocsr()
        del test_word_features, tst_str_array, test_char_features,test_title_word_features,test_char6_features,test_text_feature_word_features
        gc.collect()
        print("Fin Stack test")

    with timer("Scoring FM FTRL"):
        kf = KFold(n_splits=5, random_state=42, shuffle=True)

        model = FM_FTRL(
            alpha=0.02, beta=0.01, L1=0.00001, L2=30.0,
            D=train_features.shape[1], alpha_fm=0.1,
            L2_fm=0.5, init_fm=0.01, weight_fm=50.0,
            D_fm=200, e_noise=0.0, iters=7,#5
            inv_link="identity", e_clip=1.0, threads=6, use_avx=1, verbose=1 #inv_link="identity", e_clip=1.0, threads=1, use_avx=1, verbose=1
        )

        fold_id = -1
        X = train_features
        y = labels
        x_test = test_features

        val_predict= np.zeros(y.shape)

        for train_index, val_index in kf.split(X):
            fold_id += 1
            x_train, x_val = X[train_index], X[val_index]
            y_train, y_val = y[train_index], y[val_index]


            model.fit(x_train, y_train)


            train_pred = model.predict(x_train)
            y_pred = model.predict(x_val)
            val_predict[val_index] = y_pred
            rmse = mean_squared_error(y_val, y_pred) ** 0.5
            print('valid score: {}'.format(rmse))
            sub = pd.read_csv('../input/sample_submission.csv')
            pred = model.predict(x_test)
            sub['deal_probability'] = pred
            sub['deal_probability'].clip(0.0, 1.0, inplace=True)
            print("Output Prediction CSV")
            sub.to_csv('subm/wordbatch_fmtrl_submissionV3_{}.csv'.format(fold_id), index=False)



        # val_predicts = pd.DataFrame(data=val_predict, columns= labels)
        # val_predicts['user_id'] = train_user_ids
        # val_predicts['item_id'] = train_item_ids
        # val_predicts.to_csv('subm/wordbatch_fmtrl_submission_train.csv', index=False)
'''
valid score: 0.22830226415753876

FINAL RMSE 0.22179138519107305, char n_gram 1,1 description

FINAL RMSE 0.22265192127859398 , char n_gram 1,6
FINAL RMSE 0.22265821448778575, char n_gram 1,4
[Scoring FM FTRL] done in 347 s

FINAL RMSE 0.22547715723618647
[Scoring FM FTRL] done in 692 s

FINAL RMSE 0.22721282218651886
[Scoring FM FTRL] done in 641 s

average valid acc:0.943010665961
Save epoch9 at ../data/MTCNN_model/PNet_landmark/PNet

增加一个text_feature
DONE
Fin Stack Train
Fin Stack test
[Stacking Features] done in 22 s
Total e: 189046.20930299623
Total e: 183864.5931029993
Total e: 182518.3256617243
Total e: 181750.69846973865
Total e: 181233.67554650828
Total e: 180853.83779048597
Total e: 180559.64456537634
valid score: 0.2275048093223973
Output Prediction CSV
Total e: 188873.4560661744
Total e: 183712.39039747955
Total e: 182368.29046199698
Total e: 181600.4972747812
Total e: 181083.63226694267
Total e: 180704.6173825646
Total e: 180411.32691318056
valid score: 0.2277848426611388
Output Prediction CSV
Total e: 188844.9140664788
Total e: 183665.8783948139
Total e: 182323.13800638154
Total e: 181559.40195599885
Total e: 181046.14236964888
Total e: 180669.61394944967
Total e: 180378.27942137406
valid score: 0.22792221605280613
Output Prediction CSV
Total e: 188964.6510297849
Total e: 183787.39461554418
Total e: 182442.7112902973
Total e: 181675.55625697025
Total e: 181158.00582514986
Total e: 180777.75450087097
Total e: 180483.69289019858
valid score: 0.2276856289512003
Output Prediction CSV
Total e: 188953.84776714086
Total e: 183750.32084681623
Total e: 182401.5191464039
Total e: 181635.09029368704
Total e: 181120.39854659705
Total e: 180743.25542031662
Total e: 180451.25567295877
valid score: 0.2278115387203652
Output Prediction CSV
[Scoring FM FTRL] done in 4325 s

Process finished with exit code 0

[Stacking Features] done in 21 s
Total e: 190332.13532013315
Total e: 185176.36752191
Total e: 183803.4162622352
Total e: 183022.48111174765
Total e: 182497.66910659662
valid score: 0.22830226415753876
Output Prediction CSV
Total e: 190175.97780981776
Total e: 185029.47584343897
Total e: 183657.40182914195
Total e: 182876.1451740742
Total e: 182350.14800472654
valid score: 0.22858791598445452
Output Prediction CSV
Total e: 190163.9991700796
Total e: 185020.63139812616
Total e: 183652.56760049515
Total e: 182875.41300158898
Total e: 182352.74030912982
valid score: 0.22867367692104767
Output Prediction CSV
Total e: 190237.01160260584
Total e: 185088.29752398795
Total e: 183713.19926152908
Total e: 182928.18302477512
Total e: 182400.8309299546
valid score: 0.22857778377101393
Output Prediction CSV
Total e: 190249.99448752435
Total e: 185079.2242154467
Total e: 183707.0861143455
Total e: 182926.85280229314
Total e: 182401.34665621707
valid score: 0.2285867373708063
Output Prediction CSV
'''
