#!/usr/bin/env python

import numpy as np
import sys
import tensorflow as tf
from text_rnn import TextRNNClassifier
import preproc

NCLASS = 1999
NWORDS = 411721
SEQ_LEN = 150

def inp_fn(data):
    inp_x = []
    inp_y = []
    for inst in data:
        flds = inst.split('\t')
        label = int(flds[0])
        feats = map(int, flds[1:])
        inp_y.append(preproc.sparse2dense([label], ndim=NCLASS))
        inp_x.append(preproc.zero_padding(feats, SEQ_LEN))
    return np.array(inp_x), np.array(inp_y)

def get_fbatch(fp, batch_size=1):
    out = []
    for line in fp:
        out.append(line.rstrip('\n'))
        if len(out) >= batch_size:
            break
    return out

train_file = '/Users/tchen/projects/ZhihuCup/mdl_cnn1/feat_train/trnvld_feature.tsv'
train_fp = open(train_file)

mdl = TextRNNClassifier(
    seq_len=SEQ_LEN,
    emb_dim=128,
    nclass=NCLASS,
    vocab_size=NWORDS,
    reg_lambda=0.0,
    lr=1e-3)

sess = tf.Session()
sess.run(tf.global_variables_initializer())
niter = 0
while niter < 1000:
    niter += 1
    batch_data = get_fbatch(train_fp, 64)
    #print batch_data[0]
    if len(batch_data) <= 0:
        break
    train_x, train_y = inp_fn(batch_data)
    mdl.train_step(sess, train_x, train_y)
    #print train_x[0]
    #print train_y[0][590:600]
    print mdl.eval_step(sess, train_x, train_y)

train_fp.close()
sess.close()


@@ -46,16 +46,54 @@
 ort sys%0A
+sys.path.append('../../MLFlow/utils')%0A
 import t
@@ -154,19 +154,20 @@
 %0Aimport 
-pre
+data
 proc%0A%0ANC
@@ -177,12 +177,9 @@
 S = 
-1999
+2
 %0ANWO
@@ -188,14 +188,13 @@
 S = 
-411721
+18764
 %0ASEQ
@@ -201,17 +201,17 @@
 _LEN = 1
-5
+0
 0%0A%0Adef i
@@ -388,27 +388,28 @@
 np_y.append(
-pre
+data
 proc.sparse2
@@ -462,11 +462,12 @@
 end(
-pre
+data
 proc
@@ -546,182 +546,10 @@
 y)%0A%0A
-def get_fbatch(fp, batch_size=1):%0A    out = %5B%5D%0A    for line in fp:%0A        out.append(line.rstrip('%5Cn'))%0A        if len(out) %3E= batch_size:%0A            break%0A    return out%0A%0A
+# 
 trai
@@ -640,16 +640,65 @@
 in_f
-p = open
+ile = './rt-polarity.shuf'%0Afreader = dataproc.BatchReader
 (tra
@@ -769,11 +769,11 @@
 dim=
-128
+256
 ,%0A  
@@ -976,29 +976,26 @@
 a = 
+freader.
 get_
-f
 batch(
-train_fp, 
 64)%0A
@@ -1151,61 +1151,8 @@
 _y)%0A
-    #print train_x%5B0%5D%0A    #print train_y%5B0%5D%5B590:600%5D%0A
     

use test data