# -*- coding: utf-8 -*-
'''
Created on Feb 25, 2014

@author: Chunwei Yan @ PKU
@mail:  yanchunwei@outlook.com
'''
from __future__ import division
import sys
sys.path.append('..')
import csv
import cPickle as pickle
import time
import numpy
import theano
from theano import tensor as T
from utils import Timeit

class Dataset(object):
    def __init__(self, ori_dataset_ph=None, pk_data_ph=None):
        '''
        :parameters:
            data_ph: string
                path to train dataset

            train_prob: float
                the proportion of train set
        '''
        self.data_ph = ori_dataset_ph
        self.pk_data_ph = pk_data_ph
        self.records = []
        self.labels = []

    def load_ori_dataset(self):
        print 'load data ...'
        timeit = Timeit()
        with open(self.data_ph) as f:
            reader = csv.reader(f)
            for i,ls in enumerate(reader):
                if i == 0:
                    continue
                if i % 1000 == 0:
                    print '> load\t%d\trecords' % i
                label = int(ls[0])
                record = [int(r) for r in ls[1:]]
                self.records.append(record)
                self.labels.append(label)
        timeit.print_time()

    def tofile(self):
        print '... save data in pickle format'
        timeit = Timeit()
        dataset = (self.labels, self.records,)
        with open(self.pk_data_ph, 'wb') as f:
            pickle.dump(dataset, f)
        timeit.print_time()

    def fromfile(self):
        print '... load dataset from :\t %s' % self.pk_data_ph
        timeit = Timeit()
        with open(self.pk_data_ph, 'rb') as a:
            self.labels, self.records = pickle.load(a)
        timeit.print_time()
        print '... done'
        
    def trans_data_type(self, train_prob=0.8):
        print 'trains data ...'
        timeit = Timeit()
        self.train_prob = train_prob
        n_records = len(self.labels)
        n_trainset = int(n_records * self.train_prob)

        self.trainset = (self.labels[:n_trainset], self.records[:n_trainset],)
        # validation set
        self.validset = (self.labels[n_trainset:], self.records[n_trainset:],)
        print 'load records', len(self.labels)
        # train set
        labels, records = self.trainset
        labels = numpy.array(labels).astype(theano.config.floatX)
        records = numpy.array(records).astype(theano.config.floatX)
        self.trainset = (labels, records)
        # validate set
        labels, records = self.validset
        labels = numpy.array(labels).astype(theano.config.floatX)
        records = numpy.array(records).astype(theano.config.floatX)
        self.validset = (labels, records)
        timeit.print_time()
        return self.trainset, self.validset



if __name__ == '__main__':
    dataset = Dataset('./trainset.csv', './dataset.pk')
    dataset.load_ori_dataset()
    dataset.tofile()
    #dataset.fromfile()
    #trainset, validset = dataset.trans_data_type()
    #print trainset.shape, validset.shape

@@ -173,16 +173,42 @@
 d('..')%0A
+sys.path.append('../../')%0A
 import c
@@ -329,16 +329,96 @@
 t Timeit
+%0Afrom models.stacked_autoencoder import StackedAutoEncoder%0A%0AN_PIXEL_VALUES = 256
 %0A%0Aclass 
@@ -1347,32 +1347,713 @@
 t.print_time()%0A%0A
+    def load_dataset_to_norm_float(self):%0A        '''%0A        load original dataset and transform%0A        value of each pixel to 0-1 float%0A        '''%0A        print 'load data ...'%0A        timeit = Timeit()%0A        with open(self.data_ph) as f:%0A            reader = csv.reader(f)%0A            for i,ls in enumerate(reader):%0A                if i == 0:%0A                    continue%0A                if i %25 1000 == 0:%0A                    print '%3E load%5Ct%25d%5Ctrecords' %25 i%0A                label = int(ls%5B0%5D)%0A                record = %5Bint(r)/N_PIXEL_VALUES for r in ls%5B1:%5D%5D%0A                self.records.append(record)%0A                self.labels.append(label)%0A        timeit.print_time()%0A%0A%0A
     def tofile(s
@@ -2282,32 +2282,33 @@
 t.print_time()%0A%0A
+%0A
     def fromfile
@@ -2559,16 +2559,17 @@
 . done'%0A
+%0A
         
@@ -3582,16 +3582,226 @@
 idset%0A%0A%0A
+class Trainer(object):%0A    def __init__(self, pk_data_ph):%0A        self.dataset = Dataset(pk_data_ph = pk_data_ph)%0A%0A    def _init(self):%0A        self.dataset.fromfile()%0A        self.sA = StackedAutoEncoder()%0A%0A%0A
 %0Aif __na
@@ -3863,16 +3863,27 @@
 sv', './
+norm_float_
 dataset.
@@ -3887,24 +3887,25 @@
 et.pk')%0A    
+#
 dataset.load
@@ -3915,24 +3915,65 @@
 i_dataset()%0A
+    dataset.load_dataset_to_norm_float()%0A
     dataset.

add load_dataset_to_norm_float method