from paddle.trainer.PyDataProvider2 import *


# Define a py data provider
@provider(
    input_types={'pixel': dense_vector(28 * 28),
                 'label': integer_value(10)})
def process(settings, filename):  # settings is not used currently.
    imgf = filename + "-images-idx3-ubyte"
    labelf = filename + "-labels-idx1-ubyte"
    f = open(imgf, "rb")
    l = open(labelf, "rb")

    f.read(16)
    l.read(8)

    # Define number of samples for train/test
    if "train" in filename:
        n = 60000
    else:
        n = 10000

    for i in range(n):
        label = ord(l.read(1))
        pixels = []
        for j in range(28 * 28):
            pixels.append(float(ord(f.read(1))) / 255.0)
        yield {"pixel": pixels, 'label': label}

    f.close()
    l.close()

@@ -38,17 +38,29 @@
 mport *%0A
+import numpy
 %0A
-
 %0A# Defin
@@ -184,16 +184,55 @@
 lue(10)%7D
+,%0A    cache=CacheType.CACHE_PASS_IN_MEM
 )%0Adef pr
@@ -588,172 +588,241 @@
 000%0A
-%0A
     
-for i in range(n):%0A        label = ord(l.read(1))%0A        pixels = %5B%5D%0A        for j in range(28 * 28):%0A  
+%0A    images = numpy.fromfile(f, 'ubyte', count=n*28*28).reshape((n, 28*28)).astype('float32')%0A    images = images / 255.0 * 2.0 - 1.0    %0A    labels = numpy.fromfile(l, 'ubyte', count=n).astype(%22int%22)%0A
     
+%0A
     
-  pixels.append(float(ord(f.read(1))) / 255.0)
+for i in xrange(n):
 %0A   
@@ -846,14 +846,20 @@
 l%22: 
-pixels
+images%5Bi, :%5D
 , 'l
@@ -874,10 +874,18 @@
 abel
-%7D%0A
+s%5Bi%5D%7D%0A    
 %0A   

improve demo/mnist dataProvider speed