#!/opt/anaconda/bin/python

#
# Copyright 2013 Baxter, Lovell, Mangsingkha, Saeedi
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
import sys
#
import tabular_predDB.python_utils.data_utils as du
import tabular_predDB.python_utils.file_utils as fu
import tabular_predDB.python_utils.xnet_utils as xu
import tabular_predDB.python_utils.general_utils as gu
import tabular_predDB.LocalEngine as LE


def initialize_helper(table_data, dict_in):
    M_c = table_data['M_c']
    M_r = table_data['M_r']
    T = table_data['T']
    initialization = dict_in['initialization']
    SEED = dict_in['SEED']
    engine = LE.LocalEngine(SEED)
    M_c_prime, M_r_prime, X_L, X_D = \
               engine.initialize(M_c, M_r, T, initialization=initialization)
    #
    ret_dict = dict(X_L=X_L, X_D=X_D)
    return ret_dict

def analyze_helper(table_data, dict_in):
    M_c = table_data['M_c']
    T = table_data['T']
    X_L = dict_in['X_L']
    X_D = dict_in['X_D']
    kernel_list = dict_in['kernel_list']
    n_steps = dict_in['n_steps']
    c = dict_in['c']
    r = dict_in['r']
    max_time = dict_in['max_time']
    SEED = dict_in['SEED']
    engine = LE.LocalEngine(SEED)
    X_L_prime, X_D_prime = engine.analyze(M_c, T, X_L, X_D, kernel_list=kernel_list,
                                          n_steps=n_steps, c=c, r=r,
                                          max_time=max_time)
    #
    ret_dict = dict(X_L=X_L_prime, X_D=X_D_prime)
    return ret_dict

def time_analyze_helper(table_data, dict_in):
    start_dims = du.get_state_shape(dict_in['X_L'])
    with gu.Timer('time_analyze_helper', verbose=False) as timer:
        inner_ret_dict = analyze_helper(table_data, dict_in)
    end_dims = du.get_state_shape(inner_ret_dict['X_L'])
    T = table_data['T']
    table_shape = (len(T), len(T[0]))
    ret_dict = dict(
        table_shape=table_shape,
        start_dims=start_dims,
        end_dims=end_dims,
        elapsed_secs=timer.elapsed_secs,
        kernel_list=dict_in['kernel_list'],
        n_steps=dict_in['n_steps'],
        )
    return ret_dict

method_lookup = dict(
    initialize=initialize_helper,
    analyze=analyze_helper,
    time_analyze=time_analyze_helper,
    )

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--table_data_filename', type=str,
                        default=xu.default_table_data_filename)
    args = parser.parse_args()
    table_data_filename = args.table_data_filename
    table_data = fu.unpickle(table_data_filename)
    #
    from signal import signal, SIGPIPE, SIG_DFL 
    signal(SIGPIPE,SIG_DFL) 
    for line in sys.stdin:
        key, dict_in = xu.parse_hadoop_line(line)
        if dict_in is None:
            continue
        command = dict_in['command']
        method = method_lookup[command]
        ret_dict = method(table_data, dict_in)
        print key, ret_dict


@@ -642,16 +642,26 @@
 ense.%0A#%0A
+import os%0A
 import s
@@ -663,16 +663,16 @@
 ort sys%0A
-
 #%0Aimport
@@ -1974,24 +1974,795 @@
 n ret_dict%0A%0A
+def checkpoint_analyze_helper(table_data, dict_in):%0A    original_n_steps = dict_in%5B'n_steps'%5D%0A    original_SEED = dict_in%5B'SEED'%5D%0A    chunk_size = dict_in%5B'chunk_size'%5D%0A    checkpoint_filename_prefix = dict_in%5B'checkpoint_filename_prefix'%5D%0A    #%0A    dict_in%5B'n_steps'%5D = chunk_size%0A    steps_done = 0%0A    while steps_done %3C original_n_steps:%0A        # FIXME: modify SEED%0A        ith_chunk = steps_done / chunk_size%0A        dict_out = analyze_helper(table_data, dict_in)%0A        dict_in.update(dict_out)%0A        # write to hdfs%0A        checkpoint_filename = '%25s_%25s' %25 (checkpoint_filename_prefix, ith_chunk)%0A        fu.pickle(dict_out, checkpoint_filename)%0A        HE.put_hdfs(None, checkpoint_filename)%0A        #%0A        steps_taken += step_size%0A    return dict_out%0A    %0A
 def time_ana
@@ -3437,24 +3437,24 @@
 yze_helper,%0A
-
     time_ana
@@ -3475,24 +3475,74 @@
 yze_helper,%0A
+    checkpoint_analyze=checkpoint_analyze_helper,%0A
     )%0A%0Aif __

add checkpoint_analyze