@@ -0,0 +1,2172 @@
+import pandas as pd%0Aimport numpy as np%0Aimport cPickle as pickle%0Aimport sys%0Aimport os%0Asys.path.insert(0, '../preproc')%0Afrom df_build_func import get_all_dates%0A%0Ana_cutoff_rate = 0.4%0A#csv_df_full = os.getcwd()+'/../../data/india/csv_preprocessed/wholesale_daily/india_original_wholesale_daily.csv'%0Acsv_df_full = os.getcwd() + '/../preproc/india_original_wholesale_daily_'+str(na_cutoff_rate)+'.csv'%0A%0Adef subsample(df, begin_date):%0A    df%5B'date'%5D = pd.to_datetime(df%5B'date'%5D)%0A    begin_date = pd.to_datetime(begin_date)%0A    return df%5Bdf%5B'date'%5D %3E= begin_date%5D%0A%0Adef na_analysis(df):%0A    all_dates = get_all_dates(df,'D')%0A    all_states = list(set(df%5B'state'%5D))%0A    all_prod_subs = %5B%5D%0A    for (product, subproduct), group in df.groupby(%5B'product', 'subproduct'%5D):%0A        if subproduct != 'Other':%0A            all_prod_subs.append((product, subproduct))%0A%0A%0A    num_cities = pd.DataFrame(columns=all_states, index=all_prod_subs)%0A    best_non_na = pd.DataFrame(columns=all_states, index=all_prod_subs)%0A    #find good subproduct across all regions%0A    for product, prod_group in df.groupby('product'):%0A        for subproduct, sub_group in prod_group.groupby('subproduct'):%0A            if subproduct != 'Other':%0A                for state, stat_group in sub_group.groupby('state'):%0A                    num_cities%5Bstate%5D%5B(product, subproduct)%5D = stat_group.shape%5B0%5D / len(all_dates)%0A                    valid_counts = %5B%5D%0A                    for city, city_group in stat_group.groupby('city'):%0A                        valid_counts.append(city_group%5B'price'%5D.count() * 1. / len(all_dates))%0A                    best_non_na%5Bstate%5D%5B(product, subproduct)%5D = max(valid_counts)%0A%0A    return num_cities, best_non_na%0A%0A%0A%0Aif __name__ == '__main__':%0A    df = pd.read_csv(csv_df_full)%0A%0A    num_cities, best_non_na = na_analysis(df)%0A    num_cities.to_csv('num_cities_'+str(na_cutoff_rate)+'.csv')%0A    best_non_na.to_csv('best_non_na_'+str(na_cutoff_rate)+'.csv')%0A%0A    df_3y = subsample(df, '2011-01-01')%0A    num_cities_3y, best_non_na_3y = na_analysis(df_3y)%0A    num_cities_3y.to_csv('num_cities_3y_'+str(na_cutoff_rate)+'.csv')%0A    best_non_na_3y.to_csv('best_non_na_3y_'+str(na_cutoff_rate)+'.csv')%0A

add na_analysis script