<jupyter_start><jupyter_text>Data Explore  Load modules<jupyter_code>import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import matplotlib.pyplot as plt
import seaborn as sns
import time
from datetime import datetime
from collections import Counter
from subprocess import check_output
MEMBERS = pd.read_csv("../data/processed/MEMBERS_PRED2.csv", parse_dates=True)
MEMBERS.head()
MEMBERS.describe()
MEMBERS[MEMBERS["gender"].isna() == True]
STATUS = pd.read_csv("../data/raw/STATUS_02.csv")
STATUS.head()
STATUS['status'] = STATUS['status'].astype(str)
STATUS['status'].value_counts()
training = pd.merge(left = STATUS,right = MEMBERS,how = 'left',on=['msno'])
training.head()
training.info()
training = training.dropna()
training['registration_init_time'] = training['registration_init_time'].map(lambda x: datetime(int(x[:4]), int(x[5:7]), int(x[8:])))
training.head()<jupyter_output><empty_output><jupyter_text>Training data<jupyter_code># City count
fig1 = plt.figure(figsize=(12,12))
plt.subplot(411)
city_order = training['city'].unique()
city_order=sorted(city_order, key=lambda x: float(x))
sns.countplot(x="city", data=training , order = city_order)
plt.ylabel('Count', fontsize=12)
plt.xlabel('City', fontsize=12)
plt.xticks(rotation='vertical')
plt.title("Frequency of City Count", fontsize=12)
plt.show()
fig1.savefig("Frequency of City Count.png", dpi=fig1.dpi)
city_count = Counter(training['city']).most_common()
print("City Count " +str(city_count))

#Registered Via Count
fig2 = plt.figure(figsize=(12,12))
plt.subplot(412)
R_V_order = training['registered_via'].unique()
R_V_order = sorted(R_V_order, key=lambda x: str(x))
R_V_order = sorted(R_V_order, key=lambda x: float(x))
#above repetion of commands are very silly, but this was the only way I was able to diplay what I wanted
sns.countplot(x="registered_via", data=training,order = R_V_order)
plt.ylabel('Count', fontsize=12)
plt.xlabel('Registered Via', fontsize=12)
plt.xticks(rotation='vertical')
plt.title("Frequency of Registered Via Count", fontsize=12)
plt.show()
fig2.savefig("Frequency of Registered Via Count.png", dpi=fig2.dpi)
RV_count = Counter(training['registered_via']).most_common()
print("Registered Via Count " +str(RV_count))

#Gender count
fig3 = plt.figure(figsize=(12,12))
plt.subplot(413)
sns.countplot(x="gender", data=training)
plt.ylabel('Count', fontsize=12)
plt.xlabel('Gender', fontsize=12)
plt.xticks(rotation='vertical')
plt.title("Frequency of Gender Count", fontsize=12)
plt.show()
fig3.savefig("Frequency of Gender Count.png", dpi=fig2.dpi)
gender_count = Counter(training['gender']).most_common()
print("Gender Count " +str(gender_count))<jupyter_output><empty_output><jupyter_text>registration_init_time Trends exploration<jupyter_code>#registration_init_time yearly trend
training['registration_init_time_year'] = pd.DatetimeIndex(training['registration_init_time']).year
training['registration_init_time_year'] = training.registration_init_time_year.apply(lambda x: int(x) if pd.notnull(x) else "NAN" )
year_count=training['registration_init_time_year'].value_counts()
#print(year_count)
fig4 = plt.figure(figsize=(12,12))
plt.subplot(311)
year_order = training['registration_init_time_year'].unique()
year_order=sorted(year_order, key=lambda x: str(x))
year_order = sorted(year_order, key=lambda x: float(x))
sns.barplot(year_count.index, year_count.values,order=year_order)
plt.ylabel('Count', fontsize=12)
plt.xlabel('Year', fontsize=12)
plt.xticks(rotation='vertical')
plt.title("Yearly Trend of registration_init_time", fontsize=12)
plt.show()
fig4.savefig("Yearly Trend of registration_init_time.png", dpi=fig4.dpi)
year_count_2 = Counter(training['registration_init_time_year']).most_common()
print("Yearly Count " +str(year_count_2))

#registration_init_time monthly trend
training['registration_init_time_month'] = pd.DatetimeIndex(training['registration_init_time']).month
training['registration_init_time_month'] = training.registration_init_time_month.apply(lambda x: int(x) if pd.notnull(x) else "NAN" )
month_count=training['registration_init_time_month'].value_counts()
fig5 = plt.figure(figsize=(12,12))
plt.subplot(312)
month_order = training['registration_init_time_month'].unique()
month_order = sorted(month_order, key=lambda x: str(x))
month_order = sorted(month_order, key=lambda x: float(x))
sns.barplot(month_count.index, month_count.values,order=month_order)
plt.ylabel('Count', fontsize=12)
plt.xlabel('Month', fontsize=12)
plt.xticks(rotation='vertical')
plt.title("Monthly Trend of registration_init_time", fontsize=12)
plt.show()
fig5.savefig("Monthly Trend of registration_init_time.png", dpi=fig5.dpi)
month_count_2 = Counter(training['registration_init_time_month']).most_common()
print("Monthly Count " +str(month_count_2))

#registration_init_time day wise trend
training['registration_init_time_weekday'] = pd.DatetimeIndex(training['registration_init_time']).weekday_name
training['registration_init_time_weekday'] = training.registration_init_time_weekday.apply(lambda x: str(x) if pd.notnull(x) else "NAN" )
day_count=training['registration_init_time_weekday'].value_counts()
fig6 = plt.figure(figsize=(12,12))
plt.subplot(313)
#day_order = training['registration_init_time_day'].unique()
day_order = ['Monday','Tuesday','Wednesday','Thursday','Friday','Saturday','Sunday','NAN']
sns.barplot(day_count.index, day_count.values,order=day_order)
plt.ylabel('Count', fontsize=12)
plt.xlabel('Day', fontsize=12)
plt.xticks(rotation='vertical')
plt.title("Day-wise Trend of registration_init_time", fontsize=12)
plt.show()
fig6.savefig("Day-wise Trend of registration_init_time.png", dpi=fig6.dpi)
day_count_2 = Counter(training['registration_init_time_weekday']).most_common()
print("Day-wise Count " +str(day_count_2))<jupyter_output><empty_output><jupyter_text>Relation of between train data set and members Data set<jupyter_code>#Gender
gender_crosstab=pd.crosstab(training['gender'],training['status'])
ax = gender_crosstab.plot(kind='bar', stacked=True, grid=True)
fig = ax.get_figure()
fig.savefig('gender_crosstab')
gender_crosstab["Ratio"] =  gender_crosstab.iloc[:, 1] / gender_crosstab.iloc[:, 0]
gender_crosstab
#Registered Via
registered_via_crosstab=pd.crosstab(training['registered_via'],training['status'])
ax = registered_via_crosstab.plot(kind='bar', stacked=True, grid=True)
fig = ax.get_figure()
fig.savefig('registered_via_crosstab')
registered_via_crosstab["Ratio"] =  registered_via_crosstab.iloc[:, 1] / registered_via_crosstab.iloc[:, 0]
registered_via_crosstab
#city
city_crosstab=pd.crosstab(training['city'],training['status'])
ax = city_crosstab.plot(kind='bar', stacked=True, grid=True)
fig = ax.get_figure()
fig.savefig('city_crosstab')
city_crosstab["Ratio"] =  city_crosstab.iloc[:,1] / city_crosstab.iloc[:,0]
city_crosstab
#Birth Date
ax = sns.boxplot(x=training["status"],y=training["bd"],data=training)
fig = ax.get_figure()
fig.savefig('status_boxplot')<jupyter_output><empty_output><jupyter_text>Data Exploration in members data ( just for comparison with training merged dataset )<jupyter_code>MEMBERS = MEMBERS.drop(labels="Unnamed: 0", axis=1)
MEMBERS.info()
MEMBERS = MEMBERS.fillna("Unknown")
# City count in Members Data Set
fig = plt.figure(figsize=(12,12))
plt.subplot(311)
sns.countplot(x="city", data=MEMBERS)
plt.ylabel('Count', fontsize=12)
plt.xlabel('City', fontsize=12)
plt.xticks(rotation='vertical')
plt.title("Frequency of City Count in Members Data Set", fontsize=12)
plt.show()
fig.savefig("Frequency of City Count in Members Data Set.png", dpi=fig.dpi)
city_count = Counter(MEMBERS['city']).most_common()
print("City Count " +str(city_count))

#Registered Via Count in Members Data Set
fig = plt.figure(figsize=(12,12))
plt.subplot(312)
sns.countplot(x="registered_via", data=MEMBERS)
plt.ylabel('Count', fontsize=12)
plt.xlabel('Registered Via', fontsize=12)
plt.xticks(rotation='vertical')
plt.title("Frequency of Registered Via Count in Members Data Set", fontsize=12)
plt.show()
fig.savefig("Frequency of Registered Via Count in Members Data Set.png", dpi=fig.dpi)
RV_count = Counter(MEMBERS['registered_via']).most_common()
print("Registered Via Count " +str(RV_count))


#Gender count in Members Data Set
fig = plt.figure(figsize=(12,12))
plt.subplot(313)
sns.countplot(x="gender", data=MEMBERS)
plt.ylabel('Count', fontsize=12)
plt.xlabel('Gender', fontsize=12)
plt.xticks(rotation='vertical')
plt.title("Frequency of Gender Count in Members Data Set", fontsize=12)
plt.show()
fig.savefig("Frequency of Gender Count in Members Data Set.png", dpi=fig.dpi)
gender_count = Counter(MEMBERS['gender']).most_common()
print("Gender Count " +str(gender_count))
tmp_1=training.bd.value_counts()
tmp_1.head()
training['bd'] = training.bd.apply(lambda x: int(x) if pd.notnull(x) else "NAN" )
bd_count = Counter(training['bd']).most_common()
print("BD Count " +str(bd_count))
#training.loc[(training['bd'] <= 1), 'bd'] = -99999
#training.loc[(training['bd'] >= 100), 'bd'] = -99999
training['bd'] = training.bd.apply(lambda x: -99999 if float(x)<=1 else x )
training['bd'] = training.bd.apply(lambda x: -99999 if float(x)>=100 else x )
#Birth Date count in training Data Set
fig = plt.figure(figsize=(12,8))
bd_order = training['bd'].unique()
bd_order = sorted(bd_order, key=lambda x: str(x))
bd_order = sorted(bd_order, key=lambda x: float(x))
#above repetion of commands are very silly, but this was the only way I was able to diplay what I wanted
sns.countplot(x="bd", data=training , order = bd_order)
plt.ylabel('Count', fontsize=12)
plt.xlabel('BD', fontsize=12)
plt.xticks(rotation='vertical')
plt.title("Frequency of BD Count", fontsize=12)
plt.show()
fig.savefig("Frequency of BD Count.png", dpi=fig.dpi)
bd_count = Counter(training['bd']).most_common()
print("BD Count " +str(bd_count))
tmp_bd = training[training.bd != -99999]
print("Mean of Birth Date = " +str(np.mean(tmp_bd['bd'])))
print("Median of Birth Date = " +str(np.median(tmp_bd['bd'])))
#print("Mode of Birth Date = " +str(np.mode(tmp_bd['bd'])))
fig = plt.figure(figsize=(12,8))
plt.subplot(211)
bd_order_2 = tmp_bd['bd'].unique()
bd_order_2 = sorted(bd_order_2, key=lambda x: float(x))
sns.countplot(x="bd", data=tmp_bd , order = bd_order_2)
plt.ylabel('Count', fontsize=12)
plt.xlabel('BD', fontsize=12)
plt.xticks(rotation='vertical')
plt.title("Frequency of BD Count without ouliers and NAN values", fontsize=12)
plt.show()
fig.savefig("Frequency of BD Count without ouliers and NAN values.png", dpi=fig.dpi)

fig = plt.figure(figsize=(4,12))
plt.subplot(212)
sns.boxplot(y=tmp_bd["bd"],data=tmp_bd)
plt.xlabel('BD', fontsize=12)
plt.title("Box Plot of Birth Date without ouliers and NAN values", fontsize=12)
plt.show()
fig.savefig("Box Plot of Birth Date without ouliers and NAN values.png", dpi=fig.dpi)
del MEMBERS
del STATUS
del training
TRANSACTION = pd.read_csv("../data/processed/TRANSACTION_PRED.csv", parse_dates=True)
TRANSACTION.head()
TRANSACTION['is_cancel'] = TRANSACTION['is_cancel'].astype(str)
TRANSACTION['is_cancel'].value_counts()
TRANSACTION['is_auto_renew'] = TRANSACTION['is_auto_renew'].astype(str)
TRANSACTION['is_auto_renew'].value_counts()
Non_monthly_sub = TRANSACTION['payment_plan_days'][TRANSACTION['payment_plan_days'] != 30][TRANSACTION['payment_plan_days'] != 31]
Non_monthly_sub.astype(str)
Non_monthly_sub.value_counts()
TRANSACTION[TRANSACTION["plan_list_price"] != TRANSACTION["actual_amount_paid"]]
# payment_method_id count in transactions Data Set
fig = plt.figure(figsize=(18,6))
#plt.subplot(311)
sns.countplot(x="payment_method_id", data=TRANSACTION)
plt.ylabel('Count', fontsize=12)
plt.xlabel('payment_method_id', fontsize=12)
plt.xticks(rotation='vertical')
plt.title("Frequency of payment_method_id Count in transactions Data Set", fontsize=12)
plt.show()
fig.savefig("Frequency of payment_method_id Count in transactions Data Set.png", dpi=fig.dpi)
payment_method_id_count = Counter(TRANSACTION['payment_method_id']).most_common()
print("payment_method_id Count " +str(payment_method_id_count))
# payment_plan_days count in transactions Data Set
fig = plt.figure(figsize=(18,6))
sns.countplot(x="payment_plan_days", data=TRANSACTION)
plt.ylabel('Count', fontsize=12)
plt.xlabel('payment_plan_days', fontsize=12)
plt.xticks(rotation='vertical')
plt.title("Frequency of payment_plan_days Count in transactions Data Set", fontsize=12)
plt.show()
fig.savefig("Frequency of payment_plan_days Count in transactions Data Set.png", dpi=fig.dpi)
payment_plan_days_count = Counter(TRANSACTION['payment_plan_days']).most_common()
print("payment_plan_days Count " +str(payment_plan_days_count))
# plan_list_price count in transactions Data Set
fig = plt.figure(figsize=(18,6))
sns.countplot(x="plan_list_price", data=TRANSACTION)
plt.ylabel('Count', fontsize=12)
plt.xlabel('plan_list_price', fontsize=12)
plt.xticks(rotation='vertical')
plt.title("Frequency of plan_list_price Count in transactions Data Set", fontsize=12)
plt.show()
fig.savefig("Frequency of plan_list_price Count in transactions Data Set.png", dpi=fig.dpi)
plan_list_price_count = Counter(TRANSACTION['plan_list_price']).most_common()
print("plan_list_price Count " +str(plan_list_price_count))
# is_auto_renew count in transactions Data Set
fig = plt.figure(figsize=(4,4))
sns.countplot(x="is_auto_renew", data=TRANSACTION)
plt.ylabel('Count', fontsize=12)
plt.xlabel('is_auto_renew', fontsize=12)
plt.xticks(rotation='vertical')
plt.title("Frequency of is_auto_renew Count in transactions Data Set", fontsize=6)
plt.show()
fig.savefig("Frequency of is_auto_renew Count in transactions Data Set.png", dpi=fig.dpi)
is_auto_renew_count = Counter(TRANSACTION['is_auto_renew']).most_common()
print("is_auto_renew Count " +str(is_auto_renew_count))
# is_cancel count in transactions Data Set
fig = plt.figure(figsize=(4,4))
sns.countplot(x="is_cancel", data=TRANSACTION)
plt.ylabel('Count', fontsize=12)
plt.xlabel('is_cancel', fontsize=12)
plt.xticks(rotation='vertical')
plt.title("Frequency of is_cancel Count in transactions Data Set", fontsize=6)
plt.show()
fig.savefig("Frequency of is_cancel Count in transactions Data Set.png", dpi=fig.dpi)
is_cancel_count = Counter(TRANSACTION['is_cancel']).most_common()
print("is_cancel Count " +str(is_cancel_count))<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup