<jupyter_start><jupyter_text>Data Preprocessing<jupyter_code>import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
%matplotlib inline
import seaborn as sns
from scipy import stats
from scipy.stats import norm, skew
import warnings
def ignore(*args, **kwargs):
    """ ignore annoying warnings """
    pass
warnings.warn = ignore
train = pd.read_csv('train_revised.csv')
test = pd.read_csv('test_questions.csv')
train.head()
test.head()
train, test = train.drop_duplicates(), test.drop_duplicates()
#adding the number of tickets column
count1 = dict(train.ride_id.value_counts())
train['number_of_tickets'] = train['ride_id'].map(count1)
#storing the ride_id columns
train_id = train['ride_id']
test_id = test['ride_id']
#dropping the ride_id columns
train.drop('ride_id', axis=1, inplace=True)
test.drop('ride_id', axis=1, inplace=True)
train.drop(['seat_number', 'payment_method', 'payment_receipt'], axis=1, inplace=True)
#casting travel_date to datetime
train.travel_date = pd.to_datetime(train.travel_date)
test.travel_date = pd.to_datetime(test.travel_date)
"""splitting the travel date to month and day"""
train['year'], test['year'] = train.travel_date.dt.year, test.travel_date.dt.year
#train['month'], test['month'] = train.travel_date.dt.month, test.travel_date.dt.month
train['day'], test['day'] = train.travel_date.dt.dayofyear, test.travel_date.dt.dayofyear
train['hour'], test['hour'] = train.travel_time.map(lambda x: pd.to_datetime(x).hour), test.travel_time.map(lambda x: pd.to_datetime(x).hour)
train.head()
fig, ax = plt.subplots()
ax.scatter(x = train['hour'], y = train['number_of_tickets'])
plt.ylabel('number_of_tickets', fontsize=13)
plt.xlabel('hour of day', fontsize=13)
plt.show()<jupyter_output><empty_output><jupyter_text>Target variable analysis<jupyter_code>sns.distplot(train['number_of_tickets'], fit=norm)
#getting fitted parameters used by the function
(mu, sigma) = norm.fit(train['number_of_tickets'])
print(f'mean mu = {mu:.2f} and stan_dev sigma = {sigma:.2f}')

#plotting the distribution
plt.legend([f'Normal dist.($\mu=$ {mu:.2f} and $\sigma=$ {sigma:.2f})'],
           loc='best')
plt.ylabel('Frequency')
plt.title('ticket sales distribution')

#QQ-plot
fig = plt.figure()
res = stats.probplot(train['number_of_tickets'], plot=plt)
plt.show()<jupyter_output>mean mu = 17.28 and stan_dev sigma = 12.38<jupyter_text>Scaling the target variable<jupyter_code>train['number_of_tickets'] = np.log1p(train['number_of_tickets'])
#checking the new distribution
sns.distplot(train['number_of_tickets'], fit=norm)

#getting fitted parameters used by the function
(mu, sigma) = norm.fit(train['number_of_tickets'])
print(f'mean mu = {mu:.2f} and stan_dev sigma = {sigma:.2f}')

#plotting the distribution
plt.legend([f'Normal dist.($\mu=$ {mu:.2f} and $\sigma=$ {sigma:.2f})'],
           loc='best')
plt.ylabel('Frequency')
plt.title('ticket sales distribution')

#QQ-plot
fig = plt.figure()
res = stats.probplot(train['number_of_tickets'], plot=plt)
plt.show()<jupyter_output>mean mu = 2.66 and stan_dev sigma = 0.74<jupyter_text>Feature engineering<jupyter_code>#concatenating the train and test sets
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.number_of_tickets.values

all_data = pd.concat((train, test)).reset_index(drop=True)
all_data.drop(['number_of_tickets'], axis=1, inplace=True)
print(f'dataset size: {all_data.shape}')
#checking for missing values
all_data.isnull().sum()
#transforming variables that are categorical
all_data['year'] = all_data['year'].astype(str)
all_data['day'] = all_data['day'].astype(str)
all_data['hour'] = all_data['hour'].astype(str)
#analysing data correlation with a correlation map
corrmat = train.corr()
plt.subplots(figsize=(9,7))
sns.heatmap(corrmat, vmax=0.9, square=True)<jupyter_output><empty_output><jupyter_text>Label encoding some variables<jupyter_code>from sklearn.preprocessing import LabelEncoder
columns = ('car_type', 'max_capacity', 'travel_from')
for col in columns:
    le = LabelEncoder()
    le.fit(list(all_data[col].values))
    all_data[col] = le.transform(list(all_data[col].values))
print(f'shape: {all_data.shape}')
#adding a new feature
tount = dict(all_data['day'].value_counts())
all_data['daily_travelers'] = all_data['day'].map(tount)
all_data.drop(['travel_date', 'travel_time','travel_to'], axis=1, inplace=True)
print(f'dataset size: {all_data.shape}')
#getting dummy cat features
all_data = pd.get_dummies(all_data)
print(all_data.shape)
train = all_data[:ntrain]
test = all_data[ntrain:]<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup