<jupyter_start><jupyter_text>Week 6 IP  \ **Defining the Question**\  Specifying the Question > predicting a team score based on  their rank.  Defining the Metric for Success > Getting the prediction a team score based on their rank.  Understanding the context  > Here  Recording the Experimental Design > Here  Data Relevance > The dataset's give relevenat to this prediction   \ **Importing the Libraries**\<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import LabelEncoder
from sklearn import metrics
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import KFold
from sklearn.linear_model import LogisticRegression
from datetime import date<jupyter_output><empty_output><jupyter_text>\ **Reading the Data**\  First Dataset<jupyter_code>fifa = pd.read_csv('/content/fifa_ranking.csv')<jupyter_output><empty_output><jupyter_text>Second Dataset<jupyter_code>results = pd.read_csv('/content/results.csv')<jupyter_output><empty_output><jupyter_text>\ **Checking the Data**\  First Dataset<jupyter_code># Determining the no. of records in our dataset
#
records = fifa.shape[0]
print(f'{records} records')
# Number of columns and rows

rows = fifa.shape[0]
columns = fifa.shape[1]

print(f'{rows} rows and {columns} columns')
fifa.columns
# Previewing the top of our dataset
#
fifa.head(10)
# Previewing the bottom of our dataset
# 
fifa.tail(10)
fifa.info()
# Checking whether each column has an appropriate datatype
#
fifa.dtypes
fifa.nunique()
col = list(fifa.columns)
for i in col:
  print(f'{i}')
  print('')
  a = fifa[i].unique()
  print(a)
  print('****************************************************************************')
  print('')
fifa.describe()<jupyter_output><empty_output><jupyter_text>Second Dataset<jupyter_code># Determining the no. of records in our dataset
records = results.shape[0]
print(f'{records} records')
# Number of columns and rows
rows = results.shape[0]
columns = results.shape[1]

print(f'{rows} rows and {columns} columns')
results.columns
# Previewing the top of our dataset
results.head()
# Previewing the bottom of our dataset 
results.tail()
results.info()
# Checking whether each column has an appropriate datatype
results.dtypes
results.nunique()
col = list(results.columns)
for i in col:
  print(f'{i}')
  print('')
  a = results[i].unique()
  print(a)
  print('')
  print('****************************************************************************')
  print('')
results.describe()<jupyter_output><empty_output><jupyter_text>\ **Tidying the Dataset**\  First Dataset  Missing Data<jupyter_code># Identifying the Missing Data
#
fifa.isnull().sum()<jupyter_output><empty_output><jupyter_text>Duplicate data<jupyter_code>fifa.duplicated().sum()
fifa = fifa.drop_duplicates()
fifa.duplicated().sum()<jupyter_output><empty_output><jupyter_text>Drop coluns that are not need<jupyter_code><jupyter_output><empty_output><jupyter_text>Outliers<jupyter_code># Checking for Outliers
#
numeric = fifa.drop(['rank', 'country_full', 'country_abrv', 'confederation', 'rank_date'],axis=1)

columns = None
columns = list(numeric.columns)

fig, ax = plt.subplots(nrows=11, ncols=1, figsize=(10,30))  

i = 0
for column in columns:

  sns.boxplot(x=fifa[column], ax=ax[i])
  ax[i].set_title(f'Box plot of {column}', fontsize=20, pad=10 )
  ax[i].set_xlabel(column, fontsize=15)
  i+=1

plt.tight_layout()
plt.show()
numeric = fifa.drop(['rank', 'country_full', 'country_abrv', 'confederation', 'rank_date'],axis=1)

columns = None
columns = list(numeric.columns)

for column in columns:

  max_threshold_fifa = fifa[column].quantile(0.95)
  min_threshold_fifa = fifa[column].quantile(0.05)
  df_1 = fifa[(fifa[column]<max_threshold_fifa) & (fifa[column]>min_threshold_fifa)]
  num = fifa.shape[0]-df_1.shape[0]
  print(f'{num} rows that are outliers in {column} \n This is {"{:.2f}".format((num/(fifa.shape[0]))*100)}% percent approximately of {column} column')
  print(' ')<jupyter_output>43600 rows that are outliers in total_points 
 This is 75.49% percent approximately of total_points column
 
5989 rows that are outliers in previous_points 
 This is 10.37% percent approximately of previous_points column
 
6880 rows that are outliers in rank_change 
 This is 11.91% percent approximately of rank_change column
 
45405 rows that are outliers in cur_year_avg 
 This is 78.62% percent approximately of cur_year_avg column
 
45405 rows that are outliers in cur_year_avg_weighted 
 This is 78.62% percent approximately of cur_year_avg_weighted column
 
45522 rows that are outliers in last_year_avg 
 This is 78.82% percent approximately of last_year_avg column
 
45522 rows that are outliers in last_year_avg_weighted 
 This is 78.82% percent approximately of last_year_avg_weighted column
 
45771 rows that are outliers in two_year_ago_avg 
 This is 79.25% percent approximately of two_year_ago_avg column
 
45771 rows that are outliers in two_year_ago_weighted 
 This is 79.25% percent[...]<jupyter_text>Changing data type<jupyter_code>fifa['rank_date'] = pd.to_datetime(fifa['rank_date'], format="%Y/%m/%d")<jupyter_output><empty_output><jupyter_text>Adding the column year<jupyter_code>fifa['year'] = pd.DatetimeIndex(fifa['rank_date']).year
print(fifa['year'].unique())<jupyter_output>[1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006
 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018]<jupyter_text>Adding the column month_year<jupyter_code>fifa['month_year'] = fifa['rank_date'].dt.to_period('M')
print(fifa['month_year'].unique())<jupyter_output><PeriodArray>
['1993-08', '1993-09', '1993-10', '1993-11', '1993-12', '1994-02', '1994-03',
 '1994-04', '1994-05', '1994-06',
 ...
 '2017-09', '2017-10', '2017-11', '2017-12', '2018-01', '2018-02', '2018-03',
 '2018-04', '2018-05', '2018-06']
Length: 285, dtype: period[M]<jupyter_text>Adding the column day<jupyter_code>fifa['day'] = fifa['rank_date'].dt.day
print(fifa['day'].unique())<jupyter_output>[ 8 23 22 19 15 17 14 21 13 25 20 16 24  3 28 18 27  9 12 10  7  5  6 11
 26 30  4  1  2 31 29]<jupyter_text>Adding the column month<jupyter_code>fifa['month_number'] = pd.DatetimeIndex(fifa['rank_date']).month
fifa['month_name'] = pd.DatetimeIndex(fifa['rank_date']).month
a = list(fifa['month_number'].unique())
b = sorted(a)
print(b)
fifa['month_name'].replace({1:'January',
                       2:'February', 
                       3:'March',
                       4:'April', 
                       5:'May',
                       6:'June', 
                       7:'July',
                       8:'August', 
                       9:'September', 
                       10:'October', 
                       11:'November', 
                       12:'December'}, inplace=True)
print(fifa['month_name'].unique())<jupyter_output>['August' 'September' 'October' 'November' 'December' 'February' 'March'
 'April' 'May' 'June' 'July' 'January']<jupyter_text>Changing the title<jupyter_code>fifa = fifa.rename(columns={'rank_date': 'date'})
fifa.columns
# fifa[(fifa['country_full']=='Norway') & (fifa['year']==1993) & (fifa['confederation']=='UEFA')].sort_values(by="date", ascending=False)
fifa[(fifa['country_full']=='Norway') & (fifa['year']<=1994) & (fifa['confederation']=='UEFA')].sort_values(by="date", ascending=False)
# for i, row in fifa.iterrows():
#   year =  fifa.loc[i,'year']
#   country = fifa.loc[i,'country_full']
#   confederation = fifa.loc[i,'confederation']
  # the_record = fifa[(fifa['country_full']==country) & (fifa['year']==year) & (fifa['confederation']==confederation)].sort_values(by="date", ascending=False).head(1)
  # a = the_record['rank']
  # the_rank = a.iloc[0]
  # if the_rank == fifa.loc[i,'rank']:
  #   continue
  # fifa.loc[i,'rank'] = the_rank
# fifa[(fifa['country_full']=='Norway') & (fifa['year']==year) & (fifa['confederation']=='UEFA')].sort_values(by="date", ascending=False)<jupyter_output><empty_output><jupyter_text>Second Dataset  Missing Data<jupyter_code># Identifying the Missing Data
#
results.isnull().sum()<jupyter_output><empty_output><jupyter_text>Duplicate data<jupyter_code>results.duplicated().any()<jupyter_output><empty_output><jupyter_text>Drop columns<jupyter_code>results.head()<jupyter_output><empty_output><jupyter_text>Outliers<jupyter_code># Checking for Outliers
#
numeric = results[['away_score', 'home_score']]

columns = None
columns = list(numeric.columns)

fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(15,5))  

i = 0
for column in columns:

  sns.boxplot(x=results[column], ax=ax[i])
  ax[i].set_title(f'Box plot of {column}', fontsize=20, pad=10 )
  ax[i].set_xlabel(column, fontsize=15)
  i+=1

plt.tight_layout()
plt.show()
numeric = results[['away_score', 'home_score']]

columns = None
columns = list(numeric.columns)

for column in columns:

  max_threshold_r = results[column].quantile(0.95)
  min_threshold_r = results[column].quantile(0.05)
  df_1 = results[(results[column]<max_threshold_r) & (results[column]>min_threshold_r)]
  num = results.shape[0]-df_1.shape[0]
  print(f'{num} rows that are outliers in {column} \n This is {"{:.2f}".format((num/(results.shape[0]))*100)}% percent approximately of {column} column')
  print(' ')<jupyter_output>17498 rows that are outliers in away_score 
 This is 42.85% percent approximately of away_score column
 
12447 rows that are outliers in home_score 
 This is 30.48% percent approximately of home_score column<jupyter_text>Changing data type<jupyter_code>results['date'] = pd.to_datetime(results['date'], format="%Y/%m/%d")<jupyter_output><empty_output><jupyter_text>Adding the column month_year<jupyter_code>results['month_year'] = results['date'].dt.to_period('M')
print(results['month_year'].unique())<jupyter_output><PeriodArray>
['1872-11', '1873-03', '1874-03', '1875-03', '1876-03', '1877-03', '1878-03',
 '1879-01', '1879-04', '1880-03',
 ...
 '2018-10', '2018-11', '2018-12', '2019-01', '2019-02', '2019-03', '2019-04',
 '2019-05', '2019-06', '2019-07']
Length: 1393, dtype: period[M]<jupyter_text>Adding the column day<jupyter_code>results['day'] = results['date'].dt.day
print(results['day'].unique())<jupyter_output>[30  8  7  6  4 25  3  5  2 23 18 13 15 27 26 12 14 11 24 10 17  9 29 28
 21 20 19 22  1 31 16]<jupyter_text>Adding the column month<jupyter_code>results['month_number'] = pd.DatetimeIndex(results['date']).month
results['month_name'] = pd.DatetimeIndex(results['date']).month
a = list(results['month_number'].unique())
b = sorted(a)
print(b)
results['month_name'].replace({1:'January',
                       2:'February', 
                       3:'March',
                       4:'April', 
                       5:'May',
                       6:'June', 
                       7:'July',
                       8:'August', 
                       9:'September', 
                       10:'October', 
                       11:'November', 
                       12:'December'}, inplace=True)
print(results['month_name'].unique())
for i,row in results.iterrows():
  a = results.loc[i, 'tournament']
  b = a.lower()
  if ('world cup' in b)==True:
    results.loc[i, 'tournament'] = 'World cup'
  elif (('friendly' in b)==True):
    pass
  else:
    results.loc[i, 'tournament'] = 'Other'
results['tournament'].unique()
for i,row in results.iterrows():
  home = results.loc[i, 'home_score']
  away = results.loc[i, 'away_score']
  if (home > away):
    results.loc[i, 'status'] = 'Win'
  elif (home < away):
    results.loc[i, 'status'] = 'Lose'
  else:
    results.loc[i, 'status'] = 'Draw'
results['status'].unique()<jupyter_output><empty_output><jupyter_text>\ **Exploratory Analysis**\  Univariate(for fifa dataset)<jupyter_code>fifa['confederation'].value_counts().plot(kind='bar')<jupyter_output><empty_output><jupyter_text>> Most of the matches play come from UEFA and CAF.<jupyter_code>rank = fifa['total_points']
plt.hist(rank, bins=5, histtype='bar')

plt.ylabel('total_points')
plt.title('Histogram of total_points')
plt.show()<jupyter_output><empty_output><jupyter_text>Central Tendency<jupyter_code>fifa.columns
numeric = fifa.drop(['country_full', 'country_abrv', 'confederation', 'date', 'month_name', 'month_year'],axis=1)
col = list(numeric.columns)
col<jupyter_output><empty_output><jupyter_text>mean<jupyter_code>for i in col:
  mean = fifa[i].mean()
  print(f'This is the mean of the {i} column: ')
  print(f'{mean}')
  print('  ')<jupyter_output>This is the mean of the rank column: 
101.61811759817162
  
This is the mean of the total_points column: 
121.97115104924205
  
This is the mean of the previous_points column: 
332.33996467899436
  
This is the mean of the rank_change column: 
-0.0099383613823672
  
This is the mean of the cur_year_avg column: 
61.7513624558485
  
This is the mean of the cur_year_avg_weighted column: 
61.7513624558485
  
This is the mean of the last_year_avg column: 
60.953507341228374
  
This is the mean of the last_year_avg_weighted column: 
30.476829766604226
  
This is the mean of the two_year_ago_avg column: 
59.734768335757124
  
This is the mean of the two_year_ago_weighted column: 
17.92046938846179
  
This is the mean of the three_year_ago_avg column: 
59.114466375787316
  
This is the mean of the three_year_ago_weighted column: 
11.822921601219143
  
This is the mean of the year column: 
2006.1730209848326
  
This is the mean of the day column: 
15.018301128887042
  
This is the mean of the m[...]<jupyter_text>median<jupyter_code>for i in col:
  median = fifa[i].median()
  print(f'This is the median of the {i} column: ')
  print(f'{median}')
  print('  ')<jupyter_output>This is the median of the rank column: 
101.0
  
This is the median of the total_points column: 
0.0
  
This is the median of the previous_points column: 
272.0
  
This is the median of the rank_change column: 
0.0
  
This is the median of the cur_year_avg column: 
0.0
  
This is the median of the cur_year_avg_weighted column: 
0.0
  
This is the median of the last_year_avg column: 
0.0
  
This is the median of the last_year_avg_weighted column: 
0.0
  
This is the median of the two_year_ago_avg column: 
0.0
  
This is the median of the two_year_ago_weighted column: 
0.0
  
This is the median of the three_year_ago_avg column: 
0.0
  
This is the median of the three_year_ago_weighted column: 
0.0
  
This is the median of the year column: 
2006.0
  
This is the median of the day column: 
16.0
  
This is the median of the month_number column: 
7.0<jupyter_text>mode<jupyter_code>mod = fifa.drop(['country_abrv', 'total_points',
       'previous_points', 'rank_change', 'cur_year_avg',
       'cur_year_avg_weighted', 'last_year_avg', 'last_year_avg_weighted',
       'two_year_ago_avg', 'two_year_ago_weighted', 'three_year_ago_avg',
       'three_year_ago_weighted', 'date', 'year', 'day',
       'month_number','month_year'],axis=1)
col = list(mod.columns)
for i in col:
  mode = fifa[i].mode()[0]
  print(f'This is the mode of the {i} column: ')
  print(f'{mode}')
  print('  ')<jupyter_output>This is the mode of the rank column: 
147
  
This is the mode of the country_full column: 
Sudan
  
This is the mode of the confederation column: 
UEFA
  
This is the mode of the month_name column: 
May<jupyter_text>Dispersion  Standard Deviation<jupyter_code>numeric = fifa.drop(['country_full', 'country_abrv', 'confederation', 'date', 'month_name', 'month_year'],axis=1)
col = list(numeric.columns)

for i in col:
  mode = fifa[i].std()
  print(f'This is the Standard Deviation of the {i} column: ')
  print(f'{mode}')
  print('  ')<jupyter_output>This is the Standard Deviation of the rank column: 
58.6350962126357
  
This is the Standard Deviation of the total_points column: 
260.4791908256627
  
This is the Standard Deviation of the previous_points column: 
302.96417559526026
  
This is the Standard Deviation of the rank_change column: 
5.802787706913632
  
This is the Standard Deviation of the cur_year_avg column: 
138.04200646327516
  
This is the Standard Deviation of the cur_year_avg_weighted column: 
138.04200646327516
  
This is the Standard Deviation of the last_year_avg column: 
137.71294461767638
  
This is the Standard Deviation of the last_year_avg_weighted column: 
68.8565136251929
  
This is the Standard Deviation of the two_year_ago_avg column: 
136.3191193341892
  
This is the Standard Deviation of the two_year_ago_weighted column: 
40.895761005541885
  
This is the Standard Deviation of the three_year_ago_avg column: 
135.54013469372137
  
This is the Standard Deviation of the three_year_ago_weighted column: 
2[...]<jupyter_text>Variance<jupyter_code>for i in col:
  variance = fifa[i].var()
  print(f'This is the variance of the {i} column: ')
  print(f'{variance}')
  print('  ')<jupyter_output>This is the variance of the rank column: 
3438.0745078650457
  
This is the variance of the total_points column: 
67849.40885319203
  
This is the variance of the previous_points column: 
91787.2916941157
  
This is the variance of the rank_change column: 
33.67234517150797
  
This is the variance of the cur_year_avg column: 
19055.5955484069
  
This is the variance of the cur_year_avg_weighted column: 
19055.5955484069
  
This is the variance of the last_year_avg column: 
18964.855115271206
  
This is the variance of the last_year_avg_weighted column: 
4741.219468616377
  
This is the variance of the two_year_ago_avg column: 
18582.902296048913
  
This is the variance of the two_year_ago_weighted column: 
1672.4632682224005
  
This is the variance of the three_year_ago_avg column: 
18371.12811279213
  
This is the variance of the three_year_ago_weighted column: 
734.8454985820633
  
This is the variance of the year column: 
49.45857503764371
  
This is the variance of the day column: [...]<jupyter_text>Maximum<jupyter_code>for i in col:
  max = fifa[i].max()
  print(f'This is the largest value in the {i} column: ')
  print(f'{max}')
  print('  ')<jupyter_output>This is the largest value in the rank column: 
209
  
This is the largest value in the total_points column: 
1775.03
  
This is the largest value in the previous_points column: 
1920
  
This is the largest value in the rank_change column: 
92
  
This is the largest value in the cur_year_avg column: 
1158.66
  
This is the largest value in the cur_year_avg_weighted column: 
1158.66
  
This is the largest value in the last_year_avg column: 
1169.57
  
This is the largest value in the last_year_avg_weighted column: 
584.79
  
This is the largest value in the two_year_ago_avg column: 
1159.71
  
This is the largest value in the two_year_ago_weighted column: 
347.91
  
This is the largest value in the three_year_ago_avg column: 
1200.77
  
This is the largest value in the three_year_ago_weighted column: 
240.15
  
This is the largest value in the year column: 
2018
  
This is the largest value in the day column: 
31
  
This is the largest value in the month_number column: 
12<jupyter_text>Minimum<jupyter_code>for i in col:
  min = fifa[i].min()
  print(f'This is the small value in the {i} column: ')
  print(f'{min}')
  print('  ')<jupyter_output>This is the small value in the rank column: 
1
  
This is the small value in the total_points column: 
0.0
  
This is the small value in the previous_points column: 
0
  
This is the small value in the rank_change column: 
-72
  
This is the small value in the cur_year_avg column: 
0.0
  
This is the small value in the cur_year_avg_weighted column: 
0.0
  
This is the small value in the last_year_avg column: 
0.0
  
This is the small value in the last_year_avg_weighted column: 
0.0
  
This is the small value in the two_year_ago_avg column: 
0.0
  
This is the small value in the two_year_ago_weighted column: 
0.0
  
This is the small value in the three_year_ago_avg column: 
0.0
  
This is the small value in the three_year_ago_weighted column: 
0.0
  
This is the small value in the year column: 
1993
  
This is the small value in the day column: 
1
  
This is the small value in the month_number column: 
1<jupyter_text>Range<jupyter_code>for i in col:
  max = fifa[i].max()
  min = fifa[i].min()
  range = max - min
  print(f'This is the range of the {i} column: ')
  print(f'{range}')
  print('  ')<jupyter_output>This is the range of the rank column: 
208
  
This is the range of the total_points column: 
1775.03
  
This is the range of the previous_points column: 
1920
  
This is the range of the rank_change column: 
164
  
This is the range of the cur_year_avg column: 
1158.66
  
This is the range of the cur_year_avg_weighted column: 
1158.66
  
This is the range of the last_year_avg column: 
1169.57
  
This is the range of the last_year_avg_weighted column: 
584.79
  
This is the range of the two_year_ago_avg column: 
1159.71
  
This is the range of the two_year_ago_weighted column: 
347.91
  
This is the range of the three_year_ago_avg column: 
1200.77
  
This is the range of the three_year_ago_weighted column: 
240.15
  
This is the range of the year column: 
25
  
This is the range of the day column: 
30
  
This is the range of the month_number column: 
11<jupyter_text>Quantiles  First Quantile<jupyter_code>for i in col:
  first_quantile = fifa[i].quantile(0.25)
  print(f'This is the first quantile of the {i} column: ')
  print(f'{first_quantile}')
  print('  ')<jupyter_output>This is the first quantile of the rank column: 
51.0
  
This is the first quantile of the total_points column: 
0.0
  
This is the first quantile of the previous_points column: 
56.0
  
This is the first quantile of the rank_change column: 
-2.0
  
This is the first quantile of the cur_year_avg column: 
0.0
  
This is the first quantile of the cur_year_avg_weighted column: 
0.0
  
This is the first quantile of the last_year_avg column: 
0.0
  
This is the first quantile of the last_year_avg_weighted column: 
0.0
  
This is the first quantile of the two_year_ago_avg column: 
0.0
  
This is the first quantile of the two_year_ago_weighted column: 
0.0
  
This is the first quantile of the three_year_ago_avg column: 
0.0
  
This is the first quantile of the three_year_ago_weighted column: 
0.0
  
This is the first quantile of the year column: 
2000.0
  
This is the first quantile of the day column: 
10.0
  
This is the first quantile of the month_number column: 
4.0<jupyter_text>Second Quantile<jupyter_code>for i in col:
  second_quantile = fifa[i].quantile(0.5)
  print(f'This is the second quantile of the {i} column: ')
  print(f'{second_quantile}')
  print('  ')<jupyter_output>This is the second quantile of the rank column: 
101.0
  
This is the second quantile of the total_points column: 
0.0
  
This is the second quantile of the previous_points column: 
272.0
  
This is the second quantile of the rank_change column: 
0.0
  
This is the second quantile of the cur_year_avg column: 
0.0
  
This is the second quantile of the cur_year_avg_weighted column: 
0.0
  
This is the second quantile of the last_year_avg column: 
0.0
  
This is the second quantile of the last_year_avg_weighted column: 
0.0
  
This is the second quantile of the two_year_ago_avg column: 
0.0
  
This is the second quantile of the two_year_ago_weighted column: 
0.0
  
This is the second quantile of the three_year_ago_avg column: 
0.0
  
This is the second quantile of the three_year_ago_weighted column: 
0.0
  
This is the second quantile of the year column: 
2006.0
  
This is the second quantile of the day column: 
16.0
  
This is the second quantile of the month_number column: 
7.0<jupyter_text>Third Quantile<jupyter_code>for i in col:
  third_quantile = fifa[i].quantile(0.75)
  print(f'This is the third quantile of the {i} column: ')
  print(f'{third_quantile}')
  print('  ')<jupyter_output>This is the third quantile of the rank column: 
152.0
  
This is the third quantile of the total_points column: 
91.5
  
This is the third quantile of the previous_points column: 
525.0
  
This is the third quantile of the rank_change column: 
1.0
  
This is the third quantile of the cur_year_avg column: 
30.4475
  
This is the third quantile of the cur_year_avg_weighted column: 
30.4475
  
This is the third quantile of the last_year_avg column: 
26.4
  
This is the third quantile of the last_year_avg_weighted column: 
13.2
  
This is the third quantile of the two_year_ago_avg column: 
21.25
  
This is the third quantile of the two_year_ago_weighted column: 
6.38
  
This is the third quantile of the three_year_ago_avg column: 
20.795
  
This is the third quantile of the three_year_ago_weighted column: 
4.1625
  
This is the third quantile of the year column: 
2012.0
  
This is the third quantile of the day column: 
20.0
  
This is the third quantile of the month_number column: 
10.0
  [...]<jupyter_text>Descriptions of the distribution curve  Skewness<jupyter_code>for i in col:
  skewness = fifa[i].skew()
  print(f'This is the skewness of the {i} column: ')
  print(f'{skewness}')
  print('  ')<jupyter_output>This is the skewness of the rank column: 
0.018697560304961978
  
This is the skewness of the total_points column: 
2.588628658130345
  
This is the skewness of the previous_points column: 
1.0976209078436039
  
This is the skewness of the rank_change column: 
1.2387472729578097
  
This is the skewness of the cur_year_avg column: 
2.7847173066306636
  
This is the skewness of the cur_year_avg_weighted column: 
2.7847173066306636
  
This is the skewness of the last_year_avg column: 
2.8826204517315164
  
This is the skewness of the last_year_avg_weighted column: 
2.882613017243387
  
This is the skewness of the two_year_ago_avg column: 
2.881021528682775
  
This is the skewness of the two_year_ago_weighted column: 
2.8810175657716393
  
This is the skewness of the three_year_ago_avg column: 
2.9119344417489983
  
This is the skewness of the three_year_ago_weighted column: 
2.91192631008562
  
This is the skewness of the year column: 
-0.08381950107331076
  
This is the skewness of the d[...]<jupyter_text>Kurtosis<jupyter_code>for i in col:
  kurtosis = fifa[i].kurt()
  print(f'This is the kurtosis of the {i} column: ')
  print(f'{kurtosis}')
  print('  ')<jupyter_output>This is the kurtosis of the rank column: 
-1.1804205541924453
  
This is the kurtosis of the total_points column: 
6.895419571032518
  
This is the kurtosis of the previous_points column: 
1.3069816192284835
  
This is the kurtosis of the rank_change column: 
19.676552194198347
  
This is the kurtosis of the cur_year_avg column: 
8.449820474100262
  
This is the kurtosis of the cur_year_avg_weighted column: 
8.449820474100262
  
This is the kurtosis of the last_year_avg column: 
9.400430087283988
  
This is the kurtosis of the last_year_avg_weighted column: 
9.400377506038938
  
This is the kurtosis of the two_year_ago_avg column: 
9.34901226554119
  
This is the kurtosis of the two_year_ago_weighted column: 
9.348981641604768
  
This is the kurtosis of the three_year_ago_avg column: 
9.706373625696592
  
This is the kurtosis of the three_year_ago_weighted column: 
9.70630153793773
  
This is the kurtosis of the year column: 
-1.1329921911609366
  
This is the kurtosis of the day colum[...]<jupyter_text>Univariate(for the result dataset)<jupyter_code>results.columns<jupyter_output><empty_output><jupyter_text>> Scotland played the most matches.<jupyter_code>results['neutral'].value_counts().plot(kind='bar')<jupyter_output><empty_output><jupyter_text>> match really played in neutral grounds<jupyter_code>results['home_score'].hist()
results['month_name'].value_counts().plot(kind='bar')<jupyter_output><empty_output><jupyter_text>> In our dataset common for most matches are held in March<jupyter_code>results['away_score'].hist()<jupyter_output><empty_output><jupyter_text>Central Tendency<jupyter_code>results.columns
numeric = results.drop(['date', 'home_team', 'away_team',
       'tournament', 'city', 'country', 'neutral', 'day',
       'month_number', 'month_name', 'status','month_year'],axis=1)
col = list(numeric.columns)<jupyter_output><empty_output><jupyter_text>mean<jupyter_code>for i in col:
  mean = results[i].mean()
  print(f'This is the mean of the {i} column: ')
  print(f'{mean}')
  print('  ')<jupyter_output>This is the mean of the home_score column: 
1.7457087587844953
  
This is the mean of the away_score column: 
1.188104507945836<jupyter_text>median<jupyter_code>for i in col:
  median = results[i].median()
  print(f'This is the median of the {i} column: ')
  print(f'{median}')
  print('  ')<jupyter_output>This is the median of the home_score column: 
1.0
  
This is the median of the away_score column: 
1.0<jupyter_text>mode<jupyter_code>mod = results.drop(['date', 'day',
       'month_number','month_year'],axis=1)
col = list(mod.columns)
for i in col:
  mode = results[i].mode()[0]
  print(f'This is the mode of the {i} column: ')
  print(f'{mode}')
  print('  ')<jupyter_output>This is the mode of the home_team column: 
Brazil
  
This is the mode of the away_team column: 
Uruguay
  
This is the mode of the home_score column: 
1
  
This is the mode of the away_score column: 
0
  
This is the mode of the tournament column: 
Friendly
  
This is the mode of the city column: 
Kuala Lumpur
  
This is the mode of the country column: 
United States
  
This is the mode of the neutral column: 
False
  
This is the mode of the month_name column: 
June
  
This is the mode of the status column: 
Win<jupyter_text>Dispersion  Standard Deviation<jupyter_code>numeric = results.drop(['date', 'home_team', 'away_team',
       'tournament', 'city', 'country', 'neutral', 'day',
       'month_number', 'month_name', 'status', 'month_year'],axis=1)
col = list(numeric.columns)

for i in col:
  mode = results[i].std()
  print(f'This is the Standard Deviation of the {i} column: ')
  print(f'{mode}')
  print('  ')<jupyter_output>This is the Standard Deviation of the home_score column: 
1.74914491825338
  
This is the Standard Deviation of the away_score column: 
1.4051198000976626<jupyter_text>Variance<jupyter_code>for i in col:
  variance = results[i].var()
  print(f'This is the variance of the {i} column: ')
  print(f'{variance}')
  print('  ')<jupyter_output>This is the variance of the home_score column: 
3.0595079450516236
  
This is the variance of the away_score column: 
1.974361652626495<jupyter_text>Maximum<jupyter_code>for i in col:
  max = results[i].max()
  print(f'This is the largest value in the {i} column: ')
  print(f'{max}')
  print('  ')<jupyter_output>This is the largest value in the home_score column: 
31
  
This is the largest value in the away_score column: 
21<jupyter_text>Minimum<jupyter_code>for i in col:
  min = results[i].min()
  print(f'This is the small value in the {i} column: ')
  print(f'{min}')
  print('  ')<jupyter_output>This is the small value in the home_score column: 
0
  
This is the small value in the away_score column: 
0<jupyter_text>Range<jupyter_code>for i in col:
  max = results[i].max()
  min = results[i].min()
  range = max - min
  print(f'This is the range of the {i} column: ')
  print(f'{range}')
  print('  ')<jupyter_output>This is the range of the home_score column: 
31
  
This is the range of the away_score column: 
21<jupyter_text>Quantiles  First Quantile<jupyter_code>for i in col:
  first_quantile = results[i].quantile(0.25)
  print(f'This is the first quantile of the {i} column: ')
  print(f'{first_quantile}')
  print('  ')<jupyter_output>This is the first quantile of the home_score column: 
1.0
  
This is the first quantile of the away_score column: 
0.0<jupyter_text>Second Quantile<jupyter_code>for i in col:
  second_quantile = results[i].quantile(0.5)
  print(f'This is the second quantile of the {i} column: ')
  print(f'{second_quantile}')
  print('  ')<jupyter_output>This is the second quantile of the home_score column: 
1.0
  
This is the second quantile of the away_score column: 
1.0<jupyter_text>Third Quantile<jupyter_code>for i in col:
  third_quantile = results[i].quantile(0.75)
  print(f'This is the third quantile of the {i} column: ')
  print(f'{third_quantile}')
  print('  ')<jupyter_output>This is the third quantile of the home_score column: 
2.0
  
This is the third quantile of the away_score column: 
2.0<jupyter_text>Descriptions of the distribution curve  Skewness<jupyter_code>for i in col:
  skewness = results[i].skew()
  print(f'This is the skewness of the {i} column: ')
  print(f'{skewness}')
  print('  ')<jupyter_output>This is the skewness of the home_score column: 
2.2015038640372713
  
This is the skewness of the away_score column: 
2.59058860926579<jupyter_text>Kurtosis<jupyter_code>for i in col:
  kurtosis = results[i].kurt()
  print(f'This is the kurtosis of the {i} column: ')
  print(f'{kurtosis}')
  print('  ')<jupyter_output>This is the kurtosis of the home_score column: 
11.71410446777048
  
This is the kurtosis of the away_score column: 
15.757919940541624<jupyter_text>Bivariate **First dataset**<jupyter_code>fifa.columns
c_fifa = fifa.drop(['cur_year_avg',
       'cur_year_avg_weighted', 'last_year_avg', 'last_year_avg_weighted',
       'two_year_ago_avg', 'two_year_ago_weighted', 'three_year_ago_avg',
       'three_year_ago_weighted', 'date', 'day', 'month_year'], axis=1)
plt.subplots(figsize=(15,15))
sns.heatmap(c_fifa.corr(),annot=True, linewidths=5, linecolor='white')
sns.set(font_scale=1)
plt.show()<jupyter_output><empty_output><jupyter_text>**Second dataset**<jupyter_code>sns.heatmap(results.corr(),annot=True)
plt.show()<jupyter_output><empty_output><jupyter_text>\ **Implementing the Solution**\<jupyter_code>fifa.shape
results.columns
fifa_home = fifa[['rank', 'country_full', 'month_year']]
fifa_home.rename(columns={'country_full':'home_team'}, inplace=True)
fifa_away = fifa[['rank', 'country_full', 'month_year']]
fifa_away.rename(columns={'country_full':'away_team'}, inplace=True)

results_1_home = results[['home_team','home_score', 'month_year','tournament','status']]
results_1_home = results_1_home.merge(fifa_home, on=['month_year','home_team'], how='inner')
results_1_home.rename(columns={'rank':'home_team_rank'}, inplace=True)
results_1_away = results[['away_team','away_score', 'month_year','tournament','status']]
results_1_away = results_1_away.merge(fifa_away, on=['month_year','away_team'])
results_1_away.rename(columns={'rank':'away_team_rank'}, inplace=True)
# results_1_home = pd.read_csv('results_home.csv', index_col=0)
results_1_home[['home_team', 'home_score', 'month_year', 'tournament', 'status',
       'home_team_rank']]
# results_1_away = pd.read_csv('results_away.csv', index_col=0)
results_1_away[['away_team', 'away_score', 'month_year', 'tournament', 'status',
       'away_team_rank']]
results_1 = results_1_home.merge(results_1_away, on=['month_year','tournament','status'])<jupyter_output><empty_output><jupyter_text>Approach 1: Polynomial approach  Detecting multicollinearity<jupyter_code>results_1 = results_1.dropna()
label_encoder = LabelEncoder()

#transform to integer
results_1['tournament'] = label_encoder.fit_transform(results_1['tournament'])
independent_only = results_1[['home_team_rank','away_team_rank','tournament','away_score']]
independent_only.corr()
# Let's display the correlations between the variables
correlations = independent_only.corr()
correlations
pd.DataFrame(np.linalg.inv(correlations.values), index = correlations.index, columns=correlations.columns)<jupyter_output><empty_output><jupyter_text>Model 1: Predict how many goals the home team scores.  Residual plots and heteroskedasticity testing<jupyter_code>X = independent_only.values
y = results_1['home_score'].values
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state=0)

# Fit polynomial Regression to the dataset
poly_reg = PolynomialFeatures(degree = 2) 
X_poly = poly_reg.fit_transform(X_train)


pol_reg = LinearRegression()
pol_reg.fit(X_poly, y_train)

# This is our prediction for score based on our model
y_pred = pol_reg.predict(poly_reg.fit_transform(X_test))

# We now create the residual by substracting the test value from the predicted 
# value for each row in our dataset

residuals = np.subtract(y_pred, y_test)

# Let's describe our residual:
pd.DataFrame(residuals).describe()

residuals.mean()
import scipy as sp

test_result, p_value = sp.stats.bartlett(y_pred, residuals)

# To interpret the results we must also compute a critical value of the chi squared distribution
degree_of_freedom = len(y_pred)-1
probability = 1 - p_value

critical_value = sp.stats.chi2.ppf(probability, degree_of_freedom)

print(f'critical_value : {critical_value}')
print(f'degree_of_freedom : {degree_of_freedom}')
print(f'probability : {probability}')

# If the test_result is greater than the critical value, then we reject our null
# hypothesis. This would mean that there are patterns to the variance of the data

# Otherwise, we can identify no patterns, and we accept the null hypothesis that 
# the variance is homogeneous across our data

if (test_result > critical_value):
  print('the variances are unequal, and the model should be reassessed')
else:
  print('the variances are homogeneous!')<jupyter_output>critical_value : inf
degree_of_freedom : 97219
probability : 1.0
the variances are homogeneous!<jupyter_text>Creating modal<jupyter_code>X = independent_only
y = results_1['home_score']
from sklearn.model_selection import LeaveOneOut

X = independent_only.values
y = results_1['home_score'].values

folds = KFold(n_splits=5)

print('we are using ' +str(folds.get_n_splits(X)) + ' folds')

RMSES = [] 
count = 0 
for train_index, test_index in folds.split(X):
  
  X_train, X_test = X[train_index], X[test_index]
  y_train, y_test = y[train_index], y[test_index]
  
  # Fit polynomial Regression to the dataset
  poly_reg = PolynomialFeatures(degree = 2) 
  X_poly = poly_reg.fit_transform(X_train)


  pol_reg = LinearRegression()
  pol_reg.fit(X_poly, y_train)

  # This is our prediction for admission based on our model
  y_pred = pol_reg.predict(poly_reg.fit_transform(X_test))
  
  rmse_value =  np.sqrt(metrics.mean_squared_error(y_test, y_pred))
  RMSES.append(rmse_value)
  
  count+=1
print('trained ' + str(count) + ' models' )
np.mean(RMSES)<jupyter_output>we are using 5 folds
trained 5 models<jupyter_text>**RMSES**<jupyter_code>np.mean(RMSES)
print('Mean Absolute Error:', metrics.mean_absolute_error(y_test, y_pred))  
print('Mean Squared Error:', metrics.mean_squared_error(y_test, y_pred))  
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_test, y_pred)))<jupyter_output>Mean Absolute Error: 1.131410368794534
Mean Squared Error: 2.237506054109723
Root Mean Squared Error: 1.4958295538294872<jupyter_text>Model 2: Predict how many goals the away team scores.  Residual plots and heteroskedasticity testing<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state=0)

# Fit polynomial Regression to the dataset
poly_reg = PolynomialFeatures(degree = 2) 
X_poly = poly_reg.fit_transform(X_train)


pol_reg = LinearRegression()
pol_reg.fit(X_poly, y_train)

# This is our prediction for score based on our model
y_pred = pol_reg.predict(poly_reg.fit_transform(X_test))

# We now create the residual by substracting the test value from the predicted 
# value for each row in our dataset

residuals = np.subtract(y_pred, y_test)

# Let's describe our residual:
pd.DataFrame(residuals).describe()

residuals.mean()
import scipy as sp

test_result, p_value = sp.stats.bartlett(y_pred, residuals)

# To interpret the results we must also compute a critical value of the chi squared distribution
degree_of_freedom = len(y_pred)-1
probability = 1 - p_value

critical_value = sp.stats.chi2.ppf(probability, degree_of_freedom)

print(f'critical_value : {critical_value}')
print(f'degree_of_freedom : {degree_of_freedom}')
print(f'probability : {probability}')

# If the test_result is greater than the critical value, then we reject our null
# hypothesis. This would mean that there are patterns to the variance of the data

# Otherwise, we can identify no patterns, and we accept the null hypothesis that 
# the variance is homogeneous across our data

if (test_result > critical_value):
  print('the variances are unequal, and the model should be reassessed')
else:
  print('the variances are homogeneous!')<jupyter_output>critical_value : inf
degree_of_freedom : 97219
probability : 1.0
the variances are homogeneous!<jupyter_text>Creating modal<jupyter_code>from sklearn.model_selection import LeaveOneOut

X = independent_only.values
y = results_1['away_score'].values

folds = KFold(n_splits=5)

print('we are using ' +str(folds.get_n_splits(X)) + ' folds')

RMSES = [] 
count = 0 
for train_index, test_index in folds.split(X):
  
  X_train, X_test = X[train_index], X[test_index]
  y_train, y_test = y[train_index], y[test_index]
  
  # Fit polynomial Regression to the dataset
  poly_reg = PolynomialFeatures(degree = 2) 
  X_poly = poly_reg.fit_transform(X_train)

  pol_reg = LinearRegression()
  pol_reg.fit(X_poly, y_train)

  # This is our prediction for admission based on our model
  y_pred = pol_reg.predict(poly_reg.fit_transform(X_test))
  
  rmse_value =  np.sqrt(metrics.mean_squared_error(y_test, y_pred))
  RMSES.append(rmse_value)
  
  count+=1
print('trained ' + str(count) + ' models' )
np.mean(RMSES)<jupyter_output>we are using 5 folds
trained 5 models<jupyter_text>**RMSES**<jupyter_code>np.mean(RMSES)
print('Mean Absolute Error:', metrics.mean_absolute_error(y_test, y_pred))  
print('Mean Squared Error:', metrics.mean_squared_error(y_test, y_pred))  
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_test, y_pred)))<jupyter_output>Mean Absolute Error: 2.4502795614179403e-13
Mean Squared Error: 1.2833999438201543e-25
Root Mean Squared Error: 3.58245717883711e-13<jupyter_text>Approach 2: Logistic approach<jupyter_code>results_1 = results_1.drop(['home_team','away_team','month_year'],axis=1)

label_encoder = LabelEncoder()

#transform to integer
results_1['status'] = label_encoder.fit_transform(results_1['status'])
results_1['tournament'] = label_encoder.fit_transform(results_1['tournament'])
X = results_1.drop('status',axis=1)
y = results_1['status']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = .3, random_state=0)
LogReg = LogisticRegression()
LogReg.fit(X_train, y_train)
y_pred = LogReg.predict(X_test)
from sklearn.metrics import confusion_matrix
confusion_matrix = confusion_matrix(y_test, y_pred)
confusion_matrix<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup