<jupyter_start><jupyter_text>I. Complete Imports and Wrangle Data<jupyter_code># Import of packages and package classes

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns

from category_encoders import OneHotEncoder, OrdinalEncoder
from pandas_profiling import ProfileReport
from pdpbox.pdp import pdp_isolate, pdp_plot, pdp_interact, pdp_interact_plot
from sklearn.ensemble import RandomForestClassifier
from sklearn.impute import SimpleImputer
from sklearn.inspection import permutation_importance
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import classification_report, plot_roc_curve, roc_auc_score, plot_confusion_matrix
from sklearn.model_selection import cross_val_score, train_test_split, GridSearchCV, RandomizedSearchCV
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from xgboost import XGBClassifier
# Write a wrangle function for our dataset

def wrangle(filepath):
    # Establish path to data files
    DATA_PATH = '../data/class-project/LoanApproval/'
    
    # Read in the data
    df = pd.read_csv(DATA_PATH + filepath)
    print(df.shape)
    
    # Drop 'Loan_ID' column, it is an identifier only and will not affect model
    # No other high-cardinality colums 
    df.drop(columns='Loan_ID', inplace=True)
    
    # Clean data col names
    df.columns = [col.lower() for col in df.columns]
        
    df = df.rename(columns=
        {'applicantincome': 'applicant_income',
         'coapplicantincome': 'coapplicant_income',
         'loanamount': 'loan_amount'})
        
    # Convert 'credit_history' to binary categorical from float (ready for OHE/ordinal encoding)
    df['credit_history'].replace(to_replace={1.0: '1', 0.0: '0'}, 
                                 inplace=True)
       
    # Remove the outliers where income is > $250,000, and the two outliers where income is < $10,000
    df = df[ (df['applicant_income'] > 1_000) & 
             (df['applicant_income'] < 25_000)]
    
    # Remove the outlier for 'loan_amount' 
    df = df[df['loan_amount'] > 10]  
    
    #df = df[ (df['loan_amount'] > 80) &
    #         (df['loan_amount'] < 300_000)]
    
    # Convert 'coapplicant_income' and'loan_amount' to integers from floats
    for col in ['coapplicant_income', 'loan_amount']:
       df[col] = df[col].astype(int)
    
    # Address NaN values and prepare for OHE/Ordinal Encoding
    mode_cols = ['gender', 'married', 'dependents', 'self_employed', 'loan_amount_term', 'credit_history']
    for col in mode_cols:
        df[col].fillna(value=df[col].mode()[0], inplace=True)

    df['loan_amount_term'] = df['loan_amount_term'].astype(int).astype(str)    
    df['dependents'] = df['dependents'].str.strip('+')

    # Convert target, 'LoanStatus' to binary numeric values
    df['loan_status'].replace(to_replace={'Y': 1, 'N':0}, inplace=True) 
    
    #df.drop(columns=['gender', 'property_area', 'loan_amount_term', 'self_employed', 'education', 'married'], inplace=True)
    
    return df

train_path = 'train_data.csv'
train = wrangle(train_path)
print(train.shape)
train.head()
# Remaining NaN values are in categorical features and will be filled using an imputer with strategy='most_frequent'

train.info()
train['gender'].value_counts(dropna=False)
train['credit_history'].value_counts(dropna=False)
train['dependents'].value_counts(dropna=False)
train['loan_amount_term'].value_counts(dropna=False)
train['loan_amount'].plot(kind='hist', bins=50)
# EDA

print(train.shape)
train.head()

# 'gender', 'married', 'education', 'self_employed' are binary categorial variables
# 'loan_status' is our target feature
# 'credit_history' is indicated to mean 'credit history meets guidelines, Y/N', and should be an integer
train.info()
# Categorical variables are not high cardinality

train.describe(exclude='number')
# Many more male applicants than female

print(train['gender'].value_counts(normalize=True))

train['gender'].value_counts().plot(kind='bar')
plt.xlabel('Gender')
plt.ylabel('Count')
plt.show();
# Primary number of dependents is 0, 3+ are the most infrequent

print(train['dependents'].value_counts(normalize=True))

train['dependents'].value_counts().plot(kind='bar')
plt.xlabel('Dependents')
plt.ylabel('Count')
plt.show();
# More applicants are classified as 'Graduate' vs 'Not Graduate'

print(train['education'].value_counts(normalize=True), "\n")

train['education'].value_counts().plot(kind='bar')
plt.xlabel('Graduation Status')
plt.ylabel('Count')
plt.show();
# More applicants work for someone else versus being self-employed

print(train['self_employed'].value_counts(normalize=True), "\n")

train['self_employed'].value_counts().plot(kind='bar')
plt.xlabel('Self Employed')
plt.ylabel('Frequency')
plt.show()
# After removing outliers, can see that most applicant income are in the 30,000 to 50,000 range, skewed to the right

print(train['applicant_income'].min())
print(train['applicant_income'].unique())

train['applicant_income'].plot(kind='hist', bins=50)

plt.xlabel('Applicant Income')
plt.ylabel('Frequency')
plt.show();
# Distribution of 'coapplicant_income'

print(train['coapplicant_income'].min())
print(train['coapplicant_income'].unique())

train['applicant_income'].plot(kind='hist')

plt.xlabel('Co-applicant Income')
plt.ylabel('Frequency')
plt.show();
# Distribution of 'loan_amount'

print(train['loan_amount'].min())
print(train['loan_amount'].unique())

train['loan_amount'].plot(kind='hist')

plt.xlabel('Loan Amount')
plt.ylabel('Frequency')
plt.show();
# Most applicants apper to be applying for loans with 30-year term lengths

print(train['loan_amount_term'].value_counts(normalize=True), "\n")

train['loan_amount_term'].value_counts().plot(kind='bar')
plt.xlabel('Loan Term')
plt.ylabel('Frequency')
plt.show()
# Most applicants 'credit_history' meet guidelines

print(train['credit_history'].value_counts(normalize=True), "\n")
print(train['credit_history'].dtypes)

train['credit_history'].value_counts().plot(kind='bar')
plt.xlabel('Credity History, meets guidelines')
plt.ylabel('Frequency')
plt.show()
# Most 'property_area'

print(train['property_area'].value_counts(normalize=True), "\n")
print(train['property_area'].dtypes)

train['property_area'].value_counts().plot(kind='bar')
plt.xlabel('Property Area')
plt.ylabel('Frequency')
plt.show()
train.shape
train.head()
# Examine for correlation among continuous variables

sns.pairplot(train[['applicant_income', 'coapplicant_income', 'loan_amount']])<jupyter_output><empty_output><jupyter_text>II. Split the Data<jupyter_code># Create Feature Matrix and Target Array

target = 'loan_status'

y = train[target]
X = train.drop(columns=target)
y.shape
# Split the data
# Will use a random split; there is no datetime information included in this dataset.

X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)<jupyter_output><empty_output><jupyter_text>III. Establish a Baseline<jupyter_code># Establish a baseline for this classification problem - 'Was the loan approved?'
# The classes of the Target Vector are moderately imbalanced towards approval
# Since this is a classification problem we will be looking at accuracy 
# You have a 69.15% chance of being correct if you always decide that the loan was approved; this is our baseline
# to beat

print(y_train.value_counts(normalize=True), "\n")
print('Baseline Accuracy: {:.4f}'.format(y_train.value_counts(normalize=True).max()))
X_train.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 462 entries, 265 to 110
Data columns (total 11 columns):
 #   Column              Non-Null Count  Dtype 
---  ------              --------------  ----- 
 0   gender              462 non-null    object
 1   married             462 non-null    object
 2   dependents          462 non-null    object
 3   education           462 non-null    object
 4   self_employed       462 non-null    object
 5   applicant_income    462 non-null    int64 
 6   coapplicant_income  462 non-null    int64 
 7   loan_amount         462 non-null    int64 
 8   loan_amount_term    462 non-null    object
 9   credit_history      462 non-null    object
 10  property_area       462 non-null    object
dtypes: int64(3), object(8)
memory usage: 43.3+ KB<jupyter_text>IV. Build Models- `LogisticRegression`        - `OneHotEncoder`    - `StandardScaler` - `RandomForrestClassifier`    - `OrdinalEncoder`    - `XGBClassifier`    - `OrdinalEncoder`<jupyter_code># Model 1: Logistic Regression Model

model_lr = make_pipeline(
    OneHotEncoder(use_cat_names=True),
    #SimpleImputer(strategy='most_frequent'),
    StandardScaler(),
    LogisticRegression()
)

model_lr.fit(X_train, y_train);
# Model 1: Random Forest Classifier Model

model_rf = make_pipeline(
    OrdinalEncoder(),
    #SimpleImputer(strategy='most_frequent'),
    RandomForestClassifier(n_jobs=-1, random_state=42)
)

model_rf.fit(X_train, y_train);
# Model 2: XG-Boost Classifier

model_xgb = make_pipeline(
    OrdinalEncoder(),
    #SimpleImputer(strategy='most_frequent'),
    XGBClassifier(random_state=42, n_jobs=-1)
)

model_xgb.fit(X_train, y_train);<jupyter_output><empty_output><jupyter_text>V. Check Metrics<jupyter_code># Classification: Is your majority class frequency >= 50% and < 70% ? 
# If so, you can just use accuracy if you want. Outside that range, accuracy could be misleading. 
# What evaluation metric will you choose, in addition to or instead of accuracy?

# Our majority class is less than 70% and can just use accuracy.  Should, however, come back and build out a 
# confusion_matrix, and look at recall/precision.  Will also explore Precision, Recall, and F1 Score.
# Training and Validation accuracy of our Logistic Regression model

print('Training Accuracy (LOGR):', model_lr.score(X_train, y_train))
print('Validation Accuracy (LOGR):', model_lr.score(X_val, y_val))
# Cross Validation Score for our Logistic Regression model

lr_cvs = cross_val_score(model_lr, X_train, y_train, cv=5, n_jobs=-1)
print('Cross Validation Score (LOGR):', '\n', lr_cvs[0], '\n',
      lr_cvs[1], '\n', lr_cvs[2], '\n', lr_cvs[3],  '\n', 
      lr_cvs[4])
# Training and Validation accuracy of our Random Forest Classifier model

print('Training Accuracy (RF):', model_rf.score(X_train, y_train))
print('Validation Accuracy (RF):', model_rf.score(X_val, y_val))
# Cross Validation Score for our Random Forest Classifier model

rf_cvs = cross_val_score(model_rf, X_train, y_train, cv=5, n_jobs=-1)
print('Cross Validation Score (RF):', '\n', rf_cvs[0], '\n',
      rf_cvs[1], '\n', rf_cvs[2], '\n', rf_cvs[3],  '\n', 
      rf_cvs[4])
# Training and Validation accuracy of our XGBoost Classifier model
# Model appears to be overfit

print('Training Accuracy (XGB):', model_xgb.score(X_train, y_train))
print('Valdiation Accuracy (XGB):', model_xgb.score(X_val, y_val))
# Cross Validation Score for our XGBoost Classifier model

xgb_cvs = cross_val_score(model_xgb, X_train, y_train, cv=5, n_jobs=-1)
print('Cross Validation Score (RF):', '\n', xgb_cvs[0], '\n',
      xgb_cvs[1], '\n', xgb_cvs[2], '\n', xgb_cvs[3],  '\n', 
      xgb_cvs[4])
# LOGISTIC REGRESSION, preformance
# Not very good precision for Y, great recall for Y

print('Logistic Regression')
print(classification_report(y_val, model_lr.predict(X_val)))
# LOGISTIC REGRESSION
# Plot Confusion Matrix

plot_confusion_matrix(model_lr, X_val, y_val, values_format='.0f')

# TN FP
#
# FN TP
# LOGISTIC REGRESSION
# Calculate Precision and Recall

# Precision = TP / (TP + FP)
precision = 82 / (82 + 16)

# Recall = TP / (TP + FN)
recall = 82 / (82 + 3)

print('Logistic Regression model precision', precision)
print('Logistic Regression model recall', recall)
# RANDOM FOREST CLASSIFIER, preformance
# My comments here ************************

print('Random Forest Classifier')
print(classification_report(y_val, model_rf.predict(X_val)))
# RANDOM FOREST CLASSIFIER
# Plot Confusion Matrix

plot_confusion_matrix(model_rf, X_val, y_val, values_format='.0f') 

# TN FP
#
# FN TP
# RANDOM FOREST CLASSIFIER 
# Calculate Precision and Recall

# Precision = TP / (TP + FP)
precision = 76 / (76 + 17)

# Recall = TP / (TP + FN)
recall = 76 / (76 + 9)

print('Random Forest Classifier precision', precision)
print('Random Forest Classifier recall', recall)
# XGBoost CLASSIFIER, preformance
# My comments here**********

print('XGBoost Classifier')
print(classification_report(y_val, model_xgb.predict(X_val)))
# XGBoost CLASSIFIER
# Plot Confusion Matrix

plot_confusion_matrix(model_xgb, X_val, y_val, values_format='.0f') 

# TN FP
#
# FN TP
# XGBoost CLASSIFIER 
# Calculate Precision and Recall

# Precision = TP / (TP + FP)
precision = 74 / (74 + 15)

# Recall = TP / (TP + FN)
recall = 74 / (74 + 11)

print('XGBoost model precision', precision)
print('XGBoost model recall', recall)<jupyter_output>XGBoost model precision 0.8314606741573034
XGBoost model recall 0.8705882352941177<jupyter_text>ROC Curve- To evalute models for binary classification- To decide what probability threshold you should use when making your predictions<jupyter_code># Use VALIDATION DATA

# ROC curve is used with classification problems
# 'How far up can I go without having to go too far to the right?'
# An ROC curve let's you see how your model will perform at various thresholds
# Also allows you to compare different models 

lr = plot_roc_curve(model_lr, X_val, y_val, label='Logistic')
rf = plot_roc_curve(model_rf, X_val, y_val, ax=lr.ax_, label='Random Forest')
xgb = plot_roc_curve(model_xgb, X_val, y_val, ax=lr.ax_, label='XGBoost')

plt.plot([(0,0), (1,1)], color='grey', linestyle='--')
plt.legend();
print('Logistic: ROA-AUC Score:', roc_auc_score(y_val, model_lr.predict(X_val)))
print('Random Forest: ROC-AUC Score:', roc_auc_score(y_val, model_rf.predict(X_val)))
print('XGBoost: ROC-AUC Score:', roc_auc_score(y_val, model_xgb.predict(X_val)))
# Logisitic Regression coefficients

coefs = model_lr.named_steps['logisticregression'].coef_[0]
features = model_lr.named_steps['onehotencoder'].get_feature_names()
pd.Series(coefs, index=features).sort_values(key=abs).tail(20).plot(kind='barh')
importances = model_xgb.named_steps['xgbclassifier'].feature_importances_
features = X_train.columns
feat_imp = pd.Series(importances, index=features).sort_values()
feat_imp
feat_imp.tail(10).plot(kind='barh')
plt.xlabel('Gini Importance')
plt.ylabel('Feature')
plt.title('Feature Importances for XGBoost model')
# Calculate performance metrics using permutated data (add static-noise to features)
perm_imp = permutation_importance(
    model_xgb,
    X_val, # Always use your VALIDATION set
    y_val,
    n_jobs=-1,
    random_state=42
)
perm_imp.keys()
# Put results into a DataFrame
data = {'importances_mean': perm_imp['importances_mean'],
        'importances_std': perm_imp['importances_std']}

df = pd.DataFrame(data, index=X_val.columns)
df.sort_values(by='importances_mean', inplace=True)
df
df['importances_mean'].tail(10).plot(kind='barh')
plt.xlabel('Importance (drop in accuracy)')
plt.ylabel('Feature')
plt.title('Permutation importance for model_xgb')
feature = 'applicant_income'

# Build your 'pdp_isolate' object 
# Create and instance of the pdp_isolate class
# Always use with test or validation data, NEVER training data
isolate = pdp_isolate(
    model=model_xgb,
    dataset=X_val, #<-- Always use with VALIDATION or TEST date
    model_features=X_val.columns,
    feature=feature
)

# Build your plot
pdp_plot(isolate, feature_name=feature);
feature = 'coapplicant_income'

# Build your 'pdp_isolate' object 
# Create and instance of the pdp_isolate class
# Always use with test or validation data, NEVER training data
isolate = pdp_isolate(
    model=model_xgb,
    dataset=X_val, #<-- Always use with VALIDATION or TEST date
    model_features=X_val.columns,
    feature=feature
)

# Build your plot
pdp_plot(isolate, feature_name=feature);
feature = 'loan_amount'

# Build your 'pdp_isolate' object 
# Create and instance of the pdp_isolate class
# Always use with test or validation data, NEVER training data
isolate = pdp_isolate(
    model=model_xgb,
    dataset=X_val, #<-- Always use with VALIDATION or TEST date
    model_features=X_val.columns,
    feature=feature
)

# Build your plot
pdp_plot(isolate, feature_name=feature);
features = ['loan_amount', 'coapplicant_income']

interact = pdp_interact(
    model=model_xgb,
    dataset=X_val,
    model_features=X_val.columns,
    features=features
)

pdp_interact_plot(interact, plot_type='grid', feature_names=features);
# Tune Logistic Regression Model

# penalty=
penalty_variants = ['l2', 'none']

# C=range(1, 11)

# solver=
solver_variants = ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga']

# No effect witnessed with tuning of this hyperparameter
# for _ in penalty_variants:
#     model_lr_tune = make_pipeline(
#         OneHotEncoder(use_cat_names=True),
#         StandardScaler(),
#         LogisticRegression(penalty=_)
#     )
#     model_lr_tune.fit(X_train, y_train);
#     print(_, 'Validation Accuracy (LOGR_penalty_tuned):', model_lr.score(X_val, y_val), "\n")

# No effect witnessed with tuning of this hyperparameter
# for _ in range(1, 11):
#     model_lr_tune = make_pipeline(
#         OneHotEncoder(use_cat_names=True),
#         StandardScaler(),
#         LogisticRegression(C=_)
#     )
#     model_lr_tune.fit(X_train, y_train);
#     print(_, 'Validation Accuracy (LOGR_penalty_tuned):', model_lr.score(X_val, y_val), "\n")

# No effect witnessed with tuning of this hyperparameter
# for _ in solver_variants:
#     model_lr_tune = make_pipeline(
#         OneHotEncoder(use_cat_names=True),
#         StandardScaler(),
#         LogisticRegression(solver=_)
#     )
#     model_lr_tune.fit(X_train, y_train);
#     print(_, 'Validation Accuracy (LOGR_penalty_tuned):', model_lr.score(X_val, y_val), "\n")
params = {'randomforestclassifier__n_estimators': np.arange(20, 100, 5), 
          'randomforestclassifier__max_depth': np.arange(10, 75, 5),
          'randomforestclassifier__max_samples': np.arange(0.1, 0.99, 0.1)}

rf_rs = RandomizedSearchCV(model_rf, param_distributions=params, n_iter=10, cv=5, n_jobs=-1, verbose=1)
rf_rs.fit(X_train, y_train)
rf_rs.best_score_
rf_rs.score(X_val, y_val)
rf_rs.best_params_
# # wrangle_I, clean data and drop NaN values

# def wrangle_I(filepath):

#     # Read in the data
#     df = pd.read_csv('../data/class-project/LoanApproval/' + filepath)
#     print(df.shape)
    
#     # Drop NaN values and drop high-cardinality identifier column, 'Loan_ID'
#     df.dropna(inplace=True)
#     df.drop(columns='Loan_ID', inplace=True)
    
#     # Cleanup column names
#     df.columns = [col.lower() for col in df.columns]       
#     df = df.rename(columns=
#         {'applicantincome': 'applicant_income',
#          'coapplicantincome': 'coapplicant_income',
#          'loanamount': 'loan_amount'})
    
#     # Scale 'applicant_income' and 'coapplicant_income' to thousands
#     df['applicant_income'] = df['applicant_income'] / 100
#     df['coapplicant_income'] = df['coapplicant_income'] / 100
       
#     # Convert 'credit_history' to binary categorical from float 
#     df['credit_history'].replace(to_replace={1.0: '1', 0.0: '0'}, 
#                                  inplace=True)
    
#     # Convert 'loan_amount_term' to categorical variable (object) from float
#     df['loan_amount_term'] = df['loan_amount_term'].astype(int).astype(str) 
    
#     # Clean 'dependents' feature
#     df['dependents'] = df['dependents'].str.strip('+')

#     # Convert target, 'LoanStatus' to binary numeric values
#     df['loan_status'].replace(to_replace={'Y': 1, 'N':0}, inplace=True) 
    
#     return df

# train_path = 'train_data.csv'
# train_I = wrangle_I(train_path)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup