<jupyter_start><jupyter_text>Section 1:  Data setup  Status<jupyter_code>## CREATE SUBSETS FOR USER STATUS ANALYSIS ##

# subsets for upvotes-downvotes and upvotes+downvotes at time of request and retrieval
diffTrequest = allTData['requester_upvotes_minus_downvotes_at_request']
diffDrequest = allDData['requester_upvotes_minus_downvotes_at_request']
diffTretrieval = allTData['requester_upvotes_minus_downvotes_at_retrieval']
diffDretrieval = allDData['requester_upvotes_minus_downvotes_at_retrieval']
totalvotesTrequest = allTData['requester_upvotes_plus_downvotes_at_request']
totalvotesDrequest = allDData['requester_upvotes_plus_downvotes_at_request']
totalvotesTretrieval = allTData['requester_upvotes_plus_downvotes_at_retrieval']
totalvotesDretrieval = allDData['requester_upvotes_plus_downvotes_at_retrieval']

# subsets for flair (training data only)
Tflair = allTData['requester_user_flair'].copy()
from IPython.display import display

## SUMMARY STATISTICS FOR UPVOTES-DOWNVOTES AT REQUEST AND AT RETRIEVAL ##

# print("SUMMARY STATISTICS FOR UPVOTES-DOWNVOTES AT REQUEST AND AT RETRIEVAL")
# # UPVOTES-DOWNVOTES AT REQUEST ----------------------
# print("\nSummary for upvotes-downvotes at request:")
# print(round(diffTrequest.describe(),2), end="\n\n")

# # UPVOTES-DOWNVOTES AT RETRIEVAL -------------------
# print("-"*100)
# print("Summary for upvotes-downvotes at retrieval:")
# print(round(diffTretrieval.describe(),2), end="\n\n")
from IPython.display import display

## CREATING NEW PARAMETER: the change in upvotes-downvotes from time of request to time of retrieval ##

# user status with new parameter for training data
allTData['request_to_retrieval_upvotes_minus_downvotes'] = diffTretrieval - diffTrequest
diffTcombined = allTData[['requester_upvotes_minus_downvotes_at_request','requester_upvotes_minus_downvotes_at_retrieval',
                          'request_to_retrieval_upvotes_minus_downvotes','requester_received_pizza']].copy()
# rename column headings
diffTcombined.columns = ['Request', 'Retrieval', 'Request-Retrieval', 'Got Pizza']
# print("ADDING NEW PARAMETER - REQUEST-RETRIEVAL:  to represent change in counts from upvotes-downvotes from time of request to time of retrieval.")
# print("\nSTATUS TRAINING DATA")
# display(diffTcombined.head())

# user status with new parameter for training data
allDData['request_to_retrieval_upvotes_minus_downvotes'] = diffDretrieval - diffDrequest
diffDcombined = allDData[['requester_upvotes_minus_downvotes_at_request','requester_upvotes_minus_downvotes_at_retrieval',
                          'request_to_retrieval_upvotes_minus_downvotes','requester_received_pizza']].copy()
# rename columne headings
diffDcombined.columns = ['Request', 'Retrieval', 'Request-Retrieval', 'Got Pizza']
# print("\n")
# print("STATUS DEVELOPMENT DATA")
# display(diffDcombined.head())
import matplotlib.pyplot as plt
%matplotlib inline

## SUMMARY STATISTICS FOR UPVOTES-DOWNVOTES AT REQUEST AND AT RETRIEVAL ##

# upvotes-downvotes at request, grouped by pizza success -------------------------
print("\n\nSUMMARY STATISTICS FOR UPVOTES-DOWNVOTES AT REQUEST AND AT RETRIEVAL, GROUPED BY PIZZA SUCCESS")

diffreq_pizzas = allTData[['requester_upvotes_minus_downvotes_at_request','requester_received_pizza']].copy()
diffreq_pizzas.columns = ['Request', 'Got Pizza']
pizza_diff_request = diffreq_pizzas.groupby(['Got Pizza'])
# print("\nGroup upvotes-downvotes at request by pizza success:\n")
# print(round(pizza_diff_request.describe(), 2))

# upvotes-downvotes at retrieval, grouped by pizza success -------------------------
diffretrieve_pizzas = allTData[['requester_upvotes_minus_downvotes_at_retrieval','requester_received_pizza']].copy()
diffretrieve_pizzas.columns = ['Retrieval', 'Got Pizza']
pizza_diff_retrieval = diffretrieve_pizzas.groupby(['Got Pizza'])
# print("\n")
# print("-"*100)
# print("\nGroup upvotes-downvotes at retrieval by pizza success:\n")
# print(round(pizza_diff_retrieval.describe(), 2))
## USER FLAIR ##
# None = no pizza
#'shroom' = got pizza without 'pay it forward' (PIF)
# PIF' = got pizza and paid forward

flair = allTData[['requester_user_flair','requester_received_pizza']].copy()
flair.columns = ['Flair', 'Got Pizza']
# print("\n")
# print("Types of Flair and Pizza Success\n")
# print(flair.head(10))
# pizza_flair = flair.groupby(['Got Pizza'])
# print("\n")
# print("-"*100)
# print("\nGroup flair by pizza success:\n")
# print(round(pizza_flair.describe(),2))

# binarize categorical features in FLAIR
def flair_numeric(x):
    if x == 'None':
        return 0
    if x == 'shroom':
        return 1
    if x == 'PIF':
        return 2
bin_flair = flair['Flair'].replace(['None', 'shroom', 'PIF'], [-1, 0, 1])
# bin_flair.value_counts()
# print("\nOut of 734 pizza recipients, 683 did not PIF and 51 did PIF.")
## CONVERT UPVOTES-DOWNVOTES SUBSETS INTO Z-SCORES FOR STANDARDIZATION ##
## CREATE FINAL SUBSET FOR UPVOTES-DOWNVOTES, Z-SCORES, AND FLAIR ##

# Z-scores for training data
statusTtrain = allTData[['requester_upvotes_minus_downvotes_at_request','requester_upvotes_minus_downvotes_at_retrieval',
                          'request_to_retrieval_upvotes_minus_downvotes']].copy()
statusTtrain.columns = ['Request', 'Retrieval', 'Request-Retrieval']
for col in statusTtrain:
    zscore_col = col + " Z-score"
    statusTtrain[zscore_col] = (statusTtrain[col] - statusTtrain[col].mean() / statusTtrain[col].std(ddof=0))
statusTtrain = (statusTtrain).join(allTData['requester_user_flair'])
display(statusTtrain.head())

# Z-scores for development data
statusDtrain = allDData[['requester_upvotes_minus_downvotes_at_request','requester_upvotes_minus_downvotes_at_retrieval',
                          'request_to_retrieval_upvotes_minus_downvotes']].copy()
statusDtrain.columns = ['Request', 'Retrieval', 'Request-Retrieval']

for col in statusDtrain:
    zscore_dev_col = col + " Z-score"
    statusDtrain[zscore_dev_col] = (statusDtrain[col] - statusDtrain[col].mean() / statusDtrain[col].std(ddof=0))
statusDtrain = (statusDtrain).join(allDData['requester_user_flair'])
# display(statusDtrain.head())<jupyter_output><empty_output><jupyter_text>DateTime<jupyter_code>import datetime as dt
from IPython.display import display

# CHANGE UNIX TO DATETIME
unix = allTData[['unix_timestamp_of_request_utc', 'requester_received_pizza']].copy()

# unix --> datetime
unix['Datetime'] = pandas.to_datetime(unix['unix_timestamp_of_request_utc'], unit='s')
# just in case we wanted to add date columns to original dataframe
unix['Date'] = unix['Datetime'].dt.date
unix['Hour'] = unix['Datetime'].dt.hour
print("Converted unix timestamp to Date and Time:")
# display(unix.head(5))

# set datetime as index
unix_index = unix.set_index('Datetime')
unix_index = unix_index.drop(['unix_timestamp_of_request_utc', 'Date'], axis=1)
print("\nIndexed by Date and time:")
# display(unix_index.head(10))
# trial grouping by different dates or seasons

# find Christmas
# index by 1 date
christmas2012_pizza = unix_index['2012-12-25']
print("To index by 1 date")
# display(christmas2012_pizza)

# to index by specific month and day
# christmas_pizza = unix_index[unix_index.Date.dt.]

# index a season, ex: summer
summer_pizza = unix_index['2013-05-01':'2013-08-31']
print("\nSummer pizzas:")
# display(summer_pizza.head(5))
%matplotlib inline

# plot pizzas by month 
unix_index_base = unix_index.drop(['Hour'], axis=1)
month_pizzas = unix_index_base.resample('M').sum()
month_pizzas_area = month_pizzas.plot(kind='area')


month_pizzas_bar = month_pizzas.plot(kind='bar')
# Analyze pattern by days/weeks

# Take a look at the highest frequency months (07/31/2012 and 4/30/2013)
July2012 = unix_index['7/2012'].resample('D').mean()
# July2012_area = July2012.plot(kind='area')

April2013 = unix_index['4/2013'].resample('W').mean()
April2013_area = April2013.plot(kind='area')


# Take a look at the lowest frequency months
Nov2012 = unix_index['11/2012'].resample('D').mean()
# Nov2012_area = Nov2012.plot(kind='area')

Jun2013 = unix_index['6/2013'].resample('W').mean()
Jun2013_area = Jun2013.plot(kind='area')

# Analyze pattern by hours

hourly = unix_index['10/2012':'9/2013'].groupby('Hour')['requester_received_pizza'].sum()
print(hourly.describe())
print(hourly)
print("Pizza success by hour of the day")
hourly.plot('area')<jupyter_output>count    24.00000
mean     10.75000
std       8.78858
min       0.00000
25%       3.50000
50%       8.50000
75%      16.00000
max      32.00000
Name: requester_received_pizza, dtype: float64
Hour
0     32.0
1     23.0
2     13.0
3      8.0
4      8.0
5      5.0
6      4.0
7      2.0
8      0.0
9      1.0
10     1.0
11     1.0
12     2.0
13     4.0
14     9.0
15     8.0
16    11.0
17    13.0
18    15.0
19    15.0
20    19.0
21    19.0
22    20.0
23    25.0
Name: requester_received_pizza, dtype: float64
Pizza success by hour of the day<jupyter_text>Section 2: Status and DateTime models  LR models for Status only<jupyter_code># LOGISTIC REGRESSION MODELS
from sklearn.linear_model import LogisticRegression

lr = LogisticRegression(penalty='l2')

# create training labels for pizza success
train_labels = np.asarray(allTData['requester_received_pizza'])
dev_labels = np.asarray(allDData['requester_received_pizza'])

# Z-scored for Request-Retrieval Z-score
diff_combined = np.asarray(statusTtrain[['Request-Retrieval Z-score']].copy())
diff_dev_combined = np.asarray(statusDtrain[['Request-Retrieval Z-score']].copy())
train_combine_model = lr.fit(diff_combined, train_labels)
dev_combine_labels = train_combine_model.predict(diff_dev_combined)
combine_score = train_combine_model.score(diff_dev_combined, dev_labels)
combine_F1score = metrics.f1_score(dev_combine_labels, dev_labels)

# Z-scores for Request, Retrieval, and Request-Retrieval Z-scores and Flair
zcomb_train = statusTtrain[['Request Z-score','Retrieval Z-score','Request-Retrieval Z-score', 'requester_user_flair']]
zcomb_train['requester_user_flair'] = zcomb_train['requester_user_flair'].apply(flair_numeric).fillna(0)

zcomb_dev = statusDtrain[['Request Z-score','Retrieval Z-score','Request-Retrieval Z-score', 'requester_user_flair']]
zcomb_dev['requester_user_flair'] = zcomb_dev['requester_user_flair'].apply(flair_numeric).fillna(0)

zcomb_model = lr.fit(zcomb_train, train_labels)
zcomb_labels = zcomb_model.predict(zcomb_dev)
zcomb_score = zcomb_model.score(zcomb_dev, dev_labels)
zcomb_F1score = metrics.f1_score(zcomb_labels, dev_labels)

# Z-scores for Request, Retrieval, and Request-Retrieval Z-scores
z_status_train = statusTtrain[['Request Z-score','Retrieval Z-score','Request-Retrieval Z-score']]
z_status_dev = statusDtrain[['Request Z-score','Retrieval Z-score','Request-Retrieval Z-score']]

z_status_model = lr.fit(z_status_train, train_labels)
z_status_labels = z_status_model.predict(z_status_dev)
z_status_score = z_status_model.score(z_status_dev, dev_labels)
z_status_F1score = metrics.f1_score(z_status_labels, dev_labels)

                       
print("\nFor model with Request-Retrieval Z-Score only:")
print("Accuracy:", combine_score)
print("F1 score:", combine_F1score)

print("\nZ-scores for Request, Retrieval, Request-Retrieval Z-Score and Flair:")
print("Accuracy:", zcomb_score)
print("F1 score:", zcomb_F1score)
                        
print("\nZ-scores for Request, Retrieval, Request-Retrieval Z-Score:")
print("Accuracy:", z_status_score)
print("F1 score:", z_status_F1score)<jupyter_output>For model with Request-Retrieval Z-Score only:
Accuracy: 0.738
F1 score: 0.0

Z-scores for Request, Retrieval, Request-Retrieval Z-Score and Flair:
Accuracy: 1.0
F1 score: 1.0

Z-scores for Request, Retrieval, Request-Retrieval Z-Score:
Accuracy: 0.737
F1 score: 0.0<jupyter_text>LR models for status and time<jupyter_code># Group scoring functions

def roc_curve1(y_true, y_pred_prob):
    """This function plots the ROC curve
    Inputs: y_true, correct label
            y_pred_prob, predicted probabilities
    """
    fpr, tpr, thr = roc_curve(y_true, y_pred_prob)
   
    plt.figure()
    plt.plot(fpr,tpr)
    plt.xlabel("False positive rate")
    plt.ylabel("True positive rate")
    plt.title("ROC Curve")
    plt.show()
    

def score_rep(y_true, y_pred, desc):
    """Function to print out comprehensive report for classification test
    Inputs: y_true, correct label
            y_pred, predicted label from model
            desc, description of model
    Output: classification report
    """
    print(desc)
    print("-"*75)
    print("Accuracy: ", metrics.accuracy_score(y_true, y_pred))
    print("Area under curve of ROC: ", metrics.roc_auc_score(y_true, y_pred))
    print("Classification report:\n")
    print(metrics.classification_report(y_true, y_pred))
    print("-"*75)
    
# def roc_curve1(y_true, y_pred_prob):
# def score_rep(y_true, y_pred, desc):
# FOR STATUS AND TIME BY HOUR
lr = LogisticRegression(penalty='l2', C=0.01)

train_labels = np.asarray(allTData['requester_received_pizza'])
dev_labels = np.asarray(allDData['requester_received_pizza'])

# setup training and development datasets
# train
stathourTData = pandas.concat([statusTtrain['Request-Retrieval Z-score'], allTData['unix_timestamp_of_request_utc']], axis=1, 
                              keys=['Request-Retrieval Z-score', 'unix_timestamp_of_request_utc'])
stathourTData['Datetime'] = pandas.to_datetime(stathourTData['unix_timestamp_of_request_utc'], unit='s')
stathourTData['Hour'] = stathourTData['Datetime'].dt.hour
stathourTData = stathourTData.drop(['Datetime', 'unix_timestamp_of_request_utc'], axis=1)
# display(stathourTData.head(5))

# development
stathourDData = pandas.concat([statusDtrain['Request-Retrieval Z-score'], allDData['unix_timestamp_of_request_utc']], axis=1, 
                              keys=['Request-Retrieval Z-score', 'unix_timestamp_of_request_utc'])
stathourDData['Datetime'] = pandas.to_datetime(stathourDData['unix_timestamp_of_request_utc'], unit='s')
stathourDData['Hour'] = stathourDData['Datetime'].dt.hour
stathourDData = stathourDData.drop(['Datetime', 'unix_timestamp_of_request_utc'], axis=1)
# display(stathourDData.head(5))

# fit LR models
train_stathour_model = lr.fit(stathourTData, train_labels)
dev_stathour_labels = train_stathour_model.predict(stathourDData)
stathour_score = train_stathour_model.score(stathourDData, dev_labels)
stathour_F1score = metrics.f1_score(dev_stathour_labels, dev_labels)

print("\nScoring for Request-retrieval Z-scores and Hour")
print("Accuracy:", stathour_score)
print("F1 score:", stathour_F1score)
roc_curve1(dev_labels, train_stathour_model.predict_proba(stathourDData)[:,0])
score_rep(dev_labels, dev_stathour_labels,"Logistic Regression, C = 0.01")
# FOR STATUS AND BUCKETED HOUR
lr = LogisticRegression(penalty='l2')

# Bucket hours
def bucket(x):
    if x >= 6 and x <= 13:
        return 0
    if x >= 0 and x <=2:
        return 1
    if x >= 16 and x <=23:
        return 1
    else:
        return 2

bucketTData = stathourTData
bucketTData['Hour'] = bucketTData['Hour'].apply(bucket)
bucketDData = stathourDData
bucketDData['Hour'] = bucketDData['Hour'].apply(bucket)

# fit LR models
train_buckethr_model = lr.fit(bucketTData, train_labels)
dev_buckethr_labels = train_buckethr_model.predict(bucketDData)
buckethr_score = train_buckethr_model.score(bucketDData, dev_labels)
buckethr_F1score = metrics.f1_score(dev_buckethr_labels, dev_labels)

print("\nScoring for Request-retrieval Z-scores and bucketed Hour:")
print("Accuracy:", buckethr_score)
print("F1 score:", buckethr_F1score)
roc_curve1(dev_labels, train_buckethr_model.predict_proba(bucketDData)[:,0])
score_rep(dev_labels, dev_buckethr_labels,"Logistic Regression, C = 0.01")
# FOR STATUS, BUCKETED HOUR, and HALVES OF MONTH

# def roc_curve1(y_true, y_pred_prob)
# def score_rep(y_true, y_pred, desc)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup