<jupyter_start><jupyter_text>Filter to 2017 + data<jupyter_code>just_rr = just_rr.loc[just_rr["year"] >= 2017 ].drop("Unnamed: 0", axis = 1).sort_values(["month", "year"])
rr_cc = rr_cc.loc[rr_cc["year"] >= 2017 ].drop("Unnamed: 0", axis = 1).sort_values(["month", "year"])<jupyter_output><empty_output><jupyter_text>Filter out rows with no demand or reviews<jupyter_code>just_rr = just_rr.loc[just_rr["demand"] > 0]
just_rr = just_rr.loc[just_rr["percent_neutral"] + just_rr["percent_positive"] + just_rr["percent_negative"] > 0]

rr_cc = rr_cc.loc[rr_cc["demand"] > 0]
rr_cc = rr_cc.loc[rr_cc["percent_neutral"] + rr_cc["percent_positive"] + rr_cc["percent_negative"] > 0]
rr_cc<jupyter_output><empty_output><jupyter_text>Models<jupyter_code>test = test = rr_cc[['demand_F1','percent_1', 'percent_2',
       'percent_3', 'percent_4', 'percent_5', 'percent_negative',
       'percent_neutral', 'percent_positive', 'demand_P1',
       'demand_P2', 'demand', 'percent_Complaints',
       'percent_Compliments']]
corrplot(test.corr())<jupyter_output><empty_output><jupyter_text>R&R Data + Demand Data<jupyter_code>X = cp.deepcopy(just_rr)
y = X.pop("demand_F1")
X_train, X_test, y_train, y_test = train_test_split(X, y, shuffle = False)
lm = LinearRegression()
lm.fit(X_train, y_train)
pred = lm.predict(X_test)
metrics.r2_score(pred, y_test)
X2 = sm.add_constant(X_train)
est = sm.OLS(y_train, X2)
est2 = est.fit()

print(est2.summary())<jupyter_output>OLS Regression Results                            
==============================================================================
Dep. Variable:              demand_F1   R-squared:                       0.847
Model:                            OLS   Adj. R-squared:                  0.847
Method:                 Least Squares   F-statistic:                     2258.
Date:                Fri, 10 Jul 2020   Prob (F-statistic):               0.00
Time:                        12:01:24   Log-Likelihood:                -79247.
No. Observations:                6126   AIC:                         1.585e+05
Df Residuals:                    6110   BIC:                         1.586e+05
Df Model:                          15                                         
Covariance Type:            nonrobust                                         
=======================================================================================
                          coef    std err  [...]<jupyter_text>Just R&R Data<jupyter_code>X= X[['month', 'year', "demand", "demand_P1", "demand_P2",'percent_negative', 'percent_neutral', 'percent_positive']]
X_train, X_test, y_train, y_test = train_test_split(X, y, shuffle = False)
lm = LinearRegression()
lm.fit(X_train, y_train)
pred = lm.predict(X_test)
metrics.r2_score(pred, y_test)
X2 = sm.add_constant(X_train)
est = sm.OLS(y_train, X2)
est2 = est.fit()

print(est2.summary())<jupyter_output>OLS Regression Results                            
==============================================================================
Dep. Variable:              demand_F1   R-squared:                       0.846
Model:                            OLS   Adj. R-squared:                  0.846
Method:                 Least Squares   F-statistic:                     4793.
Date:                Fri, 10 Jul 2020   Prob (F-statistic):               0.00
Time:                        12:02:20   Log-Likelihood:                -79275.
No. Observations:                6126   AIC:                         1.586e+05
Df Residuals:                    6118   BIC:                         1.586e+05
Df Model:                           7                                         
Covariance Type:            nonrobust                                         
====================================================================================
                       coef    std err        [...]<jupyter_text>R&R + Demand + PCA<jupyter_code>X = cp.deepcopy(just_rr)
y = X.pop("demand_F1")
X = StandardScaler().fit_transform(X)
pca = PCA().fit(X)
plt.plot(np.cumsum(pca.explained_variance_ratio_))
plt.xlabel('number of components')
plt.ylabel('cumulative explained variance')
pca = PCA(n_components=12)
principalComponents = pca.fit_transform(X)
principalDf = pd.DataFrame(data = principalComponents)
X_train, X_test, y_train, y_test = train_test_split(principalDf, y, shuffle = False)
lm = LinearRegression()
lm.fit(X_train, y_train)
pred = lm.predict(X_test)
metrics.r2_score(pred, y_test)
X2 = sm.add_constant(X_train)
est = sm.OLS(y_train.values, X2)
est2 = est.fit()

print(est2.summary())
for i,x in enumerate(pca.components_):
    print("Feature " + str(i), ", Explains " + str(pca.explained_variance_[i]) + "%")
    top = x.argsort()[-5:][::-1]
    table = [list(just_rr.drop(["demand_F1"], axis = 1).columns[top]), x[top]]
    df = pd.DataFrame(np.transpose(table))
    print(df)
principalDf["y"] = y
principalDf.corr()["y"]<jupyter_output><empty_output><jupyter_text>Demand + PCA<jupyter_code>X = cp.deepcopy(just_rr)
y = X.pop("demand_F1")
X = X[['demand', 'demand_P2', 'demand_P1', 'avg_subcat_demand',
       'monthly_avg_demand', 'avg_12_month_demand', 'avg_brand_demand',
       'month', 'year']]
X = StandardScaler().fit_transform(X)
pca = PCA().fit(X)
plt.plot(np.cumsum(pca.explained_variance_ratio_))
plt.xlabel('number of components')
plt.ylabel('cumulative explained variance')
pca = PCA(n_components=6)
principalComponents = pca.fit_transform(X)
principalDf = pd.DataFrame(data = principalComponents)
X_train, X_test, y_train, y_test = train_test_split(principalDf, y, shuffle = False)
lm = LinearRegression()
lm.fit(X_train, y_train)
pred = lm.predict(X_test)
metrics.r2_score(pred, y_test)
X2 = sm.add_constant(X_train)
est = sm.OLS(y_train.values, X2)
est2 = est.fit()

print(est2.summary()) 
for i,x in enumerate(pca.components_):
    print("Feature " + str(i), ", Explains " + str(pca.explained_variance_[i]) + "%")
    top = x.argsort()[-5:][::-1]
    table = [list(just_rr.drop(["demand_F1"], axis = 1).columns[top]), x[top]]
    df = pd.DataFrame(np.transpose(table))
    print(df)<jupyter_output>Feature 0 , Explains 3.4310818814472785%
                   0                    1
0             demand   0.5073913100301518
1          demand_P1  0.49650808709856203
2          demand_P2   0.4864246675582888
3  avg_subcat_demand  0.36104681086759594
4   avg_brand_demand  0.32980521276018443
Feature 1 , Explains 1.9866058811247422%
                   0                      1
0               year     0.2906456004268644
1          demand_P2    0.10163335211569667
2          demand_P1    0.09013172704507094
3             demand    0.08270837918971535
4  avg_subcat_demand  -0.002314298119528941
Feature 2 , Explains 1.0090322033865928%
                    0                      1
0  monthly_avg_demand     0.3850219018977986
1    avg_brand_demand    0.13788134370125912
2   avg_subcat_demand  -0.008510872286961332
3              demand  -0.023995980647756304
4           demand_P2   -0.05247664825506044
Feature 3 , Explains 0.7962477474798098%
                    0                      1
0    [...]<jupyter_text>R&R + CC + Demand + PCA<jupyter_code>X = cp.deepcopy(rr_cc)
y = X.pop("demand_F1")
X = StandardScaler().fit_transform(X)
pca = PCA().fit(X)
plt.plot(np.cumsum(pca.explained_variance_ratio_))
plt.xlabel('number of components')
plt.ylabel('cumulative explained variance')
pca = PCA(n_components=80)
principalComponents = pca.fit_transform(X)
principalDf = pd.DataFrame(data = principalComponents)
X_train, X_test, y_train, y_test = train_test_split(principalDf, y, shuffle = False)
lm = LinearRegression()
lm.fit(X_train, y_train)
pred = lm.predict(X_test)
metrics.r2_score(pred, y_test)
X2 = sm.add_constant(X_train)
est = sm.OLS(y_train.values, X2)
est2 = est.fit()

print(est2.summary())<jupyter_output>OLS Regression Results                            
==============================================================================
Dep. Variable:                      y   R-squared:                       0.858
Model:                            OLS   Adj. R-squared:                  0.856
Method:                 Least Squares   F-statistic:                     455.3
Date:                Tue, 07 Jul 2020   Prob (F-statistic):               0.00
Time:                        09:47:47   Log-Likelihood:                -79029.
No. Observations:                6126   AIC:                         1.582e+05
Df Residuals:                    6045   BIC:                         1.588e+05
Df Model:                          80                                         
Covariance Type:            nonrobust                                         
==============================================================================
                 coef    std err          t      P>|[...]
StarcoderdataJupyterStructuredCleanDedup