<jupyter_start><jupyter_text>Import Libraries<jupyter_code>import os 
import math
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns 
from datetime import datetime
from itertools import product
from scipy import stats
from sklearn.metrics import mean_squared_error
import statsmodels.api as sm
from statsmodels.tsa.stattools import adfuller, kpss
from statsmodels.tsa.seasonal import seasonal_decompose
from statsmodels.graphics.tsaplots import plot_acf, plot_pacf
import warnings
plt.style.use('ggplot')
warnings.filterwarnings("ignore")<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>df = pd.read_csv('./aggregated_df_final01oct2021.csv')
df.shape
df.describe(include= 'all')
df.head(2)
df.head(2).transpose()<jupyter_output><empty_output><jupyter_text>Extract relevant features - set the 'time' column as index of the data - extract relevant features for time series analysis<jupyter_code>df = df.set_index('time')
df.head(2)
df = df.loc[:,['close_x', 'btc_s2f', 'btc_s2f_price']]
df.head(5)
# Plotting movement of OHLC over the period 2017-2021
df.loc[:,df.columns].plot(figsize = (12,6))
# Converting index to datetime format
df.index = pd.to_datetime(df.index)
df.head(5)<jupyter_output><empty_output><jupyter_text>Resampling - daily frequency - monthly frequency - annual frequency - quarterly frequency<jupyter_code># Resampling to daily frequency
df_day = df.resample('D').mean()
# df_day = df.groupby(df.index).mean()

# Resampling to monthly frequency
df_month = df.resample('M').mean()

# Resampling to annual frequency
df_year = df.resample('A-DEC').mean()

# Resampling to quarterly frequency
df_quarter = df.resample('Q-DEC').mean()
len(df_day.close_x)
# # temporary check
# pd.Series(df_day.close_x.isnull().index).apply(lambda x: x.year).value_counts().sum()
display(df_day.head(3), df_month.head(3), df_year.head(3), df_quarter.head(3))
# Plotting subplots of OHLC by days, months, quarters and years
fig = plt.figure(figsize=[20,6])
plt.suptitle('Bitcoin Exchanges (USD)', fontsize=22)

plt.subplot(221)
plt.plot(df_day.close_x, '-', label='By Days')
plt.legend()

plt.subplot(222)
plt.plot(df_month.close_x, '-', label='By Months')
plt.legend()

plt.subplot(223)
plt.plot(df_quarter.close_x, '-', label='By Quarters')
plt.legend()

plt.subplot(224)
plt.plot(df_year.close_x, '-', label='By Years')
plt.legend()

plt.show()
# Plotting subplots of OHLC by days, months, quarters and years
fig = plt.figure(figsize=[20,16])
plt.suptitle('Bitcoin Exchanges (USD)', fontsize=22)

plt.subplot(411)
plt.plot(df_day.close_x, '-', label='By Days')
plt.legend()

plt.subplot(412)
plt.plot(df_month.close_x, '-', label='By Months')
plt.legend()

plt.subplot(413)
plt.plot(df_quarter.close_x, '-', label='By Quarters')
plt.legend()

plt.subplot(414)
plt.plot(df_year.close_x, '-', label='By Years')
plt.legend()

plt.show()<jupyter_output><empty_output><jupyter_text>Check Stationarity<jupyter_code># Checking Stationarity 
plt.figure(figsize=(12,6))
sm.tsa.seasonal_decompose(df_month.close_x).plot()
print("Dickey–Fuller test: p=%f" % sm.tsa.stattools.adfuller(df_month.close_x)[1])
plt.show()<jupyter_output>Dickey–Fuller test: p=0.893560<jupyter_text>From the p-value of Dickey-Fuller test, we conclude that the series is not stationary.<jupyter_code>df_month.close_x.min()<jupyter_output><empty_output><jupyter_text>minmax norm<jupyter_code>s_mm = df_month.close_x.apply(lambda x: ( x - min(df_month.close_x) + 100 ) / ( max(df_month.close_x) - min(df_month.close_x)  ) )
s_mm
df_month['close_x'] = s_mm
df_month.head()
# Box-Cox Transformations & Checking Stationarity 
df_month['close_x'], lmbda = stats.boxcox(df_month.close_x)
print("Dickey–Fuller test: p=%f" % sm.tsa.stattools.adfuller(df_month.close_x)[1])
sm.tsa.seasonal_decompose(df_month.close_x).plot();
# Box-Cox Transformations & Checking Stationarity 
df_month['close_x'], lmbda = stats.boxcox(df_month.close_x)
print("Dickey–Fuller test: p=%f" % sm.tsa.stattools.adfuller(df_month.close_x)[1])
sm.tsa.seasonal_decompose(df_month.close_x).plot();<jupyter_output><empty_output><jupyter_text>Time series data still not stationary<jupyter_code>df_month.close_x.hist()
import pandas as pd
from sklearn import preprocessing

x = df_month.close_x #returns a numpy array
min_max_scaler = preprocessing.MinMaxScaler()   # max - min / ... 
x_scaled = min_max_scaler.fit_transform(x.values.reshape(-1,1))
dff = pd.DataFrame(x_scaled)
dff[0].hist()
# Box-Cox Transformations & Checking Stationarity 
# after minmax norm.
s_mm, lmbda = stats.boxcox(s_mm)
print("Dickey–Fuller test: p=%f" % sm.tsa.stattools.adfuller(s_mm))
sm.tsa.seasonal_decompose(s_mm).plot();
import pandas as pd
from sklearn import preprocessing

x = df_month.close_x #returns a numpy array
min_max_scaler = preprocessing.StandardScaler()   # max - min / ... 
x_scaled = min_max_scaler.fit_transform(x.values.reshape(-1,1))
dff = pd.DataFrame(x_scaled)
dff[0].hist()
dff<jupyter_output><empty_output><jupyter_text>Initial approximation - Autocorrelation - Partial Autocorrelation Plots<jupyter_code># Initial approximation of parameters using Autocorrelation and Partial Autocorrelation Plots

plt.figure(figsize=(15,7))

ax = plt.subplot(211)
sm.graphics.tsa.plot_acf(df_month.close_x, ax=ax)

ax = plt.subplot(212)
sm.graphics.tsa.plot_pacf(df_month.close_x, ax=ax)

plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>As seen in the preceding figure, the partial autocorrelation only at lag one is statistically significant while for rest the lags it is within the 95% confidence intervals. Hence, for BTC close values the order of AR models is one.<jupyter_code># Initial approximation of parameters
Qs = range(0, 2)
qs = range(0, 3)
Ps = range(0, 3)
ps = range(0, 3)
D=1
d=1
parameters = product(ps, qs, Ps, Qs)
parameters_list = list(parameters)
print(len(parameters_list))
# Model Selection

results = []
best_aic = float("inf")

for param in parameters_list:
    try:
        model=sm.tsa.statespace.SARIMAX(df_month.close_x, enforce_stationarity=False, order=(param[0], d, param[1]), 
                                        seasonal_order=(param[2], D, param[3], 12)).fit(disp=-1)
    except ValueError:
        print('wrong parameters:', param)
        continue
    aic = model.aic
    if aic < best_aic:
        best_model = model
        best_aic = aic
        best_param = param
    results.append([param, model.aic])
    
    
    
# Best Models
result_table = pd.DataFrame(results)
result_table.columns = ['parameters', 'aic']
# display(result_table.sort_values(by = 'aic', ascending=True).head())
# display(best_model.summary())
# STL-decomposition

plt.figure(figsize=(15,7))

plt.subplot(211)
best_model.resid[13:].plot()
plt.ylabel(u'Residuals')

ax = plt.subplot(212)
sm.graphics.tsa.plot_acf(best_model.resid[13:].values.squeeze(), lags=20, ax=ax)

print("Dickey–Fuller test:: p=%f" % sm.tsa.stattools.adfuller(best_model.resid)[1])

plt.tight_layout()
plt.show()
# Best Models
result_table =a pd.DataFrame(results)
result_table.columns = ['parameters', 'aic']
display(result_table.sort_values(by = 'aic', ascending=True).head())
display(best_model.summary())
# STL-decomposition

plt.figure(figsize=(15,7))

plt.subplot(211)
best_model.resid[13:].plot()
plt.ylabel(u'Residuals')

ax = plt.subplot(212)
sm.graphics.tsa.plot_acf(best_model.resid[13:].values.squeeze(), lags=20, ax=ax)

print("Dickey–Fuller test:: p=%f" % sm.tsa.stattools.adfuller(best_model.resid)[1])

plt.tight_layout()
plt.show()
# Inverse Box-Cox Transformation Function

def invboxcox(y,lmbda):
    if lmbda == 0:
        return(np.exp(y))
    else:
        return(np.exp(np.log(lmbda*y+1)/lmbda))
# Prediction

df_month2 = df_month[['close_x']]
df_month2.close_x = df_month2.close_x.apply(lambda x: invboxcox(x, lmbda))
date_list = [datetime(2021, 9, 30), datetime(2021, 10, 31), datetime(2021, 11, 30), datetime(2021, 12, 31)]
future = pd.DataFrame(index=date_list, columns= df_month.columns)
df_month2 = pd.concat([df_month2, future])
df_month2['forecast'] = invboxcox(best_model.predict(start=0, end=75), lmbda)
plt.figure(figsize=(12,6))
df_month2.close_x.plot()
df_month2.forecast.plot(color='b', ls='--', label='Predicted Close Price')
plt.legend()
plt.title('Bitcoin exchanges, by months')
plt.ylabel('mean USD')
plt.show()<jupyter_output><empty_output><jupyter_text>Evaluation<jupyter_code>df_month2.head(5)
df_month2[['close_x', 'forecast']].dropna().head(5)
df_month2[df_month2['close_x'].isnull()]
df_month2[df_month2['forecast'].isnull()]
df_month3 = df_month2[['close_x', 'forecast']].dropna()
s_a = df_month3['close_x']
s_b = df_month3['forecast']

mean_squared_error(s_a.values, s_b.values)<jupyter_output><empty_output><jupyter_text>Daily (same work)<jupyter_code>df_day.head(10)
# Checking Stationarity 
plt.figure(figsize=(12,6))
sm.tsa.seasonal_decompose(df_day.close_x, freq=1).plot()
print("Dickey–Fuller test: p=%f" % sm.tsa.stattools.adfuller(df_day.close_x)[1])
plt.show()

# Box-Cox Transformations & Checking Stationarity 
df_day['close_x'], lmbda = stats.boxcox(df_day.close_x)
print("Dickey–Fuller test: p=%f" % sm.tsa.stattools.adfuller(df_day.close_x)[1])
sm.tsa.seasonal_decompose(df_day.close_x, freq=1).plot();
plt.show()

# Initial approximation of parameters using Autocorrelation and Partial Autocorrelation Plots

plt.figure(figsize=(15,7))

ax = plt.subplot(211)
sm.graphics.tsa.plot_acf(df_day.close_x, ax=ax)

ax = plt.subplot(212)
sm.graphics.tsa.plot_pacf(df_day.close_x, ax=ax)

plt.tight_layout()
plt.show()
# Initial approximation of parameters
Qs = range(0, 2)
qs = range(0, 3)
Ps = range(0, 3)
ps = range(0, 3)
D=1
d=1
parameters = product(ps, qs, Ps, Qs)
parameters_list = list(parameters)
print(len(parameters_list))

# Model Selection

results = []
best_aic = float("inf")
parameters_list[0:10]
for param in parameters_list:
    try:
        model=sm.tsa.statespace.SARIMAX(df_day.close_x, 
                                        order=(param[0], d, param[1]), 
                                        seasonal_order=(param[2], D, param[3], 365)).fit(disp=-1)
    except ValueError:
        print('wrong parameters:', param)
        continue
    aic = model.aic
    if aic < best_aic:
        best_model = model
        best_aic = aic
        best_param = param
    results.append([param, model.aic])
    
    
    
# Best Models
result_table = pd.DataFrame(results)
result_table.columns = ['parameters', 'aic']
# display(result_table.sort_values(by = 'aic', ascending=True).head())
# display(best_model.summary())



# STL-decomposition

plt.figure(figsize=(15,7))

plt.subplot(211)
best_model.resid[13:].plot()
plt.ylabel(u'Residuals')

ax = plt.subplot(212)
sm.graphics.tsa.plot_acf(best_model.resid[13:].values.squeeze(), lags=20, ax=ax)

print("Dickey–Fuller test:: p=%f" % sm.tsa.stattools.adfuller(best_model.resid)[1])

plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>Quarterly (same work)<jupyter_code>df_quarter.head(2)
# Checking Stationarity 
plt.figure(figsize=(12,6))
sm.tsa.seasonal_decompose(df_quarter.close_x, freq=1).plot()
print("Dickey–Fuller test: p=%f" % sm.tsa.stattools.adfuller(df_quarter.close_x)[1])
plt.show()

# Box-Cox Transformations & Checking Stationarity 
df_quarter['close_x'], lmbda = stats.boxcox(df_quarter.close_x)
print("Dickey–Fuller test: p=%f" % sm.tsa.stattools.adfuller(df_quarter.close_x)[1])
sm.tsa.seasonal_decompose(df_quarter.close_x, freq=1).plot();
plt.show()

# Initial approximation of parameters using Autocorrelation and Partial Autocorrelation Plots

plt.figure(figsize=(15,7))

ax = plt.subplot(211)
sm.graphics.tsa.plot_acf(df_quarter.close_x, ax=ax)

ax = plt.subplot(212)
sm.graphics.tsa.plot_pacf(df_quarter.close_x, ax=ax)

plt.tight_layout()
plt.show()
# Initial approximation of parameters
Qs = range(0, 2)
qs = range(0, 3)
Ps = range(0, 3)
ps = range(0, 3)
D=1
d=1
parameters = product(ps, qs, Ps, Qs)
parameters_list = list(parameters)
print(len(parameters_list))

# Model Selection

results = []
best_aic = float("inf")
for param in parameters_list:
    try:
        model=sm.tsa.statespace.SARIMAX(df_quarter.close_x, order=(param[0], d, param[1]), seasonal_order=(param[2], D, param[3], 12)).fit(disp=-1)
    except ValueError:
        print('wrong parameters:', param)
        continue
    aic = model.aic
    if aic < best_aic:
        best_model = model
        best_aic = aic
        best_param = param
    results.append([param, model.aic])
    
    
    
# Best Models
result_table = pd.DataFrame(results)
result_table.columns = ['parameters', 'aic']
# display(result_table.sort_values(by = 'aic', ascending=True).head())
# display(best_model.summary())



# STL-decomposition

plt.figure(figsize=(15,7))

plt.subplot(211)
best_model.resid[13:].plot()
plt.ylabel(u'Residuals')

ax = plt.subplot(212)
sm.graphics.tsa.plot_acf(best_model.resid[13:].values.squeeze(), lags=20, ax=ax)

print("Dickey–Fuller test:: p=%f" % sm.tsa.stattools.adfuller(best_model.resid)[1])

plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>Yearly (Same)<jupyter_code>df_year
# Checking Stationarity 
plt.figure(figsize=(12,6))
sm.tsa.seasonal_decompose(df_year.close_x, freq=1).plot()
print("Dickey–Fuller test: p=%f" % sm.tsa.stattools.adfuller(df_year.close_x)[1])
plt.show()

# Box-Cox Transformations & Checking Stationarity 
df_year['close_x'], lmbda = stats.boxcox(df_year.close_x)
print("Dickey–Fuller test: p=%f" % sm.tsa.stattools.adfuller(df_year.close_x)[1])
sm.tsa.seasonal_decompose(df_year.close_x, freq=1).plot()
plt.show()

# Initial approximation of parameters using Autocorrelation and Partial Autocorrelation Plots

plt.figure(figsize=(15,7))

ax = plt.subplot(211)
sm.graphics.tsa.plot_acf(df_year.close_x, ax=ax)

ax = plt.subplot(212)
sm.graphics.tsa.plot_pacf(df_year.close_x, ax=ax)

plt.tight_layout()
plt.show()

Qs = range(0, 2)
qs = range(0, 3)
Ps = range(0, 3)
ps = range(0, 3)
D=1
d=1
parameters = product(ps, qs, Ps, Qs)
parameters_list = list(parameters)
print(len(parameters_list))

# Model Selection

results = []
best_aic = float("inf")


for param in parameters_list:
    try:   
        model=sm.tsa.statespace.SARIMAX(df_year.close_x, 
                                        order=(param[0], y, param[1]), seasonal_order=(param[2], D, param[3], 12)).fit(disp=-1)
    except ValueError:
        print('wrong parameters:', param)
        continue
    aic = model.aic
    if aic < best_aic:
        best_model = model
        best_aic = aic
        best_param = param
    results.append([param, model.aic])
    
    
    
# Best Models
result_table = pd.DataFrame(results)
result_table.columns = ['parameters', 'aic']
# display(result_table.sort_values(by = 'aic', ascending=True).head())
# display(best_model.summary())

best_model.resid[:]
# STL-decomposition

plt.figure(figsize=(15,7))

plt.subplot(211)
best_model.resid.plot()
plt.ylabel(u'Residuals')

ax = plt.subplot(212)
sm.graphics.tsa.plot_acf(best_model.resid.values[:].squeeze(), lags=2, ax=ax)

print("Dickey–Fuller test:: p=%f" % sm.tsa.stattools.adfuller(best_model.resid)[1])

plt.tight_layout()
plt.show()
df_year
df_year2 = df_year[['close_x']]
df_year2.close_x = df_year2.close_x.apply(lambda x: invboxcox(x, lmbda))

# year pred
date_list = [datetime(2020, 12, 31), datetime(2021, 12, 31), datetime(2022, 12, 31), datetime(2023, 12, 31)]
future = pd.DataFrame(index=date_list, columns= df_year.columns)
df_year2 = pd.concat([df_year2, future])
df_year2['forecast'] = invboxcox(best_model.predict(start=0, end=75), lmbda)

plt.figure(figsize=(12,6))
df_year2.close_x.plot()
df_year2.forecast.plot(color='b', ls='--', label='Predicted Close Price')
plt.legend()
plt.title('Bitcoin exchanges, by months')
plt.ylabel('mean USD')
plt.show()
df_year2 = df_year2[['close_x', 'forecast']].dropna()
s_a = df_year2['close_x']
s_b = df_year2['forecast']
mean_squared_error(s_a.values, s_b.values)<jupyter_output><empty_output><jupyter_text>gc  Graphing - only without null<jupyter_code>plt.figure(figsize=(12,6))
df_month3.close_x.plot()
plt.figure(figsize=(12,6))
df_month3.close_x.plot()
df_month3.forecast.plot(color='b', ls='--', label='Predicted Close Price')
df_month3.forecast.describe()
df_month3.forecast.sort_values(ascending=False)
df_month4 = df_month3[df_month3['forecast']!=max(df_month3.forecast)]
df_month4.head(5)
plt.figure(figsize=(12,6))
df_month4.close_x.plot()
df_month4.forecast.plot(color='b', ls='--', label='Predicted Close Price')
plt.legend()
plt.title('Bitcoin exchanges, by months')
plt.ylabel('mean USD')
plt.show()
df<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup