% !TEX root = ../main.tex

\chapter{Methodology}
\thispagestyle{myheadings}

\section{HAR model}

To capture the long-memory behaviour of volatility movement, this research uses the HAR model suggested by \cite{corsi_simple_2009} with a variation of logarithm in the form of Equation \ref{eq:HAR_model}.

\begin{equation}
RV_t = \beta_0 + \beta_1 RV_{t-1} +\beta_2 RV_{t-1|t-7} + \beta_3 RV_{t-1|t-{30}} + u_t
\label{eq:HAR_model}
\end{equation}

We can see clearly that an AR(1) process from the PACF plot also has effects in the following rolling period. As $ RV_{t-j|t-k}=\frac{1}{h+1-j}\sum_{i=j}^h RV_{t-i},\; $ with $j\leqslant{h}$. Hereby Realized Volatility is averaged on both weekly and monthly basis, to capture the Autoregressive effect in the longer period.\\

\begin{comment}
\begin{table}[H]
\setlength{\tabcolsep}{6mm}{
\begin{tabular}{|c|l|c|c|c|c|}
\hline
\multicolumn{6}{|c|}{HAR Model Result}                                                                           \\ \hline
\multicolumn{2}{|c|}{Dep. Variable:}       & logRV        & \multicolumn{2}{c|}{R-squared:}          & 0.729     \\
\multicolumn{2}{|c|}{Model:}               & OLS          & \multicolumn{2}{c|}{Adj. R-squared:}     & 0.728     \\
\multicolumn{2}{|c|}{Method:}              & LeastSquares & \multicolumn{2}{c|}{F-statistic:}        & 1157.     \\
\multicolumn{2}{|c|}{Log-Likelihood:}      & -1521.0      & \multicolumn{2}{c|}{Prob (F-statistic):} & 8.32e-276 \\
\multicolumn{2}{|c|}{No. Observations:}    & 1298         & \multicolumn{2}{c|}{AIC:}                & 3050.     \\
\multicolumn{2}{|c|}{Df Residuals:}        & 1294         & \multicolumn{2}{c|}{BIC:}                & 3071.     \\
\multicolumn{2}{|c|}{Df Model:}            & 3            & \multicolumn{2}{c|}{Covariance Type:}    & nonrobust \\ \hline
\multicolumn{2}{|c|}{}                     & coef         & std err              & t                 & P>|t|     \\
\multicolumn{2}{|c|}{const}                & -0.7749      & 0.118                & -6.545            & 0.000     \\
\multicolumn{2}{|c|}{$\log(RV_{t-1})$}      & 0.2463       & 0.029                & 8.570             & 0.000      \\
\multicolumn{2}{|c|}{$\log(RV_{t-1|t-7})$}  & 0.6804       & 0.039                & 17.571            & 0.000      \\
\multicolumn{2}{|c|}{$\log(RV_{t-1|t-30})$} & -0.0073          & 0.031                & -0.237            & 0.813     \\ \hline
\multicolumn{2}{|c|}{Omnibus:}             & 63.912       & \multicolumn{2}{c|}{Durbin-Watson:}      & 1.614     \\
\multicolumn{2}{|c|}{Prob(Omnibus):}       & 0            & \multicolumn{2}{c|}{Jarque-Bera (JB):}   & 141.160   \\
\multicolumn{2}{|c|}{Skew:}                & 0.296        & \multicolumn{2}{c|}{Prob(JB):}           & 2.23e-31  \\
\multicolumn{2}{|c|}{Kurtosis:}            & 4.503        & \multicolumn{2}{c|}{Cond. No.}           & 66.5      \\ \hline
\end{tabular}}
\caption{HAR Model Result (under 10 min interval)} \label{HAR_model_result}
\end{table}
\end{comment}

\begin{equation}
RV_t = \beta_0 + \beta_1 RV_{t-1} +\beta_2 RV_{t-1|t-7} + u_t
\label{eq:sHAR_model}
\end{equation}

For easier interpretation and comparison with models later on
\footnote{Removed $\log(RV_{t-1|t-30})$ for most other models to avoid overfitting}. Modified HAR model in Equation \ref{eq:sHAR_model} removed insignificant $\log(RV_{t-1|t-30})$ to avoid overfitting. Removing $\log(RV_{t-1|t-30})$ results in no change in Adjusted R-squared which indicates the bare contribution in explaining $RV_t$.

\begin{comment}
\begin{table}[H]
\setlength{\tabcolsep}{6mm}{
\begin{tabular}{|c|c|c|c|c|}
\hline
\multicolumn{5}{|c|}{Modified HAR Model Result}                                                                          \\
\hline
Dep. Variable:         & logRV        & \multicolumn{2}{c|}{R-squared:}          & 0.728                                 \\
Model:                 & OLS          & \multicolumn{2}{c|}{Adj. R-squared:}     & 0.728                                 \\
Method:                & LeastSquares & \multicolumn{2}{c|}{F-statistic:}        & 1737.                                 \\
Log-Likelihood:        & -1521.0      & \multicolumn{2}{c|}{Prob (F-statistic):} & 0.00                                  \\
No. Observations:      & 1298         & \multicolumn{2}{c|}{AIC:}                & 3048.                                 \\
Df Residuals:          & 1295         & \multicolumn{2}{c|}{BIC:}                & 3064.                                 \\
Df Model:              & 2            & \multicolumn{2}{c|}{Covariance Type:}    & nonrobust                             \\
\hline
                       & coef         & std err & t                              & P\textgreater{}\textbar{}t\textbar{}  \\
const                  & -0.7651      & 0.111   & -6.906                         & 0.000                                 \\
$\log RV_{t-1}$        & 0.2463       & 0.029   & 8.573                          & 0.000                                 \\
$\log (RV_{t-1|t-7})$  & 0.6748       & 0.031   & 21.990                         & 0.000                                 \\
\hline
Omnibus:               & 64.455       & \multicolumn{2}{c|}{Durbin-Watson:}      & 1.615                                 \\
Prob(Omnibus):         & 0.000        & \multicolumn{2}{c|}{Jarque-Bera (JB):}   & 141.803                               \\
Skew:                  & 0.301        & \multicolumn{2}{c|}{Prob(JB):}           & 1.61e-31                              \\
Kurtosis:              & 4.503        & \multicolumn{2}{c|}{Cond. No.}           & 52.0                                  \\
\hline
\end{tabular}}
\caption{Modified HAR Model Result (under 10 min interval)}
\label{mhar_model_result}
\end{table}
\end{comment}

\section{HARQ model}

However, due to the existence of measurement errors in realized volatilities, the coefficients in the HAR model can be optimized by spliting a share to Realized Quarticity, present in Equation \ref{eq:HARQ_model}
%$\log(RV_{t-1|t-30})$ is removed for insignificancy,

\begin{equation}
\begin{split}
\left\{
             \begin{array}{lr}
             RV_t=\beta_0+\underbrace{(\beta_1+\beta_{1Q} RQ_{t-1}^{1/2})}_{\beta_{1,t}}RV_{t-1}+\underbrace{(\beta_2+\beta_{2Q} RQ_{t-1|t-7}^{1/2})}_{\beta_{2,t}}RV_{t-1|t-7}+u_t\\
             RQ=\frac{3}{M}\sum_{i=1}^Mr_{t,i}^4,\;RQ_{t-1|t-k}=\frac{1}{k}\sum_{j=1}^kRQ_{t-j}
             \end{array}
\right.
\end{split}
\label{eq:HARQ_model}
\end{equation}

In parallel to $RV_{t-j|t-k}$, $RQ_{t-1|t-k}$ is rolling average of previous certain period of Realized Quarticity.\\

Conjoint regression of Realized Quarticity and Realized Volatility is based on the idea that as Realized Volatility consists of Integrated Volatility and Brownian Motion, and unwanted Brownian Motion makes an extra impact on observed coefficient of Volatility. Separate $RQ_t^{1/2}\,$\footnote{$RQ_t$ takes the square root, to resize its unit to the same level of $RV_t$} from HAR model quantifies the effect from Brownian Motion. Given the positive weekly array in Figure \ref{PACF}. Brownian Motion should conduct negative impact on coefficients of historical realised volatility. Intuitively, subtract Brownian Motion from Realized Volatility filtered coefficients to construct more punctual models. \\

\begin{comment}
\begin{table}[H]
\setlength{\tabcolsep}{6mm}{
\begin{tabular}{|c|l|c|c|c|c|}
\hline
\multicolumn{6}{|c|}{HARQ Model Result}                                                                                \\ \hline
\multicolumn{2}{|c|}{Dep. Variable:}                         & logRV                        & \multicolumn{2}{c|}{R-squared:}                          & 0.729                      \\
\multicolumn{2}{|c|}{Model:}                                 & OLS                          & \multicolumn{2}{c|}{Adj. R-squared:}                     & 0.728                      \\
\multicolumn{2}{|c|}{Method:}                                & LeastSquares                 & \multicolumn{2}{c|}{F-statistic:}                        & 870.2                      \\
\multicolumn{2}{|c|}{Log-Likelihood:}                        & -1519.5                      & \multicolumn{2}{c|}{Prob (F-statistic):}                 & 2.69e-276                  \\
\multicolumn{2}{|c|}{No. Observations:}                      & 1298                         & \multicolumn{2}{c|}{AIC:}                                & 3049.                      \\
\multicolumn{2}{|c|}{Df Residuals:}                          & 1293                         & \multicolumn{2}{c|}{BIC:}                                & 3075.                      \\
\multicolumn{2}{|c|}{Df Model:}                              & 4                            & \multicolumn{2}{c|}{Covariance Type:}                    & nonrobust                  \\ \hline
\multicolumn{2}{|c|}{}                                       & coef                         & std err                    & t                           & P>|t|                      \\
\multicolumn{2}{|c|}{const}                                  & -1.2690                      & 0.357                      & -3.553                      & 0.000                      \\
\multicolumn{2}{|c|}{$\log(RV_{t-1})$}                        & 0.3355                        & 0.171                         & 1.965                       & 0.050                      \\
\multicolumn{2}{|c|}{$\log(RQ_{t-1}^{1/2}\cdot RV_{t-1})$}    & -0.0459                       & 0.079                      & -0.578                       & 0.563                          \\
\multicolumn{2}{|c|}{$\log(RV_{t-1|t-7})$}                    &    0.8166                      & 0.127                     & 6.407                       & 0.000                      \\
\multicolumn{2}{|l|}{$\log(RQ_{t-1|t-7}^{1/2}\cdot RV_{t-1|t-7})$} & \multicolumn{1}{|c|}{-0.0704} & \multicolumn{1}{|c|}{0.057} & \multicolumn{1}{|c|}{-1.237} & \multicolumn{1}{|c|}{0.216} \\ \hline
\multicolumn{2}{|c|}{Omnibus:}                               & 76.823                       & \multicolumn{2}{c|}{Durbin-Watson:}                      & 1.615                      \\
\multicolumn{2}{|c|}{Prob(Omnibus):}                         & 0.000                        & \multicolumn{2}{c|}{Jarque-Bera (JB):}                   & 158.995                     \\
\multicolumn{2}{|c|}{Skew:}                                  & 0.385                        & \multicolumn{2}{c|}{Prob(JB):}                           & 2.98e-35                   \\
\multicolumn{2}{|c|}{Kurtosis:}                              & 4.532                            & \multicolumn{2}{c|}{Cond. No.}                           & 497.                       \\ \hline
\end{tabular}}
\caption{HARQ Model Result (under 10 min interval)} \label{HARQ_model_result}
\end{table}
\end{comment}

As the difficulties of estimate accurately the quarticity parameters, there is an empirical investigation on the simplified version of HARQ model, only allows daily lagged RV the coefficient to vary as a function of $RQ^{1/2}$.\\

\begin{equation}
\begin{split}
\left\{
             \begin{array}{lr}
             RV_t=\beta_0+\underbrace{(\beta_1+\beta_{1Q} RQ_{t-1}^{1/2})}_{\beta_{1,t}}RV_{t-1}+\beta_2 RV_{t-1|t-7}+u_t\\
             RQ=\frac{3}{M}\sum_{i=1}^Mr_{t,i}^4,\;RQ_{t-1|t-k}=\frac{1}{k}\sum_{j=1}^kRQ_{t-j}
             \end{array}
\right.
\end{split}
\label{eq:sHARQ_model}
\end{equation}

It is substantial for the estimation error of daily (k = 1) simulated RV. To be expected, weekly measurement error variance should be much smaller than daily RV (normalised). Therefore, the attenuation bias for short HARQ model will not be severely affected by the absence of weekly coefficients.\\

\begin{comment}
\begin{equation}
RV_t=-1.1799+(0.4325-0.0892RQ_{t-1}^{1/2})RV_{t-1}+0.6640RV_{t-1|t-7}+u_t
\label{sHARQ_equation}
\end{equation}

\begin{table}[H]
\setlength{\tabcolsep}{6mm}{
\begin{tabular}{|c|l|c|c|c|}
\hline
\multicolumn{5}{|c|}{Short HARQ Model Result}                                                                                \\
\hline
Dep. Variable:                        & logRV        & \multicolumn{2}{c|}{R-squared:}          & 0.729                                 \\
Model:                                & OLS          & \multicolumn{2}{c|}{Adj. R-squared:}     & 0.728                                 \\
Method:                               & LeastSquares & \multicolumn{2}{c|}{F-statistic:}        & 1159.                                 \\
Log-Likelihood:                       & -1520.3      & \multicolumn{2}{c|}{Prob (F-statistic):} & 2.69e-276                             \\
No. Observations:                     & 1298         & \multicolumn{2}{c|}{AIC:}                & 3049.                                 \\
Df Residuals:                         & 1294         & \multicolumn{2}{c|}{BIC:}                & 3069.                                 \\
Df Model:                             & 3            & \multicolumn{2}{c|}{Covariance Type:}    & nonrobust                             \\
\hline
                                      & coef         & std err & t                              & P\textgreater{}\textbar{}t\textbar{}  \\
const                                 & -1.1799      & 0.350   & -3.372                         & 0.001                                 \\
$\log(RV_{t-1})$                      & 0.4325       & 0.152   & 2.851                          & 0.004                                 \\
$\log(RQ_{t-1}^{1/2}\cdot RV_{t-1})$  & -0.0892      & 0.071   & -1.250                         & 0.212                                 \\
$\log(RV_{t-1|t-7})$                  & 0.6640       & 0.032   & 20.821                         & 0.000                                 \\
\hline
Omnibus:                              & 67.731       & \multicolumn{2}{c|}{Durbin-Watson:}      & 1.615                                 \\
Prob(Omnibus):                        & 0.000        & \multicolumn{2}{c|}{Jarque-Bera (JB):}   & 139.964                               \\
Skew:                                 & 0.340        & \multicolumn{2}{c|}{Prob(JB):}           & 4.05e-31                              \\
Kurtosis:                             & 4.458        & \multicolumn{2}{c|}{Cond. No.}           & 380.                                  \\
\hline
\end{tabular}}
\caption{Short HARQ Model Result (under 10 min interval)}
\label{sHARQ_model_result}
\end{table}

\begin{figure}[H]
\centering
\fbox{\includegraphics[width=0.9\linewidth]{figures/sHARQ}}
\caption{Short HARQ Model Predictability (under 10min interval)}
\label{sHARQ_Model_Predictability}
\end{figure}
\end{comment}

\section{Metrics in Bitcoin}

Considered as a comparatively new investment, the construction of price/volatility for Bitcoin has not been substantial. Features from blockchain are selected for improving the forecast of existing models.\\

Table \ref{tb:metrics_model_result} shows that metrics generates impacts on results regarding to Realized Volatility. However, Condition Number of this table is $9.32\times10^3$, and such large Condition Number commonly pertains to a signal of multicollinearity, which indicates that metrics selected in this model have overlapping effects. \\

The result of regression for all metrics is presented in Appendix Table \ref{tb:metrics_model_result}, from which we can see that part of metrics have significant impact over Realized Volatility. Metrics such as adjustedTxVolume(USD), txCount, marketcap(USD) averageDifficulty, medianTxValue(USD) are singled out in further regression as and extension to other models as X factors. To choose from the lesser of two evils\footnote{multicollinearity and overfitting}, empirically one of a similar parameters is dropped in each pair such as txVolume(USD)|adjustedTxVolume(USD), marketCap(USD)|realizedCap(USD), and paymentCount|txCount. Only metrics that have significant result in Table \ref{tb:metrics_model_result} are constructed into X-factors.\\

\section{HARQ-X model}

HARQ-X model is a mixture of HARQ with extensions of including metrics that are selected from Table \ref{tb:metrics_model_result} . Here the X stands for the combination of adjustedTxVolume(USD), txCount, marketcap(USD), averageDifficulty, and medianTxValue(USD).\\

\begin{equation}
\begin{split}
\left\{
             \begin{array}{ll}
             RV_t=&\beta_0+\underbrace{(\beta_1+\beta_{1Q} RQ_{t-1}^{1/2})}_{\beta_{1,t}}RV_{t-1}+\underbrace{(\beta_2+\beta_{2Q} RQ_{t-1|t-7}^{1/2})}_{\beta_{2,t}}RV_{t-1|t-7}+X_{t-1}+u_t\\
             X_t=&adjustedTxVolume(USD)_t+txCount_t+marketcap(USD)_t\\&+medianTxValue(USD)_t+averageDifficulty_t\\
             RQ_t=&\frac{3}{M}\sum_{i=1}^Mr_{t,i}^4,\;RQ_{t-1|t-k}=\frac{1}{k}\sum_{j=1}^kRQ_{t-j}
             \end{array}
\right.
\end{split}
\label{eq:HARQx_model}
\end{equation}

Within this HARQ-X, all metrics inside X has been normalised via algorithm $\frac{metric-\mu}{\delta}$, where $\mu$ stands for the average of the specific metric, and $\delta$ is a standard deviation in Equation \ref{eq:HARQx_model}. Normalisation of metrics makes convenience for comparison.

\begin{comment}
\begin{table}[H]
\setlength{\tabcolsep}{5mm}{
\begin{tabular}{|c|l|c|c|c|c|}
\hline
\multicolumn{6}{|c|}{HARQ-X Model Result}                                                                                \\ \hline
\multicolumn{2}{|c|}{Dep. Variable:}                         & logRV        & \multicolumn{2}{c|}{R-squared:}          & 0.731     \\
\multicolumn{2}{|c|}{Model:}                                 & OLS          & \multicolumn{2}{c|}{Adj. R-squared:}     & 0.729     \\
\multicolumn{2}{|c|}{Method:}                                & LeastSquares & \multicolumn{2}{c|}{F-statistic:}        & 388.6     \\
\multicolumn{2}{|c|}{Log-Likelihood:}                        & -1515.3      & \multicolumn{2}{c|}{Prob (F-statistic):} & 0.00       \\
\multicolumn{2}{|c|}{No. Observations:}                      & 1298         & \multicolumn{2}{c|}{AIC:}                & 3051.     \\
\multicolumn{2}{|c|}{Df Residuals:}                          & 1288         & \multicolumn{2}{c|}{BIC:}                & 3102.     \\
\multicolumn{2}{|c|}{Df Model:}                              & 9            & \multicolumn{2}{c|}{Covariance Type:}    & nonrobust \\ \hline
\multicolumn{2}{|c|}{}                                       & coef         & std err              & t                 & P>|t|     \\
\multicolumn{2}{|c|}{const}                                  & -2.1669      & 0.565                & -3.839            & 0.000     \\
\multicolumn{2}{|c|}{$\log RV_{t-1}$}                          & 0.2714       & 0.176                & 1.542             & 0.123     \\
\multicolumn{2}{|c|}{$\log RQ_{t-1}^{1/2}\cdot RV_{t-1}$}      & -0.0226      & 0.082                & -0.276            & 0.782     \\
\multicolumn{2}{|c|}{$\log (RV_{t-1|t-7})$}                     & 0.8003       & 0.134                & 5.992             & 0.000     \\
\multicolumn{2}{|c|}{$\log (RQ_{t-1|t-7}^{1/2}\cdot RV_{t-1|t-7})$} & -0.0684      & 0.059                & -1.164            & 0.245     \\
\multicolumn{2}{|c|}{adjustedTxVolume(USD)}                  & 0.2988       & 0.386                & 0.774             & 0.439     \\
\multicolumn{2}{|c|}{txCount}                                & -0.3945      & 0.281                & -1.402            & 0.161     \\
\multicolumn{2}{|c|}{marketcap(USD)}                         & -0.3222      & 0.262                & -1.231            & 0.218    \\
\multicolumn{2}{|c|}{medianTxValue(USD)}                     & 1.1473       & 0.565                & 2.031             & 0.042     \\
\multicolumn{2}{|c|}{averageDifficulty}                      & 0.2655      & 0.208                & 1.275            & 0.202     \\ \hline
\multicolumn{2}{|c|}{Omnibus:}                               & 72.995       & \multicolumn{2}{c|}{Durbin-Watson:}      & 1.622     \\
\multicolumn{2}{|c|}{Prob(Omnibus):}                         & 0.000        & \multicolumn{2}{c|}{Jarque-Bera (JB):}   & 147.482    \\
\multicolumn{2}{|c|}{Skew:}                                  & 0.374            & \multicolumn{2}{c|}{Prob(JB):}           & 9.43e-33  \\
\multicolumn{2}{|c|}{Kurtosis:}                              & 4.473            & \multicolumn{2}{c|}{Cond. No.}           & 997.   \\ \hline
\end{tabular}}
\caption{HARQ-X Model Result (under 10 min interval)} \label{harq_x_model_result}
\end{table}
\end{comment}

\section{HAR-X model}

For comparison of the effects between Metrics and Quarticities, HAR-X model is composed of Metrics and HAR model.

\begin{equation}
\begin{split}
\left\{
             \begin{array}{ll}
             RV_t=&\beta_0 + \beta_1 RV_{t-1} +\beta_2 RV_{t-1|t-7} + \beta_3 RV_{t-1|t-{30}} + X_{t-1} + u_t\\
             X_t=&adjustedTxVolume(USD)_t+txCount_t+marketcap(USD)_t\\&+medianTxValue(USD)_t+averageDifficulty_t\\
             \end{array}
\right.
\end{split}
\end{equation}

\begin{comment}
\begin{table}[H]
\setlength{\tabcolsep}{5mm}{
\begin{tabular}{|c|c|c|c|c|}
\hline
\multicolumn{5}{|c|}{HAR-X Model Result}                                                     \\ \hline
Dep. Variable:         & logRV        & \multicolumn{2}{c|}{R-squared:}          & 0.731     \\
Model:                 & OLS          & \multicolumn{2}{c|}{Adj. R-squared:}     & 0.729     \\
Method:                & LeastSquares & \multicolumn{2}{c|}{F-statistic:}        & 437.0     \\
Log-Likelihood:        & -1516.0      & \multicolumn{2}{c|}{Prob (F-statistic):} & 0.00      \\
No. Observations:      & 1298         & \multicolumn{2}{c|}{AIC:}                & 3050.     \\
Df Residuals:          & 1298         & \multicolumn{2}{c|}{BIC:}                & 3096.     \\
Df Model:              & 8            & \multicolumn{2}{c|}{Covariance Type:}    & nonrobust \\ \hline
                       & coef         & std err              & t                 & P>|t|     \\
const                  & -1.8182      & 0.451                & -4.031            & 0.000     \\
$\log RV_{t-1}$        & 0.2312       & 0.030                & 7.818             & 0.000     \\
$\log (RV_{t-1|t-7})$  & 0.6751       & 0.039                & 17.437            & 0.000     \\
$\log (RV_{t-1|t-30})$ & -0.0304      & 0.034                & -0.885            & 0.376     \\
adjustedTxVolume(USD)  & 0.3222       & 0.385                & 0.836             & 0.403     \\
txCount                & -0.5050      & 0.279                & -1.810            & 0.071     \\
marketcap(USD)         & -0.2751      & 0.262                & -1.048            & 0.295     \\
medianTxValue(USD)     & 1.0979       & 0.565                & 1.945             & 0.052     \\
averageDifficulty      & 0.2657       & 0.209                & 1.274             & 0.203     \\ \hline
Omnibus:               & 61.021       & \multicolumn{2}{c|}{Durbin-Watson:}      & 1.622     \\
Prob(Omnibus):         & 0.000        & \multicolumn{2}{c|}{Jarque-Bera (JB):}   & 130.365   \\
Skew:                  & 0.292        & \multicolumn{2}{c|}{Prob(JB):}           & 4.92e-29  \\
Kurtosis:              & 4.438        & \multicolumn{2}{c|}{Cond. No.}           & 413.      \\ \hline
\end{tabular}}
\caption{HAR-X Model Result (under 10 min interval)} \label{har_x_model_result}
\end{table}
\end{comment}


\lab{Speech Recognition using CDHMMs}{Speech Recognition using CDHMMs}
\objective{Understand how speech recognition via CDHMMs works, and implement a simplified speech recognition system.}

\subsection{Continuous Density Hidden Markov Models}
Some of the most powerful applications of HMMs (speech and voice recognition) result from allowing the observation space to be continuous instead of discrete.
These are called Continuous Density Hidden Markov Models (CDHMMs), and they have two standard formulations: Gaussian HMMs and Gaussian Mixture Model HMMs (GMMHMMs).
In fact, the former is a special case of the latter, so we will just discuss GMMHMMs in this lab.

In order to understand GMMHMMs, we need to be familiar with a particular continuous, multivariate distribution called a \emph{mixture of Gaussians}.
A mixture of Gaussians is a distribution composed of several Gaussian (or Normal) distributions with corresponding weights.
Such a distribution is parameterized by the number of mixture components $M$, the dimension $N$ of the normal distributions involved, a collection of component weights
$\{c_1, \ldots, c_M\}$ that are nonnegative and sum to 1, and a collection of mean and covariance parameters $\{(\mu_1,\Sigma_1), \ldots, (\mu_M,\Sigma_M)\}$ for each Gaussian
component. To sample from a mixture of Gaussians, one first chooses the mixture component $i$ according to the probability weights $\{c_1,\ldots,c_M\}$, and then one
samples from the normal distribution $\mathcal{N}(\mu_i, \Sigma_i)$. The probability density function for a mixture of Gaussians is given by
\[
f(x) = \sum_{i=1}^M c_iN(x;\,\mu_i,\Sigma_i),
\]
where $N(\cdot;\,\mu_i,\Sigma_i)$ denotes the probability density function for the normal distribution $\mathcal{N}(\mu_i, \Sigma_i)$.
See Figure \ref{fig:mixture} for the plot of such a density curve.
Note that a mixture of Gaussians with just one mixture component reduces to a simple normal distribution, and so a GMMHMM with just one mixture component
is simply a Gaussian HMM.

\begin{figure}
\centering
\includegraphics[width=\textwidth]{mixture}
\caption{The probability density function of a mixture of Gaussians with four components.}
\label{fig:mixture}
\end{figure}

In a GMMHMM, we seek to model a hidden state sequence $\{\mathbf{x}_1,\ldots,\mathbf{x}_T\}$ and a corresponding observation sequence $\{O_1,\ldots,O_T\}$, just as with discrete HMMs.
The major difference, of course, is that each observation $O_t$ is a real-valued vector of length $K$ distributed according to a mixture of Gaussians with $M$ components.
The parameters for such a model include the initial state distribution $\pi$ and the state transition matrix $A$ (just as with discrete HMMs).
Additionally, for each state $i=1,\ldots,N$, we have component weights $\{c_{i,1},\ldots,c_{i,M}\}$, component means $\{\mu_{i,1},\ldots,\mu_{i,M}\}$, and component covariance matrices
$\{\Sigma_{i,1},\ldots,\Sigma_{i,M}\}$.

Let's define a full GMMHMM with $N=2$ states, $K = 3$, and $M=3$ components.
\begin{lstlisting}
>>> import numpy as np
>>> A = np.array([[.65, .35], [.15, .85]])
>>> pi = np.array([.8, .2])
>>> weights = np.array([[.7, .2, .1], [.1, .5, .4]])
>>> means1 = np.array([[0., 17., -4.], [5., -12., -8.], [-16., 22., 2.]])
>>> means2 = np.array([[-5., 3., 23.], [-12., -2., 14.], [15., -32., 0.]])
>>> means = np.array([means1, means2])
>>> covars1 = np.array([5*np.eye(3), 7*np.eye(3), np.eye(3)])
>>> covars2 = np.array([10*np.eye(3), 3*np.eye(3), 4*np.eye(3)])
>>> covars = np.array([covars1, covars2])
>>> gmmhmm = [A, weights, means, covars, pi]
\end{lstlisting}

We can draw a random sample from the GMMHMM corresponding to the second state as follows:
\begin{lstlisting}
>>> sample_component = np.argmax(np.random.multinomial(1, weights[1,:]))
>>> sample = np.random.multivariate_normal(means[1, sample_component, :], covars[1, sample_component, :, :])
\end{lstlisting}

Figure \ref{fig:samples} shows an observation sequence generated from a GMMHMM with one mixture component and two states.

\begin{figure}
\centering
\includegraphics[width=\textwidth]{samples}
\caption{An observation sequence generated from a GMMHMM with one mixture component and two states.
The observations (points in the plane) are shown as solid dots, the color indicating from which
state they were generated. The connecting dotted lines indicate the sequential order of the observations.}
\label{fig:samples}
\end{figure}

\begin{problem}
Write a function which accepts a GMMHMM in the format above as well as an integer $n\_sim$, and which simulates the GMMHMM process, generating $n\_sim$ different observations.
Do so by implementing the following function declaration.
\begin{lstlisting}
def sample_gmmhmm(gmmhmm, n_sim):
    """
    Simulate sampling from a GMMHMM.

    Returns
    -------
    states : ndarray of shape (n_sim,)
        The sequence of states
    obs : ndarray of shape (n_sim, K)
        The generated observations (column vectors of length K)
    """
    pass
\end{lstlisting}
\end{problem}

The classic problems for which we normally use discrete observation HMMs can also be solved by using CDHMMs, though with continuous observations it is much more difficult to keep things numerically stable.
We will not have you implement any of the three problems for CDHMMs yourself; instead, you will use a stable module we will provide for you.
Note, however, that the techniques for solving these problems are still based on the forward-backward algorithm; the implementation may be trickier, but the mathematical
ideas are virtually the same as those for discrete HMMs.

\subsection*{Speech Recognition and Hidden Markov Models}
Hidden Markov Models are the basis of modern speech recognition systems. However, a fair amount of signal processing must precede the HMM stage, and there are other
components of speech recognition, such as language models, that we will not address in this lab.

The basic signal processing and HMM stages of the speech recognition system that we develop in this lab can be summarized as follows:
The audio to be processed is divided into small frames of approximately $30$ ms. These are short enough that we can treat the signal as being constant over these inervals. We can then take this framed signal and, through a series of transformations, represent it by mel-frequency cepstral coefficients (MFCCs), keeping only the first $K$ (say $K = 10$). Viewing these MFCCs as continuous observations in $\mathbb{R}^{K}$, we can train a GMMHMM on sequences of MFCCs for a given word, spoken multiple times. Doing this for several words, we have a collection of GMMHMMs, one for each word. Given a new speech signal, after framing and decomposing it into its MFCC array, we can score the signal against each GMMHMM, returning the word whose GMMHMM scored the highest.

Industrial-grade speech recognition systems do not train a GMMHMM for each word in a vocabulary (that would be ludicrous for a large vocabulary), but rather on \emph{phonemes}, or distinct sounds. The English language has $44$ phonemes, yielding $44$ different GMMHMMs. As you could imagine, this greatly facilitates the problem of speech recognition.  Each and every word can be represented by some combination of these $44$ distinct sounds.  By correctly classifying a signal by its phonemes, we can determine what word was spoken. Doing so is beyond the scope of this lab, so we will simply train GMMHMMs on five words/phrases: biology, mathematics, political science, psychology, and statistics.

\begin{problem}
Obtain $30$ (or more) recordings for each of the words/phrases \emph{mathematics}, \emph{biology}, \emph{political science}, \emph{psychology}, and \emph{statistics}.
These audio samples should be 2 seconds in duration, recorded at a rate of 44100 samples per second, with samples stored as 16-bit signed integers in WAV format.
Load the recordings into Python using {\tt scipy.io.wavfile.read}.

If the audio files have two channels, average these channels to obtain an array of length $88200$ for each sample.
Extract the MFCCs from each sample using code from the file {\tt MFCC.py}:
\begin{lstlisting}
>>> import MFCC
>>> # assume sample is an array of length 88200
>>> mfccs = MFCC.extract(sample)
\end{lstlisting}
Store the MFCCs for each word in a separate list. You should have five lists, each containing 50 MFCC arrays, corresponding to each of the five words
under consideration.
\end{problem}

For a specific word, given enough distinct samples of that word (decomposed into MFCCs), we can train a GMMHMM.
Recall, however, that the training procedure does not always produce a very effective model, as it can get stuck in a poor local minimum.
To combat this, we will train 10 GMMHMMs for each word (using a different random initialization of the parameters each time)
and keep the model with the highest log-likelihood.

For training, we will use the file we have provided called {\tt gmmhmm.py}, as this is a stable implementation of GMMHMM algorithms.
To facilitate random restarts, we need a function to provide initializations for the initial state distribution and the transition matrix.

Let \li{samples} be a list of arrays, where each array is the output of the MFCC extraction for a speech sample.
Using a function \li{initialize()} that returns a random initial state distribution and (row-stochastic) transition matrix, we can train a GMMHMM with $5$ states
and $3$ mixture components and view its log-likelihood as follows:
\begin{lstlisting}
>>> import gmmhmm
>>> startprob, transmat = initialize(5)
>>> model = gmmhmm.GMMHMM(n_components=5, n_mix=3, transmat=transmat, startprob=startprob, cvtype='diag')
>>> # these values for covars_prior and var should work well for this problem
>>> model.covars_prior = 0.01
>>> model.fit(samples, init_params='mc', var=0.1)
>>> print model.logprob
\end{lstlisting}

\begin{problem}
Partition each list of MFCCs into a training set of 20 samples, and a test set of the remaining 10 samples.

Using the training sets, train a GMMHMM on each of the words from the previous problem with at least $10$ random restarts, keeping the best model for each word (the one with the highest log-likelihood).
This process may take several minutes.  Since you will not want to run this more than once, you will want to save the best model for each word to disk using the {\tt pickle} module so that you can use it later.
\end{problem}

Given a trained model, we would like to compute the log-likelihood of a new sample.
Letting \li{obs} be an array of MFCCs for a speech sample we do this as follows:
\begin{lstlisting}
>>> score = model.score(obs)
\end{lstlisting}
We classify a new speech sample by scoring it against each of the 5 trained GMMHMMs, and returning the word corresponding to the GMMHMM with the highest score.
\begin{problem}
Classify the 10 test samples for each word.
How does your system perform? Which words are the hardest to correctly classify?
Make a dictionary containing the accuracy of the classification of your five testing sets.  Specifically, the words/phrases will be the keys, and the values will be the percent accuracy.
\end{problem}

\begin{comment}

Hidden Markov Models are the basis of modern speech recognition systems. We assume that a short speech signal can be viewed as a stationary signal, and so we can divide a speech signal into small frames (approximately $30$ ms or so). We can take this framed signal and through a series of transformations represent it by mel-frequency cepstral coefficients (MFCCs), keeping only the first $K$ (say $K = 10$). Viewing these MFCCs as continuous observations in $\mathbb{R}^{K}$, we can train a GMMHMM on sequences of MFCCs for a given word, spoken multiple times. Doing this for several words, we have a collection of GMMHMMs, one for each word. Given a new speech signal, after framing and decomposing it into its MFCC array, we can score the signal against each GMMHMM, returning the word whose GMMHMM scored the highest.

In practice, a GMMHMM is not trained for each word in a vocabulary (that would be ludicrous for a large vocabulary), but rather on \emph{phonemes}, or distinct sounds. The English language has $44$ phonemes, yielding $44$ different GMMHMMs. By correctly classifying a signal by its phonemes, we can determine what word was spoken. Doing so is beyond the scope of this program, so we will simply train GMMHMMs on five words/phrases: biology, mathematics, political science, psychology, and statistics.

In a Linux environment, plug in your USB microphone headset and in the command line, enter
\texttt{arecord -l}
Note under which card the USB audio device is listed, as well as the device number. This will likely be card 1, device 0.

To record audio from the command line and save it as \texttt{test.wav} in your current working directory, enter
\texttt{arecord -f S16\_LE --rate=44100 -D hw:1,0 -d 2 test.wav}
This will record the audio from your USB microphone for $2$ seconds, sampling at a rate of $44100$ samples per second, saving the samples as signed $16$-bit numbers at the file name \texttt{test.wav}. We can read the wav file into python with the \li{scipy.io.wavfile} module:
\begin{lstlisting}
>>> import numpy as np
>>> import scipy.io.wavfile as wavfile
>>> sample = wavfile.read("test.wav")[1]
\end{lstlisting}

The object \li{sample} will be a vector of integers of length $88200 = 44100*2$. This in and of itself is not very useful. We must transform the data in a series of steps before it will be useful. We must first break the sample into \emph{frames}, each of some small length ($30$ ms). We will overlap these frames to smooth out these cutoff areas. In short, each frame will overlap with the previous by $20$ ms. Doing so with a $2$ second sample yields $198$ frames, each frame containing $1323$ values from the original sample, overlapping $882$ values with each sample immediately preceding and following it. We let $f_{n}$ denote the $n^{\text{th}}$ entry of the frame $f$.

Because we have overlapped the samples, the most significant part of a frame is the middle $441$ entries. We decrease the effects of the edges of the frame by applying a window function, making the edges nearly zero, while keeping the middle large. We use the Hamming window, defined as
\begin{equation*}
w(n) = 0.54 - 0.46 \cos \left( \frac{2\pi (n + 0.5)}{N} \right)
\end{equation*}
where $N = 1323$ is the length of the frame. We window each frame, computing $\tilde{f}_{n} = f_{n}w(n)$.

After windowing each frame, we then apply a filtering process known as \emph{preemphasis} used to improve the signal-to-noise ratio as follows:
\begin{equation*}
\widehat{f}_{n} = \tilde{f}_{n} - 0.95 \tilde{f}_{n-1}
\end{equation*}

We must next perform the discrete fourier transform of each frame, though to get a more refined view of the frequency spectrum, we must pad our vector with zeros to make it of length $2048$. We also will only keep approximately half of the returned vector, then take the square of the magnitude of each complex component.

We will ultimately be taking the logarithm of this transformed frame, so we must make sure all entries are slightly positive. We must then transform this into the \emph{mel scale}, which is a scale of pitches which listeners tend to judge to be of equal distance from one another. We make this transformation by binning the frames into $40$ bins, according to the overlapping triangular binning scheme in Figure \ref{fig:binning}.

\begin{figure}
\centering
\includegraphics[width=\textwidth]{melfilterbank.jpeg}
\caption{Binning scheme to transform linearly spaced frequency to mel scale.}
\label{fig:binning}
\end{figure}

If we want $40$ bins where the length of our padded frames is $2048$, then we can compute a \emph{mel filter bank} $M$ which is a matrix used to bin our transformed frame into the mel scale. We can now bin our frame by multiplying this matrix against the transformed frame. We also take the logarithm of the binned values, after which we compute the discrete cosine transform (DCT) of the returned values via a DCT transformation matrix, and then finally truncate to only keep ten values, called the mel frequency cepstral coefficients (MFCC). Note that we do not typically keep the first MFCC.

Normally we do this for all desired frames, storing each vector of MFCCs as a row in an array, after which we normalize by subtracting the mean of the MFCC rows, and divide by the standard deviations. We have provided you with code that allows you to compute these MFCCs quickly:
\begin{lstlisting}
>>> import MFCC
>>> mfccs = MFCC.extract(sample)
\end{lstlisting}

\begin{problem}
Record the word \emph{mathematics} as a $2$ second WAV file $20$ times, and decompose each file into its MFCC array, storing these in a list. Do this also for the words/phrases \emph{biology}, \emph{political science}, \emph{psychology}, and \emph{statistics}. Be sure you complete each word/phrase in the $2$ second window.
\end{problem}

For a specific word, given enough samples of that word decomposed into its MFCCs, we can train a GMMHMM. For this, we will use the file \li{gmmhmm.py} provided, as this is a stable implementation of GMMHMM algorithms. To facilitate random restarts, we need a function to provide initializations for the initial state distribution and the transition matrix.

\begin{problem}
Write a function that initializes the initial state distribution and transition matrix for a GMMHMM with $n$ states. You may have done this in a previous lab, so feel free to copy and paste.
\end{problem}

Let \li{samples} be a list of arrays, where each row in each array is a set of MFCCs corresponding to a frame in a sample for a given word. Using a function \li{initialize()} that returns a random initial state distribution and transition matrix, we will show how to train a GMMHMM with $5$ states, each having an output distribution as a GMM with $3$ mixture components. We also look at the log-likelihood of the data, given the trained model.
\begin{lstlisting}
>>> import gmmhmm
>>> startprob, transmat = initialize(5)
>>> model = gmmhmm.GMMHMM(n_components=5, n_mix=3, transmat=transmat, startprob=startprob, cvtype='diag')
>>> model.covars_prior = 0.01
>>> model.fit(samples, init_params='mc', var=0.1)
>>> print model.logprob
\end{lstlisting}

\begin{problem}
Train a GMMHMM on each of the words you previously recorded with at least $10$ random restarts, keeping the best model for each word. You might want to do this in parallel to save time.
\end{problem}

Given a trained model, we would like to compute the log-likelihood of a new sample. Letting \li{obs} be an array, each row of which is a set of MFCCs for a frame in a sample, we do this as follows:
\begin{lstlisting}
>>> score = model.score(obs)
\end{lstlisting}

\begin{problem}
Write a function that records a sample, converts it into its MFCC array, and then scores it on each of the five trained models. Return the word corresponding to the highest scoring model. Test your speech recognition system on each of the five words multiple times. How does it perform?
\end{problem}
\end{comment}


\section{Cellular approximation, cellular homology, obstruction theory}
%pset 3 is up in part. Today I'm going to tell you how to think about CW-complexes and work with them, and I won't give a lot of details. Some good sources are lecture notes by Davis-Kirk, and a beautiful book by Glen Bredon, as well as Hatcher.
In previous sections, we saw that homotopy groups play well with (maps between) CW-complexes.
Here, we will study maps between CW-complexes themselves, and prove that they are, in some sense, ``cellular'' themselves.
\subsection{Cellular approximation}
\begin{definition}\label{cellularmaps}
    Let $X$ and $Y$ be CW-complexes, and let $A\subseteq X$ be a subcomplex.
    Suppose $f:X\to Y$ is a continuous map.
    We say that $f|_{A}$ is skeletal\footnote{Some would say cellular.} if $f(\Sigma_n)\subseteq Y_n$.
\end{definition}
Note that a skeletal map might not take cells in $A$ to cells in $Y$, but it takes $n$-skeleta to $n$-skeleta.
\begin{theorem}[Cellular approximation]\label{cellularapprox}
    In the setup of Definition \ref{cellularmaps}, the map $f$ is homotopic to some other
    continuous map $f^\prime:X\to Y$, relative to $A$, such that $f^\prime$ is skeletal on all of $X$.
\end{theorem}
To prove this, we need the following lemma.
\begin{lemma}[Key lemma]
    Any map $(D^n,S^{n-1})\to (Y,Y_{n-1})$ factors as:
    \begin{equation*}
	\xymatrix{
	(D^n,S^{n-1})\ar[r]\ar@{-->}[dr] & (Y,Y_{n-1})\\
	& (Y_n,Y_{n-1})\ar[u]
	}
    \end{equation*}
\end{lemma}
\begin{proof}[``Proof.'']
    Since $D^n$ is compact, we know that $f(D^n)$ must lie in some finite subcomplex $K$ of $Y$.
    The map $D^n\to K$ might hit some top-dimensional cell $e^m\subseteq K$, which does not have anything attached to it;
    hence, we can homotope this map to miss a point, so that it contracts onto a lower-dimensional cell.
    Iterating this process gives the desired result.
\end{proof}
Using this lemma, we can conclude the cellular approximation theorem.
\begin{proof}[``Proof'' of Theorem \ref{cellularapprox}]
    We will construct the homotopy $f\simeq f^\prime$ one cell at a time.
    Note that we can replace the space $A$ by the subspace to which we have extended the homotopy.

    Consider a single cell attachment $A\to A\cup D^m$; then, we have
    \begin{equation*}
	\xymatrix{
	A\ar[d]_{\text{skeletal}}\ar[r] & A\cup D^m\ar[dl]^{\text{ may not be skeletal}}\\
	Y &
	}
    \end{equation*}
    Using the ``compression lemma'' from above, the rightmost map factors (up to homotopy) as the composite
    $A\cup D^m\to Y_m \to Y$.
    Unfortunately, we have not extended this map to the whole of $X$, although we could do this
    if we knew that the inclusion of a subcomplex is a cofibration.
    But this is true: there is a cofibration $S^{n-1}\to D^n$, and so any pushout of these maps is a cofibration!
    This allows us to extend; we now win by iterating this procedure.
\end{proof}
As a corollary, we find:
\begin{exercise}
    The pair $(X,X_n)$ is $n$-connected.
\end{exercise}
\subsection{Cellular homology}
Let $(X,A)$ be a relative CW-complex with $A\subseteq X_{n-1}\subseteq X_n\subseteq\cdots\subseteq X$.
In the previous part \todo{provide a link!} that $H_\ast(X_n,X_{n-1})\simeq \widetilde{H}_\ast(X_n/X_{n-1})$.
More generally, if $B\to Y$ is a cofibration, there is an isomorphism (see \cite[p. 433]{bredon}):
$$H_\ast(Y,B)\simeq\widetilde{H}_\ast(Y/B).$$
Since $X_n/X_{n-1} = \bigvee_{\alpha\in \Sigma_n}S^n_\alpha$, we find that
$$H_\ast(X_n,X_{n-1})\simeq \Z[\Sigma_n] = C_n.$$
The composite $S^{n-1}\to X_{n-1}\to X_{n-1}/X_{n-2}$ is called a relative attaching map.

There is a boundary map $d:C_n\to C_{n-1}$, defined by
$$d:C_n = H_n(X_n,X_{n-1})\xrightarrow{\partial}H_{n-1}(X_{n-1})\to H_{n-1}(X_{n-1},X_{n-2}) = C_{n-1}.$$
\begin{exercise}
    Check that $d^2 = 0$.
\end{exercise}
Using the resulting chain complex, denoted $C_\ast(X,A)$, one can prove that there is an isomorphism
$$H_n(X,A)\simeq H_n(C_\ast(X,A)).$$
(In the previous part\todo{provide a link!}, we proved this for CW-pairs, but not for relative CW-complexes.)
The incredibly useful cellular approximation theorem therefore tells us that the effect of maps on homology can be computed.

Of course, the same story runs for cohomology: one gets a chain complex which, in dimension $n$, is given by
$$C^n(X,A;\pi) = \Hom(C_n(X,A),\pi) = \Map(\Sigma_n,\pi),$$
where $\pi$ is any abelian group.
\subsection{Obstruction theory}
Using the tools developed above, we can attempt to answer some concrete, and useful, questions.
\begin{question}
    Let $f:A \to Y$ be a map from a space $A$ to $Y$.
    Suppose $(X,A)$ is a relative CW-complex.
    When can we find an extension in the diagram below?
    \begin{equation*}
	\xymatrix{
	    X\ar@{-->}[dr] & \\
	    A\ar@{^(->}[u]\ar[r]_f & Y
	    }
    \end{equation*}
\end{question}
The lower level obstructions can be worked out easily:
\begin{equation*}
    \xymatrix{
	A\ar[d]\ar@{^(->}[r] & X_0\ar@{-->}[dl]\ar@{^(->}[r] & X_1\ar@{-->}[dll]\\
	\emptyset\neq Y & &
    }
\end{equation*}
Thus, for instance, if two points in $X_0$ are connected in $X_1$, we only have to check that they are also connected in $Y$.

For $n\geq 2$, we can form the diagram:
\begin{equation*}
    \xymatrix{
	\coprod_{\alpha\in\Sigma_n}S^{n-1}_\alpha\ar[r]^f\ar@{^(->}[d] & X_{n-1}\ar[r]^g\ar[d] & Y\\
	\coprod_{\Sigma_n}D^n_\alpha\ar[r] & X_n\ar@{-->}[ur] &
    }
\end{equation*}
The desired extension exists if the composite $S^{n-1}_\alpha\xar{f_\alpha} X_{n-1}\to Y$ is nullhomotopic.

Clearly, $g\circ f_\alpha\in [S^{n-1},Y]$.
To simplify the discussion, let us assume that $Y$ is simple;
then, Exercise \ref{simplequotient} says that $[S^{n-1},Y] = \pi_{n-1}(Y)$.
This procedure begets a map $\Sigma_n\xrightarrow{\theta}\pi_{n-1}(Y)$, which is a $n$-cochain, i.e., an element of
$C^n(X,A;\pi_{n-1}(Y))$.
It is clear that $\theta = 0$ if and only if the map $g$ extends to $X_n\to Y$.
\begin{prop}
    $\theta$ is a cocycle in $C^n(X,A;\pi_{n-1}(Y))$, called the ``obstruction cocycle''.
\end{prop}
\begin{proof}
    $\theta$ gives a map $H_n(X_n,X_{n-1})\to \pi_{n-1}(Y)$.
    We would like to show that the composite
    $$H_{n+1}(X_{n+1},X_n)\xrightarrow{\partial}H_n(X_n)\to H_n(X_n,X_{n-1})\xrightarrow{\theta}\pi_{n-1}(Y)$$
    is trivial.
    %OK, we know that $\pi_n(X_n,X_{n-1})\to H_n(X_n,X_{n-1})$ is surjective. This relative homotopy group holds the characteristic maps, doesn't it? So picking a representative back there is no problem; now I have the lexseq of a pair, which gives:
    We have the long exact sequence in homotopy of a pair (see Equation \eqref{lexseqhomotopy}):
    \begin{equation*}
	\xymatrix{
	    \pi_{n+1}(X_{n+1},X_n)\ar[d]\ar@{->>}[r] & H_{n+1}(X_{n+1},X_n)\ar[d]^\partial\\
	    \pi_n(X_n)\ar[r]\ar[d] & H_n(X_n)\ar[d]\\
	    \pi_n(X_n,X_{n-1})\ar@{->>}[r] \ar[d]_\partial & H_n(X_n,X_{n-1})\ar[d]^\theta\\
	    \pi_{n-1}(X_{n-1})\ar[r]_{g_\ast} & \pi_{n-1}(Y)
	    }
    \end{equation*}
    This diagram commutes, so $\theta$ is indeed a cocycle.
\end{proof}
Our discussion above allows us to conclude:
\begin{theorem}
    Let $(X,A)$ be a relative CW-complex and $Y$ a simple space.
    Let $g:X_{n-1}\to Y$ be a map from the $(n-1)$-skeleton of $X$.
    Then $g|_{X_{n-2}}$ extends to $X_n$ if and only if $[\theta(g)]\in H^n(X,A;\pi_{n-1}(Y))$ is zero.
\end{theorem}
\begin{corollary}
    If $H^n(X,A;\pi_{n-1}(Y)) = 0$ for all $n>2$, then any map $A\to Y$ extends to a map $X\to Y$ (up to homotopy\footnote{In fact, this condition is unnecessary, since the inclusion
    of a subcomplex is a cofibration.});
    in other words, there is a dotted lift in the following diagram:
    \begin{equation*}
	\xymatrix{
	A\ar[r]\ar[d] & Y\\
	X\ar@{-->}[ur] &
	}
    \end{equation*}
\end{corollary}
For instance, every map $A\to Y$ factors through the cone if $H^n(CA,A;\pi_{n-1}(Y)) \simeq \widetilde{H}^{n-1}(A;\pi_{n-1}(Y)) = 0$.
%Of course, one still needs to know the homotopy groups of $Y$ are, but often, you can get away even with partial information.
%\begin{remark}
%    Fix a prime $p$. Later we'll see (I hope) that $Y$ is simply connected (maybe even simple?), and suppose that $\widetilde{H}_\ast(Y;\Z_{(p)}) = 0$ (i.e., no $\Z$-summands and no $\Z/p^k$-summands). Then $\pi_\ast(Y)\otimes\Z_{(p)} = 0$ too. So if $H_\ast(A;\Z/\ell\Z) = 0$ for $\ell\neq p$, then $[A,Y]\simeq \ast$.
%\end{remark}


\documentclass[9pt,twocolumn]{extarticle}

\usepackage[hmargin=0.5in,tmargin=0.5in]{geometry}
\usepackage{amsmath,amssymb}
\usepackage{times}
\usepackage{graphicx}
\usepackage{subfigure}

\usepackage{cleveref}
\usepackage{color}
\newcommand{\TODO}[1]{\textcolor{red}{#1}}

\newcommand{\FPP}[2]{\frac{\partial #1}{\partial #2}}
\newcommand{\argmin}{\operatornamewithlimits{arg\ min}}
\author{Siwang Li}

\title{self-collision handling}

%% document begin here
\begin{document}
\maketitle

\setlength{\parskip}{0.5ex}

\section{Introduction}

\begin{eqnarray*}
   x_0 &=& \xi_1x_1+\xi_2x_2+\xi_3x_3+(1-\xi_1-\xi_2-\xi_3)x_4 \\
   G &=& (x_1-x_4, x_2-x_4, x_3-x_4)\\
   x &=& (x_0^T, x_1^T, x_2^T, x_3^T)^T\\
   x_0 &=& G(x)\xi+x_4 \\
   \xi &=& G^{-1}(x)(x_0-x_4) \\
   D &=& (d_1-d_4, d_2-d_4, d_3-d_4)\\
   \TODO{\hat{c}(x)} &=& \xi_1d_1+\xi_2d_2+\xi_3d_3+(1-\xi_1-\xi_2-\xi_3)d_4\\
   &=& DG^{-1}(x)(x_0-x_4)-d_4
\end{eqnarray*}
\begin{equation}
  \TODO{\tilde{x} = \arg \min_{\tilde{x}} \frac{1}{2}\|\tilde{x}-x^{*}\|_2^2 \text{ s.t. } \hat{c}(x) = 0}
\end{equation}
\begin{eqnarray*}
  \hat{c}_i(x_i) &=& \hat{c}(\tilde{x}) + \nabla_{x_i} \hat{c}(\tilde{x})^T(x_i-\tilde{x}_i)\\
  &=& n_i^Tx_i + p_i
\end{eqnarray*}
\begin{eqnarray*}
  \nabla_{x} \hat{c}(\tilde{x}) &=& (n_0^T, n_1^T, n_2^T, n_3^T, n_4^T)^T\\
  \TODO{n_i} &=& \nabla_{x_i} \hat{c}(\tilde{x})\\
  \TODO{p_i} &=& \hat{c}(\tilde{x}) - n_i^T\tilde{x}_i
\end{eqnarray*}

\begin{eqnarray*}
  x_{k+1}-x^* + J^T \lambda &=& 0\\
  \hat{c}(x_k)+J(x_{k+1}-x_k) &=& 0\\
  x_{k+1} + J^T \lambda &=& x^*\\
  J x_{k+1} &=& J x_k-\hat{c}(x_k)\\
  \left( \begin{array}{cc}
    I & J^T\\
    J & O
  \end{array} \right)
  \left( \begin{array}{c}
    x_{k+1}\\
    \lambda
  \end{array} \right) &=&
\left( \begin{array}{c}
  x^*\\
J x_k-\hat{c}(x_k)
\end{array} \right)
\end{eqnarray*}

\end{document}


\section{Historical Introduction}
\subsection{Black-body radiation}
The birth of quantum mechanics finds its way through its ability to explain phenomena that cannot be explained in classical physics.
The first example of which is the black-body radiation.
\begin{definition}
    A black body is a totally absorbing heated body at thermal equilibrium $T$.
\end{definition}
What people towards the end of 19th century intended to measure was the energy density of light emitted by a black body at this equilibrium temperature $\rho(\nu,T)$ where $\nu$ is the frequency of the light.
Experimentally, the graph of $\rho$ against $\nu$ fixing $T$ always looks like a curve that grows parabolically at the beginning near $0$ but decreases exponentially after some point.\\
Many physicists tried to model this curve.
Rayleight (1900) made the first attempt in the problem.
He assumed that the black body is a cobe with side length $L$.
The radiation is taken as a superposition of plane waves in the form $e^{i\underline{k}\cdot\underline{x}}$ where $\underline{k}$ is the called the wavenumber of the plane wave.
Obviously, we have $|\underline{k}|=2\pi/\lambda$, so $\nu=|\underline{k}|x/(2\pi)$.\\
However, we cannot take the wavenumbers arbitarily, as we require the wave to vanish at the boundary to make a thermal equilibrium.
With this condition, we obtain
$$\underline{k}=\frac{2\pi}{L}\underline{n},\underline{n}\in\mathbb Z^3\implies \nu=\frac{|\underline{n}|c}{L}$$
Let $N(\nu)$ be the number of modes (possible values of $\underline{n}$) between $\nu$ and $[\nu+\mathrm d\nu]$.
We then obtain
$$N(\nu)\,\mathrm d\nu=2\times 4\pi|\underline{n}|^2\,\mathrm d|\underline{n}|=8\pi\left(\frac{L}{c}\right)^3\nu^2\,\mathrm d\nu$$
We have, let $\langle E(\nu,T)\rangle$ be the average energy of the modes,
$$\rho(\nu,T)=N(\nu)\frac{\langle E(\nu,T)\rangle}{L^3}$$
In classical statistical dynamics, $\langle E\rangle$ is directly proportional to $T$ and does not depend on $\nu$.
So $\langle E\rangle=k_BT$ wherre $k_B$ is the Boltzmann constant.
Where does it come from?
It comes from the fact that the probability for a plane wave to have energy $E$ is
$$\mathbb P(E)\propto e^{-E/(k_BT)}\implies \langle E\rangle=\frac{\int_0^\infty Ee^{-E/(k_BT)}\,\mathrm dE}{\int_0^\infty e^{-E/(k_BT)}\,\mathrm dE}=k_BT$$
However, this shall yield us the energy density
$$\rho(\nu,T)=\frac{8\pi k_B}{c^3}T\nu^2$$
which is obviously NOT compliant with experimental results except for small frequencies.
This is known as the ultraviolet catastrophe, getting its name due to the fact that this model stops working pass the ultraviolt frequencies.\\
Planck (1905) found a workaround of the problem.
He postulated that the energy is discrete instead of continuous, so $E=nh\nu$ for nonnegative integer $n$ and constant $h$ (known as the planck constant).
This is known as a quantisation of the energy of radiation.\\
In this case, we obtain
$$\langle E\rangle=\frac{\sum_{n=0}^\infty nh\nu e^{-nh\nu/(k_BT)}}{\sum_{n=0}^\infty e^{-nh\nu/(k_BT)}}=\frac{h\nu}{e^{h\nu/(k_BT)}-1}$$
which is fascinatingly different from the continuous case.
\footnote{Well, not really.}
In this case, we obtain
$$\rho(\nu,T)=\frac{8\pi h}{c^3}\frac{\nu^3}{e^{h\nu/(k_BT)}-1}$$
which is surprisingly fitting the experimental data.
This makes people start thinking about taking certain physical quantities as discrete objects.
\subsection{Photoelectric Effect}
A long time ago, people have observed the emission of electrons when light is shine on metal surfaces.
In a classical point of view, if the intensity $I$ of the incident light is high enough, the radiation would give enough energy for electrons to emit.
However, this is not what happened in an experiment.
If the frequency is too low, no matter how intense the radiation is, there can be no emission of electron.
Even more puzzling, the rate of the emission of the electrons is not growing with the frequency of the light, but instead with intensity of the light.\\
Einstein (1905) took up the idea of Planck ad came up with the idea of viewing light as light quanta (photons), which are particles.
And each of those particles carries energy $E=h\nu=\hbar\omega$ where $\hbar=h/(2\pi)$ is the reduced Planck constant and $\omega=2\pi\nu$ is the angular frequency.
Photoelectric effect happens, as Einstein interpreted it, when an electron absorbs a photon with large enough energy to make it escape its original structure.\\
So the energy of the emitted electron is bounded by $E\le \hbar\omega-\phi$ where $\phi$ is the ``work function'' that denotes the minimal amount of energy of a single photon spent to trigger photoelectric effect.
With this theory, one can easily explain the observations of photoelectric effect.\\
This theory of viewing light as particles got developed further and helped explaining more phenomena.
In 1923, an experiment known as Compton scattering was conducted by Compton.
In this experiment, Compton used X-Rays to scatter off free electrons.
Classically, if the light is simply a wave, then there will be a re-radiation of light by electrons with a certain angular distribution, which one can obtain as $I(\nu)\propto(1+\cos^2\theta)$.
So one could expect to find a single peak at $\nu_0$ in the graph of $I$ against $\nu$.\\
However, what is observed experimentally, there was a second peak $\nu'$ such that $|\nu_0-\nu'|$ is dependent on $\theta$.\\
In quantum mechanics, if we take lights as particles, as we did in Dynamics \& Relativity, it can be explained.
Assume the deflected electron and photon (whose energy is $E'=\hbar\omega'$) make angles of $\alpha$ $\theta$ to the original path, then by using energy-momentum conservation one can obtain
$$\frac{1}{\omega'}=\frac{1}{\omega}+\frac{\hbar}{mc}(1-\cos\theta)$$
which is consistent with the experimental result.
This also shows that $\hbar$ actually measures the strength of quantum effect.
\subsection{Particles as Waves}
In previous sections, we found that light, which always manifest itself as a wave, can be interpreted as particles too.
A natural thought from that is then:
Can we interpret particles as waves?\\
In 1923, de Broglie postulated that for any particle of mass $m$, it can be associated with a wave with angular frequency $\omega=E/\hbar$ and $\underline{k}=\underline{p}/\hbar$.
People then attempt to conduct experiments to verify this theory.
For example, one can do a diffraction experiment with particle beams.
But experiments of this are usually complicated to administrate as the wavelength of particles can be very small.
But some physicists managed to get it done on small particles like electrons, which does yield a typical diffraction pattern and the calculated wavelength is complicant with de Broglie's theory.
So we can interpret particles as waves.\\
But the electron can't possibly split in space, yet even we pass the electron one-by-one, the same phenomenon happens.
This can be explained using a probabilistic interpretation of the whole thing, which we will cover later.
\subsection{Atomic Models}
In 1897, J. J. Thompson made the first atomic model by viewing atoms as a cloud of positive charges that enclose a number of electrons.
To verify (or disprove) this model, Rutherfold suggested Geiger and Marsden to conduct a scattering experiment.
They took a beam of alpha particles and shine it on a piece of gold foil.
If the model of J. J. Thompson was correct, then the alpha particles should not be deflected by any large angle, but should merely pass through the foil with very small deflection.
However, in the experiment, many deflections with very large angle were observed, and some were even scattered back.\\
After this experiment, Rutherfold came up with another model, which postulates that electrons are orbitting a common nucleus with positive charge.
This explains the phenomenon that was observed in the scattering experiment.
But there are several problems with it:
First, why is there not any radiation from the electrons resulting from the centripetal acceleration?
This radiation should make the electron's energy decrease and eventually collapse, but in reality no such thing happen.
Secondly, why is there discrete instead of continuous frequencies in the atomic spectra?
That is, when an excited atom de-excite, the angular frequencies of light emitted (the line spectra) are discrete and satisfy
$$\omega_{mn}=2\pi cR_0\left( \frac{1}{n^2}-\frac{1}{m^2} \right)$$
for $m>n$.
Here $R_0$ is a constant known as the Rydberg constant, which has order of magnitude $10^7{\rm m^{-1}}$.\\
These questions prompted another reformulation of the atomic model by Bohr (1913).
He hypothesise that the electron orbits are quantised so that the orbital angular momentum $L$ satisfies $L=\hbar n$ for $n\in\mathbb N$.
It sounds out of nowhere, but it works.
There is an explanation of it by taking electrons as waves.
Then, the wavenumber of an electron is just $\underline{k}=\underline{p}/\hbar$ so $\lambda=2\pi\hbar/|\underline{p}|$.
If we imagine the electron in the orbit of radius $r$, then we can take it as a stationary wave on this circumference, so
$$2\pi r=n\lambda=n\frac{2\pi\hbar}{|\underline{p}|}\implies L=|\underline{r}||\underline{p}|=\hbar n$$
which is just Bohr's hypothesis.\\
What is the consequences of this formulation of atomic model?
We have, by Newton's second law,
$$\frac{e^2}{4\pi\epsilon_0}\frac{1}{r^2}=m_e\frac{v^2}{r}$$
in the $\hat{r}$ component.
Hence by $L=\hbar n$, the velocity and radius are quantized as
$$v_n=\frac{n\hbar}{m_e r_n},r_n=n^2\left( \frac{4\pi\epsilon_0}{m_ee^2}\hbar^2 \right)$$
$a_0=r_1$ is called the Bohr radius.
The energy of the electron is also quantised.
$$E_n=\frac{1}{2}m_ev_n^2-\frac{e^2}{4\pi\epsilon_0}\frac{1}{r_n}=-\frac{e^2}{8\pi\epsilon_0a_0}\frac{1}{n^2}=\frac{E_1}{n^2}$$
We have $E_1=-13.6{\rm eV}$ upon calculation.
For an electron with $n>1$, we say it is excited as such states can be created by exciting (giving energy to, usually via photon) the electron in ground state, while an electron with $n=0$ is said to be in ground state.\\
In this model, an de-excitation from state $m$ to $n$ would then give light with angular frequency
$$\omega_{mn}=(E_m-E_n)\frac{1}{\hbar}=2\pi c\frac{m_ec}{2\hbar}\left(\frac{e^2}{4\pi\epsilon_0\hbar c}\right)^2\left( \frac{1}{n^2}-\frac{1}{m^2} \right)$$
which gives a prediction of $R_0$ very close to the experiment.
In particular, calculation reveals that $\omega_{n+1,n}\sim v_n/r_n$ as $n\to\infty$, which can be viewed as a kind of classical limit.
%$$\omega_{n+1,n}=\frac{m_e^3e^4}{64\pi^4\epsilon_0^2\hbar^3}\left( \frac{1}{n^2}-\frac{1}{(n+1)^2} \right)\sim\frac{m_e^3e^4}{32\pi^4\epsilon_0^2\hbar^3}\frac{1}{n^3}=\frac{v_n}{r_n}$$
%as $n\to\infty$.
%This is exactly the classical case.
\footnote{However, $\omega_{n+2,n}$ does not tend to this limit when $n\to\infty$. It does not tend to any well-known classical limit.}
\subsection{Conclusion}
Quantum mechanics is a new framework of explaining physical phenomena.
It may come in counter-intuitive as we always think in the classical way, but it has an undeniable predictivity.


%-------------------------------------------------------------------------------
% Signatures
%-------------------------------------------------------------------------------

\subsection{Signatures}

Signatures map constants to types and kinds.

\newcommand{\Sig}{\ \mathtt{sig}}
\bigskip
\framebox{$\Sigma\Sig$}
\bigskip

$$
\begin{array}{ccc}
\infer{\cdot\Sig}{}&
\infer{\Sigma,c:A\Sig}{\Sigma\Sig & \CheckTy[\cdot]{A}{\Type}}&
\infer{\Sigma,a:K\Sig}{\Sigma\Sig & \CheckTy[\cdot]{K}{\Kind}}
\end{array}
$$


\subsection{Vickrey auctions}

Vickrey are sealed-bid auctions. Participants cannot view other bids.

The price paid is the second-highest bid.


\section{Cycles}

%%%%%%%%%%
\begin{frame}{Cycles}
  \begin{exampleblock}{4-Cycle in undirected graph \pno{3.7.1}}
    \begin{itemize}
      \item undirected graph $G = (V, E)$
      \item simple cycle of length 4
      \item $O(n^{3})$
    \end{itemize}
  \end{exampleblock}

  \begin{block}{Solution.}

  \end{block}
\end{frame}
%%%%%%%%%%
\begin{frame}{Cycles}
  \begin{exampleblock}{Shortest cycle in digraph \pno{3.7.4}}

  \end{exampleblock}

  \begin{block}{Solution.}
    Floyd-Warshall: $\min_{i}D[i][i]$

    Initialization: $D^{(0)}[i][i] = \infty$
  \end{block}

  \begin{alertblock}{Remark.}
    Does not apply to undirected graph.
  \end{alertblock}
\end{frame}
%%%%%%%%%%
\begin{frame}{Cycles}
  \begin{exampleblock}{Shortest cycle in undirected graph \pno{3.7.14}}
    \begin{itemize}
      \item $G = (V, E), w(e) = 1$
      \item DFS: back edge $\iff$ cycle
      \item $u \to v$: $\text{level}[u] - \text{level}[v] + 1$
    \end{itemize}
  \end{exampleblock}

  \begin{block}{Solution.}
    A counterexample here.
  \end{block}
\end{frame}
%%%%%%%%%%
\begin{frame}{Cycles}
  \begin{exampleblock}{Shortest cycle containing a specific edge \pno{3.7.5}}
    \begin{itemize}
      \item undirected graph $G = (V, E, w), w(e) > 0, e \in E$
      \item shortest cycle containing $e$
    \end{itemize}
  \end{exampleblock}

  \begin{block}{Solution.}
    $P_{u \leadsto v} + (u,v)$
  \end{block}
\end{frame}
%%%%%%%%%%
\begin{frame}{Cycles}
  \begin{exampleblock}{Hamiltonian path in tournament graph \pno{3.7.18}}
    \begin{itemize}
      \item digraph $G = (V, E)$
      \item $\forall u,v: (u \to v \lor v \to u) \land \lnot (u \to v \land v \to u)$
      \item hamiltonial path
    \end{itemize}
  \end{exampleblock}

  \begin{block}{Solution.}
    \begin{itemize}
      \item existence: induction on $n$
      \item algorithm $1 + 2 + \cdots + (n-1) = O(n^{2})$
    \end{itemize}
  \end{block}
\end{frame}
%%%%%%%%%%


\documentclass[twoside]{MATH77}
\usepackage{multicol}
\usepackage[fleqn,reqno,centertags]{amsmath}
\begin{document}
\hyphenation{YPKNOT}
\begmath 11.4 Least-Squares Cubic Spline Fit

\silentfootnote{$^\copyright$1997 Calif. Inst. of Technology, \thisyear \ Math \`a la Carte, Inc.}

\subsection{Purpose}

A cubic spline function with NB $-$ 1 segments is a function consisting of
NB $-$ 1 pieces, each of which is a cubic polynomial. At the abscissae,
called knots, at which adjacent segments meet, the function has $C^2$
continuity, $i.e$. continuity in value, first derivative, and second
derivative.

Subroutine SC2FIT or DC2FIT will determine the (NB $-$ 1)-segment cubic spline
function, with user specified knots, that best fits a set of discrete data
in the sense of weighted least-squares, and return the values of the fitted
spline curve and its first derivative at the knots. The user can then
evaluate the curve, or its first or second derivative, at any argument using
the Hermite interpolation subroutine, SHINT or DHINT, of Chapter~12.3.

This software can be used for interpolation by setting the number of knots,
NB, to be two less than the number of data points, NXY. Setting NB $<$
NXY $-$ 2 gives least-squares approximation.

For spline fitting with more generality, see Chapter~11.5.

\subsection{Usage}

\subsubsection{Program Prototype, Single Precision}
\begin{description}
\item[INTEGER]  \ {\bf NXY, NB, LDW, IERR1}

\item[REAL]  \ {\bf X}($\geq $NXY){\bf , Y}($\geq $NXY){\bf , SD}($\geq $NXY)%
{\bf , \nolinebreak B}($\geq $NB){\bf , W}(LDW,~5){\bf , YKNOT}($\geq $NB)%
{\bf , YPKNOT}($\geq $NB){\bf , SIGFAC}
\end{description}
Assign values to X(), Y(), SD(), NXY, B(), NB, and LDW.\vspace{-15pt}
\begin{center}
\fbox{\begin{tabular}{@{\bf }c}
CALL SC2FIT(X, Y, SD, NXY, B, NB, W,\\
LDW, YKNOT, YPKNOT, SIGFAC, IERR1)\\
\end{tabular}}
\end{center}
Computed quantities are returned in YKNOT(), YPKNOT(), SIGFAC, and IERR1.

Following the use of SC2FIT the user may use SHINT of Chapter~12.3 to compute
values of the fitted curve.

\subsubsection{Argument Definitions}
\begin{description}
\item[X(), Y()]  \ [in] Data pairs (X(I), Y(I), I = 1, ..., NXY). The
contents of X() must satisfy X($1)\leq $ X(2) $\leq $ ... $\leq $ X(NXY).

\item[SD()]  \ [in] If SD($1)>0.$, each SD(I) must be positive and must be
the user's a priori estimate of the standard deviation of the uncertainty $%
(e.g.$, observational error) in the corresponding data value Y(I).

If SD($1)<0.$, $|$SD(1)$|$ will be used as the a priori standard deviation
of each data value Y(I). In this case the array SD() may be dimensioned as
SD(1).

\item[NXY]  \ [in] Number of data points. Require NXY $\geq 4.$

\item[B()]  \ [in] Set by the user to specify the knot abscissae and
endpoints for the spline curve. Must satisfy B(1) $<$ B(2)
$<$ ... $<$ B(NB).  Also require B(1) $\leq $
X(1), B(NB) $\geq $ X(NXY).

\item[NB]  \ [in] Number of knots, including endpoints. The number of
segments in the spline curve will be NB $-$ 1. The number of degrees of freedom
for the fit will be NB + 2. Require $2\leq \text{NB}\leq \text{NXY}-2.$

\item[W(\ ,\ )]  \ [scratch] Working space, dimensioned W(LDW,~5).

\item[LDW]  \ [in] Leading dimension for the work array W(,). LDW must be at
least NB + 4, but the execution time is less for larger values of LDW, up to
NB$+3+k$, where $k$ is the largest number of data points lying between any
adjacent pair of knots.

\item[YKNOT(), YPKNOT()]  \ [out] Arrays, each of length at least NB, in
which the subroutine will store a definition of the fitted spline curve as a
sequence of values of the curve and its first derivative at the knot
abscissae B(i). Letting $f$ denote the fitted curve, the elements of these
arrays will be set to

YKNOT$(i)=f(\text{B}(i))$, $i=1$, ..., NB

YPKNOT$(i)=f^{\prime }(\text{B}(i))$, $i=1$, ..., NB

\item[SIGFAC]  \ [out] Set by the subroutine as a measure of the residual
error of the fit. See Section D.

\item[IERR1]  \ [out] Error status indicator. Set on the basis of tests done
in SC2FIT as well as error indicators IERR2 set by SBACC and IERR3 set by
SBSOL. Zero indicates no errors detected. See Section E for the meaning of
nonzero values.
\end{description}
\subsubsection{Modifications for Double Precision}

For double precision usage change the REAL statement to DOUBLE PRECISION and
change the subroutine name SC2FIT to DC2FIT.

\subsection{Examples and Remarks}

{\bf Example:} Given a set of 12~data pairs $(x_i,y_i)$ compute the
uniformly weighted least-squares cubic spline fit to these data using six
uniformly spaced breakpoints, including endpoints. After determining the
spline function $f(x)$, compute and tabulate the quantities $x_i$, $y_i$, $%
f(x_i)$, and $r_i = y_i - f(x_i).$

This computation is illustrated by the program DRDC2FIT and the output
ODDC2FIT. The fitted curve is determined using DC2FIT, and is evaluated
using DHINT of Chapter~12.3.

{\bf Interpolation:} If all of the data abscissae are distinct, and one wants
interpolation, rather than the smoothing effect of least-squares
approximation, one can choose interpolation by setting NB = NXY $-$ 2. The
NB knot abscissae can be assigned in various ways, but one reasonable way is
to set B(1) = X(1), B(i) = X$(i+1)$, for $i = 2$, ..., NB $-$ 1, and B(NB) =
X(NXY).

\subsection{Functional Description}

Let knot abscissae $b_1 < b_2 < ..$.  $< b_{NB}$ be given.  A cubic spline
function defined over the interval $[b_1$, $b_{NB}]$ is a cubic polynomial
in each subinterval $[b_i$, $b_{i+1}]$, $i = 1$, ..., NB $-$ 1, with
continuity of the value, first derivative, and second derivative at each
internal knot, $b_2$, ..., $b_{NB-1}$.  The set of all cubic spline
functions defined relative to this knot set is a linear space of dimension
$d =$ NB $+$ 2.  If the knot spacing does not depart severely from
uniformity a well conditioned set of basis functions for this space is
provided by a particular set of cubic spline functions called B-splines,
$p_i(x)$, $i = 1$, ..., $d$, each of which is nonzero over at most four
adjacent subintervals, \cite{deBoor:1972:OCB}.

The problem data are $\{(x_i$, $y_i$, $s_i)$, $i=1$, ..., NXY$\}$, where $s_i$
is the a priori standard deviation of the error in the value $y_i$. The
weighted least-squares curve fitting problem then becomes one of determining
coefficients $c_j$ to minimize%
\begin{equation*}
\rho ^2({\bf c})=\sum_{i=1}^{\text{NXY}}\left[ \frac{
y_i-\sum_{j=1}^dc_jp_j(x_i)}{s_i}\right] ^2
\end{equation*}
The matrix formulation of this least-squares problem involves a matrix
having a banded form in which at most four elements are nonzero in each row.
This least-squares problem is solved using the subroutines of Chapter~4.5.

This problem will have a unique set of solution coefficients, $c_j$, if NB
$\leq \text{NXY}-2$ and the positioning of the knots is such that there
exists an indexing of some set of NB + 2 of the distinct data abscissae,
$x_i$ (not necessarily the indexing used in the subprogram) such that
$b_{i-3} < x_i < b_{i+1}$ for $i = 1$, ..., NB$+2$.  Here $b_{-2}$,
$b_{-1}$, and $b_0$ denote fictitious knots to the left of $b_1$, and
$b_{NB+1}$, $b_{NB+2}$, and $b_{NB+3}$ denote fictitious knots to the
right of $b_{NB}$, see \cite{Rice:COC:1967}.  If the solution is not
unique, no solution will be given and an error code will be returned as
described in Section E.

After determining coefficients, $c_j$, SC2FIT uses subroutine STRC2C to
evaluate the value and first derivative of the fitted curve at the knots.
These quantities are returned to the user in the arrays YKNOT() and YPKNOT()
as the defining parameters of the fitted curve.

Subroutine SC2BAS is called by both SC2FIT and STRC2C to evaluate B-spline
basis functions.

\bibliography{math77}
\bibliographystyle{math77}

\subsection{Error Procedures and Restrictions}

SC2FIT sets IERR1 and issues error messages based on internal tests as well
as propagating error information set in IERR2 by SBACC and in IERR3 by
SBSOL. See Chapter~4.5 for the meaning of IERR2 and IERR3. In all cases in
which IERR1 is set nonzero, no solution will be computed.

\begin{tabular}{ll}
{\bf IERR1} & {\bf \quad \quad Meaning}\\

\phantom{100}0 & No errors detected.\\

\phantom{1}100 & NB $< 2$ or NXY $<$ NB + 2.\\

\phantom{1}200 & B(I$) \geq $ B(I+1) for some I.\\

\phantom{1}300 & LDW $<$ NB + 4.\\

\phantom{1}400 & X(I $- 1) > $ X(I) for some I.\\

\phantom{1}500 & B($1) > $ X(1) or B(NB) $<$ X(NXY).\\

\phantom{1}600 & Need larger dimension LDW.\\

\phantom{1}700+IERR2 & SBACC set IERR2 $\neq 0$.\\

\phantom{1}800+IERR2 & SBACC set IERR2 $\neq 0$.\\

\phantom{1}900+IERR2 & SBACC set IERR2 $\neq 0$.\\

 1000+IERR3 & SBSOL set IERR3 $\neq 0$. Indicates\\
 & singularity.\\

 1100 & SD(1) = 0.0.\\

 1200 & SD(1) $>$ 0.0, and SD($i) \leq 0.0$ for\\
 & some $i \in $ {[2, NXY]}.
\end{tabular}
\subsection{Supporting Information}

\begin{tabular}{@{\bf}l@{\hspace{5pt}}l}
\bf Entry & \hspace{.35in} {\bf Required Files}\vspace{2pt} \\
DC2FIT & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
 AMACH, DBACC, DBSOL, DC2BAS, DC2FIT, DERM1, DERV1, DHTCC,
 DNRM2, DTRC2C, ERFIN, ERMSG, IERM1, IERV1\rule[-5pt]{0pt}{8pt}}\\
SC2FIT & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
 AMACH, ERFIN, ERMSG, IERM1, IERV1, SBACC, SBSOL, SC2BAS, SC2FIT,
 SERM1, SERV1, SHTCC, SNRM2, STRC2C}\\\end{tabular}

Original code designed by C. L. Lawson and R. J. Hanson, JPL, 1968.
Programmed and modified by Lawson, Hanson, T. Lang, and D. Campbell, JPL,
1968--1974. Adapted to Fortran~77 by Lawson and S. Y. Chiu, July~1987.

\begcodenp
\lstset{language=[77]Fortran,showstringspaces=false}
\lstset{xleftmargin=.8in}

\centerline{\bf \large DRDC2FIT}\vspace{10pt}
\lstinputlisting{\codeloc{dc2fit}}
\newpage

\vspace{30pt}\centerline{\bf \large ODDC2FIT}\vspace{10pt}
\lstset{language={}}
\lstinputlisting{\outputloc{dc2fit}}
\end{document}


\section{Details on Machine Learning Methods}
\subsection{Decision Tree}
\subsubsection{Introduction}
Decision trees are a statistical learning method used both in the classification and regression setting. They are non-parametric models, meaning that they don't approximate a function $\hat{f}$ to describe the relationship between the features and target, but are classifying data by inferring a hierarchical decision structure (the decision tree) \cite{introStats}.

Decision trees are based on the idea of segmenting the feature space into a number of simple regions (rectangles in 2-dimensional space, cubes in 3-dimensional space, etc.). The challenge of tree-based learning methods is to identify the regions that best separate the classes. Mathematically, a loss function, ie. cross-entropy, gini-impurity or 0-1 loss, should be minimised for each data point in the training split, st.

$$
\text{min}\left\{\sum_{j=1}^J\sum_{i\in{R_J}} \text{loss(}y_i, \hat{y}_{R_j}\text{)}\right\}
$$
\vspace{5pt}

However, it is computationally infeasible to consider every possible partition of the feature space into $J$ regions and evaluate the loss. For this reason, a top-down, greedy approach, known as recursive binary splitting is used to construct the decision tree in a computationally light fashion.

Recursive binary splitting is a recursive algorithm that splits a given partition of the training set (node) into the two (hence the name \textit{binary} splitting) partitions that best separate the classes in the original partition. It thus finds the cut point $s$ for a feature $X_i$ that forms two sets $\{X| X_i<s\}$ and $\{X|X_i\ge s\}$ that minimise the weighted loss of both regions.
The decision tree is then built by recursively applying this algorithm, until some stopping criterion is reached, or all regions are pure (ie. all training samples in a decision region are of a single class). The final prediction is based on identifying which region a new data point belongs to and then predicting the majority class in that region.

\subsubsection{Decision Tree Model Selection}
%Having implemented the \class{DecisionTreeClassifier()}, the challenge arises of which set of hyper parameters in the training phase produces a classifier, that best generalises to unseen data. Particularly, we wish to find the classifier that is neither oversimplifying (high bias, low variance model) nor overfitting (low bias, high variance model).

% To find the configuration of hyper parameters that best generalise, we used \class{GridSearchCV} from \textit{sci-kit learn}. It allows to search a set of hyper parameter configurations for the best-generalising performance using 5-fold cross validation by choosing the model that achieves best average validation performance on some scoring-criterion.

The final decision tree classifier is a pipeline of a \code{StandardScaler} instance, that was fitted on the training split and a \code{DecisionTreeClassifier} instance. For the scope of this project a combination of the hyper parameters \code{criterion}, \code{splitter}, \code{max_features} and \code{max_depth} of the tree were checked within a grid search.
\newline

\begin{comment}
\begin{lstlisting}[language=Python, caption=DecisionTreeClassifier Pipeline and GridSearch Setup, label=dt_pipeline]
    # model pipeline
    pipe = Pipeline(steps=[('scaler', scaler),
                           ('decision_tree', DecisionTreeClassifier(random_state=1))])

    # define hyper parameters to grid search
    params = {
            'decision_tree__criterion': ['gini', 'entropy'],
            'decision_tree__max_depth': list(range(1, 10)),
            'decision_tree__max_features': list(range(1, 10)),
            'decision_tree__splitter': ['best', 'random']
            }
\end{lstlisting}
\end{comment}

The grid search ran over all possible combinations of hyper parameters defined. For each setting it fitted five models on four (randomised) folds and validated the performance of the model on the remaining split. It then computed the averaged accuracy score on the validation splits that occurred within the training and returned the model configuration that maximised the performance of the model on the validation splits. The best generalising decision tree was found to use \textit{entropy-impurity} as a loss-function, is trained to maximum depth of five and considers eight features for each split.

\subsubsection{Decision Tree Performance Evaluation}
The best generalising model achieved an averaged accuracy score in the cross-validation splits of 70\%, which is the estimated performance on unseen data. Because of that, it is surprising to see a test accuracy of only 62\%. The variation in the two metrics is likely occurs, because both the training and test split are rather small, making it prone to random variation. From the classification report it becomes clear that the decision tree struggles with separating classes 1, 2 and 3 from each other. In contrast, the model does rather well on the more separated classes 6 and 7 with an F1-Score of 80\% each.
\newline

\begin{table}[!ht]
\begin{subtable}[c]{0.4\textwidth}
\footnotesize
\centering
\begin{tabular}{ c | c }
 \toprule
 Evaluation Metric & Score  \\
 \midrule
 Training Accuracy & 87\% \\
 Validation Accuracy (during CV) & 70\% \\
 Test Accuracy & 62\% \\
 \bottomrule
\end{tabular}
\captionsetup{justification=centering,margin=1cm}
\subcaption{Decision Tree Accuracy Scores on Training, Validation and Test Split}
\end{subtable}
\begin{subtable}[c]{0.6\textwidth}
\footnotesize
\centering
\begin{tabular}{c | c c c r}
Class & Precision & Recall & F-Score & Support\\
\midrule
1   &    0.68  &    0.62  &    0.65  &  21 \\
2   &    0.64  &    0.61  &    0.62  &  23 \\
3   &    0.25  &    0.20  &    0.22  &   5 \\
5   &    0.33  &    1.00  &    0.50  &   4 \\
6   &    1.00  &    0.67  &    0.80  &   3 \\
7   &    1.00  &    0.67  &    0.80  &   9 \\
\midrule
    accuracy   &           &         &    0.62   &   65 \\
   macro avg   &    0.65   &   0.63  &    0.60   &   65 \\
weighted avg   &    0.67   &   0.62  &    0.63   &   65 \\
\end{tabular}
\captionsetup{justification=centering,margin=1cm}
\subcaption{Decision Tree Classification Report on Test Split}
\end{subtable}
\caption{Decision Tree Performance Evaluation}
\label{dt_evaluation}
\end{table}

\subsubsection{Visualization of Decision Tree}
Figure \ref{dt_visualisation} shows a visualisation of the constructed decision tree. At each level of the tree the node carries information about the decision rule applied at the specific node, the loss-value (ie. gini impurity), the number of samples considered and the majority class in the node.

One interesting observation is that the feature \textit{barium} is used at depth 2 to separate a pure leaf with class 7 (headlamp) off. This was expected from inspecting the feature distribution in each class in Section 3, since the feature distribution of class 7 was distinct for the feature \textit{batrium}.

\begin{figure}[ht]
\centering
\includegraphics[scale=0.09]{figures/graphviz_sklearn_dt.png}
\caption{Visualised Decision Tree}
\label{dt_visualisation}
\end{figure}

\subsection{Neural Network}
\subsubsection{Introduction}
A feed-forward neural network is a machine learning algorithm that constructs
a nonlinear (decision) function $\hat{f}(X)$ that tries to approximate the true relationship between some feature matrix $X$ and a target vector $y$ \cite{introStats}. In each layer of the neural network, an activation function is applied to ensure the non-linearity of the decision function $\hat{f}$.

To train a neural network a loss function (e.g. cross entropy) is minimised using the technique of back-propagation. The algorithm computes the gradient of the loss with respect to all the weights and biases. This gradient
is then used to tweak the parameters to reduce the error using a gradient descent step. The process is repeated until the neural network converges to the desired solution \cite{handsonML}.

When dealing with a classification problem the response $y$ is a vector produced by the last layer (the output layer) with a soft-max function as its activation, ensuring that the sum of elements is one ($\sum_{i=1}^{i=k}y_i = 1$) and each respective element is in the range of 0 to 1 ($0\ge y_i \ge 1 \ \ y_i \forall y$).
Therefore the individual scalars of the vector $y$ can be interpreted as probabilities that a data point that was fed forward through the network belongs to the different classes. The prediction is then made to the class with the largest probability.

\subsubsection{Neural Network Model Selection}
Since the custom \class{NeuralNetworkClassifier} is not able to reproduce the exact results of other deep learning libraries, such as \textit{Pytorch} or \textit{Keras}, due randomness in the initialisation of weights, different optimisation algorithms and a different technique of back-propagation, this section will be split into model selection for the custom implementation and using the Python deep learning framework \textit{Keras} that is built on top of \textit{Tensorflow}.
\newline

% ---------------------------- CUSTOM --------------
\textbf{Custom Implementation. }
Since the training process for the custom \class{NeuralNetworkClassifier()} implementation is relatively slow due to the recursive back-propagation, the model selection process was limited. Too complex network architectures as well as too high number of training iterations were not feasible, as well as an exhaustive grid search to tune hyper parameters. The model selection was therefore primarily based on heuristics and trial-and-error. Since the custom implementation does not allow to evaluate the model on a validation split at each epoch to stop the training process, the correct number of epochs to prevent overfitting was inferred from experience.

The final configuration of the model is a neural network with a single, ReLu-activated, hidden layer of 20 nodes that is trained for a total of 100 epochs using 10 batches, a learning rate of 0.01 and computes loss using cross-entropy.
\newline

\begin{comment}
\begin{lstlisting}[language=Python, caption=Custom Neural Network Setup]
    # setting up neural network architecture
    nn = NeuralNetworkClassifier(
        layers = [DenseLayer(n_in=9, n_out=20, activation='relu', name='fc1'),
                  DenseLayer(n_in=20, n_out=6, activation='softmax', name='output')],
        loss='cross_entropy',
        name='CustomModel'
        )

    # fitting model
    nn.fit(X_train, y_train,
           epochs=100,
           lr=0.01,
           num_batches=10,
           verbose=1)
\end{lstlisting}
\end{comment}

The training history of loss and training accuracy is depicted in Figure \ref{custom_nn_history}. It can be seen that the loss is steadily decreasing and maximising the training accuracy.
\newline

\begin{figure}[ht]
\centering
\includegraphics[scale=0.8]{figures/custom_nn_training.pdf}
\caption{Training History of Custom Neural Network Implementation}
\label{custom_nn_history}
\end{figure}

\subsubsection{Custom Neural Network Performance Evaluation}
The fitted model achieves a training accuracy of 72\% and a test accuracy of 74\% (\ref{custom_nn_evaluation}). The model is best at predicting class 1, 2 and 7 correct, which is logical since these are the majority classes. Except for the minority class 3, that is very similar to class 1 and 2 (as seen from the EDA), the model does an overall good job of classifying.

\begin{table}[!ht]
\begin{subtable}[c]{0.4\textwidth}
\footnotesize
\centering
\begin{tabular}{ c | c }
 \toprule
 Evaluation Metric & Score  \\
 \midrule
 Training Accuracy & 72\% \\
 Test Accuracy & 74\% \\
 \bottomrule
\end{tabular}
\captionsetup{justification=centering,margin=1cm}
\subcaption{Custom Neural Network Accuracy Scores on Training and Test Split}
\end{subtable}
\begin{subtable}[c]{0.6\textwidth}
\footnotesize
\centering
\begin{tabular}{c | c c c r}
Class & Precision & Recall & F-Score & Support\\
\midrule
1   &    0.70  &    0.90  &    0.79   &     21 \\
2   &    0.73  &    0.70  &    0.71   &     23 \\
3   &    0.00  &    0.00  &    0.00   &      5 \\
5   &    0.60  &    0.75  &    0.67   &      4 \\
6   &    0.67  &    0.67  &    0.67   &      3 \\
7   &    1.00  &    0.89  &    0.94   &      9 \\
\midrule
    accuracy  &          &         &   0.74  &  65 \\
   macro avg  &   0.62   &   0.65  &   0.63  &  65 \\
weighted avg  &   0.69   &   0.74  &   0.71  &  65 \\
\end{tabular}
\captionsetup{justification=centering,margin=1cm}
\subcaption{Custom Neural Network Classification Report on Test Split}
\end{subtable}
\caption{Custom Neural Network Performance Evaluation}
\label{custom_nn_evaluation}
\end{table}

% ---------------------------- KERAS --------------
\textbf{Keras Implementation. }
Similarly, the neural network architecture for the Keras neural network was mostly derived through trial-and-error. A final configuration of a simple, neural network with a single hidden, ReLu-activated layer of 50 neurons, with an \class{Adam()} optimiser set to a learning rate of $0.005$ and optimising for cross-entropy loss on the entire batch (batch gradient descent) for each epoch was found to give good results.
Using the \class{EarlyStopping()} callback, the training process was monitored, such that a continuous increase of the validation loss over $2$ epochs would stop the fitting of the model. This prevented overfitting.
\newline

\begin{comment}
\begin{lstlisting}[language=Python, caption=Keras Neural Network Setup]
    # neural network architecture
    nn = Sequential()
    nn.add(Dense(50, input_dim=9, activation='relu', name='fc1'))
    nn.add(Dense(6, activation='softmax', name='output'))

    # define optimiser, loss and optimisation target
    nn.compile(optimizer=Adam(lr=0.005),
               loss='categorical_crossentropy',
               metrics=['accuracy'])

    # early stopping
    es = EarlyStopping(monitor='val_loss', patience=2, verbose=1)

    # fitting neural network
    history = nn.fit(X_train, y_train_hot,
                     batch_size=len(X_train), # batch gradient descent
                     epochs=epochs,
                     verbose=2,
                     validation_split=0.2,
                     callbacks=[es]) # train with validation split (here test data)
\end{lstlisting}
\end{comment}

With this configuration, the model callback was reached after $\sim 80$ epochs. The training history is depicted in Figure \ref{keras_nn_history} and reveals that the loss continuously decreased for the training split, and was stopped just as the validation loss was beginning to increase again.

\begin{figure}[ht]
\centering
\includegraphics[scale=0.8 ]{figures/keras_nn_training.pdf}
\caption{Training History of Keras Neural Network Implementation}
\label{keras_nn_history}
\end{figure}

\subsubsection{Keras Neural Network Performance Evaluation}
As can be seen from the history of accuracies, the final trained neural network achieves a training and validation accuracy of roughly 80\%.  The validation score for this model is a good estimate for the test accuracy, which is 75\% (Figure \ref{keras_nn_evaluation}). From the classification report (Figure \ref{keras_nn_evaluation}) it becomes clear, that the model does well on the better separated classes 6 and 7. It has more difficulty separating the closely related classes 1, 2 and 3 and, in fact, never predicts the minority class 3. Nevertheless, the overall classification result is good.
\newline

\begin{table}[!ht]
\begin{subtable}[c]{0.4\textwidth}
\footnotesize
\centering
\begin{tabular}{ c | c }
 \toprule
 Evaluation Metric & Score  \\
 \midrule
 Training Accuracy & 78\% \\
 Test Accuracy & 75\% \\
 \bottomrule
\end{tabular}
\captionsetup{justification=centering,margin=1cm}
\subcaption{Keras Neural Network Accuracy Scores on Training and Test Split}
\end{subtable}
\begin{subtable}[c]{0.6\textwidth}
\footnotesize
\centering
\begin{tabular}{c | c c c r}
Class & Precision & Recall & F-Score & Support\\
\midrule
1   &    0.72   &   0.86  &    0.78  &      21 \\
2   &    0.77   &   0.74  &    0.76  &      23 \\
3   &    0.00   &   0.00  &    0.00  &       5 \\
5   &    0.75   &   0.75  &    0.75  &       4 \\
6   &    0.75   &   1.00  &    0.86  &       3 \\
7   &    1.00   &   0.89  &    0.94  &       9 \\
\midrule
    accuracy   &           &         &    0.75   &     65 \\
   macro avg   &    0.67   &   0.71  &    0.68   &     65 \\
weighted avg   &    0.73   &   0.75  &    0.74   &     65 \\
\end{tabular}
\captionsetup{justification=centering,margin=1cm}
\subcaption{Keras Neural Network Classification Report on Test Split}
\end{subtable}
\caption{Keras Neural Network Performance Evaluation}
\label{keras_nn_evaluation}
\end{table}

\subsection{Random Forest}
\subsubsection{Introduction}
The statistical observation of the wisdom-of-the-crowd motivated the emergence of so-called ensemble methods in the cosmos of machine learning. Ensemble methods are groups of predictors, that collectively construct the ensemble's decision \cite{introStats}.

One of the most popular ensemble-methods is the so-called \class{RandomForestClassifier()}. It refers to a series of individual decision trees  independently trained on random sub-samples and randomly selected sets of features of the input data to generate non-correlated classifiers. The final-decision is then attributed to the class being predicted by the largest number of individual trees (hard voting-classifier) \cite{introStats}.

\subsubsection{Random Forest Model Selection}
The model-selection-pipeline for the \class{RandomForestClassifier()} was similar to that of the single decision tree. Again, a pipeline that involved scaling the features was used, and grid searched for a set of hyper parameters in order to find the configuration of hyper parameters that best generalise. Within training, the hyper parameters \code{n_estimators}, \code{criterion}, \code{bootstrap} (sampling from original data with or without replacement) and \code{max_depth} were searched.
\newline

\begin{comment}
\begin{lstlisting}[language=Python, caption=RandomForestClassifier Pipeline and GridSearch Setup]
    # define pipeline
    pipe = Pipeline(steps=[('scaler', scaler),
                           ('random_forest', RandomForestClassifier(random_state=1))])

    # define hyper parameters to grid search
    params = {'random_forest__n_estimators': [20, 50, 100],
              'random_forest__max_depth': list(range(5, 10)) + [None],
              'random_forest__bootstrap': [False, True],
              'random_forest__criterion': ['entropy', 'gini']
             }
\end{lstlisting}
\end{comment}

\subsubsection{Random Forest Performance Evaluation}
The grid search returned that the best-generalising random forest classifier uses 100 predictors, Gini-impurity as a loss-function, bootstrapped random sub-samples of the training data and trains each tree to a maximal depth of 8. This fitted model has a training accuracy of 100\%, while still having a high mean validation accuracy of 77\% encountered during the 5-fold cross validation. The accuracy on the final test set is 83\%.

\begin{table}[!ht]
\begin{subtable}[c]{0.4\textwidth}
\footnotesize
\centering
\begin{tabular}{ c | c }
 \toprule
 Evaluation Metric & Score  \\
 \midrule
 Training Accuracy & 100\% \\
 Validation Accuracy & 77\% \\
 Test Accuracy & 83\% \\
 \bottomrule
\end{tabular}
\captionsetup{justification=centering,margin=1cm}
\subcaption{Random Forest Accuracy Scores on Training, Validation and Test Split}
\end{subtable}
\begin{subtable}[c]{0.6\textwidth}
\footnotesize
\centering
\begin{tabular}{c | c c c r}
Class & Precision & Recall & F-Score & Support\\
\midrule
1   &  0.95  &  0.95  &  0.95  &  21\\
2   &  0.83  &  0.83  &  0.83  &  23\\
3   &  0.00  &  0.00  &  0.00  &   5\\
5   &  0.50  &  1.00  &  0.67  &   4\\
6   &  0.75  &  1.00  &  0.86  &   3\\
7   &  1.00  &  0.89  &  0.94  &   9\\
\midrule
    accuracy  &        &        &  0.83  &  65\\
   macro avg  &  0.67  &  0.78  &  0.71  &  65\\
weighted avg  &  0.80  &  0.83  &  0.81  &  65\\
\end{tabular}
\captionsetup{justification=centering,margin=1cm}
\subcaption{Random Forest Classification Report on Test Split}
\end{subtable}
\caption{Random Forest Performance Evaluation}
\label{random_forest_evaluation}
\end{table}


\subsection{Autoencoders}

Autoencoders are a type of neural network.

For autoencoders the goal for the output layer is the reconstructed input layer, rather than a classification.

By including sparsity in the neural network we can reduce the dimensions. This splits the network into an encoder and a decoder.

Middle vector is called latent variables.