%!TEX root = ../thesis.tex
%*******************************************************************************
%*********************************** Thrid Chapter *****************************
%*******************************************************************************

\chapter{Background Methodologies \label{cha:background}}  %Title of the Thrid Chapter

\nomenclature[a-GP]{$\mathcal{GP}\left(m(\cdot), k(\cdot, \cdot)\right)$}{Gaussian Process with mean function $m(\cdot)$ and covariance function $k(\cdot, \cdot)$}
\nomenclature[s-k]{$k$}{Principal component index.}

\ifpdf
    \graphicspath{{Chapter3/Figs/Raster/}{Chapter3/Figs/PDF/}{Chapter3/Figs/}}
\else
    \graphicspath{{Chapter3/Figs/Vector/}{Chapter3/Figs/}}
\fi

In the following chapter we consider the various statistical methodologies  upon which we build our CPACE model.
This chapter is structured as follows.
First we focus on common FDA techniques applicable to EO data.
We follow this by discussing smoothing methodologies which are use in the FDA techniques.
Finally we discuss Gaussian processes that are used in the CPACE framework to model correlation.

\section{Functional Principal Components Analysis \label{sec:fpca}}
A commonly used technique in multivariate statistics is Principal Components Analysis (PCA), \citep{wold_principal_1987}.
The technique finds dominant directions of variation and helps to achieve dimensionality reduction.
This offers a parsimonious way to view data which is driven by the data themselves.
The equivalent technique when the data are functional in nature is known as Functional Principal Components Analysis (FPCA), \citep{ramsay_functional_2010}.
The basic concepts were studied in the mid twentieth century.
The work of \citeauthor{karhunen_zur_1946} and independently \citeauthor{loeve_fonctions_1946} paved the basic foundations of the technique in the FDA literature, \citep{karhunen_zur_1946, loeve_fonctions_1946}.
The FPCA technique essentially stems from representing the random function $\mathcal{X}(t)$ as an infinite linear combination of orthogonal functions.
Such a representation is now known as the Karhunen-Lo\`{e}ve theorem after its discoverers.

\subsection{Formulation}
The formulation of FPCA begins by assuming that $\mathcal{X}(t)$, $t \in \mathcal{T}$ is a square integrable stochastic process over some domain $\mathcal{T}$.
Let the mean and the covariance of the stochastic process $\mathcal{X}$ be denoted by $\mu(t)$ and $G\left(s, t\right)$ respectively, where:
\begin{align}
	\mu(t) &= \E \left(\mathcal{X}(t)\right) \label{eqn:mean_fn}\\
	G\left(s, t\right) &= \text{Cov}\left(\mathcal{X}(s),  \mathcal{X}(t)\right) \label{eqn:cov_fn}
\end{align}

Associated with the covariance surface $G\left(s, t\right)$ we have the linear operator $T_G$ defined by:
\begin{align}
	T_G &: L^2\left(\mathcal{T}\right) \to L^2\left(\mathcal{T}\right) \\
	T_G&:  f \mapsto T_G f = \int_{\mathcal{T}} G\left(s, \cdot \right) f(s) ds \label{eqn:t_op}
\end{align}

As $T_G$ is a linear operator we can consider its eigenvalues and eigenfunctions which we will denote by $\lambda_k$ and $\phi_k$ respectively (following convention set out in \citep{yao_functional_2005}) for $k=1,2,\cdots$.
These are defined as the solutions to the Fredholm integral equations of the second kind, \citep{yao_functional_2005}:

\begin{equation}\label{eqn:fredholm}
	\langle G(\cdot, t), \phi_k \rangle = \lambda_k \phi_k(t)
\end{equation}
where $\langle f, g \rangle = \int_{\mathcal{T}} f(s) g(s) ds$ is the inner product in the space $L^2(\mathcal{T})$.
Then by the Karhunen-Lo\`{e}ve theorem one can express the centred process through the eigenvalues and eigenfunctions of the linear operator associated to the covariance surface, \citep{karhunen_zur_1946, loeve_fonctions_1946}.
That is:
\begin{equation}\label{eqn:fpca}
	\mathcal{X}(t) - \mu(t) = \sum_{k=1}^{\infty}\xi_k \phi_k(t)
\end{equation}
where $\xi_k$ is the $k^\text{th}$ principal component associated to the eigenfunction $\phi_k$.
The Karhunen-Lo\`{e}ve theorem assures us this $L^2$ convergence is uniform in $t$.
The principal components are given by the following:
\begin{equation}\label{eqn:principal_comp}
	\xi_k= \langle \mathcal{X} - \mu, \phi_k \rangle
\end{equation}

Further to this decomposition the Karhunen-Lo\`{e}ve theorem means we have that the principal components are independent from each other, centred, and have variance equal to their associated eigenvalue, \citep{karhunen_zur_1946, loeve_fonctions_1946}.
That is:
\begin{align}
	\E\left(\xi_k\right) &= 0 \\
	\text{Var}\left(\xi_k\right) &= \lambda_k \\
	\E\left(\xi_k \xi_l\right) &= 0,~\text{for}~ k \ne l \label{eqn:principal_comp_uncorr}
\end{align}

\subsection{Interpretation}
As with the multivariate principal components analysis the interpretation of the eigenvectors is often useful in exploratory analysis of data.
The functional principal components analysis is of a similar form to the multivariate case and as such the same interpretation of the eigenfunctions is often employed.
The first eigenfunction $\phi_1(t)$ encapsulates the dominant mode of variation in $\mathcal{X}(t)$ by construction since:
\begin{equation}\label{eqn:first_comp}
	\phi_1 = \argmax_{\lVert \phi \rVert=1} \text{Var}\left( \langle  \mathcal{X} - \mu, \phi \rangle  \right)
\end{equation}
Similarly, the $k^\text{th}$ eigenfunction is the dominant mode of variation which is orthogonal to the preceding $k-1$ components.
Therefore exploring the first few eigenfunctions often gives a parsimonious way to view the variation in the data.
Alike multivariate PCA, it is often that the structure of the eigenfunctions replicates some observed physical process.
As such the FPCA decomposition is often used widely as a tool for data exploration.

In addition to this, we can use the fact that subsequent eigenfunctions capture less and less variation of the data as a form of dimensionality reduction, like PCA, \citep{wold_principal_1987}.
In this sense we can consider truncating the full representation given in Equation~\eqref{eqn:fpca} to the $K$ leading eigenfunctions which gives an approximation to the full process which we will denote by $\mathcal{X}^K(t)$ where:
\begin{equation}\label{eqn:fpca_trun}
	\mathcal{X}^K(t)  =   \mu(t) + \sum_{k=1}^{K}\xi_k \phi_k(t)
\end{equation}
The approximation of $\mathcal{X}$ by $\mathcal{X}^K$ converges as:
\begin{equation}\label{eqn:fpca_trun_conv}
	\E \left( \langle  \mathcal{X} - \mathcal{X}^K, \mathcal{X} - \mathcal{X}^K \rangle \right)  =   \sum_{k > K}^{\infty} \lambda_k \to 0~\text{as}~K \to \infty
\end{equation}

As such using the leading principal components for reconstruction has the effect of capturing the main modes of variation of the data and ignoring smaller modes of variation.
Choosing the number of principal components is then up to the practitioner as in multivariate PCA, \citep{wold_principal_1987}.
 \citeauthor{ramsay_functional_2010} discuss in length the comparison of PCA to FPCA including commentary on the optimal choice of the number of principal components, \cite[Chapter~8]{ramsay_functional_2010}.
 The practical implementation of FPCA then involves estimating various components.
 In particular estimation of;  the mean function $\mu(t)$, the covariance surface $G\left(s,t \right)$, the $K$ eigenfunctions and eigenvalues $\phi_k(t)$, $\lambda_k$ respectively, and the principal components $\xi_k$ for each realisation of the process $\mathcal{X}$ we observe.

\section{Principal Analysis Through Conditional Expectation \label{sec:pace}}
We will assume for now that we have a sufficient method for estimating the mean  and covariance surfaces which we will denote by $\hat{\mu}(t)$ and $\hat{G}\left(s, t \right)$ respectively.
We discuss in more detail the estimation of these components in Section~\ref{sec:splines}.
Prior to the introduction of the Principal Analysis through Conditional Expectation (PACE) methodology in \citep{yao_functional_2005} FPCA decomposition was restricted due to the need for approximating the integrals in Equations~\eqref{eqn:principal_comp}.
As such it was often a requirement that the functional data were observed on a dense regular grid which meant that the principal components could be reliably estimated though some numerical integration scheme, \citep[Chapter~8]{ramsay_functional_2010}.
This very much restricted the application of the FPCA technique, however \citeauthor{yao_functional_2005} introduced the PACE method for overcoming such an obstacle using conditional expectations for sparsely observed functional data, \citep{yao_functional_2005}.
At the same time the technique of \citep{yao_functional_2005} accommodates for observation error.

Traditionally Equation~\eqref{eqn:principal_comp}, used for estimating the principal component scores for the $i^\text{th}$ realisation, is approximated through sums.
Substituting $y_{ij}$ for $\mathcal{X}(t_{ij})$, $\hat{\mu}(t_{ij})$ for $\mu(t_{ij})$, and $\hat{\phi}_k(t_{ij})$ for  $\phi_k(t_{ij})$ we obtain the estimate $\xi_i^{S} = \sum_1^{J_i}\left(y_{ij} - \hat{\mu}(t_{ij})\right)\hat{\phi}_k(t_{ij})\left(t_{ij} - t_{i(j-1)}\right)$, \citep{yao_functional_2005}, where $y_{ij}$ is as described in Equation~\eqref{eqn:fd_temporal} and setting $t_{i0}=0$.
However such an estimate breaks for the case that observations are sparse. Similarly such an approximation will be biased when the error processes from Equation~\eqref{eqn:fd_temporal}, $\varepsilon_{ij}$, is non-zero.
\citeauthor{yao_functional_2005} overcome this by first assuming that the model is as follows, \citep{yao_functional_2005}:

\begin{align}
	y_{ij} &= \chi_{i} + \varepsilon_{ij} \\
	&= \mu(t_{ij}) + \sum_{k=1}^{\infty} \xi_{ik} \phi_k(t_{ij}) + \varepsilon_{ij} \label{eqn:fd_temporal_fpca}
\end{align}
with $\varepsilon_{ij}$ being jointly Gaussian with $\xi_{ik}$.
We also require the noise process satisfies:
\begin{align}
	\E\left(\varepsilon_{ij}\right) &= 0 \\
	\text{Var}\left( \varepsilon_{ij} \right) &= \sigma_\varepsilon^2
\end{align}

The number of measurements of the $i^\text{th}$ subject is considered random which reflect sparse functional data.
Such a description follows naturally from our dataset description, given in Equation~\eqref{eqn:fd_temporal}, by using the FPCA decomposition structure of $\mathcal{X}$ as discussed in Section~\ref{sec:fpca}.
Following \citep{yao_functional_2005} we define the subsequent vector notations:
\begin{align}
	\vesub{Y}{i} &= \left(y_{i1}, y_{i2}, \cdots, y_{iJ_i}\right)^\top \label{eqn:yvec}\\
	\vesub{\phi}{ik} &= \left(\phi_{k}(t_{i1}), \phi_{k}(t_{i2}), \cdots, \phi_k(t_{iJ_i}) \right)^\top \label{eqn:phivec} \\
	\vesub{\mu}{i} &= \left(\mu(t_{i1}), \mu(t_{i2}), \cdots, \mu(t_{iJ_i})\right)^\top \label{eqn:muvec}\\
	\vesub{t}{i} &= \left( t_{i1}, t_{i2}, \cdots, t_{iJ_i} \right)^\top \label{eqn:tvec}
\end{align}
With such a model and assumptions, as stated in \citep{yao_functional_2005}, the best prediction of the principal component scores for the $i^\text{th}$  subject is given by:
\begin{equation}\label{eqn:fpc_best}
	\tilde{\xi}_{ik} = \E\left(\xi_{ik} | \vesub{Y}{i}, \vesub{t}{i} \right) = \lambda_k \vess{\phi}{ik}{\top} \vess{\Sigma}{\vesub{Y}{i}}{-1} \left( \vesub{Y}{i} - \vesub{\mu}{i} \right)
\end{equation}
where $\vesub{\Sigma}{\vesub{Y}{i}} = \text{Cov}\left(\vesub{Y}{i}, \vesub{Y}{i} \right)$.
The estimate for the principal component score can then be found by substituting in estimates for the various components in Equation~\eqref{eqn:fpc_best}.
That is:
\begin{equation}\label{eqn:fpc_est}
	\hat{\xi}_{ik}= \hat{\E}\left(\xi_{ik} | \vesub{Y}{i}, \vesub{t}{i} \right) = \hat\lambda_k \hvess{\phi}{ik}{\top} \hvess{\Sigma}{\vesub{Y}{i}}{-1} \left( \vesub{Y}{i} - \hvesub{\mu}{i} \right)
\end{equation}
The covariance matrix $\hvesub{\Sigma}{\vesub{Y}{i}}$ is formed with $\left(l, m\right)^\text{th}$ element:
\begin{equation}\label{eqn:sig_cov}
	\left[\hvesub{\Sigma}{\vesub{Y}{i}}\right]_{lm} = \hat{G}(t_{il}, t_{im}) + \hat{\sigma}_\varepsilon^2 \delta_{lm}
\end{equation}
where $\hat{\sigma}_\varepsilon^2$ is the estimated variance of the noise process.
The estimation method for this is discussed in Section~\ref{sec:splines}.
\citeauthor{yao_functional_2005} also provide asymptotic properties of such an estimator  along with asymptotic confidence bands where the mean and covariance surfaces are estimated with local linear smoothers, \citep{fan_study_1996}.

The conditional expectation technique describe above from \citep{yao_functional_2005} alleviates the issue of poor integral approximation from sparsely observed data when the estimated covariance surface is a relatively good fit to the true covariance surface.
This is a somewhat better condition as it allows one to pool data from different observed subjects to estimate such a surface and thus the requirement of dense data per subject is relaxed to having dense data from the collection over all subjects.
We discuss a particular method for estimating such surfaces in Section~\ref{sec:splines}.

\section{Penalised Regression Splines \label{sec:splines}}
 Smoothing models underpin much of FDA.
 FDA uses the smoothness of observations over a continuous domain to help inform and model observed data, \citep{ramsay_functional_2010}.
 Typically, as described in Section~\ref{sec:fr}, data is only observed discretely.
 Therefore with most FDA methodology there must be a conversion from discretely observed data and the continuous functional variable that generates it.
 This is particularly the case for our EO data since we have discrete observations specified by our data model given in Equation~\eqref{eqn:observed_data} which we assume is generated by observations of continuous functions given by our models in Equation~\eqref{eqn:fd_temporal}.
 As such many models for obtaining such a smooth of the data have been studied, such as kernel smoothing, polynomial regression, and local linear smoothing, \citep[Chapter~4]{ramsay_functional_2010}.
 In this section we consider the well studied technique of obtaining smooths of discrete data through penalised regression splines, \citep{ruppert_semiparametric_2003}.
 We will use such a method to estimate the mean and covariance surfaces present in the PACE methodology as described in Section~\ref{sec:pace}.
 We first describe the components that form the foundations of a regression spline, the spline basis.

 \subsection{Basis splines  \label{ssec:basis_splines}}
 One of the components of a penalised regression splines is the basis functions used in the regression.
 As the name suggests regression splines uses spline functions as the regression basis.
 A spline function of order $d$,  which is well documented in the monograph of \citeauthor{de_boor_practical_2001}, is a piecewise polynomial function of degree $d-1$, \cite{de_boor_practical_2001}. In the case of a spline function of order $d$, $S: \mathcal{T} \to \mathbb{R}$, over a univariate domain $\mathcal{T} = \left[a, b\right] \subset \mathbb{R}$ we have:
 \begin{equation}
 	S: t \mapsto S(t) = \begin{cases}
 		P_0(t)~\text{if}~ \tau_0 < t  \leq \tau_1,\\
 		P_1(t)~\text{if}~ \tau_1 < t  \leq \tau_2,\\
 		\vdots \\
 		P_{m-1}(t)~\text{if}~ \tau_{m-1} < t  \leq \tau_m,\\
 	\end{cases}
 \end{equation}
where $P_i: \left[\tau_i, \tau_{i+1}\right] \to \mathbb{R}$ are polynomial functions of degree $d-1$.
The vector of points $\ve{\tau} = \left(\tau_0, \tau_1, \cdots, \tau_m \right)$ is known as the knot vector for the spline and must satisfy $a=\tau_0 < \tau_1 < \cdots < \tau_m = b$.
By specifying that the piecewise polynomials must share the same derivative order up to a degree we can ensure continuity of relative smoothness over the knot points and the whole spline function.
We specify the continuity at each point in our knot vector by the continuity vector $\ve{r}=\left( r_0,\cdots, r_m \right)^\top$ where $r_i$ specifies that $P_i$ and $P_{i+1}$ share common derivative values at point $\tau_i$ for derivatives up to order $r_i$.
The spline type can be specified completely by specifying the knot locations and the continuity vector, \citep{de_boor_practical_2001}.
In fact, one can extend our definition of the knot vector to incorporate both the knot and continuity vector into one.
This is known as the extended knot vector, which will completely specify the spline type, \citep{de_boor_practical_2001}.
We define the extended knot vector as the vector of knot points which repeats the $i^\text{th}$ knot vector exactly $n - r_i$ times. That is:
\begin{equation*}
	(\tau_0,\cdots,\tau_0, \tau_1, \cdots, \tau_1 ,\cdots, \tau_{m-1},\cdots, \tau_{m-1}, \tau_m \cdots, \tau_m)
\end{equation*}
We denote the spline functions of order $d$ with extended knot vector by $S_{d, \ve{\tau}}$.

The Basis splines are more commonly referred to as B-splines, \citep{knott_interpolating_2000}.
B-splines are basis functions for splines of the same order defined over the same knots.
They are typically defined recursively, \citep{knott_interpolating_2000, de_boor_practical_2001}.
The classic algorithm for the recursive construction is known as the Cox-de Boor recursion formula, \citep{de_boor_practical_2001}, and is given as follows.
Given a knot vector $(\tau_0,\cdots,\tau_0, \tau_1,\cdots,  \tau_1, \cdots, \tau_{m-1},\cdots, \tau_{m-1}, \tau_m,\cdots, \tau_m)^\top$ the B-spline of order $1$ is given by:
\begin{equation}	\label{eqn:bspline1}
	B_{i, 1}\left( t \right) = \begin{cases} 1,~\text{for}~\tau_i \le t < \tau_{i+1} \\ 0,~\text{otherwise.} \end{cases}
\end{equation}

The higher order B-splines are defined by recursion as:
\begin{equation}\label{eqn:bspline_high}
	B_{i, q+1}(t) = w_{i, p}(t) B_{i, q}(t) + \left[ 1 - w_{i+1, q}(t) \right]B_{i+1, q}(t)
\end{equation}
where $w_{i, q}$ is a weighting for the $i^\text{th}$ B-spline of order $d$ given by:
\begin{equation}
	w_{i, q}(t) = \begin{cases} \frac{x-\tau_i}{\tau_{i+q} - \tau_i} ,~\text{for}~\tau_{i+q} \ne \tau_i \\ 0,~\text{otherwise.} \end{cases}
	\label{eqn:weighting}
\end{equation}
A B-spline basis system of size $Q$ can then be considered by choosing the extended knot vector $\ve{\tau}$ and specifying the order, $d$, of the B-spline functions, and is given by the collection:
\begin{equation}
	\{ B_{d, q}^{\ve{\tau}}(t)\}_{q=1}^{Q}
	\label{eqn:bspline_basis}
\end{equation}
where $Q$ is the number of basis functions to use in the system, $\ve{\tau}$ is the extended knot vector, and $B_{d, q}^{\ve{\tau}}$ is the $q^\text{th}$ B-spline of order $d$ defined by Equation~\eqref{eqn:bspline_high} for our knot vector $\ve{\tau}$.

\subsection{Regression splines \label{ssec:spline_reg}}
As discussed in Section~\ref{sec:pace} the PACE methodology requires estimation of both the mean function, $\mu(t)$, and covariance surface $G\left(s, t\right)$.
Estimating such functions is a problem due to their infinite dimensional nature.
A well studied and effective method for representing such functions is the use of a basis function expansion,  \cite{ramsay_functional_2010}.
That is representing the target surface using a linear combination of known basis functions.
In this work we will utilise the B-spline basis function as discussed in Section~\ref{ssec:basis_splines}.
The B-spline system is exceptionally popular due to its ease of computation and ability to reconstruct many surfaces, \citep{de_boor_practical_2001}.
Such ease of computation makes it feasible to not only create large basis systems but also alleviates many fitting procedures as we can re-evaluate the basis system at various points with ease.
Such properties are very useful when using such a basis for regression models.
Other common basis systems include the Fourier, Monomial, and Polynomial basis systems.
See \citep{ramsay_functional_2010} for details of these basis systems in the functional framework.
In the following we present the approach for estimating an arbitrary realisation of our functional random variable $\chi_i(t)$ over domain $\mathcal{T}$ and discuss how we extend the same concept to a two dimensional surface over $\mathcal{T} \times \mathcal{T}$.

We assume that our function  can be represented using an order $d$ B-spline basis system with knot vector $\ve{\tau}$:

\begin{align}
	\chi_i(t) &= \sum_{q=1}^Q c_q B_{d, q}^{\ve{\tau}}(t) \\
	&= \vesup{c}{\transpose} \vess{B}{d}{\ve{\tau}}(t) \label{eqn:basis_expansion}\\
\end{align}
where $\ve{c}=\left( c_1,\cdots, c_K \right)^\transpose$, $\vess{B}{d}{\ve{\tau}}(t) = \transpose{\left( B_{d, 1}^{\ve{\tau}}(t), B_{d, 2}^{\ve{\tau}}(t), \cdots, B_{d, Q}^{\ve{\tau}}(t) \right)}$, and $Q$ is the dimension of the expansion.
If such basis functions have nice properties like being easy to compute then such a representation for $\chi_i$ given by Equation~\eqref{eqn:basis_expansion} can be extremely useful since most problems can be reduced to involving only the finite dimensional vector $\ve{c} \in \mathbb{R}^Q$.

Our representation of $\chi_i$ using a basis system then becomes the problem of choosing the coefficients $\vesup{c}{\transpose}$ using only our set of observations of $\vesub{Y}{i}$ which are observed with error.
The most common method for fitting a basis system to discretely observed data is by choosing the coefficients of the expansion $c_q$ given in Equation~\ref{eqn:basis_expansion} by minimising the criterion, \cite{bjorck_numerical_1996}:
\begin{equation}
	\text{SSE}_{\vesub{Y}{i}}\left( \ve{c}\right) = \lVert \vesub{Y}{i} - \ve{B}\ve{c} \rVert^2
	\label{eqn:sse}
\end{equation}
where $\ve{B} = \left( \vess{B}{d}{\ve{\tau}}(t_{i1}),\vess{B}{d}{\ve{\tau}}(t_{i2}),\cdots, \vess{B}{d}{\ve{\tau}}(t_{iJ_i}) \right)^\transpose$ is the $J_i \times Q$ matrix of the basis system evaluated at observed time points corresponding to the $J_i$ length observation vector $\vesub{Y}{i}$. Minimising such a criterion is given by, \cite{bjorck_numerical_1996}:
\begin{equation}
	\hat{\ve{c}} = \left( \vesup{B}{\transpose} \ve{B} \right)^{-1}\vesup{B}{\transpose}\vesub{Y}{i}
	\label{eqn:hatc}
\end{equation}

The simple least squares approximation is a well studied and standard approach. See \citep{bjorck_numerical_1996} for a through introduction to the concept.
Such a methodology is often suitable for situations where our error process $\varepsilon(t)$ is a white noise process.
Such a process for the noise is often unrealistic; as such a simple adjustment to the least squares criterion in Equation~\ref{eqn:sse} can be used to allow for correlation among the observation errors:
\begin{equation}
	\text{SSE}_{\vesub{Y}{i}, \ve{W}}(\ve{c}) = \lVert \ve{W}^\frac{1}{2}\left( \vesub{Y}{i} - \ve{B}\vesup{c}{\transpose} \right) \rVert^2
	\label{eqn:wsse}
\end{equation}
where $\ve{W}$ is a weighting matrix for the observations. Ideally the matrix will be the inverse of the variance-covariance matrix of the observations. Minimising the adjusted criterion is given by, \cite{bjorck_numerical_1996}:
\begin{equation}
	\hat{\ve{c}} = \left( \vesup{B}{\transpose} \ve{W} \ve{B} \right)^{-1}\vesup{B}{\transpose} \ve{W} \vesub{Y}{i}
	\label{eqn:whatc}
\end{equation}
The estimate with least squares fitting can then be found by substituting $\hat{\ve{c}}$ for the $\ve{c}$ in Equation~\ref{eqn:basis_expansion}, \citep{bjorck_numerical_1996}. That is:
\begin{equation}\label{eqn;hat_basis_expansion}
	\hat{\chi}_i(t) = \vesup{\hat{c}}{\transpose} \vess{B}{d}{\ve{\tau}}(t)
\end{equation}
The selection of the knot vector is well studied and the classical choice is to choose a knot vector where knots are located at the sampling points, \citep{de_boor_practical_2001}.

An issue with the classical least squares fitting using a basis system expansion is the choice of number of basis functions, \cite{ramsay_functional_2010}.
We are constrained to choose $Q$ to be less than or equal to the number of observations, $J_i$. This is because more than $J_i$ basis functions would results in Equation~\ref{eqn:whatc} being ill defined since the matrix $\ve{B}$ would have linearly dependent columns.
However, we still have the choice to choose $Q$ between $1$ and $J_i$.
Exactly which value for $Q$ to choose is unknown and results in bias - variance trade off in the estimator, \citep{ramsay_functional_2010}.
A large number of basis functions reduces bias in the estimator $\hat{\chi}_i(t)$, but the variance of this estimator may be unacceptably high.
Conversely, a lower number of basis functions will result in high bias of the estimator but low variance.
The bias-variance trade off is well studied and there is a vast literature on the methodology of choosing the number of basis functions.
However, there is no gold standard and often the choice is made in an ad hoc fashion, \citep{ramsay_functional_2010}.
Such an issue motivates modifying the fitting criterion which determines $\ve{\hat{c}}$ in Equation~\eqref{eqn:whatc}.

\subsubsection{Penalties}
Ideally, we want to penalise estimators which have high variance, that occur naturally when we have a large number of basis functions, but keep bias low.
The naive choice of just reducing the number of basis functions, known as regression splines,  fails in this respect, \citep{ruppert_semiparametric_2003}.
One such approach to do this is to reduce the number of basis functions in conjunction with a penalty, known as penalised regression splines, \citep{ruppert_semiparametric_2003}.
Such an approach was first used in \citep{osullivan_statistical_1986} who applied such a technique to ill posed inverse problems.
\citep{ruppert_semiparametric_2003} discusses various other spline smoothing techniques as well as the penalised regression splines.

Penalised regression spline models adjust the fitting criterion in Equation~\eqref{eqn:whatc} to, \citep{ruppert_semiparametric_2003}:
\begin{equation}\label{eqn:plss}
	\text{PSSE}_{\vesub{Y}{i}, \ve{W}, \lambda}(\ve{c}) = \lVert \vesup{W}{\frac{1}{2}}\left( \vesub{Y}{i} - \ve{B}\ve{c} \right) \rVert^2 + \omega \vesup{c}{\transpose} \ve{P} \ve{c}
\end{equation}
where $\ve{P}$ is formed with $(l,m)^\text{th}$ element $\left[\ve{P}\right]_{lm} = \langle L\left(\vesub{B}{l}\right), L\left( \vesub{B}{m} \right) \rangle$ and $\omega$ is a parameter which controls the regularisation trade off.
$L$ is some linear differential operator.
Typically, one chooses $L$ to be the required smoothness of the target function and examples include simple first or second derivatives, \cite{ruppert_semiparametric_2003}.

Analytically minimising the $\text{PSSE}$ criterion in Equation~\eqref{eqn:plss} can be found via, \citep{ruppert_semiparametric_2003}:
\begin{equation}
	\hat{\ve{c}} = \left( \vesup{B}{\transpose} \ve{W} \ve{B} + \omega \ve{P} \right)^{-1}\vesup{B}{\transpose} \ve{W} \vesub{Y}{i}
	\label{eqn:phatc}
\end{equation}

Essentially such a penalisation term determines that there should be a trade off between the bias which corresponds to the first term in Equation~\eqref{eqn:plss} and the variance which is the second term.
This trade off is controlled by the regularisation parameter $\omega$.
The advantage of this method is that we can now let $Q$, our number of basis functions, be large without worrying of over fitting as the penalty term in Equation~\eqref{eqn:plss} will penalise functions with high variability in terms of the differential operator $L$.

The choice of differential operator is a well studied problem also. A common choice is the first or second order differential, denoted by  $D^1$ and  $D^2$ respectively, as this specifies a reasonable level of smoothness in the target function, \cite{ruppert_semiparametric_2003}. However, often more complex terms are used to facilitate known properties of the target functions, such a letting $L$ be the harmonic acceleration operator which forces a periodic form of the target functions.
More care must be taken when extending the linear differential operator to higher dimensions which is discussed below.
Additionally, in the case of B-spline basis system these penalty matrices are typically evaluated using a form of numerical integration, \cite{ramsay_functional_2010}.

Penalised regression splines moves our problem of selecting $Q$ to choosing our regularisation parameter, $\omega$.
Such a parameter influences the strictness with which we expect our target function to be smooth as defined by the operator $L$.
Choosing such a parameter is a problem that is present not only in spline smoothing but other penalised regression approaches, \citep{lukas_robust_2006}.
A popular method for choosing such a parameter is Generalised Cross Validation (GCV).
GCV, introduced by \citeauthor{wahba_practical_1977} in \citep{wahba_practical_1977}, is a well studied method which has good asymptotic properties as the number of observations tends to infinity, \citep{wahba_spline_1990, wahba_comparison_1985}.
GCV chooses $\omega$ as the minimiser of the GCV criterion $V(\omega)$ which is given by, \citep{wahba_spline_1990}:
\begin{equation}\label{eqn:gcv}
	V\left(\omega\right) = \frac{J_i^{-1} \lVert \left(\ve{I} - \ve{A} \right) \vesub{Y}{i} \rVert^2}{\left[ J_i^{-1} \text{tr}\left(\ve{I} - \ve{A}\right)\right]^2}
\end{equation}
where $\ve{A}$ is the influence matrix defined by:
\begin{equation}\label{eqn:inf}
	\ve{A} = \ve{B} \left( \vesup{B}{\transpose} \ve{W} \ve{B} + \omega \ve{P} \right)^{-1}\vesup{B}{\transpose} \ve{W}
\end{equation}
The GCV method can then be minimised for $\omega$ using a numerical minimisation routine.
For large $J_i$ it is known that the GCV criterion performs well in recovering a regularisation parameter which minimises variance while maintaining low bias in the reconstruction of the target function, \citep{wahba_comparison_1985}.
For the case of low $J_i$ the GCV method may not be reliable.
As such, methods to extend the GCV criterion have been considered.
The modified GCV criterion adds a further modifier to the denominator in Equation~\eqref{eqn:gcv} by multiplying the trace of the influence matrix by a factor, \citep{cummins_confidence_2001}.
The modified GCV approach effectively increases the cost associated with each effective parameter in the curve which reduces the chance of choosing an $\omega$ which under smooths the data, \citep{cummins_confidence_2001}.
A similar but separate approach to adjusting the GCV is robust GCV, introduced by \citeauthor{lukas_robust_2006}.
\citeauthor{lukas_robust_2006} uses a weighted sum of the GCV function with a term which penalises $\omega$ values that are close to zero, \citep{lukas_robust_2006}.
The performance of such methods are discussed in \citep{lukas_performance_2012}.

Choosing a basis system, a criterion to choose the regularisation parameter, and a differential operator then fully specifies the penalised regression spline approach.
In the case of one dimensional functions the procedure applies as above. As such, we can estimate our mean function $\mu(t)$ through the use of a penalised regression splines where our observation points for the mean function are the pooled mean across subjects of the union of observed time points for all curves.
However for multiple dimensions, particularly the case when we wish to smooth the covariance surface, we must make some adjustments to the approach described above.

\subsubsection{Extension to higher dimensions \label{sssec:spline_ext}}
There are two issues when extending the penalised regression spline to higher dimensions; extending the basis system and extending the penalty specification.
To alleviate the first we must specify a basis system which can cover multiple dimensions. In fact there are many such systems, \citep{wahba_spline_1990}.
One such popular approach when we have regular data for FDA is using a tensor product B-spline system, \citep{xiao_asymptotic_2020}.
We describe the extension to two dimensional surface, but the same extension will work for higher dimensional surfaces.
Consider a two dimensional surface, $\sigma \left(s, t\right)$, which we represent by the tensor product spline given by, \citep{xiao_asymptotic_2020}:
\begin{equation}\label{eqn:tensor_expansion}
	\sigma \left(s,t\right) = \sum_{1 \le q_1, q_2 \le \bar{Q}} c_{q_1, q_2} \vess{B}{d_1}{\vesub{\tau}{1}}(s)\vess{B}{d_2}{\vesub{\tau}{2}}(t)
\end{equation}
where $\vess{B}{d_i}{\vesub{\tau}{i}}$ is the B-spline basis system for the $i^\text{th}$ dimension for $i=1,2$.
For notational simplicity we assume the dimension of each marginal basis system is the same, $\bar{Q}$.
However, in general this need not be the case.
$\ve{C} \in \mathbb{R}^{\bar{Q} \times \bar{Q}}$ is a coefficient matrix to be determined.
Equation~\eqref{eqn:tensor_expansion} can be written more succinctly using a Kronecker product as, \citep{xiao_asymptotic_2020}:
\begin{equation}\label{eqn:kron_expansion}
	\sigma\left(s,t\right) = \vesup{\bar{B}}{\transpose}\left(s, t\right) \text{Vec} \left(\ve{C}\right)
\end{equation}
where $ \ve{\bar{B}}\left(s, t\right) = \vess{B}{d_2}{\vesub{\tau}{2}}(t) \otimes \vess{B}{d_1}{\vesub{\tau}{1}}(s)$ and $\text{Vec}\left(\cdot\right)$ is an operator which stacks the columns of a matrix into a vector. We use the $\bar{\cdot}$ notation to make explicit that this basis is over multiple dimensions.

The same methods now follow as in the non penalised univariate case with this Kronecker basis system, \citep{xiao_asymptotic_2020}.
However, we must still adjust the penalty matrix in Equation~\eqref{eqn:phatc} to account for smoothness across multiple dimensions.

Using the tensor product basis system as described above one might consider specifying the smoothness of the function in each dimension.
Indeed, one such approach to extending the penalty specification which was introduced by \citeauthor{wood_low-rank_2006} is to consider setting penalties on the marginal basis separately and to combine them by a weighted sum, \cite{wood_low-rank_2006}.
Such an approach known as tensor product penalties is well studied in the linear generalised additive model setting, \cite{wood_generalized_2006}.
A two dimensional penalty matrix $\bar{\ve{P}}$ may be described as follows:
\begin{equation}\label{eqn:tensor_pen}
	    \bar{\ve{P}} = \omega_1 \ve{P}_1 \otimes \ve{I}_2 + \omega_2 \ve{I}_1 \otimes \ve{P}_2
\end{equation}
where $\vesub{P}{i}$ is marginal penalty over a single basis dimension as described in Equation~\eqref{eqn:plss}, $I_i$ is the identity matrix of dimension of the $i^\text{th}$ dimension basis, and $\omega_i$ is the marginal regularisation parameter for $i=1,2$.
The properties of such a smoothness penalty are discussed in detail in \citep{wood_low-rank_2006} with the main points being such a penalty is both scale invariant and low rank.
In addition, \citep{wood_p-splines_2017} studies the use of such a penalty for the case of unevenly distributed data.
The additional complication is that we now have multiple smoothness parameters $\omega_i$, one for each dimension of the surface to be smoothed.
In this case the GCV methodology described above, can still be applied but now minimisation occurs with respect to the vector $\ve{\omega} = \left(\omega_1, \omega_2 \right)^\transpose$. Implementation details of such can be found in \cite{wood_generalized_2006}.

We can now use the above approach to estimate our covariance surface, denoted by $\hat{G}\left(s,t\right)$, for use in PACE methodology, \citep{yao_functional_2005}.
The discrete observations for the covariance surface to be smoothed are gathered by pooling individual observed covariances from across subjects, which is discussed in detail in both \citep{yao_functional_2005, xiao_asymptotic_2020}.
\citeauthor{xiao_asymptotic_2020} provides asymptotic properties of such an approach to the covariance surface of independent functional data which are on par to the asymptotic results of other smoothers used in \citep{yao_functional_2005} for the PACE methodology, \citep{xiao_asymptotic_2020}.

\section{Functional Time Series\label{sec:fts}}
As discussed in Section~\ref{sec:eo} EO data is often both spatially and temporally correlated.
The two types of correlation are often considered separately.
An area in FDA which has considered a similar case where functional data is observed and observations are correlated is functional time series, \citep{aguilera_forecasting_1999}. Typically, functional observations are naturally indexed by some time of observation and correlation may occur between observations.
Hence we may build up a time series of functional observations.
Functional time series models are some of the first in the FDA literature to start to consider correlated functional observations.
Although they limit themselves to temporal correlation many of the ideas can be considered for extensions to higher dimension correlation and so we discuss a few of the more popular methodologies in this section.

We focus on a technique introduced by \citeauthor{hyndman_forecasting_2009} in \citep{hyndman_forecasting_2009} to forecast functional time series.
Such a method is of interest as it expands methodology on how to use existing forecasting techniques in a functional setting.
In particular \citep{hyndman_forecasting_2009} uses the FPCA decomposition described in Section~\ref{sec:fpca} to decompose functional observations and then uses independent forecasting of each principal component score using standard multivariate techniques.

\citeauthor{hyndman_robust_2007} in \citep{hyndman_robust_2007} suggests to assume the principal component scores, $\xi_{ik}$, follow independent univariate time series models.
Then, conditioning on the observed data $\ve{Y}$ given in Equation~\eqref{eqn:observed_data} and the set of principal components $\ve{\phi}(t) = \left(\phi_1(t), \phi_2(t), \cdots, \phi_K(t)\right)$ they obtain the $h$-step ahead forecast of $\chi_{i+h | i}(t)$ as, \citep{hyndman_robust_2007}:

\begin{equation}\label{eqn:forecast}
	\hat{\chi}_{i+h | i} = \E\left(\chi_{i+h}(t) | \ve{Y}, \ve{\phi}\right) = \hat{\mu}(t) + \sum_{k=1}^K \hat{\xi}_{i+h | i, k} \phi_k(t)
\end{equation}
where $\hat{\xi}_{i+h | i, k}$ denotes the $h$-step ahead forecast of the $k^\text{th}$ principal component score.
The method for which $\hat{\xi}_{i+h | i, k}$ is obtained can be any univariate time series method. Such methods are extremely well studied and discussed in the monograph of \citeauthor{hyndman_forecasting_2018} in \citep{hyndman_forecasting_2018}.
\citeauthor{hyndman_stochastic_2008} highlight that the forecast, given by Equation~\eqref{eqn:forecast}, is relatively insensitive to the choice of number of components in the principal decomposition provided it is sufficiently large.
The variance of such a method can also easily be obtained through the sum of the component variances, \citep{hyndman_stochastic_2008}.
The component variance of the forecast principal component scores are generally readily available from many time series models, \citep{hyndman_forecasting_2018}.
The above forecasting methodology initially described in \citep{hyndman_robust_2007} used normal FPCA procedure with outliers weighted to zero, however this was reconsidered in \citep{hyndman_forecasting_2009} to include a geometric weighting to the principal components to allow for changes in the function over time.

Such a methodology motivates the construction of the CPACE model described in Chapter~\ref{cha:cpace} for correlated functional data by considering the case where the principal components scores obey univariate correlated models not just time series. To describe some of these such models we use the concept of a Gaussian Process. We give background to this in the following section.

\section{Gaussian Process Regression \label{sec:gp}}
The above section on functional time series shows there is scope for placing a model on the principal component scores to allow for correlation among functional observations.
The natural progression to such work is to consider what options are available when we have more complex correlation structure or a higher dimensional domain.
For example, in the case of EO data discussed in Section~\ref{sec:eo}, we have functional observations over a spatial domain indexed by some coordinate, $\ve{s} \in \mathcal{S}$.
For this the univariate time series methods discussed in \citep{hyndman_forecasting_2009} are not suitable and we look to Gaussian processes as one possible solution to model principal component scores which are indexed by space.
A such we discuss the basic concept of a Gaussian process in the following.

To describe a Gaussian process we first discuss the concept of a stochastic process.
A real valued stochastic process is a collection of real random variables defined on the same probability space $\left(\Omega, \mathcal{F}, \mathcal{P}\right)$ where $\Omega$ is a sample space, $\mathcal{F}$ is a $\sigma$-algebra, and $\mathcal{P}$ is a probability measure; and the random variables, indexed by some set $\mathcal{S}$ are all real valued. More details of such constructions can be found in \citep{billingsley_probability_1995}. A stochastic process can then be written as the collection:
\begin{equation*}
	\{ \xi(\ve{s}, w)| \ve{s} \in \mathcal{S} \}
\end{equation*}
where $w \in \Omega$. A sample function of the stochastic process is the mapping, for a point $w \in \Omega$:
\begin{equation*}
	 \xi(\cdot, w) : \mathcal{S} \to \mathbb{R}
 \end{equation*}

A Gaussian process is a stochastic process which is parametrised by a mean function $m: \mathcal{S} \to \mathbb{R}$ where $m(\ve{s}) = \E\left(\xi(\ve{x})\right)$ and its covariance function:
\begin{align*}
	k&: \mathcal{S}^2 \to \mathbb{R} \\
	k&: \ve{s} \mapsto \text{Cov}\left(\xi(\ve{s}), \xi(\ve{s})\right)
\end{align*}
where for any finite collection of points, $\vesub{s}{1}, \vesub{s}{2}, \cdots, \vesub{s}{n} \in \mathcal{S}$, the joint distribution of $\vesub{\xi}{n} = \left(\xi(\vesub{s}{1}), \xi(\vesub{s}{2}), \cdots, \xi(\vesub{s}{n})\right)^\transpose$ is a multivariate normal distribution with mean vector $\vesub{m}{n} = \left(m(\vesub{s}{1}), m(\vesub{s}{2}), \cdots, m(\vesub{s}{n})\right)^\transpose$ and covariance matrix $\vesub{K}{n}$ whose $\left(l,m\right)^\text{th}$ entry is given by $k(\vesub{s}{l}, \vesub{s}{m})$, \citep{shi_gaussian_2011}.
As such, Gaussian processes are a natural way of defining a prior distribution over spaces of functions, which are the parameter spaces for Bayesian non linear regression models.
In this work, we will denote such a Gaussian process by $\mathcal{GP}$ and write:
\begin{equation}\label{eqn:gp}
	\xi(\cdot) \sim \mathcal{GP}\left( m(\cdot), k(\cdot, \cdot) \right)
\end{equation}

One aspect of Gaussian process regression models is that under Gaussian assumptions they have a nice closed form for prediction. Let $S = \{ \vesub{s}{1}, \vesub{s}{2}, \cdots, \vesub{s}{n}\}$ denote the design matrix of the regression, and $\ve{\xi}$ denote the corresponding target vector.
Then conditioning on the joint Gaussian prior distribution on the observations gives the posterior at prediction points $S_*$, \citep{williams_gaussian_2006}:
\begin{equation}\label{eqn:gp_pred}
	\vesub{\xi}{*} | S_{*}, S, \ve{\xi} \sim \mathcal{N}\left(K(S_*, S)K(S, S)^{-1} \ve{\xi}, K(S_*, S_*) - K(S_*, S)K(S, S)^{-1}K(S, S_*)\right)
\end{equation}
where $\vesub{\xi}{*}$ is our posterior process evaluated at the collection of prediction points, and $K(\cdot, \cdot)$ is the covariance matrix formed by evaluating the covariance function $K(\cdot, \cdot)$ at all pairs of inputs. This can be extended easily to noisy observations by adjusting the observed covariance $K(S, S)$ to include a diagonal component which is the effect of model observation error. See \citep{williams_gaussian_2006} for details.

One key aspect of the Gaussian process is the covariance function $k(\cdot, \cdot)$.
The covariance function characterises various smoothness properties such as the sample path continuity and its differentiability, \citep{williams_gaussian_2006}.
As such the choice of $k(\cdot, \cdot)$ heavily influences the prediction mean and covariance as described in Equation~\eqref{eqn:gp_pred}.
There are various common forms of the covariance function but all must have the intrinsic property of being non-negative definite.
The covariance function is of such importance in Gaussian process modelling and spatial statistics that it has been widely studied.
See \citep[Chapter~4]{williams_gaussian_2006} for a detailed introduction to various covariance functions. We expand on Section~\ref{sec:st_methods} to briefly introduce the form of covariance function which we will consider throughout this work.

Of the many different covariance functions employed in Gaussian processes stationary covariance functions are most commonly employed due to their simplicity and ease of construction, \citep{cressie_statistics_2010}.
One such commonly used covariance function is the Mat\`{e}rn covariance function which is given by, \citep{abramowitz_handbook_2013}:
\begin{equation}\label{eqn:mat}
	C_\nu(d) = \sigma^2 \frac{2^{1-\nu}}{\Gamma(\nu)} \left(\sqrt{2\nu}d\right)^\nu K_\nu \left( \sqrt{2\nu}d \right)
\end{equation}
where $\Gamma$ is the gamma function, $K_\nu$ is the modified Bessel function of the second kind, $\nu$ is a shape parameter of the kernel, and $d$ is the possibly anisotropic separation between two vectors $\ve{s}, \vesup{s}{\prime}$, \citep{abramowitz_handbook_2013}.
The covariance kernel $k(\ve{s}, \vesup{s}{\prime})$ is then simply $C_\nu(d(\ve{s}, \vesup{s}{\prime}))$.

The issue with stationary covariance forms is that they are often quite restrictive in the sense that the correlation structure cannot vary across the domain.
For example, this might be a too restrictive assumption in the case of climate data where correlation structure might be quite different in different parts of the globe.
One particular way to extend the stationary Mat\`{e}rn kernel to be non-stationary is proposed in \citep{paciorek_spatial_2006}.
\citeauthor{paciorek_spatial_2006} propose a method to knit together multiple stationary correlation functions such that the resultant function is non-stationary.

They provide a form of non-stationary covariance function $k^{NS}(\cdot, \cdot)$ from a stationary covariance  function $k^{S}(\cdot, \cdot)$ as follows, \citep{paciorek_spatial_2006}:
\begin{equation}\label{eqn:mat_ns}
	k^{NS}(\ve{s}, \vesup{s}{\prime}) = \lvert \Sigma_{\ve{s}} \rvert ^{\frac{1}{4}} \lvert \Sigma_{\vesup{s}{\prime}} \rvert^{\frac{1}{4}} \lvert \frac{ \Sigma_{\ve{s}}  + \Sigma_{\vesup{s}{\prime}}}{2}\rvert^{-\frac{1}{2}} k^S(Q(\ve{s}, \vesup{s}{\prime}))
\end{equation}
where $Q(\ve{s}, \vesup{s}{\prime}) = \left(\ve{s} - \vesup{s}{\prime}\right)^\transpose \left( \frac{ \Sigma_{\ve{s}}  + \Sigma_{\vesup{s}{\prime}}}{2}\right)^{-1}  \left(\ve{s} - \vesup{s}{\prime}\right)$ and $\Sigma_{\ve{s}} = \Sigma(\ve{s})$ is the covariance matrix of the Gaussian kernel centred at $\ve{s}$. How $\Sigma_{\ve{s}}$ varies across the domain specifies how non-stationary the full covariance kernel is.

With almost all covariance functions and especially non-stationary covariances there are typically hyper parameters which must be estimated from the data.
For example in the Mat\`{e}rn covariance we have the shape parameter $\nu$ and any length scale parameters defined in the distance function $d(\cdot, \cdot)$.
These are typically estimated though maximum likelihood estimation, \citep{williams_gaussian_2006}, however fully Bayesian estimation can also be achieved through some Markov Chain Monte Carlo (MCMC) scheme, \citep{paciorek_spatial_2006}.


\SecDef{symmetries}{Rotational Invariants in NORX}

In this section I describe rotational symmetries in the permutation $F$ of NORX. They exist both on the word level (inherited from $G$) and on the state level (structural).

\subsection{State Invariants}

We can see a 4x4 NORX state $S$ as a list of 4 columns: $S = (c_1, c_2, c_3, c_4)$.

\begin{definition}[Columns Rotation]
For an integer $n$ denote by $R_n$ the function rotating of the columns left by $n$ positions. For example $R_1(c_1, c_2, c_3, c_4) = (c_2, c_3, c_4, c_1)$ for arbitrary $c_1, c_2, c_3, c_4 \in (\field{w})^4$.
\end{definition}

The following proposition shows that the permutation $F$ is column rotation-symmetric.

\begin{proposition}
The permutations $R_n$ and $F^l$ commute for any integers $n$ and $l \ge 1$:
$$
F^l \circ R_n = R_n \circ F^l.
$$
\end{proposition}
\begin{proof}
Clearly, the rotation of columns does not affect the column step $\col$, since it transforms each column separately: $\col \circ R_n = R_n \circ \col$. Such rotations do not break the diagonals as well, because the diagonals are simply reordered. Therefore, $\diag \circ R_n = R_n \circ \diag$. It follows that $F$ commutes with $R_n$ and thus $F^l$ commutes with $R_n$ too.
\end{proof}

\begin{definition}
A state $s \in (\field{w})^{16}$ is said to be \emph{column $n$-rotation invariant} if
$$
R_n(s) = s.
$$
\end{definition}

Let $s \in (\field{w})^{16}$ be a column $n$-rotation invariant state for a fixed positive integer $n$. Observe that
$$
R_n(F(s)) = F(R_n(s)) = F(s),
$$
i.e. $F(s)$ is also column $n$-rotation invariant. It follows that the property of a state being column $n$-rotation invariant is an invariant of the round function $F$. It is easy to see that this invariant corresponds to an invariant subspace.

\begin{proposition}
For a fixed integer $n, 1 \le n \le 3$, the set of all column $n$-rotation invariant states is a linear subspace of $(\field{w})^{16}$.
For $n = 1$ or $n = 3$ this is the same subspace of dimension $4w$,
for $n = 2$ the invariant subspace has dimension $8w$.
\end{proposition}
\begin{proof}
    If $n=1$ or $n=3$, then for any $c_1, c_2, c_3, c_4 \in (\field{w})^4$
    $$
    (c_1, c_2, c_3, c_4) = (c_2, c_3, c_4, c_1)
    $$
    and it follows that all columns are equal: $c_1 = c_2 = c_3 = c_4$. There are $2^{4w}$ out of $2^{16w}$ such states. The designers of NORX noted these states in~\cite{DBLP:conf/latincrypt/AumassonJN14}. A constraint $c_i = c_j$ consists of $4w$ linear equations $c_{i,y,x} \oplus c_{j,y,x} = 0$, where $1 \le y \le 4, 1 \le x \le w$. Therefore, these constraints define a linear subspace of dimension $16w - 3\cdot 4w = 4w$.

    If $n = 2$, then for any $c_1, c_2, c_3, c_4 \in (\field{w})^4$ $$
    (c_1, c_2, c_3, c_4) = (c_3, c_4, c_1, c_2)
    $$
    and it follows that the two pairs of columns are equal: $c_1 = c_3$ and $c_2 = c_4$. There are $2^{8w}$ out of $2^{16w}$ such states. Similarly, these constraints define a linear subspace of dimension $8w$.
\end{proof}

Hitting such a special state even for the case $n=2$ is not easy under the NORX security claims. However, $2^{8w}$ is a more serious fraction of states than the $2^{4w}$ weak states which were known to the designers. To illustrate possible dangers of such properties, I refer to the forgery attack~\cite{NORXfse} on the previous version of NORX exploiting this invariant, and I also describe two hypothetical attacks on NORX8~\cite{aumasson2015norx8}, a NORX version with 8-bit words for low-end devices. I remark that NORX8 is not a part of the CAESAR submission.

The fist attack shows a weak-key set, which could be exploited if the domain separation constants were rotation-invariant. The weak-key set is relatively small, $2^{32}$ keys out of $2^{80}$. The second attack is a state/key recovery attack in a known plaintext scenario. It succeeds with probability $2^{-64}$ for each two consequent known-plaintext blocks, and the total time complexity is $2^{72}$ to recover an 80-bit key. Note that the designers restrict the data per single key to $2^{24}$ message blocks, therefore, the attack can break a concrete key with probability only $2^{-40}$.

Both attacks are independent of the number of rounds $l$ used in the permutation.

\subsection{Hypothetical Weak-key Attack on NORX8 Initialization}
The initial state of NORX8 is given by
\begin{equation}
\begin{pmatrix}
    n_1 & n_2 & n_3 & n_4 \\
    k_1 & k_2 & k_3 & k_4 \\
    k_5 & k_6 & k_7 & k_8 \\
    k_9\oplus w & k_{10}\oplus l & u_{15}\oplus p  & u_{16}\oplus t \\
\end{pmatrix} \in (\field{8})^{16},
\end{equation}
where $n_i$ and $k_i$ denote bytes of the nonce and the key respectively, $u_i$ are constants and $w,l,p,t$ are constants encoding parameters of NORX. It is possible to construct valid initial states with two equal halves, i.e. a column 2-rotation invariant state. Indeed, let us fix the four key bytes $(k_3, k_4, k_7, k_8)$ arbitrarily and let us choose the two nonce bytes $(n_3, n_4)$ arbitrarily. Then we can set the left half of the state equal to the right half, i.e.
\eq{
(n_1, n_2) &= (n_3, n_4),\\
(k_1, k_2) &= (k_3, k_4),\\
(k_5, k_6) &= (k_7, k_8),\\
(k_9, k_{10}) &= (u_{14} \oplus p \oplus w, u_{15} \oplus t \oplus l).
}
There are $2^{32}$ weak keys out of $2^{80}$ and $2^{16}$ nonces that result in such a weak state. The column 2-rotation invariant of such state is preserved through arbitrary number of rounds of $F$. However, after the first $F^l$ rounds the domain separation constant will be added to the last word of the state (see~\FigRef{sponge2}). This constant is not column 2-rotation invariant and therefore it will break the property. Therefore, we consider a slightly modified version of NORX8 where the domain separation constant is column 2-rotation invariant. For example, the original constant may be added not only to the last word, but to all words of the state or to all words in the last row. In such case the invariant is preserved through the next $F^l$ rounds and the rate part of the state is then observed by an adversary. This leads to a simple distinguisher: the adversary simply compares the left and right halves of the exposed part of the state. In NORX8 the rate part consists of only 5 bytes. It allows to check only the topmost 4 words with error probability $2^{-16}$. By using a few more encryptions with another weak nonces the error probability can be decreased to negligible.

I remark that the weak key space is very small and the attack requires symmetric domain separation constants. On the other hand, it is powerful in that it is independent of the number of rounds. The attack illustrates possible dangers of having such strong invariants in the permutation.

\subsection{State Recovery Attack on NORX8}

The column $2$-rotation invariant can be used to mount a state/key recovery attack on NORX8, though exceeding the data usage limit defined by the designers.

\begin{figure}[htbp]
    \centering
    \includegraphics[height=2.8cm]{\PathFig{layout.png}}
    \FigDef{sponge2}{The NORX v2.0 AE scheme with parallelization parameter $p = 1$. NORX8 and NORX16 follow this scheme. (credits: NORX specification~\cite{NORX})}
\end{figure}

Assume that we have a two-block known-plaintext message. That is, we know the rate part before and after a call to the NORX8 core permutation $F^l$. Denote the input rate part by $a$ and the output rate part by $b$. Recall that the rate in NORX8 is 40 bits, which is five 8-bit words. With probability $2^{-16}$ we will observe $a_1 = a_3, a_2 = a_4$. Then there are two cases:
\begin{enumerate}
    \item The whole state is column rotation-2 invariant. The probability of this is equal to  $2^{-6\cdot8}=2^{-48}$, given the observed rate part. Indeed, a uniformly random state is column 2-rotation invariant with probability $2^{-64}$. In this case the output state will be also column rotation-2 invariant with probability 1 and we will observe $b_1 = b_3, b_2 = b_4$.

    \item The whole state is not column rotation-2 invariant. Then with probability $2^{-16}$ we will observe $b_1 = b_3, b_2 = b_4$ as a false positive.
\end{enumerate}
As a result, when we observe both $a_1 = a_3, a_2 = a_4$ and $b_1 = b_3, b_2 = b_4$, the probability of the state being column rotation-2 invariant is equal to $2^{-32}$ and in the other cases it is a false positive. In the first case the state before the call to $F^l$ contains 5 unknown words $x_1,\ldots,x_5 \in \field{8}$:
$$
\begin{pmatrix}
    a_1 & a_2 & a_3=a_1 & a_4=a_2 \\
    a_5 & x_1 & a_5 & x_1 \\
    x_2 & x_3 & x_2 & x_3 \\
    x_4 & x_5 & x_4 & x_5 \\
\end{pmatrix}.
$$
We can exhaustively check all $2^{40}$ possibilities for $x_1, \ldots, x_5$ by encrypting through $F^l$ and obtaining extra filter with probability $2^{-24}$ from $b$. The remaining $2^{16}$ candidates can be checked by decrypting the state up to the initial state and matching the constants and further verifying the tag.

As a result, with probability $2^{-64}$ two consequent known-plaintext blocks allow to recover the full state and the secret key. The initial filter has strength $2^{-32}$ and the time complexity of checking a block pair is $2^{40}$. Note that the designers set a limit to $2^{24}$ data, therefore the attack succeeds for a concrete key only with probability around $2^{-40}$.

\subsection{Word Invariants}

\newcommand\rr{\mathbf{r}}
A similar rotational symmetry exists on the word level too.
Let $G'$ be the permutation of $(\field{w})^4$ to itself obtained from $G$ by replacing the four left shift operations by left rotations.

\begin{proposition}
\PropLabel{GprimeG}
$G' = G$ is conditioned by 4 bit equations, where each equation holds with probability $3/4$.
\end{proposition}
\begin{proof}
The left shift by one inserts a zero in the least significant bit of the result. If the most significant bit of the input is equal to 0, then the left shift is equivalent to the left rotation. There are 4 left shifts in $G$, each yields such bit equation. The input of a left shift in $G$ is simply an AND of two state bits, which are uniformly distributed.
\end{proof}

\begin{observation}
Experimentally, it is observed that $\Pr[G' = G]$ is close to $2^{-1.82}$, where the input is sampled uniformly at random, for all word sizes $w \in \{8,16,32,64\}$.
\end{observation}

Note that this observation shows the effect of dependency of the four quarter-steps in $G$. The probability that all these bits are equal to zero can be estimated as $(3/4)^4 \approx 2^{-1.66}$. However, the actual probability is lower due to the dependency of the equations.

\begin{definition}
Let $r_n\colon \field{w} \to \field{w}$ be the mapping which rotates a word left by $n$ bits and let $\rr_n \colon (\field{w})^4 \to (\field{w})^4$ be defined as
$$
\rr_n(a,b,c,d) \eqdef (r_n(a), r_n(b), r_n(c), r_n(d)).
$$
\end{definition}

\begin{proposition}
\PropLabel{Gcommute}
For any integer $n, 1 \le n < w$, $\rr_n$ commutes with $G'$:
$$
G' \circ \rr_n = \rr_n \circ G'.
$$
Furthermore, $\rr_n$ commutes with $G$ conditioned by 8 bit equations, each holding with probability $3/4$.
\end{proposition}
\begin{proof}
First, it is easy to verify that all operations in $G'$ commute with $\rr_n$. For a binary operation to commute it is required that $\rr_n$ applied to both inputs is equivalent to $\rr_n$ applied to the output.

The second claim follows by applying \PropRef{GprimeG} to the equation $G' \circ \rr_n = \rr_n \circ G'$ two times.
\end{proof}

\begin{observation}
Experimentally, it is observed that $\Pr[G' \circ \rr_n = \rr_n \circ G']$ varies from $2^{-3.84}$ to $2^{-3.59}$ depending on the word size and rotation amount $n$. The rotation amounts corresponding to the smallest probabilities are $1$ and $w-1$.
\end{observation}

Similarly to the column $n$-rotation invariant, define the word $n$-rotation invariant.

\begin{definition}
A columns $c \in (\field{w})^4$ (resp. a state $s \in (\field{w})^{16}$) is said to be word $n$-rotation invariant if for each its word $c_i$ (resp. $s_i$) the following holds:
$$
r_n(c_i) = c_i~~(\text{resp.}~r_n(s_i) = s_i).
$$
\end{definition}

\begin{proposition}
The set of all word $n$-rotation invariant states is a linear subspace of dimension $16\cdot gcd(n, w)$.
\end{proposition}
\begin{proof}
It is easy to see that a word $v \in \field{w}$ is word $n$-rotation invariant if and only if it is made of $w/gcd(n,w)$ copies of the same vector $u \in \field{gcd(n,w)}$. Clearly, all such words form a linear subspace of $\field{w}$ of dimension $gcd(n,w)$. As there are 16 words in the state, the proposition follows.
\end{proof}

Note that the property of a state or column being invariant requires only one approximation of $G$ by $G'$, i.e. it is approximately twice as more probable than the commutation.

\begin{proposition}
Let $c \in (\field{w})^4$ be a word $n$-rotation invariant column. Then
$$
\Pr[\rr_n(G(c)) = G(c)] \ge \Pr[G(c) = G'(c)],
$$
where the probabilities are taken over $c$ sampled uniformly at random from the set of all word $n$-rotation invariant columns.
\end{proposition}
\begin{proof}
Consider the following equation:
$$
\rr_n(G(c)) \approx \rr_n(G'(c)) = G'(\rr_n(c)) = G'(c) \approx G(c).
$$
The two approximations are applied to the same input: $G(c) \approx G'(c)$, therefore the equation holds with probability at least $\Pr[G(c) = G'(c)]$.
\end{proof}

Experimentally, no difference is observed on $\Pr[G(c) = G'(c)]$ when $c$ is sampled uniformly at random from $(\field{w})^4$ and when it is sampled uniformly at random from the set of all word $n$-rotation invariant columns. Therefore, it can be expected that a word $n$-rotation invariant is preserved through $F$ with a probability approximately $(2^{-1.82})^8 = 2^{-14.56}$. The commutation of $F$ and the $n$-rotation of each word can be expected to happen with probability approximately $(2^{-3.59})^8 = 2^{-28.72}$ if $1 < n < w-1$.

It is worth noting that the word $n$-rotation invariants can be seen as probabilistic invariant subspaces of $F$.

\subsection{Hypothetical Attack on NORX128 v2.0}

As the probability of $\rr_n$ commuting with $G'$ does not seem to depend on the word size, the distinguishing property is stronger for instances with larger words and key size. I consider an existential forgery attack similar to the one proposed in~\cite{NORXfse}. Similarly, I consider NORX v2.0 since NORX v3.0 breaks the attack by injecting the key in the finalization stage.

\newcommand\rrr{\mathbf{r}}
Consider the forgery attack scenario. The finalization stage of NORX consists of 8 iterations of $F$.
Let us assume that the words in the rate part of the state before the finalization are $w/2$-rotation invariant. This happens with probability $2^{-2w}$. Then we can attempt a forgery by rotating each word in the last ciphertext block by $w/2$. Then, with probability approximately $(2^{-3.59})^{64} = 2^{-229.76}$ we expect the rotation to commute with the finalization:
$$
F^8(\rrr_n(s)) = \rrr_n(F^8(s)),
$$
where $s$ is the state before the finalization stage. Since the tag is obtained by truncating the final state and we have observed the tag in the first encryption, we can expect the new tag to be equal to the word $w/2$-rotated version of the original tag.

For NORX64, the probability of the rate to be $32$-rotation invariant is equal to $2^{-128}$. Unfortunately, the attack's success probability then is worse than for a generic attack (i.e. $2^{-256}$). For this reason, I suggest to increase the word size even more and to consider NORX128, a generalization of NORX64 by increasing the word size to 128 bits. In this hypothetical cipher, the full attack success probability is approximately $2^{-256}\cdot 2^{-229.76} < 2^{-512}$, i.e. it is better than a generic attack.

This attack on the hypothetical instance of NORX shows the possibility of exploiting the word-level symmetries as well. The attack does not apply directly to main instances of NORX.


\Lecture{Jayalal Sharma}{Sept 19, 2020}{06}{Catlan Bijections}{Anshu and Narasimha Sai}{$\alpha$}{JS}

\section{Introduction}
One of the classic examples to demonstrate the power of bijections is \emph{Catlan numbers}. The Catlan numbers form a sequence of natural numbers that occur in various counting problems and occurs in several seemingly different contexts. Historically, \emph{Euler} is the first person to study them. He was interested in counting the number of ways of dividing a polygon into triangles by drawing non-overlapping diagonals. Catlan numbers got their name from \emph{Eugene Catlan} when he used them to answer the \emph{Parenthesisation problem} which is the following: Consider a sequence $(a_1,a_2,\cdots,a_{n+1})$ of $n+1$ numbers, If we have to perform a binary operations $\odot$ $n$ times among them, how many number of ways are there to parenthesise (or bracket) them using $n$ parenthesis of single type (say $'()'$). In this lecture, we will see a few equivalent problems to this and then arrive at an explicit expression of Catlan numbers.
\section{Equivalent Bijections}
In this section, we see a few equivalent problems of the \emph{parenthesisation} problem and argue that answer to each of them is also the \emph{catlan number}
\paragraph{Full binary trees} If we observe the Parenthesisation problem carefully, we notice that every valid parentesisation of those $n+1$ numbers form a \emph{full binary tree} (a binary tree in which every node have either two children or no children) of $n+1$ leaves and $n$ internal nodes where leaves represents the numbers $a_1,\cdots,a_{n+1}$ and each internal node corresponds to one operation. Therefore, there's an implicit bijection between the set of valid parenthesisations and full binary trees with $n$ internal nodes. Therefore,
\begin{equation}
    \substack{\textrm{number of valid parenthesisations of }\\ n+1 \textrm{ elements }}  = \substack{\textrm{number of full binary trees with }\\ n \textrm{ internal nodes}}
\end{equation}

\paragraph{Balanced parenthesised strings} A balanced parenthesised string of length $2n$ is a string consists of $n$ left brackets $'('$ and $n$ right brackets $')'$ in which every prefix of the string has number of left brackets $'('$ $\geq$ number of right brackets $')'$. One can easily observe the bijection from set of balanced paranthesised string to valid parenthesisations of $n+1$ numbers

\paragraph{Euler's problem} Find the number of ways of triangulating a polygon with $n+2$ edges

\paragraph{Handshaking problem} Consider a scenario where $2n$ people are sitting around a table. How many ways they can shake hands with each other without crossing hands. We leave it as an exercise to establish bijections from \emph{Euler's} problem to \emph{Full binary tree} problem and \emph{handshaking} problem to \emph{balanced parenthesised strings} problem.
\jsay{Establish bijections from \emph{Euler's} problem to \emph{Full binary tree} problem and \emph{handshaking} problem to \emph{balanced parenthesised strings} problem}

\section{Algebraic Expression}
In this section, we are interested in arriving at a concrete expression of the $n^{th}$ \emph{catlan number} (denoted by $c_n$). Let's solve another problem and then, by establishing a bijection to one of the above problems, we can arrive at an expression for $c_n$.

\subsection{Monotone walk on $n\times n$ grid} Suppose we have a grid of size $n\times n$. How many ways are there to go from $(0,0)$ to $(n,n)$ by using only downward edges or right edges. A sample path is represented in Fig. \ref{fig:sample-path}. We observe that each step can increment the value of exactly one of the co-ordinates by $1$. Since we have to move from $(0,0)$ to $(n,n)$, we have to increase the value of both the co-ordinates by $n$ and $n$ and thus irrespective of the path you take, the length of a path from $(0,0)$ to $(n,n)$ must be of length $n+n=2n$.

\begin{figure}[h!]
    \centering
    \includegraphics[width=0.4\linewidth]{images/sample-path.png}
    \caption{A path from $(0,0)$ to $(n,n)$ using downward and right edges}
    \label{fig:sample-path}
\end{figure}

If we represent each right move as $R$ and each downward move as $D$, one can observe that there's a bijection $f$ from the set of paths to set of strings of length $2n$ over the alphabet $\{D,R\}$ with number of $D$'s = number of $R$'s = $n$. Formally, if $(u_0,v_0), (u_1,v_1),\cdots,(u_{2n},v_{2n})$ represents the path where $(u_0,v_0)=(0,0)$ and $(u_{2n},v_{2n})=(n,n)$, and $b=b_1b_2\cdots b_{2n}$ represents the string where each $b_i$ is either $D$ or $R$, our bijection $f$ takes a path as input and sets $b_i$ as
$$b_i=\begin{cases}
D &\mbox{if } u_i = u_{i-1}+1\\
R &\mbox{if } v_i = v_{i-1}+1
\end{cases}$$
\begin{description}
\item \underline{Well defined:} As we have exactly $n$ $x$ co-ordinate increments and $n$ $y$ co-ordinate increments, we will have exactly $n$ $D$'s and $n$ $R$'s in our string and thus $f$ is well defined.
\item \underline{Injective:} Two different paths from $(0,0)$ to $(n,n)$ will different in at least one $(u_{i-1},v_{i-1})$ to $(u_i,v_i)$ transition where $i=1,2,\cdots,2n$, their corresponding strings under $f$ will differ in at least $i^{th}$ position and thus $f$ is injective.
\item \underline{Surjective:} Every string over $\{D,R\}$ of length $2n$ with equal number of $D$'s and $R$'s has a pre-image under $f$ which is defined by $(u_0,v_0)=(0,0)$ and $(u_i,v_i)$ is $(u_{i-1}+1,v_{i-1})$ if $b_i=R$ and $(u_{i-1},v_{i-1}+1)$ if $b_i=D$. As there will be $n$ $D$'s and $n$ $R$'s, $(u_{2n},v_{2n})=(n,n)$ and thus $f$ is surjective .
\end{description}
Thus $f$ is bijection. As we have number of string over $\{D,R\}$ of length $2n$ with equal number of $D$'s and $R$'s equal to $\binom{2n}{n}$ (select $n$ positions out of $2n$ available and fill them with $D$'s and the rest with $R$'s). Thus the number of paths from $(0,0)$ to $(n,n)$ with only downward and rightward movements is $\binom{2n}{n}$.

Lets ask a slightly question. How many ways are there to go from $(0,0)$ to $(n+1,n-1)$ using only downward or right edges.Using a similar arguments as above, we can come up with a bijection to set of string over $\{D,R\}$ of length $2n$ with $n+1$ $D$'s and $n-1$ $R$'s. Therefore number of required paths are $\binom{2n}{n+1}=\binom{2n}{n-1}$


\documentclass[12pt]{article}
\usepackage[utf8]{inputenc}
\usepackage{parskip}
\usepackage{markdown}
\usepackage{hyperref}
\usepackage{listings}
\usepackage{color}
\usepackage[subtle]{savetrees}
\usepackage{verbatim}
\usepackage{blindtext}

\title{\vspace{-2cm} \textbf{Session 5 - Some CS-ey things} \\ UCAS Program 2020}

\author{Chloe Lau}
\date{August 2020}

\begin{document}
\setlength{\parindent}{4ex}
\setlength{\parskip}{1em}

\maketitle

\section{Big O Notation}
\begin{center}
    \includegraphics[scale=0.2]{bigo.png}
\end{center}

The \href{https://en.wikipedia.org/wiki/Big_O_notation}{Big O Notation} ($\mathcal{O} \sim$) is used in Computer Science to describe the performance or complexity of an algorithm. Big O specifically describes the worst-case scenario, and can be used to describe the execution time required or the space used (e.g. in memory or on disk) by an algorithm.

As it is hard to visualise easily what the complexity of an algorithm is, below are some common orders of growth along with descriptions and examples where possible.

\section{Examples}

\subsection{$\mathcal{O}$(1)}
$\mathcal{O}$(1) describes an algorithm that will always execute in the same time (or space) regardless of the size of the input data set.

\begin{verbatim}
    bool IsFirstElementNull(IList<string> elements)
    {
        return elements[0] == null;
    }
\end{verbatim}

This is simple huh? Now let's get to other examples:

\subsection{$\mathcal{O}$(N)}
$\mathcal{O}$(N) describes an algorithm whose performance will grow linearly and in direct proportion to the size of the input data set. The example below also demonstrates how Big O favours the worst-case performance scenario; a matching string could be found during any iteration of the \texttt{for} loop and the function would return early, but Big O notation will always assume the upper limit where the algorithm will perform the maximum number of iterations.

\begin{verbatim}
    bool ContainsValue(IList<string> elements, string value)
    {
        foreach (var element in elements)
        {
            if (element == value) return true;
        }

    return false;
    }
\end{verbatim}

Still easy? Let's play with exponentials then!

\newpage
\subsection{$\mathcal{O}$($N^2$)}
$\mathcal{O}$($N^2$) represents an algorithm whose performance is directly proportional to the square of the size of the input data set. This is common with algorithms that involve nested iterations over the data set. Deeper nested iterations will result in $\mathcal{O}$($N^3$), $\mathcal{O}$($N^4$) etc.

\begin{verbatim}
    bool ContainsDuplicates(IList<string> elements)
    {
        for (var outer = 0; outer < elements.Count; outer++)
        {
            for (var inner = 0; inner < elements.Count; inner++)
            {
                // Don't compare with self
                if (outer == inner) continue;

                if (elements[outer] == elements[inner]) return true;
            }
        }

        return false;
    }
\end{verbatim}

Makes sense right? How about some Binary Powers:

\subsection{$\mathcal{O}$($2^N$)}
$\mathcal{O}$($2^N$) denotes an algorithm whose growth doubles with each addition to the input data set. The growth curve of an $\mathcal{O}$($2^N$) function is exponential - starting off very shallow, then rising meteorically. An example of an O(2N) function is the recursive calculation of Fibonacci numbers:

\begin{verbatim}
    int Fibonacci(int number)
    {
        if (number <= 1) return number;

        return Fibonacci(number - 2) + Fibonacci(number - 1);
    }
\end{verbatim}

Wait for the finale:

\subsection{Logarithms}
Logarithms are slightly trickier to explain so I'll use a common example:

\href{https://en.wikipedia.org/wiki/Binary_search_algorithm#Derivation_of_average_case}{Binary search} is a technique used to search sorted data sets. It works by selecting the middle element of the data set, essentially the median, and compares it against a target value. If the values match it will return success. If the target value is higher than the value of the probe element it will take the upper half of the data set and perform the same operation against it. Likewise, if the target value is lower than the value of the probe element it will perform the operation against the lower half. It will continue to halve the data set with each iteration until the value has been found or until it can no longer split the data set.

This type of algorithm is described as $\mathcal{O}$(log N). The iterative halving of data sets described in the binary search example produces a growth curve that peaks at the beginning and slowly flattens out as the size of the data sets increase e.g. an input data set containing 10 items takes one second to complete, a data set containing 100 items takes two seconds, and a data set containing 1000 items will take three seconds. Doubling the size of the input data set has little effect on its growth as after a single iteration of the algorithm the data set will be halved and therefore on a par with an input data set half the size. This makes algorithms like binary search extremely efficient when dealing with large data sets.

\end{document}


\documentclass[t,usenames,dvipsnames]{beamer}
\usetheme{Copenhagen}
\setbeamertemplate{headline}{} % remove toc from headers
\beamertemplatenavigationsymbolsempty

\usepackage{amsmath, tikz, xcolor, array, graphicx}
\usetikzlibrary{arrows.meta}
\tikzset{>=stealth}
\everymath{\displaystyle}

\title{Angles and Radian Measure}
\author{}
\date{}

\AtBeginSection[]
{
  \begin{frame}
    \frametitle{Table of Contents}
    \tableofcontents[currentsection]
  \end{frame}
}

\newcommand\bigangle[2][]{%
    \draw[->,domain=0:#2,variable=\t,samples=200,>=stealth,#1]
      plot ({(\t+#2)*cos(\t)/(4*#2)},
           {(\t+#2)*sin(\t)/(4*#2)})
        ;}

\begin{document}

\begin{frame}
    \titlepage
\end{frame}

\begin{frame}{Angles in Standard Position}
\begin{center}
\begin{tikzpicture}[scale=1.2]
    \draw [<->] (-3,0) -- (3,0) node [right] {$x$};
    \draw [<->] (0,-2) -- (0,3) node [right] {$y$};
    \draw [->, >=stealth, line width = 1.25, color=blue] (0,0) -- (2.75,0) node [below, midway] {\color{blue}Initial Side};
    \draw [->, >=stealth, line width = 1.25, color = red] (0,0) -- (140:2.75) node [midway, below, sloped] {\color{red}Terminal Side};
    \draw [color=violet, fill=violet] (0,0) circle (2pt);
    \draw [->, >=stealth, color=violet] (-0.5,-0.5) node [below] {\color{violet}Vertex} -- (0,0);
    \draw [->, >=stealth] (0:0.5) arc (0:140:0.5) node [midway, above right] {$\theta$};
\end{tikzpicture}
\end{center}
\end{frame}

\begin{frame}{Angles in Standard Position}
    An angle is in \alert{standard position} if    \newline\\  \pause
\begin{itemize}
    \item Vertex is at the origin.  \newline\\  \pause
    \item Initial side runs along positive $x$-axis.    \newline\\  \pause
\end{itemize}

Positive angles open \textbf{counter-clockwise} and negative angles open \textbf{clockwise}.    \newline\\  \pause

A \alert{quadrantal angle} is one whose terminal side lies on an axis. In other words, it's a multiple of $90^\circ$.
\end{frame}

\begin{frame}{Radians}
One \textbf{radian} is the measure of the central angle of a circle in which the radius equals the length of the intercepted arc.    \newline\\

\begin{center}
    \begin{tikzpicture}
    \draw (0,0) circle (3cm);
    \draw [color = blue, ultra thick] (0,0) -- (3,0) node [below, midway] {$r$};
    \draw [color = blue, ultra thick] (0,0) -- (57.3:3) node [sloped, midway, above] {$r$};
    \draw [line width = 1.5, color = blue, ultra thick] (0:3) arc (0:57.3:3) node [midway, right] {$r$};
    \draw [->, >=stealth] (0:0.75) arc (0:57.3:0.75) node [midway, right] {1 radian};
    \end{tikzpicture}
\end{center}
\end{frame}

\begin{frame}{Radian Interpretation}
In other words, its when the length of your slice of pizza's crust is equal to the radius of the pizza.
\end{frame}

\section{ Convert degrees to radians and radians to degrees.}

\begin{frame}{Relationship Between Radians and Degrees}
    One rotation is the circumference of the circle ($2\pi r$) and also $360^\circ$:

\begin{align*}
    \onslide<2->{360^\circ &= 2\pi \text{ radians}} \\[10pt]
    \onslide<3->{180^\circ &= \pi \text{ radians} } \\[10pt]
    \onslide<4->{\frac{180^\circ}{\pi} &= \text{1 radian}} \\
\end{align*}
\end{frame}

\begin{frame}{Convert Degrees to Radians and Radians to Degrees}
\begin{itemize}
    \item To convert degrees to radians, multiply by $\frac{\pi}{180^\circ}$.   \\[18pt]  \pause
    \item To convert radians to degrees, multiply by $\frac{180^\circ}{\pi}$.
\end{itemize}
\end{frame}

\begin{frame}{Example 1}
Convert each angle to radians.  \newline\\
(a) \quad $30^\circ$
\begin{align*}
    \onslide<2->{30^\circ &    }\onslide<3->{\rightarrow30^\circ\left(\frac{\pi}{180^\circ}\right)}   \\[12pt]
    \onslide<4->{&= \frac{30\pi}{180}} \\[12pt]
    \onslide<5->{&= \frac{\pi}{6}}
\end{align*}
\end{frame}

\begin{frame}{Example 1}
Convert each angle to radians.  \newline\\
(b) \quad $90^\circ$
\begin{align*}
    \onslide<2->{90^\circ &    }\onslide<3->{\rightarrow90^\circ\left(\frac{\pi}{180^\circ}\right)}   \\[12pt]
    \onslide<4->{&= \frac{90\pi}{180}} \\[12pt]
    \onslide<5->{&= \frac{\pi}{2}}
\end{align*}
\end{frame}

\begin{frame}{Example 1}
Convert each angle to radians.  \newline\\
(c) \quad $-135^\circ$
\begin{align*}
    \onslide<2->{-135^\circ &    }\onslide<3->{\rightarrow-135^\circ\left(\frac{\pi}{180^\circ}\right)}   \\[12pt]
    \onslide<4->{&= \frac{-135\pi}{180}} \\[12pt]
    \onslide<5->{&= \frac{-3\pi}{4}}
\end{align*}
\end{frame}

\begin{frame}{Example 2}
Convert each angle to degrees. \newline\\
(a) \quad $\frac{\pi}{3}$
\begin{align*}
    \onslide<2->{\frac{\pi}{3} &    }\onslide<3->{\rightarrow\frac{\pi}{3}\left(\frac{180^\circ}{\pi}\right)}   \\[12pt]
    \onslide<4->{&= \frac{180^\circ}{3}} \\[12pt]
    \onslide<5->{&= 60^\circ}
\end{align*}
\end{frame}

\begin{frame}{Example 2}
Convert each angle to degrees. \newline\\
(b) \quad $-\frac{5\pi}{4}$
\begin{align*}
    \onslide<2->{-\frac{5\pi}{4} &    }\onslide<3->{\rightarrow-\frac{5\pi}{4}\left(\frac{180^\circ}{\pi}\right)}   \\[12pt]
    \onslide<4->{&= -\frac{900^\circ}{4}} \\[12pt]
    \onslide<5->{&= -225^\circ}
\end{align*}
\end{frame}

\section{ Draw angles in standard position.}

\begin{frame}{Example 3}
Draw and label each angle in standard position. \newline\\
(a) \quad $\alpha = \frac{\pi}{6}$  \qquad \onslide<2->{$\alpha = 30^\circ$}
\begin{center}
\begin{tikzpicture}[scale=1.2]
    \draw [<->] (-2,0) -- (2,0) node [right] {$x$};
    \draw [<->] (0,-2) -- (0,2) node [above] {$y$};
    \draw [->, color=blue, very thick] (0,0) -- (1.5,0);
    \onslide<3->{\draw [->, color=red, very thick] (0,0) -- (30:1.5);}
    \onslide<4->{\draw [->] (0.5,0) arc (0:30:0.5) node [right, yshift=-0.1cm] {$\alpha$};}
\end{tikzpicture}
\end{center}
\end{frame}

\begin{frame}{Example 3}
(b) \quad $\beta = -\frac{4\pi}{3}$ \qquad \onslide<2->{$\beta = -240^\circ$}
\begin{center}
\begin{tikzpicture}[scale=1.2]
    \draw [<->] (-2,0) -- (2,0) node [right] {$x$};
    \draw [<->] (0,-2) -- (0,2) node [above] {$y$};
    \draw [->, color=blue, very thick] (0,0) -- (1.5,0);
    \onslide<3->{\draw [->, color=red, very thick] (0,0) -- (-240:1.5);}
    \onslide<4->{\draw [->] (0.5,0) arc (0:-240:0.5) node [below, xshift=-0.5cm, yshift=-0.75cm] {$\beta$};}
\end{tikzpicture}
\end{center}
\end{frame}

\begin{frame}{Example 3}
(c) \quad   $\gamma = - \frac{9\pi}{4}$ \qquad \onslide<2->{$\gamma = -405^\circ$}
\begin{center}
\begin{tikzpicture}[scale=1.2]
    \draw [<->] (-2,0) -- (2,0) node [right] {$x$};
    \draw [<->] (0,-2) -- (0,2) node [above] {$y$};
    \draw [->, color=blue, very thick] (0,0) -- (1.5,0);
    \onslide<3->{\draw [->, color=red, very thick] (0,0) -- (-405:1.5);}
    \onslide<4->{\bigangle{-405};}
    \onslide<4->{\node at (-0.5,0.5) {$\gamma$};}
\end{tikzpicture}
\end{center}
\end{frame}

\begin{frame}{Example 3}
(d) \quad   $\delta = -\frac{5\pi}{2}$  \qquad \onslide<2->{$\delta = -450^\circ$}
\begin{center}
\begin{tikzpicture}[scale=1.2]
    \draw [<->] (-2,0) -- (2,0) node [right] {$x$};
    \draw [<->] (0,-2) -- (0,2) node [above] {$y$};
    \draw [->, color=blue, very thick] (0,0) -- (1.5,0);
    \onslide<3->{\draw [->, color=red, very thick] (0,0) -- (-450:1.5);}
    \onslide<4->{\bigangle{-450};}
    \onslide<4->{\node at (-0.5,0.5) {$\delta$};}
\end{tikzpicture}
\end{center}
\end{frame}

\section{ Find a coterminal angle to a given angle.}

\begin{frame}{Coterminal Angles}
Two angles that have the same initial and terminal side are \alert{coterminal angles}.    \newline\\  \pause

To find coterminal angles, add (or subtract) multiples of $360^\circ$ (or $2\pi$ radians).
\end{frame}

\begin{frame}{Example 4}
Find a coterminal angle between $0^\circ$ and $360^\circ$ (or 0 and $2\pi$ radians) for each.   \newline\\
(a) \quad   $400^\circ$
\begin{align*}
    \onslide<2->{400^\circ}
    \onslide<3->{&\rightarrow 400^\circ - 360^\circ} \\[10pt]
    \onslide<4->{&= 40^\circ}
\end{align*}
\end{frame}

\begin{frame}{Example 4}
(b) \quad   $-\frac{4\pi}{3}$
\begin{align*}
    \onslide<2->{-\frac{4\pi}{3}}
    \onslide<3->{&\rightarrow -\frac{4\pi}{3} + 2\pi} \\[10pt]
    \onslide<4->{&= \frac{2\pi}{3}}
\end{align*}
\end{frame}

\begin{frame}{Example 4}
(c) \quad   $\frac{9\pi}{4}$
\begin{align*}
    \onslide<2->{\frac{9\pi}{4}}
    \onslide<3->{&\rightarrow \frac{9\pi}{4} - 2\pi} \\[10pt]
    \onslide<4->{&= \frac{\pi}{4}}
\end{align*}
\end{frame}

\begin{frame}{Example 4}
(d) \quad   $-785^\circ$
\begin{align*}
    \onslide<2->{-785^\circ}
    \onslide<3->{&\rightarrow -785^\circ + 1080^\circ} \\[10pt]
    \onslide<4->{&= 295^\circ}
\end{align*}
\end{frame}

\end{document}

   \newline\\

{\color{red}\textbf{Example 4.}}
\begin{enumerate}[(a)]
    \item
    \item
    \item
    \item
\end{enumerate}