%!TEX root = ../06DL.tex

\chapter{Stochastic Gradient Descent Methods and Its Variants}

\section{Fully Gradient Descent Method}
\begin{problem}Find ${x}^{*} \in \mathcal{X}$  s.t:
\begin{equation}\label{equ:sum-opt}
{x}^* = \mathop{\arg\min}_{{x} \in \mathcal{X} \subseteq \mathbb{R}^{n}} f(x).
\end{equation}
Here $\mathcal{X}$ is a convex compact subset. (This condition is used to prove the gradient is bounded, i.e. $\|\nabla f(x)\| \le M$ for SGD method. But in fully gradient descent method, $\mathcal{X}$ can be equal to $\mathbb{R}^n$.)
\end{problem}

\begin{assumption}\label{assum:GDconvergence}
Here $f$ satisfies:
\begin{itemize}
\item $\lambda$-strongly convex, i.e.
\begin{equation}\label{equ:lambdastronglyconvex}
f(x) - f(y) \ge \nabla f(y)^T(x-y) + \frac{\lambda}{2}\|x-y\|^2, \quad \forall x, y.
\end{equation}
\begin{itemize}
\item We can even assume that:
\begin{equation}\label{eq:sufficient_lambdastronglyconvex}
y^T\nabla^2 f(x)y \ge \lambda \|y\|^2, \quad \forall x, y,
\end{equation}
and \eqref{eq:sufficient_lambdastronglyconvex} can prove $\lambda$-strongly convex \eqref{equ:lambdastronglyconvex}.
\end{itemize}
\item $\nabla f$ Lipschitz continuous with constant $L$ i.e
\begin{equation}\label{eq:lipschtizcont}
\| \nabla f(x) - \nabla f(y)\| \le L \|x- y\|
\end{equation}
%Thus we will have:
%\begin{equation}
%f(x) - f(y) \le \nabla f(y)^T(x-y) + \frac{H}{2}\|x-y\|^2, \quad \forall x, y.
%\end{equation}
\end{itemize}
\end{assumption}

Here from the Lipschitz continuity of $\nabla f$ \eqref{eq:lipschtizcont}, we have:
\begin{lemma}
If $f \in C^1$ and $\| \nabla f(x) - \nabla f(y)\| \le L \|x- y\|$, then we have
\begin{equation}
f(x) - f(y) - \nabla f(y)^T(x-y) \le |f(x) - f(y) - \nabla f(y)^T(x-y)| \le \frac{L}{2}\|x- y\|^2.
\end{equation}
\end{lemma}
\begin{proof}
Take $g(t) = f(tx + (1-t)y)$, so we heve $g^\prime(t) = \nabla f(tx + (1-t)y)^\top (x- y)$ and
\begin{equation*}
f(x) - f(y) = \int_0^1 g^{\prime}(t) {\rm d}t .
\end{equation*}
Hence,
\begin{align*}
&|f(x) - f(y) - \nabla f(y)^T(x-y)| \\
= & \left |\int_0^1 \nabla f(t + (1-t)y)^T(x- y) - \nabla f(y)^T(x-y)\rm{dt}\right | \\
\le & \int_0^1 \left | \nabla (f(t + (1-t)y)^T- \nabla f(y)^T)(x-y)\right | \rm{dt} \\
\le & L\|x - y\|^2 \int_0^1 t \rm{dt} = \frac{L}{2}\|x-y\|^2.
\end{align*}
\end{proof}

So the algorithm is:
\begin{algorithm}
\caption{ FGD}
\label{alg:FGD}
\begin{equation}\label{equ:fgd-iteration}
x_{t+1} = \Pi_{\mathcal{X}}(x_{t} - \eta_t \nabla f(x_t)), \quad t = 0:T,
\end{equation}
where $\eta_t$ is the stepsize / learning rate.
\end{algorithm}

Here $\Pi_{\mathcal{X}}$ is the nonlinear projection operator of convex compact set $\mathcal{X} \subseteq \mathbb{R}^n$:
\begin{equation}
\Pi_{\mathcal{X}} w = \mathop{\arg\min}_{x \in \mathcal{X}} \|w - x\|.
\end{equation}

\subsection{Convergence of GD}
\begin{theorem}
If $\eta_t  = c $ is a small enough constant, then there exists $ 0< \alpha < 1 $ such that
\begin{equation}
\|x_t - x^*\|^2 \le  \alpha^t \|x_0 - x^*\|^2
\end{equation}
for Algorithm \ref{alg:FGD} .
\end{theorem}

\begin{proof}
%Then we have
If we minus any $x \in \mathcal{X}$, we can only get:
\begin{equation*}
x_{t+1} - x = \Pi_{\mathcal{X}}(x_{t} - \eta_t \nabla f(x_t)) - x.
\end{equation*}
If we take $L^2$ norm for both side, we get:
\begin{equation*}
\|x_{t+1} - x \|^2 = \| \Pi_{\mathcal{X}}(x_{t} - \eta_t \nabla f(x_t)) - x \|^2.
\end{equation*}
So we cannot get $(x_t - x)$ from the right side directly. But thanks to the definition of $\Pi_{\mathcal{X}}$ and $x \in \mathcal{X}$, we have:
\begin{equation*}
\| \Pi_{\mathcal{X}}(x_{t} - \eta_t \nabla f(x_t)) - x \|^2 \le \|x_{t} - \eta_t \nabla f(x_t) - x\|^2,
\end{equation*}
so we have the following inequality and take $x = x^*$:
\begin{align*}
 \|x_{t+1} - x^* \|^2 &\le  \| x_{t} - \eta_t \nabla f(x_t) - x^* \|^2 \\
 &= \|x_t-x^*\|^2 - 2\eta_t \nabla f(x_t)^\top (x_t - x^*) + \eta_t^2 \|\nabla f(x_t) - \nabla f(x^*)\|^2 \\
 &\le \|x_t - x^*\|^2 - \eta_t \lambda \|x_t - x^*\|^2 + \eta_t ^2 L^2 \|x_t - x^*\|^2  \\
 &\le (1 - 2\eta_t \lambda + \eta_t^2 L^2) \|x_t - x^*\|.
\end{align*}
So, if $\eta_t < 2\lambda/L^2$, then $\alpha = (1 - 2\eta_t \lambda + \eta_t^2 L^2) < 1$, so it has linear convergence rate.
\end{proof}

\section{Stochastic Gradient Descent Methods}
When we consider about the deep learning problems, they are almost like:
\begin{problem}Find ${x}^{*} \in \mathcal{X}$  s.t:
\begin{equation}\label{equ:sum-opt}
{x}^* = \mathop{\arg\min}_{{x} \in \mathcal{X} \subseteq \mathbb{R}^{n}} \frac{1}{m}\sum_{i=1}^m f_i(x).
\end{equation}
Here $\mathcal{X}$ is a convex compact subset. (This condition is used to prove the gradient is bounded, i.e. $\|\nabla f_i(x)\| \le M$.)
\end{problem}
The SGD algorithm is:
\begin{algorithm}\caption{SGD}
\label{alg:SGD}
\begin{equation}\label{equ:sgd-iteration}
x_{t+1} = \Pi_{\mathcal{X}}(x_{t} - \eta_t \nabla f_{i_t}(x_t)), \quad t = 0:T,
\end{equation}
\begin{equation}
\mathbb{P}(i_t = s) = \frac{1}{m}, \quad s = 1:m,
\end{equation}
\begin{equation}
i_1, \cdots, i_T \quad \text{are independent}.
\end{equation}
\end{algorithm}

Under the next assumptions, we can prove the convergence properties for SGD:
\begin{assumption}\label{assum:SGDconvergence}
Here $f_i$ satisfies:
\begin{itemize}
\item $\lambda$-strongly convex, i.e.
\begin{equation}\label{eq:sgdlambdastrcvx}
f_i(x) - f_i(y) \ge \nabla f_i(y)^\top (x-y) + \frac{\lambda}{2}\|x-y\|^2, \quad \forall x, y, i.
\end{equation}
\begin{itemize}
\item We can even assume that:
\begin{equation}\label{eq:sgdsufficientlambdastrcvx}
y^\top \nabla^2 f_i(x)y \ge \lambda \|y\|^2, \quad \forall x, y, i,
\end{equation}
and \eqref{eq:sgdsufficientlambdastrcvx} can prove $\lambda$-strongly convex \eqref{eq:sgdlambdastrcvx}.
\end{itemize}
\item Gradient is bounded, i.e. $\|\nabla f_i(x)\| \le M$.
\begin{itemize}
\item This condition can be proved by the projection $\Pi_{\mathcal{X}}$, otherwise this is generally not true for a convex function in $R^n$.
\end{itemize}
\end{itemize}
\end{assumption}

\subsection{Convergence of SGD}
\begin{theorem}Let the problem satisfy Assumptions \ref{assum:SGDconvergence}, and denote $e_t = \|x_t - x^*\|$,
then %we can prove by induction that
\begin{equation}
\mathbb{E}e_{t}^2 \le \frac{4M^2}{\lambda^2 t},
\end{equation}
if we take $\eta_t = \frac{1}{\lambda t}$.
\end{theorem}
\begin{proof}
The $L^2$ error of SDG can be written as
\begin{equation}
      \label{equ:L2SGD}
      \begin{split}
            \mathbb{E} \|x_{t+1} - x^*\|^2 &\le \mathbb{E}\| x_{t} - \eta_t \nabla f_{i_t}(x_t) - x^* \|^2 \\
            &\le \mathbb{E} \|x_t - x^*\|^2
            - 2 \eta_t \mathbb{E} (\nabla f_{i_t}(x_t) \cdot (x_t - x^*))
            + \eta_t^2 \mathbb{E} \|\nabla f_{i_t}(x_t)\|^2 \\
            & \le \mathbb{E} \|x_t - x^*\|^2 - 2 \eta_t \mathbb{E} (\nabla f (x_t) \cdot (x_t - x^*))
            + \eta_t^2 M^2 \\
            & \le \mathbb{E} \|x_t - x^*\|^2 -  \eta_t \lambda \mathbb{E} \|x_t - x^*\|^2 + \eta_t^2 M^2 \\
            & = (1 - \eta_t\lambda) \mathbb{E} \|x_t - x^*\|^2 + \eta_t^2 M^2
      \end{split}
\end{equation}
Here we can take the expectation of $i_t$ independently from $\{x_i, i = 1,\ldots,t\}$
to obtain the second line of (\ref{equ:L2SGD}) since $i_t$ is independent from
$\{x_i, i = 1,\ldots,t\}$ which is completely determined by $\{i_j, j = 1,\ldots,t - 1\}$.
And the third line of (\ref{equ:L2SGD}) is obtained from \eqref{eq:sgdlambdastrcvx}.

Note that from Assumption \ref{assum:SGDconvergence}%inequality (\ref{equ:lambdastronglyconvex})
\begin{equation}
\frac{\lambda}{2} \|x_{1} - x^*\| \le \|\nabla f_{i_1}(x_{1})\|
\end{equation}
which implies $ \mathbb{E}e_{1}^2 \le \frac{4M^2}{\lambda^2}$.

In the case of SDG, by the inductive hypothesis,
\begin{equation}
      \begin{split}
            \mathbb{E}e_{t+1}^2 & \le 4 (1 - \frac{2}{t}) \frac{M^2}{\lambda^2 t} + \frac{M^2}{\lambda^2 t^2} \\
            & \le \frac{M^2}{\lambda^2} \frac{1}{t}(4 - \frac{8}{t} + \frac{1}{t}) \\
            & \le \frac{M^2}{\lambda^2} \frac{4}{t+1}.
      \end{split}
\end{equation}
\end{proof}

\section{Incremental Gradient Descent and Practical Training Method in
	Large-Scale Machine Learning Problems}

\subsection{Basic Gradient Descent Type Algorithms}
Now we suppose we have a machine learning model
\begin{equation}
f(x; w),
\end{equation}
where $f$ can be any machine learning model like: linear regression, SVM or deep learning models.

We have data as
$$
z_i=\{x_i, y_i\}\quad i = 1:N,
$$
and the loss between the label and prediction is:
$$
f_i(w) = l(f(x_i, w), y_i),
$$
one example of $l$ is:
$$
l(f(x_i, w), y_i) = \|f(x_i, w)- y_i\|^2.
$$
What we want to solve is:
\begin{equation}
\mathop{\min}_{{w} \in \mathbb{R}^{n}} \frac{1}{N}\sum_{i=1}N f_i(w),
\end{equation}
with $F =  \frac{1}{m}\sum_{i=1}^m f_i(w)$.

\subsubsection{Mini-Batch Method}
Now we introduce the Mini-Batch algorithm which is mostly used basic algorithm for above problem:
\begin{algorithm}[H]
\caption{Mini-Batch}
\label{alg:mini-batch}
{\bf Input}: learning rate $\eta_t$, batch size $m$, parameter Initialization $ w_0$, number of epochs $K$. \\
For iteration $t = 1: K\frac{N}{m}$ \\
For Epoch $k = 1:K$\\
Shuffle data and get mini-batch $B_1, \cdots, B_{\frac{N}{m}}$, choose mini-batch as: $B_{i_t}$ with
$$
i_t \equiv t \mod(\frac{N}{m}),
$$
Compute the gradient on $B_{i_t}$:
$$
g_t = \nabla_{w} \frac{1}{m} \sum_{i \in B_{i_t}} f_i(w_{t})
$$
Update $w$:
\begin{equation}
w_{t+1} = w_t - \eta_t g_t.
\end{equation}
\end{algorithm}

This method is a little different from pure SGD type method as we still have
$$
\mathbb{E} g_t = \nabla f,
$$
only the choice of $\nabla f_i$ and $\nabla f_{i+1}$ are dependent.
But this method works very good with some suitable batch-size. Recently, many researches show that small batch-size may lead to some flat minimizer with better generalization error. And some research suggested that, you can use a small batch size at the beginning and increase the batch size slowly with iteration.

\subsubsection{Momentum Mini-Batch}
Then we are going to introduce the momentum method to accelerate the gradient method.
{\bf Without special statement, the general setup for those next algorithms are the same to Mini-Batch}

\begin{algorithm}[H]
\caption{Momentum Mini-Batch}
\label{alg:mom}
{\bf Input}: momentum coefficient $\alpha$(we often choose as: 0.5, 0.9, 0.999) \\
Compute the gradient on $B_{i_t}$:
$$
g_t = \nabla_{w} \frac{1}{m} \sum_{i \in B_{i_t}} f_i(w_{t}).
$$
Compute the momentum:
\begin{equation}
v_t = \alpha v_{t-1} - \eta_t g_t \quad (v_0 = 0).
\end{equation}
Update $w$:
\begin{equation}
w_{t+1} = w_t + v_t.
\end{equation}
\end{algorithm}

\subsubsection{Nesterov Momentum Mini-Batch Method}
We may also use the Nesterov accelerated skill and get the Nesterov momentum Mini-Batch method.
\begin{algorithm}[H]
\caption{Nesterov Momentum Mini-Batch}
\label{alg:Nesterov}
{\bf Input}: momentum coefficient $\alpha$\\
Compute the gradient on $B_{i_t}$:
$$
g_t = \nabla_{w} \frac{1}{m} \sum_{i \in B_{i_t}} f_i(w_{t} + \alpha v_{t-1}) \quad (v_0 = 0).
$$
Compute the momentum:
\begin{equation}
v_t = \alpha v_{t-1} - \eta_t g_t.
\end{equation}
Update $w$:
\begin{equation}
w_{t+1} = w_t + v_t.
\end{equation}
\end{algorithm}

\subsection{Adaptive Learning Rate Mini-batch Based Method}
The delta-bar-delta algorithm (Jacobs, 1988) is an early heuristic approach to adapting individual learning rates for model parameters during training. The approach is based on a simple idea: if the partial derivative of the loss, with respect to a given model parameter, remains the same sign, then the learning rate should increase. If the partial derivative with respect to that parameter changes sign, then the learning rate should decrease. Of course, this kind of rule can only be applied to full batch optimization. More recently, a number of incremental (or mini-batch-based) methods have been introduced that adapt the learning rates of model parameters.

\subsubsection{AdaGrad}
The AdaGrad algorithm, individually adapts the learning rates of all model parameters by scaling them inversely proportional to the square root of the sum of all of their historical squared values (Duchi et al., 2011).
\begin{algorithm}[H]
\caption{AdaGrad}
\label{alg:AdaGrad}
{\bf Input}: small constant $\delta$(perhaps $10^{-7}$, for numerical stability)\\
Compute the gradient on $B_{i_t}$:
$$
g_t = \nabla_{w} \frac{1}{m} \sum_{i \in B_{i_t}} f_i(w_{t}).
$$
Accumulate squared gradient:
\begin{equation}
r_t = r_{t-1} + g_t \otimes g_t. \quad (r_0 = 0),
\end{equation}
with $\otimes$ means multiplication element-wise.  \\
Compute Update:
\begin{equation}
\Delta w_t = -\frac{\eta_t}{\delta + \sqrt{r_t}} \otimes g_t,
\end{equation}
with division and square root applied element-wise. \\
Update $w$:
\begin{equation}
w_{t+1} = w_t + \Delta w_t.
\end{equation}
\end{algorithm}

\subsubsection{RMSProp}
The RMSProp algorithm (Hinton, 2012) modifies AdaGrad to perform better in the non-convex setting by changing the gradient accumulation into an exponentially weighted moving average.

\begin{algorithm}[H]
\caption{RMSProp}
\label{alg:RMSProp}
{\bf Input}: decay rate $\rho$, small constant $\delta$(usually $10^{-6}$)\\
Compute the gradient on $B_{i_t}$:
\begin{equation}
g_t = \nabla_{w} \frac{1}{m} \sum_{i \in B_{i_t}} f_i(w_{t}).
\end{equation}
Accumulate squared gradient:
\begin{equation}
r_t = \rho r_{t-1} + (1-\rho)g_t \otimes g_t. \quad (r_0 = 0),
\end{equation}
Compute update:
\begin{equation}
\Delta w_t = -\frac{\eta_t}{\sqrt{\delta + {r_t}}} \otimes g_t
\end{equation}
Update $w$:
\begin{equation}
w_{t+1} = w_t + \Delta w_t.
\end{equation}
\end{algorithm}

And if we combine RMSProp  with Nesterov momentum, we can get:

\begin{algorithm}[H]
\caption{RMSProp with Nesterov momentum}
\label{alg:RMSPropNesterov}
{\bf Input}: decay rate $\rho$, momentum coefficient $\alpha$\\
Compute the gradient on $B_{i_t}$:
\begin{equation}
g_t = \nabla_{w} \frac{1}{m} \sum_{i \in B_{i_t}} f_i(w_{t} + \alpha v_{t-1}), \quad (v_0 = 0).
\end{equation}
Accumulate squared gradient:
\begin{equation}
r_t = \rho r_{t-1} + (1-\rho)g_t \otimes g_t. \quad (r_0 = 0),
\end{equation}
Compute velocity update:
\begin{equation}
v_t = \alpha v_{t-1} -\frac{\eta_t}{\sqrt{{r_t}}} \otimes g_t
\end{equation}
Update $w$:
\begin{equation}
w_{t+1} = w_t + v_t.
\end{equation}
\end{algorithm}

\subsubsection{Adam}
Adam (Kingma and Ba, 2014) is yet another adaptive learning rate optimization algorithm, and its name ``Adam'' derives from the phrase \emph{Adaptive Moment Estimation}, which adapt both learning rate $\eta$ and subgradient $g$.
Use moving average of both $g$ and $g^2$ (here $g^2$ indicates the element-wise square $g \bigodot g$) to compute first order moment vector
\begin{equation}
m_t  =\beta_1 m_{t-1} + (1-\beta_1) g_t=(1-\beta_1)\sum_{i=1}^{t}\beta_1^{t-i} g_i ,
\end{equation}
and second order moment vector
\begin{equation}
v_t  =\beta_2 v_{t-1} + (1-\beta_2) g_t^2=(1-\beta_2)\sum_{i=1}^{t}\beta_2^{t-i} g^2_i.
\end{equation}
Here $\beta_1 = 0.9$, $\beta_2 = 0.99$.

Since
\begin{align}
\mathbb{E} \left[ m_t \right] & =\mathbb{E} \left[ g_t \right] \cdot (1-\beta_1^t) +\zeta\\
\mathbb{E} \left[ v_t \right] & =\mathbb{E} \left[ g^2_t \right] \cdot (1-\beta_2^t) +\zeta.
\end{align}
Here $\zeta=0$ if the true moment is stationary.
So, we use a bias-corrected version here
\begin{align}
\tilde{m}_t=\frac{m_t}{1-\beta_1^t} \\
\tilde{v}_t=\frac{v_t}{1-\beta_2^t}.
\end{align}

The iterate can be written as
\begin{equation}\label{Adam}
w_{t+1}=w_t-\frac{\eta}{\sqrt{\tilde{v}_t}+\delta} \tilde{m}_t.
\end{equation}

\begin{algorithm}[H]
\caption{Adam}
\label{alg:Adam}
{\bf Input}: small constant $\delta$ (can be $10^{-8}$), decay rate for moment estimates $\rho_1$ and $\rho_2$ (suggested defaults: 0.9 and 0.999 respectively)\\
Compute the gradient on $B_{i_t}$:
\begin{equation}
g_t = \nabla_{w} \frac{1}{m} \sum_{i \in B_{i_t}} f_i(w_{t}).
\end{equation}
Update biased first moment estimate:
\begin{equation}
m_t = \rho_1 m_{t-1} + (1-\rho_1)g_t, \quad (m_0 = 0).
\end{equation}
Update biased second moment estimate:
\begin{equation}
v_t = \rho_2 v_{t-1} + (1-\rho_2)g_t \otimes g_t, \quad (v_0 = 0).
\end{equation}
Correct bias in first moment:
\begin{equation}
\tilde m_t = \frac{m_t}{1 - \rho_1^{t}}.
\end{equation}
Correct bias in second moment:
\begin{equation}
\tilde v_t = \frac{v_t}{1 - \rho_2^{t}}.
\end{equation}
Compute update:
\begin{equation}
\Delta w_t =  -\frac{\eta_t}{\sqrt{\tilde v_t} + \delta} \otimes \tilde m_t
\end{equation}
Update $w$:
\begin{equation}
w_{t+1} = w_t + \Delta w_t.
\end{equation}
\end{algorithm}

Now we apply Adam for the following SPD linear system
\begin{equation}
	A u-b=0
\end{equation}
where $A \in \mathbb{R}^{n \times n}$ is \emph{symmetric positive definite} matrix and $b \in \mathbb{R}^n$.
It is equal to solve the following minimization problem
\begin{equation}
	\min_u\ \left\{f(u):= \frac{1}{2} u^T Au - b^T u\right\}.
\end{equation}

Assume that we didn't take stochastic technique into consideration, then
\begin{equation}
g_k=A u_k-b.
\end{equation}
Compute first order moment vector
\begin{align}
m_k & = \beta_1 m_{k-1} +(1-\beta_1) g_k\\
&= \beta_1 m_{k-1} + (1-\beta_1) (A u_k - b)
\end{align}
The update can be written as
\begin{equation}
u_{k+1}=u_k-\eta P^{-1}_k R_k
\end{equation}
where
\begin{align}
R_k & =\frac{1-\beta_1}{1-\beta_1^k} \sum_{i=1}^{k} \beta_1^{k-i} (A u_i -b)\\
& = \frac{(1-\beta_1) \beta_1^{(k-1)}}{1-\beta_1^k} (Au_1 -b) + \frac{(1-\beta_1) \beta_1^{(k-2)}}{1-\beta_1^k} (Au_2 -b) +\cdots \\
& +\frac{(1-\beta_1) \beta_1^{0}}{1-\beta_1^k} (Au_k -b) \\
& =c_1 (Au_1-b) + c_2(Au_2-b) + \cdots + c_k (Au_k-b).
\end{align}
Here
\begin{equation}
c_i = \frac{(1-\beta_1) \beta_1^{(k-i)}}{1-\beta_1^k}
\end{equation}
is increase geometrical sequence.
And we can easily draw the conclusion that
\begin{equation}
\sum_{i=1}^{k} c_i =1.
\end{equation}
So $R_k$ is weighted average of $\left\{u_1, u_2, \dots, u_k\right\}$.

$P_k$ is a diagonal matrix
\begin{align}
P_k & =\text{diag} \left\{
\sqrt{ \frac{1-\beta_2}{1-\beta_2^k} \sum_{i=1}^{k} \beta_2^{k-i} (A u_i -b)^2 }+\epsilon
\right\}\\
& = \text{diag} \left\{ p_1, p_2, \cdots, p_n \right\},
\end{align}
where $(Au_i -b)^2$ is element-wise product.
$p_i^2$ is weighted average of $(A u_i-b)^2$.

To simplify our analysis, we will use $A u_k - b$ instead of $R_k$, as well as
\begin{equation}
	P_k=\text{diag} \left\{ |A u_k-b| \right\}
\end{equation}

Assume $u^*$ is the solution of $Au-b=0$.
So
\begin{equation}
u^* = u^* + \eta P^{-1} (A u^* -b)
\end{equation}
And
\begin{equation}
u^* -u_k = (I- \eta P_k^{-1} A) (u^* - u_k)
\end{equation}
We need estimate the upper bound of $\|I-\eta P^{-1}_k A\|$.

In other way, we will find the relation of ADAM and SDA.

\subsection{Choosing the Right Optimization Algorithm}
Schaul et al. (2014) presented a valuable comparison of a large number of optimization algorithms across a wide range of learning tasks. While the results suggest that the family of algorithms with adaptive learning rates (represented by RMSProp and Adam) performed fairly robustly, no single best algorithm has emerged.

Currently, the most popular optimization algorithms actively in use include SGD, SGD with momentum, RMSProp, RMSProp with momentum, AdaDelta and Adam. The choice of which algorithm to use, at this point, seems to depend largely on the user's familiarity with the algorithm (for ease of hyperparameter tuning).


\subsection{Definitions}
\label{subsec:formalisations:groove_formalisation:definitions}
This section discusses some definitions specific to the GROOVE formalisation. The definitions need to be in place before the formalisations of the different GROOVE graphs are given.

GROOVE internally uses a set of labels $Lab$ for each defined grammar. These labels are used by multiple graph types of GROOVE, including (but not limited to) type graphs and instance graphs.

\begin{defin}[Labels]
\label{defin:formalisations:groove_formalisation:definitions:labels}
$Lab$ is the set of labels used by GROOVE graphs. It can be subdivided into three sets:
\begin{itemize}
    \item The set of type labels $Lab_t \subseteq Lab$
    \item The set of edge labels $Lab_e \subseteq Lab$
    \item The set of flag labels $Lab_f \subseteq Lab$
\end{itemize}

The intersection of each of the sets $Lab_t$, $Lab_f$ and $Lab_e$ has to be empty:
\begin{equation*}
    Lab_t \cap Lab_f = \emptyset \land Lab_t \cap Lab_e = \emptyset \land Lab_f \cap Lab_e = \emptyset
\end{equation*}

\isabellelref{Lab}{GROOVE.Type_Graph}
\end{defin}

The set $Lab_t$ will be used to denote the types of nodes in the graph, while the $Lab_e$ set will be used to distinguish between different edges. The $Lab_f$ set are labels for particular kind of edges which always have an identical source and target node. These are used as flags on nodes to indicate that a specific property holds for a node.

Although the intersection of each of the sets $Lab_t$, $Lab_f$ and $Lab_e$ has to be empty, the sets do not necessarily form a partition of $Lab$, as one or more of the subsets can be empty. Furthermore, $Lab$ itself can be empty if the GROOVE grammar does not define any types, flags or edge labels.

Besides $Lab$ that belongs to a grammar, GROOVE uses a set of reserved primitive type labels $Lab_{prim}$ that can never be part of the label set of a grammar.

\begin{defin}[Primitive type labels]
\label{defin:formalisations:groove_formalisation:definitions:primitive_type_labels}
GROOVE has a set of reserved primitive type labels $Lab_{prim}$:
\begin{equation*}
    Lab_{prim} = \{ \type{bool}, \type{int}, \type{real}, \type{string} \}
\end{equation*}

It should hold that $Lab \cap Lab_{prim} = \emptyset$ since the primitive type labels are reserved.
\end{defin}

The primitive type labels allow the use of primitive types as attributes and values. The label $\type{bool}$ represents the type for boolean values $\mathbb{B}$, $\type{int}$ represents the type for integer values $\mathbb{Z}$, $\type{real}$ represents the type for real numbers $\mathbb{R}$ and finally $\type{string}$ represents the type for string values $\mathbb{S}$.


\documentclass[12pt]{cdblatex}
\usepackage{examples}
\usepackage{fancyhdr}
\usepackage{footer}

\begin{document}

% ============================================================================================
\bgroup
\CdbSetup{action=hide}
\begin{cadabra}
   import cdblib
   checkpoint_file = 'tests/semantic/output/example-09.json'
   cdblib.create (checkpoint_file)
   checkpoint = []
\end{cadabra}
\egroup

\clearpage

% ============================================================================================
\section*{Example 9 The Gauss equation}

\begin{cadabra}
   {a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u#}::Indices(position=independent).

   \nabla{#}::Derivative.

   K_{a b}::Symmetric.
   g^{a}_{b}::KroneckerDelta.

   # define the projection operator

   hab:=h^{a}_{b} -> g^{a}_{b} - n^{a} n_{b}.

   # 3-covariant derivative obtained by projection on 4-covariant derivative

   vpq:=v_{p q} -> h^{a}_{p} h^{b}_{q} \nabla_{b}{v_{a}}.

   # compute 3-curvature by commutation of covariant derivatives

   vpqr:= h^{a}_{p} h^{b}_{q} h^{c}_{r} ( \nabla_{c}{v_{a b}} - \nabla_{b}{v_{a c}} ).

   substitute (vpq,hab)
   substitute (vpqr,vpq)

   distribute   (vpqr)
   product_rule (vpqr)
   distribute   (vpqr)
   eliminate_kronecker (vpqr)

   # standard substitutions

   substitute (vpqr,$h^{a}_{b} n^{b} -> 0$)
   substitute (vpqr,$h^{a}_{b} n_{a} -> 0$)
   substitute (vpqr,$\nabla_{a}{g^{b}_{c}} -> 0$)
   substitute (vpqr,$n^{a} \nabla_{b}{v_{a}} -> -v_{a} \nabla_{b}{n^{a}}$)
   substitute (vpqr,$v_{a} \nabla_{b}{n^{a}} -> v_{p} h^{p}_{a}\nabla_{b}{n^{a}}$)
   substitute (vpqr,$h^{p}_{a} h^{q}_{b} \nabla_{p}{n_{q}} -> K_{a b}$)
   substitute (vpqr,$h^{p}_{a} h^{q}_{b} \nabla_{p}{n^{b}} -> K_{a}^{q}$)  # cdb(ex-09.095,vpqr)

   # tidy up

   {v_{a},h^{a}_{b},K_{a}^{b},K_{a b},R^{a}_{b c d},\nabla_{a}{v_{b}}}::SortOrder.

   sort_product   (vpqr)                                     # cdb(ex-09.096,vpqr)
   rename_dummies (vpqr)                                     # cdb(ex-09.097,vpqr)
   canonicalise   (vpqr)                                     # cdb(ex-09.098,vpqr)
   factor_out     (vpqr,$h^{a?}_{b?}$)                       # cdb(ex-09.099,vpqr)
   factor_out     (vpqr,$v_{a?}$)                            # cdb(ex-09.101,vpqr)

   checkpoint.append (vpqr)
\end{cadabra}

\begin{dgroup*}[spread={3pt}]
   \Dmath*{( D_{r}D_{q} - D_{q}D_{r} ) v_{p} = \Cdb*[\hskip 1.0cm\hfill]{ex-09.095}
                                             = \Cdb*[\hskip 1.0cm\hfill]{ex-09.096}
                                             = \Cdb*[\hskip 1.0cm\hfill]{ex-09.097}
                                             = \Cdb*{ex-09.098}
                                             = \Cdb*{ex-09.099}
                                             = \Cdb*{ex-09.101}}
\end{dgroup*}

\clearpage

\begin{cadabra}
   R{#}::LaTeXForm("{{\strut}^g R}").

   gRabcd := \nabla_{c}{\nabla_{b}{v_{a}}}
            -\nabla_{b}{\nabla_{c}{v_{a}}} -> R^{d}_{a b c} v_{d}.

   substitute     (vpqr,gRabcd)                              # cdb(ex-09.102,vpqr)
   distribute     (vpqr)                                     # cdb(ex-09.103,vpqr)
   substitute     (vpqr,$v_{a} -> h^{b}_{a} v_{b}$)          # cdb(ex-09.104,vpqr)
   substitute     (vpqr,$h^{b}_{a} K_{c}^{a} -> K_{c}^{b}$)  # cdb(ex-09.105,vpqr)
   sort_product   (vpqr)                                     # cdb(ex-09.106,vpqr)
   rename_dummies (vpqr)                                     # cdb(ex-09.107,vpqr)
   canonicalise   (vpqr)                                     # cdb(ex-09.108,vpqr)
   factor_out     (vpqr,$v_{a?}$)                            # cdb(ex-09.109,vpqr)
   substitute     (vpqr,$v_{a}->1$)                          # cdb(ex-09.110,vpqr)
   sort_product   (vpqr)                                     # cdb(ex-09.111,vpqr)

   checkpoint.append (vpqr)
\end{cadabra}

\begin{dgroup*}[spread={3pt}]
   \Dmath*{( D_{r}D_{q} - D_{q}D_{r} ) v_{p} = \Cdb*{ex-09.101}
                                             = \Cdb*{ex-09.102}
                                             = \Cdb*{ex-09.103}
                                             = \Cdb*{ex-09.104}
                                             = \Cdb*{ex-09.105}
                                             = \Cdb*{ex-09.106}
                                             = \Cdb*{ex-09.107}
                                             = \Cdb*{ex-09.108}
                                             = \Cdb*{ex-09.109}}
\end{dgroup*}

\begin{dgroup*}[spread={3pt}]
   \Dmath*{{\strut}^hR^{a}{}_{pqr} = \Cdb*{ex-09.110}}
   \Dmath*{{\strut}^hR^{a}{}_{pqr} = \Cdb*{ex-09.111}}
\end{dgroup*}

\clearpage

% ============================================================================================
% export to json format

\bgroup
\CdbSetup{action=hide}
\begin{cadabra}
   for i in range( len(checkpoint) ):
      cdblib.put ('check{:03d}'.format(i),checkpoint[i],checkpoint_file)
\end{cadabra}
\egroup

\end{document}


% !TEX root = hott_intro.tex

\section{Equivalences}

\subsection{Homotopies}
In homotopy type theory, a homotopy is just a pointwise equality between two functions $f$ and $g$.

\begin{defn}
Let $f,g:\prd{x:A}P(x)$ be two dependent functions. The type of \define{homotopies}\index{homotopy|textbf} from $f$ to $g$ is defined as
\begin{equation*}
f\htpy g \defeq \prd{x:A} f(x)=g(x).
\end{equation*}
\end{defn}

Since we formulated homotopies using dependent functions, we may also consider homotopies \emph{between}\index{homotopy!iterated} homotopies, and further homotopies between those higher homotopies.
Explicitly, if $H,K:f\htpy g$, then the type $H\htpy K$ of homotopies is just the type
\begin{equation*}
\prd{x:A} H(x)=K(x).
\end{equation*}

In the following definition we define the groupoid-like structure of homotopies. Note that we implement the groupoid-laws as \emph{homotopies} rather than as identifications.

\begin{defn}\label{defn:htpy_groupoid}\index{groupoid laws!of homotopies|textbf}
For any dependent type $B:A\to\type$ there are operations
\begin{align*}
& \mathsf{htpy\usc{}refl} & & : \prd{f:\prd{x:A}B(x)}f\htpy f \\
& \mathsf{htpy\usc{}inv} & & : \prd*{f,g:\prd{x:A}B(x)} (f\htpy g)\to(g\htpy f) \\
& \mathsf{htpy\usc{}concat} & & : \prd*{f,g,h:\prd{x:A}B(x)} (f\htpy g)\to ((g\htpy h)\to (f\htpy h)).
\end{align*}
We will write $H^{-1}$ for $\mathsf{htpy\usc{}inv}(H)$, and $\ct{H}{K}$ for $\mathsf{htpy\usc{}concat}(H,K)$.

Furthermore, we define
\begin{align*}
& \mathsf{htpy\usc{}assoc}(H,K,L) & & : \ct{(\ct{H}{K})}{L}\htpy\ct{H}{(\ct{K}{L})} \\
& \mathsf{htpy\usc{}left\usc{}unit}(H) & & : \ct{\mathsf{htpy\usc{}refl}_f}{H}\htpy H \\
& \mathsf{htpy\usc{}right\usc{}unit}(H) & & : \ct{H}{\mathsf{htpy\usc{}refl}_g}\htpy H \\
& \mathsf{htpy\usc{}left\usc{}inv}(H) & & : \ct{H^{-1}}{H} \htpy \mathsf{htpy\usc{}refl}_g \\
& \mathsf{htpy\usc{}right\usc{}inv}(H) & & : \ct{H}{H^{-1}} \htpy \mathsf{htpy\usc{}refl}_f
\end{align*}
for any $H:f\htpy g$, $K:g\htpy h$ and $L:h\htpy i$, where $f,g,h,i:\prd{x:A}B(x)$.
\end{defn}

\begin{constr}
We define
\begin{align*}
\mathsf{htpy\usc{}refl}(f) & \defeq \lam{x} \refl{f(x)} \\
\mathsf{htpy\usc{}inv}(H) & \defeq \lam{x} H(x)^{-1} \\
\mathsf{htpy\usc{}concat}(H,K) & \defeq \lam{x}\ct{H(x)}{K(x)},
\end{align*}
where $H:f\htpy g$ and $K:g\htpy h$ are homotopies. Furthermore, we define
\begin{align*}
\mathsf{htpy\usc{}assoc}(H,K,L) & \defeq \lam{x}\mathsf{assoc}(H(x),K(x),L(x)) \\
\mathsf{htpy\usc{}left\usc{}unit}(H) & \defeq \lam{x}\mathsf{left\usc{}unit}(H(x)) \\
\mathsf{htpy\usc{}right\usc{}unit}(H) & \defeq \lam{x}\mathsf{right\usc{}unit}(H(x)) \\
\mathsf{htpy\usc{}left\usc{}inv}(H) & \defeq \lam{x}\mathsf{left\usc{}inv}(H(x)) \\
\mathsf{htpy\usc{}right\usc{}inv}(H) & \defeq \lam{x}\mathsf{right\usc{}inv}(H(x)).\qedhere
\end{align*}
\end{constr}

Apart from the groupoid operations and their laws, we will occasionally need \emph{whiskering} operations.

\begin{defn}
We define the following \define{whiskering}\index{homotopy!whiskering operations|textbf}\index{whiskering operations!of homotopies|textbf} operations on homotopies:
\begin{enumerate}
\item Suppose $H:f\htpy g$ for two functions $f,g:A\to B$, and let $h:B\to C$. We define
\begin{equation*}
h\cdot H\defeq \lam{x}\ap{h}{H(x)}:h\circ f\htpy h\circ g.
\end{equation*}
\item Suppose $f:A\to B$ and $H:g\htpy h$ for two functions $g,h:B\to C$. We define
\begin{equation*}
H\cdot f\defeq\lam{x}H(f(x)):h\circ f\htpy g\circ f.
\end{equation*}
\end{enumerate}
\end{defn}

We also use homotopies to express the commutativity of diagrams. For example, we say that a triangle
\begin{equation*}
  \begin{tikzcd}[column sep=tiny]
    A \arrow[rr,"h"] \arrow[dr,swap,"f"] & & B \arrow[dl,"g"] \\
    & X
  \end{tikzcd}
\end{equation*}
commutes if it comes equipped with a homotopy $H:f\htpy g\circ h$, and we say that a square
\begin{equation*}
  \begin{tikzcd}
    A \arrow[r,"g"] \arrow[d,swap,"f"] & A' \arrow[d,"{f'}"] \\
    B \arrow[r,swap,"h"] & B'
  \end{tikzcd}
\end{equation*}
if it comes equipped with a homotopy $h \circ f~g\circ f'$.

\subsection{Bi-invertible maps}
\begin{defn}
Let $f:A\to B$ be a function. We say that $f$ has a \define{section}\index{section!of a map|textbf} if there is a term of type\index{sec(f)@{$\mathsf{sec}(f)$}|textbf}
\begin{equation*}
\mathsf{sec}(f) \defeq \sm{g:B\to A} f\circ g\htpy \idfunc[B].
\end{equation*}
Dually, we say that $f$ has a \define{retraction}\index{retraction} if there is a term of type\index{retr(f)@{$\mathsf{retr}(f)$}|textbf}
\begin{equation*}
\mathsf{retr}(f) \defeq \sm{h:B\to A} h\circ f\htpy \idfunc[A].
\end{equation*}
If a map $f:A \to B$ has a retraction, we also say that $A$ is a \define{retract}\index{retract!of a type} of $B$.
We say that a function $f:A\to B$ is an \define{equivalence}\index{equivalence|textbf}\index{bi-invertible map|see {equivalence}} if it has both a section and a retraction, i.e., if it comes equipped with a term of type\index{is_equiv@{$\isequiv$}|textbf}
\begin{equation*}
\isequiv(f)\defeq\mathsf{sec}(f)\times\mathsf{retr}(f).
\end{equation*}
We will write $\eqv{A}{B}$\index{equiv@{$\eqv{A}{B}$}|textbf} for the type $\sm{f:A\to B}\isequiv(f)$.
\end{defn}

\begin{rmk}
An equivalence, as we defined it here, can be thought of as a \emph{bi-invertible} map, since it comes equipped with a separate left and right inverse. Explicitly, if $f$ is an equivalence, then there are
\begin{align*}
g & : B\to A & h & : B\to A \\
G & : f\circ g \htpy \idfunc[B] & H & : h\circ f \htpy \idfunc[A].
\end{align*}
Clearly, if $f$ is \define{invertible}\index{invertible map} in the sense that it comes equipped with a function $g:B\to A$ such that $f\circ g\htpy\idfunc[B]$ and $g\circ f\htpy\idfunc[A]$, then $f$ is an equivalence. We write\index{is_invertible@{$\mathsf{is\usc{}invertible}$}|textbf}
\begin{equation*}
\mathsf{has\usc{}inverse}(f)\defeq\sm{g:B\to A} (f\circ g\htpy \idfunc[B])\times (g\circ f\htpy\idfunc[A]).
\end{equation*}
\end{rmk}

\begin{defn}\label{defn:inv_equiv}
Any equivalence can be given the structure of an invertible map.\index{equivalence!invertibility of}
\end{defn}

\begin{constr}
First we construct for any equivalence $f$ with right inverse $g$ and left inverse $h$ a homotopy $K:g\htpy h$. For any $y:B$, we have
\begin{equation*}
\begin{tikzcd}[column sep=huge]
g(y) \arrow[r,equals,"H(g(y))^{-1}"] & hfg(y) \arrow[r,equals,"\ap{h}{G(y)}"] & h(y).
\end{tikzcd}
\end{equation*}
Therefore we define a homotopy $K:g\htpy h$ by $K\defeq \ct{(H\cdot g)^{-1}}{h\cdot G}$.
Using the homotopy $K$ we are able to show that $g$ is also a left inverse of $f$. For $x:A$ we have the identification
\begin{equation*}
\begin{tikzcd}[column sep=large]
gf(x) \arrow[r,equals,"K(f(x))"] & hf(x) \arrow[r,equals,"H(x)"] & x.
\end{tikzcd}\qedhere
\end{equation*}
\end{constr}

\begin{cor}
The inverse of an equivalence is again an equivalence.
\end{cor}

\begin{proof}
Let $f:A\to B$ be an equivalence. By \cref{defn:inv_equiv} it follows that the section of $f$ is also a retraction. Therefore it follows that the section is itself an invertible map, with inverse $f$. Hence it is an equivalence.
\end{proof}

\begin{thm}\label{thm:id_equiv}
For any type $A$, the identity function $\idfunc[A]$ is an equivalence.\index{identity function!is an equivalence|textit}
\end{thm}

\begin{proof}
The identity function is trivially its own section and its own retraction.
\end{proof}

\begin{eg}
  For any type $C(x,y)$ indexed by $x:A$ and $y:B$, the function
\begin{equation*}
\sigma:\Big(\prd{x:A}{y:B}C(x,y)\Big)\to\Big(\prd{y:B}{x:A}C(x,y)\Big)
\end{equation*}
that swaps the order of the arguments $x$ and $y$ is an equivalence by \cref{ex:swap}.\index{swap function!is an equivalence|textit}
\end{eg}

\subsection{The identity type of a \texorpdfstring{$\Sigma$-}{dependent pair }type}

In this section we characterize the identity type of a $\Sigma$-type as a $\Sigma$-type of identity types. In this course we will be characterizing the identity types of many types, so we will follow the general outline of how such a characterization goes:
\begin{enumerate}
\item First we define a binary relation $R:A\to A\to \UU$ on the type $A$ that we are interested in. This binary relation is intended to be equivalent to its identity type.
\item Then we will show that this binary relation is reflexive, by constructing a term of type
  \begin{equation*}
    \prd{x:A}R(x,x)
  \end{equation*}
\item Using the reflexivity we will show that there is a canonical map
  \begin{equation*}
    (x=y)\to R(x,y)
  \end{equation*}
  for every $x,y:A$. This map is just constructed by path induction, using the reflexivity of $R$.
\item Finally, it has to be shown that the map
  \begin{equation*}
    (x=y)\to R(x,y)
  \end{equation*}
  is an equivalence for each $x,y:A$.
\end{enumerate}
The last step is usually the most difficult, and we will refine our methods for this step in \cref{chap:fundamental}, where we establish the Fundamental Theorem of Identity Types.

In this section we consider a type family $B$ over $A$. Given two pairs
\begin{equation*}
  (x,y),(x',y'):\sm{x:A}B(x),
\end{equation*}
if we have a path $\alpha:x=x'$ then we can compare $y:B(x)$ to $y':B(x')$ by first transporting $y$ along $\alpha$, i.e., we consider the identity type
\begin{equation*}
  \mathsf{tr}_B(\alpha,y)=y'.
\end{equation*}
Thus it makes sense to think of $(x,y)$ to be identical to $(x',y')$ if there is an identification $\alpha:x=x'$ and an identification $\beta:\mathsf{tr}_B(\alpha,y)=y'$. In the following definition we turn this idea into a binary relation on the $\Sigma$-type.

\begin{defn}
  We will define a relation
  \begin{equation*}
    \mathsf{Eq}_{\Sigma} : \Big(\sm{x:A}B(x)\Big)\to\Big(\sm{x:A}B(x)\Big)\to\UU
  \end{equation*}
  by defining
  \begin{equation*}
    \mathsf{Eq}_{\Sigma}(s,t)\defeq\sm{\alpha:\proj 1(s)=\proj 1(t)}\mathsf{tr}_B(\alpha,\proj 2(s))=\proj 2 (t).
  \end{equation*}
\end{defn}

\begin{lem}
  The relation $\mathsf{Eq}_{\Sigma}$ is reflexive, i.e., there is a term
  \begin{equation*}
    \mathsf{reflexive\usc{}Eq}_{\Sigma}:\prd{s:\sm{x:A}B(x)}\mathsf{Eq}_{\Sigma}(s,s).
  \end{equation*}
\end{lem}

\begin{constr}
  This term is constructed by $\Sigma$-induction on $s:\sm{x:A}B(x)$. Thus, it suffices to construct a term of type
  \begin{equation*}
    \prd{x:A}{y:B(x)}\sm{\alpha:x=x}\mathsf{tr}_B(\alpha,y)=y.
  \end{equation*}
  Here we take $\lam{x}{y}(\refl{x},\refl{y})$.
\end{constr}

\begin{defn}
  Consider a type family $B$ over $A$. Then for any $s,t:\sm{x:A}B(x)$ we define a map\index{pair_eq@{$\mathsf{pair\usc{}eq}$}|textbf}
  \begin{equation*}
    \mathsf{pair\usc{}eq}: (s=t)\to \mathsf{Eq}_\Sigma(s,t)
  \end{equation*}
  by path induction, taking $\mathsf{pair\usc{}eq}(\refl{s})\defeq\mathsf{reflexive\usc{}Eq}_\Sigma(s)$.
\end{defn}

\begin{thm}\label{thm:eq_sigma}
  Let $B$ be a type family over $A$. Then the map
  \begin{equation*}
    \mathsf{pair\usc{}eq}: (s=t)\to \mathsf{Eq}_\Sigma(s,t)
  \end{equation*}
  is an equivalence for every $s,t:\sm{x:A}B(x)$.\index{Sigma type@{$\Sigma$-type}!identity types of|textit}\index{identity type!of a Sigma-type@{of a $\Sigma$-type}|textit}
\end{thm}

\begin{proof}
The maps in the converse direction\index{eq_pair@{$\mathsf{eq\usc{}pair}$}}
\begin{equation*}
\mathsf{eq\usc{}pair} : \mathsf{Eq}_\Sigma(s,t)\to(\id{s}{t})
\end{equation*}
are defined by repeated $\Sigma$-induction. By $\Sigma$-induction on $s$ and $t$  we see that it suffices to define a map
\begin{equation*}
\mathsf{eq\usc{}pair} : \Big(\sm{p:x=x'}\id{\mathsf{tr}_B(p,y)}{y'}\Big)\to(\id{(x,y)}{(x',y')}).
\end{equation*}
A map of this type is again defined by $\Sigma$-induction. Thus it suffices to define a dependent function of type
\begin{equation*}
\prd{p:x=x'} (\id{\mathsf{tr}_B(p,y)}{y'}) \to (\id{(x,y)}{(x',y')}).
\end{equation*}
Such a dependent function is defined by double path induction by sending $\pairr{\refl{x},\refl{y}}$ to $\refl{\pairr{x,y}}$. This completes the definition of the function $\mathsf{eq\usc{}pair}$.

Next, we must show that $\mathsf{eq\usc{}pair}$ is a section of $\mathsf{pair\usc{}eq}$. In other words, we must construct an identification
\begin{equation*}
\mathsf{pair\usc{}eq}(\mathsf{eq\usc{}pair}(\alpha,\beta))=\pairr{\alpha,\beta}
\end{equation*}
for each $\pairr{\alpha,\beta}:\sm{\alpha:x=x'}\id{\mathsf{tr}_B(\alpha,y)}{y'}$. We proceed by path induction on $\alpha$, followed by path induction on $\beta$. Then our goal becomes to construct a term of type
\begin{equation*}
\mathsf{pair\usc{}eq}(\mathsf{eq\usc{}pair}\pairr{\refl{x},\refl{y}})=\pairr{\refl{x},\refl{y}}
\end{equation*}
By the definition of $\mathsf{eq\usc{}pair}$ we have $\mathsf{eq\usc{}pair}\pairr{\refl{x},\refl{y}}\jdeq \refl{\pairr{x,y}}$, and by the definition of $\mathsf{pair\usc{}eq}$ we have $\mathsf{pair\usc{}eq}(\refl{\pairr{x,y}})\jdeq\pairr{\refl{x},\refl{y}}$. Thus we may take $\refl{\pairr{\refl{x},\refl{y}}}$ to complete the construction of the homotopy $\mathsf{pair\usc{}eq}\circ\mathsf{eq\usc{}pair}\htpy\idfunc$.

To complete the proof, we must show that $\mathsf{eq\usc{}pair}$ is a retraction of $\mathsf{pair\usc{}eq}$. In other words, we must construct an identification
\begin{equation*}
\mathsf{eq\usc{}pair}(\mathsf{pair\usc{}eq}(p))=p
\end{equation*}
for each $p:s=t$. We proceed by path induction on $p:s=t$, so it suffices to construct an identification
\begin{equation*}
\mathsf{eq\usc{}pair}\pairr{\refl{\proj 1(s)},\refl{\proj 2(s)}}=\refl{s}.
\end{equation*}
Now we proceed by $\Sigma$-induction on $s:\sm{x:A}B(x)$, so it suffices to construct an identification
\begin{equation*}
\mathsf{eq\usc{}pair}\pairr{\refl{x},\refl{y}}=\refl{(x,y)}.
\end{equation*}
Since $\mathsf{eq\usc{}pair}\pairr{\refl{x},\refl{y}}$ computes to $\refl{(x,y)}$, we may simply take $\refl{\refl{(x,y)}}$.
\end{proof}

\begin{exercises}
%  \item Show that for any term $a:A$ the functions
%    \begin{align*}
%      \ind{\unit}(a) & : \unit \to A \\
%      \mathsf{const}_a & : \unit \to A
%    \end{align*}
%    are homotopic.
%  \item Let $A$ and $B$ be types, and consider the constant map $\mathsf{const}_b:A\to B$ for some $b:B$. Construct a homotopy
%    \begin{equation*}
%      \mathsf{ap}_{\mathsf{const}_b}(x,y)\htpy \mathsf{const}_{\refl{b}}
%    \end{equation*}
%    for any $x,y:A$.
\item \label{ex:equiv_grpd_ops}Show that the functions
  \begin{align*}
    \mathsf{inv} & :(\id{x}{y})\to(\id{y}{x}) \\
    \mathsf{concat}(p) & : (\id{y}{z})\to(\id{x}{z}) \\
    \mathsf{concat'}(q) & : (\id{x}{y}) \to (\id{x}{z}) \\
    \mathsf{tr}_B(p) & :B(x)\to B(y)
  \end{align*}
  are equivalences, where $\mathsf{concat'}(q,p)\defeq \ct{p}{q}$\index{concat'@{$\mathsf{concat'}$}}. Give their inverses explicitly.
\item Show that the maps
  \begin{align*}
    \inl & : X \to X+\emptyt &     \proj 1 & : \emptyt \times X \to \emptyt \\
    \inr & : X \to \emptyt+X &    \proj 2 & : X \times \emptyt \to \emptyt
  \end{align*}
  are equivalences.
\item
  \begin{subexenum}
  \item \label{ex:htpy_equiv}\index{equivalence!homotopic maps} Consider two functions $f,g:A\to B$ and a homotopy $H:f\htpy g$. Then
    \begin{equation*}
      \isequiv(f)\leftrightarrow\isequiv(g).
    \end{equation*}
  \item Show that for any two homotopic equivalences $e,e':\eqv{A}{B}$, their inverses are also homotopic.
  \end{subexenum}
\item \label{ex:3_for_2}\index{equivalence!three@{3-for-2 property}}\index{3-for-2 property!of equivalences}
  Consider a commuting triangle
  \begin{equation*}
    \begin{tikzcd}[column sep=tiny]
      A \arrow[rr,"h"] \arrow[dr,swap,"f"] & & B \arrow[dl,"g"] \\
      & X.
    \end{tikzcd}
  \end{equation*}
  with $H:f\htpy g\circ h$.
  \begin{subexenum}
  \item Suppose that the map $h$ has a section $s:B \to A$. Show that the triangle
    \begin{equation*}
      \begin{tikzcd}[column sep=tiny]
        B \arrow[rr,"s"] \arrow[dr,swap,"g"] & & A \arrow[dl,"f"] \\
        & X.
      \end{tikzcd}
    \end{equation*}
    commutes, and that $f$ has a section if and only if $g$ has a section.
  \item Suppose that the map $g$ has a retraction $r:X\to B$. Show that the triangle
    \begin{equation*}
      \begin{tikzcd}[column sep=tiny]
        A \arrow[rr,"f"] \arrow[dr,swap,"h"] & & X \arrow[dl,"r"] \\
        & B.
      \end{tikzcd}
    \end{equation*}
    commutes, and that $f$ has a retraction if and only if $h$ has a retraction.
  \item (The \define{3-for-2 property} for equivalences.) Show that if any two of the functions
    \begin{equation*}
      f,\qquad g,\qquad h
    \end{equation*}
    are equivalences, then so is the third.
  \end{subexenum}
\item \label{ex:neg_equiv}
  \begin{subexenum}
  \item Define the negation function on the booleans, and show that it is an equivalence.\index{negation function!is an equivalence}
  \item Use the observational equality on the booleans, defined in \cref{ex:obs_bool}, to show that $\btrue\neq\bfalse$.
  \item Show that for any $b:\bool$, the constant function $\mathsf{const}_b$ is not an equivalence.
  \end{subexenum}
\item \label{ex:succ_equiv} Show that the successor function on the integers is an equivalence.\index{successor function!on Z@{on $\Z$}!is an equivalence}
\item \label{ex:comm_prod}Construct a equivalences $\eqv{A+B}{B+A}$ and $\eqv{A\times B}{B\times A}$.\index{coproduct!is symmetric}
\item \label{ex:retr_id} Consider a section-retraction pair
  \begin{equation*}
    \begin{tikzcd}
      A \arrow[r,"i"] & B \arrow[r,"r"] & A,
    \end{tikzcd}
  \end{equation*}
  with $H:r\circ i\htpy \idfunc$. Show that $\id{x}{y}$ is a retract of $\id{i(x)}{i(y)}$.\index{retract!identity types of}
\item \label{ex:sigma_assoc}\index{Sigma type@{$\Sigma$-type}!associativity of}Let $B$ be a family of types over $A$, and let $C$ be a family of types indexed by $x:A,y:B(x)$. Construct an equivalence
  \begin{equation*}
    \Sigma\mathsf{\usc{}assoc}:\eqv{\Big(\sm{p:\sm{x:A}B(x)}C(\proj 1(p),\proj 2(p))\Big)}{\Big(\sm{x:A}\sm{y:B(x)}C(x,y)\Big)}.
  \end{equation*}
\item \label{ex:sigma_swap}Let $A$ and $B$ be types, and let $C$ be a family over $x:A,y:B$. Construct an equivalence
  \begin{equation*}
    \Sigma\mathsf{\usc{}swap}:\eqv{\Big(\sm{x:A}{y:B}C(x,y)\Big)}{\Big(\sm{y:B}{x:A}C(x,y)\Big)}.
  \end{equation*}
  %\item \label{ex:sigma_base_equiv}Consider an equivalence $e:A'\eqv A$ and a type family $B$ over $A$. Show that the map
  %\begin{equation*}
  %\lam{(x',y)}(e(x'),y) : \Big(\sm{x':A'}B(e(x'))\Big)\to\Big(\sm{x:A}B(x)\Big)
  %\end{equation*}
  %is an equivalence.
\item \label{ex:int_group_laws}\index{Z@{$\Z$}!group laws} In this exercise we will show that the laws for abelian groups hold for addition on the integers. Note: these are obvious facts, but the proof terms that show \emph{how} the group laws hold are nevertheless fairly involved. This exercise is perfect for a formalization project.
  \begin{subexenum}
  \item Show that addition satisfies the left and right unit laws, i.e., construct terms
    \begin{align*}
      \mathsf{left\usc{}unit\usc{}law\usc{}add\usc{}}\Z  & : \prd{x:\Z} 0 + x = x \\
      \mathsf{right\usc{}unit\usc{}law\usc{}add\usc{}}\Z  & : \prd{x : \Z} x + 0 = x.
    \end{align*}
  \item Show that addition respects predecessors and successor on both sides, i.e., construct terms
    \begin{align*}
      \mathsf{left\usc{}predecessor\usc{}law\usc{}add\usc{}}\Z & : \prd{x,y:\Z} \mathsf{pred}_{\mathbb{Z}}(x)+y = \mathsf{pred}_{\mathbb{Z}}(x+y) \\
      \mathsf{right\usc{}predecessor\usc{}law\usc{}add\usc{}}\Z & : \prd{x,y:\Z} x+\mathsf{pred}_{\mathbb{Z}}(y) = \mathsf{pred}_{\mathbb{Z}}(x+y) \\
      \mathsf{left\usc{}successor\usc{}law\usc{}add\usc{}}\Z & : \prd{x,y:\Z} \mathsf{succ}_{\mathbb{Z}}(x)+y = \mathsf{succ}_{\mathbb{Z}}(x+y) \\
      \mathsf{right\usc{}successor\usc{}law\usc{}add\usc{}}\Z & : \prd{x,y:\Z} x+\mathsf{succ}_{\mathbb{Z}}(y) = \mathsf{succ}_{\mathbb{Z}}(x+y).
    \end{align*}
    Hint: to avoid an excessive number of cases, use induction on $x$ but not on $y$. You may need to use the homotopies $\mathsf{succ}_{\mathbb{Z}}\circ \mathsf{pred}_{\mathbb{Z}}\htpy \idfunc$ and $\mathsf{pred}_{\mathbb{Z}}\circ\mathsf{succ}_{\mathbb{Z}}$ constructed in exercise \cref{ex:succ_equiv}.
  \item Use part (b) to show that addition on the integers is associative and commutative, i.e., construct terms
    \begin{align*}
      \mathsf{assoc\usc{}add\usc{}}\Z & : \prd{x,y,z:\Z} (x+y)+z = x + (y+z) \\
      \mathsf{comm\usc{}add\usc{}}\Z & : \prd{x,y:\Z} x+y = y+x.
    \end{align*}
    Hint: Especially in the construction of the associator there is a risk of running into an unwieldy amount of cases if you use $\Z$-induction on all arguments. Avoid induction on $y$ and $z$.
  \item Show that addition satisfies the left and right inverse laws:
    \begin{align*}
      \mathsf{left\usc{}inverse\usc{}law\usc{}add\usc{}}\Z & : \prd{x : \Z} (-x)+x=0 \\
      \mathsf{right\usc{}inverse\usc{}law\usc{}add\usc{}}\Z & : \prd{x : \Z} x + (-x)=0.
    \end{align*}
    Conclude that the functions $y \mapsto x + y$ and $x\mapsto x + y$ are equivalences for any $x:\Z$ and $y:\Z$, respectively.
  \end{subexenum}
\item \label{ex:coproduct_functor}In this exercise we will construct the \define{functorial action} of coproducts.
  \begin{subexenum}
  \item Construct for any two maps $f:A \to A'$ and $g:B \to B'$, a map
    \begin{equation*}
      f+g:A+B \to A'+B'.
    \end{equation*}
  \item Show that if $H:f\htpy f'$ and $K:g\htpy g'$, then there is a homotopy
    \begin{equation*}
      H+K:(f+g)\htpy (f'+g').
    \end{equation*}
  \item Show that $\idfunc[A]+\idfunc[B]\htpy \idfunc[A+B]$.
  \item Show that for any $f:A\to A'$, $f':A'\to A''$, $g:B\to B'$, and $g':B'\to B''$ there is a homotopy
    \begin{equation*}
      (f'\circ f)+(g'\circ g) \htpy (f'+g')\circ (f\circ g).
    \end{equation*}
  \item \label{ex:coproduct_functor_equivalence}Show that if $f$ and $g$ are equivalences, then so is $f+g$. (The converse of this statement also holds, see \cref{ex:is-equiv-is-equiv-functor-coprod}.)
  \end{subexenum}
\item Construct equivalences
  \begin{align*}
    \mathsf{Fin}(m+n) & \simeq \mathsf{Fin}(m)+\mathsf{Fin}(n) \\
    \mathsf{Fin}(mn) & \simeq \mathsf{Fin}(m)\times\mathsf{Fin}(n).
  \end{align*}
\end{exercises}


\section{A Silly Idea}

    \frame{\sectionpage}

    \begin{frame}{Ordinary Differential Equations}
        \uncover<+->{\begin{equation*}
            \dv{x} y(x) + \frac{1}{CR} y(x) = 0
        \end{equation*}}

        \uncover<+->{\begin{equation*}
            \dv[2]{x} y(x) + \gamma \dv{x} y(x) + \omega_0^2 y(x) = f(x)
        \end{equation*}}
    \end{frame}

    \begin{frame}{Livin' La Vida Loca}
        \uncover<+>{\begin{equation*}
            \dv[2]{x} y(x) + \gamma \dv{x} y(x) + \omega_0^2 y(x) = f(x)
        \end{equation*}}
        \uncover<+>{\[ \Downarrow \]
        \begin{equation*}
            \colch{\dv[2]{x} + \gamma \dv{x} + \omega_0^2} y(x) = f(x)
        \end{equation*}}
        \uncover<+>{\[ \Downarrow \]
        \begin{equation*}
            y(x) = \frac{f(x)}{\dv[2]{x} + \gamma \dv{x} + \omega_0^2}
        \end{equation*}}
    \end{frame}

    \begin{frame}{Livin' La Vida Loca}
        \centering
        \includegraphics[width = 0.8\textwidth]{images/coke.jpg}
    \end{frame}

    \begin{frame}{Pandora's Box}
        \centering
        \includegraphics[width = 0.9\textwidth]{images/fourier-1.pdf}
    \end{frame}

    \begin{frame}{Pandora's Box}
        \centering
        \includegraphics[width = 0.9\textwidth]{images/fourier-2.pdf}
    \end{frame}

    \begin{frame}{Pandora's Box}
        \centering
        \includegraphics[width = 0.9\textwidth]{images/fourier-3.pdf}
    \end{frame}

    \begin{frame}{Pandora's Box}
        \centering
        \includegraphics[width = 0.9\textwidth]{images/fourier-4.pdf}
    \end{frame}

    \begin{frame}{Pandora's Box}
        \centering
        \includegraphics[height = 0.7\textheight]{images/fourier-5.pdf}
    \end{frame}

    \begin{frame}{Box Proposal}
        \uncover<+>{\begin{equation*}
            \mathcal{F}[f](\xi) = \hat{f}(\xi) = \frac{1}{\sqrt{2\pi}} \int_{-\infty}^{+\infty} f(x) e^{-i x \xi} \dd{x}
        \end{equation*}}
        \uncover<+>{\begin{equation*}
            \mathcal{F}^{-1}[\hat{f}](x) = f(x) = \frac{1}{\sqrt{2\pi}} \int_{-\infty}^{+\infty} \hat{f}(\xi) e^{i x \xi} \dd{\xi}
        \end{equation*}}
    \end{frame}

    \begin{frame}{Quality Control}
        \uncover<+>{\begin{equation*}
            \widehat{(f + \alpha g)}(\xi) = \frac{1}{\sqrt{2\pi}} \int_{-\infty}^{+\infty} \prnt{f(x) + \alpha g(x)} e^{-i x \xi} \dd{x}
        \end{equation*}}
        \uncover<+>{\[ \Downarrow \]
        \begin{equation*}
            \widehat{(f + \alpha g)}(\xi) = \frac{1}{\sqrt{2\pi}} \int_{-\infty}^{+\infty} f(x) e^{-i x \xi} \dd{x} + \frac{\alpha}{\sqrt{2\pi}} \int_{-\infty}^{+\infty} g(x) e^{-i x \xi} \dd{x}
        \end{equation*}}
        \uncover<+>{\[ \Downarrow \]
        \begin{equation*}
            \widehat{(f + \alpha g)}(\xi) = \hat{f}(\xi) + \alpha \hat{g}(\xi)
        \end{equation*}}
    \end{frame}

    \begin{frame}{Quality Control}
        \uncover<+>{\begin{equation*}
            \widehat{f'}(\xi) = \frac{1}{\sqrt{2\pi}} \int_{-\infty}^{+\infty} f'(x) e^{-i x \xi} \dd{x}
        \end{equation*}}
        \uncover<+>{\[ \Downarrow \]
        \begin{equation*}
            \widehat{f'}(\xi) = \eval{\frac{f(x) e^{-ix\xi}}{\sqrt{2\pi}}}^{+\infty}_{-\infty} + i\xi \cdot \frac{1}{\sqrt{2\pi}} \int_{-\infty}^{+\infty} f(x) e^{-i x \xi} \dd{x}
        \end{equation*}}
        \uncover<+>{\[ \Downarrow \]
        \begin{equation*}
            \widehat{f'}(\xi) = i\xi \widehat{f}(\xi)
        \end{equation*}}
    \end{frame}

    \begin{frame}{Quality Control}
        \centering
        \huge{The inverse does work}

        \normalsize{for appropriate functions}

        \tiny{and, sometimes, the Fourier Transform of a function is not in the same set as the original function, but let's forget about this since we do not know a decent theory of integration}
    \end{frame}

200+ beamer 模板合集/DeadPhysicistsSocietyPresentationTemplate(DPS 研讨会)/chapters/a-silly-idea.tex


Due to the fact that  two of the  studied classes of systems that are studied in this paper are affine functions in terms of $f$ and $g$, we propose to solve the "one--step nonsmooth problem'' (\ref{eq:toto1}) by performing an external Newton linearization.

 \paragraph{Newton's linearization of the first line of~(\ref{eq:toto1})} The first line of the  problem~(\ref{eq:toto1}) can be written under the form of a residue $\mathcal R$ depending only on $x_{k+1}$ and $r_{k+1}$ such that
\begin{equation}
  \label{eq:NL3}
  \mathcal R (x_{k+1},r _{k+1}) =0
\end{equation}
with
\begin{equation}
\mathcal R(x,r) = M(x - x_{k}) -h\theta f( x , t_{k+1}) - h(1-\theta)f(x_k,t_k) - h\gamma r
- h(1-\gamma)r_k.
\end{equation}
The solution of this system of nonlinear equations is sought as a limit of the sequence $\{ x^{\alpha}_{k+1},r^{\alpha}_{k+1} \}_{\alpha \in \NN}$ such that
 \begin{equation}
   \label{eq:NL7}
   \begin{cases}
     x^{0}_{k+1} = x_k \\ \\
     r^{0}_{k+1} = r_k \\ \\
     \mathcal R_L( x^{\alpha+1}_{k+1},r^{\alpha+1}_{k+1}) = \mathcal
     R(x^{\alpha}_{k+1},r^{\alpha}_{k+1})  + \left[ \nabla_{x} \mathcal
     R(x^{\alpha}_{k+1},r^{\alpha}_{k+1})\right] (x^{\alpha+1}_{k+1}-x^{\alpha}_{k+1} ) +
     \left[ \nabla_{r} \mathcal R(x^{\alpha}_{k+1},r^{\alpha}_{k+1})\right] (r^{\alpha+1}_{k+1} - r^{\alpha}_{k+1} ) =0
 \end{cases}
\end{equation}
\begin{ndrva}
  What about $r^0_{k+1}$ ?
\end{ndrva}

The residu \free $\mathcal R _{\free}$ is also defined (useful for implementation only):
\[\mathcal R _{\free}(x) \stackrel{\Delta}{=}  M(x - x_{k}) -h\theta f( x , t_{k+1}) - h(1-\theta)f(x_k,t_k),\]
which yields
\[\mathcal R (x,r) = \mathcal R _{\free}(x)   - h\gamma r - h(1-\gamma)r_k.\]

\begin{equation}
  \mathcal R (x^{\alpha}_{k+1},r^{\alpha}_{k+1}) = \fbox{$\mathcal R^{\alpha}_{k+1} \stackrel{\Delta}{=}  \mathcal R
_{\free}(x^{\alpha}_{k+1})  - h\gamma r^{\alpha}_{k+1} - h(1-\gamma)r_k$}\label{eq:rfree-1}
\end{equation}

\[  \mathcal R
_{\free}(x^{\alpha}_{k+1},r^{\alpha}_{k+1} )=\fbox{$ \mathcal R _{\free, k+1} ^{\alpha} \stackrel{\Delta}{=}  M(x^{\alpha}_{k+1} - x_{k}) -h\theta f( x^{\alpha}_{k+1} , t_{k+1}) - h(1-\theta)f(x_k,t_k)$}\]

% The computation of the Jacobian of $\mathcal R$ with respect to $x$, denoted by $   W^{\alpha}_{k+1}$ leads to
% \begin{equation}
%    \label{eq:NL9}
%    \begin{array}{l}
%     W^{\alpha}_{k+1} \stackrel{\Delta}{=} \nabla_{x} \mathcal R (x^{\alpha}_{k+1},r^{\alpha}_{k+1})= M - h  \theta \nabla_{x} f(  x^{\alpha}_{k+1}, t_{k+1} ).\\
%  \end{array}
% \end{equation}
At each time--step, we have to solve the following linearized problem,
\begin{equation}
   \label{eq:NL10}
    \mathcal R^{\alpha}_{k+1} + (M-h\theta A ^{\alpha}_{k+1}) (x^{\alpha+1}_{k+1} -
    x^{\alpha}_{k+1}) - h \gamma (r^{\alpha+1}_{k+1} - r^{\alpha}_{k+1} )  =0 ,
\end{equation}
with
\begin{equation}
     \begin{array}{l}
       A^{\alpha}_{k+1} = \nabla_x f(t_{k+1}, x^{\alpha}_{k+1})
 \end{array}
\end{equation}

By using (\ref{eq:rfree-1}), we get
\begin{equation}
  \label{eq:rfree-2}
  \mathcal R
_{\free}(x^{\alpha}_{k+1},r^{\alpha}_{k+1} )  - h\gamma r^{\alpha+1}_{k+1} - h(1-\gamma)r_k  + (M-h\theta A^{\alpha}_{k+1}) (x^{\alpha+1}_{k+1} -
    x^{\alpha}_{k+1})  =0
\end{equation}

% %\fbox
% {
%   \begin{equation}
%     \label{eq:rfree-11}
%     \boxed{ x^{\alpha+1}_{k+1} = h\gamma (W^{\alpha}_{k+1})^{-1}r^{\alpha+1}_{k+1} +x^\alpha_{\free}}
%   \end{equation}
% }
% with :
% \begin{equation}
%   \label{eq:rfree-12}
%   \boxed{x^\alpha_{\free}\stackrel{\Delta}{=}x^{\alpha}_{k+1}-(W^{\alpha}_{k+1})^{-1}\mathcal (R_{\free,k+1}^{\alpha} \textcolor{red}{- h(1-\gamma) r_k})}
% \end{equation}

The matrix $W$ is clearly non singular for small $h$.

% that is

% \begin{equation}
%    \begin{array}{l}
%  h \gamma  r^{\alpha+1}_{k+1} = r_c + W^{\alpha}_{k+1} x^{\alpha+1}_{k+1}
%  .\label{eq:NL11}
%  \end{array}
% \end{equation}
% with
% \begin{equation}
%    \begin{array}{l}
% r_c \stackrel{\Delta}{=} h \gamma r^{\alpha}_{k+1} - W^{\alpha}_{k+1} x^{\alpha}_{k+1} + \mathcal R
% ^{\alpha}_{k+1}=- W^{\alpha}_{k+1} x^{\alpha}_{k+1} + \mathcal R_{\free k+1} ^{\alpha} - h(1-\gamma)r_k\\ \\
% \end{array}
% \end{equation}
% \begin{equation}
%    \begin{array}{l}
% \mathcal R ^{\alpha}_{k+1}=M( x^{\alpha}_{k+1} - x_k) -h \theta f(x^{\alpha}_{k+1})-h(1-\theta)f(x_k)
% - h \gamma r^{\alpha}_{k+1} -h(1- \gamma)r_k
%  \end{array}
%    \end{equation}
% \[x^{\alpha+1}_{k+1} = h(W^{\alpha}_{k+1})^{-1}r^{\alpha+1}_{k+1} +(W^{\alpha}_{k+1})^{-1}(\mathcal
% R_{\free k+1} ^{\alpha})+x^{\alpha}_{k+1}\]

 \paragraph{Newton's linearization of the second  line of~(\ref{eq:toto1})}
The same operation is performed with the second equation of (\ref{eq:toto1})
\begin{equation}
  \begin{array}{l}
    \mathcal R_y(x,y,\lambda)=y-h(t_{k+1},x,\lambda) =0\\ \\
  \end{array}
\end{equation}
which is linearized as
\begin{equation}
  \label{eq:NL9}
  \begin{array}{l}
    \mathcal R_{Ly}(x^{\alpha+1}_{k+1},y^{\alpha+1}_{k+1},\lambda^{\alpha+1}_{k+1}) = \mathcal
    R_{y}(x^{\alpha}_{k+1},y^{\alpha}_{k+1},\lambda^{\alpha}_{k+1}) +
    (y^{\alpha+1}_{k+1}-y^{\alpha}_{k+1})- \\[2mm] \qquad  \qquad \qquad \qquad  \qquad \qquad
    C^{\alpha}_{k+1}(x^{\alpha+1}_{k+1}-x^{\alpha}_{k+1}) - D^{\alpha}_{k+1}(\lambda^{\alpha+1}_{k+1}-\lambda^{\alpha}_{k+1})=0
  \end{array}
\end{equation}

This leads to the following linear equation
\begin{equation}
  \boxed{y^{\alpha+1}_{k+1} =  y^{\alpha}_{k+1}
  -\mathcal R^{\alpha}_{yk+1}+ \\
  C^{\alpha}_{k+1}(x^{\alpha+1}_{k+1}-x^{\alpha}_{k+1}) +
  D^{\alpha}_{k+1}(\lambda^{\alpha+1}_{k+1}-\lambda^{\alpha}_{k+1})}. \label{eq:NL11y}
\end{equation}
with,
\begin{equation}
     \begin{array}{l}
  C^{\alpha}_{k+1} = \nabla_xh(t_{k+1}, x^{\alpha}_{k+1},\lambda^{\alpha}_{k+1} ) \\ \\
  D^{\alpha}_{k+1} = \nabla_{\lambda}h(t_{k+1}, x^{\alpha}_{k+1},\lambda^{\alpha}_{k+1})
 \end{array}
\end{equation}
and
\begin{equation}\fbox{$
\mathcal R^{\alpha}_{yk+1} \stackrel{\Delta}{=} y^{\alpha}_{k+1} - h(x^{\alpha}_{k+1},\lambda^{\alpha}_{k+1})$}
 \end{equation}
 \paragraph{Newton's linearization of the third  line of~(\ref{eq:toto1})}
The same operation is performed with the third equation of (\ref{eq:toto1})
\begin{equation}
  \begin{array}{l}
    \mathcal R_r(r,x,\lambda)=r-g(t_{k+1},x,\lambda) =0\\ \\  \end{array}
\end{equation}
which is linearized as
\begin{equation}
  \label{eq:NL9}
  \begin{array}{l}
      \mathcal R_{Lr}(r^{\alpha+1}_{k+1},x^{\alpha+1}_{k+1},\lambda^{\alpha+1}_{k+1}) = \mathcal
      R_{rk+1}^{\alpha} + (r^{\alpha+1}_{k+1} - r^{\alpha}_{k+1}) -
      K^{\alpha}_{k+1}(x^{\alpha+1}_{k+1} - x^{\alpha}_{k+1})- B^{\alpha}_{k+1}(\lambda^{\alpha+1}_{k+1} -
      \lambda^{\alpha}_{k+1})=0
    \end{array}
  \end{equation}
\begin{equation}
  \label{eq:rrL}
  \begin{array}{l}
    \boxed{r^{\alpha+1}_{k+1} = g(t_{k+1},x ^{\alpha}_{k+1},\lambda ^{\alpha}_{k+1}) +
      K^{\alpha}_{k+1}(x^{\alpha+1}_{k+1} - x^{\alpha}_{k+1})
      + B^{\alpha}_{k+1}(\lambda^{\alpha+1}_{k+1} - \lambda^{\alpha}_{k+1})
    }
  \end{array}
\end{equation}
with,
\begin{equation}
     \begin{array}{l}
  K^{\alpha}_{k+1} = \nabla_xg(t_{k+1},x^{\alpha}_{k+1},\lambda ^{\alpha}_{k+1})  \\ \\
  B^{\alpha}_{k+1} = \nabla_{\lambda}g(t_{k+1},x^{\alpha}_{k+1},\lambda ^{\alpha}_{k+1})
 \end{array}
\end{equation}
and the  residue for $r$:
\begin{equation}
\boxed{\mathcal
      R_{rk+1}^{\alpha} = r^{\alpha}_{k+1} - g(t_{k+1},x^{\alpha}_{k+1},\lambda ^{\alpha}_{k+1})}
  \end{equation}

\paragraph{Reduction to a linear relation between  $x^{\alpha+1}_{k+1}$ and $\lambda^{\alpha+1}_{k+1}$}

Inserting (\ref{eq:rrL}) into~(\ref{eq:rfree-2}), we get the following linear relation between $x^{\alpha+1}_{k+1}$ and
$\lambda^{\alpha+1}_{k+1}$,
\begin{equation}
  \label{eq:rfree-3}
  \begin{array}{l}
  \mathcal R^{\alpha}_{\free, k+1}  - h\gamma\left[  g(t_{k+1},x^{\alpha}_{k+1},\lambda^{\alpha}_{k+1}) +
    B^{\alpha}_{k+1} (\lambda^{\alpha+1}_{k+1} - \lambda^{\alpha}_{k+1})+K^{\alpha}_{k+1}
    (x^{\alpha+1}_{k+1} - x^{\alpha}_{k+1})  \right] \\
  \quad\quad - h(1-\gamma)r_k  + (M-h\theta A^{\alpha}_{k+1}) (x^{\alpha+1}_{k+1} -
    x^{\alpha}_{k+1})  =0
  \end{array}
\end{equation}
that is
\begin{equation}
  \label{eq:rfree-4}
  \begin{array}[l]{lcl}
  (M-h\theta A^{\alpha}_{k+1}-h\gamma K^{\alpha}_{k+1}) (x^{\alpha+1}_{k+1}  -  x^{\alpha}_{k+1}) &=&
  -  \mathcal R^{\alpha}_{\free, k+1} -h(1-\gamma) r_k \\ & & + h\gamma \left[  g(t_{k+1},x^{\alpha}_{k+1},\lambda^{\alpha}_{k+1}) +
    B^{\alpha}_{k+1} (\lambda^{\alpha+1}_{k+1} - \lambda^{\alpha}_{k+1})  \right]
\end{array}
\end{equation}

Let us introduce some intermediate notation:
\begin{equation}
   \label{eq:NL9}
   \begin{array}{l}
     W^{\alpha}_{k+1} \stackrel{\Delta}{=} M-h\theta A^{\alpha}_{k+1}-h\gamma K^{\alpha}_{k+1})\\
   \end{array}
 \end{equation}
 \begin{equation}
   \label{eq:rfree-12}
   \boxed{x^\alpha_{\free}\stackrel{\Delta}{=}x^{\alpha}_{k+1}-(W^{\alpha}_{k+1})^{-1}\mathcal (R_{\free,k+1}^{\alpha} \textcolor{red}{- h(1-\gamma) r_k})}
 \end{equation}
and
\begin{equation}
  \boxed{x^\alpha_p \stackrel{\Delta}{=}  h\gamma(W^{\alpha}_{k+1} )^{-1}\left[g(t_{k+1},x^{\alpha}_{k+1},\lambda^{\alpha}_{k+1})
    -B^{\alpha}_{k+1} (\lambda^{\alpha}_{k+1}) \right ] +x^\alpha_{\free}}.
\end{equation}

The relation (\ref{eq:rfree-4}) can be written as
\begin{equation}
  \label{eq:rfree-13}
  \begin{array}{l}
    \boxed{   x^{\alpha+1}_{k+1}\stackrel{\Delta}{=}  x^\alpha_p +  \left[ h \gamma (W^{\alpha}_{k+1})^{-1}    B^{\alpha}_{k+1} \lambda^{\alpha+1}_{k+1}\right]}
   \end{array}
\end{equation}

\paragraph{Reduction to a linear relation between  $y^{\alpha+1}_{k+1}$ and
$\lambda^{\alpha+1}_{k+1}$.}

Inserting (\ref{eq:rfree-13}) into (\ref{eq:NL11y}), we get the following linear relation between $y^{\alpha+1}_{k+1}$ and $\lambda^{\alpha+1}_{k+1}$,
\begin{equation}
   \begin{array}{l}
     y^{\alpha+1}_{k+1} = y_p + \left[ h \gamma C^{\alpha}_{k+1} ( W^{\alpha}_{k+1})^{-1}  B^{\alpha}_{k+1} + D^{\alpha}_{k+1} \right]\lambda^{\alpha+1}_{k+1}
   \end{array}
\end{equation}
with
\begin{equation}\boxed{
    y_p = y^{\alpha}_{k+1} -\mathcal R^{\alpha}_{yk+1} + C^{\alpha}_{k+1}(x_q) -   D^{\alpha}_{k+1} \lambda^{\alpha}_{k+1} }
\end{equation}
\textcolor{red}{
  \begin{equation}
   \boxed{ x_q=x^\alpha_p -x^{\alpha}_{k+1}\label{eq:xqq}}
  \end{equation}
}

% \paragraph{With $\gamma =1$:}
% \[(W^{\alpha}_{k+1} )x^{\alpha+1}_{k+1}= hr^{\alpha+1}_{k+1}- \mathcal R_{\free, k+1} ^{\alpha}+W^{\alpha}_{k+1}x^{\alpha}_{k+1}\]
% \[x^{\alpha+1}_{k+1}= h( W^{\alpha}_{k+1})^{-1}r^{\alpha+1}_{k+1}-
% ( W^{\alpha}_{k+1})^{-1} \mathcal R_{\free k+1} ^{\alpha}+x^{\alpha}_{k+1}\]
% \[x^{\alpha+1}_{k+1}= h( W^{\alpha}_{k+1})^{-1}r^{\alpha+1}_{k+1}+x_{\free}\]
% with, using \ref{}
% \begin{equation}
% x_p-x^{\alpha}_{k+1}=h(
% W^{\alpha}_{k+1})^{-1}(g(x^{\alpha}_{k+1},\lambda^{\alpha}_{k+1},t_{k+1})-B^{\alpha}_{k+1}
% \lambda^{\alpha}_{k+1}-K^{\alpha}_{k+1} x^{\alpha}_{k}))+\tilde x_{\free}
% \end{equation}
% \[    \tilde x_{\free}= -( W^{\alpha}_{k+1})^{-1} \mathcal R _{\free k+1} ^{\alpha} \]
%       \[x_{\free} = \tilde x_{\free} + x^{\alpha}_{k+1}=\fbox{$- W^{-1}R_{\free k+1} ^{\alpha} + x^{\alpha}_{k+1}$}\]
% \[ \fbox{$x_p  = x_{\free} + h ( W^{\alpha}_{k+1})^{-1}( g(x ^{\alpha}_{k+1},\lambda ^{\alpha}_{k+1},t_{k+1}) -
%       B^{\alpha}_{k+1} \lambda^{\alpha}_{k+1}-K^{\alpha}_{k+1} x^{\alpha}_{k+1} )$} \]

\paragraph{Mixed linear complementarity problem (MLCP)}To summarize, the problem to be solved in each Newton iteration is:\\{
  \begin{minipage}[l]{1.0\linewidth}
    \begin{equation}
      \begin{cases}
      \begin{array}[l]{l}
        y^{\alpha+1}_{k+1} =   W_{mlcpk+1}^{\alpha}  \lambda^{\alpha+1}_{k+1} + b^{\alpha}_{k+1}
        \\ \\
        -y^{\alpha+1}_{k+1} \in N_{[l,u]}(\lambda^{\alpha+1}_{k+1} ).
      \end{array}
      \label{eq:NL14}
      \end{cases}
    \end{equation}
  \end{minipage}
}
with $W_{mlcpk+1}\in \RR^{m\times m}$ and $b\in\RR^{m}$ defined by
\begin{equation}
  \label{eq:NL15}
 \begin{array}[l]{l}
   W_{mlcpk+1}^{\alpha} = h \gamma C^{\alpha}_{k+1}  (W^{\alpha}_{k+1})^{-1}  B^{\alpha}_{k+1} + D^{\alpha}_{k+1} \\
   b^{\alpha}_{k+1} = y_p
\end{array}
\end{equation}

The problem~(\ref{eq:NL14}) is equivalent to a Mixed Linear Complementarity Problem (MLCP) which can be solved under suitable assumptions by many linear complementarity solvers such as pivoting techniques, interior point techniques and splitting/projection strategies. The  reformulation into a standard MLCP follows the same line as for the MCP in the previous section. One obtains,
    \begin{equation}
      \begin{array}[l]{l}
        y^{\alpha+1}_{k+1} =   - W^{\alpha}_{k+1}  \lambda^{\alpha+1}_{k+1} + b^{\alpha}_{k+1}
        \\ \\
        (y^{\alpha+1}_{k+1})_i  = 0 \qquad \textrm{ for } i \in \{ 1..n\}\\[2mm]
        0 \leq  (\lambda^{\alpha+1}_{k+1})_i\perp (y^{\alpha+1}_{k+1})_i \geq 0 \qquad \textrm{ for } i \in \{ n..n+m\}\\
      \end{array}
      \label{eq:MLCP1}
    \end{equation}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "DevNotes"
%%% End:


% Define document class
\documentclass[modern]{aastex631}

% Filler text
\usepackage{acro}
\usepackage{blindtext}

\DeclareAcronym{KDE}{
    short = {KDE},
    long = {kernel density estimate}
}

\DeclareMathOperator{\var}{Var}

% Begin!
\begin{document}

% Title
\title{A Note About Marginalizing Over Gaussian Populations With KDE Likelihood Representations}

% Author list
\author[0000-0003-1540-8562]{Will M. Farr}

% Abstract with filler text
\begin{abstract}
    I work through the marginalization over parameters whose population is
    Gaussian using a \ac{KDE} approximation to the likelihood function.  The
    result is an analytic \ac{KDE} representation of the marginal likelihood for
    the Gaussian population parameters.
\end{abstract}

% Main body with filler text
\section{Marginalization}

Suppose we are conducting a population analysis with a Gaussian population
\citep[e.g.][]{Isi2019,Miller2020}.  The hierarchical posterior for the unknown
parameter $x_i$ with the Gaussian population from each of the $i = 1, \ldots, N$
events and the population parameters $\mu$ and $\sigma$ is
\begin{equation}
    \label{eq:full-posterior}
    p\left( \left\{ x_i \right\}, \mu, \sigma \mid \left\{ d_i \right\} \right) \propto p\left( \mu, \sigma \right) \prod_{i=1}^N p\left( d_i \mid x_i \right) p\left( x_i \mid \mu, \sigma \right)
\end{equation}
where $p\left( \mu, \sigma \right)$ is the prior we impose on the population
parameters, and $d_i$ is the data for each observation.  The population is
assumed to be Gaussian with mean $\mu$ and standard deviation $\sigma$:
\begin{equation}
    p\left( x \mid \mu, \sigma \right) = N\left( x \mid \mu, \sigma \right).
\end{equation}

In problems of interest, the likelihood function is often not simple to write
down, and does not take a straightforward analytic form.  In such situations, we
usually draw \emph{samples} from the likelihood (or a posterior which is the
likelihood times some simple prior) for each event.  Let there be $j = 1,
\ldots, M_i$ samples $x_{ij}$ from (a density proportional to) the likelihood
(i.e.\ from a posterior with a flat prior):
\begin{equation}
    x_{ij} \sim p\left( d \mid x_i \right).
\end{equation}
A useful representation of the likelihood function (up to a proportionality
constant that we usually ignore---unless we are computing the Bayesian evidence)
then is a \ac{KDE} with a Gaussian kernel over the samples from the likelihood
\begin{equation}
    p\left( d \mid x_i \right) \simeq \frac{1}{M_i} \sum_{j} N\left( x_i \mid x_{ij}, \sigma_i \right)
\end{equation}
where $\sigma_i$ is a reasonable \emph{bandwidth} for the \ac{KDE}\footnote{When
in doubt, I usually follow \citet{Scott1992}, with
\begin{equation}
    \sigma_i^2 = \frac{\var x_{ij}}{M_i^{2/5}}.
\end{equation}
}.

Because we can do Gaussian integrals (though the discussion here is for a
one-dimensional parameter $x$, the same trick works is multiple dimensions;
\citet{Hogg2020} is helpful here), we can analytically integrate out the true
parameter values $x_i$ from the posterior in Eq.\ \eqref{eq:full-posterior}
using the \ac{KDE} approximation to the likelihood. The result is
\begin{equation}
    p\left( \mu, \sigma \mid \left\{ d_i \right\} \right) \propto p\left( \mu, \sigma \right) \prod_{i=1}^N \frac{1}{M_i} \sum_{j=1}^{M_i} N\left( x_{ij} \mid \mu, \sqrt{\sigma_i^2 + \sigma^2} \right)
\end{equation}
In this special case of Gaussian populations and Gaussian \ac{KDE}
approximations to the likelihood function, the above expression is considerably
more robust to differences in scale between the population and measurement
(i.e.\ $\sigma_i \ll \sigma$ or $\sigma \ll \sigma_i$) than the usual
Monte-Carlo approximation to the integral of the likelihood
\citep[e.g.][]{Miller2020}.

\bibliography{bib}

\end{document}


% file: Mat_out.tex
%
% github        : ernestyalumni
% gmail         : ernestyalumni
% linkedin      : ernestyalumni
% wordpress.com : ernestyalumni
%
% This code is open-source, governed by the Creative Common license.  Use of this code is governed by the Caltech Honor Code: ``No member of the Caltech community shall take unfair advantage of any other member of the Caltech community.''

\documentclass[10pt]{amsart}
\pdfoutput=1
\usepackage{mathtools,amssymb,caption}

\usepackage{graphicx}
\usepackage{hyperref}
\usepackage[utf8]{inputenc}
\usepackage{listings}
\usepackage[table]{xcolor}
\usepackage{pdfpages}
\usepackage{tikz}
\usetikzlibrary{matrix,arrows}

\usepackage{breqn} % for dmath

%\usepackage{cancel} % for Feynman slash notation

\hypersetup{colorlinks=true,citecolor=[rgb]{0,0.4,0}}

%\oddsidemargin=15pt
%\evensidemargin=5pt
%\hoffset-45pt
%\voffset-55pt
%\topmargin=-4pt
%\headsep=5pt
%\textwidth=1120pt
%\textheight=595pt
%\paperwidth=1200pt
%\paperheight=700pt
%\footskip=40pt

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}{Corollary}
%\newtheorem*{main}{Main Theorem}
\newtheorem{lemma}{Lemma}
\newtheorem{proposition}{Proposition}

\newtheorem{definition}{Definition}
\newtheorem{remark}{Remark}

\newenvironment{claim}[1]{\par\noindent\underline{Claim:}\space#1}{}
\newenvironment{claimproof}[1]{\par\noindent\underline{Proof:}\space#1}{\hfill $\blacksquare$}

%This defines a new command \questionhead which takes one argument and
%prints out Question #. with some space.
\newcommand{\questionhead}[1]
  {\bigskip\bigskip
   \noindent{\small\bf Question #1.}
   \bigskip}

\newcommand{\problemhead}[1]
  {
   \noindent{\small\bf Problem #1.}
   }

\newcommand{\exercisehead}[1]
  { \smallskip
   \noindent{\small\bf Exercise #1.}
  }

\newcommand{\solutionhead}[1]
  {
   \noindent{\small\bf Solution #1.}
   }

  \title[Matrices as a Tensor Product and the \emph{Mat} C++ class template]{Matrices as a Tensor Product and the \emph{Mat} C++ class template}

\author{Ernest Yeung \href{mailto:ernestyalumni@gmail.com}{ernestyalumni@gmail.com}}
\date{27 juillet 2017}
\keywords{C++, C++11, Matrices, Tensor Products, dual basis, basis, numerical computation}

\begin{document}

\definecolor{darkgreen}{rgb}{0,0.4,0}
\lstset{language=Python,
 frame=bottomline,
 basicstyle=\scriptsize,
 identifierstyle=\color{blue},
 keywordstyle=\bfseries,
 commentstyle=\color{darkgreen},
 stringstyle=\color{red},
 }
%\lstlistoflistings

\maketitle

\tableofcontents

%\begin{multicols*}{2}

\begin{abstract}
\section{Executive Summary}

I implement matrices, with a focus on implementing matrix multiplication and the taking the transpose for the matrices, in C++11.  In designing the class templates for the matrices, I begin with the mathematical formulation of the space of all matrices of matrix size dimensions $M\times N$, $\textbf{Mat}_{\mathbb{K}}(M,N)$, with $\mathbb{K}$ being the underlying field (e.g. $\mathbb{K}=\mathbb{R},\mathbb{Z}$), as a \emph{tensor product} of vector spaces,  $\bigotimes_{j=1}^P \mathbb{K}^M$ and equivalently as a tensor product of \emph{dual} vector spaces $\bigotimes_{i=1}^M \mathbb{K}^P$.  By doing so, we can implement matrix multiplication and the transpose using new C++11 features in the library header `algorithm`, and efficiently read contiguous memory addresses along each of the \verb|std::vector|s, representing rows or columns of a matrix.
\end{abstract}

\section{Mathematical formulation: matrices as tensor products of vectors (or \emph{dual} vectors)}

Denote the set of all matrices of \emph{matrix size dimensions} (i.e. number of rows $\times $ number of columns) $(M,P)$ or i.e. $M\times N$, with the underlying field $\mathbb{K}$, to be $\text{Mat}_{\mathbb{K}}(M,P)$.

Consider two matrices $A \in \text{Mat}_{\mathbb{K}}(M,P)$, $B \in \text{Mat}_{\mathbb{K}}(P,N)$.  We can multiply them together since they share equal "inner size dimensions."

Let's consider the entries of matrix $A$ (even if only to set notation), $A_{ij}$, \, $\forall \, i = 1,2, \dots M$, $j=1,2,\dots P$.  Consider the "rows" of the matrix, and the "columns" of the matrix $A$.  For the $i$th row of $A$, denote it as $A_{i*}$, so that
\begin{equation}\label{Eq:RowForm}
	A_{i*} = (A_{i1},A_{i2}, \dots A_{iP}) \qquad \, \forall \, i = 1,2, \dots M
\end{equation}
For the $j$th column of $A$, denote it as $A_{*j}$, so that
\begin{equation}\label{Eq:ColumnForm}
	A_{*j} = (A_{1j},A_{2j}, \dots A_{jM}) \qquad \, \forall \, j = 1,2, \dots P
\end{equation}

Let's treat the "row vector", $A_{i*}$ as being an element of the dual vector space to $\mathbb{K}^P$, $(\mathbb{K}^P)^*$.  Notice that matrix $A$ has $M$ of these "rows."  Then $\forall \, A \in \text{Mat}_{\mathbb{K}}(M,P)$, $A \in \bigotimes_{i=1}^M (\mathbb{K}^P)^*$.  In fact, we already described the isomorphism between these two spaces.  So
\begin{equation}
\bigotimes_{i=1}^M (\mathbb{K}^P)^* \cong \text{Mat}_{\mathbb{K}}(M,P)
\end{equation}

Let's treat the "column vector", $A_{*j}$ as being an element of the vector space $\mathbb{K}^M$.  Notice that matrix $A$ has $P$ of these "columns."  Then $\forall \, A \in \text{Mat}_{\mathbb{K}}(M,P)$, $A \in \bigotimes_{j=1}^P (\mathbb{K}^M)^*$.  In fact, we already described the isomorphism between these two spaces.  So
\begin{equation}
\bigotimes_{j=1}^P (\mathbb{K}^M) \cong \text{Mat}_{\mathbb{K}}(M,P)
\end{equation}

\subsection{Matrix Multiplication}

Again, given matrices $A \in \text{Mat}_{\mathbb{K}}(M,P)$, $B \in \text{Mat}_{\mathbb{K}}(P,N)$, we multiply them together to obtain $C \in \text{Mat}_{\mathbb{K}}(M,N)$:
\begin{equation}
\begin{gathered}
C = AB  \\
C_{ij} = \sum_{k=1}^P A_{ik}B_{kj} \equiv A_{ik}B_{kj} \quad \, \forall \, i=1,2,\dots M, \, \forall \, j = 1,2, \dots N
\end{gathered}
\end{equation}
where at the end, we use Einstein's summation notation of implicit summation of any repeated indices.

Keeping in mind that "row vector" and "column vector" (dual vector and vector) forms from Eq. \ref{Eq:RowForm}, Eq. \ref{Eq:ColumnForm}, respectively, let's try to rewrite the matrix multiplication $C=AB$:
\begin{equation}\label{Eq:MatrixMultiplybyRows}
\begin{gathered}
C_{ij} = A_{ik}B_{kj} = A_{ik} (B^T)_{jk} = A_{i*} \cdot (B^T)_{j*} \qquad \, \forall \, i = 1, \dots M, \, \forall \, j = 1, \dots N
\end{gathered}
\end{equation}
Thus, if we can get all the rows of $A$, and get all the columns of $B$, we can simply take the inner product of each row with each column (all possible row, column pairs), and we'll have obtained $C$!

\subsubsection{The case for thinking of Matrix Multiplication as the inner product of a dual vector with a vector from a computational optimization point of view}

Consider a "naive" implementation of matrix multiplication, which, regardless of your choice of programming language, mathematically is essentially the following:
\begin{equation}
\begin{gathered}
\forall \, i = 1,2, \dots M, \\
\phantom{\qquad \, }\forall \, j = 1,2, \dots N , \\
\phantom{\qquad \qquad \, } \forall \, k = 1,2,\dots P , \\
\text{sum} += A_{ik}B_{kj} , \quad \, i.e. \quad \sum_{k=1}^P A_{ik}B_{kj}
\end{gathered}
\end{equation}
A (very) cursory look at the read requirements in this matrix multiplication operation would tell us that the read, memory requirements are of order $O(M*P^2*N)$ (imagine multiply a row from $A$ with a "fixed" column from $B$.  That same "fixed" column from $B$ will have to be multiplied by the other $M-1$ rows, but we've read this particular column from $B$ already $M-1$ redundantly).  Nevertheless, it is polynomial in time.

Also, notice that if we assume \emph{row-major ordering} for how the values of the matrix entries are inserted or stored, contiguously, in memory addresses, then in accessing the values for the columns of matrix $B$, we are not efficiently accessing those values wince we're "jumping" over $P$ memory addresses to grab the next adjacent entry in a column.

However, Eq. \ref{Eq:MatrixMultiplybyRows} gives us a prescription:
\begin{equation}\label{Eq:MatrixMultiplybyRows2}
C_{ij} = A_{i*} \cdot (B^T)_{j*} \quad \, \forall \, i =1 \dots M, \, \forall \, j = 1\dots N
\end{equation}
If we can store the "columns" of $B$ as rows, $(B^T)_{j*}$, then we can access columns in a way that their respective memory addresses are contiguous.  Also, if we have an optimized routine for taking \emph{inner products} or, i.e., \emph{dot products}, we can employ that as well, according to Eq. \ref{Eq:MatrixMultiplybyRows2}.  Indeed, we have this in C++11 standard library \verb|<numeric>|, with \verb|std::inner_product|.  At this point, take a look at the code in \verb|Mat/Mat.h|, for the implementation of matrix multiplication given in the operator overloading of \verb|*| operator for class \verb|Mat|,
\[
\verb|Mat<Type> operator*(const Mat<Type>& rhs)|
\].  Each row from matrix $A$, $i$th row \verb|A_i| and each column from $B \equiv $ \verb|rhs|, \verb|col|, has its inner product computed by \verb|std::inner_product|.

\subsection{Transpose operation on matrices, as tensor products of vector spaces and dual vector spaces}

Consider the mathematical formulation of the transpose operation:
\begin{equation}\label{Eq:Transpose}
\begin{gathered}
\text{Mat}_{\mathbb{K}}(M,P) \xrightarrow{ T } \text{Mat}_{\mathbb{K}}(P,M)  \\
\bigotimes_{i=1}^P(\mathbb{K}^M) \xrightarrow{ T } \bigotimes_{j=1}^P (\mathbb{K}^M)^*  \\
A = (A_{*1}, A_{*2}, \dots A_{*P}) \xmapsto{T} A^T = (A^T_{1*}, A^T_{2*}, \dots A^T_{M*})
\end{gathered}
\end{equation}
Thus, it's clear from Eq. \ref{Eq:Transpose} that if we can simply store the "columns" or "column" vectors of matrix $A$ as "rows" or dual vectors, we can simply read off these "rows" or dual vectors as the new rows of a new matrix $A^T$, the transpose of $A$.

This is what was exactly implemented in \verb|Mat.h| in the class template method \verb|Mat<Type> T()|.  All we do is take the "columns" of a matrix, stored in private member variable \verb|Columns_| and assign them to the new matrix for the transpose, \verb|new_Rows|, and then initialize a new matrix with the class template \verb|Mat|, \verb|Atranspose|.

\section{Further isomorphisms between math and the code}

Take a look at the \verb|private| member variables of the class template \verb|Mat| in \verb|Mat/Mat.h|.  They include
\begin{itemize}
	\item \verb|std::array<unsigned int,2> Size_Dims_;|
	\item \verb|std::vector<std::vector<Type>> Rows_|
	\item \verb|std::vector<std::vector<Type>> Columns_|
	\item \verb|std::vector<Type> Entries_|
\end{itemize}
These are isomorphic to the following:
\begin{itemize}
	\item $(M,P)$ in $\text{Mat}_{\mathbb{K}}(M,P) \ni A$
	\item $(A_{1*}, A_{2*}, \dots A_{i*} \dots A_{M*}) \in \bigotimes_{i=1}^M (\mathbb{K}^P)^*$
	\item $(A^T_{1*}, A^T_{2*}, \dots A^T_{j*} \dots A^T_{P*}) \in \bigotimes_{j=1}^P (\mathbb{K}^M)^*$
	\item $(A_{ij})=A \in \mathbb{K}^{M\cdot P}$ (there is an obvious isomorphism between matrices and a vector space, i.e. $\text{Mat}_{\mathbb{K}}(M,P) \cong \mathbb{K}^{M\cdot P}$.
\end{itemize}, respectively.

Understanding these relations and the availability of these private member variables, one can implement further useful methods, including getting the values of the entries, getting a particular row (that's contiguously laid out sequentially on adjacent memory addresses), setting a specific row, and then going from \verb|Rows_| and turning it into a new \verb|Entries_|.  Also, addition for matrices (since matrices are non-commutative rings) and scalar multiplication could also be implemented in memory efficient (i.e. saving the number of reads into memory that is needed and the utilization of optimized algorithms in the standard library of C++11) with this understanding.

\section{Concluding Remarks}

I explicitly show an isomorphism between the mathematical formulation of matrices and the operations matrix multiplication and the taking of the transpose as a tensor product of dual vectors and equivalently tensor product of vectors, and the C++11 implementation of a class template \verb|Mat|.  I point out that by doing so, we've discovered a more read-memory efficient and more optimized way of doing matrix multiplication, taking the transpose, and accessing the values of the entries of the matrix, than a "naive" implementation.  I was also able to utilize and showcase a number of novel, new features from C++11; \verb|std::vector| as a container and \verb|<numerics>|.

The developer using this code or the framework for it should be clearly able to add new methods and features to the class template, as long as he or she understands the underlying mathematics.  In fact, we should agree upon the mathematics first and then try as much as we could to have a 1-to-1 correspondence between the math and the code, in object-oriented programming (OOP) design.  I argue this because it should make the code maintainable and clear because we should all agree on the same thing with the math.

Also, with these more efficient memory-accessing, we have made matrix multiplication and the taking of the transpose for matrices not only faster, but more scalable.

%\begin{thebibliography}{9}

%\end{thebibliography}

\end{document}


\section{Projecting onto more than one vector}

In \cref{sec:GEVD_overview,sec:GEVD_SNR}, we projected the multi-channel
input signal $\z_t$ onto one optimal weight vector $\what$, to yield a
one-dimensional output signal $y_t$ (see e.g. \cref{eq:linear} or
\cref{fig:GEVD_principle}).

We may also project $\z_t$ onto multiple weight vectors $\w_i$ to obtain
multiple output channels, $y_{t,i}$. When the $k$ weight vectors are gathered
as columns of a matrix $\What \in \reals^{C \cross k}$, and the different
outputs are combined into one multichannel output signal $\y_t \in \reals^k$,
\cref{eq:linear} becomes:
\[
\y_t = \What \z_t
\]
These multiple output signals can then be used as input to a further signal
processing step. Alternatively, sharp wave-ripple events may be detected in
each output signal separately, as described in \cref{sec:GEVD_overview}.
These detections can then be combined using a voting scheme, to yield a final
detection only when enough output channels yield a detection within close
temporal proximity of each other.

The question is then how to find these different weight vectors $\What$. As
in \cref{sec:GEVD_SNR}, we divide our input data into two data matrices $\Sm
\in \reals^{C \cross N_s}$ and $\Nm \in \reals^{C \cross N_n}$, with $C$ the
number of channels, and $N_s$ and $N_n$ the number of signal and noise
training samples, respectively. The output data matrices are then:
\begin{align*}
\Y_s &= \What^T \Sm \\
\Y_n &= \What^T \Nm.
\end{align*}

As before, for each pair of output data vectors $\y_{s,i}$ and $\y_{n,i}$
(i.e. rows $i$ of $\Y_s$ and $\Y_n$) we want to maximise the ratio of the
variance of the signal $\y_{s,i}$ versus the variance of the noise
$\y_{n,i}$, i.e. we want to find (see \cref{sec:GEVD_SNR}):
%
\begin{equation}
\label{eq:maxwi}
\what_i = \argmax_{\w_i} \frac{\w_i^T \Rss \w_i}
                              {\w_i^T \Rnn \w_i},
\end{equation}
%
with $\Rss = \frac{1}{N_s} \Sm \Sm^T$ and $\Rnn = \frac{1}{N_s} \Nm \Nm^T$
the signal and noise covariance matrices, as before.

The scale of $\w_i$ does not matter; only the variance ratio in
\cref{eq:maxwi} does. \Cref{eq:maxwi} is therefore equivalent to
%
\begin{gather*}
\what_i = \argmax_{\w_i} \w_i^T \Rss \w_i \\
\st \w_i^T \Rnn \w_i = 1
\end{gather*}

To combine the different $\w_i$ into one optimisation problem, we will ask to
optimise the sum of the signal-to-noise variance ratios. In matrix form, this
is written as:
\begin{gather*}
\What = \argmax_{\W} \Tr(\W^T \Rss \W) \\
\st \diag{\W^T \Rnn \W} = \diag{\I_k},
\end{gather*}
%
where $\Tr(\cdot)$ (for trace) denotes the sum of diagonal elements,
$\diag{\cdot}$ denotes the vector of diagonal elements, and $\I_k$ is the $k
\cross k$ identity matrix.

This maximisation problem yields identical $\w_1 = \w_2 = \tdots = \w_k$. If
we want different output channels, we need to introduce an additional
constraint. When we require the noise output channels to be uncorrelated
(i.e. $\Cov(\y_{n,i}, \y_{n,j}) = \y_{n,i}^T \y_{n,j} = 0$ for $i \neq j$),
our final optimisation problem becomes:
%
\begin{gather}
\What = \argmax_{\W} \Tr(\W^T \Rss \W)  \label{eq:opti_What} \\
\st \W^T \Rnn \W = \I_k.  \nonumber
\end{gather}
%
It can be shown that the solution $\What$ to this problem can be found as the
first $k$ generalised eigenvectors of $(\Rnn,\Rss)$, corresponding to the $k$
largest eigenvalues \cite{Kokiopoulou2011}.

\section{Context}

The method described in this chapter finds linear combinations of input
channels that maximise the signal-to-noise ratios in the output channels. For
brevity, we will call this method \textbf{LSM}, for Linear Signal-to-noise
Maximisation.\footnotemark{} This section shortly explores the wider context
of this alogrithm.

\footnotetext{Algorithmically, there is nothing more to LSM than calculating
a generalised eigenvalue decomposition, just as algorithmically, there is
nothing more to principal component analysis (PCA) than calculating an
ordinary eigenvalue decomposition. Why invent new names then? LSM and PCA are
little `wrapper' frameworks around the eigenvalue decompositions to usefully
interprete and reason about these operations when applied to actual data
problems (see the exposition in \cref{sec:GEVD_overview,sec:GEVD_SNR}).}

\subsection{Similarity to PCA}

Throughout this chapter, you may have noticed the strong similarities between
LSM and principal component analysis (PCA). Both are linear dimensionality
reduction methods (number of output channels $k$ < number of input channels
$C$). PCA is based on the ordinary eigenvalue decomposition (EVD) of the data
covariance matrix; LSM is based on the generalised eigenvalue decomposition
(GEVD) of the signal and noise data covariance matrices. Both methods
maximise a sum of output variances under an orthogonality constraint. And
just as PCA can also be computed using a singular value decomposition (SVD)
of the data matrix (without making a detour via the data covariance matrix),
LSM can also be calculated using a generalised singular value decomposition
(GSVD) of the signal and noise data matrices \cite{Howland2004}.

The difference between LSM (GEVD) and PCA (EVD), is that the latter is an
unsupervised algorithm, while the former is a supervised algorithm (i.e. you
need to divide your training data into `signal' and `noise' samples). For a
useful overview of different supervised, unsupervised and semi-supervised
dimensionality reduction methods, see \cite{Kokiopoulou2011}.\footnotemark{}
Of particular relevance to this chapter is their section on linear
discriminant analysis (LDA), which leads to the exact same optimisation
problem as \cref{eq:argmax_R} and \cref{eq:opti_What} (but for matrices
$\Rnn$ and $\Rss$ that have a slightly different interpretation).

\footnotetext{This paper ties together many linear and quasi-linear
dimensionality reduction methods. It shows how these methods relate to each
other in a unifying framework, that is based around trace optimisation
problems and (generalised) eigenvalue decompositions. Some of the methods
discussed are PCA, linear discriminant analysis (LDA), locally linear
embedding (LLE), Laplacean eigenmaps, multidimensional scaling (MDS),
spectral clustering, and kernel methods.}

\subsection{Other applications}

The generalised eigenvalue decomposition (GEVD) has found many applications.
Examples include blind source separation and beamforming
\cite{Tome2006,Warsitz2007}, speech enhancement \cite{Doclo2002}, linear
classification (via LDA, as described above), vibration analysis
\cite{Zhou2007} and analysis of magneto-encephalography (MEG)
\cite{Sekihara1999}, electro-encephalography (EEG) \cite{Blankertz2008} and
magnetic resonance imaging (MRI) \cite{Zhang2015} data.

\subsection{Distributed computation}

Modern neural probes support both large electrode counts and high sampling
frequencies \cite{Jun2017}, and may be used in real-time applications. The
signal processing algorithm used needs to process this large data stream fast
enough to meet the real-time requirements.

An advantage of using a signal processing algorithm based on an (adaptive)
ordinary or generalised eigenvalue decomposition, is that these
decompositions can be estimated in a distributed way
(\cite{Bertrand2014,Bertrand2015}). This allows a network of distributed,
embedded electronic circuits to process the neural probe data chunk-wise,
which may enable faster total processing, and thus larger electrode counts
and sampling frequencies, than possible with an algorithm that only works on
centralized data from all channels.

\section{Summary}

We have establised that we can find the weight vector (the linear combination
of input channels) that yields an output signal with a maximal
signal-to-noise variance ratio as the first generalised eigenvector $\w_1$
of the ordered covariance matrix pair $(\Rss,\Rnn)$, with $\Rss$ and $\Rnn$
the covariance matrices of signal and noise training data.

As a generalisation, we can also calculate a set of $k$ weight vectors that
maximise the sum of their respective output channel signal-to-noise variance
ratios (where the noise segments of each output channel are uncorrelated).
Analogously to the one output channel case, these weight vectors are found as
the first $k$ generalised eigenvectors $(\Rss, \Rnn)$. We have given some
insight in how these generalised eigenvectors are calculated, and into how
this signal-to-noise maximisation method fits into the literature.