\RequirePackage{fix-cm} 											%
\documentclass[11pt, a4paper, parskip=half*, bibliography=totoc, cleardoublepage=empty, final,
numbers=noenddot]{scrbook}
\usepackage[ansinew]{inputenc}
\usepackage[automark]{scrpage2}
\usepackage{fixltx2e}
\usepackage[linktocpage]{hyperref}
\usepackage{breakurl}
\usepackage[english]{babel}
\usepackage{multicol}
\usepackage{multirow}
\usepackage{cite}
\usepackage{bm}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amssymb}
\usepackage{subcaption}
\usepackage{rotating}
\usepackage{fixmath}
\usepackage{braket}
\DeclareMathOperator*{\argmin}{arg\,min}
\usepackage[margin=10pt,labelfont=bf]{caption}
\usepackage[top=2.5cm,left=3.5cm,right=2.5cm,bottom=3cm]{geometry}
\usepackage[onehalfspacing]{setspace}
\usepackage{lmodern}
\captionsetup{justification=raggedright,singlelinecheck=false}
\begin{document}
\chapter{Linear algebra}
Note that we consider only real values.
\section{Gram-Schmidt orthogonalization}
Using orthogonal projections $\frac{<v, u>} {<u, u>} u$ of vectors $v$ onto vectors $u$ and differences $v - \frac{<v, u>}{<u, u>} u$, the Gram-Schmidt orthogonalization constructs from a set of vectors a set of orthogonal ones in an iterative procedure. Note that the difference $v - \frac{<v, u>}{<u, u>} u$ is orthogonal to the projection (assuming $u$ is a unit vector):
\begin{align}
\langle <v, u> u, v- <v, u> u \rangle &= \langle <v, u> u, v \rangle - \langle <v, u> u, <v, u> u \rangle \\
&= <v, u>^2 - <v, u>^2 \underbrace{<u, u>}_{=1} \\
&= 0.
\label{eq:orthogonal-proj-zero}
\end{align}
Let $S_V = \{v_1, v_2, ..., v_n\}$ be a set of vectors and $d$ be the dimension of the space $V$ spanned by the vectors in $S_V$. Then the Gram-Schmidt orthogonalization (or here orthonormalization) generates an orthonormal basis set $S_U = \{u_1, u_2, ..., u_d\}$ in $V$ by:
\begin{itemize}
\item[1.] $u_1 = \frac{v_1}{\| v_1\|}$
\item[2.] For all $i>1$: first
\begin{equation}
\tilde{u}_i = v_i - \sum_{k=1}^{i-1} <v_i, u_k> u_k     ,
\label{eq:gs-step}
\end{equation}
then $u_i = \frac{\tilde{u}_i}{\| \tilde{u}_i\|}$.
\end{itemize}
Zero vectors, which occur when there is a linear dependency in $S_V$, are not added to $S_U$. The sum in Eq. \ref{eq:gs-step} gives the orthogonal projection of the vector $v_i$ onto the hyperplane spanned by the $i-1$ vectors $u_k$. Note that in the code, we implement the orthogonalization also for functions (not only arrays), i.e. in ``ortho\_basis.py''. If arrays (matrices) are used, the second step can be calculated via (in case of the standard scalar product)
\begin{equation}
\tilde{\bm{u}}_i = \bm{v}_i - \bm{U}_{i-1}  \bm{U}^T_{i-1}  \bm{v}_i     ,
\end{equation}
where $\bm{U}_{i-1}$ denotes the matrix with columns $\bm{u}_{1}, ..., \bm{u}_{i-1}$. \\
Better numerical stability can be obtained by the \textbf{modified Gram-Schmidt} orthogonalization which replaces the second step of the not-modified one by:
\begin{itemize}
\item[1.] Let $k=1$ and $\tilde{u}^{(1)}_i = v_i$.
\item[2.] Calculate, first
\begin{equation}
\tilde{u}^{(k+1)}_i = \tilde{u}^{(k)}_i -  <\tilde{u}^{(k)}_i, u_k> u_k     ,
\end{equation}
then $u^{(k+1)}_i = \frac{\tilde{u}^{(k+1)}_i}{\| \tilde{u}^{(k+1)}_i\|}$.
\item[3.] If $k=i-1$ stop and $u_i = u^{(k+1)}_i$. Otherwise, let $k = k+1$ and continue with 2. step (of this algorithm).
\end{itemize}

\section{QR decomposition}
The matrix $\bm{A} \in \mathbb{R}^{m \times n}$ is decomposed into an orthogonal matrix $\bm{Q}$ and an upper triangular  matrix $\bm{R}$:
\begin{equation}
\bm{A}=\bm{Q} \bm{R}.
\end{equation}
In our implementation both $\bm{Q}$ and $\bm{R}$ do not need to be square matrices. \\
\subsection{Using Gram-Schmidt orthogonalization}
Using the Gram-Schmidt process an orthogonal basis set is found (for the column space of $\bm{A}$) by transforming the columns of $\bm{A}$. $\bm{Q}$ is given by the new orthogonal vectors. Therefore, we use the size convention $\bm{Q} \in \mathbb{R}^{m \times \text{rank}(A)}$ and $\bm{R} \in \mathbb{R}^{\text{rank}(A) \times n}$. One way to obtain $\bm{R}$ is by calculating $\bm{R} = \bm{Q}^T \bm{A}$. However, in practice the whole matrix multiplication does not need to be carried out because some entries are known to become zero.
\subsection{Using Housholder reflections}
The QR decompositions based on Housholder reflections is known to be more numerically stable than the Gram-Schmidt process (however, I could not verify this statement through the cases I have looked into with the here presented codes.) The Housholder reflection is a linear transformation that reflects a vector $x$ about some (hyper-)plane by
\begin{equation}
x' = x - 2 <x, v> v,
\label{eq:housholder}
\end{equation}
where $v$ is a unit vector orthogonal to the hyperplane. The reflected vector $x'$ has the same length as $x$:
\begin{align}
\| x' \|^2 &= \langle x - 2 <x, v> v, x - 2 <x, v> v \rangle \\
&= <x,x> - 2 \langle x, 2 <x, v> v \rangle + 4 <x, v>^2 \underbrace{<v, v>}_{=1}\\
&= <x, x> - 4 <x, v>^2 + 4 <x, v>^2 \\
&= \| x\|^2.
\end{align}
Now, let us define:
\begin{align}
u &= x - \| x \| e, \label{eq:proj-v1}\\
v &= \frac{u}{\| u \|},
\label{eq:proj-v2}
\end{align}
where $e$ a unit vector. Then, with $<e,e>=1$,
\begin{align}
x' &= x - 2 \frac{<x, x - \| x \| e> }{<x - \| x \|  e, x - \| x \|  e>} (x - \| x \|  e) \\
&= x - 2 \frac{<x, x> - \| x \| <x,  e> }{<x,x> - 2 \| x \| <x, e> x + \| x \|^2 <e, e> } (x - \| x \|  e) \\
&= x - 2 \frac{ \| x \|^2 - \| x \| <x,  e> }{ \| x \|^2 - 2 \| x \| <x, e> x + \| x \|^2  } (x - \| x \|  e) \\
&= x - 2 \frac{1}{2} (x - \| x \|  e) \\
&=  \| x \| e \\
\end{align}
So, interestingly, with the choice in Eq. \ref{eq:proj-v1} and \ref{eq:proj-v2} for $v$ the Housholder reflection becomes a projection of $x$ onto the unit vector $e$. Crucially, if we chose for $e$ the vector $(1, 0, 0, ...)^T$ the Housholder reflection based on this $v$ will transform $x$ into a vector where only one element is non-zero. As a consequence, we can build a clever iterative procedure to apply this reflection (partially) to every column of a matrix $\bm{A}$ and transform $\bm{A}$ to the targeted upper triangular form $\bm{R}$ of the QR decompostion, as described in the following.

Consider matrix form. We can rewrite Eq. \ref{eq:housholder} to
\begin{align}
\bm{x}' = \bm{Q} \bm{x}
\end{align}
with the square matrix
\begin{align}
\bm{Q} = \bm{I} - 2 \bm{v} \bm{v}^T.
\end{align}
In the first iteration step of the QR decomposition we define
\begin{align}
\bm{u}_1 &= \bm{A}_1 - \| \bm{A}_1 \| \bm{e}_1,\\
\bm{v}_1 &= \frac{\bm{u}_1 }{\|\bm{u}_1 \|},
\end{align}
where $\bm{A}_1$ is the first column of $\bm{A}$ and $\bm{e}_1\ = (1, 0,  ..., 0)^T$ a unit vector. Then, with $\bm{Q}_1 = \bm{I} - 2 \bm{v}_1 \bm{v}^T_1$,
\begin{align}
\bm{Q}_1 \bm{A}_1 = \begin{pmatrix}
\| \bm{A}_1 \| \\
0 \\
\vdots\\
0
\end{pmatrix}
\end{align}
and
\begin{align}
\bm{Q}_1 \bm{A} = \begin{pmatrix}
\| \bm{A}_1 \| & \star &\hdots & \star \\
0 \\
\vdots & & \bm{A}' & \\
0
\end{pmatrix}.
\end{align}
In the next step this is repeated for the matrix $\bm{A}'$ ($\bm{A}$ with deleted first row and column), where now $\bm{v}_2$ is based on the first column $\bm{A}_1'$  of $\bm{A}'$, resulting in a matrix $\bm{Q}_2'$. To keep the original matrix sizes, $\bm{Q}_k'$ of any iteration step $k$ is extended by:
\begin{align}
\bm{Q}_k  = \begin{pmatrix}
\bm{I}_{k-1} & 0\\
0 &  \bm{Q}_k'\\
\end{pmatrix}.
\end{align}
After $t=\min(m-1, n)$ iterations, the transpose of the final orthogonal matrix $\bm{Q}$ and the triangular matrix $\bm{R}$ of the QR decomposition are given by
\begin{align}
\bm{Q}^T  = \bm{Q}_t \cdots \bm{Q}_2 \bm{Q}_1
\end{align}
and
\begin{align}
\bm{R}  = \bm{Q}^T \bm{A}.
\end{align}

\section{Eigenvalues and -vectors }
The eigenvalue equation is written
\begin{equation}
\bm{A} \bm{v} = \lambda \bm{v},
\end{equation}
where $\bm{A}$ is a square matrix, the non-zero vector $\bm{v}$ is the eigenvector, and the scalar $\lambda$ is the eigenvalue. The eigenvectors yield the set of vectors which are not rotated in a linear transformation based on $\bm{A}$ but only scaled or inverted in direction.
\subsection{QR algorithm}
The (practical) QR algorithm (without shifts) calculates the eigenvalues of $\bm{A}$ using QR decompositions  iteratively. We initialize $\bm{A}_0 = \bm{A}$ and calculate at each iteration
\begin{equation}
\bm{A}_{k+1} = \bm{R}_k \bm{Q}_k,
\end{equation}
where $\bm{R}_k$ and $\bm{Q}_k$ are obtained from the QR decomposition: $\bm{A}_{k} = \bm{Q}_k \bm{R}_k$. Given that all $\bm{A}_k$ are similar,
\begin{equation}
\bm{A}_{k+1} = \bm{R}_k \bm{Q}_k = \bm{Q}^{-1}_k \bm{Q}_k \bm{R}_k \bm{Q}_k = \bm{Q}^{-1}_k \bm{A}_k \bm{Q}_k,
\end{equation}
they all have the same eigenvalues, as:
\begin{align}
\bm{A}_{k+1} \bm{v} &= \bm{Q}^{-1}_k \bm{A}_k \bm{Q}_k \bm{v} \\
\lambda \bm{v} &= \bm{Q}^{-1}_k \bm{A}_k \bm{Q}_k \bm{v} \\
\lambda \bm{Q}_k \bm{v} &=  \bm{A}_k \bm{Q}_k \bm{v}.
\end{align}
So every eigenvalue $\lambda$ of $\bm{A}_{k+1}$ is also an eigenvalue of $\bm{A}_{k}$, where the corresponding eigenvector of $\bm{A}_{k}$ is  $\bm{Q}_k \bm{v}$. And one can similarly show that every eigenvalue of $\bm{A}_{k}$ is an eigenvalue of $\bm{A}_{k+1}$. \\
Under some conditions, the matrices $\bm{A}_{k}$ will converge against a triangular matrix in the iterative procedure. The eigenvalues of a triangular matrix are given by its diagonal elements. If $\bm{A}$ is symmetric, then the product of the matrices $\cdots \bm{Q}_{2} \bm{Q}_{1} \bm{Q}_{0}$ yield a matrix whose columns are the eigenvectors of $\bm{A}$.
\section{Singular value decomposition}
The matrix $\bm{A} \in \mathbb{R}^{m \times n}$ is decomposed into:
\begin{equation}
\bm{A} = \bm{U} \bm{S} \bm{V}^T,
\end{equation}
where $\bm{U} \in \mathbb{R}^{m \times m}$ and $\bm{V} \in \mathbb{R}^{n \times n}$ are orthogonal matrices and  $\bm{S} \in \mathbb{R}^{m \times n}$ is a diagonal rectangular matrix. The diagonal entries of $\bm{S}$ are given by the singular values of $\bm{A}$, i.e. the square roots of non-negative eigenvalues of $\bm{A}^T \bm{A}$. However, in this code, $\bm{U}$ and $\bm{V}$ are not necessarily square matrices but shaped by $\text{rank}(\bm{A})$ while $\bm{S}$ is square. We compute the decomposition by:
\begin{itemize}
\item[1.] Calculate the eigenvalues and -vectors of $\bm{A}^T \bm{A}$.
\item[2.] Build the matrix $\bm{V}$ of the eigenvectors as columns and the rectangular diagonal matrix $\bm{S}$  with square roots of the non-zero eigenvalues.
\item[3.] Calculate $\bm{U} = \bm{A} \bm{V} \bm{S}^{-1}$.
\end{itemize}
Interestingly, when defining a linear transformation $T$ with $\bm{A}$ from $K^n$ to $K^m$, then
\begin{align}
T(\bm{V}_i) &= \sigma_i \bm{U}_i \ \text{ for } \ i = 1,..., \text{rank}(\bm{A})\\
T(\bm{V}_i) &= 0 \ \text{ for } \ i > \text{rank}(\bm{A}).
\end{align}
This means that one can find orthonormal bases of $K^n$ and $K^m$ such that $T$ maps the $i$-th basis vector of $K^n$ to a non-negative multiple of the $i$-th basis vector of $K^m$, and yields zero for the remaining basis vectors.

\section{Pseudoinvers}
In order to calculate the generalized inverse $\bm{A}^+ \in \mathbb{R}^{n \times m}$ of a matrix  $\bm{A} \in \mathbb{R}^{m \times n}$ that is non-square or not invertible, we use the singular value decomposition $\bm{A} = \bm{U} \bm{S} \bm{V}^T$:
\begin{equation}
\bm{A}^+ = \bm{V} \bm{S}^{-1} \bm{U}^T.
\end{equation}
This definition fulfills the Moore-Penrose conditions that define a pseudoinverse:
\begin{itemize}
\item[1.] $\bm{A} \bm{A}^+ \bm{A} = \bm{A}^+$
\item[2.] $\bm{A}^+ \bm{A} \bm{A}^+ = \bm{A}$
\item[3.] $(\bm{A} \bm{A}^+)^T = \bm{A} \bm{A}^+$
\item[4.] $(\bm{A}^+ \bm{A})^T = \bm{A}^+ \bm{A}$.
\end{itemize}

\section{Cholesky decomposition} \label{sec:cholesky}
The Cholesky decomposition of a positive-definite matrix $A \in \mathbb{n \times n}$ is given by
\begin{equation}
\bm{A} = \bm{L} \bm{L}^T
\end{equation}
where $\bm{L} \in \mathbb{R}^{n \times n}$ is a lower triangular matrix with positive diagonal elements. The decomposition can be shown using the QR decomposition.
A positive-definitem matrix $\bm{A}$ can be written as a product of its square root matrix, $\bm{A} = \bm{B} \bm{B}^T$. With $\bm{B}^T = \bm{Q} \bm{R}$ and $\bm{R} = \bm{L}^T$, we have:
\begin{align}
\bm{A} = \bm{B} \bm{B}^T = \bm{R}^T \bm{Q}^T \bm{Q} \bm{R} = \bm{R}^T \bm{R} = \bm{L} \bm{L}^T.
\end{align}
The elements of $\bm{L}$ are determined by:
\begin{align}
L_{j,j} &= \sqrt{A_{j, j} - \sum_{k=1}^{j-1} L_{j,k}^2},\\
L_{i,j} &= \frac{1}{L_{j,j}} \left( A_{i, j} - \sum_{k=1}^{j-1} L_{i,k} L_{j,k}\right) \quad \text{for} \quad i>j.
\end{align}
The Cholesky decomposition is useful for solving $\bm{A} \bm{x} = \bm{b}$. The equation is rewritten: $\bm{L} \bm{L}^T \bm{x} = \bm{b}$. Then one can solve first $\bm{L}  \bm{y} = \bm{b}$ using forward substitution and next $ \bm{L}^T \bm{x} = \bm{y}$ via back substitution.

\chapter{Quadratic programming} \label{ch:qp}
Consider the optimization problem:
\begin{align}
\min_{\bm{x}} \quad &\frac{1}{2} \bm{x}^T \bm{Q} \bm{x} + \bm{p}^T \bm{x}   \\
\text{subject to} \quad &\bm{A} \bm{x} = \bm{b} \\
                  &\bm{G} \bm{x} \leq \bm{h} \\
                  &\bm{x} \geq \bm{0}.
\end{align}
This problem is solved by quadratic programming. Before that, we transform the inequality constraint $\bm{G} \bm{x} \leq \bm{h}$ to an equality constraint, by introducing slack variables $\bm{\xi} >\bm{0}$ such that  \mbox{$\bm{G} \bm{x} + \bm{\xi} \leq \bm{h}$}. This transforms the constraints to:
\begin{align}
\begin{pmatrix}
\bm{A} & \bm{0} \\
\bm{G} & \bm{I}
\end{pmatrix}
\begin{pmatrix}
\bm{x} \\
\bm{\xi}
\end{pmatrix}
&=
\begin{pmatrix}
\bm{b} \\
\bm{h}
\end{pmatrix}, \\
\begin{pmatrix}
\bm{x} \\
\bm{\xi}
\end{pmatrix}
&\geq \bm{0}.
\end{align}
From now we will assume that this is already taken care of (also the right vectorizations in the implementation) and that it is enough to consider a problem of the form:
\begin{align}
\min_{\bm{x}} \quad &\frac{1}{2} \bm{x}^T \bm{Q} \bm{x} + \bm{p}^T \bm{x}   \\
\text{subject to} \quad &\bm{A} \bm{x} = \bm{b} \\
                  &\bm{x} \geq \bm{0}.
\end{align}
We will solve this problem by the primal-dual interior-point method.\\
First we will reformulate the problem based on minimizing a function with constraints to minimizing a function with further penalty terms, using the method of Lagrange multipliers and introducing a logarithmic barrier function for the non-negative variables to force almost all iterates to stay in the feasible set (space of $\bm{x}$ fulfilling the constraints). Define the Lagrange function:
\begin{align}
\mathcal{L} = \frac{1}{2} \bm{x}^T \bm{Q} \bm{x} + \bm{p}^T \bm{x} - \bm{\lambda}^T (\bm{A} \bm{x} - \bm{b}) - \mu \sum_i \log{x_i},
\end{align}
where $\bm{\lambda} > \bm{0}$ and $\mu > 0$. For a given $\mu$, the minimum point satisfies the Karush-Kuhn-Tucker conditions:
\begin{align}
\nabla_{\bm{x}} \mathcal{L} &=  \bm{Q} \bm{x} + \bm{p} -  \bm{A} \bm{\lambda}^T -  \mu \bm{\chi} = 0,\\
\nabla_{\bm{\lambda}} \mathcal{L} &=  \bm{A} \bm{x} - \bm{b} = 0,
\end{align}
where $\chi_i = 1/x_i$. By defining $\bm{s} = \mu \bm{\chi}$ the equation to be solved becomes:
\begin{align}
\bm{f}(\bm{x}, \bm{\lambda}, \bm{s}) = \bm{0}
\end{align}
with
\begin{align}
\bm{f}(\bm{x}, \bm{\lambda}, \bm{s}) =
\begin{pmatrix}
\bm{Q} \bm{x} + \bm{p} -  \bm{A} \bm{\lambda}^T - \bm{s} \\
\bm{A} \bm{x} - \bm{b} \\
\bm{X} \bm{s} - \bm{\mu}
\end{pmatrix},
\end{align}
where $\bm{X} = \text{diag}(\bm{x})$ is a diagonal with elements of $\bm{x}$ and $\bm{\mu}$ is a vector full of $\mu$. We will solve this problem using Newtons method, i.e. an iterative procedure to find the root, where at each step, $x$ is updated by $\Delta x$, with $f(x)' \Delta x= f(x)$. The Jacobian matrix (gradients of $\bm{f}$) is denoted by:
\begin{align}
\bm{J}(\bm{x}, \bm{\lambda}, \bm{s}) =
\begin{pmatrix}
\bm{Q} &  -\bm{A}^T & - \bm{I} \\
\bm{A} & \bm{0} & \bm{0} \\
\bm{S} & \bm{0} & \bm{X}
\end{pmatrix},
\end{align}
where $\bm{S}=\text{diag}(\bm{s})$.
Then, at each iteration $k$, we solve the linear system
\begin{align}
\bm{J}(\bm{t}^k) \bm{d} = \bm{f}(\bm{t}^k),
\end{align}
where $\bm{t}^k = (\bm{x}^k, \bm{\lambda}^k, \bm{s}^k)$, and then update $\bm{t}^{k+1} = \bm{t}^{k} + \alpha \bm{t}^{k}$. Note that this gives a minimum point $\bm{x}_\mu$ only for a given $\mu$. However, by decreasing $\mu$, $x_\mu$ will converge towards the solution of the problem. In practice, $\mu$ (and also $\alpha$) could be updated at each iteration $k$ of the Newton(-like) method.

\chapter{Machine learning}
We consider a data set of $n$ input-output pairs $\{(\bm{x}_1, y_1), ..., (\bm{x}_n, y_n)\}$. $y_i$ represents the value of a targeted property of the data point $i$ to be predicted by a vectorial input (representation of the data point) $\bm{x}_i \in \mathbb{R}^m$ of the data point. Often, the goal is to identify a function $f$ out of a function space $\mathcal{F}$ in order to describe the input-output relationship. A general formulation of the ML optimization problem is given by
\begin{equation}
\argmin_{f \in \mathcal{F}} \sum_{i=1}^n L(f(\bm{x}_i), y_i)+\lambda r(f),
\label{eq:opt-ml}
\end{equation}
where $L$ is a loss function and $r(f)$ is a measurement of the \textit{complexity} of $f$. The parameter $\lambda \geq 0$ regulates the compromise between the accuracy and a low complexity of the model. One reason for regularization is to avoid \textit{overfitting} (fitting \textit{unimportant fluctuations}) in order to increase the prediction accuracy on data points out of the training set. The choice of $\mathcal{F}$, $L$ and $k$ determines the ML method. All supervised-learning based methods used in the code are based on the squared error loss $ L(f(\bm{x}_i),  y_i) = (f(\bm{x}_i)- y_i)^2$.\\

In the following sections we write many equations in matrix form and yield the $n$ data points as a target vector $\bm{y} \in \mathbb{R}^n$ and an input matrix $\bm{X} \in \mathbb{R}^{n \times m}$. We will assume that $\bm{X}$ is standardized to have zero mean and variance one, for numerical or conceptional reasons.

\section{Linear regression}
The most basic and widely used machine-learning method is linear regression. One reason is that many linear-regression problems can be solved by linear algebra and convex optimization. The least-squares problem
\begin{equation}
\argmin_{\bm{c} \in \mathbb{R}^m} \| \bm{y} - \bm{X} \bm{c} \|^2
\label{eq:ls}
\end{equation}
formulates the fundament of linear regression analysis determining the regression coefficients $\bm{c}$ of a linear model $f(\bm{x}) = \bm{x}^T \bm{c}$. Its solution is given by the closed-form expression $\bm{c}^* = (\bm{X}^\text{T}\bm{X})^{-1}\bm{X}^\text{T} \bm{y}$, which gives the orthogonal projection $\text{Proj}_{\mathcal{C}(\bm{X})}(\bm{y}) = \bm{X} \bm{c}^*$ of $\bm{y}$ onto the column space $\mathcal{C}(\bm{X}) = \{ \bm{X} \bm{c} \in \mathbb{R}^n  : \bm{c} \in \mathbb{R}^m \}$ of $\bm{X}$.\\
The fact that the orthogonal projection minimizes the Euclidean distance in Eq. \ref{eq:ls} is clear because any other vector $\bm{X} (\bm{c}^* - \bm{k})$ with nonzero $\bm{k}$ will have a higher distance to $\bm{y}$ due to the orthogonality \mbox{$<\text{Proj}_{\mathcal{C}(\bm{X})}(\bm{y}), \bm{X} \bm{k}> = 0$} (using Pytagoras theorem):
\begin{align}
\| \bm{y} -  \text{Proj}_{\mathcal{C}(\bm{X})}(\bm{y}) +\bm{X}\bm{k}\|^2 &= \| \bm{y} -  \text{Proj}_{\mathcal{C}(\bm{X})}(\bm{y}) \|^2 + \| \bm{X}\bm{k} \|^2 \\
&> \| \bm{y} -  \text{Proj}_{\mathcal{C}(\bm{X})}(\bm{y}) \|^2.
\end{align}

We can calculate the least squares solution using the singular value decomposition by (note that we use square $\bm{S}$):
\begin{align}
\bm{c}^* &= (\bm{X}^\text{T}\bm{X})^{-1}\bm{X}^\text{T} \bm{y} \\
&= (\bm{V} \bm{S} \bm{U}^T \bm{U} \bm{S} \bm{V}^T)^{-1}\bm{X}^\text{T} \bm{y}\\
&= (\bm{V} \bm{S}\bm{S} \bm{V}^T)^{-1}\bm{X}^\text{T} \bm{y}\\
&= (\bm{V} \bm{S}^{-1}\bm{S}^{-1} \bm{V}^T )\bm{X}^\text{T} \bm{y}\\
&= \bm{V} \bm{S}^{-1}\bm{S}^{-1} \bm{V}^T \bm{V} \bm{S} \bm{U}^T \bm{y} \\
&= \bm{V} \bm{S}^{-1} \bm{U}^T \bm{y}.
\label{eq:ls-svd}
\end{align}

In order to avoid overfitting (and also numerical instability), the problem \ref{eq:ls} is often extended by an $\ell_2$ penalty \mbox{$\lambda \| c \|^2_2 = \lambda \| c \|^2$} (linear ridge regression):
\begin{equation}
\argmin_{\bm{c} \in \mathbb{R}^m} \| \bm{y} - \bm{X} \bm{c} \|^2+ \lambda \| \bm{c} \|^2.
\label{eq:ridge}
\end{equation}
The solution to this problem is given by  $\bm{c} = (\bm{X}^\text{T}\bm{X}+\lambda \bm{I})^{-1}\bm{X}^\text{T} \bm{y}$, where $\bm{I} \in \mathbb{R}^{m \times m}$ is the identity matrix.
In terms of singular value decomposition, we can write the solution:
\begin{align}
\bm{c}^* &= (\bm{X}^\text{T}\bm{X} + \lambda \bm{I})^{-1}\bm{X}^\text{T} \bm{y} \\
&= (\bm{V} \bm{S} \bm{U}^T \bm{U} \bm{S} \bm{V}^T + \lambda \bm{I})^{-1}\bm{X}^\text{T} \bm{y}\\
&= (\bm{V} \bm{S}\bm{S} \bm{V}^T + \lambda \bm{I})^{-1}\bm{X}^\text{T} \bm{y}\\
&= (\bm{V} \bm{S}\bm{S} \bm{V}^T + \lambda \bm{I}\bm{V} \bm{V}^T)^{-1}\bm{X}^\text{T} \bm{y}\\
&= (\bm{V} [\bm{S}\bm{S} + \lambda \bm{I}] \bm{V}^T)^{-1}\bm{X}^\text{T} \bm{y}\\
&= \bm{V} (\bm{S}\bm{S} + \lambda \bm{I})^{-1} \bm{V}^T \bm{X}^\text{T} \bm{y}\\
&= \bm{V} (\bm{S}\bm{S} + \lambda \bm{I})^{-1} \bm{V}^T \bm{V} \bm{S} \bm{U}^T \bm{y}\\
&= \bm{V} (\bm{S}\bm{S} + \lambda \bm{I})^{-1} \bm{S} \bm{U}^T \bm{y}.
\label{eq:ridge-svd}
\end{align}
Both the least-squares and ridge solution are written in the form of $\bm{V} \bm{D} \bm{U}^T \bm{y}$ and distinguished only by the diagonal matrix $\bm{D}$. The elements of $\bm{D}$ are given by
\begin{equation}
d_{ii} = \frac{1}{s_{ii}}
\end{equation}
and
\begin{equation}
d_{ii} = \frac{s_{ii}}{s_{ii}^2 + \lambda},
\end{equation}
respectively. When the $s_{ii}$ come close to zero, the inverse singular values explode. If $\lambda>0$, the solution of the ridge regression becomes not only more stable, but also the coefficients of correlated columns (features) of $\bm{X}$ become more strongly shrinkaged. In particular, $|s_{ii}|$ is small when features are correlated, because when being close to collinearity, $\| \bm{X} \bm{v}_i \|$ is small:
\begin{align}
\| \bm{X} \bm{v}_i \|^2 &= \bm{v}_i^T \bm{X}^T \bm{X}  \bm{v}_i \\
&= \bm{v}_i^T \bm{V} \bm{S} \bm{U}^T \bm{U} \bm{S} \bm{V}^T  \bm{v}_i \\
&= \bm{v}_i^T \bm{V} \bm{S}\bm{S} \bm{V}^T  \bm{v}_i \\
&= s_{ii}^2.
\end{align}

\section{Non-negative linear regression}
Assume we want solve the linear regression (ridge regression) problem of Eq. \ref{eq:ridge}, however, with the constraint that the coefficients $\bm{c}$ should be non-negative:
\begin{equation}
\argmin_{\bm{c} \in \mathbb{R}^m} \| \bm{y} - \bm{X} \bm{c} \|^2+ \lambda \| \bm{c} \|^2
\quad \text{subject to} \quad   \bm{c} \geq \bm{0}.
\label{eq:nonnegative-ridge}
\end{equation}
Rewriting the cost function in Eq. \ref{eq:nonnegative-ridge} yields:
\begin{align}
\| \bm{y} - \bm{X} \bm{c} \|^2+ \lambda \| \bm{c} \|^2
&= (\bm{y} - \bm{X} \bm{c})^T (\bm{y} - \bm{X} \bm{c}) + \lambda \bm{c}^T \bm{c}\\
&= \bm{y}^T \bm{y} - \bm{y}^T \bm{X} \bm{c} - (\bm{X} \bm{c})^T \bm{y}  + (\bm{X} \bm{c})^T  \bm{X} \bm{c} + \lambda \bm{c}^T \bm{c} \\
&= \bm{y}^T \bm{y} -2 \bm{y}^T \bm{X} \bm{c}  + \bm{c}^T \bm{X}^T  \bm{X} \bm{c} + \lambda \bm{c}^T \bm{c}\\
&= \bm{y}^T \bm{y} -2 \bm{y}^T \bm{X} \bm{c}  + \bm{c}^T (\bm{X}^T  \bm{X} + \lambda \bm{I}) \bm{c} \\
&= \bm{y}^T \bm{y} +2 \bm{p}^T \bm{c}  + \bm{c}^T \bm{Q} \bm{c},
\label{eq:nonnegative-cost}
\end{align}
where $\bm{Q} = \bm{X}^T  \bm{X} + \lambda \bm{I} $ and $\bm{p} =- \bm{X}^T \bm{y}$. Substituting, Eq. \ref{eq:nonnegative-cost} into Eq. \ref{eq:nonnegative-ridge} gives (in equivalent form):
\begin{equation}
\argmin_{\bm{c} \in \mathbb{R}^m} \frac{1}{2} \bm{c}^T \bm{Q} \bm{c} +\bm{p}^T \bm{c} \quad \text{subject to} \quad   \bm{c} \geq \bm{0}.
\label{eq:nonnegative-ridge}
\end{equation}
This problem can be solved by quadratic programming described in Chapter \ref{ch:qp}.

\section{Orthogonal matching pursuit}
In some applications it is desirable to find linear models $f(\bm{x}) = \bm{x}^T \bm{c}$ where some, many, or most coefficients (elements of $\bm{c}$) are zero. Apart from the least absolute shrinkage and selection operator (LASSO, $\ell_1$ regularization) the orthogonal-matching-pursuit (OMP) algorithm provides a way to find such a model. It is a greedy algorithm which adds at each iteration $k$ a column of $\bm{X}$ with non-zero coefficient to the model (where all columns with zero $c_i$ are removed in some sense) that is based on all added columns during the iterative procedure. The non-zero coefficients are determined via least-squares regression. The added column $\bm{x}^r$ provides that column among all columns $\bm{x}^i$ of $\bm{X}$ which is the closest to the current residual vector $\bm{R} = \bm{y} - f_k$
\begin{align}
\|\bm{R} - \bm{x}^r c_r \|^2 \leq \|\bm{R} - \bm{x}^i c_i \|^2 \ \quad \forall i
\label{eq:omp-close}
\end{align}
where $c_i$ denotes the corresponding least squares solutions in fitting $\bm{R}$. First, let us add $\|\bm{x}^r c_r\|_2^2+ \|\bm{x}^i c_i\|_2^2$ to both sides of Eq. \ref{eq:omp-close}:
\begin{align}
\|\bm{R} - \bm{x}^r c_r \|^2 + \|\bm{x}^r c_r\|^2+ \|\bm{x}^i c_i\|^2 \leq \|\bm{R} - \bm{x}^i c_i \|^2 + \|\bm{x}^r c_r\|^2+ \|\bm{x}^i c_i\|^2
\end{align}
Recall that the least squares solution $c_i$ provides the orthogonal projection $\bm{x}^i c_i$ of $\bm{R}$ onto $\bm{x}_i$. Thus, $\bm{x}^i c_i$ and $\bm{R} - \bm{x}^i c_i$ are orthogonal, i.e. $\langle \bm{R} - \bm{x}^i c_i , \bm{x}^i c_i \rangle = 0$, see Eq. \ref{eq:orthogonal-proj-zero}. This allows us to use the Pythagorean theorem:
\begin{align}
\|\bm{R} - \bm{x}^r c_r + \bm{x}^r c_r\|^2+ \|\bm{x}^i c_i\|^2  &\leq \|\bm{R} - \bm{x}^i c_i\ + \bm{x}^i c_i\|^2 + \|\bm{x}^r c_r\|^2 \\
\Leftrightarrow \|\bm{R}\|^2+ \|\bm{x}^i c_i\|^2          &\leq \|\bm{R}\|^2 + \|\bm{x}^r c_r\|^2 \\
\Leftrightarrow \|\bm{x}^i c_i\|^2 &\leq  \|\bm{x}^r c_r\|^2 \\
\Leftrightarrow \| \bm{x}^i\|^2 | c_i|^2  &\leq  \|\bm{x}^r \|^2 | c_r|^2\\
\end{align}
Recall that the orthogonal projection coefficient is given by
\begin{equation}
c_i =  \frac{\langle \bm{R}, \bm{x}^i \rangle}{\|\bm{x}^i\|^2} .
\end{equation}
Using that and assuming furthermore that all columns have same lengths $\|\bm{x}^i\|$ (that is the case if they are for example standardized), we obtain:
\begin{align}
\| \bm{x}^i\|^2 \frac{|\langle \bm{R}, \bm{x}^i \rangle|^2}{\|\bm{x}^i\|^4}  &\leq  \|\bm{x}^r \|^2  \frac{|\langle \bm{R}, \bm{x}^r \rangle|^2}{\|\bm{x}^r\|^4} \\
\Leftrightarrow  \frac{|\langle \bm{R}, \bm{x}^i \rangle|^2}{\|\bm{x}^i\|^2}  &\leq   \frac{|\langle \bm{R}, \bm{x}^r \rangle|^2}{\|\bm{x}^r\|^2} \\
\Leftrightarrow  |\langle \bm{R}, \bm{x}^i \rangle|^2  &\leq   |\langle \bm{R}, \bm{x}^r \rangle|^2 \\
\Leftrightarrow  |\langle \bm{R}, \bm{x}^i \rangle|  &\leq   |\langle \bm{R}, \bm{x}^r \rangle|.
\end{align}
This means that in each iteration, the closest column $\bm{x}^i$ to the residual $\bm{R}$ is given by the highest absolute value of the projection score $|\langle \bm{R}, \bm{x}^i \rangle|$, i.e. the calculation expense at each iteration is only given by calculating $\bm{X}^T \bm{R}$ and determining the index with the highest absolute value.\\
The OMP algorithm is given by:
\begin{enumerate}
\item Initialize $\bm{R}_1 =\bm{y}$ and $\mathcal{X}=\emptyset$ the set of saved columns. Chose a target number $q$ of selected columns (or nonzero coefficients) for the linear model and let the iteration counter $k=1$.
\item Find the closest $\bm{x}^i$ to $\bm{R}_k$ and add it to $\mathcal{X}$.
\item Build the matrix $\tilde{\bm{X}}$ with all $\bm{x}^i \in \mathcal{X}$. Calculate $\bm{c}^* = \argmin_{\bm{c} \in \mathbb{R}^{k}} \| \bm{y} - \tilde{\bm{X}} \bm{c} \|^2$. Let $\bm{R}_{k+1} =\bm{y} - \tilde{\bm{X}} \bm{c}^*$.
\item If $k=q$, stop. Otherwise set $k=k+1$ and return to the 2. step.
\end{enumerate}

\section{Kernel ridge regression} \label{sec:krr}
The kernel ridge regression is a generalization of the linear ridge regression \ref{eq:ridge} towards considering nonlinar models $f$. Consider again the general optimization problem in Eq. \ref{eq:opt-ml} with the squared error loss $ L(f(\bm{x}_i),  y_i) = (f(\bm{x}_i)- y_i)^2$ and, moreover, $r(f) = \| f \|^2_\mathcal{F}$ for a reproducing kernel Hilbert space $\mathcal{F}$\footnote{A Hilbert space is a reproducing kernel Hilbert space if it has a bounded linear evaluation function. For more details we refer to the corresponding literature of functional analysis.}. Then, according to the representer theorem, the solution to the optimization problem has the form
\begin{equation}
f(\cdot) = \sum_{i=1}^n \alpha_i k(\cdot, \bm{x}_i)
\label{eq:krr-model}
\end{equation}
where $k: \mathcal{X} \times \mathcal{X} \rightarrow \mathbb{R}$ is the associated positive-definite real-valued kernel on our input space $\mathcal{X}$. It can be shown that the coefficients $\{ \alpha_i \}$ are a closed-form solution $\bm{\alpha} = (\bm{K}+\lambda \bm{I})^{-1} \bm{y}$ to the optimization problem (kernel ridge regression)
\begin{equation}
\argmin_{\bm{\alpha} \in \mathbb{R}^n} \| \bm{y} - \bm{K} \bm{\alpha} \|^2+ \lambda  \bm{\alpha}^T \bm{K} \bm{\alpha},
\label{eq:kernel-ridge}
\end{equation}
where $\bm{K} \in \mathbb{R}^{n \times n}$ represents the matrix with elements $K_{ij} = k(\bm{x}_i, \bm{x}_j)$. The fact that the non-linear function \ref{eq:krr-model} is learnt by a linear learning algorithm, i.e. the closed-form expression, is considered as the \textit{kernel trick}. For a linear kernel $k(\bm{x}_i, \bm{x}_j) = \bm{x}_i \cdot  \bm{x}_j$ the \mbox{model \ref{eq:krr-model}} becomes a linear function and equals the linear model that results from the optimization problem \ref{eq:ridge} of the linear ridge regression. For instance, the kernel ridge regression can alternatively be derived by kernelizing the linear ridge regression and, moreover, introducing a potentially nonlinear feature map $\phi$ such that $k(\bm{x}_i, \bm{x}_j) = < \phi(\bm{x}_i),   \phi(\bm{x}_j)>$. However, $\phi$ does not need to be known and rather the kernel is specified directly. A typical (nonlinear) kernel is given by the Gaussian kernel $k(\bm{x}_i, \bm{x}_j) = \exp(- \frac{\|\bm{x}_i -  \bm{x}_j\|^2}{2 \sigma^2})$. Note that the kernel function is often viewed as a similarity measurement between two data points.\\
In practice, we solve the linear system $\bm{K} \bm{\alpha}=\bm{y}$ using the Cholesky decomposition and forward and back substitution as described in section \ref{sec:cholesky}.

\section{Logistic regression}
Consider the binary classification problem with $y_i \in \{0, 1 \}$. The logistic regression model
\begin{align}
f(\bm{x}, y) = P(y| \bm{x}; \bm{c})= P(Y=1| \bm{x}; \bm{c})^y  P(Y=0| \bm{x}; \bm{c})^{(1-y)}
\end{align}
calculates the probability of a data point $\bm{x}$ for having the label $y$, where
\begin{align}
 P(Y=1| \bm{x}; \bm{c}) &= \frac{1}{1 + e^{- \bm{x}^T \bm{c}}} \\
P(Y=0| \bm{x}; \bm{c}) &= 1 - P(Y=1| \bm{x}; \bm{c}) = 1 - \frac{1}{1 + e^{- \bm{x}^T \bm{c}}}.
\end{align}
Assuming that the $\bm{x}_i$ are independent and identically distributed random variables, their joint probability distribution is given by
\begin{equation}
P(y_1, y_2, ..., y_n| \bm{x_1}, \bm{x}_2, ..., \bm{x}_n| \bm{c}) = \prod_{i=1}^n P(y_i| \bm{x_i}; \bm{c}).
\end{equation}
The right side is equal to the likelihood function $L(\bm{c}|y_1, y_2, ..., y_n| \bm{x_1}, \bm{x}_2, ..., \bm{x}_n)$ which, however, treats $\bm{c}$ as a parameter and the input-output pairs as given. In order to determine $\bm{c}$ such that the observed data is most probable, $L$ is maximized or, in practice, $\log L$:
\begin{align}
 \log L(\bm{c}|y_1, y_2, ..., y_n| \bm{x_1}, \bm{x}_2, ..., \bm{x}_n) &= \log \prod_{i=1}^n P(y_i| \bm{x_i}; \bm{c}) \\
 &=  \sum_{i=1}^n \log P(y_i| \bm{x_i}; \bm{c}).
\end{align}
We solve the optimization problem
\begin{align}
\max_{\bm{c} \in \mathbb{R}^m} \sum_{i=1}^n \log P(y_i| \bm{x_i}; \bm{c})
\end{align}
via gradient descent.

\section{Support vector machines}
Consider the binary classification problem with $y_i \in \{ 1, -1 \}$. Let us target the goal to find some hyperplane defined by
\begin{equation}
\bm{x} \bm{w} - b =0
\label{eq:hyper}
\end{equation}
that separates the two classes in the input space of the data represented by $\bm{X}$. In Eq. \ref{eq:hyper}, $\bm{w}$ denotes the normal vector to the hyperplane and $\frac{b}{\| \bm{w} \|}$ the offset of the hyperplane from the origin along $\bm{w}$. Assume for the moment that the data is linearly separable. The SVM determines $\bm{w}$ and $b$ (there is at least one with $y=1$ and one with $y=-1$) such that the closest data points (there is at least one with $y=1$ and one with $y=-1$) to the hyperplane have a distance to it of $\frac{1}{\| \bm{w} \|}$ along $\bm{w}$ if $y=1$ and along $-\bm{w}$ if $y=-1$:
\begin{align}
\bm{x}_i \bm{w} - b &\geq 1 \quad   \text{if} \quad   y_i=1,\\
\bm{x}_i \bm{w} - b &\leq -1 \quad   \text{if} \quad   y_i=-1
\end{align}
for all data points $i$. This can be rewritten:
\begin{align}
y_i(\bm{x}_i \bm{w} - b) &\geq 1.
\end{align}
That means the closest data points $\bm{x}_i$ (support vectors) lie on two hyperplanes with the same normal vector $\bm{w}$ which define a margin of $\frac{2}{\| \bm{w} \|}$ between the two classes. \\
In case the data is not linearly separable, we need to introduce a set of slack variables $\xi_i \geq 0$ which allow for misclassified data points:
\begin{align}
\bm{x}_i \bm{w} - b &\geq 1 - \xi_i\quad   \text{if} \quad   y_i=1,\\
\bm{x}_i \bm{w} - b &\leq -1 + \xi_i \quad   \text{if} \quad   y_i=-1
\end{align}
for all data points $i$. This can be rewritten:
\begin{align}
y_i(\bm{x}_i \bm{w} - b) &\geq 1 - \xi_i.
\end{align}
Together with the further criterion that the margin between the two classes should be maximal, the SVM-optimization problem becomes:
\begin{align}
\min_{\bm{w}, b, \bm{\xi}} \frac{1}{2} \| \bm{w} \|^2 + C \sum_{i=1}^n \xi_i \quad \text{subject to} \quad   y_i(\bm{x}_i \bm{w} - b) &\geq 1 - \xi_i \quad \text{and} \quad \xi_i \geq 0, \quad  \forall i.
\end{align}
Note that the optimization problem is fully described by those data points that do not fulfill $y_i(\bm{x}_i \bm{w} - b) > 1$. A consequence of minimizing the slack variables $\xi_i$ is that they will be zero for data points that are on the correct side of the hyperplane margin, i.e. $y_i(\bm{x}_i \bm{w} - b) \geq 1$. In case of misclassifed data points and also ones that are correctly classified but inside the margin, the consequence of minimizing the $\xi_i$ is the that their sizes are not larger than the respective distances of the positions of the data point along $\bm{w}$ to the correct side of the hyperplane margin. However, given the side constraint that $\xi_i$ should be at least the distance to the correct side of the hyperplane margin, we have $\xi_i = 1 - y_i(\bm{x}_i \bm{w} - b) $ for misclassified data points. With a large $C$, we will put more weight on minimizing the slack variables $\xi_i$. As a result, increasing $C$ will lead to decreasing the margin of the hyperplane, because the distance of the data points which are not on the correct side of the margin needs to become small (or just because less weight is put on minimizing $\| \bm{w} \|$, i.e. the inverse of the margin). If the model complexity allows such as with a flexible model based on some (nonlinear) kernel (this will be introduced a few paragraphs later), instead of a linear model as discussed till now, increasing $C$ will lead to a more \textit{wiggly} that decreases the number of mislassified data points, deacreasing the number of $\xi_i>0$. Decreasing $C$ will lead to a \textit{coarser} model that allows for misclassified data points (taken care of by sufficiently large $\xi_i$), however, might be more robust when predicting the labels of new data points. \\
We will solve this problem using the method of Lagrange multipliers, where the multipliers $\alpha_i \geq 0$ and $\mu_i \geq 0$, take care of the side constraints. The respective Lagrangian is given by:
\begin{align}
\mathcal{L} &= \frac{1}{2} \| \bm{w} \|^2 + C \sum_{i=1}^n \xi_i - \sum_{i=1}^n \alpha_i[ y_i(\bm{x}_i \bm{w} - b) -1 + \xi_i] - \sum_{i=1}^n \mu_i \xi_i \label{eq:first}\\
&= \frac{1}{2} \| \bm{w} \|^2 + C \sum_{i=1}^n \xi_i - \bm{w} \sum_{i=1}^n \alpha_i y_i \bm{x}_i  + b \sum_{i=1}^n \alpha_i y_i + \sum_{i=1}^n \alpha_i - \sum_{i=1}^n \alpha_i \xi_i - \sum_{i=1}^n \mu_i \xi_i \\
&= \frac{1}{2} \| \bm{w} \|^2 - \bm{w} \sum_{i=1}^n \alpha_i y_i \bm{x}_i  + b \sum_{i=1}^n \alpha_i y_i+ \sum_{i=1}^n \alpha_i + \sum_{i=1}^n (C - \alpha_i - \mu_i) \xi_i
\label{eq:svm-lagrangian-primary}
\end{align}
Minimizing the Lagrangian with respect to $\bm{w}$ and $b$ and $\xi_i$ yields:
\begin{align}
\frac{\partial \mathcal{L}_\text{P}}{\partial \bm{w}} = 0 \quad  \Rightarrow \bm{w} = \sum_{i=1}^n \alpha_i y_i \bm{x}_i, \label{eq:svm-sc1}\\
\frac{\partial \mathcal{L}_\text{P}}{\partial b} = 0 \quad  \Rightarrow \sum_{i=1}^n \alpha_i y_i  = 0. \label{eq:svm-sc2}\\
\frac{\partial \mathcal{L}_\text{P}}{\partial \xi_i} = 0 \quad  \Rightarrow C - \alpha_i - \mu_i =0. \label{eq:svm-sc3}
\end{align}
By substituting Eq. \ref{eq:svm-sc1}, \ref{eq:svm-sc2}, and \ref{eq:svm-sc3} into \ref{eq:svm-lagrangian-primary}, we obtain a new Lagrangian:
\begin{align}
\mathcal{L}_\text{D} &= \frac{1}{2} \| \bm{w} \|^2 - \bm{w} \sum_{i=1}^n \alpha_i y_i \bm{x}_i  + b \underbrace{\sum_{i=1}^n \alpha_i y_i}_{=0}+ \sum_{i=1}^n \alpha_i + \sum_{i=1}^n \underbrace{(C - \alpha_i - \mu_i)}_{=0} \xi_i\\
&= \frac{1}{2} \| \bm{w} \|^2 - \bm{w} \sum_{i=1}^n \alpha_i y_i \bm{x}_i +\sum_{i=1}^n \alpha_i \\
&= \frac{1}{2} \sum_{i,j} \alpha_i \alpha_j y_i y_j \bm{x}_i \bm{x}_j- \sum_{i,j} \alpha_i \alpha_j y_i y_j \bm{x}_i \bm{x}_j  + \sum_{i=1}^n \alpha_i\\
&= -\frac{1}{2} \sum_{i,j} \alpha_i \alpha_j y_i y_j \bm{x}_i \bm{x}_j  + \sum_{i=1}^n \alpha_i\\
&= -\frac{1}{2} \bm{\alpha}^T \bm{H} \bm{\alpha}  + \sum_{i=1}^n \alpha_i,
\label{eq:svm-lagrangian-dual}
\end{align}
where $H_{ij} = y_i y_j \bm{x}_i \bm{x}_j $. With that and the fact that Eq. \ref{eq:svm-sc3} and $\mu_i > 0$ lead to $ 0 \leq \alpha_i \leq C$, the optimization problem becomes the dual problem
\begin{align}
\max_{\bm{\alpha}} \sum_{i=1}^n \alpha_i -\frac{1}{2} \bm{\alpha}^T \bm{H} \bm{\alpha}  \quad \text{subject to} \quad   0 \leq \alpha_i \leq C \ \ \forall i \quad \text{and} \quad \sum_{i=1}^n \alpha_i y_i  = 0,
\end{align}
which can be solved via quadratic programming, see Chapter \ref{ch:qp}. Recall the Lagrange duality theory: the primal problem is given by
\begin{align}
\min_{\bm{w}, b, \bm{\xi}}\left( \max_{\bm{\alpha}\geq0, \bm{\mu}\geq0} \mathcal{L}(\bm{w}, b, \bm{\alpha}, \bm{\mu}, \bm{\xi}) \right) = \min_{\bm{w}, b, \bm{\xi}} \mathcal{L_\text{P}}
\end{align}
and the dual one by
\begin{align}
\max_{\bm{\alpha}\geq0, \bm{\mu}\geq0}  \left(  \min_{\bm{w}, b, \bm{\xi}} \mathcal{L}(\bm{w}, b, \bm{\alpha}, \bm{\mu}, \bm{\xi}) \right) = \max_{\bm{\alpha}\geq0, \bm{\mu}\geq0}  \mathcal{L_\text{D}}.
\end{align}
Note that when considering the side constraints, the maximum of the $\mathcal{L_\text{D}}$ with respect to $\bm{\alpha}$ (and $\bm{\mu}$) is $\frac{1}{2} \|\bm{w} \|^2$. Assume for a given $\bm{w}$ and $b$, some data points fulfill $y_i(\bm{x}_i \bm{w} - b) > 1 -\xi_i$ (then also $y_i(\bm{x}_i \bm{w} - b) > 1$ because $\xi_i$ is minimized). Then for the $\alpha_i$-based term in Eq. \ref{eq:first}, we have:
\begin{align}
 - \sum_{i=1}^n \alpha_i[ y_i(\bm{x}_i \bm{w} - b) -1 + \xi_i]  <0.
\end{align}
Maximizing with respect to $\alpha_i$ yields $\alpha_i=0$ for these (non-support-vector) data points.
We define the set $S$ of support vectors $\bm{x_s}$ which are characterized by $\alpha_s>0$. We rewrite the weight vector as a linear combination of the support vectors, i.e. we rewrite Eq. \ref{eq:svm-sc1} to
\begin{align}
\bm{w} = \sum_{s \in S}^n \alpha_s y_s \bm{x}_s.
\end{align}
Given furthermore that the support vectors fulfill $y_i(\bm{x}_i \bm{w} - b) = 1$, we obtain from
\begin{align}
y_i &= \underbrace{y_i^2}_{=1}(\bm{x}_i \bm{w} - b) \\
& = \bm{x}_i \sum_{s \in S}^n \alpha_s y_s \bm{x}_s - b
\end{align}
a way to calculate $b$:
\begin{align}
b =  \sum_{s \in S}^n \alpha_s y_s \bm{x}_i \bm{x}_s - y_i.
\end{align}

The model can be extended to a nonlinear decision boundary by kernelizing the linear problem (see kernel ridge regression section) by introducing a potentially nonlinear feature map $\phi(\bm{x}_i)$ and a kernel given by an inner product $k(\bm{x}_i ,\bm{x}_j) = <\phi(\bm{x}_i) , \phi(\bm{x}_j)>$. Then $H_{ij} = y_i y_j k(\bm{x}_i, \bm{x}_j)$ and the original problem can be reproduced by  considering a linear kernel $k(\bm{x}_i ,\bm{x}_j) = \bm{x}_i \bm{x}_j$.

\end{document}


\documentclass[
  xhtml,%
  use filename%
]{internet}

\usepackage{tutorial}
\usepackage{hyperref}

\title{A Christmas Card}
\date{\today}
\begin{document}
\maketitle

\section{Introduction}

There are several built-in shapes that you can use to make a scene on the screen, but sometimes you need a little more flexibility.
In this tutorial, we'll introduce \verb+path+s and see how we can use them to make almost any shape.

\section{The Basics}

A \verb+path+ is what it sounds like: a path on the screen.
It is built up out of pieces like lines and arcs.
To make a path, you first give it a name.
Then you add pieces to it.

For example, in the following code we define a simple shape in the \verb+setup+ function.

\begin{verbatim}
function setup()
  print("hello world")
  shape = path()
  shape:moveTo(200,200)
  shape:lineTo(300,200)
  shape:lineTo(200,300)
end

function draw()
  background(40,40,50)
  fill(150,200,30)
  stroke(200,30,150)
  strokeWidth(10)
  rect(20,20,100,100)
  shape:fill()
  shape:draw()
end
\end{verbatim}

Defining the shape is the lines:

\begin{verbatim}
shape = path()
shape:moveTo(200,200)
shape:lineTo(300,200)
shape:lineTo(200,300)
\end{verbatim}

The first line names our path \verb+shape+.
The second line \emph{moves} our path to the point \((200,200)\).
This tells our path where to start.
The third line then adds a line to the path, starting at its current point (which is \((200,200)\)) and ending at \((300,200)\).
Every time we add a piece to the path, it starts where the last one ended.
Then the last line adds another line from \((300,200)\) to \((200,300)\).

This sets up the shape.
To put something on the screen, we have the lines in the \verb+draw+ function:

\begin{verbatim}
shape:fill()
shape:draw()
\end{verbatim}

The first line \emph{fills} the path with the current \verb+fill+ colour.
As our path is not closed, it joins it up to close it before filling it.
The second line draws the path with the current \verb+stroke+ colour and \verb+strokeWidth+.
(The path isn't closed when it is drawn.)

\section{Drawing a Tree}

We can draw a Christmas tree using paths.
Let's start with naming our path: put \verb+tree = path()+ in the \verb+setup+ function.
We'll define it so that the top of the tree is at the point \verb+(0,0)+ and then use \emph{transformations} to move it to where we want it.
To make our tree start at \verb+(0,0)+, we write \verb+tree:moveTo(0,0)+.

We want to start with a nice curve.
We can do that with an arc.
To add an arc, we need to fix a centre, a radius, and starting and ending angles.
So to draw a quarter circle starting at the top and sweeping down, we could use:

\begin{verbatim}
tree:arc(-100,0,100,0,-90,true)
\end{verbatim}

This arc is drawn from a circle centred at \verb+(-100,0)+ and has radius \verb+100+.

We'll want another arc the other side.

\begin{verbatim}
tree:arc(100,0,100,-90,-180,true)
\end{verbatim}

We'll just be filling the shape so don't need to worry about joining up the arcs.

To draw the shape, we need to move it and then fill it.
In the \verb+draw+ function, we write:

\begin{verbatim}
tree:use({transformShape = true, fill})
\end{verbatim}

Putting all that together, we have:

\begin{verbatim}
function setup()
  print("hello world")
  tree = path()
  tree:moveTo(0,0)
  tree:arc(-100,0,100,0,-90,true)
  tree:arc(100,0,100,-90,-180,true)
end

function draw()
  background(40,40,50)
  fill(50,200,30)
  translate(WIDTH/2,HEIGHT-50)
  tree:use({transformShape = true, fill = true})
end
\end{verbatim}

We can add another segment of the tree with more arcs:

\begin{verbatim}
tree:arc(-200,0,200,0,-90,true)
tree:arc(200,0,200,-90,-180,true)
\end{verbatim}

We can add a third segment by changing the \verb+200+s to \verb+300+s, but that doesn't look quite right.
Better is to use a smaller arc slightly lower down.

\begin{verbatim}
tree:arc(-250,-50,250,0,-90,true)
tree:arc(250,-50,250,-90,-180,true)
\end{verbatim}

Our code at this point looks like the following:

\begin{verbatim}
function setup()
  print("hello world")
  tree = path()
  tree:moveTo(0,0)
  tree:arc(-100,0,100,0,-90,true)
  tree:arc(100,0,100,-90,-180,true)
  tree:arc(-200,0,200,0,-90,true)
  tree:arc(200,0,200,-90,-180,true)
  tree:arc(-250,-50,250,0,-90,true)
  tree:arc(250,-50,250,-90,-180,true)
end

function draw()
  background(40,40,50)
  fill(50,200,30)
  stroke(0,0,0)
  strokeWidth(5)
  translate(WIDTH/2,HEIGHT-50)
  tree:use({transformShape = true, fill = true})
end
\end{verbatim}

\section{The Trunk and Bucket}

The trunk and the bucket will be a new paths, one for each.
One reason for this is that we will fill them with different colours.
So in \verb+setup+ we need paths \verb+trunk = path()+ and \verb+bucket = path()+.
We'll actually draw the trunk first so it can overlap slightly with the tree and bucket.
So it just needs to be a rectangle.
The centre of the bottom of the tree is at \verb+(0,-300)+, so we start just left of there:

\begin{verbatim}
trunk:moveTo(-10,-300)
trunk:lineTo(-10,-350)
trunk:lineTo(10,-350)
trunk:lineTo(10,-300)
\end{verbatim}

When filling each path, we can specify the colour as part of its options.
This saves us having to remember which colour goes with which piece.
So in the \verb+draw+ function, we can write:

\begin{verbatim}
function setup()
  print("hello world")
  tree = path()
  tree:moveTo(0,0)
  tree:arc(-100,0,100,0,-90,true)
  tree:arc(100,0,100,-90,-180,true)
  tree:arc(-200,0,200,0,-90,true)
  tree:arc(200,0,200,-90,-180,true)
  tree:arc(-250,-50,250,0,-90,true)
  tree:arc(250,-50,250,-90,-180,true)
  trunk = path()
  trunk:moveTo(-10,-300)
  trunk:lineTo(-10,-350)
  trunk:lineTo(10,-350)
  trunk:lineTo(10,-300)
end

function draw()
  background(40,40,50)
  fill(50,200,30)
  stroke(0,0,0)
  strokeWidth(5)
  translate(WIDTH/2,HEIGHT-50)
  trunk:use({transformShape = true, fill = colour("brown")})
  tree:use({transformShape = true, fill = colour(50,200,30)})
end
\end{verbatim}

Of course, we can use other colours.
See the \href{Style.xhtml}{Style} tutorial for more details.

Now for the bucket.
To give it a more 3D look, we'll curve the top and bottom.
So in \verb+setup+:

\begin{verbatim}
bucket = path()
bucket:moveTo(-30,-340)
bucket:lineTo(-20,-400)
bucket:curveTo(-10,-410,10,-410,20,-400)
bucket:lineTo(30,-340)
bucket:curveTo(20,-350,-20,-350,-30,-340)
\end{verbatim}

And in \verb+draw+:

\begin{verbatim}
trunk:use({transformShape = true, fill = colour("brown")})
bucket:use({transformShape = true, fill= colour("red")})
tree:use({transformShape = true, fill = colour(50,200,30)})
\end{verbatim}

Our full code looks like this:

\begin{verbatim}
function setup()
  print("Merry Christmas!")
  tree = path()
  tree:moveTo(0,0)
  tree:arc(-100,0,100,0,-90,true)
  tree:arc(100,0,100,-90,-180,true)
  tree:arc(-200,0,200,0,-90,true)
  tree:arc(200,0,200,-90,-180,true)
  tree:arc(-250,-50,250,0,-90,true)
  tree:arc(250,-50,250,-90,-180,true)
  trunk = path()
  trunk:moveTo(-10,-300)
  trunk:lineTo(-10,-350)
  trunk:lineTo(10,-350)
  trunk:lineTo(10,-300)
  bucket = path()
  bucket:moveTo(-30,-340)
  bucket:lineTo(-20,-400)
  bucket:curveTo(-10,-410,10,-410,20,-400)
  bucket:lineTo(30,-340)
  bucket:curveTo(20,-350,-20,-350,-30,-340)
end

function draw()
  background(40,40,50)
  translate(WIDTH/2,HEIGHT-50)
  trunk:use({transformShape = true, fill = colour("brown")})
  bucket:use({transformShape = true, fill= colour("red")})
  tree:use({transformShape = true, fill = colour(50,200,30)})
end
\end{verbatim}

\section{Decorations}

One we have our basic tree, we can decorate it how we like.
We could use \verb+ellipse+ or \verb+circle+ to make baubles, and \verb+rect+ to make presents.
Or we could define new paths to make more complicated objects, like a star for the top.

If we're feeling very ambitious, we could even animate the decorations so that they move around the tree.

\end{document}


\documentclass[12pt,a4paper]{exam}

% Edit these as appropriate
\newcommand\hwnumber{1}                            % <-- homework number
\newcommand\DateOfTutorium{\DTMdate{2019-12-23}}   % <-- date of tutorium

% Load Preamble where settings and styles of document is defined
\input{Preamble.tex}

\begin{document}

\section*{The Brachistochrone Problem}

  In the following we are going to discuss the famous Brachistochrone Problem using the method of Lagrangian mechanics.

  Our goal is to find the optimal path between two points in space. For this path the time it takes an ideal object to roll down should be as short as possible. This means we want to find the minimum of the integral

    \begin{equation}
      t_{12} = \int_1^2 \text{d}t = \int_1^2 \frac{1}{v} \, \text{d}s, \label{eq:IntegralOne}
    \end{equation}
  where $v$ is the velocity at every point and $\text{d}s$ is the distance along the path. Using pythagoras we can write

    \begin{equation}
      \text{d} s^2 = \text{d} x^2 + \text{d} y^2 = \text{d} x^2 \left(1 + \left(\frac{\text{d} y}{\text{d} x}\right)\right) =: \text{d} x^2 \left(1 + y'^2\right). \label{eq:Distance}
    \end{equation}
  In order to find the velocity we can use conservation of the total energy $E$

    \begin{equation}
      E = \frac{1}{2}mv^2 + mgy \overset{!}{=} 0
    \end{equation}
  which can be set equal to zero when choosing the origin of our coordinate system appropriately. Thus the velocity is

    \begin{equation}
      v = \sqrt{2gy}. \label{eq:Velocity}
    \end{equation}
  Now we can plug in the expressions for $\text{d}s$, \eqref{eq:Distance} and the velocity, \eqref{eq:Velocity} into the integral of time, \eqref{eq:IntegralOne} which gets

    \begin{equation}
      t_{12} = \int_1^2 \frac{\sqrt{1 + y'^2}}{\sqrt{2gy}} \text{d}x. \label{eq:IntegralTwo}
    \end{equation}

  At this point we can use the tools of variational calculus. The goal is to find a function $y(x)$ which minimizes integral \eqref{eq:IntegralTwo} and thus the time it takes the object to roll down the path. Notice that the path is described by the function $y(x)$.

  We can define the Lagrange function $\mathcal{L}$ as the integrand of \eqref{eq:IntegralTwo}

    \begin{equation}
      \mathcal{L}\left(y, y'\right) := \sqrt{\frac{1 + y'^2}{2gy}} \label{eq:Lagrangian}.
    \end{equation}
  Usually solving the Euler-Lagrange equation

    \begin{equation}
      \frac{\partial \mathcal{L}}{\partial y} - \frac{\text{d}}{\text{d} x} \frac{\partial \mathcal{L}}{\partial y'} \overset{!}{=} 0 \label{eq:EulerLagrange}
    \end{equation}
  gives us the solution for $y(x)$ but since $\mathcal{L}$ is independet of $x$ ($\frac{\partial \mathcal{L}}{\partial x} = 0$) we can compute the total derivative

    \begin{equation}
      \begin{split}
        \text{d} \mathcal{L} &= \frac{\partial \mathcal{L}}{\partial y} \text{d} y + \frac{\partial \mathcal{L}}{\partial y'} \text{d} y' + \frac{\partial \mathcal{L}}{\partial x} \text{d} x \\
        \Rightarrow \frac{\text{d} \mathcal{L}}{\text{d} x} &= \frac{\partial \mathcal{L}}{\partial y} \frac{\text{d} y}{\text{d} x} + \frac{\partial \mathcal{L}}{\partial y'} \frac{\text{d} y'}{\text{d} x} + \frac{\partial \mathcal{L}}{\partial x} \\
        \Leftrightarrow \frac{\text{d} \mathcal{L}}{\text{d} x} &= \frac{\partial \mathcal{L}}{\partial y} y' + \frac{\partial \mathcal{L}}{\partial y'} y'' + \frac{\partial \mathcal{L}}{\partial x}.
      \end{split}
    \end{equation}
  Now we can insert the expression for $\frac{\partial \mathcal{L}}{\partial y}$ from \eqref{eq:EulerLagrange} and after using the product rule to rearrange the equation get

    \begin{equation}
      \begin{split}
        \frac{\text{d} \mathcal{L}}{\text{d} x} - y' \frac{\text{d}}{\text{d} x} \frac{\partial \mathcal{L}}{\partial y'}  - \frac{\partial \mathcal{L}}{\partial y'} y'' - \frac{\partial \mathcal{L}}{\partial x} &= 0 \\
        \Leftrightarrow \frac{\text{d} \mathcal{L}}{\text{d} x} -  \frac{\text{d}}{\text{d} x} y' \frac{\partial \mathcal{L}}{\partial y'} - \frac{\partial \mathcal{L}}{\partial x} &= 0 \\
        \Leftrightarrow \frac{\text{d} }{\text{d} x} \left(\mathcal{L} -   y' \frac{\partial \mathcal{L}}{\partial y'}\right) &= 0.
      \end{split}
    \end{equation}
  Thus the term in brackets is a constant,

    \begin{equation}
      \mathcal{L} -   y' \frac{\partial \mathcal{L}}{\partial y'} =: C
    \end{equation}
  and calculating the derivative

    \begin{equation}
      \frac{\partial \mathcal{L}}{\partial y'} = \frac{y'}{\sqrt{2gy\left(1+y'^2\right)}}
    \end{equation}
  gives
    \begin{equation}
      \begin{split}
        \sqrt{\frac{1 + y'^2}{2gy}} - \frac{y'^2}{\sqrt{2gy\left(1+y'^2\right)}} &= C \\
        \Leftrightarrow \frac{1}{\sqrt{2gy\left(1+y'^2\right)}} &= C.
      \end{split}
    \end{equation}
  Let's rearrange this expression to get

    \begin{equation}
      \begin{split}
        y \left(1 + y'^2\right) &= \frac{1}{2gC^2} := \frac{A}{2} \\
        \Leftrightarrow y \left(1 + \left(\frac{\text{d} y}{\text{d} x}\right)^2\right) &= \frac{A}{2}  \\
        \Leftrightarrow \frac{\text{d} y}{\text{d} x} &= \sqrt{\frac{A}{2y} - 1}.
      \end{split}
    \end{equation}

  This expression is solved by

    \begin{equation}
      x = A \left(\phi - \sin \phi \right), y = A \left(1 - \cos \phi \right),
    \end{equation}
  where $A$ is some parameter. In the following is an inverted sketch of this optimized path.

  \begin{center}
    \begin{tikzpicture}[scale=2.5]
      \tkzInit[xmin=0,xmax=3.3,xstep=1,ymin=0,ymax=2,ystep=1]
      \tkzAxeX[step=1]
      \tkzAxeY[step=1]
      \tkzFctPar[samples=100,domain=0:4*pi]{t-sin(t)}{1-cos(t)}
    \end{tikzpicture}
  \end{center}

\end{document}


\documentclass[12pt]{cdblatex}
\usepackage{fancyhdr}
\usepackage{footer}

\begin{document}

% =================================================================================================
\section*{Compute $\Gamma^{a}{}_{bcd}$ directly}

Here we will compute one of generalised connections directly from the connection. That is, we will
compute
\begin{align}
   \Gamma^{a}{}_{bcd} = \Gamma^{a}{}_{(bc,d)} - 2 \Gamma^{a}{}_{p(c} \Gamma^{p}{}_{bd)}
\end{align}
given an explict expression for the RNC connection $\Gamma^{a}{}_{bc}$.

This code was written as a check for the {\tt genGamma.tex} code. I had a discrepency between my
newly created Cadabra v2.0 codes and my old Cadabra v1.0 codes (which were the basis of my lcb09-03 paper).
I found that my new codes agreed with this code and thus my old codes were wrong. I found the errors
in the old code (see the updated codes in v1.0/rnc-new/gen-gamma.cdbp, see also the file v1.0/rnc-new/NOTES.txt).

The head-on approach shown in this code works for this simple compuation but for the higher
generalised connections it's almost certainly going to be way too slow to be use. So this code is
only a check of the results from {\tt genGamma.tex} code. The good news is -- they agree.

\clearpage

\begin{cadabra}
   {a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w#}::Indices(position=independent).

   D{#}::Derivative.
   \nabla{#}::Derivative.
   \partial{#}::PartialDerivative.

   g_{a b}::Metric.
   g^{a b}::InverseMetric.
   g_{a}^{b}::KroneckerDelta.
   g^{a}_{b}::KroneckerDelta.
   \delta^{a}_{b}::KroneckerDelta.
   \delta_{a}^{b}::KroneckerDelta.

   R_{a b c d}::RiemannTensor.
   R_{a b c d}::Depends(\nabla{#}).

   x^{a}::Depends(\partial{#}).

   \Gamma^{a}_{b c}::Depends(\partial{#}).
   \Gamma^{a}_{b c}::TableauSymmetry(shape={2}, indices={1,2}).

   Q_{a b c d}::Weight(label=numR,value=2).
   Q_{a b c d e}::Weight(label=numR,value=3).
   Q_{a b c d e f}::Weight(label=numR,value=4).
   Q_{a b c d e f g}::Weight(label=numR,value=5).

   # note: keeping numbering as is (out of order) to ensure R appears before \nabla R etc.
   def product_sort (obj):
       substitute (obj,$ A^{a}                            -> A001^{a}               $)
       substitute (obj,$ x^{a}                            -> A002^{a}               $)
       substitute (obj,$ g^{a b}                          -> A003^{a b}             $)
       substitute (obj,$ \nabla_{e f g h}{R_{a b c d}}    -> A008_{a b c d e f g h} $)
       substitute (obj,$ \nabla_{e f g}{R_{a b c d}}      -> A007_{a b c d e f g}   $)
       substitute (obj,$ \nabla_{e f}{R_{a b c d}}        -> A006_{a b c d e f}     $)
       substitute (obj,$ \nabla_{e}{R_{a b c d}}          -> A005_{a b c d e}       $)
       substitute (obj,$ R_{a b c d}                      -> A004_{a b c d}         $)
       sort_product   (obj)
       rename_dummies (obj)
       substitute (obj,$ A001^{a}                  -> A^{a}                         $)
       substitute (obj,$ A002^{a}                  -> x^{a}                         $)
       substitute (obj,$ A003^{a b}                -> g^{a b}                       $)
       substitute (obj,$ A004_{a b c d}            -> R_{a b c d}                   $)
       substitute (obj,$ A005_{a b c d e}          -> \nabla_{e}{R_{a b c d}}       $)
       substitute (obj,$ A006_{a b c d e f}        -> \nabla_{e f}{R_{a b c d}}     $)
       substitute (obj,$ A007_{a b c d e f g}      -> \nabla_{e f g}{R_{a b c d}}   $)
       substitute (obj,$ A008_{a b c d e f g h}    -> \nabla_{e f g h}{R_{a b c d}} $)

       return obj

   def truncate (obj,n):

   # I would like to assign different weights to \nabla_{a}, \nabla_{a b}, \nabla_{a b c} etc. but no matter
   # what I do it appears that Cadabra assigns the same weight to all of these regardless of the number of subscripts.
   # It seems that the weight is assigned to the symbol \nabla alone. So I'm forced to use the following substitution trick.

       tmp := @(obj).

       substitute (tmp, $\nabla_{e f g}{R_{a b c d}} -> Q_{a b c d e f g}$)
       substitute (tmp, $\nabla_{e f}{R_{a b c d}} -> Q_{a b c d e f}$)
       substitute (tmp, $\nabla_{e}{R_{a b c d}} -> Q_{a b c d e}$)
       substitute (tmp, $R_{a b c d} -> Q_{a b c d}$)

       ans = Ex(0)

       for i in range (0,n+1):
          foo := @(tmp).
          bah = Ex("numR = " + str(i))
          keep_weight (foo, bah)
          ans = ans + foo

       substitute (ans, $Q_{a b c d e f g} -> \nabla_{e f g}{R_{a b c d}}$)
       substitute (ans, $Q_{a b c d e f} -> \nabla_{e f}{R_{a b c d}}$)
       substitute (ans, $Q_{a b c d e} -> \nabla_{e}{R_{a b c d}}$)
       substitute (ans, $Q_{a b c d} -> R_{a b c d}$)

       return ans

   def get_term (obj,n):

       bah := @(obj).
       distribute (bah)

       substitute (bah, $\nabla_{e f g}{R_{a b c d}} -> Q_{a b c d e f g}$)
       substitute (bah, $\nabla_{e f}{R_{a b c d}} -> Q_{a b c d e f}$)
       substitute (bah, $\nabla_{e}{R_{a b c d}} -> Q_{a b c d e}$)
       substitute (bah, $R_{a b c d} -> Q_{a b c d}$)

       foo = Ex("numR = " + str(n))
       keep_weight (bah, foo)

       substitute (bah, $Q_{a b c d e f g} -> \nabla_{e f g}{R_{a b c d}}$)
       substitute (bah, $Q_{a b c d e f} -> \nabla_{e f}{R_{a b c d}}$)
       substitute (bah, $Q_{a b c d e} -> \nabla_{e}{R_{a b c d}}$)
       substitute (bah, $Q_{a b c d} -> R_{a b c d}$)

       return bah

   def tidy (obj,number):
      bah  = Ex(str(number))
      tmp := @(bah) @(obj).
      distribute  (tmp)
      factor_out  (tmp,$A^{a?},x^{b?}$)
      ans := @(tmp) / @(bah).
      return ans

   import cdblib

   Gamma = cdblib.get ('Gamma','../connection.json')

   defGamma := \Gamma^{d}_{a b} -> @(Gamma).

   genGam := A^{d} A^{b} A^{c} (\partial_{d}{\Gamma^{a}_{b c}} - 2 \Gamma^{a}_{p c} \Gamma^{p}_{b d}).

   substitute     (genGam,defGamma)   # cdb (genGam.001,genGam)
   distribute     (genGam)
   unwrap         (genGam)            # cdb (genGam.002,genGam)
   distribute     (genGam)            # cdb (genGam.003,genGam)
   product_rule   (genGam)            # cdb (genGam.004,genGam)
   distribute     (genGam)            # cdb (genGam.005,genGam)
   substitute     (genGam,$\partial_{a}{x^{b}}->\delta_{a}^{b}$)   # cdb (genGam.006,genGam)
   eliminate_kronecker (genGam)       # cdb (genGam.007,genGam)
   genGam = truncate     (genGam,5)   # cdb (genGam.008,genGam)
   genGam = product_sort (genGam)     # cdb (genGam.009,genGam)
   rename_dummies (genGam)            # cdb (genGam.010,genGam)
   canonicalise   (genGam)            # cdb (genGam.011,genGam)

   # all done, now for some housekeeping

   term3 = get_term (genGam,3)   # cdb (term3.001,term3)
   term4 = get_term (genGam,4)   # cdb (term4.001,term4)
   term5 = get_term (genGam,5)   # cdb (term5.001,term5)

   term3 = tidy (term3,2)
   term4 = tidy (term4,120)
   term5 = tidy (term5,180)

   genGam := @(term3) + @(term4) + @(term5).

   scaledGamma := 360 @(genGam).  # cdb (scaledGamma.001,scaledGamma)

\end{cadabra}

\clearpage

\begin{dgroup*}
   \begin{dmath*} \cdb*{term3.001} \end{dmath*}
   \begin{dmath*} \cdb*{term4.001} \end{dmath*}
   \begin{dmath*} \cdb*{term5.001} \end{dmath*}
\end{dgroup*}

\clearpage

% =================================================================================================
\section*{Summary}

\begin{dgroup*}
   \begin{dmath*} 360 A^b A^c A^d \Gamma^{a}_{b c d} = \cdb{scaledGamma.001} \end{dmath*}
\end{dgroup*}

\end{document}


% Define document class
\documentclass[modern]{aastex631}

% Filler text
\usepackage{blindtext}

% Math shortcuts
\newcommand{\like}{\mathcal{L}}
\newcommand{\order}[1]{\mathcal{O}\left( #1 \right)}

% Begin!
\begin{document}

% Title
\title{When and How Can we Fit Residuals?}

% Author list
\author[0000-0003-1540-8562]{Will M. Farr}
\email{will.farr@stonybrook.edu}
\email{wfarr@flatironinstitute.org}
\affiliation{Department of Physics and Astronomy, Stony Brook University, Stony Brook NY 11794, USA}
\affiliation{Center for Computational Astrophysics, Flatiron Institute, New York NY 10010, USA}

% Abstract with filler text
\begin{abstract}
    I discuss the effect of fixing the \emph{residuals} from a global fit in
    LISA-like data when fitting broadband signals like the inspiral of a
    high-redshift seed BBH.
\end{abstract}

% Main body with filler text
\section{Introduction}

The joint likelihood for a single BBH merger waveform $h$ and some number of
white dwarf monochromatic or near-monochromatic waveforms in LISA data $d$ is
%
\begin{equation}
    \log \like \propto - \Delta f \sum_i \frac{\left| d_i - h_i - \sum_\alpha g_{\alpha, i} \right|^2}{2 S_i},
\end{equation}
%
where the sum runs over frequency-domain components and $S_i$ is the noise PSD
at frequency $i$.  Expanding to quadratic order about $h_i = h_{i,0}$ in terms
of the parameters $\theta$ that control the waveform, we have
%
\begin{multline}
    \log \like \sim - \Delta f \sum_i \frac{1}{2 S_i} \left( \left| d_i - h_{i,0} - G_i \right|^2 - 2 \Re \left( d_i - h_{i,0} - G_i \right)^* \frac{\partial h_i}{\partial \theta^a} \left( \theta^a - \theta^a_0 \right)  \right. \\ \left. + \left(\theta^a - \theta_0^a \right) \left( \frac{\partial h_i^*}{\partial \theta^a} \frac{\partial h_i}{\partial \theta^b} + \Re \left( d_i - h_{i,0} - G_i \right)^* \frac{\partial^2 h_i}{\partial \theta^a \partial \theta^b} \right) \left( \theta^b - \theta_0^b \right) \right),
\end{multline}
%
where
%
\begin{equation}
    G_i \equiv \sum_\alpha g_{\alpha,i}.
\end{equation}
%
Collecting terms, we see that we have a Gaussian likelihood for the parameters
$\theta$, with
%
\begin{equation}
    \log \like \sim - \left( A + B_a \Delta \theta^a + \frac{1}{2} \Delta \theta^a C_{ab} \Delta \theta^b \right),
\end{equation}
%
with
%
\begin{equation}
    B_a = \Delta f \sum_i \frac{1}{S_i} \left( - \Re \left( d_i - h_{i,0} - G_i \right)^* \frac{\partial h_i}{\partial \theta^a} \right)
\end{equation}
%
and
%
\begin{equation}
    C_{ab} = \Delta f \sum_i \frac{1}{S_i} \left( \frac{\partial h_i^*}{\partial \theta^a} \frac{\partial h_i}{\partial \theta^b} + \Re \left( d_i - h_{i,0} - G_i \right)^* \frac{\partial^2 h_i}{\partial \theta^a \partial \theta^b} \right).
\end{equation}
%

Now we make the assumption that the sources making up $G$ are \emph{narrowband},
that is that their S/N accumulates over a small number of bins around $i = i_0$,
such that
%
\begin{equation}
\frac{\left| g_{\alpha,i} \right|^2}{S_i} \sim \begin{cases}
    \order{\rho_\alpha^2} & \left| i - i_0 \right| \sim \order{1} \\
    0 & \mathrm{otherwise}
\end{cases}.
\end{equation}
%
We further assume that $h$ and therefore $h_0$ are \emph{broadband}, so
%
\begin{equation}
\frac{\left| h_i \right|^2}{S_i} \sim \frac{\order{\rho_h^2}}{N}
\end{equation}
%
where $N \gg 1$ is the number of bins over which the S/N of $h$ accumulates.

Let us suppose that the data are composed of signals like $h$ and $G$ plus colored noise $n$ with PSD $S$:
%
\begin{equation}
    d_i = \bar{h}_i + \bar{G}_i + n_i
\end{equation}
%

\end{document}


\section{two scatterer folding}

Two scatterers: incident is A, reflected is B, and transmitted is G. Between are C (forward) and D (backward).

% see notes, 20080619
First, boundary conditions match for the amplitudes and the derivatives, where n is the single incident channel, which scatters into channels m. For the first scatterer,
\begin{equation}
\begin{gathered}
A_n \delta_{mn} + B_m = C_m+D_m \\
(i k_m C_m-i k_m D_m) - (i k_m \delta_{nm}^{(1)} A_m-i k_m B_m) =
\sum_{m'} \Gamma_{mm'} (A_n \delta_{nm'}+B_{m'})
\end{gathered}
\end{equation}
and similarly for the second scatterer, which is at x=a instead of zero,
\begin{equation}
\begin{gathered}
C_m \exp(i k_m a) + D_m \exp(-i k_m a) = G \exp(i k_m a) \\
(i k G_m \exp(i k_m a)-(i k_m C_m \exp(i k_m a) - i k_m D_m \exp(-i k_m a)) =
\sum_{m'} \Gamma_{mm'}^{(2)} G_{m'} \exp(i k_{m'} a)
\label{matchatxequalsa}
\end{gathered}
\end{equation}
the k variable is a matrix of diagonal elements indexed by channel number
\begin{equation}
\hat{k}_{nn} = \sqrt{(\frac{2 \pi}{\lambda})^2 -(\frac{\pi n}{w})^2}
\end{equation}
where n goes from 1 to $N+N_c$

\begin{equation}
\begin{gathered}
C_m +D_m = \delta_{mn} A_m+B_m \\
C_m -D_m = A_m \delta_{mn} - B_m +\frac{1}{i k_m} \sum_{m'}
\Gamma_{mm'}^{(1)} (\delta_{nm'} A_{m'}+B_{m'})
\end{gathered}
\end{equation}
Solve for $C_m$ and $D_m$ at x = 0.
\begin{comment}
use
x+y=A
x-y=B
==>  2x = A+B
====> x = (A+B)/2
repeat for y
\end{comment}

\begin{equation}
\begin{gathered}
C_m = A_m \delta_{nm} + \frac{1}{2 i k_m}\sum_{m'}\Gamma_{mm'}^{(1)}
(\delta_{nm'} A_{m'}+B_{m'}) \\
D_m = B_m - \frac{1}{2 i k_m}\sum_{m'}\Gamma_{mm'}^{(1)}
(\delta_{nm'} A_{m'}+B_{m'})
\end{gathered}
\end{equation}
\begin{equation}
\begin{gathered}
A_m \delta_{nm} \exp(i k_m a)+B_m \exp(-i k_m a) +
\frac{1}{k_m}(\sum_{m'}\Gamma_{mm'}^{(1)}(\delta_{nm'} A_{m'}+B_{m'})) sin(k_m a) = \\
G_m \exp(i k_m a) \\
G_m \exp(i k_m a) - (A_m \delta_{nm} \exp(i k_m a)-B_m \exp(-i k_m a) +
\frac{1}{i k_m}(\sum_{m'}\Gamma_{mm'}^{(1)}(\delta_{nm'} A_{m'}+B_{m'})) cos(k_{m'} a) = \\
\frac{1}{i k_m}(\sum_{m'}\Gamma_{mm'}^{(2)} G_{m'} \exp(i k_{m'} a)) \\
\end{gathered}
\end{equation}
divide everything by $A_m$ to get normalized incident wave
\begin{equation}
\begin{gathered}
\delta_{nm} \exp(i k_m a)+r_{mn} \exp(-i k_m a) +
\frac{1}{k_m}(\sum_{m'}\Gamma_{mm'}^{(1)}(\delta_{nm'}+r_{nm'})) sin(k_m a) = \\
t_{nm} \exp(i k_m a) \\
t_{nm} \exp(i k_m a) - (\delta_{nm} \exp(i k_m a)-r_{mn} \exp(-i k_m a) +
\frac{1}{i k_m}(\sum_{m'}\Gamma_{mm'}^{(1)}(\delta_{nm'}+r_{nm'})) cos(k_{m'} a) = \\
\frac{1}{i k_m}(\sum_{m'}\Gamma_{mm'}^{(2)} t_{nm'} \exp(i k_{m'} a)) \\
\end{gathered}
\end{equation}
where
\begin{equation}
\begin{gathered}
\frac{B}{A} \equiv r_{nm} \\
\frac{G}{A} \equiv t_{nm}
\end{gathered}
\end{equation}
Now we'll switch from indexed variable to matrix notation

\begin{equation}
\begin{gathered}
(\exp(-i \hat{k} a)+\hat{k}^{-1} sin(\hat{k} a) \hat{\Gamma}) \vec{r} +
(-\exp(i \hat{k} a)) \vec{t} = \\
-(\exp(i \hat{k} a +\hat{k}^{-1} sin(\hat{k} a) \hat{\Gamma}) \vec{h}\\
(\exp(-i \hat{k} a)+i \hat{k}^{-1} cos(\hat{k} a) \hat{\Gamma}) \vec{r} +
(\exp(i \hat{k} a)+i \hat{k}^{-1} \exp(i \hat{k} a) \hat{\Gamma}) \vec{t} = \\
(\exp(i \hat{k} a)-i \hat{k}^{-1} cos(\hat{k} a) \hat{\Gamma}) \vec{h}
\end{gathered}
\end{equation}
we clump the coefficients in that big mess to another indexed variable, ``F''.
\begin{equation}
\begin{gathered}
\hat{F_1} \vec{r}+\hat{F_2} \vec{t}=\hat{F_3} \vec{h} \\
\hat{F_4} \vec{r}+\hat{F_5} \vec{t}=\hat{F_6} \vec{h}
\end{gathered}
\end{equation}

solve for $\vec{r}$, set the two equal
\begin{equation}
\begin{gathered}
(\hat{F_1}^{-1} \hat{F_3}-\hat{F_4}^{-1} \hat{F_6}) \vec{h} =
(\hat{F_1}^{-1} \hat{F_2}-\hat{F_4}^{-1} \hat{F_5}) \vec{t}
\end{gathered}
\end{equation}
then solve for $\vec{h}$ in terms of $\vec{t}$
\begin{equation}
\vec{h} = (\hat{F_1}^{-1} \hat{F_3} - \hat{F_4}^{-1} \hat{F_6})^{-1}
(\hat{F_1}^{-1} \hat{F_2} - \hat{F_4}^{-1} \hat{F_5}) \vec{t}
\end{equation}
and we'll call that big messy matrix, which has dimension $(N+N_c)x(N+N_c)$,
\begin{equation}
(\hat{F_1}^{-1} \hat{F_3} - \hat{F_4}^{-1} \hat{F_6})^{-1}
(\hat{F_1}^{-1} \hat{F_2} - \hat{F_4}^{-1} \hat{F_5}) \equiv
\hat{I}+\frac{i}{2}\hat{k}^{-1}\Gamma^{combined}
\end{equation}
Solving for $\Gamma^{combined}$,
\begin{equation}
\Gamma^{combined} = -i 2\hat{k}(((\hat{F_1}^{-1} \hat{F_3} - \hat{F_4}^{-1} \hat{F_6})^{-1}
(\hat{F_1}^{-1} \hat{F_2} - \hat{F_4}^{-1} \hat{F_5})) - \hat{I})
\end{equation}

Now we have a general algorithm for combining any two $\Gamma$ matrices, either two reduced (open channels only, (NxN)) or two unreduced (open and closed channels, (($N+N_c$)x($N+N_c$))).

Using this ``gamma combiner'' algorithm with the above ``gamma reducer'' algorithm we can numerically compute how the two are different, and how separation (density in many scatterers) is affected when N varies, when $N_c$ is include or not.


\section{The \insertionsort algorithm}
\Label{sec:insertionsort}

Like \selectionsort,
the algorithm \insertionsort traverses the given array \inl{a[0..n-1]}
left to right, maintaining a left-adjusted,
constantly increasing range \inl{a[0..i-1]} that is already in increasing order.

Unlike \selectionsort, however, \insertionsort adds \inl{a[i]} to the
initial segment in the \inl{i}th step (see Figure~\ref{fig:insertionsort-example}).
%
It determines the (rightmost) appropriate position to insert \inl{a[i]}
by a call to \specref{upperbound} and then uses \specref{rotatei} to
perform a \emph{circular shift} to establish the insertion.

\begin{figure}[hbt]
\begin{center}
\includegraphics[width=0.65\textwidth]{Figures/insertion_sort.pdf}
\caption{An iteration of \insertionsort}
\Label{fig:insertionsort-example}
\end{center}
\end{figure}

\FloatBarrier

\subsection{Formal specification of \insertionsort}

The following listing shows our (generic sorting) contract for \insertionsort.

\input{Listings/insertion_sort.h.tex}

\clearpage

\subsection{Implementation of \insertionsort}

The implementation of \insertionsort is shown in the next listing.
%
We used an \acsl statement contract to specify those aspects of the
\rotatei contract that are needed here.
%
Properties related to the result of \insertionsort being in increasing
order are labelled \inl{increasing}.
Properties related to the rearrangement of elements are labelled \inl{reorder} and,
whenever their order isn't changed, \inl{unchanged}.

\input{Listings/insertion_sort.c.tex}

When we originally
implemented and verified \rotatei, we hadn't yet in mind to
use that function inside of \insertionsort.
%
Consequently, the properties needed for the latter
aren't directly provided by the former.
%
One approach to solve this problem is to add the new properties to
the contract of \specref{rotatei} and repeat its verification proof.
However, if \rotatei is assumed to be part of a pre-verified library,
this approach isn't feasible, since \rotatei's implementation may not
be available for re-verification.
%
Therefore, we used another approach, viz.\ to prove that \rotatei's
original specification \emph{implies} all the properties we need in
\insertionsort.
This is another use of the Hoare calculus' implication rule
(\S\ref{sec:The Implication Rule}).
%
We used several lemmas, shown below,
to make the necessary implications explicit, and to help the provers to
establish them.
%
Some of them needed manual proofs by induction.

\clearpage

Lemma \logicref{IncreasingEqual} in the following listing assumes an ordered range
\inl{a[m..n-1]} and claims that every (elementwise) equal range
range \inl{a[m+p..n+p-1]} is ordered, too.
%
It is needed to establish that the call to \specref{rotatei} preserves the order of
those elements that are shifted upwards
(cf.\ Figure~\ref{fig:insertionsort-example}).

Similarly, lemma \logicref{CountEqual} says that two elementwise equal ranges
\inl{a[m..n-1]} and \inl{a[p..p+n-m-1]} will result in the same occurrence count,
for each value \inl{v}.
%
This lemma is useful in the proof of the lemma
\logicref{CircularShiftMultisetReorder} (discussed below),
since the predicate \logicref{MultisetReorder}
is defined via the logic function \logicref{Count}.

Lemma \logicref{CircularShiftStrictLowerBound} in the next listing
is used to prove that the range \inl{a[k..i-1]} having
\inl{a[i]} as strict lower bound before our call to \rotatei ensures
that it has \inl{a[k]} as such a bound after the call.
Note that this lemma reflects that \rotatei is uses as a \emph{circular shift}
at the call site.
%
Similarly, lemma \CircularShiftMultisetReorder establishes that
a circular shift just reorders the range it is applied to.

\input{Listings/CircularShiftLemmas.acsl.tex}

\clearpage


\chapter{Additional Kolmogorov-Smirnov Two Sample Test Tables}
\label{appendix:ks-test}
This appendix lists the Kolmogorov-Smirnov tables comparing the features for intensity and texture.

\begin{table}[H]
\label{table:blob-texture-ks}
\centering
\primitiveinput{tables/texture_features_ks.tex}
\caption{Comparison of the Kolmogorov-Smirnov test results for each texture feature derived from patches of images defined by blobs across ten scales.}
\end{table}

\primitiveinput{tables/intensity_features_ks.tex}

\begin{table}[H]
\label{table:line-texture-ks}
\centering
\primitiveinput{tables/texture_features_ks_lines.tex}
\caption{Comparison of the Kolmogorov-Smirnov test results for each texture feature derived from patches of images defined by lines.}
\end{table}

\begin{table}[H]
\label{table:line-intensity-ks}
\centering
\primitiveinput{tables/line_intensity_features_ks.tex}
\caption{Comparison of the Kolmogorov-Smirnov test results for each intensity feature derived from patches of images defined by lines.}
\end{table}


\documentclass[12pt]{article}

\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{subfigure}
\usepackage{float}
\usepackage{ulem}
\usepackage{bm}
\usepackage{anysize}
\usepackage{pythonhighlight}

\marginsize{2cm}{2cm}{0.9cm}{1.8cm}

\title{EECE 5639 Computer Vision\\ [2ex] \begin{large} Homework \#2 \end{large} }
\author{Jiyu Tian}
\date{}

\begin{document}
\maketitle
%%---------------------------------------------------------------
%% Question 1
%%---------------------------------------------------------------
\section{Solution:}
As shown in Figure \ref{raw}, the estimated noise of generated images is 1.9462, which is very approximate to 2, the targeted $\sigma$. And the worst case acquisition noise is 4.0258, around $2\sigma$.
\begin{figure}[H]
\centering
\includegraphics[width = 1\textwidth]{fig1.png}
\caption{Raw Images}
\label{raw}
\end{figure}
%%---------------------------------------------------------------
%% Question 2
%%---------------------------------------------------------------
\section{Solution:}
As shown in Figure \ref{filtered}, the estimated noise of the images after $3\times3$ box filter is 0.6472, and the worst case acquisition noise is 1.3971. Compared to the original ones, the image noises are reduced significantly, which we can also easily tell from the two figures.
\begin{figure}[H]
\centering
\includegraphics[width = 1\textwidth]{fig2.png}
\caption{Raw Images}
\label{filtered}
\end{figure}
%%---------------------------------------------------------------
%% Question 3
%%---------------------------------------------------------------
\section{Solution:}
The 2D Gaussian filter mask can be expressed as:
\begin{equation*}
    G(x,y) = \frac{1}{2\pi\sigma^2}e^{-\frac{x^2+y^2}{2\sigma^2}}
\end{equation*}
which can also be separated into two 1D Gaussian filters:
\begin{equation*}
    G(x) = \frac{1}{\sqrt{2\pi}\sigma}e^{-\frac{x^2}{2\sigma^2}}, G(y) = \frac{1}{\sqrt{2\pi}\sigma}e^{-\frac{y^2}{2\sigma^2}}
\end{equation*}
The minimum filter size is $2\times ceil(2\sigma)+1=7$. With implementation in python, a $7\times 7$ 2D Gaussian filter mask with standard deviation $1.4$ can be applied onto the sample figure, as shown in Figure \ref{ouput}. Please refer to Appendix for detailed code.
\begin{figure}[H]
\centering
\includegraphics[width = 1\textwidth]{output.png}
\caption{Output of images after applying Gaussian filter mask}
\label{ouput}
\end{figure}

\vfill
\clearpage
%%---------------------------------------------------------------
%% Question 4
%%---------------------------------------------------------------
\section{Solution:}
(a) With filter (a):
\begin{equation*}
{\left[ \begin{array}{cccccccccc}
6 & 8 & 10 & 16 & 22 & 28 & 34&  40 &  32& 24
\end{array} \right]}
\end{equation*}
\noindent With filter (b):
\begin{equation*}
{\left[ \begin{array}{cccccccccc}
7 & 9 & 10 & 13 & 19 & 31 & 37 & 40 & 36 & 28
\end{array} \right]}
\end{equation*}
\noindent (b) Consider a 1D image which is large enough to ignore the borders, with additive, uncorrelated Gaussian noise with zero mean and variance $\sigma^2$. Since filter (b) has different weights upon each filtered pixel, its computation cost is higher than that of filter (a). As for the variance of the noise, with filter (a), a pixel in the output noise $O$ with respect to input noise $I$ is:
\begin{equation*}
O_i = \frac{1}{5}\sum^2_{j = -2}I_{i+j}
\end{equation*}
The expected value of a pixel after filtering is:
\begin{equation*}
E[O] = E\left[\frac{1}{5}\sum^2_{j = -2}I_{i+j}\right] = \frac{1}{5}\sum^2_{j=-2}E\left[I_{i+j}\right] = 0
\end{equation*}
The variance of a pixel after filtering is:
\begin{equation*}
\begin{aligned}
E[(O-E[O])^2] &= E[O^2] \\
&= E\left[\frac{1}{25}\left(\sum^2_{j = -2}I_{i+j}\right)^2\right]\\
&= \frac{1}{25}E\left[\sum^2_{j-2}I^2_{i+j} + \sum^2_{m,n = -2}I_{i+m}I_{i+n}\right]\\
&= \frac{1}{25}\sum^2_{j-2}E\left[I^2_{i+j}\right]\\
&=\frac{1}{25}\times5\sigma^2 = \frac{\sigma^2}{5}
\end{aligned}
\end{equation*}
With filter (b):
\begin{equation*}
O_i = \frac{1}{10}\left(I_{i-2} + 2I_{i-1} + 4I_{i} + 2I_{i+1} + I_{i+2}\right)
\end{equation*}
The expected value of a pixel after filtering is:
\begin{equation*}
\begin{aligned}
E[O] &= E\left(\frac{1}{10}\left[I_{i-2} + 2I_{i-1} + 4I_{i} + 2I_{i+1} + I_{i+2}\right]\right)\\
&= \frac{1}{10}\left[E[I_{i-2}] + E[2I_{i-1}] + E[4I_{i}] + E[2I_{i+1}] + E[I_{i+2}] \right] \\
&= 0
\end{aligned}
\end{equation*}
The variance of a pixel after filtering is:
\begin{equation*}
\begin{aligned}
E[(O-E[O])^2] &= E[O^2] \\
&=\frac{1}{100}E\left[ I^2_{i-2} + 4I^2_{i-1} + 16I^2_{i} + 4I^2_{i+1} + I^2_{i+2} +\sum_l\sum^2_{m,n = -2}\alpha_lI_{i+m}I_{i+n} \right]\\
&=\frac{1}{100}\left[ E[I^2_{i-2}] + 4E[I^2_{i-1}] + 16E[I^2_{i}] + 4E[I^2_{i+1}] + E[I^2_{i+2}] \right]\\
&=\frac{1}{100}\times26\sigma^2=\frac{13}{50}\sigma^2> \frac{\sigma^2}{5}
\end{aligned}
\end{equation*}
Therefore, after these two average filters, the expected value keeps the same. But the variance after filter (b) is larger than that of filter (a).
%%---------------------------------------------------------------
%% Question 5
%%---------------------------------------------------------------
\section{Solution:}
When operator is centered on the line of grey level 50, i.e. $I = [noise\ \ line\ \ noise]$:
\begin{equation*}
\begin{aligned}
&P(I = \left[ \begin{array}{ccc}
salt & line & salt
\end{array} \right]) = P(O = -100) = 0.49\\
&P(I = \left[ \begin{array}{ccc}
salt & line & pepper
\end{array} \right]) = P(O = 0) = 0.21\\
&P(I = \left[ \begin{array}{ccc}
pepper & line & salt
\end{array} \right]) = P(O = 0) = 0.21\\
&P(I = \left[ \begin{array}{ccc}
pepper & line & pepper
\end{array} \right]) = P(O = 100) = 0.09
\end{aligned}
\end{equation*}
And thus the PDF of output value is:
\begin{equation*}
    f(x) = \frac{49}{100}\delta(x+100) + \frac{42}{100}\delta(x) + \frac{9}{100}\delta(x-100)
\end{equation*}
When operator is centered on the line adjacent to the line, i.e. $I = [line\ \ noise\ \ noise]$:
\begin{equation*}
\begin{aligned}
&P(I = \left[ \begin{array}{ccc}
line & salt & salt
\end{array} \right]) = P(O = 50) = 0.49\\
&P(I = \left[ \begin{array}{ccc}
line & salt & pepper
\end{array} \right]) = P(O = 150) = 0.21\\
&P(I = \left[ \begin{array}{ccc}
line & pepper & salt
\end{array} \right]) = P(O = -150) = 0.21\\
&P(I = \left[ \begin{array}{ccc}
line & pepper & pepper
\end{array} \right]) = P(O = -50) = 0.09
\end{aligned}
\end{equation*}
And thus the PDF of output value is:
\begin{equation*}
    f(x) = \frac{49}{100}\delta(x-50) + \frac{21}{100}\delta(x-150) + \frac{21}{100}\delta(x+150) + \frac{9}{100}\delta(x+50)
\end{equation*}

\vfill
\clearpage
%%---------------------------------------------------------------
%% Question 6
%%---------------------------------------------------------------
\section{Solution:}
The image I is:
\begin{equation*}
I={\left[ \begin{array}{cccccccc}
0 & 1 & 2 & 3 & 4 & 5 & 6 & 7\\
1 & 0 & 1 & 2 & 3 & 4 & 5 & 6\\
2 & 1 & 0 & 1 & 2 & 3 & 4 & 5\\
3 & 2 & 1 & 0 & 1 & 2 & 3 & 4\\
4 & 3 & 2 & 1 & 0 & 1 & 2 & 3\\
5 & 4 & 3 & 2 & 1 & 0 & 1 & 2\\
6 & 5 & 4 & 3 & 2 & 1 & 0 & 1\\
7 & 6 & 5 & 4 & 3 & 2 & 1 & 0\\
\end{array} \right]}
\end{equation*}
After applying a $3\times3$ median filter, the output is
\begin{equation*}
O={\left[ \begin{array}{cccccccc}
0 & 1 & 2 & 3 & 4 & 5 & 6 & 7\\
1 & 1 & 1 & 2 & 3 & 4 & 5 & 6\\
2 & 1 & 1 & 1 & 2 & 3 & 4 & 5\\
3 & 2 & 1 & 1 & 1 & 2 & 3 & 4\\
4 & 3 & 2 & 1 & 1 & 1 & 2 & 3\\
5 & 4 & 3 & 2 & 1 & 1 & 1 & 2\\
6 & 5 & 4 & 3 & 2 & 1 & 1 & 1\\
7 & 6 & 5 & 4 & 3 & 2 & 1 & 0\\
\end{array} \right]}
\end{equation*}

%%---------------------------------------------------------------
%% Question 7
%%---------------------------------------------------------------
\section{Solution:}
The input image is:
\begin{equation*}
I={\left[ \begin{array}{cccccccc}
4 & 4 & 4 & 4 & 8 & 8 & 8 & 8
\end{array} \right]}
\end{equation*}
After applying the median filter assuming that the border pixels are not changed:
\begin{equation*}
O={\left[ \begin{array}{cccccccc}
4 & 4 & 4 & 4 & 8 & 8 & 8 & 8
\end{array} \right]}
\end{equation*}
After applying the averaging mask assuming that the border pixels are not changed:
\begin{equation*}
O={\left[ \begin{array}{cccccccc}
4 & 4 & 4 & 5 & 7 & 8 & 8 & 8
\end{array} \right]}
\end{equation*}
The median filter keeps the step image unchanged, while the averaging mask smooths the step.
\vfill
\clearpage
%%---------------------------------------------------------------
%% Appendix
%%---------------------------------------------------------------
\section*{Appendix}
\begin{python}
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import numpy as np
import matplotlib.pylab as plt
import matplotlib.image as mpimg
plt.rcParams['figure.figsize'] = 15, 6
#plt.rcParams['figure.dpi'] = 300

NUMBER = 10
IMG_SHAPE = (256, 256)
GREY_LEVEL = 128
MU = 0
SIGMA = 2

class Homework2(object):

    def __init__(self):
        pass

    @staticmethod
    def gen_one_image():
        """Generate one image."""
        img = GREY_LEVEL * np.ones(IMG_SHAPE)
        noise = np.random.normal(MU, SIGMA, IMG_SHAPE)
        image = img + noise

        return image

    def gen_image(self, nimage):
        """Generate all required images."""
        tmp = [0] * nimage
        for i in np.arange(nimage):
            tmp[i] = self.gen_one_image()

        return np.array(tmp)

    @staticmethod
    def EST_NOISE(images):
        """Implementation of EST_NOISE in Chapter 2 of Trucco and Verri."""
        num = images.shape[0]
        m_e_bar = sum(images)/num
        m_sigma = np.sqrt(sum((images - m_e_bar)**2)/(num - 1))

        return m_sigma

    def solve_p1(self):
        """Solve problem #1."""
        all_image = self.gen_image(NUMBER)
        result = self.EST_NOISE(all_image)
        print("\n\nSolution for problem #1.")
        self.plot_result(all_image, result)

        self.images = all_image

    @staticmethod
    def apply_2d_filter(bfilter, timage):
        """Apply given 2D filter onto an image.

        Parameters
        ----------
        bfilter: array-like
            The filter
        timage: array-like
            Targeted image
        """
        image_shape = timage.shape
        ovrlay = int(bfilter.shape[0] / 2)
        tmp_matrix = np.zeros(np.array(image_shape) + 2 * ovrlay)
        tmp_matrix[ovrlay:-ovrlay, ovrlay:-ovrlay] = timage
        res_matrix = np.zeros(timage.shape)
        for i in np.arange(image_shape[0]) + ovrlay:
            for j in np.arange(image_shape[1]) + ovrlay:
                local_matrix = tmp_matrix[i - ovrlay:i + ovrlay + 1,
                                          j - ovrlay:j + ovrlay + 1]
                res_matrix[i - ovrlay, j - ovrlay] = sum(sum(local_matrix * bfilter))
        return res_matrix

    def solve_p2(self):
        """Solve problem #2."""
        box_filter = np.ones((3, 3)) / 9
        all_image = self.images
        filted_image = [0] * NUMBER
        for i in np.arange(NUMBER):
            filted_image[i] = self.apply_2d_filter(box_filter, all_image[i])
        filted_image = np.array(filted_image)
        result = self.EST_NOISE(filted_image)
        print("\n\nSolution for problem #2.")
        self.plot_result(filted_image, result)

    @staticmethod
    def plot_result(images, res):
        """Plot results."""
        fig = plt.figure()
        for i in np.arange(NUMBER):
            ax = fig.add_subplot(2, 5, i + 1)
            ax.axis('off')
            ax.imshow(images[i], cmap=plt.cm.gray)
        fig.suptitle(f"Estimated Noise {res.mean():.4f}, Worst Case Noise {res.max():.4f}",
                     fontsize=12)
        fig.show()

    def solve_p3(self, sigma=1.4, show=True):
        """Solve problem #1."""
        n = int(2 * np.ceil(2 * sigma) + 1)
        # 2D Gaussian
        ovrlay = int(n / 2)
        inds = np.arange(-ovrlay, ovrlay + 1)
        x, y = np.meshgrid(inds, inds)
        mask = np.exp(-(x**2 + y**2)/(2*sigma**2))
        mask = mask/sum(sum(mask))

        # two 1D Gaussian
        gaussian_1d = np.exp(-inds**2/(2 * sigma**2))
        gaussian_1d = gaussian_1d /sum(gaussian_1d).reshape((-1, 1))
        mask2 = gaussian_1d * gaussian_1d.T

        print("\n\nSolution for problem #3.")

        if show:
            Fsize = 16
            test_img = mpimg.imread(r".\fig\test.png")
            output_img1 = self.apply_2d_filter(mask, test_img)
            output_img2 = self.apply_2d_filter(mask2, test_img)
            fig = plt.figure()
            ax1 = fig.add_subplot(131)
            ax1.axis('off')
            ax1.imshow(test_img, cmap=plt.cm.gray)
            ax1.set_title("Raw image", fontsize=Fsize)
            ax2 = fig.add_subplot(132, sharex=ax1, sharey=ax1)
            ax2.axis('off')
            ax2.imshow(output_img1, cmap=plt.cm.gray)
            ax2.set_title(f"2D {n}x{n} Gaussian", fontsize=Fsize)
            ax3 = fig.add_subplot(133, sharex=ax1, sharey=ax1)
            ax3.axis('off')
            ax3.imshow(output_img2, cmap=plt.cm.gray)
            ax3.set_title(f"Two 1D n={n} Gaussian", fontsize=Fsize)
            fig.show()

        return mask, gaussian_1d

    @staticmethod
    def apply_1d_filter(bfilter, timage):
        """Apply given 1D filter onto an image.

        Parameters
        ----------
        bfilter: array-like
            The filter
        timage: array-like
            Targeted image
        """
        image_length = len(timage)
        ovrlay = int(bfilter.shape[0] / 2)
        tmp_array = np.zeros(image_length + 2 * ovrlay)
        tmp_array[ovrlay:-ovrlay] = timage
        res_array = np.zeros(image_length )
        for i in np.arange(image_length) + ovrlay:
            local_matrix = tmp_array[i - ovrlay:i + ovrlay + 1]
            res_array[i - ovrlay] = sum(local_matrix * bfilter)
        return res_array

    @staticmethod
    def apply_1d_median_filter(n, timage):
        """Applying a 3 median flter on the image I assuming that the
        border pixels are not changed.

        Parameters
        ----------
        n: int
            Shape of median filter
        timage: array-like
            Targeted image
        """
        image_shape = timage.shape
        ovrlay = int(n / 2)
        res_matrix = np.copy(timage)
        for i in np.arange(image_shape[0])[1:-1]:
            local_matrix = timage[i - ovrlay:i + ovrlay + 1]
            median = np.median(local_matrix)
            res_matrix[i] = median
        return res_matrix

    def solve_p4(self):
        """Solve problem #4."""
        I = np.array([10] * 5 + [40] * 5)
        filter1 = np.ones(5)/5
        filter2 = np.array([1, 2, 4, 2, 1]) / 10

        O1 = self.apply_1d_filter(filter1, I).astype(int)
        O2 = self.apply_1d_filter(filter2, I).astype(int)
        print("\n\nSolution for problem #4.")
        print("Filter (a)")
        print(O1)
        print("Filter (b)")
        print(O2)
        return O1, O2

    @staticmethod
    def apply_2d_median_filter(n, timage):
        """Applying a nxn median filter on the image I assuming that the
        border pixels are not changed."""
        image_shape = timage.shape
        ovrlay = int(n / 2)
        res_matrix = np.copy(timage)
        for i in np.arange(image_shape[0])[1:-1]:
            for j in np.arange(image_shape[1])[1:-1]:
                local_matrix = timage[i - ovrlay:i + ovrlay + 1,
                                      j - ovrlay:j + ovrlay + 1]
                median = np.median(local_matrix)
                res_matrix[i, j] = median
        return res_matrix

    def solve_p5(self, show=True):
        """Solve problem #5."""
        print("\n\nSolution for problem #5.")
        if show:
            fig = plt.figure()
            ax1 = fig.add_subplot(121)
            ax1.stem([-100, 0, 100], np.array([49, 42, 9])/100)
            ax1.set_ylabel("Probability")
            ax1.set_xlabel("Output")
            ax1.set_title("On the line")
            ax2 = fig.add_subplot(122)
            ax2.stem([-150, -50, 50, 100], np.array([21, 9, 21, 49])/100)
            ax2.set_ylabel("Probability")
            ax2.set_xlabel("Output")
            ax2.set_title("On the adjacent line")
            fig.show()

    def solve_p6(self):
        """Solve problem #6."""
        I = np.zeros((8, 8)).astype(int)
        for i in np.arange(8):
            for j in np.arange(8):
                I[i, j] = np.abs(i - j)
        O = self.apply_2d_median_filter(3, I).astype(int)
        print("\n\nSolution for problem #6.")
        print(I)
        print(O)
        return I, O

    def solve_p7(self):
        """Solve problem #7."""
        I = np.array([4] * 4 + [8] * 4)
        O1 = self.apply_1d_median_filter(3, I)
        avgfilter = np.array([1, 2, 1]) / 4
        O2 = np.copy(I)
        O2[1:-1] = self.apply_1d_filter(avgfilter, I)[1:-1]

        print("\n\nSolution for problem #7.")
        print("Median Filter")
        print(O1)
        print("Average Mask")
        print(O2)
        return O1, O2

if __name__ == "__main__":
    hw2 = Homework2()
    hw2.solve_p1()
    hw2.solve_p2()
    hw2.solve_p3()
    hw2.solve_p4()
    hw2.solve_p5()
    hw2.solve_p6()
    hw2.solve_p7()

\end{python}
\end{document}

\begin{equation*}
\begin{aligned}
G(x, y) = &\left[ \begin{array}{ccccc}
0.01214612 & 0.02610994 & 0.03369732 & 0.02610994 & 0.01214612\\
0.02610994 & 0.0561273  & 0.07243752 & 0.0561273  & 0.02610994\\
0.03369732 & 0.07243752 & 0.09348738 & 0.07243752 & 0.03369732\\
0.02610994 & 0.0561273  & 0.07243752 & 0.0561273  & 0.02610994\\
0.01214612 & 0.02610994 & 0.03369732 & 0.02610994 & 0.01214612\\
\end{array} \right] = G(x)G(y)\\
= &\left[ \begin{array}{c}
0.11020946 \\
0.23691201 \\
0.30575706 \\
0.23691201 \\
0.11020946
\end{array} \right] \times \left[ \begin{array}{ccccc}
0.11020946 & 0.23691201 & 0.30575706 & 0.23691201 & 0.11020946
\end{array} \right]
\end{aligned}
\end{equation*}


\section{Model Functions}
The main functions in this model include converting from Keplerian orbital elements to Cartesian vectors and converting from Cartesian vectors to Keplerian orbital elements. Orbital elements used for this conversion include the semimajor axis, eccentricity, inclination, ascending node, argument of periapses, and true anomaly. The Cartesian parameters consist of position and velocity vectors. These conversions are able to be performed with a variety of inputs, so listed below are the orbit types available for accurate conversion.

\begin{itemize}
	\item \textbf{Elliptic Orbit} \boldmath($0<e<1.0$, \quad $a>0$)\unboldmath
	\begin{itemize}
		\item Inclined: $i>0$, \quad $\Omega>0$
		\item Equatorial: $i=0$, \quad $\Omega=0$
	\end{itemize}
	\item \textbf{Circular Orbit} \boldmath($e=0$, \quad $a>0$, \quad $\omega=0$)\unboldmath
	\begin{itemize}
		\item Inclined: $i>0$, \quad $\Omega>0$
		\item Equatorial: $i=0$, \quad $\Omega=0$
	\end{itemize}
	\item \textbf{Parabolic orbit} \boldmath($e= 1.0$, \quad $a=-r_p$)\unboldmath
	\begin{itemize}
		\item Inclined: $i>0$, \quad $\Omega>0$
		\item Equatorial: $i=0$, \quad $\Omega=0$
	\end{itemize}
	\item \textbf{Hyperbolic orbit} \boldmath($e>1.0$, \quad $a<0$)\unboldmath
	\begin{itemize}
		\item Inclined: $i>0$, \quad $\Omega>0$
		\item Equatorial: $i=0$, \quad $\Omega=0$
	\end{itemize}
\end{itemize}

\section{Model Assumptions and Limitations}
\subsection{Assumptions}
\begin{itemize}
	\item The origin of the inertial frame is coincident with the geocentric equatorial system.
	\item The attracting body is specified by the supplied gravity constant $\mu [\frac{km^3}{s^2}]$.
\end{itemize}
\subsection{Limitations}
\begin{itemize}
	\item \textbf{\boldmath For input $e \geq 1.0$}
	\begin{itemize}
		\item Rectilinear orbits are not supported with this module because their angular momentum equals zero. However, Cartesian vectors can be obtained from orbital elements because this only affects the Cartesian to element conversion. Regardless, this case will not be needed when using the orb\_elem\_convert module.
		\item The semimajor axis input must be negative.
	\end{itemize}
	\item \textbf{\boldmath For input $e = 0$}
	\begin{itemize}
		\item The argument of periapsis input must be zero.
	\end{itemize}
	\item \textbf{\boldmath For input $a = 0$}
	\begin{itemize}
		\item The ascending node input must be zero.
	\end{itemize}
	\item \textbf{\boldmath For input $a<0$}
	\begin{itemize}
		\item The eccentricity must be greater than or equal to 1.0.
	\end{itemize}
	\item \textbf{Orbital element angles ($i$, $\Omega$, $\omega$, $f$)}
	\begin{itemize}
		\item Must be greater than or equal to zero
		\item Must be radians
	\end{itemize}
\end{itemize}


\section{Server Farms}
\label{sec:server-farms}

A server farm is a collection of servers that cooperates to handle incoming requests.
A server farm is preferable to a super-fat server because it is cheaper and more flexible. This practical advantages have made server farms ubiquitous.

A server farm can be modeled with a $M/M/k/k$ queue and with a $M/M/k$ queue.

These queues are both birth-death processes, thus they are both time-reversible.