\begin{table*}[tb]
	\tbl{Set operations available in LazySets. The distinction between lazy and concrete functions is explained in Section~\ref{sec:lazy}. The result of the last three operations is generally not convex even if used with convex operands. For binary operations (marked with \binary{$\cdot$}) there is also an $n$-ary lazy version with the suffix \code{Array}, e.g., \code{MinkowskiSumArray}. Unicode symbols (as mentioned in the column ``Short form'') are entered in the Julia REPL by typing the \LaTeX\ command (e.g.: \code{\textbackslash{}oplus} for $\oplus$) followed by pressing the ``Tab'' key. See Appendix~\ref{sec:setops} for some central definitions.}{
	\begin{tabular}{l l l l l}
		\toprule
		Operation name & Math form & Lazy function (constructor) & Short form & Concrete function \\
		\midrule
		%
		\binary{Minkowski sum} & $\X \oplus \Y$ & \code{MinkowskiSum} & \code{+},\code{\textbackslash{}oplus} & \code{minkowski$\_$sum} \\
		%
		\binary{Intersection} & $\X \cap \Y$ & \code{Intersection} & \code{\textbackslash{}cap} & \code{intersection} \\
		%
		\binary{Cartesian product} & $\X \times \Y$ & \code{CartesianProduct} & \code{*},\code{\textbackslash{}times} & \code{cartesian$\_$product} \\
		%
		\binary{Convex hull} & $\CH(\X \cup \Y)$ & \code{ConvexHull} & \code{CH} & \code{convex$\_$hull} \\
		%
		Symmetric interval hull & $\boxdot(\X)$ & \code{SymmetricIntervalHull} & \code{\textbackslash{}boxdot} & \code{symmetric$\_$interval$\_$hull} \\
		%
		Linear map & $A \X$ & \code{LinearMap} & \code{*} & \code{linear$\_$map} \\
		%
		Exponential map & $e^A \X$ & \code{ExponentialMap} && \code{exponential$\_$map} \\
		%
		Translation & $\X + b$ & \code{Translation} & \code{+} & \code{translate} \\
		%
		Affine map & $A \X + b$ & \code{AffineMap} & \code{*} and \code{+} & \code{affine$\_$map} \\
		%
		Reset map & $x_i \mapsto c$ & \code{ResetMap} && - \\
		%
		Inverse linear map & $A^{-1} \X$ & \code{InverseLinearMap} && - \\
		%
		Bloating & $\X \oplus \{x : \Vert x \Vert \leq \varepsilon\}$ & \code{Bloating} && - \\
		%
		\midrule
		%
		\binary{Union} & $\X \cup \Y$ & \code{UnionSet} & \code{\textbackslash{}cup} & - \\
		%
		Complement & $\X^C$ & \code{Complement} && \code{complement} \\
		%
		Rectified linear unit & $x_i \mapsto \max(x_i, 0)$ & \code{Rectification} && \code{rectify} \\
		\bottomrule
	\end{tabular}}
	\label{tab:operations}
\end{table*}

LazySets offers support for \emph{convex} and \emph{non-convex} sets.
%
Intuitively, a set $\X$ is convex if one can draw a straight line segment between any two points in $\X$ without leaving $\X$ (see Appendix~\ref{sec:convexdef} for a formal definition).
This explains why optimization over a convex set is efficient.
Convex sets enjoy several other attractive properties, and many important geometric shapes are convex.
%

\subsection{Constructing sets}

Two basic sets are the \emph{hyperplane}
%
\[
	\{x \in \R^n \mid a^T x = b\},
\]
%
which is parametric in a vector $a \in \R^n$ and a scalar $b \in \R$, and the \emph{half-space} (or \emph{linear constraint})
%
\[
	\{x \in \R^n \mid a^T x \leq b\},
\]
%
which consists of all points on one side of the corresponding hyperplane.
In LazySets these sets are constructed from $a$ and $b$. For example, the two-dimensional hyperplane $x = 1$ (resp. the half-space $x \leq 1$) are:

\begin{minipage}{\linewidth}
\vspace{-\abovedisplayskip}
\begin{lstlisting}
julia> a = [1.0, 0.0]; b = 1.0

julia> Hyperplane(a, b)
Hyperplane{Float64,Vector{Float64}}([1.0, 0.0], 1.0)

julia> HalfSpace(a, b)
HalfSpace{Float64,Vector{Float64}}([1.0, 0.0], 1.0)
\end{lstlisting}
\end{minipage}
%
Higher-dimensional sets are defined in a similar fashion; for instance, the 100-dimensional half-space $x_1 + \ldots + x_{100} \leq 10$ is:\footnote{Sometimes we abbreviate the output to improve readability.}
%
\begin{minipage}{\linewidth}
\vspace{-\abovedisplayskip}
	\begin{lstlisting}
julia> a = fill(1.0, 100); b = 10.0

julia> HalfSpace(a, b)
HalfSpace([1.0, ..., 1.0], 10.0)
	\end{lstlisting}
\end{minipage}

The most widely used convex sets in various disciplines are (convex) \emph{polyhedra}, which are characterized as the finite intersection of half-spaces.
This is also known as the \emph{H-representation}.
For bounded polyhedra, which are called \emph{polytope}, LazySets also supports a dual way to represent such sets in \emph{V-representation} as the convex hull of points.
The functions \code{tohrep(X)} and \code{tovrep(X)} can be used to convert between these representations.

Optimization over a polyhedron with linear objective corresponds to solving a linear program.
Linear programs can model a wide variety of real-life optimization problems and can be solved efficiently \cite{dantzig1998linear,kochenderfer2019algorithms}.
On the left of Fig.~\ref{fig:supfunc} we show an example of a polytope in orange, with seven (linear) constraints, and a half-space in blue.

\smallskip

LazySets contains many (currently: $26$) different structs to represent common classes of sets (such as half-spaces).
These set types simply expect and store the corresponding parameters that represent the set.
For example, the \code{HalfSpace} stores the vector \code{a} and the scalar \code{b}.
(There are a few exceptions where the constructor performs normalization by default, e.g., \code{HPolygon}, representing a two-dimensional polytope, sorts the constraints by the vectors \code{a} in counter-clockwise order.)
Hence construction is fast and the internal representation is space efficient.
For instance, the \code{BallInf} represents a hypercube specified by the center vector $c \in \R^n$ and the radius $r \in \R$.
In $n$ dimensions, a hypercube has $2^n$ vertices, but creating an $1{,}000$-dimensional \code{BallInf} is instantaneous.

\begin{minipage}{\linewidth}
\vspace{-\abovedisplayskip}
\begin{lstlisting}
julia> @time BallInf(zeros(1000), 1.0)
0.000005 seconds (2 allocations: 7.969 KiB)
\end{lstlisting}
\end{minipage}

\subsection{Extracting information from sets}

Being able to represent sets is not useful by itself because we also want to interact with them.
For example, we may want to draw samples from a set.
A general approach to do that is rejection sampling, which picks a random point $x \in \R^n$ and checks whether $x \in \X$ holds.
We can thus use rejection sampling with any set type that implements a membership test.

\begin{minipage}{\linewidth}
\vspace{-\abovedisplayskip}
\begin{lstlisting}
julia> ones(1000) ∈ BallInf(zeros(1000), 1.0)
true
\end{lstlisting}
\end{minipage}

Other typical properties that can be checked for sets $\X$ and $\Y$ are emptiness ($\X = \emptyset$; \code{isempty}), inclusion ($\X \subseteq \Y$; \code{issubset}), and having no point in common ($\X \cap \Y = \emptyset$; \code{isdisjoint}).
The complexity of such operations depends on the representation of the set.
Take for example a polyhedron represented by the list of its linear constraints.
Emptiness can be checked by formulating a feasibility linear program.
Inclusion and disjointness can be checked using the support function (see Section~\ref{sec:supfun}).

\smallskip

We may also want to obtain information that is encoded in the set representation.
For example, we can ask for the list of vertices of a polytope.
We have seen that a hypercube is represented by the center and the radius, so the vertices need to be computed on demand.

\begin{minipage}{\linewidth}
\vspace{-\abovedisplayskip}
\begin{lstlisting}
julia> vertices_list(BallInf([1.0, 4.0], 1.0))
4-element Vector{Vector{Float64}}:
 [2.0, 5.0]
 [0.0, 5.0]
 [2.0, 3.0]
 [0.0, 3.0]
\end{lstlisting}
\end{minipage}

Equality of sets in the mathematical sense can be checked via \code{isequivalent} (which by default checks mutual inclusion):

\begin{minipage}{\linewidth}
\vspace{-\abovedisplayskip}
\begin{lstlisting}
julia> X = Interval(-1, 1) × Interval(-1, 1)
CartesianProduct{Float64,
  Interval{...}, Interval{...}}(...)

julia> Y = BallInf(zeros(2), 1.0)

julia> isequivalent(X, Y)
true
\end{lstlisting}
\end{minipage}

\subsection{Set interfaces}

Sometimes the same implementation works for several set types.
LazySets uses a hierarchy of abstract types (which we call \emph{interfaces}) to summarize common functionalities.
For example, \code{AbstractHyperrectangle} is a supertype of all hyperrectangular set types such as \code{BallInf} and provides a default implementation to compute the volume.
When adding a new set type representing a hyperrectangle, it will automatically use this implementation.

The following list is not exhaustive, but should help as a mental model of how the library is organized. Definitions are given from more specific to more general (i.e., less structured).

\smallskip

\code{AbstractHyperrectangle}: Hyperrectangular sets can be represented by a center vector $c \in \R^n$ and a radius vector $r \in \R^n$. Each $x \in \X$ can be written as $x_i = c_i + \xi_i r_i$ for $i = 1,\ldots, n$, for some $\xi_i \in [-1, 1]$. Implementations include intervals (\code{Interval}), hypercubes (\code{BallInf}), and the general \code{Hyperrectangle}.

\smallskip

\code{AbstractZonotope}: Zonotopic sets are those which admit a representation given by a center $c \in \R^n$ and a finite set of \emph{generators} $g_j \in \R^n$, $j \in 1, \ldots, m$, such that $x \in \X$ is can be written as $x = c + \sum_j \xi_j g_j$ for some $\xi_j \in [-1, 1]$. Hyperrectangular sets are also zonotopic, as well as general zonotopes (\code{Zonotope}).

\smallskip

\code{AbstractPolyhedron}: A set is called polyhedral if it can be expressed as a finite intersection of half-spaces. Special cases include hyperrectangular and zonotopic sets, as well as more general polytopes (\code{HPolytope}, \code{VPolytope}) and also possibly unbounded polyhedra (\code{HPolyhedron}).

\smallskip

\code{LazySet}: All convex set types belong to this abstract supertype to prevent type piracy when extending \code{Base} functions.
We are working toward having non-convex sets, such as set unions, in the same type hierarchy as well.

\subsection{Set operations}

We have seen that we can interact with sets by checking properties.
Importantly, we can also apply set operations to sets for constructing new sets.
(By default the result is a new set instance and the original set instance is not manipulated.)
For details about the complexity for common set representations we refer to \cite[Table~1]{althoff2020set}.
For example, one common set operation is to translate (or shift) every element in the set by a constant vector.

\begin{minipage}{\linewidth}
\vspace{-\abovedisplayskip}
\begin{lstlisting}
julia> B1 = BallInf([1.5, 2.0], 1.0)
julia> B2 = translate(B1, [1.5, -1.0])
julia> dump(B2)
BallInf{Float64, Vector{Float64}}
  center: Array{Float64}((2,)) [3.0, 1.0]
  radius: Float64 1.0
\end{lstlisting}
\end{minipage}

As seen above, a translation usually preserves the set type.
For most operations this is generally not the case.
For instance, the intersection of two half-spaces is itself not a half-space but a polyhedron.

\begin{minipage}{\linewidth}
\vspace{-\abovedisplayskip}
\begin{lstlisting}
# intersect {x | x <= 1} and {x | x >= 0}
julia> P = intersection(HalfSpace([1.0], 1.0),
                        HalfSpace([-1.0], 0.0))
julia> typeof(P)
HPolyhedron{Float64, Vector{Float64}}
\end{lstlisting}
\end{minipage}

For a complete list of the set operations available in LazySets we refer to Table~\ref{tab:operations}.


If you haven't downloaded and unzipped \href{https://libaoj.in/courses/2021f/MATH3341/zip/Math.3341.zip}{\texttt{Math.3341.zip}}. Download and unzip it under \verb|H:| (H Drive if you are working on the Remote Lab). Change the current working directory by typing \verb|cd H:\Math.3341\Math.3341.Lab.11| in the Command Window, and type \verb|edit lab_11_script| in the Command Window to edit \verb|lab_11_script.m|.

\section{Built-in Integration Functions}
\begin{enumerate}[(a)]
\item Use both \verb|polyint| and \verb|integral| to evaluate $\displaystyle \int_{-1}^{3} (x^2 - 2x + 1) \, dx$.
    \begin{enumerate}[(1)]
        \item Define the lower bound \verb|a| and upper bound \verb|b|.
        \item Define the polynomial \verb|P| as $p(x) = x^{2} - 2x + 1$.
        \item Integrate $p(x) = x^{2} - 2x + 1$ using \verb|polyint| and store the result to \verb|pI|.
        \item By fundamental theorem of calculus, evaluate the integral \verb|pI| on $[a, b]$ using \verb|polyval| and store the result to \verb|pI_value|.
        \item Define the anonymous function \verb|f| by $f(x) = x^2 - 2x + 1$, and then use \verb|integral| to evaluate $\displaystyle \int_{-1}^{3} (x^2 - 2x + 1) \, dx$ and store it to \verb|I|.
    \end{enumerate}
\item Evaluate the previous integral again, now using \verb|trapz| and \verb|cumtrapz|.
\item Use \verb|integral2| to evaluate $\displaystyle \int_{-\pi}^{-3\pi / 2} \int_{0}^{2 \pi} (y \sin{x} + x \cos{y}) \, dy \, dx$.
\item Use \verb|integral3| to evaluate $\displaystyle \int_{0}^{1} \int_{x^2}^{x} \int_{x-y}^{x+y} y \, dz \, dy \, dx$.
\end{enumerate}
\section{Gauss Quadrature}
\begin{enumerate}[(a)]
    \item Implement Gauss quadrature using $n$ Gauss nodes, which is given by Equation \eqref{eq:gauss}, in the function file \verb|gauss_quad.m|.
        \begin{equation}
            \label{eq:gauss}
            \int_{-1}^{1} f(x) \, dx \approx \sum_{i=1}^{n} w_i f(x_i).
        \end{equation}
    \item Use \verb|gauss_quad| to evaluate the integral
        $$
        \int_{1}^{1.6} \frac{2x}{x^2 - 4} \, dx,
        $$
        with $n = 1, 2, \ldots , 15$ Gauss nodes.

        Note: \verb|legendre_pair.m| is provided to calculate $x_i$ and $w_i$. Use \verb|help legendre_pair| to check the usage.
\end{enumerate}

At last, call \verb|diary('lab_11_output.txt')|, run the scripts \verb|lab_11_script.m|, then call \verb|diary off|. You will upload the script files \verb|lab_11_script.m|, \verb|lab_11_output.txt|, and \verb|gauss_quad.m| to Overleaf. Then recompile, and submit the generated .pdf file on WyoCourses.


\begin{Definition}{a-posteriori}
  Let $u\in V$ be the solution to a boundary value in weak form and
  $u_h\in V_h$ be its finite element approximation on the mesh
  $\mesh_h$. We call a quantity $\eta_h(u_h)$ \define{a posteriori
    error estimator},
  \begin{gather}
    \norm{u-u_h} \le c \eta_h(u_h).
  \end{gather}
  The estimator is \define{reliable}, if the constant $c$ is
  computable. It is \define{efficient}, if the converse estimate holds,
  namely
%  with addition of a quantity $\osc_h(f)$
  \begin{gather}
    \eta_h(u_h) \le c \norm{u-u_h}.
  \end{gather}
\end{Definition}

\subsection{Quasi-interpolation in $H^1$}

\begin{intro}
  Interpolation in Sobolev spaces in
  \slideref{Theorem}{fe-interpolation} relies on the nodal
  interpolation operator in
  \slideref{Definition}{nodal-interpolation}, which in turn requires
  point values of the interpolated function. Therefore, it is not
  defined on $H^1(\domain)$ in dimensions greater than one. Since we
  need such interpolation operators in the analysis of a posteriori
  error estimates, we provide them in this section.

  Most details in this section are from~\cite{Verfuerth13}. For the
  ease of presentation, we present the results for Dirichlet problem
  of the Laplacian, namely $V = H^1_0(\domain)$ and
  \begin{gather*}
    a(u,v) = \int_\domain \nabla u\cdot\nabla v\dvx.
  \end{gather*}
\end{intro}

\begin{Definition}{locally-quasi-uniform}
  A shape-regular family of meshes $\{\mesh_h\}$ is called
  \define{locally quasi-uniform}, if there is a constant $c$ such that
  for every pair of cells $\cell_1$ and $\cell_2$ sharing at least one
  vertex there holds
  \begin{gather}
    h_{\cell_1} \le c h_{\cell_2}.
  \end{gather}
\end{Definition}

\begin{Definition}{fem-neighborhood}
  For a vertex or higher-dimensional boundary facet $F$, we define the
  set of cells
  \begin{gather}
    \mesh_{F} = \bigl\{ \cell\in\mesh \big|F \subset\d\cell \bigr\}.
  \end{gather}
  Similarly, the set of cells sharing at least one vertex with $\cell$
  is called $\mesh_{\cell}$. Additionally, we define the subdomains
  \begin{gather}
    \overline\domain_{F} = \bigcup_{\cell\in \mesh_{F}}\overline\cell,
    \qquad
    \overline\domain_{\cell} = \bigcup_{\cell'\in \mesh_{\cell}}\overline\cell'.
  \end{gather}
\end{Definition}

\begin{Theorem*}{clement}{Clément quasi-interpolation}
  Let $\{\mesh_h\}$ be a locally quasi-uniform family of meshes with
  piecewise polynomial finite element spaces
  $V_h\subset H^1_0(\domain)$. Then, there exist bounded operators
  $\overline I_h\colon H^1_0(\domain) \to V_h$ such that for every
  function $u\in H^1_0(\domain)$, every mesh cell $\cell$, every face
  $F$, and for $m=0,1$ there holds
  \begin{align}
    \norm{u-\overline I_h u}_{m;T} &\le c h_\cell^{1-m} \snorm{u}_{1;\domain_\cell} \\
    \norm{u-\overline I_h u}_{0;F} &\le c h_\cell^{\nicefrac{1}{2}} \snorm{u}_{1;\domain_\cell}
  \end{align}
\end{Theorem*}

\begin{proof}
  We construct the quasi-interpolation operator on a mesh cell $\cell$
  into the lowest order space $\P_1$ or $\Q_1$ in two steps. First,
  for each vertex $\vertex_i$ of $\cell$, let
  \begin{gather}
    \overline u_i = \frac1{\abs{\domain_{\vertex_i}}}
    \int_{\domain_{\vertex_i}} u \dvx.
  \end{gather}
  By Poincaré inequality,
  \begin{gather}
    \label{eq:fem-aposteriori:1}
    \norm{u-\overline u_i}_{\domain_{\vertex_i}}
    \le c \diam(\domain_{\vertex_i}) \norm{\nabla u}_{\domain_{\vertex_i}}.
  \end{gather}
  For a vertex $\vertex_i$ on the boundary, we observe
  \begin{gather}
    \label{eq:fem-aposteriori:2}
    \begin{split}
      \norm{\overline u_i}_{\domain_{\vertex_i}}^2
      &= \frac1{\abs{\domain_{\vertex_i}}^2}
      \int_{\domain_{\vertex_i}}\left(\int u \dvy\right)^2\dvx\\
      & = \frac1{\abs{\domain_{\vertex_i}}}\left(\int 1 u \dvy\right)^2\\
      & \le \frac1{\abs{\domain_{\vertex_i}}} \int 1^2\dvy \int u^2\dvy \\
      & = \norm{u}^2_{{\domain_{\vertex_i}}}\\
      & \le c \diam({\domain_{\vertex_i}})^2 \norm{\nabla u}_{{\domain_{\vertex_i}}}^2,
    \end{split}
  \end{gather}
  by Friedrichs' inequality, since the subdomain
  ${\domain_{\vertex_i}}$ has at least one face on $\d\domain$.

  Now, we define the quasi-interpolation operator
  $\overline I_h\colon H^1_0(\domain)\to V_h$ cellwise on simplices by
  \begin{gather}
    \label{eq:fem-aposteriori:3}
    \overline I_h u_{|\cell}
    = \sum_{\vertex_i\in\d\cell\cap\domain} \lambda_i \overline u_i.
  \end{gather}
  Note, that zero boundary conditions are enforced by omitting
  vertices on the boundary.  On quadrilaterals, we use the bilinear
  shape functions associated with the vertices instead of the
  \putindex{barycentric coordinates} $\lambda_i$. Now, using
  $\sum\lambda_i=1$ and $0\le \lambda_i \le 1$, we estimate
  \begin{align*}
    \norm{u - \overline I_h u}_\cell
    & \le \sum_{\vertex_i\in\d\cell} \norm{\lambda_i(u-\overline u_i)}_{\cell}
      + \sum_{\vertex_i\in\d\cell\cap\d\domain}
      \norm{\lambda_u \overline u_i} \\
    & \le \sum_{\vertex_i\in\d\cell} \norm{u-\overline u_i}_{\cell}
      + \sum_{\vertex_i\in\d\cell\cap\d\domain}
      \norm{\overline u_i} \\
    & \le \diam({\domain_{\vertex_i}})\left(
      \sum_{\vertex_i\in\d\cell} \norm{\nabla u}_{\domain_{\vertex_i}}
      + \sum_{\vertex_i\in\d\cell\cap\d\domain} \norm{\nabla u}_{\domain_{\vertex_i}}
      \right),
  \end{align*}
  where we used~\eqref{eq:fem-aposteriori:1}
  and~\eqref{eq:fem-aposteriori:2} in the end. Observing that both
  sums extend over finitely many vertices and that by local
  quasi-uniformity we can bound the diameter of
  $\diam({\domain_{\vertex_i}})$ by that of $\cell$, namely
  $\diam({\domain_{\vertex_i}}) \le c h_\cell$, we obtain the estimate
  in $L^2(\cell)$. For the estimate in $H^1(\cell)$, we observe that
  the mean value calculation is a continuous operation on
  $H^1_0(\domain(\vertex_i))$, and that~\eqref{eq:fem-aposteriori:3} as a finite
  sum is continuous, therefore,
  \begin{gather}
    \snorm{u-\overline I_h u}_{1;\cell} \le \norm{\nabla u}_{\cell} +
    \norm{\nabla \overline I_h u}_{\cell}
    \le c \norm{\nabla u}_{\domain(\cell)}.
  \end{gather}
  Finally, we use the trace estimate for $u\in H^1(\cell)$
  \begin{gather}
    \norm{u}_F
    \le c \left(h^{-\nicefrac12} \norm{u}_\cell
      +h^{\nicefrac12} \norm{\nabla u}_\cell\right),
  \end{gather}
  to obtain the estimate on the edge.
\end{proof}

\begin{Theorem*}{scott-zhang}{Scott-Zhang quasi-interpolation}
  Let $\{\mesh_h\}$ be a locally quasi-uniform family of meshes with
  piecewise polynomial finite element spaces
  $V_h\subset H^1(\domain)$. Then, there exist bounded operators
  $\overline I_h\colon H^1(\domain) \to V_h$ such that for every
  function $u\in H^1(\domain)$, every mesh cell $\cell$, every face
  $F$, and for $m=0,1$ there holds
  \begin{align}
    \norm{u-\overline I_h u}_{m;T} &\le c h_\cell^{1-m} \snorm{u}_{1;\domain_\cell} \\
    \norm{u-\overline I_h u}_{0;F} &\le c h_\cell^{\nicefrac{1}{2}} \snorm{u}_{1;\domain_\cell},
  \end{align}
  and $\overline I_h u$ is a quasi-interpolation on the boundary $\d\domain$.
\end{Theorem*}

\begin{Theorem*}{schoeberl-interpolation}{Schöberl quasi-interpolation}
  Let $\{\mesh_h\}$ be a locally quasi-uniform family of meshes with
  piecewise polynomial finite element spaces
  $V_h\subset H^1(\domain)$. Then, there exist bounded
  \emph{projection} operators
  $\overline I_h\colon H^1(\domain) \to V_h$ such that for every
  function $u\in H^1(\domain)$, every mesh cell $\cell$, every face
  $F$, and for $m=0,1$ there holds
  \begin{align}
    \norm{u-\overline I_h u}_{m;T} &\le c h_\cell^{1-m} \snorm{u}_{1;\domain_\cell} \\
    \norm{u-\overline I_h u}_{0;F} &\le c h_\cell^{\nicefrac{1}{2}} \snorm{u}_{1;\domain_\cell}.
  \end{align}
\end{Theorem*}

\begin{Definition}{residual}
  Let $a(u,v) = f(v)$ be the weak formulation of a BVP on the space
  $V$.  Then, we define the \define{residual}
\begin{gather}
  \begin{split}
    R\colon V &\to V^* \\
    w&\mapsto f(\cdot) - a(w,\cdot).
  \end{split}
\end{gather}
\end{Definition}
\begin{Lemma}{residual-hm1}
  For $u\in V=H^1_0(\domain)$ solution to Poisson's equation and any
  other function $w\in V$, there holds
  \begin{gather}
    \snorm{u-v}_1 = \norm{R v}_{-1}
    := \sup_{w\in V} \frac{\left<R v,w\right>}{\snorm{w}_1}.
  \end{gather}
\end{Lemma}

\begin{proof}
  We have
  \begin{gather}
    \left<Rv,w\right>
    = f(w) - a(v,w)
    = a(u-v,w).
  \end{gather}
  Since $a(.,.)$ is s.p.d., we can apply Cauchy-Schwarz to obtain the result.
\end{proof}

\begin{Definition}{mvl-jmp}
  Let $u$ be a piecewise continuous function on a mesh $\mesh$. On a
  face $F$ between two cells $\cell_1$ and $\cell_2$, we define the
  \define{mean value operator}
  \begin{gather}
    \mvl{u}(\vx) = \frac{u_1(\vx) + u_2(\vx)}2
    = \lim_{\epsilon\searrow 0}
    \frac{u(\vx-\epsilon \n_1) + u(\vx - \epsilon\n_2)}2,
  \end{gather}
  where $\n_i$ are the outer normal vectors of the two cells. The
  \define{jump operator} is
  \begin{gather}
    \mvl{u\n} = \frac{(u_1-u_2)\n_1}2 = \frac{(u_2-u_1)\n_2}2.
  \end{gather}
\end{Definition}
\begin{Definition}{residual-strong}
  Let $u\in V=H^1_0(\domain)$ be the solution to Poisson's equation
  and let $u_h\in V_h$ be a finite element function.  The strong form
  of the residual is
  \begin{gather}
    \left<Rv,w\right> = \sum_{\cell\in\mesh_h}\int_{\cell} r_\cell(u_h) w\dvx
    - \sum_{F\in \mathbb F_h^i} 2\mvl{\n\cdot\nabla u_h} w\ds,
  \end{gather}
  where
  \begin{gather}
    r_\cell(u_h) = f + \Delta u_h.
  \end{gather}
\end{Definition}

\begin{Lemma}{residual-upper-bound}
  Let $\mesh_h$ be a locally quasi-uniform mesh.
  There is a constant $c>0$ such that the error between the true
  solution $u\in V=H^1_0(\domain)$ and the finite element solution
  $u_h\in V_h \subset V$ is bounded by
  \begin{gather}
    \label{eq:fem-aposteriori:5}
    \snorm{u-u_h}_{1;\mesh_h} \le c \left(
      \sum_{\cell\in\mesh_h} h_\cell^2 \norm{r_\cell(u_h)}_\cell^2
      + \sum_{F\in \mathbb F_h^i} h_F
      \norm{\mvl{\n\cdot\nabla u_h}}_F^2
    \right)^{\nicefrac12}.
  \end{gather}
\end{Lemma}

\begin{proof}
  We begin with the equivalence of error in $V$ and residual in
  $V^*$. For the residual, there holds Galerkin orthogonality, such
  that we obtain
  \begin{gather}
    \label{eq:fem-aposteriori:4}
    \snorm{u-u_h}_1
    = \sup_{w\in V}\frac{\left<Ru_h,w\right>}{\snorm{w}_1}
    = \sup_{w\in V}\frac{\left<Ru_h,w- \overline I_h w\right>}{\snorm{w}_1}.
  \end{gather}
  Using the strong form and the quasi-interpolation $\overline I_h$,
  we get
  \begin{align*}
    \left<Ru_h,w\right>
    &= \sum_{\cell\in\mesh_h}\int_{\cell} r_\cell(u_h) (w-\overline I_h w)\dvx
    - \sum_{F\in \mathbb F_h^i} 2\mvl{\n\cdot\nabla u_h} (w-\overline I_h w)\ds
    \\
    & \le \sum_{\cell\in\mesh_h} \norm{r_\cell(u_h)}_\cell
      \norm{w-\overline I_h w}_\cell
      + \sum_{F\in \mathbb F_h^i} \norm{\mvl{\n\cdot\nabla u_h}}_\cell
      \norm{w-\overline I_h w}_\cell
      \\
    & \le c_1 \sum_{\cell\in\mesh_h} h_\cell
      \norm{r_\cell(u_h)}_\cell
      \norm{\nabla w}_{\domain_\cell}
      + c_2 \sum_{F\in \mathbb F_h^i} h_E^{\nicefrac12}
      \norm{\mvl{\n\cdot\nabla u_h}}_\cell
      \norm{\nabla w}_{\domain_\cell}
  \end{align*}
  Applying Hölder inequality, we obtain for the first term
  \begin{gather*}
    \sum_{\cell\in\mesh_h} h_\cell
    \norm{r_\cell(u_h)}_\cell
    \norm{\nabla w}_{\domain_\cell}
    \le
    \left(\sum_{\cell\in\mesh_h} h_\cell^2
    \norm{r_\cell(u_h)}_\cell^2\right)^{\nicefrac12}
  \left(\sum_{\cell\in\mesh_h} \norm{\nabla w}_{\domain_\cell}^2
  \right)^{\nicefrac12},
  \end{gather*}
  and similar for the second term. Both contain a term of the form
  \begin{gather*}
    \sum_{\cell\in\mesh_h} \norm{\nabla w}_{\domain_\cell}^2
    = \sum_{\cell\in\mesh_h} \sum_{\cell'\in\mesh_\cell}\norm{\nabla w}_{\cell'}^2
    \le n \norm{\nabla w}_{\domain}^2,
  \end{gather*}
  where $n$ is the maximal number of occurrences of a cell $\cell'$ in
  the double sum. This $n$ is bounded uniformly on shape regular
  family of meshes, since shape regularity prohibits degeneration of
  cells. Therefore, we conclude
  \begin{align*}
    \left<Ru_h,w\right>
    & \le c \left(\sum_{\cell\in\mesh_h} h_\cell^2
      \norm{r_\cell(u_h)}_\cell^2 +
      \sum_{F\in \mathbb F_h^i} h_E
      \norm{\mvl{\n\cdot\nabla u_h}}_\cell^2\right)^{\nicefrac12}
      \snorm{w}_{1;\domain}.
  \end{align*}
  Entering into~\eqref{eq:fem-aposteriori:4} yields the proposition.
\end{proof}

\begin{remark}
  The constant $c$ in the previous theorem depends on the constant in
  the quasi-interpolation estimate, which in turn was derived using
  Poincaré and trace inequalities. Traditionally, both are derived
  using indirect arguments, but a constructive proof with computable
  bounds is possible. For details, see~\cite[Chapter
  3]{Verfuerth13}. There still remains the question, whether these
  bounds are sufficiently sharp to be applicable in practice.
\end{remark}

\begin{Definition}{osc}
  For a function $f\in L^2(\domain)$, we define the cell-wise average
  \begin{gather}
    \overline f(\vx) = \frac1{\abs{\cell}}\int_\cell f \dvx
    \qquad\vx\in\cell, \quad\cell\in\mesh_h,
  \end{gather}
  and the \define{data oscillation}
  \begin{gather}
    \operatorname{osc}_\cell f = \norm{f-\overline f}_\cell.
  \end{gather}
\end{Definition}

\begin{Definition}{residual-estimator}
  The residual based error estimator for the finite element method for
  Poisson's equation on a mesh $\mesh$ is defined as
  \begin{gather}
    \eta_{h}(u_h) = \sum_{\cell\in\mesh} \eta_{\cell}(u_h),
  \end{gather}
  where
  \begin{gather}
    \eta_{\cell}(u_h)^2 = h_{\cell}^2 \norm{\overline f + \Delta u_h}_\cell^2
    + \frac12 \sum_{F\subset \d\cell} h_F\norm{\mvl{n\cdot\nabla u_h}}_E^2.
  \end{gather}
\end{Definition}

\begin{Theorem}{residual-estimate}
  There holds with positive constants $c_1$ and $c_2$
  \begin{align}
    \label{eq:fem-aposteriori:6}
    c_1 \snorm{u-u_h}_1^2
    &\le \eta_{h}(u_h)^2
      + \sum_{\cell\in\mesh_h} h^2_\cell\operatorname{osc}_\cell^2 f
    \\
    \label{eq:fem-aposteriori:7}
    c_2 \,\eta_\cell^2(u_h)
    &\le \snorm{u-u_h}_{1;\domain_{\cell}}^2
      + \sum_{\cell\in\mesh_\cell} h^2_\cell\operatorname{osc}_\cell^2 f
  \end{align}
\end{Theorem}

\begin{proof}
  Note that the right hand side of the
  estimate~\eqref{eq:fem-aposteriori:5} in
  \slideref{Lemma}{residual-upper-bound} differs from the estimator
  $\eta_h(u_h)$ only by the replacement of $f$ by $\overline
  f$. Therefore, we can start the proof of this lemma by
  changing~\eqref{eq:fem-aposteriori:4} to
  \begin{gather}
    \snorm{u-u_h}_1
    = \sup_{w\in V}\frac{\left<Ru_h,w\right>}{\snorm{w}_1}
    = \sup_{w\in V}\frac{\left<\overline Ru_h,w\right> + \form(f-\overline f, w)}{\snorm{w}_1},
  \end{gather}
  where we ad hoc defined $\overline R$ like $R$, but replacing
  $r_\cell = f+\nabla u_h$ by $\overline f + \nabla u_h$. Due to
  equality, we can still subtract the quasi-interpolant of $w$ and
  continue through the whole proof. What remains is the estimate
  \begin{gather}
    \label{eq:fem-aposteriori:8}
    \form(f-\overline f, w - \overline I_h w)
    \le c \sum_{\cell\in\mesh} h_\cell \norm{f-\overline f}_{\cell} \norm{\nabla w}_{\domain_\cell}
    \le c \left(\sum_{\cell\in\mesh} h_\cell \norm{f-\overline f}_{\cell}\right)^{\nicefrac12}
      \snorm{w}_{1;\domain},
    \end{gather}
    which is proven by Hölder inequality and the boundedness of the
    number of cells in $\mesh_\cell$. Thus, the upper
    bound~\eqref{eq:fem-aposteriori:6} is an immediate consequence of
    \slideref{Lemma}{residual-upper-bound}.

    Note: the proof for the lower bound is for linear elements
    only. It can be generalized to higher order by generalizing the
    bubble functions below, which is mostly technical.

    We observe that the lower bound is local, that is, the estimate on
    each cell $\cell$ is bounded from above by the error on the halo
    $\domain_\cell$. Therefore, we start by constructing test
    functions with support on a cell. On a simplex, we define the
    \define{bubble function} in terms of \putindex{barycentric coordinates}
    \begin{gather}
      \label{eq:bubble-simplex}
      B_\cell = \frac1{(d+1)^{d+1}}\lambda_0\lambda_1\dots\lambda_d,
    \end{gather}
    while on the reference hypercube $(0,1)^d$ we let
    \begin{gather}
      \label{eq:bubble-tensor}
      \reference B_\cell(\refvx) = b(\refx_1)b(\refx_2)\dots b(\refx_d)
      \qquad b(x) = 4 x(1-x).
    \end{gather}
    Both share the following properties: they vanish on $\d\cell$,
    they are positive inside $\cell$, and $\max B_\cell=1$. From
    positivity, we deduce the existence of a constant $c$ depending on
    shape regularity and the shape function space $\shapespace(\cell)$, such that
    \begin{gather}
      \int_\cell p^2 B_\cell \dvx \ge c \norm{p}_\cell^2
      \qquad\forall p\in \shapespace(\cell).
    \end{gather}
    Furthermore, $B_\cell$ is polynomial, such that the inverse estimate holds.

    Choose now the test function
    $w_\cell = (\overline f + \Delta u_h) B_\cell$. Since it vanishes
    on the boundary and outside of $\cell$, the strong and weak form
    of the residual reduce to
    \begin{gather}
      \int_\cell r_\cell(u_h) w_\cell \dvx = \int_\cell \nabla(u-u_h)\cdot\nabla w_\cell \dvx.
    \end{gather}
    Adding the difference of $f$ and $\overline f$ on both sides yields
    \begin{gather}
      \int_\cell (\overline f+\Delta u_h)^2 B_\cell \dvx
      =\int_\cell \nabla(u-u_h)\cdot\nabla w_\cell \dvx
      + \int_\cell (\overline f - f) w_\cell \dvx.
    \end{gather}
    On the left, we estimate
    \begin{gather}
      c \norm{\overline f + \Delta u_h}_\cell^2 \le \int_\cell (\overline f+\Delta u_h)^2 B_\cell \dvx.
    \end{gather}
    On the right, we estimate the residual term by
    \begin{align*}
      \int_\cell \nabla(u-u_h)\cdot\nabla w_\cell \dvx
      & \le \snorm{u-u_h}_{1;\cell} \,\snorm{w_\cell}_{1;\cell}
      \\
      & \le c \snorm{u-u_h}_{1;\cell} \, h_\cell^{-1} \norm{\overline f + \Delta u_h}_\cell.
    \end{align*}
    The data oscillation is estimated in a straightforward way by
    \begin{align*}
      \int_\cell (\overline f-f) w_\cell \dvx
      &\le \operatorname{osc}_\cell f \norm{\overline f+\Delta u_h}_\cell
    \end{align*}
    Combining these three estimates yields
    \begin{gather}
      \label{eq:fem-aposteriori:9}
      c \norm{\overline f+\Delta u_h}_\cell
      \le h_\cell^{-1} \snorm{u-u_h}_{1;\cell}
      + \operatorname{osc}_\cell f,
    \end{gather}
    which is the desired estimate for the cell term of the estimator
    $\eta_{\cell}(u_h)$ if we multiply by $h_\cell$.

    The estimate for the face term is similar, using a bubble function
    for the face. Since this functon is nonzero on the face, its
    support extends over two mesh cells. On simplicial cells, it is
    constructed like the cell bubble in
    equation~\eqref{eq:bubble-simplex}, but extending the product only
    over indices of vertices on the face. Therefore, it is a
    polynomial of degree $d-1$ on the face and also in the interior of
    each cell. For hypercubes, it is the product of the quadratic
    polynomials $b(x)$ of variables on the face times a linear
    function decaying linearly from 1 to zero. Again, we normalize
    such that the maximum equals 1. Choosing\footnote{For linear
      elements, the derivative on the face is constant and thus this
      definition is obvious. For higher order polynomials, we need a
      an extension from the face into the interior, which can be
      obtained from the barycentric coordinates or the tensor product
      structure.} $w_F = 2\mvl{\n\cdot\nabla u_h}_F B_F$, we obtain
    \begin{align*}
      c\norm{\mvl{\n\cdot\nabla u_h}}_F^2
      &\le \int_F \mvl{\n\cdot\nabla u_h}^2 B_F \ds
      \\&= \int_F \mvl{\n\cdot\nabla u_h} w_F \ds
      \\&= \sum_{\cell\in\mesh_F} \int_\cell r_\cell w_F \dvx - \left<R,w_F\right>
      \\&= \sum_{\cell\in\mesh_F} \int_\cell r_\cell w_F \dvx
      - \int_{\domain_F} \nabla (u-u_h)\cdot\nabla w_F \dvx
      \\&= \sum_{\cell\in\mesh_F} \int_\cell (\overline f + \Delta u_h) w_F \dvx
      - \int_{\domain_F} \nabla (u-u_h)\cdot\nabla w_F \dvx
      + \sum_{\cell\in\mesh_F} \int_\cell (\overline f - f)w_F \dvx.
    \end{align*}
    Again, we estimate the three terms on the right hand side
    separately. For the first, we estimate the norm of $w_F$ by its
    trace on the edge $F$. From the already proven estimate for the
    cell residual~\eqref{eq:fem-aposteriori:9}, we obtain
    \begin{align*}
      \sum_{\cell\in\mesh_F} \int_\cell (\overline f + \Delta u_h) w_F \dvx
      & \le \sum_{\cell\in\mesh_F} \norm{\overline f + \Delta u_h}_\cell
        \,\norm{w_F}_{\cell}
      \\&\le \sum_{\cell\in\mesh_F} \norm{\overline f + \Delta u_h}_\cell
      \,c h_F^{\nicefrac12} \norm{\mvl{\n\cdot\nabla u_h}}_F
      \\&\le
      + c  \sum_{\cell\in\mesh_F} \left(h_F^{-\nicefrac12} \snorm{u-u_h}_{1;\cell}
      + h_F^{\nicefrac12}\operatorname{osc}_{\cell} f\right)\norm{\mvl{\n\cdot\nabla u_h}}_F
    \end{align*}
    Similarly, the inverse estimate for $w_F$ yields
    \begin{align*}
      \int_{\domain_F} \nabla (u-u_h)\cdot\nabla w_F \dvx
      &\le \snorm{u-u_h}_{1;\domain_F} \norm{\nabla w_F}_{\domain_F}
      \\&\le \snorm{u-u_h}_{1;\domain_F}
      \,c h_F^{-\nicefrac12}\norm{\mvl{\n\cdot\nabla u_h}}_F.
    \end{align*}
    Finally, the data oscillation terms becomes
    \begin{align*}
      \sum_{\cell\in\mesh_F} \int_\cell (\overline f - f)w_F \dvx
      &\le \sum_{\cell\in\mesh_F} \norm{\overline f - f}_\cell \norm{w_F}_\cell
      \\&\le \sum_{\cell\in\mesh_F} \norm{\overline f - f}_\cell
      \,c h_F^{\nicefrac12}\norm{\mvl{\n\cdot\nabla u_h}}_F.
    \end{align*}
    Summing and scaling with $h_F^{\nicefrac12}$ yields the result.
\end{proof}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "main"
%%% End:


\lab{Applications}{Applications of the Pseudo-Inverse}{Pseudo-Inverse Applications}

\objective{This section explains a few applications of the Moore-Penrose inverse, including least squares problems, norm-minimization problems and the condition number.}

The pseudo-inverse is a useful generalization for a variety of applications. The first we will consider is least squares and norm-minimization problems.

Consider the system
\[
Ax = b
\]

$A$ need not be full rank. It may be either over or under-determined. How do we come up with a ``good'' solution to this system? In the case where the system is over-determined, we can use least squares to find the solution that minimizes $\norm{Ax^* - b}_2$. In the case where $A$ is under-determined there may be an infinite number of solutions. Assuming that there is a solution, we will search for the solution that minimizes the norm of the solution, or in other words we will minimize $\norm{x^*}_2$ subject to $Ax-b = 0$.

Amazingly, the pseudo-inverse solves both of these problems. We will offer a brief proof. First, for ease of notation, let $P = A A^\dagger$. Note that $PA = A$ and $P = P^*$. We also denote $z = A^\dagger b$. Using these facts we can show that:
\begin{align*}
A^*(Az-b) &= A^*(A A^\dagger b - b) \\
&= A^*P b - A^* b \\
&= (PA)^* b - A^* b \\
&= 0
\end{align*}

Then we can use the parallelogram law to show that:

\begin{align*}
\norm{Ax-b}_2^2 &= \norm{Ax + Az-Az-b}_2^2 \\
&= \norm{Az-b}_2^2 + (A(x-z))^*(Az-b) + \left((A(x-z))^*(Az-b)\right)^* + \norm{A(x-z)}_2^2 \\
&= \norm{Az-b}_2^2 + ((x-z)^*A^*(Az-b) + \left(((x-z)^*A^*(Az-b)\right)^* + \norm{A(x-z)}_2^2 \\
&= \norm{Az-b}_2^2 + \norm{A(x-z)}_2^2 \\
&\geq \norm{Az-b}_2^2
\end{align*}

Thus, $z$ is the least squares solution.
%I thought I saw this proof somewhere else in the book - is it in here twice?
\begin{problem}
Suppose that $A$ is under-determined and that $Ax = b$ has a solution (i.e. is staisfiable). Use the same steps as we used for the least squares solution to prove that $z = A^\dagger b$ is the minimum norm solution. For the first step, show that expression $z^*(x-z) = 0$. For the second step use the parallelogram rule on $\norm{x -z +z}_2^2$.
\end{problem}

{\bf Here a good problem involving finding the minimum-norm solution of something would be perfect. Do you have any suggestions?}

\subsection*{Condition Number}

Suppose that we have the equation:
\[
Ax= b
\]
Now suppose that we have an approximate solution, namely we have $\tilde{x}$ such that $\norm{A\tilde{x} -b}$ is small. Does this imply that $\norm{x-\tilde{x}}$ is small? This is not always the case. Consider, for example, the system:
\[
\begin{pmatrix}
2 & 1\\
4 & 2.1 \\
\end{pmatrix} x =
\begin{pmatrix}
3 \\
6.01\\
\end{pmatrix}
\]

This system has the unique solution $x = [1~1]^T$. However, the vector $\tilde{x} = [1.5~0]^T$ has a very small solution error since $\norm{Ax-A\tilde{x}} = .01$, but the difference betwen $[1~1]$ and $[1.5~0]$ is pretty significant. This is an example of what is called an ill-conditioned matrix. The issue in this case, is that the matrix $A$ maps two vectors that are relatively far apart (namely $x$ and $\tilde{x}$) to vectors that are close together.

This issue becomes very important when we're dealing with floating point arithmetic. It isn't hard to imagine that in finite precision arithmetic it might be reasonable to say that $\tilde{x}$ is a solution to the linear equation above, while it is actually very different from the real solution. We note that in this case rounding errors are not to blame! The matrix itself is to blame.

We quantify this behavior with what is called the condition number of a matrix. The condition number, informally, represents the amount that the solution $x$ of the equation $Ax = b$ changes with respect to $b$. Therefore a large condition number implies that a small error in $b$ (from either measurement error, or floating point arithmetic) can yield a large error in the solution $x$. We can calculate the condition number of a matrix using the following equality:
\[
k(A) = \norm{A} \norm{A^\dagger}
\]
Where $k(A)$ represents the condition number of the of matrix $A$. In this case, the choice of norm isn't important. What we are  doing when we change norms is really change the interpretation of the condition number (i.e. what we are using to measure changes in $b$ and $x$)\footnote{The fact that $A^\dagger$ is not continuous can be problematic for the calculation above, but we are not going to treat that problem here}.

For example, in Python we can calculate the condition number of the above matrix using the following code:

\begin{lstlisting}[style=python]
A = sp.array([[2,1],[4,2.01]])
la.norm(A)*la.norm(la.pinv(A))
\end{lstlisting}

The error that we had in our example was

\[
\frac{\norm{[.5, -1]}}{\norm{[0, .01]}} = 111.8
\]

Which is a whole order of magnitude smaller than the condition number. This implies that the matrix $A$ could give solutions ten times ``worse'' than it did in our example.

Note that Python has a \li{np.linalg.cond} function to calculate the condition number of a matrix. You can verify that the output is the same for the example that we just showed. Also, Volume II offers a more in-depth discussion of conditioning.

\begin{problem}
An $n \times n$ Hilbert Matrix is a matrix with the following entries:
\[
H_{ij} = \frac{1}{i + j -1}
\]

This matrix arises in least-squares polynomial approximation. This matrix is also a canonical example of an ill-conditioned matrix. Write a function that accepts $n$ as an argument and calculates the condition number of the n-th Hilbert Matrix. Plot the growth of the condition number with respect to $n$. What conclusion can you draw about using this type of matrix? %What should be the range of n on our plot?
\end{problem}


\chapter{Relativistic spin-0 particles}
\section{The Klein-Gordon equation}
The Schr{\"o}dinger equation is the quantum mechanical equivalent of the classical $E = p^2/2m$. Now we want a relativistic version. Start from the relationship between energy, momentum, and mass,
\begin{equation}
E^2 - p^2 = m^2.
\end{equation}
Replacing the appropriate values with operators,
\begin{equation*}
E \rightarrow i \pdv{t}, \quad p \rightarrow -i \vec{\nabla}
\end{equation*}
and applying them to some general wavefunction,
\begin{equation}
\left( -\pdv[2]{t} + \nabla^2 \right)\psi = m^2 \psi.
\end{equation}
Rearranging and using equation \eqref{eq:nablaSquared} gives the Klein-Gordon equation
\begin{equation}\boxed{
\left( \partial^2 + m^2 \right)\psi = 0\label{eq:KleinGordon}
}.
\end{equation}
This is the fully relativistic equation of motion for spin-0 particles.

\subsection{4-current density}
We now wish to derive the probability 4-current from the Klein-Gordon equation. Taking the complex conjugate of \eqref{eq:KleinGordon} and multiplying by $\psi$ gives
\begin{equation}
\psi \left( \partial^2 + m^2 \right)\psi^* = 0\label{eq:KG1}.
\end{equation}
Similarly, multiplying \eqref{eq:KleinGordon} by $\psi^*$ gives
\begin{equation}
\psi^* \left( \partial^2 + m^2 \right)\psi = 0\label{eq:KG2}.
\end{equation}
Subtracting \eqref{eq:KG1} from \eqref{eq:KG2} and multiplying by $i$,
\begin{align}
i\psi^*\partial^2\psi - i\psi\partial^2\psi^* = 0 \\
\Rightarrow\quad \pdv{t} \left( i\psi^*\pdv{\psi}{t} - i\pdv{\psi^*}{t}\psi \right) - \vec{\nabla}\cdot\left( i\psi^* \vec{\nabla}\psi - i(\vec{\nabla}\psi)\psi^* \right) = 0
\end{align}
This result is a continuity equation,
\begin{equation}
\partial_\mu j^\mu = 0
\end{equation}
for the 4-current density
\begin{equation}
j^\mu = i\psi^* \partial^\mu \psi - i\psi \partial^\mu \psi^*
\end{equation}

\subsection{Application to a plane wave}
Consider a particle with wavefunction $\phi = \mathcal{N}e^{-iPX}$. Applying the above definition of 4-current density gives the values
\begin{align}
\rho = j^0 = 2\abs{\mathcal{N}}^2 \, E  \\
\vec{j} = 2\abs{\mathcal{N}}^2 \, \vec{p}.
\end{align}
That $\rho$ is proportional to $E$ is to be expected. Under a Lorentz boost the volume element transforms as
\begin{equation*}
\dd[3]{\vec{x}} \rightarrow \frac{\dd[3]{\vec{x}}}{\gamma}
\end{equation*}
so in order to preserve $\rho \dd[3]{\vec{x}}$, $\rho$ must transform as $\rho\rightarrow \gamma\rho$, in the same way energy does.

If the covariant nomalization to $2E$ particles per unit volume is used, the result is that $\rho = 1$.

\section{Negative energy particles and the Fenyman-Stueckelberg interpretation}
For the Klein-Gordon equation to be an accurate description of spin-0 particles, it should be able to describe antiparticles. Indeed, the Klein-Gordon equation does allow for particles with negative energy, i.e.~the negative solution of
\begin{equation}
E_\pm = \pm\sqrt{p^2 + m^2}.
\end{equation}
There are some problems associated with this. Firstly, the existence of negative energy states means that the energy of a particle can always be lowered. Secondly, the $E_-$ solutions are associated with a negative probability density $\rho$, which doesn't make sense and is not allowed.

Pauli and Weisskopf showed that it is possible to have a negative energy solution of the Klein-Gordon equation if the scalar electron charge $-e$ is included in $j^\mu$ and it is interpreted as a charge-current density,
\begin{equation}
j^\mu = -ie\left(\psi^* \partial^\mu \psi - i\psi \partial^\mu \psi^*\right).
\end{equation}
Now $\rho=j^0$ represents the charge density which is allowed to be negative.

Now that the theory has been fixed to allow the $E_-$ particles, we need an interpretation for them. The Feynman-Stueckelberg interpretation states that the negative energy states correspond to antiparticles. Consider the 4-charge-current of a scalar electron with a plane wavefunction,
\begin{equation}
j^\mu (\Pelectron) = -2e\abs{\mathcal{N}}^2\, \mqty(E \\ \vec{p}).
\end{equation}
In the same way, a positron has the current
\begin{align}
j^\mu (\Ppositron) &= 2e\abs{\mathcal{N}}^2\, \mqty(E \\ \vec{p})\\
&= -2e\abs{\mathcal{N}}^2\, \mqty(-E \\ -\vec{p}).
\end{align}
This is the same as for an electron with $-E$, $-\vec{p}$. So far as a system is concerned, the emission of a positron with energy $E$ is the same as the absorption of an electron with energy $-E$. In other words, positive-energy antiparticles travelling forwards in time are negative-energy particles travelling backwards in time. This gives the $E_-$ solutions an interpretation.


\documentclass{homework}
\course{Math 5522H}
\author{Jim Fowler}
\input{preamble}

\begin{document}
\maketitle

\begin{inspiration}
Le plus court chemin entre deux v\'erit\'es dans le domaine r\'eel passe par le domaine complexe.
%The shortest path between two truths in the real domain passes through the complex domain.
\byline{Jacques Hadamard}
\end{inspiration}

\section{Terminology}

\begin{problem}
  Define $\C$.  (How many ``different'' definitions do you know?)
\end{problem}

\begin{problem}
  Define $\conj{z}$ and $\abs{z}$ for $z \in \C$.
\end{problem}

\begin{problem}
  For complex numbers $z, w \in \C$, what do we mean by $z^w$ ?
\end{problem}

\section{Numericals}

\begin{problem}
  Apply \textbf{partial fractions} to write $\displaystyle\frac{1}{1-z^4}$ as a sum of terms of the form $\displaystyle\frac{A}{Bz + C}$.
\end{problem}

\begin{problem}
  Find $a, b, z \in \C$ so that $\left(z^a\right)^b \neq z^{\left(ab\right)}$.
\end{problem}

\begin{problem}
  We will often see \textbf{roots of unity}.  To practice computing with such objects, let
  \[
    \zeta := \cos \left( \frac{2\pi}{7} \right) + i \, \sin \left( \frac{2\pi}{7} \right) \mbox{ and }
    r := \zeta + \zeta^2 - \zeta^3 + \zeta^4 - \zeta^5 - \zeta^6.
  \]
  Find the integer $r^2$.  (This surprise is a \textbf{Gauss sum}.)
\end{problem}

\begin{problem}
  For which $z \in \mathbb{C}$ is it the case that $\log \left( e^z \right) = z$?  \\ (What do we mean when we write $\log$ here?)
\end{problem}

\section{Exploration}

\begin{problem}\label{tj-versus-jt}
  Let's review some linear algebra.  Define $J(x,y) = (y,-x)$ so $J$
  is counter-clockwise rotation by $90^\circ$, and suppose
  $T : \R^2 \to \R^2$ is a linear transformation with the property
  that $T \circ J = J \circ T$.  Can you relate $T$ to the complex
  numbers?
\end{problem}

\begin{problem}\label{mobius-transformations}Here is another connection to linear algebra.  Suppose we have complex-valued functions
  \[
    f(z) = \frac{az + b}{cz + d} \mbox{ and }
    F(z) = \frac{Az + B}{Cz + D}.
  \]
  Such functions are \textbf{M\"obius transformations}.  Relate the
  function $f \circ F$ to a product of certain matrices.
\end{problem}

\begin{problem}\label{abels-theorem}Let's review some real analysis.
  Consider a sequence $(a_n)$ of real numbers so that
  $\sum_{n=0}^\infty a_n$ converges to $L$.  Does the one-sided limit
  \[
    \lim_{x \to 1^{-}} \sum_{n=0}^\infty a_n x^n
  \]
  also equal $L$?  See \textbf{Abel's theorem}.
\end{problem}

\begin{problem}\label{harmonic-function}
  For an open subset $U \subset \R^2$, a \textbf{harmonic function} $f : U \to \R$ is a twice continuously differential function satisfying the Laplace's equation
  \[
    \frac{\partial^2 f}{\partial x^2} + \frac{\partial^2 f}{\partial y^2} = 0.
  \]
  Suppose $f(x,y) = Ax^3 + Bx^2 y + C xy^2 + D y^3$ is harmonic for constants $A, B, C, D \in \R$.  Relate $f$ to $z \cdot (x + iy)^3$.
\end{problem}

\section{Prove or Disprove and Salvage if Possible (PODASIP)}

\textit{You may not have met these PODASIP-style problems before.
  What follows are statements which may be ``true'' or ``false.''  If
  the statement is true, then you should provide a proof.  If the
  statement is false, find a counterexample and fix the statement and
  then supply a proof for your repaired statement.  For example,
  \ref{blaschke-factors} suffers from the domain of $f$ not being
  specified, so you can ``salvage'' \ref{blaschke-factors} by
  carefully describing the domain.}

\begin{problem}\label{blaschke-factors}
  Suppose $w \in \C$ and $\abs{w} < 1$.  Define a function by the rule
  \[
    f(z) = \frac{w - z}{1 - \conj{w}z}.
  \]
  If $\abs{z} < 1$, then $\abs{f(z)} < 1$.  (These are \textbf{Blaschke factors}.)
\end{problem}

\begin{problem} % you may assume R is complete.
  The field $\mathbb{C}$ is complete.
\end{problem}

\begin{problem}
 For all $z, w \in \C$ it is the case that $\sqrt{z} \sqrt{w} = \sqrt{zw}$.
\end{problem}

\begin{problem} % missing non-empty, compact
  Suppose $K_1 \supset K_2 \supset \cdots$ be nested subsets of $\C$ so that $\diam K_n < 1/2^n$.  Then
  \[
    \bigcap_{n=1}^\infty K_n
  \]
  is non-empty and consists of a single point.
\end{problem}

\begin{problem}
 For all $a, b, z \in \C$ it is the case that $z^a \, z^b = z^{a+b}$.
\end{problem}

\begin{problem}\label{cross-ratio}
  Define the \textbf{cross-ratio} of distinct complex numbers $z_1,z_2,z_3,z_4 \in \C$ by
\[
\left(z_1,z_2;z_3,z_4\right):=\frac {\left(z_3-z_1\right)\,\left(z_4-z_2\right)}{\left(z_3-z_2\right)\,\left(z_4-z_1\right)}.
 \]
 If $f$ is a M\"obius transformation (cf.~\ref{mobius-transformations}), then
 \[
   \left(z_1,z_2;z_3,z_4\right) =
   \left(f(z_1),f(z_2);f(z_3),f(z_4)\right).
\]
\end{problem}

\end{document}


\section{Usage in C++}

A good place to see example usages is the gtest files \textit{tests/tst\_dConvolution.cpp} \\
and \textit{tests/tst\_dConvolution\_squares.cpp}.

There are three global static variables representing different sums:
\begin{enumerate}
    \item \(\it{cnvl}\) represents the simple sum, ie $R=X+Y$.
    \item \(\it{cnvlSq}\) represents the sum of the squares, ie $R=X^2+Y^2$.
    \item \(\it{cnvlSSqrt}\) represents the sum of the squares and then take square root, ie $R=\sqrt{X^2+Y^2}$.
\end{enumerate}

\subsection{Simple Sum}

\subsubsection*{Example 1}
The sum of two RVs of standard uniform distributions is a RV of Irwin-Hall distribution.

\begin{minted}{cpp}
disStdUniform su1{}, su2{};
probDistr* rsu = cnvl.go(su1, su2);
disIrwinHall* rih = dynamic_cast<disIrwinHall*>(rsu);
\end{minted}

\subsubsection*{Example 2}
The sum of an array of RVs of standard uniform distributions is a RV of Irwin-Hall distribution.

\begin{minted}{cpp}
disStdUniform su1{}, su2{}, su3{}, su4{};
probDistr* rsu = convolve<disStdUniform>({su1, su2, su3, su4});
disIrwinHall* rih = dynamic_cast<disIrwinHall*>(rsu);
\end{minted}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Sum of the Squares}

\subsubsection*{Example}
The sum of two squares of RVs of normal distributions is a RV of Chi Square distribution.

\begin{minted}{cpp}
disNormal a(0,1);
disNormal b(0,1);
disChiSq s = *static_cast<disChiSq*>(cnvlSq.go(a,b));
\end{minted}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Sum of the Squares, and Then Take Square Root}

\subsubsection*{Example}
The sum of two squares of RVs of normal distribution (with same variance) is a RV of Rician distribution.

\begin{minted}{cpp}
disNormal a(2,9);
disNormal b(3,9);
disRician s = *static_cast<disRician*>(cnvlSSqrt.go(a,b));
\end{minted}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{How to add a new sum of random variables?}

If you do not find your distributions in currenctly supported list, you can easily add a new sum for your random variables.

... is implemented in the double dispatcher pattern. (See \textit{Modern C++ Design by Andrei Alexandrescu} for pattern detail.)

TBW

\begin{enumerate}
    \item Register the pair in \textit{dConvolution.cpp}.
    \item Write a function prototype in \textit{dConvolution.h}.
    \item Implement the sum in \textit{dConvolution.cpp}.
\end{enumerate}

\begin{minted}{cpp}
/* dConvolution.cpp */

auto ConvolutionSqDoubleDispatcherInitialization = [](){
    cnvlSq.add<disNormal,disNormal,convolveSq>();
    cnvlSq.add<disApple,disBananna,convolveSq>();  // new distribution pair.
    return true;
}();

probDistr* convolve(disApple& l, disBananna& r) {
    // Implement the sum
    // Blah, blah, blah
    probDistr* res = xxxxx;
    return res;
};
\end{minted}

\begin{minted}{cpp}
/* dConvolution.h */

probDistr* convolve(disCauchy& lhs, disCauchy& rhs);
\end{minted}


\documentclass[class=report, float=false, crop=false]{standalone}
\usepackage[subpreambles=true]{standalone}

\input{preamble}

\graphicspath{{figures/images/}}

% \begin{cbunit}

\begin{document}

\chapter{Quaternions}
\label{appendix:quaternions}

\section{Definition}

Quaternions were introduced by Hamilton as extension of complex numbers \cite{shoemake}:
\begin{equation}
q = q_0 + \sum_{i=1}^{3}q_i e_i\text{, with } \begin{cases} e_m^2 = -1& \\ e_me_n = \varepsilon_{mnk} e_k&\text{, } m \neq n \end{cases} \text{ and } q_0,q_1,q_2,q_3 \in \mathbb{R}
\end{equation}

For convenience, we will write $q \equiv [\vec{q},q_0]$ with $\vec{q} = (q_1,q_2,q_3) \in \mathbb{R}^3$, and $\mathbb{H}$ the quaternions' space.\\

Moreover, we will identify vectors $\vec{v}$ from $\mathbb{R}^3$ with $[\vec{v},0]$ and scalars $w$ from $\mathbb{R}$ with $[0,w]$. We will assume this identification throughout the document without notification.

\section{Properties}
\label{quat_properties}

\begin{description}
\item[Addition] $\boxed{q + q' = [\vec{q} + \vec{q'},q_0 + q'_0]}$
\item[Multiplication] $\boxed{qq' = [\vec{q}\times\vec{q'} + q_0\vec{q'} + q'_0\vec{q},q_0q'_0 - \vec{q}\cdot\vec{q'}]}$
\begin{align*}
qq' &= (q_0 + \sum_{i=1}^{3}q_i e_i)(q'_0 + \sum_{i=1}^{3}q'_i e_i) = q_0q'_0 + \underbrace{q_0\sum_{i=1}^{3}q'_i e_i}_{q_0\vec{q'}} + \underbrace{q'_0\sum_{i=1}^{3}q_i e_i}_{q'_0\vec{q}} + \underbrace{\sum_{i=1}^{3}q_iq'_i \underbrace{e_ie_i}_{-1}}_{-\vec{q}\cdot\vec{q'}} + \underbrace{\sum_{\substack{i,j \\ i \neq j}}q_iq'_j\underbrace{e_ie_j}_{\varepsilon_{ijk}e_k}}_{\vec{q}\times\vec{q'}}\\
&= [\vec{q}\times\vec{q'} + q_0\vec{q'} + q'_0\vec{q},q_0q'_0 - \vec{q}\cdot\vec{q'}]
\end{align*}
\item[Multiplication by a scalar] $\boxed{\lambda q = [\lambda\vec{q},\lambda q_0]}$
\item[Bilinearity] $\boxed{q''(\lambda q + \lambda q') = \lambda q''q + \lambda q''q'}$
\item[Associative property] $\boxed{(qq')q'' = q(q'q'')}$
\item[Conjugate] $\boxed{q^* = [-\vec{q},q_0]}$
\begin{align*}
q^* &= (q_0 + \sum_{i=1}^{3}q_i e_i)^* = q_0 - \sum_{i=1}^{3}q_i e_i\\
&= [-\vec{q},q_0]
\end{align*}
\item[Conjugation property] $\boxed{(qq')^* = q'^*q^*}$
\begin{align*}
(qq')^* &= ([\vec{q},q_0][\vec{q'},q'_0])^*\\
&= ([\vec{q}\times\vec{q'} + q_0\vec{q'} + q'_0\vec{q},q_0q'_0 - \vec{q}\cdot\vec{q'}])^*\\
&= [\underbrace{-\vec{q}\times\vec{q'}}_{(-1)\vec{q'}\times(-1)\vec{q}} + q_0(-\vec{q'}) + q'_0(-\vec{q}),q_0q_0' - (-1)\vec{q}\cdot(-1)\vec{q'}]\\
&= [-\vec{q'},q'_0][-\vec{q},q_0]\\
&= q'^*q^*
\end{align*}
\item[Norm] $\boxed{N^2(q) = \sum_{i=0}^{3} q_i^2}$
\begin{align*}
N^2(q) &= qq* = [\vec{q},q_0][-\vec{q},q_0] = [\cancel{-q_0 \vec{q}} + \cancel{q_0\vec{q}},q_0q_0 + \vec{q}\cdot\vec{q}]\\
&= [0,\sum_{i=0}^{3} q_i^2] = \sum_{i=0}^{3} q_i^2
\end{align*}
\item[Inverse] $\boxed{q^{-1} = q^*/N^2(q)}$
\begin{align*}
q\frac{q*}{N^2(q)} = \frac{q*}{N^2(q)}q = 1 \Leftrightarrow q^{-1} = \frac{q*}{N^2(q)}
\end{align*}
\item[Action of an unit quaternion] From the property of the inverse of a quaternion, it is trivial that the action of any quaternion $q$
\begin{align*}
\mathcal{A}_q \colon &\mathbb{H} \to \mathbb{H}\\     &\phantomarrow{\mathbb{H}}{p} qpq^{-1}
\end{align*}
will be the same as the action of the associated unit quaternion $q/N(q)$. Therefore, we will only consider unit quaternion from now on and use $q^*$ instead of $q^{-1}$.\\

We then have that for any $p = [\vec{v},w]$, $\boxed{qpq^* = [\vec{v'},w]}$ with $N(\vec{v}) = N(\vec{v'})$
\begin{align*}
&\text{Indeed, with $S(q) \equiv \frac{q+q^*}{2}$ the scalar part of any quaternion $q$ we have}\\
%\vspace{-30pt}
&2S(qpq^*)\\
&= qpq^* + (qpq^*)^* = qpq^* + qp*q^*\\
&= q\underbrace{(p + p^*)}_{2w}q^* = 2w\cancel{qq^*} = 2w\\
&= 2S(p)\\
&\text{The conjugation property ensures that the multiplication conserves the}\\
&\text{norm, then $N(qpq^*)=N(p)$, hence the property.}
\end{align*}
Therefore the action of any quaternion on any vector of $\mathbb{R}^3$ is a vector of $\mathbb{R}^3$.
\end{description}

\section{Actions and rotations}
\label{action_rotation}

Consider $\vec{v_0},\vec{v_1} \in \mathbb{R}^3$ with $\vec{v_0} \times \vec{v_1} \neq 0$ and $N(\vec{v_0}) = N(\vec{v_1}) = 1$. We define $q \equiv \vec{v_1}\vec{v_0}^* = [\vec{v_0}\times\vec{v_1},\vec{v_0}\cdot\vec{v_1}]$, $\theta \equiv (\vec{v_0};\vec{v_1})$ and $\vec{v} = \frac{\vec{v_0} \times \vec{v_1}}{||\vec{v_0} \times \vec{v_1}||}$ so that $q = [\vec{v}\sin\theta,\cos\theta]$. We want to know what is the action of $q$, $\mathcal{A}_q$, on any vector of $\mathbb{R}^3$.\\

We have $(\vec{v_0},\vec{v_1},\vec{v})$  a basis of $\mathbb{R}^3$. Thanks to the bilinearity of the product of quaternions, we can infer $\mathcal{A}_q(\vec{u}) \forall \vec{u} \in \mathbb{R}^3$ from the action of $q$ on this basis. The following demonstration was originally made in \cite{shoemake}.\\

\begin{description}
\item[Action on $\vec{v}$\hspace{20pt}]
\begin{align*}
\mathcal{A}_q(\vec{v}) &= q\vec{v}q^*\\
&= [\vec{v}\sin\theta,\cos\theta][\vec{v},0][-\vec{v}\sin\theta,\cos\theta]\\
&= [\vec{v}\cos\theta,-\underbrace{|\vec{v}|^2}_{=1}\sin\theta][-\vec{v}\sin\theta,\cos\theta]\\
&= [\underbrace{\vec{v}\sin^2\theta + \vec{v}\cos^2\theta}_{=\vec{v}},\cancel{-\sin\theta\cos\theta} + \underbrace{|\vec{v}|^2}_{=1}\cancel{\sin\theta\cos\theta}]\\
&= [\vec{v},0]\\
&= \vec{v}
\end{align*}
\item[Action on $\vec{v_0}$\hspace{20pt}] We define $\vec{v_2} \equiv \mathcal{A}_q(\vec{v_0}) = q\vec{v_0}q^*$ and notice that
\begin{align*}
\vec{v_2}\vec{v_1}^* &= (q\vec{v_0}q^*)\vec{v_1}^*\\
&= (q\vec{v_0}(\vec{v_1}\vec{v_0}^*)^*)\vec{v_1}^*\\
&= q\vec{v_0}(\vec{v_0}\vec{v_1}^*)\vec{v_1}^*\\
&=q\underbrace{(\vec{v_0}\vec{v_0})}_{=-1}\underbrace{(\vec{v_1}^*\vec{v_1}^*)}_{=-1}\\
&=q=\vec{v_1}\vec{v_0}^*
\end{align*}
so \(\begin{cases} \vec{v_1}\times\vec{v_2} &= \vec{v_0}\times\vec{v_1} \\ \vec{v_1}\cdot\vec{v_2} &= \vec{v_0} \cdot \vec{v_1} \end{cases} \). Therefore, $\vec{v_0}, \vec{v_1}$ and $\vec{v_2}$ belong to the same plane and furthermore, $(\vec{v_1};\vec{v_2}) = (\vec{v_0};\vec{v_1}) = \theta$. Then, $\vec{v_2}$ is the rotation of $\vec{v_0}$ of an angle $2\theta$ around $\vec{v}$.\\
\item[Action on $\vec{v_1}$\hspace{20pt}] We define $\vec{v_3} \equiv \mathcal{A}_q(\vec{v_1}) = q\vec{v_1}q^*$ and notice that
\begin{align*}
\vec{v_3}\vec{v_2}^* &= (q\vec{v_1}q^*)(q\vec{v_0}q^*)^*\\
&= (q(q\vec{v_0})q^*)(q\vec{v_0}q^*)^*\\
&=  q(q\vec{v_0}q^*)(q\vec{v_0}q^*)^*\\
&=q=\vec{v_1}\vec{v_0}^*
\end{align*}
so \(\begin{cases} \vec{v_2}\times\vec{v_3} &= \vec{v_0}\times\vec{v_1} \\ \vec{v_3}\cdot\vec{v_3} &= \vec{v_0} \cdot \vec{v_1} \end{cases} \). Therefore, $\vec{v_0}, \vec{v_1}, \vec{v_2}$ and $\vec{v_3}$ belong to the same plane and furthermore, $(\vec{v_2};\vec{v_3}) = (\vec{v_0};\vec{v_1}) = \theta$. Then, $\vec{v_3}$ is the rotation of $\vec{v_1}$ of an angle $2\theta$ around $\vec{v}$.\\
\end{description}

Finally, we have that $\mathcal{A}_q$ corresponds to the rotation of angle $2\theta$ around $\vec{v}$. The corollary to that property is that

\begin{center}$\boxed{\text{\textbf{every 3D rotation can be expressed as the action of some unit quaternion}}}$.\end{center}

% \input{references/biblio}

\end{document}

% \end{cbunit}


\section[Linear recurrences]{Linear recurrences - The Fibonacci sequence}\label{fibonacci}

We are going to discuss the Fibonacci sequence in the following section.

\subsection{Fibonacci's rabbit problem}

Suppose we have a pair of rabbits that are adults after one month. In the month after they are fully grown,
they give birth to two more rabbits. The newborns are again fully grown after one month and give birth to two more
rabbits, and so on. Consider the following figure as an example.
\\
\\
$
\begin{array}{l}
    \includegraphics[scale=0.1]{pictures/rabbits.jpg}
\end{array}
n=1$\\
$
\begin{array}{l}
    \includegraphics[scale=0.1]{pictures/rabbits.jpg}
\end{array}
n=2 \text{ (grown up)}$
\\
$
\begin{array}{l}
    \includegraphics[scale=0.1]{pictures/rabbits.jpg}
    \includegraphics[scale=0.1]{pictures/rabbits.jpg}
\end{array}
n=3$
\\
$
\begin{array}{l}
    \includegraphics[scale=0.1]{pictures/rabbits.jpg}
    \includegraphics[scale=0.1]{pictures/rabbits.jpg}
    \includegraphics[scale=0.1]{pictures/rabbits.jpg}
\end{array}
n=4$
\\
\\
\noindent
\textbf{Question:} How many pairs of rabbits will we have after one year?
\\
\\
\noindent
Let us introduce the following notation. Denote the number of pairs of rabbits during
the $n$-th month by $F(n)$. Then, we could express $F(n)$ by the number of rabbits in
the $(n-1)$-th month and $(n-2)$-th month by the following equation.
$$
F(n)=F(n-1)+F(n-2)
$$
Thus, by setting $F(2)=1$ the following sequence of numbers follows.
$$
1,1,2,3,5,8,13,21,36,55,89,144, \cdots
$$
\noindent
The answer follows for $F(12)=144$, hence after $12$ months we have $144$ pairs of rabbits.
\\
\\
The above sequence is called the "Fibonacci sequence" and we are going to investigate some
of its fascinating properties in the next sections.

\newpage

\subsection[Pascal triangle]{Fibonacci numbers and the Pascal triangle}

We consider once again the Pascal triangle. Let us sum up its diagonals. This is done in the following figure for
the first few diagonals.
\\
\\
\begin{tabular}{>{$n=}l<{$\hspace{12pt}}*{13}{c}}
    0 &&&&&&&1&&&&&&\\
    1 &&&&&&1&&\color{red}1\textsuperscript{\color{red}=2}&&&&&\\
    2 &&&&&\color{red}1&&\color{yellow}2\textsuperscript{\color{yellow}=3}&&1&&&&\\
    3 &&&&\color{yellow}1&&\color{green}3\textsuperscript{\color{green}=4}&&\color{blue}3\textsuperscript{\color{blue}=8}&&\color{cyan}1\textsuperscript{\color{cyan}=13}&&&\\
    4 &&&\color{green}1&&\color{blue}4&&\color{cyan}6&&4&&1&&\\
    5 &&\color{blue}1&&\color{cyan}5&&10&&10&&5&&1&\\
    6 &\color{cyan}1&&6&&15&&20&&15&&6&&1
\end{tabular}
\\
\\
\noindent
The following sequence is the result.
$$
1,1,
\color{red} 1 \color{black}+ \color{red}1\color{black}=\color{red}2 \color{black}, \color{yellow} 1 \color{black}+ \color{yellow}2\color{black}=\color{yellow}3 \color{black},
\color{green} 1 \color{black}+ \color{green}3\color{black}=\color{green}4 \color{black},
\color{blue} 1 \color{black}+ \color{blue}4\color{black}+ \color{blue}3=\color{blue}8 \color{black},
\color{cyan} 1 \color{black}+ \color{cyan}5\color{black}+ \color{cyan}6\color{black}+ \color{cyan}1=\color{cyan}13 \color{black}, \cdots
$$
The general formula for the sum of the diagonals in the Pascal triangle is easily derived. It is as follows.
$$
\sum_{k=0}^{\floor{\frac{n-1}{2}}} {n-k-1 \choose k}
$$

\subsection{Domino tilings}

\subsection{Vending machine problem}

Suppose that we only have nickels ($5$), dimes ($10$) and quarters ($25$). In how many ways can we pay $\$1$ (or any other amount)?
Notice first that, unlike in the integer composition problem, the sums $5 + 10 + 10 + 25 + 25 + 25$ and $10 + 25 + 10 + 5 + 25 + 25$ are not considered identical.
Let us denote the number of ways to pay $5n$ cents by $A(n)$ and set $A(0)=1$. Let us make a table with the $n$ ranging to $6$.
\bigskip
\begin{center}
    \begin{tabular}{ |c|c|c|c|c|c|c|c| }
     \hline
     $n$ & $0$ & $1$ & $2$ & $3$ & $4$ & $5$ & $6$ \\
     $A(n)$ & $1$ & $1$ & $2$ & $3$ & $5$ & $9$ & $15$ \\
     \hline
    \end{tabular}
\end{center}
\bigskip
\noindent
Let us examine $n=5$, the number of ways to pay $25$ cents, in more detail. There is only one way to pay with a quarter. There are three ways
to pay $25$ cents if we start with a dime, namely $10 + 10 + 5, 10 + 5 + 10, 5 + 10 + 10$, and five ways to pay starting with a nickel.
It is not hard to see that this yields the following recurrence relation.
$$
A(n) = A(n-1)+A(n-2)+A(n-5)
$$

\subsection{Linear recurrence relations}

In the following section, we are going to introduce the formal definition of a linear recurrence relation.

\begin{defn}
A linear recurrence relation of order $k$ is a sequence satisfying
$$
A(n)=c_1A(n-1)+c_2A(n-2)+\cdots+c_kA(n-k)
$$
where $c_1,\cdots, c_k \in \mathbb{R}$.
\label{defn:rec}
\end{defn}

\begin{exmp}
The Fibonacci sequence $F(n)=F(n-1)+F(n-2)$ is a linear recurrence relation of order $2$. The sequence
$A(n)=A(n-1)+A(n-2)+A(n-5)$ from the last section is a linear recurrence relation of order $5$.
\end{exmp}

\begin{remark}
A sequence defined by a linear recurrence relation is uniquely determined by the first $k$ terms $A(0), A(1), \cdots, A(k-1)$.
Why is that the case? Suppose that $a_0, a_1, a_2, \cdots$ and $a_0', a_1', a_2', \cdots$ satisfy the relation $(*)$,
then $a_0+a_0', a_1+a_1',a_2+a_2', \cdots$ also satisfy $(*)$ and so does $c \cdots a_0, c \cdot a_1, c \cdot a_2, \cdots$.
In other words, the solutions of a linear recurrence relation form a vector space.
\end{remark}

\subsection[The characteristic equation]{The characteristic equation}

In the following section we want to solve the following problem.\\
\textbf{Problem:} Given a recurrence relation
$$
(*)\ A(n)=c_1A(n-1)+c_2A(n-2)+\cdots+c_kA(n-k)
$$
as in definition \ref{defn:rec}. Find \underline{all} the sequences $(a_0, a_1, a_2, \cdots)$ satisfying this relation.
\\
For $k=1$ the linear recurrence relation yields $A(n)=c \cdot A(n-1)$, and the answer for our problem is just the geometric progression
given by $a_n=c^n a_0$ where $a_0 \in \mathbb{R}$.
\\
\\
\noindent
\textbf{Idea:} Find a geometric progression $(a_0, \lambda a_0, \lambda^2 a_0, \cdots)$ satisfying $(*)$. Suppose $\lambda \neq 0$ and $a_0 \neq 0$.
Then,
$$
a_0 \lambda^n=c_1 a_0 \lambda^{n-1}+c_2 a_0 \lambda^{n-2} + \cdots + c_k a_0 \lambda^{n-k}, \forall n \geq k.
$$
Dividing the above equation by $a_0 \lambda^{n-k}$ yields
$$
\lambda^k=c_1 \lambda^{k-1} + c_2 \lambda^{k-2} + \cdots + c_k.
$$
Then $\lambda$ is a root of the so called \underline{characteristic equation} which is given by
$$
t^k=c_1t^{k-1}+c_2 t^{k-2} + \cdots + c_k.
$$
The coefficients of the characteristic equation are the same as in the original recurrence relation.

\subsection{Linear recurrence relations of order $2$}

Let $A(n)=c_1 A(n-1) + c_2 A(n-2)\ (*)$ and $t^2=c_1t +c_2\ (**)$. Suppose that $\lambda$ and $\mu$ are roots of $(**)$ where $\lambda \neq \mu$.
Then,
\begin{align*}
1, \lambda, \lambda^2, \lambda^3, \cdots \\
1, \mu, \mu^2, \mu^3, \cdots
\end{align*}
are solutions of equation $(*)$. Since the solutions of a recurrence relation form a vector space
$$
a_n=C \cdot \lambda^n + D \cdot \mu^n
$$
is also a solution (the linear combinations of the above solutions).\\\\
\textbf{Question:} Are there any other solutions?
\\
\\
Well, the answer is "No", because for any solution $(a_0, a_1, \cdots)$ of $(*)$, there exist $C$
 and $D$ such that $a_n=C \dot \lambda^n + D \cdot \mu^n$ which is called the \textit{general solution} of $(*)$. But why is that so?
Since $(*)$ is a linear recurrence relation of order $k=2$, $(a_0, a_1, a_2, \cdots, a_n, \cdots)$ is uniquely determined by its first two terms $a_0$ and $a_1$.
We need to show that for any $a_0$ and $a_1$ there exist a $C$ and $D$ such that
\begin{align*}
&\begin{cases}
    a_0 = C + D\\
    a_1 = C \dot \lambda + D \cdot \mu
\end{cases}\\
&\begin{cases}
    D = a_0 - C\\
    a_n = C \cdot \lambda + (a_0 - C) \cdot \mu
\end{cases}\\
&
C=\frac{a_1 - a_0 \mu}{\lambda - \mu},\ D=\frac{a_0 \lambda - \mu}{\lambda - \mu}
\end{align*}
\noindent
Therefore, $C$ and $D$ are uniquely determined, hence there are no other solutions then the above.

\subsection{The Binet formula}
In the following section we are going to go over two examples of the characteristic equation.
\begin{exmp}
Let $A(n)=5 A(n-1)-6A(n-2)$. Find a solution for $a_0=3$ and $a_1=7$. First, define the characteristic which is quickly found.
\begin{align*}
t^2=5t-6\\
t^2-5t+6=0
\end{align*}
The roots are $\lambda=2$ and $\mu=3$, thus the general solution of our characteristic equation is given by
$$
a_n=C \cdot 2^n + D \cdot 3^n.
$$
Solving the equation for $C$ and $D$ yields the following system of linear equations.
\begin{align*}
    &\begin{cases}
        a_0 = 3 = C + D\\
        a_1 = 7 = 2C + 3D
    \end{cases}\\
    &\begin{cases}
        C=2\\
        D=1
    \end{cases}
\end{align*}
Therefore,
$$
a_n=\underbrace{2^{n+1}}_{2 \cdot 2^n} + \underbrace{3^n}_{1 \cdot 3^n}.
$$
\end{exmp}

\begin{exmp}
The Fibonacci sequence is given by the linear recurrence relation $F(n)=F(n-1)+F(n-2)$ with $f_0=0$ and $f_1=1$. Thus,
the characteristic equation is given by
\begin{align*}
t^2=t+1\\
t^2-t-1=0
\end{align*}
and the roots, call them $\varphi$ and $\psi$, follow
\begin{align*}
\varphi &= \frac{1 + \sqrt(5)}{2}\\
\psi &= \frac{1 - \sqrt(5)}{2}
\end{align*}
And as before
\begin{align*}
    &\begin{cases}
        0 = C + D\\
        1 = C \cdot \varphi + D \cdot \psi
    \end{cases}\\
    &\begin{cases}
        D = -C\\
        1 = C (\varphi - \psi)
    \end{cases}\\
    &\begin{cases}
        C = \frac{1}{\varphi - \psi}\\
        D = \frac{1}{\psi - \varphi}
    \end{cases}\\
    &\begin{cases}
        C = \frac{1}{\sqrt{5}}\\
        D = -\frac{1}{\sqrt{5}}
    \end{cases}\\
\end{align*}
The solution follows.
$$
f_n= \frac{\varphi^n - \psi^n}{\varphi - \psi} = \frac{1}{\sqrt{5}}\biggr (\biggr(\frac{1 + \sqrt{5}}{2}\biggl )^n - \biggr( \frac{1- \sqrt{5}}{2} \biggr )^n \biggr )
$$
This formula is called "The Binet Formula".
\end{exmp}

\subsection{Linear recurrence relations of arbitrary order}

It is no surprise that we can generalize the results of the last sections to linear recurrence relations of an arbitrary order $k$.
Given a linear recurrence relation
$$
A(n)=c_1A(n-1)+c_2A(n-2)+\cdots+c_kA(n-k)\ (*)
$$
then $1, \lambda, \lambda^2, \cdots, \lambda^n, \cdots$ is a solution of $(*)$ where $\lambda$ is a root of the characteristic equation
$$
t^k=c_1 t^{k-1} + \cdots + c_k.
$$
Suppose it has $k$ distinct roots $\lambda_1, \cdots, \lambda_k$, then
$$
a_n=c_1 \lambda_1^n + c_2 \lambda_2^n + \cdots + c_k \lambda_k^n
$$
is the general solution. Suppose that we are given $a_0, \cdots, a_{k-1}$. We want a solution with the following initial solutions.
$$
\begin{cases}
    a_0 = c_1 + c_2 + \cdots + c_k\\
    a_1 = c_1 \lambda_1 + c_2 \lambda_2 + \cdots + c_k \lambda_k\\
    \vdots \text{ } \cdots\\
    a_{k-1}= c_1 \lambda_1^{k-1} + c_2 \lambda_2^{k-2} + \cdots + c_k \lambda^{k-1}
\end{cases}
$$
The above system of linear equations has a unique solution if and only if its determinant is equal to
$$
\det\begin{pmatrix} 1 & 1 & \cdots & 1
    \\ \lambda_1 & \lambda_2 & \cdots & \lambda_k \\
    \lambda_1^2 & \lambda_2^2 & \cdots & \lambda_k^2 \\
    \vdots & \vdots & \vdots & \vdots \\
    \lambda_1^{k-1} & \cdots & \cdots & \lambda_k^{k-1}
\end{pmatrix}
= \prod_{k \geq 1 > j \geq 1} (\lambda_i - \lambda_j)
$$
The above is known as the "Vandermont determinant" and since all $\lambda_i$ and $\lambda_j$ are distinct, the above has an unique solution.

\subsection{The case of roots with multiplicities}

What happens if some roots of the characteristic equation occur with multiplicities? Given $A(n)=c_1 A(n-1)+ c_2 A(n-2)$ yields $t^2-c_1t -c_2=0$.
Suppose that $\lambda$ is a unique (double) root of this equation, then
$$
\begin{cases}
    -c_2=\lambda^2\\
    c_1 = 2 \lambda
\end{cases}
$$
since $(t-\lambda)^2=t^2 -c_1t-c_2$ and $1, \lambda, \lambda^2, \cdots, \lambda^n$ is a solution.

\begin{claim}
$0, \lambda, 2 \lambda^2, 3 \lambda^3, \cdots, n \lambda^n$ is another solution.
\end{claim}
\begin{proof}
The above yields to
$$
n \lambda^n = c_1 (n-1)\lambda^{n-1} + c_2 (n-2)\lambda^{n-2}.
$$
The solution $-c_2=\lambda^2$ and $c_1=2 \lambda$ yields in the following.
$$
n \lambda^n = 2 \lambda(n-1)\lambda^{n-1} + (-\lambda^2)(n-2)\lambda^{n-2}
$$
Both sides are divisible by $\lambda^n$, thus $n=2(n-1)-(n-2)$ is a solution which is linearly independent of the previous one.
\end{proof}

In general, if $\lambda$ is a root of the characteristic polynomial of order $r$, then the characteristic polynomial is divisible by $(t- \lambda)^r$.
Hence, the following are all solutions of the above.
\begin{align*}
1, \lambda, \lambda^2, \cdots, \lambda^n, \cdots \\
0, \lambda, 2 \lambda^2, \cdots, n \lambda^n, \cdots \\
0, \lambda, 4 \lambda^4, 9 \lambda^3, \cdots, n^2 \lambda^n, \cdots \\
0, \lambda, 2^{r-1} \lambda^2, \cdots, n^{r-1}\lambda^n, \cdots
\end{align*}

\subsection{Exercises}

\begin{enumerate}
\item Which of the following is a Fibonacci sequence?
\begin{enumerate}
\item The number of partitions, i.e. the presentations of a natural number as a sum of positive non-increasing summands (i.e., $3=2+1=1+1+1$)
\item The number of sequences of $0$'s and $1$'1 with $n$ digits that contain no two consecutive zeroes
\item The number of subsets of $\{1,2, \cdots, n\}$ that contain no consecutive integers
\item The number of partitions of a rectangle $2 \times n$ into rectangles $2 \times 1$
\item The number of compositions of a natural number into positive odd summands (i.e., $4=1+1+1+1=1+3=3+1$)
\end{enumerate}
\item The Fibonacci sequence can be continues "backwards" using the same rule: $F_n=F_{n-1}+F_{n-2}$. For example, $F_0=0, F_{-1}=1$. Find $F_{-10}$.
\item Find the maximal common ration of a geometric progression $a_n$ satisfying the following equation: $a_{n+2}=3a_{n+1}-2a_n$.
\item The sequence $a_n$ is defined by the recurrence relation $a_{n+3}=3a_{n+2}-3a_{n+1}+a_n$ with initial values $a_0=a_1=0, a_2=1$. Find $a_{100}$.
\item The sequence $a_n$ is defined by the recurrence relation $a_{n+4}=a_{n+3}-a_{n+2}+a_{n+1}-a_n$ with initial values $a_0=1607, a_1=1707, a_2=1814, a_3=1914$. Find $a_{100}$.
\end{enumerate}


\section{Support vector machine}\label{sec:SVMintro}% and the relation with Binary Logistic Regression}
There is a lot of work about SVM in literature , see \cite{drucker1997support,ben2001support,cortes1995support,cristianini2000introduction} for example.
Given a binary linearly separable classification dataset ${(x_i,y_i)}_{i = 1}^N$, where $x_i\in \mathbb{R}^d, y_i\in \left \{\begin{pmatrix}1\\0\end{pmatrix}, \begin{pmatrix}0\\1\end{pmatrix}\right \}$. We use $A_1,A_2$ to denote the data with label $\begin{pmatrix}1\\0\end{pmatrix}, \begin{pmatrix}0\\1\end{pmatrix}$, respectively. Our goal is to find a $\theta = (w,b)$ where $w\in \mathbb{R}^{1\times d}, b\in \mathbb{R}$ such that the hyperplane $H_{\theta} = \{x:wx + b = 0\}$ can separate $A_1,A_2$.

\subsection{Binary SVM}\label{sec:SVM}
Binary Support Vector Machine (SVM for short hereinafter) wants to find the classifiable hyperplane which has the biggest distance with $A_1$ and $A_2$. Assume that we have the hyperplanes $wx+b=\pm 1$ with
$$
wx_i+b\ge 1 \quad \mbox{for}\quad x_i\in A_1,\quad wx_i+b\le -1 \quad \mbox{for}\quad x_i\in A_2,
$$
which is similar to the definition \eqref{2classH}.
Let $y_1=\begin{pmatrix}1\\0\end{pmatrix}$ for $x_i\in A_1$ and $y_2=\begin{pmatrix}0\\1\end{pmatrix}$ for $x_i\in A_2$. %Then, we have the following inequality
%$$
%y_i(wx_i+b)\ge 1 \quad \mbox{for all }\quad i.
%$$
Note that $w$ is normal to the hyperplane and the distance  between the points satisfying
$$
wx_i+b=\pm 1
$$
and the hyperplane $wx+b=0$ is $\displaystyle {1\over \|w\|_2}$. Thus, the width of the margin is $\displaystyle {2\over \|w\|_2}$ as shown in Figure \ref{fig:margin}.
\begin{figure}
	\centering
	\includegraphics[width = 2in]{margin}
	\caption{SVM}
	\label{fig:margin}
\end{figure}
For any $w$ and $b$, the smallest distance between points and the hyperplane is
\begin{equation}
\frac{\min_{i} \ell_i(wx_i+b)}{\|w\|_2},
\end{equation}
where $\ell_i=1-2e_2^Ty_i$.
Note that
$$
\ell_i=\begin{cases}
1 & \mbox{ if } y_i=\begin{pmatrix}1\\0\end{pmatrix},
\\
-1 &\mbox{ if } y_i=\begin{pmatrix}0\\1\end{pmatrix}.
\end{cases}
$$
Consider the problem
\begin{equation}
	\max_{w,b} \frac{\min_{i} \ell_i(wx_i+b)}{\|w\|_2}.
\end{equation}
Intuitively, the best separating hyperplane $H$ is only determined by those data points who are closest to $H$. Those data points are called {support vector}, and this method are called {support vector machine}.

Without loss of generality, we may restrict the norm of $\|w\|$ to be 1, which leads to a equivalent optimization problem
\begin{equation}
\max_{\|w\|_2 = 1} \min_{i} \ell_i(wx_i+b)
\end{equation}
Actually, {we can prove $\displaystyle \mathop{\rm argmax}_{\|w\|_2 = 1} \min_{i} \ell_i(wx_i+b)$ is nonempty, but here we just admit this fact and only prove the uniqueness of the solution.}

\begin{lemma}
	If $A_1,A_2$ are linearly separable, then
	\begin{equation}\label{binarySVM}
	\mathop{\rm argmax}_{\|w\|_2 = 1} \min_{i} \ell_i(wx_i+b)
	\end{equation}
	is nonempty.
\end{lemma}

\begin{proof}
	Take $x_{i_1} \in A_1$ and  $x_{i_2} \in A_2$,  given $(w,b)\in \{(w,b): l_i(wx_i +b)>0, \forall i\}$, we have
	\begin{equation}
		\begin{cases}
		wx_{i_1} + b > 0,\\
		wx_{i_2} + b < 0
		\end{cases}
	\end{equation}
	which implies $|b| < \max_{i} \|x_i\|_2$. So we have
	\begin{equation}
		\mathop{\rm argmax}_{\|w\|_2 = 1} \min_{i} \ell_i(wx_i+b) = \mathop{\rm argmax}_{\|w\|_2 = 1, ~|b|\leq \max_{i} \|x_i\|_2} \min_{i} \ell_i(wx_i+b) \neq \emptyset.
	\end{equation}
\end{proof}

\begin{lemma}
	If $A_1,A_2$ are linearly separable, then
	\begin{equation}
		\mathop{\rm argmax}_{\|w\|_2 = 1} \min_{i} \ell_i(wx_i+b)
	\end{equation}
	 is a singleton set.
\end{lemma}

\begin{proof}
	Denote $\displaystyle m(w,b) = \min_{i} \ell_i(wx_i+b)$. Notice that $m(w,b)$ is a concave homogeneous function w.r.t $w,b$ and $\|\cdot\|_2$ is a strictly convex norm. Suppose there are two solution $(w_1,b_1)$ and $(w_2,b_2)$ such that $w_1 \neq w_2$, take $\overline{w} = \frac{w_1 + w_2}{2}, \overline{b} = \frac{b_1 + b_2}{2}$, we must have
	\begin{equation}
		m(\overline{w},\overline{b}) \geq \frac{m(w_1,b_1)+ m(w_2,b_2)}{2} = \max_{\|w\|_2 = 1} m(w,b),
	\end{equation}
	and
	\begin{equation}
		\|\overline{w}\|_2 < 1.
	\end{equation}
	So
	\begin{equation}
		m(\frac{\overline{w}}{\|\overline{w}\|_2},\frac{\overline{b}}{\|\overline{w}\|_2}) = \frac{m(\overline{w},\overline{b}) }{\|\overline{w}\|_2} > \max_{\|w\|_2 = 1} m(w,b),
	\end{equation}
	which leads to a contradiction. So all the solutions must have the same $w$, we denote it as $w^*$. Then if $(w^*,b^*)$ is a solution of problem (\ref{binarySVM}), we must have
	\begin{equation}
		b^* \in \mathop{\rm argmax}_{b} m(w^*,b)
	\end{equation}
	Actually,
	\begin{equation}
		m(w^*,b) = \min\{b+\min_{x\in A_1} w^*x, -b +\min_{x\in A_2} (-w^*x)\}.
	\end{equation}
	It is easy to observe that $\displaystyle \mathop{\rm argmax}_{b} m(w^*,b)$ is a singleton set and
	\begin{equation}
		b^* = \frac{\min_{x\in A_2} (-w^*x) - \min_{x\in A_1} w^*x}{2}.
	\end{equation}
\end{proof}

Denote
\begin{equation}\label{maxSVM}
	\theta^*_{SVM} = (w_{SVM}^*,b_{SVM}^*) = \mathop{\rm argmax}_{\|w\|_2 = 1} \min_{i} \ell_i(wx_i+b).
\end{equation}

\begin{theorem}[Representation Theorem]
	Let $\theta^*_{SVM} = (w_{SVM}^*,b_{SVM}^*) $ be the solution of \eqref{maxSVM}. Then, $w_{SVM}^*$ must be a linear combination of $x_i^T, i = 1,2,\cdots,N$.
\end{theorem}

\begin{proof}
	Denote
	\begin{equation}
		S = {\rm span} \{x_i^T\}_{i=1}^N.
	\end{equation}
	Then we have
	\begin{equation}
		\mathbb{R}^{1\times d} = S \oplus^{\perp} S^{\perp}.
	\end{equation}
	So $w_{SVM}^*$ can be uniquely decomposed as $w_{SVM}^* = w^*_S + w^*_{S^{\perp}}$ where $w_S\in S$ and $w^*_{S^{\perp}}\in S^{\perp}$.
	We will prove that $w^*_{S^{\perp}} = 0$. Suppose not, we have
	\begin{equation}
		\|w^*_S\|_2 < \|w_{SVM}^*\|_2 = 1.
	\end{equation}
	Notice that
	\begin{equation}
		w_{SVM}^* x_i = w_S^* x_i,\ \forall i = 1,2,\cdots,N.
	\end{equation}
	Thus we have
	\begin{equation}
		\min_{i} \ell_i(w_{SVM}^*x_i+b_{SVM}^*) = \min_{i} \ell_i(w_S^*x_i+b_{SVM}^*).
	\end{equation}
	So
	\begin{equation}
	\min_{i} \ell_i(w_{SVM}^*x_i+b_{SVM}^*) < \frac{\min_{i} \ell_i(w_S^*x_i+b_{SVM}^*)}{\|w_S^*\|} = \min_{i} \ell_i(\frac{w^*_S}{\|w_S^*\|_2}x_i+\frac{b_{SVM}^*}{\|w^*_S}\|_2),
	\end{equation}
	which leads to a contradiction to the definition of $\theta_{SVM}^*$.
\end{proof}

%For any $w$ and $b$, the smallest distance between points and the hyperplane is
%\begin{equation}
%\frac{\min_{i} y_i(wx_i+b)}{\|w\|_2}.
%\end{equation} Consider the problem
%\begin{equation}
%	\max_{w,b} \frac{\min_{i} y_i(wx_i+b)}{\|w\|_2}.
%\end{equation}
%Intuitively, the best separating hyperplane $H$ is only determined by those data points who are closest to $H$. Those data points are called {support vector}, and this method are called {support vector machine}.
%
%Without loss of generality, we may restrict the norm of $\|w\|$ to be 1, which leads to a equivalent optimization problem
%\begin{equation}
%\max_{\|w\|_2 = 1} \min_{i} y_i(wx_i+b)
%\end{equation}
%Actually, {we can prove $\displaystyle \mathop{\rm argmax}_{\|w\|_2 = 1} \min_{i} y_i(wx_i+b)$ is nonempty, but here we just admit this fact and only prove the uniqueness of the solution.}
%
%\begin{lemma}
%	If $A_1,A_2$ are linearly separable, then
%	\begin{equation}\label{binarySVM}
%		\mathop{\rm argmax}_{\|w\|_2 = 1} \min_{i} y_i(wx_i+b)
%	\end{equation}
%	 is a singleton set.
%\end{lemma}
%
%\begin{proof}
%	Denote $\displaystyle m(w,b) = \min_{i} y_i(wx_i+b)$. Notice that $m(w,b)$ is a concave homogeneous function w.r.t $w,b$ and $\|\cdot\|_2$ is a strictly convex norm. Suppose there are two solution $(w_1,b_1)$ and $(w_2,b_2)$ such that $w_1 \neq w_2$, take $\overline{w} = \frac{w_1 + w_2}{2}, \overline{b} = \frac{b_1 + b_2}{2}$, we must have
%	\begin{equation}
%		m(\overline{w},\overline{b}) \geq \frac{m(w_1,b_1)+ m(w_2,b_2)}{2} = \max_{\|w\|_2 = 1} m(w,b),
%	\end{equation}
%	and
%	\begin{equation}
%		\|\overline{w}\|_2 < 1.
%	\end{equation}
%	So
%	\begin{equation}
%		m(\frac{\overline{w}}{\|\overline{w}\|_2},\frac{\overline{b}}{\|\overline{w}\|_2}) = \frac{m(\overline{w},\overline{b}) }{\|\overline{w}\|_2} > \max_{\|w\|_2 = 1} m(w,b),
%	\end{equation}
%	which leads to a contradiction. So all the solutions must have the same $w$, we denote it as $w^*$. Then if $(w^*,b^*)$ is a solution of problem (\ref{binarySVM}), we must have
%	\begin{equation}
%		b^* \in \mathop{\rm argmax}_{b} m(w^*,b)
%	\end{equation}
%	Actually,
%	\begin{equation}
%		m(w^*,b) = \min\{b+\min_{x\in A_1} w^*x, -b +\min_{x\in A_2} (-w^*x)\}.
%	\end{equation}
%	It is easy to observe that $\displaystyle \mathop{\rm argmax}_{b} m(w^*,b)$ is a singleton set and
%	\begin{equation}
%		b^* = \frac{\min_{x\in A_2} (-w^*x) - \min_{x\in A_1} w^*x}{2}.
%	\end{equation}
%\end{proof}
%
%Denote
%\begin{equation}\label{maxSVM}
%	\theta^*_{SVM} = (w_{SVM}^*,b_{SVM}^*) = \mathop{\rm argmax}_{\|w\| = 1} \min_{i} y_i(wx_i+b).
%\end{equation}
%
%
%\begin{theorem}[Representation Theorem]
%	Let $\theta^*_{SVM} = (w_{SVM}^*,b_{SVM}^*) $ be the solution of \eqref{maxSVM}. Then, $w_{SVM}^*$ must be a linear combination of $x_i^T, i = 1,2,\cdots,N$.
%\end{theorem}
%
%\begin{proof}
%	Denote
%	\begin{equation}
%		S = {\rm span} \{x_i^T\}_{i=1}^N.
%	\end{equation}
%	Then we have
%	\begin{equation}
%		\mathbb{R}^{1\times d} = S \oplus^{\perp} S^{\perp}.
%	\end{equation}
%	So $w_{SVM}^*$ can be uniquely decomposed as $w_{SVM}^* = w^*_S + w^*_{S^{\perp}}$ where $w_S\in S$ and $w^*_{S^{\perp}}\in S^{\perp}$.
%	We will prove that $w^*_{S^{\perp}} = 0$. Suppose not, we have
%	\begin{equation}
%		\|w^*_S\|_2 < \|w_{SVM}^*\|_2 = 1.
%	\end{equation}
%	Notice that
%	\begin{equation}
%		w_{SVM}^* x_i = w_S^* x_i,\ \forall i = 1,2,\cdots,N.
%	\end{equation}
%	Thus we have
%	\begin{equation}
%		\min_{i} y_i(w_{SVM}^*x_i+b_{SVM}^*) = \min_{i} y_i(w_S^*x_i+b_{SVM}^*).
%	\end{equation}
%	So
%	\begin{equation}
%	\min_{i} y_i(w_{SVM}^*x_i+b_{SVM}^*) < \frac{\min_{i} y_i(w_S^*x_i+b_{SVM}^*)}{\|w_S^*\|} = \min_{i} y_i(\frac{w^*_S}{\|w_S^*\|_2}x_i+\frac{b_{SVM}^*}{\|w^*_S}\|_2),
%	\end{equation}
%	which leads to a contradiction to the definition of $\theta_{SVM}^*$.
%\end{proof}

\subsection{Soft margin maximization and kernel methods}

We may rewrite the SVM problem as
\begin{align}
	\max_{w,b}&\ {2\over \|w\|},\\
	s.t.&\ \ell_i(wx_i+b) \geq 1,\ \forall i.
\end{align}
or equivalently,
\begin{align}{\label{SVM_Quad}}
	\min_{w,b}&\ \|w\|^2,\\
	s.t.&\ \ell_i(wx_i+b) \geq 1,\ \forall i.
\end{align}
Notice that the feasible domain of margin maximization is nonempty if and only if dataset is linearly separable. So when the data is linearly nonseparable, this method can't even get a classifier even though it may not be good. One way to handle this problem is to relax the constraint by adding relaxation variables.

Define soft margin maximization problem
\begin{align}{\label{SVM_Quad_soft}}
\min_{w,b,\xi}&\ \|w\|^2 + \lambda^{-1} \sum_{i = 1}^N\xi_i,\\
s.t.&\ \ell_i(wx_i+b) + \xi_i \geq 1,\ \forall i. \\
     &\ \xi_i \geq 0.
\end{align}
where $\lambda>0$. The above problem is equivalent to
 \begin{align}
 \min_{w,b}&\ \|w\|^2 + \lambda^{-1} \sum_{i = 1}^N {\rm ReLU}(1-\ell_i(wx_i+b)).
 \end{align}
Thus, soft margin maximization problem \eqref{SVM_Quad_soft} can be reformulated as
 \begin{align}{\label{SVM_soft}}
\min_{w,b}&\  \sum_{i = 1}^N {\rm ReLU}(1-\ell_i(wx_i+b)) + \lambda \|w\|^2.
\end{align}
We can still prove that the solution of (\ref{SVM_soft}) satisfies the representation theorem. Thus we can restrict $w$ to be in the set $S$. Assume that
\begin{equation}
	w = \sum_{i = 1}^N \alpha_i x_i^T,
\end{equation}
Denote $\alpha = (\alpha_1,\cdots,\alpha_N)^T$. We can rewrite the problem (\ref{SVM_soft}) as
\begin{equation}
\min_{\alpha}\ \sum_{i = 1}^N {\rm ReLU}(1-\ell_i(\sum_{j = 1}^N \langle x_i,x_j\rangle \alpha_j+b)) + \lambda\alpha^T \big(\langle x_i,x_j\rangle\big)_{N\times N} \alpha\\
\end{equation}
We can see that the whole problem is only determined by the inner product of data points but not the data itself directly. \\

Use the above formulation, we can induce nonlinearity in SVM. Denote the input space as $X$ where $\{x_i\}_{i=1}^N \subset X$. We use two steps to obtain a nonlinear classification model. First, we use a nonlinear feature mapping $\phi: X\rightarrow \mathcal{H}$ to map input space $X$ to a feature space $\mathcal{H}$. Second, we use linear SVM to do classification on $\{\phi(x_i)\}_{i=1}^N\subset \mathcal{H}$.\\

We may just asssume dataset after feature mapping $\phi$ is linearly separable. Then, the SVM problem after doing feature mapping can be formulated as
problem (\ref{SVM_soft}) as
\begin{equation}
\min_{\alpha}\ \sum_{i = 1}^N {\rm ReLU}(1-\ell_i(\sum_{j = 1}^N \langle \phi(x_i),\phi(x_j)\rangle \alpha_j+b)) + \lambda\alpha^T \big(\langle \phi(x_i),\phi(x_j)\rangle\big)_{N\times N} \alpha\\
\end{equation}

Notice that to obtain the above problem we don't really need to know what exactly is the nonlinear mapping $\phi$, but only need to compute the value of $<\phi(x_i),\phi(x_j)>$. So we define a kernel function $k: X\times X\rightarrow \mathbb{R}$ such that
\begin{equation}
	k(x,y) = \langle\phi(x),\phi(y)\rangle,\ x,y\in X.
\end{equation}
Then the kernel SVM can be formulated as
\begin{equation}
\min_{\alpha}\ \sum_{i = 1}^N {\rm ReLU}(1-\ell_i(\sum_{j = 1}^N  k(x_i,x_j) \alpha_j+b)) + \lambda\alpha^T \big(k(x_i,x_j)\big)_{N\times N} \alpha\\
\end{equation}
In practice, we just need to find a proper kernel function instead of a good nonlinear feature mapping. Here we list some common used kernel functions:
\begin{itemize}
	\item Polynomial kernel: $k(x,y) = (a\langle x,y\rangle+ b)^n, a > 0, b\geq 0, n\in \mathbb{N}^+$.
	\item Gaussian kernel: $k(x,y) = e^{-\gamma\|x-y\|^2}, \gamma > 0$.
	\item Laplacian kernel: $k(x,y) = e^{-\gamma\|x-y\|}, \gamma > 0$
	\item Tanh kernel: $k(x,y) = \tanh(a\langle x,y\rangle+b), a>0, b\geq 0.$
\end{itemize}

\subsection{Binary logistic regression}
In multi-class Logistic regression, if we use $\|W\|$ to replace $\|\bm\theta\|$ in regularization term,  we can get another version of logistic regression:
\begin{equation}
	\mathcal L_\lambda(\bm \theta) = - \sum_{i=1}^k \sum_{x\in A_i} \log p_{i}(x;\bm \theta) + \lambda R(\|W\|),
\end{equation}
where $p_i(x;\bm \theta)$ and $R(\cdot)$ share the same definitions as in previous sections of logistic regression.
Let
\begin{equation}
\bm\Theta_{\lambda} = \mathop{{\arg\min}}_{\bm\theta}  \mathcal L_\lambda(\bm\theta).
\end{equation}
The following lemma follows directly from the definition of $p_{i}(x;\bm \theta)$.
\begin{lemma}
	For any $W\in \mathbb{R}^{k\times d}, b\in \mathbb{R}^k, \alpha \in \mathbb{R}$, we have
	\begin{equation}
		\mathcal L_\lambda(W,b) = \mathcal L_\lambda(W,b + \alpha \bm 1),
	\end{equation}
	where $\bm 1 = (1,1,\cdots,1)^T\in\mathbb{R}^k.$
\end{lemma}

Binary logistic regression refers to the case when $k=2$.
\begin{lemma}
	If $k = 2$, given any $\bm\theta_\lambda = \begin{pmatrix}
	w_1 &  b_1\\
	w_2 & b_2
	\end{pmatrix} \in \bm\Theta_\lambda$, we have
	\begin{equation*}
         w_1 = -w_2.
	\end{equation*}
\end{lemma}

According to the above two lemmas, we can restrict $\bm\theta$ to have the form
\begin{equation}
	\bm\theta = \begin{pmatrix}
	\frac{w}{2}\ &\frac{b}{2}\\
	-\frac{w}{2}\ &-\frac{b}{2}
	\end{pmatrix},
\end{equation}
so our score mapping can be written as
\begin{equation}\label{key}
\bm p(x;\theta) = \begin{pmatrix}
	\frac{1}{1+ e^{-(wx+b)}}\\
	\frac{1}{1+e^{wx+b}}
\end{pmatrix}.
\end{equation}
Denote $\theta = (w,b)$ where $w\in \mathbb{R}^d, b\in \mathbb{R}$, correspondingly, we have
\begin{equation}
P(\theta) = \prod_{i = 1}^N \frac{1}{1+ e^{-y_i(wx+b)}}.
\end{equation}
Here we have the new ``label'' $y_i$ defined by
\begin{equation}\label{key}
y_i = \begin{cases}
1, \quad &\text{if}  \quad x_i \in A_1 \\
-1, \quad &\text{if} \quad x_i \in A_2
\end{cases}.
\end{equation}
Thus we have
\begin{equation}
L(\theta) = -\log P(\theta) = \sum_{i = 1}^N \log(1+ e^{-y_i(wx+b)}),
\end{equation}
and take $R(t) = t^2$, we have
\begin{equation}
\mathcal L_{\lambda}(\theta)  = L(\theta) + \lambda \|w\|_2^2 = \sum_{i = 1}^N \log(1+ e^{-y_i(wx+b)}) + \lambda \|w\|_2^2.
\end{equation}
Here $L(\theta)$ is a strictly convex function without any global minima.

\begin{lemma}
Assume that $A_1,A_2$ are linearly separable, and we follow the same definition
of linearly classifiable weights:
\begin{equation}
\bm{\Theta} = \left\{\bm\theta: p_i(x; \bm\theta)>p_j(x; \bm\theta),~\forall x\in A_i, j\neq i, i= 1,2\right\},
\end{equation}
where
\begin{equation}\label{key}
\bm p(x;\theta) = \begin{pmatrix}
\frac{1}{1+ e^{-(wx+b)}}\\
\frac{1}{1+e^{wx+b}}
\end{pmatrix}.
\end{equation}
Then, we have the following statement:\\

\begin{enumerate}

\item  $\theta = (w,b) \in \Theta$ if and only if
		\begin{equation*}
		\frac{1}{1+ e^{-y_i(wx+b)}} > \frac{1}{2},\ \forall i = 1,2\cdots,N.
		\end{equation*}
\item  If $P(\theta) > \frac{1}{2}$, then $\theta$ must be classifiable, i.e.
\begin{equation*}
	\{\theta: P(\theta)>\frac{1}{2}\}\subset \bm \Theta.
\end{equation*}
\item  Prove that
\begin{equation*}
	\bm\Theta = \{\theta: \lim_{\alpha\rightarrow +\infty} P(\alpha \theta) = 1\}.
\end{equation*}
\end{enumerate}
\end{lemma}

	If $A_1,A_2$ are linearly separable, then
$\displaystyle
\mathop{\rm argmin}_{w,b} \mathcal L_{\lambda}(\theta)
$
	is nonempty for $\lambda $ sufficiently small.

\begin{lemma}
		If $A_1,A_2$ are linearly separable, then
	\begin{equation}\label{binaryLR}
	\mathop{\rm argmin}_{w,b} \mathcal L_{\lambda}(\theta)
	\end{equation}
	is a singleton set for $\lambda $ sufficiently small.
\end{lemma}

\begin{proof}
Because $L(\theta)$ is strictly convex w.r.t. $\theta$ and $\|w\|^2$ is convex w.r.t. $\theta$, so $\mathcal L(\theta,\lambda)  = L(\theta) + \lambda \|w\|_2^2$ is stricly convex w.r.t. $\theta$, which implies our result directly.

\end{proof}

For $\lambda$ sufficiently small, denote
\begin{equation}
	\theta_{LR}(\lambda) = (w_{LR}(\lambda),b_{LR}(\lambda)) = \mathop{\rm argmin}_{w,b} \mathcal L_{\lambda}(\theta).
\end{equation}

\begin{lemma}
	If $A_1,A_2$ are linearly separable,
	\begin{enumerate}
	\item $\mathcal L_{\lambda}(\theta) \rightarrow 0$ as $\lambda \rightarrow 0$.
	\item $\|w_{LR}(\lambda)\|\rightarrow \infty$ as $\lambda \rightarrow 0$.
	\item $\displaystyle \min_{i} y_i(w_{LR}(\lambda) x_i + b_{LR}(\lambda))\rightarrow \infty$ as $\lambda \rightarrow 0$.
	\item If $A_1,A_2$ are also nonempty,
	\begin{equation}
		\|b_{LR}(\lambda)\|\leq \max_{i} \|x_i\| \|w_{LR}(\lambda)\|
	\end{equation}
	for $\lambda$ sufficiently small.
	\end{enumerate}
\end{lemma}

The above lemma implies that $\theta_{LR}(\lambda)/\|w_{LR}(\lambda)\|$ is bounded for $\lambda$ sufficiently small.

\begin{theorem}
	If $A_1,A_2$ are linearly separable, then $\frac{\theta_{LR}(\lambda)}{\|w_{LR}(\lambda)\|}$ converge to $\theta^*_{SVM}$ as $\lambda \rightarrow 0$, i.e.
	\begin{equation}
		\theta^*_{SVM} = \lim_{\lambda\rightarrow 0} \frac{\theta_{LR}(\lambda)}{\|w_{LR}(\lambda)\|}.
	\end{equation}
\end{theorem}

\begin{proof}
	Because $\frac{\theta_{LR}(\lambda)}{\|w_{LR}(\lambda)\|}$ is bounded for $\lambda$ sufficiently small, we only need to prove that it has no convergence points other than $\theta^*_{SVM}$ as $\lambda \rightarrow 0$.\\
	We first introduce a soft margin function $m: \mathbb{R}^n \rightarrow \mathbb{R}$ such that
	\begin{equation}
		m(\theta) = \min_{i} y_i(wx_i + b).
	\end{equation}
	Suppose that there exist a sequence $\{\lambda_n\}_{n = 1}^\infty$ and $\bar{\theta} = (\bar{w},\bar{b}) \neq \theta^*_{SVM}$ so that  $\lambda_n \downarrow 0$ and $\frac{\theta_{LR}(\lambda_n)}{\|w_{LR}(\lambda_n)\|} \rightarrow \bar{\theta}$ as $n\rightarrow \infty$. Obviously $\|\bar{w}\| = 1$, and
    \begin{equation}
    	0 \leq m(\bar{\theta}) < m(\theta^*_{SVM}).
    \end{equation}
     Take a positive number $\epsilon < m(\theta^*_{SVM}) - m(\bar{\theta})$. Then there exists $K_0\in \mathbb{N}^+$ such that
    \begin{equation}
    0 < m(\frac{\theta_{LR}(\lambda_n)}{\|w_{LR}(\lambda_n)\|}) <  m(\theta^*_{SVM})- \epsilon,
    \end{equation}
    for all $n \geq K_0$.\\

    Consider $\theta_n = (w_n,b_n) = \|w_{LR}(\lambda_n)\|\theta^*_{SVM}$. Then
    \begin{equation}
    \lim_{n\rightarrow \infty} \frac{\log(1 + e^{-m(\theta_n)})}{\log(1 + e^{-m(\theta_{LR}(\lambda_n))}} = \lim_{n \rightarrow \infty} e^{\|w_{LR}(\lambda_n)\| (m(\frac{\theta_{LR}(\lambda_n)}{\|w_{LR}(\lambda_n)\|}) -  m(\theta^*_{SVM}))} \leq \lim_{n \rightarrow \infty} e^{-\|w_{LR}(\lambda_n)\| \epsilon} = 0.
    \end{equation}
    So there exists a $K_1 \in \mathbb{N}^+$ such that
    \begin{equation}
    	N \log(1 + e^{-m(\theta_n)}) < \log(1 + e^{-m(\theta_{LR}(\lambda_n))})
    \end{equation}
    for all $n \geq K_1$. Take $K  = \max\{K_0, K _1\}$. Then for all $n \geq K$, we have
    \begin{equation}
    	\mathcal{L}(\theta_n) \leq N \log(1 + e^{-m(\theta_n)}) + \lambda \|w_{\lambda_n}\|^2 < \log(1 + e^{-m(\theta_{LR}(\lambda_n))})+ \lambda \|w_{\lambda_n}\|^2 \leq \mathcal{L}(\theta_{LR}(\lambda_n)),
    \end{equation}
    which contradicts the definition of $\theta_{LR}(\lambda)$.
\end{proof}

For the proof of the above theorem, you can also refer to the paper by Rosset Saharon, Zhu Ji and Trevor J. Hastie~ \cite{rosset2004margin}.

497-6DL/2 Linear Machine Learning Models/2.4-binaryLRSVM.tex