\chapter{Fundamentals}
\label{chap:fund}
This chapter documents the definitions for fields, potentials, and
Fourier transforms that are employed in remainder of this document.

We restrict consideration to time-harmonic sources in simple, linear media,
assuming and suppressing
a time dependence of $e^{j\omega t}$.  RMS phasers are used throughout, employing
rationalized MKS units. Thus,
if $V$ is a (complex) phaser voltage, then the corresponding function of time
is $v(t) = \sqrt{2} \Real{V e^{j\omega t}}$.

A right-handed Cartesian  coordinate system is adopted, with $x$, $y$, and
$z$ axes, and unit vectors $\x$, $\y$, and $\z$.  A point $P = (x,y,z)$ is
typically identified by the vector $\r = x\x + y\y + z\z$ which measures
the displacement of $P$ from the origin.

The medium under consideration is characterized by its scalar, complex permittivity
$\epsilon$ [F/m] and its scalar, complex permeability $\mu$ [H/m],
both of which may be functions
of position, $\epsilon = \epsilon(\r)$, $\mu = \mu(\r)$. Although these parameters are
both positive for lossless media, in the presence of electric and/or magnetic
losses the imaginary part of $\epsilon$ and/or $\mu$, respectively, is negative.
Thus, $\epsilon$ and $\mu$ both lie in the fourth quadrant (or positive real
axis) of the complex plane.

For convenience we define the medium's intrinsic wavenumber
\begin{equation}
  k = \omega \sqrt{\mu\epsilon} \qquad \text{(fourth quadrant)}
\end{equation}
and intrinsic impedance
\begin{equation}
  \eta = \sqrt{\mu/\epsilon} \qquad \left(\abs{\arg\eta} < \frac{\pi}{4} \right),
\end{equation}
which, of course, vary with position if $\epsilon$ or $\mu$ do.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Maxwell's Equations and Potentials for Electric Sources}
Under the assumptions listed above, and postulating the existence of only
electric sources, Maxwell's curl equations
(Ampere's Law and Faraday's Law) take the form
\begin{subequations}
  \begin{align}
    \curl \H &= j\omega\epsilon \E + \J \label{eq:AmpereE} \\
    \curl \E &= -j\omega\mu \H  \label{eq:FaradayE}
  \end{align}
\end{subequations}
where $\E$ is the electric field vector [V/m], $\H$ is the magnetic field
vector [A/m], and $\J$ is the electric current density [A/m$^2$].
When combined with the equation of continuity
\begin{equation}
  \divergence \J + j\omega\qe = 0
\end{equation}
we obtain the divergence relations
\begin{subequations}
  \begin{align}
    \divergence \epsilon\E &= \qe  \label{eq:divE}\\
    \divergence \mu\H       &= 0,
  \end{align}
\end{subequations}
where $\qe$ is the electric charge density,
with units of [C/m$^3$].
The fact that $\mu\H$ is divergenceless leads to
the introduction of the magnetic vector potential $\A$
having units of [Vs/m]:
\begin{equation}
  \boxed{
  \mu\H = \curl \A.
  } \label{eq:curlA}
\end{equation}
Substituting \eqref{eq:curlA} into \eqref{eq:FaradayE} we find that
$-\E - j\omega\A$ is curl-free, and so can be written as the gradient
of the so-called electric scalar potential $\Phi$ [V]:
\begin{equation}
  \label{eq:PhiE}
  \boxed{
  \E = -j\omega\A-\gradient\Phi.
  }
\end{equation}
To derive the differential equations for $\A$ and $\Phi$ we
begin with the identity
\begin{equation}
  \curl \curl \A = \curl \mu\H = \mu \curl \H + \gradient\mu \cross \H
\end{equation}
and use Equations \eqref{eq:AmpereE} and \eqref{eq:curlA} to eliminate
$\H$:
\begin{equation}
  \gradient\divergence\A - \vlaplace\A =
  j\omega\epsilon\mu\E + \mu\J + \gradient\mu \cross
  \left(
    \frac{1}{\mu} \curl\A
  \right).
\end{equation}
Note that we also employed the identity $\curl\curl\A =
  \gradient\divergence\A - \vlaplace\A$.
Now eliminating $\E$ using \eqref{eq:PhiE} we obtain
\begin{equation}
  \vlaplace\A + k^2 \A + \gradient\mu \cross
  \left(
    \frac{1}{\mu} \curl\A
  \right)
  - \gradient
  \left[
    \divergence\A + j\omega\mu\epsilon\Phi
  \right]
  = -\mu\J.
\end{equation}
Since the divergence of the magnetic vector potential is as yet unspecified,
we may apply the Lorentz gauge, $\divergence\A = -j\omega\epsilon\mu\Phi$,
and set the quantity in square brackets above to zero:
\begin{equation}
  \boxed{%
  \vlaplace\A + k^2 \A + \frac{\gradient\mu}{\mu} \cross \curl\A
  = -\mu\J.  \label{eq:WaveAinhom}
  }
\end{equation}
Equation~\eqref{eq:WaveAinhom} is the fundamental wave equation for
the magnetic vector potential under the Lorentz gauge in an inhomogeneous
medium.

An equation for $\Phi$ is now obtained by employing the
identity $\divergence \epsilon\E = \E\bdot\gradient \epsilon
+ \epsilon \divergence\E$ in \eqref{eq:divE} and then
eliminating $\E$ using \eqref{eq:PhiE}:
\begin{align}
  \qe
  &= \E \bdot \gradient \epsilon + \epsilon \divergence\E \notag \\
  &= -(j\omega\A + \gradient\Phi) \bdot \gradient\epsilon
  - \epsilon \divergence (j\omega\A + \gradient\Phi).
\end{align}
After invoking the Lorentz gauge this can be written as
\begin{equation}
  \boxed{%
  \laplace\Phi + k^2\Phi + \frac{\gradient\Phi \bdot \gradient\epsilon}{\epsilon}
  =
  -\frac{\qe}{\epsilon} - \frac{j\omega\A \bdot \gradient\epsilon}{\epsilon}.
  } \label{eq:WavePhiinhom}
\end{equation}
Equation~\eqref{eq:WavePhiinhom} is the wave equation for
the electric scalar potential under the Lorentz gauge in an inhomogeneous
medium.

\subsection{Piecewise Homogeneous Medium}
Suppose that the spatial domain $U$ of the boundary value problem
for which Maxwell's Equations are to be
solved consists of a disjoint union of a finite number $N$ of
homogeneous regions $U_i$, as in the case of a stratified medium:
\begin{equation}
  U = \union_{i=1}^{N} U_i,
\end{equation}
and suppose that the permittivity and permeability of the $i$th region
are the constants $\epsilon_i$ and $\mu_i$, respectively, with corresponding
wavenumber $k_i$.  Then, for
points within the $i$th medium, the terms involving the gradient of
the permittivity and permeability are zero, and the
potentials within the $i$th region are solutions to
\begin{subequations}
  \begin{gather}
    \vlaplace\A^{(i)} + k_i^2 \A^{(i)}  = -\mu_i\J, \\
    \laplace\Phi^{(i)} + k_i^2 \Phi^{(i)}  = -\qe/\epsilon_i.
  \end{gather}
\end{subequations}
If the $i$th region contains no sources, then the potentials
in that region are solutions to the Helmholtz equation:
\begin{subequations}
  \begin{gather}
    \vlaplace\A^{(i)} + k_i^2 \A^{(i)}  = \0, \\
    \laplace\Phi^{(i)} + k_i^2 \Phi^{(i)}  = 0.
  \end{gather}
\end{subequations}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Maxwell's Equations for Magnetic Sources}
Under the assumptions listed in the introduction, and postulating the existence of only
magnetic sources, Maxwell's curl equations
(Ampere's Law and Faraday's Law) take the form
\begin{subequations}
  \begin{align}
    \curl \H &= j\omega\epsilon \E \label{eq:AmpereM} \\
    \curl \E &= -j\omega\mu \H - \M \label{eq:FaradayM}
  \end{align}
\end{subequations}
where $\E$ is the electric field vector [V/m], $\H$ is the magnetic field
vector [A/m], and $\M$ is the magnetic current density [V/m$^2$].
When combined with the equation of continuity
\begin{equation}
  \divergence \M + j\omega\qm = 0
\end{equation}
we obtain the divergence relations
\begin{subequations}
  \begin{align}
    \divergence \epsilon\E &= 0  \label{eq:divEm}\\
    \divergence \mu\H &= \qm,    \label{eq:divHm}
  \end{align}
\end{subequations}
where $\qm$ is the magnetic charge density,
with units of [Wb/m$^3$].
The fact that $\epsilon\E$ is divergenceless leads to
the introduction of the electric vector potential $\F$
having units of [As/m]:
\begin{equation}
  \boxed{
  \epsilon\E = \curl \F.
  } \label{eq:curlF}
\end{equation}
Substituting \eqref{eq:curlF} into \eqref{eq:AmpereM} we find that
$j\omega\F - \H$ is curl-free, and so can be written as the gradient
of the so-called magnetic scalar potential $\Psi$ [A]:
\begin{equation}
  \label{eq:PsiM}
  \boxed{
  \H =  j\omega\F - \gradient\Psi.
  }
\end{equation}
To derive the differential equations for $\F$ and $\Psi$ we
begin with the identity
\begin{equation}
  \curl \curl \F = \curl \epsilon\E = \epsilon \curl \E + \gradient\epsilon \cross \E
\end{equation}
and use Equations \eqref{eq:FaradayM} and \eqref{eq:curlF} to eliminate
$\E$:
\begin{equation}
  \gradient\divergence\F - \vlaplace\F =
  -j\omega\epsilon\mu\H - \epsilon\M + \gradient\epsilon \cross
  \left(
    \frac{1}{\epsilon} \curl\F
  \right).
\end{equation}
Note that we also employed the identity $\curl\curl\F =
  \gradient\divergence\F - \vlaplace\F$.
Now eliminating $\H$ using \eqref{eq:PsiM} we obtain
\begin{equation}
  \vlaplace\F + k^2 \F + \gradient\epsilon \cross
  \left(
    \frac{1}{\epsilon} \curl\F
  \right)
  - \gradient
  \left[
    \divergence\F - j\omega\mu\epsilon\Psi
  \right]
  = \epsilon\M.
\end{equation}
Since the divergence of the electric vector potential is as yet unspecified,
we may apply the Lorentz gauge, $\divergence\F = j\omega\epsilon\mu\Psi$,
and set the quantity in square brackets above to zero:
\begin{equation}
  \boxed{%
  \vlaplace\F + k^2 \F + \frac{\gradient\epsilon}{\epsilon} \cross
  \curl\F
  = \epsilon\M.
  \label{eq:WaveFinhom}
  }
\end{equation}
Equation~\eqref{eq:WaveFinhom} is the fundamental wave equation for
the electric vector potential under the Lorentz gauge in an inhomogeneous
medium.

An equation for $\Psi$ is now obtained by employing the
identity $\divergence \mu\H = \H\bdot\gradient \mu
+ \mu \divergence\H$ in \eqref{eq:divHm} and then
eliminating $\H$ using \eqref{eq:PsiM}:
\begin{align}
  \qm
  &= \H \bdot \gradient \mu + \mu \divergence\H \notag \\
  &= (j\omega\F - \gradient\Psi) \bdot \gradient\mu
  + \mu \divergence (j\omega\F - \gradient\Psi).
\end{align}
After invoking the Lorentz gauge this can be written as
\begin{equation}
  \boxed{%
  \laplace\Psi + k^2\Psi + \frac{\gradient\Psi \bdot \gradient\mu}{\mu}
  =
  -\frac{\qm}{\mu} + \frac{j\omega\F \bdot \gradient\mu}{\mu}.
  } \label{eq:WavePsiinhom}
\end{equation}
Equation~\eqref{eq:WavePsiinhom} is the wave equation for
the electric scalar potential under the Lorentz gauge in an inhomogeneous
medium.

\subsection{Piecewise Homogeneous Medium}
Suppose that the spatial domain $U$ of the boundary value problem
for which Maxwell's Equations are to be
solved consists of a disjoint union of a finite number $N$ of
homogeneous regions $U_i$, as in the case of a stratified medium:
\begin{equation}
  U = \union_{i=1}^{N} U_i,
\end{equation}
and suppose that the permittivity and permeability of the $i$th region
are the constants $\epsilon_i$ and $\mu_i$, respectively, with corresponding
wavenumber $k_i$.  Then, for
points within the $i$th medium, the terms involving the gradient of
the permittivity and permeability are zero, and the
potentials within the $i$th region are solutions to
\begin{subequations}
  \begin{gather}
    \vlaplace\F^{(i)} + k_i^2 \F^{(i)}  = \epsilon_i\M, \\
    \laplace\Psi^{(i)} + k_i^2 \Psi^{(i)}  = -\qm/\mu_i.
  \end{gather}
\end{subequations}
If the $i$th region contains no sources, then the potentials
in that region are solutions to the Helmholtz equation:
\begin{subequations}
  \begin{gather}
    \vlaplace\F^{(i)} + k_i^2 \F^{(i)}  = \0, \\
    \laplace\Psi^{(i)} + k_i^2 \Psi^{(i)}  = 0.
  \end{gather}
\end{subequations}

\section{Duality}
We note that the cases of electric-only and magnetic-only sources are duals.
Any valid equation involving electromagnetic quantities has a dual equation
which can be obtained by applying the following rules:
\begin{enumerate}
\item Interchange $\mu$ and $\epsilon$.
\item Electric quantities are replaced by their corresponding magnetic quantity.
\item Magnetic quantities are replaced by the negative of their corresponding electric quantity.
\end{enumerate}
The mappings between original and dual quantities are given in
Table~\ref{tab:duals}.
\begin{table}[htbp]
  \begin{center}
    \leavevmode
    \begin{tabular}{|c|} \hline
      \bfseries Original $\boldsymbol{\rightarrow}$ Dual \\ \hline \hline
      $\mu \rightarrow \epsilon$ \\ \hline
      $\epsilon \rightarrow \mu$ \\ \hline
      $k \rightarrow k$ \\ \hline
      $\eta \rightarrow 1/\eta$ \\ \hline
      $\E \rightarrow \H$ \\ \hline
      $\J \rightarrow \M$ \\ \hline
      $\F \rightarrow \A$ \\ \hline
      $\Phi \rightarrow \Psi$ \\ \hline
      $\qe \rightarrow \qm$ \\ \hline
      $\H \rightarrow -\E$ \\ \hline
      $\M \rightarrow -\J$ \\ \hline
      $\A \rightarrow -\F$ \\ \hline
      $\Psi \rightarrow -\Phi$ \\ \hline
      $\qm \rightarrow -\qe$ \\ \hline
    \end{tabular}
    \caption{Electromagnetic dual quantities.}
    \label{tab:duals}
  \end{center}
\end{table}

\section{Fourier Transform Definitions}
\subsection{One-dimensional Transform}
The Fourier transform of a function $f: \Realnum \rightarrow \Complexnum$
is $\tilde{f} = \Fourier \{f\}$, where
\begin{equation}
  \tilde{f}(k) = \int_{-\infty}^{\infty} f(x) e^{jkx} \d x,
\end{equation}
so that
\begin{equation}
  f(x) = \frac{1}{2\pi} \int_{-\infty}^{\infty} \tilde{f}(k) e^{-jkx} \d k.
\end{equation}

The completeness statement is
\begin{equation}
  \delta(x) = \frac{1}{2\pi} \int_{-\infty}^{\infty} e^{\pm jkx} \d k
\end{equation}
and Parseval's relation is
\begin{equation}
\int_{-\infty}^{\infty} f(x) g^*(x) \, \d x =
\frac{1}{2\pi} \int_{-\infty}^{\infty} \tilde{f}(k) \tilde{g}^*(k) \, \d k.
\end{equation}
Finally, if
\begin{equation}
  h(x) = \int_{-\infty}^{\infty} f(x') g(x-x') \, \d x'
\end{equation}
then the convolution theorem states that
\begin{equation}
  \tilde{h}(k) = \tilde{f}(k) \tilde{g}(k).
\end{equation}

\subsection{Two-dimensional Transform}
The Fourier transform of a function $f: \Realnum \times \Realnum
\rightarrow \Complexnum$
is $\tilde{f} = \Fourier \{f\}$, where
\begin{equation}
  \tilde{f}(k_x,k_y) =
  \int_{-\infty}^{\infty}\int_{-\infty}^{\infty}
  f(x,y) e^{j(k_x x + k_y y)} \d x  \d y,
\end{equation}
so that
\begin{equation}
  f(x,y) = \frac{1}{4\pi^2}
  \int_{-\infty}^{\infty}\int_{-\infty}^{\infty}
 \tilde{f}(k_x,k_y) e^{-j(k_x x + k_y y)} \d k_x  \d k_y.
\end{equation}

The completeness statement is
\begin{equation}
  \delta(x) \delta(y) = \frac{1}{4\pi^2}
  \int_{-\infty}^{\infty}\int_{-\infty}^{\infty}
  e^{\pm j(k_x x + k_y y)} \d k_x  \d k_y
\end{equation}
and Parseval's relation is
\begin{equation}
\int_{-\infty}^{\infty} \int_{-\infty}^{\infty}
 f(x,y) g^*(x,y) \, \d x  \d y =
\frac{1}{4\pi^2} \int_{-\infty}^{\infty} \int_{-\infty}^{\infty}
\tilde{f}(k_x,k_y) \tilde{g}^*(k_x,k_y) \, \d k_x  \d k_y.
\end{equation}
Finally, if
\begin{equation}
  h(x,y) = \int_{-\infty}^{\infty}\int_{-\infty}^{\infty}
 f(x',y') g(x-x',y-y') \, \d x'  \d y'
\end{equation}
then the convolution theorem states that
\begin{equation}
  \tilde{h}(k_x,k_y) = \tilde{f}(k_x,k_y) \tilde{g}(k_x,k_y).
\end{equation}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CS624: Analysis of Algorithms
% Copyright 2015 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/beacon
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Question 7}

Give a counter-example to the conjecture that if a directed graph $G$ contains a path from $u$ to $v$, then any depth-first search must result in $v.d \leq u.f$.

\subsection*{Solution}

The counter-example proposed for previous question depicted in Figure \ref{fig61} can be proposed again.
Starting from node $a$ as the source vertex, if depth-first algorithm continues with vertex $b$ ($b.d = 2$), there is no child of $b$ not already visited and therefore, node $b$ will be marked as visited ($b.f = 3 $).
Going back to the partially visited vertex $a$, the algorithm would continue by discovering $c$ ($c.d = 5$).
And this shows although there is a path from $b$ to $c$, $c.d \nleq b.f$.


\documentclass[11pt,a4paper]{article}

\usepackage{tikz}
\usepackage[parfill]{parskip}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{placeins}
\usepackage{cleveref}
\usepackage{bm}

\newcommand{\norm}[1]{\left\lVert#1\right\rVert}
\newcommand{\p}[1]{\left(#1\right)}

\begin{document}

\usetikzlibrary{math}

\section{Introduction}
This document describes how we can generate and transform
various shapes for rendering on a GPU. The output of each
shape is a list of vertices and some parameters to tweak
how the shape is rendered.

\section{Transformations}
There are two main ways to transform a shape. We can transform
the underlying data that describes the shape, or we can transform
the shape itself. All shapes discussed here are described by a set
of 2D points. When we transform the underlying data, it is these
points we refer to. Consider a point $\mathbf{x} \in \mathbb{R}^2$.
Any linear transformation can be described by
%
\begin{align}
  \mathbf{y} = \mathbf{A} \mathbf{x} + \mathbf{b}
\end{align}
%
If we apply this to all points before rendering, we have
transformed the underlying data. To transform the shape
we instead apply it to the points defining the triangles
that are rendered by the GPU. Of course, both approaches
can be combined.

\subsection{Homogenous Coordinates}
It is often simpler to work in homogenous coordinates,
since chaining transformations can be done by multiplying
matrices. The point $\mathbf{x}$ is represented as
%
\begin{align}
  \mathbf{x}' = \begin{bmatrix} \mathbf{x} \\ 1 \end{bmatrix}
\end{align}
%
Define a matrix
%
\begin{align}
  \mathbf{A}' = \begin{bmatrix}
    \mathbf{A} & \mathbf{b} \\
    \mathbf{0}^T & 1
    \end{bmatrix}
\end{align}
%
Then
%
\begin{align}
  \mathbf{y}' = \mathbf{A}' \mathbf{x}'
\end{align}
%
where $\mathbf{y}'$ is also a homogenous coordinate. Hence,
transformations are easily chained. We can also reverse the
transformation by inverting $\mathbf{A}'$.

\subsection{Decomposition}
If we restrict ourselves to the transformations scaling, rotation and translation,
then any transformation can be decomposed as
%
\begin{align}
  \mathbf{y} = \mathbf{R} \mathbf{S} \mathbf{x} + \mathbf{b}
\end{align}
%
where
%
\begin{align}
  \mathbf{R} = \begin{bmatrix}
    \cos(\theta) & -\sin(\theta) \\
    \sin(\theta) & \cos(\theta)
  \end{bmatrix}\quad,\quad
  %
  \mathbf{S} = \begin{bmatrix}
    s_x & 0 \\ 0 & s_y
  \end{bmatrix}
\end{align}
%
where $\theta$ is rotation, $s_x$ and $s_y$ is $x$ and $y$ scaling,
and $\mathbf{b}$ is translation. We thus have
%
\begin{align}
  \mathbf{A} = \mathbf{RS} = \begin{bmatrix}
    s_x \cos(\theta) & -s_y \sin(\theta) \\
    s_x \sin(\theta) & s_y \cos(\theta)
  \end{bmatrix}
\end{align}
%
which yields $4$ equations with $3$ unknowns. We begin solving for $\theta$.
%
\begin{align}
  \frac{s_x \sin(\theta)}{s_x \cos(\theta)} = \frac{A_{00}}{A_{10}}
\end{align}
%
which leads to
%
\begin{align}
  \theta = \tan^{-1}\p{\frac{A_{00}}{A_{10}}}
\end{align}
%
$A_{10}$ might be problematic. Hence, we also consider the solution
%
\begin{align}
  \theta = \tan^{-1}\p{\frac{A_{01}}{A_{11}}}
\end{align}
%
With the transformations considered, $A_{10}$ and $A_{11}$ will never
be zero simultaneously.  From here it's easy to see that
%
\begin{align}
  s_x &= \frac{A_{00}}{\cos(\theta)}\\
  s_y &= \frac{A_{11}}{\cos(\theta)}
\end{align}
%
Thus, the decomposed quantities can easily be extracted from any
homogenous transformation matrix $\mathbf{A}'$.

\FloatBarrier
\section{Lines}
\subsection{Single Line}
Let $\mathbf{x}_1$ be one side of the line and $\mathbf{x}_2$ be the
other.  Let $w$ be desired with of the line. \Cref{fig:single-line}
shows an example of a single drawn line. To draw this line on the GPU
we need to pass the points $a$, $b$, $c$, and $d$ to a vertex shader
somehow.  It is important that both the transformation matrix
$\mathbf{A}'$ and the line width $w$, is \emph{only} used in the GPU.
With this constraint, the vertices need only be computed once.
%
\begin{figure}
  \centering
  \begin{tikzpicture}
    \tikzmath{
      \x1 = 0;
      \y1 = 0;
      \x2 = 4;
      \y2 = 2;
      \l = sqrt((\x2-\x1)^2 + (\y2-\y1)^2);
      \w = 0.4;
      \dx = \w * (\x2 - \x1) / \l;
      \dy = \w * (\y2 - \y1) / \l;
    }

    \coordinate (a) at (\x1-\dy, \y1+\dx);
    \coordinate (b) at (\x1+\dy, \y1-\dx);
    \coordinate (c) at (\x2+\dy, \y2-\dx);
    \coordinate (d) at (\x2-\dy, \y2+\dx);
    \filldraw[black] (\x1, \y1) circle (2pt) node [anchor=east] {$\mathbf{y}_1$};
    \filldraw[black] (\x2, \y2) circle (2pt) node [anchor=west] {$\mathbf{y}_2$};

    \draw[gray,thick] (a) node [anchor=south] {a} --
    (b) node [anchor=north] {b} --
    (c) node [anchor=north] {c} --
    (d) node [anchor=south] {d} -- (a);
    \draw[black,thick,dashed] (\x1, \y1) -- (\x2, \y2);
  \end{tikzpicture}
  \caption{Single line}
  \label{fig:single-line}
\end{figure}

First, define
%
\begin{align}
  \mathbf{y}_1 &= \mathbf{A} \mathbf{x}_1 + \mathbf{b} \\
  \mathbf{y}_2 &= \mathbf{A} \mathbf{x}_2 + \mathbf{b} \\
\end{align}
%
and
%
\begin{align}
  \bm{\Delta} &= \mathbf{y}_2 - \mathbf{y}_1\\
  l &= \norm{\bm{\Delta}}
\end{align}
%
Next, we define a matrix
%
\begin{align}
  \mathbf{R} = \begin{bmatrix} 0 & -1 \\ 1 & 0 \end{bmatrix}
\end{align}
%
In practice, applying this to a vector rotates the vector by 90 degrees
counter-clockwise. We now have what we need to define the corners
of the rectangle defining the line.
%
\begin{align}
  \mathbf{a} &= \mathbf{y}_1 + \frac{w}{l} \mathbf{R} \bm{\Delta}\\
  \mathbf{b} &= \mathbf{y}_1 - \frac{w}{l} \mathbf{R} \bm{\Delta}\\
  \mathbf{c} &= \mathbf{y}_2 + \frac{w}{l} \mathbf{R} \bm{\Delta}\\
  \mathbf{d} &= \mathbf{y}_2 - \frac{w}{l} \mathbf{R} \bm{\Delta}\\
\end{align}
%
There is a problem here however. $\mathbf{A}'$ is baked in to several
of the elements. We replace the $\mathbf{y}$s with $\mathbf{x}$s. We
define
%
\begin{align}
  \bm{\Delta}_x = \p{\mathbf{x}_2 - \mathbf{x}_1}
\end{align}
%
which yields
%
\begin{align}
  \bm{\Delta} &= \mathbf{A} \bm{\Delta}_x\\
  l &= \norm{\mathbf{A} \bm{\Delta}_x}
\end{align}
%
We can now rewrite the rectangles as
%
\begin{align}
  \mathbf{a} &= \mathbf{A} \mathbf{x}_1 + \mathbf{b} +
               \frac{w}{l} \mathbf{R} \mathbf{A} \bm{\Delta}_x\\
  %
  \mathbf{b} &= \mathbf{A} \mathbf{x}_1 + \mathbf{b} -
               \frac{w}{l} \mathbf{R} \mathbf{A} \bm{\Delta}_x\\
  %
  \mathbf{c} &= \mathbf{A} \mathbf{x}_2 + \mathbf{b} +
               \frac{w}{l} \mathbf{R} \mathbf{A} \bm{\Delta}_x\\
  %
  \mathbf{d} &= \mathbf{A} \mathbf{x}_2 + \mathbf{b} -
               \frac{w}{l} \mathbf{R} \mathbf{A} \bm{\Delta}_x\\
\end{align}
%
From this, we see that each vertex needs five attributes, its
position, $\bm{\Delta}$, and its offset direction.  Let
$\mathbf{v} \in \mathbb{R}^5$ represent a vertex. The 2D position to
draw each point is
%
\begin{align}
  \mathbf{p} &= \mathbf{A} \begin{bmatrix} v_0 \\ v_1 \end{bmatrix} + \mathbf{b} +
  \frac{v_4 w}{l} \mathbf{R} \mathbf{A} \begin{bmatrix} v_2 \\ v_3 \end{bmatrix}\\
  l &= \norm{\mathbf{A} \begin{bmatrix} v_2 \\ v_3 \end{bmatrix}}
\end{align}
%
where
%
\begin{align}
  \mathbf{a} \quad &=> \quad \mathbf{v} =
            \begin{bmatrix} \mathbf{x}_1^T & \bm{\Delta}^T & 1 \end{bmatrix}^T\\
  %
  \mathbf{b} \quad &=> \quad \mathbf{v} =
            \begin{bmatrix} \mathbf{x}_1^T & \bm{\Delta}^T & -1 \end{bmatrix}^T\\
  %
  \mathbf{c} \quad &=> \quad \mathbf{v} =
            \begin{bmatrix} \mathbf{x}_2^T & \bm{\Delta}^T & 1 \end{bmatrix}^T\\
  %
  \mathbf{d} \quad &=> \quad \mathbf{v} =
            \begin{bmatrix} \mathbf{x}_2^T & \bm{\Delta}^T & -1 \end{bmatrix}^T\\
\end{align}
%
and $\mathbf{A}$, $\mathbf{b}$ and $w$ are passed as uniforms. This way, the
vertices only need to be computed once.

\FloatBarrier
\subsection{Multiple Lines}
When drawing multiple lines, we have to make sure they intersect nicely.
An example is shown in \cref{fig:multi-line}, and there is a gap between
the lines which must be filled in. To do this we draw an additional triangle
between the black dot, corner d and corner e. If the angle between the
lines went the other way the gap would occur between the black dot, corner
f and corner c. Hence, we draw a triangle here as well.
%
\begin{figure}
  \centering
  \begin{tikzpicture}
    \tikzmath{
      \x1 = 0;
      \y1 = 0;
      \x2 = 4;
      \y2 = 2;
      \x3 = 6;
      \y3 = 0;
      \w = 0.4;
      %
      \l1 = sqrt((\x2-\x1)^2 + (\y2-\y1)^2);
      \dx1 = \w * (\x2 - \x1) / \l1;
      \dy1 = \w * (\y2 - \y1) / \l1;
      %
      \l2 = sqrt((\x3-\x2)^2 + (\y3-\y2)^2);
      \dx2 = \w * (\x3 - \x2) / \l2;
      \dy2 = \w * (\y3 - \y2) / \l2;
    }

    \coordinate (a) at (\x1-\dy1, \y1+\dx1);
    \coordinate (b) at (\x1+\dy1, \y1-\dx1);
    \coordinate (c) at (\x2+\dy1, \y2-\dx1);
    \coordinate (d) at (\x2-\dy1, \y2+\dx1);

    \coordinate (e) at (\x2-\dy2, \y2+\dx2);
    \coordinate (f) at (\x2+\dy2, \y2-\dx2);
    \coordinate (g) at (\x3+\dy2, \y3-\dx2);
    \coordinate (h) at (\x3-\dy2, \y3+\dx2);

    \filldraw[black] (\x1, \y1) circle (2pt);
    \filldraw[black] (\x2, \y2) circle (2pt);
    \filldraw[black] (\x3, \y3) circle (2pt);

    \draw[gray,thick]
      (a) node [anchor=east] {a} --
      (b) node [anchor=north] {b} --
      (c) node [anchor=west] {c} --
      (d) node [anchor=south] {d} --
      (a);
    \draw[gray,thick]
      (e) node [anchor=south] {e} --
      (f) node [anchor=east] {f} --
      (g) node [anchor=north] {g} --
      (h) node [anchor=west] {h} --
      (e);
  \end{tikzpicture}
  \caption{Multiple lines}
  \label{fig:multi-line}
\end{figure}

\end{document}


\newcommand\vol{\mathit{vol}}

\chapter{Conductance, Expanders and Cheeger's Inequality}
\label{chap:cheegersInequality}
\sloppy

A common algorithmic problem that arises is the problem of partitioning the vertex set $V$ of a graph $G$ into clusters $X_1, X_2, \dots, X_k$ such that
\begin{itemize}
    \item for each $i$, the \emph{induced} graph $G[X_i] = (X_i, E \cap (X_i \times X_i))$ is "well-connected", and
    \item only an $\epsilon$-fraction of edges $e$ are not contained in any induced graph $G[X_i]$ (where $\epsilon$ is a very small constant).
\end{itemize}

\begin{figure}[!ht]
    \centering
    \includegraphics[scale=0.25]{fig/fig1_lecCheeger}
    \caption{After removing the red edges (of which there are few in relation to the total number of edges), each connected component in $G$ is "well-connected".}
\end{figure}

In this lecture, we make precise what "well-connected" means by introducing the notions of \emph{conductance} and \emph{expanders}.

Building on the last two lectures, we show that the second eigenvalue of the Laplacian $L$ associated with graph $G$ can be used to certify that a graph is "well-connected" (more precisely the second eigenvalue of a normalized version of the Laplacian). This result, called Cheeger's inequality, is one of the key tools in Spectral Graph Theory. Moreover, it can be turned into an algorithm that computes the partition efficiently!

\section{Conductance and Expanders}

\paragraph{Graph Definitions.} In this lecture, we let $G=(V,E)$ be unweighted\footnote{Everything we present here also works for weighted graphs, however, we focus on unweighted graphs for simplicity.} and always be \emph{connected}, and let $\dd(v)$ be the degree of a vertex $v$ in $G$. We define the \emph{volume} $\vol(S)$ for any vertex subset $S \subseteq V$, to be the sum of degrees, i.e. $\vol(S) = \sum_{v \in S} \dd(v)$.

For any  $A, B \subseteq V$, we define $E(A, B)$ to be the set of edges in $E \cap (A \times B)$, i.e. with one endpoint in $A$ and one endpoint in $B$. We let $G[A]$ be the \emph{induced} graph $G$ by $A \subseteq V$, which is the graph $G$ restricted to the vertices $A$, i.e. an edge $e$ in $G$ is in $G[A]$ iff both endpoints are in $A$.

\paragraph{Conductance.} Given set $\emptyset \subset S \subset V$, then we define the conductance $\phi(S)$ of $S$ by
\[
\phi(S) = \frac{|E(S, V \setminus S)|}{ \min\{\vol(S), \vol(V \setminus S) \} }.
\]
It can be seen that $\phi(\cdot)$ is symmetric in the sense that $\phi(S) = \phi(V \setminus S)$. We define the conductance of the graph $G$ denoted $\phi(G)$ by
\[
    \phi(G) = \min_{\emptyset \subset S \subset V} \phi(S).
\]
We note that finding the conductance of a graph $G$ is NP-hard. However, good approximations can be found as we will see today (and in a later lecture).

\paragraph{Expander and Expander Decomposition.} For any $\phi \in (0, 1]$, we say that a graph $G$ is a \emph{$\phi$-expander} if $\phi(G) \geq \phi$. We say that the partition $X_1, X_2, \dots, X_k$ of the vertex set $V$ is a \emph{$\phi$-expander decomposition} of quality $q$ if
\begin{itemize}
    \item each induced graph $G[X_i]$ is a $\phi$-expander, and
    \item the number of edges not contained in any $G[X_i]$ is at most $q \cdot \phi \cdot m$.
\end{itemize}
Today, we obtain a $\phi$-expander decomposition of quality $q = O(\phi^{-1/2} \cdot \log n)$. In a few lectures, we revisit the problem and obtain quality $q = O(\log^c n)$ for some small constant $c$. In practice, we mostly care about values $\phi \approx 1$.

\paragraph{An Algorithm to Compute Conductance and Expander Decomposition.}
In this lecture, the main focus is \emph{not} to obtain an algorithm to compute conductance but rather only to show that the conductance of a graph can be approximated using the eigenvalues of the "normalized" Laplacian.

However, this proof gives then rise to an algorithm $\textsc{CertifyOrCut}(G, \phi)$ that given a graph $G$ and a parameter $\phi$ either:
\begin{itemize}
    \item \emph{Certifies} that $G$ is a $\phi$-expander, or
    \item Presents a \emph{cut} $S$ such that $\phi(S) \leq \sqrt{2\phi}$.
\end{itemize}
In the graded homework, we ask you to make the procedure $\textsc{CertifyOrCut}(G, \phi)$ explicit, and then to show how to use it to compute a $\phi$-expander decomposition.

\section{A Lower Bound for Conductance via Eigenvalues}

\paragraph{An Alternative Characterization of Conductance.} Let us now take a closer look at the definition of conductance and observe that if a set $S$ has $\vol(S) \leq \vol(V)/2$ then
\[
\phi(S) = \frac{|E(S, V \setminus S)|}{ \min\{\vol(S), \vol(V \setminus S)\} } = \frac{|E(S, V \setminus S)|}{ \vol(S) } = \frac{\vecone_S^\trp \LL \vecone_S}{\vecone_S^\trp \DD \vecone_S}.
\]
To see this, observe that we can rewrite the numerator above using the Laplacian of $G$ as
\[
|E(S, V \setminus S)| = \sum_{(u,v)\in E} (\vecone_S(u) - \vecone_S(v))^2 =  \vecone_S^\trp \LL \vecone_S
\]
where $\vecone_S$ is the characteristic vector of $S$. Further, we can rewrite the denominator as
\[
\vol(S) = \vecone_S^\trp \dd = \vecone_S^\trp \DD \vecone_S
\]
where $\DD = \diag(\dd)$ is the degree-matrix. We can now alternatively define the graph conductance of $G$ by
\begin{equation}\label{eq:conductanceAltDef}
\phi(G) = \min_{\stackrel{\emptyset \subset S \subset V,} {\vol(S) \leq \vol(V)/2}} \frac{\vecone_S^\trp \LL \vecone_S}{\vecone_S^\trp \DD \vecone_S}
\end{equation}
where we use that $\phi(S) = \phi(V \setminus S)$ such that the objective value is unchanged as long as for each set $\emptyset \subset S \subset V$ either $S$ or $V \setminus S$ is in the set that we minimize over.

\paragraph{The Normalized Laplacian.} Let us next define the \emph{normalized} Laplacian \[
\NN = \DD^{-1/2} \LL \DD^{-1/2} = \II -  \DD^{-1/2} \AA \DD^{-1/2}.
\]
To learn a bit about this new matrix, let us first look at the first eigenvalue where we use the test vector $y = \DD^{1/2}\vecone$, to get by Courant-Fischer (see \Cref{thm:courant-fischer-eigvec}) that
\begin{align}\label{eq:plugInNormalizedVector}
    \lambda_1(\NN) =
    \min_{
      \substack{ \xx \neq \veczero}
    }
    \frac{\xx^\trp \NN\xx}{\xx^\trp\xx}  \leq \frac{\yy^\trp\DD^{-1/2} \LL \DD^{-1/2} \yy}{\yy^\trp\yy} = \frac{\vecone^\trp \LL \vecone}{\yy^\trp\yy} = 0
\end{align}
because $\DD^{-1/2} \DD^{1/2} = I$ and $\LL \vecone = 0$ (for the former we use the assumption that $G$ is connected). Since $\NN$ is PSD (as you will show in the exercises), we also know $\lambda_1(\NN) \geq 0$, so $\lambda_1(\NN) = 0$.

Let us use Courant-Fischer again to reason a bit about the second eigenvalue of $\NN$:
\begin{align}\label{eq:courantFisherSecondEig}
    \lambda_2(\NN) =
    \min_{
      \substack{ \xx \perp \DD^{1/2}\vecone \\ \xx \neq \veczero}
    }
    \frac{\xx^\trp \NN\xx}{\xx^\trp\xx} =
    \min_{
      \substack{ \zz \perp \dd \\ \zz \neq \veczero}
    }
    \frac{\zz^\trp \DD^{1/2}  \NN \DD^{1/2} \zz}{\zz^\trp \DD^{1/2}\DD^{1/2}\zz} = \min_{
      \substack{ \zz \perp \dd \\ \zz \neq \veczero}
    } \frac{\zz^\trp \LL \zz}{\zz^\trp \DD \zz}.
\end{align}

\paragraph{Relating Conductance to the Normalized Laplacian.} At this point, it might become clearer why $\NN$ is a natural matrix to consider when arguing about conductance: if we could argue that for every $\emptyset \subset S \subset V, \vol(S) \leq \vol(V)/2$, we have $\vecone_S \perp \dd$, then it would be easy to see that taking the second eigenvalue of $\NN$ in equation \ref{eq:courantFisherSecondEig} is a relaxation of the minimization problem \ref{eq:conductanceAltDef} defining $\phi(G)$.

While this is clearly not true, we can still argue along these lines.

\begin{theorem}[Cheeger's Inequality, Lower Bound]\label{thm:cheegerInequLowerBound}
We have $\frac{\lambda_2(\NN)}{2} \leq \phi(G)$.
\end{theorem}
\begin{proof}
Instead of using $\vecone_S$ directly, we shift $\vecone_S$ by $\vecone$ such that it is orthogonal to $\dd$: we define $\zz_S = \vecone_S - \alpha \vecone$ where $\alpha$ is the scalar that solves
\begin{align*}
       0 &= \dd^\trp \zz_S \\
       \iff 0 &= \dd^\trp(\vecone_S - \alpha \vecone)\\
       \iff 0 &= \dd^\trp \vecone_S - \alpha  \dd^\trp\vecone \\
       \iff \alpha &= \frac{\dd^\trp \vecone_S}{\dd^\trp\vecone} = \frac{\vol(S)}{\vol(V)}.
\end{align*}

To conclude the proof, it remains to argue that $\frac{\vecone_S^\trp \LL \vecone_S}{\vecone_S^\trp\DD\vecone_S} \geq \frac{1}{2} \cdot \frac{\zz_S^\trp \LL \zz_S}{\zz_S^\trp\DD\zz_S}$:
\begin{itemize}
    \item Numerator: since $\vecone^\trp \LL \vecone = 0$, we have that $\vecone_S^\trp \LL \vecone_S = \zz_S^\trp \LL \zz_S$.
    \item Denominator: observe by straight-forward calculations that \begin{align*}
        \zz_S^\trp\DD\zz_S &= \vol(S) \cdot (1-\alpha)^2 + \vol(V \setminus S) \cdot (-\alpha)^2\\
        &= \vol(S) - 2\vol(S) \cdot \alpha + \vol(V) \cdot \alpha^2\\
        & = \vol(S) -   \frac{\vol(S)^2}{\vol(V)} \\
        & = \vol(S) - \vol(S) \cdot  \frac{\vol(S)}{\vol(V)}\\
        & \geq \frac{1}{2}\vol(S) = \frac{1}{2}\vecone_S^\trp\DD\vecone_S
    \end{align*}
    where we use the assumption that $\vol(S) \leq \vol(V)/2$.
\end{itemize}
\end{proof}

\section{An Upper Bound for Conductance via Eigenvalues}

Slightly more surprisingly, we can also show that the second eigenvalue $\lambda_2(\NN)$ can be used to upper bound the conductance.

\begin{theorem}[Cheeger's Inequality, Upper Bound]\label{thm:cheegerInequUpperBound}
We have $\phi(G) \leq \sqrt{2 \cdot \lambda_2(\NN)}$.
\end{theorem}
\begin{proof}
To prove the theorem, we want to show that for \emph{any} $\zz \perp \dd$, we can find a set $\emptyset \subset S \subset V$, such that
\begin{equation}\label{eq:findTightCutCheeger}
\frac{\vecone_S^\trp \LL \vecone_S}{\vecone_S^\trp \DD \vecone_S}  \leq \sqrt{2 \cdot \frac{\zz^\trp \LL \zz}{\zz^\trp \DD \zz}}.
\end{equation}

As a first step, we would like to change $\zz$ slightly to make it more convenient to work with:
\begin{itemize}
    \item we \emph{renumber} the vertices in $V$ such that we have
    \[
    \zz(1) \leq \zz(2) \leq \dots \leq \zz(n).
    \]
    \item we \emph{center} $\zz$, that is we let $\zz_c = \zz - \alpha \vecone$ where $\alpha$ is chosen such that \begin{align*}
        \sum_{\zz_c(i) < 0} \dd(i) < \vol(V)/2 \text{ and }
        \sum_{\zz_c(i) \leq 0} \dd(i) \geq \vol(V)/2
    \end{align*}
    i.e. $\sum_{\zz_c(i) > 0} \dd(i) \leq \vol(V)/2$.
    \item we \emph{scale}, let $\zz_{sc} = \beta \zz_c$ for some scalar $\beta$ such that $\zz_{sc}(1)^2 + \zz_{sc}(n)^2 = 1$.
\end{itemize}
In the exercises, you will show that changing $\zz$ to $\zz_{sc}$ can only make the ratio we are interested in smaller, i.e. that
$\frac{\zz^\trp \LL \zz}{\zz^\trp \DD \zz} \geq \frac{\zz_{sc}^\trp \LL \zz_{sc}}{\zz_{sc}^\trp \DD \zz_{sc}}$. Thus, if we can show that equation \ref{eq:findTightCutCheeger} holds for $\zz_{sc}$ in place of $\zz$, then it also follows for $\zz$ itself.

We now arrive at the main idea of the proof: we define the set $S_{\tau} = \{ i \in V \;|\; \zz_{sc}(i) < \tau\}$ for some random variable $\tau$ with distribution with probability density function
\begin{align}\label{eq:cheegerDefineDensityFun}
    p(t) = \begin{cases}
        2 \cdot |t| &t \in [\zz_{sc}(1), \zz_{sc}(n)],
        \\
        0 &\text{otherwise}.
        \end{cases}
\end{align}
So, we have probability $\mathbb{P}[a < \tau < b] = \int_{t=a}^b p(t)\; dt$.

Since the volume incident to $S_{\tau}$ might be quite large, let us define $S$ for convenience by
\[
S = \begin{cases}
        S_{\tau} & \vol(S_{\tau}) < \vol(V)/2,
        \\
        V \setminus S_{\tau} &\text{otherwise}.
        \end{cases}
\]

\begin{claim}\label{clm:cheegersInequInExpectation}
We have  $\frac{\mathbb{E}_{\tau}\left[\vecone_S^\trp \LL \vecone_S\right]}{\mathbb{E}_{\tau}\left[{\vecone_S^\trp \DD \vecone_S}  \right]} \leq \sqrt{2 \cdot \frac{\zz_{sc}^\trp \LL \zz_{sc}}{\zz_{sc}^\trp \DD \zz_{sc}}}$.
\end{claim}
\begin{proof}
Recall $\vecone_S^\trp \LL \vecone_S = E(S_{\tau}, V \setminus S_{\tau})$, and by choice of $\tau$, we have for any edge $e = \{i,j\} \in E$ where $\zz_{sc}(i) \leq \zz_{sc}(j)$,
\begin{align*}
    \mathbb{P}_{\tau}[e \in E(S_{\tau}, V \setminus S_{\tau})] &= \mathbb{P}_{\tau}[\zz_{sc}(i) < \tau \leq \zz_{sc}(j)] \\ &= \int_{t=i}^j 2|t|\; dt = \sgn{j} \cdot \zz_{sc}(j)^2 - \sgn{i} \cdot \zz_{sc}(i)^2.
\end{align*}
Distinguishing by cases, we get
\[
\sgn{j} \cdot \zz_{sc}(j)^2 - \sgn{i} \cdot \zz_{sc}(i)^2 = \begin{cases}
    |\zz_{sc}(i)^2 - \zz_{sc}(j)^2| & \sgn{i} = \sgn{j},
    \\
    \zz_{sc}(i)^2 + \zz_{sc}(j)^2 &\text{otherwise}.
    \end{cases}
\]
We can further upper bound either case by $ |\zz_{sc}(i) - \zz_{sc}(j)| \cdot (|\zz_{sc}(i)| + |\zz_{sc}(j)|)$ (we leave this as an exercise).

Using our new upper bound, we can sum over all edges $e \in E$ to conclude that
\begin{align*}
   \mathbb{E}_{\tau}[|E(S_{\tau}, V \setminus S_{\tau})|] &\leq \sum_{i \sim j} |\zz_{sc}(i) - \zz_{sc}(j)| \cdot (|\zz_{sc}(i)| + |\zz_{sc}(j)|) \\
   &\leq \sqrt{\sum_{i \sim j} (\zz_{sc}(i) - \zz_{sc}(j))^2 \cdot \sum_{i \sim j} (|\zz_{sc}(i)| + |\zz_{sc}(j)|)^2 }
\end{align*}
where the last line follows from $\langle \xx, \yy \rangle^2 \leq \langle \xx, \xx \rangle \cdot \langle \yy, \yy \rangle$ (i.e. Cauchy-Schwarz).\\

The first sum should look familiar by now: it is simply the Quadratic Laplacian Form $\sum_{i \sim j} (\zz_{sc}(i) - \zz_{sc}(j))^2 = \zz_{sc}^\trp \LL \zz_{sc}$. \\

It is not hard to reason about the second term either
\[
\sum_{i \sim j} (|\zz_{sc}(i)| + |\zz_{sc}(j)|)^2 \leq 2\sum_{i \sim j} \zz_{sc}(i)^2 + \zz_{sc}(j)^2 = 2\sum_{i \in V} \dd(i) \zz_{sc}(i)^2 = 2 \zz_{sc}^{\trp} \DD \zz_{sc}.
\]

Putting everything together, we obtain
\begin{align}\label{eq:cheegerInequAlmostDone}
   \mathbb{E}_{\tau}[|E(S_{\tau}, V \setminus S_{\tau})|] \leq \sqrt{\zz_{sc}^\trp \LL \zz_{sc} \cdot 2 \zz_{sc}^{\trp} \DD \zz_{sc}} = \sqrt{2 \cdot \frac{\zz_{sc}^\trp \LL \zz_{sc}}{\zz_{sc}^{\trp} \DD \zz_{sc}} }\; \cdot \; \zz_{sc}^{\trp} \DD \zz_{sc}
\end{align}

While this almost looks like what we want, we still have to argue that $\zz_{sc}^\trp \DD \zz_{sc} = \mathbb{E}_{\tau}[\vecone_S^\trp \DD \vecone_S]$ to finish the proof.

To this end, when unrolling the expectation, we use a simple trick that splits by cases:
\begin{align*}
    \mathbb{E}_{\tau}[\vecone_S^\trp \DD \vecone_{S}] &=\sum_{i \in V} \dd(i) \cdot \mathbb{P}[i \in S] \\
    &=\sum_{i \in V, \zz_{sc}(i) < 0} \dd(i) \cdot \mathbb{P}[i \in S \land S=S_{\tau}] + \sum_{i \in V, \zz_{sc}(i) \geq 0} \dd(i) \cdot \mathbb{P}[i \in S \land S \neq S_{\tau}]\\
     &=\sum_{i \in V, \zz_{sc}(i) < 0} \dd(i) \cdot \mathbb{P}[\zz_{sc}(i) < \tau \land \tau < 0] + \sum_{i \in V, \zz_{sc}(i) \geq 0} \dd(i) \cdot \mathbb{P}[\zz_{sc}(i) \geq \tau \land \tau \geq 0]
\end{align*}
where we use the centering of $\zz_{sc}$ the definition of $S$ and that the event $\{i \in S \land S=S_{\tau}\}$ can be rewritten as the event $\{i < \tau \land \tau < 0\}$ (the other case is analogous).

Let $i$ be a vertex with $\zz_{sc}(i) < 0$, then the probability $\mathbb{P}[i \in S \land S=S_{\tau}]$ is exactly $\zz_{sc}(i)^2$ by choice of the density function of $\tau$ (again the case for $i$ with $\zz_{sc}(i)$ non-negative is analgous). Thus, summing over all vertices, we obtain
\begin{align*}
    \mathbb{E}_{\tau}[\vecone_S^\trp \DD \vecone_{S}]
     &=\sum_{i \in V, \zz_{sc}(i) < 0} \dd(i) \cdot \mathbb{P}[\zz_{sc}(i) < \tau \land \tau < 0] + \sum_{i \in V, \zz_{sc}(i) \geq 0} \mathbb{P}[\zz_{sc}(i) \geq \tau \land \tau \geq 0]\\
     & = \sum_{i \in V } \dd(i) \cdot \zz_{sc}(i)^2 = \zz_{sc}^\trp \DD \zz_{sc}.
\end{align*}
Therefore, we can plug in our result directly into Equation
\ref{eq:cheegerInequAlmostDone} and the proof is completed by dividing both sides by $\mathbb{E}_{\tau}[\vecone_S^\trp \DD \vecone_{S}]$.
\end{proof}

While \Cref{clm:cheegersInequInExpectation} only ensures our claim in expectation, this is already sufficient to conclude that there exists some set $S$ that satisfies the same guarantees deterministically, as you will prove in Problem Set 4. This is often called the \emph{probabilistic method of expectation} and can be seen from the definition of expectation. We have thus proven the upper bound of Cheeger's inequality.
\end{proof}

\section{Conclusion}

Today, we have introduced the concepts of conductance and formalized expanders and expander decompositions. These are crucial concepts that you will encounter often in literature and also again in this course. They are a key tool in many recent breakthroughs in Theoretical Computer Science.

In the second part of the lecture (the main part), we discussed Cheeger's inequality which allows to relate the second eigenvalue of the normalized Laplacian to a graphs conductance. We summarize the full statement here.

\begin{theorem}[Cheeger's Inequality]\label{thm:cheegerInequFull}
We have $\frac{\lambda_2(\NN)}{2} \leq \phi(G) \leq \sqrt{2 \cdot \lambda_2(\NN)}$.
\end{theorem}

We point out that this Theorem is tight as you will show in the exercises. The proof for Cheeger's inequality is probably the most advanced proof, we have seen so far in the course. The many tricks that make the proof work might sometimes seem a bit magical but it is important to remember that they are a result of many people polishing this proof over and over. The proof techniques used are extremely useful and can be re-used in various contexts. We therefore strongly encourage you to really understand the proof yourself!


\section{Support Vector Machine (SVM)}

\iffalse
    % Here: Data is assumed linearly separable
    % Below: additionally (orange) soft margin when not
    % ===
    \textit{Primal}: \: $\min\limits_{\bm w, w_0} \frac12 \norm{\bm w}^2$ \: s.t. \: $y_i(\bm w^\top \bm x_i + w_0) \geq 1$\\
    \quad $\hookrightarrow$ intractable if $\varphi(\bm x_i)$ instead of $\bm x_i$

    \textit{Dual}: \: $\max\limits_{\bm\alpha} \sum_i \alpha_i - \frac12 \sum_{i,j} \alpha_i \alpha_j y_i y_j \bm x_i^\top \bm x_j$\\
    \qquad\qquad s.t. \: $\alpha_i \geq 0$; \: $\sum_i \alpha_i y_i = 0$\\
    \quad $\hookrightarrow$ solve $\alpha$ via quadratic optimisation

    Optimal hyperplane: \: $\bm w^\ast = \sum_i \alpha_i^\ast y_i \bm x_i$\\
    \quad $\hookrightarrow$ $\alpha_i^\ast \neq 0$ only for \textit{support vectors}
\fi

% ===
\textit{Primal} \textcolor{OrangeRed}{(soft margin)}: \: $\min\limits_{\bm w, w_0 \color{OrangeRed}, \xi} \frac12 \norm{\bm w}^2 \color{OrangeRed} + C\sum_i \xi_i$\\
\qquad s.t. \: $y_i(\bm w^\top \bm x_i + w_0) \geq 1 \color{OrangeRed} - \xi_i$ \: and \: $\color{OrangeRed} \xi_i \geq 0$\\
\quad $\hookrightarrow$ intractable if $\varphi(\bm x_i)$ instead of $\bm x_i$\\
\quad $\hookrightarrow$ $\xi_i = 0$ means $\bm x_i$ was \underline{not} neglected

\textit{Dual}: \: $\max\limits_{\bm\alpha} \sum_i \alpha_i - \frac12 \sum_{(i,j)} \alpha_i \alpha_j y_i y_j \bm x_i^\top \bm x_j$\\
\qquad\qquad s.t. \: $0 \leq \alpha_i \color{OrangeRed} \leq C$; \: $\sum_i \alpha_i y_i = 0$\\
\quad $\hookrightarrow$ solve $\alpha$ via quadratic optimisation

Optimal hyperplane: \: $\bm w^\ast = \sum_i \alpha_i^\ast y_i \bm x_i$\\
\quad $\hookrightarrow$ $\alpha_i^\ast \neq 0$ only for \textit{support vectors}

Optimal slack: \: $\xi_i^\ast = \max(0, 1 - y_i(\bm w^{\ast\top} \!\bm x_i {+} w_0^\ast))$

% ===
\subsection{Structural SVMs}

$\min\limits_{\bm w \color{OrangeRed}, \xi} \frac12 \norm{\bm w}^2 \color{OrangeRed} + \frac{C}{n} \sum\limits_{i\leq n} \xi_i$
\quad s.t. $\color{OrangeRed} \xi_i \geq 0$ and $\forall y' \neq y_i :$\\
\quad $\bm w^\top \Psi(x_i, y_i) \geq \Delta(y_i, y') + \underbracket[.7pt][.7pt]{\bm w^\top \Psi(x_i, y')}_{\hookrightarrow \text{ mislabelings}} \color{OrangeRed} - \xi_i \color{Green} - \epsilon$\\

% ---
%\columnbreak
% ---

$\Psi$ : \textit{joint-feature map}; \:
$\Delta$ : loss / class dissimilarity func.; \:
$\bm w^\top \Psi(x,y)$ : compatibility score btw. $x$ and $y$; \:
$\color{Green} \epsilon$ : tolerance / universal slack variable.

\textbf{Prediction:} \: $c(x) = \argmax_y \bm w^\top \Psi(x,y)$
\vspace{-3pt}

\textit{Note:} For optimal $\bm w^\ast$, $\xi^\ast$, $\textrm{emp. risk}(\bm w^\ast) \leq \frac1n \sum_i \xi_i^\ast$

\textbf{Training:} Start without any constraints.\\
In each iteration, add for each $(x_i,y_i)$ the constraint with $y' {\neq} y_i$ that is the ``most violated'' and solve again with quadr. optimisation.

% ===


\section{Data Mining}
\label{sec:dataMining}

Data mining is the computational process of discovering patterns in large data sets (``big data'') involving methods at the intersection of artificial intelligence, machine learning, statistics, and database systems. The overall goal of the data mining process is to extract information from a data set and transform it into an understandable structure for further use.
\\RAVEN has support of several different data mining algorithms,
such as:
\begin{enumerate}
  \item \textit{Hierarchical methodologies}
  \item \textit{K-Means}
  \item \textit{Mean-Shift}, etc.
\end{enumerate}
In this section only few algorithms will be explained

\subsection{Clustering}
\label{clustering}
A loose definition of clustering is the process of organizing objects into groups whose members are, in some way, similar.
Therefore, a cluster is a collection of objects that are similar to each other and are dissimilar to the objects belonging to other clusters~\cite{SurveyClustering,MandelliClusteringRESS}.

The similarity criterion is distance. Two or more objects belong to the same cluster if they are ``close'' according to a specified distance. The approach of using distance metrics to clustering is called distance-based clustering and is used in this work.

The notion of distance implies that the data points lay in a metric space \cite{Mendelson75introduction}:

    \begin{mydef}[Metric Space]
    A metric space is a space X provided with a function \emph{d}:
    \begin{math}
    f: X\times X\rightarrow \mathbb{R}
    \end{math}
    satisfying the following properties \begin{math}\forall \vec{x},\vec{y} \in X \end{math} :

    \begin{itemize}
      \item \begin{math} d(\vec{x},\vec{y}) \geqslant 0 \end{math}
      \item \begin{math} d(\vec{x},\vec{y}) = d(\vec{y},\vec{x}) \end{math}
      \item \begin{math} d(\vec{x},\vec{y}) \leqslant d(\vec{x},\vec{z}) + d(\vec{z},\vec{y}) \end{math}
    \end{itemize}

    \end{mydef}

The function \begin{math} d(\vec{x},\vec{y}) \end{math} is usually called the distance function. In a 2-dimensional Euclidean space ($\mathbb{R}^{2}$), the distance between points can be calculated using the Pythagorean theorem which is the direct application of the Euclidean distance and is a special case of the most general Minkowski distance \begin{math}\ d_{2}(\vec{x},\vec{y}) = \sqrt{(x_{1}-y_{1})^{2}+(x_{2}-y_{2})^{2}} \end{math} between two points $\vec{x}=(x_{1},x_{2})$ and $\vec{y}=(y_{1},y_{2})$ in $\mathbb{R}^{2}$.

In the literature~\cite{Mendelson75introduction}, it is possible to find several types of distances other than the Euclidean and the Minkowski distance as shown in Table~\ref{table:tableDist}. The approach of using distance metrics is called distance-based clustering and will be used in this dissertation.

    \begin{table}[ht]
    \caption {\small Summary of the commonly used measures \cite{Mendelson75introduction}.}
    \centering
    \begin{tabular}{c c }
    \hline\hline
    Measure & Form \\ [0.5ex]
    \hline
    \hline
    Minkowski distance & \begin{math}\ d_{n}(\vec{x},\vec{y}) = (\displaystyle \sum_{k=1}^\delta |x_{k}-y_{k}|^{n})^{\frac{1}{n}} \end{math} \\
    Euclidean distance & \begin{math}\ d_{2}(\vec{x},\vec{y}) = (\displaystyle \sum_{k=1}^\delta |x_{k}-y_{k}|^{2})^{\frac{1}{2}} \end{math} \\
    Taxicab distance & \begin{math}\ d_{1}(\vec{x},\vec{y}) = \displaystyle \sum_{k=1}^\delta |x_{k}-y_{k}| \end{math} \\
    Supremum distance & \begin{math}\ d_{0}(\vec{x},\vec{y}) = \displaystyle max_{k} |x_{k}-y_{k}| \end{math} \\
    Mahalanobis distance & \begin{math}\ d_{M}(\vec{x},\vec{y}) = (\vec{x}-\vec{y})^{T} S^{-1} (\vec{x}-\vec{y}) \end{math} \\
    \hline
    \end{tabular}
    \label{table:tableDist}
    \end{table}

From a mathematical viewpoint, the concept of clustering~\cite{SurveyClustering} aims to find a partition $\mathbf{C}=\{C_{1},\ldots,C_{l},\ldots,{C_{L}}\}$
of the set of $I$ scenarios
    $\mathbf{X} = \{\vec{x_{1}},\ldots,\vec{x_{i}},\ldots,\vec{x_{I}}\}$
where each scenario $\vec{x_{i}}$ is represented as a $\delta$-dimensional vector.
Each $C_{l}$ $(l=1,\ldots,L)$ is called a cluster. The partition
    $ \mathbf{C} $ of $ \mathbf{X} $
is given as follows\footnote{In most clustering algorithms each scenario belongs to only one cluster. However this is not always the case. In fuzzy clustering methodologies~\cite{ZioMaio} a scenario may be allowed to belong to more than one cluster with a degree of membership
\begin{math} u_{i,j}\in [0,1] \end{math} which represents the member coefficient of the $j$ scenario for the $i^{th}$ cluster and satisfies the following properties:

$ \sum_{i=1}^{K}u_{i,j}=1,  \text{ and }  \sum_{j=1}^{N}u_{i,j}<N, \forall j $}:

    \begin{equation}\label{eq: ClassRequier}
        \begin{cases} \mathbf{C}_{l}\neq\varnothing, l=1,\ldots,L \\
                        \\
                     \bigcup_{l=1}^{L}\mathbf{C}_{l}= \mathbf{X} \\
        \end{cases}
    \end{equation}

\subsection{Hierarchical Methodologies}
\label{Hierarchical}

These methodologies organize the data set into a hierarchical structure according to a proximity matrix. Each element $d(i,j)$ of this matrix contains the distance between the the $i^{th}$ and the $j^{th}$ cluster center. The final results of this technique is a tree commonly called a dendrogram. This kind of representation has the advantages of providing a very informative description and visualization of the data structure even for high values of dimensionality.

The procedure to determine the dendrogram for a data set of $I$ points in an $\delta$-dimensional space is the following:

\begin{enumerate}
  \item Start the analysis with a set of $I$ clusters (i.e., each point is considered as a cluster).
  \item Determine the proximity matrix $M$ (dimension: $I\times I$): $M(i,j)= d(\vec{x_{i}},\vec{x_{j}})$ where $\vec{x_{i}}$ and $\vec{x_{j}}$ are the position of the $i^{th}$ and the $j^{th}$ cluster.
  \item For each point $p$ find the closest neighbor $q$  from the proximity matrix $M$
  \item Combine the points $p$ and $q$
  \item Repeat Steps 2, 3 and 4 until all the points of the data set are in the same cluster
\end{enumerate}

The advantage of this kind of algorithm is the nice visualization of the results that show the underlying structure of the data set. However, the computational complexity for most of the hierarchical algorithm is of the order of $\mathcal{O}(I^{2})$ (where \emph{I} is the number of points in the data set).

\subsection{\emph{K}-Means}
\label{KMeans}

\emph{K}-Means clustering algorithms belong to the more general family of Squared Error algorithms. The goal is to partition $I$ data points $\vec{x_{i}}$ $(i=1,\ldots,I)$ into \emph{K} clusters in which each data point maps to the cluster with the nearest mean. The stopping criterion is to find the global minimum of the error squared function $\chi$ defined as:

\begin{equation}
    \chi = \displaystyle \sum_{i=1}^K \displaystyle \sum_{x_{j}\in C_{i}} |\vec{x_{j}}-\vec{\mu_{i}}|^{2}
\end{equation}

where $\vec{\mu_{i}}$ is the centroid (i.e., the center) of the cluster $C_{i}$.

The procedure to determine the centroids $\vec{\mu_{i}}$ of \emph{K} clusters (${C_{1},\ldots,C_{K}}$) is the following:

\begin{enumerate}
  \item Start with a set of $K$ random centroids distributed in the state space
  \item Assign each pattern to the the closest centroid
  \item Determine the new $K$ centroids according to the point-centroid membership

    \begin{equation}
        \mu_{i} = \displaystyle \frac{1}{N_{i}} \sum_{\vec{x_{j}}\in C_{i}} \vec{x_{j}}
    \end{equation}

    where $N_{i}$ corresponds to the number of of data points in the $i^{th}$ cluster.

  \item Repeat Steps 2 and 3 until convergence is met (i.e., until a minima of the $\chi$ function is reached)
\end{enumerate}

\emph{K}-Means algorithm is one of the most popular and used methodologies also due to the fact that is very straightforward to implement and the computational time is directly proportional to the cardinality of data points (i.e., $\mathcal{O}(I)$ where \emph{I} is the number of data points). The main disadvantage is that the algorithm is sensitive to the choice of the initial partition and may converge to a local minimum of the error squared function~\cite{JainAlgor88}. Another disadvantage of this algorithm is that is only able to identify clusters having spherical or ellipsoidal geometry. Thus, \emph{K}-Means is not able to identify clusters of points having arbitrary shapes. Moreover, the number of cluster \emph{K} to be obtained is specified by the user prior the clustering process.

%\subsection{Fuzzy C-Means}
%\label{FuzzyCMeansn}
%
%Fuzzy \emph{C}-Means clustering is a clustering methodology that is based on fuzzy sets and, hence, it allows a data point to belong to more that one cluster~\cite{FuzzyBezdek,DunnFuzzy}. Similar to the \emph{K}-Means clustering, the objective is to find a partition of $C$ fuzzy centers to minimize the function $J$ defined as following:
%
%\begin{equation}
%    J = \displaystyle \sum_{i=1}^I \displaystyle \sum_{j=1}^C u_{ij}^{m}|\vec{x_{i}}-\vec{\mu_{j}}|^{2}
%\end{equation}
%
%where:
% \begin{itemize}
%   \item $u_{ij}^{m}\in[0,1]$ is the membership coefficient of the data point $\vec{x_{i}}$ for the $j^{th}$ cluster having centroid $\vec{\mu_{j}}$,
%   \item $m\in[0,\infty)$ is the fuzzification parameter (usually set to $m=2$), and,
%   \item $\mu_{j}$ is the centroid of the $j^{th}$ cluster center
% \end{itemize}
%
%The procedure to determine the centroids (or, equivalently, cluster centers)
%$ \vec{\mu_{j}}$ $(j=1,\ldots,C)$ of $C$ clusters is the following:
%
%\begin{enumerate}
%  \item Initialize the $U=[u_{ij}^{m}]$ matrix
%  \item Calculate the set of $C$ centroids  as following:
%
%          \begin{equation}
%            \mu_{j}=\frac{\sum_{i=1}^N u_{ij}^{m}x_{i}}{\sum_{i=1}^N u_{ij}^{m}}
%          \end{equation}
%
%  \item Update the matrix $U=[u_{ij}^{m}]$ as following:
%
%          \begin{equation}
%            u_{ij}^{m}=\frac{1}{\sum_{k=1}^C (\frac{|x_{i}-\mu_{j}|}{|x_{i}-\mu_{k}|})^{\frac{2}{m-1}}}
%          \end{equation}
%
%  \item Repeat Steps 2 and 3 until convergence, i.e. if \begin{math} |U^{(K+1)}-U^{(K)}|<\epsilon \end{math}
%\end{enumerate}
%
%Fuzzy \emph{C}-Means clustering is very similar to the \emph{K}-Means. As seen for the \emph{K}-Means, Fuzzy \emph{C}-Means can also converge to a local minima of the convergence criterion function~\cite{FuzzyBezdek}.
%Like \emph{K}-Means, it is not able to identify cluster of points having arbitrary shapes but only clusters having ellipsoidal or spherical geometry and the number of clusters \emph{C} to be obtained is specified by the user prior the clustering process.
%Fuzzy \emph{C}-Means algorithms can be useful when the boundaries among clusters are ambiguous and not well defined.

\subsection{Mean-Shift}
\label{Mean-Shift}

The Mean-Shift algorithm~\cite{EstimationGradient} is a non-parametric iterative procedure that can be used to assign each point to one cluster center through a set of local averaging operations~\cite{EstimationGradient}. The local averaging operations provide empirical cluster centers within the locality and define the vector which denotes the direction of increase for the underlying unknown density function.

The underlying idea is to treat each point $\vec{x_{i}}$ $(i=1,\ldots, I)$ of the dataset as an empirical probability distribution function using  kernel $K(\vec{x}): \mathbb{R}^{M\cdot K}\rightarrow \mathbb{R}$. This multivariate kernel density resides in a multidimensional space where regions with high data density (i.e., modes) correspond to local maxima of the density estimate $f_{I}(\vec{x})$~\cite{CacoullosEstimation}  defined by:
\begin{equation}
    f_{I}(\vec{x})=\frac{1}{Ih^{d}}\sum_{i=1}^{I} K\left(\frac{\vec{x}-\vec{x_{i}}}{h}\right),
    \label{eq:density:estimate}
\end{equation}
where $\vec{x}\in \mathbb{R}^{M\cdot K}$ and $h$ is often referred as the bandwidth associated with the kernel.

The kernel in Equation ~\ref{eq:density:estimate} serves as a weighting function~\cite{CacoullosEstimation} associated with each data point and is expressed as:
\begin{equation}
    K(\vec{x})=c_{k} k(\norm{\vec{x}}^{2})
\end{equation}
where $k(x):[0,\infty]\rightarrow \mathbb{R}$ is referred as the \emph{kernel profile} and $c_{k}$ is a normalization constant. The profile satisfies the following properties:
\begin{itemize}
  \item $k(x)$ is non negative
  \item $k(x)$ is non increasing (i.e., $k(a)\geq k(b)$ if $a<b$)
  \item $k(x)$ is piecewise continuous and $\int_0^\infty \! k(x) \, dx < \infty$
\end{itemize}

In order to estimate the data points with highest probability from an initial estimate (i.e., the modes of $f_{I}(\vec{x})$), consider the gradient of the density function $\nabla_{x} f_{I}(\vec{x})=0$ ~\cite{EstimationGradient} where
\begin{eqnarray}
\label{gradient}
    \nabla_{x} f_{I}(\vec{x}) & = & \frac{2 c_{k}}{I h^{d+2}} \sum_{i=1}^{I}(\vec{x}-\vec{x_{i}})k'\left(\norm{\frac{\vec{x}-\vec{x_{i}}}{h}}^{2}\right) \nonumber \\
    & = & \underbrace{\frac{2 c_{k}}{I h^{d+2}} \left(\sum_{i=1}^{I} g\left(\norm{\frac{\vec{x}-\vec{x_{i}}}{h}}^{2}\right)\right)}_{A}
    \underbrace{\left(\frac{\sum_{i=1}^{I}
    \vec{x} g\left(\norm{\frac{\vec{x}-\vec{x_{i}}}{h}}^{2}\right)}{\sum_{i=1}^{I} g\left(\norm{\frac{\vec{x}-\vec{x_{i}}}{h}}^{2}\right)}-\vec{x}\right)}_{B},
\end{eqnarray}
which points in the direction of the increase in kernel density estimate. The kernel $K(\vec{x})$ is also referred to as the shadow of $G(\vec{x})=c_{g} g(\norm{\vec{x}}^{2})$~\cite{Mode-seekingMedoidshifts} where $c_{g}$, similar to $c_{k}$, is a normalization constant and $g(x)$ is the derivative of $k(x)$ over $x$, i.e., $g(x)=k'(x)$. In the equation above, the first term denoted as $A$ is a scalar proportional to the density estimate computed with the kernel $G(\vec{x})$ and does not provide information regarding where the mode resides. Unlike $A$, the vector quantity $B$, which is the second term in the equation above, is difference between the weighted mean
\begin{equation}
	m(\vec{x})=\frac{\sum_{i=1}^{I}\vec{x} g(\norm{\frac{\vec{x}-\vec{x_{i}}}{h}}^{2})}{\sum_{i=1}^{I} g(\norm{\frac{\vec{x}-\vec{x_{i}}}{h}}^{2})}.
\end{equation}

and the initial estimate $\vec{x}$. This term points in the direction of local increase in density using kernel $G(\vec{x})$, hence provides a means to find the mode of the density. Note that all points used to compute a particular mode are considered to reside in the same cluster.

Since each each data point $\vec{x_{i}}$ (or scenario) is considered as an empirical probability distribution function, this consideration allows to include in the scenario clustering analysis also the possible uncertainty associated with each scenario.

\subsection{DBSCAN }
\label{DBSCAN }

The Density-Based Spatial Clustering of Applications with Noise (DBSCAN) algorithm views clusters as areas of high density of data points. The data points in the low-density areas are seen as noise and border points, which are actually separating the clusters. Clusters found by DBSCAN can be any shape because of this approach.
The main element of the DBSCAN algorithm is the concept of core samples, which are samples that are in areas of high density. Therefore, a cluster is a set of core samples, each close to each other (measured by some distance measure) and a set of non-core samples that are close to a core sample (but are not themselves core samples). There are two parameters to the algorithm: $min_samples$ and $eps$. Higher $min_samples$ or lower $eps$ indicate higher density necessary to form a cluster.
A cluster is a set of core samples, that can be built by recursively by taking a core sample, finding all of its neighbors that are core samples, finding all of their neighbors that are core samples, and so on. A cluster also has a set of non-core samples, which are samples that are neighbors of a core sample in the cluster but are not themselves core samples; these are on the borders of a cluster.
The DBSCAN algorithm finds core samples of high density and expands clusters from them. It is good for data, which contains clusters of similar density.

\subsection{Dimensionality Reduction}
\label{sec:6DimRed.section}

The dimensionality $\delta$ of each data point (i.e., each scenario) is equal to the product of the number of variables (i.e., $M$) chosen to represent each scenario multiplied by the number of times each variable has been sampled.
In order to reduce the computational time due to the high data dimensionality, the use of dimensionality reduction techniques was to reduce the number of variables $M$\footnote{Other possible options are to reduce the number of sample instants $K$ or to observe the local properties of the covariance matrix $S$.}.

The raw data generated by DET methodologies contain the temporal behavior of a vast set of variables (e.g., temperature, pressure). These variables are often heavily correlated and, consequently, the information contained in the set of $M$ variables comprising the full state space can be condensed to a set of $N$ variables where $N <M$. The objective of the dimensionality reduction process is to determine those $N$ variables by finding the correlations among the original $M$ variables\footnote{Note that those $N$ variables are not necessarily a subset of the original $M$ variables but, more likely, a combination of those $M$ variables.}.

Linear algorithms, such as PCA~\cite{JolliffePCA} or multidimensional scaling (MDS)~\cite{MDS}, have the advantage that they are easier to implement but they can only identify linear correlation among variables. On the other hand, methodologies such as Local Linear Embedding~\cite{lle} and ISOMAP~\cite{isomap} are more computationally intensive but they are able to identify non-linear correlations.

Dimensionality reduction is the process of finding a bijective mapping function $\mathfrak{F}$
\begin{equation}\label{eq:dimRed}
    \mathfrak{F}:\mathbb{R}^{D}\mapsto\mathbb{R}^{d} \mbox{ (where $d<D$)}
\end{equation}
which maps the data points from the $D$-dimensional space into a reduced $d$-dimensional space (i.e. embedding on a manifold) in such a way that the distances between each point and its neighbors are preserved. In our applications $D = M+1$, i.e. $M$ state variables plus time $t$.

\subsection{Dimensionality Reduction: Linear Algorithms}
\label{dimRed}
This section describes the two most important algorithms for dimensionality reduction:
\begin{enumerate}
  \item PCA (see Section~\ref{pca}), and,
  \item MDS (see Section~\ref{mds}).
\end{enumerate}

\subsection{Principal Component Analysis (PCA)}
\label{pca}

The main idea behind PCA~\cite{JolliffePCA} is to perform a linear mapping of the data set onto a lower dimensional space such that the variance of the data in the low-dimensional representation is maximized.

This is accomplished by determining the eigenvectors and their corresponding eigenvalues of the data covariance matrix\footnote{Given a data set in form of a vector $Z$, rows correspond to data dimensions ($D$) and columns correspond to data observations ($\Lambda$), the covariance matrix $S$ is determined as: $S=\frac{1}{\Lambda-1}Z'Z$.}
$S$.
The eigenvectors that correspond to the largest eigenvalues (i.e., the principal components) can be used as a set of basis functions. Thus, the original space is reduced to the space spanned by a few eigenvectors.

The algorithm is very straightforward to implement but, on the other hand, PCA is not able to identify non-linear correlations of more complex data sets.

\subsection{Multidimensional Scaling (MDS)}
\label{mds}

Multidimensional scaling~\cite{MDS} is a popular technique used to analyze the properties of data sets. The scope of this methodology is to find a set of dimensions that preserve distances between data points.

This is performed by:
\begin{enumerate}
  \item Creating dissimilarity matrix $D=[d_{ij}]$ where $d_{ij}$ is the distance between two points $x_i$ and $x_j$.
  \item Finding the hyper-plane that preserves the dissimilarity matrix $D$ (i.e., the \emph{nearness} of points)
\end{enumerate}

As in PCA analysis, the algorithm can be easily implemented but it is not able to identify non-linear correlations of more complex data sets.


\section{The Axioms}

\subsection{Terms \& Definitions}

\begin{itemize}

	\item A \textit{term} is well-defined in Set Theory if it \textit{exists} and is \textit{unique}

	\item Term: $ x $ is a \textit{subset} of $ y $ ($ x \subseteq y$) if $ \forall z (z \in x \implies z \in y) $

\end{itemize}

\subsection{Extensionality}
\begin{equation*}
\forall x \forall y (\forall z (z \in x \iff z \in y) \implies x = y)
\end{equation*}

\begin{itemize}

	\item If sets $ x $ and $ y $ have exactly the same members, then they are the same set

\end{itemize}

\subsection{Foundation}
\begin{equation*}
\forall x (\exists y \in x \implies \exists z (z \in x \land \forall w (w \in x \implies w \notin z)))
\end{equation*}

\begin{itemize}

	\item If $ x $ is non-empty, then it has an $ \in $-minimal member (there is some $ z \in x $ such that every member of $ x $ is not a member of $ z $)

\end{itemize}

\subsection{Separation}
\begin{equation*}
\forall x_0 \dots \forall x_n \forall w \exists y \forall z (z \in y \iff z \in w \land \varphi (z, x_0 \dots, x_n))
\end{equation*}

\begin{itemize}

	\item Given any set $ w $,  there is a set $ y $ consisting of  exactly the elements $ z $ from $ w $ such that $ \varphi(z) $. The $ \varphi $s are \textit{separated out} from $ w $ to get $ y $

	\item Why do we need a $ w $? We would like to do na\"ive comprehension: $ \exists y \forall x (x \in y \iff \varphi(x)) $

	\begin{itemize}

		\item \textit{(Russell) Any theory including na\"ive comprehension is inconsistent}

		%	\begin{proof}
		%		Let $ \varphi(x) = x \notin x $ (i.e. $ \varphi $ says $ x $ is not a member of itself).
		%
		%		Fix $ r $ such that for all $ x $,  $ x \in r \iff x \notin x $. But then $ r \in r \iff r \notin r $ which is a contradiction.
		%	\end{proof}

		\item This also means there is no universal set, i.e. a set $ x $ such that $ \forall y (y \in x \iff y = y) $

		%	\begin{proof}
		%		Suppose there is such an $ x $. Then by separation we get $ \RussellSet $.
		%
		%		But for all $ w $, $ w \in \RussellSet \iff w \notin w $. Since $ x $ contains all of the sets, $ \RussellSet $ is Russell's set which is a contradiction.
		%	\end{proof}

	\end{itemize}

\end{itemize}

\subsection{Pairing}
\begin{equation*}
\forall x \forall y \exists z (x \in z \land y \in z)
\end{equation*}

\begin{itemize}

	\item

\end{itemize}

\subsection{Union}
\begin{equation*}
\forall x \exists z \forall y \forall w (y \in w \land w \in x \implies y \in z)
\end{equation*}

\begin{itemize}

	\item

\end{itemize}

\subsection{Powerset}
\begin{equation*}
\forall x \exists y \forall z (z \subseteq x \implies z \in y)
\end{equation*}

\begin{itemize}

	\item

\end{itemize}

\subsection{Replacement}
\begin{equation*}
aa
\end{equation*}

\begin{itemize}

	\item

\end{itemize}

\subsection{Infinity}
\begin{equation*}
\exists x (\exists y y \in x \land \forall z (z \in x \implies \set{z} \in x))
\end{equation*}

\begin{itemize}

	\item

\end{itemize}

\subsection{Choice}
\begin{equation*}
aa
\end{equation*}

\begin{itemize}

	\item

\end{itemize}

\section{Models, Structures \& Sequences}

\section{The Ordinals}

\section{Transfinite Induction and Recursion}

\section{The Cardinals}

\section{Infinite Cardinals \& The Axiom of Choice}


\lab{Invertible Affine Transformations and Linear Systems}{Invertible Affine Transformations and Linear Systems}
\label{lab:ChangeBasis}
\objective{Apply affine transformations to a set of vectors in $\mathbb{R}^2$ and solve linear systems.}

\section*{Linear transformations in $\mathbb{R}^2$}
\subsection*{Dilations}
A \emph{dilation} of the vector space rescales the vectors.
Graphically, a dilation stretches or compresses the space.
A linear transformation is a dilation if and only if its matrix representation is diagonal, so in particular all Type II elementary matrices are dilations.
The matrix $\begin{pmatrix}
1.5 & 0\\
0 & 1.5 \end{pmatrix}$ corresponds to the dilation in Figure \ref{fig:dilation}.
\begin{figure}
\includegraphics[width=\textwidth]{stretch.pdf}
\caption{An example of a dilation.
The top image was stretched by a factor of $1.5$ in all directions, producing the bottom image.}
\label{fig:dilation}
\end{figure}

\begin{problem}\label{prob:dilation}
Write a function that accepts an array of points and an array giving the stretching factors in each direction.
Your function should return the dilated points.
Hint: To check your work, plot the original points and their images under the transformation using the function \li{plotOldNew()} defined below.
\begin{lstlisting}
import numpy as np
from matplotlib import pyplot as plt

def plotOldNew(old, new):
    '''Inputs:
    new -- a (2,n) numpy array containing x-coordinates on the
            first row and y-coordinates on the second row.
    old -- a (2,n) numpy array containing x-coordinates on the first
            row and y-coordinates on the second row.
    '''

    plt.subplot(2, 1, 1)
    plt.scatter(old[0], old[1])
    plt.axis('equal')
    plt.subplot(2, 1, 2)
    plt.scatter(new[0], new[1])
    plt.show()
\end{lstlisting}
\end{problem}

\subsection*{Rotations}
A second type of linear transformation is to rotate vectors around the origin.
A rotation of $\theta$ radians counterclockwise corresponds to the matrix $\begin{pmatrix}
\cos(\theta) & -\sin(\theta) \\
\sin(\theta) & \cos(\theta)
\end{pmatrix}.$
When $\theta = \pi/3$ we get the rotation matrix illustrated in Figure \ref{fig:rotate}.

\begin{figure}
\includegraphics[width=\textwidth]{rotate.pdf}
\caption{An example of a rotation.
The top image was rotated by $\pi/3$, producing the bottom image.}
\label{fig:rotate}
\end{figure}

\begin{problem}
 Write a function that accepts an array of points and the angle of rotation (in radians).
 Your function should return the rotated points.
 Hint: To check your work, plot the original points and their images under the transformation using the function \li{plotOldNew()} defined in Problem \ref{prob:dilation}.
\end{problem}

\subsection*{Shears}

\begin{figure}
\includegraphics[width=\textwidth]{shear.pdf}
\caption{An example of a shear.
The top image was sheared horizontally to produce the bottom image.}
\label{fig:shear}
\end{figure}

A third type of linear transformation is a \emph{shear}, which ``slants'' a set of vectors.
The corresponding matrix is a Type III elementary matrix.
A horizontal shear has the form $\begin{pmatrix}
1 & c \\
0 & 1
\end{pmatrix}$ and a vertical skew has the form $
 \begin{pmatrix}
1 & 0 \\
c & 1
\end{pmatrix}
$.
Notice that horizontal skews fix the $y$-coordinate of a vector while vertical skews fix the $x$-coordinate.
The horizontal shear in Figure \ref{fig:shear} corresponds to the matrix $\begin{pmatrix}
1 & 1.02 \\
0 & 1
\end{pmatrix}
$.

\begin{problem}[Optional]
Write a function that accepts an array of points, a floating point argument that indicates the shearing amount, and an integer argument that indicates the direction of the shear (0 for horizontal, 1 for vertical).
Your function should return the sheared points.
Hint: To check your work, plot the original points and their images under the transformation using the function \li{plotOldNew()} defined in Problem \ref{prob:dilation}.
\end{problem}

\subsection*{Reflections}

\begin{figure}
\includegraphics[width=\textwidth]{reflect.pdf}
\caption{An example of a reflection.
The top image was reflected about the line $y = (1/\sqrt{3})x$, producing the bottom image.}
\label{fig:reflect}
\end{figure}
A fourth type of linear transformation is reflections about a line, also called \emph{Householder transformations}.
Reflecting about a line spanned by $(l_1, l_2)$ corresponds to the matrix
\[
\frac{1}{l_1^2 + l_2^2}
\begin{pmatrix}
l_1^2 - l_2^2 & 2l_1l_2 \\
2l_1l_2 & l_2^2 - l_1^2
\end{pmatrix}.
\]

For example, the line $y=x$ is spanned by $(1, 1)$.
In this case the corresponding matrix $\begin{pmatrix}
0 & 1\\
1 & 0
\end{pmatrix}$ is a Type I elementary matrix, in fact the only one of size $2 \times 2$.
As another example, the reflection in Figure \ref{fig:reflection} about the line $y = (1/\sqrt{3})x$ corresponds to the matrix $\frac{1}{4}\begin{pmatrix}
2 & 2\sqrt{3}\\
2\sqrt{3} & -2
\end{pmatrix}$.

\begin{problem}[Optional]
Write a function that accepts an array of points and a 1-D array describing the axis of reflection (in the notation above, this argument is $(l_1, l_2)$.
Your function should return the reflected points.
Hint: To check your work, plot the original points and their images under the transformation using the function \li{plotOldNew()} defined in Problem \ref{prob:dilation}.
\end{problem}

\subsection*{Composition of linear transformations}
Recall that composition of linear transformations corresponds to matrix multiplication.
For example, if $S$ is a matrix representing a shear and $R$ is a matrix representing a rotation, then $RS$ represents a shear followed by a rotation.

In fact, any linear transformation of $\mathbb{R}^2$ is a composition of the transformations discussed in this lab.
This is because reflections, dilations, and shears provide us with all the elementary matrices, and every matrix is a product of elementary matrices.

\section*{Affine transformations}
\subsection*{Translations}

\begin{figure}
\includegraphics[width=\textwidth]{translate.pdf}
\caption{
An example of a translation.
The top image was translated by the vector $(2, 0)\trp$ to produce the bottom image.}
\label{fig:translation}
\end{figure}

A translation is a map $T: \mathbb{R}^2 \rightarrow \mathbb{R}^2$ defined by $T(\mathbf{x}) = \mathbf{x}+\mathbf{b}$ where $\mathbf{b} \in \mathbb{R}^2$.
For example, if $\mathbf{b} = (2, 0)\trp$, then applying $T$ to an image will shift it left by 2.
This translation is illustrated in Figure \ref{fig:translation}.

Translations are usually NOT linear maps.
Therefore, they cannot be represented as matrix multiplication.

\begin{problem}
Write a function that accepts an array of points and an array indicating how much to shift them in each direction.
The function should return the translated points.
Hint: You can construct a $2 \times 1$ array using the syntax \li{np.array([[a], [b]])}.
This may be more convenient for broadcasting.
Another hint: To check your work, plot the original points and their images under the transformation using the function \li{plotOldNew()} defined in Problem \ref{prob:dilation}.
\end{problem}

\subsection*{Affine transformations}
Translations, together, with linear transformations, make up the broader class of transformations called ``affine transformations."
These are transformations of the form $T: \mathbb{R}^2 \to \mathbb{R}^2$, $T(X) = AX + b$ where $A$ is an $n\times n$ matrix and $b \in \mathbb{R}^n$.
Affine transformations include all compositions of scalings, rotations, dilations, reflections, and translations.
For example, if $S$ represents a shear and $R$ a rotation, and if $\mathbf{b}$ is a vector in $\mathbb{R}^2$, then $T(\mathbf{x}) = RS\mathbf{x} + \mathbf{b}$ first shears $\mathbf{x}$, then rotates it, and finally translates it by $\mathbf{b}$.

\begin{problem}
Imagine a particle $p_1$ rotating around a second particle $p_2$ which is moving through $\mathbb{R}^2$ in a straight line.
Suppose $p_2$ begins at the origin and $p_1$ begins at $(1, 0)$.
We can compute the trajectory of $p_1$ using affine transformations.

\begin{enumerate}\label{prob:trajectory}
\item Write a function that returns the position of $p_1$ at a time $t$.
Your function should accept a time $t$, an angular velocity $\omega$, a direction vector $\mathbf{v}$, and a speed $s$.
Assume $p_1$ rotates with angular velocity $\omega$ and $p_2$ moves in the direction of $\mathbf{v}$ with speed $s$.
The location of $p_1$ at time $t$ can be computed as follows:
\begin{itemize}
\item Calculate the position of $p_2$ at time $t$ with the formula $(st/\|\mathbf{v}\|) \mathbf{v}$.
\item Calculate the position of $p_1$ as follows:
\begin{itemize}
\item Rotate $p_1$ by $t\omega$ radians.
\item Translate the resulting vector by the vector equal to the position of $p_2$ at time $t$.
\end{itemize}
\end{itemize}
\end{enumerate}
\item Plot the trajectory of $p_1$ on the time interval $(0, 10)$ assuming $\omega=\pi$, $v=(1, 1)$, and $s=3$.
Your graph should look something like Figure \ref{fig:trajectory}.
\begin{figure}[H]
\includegraphics[width=\textwidth]{trajectory.pdf}
\caption{Solution to Problem \ref{prob:trajectory}.}
\label{fig:trajectory}
\end{figure}
\end{problem}

\section*{Linear systems}
This section describes efficient algorithms for solving systems of linear equations.

\subsection*{Programming elementry row operations}
When you perform a row operation on a matrix, you are really left-multiplying by some elementary matrix.
However, matrix multiplication is not the most efficient way to implement row operations. It is much faster to perform row operations by modifying the array in place, and only changing those entries that are affected by the operation.
The following code implements the three elementary row operations by modifying the array in place.

\lstinputlisting[style=fromfile]{row_opers.py}

\subsection*{Programming row reduction}
When you solve a linear system by reducing the matrix with row operations, it is most efficient to reduce only to row echelon form (REF)
\footnote{We do not require leading coefficients to be 1 in the REF.}
and then use back substitution.
Here is some code that reduces a matrix to REF.

\begin{lstlisting}
>>> A = np.array([[4., 5., 6., 3.],[2., 4., 6., 4.],[7., 8., 0., 5.]])
array([[ 4.,  5.,  6.,  3.],
       [ 2.,  4.,  6.,  4.],
       [ 7.,  8.,  0.,  5.]])
>>> A[1] -= (A[1,0]/A[0,0]) * A[0]
>>> A[2] -= (A[2,0]/A[0,0]) * A[0]
>>> A[2,1:] -= (A[2,1]/A[1,1]) * A[1,1:]
>>> A
array([[ 4. ,  5. ,  6. ,  3. ],
       [ 0. ,  1.5,  3. ,  2.5],
       [ 0. ,  0. , -9. ,  1. ]])
\end{lstlisting}

In the third row operation modified only a part of the third row because we knew that the first value would still be 0.
Modifying only those entries of the array that are affected by a row operation will save a lot of time when you are row reducing a large matrix.

Beware that round-off error in row reduction can cause serious problems.
Suppose we wish to row reduce a matrix $A$ as follows.

\begin{lstlisting}
>>> A = np.array([[4., 5., 6., 3.],[2., 2.5, 6., 4.],[7., 8., 0., 5.]])
array([[ 4.,  5.,  6.,  3.],
       [ 2.,  2.5,  6.,  4.],
       [ 7.,  8.,  0.,  5.]])
>>> A[1] -= (A[1,0]/A[0,0]) * A[0]
>>> A[2] -= (A[2,0]/A[0,0]) * A[0]
\end{lstlisting}

If we work this out by hand, at this point we have

\begin{equation}\label{equ:correct_ans}
A = \begin{pmatrix}
4&5&6&3 \\
0&0&3&2.5 \\
0&-7.5&-10.5&-.25
\end{pmatrix}.
\end{equation}

If we swap the second and third rows, then the matrix is in row echelon form.
However, suppose that due to round-off error, the machine instead computes

\[
A = \begin{pmatrix}
4&5&6&3 \\
0&10^{-15}&3&2.5 \\
0&-7.5&-10.5&-.25
\end{pmatrix}.
\]

The algorithm would then attempt to pivot on the \li{A[1,1]} entry as follows.

\begin{lstlisting}
>>> A[2,1:] -= (A[2,1]/A[1,1]) * A[1,1:]
>>> A
array([[ 4. ,  5.0e+00 , 6.00e+00 ,  3.000e-00 ],
       [ 0. ,  1.0e-14 , 3.00e+00 ,  2.500e-00 ],
       [ 0. ,  0.0e+00 , 2.25e+14 ,  1.875e+14 ]])
\end{lstlisting}

The round-off error in the \li{A[1,1]} entry has affected the third row, and the matrix is now much different than the correct answer in (\ref{equ:correct_ans}).
In larger matrices, round-off error can propagate through many steps in a calculation, resulting in garbage output.

Some of NumPy's matrix algorithms use row reduction.
NumPy's methods use several clever tricks to minimize the impact of round-off errors.
However, these methods may still give you garbage output due to round-off error, especially if the matrix is \emph{ill-conditioned}.
You should always be aware of this possibility.

\begin{problem}
\label{prob:REF}
Write a function which reduces a square matrix to REF.
You may make the following assumptions:
\begin{enumerate}
\item The matrix is invertible.
\item During your row reduction, a zero will never appear on the main diagonal.
\item All round-off errors may be ignored.
\end{enumerate}
During a row operation, do not modify any entries that you know will be zero before and after the operation.
\end{problem}

\subsection*{The LU decomposition}
The LU decomposition of a square matrix $A$ is a factorization $A=LU$ where $U$ is an upper triangular and $L$ is a lower triangular matrix.
In fact, $U$ is the REF of $A$ and $L$ is a product of Type III elementary matrices whose inverses reduce $A$ to $U$.
Thus, the LU decomposition is a way of storing the REF and ``how we got there.''

The LU factorization of $A$ exists only when $A$ can be reduced to REF using only Type III elementary matrices (no row swaps).
However, we can always permute the rows of $A$ to obtain a matrix that has an LU decomposition.
If $P$ encodes the row swaps, then a decomposition $PA = LU$ always exists.

If $A$ has an LU decomposition (not requiring row swaps), then we can find it as follows.
Reduce $A$ to REF with $k$ row operations, corresponding to matrices $E_1, \ldots, E_k$.
Then $U = E_k \ldots E_2E_1A,$ where $U$ is the REF of $A$.
Because there were no row swaps, each $E_i$ is a lower triangular Type III matrix. Then $E_i^{-1}$ is also lower triangular.
Thus, $L=(E_k \ldots E_2E_1)^{-1}$ is lower triangular, and $LU=A.$

Because $L=(E_k \ldots E_2E_1)^{-1} = IE_1^{-1}E_2^{-1}\ldots E_k^{-1}$, we can compute $L$ by right-multiplying the identity by the matrices we used to reduce $U$.
In fact, in this special situation, each right-multiplication will change only one entry of $L$.
We have the following algorithm for the LU decomposition, assuming it exists.

%
%\begin{itemize}
%\item Make a copy, $U$, of $A$.
%\item Make an identity matrix $L$ that is the same shape as $A$.
%\item Iterate through the entries below the diagonal of $U$.
%
%Now for each entry below the main diagonal of $U$ do the following:
%	\begin{itemize}
%	\item Set the corresponding entry of $L$ to the quotient of the current entry of $U$ and the entry of the main diagonal of $U$ located above the current entry.
%	\item Perform the type 3 row operation to set the current entry of $U$ to 0.
%		Do not modify any entries known to be 0.
%	\end{itemize}
%\item Return $L$ and $U$
%\end{itemize}

\begin{algorithm}
\begin{algorithmic}[1]
\Procedure{LU Decomposition}{$A$}
\State $m, n \gets \shape{A}$
\State $U \gets \makecopy{A}$
\State $L \gets \Id{n}$
\For{$j=0 \ldots n-1$}
    \For{$i=j+1 \ldots m-1$}
    \State $L[i,j] \gets U[i, j]/U[j, j]$
    \State $U[i,j:] \gets U[i,j:] - L[i,j]U[j, j:]$
	\EndFor
\EndFor
\State \pseudoli{return} $L, U$
\EndProcedure
\end{algorithmic}
\caption{The algorithm for LU decomposition of a matrix $A$. This algorithm returns lower triangular $L$ and upper triangular $U$ such that $A = LU$.}
\label{Alg:gram_schmidt}
\end{algorithm}

\begin{problem}
\label{prob:LU_copy}
Write a function that finds the LU decomposition of a square matrix.
You may assume that the matrix has an LU decomposition.
\end{problem}

The LU decomposition can be performed in-place by storing $U$ on and above the main diagonal of the array and storing $L$ below it.
The main diagonal of $L$ does not need to be stored since all its entries are ones.

\begin{problem}[Optional]
Modify your solution to Problem \ref{prob:LU_copy} so that the function computes the LU decomposition in place.
\end{problem}

\subsection*{Applications of the LU decomposition}
The LU decomposition is a more efficient way to solve linear systems than row reduction, and it can also be used to quickly compute inverses and determinants.
SciPy implements these methods in the \li{linalg} module, specifically in the functions \li{linalg.lu_factor}, \li{linalg.solve}, \li{linalg.inv}, and \li{linalg.det}.

Let us see how to use the LU decomposition to solve matrix equations.
Suppose that after row swaps, $A$ has the decomposition $PA = LU$.
Then $Ax=b$ is equivalent to $LUx=Pb$.
We can solve this system by first solving $Ly = Pb$ and then $Ux = y$.
Since $L$ and $U$ are triangular, these systems can be solved with backward and forward substitution, which is faster than row reduction.
Thus, we can perform row reduction once to compute the $LU$ factorization of $A$, and then we can use substitution to solve $Ax=b$ many different values of $b$.
This technique is significantly faster than storing $A^{-1}$ and then computing $A^{-1}b$ (see Problem \ref{prob:solve}).

\begin{problem}\label{prob:solve}
In this problem you will solve the system $Ax = b$ for fixed $A$ and many different values of $b$.
You will do this in two ways.
For a random $1000 \times 1000$ array $A$ and a random $1000 \times 500$ array $B$ do the following.
\begin{enumerate}
\item Time \li{la.lu_factor(A)}.
\item Time \li{la.inv(A)}.
\item Store the output of \li{la.lu_factor()}. Time \li{la.lu_solve()} on this stored output and $B$.
\item Store the inverse of $A$. Time how long it takes to multiply $A^{-1}$ by $B$.
\item What can you conclude about the more efficient way to solve linear systems?
\end{enumerate}
\end{problem}

Our technique for solving linear equations can also be used to invert matrices.
We can compute $A^{-1}$ by solving $LUx_i = P_i$ for every $P_i$ that is a column of $P$.
Then $A^{-1}$ is the matrix with columns $x_1, x_2, \ldots, x_n$.

Finally, the LU decomposition also gives us an efficient way to compute determinants.
For if $PA=LU$, then $\det(A) = [\det(P)]^{-1}\det(L)\det(U)$.
But the determinant of a triangular matrix is the product of its diagonal entries, and every diagonal entry of $L$ is 1.
Also, $\det(P)$ is the number of row swaps we applied to $A$ to put it in the appropriate form.
So if $U$ has diagonal entries $u_{ii}$ for $i=1, \ldots, n$, then

\[
\det(A) = (-1)^S\left(\displaystyle\prod_{i=1}^nu_{ii}\right),
\]

where $S$ is the number of row-swaps.

%\begin{problem}
%\label{prob:lusolve}
%Write a function that takes the LU decomposition computed by the second function you made in Problem \ref{prob:LU} and another array representing the right hand side of a linear system and modifies the second array in place so that it represents the solution to the linear system.
%No changes to the array storing the LU decomposition are necessary.
%\end{problem}
%

\begin{problem}[Optional]
\label{prob:det}
Write a function that finds the determinant of a matrix using \li{la.lu_factor()} and the algorithm described above.
Read the documentation to learn about the output of \li{la.lu_factor()}.
Hint: If the $i^{th}$ entry of the output \li{piv} does not equal $i$, this means that a row swap occurred.
\end{problem}

\subsection*{The Cholesky decomposition (Optional)}
The Cholesky decomposition requires half the calculations and memory of the LU decomposition.
Furthermore, it is \emph{numerically stable}, which means that round-off errors do not propagate throughout the computation.
Because of its efficiency and numerical stability, the Cholesky decomposition is used to solve least squares, optimization, and state estimation problems.

However, the Cholesky decomposition is only applicable to Hermitian positive definite matrices.
A matrix $A$ is positive definite if  $\mathbf{z}\trp A\mathbf{z} > 0$ for all $\mathbf{z} \neq 0$.
Furthermore, $A$ is Hermitian if $A = A^*$ where $A^* = \overline{A\trp }$, so a real Hermitian matrix is just a symmetric matrix.
The Cholesky decomposition is the matrix equivalent to taking the square root of a positive real number.

The Cholesky decomposition of a $A$ is a lower-triangular matrix $L$ such that

\begin{equation*}
 A = LL^*.
\end{equation*}

The entries of $L$ are calculated as follows.

\begin{align*}
&L_{i,j} = \frac{1}{L_{j,j}}\left(A_{i,j} -\sum_{k=1}^{j-1}{L_{i,k}L_{j,k}}\right) \mbox{ for $i>j$} \\ \\
&L_{i,i} = \sqrt{A_{i,i} - \sum_{k=1}^{i-1}{L_{i,k}L_{i,k}}}.
\end{align*}

Notice that the entries of $L$ are defined recursively, with dependencies as diagrammed in Figure \ref{fig:cholesky}.
Thus, an implementation of the Cholesky decomposition must compute the entries of $L$ in the correct order.

\begin{figure}
\begin{tikzpicture}[red dot/.style={draw, circle, fill=red, red},
	norm/.style={draw=none}, xscale=1.5, yscale=1.5]

\begin{scope}[shift={(4,0)}]
\draw [-,ultra thick](-.2,0)--(-.2,2.5);
\draw [-,ultra thick](2.7,0)--(2.7,2.5);
\draw [-,ultra thick](-.2,0)--(0,0);
\draw [-,ultra thick](-.2,2.5)--(0,2.5);
\draw [-,ultra thick](2.7,2.5)--(2.5,2.5);
\draw [-,ultra thick](2.7,0)--(2.5,0);

\node[norm,black](bk1)at(.25,2.25){\LARGE \textbullet};
\node[norm,black](bk2)at(.75,1.75){\LARGE \textbullet};
\node[norm,black!25!](b1)at(1.25,1.25){\LARGE \textbullet};
\node[norm,black](bk3)at(1.75,.75){\LARGE \textbullet};
\node[norm, black](bk4)at(2.25,.25){\LARGE \textbullet};
\node[norm, black!25!](b2)at(.25,1.25){\LARGE \textbullet};
\node[norm,black!25!](b3)at(.75,1.25){\LARGE \textbullet};
\node[norm,black!25!](b4)at(.25,.25){\LARGE \textbullet};
\node[norm,black!25!](b3)at(.75,.25){\LARGE \textbullet};
\node[norm, shadecolor](r1)at(1.25,.25){\Huge \textbullet};

\end{scope}

\begin{scope}
\draw [-,ultra thick](-.2,0)--(-.2,2.5);
\draw [-,ultra thick](2.7,0)--(2.7,2.5);
\draw [-,ultra thick](-.2,0)--(0,0);
\draw [-,ultra thick](-.2,2.5)--(0,2.5);
\draw [-,ultra thick](2.7,2.5)--(2.5,2.5);
\draw [-,ultra thick](2.7,0)--(2.5,0);

\node[norm, shadecolor](r1)at(1.75,.75){\Huge \textbullet};
\node[norm,black](bk1)at(.25,2.25){\LARGE \textbullet};
\node[norm,black](bk2)at(1.25,1.25){\LARGE \textbullet};
\node[norm,black](bk3)at(.75,1.75){\LARGE \textbullet};
\node[norm,black](bk4)at(2.25,.25){\LARGE \textbullet};
\node[norm,black!25!](bk1)at(.25,.75){\LARGE \textbullet};
\node[norm,black!25!](bk1)at(.75,.75){\LARGE \textbullet};
\node[norm, black!25!](bk1)at(1.25,.75){\LARGE \textbullet};
\end{scope}
\end{tikzpicture}
\label{fig:cholesky}
\caption{The entries of $L$ in the Cholesky decomposition are defined recursively, as illustrated in this picture.
To calculate the green entry, you need to know each of the light gray entries.}
\end{figure}

\begin{problem}[Optional]
Write a function that finds the Cholesky decomposition of a Hermitian positive definite matrix.
Hint: To generate symmetric positive definite matrices on which to test your function, recall that for any matrix $A$, the matrix $A\trp A$ is symmetric and positive definite.
\end{problem}

The \li{linalg} module of SciPy includes an optimized Cholesky decomposition.
As with the LU decomposition, SciPy has the methods \li{la.cho_factor} and \li{la.cho_solve}.

\begin{problem}[Optional]
Repeat problem \ref{prob:solve}, this time using the methods \li{la.cho_factor} and \li{la.cho_solve}.
\end{problem}


\chapter{Classification}
\begin{multicols*}{2}

\section{Naive Bayes Classifier}

\noindent Bayes' rule:
$$P(c|d) = \frac{P(d|c) P(c)}{P(d)}$$

\noindent Naive Bayes classifier:
$$c=\arg\!\max_{c_j \in C} P(x_1, x_2, \ldots,x_n|c_j)P(c_j)$$
$$c=\arg\!\max_{c_j \in C} P(x_1|c_j)P(x_2|c_j)\ldots P(x_n|c_j)P(c_j)$$
$$c=\arg\!\max_{c_j \in C} P(x_j) \prod_{i=1}^n P(x_i|c_j)$$

\noindent Smoothing:
$$P(x_i|c_j) = \frac{P(x_i,c_j)}{P(c_j)}$$
$$P(x_i|c_j) = \frac{P(x_i,c_j)+1}{P(c_j)+k}$$

\noindent where $k$ is the size of vocabulary

\section{K Nearest Neighbour Classification}
\noindent Algorithm:
\begin{itemize}
    \item Define $N$ as $k$ nearest neighbours of $d$
    \item Use majority vote in $N$ to classify $d$
\end{itemize}
\noindent kNN has high variance and low bias (memories more than generalise)

\section{Support Vector Machine (SVM)}
\noindent Idea: the original feature space can be mapped to some higher-dimensional feature space where the training set is separable\\

\noindent Linear separability means classes can be separated by a lines or hyperplanes\\

\noindent SVM maximises  the margin around the separating hyperplane. The decision function is specified by a subset of training samples called support vectors.

\section{Feature Selection: Chi-square}
\noindent Motivation: to reduce training time, improve generalisation, eliminate noise features, and avoids overfitting\\

\noindent The chi-square test is used to determine whether there is a significant difference between the expected frequencies and the observed frequencies in one or more categories.

\begin{center}
\begin{tabular}{ |c|c| }
    \hline
    $A = \#(t,c)$ & $B = \#(\sim t,c)$ \\
    \hline
    $C = \#(t,\sim c)$ & $D = \#(\sim t,\sim c)$ \\
    \hline
\end{tabular}
\end{center}

$$\chi^2(t,c) = \frac{N\times (AD - BC)^2}{(A+B) \times (C+D) \times (A+C) \times (B+D)}$$
$$N=A+B+C+D$$

\noindent If $\chi^2 > 10.83$, we reject with 99\% condident that the two variables / features are independent. Therefore, they are dependent and should be helpful as features for the classification.

\end{multicols*}