\section*{Exercises}

\begin{ex}
  Find the vector equation for the line through $(-7,6,0)$ and
  $(-1,1,4)$. Then, find the parametric equations for this
  line.
  % \begin{sol}
  % \end{sol}
\end{ex}

\begin{ex}
  Find parametric equations for the line through the point
  $(7,7,1)$ with direction vector
  $\vect{d} = \begin{mysmallmatrix}{r} 1 \\ 6 \\ 2 \end{mysmallmatrix}$.
  % \begin{sol}
  % \end{sol}
\end{ex}

\begin{ex}
  Parametric equations of the line are
  \begin{equation*}
    \begin{array}{c}
      x = t+2, \\
      y = 6-3t, \\
      z = -t-6.
    \end{array}
  \end{equation*}
  Find a direction vector for the line and a point on the line.
  % \begin{sol}
  % \end{sol}
\end{ex}

\begin{ex}
  The equation of a line in two dimensions is written as $y=x-5$. Find
  a vector equation for this line.
  % \begin{sol}
  % \end{sol}
\end{ex}

\begin{ex}
  Find parametric equations for the line through $(6, 5, -2, 3)$
  and $(5, 1, 2, 1)$.
  % \begin{sol}
  % \end{sol}
\end{ex}

\begin{ex}
  Consider the following vector equation for a line in $\R^3$:
  \begin{equation*}
    \begin{mymatrix}{c} x\\y\\z \end{mymatrix}
    = \begin{mymatrix}{r} 1\\2\\0 \end{mymatrix}
    + t\,\begin{mymatrix}{r} 1\\0\\1 \end{mymatrix}.
  \end{equation*}
  Find a new vector equation for the same line by doing the change of
  parameter $t=2-s$.
  \begin{sol}
    We have
    \begin{equation*}
      \begin{mymatrix}{c} x\\y\\z \end{mymatrix}
      = \begin{mymatrix}{r} 1\\2\\0 \end{mymatrix}
      + (2-s)\,\begin{mymatrix}{c} 1\\0\\1 \end{mymatrix}
      = \begin{mymatrix}{r} 3\\2\\2 \end{mymatrix}
      + s\,\begin{mymatrix}{r} -1\\0\\-1 \end{mymatrix}.
    \end{equation*}
  \end{sol}
\end{ex}

\begin{ex}
  Consider the line given by the following parametric equations:
  \begin{equation*}
    \begin{array}{c}
      x = 2t+2, \\
      y = 5-4t, \\
      z= -t-3.
    \end{array}
  \end{equation*}
  Find symmetric equations for the line.
  % \begin{sol}
  % \end{sol}
\end{ex}

\begin{ex}
  Find the point on the line segment from $P = (-4, 7, 5)$ to
  $Q = (2, -2, -3)$ which is $\frac{1}{7}$ of the way from $P$
  to $Q$.
  % \begin{sol}
  % \end{sol}
\end{ex}

\begin{ex} Suppose a triangle in $\R^n$ has vertices at $P$, $Q$,
  and $R$.  Consider the lines which are drawn from a vertex to the
  mid point of the opposite side. Show these three lines intersect in
  a point and find the coordinates of this point.
  % \begin{sol}
  % \end{sol}
\end{ex}

\begin{ex}
  Determine whether the lines
  \begin{equation*}
    \begin{mymatrix}{c} x \\ y \\ z \end{mymatrix}
    = \begin{mymatrix}{c} 1 \\ 1 \\ 2 \end{mymatrix}
    + t \begin{mymatrix}{c} 1 \\ 2 \\ 2 \end{mymatrix}
    \quad\mbox{and}\quad
    \begin{mymatrix}{c} x \\ y \\ z \end{mymatrix}
    = \begin{mymatrix}{c} 1 \\ -1 \\ -4 \end{mymatrix}
    + s \begin{mymatrix}{c} 1 \\ 1 \\ -1 \end{mymatrix}
  \end{equation*}
  intersect. If yes, find the point of intersection.
\end{ex}

\begin{ex}
  Determine whether the lines
  \begin{equation*}
    \begin{mymatrix}{c} x \\ y \\ z \end{mymatrix}
    = \begin{mymatrix}{c} 2 \\ -1 \\ 0 \end{mymatrix}
    + t \begin{mymatrix}{c} 1 \\ 3 \\ 2 \end{mymatrix}
    \quad\mbox{and}\quad
    \begin{mymatrix}{c} x \\ y \\ z \end{mymatrix}
    = \begin{mymatrix}{c} 1 \\ 1 \\ 3 \end{mymatrix}
    + s \begin{mymatrix}{c} 1 \\ 2 \\ 0 \end{mymatrix}
  \end{equation*}
  intersect. If yes, find the point of intersection.
\end{ex}

\begin{ex}
  Find the angle between the two lines
  \begin{equation*}
    \begin{mymatrix}{r} x \\ y \\ z \end{mymatrix}
    = \begin{mymatrix}{r} 3 \\ 0 \\ 1 \end{mymatrix}
    + t \begin{mymatrix}{r} 3 \\ -3 \\ 0 \end{mymatrix}
    \quad\mbox{and}\quad
    \begin{mymatrix}{r} x \\ y \\ z \end{mymatrix}
    = \begin{mymatrix}{r} 3 \\ 0 \\ 1 \end{mymatrix}
    + s \begin{mymatrix}{r} -1 \\ 2 \\ 2 \end{mymatrix}.
  \end{equation*}
  % \begin{sol}
  % \end{sol}
\end{ex}

\begin{ex} Let $P = (1,2,3)$ be a point in $\R^3$. Let $L$ be the line
  through the point $P_0 = (1, 4, 5)$ with direction vector
  $\vect{d} = \begin{mysmallmatrix}{r} 1 \\ -1 \\
    1 \end{mysmallmatrix}$. Find the shortest distance from $P$ to
  $L$, and find the point $Q$ on $L$ that is closest to $P$.
  % \begin{sol}
  % \end{sol}
\end{ex}

\begin{ex} Let $P = (0,2,1)$ be a point in $\R^3$. Let $L$ be the line through the points $P_0 = (1, 1, 1)$ and $P_1 = (4, 1, 2)$. Find the shortest distance from $P$ to $L$, and find the point $Q$ on $L$ that is closest to $P$.
  % \begin{sol}
  % \end{sol}
\end{ex}

\begin{ex}\label{ex:angle-lines}
  When we computed the angle between two lines in
  Example~\ref{exa:angle-between-two-lines}, we calculated two
  different angles and took the smaller of the two. Show that one can
  get the same answer by taking the absolute value of the dot product,
  i.e., by solving
  \begin{equation*}
    \cos\theta =
    \frac{\abs{\vect{u}\dotprod\vect{v}}}{\norm{\vect{u}}\norm{\vect{v}}}.
  \end{equation*}
  \begin{sol}
    From trigonometry, we have the following properties of the cosine
    function:
    \begin{itemize}
    \item $\cos\theta$ is positive for $0\leq\theta\leq\frac{\pi}{2}$,
      and negative for $\frac{\pi}{2}\leq\theta\leq\pi$.
    \item $\cos(\pi-\theta) = -\cos\theta$.
    \end{itemize}
    By the method in Example~\ref{exa:angle-between-two-lines}, we
    calculated $\theta$ such that
    \begin{equation*}
      \cos\theta =
      \frac{\vect{u}\dotprod\vect{v}}{\norm{\vect{u}}\norm{\vect{v}}}.
    \end{equation*}
    If $0\leq\theta\leq\frac{\pi}{2}$, the answer is $\theta$. If
    $\frac{\pi}{2}\leq\theta\leq\pi$, the answer is $\phi=\pi-\theta$.
    But in the last case, the dot product is negative and we have
    \begin{equation*}
      \cos\phi = \cos(\pi-\theta) = -\cos\theta = \frac{-\vect{u}\dotprod\vect{v}~~}{\norm{\vect{u}}\norm{\vect{v}}}
      =
      \frac{\abs{\vect{u}\dotprod\vect{v}}}{\norm{\vect{u}}\norm{\vect{v}}}.
    \end{equation*}
    So in either case we get the correct answer by taking the absolute
    value.
  \end{sol}
\end{ex}


\section{LLN and CLT}
Let $X_1, ..., X_n \stackrel{iid}{\sim} P_{\mu}$, where $E(X_i)=\mu$ and $Var(X_i)=\sigma^2$ for all $i=1,2,...,n$ and $\bar{X_n}= \frac{1}{n} \sum_{i=1}^{n} X_i$.\\
\textbf{Law of large numbers:}
\begin{align*}
\bar{X_n}& \xrightarrow[n \rightarrow \infty]{P, a.s.} \mu\\
\frac{1}{n} \sum_{i=1}^{n} g(X_i)& \xrightarrow[n \rightarrow \infty]{P, a.s.} \mathbb{E}[g(X)]
\end{align*}
\textbf{Central Limit Theorem for Mean:}
\begin{align*}
\sqrt(n)\frac{\bar{X_n}-\mu}{\sqrt(\sigma^2)}& \xrightarrow[n \rightarrow \infty]{(d)} N(0,1)\\
\sqrt(n)(\bar{X_n}-\mu)& \xrightarrow[n \rightarrow \infty]{(d)} N(0,\sigma^2)
\end{align*}

\textbf{Central Limit Theorem for Sums:}

\begin{align*}
\sum{X}_{i=1}^{n} & \xrightarrow[n \rightarrow \infty]{(d)} N(n \mu, \sqrt(n)\sqrt(\sigma^2))\\
\frac{\sum{X}_{i=1}^{n} - n\mu}{\sqrt(n) \sqrt(\sigma^2)} & \xrightarrow[n \rightarrow \infty]{(d)} N(0, 1)\\
\end{align*}

\textbf{Variance of the Mean:}
\begin{align*}
Var(\overline{X_n})&=(\frac{\sigma^2}{n})^2 Var(X_1 + X_2,...,X_n) =\frac{\sigma^2}{n}
\end{align*}
\textbf{Expectation of the mean:}
\begin{align*}
E[\bar{X_n}] & =\frac{1}{n}E[X_1 + X_2,...,X_n] =\mu.
\end{align*}


\section{Masked attention
implementation}

In this section we will detail the prefix sum algorithm proposed by
\citet{choromanski2021rethinking} for
 masked kernelized attention, and give an implementation with usual
functions of neural network frameworks.

\begin{equation}
A^{masked} = \mathrm{masked} \left( \phi(Q) \times \phi(K^T) \right) \times V
\end{equation}

In this expression, $\mathrm{masked}$ being the operation that set all cells
 above the diagonal to 0 in a matrix. The naive implementation of this
 operation has complexity $O(L_QL_Kd)$.

We can change the complexity using the operation proposed by
 \citet{choromanski2021rethinking}. We
 will derive its formulation here.
 To start with it, $A^{masked}$ is defined as

\begin{equation}
A^{masked} = S^{masked} \times V
\end{equation}

\noindent{}which
in summation form is expressed as

\begin{equation}
A^{masked}_{ij} = \sum_k V_{kj} \times S^{masked}_{ik}
\end{equation}

\noindent{}and the
elements of S are defined as:

\begin{equation}
S^{masked}_{ik} =
\begin{cases}
k \leq i & \sum_l \left( \phi(Q)_{il} \times \phi(K)_{kl} \right) \\
\text{otherwise} &{}0
\end{cases}
\end{equation}

Putting these elements together leads to

\begin{equation}
A^{masked}_{ij}= \sum_{k=1}^i V_{kj} \times \sum_l \left( \phi(Q)_{il} \times \phi(K)_{kl} \right)
\end{equation}

\noindent{}which can be reworked as

\begin{equation}
A^{masked}_{ij}= \sum_l \phi(Q)_{il} \times \sum_{k=1}^i \left(V_{kj} \times \phi(K)_{kl} \right)
\end{equation}

In this work we make use of these ideas to implement the calculation of
 $A^{masked}$ with complexity $O(max(L_Q, L_K) \times d^2)$ without
 custom GPU code as per the following algorithm:

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\item
$\phi(Q)$, $\phi(K)$, $V$ tensors of shape $(L_Q, 1, d)$,
$(L_K, d)$, $(L_K, d)$
\item
$Unrolled_{kjl} = V_{kj} \times \phi(K)_{kl}$ tensor of shape
$(L_K, d, d)$
\item
$Right = cumsum(Unrolled,\text{ dim=0})$ tensor of shape
$(L_K, d, d)$
\item
$Right = align(Right, L_Q)$ tensor of shape $(L_Q, d, d)$
\item
$A^{masked} = \phi(Q) \otimes Right$ tensor of shape $(L_Q, 1, d)$
\end{enumerate}

\noindent{}with

\begin{itemize}
\item
$\otimes$ the batch matrix product along the last two dimensions
\item
$cumsum(\_, dim=0)$ the function that calculates the cumulated sum
along the first dimension
\item
$align(\_, L_Q)$ the function that extend the first dimensions to
size $L_Q$ by repeating the last element if $L_Q > L_K$, or
truncate to size $L_Q$ if $L_Q < L_K$. (Can be implemented with
slicing and concatenation)
\end{itemize}

\endinput


\subsection{Support}

\(f X\rightarrow R\)

Support of \(f\) is \(x \in X\) where \(f(x)\ne 0\)


\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage[margin=1in]{geometry}
\usepackage{amsmath}
\usepackage{graphicx}
\graphicspath{{plots/}}
\setlength{\parindent}{0em}
\setlength{\parskip}{0.5em}

\title{CTA200 Assignment 2}
\author{Emily Crawley}
\date{05/05/2021}

\begin{document}

\maketitle

\section*{Question 1}

Using the approximation given for evaluating a function $f(x)$ at $x_0$ for small finite $h$, we define \texttt{deriv(f, x0, h)} which computes

$$ \frac{f(x_0 + h) - f(x_0 - h)}{2h} $$

We then define our function to be differentiated, \texttt{f(x) = sin(x)} and its analytical derivative \texttt{dfdx(x) = cos(x)}. Values of $h < 1$ were chosen ranging from 0.1 to 1e-8.

We compute the analytical derivative $cos(0.1)$, and the numerical derivatives at each value of $h$, plotting them each against the analytical solution. Since the $h$ values vary by orders of 10 we plot the error on logarithmic axes using \texttt{pyplot.loglog}.

\includegraphics[scale=0.7]{plot1}

From the plot we can see that the error in the numerical solution decreases linearly for smaller values of $h$, until around $h < 10^{-6}$ when the larger relative error presumably comes from the precision limitations of the \texttt{float} type. The slope of the linear part of the graph represents how quickly the error decreases as $h$ decreases, so in some sense represents its effectiveness in terms of accuracy. That is, we want to be able to greatly decrease error with only one or two order-of-magnitude decrements of $h$, so a steeper slope is preferable.

\section*{Question 2}

We cover the complex plane in the square $|z|^2 = \Re(z)^2 + \Im(z)^2$ by taking $c = x + iy$ for numerous combinations of x- and y-values between -2 and 2.

For each point $c$ we iterate the expression $z_{i + 1} = z_i^2 + c$ starting from $z_0 = 0$ and test the absolute square of each $z_i$ to see if it diverges from the boundary. If so, we assign it a value of 0 in a 2D boolean array representing whether each point is bounded or not. We also record the iteration number in a separate 2D array. The iteration number array contains values of -1 for points that stay bounded. We then plot the results using \texttt{pyplot.imshow}.

When tested running for 1000 iterations, none of the points in the sample set take more than 32 iterations to diverge, so to save time we will only iterate for 32 steps.

The only obvious pattern from the resulting images is that there are no points that remain bounded from our iteration in an area centered around $c = -0.5i$. This same area on the iteration numbers plot also notably contains the brightest points, i.e. the points in these areas take a higher number of iterations to diverge. There is a single point near the center of this area that notably takes the single highest number of iterations to diverge. These results are taken from a finite number of points across the area and not every possible point, so there could be more "interesting" points that are missed by numerical analysis.

\includegraphics[scale=0.7]{plot2_1}

\includegraphics[scale=0.7]{plot2_2}

\section*{Question 3}

Our initial value problem is given as:

\begin{align}
    \frac{dS}{dt} &= -\frac{\beta S I}{N},\\
    \frac{dI}{dt} &= \frac{\beta S I}{N} - \gamma I,\\
    \frac{dR}{dt} &= \gamma I
\end{align}

$I(0) = 1, S(0)=999, R(0) = 0$

$N=1000$, $0\leq t \leq 200$

Using \texttt{scipy.integrate.solve\_ivp} as the ODE integrator, which takes parameters \texttt{(fun, tspan, y0)}, we set up the time derivative functions as a vector $[\frac{dS}{dt},\frac{dI}{dt},\frac{dR}{dt}]$ for the \texttt{fun} parameter, and the initial conditions vector $[S(0), I(0), R(0)]$ as \texttt{y0}. We also specify the parameter \texttt{vectorized=True} so that the integrator handles the vectors properly. We will use its default Runge-Kutta method, \texttt{method='RK45'}.

$\beta$ represents an infection rate and $\gamma$ represents a recovery rate, so we will select some meaningful variations on these parameters, i.e. a disease that spreads quickly but has a fast recovery rate, a disease that spreads quickly with a slower recovery rate, etc. The solutions show us the different progressions for each of these sets of parameters, for example with a high infection rate and low recovery rate there is a large spike in $I(t)$, whereas with the same infection rate but a higher recovery rate the $I(t)$ "spike" is much lower as the disease does not have the opportunity to progress. The first three scenarios plotted all have the recovered and susceptible populations become constant when the infected population reaches zero. With a zero recovery rate, the entire population eventually becomes infected.

\includegraphics[scale=0.7]{plot3}

\end{document}


\documentclass{article}
\title{Learning}
\author{Samuel Schlesinger}

\usepackage{amsmath}
\usepackage{amssymb}

\begin{document}

\maketitle

\section{Problems}

\subsection{Linear Regression}

In linear regression, we wish to relate two sets of data by a linear estimator. Specifically,
we have some set of vectors $\{(x_i \in \mathbb{R}^D, y_i \in \mathbb{R})\}_{i \in [N]}$ and
we want to find some $\beta \in \mathbb{R}^{D + 1}$ such that we minimize the sum of squared
errors. From now on, for notational parsimony, I'll assume that $x_i \in \mathbb{R}^{D+1}$ and
say, the last dimension, is reserved for a $1$, to multiplicatively interact with the extra
place in $\beta$, which will perform the role of a "bias" in our estimate. We're learning an
affine transformation rather than simply a linear one this way. We relate our variables in the
following way, where $\epsilon \in \mathbb{R}$ is the error term.

\[ y = x\beta + \epsilon \]
\[ SSE = \epsilon^T\epsilon \]

We'll go through a quick derivation of the closed formula for $\beta$ which minimizes $SSE$.
First, we recognize:

\[ \epsilon = y - x\beta \]

Thus, we can write $SSE$ as:

\[ SSE = (y - x\beta)^T(y - x\beta) \]

We can distribute the transposition:

\[ SSE = (y^T - \beta^Tx^T)(y - x\beta) \]

Finally, we expand:

\[ SSE = y^Ty - y^Tx\beta - \beta^Tx^Ty + \beta^Tx^Tx\beta \]

And by transposing one of the inner terms (not changing its value, as this is the error which
is a scalar):

\[ SSE = y^Ty - 2\beta^Tx^Ty + \beta^Tx^Tx\beta \]

So we have a nice formula for the error, but we have to minimize!

\[ \frac{\partial SSE}{\partial \beta} = -2x^Ty + 2x^Tx\beta = 0 \]
\[ \beta = (x^Tx)^{-1}x^Ty \]

We have found our $\beta$ in closed form, so then my Haskell code initially reflected this.
That being said, once one observes a property of $\beta$ its clear that something better
can be done computationally. Critically, $x^Tx$ is always going to be positive definite, so
we can use the Cholesky factorization and get a matrix $L^*L = x^Tx$ and write:

\[ \beta = (L^*L)^{-1}x^Ty = (L^{-1})(L^{-1})^*x^Ty \]

The property of $L$ which is important here is that it is triangular, making the inversion
much more numerically stable than it otherwise could be. This becomes incredibly important
and because of this implementation change I went from being able to deal with thousands of
data points of dimension $3$, now I can handle millions of dimension $100$.

\subsection{KMeans Clustering}

When we kmeans-cluster data, we are given a set of data points and an integer:

\[X = \{x_i \in \mathbb{R}^D\}_{i \in [N]}, k \in \mathbb{N}, k \leq N \]

Our objective is to produce $S = \{S_i \subset X\}_{i \in [k]}$ a partition of $X$ with means
$\mu_i = \frac{\sum_{x \in S_i} x}{|S_i|}$ such that it
minimizes the kmeans error function:

\[ KME(S) = \sum_{i = 1}^k \sum_{x \in S_i} ||x - \mu_i||^2 \]

Solving this problem is not easy, in fact its NP-hard if you squint at it long enough, so
we'll use a simple greedy heuristic. Lets assume that we have some candidate set of $\mu_i$,
we can use these to generate a clustering by taking
$S_i = \{ x \in X \mid argmin_j d(x, \mu_j) = i\}$. That being said, we must then check if
these $\mu_i$ actually are the means for these clusters, and thus we must check that $\mu_i$
truly is the average of $S_i$. Thus we compute the new mean of $S_i$, $\mu'_i$, and we check
if $\mu'_i = \mu_i$. If this is the case for all $i$, we are finished, but if it is not, we
must undergo the hassle of reconstructing $S$, as a different partition will result for the
new means we've found. This process repeats and repeats and repeats until it reaches a stable
point, which we say is the kmeans clustering of the dataset according to our heuristic. It
turns out this process works well if you choose the points right, and there are a variety of
ways to do this, so I've left this choice up to the user of the algorithm. In the test, I use
a simple uniform choice selection, which works decently well in practice but lacks some perks
for theory. This algorithm performs much more poorly than linear regression simply because of
the nature of the problem, and because I also wasn't able to delegate enough of this work to
the fast processes in hmatrix. I really want to give this another whack sometime and make it
much more efficient.

\end{document}


\documentclass{ximera}
\input{../preamble}
\title{Exercises: General Slicing}
%%%%%\author{Philip T. Gressman}

\begin{document}
\begin{abstract}
Exercises computing volume by cross-sectional area.
\end{abstract}
\maketitle

\begin{exercise}
The base of a solid region is bounded by the curves $x = 0$, $y = x^2$ and $y = x$. The cross sections perpendicular to the $x$-axis are squares. Compute the volume of the region.
\begin{itemize}
\item A typical square cross section has side length $L = \answer{x - x^2}$ and area $A = \answer{(x-x^2)^2}$.
\item Possible numerical values of the $x$-coordinates of points in the base range from a minimum value of $x = \answer{0}$ up to a maximum of $x = \answer{1}$.
\item To compute volume, integrate:
\[ V = \int_{\answer{0}}^{\answer{1}} \answer{(x-x^2)^2} d\answer{x} = \answer{\frac{1}{30}}. \]
\end{itemize}
\end{exercise}

\begin{exercise}
Find the volume of the region in three-dimensional space defined by the inequalities
\begin{align*}
0 & \leq x \leq 1, \\
0 & \leq y \leq z^2, \\
0 & \leq z \leq 3.
\end{align*}
\begin{itemize}
\item Cross sections perpendicular to the $z$-axis are \wordChoice{\choice{square}\choice[correct]{rectangular}\choice{triangluar}} with length $\answer{1}$ in the $x$-direction and width $\answer{z^2}$ in the $y$-direction.
\item The area of a $z$ cross section is $A(z) = \answer{z^2}$.
\item To compute volume, integrate:
\[ V = \int_{\answer{0}}^{\answer{3}} \answer{z^2} dz = \answer{9}. \]
\end{itemize}
\end{exercise}

\begin{exercise}
A right circular cylinder of radius $1$ and height $3$ is twisted along its axis so that the disk at height $z$ is centered on the axis $ x = \cos (2 \pi z/3), y = \sin (2 \pi z/3))$, which corresponds to one full twist along the axis. Compute the volume of this twisted cylinder.
\[ V = \answer{3 \pi}. \]
\end{exercise}

\begin{exercise}
A certain three-dimensional region has a base in the $xy$-plane which is bounded above by the graph $y = 1-x^2$ and below by $y=0$. Slices perpendicular to the $y$-axis are equilateral triangles whose base lies in the $xy$-plane as well. Compute the volume of the region.
\[ V = \answer{\frac{\sqrt{3}}{2}}. \]
\end{exercise}

\section*{Sample Exam Questions}

\begin{question}(2018 Midterm 1)
Compute the volume of the region in 3-dimensional space which satisfies the inequalities
\begin{align*}
0 \leq   x \leq (1 - z^2) \qquad \mbox{ and } \qquad
0 \leq  y \leq (1 + z^2) \qquad \mbox{ and } \qquad
0 \leq z \leq 1.
\end{align*}
\vspace{-12pt}
\begin{multipleChoice}
\choice{$\displaystyle \frac{2}{3}$}
\choice{$\displaystyle \frac{3}{4}$}
\choice[correct]{$\displaystyle \frac{4}{5}$}
\choice{$\displaystyle \frac{5}{6}$}
\choice{$\displaystyle \frac{6}{7}$}
\choice{none of these}
\end{multipleChoice}
\end{question}

\begin{question}(2019 Midterm 1)
The inequality
\[ \frac{x^2}{a^2} + \frac{y^2}{b^2} \leq 1 \]
defines an ellipse in the $xy$-plane whose area is $\pi a b$ for any positive values of the constants $a$ and $b$.  Compute the three dimensional volume of the region defined by
\[ 4 x^2 + z^2 y^2 \leq z^2 \mbox{ for } 0 \leq z \leq 1. \]
(Hints won't reveal until after you choose a response.)
\begin{multipleChoice}
\choice{$\displaystyle 4 \pi z^2$}
\choice{$4 \pi$}
\choice{$\displaystyle \frac{\pi}{4z}$}
\choice[correct]{$\displaystyle \frac{\pi}{4}$}
\choice{$\displaystyle {\pi z}$}
\choice{$\displaystyle {\pi}$}
\end{multipleChoice}
\begin{feedback}
Dividing the first inequality by $z^2$ on both sides gives
\[ \frac{x^2}{\frac{z^2}{4}} + \frac{y^2}{1} \leq 1, \]
\begin{hint}
which means that slices in the $z$-direction are ellipses with area $\pi \frac{z}{2} \cdot 1 = \frac{\pi z}{2}$.
\begin{hint}
Volume is obtained by integration:
\[ V = \int_0^1 \frac{\pi z}{2} dz = \left. \frac{\pi z^2}{4} \right|_{0}^1 = \frac{\pi}{4}. \]
\end{hint}\end{hint}
\end{feedback}
\end{question}

\end{document}


\documentclass{article}
\usepackage{graphicx}
\usepackage{titletoc}
\usepackage{titlesec}
\usepackage{geometry}
\usepackage{fontspec, xunicode, xltxtra}
\usepackage{float}
\usepackage{cite}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{listings}
\usepackage{titletoc}
\usepackage{bm}

\geometry{left=3cm,right=3cm,top=3cm,bottom=3cm}
\DeclareMathOperator*{\argmin}{argmin}
\DeclareMathOperator*{\argmax}{argmax}
\DeclareMathOperator*{\var}{var}
\DeclareMathOperator*{\expec}{E}

\begin{document}
\title{\textsf{Homework 7 for Pattern Recognition}}
\author{Fan JIN\quad (2015011506)}
\maketitle

\section*{Question 1}
{
    \subsection*{Notations}
    {
        The multidimensional scaling (MDS) seeks to find a centered configuration $X = [x_1, \cdots, x_n]$,
        where $x_1, \cdots, x_n \in \mathbb{R}^2$,
        such that its distance matrix $\hat{D} = (\hat{d}_{ij}) \approx D$ where $$\hat{d}_{ij}^2 = (x_i - x_j)^T (x_i - x_j)$$ for $1 \leq i \leq n$ and $1 \leq j \leq n$.

        Define the Gram matrix $$B = (b_{ij}) = X^T X,$$
        and it follows that $$\hat{d}_{ij}^2 = x_i^T x_i + x_j^T x_j - 2 x_i^T x_j = b_{ii} + b_{jj} - 2 b_{ij}.$$

        By algebra manipulations\footnote{https://en.wikipedia.org/wiki/Multidimensional\_scaling},
        we can express $b_{ij}$ in terms of $d_{ij}$:
        $$\hat{B} = -\frac{1}{2} J \hat{D} J,$$
        where $$J = I_n - \frac{1}{n} 1 \cdot 1^T.$$

        Thus, we take $B$ as an estimate of $\hat{B}$, by reducing the dimension using the PCA approach.
        Apply eigen decomposition and we obtain
        $$B = V * D * V^T,$$ where the eigenvalues are sorted in descending order.
        Retain the first two eigenvalues:
        $$\hat{B} = \hat{V} * \hat{D} * \hat{V}^T,$$
        where $\hat{V}$ is the first two columns of $V$, and $\hat{D}$ is the first two eigenvalues of the diagonal matrix $D$. Then, we have
        $$X = \hat{D}^{1/2} \hat{V}^T$$ which satisfies $$B = X^T X.$$
    }

    \subsection*{Visualization}
    {
        \begin{figure}[H]
            \centering
            \includegraphics[width = 1\linewidth]{Q1/result.png}
            \caption{Raw result}
        \end{figure}

        To compare it with the actual satellite maps, we rotate it such that Zhengzhou is located roughly on the north of Wuhan.

        We see the cities in main lines (e.g. Beijing, Zhengzhou, Wuhan) preserve their relative location better, compared to those in branch lines.
        This is mainly because the linearity between distance and traveling duration differs in the railway level. For main line railways, the traveling duration is usually linear with the distance on map, for they have
        \begin{itemize}
            \item routemaps much more straightforward,
            \item fewer stops,
            \item faster train speed.
        \end{itemize}

        It is noticed that the city Hanzhong seems like an outlier. This city, located in the middle of Mount Qinling and Mount Daba, was connected to other cities by a very slow branch line railway, before the construction of highspeed railways. This is why it is far away from other cities in the MDS map, although it is actually not so on the fringe.

        \begin{figure}[H]
            \centering
            \includegraphics[width = 1\linewidth]{Q1/rotated.png}
            \includegraphics[width = 1\linewidth]{Q1/map.jpg}
            \caption{Comparison}
        \end{figure}
    }
}

\section*{Question 2a}
{
    \subsection*{PCA}
    {
        The PCA (principal component analysis) applies eigen decomposition to the covariance matrix (unnormalized) or correlation matrix (normalized) to the original data in $P$ dimensions, and only retains the first $p$ largest eigenvalues and their corresponding eigenvectors, which reconstructs the compressed data in $p$ dimensions.
    }

    \subsection*{t-SNE}
    {
        The basic idea of t-SNE (t-distribution stochastic neighbor embedding) is to map the data to a probability distribution by affine transformation.

        Given $N$ high dimensional objects $x_1, \cdots, x_N$, the t-SNE algorithm first computes a conditional probability,
        which indicates the dissimilarity between two objects,
        $$p_{j|i} = \frac{ \exp{(- \lVert x_i - x_j \rVert^2 / 2\sigma_i^2)} }{ \sum_{k\neq i} \exp{(- \lVert x_i - x_k \rVert^2 / 2\sigma_i^2)} },$$
        where the bandwidth of the Gaussian kernels $\sigma_i$ is set such that the perplexity of the conditional distribution equals a predefined perplexity.
        This can be done using the bisection method. \footnote{https://en.wikipedia.org/wiki/T-distributed\_stochastic\_neighbor\_embedding}

        Given the conditional distributions, the joint distribution of two objects is computed as follows.
        $$p_{ij} = \frac{ p_{j|i} + p_{i|j} }{2N}.$$

        Unlike the classical SNE algorithm, the joint probabilities in low dimension space are computed in t-SNE using the t-distribution (heavy-tailed).
        $$q_{ij} = \frac{ (1 + \lVert y_i - y_j \rVert^2 )^{-1} }{ \sum_{k\neq i} (1 + \lVert y_i - y_j \rVert^2 )^{-1} }.$$

        The final step is to train the low dimensional objects $y_1, \cdots, y_N$ such that the KL distance below between the two distributions is minimized:
        $$\mathrm{KL}(P\Vert Q) = \sum_{i\neq j} p_{ij} \log{\frac{p_{ij}}{q_{ij}}}.$$

    }

    \subsection*{LLE}
    {
        The basic idea of LLE (locally linear embedding) is to map the data to a lower dimensional space while retaining the nonlinear manifold.
        \footnote{https://cs.nyu.edu/~roweis/lle/algorithm.html}
        It hypothesize that each data point is a weighted linear combination of its neighboring objects,
        and that such weights are shared in both original space and lower dimensional space. It consists of 3 steps:

        \begin{itemize}
            \item Calculate the $k$ nearest neighbors using KNN algorithms.

            \item Minimize the reconstruction loss
            $$\epsilon(W) = \sum_{i}{ \lVert X_i - \sum_{j}{W_{ij} X_j} \rVert ^2}$$
            subject to $$\sum_{j} {\frac{ \sum_k C_{jk}^{-1} }{ \sum_{lm} C_{lm}^{-1} }} = 1$$
            where the local convariance matrix $$C_{jk} = (x - \eta_j)^T (x - \eta_k)$$ with $\eta$ is the nearest neighbors of $x$.
        \end{itemize}

    }
}

\section*{Question 2b}
{
    \begin{figure}[H]
        \centering
        \includegraphics[width = 0.8\linewidth]{Q2/t-SNE.png}
        \caption{2D mapped data by t-SNE}
    \end{figure}

    \begin{figure}[H]
        \centering
        \includegraphics[width = 0.8\linewidth]{Q2/LLE_2.png}
        \caption{2D mapped data by LLE with k=2}
    \end{figure}

    \begin{figure}[H]
        \centering
        \includegraphics[width = 0.8\linewidth]{Q2/LLE_3.png}
        \caption{2D mapped data by LLE with k=3}
    \end{figure}

    \begin{figure}[H]
        \centering
        \includegraphics[width = 0.8\linewidth]{Q2/LLE_4.png}
        \caption{2D mapped data by LLE with k=4}
    \end{figure}

    \begin{figure}[H]
        \centering
        \includegraphics[width = 0.8\linewidth]{Q2/LLE_5.png}
        \caption{2D mapped data by LLE with k=5}
    \end{figure}

    \begin{figure}[H]
        \centering
        \includegraphics[width = 0.8\linewidth]{Q2/LLE_6.png}
        \caption{2D mapped data by LLE with k=6}
    \end{figure}

    \begin{figure}[H]
        \centering
        \includegraphics[width = 0.8\linewidth]{Q2/LLE_8.png}
        \caption{2D mapped data by LLE with k=8}
    \end{figure}

    \begin{figure}[H]
        \centering
        \includegraphics[width = 0.8\linewidth]{Q2/LLE_10.png}
        \caption{2D mapped data by LLE with k=10}
    \end{figure}

    \begin{figure}[H]
        \centering
        \includegraphics[width = 0.8\linewidth]{Q2/LLE_12.png}
        \caption{2D mapped data by LLE with k=12}
    \end{figure}

    \begin{figure}[H]
        \centering
        \includegraphics[width = 0.8\linewidth]{Q2/LLE_15.png}
        \caption{2D mapped data by LLE with k=15}
    \end{figure}

    \begin{figure}[H]
        \centering
        \includegraphics[width = 0.8\linewidth]{Q2/LLE_20.png}
        \caption{2D mapped data by LLE with k=20}
    \end{figure}

    \begin{figure}[H]
        \centering
        \includegraphics[width = 0.8\linewidth]{Q2/LLE_25.png}
        \caption{2D mapped data by LLE with k=25}
    \end{figure}

    \begin{figure}[H]
        \centering
        \includegraphics[width = 0.8\linewidth]{Q2/LLE_30.png}
        \caption{2D mapped data by LLE with k=30}
    \end{figure}

    \begin{figure}[H]
        \centering
        \includegraphics[width = 0.8\linewidth]{Q2/LLE_50.png}
        \caption{2D mapped data by LLE with k=50}
    \end{figure}
}

\section*{Question 2c}
{
    According to the author
    \footnote
    {
        Once I have a t-SNE map, how can I embed incoming test points in that map? \\
        t-SNE learns a non-parametric mapping, which means that it does not learn an explicit function that maps data from the input space to the map.
        Therefore, it is not possible to embed test points in an existing map (although you could re-run t-SNE on the full dataset).
        A potential approach to deal with this would be to train a multivariate regressor to predict the map location from the input data.
        Alternatively, you could also make such a regressor minimize the t-SNE loss directly, which is what I did in this paper.
    },
    the incoming test points cannot be embedded in the t-SNE map. There are two workarounds.
    \begin{itemize}
        \item Re-run t-SNE on the full dataset. But the classifier we trained on training set would not work, since the re-run t-SNE map would differ from the previous one.
        \item Train a multivariate regressor to predict the map location.
    \end{itemize}

    In this project, we choose the latter workaround. We use the KNN classifier provided by the Classification Learner Toolbox in MATLAB.
    And we achieve a testing accuracy of 0.983051 in the reduced model in 2 dimensional space, while it is 0.947034 in the full model in 784 dimensional space.
}

\section*{Question 2d}
{
    In section 2b, we experimented the t-SNE algorithm, and the LLE algorithm with various hypermaramter $k$s.

    \begin{itemize}
        \item The LLE requires a hyperparamter $k$, and the performance is quite sensitive to $k$. This means hypertuning is necessary, and a proper $k$ is crucial to the success.
        However, the output is separable within a range from $k=8$ to $k=20$. Many hypervalues are available for the algorithm.

        \item The t-SNE has no hyperparamters. But it has some weakness in that it is impossible to embed incoming test points to a pretrained map.
        When a test point comes in, we have to re-train the model with the training set combined with the testing point, which is time consuming.
    \end{itemize}
}

\section*{Question 3}
{
    \begin{itemize}
        \item Classifier: Medium KNN (Classification Toolbox in MATLAB)
        \item Feature Selection by: KL Divergence
        \item Features to select from: 1000
        \item Validation: 10-fold
        \item Features to select: 20. We experimented from $p=2$ to $p=30$, and $p=20$ has the best performance in terms of validation accuracy.
    \end{itemize}

    The features selected are
    \begin{table}[!hbp]
        \centering
        \begin{tabular}{|c|c|c|c|c|}
        \hline
        361 & 425 & 662 & 166 & 219 \\
        \hline
        329 & 592 & 957 & 663 & 804 \\
        \hline
        542 & 817 & 395 & 281 & 778 \\
        \hline
        720 & 626 & 206 & 844 & 48 \\
        \hline
        \end{tabular}
        \caption{Features selected}
    \end{table}

    \begin{table}[!hbp]
        \centering
        \begin{tabular}{|c|c|c|}
        \hline
        & Full features & 20 selected features \\
        \hline
        Validation accuracy & 0.697500 & 0.797500 \\
        \hline
        \end{tabular}
        \caption{Comparison with 10-fold}
    \end{table}

    By feature selection with KL divergence, the accuracy of Medium KNN increases from 69.75\% to 79.75\%. What an improvement!
}

\section*{Source Code}
{
    Please download the souece code from http://39.106.23.58/files/PR7\_2015011506.7z

    For Question 2, please cd into ``Q1'' directory and run ``main.m''.

    For Question 2, please cd into ``Q2'' directory run ``main.m''. It may take \emph{less than a minute} to train the network, but the result is reproducible because of the random seed.

    For Question 3, please cd into ``Q3'' directory run ``main.m''.

    For each model, I clicked ``Generate code'' button to transcript my operations into MATLAB codes, and stored each of them in the corresponding ``.m'' file. These files include:
    \begin{itemize}
        \item trainClassifierFull.m
        \item trainClassifierReduced.m
    \end{itemize}
    Thus, the steps above can be easily reproduced without using the GUI of the toolbox.

}

\clearpage
\end{document}


\documentclass[12pt]{cdblatex}
\usepackage{exercises}
\usepackage{fancyhdr}
\usepackage{footer}

\begin{document}

% --------------------------------------------------------------------------------------------
\section*{Exercise 3.1 Some symmetries of Riemann}

\begin{cadabra}
   {a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w#}::Indices(position=independent).

   ;::Symbol;

   \partial{#}::PartialDerivative.

   \Gamma^{a}_{b c}::TableauSymmetry(shape={2}, indices={1,2}).

   Rabcd := R^{a}_{b c d} ->   \partial_{c}{\Gamma^{a}_{b d}}
                             - \partial_{d}{\Gamma^{a}_{b c}}
                             + \Gamma^{e}_{b d} \Gamma^{a}_{c e}
                             - \Gamma^{e}_{b c} \Gamma^{a}_{d e}.       # cdb(Rabcd.000,Rabcd)

   dRabcd := R^{a}_{b c d ; e} -> \partial_{e}{R^{a}_{b c d}}
                                + \Gamma^{a}_{f e} R^{f}_{b c d}
                                - \Gamma^{f}_{b e} R^{a}_{f c d}
                                - \Gamma^{f}_{c e} R^{a}_{b f d}
                                - \Gamma^{f}_{d e} R^{a}_{b c f}.       # cdb(dRabcd.000,dRabcd)

\end{cadabra}

\clearpage

% --------------------------------------------------------------------------------------------
\section*{Exercise 3.1 Antisymmetry on last pair of indices}

\begin{cadabra}
   expr := R^{a}_{b c d} + R^{a}_{b d c}.                               # cdb(ex-0301.101,expr)

   substitute (expr, Rabcd)                                             # cdb(ex-0301.102,expr)
\end{cadabra}

\begin{dgroup*}[spread={3pt}]
   \Dmath*{\cdb{ex-0301.101} = \Cdb*{ex-0301.102}}
\end{dgroup*}

\clearpage

% --------------------------------------------------------------------------------------------
\section*{Exercise 3.1 First Bianchi identity}

\begin{cadabra}
   expr := R^{a}_{b c d} + R^{a}_{d b c} + R^{a}_{c d b}.               # cdb(ex-0301.201,expr)

   substitute   (expr, Rabcd)                                           # cdb(ex-0301.202,expr)
   canonicalise (expr)                                                  # cdb(ex-0301.203,expr)
\end{cadabra}

\begin{dgroup*}[spread={3pt}]
   \Dmath*{\cdb{ex-0301.201} = \Cdb*[\hskip2.5cm\hfill]{ex-0301.202}
                             = \Cdb*{ex-0301.203}}
\end{dgroup*}

\clearpage

% --------------------------------------------------------------------------------------------
\section*{Exercise 3.1 Second Bianchi identity}

\begin{cadabra}
   expr := R^{a}_{b c d ; e} + R^{a}_{b e c ; d} + R^{a}_{b d e ; c}.   # cdb(ex-0301.301,expr)

   substitute     (expr, dRabcd)                                        # cdb(ex-0301.302,expr)
   substitute     (expr,  Rabcd)                                        # cdb(ex-0301.303,expr)
   distribute     (expr)                                                # cdb(ex-0301.304,expr)
   product_rule   (expr)                                                # cdb(ex-0301.305,expr)
   sort_product   (expr)                                                # cdb(ex-0301.306,expr)
   rename_dummies (expr)                                                # cdb(ex-0301.307,expr)
   canonicalise   (expr)                                                # cdb(ex-0301.308,expr)
\end{cadabra}

\begin{dgroup*}[spread={3pt}]
   \Dmath*{\cdb{ex-0301.301} = \Cdb*[\hskip2.0cm\hfill]{ex-0301.302}
                             = \Cdb*{ex-0301.303}
                             = \Cdb*{ex-0301.304}}
\end{dgroup*}

\clearpage

\begin{dgroup*}[spread={3pt}]
   \Dmath*{\cdb{ex-0301.301} = \Cdb*[\hskip4.5cm\hfill]{ex-0301.305}
                             = \Cdb*[\hskip4.5cm\hfill]{ex-0301.306}}
\end{dgroup*}

\clearpage

\begin{dgroup*}[spread={3pt}]
   \Dmath*{\cdb{ex-0301.301} = \Cdb*[\hskip3.5cm\hfill]{ex-0301.307}
                             = \Cdb*{ex-0301.308}}
\end{dgroup*}

\end{document}


\title{Probabilistic PCA}

\subsection{Probabilistic PCA}

Probabilistic principal components analysis (PCA) is a
dimensionality reduction technique that
analyzes data via a lower dimensional latent space
\citep{tipping1999probabilistic}. It is often
used when there are missing values in the data or for multidimensional
scaling.

We demonstrate with an example in Edward.
An interactive version with Jupyter notebook is available
\href{http://nbviewer.jupyter.org/github/blei-lab/edward/blob/master/notebooks/probabilistic_pca.ipynb}{here}.

\subsubsection{Data}

We use simulated data. We'll talk about the individual variables and
what they stand for in the next section. For this example, each data
point is 2-dimensional, $\mathbf{x}_n\in\mathbb{R}^2$.

\begin{lstlisting}[language=Python]
def build_toy_dataset(N, D, K, sigma=1):
  x_train = np.zeros((D, N))
  w = np.random.normal(0.0, 2.0, size=(D, K))
  z = np.random.normal(0.0, 1.0, size=(K, N))
  mean = np.dot(w, z)
  for d in range(D):
    for n in range(N):
      x_train[d, n] = np.random.normal(mean[d, n], sigma)

  print("True principal axes:")
  print(w)
  return x_train

N = 5000  # number of data points
D = 2  # data dimensionality
K = 1  # latent dimensionality

x_train = build_toy_dataset(N, D, K)
\end{lstlisting}

\begin{lstlisting}
## True principal axes:
## [[ 0.25947927]
##  [ 1.80472372]]
\end{lstlisting}

We visualize the data set.

\begin{lstlisting}[language=Python]
plt.scatter(x_train[0, :], x_train[1, :], color='blue', alpha=0.1)
plt.axis([-10, 10, -10, 10])
plt.title("Simulated data set")
plt.show()
\end{lstlisting}

\includegraphics[width=450px]{/images/probabilistic-pca-fig0.png}

\subsubsection{Model}

Consider a data set $\mathbf{X} = \{\mathbf{x}_n\}$ of $N$ data
points, where each data point is $D$-dimensional, $\mathbf{x}_n \in
\mathbb{R}^D$. We aim to represent each $\mathbf{x}_n$ under a latent
variable $\mathbf{z}_n \in \mathbb{R}^K$ with lower dimension, $K <
D$. The set of principal axes $\mathbf{W}$ relates the latent variables to
the data.

Specifically, we assume that each latent variable is normally distributed,
\begin{equation*}
\mathbf{z}_n \sim N(\mathbf{0}, \mathbf{I}).
\end{equation*}
The corresponding data point is generated via a projection,
\begin{equation*}
\mathbf{x}_n \mid \mathbf{z}_n
\sim N(\mathbf{W}\mathbf{z}_n, \sigma^2\mathbf{I}),
\end{equation*}
where the matrix $\mathbf{W}\in\mathbb{R}^{D\times K}$ are known as
the principal axes. In probabilistic PCA, we are typically interested in
estimating the principal axes $\mathbf{W}$ and the noise term
$\sigma^2$.

Probabilistic PCA generalizes classical PCA. Marginalizing out the the
latent variable, the distribution of each data point is
\begin{equation*}
\mathbf{x}_n \sim N(\mathbf{0}, \mathbf{W}\mathbf{W}^Y + \sigma^2\mathbf{I}).
\end{equation*}
Classical PCA is the specific case of probabilistic PCA when the
covariance of the noise becomes infinitesimally small, $\sigma^2 \to
0$.

We set up our model below. In our analysis, we fix $\sigma=2.0$, and
instead of point estimating $\mathbf{W}$ as a model parameter, we
place a prior over it in order to infer a distribution over principal
axes.

\begin{lstlisting}[language=Python]
from edward.models import Normal

w = Normal(loc=tf.zeros([D, K]), scale=2.0 * tf.ones([D, K]))
z = Normal(loc=tf.zeros([N, K]), scale=tf.ones([N, K]))
x = Normal(loc=tf.matmul(w, z, transpose_b=True), scale=tf.ones([D, N]))

\end{lstlisting}

\subsubsection{Inference}

The posterior distribution over the principal axes $\mathbf{W}$ cannot
be analytically determined. Below, we set up our inference variables
and then run a chosen algorithm to infer $\mathbf{W}$. Below we use
variational inference to minimize the $\text{KL}(q\|p)$ divergence
measure.

\begin{lstlisting}[language=Python]
qw = Normal(loc=tf.get_variable("qw/loc", [D, K]),
            scale=tf.nn.softplus(tf.get_variable("qw/scale", [D, K])))
qz = Normal(loc=tf.get_variable("qz/loc", [N, K]),
            scale=tf.nn.softplus(tf.get_variable("qz/scale", [N, K])))

inference = ed.KLqp({w: qw, z: qz}, data={x: x_train})
inference.run(n_iter=500, n_print=100, n_samples=10)
\end{lstlisting}

\subsubsection{Criticism}

To check our inferences, we first inspect the model's learned
principal axes.

\begin{lstlisting}[language=Python]
sess = ed.get_session()
print("Inferred principal axes:")
print(sess.run(qw.mean()))
\end{lstlisting}

\begin{lstlisting}
## Inferred principal axes:
## [[-0.24093632]
##  [-1.76468039]]
\end{lstlisting}

The model has recovered the true principal axes up to finite data and
also up to identifiability (there's a symmetry in the
parameterization).

Another way to criticize the model is to visualize the observed data
against data generated from our fitted model.

\begin{lstlisting}[language=Python]
x_post = ed.copy(x, {w: qw, z: qz})
x_gen = sess.run(x_post)

plt.scatter(x_gen[0, :], x_gen[1, :], color='red', alpha=0.1)
plt.axis([-10, 10, -10, 10])
plt.title("Data generated from model")
plt.show()
\end{lstlisting}

\includegraphics[width=450px]{/images/probabilistic-pca-fig1.png}

The generated data looks close to the true data.

\subsubsection{Acknowledgments}

We thank Mayank Agrawal for writing the initial version of this
tutorial.

\subsubsection{References}\label{references}


\section{Trim and linearisation}

\subsection{Trimming}
The first step when designing the control system of an aircraft is to study the behavior of the aircraft due to control inputs or external disturbances from an equilibrium condition. If the aircraft we're not to be in equilibrium, deviation from the initial conditions would occur that are unrelated to the control inputs making the analysis more difficult.

In the case of an aircraft this equilibrium condition is known as a trimmed flight condition. In order to determine the trimmed flight condition the states and inputs much be chosen such that the linear- and angular-accelerations are zero. For this assignment this is done by minimizing the following cost function.

\begin{equation}
    \label{eq:trim_cost}
    cost = 5\dot{h}^2 +
           W_{\phi}\dot{\phi}^2 +
           W_{\theta}\dot{\theta}^2 +
           W_{\psi}\dot{\psi}^2 +
           2\dot{V_{tot}}^2 +
           10\dot{\alpha}^2 +
           10\dot{\beta}^2 +
           10\dot{P}^2 +
           10\dot{Q}^2 +
           10\dot{R}^2
\end{equation}

All the flight states in the cost function are squared, which is what allows the trim condition to be found by minimizing it. Once the cost function returns zero, the trimmed flight condition states and inputs have been found.

Performing the trimming procedure for 5 iterations in level flight for both flight conditions and both the high and low fidelity models yields the following final values for the cost function.

\begin{center}
    \begin{tabular}{ r | c | c }
                               & high fidelity & low fidelity \\ \hline \hline
     Assigned flight condition & $0.0506$ & $4.2997\cdot10^{-29}$ \\
     APA flight condition      & $7.1856\cdot10^{-6}$ & $5.1572\cdot10^{-29}$
    \end{tabular}
\end{center}

The results for the low fidelity models are smaller than the machine epsilon $2.2204\cdot10^{-16}$ of the computer used to calculate these, so it's safe to assume the trimmed flight condition has been successfully found.

The final costs for the high fidelity model are higher. The results for the APA flight conditions are in the order of $10^{-6}$ and for the assigned flight conditions in the order of $10^{-2}$. This indicates that the resulting state and inputs values are not perfect, but since the cost is close to zero, it may be close enough for further analysis.

\subsection{Accelerometer Position Analysis}
No changes are seen in the $A$ and $B$ matrix after adding the new vertical accelerometer output to the Simulink model. This is expected since the addition of the accelerometer output didn't change the dynamics of the aircraft. However the C and D matrices now have an extra row which corresponds to the new output. Equation~\ref{eq:anss} shows the linearized output equation of the normal acceleration at the center of gravity ($x_a=0$).

\begin{equation}
    \label{eq:anss}
    y = \begin{bmatrix}
        0 \\ 0 \\ -3.24322220018077e-05 \\ 0 \\ -9.67969758700180e-06 \\ 0 \\
        0.00398736185031356 \\ 9.92978181907083 \\ 0 \\ 0 \\ 0.966415396225217 \\
        0 \\ 0 \\ 0.0208407616972783 \\ 0 \\ 0 \\ 0 \\ 0
        \end{bmatrix}^T x +
        \begin{bmatrix}
        0 & 0 & 0 & 0
    \end{bmatrix} u
\end{equation}

The accelerometer output primarily depends on the velocity, angle of attack, pitch rate and the normal load factor. The altitude and pitch angle also have a minor contribution, however since these are small it could be caused by errors in the model or linearizion process. Equation~\ref{eq:tf_el_an} shows the transfer function from the elevator to normal acceleration.

\begin{equation}
    \label{eq:tf_el_an}
    \frac{
    \begin{matrix}
        0.421 s^{17} + 22.9 s^{16} + 404.6 s^{15} + 1728 s^{14} - 2.376e04 s^{13} - 3.177e05 s^{12} \\
        - 1.64e06 s^{11} - 5.127e06 s^{10} - 1.168e07 s^{9} - 1.574e07 s^{8} - 7.901e06 s^{7} \\
        - 5.914e04 s^{6} +  746.3 s^{5} + 4.87 s^{4} - 1.464e-11 s^{3}
    \end{matrix}
    }{
    \begin{matrix}
        s^{18} + 80.51 s^{17} + 2581 s^{16} + 4.234e04 s^{15} + 3.876e05 s^{14} + 2.115e06 s^{13} \\
        + 7.644e06 s^{12} + 2.059e07 s^{11} + 3.952e07 s^{10} + 5.029e07 s^{9} + 4.057e07 s^{8} \\
        + 1.553e07 s^{7} + 5.631e05 s^{6} + 1.073e05 s^{5} + 1159 s^{4} - 8.86e-11 s^{3}
    \end{matrix}
    }
\end{equation}

The transfer function has a zero on the right hand side of the imaginary axis at $9.76+0j$. This zero is the one responsible for the non-minimum-phase behavior. The physical explanation of this is that a pitch up maneuverer with the elevator produces a downwards force which causes the aircraft to accelerate downwards. Eventually the angle of attack will increase as the nose pitches up and the extra lift will accelerate the aircraft upwards.

\begin{figure}[ht]
    \centering
    \includegraphics[width=0.6\textwidth]{figures/an_elev_step}
    \label{fig:an_elev_step}
    \caption{Normal acceleration after a negative step input on the elevator.}
\end{figure}

Repeating the simulation for increasing values of $x_a$ shows that the zero moves away from the origin indicating that its influence becomes less dominant. When $x_a=5.9$ the zero has moved to the left side of the imaginary axis and the non-minimum-phase behavior disappears. Since the value of the zero much further to the left then the other zeros and poles, its motion is negligible. Increasing $x_a$ further moves the zero closer to origin and its motion reappears, this time in the direction of the reference signal.

\begin{figure}[ht]
    \centering
    \includegraphics[width=0.6\textwidth]{figures/an_elev_step_mult}
    \caption{Normal acceleration after a negative step input on the elevator for different values of $x_a=5.9$.}
    \label{fig:an_elev_step_mult}
\end{figure}

One property of the \emph{instantaneous center of rotation} is that the linear accelerations at that point are the same as that of the overall aircraft. A point in front of it will accelerate upwards with a pitch up manoeuvre and a point behind it will accelerate downwards during the same manoeuvre. Thus based on the effects of moving $x_a$, it fair to conclude that the instantaneous center of rotation must be near $x=5.9$. Another way to arrive at the same conclusion is that the acceleration due to rotation is caused by the zero mentioned in the previous section since this is the only zero that changes significantly with $x_a$. Thus the instantaneous center of rotation will be at the location where this zero is infinitely far away from the origin.

The pilot should be placed at or after the instantaneous center of rotation ($x_a\geq5.9\ ft$), this is because if he is placed before, he will initially feel the aircraft moving into the opposite direction he is commanding it to and this might confuse the pilot resulting in him compensating for an error that is not there. This could make the combined human-aircraft system unstable or at least difficult to control.

It is important to place the accelerometer close a node of the most important fuselage bending moment. The reason is because, as the aircraft flies, loads and vibrations acting on the aircraft will make the fuselage bend. The nodes are locations where the displacements due to vibrations or bending are zero. Thus placing the accelerometer far away from a node, will result in the accelerometer measuring extra accelerations due to vibrations or bending.

\clearpage


%!TEX root = head-full.tex

%  On one hand, the development of the broadband services greatly
% improves the user experience of

\section{Introduction} \label{sec:introduction}

\para{Markov Chain}
A Markov Chain is a special stochastic process in which the current state only depends on its previous state, we call this property the Markov property or memorylessness. i.e.
\begin{equation}
P(X_{t+1}=x|X_t, X_{t-1}, \cdots) =P(X_{t+1}=x|X_t)
\end{equation}

Given a Markov Chain, if a vector $\pi$, has the property:
\begin{equation}
\pi=\pi P
\end{equation}
then we call $\pi$ the stationary distribution, it denotes the final state distribution of the stochastic process in a Markov Chain.

Markov Chain has a wide application in many fields in the real world, such as social science\cite{acemoglu2011political}, econmics \& finance\cite{hamilton1989new} and of course, computer science\cite{page1999pagerank}, etc.

\para{Markov Chain Monte Carlo}
If we are given a probability distribution $p(x)$, it would be a great thing if we could generate some samples of it by a simple method, so here comes the Markov Chain Monte Carlo(MCMC) method. If we could construct a Markov Chain which its stationary distribution $\pi$ just equals to $p(x)$, then we could use this Markov Chain to sample from this distribution $p(x)$. And this is the main idea of MCMC.

In this paper, we implement the Metropolis-Hastings Algorithm\cite{metropolis1953equation,hastings1970monte}, which is one of the most widely used sampling method. We also deal with the accepting rate, which I will introduce later, for previous work\cite{roberts1997weak} have shown that the accepting rate may influence the result of the experiment and there is theoretical support in choosing an optimal accepting rate.

\para{Restricted Boltzmann Machine}
A Restricted Bolztmann Machine (RBM)\cite{mcclelland1987parallel} is a significant work bringing theory in statistical physics to computer science. By stacking several layers of RBM, we will get a fundamental model, Deep Belief Network\cite{hinton2006reducing}, in the field of Deep Learning, which is nowadays the hottest class of algorithms used in Machine Learning.

\para{Estimating Partition functions}
In the process of training an RBM, however, will include incontractable computation of the partition function. When the model grows large, the complexity of this work will be incompletable. The good news is, researches have shown that there are ways to avoid this by using an MCMC approach instead, to estimate it.

In this paper, we implement three prevalent MCMC methods of estimating a partition function, Thouless-Anderson-Palmer Sampling(TAP)\cite{gabrie2015training}, Annealed Importance Sampling (AIS)\cite{neal2001annealed,salakhutdinov2009learning}, Rao-Blackwellized Tempered Sampling(RTS)\cite{carlson2016partition}, respectively, and give an overall comparison on the theory \& performance between them.


\chapter{BSS}

\section{Mixing and demixing}

The two sources are mixed as follows:
\begin{align}
  y_1(t) & = s_1(t) + s_2(t) \\
  y_2(t) & = s_1(t+\delta_1) + s_2(t+\delta_2)
\end{align}
where $\delta_1$ and $\delta_2$ are delays that depend on the distance
between the microphones and the directions of the sources.

In the frequency domain:
\begin{align}
  Y_1(\omega) & = S_1(\omega) + S_2(\omega) \\
  Y_2(\omega) & = e^{-\omega\delta_1}S_1(\omega) + e^{-\omega\delta_2}S_2(\omega)
\end{align}

This is a scalar mixture! So we can just do PCA and we are done. Not
so fast. There is a permutation problem. There is no guarantee that
signal 1 will appear on ouput channel 1, or signal 2 on output channel
2. If we were to apply PCA to all the different frequencies then we
would still have the problem of determining which output should be on
which channel. If there are $N$ different frequencies then there will
be $2^N$ different possibilities to choose from.

In radar, there is only one frequency so this doesn't matter so we can
use PCA. Speech is broadband. We need a different approach.

Let us write the delay operation as follows:
\begin{align}
  z_1 & =  e^{-\omega\delta_1}\\
  z_2 & =  e^{-\omega\delta_2}
\end{align}

We can write the mixing process as:
\begin{equation}
  \begin{bmatrix}
    Y_1(\omega)\\
    Y_2(\omega)
  \end{bmatrix}
  =
  \begin{bmatrix}
  1 & 1\\
  z_1 & z_2
  \end{bmatrix}
  \begin{bmatrix}
  S_1(\omega)\\
  S_2(\omega)
    \end{bmatrix}
\end{equation}

The inverse is
\begin{equation}
  \begin{bmatrix}
    S_1(\omega)\\
    S_2(\omega)
  \end{bmatrix}
  =
  \frac{1}{z_2-z_1}
  \begin{bmatrix}
  z_2 & -1\\
  -z_1 & 1
  \end{bmatrix}
  \begin{bmatrix}
  Y_1(\omega)\\
  Y_2(\omega)
    \end{bmatrix}
 \end{equation}

We are going to ignore the determinant term because of the possibility of dividing by zero (or by some number close to zero).

\begin{equation}
  \begin{bmatrix}
    X_1(\omega)\\
    X_2(\omega)
  \end{bmatrix}
  =
  \begin{bmatrix}
  z_2 & -1\\
  -z_1 & 1
  \end{bmatrix}
  \begin{bmatrix}
  Y_1(\omega)\\
  Y_2(\omega)
    \end{bmatrix}
 \end{equation}

So, if we could just find $\delta_1$ and $\delta_2$, we would demix as follows:
\begin{align}
  x_1(t) & = y_1(t+\delta_2) - y_2(t) \\
  x_2(t) & = -y_1(t+\delta_1) + y_2(t)
\end{align}

\section{The cost function}

We know that $x_1(t)$ and $x_2(t)$ are uncorrelated. That means that
we should find $\delta_1$ and $\delta_2$ that minimize the correlation
between $x_1(t)$ and $x_2(t)$.

Jumping back into the frequency domain, we need to minimize the
magnitude of $X_1(\omega) \overline{X_2(\omega)}$. We take the complex
conjugate because this is not a convolution - it's a correlation:
\begin{align}
  X_1(\omega) \overline{X_2(\omega)} & = \left(z_2 Y_1(\omega) - Y_2(\omega)\right) \overline{\left(-z_1 Y_1(\omega) + Y_2(\omega)\right) }\\
  & = -z_2 \overline{z_1} Y_1 \overline{Y_1} + z_2 Y_1 \overline{Y_2} + \overline{z_1} Y_2 \overline{Y_1} - Y_2 \overline{Y_2}
\end{align}

We sum the square of the magnitude of $X_1(\omega)
\overline{X_2(\omega)}$ over all frequencies and call that the cost
function.  Summing the square of the magnitude has another
advantage. According to Parseval's Theorem, the cost function is
identical to the sum of the squares of the cross correlations between
$x_1(t)$ and $x_2(t)$ in the time domain. This is a nice result
because the time domain justification for this cost function is easy
to understand.

\subsection{Cost function details}

As always, the devil is in the details:

\begin{itemize}

\item The cost function is multimodal. That's a bad thing if we are
  using a standard optimization routine because, depending on starting
  position, it might not find the best solution at the global
  minimum. Beware the local minimum. To mitigate against this, we do a coarse
  grid search first. And then use an optimizer to clean up the answer.

\item We take the logarithm of the sum of squares. This is done for the sake of numerical stability because the cost function changes very quickly.

  \item Note that only $z_1$ and $z_2$ are functions of $\delta_1$ and $\delta_2$. We can precompute auto- and cross-correlations of  $Y_1 \overline{Y_1}$, $Y_1 \overline{Y_2}$, $Y_2 \overline{Y_1}$, and $Y_2 \overline{Y_2}$ in advance to save computation.

\end{itemize}


\subsection{Monte Carlo method}
There are complex problems in a variety of research fields which could take up years or even centuries to compute with simple deterministic methods. For some problems there is an algorithmic solution which makes it possible to cut down computation time significantly, but when no deterministic algorithm is available to speed up the process an empirical probabilistic method might be able to approximate the desired results.

With the Monte Carlo method random samples are taken from the parameter space ,that describe a data set, and fed into a model which produces a potential outcome. By repeating the process more results are generated until at some point the data can display a pattern that describes the outcome. The result is a quantified probability which describes the chance that something might occur based on the quantity of occurrence generated by the model \cite{stephanie_monte_2015,wikipedia_monte_2019,wikipedia_monte-carlosimulatie_2018}.
%Monte Carlo Simulation / Method, Stephanie
%Monte Carlo method, Wikipedia
%Monte-Carlosimulatie, Wikipedia
\newline
\newline
The Monte Carlo methods can differ depending on the algorithm and application in which it is used, but in summary most implementations will follow a general pattern \cite{wikipedia_monte_2019}.
%Monte Carlo method, Wikipedia
\begin{enumerate}
	\setcounter{enumi}{-1}
	\item Construct a model which is able to describe an outcome of the problem.
	\item Define the space of which inputs can be used by the model to get an outcome (creating a parameter space).
	\item Use the model to generate results based on random sampled input from the parameter space.
	\item Order and determine which results are part of a certain outcome and draw conclusions on the generated statistical evidence.
\end{enumerate}

\label{subsec:Monte_Carlo_Method}

\subsection{The use of the Monte Carlo method and its pitfalls}
Thet MOnte Carlo method is widely used within various applications in different fields of science but is suitable for problems. Where all inputs are known but it is too inefficient to compute the problem with a deterministic method. Problems that require uncertainty to be incorporated into the analysis or prediction and is suitable for discovering new parameters in a model to improve the current model. Monte Carlo is not good at solving problems where answers are known but the inputs are not and in solving problems where no uncertainty should be in the answer produced.

All  mentioned  problems it can solve all tend to rely on significant quantities of data which makes it a relative time consuming process for generating results. Meaning of the generated result is highly depended on the model and random sampling technique that is used and both contribute to biases in the results \cite{stephanie_monte_2015,wikipedia_monte_2019,alon_honig_introduction_nodate}.
%Monte Carlo Simulation / Method, Stephanie
%Monte Carlo method, Wikipedia
%Introduction to Monte Carlo Methods, Alon Honig


\section{Graphs}

\begin{code}{Topological Sort}{$\mathcal{O}(|V| + |E|)$}{graphs/topoSort.cc}
  A priority queue can be used if further sorting is necessary.
\end{code}

\lst{SCC Tarjan}{$\mathcal{O}(|V| + |E|)$}{graphs/scc.cc}
\lst{Articulation Points}{$\mathcal{O}(| V | + | E |)$}{graphs/articulationPoints.cc}
\lst{Bridges}{$\mathcal{O}(|V|+|E|)$}{graphs/bridges.cc}

\lst{Minimal Spanning Tree -- Kruskal}{$\mathcal{O}(|E|\log|V|)$}{graphs/kruskal.cc}

\subsection{Shortest Paths}

\lst{Dijkstra}{$\mathcal((|E| + |V|)\log|V|)$}{graphs/dijkstra.cc}

\begin{code}{Bellman Ford}{$\mathcal{O}(|E||V|)$}{graphs/bellmanFord.cc}
	Check for negative cycles: \\
	$dist$ still changes in a $\lvert V \rvert$'th relaxation step with $dist_i = 0$ initially for all $i$.
\end{code}

\begin{code}{Bellman Ford with Queue}{$\mathcal{O}(|E||V|)$}{graphs/bellmanFordQueue.cc}
  This approach may be faster
\end{code}

\lst{Floyd Warshall}{$\mathcal{O}(|V|^3)$}{graphs/floydWarshall.cc}

\subsection{Forest}
\lst{Lowest Common Ancestor}{build: $\mathcal{O}(|V| \log|V|)$ query: $\mathcal{O}(1)$}{graphs/LCA.cc}
\lst{LCA with binary lifting}{build: $\mathcal{O}(|V|\log |V|)$ query: $\mathcal{O}(\log|V|)$}{graphs/LCABL.cc}

\lst{Heavy-light decomposition}{build: $\mathcal{O}(|V|)$, query/update: $\mathcal{O}(\log^2|V|)$/$\mathcal{O}(\log|V|)$}{graphs/HLD.cc}

\subsection{Flow}

\textbf{Max-flow min-cut theorem.} The maximum value of an $s$-$t$ flow is
equal to the minimum capacity over all $s$-$t$ cuts

\lst{Edges for flow algorithms}{}{graphs/flowedge.cc}
\lst{Edmonds Karp}{$\mathcal{O}(|V||E|^2)$}{graphs/edmondsKarp.cc}
\lst{Dinic}{$\mathcal{O}(|V|^2|E|)$}{graphs/dinic.cc}
\lst{Push Relabel}{$\mathcal{O}(|V|^3)$}{graphs/pushRelabel.cc}

\subsubsection{Minimum s-t cut}

To find a minimal $s$-$t$ cut find all nodes that are reachable in the
residual network for a network w/ maximum flow from $s$.  This is the
$s$ part of the cut.  All other nodes belong to the $t$ part.

\subsubsection{Closure Problem}

A closure of a directed graph is a set of vertices with no outgoing
edges.  The closure problem is the task to find the maximum weighted
closure.  Solvable through reduction to a maximum flow problem: Add
source and target, connect all the vertices with positive weight $w$
to the source with capacity $w$ and connect all the vertices with
negative weight $w$ to the target with capacity $-w$.  All of the
edges in the original graph have infinite capacity in the new
graph.  The weight of the maximum weighted closure is equal to the sum
of all positive weighted vertices in the original graph minus the
maximum flow in the constructed graph.