\section*{Bayesian teting and ``interval'' estimation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{The duality between estimation and testing}
Similarly to the frequentist case, in Bayesian inference there is an intimate relationship between testing hypotheses an estimating measurable functions of the parameters.
\begin{defn}[Test]
 Consider a statistical model $f(x \mid \theta)$  with $\theta \in \boldsymbol{\Theta}$.
 Given $\boldsymbol{\Theta}_0 \subset \boldsymbol{\Theta}$, a \textit{test} consists in answering the question of whether
 $$ H_0 : \theta \in \boldsymbol{\Theta}_0 $$
 is true.
 We call $H_0$ the \textit{null hypothesis} and $\boldsymbol{\Theta}_0$ can often be a point, i.e. $\boldsymbol{\Theta}_0 = \{\theta_0 \}$.
\end{defn}
Notice that $\mathbb{I}_{\boldsymbol{\Theta}_0}(\theta)$ is measurable and thus we can define, for instance
\begin{equation*}
L_1(\theta, \varphi) = \begin{cases}
1, \varphi = \mathbb{I}_{\boldsymbol{\Theta}_0}(\theta),\\
0, \: \text{otherwise},
\end{cases}
\end{equation*}
which in turn leads to
\begin{equation*}
\varphi_1 = \begin{cases}
1, \pr(\theta \in \boldsymbol{\Theta}_0 \mid x) > \pr(\theta \in \boldsymbol{\Theta}_0^c \mid x),\\
0, \: \text{otherwise}.
\end{cases}
\end{equation*}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{A refinement}
 The loss function just seen can be refined to
 \begin{equation*}
L_2(\theta, \varphi) = \begin{cases}
0, \varphi = \mathbb{I}_{\boldsymbol{\Theta}_0}(\theta),\\
a_0, \theta \in \boldsymbol{\Theta}_0, \varphi = 0\\
a_1, \theta \in \boldsymbol{\Theta}_0^c, \varphi = 1.
\end{cases}
\end{equation*}
Under this loss, we have
\begin{equation*}
\varphi_2 = \begin{cases}
1, \pr(\theta \in \boldsymbol{\Theta}_0 \mid x) >  a_1/(a_0 + a_1),\\
0, \: \text{otherwise}.
\end{cases}
\end{equation*}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Example}
\begin{example}[\textit{One} Normal test]
  Take, for example, $x \sim \operatorname{Normal}(\theta, \sigma^2)$, with $\theta \sim \operatorname{Normal}(\mu_0, \tau^2)$.
This implies $\theta \mid x \sim \operatorname{Normal}(\mu(x), \omega^2)$, where
\begin{align*}
 \mu(x) &= \frac{\sigma^2\mu_0 + \tau^2x}{\sigma^2 + \tau^2};
 \omega^2 = \frac{\sigma^2\tau^2}{\sigma^2 + \tau^2}.
\end{align*}
To test $H_0: \theta < 0$, we can compute
\begin{align*}
 \pr(\theta < 0 \mid x) &= \pr \left(\frac{\theta-\mu(x)}{\omega} < \frac{\mu(x)}{\omega}\right), \\
 &= \Phi\left(\frac{-\mu(x)}{\omega}\right).
\end{align*}
This means that if $z_{a_0, a_1}$ is such that $\Phi(z_{a_0, a_1}) = a_1/(a_0 + a_1)$, we can accept $H_0$ if
$$\mu(x) < -z_{a_0, a_1}\omega. $$
\end{example}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Bayes factors}
A central tool in Bayesian testing is the \textbf{Bayes factor} -- see \cite{Kass1995} for a review and guide for interpretation.
\begin{defn}[Bayes factor]
 \label{def:Bayes_factor}
 The Bayes factor is the ratio of posterior odds and the prior odds over the null and the alternative:
 \begin{align*}
  B^\pi_{01}(x) &= \frac{\pr(\theta \in\boldsymbol{\Theta}_0 \mid x)}{\pr(\theta \in\boldsymbol{\Theta}_1 \mid x)}\bigg/\frac{\pr(\theta \in\boldsymbol{\Theta}_0)}{\pr(\theta \in\boldsymbol{\Theta}_1)},\\
  &= \frac{\pr(\theta \in\boldsymbol{\Theta}_0 \mid x)\cdot\pr(\theta \in\boldsymbol{\Theta}_1)}{\pr(\theta \in\boldsymbol{\Theta}_1 \mid x)\cdot\pr(\theta \in\boldsymbol{\Theta}_0)}.
 \end{align*}
\begin{remark}
 When $\boldsymbol{\Theta}_0 = \{\theta_0\}$ and  $\boldsymbol{\Theta}_1 = \{\theta_1\}$ the Bayes factor simplifies to
 \begin{equation*}
  r_{01}(x) = \frac{f(x\mid\theta_0)}{f(x\mid\theta_1)},
 \end{equation*}
 also known as the \textbf{likelihood ratio}.
\end{remark}
\end{defn}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{A few more considerations on the Bayes factor}
The Bayes factor can also be written as
\begin{align*}
  B^\pi_{01}(x) &=  \frac{\int_{\boldsymbol{\Theta}_0} f(x\mid t)\pi_0(t)\,dt }{\int_{\boldsymbol{\Theta}_1} f(x\mid t)\pi_1(t)\,dt} = \frac{m_0(x)}{m_1(x)},
\end{align*}
where $\pi_0$ and $\pi_1$ are the prior distributions under each hypothesis.
Also, if $\hat{\theta}_0$ and $\hat{\theta}_1$ are the MLE under each hypothesis, by making $\pi_0$ and $\pi_1$ Dirac masses at $\hat{\theta}_0$ and $\hat{\theta}_1$, respectively, we recover
\begin{equation}
\label{eq:bayes_lrt}
 R(x) = \frac{\sup_{\theta \in \boldsymbol{\Theta}_0}f(x \mid \theta)}{\sup_{\theta \in \boldsymbol{\Theta}_1}f(x \mid \theta)}
\end{equation}
\begin{exercise}[Bayesian justifcation of LRT]
 Does (\ref{eq:bayes_lrt}) offer a Bayesian justifcation for likelihood ratios?
\end{exercise}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Testing point-null hypotheses}
Hypotheses of the form $H_i : \theta \in \{ \theta_i \}$, called point-null hypotheses, are hard to deal with from a probabilistic point of view.
\begin{remark}[Point-null hypotheses under continuous priors]
 Point-null cannot be tested under continuous prior distributions.
 More generally, if either $H_0$ or $H_1$ are \textbf{impossible} \textit{a priori}, then no amount of data can change that belief.
\end{remark}
\begin{idea}[Cromwell's law\footnote{This idea is attributed to British statistician Dennis Lindley (1923-2013), one of the founders of modern Bayesian theory.}]
 In general, one not assign probability zero to events that are not logically or physically demonstrably impossible.
 Or, more eloquently, as  Oliver Cromwell writes to the General Assembly of the Church of Scotland on 3 August 1650:
 \begin{quotation}
  I beseech you, in the bowels of Christ, think it possible that you may be mistaken.
 \end{quotation}
\end{idea}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Point-null hypotheses: modification of the prior}
Testing point-null hypotheses involves a \textbf{modification of the prior}
 If $H_0: \theta \in \{\theta_0\}$  we can write $\rho_0 = \pr(\theta = \theta_0)$ and then
 \begin{equation*}
  \tilde{\pi}(\theta) = \rho_0 \mathbb{I}_{\boldsymbol{\Theta}_0}(\theta) + (1-\rho_0)\pi_1(\theta),
 \end{equation*}
is our new prior, where $\pi_1$ is the distribution with density $g_1(\theta) \propto \pi(\theta)\mathbb{I}_{\boldsymbol{\Theta}_1}(\theta)$ with respect to the dominating measure on $\boldsymbol{\Theta}_1$.
This gives a posterior probability
\begin{equation*}
 \tilde{\pi}(\boldsymbol{\Theta}_0 \mid x) = \frac{f(x \mid \theta_0)\rho_0}{f(x \mid \theta_0)\rho_0 + (1-\rho_0)m_1(x)}.
\end{equation*}
where $m_1(x) = \int_{\boldsymbol{\Theta}_1} f(x \mid t)g_1(t)\,dt$.
It can be shown that
\begin{equation*}
 \tilde{\pi}(\boldsymbol{\Theta}_0 \mid x) = \left[1 + \frac{1-\rho_0}{\rho_0}\frac{1}{B^\pi_{01}(x)}\right]^{-1},
\end{equation*}
which makes clear the relationship between posterior probabilities and Bayes factors.
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Example}
Consider $x \sim \operatorname{Binomial}(n, p)$ and consider testing $H_0: p = 1/2$ against $H_1: p \neq 1/2$.
Taking $g_1(p) = 1$, we have
\begin{equation*}
  \tilde{\pi}(\boldsymbol{\Theta}_0 \mid x) = \left[1 + \frac{1-\rho_0}{\rho_0}2^n B(x+1, n-x+1)\right]^{-1}.
\end{equation*}
 \begin{center}
 \includegraphics[scale=0.45]{figures/posterior_prob_half.pdf}
\end{center}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Testing with improper priors}
 \begin{idea}[Bayesian hypothesis testing with improper priors]
  No. Just... No.
 \end{idea}
See~\cite{Degroot1973} for the many reasons why this is just a bad idea.
If you insist, please see Section 5.2.5 in \cite{Robert2007} and references therein.
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{An interesting little paradox}
\begin{idea}[The Jeffreys-Lindley paradox]
Consider $x \sim \operatorname{Normal}(\theta, \sigma^2)$ with $\sigma^2$ known and suppose we are interested in testing $H_0: \theta = \theta_0$ against $H_1: \theta \neq \theta_0$.
We can summarise the data using the sample mean $\bar{x}$ and then compute $t_n = \sqrt{n}(\bar{x}-\theta_0)/\sigma$.
Employing a conjugate prior $\theta \sim \operatorname{Normal}(\mu_0, \sigma^2)$,
the Bayes factor is
\begin{equation*}
B_{01}(\boldsymbol{x}) = \sqrt{1 + n}\exp\left(-\frac{nt_n^2}{2(1+n)}\right),
\end{equation*}
which goes to infinity with $n$, while the p-value:
\begin{equation*}
 p(t_n) = 1-2\Phi(|t_n|),
\end{equation*}
is constant in $n$.
In practice this means that, for instance $t_n = 1.96$ and $n = 16, 818$, we have 95\% frequentist confidence that $\theta \neq \theta_0$ whilst \textbf{at the same time} having 95\% belief that $\theta = \theta_0$.
\end{idea}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Another look at principled Bayesian testing}
 Before we were doing
 \begin{equation*}
  L_3(\theta, \varphi) = |\varphi - \mathbb{I}_{\boldsymbol{\Theta}_0}(\theta)|.
 \end{equation*}
But considering a strictly convex loss such as the quadratic loss
 \begin{equation*}
  L_4(\theta, \varphi) = \left(\varphi - \mathbb{I}_{\boldsymbol{\Theta}_0}(\theta)\right)^2,
 \end{equation*}
 leads to better (more adaptable) estimators in general.
 For instance, the Bayes estimator under $L_4$ is
 \begin{equation*}
  \varphi_\pi(x) = \pr(\theta \in  \boldsymbol{\Theta}_0 \mid x).
 \end{equation*}
 \end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Credibility regions}
After all of this work, we are finally ready to define credibility regions, the main object in Bayesian interval estimation.
\begin{defn}[Credibility region]
For a prior $\pi$, a set $C_x$ is called an $\alpha$-credible set if
\begin{equation*}
 \pr(\theta \in C_x \mid x) \geq 1-\alpha.
\end{equation*}
We call $C_x$ a highest posterior density (HPD) $\alpha$-credible region if
\begin{equation*}
 \left\{\theta : p(\theta \mid x) > k_\alpha \right\} \subset C_x \subset \left\{\theta : p(\theta \mid x) \geq k_\alpha \right\},
\end{equation*}
subject to the restriction that
\begin{equation*}
 \pr(\theta \in C_x^\alpha) \geq 1-\alpha.
\end{equation*}
\end{defn}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{A couple remarks}
Credibility regions have a few desirable properties that make them quite attractive as ``interval'' estimates.
\begin{remark}[No randomisation]
 One nice feature of credibility regions for discrete distributions is that, contrary to the frequentist approach, no randomisation is needed to attain a certain level $\alpha$.
\end{remark}
Also,
\begin{remark}[Improper priors and credibility regions]
 In principle, the use of improper priors poses no problem for the derivation of credibility regions.
\end{remark}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Credibility regions: Example I}
Sometimes we will be able to provide Bayesian justifcation for frequentist confidence regions/intervals.
\begin{example}[Credibility intervals for the variance in the Normal]
\label{ex:cred_var_normal_Jeffreys}
Consider $\boldsymbol{x} = \{ x_1, \ldots, x_n \}$, $x_i \sim \operatorname{Normal}(\theta, \sigma^2)$, with both parameters unknown.
Consider
$$ \pi(\theta, \sigma^2) \propto \frac{1}{\sigma^2}. $$
Make $s^2 = \sum_{i=1}^n (x-\bar{x})^2$.
It can be shown that $p(\sigma^2 \mid  s^2) \equiv \operatorname{Gamma}(\sigma^2; (n-1)/2, s^2/2)$.
In particular, this implies
\begin{equation*}
 \frac{s^2}{\sigma^2} \mid \bar{x} \sim \operatorname{Chi-square}(n-1),
\end{equation*}
which the attentive student will notice leads to the same solution as the classical confidence approach.
\end{example}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Credibility regions: Example II}
\begin{example}[HPD for the normal mean]
\label{ex:cred_mean_normal_Jeffreys}
 Consider again the setting of example~\ref{ex:cred_var_normal_Jeffreys}.
 Define $\bar{s}^2 = s^2/(n-1)$ and take $t = F_{\text{Student}}^{-1}(\alpha; n-1)$.
 The classical ``T'' interval,
 \begin{equation*}
  C_t(\bar{x}, \bar{s}^2) = \left(\bar{x} - t\sqrt{\frac{\bar{s}^2}{n}}, \bar{x} + t\sqrt{\frac{\bar{s}^2}{n}}\right),
 \end{equation*}
 is a HPD region under the Jeffreys's prior.
 Again, we can show that
 \begin{equation*}
  \sqrt{n}\frac{\theta -\bar{x}}{\sqrt{\bar{s}^2}} \mid \bar{x}, \sqrt{\bar{s}^2} \sim \operatorname{Student-t}(n-1).
 \end{equation*}
\end{example}

\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{A little decision theory can't hurt... Or can it?}
 Consider the loss
 \begin{equation*}
  L_1(C, \theta) = \operatorname{vol}(C) + (1-\mathbb{I}_{C}(\theta))a,
 \end{equation*}
 which leads to the risk
 \begin{equation*}
  R(C_x, \theta) = E[\operatorname{vol}(C_x)] + \pr(\theta \notin C_x).
 \end{equation*}
Under this loss, the interval in Example~\ref{ex:cred_mean_normal_Jeffreys} is dominated by
\begin{equation*}
 C_t^\prime(\bar{x}, \bar{s}^2) = \begin{cases}
C_t(\bar{x}, \bar{s}^2), \sqrt{\bar{s}^2} < \sqrt{n}c/(2t),\\
\{\bar{x}\}, \: \text{otherwise},
\end{cases}
\end{equation*}
which is a bit weird -- why?

Now, consider what happens under a \textit{rational loss}
\begin{equation*}
 L_k(C, \theta) = \frac{\operatorname{vol}(C)}{\operatorname{vol}(C) + k} + (1-\mathbb{I}_{C}(\theta)), k >0.
\end{equation*}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{HPD (or HDI in one dimension)}
 \begin{center}
 \includegraphics[scale=0.5]{figures/HDI.pdf}
\end{center}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Recommended reading}
\begin{itemize}
  \item[\faBook] \cite{Robert2007}, Ch. 5.
%  \item
 \item[\faForward] Next lecture: \cite{Robert2007} Ch. 7.
 \end{itemize}
\end{frame}


\section{Spherical and cylindrical coordinates}

\begin{outcome}
  \begin{enumerate}
  \item Understand cylindrical and spherical coordinates.
  \item Convert points between Cartesian, cylindrical, and spherical
    coordinates.
  \end{enumerate}
\end{outcome}

Spherical and cylindrical coordinates are two generalizations of polar coordinates to three dimensions. We will first look at \textbf{cylindrical coordinates}\index{cylindrical coordinates}.

When moving from polar coordinates in two dimensions to cylindrical coordinates in three dimensions, we use the polar coordinates in the $xy$
plane and add a $z$ coordinate. For this reason, we use the notation $(r, \theta, z)$ to express cylindrical coordinates.
The relationship between Cartesian coordinates $(x,y,z)$ and cylindrical coordinates $(r, \theta, z)$ is given by
\begin{align*}
x& =r\cos (\theta)  \\
y& =r\sin (\theta)  \\
z& =z
\end{align*}
where $r\geq 0$, $\theta \in \lbrack 0,2\pi )$, and $z$ is simply the Cartesian
coordinate. Notice that $x$ and $y$ are defined as the usual polar coordinates in the $xy$-plane. Recall that $r$ is defined as the length of the ray from the origin to the point $(x,y,0)$, while $\theta$ is the angle between the positive $x$-axis and this same ray.

To illustrate this coordinate system, consider the following two pictures.
In the first of these, both $r$ and $z$ are known. The cylinder corresponds to a given value for $r$. A useful way to think of $r$ is
as the distance between a point in three dimensions and the $z$-axis. Every point on the cylinder shown is at the same distance from the $z$-axis. Giving a value for $z$ results in a horizontal circle, or cross section of the cylinder at the given height on the $z$ axis (shown below as a black line on the cylinder). In the second picture, the point is specified completely by also knowing $\theta$ as shown.

\begin{center}
\begin{tikzpicture}
\node at (-5, 2.5){$z$};
\node at (-2.5, -2){$y$};
\node at (-7.5, -2){$x$};
\node at (-5,0){\includegraphics[width=.25\textwidth]{figures/cylinder.eps}};
\node at (-5,-3){$r$ and $z$ are known};
\draw(-1,-1,0)--(3,-1,0);
\draw(1,-1,0)--(1,2,0);
\draw[dotted](1,-1,-2)--(1,-1,0);
\draw(1,-1,0)--(1,-1,2);
\draw(1,1,0) circle [x radius=1.5cm, y radius=0.5cm];
%\draw[red](1,-1,0)--(1.5,0.55,0);
\draw[fill, red](1.5,0.55,0) circle [radius=2pt];
\draw[blue](1,-1,0)--(1.5,-1.5,0);
\draw[fill,blue](1.5,-1.5,0) circle [radius=2pt];
\draw[red](1.5,0.55,0)--(1.5,-1.5,0);
\draw[purple, dotted](1.5,0.55,0)--(1,0.65,0);
\draw[fill,purple](1,0.65,0) circle [radius=2pt];
\node[below right] at (1.5,0.55,0){$(x,y,z)$};
\node[right] at (1.5,-1.5,0){$(x,y,0)$};
\node[below] at (1,-1.2,0){$\theta$};
\node[right] at (3,-1,0){$y$};
\node[above] at (1,2,0){$z$};
\node[below] at (1,-1,2){$x$};
\node[above] at (1.4,-1,0){$r$};
\node[left] at (1,0.65,0){$z$};
\node at (1, -3){$r$, $\theta$ and $z$ are known};
\end{tikzpicture}
\end{center}

 Every point of $3$-dimensional
space other than the $z$ axis  has  unique cylindrical coordinates. Of course there are infinitely many cylindrical coordinates for the
origin and for the $z$-axis. Any $\theta $ will work if $r=0$ and $z$ is given.

Consider now \textbf{spherical coordinates}\index{spherical coordinates}, the second generalization of polar form in three dimensions. For a point $(x,y,z)$ in $3$-dimensional space, the spherical coordinates are defined as follows.
\begin{equation*}
\begin{array}{l}
\rho: \mbox{the length of the ray from the origin to the point}\\
\theta: \mbox{the angle between the positive $x$-axis and the ray from the origin to the point $(x,y,0)$}\\
\phi: \mbox{the angle between the positive $z$-axis and the ray from the origin to the point of interest}
\end{array}
\end{equation*}
The spherical coordinates are determined by $(\rho ,\phi
,\theta)$. The relation between these and the Cartesian coordinates $(x,y,z)$ for a point
are as follows.
\begin{align*}
x& =\rho \sin (\phi) \cos (\theta) ,\ \phi \in
\mat{0,\pi}  \\
y& =\rho \sin (\phi) \sin (\theta) ,\text{ }\theta
\in \lbrack 0,2\pi ) \\
z& =\rho \cos \phi \text{, }\rho \geq 0.
\end{align*}

Consider the pictures below. The first illustrates the surface when $\rho$ is known, which is a sphere of radius $\rho$. The second picture corresponds to knowing both $\rho $ and $\phi$, which results in a circle about the $z$-axis. Suppose the first picture demonstrates a graph of the Earth. Then the circle in the second picture would
correspond to a particular latitude.

\begin{center}
\begin{tikzpicture}
\node at (-6, 2.5){$z$};
\node at (-3.5, -1){$y$};
\node at (-8.5, -1.5){$x$};
\node at (-6,0){\includegraphics[width=.25\textwidth]{figures/sphericalcoordinates.eps}};
\node at (-6,-3){$\rho$ is known};
\node at (2,-3){$\rho$ and $\phi$ are known};
\node at (2,0){\includegraphics[width=.25\textwidth]{figures/rhoandphigiven.eps}};
\draw[red, ultra thick] (2,-0.9)--(3,0);
\node[right] at (1.9, -0.5){$\phi$};
\node at (2, 2.5){$z$};
\node at (4.5, -2){$y$};
\node at (-0.5, -2){$x$};
\end{tikzpicture}
\end{center}

Giving the third coordinate, $\theta $ completely specifies the point of interest. This is demonstrated in the following picture. If the latitude corresponds to $\phi$, then we can think of $\theta$ as the longitude.

\begin{center}
\begin{tikzpicture}
\draw(-1,-1,0)--(3,-1,0);
\draw(1,-1,0)--(1,2,0);
\draw[dotted](1,-1,-2)--(1,-1,0);
\draw(1,-1,0)--(1,-1,2);
\draw(1,1,0) circle [x radius=1.5cm, y radius=0.5cm];
\draw[red](1,-1,0)--(1.5,0.55,0);
\draw[fill, red](1.5,0.55,0) circle [radius=2pt];
\draw[blue](1,-1,0)--(1.5,-1.5,0);
\draw[fill,blue](1.5,-1.5,0) circle [radius=2pt];
\draw[dashed](1.5,0.55,0)--(1.5,-1.5,0);
\node[below] at (1,-1.2,0){$\theta$};
\node[right] at (3,-1,0){$y$};
\node[above] at (1,2,0){$z$};
\node[below] at (1,-1,2){$x$};
\node[above] at (1.1,-0.4,0){$\phi$};
%\node[right] at (1.4,0.1,0){$\rho$};
\node at (1, -3){$\rho$, $\phi$ and $\theta$ are known};
\end{tikzpicture}
\end{center}

The following picture summarizes the geometric meaning of the three coordinate systems.

\begin{center}
\begin{tikzpicture}
\draw[->](0,0,0)--(3,0,0);
\draw[->](0,0,0)--(0,3,0);
\draw[->](0,0,0)--(0,0,4);
\draw[ultra thick, red](0,0,0)--(1.5,1.5,0);
\draw[fill, red](1.5,1.5,0) circle [radius=2pt];
\draw[ultra thick, blue](0,0,0)--(1.5,-1.5,0);
\draw[fill, blue](1.5,-1.5,0) circle [radius=2pt];
\draw[dotted](-1.25,-1.25,0)--(1.5,-1.5,0)--(2.5,0,0);
\draw[dotted](1.5,-1.5,0)--(1.5,1.5,0);
\node[right] at (3,0,0){$y$};
\node[above] at (0,3,0){$z$};
\node[below] at (0,0,4){$x$};
\node[right] at (1.5,-1.5,0){$(x,y,0)$};
\node[below] at (0,0,0){$\theta$};
\node[above right] at (0,0.2,0){$\phi$};
\node[right] at (0.75,0.75,0){$\rho$};
\node[right] at (0.75,-0.75,0){$r$};
\node[right] at (1.5,1.5,0){$(x,y,z)$};
\node[right] at (1.5,2,0){$(r, \theta, z)$};
\node[right] at (1.5,2.5,0){$(\rho, \phi, \theta)$};
\end{tikzpicture}
\end{center}

Therefore, we can represent the same point in three ways, using Cartesian coordinates, $(x,y,z)$, cylindrical coordinates, $(r, \theta, z)$, and spherical coordinates $(\rho, \phi, \theta)$.

Using this picture to review, call the point of interest $P$ for convenience. The Cartesian coordinates for $P$ are $(x,y,z)$. Then $\rho $ is the distance between the origin and the point $P$. The angle between
the positive $z$ axis and the line between the origin and $P$
 is denoted by $\phi$. Then $\theta $ is the angle
between the positive $x$ axis and the line joining the origin to the point
$(x,y,0) $ as shown. This gives the spherical coordinates, $( \rho, \phi, \theta)$. Given the line from the origin to $(x,y,0)$,  $r=\rho \sin(\phi)$ is the length of this
line. Thus $r$ and $\theta $ determine a point in the $xy$-plane. In other words, $r$ and $\theta $ are the usual polar coordinates and $r\geq 0$ and $\theta \in \lbrack 0,2\pi )$. Letting $z$ denote the usual
$z$ coordinate of a point in three dimensions,
$(r,\theta ,z) $ are the cylindrical coordinates of $P$.

The relation between spherical and cylindrical coordinates is that $r=\rho
\sin(\phi)$ and the $\theta$ is the same as the $\theta$ of cylindrical and polar
coordinates.

We will now consider some examples.

\begin{example}{Describing a surface in spherical coordinates}{}
Express the surface $z=\frac{1}{\sqrt{3}}\sqrt{x^{2}+y^{2}}$ in spherical
coordinates.
\end{example}

\begin{solution}
We will use the equations from above:
\[
\begin{array}{l}
x =\rho \sin (\phi) \cos (\theta), \phi \in
\mat{0,\pi}\\
 y =\rho \sin (\phi) \sin (\theta) ,\text{ }\theta
\in \lbrack 0,2\pi ) \\
 z =\rho \cos \phi \text{, }\rho \geq 0
\end{array}
\]

To express the surface in spherical coordinates, we substitute these expressions into the equation.
This is done as follows:
\begin{equation*}
\rho \cos (\phi) =\frac{1}{\sqrt{3}}\sqrt{(\rho \sin
(\phi) \cos (\theta)) ^{2}+(\rho \sin
(\phi) \sin (\theta)) ^{2}}=\allowbreak
\frac{1}{3}\sqrt{3}\rho \sin (\phi).
\end{equation*}
This reduces to
\begin{equation*}
\tan (\phi)=\sqrt{3}
\end{equation*}
and so $\phi =\pi /3$.
\end{solution}

\begin{example}{Describing a surface in spherical coordinates}{}
Express the surface $y=x$ in terms of spherical coordinates.
\end{example}

\begin{solution}
Using the same procedure as the previous example, this says $\allowbreak \rho \sin (\phi) \sin (\theta
) =\rho \sin (\phi) \cos (\theta)$. Simplifying,  $\sin (\theta) =\cos (\theta)$, which you could also write $\tan (\theta)=1$.
\end{solution}

We conclude this section with an example of how to describe a surface using cylindrical coordinates.

\begin{example}{Describing a surface in cylindrical coordinates}{}
Express the surface $x^{2}+y^{2}=4$ in cylindrical coordinates.
\end{example}

\begin{solution}
Recall that to convert from Cartesian to cylindrical coordinates, we can use the following equations:
\[
x =r\cos (\theta) , y=r\sin (\theta) , z =z
\]

Substituting these equations in for $x,y,z$ in the equation for the surface, we have
\[
r^{2}\cos ^{2} (\theta) +r^{2}\sin ^{2} (\theta)=4
\]
This can be written as $r^2 ( \cos^{2} (\theta)+ \sin^{2} (\theta) ) = 4$. Recall that $ \cos^{2} (\theta)+ \sin^{2} (\theta)=1$.  Thus $r^{2} = 4$ or $r=2$.
\end{solution}


\section{Resolution and Leakage of Periodogram-based Methods}

\begin{enumerate}[label=\alph*), leftmargin=*]
%% a)
\item
%

We investigate the magnitude spectrum of the $N$-points Bartlett window, $W_{B}(w)$, for several values of $N$, illustrated at figure \ref{fig:1_3_a_1}.
We note that its $3dB$ width of the main lobe varies as a function of $N$. Their inverse relationship is empirically shown in figure \ref{fig:1_3_a_2}.
Lastly, the peaks of the side lobes as a function of $N$ are depicted in figure \ref{fig:1_3_a_3}. Interestingly, the side lobes peaks are almost unaffected
by $N$ ($1.5 dB$ change from $N=8$ to $N=1024$), unlike the $3dB$ width of the main lobe, which shrinks considerably for increasing values of $N$.

\begin{figure}[h]
    \centering
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/a/bartlett-linear-N_8}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/a/bartlett-linear-N_128}
    \end{subfigure}
    ~
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/a/bartlett-log-N_8}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/a/bartlett-log-N_128}
    \end{subfigure}
    \caption{$N$-points magnitude spectrum of Bartlett window, $W_{B}(w)$.}
    \label{fig:1_3_a_1}
\end{figure}

\begin{figure}[h]
    \centering
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/a/bartlett-3db-vs-N}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/a/bartlett-3db-vs-1_over_N}
    \end{subfigure}
    \caption{Bartlett window $3dB$ width of the main lobe as a function of $N$ and $1/N$.}
    \label{fig:1_3_a_2}
\end{figure}

\begin{figure}[h]
    \centering
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/a/bartlett-3db-peak-vs-N-linear}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/a/bartlett-3db-peak-vs-N-log}
    \end{subfigure}
    \caption{Bartlett window peaks of the side lobes as a function of $N$.}
    \label{fig:1_3_a_3}
\end{figure}

%% b)
\item
%

Let the signal

\begin{equation}
    x(n) = sin(2 \pi f_{0} n) + sin(2 \pi (f_{0} + \frac{\alpha}{N}) n)
\end{equation}

where $\alpha$ is a varying parameter and $N=256$ the fixed signal length. The PSD of $x(n)$ is ideally expected to have two peaks at frequencies
$f_{0}$ and $f_{0} + \frac{a}{N}$ (in Hz), therefore if the frequency resolution $\Delta f$ is not sufficiently small, the two peaks are not
distinguishable, thus $\Delta f \leq \frac{\alpha}{N}$ is required.
Moreover, due to the spectral leakage, the side lobes height make differentiation even harder, thus we will find the minimum $\alpha$ value
by experiments. Figure \ref{fig:1_3_b_1} shows the empirical determination of $\alpha$ using a \textbf{rectangular window periodogram}.
We note that for $\alpha \lessapprox 0.62$ the two peaks are indistinguishable, while for greater $\alpha$ values the peaks can be discriminated.
Lastly, figure \ref{fig:1_3_b_2} illustrates some examples for different $\alpha$ values.

\begin{figure}[h]
    \centering
    \includegraphics[height=1.5in]{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/b/periodogram-resolution-threshold-alpha}
    \caption{Rectangular Window: Number of Peaks in Periodogram for varying $\alpha$.}
    \label{fig:1_3_b_1}
\end{figure}

\begin{figure}[h]
    \centering
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/b/periodogram-example-alpha_0.2}.pdf}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/b/periodogram-example-alpha_0.6}.pdf}
    \end{subfigure}
    ~
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/b/periodogram-example-alpha_0.7}.pdf}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/b/periodogram-example-alpha_1.0}.pdf}
    \end{subfigure}
    \caption{Rectangular window: Periodograms of $x(n)$ for varying $\alpha$.}
    \label{fig:1_3_b_2}
\end{figure}

%% c)
\item
%

Repeating the experiment using the \textbf{Hamming-windowed periodogram method}, we obtain figures \ref{fig:1_3_c_1} and \ref{fig:1_3_c_2}.
Note that since the Hamming window has a wider main lobe than the rectangular window, a larger value of $\alpha$ is required, $\alpha \lessapprox 0.71$, in order to
distinguish the two peaks. Nonetheless, the Hamming window has better attenuation of the side lobes, depicted in figure \ref{fig:1_3_c_2}.

\begin{figure}[h]
    \centering
    \includegraphics[height=1.5in]{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/c/periodogram-hamming-resolution-threshold-alpha}
    \caption{Hamming Window: number of peaks in periodogram for varying $\alpha$.}
    \label{fig:1_3_c_1}
\end{figure}

\begin{figure}[h]
    \centering
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/c/periodogram-hamming-example-alpha_0.2}.pdf}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/c/periodogram-hamming-example-alpha_0.7}.pdf}
    \end{subfigure}
    ~
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/c/periodogram-hamming-example-alpha_0.8}.pdf}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/c/periodogram-hamming-example-alpha_1.0}.pdf}
    \end{subfigure}
    \caption{Hamming Window: periodograms of $x(n)$ for varying $\alpha$.}
    \label{fig:1_3_c_2}
\end{figure}

%% d)
\item

Let the signal

\begin{equation}
    x(n) = sin(2 \pi f_{0} n) + a_{2} sin(2 \pi (f_{0} + \frac{\alpha}{N}) n)
\end{equation}

where $a_{2} \in \{1, 0.1, 0.01, 0.001\}$ and $\alpha \in \{4, 12\}$ are varying parameters and $N=256$ the fixed signal length. Ideally, for an infinitely long $x(n)$,
the PSD is expected to comprise of two dirac deltas, however, since a finitely long sequence is used, spectral leakage is expected to degrade the quality of the periodogram.
Different window functions trade $3dB$ width of the main lobe and the relative heights of the side lobes. The rectangular window used, has the smallest main lobe,
however it also has the highest side lobes, resulting in the greatest spectral leakage. Consequently, the amplitude of the second sinusoid, $a_{2}$, affects significantly our
ability to identify the second sinusoidal term in the spectral estimate. Figure \ref{fig:1_3_d} illustrates the periodograms for the different values of $a_{2}$ and $\alpha$.
We observe that for higher $a_{2}$ values, $1.0$ and $0.1$, the second peak can be identified for both $\alpha$ values. When $\alpha=12$, the peak can also be observed for
$a_{2}=0.01$, however it is rather unclear. Hence, even when the peaks are moved further apart (increased $\alpha$) the spectral leakage deteriorates peak identification
considerably.

\begin{figure}[h]
    \centering
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/d/periodogram-leakage-rect-alpha_4.0-a2_1.000}.pdf}
    \end{subfigure}
    ~
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/d/periodogram-leakage-rect-alpha_4.0-a2_0.100}.pdf}
    \end{subfigure}
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/d/periodogram-leakage-rect-alpha_4.0-a2_0.010}.pdf}
    \end{subfigure}
    ~
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/d/periodogram-leakage-rect-alpha_4.0-a2_0.001}.pdf}
    \end{subfigure}
    ~
    ~
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/d/periodogram-leakage-rect-alpha_12.0-a2_1.000}.pdf}
    \end{subfigure}
    ~
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/d/periodogram-leakage-rect-alpha_12.0-a2_0.100}.pdf}
    \end{subfigure}
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/d/periodogram-leakage-rect-alpha_12.0-a2_0.010}.pdf}
    \end{subfigure}
    ~
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/d/periodogram-leakage-rect-alpha_12.0-a2_0.001}.pdf}
    \end{subfigure}
    \caption{Rectangular window spectral leakage: periodograms of $x(n)$ for varying $a_{2}$ and $\alpha$.}
    \label{fig:1_3_d}
\end{figure}

%% e)
\item

In figure \ref{fig:1_3_e} the amplitude of the Fourier Transform of the Bartlett window is provided. We note that at frequencies $k\frac{2}{N}, k \in \mathbb{Z}$ there are zeros and thus
at frequencies $\frac{4}{N}$ and $\frac{12}{N}$, too. Additionally, we observe that the side lobes of the window are not constant and they decrease as distance from the main lobe increases (increasing $\alpha$).
As a result, the amplitude threshold identification is slightly easier at $\alpha = 12$ because the peak of side lobes around $f = \frac{12}{N}$ is slightly lower than the peak of side lobes around $f = \frac{4}{N}$.

\begin{figure}[h]
    \centering
    \includegraphics[height=1.5in]{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/e/bartlett-zeros-alpha}
    \caption{Bartlett window: periodogram and zeros at $f = \frac{12}{N}$ and $f = \frac{4}{N}$.}
    \label{fig:1_3_e}
\end{figure}

%% f)
\item

Repeating the experiment using the \textbf{Chebyshev-windowed periodogram method}, we obtain figure \ref{fig:1_3_f_1}. Thanks to the Chebyshev window side lobes high attenuation,
we notice that the two frequency components are distinguishable even in the case of the smallest $a_{2} = 0.001$. However, given the trade-off between side lobes attenuation
and main lobe bandwidth, for $\alpha = 4$, the two peaks overlap, making discrimination more difficult. Overall, this represents the tradeoff that windows have to make between
the width of the mainlobe as the height of the sidelobes.The rectangular window has a small mainlobe and thus the trouble in identification comes about because of the leakage effects
whereas the Chebyshev window cause more smearing and less leakage.

\begin{figure}[h]
    \centering
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/f/periodogram-leakage-chebyshev-alpha_4.0-a2_1.000}.pdf}
    \end{subfigure}
    ~
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/f/periodogram-leakage-chebyshev-alpha_4.0-a2_0.100}.pdf}
    \end{subfigure}
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/f/periodogram-leakage-chebyshev-alpha_4.0-a2_0.010}.pdf}
    \end{subfigure}
    ~
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/f/periodogram-leakage-chebyshev-alpha_4.0-a2_0.001}.pdf}
    \end{subfigure}
    ~
    ~
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/f/periodogram-leakage-chebyshev-alpha_12.0-a2_1.000}.pdf}
    \end{subfigure}
    ~
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/f/periodogram-leakage-chebyshev-alpha_12.0-a2_0.100}.pdf}
    \end{subfigure}
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/f/periodogram-leakage-chebyshev-alpha_12.0-a2_0.010}.pdf}
    \end{subfigure}
    ~
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/f/periodogram-leakage-chebyshev-alpha_12.0-a2_0.001}.pdf}
    \end{subfigure}
    \caption{Chebyshev window: periodograms of $x(n)$ for varying $a_{2}$ and $\alpha$.}
    \label{fig:1_3_f_1}
\end{figure}

Using the \textbf{Blackman-Tukey periodogram method} with $M = \frac{N}{4} = 64$ lags for spectral estimation of $x(n)$, figure \ref{fig:1_3_f_2} is obtained.
We notice that the two frequency peaks are obtained only in the case of $a_{2} = 1.0$, failing in all other cases, regardless $\alpha$.
This method trades resolution, $\Delta f_{BT} \sim \frac{1}{M}$ compared to $\Delta f_{Per} \sim \frac{1}{N}$ for a rectangular window periodogram, for variance.
However, the signal under investigation, $x(n)$, is purely deterministic (no stochastic term, since $\sigma^{2} = 0$) and thus the reduction in variance does noy add any value to our estimate,
leading solely to reduced resolution and hence unsuccessful identification of the two frequency components.

\begin{figure}[h]
    \centering
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/f/periodogram-leakage-blackman-alpha_4.0-a2_1.000}.pdf}
    \end{subfigure}
    ~
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/f/periodogram-leakage-blackman-alpha_4.0-a2_0.100}.pdf}
    \end{subfigure}
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/f/periodogram-leakage-blackman-alpha_4.0-a2_0.010}.pdf}
    \end{subfigure}
    ~
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/f/periodogram-leakage-blackman-alpha_4.0-a2_0.001}.pdf}
    \end{subfigure}
    ~
    ~
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/f/periodogram-leakage-blackman-alpha_12.0-a2_1.000}.pdf}
    \end{subfigure}
    ~
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/f/periodogram-leakage-blackman-alpha_12.0-a2_0.100}.pdf}
    \end{subfigure}
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/f/periodogram-leakage-blackman-alpha_12.0-a2_0.010}.pdf}
    \end{subfigure}
    ~
    \begin{subfigure}{0.225\textwidth}
        \centering
        \includegraphics[height=0.75in]{{report/spectrum-estimation/resolution-and-leakage-of-periodogram-based-methods/assets/f/periodogram-leakage-blackman-alpha_12.0-a2_0.001}.pdf}
    \end{subfigure}
    \caption{Blackman-Tukey method: periodograms of $x(n)$ for varying $a_{2}$ and $\alpha$.}
    \label{fig:1_3_f_2}
\end{figure}

%
\end{enumerate}


\subsection{Bayesian parameter estimation for dependent models}

\subsubsection{Recap}

For non-dependent models we had:

\(P(\theta |y)=\dfrac{P(y, \theta)}{P(y)}\)

\(P(\theta |y)=\dfrac{P(y| \theta)P(\theta )}{P(y)}\)

The bottom bit is a normalisation factor, and so we can use:

\(P(\theta |y)\propto P(y| \theta)P(\theta )\)

We have here:

\begin{itemize}
\item Our prior - \(P(\theta )\)
\item Our posterior - \(P(\theta |y)\)
\item Our likelihood function - \(P(y| \theta)\)
\end{itemize}

\subsubsection{Bayesian regression for generative models}

We know:

\(P(\theta |y,X)=\dfrac{P(y, \theta, X )}{P(y, X)}\)

\(P(\theta |y,X)=\dfrac{P(y, X |\theta )P(\theta )}{P(y, X)}\)

The bottom bit is a normalisation factor, and so we can use:

\(P(\theta |y,X)\propto P(y, X| \theta)P(\theta)\)

We have here:

\begin{itemize}
\item Our prior - \(P(\theta )\)
\item Our posterior - \(P(\theta |y,X)\)
\item Our likelihood function - \(P(y, X| \theta )\)
\end{itemize}

\subsubsection{Bayesian regression for discriminative models}

We know:

\(P(\theta |y,X)=\dfrac{P(y, \theta, X )}{P(y, X)}\)

\(P(\theta |y,X)=\dfrac{P(y| \theta, X )P(\theta, X)}{P(y, X)}\)

\(P(\theta |y,X)=\dfrac{P(y| \theta, X )P(\theta )P(X|\theta )}{P(y, X)}\)

We assume \(P(X|\theta )=X\) and so:

\(P(\theta |y,X)=\dfrac{P(y| \theta, X )P(\theta )P(X)}{P(y, X)}\)

The bottom bit is a normalisation factor, and so we can use:

\(P(\theta |y,X)\propto P(y| X, \theta)P(\theta)\)

We have here:

\begin{itemize}
\item Our prior - \(P(\theta )\)
\item Our posterior - \(P(\theta |y,X)\)
\item Our likelihood function - \(P(y| X, \theta )\)
\end{itemize}


\section{The SkipBlock Model}
\label{sec:skipbock}

The Skip List model works on a record unit, i.e., a synchronization point has a
probability p to appear in an inverted list of n records.
The SkipBlock model operates instead on \emph{blocks} of records of a fixed
size, in place of the records themselves. Consequently, the probabilistic parameter
$p$ is defined with respect to a block unit. A synchronization point is
created every $\frac{1}{p^i}$ blocks on a level $i$, thus every $\frac{\vert B
\vert}{p^i}$ records where $\vert B \vert$ denotes the block size. A
synchronization point links to the first record of a block interval. Compared
to Figure~\ref{fig:skiplists}, a SkipBlock structure with $p=\frac{1}{8}$ and
$\vert B \vert=2$ also has an interval of $\frac{\vert B \vert}{p^1}=16$
records. However, on level 2, the synchronization points are separated by
$\frac{\vert B \vert}{p^2}=128$ instead of 256 records. We note that with
$\vert B \vert= 1$, the SkipBlock model is equivalent to the original Skip
Lists model, and therefore it is a generalization. The two properties of the
Skip Lists are then translated to this model:
\begin{enumerate}
  \item the number of levels is defined by
\begin{equation}
L_B(n)=\left\lfloor \ln_{\frac{1}{p}}\left(\frac{n}{\vert B \vert}\right) \right\rfloor
\label{eq:skipblock-maxlevels}
\end{equation}
  \item the structure's size
\begin{equation}
S_B(n)=\sum_{i=1}^{
L_B\left(n\right)}{\left\lfloor \frac{n\times p^i}{\vert B \vert}
\right\rfloor}
\label{eq:skipblock-size}
\end{equation}
\end{enumerate}

\subsection{SkipBlock Search Algorithm}
\label{sec:search-skipblock}

With the block-based Skip Lists model, the search algorithm returns an
interval of blocks containing the target record. We discuss the search in
that interval in Section~\ref{sec:search-interval}. The search walk is identical
to the one presented in Section~\ref{sec:search-skiplists}: we walk from the
top to the bottom level, and compare at each step the current synchronization
point with the target. The walk stops at the same termination criteria as in
Skip Lists. The search complexity in the worst case becomes
\begin{equation}
C_{SB}=\frac{L_B(n)}{p}
\label{eq:skipblock-complexity}
\end{equation}

\subsection{Impact of the Self-Indexing Parameters}
\label{sec:block-probas-impact}

In this section, we discuss the consequences of the probabilistic parameter on
the Skip Lists data structure.
Table~\ref{tab:skiplists-proba} reports for low (i.e., $\frac{1}{1024}$) and
high (i.e., $\frac{1}{2}$) probabilities
\begin{inparaenum}[(1)]
\item the complexity (\ref{eq:skiplists-complexity}) to find the interval
containing the target record, and
\item the size (\ref{eq:skiplists-size}) of the Skip Lists structure.
\end{inparaenum}
For example with an interval $\vert I \vert = 1024$ and an
inverted list of size $10^6$ (i.e., Skip Lists with one level), reaching a record
located at the end of the inverted list incurs a lot of synchronization points
at level 1 being read. With a smaller interval, e.g. 16, the number of
synchronization points read decreases because the $L(10^6)=4$ levels provide
more accessing points to the inverted list.

Large intervals give the possibility to skip over a large number of records by
reading a few synchronization points. However this leads to less use cases of
the Skip Lists structure when the searched records are close to each other. In
the latter case, small intervals are then more adapted, but at the cost of a
bigger structure. Moreover the more levels there are, the more walks up and
down on them there are.

There is a trade-off to achieve when selecting $p$: a high probability
provides a low search complexity but at a larger space cost, and a low
probability reduces considerably the required space at the cost of higher
search complexity.

The SkipBlock model provides two parameters to control its Skip Lists
structure: the probabilistic parameter $p$ and the block size $\vert B \vert$.
The block size parameter enables more control over the Skip Lists structure. For
example, to build a structure with an interval of length 64, the original Skip
Lists model proposes only one configuration given by $p=\frac{1}{64}$. For
this same interval length, SkipBlock proposes all the configurations that
verify the equation $\frac{\vert B \vert}{p}=64$.
Table~\ref{tab:skiplists-proba-block} reports statistics of some SkipBlock
configurations for the same interval lengths as in
Table~\ref{tab:skiplists-proba}. Compared to Skip Lists on a same interval
length, SkipBlock shows a lower search complexity in exchange of a larger
structure.

\begin{table}
\begin{center}
\subfloat[Skip Lists with $\vert I\vert=\frac{1}{p}$.]{
\centering
\ra{1.1}
\resizebox{0.39\linewidth}{!}{%
\begin{tabular}{@{}lcc@{\hs}c@{\hs}c@{\hs}c@{\hs}c@{\hs}}
\toprule
$\vert I \vert$ & \phantom{a} & 2 & 16 & 64 & 128 & 1024\\
\cmidrule{2-7}
$S(n)$ & \phantom{a} &
\numprint{99999988}&\numprint{6666664}&\numprint{1587300}&\numprint{787400}&\numprint{97751}\\
$C$ & \phantom{a} & 54 & 112 & 320 & 512 & 3072 \\
\bottomrule
\end{tabular}
\label{tab:skiplists-proba}
}}\quad%
\subfloat[SkipBlock with $\vert I\vert=\frac{\vert B \vert}{p}$.]{
\ra{1.1}
\resizebox{0.54\linewidth}{!}{%
\begin{tabular}{@{}lcc@{\hs}c@{\hs}cc@{\hs}c@{\hs}cc@{\hs}c@{\hs}cc@{\hs}c@{\hs}}
\toprule
$\vert I \vert$ & \phantom{a} & \multicolumn{2}{c}{16} & \phantom{a} &
\multicolumn{2}{c}{64} & \phantom{a} & \multicolumn{2}{c}{128} & \phantom{a} &
\multicolumn{2}{c}{1024} \\
\cmidrule{3-4} \cmidrule{6-7} \cmidrule{9-10} \cmidrule{12-13}
p:$\vert B \vert$ & \phantom{a} & $\frac{1}{4}$:4 & $\frac{1}{8}$:2 & \phantom{a} &
$\frac{1}{4}$:16 & $\frac{1}{8}$:8 & \phantom{a} & $\frac{1}{4}$:32 &
$\frac{1}{8}$:16 & \phantom{a} & $\frac{1}{4}$:256 & $\frac{1}{8}$:128 \\
$S_B(n)$ & \phantom{a} & \numprint{8333328} & \numprint{7142853} &
\phantom{a} & \numprint{2083328} & \numprint{1785710} & \phantom{a} &
\numprint{1041660} & \numprint{892853} & \phantom{a} &
\numprint{130203} & \numprint{111603} \\
$C$ & \phantom{a} & 48 & 64 & \phantom{a} & 44 & 56 & \phantom{a} &
40 & 56 & \phantom{a} & 36 & 48 \\
\bottomrule
\end{tabular}
\label{tab:skiplists-proba-block}
}}%
\end{center}
\caption{Search and size costs of Skip Lists and SkipBlock with $n=10^8$.
$\vert I \vert$ stands for an interval length. $C$ reports the search
complexity to find an interval (Sections~\ref{sec:search-skiplists}
and~\ref{sec:search-skipblock}).}
\end{table}

\section{Searching within an Interval}
\label{sec:search-interval}

The Skip Lists and SkipBlock techniques enables the retrieval of an interval
given a target record. The next step is to find the target record within that
interval. A first strategy (S1) is to linearly scan all the records within
that interval until the target is found. Its complexity is therefore $O(\vert I \vert)$.

SkipBlock takes advantage of the block-based structure of the
interval to perform more efficient search strategy. Here are defined four
additional strategies for searching a block-based interval, parameterized to a
probability p. The second strategy (S2) performs
\begin{enumerate}[(a)]
\item a linear scan over the blocks of the interval to find the block holding
the target and
\label{s2-linear-search-blocks}
\item a linear scan of the records of that block to find the target.
\label{s2-target-block}
\end{enumerate}
The search complexity is $\frac{1}{p}+\vert B \vert$ with $\frac{1}{p}$ denoting
the linear scan over the blocks and $\vert B \vert$ the linear scan over the
records of one block. Similarly to S2, the third strategy (S3) performs the
step (\ref{s2-linear-search-blocks}). Then, it uses to find the target an
inner-block Skip Lists structure restricted to one level only. The complexity
is $\frac{1}{p}+\frac{1}{q}+\left\lfloor\vert B\vert\times q\right\rfloor$
with q the probability of the inner Skip Lists. In contrast to S3, the fourth
strategy (S4) uses a non-restricted inner-block Skip Lists structure. The
complexity is $\frac{1}{p}+\frac{L(\vert B \vert)+1}{q}$ with q the inner Skip
Lists probability. The fifth one (S5) builds a Skip Lists structure on the
whole interval instead of on a block. Its complexity is then
$\frac{L\left(\frac{\vert B\vert}{p}\right)+1}{q}$, with $q$ the inner Skip
Lists probability. The strategies S3, S4 and S5 are equivalent to S2 when the
block size is too small for creating synchronization points.

\section{Cost-Based Comparison}
\label{sec:cost-based-cmp}

In this section, we define a cost model that we use to compare five
SkipBlock implementations and the original Skip Lists implementation.

\paragraph{Cost Model}
\label{sec:skipblock-cost-model}
For both the Skip Lists and the SkipBlock, we define a cost model by
\begin{inparaenum}[(a)]
\item the cost to search for the target, and
\label{cost-search}
\item the cost of the data structure's size.
\label{cost-size}
\end{inparaenum}
The search cost consists in the number of synchronization points traversed to
reach the interval containing the target, plus the number of records scanned
in that interval to find the target. The size cost consists in the total number
of synchronization points in the data structure, including the additional ones
for S3, S4 and S5 in the intervals.

\paragraph{Implementations}
\label{sec:impl}
We define as the baseline implementation, denoted $I_1$, the Skip
Lists model using the strategy (\emph{S1}). We define
five implementations of the SkipBlock model, denoted by $I_2$, $I_3$,
$I_4$, $I_5$ and $I_6$, based on the five interval search strategies, i.e.,
\emph{S1}, \emph{S2}, \emph{S3},\emph{S4} and \emph{S5} respectively. The
inner Skip Lists in implementations $I_4$, $I_5$ and $I_6$ is
configured with probability $q=\frac{1}{16}$. The inner Skip Lists in $I_5$ and
$I_6$ have at least 2 levels. The size costs of the six implementations are
\begin{itemize}
  \item[$I_1$:] $S(n)$
  \item[$I_2$:] $S_B(n)$
  \item[$I_3$:] $S_B(n)+\frac{n}{\vert B \vert}$
  \item[$I_4$:] $S_B(n)+\left\lfloor n\times q\right\rfloor$
  \item[$I_5$:] $S_B(n)+\frac{S(\vert B \vert)\times n}{\vert B \vert}$
  \item[$I_6$:]$S_B(n)+\frac{S(p\times\vert B \vert)\times n}{p\times\vert B
  \vert}$
\end{itemize}

\paragraph{Comparison}

With respect to the SkipBlock model, we tested all the possible
configurations for a given interval length. We report that all of them were
providing better search cost than the baseline. We only report in
Table~\ref{tab:cmp-costs} the configurations providing the best search cost
given an interval length with the associated size cost. We observe that $I_2$
already provides better search cost than the baseline $I_1$ using the same
search strategy \emph{S1}, in exchange of a larger size cost. The other
implementations, i.e., $I_3$, $I_4$, $I_5$ and $I_6$ which use a more
efficient interval search strategies further decrease the search cost. In
addition, their size cost decreases significantly with the size of the
interval. On a large interval ($512$), $I_4$ is able to provide a low search
cost ($64$) while sustaining a small size cost ($6.5e6$). The inner Skip
Lists of $I_5$ and $I_6$ are built on lists too small to provide benefits. To
conclude, $I_4$ seems to provide a good compromise between search cost and
size cost with large intervals.

\begin{table}
\centering
\ra{1.1}
\resizebox{\linewidth}{!}{%
\begin{tabular}{@{}lcc@{\hs}c@{\hs}c@{\hs}c@{\hs}c@{\hs}c@{\hs}cc@{\hs}c@{\hs}c@{\hs}c@{\hs}c@{\hs}c@{\hs}cc@{\hs}c@{\hs}c@{\hs}c@{\hs}c@{\hs}c@{\hs}cc@{\hs}c@{\hs}c@{\hs}c@{\hs}c@{\hs}c@{\hs}}
\toprule
$\vert I\vert$&\phantom{a}&\multicolumn{6}{c}{8}
&\phantom{a}&\multicolumn{6}{c}{16}
&\phantom{a}&\multicolumn{6}{c}{512}
&\phantom{a}&\multicolumn{6}{c}{1152}\\
&\phantom{a}& $I_1$ & $I_2$ & $I_3$ & $I_4$ & $I_5$ & $I_6$  &\phantom{a}& $I_1$ & $I_2$ & $I_3$
& $I_4$ & $I_5$ & $I_6$   &\phantom{a}& $I_1$ & $I_2$ & $I_3$ & $I_4$ & $I_5$ & $I_6$  &\phantom{a}& $I_1$ &
$I_2$ & $I_3$ & $I_4$ & $I_5$ & $I_6$  \\
\cmidrule{3-8}\cmidrule{10-15}\cmidrule{17-22}\cmidrule{24-29}
SC&\phantom{a}& $72$ & $56$ & \multicolumn{4}{c}{$54$}
&\phantom{a}& $112$ & $62$ & \multicolumn{4}{c}{$56$}
&\phantom{a}& $1536$ & $548$ & $120$ & $64$ & $86$ & $84$
&\phantom{a}& $3456$ & $1186$ & $154$ & $74$ & $84$ & $81$ \\
\cmidrule{3-8}\cmidrule{10-15}\cmidrule{17-22}\cmidrule{24-29}
ZC$\times e6$&\phantom{a}& $14.3$ & $16.7$ & \multicolumn{4}{c}{$50.0$}
&\phantom{a}& $6.7$ & $12.5$ & \multicolumn{4}{c}{$25.0$}
&\phantom{a}& $0.2$ & $0.3$ &$1.8$ & $6.5$ & $7.0$ & $6.9$
&\phantom{a}& $0.09$ &$0.17$ & $1.7$&$6.4$ & $6.6$&$6.7$\\
\bottomrule
\end{tabular}
}%
\caption{Search (i.e., SC) and size (i.e., ZC, in million) costs with $n=10^8$.
SkipBlock implementations report the best search cost with the associated
size cost.}
\label{tab:cmp-costs}
\end{table}

\subsection{Skipping Analysis}

The SkipBlock structure, as a generalization of the Skip List, possesses the
same hierarchical layout. However the difference between the two is concrete:
SkipBlock jumps over groups of blocks at a time, whereas Skip Lists jumps over
groups of records. The Table~\ref{tab:skip-levels} reports the intervals at
each levels for both structures, with a constant interval between towers of
256 records. For a same interval length (e.g., 256 records in the table), we are
able to yield more configurations with the SkipBlock than with the Skip List
model. We remark that in both configurations of the SkipBlock, in-between
levels have been added in comparison to the original Skip List. These
additional levels give more skipping possibilities while searching. Moreover
this implies that more data will be read from a tower in a SkipBlock structure
than with the Skip List, for a same interval length.

\begin{table}
\ra{1.1}
\centering
\resizebox{0.45\linewidth}{!}{%
\begin{tabular}{lc@{\hs}rc@{\hs}rr}
\toprule
Level & \phantom{a} & Skip Lists & \phantom{a} & \multicolumn{2}{c}{SkipBlock}\\
\cmidrule{5-6}
& & p=256 & \phantom{a} & B=16 p=16 & B=64 p=4 \\
\cmidrule{5-6}
1 & \phantom{a} & 256 & \phantom{a} & 256 & 256 \\
2 & \phantom{a} & \uwave{\numprint{65536}} & \phantom{a} & \numprint{4096} &
\numprint{1024} \\
3 & \phantom{a} & \uwave{\numprint{16777216}} & \phantom{a} &
\uwave{\numprint{65536}} & \numprint{4096} \\
4 & \phantom{a} & \ldots & \phantom{a} & \numprint{1048576} & \numprint{16384}\\
5 & \phantom{a} & \ldots & \phantom{a} & \uwave{\numprint{16777216}} &
\uwave{\numprint{65536}}\\
\bottomrule
\end{tabular}
}
\caption{Interval length for each skip levels.}
\label{tab:skip-levels}
\end{table}

The SkipBlock structure will behave differently depending on the parameter
configurations in comparison to the Skip list, since they yield different skip
levels. To have a better understanding of the differences between the two
models, we discuss here the cost of reading skip data into memory and in which
case it outweighs the CPU saving thanks to an adaptation of the method from
\cite{moffat:96}.

Let $t_r$ be the cost of reading one record or a synchronization point, both as
part of a bulk read. Let also $t_d$ be the cost of decoding a
record/synchronization point. Let further $k$ be the number of
\emph{candidates}, i.e., the records that will be lookup from the inverted
list thanks to the self-indexing structure. Then the total time $T$ required
to search one inverted list of size $n$ in the worst case scenario (i.e., all
levels up to the maximum $L(n)$ are read) is given by the formula
\begin{displaymath}
T=T_d+T_r=kt_d\left( C + \vert I \vert \right)+t_r\left(n+2\times S\right)
\end{displaymath}
where $T_d$ is the total time to decode pointers, $T_r$ the total time to read
into memory the inverted list and synchronization points. $ZC$ and $SC$ reports
respectively the size and the search costs. The numerical values of $t_d$ and
$t_r$ are respectively $2.5\times 10^{-6}$ and $0.5\times 10^{-6}$, values
taken from \cite{moffat:96}.

The Table~\ref{tab:predicted-times} reports a numerical comparison between the
two self-indexing models, with $n=60,000$ and $k=\{60,\;6000\}$. With a bigger
number of candidates we can expect a bigger processing time, since the
self-indexing structure will be accessed more often. The SkipBLock structure's
values are reported with the strategy S1 for the configurations that yields
the best processing time T. On small intervals (e.g., 16) the SkipBlock model
does not provide significant better performance even though the complexity is
better than the Skip List. This is explained by the difference in size, as the
amount of data to be read is greater, which is reflected by the greater
required time to read data $T_r$. As the interval increases, we can see that
the processing times increase for both models. However the expected
processing time is inferior with SkipBlock than with the original Skip List by
2 times.
The reason is that even if the strategies for searching within intervals is
the same (i.e., linear), the additional levels that the SkipBlock model
provides give more skipping possibilities. This comforts the fact that the
SkipBlock complexities are lower than the Skip List's. With the interval
$\vert I \vert = 1024$, the structure's size increases by 30\%, however both
the search complexity and the estimated processing time are twice as low as
the original model.

\begin{table}
\ra{1.1}
\centering
\resizebox{\linewidth}{!}{%
\begin{tabular}{lc@{\hs}rrrc@{\hs}rrrc@{\hs}rrc@{\whs}rrrc@{\hs}rrrc@{\hs}rr}
\toprule
$\vert I \vert$ & \phantom{a} & \multicolumn{10}{c}{Skip List} & \phantom{a} &
\multicolumn{10}{c}{SkipBlock} \\
\cmidrule{3-12} \cmidrule{14-23}
k &\phantom{a}& \multicolumn{3}{c}{60} &\phantom{a}& \multicolumn{3}{c}{6000}
&\phantom{a}& & & \phantom{a} & \multicolumn{3}{c}{60} &\phantom{a}&
\multicolumn{3}{c}{6000} & & \\
\cmidrule{3-5} \cmidrule{7-9} \cmidrule{14-16} \cmidrule{18-20}
& \phantom{a} & $T_d$ & $T_r$ & T & \phantom{a} & $T_d$ & $T_r$ & T &
\phantom{a} & SC & ZC & \phantom{a} & $T_d$ & $T_r$ & T & \phantom{a} &
$T_d$ & $T_r$ & T & \phantom{a} & SC & ZC \\
16&\phantom{a}&0.010&0.034&0.044
&\phantom{a}& 0.960 & 0.034 & 0.994 & \phantom{a}& 64 & 3998
&\phantom{a}&0.006&0.035&0.041
&\phantom{a}& 0.600 & 0.035 & 0.635 & \phantom{a}&40 & 4996 \\

32&\phantom{a}&0.019&0.032&0.051
&\phantom{a}& 1.92 & 0.032 & 1.952& \phantom{a} & 128 & 1934
&\phantom{a}&0.008&0.034&0.042
&\phantom{a}& 0.81 & 0.034 & 0.844 & \phantom{a}& 54 & 3743 \\

256&\phantom{a}&0.077&0.030&0.107
&\phantom{a}& 7.68 & 0.030 & 7.710& \phantom{a} & 512 & 234
&\phantom{a}&0.041&0.030&0.071
&\phantom{a}& 4.08 & 0.030 & 4.110& \phantom{a} & 272& 309 \\

512&\phantom{a}&0.154&0.030&0.184
&\phantom{a}& 15.36 & 0.030 & 15.390& \phantom{a} & 1024 & 117
&\phantom{a}&0.079&0.030&0.109
&\phantom{a}& 7.89 & 0.030 & 7.920 & \phantom{a}& 526 & 229 \\

1024&\phantom{a}&0.307&0.030&0.337
&\phantom{a}&30.72 & 0.030 & 30.750& \phantom{a} & 2048 & 58
&\phantom{a}&0.155&0.030&0.185
&\phantom{a}& 15.54 & 0.030 & 15.570 & \phantom{a}& 1036 & 75 \\

\bottomrule
\end{tabular}}
\caption{Predicted processing times in seconds (n=60,000), with the
corresponding structure's search (i.e., SC) and size (i.e., ZC) costs. $T_d$
stands for the time for decoding a record, $T_r$ for the time of reading into memory a
record/synchronization point and T the sum of these two times. $k$ indicate the
number of candidates.}
\label{tab:predicted-times}
\end{table}


% Section 2: standard form LP

\clearpage
\section{Standard Form LP}
\begin{equation}
    \begin{aligned}
      \min _{\pi \in \mathbb{R}^{m \times n}} & \sum_{i=1}^{m} \sum_{j=1}^{n} c_{i j} \pi_{i j} \\
      \text { s.t. } & \sum_{j=1}^{n} \pi_{i j}=\mu_{i}, \quad \forall i=1, \ldots, m \\
      & \sum_{i=1}^{m} \pi_{i j}=\nu_{i}, \quad \forall j=1, \ldots, n \\
      & \pi_{i j} \geq 0
    \end{aligned}
  \end{equation}

  \vspace{5ex}

\subsection{Calling Mosek and Gurobi}

The standard form can be vectorized as:
\begin{equation}
    \begin{aligned}
      \min _{\pi \in \mathbb{R}^{m \times n}} & c^T \pi \\
      \text { s.t. } & A \pi = b \\
      & \pi_{i j} \geq 0
    \end{aligned}
  \end{equation}

where
$$
c=\left(c_{11}, c_{21}, \dots c_{n 1}, \dots, c_{1 n}, \dots c_{n n}\right)^{T}
$$

$$
\pi=\left(\pi_{11}, \pi_{21}, \ldots \pi_{n 1}, \ldots, \pi_{1 n}, \ldots \pi_{n n}\right)^{T}
$$

$$
A=\left(\begin{array}{ccc}
    {I_{m}} & {\cdots} & {I_{m}} \\
    {E_{1}} & {\cdots} & {E_{n}}
    \end{array}\right),\left(E_{j}\right)_{s t}=\delta_{s j}
$$

$$
b=\left(\mu_{1}, \dots \mu_{m}, \nu_{1}, \dots \nu_{n}\right)^{T}
$$

Then we could directly call linear programming solvers provided by Mosek and Gurobi.

\subsection{First Order Method: ADMM}
For convenience, we reformulate the original primal problem as:

\begin{equation}
\begin{array}{rl}
{\mathrm{(primal)}} & {\min_\pi \sum_{i=1}^{m} \sum_{j=1}^{n} c_{i j} \pi_{i j} + I_{\mathbb{R}_+^{m\times n}}(\pi^\dagger)} \\
{\text { subject to }} & {\sum_{j=1}^{n} \pi_{i j}=\mu_{i} \forall i=1, \ldots, m} \\
{} & {\sum_{i=1}^{m} \pi_{i j}=\nu_{j} \forall j=1, \ldots, n} \\
& \pi = \pi^{\dagger}
\end{array}
\end{equation}

Then we have the augmented Lagrangian:

\begin{equation}
  \begin{aligned}L(\pi, \pi^{\dagger}, u, v, w)=& \min_\pi \sum_{i=1}^{m} \sum_{j=1}^{n} c_{i j} \pi_{i j} + I_{\mathbb{R}_+^{m\times n}}(\pi^\dagger) \\&+\sum_{i=1}^{m} u_{i}\left(\mu_{i}-\sum_{j=1}^{n} \pi_{i j}\right)+\sum_{j=1}^{n} v_{j}\left(\nu_{j}-\sum_{i=1}^{m} \pi_{i j}\right)+\sum_{i=1}^{m} \sum_{j=1}^{n} w_{i j}\left(\pi_{i j}-\pi^{\dagger}_{i j}\right) \\&+\frac{\rho}{2} \sum_{i=1}^{m}\left(\mu_{i}-\sum_{j=1}^{n} \pi_{i j}\right)^{2}+\frac{\rho}{2} \sum_{j=1}^{n}\left(\nu_{j}-\sum_{i=1}^{m} \pi_{i j}\right)^{2}+\frac{\rho}{2} \sum_{i=1}^{m} \sum_{j=1}^{n}\left(\pi_{i j}-\pi^{\dagger}_{i j}\right)^{2}\end{aligned}
\end{equation}

$\partial_{\pi_{ij}} L = 0 $ gives:

\begin{equation}
  \sum_{k=1}^{n} \pi_{i k}+\sum_{k=1}^{m} \pi_{k j}+\pi_{i j}=\frac{1}{\rho}\left(-e_{i j}+u_{i}+v_{j}-c_{i j}\right)+\mu_{i}+\nu_{j}+\pi_{ij}^{\dagger}
\end{equation}

$\partial_{\pi^{\dagger}_{ij}} = 0$ gives:

\begin{equation}
  {\pi^{\dagger}_{ij}} = (\pi_{ij} + \frac{w_{ij}}{\rho})_+
\end{equation}

\vspace{2ex}
    \begin{algorithm}[htbp]
        \SetAlgoNoLine
        \caption{ADMM method for primal problem}
        \KwIn{parameters $\mu$, $\nu$, $c$}
        \KwIn{step size $\alpha$, penalty $\rho$}
        Initialize variables $\pi, \pi^{\dagger} = \boldsymbol{0}$\\
        Initialize variables $u, v, w = \boldsymbol{0}$\\
        \While{ stopping criterion not met }
        {
            Update $\pi$: $\boldsymbol{\pi} \leftarrow \operatorname{argmin}_{\pi} L(\pi, \pi^{\dagger}, u, v, w)$\\
            Update $\pi^{\dagger}$: $\boldsymbol{\pi}^{\dagger} \leftarrow \operatorname{argmin}_{\pi^{\dagger}} L(\pi, \pi^{\dagger}, u, v, w)$\\
            Update $u$: $\boldsymbol{u} \leftarrow u + \rho\cdot \alpha (\mu - \sum_j \pi_{ij})$\\
            Update $v$: $\boldsymbol{v} \leftarrow v + \rho\cdot \alpha (\nu - \sum_i \pi_{ij})$\\
            Update $w$: $\boldsymbol{w} \leftarrow w + \rho\cdot \alpha (\pi - \pi^{\dagger})$\\
        }
    \end{algorithm}


\documentclass{article}

\usepackage{amsmath,amssymb,siunitx,graphicx}
\usepackage[margin=1in]{geometry}
\DeclareSIUnit\ergs{ergs}
\DeclareSIUnit\yr{yr}
\DeclareSIUnit\AU{AU}
\DeclareSIUnit\msun{\ensuremath{\mathrm{M}_{\odot}}}

\title{Asteroid Defense}
\author{Matthias J. Raives}

\begin{document}

        \maketitle{}

	\section{Deflection}
	Consider an asteroid at some distance $R$ from Earth, heading directly towards the center of the Earth at a speed $v_{0}$.  To deflect it, we need to change its velocity such that it path just barely intersects the gravitational focusing radius $b$.

	The gravitational focusing radius can be derived from a conservation of energy and angular momentum argument:
        \begin{align}
                \frac{1}{2}mv_{0}^{2} &= \frac{1}{2}mv_{max}^{2} - \frac{GM_{\oplus}m}{R_{\oplus}}\\
                bv_{0} &= v_{max}R_{\oplus}
        \end{align}
        where $v_{max}$ is the velocity at impact.  We solve for $b$ as:
        \begin{equation}
                b = R_{\oplus}\left(1+\frac{v_{esc}^{2}}{v_{0}^{2}}\right)
        \end{equation}
        where $v_{esc}$ is the escape velocity from the surface of the Earth
        \begin{align}
                v_{esc}^{2} = \frac{2GM_{\oplus}}{R_{\oplus}}.
        \end{align}

        Now consider the incoming asteroid.  It has an initial momentum $\mathbf{p}_{0}=m\mathbf{v}_{0}$.  Consider our deflection to change it's momentum by some $\Delta \mathbf{p}$ but without changing the total energy, i.e., $|\mathbf{p}_{0}|=|\mathbf{p}_{1}|$.  This defines a triangle that can be solved as:
        \begin{equation}
                \sin\frac{\theta}{2} = \frac{\Delta p}{p_{0}}
        \end{equation}
        where $\theta$ is determined by
        \begin{equation}
                \sin\theta = \frac{b}{R}.
        \end{equation}
        Using the small-angle approximation (valid if $R\gg b$, i.e., $R\gg R_{\oplus}$ and $v_{0}\gg v_{esc}$), we obtain
        \begin{equation}
                \Delta p = p_{0}\frac{b}{R} = mv_{0}\frac{R_{\oplus}}{R}\left(1+\frac{v_{esc}}{v_{0}}\right)
        \end{equation}
        In the reference frame initially co-moving with the asteroid, the magnitudes of the initial and final momenta are not the same, and the change in energy (and thus, by conservation of energy, the energy required to change it's trajectory) is just:
        \begin{equation}
                \Delta E = \frac{\Delta p^{2}}{2m} = \frac{1}{2}mv_{0}^{2}\frac{R_{\oplus}^{2}}{R^{2}}\left(1+\frac{v_{esc}}{v_{0}}\right)^{2}
        \end{equation}
        Suppose the asteroid is the same size as the Chicxulub impactor, and is detected at the moon's orbital distance.  The asteroid's mass can be determined assuming it has an average density of $\rho=\SI{5}{\gram\per\cm\cubed}$:
        \begin{equation}
                m = \frac{4\pi}{3}\rho R^{3} \sim \SI{3e18}{\gram}.
        \end{equation}
        If the earth-moon orbital distance is not known, it can be determined from Kepler's 3rd law:
        \begin{align}
                \left(\frac{P}{\si{\yr}}\right)^{2} &= \left(\frac{\si{\msun}}{M}\right)\left(\frac{R}{\si{\AU}}\right)^{3}\\
                R &= \left(\frac{P}{\si{\yr}}\right)^{2/3}\left(\frac{M}{\si{\msun}}\right)^{1/3}\:\si{\AU}\\
                R &\sim \SI{2e-3}{\AU} \sim \SI{3e5}{\km}
        \end{align}
        The initial velocity can be assumed to be the free-fall velocity at \SI{1}{\AU}, i.e.,
        \begin{equation}
                v_{0} = \sqrt{\frac{2G\si{\msun}}{\SI{1}{\AU}}} \sim \SI{4e6}{\cm\per\second}.
        \end{equation}
        Thus, the energy required is:
        \begin{equation}
                \Delta E \sim \SI{1.5e28}{\ergs}.
        \end{equation}
        Note that gravitational focusing had a minimal effect on the answer.  This is because the asteroid is moving quickly; an object with a small relative velocity (say, small bodies near Earth during solar system formation) will be focused much more effectively.

        \section{Destruction}
        Smaller asteroids will have a smaller impact velocity as they have a smaller terminal velocity.  Terminal velocity is defined as the velocity at which the drag force and gravitational force balance out, so:
        \begin{align}
                mg &= \frac{1}{2}\rho_{A} v_{T}^{2}c_{D}A\\
                v_{T}^{2} &= {\frac{2mg}{\rho_{A} c_{D}A}},
        \end{align}
        where $\rho_{A}$ is the density of air, $A$ is the cross-sectional area of the falling object, and $c_{D}$ is the drag coefficient.  We can use $mg$ for the gravitational force because the height of the atmosphere is small compared to the radius of the earth, especially when you consider that the bulk of air resistance is in the lower atmosphere, where the air is more dense. The energy of the impact thus goes as:
        \begin{equation}
                E = \frac{1}{2}mv_{T}^{2} \propto m^{2}A^{-1}.
        \end{equation}
        The area can be written in terms of the mass if we know the density of the asteroid:
        \begin{align}
                A &= \pi R^{2}\\
                m &= \frac{4\pi}{3}\rho R^{3}\\
                A &= \pi \left(\frac{3m}{4\pi\rho}\right)^{2/3}.
        \end{align}
        Thus, we have:
        \begin{equation}
                E\propto m^{4/3}
        \end{equation}
        Consider now, instead of one large asteroid of mass $m$, we have $N$ smaller asteroids, each of mass $m_{N}\equiv \frac{m}{N}$.  The energy then goes as
        \begin{equation}
                \Sigma{E} \propto N\left(\frac{m_{N}}{N}\right)^{4/3} \propto N^{-1/3}
        \end{equation}
        Thus, to reduce the total impact energy by a factor of 10, we would need to break the asteroid into $N=1000$ smaller impactors.

\end{document}


\subsection{Introduction}

OLS:

\(\hat \theta =\frac{\sum_i (X_i-\mu_X)(y_i-\mu_y)}{\sum_i(x_i-\mu_X)^2}\)

Trimming

\(\hat \theta =\frac{n^{-1}\sum_i (X_i-\mu_X)(y_i-\mu_y)\mathbf 1_i}{n^{-1}\sum_i(x_i-\mu_X)^2\mathbf 1_i}\)

Where:

\(\mathbf 1_i=\mathbf 1(\hat f(z_i)\ge b)\)

Where \(b=b(n)\) is a trimming parameter, where:

\(b\rightarrow 0\) as \(n\rightarrow \infty \)


\chapter{Polynomial and Rational Functions}
\section{Polynomials}
A polynomial function of degree $n$ is defined as follows:
\begin{equation}
  f(x)=a_nx^n+a_{n-1}x^{n-1}+a_{n-2}x^{n-2}+...+a_1x+a_0
\end{equation}

Functions of this classification for which $n=2$ are defined as
\textit{quadratic functions}, and look as follows: $f(x)=ax^2+bx+c$.

\begin{itemize}
  \item For all $a>0$, the parabola opens upward
  \item For all $a<0$, the parabola opens downwards
\end{itemize}

The standard form of a quadratic equation is given as follows:
\begin{equation}
  f(x)=a(x-h)^2+k
\end{equation}

To identify the vertex of a function in this classification, simply use the
coordinate pair $(h,k)$.  If the quadratic is not in this format, simply
complete the square, and you will have your $h$, $k$ pair.

When a quadratic opens upwards and you have the vertex, then that is the
relative minimum of the function.  In the other case, you have the relative
maximum of that function.

\section{Higher-Order Functions}
If you are presented with a function that has a degree higher than 2, (i.e., it
is not a quadratic), then you can at least determine the end-behavior of that
function.  Consider the function $f(x)=3x^3+7x^2+6$:

\graph{3*x^3 + 7*x^2 + 6}

We can see that the function has two limits, namely:
\begin{itemize}
  \item $$\lim_{x\to\infty} f(x)=\infty$$
  \item $$\lim_{x\to-\infty} f(x)=-\infty$$
\end{itemize}

We can evaluate the limits of a function at $\infty$ and $-\infty$ simply by
evaluating the leading term (by power), which in the case of $f(x)$ as defined
above is $3x^3$.  If we evaluate that function at both a large positive, and
small negative value, the sign of the output will tell us whether the limit
approaches positive or negative infinity.

\subsection{Zeros}
\begin{enumerate}
  \item $x=a$ is a zero of the function $f(x)$
  \item $x=a$ is a solution of the equation $f(x)=0$
  \item $(x-a)$ is a factor of the function $f(x)$
  \item $(a,0)$ is an $x$-intercept of the function $f(x)$
\end{enumerate}

\subsection{Multiplicity}
For any zero of a function, you can determine what it does at the x-axis
(whether it crosses it or "bounces" across it) by determing the multiplicty of
that factor.  Consider the function $f(x)=x^3-7x^2+16x-12$ which factors into
$f(x)=(x-2)^2(x-3)$.  The factor $(x-2)$ follows the form $(x-a)^k$, where
$a=-2$ and $k=2$.  Therefore, at $x=2$, the function will bounce across the
x-axis because $k$ is even.  If $k$ were to be odd, it would cross through the
axis, as it does at $x=3$.

\subsection{Intermediate Value Theorem}
If $a$ and $b$ are real numbers, and $f(x)$ is continuous and defined upon the
interval $[a,b]$, there exists a $c$ such that $a<c<b$ and $f(a)<f(c)<f(b)$.

\section{Real Zeros of Polynomial Functions}
\subsection{Long Division}
Much like you did in second grade, you can prefom long division upon
polynomials.

TODO

You will end up with a function that can be composed as follows:
\begin{equation}
  f(x)=d(x)q(x)+r(x)
\end{equation}

\subsection{Synthetic Division}
If the monomial that you are trying to divide into follows the form $(x-k)$,
then you can preform what is known as synthetic division.  To do synthetic
division, take note of the following steps:

\begin{enumerate}
  \item Write all of the terms, by power order (0 for powers that don't exist in
  the original equation) for all powers $n$ in the original polynomial
  \item Write $k$ on the left side of all of that such that $k$ is found in
  $(x-k)$
  \item Drop the first term
  \item Multiply it by $k$ ($ka_n$)
  \item Place it up under the coeffiient $a_{n-1}$
  \item Add the two ($ka_n+a_{n-1}$)
  \item Repeat
\end{enumerate}

\subsection{The Remainder Theorem}
If a polynomial $f(x)$ is divided into $(x-k)$, the remainder is $r=f(k)$.

\subsection{The Ratioinal Zero Test}
If a polynomial is defined as follows:
\begin{equation}
f(x)=a_nx^n+a_{n-1}x^{n-1}+a_{n-2}x^{n-2}+...+a_1x+a_0
\end{equation}

Then all of the rational zeros must be defined in the set of numbers that look
like the following:

\begin{equation}
  \text{rational zeros}=\frac{\text{factors of the constant}}{
    \text{factors of the leading coefficient}}
\end{equation}

\section{Complex Numbers}
So far, we have learned about numbers that come in \textit{standard form}, such
as $1$, $3$, and others.  There is another classificaiton of numbers, called
\textit{complex numbers} that come in the form $a+bi$.

$i$ is defined as follows:

\begin{equation}
  i=\sqrt{-1}
\end{equation}

You can easily raise $i$ to several different powers (1..4) and it will obey
certain, easily derivable rules:

\begin{enumerate}
  \item{$i^1=i$}
  \item{$i^2=-1$}
  \item{$i^3=i^2i^2=-i$}
  \item{$i^4=i^2i^2=1$}
\end{enumerate}

Adding and subtracting complex numbers works as you'd expect. You add the real
terms, and then add the complex terms, factoring out the $i$.

Multiplying complex numbers is only slightly more complicated.  While it works
normally as you'd expect, you have to remember the power rules of $i$ as defined
above: sometimes you can end up with a negative number.

\subsection{Complex Conjugates}
To remove the complex part of a complex number, you can multiply it by the
\textit{complex conjugate} of itself.  The complex conjugate of $(a+bi)$ is
defined as $(a-bi)$.

\subsection{Principal Square Root}
To take the square root of a negative number, say $\sqrt{-2}$, you can remember
the rules of the square-root, and expand the above expression into
$\sqrt{2}\sqrt{-1}$, which is just $i\sqrt{2}$.

\section{The Fundamental Theorem of Algebra}
The \textit{Fundamental Theorem of Algebra} is defined such that for any
function $f(x)$ which has degree $n$, there is at least one zero of $f$ in the
complex number system.

\section{Asymptotes}
An asymptote of a function is defined as a part of a function ($f(x)$) such that
$f(x)$ comes arbitrairly close to the asymptote, but never actually approaches
it.  For example, $f(x)=ln(x)$ has a vertical asymptote at $x=0$.

\graph{ln(x)}

\begin{enumerate}
  \item{A horizontal asymptote can be evaluated by taking the limit of a
      function as it approaches some endpoint.  As $x$ moves to some endpoint,
    the function $f(x)$ will approach some value, $g(x)=b$.}
  \item{A vertical asymptote is evaluated by finding all of the $x$-values
      outside the domain of $f(x)$.  If $f(x)$ is not defined for $x=a$, then
      $x=a$ is a vertical asymptote of that function.}
\end{enumerate}

\section{Graphing Rational Functions}
To graph a rational function, one must simply take note of the following steps:
\begin{enumerate}
  \item{Simplify $f$, if possible}
  \item{Plot the $y$-intercept}
  \item{Plot the $x$-intercept(s)}
  \item{Sketch any asymptotes}
  \item{Use curves to complete the graph}
\end{enumerate}


\lesson{1}{Oct 11 2021 Mon (10:42:23)}{Greatest Common Factors and Special Products}{Unit 2}

\subsubsection*{Greatest Common Factor}

A greatest common factor of two or more terms is the largest factor that all terms have in common. The greatest common factor of a polynomial should be factored out first before any further factoring is completed.

\begin{example}[GCF]
    \begin{align}
        3r^6 + 27r^4 + 15r^2 &= 3r^2(r^4 + 9r^2 + 5) \\
    \end{align}

    When you multiply ($\times$) variables, add the exponents:

    \begin{align}
         r^2 \times r^4 &= rr \times rrrr \\
                        &= r^6
    \end{align}

    When factoring a \bf{GCF}, subtract the exponents:

    \begin{align}
        \rm{To factor $r^2$ from $r^6$: $r^{6 - 2} = r^4$} \\
        rrrrrr = (rr)(rrrr) = r^2(r^4)
    \end{align}
\end{example}

\subsubsection*{Difference of Squares Binomials}

A difference of \bf{squares binomial} includes a \bf{perfect square} term subtracted by another \bf{perfect square term}.

\begin{example}[Difference of Squares Binomials]
    \begin{align}
        a^2 - b^2 = (a + b)(a - b) \\
        r^2 - 4 = (r + 2)(r - 2)
    \end{align}
\end{example}

\subsubsection*{Perfect Square Trinomials}

A \bf{perfect square trinomial} is a polynomial of \bf{three terms} where the \it{first} and \it{last} terms are \bf{perfect squares} and the \bf{middle term} is \bf{twice the product of the square roots} of those terms.

\begin{example}[Perfect Square Trinomials]
    \begin{center}
        Pattern:
    \end{center}

    \begin{align}
        a^2 + 2ab + b^2 = (a + b)^2 \rm{ OR } a^2 - 2ab + b^2 = (a - b)^2
    \end{align}

    \begin{center}
        Example:
    \end{center}

    \begin{align}
        r^2 + 12r + 36 = (r + 6)^2 \rm{ OR } r^2 - 12r + 36 = (r - 6)^2
    \end{align}
\end{example}

\newpage


% !TEX root = index.tex

\section{Principal Curvatures}

\epigraph{If you are receptive and humble, mathematics will lead you by the hand.}{Paul Dirac}

For the graph $z=f(x,y)$ the Gaussian and Mean curvatures are the determinant and trace of $\hess(f)$ respectively, and are invariant under rotation of the $xy$-plane. We can ask: how much can we simplify $\hess(f)$ by replacing it with a suitable $P^{-1} \hess(f) P$?\\ Answer: A Lot.\\

The Hessian $ \hess(f)$ is a symmetric $ 2 \times 2$ matrix, hence by the \textbf{Spectral Theorem}, there exist orthonormal vectors (eigenvectors) $ v_1, v_2$ and constants (eigenvalues) $ \kappa_1, \kappa_2$ such that for the matrix $ P$ whose columns are $v_1, v_2$ we have
\begin{align}
	P^{-1} \hess(f) P  = \begin{bmatrix} \kappa_1 & \\ & \kappa_2 \end{bmatrix}
\end{align}
The set $\{ \kappa_1, \kappa_2 \}$ is uniquely determined by $\hess(f)$.

\begin{definition}
	The eigenvalues $ \kappa_1, \kappa_2$ of $ A$ are called the \textbf{principal curvatures} and the eigenvectors $ v_1, v_2$ are called the \textbf{principal directions} at $p$.
\end{definition}
\begin{remark}
	In the case $ \kappa_1 = \kappa_2$ all directions are principal.
\end{remark}
\begin{ques}
	Show that the various curvatures are related as follows:
	\begin{align}
		\nonumber H &= (\kappa_1 + \kappa_2)/2 \\
		\nonumber K &= \kappa_1 \kappa_2 \\
			\label{eq:eigenvalues}
		&\kappa_1, \kappa_2 \mbox{ are the roots of } \kappa^2 - 2H \kappa + K
	\end{align}
\end{ques}
\begin{ques}
	Find the Principal Curvatures and Principal Directions of the curves you analyzed yesterday
	\begin{description}
		\item[The Perfect \textbf{Potato Chip}: ]  $ z = x^2 - y^2 $
		\item[Cylindrical Potato: ] $z = -\sqrt{r^2 - x^2} $
		\item[Spherical Potato: ] $z = -\sqrt{r^2 - x^2 - y^2}$
		\item[Parabolic Cylinder: ] $z =  x^2$
	\end{description}
	(You can use the estimate $-\sqrt{r^2 - \alpha} \approx -r + \frac{\alpha}{2r} $.)
\end{ques}
\noindent This is all the algebra we'll be needing. We'll now start analyzing the curvatures geometrically.

% \subsection{Principal Curvatures as Good Approximations}
% We can rotate the $xy$-plane about the $z$-axis so that the two axes now point in the direction of the vectors $v_1$ and $v_2$. Without loss of generality let us assume that $v_1$, $v_2$ point in the $x, y$ direction to begin with. Then the Hessian becomes
% \begin{alignat*}{4}
% \hess(f) = &	\begin{bmatrix}
% 		f_{xx} & f_{xy} \\
% 		f_{xy} & f_{yy}
% 	\end{bmatrix} & \quad = \quad &
% 	\begin{bmatrix}
% 		\kappa_1 & 0         \\
% 		0         & \kappa_2
% 	\end{bmatrix}                                                   \\
% 	\Rightarrow \quad &
% 	f(x, y)                & \approx \quad & f(p) + \kappa_1 \cdot \dfrac{x^2}{2} + \kappa_2 \cdot \dfrac{y^2}{2}
% \end{alignat*}
% We've seen such expressions before. The right hand side is the quadratic approximation of the function
% \begin{align*}
% 	f(x,y) &= \sqrt{ f(p)^2 - }
% \end{align*}
%
% This is the geometric significance of the principal directions and principal curvatures:
% \begin{prop}
% 	\label{thm:principal_curvatures}
% 	If $ \kappa_1, \kappa_2 $ are the principal curvatures of the graph of $f(x,y)$ at a critical point $ p$ then near $ p$, after possibly rotating the $xy$-plane about the $z$-axis, the quadratic function that \textbf{best approximates} $f$ is
% 	\begin{align*}
% 		f(p) + \kappa_1\dfrac{x^2}{2} + \kappa_2\dfrac{y^2}{2}
% 	\end{align*}
% \end{prop}
% This is completely analogous the situation in one variable where the circle of radius $1/\kappa$ provided a best approximation for a curve. In 2 dimensions, it's not enough to look at circles alone instead we need to allow for 2 variables {\color{red}something}.
%
% \begin{ques}
% 	For the graph $ z =f(x,y)$ show that the equation of the tangent plane at $ p = (0,0)$ is given by
% 	\begin{align*}
% 		z = f(p) + f_x(p) x + f_x(p) y
% 	\end{align*}
% 	Compare this to Proposition \ref{thm:principal_curvatures}.
% \end{ques}

\begin{figure}[H]
	\centering
	\includegraphics[width=8cm]{normal_curvature}
	\caption{Principal Directions and Normal Curvatures. Image from Wikipedia.}
\end{figure}

\subsection{Curves on a Potato}
Without any loss of generality assume that the $x, y$ axes are the principal directions with principal curvatures $\kappa_1$, $\kappa_2$ respectively. This is equivalent to saying that the degree 2 Taylor approximation of $ f$ is
\begin{align*}
	f(x,y) &\approx f(0,0) + \kappa_1 \cdot \dfrac{x^2}{2} + \kappa_2 \cdot \dfrac{y^2}{2}
\end{align*}
Let us look at the \emph{curves on the surface} passing through $(0,0)$ and compute their curvature in terms of $ \kappa_1, \kappa_2$. We'll consider curves of the form
\begin{align*}
	c_\theta(t)
	 & = (t \cos \theta, t \sin \theta, f(t\cos \theta,t\sin \theta))
\end{align*}
Note that upon projecting onto the $xy$-plane $c_\theta(t)$ projects onto the straight line $y=\tan \theta \cdot x$. Applying the formula for curvature we get:
\begin{align*}
	c'_\theta(t) &= (\cos \theta, \sin \theta, f'(t\cos \theta,t\sin \theta) \\
	c''_\theta(t) &= (0, 0, f''(t\cos \theta,t\sin \theta))
\end{align*}
Because $c(0)=(0,0)$ is the critical point, the tangent plane to $S$ at $(0,0)$ is horizontal and hence we must have $f'(t\cos \theta,t\sin \theta)|_{t=0}=0$ so that
\begin{align*}
	c'_\theta(0) &= (\cos \theta, \sin \theta, 0)
\end{align*}
which is unit length. By explicitly computing the cross product we get
\begin{align*}
	\kappa
	&= |c''_\theta \times c'_\theta|_{t=0} \\
	&= |f''(t\cos \theta,t\sin \theta)|_{t=0}
\end{align*}
We'll remove the absolute value $\lvert - \rvert$ and compute the \textbf{signed curvature}
\begin{align*}
	\kappa &= f''(t\cos \theta,t\sin \theta)|_{t=0}
\end{align*}
\begin{definition}
	\label{def:normal_curvature}
	The signed curvature of $c_\theta(t)$ at $t=0$ is called the \textbf{normal curvature} at $p$ along the direction $(\cos \theta, \sin \theta)$ (we will abbreviate this as just the direction $\theta$).
\end{definition}
We can compute this using the Taylor approximation
\begin{align*}
	f(t\cos \theta,t\sin \theta) &\approx f(0,0) + \kappa_1 \cdot \dfrac{(t \cos \theta)^2}{2} + \kappa_2 \cdot \dfrac{(t \sin \theta)^2}{2} + \cdots \\
	\Rightarrow \qquad
	f''(t\cos \theta,t\sin \theta) &\approx  \kappa_1 \cdot \cos^2 \theta + \kappa_2 \cdot \sin^2 \theta + \cdots
\end{align*}
When we plug in $t=0$ the higher degree terms in the Taylor approximation vanish and we get
\begin{align*}
	\kappa = f''(t\cos \theta,t\sin \theta)|_{t=0} &=  \kappa_1 \cdot \cos^2 \theta + \kappa_2 \cdot \sin^2 \theta
\end{align*}

\begin{prop}
	With the notation as above, the normal curvature at the point $p$ in the direction $\theta$ equals
	  \begin{align*}
			\kappa &=  \kappa_1 \cdot \cos^2 \theta + \kappa_2 \cdot \sin^2 \theta
	  \end{align*}
\end{prop}

\begin{thm}
	\label{thm:extremal_curvatures}
	Without any loss of generality assume that $\kappa_1 \ge \kappa_2$. Then the maximum and minimum normal curvatures at the point $p$ are $\kappa_1$ and $\kappa_2$ respectively, in the corresponding principal directions.
\end{thm}
\noindent Proof is in the following exercise.
\begin{ques}
	Assume $\kappa_1 \ge \kappa_2$. Prove that $\kappa_1 \cos^2 \theta  + \kappa_2 \sin^2 \theta$ attains it maximum at $ \theta = 0, \pi$ and minimum at $ \theta = \pi/2, 3 \pi/2$ and the maximum value is $ \kappa_1$ and the minimum value is $ \kappa_2$.
\end{ques}
\begin{ques} There is another sense in which the Mean Curvature is the mean: it is the mean of all normal curvatures.
	\begin{enumerate}
		\item Plot $ r=\kappa_1 \cos^2 \theta  + \kappa_2 \sin^2 \theta$ in polar coordinates. Interpret this geometrically.
		\item Show that
		\begin{align*}
			H = \dfrac{1}{2 \pi} \int \limits_{0}^{2 \pi } \kappa_1 \cos^2 \theta  + \kappa_2 \sin^2 \theta \: d \theta
		\end{align*}
	\end{enumerate}
\end{ques}
\begin{remark}
	Because curvature does not change when we rotate $\R^3$ we can drop the condition that $p$ is a critical point from the above theorem. It is worth rephrasing the theorem to state this.
\end{remark}
\begin{thm}
	At every point $p$ on a surface $S$ there are two perpendicular directions (principal directions) along which the normal curvatures attain the maximum and minimum (principal curvatures).
\end{thm}

\subsection{Classification of Points}
The signs of the principal curvatures allow us to classify the points on the surface.
\begin{figure}[H]
	\centering
\begin{tabular}{|l|l|l|}
	\hline
	& the surface looks like a ... & such points are called ...\\ \hline
	$\kappa_1, \kappa_2 $ both positive/negative & ellipsoid &  elliptic \\ \hline
	$\kappa_1, \kappa_2$ have opposite signs & hyperboloid & hyperbolic \\ \hline
	$\kappa_1 \kappa_2 = 0$ & cylinder/plane & parabolic \\\hline
\end{tabular}
\end{figure}
\begin{figure}[H]
  \centering
  \begin{subfigure}[t]{0.30\textwidth}
    \centering
    \includegraphics[width=4cm]{sphere}
    \caption{$ K > 0$}
  \end{subfigure}
	\begin{subfigure}[t]{0.30\textwidth}
    \centering
    \includegraphics[width=3cm]{wafer}
    \caption{$ K < 0 $}
  \end{subfigure}
	\begin{subfigure}[t]{0.30\textwidth}
    \centering
    \includegraphics[width=3cm]{parabola1}
    \caption{$K = 0$}
  \end{subfigure}
\end{figure}
Note that because $K = \kappa_1 \kappa_2$ the above three classifications are essentially the classifications based on Gaussian curvature. This already suggests that the Gaussian curvature sees some \emph{intrisic} curvature of surfaces.\\

\begin{ques}
	Classify the points on a torus as elliptic, hyperbolic, or parabolic.
	\begin{figure}[H]
    \centering
    \includegraphics[width=10cm]{torus}
  \end{figure}
\end{ques}

\newpage
\subsection{Appendix: Spectral Theorem}
A basis $ v_1, v_2, \cdots, v_n$ of $ \R^n$ is said to be \textbf{orthonormal} if $ \norm{v_i}=1 $ for each $ i$ and $ v_i \cdot v_j = 0$ for $ i \neq j$.

\begin{thm}[Spectral Theorem for Symmetric Matrices]
	Let $ A$ be a \textbf{symmetric} $ n \times n$ matrix with real entries. Then there exists an orthonormal basis $ v_1, v_2, \cdots, v_n$ basis of $ \R^n$ and real numbers $ \kappa_i$ such that
	\begin{align*}
		A v_i = \kappa_i v_i
	\end{align*}
	Equivalently, if $P$ is the matrix with columns $ v_i$ then
	\begin{align*}
		P^{-1} A P =
		\begin{bmatrix} \kappa_1 &           &        &           \\
			          & \kappa_2 &        &           \\
			          &           & \ddots &           \\
			          &           &        & \kappa_n
		\end{bmatrix}
	\end{align*}
	The $ \kappa_i$ (\textbf{eigenvalues}) are unique up to permutations and the orthonormal basis $ v_1, v_2, \cdots, v_n$ (\textbf{eigenvectors}) and the matrix $ P$ are `essentially' unique.
\end{thm}
The following exercise outlines the proof for $n=2$.

\begin{ques}
	Consider the symmetric matrix $A = \begin{bmatrix} a & b \\ b & c\end{bmatrix}$. Assume that $A$ is not a scalar matrix i.e. $A \not= \begin{bmatrix} a & 0 \\ 0 & a\end{bmatrix}=aI$ (this case is trivial).
	\begin{enumerate}
		\item Show that the equation
		\begin{align*}
			\det(A - \kappa I) = \det \begin{bmatrix} a -\kappa & b \\ b & c - \kappa \end{bmatrix} = 0
		\end{align*}
		has 2 distinct real solutions $\kappa_1, \kappa_2$.
		\item Argue that there exist 2 linearly independent unit length vectors $v_1, v_2$ satisfying
		$ A v_i = \kappa_i v_i$
		for $i = 1, 2$.\hint{Look at the kernel of $A - \kappa_i I$.}
		\item Prove that $v_1 \perp v_2$.\hint{Find $v_1 ^ T A v_2$ in 2 different ways.}
	\end{enumerate}
	Let $P$ be the matrix with column vectors $v_1, v_2$.
	\begin{enumerate}[resume]
		\item Prove that $P^T = P^{-1}$.
		\item By an explicit computation show $P^T A P = \begin{bmatrix} \kappa_1 & 0 \\ 0 & \kappa_2 \end{bmatrix}$
	\end{enumerate}
\end{ques}


\subsection{Definition and existence}
\begin{Definition}{qr-decomposition}
  The \define{QR decomposition} of a matrix $\mata\in\C^{m\times n}$
  with $m\ge n$ is the product
  \begin{gather}
    \mata = \matq\matr,
  \end{gather}
  such that $\matq \in\C^{m\times n}$ is unitary and
  $\matr\in \C^{n\times n}$ is upper triangular.
\end{Definition}

\begin{Lemma}{qr-columns}
  Let $\mata = \matq\matr$. Then, the column vectors of $\mata$ and
  $\matq$ admit the relation
  \begin{gather}
    \va_k = \sum_{i=1}^k r_{ik} \vq_i.
  \end{gather}
  If $r_{ii}\neq 0$ for $i=1,\dots,k$, this relation is uniquely
  invertible. In particular,
  \begin{gather}
    \operatorname{span}\{\vq_1,\dots,\vq_k\}
    =
    \operatorname{span}\{\va_1,\dots,\va_k\}.
  \end{gather}
\end{Lemma}

\begin{Theorem}{qr-existence}
  Every matrix $\mata\in\C^{m\times n}$ with $m\ge n$ of full rank
  admits a QR decomposition. It is unique under the condition that for
  all $i$ there holds $r_{ii} > 0$.
\end{Theorem}

\subsection{Householder transformations}

\begin{Definition}{householder-transformation}
  The \define{Householder transformation}
  associated with a vector $\vw\in\C^n$ is
  \begin{gather}
    \matq_w = \id - 2\frac{\vw\vw^*}{\vw^*\vw}
  \end{gather}
  It is also called \define{Householder matrix} or, particularly in
  the real case, \define{Householder reflection}.
\end{Definition}

\begin{Lemma}{householder-symmetry}
  For any vector $\vw\in\C^n$ the Householder transformation
  $\matq_{\vw}$ is Hermitian and orthogonal, that is,
  \begin{gather}
    \matq^{-1} = \matq^* = \matq.
  \end{gather}
\end{Lemma}

\begin{Lemma}{householder-qr}
  For any vector $\vy\in\C^n$ there are vectors $\vw_\phi\in\C^n$ such
  that $\matq_{\vw_\phi} \vy$ is a multiple of $\ve_1$.

  The vector of choice for numerical stability is
  \begin{gather}
    \vw = \vy + e^{i\phi} \norm{\vy}_2\ve_1,
  \end{gather}
  where $\phi$ is the phase of $y_1$.
\end{Lemma}

\begin{proof}
  The statement of the lemma says that for a suitable vector $\vw$ there is a complex number
  $\alpha$ such that $\matq_{\vw} \vy = \alpha \ve_1$. Since
  $\matq_{\vw}$ preserves the Euclidean norm, we already know
  \begin{gather}
    \abs{\alpha} = \norm{\vy}_2.
  \end{gather}
  There holds
  \begin{gather}
    \alpha\ve_1 = \matq_{\vw} \vy
    = \vy - 2 \frac{\vw\vw^*\vy}{\vw^*\vw}
    = \vy - w \frac{\vw^*\vy}{\vw^*\vw}\vw.
  \end{gather}
  Thus, $\vw$ is in the span of $\vy-\alpha \ve_1$. Since we divide by
  its norm, its length does not matter and we let
  \begin{gather}
    \vw_\phi =
    \begin{pmatrix}
      y_1 - e^{i\phi} \norm{\vy}_2\\y_2\\\vdots\\y_n
    \end{pmatrix},
    \qquad
    \matq_{\vw_\phi}\vy =
    \begin{pmatrix}
      e^{i\phi} \norm{\vy}_2\\0\\\vdots\\0
    \end{pmatrix}.
  \end{gather}
  Since the computation of the first component of $\vw_\phi$ is prone to loss of significance, we choose $\phi$ as the phase of $-y_1$.
\end{proof}

\subsection{Givens rotation}

\begin{Definition}{givens}
  The \define{Givens-Rotation} $\Omega_{jk}$ for $j<k$ with angle $\theta$ is the matrix
  \begin{gather}
      \Omega_{jk} =
    \begin{pmatrix}
      \id \\
      &c&\cdots&s\\
      &\vdots&\id &\vdots\\
      &-s&\cdots&c\\
      &&&&\id
    \end{pmatrix}
    \in\Cnn.
  \end{gather}
  The corresponding mapping $\Omega_{jk}\colon x\mapsto y$ is defined by
  \begin{gather}
    y_i =
    \begin{cases}
      c x_j + s x_k & i=j\\
      -s x_j + c x_k & i=k\\
      x_i &\text{else}
    \end{cases}
  \end{gather}
  where $c = \cos\theta$ and $s = \sin\theta$.
\end{Definition}

\begin{remark}
  The entries $c$ and $s$ of the rotation matrix are in rows and
  columns $j$ and $k$.  The identity matrices $\id$ are of
  corresponding dimensions.

  Applying $\Omega_{jk}$ to a matrix $\mata$ from the left modifies rows  $j$ and $k$ of $\mata$. Thus, it is a row operation similar to Gauss elimination.

  The action of $\Omega_{jk}$ on a vector corresponds to the rotation
  in the plane spanned by $\ve_j$ und $\ve_k$. Thus, it is sufficient to investigated $2\times2$-matrices.

  Note that the notation $\Omega_{jk}$ misses the angle $\theta$. This
  is due to the fact that it is always determined such that $y_k=0$.
\end{remark}

\begin{Lemma}{givens-computation}
  Givens rotation $\Omega_{jk}^*$ eliminates the second component of the vector
  $(x_j,x_k)^T$ by choosing
  \begin{gather}
    r = \sqrt{x_j^2+x_k^2},\qquad
    c = \frac{x_j}r,\quad s = \frac{x_k}r.
  \end{gather}
  We obtain
  \begin{gather}
    \begin{pmatrix}
      c & s \\ -s & c
    \end{pmatrix}
    \begin{pmatrix}
      x_j\\x_k
    \end{pmatrix}
    =
    \begin{pmatrix}
      r\\0
    \end{pmatrix}
    .
  \end{gather}
\end{Lemma}

\begin{remark}
  Computation of $r$ in the previous lemma is prone to numerical
  overflow due to computation of $x_j^2$ or $x_k^2$, even if $r$
  itself is within the numerical range. For the implementation, we can
  use the function \lstinline!hypot!. It computes the hypothenuse of a
  right-angled triangle without overflow.
\end{remark}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "main"
%%% End: