% Copyright 2018 Melvin Eloy Irizarry-Gelpí
\chapter{Reals}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
The real numbers are the set of familiar numbers. There are integers, rationals, and irrationals.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Arithmetic Operations}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
For future reference, here you can find the familiar arithmetic operations on real numbers.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Addition}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
The addition operation is familiar. If $a$ and $b$ are two real numbers, then the result $c$ of the addition of $a$ and $b$ is also a real number:
\begin{equation}
    c = a + b
\end{equation}
There is an element $0$ in the reals called zero that does not contribute to the addition:
\begin{equation}
    a = a + 0 = 0 + a
\end{equation}
Note that zero does not have a sign.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Multiplication}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
The multiplication operation is also familiar. If $a$ and $b$ are two real numbers, then the result $c$ of the multiplication of $a$ and $b$ is also a real number:
\begin{equation}
    c = a \wedge b
\end{equation}
There is an element $1$ in the reals called one that does not contribute to the multiplication:
\begin{equation}
    a = a \wedge 1 = 1 \wedge a
\end{equation}
Note that one is positive.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Negation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
There is an element $-1$ in the reals that when multiplied with a real number $a$ gives the negative of $a$:
\begin{equation}
    {-a} = a \wedge {-1} = {-1} \wedge a
\end{equation}
Note that one is negative. This operation is a 2-involution.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conjugation Operations}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
The conjugation operations on real numbers are trivial, but are needed for Cayley-Dickson construct where they lead to non-trivial conjugations.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Asterisk Conjugation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
The asterisk conjugate operation on a real number $a$ is itself:
\begin{equation}
    a^{\ast} = a
\end{equation}
That is, all real numbers are asterisk-self-conjugate. This operation is trivially a 2-involution.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Quadrance}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
The quadrance of a real number $a$ is defined as the product of $a$ and the asterisk conjugate of $a$:
\begin{equation}
    \Vert a \Vert^{2} = a^{2} = a \wedge a^{\ast} = a^{\ast} \wedge a
\end{equation}
Note that the quadrance of any non-zero real number is positive, and that zero is the only real number with zero quadrance.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Cloak Conjugation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
The cloak conjugate operation on a real number $a$ is equivalent to the negation operation:
\begin{equation}
    a^{\diamond} = {-a}
\end{equation}
This operation is a 2-involution.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Dagger Conjugation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
The dagger conjugate operation on a real number $a$ is itself:
\begin{equation}
    a^{\dagger} = a
\end{equation}
That is, all real numbers are dagger-self-conjugate. This operation is trivially a 2-involution.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Hodge Star}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
The Hodge star operation on a real number $a$ is itself:
\begin{equation}
    a^{\star} = a
\end{equation}
That is, all real numbers are star-self-conjugate. This operation is trivially a 2-involution.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Differential Operators}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
You can define a real variable $z$ and a \textbf{Wirtinger} differential operator $\nabla$ such that
\begin{equation}
    \nabla \wedge z = 1
\end{equation}
That is,
\begin{equation}
    \nabla = \frac{\partial}{\partial z}
\end{equation}
Since real numbers are asterisk-self-conjugate, the asterisk Wirtinger differential operator $\nabla^{\ast}$ is equivalent to $\nabla$:
\begin{equation}
    \nabla^{\ast} = \nabla
\end{equation}
The quadrance of the Wirtinger differential operator is the \textbf{Laplace} differential operator:
\begin{equation}
    \Vert \nabla \Vert^{2} = \frac{\partial^{2}}{\partial z^{2}} = \nabla \wedge \nabla^{\ast} = \nabla^{\ast} \wedge \nabla
\end{equation}


\lab{Algorithms}{Functions and Logic}{Functions and Logic}

\objective{This section will teach how to build functions. It will also introduce logical statements used in programming.}

Up to this point we have only run scripts. We now introduce a much more powerful and versatile method of problem solving: writing functions.

A function, in the programming sense, is a block of code that accepts input (arguments) and returns output.  When you need a function, you simply define one.  Python functions begin with the keyword \li{def}, followed by the function name.  Input arguments are listed inside parentheses separated by commas.  Finally the function definition ends with a colon.  In Python, every function returns a value. This value can be explicitly returned using a \li{return} statement.  If no \li{return} is defined, the function will always return \li{None}.  Below is a simple function that returns what it receives
\begin{lstlisting}[style=python]
: def returnInput(x):
....: return x
\end{lstlisting}

Note that all code inside the function is indented.  This is very important in Python.  Any code not indented under the function definition is not part of the function.  To execute a function, you call the function by name and pass input parameters.  Try calling the function and passing different arguments.
\begin{lstlisting}[style=python]
: returnInput("This is a string")
'This is a string'
: returnInput(37)
37
: returnInput(3**2)
9
\end{lstlisting}

This function is defined for as long as we have IPython running.  If we happen to close IPython, our function definition will be lost.  To avoid losing our function definitions, we can place them in a Python script file (a *.py file).

\begin{problem}
In a file, functions.py, define functions that will do the following:
\begin{itemize}
\item Accept anything, but always return the number 42
\item Accept two numbers and return their product
\item Accepts no arguments and prints ``You called!"
\end{itemize}
\end{problem}

Restart IPython.  Now, how do we use our functions defined in functions.py without having to redefine each function inside IPython.  Fortunately, Python allows us to import functions.  Each *.py is really a Python module which can be imported in the same fashion as the SciPy library.
\begin{lstlisting}[style=python]
: import functions
\end{lstlisting}

Now our function are accessible as \li{functions.<function_name>}.  Using the \li{as} keyword, we can assign an alias to our module.  Let us define a function in \li{functions.py} that will compute the roots of a quadratic equation.  This function will return a tuple containing multiple values.  Note that before using the \li{sqrt} function, we have to define it, which we do by importing \li{sqrt} from Python's \li{math} library.
\begin{lstlisting}[style=python]
from math import sqrt
def quadrForm(a,b,c):
    descr = sqrt(b**2-4*a*c)
    x1 = (-b+descr)/2.0*a
    x2 = (-b-descr)/2.0*a
    return (x1, x2)
\end{lstlisting}

This function takes as inputs the coefficients of a quadratic function and returns the output of the quadratic formula.
\begin{lstlisting}[style=python]
: functions.quadrForm(1,-2,1)
(1.0, 1.0)
\end{lstlisting}

We note that this function is susceptible to floating point error. For example, try to find the roots of the polynomial $x^2 - (10^7 + 10^-7)x + 1$ using our function:

\begin{lstlisting}[style=python]
>> [x1 x2] = quadrForm(1,-(1e7 + 1e-7),1)
(10000000.0, 9.96515154838562e-08)
\end{lstlisting}

Clearly the first root is correct, but the second root is clearly in error (admittedly the error is small, but in this case it is easily fixed). The second root shows the error because it is calculated by subtracting two numbers that are very close together.

We can solve this problem by using slightly different approach. We first calculate the root that is farther from zero using the formula.  Note that we will have to write our own \li{sign()} function in Python.  The \li{sign()} function should return $-1.0$ if the input is negative, $0.0$ if the input is zero, or $1.0$ if the input is positive.
\[
x_1 = \frac{-b - \text{sign}(b)*\text{descr}}{2a}
\]
We can then use a formula known as Viete's formula to calculate the other root (solving for x2).
\[
x_1 x_2 = c/a
\]

\begin{problem}
Write a function that implements this second approach to finding the roots. Call it \li{quadrForm2}.  Note the improved accuracy.
\end{problem}

\section*{Logic, Conditionals and Loops}
Three basic operators in logic are \li{and} (\&), \li{or} (\textbar), and \li{not} (!). We can use relational operators from the~\ref{tbl:relops} to build logical statements.

\begin{table}[h!]
\begin{center}
\begin{tabular}{|c|c|}
	\hline
	\li{==} & equal to\\
        \li{<} & less than\\
	\li{<=} & less than or equal to\\
	\li{\!=} & not equal to\\
	\li{>} & greater than\\
	\li{>=} & greater than or equal to\\
	\hline
\end{tabular}
\caption{Some relational operators}
\label{tbl:relops}
\end{center}
\end{table}

Using these building blocks we can build complicated statements. For example, consider a vector $x$ and the statement \li{(x>1) & (x<10)}.  The statement will evaluate \li{True} if $x_i$ is between one and ten, and \li{False} otherwise.  This technique is called \emph{masking} and is useful for operating only on parts of a vector or an array that meet certain conditions.  Applying the mask to our vector $x$ will yield a boolean vector (a vector with only \li{True} or \li{False} values).  We can assign an operation for when we encounter a \li{True} value.  For example, suppose we want to add $5.5$ to any number that is less than $0.6$ in our vector $x$.  We can do this simply with the following statement:
\begin{lstlisting}[style=python]
: a = sp.rand(10); a
array([ 0.90492936,  0.42251211,  0.80463372,  0.45087988,  0.94395439,
        0.11372628,  0.17177908,  0.42053736,  0.94759312,  0.59030686])
: b = (a<.6); b
array([False,  True, False,  True, False,  True,  True,  True, False,  True], dtype=bool)
: a[b]  #the values that correspond to the True values of our mask
array([ 0.42251211,  0.45087988,  0.11372628,  0.17177908,  0.42053736,
        0.59030686])
: a[b] += 5.5; a   # a += 1 is equivalent to a = a + 1. a[b] makes it so that the operation is only applied to values which correspond to a 'True' value in b.
array([ 0.90492936,  5.92251211,  0.80463372,  5.95087988,  0.94395439,
        5.61372628,  5.67177908,  5.92053736,  0.94759312,  6.09030686])
\end{lstlisting}

\begin{problem}
Create logical statements for the following:
\begin{itemize}
%\item True if x is prime and less than one-thousand, false otherwise (use the function \li{isprime}).
\item True if $x^2$ is greater than 10 or x is positive and smaller than 2.
\item True if the Bessel function of the second kind evaluated at x, with nu = 1, has magnitude greater than 1 (you will need to import \li{special.yn()}).
\end{itemize}
\end{problem}

\begin{problem}
Create a function that takes an input vector x and shuffles it like a deck of cards. You may assume that x has even length. The key is to create a mask using some random vector and then assign specific cards to even or odd slots of an output vector using that mask. Then see how many shuffles it takes to make the output $y$ ``random." (you can do this by looking at the offdiagonal entry of corrcoef(x,y), which should be close to zero if the output is random).
\end{problem}


% !TeX root = thoughts.tex
\section{Recap of probabilistic inversion and Hamiltonian Mechanics}

\paragraph{Probabilistic inversion and classical sampling}
In probabilistic inversion one does not require regularization, in contrast to deterministic inversion. Instead, one explores the model space by quantifying uncertainties and calculating probabilities for any model. In a sense, this can be considered as the `complete' solution. It is defined using Bayes' Theorem as follows;

\begin{gather}
	p(\mathbf{q}|\mathbf{d}) = \frac{p(\mathbf{d}|\mathbf{q})\;p(\mathbf{q})}{ p(\mathbf{d})}
\end{gather}
Generally, we ignore the data evidence ($p(\mathbf{d})$) as it only provides scaling of the posterior.

If the prior is normally distributed in model space, we can define it by

\begin{gather}
	p(\mathbf{q}) = k \cdot \exp{\left( -\frac{1}{2} \left[ \mathbf{q} -\mathbf{q_0}\right]^T \MatrixVariable{C}_M ^{-1}\left[ \mathbf{q} -  \mathbf{q_0}\right]\right)}
\end{gather}
where $k$ is a scaling constant (i.e. for normalization), $\mathbf{q_0}$ the mean of the prior distribution and $\MatrixVariable{C}_M ^{-1}$ the inverse parameter covariance matrix. This inverse parameter covariance matrix is given by
\begin{gather}
	\left[ \MatrixVariable{C}_M^{-1} \right]_{ij} = r_{ij} \sigma_i \sigma_j
\end{gather}
with $r_{ij}$ the correlation between parameters $q_i$ and $q_j$ and $\sigma_i$ the standard deviation of parameter $q_i$.

The prior in data space, $p(\mathbf{d}|\mathbf{q}) $, can be seen as the likelihood to see data based on a selected model. This generally incorporates a forward model, measurement uncertainties and forward modeling uncertainties. Because the quantification of forward modeling uncertainties is generally very hard to estimate usually the actual implementation of this is ignored. With only the forward model and measurement uncertainties (normally distributed) the prior in data space is as follows:
\begin{gather}
	p(\mathbf{d}|\mathbf{q}) =  k \cdot \exp{\left( -\frac{1}{2} \left[ G\left(\mathbf{q}\right) -\mathbf{d_0}\right]^T \MatrixVariable{C}_D ^{-1}\left[G\left(\mathbf{q}\right) -  \mathbf{d_0}\right]\right)}
\end{gather}
with $k$ again a scaling constant, $\MatrixVariable{C}_D$ the data covariance matrix, $d_0$ the observed data and $G\left(\mathbf{q}\right)$ the forward modeled data based on parameters $q$ and forward model $G$. This forward model $G$ can be any non-linear model, but \gls{HMC} inversion algorithms are greatly simplified if they are actually linear systems, as will be illustrated later.

When these prior data functions are combined one obtains the (improperly scaled) posterior. The negative exponent of the posterior is generally called the misfit and has a special role in many inversions. In deterministic inversion, the aim is usually to find the global minimum of this function. In probabilistic inversion,one tries to map the entire misfit by (pseudo) random sampling. The misfit is given by

\begin{gather}
	\chi(\mathbf{q}) = \frac{1}{2} \left[ \mathbf{q} -\mathbf{q_0}\right]^T \MatrixVariable{C}_M ^{-1}\left[ \mathbf{q} -  \mathbf{q_0}\right] + \frac{1}{2} \left[ G\left(\mathbf{q}\right) -\mathbf{d_0}\right]^T \MatrixVariable{C}_D ^{-1}\left[G\left(\mathbf{q}\right) -  \mathbf{d_0}\right].
\end{gather}

Exploring this misfit is typically done based on prior information. The Metropolis-Hastings algorithm draws new models entirely from the prior in model space, and accepts either by having a lower misfit or randomly based on the exponential increase in misfit if the proposed model has a larger misfit.

If the prior model is far off or has large uncertainties, classical samplers might have a hard time finding proper acceptable models. An example which will be expanded on in the first section is a linear model where prior data is off by double the forward modeled value, which on 10,000 samples lead to less than 100 accepted models. These acceptance rates are very wasteful of computing power. On the other side, drawing new models is computationally cheap relative to the more intensive \gls{HMC} sampling.

A rather helpful way to write the misfit for a linear forward model with Gaussian uncertainties allows for easy computation of derivatives, as well as keeping notations clean. I start by rewriting the misfit functional as;

\begin{align}
	\chi(\mathbf{q}) =& \frac{1}{2} \left[ \mathbf{q} -\mathbf{q_0}\right]^T \MatrixVariable{C}_M ^{-1}\left[ \mathbf{q} -  \mathbf{q_0}\right]
	+ \frac{1}{2} \left[ \MatrixVariable{G}\mathbf{q} -\mathbf{d_0}\right]^T \MatrixVariable{C}_D ^{-1}\left[\MatrixVariable{G}\mathbf{q} -  \mathbf{d_0}\right]\nonumber\\
	=&\frac{1}{2} \mathbf{q}^T \MatrixVariable{C}_M ^{-1}\mathbf{q}
	- \frac{1}{2} \mathbf{q_0}^T \MatrixVariable{C}_M ^{-1}\mathbf{q}
	- \frac{1}{2} \mathbf{q}^T \MatrixVariable{C}_M ^{-1}\mathbf{q_0}
	+ \frac{1}{2} \mathbf{q_0}^T \MatrixVariable{C}_M ^{-1}\mathbf{q_0}\nonumber\\
	&+ \frac{1}{2}\left( \MatrixVariable{G} \mathbf{q} \right)^T \MatrixVariable{C}_D ^{-1}\left( \MatrixVariable{G} \mathbf{q} \right)
	-\frac{1}{2} \mathbf{d_0} ^T \MatrixVariable{C}_D ^{-1}\left( \MatrixVariable{G} \mathbf{q} \right)\nonumber\\
	&-\frac{1}{2}\left( \MatrixVariable{G} \mathbf{q} \right)^T \MatrixVariable{C}_D ^{-1}\mathbf{d_0}
	+\frac{1}{2}\mathbf{d_0}^T \MatrixVariable{C}_D ^{-1}\mathbf{d_0}\nonumber\\
	=&\frac{1}{2} \mathbf{q}^T \MatrixVariable{C}_M ^{-1}\mathbf{q}
	- \frac{1}{2} \mathbf{q_0}^T \MatrixVariable{C}_M ^{-1}\mathbf{q}
	- \frac{1}{2} \mathbf{q}^T \MatrixVariable{C}_M ^{-1}\mathbf{q_0}
	+ \frac{1}{2} \mathbf{q_0}^T \MatrixVariable{C}_M ^{-1}\mathbf{q_0\nonumber}\\
	&+ \frac{1}{2}\mathbf{q}^T \MatrixVariable{G}^T  \MatrixVariable{C}_D ^{-1} \MatrixVariable{G} \mathbf{q}
	-\frac{1}{2} \mathbf{d_0} ^T \MatrixVariable{C}_D ^{-1} \MatrixVariable{G} \mathbf{q} %\nonumber\\
	-\frac{1}{2} \mathbf{q}^T  \MatrixVariable{G}^T \MatrixVariable{C}_D ^{-1}\mathbf{d_0}
	+\frac{1}{2}\mathbf{d_0}^T \MatrixVariable{C}_D ^{-1}\mathbf{d_0}.
\end{align}
Now the trick is to realize that all the components can be individually transposed without altering the equation. The fact that $ \mathbf{a}^T \mathbf{b} = \mathbf{a} \cdot \mathbf{b} = \mathbf{b} \cdot \mathbf{a} = \mathbf{b}^T \mathbf{a}$ should be enough proof. Also realizing that  covariance matrices are always symmetric, we rearrange the equation in second, first and zeroth order terms of $\mathbf{q}$:
\begin{align}
	\chi(\mathbf{q}) =
	&\frac{1}{2} \mathbf{q}^T \left[\MatrixVariable{C}_M^{-1} +  \MatrixVariable{G}^T  \MatrixVariable{C}_D ^{-1} \MatrixVariable{G} \right]\mathbf{q} -\left(\mathbf{q_0}^T \MatrixVariable{C}_M ^{-1} + \mathbf{d_0} ^T \MatrixVariable{C}_D ^{-1} \MatrixVariable{G}\right) \mathbf{q}
	 \nonumber\\
	&
	+ \frac{1}{2} \mathbf{q_0}^T \MatrixVariable{C}_M ^{-1}\mathbf{q_0}
	+\frac{1}{2}\mathbf{d_0}^T \MatrixVariable{C}_D ^{-1}\mathbf{d_0}.
\end{align}
Or, substituting the different components;
\begin{align}
	\chi(\mathbf{q}) =
	&\frac{1}{2}\mathbf{q}^T \MatrixVariable{A} \; \mathbf{q} -\mathbf{b}  \mathbf{q} + c,
\end{align}
where
\begin{align}
	\MatrixVariable{A} &= \MatrixVariable{C}_M^{-1} +  \MatrixVariable{G}^T  \MatrixVariable{C}_D ^{-1} \MatrixVariable{G} \label{eq:linear_system.misfit_A}\\
	\mathbf{b} &= \mathbf{q_0}^T \MatrixVariable{C}_M ^{-1} + \mathbf{d_0} ^T \MatrixVariable{C}_D ^{-1} \MatrixVariable{G} \label{eq:linear_system.misfit_b}\\
	c &=  \frac{1}{2} \mathbf{q_0}^T \MatrixVariable{C}_M ^{-1}\mathbf{q_0}
	+\frac{1}{2}\mathbf{d_0}^T \MatrixVariable{C}_D ^{-1}\mathbf{d_0}.
\end{align}
Note that $\mathbf{b}$ is a row vector, so $\mathbf{b}\mathbf{q}$ is actually the dot product between $\mathbf{b}^T$ and $\mathbf{q}$.

Now using this notation, one can easily compute the gradient of the misfit as
\begin{gather}
	\frac{\partial \chi}{\partial q_i} = A_{ij} q_j - b_i.
\end{gather}
\index{Algorithm performance}Note that repeated indices are summed. Note that using these quantities (which are precomputed every inversion) sped up propagation by 10 times.

\paragraph{Hamiltonian Mechanics and sampling} One way to propose more acceptable models is to use \gls{HMC} sampling. In this algorithm, the model is considered as a particle in $n$-dimensional space, where n denotes the number of inversion parameters. By not assigning new parameters but momenta in each dimension and propagating the model for a certain amount of time one proposes new models.

The propagation is based on Hamilton's Equations. These two equations interrelate energy of a system to it's position and momentum.\index{Hamilton's equations} Hamilton's equations for a trajectory of a particle are given in this $n$-dimensional space as;

\begin{gather}
	\frac{d q_i}{dt} = \frac{\partial H}{\partial p_i},\label{eq:ham1}\\
	\frac{d p_i}{dt} = - \frac{\partial H}{\partial q_i}\label{eq:ham2}.
\end{gather}
In these equations, $q_i$ is the position in dimension $i$, $p_i$ is the momentum in dimension $i$ (given by $p_i = \mu_i \frac{d q_i}{dt}$). $H$ stands for the Hamiltonian, which in physical Hamiltonian Mechanics is the sum of potential en kinetic energy.

The trick to \gls{HMC} sampling is to consider the misfit functional $\chi$ as a gravitational potential. This allows us to propagate our model over model space as if it were a particle moving under the influence of gravity defined by the posterior distribution. The actual definitions for potential and kinetic energy then become:

\begin{gather}
	K(\mathbf{p}) = \frac{1}{2} \sum_{i=1}^{n} \frac{p_i^2}{\mu_i},\label{eq:kineticsimple}\\
	U(\mathbf{q}) = \chi(\mathbf{q}).
\end{gather}

There are already many interesting options, remarks and conclusions to draw from this framework, but as it is usually nicely illustrated using a few examples, we will come to that later. Noteworthy however that a simplification is done on the calculation of momenta. In a later analysis, I extend this definition. See also the note about equation \eqref{eq:hamsimple1}.

One thing which is useful to note now is that the derivatives on the right hand sides of equations \eqref{eq:ham1} and \eqref{eq:ham2} now simplify, for the Hamiltonian derivative with respect to momenta only depends on kinetic energy, while the derivative with respect to position only depends on potential energy. The simplified representation is:

\begin{gather}
\frac{d q_i}{dt} = \frac{p_i}{\mu_i},\label{eq:hamsimple1}\\
\frac{d p_i}{dt} = - \left[\nabla_{q} \chi \right]_i\label{eq:hamsimple2}.
\end{gather}
An extension on this with a \index{Mass matrix}non-diagonal mass matrix will allows us to `link' parameters together in the propagation. This will be analyzed later on. Typically, however, for simple cases one chooses a diagonal positive definite mass matrix, which leads to equation \eqref{eq:hamsimple1}.

\index{Model propagation}The propagation of models is done using a leapfrog scheme, in which one splits up each time step in three separate calculations. First the momentum is propagated half an original time step, then the model parameters are propagated a full step, after which the momentum catches up again. Since I will not alter much on this side of the algorithm, for specifics I refer to section~5.2.3.3 of \cite{neal2011mcmc}.

\index{Acceptance rate}Acceptance of a new proposition works almost equal with \gls{HMC} as it does with the Metropolis-Hastings algorithm. The difference is that one does not compare misfit magnitudes, but the actual Hamiltonian, or energy of the system. Mathematically, this can be expressed as
\begin{gather}
	\min\left( 1, \frac{\exp\left[-H(\mathbf{q_\tau},\mathbf{p_\tau}) \right]}{\exp\left[-H(\mathbf{q},\mathbf{p}) \right]} \right).
\end{gather}
I like this better in words. As we explore model space, we assign new momenta in each iteration. This will result in a different energy of the system. If the Hamiltonian (the system's energy) has decreased from the previous sample, than it is accepted unconditionally (because $\exp (H - H_\tau)>1$). If, however, energy has increased, it has a chance of $\exp \left(H - H_\tau\right)$ to be accepted. This can be expressed as exploration of energy levels.

\index{Measure of exploration}What is noteworthy is that through the conservation of energy the Hamiltonian will not change over the course of the trajectory. This allows two things; a quality control to ensure that propagation of the model is performed correctly. Moreover, the chance of accepting a model is completely defined as soon as the momentum is assigned. This is very important, as it now also follows that propagation time doesn't influence the acceptance rate \textit{at all}. As one will see, trajectory length is more a tuning parameter which determines the algorithm's measure of exploration.

\index{Mass matrix}Mass matrices can be chosen in two ways. A standard, non-optimized option would be to choose the unit matrix. A simple analysis given in Andreas Fichtner's reader also reveal that parameters with different forward model derivatives oscillate non-equally during a trajectory. A mitigation is to assign the mass matrix according to the forward model. The mass matrix that would result in equal oscillations would be
\begin{gather}\label{eq:massMatrixForward}
\MatrixVariable{M} = \MatrixVariable{G}^T \MatrixVariable{G}
\end{gather}
where just taking the trace of this matrix would `unlink' the parameters again and make oscillations roughly equal. We'll see later that this assumption is not fully correct.

\index{Momenta!Drawing} Momenta are drawn from the mass matrix diagonal (correlation is assumed to be non-existent). By using the square root of the mass as standard deviation and zero mean, $n$ momenta are drawn corresponding to $n$ parameters.


\chapter{Rewriting math equations into basic UREs}

Based on the work of Gusev and Evans~\cite{broadcastElim,broadcastElimQR},  we can translate math equations into UREs, following the simple steps below. Going through an example, we would find that this process is pretty much mechanical and intuitive, and only middle-school level of math knowledge is needed. So do not be scared by the math symbols. In fact, you want to work out the UREs through the simple math process, since the correctness can be proved and that would save you time in debugging.

We strongly recommend a beginner tries one example so as to quickly master the skills. These skills are generally applicable to real world problems.

Use auto-regressive filter~\cite{autoRegressiveModel} for example:
\begin{equation}
 X_t=c+\sum\limits^I_{i=1}\varphi_i X_{t-i}+\varepsilon_t
\end{equation}
where $c$ is a constant, $\varphi_1, ... \varphi_I$ are the parameters, $\varepsilon _t$ is white noise, and $X_t$ is the output at the current time $t$ and is dependent on the previous $I$ outputs $ X_{t-1}, ..., X_{t-I}$.

Table~\ref{tab:deriving-ures-for-auto-regressive-filter} shows a complete process how UREs are derived for the problem, and the steps are explained in more detail below:

\begin{table*}[!ht]
\begin{tabular}{|l|ll|ll|}
\hline
    \multicolumn{1}{|c|}{\textbf{Step}} & \multicolumn{2}{|c|}{\textbf{Equations}} & \multicolumn{2}{|c|}{\textbf{Initial values}} \\\hline\hline
     \multirow{1}{*}{1: Iterative form}
        & $X_t=c+\sum\limits^I_{i=1}\varphi_i X_{t-i}+\varepsilon_t$    & $t=1, ..., T$   & $X_{t-i}=0$ &if $t-i \le 0$          \\\hline
     \multirow{2}{*}{2: Recursive form}
        & $X_t=X_t+\varphi_i X_{t-i}$                   & $t=1,...,T$ & $X_t=c+\varepsilon_t$ &  \\
        &                                         &   $i=1,..,I$      &  $X_{t-i}=0$ &if $t-i \le 0$ \\\hline
     \multirow{2}{*}{3: DSA}
        & $X_t^i=X_t^{i-1}+\varphi_i X_{t-i}^I$         & $t=1,...,T$  & $X_t^{0}=c+\varepsilon_t$ & \\
        &                                         &  $i=1,..,I$& $X_{t-i}^I=0$ &if $t-i \le 0$\\\hline
     \multirow{2}{*}{4: Full index form}
         & $X_t^i=X_t^{i-1}+\varphi_i^0 X_{t-i}^I$      & $t=1,...,T$ & $X_t^{0}=c+\varepsilon_t^0$  & \\
         & &$i=1,..,I$ & $X_{t-i}^I=0$ &if $t-i \le 0$\\\hline
     \multirow{4}{*}{5: UREs}
         & \multicolumn{2}{|l|}{$\Phi_t^i=(t-1=0) ? \varphi_i^0 : \Phi_{t-1}^i $}             &    & \\
         & \multicolumn{2}{|l|}{$\chi_t^i=(t-1=0) ? 0 : (i-1=0 ? X_{t-1}^{i-(1-I)} : \chi_{t-1}^{i-1})$}&&\\
         & $X_t^i=(i-1=0 ? c+\varepsilon_t^0 :  X_t^{i-1})+\Phi_t^i\chi_t^i$ &$t=1,...,T$ & &\\
         & & $i=1,..,I$& & \\\hline
\end{tabular}
\caption{Deriving UREs for auto-regressive filter. Here $c? a : b$ is an expression returning $a$ when condition $c$ is true, and $b$ otherwise.}
\label{tab:deriving-ures-for-auto-regressive-filter}
\end{table*}

\section{Step 1: Iterative form}

First, write down the math equation(s) of the original problem. Usually, in an  equation, a domain  is iterated (like $i=1,...,I$ and $t=1,...,T$),  and some variable (like $X$) is computed. A variable might have some initial values (like $X_{t-i}=0$ for $t-i \le 0$).

\section{Step 2: Recursive form}

Translating the iterative form to a recursive form is straightforward: according to the iterative form, initialize a variable (e.g. $X_t=c+\varepsilon_t$), and update the variable every iteration with a new value based on its previous value (in the form of $X_t=X_t+...$).

\section{Step 3: DSA (Dynamic Single Assignment)}

When updating a variable every iteration, save it to a distinct memory location. Every reference to a variable thus exposes the iteration in which the variable is defined. For example, $X_t=X_t + ... X_{t-i}$ is changed into  $X_t^i=X_t^{i-1} + ... X_{t-i}^I$. After this renaming, it is clear that the 3 references to $X$ are referring to the $X$ values defined in the current iteration $\begin{psmallmatrix}i\\t\end{psmallmatrix}$ and previous iterations $\begin{psmallmatrix}i-1\\t\end{psmallmatrix}$ and $\begin{psmallmatrix}I\\t-i\end{psmallmatrix}$, respectively.
Consequently, dataflow/dependences between these iterations are made explicit.

For another example, the initialization $X_t=c+\varepsilon_t$ is changed into  $X_t^0=c+\varepsilon_t$ by adding one 0 for the missing index $i$. $X_t^0$ refers to the $X$ value defined in iteration $\begin{psmallmatrix}0\\t\end{psmallmatrix}$, which is outside the domain, because index $i$ starts from 1 with a step of 1. In general, if index $i$ starts from $s$ with a step $h$, the initial value should be $X_t^{s-h}$.

\section{Step 4: Full index form}

Now variables are referenced with full indices, but constants are not: $\varphi_i$ and $\varepsilon_t$ are inputs never modified when computing $X_t$ (The other constant $c$ is just a number and we do not care). Similar to the handling of initialization in step 3, we give these constants full indices by adding 0's for the missing index $i$:  change $\varphi_i$ and $\varepsilon_t$  into $\varphi_i^0$ and $\varepsilon_t^0$. They refer to the $\varphi$ and $\varepsilon$ values defined in iteration $\begin{psmallmatrix}0\\i\end{psmallmatrix}$ and $\begin{psmallmatrix}0\\t\end{psmallmatrix}$, respectively,  which are outside the domain. In general, if iteration index $i$ starts from $s$ with a step $h$, we should rename $\varphi_i$ and $\varepsilon_t$ into $\varphi_i^{s-h}$ and $\varepsilon_t^{s-h}$.

After being full indexed, variables and constants will be processed in the same way.

At this point, the equations we get are AREs (Affine Recurrence Equations), that is, the current iteration $\begin{psmallmatrix}i\\t\end{psmallmatrix}$ reads a value defined in a previous iteration with a distance $d$ that is in the form of  $d=A\begin{psmallmatrix}i\\t\end{psmallmatrix}+d_0$, where $A$ is a matrix and $d_0$ a constant vector. In other words, there is a read-after-write affine dependence between the two iterations. The dependence distance can be calculated as the current iteration  - the previous iteration  = the write's index - the read's index: Remember that every write/read is fully indexed with the iteration that defines its value; thus the write is indexed with the current iteration, the read is indexed with the previous iteration.  See Table~\ref{tab:deps-full-index-form-auto-regressive-filter} for all the dependences:

\begin{table*}[!ht]
\begin{tabular}{|c|c|c|l|}
\hline
    \multicolumn{1}{|c|}{\textbf{Dependence No.}} & \multicolumn{1}{|c|}{\textbf{Write}} & \multicolumn{1}{|c|}{\textbf{Read}} & \multicolumn{1}{|c|}{\textbf{Dependence distance}}\\\hline\hline
     \multirow{1}{*}{1} & $X_t^i$ & $X_t^{i-1}$ &  $\begin{pmatrix} i\\t\end{pmatrix} -  \begin{pmatrix} i-1 \\ t\end{pmatrix} =  \begin{pmatrix} 1 \\ 0\end{pmatrix}$ \\\hline
     \multirow{1}{*}{2} & $X_t^i$ & $\varphi_i^{0}$ &  $\begin{pmatrix} i\\t\end{pmatrix} -  \begin{pmatrix} 0 \\ i \end{pmatrix} =  \begin{pmatrix} i \\ t-i\end{pmatrix} =  \begin{pmatrix} 1 & 0 \\ -1 & 1\end{pmatrix}  \begin{pmatrix} i \\ t\end{pmatrix}$ \\\hline
     \multirow{1}{*}{3} & $X_t^i$ & $X_{t-i}^{I}$ &  $\begin{pmatrix} i\\t\end{pmatrix} -  \begin{pmatrix} I \\ t-i \end{pmatrix} =  \begin{pmatrix} i-I \\ i\end{pmatrix} =  \begin{pmatrix} 1 & 0 \\ 1 & 0\end{pmatrix}  \begin{pmatrix} i \\ t\end{pmatrix} + \begin{pmatrix} -I \\ 0 \end{pmatrix}$  \\\hline
\end{tabular}
\caption{Dependences of the full index form in  step 4 of  Table~\ref{tab:deriving-ures-for-auto-regressive-filter}}.
\label{tab:deps-full-index-form-auto-regressive-filter}
\end{table*}

\section{Step 5: UREs}

We translate AREs into UREs by converting a broadcast into a pipeline. After that, every dependence has a constant distance uniformly in the entire domain.

There are two ways to convert a broadcast into a pipeline, either graphically or mathematically.

\subsection{First way to translate AREs into UREs: drawing a dataflow graph}
\label{sec:are-to-ures-with-dfg}

We can draw the dataflow and intuitively figure out how to change a broadcast into a pipeline, as exemplified in Fig.~\ref{fig:broadcast-to-pipeline-arf}. According to the 2nd dependence in Table~\ref{tab:deps-full-index-form-auto-regressive-filter}, originally, a datum $\varphi_i^0$ is broadcast to iterations $X_t^i$ for all $t$, as shown in the left of the figure.

Equivalently, the same datum can be loaded in an iteration at a boundary of the domain, and from that iteration, propagated in a pipeline fashion to all the other iterations, as shown in the right of the figure. As we can see, $\varphi_i^0$ is  loaded at a bounary iteration $(i, 1)$, and then is propagated to iteration $(i, 2)$, and from there to iteration $(i, 3)$, etc.

\begin{figure}[!ht]
    \centering
    \includegraphics[width=\textwidth]{./img/broadcast-to-pipeline-arf.png}
    \caption{For the 2nd dependence of auto-regressive filter in Tabel~\ref{tab:deps-full-index-form-auto-regressive-filter}, the broadcasts due to this dependence are changed to pipelines. Here we assume $T=3$, and every point is an iteration annotated with its indices $\begin{psmallmatrix} i\\ t\end{psmallmatrix}$.}
    \label{fig:broadcast-to-pipeline-arf}
\end{figure}

\subsubsection{Expressing a pipeline}
\label{sec:express-pipeline}

Now we can modify the full index form

\begin{equation}
X_t^i=...\varphi_i^0...
\end{equation}

into

\begin{equation}
X_t^i=...\Phi_t^i...
\end{equation}

where $\Phi$ values are propagated along the $t$ dimension until out of the domain:

\begin{equation}
\Phi_t^i=(t-1=0) ? \varphi_i^0 : \Phi_{t-1}^i
\end{equation}

As you can see, the keys to change a broadcast into a pipeline are: (1) the direction of the pipeline, along which data would be propagated from one iteration to the next, and (2) the boundary conditions when the pipeline is fed with some initial values.

In the same way, we can convert a broadcast due to the third dependence into a pipeline (Could you do it?). After that, we get the UREs shown in step 5 of  Table~\ref{tab:deriving-ures-for-auto-regressive-filter}.

\subsection{Second way to translate AREs into UREs: Calculating a propagation direction}
\label{sec:are-to-ures-with-math}

We can generalize the example in Fig.~\ref{fig:broadcast-to-pipeline-arf}, and directly find out a propagation direction vector in simple math:

\begin{quotation}
\noindent If a read-after-write dependence is affine, i.e. the distance $d$ is in the form of $Az+d_0$, where $A$ is a matrix, $z$ is the current iteration, and $d_0$ is a constant vector, then the dependence incurs broadcasts of values, and such a broadcast can be changed into a pipeline by propagating a value along a direction $r$ that is a solution of $(E-A)r=\mathbf{0}$, where $E$ is the identity matrix.
\end{quotation}

\begin{figure}[!ht]
    \centering
    \includegraphics[width=\textwidth]{./img/broadcast-to-pipeline.png}
    \caption{Changing a broadcast into a pipeline}
    \label{fig:broadcast-to-pipeline}
\end{figure}

The left part of Fig.~\ref{fig:broadcast-to-pipeline} shows that due to an affine dependence, some data are broadcast to multiple consumers, including an iteration $b$ at the boundary of the domain, and two other iterations $c_1$ and $c_2$, etc. Remember that data are fully indexed, i.e. they can be considered having been defined in an iteration $y$, even though that iteration is actually out of the domain.

Let the dependence distance be $d=Az+d_0$.  Because both iteration $c1$ and $c_2$ get data from the same iteration $y$, using the  dependence distance, we have
\begin{equation}
y = c_2 - (Ac_2+d_0) = c_1 - (Ac_1+d_0)
\end{equation}

So
\begin{equation}
(E-A)c_2 = (E-A)c_1
\end{equation}

Let $r=c_2-c1$, we have
\begin{equation}
(E-A)r = 0.
\end{equation}

Therefore, we can imagine that a datum that is defined outside the domain is first sent to a boundary iteration $b$, and then following a propagation direction $r$ to the next iteration $b+r$, and so on, and eventually the data reach iteration $c_1$, and then $c_2$, etc. This constructs a pipeline as shown in the right part of Fig.~\ref{fig:broadcast-to-pipeline}.

For example, the second dependence for the auto-regressive filter is $\begin{psmallmatrix} 1 & 0 \\ -1 & 1\end{psmallmatrix}  \begin{psmallmatrix} i \\ t\end{psmallmatrix}$ (Table~\ref{tab:deps-full-index-form-auto-regressive-filter}), and thus $A=\begin{psmallmatrix} 1 & 0 \\ -1 & 1\end{psmallmatrix}$. Solve the equation $(E-A)r=\begin{psmallmatrix} 0 & 0 \\ 1 & 0\end{psmallmatrix}r=\mathbf{0}$. We get a solution $r=\begin{psmallmatrix} 0 &\\ *\end{psmallmatrix}$, where the second element (the $t$ dimension)  can be arbitrary. Take a non-zero solution $r=\begin{psmallmatrix} 0 &\\ 1\end{psmallmatrix}$, and we see exactly from the right part of Fig.~\ref{fig:broadcast-to-pipeline-arf} that a  pipeline can be constructed along the $t$ dimension for a datum.

With this propagation direction, we convert a broadcast due to the second dependence into a pipeline in the same way as shown in Section~\ref{sec:express-pipeline}. Following the same approach, for the third dependence, we can find a propagation direction $r=\begin{psmallmatrix} 1 &\\ 1\end{psmallmatrix}$, and convert the broadcasts due to this dependence into pipelines as well (Could you do it?). After that, we get the UREs shown in step 5 of  Table~\ref{tab:deriving-ures-for-auto-regressive-filter}.

\subsection{Testing correctness of UREs in standard C}
\label{sec:test-correctness-ures}

We can express the UREs in Table~\ref{tab:deriving-ures-for-auto-regressive-filter} in standard C, adding necessary helping code for testing, and see if the UREs can produce correct results. See Listing~\ref{lst:test-correctness-ures-auto-regressive-filter-in-c}.

\lstinputlisting[language=c, caption={Testing the correctness of the UREs in Table~\ref{tab:deriving-ures-for-auto-regressive-filter} in standard C code.}, label={lst:test-correctness-ures-auto-regressive-filter-in-c}]{code/auto-regressive-filter-testing-ures-in-c.c}

Now test it:
\begin{verbatim}
    $gcc auto-regressive-filter-testing-ures.c
    $./a.out
    Success!
\end{verbatim}

\subsection{Expressing the UREs in T2S}
\label{sec:express-ures-in-t2s}

It is straightforward to express the UREs in T2S. See Listing~\ref{lst:test-correctness-ures-auto-regressive-filter-in-t2s}. The code is similar to the C code before. The major difference is that the T2S code builds a symbolic dataflow graph where inputs are symbolic parameters and computation is symbolic expressions (Line 13-27), then instantiates the graph to compute with concrete inputs (Line 29-37), and after that compares with a golden baseline for correctness.

\lstinputlisting[language=c, caption={Testing the correctness of the UREs in Table~\ref{tab:deriving-ures-for-auto-regressive-filter} in T2S.}, label={lst:test-correctness-ures-auto-regressive-filter-in-t2s}]{code/auto-regressive-filter-testing-ures-in-t2s.cpp}

Now test it:
\begin{verbatim}
    $ export T2S_PATH=path_to_your_t2s_installation
    $ export PATH=$T2S_PATH/Halide/bin:$T2S_PATH/install/gcc-7.5.0/bin:$PATH
    $ export LD_LIBRARY_PATH=$T2S_PATH/Halide/bin:$LD_LIBRARY_PATH
    $ g++ -I$T2S_PATH/Halide/include -L$T2S_PATH/Halide/bin -lHalide  -std=c++11 \
          auto-regressive-filter-testing-ures-in-t2s.cpp
    $ ./a.out
    Success!
\end{verbatim}


\section{Bonus Lecture: Simple Groups of Order 60}
Consider $\operatorname{GL}_2(\mathbb Z_5)$.
We first want to find the size of this group.
It is easy to see that
$$\begin{pmatrix}
    a&b\\
    c&d
\end{pmatrix}\in\operatorname{GL}_2(\mathbb Z_5)$$
is invertible if annd only if $ad-bc$ has a multiplicative inverse, that is, is nonzero.
Take $U_5\subset \mathbb Z_5$ to be the set of elements in $\mathbb Z_5$ having multiplicative inverse, that is, $U_5=\mathbb Z_5^\times=\mathbb Z_5\setminus\{0\}$.
There are $5^4=625$ $2\times 2$ matrices in total.
Amongst them, the number of non-invertible ones satisfies $ad\equiv bc\pmod{5}$.\\
Case 1: $a=0$, then $bc=0$, so either $b=0$ (which gives $25$ choices) or $c=0$ (which gives, again, $25$ choices).
There are double-counted cases where $b=c=0$ and there are $5$ cases, so there is a total of $45$ choices.\\
Case 2: $a\neq 0$, then we can solve for $d$ given $b,c$.
Indeed, for any $b,c$, we can have an unique corresponding $d$, hence there are $4\times 5^2=100$ choices.\\
So there are a total of $145$ non-invertible matrices and thus $480$ invertible matrices.
There are still a lot of elements, so we want to think about $\operatorname{SL}_2(\mathbb Z_5)$.
But since $\det:\operatorname{GL}_2(\mathbb Z_5)\to\mathbb Z_5^\times$ is a surjective homomorphism with kernel $\operatorname{SL}_2(\mathbb Z_5)$, so $|\operatorname{SL}_2(\mathbb Z_5)|=120$.
Also note that $\operatorname{PSL}_2(\mathbb Z_5)\cong\operatorname{SL}_2(\mathbb Z_5)/\{\pm I\}$, then $|\operatorname{PSL}_2(\mathbb Z_5)|=60$.
We can analyze the conjugacy classes in $\operatorname{PSL}_2(\mathbb Z_5)$ to find out that they look exactly like that in $A_5$.
In fact they are isomorphic.
\begin{theorem}
    $\operatorname{PSL}_2(\mathbb Z_5)\cong A_5$.
\end{theorem}
\begin{proof}
    Let $G=\operatorname{PSL}_2(\mathbb Z_5)$.
    Take the subgroup
    $$H=\left\{ \begin{pmatrix}
        1&0\\
        0&1
    \end{pmatrix},
    \begin{pmatrix}
        2&0\\
        0&3
    \end{pmatrix},
    \begin{pmatrix}
        0&2\\
        2&0
    \end{pmatrix},
    \begin{pmatrix}
        0&1\\
        4&0
    \end{pmatrix}\right\}\le G$$
    so the three non-identity elements in $H$ has order $2$ and since there is only one conjugation class of element of order $2$, so the conjugates of $H$ contains all element of order $2$.
    Since there are $15$ elements of order $2$ and $H$ contains $3$ elements of order $2$, there must be at least $5$ conjugates of $H$.
    We want to show there are precisely $5$ of them.
    Consider the action $G$ on the set $X$ of the subgroups of $G$ by conjugation, then the stabiliser $G_H$ (the normalizer) satisfies $|G_H|\times |G\star H|=|G|=60$.
    Note that $H\le G_H$, hence $4||G_H|$, therefore $|G\star H||15$.
    If $|G\star H|=15$, we know that there are only $15$ elements of order $2$, some pair of conjugates have $3$ elements in common (counting identity), but then they must be the same, contradiction.
    So $|G\star H|=5$ as claimed.
    The action of $G$ on $G\star H$ then gives a homomorphism $\rho:G\to S_5$.\\
    Now $G$ is simple by the same argument we used to show the simplicity of $A_5$ since they have the same table of sizes of conjugacy classes.
    Hence $\rho$ is injective (since $\rho$ is obviously not constant), so $G\cong\operatorname{Im}\rho$ which has index $2$ in $S_5$, hence is normal in $S_5$.\\
    Suppose $\operatorname{Im}\rho\neq A_5$, then $\operatorname{Im}\rho\cap A_5\le A_5$ has index $2$, so $\{e\}\neq\operatorname{Im}\rho\cap A_5\lhd A_5$, contradiction.
    Therefore $G\cong\operatorname{Im}\rho\cong A_5$.
\end{proof}
In fact, there is only one simple group of order $60$ up to isomorphism.\\
Now we turn to the symmetry of platonic solids.
Note that dual solids have isomorphic symmetries.
Let $G$ be the group of all isometries of a platonic solid and and $SG$ the group of all rotational isometries of it.
It is fact that $\operatorname{O}(3)\cong \operatorname{SO}(3)\times C_2$.
Note $S_4\not\cong A_4\times C_2$, since $x\mapsto -x$ is no longer a symmetry of the tetrahedron.
Otherwise, we have $G\cong SG\times C_2$.\\
For cube, we have seen that $SG$ has $24$ elements.
In fact, $SG\cong S_4$ since it permutes the set of long diagonals (pairs of opposite vertices).\\
Now symmetries of a regular isocahedron.
Let $SG$ act (transitively) on its $12$ vertices.
The stabiliser of the vertex are the rotations though the axis through the vertex, so it is isomorphic to $C_5$.
So $|SG|=5\times 12=60$.
In fact, again we have $SG\cong A_5$.
\begin{proposition}
    $SG$ is simple.
\end{proposition}
\begin{proof}
    $SG$ contains rotations of order $5$ through a vertex and of order $3$ through the centre of a face or order $2$ through the centre of an edge.
    So if $H\unlhd SG$ contains a rotation of order $5$ around some vertex, then by conjugating we can contain all rotations of order $5$ around any vertex.
    But then $H$ acts transitively on the vertices, and on any pair of vertices, but then one must get all rotations, so $H=SG$.
    Similar arguments hold if $H$ contains an element of order $2$ or $3$, hence $SG$ is simple.
\end{proof}
In fact, there are five inscribed tetrahedra in an isocahedra that are permuted by the action of $SG$, so by the same argument used in showing $\operatorname{PSL}_2(\mathbb Z_5)\cong A_5$, we have $SG\cong A_5$.


\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{physics}
\usepackage{amsmath}
\usepackage{showlabels}
\usepackage{amssymb}
\title{Statistical Computing for Scientists and Engineers\\[1em] Homework 5}
\author{Jiale Shi}
\date{Nov/12/2018}

\usepackage{natbib}
\usepackage{graphicx}
\usepackage{array}
\begin{document}
\maketitle

\newpage

\section{Importance Sampling}
Use importance sampling to approximate expectation $\mathbb{E}_{p}[f(x)]$ where
\begin{equation}
    f(x) = 2\sin(\frac{\pi}{1.5}x) , x\geq 0
\end{equation}
and target distribution is defined as
\begin{equation}
    p(x) = x^{1.65-1}\exp(-x^2/2), x\geq 0
\end{equation}

write down your choice of sampling distribution. Plot sampling distribution and target distribution on a plot. Write down the algorithm of your implementation.

Solution:
The target distribution
\begin{equation}
    p(x) = x^{1.65-1}\exp(-x^2/2) =x^{0.65}\exp(-x^2/2) , x\geq 0
\end{equation}

The shape of the target distribution is similar to the Gamma distribution Gamma(3,2.5) to some extent. Therefore, we choose Gamma distribution Gamma(3,2.5) as our proposed distribution.

\begin{figure}[h!]
\centering
\includegraphics[scale=0.6]{HW5P1_1.png}
\caption{sampling distribution Gamma(3,2.5), target distribution}
%\label{fig:universe}
\end{figure}

We samples for $N = 10^6$ times, collect samples(x), calculate weight$W$.
\begin{equation}
    W(samples) = \frac{p(samples)}{Gamma(3,2.5)}
\end{equation}

\begin{equation}
    E(f(x)) \approx \frac{1}{N}\sum_{i=1}^{N}W(samples)f(samples) \approx 0.7758554717760647
\end{equation}

\newpage
\section{Sequential Monte Carlo Sampler}
Implement a sequential Monte Carlo sampler to draw samples from target distribution $\pi$, which is an equal mixture of 5 normal distributions in $\mathbb{R}^2$ with unit covariance and centers that are equally distributed on the circumference of a circle with diameter 40.

Initialize 3000 particles with points located at the center of the circle.

Using 100 bridging densities $\pi_{0}$,... ,$\pi_{100}$ where
\begin{equation}
    \pi_{k}(x) \propto \pi(x)^{\alpha_{k}}
\end{equation}
and $0 \leq \alpha_{1} \leq ... \leq \alpha_{n} = 1$.

Use a normal random walk proposal with step size $\sqrt{6}$ to evolve particles

Measure degeneracy using effective sample size with a threshold of $N/2$ and resample accordingly.

To show the evolution of the points, including distribution of points for $\pi_{0}$, $\pi_{50}$, $\pi_{100}$. Additionally, generate an animation showing the points evolving from $\pi_{0}$ to $\pi_{n}$. Write down algorithm of your implementation.

Solution:
Target distribution $\pi$, is an equal mixture of 5 normal distributions in $\mathbb{R}^2$ with unit covariance and centers that are equally distributed on the circumference of a circle with diameter 40.

The five center points should be

$(x_1,y_1) = (20\cos(0),20 \sin(0))$ ;

$(x_2,y_2) = (20\cos(\frac{2\pi}{5}),20 \sin(\frac{2\pi}{5}))$;

$(x_3,y_3) = (20\cos(\frac{4\pi}{5}),20 \sin(\frac{4\pi}{5}))$;

$(x_4,y_4) = (20\cos(\frac{6\pi}{5}),20 \sin(\frac{6\pi}{5}))$;

$(x_5,y_5) = (20\cos(\frac{8\pi}{5}),20 \sin(\frac{8\pi}{5}))$

The final target distribution $\pi(x,y)$ should  be
\begin{equation}
\begin{aligned}
    \pi(x,y) =   & \exp\{ -[(\frac{(x-x_1)^2}{2})+(\frac{(y-y_1)^2}{2})] \} \\
     + & \exp\{-[(\frac{(x-x_2)^2}{2})+(\frac{(y-y_2)^2}{2})] \} \\
     + & \exp\{-[(\frac{(x-x_3)^2}{2})+(\frac{(y-y_3)^2}{2})] \} \\
     + & \exp\{-[(\frac{(x-x_4)^2}{2})+(\frac{(y-y_4)^2}{2})] \}   \\
     + & \exp\{-[(\frac{(x-x_5)^2}{2})+(\frac{(y-y_5)^2}{2})] \}
\end{aligned}
\end{equation}

All the 3000 particles start from the original point $(0,0)$.
For $K = [1,2,3,...,100]$, $a_{K} = [0.01,0.02,0.03,..,1]$ the target distribution $\pi(x,y)_{K} = \pi(x,y)^{a_K}$. The proposal distribution is a 2D normal distribution with step size is $\sqrt{6}$. The update follows the MH algorithm. The detail is in the following figure.

\begin{figure}[h!]
\centering
\includegraphics[scale=0.46]{SMCalgorithm.png}
\caption{SMC Algorithm}
%\label{fig:universe}
\end{figure}

The distribution of points for $\pi_{0}$, $\pi_{50}$,$\pi_{100}$.

\begin{figure}[h!]
\centering
\includegraphics[scale=0.3]{f0.png}
\caption{$\pi_{0}$ start from the center of the circle}
%\label{fig:universe}
\end{figure}

\begin{figure}[h!]
\centering
\includegraphics[scale=0.3]{f50.png}
\caption{$\pi_{50}$}
%\label{fig:universe}
\end{figure}

\begin{figure}[h!]
\centering
\includegraphics[scale=0.3]{f100.png}
\caption{$\pi_{100}$}
%\label{fig:universe}
\end{figure}

\newpage
\section{Hamiltonian Monte Carlo (HMC)}
HMC introduces an momentum variable $q$, and uses Hamiltonian dynamics to generate samples.

The potential energy $U(x) = -\log p(x)$, the kinetic energy $K(q) = -\log p(q)$.
$p(x)$ is the target density and $p(q)$ is the proposal density for $q$.
The summation $H(x,q) = U(x) + K(q)$. If we can generate samples $\propto \exp(-H(x,q)) = p(x)p(q)$, the resulting x samples will be distributed according to the target one.

To generate new candidate samples based on the Hamilton's equation of motion:

\begin{equation}
\begin{aligned}
& \frac{\partial x_i}{\partial t} = & \frac{\partial H}{\partial q_i} = & \frac{\partial K}{\partial q_{i}} \\
& \frac{\partial q_i}{\partial t} = & -\frac{\partial H}{\partial x_i} = & -\frac{\partial U}{\partial x_{i}}
\end{aligned}
\end{equation}

For numerical implementation, Hamilton's equations must be approximated by non-continual time, using small step $\epsilon$.
It starts with half step update for the momentum variable, and then do a full step for $x$ using the update momentum, and then do the other half step for momentum.
\begin{equation}
\begin{aligned}
& q_{i}(t+\epsilon/2) = q_i(t) -(\epsilon/2)\frac{\partial U}{\partial x_{i}(t)} \\
& x_{i}(t+\epsilon) = x_{i}(t)+ \epsilon   \frac{\partial H}{\partial q_i}  & \mbox{where}  &  \frac{\partial H}{\partial q_i}=q_i(t+\epsilon/2) \\ %\frac{\partial K}{\partial q_{i}}
& q_{i}(t+\epsilon) = q_i(t+\epsilon/2) -(\epsilon/2)\frac{\partial U}{\partial x_{i}(t+\epsilon)}
\end{aligned}
\end{equation}

\begin{figure}[h!]
\centering
\includegraphics[scale=0.6]{HW5P3_1.png}
\caption{target distribution, sampled distribution}
%\label{fig:universe}
\end{figure}

\begin{figure}[h!]
\centering
\includegraphics[scale=0.50]{HW5P3_2.png}
\caption{$p(x_1 | x_2)$}
%\label{fig:universe}
\end{figure}

\begin{figure}[h!]
\centering
\includegraphics[scale=0.55]{HW5P3_3.png}
\caption{$p(x_2 | x_1)$}
%\label{fig:universe}
\end{figure}

\begin{figure}[h!]
\centering
\includegraphics[scale=0.55]{HW5P3_4.png}
\caption{The first 50 steps}
%\label{fig:universe}
\end{figure}

\begin{figure}[h!]
\centering
\includegraphics[scale=0.5]{HW5P3_5.png}
\caption{Convergence of $<x_{1}>, <x_{2}>$}
%\label{fig:universe}
\end{figure}

\newpage
\section{Sequential Importance Sampling for Solving Integral Equations}
Devise and implement a (sequential) importance sampling Monte Carlo scheme to solve the following integral equation for $f(x)$ (defined in [-1,1]) at $x=0$:

\begin{equation}
    f(x) = x+\frac{1}{2}\int^{1}_{-1}(t-x)f(t)dt
\end{equation}

and compare results with the actual solution:

\begin{equation}
    f(x) = \frac{3}{4}x+\frac{1}{4}
\end{equation}

Consider uniform distribution $U[-1-\alpha,1+\alpha]$ as the transition kernel and with the appropriate stopping rule (similarly to the example considered in class in solving $Ax=b$). Comment on the performance for $[\alpha=0.001,0.1,1,1.5,2]$

\textbf{Solution:}

According to the actual solution, $f(x) = \frac{3}{4}x+\frac{1}{4}$.
The true value $f(0) = 0.25$.

The integral equation can be transformed into solving
\begin{equation}
    f(x) = x + \sum_{n=1}^{\infty} \int_{-1}^{1} (\prod_{k=1}^{n}(t_{k}-t_{k-1}))t_k dt_{1:n}
\end{equation}
It involves an infinite sum of integrals of increasing dimension, which can be solved by SIS.
Consider uniform distribution $U[-1-\alpha,1+\alpha]$ as the transition kernel.

\begin{math}
\begin{aligned}
& P_{d} = \alpha \\
& M = \frac{1}{(2+2\cdot\alpha)} \\
\end{aligned}
\end{math}

simulate a path using Markov chain. Start from $x^{(i)}=0$, then generate sample $t_{1}^{(i)} \sim M(t_{1}^{(i)},t)$, $t_{2}^{(i)} \sim M(t_{2}^{(i)},t)$, until $t_{k+1}^{(i)}$ reaches $s$, the cemetery state.

Calculate the associated weight

\begin{math}
\begin{aligned}
W^{(i)} (x,t_{1},..,t_{k}) =     \left\{ \begin{array}{rcl}
     \frac{0.5((t_{1}-x))}{M} (\prod^{n}_{k=1} \frac{0.5(t_{k}-t_{k-1})}{M})\frac{t_{k}}{P_{d}} & \mbox{for}
   & k>0 \\
   \frac{0.5((t_{1}-x))}{M} \frac{t_{k}}{P_{d}} & \mbox{for} & k=0
 \end{array}\right.
\end{aligned}
\end{math}

\begin{math}
\begin{aligned}
f(x) = y^{(N)}  = \frac{y_{0} + \sum_{i}^{N}W^{(i)}}{N} & \mbox{where} & y_{0}=0
\end{aligned}
\end{math}

For $[\alpha=0.001,0.1,1,1.5,2]$, the performance is:
\begin{equation}
\begin{aligned}
& \alpha = 0.001, & f(0) = 0.2764 & \mbox{slow}\\
& \alpha = 0.01, & f(0) = 0.2527 \\
& \alpha = 0.015, & f(0) = 0.2501 \\
& \alpha = 0.02, & f(0) = 0.2410 \\
& \alpha = 0.04, & f(0) = 0.2368 \\
& \alpha = 0.1, & f(0) = 0.2305 \\
& \alpha = 1, & f(0) = 0.1205 \\
& \alpha = 1.5, & f(0) = 0.0982 \\
& \alpha = 2, & f(0) = 0.0820 & \mbox{fast}
\end{aligned}
\end{equation}

\begin{figure}[h!]
\centering
\includegraphics[scale=0.5]{HW5P4.png}
\caption{Convergence of $<x_{1}>, <x_{2}>$}
%\label{fig:universe}
\end{figure}

When $\alpha$ becomes larger and larger, the speed becomes faster and faster.
From $\alpha = [0.001,0.01,0.015,0.02, 0.04, 0.1,1,1.5,2]$, when $\alpha$ becomes larger and larger, $f(0)$ initially a little bigger than 0.25, and then $f(0)$ decreases. At $\alpha = 0.015$, $f(0)$ is closest to the true value 0.25.

%\bibliographystyle{plain}
%\bibliography{references}
\end{document}


\subsection{Evaluating clusterings}
\subsubsection{Davies-Bouldin index}

The Davies-Bouldin index is a method for evaluating  clustering algorithms, such as k-means.

It examines the distance between centroids, and the tightness of centroids.


\clearpage
\subsection{Comparison between 1D and 2D finite element discretization}
\begin{table}[]
\centerline{\bf 1D and 2D Comparison for Finite Element and Multigrid }\medskip
\centering
\begin{tabular}{ p{7cm}<{\centering}|p{7cm}<{\centering}}
\hline
%\smallskip
   $1D: \Omega=(0, 1)$      \smallskip &    $2D: \Omega=(0, 1)\times (0, 1)$
%\smallskip
  \\\hline
 $\begin{cases}
 -u''=f&x\in \Omega\\
 u(0)=u(1)=0&
 \end{cases}$        &
 $\begin{cases}
 -\triangle u=f&x\in \Omega\\
 u=0&x\in \partial \Omega
 \end{cases}$
 \\\hline
  \multicolumn{2}{c}{$\displaystyle u=\argmin_{v\in V}   J(v)$}
 \\\hline
  $\displaystyle J(v)={1\over 2}\int_0^1  |v'|^2dx - \int_0^1  fv dx $
  &     $\displaystyle J(v)={1\over 2}\int_\Omega  |\nabla v|^2dx - \int_\Omega fv
  dx $
 \\\hline
 \multicolumn{2}{c}{$V=\{v: \Omega\rightarrow \mathbb{R} \mbox{ is continuous and piecewise smooth, } v|_{\partial \Omega}=0\}$}
 \\\hline
  \multicolumn{2}{c}{FE space: $V_h=\{v_h\in V, v_h \mbox{ is piecewise linear w.r.t. } \mathcal{T}_h\}$}
\\\hline
\begin{minipage}{0.5\textwidth}\centering
      	\includegraphics[width=4cm]{figures/grid1d.png}
   	 \end{minipage}
    	&
	\begin{minipage}{0.2\textwidth}\centering
      	\includegraphics[width=2cm]{figures/grid1.png}
   	 \end{minipage}
\\\hline
 $\phi_i(x)$   & $\phi_{ij}(x, y)$
 \\\hline
 \begin{minipage}{0.4\textwidth}\centering
      	\includegraphics[height=2cm]{figures/basisfunction.pdf}
   	 \end{minipage}
    & \begin{minipage}{0.5\textwidth}\centering
      	\includegraphics[height=2cm]{figures/nodalbasis.pdf}
   	 \end{minipage}
\\\hline
 \multicolumn{2}{c}{Find $u_h\in V_h$ s.t. $\displaystyle J(u_h)=\min_{v_h\in V_h} J(v_h)$}
 \\\hline
 $\displaystyle u_h=\sum_{i=1}^n \mu_i\phi_i(x)$         &    $\displaystyle u_h=\sum_{i, j=1}^n \mu_{ij}\phi_{ij}(x, y)$
  \\\hline
 Find $\mu\in \mathbb{R}^n$ s.t. $\displaystyle  I(\mu)=\min_{\nu\in \mathbb{R}^n} I(\nu)$
 &   Find $\mu\in \mathbb{R}^{n\times n}$ s.t. $\displaystyle  I(\mu)=\min_{\nu\in \mathbb{R}^{n\times n}} I(\nu)$
 \\\hline
 \multicolumn{2}{c}{$I(\nu)={1\over 2}(A\ast \nu, \nu)_{l^2} - (b, v)_{l^2} $}
 \\\hline
 $A={1\over h}(-1, 2, -1)$        &
$\scriptsize
 A=\begin{pmatrix}
 0&-1&0\\
 -1&4&-1\\
 0&-1&0
 \end{pmatrix}
$
 \\\hline
  \multicolumn{2}{c}{$\mu=\argmin I(\nu) \Longleftrightarrow \nabla J(\mu)=A\ast \mu -b=0$}    \\\hline
\multicolumn{2}{c}{ GD Method:   $ \mu^{(m+1)}=\mu^{(m)} -\eta(A\ast \mu^{(m)}-b)$}
 \\\hline
$\eta={h\over 4}$       &    $\eta={1\over 8}$      \\\hline
\end{tabular}
\end{table}

\renewcommand\arraystretch{2}
\begin{table}[]
\centerline{\bf Basic multigrid components}
\medskip
\centering
\begin{tabular}{ p{7cm}<{\centering}|p{7cm}<{\centering}}
\hline
 \begin{minipage}{0.5\textwidth}
      	\includegraphics[width=7cm,height=3.5cm]{figures/two-grids1.png}
   	 \end{minipage}
    & \begin{minipage}{0.5\textwidth}
      	\includegraphics[width=7cm,height=7cm]{figures/two-grids2.png}
   	 \end{minipage}
\\\hline
  $\phi_{i}^{2h}={1\over 2} \phi_{2i-1}^{h} + \phi_{2i}^{h} + {1\over 2} \phi_{2i+1}^{h}$        &     { \small$ \phi_{i,j}^{2h} =\phi_{2i,2j}^{h}
+\frac{1}{2}\left(\phi_{2i-1,2j-1}^{h} +\phi_{2i+1,2j+1}^{h} \right)
+\frac{1}{2}\left(\phi_{2i-1,2j}^{h} +\phi_{2i,2j-1}^{h}
+\phi_{2i+1,2j}^{h} +\phi_{2i,2j+1}^{2h} \right)$}
      \\\hline
  \multicolumn{2}{c}{ $\Phi^{2h}=R\ast_2 \Phi^h$  }    \\\hline
 $R=({1\over 2}, 1, {1\over 2})$      &   $R=\begin{pmatrix}
0&{1\over 2}&{1\over 2}\\
{1\over 2}&1&{1\over 2}\\
{1\over 2}&{1\over 2}&0
\end{pmatrix}$       \\\hline
\end{tabular}
\end{table}

\clearpage

497-6DL/8 Convolutional Multigrid Method/8.5-1D2DFE_MgNet.tex


\subsection{Introduction}

The best linear predictor is the one which minimises:

\(E[Y-X\theta ]\)

Under what circumstances is this the same as OLS? When n>>p. When n is not, then other linear estimators (like LASSO) can be better.


\subsection{Predictions}

Our data \((\mathbf y, \mathbf X)\) is divided into \((y_i, \mathbf x_i)\).

We create a function \(\hat y_i = f(\mathbf x_i)\).

The best predictor of \(y\) given \(x\) is:

\(g(X)=E[Y|X]\)

The goal of regression is to find an approximation of this function.

\subsection{Residuals}

\(\epsilon_i = y_i- \hat y_i\)

\subsection{Residual sum of squares (RSS)}

\(RSS=\sum_i \epsilon_i^2\)

\(RSS=\sum_i (y_i-\hat y_i)^2\)

\subsection{Explained sum of squares (ESS)}

\(ESS=\sum_i (\bar y-\hat y_i)^2\)

\subsection{Total sum of squares (TSS)}

\(TSS=\sum_i (y_i-\bar y)^2\)

\subsection{Relationship between prediction and probability distribution}

\(P(y|X, \theta )\)

\(\hat y =f(\mathbf x)\)

Through integration?

\(E[y] = \int P(y|X, \theta ) dy\)