\newpage
\section{Deep Boltzmann Machines \cite{salakhutdinov2013learning, salakhutdinov2009deep, goodfellow2016deep}}
\subsection{More efficient learning algorithm for general binary-binary BMs}
\subsubsection{PCD-k}
To recap:
\bg
\frac{1}{N}\sum_{n=1}^N\nabla_{\mb{W}}\log p(\mb{x}_n;\bs{\psi}) = \E_{\mb{v},\mb{h}\sim P_{\text{data}}(\mb{v},\mb{h};\bs{\psi})}\l[\mb{v}\mb{h}^T\r]-\E_{\mb{v},\mb{h}\sim P_{\text{model}}(\mb{v},\mb{h};\bs{\psi})}\l[\mb{v}\mb{h}^T\r]
\eg
and similar formulae for log-likelihood gradients w.r.t. $\mb{L},\mb{J},\mb{b},\mb{c}$, see (17).
\\[0.5em]
Now, instead of CD-k we use PCD-k (i.e. we keep one Markov chain without restarting its state between the updates), which falls into class of so-called \emph{Stochastic approximation procedures} (SAP) to estimate \tb{model's} expectations.
\\[1em]
Let $\bs{\psi}^t$ and $\mb{x}^t$ be the current model parameters and the state. Then $\mb{x}^t$ and $\bs{\psi}^t$ are updated sequentially as follows:
\begin{itemize}
	\item given $\mb{x}^t$, a new state $\mb{x}^{t+1}$ is sampled from a transition operator $T_{\bs{\psi}^t}(\mb{x}^{t+1}\leftarrow \mb{x}^t)$ that leaves $p(\cdot,\cdot;\bs{\psi}^t)$ invariant (which in our case is performing Gibbs sampling using equations (11) and (12) for $k$ steps);
	\item a new parameter $\bs{\psi}^{t+1}$ is then obtained by replacing the intractable model's expectation by the point estimate $\mb{x}^t$ (see also subsubsection 2.2.2 for whether to sample or use probabilities/means instead of sampling, and for which type of states)
\end{itemize}
In practice, we typically maintain a set of $P$ "persistent" sample "particles" $X^t=\{\mb{x}_1^t\ldots \mb{x}_{P}^t\}$ and use average over those particles.
\\[1em]
The intuition behind why this procedure works is the following: as the learning rate becomes sufficiently small compared with the mixing rate of the Markov chain, this "persistent" chain will always stay very close to the stationary distribution even if it is only run for a few MCMC updates per parameter update.
\\[1em]
Provided $\|\bs{\psi}^t\|$ is bounded and Markov chain, governed by a transition kernel $T_{\bs{\psi}^t}$ is ergodic (which is typically true in practice), and sequence of learning rates $\alpha_t$ satisfies $\sum_{t}\alpha_t=\infty$, $\sum_{t}\alpha_t^2<\infty$, this stochastic approximation procedure procedure is almost surely convergent to an asymptotically stable point.
\\
Note that in practice, $\alpha_t$ is not approached to zero, but rather to some small but positive constant $\varepsilon$ (e.g. $10^{-6}, 10^{-5}$).
\subsubsection{Variational learning}
Another approach is used to approximate \tb{data}-dependent expectations. We approximate true posterior over latent variables $p(\mb{h}|\mb{v};\bs{\psi})$ (which is intractable in general BM, tractable in RBM, but will be again intractable in DBM) by approximate posterior $q(\mb{h};\bs{\mu})$ and the variational parameters $\bs{\mu}$ are updated to follow the gradient of a \emph{lower bound on the log-likelihood}:
\\[1em]
In general:
\begin{empheq}[box={\mybox[1em][1em]}]{gather*}
\log p(\mb{v};\bs{\psi})=\int q(\mb{h};\bs{\mu})\log p(\mb{v};\bs{\psi})\mathrm{d}\mb{h}
=\int q(\mb{h};\bs{\mu})\log \frac{p(\mb{v},\mb{h};\bs{\psi})}{p(\mb{h}|\mb{v};\bs{\psi})}\mathrm{d}\mb{h}=\\
=\int q(\mb{h};\bs{\mu})\log \l(\frac{p(\mb{v},\mb{h};\bs{\psi})}{q(\mb{h};\bs{\mu})}\cdot \frac{q(\mb{h};\bs{\mu})}{p(\mb{h}|\mb{v};\bs{\psi})}\r)\mathrm{d}\mb{h}=\\
=\int q(\mb{h};\bs{\mu})\log p(\mb{v},\mb{h};\bs{\psi})\mathrm{d}\mb{h}
\underbrace{-\int q(\mb{h};\bs{\mu})\log q(\mb{h};\bs{\mu})\mathrm{d}\mb{h}}_{\mc{H}(q)}
+\underbrace{\int q(\mb{h};\bs{\mu})\log \frac{q(\mb{h};\bs{\mu})}{p(\mb{h}|\mb{v};\bs{\psi})}\mathrm{d}\mb{h}}_{D_{\text{KL}}(q(\mb{h};\bs{\mu}) \;\|\; p(\mb{h}|\mb{v};\bs{\psi}))\geq 0}\geq\\
\geq \int q(\mb{h};\bs{\mu})\log p(\mb{v},\mb{h};\bs{\psi})\mathrm{d}\mb{h} + \mc{H}(q) =: \mc{L}_{\text{ELBO}}(\bs{\mu}; \bs{\psi})
\end{empheq}
For Boltzmann Machine:
\begin{empheq}[box={\mybox[1em][1em]}]{gather*}
\text{\textbullet{} }[1]=\sum_{\mb{h}} q(\mb{h};\bs{\mu})\l[-E(\mb{v},\mb{h};\bs{\psi})=-\log Z(\bs{\psi}) \r]-\underbrace{\log Z(\bs{\psi})}_{=\text{ const w.r.t.}\bs{\mu}}\cdot\underbrace{\sum_{\mb{h}} q(\mb{h};\bs{\mu})}_{=1}+\\
+\sum_{\mb{h}} q(\mb{h};\bs{\mu})\l[\sum_{j<k}L_{jk}v_jv_k +\sum_{l<m}J_{lm}h_lh_m +\sum_{j,l}W_{jl}v_jh_l+\sum_jb_jv_j+\sum_lc_lh_l  \r]=\\
=\sum_{l<m}J_{lm}\E_{\mb{h}\sim q(\mb{h};\bs{\mu})}[h_lh_m]+\sum_{j,l}W_{jl}v_j\E_{\mb{h}\sim q(\mb{h};\bs{\mu})}[h_l]+\sum_lc_l\E_{\mb{h}\sim q(\mb{h};\bs{\mu})}[h_l]+\text{const}
\end{empheq}
For Boltzmann Machine and fully-factorizable $q(\mb{h};\bs{\mu})=\prod_l q(h_l;\mu_l), q(h_l=1;\mu_l)=\mu_l$ (\emph{mean-field} approach):
\begin{empheq}[box={\mybox[1em][1em]}]{gather*}
\text{\textbullet{} }[1]=\sum_{l<m}J_{lm}\mu_l\mu_m+\sum_{j,l}W_{jl}v_j\mu_l+\sum_lc_l\mu_l+\text{const}
\\
\text{\textbullet{} }[2]=\mc{H}(q)=-\sum_{\mb{h}}q(\mb{h};\bs{\mu})\log q(\mb{h};\bs{\mu})=-\sum_{\mb{h}}q(\mb{h};\bs{\mu})\sum_{j}\log q(h_j;\mu_j)=\\
=-\sum_j \sum_{h_j\in\{0,1\}}q(h_j;\mu_j)\log q(h_j;\mu_j) \underbrace{\sum_{\mb{h}_{-j}}q(\mb{h}_{-j};\bs{\mu}_{-j})}_{=1}=-\sum_j \mu_j\log \mu_j+(1-\mu_j)\log(1-\mu_j)
\end{empheq}
\bg
\boxed{\mc{L}_{\text{ELBO}}(\bs{\mu}; \bs{\psi})=\sum_{l<m}J_{lm}\mu_l\mu_m+\sum_{j,l}W_{jl}v_j\mu_l+\sum_lc_l\mu_l-\sum_j \mu_j\log \mu_j+(1-\mu_j)\log(1-\mu_j)+\text{C}}
\eg
Let us maximize (55) for $\bs{\mu}$ for fixed $\bs{\psi}$:
\begin{empheq}[box={\mybox[1em][1em]}]{gather*}
0 \doteq \frac{\partial}{\partial\mu_{i}}\mc{L}=\underbrace{\comment{$l=i$}\sum_{m>i}J_{im}\mu_m+\comment{$m=i$}\sum_{l<i}J_{li}\mu_l}_{=\comment{$J_{ij}=J_{ji},J_{ii}=0$}\sum_l J_{il}\mu_l}+\sum_{j}W_{ji}v_j+c_i
-\log \mu_i-1+\log(1-\mu_i)+1
\\
\Leftrightarrow
\\
\text{sigm}^{-1}(\mu_i)=\log\frac{\mu_i}{1-\mu_i}=\sum_{l<i}J_{li}\mu_l+\sum_{j}W_{ji}v_j+c_i
\end{empheq}
\bg
\boxed{\mu_i\leftarrow \text{sigm}\l(\sum_{l<i}J_{li}\mu_l+\sum_jW_{ji}v_j+c_i\r)}
\eg
Note that this is exactly the formula for (12) for computing $p(h_j=1|\mb{v},\mb{h}_{-i})$ in BM! So, updates of variational parameters can be computed using Gibbs sampler. This is not a coincidence, but the same holds if replace types of units, use RBM or even DBM (see below)!
\\[1em]
Note, that this variational approach cannot be used to approximate model-expectations because of minus sign in formulae (54),(17). This would cause variational learning to change the parameters so as to \emph{maximize} $D_{\text{KL}}(q(\mb{h};\bs{\mu}) \;\|\; p(\mb{h}|\mb{v};\bs{\psi}))$.
\\[0.5em]
The naive mean-field approach was chosen because:
\begin{itemize}
	\gooditem its convergence is usually fast;
	\gooditem it is unimodal.
\end{itemize}
Note that in general, we don't have to provide a parametric form of the approximating distribution beyond enforcing the independence assumptions. The variational approximation procedure is generally able to recover the functional form of the approximate distribution \cite{goodfellow2016deep}.

\subsection{Deep Boltzmann Machine}
Again assume unless specifically mentioned that DBM contains all binary units.
\subsubsection{High-level overview}
\textbullet{} DBM is a deep generative model, that consists of a layer of visible units and a series of layers of hidden units.
\begin{figure}[h]
\begin{mdframed}
\includegraphics[scale=0.4]{img/dbn_dbm.png}
\centering
\caption{A three-layer Deep Belief Network and a three-layer Deep Boltzmann Machine.}
\label{fig:dbn_dbm}
\end{mdframed}
\end{figure}
In comparison to another deep generative model, DBN (which is hybrid, and has directed layers and one undirected), DBM is entirely undirected model, see Fig. \ref{fig:dbn_dbm}. DBN is trained using greedy, layer by layer training of corresponding RBMs (one bottom-top pass). At the same time, All parameters in DBM are learned \tb{jointly}, which greatly facilitates learning better generative models. Even though both models have a potential of learning series of internal representations that become increasingly complex, DBM's approximate bottom-top and top-bottom inference better propagate uncertainty $\Rightarrow$ deal more robustly with ambiguous inputs, than DBN.
\\
\textbullet{} Formally, suppose number of (hidden) layers $L=3$.
$$
\mb{v}\in\R^D, \mb{h}=\{\mb{h^{(1)}}, \mb{h^{(2)}}, \mb{h^{(3)}}\}, \mb{h^{(s)}}\in\R^{H_s}, s\in\{1,2,3\};
$$
Energy function:
\bg
E(\mb{v},\mb{h};\bs{\psi})=-\mb{v}^T\mb{W^{(1)}}\mb{h^{(1)}}-\mb{h^{(1)}}^T\mb{W^{(2)}}\mb{h^{(2)}}-\mb{h^{(2)}}^T\mb{W^{(3)}}\mb{h^{(3)}}-\mb{b}\cdot\mb{v}-\mb{c^{(1)}}\cdot\mb{h^{(1)}}-\mb{c^{(2)}}\cdot\mb{h^{(2)}}-\mb{c^{(3)}}\cdot\mb{h^{(3)}},
\eg
where $\bs{\psi}=\{\mb{W^{(1)}}, \mb{W^{(2)}}, \mb{W^{(3)}}, \mb{b}, \mb{c^{(1)}}, \mb{c^{(2)}}, \mb{c^{(3)}}\}$.
Probability that the model assign to a configuration $(\mb{v},\mb{h})$:
\bg
p(\mb{v},\mb{h};\bs{\psi})\;\propto\;\exp(-E(\mb{v},\mb{h};\bs{\psi}))
\eg
\\
\textbullet{} Now observe that conncections between units in the DBM are restricted in such a way, that unit from a layer depends only on the units from the \emph{neighboring} layers, and does not depend from other units in the same layer or in the layers beyond. This is a multi-layer generalization of RBM, and allows to compute probabilities of units on given the others efficiently. For instance,
\bg
p(h^{(1)}_j=1|\mb{v},\mb{h^{(2)}})=\text{sigm}\l(\sum_iW^{(1)}_{ij}v_i+\sum_iW^{(2)}_{jl}h_l^{(2)}+c_j^{(1)}\r)
\eg
Observe how this formula resembles formulae (21),(22). This is also easily generalizes to other layers and other types of layers:
\\[1em]
\noindent\fbox{%
	\parbox{\textwidth}{%
	To compute probability of unit being on given all the others, \tb{add} linear combinations of states of units from neighboring layers + bias and apply \tb{activation function of a unit} (e.g. sigmoid for binary, softmax for softmax/multinomial, affine for gaussian etc.).
	}%
}
\\[1em]
Note, however, that the distribution over \tb{all} hidden layers generally does not factorize because of interactions between layers. For instance, for $L=2$, $p(\mb{h}^{(1)},mb{h}^{(2)}|\mb{v};\bs{\psi})$ does not factorize due to interaction weights $\mb{W}^{(2)}$ between $\mb{h}^{(1)}$ and $\mb{h}^{(2)}$ which  render those variables mutually dependent.
\\
\textbullet{} Formulae for log-likelihood gradients are derived the same way as for RBM and have similar form. For instance:
\bg
\frac{1}{N}\sum_{n=1}^N\nabla_{\mb{W^{(2)}}}\log p(\mb{x}_n;\bs{\psi}) = \E_{\mb{h^{(1)}},\mb{h^{(2)}}\sim P_{\text{data}}(\mb{h^{(1)}},\mb{h^{(2)}};\bs{\psi})}\l[\mb{h^{(1)}}\mb{h^{(2)}}^T\r]-\E_{\mb{h^{(1)}},\mb{h^{(2)}}\sim P_{\text{model}}(\mb{h^{(1)}},\mb{h^{(2)}};\bs{\psi})}\l[\mb{h^{(1)}}\mb{h^{(2)}}^T\r]
\eg
\textbullet{} Finally, we apply new learning algortihms for BMs described in the previous subsection with fully-factorizable mean-field approach:
\bg
q(\mb{h};\bs{\mu})=\prod_j \prod_l \prod_m q(h_j^{(1)};\mu_j^{(1)}) \cdot q(h_l^{(2)};\mu_l^{(2)}) \cdot q(h_m^{(3)};\mu_m^{(3)})
\eg
Thanks to lack of intra-layer interaction makes it
possible to use fixed point equations (just like for general BM algorithm) to actually optimize the variational lower bound and find the true optimal mean field expectations.
\\
\textbullet{} Further we will use DBM with Gaussian visible units, Multinomial top-most layer hidden unist, and Bernoulli hidden units for intermediate layers. In this setting, again the learning algortihms remains the same, the difference is only in the way probabilities are computed, and samples are made.
\\
\textbullet{} One unfortunate property of DBMs is that sampling from them is relatively
difficult. DBNs only need to use MCMC sampling in their top pair of layers. The
other layers are used only at the end of the sampling process, in one efficient
ancestral sampling pass. To generate a sample from a DBM, it is necessary to
use MCMC across all layers, with every layer of the model participating in every
Markov chain transition.

\subsubsection{Gibbs sampling in DBMs}
\textbullet{} Similar to RBM, Gibbs sampling using equations (59) can be made in parallel thus allowing to perform block Gibbs sampling for each layer of units. In addition to that, as illustrated in Fig. \ref{fig:dbm_gibbs}, the DBM layers
can be organized into a bipartite graph, with odd layers on one side and even layers
on the other. This immediately implies that when we condition on the variables in
the even layer, the variables in the odd layers become conditionally independent. In conjuction with block Gibbs sampling for each layer, this allow to perform a Gibbs sampling in the whole DBM in \tb{only 2 iterations}, instead of $L + 1$, as one might naively think at first.
\\
\textbullet{} Good news that in TF no additional work need to be done beyond implementing block Gibbs sampling for each layer. Each independent branch in the computational graph should be executed in parallel.
\begin{figure}[h]
\begin{mdframed}
\includegraphics[scale=0.4]{img/dbm_gibbs.png}
\centering
\caption{A deep Boltzmann machine, re-arranged to reveal its bipartite graph structure.}
\label{fig:dbm_gibbs}
\end{mdframed}
\end{figure}
\textbullet{} Note that Contrastive Divergence algorithm is slow for DBMs because they do not allow efficient sampling of the hidden states given the visible units -- instead, CD would require burning in a Markov chain every time a new negative phase sample is needed.
\subsubsection{Greedy layerwise pretraining of DBMs}
DBM can be trained using the aforementioned learning algorithm from random initialization (typically the results are quite bad even on MNIST, see \cite{goodfellow2012joint, goodfellow2016deep}), but it works much better if weights are initialized sensibly. Greedy layerwise pretraining = learning procedure that consists of learning a stack of RBM's one layer at a time. After the stack is learned, the whole stack can be viewed as a single probabilistic model, called Deep Belief Net. Thus, pre-training for DBN is straightforward. In case of DBM though, a layer in the middle of the stack of RBMs is trained with only bottom-up input, but after the stack is combined to form DBM, the layer will have both bottom-up and top-down input. To account for this so-called \emph{evidence double counting problem} \cite{salakhutdinov2009deep, goodfellow2016deep}, Fig. \ref{fig:dbm_pretraining}, two modifications are required:
\begin{figure}[h]
\begin{mdframed}
\includegraphics[width=1.5in]{img/dbm_pretraining2.png}
\quad
\quad
\includegraphics[width=3.5in]{img/dbm_pretraining.png}
\centering
\caption{Pre-training consists of learning a stack of modified RBMs, that are then composed to create a DBM.}
\label{fig:dbm_pretraining}
\end{mdframed}
\end{figure}
\begin{itemize}
	\item bottom RBM should be trained using two "copies" of each visible unit and the weights tied to be equal between these two copies ($\cong$ simply double the total input to hidden layer during upward pass); similarly, top RBM should be trained with two copies of topmost layer. Training of all intermediate RBMs if there are any, should not be modified.
	\item the weights of all intermediate RBMs though, should be divided by 2 before inserting into DBM
\end{itemize}
\begin{figure}[h]
\begin{mdframed}
\includegraphics[scale=0.08]{dbm/dbm_init.jpg}
\centering
\caption{A more detailed scheme how to initialize 3-layer DBM from learned stack or RBMs, including biases. Black circles -- visible units, blue -- hidden units, red circel -- visible bias, black square -- hidden bias. Biases can be summed or averaged.}
\label{fig:dbm_init}
\end{mdframed}
\end{figure}

\subsubsection{Joint training of DBMs}
Classic DBMs require greedy unsupervised pretraining, and to perform classification
well, require a separate MLP-based classifier on top of the hidden features they
extract. It is hard to track performance
during training because we cannot evaluate properties of the full DBM while
training the first RBM. Software implementations
of DBMs need to have many different components for CD training of individual
RBMs, PCD training of the full DBM, and training based on back-propagation
through the MLP. Finally, the MLP on top of the Boltzmann machine loses many
of the advantages of the Boltzmann machine probabilistic model, such as being
able to perform inference when some input values are missing.
\\
There are two main ways to resolve the joint training problem of the deep
Boltzmann machine: \tb{multi-prediction DBMs}\cite{goodfellow2013multi}, which is currently beyond the scope of this project, and the \tb{centering trick} \cite{montavon2012deep}, which reparametrizes the model in order to make the Hessian of the cost function better-conditioned at the beginning of the learning process. More specifically, if we consider energy function of generalized Boltzmann Machine (BM, RBM, DBM can all be represented by appropriate choice of $\mb{x}$ -- states, $\mb{U}$ -- weights, $\mb{a}$ -- biases):
\bg
E(\mb{x};\bs{\psi})=-\mb{x}^T \mb{U}\mb{x}-\mb{a}\cdot\mb{x}
\eg
Then the idea of centering trick is simply to reparameterize this energy function as
\bg
E(\mb{x};\bs{\psi})=-(\mb{x}-\bs{\beta})^T \mb{U}(\mb{x}-\bs{\beta})-\mb{a}\cdot(\mb{x}-\bs{\beta})
\eg
Where new hyperparameter vector $\bs{\beta}$ is chosen to be $\mb{x}-\bs{\beta}\approx \mb{0}$ at the beginning of training. This does not change the set of probability distributions that the model can represent, but it does change the learning dynamics so much, that it is actually possible to train DBM from random initialization w/o pre-training and achieve sensible results. However, in \cite{goodfellow2013multi} they say when DBM is trained using centering trick, they have never shown to have good classification performance, if this was the primary goal.

\subsubsection{Annealed importance sampling \cite{salakhutdinov2008, salakhutdinov2009deep, hinton2012better, upadhya2015empirical}}
Let $p_A(\mb{x})=\frac{p^*_A(\mb{x})}{\mc{Z}_A}$ be simple proposal distribution from which we can sample easily, and $p_{B}(\mb{x})=\frac{p^*_B(\mb{x})}{\mc{Z}_B}$ our complex target distribution. We also have to make sure $p_B \ll p_A$, which is easy in our case, since we can always choose $p_A$ to be uniform pmf, which dominates every other probability mass function on discrete units (of finite cardinality).
\\[0.5em]
\u{(Classical) Importance Sampling}
\\
The ratio of partition functions can be estimated as follows
\bg
\frac{\mc{Z}_B}{\mc{Z}_A}=\frac{p^*_B(\mb{x})}{p^*_A(\mb{x})}=\sum_{\mb{x}}\frac{p^*_B(\mb{x})}{p^*_A(\mb{x})} p_A(\mb{x})=\E_{\mb{x}\sim p_A}\l[\frac{p^*_B(\mb{x})}{p^*_A(\mb{x})} \r] \approx \frac{1}{N}\sum_{i=1}^N \frac{p^*_B(\mb{x}_i)}{p^*_A(\mb{x}_i)}
\eg
The problem is when $p_A$ and $p_B$ are very different, as in our case, this estimator is very poor: its variance is very large, possibly infinite.
\\[0.5em]
\u{Annealed Importance Sampling}
\\
To handle this issue, we define sequence of probability mass functions $\l(p_m\r)_{m=0:M}$ such that $p_0 = p_A$ and $p_M = p_B$, and for which we know unnormalized probabilities $p_m^*$, which typically are mixtures of target and proposal:
\bg
p_m(\mb{x})=p_B(\mb{x})^{\beta_m}\cdot p_A(\mb{x})^{1-\beta_m}, \beta_m=\frac{m}{M}
\eg
Also, in order not to sample from $p_s$ we also need sequence of transition operators $\l(T_i(\mb{x}_{i + 1} \leftarrow \mb{x}_i)\r)_{i=1:M-1}$ each that leaves the corresponding $p_i$ invariant. The importance weight can then be computed as
\bg
\omega_{\text{AIS}} \leftarrow \prod_{m=1}^M \frac{p_m^*(\mb{x}_m)}{p_{m-1}^*(\mb{x}_m)}
\eg
where $\mb{x}_1 \sim p_0=p_A;\; \mb{x}_2 \sim T_1(\mb{x}_2 \leftarrow \mb{x}_1);\; \ldots \; \mb{x}_M \sim T_{M-1}(\mb{x}_M \leftarrow \mb{x}_{M-1})$.
The ratio of partition functions can then be estimated as average over many AIS runs:
\bg
\frac{\mc{Z}_B}{\mc{Z}_A}=\frac{\mc{Z}_M}{\mc{Z}_0}\approx \frac{1}{L}\sum_{l=1}^L \omega_{\text{AIS}}^{(l)}
\eg
Notice also that we don't need to compute partition functions of any of the intermediate distributions.
\\
\tb{Note}: to avoid numerical problems and overflow errors (partition functions are very large numbers even for very moderate sized BM), all computations are performed in $\log$-domain, as usual.
\\[0.5em]
\u{Annealed Importance Sampling for 2-layer Bernoulli BM}
\\
It turns out that we can reduce state space of AIS to only hidden units in first layer $\mb{x}=\{\mb{h}^{(1)}\}$ by explicitly summing out visible and top-most layer hidden units:
\begin{empheq}[box={\mybox[1em][1em]}]{align*}
\log p^*\l(\mb{h}^{(1)}\r) &= \log \sum_{\mb{v},\mb{\mb{h}^{(2)}}} p^*\l(\mb{v}, \mb{h}^{(1)}, \mb{h}^{(2)}\r)=
\\
&= \log \sum_{\mb{v},\mb{\mb{h}^{(2)}}} \exp \l( \mb{v}^T\mb{W}^{(1)}\mb{h}^{(1)} +
\mb{h}^{(1)^T}\mb{W}^{(2)}\mb{h}^{(2)} + \mb{b}\cdot\mb{v} + \mb{c}^{(1)}\cdot\mb{h}^{(1)} + \mb{c}^{(2)}\cdot\mb{h}^{(2)} \r)
\\
&= \mb{c}^{(1)}\cdot\mb{h}^{(1)} + \log\l[ \sum_{\mb{v}}\exp\l( \mb{v}^T\mb{W}^{(1)}\mb{h}^{(1)} + \mb{b}\cdot\mb{v} \r)\sum_{\mb{h}^{(2)}} \exp\l( \mb{h}^{(1)^T}\mb{W}^{(2)}\mb{h}^{(2)} + \mb{c}^{(2)}\cdot\mb{h}^{(2)} \r) \r]
\\
&= \mb{c}^{(1)}\cdot\mb{h}^{(1)} + \sum_i^V \text{softplus}\l(\sum_j^{H_1} W_{ij}^{(1)}h_j^{(1)}+b_i \r) + \sum_k^{H_2} \text{softplus}\l(\sum_j^{H_1} W_{jk}^{(2)}h_k^{(2)}+c_k^{(2)} \r)
\end{empheq}
\bg
\boxed{\log p^*\l(\mb{h}^{(1)}\r)=\mb{c}^{(1)}\cdot\mb{h}^{(1)} + \sum_i^V \text{softplus}\l(\sum_j^{H_1} W_{ij}^{(1)}h_j^{(1)}+b_i \r) + \sum_k^{H_2} \text{softplus}\l(\sum_j^{H_1} W_{jk}^{(2)}h_k^{(2)}+c_k^{(2)} \r)}
\eg
From this we can easily derive equation for $\log p_{\textcolor{red}{m}}^*$ by simply scaling all weights by $\beta_m$:
\begin{equation}
\begin{aligned}
\log p_{\textcolor{red}{m}}^*\l(\mb{h}^{(1)}\r) =
\textcolor{red}{\beta_m}\mb{c}^{(1)}\cdot\mb{h}^{(1)} + \sum_i^V \text{softplus}\l(\textcolor{red}{\beta_m}\cdot\l(\sum_j^{H_1} W_{ij}^{(1)}h_j^{(1)}+b_i \r)\r) +
\\
+ \sum_k^{H_2} \text{softplus}\l(\textcolor{red}{\beta_m}\cdot\l(\sum_j^{H_1} W_{jk}^{(2)}h_k^{(2)}+c_k^{(2)} \r)\r)
\end{aligned}
\end{equation}
When $\beta_m=1$ we obtain target distribution, when $\beta_m=0$ we obtain uniform distribution:
\bg
\log p_0^*\l(\mb{h}^{(1)}\r) \equiv 0+\sum_i^V \text{softplus}(0) + \sum_k^{H_2} \text{softplus}(0)=(V+H_2)\log 2
\eg
thus $ \log \mc{Z}_0=(V+H_1+H_2)\log 2$.
\\[1em]
Thus we gradually increase "inverse temperature" $\beta$ from 0 to 1 and can estimate partition function using procedure described above. Staring from randomly initialized $\mb{h}^{(1)}$, we apply sequence of transition operators $T_i$ which are simply alternating Gibbs sampler with weights scaled by $\beta_i$.
\\[1em]
We can do the same for different types of units and larger number of layers. In the latter case we can again analytically sum out visible and top-most hidden units.
\\[0.5em]
\u{Variational lower-bound}
\\
Having estimate of partition function $\t{\mc{Z}}$, we can estimate variational lower-bound on test vector $\mb{v}^*$ as follows
\begin{equation}
\begin{aligned}
\log p(\mathbf{v}^{*};\boldsymbol{\psi})\geq\;&-\sum_{\mathbf{h}} q(\mathbf{h};\boldsymbol{\mu})E(\mathbf{v}^{*}, \mathbf{h};\boldsymbol{\psi})+\mathcal{H}(\boldsymbol{\mu})-\log\mathcal{Z}(\boldsymbol{\psi})
\\
=\;& \mathbf{v}^{*^{T}}\mathbf{W}^{(1)}\boldsymbol{\mu}_{\mathbf{v}^*}^{(1)}+\boldsymbol{\mu}_{\mathbf{v}^*}^{(1)^{T}}\mathbf{W}^{(2)}\boldsymbol{\mu}_{\mathbf{v}^*}^{(2)}+\mathbf{b}\cdot\mathbf{v}^{*}+\mathbf{c}^{(1)}\cdot\boldsymbol{\mu}_{\mathbf{v}^*}^{(1)}+\mathbf{c}^{(2)}\cdot\boldsymbol{\mu}_{\mathbf{v}^*}^{(2)}+\mathcal{H}(\boldsymbol{\mu}_{\mathbf{v}^*})-\log\mathcal{Z}(\boldsymbol{\psi})
\\
\approx\;& \mathbf{v}^{*^{T}}\mathbf{W}^{(1)}\boldsymbol{\mu}_{\mathbf{v}^*}^{(1)}+\boldsymbol{\mu}_{\mathbf{v}^*}^{(1)^{T}}\mathbf{W}^{(2)}\boldsymbol{\mu}_{\mathbf{v}^*}^{(2)}+\mathbf{b}\cdot\mathbf{v}^{*}+\mathbf{c}^{(1)}\cdot\boldsymbol{\mu}_{\mathbf{v}^*}^{(1)}+\mathbf{c}^{(2)}\cdot\boldsymbol{\mu}_{\mathbf{v}^*}^{(2)}+\mathcal{H}(\boldsymbol{\mu}_{\mathbf{v}^*})-\log\widehat{\mathcal{Z}}
\end{aligned}
\end{equation}
where $\bs{\mu}_{\mathbf{v}^*}$ are variational parameters obtained by running fixed-point equations using Gibbs sampler unitl convergence with visible units clamped to $\mathbf{v}^*$.
\\[1em]
One can also estimate true log-probability using AIS by clamping visible units to test example (estimating log-probability for one test example is computationally equivalent to estimating a partition function).

\subsubsection{Additional facts}
\textbullet{} In \cite{goodfellow2016deep} they say that obtaining sota results with DBM requires an additional partial mean field in negative phase, more details in \cite{goodfellow2013multi}.
\\
\textbullet{} The inference can further be accelerated using separate \emph{recognition model}, see \cite{salakhutdinov2010efficient} for details.
\\
\textbullet{} DBMs were developed after DBNs. Compared to DBNs, the posterior distribution $p(\mb{h}|\mb{v})$ is simpler for DBMs. Somewhat counterintuitively, the simplicity of this posterior distribution allows richer approximations of the posterior \cite{goodfellow2016deep}
\\
\textbullet{} The use of proper mean field allows the approximate inference procedure for DBMs to capture the influence of top-down feedback interactions. This makes
DBMs interesting from the point of view of neuroscience, because the human brain
is known to use many top-down feedback connections\cite{goodfellow2016deep}.
\\
\textbullet{} In \cite{goodfellow2013joint} they observe that energy function $E(\mb{v},\mb{h};\bs{\psi})$ inevitably induces some prior $p(\mb{h};\bs{\psi})$ that is not motivated by the structure of any kind of data. The role of deeper layers in DBM is simply to provide a better prior on the first layer hidden units.

\clearpage
\begin{figure}[h]
\begin{mdframed}
\centering
\includegraphics[width=6.4in]{dbm/tf_graph.png}
\caption{High-level computational graph for DBM model.}
\end{mdframed}
\end{figure}


\documentclass{article}
\usepackage{graphicx}
\usepackage{titletoc}
\usepackage{titlesec}
\usepackage{geometry}
\usepackage{fontspec, xunicode, xltxtra}
\usepackage{float}
\usepackage{cite}
\usepackage{amsmath}
\usepackage{listings}
\usepackage{titletoc}

\geometry{left=3cm,right=3cm,top=3cm,bottom=3cm}
\DeclareMathOperator*{\argmin}{argmin}
\DeclareMathOperator*{\argmax}{argmax}
\DeclareMathOperator*{\logit}{logit}
\DeclareMathOperator*{\var}{var}
\DeclareMathOperator*{\expec}{E}

\begin{document}
\title{\textsf{Homework 2 for Bayesian Data Analysis}}
\author{Fan JIN\quad (2015011506)}
\maketitle

\section*{Question 2.7a}
{
    $$p(\theta) \propto \theta^{-1} (1-\theta)^{-1}.$$

    Denote $\phi = \logit(\theta) = \log{(\theta / (1-\theta))}$, one has $\theta = 1/(1+\exp{(-\phi)})$, and therefore,
    $$p(\phi) \propto (\frac{1}{1 + \exp{(-\phi)}})^{-1} (1-\frac{1}{1 + \exp{(-\phi)}})^{-1} \cdot \left| \frac{\exp{(-\phi)}}{(1+\exp{(-\phi)})^2} \right| = 1,$$ which gives a uniform prior distribution for $\logit(\theta)$, or the natural parameter of the exponential family.
}

\section*{Question 2.7b}
{
    $$p(\theta|y) \propto \theta^{-1} (1-\theta)^{-1} \cdot \theta^{y} (1-\theta)^{n-y} = \theta^{y-1} (1-\theta)^{n-y-1}.$$

    If $y=0$, then we have $$p(\theta|y) \propto \theta^{-1} (1-\theta)^{n-1},$$ whose integral in interval $[0, 1]$ is
    $$\int_0^1 {\theta^{-1} (1-\theta)^{n-1} \mathrm{d}\theta} = +\infty,$$ since $p(\theta|y)$ is of the same order as $1/\theta$ when $\theta \rightarrow 0$.

    Likewise, if $y=n$, the integral is infinite when $\theta \rightarrow 1$. Therefore, the posterior distribution is improper if $y=0$ or $y=n$.
}

\section*{Question 2.12}
{
    $$\log{(p(y|\theta))} = \theta + y \log{(\theta)} - \log{(y!)},$$

    $$J(\theta) = -\expec{( \frac{\mathrm{d}^2 \log{(p(y|\theta))}}{\mathrm{d}\theta^2} | \theta)} = -\expec{( -\frac{y}{\theta^2} | \theta)} = \frac{\expec{(y | \theta)}}{\theta^2} = \frac{1}{\theta}.$$

    Hence, the Jeffery's prior density for $\theta$ is $$p(\theta) \propto [J(\theta)]^{1/2} = \theta^{-1/2}.$$

    Compared with a $\mathrm{Gamma}(\alpha, \beta)$ distribution with prior $p(\theta) \propto \theta^{\alpha-1} \exp{(-\beta \theta)}$, we have $\alpha=1/2$ and $\beta=0$.
}

\section*{Question 3.1a}
{
    $$p(\theta) \propto \prod_1^J {{\theta_j}^{\alpha_j-1}},$$
    $$p(y|\theta) \propto \prod_1^J {{\theta_j}^{y_j}},$$
    $$p(\theta|y) \propto \prod_1^J {{\theta_j}^{y_j+\alpha_j-1}}.$$

    Thus, by integrating $y_3, \cdots, y_J$, the joint posterior distribution of $\theta_1$ and $\theta_2$ is $$p(\theta_1, \theta_2 | y) \propto \theta_1^{y_1+\alpha_1-1} \theta_2^{y_2+\alpha_2-1}.$$

    Under the variable substitution $\alpha = \theta_1 / (\theta_1+\theta_2)$ and $\beta = \theta_1 + \theta_2$, we have $$\theta_1 = \alpha \beta, \quad \theta_2 = (1-\alpha) \beta,$$ and
    $$p(\alpha, \beta | y) = p(\theta_1, \theta_2) \cdot \left| \frac{\partial (\theta_1, \theta_2)}{\partial (\alpha, \beta)} \right| \propto (\alpha \beta)^{y_1+\alpha_1-1} \left((1-\alpha) \beta\right)^{y_2+\alpha_2-1} \cdot \left| \beta \right| $$
    $$= \alpha^{y_1+\alpha_1-1} (1-\alpha)^{y_2+\alpha_2-1} \beta^{y_1+y_2+\alpha_1+\alpha_2-1}.$$

    Therefore, the marginal posterior distribution of $\alpha$ is $$p(\alpha|y) = \int {p(\alpha, \beta | y) \mathrm{d}\beta} \propto \alpha^{y_1+\alpha_1-1} (1-\alpha)^{y_2+\alpha_2-1},$$ which is $\alpha|y \sim \mathrm{Gamma}(y_1+\alpha_1, y_2+\alpha_2).$
}

\section*{Question 3.1b}
{
    For a $\mathrm{Gamma}(\alpha_1, \alpha_2)$ prior distribution, and a $\mathrm{Binomial}$ sample with $y_1$ independent observations out of $y_1+y_2$ tests, we have the posterior distribution for the probability $\alpha$ is a $\mathrm{Gamma}(y_1+\alpha_1, y_2+\alpha_2)$. (See Homework 1) This posterior distribution is identical to the distribution obtained in (a).
}

\section*{Question 3.9}
{
    It is known that
    $$p(y | \mu, \sigma^2) \propto \sigma^{-n} \exp{\left(-\frac{1}{2\sigma^2} \sum_1^n {(y_i-\mu)^2}\right)}$$
    $$= \sigma^{-n} \exp{\left(-\frac{1}{2\sigma^2} \left[\sum_1^n {(y_i-\bar{y})^2} + n (\bar{y}-\mu)^2 \right]\right)}$$
    $$= \sigma^{-n} \exp{\left(-\frac{1}{2\sigma^2} \left[(n-1)s^2 + n (\bar{y}-\mu)^2 \right]\right)},$$
    and
    $$p(\mu, \sigma^2) = p(\sigma^2) p(\mu | \sigma^2) \propto \sigma^{-1} (\sigma^2)^{-(\nu_0/2+1)} \exp{\left( -\frac{1}{2\sigma^2} \left[ \nu_0\sigma_0^2 + \kappa_0(\mu_0-\mu)^2 \right] \right)}.$$

    Therefore, we have the joint posterior distribution
    $$p(\mu, \sigma^2 | y) \propto p(y | \mu, \sigma^2) p(\mu, \sigma^2)$$
    $$\propto \sigma^{-1} (\sigma^2)^{-((\nu_0+n)/2+1)} \exp{\left( -\frac{1}{2\sigma^2} \left[ (n-1)s^2 + n (\bar{y}-\mu)^2 + \nu_0\sigma_0^2 + \kappa_0(\mu_0-\mu)^2 \right] \right)},$$
    which is identical to a N-Inv-$\chi^2 (\mu_n, \sigma_n^2; \nu_n, \sigma_n^2)$ distribution, or
    $$\sigma^{-1} (\sigma^2)^{-(\nu_n/2+1)} \exp{\left( -\frac{1}{2\sigma^2} \left[ \nu_n\sigma_n^2 + \kappa_n(\mu_n-\mu)^2 \right] \right)}.$$

    Thus, by comparing the coefficients, we have $$\nu_n = \nu_0+n$$ and
    $$(n-1)s^2 + n (\bar{y}-\mu)^2 + \nu_0\sigma_0^2 + \kappa_0(\mu_0-\mu)^2 = \nu_n\sigma_n^2 + \kappa_n(\mu_n-\mu)^2,$$ or
    $$n+\kappa_0 = \kappa_n,$$
    $$-2n\bar{y} - 2\kappa_0 \mu_0 = -2\kappa_n \mu_n,$$
    $$(n-1)s^2 + n\bar{y}^2 + \nu_0\sigma_0^2 + \kappa_0 \mu_0^2 = \nu_n\sigma_n^2 + \kappa_n \mu_n^2,$$
    the solution to which is
    $$\nu_n = \nu_0+n,$$
    $$\kappa_n = \kappa_0+n,$$
    $$\mu_n = \frac{n}{n+\kappa_0} \bar{y} + \frac{\kappa_0}{n+\kappa_0} \mu_0,$$
    $$\nu_n\sigma_n^2 = (n-1)s^2 + \nu_0\sigma_0^2 + n\bar{y}^2 + \kappa_0 \mu_0^2 - \kappa_n \mu_n^2 = (n-1)s^2 + \nu_0\sigma_0^2 + \frac{\kappa_0 n}{\kappa_0 + n} (\bar{y}-\mu_0)^2.$$

}

\section*{Question 3.10}
{
    From the independency conditions, we have the joint posterior distribution
    $$p(\mu_1, \sigma_1^2, \mu_2, \sigma_2^2 \mid y) \propto \sigma_1^{-n_1-2} \sigma_2^{-n_2-2} \exp{\left( - \frac{1}{2\sigma_1^2} \sum_{j=1}^{n_1}{(y_{1j}-\mu_1)^2} - \frac{1}{2\sigma_2^2} \sum_{j=1}^{n_2}{(y_{2j}-\mu_2)^2} \right)}$$
    $$= \sigma_1^{-n_1-2} \sigma_2^{-n_2-2} \exp{\left( - \frac{1}{2\sigma_1^2} {\left[ (n_1-1)s_1^2 + n_1 (\bar{y_1} - \mu_1)^2 \right]} \right)} \exp{\left( - \frac{1}{2\sigma_2^2} {\left[ (n_2-1)s_2^2 + n_2 (\bar{y_2} - \mu_2)^2 \right]} \right)}.$$

    Recall that $$\int {\exp{\left(-\frac{n_1}{2\sigma_1^2} (\bar{y_1}^2 - \mu_1)^2 \right)} \mathrm{d}\mu_1} = (2\pi)^{1/2} \sigma_1 / \sqrt{n_1},$$ we have the integral
    $$p(\sigma_1^2, \sigma_2^2 \mid y) = \int {p(\mu_1, \sigma_1^2, \mu_2, \sigma_2^2 | y) \mathrm{d}\mu_1 \mathrm{d}\mu_2} $$
    $$\propto \sigma_1^{-n_1-2} \sigma_2^{-n_2-2} \exp{\left( - \frac{1}{2\sigma_1^2} {\left[ (n_1-1)s_1^2 \right]} \right)}  \exp{\left( - \frac{1}{2\sigma_2^2} {\left[ (n_2-1)s_2^2 \right]} \right)} \cdot \sigma_1 \sigma_2$$
    $$\propto \sigma_1^{-n_1-1} \sigma_2^{-n_2-1} \exp{\left( - \frac{(n_1-1)s_1^2}{2\sigma_1^2} \right)} \exp{\left( - \frac{(n_2-1)s_2^2}{2\sigma_2^2} \right)}$$
    $$\propto \left(\frac{s_1^2}{\sigma_1^2}\right)^{(n_1+1)/2} \left(\frac{s_2^2}{\sigma_2^2}\right)^{(n_2+1)/2} \exp{\left( - \frac{(n_1-1)s_1^2}{2\sigma_1^2} \right)} \exp{\left( - \frac{(n_2-1)s_2^2}{2\sigma_2^2} \right)}.$$

    Denote $u_1 = \frac{s_1^2}{\sigma_1^2}$, $u_2 = \frac{s_2^2}{\sigma_2^2}$, we have
    $$p(u_1, u_2 \mid y) \propto \left(u_1\right)^{(n_1+1)/2} \left(u_2\right)^{(n_2+1)/2} \exp{\left( - \frac{(n_1-1)}{2} u_1 \right)} \exp{\left( - \frac{(n_2-1)}{2} u_2 \right)} \cdot \left| \frac{1}{u_1^2} \frac{1}{u_1^2} \right|$$
    $$= \left(u_1\right)^{(n_1-3)/2} \left(u_2\right)^{(n_2-3)/2} \exp{\left( - \frac{(n_1-1)}{2} u_1 \right)} \exp{\left( - \frac{(n_2-1)}{2} u_2 \right)}$$
    and that $u_1$ is independent of $u_2$.

    By variable substitution $v_1 = u_1/u_2$ and $v_2 = u_2$, or $u_1 = v_1 v_2$ and $u_2 = v_2$, we have
    $$p(v_1, v_2 \mid y) \propto \left(v_1 v_2\right)^{(n_1-3)/2} \left(v_2\right)^{(n_2-3)/2} \exp{\left( - \frac{(n_1-1)}{2} v_1 v_2 \right)} \exp{\left( - \frac{(n_2-1)}{2} v_2 \right)} \cdot \left| v_2 \right|$$
    $$ = (v_1)^{(n_1-2)/2} \cdot (v_2)^{(n_1+n_2-2)/2} \cdot \exp{\left( -\frac{(n_1-1)v_1 + (n_2-1)}{2} v_2 \right)}.$$

    Note that the gamma distribution\footnote{https://en.wikipedia.org/wiki/Gamma\_distribution}, we have
    $$ p(v_1 \mid y) = \int_0^\infty {p(v_1, v_2 \mid y) \mathrm{d}v_2} $$
    $$= \int_0^\infty {(v_1)^{(n_1-3)/2} \cdot (v_2)^{(n_1+n_2-4)/2} \cdot \exp{\left( -\frac{(n_1-1)v_1 + (n_2-1)}{2} v_2 \right)} \mathrm{d}v_2}$$
    $$= (v_1)^{(n_1-3)/2} \cdot \frac{\Gamma(\alpha_0)}{\beta_0^{\alpha_0}}$$
    $$ \propto (v_1)^{(n_1-3)/2} \cdot \left(\frac{(n_1-1)v_1 + (n_2-1)}{2} \right)^{-(n_1+n_2-2)/2},$$
    where $\alpha_0 = (n_1+n_2-2)/2$ and $\beta_0 = (n_1-1)v_1 + (n_2-1))/2$.

    Compare the expression above with the pdf of $F$ distributions\footnote{https://en.wikipedia.org/wiki/F-distribution}, we have $$v_1 \mid y \sim F(n_1-1, n_2-1)$$.
}

\clearpage
\end{document}


%% SECTION HEADER /////////////////////////////////////////////////////////////////////////////////////
\section{3D Spectral Modelling}
\label{sec:3Dmodel}

%% SECTION CONTENT ////////////////////////////////////////////////////////////////////////////////////
The displacement vector of the \ac{3d} element is composed of three translational displacements defined as:
\begin{eqnarray}
	\left \{ \begin{array}{c}
		\textbf{u}^e(\xi,\eta,\zeta) \\
		\textbf{v}^e(\xi,\eta,\zeta) \\
		\textbf{w}^e(\xi,\eta,\zeta)
	\end{array} \right\}
	= \textbf{N}^e(\xi,\eta, \zeta)\widehat{\textbf{d}}^e
	= \sum_{l=1}^r\sum_{n=1}^q\sum_{m=1}^p\textbf{N}_m^e(\xi)\textbf{N}_n^e(\eta)\textbf{N}_l^e(\zeta)
	\left \{ \begin{array}{c}
		\widehat{\textbf{u}}^e(\xi_m,\eta_n,\zeta_l) \\
		\widehat{\textbf{v}}^e(\xi_m,\eta_n,\zeta_l) \\
		\widehat{\textbf{w}}^e(\xi_m,\eta_n,\zeta_l)
	\end{array} \right\},
	\label{eq:3D_displ}
\end{eqnarray}
where \(\widehat{\textbf{u}}^e\), \(\widehat{\textbf{v}}^e\) and
\(\widehat{\textbf{w}}^e\) are displacements of the element nodes in \(\xi,\eta\) and \(\zeta\) direction.

The nodal strain--displacement relations are given as \cite{kudela20093d}:
\begin{eqnarray}
	\boldsymbol{\epsilon}^e=\textbf{B}_{d}^e\widehat{\textbf{d}}^e=
	\left [
	\begin{array}{ccc}
		\frac{\partial N^e}{\partial x} & 0 & 0\\
		0 & \frac{\partial N^e}{\partial y} & 0\\
		0 & 0 & \frac{\partial N^e}{\partial z}\\
		0 & \frac{\partial N^e}{\partial z} & \frac{\partial N^e}{\partial y}\\
		\frac{\partial N^e}{\partial z} & 0 & \frac{\partial N^e}{\partial x}\\
		\frac{\partial N^e}{\partial y} & \frac{\partial N^e}{\partial x} & 0
	\end{array} \right]
	\left \{ \begin{array}{c}
		\widehat{\textbf{u}}^e \\
		\widehat{\textbf{v}}^e \\
		\widehat{\textbf{w}}^e
	\end{array} \right\}.
\end{eqnarray}
The formulae of the structural matrices for 3D elements are:
\begin{eqnarray}
	\textbf{M}_{dd}^e & = & \int_{V_e}\textbf{N}^T\rho \textbf{N} \diff V_e,\\
	\textbf{K}_{dd}^e & = & \int_{V_e}{\textbf{B}_d^e}^T\textbf{C}\textbf{B}_d^e \diff V_e,
\end{eqnarray}
where \textbf{C} is the stiffness tensor, \(\rho\) is mass density, and \(V_e\) is the element volume.


\documentclass[10pt, a4paper]{article}

\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{graphicx}
\usepackage{hyperref}
\usepackage{cite}
\usepackage{fullpage}

\graphicspath{ {./images/} }

\title{Computational Physics Assignment Answers}
\author{Tilman Roeder}
\date{\today}

\renewcommand\thesection{Question \arabic{section}}
\renewcommand\thesubsection{\thesection{} (\alph{subsection})}

\newcommand{\plot}[3]{\begin{figure}[ht]\centering\includegraphics[width=10cm]{#1}\caption{#2}\label{#3}\end{figure}}

\begin{document}
\maketitle

% Question 1
\section{}

  \subsection{}
  \label{sec:cgo}
  The program can be found in \texttt{/assignment/q-1}.

  \subsection{}
  The program written for \ref{sec:cgo} reports the smallest floating point number
  $a$ such that $1 + a > 1$ as\footnotemark{}:
  \begin{equation}
    a = 2^{-63}.
  \end{equation}

  Note that this is for the C numeric type \texttt{long double}, on a 2017 MacBook Pro using clang.
  This size is the one that would be expected for an 80-bit extended precision floating point (IEEE 754).

  Go supports 32 and 64 bit IEEE floating point numbers. For these types we find:
  \begin{itemize}
    \item \texttt{float32} $a \approx 1.192093 \times 10^{-7}$
    \item \texttt{float64} $a \approx 2.220446 \times 10^{-16}$
  \end{itemize}

  Note that Go does not support extended precision floating point values (although they can be used through
  CGo, as is done for \ref{sec:cgo}). These values match the expected values (where `expected' means `values
  they need to be to be IEEE compliant'). The theoretical values given our above definition are:
  \begin{itemize}
    \item \texttt{float32} $a = 2^{-23}$
    \item \texttt{float64} $a = 2^{-52}$
  \end{itemize}

  \footnotetext{Machine $\epsilon$ is also sometimes defined as $\frac{a}{2}$, with a defined as above.
  The values given for $a$ are valid for the definition given above.}

% Question 2
\section{}
  \subsection{}
  The program can be found in \texttt{/assignment/q-2}. The LU decomposition is implemented in the
  \texttt{/assignment/comply} package.

  \subsection{}
  Using the LU decomposition routine, one can find\footnotemark:
  \begin{equation}
    L = \begin{bmatrix}1 & 0 & 0 & 0 & 0 \\ 1 & 1 & 0 & 0 & 0 \\ 0 & 1.125 & 1 & 0 & 0 \\ 0 & 0 & -1.419\cdots & 1 & 0 \\ 0 & 0 & 0 & -1.216\cdots & 1\end{bmatrix}
  \end{equation}
  \begin{equation}
    U = \begin{bmatrix}3 & 1 & 0 & 0 & 0 \\ 0 & 8 & 4 & 0 & 0 \\ 0 & 0 & 15.5 & 10 & 0 \\ 0 & 0 & 0 & 45.193\cdots & -25 \\ 0 & 0 & 0 & 0 & 30.575\cdots\end{bmatrix},
  \end{equation}
  where $LU = A$, with $A$ being the matrix given in the assignment problem.

  \footnotetext{Some of the numeric answers for these questions have very long decimal representations
    when the full result is quoted to machine precision. These numbers have been truncated at three decimal
    figures and are reported as $0.123\cdots$. These merely indicate truncation and no rounding has taken place. To
    see the full results to machine precision, please run \texttt{make assignment}.}

  Further, it can be obtained:
  \begin{equation}
    \det(A) = \det(L) \times \det(U) = \det(U) = \prod_{i=1}^5 U_{ii} = 514032.
  \end{equation}

  \subsection{}
  The solver is implemented in \texttt{/assignment/comply}.

  \subsection{}
  Using the solver, $x$ is determined to be:
  \begin{equation}
    x \approx \begin{bmatrix}0.4565707971488156 \\ 0.6302876085535531 \\ -0.5105752171071062 \\ 0.05389158651601452 \\ 0.19613175833411145\end{bmatrix}.
  \end{equation}

  \subsection{}
  Using the solver, the matrix inverse is determined as:
  \begin{equation}
    A^{-1} \approx \begin{bmatrix}0.379\cdots & -0.046\cdots & 0.004\cdots & -0.004\cdots & -0.001\cdots \\ -0.138\cdots & 0.138\cdots & -0.012\cdots & 0.014\cdots & 0.005\cdots \\ 0.027\cdots & -0.027\cdots & 0.024\cdots & -0.028\cdots & -0.011\cdots \\ 0.070\cdots & -0.070\cdots & 0.062\cdots & 0.044\cdots & 0.018\cdots \\ 0.063\cdots & -0.063\cdots & 0.056\cdots & 0.039\cdots & 0.032\cdots\end{bmatrix}
  \end{equation}

% Question 3
\section{}
  \subsection{}
  Linear interpolation is implemented in \texttt{/pkg/interpolate}.

  \subsection{}
  Cubic spline interpolation is implemented in \texttt{/pkg/interpolate}. However this implementation is
  not fully compliant with the question requirements, so there is an additional (compliant but slower and
  more memory intensive) implementation in \texttt{/assignment/comply}.

  \subsection{}
  The data and interpolations are plotted in figure \ref{fig:interpolate}.

  \plot{assignment-q-3}{
    Linear and natural cubic spline interpolation on the given data.
  }{fig:interpolate}

% Question 4
\section{}
  \subsection{}
  The program can be found in \texttt{/assignment/q-4}. The convolutions are implemented in \texttt{/pkg/signal}.

  The program samples both functions over the range $[-10; 10]$ and takes $2^{10} = 1024$ samples. This
  is an exact power of two, which makes good use of the fast Fourier transform.

  A sample density of $1000$ was chosen for two reasons: firstly, this returns a convolution
  sampled at a rate which results in a smooth-looking graph. Secondly, trials using the Fourier
  transform show that for the chosen range and sample-density, one retrieves the original series very
  precisely with minimal edge-effects or aliasing. This should then lead to good results when
  using the Fourier transform to compute the convolution.

  These choices are validated, when comparing the numerical result to the exact result, which can
  be obtained by computing the convolution integral directly. (See figure \ref{fig:conv} and equation
  \ref{eq:conv}.)

  \subsection{}
  The plot showing $h(t)$, $g(t)$, and $(g * h)(t)$ can be seen in figure \ref{fig:conv}.

  % Given by: integrate(4 * exp((-(t-tau)**2)/2) / sqrt(2*pi), (tau, 3, 5))
  The theoretical result
  \begin{equation}
    \label{eq:conv}
    (g * h)(t) = - 2 \operatorname{erf}{\left(\frac{\sqrt{2} \left(3 - t\right)}{2} \right)} + 2 \operatorname{erf}{\left(\frac{\sqrt{2} \left(5 - t\right)}{2} \right)}
  \end{equation}
  is also shown in the plot.

  \plot{assignment-q-4}{
    Convolution of the function g and h. The plot depicts the functions themselves, their exact
    convolution, and the result of their numerical convolution.
  }{fig:conv}

% Question 5
\section{}
  \subsection{}
  The program is implemented in \texttt{/assignment/q-5}, and the resulting distribution can be seen in
  figure \ref{fig:uniform}. The used random sampling is ultimately based on \texttt{PCG XSL RR 128/64},
  which is a random number generator with very robust statistical properties and small memory
  requirements\cite{pcg}.

  \plot{assignment-q-5-a}{
    Random samples drawn from a uniform distribution over $[0,1]$. The red line indicates the shape of
    the sample distribution.
  }{fig:uniform}

  \subsection{}
  \label{sec:sample}
  Starting from a uniform variate $x \in [0,1]$ with $x \sim p(x) = 1$, consider:

  \begin{equation}
    \int_0^x d\zeta p_x(\zeta) = \int_0^{y(x)} d\gamma p_y(\gamma) = C_y(y(x)),
  \end{equation}
  from where it follows that
  \begin{equation}
    y(x) = C_y^{-1}(x) = \left(\int_0^y d\gamma p_y(\gamma)\right)^{-1}(x).
  \end{equation}

  Given $p_y(y) = \frac{1}{2} \cos(\frac{y}{2})$, then:
  \begin{equation}
    C_y(y) = \int_0^y d\gamma \frac{1}{2} \cos(\frac{\gamma}{2}) = \sin(\frac{y}{2}),
  \end{equation}
  from which finally:
  \begin{equation}
    y(x) = C_y^{-1}(x) = 2 \times \arcsin(x).
  \end{equation}

  The resulting sample distribution can be seen in figure \ref{fig:sample}.

  \plot{assignment-q-5-b}{
    Samples of $x \sim \frac{1}{2} \cos(\frac{x}{2})$. The red line indicates the shape of
    the sample distribution.
  }{fig:sample}

  \subsection{}
  The rejection method is implemented in \texttt{/assignment/comply}. The implementations
  interface is modeled on the rejection sampling interface provided by Gonum.

  We use the distribution from \ref{sec:sample} as the proposal distribution, with a value
  of $c = 1.3$. (This $c$ is chosen to minimize the rejection probability\footnotemark{}.) The resulting
  sample distribution can be seen in figure \ref{fig:reject}.

  \footnotetext{Notice that $\frac{4}{\pi} \approx 1.3$.}

  \plot{assignment-q-5-c}{
    Samples of $x \sim \frac{2}{\pi} \cos^2(\frac{x}{2})$. The red line indicates the shape of
    the sample distribution.
  }{fig:reject}

  The ration of the time taken versus \ref{sec:sample} falls around $1.3$. Knowing that the
  expected number of samples taken is $\mathbb{E}(N) = n \times c$, where $n$ is the number of samples
  to generate, the expected ratio is $1.3$. Note that the relative time taken varies with every run,
  and depends on additional factors that lie outside the program itself. Running the binary multiple
  times shows that the average ration is $\approx 1.3$ as expected.

\bibliography{assignment.bib}{}
\bibliographystyle{plain}

\appendix{}

\section{Unit Tests}
The code written for the assignments, is extensively unit-tested. To run the unit tests, use
\texttt{make test} and to inspect test-coverage on a line-by-line basis, run \texttt{make cover}.

Below, the tests written for the different routines are outlined. These tests aim to validate the
implementations submitted as part of this assignment. (Tests can be found in files named
\texttt{*\_text.go}.)

  \subsection{LU Decomposition}
  The tests approaches are the following:
  \begin{itemize}
    \item $LU = M$, so one can check if $L$ and $U$ are correct by performing a matrix multiplication
    and comparing with $M$.
    \item $M\vec{x} = \vec{y}$, so one can similarly check if $\vec{x}$ is correct, by performing
    a dot product and comparing the resulting vector with $\vec{y}$.
    \item $M^{-1}M = \mathbb{I}$, so one can verify any obtained inverse by performing a matrix
    multiplication and comparing the result to unity.
  \end{itemize}

  \subsection{Interpolation}
  The cubic splines and linear interpolation should be the exact result if the function being interpolated
  is linear or cubic respectively. This allows one to compare values directly and verify.

  \subsection{Convolution}
  If one attempts a convolution for which the analytic result is known, one can directly compare the
  obtained values and verify the convolution is computed correctly.

  \subsection{Sampling}
  There are no unit-tests for the sampling routines. However, above the resulting distributions
  are plotted and compared to the theoretical distributions sampled from (see e.g. figure \ref{fig:reject}).

  This demonstrates, that with regards to the resulting distribution, the samples work as advertised.
  There are many more statistics which one could compare to known values. And depending on the sampling
  routine and quality of our random numbers some may show a discrepancy. However for the purposes of
  this assignment, generating the correct distribution is sufficient\footnotemark{}.

  \footnotetext{Since a random number generator with good statistical properties is used, one might expect
  that our sampler will have reasonable statistical properties. Seeing that the assignment required to
  use a good random number generator and justify the choice, one might argue that more statistics are
  expected to be correct. However the only statistic relevant to the desired output (histogram) is the
  distribution, which is correct. So I felt that no further tests were necessary.}

\section{Word Count}
\texttt{wc} reports 1526 words for the \LaTeX{} source file.

\end{document}


\setchapterpreamble[u]{\margintoc}
\chapter{Mathematical Background}

The character of quantum mechanics is to a large extent dependent on the mathematics used to describe it. At the end of the day, all theory needs to be supported by experimental evidence. That is, by actual measurements. We quantify our measurements with numbers to speak in a standardized, precise way. When we ask questions like, ``How long?'' or ``How much?'', we typically use numbers to give an answer.

In quantum mechanics, we will use increasingly sophisicated mathematical constructs to describe the world around us, including experiments physicists perform. Some of properties of numbers that we take for granted will remain, while others will disappear. Here we begin with a exploration of two particular useful number systems: the \emph{real} and the \emph{complex} numbers.

\section{Real Numbers}\marginnote{Real numbers. Communitivity, distributivity. (Figures showing geometric equivalence.) \textbf{Exercises.} Field properties. Example, the real numbers. Binary. Finite fields. $\mathbf{Z}_2$. $V_4$. \textbf{Exercises.} Cartesian plane. Absolute value. Distance. Circles. \textbf{Exercises.}}

We start of with our old friend, the \emph{real line}. The real line is an ordinary geometric straight line. We label the points on the line with the set $\mathbf{R}$ of all \emph{real numbers}. That is, we identify each point on the line with a unique real number. And  we often speak as if the numbers were actually on a line, and as if the points on the line are actually numbers.

\begin{figure}[h]
\input{figures/introduction/real-line.tex}
\caption{The real line}
\end{figure}

The structure of the real line is anything but simple. Do not be fooled. You may wonder what makes the real numbers ``real''? Begin with integers.

The Greeks restricted themselves to the numbers that could be constructed using a compass and an unmarked straight edge. The began with the so-called \emph{natural numbers}, usually denoted as a set by $\mathbf{N}$. The construction went like so [FIGURE]. Because the straight edge had no unit measure on it, the results they produced are \emph{coordinate-free}; that is, true no matter which coordinates we choose, be it inches, centimeters, or pikas.

Then fractions. Pythagoras' cult discovered the irrationality of square root of two. Are there any holes? Answer to this question can be found in the study of real analysis. For now, the real line will suffice.  We will review a few properties of the real numbers.

\begin{marginfigure}
\input{figures/introduction/integer-line-construction.tex}
\caption{Construction of the natural numbers}
\end{marginfigure}

\section{Complex Numbers}

\marginnote{\textsf{Jordan Chapter 2: Imaginary Numbers}}


\chapter{Complex Numbers}

\Section{1}{1}{Rational numbers.}

The idea of a set of numbers is derived in the first instance from the
consideration of the set of positive* integral numbers, or positive
integers; that is to say, the nnmbL-rs 1. 2, 8, 4, .... Positive
integers have many properties, which will be found in treatises on the
Theory of Integral Numbers; but at a very early stage in the
development of mathematics it was found that the operations of
Subtraction and Division could only be performed among them subject to
inconvenient restrictions; and consequently, in elementary
Arithmetic, classes of numbers are constructed such that the
operations of subtraction and division can always bB performed among
them.

To obtaiti a class of numbers among which the operation of subtraction
can be performed without restraint wc construct the class of integers,
which consists of the class of positive f integers (+1, +2, +3, ...)
and of the class of negative integers (-1, -2, -3, ...) and the number
0.

To obtain a class of numbers among which the operations both of sub-
traction and of division can be performed freely:):, we construct the
class of rational numbers. Symbols which denote members of this class
are |, 3,

0, -Y--

We have thus introduced three classes of numbers, (i) the signless
integers, (ii) the integers, (iii) the rational numbers.

It is not part of the scheme of this work to discuss the construction
of the class of integers or the logical foundations of the theory of
I'ational numbers .

The extension of the idea of number, which has just been described,
was not effected without some opposition from the more conservative
mathematicians. In the latter half of the eighteenth century, Maseras
(1731-1824) and Frend (1757-1841) published works on Algebra,
Trigonometry, etc., in which the use of negative numbers was
disallowed, although Descartes had used them imrestrictedly more than
a hundred years before.

* Strictly speakinpr, a more appropriate epithet would be, not
positive, but signless.

t In the strict sense.

:*: With the exception of division by the rational number 0.

§ Such a discussion, defining a rational number as an ordered
number-pair of iute ers in a similar manner to that in which a complex
number is defined in \hardsectionref{1}{3} as an ordere i number-pair of real numbers,
will be found in Hobsou's Functions of a Real Variable, §§ 1-12.

1-2

%
% 4
%

A rational number x may be represented to the eye in the following
manner :

If, on a straight line, we take an origin and a fixed segment OPi (Pi
being on the right of 0), we can measure from a length OFx such that
the ratio OPx/OPi is equal to x; the point P is taken on the right or
left of according as the number x is positive or negative. We may
regard either the point P or the displacement OP (which Avill be
written OPx) as repre- senting the number x.

All the rational numbers can thus be represented by points on the
line, but the converse is not true. For if we measure off on the line
a length OQ equal to the diagonal of a square of which OPi is one
side, it can be proved that Q does not correspond to any rational
number.

Points oil the line which do not represent rational numbers may be
said to represent irrational numbers; thus the jjoint Q is said to
rej resent the irrational number, /2 = l 414213.... But while such
an explanation of the existence of irrational numbers satisfied the
mathematicians of the eighteenth century and may still be sufficient
for those whose interest lies in the applications of mathematics
rather than in the logical upbuilding of the theory, yet from the
logical standpoint it is improper to introduce geometrical intuitions
to supply deficiencies in arithmetical arguments; aud it was shewn by
Dedekind in 1858 that the theory of irrational numbers can be
established on a purely arithmetical basis without any appeal to
geometry.

\Section{1}{2}{Dedekind's* theory of irrational numbers.}

The geometrical property of points on a line which suggested the
starting point of the arithmetical theory of irrationals was that, if
all points of a line are separated into two classes such that every
point of the first class is on the right of every point of the second
class, there exists one and only one point at which the line is thus
severed.

Following up this idea, Dedekind considered rules by which a
separationf or section of all rational numbers into two classes can be
made, these classes (which will be. called the Z-class and the
P-class, or the left class and the right class) being such that they
possess the following properties :

(i) At least one member of each class exists.

(ii) Every member of the X-class is less than every member of the
P-class.

It is obvious that such a section is made b ' an rational number x;
and X is either the greatest number of the Z-class or the least number
of the

* The theory, though elaborated in 1858, was not published before the
appearance of Dede- kind's tract, Stetigkeit und irrationale Zahlen,
Brunswick, 1872. Other theories are due to Weitrstrass [see von
Dantscher, Die Weierstrasa'sche Theorie der irrationulen Zahlen
(Leipzig, 1908)] and Cantor, Math. Ann. v. (1872), pp. 123-130.

t This procedure formed the basis of the treatment of irrational
numbers by the Greek mathematicians in the sixth aud fifth centuries
b.c. The advance made by Dedekind consisted in observing that a purely
arithmetical theory could be built ap on it. *

%
% 5
%

i?-class. But sections can be made in which no rational number x plays
this part. Thus, since there is no rational number* Avhose square is
2, it is easy to see that we may form a section in which the i?-c]ass
consists of the positive rational numbers whose squares exceed 2, and
the Z-class consists of all other rational numbers.

Then this section is such that the i?-class has no least member and
the Z-class has no greatest member; for, if x be any positive
rational fraction,

and 2 are in order of magnitude; and therefore given any member x of
the Zr-class, we can always find a greater member of the Z-class, or
given any member x of the 7?-class, we can always find a smaller
member of the /i- class, such numbers being, for instance, y and \ j,
where y' is the same function of x' as y of x.

If a section is made in which the i -chiss has a least member Jo, or
if the /> class has a greatest member J,, the section determines a
rational-real number, which it is convenient to denote by the samef
symbol Ao\ or A .

If a section is made, such that the i -class has no least member and
the Z-class has no greatest member, the section determines an
irrational-real number I.

If X, y are real numbers (defined by sections) we say that x is
greater than 3/ if the Z-class defining x contains at least two§
members of the i -class defining y.

Lot a,, ... be real numbers and let .4j, i?,. ... be any members of
the corresponding Z-classes while A., B... are any members of the
corresponding i -classes. The classes of which A, Ao, ... are
respectively members will be denoted by the symbols A ), A. ), ....

Then the sum (written a + ) of two real numbers a and /3 is defined as
the real number (rational or irrational) which is determined by the
Z-class A + B,) and the E-class A. + B.,).

It is, of course, necessary to prove that these classes determine a
section of the rational numbers. It is evident that A i + 5, < A, +
B., and that at least one member of each of the classes Ai + B ),
A.,-\-B.,) exists. It remains to prove that there is, at most, one
rational

* For if piq be such a number, this fraction being in its lowest
terms, it may be seen that (2'if-i')/(p-9) is another such number, and
0<p-q<q, so that pjq is not in its lowest terms. The contradiction
implies that such a rational number does not exist.

+ This causes no confusion in practice.

X B. A, W. Russell defines the class of real numbers as actualhj being
the class of all L-classes; the class of real numbers whose i classes
have a greatest member corresponds to the class of rational numbers,
and though the rational-real number x which corresponds to a rational
number .r is conceptually distinct from it, no confusion arises from
denoting both by the same symbol.

§ If the classes had only one member in common, that member might be
the greatest member of the JL-class of .r and the least member of the
i -class of y.

%
% 6
%

number which is greater than every Ai + B and less than every A., +
B.,; suppose, if possible, that tliere are two, .v and y (?/>.r). Let
cfj be a member of (.Ij) and let a-> be a member of (Ao); and let lY
be the integer next greater than (a2- i)/ i(y-. ) - Take the last of

the numbers ai-|- ((,,\ flj), (where m=0, 1, ... lY), which belongs to
(A ) and the first of

them which belongs to (Ao); let these two numbers be Cj, c,. Then

 2 -' i = Jr ao - i) < i (3/ - )- Choose c/i, do in a similar maimer
from the classes defining /3; then

C2 + cL - f 1 - c?i <y - A'. But C2 + d., i Ci+di x, and therefore C2
+ c?2-Ci-c/i > i/-.?-; we have therefore arrived at a contradiction by
supjjosing that two rational numbers a;, y exist belonging neither to
( Jj + B ) nor to ( 2 + 2)-

If every rational number belongs either to the class (J 1 + 1) or to
the class ( 2+ 2), then the classes (Jj + i), ( 2 + 2) define an
irrational number. If one rational number.)- exists belonging to
neither class, then the Z-class formed by x and (Jj+ j) and the
i2-class ( 2 + -52) define the rational-real number x. In either
ca.se, the number defined is called the sum a + /3.

The difference a-/3 of two real numbers is defined by the Z-class
(J1-Z2) and the Z'-class ( 2- 1).

The product of two positive real numbers a, /3 is defined by the 7
class A B-i) and the Z-class of all other rational numbers.

The reader will see without difficulty how to define the product of
negative real num- bers and the quotient of two real numbers; and
further, it may be shewn that real numbers may be combined in
accordance with the associative, distributive and commuta- tive laws.

The aggregate of rational-real and irrational-real numbers is called
the aggregate of real numbers; for brevity, rational-real numbers and
irrational- real numbers are called rational and irrational numbers
respectively.

\Section{1}{3}{Complex numbers.}

We have seen that a real number may be visualised as a displacement
along a definite straight line. If, however, P and Q are any two
points in a plane, the displacement PQ needs two real numbers for its
specification; for instance, the differences of the coordinates of P
and Q referred to fixed rectangular axes. Ifthe coordinates of P be (
, 77) and those oi Q( + cc,7 +i/), the displacement PQ may be
described by the symbol [x, y]. We are thus led to consider the
association of real numbers in ordered* pairs. The natural definition
of the sum of two displacements [x, y\ [x, y'] is the displacement
which is the result of the successive applications of the two
displacements; it is therefore convenient to define the sum of two
number-pairs by the equation

[.c, y] + [x', y'] = [x + x\ y -f y'].

The order of the two terms distiuguishes the ordered number-pair [.r,
ij] from the ordered number-pair [?,r].

%
% 7
%

The product of a number-pair and a real, number x is then naturally
defined by the equation

x X \ x, y\ = \ x'x, x'y\

We are at liberty to define the product of two number-pairs in any
convenient manner; but the only definition, which does not give rise
to results that are merely trivial, is that symbolised by the equation

[x, y] X [x, ij] = [xx' - i/t/', xy + xy\

It is then evident that

[x, 0] X [x, y ] = [xx, xy' = xx [x, y']

and [0, y] x [x, y] = [- yy', x'y] = y x [- y', x'].

The geometrical interpretation of these results is that the effect of
multiplying by the displacement [x, 0] is the same as that of
multiplying by the real number x; but the effect of multiplying a
displacement by [0, y] is to multiply it by a real number y and turn
it through a right angle.

It is convenient to denote the number-pair [x, y] by the compound
symbol x + iy; and a number-pair is now conveniently called (after
Gauss) a complex number; in the fundamental operations of Arithmetic,
the complex number x+ iO may be replaced by the real number x and,
defining i to mean + il, we have i- = [0, 1] x [0, 1] = [- 1, 0]; and
so r may be replaced by - 1.

The reader will easily convince himself that the definitions of
addition and multiplication of numbei--pairs have been so framed that
we may perform the ordinary operations of algebra with complex numbers
in exactly the same way as with real numburs, treating the symbol i as
a number and replacing the product it by - I wherever it occurs.

Thus he will verify that, if a, b, c are complex numbers, we have

a + b = b + a,

ab = b(i,

(a + b) +c = a +(b -\-c),

ab .c = a.bc,

a b -\- c)= ab + ac,

and if ab is zero, then either a or 6 is zero.

It is found that algebraical operations, direct or inverse, when
applied to complex numbers, do not suggest numbers of any fresh type;
the complex number will therefore for our purposes be taken as the
most general type of number.

The introduction of the complex number has led to many important
developments in mathematics. Functions which, when real variables only
are considered, appear as essentially distinct, are seen to be
connected when complex variables are introduced :

%
% 8
%

thus the circular functions are found to be expressible in terms of
exponential functions of a complex argument, by the equations

TODO

Again, many of the most important theorems of modern analysis are not
true if the numbers concerned are restricted to be real; thus, the
theorem that every algebraic equation of degree n has % roots is true
in general only when regarded as a theorem concerning complex numbers,

Hamilton's quaternions furnish an example of a still fiu'ther
extension of the idea of number. A quaternion

iv+xi+1/j + zk

is formed from four real numbers w, .i; y, z, and four number- units
1, i, j, l\ in the same way that the ordinary complex number x-\-iy
might be regarded as being formed from two real numbers x, y, and two
number-units 1, i. Quaternions however do not obey the commutative law
of multiplication.

\Section{1}{4}{The modulus of a complex number.}

Let X + iy be a complex number, x and y being real numbers. Then the
positive square root of x- 4- y- is called the modulus of x + iy), and
is

written

x + iy.

Let us consider the complex number which is the sum of two given
complex numbers, x + iy and u + iv. We have

TODO

The modulus of the sum of the two numbers is therefore

\ \ {X + Uf + ((/ + V) 2,

or [ x- + y-) + ur + V-) -+ 2 xu + ijv)] .

But

 \ x-\-iy\-\-\ u + iv\ ] -= [ x- + /y"') + (w- + v' yf

= ( 2 4- y ) + ( <2 +;2 + 2 x' + t/'f tC + V') '

= (x- + y-) + (u + tj2) + 2 (xu + yv)- -h (xv - yu)-], and this
latter expression is greater than (or at least equal to) x- + y' ) +
(u" + V-) + 2 (xu + yv). We have therefore

TODO

i.e. the moduhis of the sum of two complex numbers cannot be greater
than the sum of their moduli; and it follows by induction that the
modulus of the sum of any number of complex numbers cannot be greater
than the sum of their moduli.

%
% 9
%

Let us consider next the complex number which is the product of two
given complex numbers, x + iy and u + iv; we have

 pG 4- iy) u + iv) = xu - yv) + i xv + yu), and therefore

TODO

The modulus of the product of two complex numbers (and hence, by in-
duction, of any number of complex numbers) is therefore equal to the
product of their moduli.

\Section{1}{5}{The Argand diagram.}

We have seen that complex numbers may be represented in a geometrical
diagram by taking rectangular axes Ox, Oy in a plane. Then a point P
whose coordinates referred to these axes are x, y may be regarded as
representing the complex number x + iy. In this way, to every point of
the plane there corresponds some one complex number; and, conversely,
to every possible complex number there corresponds one, and only one,
point of the plane.

The complex number x + iy may be denoted by a single letter* z. The
point P is then called the representative point of the number z; we
shall also speak of the number z as being the ajfx of the point P.

If we denote (jf + /) by r and choose 6 so that r cos 6 = x, r sin 9
=y, then r and 6 are clearly the radius vector and vectorial angle of
the point P, referred to the origin and axis Ox.

The representation of complex numbers thus afforded is often called
the Argand diagram .

By the definition already given, it is evident that r is the modulus
of z. The angle 6 is called the argument, or amplitude, or phase, of
z.

We write 6 - arg z.

From geometrical considerations, it appears that (although the modulus
of a complex number is unique) the argument is not unique;; if be a
value of the argument, the other values of the argument are comprised
in the expression 2mr + d where a is any integer, not zero. The
principal value of arg 2 is that which satisfies the inequality TODO.

* It is convenient to call .r and y the real and imaginary parts of z
respectively. We fre- quently write x = R(~), y = I(z).

t .J. E. Argand published it in 1806; it had however previously been
used by Gauss, and by Caspar Wessel, who discussed it in a memoir
presented to the Danish Academy in 1797 and published by that Society
in 1798-9.

X See the Appendix, § A-521.

%
% 10
%

If P, and P2 are the representative points corresponding to values z-
and Z.2 respectively of z, then the point which represents the value
z- + z. is clearly the terminus of a line drawn from P, equal and
parallel to that which joins the origin to P. 2.

To find the point which represents the complex number z Zo, where z
and Zo are two given complex numbers, we notice that if

z = 1 (cos 6 + i sin 6 ),

z.2, = ?-2 (cos f 2 + * sin o) then, by multiplication,

Z\ Z% = n 2 [cos ((9i + 2) + sin ( j + 6. ]. The point which
represents the number z z has therefore a radius vector measured by
the product of the radii vectores of Pj and P2, and a vectorial angle
equal to the sum of the vectorial angles of Pj and Pg.

REFERENCES.

Tke gical foundations of the theory of number. u . X. Whitehead axd B.
A. W. Russell, Principia Mathematica (1910-1913). 'B. A. W. Russell,
Introduction to Mathematical Philosophy (1919).

On irrational numbers.

R. Dedekixd, Stetigkeit unci irrationale Zahlen. (Brunswick, 1872.)
''V. VON Dantscher, Vorlesungen ueber die Weierstrass'sehe Theorie der
irrationalen

Zahlen. (Leipzig, 1908.) E .W. HoBSON, Functions of a Real Variable
(1907), Ch. i. T.'*?. I'a. Bromwich, Theory of Infinite Series (1908),
Appendix i.

On omplex numbers.

H. Hankel, Theorie der complexen Zahlen-systeme. (Leipzig, 1867.) O.
Stolz, Yorlesungen iiber allgemeine Arithmetik, II. (Leipzig, 1886.)
G. H. Hardy, A course of Pure Mathematics (1914), Ch. in.

Miscellaneous Examples.

1. Shew that the representative points of the complex numbers 1 + 4 2
4-7?', 34-10 are coHinear.

2. Shew that a paralwla can be drawn to pass through the
representative points of the complex numbers

2 + V, 44-4t, 6 + 9 8+ 16 10 + 2.5i.

3. Determine the nth. roots of unity by aid of the Argand diagram;
and shew that the number of primitive roots (roots the powers of each
of which give all the roots) is the number of integers (including
unity) less than n and prime to it.

Prove that if j, 2? 3,  he the arguments of the primitive roots, 2
cosjo =0 when

p m \& positive integer less than -7 ., where a, b, c, ... k are the
difterent constituent

n ( - 'f n

primes of 71; and that, when p-, - -?, 2 cos p6=,-,, where /x is
the number of

(toe, m iC ccoc,,, fc

the constituent primes. \addexamplecitation{Math. Trip. 1895}


Gumbel statistics are often used to estimate the statistical
significance of local alignment scores.

The Gumbel distribution is the so-called Type I extreme value
distribution (EVD). It occurs so frequently in sequence analysis
applications, compared to the type II (Fr\'{e}chet) and type III
(Weibull) extreme value distributions, that ``Gumbel'' and ``EVD'' are
often used interchangeably in bioinformatics. Easel has a separate
module, the \eslmod{gev} module, that implements the generalized
extreme value distribution.

Karlin/Altschul statistics are a special case of the Gumbel
distribution that apply to the scores of ungapped local alignments
between infinitely long random sequences. Empirically, Karlin/Altschul
statistics also apply reasonably well to the more useful case of
gapped alignment of finite-length sequences. Karlin/Altschul
statistics predict how the Gumbel's two parameters depend on the
length of the query and target sequences. In the case of ungapped
alignments, Karlin/Altschul statistics allow the Gumbel parameters to
be estimated directly, without the need for a compute-intensive
simulation.

\subsection{The gumbel API}

The \eslmod{gumbel} API consists of the following functions:

\vspace{0.5em}
\begin{center}
\begin{tabular}{ll}\hline
    \multicolumn{2}{c}{\textbf{evaluating densities and distributions:}}\\
\ccode{esl\_gumbel\_pdf()}     & Returns the probability density, $P(S=x)$.\\
\ccode{esl\_gumbel\_logpdf()}  & Returns the log of the pdf, $\log P(S=x)$.\\
\ccode{esl\_gumbel\_cdf()}     & Returns the cumulative probability distribution, $P(S \leq x)$.\\
\ccode{esl\_gumbel\_logcdf()}  & Returns the log of the cdf, $\log P(S \leq x)$.\\
\ccode{esl\_gumbel\_surv()}    & Returns right tail mass, 1-cdf, $P(S > x)$\\
\ccode{esl\_gumbel\_logsurv()} & Returns log of 1-cdf, $\log P(S > x)$.\\
    \multicolumn{2}{c}{\textbf{sampling:}}\\
\ccode{esl\_gumbel\_Sample()}  & Returns a Gumbel-distributed random sample.\\
    \multicolumn{2}{c}{\textbf{maximum a posteriori parameter fitting:}}\\
\ccode{esl\_gumbel\_FitComplete()} & Estimates $\mu,\lambda$ from complete data.\\
\ccode{esl\_gumbel\_FitCompleteLoc()} & Estimates $\mu$ when $\lambda$ is known.\\
\ccode{esl\_gumbel\_FitCensored()} & Estimates $\mu,\lambda$ from censored data.\\
\ccode{esl\_gumbel\_FitCensoredLoc()} & Estimates $\mu$ when $\lambda$ is known.\\
\ccode{esl\_gumbel\_FitTruncated()}& Estimates $\mu,\lambda$ from truncated data.\\\hline
\end{tabular}
\end{center}
\vspace{0.5em}

The Gumbel distribution depends on two parameters, $\mu$ and
$\lambda$. When $\mu$ and $\lambda$ are known, the statistical
significance (P-value) of a single score $x$ is $P(S>x)$, obtained by
a call to \ccode{esl\_gumbel\_surv()}.  The E-value for obtaining that
score or better in searching a database of $N$ sequences is just
$NP(S>x)$.

When $\mu$ and $\lambda$ are unknown, they are estimated from scores
obtained from comparisons of simulated random data. (Analytical
solutions for $\mu$ and $\lambda$ are only available in the case of
ungapped sequence alignments.)  The \ccode{esl\_evd\_Fit*()} functions
provide maximum likelihood parameter fitting routines for different
types of data.

\subsubsection{Augmentations: random, minimizer}

The \ccode{esl\_gumbel\_Sample()} function requires augmenting with the
\eslmod{random} module.

The \ccode{esl\_gumbel\_FitTruncated()} function requires augmenting
with the \eslmod{minimizer} module.

\subsection{Example of using the gumbel API}

An example that samples 10,000 data points from a Gumbel distribution
with $\mu=-20$, $\lambda=0.4$; reports the min and max samples, and
the probability mass to the left of the min and to the right of the
max (both of which should be about $\frac{1}{10000}$, since we took
10,000 samples); and then fits those simulated data to a Gumbel and
reports the fitted $\mu$ and $\lambda$:

\input{cexcerpts/gumbel_example}

\subsection{Gumbel densities}

The probability density function (pdf) and the cumulative distribution
function (cdf) of the extreme value distribution are:

\begin{equation}
P(x) = \lambda \exp \left[ -\lambda (x - \mu) - e^{- \lambda (x - \mu)} \right]
\label{eqn:gumbel_density}
\end{equation}

\begin{equation}
P(S < x) = \exp \left[ -e^{-\lambda(x - \mu)} \right]
\label{eqn:gumbel_distribution}
\end{equation}

The extreme value density and distribution functions for $\mu = 0$ and
$\lambda = 1.0$ are shown below.

\begin{center}
\includegraphics[width=3in]{figures/evd_basic}
\end{center}

The $\mu$ and $\lambda$ parameters are {\em location} and {\em scale}
parameters, respectively:

\centerline{
\begin{minipage}{3in}
\includegraphics[width=2.8in]{figures/evd_location}
\end{minipage}
\begin{minipage}{3in}
\includegraphics[width=2.8in]{figures/evd_scale}
\end{minipage}
}

For more details, a classic reference is \citep{Lawless82}.  Gumbel
distributions can have their long tail to the right or to the
left. The form given here is for the long tail to the right.  This is
the form that arises when the extreme value is a maximum, such as when
our score is the maximum over the individual scores of all possible
alignments. The equations in \citep{Lawless82} are for extremal
minima; use $(x - u) = -(x - \mu)$ and $b = 1 / \lambda$ to convert
Lawless' notation to the notation used here.

\subsection{Fitting Gumbel distributions to observed data}

Given a set of $n$ observed samples $\mathbf{x}$, we may want to
estimate the $\mu$ and $\lambda$ parameters.

One might try to use linear regression to fit to a $\log \log$
transformation of the $P(S < x)$ histogram, which gives a straight
line with slope $-\lambda$ and $x$ intercept $\mu$:

\begin{equation}
\log \left[ -\log P(S<x) \right] = -\lambda x + \lambda \mu
\end{equation}

However, the linear regression method is undesirable because it is
sensitive to outliers. The following table shows the \% error for
estimating $\hat{\mu}$ and $\hat{\lambda}$ from 500 simulated complete
datasets, sampled from a Gumbel with $\mu = -20.0$ and $\lambda =
0.4$, for four different dataset sizes:

\begin{center}
\begin{tabular}{lrrrr} \hline
                              & \multicolumn{4}{c}{\# of samples}\\
                              & 100 & 1000  & 10,000 & 100,000 \\
\% error in $\hat{\mu}$       &  2\%&   1\% & 0.9\%  &  0.9\%  \\
max error in $\hat{\mu}$      & 24\%&  13\% &  10\%  &   10\%  \\
\% error in $\hat{\lambda}$   & 12\%&   7\% &   5\%  &    3\%  \\
max error in $\hat{\lambda}$  & 49\%&  33\% &  25\%  &   20\%  \\ \hline
\end{tabular}
\end{center}

A better rough estimate of $\hat{\mu}$ and $\hat{\lambda}$ can be
obtained from the sample mean $m$ and variance $s^2$ of the observed
data \citep{Evans00}:\footnote{All simulation data are generated by
the \eslmod{evd} module's stats driver. The only exception is the
linear regression fit data, which come from an old version of HMMER.}

\begin{eqnarray*}
  \hat{\lambda} & = & \frac{\pi}{\sqrt{6s^2}}\\
  \hat{\mu}     & = & m - \frac{0.57722}{\hat{\lambda}}
\end{eqnarray*}

The mean/variance method is more accurate than linear regression, as
shown by the following simulation results:

\begin{center}
\begin{tabular}{lrrrr} \hline
                              & \multicolumn{4}{c}{\# of samples}\\
                              & 100 & 1000  & 10,000 & 100,000 \\
\% error in $\hat{\mu}$       &  1\%& 0.3\% &  0.1\% & 0.03\%  \\
max error in $\hat{\mu}$      &  5\%&   1\% &  0.4\% &  0.1\%  \\
\% error in $\hat{\lambda}$   &  9\%&   3\% &  0.8\% &  0.3\%  \\
max error in $\hat{\lambda}$  & 40\%&  12\% &    3\% &  0.9\%  \\ \hline
\end{tabular}
\end{center}

Still, the mean/variance method is not as accurate as a maximum
likelihood estimation (especially for $\lambda$). Also, it requires
complete data, whereas we also need to solve problems where we fit to
\emph{truncated} or \emph{censored} data. Easel's main estimation
methods are therefore maximum likelihood methods.

\subsubsection{Maximum likelihood estimation, complete data}

Given $n$ samples $x_1..x_n$ from some distribution that depends on
parameters $\theta$, we want to estimate maximum likelihood parameter
estimates $\hat{\theta}$ that maximize the log likelihood:

\[
   \hat{\theta} = \argmax_{\theta} \sum_{i=1}^{n} \log P(x_i \mid \theta)
\]

These are also \emph{maximum a posteriori} parameter estimates, if we
assume a uniform prior $P(\theta)$.

Specifically, for samples $x_i$ drawn from an extreme value
distribution, the log likelihood to optimize is:

\begin{equation}
\log L(\lambda, \mu) = n \log \lambda - \sum_{i=1}^{n} \lambda(x_i -
\mu) - \sum_{i=1}^{n} e^{-\lambda(x_i - \mu)}
\label{eqn:gev_logL}
\end{equation}

This objective function is differentiable with respect to $\mu$ and
$\lambda$:

\begin{eqnarray}
\frac{\partial \log L}{\partial \mu} & = &
n \lambda - \lambda \sum_{i=1}^{n} e^{-\lambda (x_i - \mu)}\\%
\\%
\label{eqn:mupartial}
\frac{\partial \log L}{\partial \lambda} & = &
\frac{n}{\lambda} - \sum_{i=1}^{n} (x_i - \mu) +
\sum_{i=1}^{n} (x_i - \mu) e^{-\lambda (x_i - \mu)}
\label{eqn:lambdapartial}
\end{eqnarray}

The maximum likelihood estimates $\hat{\lambda}$ and $\hat{\mu}$ are
the solutions to $\frac{\partial \log L}{\partial \mu} = 0$ and
$\frac{\partial \log L}{\partial \lambda} = 0$. Lawless
\citep{Lawless82} gives a useful trick here that lets us solve both of
these simultaneously. When (\ref{eqn:mupartial}) is set to zero, it
can be used to get $\hat{\mu}$ in terms of $\hat{\lambda}$:

\begin{eqnarray}
e^{-\hat{\lambda} \hat{\mu}} & = & \frac{1}{n} \sum_{i=1}^{n} e^{-\hat{\lambda} x_i}
\label{eqn:substitute}\\
\hat{\mu} & = & - \frac{1}{\hat{\lambda}}
	\log \left[ \frac{1}{n} \sum_{i=1}^{n} e^{-\hat{\lambda} x_i} \right]
\label{eqn:solvemu}
\end{eqnarray}

Substituting (\ref{eqn:substitute}) into (\ref{eqn:lambdapartial}),
gives us an equation for solving $\hat{\lambda}$ in terms of the
$x_i$'s:

\begin{eqnarray}
\frac{1}{\hat{\lambda}} - \frac{1}{n} \sum_{i=1}^{n} x_i +
\frac{\sum_{i=1}^{n} x_i e^{-\hat{\lambda} x_i}}
     {\sum_{i=1}^{n} e^{-\hat{\lambda} x_i}}
& = & 0
\label{eqn:newtontarget}
\end{eqnarray}

This is our target function. We could solve it readily enough (by
bisection search, for example) and obtain $\hat{\lambda}$. We can
solve it even faster using the Newton/Raphson algorithm, because it is
differentiable with respect to lambda:

\begin{equation}
\frac{d}{d\hat{\lambda}} =
\frac{\left( \sum_{i=1}^{n} x_i e^{-\hat{\lambda} x_i} \right)^2 }
     {\left( \sum_{i=1}^{n} e^{-\hat{\lambda} x_i}     \right)^2 }
-
\frac{\sum_{i=1}^{n} x_i^2 e^{-\hat{\lambda} x_i}}
     {\sum_{i=1}^{n} e^{-\hat{\lambda} x_i}}
-
\frac{1}{\hat{\lambda}^2}
\label{eqn:newtonderivative}
\end{equation}

Now, the key equations are (\ref{eqn:solvemu}),
(\ref{eqn:newtontarget}), and (\ref{eqn:newtonderivative}). In
summary, the inference procedure is the following:

\begin{itemize}
\item Guess an initial $\hat{\lambda}$ (using the mean/variance
  method, for example, but any reasonable guess works).
\item Use Newton/Raphson iterations to find the $\hat{\lambda}$ that satisfies
      (\ref{eqn:newtontarget}):
	\begin{itemize}
	\item calculate the target function $f$ and
         its first derivative $f'$ at $\hat{\lambda}$, using
	(\ref{eqn:newtontarget}) to calculate $f$ and
	(\ref{eqn:newtonderivative}) to calculate $f'$.
	\item If $f$ is within some absolute tolerance of zero
	(e.g., $10^{-6}$), stop; we have found $\hat{\lambda}$.
	\item Else, estimate a new $\hat{\lambda} = \hat{\lambda} - \frac{f}{f'}$,
	  and do another iteration.
	\end{itemize}
\item Plug $\hat{\lambda}$ into (\ref{eqn:solvemu}) to get $\hat{\mu}$.
\end{itemize}

This algorithm is implemented in \ccode{esl\_evd\_FitComplete()}.  An
auxiliary function, \ccode{lawless416()}, calculates the target
function and its derivative (equations (\ref{eqn:newtontarget}) and
(\ref{eqn:newtonderivative})) given the current estimate of
$\hat{\lambda}$.  The name comes from Lawless' equation 4.1.6, the
target function \citep{Lawless82}.

The accuracy of fitting to simulated data (generated with $\mu=-20$
and $\lambda=0.4$), collated over 500 simulations, is shown in the
following table:

\begin{center}
\begin{tabular}{lrrrr} \hline
                              & \multicolumn{4}{c}{\# of samples}\\
                              & 100 & 1000  & 10,000 & 100,000 \\
\% error in $\hat{\mu}$       &  1\%& 0.3\% &  0.1\% & 0.03\%  \\
max error in $\hat{\mu}$      &  4\%&   2\% &  0.5\% &  0.1\%  \\
\% error in $\hat{\lambda}$   &  6\%&   2\% &  0.6\% &  0.2\%  \\
max error in $\hat{\lambda}$  & 36\%&   9\% &    2\% &  0.8\%  \\ \hline
\end{tabular}
\end{center}

This is in accord with theoretical expectation. The distribution of
$\frac{\lambda}{\hat{\lambda}}$ is approximately normal with mean 1 and
standard error $\frac{0.78}{\sqrt{N}}$ \citep{Lawless82,Altschul01}.

% Altschul says \frac{\hat{\lambda}}{\lambda}, actually, but I believe
% that's wrong. xref J1/46.

\subsubsection{Maximum likelihood fitting to censored data}

A \emph{censored} data problem is when we have $N$ samples, but we
only observe the values of a subset of $n$ samples $x_1..x_n$ that are
greater or equal to some cutoff $\phi$. The remaining $z = N-n$
samples are \emph{censored}, and for these we only know that $x <
\phi$.  $x_i..x_n$, $n$, $\phi$, and $z$ are all known in a censored
data problem.

To estimate maximum likelihood parameters $\hat{\theta}$ for some
distribution from censored data \citep{Gelman95}, the log likelihood
to maximize is:

\[
  \hat{\theta} = \argmax_{\theta} z \log P(x<\phi \mid \theta)
                         + \sum_{i=1}^n \log P(x_i \mid \theta)
\]

Specifically, when fitting a Gumbel distribution, the log likelihood
to optimize is:

\begin{equation}
  \log L(\lambda, \mu) =
    n \log \lambda
     - z e^{-\lambda(\phi - \mu)}
     - \sum_{i=1}^{n} \lambda(x_i - \mu)
     - \sum_{i=1}^{n} e^{-\lambda(x_i - \mu)}
\label{eqn:censor_logL}
\end{equation}

To optimize this, we follow a similar procedure as used for complete
data \citep{Lawless82}. The log likelihood is differentiable with
respect to $\lambda$ and $\mu$:

\begin{eqnarray}
\frac{\partial \log L}{\partial \mu} & = &
n \lambda
- z \lambda e^{-\lambda (\phi - \mu)}
- \lambda \sum_{i=1}^{n} e^{-\lambda (x_i - \mu)}
\label{eqn:censor_dmu}
\\%
\frac{\partial \log L}{\partial \lambda} & = &
\frac{n}{\lambda}
+ z (\phi - \mu) e^{-\lambda (\phi - \mu)}
- \sum_{i=1}^{n} (x_i - \mu)
+ \sum_{i=1}^{n} (x_i - \mu) e^{-\lambda (x_i - \mu)}
\label{eqn:censor_dlambda}
\end{eqnarray}

Setting (\ref{eqn:censor_dmu}) to zero and solving for $\hat{\mu}$ in
terms of $\hat{\lambda}$ gives:

\begin{equation}
\hat{\mu}  =  - \frac{1}{\hat{\lambda}}
	\log \left[ \frac{1}{n}
	\left( z e^{-\hat{\lambda} \phi}
               + \sum_{i=1}^{n} e^{-\hat{\lambda} x_i} \right)
	\right]
\label{eqn:censor_solvemu}
\end{equation}

Substituting (\ref{eqn:censor_solvemu}) into
(\ref{eqn:censor_dlambda}) gives the target equation:

\begin{equation}
\frac{1}{\hat{\lambda}}
- \frac{1}{n} \sum_{i=1}^{n} x_i +
\frac{z \phi e^{-\hat{\lambda} \phi} + \sum_{i=1}^{n} x_i e^{-\hat{\lambda} x_i}}
     {z e^{-\hat{\lambda} \phi} + \sum_{i=1}^{n} e^{-\hat{\lambda} x_i}}
 =  0
\label{eqn:censor_newtontarget}
\end{equation}

To use Newton-Raphson root finding (instead of a slower bisection
search) we also need the first derivative of this target equation with
respect to $\lambda$:

\begin{equation}
\frac{d}{d\hat{\lambda}} =
\frac{\left(
        z \phi e^{-\hat{\lambda} \phi}
        + \sum_{i=1}^{n} x_i e^{-\hat{\lambda} x_i}
       \right)^2 }
     {\left(
        z e^{-\hat{\lambda} \phi}
        + \sum_{i=1}^{n} e^{-\hat{\lambda} x_i}
       \right)^2 }
-
\frac{z \phi^2 e^{-\hat{\lambda} \phi} + \sum_{i=1}^{n} x_i^2 e^{-\hat{\lambda} x_i}}
     {z  e^{-\hat{\lambda} \phi} + \sum_{i=1}^{n} e^{-\hat{\lambda} x_i}}
-
\frac{1}{\hat{\lambda}^2}
\label{eqn:censor_newtonderiv}
\end{equation}

In summary: given $n$ observed samples $x_1..x_n$ from a total sample
of $N$ samples, $z = N-n$ of which were censored because they have
values $< \phi$, we solve for maximum likelihood estimates
$\hat{\lambda}$ and $\hat{\mu}$ using the same procedure we used for
complete data, by using equations (\ref{eqn:censor_solvemu}),
(\ref{eqn:censor_newtontarget}), and (\ref{eqn:censor_newtonderiv}) in
place of equations (\ref{eqn:solvemu}), (\ref{eqn:newtontarget}), and
(\ref{eqn:newtonderivative}). Easel implements this procedure in
\ccode{esl\_evd\_FitCensored()}.  The target function
(\ref{eqn:censor_newtontarget}) and its derivative
(\ref{eqn:censor_newtonderiv}) are implemented in the auxiliary
function \ccode{lawless422()} \citep{Lawless82}.

Results on 500 simulated datasets with $\mu = -20, \lambda = 0.4$,
censored at $\phi = -20$ -- the expected peak of the histogram; that
is, a censored fit only to the right tail, which contains about 63\%
of the samples:

\begin{center}
\begin{tabular}{lrrrr} \hline
 & \multicolumn{4}{c}{\# samples in EVD histogram}\\
                        & 100 & 1000  & 10,000 & 100,000 \\
\% error in $\mu$       &  1\%& 0.4\% &  0.1\% &  0.04\%  \\
max error in $\mu$      &  5\%&   2\% &  0.5\% &  0.2\%  \\
\% error in $\lambda$   &  9\%&   3\% &  0.9\% &  0.3\%  \\
max error in $\lambda$  & 33\%&  11\% &    3\% &    1\%  \\ \hline
\end{tabular}
\end{center}

\subsubsection{Maximum likelihood fitting to truncated data}

A \emph{truncated} dataset is when we only observe $n$ samples $x_i$,
and an \emph{unknown} number $z$ of samples less than some threshold
$\phi$ were unobserved. Thus, only the right tail of $n$ samples $x_i
\geq \phi$ as observed. In a truncated dataset, $x_1..x_n$, $n$, and
$\phi$ are known, but $z$ is unknown.

Solving a truncated data problem motivates a Bayesian approach,
because we need to integrate out (marginalize) the nuisance $z$
parameter, and to do this, we have to specify a prior distribution for
$P(z)$. Gelman \emph{et al.} describe a general Bayesian framework for
thinking about various types of missing data problems, including
censored and truncated data \citep{Gelman95}.

In short, to obtain maximum likelihood parameters $\hat{\theta}$ for
some distribution, given truncated data, the log likelihood we wish to
maximize is:

\begin{equation}
  \hat{\theta} = \argmax_\theta -n \log P(x \geq \phi \mid \theta)
                   + \sum_{i=1}^n \log P(x_i \mid \theta).
\label{eqn:truncated_objective}
\end{equation}

\textbf{Detour: derivation of the truncated data likelihood}

The derivation of the above equation may not be immediately obvious.
The presence of the $n P(x \geq \phi \mid \theta)$ term may be
counterintuitive, as opposed to the more intuitive $z P(x < \phi \mid
\theta)$ term that accounts for the missing data in a censored data
problem. Gelman \emph{et al.} actually don't even show the equation in
their book; I obtained it from an exercise solution on their web site.
To convince you (and to remind me) of its correctness, a sketch of the
derivation follows.

We start with the same likelihood equation that arises in censored
data for a \emph{known} total number of samples $N$ (where $N=n+z$),
but since $N$ is unknown, we need to integrate over all possible $N$
from $n$ to $\infty$:

\begin{eqnarray*}
   P(\mathbf{x} \mid \theta, \phi) & = &
    \sum_{N=n}^{\infty}   P(\mathbf{x} \mid \theta, \phi, N) P(N)\\
   & = &
    \prod_{i=1}^n P(x_i \mid \theta)
    \left[
      \sum_{N=n}^\infty {N \choose n} P(x < \phi \mid \theta)^{N-n} P(N)
    \right]\\
\end{eqnarray*}

The $\prod_{i=1}^n P(x_i \mid \theta)$ is straightforward; that sum is
our problem. The trick is to rearrange it so we can treat it as a
convergent negative binomial series:

\[
   (1-p)^{-a} = 1 + ap + \frac{a(a+1)}{2!} p^2 +
   \frac{a(a+1)(a+2)}{3!} p^3...
\]

To get the sum into the form of this series, Gelman \emph{et al.}
suggest using an informative prior $P(N) = \frac{1}{N}$, an apparently
unmotivated choice that happens to make the sum collapse nicely:

\begin{eqnarray*}
 &=& P(N=n)
    + (n+1) P(x < \phi \mid \theta) P(N=n+1)
    + \frac{(n+1)(n+2)}{2!} P(x < \phi \mid \theta)^2 P(N=n+2) ...\\
 &= & \frac{1}{n} \left[
      1
      + n P(x < \phi \mid \theta)
      + \frac{n(n+1)}{2!} P(x < \phi \mid \theta)^2
      + \frac{n(n+1)(n+2)}{3!} P(x < \phi \mid \theta)^3 \right]\\
 &=& \frac{1}{n} (1 - P(x < \phi \mid \theta))^{-n}\\
 &=& \frac{1}{n} P(x \geq \phi \mid \theta)^{-n}\\
\end{eqnarray*}

The $\frac{1}{n}$ is a constant, so we drop it from the likelihood
equation we'll maximize. Putting this term back together with the
probability of the observed data and taking the log, we obtain the log
likelihood in equation (\ref{eqn:truncated_objective}).

Alternatively, we might choose an uninformative improper uniform prior
$P(N) \propto 1$. This gives a log likelihood that only differs by a
term of $n+1$ versus $n$:

\begin{equation}
  \hat{\theta} = \argmax_\theta -(n+1) \log P(x \geq \phi \mid \theta)
                   + \sum_{i=1}^n \log P(x_i \mid \theta).
\end{equation}

However, empirically, this form is ineffective, at least for fitting
Gumbels. The $\frac{1}{N}$ prior performs much better, probably
because it constrains the solutions to favor smaller, finite, more
reasonable choices of $N$.

\textbf{Back to fitting a truncated Gumbel}

For the specific case of fitting a truncated Gumbel, the log
likelihood (\ref{eqn:truncated_objective}) to optimize is:

\[
  \log L(\lambda, \mu) =
     n \log \lambda
     - \sum_{i=1}^{n} \lambda(x_i - \mu)
     - \sum_{i=1}^{n} e^{-\lambda(x_i - \mu)}
     - n \log (1 - \exp(-e^{-\lambda(\phi - \mu)}))
\label{eqn:truncated_logL}
\]

This is differentiable with respect to $\lambda$ and $\mu$, but it's
not going to reduce to the clean root-finding problem that we obtained
for complete data or censored data. Instead we're going to be left
with a numerical optimization problem. We can use standard numerical
optimization code, such as steepest descent or conjugate gradient
descent. There's just one hitch. These algorithms assume unconstrained
parameters, but $\lambda$ is constrained to values $>0$. We do a
change of variables, and use the transformation $\lambda = e^w$ so we
can optimize the unconstrained parameter $w = \log \lambda$ instead of
optimizing $\lambda$ directly.  The necessary partial derivatives are
then:

\begin{eqnarray}
\frac{\partial \log L}{\partial \mu} & = &
n \lambda
- \lambda \sum_{i=1}^{n} e^{-\lambda (x_i - \mu)}
- \frac{n \lambda \exp \left[ -\lambda (\phi - \mu) - e^{- \lambda (\phi - \mu)} \right]}
       {1 - \exp(-e^{-\lambda(\phi - \mu)})}
\label{eqn:truncated_dmu}
\\%
\frac{\partial \log L}{\partial w} & = &
n
- \sum_{i=1}^{n} \lambda(x_i - \mu)
+ \sum_{i=1}^{n} \lambda(x_i - \mu) e^{-\lambda (x_i - \mu)}
+ \frac{n\lambda (\phi-\mu) \exp \left[ -\lambda (\phi - \mu) - e^{- \lambda (\phi - \mu)} \right]}
       {1 - \exp(-e^{-\lambda(\phi - \mu)})}
\label{eqn:truncated_dw}
\end{eqnarray}

This optimization is carried out by \ccode{esl\_evd\_FitTruncated()}.
The likelihood (\ref{eqn:truncated_logL}) is implemented in
\ccode{tevd\_func()}, and the derivatives (\ref{eqn:truncated_dmu}) and
(\ref{eqn:truncated_dw}) are implemented in \ccode{tevd\_dfunc()}.
\ccode{esl\_evd\_FitTruncated()} simply sets up the problem and passes
it all off to a conjugate gradient descent optimizer.

Results on 500 simulated datasets with $\mu = -20, \lambda = 0.4$,
truncated at $\phi = -20$ (leaving the right tail, containing about
63\% of the samples):

\begin{center}
\begin{tabular}{lrrrr} \hline
                              & \multicolumn{4}{c}{\# samples}\\
                              & 100 & 1000  & 10,000 & 100,000 \\
\% error in $\hat{\mu}$       & 13\%&   2\% &  0.8\% &  0.3\%  \\
max error in $\hat{\mu}$      &260\%&  42\% &    3\% &    1\%  \\
\% error in $\hat{\lambda}$   & 15\%&   5\% &    2\% &  0.6\%  \\
max error in $\hat{\lambda}$  & 68\%&  18\% &    6\% &    2\%  \\ \hline
\end{tabular}
\end{center}

Fitting truncated Gumbel distributions is difficult, requiring much
more data than fitting complete or censored data. The problem is that
the right tail becomes a scale-free exponential when $\phi >> \mu$,
and $\mu$ becomes undetermined. Fits become very inaccurate as $\phi$
gets larger than $\mu$, and for sufficiently large $\phi$, the
numerical optimizer will completely fail.