\documentclass[12pt,notitlepage]{article}
\usepackage{hyperref}
\usepackage{amsmath,amsfonts,amssymb,amsthm}
% \usepackage{kbordermatrix}
% theorem preambles
\newtheorem{mydef}{Definition}
\usepackage{graphicx}			% allows us to import images
\usepackage{listings}
\lstset{
  breaklines=true
}
\usepackage{tikz}
\usepackage{verbatim}
\usepackage{cmds}
\hypersetup{colorlinks=true}
\usepackage[text={7in,10in}]{geometry}
% \setlength{\parindent}{0in}

% matrix columns
\newcommand\scalemath[2]{\scalebox{#1}{\mbox{\ensuremath{\displaystyle #2}}}}
\setcounter{MaxMatrixCols}{20}

\begin{document}
\title{Neural Network Notes}
\author{Kevin J. Marshall}
\maketitle

\section{Feedforward Neural Networks}
\label{sec:ff-=nn}
The basic computational units of a neural network (NN) are called
neurons.  Neurons are grouped together in finite sets called layers.
The structure of a neural network consists of an input layer followed
by a finite set of hidden computational layers and a final output
layer.  In feedforward NNs, information flows in a single direction from the
inputs to the outputs and the computation graph may always be
interpreted as a directed acyclic
graph (DAG).  Since the computational graph is a DAG, the neural
network may always be arranged such that each layer of the feedforward
NN only depends on  the layer to its left (the input side).  This can
be accomplished by performing a topological sort on the neurons
(nodes) of the DAG and
grouping them into layers $l_{k}$ with depth $k$ where $l_{k} \equiv \{n_{0},\dots,n_{k-1}\}$ is the
set of all neurons in layer $k$.  Upon arranging the $K$ layers from
$l_{0},l_{1},\dots,l_{K}$ each layer's neuron set $l_{k}$ will only depend
on neurons from the set of layers $\{l_{j<k}\}$.

\subsection{Matrix Representation}
\label{sec:matrix-rep}

During training, a set of $N$ vector valued input-output pairs $\{(\vx_{0},\vy_{0}), \dots,
(\vx_{N-1},\vy_{N-1})\}$ is fed into the network.  For each layer, $k$, the
following computation is carried out,
\begin{align}
  \label{eq:activation-component}
  o_{j}^{k} &= w_{ij}^{k}a_{i}^{k-1}\\
  \label{eq:output-component}
  a_{j}^{k} &= f( o_{j}^{k})
\end{align}
where we have used Einstein summation notation in
Eqns.~\ref{eq:activation-component} and \ref{eq:output-component} and
where the associated vector expressions are
\begin{align}
  \label{eq:activation-matrix}
  \vect{o}^{k} &= (\vect{W}^{k})^{T}\vect{a}^{k-1}\\
  \label{eq:output-matrix}
  \vect{a}^{k} &= f(\vect{o}^{k})
\end{align}
In these equations
\begin{itemize}
\item $o_{j}^{k}$ is the output computation of neuron $j$ in layer $k$
  and $\vect{o}^{k}$ is a vector with row dimension $|n_{k}+1|$ including a bias term.
\item $a_{j}^{k}$ is the nonlinear output or activation of neuron $j$ in layer $k$ after
  passing through the nonlinear function $f(\cdot)$
\item $\vect{W}^{k}$ is the weight matrix of layer $k$ such that
  elements $W_{ij}^{k}$ describe the weight associated with the $i$'th
  neuron input in layer $k-1$ to the $j$'th neuron in layer $k$.
  The dimensions of the weight matrix are $( d_{k-1} + 1 ) \times
  d_{k}$ where $d_{k-1} = |l_{k-1}|$ and $d_{k}
  = |l_{k}|$ are the number of neurons in layers $k-1$ and $k$ and
  $+1$ accounts for weights associated with the bias neuron.
\item $f(\cdot;j,k)$ is a nonlinear activation function $f: \mathbb{R}^{n_{k}} \to
  \mathbb{R}^{n_{k}}$ which may be different for each layer and each neuron.  Historically
  this function is taken to be the same for all neurons in all hidden
  layers,
  $f( \cdot; j,k ) = f( \cdot )$, and often takes the form of the
  logistic function
  \begin{equation}
    \label{eq:logistic-fun}
    f(x) = \frac{A}{1+\exp(-r(x-x_{0}))}
  \end{equation}
  where $A$ is the curve's maximum value, $x_{0}$ is the x-value of
  the functions midpoint, and $r$ is the logistic growth rate
  (steepness of the curve).  Standard practice uses $A=1, r=1, x_{0} =
  0$ which centers the function at $x=0$ and scales the input between
  to $(0,1)$ however the shape of this curve may adjusted in order to
  transform and/or scale layer outputs.
\end{itemize}
Once the information flow reaches the output layer, $l_{K}$, an error
may be computed for each input based on the nodal output values
$\hat{y}_{n}$ and the target values $y_{n}$.  Gradient decent
techniques attempt to minimize the error by adjusting the weights.
Weight update rules involve computing
\begin{equation}
  \label{eq:error-grad}
  \begin{split}
    \pderiv{E}{w^{k}_{ij}} &=
    \pderiv{E}{o_{l}^{k}}\pderiv{o_{l}^{k}}{w_{ij}^{k}}\\
    &= \pderiv{E}{o_{l}^{k}} \pderiv{}{w_{ij}^{k}}\left( w_{ml}^{k}o^{k-1}_{m} \right)\\
    &= \pderiv{E}{o_{l}^{k}}\delta_{mi}\delta_{jl}o_{m}^{k-1}\\
    &= \delta_{j}^{k}o_{i}^{k-1}
  \end{split}
\end{equation}

\subsection{Bias}
\label{sec:bias}
A bias term has been hidden in Eqn.~\ref{eq:activation-matrix} such
that $\vect{W}^{T}$ is given by
\begin{equation}
  \label{eq:wmat-explicit}
  (\vect{W}^{k})^{T}\vect{a}^{k-1} =
  \begin{bmatrix}
    w_{00} & w_{10} & \dots & w_{n_{k-1}0} & w_{b_{k-1}0} \\
    \vdots & \vdots & \vdots & \vdots & \vdots \\
    w_{0n_{k}} & w_{1n_{k}} & \dots & w_{n_{k-1}n_{k}} & w_{b_{k-1}n_{k}}
  \end{bmatrix}
  \begin{bmatrix}
    a_{0}^{k-1}\\
    \vdots\\
    a_{n_{k-1}}^{k-1}\\
    1
  \end{bmatrix}
\end{equation}
for $|l_{k}|=n_{k}$ and $|l_{k-1}|=n_{k-1}$ neurons in layers $k$ and $k-1$ and a bias
weight vector of $[w_{b_{k-1}0},\dots,w_{b_{k-1}n_{k}}]^{T}$.  In
general, layers contain a decreasing number of nodes as the depth
increases, i.e. $|l_{k}| \le |n_{k-1}|$.

\subsection{Activation Functions}
\label{sec:activation-fun}
Activation functions act on neuron layer outputs and may be interpreted as
individual layers themselves.  These functions introduce
non-linearities into the prediction model.  Several common activation
functions and their derivatives are given below.  In what follows we
assume $\vect{o}^{k}$ to be the input to the activation layer and
$\vect{a}^{k}$ to be the associated output.

\begin{itemize}
\item Identity:
  \begin{align}
    \label{eq:identity}
    a_{i}^{k} &= f(o_{i}^{k}) = o_{i}^{k}\\
    \pderiv{a_{i}^{k}}{o_{i}^{k}} &= 1
  \end{align}
\item Logistic:
  \begin{align}
    \label{eq:logistic-fun}
    a_{i}^{k} &= f(o_{i}^{k}) = \frac{1}{1+\exp(-o_{i}^{k})}\\
    \pderiv{a_{i}^{k}}{o_{i}^{k}} &=
                                    \frac{\exp(-o_{i}^{k})}{(1+\exp(-o_{i}^{k}))^{2}}
                                    = f(o_{i}^{k})(1-f(o_{i}^{k}))
  \end{align}
\item TanH:
  \begin{align}
    \label{eq:tanh-fun}
    a_{i}^{k} &= f(o_{i}^{k}) = \tanh(o_{i}^{k})\\
    \pderiv{a_{i}^{k}}{o_{i}^{k}} &= \sech^{2}(o_{i}^{k}) = 1 - f^{2}(o_{i}^{k})
  \end{align}
\item arcTan:
  \begin{align}
    \label{eq:arctan-fun}
    a_{i}^{k} &= f(o_{i}^{k}) = \arctan(o_{i}^{k})\\
    \pderiv{a_{i}^{k}}{o_{i}^{k}} &= \frac{1}{(o_{i}^{k})^{2}+1}
  \end{align}
\item ReLU (rectified linear unit)
  \begin{align}
    \label{eq:relu-fun}
    a_{i}^{k} &= f(o_{i}^{k}) =
                \begin{cases}
                  o_{i}^{k}, \qquad & o_{i}^{k} \ge 0\\
                  0, \qquad & o_{i}^{k} < 0
                \end{cases}\\
    \pderiv{a_{i}^{k}}{o_{i}^{k}} &=
                                    \begin{cases}
                                      1, \qquad & f(o_{i}^{k}) \ge 0\\
                                      0, \qquad & f(o_{i}^{k}) < 0
                                    \end{cases}
  \end{align}
\item PLU (parametric linear unit):
  \begin{align}
    \label{eq:plu-fun}
    a_{i}^{k} &= f(o_{i}^{k}) =
                \begin{cases}
                  \alpha o_{i}^{k}, \qquad & o_{i}^{k} \ge 0\\
                  0, \qquad & o_{i}^{k} < 0
                \end{cases}\\
    \pderiv{a_{i}^{k}}{o_{i}^{k}} &=
                                    \begin{cases}
                                      \alpha, \qquad & f(o_{i}^{k}) \ge 0\\
                                      0, \qquad & f(o_{i}^{k}) < 0
                                    \end{cases}
  \end{align}
\item ELU (exponential linear unit):
  \begin{align}
    \label{eq:elu-fun}
    a_{i}^{k} &= f(o_{i}^{k}) =
                \begin{cases}
                  \alpha ( \exp(o_{i}^{k}) - 1 ), \qquad & o_{i}^{k} \ge 0\\
                  0, \qquad & o_{i}^{k} < 0
                \end{cases}\\
    \pderiv{a_{i}^{k}}{o_{i}^{k}} &=
                                    \begin{cases}
                                      f(o_{i}^{k}) + \alpha, \qquad & f(o_{i}^{k}) \ge 0\\
                                      0, \qquad & f(o_{i}^{k}) < 0
                                    \end{cases}
  \end{align}
\item SoftMax:
  \begin{align}
    \label{eq:relu-fun}
    a_{i}^{k} &= f(o_{i}^{k}) = \frac{\exp(o_{i}^{k})}{\sum_{j}\exp(o_{j}^{k})}\\
    \pderiv{a_{i}^{k}}{o_{j}^{k}} &= f(o_{i}^{k})\delta_{ij} - f(o_{i}^{k})f(o_{j}^{k})
  \end{align}
\item LogSoftMax:
  \begin{align}
    \label{eq:relu-fun}
    a_{i}^{k} &= f(o_{i}^{k}) = o_{i}^{k} - \log\left(
                \sum_{j}\exp(o_{j}^{k}) \right)\\
    \pderiv{a_{i}^{k}}{o_{j}^{k}} &= \delta_{ij} - \exp_{i}(f(o_{j}^{k}))
  \end{align}
\end{itemize}

\subsection{Loss Functions}
\label{sec:loss-fun}

\subsubsection{Mean Squared Error}
\label{sec:error-mse}
The mean squared error is a good classifier for the numeric output in
which case weight updates are computed by minimizing the error
\begin{equation}
  \label{eq:error}
  \begin{split}
    E &= \frac{1}{2N}\sum_{n}(\hat{\vy}^{n} - \vy^{n} )^{2}\\
    &= \frac{1}{2N}\sum_{n}(f(\vect{a}^{K};K)^{n} - \vy^{n} )^{2}\\
    &= \frac{1}{N}\sum_{n}E_{n}\\
  \end{split}
\end{equation}
where $\hat{\vy}^{n}$ and $\vy^{n}$ are respectively the
output (prediction) vector and target vector of the $n$'th input
$(\vx^{n},\vy^{n})$.  For the output layer we find that
Eqn.~\ref{eq:error-grad} becomes
\begin{equation}
  \label{eq:error-grad-output}
  \begin{split}
    \pderiv{E_{n}}{w^{K}_{ij}} &=
    \pderiv{E_{n}}{\hat{y}_{l}^{K}}\pderiv{\hat{y}_{l}^{K}}{o_{m}^{K}}\pderiv{\hat{o_{m}^{K}}}{w_{ij}^{K}}\\
    &= (\hat{y}_{l}^{K} - y^{p}_{l} )f'(o_{l}^{K})\delta_{ml}\delta_{in}\delta_{jm}a_{n}^{K-1}\\
    &= (\hat{y}_{j}^{K} - y^{p}_{j} )f'(o_{j}^{K})a_{i}^{K-1}
  \end{split}
\end{equation}
We note that equation \ref{eq:error-grad-output} actually describes
three layer operations which take place sequentially.  These
operations may be described from right to left in accordance with back
propagation as
\begin{enumerate}
\item Error propagation over the loss function layer,
  \begin{equation}
    \label{eq:error-loss-fcn}
    \pderiv{E^{p}}{\hat{y}_{l}^{p}} = (\hat{y}_{l}^{K} - y^{p}_{l})
  \end{equation}
\item Error propagation over the activation layer
  \begin{equation}
    \label{eq:error-act-fun}
    \pderiv{E^{p}}{a_{m}^{K}} =
    \pderiv{E^{p}}{\hat{y}_{l}^{p}}f'(o_{l}^{K})\delta_{ml} = \pderiv{E^{p}}{\hat{y}_{m}^{p}}f'(o_{m}^{K})
  \end{equation}
\item Error propagation over the last layer $K$ to find gradient updates
  \begin{equation}
    \label{eq:error-last-layer}
    \pderiv{E^{p}}{w_{ij}^{K}} =
    \pderiv{E^{p}}{o_{m}^{K}}\delta_{in}\delta_{jm}a_{n}^{K-1} = \pderiv{E^{p}}{o_{j}^{K}}a_{i}^{K-1}
  \end{equation}
\end{enumerate}
The key here is to notice that back propagation inputs are error
gradients computed on the forward pass inputs.  Gradient update rules for
weight matrices are obtained using stochastic variations on Euler's method.
\subsection{Hidden Layers}
\label{sec:hidden-layers}

For hidden layers $l_{k}$, $0 \le k < K$, the calculation becomes,
\begin{equation}
  \label{eq:error-grad-hidden}
  \begin{split}
    \pderiv{E_{n}}{w^{k}_{ij}} &=
    \pderiv{E_{n}}{o_{l}^{k}}\pderiv{o_{l}^{k}}{w_{ij}^{k}}\\
    &=
    \pderiv{E_{n}}{o_{m}^{k+1}}\pderiv{o_{m}^{k+1}}{o_{l}^{k}}\pderiv{o_{l}^{k}}{w_{ij}^{k}}\\
    &=
    \pderiv{E_{n}}{o_{m}^{k+1}}\pderiv{o_{m}^{k+1}}{o_{l}^{k}}\delta_{jl}a_{i}^{k-1}\\
    &=
    \pderiv{E_{n}}{o_{m}^{k+1}}\pderiv{o_{m}^{k+1}}{o_{j}^{k}}a_{i}^{k-1}\\
    &=
    \pderiv{E_{n}}{o_{m}^{k+1}}\pderiv{o_{m}^{k+1}}{a_{p}^{k}}\pderiv{a_{p}^{k}}{o_{j}^{k}}o_{i}^{k-1}\\
    &=
    \pderiv{E_{n}}{o_{m}^{k+1}}w_{pm}^{k+1}f'(o_{p}^{k})\delta_{pj}o_{i}^{k-1}\\
    &= \delta^{k+1}_{m}w_{jm}^{k+1}f'(a_{j}^{k})o_{i}^{k-1}\\
  \end{split}
\end{equation}

\end{document}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: t
%%% End:


\documentclass[letterpaper]{article}

\usepackage{fullpage}
\usepackage{nopageno}
\usepackage{amsmath}
\usepackage{amssymb}
\allowdisplaybreaks

\newcommand{\abs}[1]{\left\lvert #1 \right\rvert}

\begin{document}
\title{Notes}
\date{August 29, 2014}
\maketitle

last time assigned gcd stuff and assigned to read results

\section*{gcd definition}
$a,b$ not both 0, notation $\text{gcd}(a,b)=(a,b)$

\subsection*{facts}
\begin{enumerate}
\item
gcd exists and  is unique

follows from assigned theorems.

\subsubsection*{example}
$\text{gcd}(6,14)=2$
\item
the gcd of $a$ and $b$ is a linear combination of $a$ and $b$. ie, there exist $m,n\in\mathbb{Z}$ such that $(a,b)=ma+nb$

in fact $\text{gcd}(a.b)$ is the smallest positive inegerthat is a linear combination of $a$ and $b$
\begin{align*}
  \{ma+nb\mid m,n\in\mathbb{Z}\}
\end{align*}
\end{enumerate}

\subsection*{euclidean algorithm}
\begin{align*}
  (a,b)&=(\abs{a},\abs{b})
\end{align*}
we may assume that $a\ge b\ge 0$. $a=bq_1+r_1$.

\subsubsection*{claim}
\begin{align*}
  (a,b)&=(b,r_1)\\
  d_1=(a,b),&\quad d_2=(b,r_1)\\
  d_1\vert a&\rightarrow d_1\vert(bq_1+r_1)\\
  d_1\vert b&\rightarrow d_1\vert r_1\\
  &\rightarrow d_1\vert d2 \text{ because } d_2=(b,r_1)
\end{align*}
similarly show that $d2\vert d1$ hence d1=d2

now we see
\begin{align*}
  a&=bq_1+r_1\\
  b&=rq_2+r_2\\
  r_1&=r_2q3+r_3\\
  &\vdots\\
  r_n&=\text{zero remainder because remainders are shrinking}
\end{align*}
so $(a,b)=(r_{n-1},0)=r_{n-1}$
\subsection*{example}
find $(33,9)$
\begin{align*}
  33&=9*3+6\\
  9&=6*1+3\\
  6&=3*2+0\\
  (33,9)&=3
  3&=9-1*6\\
  &=9-1*(33-3*9)\\
  &=9-1*33+3*9\\
  &=4*9+(-1)*33
\end{align*}
can also use euclidean algorithm to generate linear combination from gcd
\section*{1.2 prime numbers}
\subsection*{definition}
$p>1$ is prime if the only positive divisors of $p$ are $1$ and $p$

$p>1$ is prime if the only divisors of $p$ are $\pm1$ and $p$

\subsection*{definition}
we say that $a$ and $b$ are relatively prime if $\text{gcd}(a,b)=1$
\subsection*{proposition}
let $p>1,p\in\mathbb{Z}$ then $p$ is prime iff the following property holds:

$a,b\in\mathbb{Z}$ and $p\vert ab$ then $p\vert a$ or $p\vert b$

only true if p is prime, ${4}{\not\vert}{6\cdot6}$
\subsubsection*{proof}
assume $p$ is prime.  assume $p\vert ab$, then $(p,a)=1$ or $(p,a)=p$. this is because the only divisor of $p$ is $p$ or 1.

case 1, $(p,a)=p$. then $p|a$ and we are done.

case 2, $(p,a)=1$. then there exists $m,n\in\mathbb{Z}$ such that $mp+na=1$.
\begin{align*}
  mp+na&=1\\
  bmp+bna&=b\\
  p|ab&\rightarrow p|ab
  p|bmp
\end{align*}
since $p|bmp$ and $p|bna$ therefore $p|b$

conversely

assume $\alpha|p$ with $\alpha>0$. Nee to prove that $\alpha=1$ or $\alpha=p$

$\alpha|p\rightarrow p=\alpha\cdot\beta$ with  $\beta\in\mathbb{Z}$

by the property satisfied $p|\alpha or p|\beta$. if $p|\alpha$ since $\alpha|p$ we have $\alpha=p$. if $p|\beta$, since $\beta|p$ we have $\beta=p$

if $\beta=p$ then $\alpha=1$
\end{document}


\documentclass{article}
\usepackage[margin=2cm]{geometry}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{hyperref}
\title{Flexible Snow Model (FSM2) scientific documentation}
\author{Richard Essery}
\date{2 January 2019}
\begin{document}
\maketitle

The Flexible Snow Model allows alternative process parametrizations to be combined in a complete model of the mass and energy balances of snow on the ground and in forest canopies. Existing models from which parametrizations are taken for inclusion in FSM2 include
\href{https://www.tandfonline.com/doi/abs/10.3137/ao.440301}{CLASS},
\href{http://www.cesm.ucar.edu/models/cesm1.2/clm/CLM45_Tech_Note.pdf}{CLM},
\href{https://www.geosci-model-dev.net/5/773/2012/}{Crocus},
\href{https://www.umr-cnrm.fr/IMG/pdf/snowdoc_v2.pdf}{ISBA} and
\href{https://link.springer.com/article/10.1007/s003820050276}{MOSES}.
Parametrizations are selected by setting option numbers in a text file before the model is compiled. Parameter values and input data are read from text files when the model is run. Physical constants, meteorological driving variables, site characteristics, model state variables and parameters are listed in tables 1 to 5; refer to these tables for any variables that are not explicitly defined in the text.

\section{Snow-free and snow-covered ground albedos}

Bare ground with albedo $\alpha_0$ and snow cover fraction $f_s$ with albedo $\alpha_s$ have average albedo
\begin{equation}
\alpha_g = (1 - f_s)\alpha_0 +f_s\alpha_s.
\end{equation}
The snow cover is a function of snow depth $h$ parametrized as
\begin{equation}
f_s(h) = \frac{h}{h + h_f}
\end{equation}
if option {\tt SNFRAC=0} is set in the compilation file, and
\begin{equation}
f_s(h) = \tanh\left(\frac{h}{h_f}\right)
\end{equation}
if {\tt SNFRAC=1}.

\subsection{Diagnosed snow albedo (option {\tt ALBEDO=0})}

Following a common approach in earlier generations of climate models, snow albedo is diagnosed as a function of surface temperature
\begin{equation}
\alpha_s(T_*) = \alpha_{\rm min} + (\alpha_{\rm max} - \alpha_{\rm min})
                \min\left(\frac{T_* - T_m}{T_\alpha}, 1\right).
\end{equation}

\subsection{Prognostic snow albedo (option {\tt ALBEDO=1})}

Based on CLASS and ISBA, decreasing albedo as snow ages with time $t$ and increasing albedo as fresh snow falls at rate $S_f$ are parametrized by
\begin{equation}
\frac{d\alpha_s}{dt} = \frac{1}{\tau_\alpha}(\alpha_{\rm min} - \alpha_s) + \frac{S_f}{S_\alpha}(\alpha_{\rm max} - \alpha_s),
\label{eq:albedo}
\end{equation}
where the timescale $\tau_\alpha$ has different values $\tau_{\rm cold}$ and $\tau_{\rm melt}$ for cold and melting snow. Equation (\ref{eq:albedo}) is implemented by integrating over a timestep of length $\delta t$ to give the change in snow albedo as
\begin{equation}
\delta\alpha_s = (\alpha_{\rm lim} - \alpha_s)(1-e^{-\gamma\delta t}),
\end{equation}
where
\begin{equation}
\gamma = \frac{1}{\tau_\alpha}+\frac{S_f}{S_\alpha}
\end{equation}
and
\begin{equation}
\alpha_{\rm lim} = \frac{1}{\gamma}\left(\frac{1}{\tau_\alpha}\alpha_{\rm min} +\frac{S_f}{S_\alpha}\alpha_{\rm max}\right).
\end{equation}

\section{Snow density}

\subsection{Compaction}
Snow may be assumed to have constant density $\rho_0$ (option {\tt DENSTY=0}) or to compact at rate
\begin{equation}
\frac{d\rho_s}{dt} = f_\rho(\rho_s,T_s).
\label{eq:drhodt}
\end{equation}
Snow density $\rho_s$ in layer $k$ with ice mass $I$ and liquid water mass $W$ at the beginning of timestep $n$ is diagnosed as
\begin{equation}
\rho_{s,k}^{(n)} = \frac{I_k^{(n)} + W_k^{(n)}}{D_{s,k}}.
\label{eq:rhok}
\end{equation}
Equation (\ref{eq:drhodt}) is then implemented as
\begin{equation}
\rho_{s,k}^{(n+1)} = \rho_{s,k}^{(n)} + f_\rho\left(\rho_{s,k}^{(n)},T_{s,k}^{(n)}\right)\delta t.
\end{equation}
Finally, the thickness of the compacted layer at the end of the timestep is calculated by inverting Equation (\ref{eq:rhok}).

\subsubsection{Empirical maximum densities (option {\tt DENSTY=1})}

Based on CLASS, the compaction rate function is
\begin{equation}
f_\rho = \frac{1}{\tau_\rho}(\rho_{\max} - \rho_s)
\end{equation}
with the same time constant $\tau_\rho$ but different asymptotic values $\rho_{\rm cold}$ and $\rho_{\rm melt}$ for $\rho_{\max}$ in cold and melting snow.

\subsubsection{Viscous compaction by overburden and thermal metamorphism (option {\tt DENSTY=2})}

Following ISBA, the compaction rate function is
\begin{equation}
f_\rho = \rho_s\left\{\frac{gm}{\eta} + c_1 \exp\left[\frac{(T_s - T_m)}{23.8}
                - \max\left(\frac{\rho_s - 150}{21.7}, 0\right)\right]\right\}
\end{equation}
where
\begin{equation}
\eta = \eta_0\exp\left[-\frac{(T_s - T_m)}{12.4} + \frac{\rho_s}{55.6}\right].
\end{equation}
and the snow mass overlying the middle of a layer is
\begin{equation}
m_k = \sum_{j=1}^{k-1}\left[I_j^{(n)} + W_j^{(n)}\right] + 0.5\left[I_k^{(n)} + W_k^{(n)}\right].
\end{equation}

\subsection{Fresh snow density}

Fresh snow deposited with density $\rho_f$ over a timestep increases the snow depth before compaction by $\rho_f^{-1}S_f\delta t$. Unless snow density is fixed, the density of fresh snow is taken from
\begin{equation}
\rho_f = \max[\rho_0 + b_\rho (T_a - T_m) + c_\rho U_a^{1/2}, 50].
\end{equation}
Parameter values can be selected to match CLM ($\rho_0=100$ kg m$^{-3}$) or Crocus $(\rho_0=109$ kg m$^{-3}$, $a_\rho=6$ kg m$^{-3}$ K$^{-1}$ and $b_\rho=26$ kg m$^{-7/2}$ s$^{1/2}$). Snow unloading from a forest canopy is added to snow on the ground with the bulk density of the existing snow.

\section{Canopy radiative transfer}

Forest structure is defined by canopy height $h$, vegetation area index $\Lambda$ (including leaves and stems) and vegetation fraction $f_v$, which is either parametrized as $1-\exp(-k_{\rm veg}\Lambda)$ or specified as an input. The fraction of radiation incident from above at elevation angle $\theta$ that is transmitted through the canopy without interception is parametrized as
\begin{equation}
\tau_{\rm dir} = \exp[-(G_1 + G_2\sin\theta)\Lambda/\sin\theta].
\end{equation}
The canopy transmissivity for diffuse radiation is parametrized as
\begin{equation}
\tau_{\rm dif} = \exp(-k_{\rm dif}\Lambda)
\label{eq:fsky}
\end{equation}
by default but can also be specified as a sky view fraction $f_{\rm sky}$ for sites which are not directly under trees ($\Lambda=0$) but which are shaded by surrounding vegetation or topography ($f_{\rm sky}<1$). In principle, the parameters $G_1$, $G_2$ and $k_{\rm dif}$ are not independent.

\subsection{Shortwave radiation}

A forest canopy with intercepted snow mass $S_v$ and interception capacity $S_{\rm cap}$ is assumed to have snow cover fraction
\begin{equation}
f_{cs} = \frac{S_v}{S_{\rm cap}}
\end{equation}
and albedo
\begin{equation}
\alpha_c = f_v[(1 - f_{cs})\alpha_{c0} + f_{cs}\alpha_{cs}].
\end{equation}
As illustrated in Figure \ref{fig:SWcan}, the canopy is assumed to reflect fraction $\alpha_c$ of incoming shortwave radiation, but radiation reflected from the ground is transmitted and absorbed by the canopy without further reflections. For incoming shortwave radiation made up of a diffuse component $S_{\rm dif}$ and a direct-beam component $S_{\rm dir}$, the net absorption is
\begin{align}
SW_v = & (1 - \alpha_c)[1 - (1 - \alpha_g)\tau_{\rm dif}
                          - \alpha_g\tau_{\rm dif}\tau_{\rm dif}]S_{\rm dif} + \\
       & (1 - \alpha_c)[1 - (1 - \alpha_g)\tau_{\rm dir}
                          -  \alpha_g\tau_{\rm dif}\tau_{\rm dir}]S_{\rm dir}
\label{eq:SWv}
\end{align}
by the forest canopy and
\begin{equation}
SW_g = (1 - \alpha_c)(1 - \alpha_g)(\tau_{\rm dif}S_{\rm dif} + \tau_{\rm dir}S_{\rm dir}).
\label{eq:SWg}
\end{equation}
by the ground or snow surface. The effective above-canopy albedo, including shortwave radiation reflected from the canopy and the ground, is
\begin{equation}
\alpha = \alpha_c + (1 - \alpha_c)\alpha_g\tau_{\rm dif}\left(
         \frac{\tau_{\rm dif}S_{\rm dif} + \tau_{\rm dir}S_{\rm dir}}
              {S_{\rm dif} + S_{\rm dir}}\right).
\end{equation}

\begin{figure}[t]
\includegraphics[width=6cm]{SWcan.png}
\caption{Reflection and transmission of diffuse and direct-beam shortwave radiation by a forest canopy and diffuse reflection by the ground surface.}
\label{fig:SWcan}
\end{figure}

Usually, only measurements of global shortwave radiation are available. Option {\tt SWPART=0} treats all of the global radiation as diffuse and option {\tt SWPART=1} partitions global shortwave radiation into direct and diffuse components as described in Appendix A. If separate measurements of direct and diffuse components are available, they can be read as inputs using option {\tt SWPART=2}. {\it Two-stream radiative transfer to be added as an option}

\subsection{Longwave radiation}
The net absorption of longwave radiation is
\begin{equation}
LW_v = (1 - f_{\rm sky}) (LW_\downarrow  + \sigma T_*^4 - 2\sigma T_v^4)
\end{equation}
by the forest canopy and
\begin{equation}
LW_g = f_{\rm sky}LW_\downarrow - \sigma T_*^4  + (1 - f_{\rm sky})\sigma T_v^4
\label{eq:LWg}
\end{equation}
by the ground or snow surface. Upwelling longwave radiation above a forest canopy is
\begin{equation}
LW_\uparrow = f_{\rm sky}\sigma T_*^4  + (1 - f_{\rm sky})\sigma T_v^4.
\end{equation}

\section{Turbulent fluxes}

\subsection{Roughness lengths and aerodynamic resistances}

Roughness lengths for snow-free ground and snow on fraction $f_s$ of the ground are combined to give a ground roughness length
\begin{equation}
z_{0g} =  z_{0f}^{1-f_s} z_{0s}^{f_s}.
\end{equation}
For vegetation of height $h$ covering fraction $f_v$ of the ground, the roughness length and displacement height are $z_{0v} = R_{z0}h$ and $d = f_vR_dh$. The combined surface roughness length is
\begin{equation}
z_0 = z_{0g}^{1-f_v} z_{0v}^{f_v}.
\end{equation}

The aerodynamic resistance network for turbulent exchanges of heat between the ground, vegetation and the atmosphere is shown in Figure \ref{fig:rescan}. The resistance between the canopy air space and the atmosphere is
\begin{equation}
r_{aa} = \frac{1}{f_H k u_*}\ln\left(\frac{z_T-d}{z_0}\right)
\label{eq:raa}
\end{equation}
with atmospheric stability factor $f_H$ and friction velocity
\begin{equation}
u_* = kU_a\left[\ln\left(\frac{z_U-d}{z_0}\right)\right]^{-1}.
\end{equation}
Vegetated and unvegetated fractions of the ground share a common surface temperature but have separate aerodynamic resistances that combine in parallel to give resistance
\begin{equation}
r_{ag} = \frac{1}{ku_*}\left[\frac{(1-f_v)f_h}{\ln(z_{0g}/z_{0h})} +
                             f_vf_cC_{\rm dense}\right]^{-1}
\label{eq:rag}
\end{equation}
between the ground and the canopy air space, with sub-canopy stability factor $f_c$. The roughness length for heat $z_{0h}$ is assumed to be equal to $0.1z_{0g}$. Between vegetation and the canopy air space, the aerodynamic resistance is
\begin{equation}
r_{av} = \frac{C_{\rm veg}}{\Lambda u_*^{1/2}}.
\end{equation}
$C_{\rm dense}$, $C_{\rm veg}$ are constant parameters. In the absence of vegetation ($\Lambda=f_v=0)$, resistances for heat transfer combine to give
\begin{equation}
r_h \equiv r_{aa} + r_{ag} = \frac{1}{f_hk^2U_a}\ln\left(\frac{z_U}{z_{0g}}\right)
                                                \ln\left(\frac{z_T}{z_{0h}}\right).
\label{eq:rh}
\end{equation}

\subsubsection{Neutral stability (option {\tt EXCHNG=0})}

Atmospheric stability is neglected by setting $f_H=1$.
%and $\psi_m=\psi_H=0$.

\subsubsection{Richardson number stability functions (option {\tt EXCHNG=1})}

Atmospheric stability is characterized by a bulk Richardson number
\begin{equation}
{\rm Ri_B} = \frac{g(z_U-d)^2[T_a - f_vT_v - (1-f_v)T_*]}{(z_T-d) T_a U_a^2}.
\end{equation}
The aerodynamic resistances in Equation (\ref{eq:raa}) and (\ref{eq:rag}) are adjusted by factor
\begin{equation}
f_H  = \begin{cases}
       [1 + 3b_h{\rm Ri_B}(1 + b_h{\rm Ri_B})^{1/2}]^{-1}  & {\rm Ri_B} \geq 0 \\
        1 - 3b_h{\rm Ri_B}[1 + c(-{\rm Ri_B})^{1/2}]^{-1}  & {\rm Ri_B} < 0
\end{cases}
\end{equation}
with
\begin{equation}
c = 3b_h^2k^2\left(\frac{z_U}{z_0}\right)^{1/2}\left[\ln\left(\frac{z_U}{z_0}\right)\right]^{-2}.
\end{equation}
The sub-canopy stability factor is
\begin{equation}
f_c = \frac{1}{1 + {\rm Ri}_c/2}
\end{equation}
with Richardson number
\begin{equation}
{\rm Ri}_c = \frac{gh(T_c - T_*)}{T_c u_*^2}
\end{equation}
limited to values in the range 0 - 10.

\subsubsection{Obukhov length stability functions ({\it option to be added})}

\begin{figure}[t]
\includegraphics[width=8cm]{rescan.png}
\caption{Aerodynamic resistance network for heat exchanges between the ground, vegetation and the atmosphere.}
\label{fig:rescan}
\end{figure}

\subsection{Snow on short vegetation and bare ground}

Sensible heat and moisture fluxes from the ground or snow surface without exposed vegetation to the atmosphere are
\begin{equation}
H_g = \frac{\rho c_p}{r_h}(T_* - T_a)
\label{eq:Hg}
\end{equation}
and
\begin{equation}
E_g = \frac{\rho\psi_g}{r_h}[Q_{\rm sat}(T_*,P_s) - Q_a]
\label{eq:Eg}
\end{equation}
where $Q_{\rm sat}$ is the specific humidity at saturation over water or ice (Appendix B) and $\rho=P_s/(R_{\rm air}T_a)$ is the air density. The moisture availability factor $\psi_g$ is set to 1 if there is snow on the ground or if $Q_{\rm sat}(T_*,P_s)<Q_a$ (moisture flux onto the surface) and
\begin{equation}
\psi_g = \frac{r_h}{r_h+r_{sg}}
\label{eq:psig}
\end{equation}
otherwise, where $r_{sg}$ is a resistance for evaporation from soil moisture. Without exposed vegetation, moisture and sensible heat fluxes to the atmosphere are $E=E_g$ and $H=H_g$. The latent heat flux is $LE$, with $L$ taken to be the latent heat of vapourisation if $T_*>T_m$ and the latent heat of sublimation otherwise.

\subsection{Forest canopies and underlying ground}

Sensible heat fluxes are
\begin{equation}
H = \frac{\rho c_p}{r_{aa}}(T_c - T_a)
\label{eq:H}
\end{equation}
upwards from the canopy air space,
\begin{equation}
H_g = \frac{\rho c_p}{r_{ag}}(T_* - T_c)
\end{equation}
from the ground to the canopy air space, and
\begin{equation}
H_v = \frac{\rho c_p}{r_{av}}(T_v - T_c)
\label{eq:Hv}
\end{equation}
from the vegetation to the canopy air space. Similarly, moisture fluxes are
\begin{equation}
E = \frac{\rho}{r_{aa}}(Q_c - Q_a)
\end{equation}
upwards from the canopy air space,
\begin{equation}
E_g = \frac{\rho \psi_g}{r_{ag}}[Q_{\rm sat}(T_*,P_s) - Q_c]
\end{equation}
from the ground to the canopy air space, and
\begin{equation}
E_v = \frac{\rho \psi_v}{r_{av}}[Q_{\rm sat}(T_v,P_s) - Q_c]
\label{eq:Ev}
\end{equation}
from the vegetation to the canopy air space, with moisture availability factor $\psi_v=1$ if there is snow on the canopy or if $Q_{\rm sat}(T_v)<Q_c$ (moisture flux onto the canopy) and
\begin{equation}
\psi_v = \frac{r_{av}}{r_{av}+r_{sg}}
\end{equation}
otherwise.

\section{Energy balance}

\subsection{Snow on short vegetation and bare ground}

The surface energy balance
\begin{equation}
f(T_*) = LW_g + SW_g - G - H_g - LE_g - L_fM = 0
\label{eq:ebal}
\end{equation}
with flux parametrizations substituted from Equations (\ref{eq:LWg}), (\ref{eq:Hg}) and (\ref{eq:Eg}) is a nonlinear equation for the unknown surface temperature and snowmelt rate $M$. From an initial guess of temperature $T_{*0}$ and no melt, a linear estimate of $T_*$ is given by
\begin{equation}
T_* = T_{*0} - f(T_{*0})\left(\frac{df}{dT_*}\right)^{-1}.
\label{eq:Newton}
\end{equation}
A single application of Equation (\ref{eq:Newton}) gives an approximate solution, and repeated applications with $T_*$ calculated in each iteration being used as $T_{*0}$ in the next is the Newton-Raphson method for solving Equation (\ref{eq:ebal}). Neglecting the complicated temperature dependence of $r_{aa}$, this gives
\begin{equation}
T_* = T_{*0} + \frac{LW_g + SW_g - G - H_g - LE_g - L_fM}
                    {4\sigma T_{*0}^3 + 2\lambda_1/\Delta h_1
                                      + \rho(c_p + LD_g\psi_g)/r_h}.
\label{eq:Tsurf}
\end{equation}
where the fluxes in the numerator are calculated using $T_* = T_{*0}$, and
\begin{equation}
D_g = \left.\frac{dQ_{\rm sat}}{dT}\right\rvert_{T=T_{*0}}
    = \frac{LQ_{\rm sat}(T_{*0})}{R_{\rm wat}T_{*0}^2}.
\end{equation}
Equation (\ref{eq:Tsurf}) is first evaluated with $M=0$. If this gives $T_* > T_m$ and there is snow with ice mass $I$ on the ground, Equation (\ref{eq:Tsurf}) is re-evaluated assuming that all of the snow melts and $M = I / \delta t$. If this gives $T_* < T_m$, the snow does not all melt and $T_* = T_m$ is known; Equation (\ref{eq:ebal}) is solved instead for the unknown melt rate by substitution of $T_* = T_m$ in the equations for the other fluxes.

\subsection{Forest canopies and underlying ground}

\subsubsection{Zero-layer canopy model (option {\tt CANMOD=0})}

The zero-layer canopy model does not attempt to calculate the canopy energy balance, instead assuming that the canopy temperature is equal to the air temperature. Substituting $T_v = T_a$ in Equations (\ref{eq:H}) to (\ref{eq:Ev}) and rearranging gives
\begin{equation}
H_g = \frac{\rho c_p}{r_h}(T_* - T_a)
\end{equation}
with
\begin{equation}
r_h = r_{ag} + \left(\frac{1}{r_{aa}}+\frac{1}{r_{av}}\right)^{-1}.
\label{eq:modrh}
\end{equation}
and
\begin{equation}
E_g = \frac{\rho\psi_g}{r_h}(T_* - T_a)
\end{equation}
with
\begin{equation}
\frac{r_h}{\psi_g} = r_{ag} + r_{sg} + \left(\frac{1}{r_{aa}}+\frac{\psi_v}{r_{av}}\right)^{-1}.
\label{eq:modpsi}
\end{equation}
Surface temperature and melt rate are found by solving Equations (\ref{eq:Tsurf}) and (\ref{eq:ebal}) with the modified aerodynamic resistance and moisture factor from Equations (\ref{eq:modrh}) and (\ref{eq:modpsi}) instead of Equations (\ref{eq:rh}) and (\ref{eq:psig}). The rate of moisture transfer from or to the canopy is
\begin{equation}
E_v = -\left(\frac{\psi_v r_{aa}}{\psi_v r_{aa} + r_{av}}\right)E_g.
\end{equation}

\subsubsection{One-layer canopy model (option {\tt CANMOD=1})}

Energy and mass conservation equations
\begin{equation}
f_1 = (H - H_g - H_v)/(\rho c_p) = 0,
\end{equation}
\begin{equation}
f_2 = (E - E_g - E_v)/\rho = 0,
\end{equation}
\begin{equation}
f_3 = LW_g + SW_g - G - H_g - LE_g = 0,
\label{eq:f3}
\end{equation}
and
\begin{equation}
f_4 = LW_v + SW_v - H_v - LE_v - C_{\rm can}\frac{dT_v}{dt} = 0,
\end{equation}
form a set of four nonlinear equations with four unknowns: $Q_c$, $T_c$, $T_*$ and $T_v$.
Writing vectors ${\bf f} = (f_1, f_2, f_3, f_4)^T$ and ${\bf x} = (Q_c, T_c, T_*, T_v)^T$, a solution is found by iterating
\begin{equation}
{\bf x} = {\bf x}_0 - {\rm J}^{-1}{\bf f}({\bf x}_0)
\label{eq:multiNR}
\end{equation}
where ${\rm J}$ is the Jacobian matrix of {\bf f} with elements
\begin{equation}
{\rm J}_{ij} = \frac{\partial f_i}{\partial x_j}
\end{equation}
or
\begin{equation}
{\rm J} =
\begin{bmatrix}
0 & -\left(r_{aa}^{-1} + r_{ag}^{-1} + r_{av}^{-1} \right) & r_{ag}^{-1} & r_{av}^{-1} \\
-\left(r_{aa}^{-1} + \psi_g r_{ag}^{-1} + \psi_v r_{av}^{-1}\right) & 0
& \psi_g D_g r_{ag}^{-1} & \psi_v D_v r_{av}^{-1} \\
-L\rho\psi_g r_{ag}^{-1} & -\rho c_p r_{ag}^{-1} & J_{33} & -4f_v\sigma T_v^3\\
-L\rho\psi_v r_{av}^{-1} & -\rho c_p r_{av}^{-1} & -4f_v\sigma T_*^3
& J_{44} \\
\end{bmatrix}
\end{equation}
with
\begin{equation}
J_{33} = (c_p+L\psi_gD_g)\frac{\rho}{r_{ag}}+4\sigma T_*^3 + 2\frac{\lambda_1}{\Delta h_1}
\end{equation}
and
\begin{equation}
J_{44} = \frac{C_{\rm can}}{\delta t}+(c_p+L\psi_v D_v)\frac{\rho}{r_{av}}+8f_v\sigma T_v^3
\end{equation}
Equation (\ref{eq:multiNR}) is implemented by solving
\begin{equation}
{\rm J}({\bf x - x}_0) = {\bf f}({\bf x}_0)
\label{eq:increments}
\end{equation}
for ${\bf x}$ by LU decomposition. If this gives $T_* > T_m$ and there is snow with ice mass $I$ on the ground, Equation (\ref{eq:f3}) is replaced by
\begin{equation}
f_3 = LW_g + SW_g - G - H_g - LE_g - L_fM.
\end{equation}
It is first assumed that all of the snow melts and Equation (\ref{eq:increments}) is solved again with $M = I/\delta t$. If this gives $T_* < T_m$, the snow does not all melt and the surface temperature is known but the melt rate is unknown. Fluxes in $f$ are recalculated with $T_* = T_m$, elements in the third column of ${\rm J}$ are replaced by $J_{13}=J_{23}=J_{43}=0$ and $J_{33}=1$, and $M=x_3/L_f$ is found from the solution of Equation (\ref{eq:increments}).

\subsubsection{Two-layer canopy model ({\it option to be added})}

\section{Mass balance}

A forest canopy can intercept a fraction of falling snow up to a maximum capacity which is either parametrized as $S_{\rm cap} = c_{\rm vai}\Lambda$ or read from a file. If the canopy holds snow mass $S_v$ at the beginning of a timestep, the amount of snow intercepted in the timestep is
\begin{equation}
\delta S_v = (S_{\rm cap} - S_v)\left[1 - \exp\left(-\frac{f_vS_f\delta t}{S_{\rm cap}}
                                \right)\right]
\end{equation}
and the rate of snowfall reaching the ground is reduced to
\begin{equation}
T_f = S_f - \frac{\delta S_v}{\delta t}.
\end{equation}
Snow sublimates from the canopy at rate $E_v$ and unloads at rate $U_c = \tau_{\rm can}^{-1}S_v$ with different values of the time constant $\tau_{\rm can}$ for cold and melting snow; if $\tau_{\rm can} \leq \delta t$, all of the snow will unload immediately from the canopy. Interception and storage of liquid water in the canopy are neglected. The combined mass balance equation for canopy snow is
\begin{equation}
\frac{dS_v}{dt} = S_f - T_f - U_c - E_v.
\end{equation}

If there is no forest canopy, the throughfall and unloading rates are obviously $T_f=S_f$ and $U_c=0$. The mass balance equations for ice and liquid water in snow on the ground are
\begin{equation}
\frac{dI}{dt} = T_f + U_c - E_g - M + F
\end{equation}
and
\begin{equation}
\frac{dW}{dt} = R_f + M - F - R_o.
\end{equation}
If storage of liquid water in snow is neglected (option {\tt HYDROL=0}), internal refreezing rate in the snow $F=0$ and runoff rate at the base of the snow $R_o=R_f+M$. If bucket storage is selected (option {\tt HYDROL=1}), snow layer $k$ with porosity
\begin{equation}
\phi_k = 1 - \frac{I_k}{\rho_{\rm ice}D_{s,k}}
\end{equation}
can hold a maximum mass $\rho_{\rm wat}\phi_k W_{\rm irr}$ of liquid water.

\section*{Tables}
\parindent0pt
%\begin{table}
\begin{tabular}{|l|l|l|}
\hline
Documentation & Code & Value \\
\hline
Heat capacity of air $c_p$            & {\tt cp}        & 1005 J K$^{-1}$ kg$^{-1}$  \\
Heat capacity of ice $c_{\rm ice}$    & {\tt hcap\_ice} & 2100 J K$^{-1}$ kg$^{-1}$  \\
Heat capacity of water $c_{\rm wat}$  & {\tt hcap\_wat} & 4180 J K$^{-1}$ kg$^{-1}$  \\
Acceleration due to gravity $g$       & {\tt g}         & 9.81 m s$^{-2}$            \\
Solar constant $I_0$                  & {\tt I0}        & 1367 W m$^{-2}$            \\
Von K\'arm\'an constant $k$           & {\tt vkman}     & 0.4                        \\
Latent heat of fusion of water $L_f$  & {\tt Lf}        & $0.334 \times 10^6$ J kg$^{-1}$ \\
Latent heat of sublimation of ice $L_s$ & {\tt Ls}      & $2.835 \times 10^6$ J kg$^{-1}$ \\
Latent heat of vapourisation of water $L_v$ & {\tt Lv}  & $2.501 \times 10^6$ J kg$^{-1}$ \\
Gas constant for air $R_{\rm air}$    & {\tt Rair}      & 287 J K$^{-1}$ kg$^{-1}$   \\
Gas constant for water vapour $R_{\rm wat}$ & {\tt Rwat} & 462 J K$^{-1}$ kg$^{-1}$ \\
Melting point of ice $T_m$            & {\tt Tm}        & 273.15 K                  \\
Thermal conductivity of air $\lambda_{\rm air}$   & {\tt hcon\_air}  & 0.025 W m$^{-1}$ K$^{-1}$ \\
Thermal conductivity of clay $\lambda_{\rm clay}$ & {\tt hcon\_clay} & 1.16 W m$^{-1}$ K$^{-1}$ \\
Thermal conductivity of ice $\lambda_{\rm ice}$   & {\tt hcon\_ice}  & 2.24 W m$^{-1}$ K$^{-1}$ \\
Thermal conductivity of sand $\lambda_{\rm sand}$ & {\tt hcon\_sand} & 1.57 W m$^{-1}$ K$^{-1}$ \\
Thermal conductivity of water $\lambda_{\rm wat}$ & {\tt hcon\_wat}  & 0.56 W m$^{-1}$ K$^{-1}$ \\
Density of ice $\rho_{\rm ice}$ & {\tt rho\_ice} & 917 kg m$^{-3}$ \\
Density of water $\rho_{\rm wat}$ & {\tt rho\_wat} & 1000 kg m$^{-3}$ \\
Stefan-Boltzmann constant $\sigma$ & {\tt sb} & $5.26 \times 10^{-8}$ W m$^{-2}$ K$^{-4}$ \\
\hline
\end{tabular}

Table 1. Physical constants and quantities assumed to be constant in the code and documentation.
%\label{table:constants}
%\end{table}

\vskip20pt
\begin{tabular}{|l|l|l|}
\hline
Documentation                                & Code       & Units                  \\
\hline
Incoming longwave radiation $LW_\downarrow$  & {\tt LW}   & W m$^{-2}$             \\
Surface air pressure $P_s$                   & {\tt Ps}   & Pa                     \\
Specific humidity $Q_a$                      & {\tt Qa}   & kg kg$^{-1}$           \\
Rainfall rate $R_f$                          & {\tt Rf}   & kg m$^{-2}$ s$^{-1}$   \\
Snowfall rate $S_f$                          & {\tt Sf}   & kg m$^{-2}$ s$^{-1}$   \\
Diffuse shortwave radiation $S_{\rm dif}$    & {\tt Sdif} & W m$^{-2}$             \\
Direct-beam shortwave radiation $S_{\rm dir}$& {\tt Sdir} & W m$^{-2}$             \\
Incoming shortwave radiation $SW_\downarrow$ & {\tt SW}   & W m$^{-2}$             \\
Air temperature $T_a$                        & {\tt Ta}   & K                      \\
Wind speed $U_a$                             & {\tt Ua}   & m s$^{-1}$             \\
Wind direction (not yet used)                & {\tt Udir} & degrees clockwise of N \\
\hline
\end{tabular}

Table 2. Meteorological driving variables in the code and documentation

\vskip20pt
\begin{tabular}{|l|l|l|}
\hline
Documentation & Code & Units \\
\hline
\multicolumn{3}{|c|}{Forest canopy} \\
\hline
Canopy air space specific humidity $Q_c$ & {\tt Qcan} & kg kg$^{-1}$ \\
Snow mass on canopy $S_v$                & {\tt Sveg} & W m$^{-2}$   \\
Canopy air space $T_c$                   & {\tt Tveg} & K            \\
Vegetation temperature $T_v$             & {\tt Tveg} & K            \\
\hline
\multicolumn{3}{|c|}{Surface} \\
\hline
Surface skin temperature $T_*$           & {\tt Tsrf} & K            \\
\hline
\multicolumn{3}{|c|}{Snow on the ground (up to {\tt Nsmax} layers)}  \\
\hline
Number of snow layers $N_{\rm snow}$     & {\tt Nsnow} & -           \\
Albedo of snow $\alpha_s$                & {\tt albs}  & -           \\
Thickness of snow layers $D_s$           & {\tt Ds}    & m           \\
Radii of grains in snow layers $r$       & {\tt rgrn}  & m           \\
Ice content of snow layers $I$           & {\tt Sice}  & kg m$^{-2}$ \\
Liquid water content of snow layers $W$  & {\tt Sliq}  & kg m$^{-2}$ \\
Temperature of snow layers $T_s$         & {\tt Tsnow} & K           \\
\hline
\multicolumn{3}{|c|} {Soil ({\tt Nsoil} layers)} \\
\hline
Temperature of soil layers $T_g$ & {\tt Tsoil} & K \\
Volumetric moisture content of soil layers $\theta_g$ & {\tt theta} & - \\
\hline
\end{tabular}

Table 3. Model state variables in the code and documentation.

\vskip20pt
\begin{tabular}{|l|l|l|}
\hline
Documentation & Code & Default \\
\hline
Snow-free albedo $\alpha_0$         & {\tt alb0} & 0.2                                 \\
Canopy heat capacity $C_{\rm can}$  & {\tt canh} & $2500\Lambda$ (J K$^{-1}$ m$^{-2}$) \\
Soil clay fraction $f_{\rm clay}$   & {\tt fcly} & 0.3                                 \\
Soil sand fraction $f_{\rm sand}$   & {\tt fsnd} & 0.6                                 \\
Sky view fraction $f_{\rm sky}$     & {\tt fsky} & $\exp(-k_{\rm dif}\Lambda)$         \\
Canopy cover fraction $f_{\rm veg}$ & {\tt fveg} & $1 - \exp(-\Lambda)$                \\
Canopy height $h_{\rm can}$         & {\tt hcan} & 0 m                                 \\
Latitude $\phi$                     & {\tt lat}  & 0$^\circ$                           \\
Vegetation area index $\Lambda$     & {\tt VAI}  & 0                                   \\
Snow-free ground roughness length $z_{0g}$ & {\tt z0sf} & 0.1 m                        \\
Timestep $\delta t$                 & {\tt dt}   & 3600 s                              \\
Temperature and humidity measurement height $z_T$& {\tt zT} & 2 m                      \\
Wind speed measurement height $z_U$ & {\tt zU}   & 10 m                                \\
\hline
\end{tabular}

Table 4. Site and driving data characteristics in the code and documentation.

\vskip20pt
\begin{tabular}{|l|l|l|}
\hline
Documentation                                      & Code       & Default                \\
\hline
Maximum albedo for fresh snow $\alpha_{\rm max}$   & {\tt asmx} & 0.8                    \\
Minimum albedo for melting snow $\alpha_{\rm min}$ & {\tt asmn} & 0.5                    \\
Snow-free vegetation albedo $\alpha_{c0}$          & {\tt avg0} & 0.1                    \\
Snow-covered vegetation albedo $\alpha_{cs}$       & {\tt avgs} & 0.4                    \\
Atmospheric stability adjustment parameter $b_h$   & {\tt bstb} & 5                      \\
Snow thermal conductivity exponent $b_\lambda$     & {\tt bthr} & 2                      \\
Dense canopy turbulent transfer coefficient $C_{\rm dense}$ & {\tt cden} & 0.004         \\
Canopy snow capacity per unit VAI $c_{\rm vai}$    & {\tt cvai} & 4.4 kg m$^{-2}$        \\
Vegetation turbulent transfer coefficient $C_{\rm veg}$ & {\tt cveg} & 20                \\
Snow cover fraction depth scale $h_f$ & {\tt hfsn} & 0.1 m                               \\
Reference snow viscosity $\eta_0$                  & {\tt eta0} & $3.7 \times 10^7$ Pa s \\
Surface conductance for saturated soil $g_{\rm sat}$ & {\tt gsat} & 0.01 m s$^{-1}$      \\
Leaf angle distribution parameter $G_1$            & {\tt Gcn1}   & 0.5                  \\
Leaf angle distribution parameter $G_2$            & {\tt Gcn2}   & 0                    \\
Diffuse radiation extinction coefficient $k_{\rm dif}$ & {\tt kdif} &  0.5               \\
Fixed snow thermal conductivity $\lambda_0$        & {\tt kfix} & 0.24 W m$^{-1}$ K$^{-1}$ \\
Canopy cover coefficient $k_{\rm veg}$             & {\tt kveg}   &  1 \\
Displacement height to canopy height ratio $R_d$   & {\tt rchd}   & 0.67 \\
Roughness length to canopy height ratio $R_{z0}$   & {\tt rchz}   & 0.1 \\
Fresh snow grain radius $r_0$                      & {\tt rgr0}   & $5\times10^{-5}$ m \\
Fixed snow density $\rho_0$                        & {\tt rho0}   & 300 kg m$^{-3}$ \\
Temperature factor in fresh snow density $b _\rho$ & {\tt rhob}   & 0 kg m$^{-3}$ K$^{-1}$ \\
Wind factor in fresh snow density $c_\rho$         & {\tt rhoc}   & 0 kg m$^{-7/2}$ s$^{1/2}$\\
Fresh snow density $\rho_f$ & {\tt rhof}           & 100 kg m$^{-3}$ \\
Maximum density for cold snow $\rho_{\rm cold}$    & {\tt rcld} & 300 kg m$^{-3}$ \\
Maximum density for melting snow $\rho_{\rm melt}$ & {\tt rmlt} & 500 kg m$^{-3}$ \\
Snowfall to refresh albedo $S_\alpha$ & {\tt Salb} & 10 kg m$^{-2}$ \\
Thermal metamorphism parameter $c_1$ & {\tt snda}  & $2.8 \times 10^{-6}$ s$^{-1}$ \\
Snow albedo decay temperature threshold $T_\alpha$ & {\tt Talb} & -2$^\circ$C \\
Canopy unloading time scale $\tau_{\rm can}$ for cold snow & {\tt tcnc} & 240 h \\
Canopy unloading time scale $\tau_{\rm can}$ for melting snow & {\tt tcnm} & 2.4 h \\
Cold snow albedo decay time scale $\tau_{\rm cold}$ & {\tt tcld} & 1000 h \\
Melting snow albedo decay time scale $\tau_{\rm melt}$ & {\tt tmlt} & 100 h \\
Snow compaction time scale $\tau_\rho$ & {\tt trho} & 200 h \\
Irreducible liquid water content of snow $W_{\rm irr}$ & {\tt Wirr} & 0.03 \\
Ratio of roughness lengths for momentum and heat $z_0/z_{0h}$ & {\tt z0zh} & 10 \\
Snow surface roughness length $z_{0s}$ & {\tt z0sn} & 0.01 m \\
\hline
\end{tabular}

Table 5. Model parameters in the code and documentation.

\section*{Appendix: Solar radiation}

Following \href{https://www.elsevier.com/books/an-introduction-to-solar-radiation/iqbal/978-0-12-373750-2}{Iqbal (1983)}, solar declination $\delta$ (in radians) and equation of time $E_t$ (in hours) are approximated by Fourier series
\begin{align}
\delta = 0.006918 &- 0.399912\cos\Gamma  + 0.070257\sin\Gamma  \nonumber \\
                  &- 0.006758\cos2\Gamma + 0.000907\sin2\Gamma \nonumber \\
                  &- 0.002697\cos3\Gamma + 0.001480\sin3\Gamma
\end{align}
and
\begin{align}
E_t = (12/\pi)(0.000075 &+ 0.001868\cos\Gamma  - 0.032077\sin\Gamma \nonumber \\
                        &- 0.014615\cos2\Gamma - 0.04089\sin2\Gamma)
\end{align}
for day of year $d_n$ and day angle $\Gamma = 2\pi(d_n - 1)/365$. From a date given by intergers $y$, $m$ and $d$ for year, month and day, the day of year can be found using integer division in the magic formula
\begin{equation}
d_n = (7y)/4 - 7(y+(m+9)/12)/4 + (275m)/9 + d - 30.
\end{equation}

For hour $h$ of a day at latitude $\phi$, the hour angle is defined by
\begin{equation}
\omega = (\pi/12)(12 - h - E_t),
\end{equation}
the sine of the solar elevation is
\begin{equation}
\sin\theta = \sin\delta\sin\phi + \cos\delta\cos\phi\cos\omega
\end{equation}
and the cosine of the solar azimuth is
\begin{equation}
\cos\psi = (\sin\theta\sin\phi - \sin\delta)/(\cos\theta\cos\phi).
\end{equation}
Azimuth is measured anticlockwise from south, so $\psi$ has the same sign as $\omega$ (positive in the morning and negative in the afternoon).

Separate direct and diffuse components of shortwave radiation are estimated by the empirical method of \href{https://www.sciencedirect.com/science/article/pii/0038092X82903024}{Erbs et al. (1982)} if they are required but have not been measured. First, an atmospheric clearness parameter is found by dividing the global radiation by the radiation at the top of the atmosphere, giving
\begin{equation}
k_t = \frac{SW_\downarrow}{I_o\sin\theta},
\end{equation}
where $I_0$ = 1367 Wm$^{-2}$ is the solar constant. The diffuse fraction is then estimated as
\begin{equation}
\frac{S_{\rm dif}}{S_0} = \begin{cases}
    1 - 0.09k_t &  k_t<0.22  \\
    0.95-0.16k_t+4.39k_t^2-16.64k_t^3+12.34k_t^4 &  0.22<k_t<0.8  \\
    0.165 &  k_t>0.8
\end{cases}
\end{equation}
Direct radiation is the remainder $S_{\rm dir} = S_0 - S_{\rm dif}$.

If a digital elevation model is provided as an input, the array of elevations is assumed to follow the convention that the first row is at the north of the model domain and the first column is at the west.
\begin{equation}
\tan^{-1}\left[\sqrt{\left(\frac{dz}{dx}\right)^2 + \left(\frac{dz}{dy}^2\right)}\right]
\end{equation}
\begin{equation}
- \tan^{-1}\left(\frac{dz}{dy}\middle/\frac{dz}{dx}\right) - \frac{\pi}{2}
\end{equation}
Aspect is calculated to follow the same convention as solar azimuth and the Fortran {\tt atan2} function is used to obtain the correct quadrant. Grid cells are indexed by row number $i$ from north to south and column number $j$ from west to east. The surface gradient is calculated by centred differences
\begin{equation}
\left(\frac{dz}{dx},\frac{dz}{dy}\right)_{ij} =
\left(\frac{z_{i,j+1} - z_{i,j-1}}{2\Delta x}, \frac{z_{i+1,j} - z_{i-1,j}}{2\Delta x}\right)
\end{equation}
for grid spacing $\Delta x$.

\end{document}


\SetPicSubDir{ch-Intro}

\chapter{Introduction}
\vspace{2em}

The concept of graphs was first used by Leonhard Euler to study the Königsberg bridges problem \cite{euler:konis} in 1735. This laid the foundation for field of mathematics known as graph theory.

\noindent
These days graphs are used to model and study complex networks. Networks like the internet, electric grid, road networks, social networks etc. are too large to be directly studied.  In such situations a mathematical modelling of the structure allows us to study it's properties more efficiently.

\noindent
In this thesis we shall be studying one such property of a graph known as it's hamiltonicity, on particular classes of graphs.

\section{Preliminaries}
In the following section we shall briefly introduce some of the preliminaries and conventions that shall be used throughout the thesis.

\subsection{Graph Theory}

\begin{defn}
We define a \textbf{Graph} $G$ as the pair of sets $(V, E)$ where $V$ is the \textit{vertex set}, and the \textit{edge set}, $E \subseteq V \times V$.
\end{defn}
In an \textit{undirected graph} the order of vertices in an edge does not matter. In such graphs we represent an edge as $\{u, v\}$. On the other hand, in a \textit{directed graph} (or digraph) the order of vertices in an edge matters. Edges in such a graph are represented as $(u, v)$.

\begin{defn}
The number of edges \textit{adjacent} to a vertex, is known as the \textbf{Degree} of that vertex.
\end{defn}
Note that in a directed graph, we have separate in and out-degrees for each vertex. Henceforth, we shall be using $\delta$ to represent the minimum degree of a vertex in a graph, and similarly $\Delta$ to represent the maximum degree.

\begin{defn}
A sequence of distinct vertices $P = v_0, v_1, \cdots v_k$ in a directed graph $G(V, E)$, where $\forall i<k, e_i = (v_i, v_{i+1}) \in E$ is known as a \textbf{Path} in the graph.
\end{defn}

\begin{defn}
A path $C = v_0, v_1, \cdots v_k$ in graph $G(V, E)$ where $v_0 = v_k$ is known as a \textbf{Cycle}.
\end{defn}

\begin{defn}
A cycle $C$ such that it passes through every vertex of the graph, is called a \textbf{Hamiltonian Cycle}.
A graph which contains a Hamiltonian cycle, is said to be a \textbf{Hamiltonian Graph}.
\end{defn}

\subsection{Random Graph Models}
The graphs we shall be looking at in this thesis shall all be sampled from one of the distributions described below.
\begin{description}
    \item[$D_{n, m}$ model: ] A graph is picked uniformly at random from the set of all digraphs with $n$ vertices and $m$ edges.
    \item[$k$-in, $k$-out: ] In the $D_{k-in, k-out}$ model, for a graph with $n$ vertices, $k$ in-neighbours and $l$ out-neighbours are chosen independently at random from the vertex set $V_n$.
    \item[$k$-regular digraph: ] In a random regular digraph $D_{n, k}$, each of the n vertices has in-degree and out-degree exactly k.
    \item[Powerlaw graphs: ] Given a fixed degree sequence that has a power law tail, we pick a graph uniformly at random from the set of all graphs that realize the given degree seqence.
    \item[Random Tournaments: ] In a complete graph $K_n$, we assign a direction uniformly at random to each edge.
    \item[Random n-lifts: ] The \textit{n-lift} of a given graph $G(V, E)$, is obtained by replacing each vertex in $V$ by a set of n vertices, and by adding a random perfect matching (between the corresponding sets of $u \& v$) for every $e_i = (u, v) \in E$, .
\end{description}


\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{vntex}
\usepackage{amsmath}
\usepackage{units}

\title{Machine Learning Homework Week 1}
\author{Tô Đức Anh}
\date{September 2021}

\begin{document}

\maketitle

\section{Problems}
    \begin{enumerate}
        \item To evaluate a new test for detecting Hansen’s disease, a group of people 5\% of which are known to have Hansen’s disease are tested. The test finds Hansen’s disease among 98\% of those with the disease and 3\% of those who don’t. What is the probability that someone testing positive for Hansen’s disease under this new test actually has it?

        \item Proof the following distributions are normalized then calculate the mean and standard deviation of these distribution:
        \begin{itemize}
            \item Univariate normal distribution.
            \item (Optional) Multivariate normal distribution.
        \end{itemize}

    \end{enumerate}

\section{Answers}
    \subsection{Question 1}
        \begin{itemize}
            \item Probability that someone testing positive for Hansen’s disease under this new test actually has it:
               \begin{equation} \label{eq:1}
                    \begin{split}
                        P (actually \ has \ disease \mid tested) = \\ \frac{P(tested \mid actually \ has \ disease)*P(actually\ has \ disease)}{P(tested)}
                    \end{split}
                \end{equation}
            \item Probability of people with Hansen's disease in the 5\% are known to have Hansen's disease is:
                $$
                    5\%\times 98\% = 4.9\%
                $$
            \item Probability of people with Hansen's disease in the 95\% are known to not have Hansen's disease is:
                $$
                    95\%\times 3\%  = 2.85\%
                $$
            \item Probability of people testing positive for Hansen's disease is:
                $$
                    4.9\% + 2.85\% = 7.75\%
                $$
            \item Probability that someone testing positive for Hansen’s disease under this new test actually has it is:
                $$
                    \frac{5\%\times 98\%}{7.75\%} \approx 63.2\%
                $$
            \end{itemize}

    \subsection{Question 2}
        \begin{itemize}
            \item Univariate normal distribution
            \subsubsection{Formula for univariate normal distribution}
                    \[
                        f(x) = \frac{1}{\sigma\sqrt{2\pi}}\times e^{ -\frac{1}{2}(\frac{x - \mu}{\sigma})^2}
                    \]
            With respect to $\mu$ as Mean and $\sigma$ as Standard Deviation

            \subsubsection {Prove that f(x) always > 0 $\forall \mu $ and $ \sigma $}
            We can see that $\frac{1}{\sigma\sqrt{2\pi}}$ will always > 0 since the standard deviation $\sigma$ always $\ge$ 0. Therefore $ \frac{1}{\sigma\sqrt{2\pi}}\times e^{ -\frac{1}{2}(\frac{x - \mu}{\sigma})^2} $ will always > 0 $\forall \mu$ and $\sigma$
            \subsubsection{Prove that $\int_{-\infty}^{\infty} f(x)dx$ = 1}

            \subsubsection{Prove mean = $\mu$:}
            We have:
                \[
                        f(x) = \frac{1}{\sigma\sqrt{2\pi}}\times e^{ -\frac{1}{2}(\frac{x - \mu}{\sigma})^2}
                \]
            So that expectation of f(x) is:
                \begin{equation}
                    \begin{split}
                        E[f(x)] & = \int_{-\infty}^{\infty} x \times \frac{1}{\sigma\sqrt{2\pi}}\times e^{ -\frac{1}{2}(\frac{x - \mu}{\sigma})^2}dx \\
                                & = \frac{1}{\sigma\sqrt{2\pi}} \times \int_{-\infty}^{\infty} x  \times e^{ -\frac{1}{2}(\frac{x - \mu}{\sigma})^2}dx \\
                                & = \frac{\sqrt{2}\sigma}{\sigma\sqrt{2\pi}} \times \int_{-\infty}^{\infty} \Big(\sqrt{2} \sigma t + \mu\Big) \times e^{-t^2}dt \\
                                & = \frac{1}{\sqrt{\pi}} \times \Bigg(\sqrt{2} \sigma \times
                                \int_{-\infty}^{\infty}  t \times e^{-t^2} dt + \mu \int_{-\infty}^{\infty} e^{-t^2}dt\Bigg) \\
                                & = \frac{1}{\sqrt{\pi}} \times \Bigg(\sqrt{2} \sigma \times
                                \Bigg[\frac{-1}{2} \times e^{-t^2}\Bigg] + \mu \times \sqrt{\pi}\Bigg)\\
                                & = \frac{\mu \sqrt{\pi}}{\sqrt{\pi}}\\
                                & = \mu
                    \end{split}
                \end{equation}
            \subsubsection{Prove Standard deviation = $\sigma^2$}
                \begin{equation}
                    \begin{split}
                        E[f(x)] & = \int_{-\infty}^{\infty} x \times \frac{1}{\sigma\sqrt{2\pi}}\times e^{ -\frac{1}{2}(\frac{x - \mu}{\sigma})^2}dx \\
                                & = \frac{1}{\sigma\sqrt{2\pi}} \times \int_{-\infty}^{\infty} x  \times e^{ -\frac{1}{2}(\frac{x - \mu}{\sigma})^2}dx \\
                                & = \frac{\sqrt{2}\sigma}{\sigma\sqrt{2\pi}} \times \int_{-\infty}^{\infty} \Big(\sqrt{2} \sigma t + \mu\Big) \times e^{-t^2}dt \\
                                & = \frac{1}{\sqrt{\pi}} \times \Bigg(\sqrt{2} \sigma \times
                                \int_{-\infty}^{\infty}  t \times e^{-t^2} dt + \mu \int_{-\infty}^{\infty} e^{-t^2}dt\Bigg) \\
                                & = \frac{1}{\sqrt{\pi}} \times \Bigg(\sqrt{2} \sigma \times
                                \Bigg[\frac{-1}{2} \times e^{-t^2}\Bigg] + \mu \times \sqrt{\pi}\Bigg)\\
                                & = \frac{\mu \sqrt{\pi}}{\sqrt{\pi}}\\
                                & = \mu
                    \end{split}
                \end{equation}
            \item (Optional) Multivariate normal distribution.
                \begin{itemize}
                    \item Proof that the distribution are normalized:
                    \item Mean:
                    \item Standard Deviation:
                \end{itemize}
        \end{itemize}

\end{document}

Homework/Homework W1/To Duc Anh W1 HW.tex


\section{Missing data
\normalfont\sffamily, w/o labels $\color{section-text-color}\to$ unsupervised}

% ===
\emph{Mixture modeling}

Model each class as prob. distribution $P(x|\theta_j)$\\
$P(D|\theta) = \prod_{i=1}^n \sum_{j=1}^c w_j P(x_i|\theta_j)$\\
$L(w, \theta) = - \sum_{i=1}^n \log  \sum_{j=1}^c w_j P(x_i| \theta_j)$\\
$\implies \theta^\ast = \arg\!\min L(\theta)$\enskip\emph{$\to$ non convex}

% ===
\emph{Gaussian-Mixture Bayes classifiers}

Estimate prior $P(y)$; Est. cond. distr. for each class:
$P(x|y) = \sum_{j=1}^{k_y} w_j^{(y)} \mathcal{N}(x; \mu_j^{(y)}, \Sigma_j^{(y)})$\\

% ===
\emph{Hard-EM algorithm}\vspace{1pt}
\begin{highlightbox}
	Initialize $\theta^{(0)}$; \enskip Let $Q^{(t)} (z) = P(z\vert x,\theta^{(t)})$\\
	\textbf{For $t=1,2,...$ do:}\\
	$\bullet$ \textbf{E-step}: estimate log-likelihood\\
	\phantom{$\bullet$} Predict most likely class for each point $x_i$\\
	\phantom{$\bullet$} $\circ$
		$z_i^{(t)} = \arg\!\max\limits_z P(z\vert x_i,\theta^{(t-1)})$\vspace{-5pt}\\
	\phantom{$\bullet$ $\circ$}
		\phantom{$z_i^{(t)}$}$\;= \arg\!\max\limits_z P(z\vert\theta^{(t-1)}) \, P(x_i\vert z,\theta^{(t-1)})$\\
	$\bullet$ \textbf{M-step}: Maximize (MLE)\\
	\phantom{$\bullet$} $\circ$
		$L^{(t)} (\theta) = \mathbb{E}_{Q^{(t)}} [\log P(x,y\vert \theta^{(t)})]$\\
	\phantom{$\bullet$} $\circ$
		$\theta^{(t)} = \arg\!\max\limits_\theta L^{(t)}$
		$\to \pderiv{~}{\theta}L = 0$
		$\to \theta^{(t)} = ...$
\end{highlightbox}
$\theta^{(t)} = \underset{\theta}{\operatorname{argmax}} P(D^{(t)}|\theta)$, i.e. $\mu_j^{(t)} = \frac{1}{n_j} \sum_{i: z_i = j x_j}$

% ===
\emph{Soft-EM algorithm}

E-step: Calc p for each point and cls.: $\gamma_j^{(t)}(x_i)$\\
M-step: Fit clusters to weighted data points:\\
$w_j^{(t)} = \frac{1}{n} \sum_{i=1}^n \gamma_j^{(t)} (x_i)$;
$\mu_j^{(t)} = \frac{\sum_{i=1}^n \gamma_j^{(t)} (x_i) x_i}{\sum_{i=1}^n \gamma_j^{(t)} (x_i)}$\\
$\sigma_j^{(t)} = \frac{\sum_{i=1}^n \gamma_j^{(t)}(x_i) (x_i - \mu_j^{(t)})^T (x_i - \mu_j^{(t)})}{\sum_{i=1}^n \gamma_j^{(t)}(x_i)}$

% ===
\emph{Soft-EM for semi-supervised learning}

labeled $y_i$: $\gamma_j^{(t)}(x_i) = [j = y_i]$,
unlabeled:\\ $\gamma_j^{(t)}(x_i) = P(Z=j|x_i, \mu^{(t-1)}, \Sigma^{(t-1)}, w^{(t-1)})$

%If enough space put this on summary.
\iffalse
\subsection*{Log-likelihood}
$l(\theta) = log P(\mathcal{D})$ \\
$=\sum_{\overset{i=1}{y_i=\times}}^n log P(x_i;\theta) + \sum_{\overset{i=1}{y_i\not=\times}}^n log P(x_i,y_i;\theta)$\\
$=\sum_{\overset{i=1}{y_i=\times}}^n log \sum_{i=1}^m P(x_i, Y=j;\theta) +$\\
$ \sum_{\overset{i=1}{y_i\not=\times}}^n log P(x_i,y_i;\theta)$\\
$=\sum_{\overset{i=1}{y_i=\times}}^n log \sum_{i=1}^m P(x_i|Y=j;\theta)P(Y=j|\theta) +$\\
$ \sum_{\overset{i=1}{y_i\not=\times}}^n log P(x_i,y_i;\theta)$

\subsection*{Latent variable}
We denote the latent variable indicating the component the point is sampled from by Z, which takes on values in $\{1,...,k\}$.

\subsection*{E-step: Posterior probabilities}
$\gamma_j^t(x_i) = P(Z=j|x_i, \theta_t) = \frac{P(x_i|Z=j, \theta_t) P(Z=j|\theta_t)}{P(x_i;\theta_t)}$

\subsection*{M-step: maximizing expected log likelihood}
$\mathbb{E}_{\gamma^t}[\log P(\mathcal{D;\theta})] =
\mathbb{E}_{\gamma^t}[\log \Pi_{i=1}^nP(x_i,z_i;\theta)] = $ \\
$\sum_{i=1}^n \mathbb{E}_{\gamma^t}[\log P(x_i,z_i;\theta)] = $ \\
$\sum_{i=1}^n \sum_{j=1}^k \gamma_j^t(x_i) \log (P(x_i|z_i=j;\theta) P(z_i=j;\theta))$ \\
$\theta_{t+1} = \underset{\theta}{\operatorname{argmax}} \mathbb{E}_{\gamma^t}[\log P(\mathcal{D;\theta})]$
\fi


\documentclass[11pt]{article}
%Gummi|065|=)
\title{\textbf{Algorithms in Julia}}
\author{Valentin Churavy \and James Schloss}
\date{}

\usepackage{graphicx}
\usepackage{hyperref}
\usepackage{listings}
\usepackage{color}

\definecolor{dkgreen}{rgb}{0,0.6,0}
\definecolor{gray}{rgb}{0.5,0.5,0.5}
\definecolor{mauve}{rgb}{0.58,0,0.82}

\lstset{frame=tb,
  language=python,
  aboveskip=3mm,
  belowskip=3mm,
  showstringspaces=false,
  columns=flexible,
  basicstyle={\small\ttfamily},
  numbers=none,
  numberstyle=\tiny\color{gray},
  keywordstyle=\color{blue},
  commentstyle=\color{dkgreen},
  stringstyle=\color{mauve},
  breaklines=true,
  breakatwhitespace=true,
  tabsize=3
}

\begin{document}
\maketitle
In this document, we have descriptions of the following algorithms:
\begin{itemize}
\item Monte Carlo
\item Convolution / Edge Detection for simple images
\item Split-Operator method for solving the Schrodinger Equation
\item Julia Fractal
\item Force Integration
\item Barnes-Hut (N-body galaxy simulation)
\item FFT
\end{itemize}

All algorithms can be found in the examples directory of the \texttt{skillpill-julia} repository on github

\newpage
\section*{Monte Carlo}

There are many different methods that all work under the basic Monte Carlo principle of using random numbers to integrate systems.
For the sake of time and brevity, please refer to the following link for more information on Monte Carlo integration: \url{http://leios.github.io/Batman_Montecarlo/}. From there, modify the provided code to work in Julia and to integrate $x^2$ where $-3 < x < 3$.

My function reads in a total range in x along with the number of points to sample. Try to match (or beat) the following benchmark:

\begin{lstlisting}
julia> @benchmark monte_carlo(6.0, 100000)
BenchmarkTools.Trial:
  memory estimate:  0 bytes
  allocs estimate:  0
  --------------
  minimum time:     573.931 Î¼s (0.00% GC)
  median time:      578.105 Î¼s (0.00% GC)
  mean time:        596.870 Î¼s (0.00% GC)
  maximum time:     1.125 ms (0.00% GC)
  --------------
  samples:          8326
  evals/sample:     1

\end{lstlisting}

\newpage
\section*{Edge Detection}
This is more about technical implementation than algorithms; however, Canny Edge Detection is a staple algorithm used in many areas -- most importantly image analysis. It is basically composed of 5 steps:
\begin{enumerate}
\item Apply Gaussian filter to smooth the image in order to remove the noise
\item Find the intensity gradients of the image
\item Apply non-maximum suppression to get rid of spurious response to edge detection
\item Apply double threshold to determine potential edges
\item Track edge by hysteresis: Finalize the detection of edges by suppressing all the other edges that are weak and not connected to strong edges.
\end{enumerate}

For the most part, this edge detection can be done relatively simply with the following code:

\begin{lstlisting}
# First, we need to use the appropriate packages
using Images  # All filtering and image stuff
using ImageView # imshow for showing images, run in REPL
using TestImages # Standart example images in julia

# simple implementation of canny edge detection using in-built Julia functions
function simple_edge_detection()
    img = testimage("fabio_color_256")
    img_edge = canny(img)
    save(string("fabio_edges.png"), img_edge)
end

simple_edge_detection()

\end{lstlisting}

However, that doesn't really show the \textbf{power} of Julia. Perform each of the 5 above steps without using the \texttt{canny} function.

\newpage
\section*{Split-Operator Method}
The Split-Operator Method is a psuedo-spectral method that is important for many areas of physics, including quantum mechanics, where it is used to solve the Schr\"odinger equation,

$$
i \hbar \frac{\partial \Psi(\mathbf{r}, t)}{\partial t} = \left[\frac{-\hbar^2}{2m} \nabla^2 + V(\mathbf{r},t) \right] \Psi(\mathbf{r},t)
$$

To solve this equation, we basically split the time evolution operator ($\hat U = e^{-i \hat H t / \hbar}$) into 2 via the Baker-Campbell-Hausdorff relation:
$$\hat U_k = e^{-\mathbf{\hat p}^2 t / 2} \quad \hat U_V = e^{-V_0(\mathbf{\hat r})t/2}$$

From here, the method simply involves initializing the wavefunction and then multiplying it by half of $\hat U_V$ before flipping to momentum space via a FFT and multiplying it by all of $\hat U_k$ and then flipping back to real space with an inverse FFT and multiplying by the other half of $\hat U_V$, as shown in figure \ref{fig:split-op}

\begin{figure}
\begin{center}
\includegraphics[width = 0.5\textwidth]{split-op.png}
\end{center}
\caption{A single step in the split-op method as described in the description}
\label{fig:split-op}
\end{figure}

\newpage
\section*{Julia Fractal}
Fractals are cool. They are structures that continually repeat themselves no matter how far you zoom in. Because we are using Julia, the obvious fractal to work with is the Julia fractal. Full implementation details in Python can be found here: \url{https://batchloaf.wordpress.com/2013/02/10/creating-julia-set-images-in-python/}. Basically, all we need to do is iteratively solve the following formula:
$$z_{n+1} = z_n^2 + c$$
Every iteration, we reduce pixel color by 5 and recalculate z until either the pixel color is 0 or $|z|$ exceeds a provided cutoff value (somewhat arbitrary). It should be noted here that $c$ is entirely complex and that the value dratically changes the output fractal.

Create the following:
\begin{enumerate}
\item A function that sweeps through different $c$ values and outputs an image for each value
\item A function that zooms into a fractal of $c=1.0$ to see it's self-similar nature
\end{enumerate}

If you want to, you can even combine the images into a gif by using some additional tool (like ffmpeg)
\begin{figure}
\begin{center}
\includegraphics[width=0.45\textwidth]{fractal_zoom00030.png}
\includegraphics[width=0.45\textwidth]{c_scan00030.png}
\end{center}
\end{figure}

\newpage
\section*{Force Integration}
Integrating forces is a staple of any physics engine and most physics simulations. Though there are many different algorithms to do this, here we focus on the Verlet algorithm (because it's conceptually simple); however, if you want to implement Runge-Kutta (or whatever), feel free to do so!

First, let's start with the Taylor Series expansion:

$$x = x_0 + v_0 t + \frac{1}{2}at^2 + \ldots$$

If we are looking for $x(t + \Delta t)$, we can start by looking at the timesteps immediately before it:
$$x(t+\Delta t) = x(t) + v \cdot \Delta t + \frac{1}{2} a\cdot\Delta t^2$$
$$x(t-\Delta t) = x(t) - v \cdot \Delta t + \frac{1}{2} a\cdot\Delta t^2$$

Adding these together and solving for $x(t + \Delta t)$, we get:

$$x(t + \Delta t) = 2x(t) - x(t - \Delta t) + a \Delta t^2$$

This means we can solve for any object's new position if we know it's acceleration and where it's been! Of course, if we need the velocity (possibly to solve for the acceleration of a damped system), we can find it with:

$$v(t) = \frac{x(t + \Delta t) - x(t - \Delta t)}{2\Delta t}$$

Now, let's integrate some forces! Start with a ball at a height of 5m. How long does it take to hit the ground? Also, plot it's trajectory with time.

\newpage
\section*{Barnes-Hut}
Using the previous section on force integration, we know that all we really need for an N-body gravity simulator is a method to calculate the acceleration of an object in space. In principle, this is straightforward: We just add up all the particles and calculate

$$a = \frac{GMm}{r^2}$$

In practice, this means that we have an $O(n^2)$ problem on our hands! That said, we can take our N-body simulation and speed it up drastically with the help of a rather famous data structure known as an octree (or quadtree in 2d).

These data structures are straightforward. The root node is the entire simulation box, it's children are the octants of that box. Each octant has 8 octchildren, and we keep splitting up the simulation until every particle has it's own box. At this point, it's not obvious how this speed anything up; however, each box has a Center of Mass equal to all the elements on the inside of it. If our objects are sufficiently far away from each other, we can take a larger parent box instead of a smaller child box. This cuts computational time tremendously!

In particular, the cutoff value is defined as $\theta = s/d$ where $s$ is the width of the region represented by the internal node and $d$ is the distance between the body and it's Center of Mass. By playing with $\theta$, we can have either a fast, inaccurate simulation or a slow, accurate one. The trick is finding the happy-medium between the two!

\newpage
\section*{FFT}
FFT's are everywhere nowadays. In fact, we used it previously in this document when talking about the Split-Operator method. For the most part, FFT's use an implementation of the Cooley-Tukey algorithm, which basically involves recursively dividing the array we wish to transorm into even and odd components and solving them with a slow Discrete Fourier Transform (DFT) when we get below a certain number of elements.

By solving our system recursively, we can take an $O(n^2)$ process to $(n\log n)$, which is hughe savings! Full implementation details can be found here: \url{https://leios.gitbooks.io/algorithm-archive/content/chapters/computational_mathematics/cooley_tukey.html} (you might need to refresh the page to get the formulas to work)

\end{document}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Written By Michael Brodskiy
% Class: Analytic Geometry & Calculus III (Math-292)
% Professor: V. Cherkassky
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\documentclass[12pt]{article}
\usepackage{alphalph}
\usepackage[utf8]{inputenc}
\usepackage[russian,english]{babel}
\usepackage{titling}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{enumitem}
\usepackage{amssymb}
\usepackage[super]{nth}
\usepackage{everysel}
\usepackage{ragged2e}
\usepackage{geometry}
\usepackage{fancyhdr}
\geometry{top=1.0in,bottom=1.0in,left=1.0in,right=1.0in}
\newcommand{\subtitle}[1]{%
  \posttitle{%
    \par\end{center}
    \begin{center}\large#1\end{center}
    \vskip0.5em}%

}
\usepackage{hyperref}
\hypersetup{
colorlinks=true,
linkcolor=blue,
filecolor=magenta,
urlcolor=blue,
citecolor=blue,
}

\urlstyle{same}

\title{Lecture VIII Notes}
\date{\today}
\author{Michael Brodskiy\\ \small Professor: V. Cherkassky}

% Mathematical Operations:

% Sum: $$\sum_{n=a}^{b} f(x) $$
% Integral: $$\int_{lower}^{upper} f(x) dx$$
% Limit: $$\lim_{x\to\infty} f(x)$$

\begin{document}

\maketitle

\section{Projectile Motion $-$ 13.4}

Given an acceleration, $\overrightarrow{a}(t)$, one may find the velocity and position formulas. In projectile motion, the acceleration is always $\overrightarrow{a}(t) = g = 9.80665\left[\frac{m}{s^2}\right]$\\

From here, one may find that: $\Delta v = \int \overrightarrow{a}(t) dt \Longrightarrow v-v_o=\overrightarrow{a}(t)t\Longrightarrow \overrightarrow{v}(t)=v_o+\overrightarrow{a}(t)t$\\

Furthermore, one may find the position vector by integrating once more: $\Delta r = \int_{}^{}  v_o + \overrightarrow{a}(t)t dt \Longrightarrow r-r_o=v_ot+\frac{1}{2}\overrightarrow{a}(t)t^2\Longrightarrow \overrightarrow{r}(t)=r_o + v_ot+\frac{1}{2}\overrightarrow{a}(t)t^2$\\

In projectile motion, acceleration only acts in the vertical direction, and, therefore:
\begin{tabular}{c}

  $\overrightarrow{r}_y(t)=r_{oy}+v_{oy}t-\frac{1}{2}gt^2$ \\
  $\overrightarrow{v}_y(t)=v_{oy}+gt$ \\
  $\overrightarrow{r}_x(t)=r_{ox}+v_{ox}t$ \\
  $\overrightarrow{v}_x(t)=v_{ox}$

\end{tabular}

One may find the acceleration through a different method by using the curvature, $\kappa$. $\kappa = \frac{|\overrightarrow{T}'(t)|}{|\overrightarrow{r}'(t)|}\Longrightarrow |\overrightarrow{T}'(t)|=\kappa |\overrightarrow{v}(t)|$\\
Simplifying this leaves us with: $\overrightarrow{a}=|\overrightarrow{v}'|\overrightarrow{T} + \kappa |\overrightarrow{v}|^2 \overrightarrow{N}$\\
This makes sense, because on a straight line, $\kappa = 0$, and, therefore, on a straight line, $\overrightarrow{a}=|\overrightarrow{v}'|\overrightarrow{T}$\\

The components of acceleration, $a_N$ and $a_T$ can be found using the formulas: $a_N=\frac{|\overrightarrow{r}'(t)\text{ x } \overrightarrow{r}''(t)|}{|\overrightarrow{r}'(t)|}$ and $a_T = \frac{\overrightarrow{r}'(t)\cdot \overrightarrow{r}''(t)}{|\overrightarrow{r}'(t)|}$

\end{document}


\section{Introduction}
\label{sec:introduction}

Perceptrons are devices used to solve binary classification problems.
A perceptron tries to learn a hyperplane that divides the training examples in $2$ groups, depending on their label.
In general, if the dataset is linear separable, there is an infinite number of hyperplanes that perfectly separates the examples based on their labels.
The MinOver algorithm \cite{minover} tries to find the hyperplane with the maximum stability, i.e. maximize the distance of the closest examples from the hyperplane.

In this assignment, we implement the MinOver training algorithm and measured its generalization error on a linearly separable dataset as a function of the rate between the number of examples and their dimension.
We also compare its performances with the Rosenblatt algorithm \cite{rosenblatt} implemented in the previous assignment.
Finally, we introduce different degrees of noise in the datasets and observe the performances of both algorithms.

\cref{sec:theory} introduces the MinOver training algorithm.
\cref{sec:implementation} describes the implementation of the various experiments.
\cref{sec:evaluation} discusses the obtained results.
Finally, \cref{sec:conclusion} summarizes the most interesting results.


% This is part of the TFTB Reference Manual.
% Copyright (C) 1996 CNRS (France) and Rice University (US).
% See the file refguide.tex for copying conditions.

\markright{zak}
\section*{\hspace*{-1.6cm} zak}

\vspace*{-.4cm}
\hspace*{-1.6cm}\rule[0in]{16.5cm}{.02cm}
\vspace*{.2cm}

{\bf \large \sf Purpose}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
Zak transform.
\end{minipage}
\vspace*{.5cm}

{\bf \large \sf Synopsis}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
\begin{verbatim}
dzt = zak(sig)
dzt = zak(sig,N)
dzt = zak(sig,N,M)
\end{verbatim}
\end{minipage}
\vspace*{.5cm}

{\bf \large \sf Description}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
        {\ty zak} computes the Zak transform of a signal. Its definition is
given by
\[Z_{sig}(t,\nu)=\sum_{n=-\infty}^{+\infty} sig(t+n)\ e^{-j2\pi n\nu}.\]

\hspace*{-.5cm}\begin{tabular*}{14cm}{p{1.5cm} p{8.5cm} c}
Name & Description & Default value\\
\hline
        {\ty sig} & Signal to be analyzed {\ty (length(sig)=N1)}\\
        {\ty N}   & number of Zak coefficients in time ({\ty N1} must be a multiple
              of {\ty N})          & {\ty divider(N1)}\\
        {\ty M}   & number of Zak coefficients in frequency ({\ty N1} must be a
              multiple of {\ty M}) & {\ty N1/N}\\
 \hline {\ty dzt} & Output matrix {\ty (N,M)} containing the discrete Zak transform\\

\hline
\end{tabular*}

\end{minipage}
\vspace*{1cm}

{\bf \large \sf Example}
\begin{verbatim}
         sig=fmlin(256);
         DZT=zak(sig);
         imagesc(DZT);
\end{verbatim}
\vspace*{.5cm}

{\bf \large \sf See Also}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
\begin{verbatim}
izak, tfrgabor.
\end{verbatim}
\end{minipage}
\vspace*{.5cm}

{\bf \large \sf Reference}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
[1] L. Auslander, I. Gertner, R. Tolimieri, ``The Discrete Zak Transform
Application to Time-Frequency Analysis and Synthesis of Nonstationary
Signals'' IEEE Trans. on Signal Proc., Vol. 39, No. 4, pp. 825-835, April
1991.
\end{minipage}


\documentclass{article}

\usepackage[a4paper, total={6in, 9in}]{geometry}
\usepackage{amsmath, amssymb, graphicx, blindtext, tabularx, caption, hyperref, blindtext, physics, amsthm, wrapfig, subcaption}
\usepackage[noend, ruled]{algorithm2e}
\usepackage[square,numbers]{natbib}
\title{A vertical federated learning algorithm for classfication problems with gradient-based optimization}
\author{Zhu Xiaochen \qquad National University of Singapore}
\date{}
\begin{document}
\maketitle
\section{The proposed \texttt{vFedCCE} algorithm}\label{secvfedcce}
  To apply the general vertical federated learning architecture to the setting of classification problem, we studied the categorical cross entropy loss function to deploy the gradient-based optimizer on the clients, instead of the centralized server, in order to preserve privacy. Following this method, we proposed the following algorithm of vertical federated learning with categorical cross entropy loss (\texttt{vFedCCE}). To simplify, we assume that the two clients have already aligned their datasets and train on a shared ID space $I$, which is the result after the preprocessing of private entity resolution.

  Before presenting the algorithm, we need to clarify some of the notations that will be used later in this section. For an $m\times n$ matrix $x$, $x_i$ is its $i$-th row vector and $x_{ij}$ is its entry at $i$-th row and $j$-th column, which means that $x_i=\langle x_{i1},x_{i2},\ldots,x_{in}\rangle$. For real matrices $x,y$ of the same shape, $x\oslash y$ is the Hadamard division (element-wise division) of $x$ and $y$, that is, $(x\oslash y)_{ij}={x_{ij}}/{y_{ij}}$; $\langle x,y\rangle_\mathrm{F}$ is the Frobenius inner product (element-wise dot product) of $x$ and $y$, that is $\langle x,y\rangle_\mathrm{F}=\sum_{i,j}x_{ij}y_{ij}$. With these notations, the algorithm \texttt{vFedCCE} is as follows:

  \noindent\begin{minipage}{\textwidth}
    \begin{algorithm}[H]
    \caption{\texttt{vFedCCE}. $m$-class classification. Shared ID space $I$. Epoch number $E$. Batch size $N$. Batch examples indexed as $x_i^1,x_i^2,y_i$, $i\in\{1,\ldots,N\}.$ $w_1,w_2$ are the client weight vectors. Client 2 stores $y_i\in\mathbb{R}^m$ as one hot arrays of length $m$.}
    \label{vflalgo}
    \DontPrintSemicolon
    \SetAlgoNoLine
    \SetArgSty{textnormal}
    server executes:\;
    \For {each epoch $e=1,2,\ldots, E$} {
      $\mathcal{B}\leftarrow$ divide $I$ into batches of size $N$\;
      \For{batch in $\mathcal{B}$}{
        client 1 sends $a\leftarrow\begin{bmatrix} w_1x_1^1& w_1x_2^1& \ldots &w_1x_N^1 \end{bmatrix}^T$ and $\frac{\partial a_{ij}}{\partial w_1}$ to client 2\;
        server logs $\mathcal{L}\leftarrow$ client2.CalculateLossAndUpdate($a$)\;
        $g\leftarrow$ client2.AssembleGradient($\frac{\partial a_{ij}}{\partial w_1}$)\;
        client1.UpdateWithGradient($g$)\;
      }
    }\;

    \textbf{CalculateLossAndUpdate}$(a)$:\quad // client 2 executes\;
    $b\leftarrow\begin{bmatrix} w_2x_1^2& w_2x_2^2& \ldots &w_2x_N^2 \end{bmatrix}^T$\;
    shared model output $p\leftarrow(a+b)/2$\;
    $\mathcal{L}=-\frac{1}{N}\sum_{i=1}^{N}y_i\cdot \log(p_i)=-\frac{1}{N}\sum_{i=1}^N\sum_{j=1}^m y_{ij}\log(p_{ij})$\;
    $c\leftarrow\begin{bmatrix}y_1&y_2&\ldots&y_N\end{bmatrix}^T\oslash p$\;
    client2.UpdateWithGradient$\left(\frac{\partial}{\partial w_2}\left(-\frac{1}{N}\left\langle c,b\right\rangle_\mathrm{F}\right)\right)$\;
    \Return{$\mathcal{L}$}\;\;

    \textbf{AssembleGradient}($\frac{\partial a_{ij}}{\partial w_1}$):\quad // client 2 executes\;
    \Return{$-\frac{1}{N}\sum_{i=1}^N\sum_{i=1}^mc_{ij}\frac{\partial a_{ij}}{\partial w_1}$}\;\;

    \textbf{UpdateWithGradient}($g$):\quad // client 1 or 2 executes\;
    update client weights via a specified optimizer with gradient $g$\;
    \end{algorithm}
\end{minipage}

\begin{wrapfigure}{r}{0.6\textwidth}
  \centering
  \includegraphics[width=0.6\textwidth]{./images/vfedcce_diagram2.png}
  \caption{Sequence diagram for \texttt{vFedCCE}}
  \label{vflseqdia}
\end{wrapfigure}

\paragraph{Correctness.} The correctness of \texttt{vFedCCE} depends on the gradients calculated or assembled locally on clients to update their respective weight vectors.

\begin{proof}
We study a batch of $N$ examples, where features $x_1^1,x_2^1,\ldots,x_N^1$ are stored on client 1, features $x_1^2,x_2^2,\ldots,x_N^2$ and labels $y_1,y_2,\ldots,y_N$ are stored on client 2 as one-hot arrays in $\mathbb{R}^m$, where $m$ is the number of classes to predict. On a single example $i$, the probability distribution output of client 1 on its feature set $x_i^1$ will be $a_i=w_1x_i^1$ and the probability distribution output of client 2 on its feature set $x_i^2$ will be $b_i=w_2x_i^2$. For the shared model, we aim to train the model such that the average output of the client outputs converges to the actual labels $y_i$. Therefore, the predicted probability distribution of the shared model will be $p_i=\frac{1}{2}(a_i+b_i)$. Note that all probability distributions, i.e. $a_i,b_i,y_{\text{pred},i}$ are vectors in $\mathbb{R}^m$.

Therefore, on this batch of $N$ examples, probability distribution outputs of client 1, client 2 and the shared model on them will be $N\times m$ matrices: $a=\begin{bmatrix}a_1&a_2&\ldots&a_N\end{bmatrix}^T$, $b=\begin{bmatrix}b_1&b_2&\ldots&b_N\end{bmatrix}^T$ and $p=\frac{1}{2}(a+b)$. On the other hand, the actual labels $y$ stored on client 2 is also a $N\times m$ matrix where $y=\begin{bmatrix}y_1&y_2&\ldots&y_N\end{bmatrix}^T$.

As previously mentioned, the shared model aims to converge $p$ towards $y$, and therefore, the optimization goal is to minimize the categorical cross entropy loss $p$ against $y$, which is
$$
  \mathcal{L}=-\frac{1}{N}\sum_{i=1}^N y_i \cdot \log p_i = -\frac{1}{N}\sum_{i=1}^N \sum_{j=1}^m y_{ij}\log p_{ij}=-\frac{1}{N}\sum_{i=1}^s \sum_{j=1}^m (y_i)_j\log \frac{(a_i)_j+(b_i)_j}{2}.
$$
When using a gradient-based optimizer to minimize $\mathcal{L}$, one must calculate the gradients $\frac{\mathcal{L}}{\partial w_1}$ and $\frac{\mathcal{L}}{\partial w_2}$. Note that only $a_{ij}$ instead of $b_{ij}$ or $y_{ij}$ is a function of $w_1$, then by calculus chain rule we have
\begin{equation}
  \label{client1}
  \frac{\partial\mathcal{L}}{\partial w_1}=-\frac{1}{N}\sum_{i=1}^N \sum_{j=1}^m y_{ij}\frac{2}{a_{ij}+b_{ij}}\frac{\partial a_{ij}}{\partial w_1}=-\frac{1}{N}\sum_{i=1}^N \sum_{j=1}^m\frac{y_{ij}}{p_{ij}}\frac{\partial a_{ij}}{\partial w_1}.
\end{equation}
Similarly, for client 2,
\begin{equation}
  \label{client2}
  \frac{\partial\mathcal{L}}{\partial w_2}=-\frac{1}{N}\sum_{i=1}^N \sum_{j=1}^m y_{ij}\frac{2}{a_{ij}+b_{ij}}\frac{\partial b_{ij}}{\partial w_2}=-\frac{1}{N}\sum_{i=1}^N \sum_{j=1}^m\frac{y_{ij}}{p_{ij}}\frac{\partial b_{ij}}{\partial w_2}.
\end{equation}

Note that for privacy reasons, the labels $y$ stored on client 2 are not allowed to leave client 2. Therefore, we cannot directly send $y$ to client 1 in order to complete the calculations from (\ref{client1}) entirely locally on client 1. However, it is possible to send example-wise partial gradients $\frac{\partial a_{ij}}{\partial w_1}$ to client 2 to assemble into the total gradient $\frac{\partial\mathcal{L}}{\partial w_1}$ with the assistance of $y$ and $p$, which is what the function \texttt{AssembleGradient()} does in Algorithm \ref{vflalgo}.

As for client 2, since the labels $y$ are originally stored there and client 1's model output $a$ is safe to send, it can complete the calculations from (\ref{client2}) entirely locally. If we denote the element wise division $y\oslash p$ as a constant matrix $c$, client 2's gradient can be calculated as follows:

\begin{align*}
  \frac{\partial\mathcal{L}}{\partial w_2}&=-\frac{1}{N}\sum_{i=1}^N \sum_{j=1}^m\frac{y_{ij}}{p_{ij}}\frac{\partial b_{ij}}{\partial w_2}=-\frac{1}{N}\sum_{i=1}^N \sum_{j=1}^m c_{ij}\frac{\partial b_{ij}}{\partial w_2}\\
  &=\frac{\partial}{\partial w_2}\left(-\frac{1}{N}\sum_{i=1}^N\sum_{j=1}^m c_{ij} b_{ij}\right)
  =\frac{\partial}{\partial w_2}\left(-\frac{1}{N}\left\langle c,b\right\rangle_\mathrm{F}\right)
\end{align*} which is what the function \texttt{CalculateLossAndUpdate()} does in Algorithm \ref{vflalgo} to calculate the gradient for client 2.

Therefore, the gradients calculated locally on the clients in Algorithm \ref{vflalgo} equals the actual gradients of the loss funciton w.r.t. $w_1$ and $w_2$ respectively. With a specified gradient-based optimizer, the output of the shared model, i.e. the average model of clients' models, $p$ will eventually converge to $y$ to achieve desired learning outcomes.
\end{proof}

\paragraph{Theoretical performance.} Unlike \cite{hardy2017private} where a Taylor approximation of the loss function is used to apply vertical federated learning, \texttt{vFedCCE}, according to the previous proof, calculates the accurate gradients without approximation. Therefore, theoretically \texttt{vFedCCE} should produce the same training outcomes (i.e. accuracy et al) compared to a conventional machine learning algorithm with the same model over a combined dataset.

Except preserving privacy, one major advantage of vertical federated learning is to reduce communication cost by avoiding sending huge datasets from the clients to the server to combine and then apply conventional deep learning over it. Although compared with the VFL example from \cite{yang2019federated} which only sends local gradients $\frac{\partial L}{\partial w}$ (it is possible because they are dealing with a linear regression model and an $L^2$ regularized MSE loss function), much more amount of intermediate data is being sent in Algorithm \ref{vflalgo} when sending $\frac{\partial a_{ij}}{\partial w_1}$, we expect the communication cost will still be reduced when the dataset is too huge to efficiently send and combine on the server. This is because on each batch, \texttt{vFedCCE} sends $Nm$ different partial gradients $\frac{\partial a_{ij}}{\partial w_1}$, whose amount of data is independent of the number of features in the datasets.

One potential issue with Algorithm \ref{vflalgo} is that, when assembling client 1's gradient from all the partial gradients $\frac{\partial a_{ij}}{\partial w_1}$, a lot of floating point arithmetic calculation is being carried out and manually summed up, which may lead to a precision problem compared with a conventional approach. This concern has been dismissed by the experimental results shown later in section \ref{vflexp}.

\paragraph{Privacy.} To analyze whether Algorithm \ref{vflalgo} preserves privacy, we need to evaluate the data being sent across the clients, which, referring to Figure \ref{vflseqdia}, is:
\begin{itemize}
  \item Client 1's model output $a=\begin{bmatrix} w_1x_1^1& w_1x_2^1& \ldots &w_1x_N^1 \end{bmatrix}^T$. This data is safe and necessary to send as the shared model needs to average the two outputs for predictions.
  \item Client 1's partial gradients $\frac{\partial a_{ij}}{\partial w_1}$. This is safe to send as client 2 cannot restore the $x$ values from partial derivatives $\frac{\partial a_{ij}}{\partial w_1}$. Also, gradients are being sent in other literature such as \cite{yang2019federated} and \cite{hardy2017private}.
  \item Client 1's total gradient, sent from client 2 to client 1. This is also safe and necessary to sent as this is the gradient client 1 needs to update its own weight vectors.
\end{itemize}

Although Algorithm \ref{vflalgo} only sent secure intermediate data instead of sensitive raw data, it is still important to discuss the feasibility of applying some encryption scheme on top of \texttt{vFedCCE}. In our case, since all the intermediate data (i.e. $a$ and $\frac{\partial a_{ij}}{w_1}$) is sent for linear operations, it is possible to apply additively homomorphic encryption on those data, such that client 2 receives encrypted values, applies linear operations on the encrypted values and returns the encrypted assembled gradient to client 1, without even knowing the decrypted values of the data it is being sent. More details on encryption will be discussed later.

\section{Implementation and experimental results}\label{vflexp}
\paragraph{Implementation details.} The proposed \texttt{vFedCCE} algorithm is implemented in Python with the TensorFlow Keras library on the vertically partitioned SGCP dataset from \cite{groemping2019south}. In the experiments, the local models on client 1 and client 2 are the same neural network model:
\begin{itemize}
  \setlength\itemsep{0em}
  \item one normalizer layer;
  \item two fully connected L2 regularized dense layers with dropout activated by \texttt{elu};
  \item one more dense layer of unit $2$ to convert output into logits;
  \item one \texttt{Softmax} layer to output probability distributions.
\end{itemize}
This neural network model is chosen because it leads to acceptable results with a conventional approach on the combined dataset, denoted by \texttt{combinedNN}, which is the baseline method to benchmark the performance of \texttt{vFedCCE}.

For hyperparameters, no deliberate tuning is being applied since the goal of experiments is to verify that the learning outcome of \texttt{combinedNN} and \texttt{vFedCCE} are similar, instead of to achieve perfect performance. Both methods are implemented with the same hyperparameters: the learning rate (of Adam optimizer) $\eta=0.001$, the number of epochs $E=50$ and the batch size $N=32$.

\paragraph{Experimental results.} Figure \ref{fig:vflexp} demonstrates the training loss and accuracy of \texttt{combinedNN} and \texttt{vFedCCE} under the implementation specified previously. It shows that the two methods are able to minimize loss and increase accuracy over epochs, with similar speed. Table \ref{vfltest} presents the learning outcome of the two methods, which, expectedly, shows both similar and acceptable accuracy, precision, recall, F-score and AUC values. This also dismisses the concern on floating-point precision raised in section \ref{secvfedcce}.

\begin{figure}[ht]
\centering
\begin{subfigure}{.5\textwidth}
  \centering
  \includegraphics[width=\linewidth]{./images/vfl_train_loss.eps}
  \caption{Training loss over 50 epochs}
  \label{fig:vflloss}
\end{subfigure}%
\begin{subfigure}{.5\textwidth}
  \centering
  \includegraphics[width=\linewidth]{./images/vfl_train_acc.eps}
  \caption{Training accuracy over 50 epochs}
  \label{fig:vflacc}
\end{subfigure}
\caption{Training process of \texttt{vFedCCE} against \texttt{combinedNN}}
\label{fig:vflexp}
\end{figure}

\begin{table}[ht]
\centering
  \resizebox{0.7\textwidth}{!}{%
  \begin{tabular}{|c|c|c|c|c|c|}
    \hline
                        & Accuracy & Precision & Recall & F-score & AUC    \\ \hline
    \texttt{combinedNN} & 0.74     & 0.7881    & 0.8561 & 0.9207  & 0.7720 \\ \hline
    \texttt{vFedCCE}    & 0.74     & 0.7669    & 0.8993 & 0.8278  & 0.7561 \\ \hline
  \end{tabular}%
  }
\caption{Learning outcome on testing data of \texttt{vFedCCE} against \texttt{combinedNN}}
\label{vfltest}
\end{table}

\bibliographystyle{abbrvnat}
\bibliography{references}
\end{document}