%!TEX root = ../06DL.tex

% Stochastic and Incremental Gradient Methods by Junchi Li
\section{Problem Set-Up}
The problem set up is more the less the same as in \cite{Hazan2011} as are the derivations. We want to minimize a function
\begin{equation}\label{eq:origpro}
\min_x \mathbb{E}_\xi [F(x,\xi)]+P(x)
\end{equation}
but we only get access to subgradients $g(x,\xi)$ of $F(x,\xi)$ with $\xi$ sampled at random. Examples of this set-up include
\begin{enumerate}
\item {\bf Noisy gradients.} We want to minimize a smooth function $f(x)$. At every iteration, we compute or gain access to a noisy gradient $g_k=\nabla f(x_k)+\omega_k$ where $\omega_k$ is some zero-mean noise process which is independent of $x_k$.
\item {\bf Incremental gradients.} We want to minimize a function of the form
$$
f(x)=\sum_{i=1}^{m}f_i(x)
$$
At every iteration, we choose a random index $i_k$ uniformly at random from $\{1,\ldots,m\}$, and we take a step along the gradient of $f_{i_k}$ rather than of the full function $f$. This is obviously faster to compute when $m$ is large. When does this approach find a minimum of $f$?
\end{enumerate}

Throughout we assume
\begin{enumerate}
\item $f(x):=\mathbb{E}[F(x,\xi)]$ is differentiable and strongly convex. So there exists a constant $l>0$ such that
\begin{equation}
f(z)\geq f(x)+\nabla f(x)^\top (z-x)+\frac{l}{2}\|z-x\|^2
\end{equation}
\item $\nabla f$ is Lipschitz so that $\|\nabla f(x) - \nabla f(y)\|\leq L \|x-y\|$.
\item $P(x)$ is a convex extended real valued function.
\end{enumerate}
Note that the results apply to the case where there is only one value of $\xi$. That is, the non-stochastic setting. In this case we would have a differentiable convex function plus an arbitrary convex function. Also note that we can enforce the constraint $x \in X$ for some convex set $X$ by letting $P(x)=0$ for $x\in X$ and $P(x)=\infty$ for $x\notin X$.

Let us define a stochastic projected gradient scheme to solve this problem. Let
\begin{equation}\label{eq:stoprojgra}
{\rm prox}_{\nu P}(z)={\rm argmin}_x \|x-z\|^2+\nu P(x)
\end{equation}
Let $\gamma_0,\ldots,\gamma_T,\ldots$ be a sequence of positive numbers. Choose $x_0 \in X$, and iterate
\begin{equation}\label{eq:stoprojgraiter}
x_{k+1} ={\rm prox}_{\gamma_k P}(x_k-\gamma_k G(x_k,\xi_k))
\end{equation}

\section{Analysis of Unconstrained Stochastic Gradient Descent.}
First, let's examine the case with $P = 0$ and let's make no assumptions about strong convexity. Assume $\|G(x, \xi)\| \leq M$ for all $x$ and $\xi$. Let $x_*$ denote any optimal solution of \eqref{eq:origpro}. Then we have
\begin{eqnarray}
\mathbb{E}[\|x_{k+1}-x_*\|^2]&=&\mathbb{E}[\|x_{k}-\gamma_k G(x_k,\xi_k)-x_*\|^2]  \nonumber \\
&=&\mathbb{E}[\|x_{k}-x_*\|^2]- 2\gamma_k\mathbb{E}[\langle G(x_k,\xi_k), x_k-x_*\rangle ]+ \gamma_k^2 \mathbb{E}[\|\gamma_k G(x_k,\xi_k)\|^2]  \\
&\leq&\mathbb{E}[\|x_{k}-x_*\|^2]- 2\gamma_k\mathbb{E}[\langle G(x_k,\xi_k), x_k-x_*\rangle ]+ \gamma_k^2 M^2  \\
&=&\mathbb{E}[\|x_{k}-x_*\|^2]- 2\gamma_k\mathbb{E}[\langle \nabla f(x_k), x_k-x_*\rangle ]+ \gamma_k^2 M^2 \label{eq:eqnproof5} \\
&\leq& \mathbb{E}[\|x_{k}-x_*\|^2]- 2\gamma_k\mathbb{E}[f(x_k)-f(x_*)]+ \gamma_k^2 M^2 \label{eq:eqnproof6}
\end{eqnarray}
\eqref{eq:eqnproof5} follows because
\begin{eqnarray}
\mathbb{E}[\langle G(x_k,\xi_k), x_k-x_*\rangle ]&=&\mathbb{E}_{\xi_0,\ldots,\xi_{k-1}}[\mathbb{E}_{\xi_k}[\langle G(x_k,\xi_k), x_k-x_*\rangle\ |\ \xi_0,\ldots,\xi_{k-1} ]] \nonumber\\
&=&\mathbb{E}_{\xi_0,\ldots,\xi_{k-1}}[\langle \nabla f(x_k), x_k-x_*\rangle\ |\ \xi_0,\ldots,\xi_{k-1} ] \\
&=&\mathbb{E}[\langle \nabla f(x_k), x_k-x_*\rangle]
\end{eqnarray}
by the law of iterated expectation. \eqref{eq:eqnproof6} is a consequence of the inequality
\begin{equation}
\langle \nabla f(x_k), x_k-x_*\rangle \geq f(x_k)-f(x_*)
\end{equation}
which holds because $f$ is convex.

Arranging the bound: by telescoping $k$ from $0$ to $n$,
\begin{equation}
2\gamma_k\mathbb{E}[f(x_k)-f(x_*)] \leq -\mathbb{E}[\|x_{k+1}-x_*\|^2-\|x_{k}-x_*\|^2] + \gamma_k^2 M^2,
\end{equation}
we have since $\|x_{n+1}-x_*\|^2\geq 0$
\begin{equation}
2\sum_{k=0}^n \gamma_k\mathbb{E}[f(x_k)-f(x_*)] \leq -\mathbb{E}[\|x_{n+1}-x_*\|^2-\|x_{0}-x_*\|^2] + M^2 \sum_{k=0}^{n} \gamma_k^2\leq D^2+M^2  \sum_{k=0}^{n} \gamma_k^2.
\end{equation}
Dividing by the sum of the $\gamma_k$, we have for any $n$
\begin{equation}
\frac{1}{\sum_{k=0}^n \gamma_k}\sum_{k=0}^n \gamma_k\mathbb{E}[f(x_k)-f(x_*)] \leq \frac{D^2+M^2  \sum_{k=0}^{n} \gamma_k^2}{2\sum_{k=0}^n \gamma_k}.
\end{equation}
where $D=\|x_0-x_*\|$. Let $\bar{x}:=(\sum_{k=0}^n \gamma_k)^{-1}\sum_{k=0}^n \gamma_k x_k$. Then, by convexity (Jensen's inequality), we have
\begin{equation}
\mathbb{E}[f(\bar{x})-f(x_*)] \leq \frac{D^2+M^2  \sum_{k=0}^{n} \gamma_k^2}{2\sum_{k=0}^n \gamma_k}.
\end{equation}
This is precisely the bound rate of convergence for deterministic subgradient descent.

\section{Analysis of Projected Stochastic Gradient.}
In the case of constrained optimization problem, recall from \eqref{eq:stoprojgra} and \eqref{eq:stoprojgraiter} that $\Pi_X (z) ={\rm argmin}_{x\in X} \|x - z\|^2$ and the iteration updates
\begin{equation}
x_{k+1}=\Pi_X (x_k-\gamma_k G(x_k, \xi_k)).
\end{equation}
Let $x_*$ denote the optimal solution of \eqref{eq:origpro}. $x_*$ is unique because of strong convexity. Observe that
\begin{eqnarray}
\mathbb{E}[\|x_{k+1}-x_*\|^2]&=&\mathbb{E}\left[\|\Pi_{\gamma_k} (x_{k}-\gamma_k G(x_k,\xi_k))-\mathbb{E}[\|\Pi_{\gamma_k}(x_*-\gamma_k \nabla f(x_*))\|^2\right]  \nonumber \\
&\leq&\mathbb{E}[\|x_{k}-\gamma_k G(x_k,\xi_k)-x_*+\gamma_k \nabla f(x_*)\|^2]  \label{eq:eqproof13} \\
&=&\mathbb{E}[\|x_{k}-\gamma_k \nabla f(x_k)+\gamma_k (\nabla f(x_k)-G(x_k,\xi_k))-x_*+\gamma_k \nabla f(x_*)\|^2]  \label{eq:eqproof14} \\
&=& \mathbb{E}\left[ \| x_k -\gamma_k \nabla f(x_k) - x_* +\gamma_k \nabla f(x_*)\|^2\right]  \\
&& +2\gamma_k\mathbb{E}[\langle x_k- \gamma_k \nabla f(x_k) - x_* + \gamma_k \nabla f(x_*), \nabla f(x_k)-G(x_k,\xi_k)\rangle ] \nonumber \\
&&+\gamma_k^2 \mathbb{E}[\|\nabla f(x_k) - G(x_k,\xi_k)\|^2] \nonumber \\
&=&\mathbb{E}[\|x_{k}-\gamma_k \nabla f(x_k)-x_*+\gamma_k \nabla f(x_*)\|^2]+ \gamma_k^2 \mathbb{E}[\|\nabla f(x_k)-G(x_k,\xi_k)\|^2] \label{eq:eqnproof15}
\end{eqnarray}
Here, the first equality follows by the definition of $x_{k+1}$ and because $x_*$ is optimal. \eqref{eq:eqproof13} follows because the proximity operator is non-expansive. \eqref{eq:eqproof14} follows because $\mathbb{E}[G(z,\xi_k)] = \nabla f(z)$ for all $z$ and $G(z,\xi_k)$ is independent of $\xi_k$. Thus we have
\begin{eqnarray}
&&\mathbb{E}[\langle \nabla f(x_k)-G(x_k,\xi_k), x_k-\gamma_k\nabla f(x_k)-x_*+\gamma_k \nabla f(x_*) \rangle] \\
&=& \mathbb{E}_{\xi_0,\ldots,\xi_{k-1}}[\mathbb{E}_{\xi_k}[\langle \nabla f(x_k)-G(x_k,\xi_k), x_k-\gamma_k\nabla f(x_k)-x_*+\gamma_k \nabla f(x_*) \rangle \\
 && |\ \xi_0,\ldots,\xi_{k-1}] \nonumber\\
&=& 0.
\end{eqnarray}

Note that the first term in \eqref{eq:eqnproof15}  is completely independent of $\xi_k$ while the second term is a variance term concerning the second moments of the subgradients at the current iterate and at the optimum. We can bound each of these terms separately. First, since $f$ is strongly convex and has a Lipschitz continuous gradient, it follows that
\begin{equation} \label{eq:eqproof16}
\mathbb{E}[\| x_k-\gamma_k\nabla f(x_k)-x_*+\gamma_k \nabla f(x_*)\|^2] \leq \max\{ |1-\gamma_k L|, |1-\gamma_k l |\}^2 \mathbb{E}[\|x_k-x_*\|^2]
\end{equation}
For the second term, we must make some assumption about the statistics of the random function
\begin{equation} \label{eq:eqproof17}
\varphi(z;\xi_k):=G(z,\xi_k)-\nabla f(z)
\end{equation}

Let's explore some possibilities.
\subsection{$\varphi\equiv 0$}
In the case when there is no randomness at all and we are just following the gradient,
we only need upper bound \eqref{eq:eqproof16}. In this case, setting $\gamma_k = \frac{2}{L+l}$ for all $k$, we find that
\begin{equation} \label{eq:eqproof18}
\|x_{k+1}-x_*\|\leq \left(\frac{L-l}{L+l}\right) \|x_k-x_*\|
\end{equation}
or, letting $\kappa=\frac{L}{l}$ and $D_0=\| x_0-x_* \|$,
\begin{equation} \label{eq:eqproof19}
\|x_{k}-x_*\|\leq \left(\frac{\kappa-1}{\kappa+1}\right)^k D_0
\end{equation}

\subsection{$\varphi$ bounded.}
The simplest non-trivial assumption is that the deviations are bounded:
\begin{equation} \label{eq:eqproof20}
\|\varphi(z;\xi_k)\|\leq M
\end{equation}
for some universal constant $M$. In this case, we have the upper bound
\begin{eqnarray}
\mathbb{E}[\|x_{k+1}-x_*\|^2] &\leq& \max \{ |1-\gamma_k L|, |1-\gamma_k l| \}^2 \mathbb{E}[\|x_k-x_*\|^2] +\gamma_k^2 M^2 \label{eq:eqproof21}\\
&\leq& (1-2\gamma_k l+\gamma_k^2 L^2) \mathbb{E}[\|x_k-x_*\|^2]+\gamma_k^2 M^2 \label{eq:eqproof22}
\end{eqnarray}

With such a bound, we can achieve the so-called ``optima'' $O(1/k)$ rate by choosing
\begin{equation} \label{eq:eqproof23}
\gamma_k = \frac{1}{kl}
\end{equation}
Indeed, in this case, it follows by induction that
\begin{equation} \label{eq:eqproof24}
\mathbb{E}[\|x_k-x_*\|^2] = \frac{M^2+D_0^2 L^2}{kl^2}
\end{equation}
where $D_0$ again equals $\|x_0 - x_*\|$. To verify this inequality, note that for $k = 0$, the right hand is greater than $D_0^2$. Assuming that the inequality holds for $k \leq K$, observe
\begin{eqnarray}
\mathbb{E}[\|x_{K+1}-x_*\|^2] &\leq& \left( 1-\frac{2}{K} + \frac{L^2}{K^2 l^2} \right) \mathbb{E}[\|x_K-x_*\|^2] + \frac{M^2}{K^2 l^2} \\
&\leq& \left( 1-\frac{2}{K} \right) \mathbb{E}[\|x_K-x_*\|^2]+\frac{M^2+L^2 \mathbb{E} [\|x_K-x_*\|^2]}{K^2 l^2} \\
&\leq& \left( 1-\frac{2}{K} \right) \frac{M^2+D_0^2 L^2}{K l^2}+\frac{M^2+D_0
^2 L^2}{K^2 l^2} \\
&=& \frac{K^2-1}{K^2}\cdot \frac{M^2 +D_0^2 L^2}{(K+1) l^2} \leq \frac{M^2 +D_0^2 L^2}{(K+1)l^2}
\end{eqnarray}

NB: This matches the lower bound by {\color{red} agarwal2012information: Alekh Agarwal, Peter L. Bartlett, Member, IEEE, Pradeep Ravikumar, and Martin J. Wainwright, Senior Member, IEEE.}

\section{Introduction}
We want to minimize the function $f: \mathcal{C} \to \mathbb{R}$. Assume $f$ is convex and possibly strongly convex on $\mathcal{C}$. Assume that the minimum function value $f_*$ is attained in $\mathcal{C}$ and let $x_*$ denote any optimal solution. Let $D_0$ denote the distance from the initial iterate to this optimal solution: $D_0 = \|x_0 - x_*\|^2$.

Assume that the gradient of $f$ is Lipschitz with constant $L$ (i.e., $\|\nabla f(x)- \nabla f(y)\|^2 \leq L\|x-y\|^2$).
We may sometimes additionally assume that $f$ is strongly convex with parameter $l > 0$, so that
$f(y)\geq f(x)+\nabla f(x)^\top (y-x)+ \frac{l}{2}\|x-y\|^2_2$.

In stochastic gradient methods, we are allowed to query an oracle that takes a point in $\mathcal{C}$ and returns an unbiased estimate of the gradient $g(x)$. That is $\mathbb{E}[g(x)] = \nabla f(x)$. We assume the randomness is independent of the prior queries of the oracle. As is standard, assume that we know that $\|g(x)\|_2 \leq M$ almost surely for all $x \in \mathcal{C}$.

Let $\alpha_0,\ldots,\alpha_k,\ldots$, be a sequence of positive numbers. Choose $x_0 \in X$, and iterate
\begin{equation}
x_{k+1}=x_k-\alpha_k g(x_k).
\end{equation}

\section{Preliminaries}
Let us use this section to collect some standard results from prior art. Most of this material appeared in Nemirovski and Yudin \cite{Nemirovski1978, Nemirovski1983}. More contemporary treatments are provided by Nemirovski et al \cite{Nemirovski2009}, Nesterov \cite{Nesterov2009}, and Hazan and Kale \cite{Hazan2011}.

Suppose we run the incremental gradient method with stepsize $\alpha$ for $S$ iterations. Then note that
\begin{eqnarray}
\|x_{k+1}-x_*\|^2 &=& \| \Pi_{\mathcal{C}} (x_k-\alpha g_k (x_k)) - \Pi_{\mathcal{C}} (x_*) \|^2 \\
&\leq& \|x_k -\alpha g_k(x_k)-x_*\|^2 \\
&=& \|x_k -x_*\|^2 -2\alpha \langle g_k(x_k), x_k-x_* \rangle +\alpha^2 \|g_k (x_k)\|^2 \\
&\leq& \|x_k-x_*\|^2 -2\alpha \langle g_k(x_k), x_k-x_* \rangle +\alpha^2 M^2
\end{eqnarray}
Note that if we iterate the expectation
\begin{eqnarray}
\mathbb{E}[\langle g_k(x_k), x_k-x_* \rangle] &=& \mathbb{E}[\mathbb{E}[\langle g_k(x_k), x_k-x_* \rangle\ |\ g_0,\ldots,g_{k-1}]] \\
&=&\mathbb{E}[\mathbb{E}[\langle g_k(x_k) \ |\ g_0,\ldots,g_{k-1}], x_k-x_* \rangle] \\
&=& \mathbb{E}[\langle \nabla f(x_k), x_k-x_* \rangle].
\end{eqnarray}
Letting $a_k = \mathbb{E}[\| x_{k+1} - x_*\|^2]$, this gives
\begin{equation}\label{eq:eqproof30}
a_{k+1}\leq a_k -2 \alpha \mathbb{E}[\langle \nabla f(x_k), x_k-x_* \rangle ]+\alpha^2 M^2.
\end{equation}
We can now bound
\begin{eqnarray}
\mathbb{E}\left[ f \left( \frac{1}{S} \sum_{t=1}^S x_t \right) -f(x_*)\right] &\leq& \mathbb{E}\left[ \frac{1}{S} \sum_{t=1}^S f{x_t}  -f(x_*)\right] \label{eq:eqproof31}\\
&\leq& \frac{1}{S} \sum_{t=1}^S \mathbb{E}[\langle \nabla f(x_t), x_*-x_t \rangle] \label{eq:eqproof32}\\
&\leq& \frac{1}{S}\sum_{t=1}^S \frac{a_t-a_{t+1}}{2\alpha} +\frac{1}{2} \alpha M^2 \label{eq:eqproof33} \\
&=& \frac{a_0-a_S}{2S\alpha} +\frac{1}{2} \alpha M^2 \label{eq:eqproof34}\\
&\leq& \frac{D_0^2}{2S\alpha} +\frac{1}{2} \alpha M^2 \label{eq:eqproof35}\\
&\leq& \frac{f(x_0)-f_*}{lS\alpha}+\frac{1}{2} \alpha M^2 \label{eq:eqproof36}
\end{eqnarray}
Here, \eqref{eq:eqproof31} follows because f is convex, \eqref{eq:eqproof32} is the first order condition for convexity, \eqref{eq:eqproof33} uses \eqref{eq:eqproof30}, and \eqref{eq:eqproof36} uses the definition of strong convexity.

\section{Weakly convex case}
Suppose we run $E$ epochs of stochastic gradient, with epoch $k$ of length $2^{k-1}S$ and with stepsize $\alpha_k$. Let $\bar{x}_k$ denote the average of all of the iterates from epoch $k$. Let $T$ denote the total number of SGD steps. Nemirovski et al proved the following proposition for general convex $f$.

\begin{proposition} (Nemirovski et al \cite{Nemirovski2009}) Suppose we run $1$ epoch of stochastic gradient of length $S$ and with stepsize $\alpha$. Define
\begin{equation}
\theta_k:=\frac{M\sqrt{S}}{D_0} \alpha.
\end{equation}
Then we have the bound
\begin{equation}
\mathbb{E}[f(\bar{x}_1)-f_*] \leq \left( \frac{1}{2}\theta +\frac{1}{2}\theta^{-1} \right) \frac{MD_0}{\sqrt{T}}.
\end{equation}
That is, we pay linearly for errors in selecting the optimal constant stepsize. In the context of best ball, we refine our estimate of the optimal stepsize at each epoch. By doubling epochs, and always allowing $1/2$ the previous stepsize, best ball is able to tune this stepsize.
\end{proposition}
\begin{proof}
We can prove this just by minimizing \eqref{eq:eqproof35}. Plugging in our stepsize, we see that
\begin{equation}
\mathbb{E}[f(\frac{1}{S} \sum_{t=1}^S x_t)-f(x_*)] \leq \frac{D_0^2}{2S\alpha}+\frac{1}{2}\alpha M^2=\left( \frac{1}{2}\theta +\frac{1}{2}\theta_{-1} \right) \frac{MD_0}{\sqrt{T}}.
\end{equation}
\end{proof}

\begin{thebibliography}{999}
\bibitem{Hazan2011}
{E. Hazan and S. Kale}. {Beyond the regret minimization barrier: an optimal algorithm for stochastic strongly-convex optimization}. {Journal of Machine Learning Research}, 19: 421-436,  2011.

\bibitem{Nemirovski2009}
{A. Nemirovski, A. Juditsky, G. Lan, and A. Shapiro}. {Robust stochastic approximation approach to stochastic programming}. {SIAM Journal on Optimization}, 19(4):1574-1609, 2009.

\bibitem{Nemirovski1978}
{A. Nemirovski and D. Yudin}. {On cezari's convergence of the steepest descent method for approximating saddle point of convex-concave functions}. {Soviet Math Dkl.}, 19, 1978.

\bibitem{Nemirovski1983}
{A. Nemirovski and D. Yudin}. {Problem complexity and method efficiency in optimization.}. {Wiley, New York}, 1983

\bibitem{Nesterov2009}
{Y. Nesterov}. {Primal-dual subgradient methods for convex problems.}. {Mathematical Programming}, 120(1):221-259, 2009.

\bibitem{Niu2011}
{F. Niu, B. Recht, C. R$\acute{\rm e}$, and S. J. Wright}. {HOGWILD!: A lock-free approach to parallelizing stochastic gradient descent}. {In Advances in Neural Information Processing}, 2011.

\bibitem{Ben2010}
 {Ben Recht's note ``Stochastic and Incremental Gradient Methods''}.
{dated November 22, 2010.}

\bibitem{Ben2014}
 {Ben Recht's note ``Notes on SGD''},
{dated March 18, 2014.}

\end{thebibliography}


\section{Total differentiation of scalar fields}


\section{Some existing Galant animations}
\label{app:galant_animations}

The purpose of these examples is mainly to illustrate the capabilities of
Galant. Many other algorithms are easily animated and some, such as
breadth-first search and Kruskal's minimum spanning tree algorithm have
already been implemented.

\subsection{Depth-first search}

Fig.~\ref{fig:dfs} illustrates an animation of depth-first search.
The code, in the current version of Galant, requires a few (four to be exact) lines lines at the beginning to set up the global variable \verb$time$ and the arrays \verb$discovery$ and \verb$finish$.
See the programmer guide (Appendix~\ref{app:programmer_guide}) for more
detailed instructions on how to do this properly.

The animation follows the conventions of Cormen et al. --- tree edges are selected (shown as thick red lines) and non-tree edges are labeled as B = back edge,
F = forward edge, and C = cross edge.
White nodes, not yet visited,
are neither marked nor selected;
gray nodes, visited but the visit is not finished, are selected
(thick red boundary);
and black nodes, visit is completed, are both marked and selected (red boundary and gray fill).
Labels on nodes indicate the discovery and finish times, separated by a slash.

Note that the animation begins with the statement\\
\hspace{3em} \verb+setDirected(true)+\\
which ensures that the graph is displayed and treated as a directed graph.
Depth-first search on an undirected graph requires keeping track of parents of visited nodes
or distinguishes visited from unvisited edges.

\input{Y_dfs}

\subsection{Dijkstra's algorithm}

Fig.~\ref{fig:dijkstra} illustrates an animation of Dijkstra's shortest path algorithm.

\cmt{mention the role of node 0, the fact that decrease key is done in clumsy fashion to avoid a pointer into the queue (the actual decreaseKey function is not shown; not a big deal for small graphs, but
could be avoided if animator provided a binary heap implementation (which could
be an external class) and added an integer attribute.}

\input{Y_dijkstra}

\subsection{Bubble sort}

% [Last modified: 2013 07 12 at 21:05:01 GMT]


\section{Sorting}

\begin{frame}{Sorting 1 / 2}
  \textbf{Problem:}
  \begin{itemize}
    \item
      Input: $n$ elements $x_1, \ldots, x_n$
    \item
      Transitive operator \enquote{\textbf{\color{Mittel-Blau}<}} which returns
      \textbf{\color{Mittel-Blau}true} if the left value is smaller than the
      right one
      \begin{itemize}
        \item
          Transitivity: $x < y, \; y < z \rightarrow x < z$
      \end{itemize}
    \item
      Output: $x_1, \ldots, x_n$ sorted with operator
  \end{itemize}
  \onslide<2- |handout:1>{
    \begin{exampleblock}{Example}
      Input:\hspace*{1.5em}14, 4, 32, 19, 8, 44, 65\\
      Output:
    \end{exampleblock}
  }
\end{frame}

%-------------------------------------------------------------------------------

\begin{frame}{Sorting 2 / 2}
  \textbf{Why do we need sorting?}
  \begin{itemize}
    \item
      Nearly {\color{Mittel-Blau}every} program needs a sorting algorithm
    \item
      \textbf{Examples:}
      \begin{itemize}
        \item
          Index of a search engine
        \item
          Listing filesystem in explorer / finder
        \item
          (Music) library
        \item
          Highscore list
      \end{itemize}
  \end{itemize}
\end{frame}


\section{Inverse Laplace Transform}
\noindent
Similar to how the derivative is the inverse of the integral, we can think about an inverse Laplace transform that allows us to go back from the $s$ domain to the $t$ domain.
\begin{definition}
	Let $F(s) = \Laplace{f(t)}$. Then $f(t)$ is the inverse Laplace transform of $F(s)$.
	\begin{equation*}
		\inverseLaplace{F(s)} = \inverseLaplace{\Laplace{f(t)}} = f(t)
	\end{equation*}
\end{definition}

\noindent
We don't need to work through the derivations of inverse Laplace transforms because we already did the derivations for Laplace transforms.


%% LyX 1.5.6 created this file.  For more info, see http://www.lyx.org/.
%% Do not edit unless you really know what you are doing.
\documentclass[english,preprint]{revtex4}
\usepackage[T1]{fontenc}
\usepackage[latin9]{inputenc}
\usepackage{babel}

\begin{document}

\title{MATLAB example: The Born-ion model}

\begin{abstract}
Let's consider the born-ion model at room temperature (T=298.15 C).
Inside the born radius (r=3A) the low dielectric coefficient representing
the molecule is set equal to $\epsilon^{x}=\epsilon^{y}=\epsilon^{z}=1.0$.
This ion is located at the origin of the reference system and it is
immersed in a solvent medium of dielectric coefficient that mimic
water, e.g., with $\epsilon^{x}=\epsilon^{y}=\epsilon^{z}=78.54$.
The bulk ionic strength is set equal to zero. The born ion charge
valence is set equal to $z=+1$. To solve the PB equation we will
use a cubic grid of 65 points and a box length of 12 A centered at
the ion position. We will use the linear B-spline to obtain the fractional
charge at the node points.
\end{abstract}
\maketitle

\part{Using the MATLAB PB SOLVER}

\subsection*{Input files as generated by the APBS and pdb2pqr codes}

The {}``born-ion.pqr'' file corresponding to the model described
above reads

\begin{quotation}
ATOM 1 I ION 1 0.000 0.000 0.000 1.00 3.00
\end{quotation}
As an example, the {}``born.in'' input file used by the APBS reads

\begin{quotation}
\# READ IN MOLECULES

read

mol pqr born-ion.pqr

end

\# COMPUTE POTENTIAL FOR SOLVATED STATE

elec name solvated

mg-auto

dime 65 65 65

cglen 50 50 50

fglen 12 12 12

fgcent mol 1

cgcent mol 1

mol 1

lpbe

bcfl mdh

pdie 1.0

sdie 78.54

chgm spl0

srfm spl2

srad 1.4

swin 0.3

sdens 10.0

temp 298.15

calcenergy total

calcforce no

write pot dx APBS-born-pot

write charge dx born-charge

write dielx dx born-dielx

write diely dx born-diely

write dielz dx born-dielz

write kappa dx born-kappa

end

quit
\end{quotation}
For example, if you saved both files 'born-ion.pqr'' and 'born.in'
in the directory 'C:/Users/me/Documents/APBS\_PB\_solver/', you may
type in your command prompt window

\begin{quotation}
C:/Users/me/Documents/Matlab\_PB\_solver>apbs born.in
\end{quotation}
In this way we obtain the input files in dx format required by the
MATLAB version of the APBS to solve the PB equation. Specifically,
we obtain the shifted dielectric coefficients maps born-dielx.dx,
born-diely.dx, born-dielz.dx, and the ion accessibility map born-kappa.dx.
We also obtain the charge density born-charge.dx and the electrostatic
potential born-pot.dx maps which will be used later for testing purpose.

\subsection*{MATLAB input file}

Now we are ready to write the input file {}``born.inm'' as follows

\begin{quotation}
65 65 65

12 12 12

298.15

born-dielx.dx

born-diely.dx

born-dielz.dx

born-kappa.dx

born-ion.pqr

born\_ion\_model
\end{quotation}
The first line corresponds to the number of grid of points $[N_{x}N_{y}N_{z}]$;
the second line corresponds to the length in Anstrong of the boxsides
$[L_{x}L_{y}L_{z}]$; the next line corresponds to the temperature
in Celsius degree; next is the name of the dx file containing the
shifted dielectric coefficients along the x direction; next is for
the one corresponding to the y-direction and next the one in the z-direction.
The seventh line corresponds to the name of the ion accessibility
coefficient map. The next one corresponds to the name of the pqr file
generated by the pdb2pqr code and the last line corresponds to the
name of the folder that will be created into the current (working)
folder (for instances 'Matlab\_PB\_solver') where all the Matlab files
of our code are saved. In the folder {}``born\_ion\_model'' you
will find all the files generated by our Matlab code, namely electrostatic
potential and the charge maps in dx format, and the electrostatic
potential surface $u_{ij(N_{z}+1)/2}$ in both jpg and fig format.

\subsection*{Before using our Matlab code}

Before running the {}``main.m'' file in the Matlab console you must
create a folder (for instances {}``Matlab\_Input\_Files'') in the
same directory (For instances 'C:/Users/me/Documents/Matlab\_PB\_solver/')
in which all the Matlab files of our code are saved. Then you must
save the input files born-ion.inm, born-ion.pqr, born-dielx.dx, born-diely.dx,
born-dielz.dx, and born-kappa.dx in the folder {}``Matlab\_Input\_Files''.

Next, you must edit two lines in the {}``main.m'' file to know:

\begin{itemize}
\item The line number 57 contains the path to the folder {}``Matlab\_Input\_Files''.
In our example you have to set the path as follows
\end{itemize}
\begin{quotation}
MYPATH='C:/Users/me/Documents/Matlab\_PB\_solver/Matlab\_Input\_Files';
\end{quotation}
\begin{itemize}
\item The line number 65 contains the name of the inm input file. In our
example we termed it as {}``born-ion.inm'' such that we have to
set the name of the inputfile as follows
\end{itemize}
\begin{quotation}
inputfile='born-ion.inm';
\end{quotation}
Now we are ready to run the main.m file. In the command window of
Matlab you may type

\begin{quotation}
>\textcompwordmark{}>run C:/Users/me/Documents/Matlab\_PB\_solver/main
\end{quotation}
You are done. On the same command window you will see information
about the different steps that our code performs to get the required
solution.

\subsection*{Matlab output files}

In this example you will find a created folder named 'born\_ion\_model'
in the directory 'C:/Users/me/Documents/Matlab\_PB\_solver/' containing
the following files

\begin{quotation}
born\_ion\_model\_MATLAB\_charge\_density.dx

born\_ion\_model\_MATLAB\_potential\_solution.dx

born\_ion\_model\_MATLAB\_potential\_solution.jpg

born\_ion\_model\_MATLAB\_potential\_solution.fig
\end{quotation}

\subsection*{Accuracy and efficiency of our Matlab code}

You may change same parameters of our code in order to make it faster
and more efficient. In particular, the default accuracy is set equal
to 10\textasciicircum{}-9. You may change it by editing the main.m
file and change the variable 'accuracy' for the required precision.
You may also change the tolerance in the inexact LU decompostion.
The default value for the variable 'tolerance' is set equal to 0.25.

\newpage

\part{Testing our code}

\subsection*{Before using our Matlab code}

Before running the {}``comparison.m'' file in the Matlab console
you must create a folder (for instances {}``Potential'') in the
same directory (For instances 'C:/Users/me/Documents/Matlab\_PB\_solver/')
in which all the Matlab files of our code are saved. Then you must
save in that folder the APBS and MATLAB electrostatic potential solutions.
In our example they are APBS-born-pot.dx and born\_ion\_model\_MATLAB\_potential\_solution.dx.

Next, you must edit three lines in the {}``comparison.m'' file to
know:

\begin{itemize}
\item The line number 19 contains the path to the folder {}``Potential''.
In our example you have to set the path as follows
\end{itemize}
\begin{quotation}
MYPATH='C:/Users/me/Documents/Matlab\_PB\_solver/Potential';
\end{quotation}
\begin{itemize}
\item The line number 27 contains the number of grid points in the format
$[N_{x}N_{y}N_{z}]$. In our example we have to set
\end{itemize}
\begin{quotation}
dime= {[}65 65 65];
\end{quotation}
\begin{itemize}
\item The line number 29 contains the length of the boxside in the format
$[L_{x}L_{y}L_{z}]$ in anstromgs. In our example we have to set
\end{itemize}
\begin{quotation}
glen= {[}12 12 12];
\end{quotation}
Now we are ready to run the comparison.m file. In the command window
of Matlab you may type

\begin{quotation}
>\textcompwordmark{}>run C:/Users/me/Documents/Matlab\_PB\_solver/comparison
\end{quotation}
You are done. On the same command window you will see information
about the different steps that our code performs to get the required
solution.

\subsection*{Matlab output files}

In this example you will find a created folder named 'COMPARATIVE\_ANALYSIS'
in the directory 'C:/Users/me/Documents/Matlab\_PB\_solver/' containing
the following files

\begin{quotation}
Absolute Error between MATLAB and APBS solutions.dx

Absolute\_error.fig

Absolute\_error.tiff

APBS-born-pot\_and\_born\_ion\_model\_MATLAB\_potential\_solution.fig

APBS-born-pot\_and\_born\_ion\_model\_MATLAB\_potential\_solution.tiff
\end{quotation}

\end{document}


% \section{Approximation of the variogram}\label{app:approx}
% For a data matrix ${\bf Y}$ in its disjunctive form, we have ${\bf Y}_{i,j} = 0,1$ for all $l = 1,..,(p+1)L$.
% We give here arguments to justify that the function defined as

% $$
% \gamma(h) = \frac{1}{2 |N(h)|} \sum_{i,j \in N(h)} \frac{1}{L} \sum_{l = 1}^{(p+1)L} |Y_{i,l} - Y_{j,l}|
% $$
% can be approximated by the function

% $$\hat{\gamma}(h) = {C}_0 \frac{1}{2 |N(h)|} \sum_{i,j \in N(h)} \| {\bf Q}_{i,.} - {\bf Q}_{i,.}\|^2 + {C}_1.
% $$
% First, note that the sum

% $$
% S = \frac{1}{L} \sum_{l = 1}^{(p+1)L} |{\bf Y}_{i,l} - {\bf Y}_{j,l}|
% $$
% can be approximated by its expected value

% $$
% E[S] = \frac{1}{L} \sum_{l = 1}^{(p+1)L} P(|{\bf Y}_{i,l} - {\bf Y}_{j,l}| = 1)
% $$
% because

% $$
% Var(S) = \frac{1}{L^2} \sum_{l = 1}^{(p+1)L} P(|{\bf Y}_{i,l} - {\bf Y}_{j,l}| = 1)
% ( 1 - P(|{\bf Y}_{i,l} - {\bf Y}_{j,l}| = 1)) \xrightarrow[L \rightarrow \infty]{} 0.
% $$
% Then, we have

% $$
% P(|{\bf Y}_{i,l} - {\bf Y}_{j,l}| = 1) = ({\bf P}_{i,l} - {\bf P}_{j,l})^2 + {\bf P}_{i,l} (1 - {\bf P}_{i,l}) + {\bf P}_{j,l} (1 - {\bf P}_{j,l}),
% $$
% \noindent where ${\bf P}$ is the matrix of probability, for which entries ${\bf P} = P({\bf Y}_{i,l} = 1)$. Since ${\bf P} = {\bf Q} {\bf G}^T$, we have
% $$
% \sum_{i,j \in N(h)} \frac{1}{L} \sum_{l = 1}^{(p+1)L} ({\bf P}_{i,l} - {\bf P}_{j,l})^2 = {C}_0 \sum_{i,j \in N(h)} \| {\bf Q}_{i,.} - {\bf Q}_{j,.}\|^2
% $$
% where ${C}_0 = \frac{1}{L} \sum_{l = 1}^{(p+1)L} \sum_{k=1}^K {\bf G}_{k,l}^2$.
% Finally, assuming
% $$
% {C}_1 = \frac{1}{|N(h)|} \sum_{i,j \in N(h)} \frac{1}{L} \sum_l  {\bf P}_{i,l} (1 - {\bf P}_{i,l})
% $$
% is constant $\forall h$ we can approximate $\gamma(h)$ by $\hat{\gamma}(h)$.

\section{Algorithms}\label{app:algo}

\begin{algo}
	AQP algorithm pseudo code. To solve optimization problem~\eqref{eq:LS}.

\begin{algorithm}[H]
 \KwIn{ the data matrix ${\bf Y} \in \{0,1\}^{n \times (p+1) L}$, the Laplacian matrix  ${\bf \Lambda} \in \mathbb{R}^{n \times n}$, the number of ancestral populations $K$, the regularization coefficient  $\alpha$, the maximum number of iteration $itMax$}

 \KwOut{the admixture coefficients matrix ${\bf Q} \in \mathbb{R}^{n \times K}$, the ancestral genotype frequencies matrix ${\bf G} \in \mathbb{R}^{K \times (p+1) L}$ }

 \BlankLine

 Initialize ${\bf Q}$ at random \;

 \For{$it = 1..itMax$}{
	 \BlankLine
	 // G optimization step

      \For{$l = 1..L$}{
			$Y^l \leftarrow {\bf Y}_{., (p+1)l..(p+1)l + d}$ \;
		    $ {\bf D}_Q \leftarrow {\bf I}_{p+1} \otimes {\bf Q}^T {\bf Q}$\;

			$ v_Q \leftarrow Vec({\bf Q}^T Y^l)$\;

	      $g^\star \in \argmin_{
			       \substack{
	          g \in \Delta_G
	       }
	     }
              - 2 v_Q^T g + g^T {\bf D}_Q g$ \;

	$Vec({\bf G}_{(p+1)l..(p+1)l + d,.}) \leftarrow g^\star$ \;
	 }
	 \BlankLine
	// Q optimization step

	$ {\bf D}_G \leftarrow Id_{n} \otimes {\bf G}^T {\bf G} + \alpha {\bf \Lambda} \otimes {\bf I}_{K}$ \;

	 $ v_G \leftarrow Vec({\bf G}^T {\bf Y}^T)$ \;

      $Vec({\bf Q}^T) \in \argmin_{%
	       \substack{%
	          q \in \Delta_Q
	       }
	     }
	     - 2 v_G^T q + q^T {\bf D}_G q$\;
	 }
\end{algorithm}
\label{algo:aqp}
\end{algo}

\begin{algo}
APLS algorithm pseudo code. To solve the optimization problem~\eqref{eq:LS}.

\begin{algorithm}[H]
\KwIn{ the data matrix ${\bf Y} \in \{0,1\}^{n \times (d+1) L}$, the eigen values and vectors matrices ${\bf U}$ and ${\bf \Delta}$ such that ${\bf \Lambda} = {\bf U}^T {\bf \Delta} {\bf U}$, the number of ancestral populations $K$, the regularization coefficient  $\alpha$, the maximum number of iteration $itMax$}

\KwOut{the admixture coefficients matrix ${\bf Q} \in \mathbb{R}^{n \times K}$, the ancestral genotype frequencies matrix ${\bf G} \in \mathbb{R}^{K \times (d+1) L}$ }

 \BlankLine

 Initialize ${\bf Q}$ at random \;

 $proj({\bf Y}) \leftarrow {\bf R} {\bf Y}$ \;

 \For{$it = 1..itMax$}{
 	\BlankLine
	 // G optimization step

	\For{$j = 1..(p+1)L$}{

	$g^\star \in \argmin_{%
	       \substack{%
	          g \in \mathbb{R}^{K}
	       }
	       }
	     || {\bf Y}_{., j} - {\bf Q} g||^2$\;

 ${\bf G}_{j,.} \leftarrow g^\star $\;
 }

 Project ${\bf G}$ such that ${\bf G} \in \Delta_G$ \;
 	\BlankLine
	 // Q optimization step

 \For{$i = 1..n$}{
 $g^\star_i \in \argmin_{%
       \substack{%
          q \in \mathbb{R}^{K}
       }
       }
     || proj({\bf Y})_{i,.} - {\bf G}^T q ||^2 + \alpha {\bf \Delta}_{i,i} ||q||^2$\;

 $proj({\bf Q})_{i, .} \leftarrow g^\star_i $\;
 }

 ${\bf Q} \leftarrow {\bf U}^T proj({\bf Q})$\;

Project ${\bf Q}$ such that ${\bf Q} \in \Delta_Q$ \;
 }

 \end{algorithm}
\label{algo:apls}
\end{algo}


\documentclass{article}
\usepackage{amsmath}
\usepackage{bm}

\newcommand{\M}[3]{{^#1}M^{#2}_{#3}}
\newcommand{\Minv}[3]{{\left( \M{#1}{#2}{#3} \right) }^{-1}}

\newcommand{\oM}[2]{M^{#1}_{#2}}
\newcommand{\oMinv}[2]{{\left( \oM{#1}{#2} \right) }^{-1}}

\begin{document}

\section{Visual servoing}

\subsection{Method 1}

Let $T_1, T_2, G, H, $ be a visual tag on the robot gripper,
a visual tag on the object, the gripper and the handle frames.
Let ${^o}M^a_1 \in SE(3)$ be the transformation between joint 1 and the world.
Joint 1 refers to the end-effector joint.
Joint 2 refers to the object joint.
The camera frame is refered to by 3.
$a$ takes values in $\left\{ p, c, v, \empty \right\}$
meaning the transformation refers to Planning, Control, Vision or is independant of the considered framework.
For instance, $$ {^1}M_G = {^1}M^p_G = {^1}M^C_G = {^1}M^v_G $$
but, $$ {^o}M^p_1 \neq {^o}M^C_1 \neq {^o}M^v_1 $$ and ${^o}M_1 $ is meaningless.

The control variable is:
$$ {^G}M^v_H = {^G}M_{T_1} {^{T_1}}M^v_{T_2} {^{T_2}}M_H $$
where ${^{T_1}}M^v_{T_2} $ is the input.

The reference is:
$$ {^G}M^p_H = \left( {^o}M^p_{1} {^1}M_{G} \right)^{-1} \left( {^o}M^p_{2} {^2}M_{H} \right) $$

The derivative of the control variable is the derivative of:
$$ {^{T_1}}M^C_{T_2} = \left( {^3}M^C_{1} {^1}M_{T_1} \right)^{-1} \left( {^3}M^v_{T_2} \right) $$

Formulated as a task, it gives:

$$ e = \left( {^G}M^v_H \right)^{-1} {^G}M^p_H $$
%$$ J = \left( {^G}M^v_H \right)^{-1} {^G}M^p_H $$

\subsection{Method 2}

We assume $\oM{c}{2} = \oM{p}{2} $. This makes sense because the object is not actuated, so there is no control variable for it.
We define the two following errors between the control variables and the vision.
$$ \delta_g = \oMinv{c}{1} \oM{v}{1} $$
$$ \delta_h = \oMinv{c}{2} \oM{v}{2} $$
They account for kinematic calibration and localization errors between control and vision.

The desired behaviour of the system is:
$$ \oMinv{v}{h} {\bm{\oM{v}{g}}}^* = \oMinv{p}{h} {\oM{p}{g}} $$

The desired gripper position is then:
\begin{align*}
{\bm{\oM{c}{g}}}^* &= {\bm{\oM{c}{1}}}^*                                                    . g \\
                   &= {\bm{\oM{v}{1}}}^*                                      .\delta_g^{-1}. g \\
                   &= {\bm{\oM{v}{g}}}^*                               .g^{-1}.\delta_g^{-1}. g \\
                   &= \oM{v}{h}.\oMinv{p}{h}.{\oM{p}{g}}               .g^{-1}.\delta_g^{-1}. g \\
                   &= \oM{v}{2}.       h.\oMinv{p}{h}.\oM{p}{g}        .g^{-1}.\delta_g^{-1}. g \\
                   %&= \oM{p}{2}.\delta_h.\oMinv{p}{2}.\oM{p}{1}               .\delta_g^{-1}. g \\
                   &= \oM{p}{2}.\delta_h.h.\oMinv{p}{h}.\oM{p}{1}               .\delta_g^{-1}. g \\
                   &= \oM{p}{h}.h^{-1}.\delta_h.h.\oMinv{p}{h}.\oM{p}{1}               .\delta_g^{-1}. g \\
\end{align*}

\paragraph{The evaluation of $\delta_g$ and $\delta_h$} is made in the camera frame.
$$ \delta_g = \Minv{3}{c}{1} \M{3}{v}{1} $$
$$ \delta_h = \Minv{3}{c}{2} \M{3}{v}{2} $$

\end{document}


\documentclass[12pt]{article}
\usepackage{amsmath}
\usepackage{pdfpages}
\usepackage{lscape}
\usepackage{hyperref}
\title{2D Graphics with Asymptote}
\author{The Asymptote Project}
\newcommand{\insertrep}[1]{%
	\hspace*{-2.4cm}
	\fbox{\includegraphics[page=1,scale=1.5]{#1}}
}
\begin{document}
	Exercise 12:\\
	How many corners does a cube have in 4 dimensions? How many 3D faces? How many edges?

	\section{Vertices}
	A typical corner is $(0,0,1,0)$. A typical edge goes to $(0,1,0,0)$.\\
	\\
	In 3D, all cube corners are:\\
	$(0,0,0)$\\
	$(0,0,1)$\\
	$(0,1,0)$\\
	$(0,1,1)$\\
	$(1,0,0)$\\
	$(1,0,1)$\\
	$(1,1,0)$\\
	$(1,1,1)$\\
	\\
	Which follows the permutation logic, and we know that in a set with three members (3D space), each with two possibilities (0 or 1) we have $2^3=8$ permutations.\\

	\section{Edges}
	In a 4D space we will have $2^4=16$ permutations, so we will have 16 corners.\\
	If we use the technique of imagining a 4D-Cube as a moving 3D Cube, we will have:\\
	1 - 12 edges from the "starting" cube;\\
	2 - 12 edges from the "final" cude;\\
	3 - 8 edges from the "moving vertices"\\
	\\
	What will give us 32 edges on a 4D-Cube.
	\newpage
	The recursive formula is: $f(N) = 2*f(N-1) + 2^{N-1}$
	\\
	\begin{align*}
		f(N) = 2f(N-1)+2^{N-1}\\
		f(N+1) = 2f(N)+2^{N}\\
		f(N+1) - 2f(N) = 2^{N}\\
		(E-2)f = 2^{N}\\
		(E-2)(E-2)f = (E-2)*2^{N}\\
		(E-2)^2f = 0\\
		\\
		(\alpha n + \beta)2^n\\
		\\
		(1\alpha + \beta)2^1 = 1\\
		(2\alpha + \beta)2^2 = 4\\
		(3\alpha + \beta)2^3 = 12\\
		\\
		2\alpha + 2\beta = 1\\
		8\alpha + 4\beta = 4\\
		24\alpha + 8\beta = 12\\
		\\
		-2(2\alpha + 2\beta) = -2*1\\
		8\alpha + 4\beta = 4\\
		\\
		-4\alpha - 4\beta = -2\\
		8\alpha + 4\beta = 4\\
		\\
		-4\alpha - 4\beta = -2\\
		8\alpha + 4\beta = 4\\
		\\
		4\alpha + 0\beta = 2\\
		\\
	\end{align*}
	\begin{align*}
		4\alpha = 2\\
		\alpha = \frac{2}{4}\\
		\alpha = \frac{1}{2}\\
		\\
		2\frac{1}{2} + 2\beta = 1\\
		\frac{2}{2} + 2\beta = 1\\
		1 + 2\beta = 1\\
		2\beta = 1 - 1\\
		2\beta = 0\\
		\beta = 0
		\\
		(\frac{n}{2})2^n\\
		\frac{n*2^n}{2}\\
	\end{align*}
	Which is a "closed form" for the quantities of edges of "cube" in any dimension:
	$$ \text{edges}(n) = \frac{n*2^n}{2}$$
	Which can be interpreted as the dimension times half the vertices. This makes total sense because at each vertex there are three edges, one for each each axis in that dimension. This counts each edge twice, at the start and at the end, so we divide by two.
	\newpage
	\\
	Reference:\\
	\\
	Beyond the Third Dimension\\
	by \\
	Thomas F. Banchoff \\
	\begin{quote}
	It is helpful to think of cubes as generated by lower-dimensional cubes in motion. A point in motion generates a segment; a segment in motion generates a square; a square in motion generates a cube; and so on. From this progression, a pattern develops, which we can exploit to predict the numbers of vertices and edges.\\
	Each time we move a cube to generate a cube in the next higher dimension, the number of vertices doubles. That is easy to see since we have an initial position and a final position, each with the same number of vertices. Using this information we can infer an explicit formula for the number of vertices of a cube in any dimension, namely 2 raised to that power.\\
	What about the number of edges? A square has 4 edges, and as it moves from one position to the other, each of its 4 vertices traces out an edge. Thus we have 4 edges on the initial square, 4 on the final square, and 4 traced out by the moving vertices for a total of 12. That basic pattern repeats itself. If we move a figure in a straight line, then the number of edges in the new figure is twice the original number of edges plus the number of moving vertices. Thus the number of edges in a four-cube is 2 times 12 plus 8 for a total of 32. Similarly we find 32 + 32 + 16 = 80 edges on a five-cube and 80 + 80 + 32 = 192 edges on a six-cube.\\
	This presentation definitely suggests a pattern, namely that the number of edges of a hypercube of a given dimension is the dimension multiplied by half the number of vertices in that dimension. Once we notice a pattern like this, it can be proved to hold in all dimensions by mathematical induction.
	\end{quote}
	\\
	\url{http://www.math.brown.edu/~banchoff/Beyond3d/chapter4/section05.html}
	\\
	\section{Faces}
	The easiest way to calculate the quantity of faces of a 4D cube is to realize that from each vertex starts four edges and each face is a combination of two of theses edges. This gives us ${4 \choose 2}=6$ faces per vertex.\\
	A 4D Cube has $2^4=16$ vertices each containing 6 faces. But this algotihm counts each face 4 times, so we actually have:
	$$f(n) = \frac{2^n*{n \choose 2}}{4}$$
	Which gives us the correct value for $R^4$, $\frac{2^4*{4 \choose 2}}{4}=24$.\\
	\\
	Reference:\\
	\begin{quote}
		The hypercube is so highly symmetric that every vertex looks like every other vertex. If we know what happens at one vertex, we can figure out what is going to happen at all vertices. At each vertex there are as many square faces as there are ways to choose 2 edges from among the 4 edges at the point, namely 6. Since there are 16 vertices, we can multiply 6 by 16 to get 96, but this counts each square four times, once for each of its vertices. The correct number of squares in a hypercube is then 96/4, or 24.
	\end{quote}
	http://www.math.brown.edu/~banchoff/Beyond3d/chapter4/section07.html
\end{document}


\input{latexmacros.tex}
\title{Small strain elastic-plastic model}
\begin{document}
\DeclareGraphicsExtensions{.jpg,.pdf}
\maketitle
\tableofcontents
\section{Preamble}
Let $\boldsymbol{F}$ be the deformation gradient, $\boldsymbol{\sigma}$ be the
Cauchy stress, and $\boldsymbol{d}$ be the rate of deformation tensor.  We
first decompose the deformation gradient into a stretch and a rotations using
$\BF = \BR\cdot\BU$.  The rotation $\BR$ is then used to rotate the stress and
the rate of deformation into the material configuration to give us
\Beq
  \widehat{\Bsig} = \BR^T\cdot\Bsig\cdot\BR ~;~~
  \widehat{\Bd} = \BR^T\cdot\Bd\cdot\BR
\Eeq
This is equivalent to using a Green-Naghdi objective stress rate.  In the
following all equations are with respect to the hatted quantities and we
drop the hats for convenience.

\section{Elastic relation}
Let us split the Cauchy stress into a volumetric and a deviatoric part
\Beq
  \Bsig = p~\Bone + \Bs ~;~~ p = \Third~\Tr(\Bsig) ~.
\Eeq
Taking the time derivative gives us
\Beq
  \dot{\Bsig} = \dot{p}~\Bone + \dot{\Bs} ~.
\Eeq
We assume that the elastic response of the material is isotropic.  The constitutive
relation for a hypoelastic material of grade 0 can be expressed as
\Beq
  \dot{\Bsig} = \left[\lambda~\Tr(\Bd^e) - 3~\kappa~\alpha~\Deriv{}{t}(T-T_0)\right]~\Bone
    + 2~\mu~\Bd^e  ~;~~ \Bd = \Bd^e + \Bd^p
\Eeq
where $\Bd^e, \Bd^p$ are the elastic and plastic parts of the rate of deformation
tensor, $\lambda,\mu$ are the Lame constants, $\kappa$ is the bulk modulus, $\alpha$
is the coefficient of thermal expansion, $T_0$ is the reference temperature, and $T$ is the
current temperature.  If we split $\Bd^e$ into volumetric and deviatoric parts as
\Beq
  \Bd^e = \Third~\Tr(\Bd^e)~\Bone + \Beta^e
\Eeq
we can write
\Beq
  \dot{\Bsig} = \left[\left(\lambda + \cfrac{2}{3}~\mu\right)~\Tr(\Bd^e)
     - 3~\kappa~\alpha~\Deriv{}{t}(T-T_0)\right]~\Bone + 2~\mu~\Beta^e  =
  \kappa~\left[\Tr(\Bd^e) - 3~\alpha~\Deriv{}{t}(T-T_0)\right]~\Bone + 2~\mu~\Beta^e
\Eeq
Therefore, we have
\Beq
  \dot{\Bs} = 2~\mu~\Beta^e ~.
\Eeq
and
\Beq
  \dot{p} = \kappa~\left[\Tr(\Bd^e) - 3~\alpha~\Deriv{}{t}(T-T_0)\right] ~.
\Eeq
We will use a standard elastic-plastic stress update algorithm to integrate the rate
equation for the deviatoric stress.  However, we will assume that the volumetric part
of the Cauchy stress can be computed using an equation of state.  Then the final
Cauchy stress will be given by
\Beq
  \Bsig = \left[p(J) - 3~J~\Deriv{p(J)}{J}~\alpha~(T-T_0)\right]~\Bone + \Bs ~;~~
  J = \det(\BF)~.
\Eeq
(Note that we assume that the plastic part of the deformation is volume preserving.
This is not true for Gurson type models and will lead to a small error in the
computed value of $\Bsig$.)

\section{Flow rule}
We assume that the flow rule is given by
\Beq
  \Bd^p = \dot{\gamma}~\Br
\Eeq
We can split $\Bd^p$ into a trace part and a trace free part, i.e.,
\Beq
  \Bd^p = \Third~\Tr(\Bd^p)~\Bone + \Beta^p
\Eeq
Then, using the flow rule, we have
\Beq
  \Bd^p = \Third~\dot{\gamma}~\Tr(\Br)~\Bone + \Beta^p  ~.
\Eeq
Therefore we can write the flow rule as
\Beq
  \Beta^p = \dot{\gamma}~\left(-\Third~\Tr(\Br)~\Bone + \Br\right) ~.
\Eeq
Note that
\Beq
  \Bd = \Bd^e + \Bd^p \quad \implies \quad
  \Tr(\Bd) = \Tr(\Bd^e) + \Tr(\Bd^p) ~.
\Eeq
Also,
\Beq
  \Bd = \Bd^e + \Bd^p \quad \implies \quad
  \Third~\Tr(\Bd)~\Bone + \Beta =
  \Third~\Tr(\Bd^e)~\Bone + \Beta^e  +
  \Third~\Tr(\Bd^p)~\Bone + \Beta^p ~.
\Eeq
Therefore,
\Beq
  \Beta = \Beta^e + \Beta^p ~.
\Eeq

\section{Isotropic and Kinematic hardening and porosity evolution rules}
We assume that the strain rate, temperature, and porosity can be fixed at the
beginning of a timestep and consider only the evolution of plastic strain and
the back stress while calculating the current stress.

We assume that the plastic strain evolves according to the relation
\Beq
  \dot{\Ve^p} = \dot{\gamma}~h^{\alpha}
\Eeq

We also assume that the back stress evolves according to the relation
\Beq
  \dot{\widehat{\Bbeta}} = \dot{\gamma}~\Bh^{\beta}
\Eeq
where $\hat{\Bbeta}$ is the back stress.  If $\Bbeta$ is the deviatoric part of
$\widehat{\Bbeta}$, then we can write
\Beq
  \dot{\Bbeta} = \dot{\gamma}~\Dev(\Bh^{\beta}) ~.
\Eeq

The porosity $\phi$ is assumed to evolve according to the relation
\Beq
  \dot{\phi} = \dot{\gamma}~h^{\phi} ~.
\Eeq

\section{Yield condition}
The yield condition is assumed to be of the form
\Beq
  f(\Bs, \Bbeta, \Ve^p, \phi, \dot{\Ve}, T, \dots)
    = f(\Bxi, \Ve^p, \phi, \dot{\Ve}, T, \dots) = 0
\Eeq
where $\Bxi = \Bs - \Bbeta$ and $\Bbeta$ is the deviatoric part of $\widehat{\Bbeta}$.
The Kuhn-Tucker loading-unloading conditions are
\Beq
  \dot{\gamma} \ge 0 ~;~~  f \le 0 ~;~~ \dot{\gamma}~f = 0
\Eeq
and the consistency condition is $\dot{f} = 0$.

\section{Temperature increase due to plastic dissipation}
The temperature increase due to plastic dissipation is assume to be
given by the rate equation
\Beq
  \dot{T} = \cfrac{\chi}{\rho~C_p}~\sigma_y~\dot{\Ve^p} ~.
\Eeq
The temperature is updated using
\Beq
  T_{n+1} = T_n +
   \cfrac{\chi_{n+1}~\Delta t}{\rho_{n+1}~C_p}~\sigma^{n+1}_y~\dot{\Ve^p}_{n+1} ~.
\Eeq

\section{Continuum elastic-plastic tangent modulus}
To determine whether the material has undergone a loss of stability we need to compute
the acosutic tensor which needs the computation of the continuum elastic-plastic tangent
modulus.

To do that recall that
\Beq
  \Bsig = p~\Bone + \Bs \quad \implies \quad \dot{\Bsig} = \dot{p}~\Bone ~.
\Eeq
We assume that
\Beq
  \dot{p} = J~\Partial{p}{J}~\Tr(\Bd) \qquad \Tand \quad
  \dot{\Bs} = 2~\mu~\Beta^e ~.
\Eeq
Now, the consistency condition requires that
\Beq
  \dot{f}(\Bs, \Bbeta, \Ve^p, \phi, \dot{\Ve}, T, \dots) = 0 ~.
\Eeq
Keeping $\dot{\Ve}$ and $T$ fixed over the time interval, we can use the chain rule
to get
\Beq
  \dot{f} = \Partial{f}{\Bs}:\dot{\Bs} + \Partial{f}{\Bbeta}:\dot{\Bbeta} +
    \Partial{f}{\Ve^p}~\dot{\Ve^p} + \Partial{f}{\phi}~\dot{\phi} = 0~.
\Eeq
The needed rate equations are
\Beq
  \Bal
    \dot{\Bs} & = 2~\mu~\Beta^e = 2~\mu~(\Beta - \Beta^p) =
      2~\mu~[\Beta - \dot{\gamma}~\Dev(\Br)] \\
    \dot{\Bbeta} & = \dot{\gamma}~\Dev(\Bh^{\beta}) \\
    \dot{\Ve^p} & = \dot{\gamma}~h^{\alpha} \\
    \dot{\phi} & = \dot{\gamma}~h^{\phi}
  \Eal
\Eeq
Plugging these into the expression for $\dot{f}$ gives
\Beq
  2~\mu~\Partial{f}{\Bs}:[\Beta - \dot{\gamma}~\Dev(\Br)]
    + \dot{\gamma}~\Partial{f}{\Bbeta}:\Dev(\Bh^{\beta})
    + \dot{\gamma}~\Partial{f}{\Ve^p}~h^{\alpha}
    + \dot{\gamma}~\Partial{f}{\phi}~h^{\phi}  = 0
\Eeq
or,
\Beq
  \dot{\gamma} = \cfrac{2~\mu~\Partial{f}{\Bs}:\Beta}
    {2~\mu~\Partial{f}{\Bs}:\Dev(\Br) - \Partial{f}{\Bbeta}:\Dev(\Bh^{\beta})
     - \Partial{f}{\Ve^p}~h^{\alpha} - \Partial{f}{\phi}~h^{\phi}} ~.
\Eeq
Plugging this expression for $\dot{\gamma}$ into the equation for $\dot{\Bs}$,
we get
\Beq
  \dot{\Bs} = 2~\mu~\left[\Beta - \left(
    \cfrac{2~\mu~\Partial{f}{\Bs}:\Beta}
    {2~\mu~\Partial{f}{\Bs}:\Dev(\Br) - \Partial{f}{\Bbeta}:\Dev(\Bh^{\beta})
     - \Partial{f}{\Ve^p}~h^{\alpha} - \Partial{f}{\phi}~h^{\phi}}\right)~\Dev(\Br)\right]~.
\Eeq
At this stage, note that a symmetric $\Bsig$ implies a symmetric $\Bs$ and hence a
symmetric $\Beta$.  Also we assume that $\Br$ is symmetric (and hence $\Dev(\Br)$), which
is true if the flow rule is associated.  Then we can write
\Beq
  \Beta = \SfI^{4s}:\Beta \qquad \Tand \qquad \Dev(\Br) = \SfI^{4s}:\Dev(\Br)
\Eeq
where $\SfI^{4s}$ is the fourth-order symmetric identity tensor.  Also note that
if $\BA$, $\BC$, $\BD$ are second order tensors and $\SfB$ is a fourth order tensor,
then
\Beq
  (\BA:\SfB:\BC)~(\SfB:\BD) \equiv A_{ij}~B_{ijkl}~C_{kl}~B_{mnpq}~D_{pq}
     = (B_{mnpq}~D_{pq})~(A_{ij}~B_{ijkl})~C_{kl} \equiv [(\SfB:\BD)\otimes(\BA:\SfB)]:\BC ~.
\Eeq
Therefore we have
\Beq
  \dot{\Bs} = 2~\mu~\left[\SfI^{4s}:\Beta - \left(
    \cfrac{2~\mu~[\SfI^{4s}:\Dev(\Br)]\otimes[\Partial{f}{\Bs}:\SfI^{4s}]}
    {2~\mu~\Partial{f}{\Bs}:\Dev(\Br) - \Partial{f}{\Bbeta}:\Dev(\Bh^{\beta})
     - \Partial{f}{\Ve^p}~h^{\alpha} - \Partial{f}{\phi}~h^{\phi}}\right):\Beta\right]~.
\Eeq
Also,
\Beq
  \SfI^{4s}:\Dev(\Br) = \Dev(\Br) \qquad \Tand \qquad
  \Partial{f}{\Bs}:\SfI^{4s} = \Partial{f}{\Bs} ~.
\Eeq
Hence we can write
\Beq
  \dot{\Bs} = 2~\mu~\left[\SfI^{4s} - \left(
    \cfrac{2~\mu~\Dev(\Br)\otimes\Partial{f}{\Bs}}
    {2~\mu~\Partial{f}{\Bs}:\Dev(\Br) - \Partial{f}{\Bbeta}:\Dev(\Bh^{\beta})
     - \Partial{f}{\Ve^p}~h^{\alpha} - \Partial{f}{\phi}~h^{\phi}}\right)\right]:\Beta
\Eeq
or,
\Beq
  \dot{\Bs} = \SfB^{ep}:\Beta = \SfB^{ep}:\left[\Bd - \Third~\Tr(\Bd)~\Bone\right]
\Eeq
where
\Beq
  \SfB^{ep} :=
    2~\mu~\left[\SfI^{4s} - \left(
    \cfrac{2~\mu~\Dev(\Br)\otimes\Partial{f}{\Bs}}
    {2~\mu~\Partial{f}{\Bs}:\Dev(\Br) - \Partial{f}{\Bbeta}:\Dev(\Bh^{\beta})
     - \Partial{f}{\Ve^p}~h^{\alpha} - \Partial{f}{\phi}~h^{\phi}}\right)\right] ~.
\Eeq
Adding in the volumetric component gives
\Beq
  \Bal
  \dot{\Bsig} & = \dot{p}~\Bone + \dot{\Bs} \\
   & = J~\Partial{p}{J}~\Tr(\Bd)~\Bone +
       \SfB^{ep}:\left[\Bd - \Third~\Tr(\Bd)~\Bone\right] \\
   & = \left[3~J~\Partial{p}{J}~\Bone - \SfB^{ep}:\Bone\right]~\cfrac{\Bd:\Bone}{3}
       + \SfB^{ep}:\Bd \\
   & = J~\Partial{p}{J}~(\Bone\otimes\Bone):\Bd - \Third~[\SfB^{ep}:(\Bone\otimes\Bone)]:\Bd
       + \SfB^{ep}:\Bd ~.
  \Eal
\Eeq
Therefore,
\Beq
  \dot{\Bsig} = \left[J~\Partial{p}{J}~(\Bone\otimes\Bone) -
      \Third~[\SfB^{ep}:(\Bone\otimes\Bone)] + \SfB^{ep}\right]:\Bd
   = \SfC^{ep}:\Bd ~.
\Eeq
The quantity $\SfC^{ep}$ is the continuum elastic-plastic tangent modulus.  We also use the
continuum elastic-plastic tangent modulus in the implicit version of the code.  However,
for improved accuracy and faster convergence, an algorithmically  consistent tangent modulus
should be used instead.  That tangent modulus can be calculated in the usual manner and
is left for development and implementation as an additional feature in the future.

\section{Stress update}
A standard return algorithm is used to compute the updated Cauchy stress.  Recall
that the rate equation for the deviatoric stress is given by
\Beq
  \dot{\Bs} = 2~\mu~\Beta^e ~.
\Eeq
Integrating the rate equation using a Backward Euler scheme gives
\Beq
  \Bs_{n+1} - \Bs_n = 2~\mu~\Delta~t~\Beta^e_{n+1}
    = 2~\mu~\Delta~t~(\Beta_{n+1} - \Beta^p_{n+1})
\Eeq
Now, from the flow rule, we have
\Beq
  \Beta^p = \dot{\gamma}~\left(\Br -\Third~\Tr(\Br)~\Bone\right) ~.
\Eeq
Define the deviatoric part of $\Br$ as
\Beq
  \Dev(\Br) := \Br -\Third~\Tr(\Br)~\Bone ~.
\Eeq
Therefore,
\Beq
  \Bs_{n+1} - \Bs_n
    = 2~\mu~\Delta~t~\Beta_{n+1} - 2~\mu~\Delta\gamma_{n+1}~\Dev(\Br_{n+1}) ~.
\Eeq
where $\Delta\gamma := \dot{\gamma}~\Delta t$.  Define the trial stress
\Beq
  \Bs^{\Trial} := \Bs_n + 2~\mu~\Delta~t~\Beta_{n+1} ~.
\Eeq
Then
\Beq
  \Bs_{n+1} = \Bs^{\Trial} - 2~\mu~\Delta\gamma_{n+1}~\Dev(\Br_{n+1}) ~.
\Eeq
Also recall that the back stress is given by
\Beq
  \dot{\Bbeta} = \dot{\gamma}~\Dev{\Bh}^{\beta}
\Eeq
The evolution equation for the back stress can be integrated to get
\Beq
  \Bbeta_{n+1} - \Bbeta_n = \Delta\gamma_{n+1}~\Dev(\Bh)^{\beta}_{n+1} ~.
\Eeq
Now,
\Beq
  \Bxi_{n+1} = \Bs_{n+1} - \Bbeta_{n+1} ~.
\Eeq
Plugging in the expressions for $\Bs_{n+1}$ and $\Bbeta_{n+1}$, we get
\Beq
  \Bxi_{n+1} = \Bs^{\Trial} - 2~\mu~\Delta\gamma_{n+1}~\Dev(\Br_{n+1})
     - \Bbeta_n - \Delta\gamma_{n+1}~\Dev(\Bh)^{\beta}_{n+1} ~.
\Eeq
Define
\Beq
  \Bxi^{\Trial} := \Bs^{\Trial} - \Bbeta_n ~.
\Eeq
Then
\Beq
  \Bxi_{n+1} = \Bxi^{\Trial} - \Delta\gamma_{n+1}(2~\mu~\Dev(\Br_{n+1}) + \Dev(\Bh)^{\beta}_{n+1}) ~.
\Eeq
Similarly, the evolution of the plastic strain is given by
\Beq
  \Ve^p_{n+1} = \Ve^p_{n} + \Delta\gamma_{n+1}~h^{\alpha}_{n+1}
\Eeq
and the porosity evolves as
\Beq
  \phi_{n+1} = \phi_n + \Delta\gamma_{n+1}~h^{\phi}_{n+1} ~.
\Eeq
The yield condition is discretized as
\Beq
  f(\Bs_{n+1}, \Bbeta_{n+1}, \Ve^p_{n+1}, \phi_{n+1}, \dot{\Ve}_{n+1}, T_{n+1}, \dots) =
  f(\Bxi_{n+1}, \Ve^p_{n+1}, \phi_{n+1}, \dot{\Ve}_{n+1}, T_{n+1}, \dots) = 0 ~.
\Eeq
{\bf Important:} We assume that the derivatives with respect to $\dot{\Ve}$ and $T$ are
small enough to be neglected.

\subsection{Newton iterations}
We now have the following equations that have to be solved for $\Delta\gamma_{n+1}$:
\Beq
  \Bal
  \Bxi_{n+1} & = \Bxi^{\Trial} - \Delta\gamma_{n+1}(2~\mu~\Dev(\Br_{n+1}) + \Dev(\Bh)^{\beta}_{n+1})\\
  \Ve^p_{n+1} & = \Ve^p_{n} + \Delta\gamma_{n+1}~h^{\alpha}_{n+1} \\
  \phi_{n+1} & = \phi_n + \Delta\gamma_{n+1}~h^{\phi}_{n+1}  \\
  f(\Bxi_{n+1}, \Ve^p_{n+1}, \phi_{n+1}, \dot{\Ve}_{n+1}, T_{n+1}, \dots) & = 0 ~.
  \Eal
\Eeq

Recall that if $g(\Delta\gamma) = 0$ is a nonlinear equation that we have to solve
for $\Delta\gamma$, an iterative Newton method can be expressed as
\Beq
  \Delta\gamma^{(k+1)} = \Delta\gamma^{(k)} - \left[\Deriv{g}{\Delta\gamma}\right]^{-1}_{(k)}~g^{(k)} ~.
\Eeq
Define
\Beq
  \delta\gamma := \Delta\gamma^{(k+1)} - \Delta\gamma^{(k)} ~.
\Eeq
Then, the iterative scheme can be written as
\Beq
  g^{(k)} + \left[\Deriv{g}{\Delta\gamma}\right]^{(k)}~\delta\gamma  = 0~.
\Eeq
In our case we have
\Beq
  \Bal
  \Ba(\Delta\gamma) = 0 &= -\Bxi + \Bxi^{\Trial} - \Delta\gamma(2~\mu~\Dev(\Br) + \Dev(\Bh)^{\beta})\\
  b(\Delta\gamma) = 0 &= -\Ve^p + \Ve^p_{n} + \Delta\gamma~h^{\alpha} \\
  c(\Delta\gamma) = 0 & = -\phi + \phi_n + \Delta\gamma~h^{\phi}  \\
  f(\Delta\gamma) = 0 &= f(\Bxi, \Ve^p, \phi, \dot{\Ve}, T, \dots)
  \Eal
\Eeq
Therefore,
\Beq
  \Bal
  \Deriv{\Ba}{\Delta\gamma} & =
   -\Partial{\Bxi}{\Delta\gamma}  - (2~\mu~\Dev(\Br) + \Dev(\Bh)^{\beta})
   - \Delta\gamma~\left(2~\mu~\Partial{\Dev(\Br)}{\Delta\gamma} +
        \Partial{\Dev(\Bh)^{\beta}}{\Delta\gamma}\right) \\
   & =
   -\Partial{\Bxi}{\Delta\gamma}  - (2~\mu~\Dev(\Br) + \Dev(\Bh)^{\beta})
   - \Delta\gamma~\left(
      2~\mu~\Partial{\Dev(\Br)}{\Bxi}:\Partial{\Bxi}{\Delta\gamma} +
      2~\mu~\Partial{\Dev(\Br)}{\Ve^p}~\Partial{\Ve^p}{\Delta\gamma} +
      2~\mu~\Partial{\Dev(\Br)}{\phi}~\Partial{\phi}{\Delta\gamma} +
      \right. \\
   & \qquad \qquad
      \left.
      \Partial{\Dev(\Bh)^{\beta}}{\Bxi}:\Partial{\Bxi}{\Delta\gamma} +
      \Partial{\Dev(\Bh)^{\beta}}{\Ve^p}~\Partial{\Ve^p}{\Delta\gamma} +
      \Partial{\Dev(\Bh)^{\beta}}{\phi}~\Partial{\phi}{\Delta\gamma}
      \right) \\
  \Deriv{b}{\Delta\gamma} & = -\Partial{\Ve^p}{\Delta\gamma} +  h^{\alpha}
    + \Delta\gamma~\left(\Partial{h^{\alpha}}{\Bxi}:\Partial{\Bxi}{\Delta\gamma} +
                        \Partial{h^{\alpha}}{\Ve^p}~\Partial{\Ve^p}{\Delta\gamma} +
                        \Partial{h^{\alpha}}{\phi}~\Partial{\phi}{\Delta\gamma}\right) \\
  \Deriv{c}{\Delta\gamma} & = -\Partial{\phi}{\Delta\gamma} +  h^{\phi}
    + \Delta\gamma~\left(\Partial{h^{\phi}}{\Bxi}:\Partial{\Bxi}{\Delta\gamma} +
                        \Partial{h^{\phi}}{\Ve^p}~\Partial{\Ve^p}{\Delta\gamma} +
                        \Partial{h^{\phi}}{\phi}~\Partial{\phi}{\Delta\gamma}\right) \\
  \Deriv{f}{\Delta\gamma} &
     = \Partial{f}{\Bxi}:\Partial{\Bxi}{\Delta\gamma} +
          \Partial{f}{\Ve^p}~\Partial{\Ve^p}{\Delta\gamma} +
          \Partial{f}{\phi}~\Partial{\phi}{\Delta\gamma} ~.
  \Eal
\Eeq
Now, define
\Beq
   \Delta\Bxi := \Partial{\Bxi}{\Delta\gamma}~\delta\gamma ~;~~
   \Delta\Ve^p := \Partial{\Ve^p}{\Delta\gamma}~\delta\gamma ~;~~
   \Delta\phi := \Partial{\phi}{\Delta\gamma}~\delta\gamma ~.
\Eeq
Then
\Beq
  \Bal
  \Ba^{(k)} & - \Delta\Bxi - [2~\mu~\Dev(\Br^{(k)}) + \Dev(\Bh)^{\beta (k)}]~\delta\gamma \\
   & \qquad \qquad
    - 2~\mu~\Delta\gamma~\left(
      \Partial{\Dev(\Br^{(k)})}{\Bxi}:\Delta\Bxi +
      \Partial{\Dev(\Br^{(k)})}{\Ve^p}~\Delta\Ve^p +
      \Partial{\Dev(\Br^{(k)})}{\phi}~\Delta\phi
      \right) \\
   & \qquad \qquad
    - \Delta\gamma~\left(
      \Partial{\Dev(\Bh)^{\beta (k)}}{\Bxi}:\Delta\Bxi +
      \Partial{\Dev(\Bh)^{\beta (k)}}{\Ve^p}~\Delta\Ve^p +
      \Partial{\Dev(\Bh)^{\beta (k)}}{\phi}~\Delta\phi
      \right)  = 0\\
  b^{(k)} & - \Delta\Ve^p + h^{\alpha}~\delta\gamma
    + \Delta\gamma~\left(\Partial{h^{\alpha (k)}}{\Bxi}:\Delta\Bxi +
                        \Partial{h^{\alpha (k)}}{\Ve^p}~\Delta\Ve^p +
                        \Partial{h^{\alpha (k)}}{\phi}~\Delta\phi\right)
     = 0 \\
  c^{(k)} & - \Delta\phi + h^{\phi}~\delta\gamma
    + \Delta\gamma~\left(\Partial{h^{\phi (k)}}{\Bxi}:\Delta\Bxi +
                        \Partial{h^{\phi (k)}}{\Ve^p}~\Delta\Ve^p +
                        \Partial{h^{\phi (k)}}{\phi}~\Delta\phi\right)
     = 0 \\
  f^{(k)} & + \Partial{f^{(k)}}{\Bxi}:\Delta\Bxi +
          \Partial{f^{(k)}}{\Ve^p}~\Delta\Ve^p +
          \Partial{f^{(k)}}{\phi}~\Delta\phi
      = 0
  \Eal
\Eeq
Because the derivatives of $\Br^{(k)}, \Bh^{\alpha (k)}, \Bh^{\beta (k)}, \Bh^{\phi (k)}$ with respect
to $\Bxi, \Ve^p, \phi$ may be difficult to calculate, we instead use a semi-implicit scheme in our
implementation where the quantities $\Br$, $h^{\alpha}$, $\Bh^{\beta}$, and $h^{\phi}$ are evaluated
at $t_n$.  Then the problematic derivatives disappear and we are left with
\Beq
  \Bal
  \Ba^{(k)} - \Delta\Bxi - [2~\mu~\Dev(\Br_n) + \Dev(\Bh)^{\beta}_n]~\delta\gamma & = 0\\
  b^{(k)} - \Delta\Ve^p + h^{\alpha}_n~\delta\gamma & = 0 \\
  c^{(k)} - \Delta\phi + h^{\phi}_n~\delta\gamma & = 0 \\
  f^{(k)} + \Partial{f^{(k)}}{\Bxi}:\Delta\Bxi +
          \Partial{f^{(k)}}{\Ve^p}~\Delta\Ve^p +
          \Partial{f^{(k)}}{\phi}~\Delta\phi & = 0
  \Eal
\Eeq
We now force $\Ba^{(k)}$, $b^{(k)}$, and $c^{(k)}$ to be zero at all times, leading
to the expressions
\Beq
  \Bal
  \Delta\Bxi & = - [2~\mu~\Dev(\Br_n) + \Dev(\Bh)^{\beta}_n]~\delta\gamma \\
  \Delta\Ve^p & =  h^{\alpha}_n~\delta\gamma \\
  \Delta\phi & =  h^{\phi}_n~\delta\gamma \\
  f^{(k)} + \Partial{f^{(k)}}{\Bxi}:\Delta\Bxi +
          \Partial{f^{(k)}}{\Ve^p}~\Delta\Ve^p +
          \Partial{f^{(k)}}{\phi}~\Delta\phi & = 0
  \Eal
\Eeq
Plugging the expressions for $\Delta\Bxi$, $\Delta\Ve^p$, $\Delta\phi$ from the
first three equations into the fourth gives us
\Beq
  f^{(k)} -\Partial{f^{(k)}}{\Bxi}:[2~\mu~\Dev(\Br_n) + \Dev(\Bh)^{\beta}_n]~\delta\gamma +
          h^{\alpha}_n~\Partial{f^{(k)}}{\Ve^p}~\delta\gamma  +
          h^{\phi}_n~\Partial{f^{(k)}}{\phi}~\delta\gamma  = 0
\Eeq
or
\Beq
  \Delta\gamma^{(k+1)} - \Delta\gamma^{(k)} = \delta\gamma =
   \cfrac{f^{(k)}}
   {\Partial{f^{(k)}}{\Bxi}:[2~\mu~\Dev(\Br_n) + \Dev(\Bh^{\beta}_n)] -
   h^{\alpha}_n~\Partial{f^{(k)}}{\Ve^p} -
   h^{\phi}_n~\Partial{f^{(k)}}{\phi} } ~.
\Eeq

\subsection{Algorithm}
The following stress update algorithm is used for each (plastic) time step:
\begin{enumerate}
  \item Initialize:
  \Beq
    k = 0 ~;~~ (\Ve^p)^{(k)} = \Ve^p_n ~;~~ \phi^{(k)} = \phi_n ~;~~
    \Bbeta^{(k)} = \Bbeta_n ~;~~ \Delta\gamma^{(k)} = 0 ~;~~
    \Bxi^{(k)} = \Bxi^{\Trial}~.
  \Eeq
  \item Check yield condition:
  \Beq
    f^{(k)} := f(\Bxi^{(k)}, (\Ve^p)^{(k)}, \phi^{(k)}, \dot{\Ve}_n, T_n, \dots)
  \Eeq
  If $f^{(k)} < \text{tolerance}$ then
  go to step 5 else go to step 3.
  \item Compute updated $\delta\gamma^{(k)}$ using
  \Beq
    \delta\gamma^{(k)} =
     \cfrac{f^{(k)}}
     {\Partial{f^{(k)}}{\Bxi}:[2~\mu~\Dev(\Br_n) + \Dev(\Bh^{\beta}_n)] -
     h^{\alpha}_n~\Partial{f^{(k)}}{\Ve^p} -
     h^{\phi}_n~\Partial{f^{(k)}}{\phi}} ~.
  \Eeq
  Compute
  \Beq
  \Bal
    \Delta\Bxi^{(k)} & = -[2~\mu~\Dev(\Br_n) + \Dev(\Bh^{\beta}_n)]~\delta\gamma^{(k)} \\
    (\Delta\Ve^p)^{(k)} & =  h^{\alpha}_n~\delta\gamma^{(k)}   \\
    \Delta\phi^{(k)} & =  h^{\phi}_n~\delta\gamma^{(k)}
  \Eal
  \Eeq
  \item Update variables:
  \Beq
    \Bal
    (\Ve^p)^{(k+1)} & = (\Ve^p)^{(k)} + (\Delta\Ve^p)^{(k)} \\
    \phi^{(k+1)} & = \phi^{(k)} + \Delta\phi^{(k)} \\
    \Bxi^{(k+1)} & = \Bxi^{(k)} + \Delta\Bxi^{(k)} \\
    \Delta\gamma^{(k+1)} & = \Delta\gamma^{(k)} + \delta\gamma^{(k)}
    \Eal
  \Eeq
  Set $k \leftarrow k+1$ and go to step 2.
  \item Update and calculate back stress and the deviatoric part of Cauchy stress:
  \Beq
    \Ve^p_{n+1} = (\Ve^p)^{(k)} ~;~~
    \phi_{n+1} = \phi^{(k)} ~;~~
    \Bxi_{n+1} = \Bxi^{(k)} ~;~~
    \Delta\gamma_{n+1} = \Delta\gamma^{(k)}
  \Eeq
  and
  \Beq
    \Bal
    \widehat{\Bbeta}_{n+1} & = \widehat{\Bbeta}_n + \Delta\gamma_{n+1}~\Bh^{\beta}(\Bxi_{n+1}, \Ve^p_{n+1}, \phi_{n+1}) \\
    \Bbeta_{n+1} & = \widehat{\Bbeta}_{n+1} - \Third~\Tr(\widehat{\Bbeta}_{n+1})~\Bone \\
    \Bs_{n+1} & = \Bxi_{n+1} + \Bbeta_{n+1}
    \Eal
  \Eeq
  \item Update the temperature and the Cauchy stress
  \Beq
    \Bal
    T_{n+1} & = T_n +
     \cfrac{\chi_{n+1}~\Delta t}{\rho_{n+1}~C_p}~\sigma^{n+1}_y~\dot{\Ve^p}_{n+1}
     = T_n +
     \cfrac{\chi_{n+1}~\Delta\gamma_{n+1}}{\rho_{n+1}~C_p}~\sigma^{n+1}_y~h^{\alpha}_{n+1} \\
    p_{n+1} & = p(J_{n+1}) \\
    \kappa_{n+1} & = J_{n+1}~\left[\Deriv{p(J)}{J}\right]_{n+1} \\
    \Bsig_{n+1} & = \left[p_{n+1} - 3~\kappa_{n+1}~\alpha~(T_{n+1}-T_0)\right]~\Bone + \Bs_{n+1}
    \Eal
  \Eeq
\end{enumerate}

\section{Examples}
Let us now look at a few examples.
\subsection{Example 1}
Consider the case of $J_2$ plasticity with the yield condition
\Beq
  f := \sqrt{\frac{3}{2}} \Norm{\Bs-\Bbeta}{} - \sigma_y(\Ve^p, \dot{\Ve}, T, \dots) =
       \sqrt{\frac{3}{2}} \Norm{\Bxi}{} - \sigma_y(\Ve^p, \dot{\Ve}, T, \dots) \le 0
\Eeq
where $\Norm{\Bxi} = \sqrt{\Bxi:\Bxi}$. Assume the associated flow rule
\Beq
  \Bd^p = \dot{\gamma}~\Br = \dot{\gamma}~\Partial{f}{\Bsig} = \dot{\gamma}~\Partial{f}{\Bxi} ~.
\Eeq
Then
\Beq
  \Br = \Partial{f}{\Bxi} = \sqrt{\frac{3}{2}}~\cfrac{\Bxi}{\Norm{\Bxi}{}}
\Eeq
and
\Beq
  \Bd^p = \sqrt{\frac{3}{2}}~\dot{\gamma}~\cfrac{\Bxi}{\Norm{\Bxi}{}} ~;~~
  \Norm{\Bd^p}{} = \sqrt{\frac{3}{2}}~\dot\gamma ~.
\Eeq
The evolution of the equivalent plastic strain is given by
\Beq
  \dot{\Ve^p} = \dot{\gamma}~h^{\alpha} = \sqrt{\cfrac{2}{3}}~\Norm{\Bd^p}{} = \dot{\gamma}~.
\Eeq
This definition is consistent with the definition of equivalent plastic strain
\Beq
  \Ve^p = \int_0^t \dot{\Ve}^p~d\tau =
   \int_0^t \sqrt{\cfrac{2}{3}}~\Norm{\Bd^p}{}~d\tau ~.
\Eeq
The evolution of porosity is given by (there is no evolution of porosity)
\Beq
  \dot{\phi} = \dot{\gamma}~h^{\phi} = 0
\Eeq
The evolution of the back stress is given by the Prager kinematic hardening rule
\Beq
  \dot{\widehat{\Bbeta}} = \dot{\gamma}~\Bh^{\beta} = \frac{2}{3}~H'~\Bd^p
\Eeq
where $\widehat{\Bbeta}$ is the back stress and
$H'$ is a constant hardening modulus.  Also, the trace of $\Bd^p$ is
\Beq
  \Tr(\Bd^p) = \sqrt{\frac{3}{2}}~\dot{\gamma}~\cfrac{\Tr(\Bxi)}{\Norm{\Bxi}{}}~.
\Eeq
Since $\Bxi$ is deviatoric, $\Tr(\Bxi) = 0$ and hence $\Bd^p = \Beta^p$.
Hence, $\widehat{\Bbeta} = \Bbeta$ (where $\Bbeta$ is the deviatoric part of $\widehat{\Bbeta}$), and
\Beq
  \dot{\Bbeta} = \sqrt{\frac{2}{3}}~H'~\dot{\gamma}~\cfrac{\Bxi}{\Norm{\Bxi}{}} ~.
\Eeq

These relation imply that
\Beq
  \boxed{
  \Bal
    \Br & = \sqrt{\frac{3}{2}}~\cfrac{\Bxi}{\Norm{\Bxi}{}} \\
     h^{\alpha} & = 1 \\
     h^{\phi} & = 0 \\
    \Bh^{\beta} & = \sqrt{\frac{2}{3}}~H'~\cfrac{\Bxi}{\Norm{\Bxi}{}} ~.
  \Eal
  }
\Eeq
We also need some derivatives of the yield function.  These are
\Beq
  \Bal
  \Partial{f}{\Bxi} & = \Br \\
  \Partial{f}{\Ve^p} & = -\Partial{\sigma_y}{\Ve^p} \\
  \Partial{f}{\phi} & = 0 ~.
  \Eal
\Eeq

Let us change the kinematic hardening model and use the Armstrong-Frederick
model instead, i.e.,
\Beq
  \dot{\Bbeta} = \dot{\gamma}~\Bh^{\beta} = \frac{2}{3}~H_1~\Bd^p - H_2~\Bbeta~\Norm{\Bd^p}{} ~.
\Eeq
Since
\Beq
  \Bd^p = \sqrt{\frac{3}{2}}~\dot{\gamma}~\cfrac{\Bxi}{\Norm{\Bxi}{}}
\Eeq
we have
\Beq
  \Norm{\Bd^p}{} =
   \sqrt{\frac{3}{2}}~\dot{\gamma}~\cfrac{\Norm{\Bxi}{}}{\Norm{\Bxi}{}} =
   \sqrt{\frac{3}{2}}~\dot{\gamma} ~.
\Eeq
Therefore,
\Beq
  \dot{\Bbeta} = \sqrt{\frac{2}{3}}~H_1~\dot{\gamma}~\cfrac{\Bxi}{\Norm{\Bxi}{}}
    - \sqrt{\frac{3}{2}}~H_2~\dot{\gamma}~\Bbeta ~.
\Eeq
Hence we have
\Beq
  \boxed{
  \Bh^{\beta} = \sqrt{\frac{2}{3}}~H_1~\cfrac{\Bxi}{\Norm{\Bxi}{}}
    - \sqrt{\frac{3}{2}}~H_2~\Bbeta ~.
   }
\Eeq

\subsection{Example 2}
Let us now consider a Gurson type yield condition with kinematic hardening.  In this
case the yield condition can be written as
\Beq
  f := \cfrac{3~\Bxi:\Bxi}{2~\sigma_y^2} +
     2~q_1~\phi^{*}~\cosh\left(\cfrac{q_2~\Tr(\Bsig)}{2~\sigma_y}\right)
     - [1 + q_3~(\phi^*)^2]
\Eeq
where $\phi$ is the porosity and
\Beq
  \phi^* = \begin{cases}
             \phi & \text{for}~ \phi \le \phi_c \\
             \phi_c - \cfrac{\phi_u^* - \phi_c}{\phi_f - \phi_c}~(\phi - \phi_c) &
              \text{for}~ \phi > \phi_c
           \end{cases}
\Eeq
Final fracture occurs for $\phi = \phi_f$ or when $\phi_u^* = 1/q_1$.

Let us use an associated flow rule
\Beq
  \Bd^p = \dot{\gamma}~\Br = \dot{\gamma}~\Partial{f}{\Bsig} ~.
\Eeq
Then
\Beq
  \Br = \Partial{f}{\Bsig} = \cfrac{3~\Bxi}{\sigma_y^2} + \cfrac{q_1~q_2~\phi^{*}}{\sigma_y}~
   \sinh\left(\cfrac{q_2~\Tr(\Bsig)}{2~\sigma_y}\right)~\Bone ~.
\Eeq
In this case
\Beq
  \Tr(\Br) = \cfrac{3~q_1~q_2~\phi^{*}}{\sigma_y}~\sinh\left(\cfrac{q_2~\Tr(\Bsig)}{2~\sigma_y}\right)
  \ne 0
\Eeq
Therefore,
\Beq
  \Bd^p \ne \Beta^p ~.
\Eeq

For the evolution equation for the plastic strain we use
\Beq
  (\Bsig-\widehat{\Bbeta}):\Bd^p = (1 - \phi)~\sigma_y~\dot{\Ve}^p
\Eeq
where $\dot{\Ve}^p$ is the effective plastic strain rate in the matrix material.  Hence,
\Beq
  \dot{\Ve}^p = \dot{\gamma}~h^{\alpha}
    = \dot{\gamma}~\cfrac{(\Bsig - \widehat{\Bbeta}):\Br}{(1 - \phi)~\sigma_y} ~.
\Eeq

The evolution equation for the porosity is given by
\Beq
  \dot{\phi} = (1 - \phi)~\Tr(\Bd^p) + A~\dot{\Ve^p}
\Eeq
where
\Beq
A = \cfrac{f_n}{s_n \sqrt{2\pi}} \exp [-1/2 (\Ve^p - \Ve_n)^2/s_n^2]
\Eeq
and $ f_n $ is the volume fraction of void nucleating particles,
$ \Ve_n $ is the mean of the normal distribution of nucleation strains, and
$ s_n $ is the standard deviation of the distribution.

Therefore,
\Beq
  \dot{\phi} = \dot{\gamma}~h^{\phi} =
    \dot{\gamma}~\left[(1 - \phi)~\Tr(\Br) + A~
    \cfrac{(\Bsig - \widehat{\Bbeta}):\Br}{(1 - \phi)~\sigma_y}\right] ~.
\Eeq

If the evolution of the back stress is given by the Prager kinematic hardening rule
\Beq
  \dot{\widehat{\Bbeta}} = \dot{\gamma}~\Bh^{\beta} = \frac{2}{3}~H'~\Bd^p
\Eeq
where $\widehat{\Bbeta}$ is the back stress, then
\Beq
  \dot{\widehat{\Bbeta}} = \frac{2}{3}~H'~\dot{\gamma}~\Br ~.
\Eeq
Alternatively, if we use the Armstrong-Frederick model, then
\Beq
  \dot{\widehat{\Bbeta}} = \dot{\gamma}~\Bh^{\beta} =
   \frac{2}{3}~H_1~\Bd^p - H_2~\widehat{\Bbeta}~\Norm{\Bd^p}{} ~.
\Eeq
Plugging in the expression for $\Bd^p$, we have
\Beq
  \dot{\widehat{\Bbeta}} = \dot{\gamma}~
  \left[\frac{2}{3}~H_1~\Br - H_2~\widehat{\Bbeta}~\Norm{\Br}{}\right] ~.
\Eeq
Therefore, for this model,
\Beq
  \boxed{
  \Bal
  \Br & = \cfrac{3~\Bxi}{\sigma_y^2} + \cfrac{q_1~q_2~\phi^{*}}{\sigma_y}~
   \sinh\left(\cfrac{q_2~\Tr(\Bsig)}{2~\sigma_y}\right)~\Bone  \\
  h^{\alpha} &
    = \cfrac{(\Bsig - \Bbeta):\Br}{(1 - \phi)~\sigma_y} \\
  h^{\phi} & =
    (1 - \phi)~\Tr(\Br) + A~
    \cfrac{(\Bsig - \widehat{\Bbeta}):\Br}{(1 - \phi)~\sigma_y}  \\
  \Bh^{\beta} & =
   \frac{2}{3}~H_1~\Br - H_2~\widehat{\Bbeta}~\Norm{\Br}{}
  \Eal
  }
\Eeq
The other derivatives of the yield function that we need are
\Beq
  \Bal
  \Partial{f}{\Bxi} & = \cfrac{3~\Bxi}{\sigma_y^2} \\
  \Partial{f}{\Ve^p} & = \Partial{f}{\sigma_y}~\Partial{\sigma_y}{\Ve^p}
   = -\left[\cfrac{3~\Bxi:\Bxi}{\sigma_y^3} +
     \cfrac{q_1~q_2~\phi^*~\Tr(\Bsig)}{\sigma_y^2}~
     \sinh\left(\cfrac{q_2~\Tr(\Bsig)}{2~\sigma_y}\right)\right]~
     \Partial{\sigma_y}{\Ve^p}\\
  \Partial{f}{\phi} & = 2~q_1~\Deriv{\phi^*}{\phi}~
    \cosh\left(\cfrac{q_2~\Tr(\sigma)}{2~\sigma_y}\right)
    - 2~q_3~\phi^*~\Deriv{\phi^*}{\phi} ~.
  \Eal
\Eeq

\end{document}


\section{Design}
In this section we will first discuss an high-level design (\ref{sec:high-level})
and then propose some optimizations (\ref{sec:optimizations}).

\subsection{High-level design idea}
\label{sec:high-level}

\begin{figure}[h!]
  \centering
  \includegraphics{figs/vad_simple_schematic.pdf}
  \caption{High-level schematic of our VAD design.}
  \label{fig:simple_schematic}
\end{figure}

\cref{fig:simple_schematic} shows the high-level concept of the design, where we
can see:
\begin{itemize}
  \item \texttt{absnetwork}: a combinatorial network that extracts $|z|$. Both
    its input and its output have 16 bits since its maximum value is $2^{15}$.
    In \cref{sec:opt-abs}, we will evaluate whether to use an approximated
    but more efficient \texttt{absnetwork} that does not sum 1 to the complement.
  \item \texttt{squarepowernetwork}: a combinatorial network that evaluates
    $|z|^2$ on 32 bits. Note that, 31 bits would be enough since
    $|z|^2 \le (2^{15})^2 = 2^{30}$.
  \item \texttt{accumulator}: performs the recursive sum $S[n] = |z[n]|^2 + S[n - 1]$.
    The maximum sum value is $N \cdot 2^{30} = 256 \cdot 2^{30} = 2^{38}$,
    requiring 39 bits.
  \item \texttt{comparator}: combinatorial network that compares the output
    of the \texttt{accumulator} with $E_{th}'$, which must be represented on 39
    bits as well in order to be compared to the sum.
\end{itemize}

\subsection{Optimizations}
\label{sec:optimizations}

In order to reduce the number of resources used in the implementation, we can
check for possible optimizations.

\subsubsection{Comparison}
We note that $E_{th}' \simeq 13743895347$ and $\lceil\log_2 E_{th}' + 1\rceil = 34$.
This means that 34 bits would be enough to represent $E_{th}'$. This also implies
that we don't need the accumulator to sum up to more than $2^{34} - 1$, because
if the sum exceeds $2^{34}$, the sum is grater than the threshold.

Moreover we can initialize the initial value of the sum to the overflow value
minus the threshold, so that the overflow will indicate that the threshold has
been exceeded, without using the comparator logic. The initial value of the sum
is $S'[0] = 2^{34} - E'_{th} = 3435973837$ and we compute:
\begin{align*}
  S'[n] &= \bigg| S'[n-1] + |z[n]|^2 \bigg|_{2^{34}}\\
  ovf[n] &= \bigg\lfloor \frac{S'[n-1] + |z[n]|^2}{2^{34}} \bigg\rfloor
\end{align*}
$ovf[n] = 1$ does not imply that $ovf[n + 1] = 1$, but since we are summing
positive integers, the sum will obviously exceed $2^{34}$, so we must remember
that the threshold has been exceeded until the end of the frame.
For this purpose we used a Set-Reset Flip Flop, which is reset at the beginning
of the frame processing and eventually set by $ovf[n]$.

\subsubsection{Absolute value}
\label{sec:opt-abs}

Calling $Z$ the representation of $z[n]$ on 16 bits and $Z_a$ the representation
of $|z[n]|$ on 16 bits as well, we have that:
\begin{equation}
  Z_a = \begin{cases}
    Z & z[n] \ge 0 \\
    \bar{Z}+1 & z[n] < 0
  \end{cases}
\end{equation}
where $\bar{Z}$ is the complement of $Z$.

In order to simplify it, we could avoid adding 1 in the lower branch, achieving
two optimizations: a lower number of bits for the output (15 vs 16) and a
simpler logic.

\begin{equation}
  Z_a \simeq \widetilde{Z}_a = \begin{cases}
    Z & z[n] \ge 0 \\
    \bar{Z} & z[n] < 0
  \end{cases}
\end{equation}

This approximation introduces an error on the energy of the sample.
This error for negative $x[n]$ can be evaluated as
\begin{equation}
  \epsilon[n] = |z[n]|^2 - \tilde{z}^2[n] = Z_a^2 - \widetilde{Z}_a^2 = (\bar{Z}[n] + 1)^2 - \bar{Z}^2[n] = 2\bar{Z}[n] + 1
\end{equation}
The worst error we can make on one sample corresponds to $\bar{Z}[n] = 2^{15}$
so $\epsilon_{max} = 2^{16}+1$, which translates to a maximum error on the total
energy of $256 \cdot \epsilon_{max} \simeq 2^{24}$, which is $\sim 0.12\%$ of the
threshold.

In conclusion, the approximation can be safely introduced in order to save
resources on the FPGA.

\subsection{Clock and sampling}
The sampling rate is 16\si{\kilo\hertz}, but we can process the data at a faster
rate. If we used a 16\si{\kilo\hertz} clock and after a frame is processed,
another one followed, a clock cycle would be required for \texttt{FRAME\_START}.
So we loose a clock cycle at every frame. In order to avoid so, we could use a
faster clock and divide it, so that one short clock cycle is spent in resetting
everything at the \texttt{FRAME\_START}, but more clock cycles are
available to process the current input sample before it changes.

In order to perform the clock frequency division a properly configured counter
has been used. The overflow of the counter is used to enable the accumulator,
which must accumulate at a rate equal to the sampling one.
The accumulation should occur at instants which are in the middle of two sample
instants in order to improve reliability.


\documentclass[twocolumn,draft]{article}

\usepackage{savetrees}

\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{microtype}

\usepackage{amssymb}
\usepackage{amsfonts}
\usepackage{amsmath}
\usepackage{amsthm}

\usepackage[english]{babel}

\RequirePackage[l2tabu, orthodox]{nag}
\usepackage[all,warning]{onlyamsmath}

\usepackage{hyperref}

\usepackage{cuted}

\usepackage[style=numeric-comp]{biblatex}
\bibliography{notes}

\title{Genome-wide scan for linear mixed models}
\author{Danilo Horta and Francesco P. Casale}
\date{\today}

\include{definitions}

\begin{document}
	\maketitle

\section{Least squares}

We apply the method of least squares to estimate the fixed-effect sizes.

\subsection{Single block}

Let
\begin{equation}\label{eq:lmm}
  \bfy \sim \normal{\rmF\bfalpha}{\rmK}
\end{equation}
be the marginal likelihood of a LMM with a single covariate block $\rmF$.
The maximum likelihood estimation of the fixed effects is
\begin{equation*}
	\hat\bfalpha = \pinv{(\trans{\rmF}\pinv{\rmK}\rmF)}
	               \trans{\rmF}\pinv{\rmK}\bfy.
\end{equation*}
In practice, the method of least squares can be used to solve the above
equation without explicitly finding pseudoinverses.

\subsection{Double block}

Let
\begin{equation*}
  \bfy \sim \normal{\rmF\bfalpha + \rmG\bfbeta}{\rmK}
\end{equation*}
be the marginal likelihood of a LMM with two covariate blocks $\rmF$ and
$\rmG$.
We want to solve
\begin{equation*}
	\begin{bmatrix}
		\hat\bfalpha \\
		\hat\bfbeta
	\end{bmatrix} = \pinv{(\trans{\rmX}\rmX)}\trans{\rmX}\pinv{\rmK^{\haf}}\bfy
\end{equation*}
for
\begin{equation*}
\rmL = \pinv{\rmK^{\haf}}\rmF,~~ \rmR = \pinv{\rmK^{\haf}}\rmG\texto{and}
	\rmX =
		\begin{bmatrix}
			\rmL & \rmR
		\end{bmatrix}.
\end{equation*}

We have

\begin{strip}
	\begin{equation*}
		\pinv{(\trans{\rmX}\rmX)} =
		\begin{bmatrix}
			\pinv{(\trans{\rmL}\rmL)} + \pinv{(\trans{\rmL}\rmL)} (\trans{\rmL}\rmR)
				\pinv{\rmW} \trans{(\trans{\rmL}\rmR)} \pinv{(\trans{\rmL}\rmL)}
				& - \pinv{(\trans{\rmL}\rmL)} (\trans{\rmL}\rmR)
				\pinv{\rmW} \\
			- \pinv{\rmW} \trans{(\trans{\rmL}\rmR)}
			\pinv{(\trans{\rmL}\rmL)} & \pinv{\rmW}
		\end{bmatrix},
	\end{equation*}
\end{strip}
from Eq. 3 of~\cite{rohde1965generalized}, where
$\rmW = \trans{\rmR}\rmR - \trans{\rmR}\rmL\pinv{(\trans{\rmL}\rmL)}
\trans{\rmL}\rmR$.

Defining $\rmA = \trans{\rmL}\rmL$ and $\rmB = \trans{\rmL}\rmR$ leads us to
\begin{equation*}
	\pinv{(\trans{\rmX}\rmX)}\trans{\rmX} =
		\begin{bmatrix}
			\pinv{\rmA}\trans{\rmL}
			+ \pinv{\rmA}\rmB\pinv{\rmW}\trans{\rmB}\pinv{\rmA}\trans{\rmL}
			-\pinv{\rmA}\rmB\pinv{\rmW}\trans{\rmR}\\
			-\pinv{\rmW}\trans{\rmB}\pinv{\rmA}\trans{\rmL}
			+ \pinv{\rmW}\trans{\rmR}
		\end{bmatrix}.
\end{equation*}
Finally,
\begin{align*}
	&\pinv{(\trans{\rmX}\rmX)}\trans{\rmX} \pinv{\rmK^{\haf}}\bfy = \\
	&\begin{bmatrix}
		\pinv{\rmA}\trans{\rmF}\pinv{\rmK}\bfy
		+ \pinv{\rmA}\rmB\pinv{\rmW}\trans{\rmB}\pinv{\rmA}\trans{\rmF}\pinv{\rmK}\bfy
		-\pinv{\rmA}\rmB\pinv{\rmW}\trans{\rmG}\pinv{\rmK}\bfy \\
		\pinv{\rmW}\trans{\rmG}\pinv{\rmK}\bfy
		-\pinv{\rmW}\trans{\rmB}\pinv{\rmA}\trans{\rmF}\pinv{\rmK}\bfy
	\end{bmatrix}.
\end{align*}
A robust implementation of the above equation has to:
(i) associate matrix multiplications in such a way that a sequence of
$\pinv{\rmK}\dots\pinv{\rmK}$ is avoided;
and (ii) handle low-rank matrices $\rmW$.
A better association of matrix multiplications is given by
\begin{align*}
	&\pinv{(\trans{\rmX}\rmX)}\trans{\rmX} \pinv{\rmK^{\haf}}\bfy = \\
	&\begin{bmatrix}
		\pinv{\rmA}\trans{\rmF}\pinv{\rmK}\bfy
		+ \pinv{\rmA}\rmB\pinv{\rmW}(\trans{\rmB}\pinv{\rmA}\trans{\rmF}\pinv{\rmK}
		\bfy-\trans{\rmG}\pinv{\rmK}\bfy) \\
		\pinv{\rmW}(\trans{\rmG}\pinv{\rmK}\bfy
		-\trans{\rmB}\pinv{\rmA}\trans{\rmF}\pinv{\rmK}\bfy)
	\end{bmatrix}.
\end{align*}
$\pinv{\rmW}$ can be found via economic SVD decomposition.

\subsection{Batch scan}

Given a $n\times p$ covariate block $\rmG$, we want to quickly infer the
maximum likelihood estimations of $\bfalpha_i$ and $\beta_i$, for
$i \in \{1, \dots, p\}$ denoting the $\rmG$ columns.
We define a diagonal matrix
\begin{equation*}
	\rmW = \dotd{\trans{\rmG}}{\pinv{\rmK}\rmG} -
	       \dotd{\trans{\rmG}}{\pinv{\rmK}\rmF
				 \pinv{(\trans{\rmL}\rmL)}\trans{\rmF}\pinv{\rmK}\rmG},
\end{equation*}
where $\dotd{\cdot}{\cdot}$ is a function that returns the diagonal elements
of a matrix multiplication with asymptotically lower computational cost and
memory use.

Clearly,
\begin{equation*}
	\hat\bfbeta =
	\begin{bmatrix}
		\hat\beta_1 \\
		\hat\beta_2 \\
		\vdots \\
		\hat\beta_p
	\end{bmatrix}
	= \pinv{\rmW}(\trans{\rmG}\pinv{\rmK}\bfy
	-\trans{\rmB}\pinv{\rmA}\trans{\rmF}\pinv{\rmK}\bfy).
\end{equation*}
We know that
\begin{equation*}
	\hat\bfalpha_i = \pinv{\rmA}\trans{\rmF}\pinv{\rmK}
	(\bfy + (\rmG_i \rmW_i \trans{\rmG_i})
	(\pinv{\rmK}\rmF\pinv{\rmA}\trans{\rmF}\pinv{\rmK}\bfy - \pinv{\rmK}\bfy)),
\end{equation*}
where $\rmG_i$ is the $i$-th column of $\rmG$.
For one-shot computation we do
\begin{equation*}
	\hat\bfalpha = \pinv{\rmA}\trans{\rmF}\pinv{\rmK}
	\paren{\bfy \oplus \rmG \otimes
	\trans{\paren{(\rmW \trans{\rmG})
	  (\pinv{\rmK}\rmF\pinv{\rmA}\trans{\rmF}\pinv{\rmK}\bfy
		 - \pinv{\rmK}\bfy)}}},
\end{equation*}
where $\oplus$ and $\otimes$ are element-wise summation and multiplication
with broadcasting.

\section{Marginal likelihood}

Replace $\rmK$ by $\sigma^2\rmK$ in Eq.~\eqref{eq:lmm}.
Log of the marginal likelihood is given by
\begin{align*}
  \calL(\bfbeta, \sigma^2) &=
	-\haf\big(\log(2\pi) - \log\det|\sigma^2\rmK| \\
	&- \trans{(\bfy - \rmF\bfbeta)} \pinv{(\sigma^2\rmK)} (\bfy - \rmF\bfbeta)
	\big).
\end{align*}
The maximum likelihood estimation and the restricted likelihood estimation of
$\sigma^2$ are give by
\begin{equation*}
	\hat\sigma^2 = \frac{
		\trans{(\bfy - \rmF\hat\bfbeta)}
			\pinv{\rmK}
		(\bfy - \rmF\hat\bfbeta)
	}{k}
\end{equation*}
for $k = n$ and $k = n - p$, respectively.

\section{Likelihood ratio test}

Suppose that the null hyphotesis is given by
$\nullH: \bfbeta \in \Theta_0$ and the alternative one is given by
$\altH: \bfbeta \in \Theta_1$, where $\Theta_0$ is a subspace of $\Theta_1$,
and let $d$ be the difference between the $\Theta_1$ and $\Theta_0 $
dimensions.
Give that the null hyphotesis is the true one,
it follows from classical likelihood theory that
\begin{equation*}
	\til{\chi}^2_d \sim -2\ln
	\Bigg(
		\frac{\calL(\hat\bfbeta_0, \hat\sigma_0^2)}
		     {\calL(\hat\bfbeta_1, \hat\sigma_1^2)}
	\Bigg),
\end{equation*}
asymptotically, for maximum likelihood parameter estimation\footnote{This is
not true for rescricted maximum likelihood estimation.}
subject to $\hat\bfbeta_0 \in \Theta_0$ and $\hat\bfbeta_1 \in \Theta_1$
\cite{verbeke2009linear}.

\printbibliography\

\end{document}


\section{}

\subsection{Initial weights and biases}
\label{sec:weights}

Weights between input layer and hidden layer
\[
\begin{bmatrix}
    0.39653235 & -0.06798961 & -0.03110093 & 0.35617649 \\
    -0.2372905 & -0.08188699 & -0.2610652 & 0.1051389
\end{bmatrix}
\]

Biases between input layer and hidden layer
\[
\begin{bmatrix}
    0.2304629 \\
    -0.40825189 \\
    0.48841448 \\
    -0.49694009 \\
\end{bmatrix}
\]

Weights between hidden layer and output layer
\[
\begin{bmatrix}
-0.26704499 \\
0.00909349 \\
-0.10233779 \\
0.22680813 \\
\end{bmatrix}
\]

Biases between hidden layer and output layer
\[
\begin{bmatrix}
-0.35936101
\end{bmatrix}
\]

\subsection{}
\label{sec:sgd}
At first a neural network with single hidden layer having 4 neurons was constructed. The weights for the network was
randomly initialized in the range [-0.5, 0.5]. Sigmoid function was selected as the
activation function for the network. Then, stochastic gradient descent without
any modification was used to train the network. Plot of training cost with respect
to iteration is shown in figure \ref{fig:cost_sgd}.

A minimum threshold of 0.01 is selected as a stopping criteria for training algorithm.
The training is stopped as soon as average training error for samples was less than the threshold. The training algorithm
uses a learning rate($\eta$)=0.01.

\section{}
The neural network described in section \ref{sec:sgd} was again initialized with weights
presented in \ref{sec:weights}. It was then trained on the given samples using stochastic gradient descent with momentum
having momentum coefficient($\mu$)$=0.5$. The plot of training cost with respect to iteration is shown in \ref{fig:cost_sgd_momentum}.

Comparing the figures \ref{fig:cost_sgd} and \ref{fig:cost_sgd_momentum} it is clear that the SGD algorithm
with momentum reaches the minimum threshold of 0.01 in lesser number of iterations than unmodified SGD algorithm.


\documentclass[a4paper]{article}
\usepackage[english]{babel}
\usepackage{graphicx}
\usepackage{multicol}
\usepackage{amsmath}
\usepackage{hyperref}
\usepackage{amsthm}
\usepackage{geometry}
\geometry{a4paper}
\usepackage{fancyhdr}
\usepackage{xcolor}
\usepackage{amssymb}
\usepackage{multicol}
\theoremstyle{definition}
\newtheorem{exmp}{Example}[section]
\newtheorem{theorem}{Theorem}

\begin{document}
\author{Fractals}
\title{\textbf{}}
\maketitle
\tableofcontents
\noindent
\section{Basic Algebra}
\subsection{Simon's Factroring Trick}
\textbf{Simon’s Favorite Factoring Trick} (SFFT) is best explained with an example:
\begin{exmp}
    Find all positive integers \(x, y\) that satisfy
    \[
        xy- 2x - 4y = 0.
    \]
    \(Sloution\) Let us factor the first two terms:
    \[
        x(y-2) - 4y = 0.
    \]
    We want to find some way we can turn the \(y\) into a \(y - 2\). Let’s see what
    happens if we do that:
    \[
        x(y-2) - 4(y-2+2) = 0.
    \]
    \[
        x(y-2) - 4(y-2)  - 8= 0.
    \]
    \[
        x(y-2) - 4(y-2) = 8.
    \]
    Now, we can factor:
    \[
        (x-4)(y-2) = 8.
    \]
    Because \(x, y\) are positive integers,
    we know that \(x - 4 \text{ and } y - 2\) are simply the positive factors of 8
    \[
        x-4 = 1, y-2=8,
    \]
    \[
        x-4 = 2, y-2=4,
    \]
    \[
        x-4 = 4, y-2=2,
    \]
    \[
        x-4 = 8, y-2=1,
    \]
    Solving we get \framebox[1.04\width]{\((x,y)\in  \{(5,10), (6,6), (8,4), (12,3)\} \).}
    \\

    \noindent
    Now for the formal statement:
\end{exmp}
\begin{theorem}[\textbf{SFFT}]
    For all real numbers (although commonly used only for integers) \(x, y, a, b, \)
    \[
        xy + xa + yb + ab = (x + a)(y + bk).
    \]
    Two special common cases are: \(xy + x + y + 1 = (x+1)(y+1)\) and \(xy - x - y + 1 = (x-1)(y-1)\).
\end{theorem}
\end{document}


\documentclass[amsmath,amssymb,aps,pra,reprint,groupedaddress,showpacs]{revtex4-1}

\usepackage{multirow}
\usepackage{verbatim}
\usepackage{color,graphicx}

\input{../Common}

\begin{document}

\title{Fibonacci Sequence Alternative Representation}

\author{Lucchi Manuele}
\email[]{manuele.lucchi@studenti.unimi.it}
\affiliation{IT Department Students, Universita' degli Studi di Milano, Citta' degli Studi, Milano, Italia}

\date{\today}

\begin{abstract}
In this Research we will introduce a new representation of the Fibonacci Sequence starting from its Generative Function, the Binet Formula.
The result will be an equation that can be parallelized in a multithreaded system and that only uses positive integers.
\end{abstract}

\maketitle

\section{Introduction}
The Fibonacci Sequence [1] is a simple, but really important, Sequence in Natural Numbers that can be found in a lot of places in nature,
for example flowers' petals follow it in their numeration. The sequence is just the sum of its previous terms:
$$ F_{n} = F_{n-1} + F_{n-2} $$
In 1843 Binet introduced its Generative Function called the Binet Formula [2] (even if it was already known a century earlier by Euler, Daniel Bernoulli and de Moivre) that allows everyone to
calculate the $n_{th}$ Fibonacci Number without using a recursive approach
$$f(n) = \frac{1}{\sqrt{5}} \left[ \left( \frac{1+\sqrt{5}}{2} \right) ^n - \left( \frac{1-\sqrt{5}}{2} \right)^n \right] $$
However this representation uses irrational numbers like $\sqrt{5}$ that causes a certain level of imprecision in the result that should be a positive integer.
Our approach consists in using the Newton Binomial [3] to simplify all the irrational numbers and then optimize the result.

\section{Transformation}

The first step is to take out the $\frac{1}{2^n}$
$$f(n) = \frac{1}{2^n\sqrt{5}} \left[ \left( 1+\sqrt{5} \right) ^n - \left( 1-\sqrt{5} \right)^n \right]$$
Now we can notice that $(1 + \sqrt{5})^n$ and $(1 - \sqrt{5})^n$ are both in the form $(a + b)^n$ so they are suitable for the Newton Binomial transformation,
so we can write them like \\
$ A_n:= \sum_{k=0}^n \binom{n}{k} \left(\sqrt{5} \right) ^k $ and $ B_n:= \sum_{k=0}^n \binom{n}{k} \left( -\sqrt{5} \right) ^k $ both with $k \in N$.
If we unroll the sums we'll find some similarities between A and B.\\
So
\begin{gather*}
A = a\sqrt{5} +b5 + c\sqrt{5^3} + d5^2 + ... + z\sqrt{ 5 }^k \\
B = -a\sqrt{5} +b5 + -c\sqrt{5^3} + d5^2 + ... + z \left( -\sqrt{5} \right) ^k
\end{gather*}
with $a, b, c, d,$  ..., $z \in N$. \\

Since it's $f(n)=\frac{1}{2^n\sqrt{5}} \left[ A - B \right]$ we'll have
\begin{align*}
A - B &= a\sqrt{5} +b5 + c\sqrt{5^3} + d5^2 + ... + z\sqrt{5}^k  \\
&- \left( -a\sqrt{5} +b5 + -c\sqrt{5^3} + d5^2 + ... + z \left( -\sqrt{5} \right)^k \right) \\
&= a\sqrt{5} +b5 + c\sqrt{5^3} + d5^2 + ... + z\sqrt{5}^k +a\sqrt{5} -b5 \\
&+ c\sqrt{5^3} - d5^2 + ... - z \left( -\sqrt{5} \right) ^k
\end{align*}

Now we can see that all members that have an integer exponent can be reduced. If we define $C:= A - B$ we get
\begin{align*}
C_n &= \sum_{k=0}^n \binom{n}{k} \left(\sqrt{5} \right) ^k - \sum_{k=0}^n \binom{n}{k} \left(-\sqrt{5} \right) ^k\\
&= \sum_{k=0}^n \binom{n}{k} \left[ \left(\sqrt{5} \right) ^k - \left(-\sqrt{5} \right) ^k \right]
\end{align*}
We can now split the sum again, this time differentiating between odd and even steps

\begin{align*}
C_n &= \sum_{k=0}^{n/2} \binom{n}{2k} \left[ \left(\sqrt{5} \right) ^{2k} - \left(-\sqrt{5} \right) ^{2k} \right]\\
&+ \sum_{k=0}^{n/2} \binom{n}{2k + 1} \left[ \left(\sqrt{5} \right) ^{2k + 1} - \left(-\sqrt{5} \right) ^{2k +1} \right]
\end{align*}

Now if we analyze the sums one by one, we can observe that the first one converges to 0, $\forall n \in{N}$ where $n$ is odd
$$ \sum_{k=0}^{\frac{n}{2}} \binom{n}{2k} \left[ \left(\sqrt{5} \right) ^{2k} - \left(-\sqrt{5} \right) ^{2k} \right] = 0 $$
while the second one will become

\begin{align*}
C &= \sum_{k=0}^{\frac{n}{2}} \binom{n}{2k + 1} \left[ \left(\sqrt{5} \right) ^{2k + 1} - \left(-\sqrt{5} \right) ^{2k + 1} \right]\\
&= 2 \sum_{k=0}^\frac{n}{2} \binom{n}{2k + 1} \left(\sqrt{5} \right) ^{2k + 1}
\end{align*}

Since $2k + 1$ is always odd, we know that

\begin{align*}
&\left(-\sqrt{5} \right) ^{2k + 1} =  - \left( \sqrt{5} \right) ^{2k + 1} \implies \\
&\left( \sqrt{5} \right) ^{2k + 1} - \left(-\sqrt{5} \right) ^{2k + 1} = 2\left(\sqrt{5} \right) ^{2k + 1}
\end{align*}

In the end what remains is our $S_n$
\begin{align*}
S_n &= \frac{2}{2^n\sqrt{5}} \sum_{k=0}^\frac{n}{2} \binom{n}{2k + 1} \left( \sqrt{5} \right) ^{2k +1} \\
&=\frac{1}{2^{n-1}} \sum_{k=0}^\frac{n}{2} \binom{n}{2k + 1}5^{k}
\end{align*}

This is a first version of the alternative formula that already supports a certain level of parallelization and, since every Fibonacci's Number is an integer, doesn't need to handle real numbers like the Binet one

\begin{thebibliography}{24}

  \bibitem{fibonaccigeneral}
    {OEIS},
    \textit{Sequence A000045 (Fibonacci Numbers)}

  \bibitem{binetgeneral}
    {DA VEDERE},
    \textit{DA VEDERE}

  \end{thebibliography}

\end{document}


\section{conclusion}\label{sec:conclusion}
In this paper, we have presented a rigorous application of an artificial neural network methodology to the mitigation of the observational systematics in galaxy clustering measurements of an eBOSS-like ELG sample selected from DR7 (see \S~\ref{sec:data}). We have investigated the galaxy density dependency on 18 imaging attributes of the data (see Fig. \ref{fig:eboss_dr7}). We compare the performance of the neural network with that of the traditional, linear and quadratic multivariate regression methods. The key aspects of our neural network methodology are:\\

\begin{itemize}
    \item The application of k-fold cross-validation, which implements the training-validation-test split to tune the hyper parameters by evaluating how well the trained network generalizes to the unseen,  validation data set and therefore to suppress overfitting when applied to the test set;

    \item The repeated split process until we cover the entire data footprint as test sets;

    \item The elimination of redundant imaging maps by the feature selection procedure to further reduce the overfitting problem and therefore protect the cosmological clustering signal.
\end{itemize}

We apply the output of our pipeline, i.e., the selection mask for the DR7 footprint to the observed galaxy density field. Benchmark selection masks are also produced employing the linear and quadratic polynomial regression. Comparing statistical results before and after applying the selection masks, we find that:\\
\begin{itemize}
    \item Galactic foregrounds are the most dominant source of contamination in this imaging dataset (see Figs. \ref{fig:nnbar}, \ref{fig:clcross}, and \ref{fig:xicross}).

    \item This contamination causes an excess clustering signal in the auto power spectrum and correlation function of the galaxy density field on large scales (see Fig. \ref{fig:clxi}).

    \item All mitigation techniques e.g., the neural network method as well as the linear multivariate models using the linear and quadratic polynomial functions, are able to reduce the auto and cross clustering signals (see Figs. \ref{fig:xicross} and \ref{fig:clcross});

    \item However, the neural network removes the excess clustering more effectively in the auto power spectrum and correlation function of galaxies (see Fig. \ref{fig:clxi}).
\end{itemize}

The last result implies that our neural network method has a higher flexibility than both linear multivariate models we tested, and it is therefore capable of capturing the non-linear systematic effects in the observed galaxy density field.\\

We apply our methodology on two sets of 100 log-normal mock datasets with (`contaminated mocks') and without (`null mocks') imaging contamination to evaluate how well the ground truth cosmological clustering can be reconstructed in both cases, and therefore to validate the systematic mitigation techniques. All mitigation techniques are applied in the same way we treat the real data. The key results of our mock test are as follows:\\

\begin{itemize}
    \item The feature selection procedure is able to identify most of the ten contamination input maps as important for the contaminated mocks while correctly identifying most of the maps as redundant for the null mocks (see Fig. \ref{fig:mockablation}).

    \item All three mitigation methods, i.e., the linear polynomial, quadratic polynomial, and neural network methods, perform similarly in terms of the residual bias in the presence of contamination. This is expected since the contamination model is based on the linear polynomial model which all three methods are capable of reproducing. The default neural network tends to slightly under-correct which is the outcome of the feature selection procedure. On the other hand, the linear and quadratic polynomial methods tend to slightly over-correct (see the right panel of Fig. \ref{fig:deltaclmock}).;

    \item In the absence of contamination, the neural network is the most robust against regressing out the cosmological clustering. This is mainly due to the feature selection process that appropriately reduces the flexibility of the mitigation (see the left panel of Fig. \ref{fig:deltaclmock}). Based on this result, we implement the feature selection procedure for DR7.

    \item Using $\chi^{2}$ statistics, we quantify the bias and find that for the null mocks, the default neural network recovers the underlying clustering within $1\sigma$ C.L. (see Eq. \ref{eq:chi2sigma}) while the other methods return more than $4\sigma$ C.L. bias. For the contaminated mocks, all of the methods return biased clustering with $2.8-3.9\sigma$ C.L. which indicates that it is crucial for cosmological parameter estimation to determine the residual systematic uncertainty in scales sensitive to the parameters of interest (see the middle panels of Figs \ref{fig:deltaclmock} and \ref{fig:mockdclextra}).

    \item All methods do not increase fractional variance during the mitigation process (see the bottom row of Fig. \ref{fig:deltaclmock}).\\

\end{itemize}

We also employ the mocks to investigate the remaining systematic effects in the data (Figs \ref{fig:nnbar}-\ref{fig:xicross}). While the neural network methods outperform the conventional methods, we conclude that the data exhibit around 19\% residual systematics in the target number density (Figs \ref{fig:chi2pdf} \& \ref{fig:chi2breakdown}). Our analysis suggests that a more rigorous masking on  $depth-g$ (e.g., $depth-g>24.95$) improves the mean density at the cost of losing $9\%$ of data. To use this sample for cosmology, we therefore suggest a) accounting for 19\%  (or more, depending on the assumption on the baseline) additional systematic errors to the statistical errors in the density field level; b) performing further analysis of systematics and improvement of the mitigation method to deal with the depth and sky background issues.\\

To conclude, our analyses illustrate that the neural network method we developed in this paper is a promising tool for the mitigation of the large-scale spurious clustering that is likely raised by the imaging systematics. Our method is more robust against regressing out the cosmological clustering than the traditional, linear multivariate regression methods. Such improvement will be particularly crucial for an accurate measurement of non-Gaussianity from the large-scale clustering of current eBOSS and upcoming DESI and the LSST surveys. Our method is computationally less intensive than other approaches such as the Monte Carlo injection of fake galaxies: analyzing DR7 using our default neural network method requires less than six CPU hours. Application of our methodology on any imaging dataset would be straightforward. Our systematics mitigation methodology pipeline is publicly available at \url{https://github.com/mehdirezaie/SYSNet}.