\section{The Lemke-Howson Algorithm}
\label{lh-sect}

Theorem \ref{nash-ppad-complete-thm} suggests that the study of solutions of
$n$-{\sc Nash} as endpoints of paths can yield interesting results about
the complexity of the problem itself. In this
section we will study an algorithm that describes exactly this idea.

Let $P$ be a simple $d$-polytope with $n$ facets.
We {\em pivot on the vertices} of $P$ by moving from a
vertex $x$ to another vertex $y$ connected to $x$ by an edge,
see Figure \ref{pivot-vertex-fig}.
Note that, since $P$ is simple, there are exactly
$d$ possible choices for $y$.
Analogously, we {\em pivot on the facets} of a simplicial
polytope $P^\Delta$ in
dimension $d$ by moving from a facet $F$ to a facet $G$ that
shares all vertices but one with~$F$, see Figure \ref{pivot-facet-fig}.
As above, since $P^\Delta$ is simplicial, there are $d$ possible choices
for $G$.

\begin{figure}[hp]
\strut\hfill
\includegraphics[width=38ex]{chapter-3/fig-lh/cube-pivot.pdf}%
\hfill\strut
\caption[A pivot on the vertices of the cube]{%
A pivot from vertex $x$ to vertex $y$ on the edge of a cube.
}
\label{pivot-vertex-fig}
\end{figure}

\begin{figure}[hp]
\strut\hfill
\includegraphics[width=48ex]{chapter-3/fig-lh/octahedron-pivot.pdf}%
\hfill\strut
\caption[A pivot on the facets of the octahedron]{%
A pivot from facet $F$ of an octahedron to facet $G$.
}
\label{pivot-facet-fig}
\end{figure}

\clearpage

Suppose now that there is a labeling $l_f:[n]\to [d]$ of the facets of the
simple polytope $P$.
If we pivot from vertex $x$ to vertex $x'$ we ``leave behind'' a facet $F$
with label $k$ to which $x$ belongs, but $x'$ does not. At the same
time, we ``reach'' a facet $F'$ with label $h$, to which $x$ does not
belong, but $x'$ does.
Therefore, if $x$ has labels $(l_1,\ldots,k,\ldots,l_d)$, then
$x'$ has labels $(l_1,\ldots,h,\ldots,l_d)$. We call this
{\em dropping label $k$ and picking up label $h$}, or
{\em pivoting on label $k$}; see Figure \ref{pivot-vertex-label-fig}.
Analogously, if there is a labeling $l_v:[n]\to [d]$ of the vertices
of the simplicial poytope $P^\Delta$
and we pivot from a facet $F$ with labels $(l_1,\ldots,k,\ldots,l_d)$
to a facet $F'$ with labels $(l_1,\ldots,h,\ldots,l_d)$, we say that we
{\em drop label $k$ and pick up label $h$}, or that we
{\em pivot on label $k$}; see Figure \ref{pivot-facet-label-fig}.

\begin{figure}[ht]
\strut\hfill
\includegraphics[width=40ex]{chapter-3/fig-lh/cube-pivot-label.pdf}%
\hfill\strut
\caption[A pivot on the vertices of the labeled cube]{%
A pivot on label $k$: drop vertex $x$ with labels $(l_1,l_2,k)$ and
pick up vertex $x'$ with labels $(l_1,l_2,h)$.
}
\label{pivot-vertex-label-fig}
\end{figure}

\clearpage

\begin{figure}[htb]
\strut\hfill
\includegraphics[width=48ex]{chapter-3/fig-lh/octahedron-pivot-label.pdf}%
\hfill\strut
\caption[A pivot on the facets of the labeled octahedron]{%
A pivot on label $k$: drop a facet with labels $(l_1,l_2,k)$ and
pick up a facet with labels $(l_1,l_2,h)$.
}
\label{pivot-facet-label-fig}
\end{figure}

Consider a labeling function $l:[n]\to[d]$, and a subset $S$
of $[n]$ with $|S|=d$.
Then $S$ is called {\em almost completely labeled} if
\begin{equation}
\label{acl-equation}
l(S)~=~\{\,l(s)\mid s\in S\}~=~[d]\setminus\{k\}\,
\end{equation}
that is, all labels appear once in $S$ except for one {\em
missing label} $k\in [d]$.
Since $|S|=d$, in that case there is one
{\em duplicate label} $h\in [d]$ that appears twice in~$S$.

Let $S$ be the set of labels of a vertex in a
simple polytope, or the set of labels of a facet in a
simplicial polytope.
We call this vertex (or, respectively, facet) {\em almost
completely labeled vertex (or facet)} if it is almost completely
labeled by $S$ with respect to the labeling of the facets
(or, respectively, vertices) of the polytope.
It is easy to see that if we pivot from an almost completely
labeled vertex (or facet) on the duplicate label, or from a
completely labeled vertex (or facet) on any label, this
becomes the missing label~$k$, and we reach either an
almost completely labeled or a completely labeled vertex
(or facet).

The algorithm by Lemke and Howson \cite{lh} finds one Nash
equilibrium of a bimatrix game.
In a modern description (e.g., Savani and von Stengel \cite{svs}),
it employs pivoting on the vertices of a simple polytope,
moving through a succession of almost completely labeled
vertices with missing label~$k$, where this polytope is
the product $P\times Q$ of the best-response polytopes.
This can be abstracted slightly further by considering only
a single polytope $P$ in dimension $d$ with facets labels
from $[d]$.
Algorithm~\ref{lh-alg} gives this latter version; for simplicity
of notation, we will call it {\em Lemke-Howson Algorithm}.
Algorithm~\ref{lh-alg} also computes a ``Lemke
path'', in the terminology of Morris~\cite{morris}; this, in turn,
can be used to prove some fundamental properties
of both the Lemke-Howson Algorithm and the Nash equilibria of a
bimatrix game.

\begin{algorithm}[htb]
\SetKwInOut{Input}{input}
\SetKwInOut{Output}{output}
\Input{%
A simple $d$-polytope $P$ with $n$ facets and a
labeling $l_f:[n]\to [d]$ of the facets of $P$.
A vertex $x_0$ of $P$ that is completely labeled by $l_f$.
}
\Output{%
A vertex $x\neq x_0$ of $P$ that is completely labeled by $l_f$.
}
\BlankLine
choose any label $k\in [d]$ as missing label \\
pivot on label $k$ from $x_0$ to $x$ reaching a new facet
with label $h$\\
\While{ $h\ne k$, so $x$ is not completely labeled ~}
{
% you don't use x_0 in the while loop
pivot away from the other facet with label $h$ from $x$ to $x'$  \\
let $h$ be the label of the new facet of $x'$ \\
set $x = x'$
}
\Return $x$
\caption{Lemke-Howson}
\label{lh-alg}
\end{algorithm}

\begin{proposition}\label{lh-works-ppa-thm}
The Lemke-Howson Algorithm \ref{lh-alg} returns a solution to
the {\bf PPA} problem {\sc Another Completely Labeled Vertex}.
Furthermore, the number of completely labeled vertices in a simple
polytope with labeled facets is even.
\end{proposition}

\begin{proof}
We first show that the Lemke-Howson Algorithm works.
From the completely labeled vertex $x_0$, there is a unique
edge that leaves the facet with label~$k$ which leads to a
new vertex $x$, as in step~2 of the algorithm.
If $x$ is completely labeled, then the algorithm terminates
with output $x$, and it is trivial to see that $x\ne x_0$.
Otherwise, $x$ is an almost completely labeled vertex with
duplicate label~$h$, where one of the facets that contain $x$
and have label~$h$ is a ``new'' facet that did not contain
the preceding vertex on the Lemke path.
Since $S$ is simple, $x$ is always on exactly $d$ facets
and the duplicate label is unique.
Hence no vertex, including $x_0$, can ever be re-visited
on the path because it would otherwise offer an alternative
way to proceed when the vertex was encountered for the first
time.

The parity result is proven by the following argument: each Lemke path is
uniquely determined by its missing label and its starting point, so the
Lemke path from the endpoint with the same missing label will lead back
to the starting point. Since the endpoint and the starting point are
different, the Lemke paths must connect an even number of points.

Finally, for each label $k\in [d]$ chosen in line 1 of
Algorithm~\ref{lh-alg}, the Lemke paths are disjoint paths connecting
all the completely labeled vertices of $P$, with a standard
starting point $x_0$.
The problem {\sc Another Completely Labeled Vertex} corresponds to finding
a non-standard endpoint of this graph, which is a {\bf PPA} problem.
\end{proof}

Proposition \ref{lh-works-ppa-thm} can be extended. Lemke paths can be
used to prove that {\sc Another Completely Labeled Vertex} is in {\bf PPAD},
not just in {\bf PPA}. This is done
by giving a {\em sign} (positive or negative) to the vertices.
It can be proven that the endpoints of the Lemke paths have opposite
sign, and the paths can therefore be oriented accordingly.
Shapley \cite{shapley} proved an analogous result:
two Nash equilibria at the ends of a
Lemke path have opposite {\em index}, a concept analogous to sign but
defined using determinants on the payoff matrices for the equilibrium
support.
The index of a Nash equilibrium is usually normalized, by
multiplication with $-1$ of all signs if necessary, so that
the artificial equilibrium has index $-1$;
then a nondegenerate game with $n$ Nash equilibria
with index $+1$ has $n-1$ Nash equilibria with index $-1$.
For an in-depth study of the topics related to sign in the Lemke-Howson
and other algorithms, we refer to V\'{e}gh and von Stengel \cite{vvs}.

Applying the parity result of Proposition \ref{lh-works-ppa-thm} to the
case of a bimatrix game (not necessarily a unit vector game), and
remembering that the point $(\0,\0)$ corresponds to the
``artificial'' equilibrium, we have the following result, due to Lemke and
Howson \cite{lh}.

\begin{theorem}{\rm (Lemke-Howson \cite{lh})}
Every non-degenerate bimatrix game has an odd number of Nash equilibria.
\end{theorem}

There are two ways of using the Lemke-Howson Algorithm to find a Nash
equilibrium of a bimatrix game $(A,B)$.
The first one is to ``symmetrize'' the game as in Proposition
\ref{symmetrize-c}. Let
$R = \{ z\in\reals^{m+n}\ |\ z\geq\0,\ Cz\leq\1 \}$ be the
polytope associated to the game $(C,C\T)$, where
\[
C=\binom{~0~~~ A\,}{B\T~ 0\,}.
\]
The facets of $C$ correspond to $2(m+n)$ inequalities. We label both
the $i$-th and the $(m+n+i)$-th inequality as $i\in [m+n]$ and we apply the
Lemke-Howson algorithm starting from the vertex $\0$. This returns a Nash
equilibrium $(z,z)$ of $C$, which corresponds to a Nash equilibrium
$(x,y)=z$ of $(A,B)$.
We can also follow the ``traditional'' exposition of the Lemke-Howson
Algorithm given by Shapley \cite{shapley}. In this version,
we alternate a move on the best response polytopes $P$ and a move
on the best response polytope $Q$ of (\ref{br-polytopes}).
Since the polytopes $P$ and $Q$ are in $\reals^m$ and $\reals^n$, whereas $R$
is a polytope in $\reals^{m+n}$, the second version is much easier to
visualize.

\clearpage

\begin{example}({\rm Savani and von Stengel \cite{uvg}})
Consider the $3\times 3$ game $(A,B)$ of Example \ref{br-game-ex}.
\begin{equation*}
A = \left(\begin{matrix}1&0&0\\ 0&1&0\\
0&0&1\end{matrix}\right),
\qquad
B = \left(\begin{matrix}0&2&4\\ 3&2&0\\
0&2&0\end{matrix}\right).
\end{equation*}
The best response polytopes can be represented as the best response regions
of Figure \ref{br-regions-fig} extended to the origin $\0$,
as in Figure~\ref{lh-path-fig}.

The path starts from $(\0,\0)$.
We choose the missing label~1 and move in the polytope $P$.
Then label~6 is duplicate; so we drop it and we make the next move on the
polytope $Q$, and so on until we reach the point $x$ in $P$
and $y$ in~$Q$, which gives here the only Nash equilibrium $(x,y)$ of $(A,B)$.

\begin{figure}[hbt]
\strut\hfill
\includegraphics[width=70ex]{chapter-3/fig-lh/lemke-path.pdf}%
\hfill\strut
\caption[A Lemke path for a bimatrix game]{%
Lemke path for missing label 1 on the best response polytopes of
player 1 (left) and player 2 (right) of game (\ref{AB}).
}
\label{lh-path-fig}
\end{figure}
\end{example}

It is possible to have an equilibrium that cannot be reached applying the
Lemke-Howson Algorithm from the artificial equilibrium, or even from
the endpoint of a Lemke path from the artificial equilibrium. This
can be seen in the next example, where the Lemke paths form two disconnected
components. Notice that, by the parity result of
Proposition \ref{lh-works-ppa-thm}, each one of these components must
contain an even number of equilibria (either Nash or artificial), since all
these equilibria are endpoints of Lemke paths.

\begin{example}(R. Wilson, cited in Shapley \cite{shapley})
Consider the symmetric game $(C,C\T)$ with
\begin{equation}
\label{disj-lp-game}
C = \left(
    \begin{matrix}
        0&3&0 \\
        2&2&0 \\
        3&0&1
    \end{matrix}
    \right).
\end{equation}
There are three equilibria of $(C,C\T)$, all of them symmetric,
at $(x_i,x_i)$ with
$x_1=(0,0,1)$,
$x_2=(1/6,1/3,1/2)$ and
$x_3=(1/3,2/3,0)$.

All Lemke paths from the
artificial equilibrium $(0,0)$ end at $(x_1,x_1)$, and consequently all
other Lemke paths connect $(x_2,x_2)$ and $(x_3,x_3)$; see
Figure \ref{disj-lp-br-poly}.

\begin{figure}[hbt]
\strut\hfill
\includegraphics[width=70ex]{chapter-3/fig-lh/shapley-game.pdf}%
\hfill\strut
\caption[A game with disjoint Lemke paths]{%
The Lemke paths for missing label 1 (yellow), 2 (green) and 3 (pink) on the best
response polytopes of game (\ref{disj-lp-game}).

The paths for missing label
4, 5 and 6 on the best response polytope of player 1 are the same
as the paths of 1, 2 and 3 on the best response polytope of player 2,
and vice versa.
}
\label{disj-lp-br-poly}
\end{figure}
\end{example}

The dual version of the Lemke-Howson Algorithm \ref{lh-alg} and of
Proposition \ref{lh-works-ppa-thm} is straightforward; analogously,
the proof can be extended to show that {\sc Another Completely Labeled Facet}
is {\bf PPAD}.

\clearpage

\begin{algorithm}[ht]
\SetKwInOut{Input}{input}
\SetKwInOut{Output}{output}
\Input{%
A simplicial $m$-polytope $P^\Delta$ with $n$ vertices and a
labeling $l_v:[n]\to [d]$ of the vertices of $P^\Delta$.
A facet $F_0$ of $P^\Delta$ that is completely labeled by $l_v$.
}
\Output{%
A facet $F\neq F_0$ of $P^\Delta$ that is completely labeled by $l_v$.
}
\BlankLine
choose any label $k\in [d]$ as missing label \\
pivot on label $k$ from $F_0$ to $F$ which has a new vertex
with label $h$ \\
\While{ $h\ne k$, so $F$ is not completely labeled ~}
{
pivot away from the other vertex with label $h$ from $F$ to $F'$  \\
let $h$ be the label of the new vertex of $F'$ \\
set $F = F'$
}
\Return $F$
\caption{Dual Lemke-Howson}
\label{lh-dual-alg}
\end{algorithm}

\begin{proposition}\label{lh-dual-works-ppa-thm}
The Dual Lemke-Howson Algorithm \ref{lh-dual-alg} returns
a solution to the {\bf PPAD}
problem {\sc Another Completely Labeled Facet}.
Furthermore, the number of completely labeled facets
in a simplicial polytope with labeled vertices is even.
\end{proposition}

By Theorem \ref{unit-vector-thm} and Theorem \ref{unit-vector-dual-thm},
in the case of unit vector games it is enough to apply the
Lemke-Howson Algorithm~\ref{lh-alg} to the polytope $P^l$ in
(\ref{p-l}), or the Dual Lemke-Howson Algorithm \ref{lh-dual-alg}
to the polytope $P^\Delta$ in (\ref{p-l-dual}).
The following theorem by Savani and von Stengel \cite{uvg} guarantees that
not only does this yield a Nash equilibrium, but no potential solutions
are ``lost'' considering the polytope $P^l$ with $m$ labels
instead of the product of polytopes $P\times Q$ with $m + n$ labels;
an analogous result holds for the dual case.

\begin{theorem}\label{unit-paths}
Let $(U,B)$ be a unit vector game, with
$U=[e_{l(1)}\cdots e_{l(n)}]$ for a labeling $l:[n]\to [m]$.
Let
\[
\arraycolsep.3em
\begin{array}{rcll}
P&=&\{ x\in\reals^m&\mid\ x\geq\0,\ B\T x\leq\1 \}, \\
Q&=&\{ y\in\reals^n&\mid\ A y\leq\1,~y\geq\0 \}, \\
\end{array}
\]
as in $(\ref{br-polytopes})$, and let
\[
P^l=\{ x\in\reals^m \mid\ x\geq\0,\ B\T x\leq\1 \}
\quad \hbox{with labels in $[m]$ as in $(\ref{facet-labeling-unitv})$}
\]
as in $(\ref{p-l})$.
Then for the missing label $k\in [m]$
the Lemke path on $P\times Q$ projects to a path on $P$ that corresponds
to the Lemke path on $P^l$ for the missing label~$k$. For the missing label
$k=m+j$, where $j\in [n]$, the Lemke path on $P\times Q$ projects to a
path on $Q$ that corresponds to the Lemke path on $P^l$ for the
missing label~$l(j)$.
\end{theorem}

We finally focus on the case of Gale games.
In line with Proposition \ref{galenash-to-another-gale}, we look for
solutions of the problem \anothergale, see Table \ref{another-gale}.
By Proposition \ref{d-even-another-gale}, it is enough to study the
case of Gale strings $s\in G(d,n)$ with $d$ even.
We will consider these as ``wrapped-around strings''.

Let $s(i)=1$ for an index $i\in [n]$. Then, by the Gale evenness condition,
there is an odd run of \1's either on the left or on the right
of position $i$ in $s$. Let $j$ be the first index after this run.
A {\em pivot from $s$ to $s'$} is given by setting $s'(i)=0$
and $s'(j)=1$ to yield the new string~$s'$ which otherwise
agrees with~$s$.
If there is a labeling $l_s:[n]\to [d]$, we say that we
{\em drop label $l_s(i)$} and
{\em pick up label $l_s(j)$}, or that we {\em pivot on label $l_s(i)$},
specifying the index $i$ when the label $l_s(i)$ is not enough to
identify it.
The {\em Lemke-Howson for Gale Algorithm} is given
in Algorithm~\ref{lhg-alg}.

\clearpage

\begin{algorithm}%\label{lhg-alg}
\SetKwInOut{Input}{input}
\SetKwInOut{Output}{output}
\Input{%
A labeling $l_s:[n]\to [d]$, where $d$ is even, such that there is a
completely labeled Gale string $s_0 \in G(d,n)$.
}
\Output{%
A Gale string $s\in G(d,n)$ that $s$ is completely labeled by $l_s$,
such that $s\neq s_0$.
}
\BlankLine
choose a missing label $k\in [d]$ \\
pivot on label $k$ from $s_0$ to $s$ reaching a new \1 bit
with label $h$\\
\While{ $h\ne k$, so $s$ is not completely labeled ~}
{
pivot away from the other \1 bit in $s$ with label $h$ from $s$ to $s'$  \\
let $h$ be the label of the new \1 bit in $s'$ \\
set $s = s'$
}
\Return $s$
\caption{Lemke-Howson for Gale}
\label{lhg-alg}
\end{algorithm}

The next example illustrates the correspondence between the Dual
\linebreak[5]
Lemke-Howson Algorithm and the Lemke-Howson for Gale Algorithm.

\begin{example}
Figure \ref{lhg-123432-fig} shows the cyclic polytope $C_4(6)$
with the labeling
\[
\begin{array}{rll}
l_v(i)= & i\quad & \text{ for }i\in [4], \\
l_v(5)= & 3,  \\
l_v(6)= & 2.
\end{array}
\]
This corresponds to the labeling $l_s=123432$ for $G(4,6)$ given in
Example \ref{c46-123432-ex}, for which there are four completely labeled
Gale strings:
$s_A=\1\1\1\100$, $s_B=\1\10\1\10$, $s_C=\100\1\1\1$ and
$s_D=\10\1\10\1$. These, in turn, correspond to the facets $A$, $B$, $C$
and $D$ of $C_4(6)$, that are exactly the completely labeled facets for $l_v$.

From the point of view of Gale strings, pivoting on label 3
from
\linebreak[4]
$s_A=\1\1\1\100$ returns $s_B=\1\10\1\10$. Analogously,
pivoting on label 3 from facet $A$ returns facet $B$.

\begin{figure}[ht]
\strut\hfill
\includegraphics[width=40ex]{chapter-3/fig-lh/123432-lh-pivot.pdf}%
\hfill
\small
\begin{tabular}{c | c @{ } c @{ } c @{ } c @{ } c @{ } c @{ } c }
facet & {\bf 1} & {\bf 2} & {\bf 3} & {\bf 4} & {\bf 3} & {\bf 2}\\
\hline
{\bf A} & \1 & \1 & $\underline{\1}$ & \1 & 0               & 0 \\
{\bf B} & \1 & \1 & 0                & \1 & $\overline{\1}$ & 0 \\
\hline
facet & {\bf 1} & {\bf 2} & {\bf 3} & {\bf 4} & {\bf 3} & {\bf 2}
\end{tabular}
\hfill\strut
\caption[A pivot on $G(4,6)$ and on $C_4(6)$]{%
Pivoting on label 3 from $s_A=\1\1\1\1 00$ to $s_B=\1\10\1\10$ in the
Lemke-Howson for Gale Algorithm corresponds to the pivoting from
facet $A$ (edges in green) to facet $B$ (edges in blue) in the
Dual Lemke Path Algorithm.

The indices $i=1,2,4$ correspond to the 2-dimensional intersection of
$A$ and $B$ (edges in pink).
}
\label{lhg-123432-fig}
\end{figure}
\end{example}

The membership of \anothergale\ in the complexity class
{\bf PPA} follows from an argument similar to Proposition
\ref{lh-works-ppa-thm}. We give here the full proof that it is
in {\bf PPAD}, following the exposition given in Merschen \cite{jm}.

A {\em permutation} of elements of an ordered set $S$ is a sequence
without repetition; this gives a rearrangement of the elements of $S$.
A {\em transposition} is a permutation of exactly two elements.
The {\em sign of a permutation} is
$\sign(\sigma)=(-1)^m$, where $m$ is the number of transpositions needed
to get the {\em natural order} $\sigma_0=1\ldots n$ from $\sigma$.
It is immediate to see that any two permutations that differ in only
one transposition have opposite sign.

We define the {\em sign of a completely labeled Gale string} $s\in G(d,n)$
%% what is "relative labeling"?
% as follows: let $l:[n]\to [d]$ be the relative labeling of $G(d,n)$, and
as follows: let $l:[n]\to [d]$ be the labeling of $G(d,n)$, and
let $l_0$ be the string of labels $l(i)$ such that $s(i)=\1$ and that
two labels corresponding to a run in $l$ are adjacent in $l_0$. Then we
define $\sign(s)=\sign(l_0)$.
Notice that if $l(i)=i$ for $i\in [d]$ then the sign of the completely
labeled Gale string $\1^d 0^{(n-d)}$ is always positive.

The {\em sign of an almost completely labeled Gale string} $s\in G(d,n)$ with
missing label~$k$ and duplicate label $h$ is defined on two different strings.
Let $i_1$ be the index of $h$ reached by the last pivot
(the ``new'' position of the \1) and let $i_2$ be the index of $h$ such that
$s(i_2)=\1$ before the last pivot (the ``old'' position of the \1).
Let $l_1$ be the string obtained as $l_0$ substituting $k$ to
$h$ at index $i_1$, and let $l_2$ be the string obtained as $l_0$
substituting $k$ to $h$ at index $i_2$. Notice that $\sign(l_1)=-\sign(l_2)$,
since they can be obtained from each other applying the transposition $(kh)$.

Consider now the steps of the Lemke paths in the Lemke Path for Gale
Algorithm in the case where $\sign(s_0)=+1$;
the negative case is analogous, with opposite signs.
If the first pivot returns another completely labeled Gale
string $s$, this must have negative sign because it has been obtained
``jumping'' over an odd number of \1's.
For the same reason, if the pivoting returns an almost completely labeled
Gale string, we have that $\sign(l_1)=-1$, which implies $\sign(l_2)=+1$.
The next pivoting step drops the label $h$ from index $i_2$, so again we
change sign. This shows that
the Lemke Path for Gale Algorithm results in the sign
of the completely and almost completely labeled Gale strings
``swinging'' as in Table \ref{lhg-sign-figure}.
Notice that all the steps of this construction can be done in polynomial
time.
Orienting all Lemke paths from positive to negative reduces the problem
\anothergale\ to {\sc End Of The Line}.

\begin{figure}[h]
\strut\hfill
\includegraphics[width=70ex]{chapter-3/fig-lh/lhg-sign.pdf}%
\hfill\strut
\caption[Sign switching of the Lemke Path for Gale Algorithm]
{Sign switching of the Lemke Path for Gale Algorithm.}
\label{lhg-sign-figure}
\end{figure}

\begin{proposition}\label{lhg-works-ppad-thm}
The Lemke-Howson for Gale Algorithm~\ref{lhg-alg} returns a
solution to the {\bf PPAD} problem \anothergale.
% d even assumed in Algo
Furthermore, the number of completely labeled Gale strings
$s\in G(d,n)$ is even, and the two completely labeled Gale
strings at opposite ends of any Lemke path have opposite
sign.
\end{proposition}

\clearpage

\begin{example}
Let $l_s=123432$. Consider the Lemke path from the completely labeled
Gale string $s=\1\1\1\100$ with missing label $4$.
Figure \ref{lhg-sign-ex-figure} shows the graph of Table
\ref{lhg-sign-figure}.

Notice that
$\sign(\10\1\10\1)=\sign(l(6)l(1)l(3)l(4))=\sign((2134))$,
since $s(6)=s(1)=1$ and therefore the indices $6$ and $1$ are consecutive
in the same run.

\begin{figure}[h]
\strut\hfill
\includegraphics[width=70ex]{chapter-3/fig-lh/lhg-sign-example.pdf}%
\hfill\strut
\caption[Pivoting with sign]
{Pivoting with sign on the labeling $l=123432$ for $G(4,6)$.}
\label{lhg-sign-ex-figure}
\end{figure}
\end{example}

\clearpage

Morris \cite{morris} has given an example of a labeling
$l:[2d]\to [d]$ where the length of the Lemke paths on the
cyclic polytope $C_d(2d)$ for the Lemke-Howson
Algorithm~\ref{lh-alg}
grows exponentially in $d$ for every missing label.
These paths are therefore also exponential on $G(d,2d)$ for
the Lemke-Howson for Gale Algorithm~\ref{lhg-alg}.
Without repetitions of labels in consecutive positions,
Morris's labeling is equivalent to the following:
\[
\arraycolsep.2em
\begin{array}{rcll}
l(k)&=&k          & \quad\quad\text{for }k\in [d],    \\
l(d+k)&=&d-k+1    & \quad\quad\text{for }k\in [d - 1]\text{ and even},   \\
l(d+k)&=&d-k-1    & \quad\quad\text{for }k\in [d - 1]\text{ and odd}.
\end{array}
\]

\begin{example}
\label{morris-ex}
Consider the labeling $l=1234564523$ for $G(6,10)$. The only two completely
labeled Gale string  are $s=\1\1\1\1\1\100$ and $s'=\1 00000\1\1\1\1\1$.
Table \ref{morris-6-fig} shows the Lemke path for missing label~1.
\begin{table}[hbt]
\begin{center}
\large
\begin{tabular}{c @{ } c @{ } c @{ } c @{ } c @{ } c @{ } c @{ } c @{ } c @{ } c @{ } c }
{\bf 1} & {\bf 2} & {\bf 3} & {\bf 4} & {\bf 5} & {\bf 6} & {\bf 4} & {\bf 5} & {\bf 2} & {\bf 3} \\
\hline
\d1 & \1 & \1 & \1 & \1 & \1 & 0 & 0 & 0 & 0 \\
0 & \1 & \1 & \d1 & \1 & \1 & \u1 & 0 & 0 & 0 \\
0 & \1 & \1 & 0 & \d1 & \1 & \1 & \u1 & 0 & 0 \\
0 & \d1 & \1 & 0 & 0 & \1 & \1 & \1 & \u1 & 0 \\
0 & 0 & \1 & \u1 & 0 & \1 & \d1 & \1 & \1 & 0 \\
0 & 0 & \1 & \1 & \u1 & \1 & 0 & \d1 & \1 & 0 \\
0 & 0 & \d1 & \1 & \1 & \1 & 0 & 0 & \1 & \u1 \\
0 & 0 & 0 & \d1 & \1 & \1 & \u1 & 0 & \1 & \1 \\
0 & 0 & 0 & 0 & \d1 & \1 & \1 & \u1 & \1 & \1 \\
\u1 & 0 & 0 & 0 & 0 & \1 & \1 & \1 & \1 & \1 \\
\hline
{\bf 1} & {\bf 2} & {\bf 3} & {\bf 4} & {\bf 5} & {\bf 6} & {\bf 4} & {\bf 5} & {\bf 2} & {\bf 3} \\
\end{tabular}
\normalsize
\end{center}
\caption[A Lemke path for the Morris labeling]
{The Lemke path for the Morris labeling on $G(4,6)$ with missing label 1.
The bit position \1 that is dropped in the next pivoting step
is underlined, the bit \1 that has just been picked up is
overlined.}
\label{morris-6-fig}
\end{table}
\end{example}

Savani and von Stengel \cite{svs} \cite{uvg} extended Morris'
example in order to construct a series of ``hard to solve'' games.
Their results point to the importance of studying the
complexity of \anothergale\ to understand the complexity of 2-{\sc Nash}.
Our main result, in the next section, will give a
{\bf FP} algorithm that circumvents the problem of any exponential running
time of the Lemke-Howson for Gale Algorithm by using a very
different approach.


% Standard Article Definition
\documentclass[]{article}

% Page Formatting
\usepackage[margin=1in]{geometry}
\setlength\parindent{0pt}

% Graphics
\usepackage{graphicx}

% Math Packages
\usepackage{physics}
\usepackage{amsmath, amsfonts, amssymb, amsthm}
\usepackage{mathtools}

% Extra Packages
\usepackage{listings}
\usepackage{hyperref}

% Section Heading Settings
\usepackage{enumitem}
\renewcommand{\theenumi}{\alph{enumi}}
\renewcommand*{\thesection}{Problem \arabic{section}}
\renewcommand*{\thesubsection}{\alph{subsection})}
\renewcommand*{\thesubsubsection}{\quad \quad \roman{subsubsection})}

%Custom Commands
\newcommand{\Rel}{\mathcal{R}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\C}{\mathbb{C}}
\newcommand{\N}{\mathbb{N}}
\newcommand{\Z}{\mathbb{Z}}
\newcommand{\Q}{\mathbb{Q}}

\newcommand{\toI}{\xrightarrow{\textsf{\tiny I}}}
\newcommand{\toS}{\xrightarrow{\textsf{\tiny S}}}
\newcommand{\toB}{\xrightarrow{\textsf{\tiny B}}}

\newcommand{\divisible}{ \ \vdots \ }
\newcommand{\st}{\ : \ }

% Theorem Definition
\newtheorem{definition}{Definition}
\newtheorem{assumption}{Assumption}
\newtheorem{theorem}{Theorem}
\newtheorem{lemma}{Lemma}
\newtheorem{proposition}{Proposition}
\newtheorem{example}{Example}

%opening
\title{MATH 5301 Elementary Analysis - Homework 10}
\author{Jonas Wagner}
\date{2021, November 12\textsuperscript{th}}

\begin{document}

\maketitle

% Problem 1 ----------------------------------------------
\section{}
Prove that the closure and the interior of a convex set $A \subset \R^n$ are also convex.

% Convex Set
\begin{definition}
    The set $A$ is called \emph{\underline{convex}} if
    \[
        \forall_{x, y \in A} \forall_{t \in [0, 1]}
        \qty((t) x + (1 - t) y) \in A
    \]
\end{definition}

% Interior and Closure
\begin{definition}
    For a given set $A \subseteq (S,d)$,
    \begin{enumerate}
        \item the \emph{\underline{interior}} of $A$ is defined as
        \[
            \text{int}(A) = \qty{
                x \in A \st \exists_{\epsilon > 0} B_{\epsilon}(x) \subset A
            }
        \]
        \item the \emph{\underline{closure}} of $A$ is defined as
        \[
            \overline{A} = \qty{
                x \in S \st \forall_{\epsilon > 0} B_\epsilon(x) \cap A \neq \emptyset
            }
        \]
    \end{enumerate}
\end{definition}

% Problem 1 Theorem
\begin{theorem}
    If $A \subset \R^n$ is a convex set,
    then the closure of $A$, $\overline{A}$, is also convex.
    \begin{proof}
        $A$ being convex means that\[
            \forall_{x, y \in A} \forall_{t \in [0, 1]}
            \qty((t) x + (1 - t) y) \in A
        \]
        $\overline{A}$ is defined by\[
            \overline{A} = \qty{
                x \in \R^n \st \forall_{\epsilon > 0} B_\epsilon(x) \cap A \neq \emptyset
            }
        \]
        For $\overline{A}$ to be convex, the following would be true:\[
            \forall_{x, y \in \overline{A}} \forall_{t \in [0,1]}
                \qty((t) x + (1 - t) y) \in \overline{A}
        \]
        Additionally, since $\overline{A} = A \cup \partial A$, $\overline{A}$ is convex if
        \[
            \qty(
                \forall_{x \in A} \forall_{y \in \overline{A}} \forall_{t \in [0,1]}
                    \qty((t) x + (1 - t) y) \in \overline{A}
            ) \land \qty(
                \forall_{x \in \partial A} \forall_{y \in \overline{A}} \forall_{t \in [0,1]}
                    \qty((t) x + (1 - t) y) \in \overline{A}
            )
        \]
        Since $A \subset \overline{A}$, by definition the first statement is true, \[
            \forall_{x \in A} \forall_{y \in \overline{A}} \forall_{t \in [0,1]}
                \qty((t) x + (1 - t) y) \in \overline{A}
        \]
        Additionally, since the boundary of $A$, $\partial A$, is the collection of limit points of $A$ and the limit points all exist within the neighborhood of elements in $A$, \[
            \forall_{x \in \partial A} \forall_{y \in \overline{A}} \forall_{t \in [0,1]}
                    \qty((t) x + (1 - t) y) \in \overline{A}
        \]
        Therefore,
        \[
            \forall_{x, y \in \overline{A}} \forall_{t \in [0,1]}
                \qty((t) x + (1 - t) y) \in \overline{A}
        \]
    \end{proof}
\end{theorem}

% Problem 2 ----------------------------------------------
\newpage
\section{}
Prove that the intersection of an arbitrary collection of convex sets $\cap_{i \in I} C_i$ is also convex.

% Problem 2 Theorem
\begin{theorem}
    If each of the sets within the collection $C_i \subset (S,d)$ are convex,
    then the intersection of the collection, $\cap_{i \in I}$ is also convex.
    \begin{proof}
        For $\cap_{i \in I}$ to be convex, the following must be true: \[
            \forall_{x, y \in \cap_{i \in I} C_i}
                \forall_{t \in [0,1]} (t) x + (1 - t) y \in \cap_{i \in I} C_i
        \]
        Which is the same as: \[
            \forall_{x,y \in S} \st \forall_{i \in I} x,y \in C_i
                \implies \forall_{t \in [0,1]} \forall_{i \in I} (t) x + (1 - t) y \in C_i
        \]
        Since all the sets $C_i$ are convex, by definition: \[
            \forall_{x, y \in C_i} \forall_{t \in [0,1]}
                (t) x + (1 - t) y \in C_i
        \]
        Therefore this is true $\forall_{i \in I}$: \[
            \land_{i \in I} \forall_{x,y \in C_i}
                \implies \forall_{t \in [0,1]} (t) x + (1 - t) y \in C_i
        \]
        Which is equivalent to: \[
            \forall_{x, y \in \cap_{i \in I} C_i}
                \forall_{t \in [0,1]} (t) x + (1 - t) y \in \cap_{i \in I} C_i
        \]

    \end{proof}
\end{theorem}

% Problem 3 ----------------------------------------------
\newpage
\section{}
Let $\{C_i\}_{i \in \N}$ be a sequence of nested convex sets in $\R^n$, i.e. $C_i \subset C_{i+1}$.
Prove that $\cup_{i = 1}^\infty C_i$ is also convex.

% Problem 3 Theorem
\begin{theorem}
    For the sequence of nested convex sets in $\R^n$, $\qty{C_i}_{i \in \N}$, a union of all the elements, $\cup_{i = 1}^\infty C_i$, is also convex.
    \begin{proof}
        Proof by induction.

        For $n = 1$, the set $\cup_{i = 1}^n C_i = C_1$ is convex.

        For $n = 2$, the set $\cup_{i = 1}^n C_i = C_1 \cup C_2$ is convex.
        \begin{proof}
            Since $C_1 \subset C_2$, $C_1 \cup C_2 = C_2$ and $C_2$ is convex.
        \end{proof}

        Assuming for $n = k$, $\cup_{i = 1}^k C_i = C_k$ is convex,
        then for $n = k+1$, $\cup_{i = 1}^{k + 1} C_i = C_{k + 1}$ is convex.
        \begin{proof}
            Since $C_k \subset C_{k+1}$, \[
                \cup_{i = 1}^{k + 1} C_i = \cup_{i = 1}^{k} C_i \cup C_{k + 1} = C_{k + 1}
            \]
            which is convex.
        \end{proof}

        Therefore, by induction, \[
            \forall_{n \in \N} \cup_{i = i}^n C_i
        \]
        is convex.
        This implies $\cup_{i = 1}^\infty C_i$.
    \end{proof}
\end{theorem}

% Problem 4 -----------------------------------------------
\newpage
\section{}

% Convex Hull
\begin{definition}
    The \emph{\underline{convex hull}} for set $A \in (S,d)$ is defined as \[
        \textnormal{conv}(A) = \cap_{C \supseteq A \st C \textnormal{ convex}}
    \]
    Additionally, for $A \subset \R^n$, \[
        \textnormal{cov}(A) = \cup_{m = 1}^{\infty} C_m
    \]\[
        C_m = \qty{
            x \in \R^n \st
            x = \alpha_1 a_1 + \cdot + \alpha_m a_m, \
            a_1, \dots, a_m \in A, \
            \alpha_i \geq 0, \
            \sum_{i} \alpha_i = 1
        }
    \]
\end{definition}
% Open Set
\begin{definition}
    The set $A \subset V$ is called \underline{open} if \[
        \forall_{x\in A} \exists_{\epsilon>0} : B_\epsilon(x)\subset A
    \]
    or equivalently, \[
        \forall_{x \in A} \exists_{\epsilon>0} : \forall_{y \in V}\norm{x - y} < \epsilon \implies y \in A
    \]
\end{definition}
% Closed Set
\begin{definition}
    The set $A \subset V$ is called \underline{closed} if $A^c$ is open.
\end{definition}

% Part a
\subsection{}
Show that the convex hull of any open sets in $\mathcal{R}^n$ is open.

% Problem 4a Theorem
\begin{theorem}
    For any open set $A \subset \R^n$,
    then the convex hull, $\textnormal{conv}(A)$,
    is open.
    \begin{proof}
        By definition, $A$ will satisfy the open condition:\[
            \forall_{x \in A} \exists_{\epsilon>0} : \forall_{y \in \R^n}\norm{x - y} < \epsilon \implies y \in A
        \]
        The convex hull of $A$, $\textnormal{conv}(A)$, is the intersection of all convex sets that contain $A$.
        This means that the smallest convex set containing $A$ will share a portion of the boundary of $A$.
        Then the open condition will be true along the portion of the boundary in which $A$ and $\textnormal{conv}(A)$ share will be open.
        The smallest convex set that shares this boundary will then also be open since the open $\textnormal{int}(C)$ is contained by any closed set with the same boundary.
        Therefore, the convex hull would be open.
    \end{proof}
\end{theorem}

% Part b
\subsection{}
Provide an example of a closed set $A \subset \mathcal{R}^n$, such that its convex hull is not closed.

\begin{example}
    Let closed set $A \subset \R^2$ be defined as\[
        A := \qty{
            (x,y) \in \R^2 \st y \geq e^{-x^2}
        }
    \]
    Clearly, $A$ is closed, however the convex hull of $A$ is actually open due to the asymptote that occurs on the $x$-axis:\[
        \textnormal{conv}(A) := \qty{
            (x,y) \in \R^2 \st y > 0
        }
    \]
\end{example}

% Problem 5 -----------------------------------------------
\newpage
\section{}
Let $f : \R^n \to \R$ be a convex function and $A \subset \R^n$ be a bounded set.
Prove that $f(A)$ is bounded in $\R$.

% Convex Function
\begin{definition}
    Function $f : [a,b] \to \R$ is considered a \emph{\underline{convex function}} if\[
        \forall_{x_1,x_2 \in [a,b]} \forall_{t \in [0,1]}
        f((t) x_1 + (1 - t) x_2) \leq (t) f(x_1) + (1 - t) f(x_2)
    \]
\end{definition}

% Problem 5 Theorem
\begin{theorem}
    If the function $f : \R^n \to \R$ is a convex function,
    then for a bounded set $A \subset \R^n$, the image $f(A)$ is bounded in $\R$.
    \begin{proof}
        $f$ being convex means that\[
            \forall_{\va{x},\va{y} \in \R^n} \forall_{t \in [0,1]}
                f((t) \va{x} + (1 - t) \va{y}) \leq (t) f(\va{x}) + (1 - t) f(\va{y})
        \]
        $A$ being bounded means\[
            \exists_{N} \st \forall_{\va{x} \in A}  \norm{\va{x}} \leq N
        \]
        It is known that the a convex function with a non-constant output cannot obtain its maximum within $\textnormal{int}(A)$;
        therefore, $\arg \max_{\va{x} \in A} f(x) \in \partial A$.
        Since $A$ is bounded, \[
            \exists_N \st \forall_{x \in \partial A} \norm{x} < N
        \]
        Therefore, \[
            \exists_N \st \forall_{\va{x} \in A} f(x) < \max_{\va{x} \in \partial A} f(x) < N
        \]
        Which means $f(A)$ is bounded.
    \end{proof}
\end{theorem}

% Problem 6 -----------------------------------------------
\newpage
\section{}
Show that the convex hull of a compact set $A \subset \R^n$ is compact.
(\textit{Hint:} Caratheodory theorem)

\end{document}


%% SECTION HEADER /////////////////////////////////////////////////////////////////////////////////////
\section{Parallel Implementation of the Internal Force Vector Calculation}
\label{sec:gpu}

%% SECTION CONTENT ////////////////////////////////////////////////////////////////////////////////////

The most time-consuming operation in the equation (\ref{eq:motion}) is calculating the internal force vector $\textbf{F}_{int}=\textbf{Kd}$, as the stiffness matrix \textbf{K} occupies a large amount of memory, and it is defined as:
\begin{eqnarray}
	\label{eq:K}
	\textbf{K}^P=\int_{V^P}\textbf{B}^T\textbf{C}\textbf{B}dV^P
\end{eqnarray}
where \textbf{c} is the elastic constant matrix, and \textbf{B} is the strain-displacement matrix.
Instead of allocating the matrix \textbf{K} defined by equation~\ref{eq:K}, Kudela proposed a parallelized computation of the internal force vector \cite{kudela2016parallel}.
In the pre-processing, the natural derivatives matrix, the vector of inverted components of the Jacobian matrix, and the integration weights multiplied by the Jacobian determinant is rearranged from global to the local form:
\begin{eqnarray}
	\label{eq:isoparametric}
	\textbf{N}^P_{,\xi} & = & \left[ \begin{array}{cccc}
		\textbf{N}^{e=1}_{,\xi} & \textbf{0} & \ldots & \textbf{0}\\
		\textbf{0} & \textbf{N}^{e=2}_{,\xi} & \ldots & \textbf{0}\\
		\vdots & \vdots &  \ddots & \vdots\\
		\textbf{0} & \textbf{0} & \ldots & \textbf{N}^{e=n}_{,\xi}
	\end{array}\right],\\
	\label{eq:jacob}
	\left(\textbf{J}^P\right)^{ij}_{inv} & = & \left\{ \begin{array}{c}
		\left(\textbf{J}^{e=1}\right)^{ij}_{inv}\\
		\left(\textbf{J}^{e=2}\right)^{ij}_{inv}\\
		\vdots\\
		\left(\textbf{J}^{e=n}\right)^{ij}_{inv} \end{array}\right\}\\
	\label{eq:intWeights}
	\textbf{w}^P & = & \left\{ \begin{array}{c}
		\textbf{w}^{e=1}\\
		\textbf{w}^{e=2}\\
		\vdots\\
		\textbf{w}^{e=n} \end{array}\right\} \circ
	\left\{ \begin{array}{c}
		det(\textbf{J})^{e=1}\\
		det(\textbf{J})^{e=2}\\
		\vdots\\
		det(\textbf{J})^{e=n} \end{array}\right\}
\end{eqnarray}
where $n$ is the spectral elements number in modeled domain; \textbf{J} is the Jacobian matrix; $i,j=1\ldots3$; and $\circ$ denotes element-wise multiplication.
The $\textbf{N}^P_{,\xi}$ is a block-diagonal sparse matrix, and the equality of $\textbf{N}^1_{,\xi}=\textbf{N}^2_{,\xi}=\ldots=\textbf{N}^n_{,\xi}$ holds if the same order of interpolation shape function is used for the all elements.
Besides, a vector of local node indices $\textbf{I}_L$ and corresponding global node indices $\textbf{I}_G$ must be defined in the preprocessing process.

Adjacent elements in the mesh share nodes, so one node in the global system can correspond to several nodes in the local system. Since independent operations on vectors are necessary for parallel computation on \ac{gpu}, $I_{G}$ must be rearranged to separate all duplicated nodes. Therefore, the matrix $I_{G}$ is created in which no column has repeated indices of the nodes. Then, the corresponding local map $I_{L}$ must also be created. For the rearrangement algorithm presented in \cite{kudela2016parallel} was used.

The following computational operations are performed during the time integration algorithm. Firstly, the global vector of nodal displacements is transferred to the element nodes displacements such as:

\begin{eqnarray}
	\widehat{\textbf{d}}_i^P = \left\{ \begin{array}{c}
		\widehat{\textbf{d}}_i^{e=1}\\
		\widehat{\textbf{d}}_i^{e=2}\\
		\vdots\\
		\widehat{\textbf{d}}_i^{e=n} \end{array}\right\}
\end{eqnarray}
where $\widehat{\textbf{d}}_i^e$ is the nodal displacements in element for $i^{th}$ time step.
Next, the strain and stress vectors are calculated as:
\begin{eqnarray}
	\label{eq:strain}
	\boldsymbol{\epsilon}=\left[\boldsymbol{\epsilon}_{xx},\ \boldsymbol{\epsilon}_{yy},\ \boldsymbol{\epsilon}_{zz},\ \boldsymbol{\gamma}_{yz},\ \boldsymbol{\gamma}_{xz},\ \boldsymbol{\gamma}_{xy}\ \right]^T&=&\textbf{B}^e\widehat{d}^e\\
	\label{eq:stress}
	\boldsymbol{\sigma}=\left[\boldsymbol{\sigma}_{xx},\ \boldsymbol{\sigma}_{yy},\ \boldsymbol{\sigma}_{zz},\ \boldsymbol{\tau}_{yz},\ \boldsymbol{\tau}_{xz},\ \boldsymbol{\tau}_{xy},\ \right]^T&=&\textbf{c}\boldsymbol{\epsilon}
\end{eqnarray}
The formulation of equation~\ref{eq:strain} and equation~\ref{eq:stress} for 3D and first-order shear deformation model can be found in \cite{kudela2016parallel} and \cite{kudela2020parallel}, respectively.
Then, the internal forces vector is calculated as:
\begin{eqnarray}
	\label{eq:forces}
	\textbf{F}^P_{int}=\left[\textbf{F}^P_1,\ \textbf{F}^P_2,\ \ldots\ \textbf{F}^P_{n} \right]^T={\textbf{B}^e}^T\sigma
\end{eqnarray}
where $n$ is the nodal degree of freedom.
It should be mentioned that $\epsilon$, $\sigma$ and $\textbf{F}^P_{int}$ components are calculated separately, with the appropriate order of performing the element-wise multiplication of the particular vectors.
This approach is essential in order to keep the calculations matrix-free.

Finally, the assembly of internal forces vector is performed using the $\textbf{I}_G$ and $\textbf{I}_L$ as follows:

\begin{eqnarray}
	\label{eq:Fint}
	{\left(\textbf{F}_{int}\right)}^i_{\textbf{I}^m_G} = {\left(\textbf{F}_{int}\right)}^i_{\textbf{I}^m_G} + {\left(\textbf{F}^P_{int}\right)}^i_{\textbf{I}^m_L},\quad for\ m=1\ldots col
\end{eqnarray}
where $col$ is the column number of $\textbf{I}_G$.

In the presented paper, some improvements have been implemented to the above algorithm to make it more computationally efficient.
Instead of calculating the internal forces vector in the for loop like in equation~(\ref{eq:Fint}), it is recommended to assign all local forces into the matrix as:
\begin{eqnarray}
	\label{eq:Fmatrix}
	{\left(\textbf{F}_{int}\right)}^i_{\textbf{I}_G} ={\left(\textbf{F}^P_{int}\right)}^i_{\textbf{I}_L}
\end{eqnarray}
and then return the column vector containing the sum of each row of matrix ${\left(\textbf{F}^P_{int}\right)}^i_{\textbf{I}_L}$.
For example in Matlab, it can be done by built-in function $sum$ as:
\begin{eqnarray}
	\label{eq:Fsum}
	{\left(\textbf{F}_{int}\right)}^i = sum \left({\left(\textbf{F}^P_{int}\right)}^i_{\textbf{I}_L},2\right);
\end{eqnarray}
Fixed number of columns in equation~(\ref{eq:Fint}) was proposed in \cite{kudela2016parallel}. In the current approach the number of columns is chosen adaptively according to the given mesh. It should be chosen as the smallest divisor of the number of nodes in an element but not less than the maximum number of common elements for a node. In this way, less serial operations are performed and \ac{gpu} resources are better utilized.

Further code modifications included storage scheme. Instead of storing in memory both isoparametric derivatives equation~(\ref{eq:isoparametric}) and inverted components of Jacobian matrix shown in equation~(\ref{eq:jacob}), it is recommended to calculate derivatives in global coordinates system as:
\begin{eqnarray}
	\textbf{N}^P_{,X} = \textbf{J}^{-1}\,\textbf{N}^P_{\xi}
\end{eqnarray}
Also, a multiplication of elastic constants $\textbf{c}$ with integration weights defined in equation (\ref{eq:intWeights}) can be performed in preprocessing stage before main loop through integration time steps.


\section{Declarative System}

\begin{figure}[t]
    \begin{gather*}
    \begin{aligned}
        \text{Type variables}\qquad&a, b\\
        \text{Types}\qquad&A, B, C &::=&\quad 1 \mid \top \mid \bot \mid a \mid \all A \mid A\to B\\
        \text{Monotypes}\qquad&\tau &::=&\quad 1 \mid \top \mid \bot \mid a \mid \tau_1\to \tau_2\\
        \text{Expressions}\qquad&e &::=&\quad x \mid () \mid \lam e \mid e_1~e_2 \mid (e:A)\\
        \text{Context}\qquad&\Psi &::=&\quad \cdot \mid \Psi, a \mid \Psi, x:A
    \end{aligned}
    \end{gather*}
\Description{Declarative Syntax}
\caption{Declarative Syntax}\label{fig:top_decl_syntax}
\end{figure}

\paragraph{Syntax}
The syntax of the declarative system, shown in Figure~\ref{fig:top_decl_syntax},
is similar to the previous systems by having
a primitive type $1$, type variables $a$,
polymorphic types $\all A$ and function types $A \to B$.
Additionally, top and bottom types are introduced to the type system.
% intro of top/bot moved to intro

The well-formedness formalization of the system is standard
and almost identical to the previous systems,
therefore we omit the formal definitions.

\begin{figure}[t]
    \framebox{$\Psi \vdash A \le B$}
    \begin{gather*}
    \inferrule*[right=$\mathtt{{\le}Var}$]
    {a\in\Psi}{\Psi\vdash a\le a}
    \qquad
    \inferrule*[right=$\mathtt{{\le}Unit}$]
    {~}{\Psi \vdash 1 \le 1}
    \qquad
    \inferrule*[right=$\mathtt{{\le}{\to}}$]
    {\Psi \vdash B_1 \le A_1 \quad \Psi \vdash A_2 \le B_2}
    {\Psi\vdash A_1\to A_2 \le B_1\to B_2}
    \\
    \inferrule*[right=$\mathtt{{\le}\forall L}$]
    {\Psi\vdash \tau \quad \Psi\vdash [\tau/a] A \le B}
    {\Psi\vdash \all A \le B}
    \qquad
    \inferrule*[right=$\mathtt{{\le}\forall R}$]
    {\Psi, b\vdash A\le B}
    {\Psi\vdash A \le \all[b]B}
    \\
    \pmb{
    \inferrule*[right=$\mathtt{{\le}Top}$]
    {~}
    {A \le \top}
    }
    \qquad
    \pmb{
    \inferrule*[right=$\mathtt{{\le}Bot}$]
    {~}
    {\bot \le A}
    }
    \end{gather*}
\Description{Declarative Subtyping}
\caption{Declarative Subtyping}\label{fig:top_decl_subtyping}
\end{figure}

\paragraph{Declarative Subtyping}
Shown in Figure~\ref{fig:top_decl_subtyping},
the declarative subtyping extends the polymorphic subtyping relation
originally proposed by \citet{odersky1996putting}
by adding Rules $\mathtt{{\le}Top}$ and $\mathtt{{\le}Bot}$,
defining the properties of the $\top$ and $\bot$ types, respectively.
Although the new rules seem quite simple,
they may increase the uncertainty of polymorphic instantiations.
For example, the subtyping judgment
\[\all a \to a \le \bot \to \top\]
accepts any well-formed instantiation on the polymorphic type $\all a \to a$.

\begin{figure}[t]
    \begin{tabular}{rl}
        \framebox{$\Psi \vdash e \Lto A$} & $e$ checks against input type $A$.\\[0.5mm]
        \framebox{$\Psi \vdash e \To A$} & $e$ synthesizes output type $A$.\\[0.5mm]
        \framebox{$\Psi \vdash \appInf{A}{e}{C}$} & Applying a function of type $A$ to $e$ synthesizes type $C$.
    \end{tabular}
    \begin{gather*}
    \inferrule*[right=$\mathtt{DeclVar}$]
        {(x:A)\in\Psi}{\Psi\vdash x\To A}
    \qquad
    \inferrule*[right=$\mathtt{DeclSub}$]
    %e \neq \lam e' \quad B \neq \all B' \quad
        {\Psi\vdash e\To A \quad \Psi\vdash A\le B}
        {\Psi \vdash e\Lto B}
    \\
    \inferrule*[right=$\mathtt{DeclAnno}$]
        {\Psi \vdash A \quad \Psi\vdash e\Lto A}
        {\Psi\vdash (e:A)\To A}
    \qquad
    \inferrule*[right=$\mathtt{Decl1I{\To}}$]
        {~}{\Psi\vdash () \To 1}
    \\
    \inferrule*[right=$\mathtt{Decl1I}$]
        {~}{\Psi\vdash () \Lto 1}
    \qquad
    \inferrule*[right=$\mathtt{Decl\top}$]
        {\Psi \vdash e}
        {\Psi\vdash e \Lto \top}
    \qquad
    \inferrule*[right=$\mathtt{Decl{\bot}App}$]
        {\Psi\vdash e}
        {\Psi\vdash \appInf{\bot}{e}{\bot}}
    \\
    \inferrule*[right=$\mathtt{Decl\forall I}$]
        {\Psi,a \vdash e \Lto A}
        {\Psi\vdash e\Lto \all A}
    \qquad
    \inferrule*[right=$\mathtt{Decl\forall App}$]
        {\Psi \vdash \tau \quad \Psi\vdash \appInf{[\tau/a]A}{e}{C} }
        {\Psi\vdash \appInf{\all A}{e}{C}}
    \\
    \inferrule*[right=$\mathtt{Decl{\to}I}$]
        {\Psi,x:A \vdash e\Lto B}
        {\Psi\vdash \lam e \Lto A \to B}
    \qquad
    \inferrule*[right=$\mathtt{Decl{\to}I{\To}}$]
        {\Psi\vdash \sigma\to\tau \quad \Psi,x:\sigma \vdash e\Lto \tau}
        {\Psi\vdash \lam e \To \sigma\to\tau}
    \\
    \inferrule*[right=$\mathtt{Decl{\to} E}$]
        {\Psi\vdash e_1\To A \quad \Psi\vdash \appInf{A}{e_2}{C}}
        {\Psi\vdash e_1~e_2 \To C}
    \qquad
    \inferrule*[right=$\mathtt{Decl{\to}App}$]
        {\Psi\vdash e \Lto A}
        {\Psi\vdash \appInf{A \to C}{e}{C}}
    \end{gather*}
\Description{Declarative Typing}
\caption{Declarative Typing}\label{fig:top_decl_typing}
\end{figure}

\paragraph{Declarative Typing}

The declarative typing rules, shown in Figure~\ref{fig:top_decl_typing},
extend DK's higher-ranked type system in order to support the top and bottom types.
Rule $\mathtt{Decl\top}$ allows any well-formed expression to check against $\top$.
Rule $\mathtt{Decl{\bot}App}$ returns the $\bot$ type
when a function of $\bot$ type is applied to any argument.
All other rules remain exactly the same as the system of
Chapters~\ref{chap:ITP} and \ref{chap:ICFP}.

It's worth mentioning that the design of the two new rules
is driven by the subsumption property described in Section~\ref{sec:meta:decl}.
They maintain the property in presence of a more powerful declarative subtyping,
and we will discuss further later in that part.

\setcounter{algRuleCounter}{0}


\section{Macroscale Constitutive Model}

In this section we describe the macroscale stress-strain relationship, $\dot{\boldsymbol{\sigma}}^M=\dot{\boldsymbol{\sigma}}^M\left(\dot{\boldsymbol{\epsilon}}^M, \boldsymbol{\chi},\mathbf{h}\right)$, used in the validation examples. The model is chosen to be complex enough to make the validation of the framework meaningful; however, we do not claim that this is the "best" macroscale model. The framework presented is general and the macroscale constitutive model described here can be replaced in particular applications by a different one. To simplify the discussion and notation in this section, the superscript "$M$" is omitted since all quantities defined describe macroscale behaviour.

A Continuum Damage Mechanics (CDM) constitutive model was chosen to represent the NFR at the macroscale. CDM is a branch of continuum mechanics that is concerned with modeling the progressive failure and stiffness degradation in solid materials. CDM in this investigation is used to help describe the micro-mechanical degradation of the rock mass due to the nucleation and growth of cracks and voids. This micro-mechanical degradation is represented in a CDM model by using macroscopic state variables to represent a spatial average of the effects of this degradation. These state variables used in this context with respect to CDM are known as damage variables.

The damage variables in a CDM model can be described in different capacities. Often, for mathematical and physical simplicity, a single scalar damage variable is used to characterize the state of damage in the material. In this case, the damage variable, $D$, takes a value between 0 and 1 to represent the degree of damage to the material, where $D=0$ represents a completely undamaged material (original stiffness) and $D=1$ represents a completely damaged material with no stiffness. A scalar damage description limits the applicability of the CDM model to an isotropically damaged state, which may not be appropriate in some circumstances. More sophisticated CDM models use 2\textsuperscript{nd} and 4\textsuperscript{th} order tensorial representations of the damage variables as well as distinguishing between compressive damage and tensile damage states in order to more accurately characterize anisotropic damage evolution. In this paper, a ductile isotropic damage formulation is prescribed using a modified Johnson-Cook damage initiation criterion and a linear stiffness degradation model.

In addition to damage, the elasto-plastic behaviour of the rock is also considered using an extended Drucker-Prager model with a linear yield criterion and a Barcelona hardening function. Models that incorporate theories of plasticity and damage mechanics in a unified approach to damage evolution and constitutive relationships are often referred to as damage-plasticity models \citep{zhang_continuum_2010}. In general, the constitutive relation for these damage-plasticity models describes the relationship between the stress, $\boldsymbol{\sigma}$, and the strain, $\boldsymbol{\epsilon}$ as a function of the damage variable, the original elastic stiffness tensor, $\mathbf{E}$, and the plastic strain, $\boldsymbol{\epsilon}^{pl}$:

\begin{equation}
\boldsymbol{\sigma}=\left(1-D\right)\mathbf{E}:\left(\boldsymbol{\epsilon}-\boldsymbol{\epsilon}^{pl}\right)
\label{eqn:const5}
\end{equation}

where an additive decomposition of the elastic and plastic strain, $\boldsymbol{\epsilon}=\boldsymbol{\epsilon}^{el}+\boldsymbol{\epsilon}^{pl}$, is assumed.

In CDM, the notion of effective stress, $\boldsymbol{\sigma}$, becomes useful to describe the mechanics of the system as it refers to the stress that the system would be experiencing without damage. This effective stress can be related to the actual Cauchy stress through the scalar damage variable: $\boldsymbol{\sigma}=\left(1-D\right)\bar{\boldsymbol{\sigma}}$.


%!TEX root = inversion.tex
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Copyright (c) 2003-2018 by The University of Queensland
% http://www.uq.edu.au
%
% Primary Business: Queensland, Australia
% Licensed under the Apache License, version 2.0
% http://www.apache.org/licenses/LICENSE-2.0
%
% Development until 2012 by Earth Systems Science Computational Center (ESSCC)
% Development 2012-2013 by School of Earth Sciences
% Development from 2014 by Centre for Geoscience Computing (GeoComp)
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{MT inversion: 2D TE Mode}\label{sec:forward 2DMT TEMode}
In this section we present the forward model for the magnetotelluric (MT)\index{magnetotelluric}\index{MT}inversion case~\cite{chave2012magnetotelluric}
where we particularly looking into the two-dimensional case of the electrical field polarization\index{electrical field polarization}.
For this case the horizontal electrical field $E_x$\index{electrical field} and the magnetic field $H_y$\index{magnetic field} component
for a given frequency $\omega$ are measured and the horizontal impedance $Z_{xy}$\index{impedance}  is recorded as the ratio of
electrical and magentical field:
\begin{equation}\label{ref:2DMTTE:EQU:1}
E_x  = Z_{xy} \cdot H_y
\end{equation}
It is common practice to record the impedance using the apparent resistivity $\rho_a$\index{resistivity!apparent} and the
phase $\phi$ as
\begin{equation}\label{ref:2DMTTE:EQU:2}
Z_{xy} = (\rho_a \cdot \omega \cdot \mu) ^{\frac{1}{2}} \cdot e^{\mathbf{i} \phi}
\end{equation}
where $\mu$ is the permeability (e.g. $mu = 4 \pi \cdot 10^{-7} \frac{Vs}{Am}$). Notice that the impedance is independent of the scale of the $E_x$ and $E_y$.
The electrical field $E_x$ as function of depth $z$ and horizontal coordinate $y$ is given as the solution of the
PDE
\begin{equation}\label{ref:2DMTTE:EQU:3}
- E_{x,kk}  + \mathbf{i} \omega \mu \sigma \cdot E_x = 0
\end{equation}
where $\mathbf{i}$ is the complex unit, and $k=0$ and $k=1$ correspond to the $y$ and $z$ direction.
$\sigma$ is the unknown conductivity to be calculated through the inversion. The domain
of the PDE is comprised of the subsurface region in which the conductivity is to be calculated
and a sufficient high airlayer in which the conductivity is assumed to be zero.

It is assumed that $E_x$ takes the value $1$ at the top of the domain $\Gamma_0$ representing an incoming
electro-magnetic wave front. On the other faces of the domain homogeneous Neuman conditions
are set to model the assumption that the electrical field is constant away from the domain.
\begin{equation}\label{ref:2DMTTE:EQU:4}
H_y = - \frac{1}{\mathbf{i} \omega \mu} E_{x,z}
\end{equation}

The impedance $Z_{xy}^{(s)}$ is measured at certain points ${\mathbf x}^{(s)}$. The defect
of the measurements for the prediction $E_x$ is given as
\begin{equation}\label{ref:2DMTTE:EQU:5}
J^{MT}(\sigma) = \frac{1}{2}\sum_{s} \int_{\Omega}
w^{(s)} \cdot \| E_x - Z_{xy}^{(s)} \cdot H_y \|^2 d{\mathbf x}
\end{equation}
where $w^{(s)}$ is a spatially dependent weighting faction. Here we use the Gaussian profile
\begin{equation}\label{ref:2DMTTE:EQU:6}
w^{(s)}({\mathbf x}) = \frac{w^{(s)}_0}{2\pi (\eta^{(s)})^2} \cdot e^{-\frac{\|{\mathbf x} - {\mathbf x}^{(s)} \|^2}{2(\eta^{(s)})^2}}
\end{equation}
where $\eta^{(s)}$ is measuring the spatial validity
and $w^{(s)}_0$ is confidence of the measurement $s$ (e.g. $w^{(s)}_0$ is set to the inverse of the measurement error. )

\subsection{Usage}

\begin{classdesc}{MT2DModelTEMode}{domain, omega, x, Z_XY, eta,
        \optional{, w0=1}
        \optional{, mu=4E-7 * PI }
        \optional{, coordinates=\None}
        \optional{, fixAtBottom=False}
        \optional{, tol=1e-8}
}
\end{classdesc}

\subsection{Gradient Calculation}

For the implemtation we set
\begin{equation}\label{ref:2DMTTE:EQU:100}
E_x  = u_0 + \mathbf{i} \cdot u_1
\end{equation}
which translates teh forward model~\ref{ref:2DMTTE:EQU:3}
\begin{align}\label{ref:2DMTTE:EQU:103}
- u_{0,kk}  - \omega \mu \sigma u_1 = 0 \\
- u_{1,kk}  + \omega \mu \sigma u_0 = 0
\end{align}
In weak form this takes the form
\begin{equation}\label{ref:2DMTTE:EQU:104}
\int_{\Omega}
\left(
v_{0,k}u_{0,k}
+ v_{1,k}u_{1,k}
+ \omega \mu \sigma \cdot ( v_1 u_0 - v_0 u_1) \right) dx =0
\end{equation}
for all test function $v_i$ with $v_i$ zero on $\Gamma_0$.
Using the complex data
\begin{equation}\label{ref:2DMTTE:EQU:105}
d^{(s)} = - \frac{1}{\mathbf{i} \omega \mu} \cdot Z_{xy}^{(s)} = d^{(s)}_0 +  \mathbf{i} \cdot d_1^{(s)}
\end{equation}
the cost function~\ref{ref:2DMTTE:EQU:6} takes the form
\begin{equation}\label{ref:2DMTTE:EQU:106}
J^{MT}(\sigma)=
\frac{1}{2}\sum_{s} \int_{\Omega} w^{(s)} \cdot \left(  (u_0- u_{0,1} \cdot d_0^{(s)} + u_{1,1} \cdot d_1^{(s)}  )^2
 + ( u_1- u_{0,1} \cdot d_1^{(s)} -u_{1,1} \cdot d_0^{(s)} )^2 \right) dx
\end{equation}

We need to calculate the gardient
$\frac{\partial J^{MT}}{\partial \sigma}$ of the cost function $J^{{MT}}$ with
respect to the susceptibility $\sigma$.  We follow the concept as outlined in section~\ref{chapter:ref:inversion cost function:gradient}.

If $\Gamma_{\sigma}$ denotes the region of the domain where the susceptibility is
known and for a given direction $\hat{\sigma}$ with $\hat{\sigma}=0$ on $\Gamma_{\sigma}$ one has
\begin{align}\label{ref:2DMTTE:EQU:201aa}
\int_{\Omega}   \frac{\partial J^{MT}}{\partial \sigma} \cdot \hat{\sigma} \; dx  & = &
\sum_{s} \int_{\Omega}  w^{(s)} \cdot \left(  u_0 - u_{0,1} \cdot d_0^{(s)} + u_{1,1} \cdot d_1^{(s)}  \right ) \left(  \hat{u}_0 - \hat{u}_{0,1} \cdot d_0^{(s)} + \hat{u}_{1,1}\cdot d_1^{(s)}  \right )  \\
& + &  w^{(s)} \cdot \left(u_1- u_{0,1} \cdot d_1^{(s)} -u_{1,1} \cdot d_0^{(s)}  \right)   \left(\hat{u}_1- \hat{u}_{0,1} \cdot d_1^{(s)} -\hat{u}_{1,1} \cdot d_0^{(s)} \right) \; dx
\end{align}
with
\begin{equation}\label{ref:2DMTTE:EQU:201A}
\int_{\Omega}
\left(
q_{0,k}\hat{u}_{0,k}
+ q_{1,k}\hat{u}_{1,k}
+ \omega \mu \sigma \cdot ( q_1 \hat{u}_0 - q_0 \hat{u}_1)
+ \omega \mu \hat{\sigma} \cdot ( q_1 u_0 - q_0 u_1) \right) dx =0
\end{equation}
for all $q_i$ with $q_i=0$ on $\Gamma_{0}$. This equation is obtained from equation~(\ref{ref:2DMTTE:EQU:104}).
With
\begin{align}\label{ref:2DMTTE:EQU:202b}
Y_0  = & \sum_{s} w^{(s)} \cdot \left(  u_0 - u_{0,1} \cdot d_0^{(s)} + u_{1,1} \cdot d_1^{(s)}  \right) \\
Y_1  = & \sum_{s} w^{(s)} \cdot \left(  u_1 - u_{0,1} \cdot d_1^{(s)} - u_{1,1} \cdot d_0^{(s)}  \right)   \\
X_{01}  = &  \sum_{s} w^{(s)} \cdot \left(    u_{0,1} \cdot ( ( d_0^{(s)})^2 + (d_1^{(s)})^2) -  u_0 \cdot d_0^{(s)} -  u_1 \cdot d_1^{(s)} \right)    \\
X_{11}  = & \sum_{s} w^{(s)} \cdot \left(  u_{1,1} \cdot ( ( d_0^{(s)})^2 + (d_1^{(s)})^2)  + u_0 \cdot d_1^{(s)} - u_1 \cdot d_0^{(s)} \right)
\end{align}
and $X_{00}=X_{10}=0$ we can write Equation~(\ref{ref:2DMTTE:EQU:201aa}) as
\begin{equation}\label{ref:2DMTTE:EQU:202c}
\int_{\Omega}   \frac{\partial J^{MT}}{\partial \sigma} \cdot \hat{\sigma} \; dx  =
\int_{\Omega}  Y_i \hat{u}_i + X_{ij}  \hat{u}_{i,j} \; dx
\end{equation}
We then set adjoint function $u_i^*$ with $u_i^*=0$ on $\Gamma_{0}$ as the solution of the variational problem
\begin{equation}\label{ref:2DMTTE:EQU:202d}
\int_{\Omega}
\left(
u^*_{0,k} v_{0,k}
+ u^*_{1,k} v_{1,k}
+ \omega \mu \sigma \cdot ( u^*_1 v_0 - u^*_0 v_1) \right) dx = \int_{\Omega}  Y_i v_i + X_{ij}  v_{i,j} \; dx
\end{equation}
for all $v_i$ with $v_i=0$ on $\Gamma_{0}$. Setting $q=u^*$ in Equation~(\ref{ref:2DMTTE:EQU:201A}) and
$v_i=\hat{u}_i$ in Equation~(\ref{ref:2DMTTE:EQU:202d}) Equation~(\ref{ref:2DMTTE:EQU:201A}) gives
\begin{equation}\label{ref:2DMTTE:EQU:290}
\int_{\Omega} \omega \mu \hat{\sigma} \cdot ( u^*_0 u_1 - u^*_1 u_0) \; dx
= \int_{\Omega}  Y_i \hat{u}_i + X_{ij}  \hat{u}_{i,j} \; dx
\end{equation}
This gives
\begin{equation}\label{ref:2DMTTE:EQU:300}
\int_{\Omega}   \frac{\partial J^{MT}}{\partial \sigma} \cdot \hat{\sigma} \; dx  =
\int_{\Omega} \omega \mu \hat{\sigma} \cdot ( u^*_0 u_1 - u^*_1 u_0) \; dx
\end{equation}
or
\begin{equation}\label{ref:2DMTTE:EQU:301}
\frac{\partial J^{MT}}{\partial \sigma} = \omega \mu \cdot ( u^*_0 u_1 - u^*_1 u_0)
\end{equation}


\subsection{Variational updates}
In this section, we give the explicit update equations for every hidden variable that is applied at each iteration of the (stochastic) variational bayes algorithm.\\
Additionally, we write down the analytical form of the ELBO. Although computing the ELBO is not necessary in order to estimate the posterior distribution of the parameters, it is used to monitor the convergence of the algorithm.

\subsection{Variational Updates}

\subsubsection{Non-sparse factors}
For every group $g$, sample $n$ and factor $k$: \\

Prior distribution $p(z_{nk}^g)$:\\
\[
	p(z_{nk}^g), = \mathcal{N} (z_{nk}^g |0,1)
\]
Variational distribution $q(z^g_{nk})$:\\
\begin{equation}
   q(z^g_{nk}) = \mathcal{N} ( z_{nk}^g | \mu_{z_{nk}^g}, \sigma_{z_{nk}^g} )
\end{equation}
where
\begin{equation} \begin{aligned}
	\sigma_{z_{nk}^{g}}^2 &= \left( \sum_{m=1}^{M} \sum_{d=1}^{D_m} \la\tau_{dg}^m \ra \la (w_{dk}^m)^2 \ra + 1 \right)^{-1}\\
	\mu_{z_{nk}^g} &= \sigma_{z_{nk}^g}^2 \sum_{m=1}^{M} \sum_{d=1}^{D_m} \la\tau_{dg}^m\ra \la w_{dk}^m \ra \left(\sum_{g=1}^{G} y_{nd}^{gm} - \sum_{j \neq k} \la w_{dj}^m \ra \la z_{nj}^g \ra \right)
\end{aligned} \end{equation}

\subsubsection{Sparse factors}
For every group $g$, sample $n$ and factor $k$: \\

Prior distribution $p(\hat{z}_{nk}^g,s_{nk}^g)$:\\

\begin{align}
	p(\hat{z}_{nk}^g,s_{nk}^g) &= \mathcal{N} (\hat{z}_{nk}^g \,|\, 0, 1/\alpha_k^g)\, \text{Ber}(s_{nk}^g \,|\,\theta_k^g) \\
	p(\theta_k^g) &= \Bdist{\theta_k^g}{a_0^\theta,b_0^\theta}\\
	p(\alpha_k^g) &= \Gdist{\alpha_k^g}{a_0^\alpha, b_0^\alpha},
\end{align}

Variational distribution:\\
TO-DO

\subsubsection{group-wise ARD precision for the factors}
\subsubsection{Spike-and-slab sparsity for the factors}

    % \paragraph{Sample-wise}

    %     Variational distribution:\\

    %     \begin{equation}
    %         q(\alpha^g_{k}) = \Gamma \left( \alpha_k^g \middle| \hat{a}_{g,k}^{\alpha}, \hat{b}_{g,k}^{\alpha} \right)
    %     \end{equation}

    %     where\\

    %     \begin{equation}
    %       \begin{aligned}
    %           \hat{a}_{g,k}^\alpha &= a_0^\alpha + \frac{N_g}{2}\\
    %           \hat{b}_{g,k}^\alpha &= b_0^\alpha +\frac{ \sum_{n=1}^{N_g} \la (z_{nk}^g)^2 \ra }{2}
    %       \end{aligned}
    %     \end{equation}

% START COPIED
% Prior distribution $p(\hat{w}_{dk}^m,s_{dk}^m)$:\\
% \begin{align}
% 	p(\hat{w}_{dk}^m,s_{dk}^m) &= \mathcal{N} (\hat{w}_{dk}^m \,|\, 0, 1/\alpha_k^m)\, \text{Ber}(s_{dk}^m \,|\,\theta_k^m)
% \end{align}

% Variational distribution $q(\hat{w}_{dk}^m,s_{dk}^m)$:\\

% Update for $q(s_{dk}^m)$:\\

% \begin{equation}
% 	q(s^m_{dk}) = \mathrm{Ber}(s^m_{dk}|\gamma^m_{dk})
% \end{equation}
% with
% \begin{equation} \begin{aligned}
% 	&\gamma^m_{dk} = \frac{1}{1+\exp(-\lambda_{dk}^m)}\\
% 	& \lambda_{dk}^m = \la \ln\frac{\theta}{1-\theta} \ra + 0.5\ln\frac{\la\alpha_k^m\ra}{\la\tau_d^m\ra} - 0.5\ln\left( \sum_{g=1}^G\sum_{n=1}^{N_g} \la z_{nk}^2 \ra + \frac{\la\alpha_k^m\ra}{\la\tau_d^m\ra} \right) \\
% 	&+ \frac{\la\tau_d^m\ra}{2} \frac{ \left( \sum_{g=1}^G\sum_{n=1}^{N_g} y_{nd}^m \la z_{nk} \ra - \sum_{j \neq k} \la s_{dj}^m\hat{w}_{dj}^m\ra \sum_{g=1}^G\sum_{n=1}^{N_g} \la z_{nk} \ra \la z_{nj} \ra \right)^2} {\sum_{g=1}^G\sum_{n=1}^{N_g} \la z_{nk}^2 \ra + \frac{\la\alpha_k^m\ra}{\la\tau_d^m\ra} }
% \end{aligned} \end{equation}

% Update for $q(\hat{w}_{dk}^m)$:\\

% \begin{equation} \begin{aligned}
%       q(\hat{w}_{dk}^m|s_{dk}^m=0) &= \mathcal{N} \left(\hat{w}_{dk}^m \middle| 0, 1/\alpha_k^m \right) \\
%       q(\hat{w}_{dk}^m|s_{dk}^m=1) &= \mathcal{N} \left( \hat{w}_{dk}^m \middle| \mu_{w_{dk}^m}, \sigma_{w_{dk}^m}^2\right)
%   \end{aligned} \end{equation}
% with
% \begin{equation} \begin{aligned}
%   	\mu_{w_{dk}^m} &= \frac{ \sum_{g=1}^G\sum_{n=1}^{N_g} y_{nd}^m \la z_{nk} \ra - \sum_{j \neq k} \la s_{dj}^m\hat{w}_{dj}^m \ra \sum_{g=1}^G\sum_{n=1}^{N_g} \la z_{nk} \ra \la z_{nj} \ra } { \sum_{g=1}^G\sum_{n=1}^{N_g} \la z_{nk}^2 \ra + \frac{\la\alpha_k^m\ra}{\la\tau_d^m\ra} }\\
%   	\sigma_{w_{dk}^m} &= \frac{ \la\tau_d^m\ra^{-1} } { \sum_{g=1}^G\sum_{n=1}^{N_g} \la z_{nk}^2 \ra + \frac{\la\alpha_k^m\ra}{\la\tau_d^m\ra} }
% \end{aligned} \end{equation}
% where $N$ is the total number of samples (across all groups).

% END COPIED

\subsubsection{Sparse weights}
For every view $m$, feature $d$ and factor $k$: \\

Prior distribution $p(\hat{w}_{dk}^m,s_{dk}^m)$:\\
\begin{align}
	p(\hat{w}_{dk}^m,s_{dk}^m) &= \mathcal{N} (\hat{w}_{dk}^m \,|\, 0, 1/\alpha_k^m)\, \text{Ber}(s_{dk}^m \,|\,\theta_k^m)
\end{align}

Variational distribution $q(\hat{w}_{dk}^m,s_{dk}^m)$:\\

Update for $q(s_{dk}^m)$:\\

\begin{equation}
	q(s^m_{dk}) = \mathrm{Ber}(s^m_{dk}|\gamma^m_{dk})
\end{equation}
with
\begin{equation} \begin{aligned}
	&\gamma^m_{dk} = \frac{1}{1+\exp(-\lambda_{dk}^m)}\\
	& \lambda_{dk}^m = \la \ln\frac{\theta}{1-\theta} \ra + 0.5\ln\frac{\la\alpha_k^m\ra}{\la\tau_d^m\ra} - 0.5\ln\left( \sum_{g=1}^G\sum_{n=1}^{N_g} \la z_{nk}^2 \ra + \frac{\la\alpha_k^m\ra}{\la\tau_d^m\ra} \right) \\
	&+ \frac{\la\tau_d^m\ra}{2} \frac{ \left( \sum_{g=1}^G\sum_{n=1}^{N_g} y_{nd}^m \la z_{nk} \ra - \sum_{j \neq k} \la s_{dj}^m\hat{w}_{dj}^m\ra \sum_{g=1}^G\sum_{n=1}^{N_g} \la z_{nk} \ra \la z_{nj} \ra \right)^2} {\sum_{g=1}^G\sum_{n=1}^{N_g} \la z_{nk}^2 \ra + \frac{\la\alpha_k^m\ra}{\la\tau_d^m\ra} }
\end{aligned} \end{equation}

Update for $q(\hat{w}_{dk}^m)$:\\

\begin{equation} \begin{aligned}
      q(\hat{w}_{dk}^m|s_{dk}^m=0) &= \mathcal{N} \left(\hat{w}_{dk}^m \middle| 0, 1/\alpha_k^m \right) \\
      q(\hat{w}_{dk}^m|s_{dk}^m=1) &= \mathcal{N} \left( \hat{w}_{dk}^m \middle| \mu_{w_{dk}^m}, \sigma_{w_{dk}^m}^2\right)
  \end{aligned} \end{equation}
with
\begin{equation} \begin{aligned}
  	\mu_{w_{dk}^m} &= \frac{ \sum_{g=1}^G\sum_{n=1}^{N_g} y_{nd}^m \la z_{nk} \ra - \sum_{j \neq k} \la s_{dj}^m\hat{w}_{dj}^m \ra \sum_{g=1}^G\sum_{n=1}^{N_g} \la z_{nk} \ra \la z_{nj} \ra } { \sum_{g=1}^G\sum_{n=1}^{N_g} \la z_{nk}^2 \ra + \frac{\la\alpha_k^m\ra}{\la\tau_d^m\ra} }\\
  	\sigma_{w_{dk}^m} &= \frac{ \la\tau_d^m\ra^{-1} } { \sum_{g=1}^G\sum_{n=1}^{N_g} \la z_{nk}^2 \ra + \frac{\la\alpha_k^m\ra}{\la\tau_d^m\ra} }
\end{aligned} \end{equation}
where $N$ is the total number of samples (across all groups).

  	% Taken together this means that we can update $q(\hat{w}_{dk}^m,s_{dk}^m)$ using:
  	% \begin{equation*}
  	% q(\hat{w}_{dk}^m|s_{dk}^m) q(s_{dk}^m) = \Ndist{ \hat{w}_{dk}^m } { s_{dk}^m \mu_{w_{dk}^m}, s_{dk}^m\sigma_{w_{dk}^m}^2 + (1-s_{dk}^m)/\alpha_k^m}    (\new{\gamma_{dk}^m})^{s_{dk}^m} (1-\new{\gamma_{dk}^m})^{1-s_{dk}}
  	% \end{equation*}

\subsubsection{ARD precision for the loadings}
For every view $m$ and factor $k$: \\

Prior distribution $p(\alpha_k^m)$:
\[
	p(\alpha_k^m) = \Gdist{\alpha_k^m}{a_0^\alpha, b_0^\alpha},
\]
Variational distribution $q(\alpha_k^m)$:
\begin{equation}
    q(\alpha^m_{k}) = \Gamma \left( \alpha_k^m \middle| \hat{a}_{mk}^{\alpha}, \hat{b}_{mk}^{\alpha} \right)
\end{equation}
where:
\begin{equation} \begin{aligned}
	\hat{a}_{mk}^\alpha &= a_0^\alpha + \frac{D_m}{2}\\
	\hat{b}_{mk}^\alpha &= b_0^\alpha +\frac{ \sum_{d=1}^{D_m} \la (\hat{w}_{dk}^m)^2 \ra }{2}
\end{aligned} \end{equation}

\subsubsection{Spike-and-slab sparsity for the loadings}
For every view $m$ and factor $k$: \\

Prior distribution:
\[
	p(\theta_k^m) = \Bdist{\theta_k^m}{a_0^\theta,b_0^\theta}
\]
Variational distribution:
\begin{equation}
	q(\theta_k^m) = \Bdist{\theta_k^m}{\hat{a}_{mk}^{\theta}, \hat{b}_{mk}^{\theta}}
\end{equation}
where
\begin{equation}
     \begin{aligned}
  	\hat{a}_{mk}^\theta &= \sum_{d=1}^{D_m} \la s^m_{dk}\ra + a_0^\theta\\
  	\hat{b}_{mk}^\theta &= b_0^\theta - \sum_{d=1}^{D_m} \la s^m_{dk}\ra + D_m
     \end{aligned}
\end{equation}

\subsubsection{Noise}
For every view $m$, group $g$ and feature $d$:\\

Prior distribution $p(\tau_{dg}^m)$:
\[
	p(\tau_{dg}^m) = \Gdist{\tau_{dg}^m}{a_0^\tau,b_0^\tau},
\]

Variational distribution $q(\tau_d^m)$:
\begin{equation}
	q(\tau_{dg}^m) = \Gdist{\tau_{dg}^m}{\hat{a}_{dg}^{m} , \hat{b}_{dg}^{m}}
\end{equation}
where:
\begin{equation} \begin{aligned}
	\hat{a}_{dg}^{m} &= a_0^{\tau} + \frac{N_g}{2}\\
	\hat{b}_{dg}^{m} &= b_0^{\tau} + \frac{1}{2} \sum_{n=1}^{N_g}  \la\left(y_{nd}^{gm} - \sum_k^{K} \hat{w}_{dk}^m s_{dk}^m z_{nk}^{g}\right)^2 \ra
\end{aligned} \end{equation}

\subsection{Evidence Lower Bound}
As shown in~\ref{section:technical:elbo_interpretation}, the ELBO can be decomposed into a sum of two terms: (1) the expected log likelihood under the current estimate of the posterior distribution of the parameters and (2) the KL divergence between the prior and the variational distributions of the parameters:\\

\begin{equation} \begin{aligned}
    \Lagr = \E_{q(\theta)} \ln P(Y|\Theta) - \KL\left(q(\Theta) \middle|\middle| p(\Theta) \right)
\end{aligned} \end{equation}

\subsection{Log likelihood term}
Assuming a gaussian likelihood:
\begin{equation} \begin{aligned}
	\E_{q(\theta)} \ln P(Y|\Theta) = & -\sum_{m=1}^M \frac{ND_m}{2} \ln(2\pi) + \sum_{g=1}^G \frac{N_g}{2} \sum_{m=1}^M \sum_{d=1}^{D_m} \la \ln(\tau_{dg}^m) \ra \\
	&-\sum_{g=1}^G \sum_{m=1}^M \sum_{d=1}^{D_m} \frac{\la \tau_{dg}^m \ra}{2} \sum_{n=1}^{N_g} \big( y_{nd}^{m,g} - \sum_{k=1}^{K}\la s_{dk}^m \hat{w}_{dk}^m \ra \la z_{nk}^g \ra \big)^2
\end{aligned} \end{equation}

\subsection{KL divergence terms}

Note that $\KL\left(q(\Theta) \middle|\middle| P(\Theta) \right) = \E_q(q(\Theta)) - \E_q(P(\Theta))$. \\
Below, we will write the analytical form for these two expectations.

\subsubsection*{Sparse loadings}

\begin{equation} \begin{aligned}
    \E_q[\ln p(\hat{W},S)] =& -\sum_{m=1}^{M}\frac{KD_m}{2}\ln(2\pi) + \sum_{m=1}^{M}\frac{D_m}{2}\sum_{k=1}^{K} \ln(\alpha_k^m) - \sum_{m=1}^{M} \frac{\alpha_k^m}{2} \sum_{d=1}^{D_m} \sum_{k=1}^{K} \la (\hat{w}_{dk}^m)^2 \ra \\
    & + \la \ln(\theta) \ra \sum_{m=1}^{M} \sum_{d=1}^{D_m} \sum_{k=1}^{K} \la s_{dk}^m \ra + \la \ln(1-\theta) \ra \sum_{m=1}^{M} \sum_{d=1}^{D_m}\sum_{k=1}^{K} (1- \la s_{dk}^m \ra)
\end{aligned} \end{equation}

\begin{equation} \begin{aligned}
	\E_q[\ln q(\hat{W}, S)] =&-\sum_{m=1}^{M}\frac{KD_m}{2}\ln(2\pi) + \frac{1}{2}\sum_{m=1}^{M}\sum_{d=1}^{D_m}\sum_{k=1}^{K}\ln(\la s_{dk}^m \ra \sigma_{w_{dk}^m}^2 + (1-\la s_{dk}^m \ra)/\alpha_k^m) \\
	&+ \sum_{m=1}^{M} \sum_{d=1}^{D_m} \sum_{k=1}^{K} (1-\la s_{dk}^m \ra) \ln(1 - \la s_{dk}^m \ra) - \la s_{dk}^m \ra \ln \la s_{dk}^m \ra
\end{aligned} \end{equation}

\subsubsection*{Sparse factors}

\begin{equation} \begin{aligned}
    \E_q[\ln p(\hat{Z},S)] =& -\sum_{g=1}^{G}\frac{N_g K}{2}\ln(2\pi) + \sum_{g=1}^{G}\frac{N_g}{2}\sum_{k=1}^{K} \ln(\alpha_k^g) - \sum_{g=1}^{G} \frac{\alpha_k^g}{2} \sum_{n=1}^{N_g} \sum_{k=1}^{K} \la (\hat{z}_{nk}^g)^2 \ra \\
    & + \la \ln(\theta) \ra \sum_{g=1}^{G} \sum_{n=1}^{N_g} \sum_{k=1}^{K} \la s_{nk}^g \ra + \la \ln(1-\theta) \ra \sum_{g=1}^{G} \sum_{n=1}^{N_g}\sum_{k=1}^{K} (1- \la s_{nk}^g \ra)
\end{aligned} \end{equation}

\begin{equation} \begin{aligned}
	\E_q[\ln q(\hat{Z}, S)] =&-\sum_{g=1}^{G}\frac{N_g K}{2}\ln(2\pi) + \frac{1}{2}\sum_{g=1}^{G}\sum_{n=1}^{N_g}\sum_{k=1}^{K}\ln(\la s_{nk}^g \ra \sigma_{z_{nk}^g}^2 + (1-\la s_{nk}^g \ra)/\alpha_k^g) \\
	&+ \sum_{g=1}^{G} \sum_{n=1}^{N_g} \sum_{k=1}^{K} (1-\la s_{nk}^g \ra) \ln(1 - \la s_{nk}^g \ra) - \la s_{nk}^g \ra \ln \la s_{nk}^g \ra
\end{aligned} \end{equation}

\subsubsection*{Non-sparse factors}
\begin{equation} \begin{aligned}
	\E_q [\ln p(Z)] &= -\frac{NK}{2}\ln(2\pi) -\frac{1}{2} \sum_{g=1}^G\sum_{n=1}^{N_g}\sum_{k=1}^{K} \la z_{nk}^2 \ra \\
	\E_q [\ln q(Z)] &= - \frac{NK}{2}(1 + \ln(2\pi)) - \frac{1}{2}\sum_{g=1}^G\sum_{n=1}^{N_g}\sum_{k=1}^{K} \ln(\sigma_{z_{nk}}^2)
\end{aligned} \end{equation}

\subsubsection*{Automatic relevance determination for the loadings}
\begin{equation} \begin{aligned}
	\E_q [\ln p(\balpha)] &= \sum_{m=1}^{M}\sum_{k=1}^{K}\Big(a_0^\alpha\ln b_0^\alpha +   (a_0^\alpha - 1) \la \ln \alpha_k \ra - b_0^\alpha \la \alpha_k \ra - \ln \Gamma(a_0^\alpha) \Big) \\
	\E_q [\ln q(\balpha)] &= \sum_{m=1}^{M}\sum_{k=1}^{K} \Big( \hat{a}_{k}^\alpha \ln \hat{b}_{k}^\alpha + (\hat{a}_{k}^\alpha - 1) \la \ln \alpha_k \ra - \hat{b}_{k}^\alpha \la \alpha_k \ra - \ln \Gamma(\hat{a}_{k}^\alpha) \Big)
\end{aligned} \end{equation}

\subsubsection*{Noise}
\begin{equation} \begin{aligned}
	\E_q [\ln p(\btau)] &= \sum_{m=1}^{M} D_m a_0^\tau \ln b_0^\tau + \sum_{g=1}^{G}\sum_{m=1}^{M}\sum_{d=1}^{Dm} (a_0^\tau - 1) \la \ln \tau_d^{mg} \ra - \sum_{g=1}^{G}\sum_{m=1}^{M}\sum_{d=1}^{Dm} b_0^\tau \la \tau_d^{mg} \ra - \sum_{m=1}^{M} D_m \ln \Gamma(a_0^\tau)\\
	\E_q [\ln q(\btau)] &= \sum_{g=1}^{G} \sum_{m=1}^{M} \sum_{d=1}^{D_m} \left( \hat{a}_{dmg}^\tau \ln \hat{b}_{dmg}^\tau + (\hat{a}_{dmg}^\tau - 1) \la \ln \tau_d^{mg} \ra - \hat{b}_{dmg}^\tau \la \tau_d^{mg} \ra - \ln \Gamma(\hat{a}_{dmg}^\tau) \right)
\end{aligned} \end{equation}

\subsubsection*{Sparsity of the loadings}
\begin{equation} \begin{aligned}
	  &\E_q\left[ \ln p(\btheta) \right] = \sum_{m=1}^M \sum_{k=1}^K\sum_{d=1}^{D_m}\left( (a_0 - 1) \times \la \ln(\pi^m_{d, k}) \ra + (b_0 -1) \la \ln(1 - \pi^m_{d, k}) \ra - \ln (\mathrm{B} (a_0, b_0))\right) \\
	  &\E_q\left[ \ln q(\btheta) \right] = \sum_{m=1}^M \sum_{k=1}^K\sum_{d=1}^{D_m}\left( (a^m_{k,d} - 1) \times \la \ln(\pi^m_{d, k}) \ra + (b^m_{k,d} -1) \la \ln(1 - \pi^m_{d, k}) \ra - \ln (\mathrm{B} (a^m_{k,d}, b^m_{k,d})) \right) \\
\end{aligned} \end{equation}