%!TEX root = ceres.tex
\chapter{Theory}
\label{chapter:theory}
Effective use of Ceres requires some familiarity with the underlying theory. In this chapter we will provide a brief exposition to Ceres's approach to solving non-linear least squares optimization. Much of the material in this section comes from~\cite{Agarwal10bal,wu2011multicore,kushal2012}.

\section{The Levenberg-Marquardt Algorithm}

The Levenberg-Marquardt algorithm~\cite{levenberg1944method, marquardt1963algorithm} is the most popular algorithm for solving non-linear least squares problems. Ceres implements an exact step~\cite{madsen2004methods} and an inexact step variant of the Levenberg-Marquardt algorithm~\cite{wright1985inexact,nash1990assessing}.  We begin by taking a brief look at how the Levenberg-Marquardt algorithm works.

Let $x \in \mathbb{R}^{n}$ be an $n$-dimensional vector of variables, and
$ F(x) = \left[f_1(x),   \hdots,  f_{m}(x) \right]^{\top}$ be a $m$-dimensional function of $x$.  We are interested in solving the following optimization problem,
\begin{equation}
        \arg \min_x \frac{1}{2}\|F(x)\|^2\ .
        \label{eq:nonlinsq}
\end{equation}
The Jacobian $J(x)$ of $F(x)$ is an $m\times n$ matrix, where $J_{ij}(x) = \partial_j f_i(x)$  and the gradient vector $g(x) = \nabla  \frac{1}{2}\|F(x)\|^2 = J(x)^\top F(x)$. Since the efficient global optimization of~\eqref{eq:nonlinsq} for general $F(x)$ is an intractable problem, we will have to settle for finding a local minimum.

The general strategy when solving non-linear optimization problems is to solve a sequence of approximations to the original problem~\cite{nocedal2000numerical}. At each iteration, the approximation is solved to determine a correction $\Delta x$ to the vector $x$. For non-linear least squares, an approximation can be constructed by using the linearization $F(x+\Delta x) \approx F(x) + J(x)\Delta x$, which leads to the following linear least squares  problem:
\begin{equation}
         \min_{\Delta x} \frac{1}{2}\|J(x)\Delta x + F(x)\|^2
        \label{eq:linearapprox}
\end{equation}
Unfortunately, na\"ively solving a sequence of these problems and updating $x \leftarrow x+ \Delta x$ leads to an algorithm that may not converge.  To get a convergent algorithm, we need to control the size of the step $\Delta x$.  One way to do this is to introduce a regularization term:
\begin{equation}
         \min_{\Delta x} \frac{1}{2}\|J(x)\Delta x + F(x)\|^2 + \mu \|D(x)\Delta x\|^2\ .
        \label{eq:lsqr}
\end{equation}
Here, $D(x)$ is a non-negative diagonal matrix, typically the square root of the diagonal of the matrix $J(x)^\top J(x)$ and $\mu$ is a non-negative parameter that controls the strength of regularization. It is straightforward to show that the step size $\|\Delta x\|$ is inversely related to $\mu$. Levenberg-Marquardt updates the value of $\mu$ at each step based on how well the Jacobian $J(x)$ approximates $F(x)$. The quality of this fit is measured by the ratio of  the actual decrease in the objective function to the decrease in the value of the linearized model $L(\Delta x) = \frac{1}{2}\|J(x)\Delta x + F(x)\|^2$.
\begin{equation}
\rho = \frac{\|F(x + \Delta x)\|^2 - \|F(x)\|^2}{\|J(x)\Delta x + F(x)\|^2 - \|F(x)\|^2}
\end{equation}

If $\rho$ is large, that means the linear model is a good approximation to the non-linear model and it is worth trusting it more in the computation of $\Delta x$, so we decrease $\mu$. If $\rho$ is small, the linear model is a poor approximation and $\mu$ is increased. This kind of reasoning is the basis of Trust-region methods, of which Levenberg-Marquardt is an early example~\cite{nocedal2000numerical}.

Before going further, let us make some notational simplifications. We will assume that the matrix $\sqrt{\mu} D$ has been concatenated at the bottom of the matrix $J$ and similarly a vector of zeros has been added to the bottom of the vector $f$ and the rest of our discussion will be in terms of $J$ and $f$, \ie the linear least squares problem.
\begin{align}
 \min_{\Delta x} \frac{1}{2} \|J(x)\Delta x + f(x)\|^2 .
 \label{eq:simple}
\end{align}
Further, let $H(x)= J(x)^\top J(x)$ and $g(x) = -J(x)^\top  f(x)$. For notational convenience let us also drop the dependence on $x$. Then it is easy to see that solving~\eqref{eq:simple} is equivalent to solving the {\em normal equations}
\begin{align}
H \Delta x  &= g \label{eq:normal}
\end{align}

For all but the smallest problems the solution of~\eqref{eq:normal} in each iteration of the Levenberg-Marquardt algorithm is the dominant computational cost in Ceres. Ceres provides a number of different options for solving~\eqref{eq:normal}.

\section{\texttt{DENSE\_QR}}
For small problems (a couple of hundred parameters and a few thousand residuals) with relatively dense Jacobians, \texttt{DENSE\_QR} is the method of choice~\cite{bjorck1996numerical}. Let $J = QR$ be the QR-decomposition of $J$, where $Q$ is an orthonormal matrix and $R$ is an upper triangular matrix~\cite{trefethen1997numerical}. Then it can be shown that the solution to~\eqref{eq:normal} is given by
\begin{align}
	\Delta x^* = -R^{-1}Q^\top f
\end{align}
Ceres uses \texttt{Eigen}'s dense QR decomposition routines.

\section{\texttt{SPARSE\_NORMAL\_CHOLESKY}}
Large non-linear least square problems are usually sparse. In such cases, using a dense QR factorization is inefficient. Let $H = R^\top R$ be the Cholesky factorization of the normal equations, where $R$ is an upper triangular matrix, then the  solution to ~\eqref{eq:normal} is given by
\begin{equation}
	\Delta x^* = R^{-1} R^{-\top} g.
\end{equation}
The observant reader will note that the $R$ in the Cholesky factorization of $H$ is the same upper triangular matrix $R$ in the QR factorization of $J$. Since $Q$ is an orthonormal matrix, $J=QR$ implies that $J^\top J = R^\top Q^\top Q R = R^\top R$.

There are two variants of Cholesky factorization -- sparse and dense. \texttt{SPARSE\_NORMAL\_CHOLESKY}, as the name implies performs a sparse Cholesky factorization of the normal equations. This leads to substantial savings in time and memory for large sparse problems. We use the Professor Tim Davis' \texttt{CHOLMOD} library (part of the \texttt{SuiteSparse} package) to perform the sparse cholesky~\cite{chen2006acs}.

\section{\texttt{DENSE\_SCHUR} \& \texttt{SPARSE\_SCHUR}}
While it is possible to use \texttt{SPARSE\_NORMAL\_CHOLESKY} to solve bundle adjustment problems, bundle adjustment problem have a special structure, and a more efficient scheme for solving~\eqref{eq:normal} can be constructed.

Suppose that the SfM problem consists of $p$ cameras and $q$ points and the variable vector $x$ has the  block structure $x = [y_{1},\hdots,y_{p},z_{1},\hdots,z_{q}]$. Where, $y$ and $z$ correspond to camera and point parameters, respectively.  Further, let the camera blocks be of size $c$ and the point blocks be of size $s$ (for most problems $c$ =  $6$--$9$ and $s = 3$). Ceres does not impose any constancy requirement on these block sizes, but choosing them to be constant simplifies the exposition.

A key characteristic of the bundle adjustment problem is that there is no term $f_{i}$ that includes two or more point blocks.  This in turn implies that the matrix $H$ is of the form
\begin{equation}
        H =  \left[
                \begin{matrix} B & E\\ E^\top & C
                \end{matrix}
                \right]\ ,
\label{eq:hblock}
\end{equation}
where, $B \in \reals^{pc\times pc}$ is a block sparse matrix with $p$ blocks of size $c\times c$ and  $C \in \reals^{qs\times qs}$ is a block diagonal matrix with $q$ blocks of size $s\times s$. $E \in \reals^{pc\times qs}$ is a general block sparse matrix, with a block of size $c\times s$ for each observation. Let us now block partition $\Delta x = [\Delta y,\Delta z]$ and $g=[v,w]$ to restate~\eqref{eq:normal} as the block structured linear system
\begin{equation}
        \left[
                \begin{matrix} B & E\\ E^\top & C
                \end{matrix}
                \right]\left[
                        \begin{matrix} \Delta y \\ \Delta z
                        \end{matrix}
                        \right]
                        =
                        \left[
                                \begin{matrix} v\\ w
                                \end{matrix}
                                \right]\ ,
\label{eq:linear2}
\end{equation}
and apply Gaussian elimination to it. As we noted above, $C$ is a block diagonal matrix, with small diagonal blocks of size $s\times s$.
Thus, calculating the inverse of $C$ by inverting each of these blocks is  cheap. This allows us to  eliminate $\Delta z$ by observing that $\Delta z = C^{-1}(w - E^\top \Delta y)$, giving us
\begin{equation}
        \left[B - EC^{-1}E^\top\right] \Delta y = v - EC^{-1}w\ .  \label{eq:schur}
\end{equation}
The matrix
\begin{equation}
S = B - EC^{-1}E^\top\ ,
\end{equation}
is the Schur complement of $C$ in $H$. It is also known as the {\em reduced camera matrix}, because the only variables participating in~\eqref{eq:schur} are the ones corresponding to the cameras. $S \in \reals^{pc\times pc}$ is a block structured symmetric positive definite matrix, with blocks of size $c\times c$. The block $S_{ij}$ corresponding to the pair of images $i$ and $j$ is non-zero if and only if the two images observe at least one common point.

Now, \eqref{eq:linear2}~can  be solved by first forming $S$, solving for $\Delta y$, and then back-substituting $\Delta y$ to obtain the value of $\Delta z$.
Thus, the solution of what was an $n\times n$, $n=pc+qs$ linear system is reduced to the inversion of the block diagonal matrix $C$, a few matrix-matrix and matrix-vector multiplies, and the solution of block sparse $pc\times pc$ linear system~\eqref{eq:schur}.  For almost all  problems, the number of cameras is much smaller than the number of points, $p \ll q$, thus solving~\eqref{eq:schur} is significantly cheaper than solving~\eqref{eq:linear2}. This is the {\em Schur complement trick}~\cite{brown-58}.

This still leaves open the question of solving~\eqref{eq:schur}. The
method of choice for solving symmetric positive definite systems
exactly is via the Cholesky
factorization~\cite{trefethen1997numerical} and depending upon the
structure of the matrix, there are, in general, two options. The first
is direct factorization, where we store and factor $S$ as a dense
matrix~\cite{trefethen1997numerical}. This method has $O(p^2)$ space complexity and $O(p^3)$ time
complexity and is only practical for problems with up to a few hundred
cameras. Ceres implements this strategy as the \texttt{DENSE\_SCHUR} solver.

 But, $S$ is typically a fairly sparse matrix, as most images
only see a small fraction of the scene. This leads us to the second
option: sparse direct methods. These methods store $S$ as a sparse
matrix, use row and column re-ordering algorithms to maximize the
sparsity of the Cholesky decomposition, and focus their compute effort
on the non-zero part of the factorization~\cite{chen2006acs}.
Sparse direct methods, depending on the exact sparsity structure of the Schur complement,
allow bundle adjustment algorithms to significantly scale up over those based on dense
factorization. Ceres implements this strategy as the \texttt{SPARSE\_SCHUR} solver.

\section{\texttt{ITERATIVE\_SCHUR}}

The factorization methods described above are based on computing an exact solution of~\eqref{eq:lsqr}. But it is not clear if an exact solution of~\eqref{eq:lsqr} is necessary at each step of the LM algorithm to solve~\eqref{eq:nonlinsq}. In fact, we have already seen evidence that this may not be the case, as~\eqref{eq:lsqr} is itself a regularized version of~\eqref{eq:linearapprox}. Indeed, it is possible to construct non-linear optimization algorithms in which the linearized problem is solved approximately. These algorithms are known as inexact Newton or truncated Newton methods~\cite{nocedal2000numerical}.

An inexact Newton method requires two ingredients. First, a cheap method for approximately solving systems of linear equations. Typically an iterative linear solver like the Conjugate Gradients method is used for this purpose~\cite{nocedal2000numerical}. Second, a termination rule for the iterative solver. A typical termination rule is of the form
\begin{equation}
        \|H(x) \Delta x + g(x)\| \leq \eta_k \|g(x)\|. \label{eq:inexact}
\end{equation}
Here, $k$ indicates the Levenberg-Marquardt iteration number and $0 < \eta_k <1$ is known as the forcing sequence.  Wright \& Holt \cite{wright1985inexact} prove that a truncated Levenberg-Marquardt algorithm that uses an inexact Newton step based on~\eqref{eq:inexact} converges for any sequence $\eta_k \leq \eta_0 < 1$ and the rate of convergence depends on the choice of the forcing sequence $\eta_k$.

The convergence rate of Conjugate Gradients  for solving~\eqref{eq:normal} depends on the distribution of eigenvalues of $H$~\cite{saad2003iterative}. A useful upper bound is $\sqrt{\kappa(H)}$, where, $\kappa(H)$f is the condition number of the matrix $H$. For most bundle adjustment problems, $\kappa(H)$ is high and a direct application of Conjugate Gradients to~\eqref{eq:normal} results in extremely poor performance.

The solution to this problem is to replace~\eqref{eq:normal} with a {\em preconditioned} system.  Given a linear system, $Ax =b$ and a preconditioner $M$ the preconditioned system is given by $M^{-1}Ax = M^{-1}b$. The resulting algorithm is known as Preconditioned Conjugate Gradients algorithm (PCG) and its  worst case complexity now depends on the condition number of the {\em preconditioned} matrix $\kappa(M^{-1}A)$.

\subsection{Preconditioning}

The computational cost of using a preconditioner $M$ is the cost of computing $M$ and evaluating the product $M^{-1}y$ for arbitrary vectors $y$. Thus, there are two competing factors to consider: How much of $H$'s structure is captured by $M$ so that the condition number $\kappa(HM^{-1})$ is low, and the computational cost of constructing and using $M$.  The ideal preconditioner would be one for which $\kappa(M^{-1}A) =1$. $M=A$ achieves this, but it is not a practical choice, as applying this preconditioner would require solving a linear system equivalent to the unpreconditioned problem.  It is usually the case that the more information $M$ has about $H$, the more expensive it is use. For example, Incomplete Cholesky factorization based preconditioners  have much better convergence behavior than the Jacobi preconditioner, but are also much more expensive.

The simplest of all preconditioners is the diagonal or Jacobi preconditioner, \ie,  $M=\operatorname{diag}(A)$, which for block structured matrices like $H$ can be generalized to the block Jacobi preconditioner. Another option is to apply PCG to the reduced camera matrix $S$ instead of $H$. One reason to do this is that $S$ is a much smaller matrix than $H$, but more importantly, it can be shown that $\kappa(S)\leq \kappa(H)$.  Ceres implements PCG on $S$ as the \texttt{ITERATIVE\_SCHUR} solver. When the user chooses \texttt{ITERATIVE\_SCHUR} as the linear solver, Ceres automatically switches from the exact step algorithm to an inexact step algorithm.

There are two obvious choices for block diagonal preconditioners for $S$. The block diagonal of the matrix $B$~\cite{mandel1990block} and the block diagonal $S$, \ie the block Jacobi preconditioner for $S$. Ceres's implements both of these preconditioners and refers to them as  \texttt{JACOBI} and \texttt{SCHUR\_JACOBI} respectively.

As discussed earlier, the cost of forming and storing the Schur complement $S$ can be prohibitive for large problems. Indeed, for an inexact Newton solver that computes $S$ and runs PCG on it, almost all of its time is spent in constructing $S$; the time spent inside the PCG algorithm is negligible in comparison. Because  PCG only needs access to $S$ via its product with a vector, one way to evaluate $Sx$ is to observe that
\begin{align}
  x_1 &= E^\top x \notag \\
  x_2 &= C^{-1} x_1 \notag\\
  x_3 &= Ex_2 \notag\\
  x_4 &= Bx \notag\\
  Sx &= x_4 - x_3\ .\label{eq:schurtrick1}
\end{align}
Thus, we can run PCG on $S$ with the same computational effort per iteration as PCG on $H$, while reaping the benefits of a more powerful preconditioner. In fact, we do not even need to compute $H$, \eqref{eq:schurtrick1} can be implemented using just the columns of $J$.

Equation~\eqref{eq:schurtrick1} is closely related to {\em Domain Decomposition methods} for solving large linear systems that arise in structural engineering and partial differential equations. In the language of Domain Decomposition, each point in a bundle adjustment problem is a domain, and the cameras form the interface between these domains. The iterative solution of the Schur complement then falls within the sub-category of techniques known as Iterative Sub-structuring~\cite{saad2003iterative,mathew2008domain}.

For bundle adjustment problems, particularly those arising in reconstruction from community photo collections, more effective preconditioners can be constructed by analyzing and exploiting the camera-point visibility structure of the scene~\cite{kushal2012}. Ceres implements the two visibility based preconditioners described by Kushal \& Agarwal as \texttt{CLUSTER\_JACOBI} and \texttt{CLUSTER\_TRIDIAGONAL}. These are fairly new preconditioners and Ceres' implementation of them is in its early stages and is not as mature as the other preconditioners described above.

\section{\texttt{CGNR}}
For general sparse problems, if the problem is too large for \texttt{CHOLMOD} or \texttt{SuiteSparse} is not linked into Ceres, another option is the \texttt{CGNR} solver. This solver uses the Conjugate Gradients solver on the {\em normal equations}, but without forming the normal equations explicitly. It exploits the relation
\begin{align}
	H x = J^\top J x = J^\top(J x)
\end{align}
Currently only the \texttt{JACOBI} preconditioner is available for use with this solver. It uses the block diagonal of $H$ as a preconditioner.

\section{Ordering}
All three of the Schur based solvers depend on the user indicating to the solver, which of the parameter blocks correspond to the points and which correspond to the cameras. Ceres refers to them as \texttt{e\_block}s and \texttt{f\_blocks}. The only constraint on \texttt{e\_block}s is that there should be no term in the objective function with two or more \texttt{e\_block}s.

As we saw in Section~\ref{sec:tutorial:bundleadjustment}, there are two ways to indicate \texttt{e\_block}s to Ceres. The first is to explicitly create an ordering vector \texttt{Solver::Options::ordering} containing the parameter blocks such that all the \texttt{e\_block}s/points occur before the \texttt{f\_blocks}, and setting \texttt{Solver::Options::num\_eliminate\_blocks} to the number \texttt{e\_block}s.

For some problems this is an easy thing to do and we recommend its use. In some problems though, this is onerous and it would be better if Ceres could automatically determine \texttt{e\_block}s. Setting \texttt{Solver::Options::ordering\_type} to \texttt{SCHUR} achieves this.

The \texttt{SCHUR} ordering algorithm is based on the observation that
the constraint that no two \texttt{e\_block} co-occur in a residual
block means that if we were to treat the sparsity structure of the
block matrix $H$ as a graph, then the set of \texttt{e\_block}s is an
independent set in this graph. The larger the number of
\texttt{e\_block}, the smaller is the size of the Schur complement $S$. Indeed the reason Schur based solvers are so efficient at solving bundle adjustment problems is because the number of points in a bundle adjustment problem is usually an order of magnitude or two larger than the number of cameras.

Thus, the aim of the \texttt{SCHUR} ordering algorithm is to identify the largest independent set in the graph of $H$. Unfortunately this is an NP-Hard problem. But there is a  greedy approximation algorithm that performs well~\cite{li2007miqr} and we use it to identify \texttt{e\_block}s in Ceres.

\section{Automatic Differentiation}
TBD
\section{Loss Function}
TBD
\section{Local Parameterizations}
TBD


% Created 2021-08-24 Tue 19:02
% Intended LaTeX compiler: pdflatex
\documentclass[presentation,aspectratio=169]{beamer}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{grffile}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{textcomp}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\usepackage{khpreamble}
\usepackage{amssymb}
\usepgfplotslibrary{groupplots}
\newcommand*{\shift}{\operatorname{q}}
\usetheme{default}
\author{Kjartan Halvorsen}
\date{\today}
\title{Process Automation Laboratory - Modeling first-order systems}
\hypersetup{
 pdfauthor={Kjartan Halvorsen},
 pdftitle={Process Automation Laboratory - Modeling first-order systems},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 26.3 (Org mode 9.4.6)},
 pdflang={English}}
\begin{document}

\maketitle

\section{Fitting first-order model}
\label{sec:org2a0f0ec}
\begin{frame}[label={sec:org01d4e0f}]{First-order system example: Level control of a tank}
\begin{center}
\includegraphics[width=0.7\linewidth]{../../figures/tank-with-hole-no-variables}
\end{center}

What is the \alert{input signal} and \alert{output signal} to the system?
\end{frame}

\begin{frame}[label={sec:orga76c6fa}]{First-order system example: Level control of a tank}
\begin{center}
\includegraphics[width=0.7\linewidth]{../../figures/tank-with-hole-simple}
\end{center}

\begin{align*}
\frac{d}{dt} (Ah) &=  z(t) - x(t) = z(t) - a \sqrt{2gh}\quad \Rightarrow\\
\frac{d}{dt} h(t) &= - \frac{a\sqrt{2g}}{A} \sqrt{h(t)} + \frac{1}{A} z(t)
\end{align*}
\end{frame}

\begin{frame}[label={sec:orgeaac400}]{Intuition}
\includegraphics[width=0.2\linewidth]{../../figures/tank-with-hole-no-variables}

\alert{Individual activity} A constant inflow has been present since forever, but at time \(t_1\) the flow in is suddenly shut off. Which of the responses of the water level \(h(t)\) below is correct?

\begin{tikzpicture}
\small

\begin{axis}[
width=7cm,
height=2.5cm,
xlabel={$t$},
ylabel={$h(t)$},
xmin=-3.5,
xmax=10.5,
ytick = {0},
xtick = {0},
xticklabels = {$t_1$},
]
\addplot+[black, no marks, domain=-4:10, samples=400,variable=k] { (k < 0) + (k>0)*(1+exp(-4))/(1+exp(4*(0.5*k-1)))};

\node[black!40!red] at (axis cs: 5, 0.5) {\huge 1};
\end{axis}

\begin{axis}[
xshift=7cm,
width=7cm,
height=2.5cm,
xlabel={$t$},
ylabel={$h(t)$},
xmin=-3.5,
xmax=10.5,
ytick = {0},
xtick = {0},
xticklabels = {$t_1$},
]
\addplot+[black, no marks, domain=-4:10, samples=400,variable=k] { (k<0) + ((k>=0) - (k>4))*(1/4*(4-k)) };
\node[black!40!red] at (axis cs: 5, 0.5) {\huge 2};
\end{axis}

\begin{axis}[
xshift=0cm,
yshift=-2.5cm,
width=7cm,
height=2.5cm,
xlabel={$t$},
ylabel={$h(t)$},
xmin=-3.5,
xmax=10.5,
ytick = {0},
xtick = {0},
xticklabels = {$t_1$},
]
\addplot+[black, no marks, domain=-4:10, samples=400,variable=k] { (k<0) + (k>0)*exp(-0.9*k)};
\node[black!40!red] at (axis cs: 5, 0.5) {\huge 3};
\end{axis}

\begin{axis}[
xshift=7cm,
yshift=-2.5cm,
width=7cm,
height=2.5cm,
xlabel={$t$},
ylabel={$h(t)$},
xmin=-3.5,
xmax=10.5,
ytick = {0},
xtick = {0},
xticklabels = {$t_1$},
]
\addplot+[black, no marks, domain=-4:10, samples=400,variable=k] { (k<0) + ((k>=0) - (k>4))*(1-1/16*pow(-k,2)) };
\node[black!40!red] at (axis cs: 5, 0.5) {\huge 4};
\end{axis}

\end{tikzpicture}
\end{frame}

\begin{frame}[label={sec:orgc81a87f}]{Deviation variables}
\begin{center}
\includegraphics[width=0.7\linewidth]{../../figures/tank-with-hole}
\end{center}

Flow in: \(z(t) = z_0 + w(t)\). Level of water: \(h(t) = h_0 + y(t)\). The constants \(h_0\) and \(z_0\) define an \emph{operating point}.

\begin{align*}
\frac{d}{dt} h(t) &= - \frac{a\sqrt{2g}}{A} \sqrt{h(t)} + \frac{1}{A} z(t)
\end{align*}

\alert{Individual activity} Given \(h_0\) determine the operating point for the inflow, \(z_0\), such that the system is in equilibrium at the operating point.
\end{frame}

\begin{frame}[label={sec:org5f0b24a}]{Intuition}
Which change \(y(t)\) in the water level corresponds to a step change \(w(t)\) in the inflow?

\begin{center}
\includegraphics[width=0.7\linewidth]{../../figures/dc-response-exercise}
\end{center}
\end{frame}

\begin{frame}[label={sec:orga6c9c55}]{Fitting a first-order model}
Assuming a plant model of first-order with time-constant \(T\)
\[  \quad \textcolor{green!50!black}{Y(s)} = \frac{K}{sT + 1}\textcolor{blue!80!black}{U(s)} \quad \overset{U(s) = \frac{u_f}{s}}{\Longrightarrow} \quad \textcolor{green!50!black}{y(t)} = u_f K\big( 1 - \mathrm{e}^{-\frac{t}{T}}\big)u_H(t)\]
\def\Tcnst{3}
\def\tdelay{0.0}
\def\ggain{2}
\def\uampl{0.8}
\pgfmathsetmacro{\yfinal}{\uampl*\ggain}
\pgfmathsetmacro{\yone}{0.283*\yfinal}
\pgfmathsetmacro{\ytwo}{0.632*\yfinal}
\pgfmathsetmacro{\tone}{\tdelay + \Tcnst/3}
\pgfmathsetmacro{\two}{\tdelay + \Tcnst}

\begin{center}
  \begin{tikzpicture}
    \begin{axis}[
    width=14cm,
    height=4.5cm,
    grid = both,
    xtick = {0,  \two},
    xticklabels = {0, $T$},
    ytick = {0, \ytwo, \uampl, \yfinal},
    yticklabels = {0,  $ $, $u_f$, $y_f$},
    xmin = -0.2,
    %minor y tick num=9,
    %minor x tick num=9,
    %every major grid/.style={red, opacity=0.5},
    xlabel = {$t$},
    ]
      \addplot [thick, green!50!black, no marks, domain=0:10, samples=100] {\uampl*\ggain*(x>\tdelay)*(1 - exp(-(x-\tdelay)/\Tcnst)} node [coordinate, pos=0.9, pin=-90:{$y(t)$}] {};
      \addplot [const plot, thick, blue!80!black, no marks, domain=-1:10, samples=100] coordinates {(-1,0) (0,0) (0,\uampl) (10,\uampl)} node [coordinate, pos=0.9, pin=-90:{$u(t)$}] {};
    \end{axis}
  \end{tikzpicture}
\end{center}

\alert{Individual activity} Evaluate the response \(y(t)\) at the time instant \(t=T\) and for \(t\to\infty\)!
\end{frame}

\begin{frame}[label={sec:org41c055a}]{Fitting a first-order model}
Assuming a plant model of first-order with time-constant \(T\)
\[  \quad \textcolor{green!50!black}{Y(s)} = \frac{K}{sT + 1}\textcolor{blue!80!black}{U(s)} \quad \overset{U(s) = \frac{u_f}{s}}{\Longrightarrow} \quad \textcolor{green!50!black}{y(t)} = u_f K\big( 1 - \mathrm{e}^{-\frac{t}{T}}\big)u_H(t)\]
\def\Tcnst{3}
\def\tdelay{0.0}
\def\ggain{2}
\def\uampl{0.8}
\pgfmathsetmacro{\yfinal}{\uampl*\ggain}
\pgfmathsetmacro{\yone}{0.283*\yfinal}
\pgfmathsetmacro{\ytwo}{0.632*\yfinal}
\pgfmathsetmacro{\tone}{\tdelay + \Tcnst/3}
\pgfmathsetmacro{\two}{\tdelay + \Tcnst}

\begin{center}
  \small
  \begin{tikzpicture}
    \begin{axis}[
    width=14cm,
    height=3.5cm,
    grid = both,
    xtick = {0,  \two},
    xticklabels = {0, $T$},
    ytick = {0, \ytwo, \uampl, \yfinal},
    yticklabels = {0,  $0.632y_f$, $u_f$, $y_f$},
    xmin = -0.2,
    %minor y tick num=9,
    %minor x tick num=9,
    %every major grid/.style={red, opacity=0.5},
    xlabel = {$t$},
    ]
      \addplot [thick, green!50!black, no marks, domain=0:10, samples=100] {\uampl*\ggain*(x>\tdelay)*(1 - exp(-(x-\tdelay)/\Tcnst)} node [coordinate, pos=0.9, pin=-90:{$y(t)$}] {};
      \addplot [const plot, thick, blue!80!black, no marks, domain=-1:10, samples=100] coordinates {(-1,0) (0,0) (0,\uampl) (10,\uampl)} node [coordinate, pos=0.9, pin=-90:{$u(t)$}] {};
    \end{axis}
  \end{tikzpicture}
\end{center}

\alert{Time-constant:} Find the time \(t=T\) at which the response has reached 63.2\% of its final value

\alert{Gain:} \(y_f = \lim_{t\to\infty}y(t) = Ku_f \quad \Rightarrow \quad K = \frac{y_f}{u_f}\)
\end{frame}

\section{The lab activity}
\label{sec:org6acf095}

\begin{frame}[label={sec:orged6ff34}]{About the tolerance of components and propagation of errors}
\setlength{\tabcolsep}{1cm}

\begin{center}
\begin{tabular}{cc}
\includegraphics[width=3cm]{../../figures/resistor-color-code-4-band.png} & \includegraphics[width=3cm]{../../figures/capacitor.jpg}\\
\(R = R_0 \pm \Delta R\) & \(C = C_0 \pm \Delta C\)\\
\end{tabular}
\end{center}

\[\tau = RC = \tau_0 + \Delta\tau\]

\pause
Assume the tolerance for the resistor is \(\frac{\Delta R}{R_0}=\)5\% and for the capacitor \(\frac{\Delta C}{C_0}=\)20\%. What will the tolerance for the time-constant \(\frac{\Delta\tau}{\tau_0}\) be?
\setlength{\tabcolsep}{1cm}
\begin{center}
\begin{tabular}{cccc}
1. 5\% & 2. 20\% & 3. 25\% & 4. 100\%\\
\end{tabular}
\end{center}
\end{frame}

\begin{frame}[label={sec:org6db2167}]{About the tolerance of components amd propagation of errors}
\small
\begin{columns}
\begin{column}{0.3\columnwidth}
\begin{center}
 \includegraphics[width=.7\linewidth]{../../figures/resistor-color-code-4-band.png}
\end{center}

\begin{center}
\begin{tabular}{ll}
Color & Meaning\\
\hline
Brown & First digit 1\\
Black & Second digit 0\\
Orange & Multiply with \(10^3\)\\
Gold & Tolerance 5\%\\
\end{tabular}
\end{center}

\begin{align*}
R &= R_0 \pm \Delta R\\
&=10\, \text{k}\Omega \pm 5\% = (10 \pm 0.5)\, \text{k}\Omega
\end{align*}
\end{column}

\begin{column}{0.7\columnwidth}
\[ \tau = RC = R_0C_0 \pm \Delta\tau = \tau_0 \pm \Delta \tau \]

\pause

Two ways to calculate \(\Delta\tau\):
\footnotesize
\begin{enumerate}
\item Direct calculation \begin{align*} \tau &= RC = (R_0 + \Delta R)(C_0 + \Delta C)\\
   &= R_0C_0 + R_0\Delta C  + C_0\Delta R + \Delta R \Delta C\\
   &\approx \tau_0 + \underbrace{R_0\Delta C  + C_0\Delta R}_{\Delta\tau}
   \end{align*}
\item Total derivative \begin{align*} \Delta\tau &= \frac{\partial \tau}{\partial R} \Big|_{R_0, C_0} \Delta R + \frac{\partial \tau}{\partial C} \Big|_{R_0, C_0} \Delta C \\
&=C_0\Delta R + R_0\Delta C.
\end{align*}
\end{enumerate}

\small
\pause
\[\frac{\Delta\tau}{\tau_0} = \frac{C_0\Delta R + R_0\Delta C}{R_0C_0} = \frac{\Delta R}{R_0} + \frac{\Delta C}{C_0}\]
\end{column}
\end{columns}
\end{frame}

\begin{frame}[label={sec:orgd445274}]{About the tolerance of components and propagation of errors}
\setlength{\tabcolsep}{1cm}

\begin{center}
\begin{tabular}{cc}
\includegraphics[width=3cm]{../../figures/resistor-color-code-4-band.png} & \includegraphics[width=3cm]{../../figures/capacitor.jpg}\\
\(R = R_0 \pm \Delta R\) & \(C = C_0 \pm \Delta C\)\\
\end{tabular}
\end{center}

\[\tau = RC = \tau_0 + \Delta\tau\]

\pause
Assume the tolerance for the resistor is \(\frac{\Delta R}{R_0}=\)5\% and for the capacitor \(\frac{\Delta C}{C_0}=\)20\%. What will the tolerance for the time-constant \(\frac{\Delta\tau}{\tau_0}\) be?
\setlength{\tabcolsep}{1cm}
\begin{center}
\begin{tabular}{cccc}
1. 5\% & 2. 20\% & 3. 25\% & 4. 100\%\\
\end{tabular}
\end{center}
\end{frame}
\end{document}


%-------------------------------------------------------------------------------
% Uni-level
%-------------------------------------------------------------------------------

\subsection{Uni-level representation}

Note that the rules for types and kinds are very similar.
Pfenning's experience with implementing Twelf is that if they
are separate much of the code has to be duplicated for
the two levels.  These concerns lead to the final
grammar for Spine-Form LF, a single class of expressions for
terms, types, and kinds, classified by level.

$$
\begin{array}{llll}
\mathbf{Levels} & L & ::= & \Type \Spb \Kind \\
\mathbf{Expressions} & U & ::= & L \Spb \Pi(U_1,U_2) \Spb \lambda U \Spb H\cdot S \\
\mathbf{Heads} & H & ::= & c \Spb i\\
\mathbf{Spines} & S & ::= & \Nil \Spb U;S\\
\end{array}
$$


\documentclass{beamer}

\usetheme{uhh}
\showtotalframenumber
\showuhhlogoeachframe
\showsections

\usepackage{amsmath}
\DeclareMathOperator*{\argmin}{arg\,min}

\usepackage{listings}
\lstset{
language=python
}

\title{Tensorflow -- Regression Models}
\subtitle{Based on slides from Fabian Barteld, Benjamin Milde}
\author{ Fabian Barteld, Benjamin Milde, Prof. Dr. Chris Biemann}
\date[18.10.2018]{Oct 18, 2018}

\AtBeginSection[]
{
  %%%%% section title
  % This is how it would look like in Beamer:
  % \begin{frame}
  %     \frametitle{Overview}
  %     \tableofcontents[sections={2-3},currentsection,sectionstyle=show/hide,subsectionstyle=hide]
  % \end{frame}
\begin{frame}[plain]
\begin{tikzpicture}[overlay]
  \relax%
  \fill[blueuhh,opacity=1] (-10,-10)
  rectangle(\the\paperwidth,\the\paperheight);
\end{tikzpicture}
  \begin{tikzpicture}[overlay]
  \relax%
  \fill[white,opacity=1] (-5,-1.2)
  rectangle(\the\paperwidth,0.5) node[pos=0.5,black]{\LARGE\insertsectionhead};
\end{tikzpicture}
\end{frame}

%%%% add subsection to show navigation dots
\subsection{}
}

\begin{document}

\maketitle

\section{Linear Regression}

% https://medium.com/@saxenarohan97/intro-to-tensorflow-solving-a-simple-regression-problem-e87b42fd4845
% https://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/2_BasicModels/linear_regression.py
\begin{frame}
\frametitle{Linear Regression}

\begin{itemize}
\item Given: $(x_1, y_1)$, \ldots, $(x_n,y_n)$
\item Goal: find $w$ and $b$ such that
  \begin{displaymath}
    \hat{y}_i = wx_i + b
  \end{displaymath}
  fits the data, i.e.
  \begin{displaymath}
    \argmin_{w, b} \frac{\sum^n_{i=1} (\hat{y}_i - y_i)^2}{n}
  \end{displaymath}
\end{itemize}

\end{frame}

\begin{frame}[fragile]
\frametitle{Define model parameters}
Model: $\hat{y}_i = wx_i + b$\\
Parameters: $w$, $b$, tensors of rank $0$

\begin{lstlisting}
w = tf.Variable(tf.ones([]),
  name="weight")
b = tf.Variable(tf.zeros([]),
  name="bias")
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\frametitle{Define the model}

\begin{displaymath}
  \begin{pmatrix} \hat{y}_1\\\vdots\\\hat{y}_n\end{pmatrix} =
  \begin{pmatrix} x_1\\\vdots\\x_n\end{pmatrix} \odot
  \begin{pmatrix} w\\\vdots\\w\end{pmatrix} +
  \begin{pmatrix} b\\\vdots\\b\end{pmatrix}
\end{displaymath}

\begin{lstlisting}
yhat = tf.add(tf.multiply(X, w), b)
\end{lstlisting}

{\footnotesize The scalars $w$ and $b$ are converted into vectors of the same
length as X (broadcast); \url{https://www.tensorflow.org/performance/xla/broadcasting}}

\end{frame}

\begin{frame}[fragile]
\frametitle{Define the loss}

\begin{displaymath}
  \frac{\sum^n_{i=1} (\hat{y}_i - y_i)^2}{n}
\end{displaymath}

\begin{lstlisting}
loss = tf.reduce_mean(tf.square(yhat - Y))
\end{lstlisting}

\end{frame}

\begin{frame}[fragile]
\frametitle{Optimization}

\begin{lstlisting}
## Optimizer
optimizer = tf.train.GradientDescentOptimizer(
  0.01 # learning rate
  ).minimize(loss)

with tf.Session() as sess:
  ## initalize parameters
  sess.run(tf.global_variables_initializer())

  for i in range(20):
      ## run one epoch
      sess.run(optimizer)
\end{lstlisting}

\end{frame}

\begin{frame}[fragile]
\frametitle{Hands on}

Do a linear regression to learn $y = 2x + 1$

\begin{lstlisting}
X_data = np.array([1., 2., 3., 4., 5., 6.],
      dtype=np.float32).reshape(6, 1)
Y_data = 2*X_data + 1
\end{lstlisting}
\end{frame}

\section{Multiple Linear Regression}

\begin{frame}[fragile]
\frametitle{Defining the input}

Tensorflow graphs use placholders for input values

\begin{lstlisting}
input_dim = 13

X = tf.placeholder(tf.float32, [None, input_dim])
Y = tf.placeholder(tf.float32, [None, 1])
\end{lstlisting}

Defines placeholders for two tensors of rank 2,\\
the shape is [Number of examples, Dimension]
\end{frame}

\begin{frame}[fragile]
\frametitle{Adapting the model}

\begin{displaymath}
  \begin{pmatrix} \hat{y}_1\\\vdots\\\hat{y}_n\end{pmatrix} =
  \begin{pmatrix} x_{1,1} & \hdots & x_{1,input\_dim}\\\vdots && \vdots\\x_{n,1} & \hdots & x_{n,input\_dim}\end{pmatrix} \times
  \begin{pmatrix} w_1 \\\vdots\\w_{input\_dim}\end{pmatrix} +
  \begin{pmatrix} b\\\vdots\\b\end{pmatrix}
\end{displaymath}

\begin{lstlisting}
w = tf.Variable(tf.ones(input_dim))
yhat = tf.add(tf.matmul(X, w), b)
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\frametitle{Getting data into the model}

\begin{lstlisting}
## Optimizer
optimizer = tf.train.GradientDescentOptimizer(
  0.01 # learning rate
  ).minimize(loss)

with tf.Session() as sess:
  ## initalize parameters
  sess.run(tf.global_variables_initializer())

  for i in range(20):
      ## run one epoch
      sess.run(optimizer, {X: data_X, Y: data_Y})
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\frametitle{Hands on}

Do a multiple linear regression with Boston housing prices

\begin{lstlisting}
from sklearn.datasets import load_boston
from sklearn.preprocessing import scale

data_X, data_Y = load_boston(True)
data_X = scale(data_X)
data_Y = data_Y.reshape(len(data_Y), 1)
\end{lstlisting}
\end{frame}

\section{Logistic regression}

% https://www.tensorflow.org/tutorials/wide
% https://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/2_BasicModels/logistic_regression.py
\begin{frame}[fragile]
\frametitle{Multiple Logistic regression}

$p_i = S(WX_i + b)$

Loss (Binary-crossentropy):

\[\frac{1}{N}\sum_{i=1}^N (y_i\log{p_i} + (1-y_i)(\log{1-p_i}))\]

In tensorflow:\\
\footnotesize{\textcolor{reduhh}{Don't use -- numerical problems!}}

\begin{lstlisting}
p = tf.sigmoid(yhat)
loss = tf.reduce_mean(y*tf.log(p) + (1-y)*tf.log(1-p))
\end{lstlisting}

\pause

Optimized version (\footnotesize{\textcolor{green}{Use this instead!}})
\begin{lstlisting}
loss = tf.reduce_mean(
  tf.nn.sigmoid_cross_entropy_with_logits(
      labels=y, logits=yhat))
\end{lstlisting}

\end{frame}

\begin{frame}[fragile]
\frametitle{Scaling the input data}

\begin{lstlisting}
from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()
scaler.fit(x_train)
x_train = scaler.transform(x_train)
x_test = scaler.transform(x_test)
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\frametitle{Hands on: Binary classification}

Dataset: \url{http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_breast_cancer.html}

\begin{lstlisting}
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split

## load the data
bc = load_breast_cancer()
x_data = bc['data']       # shape: (569,30)
y_data = bc['target'].reshape(
  len(bc['target']), 1) # shape: (569, 1)

x_train, x_test, y_train, y_test =
  train_test_split(x_data, y_data)

\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
\frametitle{One-hot encoding of nominal features}
Names dataset
\url{http://www.nltk.org/book/ch06.html}

\begin{lstlisting}
def gender_features(word):
  return {'last_letter': word[-1]}

def gender_features(word):
  return {'suffix1': word[-1:],
          'suffix2': word[-2:]}
\end{lstlisting}
\pause
\vspace{-1.5ex}

\begin{lstlisting}
from sklearn.feature_extraction import DictVectorizer
feat_vectorizer = DictVectorizer(
  dtype=numpy.int32, sparse=False)
train_X = feat_vectorizer.fit_transform(
  train_feats)
test_X = feat_vectorizer.transform(test_feats)
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Stochastic gradient descent}

\begin{lstlisting}
with tf.Session() as sess:
    ## initalize parameters
    sess.run(tf.global_variables_initializer())

    for i in range(20):
        ## run one epoch
        ## update for each training example
        for x, y in zip(x_data, y_data):
            sess.run(optimizer, {X: x, Y: y})
\end{lstlisting}

\pause
Usually the data is shuffled and\\ passed in small batches to the optimizer.

\end{frame}

\begin{frame}[fragile]
\frametitle{Hands on}

Fit a logistic regression model to the names dataset

\url{http://www.nltk.org/book/ch06.html}
\begin{lstlisting}
import nltk
## names must be installed by running
## nltk.download('names')
from nltk.corpus import names
import random

labeled_names = (
  [(n, 0) for n in names.words('male.txt')] +
  [(n, 1) for n in names.words('female.txt')])
random.shuffle(labeled_names)

\end{lstlisting}
\end{frame}

\end{document}

%%% Local Variables:
%%% mode: latex
%%% TeX-engine: luatex
%%% End:


\documentclass[letterpaper]{article}

\usepackage{fullpage}
\usepackage{nopageno}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage[utf8]{inputenc}
\allowdisplaybreaks

\newcommand{\abs}[1]{\left\lvert #1 \right\rvert}

\begin{document}
\title{Notes}
\date{November 14, 2014}
\maketitle
\section*{5.7 monotone functions}
a function $f:\mathbb{R}\to\mathbb{R}$ is monotonic (strictly) increasing (decreasing) if $\forall x<y, f(x)\le f(y)$ changing $f(x)$ and $f(y)$ relation as necessary

note $\mathbb{R}^n$ has no natural order, so this is only defined in $\mathbb{R}$

\subsection*{proposition 5.7.2}
if $f:(a,b)\to \mathbb{R}$ is increasing then $\alpha=\lim_{x\to a^+}f(x)$ and $\lim_{x\to b^-}f(x)$ exists and $\forall x\in (a,b)$ we have $\alpha\le f(x)\le \beta$. and every element in $(a,b)$ has a left and right limit
\subsubsection*{proof}
let $c\in(a,b)$. let $F:\{f(x):x\in(a,c)\}$. because $f$ is increasing and $x<c$, then $f(x)\le f(c)\forall x\in(a,c)$. therefore $f(c)$ is an upper bound  for $F$. so $F$ has a supremum $L$. also $L\le f(c)$ because $f(c)$ is an upper bound and $L$ is the least upper bound. since $L-\varepsilon$ is not an upper bound for $F$ then $\exists y\in(a,c)$ such that $L-\varepsilon\le f(y)\le L$. take $\varepsilon=\frac{1}{n}$, for each $\varepsilon=\frac{1}{n}$, a corresponding $y_n\in(a,c)$. $L-\frac{1}{n}<f(y_n)\le L\to |f(y_n)-L|<\frac{1}{n}$ and so the limit exists.
\subsection*{corollary}
monotonic functions have only jump disccontinuities. and the number of these continuities is countable
\subsubsection*{proof}
 by 5.7.2 if $f$ has a discontinuity at a point $c\in(a,b)$ since $\lim_{x\to c^+}f(x)$ and $\lim_{x\to c^-}f(x)$ exists, they must be different. thus the discontinuity is a jump discontinuity. (if the limits were equal, then it would be continuous at that point)

let $c$ be a point where $f$ has a discontinuity. then wlog $f(x)$ is increasing. $\gamma_1\lim_{x\to c^-}f(x)<\lim_{x\to c^+}=\gamma_2$. to the discontinuity point $c$ we can associate the interval $(\gamma_1,\gamma_2)$ which  is not in the image of $f$. if $d\ne c$ is another point of discontinuity, it's corresponding interval $(\sigma_1,\sigma_2)$ can not intersect $(\gamma_1,\gamma_2)$. if wlog $c\le d$ then $\gamma_2=\lim_{x\to c^+}\le\lim_{x\to d^-}f(x)=\sigma_1$ and so $\gamma_2<\sigma_1$ and they don't intersect.

let $F:\{\text{discontinuities}\}\to\mathbb{Q}$. then $c\to c'\in(\gamma_1,\gamma_2)$. $F$ is injective because the intervals are disjoint, $|F|\le|\mathbb{Q}|$

\section*{example 5.7.8 cantor function}

in general the limit of a sequence of continous functions is not continuous. $f_n(x)=\begin{cases}0&x\le0\\x^n&x\in[0,1]\\1&x\ge 1\end{cases}$

in the limt if $x\in[0,1)$ then $f_n(x)=x^n\rightarrow0$ but $f_n(1)=1$.
\end{document}


\chapter{Optimizing network-utilization by balancing workload}
\label{chap:balancing}

The last chapter brings an overview of different notations, algorithms and techniques.
In this chapter, all these instruments are combined to a chain for distributing routes more evenly.
The goal is to augment a given graph, such that diverse alternative paths can be found and used to spread too much load while limiting the detours of individual \glspl{stpair}.
After the theory is explained, implementation-details are discussed.
When talking about performance, it is referred to the used testing-machine, which is an ordinary, but good home-computer.
Further details about the testing-machine and experimental results can be found in \vref{chap:experiments}.

For a given \gls{stpair} and a street-network given as a graph $G = (V, E)$ with vertices $V$, edges $E$ and a cost-function $c: E \to \mathbb{R}_+^d$, that returns a cost-vector for each edge $e \in E$, a path should be found through an algorithm, that implicitly tends to distribute found paths over the network, while keeping the paths' costs sufficiently near every cost-dimension's optimum.
Note, that $\mathbb{R}_+^d$ doesn't contain zero, which would, depending on the metric, result in bad alternative routes otherwise.

To achieve this, a computation-phase called balancing analyzes the graph $G$ and computes a new metric, called workload-metric.
With this workload-metric, routing-algorithms like \gls{repr} and Dijkstra (with personalized routing) can compute paths as usual.
For comparison, multiple scenarios using different combinations of these two routing-algorithms will be used.
Please note, that Dijkstra doesn't guarantee a tolerance when used with personalized routing and hence, some combinations don't provide an upper bound for path-costs.
This will also be pointed out in the respective experiments in \cref{chap:experiments}.

This thesis focusses only on distance and travel-time, where travel-time is the only metric with a provided tolerance.

It should be noted, that Dijkstra (bidirectional) instead of A* is used for every occuring routing-query, may it be in \gls{repr} or anywhere else.
The reason is, that A* works with heuristics, which are difficult to generalize over custom, artificial metrics.
The only benefit from using A* is the reduction of the search-space, which can be reduced much more using bidirectional Dijkstra on graphs contracted via contraction-hierarchies.
Besides that, the graph has multidimensional metrics, for which reason Dijkstra is used with personalized routing.

\section{Balancing to create a new metric}

    This balancing is depicted in \vref{fig:balancing:algorithm} and explained briefly in the caption or detailled in the following.
    It basically runs routing-queries, counts the resulting workload and adjusts a third metric (in addition to travel-distance and travel-time) to influence future query-executions.
    After the new workload-metric will have been settled, routing-algorithms for multidimensional metrics can run queries on the balanced graph $G$ as before, now considering the new workload-metric.
    Therefore, queries will be run in the experiments (\cref{chap:experiments}) to evaluate the new spread of the street-network.
    The set of \glspl{stpair} will be chosen differently than the set used in balancing, to reduce the dependence on it.

    \begin{figure}
        \centering
        \input{resources/graphics/balancing/algorithm}
        \caption[Overview of balancing a graph]{%
            This flow shows the balancing, that analyzes a given graph $G$.
            Here, an evaluation-phase is included afterwards.
            Shapes for actions are rectangular and blue, shapes for data are elliptical and red.
            A new, artificial metric is created for $G$ and gets updated iteratively.
            In the end, this new metric allows a routing-algorithm to distribute upcoming workload (like during rush-hour) more evenly over the underlying street-network in $G$.
            This more spreaded workload can be seen in the evaluation-step, where paths for a set of \glspl{stpair} are searched, once using and once ignoring the new metric.

            Note, that the action doing the path-searches ignores the new metric in the first balancing-iteration.
            \label{fig:balancing:algorithm}
        }
    \end{figure}

    \subsection{Selection of route-pairs}

        Let a street-network be given as a graph $G = (V, E)$ as defined in \cref{chap:preliminaries:graphs}.
        Before such a graph can be balanced, a set of \glspl{stpair} has to be chosen.
        Here, different strategies can be applied.
        For simplicity, the set of \glspl{stpair}, which are used for balancing $G$, might consist of $s$ and $t$ chosen \gls{uar}\ from $V$.
        This has the disadvantage, that a sufficiently large set of \glspl{stpair} is needed to overload $G$.
        This clearly becomes a bottleneck for performance on larger graphs (beginning with small German counties like Saarland with almost $|V| \approx \num{580000}$ and $|E| \approx \num{1160000}$ after parsing), especially when using \gls{repr}, that computes multiple Dijkstra-queries per run.
        To alleviate the computational burden of a large number of \glspl{stpair}, knowledge of the typical workloads in this graph could be used.
        Instead of picking \gls{uar}\ from $|V|$, other distributions could be chosen by weighting $v \in V$ differently or by using some \glspl{stpair} multiple times (on purpose, not only by random).
        For example, real, daily, critical \glspl{stpair} are not distributed \gls{uar}\ over $V$.
        The rush-hour in the morning, when everybody travels from their home to work, is a good counterexample here.
        The set of chosen $t$ might be much smaller than the set of chosen $s$, and less people live near industry or motorways.
        Heuristics as described in~\cite{bakillah:population_from_osm} can return \glspl{stpair} based on approximating population-data from street-networks.
        So considering more specific sets of \glspl{stpair} might be interesting and helpful to boost the balancing up, but is not covered in this thesis.
        The reason for keeping the choice \gls{uar}\ is the easy implementation, while results are sufficiently good enough, as long as $G$ shows differently (over-) loaded streets.

    \subsection{First steps of the balancing-process}
    \label{chap:balancing:balancing}

        At first, the graph $G$, that should be balanced, has to be read in.
        Before continuing, every metric is divided by the respective metric-mean resulting in $G = (V, E)$ of normalized metrics.
        Non-normalized metrics of different scales would cause \gls{repr} to find $\alpha$-values of different scale, making some metrics more important than others just based on their different scales.
        With metrics of different importance, the chosen paths tend to the respective metrics and distort the spread.
        So normalization doesn't just allow to compare different graphs (of different metrics), but also improves the spread in \gls{repr}.

        This graph $G$ is contracted via contraction-hierarchies to $G_{CH}$.
        The step of contracting is not needed for correctness, but reduces the needed runtime (even if $G$ is not being fully contracted).
        For this reason, $G$ instead of $G_{CH}$ is used in the following, to emphasize, that contraction is not necessary from a theoretical point of view.

        Given the \glspl{stpair} and a graph $G$, a path in $G$ is searched for every \gls{stpair}.
        The search may use either Dijkstra or \gls{repr}.
        Multiple paths are found by \gls{repr}, so one of them is chosen \gls{uar}\ after filtering all found paths by a given tolerance.
        Note, that Dijkstra with personalized routing doesn't consider any tolerance.
        But since \gls{repr} calls Dijkstra multiple times, just Dijkstra is faster.
        On the other hand, when using \gls{repr} with choosing a path \gls{uar}, the resulting paths are more spreaded over the network.
        This results in a better coverage of $E$ when updating the workload-metric, which allows the \gls{repr} to find more pareto-optimal personalized routes in the convex hull.
        Both methods are compared in the experiments in \vref{chap:experiments}.

        When using \gls{repr}, the quality of the balanced graph highly depends on the choice of initial paths for \gls{repr}.
        The basic approach in~\cite{barth:alternative_multicriteria_routes} suggests to use $(d+1)$ initial alphas to define one initial convex hull for a graph of $d$-dimensional metrics.
        \begin{equation}
        \label{eq:init_alphas}
        \begin{aligned}
            \left( \alpha^{(1)}, \alpha^{(2)}, \dots, \alpha^{(d)} \right) &= \mathbb{I}^d\\
            \alpha^{(d+1)} &= \left( \frac{1}{d}, \dots, \frac{1}{d} \right)^T
        \end{aligned}
        \end{equation}
        This suggestion is nice to get a tolerance, because every metric's optimal path is computed.
        However, in the balancing, this suggestion needs some adjustment.
        When the new workload-metric is added (hence let the graph's metrics' dimension be $d+1$), $\alpha^{(d+1)}$ becomes $\alpha^{(d+2)}$, which is a different direction in the $(d+1)$-cost-space than the previous $\alpha^{(d+1)}$ points to in the $(d+1)$-cost-space.
        For example (considering only \alpha's direction, not its length), let $d=2$ become $d=3$, hence $\alpha^3=(1, 1) \in \mathbb{R}_+^2$ becomes $\alpha^4=(1, 1, 1) \in \mathbb{R}_+^3$ and respectively $(1, 1, 0) \in \mathbb{R}_+^3$ from the previous case $d=2$ is missing now.
        In the case of Saarland in the experiments, this lead to the issue, that the initially found paths (as suggested in~\cite{barth:alternative_multicriteria_routes}) weren't enough to get an initial convex hull.
        Because of this, less paths have been found on Saarland in the experiments.
        Therefore, the balancing extends the original suggestion by~\cite{barth:alternative_multicriteria_routes} and takes the initial $\alpha^{(j)}$ from $\left\{ 0, 1 \right\}^{(d+1)}$ excluding $(0, \dots, 0)^T$.
        Only the direction of $\alpha$ is relevant, so the previous notation in \cref{eq:init_alphas} can be simplified.
        \begin{equation}
            \label{eq:new_init_alphas}
            \alpha \in \left\{ 0, 1 \right\}^{(d+1)} \setminus \left\{ \left( 0, \dots, 0 \right)^T \right\} \Rightarrow \alpha \text{ is an initial } \alpha^{(j)}
        \end{equation}

        After finding a set of paths for the given \glspl{stpair}, every edge $e \in E$ counts the number of found paths, that contain this edge $e$.
        This workload of balancing-iteration $i$ is normalized by all workloads' mean and the result is used to update the graph's workload-metric.
        \begin{equation}
        \label{eq:workloads}
        \begin{aligned}
            &\mathit{workloads}_{e,i} = \left| \left\{ (s, t) \in \text{\glspl{stpair}} : \text{found path from $s$ to $t$ uses } e \in E \right\} \right|\\
            \Rightarrow\ &\mathit{new}_{e,i} = \frac{\mathit{workloads}_{e,i}}{\mathit{mean}(\mathit{workloads}_i)}
        \end{aligned}
        \end{equation}

    \subsection{Updating the new metric with the collected workload}
    \label{chap:balancing:update}

        Updating the old workload-metric is a very small step in the balancing with a large impact on the ongoing balancing's performance and the quality of the balanced graph.
        Two approaches have been tested and are depicted in the following, after talking about the interpretation of the new artificial workload-metric.

        The workload-metric can be associated with popularity, because it refers to high usage.
        A path of high popularity should be chosen less often than without the workload-metric.
        So, hopefully, updating the workload-metric dependent on the current workload-metric reveals the final, balanced state of the network eventually.
        You could suggest, that a street's capacity should be considered as well, because large streets like motorways can support more vehicles than a street in a city and thus should tolerate a higher popularity than smaller streets.
        A street-capacity could be approximated using a street's number of lanes or the respective street-type (\eg\ motorways) and its length.
        Herewith, the workload-metric, penalizing popularity, would be corrected in favor of larger streets with (usually) higher speed-limit.
        That is exactly contraproductive, because the travel-time does already consider the capacity of streets with the speed-limit.
        In other words, the popularity-penalization should compensate the popularity of larger streets, that have a better travel-time.
        This wished effect would be disturbed by the consideration of capacity.

        The first approach is inspired by a simpler numerical method for solving initial-value-problems called explicit Euler.
        Let $\mathit{old}_i$ be the workload-metric (already normalized after $G$ has been initialized) already stored in the graph and $\mathit{new}_i$ be the new workload-metric (normalized by their mean, see \vref{eq:workloads}).
        \begin{equation}
        \label{eq:euler}
            \mathit{old}_{e,i+1} = \mathit{old}_{e,i} + (\mathit{new}_{e,i} - \mathit{old}_{e,i}) \cdot \mathit{correction}
        \end{equation}
        In each balancing-iteration, after the workload-metric is updated (as in \cref{eq:euler}), every value of the workload-metric smaller than a predefined minimum value $\epsilon$ (\eg\ \num{0.1}) is set to $\epsilon$.
        This removes the metrics of value \num{0.0}, which improves the set of computed shortcuts when applying contraction-hierarchies and thus performance.
        In addition, the result isn't normalized by its mean in general, for which reason it is finally normalized.
        \begin{equation}
        \label{eq:metric_cleanup}
        \begin{aligned}
            \mathit{old}_{e,i+1} &\leftarrow \mathit{max} \left( \epsilon \text{,\ } \mathit{old}_{e,i+1} \right)\\
            \mathit{old}_{e,i+1} &\leftarrow \frac{\mathit{old}_{e,i+1}}{\mathit{mean}(\mathit{old}_{i+1})}
        \end{aligned}
        \end{equation}

        In fact, this approach works okay in smaller, overloaded networks like Isle~of~Man ($|V| \approx \num{50000}$ and $|E| \approx \num{100000}$ after parsing), but lacks in convergence.
        In the first update-iteration, popular paths are ignored as much as possible, before they are highly preferred in the second iteration, ignored in the third iteration and so on.
        This behaviour could be reduced by adding the $\mathit{correction}$, but larger networks (like the small German county Saarland, $|V| \approx \num{580000}$ and $|E| \approx \num{1160000}$ after parsing) worsen the convergence.
        Further, adding $\mathit{correction}$ leads to the need of tuning it, for which reason only the second approach is shown in the experiments in \vref{chap:experiments}.

        Therefore, another approach, replacing the explicit Euler from \vref{eq:euler}, is presented in \vref{eq:averaging}, that does spread the workload well and within only two iterations.
        \begin{equation}
        \label{eq:averaging}
            \mathit{old}_{e,i+1} = \frac{i \cdot \mathit{old}_{e,i} + \mathit{new}_{e,i}}{i+1}
        \end{equation}
        This result is treated as before, meaning Equations~\ref{eq:metric_cleanup} are also applied.
        From \vref{eq:averaging}, both $\mathit{old}_{e,1}=\mathit{new}_{e,0}$ and $\mathit{old}_{e,2}=\frac{\mathit{old}_{e,1} + \mathit{new}_{e,1}}{2}$ hold before normalization.
        The interpretation of these is quite intuitive.
        The first iteration favors popular routes.
        After the first update, the workload-metric is just the normalized workload from the original, unbalanced graph.
        Hence the following iteration has an aversion to popular routes and tends to avoid them.
        This second iteration favors alternative routes and the outcoming workload completes the previously generated workload-metric.
        A third iteration has already a workload-metric with a balance between popular and alternative routes, so further iterations appear disruptive.

\section{Details on implementation}
\label{chap:balancing:implementation}

    The implementation has the goal to be efficient, easy to use and maintainable.
    For this reason, some important decisions were made and the most important ones are explained (from a theoretical point of view) in the following.

    \subsection{Configuration-files}

        Main concept of the implementation is the use of configuration-files.
        The whole parser, simulation and computation can be adjusted with these files.
        One big advantage are the parser-related settings to define the graphs nodes and edges.
        Both meta-data and metric-data is defined in the configuration-file.
        This allows the graph to be static, meaning its allocated memory doesn't change during runtime after the graph is built and shrinked to fit its needs.
        However, its values can be changed, which is necessary, since the balancing updates a metric.

    \subsection{Storing and accessing graphs}
    \label{chap:balancing:implementation:graphs}

        The underlying graph-structure is implemented as adjacency-array with several extensions.
        The adjacency-array is explained in \cref{chap:preliminaries:graphs}.
        This data-structure fits well, because it reduces redundancy by optimizing the graph's memory-consumption.
        At the same time, accessing the graph can be done in constant or logarithmic time (depending on access via indices or via ids) through indexing-logic.
        The indexing-logic from adjacency-arrays has been extended by a mapping to support forward-, backward- and shortcut-edges in the graph, which is necessary for the bidirectional-Dijkstra.

        First, the graph from preliminaries is extended by metrics.
        For every edge in $E$, a vector of floats is stored.
        To formalize this, the cost-function is slightly redefined (and duplicated edges are being removed) to have edge-costs being associated as metrics.
        Let $C \subset E \times \mathbb{R}_+^d$ be the set of metrics and $c$ be the bijective function $c: E \to C$.
        With this definition, $C[i_E]$ can be associated as the cost-vector of the edge $E$ at edge-index $i_E$, where $C$ is stored as two-dimensional array of floats.

        The vertices $V$ are an array of vertex-ids sorted in ascending order.
        This is required by the adjacency-array, but also allows the search for nodes by their ids in logarithmic time.
        However, this is only needed for users outside the graph, because inside, the graph uses only vertex-indices.

        A backward-graph is a graph, whose edges are reversed according to the original graph.
        So a backward-graph maps edges $(s, d)$ to the reversed edge $(d, s)$.
        Since this graph is used for bidirectional routing-queries, it has to support the access to both forward-edges and backward-edges.
        The backward-edges could be stored accordingly to the forward-edges, meaning storing the source-indices $S$ in addition to the destination-indices $D$.
        The occuring issue with this is the sort-order of vertices $V$.
        All vertices are sorted by their source-id to match the offset-array $O_D$ for the forward-edges and for $D$.
        Respectively, all vertices should be sorted by their destination-id to match the offset-array $O_S$ for the backward-edges and for $S$.
        This would need a copy of all vertices and an edge-index $i_E$ wouldn't point to the same edge in $S$ and $D$, leading to two access-implementations doing basically the same offset-mapping.
        The metric-array $C$ could be accessed via $i_E$ only by Dijkstra's forward-search, not the backward-search, unless a mapping from backward-edge-indices to forward-edge-indices is built (or $C$ is also stored twice).
        Therefore, this issue is solved by using this very mapping, such that an edge-index $i_E$ belongs to $S[i_E]$, $D[i_E]$, $C[i_E]$ independent of the edge's orientation and the graph can be stored without data-redundancy.
        The decision towards no data-redundancy implies consistency, which is important for the metric-update.
        Further, it improves maintainability and usage, because accessing the forward-edges and backward-edges can be done with the same accessor simply using different arrays.
        The indices-mappings are illustrated in \cref{table:balancing:indices_mapping} and explained afterwards with two example-queries.
        The graph from \cref{chap:preliminaries:graphs} contains only bidirectional edges, thus the edge $(s, d)$ with $(V[s], V[d])=(1, 2)$ is removed to make the graph containing unidirectional edges.

        \begin{table}[htbp]
            \centering
            \begin{tabular}{ l || c | c c | c c c | c c c | c c | c }
                $i_E = i_\mathit{fwd}$ & \multicolumn{12}{c}{Storing data with respect to forward-edges} \\
                \hline
                \hline
                $V$ & $\mathit{id}_0$ & \multicolumn{2}{c |}{$\mathit{id}_1$} & \multicolumn{3}{c |}{$\mathit{id}_2$} & \multicolumn{3}{c |}{$\mathit{id}_3$} & \multicolumn{2}{c |}{\cellcolor[gray]{0.8} $\mathit{id}_4$} & - \\
                \hline
                $S$ & \cellcolor[gray]{0.8} 0 & 1 & 1 & \cellcolor[gray]{0.8} 2 & 2 & 2 & \cellcolor[gray]{0.8} 3 & 3 & 3 & 4 & 4 & - \\
                $D$ & 1 & 0 & 3 & 1 & 3 & 4 & 1 & 2 & 4 & \cellcolor[gray]{0.8} 2 & \cellcolor[gray]{0.8} 3 & - \\
                $C$ & a & b & c & d & e & f & g & h & i & j & k & - \\
                & & & & & & & & & & & & \\
                $i_\mathit{fwd}$ & 0 & 1 & 2 & 3 & 4 & 5 & 6 & 7 & 8 & \cellcolor[gray]{0.8} 9 & \cellcolor[gray]{0.8} 10 & - \\
                $\mathit{map}_\mathit{bwd}: i_\mathit{fwd} \mapsto i_\mathit{bwd}$ & 1 & 0 & 6 & 2 & 7 & 9 & 3 & 4 & 10 & 5 & 8 & - \\
                & & & & & & & & & & & & \\
                $O_D: i_V \mapsto i_\mathit{fwd}$ & 0 & \multicolumn{2}{c |}{1} & \multicolumn{3}{c |}{3} & \multicolumn{3}{c |}{6} & \multicolumn{2}{c |}{\cellcolor[gray]{0.8} 9} & \cellcolor[gray]{0.8} 11 \\
                \multicolumn{13}{c}{} \\
            \end{tabular}
            \begin{tabular}{ l || c | c c c | c c | c c c | c c | c }
                $i_E = i_\mathit{bwd}$ & \multicolumn{12}{c}{Storing data with respect to backward-edges} \\
                \hline
                \hline
                $V$ & $\mathit{id}_0$ & \multicolumn{3}{c |}{\cellcolor[gray]{0.8} $\mathit{id}_1$} & \multicolumn{2}{c |}{$\mathit{id}_2$} & \multicolumn{3}{c |}{$\mathit{id}_3$} & \multicolumn{2}{c |}{$\mathit{id}_4$} & - \\
                \hline
                $D_\mathit{bwd}$ & 0 & 1 & 1 & 1 & 2 & 2 & 3 & 3 & 3 & 4 & 4 & - \\
                $S_\mathit{bwd}$ & 1 & 0 & 2 & 3 & 3 & 4 & 1 & 2 & 4 & 2 & 3 & - \\
                $C_\mathit{bwd}$ & b & a & d & g & h & j & c & e & k & f & i & - \\
                & & & & & & & & & & & & \\
                $\mathit{map}_\mathit{fwd}: i_\mathit{bwd} \mapsto i_\mathit{fwd}$ & 1 & \cellcolor[gray]{0.8} 0 & \cellcolor[gray]{0.8} 3 & \cellcolor[gray]{0.8} 6 & 7 & 9 & 2 & 4 & 10 & 5 & 8 & - \\
                $i_\mathit{bwd}$ & 0 & 1 & 2 & 3 & 4 & 5 & 6 & 7 & 8 & 9 & 10 & - \\
                & & & & & & & & & & & & \\
                $O_S: i_V \mapsto i_\mathit{bwd}$ & 0 & \multicolumn{3}{c |}{\cellcolor[gray]{0.8} 1} & \multicolumn{2}{c |}{\cellcolor[gray]{0.8} 4} & \multicolumn{3}{c |}{6} & \multicolumn{2}{c |}{9} & 11 \\
            \end{tabular}
            \caption[Indices-mapping for accessing graph-data via offset-arrays]{%
                This table shows one graph stored with respect to forward-edges above and with respect to backward-edges below.
                Note that a backward-graph of a graph has just the edges being reversed, so $(s, d) \mapsto (d, s)$.
                The more intuitive way is storing the graph and necessary indices forwardly.
                That's why the forward graph's notation is used as default (\eg\ $S$ is used instead of $S_\mathit{fwd}$ according to $S_\mathit{bwd}$).
                However, both offset-arrays $O_D$ and $O_S$ and $\mathit{map}_\mathit{fwd}$ are needed and hence each table is needed.

                The tables highlight important values of the two example-queries from the text.
                \label{table:balancing:indices_mapping}
            }
        \end{table}

        Two example-queries are explained in the following, assuming the graph being stored with respect to forward edges.
        In general, leaving edges of vertex $V[s]$ or incoming edges of a vertex $V[d]$ can be determined by following mappings (referring to \cref{table:balancing:indices_mapping}):
        \begin{equation}
        \label{eq:balancing:indices_mapping}
        \begin{aligned}
            \mathit{offset} &\in \left\{ O_D \left[ s \right], O_D \left[ s \right] + 1, \dots, O_D \left[ s+1 \right] - 1 \right\} \\
            d_\mathit{offset} &= D \left[ \mathit{identity}_\mathit{fwd} \left[ \mathit{offset} \right] \right] \\
            s &\mapsto \left\{ \left( s, d_\mathit{offset} \right) \right\} \\
            \\
            \mathit{offset} &\in \left\{ O_S \left[ d \right], O_S \left[ d \right] + 1, \dots, O_S \left[ d+1 \right] - 1 \right\} \\
            s_\mathit{offset} &= S \left[ \mathit{map}_\mathit{fwd} \left[ \mathit{offset} \right] \right] \\
            d &\mapsto \left\{ \left( s_\mathit{offset}, d \right) \right\} \\
        \end{aligned}
        \end{equation}
        The mapping $\mathit{map}_\mathit{fwd}$ can be computed by sorting all edges according to the forward-graph (first source-id, than destination-id) before sorting all edges again according to the backward-graph (first destination-id, than source-id).
        In between, the edge-positions from the first sorted edges are linked to the edges, such that they get reordered by sorting a second time.
        After sorting the second time, the remembered positions become the mapping $\mathit{map}_\mathit{fwd}$.

        In the examples, one query is asking for all leaving edges starting in $V[s=4]=\mathit{id}_4$ and one is asking for all incoming edges ending in $V[d=1]=\mathit{id}_1$.
        The choice of $d=1$ is handy, because $(2, 1) \in E \wedge (1, 2) \notin E$ holds, which might help with understanding the examples.
        Important values in these examples are highlighted in the tables.

        To get all leaving edges starting in $V[s=4]=\mathit{id}_4$, the offset-array $O_D$ is used.
        All forward-indices within $\left\{ O_D[s], O_D[s] + 1, \dots, O_D[s+1] - 1 \right\}$ refer to the leaving edges.
        Since $O_D[4]=9$ and $O_D[5]=11$, all leaving-edges are at positions $\left\{ 9, 10 \right\}$ with respect to forward-edges.
        Because the graph is stored with respect to forward-edges, these resulting positions are already the edge-indices $i_E$ of the leaving-edges.
        The \cref{table:balancing:indices_mapping} still marks the respective values \num{9} and \num{10} in the forward-table.
        Thus, all leaving-edges are $\left\{ (4, D[9]=2), (4, D[10]=3) \right\} = \left\{ (4,2), (4,3) \right\}$.

        Getting all incoming edges ending in $V[d=1]=\mathit{id}_1$ works similar and uses the offset-array $O_S$ respectively.
        All backward-indices within $\left\{ O_S[d], O_S[d] + 1, \dots, O_S[d+1] - 1 \right\}$ refer to the incoming edges.
        The difference here is the outcome from $O_S$.
        Due to $O_S[1]=1$ and $O_S[2]=4$, all incoming-edges are at positions $\left\{ 1, 2, 3 \right\}$ with respect to backward-edges, but the graph is stored with respect to forward-edges.
        Therefore, the indices are not matching and the resulting positions $\left\{ 1, 2, 3 \right\}$ need to be mapped to the respective (forward-) edge-indices $i_E$.
        For this, $\mathit{map}_\mathit{fwd}$ is used.
        \begin{equation}
        \begin{aligned}
            \mathit{map}_\mathit{fwd}[1] &= 0 \Rightarrow S[0] = 0 \\
            \mathit{map}_\mathit{fwd}[2] &= 3 \Rightarrow S[3] = 2 \\
            \mathit{map}_\mathit{fwd}[3] &= 6 \Rightarrow S[6] = 3 \\
        \end{aligned}
        \end{equation}
        Thus, all incoming-edges are $\left\{ (0, 1), (2, 1), (3, 1) \right\}$.

    \subsection{Storing and accessing shortcuts in graphs}

        Shortcuts have metrics and source-/destination-vertices as well, so they are full edges and can be added to the current data-structure.
        However, due to construction of a contracted graph, a shortcut replaces two edges, which may be shortcuts.
        This functionality is missing in the graph's data-structure yet.
        A first intuition may be extending edges by a tuple of two edge-indices.
        Implementing this causes much redundancy, because normal edges would need a placeholder for their tuple.
        The street-networks from the experiments in \cref{chap:experiments} have less than $|E|$ shortcuts added.
        So after adding all shortcuts, more than half of all edges will have unused allocated memory of two indices.
        For example, Saarland from the experiments contains over $\num{1000000}$~edges.
        If these placeholders would be used, each of these million edges would have two unused indices.
        Assuming one index needs $\si{\num{64} \bit}$, this sums up to $\si{\num{16} \mega\byte}$ of unused memory.
        This number scales with the graph's size, so Germany with over $\num{100000000}$~edges would have already $\si{\num{1.6} \giga\byte}$ of completely unused, but allocated memory.

        Therefore, since edge-indices $i_E$ can be used for all edge-data ($S$, $D$, $C$), this whole indexing-procedure from \cref{chap:balancing:implementation:graphs} can be and is used in this thesis for storing and accessing shortcuts without such placeholders.
        Like the offsets $O_D$ and $O_S$, another offset-array is created.
        Let this new offset-array be called $O_L$ ($L$ for link, since $S$ is already used).
        The significant difference between $O_D$ and $O_S$ on one side and $O_L$ on the other side is, that $O_L$ contains an offset-value per edge, whereas $O_D$ and $O_S$ contain offset-values per vertex.
        Lining all shortcut-tuples of edge-indices up according to the forward-edges' order in the graph, this results in the data-array $L$.
        This array $L$ behaves exactly similar as the array $S$ or $D$.
        So, despite the indexing-mapping, the shortcut-indices of an edge-index $i_E$ can be found in $L$ between $O_L \left[ i_E \right]$ (including) and $O_L \left[ i_E + 1 \right]$ (excluding).
        A shortcut can be determined by comparing neighbouring values in $O_L$.
        If $O_L \left[ i_E \right] = O_L \left[ i_E + 1 \right]$ holds, the edge at $i_E$ is no shortcut.

        When analyzing the memory-usage, this method has improved the other approach of storing tuples in the experiments.
        Let $m$ be the number of edges without shortcuts and $m_L$ the number of shortcuts in the graph.
        Previously, the number of needed indices was $2 \cdot (m + m_L)$.
        The offset-procedure stores $2 \cdot m_L$ replaced edge-indices and $(m + m_L)$ offset-indices, resulting in $m + m_L + 2 \cdot m_L$.
        As long as $m_L < m$ holds, the offset-procedure saves memory, which was the case in all experimental graphs.

    \subsection{Multithreading}

        The implementation of searching for paths in parallel has brought a huge performance-boost.
        Optimal paths of different \glspl{stpair} may have different costs and thus need a different amount of runtime.
        This holds especially for \gls{repr}, where longer paths lead to more alternative paths.
        Therefore, the number of \glspl{stpair} is not distributed evenly over all threads.
        In fact, there is one single master-thread pulling some \glspl{stpair} from the user-provided set of \glspl{stpair} and sends this work-package to one of several worker-threads.
        Every worker-thread has its own instance of the routing-algorithm, searches for the optimum paths, and returns a simple list of all occurred edge-indices $i_E$ to the master-thread.
        The master-thread counts the workloads for each edge and pulls the next work-package until all \glspl{stpair} are processed.

        With this procedure, the workload and not the number of \glspl{stpair} is distributed evenly.
        The communication-overhead when using smaller work-package-sizes is negligible compared to idle worker-threads, that wait for the last few worker-threads to finish their larger work-package.


\section{Algebraic Structures}

\subsection{OFE}

The model of Iris lives in the category of \emph{Ordered Families of Equivalences} (OFEs).
This definition varies slightly from the original one in~\cite{catlogic}.

\begin{defn}
  An \emph{ordered family of equivalences} (OFE) is a tuple $(\ofe, ({\nequiv{n}} \subseteq \ofe \times \ofe)_{n \in \nat})$ satisfying
  \begin{align*}
    \All n. (\nequiv{n}) ~& \text{is an equivalence relation} \tagH{ofe-equiv} \\
    \All n, m.& n \geq m \Ra (\nequiv{n}) \subseteq (\nequiv{m}) \tagH{ofe-mono} \\
    \All x, y.& x = y \Lra (\All n. x \nequiv{n} y) \tagH{ofe-limit}
  \end{align*}
\end{defn}

The key intuition behind OFEs is that elements $x$ and $y$ are $n$-equivalent, notation $x \nequiv{n} y$, if they are \emph{equivalent for $n$ steps of computation}, \ie if they cannot be distinguished by a program running for no more than $n$ steps.
In other words, as $n$ increases, $\nequiv{n}$ becomes more and more refined (\ruleref{ofe-mono})---and in the limit, it agrees with plain equality (\ruleref{ofe-limit}).

Notice that OFEs are just a different presentation of bisected 1-bounded ultrametric spaces, where the family of equivalence relations gives rise to the distance function (two elements that are equal for $n$ steps are no more than $2^{-n}$ apart).

\begin{defn}
  An element $x \in \ofe$ of an OFE is called \emph{discrete} if
  \[ \All y \in \ofe. x \nequiv{0} y \Ra x = y\]
  An OFE $A$ is called \emph{discrete} if all its elements are discrete.
  For a set $X$, we write $\Delta X$ for the discrete OFE with $x \nequiv{n} x' \eqdef x = x'$
\end{defn}

\begin{defn}
  A function $f : \ofe \to \ofeB$ between two OFEs is \emph{non-expansive} (written $f : \ofe \nfn \ofeB$) if
  \[\All n, x \in \ofe, y \in \ofe. x \nequiv{n} y \Ra f(x) \nequiv{n} f(y) \]
  It is \emph{contractive} if
  \[ \All n, x \in \ofe, y \in \ofe. (\All m < n. x \nequiv{m} y) \Ra f(x) \nequiv{n} f(y) \]
\end{defn}
Intuitively, applying a non-expansive function to some data will not suddenly introduce differences between seemingly equal data.
Elements that cannot be distinguished by programs within $n$ steps remain indistinguishable after applying $f$.

\begin{defn}
  The category $\OFEs$ consists of OFEs as objects, and non-expansive functions as arrows.
\end{defn}

Note that $\OFEs$ is bicartesian closed, \ie it has all sums, products and exponentials as well as an initial and a terminal object.
In particular:
\begin{defn}
  Given two OFEs $\ofe$ and $\ofeB$, the set of non-expansive functions $\set{f : \ofe \nfn \ofeB}$ is itself an OFE with
  \begin{align*}
    f \nequiv{n} g \eqdef{}& \All x \in \ofe. f(x) \nequiv{n} g(x)
  \end{align*}
\end{defn}

\begin{defn}
  A (bi)functor $F : \OFEs \to \OFEs$ is called \emph{locally non-expansive} if its action $F_1$ on arrows is itself a non-expansive map.
  Similarly, $F$ is called \emph{locally contractive} if $F_1$ is a contractive map.
\end{defn}
The function space $(-) \nfn (-)$ is a locally non-expansive bifunctor.
Note that the composition of non-expansive (bi)functors is non-expansive, and the composition of a non-expansive and a contractive (bi)functor is contractive.

One very important OFE is the OFE of \emph{step-indexed propositions}:
For every step-index, such a proposition either holds or does not hold.
Moreover, if a propositions holds for some $n$, it also has to hold for all smaller step-indices.
\begin{align*}
  \SProp \eqdef{}& \psetdown{\nat} \\
    \eqdef{}& \setComp{X \in \pset{\nat}}{ \All n, m. n \geq m \Ra n \in X \Ra m \in X } \\
  X \nequiv{n} Y \eqdef{}& \All m \leq n. m \in X \Lra m \in Y \\
  X \nincl{n} Y \eqdef{}& \All m \leq n. m \in X \Ra m \in Y
\end{align*}

\subsection{COFE}

COFEs are \emph{complete OFEs}, which means that we can take limits of arbitrary chains.

\begin{defn}[Chain]
  Given some set $\cofe$ and an indexed family $({\nequiv{n}} \subseteq \cofe \times \cofe)_{n \in \nat}$ of equivalence relations, a \emph{chain} $c \in \Chains(\cofe)$ is a function $c : \nat \to \cofe$ such that $\All n, m. n \leq m \Ra c (m) \nequiv{n} c (n)$.
\end{defn}

\begin{defn}
  A \emph{complete ordered family of equivalences} (COFE) is a tuple $(\cofe : \OFEs,  \lim : \Chains(\cofe) \to \cofe)$ satisfying
  \begin{align*}
    \All n, c.& \lim(c) \nequiv{n} c(n) \tagH{cofe-compl}
  \end{align*}
\end{defn}

\begin{defn}
  The category $\COFEs$ consists of COFEs as objects, and non-expansive functions as arrows.
\end{defn}

The function space $\ofe \nfn \cofeB$ is a COFE if $\cofeB$ is a COFE (\ie the domain $\ofe$ can actually be just an OFE).
$\SProp$ as defined above is complete, \ie it is a COFE.

Completeness is necessary to take fixed-points.

\begin{thm}[Banach's fixed-point]
\label{thm:banach}
Given an inhabited COFE $\ofe$ and a contractive function $f : \ofe \to \ofe$, there exists a unique fixed-point $\fixp_T f$ such that $f(\fixp_T f) = \fixp_T f$.
Moreover, this theorem also holds if $f$ is just non-expansive, and $f^k$ is contractive for an arbitrary $k$.
\end{thm}

\begin{thm}[America and Rutten~\cite{America-Rutten:JCSS89,birkedal:metric-space}]
\label{thm:america_rutten}
Let $1$ be the discrete COFE on the unit type: $1 \eqdef \Delta \{ () \}$.
Given a locally contractive bifunctor $G : \COFEs^{\textrm{op}} \times \COFEs \to \COFEs$, and provided that \(G(1, 1)\) is inhabited,
then there exists a unique\footnote{Uniqueness is not proven in Coq.} COFE $\ofe$ such that $G(\ofe^{\textrm{op}}, \ofe) \cong \ofe$ (\ie the two are isomorphic in $\COFEs$).
\end{thm}

\subsection{RA}

\begin{defn}
  A \emph{resource algebra} (RA) is a tuple \\
  $(\monoid, \mvalFull :  \monoid \to \mProp, \mcore{{-}}:
  \monoid \to \maybe\monoid, (\mtimes) : \monoid \times \monoid \to \monoid)$ satisfying:
  \begin{align*}
    \All \melt, \meltB, \meltC.& (\melt \mtimes \meltB) \mtimes \meltC = \melt \mtimes (\meltB \mtimes \meltC) \tagH{ra-assoc} \\
    \All \melt, \meltB.& \melt \mtimes \meltB = \meltB \mtimes \melt \tagH{ra-comm} \\
    \All \melt.& \mcore\melt \in \monoid \Ra \mcore\melt \mtimes \melt = \melt \tagH{ra-core-id} \\
    \All \melt.& \mcore\melt \in \monoid \Ra \mcore{\mcore\melt} = \mcore\melt \tagH{ra-core-idem} \\
    \All \melt, \meltB.& \mcore\melt \in \monoid \land \melt \mincl \meltB \Ra \mcore\meltB \in \monoid \land \mcore\melt \mincl \mcore\meltB \tagH{ra-core-mono} \\
    \All \melt, \meltB.& \mvalFull(\melt \mtimes \meltB)  \Ra \mvalFull(\melt)  \tagH{ra-valid-op} \\
    \text{where}\qquad %\qquad\\
    \maybe\monoid \eqdef{}& \monoid \uplus \set{\mnocore} \qquad\qquad\qquad \melt^? \mtimes \mnocore \eqdef \mnocore \mtimes \melt^? \eqdef \melt^? \\
    \melt \mincl \meltB \eqdef{}& \Exists \meltC \in \monoid. \meltB = \melt \mtimes \meltC \tagH{ra-incl}
  \end{align*}
\end{defn}
Here, $\mProp$ is the set of (meta-level) propositions.
Think of \texttt{Prop} in Coq or $\mathbb{B}$ in classical mathematics.

RAs are closely related to \emph{Partial Commutative Monoids} (PCMs), with two key differences:
\begin{enumerate}
\item The composition operation on RAs is total (as opposed to the partial composition operation of a PCM), but there is a specific subset of \emph{valid} elements that is compatible with the composition operation (\ruleref{ra-valid-op}).
These valid elements are identified by the \emph{validity predicate} $\mvalFull$.

This take on partiality is necessary when defining the structure of \emph{higher-order} ghost state, \emph{cameras}, in the next subsection.

\item Instead of a single unit that is an identity to every element, we allow
for an arbitrary number of units, via a function $\mcore{{-}}$ assigning to an element $\melt$ its \emph{(duplicable) core} $\mcore\melt$, as demanded by \ruleref{ra-core-id}.
  We further demand that $\mcore{{-}}$ is idempotent (\ruleref{ra-core-idem}) and monotone (\ruleref{ra-core-mono}) with respect to the \emph{extension order}, defined similarly to that for PCMs (\ruleref{ra-incl}).

  Notice that the domain of the core is $\maybe\monoid$, a set that adds a dummy element $\mnocore$ to $\monoid$.
%  (This corresponds to the option type.)
  Thus, the core can be \emph{partial}: not all elements need to have a unit.
  We use the metavariable $\maybe\melt$ to indicate elements of  $\maybe\monoid$.
  We also lift the composition $(\mtimes)$ to $\maybe\monoid$.
  Partial cores help us to build interesting composite RAs from smaller primitives.

Notice also that the core of an RA is a strict generalization of the unit that any PCM must provide, since $\mcore{{-}}$ can always be picked as a constant function.
\end{enumerate}

\begin{defn}
  It is possible to do a \emph{frame-preserving update} from $\melt \in \monoid$ to $\meltsB \subseteq \monoid$, written $\melt \mupd \meltsB$, if
  \[ \All \maybe{\melt_\f} \in \maybe\monoid. \mvalFull(\melt \mtimes \maybe{\melt_\f}) \Ra \Exists \meltB \in \meltsB. \mvalFull(\meltB \mtimes \maybe{\melt_\f}) \]

  We further define $\melt \mupd \meltB \eqdef \melt \mupd \set\meltB$.
\end{defn}
The proposition $\melt \mupd \meltsB$ says that every element $\maybe{\melt_\f}$ compatible with $\melt$ (we also call such elements \emph{frames}), must also be compatible with some $\meltB \in \meltsB$.
Notice that $\maybe{\melt_\f}$ could be $\mnocore$, so the frame-preserving update can also be applied to elements that have \emph{no} frame.
Intuitively, this means that whatever assumptions the rest of the program is making about the state of $\gname$, if these assumptions are compatible with $\melt$, then updating to $\meltB$ will not invalidate any of these assumptions.
Since Iris ensures that the global ghost state is valid, this means that we can soundly update the ghost state from $\melt$ to a non-deterministically picked $\meltB \in \meltsB$.

\subsection{Cameras}

\begin{defn}
  A \emph{camera} is a tuple $(\monoid : \OFEs, \mval : \monoid \nfn \SProp, \mcore{{-}}: \monoid \nfn \maybe\monoid,\\ (\mtimes) : \monoid \times \monoid \nfn \monoid)$ satisfying:
  \begin{align*}
    \All \melt, \meltB, \meltC.& (\melt \mtimes \meltB) \mtimes \meltC = \melt \mtimes (\meltB \mtimes \meltC) \tagH{camera-assoc} \\
    \All \melt, \meltB.& \melt \mtimes \meltB = \meltB \mtimes \melt \tagH{camera-comm} \\
    \All \melt.& \mcore\melt \in \monoid \Ra \mcore\melt \mtimes \melt = \melt \tagH{camera-core-id} \\
    \All \melt.& \mcore\melt \in \monoid \Ra \mcore{\mcore\melt} = \mcore\melt \tagH{camera-core-idem} \\
    \All \melt, \meltB.& \mcore\melt \in \monoid \land \melt \mincl \meltB \Ra \mcore\meltB \in \monoid \land \mcore\melt \mincl \mcore\meltB \tagH{camera-core-mono} \\
    \All \melt, \meltB.& \mval(\melt \mtimes \meltB) \subseteq \mval(\melt)  \tagH{camera-valid-op} \\
    \All n, \melt, \meltB_1, \meltB_2.& \omit\rlap{$n \in \mval(\melt) \land \melt \nequiv{n} \meltB_1 \mtimes \meltB_2 \Ra {}$} \\
    &\Exists \meltC_1, \meltC_2. \melt = \meltC_1 \mtimes \meltC_2 \land \meltC_1 \nequiv{n} \meltB_1 \land \meltC_2 \nequiv{n} \meltB_2 \tagH{camera-extend} \\
    \text{where}\qquad\qquad\\
    \melt \mincl \meltB \eqdef{}& \Exists \meltC. \meltB = \melt \mtimes \meltC \tagH{camera-incl} \\
    \melt \mincl[n] \meltB \eqdef{}& \Exists \meltC. \meltB \nequiv{n} \melt \mtimes \meltC \tagH{camera-inclN}
  \end{align*}
\end{defn}

This is a natural generalization of RAs over OFEs\footnote{The reader may wonder why on earth we call them ``cameras''.
The reason, which may not be entirely convincing, is that ``camera'' was originally just used as a comfortable pronunciation of ``CMRA'', the name used in earlier Iris papers.
CMRA was originally supposed to be an acronym for ``complete metric resource algebras'' (or something like that), but we were never very satisfied with it and thus ended up never spelling it out.
To make matters worse, the ``complete'' part of CMRA is now downright misleading, for whereas previously the carrier of a CMRA was required to be a COFE (complete OFE), we have relaxed that restriction and permit it to be an (incomplete) OFE.
For these reasons, we have decided to stick with the name ``camera'', for purposes of continuity, but to drop any pretense that it stands for something.}.
All operations have to be non-expansive, and the validity predicate $\mval$ can now also depend on the step-index.
We define the plain $\mvalFull$ as the ``limit'' of the step-indexed approximation:
\[ \mvalFull(\melt) \eqdef \All n. n \in \mval(\melt) \]

\paragraph{The extension axiom (\ruleref{camera-extend}).}
Notice that the existential quantification in this axiom is \emph{constructive}, \ie it is a sigma type in Coq.
The purpose of this axiom is to compute $\melt_1$, $\melt_2$ completing the following square:

% RJ FIXME: Needs some magic to fix the baseline of the $\nequiv{n}$, or so
\begin{center}
\begin{tikzpicture}[every edge/.style={draw=none}]
  \node (a) at (0, 0) {$\melt$};
  \node (b) at (1.7, 0) {$\meltB$};
  \node (b12) at (1.7, -1) {$\meltB_1 \mtimes \meltB_2$};
  \node (a12) at (0, -1) {$\melt_1 \mtimes \melt_2$};

  \path (a) edge node {$\nequiv{n}$} (b);
  \path (a12) edge node {$\nequiv{n}$} (b12);
  \path (a) edge node [rotate=90] {$=$} (a12);
  \path (b) edge node [rotate=90] {$=$} (b12);
\end{tikzpicture}\end{center}
where the $n$-equivalence at the bottom is meant to apply to the pairs of elements, \ie we demand $\melt_1 \nequiv{n} \meltB_1$ and $\melt_2 \nequiv{n} \meltB_2$.
In other words, extension carries the decomposition of $\meltB$ into $\meltB_1$ and $\meltB_2$ over the $n$-equivalence of $\melt$ and $\meltB$, and yields a corresponding decomposition of $\melt$ into $\melt_1$ and $\melt_2$.
This operation is needed to prove that $\later$ commutes with separating conjunction:
\begin{mathpar}
  \axiom{\later (\prop * \propB) \Lra \later\prop * \later\propB}
\end{mathpar}

\begin{defn}
  An element $\munit$ of a camera $\monoid$ is called the \emph{unit} of $\monoid$ if it satisfies the following conditions:
  \begin{enumerate}[itemsep=0pt]
  \item $\munit$ is valid: \\ $\All n. n \in \mval(\munit)$
  \item $\munit$ is a left-identity of the operation: \\
    $\All \melt \in M. \munit \mtimes \melt = \melt$
  \item $\munit$ is its own core: \\ $\mcore\munit = \munit$
  \end{enumerate}
\end{defn}

\begin{lem}\label{lem:camera-unit-total-core}
  If $\monoid$ has a unit $\munit$, then the core $\mcore{{-}}$ is total, \ie $\All\melt. \mcore\melt \in \monoid$.
\end{lem}

\begin{defn}
  It is possible to do a \emph{frame-preserving update} from $\melt \in \monoid$ to $\meltsB \subseteq \monoid$, written $\melt \mupd \meltsB$, if
  \[ \All n, \maybe{\melt_\f}. n \in \mval(\melt \mtimes \maybe{\melt_\f}) \Ra \Exists \meltB \in \meltsB. n \in\mval(\meltB \mtimes \maybe{\melt_\f}) \]

  We further define $\melt \mupd \meltB \eqdef \melt \mupd \set\meltB$.
\end{defn}
Note that for RAs, this and the RA-based definition of a frame-preserving update coincide.

\begin{defn}
  A camera $\monoid$ is \emph{discrete} if it satisfies the following conditions:
  \begin{enumerate}[itemsep=0pt]
  \item $\monoid$ is a discrete COFE
  \item $\mval$ ignores the step-index: \\
    $\All \melt \in \monoid. 0 \in \mval(\melt) \Ra \All n. n \in \mval(\melt)$
  \end{enumerate}
\end{defn}
Note that every RA is a discrete camera, by picking the discrete COFE for the equivalence relation.
Furthermore, discrete cameras can be turned into RAs by ignoring their COFE structure, as well as the step-index of $\mval$.

\begin{defn}[Camera homomorphism]
  A function $f : \monoid_1 \to \monoid_2$ between two cameras is \emph{a camera homomorphism} if it satisfies the following conditions:
  \begin{enumerate}[itemsep=0pt]
  \item $f$ is non-expansive
  \item $f$ commutes with composition:\\
    $\All \melt_1 \in \monoid_1, \melt_2 \in \monoid_1. f(\melt_1) \mtimes f(\melt_2) = f(\melt_1 \mtimes \melt_2)$
  \item $f$ commutes with the core:\\
    $\All \melt \in \monoid_1. \mcore{f(\melt)} = f(\mcore{\melt})$
  \item $f$ preserves validity: \\
    $\All n, \melt \in \monoid_1. n \in \mval(\melt) \Ra n \in \mval(f(\melt))$
  \end{enumerate}
\end{defn}

\begin{defn}
  The category $\CMRAs$ consists of cameras as objects, and camera homomorphisms as arrows.
\end{defn}
Note that every object/arrow in $\CMRAs$ is also an object/arrow of $\OFEs$.
The notion of a locally non-expansive (or contractive) bifunctor naturally generalizes to bifunctors between these categories.
%TODO: Discuss how we probably have a commuting square of functors between Set, RA, CMRA, COFE.

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "iris"
%%% End:


% When using TeXShop on the Mac, let it know the root document.
% The following must be one of the first 20 lines.
% !TEX root = ../design.tex

\chapter{Low-rank Matrix Factorization}

\begin{moduleinfo}
\item[Author] \href{mailto:xfeng@cs.wisc.edu}{Xixuan (Aaron) Feng} (version 0.5 only)
\item[History]
	\begin{modulehistory}
		\item[v0.5] Initial revision of design document, Implementation based on incremental gradient descent
		\item[v0.1] Initial revision (somewhat misleadingly called SVD matrix factorization at that time)
	\end{modulehistory}
\end{moduleinfo}

% Abstract. What is the problem we want to solve?
This module implements "factor model" for representing an incomplete matrix using a low-rank approximation \cite{DBLP:conf/icml/SrebroJ03}.
Mathematically, this model seeks to find matrices U and V (also referred as factors) that, for any given incomplete matrix A, minimizes:
\[ \|\boldsymbol A - \boldsymbol UV^{T} \|_2 \]
subject to $rank(\boldsymbol UV^{T}) \leq r$, where $\|\cdot\|_2$ denotes the Frobenius norm.
Let $A$ be a $m \times n$ matrix, then $U$ will be $m \times r$ and $V$ will be $n \times r$, in dimension, and $1 \leq r \ll \min(m, n)$.
This model is not intended to do the full decomposition, or to be used as part of inverse procedure.
This model has been widely used in recommendation systems (e.g., Netflix \cite{:TheNetflixPrize07}) and feature selection (e.g., image processing \cite{DBLP:conf/nips/WrightGRPM09}).

\section{Incremental Gradient Descent}

% Background. Why can we solve the problem with incremental gradient?
\subsection{Solving as a Convex Program}
Recent work \cite{DBLP:journals/cacm/CandesR12, DBLP:journals/siamrev/RechtFP10} has demonstrated that the low-rank matrix factorization can be solved as a convex programming problem.
This body of work enables us to solve the problem by using gradient-based line search algorithms.
Among many of these algorithms, incremental gradient descent algorithm is a popular choice, especially for really large input matrices \cite{DBLP:conf/sigmod/FengKRR12, DBLP:conf/kdd/GemullaNHS11}.

\subsection{Formal Description}
\begin{algorithm}[lmf-igd$(r, A, \alpha)$] \label{alg:lmf-igd}
\alginput{Sparse matrix $A$,
\\step size $\alpha$,
\\low-rank constraint $r$,
\\convergence criterion $\mathit{Convergence}$,
\\random factor generator $\mathit{GenerateFactor}$}
\algoutput{Factors $U$ ($m \times r$) and $V$ ($n \times r$)}
\algprecond{$\mathit{iteration} = 0$}
\begin{algorithmic}[1]
	\State $U \set \mathit{GenerateFactor}(m, r)$
	\State $V \set \mathit{GenerateFactor}(n, r)$
	\Repeat
		\State $\mathit{iteration} \set \mathit{iteration} + 1$
		\State $U_\text{old} \set U$
		\State $V_\text{old} \set V$
		\For{$(i, j, y) \in A$} \Comment{Single entry in sparse matrix A}
			\State $e \set U_i \cdot V_j - y$
			\State $\mathit{temp} \set U_i - \alpha e V_j$
			\State $V_j \set V_j - \alpha e U_i$ \Comment{In-place update of V}
			\State $U_i \set \mathit{temp}$ \Comment{In-place update of U}
		\EndFor
	\Until{$Convergence(U_\text{old}, V_\text{old}, U, V, \mathit{iteration})$}
\end{algorithmic}
\end{algorithm}

\begin{description}
	\item[Runtime] $O(N_{A} (m + n) r + m n r)$ for one iteration,
        where $N_{A}$ is the number of nonzero elements in $A$.

	\item[Space] Store the $\mathit{temp}$, an $r$-floating-point vector.

	\item[Parallelism] The outer loop is inherently sequential.
        The inner loop is data-parallel and model averaging \cite{DBLP:conf/nips/DuchiAW10} is used.

    \item[Factor initialization] The author of this document is not aware that significant differences are caused if random factors are initialized by different distributions.
        But zero values should be avoided.
        And entries in factors should not be initialized as the same value; otherwise, factors will always be rank 1.

    \item[Convergence criterion] Usually, following conditions are combined by AND, OR, or NOT.
        \begin{enumerate}
            \item The change in the objective drops below a given threshold (E.g., RMSE).
            \item The value of the objective matches some pre-computed value.
            \item The maximum number of iterations has been reached.
            \item There could be more.
        \end{enumerate}
\end{description}