\documentclass[12pt]{article}%
\usepackage{hyperref}
\usepackage{listings}
\usepackage{color}
\usepackage{multicol}
\usepackage{amsfonts}
\usepackage{fancyhdr}
\usepackage{comment}
\usepackage[a4paper, top=2.2cm, bottom=2.2cm, left=2.2cm, right=2.2cm]%
{geometry}
\usepackage{times}
\usepackage{changepage}
\usepackage{amssymb}
\usepackage{graphicx}%
\newtheorem{theorem}{Theorem}
\newtheorem{acknowledgement}[theorem]{Acknowledgement}
\newtheorem{algorithm}[theorem]{Algorithm}
\newtheorem{axiom}{Axiom}
\newtheorem{case}[theorem]{Case}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{conclusion}[theorem]{Conclusion}
\newtheorem{condition}[theorem]{Condition}
\newtheorem{conjecture}[theorem]{Conjecture}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{criterion}[theorem]{Criterion}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{example}[theorem]{Example}
\newtheorem{exercise}[theorem]{Exercise}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{notation}[theorem]{Notation}
\newtheorem{problem}[theorem]{Problem}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{remark}[theorem]{Remark}
\newtheorem{solution}[theorem]{Solution}
\newtheorem{summary}[theorem]{Summary}
\usepackage{commath}
\usepackage{url}
\usepackage{hyperref}
\usepackage[style=numeric]{biblatex}
\usepackage{subfig}
\usepackage{minted}
% \usepackage[utf8]{inputenc}
% \usepackage[english]{babel}
\usepackage{esvect}
\addbibresource{reference.bib}

\usepackage{commath}

\newenvironment{proof}[1][Proof]{\textbf{#1.} }{\ \rule{0.5em}{0.5em}}
\usepackage[utf8]{inputenc}

% \usepackage{algorithm}
% \usepackage{algorithmic} %format of the algorithm

% Default fixed font does not support bold face
\DeclareFixedFont{\ttb}{T1}{txtt}{bx}{n}{12} % for bold
\DeclareFixedFont{\ttm}{T1}{txtt}{m}{n}{12}  % for normal

\usepackage{graphics}

% Custom colors
\usepackage{color}
\definecolor{deepblue}{rgb}{0,0,0.5}
\definecolor{deepred}{rgb}{0.6,0,0}
\definecolor{deepgreen}{rgb}{0,0.5,0}

\usepackage{listings}
\newcommand{\Q}{\mathbb{Q}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\C}{\mathbb{C}}
\newcommand{\Z}{\mathbb{Z}}

% Python style for highlighting
\newcommand\pythonstyle{\lstset{
language=Python,
basicstyle=\ttm,
otherkeywords={self},             % Add keywords here
keywordstyle=\ttb\color{deepblue},
emph={MyClass,__init__},          % Custom highlighting
emphstyle=\ttb\color{deepred},    % Custom highlighting style
stringstyle=\color{deepgreen},
frame=tb,                         % Any extra options here
showstringspaces=false            %
}}

% Python environment
\lstnewenvironment{python}[1][]
{
\pythonstyle
\lstset{#1}
}
{}

% Python for external files
\newcommand\pythonexternal[2][]{{
\pythonstyle
\lstinputlisting[#1]{#2}}}
\usepackage[utf8]{inputenc}
\usepackage{amsmath, nccmath}
\usepackage{geometry}

\usepackage{algorithm}
\usepackage{arevmath}     % For math symbols
\usepackage[noend]{algpseudocode}

\begin{document}

\title{Institute of Robotics,  University of Innopolis}
\author{Computational Intelligence \\ Least Squares Estimation, Convex Operations , and Convex Programming}
\date{\today}
\maketitle

\subsection{Task 01}
In least-squares, given the measurements $A \in \mathbb{R}^{m\times n}$ and $b \in \mathbb{R}^{n}$, seek a vector $x \in \mathbb{R}^m$ that project $Ax$ on $b$ (or such that $Ax$ close to $b$). Such a closeness is defined as:
\begin{equation} \label{eq:least_squares}
    \sum_{i=1}^m (a_i^Tx-b_i)^2 = \min_x \left \| Ax -b \right \|_2^2
\end{equation}

\begin{enumerate}
    \item Using CVXPY formulate the (\ref{eq:least_squares}), you may generate some random matrix and vector for $A$ and $b$, respectively

\begin{minted}
[frame=lines, framesep=2mm, baselinestretch=1.2,]
{python}
 m = 20
 n = 15
 np.random.seed(1)
 A = np.random.randn(m, n)
 b = np.random.randn(m)
\end{minted}
% # Define and solve the CVXPY problem.
% x = cp.Variable(n)
% cost = cp.sum_squares(A @ x - b)
% prob = cp.Problem(cp.Minimize(cost))
% prob.solve()

% # Print result.
% print("\nThe optimal value is", prob.value)
% print("The optimal x is")
% print(x.value)
% print("The norm of the residual is ", cp.norm(A @ x - b, p=2).value)
    \item If $x^*$ is the optimal solution you obtained, comment on $Ax^*-b$
    % zero mean perfect fitting
\end{enumerate}

\subsection{Task 02}
Let's try adding a regularization term in the objective (\ref{eq:least_squares}) as follows:

\begin{equation}\label{eq:lasso}
    \min_{x} \sum_{i=1}^m (a_i^Tx-b_i)^2+\lambda\|x\|_1
\end{equation}

\begin{enumerate}
    \item Using CVXPY formulate the (\ref{eq:lasso}), use the same matrix and vector you used in task 01 for $A$ and $b$, respectively. $\lambda$ is a regularization parameter, e.g., $1/\sqrt{n}$

    \item Compare the $x^*$ in both cases
    \item Repeat the Task 01 and Task 02 using the provided dataset
    % zero mean perfect fitting
\end{enumerate}

\subsection{Task 03}
Consider the following minimization problem
\begin{equation}
    \min_x (Ax-b)^{\top} (Ax-b)
\end{equation}

\begin{equation}\label{eq:01}
\begin{aligned}
 \min_{x} \quad & (Ax-b)^{\top} (Ax-b) \\
\textrm{s.t.} \quad  & Gx \leq h,
\end{aligned}
\end{equation}
\begin{enumerate}
    \item Let's consider with no constraints, try to solve analytically ($x^* = (A^TA)^{ -1}A^T$) and as well as a QP problem and compare your answers? Take $A=\begin{bmatrix}
1 &1 \\
2 & 1\\
3 & 2
\end{bmatrix} $ and $b = \begin{bmatrix}
2\\
3\\
4
\end{bmatrix}$
    \item Constraint x within $-0.9 \leq x \leq 0.9$ and solve it again,
\end{enumerate}

\subsection{Task 04}
A sphere is described by $\{ x \in \mathbb{R}^n \mid \left \| x - x_c \right \|_2 = r\}$. Let's try to fit a sphere in $\mathbb{R}^n$ for a given m number of points ($u_1, u_2,...,u_m \in \mathbb{R}^n$), by minimizing the following error function:
\begin{equation}
    \sum_{i=1}^m (\left \| u_i -x_c \right \|_2^2-r^2)^2
\end{equation} over the variables $x_c \in \mathbb{R}^n, \; r \in \mathbb{R}$

\begin{enumerate}
    \item Formulate the problem as a least squares problem in the form : $\min_x \left \| Ax -b \right \|_2^2$
    \item If $x = (x_c, t)$, define the $t$ in terms of $r$ and $x_c$
    \item Define the $A$ and $b$  and show that
    \item Use the optimality condition $A^T(Ax-b) = 0$ and show that
    \begin{equation}\label{eq:shpere}
        r^2  = \frac{1}{m} \sum_{i=1}^m  \left \| u_i -x_c \right \|_2^2
    \end{equation}
    \item  Using CVXPY formulate the (\ref{eq:shpere}) for $\mathbb{R}^2$ where m number of points can be generated as follows:
    \begin{minted}
[frame=lines, framesep=2mm, baselinestretch=1.2,]
{python}
m = 50
r = 1
xc = (3,4)

t = np.linspace(0, 2*np.pi, m, endpoint=False)
x = xc[0] + r * np.cos(t) + np.random.uniform(-0.2,0.2,t.shape[0])
y = xc[1] + r * np.sin(t) + np.random.uniform(-0.2,0.2,t.shape[0])
U = np.vstack((x,y))

\end{minted}
\end{enumerate}

\begin{figure}[H]%
    \centering
    \subfloat[\centering \label{f:init_space}  ]{{\includegraphics[width=5cm]{circle.png} }}%
    \qquad
    \subfloat[\centering\label{f:end_pose} ]{{\includegraphics[width=7cm]{shpere.png} }}%
    \label{fig:example}\caption{Expected output for $\mathbb{R}^2$ and $\mathbb{R}^3$} %
\end{figure}

\subsection{Task 05}
Let's try to find the Chebyshev center of a polyhedron. Consider the following polyhedron:
\begin{equation}
    P = \{ x\mid a_i^{\top}x \leq b_i, \; i=1,...,m\}
\end{equation} The Chebyshev center is the center of the largest ball that can fit within the P
\begin{equation}
    Cb = \{ x_c+u\mid \left \| u \right \|_2 \leq r\}
\end{equation}, where $x_c$ is the center and $u = x-x_c$. Hint: Cauchy-Schwarz Inequality: all vectors $\mathbf{a}$ and $\mathbf{u}$ of an inner product space it is true that
\begin{equation}
    \mathbf{a}^T\mathbf{u} \leq \left \| \mathbf{a} \right \|_2 \left \| \mathbf{u} \right \|_2
\end{equation}

\begin{enumerate}
    \item To Cb be inside P, $a_i^{\top}x\leq b_i$ for all $x \subset Cb$, how can you define this condition?
    \item Solve this optimization problem considering these constraints: $A = \begin{bmatrix}
-1 & -1\\
-0.5 & 1\\
 2& -1
\end{bmatrix}, \; b = \begin{bmatrix}
1\\
2\\
4
\end{bmatrix}$, and compare your result with MPT toolbox as follows:
    \begin{minted}
[frame=lines, framesep=2mm, baselinestretch=1.2,]
{matlab}

P = Polyhedron('A', A, 'b', b);
hold on
plot(P);
x = sdpvar(2,1);
data = P.chebyCenter();
S = YSet(x, norm(x - data.x) <= data.r);
S.plot('color', 'lightgreen');
plot(data.x(1), data.x(2), 'ko', 'MarkerSize'
                    , 10, 'MarkerFaceColor', 'k');
\end{minted}

\begin{figure}[H]
    \begin{center}
    \includegraphics[width=12cm]{ball.png}
    \caption{Chebyshev center}\label{f:farthest-dis}
    \end{center}
\end{figure}

\end{enumerate}

\subsection{Task 06}
Let $S=\{y_1\mathbf{a}_1+y_2\mathbf{a}_2 \mid -1<y_1<1, -1<y_2<1\}$, where $\mathbf{a}_1, \mathbf{a}_2 \in \mathbb{R}^n$ be a polyhedron. Your task is to formulate S in the standard form, namely $S=\{ x \mid Ax \leq b, Fx=g\}$. For simplicity assume that $\mathbf{a}_1$ and $\mathbf{a}_2$ are independent, and S can be seen as a intersection of following three sets:
\begin{enumerate}
    \item $S_1$ plane defined by $\mathbf{a}_1$ and $\mathbf{a}_2$
    \item $S_2 = \{ z+y_1\mathbf{a}_1+y_2\mathbf{a}_2 \mid \mathbf{a}_1^Tz=\mathbf{a}_2^Tz=0, \; -1 \leq y_1 \leq 1\}$ parallel to $\mathbf{a}_2$ and orthogonal to $S_1$
    \item $S_3 = \{ z+y_1\mathbf{a}_1+y_2\mathbf{a}_2 \mid \mathbf{a}_1^Tz=\mathbf{a}_2^Tz=0, \; -1 \leq y_2 \leq 1\}$ parallel to $\mathbf{a}_1$ and orthogonal to $S_1$
\end{enumerate} \textbf{Hint}: a vector $c_1$ that is in the plane that defined by $\mathbf{a}_1$ and $\mathbf{a}_2$, and orthogonal to $\mathbf{a}_2$ can be described by $ c_1 = \mathbf{a}_1 - \frac{\mathbf{a}_1^T\mathbf{a}_2}{\left \| \mathbf{a}_2 \right \|_2^2}\mathbf{a}_2$

For visualization you may use following script:
\begin{minted}
[frame=lines, framesep=2mm, baselinestretch=1.2,]
{matlab}
S = Polyhedron('A', A, 'b', b);
plot(S)
\end{minted}
\begin{figure}[H]
    \begin{center}
    \includegraphics[width=6cm]{polycons.png}
    \caption{S}\label{f:farthest-dis}
    \end{center}
\end{figure}

\section{Appendix A}

\begin{figure}[H]%
    \centering
    \subfloat[\centering \label{f:init_space}  ]{{\includegraphics[width=7.5cm]{init_pose.png} }}%
    \qquad
    \subfloat[\centering\label{f:end_pose} ]{{\includegraphics[width=8cm]{end_pose.png} }}%
    \subfloat[\centering\label{f:convex_opt} ]{{\includegraphics[width=8cm]{convex_opt.png} }}%
    \label{fig:example}\caption{How can we formulate the this trajectory planning as a optimization problem?} %
\end{figure}

\begin{equation}
     J(\Gamma) =  \xi_{obs}J_{obs}(\Gamma) + \xi_{smooth}J_{smooth}(\Gamma) + \xi_{soft}J_{soft}(\Gamma), \quad J_{soft}(\Gamma) = J_{v}(\Gamma) + J_{a}(\Gamma),
\end{equation} where $J_{soft}(\Gamma)$ is determined by soft limits on acceleration and velocity. $J_{smooth}(\Gamma)$ is defined by considering geometric information and/or minimizing snap and/or jerk. $\xi_{obs}J_{obs}(\Gamma)$ helps to avoid collision detection. So how can we define those sub objective functions? What kind of things to be consider? There are so much to think aren't they? Let's get to the problem formulation

\subsection{Convex Set and Convex Functions}
A set $\Omega \subseteq \mathbb{R}^n$ is convex if and only if the line segment between any two points in $\Omega$ lies in $\Omega$, i.e., $\forall x_1,x_2 \Omega$ and $0 \leq \lambda \leq 1$
\begin{equation}
    \lambda x_1 + (1-\lambda)x_2 \in \Omega
\end{equation}$ \lambda x_1 + (1-\lambda)x_2, \; \lambda \in [0,1]$ is called convex combination of $x_1$ and $x_2$. This can be generalized up to n points $\lambda_1 x_1 + ... + \lambda_n x_n, \; \lambda_1 + ...+ \lambda_n = 1$
\begin{figure}[H]
    \begin{center}
    \includegraphics[width=12cm]{convex_set_non.png}
    \caption{Some convex and nonconvex sets~\cite{boyd2004convex}}
    \end{center}
\end{figure}

A function $f:\mathbb{R}^n \rightarrow \mathbb{R}$ is a convex function if whose domain dom(f) is a convex set and $\forall x,y \in dom(f)$ and $0 \leq \lambda \leq 1$
\begin{equation}
    f(\lambda x_1 + (1-\lambda)x_2) \leq \lambda f(x_1) + (1-\lambda)f(x_2)
\end{equation} Geometrically, the line segments connecting $(x_1, f(x_1))$ to $(x_2, f(x_2))$ is sit above the graph, namely $epi f(x)$ of the function f, refer to Fig.~\ref{f:convex_fun}. Now we are ready to define given optimization problem as convex optimization problem as follows:
\begin{equation}
\begin{aligned}
\min_{} \quad & f(x)\\
\textrm{s.t.} \quad & x \in \Omega,\\
\end{aligned}
\end{equation} where f is a convex function and $\Omega$ is a convex set. Such a problem guarantee to have a global solution due to the convexity nature.

\begin{figure}[H]
    \begin{center}
    \includegraphics[width=12cm]{convex-fun.png}
    \caption{Definition of a convex function}\label{f:convex_fun}
    \end{center}
\end{figure}

\subsection{Some importance examples of convex sets}
\subsubsection{Hyperplanes and halfspaces}
Hyperplanes and halfspaces are extremely importance when defining the constrained set for optimization problems. A hyperplane is form as:
\begin{equation}
    Hyperplanes: \{ x|\; a^Tx =b\} \; (a\in \mathbb{R}^n, b \in \mathbb{R}, a\neq 0)
\end{equation} Geometrically, the hyperplanes can be interpreted as the set of points constant inner product to a given vector a (or normal vector) whereas b determine the offset from the origin to the hyperplane. For any point in the hyperplane, namely $x_0$, geometrical interpretation can be understood as \begin{equation}
     \{ x|\; a^T(x-x_0) = 0\} = x_0 + a^{\perp}\; (a\in \mathbb{R}^n, a\neq 0), \quad a^Tx_0 = b,
\end{equation} where $a^{\perp}$ denotes the orthogonal component of a, i.e., $a^{\perp}=\{ v | a^Tv = 0\}$. This interpretation is illustrated in Fig.~\ref{f:hyperplane_1}.

\begin{figure}[H]
    \begin{center}
    \includegraphics[width=6cm]{hyperplane_1.png}
    \caption{Hyperplane in $\mathbb{R}^2$, with normal vector a and a point $x_0$. THe darker arrow depicts the any point x in the hyperplane, $x-x_0$ ~\cite{boyd2004convex}}\label{f:hyperplane_1}
    \end{center}
\end{figure}

A hyperplanecan be devided into two halfspaces. A closed halfspace is a set of the from
\begin{equation}
    \begin{aligned}
     \{x | a^Tx \leq b\}, \quad a \neq 0, or\\
      \{x | a^T(x-x_0) \leq 0\}, \quad a \neq 0, a^Tx_0 = b\\
     \end{aligned}
\end{equation}. Halfspaces are convex but not affine (Fig.~\ref{f:hyperplane_2})

\begin{figure}[H]
    \begin{center}
    \includegraphics[width=6cm]{hyperplane_2.png}
    \caption{A halfspace determined by $a^Tx \leq b$ ~\cite{boyd2004convex}}\label{f:hyperplane_2}
    \end{center}
\end{figure} The halfspace consists of $x_0$ plus any vector that makes an obtues (or right) angle with the vector a as shown in Fig.~\ref{f:hyperplane_3}.
\begin{figure}[H]
    \begin{center}
    \includegraphics[width=6cm]{hyperplane_3.png}
    \caption{The vector $x_2-x_0$ makes an obtuse angle with a, whereas the vector $x_1-x_0$ makes an acute angle with a. Hence, both $x_1$ and $x_2$ can not be in the halfspace ~\cite{boyd2004convex}}\label{f:hyperplane_3}
    \end{center}
\end{figure}
What is the different between plane and hyperplane? What is the different between hyperplane and halfspace?

\subsection{Euclidean balls and ellipsoids}
A ball in $\mathbf{R}^n$ has the form
\begin{equation}
    Ball = B(x_c, r) = \{x| \left \| x-x_c \right \|_2 \leq r\} = \{x|(x-x_c)^T(x-x_c) \leq r^2\} = \{x_c+ru| \left \| u \right \|_2 \}\; r>0
\end{equation} Can you try to show ball is a convex set?
A ellipsoid in $x^n \in \mathbf{R}^n$ has the form
\begin{equation}\label{ellipsoid}
    Ellipsoid = \{x | (x-x_c)^TP^{-1}(x-x_c) \leq 1\} = \{x_c+Au|\left \| u \right \|_2 \leq 1 \}\; P=P^T \succ 0
\end{equation} where A is square and nonsingular ($A=P^{1/2}$). The matrix P determines the how far the ellipsoid extends in every direction from $x_c$ whose length is determined by $\sqrt{\lambda_i}$, where $\lambda_i$ is the $i^{th}$ eiegen value. Ellipsoid can be a ball when $P=r^2I$.

\subsection{Polyhedra}
Given the halfspace representation (H-rep), i.e., $Ax \leq b$, corresponding representation is defined in three different ways: Polyhedron ($P = \{x|Ax\leq b\}$), Polyhedral cone ($P={x|Ax\leq 0}$), and Polytope ($P=\{Ax\leq 1\}$). A polyhedron has the form
\begin{equation}
    Polyhedron = \{x|a_jx^T \leq b_j, j=1,...,m, c_j^Tx =d_j, j=1,...,p\} = \{x|Ax \preceq b, Cx=d\}.
\end{equation} Hence, polyhedron is the intersection of a set of halfspaces and hyperplanes. In general, subspaces, hyplerplanes line segments halfspaces are all polyhedra. Let's try to visualize polyhedron, considering following constraints:
\begin{equation}\label{polyconst}
    A = \begin{bmatrix}
 -0.2936  & -1.3260 \\
    0.8245  & -1.4999 \\
    0.1941  &  1.0160 \\
    0.2977  & -0.0275 \\
   -0.7101  & -0.1604 \\
   -0.6877  &  0.3788 \\
    0.5728  & -0.1072 \\
    0.4452  &  0.2128
\end{bmatrix}, b = \begin{bmatrix}
2.0970 \\
    0.2372 \\
    1.5282 \\
    1.4607 \\
    2.5676 \\
    1.8432 \\
    0.3522 \\
    1.8206
\end{bmatrix}
\end{equation}

\begin{figure}[H]
    \begin{center}
    \includegraphics[width=10cm]{polyhedron.png}
    \caption{The polyhedron with respect to (\ref{polyconst})}\label{f:polyhedron}
    \end{center}
\end{figure}

\begin{minted}
[frame=lines, framesep=2mm, baselinestretch=1.2,]
{matlab}
A = randn(8,2);
b = 3*rand(8,1);
P = polytope(A,b);
plot(P);
\end{minted}

Let's try to define polyhedron for following constraints set:
\begin{equation}
P = \left\{\begin{matrix}
6x+y \leq 11\\
8x+2y \leq 29 \\
x-y \geq 11\\
2x+y \leq -4\\
y \leq 2\\
y < 21
\end{matrix}\right.
\end{equation} you may use $P = Polyhedron( 'A', [*], 'b', [*], 'Ae', [*], 'be', *)
$~\cite{poly_ref} notation to define the polyhedron
\subsection{Convex hull of polyhedra}
The convex hull of the given a set of polyhedron is defined as follows:
\begin{equation}
    conv\{v_1,...,v_k\} = \{\theta_1 v_1+...+\theta_k v_k | \theta \succeq 0, 1^T\theta = 1\}
\end{equation}

Let's say you are given a set of points (or vertices) and task is to construct the convex hull with respect to those vertices V and corresponding faces F
\begin{equation}
    V = \begin{bmatrix}
14.2347  & 12.5802 &  13.1171 \\
   12.7639 &  13.2543 &  15.3665 \\
   16.4311  & 15.7984  & 16.0939 \\
   12.1569 &  16.4915 &  16.7446
\end{bmatrix}, \quad F = \begin{bmatrix}
1   &  2  &   4 \\
     1   &  3  &   2 \\
     1  &   4  &   3 \\
     2  &   3  &   4
\end{bmatrix}
\end{equation}

\begin{figure}[H]
    \begin{center}
    \includegraphics[width=10cm]{convex_hull.png}
    \caption{}\label{f:convex_hull}
    \end{center}
\end{figure}

\begin{minted}
[frame=lines, framesep=2mm, baselinestretch=1.2,]
{matlab}
V = (5).*rand(4,3) + 12;
F = convhull(V);
S.Vertices = V;
S.Faces = F;
S.FaceVertexCData = jet(size(V,1));
S.FaceColor = 'interp';
patch(S);
\end{minted}

\subsection{Let's try some operations that preserve convexity}
Minkowski sum or different is interesting property for convex sets. There are various applications in which Minkowski sum is being used. When defining terminal constraints set in  Robust Model Predictive Control (RMPC), Minkowski sum is quite useful. Further, in motion planning, to differentiate free-space from obstacle space this sum is being used. Moreover, in collision detection also this property can be used.

\begin{minted}
[frame=lines, framesep=2mm, baselinestretch=1.2,]
{matlab}
A = randn(10,2);
b = 3*rand(10,1);
P = polytope(A,b);
hold on
plot(P, 'r');

E = randn(10,2);
f = 0.1*rand(10,1);
S = polytope(E,f);
plot(P-S,'g');
\end{minted}

\begin{figure}[H]
    \begin{center}
    \includegraphics[width=10cm]{minkowski_diff.png}
    \caption{Minkowski difference between two polyhedrons, namely P and E}\label{f:minkowski_diff}
    \end{center}
\end{figure}

Let's try to calculate the center corresponds to inscribing largest ball inside the polyhedron. The ball is described as $\{x | \left \| x-x_c \right \|_2 \leq r\}$.

\printbibliography
\end{document}

Practice 2021/Least Squares Estimation, Convex Operations , and Convex Programming/main.tex


\documentclass[24pt, a4]{article}
% To compile latex inside of vim
% ! clear; pdlatex name_of_file.tex or "%"

% \usepackage[utf8]{inputenc}
\usepackage[parfill]{parskip}
\usepackage{listings}
\usepackage{geometry}

\title{Binary Search}
\author{Mustafa Muhammad}
\date{10th October 2021}

\begin{document}

\maketitle

\newpage

List of Questions:
\begin{enumerate}
    \item Binary search
    \item Binary search on reverse sorted array
    \item Order not known search
    \item First and last occurent of an element
    \item Count of an element in a sorted array
    \item Find floor of an element in a sorted array
    \item Ceil of an element in a sorted array
    \item Next alphabetical element
    \item Find position of an element in an infinite sorted array
    \item Index of first in sorted infite array (binary)
    \item Minimum difference element in a sorted array
    \item Peak element
    \item Find maximum element in a bitonic array
    \item Search an element in a bitonic array
    \item Search in row and column wise sorted array
    \item Allocate minimum number of pages
    \end{enumerate}

\newpage
\section{Identification}
If you see that the given array is sorted, think binary search.
\begin{lstlisting}
def searchInSorted(self,arr, N, K):
        left = 0
        right = len(arr)-1

        while(left <= right):
            mid = int(left + (right-left)/2)

            if arr[mid] == K:
                return 1

            elif arr[mid] < K:
                left = mid+1

            else:
                right = mid -1

        return -1
\end{lstlisting}

\section{Binary search - Reverse Sorted Array}

\begin{lstlisting}
def binary_reverse_search(arr, target):
    left = 0
    right = len(arr)-1

    while(left <= right):
        mid = int(left+(right-left)/2)

        if arr[mid] < target:
            right = mid - 1

        elif arr[mid] > target:
            left = mid + 1

        else:
            return mid

    return -1
\end{lstlisting}
\newpage

\section{Order Not Known}

We can check the difference between the first and last element. If the first
element is smaller than the last, we do normal binary search, else we simply
reverse the order.

\section{First and Last Occurence of An element}

We check the first occurence of an element by finding the occurence of an
element and then proceeding to go left.
Similarly we find the last occurence of an element by going right once found.

\begin{lstlisting}
def find(arr,n,x):
    # code here
    def first_occurence(arr, x):
        left = 0
        right = len(arr)-1
        res = -1
        while(left <= right):
            mid = int(left+ (right-left)/2)
            if arr[mid] == x:
                res = mid
                right = mid - 1
            elif arr[mid] < x:
                left = mid+1
            elif arr[mid] > x:
                right = mid - 1
        return res
    def last_occurence(arr, x):
        left = 0
        right = len(arr)-1
        res = -1
        while(left <= right):
            mid = int(left+ (right-left)/2)
            if arr[mid] == x:
                res = mid
                left = mid + 1
            elif arr[mid] < x:
                left = mid+1
            elif arr[mid] > x:
                right = mid - 1
        return res
    results = []
    results.append(first_occurence(arr, x))
    results.append(last_occurence(arr, x))
    return results
\end{lstlisting}
\newpage
\section{Count of an element in a sorted array}
We find the first occurence of an element, then we find the last occurence of
an element. Formula for count = Last - FirstOccurence + 1 if neither element
is -1, else we return 0.

\section{Number of times a sorted array is rotated}
Input: nums = [3,4,5,1,2]

Output: 1

Explanation: The original array was [1,2,3,4,5] rotated 3 times. We know this
because the minimum elements index is 3. We perform binary search to find the
smallest element. Its index in the number of rotations.

\begin{lstlisting}
class Solution:
    def findMin(self, nums: List[int]) -> int:
        # minimum in rotated sorted array
        left = 0
        right = len(nums)-1
        n = len(nums)

        while left <= right:
            mid = int(left + (right-left)/2)

            # To avoid index out of bounds
            next_ = (mid+1)%n
            prev = (mid-1+n)%n

            # Minimum element will be smaller than both its neighbors.
            if nums[mid]<= nums[next_] and nums[mid] <= nums[prev]:
                return nums[mid]

            #We go towards the unsorted array (location of min element)

            elif nums[0] <= nums[mid]:
                left = mid+1

            elif nums[mid] <= nums[-1]:
                right = mid -1

        return nums[0]
\end{lstlisting}

\newpage
\section{Find an element in a rotated sorted array}

Building upon the previous question on rotation. We find the minimum index
and divide the array in half and apply binary search on both sides.

\begin{lstlisting}
class Solution:
    def search(self, nums: List[int], target: int) -> int:
        # find minimum index , then binary search on both sides
        def find_index(nums):
            left = 0
            right = len(nums)-1
            n = len(nums)
            while left <= right:
                mid = int(left + (right-left)/2)
                next_ = (mid+1)%n
                prev = (mid-1+n)%n

                if nums[mid]<=nums[next_] and nums[mid]<=nums[prev]:
                    return mid

                elif nums[0] <= nums[mid]:
                    left = mid + 1

                elif nums[mid] <= nums[-1]:
                    right = mid -1

            return -1

        def binary_search(nums, target):
            left = 0
            right = len(nums)-1
            while(left <= right):
                mid = int(left + (right-left)/2)
                if nums[mid] == target:
                    return mid
                elif nums[mid] < target:
                    left = mid + 1
                elif nums[mid] > target:
                    right = mid -1
            return -1
        min_index = find_index(nums)
        left_arr = binary_search(nums[0:min_index], target)
        right_arr = binary_search(nums[min_index:], target)

        return right_arr+len(nums[0:min_index]) if right_arr != -1 else left_arr
\end{lstlisting}
\newpage
\section{Searching in a nearly sorted array}
\begin{lstlisting}
class Solution:
    def binary_search_on_nearly_sorted_array(arr, target):
        left = 0
        right = len(arr)-1

        while left <= right:
            mid = int(left + (right-left)/2)
            if arr[mid] == target:
                return mid
            elif mid-1 >= left and arr[mid-1] == target:
                return mid-1
            elif mid+1 <= right and arr[mid+1] == target:
                return mid+1
            elif arr[mid] < target:
                left = mid+2
            elif arr[mid] > target:
                right = mid -2
        return -1
\end{lstlisting}
\newpage
\section{Find floor of an element in a sorted array}
Input:

N = 7, x = 5

arr[] = {1,2,8,10,11,12,19}

Output: 1

Explanation: Largest Number less than 5 is
2 (i.e K = 2), whose index is 1(0-based
indexing).
\begin{lstlisting}
class Solution:
    def findFloor(self,arr,N,X):
            #Your code here
            left = 0
            right = len(arr)-1
            res = -1
            while left <= right:
                mid = int(left + (right-left)/2)
                if arr[mid] == X:
                    return mid
                if arr[mid] < X:
                    res = mid
                    left = mid + 1

                elif arr[mid] > X:
                    right = mid - 1
            return res
\end{lstlisting}

\newpage
\section{Find ceil of an element in a sorted array}
For example, let the input array be {1, 2, 8, 10, 10, 12, 19}

For x = 0:    floor doesn't exist in array,  ceil  = 1

For x = 1:    floor  = 1,  ceil  = 1

For x = 5:    floor  = 2,  ceil  = 8

For x = 20:   floor  = 19,  ceil doesn't exist in array
\begin{lstlisting}
class Solution:
    def findFloor(self,arr,N,X):
            #Your code here
            left = 0
            right = len(arr)-1
            res = -1
            while left <= right:
                mid = int(left + (right-left)/2)
                if arr[mid] == X:
                    return mid
                if arr[mid] < X:
                    left = mid + 1
                elif arr[mid] > X:
                    res = mid
                    right = mid - 1
            return res
\end{lstlisting}

\section{Next alphabetical element}
Given an array of letters sorted in ascending order, find the smallest letter
in the the array which is greater than a given key letter.
Problem is very similar to ceil, as in this particular problem we just find
the next possible alphabet.

Changes:
1. Alphabets instead of numbers
2. Even if we find the target alphabet, we return the next alphabet.
\begin{lstlisting}
if arr[mid] == key:
    left  = mid+1
\end{lstlisting}

\newpage
\section{Find position of an element in an infinite sorted array}
\begin{lstlisting}
class Solution:
    def binary_search_on_inf(arr, target):
        start = 0
        end = 1

        # Moving up the end index and making start the previous end
        while arr[end] < target:
            start = end
            end = end*2

        while start <= end:
            mid = int(left + (right-left)/2)
            if arr[mid] == target:
                return mid
            elif arr[mid] < target:
                left = mid + 1
            elif arr[mid] > target:
                right = mid -1
        return -1
\end{lstlisting}

\newpage
\section{Index of First "1" in an Infinite 0/1 Sorted Arr}

Combination of binary search on an infinite array and first occurence of an
element.
\begin{lstlisting}
class Solution:
    def binary_search_on_inf(arr):
        start = 0
        end = 1

        # Moving up the end index and making start the previous end
        while arr[end] != 0:
            start = end
            end = end*2

        res = -1
        while start <= end:
            mid = int(left + (right-left)/2)
            if arr[mid] == target:
                res = mid
                right = mid - 1
            elif arr[mid] < target:
                left = mid + 1
            elif arr[mid] > target:
                right = mid -1
        return res
\end{lstlisting}

\newpage
\section{Minimum Difference Element in a Sorted Array}

We run binary search normally as we do, in the end, left and right will be
in between the key value, all we have to do is return the value with the min.
difference between the two.

Another way to think about this is to find the ceil and floor of the array
using binary search.

\begin{lstlisting}
class Solution:
    def binary_search(arr, target):
        start = 0
        end = 1
        while start <= end:
            mid = int(left + (right-left)/2)
            if arr[mid] == target:
                return mid
            elif arr[mid] < target:
                left = mid + 1
            elif arr[mid] > target:
                right = mid -1
        return min(abs(target - arr[left]), abs(target-arr[right]))
\end{lstlisting}

\section{Binary search on answer}

There are some cases where the array is not sorted but we can still apply
binary search.

We need to find a criteria inorder to split the array and find our answer.

We also need to decide how to move between the array.

Best example of this Concept is "Peak Element"

\newpage
\section{Peak Element}
Input: nums = [1,2,3,1]

Output: 2

Explanation: 3 is a peak element and your function should return the
index number 2.

We use binary search. We find the peak with the condition
if nums[mid] > nums[mid+1] and nums[mid] > nums[mid-1]

We move in the direction of the larger value as there is a higher chance
of encountering our result in that area.

\begin{lstlisting}
class Solution:
    def findPeakElement(self, nums: List[int]) -> int:
        left = 0
        right = len(nums)-1
        if len(nums) == 1: return 0
        while left <= right:
            mid = int(left + (right-left)/2)
            if mid != 0 and mid != len(nums)-1:
                if nums[mid] > nums[mid-1] and nums[mid] > nums[mid+1]:
                    return mid
                elif nums[mid] < nums[mid+1]:
                    left = mid + 1
                elif nums[mid] < nums[mid-1]:
                    right = mid - 1
            if mid == 0:
                if nums[mid]>nums[1]:
                    return mid
                else:
                    return 1
            if mid == len(nums)-1:
                if nums[mid] > nums[mid-1]:
                    return mid
                else:
                    return mid-1
        return -1
\end{lstlisting}
\section{Find maximum in a bitonic array}
Given a bitonic array find the maximum value of the array. An array is said
to be bitonic if it has an increasing sequence of integers followed
immediately by a decreasing sequence of integers.

Hence this problem is the same as finding the peak element.

Input:
1 4 8 3 2

Output:
8
\newpage
\section{Search an element in a bitonic array}
This problem is a mix of both peak element and binary search. All the elements
before the peak are sorted in ascending order, while the one's after the peak
are in descending order. Hence we just need to find the peak index and after
excluding it, we run binary search on the two arrays, if the key is not the
peak.

\begin{lstlisting}
class Solution:
    def findPeakElement(self, nums: List[int]) -> int:
        left = 0
        right = len(nums)-1
        if len(nums) == 1: return 0
        while left <= right:
            mid = int(left + (right-left)/2)
            if mid != 0 and mid != len(nums)-1:
                if nums[mid] > nums[mid-1] and nums[mid] > nums[mid+1]:
                    return mid
                elif nums[mid] < nums[mid+1]:
                    left = mid + 1
                elif nums[mid] < nums[mid-1]:
                    right = mid - 1
            if mid == 0:
                if nums[mid]>nums[1]:
                    return mid
                else:
                    return 1
            if mid == len(nums)-1:
                if nums[mid] > nums[mid-1]:
                    return mid
                else:
                    return mid-1
        return -1

    def binary_search_asc(self, arr, target):
        start = 0
        end = len(arr)-1
        while start <= end:
            mid = int(left + (right-left)/2)
            if arr[mid] == target:
                return mid
            elif arr[mid] < target:
                left = mid+1
            elif arr[mid] > target:
                right = mid-1
        return -1

    def binary_search_desc(self, arr, target):
        start = 0
        end = len(arr)-1
        while start <= end:
            mid = int(left + (right-left)/2)
            if arr[mid] == target:
                return mid
            elif arr[mid] < target:
                right = mid -1

            elif arr[mid] > target:
                left = mid + 1
        return -1
\end{lstlisting}
\section{Search in row wise and column wise sorted array}
Write an efficient algorithm that searches for a target value in an m x n
integer matrix. The matrix has the following properties:

Integers in each row are sorted in ascending from left to right.
Integers in each column are sorted in ascending from top to bottom.

Input: matrix = [[1,4,7,11,15],[2,5,8,12,19],[3,6,9,16,22],[10,13,14,17,24]
,[18,21,23,26,30]], target = 5

Output: true
\begin{lstlisting}
class Solution:
    def searchMatrix(self, matrix: List[List[int]], target: int) -> bool:
        i = 0
        j = len(matrix[0])-1
        while i>=0 and j>=0 and i<= len(matrix)-1 and j<= len(matrix[0])-1:
            if matrix[i][j] == target:
                return True
            elif matrix[i][j] > target:
                j-=1
            elif matrix[i][j] < target:
                i+=1
        return False
\end{lstlisting}
\newpage
\section{Allocate minimum number of pages}

You are given N number of books. Every ith book has Ai number of pages.
You have to allocate contagious books to M number of students. There can be
many ways or permutations to do so. In each permutation, one of the M
students will be allocated the maximum number of pages. Out of all these
permutations, the task is to find that particular permutation in which the
maximum number of pages allocated to a student is minimum of those in all
the other permutations and print this minimum value.

Each book will be allocated to exactly one student. Each student has to be
allocated at least one book.

Note: Return -1 if a valid assignment is not possible, and allotment should
be in contiguous order (see the explanation for better understanding).

Input:

N = 4

A[] = {12,34,67,90}

M = 2

Output:

113
Explanation:

Allocation can be done in following ways:
{12} and {34, 67, 90} Maximum Pages = 191

{12, 34} and {67, 90} Maximum Pages = 157

{12, 34, 67} and {90}  Maximum Pages =113

Therefore, the minimum of these cases is

113, which is selected as the output.

Main challenge is to create the is\_valid function, rest of the problem is
trivial.

\newpage
\begin{lstlisting}
class Solution:
    def findPages(self,A, N, M):
        start = max(A)
        end = sum(A)
        res = -1

        def is_valid(arr, k, mid):
            student = 1
            curr_sum = 0
            for i in range(0, len(arr)):
                curr_sum += arr[i]
                if curr_sum > mid:
                    student+= 1
                    curr_sum = arr[i]

            if student > k:
                return False
            return True

        while start <= end:
            mid = int(start + (end-start)/2)
            if is_valid(A, M, mid):
                res = mid
                end = mid - 1
            else:
                start = mid + 1
        return res
\end{lstlisting}
\end{document}


\section{Sensitivity Analysis for the Structural Behavioral Model} \label{sec:3}
\thispagestyle{plain} % surpress header on first page

In this chapter, we introduce the basic concepts, terminologies, methods of GSA in Section \ref{sec:3.1}, including the general model structure and the input factors. Then we demonstrate the mathematical descriptions of the variance-based and quantile-based sensitivity measures in Section \ref{sec:3.2} and Section \ref{sec:3.3}.

\subsection{Global sensitivity analysis: the framework} \label{sec:3.1}

The sensitivity analysis is “The study of how the uncertainty in the output of a model (numerical or otherwise) can be apportioned to different sources of uncertainty in the model input”\citep{saltelli2004SensitivityAnalysisPractice}. Thus, the definition of sensitivity analysis includes models, model inputs and model outputs. Throughout this work, a structural economic model will be regarded as a general function that defines a relationship between inputs and output(s):

\begin{equation} \label{eq:6}
\mathcal{M}: \theta \mapsto y=\mathcal{M}(\theta)
\end{equation}

\noindent
where $\boldsymbol{\theta} \in \Theta \subset \mathbb{R}^{d}$ is a vector of model parameters. The output of interest is a vector $y$. To produce a counterfactual prediction, a policy $g \in \mathcal{G}$ changes the mapping to $\mathcal{M}_g(\theta)$\citep{eisenhauer2021StructuralModelsPolicymaking}. Consequently, the differences between the prediction before and after the policy intervention yield a structural estimate of policy effects. \\

\noindent
Unlike LSA which analyzes how a tiny change near an input space value affects the scalar output, GSA identifies such effects in the whole input space. To be more precise, each input parameter is treated as a random variable assigned with a distribution of all possible values. Consequently, the uncertainty coming from model parameters is transmitted through the model to generate an empirical distribution of the output of interest $Y = M(\Theta)$ with a joint probability density function $f_Y(y)$. So far, this is how uncertainty propagation occurs. \\

\noindent
Once uncertainty propagation characterizes the uncertainty of the model output, a sensitivity analysis then can be applied to identify which parameters are primarily responsible for the variability of output. \\

\noindent
To date, a wide range of GSA methods has been developed. In this paper, we focus on variance-based sensitivity measures\citep{sobol1993SensitivityEstimatesNonlinear} and quantile-based sensitivity measures\citep{kucherenko2019QuantileBasedGlobal}. A comprehensive discussion and comparison of the GSA methods can be found in \cite{razavi2021FutureSensitivityAnalysis}.

\subsection{Variance-based sensitivity measures}  \label{sec:3.2}

The model function $Y=f\left(\theta_{1}, \ldots, \theta_{d}\right)$ is defined in $d$-dimensional real coordinate space $R^d$ with an input vector $\boldsymbol{\theta}=(\theta_1, \dots, \theta_{d})$.Note that $\mathbf{\theta}$ is a random variable with a continuous probability distribution function(PDF). To quantify the effect of variations of $\theta$ on the variation of $Y$, let us consider the conditional expectation  $\mathrm{E}\left[Y \mid \Theta_{i}=\theta_{i}\right]$. It is the mean value of output $Y$ over the probability distribution of  $\Theta_k(k \neq i)$ with the condition that $\Theta_i$ is fixed to $\theta_i$. If we consider the variations of $\Theta_i$,  the associated random variable is  $\mathrm{E}\left[Y \mid \Theta_{i}\right]$, whose variance quantifies the effect of $\Theta_i$ on the variation of Y. \\

\noindent
According to the result by \cite{sobol1993SensitivityEstimatesNonlinear}, given $k$ input variables mutually independent, the variance of the output can be decompose as the sum of variances of increasing order:

\begin{equation} \label{eq:7}
\operatorname{Var}(Y)=\sum_{i} V_{i}+\sum_{i j} V_{i j}+\sum_{i j h} V_{i j k}+\cdots+V_{1,2}, k
\end{equation}

\noindent
where $V_i = \operatorname{Var}(E(Y \mid \Theta_i))$ is first order variance and $V_{ij} = \operatorname{Var}(E(Y \mid \Theta_i,\Theta_j))$ is second order variance, etc. Note that in an additive model, this decomposition includes only first order variances. \\

\noindent
Thus, \textit{first-order indice}s of input parameter $\Theta_i$ is defined as:

\begin{equation} \label{eq:8}
S_i = \frac{V_i}{\operatorname{Var}(Y)}=\frac{\operatorname{Var}(E(Y \mid \Theta_i))}{\operatorname{Var}(Y)}
\end{equation}

\noindent
Accordingly, \textit{second-order indices} of input parameter $\Theta_i$ and $\Theta_j$ is defined as:

\begin{equation} \label{eq:9}
S_{ij} = \frac{V_{ij}}{\operatorname{Var}(Y)}=\frac{\operatorname{Var}(E(Y \mid \Theta_i,\Theta_j))}{\operatorname{Var}(Y)}
\end{equation}

\noindent
First-order indices measures the proportion of the total variance which is due to the main effect of $\Theta_i$ on Y. Whereas, second-order index measures the proportion of the total variance which is explained by the interaction between the two inputs. \\

\noindent
In 1996, \cite{homma1996ImportanceMeasuresGlobal} introduced the \textit{total variance index} which measures the proportion of the total variance due to the
main effect of $\Theta$, and all its interactions with the other inputs:

\begin{equation} \label{eq:10}
S_{Ti}=\frac{\sum_{i} V_{i}+\sum_{j h} V_{i j h}+\cdots}{\operatorname{Var}(Y)} = \frac{E(Var(Y \mid \Theta_{\sim i}))}{Var(Y)}=1-\frac{Var(E(Y \mid \Theta _{\sim i})}{Var(Y)}
\end{equation}

\noindent
where $\Theta_{\sim i}=(\Theta_1, \Theta_2, \dots, \Theta_{i-1}, \Theta_{i+1}, \dots ,  \Theta_k)$ \\

\noindent
There are three features of total variance index to be aware of. Firstly, The condition $S_{Ti}=0$ is necessary and sufficient for $\Theta_i$ to be a non-influential input(it can be treated as a fixed input). Secondly, if $S_{Ti} \approx S_i$ the interaction between $\Theta_i$ and the other inputs does not affect the variability of the output. Lastly, it is obvious that the sum of the total indexes is in general greater than \cite{homma1996ImportanceMeasuresGlobal}.

\subsection{Quantile-based sensitivity measures}  \label{sec:3.3}

Now we consider the scenarios where only a specific range of output is important to analyst. For instance, $Y = M(\boldsymbol(\Theta) \leq a )$ or $Y = M(\boldsymbol(\Theta) \geq b$. We reformulate such problems to $\alpha$-th quantile of the output CDF $q_Y(\alpha)$:

\begin{equation} \label{eq:11}
\alpha=\int_{-\infty}^{q_{Y}(\alpha)} \rho_{Y}(y) d y=P\left\{Y \leq q_{Y}(\alpha)\right\}
\end{equation}

\noindent
alternatively, to be more formal:

\begin{equation} \label{eq:12}
q_{Y}(\alpha)=F_{Y}^{-1}(\alpha)=\inf \{y \mid F(Y \leq y) \geq \alpha\}
\end{equation}

\noindent
where $\rho_Y(y)$ denotes the PDF of the output $Y$ and $F_Y(y)$ denotes to the CDF of the output $Y$.To solve such problems, \cite{kucherenko2019QuantileBasedGlobal} introduced QBSM $\bar{q}_{i}^{(1)}$ and $\bar{q}_{i}^{(2)}$

\begin{equation} \label{eq:13}
\bar{q}_{i}^{(1)}(\alpha)=E_{\theta_{i}}\left(\left|q_{Y}(\alpha)-q_{Y \mid \Theta_{i}}(\alpha)\right|\right)=\int\left|q_{Y}(\alpha)-q_{Y \mid \Theta_{i}}(\alpha)\right| d F_{\theta_{i}}
\end{equation}

\begin{equation} \label{eq:14}
\bar{q}_{i}^{(2)}(\alpha)=E_{\theta_{i}}\left[\left(q_{Y}(\alpha)-q_{Y \mid \Theta_{i}}(\alpha)\right)^{2}\right]=\int\left(q_{Y}(\alpha)-q_{Y \mid \Theta_{i}}(\alpha)\right)^{2} d F_{\theta_{i}}
\end{equation}

\noindent
Here,  $F_{\Theta_i}$ denotes to the CDF of input variable, $q_{Y \mid \Theta_{i}}({\alpha})$ denotes to the conditional PDF with $\Theta_{i}$ being fixed at $X_{i}=x_{i}^{\mbox{*}}$ \citep{song2021QuantileSensitivityMeasures}

\begin{equation}\label{eq:15}
q_{Y \mid \Theta_{i}}(\alpha)=F_{Y \mid \Theta_{i}}^{-1}(\alpha)=\inf \left\{P\left(Y \leq y \mid \Theta_{i}=\theta_{i}^{*}\right) \geq \alpha\right\}
\end{equation}

\noindent
Additionally, a normalized version of QBSM $Q_{i}^{(1)}(\alpha)$ and $Q_{i}^{(2)}(\alpha)$ was also presented in \cite{kucherenko2019QuantileBasedGlobal}:

\begin{equation}\label{eq:16}
Q_{i}^{(1)}(\alpha)=\frac{\bar{q}_{i}^{(1)}(\alpha)}{\sum_{j=1}^{d} \bar{q}_{j}^{(1)}(\alpha)}
\end{equation}

\begin{equation}\label{eq:17}
Q_{i}^{(2)}(\alpha)=\frac{\bar{q}_{i}^{(2)}(\alpha)}{\sum_{j=2}^{d} \bar{q}_{j}^{(2)}(\alpha)}
\end{equation}

\noindent
with $\left\{Q_{i}^{(1)}(\alpha), Q_{i}^{(2)}(\alpha)\right\} \in[0,1]$.


\section{Logical Paradoxes}
\newcommand{\starttoken}{\textsc{s}}
\newcommand{\finishtoken}{\textsc{f}}

In this section we provide proofs of some logical inconsistencies that arise when slight changes are made to the Iris logic.

\subsection{Saved Propositions without a Later}
\label{sec:saved-prop-no-later}

As a preparation for the proof about invariants in \Sref{app:section:invariants-without-a-later}, we show that omitting the later modality from a variant of \emph{saved propositions} leads to a contradiction.
Saved propositions have been introduced in prior work~\cite{dodds:higher-order-sync,iris2} to prove correctness of synchronization primitives; we will explain all that is necessary here.
The counterexample assumes a higher-order logic with separating conjunction, magic wand and the modalities $\always$ and $\upd$ satisfying the rules in \Sref{sec:base-logic}.

\begin{thm}
\label{thm:counterexample-1}
If there exists a type $\GName$ and a proposition $\_ \Mapsto \_ : \GName \to \Prop \to \Prop$ associating names $\gamma : \GName$ to propositions and satisfying:
\begin{align}
    \proves{}& \upd \Exists \gname : \GName. \gname \Mapsto P(\gname)
               \tagH{sprop-alloc} \\
    \gname \Mapsto P \proves{}& \always (\gname \Mapsto P)
               \tagH{sprop-persist} \\
    \gname \Mapsto \prop * \gname \Mapsto \propB \proves{}
             &
               \prop \Lra \propB
               \tagH{sprop-agree}
\end{align}
then $\proves\upd \FALSE$.
\end{thm}

The type $\GName$ should be thought of as the type of ``locations'' and $\gname \Mapsto P$ should be read as stating that location $\gname$ ``stores'' proposition $P$.
Notice that these are immutable locations, so the maps-to proposition is persistent.
The rule \ruleref{sprop-alloc} is then thought of as allocation, and the rule \ruleref{sprop-agree} states that a given location $\gname$ can only store \emph{one} proposition, so multiple witnesses covering the same location must agree.

%Compared to saved propositions in prior work, \ruleref{sprop-alloc} is stronger since the stored proposition can depend on the name being allocated.
%\derek{Can't we cut the above sentence?  This makes it sound like we are doing something weird that we ought not to be since prior work didn't do it.  But in fact, I thought that in our construction we do not really need to rely on this feature at all!  So I'm confused.}
The conclusion of \ruleref{sprop-agree} usually is guarded by a $\later$.
The point of this theorem is to show that said later is \emph{essential}, as removing it introduces inconsistency.
%
The key to proving \thmref{thm:counterexample-1} is the following proposition:
\begin{defn}
$A(\gname) \eqdef \Exists \prop : \Prop. \always\lnot \prop \land \gname \Mapsto \prop$.
\end{defn}
Intuitively, $A(\gname)$ says that the saved proposition named $\gname$ does \emph{not} hold, \ie we can disprove it.
Using \ruleref{sprop-persist}, it is immediate that $A(\gname)$ is persistent.

Now, by applying \ruleref{sprop-alloc} with $A$, we obtain a proof of $\prop \eqdef \gname \Mapsto A(\gname)$: this says that the proposition named $\gname$ is the proposition saying that it, itself, does not hold.
In other words, $\prop$ says that the proposition named $\gname$ expresses its own negation.
Unsurprisingly, that leads to a contradiction, as is shown in the following lemma:
\begin{lem}   \label{lem:saved-prop-counterexample-not-agname}   We have $\gname \Mapsto A(\gname) \proves \always\lnot A(\gname)$ and $\gname \Mapsto A(\gname) \proves A(\gname)$. \end{lem}
\begin{proof}%[\lemref{lem:saved-prop-counterexample-not-agname}]
\leavevmode
  \begin{itemize}
  \item First we show $\gname \Mapsto A(\gname) \proves \always\lnot A(\gname)$.
    Since $\gname \Mapsto A(\gname)$ is persistent it suffices to show $\gname \Mapsto A(\gname) \proves \lnot A(\gname)$.
    Suppose $\gname \Mapsto A(\gname)$ and $A(\gname)$.
    Then by definition of \(A\) there is a $\prop$ such that $\always \lnot \prop$ and $\gname \Mapsto \prop$.
    By \ruleref{sprop-agree} we have $\prop \Lra A(\gname)$ and so from $\lnot \prop$ we get $\lnot A(\gname)$, which leads to a contradiction with $A(\gname)$.

  \item Using the first item we can now prove $\gname \Mapsto A(\gname) \proves A(\gname)$.
    We need to prove
    \begin{align*}
      \Exists \prop : \Prop. \always \lnot \prop \land \gname \Mapsto \prop.
    \end{align*}
    We do so by picking $\prop$ to be $A(\gname)$, which leaves us to prove \(\always \lnot A(\gname) \land \gname \Mapsto A(\gname)\).
    The last conjunct holds by assumption, and the first conjunct follows from the previous item of this lemma.
  \end{itemize}
\end{proof}

With this lemma in hand, the proof of \thmref{thm:counterexample-1} is simple.
\begin{proof}[\thmref{thm:counterexample-1}]
  Using the previous lemmas we have
  \begin{align*}
    \proves \All \gname. \lnot (\gname \Mapsto A(\gname)).
  \end{align*}
  Together with the rule \ruleref{sprop-alloc} we thus derive $\upd \FALSE$.
\end{proof}

\subsection{Invariants without a Later}
\label{app:section:invariants-without-a-later}

Now we come to the main paradox: if we remove the $\later$ from \ruleref{inv-open}, the logic becomes inconsistent.
The theorem is stated as general as possible so that it also applies to previous, less powerful versions of Iris.

\begin{thm}
  \label{thm:counterexample-2}
  Assume a higher-order separation logic with $\always$ and an update modality with a binary mask ${\upd}_{\set{0,1}}$ (think: empty mask and full mask) satisfying strong monad rules with respect to separating conjunction and such that:
  \begin{mathpar}
    \inferhref{weaken-mask}{eq:update-weaken-mask}
    {}{{\upd}_0 \prop \proves {\upd}_1 \prop}
  \end{mathpar}

\noindent
  Assume a type $\InvName$ and a proposition $\knowInv{\cdot}{\cdot} : \InvName \to \Prop \to \Prop$ satisfying:
%
  \begin{mathpar}
    \inferhref{inv-alloc}{eq:inv-alloc}
    {}
    {\prop \proves {\upd}_1 \Exists \iname. \knowInv \iname \prop}
    \and
    \inferhref{inv-persist}{eq:inv-persistent}
    {}
    {\knowInv \iname \prop \proves \always \knowInv \iname \prop}
    \and
    \inferhref{inv-open-nolater}{eq:inv-open}
    {\prop * \propB \proves {\upd}_0 (\prop * \propC) }
    {\knowInv \iname \prop * \propB \proves {\upd}_1 \propC}
  \end{mathpar}

\noindent
  Finally, assume the existence of a type $\GName$ and two tokens $\ownGhost{\cdot}{\starttoken} : \GName \to \Prop$ and $\ownGhost{\cdot}{\finishtoken}: \GName \to \Prop$ parameterized by $\GName$ and satisfying the following properties:
  \begin{mathpar}
    \inferhref{start-alloc}{eq:start-alloc}
    {}{\proves {\upd}_0 \Exists \gname. \ownGhost \gname \starttoken}
    \and
    \inferhref{start-finish}{eq:start-finish}
    {}{\ownGhost \gname \starttoken \proves {\upd}_0 \ownGhost \gname \finishtoken}
    \and
    \inferhref{start-not-finished}{eq:start-not-finished}
    {}{\ownGhost \gname \starttoken * \ownGhost \gname \finishtoken \proves \FALSE}
    \and
    \inferhref{finished-dup}{eq:finished-dup}
    {}{\ownGhost \gname \finishtoken \proves \ownGhost \gname \finishtoken * \ownGhost \gname \finishtoken}
  \end{mathpar}

\noindent
  Then $\TRUE \proves{\upd}_1 \FALSE$.
\end{thm}

The core of the proof is defining the $\Mapsto$ from the previous counterexample using invariants.
Then, using the standard proof rules for invariants, we show that it satisfies \ruleref{sprop-alloc} and \ruleref{sprop-persist}.
Furthermore, assuming the rule for opening invariants without a $\later$, we can prove a slightly weaker version of \ruleref{sprop-agree}, which is sufficient for deriving a contradiction.

% Taking ${\upd}_0$ and ${\upd}_1$ to be the fancy update modalities $\pvs[\emptyset]$
% and $\pvs[\nat]$, respectively, we can see that Iris
% \emph{almost} satisfies these axioms.  First, to implement the tokens,
% we can use the RA with the carrier
% $\{\mundef,\epsilon,\starttoken,\finishtoken\}$ and operation
% $\epsilon \mtimes x = x \mtimes \epsilon = x$,
% $\finishtoken \mtimes \finishtoken = \finishtoken$ and otherwise
% $x \mtimes y = \mundef$.  Then, observe that the rules for
% $\knowInv{\cdot}{\cdot}$ are special cases of (derivable) invariant
% rules in Iris.  The fly in the ointment is the \ruleref{eq:inv-open}
% rule: in Iris, this rule would protect each occurrence of $\prop$
% in the premise of the rule with a $\later$, whereas here they are
% unprotected.

We start by defining $\Mapsto$ satisfying (almost) the assumptions of \lemref{lem:counterexample-invariants-saved-prop-agree}.
%
\begin{defn}
We define $\_ \Mapsto \_ : \GName \to \Prop \to \Prop$ as:
%
\begin{align*}
  \gname \Mapsto \prop \eqdef \Exists \iname. \knowInv \iname {\ownGhost \gname \starttoken \lor \ownGhost \gname \finishtoken * \always \prop}.
\end{align*}
\end{defn}
Note that using \ruleref{eq:inv-persistent}, it is immediate that $\gname \Mapsto \prop$ is persistent.

We use the tokens $\ownGhost \gname \starttoken$ and $\ownGhost \gname \finishtoken$ to model invariants that can be initialized ``lazily'': $\ownGhost \gname \starttoken$ indicates that the invariant is still not initialized, whereas the duplicable $\ownGhost \gname \finishtoken$ indicates it has been initialized with a resource satisfying $\prop$.%
%\footnote{We would usually require the token to be persistent, but it turns out the proof also works with the weaker assumption of duplicability.}
% RK: cut the footnote, it takes space. Maybe restore later

% TODO, explain this ...

We can show variants of \ruleref{sprop-agree} and \ruleref{sprop-alloc} for the defined $\Mapsto$.
\begin{lem}
  \label{lem:counterexample-invariants-saved-prop-alloc}
We have
  \(\proves {\upd}_1 \Exists \gname. \gname \Mapsto \prop(\gname)\).
\end{lem}
\begin{proof}
  We have to show the allocation rule \[\proves {\upd}_1 \Exists \gname. \gname \Mapsto \prop.\]
    From \ruleref{eq:start-alloc} we have a $\gname$ such that ${\upd}_0 \ownGhost \gname \starttoken$ holds and hence from \ruleref{eq:update-weaken-mask} we have ${\upd}_1\ownGhost\gname \starttoken$.
    Since we are proving a goal of the form ${\upd}_1 R$ we may assume $\ownGhost \gname \starttoken$.
    Thus for any $\prop$ we have ${\upd}_1\left(\ownGhost{\gname}{\starttoken} \lor \ownGhost \gname \finishtoken * \prop\right)$.
    Again since our goal is still of the form ${\upd}_1$ we may assume $\ownGhost{\gname}{\starttoken} \lor \ownGhost \gname \finishtoken * \always \prop$.
    The rule \ruleref{eq:inv-alloc} then gives us precisely what we need.
 \qed \end{proof}

%
\begin{lem}
\label{lem:counterexample-invariants-saved-prop-agree}
We have
  \(
  \gname \Mapsto \prop * \gname \Mapsto \propB * \always \prop \proves {\upd}_1 \always \propB
  \)
and thus
  \(
  \gname \Mapsto \prop * \gname \Mapsto \propB \proves ({\upd}_1 \always \prop) \Lra ({\upd}_1 \always \propB).
  \)
\end{lem}

\begin{proof}[\lemref{lem:counterexample-invariants-saved-prop-agree}]
\begin{itemize}
  \item We first show
    \[\gname \Mapsto \prop * \gname \Mapsto \propB * \always \prop \proves {\upd}_1 \always \propB.\]
    We use \ruleref{eq:inv-open} to open the invariant in $\gname \Mapsto \prop$ and consider two cases:
    %
    \begin{enumerate}
    \item $\ownGhost \gname \starttoken$(the invariant is ``uninitialized'') : In this case, we use \ruleref{eq:start-finish} to ``initialize'' the invariant and obtain $\ownGhost{\gname}{\finishtoken}$.
      Then we duplicate $\ownGhost \gname \finishtoken$, and use it together with $\always \prop$ to close the invariant.
    \item $\ownGhost \gname \finishtoken * \always \prop$ (the invariant is ``initialized''): In this case we duplicate $\ownGhost \gname \finishtoken$, and use a copy to close the invariant.
    \end{enumerate}
    %
    After closing the invariant, we have obtained $\ownGhost \gname \finishtoken$.
    Hence, it is sufficient to prove
    \[
      \ownGhost{\gname}{\finishtoken} * \gname \Mapsto \prop * \gname \Mapsto \propB * \always \prop \proves {\upd}_1 \always \propB.\]
    We proceed by using \ruleref{eq:inv-open} to open the other invariant in $\gname \Mapsto \propB$, and we again consider two cases:
    \begin{enumerate}
    \item $\ownGhost{\gname}{\starttoken}$ (the invariant is ``uninitialized''): As witnessed by \ruleref{eq:start-not-finished}, this cannot happen, so we derive a contradiction.
      Notice that this is a key point of the proof: because the two invariants ($\gname \Mapsto \prop$ and $\gname \Mapsto \propB$) \emph{share} the ghost name $\gname$, initializing one of them is enough to show that the other one has been initialized.
      Essentially, this is an indirect way of saying that really, we have been opening the same invariant two times.
    \item $\ownGhost{\gname}{\finishtoken} * \always \propB$ (the invariant is ``initialized''):
      Since $\always \propB$ is duplicable we use one copy to close the invariant, and retain another to prove ${\upd}_1 \always \propB$.
    \end{enumerate}
\item By applying the above twice, we easily obtain
\[ \gname \Mapsto \prop * \gname \Mapsto \propB \proves ({\upd}_1 \always \prop) \Lra ({\upd}_1 \always \propB) \]
\end{itemize}
\qed \end{proof}
% When allocating $\gname \Mapsto \prop(\gname)$ in \lemref{lem:counterexample-invariants-saved-prop-alloc}, we will start off in ``state'' $\ownGhost \gname \starttoken$, and once we have $P$ in \lemref{lem:counterexample-invariants-saved-prop-agree} we use \ruleref{eq:start-finish} to transition to $\ownGhost\gname \finishtoken$, obtaining ourselves a copy of said token.
% Finally, we use this token with $\gname \Mapsto \propB$ to obtain a proof of $\propB$.
Intuitively, \lemref{lem:counterexample-invariants-saved-prop-agree} shows that we can ``convert'' a proof from $\prop$ to $\propB$.

We are now in a position to replay the counterexample from \Sref{sec:saved-prop-no-later}.
The only difference is that because \lemref{lem:counterexample-invariants-saved-prop-agree} is slightly weaker than the rule \ruleref{sprop-agree} of \thmref{thm:counterexample-1}, we need to use ${\upd}_1 \FALSE$ in place of $\FALSE$ in the definition of the predicate $A$:
we let \(
  A(\gname) \eqdef \Exists \prop : \Prop. \always (\prop \Ra {\upd}_1 \FALSE) \land \gname \Mapsto \prop\)
and replay the proof that we have presented above.

%TODO: What about executing a view shift under a later?

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "iris"
%%% End:


\chapter{Results}
\label{cha:results}
\epigraph{
  This chapter will analyze the practical capabilities of the echo state
  network for anomaly detection in time series.  First we analyze the
  \emph{memory capacity} (MC) of the ESN and show how well it can predict
  a scalar chaotic time series. A tradeoff between networks with large MC and
  good capability to predict non-linearities is discussed before moving on to
  high-dimensional input images and sea surface height prediction.
}

\section{Short-term Memory}
\label{sec:short_term_memory}

The short-term memory capacity (\emph{MC}) of the internal state can be
estimated through the so-called coefficient of determination $R^2$  and a
simple experiment.  $R^2$ is also called the squared correlation coefficient.
For two variables $X$ and $Y$ it is defined by:
\begin{equation}
  \label{eq:detcoeff}
  R^2 = \text{detCoeff}(X, Y)
      = \frac{\text{cov}^2(X, Y)}{\sigma^2(X) \sigma^2(Y)}
\end{equation}

The network receives a random sequence $\mathbf{u}$ created from a uniform
distribution and is trained to extract the last $n$ inputs from the internal
state $\vt{x}$ with 20 units.  After training, the network extracts the last 40
time steps only from the last internal state
(Fig.~\ref{fig:random_timeseries_recovery}).
By creating $m$ sequences and collecting them in a matrix $U$
\begin{equation}
  U = \begin{bmatrix}
    u^{t=-n}_{0}  & \dots & u^{t=0}_{m} \\
    \vdots & \ddots & \vdots \\
    u^{t=-n}_{0}  & \dots & u^{t=0}_{m}
  \end{bmatrix}
\end{equation}
and the reconstructions of the network in a corresponding matrix $Y$, the
coefficient of determination can be calculated for each time step by treating
the columns of $U$ and $Y$ as the variables $X$ and $Y$.
The memory capacity can then be estimated by:
\begin{equation}
  MC = \sum_{-n < i < 0} \text{detCoeff}(U_i, Y_i),
\end{equation}

The determination coefficient between extracted and targeted values is one if
they are equal and zero if there is no correlation between them at all.
\begin{figure}
  \includegraphics[width=\linewidth]{mc_seq.pdf}
  \caption{Determination coefficient over time for various spectral radii of
  the reservoir and two different learning algorithms. On the left the network
  was trained with SGD and on the right with pseudo-inverse regression. The SGD
  trained network does not show a clear depedence on spectral radius, while in
  the LMS network it is clearly visible.}
  \label{fig:mc_seq}
\end{figure}

\begin{listing}
  \inputminted{json}{pseudocode/model_setups/memorize_setup.json}
  \label{lst:memorize_setup}
  \caption{ESN setup parameters for the memorization task. The \ttt{"False"}
    value of the Tikohnov regularization parameter $\beta$ means that the
    pseudo-inverse method was used. The spectral radius is varied from experiment
    to experiment.
  }
\end{listing}

To find the MC of the ESN it was trained with batches of $m=2000$ random
sequences.  Once with the Adam algorithm and once with the pseudo-inverse
method (which of course only need a single batch of random sequences to be
trained). After the training phase, another batch is fed to the network for
evaluation. An exemplary reconstruction is shown in
Fig.~\ref{fig:random_timeseries_recovery}.  It becomes evident that the most
recent time steps can be reconstructed almost perfectly and the memory of the
network becomes weaker further back in time.  With this setup one can try to
find the optimal spectral radius, which maximizes the memory capacity.  Both
plots in Fig.~\ref{fig:mc_seq} show the coefficient of determination over the
last 25 time steps at varying spectral radius $\rho$. Training
the ESN via a least mean squares approach (LMS, pseudo-inverse in this case)
seems to be much more efficient than training with Adam in this case.  Which is
also visible in the scaling of $\rho$, which has only a very small effect in
the Adam-optimized network, while it is clearly visible in the LMS ESN.
Figure~\ref{fig:mc_rho} suggests that MC increases as $\rho$ approaches one and
then quickly degenerates. This is thoroughly investigated computationally in a
paper by [\cite{farkavs2016}], which in essence confirms the conjectures that
were made before.

\begin{figure}
  \begin{minipage}[t]{.48\textwidth}
    \includegraphics[width=\linewidth]{random_timeseries_recovery.pdf}
    \caption{
      True vs. reconstructed random time series. Most recent time step at $t=0$.
    }
    \label{fig:random_timeseries_recovery}
  \end{minipage}
  \hspace{.02\textwidth}
  \begin{minipage}[t]{.48\textwidth}
    \includegraphics[width=\linewidth]{mc_rho.pdf}
    \caption{MC over spectral radius $\rho$ for the LMS and GD trained networks.}
    \label{fig:mc_rho}
  \end{minipage}
\end{figure}

\newpage
\section{Mackey-Glass System}%
\label{sec:res_mackey_glass_system}

\begin{figure}
  \centering
  \includegraphics[width=\linewidth]{mackey_esn_ba_var_loss.pdf}
  \caption{Hyper-parameters over MSE. The very large
    values are caused by diverging predictions that were clipped to a maximal
    $\mathrm{MSE}=10^{20}$. The weight initialization and the sparsity
    parameters yield good performance in a certain range, the spectral radius
  only allows good performance for values larger than one.}
  \label{fig:mackey_esn_ba_var_loss}
\end{figure}

The first actual prediction task that will be solved by the ESN is the one of
the Mackey-Glass system that was described in
section~\ref{sec:mackey_glass_system}.  The reservoir is initialized with 500
units, which should be more than enough given that the period of the
considered time series is below 100 time steps, and that the ESN should have a
memory capacity slightly smaller than 500 steps.  The ESN is fed with a
sequence of length 2200, which creates the same number of internal states. To
avoid transients in the internal state, the first 200 steps are discarded. The
remaining 2000 internal states can be used to train the readout layer
$\wmatr{out}$ via the pseudo-inverse method. Three hyper-parameters (HP) remain
to be set: spectral radius, reservoir density, and input weight initialization
parameter, which are found with the help of Bayesian Optimization (BO).  For
this purpose, the package \ttt{scikit-optimize} was used to minimize the
RMSE
\begin{equation}
  \text{RMSE} = \frac{1}{N} \sum_{i=0}^N || d_i - y_i ||_2
\end{equation}
over several predictions.  The scikit algorithm needs an interval for every
parameter to form an HP space that it can sample from, which where chosen as
specified in table~\ref{tab:mackey_bo}. The resulting optimal HPs were obtained
over five BO runs.  Each BO run evaluated 100 trained ESNs at different points
in the HP space (Fig.~\ref{fig:mackey_esn_ba_var_loss}).  A surprising result
is that the optimal spectral radius is significantly larger than one.
In fact, the predicted signal just converges towards a value close to the mean
of the series if the spectral radius is close to one.
\begin{table}[h]
  \centering
  \rowcolors{2}{gray!25}{white}
  \begin{tabular}{|l c c c c|}
    \hline \rowcolor{gray!50}
    Parameter        & Best              & Min  & Max & Prior \\ \hline
    Spectral radius  & $1.40$   & 0.5  & 2.0 & uniform \\
    Weight init.     & $0.48$   & 0.1  & 1.0 & uniform \\
    Density          & $0.03$   & 0.01 & 1.0 & logarithmic \\
    \hline
  \end{tabular}
  \caption{Hyper-parameter results obtained via Bayesian Optimization}
  \label{tab:mackey_bo}
\end{table}

This is peculiar, as section~\ref{sec:short_term_memory} showed a spectral
radius close to one to be optimal for the memorization of sequences.  It
suggests that the crucial ingredient to predicting chaotic time series might
not only be a very good knowledge of the past, but something else. In fact this
is were the memory non-linearity tradeoff that was mentioned in
section~\ref{sub:short_term_memory} comes into play.  The more non-linear a
prediction task becomes the higher must the spectral radius of the reservoir be
to perform well. Unfortunately a large spectral radius degrades the MC of the
ESN. A mathematically sound framework for reasoning about the memory
non-linearity tradeoff is yet to be developed, but a rigorous computational
analysis of the problem is carried out in an article by [\cite{verstraeten2010}].
For the purpose of an anomaly detection it is enough that we found good
parameters for the prediction task. The quality of the predictions is depicted
in Fig.~\ref{fig:mackey1d_predictions}.
The predictive power of the ESN varies slightly over different parts of the
Mackey-Glass system. By randomly choosing different starting points in the
sequence and creating predictions for the next 500 frames, we can evaluate the
average error that the ESN makes at each frame.
Fig.~\ref{fig:mackey1d_predictions} shows the best and worst predictions in the
two upper plots and the average deviation of prediction and target in the
bottom.  An anomaly detection might be feasible within the first 100 frames, an
anomaly that is caught beyond that point might as well be caused by a degraded
prediction.\\

\begin{figure}
  \centering
  \includegraphics[width=\linewidth]{mackey1d_predictions.pdf}
  \caption{The best (top) and worst (middle) predictions from 20 randomly
    chosen starting points of the Mackey-Glass series. All predictions were
    made by an ESN with optimal hyper-parameters.  The target values $d$ are
    shown in blue, prediction $y$ in yellow, and the error $e = |d - y|$ in
    green. The plot in the bottom shows the variability of $e$ over different
    predictions.
  }
  \label{fig:mackey1d_predictions}
\end{figure}

At this point, \emph{finally}, all necessary parts to try and find artificially
introduced anomalies in the MG system are in place. The anomalies are created
by varying the parameter $\gamma$ of the MG Eq.~\ref{eq:mackey_glass}.  By
default it has a value of $\gamma = 0.1$.  Every 400 steps it is decreased by
an amount $\delta$ set back to its initial value after 50 steps. Depending on
the magnitude of $\delta$ this creates rather smooth looking anomalies. Easily
visual anomalies are created with a value of $\delta = 0.05$. To search the
outliers, the trained network receives a sliding window of 300 steps of the
anomalous sequence.  The predictions for the next $N_p = 100$ steps into the
future are collected and the \emph{mean prediction error} (MPE) is calculated
for every sequence that was fed to the ESN:
\begin{equation}
  \text{MPE} = \frac{1}{N_p} \sum_{t=1}^{N_p} | d_t - y_t |.
\end{equation}
The MPE is recorded for the whole dataset and the normality score $\Sigma$
(Eq.~\ref{eq:normality_score}) is computed with a large window size of 100
steps and a small window size of 5 steps.  The result is shown in
Fig.~\ref{fig:mackey_visible_anomalies}. With delight we can find that all
anomalies are easily caught.  To demonstrate the capabilities of the ESN to
efficiently find outliers in chaotic systems like the MG system,
Fig.~\ref{fig:mackey_anomalies} depicts how the algorithm performs on
anomalies that would, most probably, not be spotted by a human.

\begin{figure}
  \centering
  \includegraphics[width=\linewidth]{mackey_visible_anomalies.pdf}
  \caption{Mackey Glass system with artificially introduced anomalies at every
    400th step with $\delta=0.05$. The detected anomalies are indicated by the
    yellow bars in the plots of the score $\Sigma$. The anomaly score catches
    the irregularly high error regions slightly too early, because the
    prediction error goes up as soon as the prediction window reaches into the
  anomalous regions.}
  \label{fig:mackey_visible_anomalies}
\end{figure}

\begin{figure}
  \centering
  \includegraphics[width=\linewidth]{mackey_anomalies.pdf}
  \caption{Outliers that would have probably not been caught by humans. Created
  with $\delta = 0.03$. One false positive at $t=1500$.}
  \label{fig:mackey_anomalies}
\end{figure}

\begin{listing}
  \inputminted{json}{pseudocode/model_setups/mackey_setup.json}
  \label{lst:mackey_setup}
  \caption{ESN setup parameters for MG prediction. The \texttt{DenseESNCell}
    implies that the reservoir matrix was created with a dense matrix representation
  }
\end{listing}

\clearpage
\section{Dansgaard-Oeschger Events}%
\label{sec:res_dansgaard_oeschger_events}

The second dataset consisting of ice-core records that reach back 100k years
has two components. The $\delta^{18}$O sequence, which is directly connected to
surface temperature and the Ca$^{2+}$ series, that measures dust content and
shows similar patterns.  Both sequences lack a clear reappearing pattern, which
was clearly visible in the Mackey-Glass series, despite the chaotic nature of
the MG system. This makes a one time training over a part of the dataset that
contains most of variability impossible.  It should however still be possible
to project several dozen years ahead by moving to an online learning mode. In
the online mode, the output weights are recomputed with every new time slice
that is fed to the network.  The resulting short-term predictions $y$ are shown
in Fig.~\ref{fig:d18O_anomalies}.
\begin{listing}
  \inputminted{json}{pseudocode/model_setups/DO_setup.json}
  \label{lst:DO_setup}
  \caption{ESN setup parameters for DO event detection. The hyper-parameters were
  found via Bayesian Optimization.}
\end{listing}
They were achieved by training the ESN for 1500 input steps and predicting the
next 200 steps (the time step of the series is one year). Due to the very
irregular dataset the predictions are not very good, but at points where the
sequences change abruptly they deviate much more from the target values than
they do otherwise. This can be exploited to detect the DO events which are
located exactly at these abruptly changing points. Apart from the online weight
adaption, the procedure is the same as with the MG system. The score is
calculated from the MPE and thresholded, resulting in the detection of
anomalies at the yellow colored regions of Fig.~\ref{fig:d18O_anomalies}.
Abrupt changes after steady periods are easily detected by the algorithm.  DO
events that lie close to each other tend to slip through. This is due to the
fact that the ESN learns from the previous DO events and as it is trained on
these anomalies, they are considered normal.
\begin{figure}
  \centering
  \includegraphics[width=\linewidth]{d18O_Ca2_anomalies.pdf}
  \caption{Results of the DO dataset. The DO events are marked with
  the grey background. Detected anomalies are indicated by the yellow bars in
  the plots of the normality score $\Sigma$}
  \label{fig:d18O_anomalies}
\end{figure}

\newpage
\section{Kuroshio}%
\label{sec:res_kuroshio}
\begin{figure}
  \begin{minipage}[b]{0.5\linewidth}
    \centering
    \includegraphics[width=.9\linewidth]{kuro_example_pred_stepidx200.pdf}
  \end{minipage}%%
  \begin{minipage}[b]{0.5\linewidth}
    \centering
    \includegraphics[width=.9\linewidth]{kuro_example_pred_stepidx470.pdf}
  \end{minipage}
  \begin{minipage}[b]{0.5\linewidth}
    \centering
    \includegraphics[width=.9\linewidth]{kuro_example_pred_stepidx0.pdf}
  \end{minipage}%%
  \begin{minipage}[b]{0.5\linewidth}
    \centering
    \includegraphics[width=.9\linewidth]{kuro_example_pred_stepidx100.pdf}
  \end{minipage}
  \caption{Four exemplary predictions and corresponding desired outputs in the
    Kuroshio region. Shown is the 150th prediction frame.}
  \label{fig:kuro_pred_examples}
\end{figure}

The prediction of the Kuroshio region involves significantly more data that
needs to be processed by the ESN. This means that the reservoir size has to
grow appropriately.  As a rule of thumb, the size of the internal state
$\vt{x}$ should be at least as big as the period of the dataset is long. The
most obvious period in climate simulations is typically the annual signal
which, in our case of 3-day means, is 122 steps long. To be able to remember
whole video sequences, this number should be multiplied by the number of pixels
in each frame to obtain the state size. Even if we consider raw inputs of 100 x 100
pixels and resample them to a size of 30 x 30, this would still result in a
state size greater than 100 000, which would result in a memory use of more
than 40 GB of the internal square reservoir matrix alone. This is clearly
unacceptable. Luckily, the pixels are highly correlated and we can hope that
the ESN is able to compress some of the information of the input images it
receives. Sizing the reservoir down to 10000 units and keeping it very sparse
will decrease the memory usage of the network to an acceptable amount of a few
hundred MB.

\begin{figure}[p]
  \centering
  \includegraphics[width=1.1\linewidth]{kuro_pred_stepidx290}
  \caption{Exemplary prediction 200 steps into the future.}
  \label{fig:kuro_pred_stepidx290}
\end{figure}

\begin{figure}[p]
  \centering
  \includegraphics[width=1.1\linewidth]{kuro_pred_stepidx20}
  \caption{Another prediction example. Here it looks like the internal state of
  the ESN has reached a fixed point, so the prediction remains constant after
  day 10100.  This fixed point produces predictions that are very similar to
  the mean contracted state of the Kuroshio, if viewed in 2D.}
  \label{fig:kuro_pred_stepidx20}
\end{figure}

Apart from using a sparsely represented ESN reservoir, the approach is again
the same as before.  The ESN receives 1300 input frames and predicts the next
200, which amounts to a prediction of roughly two years. Exemplary predictions
and the corresponding desired target outputs are shown in
Fig.~\ref{fig:kuro_pred_examples}. The figures ~\ref{fig:kuro_pred_stepidx290}
and ~\ref{fig:kuro_pred_stepidx20} show the 25th row of two more exemplary
target and prediction evolutions over time. The prediction error that is shown
is quite low for the predicted two years, but just to be sure only the first
year, meaning the first $N_p = 100$ frames, will be used for the subsequent
anomaly detection.

Fig.~\ref{fig:kuro_mean_pred} compares the target and prediction values for the
last prediction step ($N_p=100$) that is considered for the anomaly detection.
The error plot shows the MPE for each prediction. A large deviation of
prediction and target is visible around day 15500. The normality score (plotted
in a logarithmic color scale) that was calculated from the MPE sequence is very
low in this region as well. It seems to be a promising candidate for the
Kuroshio anomaly. By summing up all instances where the anomaly score $\Sigma <
0.01$ we can create a map of outliers. The result is shown in
Fig.~\ref{fig:kuro_anomaly_count}.  It shows a region with a lot of outliers
just where we expect them to be.  The similarity between the anomaly count and
the difference of elongated and contracted states is obvious and the result
will be regarded as a successful detection of the Kuroshio anomaly.

The second region of high anomaly counts is where the Kuroshio turns towards
the North Pacific basin. This anomaly is caused by a large unpredicted  eddy
that enters the analyzed area from the East. This anomalous region would
probably vanish if the analysis area is shifted eastward, away from Japan.
This would be the obvious next step of the outlier search, but unfortunately a
thorough analysis of the whole world is out of the scope of this work and will
remain as one path that a further studies of this problem could take.
\begin{figure}
  \centering
  \includegraphics[width=\linewidth]{kuro_anomaly_count.pdf}
  \caption{Count of all pixels with a normality score that is lower than 0.01,
  compared to the assumed true form of the Kuroshio anomaly (obtained from the
  difference of its two states).}
  \label{fig:kuro_anomaly_count}
\end{figure}

\begin{figure}
  \centering
  \includegraphics[width=1.1\linewidth]{kuro_mean_pred.pdf}
  \caption{The target plot shows the true values of one row of the SSH
    frames that were fed to the ESN.  In the prediction plot we can see the
    prediction that was made based on the internal state 100 steps before.
    The error plot depicts the mean error of one prediction sequence and in the
    bottom we can see the resulting anomaly score on a logarithmic color scale.
  }
  \label{fig:kuro_mean_pred}
\end{figure}

\begin{listing}
  \inputminted{json}{pseudocode/model_setups/kuro_setup.json}
  \label{lst:kuro_setup}
  \caption{ESN setup parameters for Kuroshio anomaly detection. The Bayesian
  Optimization resulted a large range of parameters with similar performance as
  long as the spectral radius was larger than 1.3 and the weight initialization
  parameter larger than 0.8. The chosen hyper-parameters reflect the need for a
  sufficiently non-linear reservoir.}
\end{listing}


\hypertarget{group__numpp__structures__matrices__sparse}{}\section{Sparse}
\label{group__numpp__structures__matrices__sparse}\index{Sparse@{Sparse}}

Module containg sparse matrix implementations.

\subsection*{Classes}
\begin{DoxyCompactItemize}
\item
class \hyperlink{classnumpp_1_1matrix_1_1sparse_1_1block}{numpp\+::matrix\+::sparse\+::block$<$ T, Rows, Columns $>$}
\begin{DoxyCompactList}\small\item\em Block version of sparse matrix. \end{DoxyCompactList}\item
class \hyperlink{classnumpp_1_1matrix_1_1sparse_1_1nested}{numpp\+::matrix\+::sparse\+::nested$<$ T, Rows, Columns $>$}
\begin{DoxyCompactList}\small\item\em Block nested version of sparse matrix. \end{DoxyCompactList}\end{DoxyCompactItemize}

\subsection{Detailed Description}
Module containg sparse matrix implementations.

\begin{DoxyWarning}{Warning}
Warnings about compilation times do not apply to blocked and nested\+\_\+block classes as those are runtime only
\end{DoxyWarning}


% !TEX root = ../main.tex

\section{Network models}

\subsection{Deterministic networks}

\begin{itemize}
  \item Complete graph $K_N$ on $N$
  \item Lattices (in $D$-dimensions): each node has 2D neighbours
\end{itemize}

\subsection{Measurements of networks}

\begin{itemize}
  \item Choose 8 persons
  \item Link = friendship
  \item Properties of friendship graphs?
\end{itemize}

Why do we need this? Reality is just a measurement. You have a lot of friendship networks,
it depends on the realization.

Motivation to study \emph{random graphs}
e.g. realizations of $G_{0.4}(8)$

\subsection{Complex network models}

\begin{itemize}
  \item Random Graph - Erdös-Rényi (1959-1960)
  \item Small-World Graph - Watts-Strogatz (1998). Introduces the idea of rewiring
  \item Scale-Free Graph - Barabási-Albert (1999)
\end{itemize}

\subsection{Erdös-Rényi random graph}
The ER random graph $G_p(N)$ is a graph with $N$ nodes and each node pair is connected
independently with probability $p$

Any so generated graph belongs to the class ER random graph $G_p(N)$ with same $N$ and $p$.

There are two variants of ER random graph:
\begin{itemize}
  \item $G_p(N)$ link existence probability $p$
  \item $G(N,L)$ precisely $L$ links?
\end{itemize}

How would you describe or deduce the adjacency matrix $A$ for $G_p(N)$?
$a_{ij}$ (with $j \ne i$) is a Bernoulli random variable with mean $p$

\begin{align*}
  Pr[a_{ij} = 1] &= p \\
  Pr[a_{ij} = 0] &= 1 - p \\
  E[a_{ij}] &= p
\end{align*}
A is a $N \times N$ matrix

The complement graph of $G_p(N)$ is $G_{1-p}(N)$

\begin{itemize}
  \item If $p=1$, then $G_1(N)$ is the complete graph $K_N$
  \item If $p=0$, then $G_0(N)$ is the empty or null graph
  \item The average number of links is
  \begin{align*}
    E[L] = \frac{N(N-1)}{2}p \implies p = \frac{L}{L_{\max}} = \text{link density}
  \end{align*}
  \item The average clustering coefficient is:
  \begin{align*}
    E[c_{G_p(N)}] = p
  \end{align*}
  \item Degree distribution: Binomial distribution
  \begin{align*}
    Pr[D_{rg} = k] &=
    \begin{pmatrix}
      N - 1 \\ k
    \end{pmatrix}
    p^k ( 1- p)^{N-1-k} \simeq
    \frac{1}{\sigma \sqrt{2\pi}}e^{-\left( \frac{(k-\mu)^2}{2\sigma^2} \right)} \\
    \mu &= E[D] = (N-1)p \\
    \sigma^2 &= Var[D] = (N-1)p(1 - p)
  \end{align*}
\end{itemize}

\subsubsection{Distribution of eigenvalues of adjacency matrix of $G_p(N)$}

Consider set of eigenvalues as realizations of eigenvalue random variable $\lambda$

Histogram approximates the probability density function $f_{\lambda}(x)$ of eigenvalues
of the adjacency matrix $A$. The largest eigenvalue of a ER graph is really large compared to
the other ones, the distance is called spectral value.

\subsubsection{Observations from the spectrum}

\begin{itemize}
  \item The probability density function $f_{\lambda}(x)$ of the eigenvalues of the adjacency
  matrix
  \begin{itemize}
    \item Peaks refer to a specific structure or pattern in the graph
    \item A broader, bell-shape form of $f_{\lambda}(x)$ around the origin ($x=0$) is
    a fingerprint of randomness
    \begin{itemize}
      \item Broadening of peaks in spectra to bell-shape forms i mostly due to randomness
    \end{itemize}
  \end{itemize}
  \item If $f_{\lambda}(-x) = f_{\lambda}(x)$ an even function of $x$, then the graph is a tree
  (no triangles, thus skewness $s_{\lambda} = 0$)
  \begin{itemize}
    \item Any tree can be represented by a bipartite graph
    \item A bipartite graph has a symmetric spectrum (for each eigenvalue $\lambda$, there
    exists an eigenvalue $-\lambda$)
  \end{itemize}
\end{itemize}

\subsection{Power law graphs}

Usually, we prefer normalized random variables with mean 0 and variance 1.

A power law degree distribution is also called scale-free:

Any number $a$ just multiplies the probability density; there is no characteristic length.
The mean is not representative, because the variance is large.

The simplest family of ``power law'' graphs have been proposed by Barbasi-Albert:

\begin{itemize}
  \item Start with $n << N$ nodes
  \item Attach a new node with $m$ links; each link to an already existing node
  randomly and proportionally to its degree
  \item Repeat 2) until size $N$ is reached
\end{itemize}

\subsubsection{Power law exponent $\tau$ in $Pr[D=k] \simeq ck^{-\tau}$}

\begin{itemize}
  \item Critical point 1
  \begin{itemize}
    \item $E[D]$ diverges
    \item $E[D^2]$ diverges
    \item $D_{max}$ grows faster than $N$
    \item $E[H] \sim $ const
  \end{itemize}
  \item Critical point 2 (needs to be fixed)
  \begin{itemize}
    \item $E[D]$ finite
    \item $E[D^2]$ diverges
    \item $D_{max}$ grows faster than $N$
    \item $E[H] \sim log(log N)$
  \end{itemize}
  \item Critical point 3 (needs to be fixed)
  \begin{itemize}
    \item $E[D]$ diverges
    \item $E[D^2]$ diverges
    \item $D_{max}$ grows faster than $N$
    \item $E[H] \sim $ const
  \end{itemize}
\end{itemize}

\subsection{Observed common properties of real-world complex networks}

\begin{itemize}
  \item small-world property
  \begin{itemize}
    \item Average length/hopcount of a path is short compared to the size $N$ of the network
    ($E[H] = O(log N))$
  \end{itemize}
  \item scale-free degree distribution
  \begin{itemize}
    \item heavy tails (non-Gaussian behavior)
  \end{itemize}
  \item clustering and community structure
  \begin{itemize}
    \item network of networks
  \end{itemize}
  \item robustness to random node failure
  \item vulnerability to targeted hub attacks and cascading failures
\end{itemize}


\section{Critical points of Energy in the sphere}