\documentclass[a4paper,10pt]{article}
\usepackage[utf8]{inputenc}

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{url}

\newcommand{\real}{\mathbb{R}}

\title{Some notes on Johnson Lindenstrauss and generalization}
\author{Nazarov Ivan}

\begin{document}
\maketitle

\section{Bounding the tail probability of $\chi^2_p$} % (fold)
\label{sec:bounding_the_tail_probability}

In this section we shall provide some bounds on the probability of deviations for
a $\chi^2_p$ random variable, which reflects the distribution of the squared
$\ell_2$ norm of any unit-norm vector, transformed by an iid Gaussain matrix.

% Chernoff's bound states that for any rv $y$ we have
% $$
%   \mathbb{P}\bigl( y \geq t \bigr)
%     % = \mathbb{P}\bigl( e^{\lambda y} \geq e^{\lambda t} \bigr)
%     \leq \inf_{\lambda > 0} e^{-\lambda t} \mathbb{E} e^{\lambda y}
%     \,. $$
% For $y \sim \chi^2_p$ the moment generating function $\mathbb{E} e^{\lambda y}$
% is given by $\lambda \mapsto (1 - 2\lambda)^{-\tfrac{p}2}$ for $\lambda \leq \tfrac12$.

\subsection{A bound from subexponentiality} % (fold)
\label{sub:a_bound_from_subexponentiality}

Consider a $\chi^2_p$ random variable $x$. Then $x$ is subexponential with parameters
$(\sigma^2, b) = (4d, 4)$, i.e. for any $\lvert\lambda\rvert < \tfrac1b$ we have
$$
  \log \mathbb{E}_{x\sim \chi^2_p} e^{\lambda x}
    \leq \lambda \mathbb{E} x + \frac{\sigma^2}2 \lambda^2
    \,. $$
Therefore, by Chernoff's inequality we have
\begin{equation*}
  \mathbb{P}_{x \sim \chi^2_p}\bigl( x \geq t \bigr)
    % = \mathbb{P}_{x \sim \chi^2_d}\bigl(
    %     e^{\lambda x} \geq e^{\lambda t}
    % \bigr)
    \leq \inf_{\lambda > 0} e^{- \lambda t} \mathbb{E}_{x \sim \chi^2_p} e^{\lambda x}
    % \leq \inf_{0 < \lambda < \tfrac14} e^{- \lambda t} e^{\lambda p + \frac{4 p}2 \lambda^2}
    \leq \inf\Bigl\{
      e^{\lambda (p - t) + 2p \lambda^2}
      \colon 0 < \lambda < \tfrac14
    \Bigr\}
    \,.
\end{equation*}
The optimal $\lambda = \tfrac{t - p}{4p} \in (0, \tfrac14)$ for $t > p$, whence
for $t = (1 + \varepsilon) p$
\begin{equation*}
  \mathbb{P}_{x \sim \chi^2_p}\bigl( x \geq t \bigr)
    \leq \exp{\Bigl\{
        -\tfrac{(t - p)^2}{4p} + 2p \tfrac{(t - p)^2}{16 p^2}
      \Bigr\}}
    % = \exp{\Bigl\{
    %     -\tfrac{(t - p)^2}{8 p}
    %   \Bigr\}}
    = e^{-\tfrac{p \varepsilon^2}8}
    \,.
\end{equation*}
In the opposite direction, we get the following:
\begin{equation*}
  \mathbb{P}_{x \sim \chi^2_p}\bigl( x \leq t \bigr)
    % = \mathbb{P}_{x \sim \chi^2_d}\bigl(
    %     e^{-\lambda x} \geq e^{-\lambda t}
    % \bigr)
    \leq \inf_{\lambda > 0} e^{\lambda t} \mathbb{E}_{x \sim \chi^2_p} e^{- \lambda x}
    % \leq \inf_{0 < \lambda < \tfrac14} e^{\lambda t} e^{-\lambda p + \frac{4 p}2 \lambda^2}
    \leq \inf\Bigl\{
      e^{\lambda (t - p) + 2p \lambda^2}
      \colon 0 < \lambda < \tfrac14
    \Bigr\}
    \,.
\end{equation*}
The optimal $\lambda = \tfrac{p - t}{4p} \in (0, \tfrac14)$ for $t < p$, whence
for $t = (1 - \varepsilon) p$
\begin{equation*}
  \mathbb{P}_{x \sim \chi^2_p}\bigl( x \leq t \bigr)
    \leq \exp{\Bigl\{
        -\tfrac{(p - t)^2}{4p} + 2p \tfrac{(p - t)^2}{16 p^2}
      \Bigr\}}
    % = \exp{\Bigl\{
    %     -\tfrac{(p - t)^2}{8 p}
    %   \Bigr\}}
    = e^{-\tfrac{p \varepsilon^2}8}
    \,.
\end{equation*}
Hence, for any $\varepsilon \in (0, 1)$ the union bound implies
\begin{equation*}
  \mathbb{P}_{x \sim \chi^2_p}\bigl(
      \lvert x - p\rvert \geq \varepsilon p
    \bigr)
    % \leq \mathbb{P}_{x \sim \chi^2_p}\bigl(
    %     x \geq (1 + \varepsilon) p
    %   \bigr)
    %   + \mathbb{P}_{x \sim \chi^2_p}\bigl(
    %     x \leq (1 - \varepsilon) p
    %   \bigr)
    \leq 2 e^{-\tfrac{p}{12} \tfrac{3 \varepsilon^2}2}
    \,.
\end{equation*}
% subsection a_bound_from_subexponentiality (end)

\subsection{A tighter bound from the mgf} % (fold)
\label{sub:a_tighter_bound_from_the_mgf}

These lectures\footnotemark ~cite
\footnotetext{\url{https://cs.stanford.edu/people/mmahoney/cs369m/Lectures/lecture1.pdf}}
Dasgupta, Gupta (1999)\footnotemark ~and refer to
\footnotetext{\url{http://citeseer.ist.psu.edu/viewdoc/summary?doi=10.1.1.45.3654}}
the moment generating function of a $\chi^2_p$ variable when bounding the absolute
deviation of the sample mean from the theoretical. We shall follow the same steps,
in order to derive the tail deviation bound for $\varepsilon > 0$
$$
  \mathbb{P}_{z \sim \mathcal{N}_p(0, I_p)}\bigl(
    \lvert \|z\|^2 - p \rvert \geq \varepsilon p
  \bigr)
  \,. $$
In the following $\|\cdot\|$ denotes the $\ell_2$ norm, unless specified otherwise.

Consider a random variable $z\sim \mathcal{N}_p(0, I_p)$. Then $z_i$, $i=1,\,\ldots,\,p$
are iid $\mathcal{N}(0, 1)$ and by definition $\|z\|^2 = \sum_{i=1}^p z_i^2 \sim \chi^2_p$.
Then for $\lambda \in \mathbb{R}$ the moment generating function of $\chi^2_p$ is:
\begin{align*}
  \mathbb{E}_{x \sim \chi^2_p} e^{\lambda x}
    &= \mathbb{E}_{z\sim \mathcal{N}_p(0, I_p)} e^{\lambda \|z\|^2}
    = \mathbb{E}_{z\sim \mathcal{N}_p(0, I_p)} e^{\lambda \sum_{i=1}^p z_i^2}
    \\
    &= \prod_{i=1}^p \mathbb{E}_{z_i\sim \mathcal{N}(0, 1)} e^{\lambda z_i^2}
    = \biggl( \mathbb{E}_{z\sim \mathcal{N}(0, 1)} e^{\lambda z^2} \biggr)^p
    \,.
\end{align*}
Now for $2 \lambda < 1$ we have
\begin{align*}
  \mathbb{E}_{z\sim \mathcal{N}(0, 1)} e^{\lambda z^2}
    &= \tfrac1{\sqrt{2 \pi}} \int_{- \infty}^{+ \infty}
      e^{- (1 - 2 \lambda) \tfrac{z^2}2} dz
    \\
    &= \tfrac1{\sqrt{2 \pi}} \int_{-\infty}^{+\infty}
      e^{-\tfrac{z^2}2} \tfrac1{\sqrt{1 - 2 \lambda}} dz
    = \tfrac1{\sqrt{1 - 2 \lambda}}
    \,,
\end{align*}
which implies that $\mathbb{E}_{x\sim \chi^2_p} e^{\lambda x} = \bigl(1 - 2 \lambda\bigr)^{-\tfrac{p}2}$.

Let's apply Chernoff's bound to get a bound on the tail probability of $\chi^2_p$.
\begin{equation*}
  \mathbb{P}_{z \sim \mathcal{N}_p(0, I_p)}\bigl( \|z\|^2 \geq t \bigr)
    % = \mathbb{P}_{x \sim \chi^2_d}\bigl(
    %     e^{\lambda x} \geq e^{\lambda t}
    % \bigr)
    % \leq \inf_{\lambda > 0} e^{- \lambda t} \mathbb{E}_{x \sim \chi^2_d} e^{\lambda x}
    % = \inf_{0 < \lambda < \tfrac12} (1 - 2\lambda)^{-\tfrac{p}2} e^{- \lambda t}
    = \inf_{0 < \lambda < \tfrac12} \Bigl[(1 - 2\lambda) e^{\lambda \tfrac{2 t}{p}} \Bigr]^{-\tfrac{p}2}
    \,.
\end{equation*}
The expression under the infimum is lower the higher the expression in the square
brackets is. For $t > p$ its maximum is attained at $\lambda = \tfrac{t - p}{2 t} < \tfrac12$.
and is equal to
$$
  \Bigl[(1 - 2\lambda) e^{\lambda \tfrac{2 t}{p}} \Bigr]^{-\tfrac{p}2}
    % = \Bigl[\tfrac{p}{t} e^{\tfrac{t - p}{p}} \Bigr]^{-\tfrac{p}2}
    = \Bigl[\tfrac{t}{p} e^{\tfrac{p - t}{p}} \Bigr]^{\tfrac{p}2}
    = \Bigl( \exp{(\log{(1 + \varepsilon)} - \varepsilon)} \Bigr)^{\tfrac{p}2}
    \,, $$
where $\varepsilon = \tfrac{t-p}{p} > 0$. Note that for all $\varepsilon > 0$ the exponent
can be bounded by
$$
  \log{(1 + \varepsilon)} - \varepsilon
    \leq - \varepsilon + \varepsilon - \tfrac{\varepsilon^2}2 + \tfrac{\varepsilon^3}3
    = \tfrac16 \bigl( 2\varepsilon^3 - 3\varepsilon^2 \bigr)
    \,. $$
Therefore, for $t = (1 + \varepsilon) p$ we have
$$
  \mathbb{P}_{x \sim\chi^2_p} \bigl( x \geq (1 + \varepsilon) p \bigr)
    \leq e^{\tfrac{p}{12} (2 \varepsilon^3 - 3 \varepsilon^2)}
    \,. $$
This bound is non-vacuous for $\varepsilon < \tfrac32$, which well covers the region
of interest that is $\varepsilon \in (0, 1)$. Although for $\varepsilon \approx 0$
the exponent is close to $0$ from below.

The bound for the opposite deviation can be derived in a similar fashion: for all
admissible $\lambda > 0$ Chernoff's bound implies
\begin{equation*}
  \mathbb{P}_{z \sim \mathcal{N}_p(0, I_p)}\bigl(
      \|z\|^2 \leq t
  \bigr)
    % = \mathbb{P}_{x \sim \chi^2_p}\bigl(
    %     e^{-\lambda x} \geq e^{- \lambda t}
    % \bigr)
    \leq \inf_{\lambda > 0} \frac1{e^{-\lambda t}} \mathbb{E}_{x \sim \chi^2_p} e^{-\lambda x}
    % \leq \inf_{\lambda > 0} (1 + 2\lambda)^{-\tfrac{p}2} e^{\lambda t}
    = \inf_{\lambda > 0} \Bigl[(1 + 2\lambda) e^{- \lambda \tfrac{2 t}{p}} \Bigr]^{-\tfrac{p}2}
    \,.
\end{equation*}
Therefore for $\lambda = \tfrac{p - t}{2 t}$, $t < p$, the right-hand side evaluates to
$$
  \Bigl[(1 + 2\lambda) e^{- \lambda \tfrac{2 t}{p}} \Bigr]^{-\tfrac{p}2}
    % = \Bigl[(1 + 2 \tfrac{p-t}{2 t}) e^{- \tfrac{p-t}{2 t} \tfrac{2 t}{p}} \Bigr]^{-\tfrac{p}2}
    % = \Bigl[\tfrac{p}{t} e^{\tfrac{t-p}{p}} \Bigr]^{-\tfrac{p}2}
    = \Bigl[\tfrac{p}{t} e^{- \tfrac{p - t}{p}} \Bigr]^{-\tfrac{p}2}
    % = \Bigl[ (1 - \varepsilon) e^\varepsilon \Bigr]^{\tfrac{p}2}
    = \Bigl( \exp{(\log{(1 - \varepsilon)} + \varepsilon)} \Bigr)^{\tfrac{p}2}
    \,, $$
where $\varepsilon = \tfrac{p - t}{p} > 0$. For any $\varepsilon \in (0, 1)$ we have
$$
  \log{(1 - \varepsilon)} + \varepsilon
    \leq \varepsilon - \varepsilon - \tfrac{\varepsilon^2}2
    \,, $$
whence for $t = (1 - \varepsilon) p$ we have
$$
  \mathbb{P}_{x \sim\chi^2_p} \bigl( x \leq (1 - \varepsilon) p \bigr)
    \leq e^{\tfrac{p}{12} (- 3 \varepsilon^2)}
    \,. $$
Therefore, for $\varepsilon\approx 0$ the union bound implies
\begin{align*}
  \mathbb{P}_{z \sim \mathcal{N}_p(0, I_p)}\bigl(
      \lvert \|z\|^2 - p \rvert \geq \varepsilon p
  \bigr)
    &\leq \mathbb{P}_{z}\bigl(
        \|z\|^2 \geq (1 + \varepsilon) p
      \bigr)
      + \mathbb{P}_{z}\bigl(
        \|z\|^2 \leq (1 - \varepsilon) p
      \bigr)
    \\
    &\leq e^{\tfrac{p}{12} (2 \varepsilon^3 - 3 \varepsilon^2)} + e^{\tfrac{p}{12} (- 3 \varepsilon^2)}
    \leq 2 e^{\tfrac{p}{12} (2 \varepsilon^3 - 3 \varepsilon^2)}
    \,.
\end{align*}

% subsection a_tighter_bound_from_the_mgf (end)

% section bounding_the_tail_probability (end)

\section{Applying the bound to get a J-L result} % (fold)
\label{sec:applying_the_bound_to_get_a_j_l_result}

Suppose $A \subset \real^d$ is a finite collection of vectors. Then there exists
a linear operator $L\colon\real^d\to \real^p$ with $d \gg p$ such that for all $\varepsilon > 0$
the following holds with high probability depending on $\varepsilon$, $p$, and
$\lvert A \rvert$:
$$
  (1 - \varepsilon) \|u - v\|^2
    \leq \bigl\| L u - L v \bigr\|^2
    \leq (1 + \varepsilon) \|u - v\|^2
    \,, $$
for all $u, v \in A$.

\medskip
Consider an iid sample $S = (z_i)_{i=1}^m$ from $\mathcal{D} = \mathcal{N}_d(0, I_d)$.
Then for any $u \in \real^d$ with $\|u\| = 1$, then $z_i^\top u \sim \mathcal{N}(0, u^\top u)$
are iid Gaussian, since linear combinations of Gaussians are Gaussian. Now, if the
sample is collected into a $m\times d$ matrix $Z_\mathcal{S} = (z_i)_{i=1}^m$, then
for any $u \in \real^d$ the rv $Z_\mathcal{S} u$ is $\mathcal{N}_m(0, \|u\|^2 I_m)$.
Therefore the squared norm of the image of a unit-norm $u$ under the random linear
transformation $Z_\mathcal{S}$ is a $\chi^2_m$ rv. For the random linear operator
\begin{equation} \label{eq:random_operator}
  L_\mathcal{S} \colon \real^d \to \real^m
    \colon a \mapsto \tfrac1{\sqrt{m}} Z_\mathcal{S} a
      = \Bigl(
        \bigl\langle \tfrac{z_i}{\sqrt{m}}, a \bigr\rangle
      \Bigr)_{i=1}^m
    \,,
\end{equation}
depending on the sample $S \sim \mathcal{D}^m$, and for any $v \in \real^d$ with
$u = \tfrac{v}{\|v\|}$ we have
\begin{align*}
  \mathbb{P}_{S \sim \mathcal{D}^m} \Bigl(
    \bigl\lvert \| L_\mathcal{S} v \|^2 - \|v\|^2 \bigr \rvert
      \geq \varepsilon \|v\|
  \Bigr)
    % &= \mathbb{P}_{S \sim \mathcal{D}^m} \biggl(
    %   \Bigl\lvert \bigl\| L_\mathcal{S} \tfrac{v}{\|v\|} \bigr\|^2 - 1 \Bigr \rvert
    %     \geq \varepsilon
    % \biggr)
    % \\
    &= \mathbb{P}_{S \sim \mathcal{D}^m} \bigl(
        \lvert \|Z_\mathcal{S} u\|^2 - m \rvert \geq \varepsilon m
      \bigr)
    \\
    &= \mathbb{P}_{x \sim \chi^2_m} \bigl(
        \lvert x - m \rvert \geq \varepsilon m
      \bigr)
      % \leq 2 e^{\tfrac{m}{12} (2 \varepsilon^3 - 3 \varepsilon^2)}
    \,,
\end{align*}
and the $\chi^2_m$ distribution does not depend on $v$. Therefore, for a finite
collection $A \subset \real^d$ the Union bound implies
\begin{align*}
  \mathbb{P}_\mathcal{S}\Bigl(
    \exists{a, b\in A} \colon
    \Bigl\lvert \tfrac{\| L_\mathcal{S} (a - b) \|^2}{\|a - b\|^2} - 1 \Bigr\rvert
      \geq \varepsilon
  \Bigr)
    &= \mathbb{P}_\mathcal{S}\Bigl(
      \cup_{v\in (A - A)} \Bigl\{
        \Bigl\lvert \tfrac{\| L_\mathcal{S} v \|^2}{\|v\|^2} - 1 \Bigr\rvert
          \geq \varepsilon
      \Bigr\}
    \Bigr)
    \\
    &\leq \sum_{v\in (A - A)}
      \mathbb{P}_\mathcal{S}\Bigl(
        \Bigl\lvert \| L_\mathcal{S} v \|^2 -  \|v\|^2 \Bigr\rvert
          \geq \varepsilon\|v\|^2
      \Bigr)
    \\
    &\leq \tfrac{n (n - 1)}2 \mathbb{P}_{x \sim \chi^2_m} \bigl(
        \lvert x - m \rvert \geq \varepsilon m
      \bigr)
    \,,
\end{align*}
where $A - B = \{a - b\colon a\in A,\,b\in B\}$ and $n = \lvert A \rvert$.
Section~\ref{sub:a_tighter_bound_from_the_mgf} implies
\begin{equation*}
  \mathbb{P}_{x \sim \chi^2_m} \bigl(
      \lvert x - m \rvert \geq \varepsilon m
    \bigr)
    \leq 2 e^{\tfrac{m}{12} (2 \varepsilon^3 - 3 \varepsilon^2)}
    \,,
\end{equation*}
whence for all $\varepsilon \in (0, 1)$
\begin{equation*}
  \mathbb{P}_\mathcal{S}\Bigl(
      \exists{a, b\in A} \colon
      \Bigl\lvert \tfrac{\| L_\mathcal{S} (a - b) \|^2}{\|a - b\|^2} - 1 \Bigr\rvert
        \geq \varepsilon
    \Bigr)
    \leq n (n - 1) e^{\tfrac{m}{12} (2 \varepsilon^3 - 3 \varepsilon^2)}
    \,.
\end{equation*}
The bound from sec.~\ref{sub:a_bound_from_subexponentiality} implies a slightly
looser bound: for all $\varepsilon \in (0, 1)$
\begin{equation*}
  \mathbb{P}_\mathcal{S}\Bigl(
      \exists{a, b\in A} \colon
      \Bigl\lvert \tfrac{\| L_\mathcal{S} (a - b) \|^2}{\|a - b\|^2} - 1 \Bigr\rvert
        \geq \varepsilon
    \Bigr)
    \leq n (n - 1) e^{- \tfrac{m}{8} \varepsilon^2}
    \,.
\end{equation*}

Therefore for $\delta, \varepsilon \in (0, 1)$ the $(\varepsilon, \delta)$-PAC
lower bounds on the required number of projected dimensions are given by
\begin{align*}
  m &\geq
      \biggl( \log\frac{n (n-1)}{\delta} \biggr)
      \frac{8}{\varepsilon^2}
    \,, \tag{sec.~\ref{sub:a_bound_from_subexponentiality}}
    \\
  m &\geq
      \biggl( \log\frac{n (n-1)}{\delta} \biggr)
      \frac{12}{3 \varepsilon^2 - 2 \varepsilon^3}
    \,, \tag{sec.~\ref{sub:a_tighter_bound_from_the_mgf}}
\end{align*}
These sample size lower bounds ensure that the linear operator $L_\mathcal{S}$
in~\eqref{eq:random_operator} has the $\varepsilon$-near isometry property for all
vectors in $A$ with probability at least $1 - \delta$. However, the one that uses
sec.~\ref{sub:a_tighter_bound_from_the_mgf} is tighter than the other one for all
$\varepsilon \in (0, \tfrac34)$.

\bigskip\noindent
The alternative forms of this sum are
$$
  \hat{\mathbb{E}}_{x\sim S} \frac{(u^\top x)^2}{\|u\|^2}
    = \tfrac1{\|u\|^2} \tfrac1p \sum_{i=1}^p x_i^\top u u^\top x_i
    = \tfrac{u^\top}{\|u\|}
      \Bigl( \tfrac1p \sum_{i=1}^p x_i x_i^\top \Bigr)
      \tfrac{u}{\|u\|}
    = \frac{\langle u, \hat{\Sigma} u \rangle}{\|u\|^2}
    \,, $$
where $\hat{\Sigma} = \tfrac1p \sum_{i=1}^p x_i x_i^\top$ is the sample covariance
matrix of $S$. Note that $\mathbb{E}_{x\sim \mathcal{D}} \frac{(u^\top x)^2}{\|u\|^2} = 1$.
\begin{align*}
  \mathbb{P}_{S \sim \mathcal{D}^m} \Bigl(
    \bigl\lvert \| L_\mathcal{S} a \|^2 - \|a\|^2 \bigr \rvert
      \geq \varepsilon \|a\|
  \Bigr)
    % &= \mathbb{P}_{S \sim \mathcal{D}^m} \biggl(
    %   \Bigl\lvert \bigl\| L_\mathcal{S} \tfrac{a}{\|a\|} \bigr\|^2 - 1 \Bigr \rvert
    %     \geq \varepsilon
    % \biggr)
    % \\
    &= \Bigl[u = \tfrac{a}{\|a\|}\Bigr]
    = \mathbb{P}_{S \sim \mathcal{D}^m} \bigl(
        \lvert \|Z_\mathcal{S} u\|^2 - m \rvert \geq \varepsilon m
      \bigr)
    \\
    &= \mathbb{P}_{x \sim \chi^2_m} \bigl(
        \lvert x - m \rvert \geq \varepsilon m
      \bigr)
      \leq 2 e^{\tfrac{m}{12} (2 \varepsilon^3 - 3 \varepsilon^2)}
    \,.
\end{align*}
Therefore, for
\begin{align*}
  \mathbb{P}_\mathcal{S}\Bigl(
    \exists{a, b\in A} \colon
    \Bigl\lvert \tfrac{\| L_\mathcal{S} (a - b) \|^2}{\|a - b\|^2} - 1 \Bigr\rvert
      \geq \varepsilon
  \Bigr)
    &= \mathbb{P}_\mathcal{S}\Bigl(
      \cup_{v\in (A - A)} \Bigl\{
        \Bigl\lvert \tfrac{\| L_\mathcal{S} v \|^2}{\|v\|^2} - 1 \Bigr\rvert
          \geq \varepsilon
      \Bigr\}
    \Bigr)
    \\
    &\leq \sum_{v\in (A - A)}
      \mathbb{P}_\mathcal{S}\Bigl(
        \Bigl\lvert \tfrac{\| L_\mathcal{S} v \|^2}{\|v\|^2} - 1 \Bigr\rvert
          \geq \varepsilon
      \Bigr)
    \\
    &\leq n (n - 1) e^{\tfrac{m}{12} (2 \varepsilon^3 - 3 \varepsilon^2)}
    \,,
\end{align*}

% section applying_the_bound_to_get_a_j_l_result (end)

\section*{Appendices} % (fold)
\label{sec:appendices}

\subsection*{Appendix A: the $\log(1+\varepsilon)$ bound} % (fold)
\label{sub:appendix_a_the_log_bound}

Consider the Taylor series expansion of $x\mapsto\log(1 + x)$ around $0$:
$$
    \log(1 + x)
        = \sum_{n\geq 1} (-1)^{n+1} \frac{x^n}{n}
    \,. $$
Observe that for any $k\geq 1$ we have
$$
    \sum_{n=1}^k (-1)^{n+1} \tfrac{x^n}{n}
        = \sum_{n=1}^k (-1)^{n-1+2} \int_0^x t^{n-1} dt
        % = \int_0^x \sum_{n=1}^k (-1)^{n+1} t^{n-1} dt
        % = \int_0^x \sum_{n=1}^k (-t)^{n-1} dt
        % = \int_0^x \sum_{n=0}^{k-1} (-t)^n dt
        = \int_0^x \frac{1 - (-1)^k t^k}{1 + t} dt
    \,. $$
Therefore for $x\geq 0$ we have
\begin{align*}
    \sum_{n=1}^k (-1)^{n+1} \tfrac{x^n}{n}
        &
        % = \int_0^x \frac{1 - (-1)^k t^k}{1 + t} dt
        \leq \int_0^x \frac1{1 + t} dt
        = \log(1+x)
        \,, \text{ for}~k~\text{even}
        \,; \\
    \sum_{n=1}^k (-1)^{n+1} \tfrac{x^n}{n}
        &
        % = \int_0^x \frac{1 - (-1)^k t^k}{1 + t} dt
        \geq \int_0^x \frac1{1 + t} dt
        = \log(1+x)
        \,, \text{ for}~k~\text{odd}
        \,.
\end{align*}
The Taylor series expansion for $x\mapsto \log(1 - x)$ around $0$ follows from the
one above, but easily lends an upper bound for all $k\geq 1$ and $x \in (0, 1)$
$$
    \log(1 - x)
        = \sum_{n\geq 1} (-1)^{n+1} \frac{(-x)^n}{n}
        = \sum_{n\geq 1} \frac{- x^n}{n}
        \leq \sum_{n=1}^k \frac{- x^n}{n}
    \,. $$

% subsection* appendix_a_the_log_bound (end)

% section* appendices (end)

\end{document}


\documentclass[onecolumn]{article}
\usepackage{amsmath}
\usepackage{lscape}
% wide page for side by side figures, tables, etc
\usepackage{changepage}
\newlength{\offsetpage}
\setlength{\offsetpage}{2.5cm}
\newenvironment{widepage}{\begin{adjustwidth}{-\offsetpage}{-\offsetpage}%
    \addtolength{\textwidth}{2\offsetpage}}%
{\end{adjustwidth}}
\begin{document}
\title{05 - Example 03 - Davies model}
\author{}
\date{}
\maketitle
Example, 5 components (4 + solvent), 3 reactions: \\
\section{Variables}
5 (components + solvent) compositions $n_0$, $n_1$, ..., $n_4$. \\
5 (components + solvent) activity coefficients
$\gamma_0$, $\gamma_1$, ..., $\gamma_4$. \\
3 reaction extents $\xi_1$, $\xi_2$, ..., $\xi_{3}$. \\
1 Ionic strength $I$. \\
Tot. $2(5) + 3 + 1 = 14$ \\
\\
$x =[$
\begin{tabular}{cccccccccccccc}
$n_0$ & $m_1$ & $m_2$ & $m_3$ & $m_4$ & $\xi_1$ & $\xi_2$ & $\xi_3$ &
$\gamma_0$ & $\gamma_1$ & $\gamma_2$ & $\gamma_3$ & $\gamma_4$ & $I$
$]^T$
\end{tabular}
\section{Objective function}
5 mole balances. \\
5 activity coefficient expressions. \\
3 equilibrium expresisons. \\
1 Ionic strength expression. \\
Tot. $2(5) + 3 + 1 = 14$\\
\\
\[
f(x) = 0 = \left(
\begin{tabular}{l}
$-n_0 + n_{0,0} + \nu_{01}\xi_1 + \nu_{02}\xi_2 + \nu_{03}\xi_3  $\\
$-m_1 n_0 M_0 + n_{1,0} + \nu_{11}\xi_1 + \nu_{12}\xi_2 + \nu_{13}\xi_3  $\\
$-m_2 n_0 M_0 + n_{2,0} + \nu_{21}\xi_1 + \nu_{22}\xi_2 + \nu_{23}\xi_3  $\\
$-m_3 n_0 M_0 + n_{3,0} + \nu_{31}\xi_1 + \nu_{32}\xi_2 + \nu_{33}\xi_3  $\\
$-m_4 n_0 M_0 + n_{4,0} + \nu_{41}\xi_1 + \nu_{42}\xi_2 + \nu_{43}\xi_3  $\\
$-K_1 + (\frac{1}{m^{\circ}})^{\sum_j{\nu_{j1}}}\times
\gamma_0^{\nu_{01}}\gamma_1^{\nu_{11}}\gamma_2^{\nu_{21}}
\gamma_3^{\nu_{31}}\gamma_4^{\nu_{41}}\times
\left(\frac{1}{M_0}\right)^{\nu_{01}}m_1^{\nu_{11}}m_2^{\nu_{21}}
m_3^{\nu_{31}}m_4^{\nu_{41}}$\\
$-K_1 + (\frac{1}{m^{\circ}})^{\sum_j{\nu_{j2}}}\times
\gamma_0^{\nu_{02}}\gamma_1^{\nu_{12}}\gamma_2^{\nu_{22}}
\gamma_3^{\nu_{32}}\gamma_4^{\nu_{42}}\times
\left(\frac{1}{M_0}\right)^{\nu_{02}}m_1^{\nu_{12}}m_2^{\nu_{22}}
m_3^{\nu_{32}}m_4^{\nu_{42}}$\\
$-K_1 + (\frac{1}{m^{\circ}})^{\sum_j{\nu_{j3}}}\times
\gamma_0^{\nu_{03}}\gamma_1^{\nu_{13}}\gamma_2^{\nu_{23}}
\gamma_3^{\nu_{33}}\gamma_4^{\nu_{43}}\times
\left(\frac{1}{M_0}\right)^{\nu_{03}}m_1^{\nu_{13}}m_2^{\nu_{23}}
m_3^{\nu_{33}}m_4^{\nu_{43}}$\\
$-\gamma_0 + e^{-1.0\times M_0 \sum_{j\neq0}{m_j}}$ \\
$-\gamma_1 +10^{- 0.510z_1^2 \times
\left(\frac{\sqrt{I}}{1+\sqrt{I}}-0.3I\right) + (1-sign(z_1)^2)b_1 I}$ \\
$-\gamma_2 +10^{- 0.510z_2^2 \times
\left(\frac{\sqrt{I}}{1+\sqrt{I}}-0.3I\right) + (1-sign(z_2)^2)b_2 I}$ \\
$-\gamma_3 +10^{- 0.510z_3^2 \times
\left(\frac{\sqrt{I}}{1+\sqrt{I}}-0.3I\right) + (1-sign(z_3)^2)b_3 I}$ \\
$-\gamma_4 +10^{- 0.510z_4^2 \times
\left(\frac{\sqrt{I}}{1+\sqrt{I}}-0.3I\right) + (1-sign(z_4)^2)b_4 I}$ \\
$-I + \frac{1}{2}(m_0 z_0^2 +  m_1 z_1^2 + m_2 z_2^2 + m_3 z_3^2 + m_4 z_4^2)$
\end{tabular}
\right)
\]
For convenience, define the reaction quotient:\\
\begin{equation}
\label{eq:reaction_quotient}
Q_j \equiv \left(\frac{1}{m^{\circ}}\right)^{\sum_i{\nu_{ij}}}\times
\gamma_0^{\nu_{0j}}\gamma_1^{\nu_{1j}}\gamma_2^{\nu_{2j}}
\gamma_3^{\nu_{3j}}\gamma_4^{\nu_{4j}}\times
\left(\frac{1}{M_0}\right)^{\nu_{0j}}m_1^{\nu_{1j}}m_2^{\nu_{2j}}
m_3^{\nu_{3j}}m_4^{\nu_{4j}}
\end{equation}
Non-electrolyte solvent: $z_0 = 0$ \\
Reference: $m^{\circ} = 1\frac{mol}{kg_{solvent}}$
\section{Jacobian}
\[
J(x) = \left(
\begin{tabular}{llll}
$\frac{\partial{f_1}}{\partial{x_1}}$ &
$\frac{\partial{f_1}}{\partial{x_2}}$ & ... &
$\frac{\partial{f_1}}{\partial{x_{14}}}$\\
$\frac{\partial{f_2}}{\partial{x_1}}$ &
$\frac{\partial{f_2}}{\partial{x_2}}$ & ... &
$\frac{\partial{f_2}}{\partial{x_{14}}}$\\
$...$ & $...$ & $...$ & $...$\\
$\frac{\partial{f_{14}}}{\partial{x_1}}$ &
$\frac{\partial{f_{14}}}{\partial{x_2}}$ & ... &
$\frac{\partial{f_{14}}}{\partial{x_{14}}}$\\
\end{tabular}
\right)
\]\\
\[
J_{1,1}(x) = \frac{\partial{f_{1}}}{\partial{x_1}} =  -1
\]
\[
J_{1,2}(x) = \frac{\partial{f_{1}}}{\partial{x_2}} =  0
\]
\[
J_{1,3}(x) = \frac{\partial{f_{1}}}{\partial{x_2}} =  0
\]
(...)
\[
J_{1,6}(x) = \frac{\partial{f_{1}}}{\partial{x_6}} =  \nu_{01}
\]
(...)
\[
J_{2,1}(x) =  -m_1 M_0;
J_{2,2}(x) =  -n_0 M_0;
J_{2,3}(x) =  0;
J_{2,6}(x) = \nu_{11}
\]
(...)
\[
J_{3,1}(x) =  -m_2 M_0;
J_{3,2}(x) =  -n_0 M_0;
J_{3,3}(x) =  0;
J_{3,6}(x) = \nu_{21}
\]
(...)
\[
\begin{aligned}
J_{6,1}(x) = & 0 \\
J_{6,2}(x) = &
\left(\frac{1}{m^{\circ}}\right)^{\sum_i{\nu_{i1}}}\times
\gamma_0^{\nu_{01}}\gamma_1^{\nu_{11}}\gamma_2^{\nu_{21}}
\gamma_3^{\nu_{31}}\gamma_4^{\nu_{41}}\times
\left(\frac{1}{M_0}\right)^{\nu_{01}}
\nu_{11}m_1^{\nu_{11}-1}m_2^{\nu_{21}}
m_3^{\nu_{31}}m_4^{\nu_{41}}\\
= & \frac{\nu_{11}}{m_{1}} \times Q_1\\
J_{6,3}(x) =& \frac{\nu_{21}}{m_2} \times Q_1\\
\text{(...)}\\
J_{6,6}(x) =& 0\\
J_{6,7}(x) =& 0\\
J_{6,8}(x) =& 0\\
J_{6,9}(x) =&
\left(\frac{1}{m^{\circ}}\right)^{\sum_i{\nu_{i1}}}\times
\nu_{01}\gamma_0^{\nu_{01}-1}\gamma_1^{\nu_{11}}\gamma_2^{\nu_{21}}
\gamma_3^{\nu_{31}}\gamma_4^{\nu_{41}}\times
\left(\frac{1}{M_0}\right)^{\nu_{01}}m_1^{\nu_{11}}m_2^{\nu_{21}}
m_3^{\nu_{31}}m_4^{\nu_{41}}\\
= & \frac{\nu_{01}}{\gamma_{0}} \times Q_1\\
J_{6,10}(x) =&
\left(\frac{1}{m^{\circ}}\right)^{\sum_i{\nu_{i1}}}\times
\gamma_0^{\nu_{01}}\nu_{11}\gamma_1^{\nu_{11}-1}\gamma_2^{\nu_{21}}
\gamma_3^{\nu_{31}}\gamma_4^{\nu_{41}}\times
\left(\frac{1}{M_0}\right)^{\nu_{01}}m_1^{\nu_{11}}m_2^{\nu_{21}}
m_3^{\nu_{31}}m_4^{\nu_{41}}\\
= & \frac{\nu_{11}}{\gamma_{1}} \times Q_1\\
\text{(...)}\\
J_{6,14}(x) =&  0\\
\text{(...)}\\
\end{aligned}
\]
\[
\begin{aligned}
J_{9,1}(x) =&  0\\
J_{9,2}(x) =& -1.0 M_0 m_1 e^{-1.0\times M_0 \sum_{j\neq0}{m_j}}\\
J_{9,3}(x) =& -1.0 M_0 m_2 e^{-1.0\times M_0 \sum_{j\neq0}{m_j}}\\
\text{(...)}\\
J_{9,5}(x) =& -1.0 M_0 m_4 e^{-1.0\times M_0 \sum_{j\neq0}{m_j}}\\
J_{9,6}(x) =& 0\\
\text{(...)}\\
J_{9,9}(x) =& -1\\
J_{9,10}(x) =& 0\\
J_{9,11}(x) =& 0\\
\text{(...)}\\
J_{9,14}(x) =& 0\\
J_{10,1}(x) =& 0\\
J_{10,2}(x) =& 0\\
\text{(...)}\\
J_{10,9}(x) =& 0\\
J_{10,10}(x) =& -1\\
J_{10,11}(x) =& 0\\
\text{(...)}\\
J_{10,14}(x) =&
-0.510 z_1^2\times\left(\frac{1}{2\sqrt{I}(1+\sqrt{I})^2} -0.3\right)
\times ln(10) \times 10^{- 0.510z_1^2 \times
\left(\frac{\sqrt{I}}{1+\sqrt{I}}-0.3I\right) + (1-sign(z_1)^2)b_1 I}\\
=& -0.510 z_1^2\times\left(\frac{1}{2\sqrt{I}(1+\sqrt{I})^2} -0.3\right)
 \times ln(10) \times (\gamma_1+f_{10}(x)) \\
\text{(...)}\\
J_{11,1}(x) =& 0\\
\text{(...)}\\
J_{11,11}(x) =& -1\\
J_{11,12}(x) =& 0\\
\text{(...)}\\
J_{11,14}(x) =&
-0.510 z_2^2\times\left(\frac{1}{2\sqrt{I}(1+\sqrt{I})^2} -0.3\right)
\times ln(10) \times 10^{- 0.510z_2^2 \times
\left(\frac{\sqrt{I}}{1+\sqrt{I}}-0.3I\right) + (1-sign(z_2)^2)b_2 I}\\
=& -0.510 z_2^2\times\left(\frac{1}{2\sqrt{I}(1+\sqrt{I})^2} -0.3\right)
 \times ln(10) \times (\gamma_2+f_{11}(x)) \\
\text{(...)}\\
\end{aligned}
\]
Using the reaction quotients $Q_i$ for convenience as defined above (eq.
\ref{eq:reaction_quotient}), the Jacobian matrix results as follows,
considering non-electrolyte solvent (charge $z_0=0$):
\begin{landscape}
\[
J(x) = \left(
\begin{tabular}{llllllllllllll}
$-1$ & 0 & 0 & 0 & 0 &
$\nu_{01}$ & $\nu_{02}$ & $\nu_{03}$ &
0 & 0 & 0 & 0 & 0 &
0\\
$-m_1 M_0$ & $-n_0 M_0$ & 0 & 0 & 0 &
$\nu_{11}$ & $\nu_{12}$ & $\nu_{13}$ &
0 & 0 & 0 & 0 & 0 &
0\\
$-m_2 M_0$ & 0 & $-n_0 M_0$  & 0 & 0 &
$\nu_{21}$ & $\nu_{22}$ & $\nu_{23}$ &
0 & 0 & 0 & 0 & 0 &
0\\
$-m_3 M_0$ & 0 & 0 & $-n_0 M_0$ & 0 &
$\nu_{31}$ & $\nu_{32}$ & $\nu_{33}$ &
0 & 0 & 0 & 0 & 0 &
0\\
$-m_4 M_0$ & 0 & 0 & 0 & $-n_0 M_0$ &
$\nu_{41}$ & $\nu_{42}$ & $\nu_{43}$ &
0 & 0 & 0 & 0 & 0 &
0\\
0 &
$\frac{\nu_{11}}{m_1}Q_1$ & $\frac{\nu_{21}}{m_2}Q_1$ &
$\frac{\nu_{31}}{m_3}Q_1$ & $\frac{\nu_{41}}{m_4}Q_1$ &
0 & 0 & 0 &
$\frac{\nu_{01}}{\gamma_0}Q_1$ & $\frac{\nu_{11}}{\gamma_1}Q_1$ &
$\frac{\nu_{21}}{\gamma_2}Q_1$ & $\frac{\nu_{31}}{\gamma_3}Q_1$ &
$\frac{\nu_{41}}{\gamma_4}Q_1$ &
0\\
0 &
$\frac{\nu_{12}}{m_1}Q_2$ & $\frac{\nu_{22}}{m_2}Q_2$ &
$\frac{\nu_{32}}{m_3}Q_2$ & $\frac{\nu_{42}}{m_4}Q_2$ &
0 & 0 & 0 &
$\frac{\nu_{02}}{\gamma_0}Q_2$ & $\frac{\nu_{12}}{\gamma_1}Q_2$ &
$\frac{\nu_{22}}{\gamma_2}Q_2$ & $\frac{\nu_{32}}{\gamma_3}Q_2$ &
$\frac{\nu_{42}}{\gamma_4}Q_2$ &
0\\
0 &
$\frac{\nu_{13}}{m_1}Q_3$ & $\frac{\nu_{23}}{m_2}Q_3$ &
$\frac{\nu_{33}}{m_3}Q_3$ & $\frac{\nu_{43}}{m_4}Q_3$ &
0 & 0 & 0 &
$\frac{\nu_{03}}{\gamma_0}Q_3$ & $\frac{\nu_{13}}{\gamma_1}Q_3$ &
$\frac{\nu_{23}}{\gamma_2}Q_3$ & $\frac{\nu_{33}}{\gamma_3}Q_3$ &
$\frac{\nu_{43}}{\gamma_4}Q_3$ &
0\\
0 & $J_{9,2}(x)$ & $J_{9,3}(x)$ & $J_{9,4}(x)$ & $J_{9,5}(x)$ &
0 & 0 & 0 &
-1 & 0 & 0 & 0 & 0 &
0\\
0 & 0 & 0 & 0 & 0 &
0 & 0 & 0 &
0 & -1 & 0 & 0 & 0 &
$J_{10,14}(x)$\\
0 & 0 & 0 & 0 & 0 &
0 & 0 & 0 &
0 & 0 & -1 & 0 & 0 &
$J_{11,14}(x)$\\
0 & 0 & 0 & 0 & 0 &
0 & 0 & 0 &
0 & 0 & 0 & -1 & 0 &
$J_{12,14}(x)$\\
0 & 0 & 0 & 0 & 0 &
0 & 0 & 0 &
0 & 0 & 0 & 0 & -1 &
$J_{13,14}$\\
0 &
$\frac{1}{2}z_1^2$ & $\frac{1}{2}z_2^2$ &
$\frac{1}{2}z_3^2$ & $\frac{1}{2}z_4^2$ &
0 & 0 & 0 &
0 & 0 & 0 & 0 & 0 &
-1\\
\end{tabular}
\right)
\]
\[
\begin{aligned}
J_{9,2}(x) =& -1.0 M_0 e^{-1.0\times M_0 \sum_{j\neq0}{m_j}}\\
J_{9,3}(x) =& J_{9,4}(x) = J_{9,5}(x) = J_{9,2}(x)\\
J_{10,14}(x) =& \left[
-0.510 ln(10)\times z_1^2\times\left(\frac{1}{2\sqrt{I}(1+\sqrt{I})^2} -0.3\right)
+ (1-sign(z_1)^2)b_1 \right] \times 10^{- 0.510z_1^2 \times
\left(\frac{\sqrt{I}}{1+\sqrt{I}}-0.3I\right) + (1-sign(z_1)^2)b_1 I}\\
J_{11,14}(x) =& \left[
-0.510 ln(10)\times z_2^2\times\left(\frac{1}{2\sqrt{I}(1+\sqrt{I})^2} -0.3\right)
+ (1-sign(z_1)^2)b_2 \right]
\times 10^{- 0.510z_2^2 \times
\left(\frac{\sqrt{I}}{1+\sqrt{I}}-0.3I\right) + (1-sign(z_2)^2)b_2 I}\\
J_{12,14}(x) =& \left[
-0.510 ln(10)\times z_3^2\times\left(\frac{1}{2\sqrt{I}(1+\sqrt{I})^2} -0.3\right)
+ (1-sign(z_1)^2)b_3 \right]
\times 10^{- 0.510z_3^2 \times
\left(\frac{\sqrt{I}}{1+\sqrt{I}}-0.3I\right) + (1-sign(z_3)^2)b_3 I}\\
J_{13,14}(x) =& \left[
-0.510 ln(10)\times z_4^2\times\left(\frac{1}{2\sqrt{I}(1+\sqrt{I})^2} -0.3\right)
+ (1-sign(z_1)^2)b_4 \right]
\times 10^{- 0.510z_4^2 \times
\left(\frac{\sqrt{I}}{1+\sqrt{I}}-0.3I\right) + (1-sign(z_4)^2)b_4 I}\\
\end{aligned}
\]
\end{landscape}
For calculation, submatrixes of the Jacobian matrix are identified:
\begin{itemize}
\item $J_{1,1}(x)$ to $J_{n+1,n+1}(x)$ is a diagonal matrix of $n_i/n_0$
as function of molality:
\[
\begin{aligned}
J_{1,1}(x) \text{ to } J_{n+1,n+1}(x) = &
-diag( \left[
\begin{tabular}{ccccc}
1 & $m_1 M_0$ & $m_2 M_0$ & $m_3 M_0$ & $m_4 M_0$\\
\end{tabular}
\right]
)\\
= & \left(
\begin{tabular}{ccccc}
-1 & 0 & 0 & 0 & 0\\
0 & -$m_1 M_0$ & 0 & 0 & 0\\
0 & 0 & -$m_2 M_0$ & 0 & 0\\
0 & 0 & 0 & -$m_3 M_0$ & 0\\
0 & 0 & 0 & 0 & -$m_4 M_0$\\
\end{tabular}
\right)
\end{aligned}
\]
\item $J_{n+2+nr,n+2+nr}(x)$ to $J_{n+2+nr+n+1,n+2+nr+n+1}(x)$
is the negative of an identity matrix, size $(n+1) \times (n+1)$
\[
J_{n+2+nr,n+2+nr}(x) \text{ to } J_{n+2+nr+n+1,n+2+nr+n+1}(x) =
-I_{n+1} = -1\times \left(
\begin{tabular}{ccccc}
1 & 0 & 0 & 0 & 0\\
0 & 1 & 0 & 0 & 0\\
0 & 0 & 1 & 0 & 0\\
0 & 0 & 0 & 1 & 0\\
0 & 0 & 0 & 0 & 1\\
\end{tabular}
\right)
\]
\item $J_{1,n+2}(x)$ to $J_{n+1,n+2+nr}(x)$ is the stoichiometric coefficients
matrix
\[
J_{1,n+2}(x) \text{ to } J_{n+1,n+2+nr}(x) = \nu_{ij} =\left(
\begin{tabular}{ccc}
$\nu_{01}$ & $\nu_{02}$ & $\nu_{03}$\\
$\nu_{11}$ & $\nu_{12}$ & $\nu_{13}$\\
$\nu_{21}$ & $\nu_{22}$ & $\nu_{23}$\\
$\nu_{31}$ & $\nu_{32}$ & $\nu_{33}$\\
$\nu_{41}$ & $\nu_{42}$ & $\nu_{43}$\\
\end{tabular}
\right)
\]
\item $J_{n+2,1}(x)$ to $J_{n+2+nr,n+1}(x)$ is expressed as a product of the
diagonalized quotients matrix, the transposed coefficients matrix,
and a diagonalized inverse molality matrix, with no variation resulting from
solvent:
\[
\begin{aligned}
J_{n+2,1}(x) & \text{ to } J_{n+2+nr,n+1}(x) \\
= & diag([Q_1 Q_2 Q_3])\times \nu_{ij}^T \times
diag([0 \frac{1}{m_1} \frac{1}{m_2} \frac{1}{m_3} \frac{1}{m_4}]) \\
= & \left(
\begin{tabular}{ccc}
$Q_1$ & 0 & 0\\
0 & $Q_2$ & 0\\
0 & 0 & $Q_3$\\
\end{tabular}
\right) \times \left(
\begin{tabular}{ccccc}
$\nu_{01}$ & $\nu_{11}$ & $\nu_{21}$ & $\nu_{31}$ & $\nu_{41}$\\
$\nu_{02}$ & $\nu_{12}$ & $\nu_{22}$ & $\nu_{32}$ & $\nu_{42}$\\
$\nu_{03}$ & $\nu_{13}$ & $\nu_{23}$ & $\nu_{33}$ & $\nu_{43}$\\
\end{tabular}
\right) \times \left(
\begin{tabular}{ccccc}
0 & 0 & 0 & 0 & 0\\
0 & $\frac{1}{m_1}$ & 0 & 0 & 0\\
0 & 0 & $\frac{1}{m_2}$ & 0 & 0\\
0 & 0 & 0 & $\frac{1}{m_3}$ & 0\\
0 & 0 & 0 & 0 & $\frac{1}{m_4}$\\
\end{tabular}
\right)\\
= & \left(
\begin{tabular}{ccccc}
0 &
$\frac{\nu_{11}}{m_1}Q_1$ & $\frac{\nu_{21}}{m_2}Q_1$ &
$\frac{\nu_{31}}{m_3}Q_1$ & $\frac{\nu_{41}}{m_4}Q_1$\\
0 &
$\frac{\nu_{12}}{m_1}Q_2$ & $\frac{\nu_{22}}{m_2}Q_2$ &
$\frac{\nu_{32}}{m_3}Q_2$ & $\frac{\nu_{42}}{m_4}Q_2$\\
0 &
$\frac{\nu_{13}}{m_1}Q_3$ & $\frac{\nu_{23}}{m_2}Q_3$ &
$\frac{\nu_{33}}{m_3}Q_3$ & $\frac{\nu_{43}}{m_4}Q_3$\\
\end{tabular}
\right)
\end{aligned}
\]
\item $J_{n+2,n+2+nr}(x)$ to $J_{n+2+nr,n+2+nr+n+1}(x)$ is analogous for
activity coefficient partial variations:
\[
\begin{aligned}
J_{n+2,n+2+nr}(x) & \text{ to } J_{n+2+nr,n+2+nr+n+1}(x) \\
= & diag([Q_1 Q_2 Q_3])\times \nu_{ij}^T \times
diag([\frac{1}{\gamma_0} \frac{1}{\gamma_1} \frac{1}{m_2}
\frac{1}{m_3} \frac{1}{m_4}]) \\
= & \left(
\begin{tabular}{ccc}
$Q_1$ & 0 & 0\\
0 & $Q_2$ & 0\\
0 & 0 & $Q_3$\\
\end{tabular}
\right) \times  \\
& \left(
\begin{tabular}{ccccc}
$\nu_{01}$ & $\nu_{11}$ & $\nu_{21}$ & $\nu_{31}$ & $\nu_{41}$\\
$\nu_{02}$ & $\nu_{12}$ & $\nu_{22}$ & $\nu_{32}$ & $\nu_{42}$\\
$\nu_{03}$ & $\nu_{13}$ & $\nu_{23}$ & $\nu_{33}$ & $\nu_{43}$\\
\end{tabular}
\right) \times
\left(
\begin{tabular}{ccccc}
$\frac{1}{\gamma_0}$ & 0 & 0 & 0 & 0\\
0 & $\frac{1}{\gamma_1}$ & 0 & 0 & 0\\
0 & 0 & $\frac{1}{\gamma_2}$ & 0 & 0\\
0 & 0 & 0 & $\frac{1}{\gamma_3}$ & 0\\
0 & 0 & 0 & 0 & $\frac{1}{\gamma_4}$\\
\end{tabular}
\right)\\
= & \left(
\begin{tabular}{ccccc}
$\frac{\nu_{01}}{\gamma_0}Q_1$ & $\frac{\nu_{11}}{\gamma_1}Q_1$ &
$\frac{\nu_{21}}{\gamma_2}Q_1$ & $\frac{\nu_{31}}{\gamma_3}Q_1$ &
$\frac{\nu_{41}}{\gamma_4}Q_1$\\
$\frac{\nu_{02}}{\gamma_0}Q_2$ & $\frac{\nu_{12}}{\gamma_1}Q_2$ &
$\frac{\nu_{22}}{\gamma_2}Q_2$ & $\frac{\nu_{32}}{\gamma_3}Q_2$ &
$\frac{\nu_{42}}{\gamma_4}Q_2$\\
$\frac{\nu_{03}}{\gamma_0}Q_3$ & $\frac{\nu_{13}}{\gamma_1}Q_3$ &
$\frac{\nu_{23}}{\gamma_2}Q_3$ & $\frac{\nu_{33}}{\gamma_3}Q_3$ &
$\frac{\nu_{43}}{\gamma_4}Q_3$\\
\end{tabular}
\right)
\end{aligned}
\]
\end{itemize}
\section{Solution}
Use Newton-Rhapson method with objective function $f(x)=0$, Jacobian $J(x)$,
with LR=PDA factorization for optimal jacobian condition and line search
(batcracking) to remain on valid steps fulfilling all $m_i>0$
\end{document}


    \filetitle{gamma}{Create function proportional to log of Chi-Squared distribution}{logdist/chisquare}

	\paragraph{Syntax}\label{syntax}

\begin{verbatim}
F = logdist.chisquare(Df)
\end{verbatim}

\paragraph{Input arguments}\label{input-arguments}

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  \texttt{Df} {[} integer {]} - Degrees of freedom of Chi-squared
  distribution.
\end{itemize}

\paragraph{Output arguments}\label{output-arguments}

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  \texttt{F} {[} function\_handle {]} - Function handle returning a
  value proportional to the log of the gamma density.
\end{itemize}

\paragraph{Description}\label{description}

See \href{logdist/Contents}{help on the logdisk package} for details on
using the function handle \texttt{F}.

\paragraph{Example}\label{example}


\documentclass[11pt]{article}

\usepackage{amsmath, amssymb, amsthm, mathtools}
\usepackage{lmodern, microtype}
\usepackage{enumitem}

% theorems
\usepackage{amsthm}

\theoremstyle{definition}
\newtheorem{definition}{Definition}
\newtheorem{exercise}{Exercise}

\theoremstyle{plain}
\newtheorem{theorem}{Theorem}
\newtheorem{lemma}{Lemma}
\newtheorem{corollary}{Corollary}

\newtheoremstyle{myremark}%
{}%
{}%
{}%
{}%
{\itshape}%
{.}%
{ }%
{\thmname{#1}\thmnumber{ #2}\thmnote{ (#3)}}%

\theoremstyle{myremark}
\newtheorem*{example}{Example}
\newtheorem*{remark}{Remark}

\numberwithin{equation}{section}
\renewcommand\qedsymbol{$\blacksquare$}

% Github logo
\usepackage{fontawesome5}

% hyperlinks
\usepackage{hyperref}
\hypersetup{
	colorlinks=true,
	linkcolor = {magenta}
}

% commands
\renewcommand{\vec}[1]{\boldsymbol{#1}}
\newcommand{\mat}[1]{\boldsymbol{#1}}

\newcommand{\N}{\mathbb{N}}
\newcommand{\Z}{\mathbb{Z}}
\newcommand{\R}{\mathbb{R}}

\DeclarePairedDelimiterX{\innerp}[2]{\langle}{\rangle}{#1, #2}
\DeclarePairedDelimiterX{\norm}[1]{\lVert}{\rVert}{#1}

\begin{document}

\begin{center}
	\Large
	Chapter 3 Notes for  \\
	\textit{Mathematics for Machine Learning} \\
	by Deisenroth, Faisal, Ong

	\vspace{1ex}
	{
		\normalsize alstonlo \href{https://github.com/alstonlo}{\faGithub}
	}
\end{center}

\paragraph{Disclaimer} The following highlights some \textit{key} ideas and examples from the text. It is by no means comprehensive!

\paragraph{Notation} Let $V$ be an $n$-dimensional vector space over $\R$.

\begin{definition}
A norm on $V$ is a map $\norm{\cdot}\colon V \to \R$ that is
\begin{itemize}
	\item Absolutely homogeneous:
	$\forall \vec{x}\in V, \, \forall \lambda \in \R, \, \norm{\lambda \vec{x}} = |\lambda|\norm{\vec{x}}$

	\item Triangle inequality:
	$\forall \vec{x}, \vec{y} \in V, \, \norm{\vec{x} + \vec{y}} \leq \norm{\vec{x}} + \norm{\vec{y}}$

	\item Positive definite:
	$\forall \vec{x}\in V, \, \norm{\vec{x}} \geq 0$ and $\norm{\vec{x}} = 0 \iff \vec{x} = \vec{0}$
\end{itemize}
\end{definition}

\begin{definition}
An inner product on $V$ is a map $\innerp{\cdot}{\cdot}\colon V \times V \to \R$ that is
\begin{itemize}
	\item Bilinear, i.e.\@ linear in its first and second slot

	\item Symmetric:
	$\forall \vec{x}, \vec{y} \in V, \, \innerp{\vec{x}}{\vec{y}} = \innerp{\vec{y}}{\vec{x}}$

	\item Positive definite:
	$\forall \vec{x}\in V, \, \innerp{\vec{x}}{\vec{x}} \geq 0$ and $\innerp{\vec{x}}{\vec{x}} = 0 \iff \vec{x} = \vec{0}$
\end{itemize}
\end{definition}

\begin{theorem}
	Given an ordered basis $B$ of $V$, a map $\innerp{\cdot}{\cdot}$ is an inner product if and only if there exists a symmetric, positive-definite matrix $\mat{A} \in \R^{n \times n}$ with $\innerp{\vec{x}}{\vec{y}} = \hat{\vec{x}}^\top \mat{A} \hat{\vec{y}}$ for all $\vec{x}, \vec{y} \in V$. Here, $\hat{\vec{x}}, \hat{\vec{y}}$ are the coordinate vectors of $\vec{x}, \vec{y}$ with respect to $B$.
\end{theorem}

\begin{definition}
	An inner product space is a pair $(V, \innerp{\cdot}{\cdot})$.
\end{definition}

\begin{remark}
	Any inner product $\innerp{\cdot}{\cdot}$ induces a norm $n(\vec{x}) = \innerp{\vec{x}}{\vec{x}}^{1/2}$. Herein, we will always assume the norm is the induced one, unless stated otherwise.
\end{remark}

\begin{example}
	The dot product (i.e.\@ $\vec{x} \cdot \vec{y} = \vec{x}^\top \vec{y}$) is an inner product that induces the $\ell_2$ (Euclidean) norm $\norm{\vec{x}}_2 = (\vec{x}^\top \vec{x})^{1/2}$. The inner product space $(V, \cdot)$ is called a Euclidean vector space in particular.
\end{example}

\begin{theorem}[Cauchy-Schwarz Inequality]
	Given $(V, \innerp{\cdot}{\cdot})$, the induced norm satisfies $|\innerp{\vec{x}}{\vec{y}}| \leq \norm{\vec{x}}\norm{\vec{y}}$ for all $\vec{x}, \vec{y} \in V$.
\end{theorem}

\begin{remark}
	Given $(V, \innerp{\cdot}{\cdot})$, by the Cauchy-Schwarz inequality,
	\[\forall \vec{x}, \vec{y} \in V \setminus \{\vec{0}\}, \; -1 \leq \frac{\innerp{\vec{x}}{\vec{y}}}{\norm{\vec{x}}\norm{\vec{y}}} \leq 1\]
	Then the angle between $\vec{x}$ and $\vec{y}$ is defined to be the unique $\alpha \in [0, \pi]$ such that $\cos \alpha$ has the value above.
\end{remark}

\begin{definition}
	A metric on $V$ is a map $d\colon V \times V \to \R$ that is
	\begin{itemize}
		\item Positive definite: $\forall \vec{x}, \vec{y} \in V, \, d(\vec{x}, \vec{y}) \geq 0$ and $d(\vec{x}, \vec{y}) = 0 \iff \vec{x} = \vec{y}$

		\item Symmetric:  $\forall \vec{x}, \vec{y} \in V, \, d(\vec{x}, \vec{y}) = d(\vec{y}, \vec{x})$

		\item Triangle inequality:
		$\forall \vec{x}, \vec{y}, \vec{z} \in V, \, d(\vec{x}, \vec{z}) \leq d(\vec{x}, \vec{y}) + d(\vec{y}, \vec{z})$
	\end{itemize}
\end{definition}

\begin{remark}
	Given $(V, \innerp{\cdot}{\cdot})$, the function $d(\vec{x}, \vec{y}) = \norm{\vec{x} - \vec{y}}$ is a metric on $V$. The value $d(\vec{x}, \vec{y})$ is called the distance between $\vec{x}$ and $\vec{y}$, and in particular, the Euclidean distance if $(V, \innerp{\cdot}{\cdot})$ is Euclidean.
\end{remark}

\begin{definition}
	Given $(V, \innerp{\cdot}{\cdot})$, if $\innerp{\vec{x}}{\vec{y}} = 0$, then we call $\vec{x}$ and $\vec{y}$ orthogonal, denoted $\vec{x} \perp \vec{y}$. If $\norm{\vec{x}} = \norm{\vec{y}} = 1$ as well, then $\vec{x}$ and $\vec{y}$ are orthonormal.
\end{definition}

\begin{definition}
	Given $(V, \innerp{\cdot}{\cdot})$, an orthonormal basis (ONB) of $V$ is a basis whose vectors are mutually orthonormal.
\end{definition}

\begin{definition}
	An orthogonal matrix is a matrix $\mat{A} \in \R^{n \times n}$ that satisfies $\mat{A} \mat{A}^\top = \mat{A}^\top \mat{A} = \mat{I}$, or equivalently, $\mat{A}^{-1} = \mat{A}^\top$.
\end{definition}

\begin{theorem}
	Given $(V, \innerp{\cdot}{\cdot})$ and an ordered ONB $B$, if $\Phi\colon V \to V$ is a linear transformation such that $A_\Phi$ (with respect to $B$) is orthogonal, then $\Phi$ preserves inner products, i.e.\@ $\innerp{\Phi\vec{x}}{\Phi\vec{y}} = \innerp{\vec{x}}{\vec{y}}$). Hence, $\Phi$ also preserves distances and angles.
\end{theorem}

\begin{definition}
	Given $(V, \innerp{\cdot}{\cdot})$, if $U$ is a $k$-dimensional subspace of $V$, then its orthogonal complement is $U^\perp = \{\vec{w} \in V \mid \vec{w} \perp \vec{u} \text{ for all } \vec{u} \in U\}$.
\end{definition}

\begin{remark}
	$U^\perp$ is an $(n - k)$-dimensional subspace of $V$ with $U \oplus U^\perp = V$.
\end{remark}

\begin{definition}
	Given $(V, \innerp{\cdot}{\cdot})$ and subspace $U \subseteq V$, an orthogonal projection is the linear transformation $\pi_U \colon V \to U$ that maps $\pi_U \vec{v} = \vec{u}$, where $\vec{u} \in U$ satisfies $\vec{v} = \vec{u} + \vec{w}$ for some $\vec{w} \in U^\perp$.
\end{definition}

\begin{remark}
	If $\pi_U$ is a orthogonal projection, then $(\pi_U)^2 = \pi_U$, and for all $\vec{v} \in V$ and $\vec{u} \in U$, we have $\norm{\vec{v} - \pi_U\vec{v}} \leq \norm{\vec{v} - \vec{u}}$.
\end{remark}

\end{document}


\chapter{Bayesian Data Analysis}

\begin{multicols}{2}[\subsubsection*{Contents of this chapter}]
   \printcontents{}{1}{\setcounter{tocdepth}{2}}
\end{multicols}

\section{Markov Chain Monte Carlo Methods}
Markov Chain Monte Carlo (MCMC) methods allow for the approximate solution of Bayesian inference problems by drawing samples from the posterior distribution.

Plain vanilla MCMC works as follows:

\begin{enumerate}
\item Make an initial guess $\theta_0$ for the value of the latent variables. This is the starting point for the Markov Chain, which could be picked randomly or, for example, the maximum a posteriori estimate.
\item Calculate the probability of observing the data based on these parameters ($p(\{\mathrm{data}\}|\theta_0)$).
\item Suggest values $\theta$ where the Markov Chain might jump next. (The way guesses are generated is where optimized sampling might comes in.)
\item Calculate $p(\{\mathrm{data}\}|\theta)$.
\item Calculate a probability of jumping to the new values, $p_{\mathrm{jump}} = \min\left(\frac{p(\{\mathrm{data}\}|\theta)}{p(\{\mathrm{data}\}|\theta_0)}, 1\right)$.
\item With probability $p_\mathrm{jump}$, let the Markov Chain jump $\theta_0 \rightarrow \theta$.
\item Repeat steps 3-6
\end{enumerate}

It can be shown that upon convergence, the probability of the Markov Chain reaching particular values of the latent variables is given by the posterior distribution. In other words, MCMC is a trick to use likelihood (and priors) to sample the posterior distribution. Certain pathological posteriors can make it difficult or impossible for the Markov Chain to sample the full posterior, and there is also no completely certain way to say that convergence has been achieved. I personally sample using multiple independent chains, and assume convergence when the posteriors sampled by all chains looks identical. I also test how robust the results are to changes in the priors.


\section{Lake at rest with an immersed bump}

This is a simple test if the method is well-balanced.
The initial condition is a lake at rest with water depth $0.5$. The topography is
\begin{equation}
z(x)= \left\{ \begin{array}{ll}
 0.2-0.05\left(x-10\right)^2& ~\textrm{if}\quad 8 \leq x \leq 12\,,\\
 0& ~\textrm{otherwise,}\\
\end{array} \right.
\end{equation}
The analytical solution is obviously a lake at rest, that is, $w=0.5$ and $u=v=0$.

\subsection{Results}

Setting up the boundaries to be reflective, we should see excellent agreement between the analytical and numerical solutions if the method is well-balanced. Some oscillations may occur, but if the method is well-balanced, they should be very close to the order of the machine precision. The following three figures show the stage, $x$-momentum, and $x$-velocity after running \anuga{} for some time.

\begin{figure}
\begin{center}
\includegraphics[width=0.9\textwidth]{stage_plot.png}
\end{center}
\caption{Stage results}
\end{figure}

\begin{figure}
\begin{center}
\includegraphics[width=0.9\textwidth]{xmom_plot.png}
\end{center}
\caption{Xmomentum results}
\end{figure}

\begin{figure}
\begin{center}
\includegraphics[width=0.9\textwidth]{xvel_plot.png}
\end{center}
\caption{Xvelocity results}
\end{figure}

\endinput


\section{Generalizable Robustness by Confidence Calibration of Adversarial Training}
\label{sec:main}

To start, we briefly review adversarial training on $L_\infty$ adversarial examples \cite{MadryICLR2018}, which has become standard to train robust models, \cf \secref{sec:main-at}.
However, robustness does not generalize to larger perturbations or unseen attacks. We hypothesize this to be the result of enforcing high-confidence predictions on adversarial examples.
\ConfTrain addresses this issue with minimal modifications, \cf \secref{subsec:main-ccat} and \algref{alg:main-ccat}, by encouraging low-confidence predictions on adversarial examples. During testing, adversarial examples can be rejected by confidence thresholding.

\textbf{Notation:}
%
We consider a classifier $f:\R^d \rightarrow \R^K$ with $K$ classes where $f_k$ denotes the confidence for class $k$. While we use the cross-entropy loss $\cL$ for training, our approach also generalizes to other losses. Given $x \in \R^d$ with class $y{\,\in\,}\{1,\ldots,K\}$, we let $f(x) :=\argmax_k f_k(x)$ denote the predicted class for notational convenience. For $f(x) = y$, an adversarial example $\tilde{x} = x+\delta$ is defined as a ``small'' perturbation $\delta$ such that $f(\tilde{x}) \neq y$, \ie, the classifier changes its decision. The strength of the change $\delta$ is measured by some $L_p$-norm, $p \in \{0,1,2,\infty\}$. Here, $p=\infty$ is a popular choice as it leads to the smallest perturbation per pixel.

\subsection{Problems of Adversarial Training}
\label{sec:main-at}

Following \cite{MadryICLR2018}, adversarial training is given as the following min-max problem:
\vspace*{0px}
\begin{align}
    \min\limits_w \Exp\left[\max\limits_{\|\delta\|_\infty \leq \epsilon}\, \cL(f(x + \delta; w), y)\right]\label{eq:adversarial-training}
\end{align}
\vskip -4px
with $w$ being the classifier's parameters. During mini-batch training the inner maximization problem,
\vspace*{0px}
\begin{align}
    \max\limits_{\|\delta\|_\infty \leq \epsilon} \cL(f(x + \delta; w), y),\label{eq:attack}
\end{align}
\vskip -4px
is approximately solved. In addition to the $L_\infty$-constraint, a box constraint is enforced for images, \ie, $\tilde{x}_i = (x + \delta)_i \in [0,1]$. Note that maximizing the cross-entropy loss is equivalent to finding the adversarial example with \emph{minimal} confidence in the true class. For neural networks, this is generally a non-convex optimization problem. In \citep{MadryICLR2018} the problem is tackled using projected gradient descent (\PGD), initialized using a random $\delta$ with $\|\delta\|_\infty \leq \epsilon$.

In contrast to adversarial training as proposed in \citep{MadryICLR2018}, which computes adversarial examples for the \emph{full} batch in each iteration, others compute adversarial examples only for \emph{half} the examples of each batch \citep{SzegedyICLR2014}. Instead of training \emph{only} on adversarial examples, each batch is divided into $50\%$ clean and $50\%$ adversarial examples. Compared to \eqnref{eq:adversarial-training}, $50\%$/$50\%$ adversarial training effectively minimizes
\vspace*{0px}
\begin{align}
    \underbrace{\Exp\Big[\max\limits_{\|\delta\|_\infty \leq \epsilon} \cL(f(x + \delta; w), y)\Big]}_{\text{50\% adversarial training}} + \underbrace{\Exp\big[\cL(f(x; w), y)\big]}_{\text{50\% ``clean'' training}}.\label{eq:50-50-adversarial-training}
\end{align}
\vskip -4px
This improves test accuracy on clean examples compared to $100\%$ adversarial training but typically leads to worse robustness. Intuitively, by balancing both terms in \eqnref{eq:50-50-adversarial-training}, the trade-off between accuracy and robustness can already be optimized to some extent \citep{StutzCVPR2019}.

\begin{figure}[t]
    \vspace*{-8px}

    \centering
    \begin{minipage}{0.49\textwidth}
        \hspace*{-0.2cm}
        \includegraphics[width=1\textwidth]{fig_mmnist_advtrain_0_interpolation}

        \hspace*{-0.2cm}
        \includegraphics[width=1\textwidth]{fig_mmnist_ours10_0_interpolation}
    \end{minipage}
    \vspace*{-10px}

    \caption{\textbf{Extrapolation of Uniform Predictions.} We plot the confidence in each class along an interpolation between two test examples $x$ and $x'$, ``2'' and ``7'', on MNIST \cite{LecunIEEE1998}: $(1 - \kappa)x + \kappa x'$ where $\kappa$ is the interpolation factor. \ConfTrain quickly yields low-confidence, uniform predictions in between both examples, extrapolating the behavior enforced within the $\epsilon$-ball during training. Regular adversarial training, in contrast, consistently produces high-confidence predictions, even on unreasonable inputs.}
    \label{fig:interpolation}
    \vspace*{-2px}
\end{figure}

\textbf{Problems:}
%
Trained on $L_\infty$ adversarial examples, the robustness of adversarial training does not generalize to previously unseen adversarial examples, including larger perturbations or other $L_p$ adversarial examples. We hypothesize that this is because adversarial training explicitly enforces high-confidence predictions on $L_\infty$ adversarial examples within the $\epsilon$-ball seen during training (``seen'' in \figref{fig:introduction}). However, this behavior is difficult to extrapolate to arbitrary regions in a meaningful way. Thus, it is not surprising that adversarial examples can often be found right beyond the $\epsilon$-ball used during training, \cf \figref{fig:introduction} (top left). This can be described as ``overfitting'' to the $L_\infty$ adversarial examples used during training. Also, larger $\epsilon$-balls around training examples might include (clean) examples from other classes. Then, \eqnref{eq:attack} will focus on these regions and reduce accuracy as considered in our theoretical toy example, see Proposition \ref{prop:toy-example}, and related work \citep{JacobsenICLR2019,JacobsenARXIV2019}.

As suggested in \figref{fig:introduction}, both problems can be addressed by enforcing low-confidence predictions on adversarial examples in the $\epsilon$-ball. In practice, we found that the low-confidence predictions on adversarial examples within the $\epsilon$-ball are extrapolated beyond the $\epsilon$-ball, \ie, to larger perturbations, unseen attacks or distal adversarial examples. This allows to reject adversarial examples based on their low confidence. We further enforce this behavior by explicitly encouraging a ``steep'' transition from high-confidence predictions (on clean examples) to low-confidence predictions (on adversarial examples). As result, the (low-confidence) prediction is almost flat close to the boundary of the $\epsilon$-ball. Additionally, there is no incentive to deviate from the uniform distribution outside of the $\epsilon$-ball. For example, as illustrated in \figref{fig:interpolation}, the confidence stays low in between examples from different classes and only increases if necessary, \ie, close to the examples.

\begin{algorithm}[t]
	\caption{\textbf{Confidence-Calibrated Adversarial Training (\ConfTrain).}  The only changes compared to standard adversarial training are the attack (line \ref{line:attack}) and the probability distribution over the classes (lines \ref{line:lambda} and \ref{line:label}), which becomes more uniform as distance $\norm{\delta}_\infty$ increases. During testing, low-confidence (adversarial) examples are rejected.}
	\label{alg:main-ccat}
	\begin{algorithmic}[1]
		\WHILE{true}
		\STATE choose random batch $(x_1,y_1),\ldots,(x_B,y_B)$.
		\FOR{$b = 1,\ldots,\nicefrac{B}{2}$}
		\STATE $\delta_b:=\argmax\limits_{\|\delta\|_\infty \leq \epsilon} \max\limits_{k \neq y_b} f_k(x_b{+}\delta)$\label{line:attack} (\eqnref{eq:conf-attack})
		\STATE $\tilde{x}_b := x_b + \delta_b$
		\STATE $\lambda(\delta_b) := (1 - \min(1, \nicefrac{\|\delta_b\|_\infty}{\epsilon}))^\rho$ (\eqnref{eq:lambda})\label{line:lambda}
		\STATE $\tilde{y_b}\,{:=}\,\lambda(\delta_b)\,\text{one\_hot}(y_b)\,{+}\,(1\,{-}\,\lambda(\delta_b)) \frac{1}{K}$ (\eqnref{eq:distribution})\label{line:label}
		\ENDFOR
		\STATE update parameters using \eqnref{eq:50-50-adversarial-training}:\\\hspace*{1cm}$\sum_{b = 1}^{\nicefrac{B}{2}} \mathcal{L}(f(\tilde{x}_b), \tilde{y}_b) + \sum_{b = \nicefrac{B}{2}}^{B} \mathcal{L}(f(x_b), y_b)$
		\ENDWHILE
	\end{algorithmic}
\end{algorithm}

\subsection{Confidence-Calibrated Adversarial Training}
\label{subsec:main-ccat}

\textbf{Confidence-calibrated adversarial training (\ConfTrain)} addresses these problems with minimal modifications, as outlined in \algref{alg:main-ccat}. During training, we train the network to predict a convex combination of (correct) one-hot distribution on clean examples and uniform distribution on adversarial examples as target distribution within the cross-entropy loss. During testing, adversarial examples can be rejected by confidence thresholding: adversarial examples receive near-uniform confidence while test examples receive high-confidence. By extrapolating the uniform distribution beyond the $\epsilon$-ball used during training, previously unseen adversarial examples such as larger $L_\infty$ perturbations can be rejected, as well. In the following, we first introduce an alternative objective for generating adversarial examples. Then, we specifically define the target distribution, which becomes more uniform with larger perturbations $\|\delta\|_\infty$. In \algref{alg:main-ccat}, these changes correspond to lines \ref{line:attack}, \ref{line:lambda} and \ref{line:label}, requiring only few lines of code in practice.

Given an example $x$ with label $y$, our adaptive attack during training maximizes the confidence in any other label $k \neq y$. This results in effective attacks against \ConfTrain, as \ConfTrain will reject low-confidence adversarial examples:
\vspace*{0px}
\begin{align}
 \max\limits_{\|\delta\|_\infty \leq \epsilon} \max\limits_{k\neq y}f_k(x + \delta;w) \label{eq:conf-attack}
\end{align}
\vskip -4px
Note that \eqnref{eq:attack}, in contrast, minimizes the confidence in the true label $y$. Similarly, \citep{GoodfellowOPENREVIEW2019} uses targeted attacks in order to maximize confidence, whereas ours is untargeted and, thus, our objective is the maximal confidence over all other classes.

Then, given an adversarial example from \eqnref{eq:conf-attack} during training, \ConfTrain uses the following combination of uniform and one-hot distribution as target for the cross-entropy loss:
\vspace*{-8px}
\begin{align}
	\tilde{y} = \lambda(\delta) \,\,\text{one\_hot}(y) + \big(1-\lambda(\delta)\big) \frac{1}{K},\label{eq:distribution}
\end{align}
\vskip -4px
with $\lambda(\delta) \in [0,1]$ and $\text{one\_hot}(y) \in \{0,1\}^K$ denoting the one-hot vector corresponding to class $y$. Thus, we enforce a convex combination of the original label distribution and the uniform distribution which is controlled by the parameter~$\lambda = \lambda(\delta)$, computed given the perturbation~$\delta$. We choose $\lambda$ to decrease with the distance $\|\delta\|_\infty$ of the adversarial example to the attacked example $x$ with the intention to enforce uniform predictions when $\|\delta\|_\infty = \epsilon$. Then, the network is encouraged to extrapolate this uniform distribution beyond the used $\epsilon$-ball. Even if extrapolation does not work perfectly, the uniform distribution is much more meaningful for extrapolation to arbitrary regions as well as regions between classes compared to high-confidence predictions as encouraged in standard adversarial training, as demonstrated in \figref{fig:interpolation}. For controlling the trade-off $\lambda$ between one-hot and uniform distribution, we consider the following ``power transition'':
\vspace*{0px}
\begin{align}
    \begin{split}
        \lambda(\delta) :=& \Big(1 - \min\Big(1, \frac{\|\delta\|_\infty}{\epsilon}\Big)\Big)^\rho
    \end{split}\label{eq:lambda}
\end{align}
\vskip -4px
This ensures that for $\delta = 0$ we impose the original (one-hot) label. For growing $\delta$, however, the influence of the original label decays proportional to $\|\delta\|_\infty$. The speed of decay is controlled by the parameter $\rho$.
For $\rho=10$, \figref{fig:introduction} (top right) shows the transition as approximated by the network.
The power transition ensures that for $\|\delta\|_\infty \geq \epsilon$, \ie, perturbations larger than encountered during training, a uniform distribution is enforced as $\lambda$ is $0$. We train on $50\%$ clean and $50\%$ adversarial examples in each batch, as in \eqnref{eq:50-50-adversarial-training}, such that the network has an incentive to predict correct labels.

The convex combination of uniform and one-hot distribution in \eqnref{eq:distribution} resembles the label smoothing regularizer introduced in \cite{SzegedyCVPR2016}. In concurrent work, label smoothing has also been used as regularizer for adversarial training \cite{ChengARXIV2020}. However, in our case, $\lambda = \lambda(\delta)$ from \eqnref{eq:lambda} is not a fixed hyper-parameter as in \cite{SzegedyCVPR2016,ChengARXIV2020}. Instead, $\lambda$ depends on the perturbation $\delta$ and reaches zero for $\|\delta\|_\infty = \epsilon$ to encourage low-confidence predictions beyond the $\epsilon$-ball used during training. Thereby, $\lambda$ explicitly models the transition from one-hot to uniform distribution.

\subsection{Confidence-Calibrated Adversarial Training Results in Accurate Models}
\label{sec:main-proposition}

Proposition \ref{prop:toy-example} discusses a problem where standard adversarial training is unable to reconcile robustness and accuracy while \ConfTrain is able to obtain \emph{both} robustness and accuracy:

\begin{proposition}\label{prop:toy-example}
    We consider a classification problem with two points $x=0$ and $x=\epsilon$ in $\mR$ with deterministic labels, \ie,
    $p(y=2|x=0)=1$ and $p(y=1|x=\epsilon)=1$, such that the problem is fully determined by the probability $p_0=p(x=0)$.
    The Bayes error of this classification problem is zero. Let the predicted probability distribution over classes be $\tilde{p}(y|x)=\frac{e^{g_y(x)}}{e^{g_1(x)}+e^{g_2(x)}}$, where $g:\R^d \rightarrow \R^2$ is the classifier and we assume that the function $\lambda:\mR_+ \rightarrow [0,1]$ used in \ConfTrain is monotonically decreasing and $\lambda(0)=1$. Then, the error of the Bayes optimal classifier (with cross-entropy loss) for
    \vspace*{-8px}
    \begin{itemize}
    \item adversarial training on $100\%$ adversarial examples, \cf \eqnref{eq:adversarial-training},
    is $\min\{p_0,1-p_0\}$.
    \vspace*{-3px}
    \item adversarial training on $50\%$/$50\%$ adversarial/clean examples per batch, \cf \eqnref{eq:50-50-adversarial-training},
    is $\min\{p_0,1 - p_0\}$.
    \vspace*{-3px}
    \item \ConfTrain on $50\%$ clean and $50\%$ adversarial examples, \cf \algref{alg:main-ccat},
    is \emph{zero}
    if $\lambda(\epsilon) < \min\left\{\nicefrac{p_0}{1-p_0},\nicefrac{1-p_0}{p_0}\right\}$.
    \vspace*{-6px}
    \end{itemize}
\end{proposition}

Here, $100\%$ and $50\%$/$50\%$ standard adversarial training are unable to obtain \emph{both} robustness and accuracy: The $\epsilon$-ball used during training contains examples of different classes such that adversarial training enforces high-confidence predictions in contradicting classes. \ConfTrain addresses this problem by encouraging low-confidence predictions on adversarial examples within the $\epsilon$-ball. Thus, \ConfTrain is able to improve accuracy while preserving robustness.

\section{Detection and Robustness Evaluation with Adaptive Attack}
\label{sec:evaluation-attack}

\ConfTrain allows to reject (adversarial) inputs by confidence-thresholding before classifying them. As we will see, this ``reject option'', is also beneficial for standard adversarial training (\AdvTrain). Thus, evaluation also requires two stages: First, we fix the confidence threshold at $99\%$ true positive rate (TPR), where correctly classified clean examples are positives such that at most $1\%$ (correctly classified) clean examples are rejected.
Second, on the non-rejected examples, we evaluate accuracy and robustness using \emph{confidence-thresholded} (robust) test error.

\subsection{Adaptive Attack}
\label{subsec:evaluation-attack}

As \ConfTrain encourages low confidence on adversarial examples, we use \PGD to maximize the confidence of adversarial examples, \cf \eqnref{eq:conf-attack}, as effective adaptive attack against \ConfTrain. In order to effectively optimize our objective, we introduce a simple but crucial improvement: after each iteration, the computed update is only applied if the objective is improved; otherwise the learning rate is reduced.
Additionally, we use momentum \citep{DongCVPR2018} and run the attack for exactly $T$ iterations, choosing the perturbation corresponding to the best objective across all iterations. In addition to random initialization, we found that $\delta = 0$ is an effective initialization against \ConfTrain. We applied the same principles for \cite{IlyasICML2018}, \ie, \PGD with approximated gradients, \eqnref{eq:conf-attack} as objective, momentum and backtracking; we also use \eqnref{eq:conf-attack} as objective for the black-box attacks of \cite{AndriushchenkoARXIV2019,NarodytskaCVPRWORK2017,KhouryARXIV2018}.

\subsection{Detection Evaluation}

In the first stage, we consider a detection setting: adversarial example are \emph{negatives} and correctly classified clean examples are \emph{positives}. The confidence threshold $\tau$ is chosen extremely conservatively by requiring a \textbf{$\boldsymbol{99\%}$ true positive rate (TPR)}: at most $1\%$ of correctly classified clean examples can be rejected. As result, the confidence threshold is determined \emph{only} by correctly classified clean examples, independent of adversarial examples. Incorrectly rejecting a significant fraction of correctly classified clean examples is unacceptable. This is also the reason why we do not report the area under the receiver operating characteristic (ROC) curve as related work \citep{LeeNIPS2018,MaICLR2018}.
Instead, we consider the \textbf{false positive rate (FPR)}.
The supplementary material includes a detailed discussion.

\subsection{Robustness Evaluation}

In the second stage, after confidence-thresholding, we consider the widely used robust test error (\RTE) \cite{MadryICLR2018}. It quantifies the model's test error in the case where all test examples are allowed to be attacked, \ie, modified within the chosen threat model, \eg, for $L_p$:
\begin{align}
    \text{``Standard'' }\RTE = \frac{1}{N}\sum_{n = 1}^N \;\max\limits_{\|\delta\|_p\leq \epsilon} \Id_{f(x_n + \delta)\neq y_n}
\end{align}
where $\{(x_n,y_n)\}_{n = 1}^N$ are test examples and labels. In practice, \RTE is computed empirically using adversarial attacks. Unfortunately, standard \RTE does not take into account the option of rejecting (adversarial) examples.

We propose a generalized definition adapted to our confidence-thresholded setting where the model can reject examples. For fixed confidence threshold $\tau$ at $99\%$TPR, the \textbf{confidence-thresholded \RTE} is defined as
\begin{align}
    \RTE(\tau) = \frac{
    	\sum\limits_{n=1}^N \;\max\limits_{\|\delta\|_p\leq \epsilon, c(x_n + \delta)\geq \tau} \Id_{f(x_n + \delta)\neq y_n}
    }
    {
    	\sum\limits_{n=1}^N \;\max\limits_{\|\delta\|_p\leq \epsilon} \Id_{c(x_n + \delta)\geq \tau}
    }\label{eq:conf-rte}
\end{align}
with $c(x) = \max_k f_k(x)$ and $f(x)$ being the model's confidence and predicted class on example $x$, respectively. Essentially, this is the \textbf{test error on test examples that can be modified within the chosen threat model \emph{and} pass confidence thresholding}. For $\tau=0$ (\ie, all examples pass confidence thresholding) this reduces to the standard \RTE, comparable to related work. We stress that our adaptive attack in \eqnref{eq:conf-attack} directly maximizes the numerator of \eqnref{eq:conf-rte} by maximizing the confidence of classes not equal $y$. A (clean) \textbf{confidence-thresholded test error ($\TE(\tau)$)} is obtained similarly. In the following, if not stated otherwise, we report \emph{confidence-thresholded} \RTE and \TE as default and omit the confidence threshold $\tau$ for brevity.

\textbf{FPR and \RTE:}
%
FPR quantifies how well an adversary can perturb (correctly classified) examples while not being rejected. The confidence-thresholded \RTE is more conservative as it measures \emph{any} non-rejected error (adversarial or not). As result, \RTE implicitly includes FPR \emph{and} \TE. Therefore, we report only \RTE and include FPRs for all our experiments in the supplementary material.

\begin{figure}[t]
    \vspace*{0px}
    \centering

    \hspace*{-0.3cm}
    \begin{minipage}[t]{0.23\textwidth}
        \vspace*{0px}

        \centering
        \includegraphics[width=1.05\textwidth]{fig_msvhn_roc}
    \end{minipage}
    \begin{minipage}[t]{0.25\textwidth}
        \vspace*{0px}

        \centering
        \includegraphics[width=1.05\textwidth]{fig_msvhn_rte}
    \end{minipage}\\

    \hspace*{-0.4cm}
    \begin{minipage}[t]{0.45\textwidth}
        \fbox{
            \hspace*{1.6cm}\includegraphics[width=0.575\textwidth]{fig_msvhn_legend}\hspace*{1.6cm}
        }
    \end{minipage}
    \vskip -4px
    \caption{\textbf{ROC and \RTE Curves.} On SVHN, we show ROC curves when distinguishing \emph{correctly classified} test examples from adversarial examples by confidence (left) and (confidence-thresholded) \RTE against confidence threshold $\tau$ (right) for worst-case adversarial examples across $L_\infty$ attacks with $\epsilon = 0.03$. The confidence threshold $\tau$ is chosen exclusively on correctly classified clean examples to obtain $99\%$TPR. For \ConfTrain, this results in $\tau \approx 0.6$. Note that \RTE subsumes both \TE and FPR.}
    \label{fig:experiments-evaluation}
    \vspace*{-2px}
\end{figure}

\textbf{Per-Example Worst-Case Evaluation:}
%
Instead of reporting average or per-attack results, we use a per-example \emph{worst-case} evaluation scheme: For each individual test example, all adversarial examples from all attacks (and restarts) are accumulated. Subsequently, \emph{per test example}, only the adversarial example with highest confidence is considered, resulting in a significantly stronger robustness evaluation compared to related work.


\hypertarget{sparse}{%
\section{Sparse}\label{sparse}}

The Sparse class provides support for sparse matrices. An empty sparse
matrix can be initialized with a given size,

\begin{lstlisting}
var a = Sparse(nrows,ncols)
\end{lstlisting}

Alternatively, a matrix can be created from an array of triplets,

\begin{lstlisting}
var a = Sparse([[row, col, value] ...])
\end{lstlisting}

For example,

\begin{lstlisting}
var a = Sparse([[0,0,2], [1,1,-2]])
\end{lstlisting}

creates the matrix

\begin{lstlisting}
[ 2 0 ]
[ 0 -2 ]
\end{lstlisting}

Once a sparse matrix is created, you can use all the regular arithmetic
operators with matrix operands, e.g.

\begin{lstlisting}
a+b
a*b
\end{lstlisting}


Trust Region refers to a family of optimization methods that operate by assuming a quadratic model is accurate within a local "trust region". The trust region's size is adjusted based on the quadratic model's performance in previous iterations. A summary of Trust Region, as implemented in DDogleg, is found in Algorithm \ref{alg:trust_region}. This implementation\footnote{The more traditional variant described in \cite{numopt2006,fletcher1987} were considered but found to converge slower in test problems.} is primarily based on the description found in \cite{IMM2004}.

\begin{algorithm}{}
\caption{\label{alg:trust_region}Trust Region}
\begin{algorithmic}[1]
	\State $k \gets 0$, $\Delta_0 \in (0,\Delta_{max})$
	\State \quad $\Delta_{max}$ is the maximum trust region size
	\State \quad $\Delta_{0}$ is the initial trust region size. \Comment{Section \ref{section:init_region_size}}
	\While{$k < k_{\mbox{max}}$ and not $done$}
	\State $p_k$ update by optimizing Eq. \ref{eq:trust_region_subproblem} \Comment{Sections \ref{section:cauchy} and \ref{section:dogleg} }
	\State $\delta_f \gets f(x_k) - f(x_k + p_k)$ \Comment{Actual reduction in score}
	\State $\delta_m \gets m_k(0)-m_k(p_k) = -g^T_k p - \frac{1}{2}p^T B_k p$ \Comment{Predicted reduction in score}
	\State $\nu \gets \delta_f / \delta_f$ \Comment{Score reduction ratio}
	\If{ $\delta_f \le 0$ or $\nu <\frac{1}{4}$} \Comment{Score got worse or the model poor?}
		\State $\Delta_{k+1} \gets \frac{1}{2}\Delta_k$
	\Else
		\If{$\nu>\frac{3}{4}$}
			\Comment{The model is good. Increase the region size?}
			\State $\Delta_{k+1} \gets \mbox{min}(\mbox{max}(3\norm{p_k},\Delta_k),\Delta_{\mbox{max}})$
		\Else
			\State $\Delta_{k+1} \gets \Delta_k$
		\EndIf
	\EndIf
	\If{$\delta_f > 0$ and $\nu > 0$} \Comment{Is the solution acceptable?}
		\State $x_{k+1} \gets x_k + p_k$ \Comment{Update the state}
		\State $done$ $\gets$ $\mbox{F-Test}$ or $\mbox{G-Test}$ \Comment{Convergence testing}
	\Else
		\State $x_{k+1} \gets x_k$
	\EndIf

	\State $k \gets k + 1$
	\EndWhile
\end{algorithmic}
\end{algorithm}

At every iteration the Trust Region subproblem is solved for, either exactly or approximately:
\begin{equation}
\begin{array}{lr}
\min\limits_{p\in \R^n} m_k(p) = f_k + g^T_k p + \frac{1}{2}p^T B_k p & s.t. \norm{p} \le \Delta_k
\end{array}
\label{eq:trust_region_subproblem}
\end{equation}
where $m(p) \in \R$ is a quadratic model approximating $f(x_k)$, $p \in \R^N$ is the step or change in state, $B \in \R^{N \times N}$ is a symmetric matrix representing the Hessian or an approximation, and $\Delta_k \in \R^+$ is the trust region size. The unconstrained solution to Eq. \ref{eq:trust_region_subproblem} is easily found by setting the first derivative to zero:
\begin{equation}
p = -B^{-1}_k g_k
\label{eq:TR_unconstrained_solution}
\end{equation}
An exact solution to (\ref{eq:trust_region_subproblem}) is expensive to compute and approximate methods are typically used instead. The Cauchy Point and Dogleg are approximate methods and included in the DDogleg library.

\subsubsection{Cauchy Point}
\label{section:cauchy}

The Cauchy Point is the solution which minimizes (\ref{eq:trust_region_subproblem}) along the steepest descent direction. It is defined as $p^s_k = \tau_k \hat{p}^s_k$ and is relative to $x_{k-1}$, where $\hat{p}^s_k$ is a unit vector, and $\tau_k$ is a scalar.
\begin{equation}
\begin{array}{lr}
\hat{p}^s_k = \min\limits_{p\in \R^n} f_k + g_k^T p & s.t. \norm{p} \le \Delta_k
\end{array}
\end{equation}
The length $\tau_k$ is found by minimizing (\ref{eq:trust_region_subproblem}) along direction $\hat{p}^s_k$
\begin{equation}
\begin{array}{lr}
\tau_k = \min\limits_{\tau \ge 0} m_k(\tau v^s_k) & s.t. \norm{\tau v^s_k} \le \Delta_k
\end{array}
\end{equation}

The solution (see Chapter 4 of \cite{numopt2006} for details and diagrams) is as follows:
\begin{equation}
p^s_k = -\tau_k \frac{\Delta_k}{\norm{g_k}}g_k
\label{eq:cauchy_p}
\end{equation}
\begin{equation}
\tau_k =
	\begin{cases}
		\quad 1 & g_k^T B_k g_k \le 0 \\
		\quad \min\left(1,\norm{g_k}^3/(\Delta_k g_k^T B_k g_k)\right) & g_k^T B_k g_k > 0
	\end{cases}
	\label{eq:cauchy_tau}
\end{equation}

The formulas in (\ref{eq:cauchy_p}) and (\ref{eq:cauchy_tau}) can be improved upon to avoid numerical issues by removing powers of three and division by $\Delta_k$:
\begin{eqnarray}
\hat{g}_k &=& \frac{g_k}{\norm{g_k}} \\
p^s_k &=& -\bar{\tau}_k \hat{g}_k
\end{eqnarray}
\begin{equation}
\bar{\tau}_k = \begin{cases}
		\quad \Delta_k & \hat{g}_k^T B_k \hat{g}_k\le 0 \\
		\quad \min\left(\Delta_k,\norm{g_k}/(\hat{g}_k^T B_k \hat{g}_k)\right) & \hat{g}_k^T B_k \bar{g}_k > 0
	\end{cases}
\end{equation}
The predicted reduction in score is found using:
\begin{equation}
m_k(0)-m_k(p_k) = \bar{\tau}_k \left(\norm{g_k} - \frac{\tau_k \hat{g}_k^T B_k \hat{g}_k}{2} \right)
\end{equation}

\subsubsection{Dogleg}
\label{section:dogleg}

The Dogleg method considers second order terms to provide a more accurate solution to Eq. \ref{eq:trust_region_subproblem}. The optimal solution, as a function of region size, is a curved trajectory. The Dogleg method approximates this curved trajectory using two line segments. The first line starts at the $x_{k-1}$ and ends at the unconstrained Cauchy point. The second heads towards $p^b$ the solution to (\ref{eq:TR_unconstrained_solution}), which is the Gauss-Newton solution. As with equations from Cauchy Point, these equations are not traditional (see \cite{numopt2006,IMM2004}) and have been reformulated to avoid powers of three.
\begin{eqnarray}
\hat{g_k} &=& \frac{g_k}{\norm{g_k}} \\
p^u_k &=& -\frac{g_k}{\hat{g_k}^T B_k \hat{g_k}} \\
p^b_k &=& -B^{-1}_k g_k \\
p^{dog}_k &=&
\begin{cases}
	\tau p^u_k & 0 \le \tau < 1 \\
	p^u_k + (\tau -1)(p^b_k-p^u_k) & 1 \le \tau \le 2
\end{cases}
\end{eqnarray}
where $B_k$ is positive definite, and $p^{dog}_k$ is the point selected by the Dogleg method. The solution to $\tau$ can be easily found by solving along each line segment. If $B_k$ is not positive definite the gradient descent is used instead.

\begin{algorithm}{}
\caption{\label{alg:dogleg_step}Selection of Dogleg Step}
\begin{algorithmic}[1]
  \If{ $B$ is positive definite}
    \If{$\norm{p^b} < \Delta$} \Comment{Gauss-Newton solution inside the trust-region?}
      \State $p^{dog} \gets p^b$
    \ElsIf{$\norm{p^u} \geq \Delta$} \Comment{Cauchy point outside the trust-region?}
    \State $p^{dog} \gets \Delta \frac{p^u}{\norm{p^u}}$
    \Else
    \State $p^{dog} \gets $ intersection of $p^u \rightarrow p^b$ and trust-region
    \EndIf
  \Else
   \State $p^{dog} \gets -\Delta\frac{g}{\norm{g}}$ \Comment{Follow gradient to end of trust region}
  \EndIf
\end{algorithmic}
\end{algorithm}

\subsubsection{Initial Region Size}
\label{section:init_region_size}

Selection of the initial trust region size $\Delta_0$ is important but typically not discussed in reference material \cite{fletcher1987,numopt2006,IMM2004} in detail. Initial region size is typically considered a tuning parameter that the user is supposed to select through trial and error. While the Trust region size is dynamically adjusted at each iteration in the Trust Region approach, the initial selection of the trust region size can significantly influence the final convergence.

Here is an example of a possible failure mode when the trust region's size is poorly selected. With the dogleg method, if $\Delta_0$ is too small then a Cauchy step is selected repeatedly. The Cauchy Point takes much smaller steps, increasing the chances of getting stuck in a local minimum.

DDogleg provides two automatic methods for finding the initial region size, with unreliable results. 1) \emph{Unconstrained initial step} and 2) \emph{Cauchy initial step}. With the unconstrained method, the selected algorithm (e.g. Dogleg or Cauchy) selects a step when given trust region of MAX\_VALUE. The step it selects is used and the trust region is then set to the length of that step. This works well in many problems but can be overly agressive and take a very large step into a distant plateau. The Cauchy initial step method computes the length of a Cauchy step, then sets the region size to be 10x that. This estimate tends to be conservative will in general converge but can converge slowly.

If the automatic methods fail to produce acceptable results then manual tuning will be necessary. One possible manual tuning procedure is to start with $\Delta_0=1$ then trying $\Delta_0=100$, and if results improved try $\Delta_0=10000$. If results don't get better try $0.1$ or other fractions of one.

Recommended Procedure for Selection of Initial Trust Region Size:
\begin{enumargin}{0.2}
\item Turn on verbose output and examine the progress
\item Start with automatic selection using \emph{unconstrained initial step}
\item If this fails then try \emph{Cauchy initial step}
\item If performance is still poor follow manual tuning procedure
\end{enumargin}
For instructions on how to switch between the methods described here consult the JavaDoc of ConfigTrustRegion.

A comparison of different initial conditions for different 'toy' problems is shown in Table \ref{results:initial_region}. In these scenarios, the Automatic Unconstrained method correctly selected the best initial conditions while all the other methods either tied unconstrained's performance or clearly made a poor choice. Unfortunately, these results don't extrapolate to all problems and there are situations where the unconstrained method results in failure. For that reason, the default method is the more conservative Automatic Cauchy.

\begin{table}[h]
\centering
\begin{threeparttable}
\caption{\label{results:initial_region}Comparison of Initial Region Size}
\begin{tabular}{|l||c|c|c||c|c|c||c|c|c||c|c|c|}
\hline
Problem        & \multicolumn{3}{c||}{Automatic} & \multicolumn{3}{c||}{Automatic} & \multicolumn{3}{c||}{Manual} & \multicolumn{3}{c|}{Manual}\\
               & \multicolumn{3}{c||}{Unconstrained} & \multicolumn{3}{c||}{Cauchy} & \multicolumn{3}{c||}{1} & \multicolumn{3}{c|}{100} \\
\hline
               & Fit     & G  & B  & Fit     & G  & B   & Fit     & G  & B  & Fit     & G  & B\\
\hline
Powel          & 2.0e-17 & 17 & 17 & 2.9e-17 & 30 & 26  & 4.9e-18 & 23 & 19 & 2.0e-17 & 17 & 17 \\
Powel Singular & 7.3e-11 & 11 & 11 & 2.3e-10 & 13 & 13  & 6.7e-11 & 11 & 11 & 7.3e-11 & 11 & 11 \\
Helical Valley & 2.5e-26 & 11 & 8  & 5.1e-18 & 11 & 11  & 4.1e-33 & 9  & 8  & 2.7e-26 & 16 & 8\\
B.S. Powell    & 4.2e-31 & 25 & 18 & 3.7e-23 & 73 & 58  & 2.2e-31 & 25 & 18 & 0       & 62 & 43 \\
Bundle 2D      & 3.7e-10 & 111& 36 & 7.0e-18 & 111& 36  & 9.0e-10 & 251& 93 & 3.3e-10 & 112 & 37 \\
Bundle 2D [1]  & 7.0e-19 & 4  & 4  & 7.0e-18 & 4  & 4   & 1.5e-15 & 5  & 5  & 7.0e-18 & 4 & 4 \\ \hline
\end{tabular}
\begin{tablenotes}
\small
\item \emph{Fit} is the final fit score where zero is a perfect fit. \emph{G} is the number of times the gradient was computed. \emph{B} is the number of times the Hessian was computed. B is by far the most expensive step.
\item Unless specified otherwise, all methods use Dogleg with a Cholesky solver.
\item [1] Uses QR with column pivots instead of Cholesky and can handle the nearly singular initial state.
\end{tablenotes}
\end{threeparttable}
\end{table}


% Created 2020-04-08 mié 12:37
% Intended LaTeX compiler: pdflatex
\documentclass[presentation,aspectratio=169]{beamer}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{grffile}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{textcomp}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\usepackage{khpreamble}
\usepackage{epigraph}
\definecolor{inputclr}{rgb}{1, 0.49, 0.0}
\definecolor{outputclr}{rgb}{0., 0.3, 0.6}
\usetheme{default}
\author{Kjartan Halvorsen}
\date{\today}
\title{Process automation laboratory - linearization}
\hypersetup{
 pdfauthor={Kjartan Halvorsen},
 pdftitle={Process automation laboratory - linearization},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 26.3 (Org mode 9.3.6)},
 pdflang={English}}
\begin{document}

\maketitle

\section{Intro}
\label{sec:org2a31e83}
\begin{frame}[label={sec:org5961aaf}]{Why linear systems?}
\setlength\epigraphwidth{.8\textwidth}
\epigraph{Finally, we make some remarks on why linear systems are so important. The answer is simple: because we can solve them!}{\textit{Richard Feynman}\\\url{https://www.feynmanlectures.caltech.edu/I_25.html}}
\end{frame}

\begin{frame}[label={sec:orgce40fa2}]{Linear first-order system}
\[ x + \tau \dot{x} = u \qquad \Leftrightarrow \qquad \dot{x} = \underbrace{-\frac{1}{\tau}x + \frac{1}{\tau} u}_{f(x, u)} \]

\begin{columns}
\begin{column}{0.4\columnwidth}
Step response \[u(t) = \begin{cases} u_0, & t \ge 0,\\ 0, & \text{otherwise} \end{cases}\]
\end{column}

\begin{column}{0.6\columnwidth}
\begin{center}
  \begin{tikzpicture}
    \pgfmathsetmacro{\tconst}{2}
    \pgfmathsetmacro{\kgain}{1}
    \pgfmathsetmacro{\uconst}{1}
    \pgfmathsetmacro{\xmax}{\uconst}
    \pgfmathsetmacro{\ymax}{\uconst/\tconst}
    \begin{axis}[
      %yshift=-5cm,
      clip=false,
      axis lines=middle,
      width = 8cm,
      height = 6cm,
      xlabel = {$x$},
      ylabel = {$\dot{x}$},
      xtick={0, \xmax},
      xticklabels={0, $u_0$},
      ytick={0, \ymax},
      yticklabels={0, $\frac{u_0}{\tau}$},
      %title={$\dot{x} = f(x) = -\frac{1}{\tau} x + \frac{1}{\tau}u_0$},
      ]
      \addplot[outputclr, thick, no marks, domain=-0.2:\xmax+0.2, samples=10] {-1.0/\tconst * x + 1.0/\tconst * \uconst} node[coordinate, pin=0:{$\dot{x}=f(x,u=u_0) = -\frac{1}{\tau} x + \frac{1}{\tau}u_0$}, pos=0.3] {};
    \end{axis}
  \end{tikzpicture}
\end{center}
\end{column}
\end{columns}
\end{frame}

\begin{frame}[label={sec:org3599a14}]{Another relevant first-order system: Logistic growth}
\[ \dot{x} = \underbrace{a\big(1 - \frac{x}{x_{max}}\big)x}_{f(x)} \]

\begin{center}
  \begin{tikzpicture}
    \pgfmathsetmacro{\gconst}{2}
    \pgfmathsetmacro{\xmax}{1}
    \pgfmathsetmacro{\ymax}{\gconst*\xmax/4}
    \begin{axis}[
      %yshift=-5cm,
      clip=false,
      axis lines=middle,
      width = 8cm,
      height = 6cm,
      xlabel = {$x$},
      ylabel = {$\dot{x}$},
      xtick={0, \xmax},
      xticklabels={0, $x_{max}$},
      ytick={0, \ymax},
      yticklabels={0, $a\frac{x_{max}}{4}$},
      ymax=\ymax+0.1,
      xmax=\xmax+0.1,
      %title={$\dot{x} = f(x) = -\frac{1}{\tau} x + \frac{1}{\tau}u_0$},
      ]
      \addplot[outputclr, thick, no marks, domain=-0.05:\xmax+0.05, samples=100] {\gconst * (1- x/\xmax)*x} node[coordinate, pin=0:{$\dot{x}=f(x) = a\big(1 - \frac{x}{x_{max}}\big)x$}, pos=0.7] {};
    \end{axis}
  \end{tikzpicture}
\end{center}
{\footnotesize See 3Blue1Brown \texit{Exponential growth and epidemics} \url{https://youtu.be/Kas0tIxDvrg}}
\end{frame}

\begin{frame}[label={sec:orgf8836be}]{Do in groups}
\begin{center}
  \begin{tikzpicture}
    \pgfmathsetmacro{\gconst}{2}
    \pgfmathsetmacro{\xmax}{1}
    \pgfmathsetmacro{\ymax}{\gconst*\xmax/4}
    \begin{axis}[
      %yshift=-5cm,
      clip=false,
      axis lines=middle,
      width = 8cm,
      height = 6cm,
      xlabel = {$x$},
      ylabel = {$\dot{x}$},
      xtick={0, \xmax},
      xticklabels={0, $x_{max}$},
      ytick={0, \ymax},
      yticklabels={0, $a\frac{x_{max}}{4}$},
      ymax=\ymax+0.1,
      xmax=\xmax+0.1,
      %title={$\dot{x} = f(x) = -\frac{1}{\tau} x + \frac{1}{\tau}u_0$},
      ]
      \addplot[outputclr, thick, no marks, domain=-0.05:\xmax+0.05, samples=100] {\gconst * (1- x/\xmax)*x} node[coordinate, pin=0:{$\dot{x}=f(x) = a\big(1 - \frac{x}{x_{max}}\big)x$}, pos=0.7] {};
    \end{axis}
  \end{tikzpicture}
\end{center}
\begin{enumerate}
\item In breakout rooms: One of you shares this slide (found on canvas, link in the program for the session)
\item Sketch the solution \(x(t)\) using the "bead-on-a-wire" idea for the inital value problem \(x(0) = 0.1x_{max}\).
\end{enumerate}
\end{frame}

\section{Linearization}
\label{sec:org14863e2}
\begin{frame}[label={sec:org43156cb}]{The general idea}
Given a dynamical system described by a nonlinear differential equation
\begin{center}
  \begin{tikzpicture}[node distance=22mm, block/.style={rectangle, draw, minimum width=15mm, minimum height=12mm}, sumnode/.style={circle, draw, inner sep=2pt}]

    \node[coordinate] (input) {};
    \node[block, right of=input, node distance=20mm] (plant)  {System};
    \node[coordinate, right of=plant, node distance=20mm] (output) {};

    \draw[->] (input) -- node[above, pos=0.3, color=inputclr] {$u(t)$} (plant);
    \draw[->] (plant) -- node[above, near end, color=outputclr] {$x(t)$} (output);
  \end{tikzpicture}
\end{center}
\[ \dot{\textcolor{outputclr}{x}} = f(\textcolor{outputclr}{x}, \textcolor{inputclr}{u})\]

Find a linear approximation to the differential equation about an \alert{operating point} \((\textcolor{outputclr}{x_0}, \, \textcolor{inputclr}{u_0})\)
\end{frame}

\begin{frame}[label={sec:org5f5417f}]{The general picture}
\begin{center}
  \begin{tikzpicture}
    \pgfmathsetmacro{\xnoll}{1.5}
    \pgfmathsetmacro{\xmax}{2}
    \pgfmathsetmacro{\fnoll}{sqrt(2-\xnoll)}
    \begin{axis}[
      %yshift=-5cm,
      clip=false,
      axis lines = middle,
      width = 12cm,
      height = 8cm,
      %xlabel = {$x$},
      %ylabel = {$\dot{x}$},
      xtick={0, \xnoll, \xmax},
      xticklabels={0, $x_0$, $x_{max}$},
      ytick={0},
      %title={$\dot{x} = f(x) = \sqrt{x_{max} -  x}$},
      ]
      \addplot[outputclr, thick, no marks, domain=0:\xmax, samples=100] {sqrt(\xmax-x)} node[coordinate, pin=-120:{$\dot{x}=f(x)= \sqrt{x_{max} -  x}$}, pos=0.1] {};
      \addplot[green!70!black, thick, no marks, domain=0.8:\xmax, samples=10] {sqrt(\xmax-\xnoll) - 0.5/sqrt(\xmax-\xnoll)*(x-\xnoll)} node[coordinate, pin=0:{$\dot{x}\approx f(x_0) + \frac{d}{dx}f|_{x_0}(x-x_0)$}, pos=0.1] {};
      \node[coordinate, pin=180:{$f(x_0)$},] at (axis cs: 0.02, \fnoll) {};
      \node at (axis cs: -0.3, 1.5) {$\dot{x}$};
      \node at (axis cs: 1.9, -0.3) {$x$};
      %\node[coordinate, pin=-90:{$x_0$},] at (axis cs: \xnoll, -0.2) {};
    \end{axis}

  \end{tikzpicture}
\end{center}
\end{frame}

\begin{frame}[label={sec:orgcb3dd14}]{Linearizing the tank-valve nonlinear model}
\[ \dot{p} = a_0(u_v - 5)|p_s - p|^{a_1} = f(p, u_v), \quad \text{with} \; a_0=1.1\; \text{and}\; a_1 = 0.47\]
\begin{enumerate}
\item Given operating pressure \(p_0\). Choose operating point \(u_0\) which gives equilibrium \(f(p_0, u_0) = 0\).
\end{enumerate}
\end{frame}

\begin{frame}[label={sec:org56a6724}]{Linearizing the tank-valve nonlinear model}
\[ \dot{p} = a_0(u_v - 5)|p_s - p|^{a_1} = f(p, u_v), \quad \text{with} \; a_0=1.1\; \text{and}\; a_1 = 0.47\]
\begin{enumerate}
\item Given operating pressure \(p_0\). Choose operating point \(u_0\) which gives equilibrium \(f(p_0, u_0) = 0\).
\item Introduce deviation variables: \(u_v = 5 + \textcolor{inputclr}{u}\) and \(p = p_0 + \textcolor{outputclr}{y}\).
\end{enumerate}
\end{frame}

\begin{frame}[label={sec:orga9fd2fc}]{Linearizing the tank-valve nonlinear model}
\[ \dot{p} = a_0(u_v - 5)|p_s - p|^{a_1} = f(p, u_v), \quad \text{with} \; a_0=1.1\; \text{and}\; a_1 = 0.47\]
\begin{enumerate}
\setcounter{enumi}{2}
\item Determine partial derivatives
\begin{align*}
\frac{\partial f}{\partial p} &= a_0(u_v-5)a_1|p_s - p|^{a_1-1}(-1)\\
\frac{\partial f}{\partial u_v} &= a_0|p_s - p|^{a_1}
\end{align*}
\item Evaluate partial derivatives at the operating point \((p_0, u_0\).
\begin{align*}
\frac{\partial f}{\partial p}\big|_{p_0, u_0} &= 0\\
\frac{\partial f}{\partial u_v}\big|_{p_0, u_0} &=  a_0|p_s - p_0|^{a_1}
\end{align*}
\end{enumerate}
\end{frame}

\begin{frame}[label={sec:orgadabd43}]{Linearizing the tank-valve nonlinear model}
\[ \dot{p} = a_0(u_v - 5)|p_s - p|^{a_1} = f(p, u_v), \quad \text{with} \; a_0=1.1\; \text{and}\; a_1 = 0.47\]
\begin{enumerate}
\setcounter{enumi}{3}
\item Evaluate partial derivatives at the operating point \((p_0, u_0\).
\begin{align*}
\frac{\partial f}{\partial p}\big|_{p_0, u_0} &= 0\\
\frac{\partial f}{\partial u_v}\big|_{p_0, u_0} &=  a_0|p_s - p_0|^{a_1}
\end{align*}
\item Form the linearized model
\begin{equation}
\begin{aligned} \dot{p} = \dot{\textcolor{outputclr}{y}} &= f(p, u_v) \approx f(p_0, u_0) + \frac{\partial f}{\partial p}|_{p_0, u_0}(p-p_0) + \frac{\partial f}{\partial u_v}|_{p_0, u_0}(u_v - u_0)\\
 &=  a_0|p_s - p_0|^{a_1} \textcolor{inputclr}{u}.
\end{aligned}
\end{equation}
\end{enumerate}
\end{frame}

\begin{frame}[label={sec:orgccfd0c1}]{Linearizing the tank-valve nonlinear model}
We arrive at the linear model
\begin{equation*}
\begin{aligned}
 \dot{\textcolor{outputclr}{y}} &= a_0|p_s - p_0|^{a_1} \textcolor{inputclr}{u}, \qquad \text{which in the Laplace domain is}\\[4mm]
 \textcolor{outputclr}{Y(s)} &= \frac{a_0|p_s - p_0|^{a_1}}{s} \textcolor{inputclr}{U(s)}
\end{aligned}
\end{equation}
\end{frame}

\begin{frame}[label={sec:orgdf6bf5d}]{Do in groups}
\[ \dot{p} = a_0(u_v - 5)|p_s - p|^{a_1}\; \textcolor{red!90!black}{-\, v} = f(p, u_v)\]

\begin{enumerate}
\item Given operating pressure \(p_0\). Choose operating point \(u_0\) which gives equilibrium \(f(p_0, u_0) = 0\).
\item Introduce deviation variables: \(u_v = u_0 + \textcolor{inputclr}{u}\) and \(p = p_0 + \textcolor{outputclr}{y}\).
\item Determine partial derivatives.
\item Evaluate partial derivatives at the operating point.
\item Form the linearized model
\end{enumerate}
\end{frame}
\end{document}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% UMB-CS110-2015S: Introduction to Computing
% Copyright 2015 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/UMB-CS110-2015S
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\def \topDirectory {.}
\def \texDirectory {\topDirectory/src/main/tex}

\documentclass[12pt,letterpaper,twoside]{article}
\usepackage{\texDirectory/template/style/directives}
\usepackage{\texDirectory/template/style/assignment}
\input{\texDirectory/template/config}

\begin{document}

\doc{title}{Solution to Quiz 2(b)}
\doc{date-pub}{Mar 05, 2015 at 01:00 PM}
\doc{date-due}{Mar 05, 2015 at 11:00 PM}
\doc{points}{4}

\prepare{header}

\section*{Question 1}

Write a program \texttt{PrimeCounter.java} that takes a command-line argument \texttt{N} and finds the number of primes less than or equal to \texttt{N}. Use it to \textbf{efficiently} print out the number of primes less than or equal to 10 million.

\subsection*{Solution}

\lstset{language=Java,tabsize=2}
\begin{lstlisting}
public class PrimeCounter {
	public static void main(String args[]) {
		int i; // first loop (number counter)
		int j; // second loop (prime checker)
		int[] primeList = new int[100000]; // array of primes
		int counter = 0; // counter for array of primes
		boolean definitelyNotPrime; // flag for prime checking
		// args[0] is a string. We have to convert it to an integer first.
		int maxNumber = Integer.parseInt(args[0]);
		System.out.println("Prime numbers less than " + args[0] + " are: ");
		for (i = 2; i <= maxNumber; i++) {
			// check if a number is prime
			// a number is prime if is not divisible by any smaller prime number
			// note that we can't afford to check if any smaller number divides our number
			definitelyNotPrime = false;
			for (j = 0; j < counter; j++) {
				if (i % primeList[j] == 0) {
					definitelyNotPrime = true;
				}
			}
			if (!definitelyNotPrime) {
				primeList[counter++] = i;
				System.out.printf("Prime %3d: %d\n", counter, i);
			}
		}
	}
}
\end{lstlisting}

\section*{Question 2}

Write a program \texttt{MatrixDeterminant.java} that asks for elements of a three-by-three matrix and prints its determinant.

\subsection*{Solution}

\lstset{language=Java,tabsize=2}
\begin{lstlisting}
import java.util.Scanner;
public class MatrixMultiply {
	public static void main(String[] args) {
		// Declaration of Variables
		int i; // counter
		int j; // counter
		int p; // counter
		int q; // counter
		int k;
		int row; // row of matrix
		int col; // column of matrix
		double matrix1[][] = new double[3][3];
		double matrix2[][] = new double[3][3];
		double multiplied[][] = new double[3][3];
		Scanner input = new Scanner(System.in);
		// ask for matrices
		for ( i = 0; i < 2; i++) {
			for ( j = 0; j < 9; j++) {
				row = j / 3;
				col = j % 3;
				if (i == 0) {
					System.out.print("Enter matrix1[" + row + "][" + col + "]: ");
					matrix1[row][col] = input.nextInt();
				}
				else {
					System.out.print("Enter matrix2[" + row + "][" + col + "]: ");
					matrix2[row][col] = input.nextInt();
				}
			}
			System.out.println("Thank you. Matrix initialized as follows:");
			for ( p = 0; p < 3; p++) {
				for ( q = 0; q < 3; q++) {
					System.out.printf("%.1f ",matrix1[p][q]);
				}
				System.out.println("");
			}
			System.out.println("");
		}
		// Scanner input shall be closed
		input.close();
		for ( i = 0; i < 3; i++) {
			for ( j = 0; j < 3; j++) {
				for ( k = 0; k < 3; k++) {
					multiplied[i][j] += matrix1[i][k] * matrix2[k][j];
				}
			}
		}
		// Multiply Matrices
		System.out.println("Answer is: ");
		for ( p = 0; p < 3; p++) {
			for ( q = 0; q < 3; q++) {
				System.out.printf("%.1f ",multiplied[p][q]);
			}
			System.out.println("");
		}
	}// end of main method
}// end of class MatrixMultiply
\end{lstlisting}

The following code computes multiplication of two \texttt{n}-by-\texttt{n} matrices.

\lstset{language=Java,tabsize=2}
\begin{lstlisting}
import java.util.Scanner;
public class MatrixMultiplyGeneral {
	public static void main(String[] args) {
		// Declaration of Variables
		int i; // counter
		int j; // counter
		int p; // counter
		int q; // counter
		int k;
		int row; // row of matrix
		int col; // column of matrix
		String str = "";
		double matrix1[][];
		double matrix2[][];
		double multiplied[][];
		int matrixSizes[][] = new int[2][2]; // contains number of rows and columns of matrix1 and matrix2
		Scanner input = new Scanner(System.in);
		// ask for dimensions of two matrices
		for (i = 0; i < 2; i++) {
			for (j = 0; j < 2; j++) {
				switch (j) {
					case 0: str = "row"; break;
					case 1: str = "column"; break;
				}
				System.out.print("Enter number of "+ str +"s of matrix "+ (i+1) + ": " );
				matrixSizes[i][j] = input.nextInt();
			}
		}
		// initialize matrices according to their number of rows and columns
		matrix1 = new double[matrixSizes[0][0]][matrixSizes[0][1]];
		matrix2 = new double[matrixSizes[1][0]][matrixSizes[1][1]];
		multiplied = new double[matrixSizes[0][0]][matrixSizes[1][1]];
		// ask for matrices
		for ( i = 0; i < 2; i++) {
			for ( j = 0; j < matrixSizes[i][0]*matrixSizes[i][1]; j++) {
				row = j / matrixSizes[i][0];
				col = j % matrixSizes[i][1];
				if (i == 0) {
					System.out.println("Enter matrix1[" + row + "][" + col + "]");
					matrix1[row][col] = input.nextInt();
				}
				else {
					System.out.println("Enter matrix2[" + row + "][" + col + "]");
					matrix2[row][col] = input.nextInt();
				}
			}
			System.out.println("Thank you. Matrix initialized as follows:");
			for ( p = 0; p < matrixSizes[i][0]; p++) {
				for ( q = 0; q < matrixSizes[i][1]; q++) {
					System.out.printf("%.1f ",matrix1[p][q]);
				}
				System.out.println("");
			}
		}
		// Scanner input shall be closed
		input.close();
		// check if number of columns of first matrix matches number of rows of second matrix
		if (matrix1[0].length == matrix2.length) {
			for ( i = 0; i < matrix2[0].length; i++) {
				for ( j = 0; j < matrix1.length; j++) {
					for ( k = 0; k < matrix1[0].length; k++) {
						multiplied[i][j] += matrix1[i][k] * matrix2[k][j];
					}
				}
			}
		}
		else {
			System.out.println("Matrices Dimensions Mismatch.");
		}
		// Multiply Matrices
		System.out.println("Answer is: ");
		for ( p = 0; p < matrixSizes[0][0]; p++) {
			for ( q = 0; q < matrixSizes[1][1]; q++) {
				System.out.printf("%.1f ",multiplied[p][q]);
			}
			System.out.println("");
		}
	}
}
\end{lstlisting}

\end{document}