\chapter{Real Analysis} \label{app:RealAnalysis}

This appendix is devoted to present some definitions and results from real analysis that are used in the manuscript.
%
Further details and proofs can be found in \cite{rudin1976principles}.

\begin{definition}[\textbf{Metric Space}] \label{def:metric_space}
    A metric space is an ordered pair $(M,d)$, where $M$ is a set and $d:M \times M \longrightarrow \R$ is a metric on $M$ that for any $x,y,z \in M$ satisfies:
    \begin{align*}
        1.&~d(x,y) = 0 \text{ if and only if } x=y, &&\text{(identity of indiscernibles)}\\
        2.&~d(x,y) = d(y,x), &&\text{(symmetry)}\\
        3.&~d(x,z) \le d(x,y) + d(y,z), &&\text{(triangle inequality)}.
    \end{align*}
\end{definition}

The non-negativity property (i.e., $d(x,y) \ge 0$ for any $x,y\in M$) follows from the axioms.

\begin{definition}[\textbf{Neighborhood}] \label{def:neighborhood}
    In a metric space $(M,d)$, a set $V$ is a \textit{neighbourhood} of a point $x_0\in M$ if there exists an open ball $B_r(x_0) \triangleq \{x \in M \mid d(x,x_0) < r\}$ with centre $x_0$ and radius $r>0$, such that the set $B_r(x_0) \subset V$.
\end{definition}

Intuitively, a neighborhood $V$ of a point $x_0$ is a set that encloses $x_0$ in the sense that one can move a certain amount in any direction starting from $x_0$ without leaving $V$.

\begin{theorem}[\textbf{Intermediate value theorem}] \label{th:IVT}
    Let $a < b$. Let the interval $I = [a,b]$ of $\R$ and a continuous function $f:I\longrightarrow\R$.
    %
    If $u\in\R$ is such that \[\min\{f(a),f(b)\} < u < \max\{f(a),f(b)\},\] then there exists $c\in(a,b)$ such that $f(c) = u$.
\end{theorem}

Above we state the well known property of continuous functions that roughly says a function $f$ assumes at least all values between $f(a)$ and $f(b)$ if we consider the interval $[a,b]$.

\begin{theorem}[\textbf{Banach fixed point theorem}] \label{th:banach}
    Let $(M,d)$ be a non-empty \textit{complete}\footnote{A metric space $M$ is \textit{complete} if every limit point of $M$ is in $M$.} metric space with a function $\varphi:M\longrightarrow M$ such that for some $c\in[0,1)$ we have that $d(\varphi(x),\varphi(y))\le c\,d(x,y)$ for all $x,y\in M$.
    %
    Then, $\varphi$ admits a unique fixed point in $M$.
\end{theorem}

Above we state an extremely important tool to prove existence and uniqueness of solutions to equations, integral and differential equations, recurrence equations, and so on.

It is interesting to note that the unicity comes trivially as follows.
%
Let $x$ and $y$ be two fixed points. Then, $d(\varphi(x),\varphi(y)) = d(x,y) \le c\,d(x,y)$, which is only true if $x=y$.

% % % % % % % % % % % % % % % %
% % % % % % % % % % % % % % % %
\chapter{Queueing Theory}

This appendix is devoted to present some definitions and results from queueing theory that are used in the manuscript.
%
Further details and proofs can be found in \cite{baccelli2013elements}.

Let us define a queue $\{Q(t)\}_t$ as a Markov chain (Definition~\ref{def:markov_chain}) with state space $\N$ and index space $\T$ (usually $\N$ or $\R_+$).
%
The number of packets (or entities) in a queue at time $t\in \T$ is denoted by the random variable $Q(t)$ and it must satisfy an equation of the kind
\begin{align}
    Q(t) = Q(t_0) + A(t_0,t] - E(t_0,t], \qquad \text{for every}~ t_0,t\in\T,
\end{align}
where $A$ and $E$ are point processes on $\T$ and represent the arrival and the departure of packets, respectively.
%
Note that $E(t_0,t]$ cannot be independent from $Q(t_0)$, otherwise we might have a negative amount of queued packets at a moment.

To eliminate this dependence in our context of telecommunication, we usually define a point process $T$ on $\T$ that represents the moments the transmission can begin.
%
Let $T$ be the times for which a transmitter can access the channel.
%
This transmitter will not access the channel if its queue is empty.
Then, we use the function $\ind\{Q(\cdot) > 0\}$ that is $1$ if the queue is non-empty, and $0$ otherwise.
%
% Let $T^*$ be a \textit{thinning} of $T$ without the points for which the queue is empty, i.e., $T^* = T^{(\ind\{Q(\cdot)>0\})}$.
%
We also define a \textit{measurable} success function $f_s:\T\to[0,1]$ that represents the probability for which a transmission is successful at a given time.
%
Supposing the transmission time is $\tau>0$, a simple form to define a queue process is using the \textit{thinning} of the point process $T$ through the functions $f_s$ and $\ind\{Q(\cdot) > 0\}$. Then,
\begin{align} \label{eq:def_queue}
    Q(t+\tau) = Q(t) + A(t,t+\tau] - T^{(\ind\{Q(\cdot) > 0\} f_s)}(t-\tau,t], \qquad \text{for every} ~ t\in\T.
\end{align}

A handful theorem to calculate the mean delay in queueing systems is known as Little's law \cite{little1961proof}, which is stated below.

\begin{theorem}[\textbf{Little's law}] \label{th:little}
    For a queueing system\footnote{A queueing system is represented by a collection of queues $\{Q_1(t), Q_2(t), \dots\}_{t\in\T}$ and we assume the arrival and departure processes are strictly jointly stationary.}, $L = a\,W$ holds, where $L$ is the time average of the number of packets in the system, $a$ is the average number of packets that arrived per unit of time, and $W$ is the average time each packet waited in the system.

    For a queue $\{Q(t)\}_{t\in\T}$, $\T=[0,\infty)$, these quantities are given by\footnote{For general \textit{measure space} $(\T,\cal{B}(\T),\mu_T)$, take the integrals in respect to the measure $\mu_\T$.}
    \begin{align*}
        L = \lim_{\tau\to\infty} \frac{1}{\tau} \int_0^\tau Q(t)\,\d t,
        \quad
        a = \lim_{\tau\to\infty} \frac{A[0,\tau]}{\tau},
        \quad
        W = \lim_{\tau\to\infty} \frac{1}{A[0,\tau]} \int_0^\tau Q(t)\,\d t,
    \end{align*}
    from which $L = a\,W$ immediately follows.
\end{theorem}

Usually it is easier to calculate the mean number of packets and the mean arrival rate of packets in the system. Then, using Little's law, the mean delay is easily obtained from those.

Now, we present the important concept of stability in a queueing system. Let us adopt the definition used by Szpankowski \cite{szpankowski1994stability}, presented next.

\begin{definition}[\textbf{Stability}] \label{def:stability}
	A queue $\{Q(t)\}_t$ is stable if for $x\in\N$
    \begin{equation*}
    	\lim_{t\to\infty} \P(Q(t) \le x)
    	    = F(x) \quad\text{and}\quad \lim_{x\to\infty} F(x) = 1,
    \end{equation*}
    where $F:\R_+\longrightarrow[0,1]$ is the limiting distribution function.

    A queueing system $\{Q_1(t), Q_2(t), \dots\}_t$ is stable if, for every $i$, each queue $\{Q_i(t)\}_t$ is stable \textit{almost surely}.
    %
    % Then, the \textit{stability region} is the set of mean arrival rates $\bm{a} = (a_1, a_2, \dots)$ that makes the queueing system stable.
\end{definition}

More succinctly, we can say a queueing system is stable if it admits a \textit{proper} limiting distribution.
%
An important form to verify whether a queue or a queueing system is stable is through the following theorem, known as Loyne's theorem \cite{loynes1962stability}.

\begin{theorem}[\textbf{Loyne's theorem}] \label{th:loynes}
    Let $\{Q(t)\}_{t\in\T}$ as defined in \eqref{eq:def_queue} be a queue with ergodic arrival and access point processes $A$ and $T$, respectively, and success function $f_s$.
    %
    Let \vspace{-5mm}
    \begin{align*}
        \rho\triangleq \displaystyle\lim_{t\to\infty}\frac{\E[A[0,t)]}{\E[T^{(f_s)}[0,t)]}.
    \end{align*}

    If $\rho < 1$, then the queue $\{Q(t)\}_t$ is stable.
\end{theorem}

In fact, Loyne's theorem is more general and deep than what we stated here.
%
It guarantees existence and uniqueness of the delay distribution and deals with the cases $\rho=1$ and $\rho>1$.
%
However, for our purposes, the above statement is enough.

A queue that is evoked throughout the thesis is the Geo/Geo/1 queue. The definition follows.
%
\begin{definition}[\textbf{Geo/Geo/1}] \label{def:geo/geo/1}
    The Geo/Geo/1 queue is a queue $\{Q(t)\}_{t\in\T}$, as defined in \eqref{eq:def_queue}, for which we have $\T=\N$, $A$ and $T$ are discrete Bernoulli point processes\footnote{A discrete Bernoulli point process $\Phi$ is a point process on a discrete set $\T$, such that, for every $t\in\T$, $\Phi(\{t\})$ follows iid Bernoulli distributions.} on $\T$, $f_s$ is a constant function, and $\tau = 1$.

    We let the constants $a,p,p_s\in[0,1]$ represent the arrival rate, access rate and success probability. More precisely, for every $t\in\T$, $A(\{t\})\sim\mathscr{B}(a)$, $T(\{t\})\sim\mathscr{B}(p)$, and $f_s \equiv p_s$.
\end{definition}

The delay and the distribution of the elements of this queue at stationary are given by the following theorem.

\begin{theorem} \label{th:geo/geo/1}
    Let $\{Q(t)\}_t$ be a \textrm{Geo/Geo/1} queue.
    %
    If this queue satisfies the conditions of the Loyne's theorem, i.e., $\rho = a/(p\,p_s) < 1$.
    %
    Then, the stationary distribution of the number of elements in the queue is given by
    \[  \lim_{t\to\infty} \P\big(Q(t)=n\big) =
        \begin{cases}
            \dfrac{\rho\,(1-\rho)}{\rho-a}\left(\dfrac{\rho-a}{1-a}\right)^n, & n\in\N^*\\
            1-\rho, & n = 0.
        \end{cases}
    \]

    Furthermore, from Little's law, the average time each packet waits in the system is
    \begin{align*}
        W = \lim_{t\to\infty}\frac{\E[Q(t)]}{a} = \frac{\rho}{a}\, \frac{1-a}{1-\rho} = \frac{1-a}{p\,p_s - a}.
    \end{align*}
\end{theorem}


%================================
\section{The Definition of Topological Space}
%================================

%--------------------------------
\begin{definition}
	Let $X$ be a set. A \textit{topological} on $X$ is a collection $\mathcal T \in \mathcal P(X)$ with the following properties.
	\begin{enumerate}[T1.]
		\item $\mathcal T$ is closed under arbitrary union;
		\item $\mathcal T$ is closed under finite intersection;
		\item $X \in \mathcal T$.
	\end{enumerate}

	The \textit{Topological Space} $(X, \mathcal T)$ is a set $X$ with a topology $\mathcal T$ on $X$. All $\mathcal T$-sets are said to be \textit{open} in $(X, \mathcal T)$.
\end{definition}
%--------------------------------

%--------------------------------
\begin{lemma}
	$\emptyset \in \mathcal T$.
\end{lemma}
%--------------------------------

%--------------------------------
\begin{proof}
	By T1, given $I$ as any index set, if for all $i \in I$, $U_i \in \mathcal T$, then
	$$
	U = \bigcup_{i \in I} U_i \in \mathcal T.
	$$
	If $I = \emptyset$, then $U = \emptyset$.
\end{proof}
%--------------------------------

%--------------------------------
\begin{note}
	Let $X = \{1,2,3\}$ with topology
	$$
	\mathcal T = \big\{ \{1,2\}, \{3\} \big\}.
	$$
	$\{1,2\} \in \mathcal T$ implies $\{3\} = X \setminus \{1,2\}$ is closed; $\{3\} \in \mathcal T$ implies that $\{1,2\} = X \setminus \{3\}$ is closed. $\{2\} \in \mathcal P(X)$, but $\{2\} \notin \mathcal T$, so $\{2\}$ is not open in $(X, \mathcal T)$, $\{1,3\} = X \setminus \{2\}$ is not closed. For any $U \in \mathcal T$, $\{2\} \ne X \setminus U$, so $\{2\}$ is not open.
\end{note}
%--------------------------------

%--------------------------------
\begin{definition}
	Given $(X, \rho)$ as a metric space, the topology
	$$
	\mathcal T_\rho = \left\{U \in \mathcal P(X) : U = \bigcup_{x \in U} B(x, \delta)\right\},
	$$
	then we call $\mathcal T_\rho$ the topology \textit{induced} by $\rho$, and $(X, \mathcal T_\rho)$ the \textit{underlying topological space} of metric space $(X, \rho)$.
\end{definition}
%--------------------------------

%--------------------------------
\begin{note}
	These topology is induced by metric.
	%--------------------------------
	\begin{enumerate}
		%--------------------------------
		\item In this case, $U$ is open in $(X, \rho)$ iff $U \in \mathcal T_\rho$.
		%--------------------------------

		%--------------------------------
		\item The metric $\rho_p :\mathbb R \times \mathbb R^n \to \mathbb R_{> 0}$ (surjective) induces $\mathcal T_{\rho_p} \subseteq \mathcal P(X)$. And we'll see that for all $p, q \ge 1$, $\mathcal T_{\rho_p} = \mathcal T_{\rho_q}$.
		%--------------------------------

		%--------------------------------
		\item The discrete topology $\rho_\mathrm{disc} : X \times X \to \mathbb R_{> 0}$ (non-sujective) induces $\mathcal T_{\rho_\mathrm{disc}} = \mathcal P(X)$. It is the largest topology on $X$, and $\rho[X \times X] \subseteq \{0,1\}$.
		%--------------------------------

		%--------------------------------
		\item The metric $\rho_p: C[a,b] \times C[a,b] \to \mathbb R_{>0}$ (surjective) induces $\mathcal T_{\rho_p} \subseteq \mathcal P(X)$. And we'll see that $\mathcal T_{\rho_1} \ne \mathcal {T}_{\rho_\infty}$.
		%--------------------------------

		%--------------------------------
		\item Given $X$ as a space, the Hausdorff metric $\rho_H: \mathcal P(X) \times \mathcal P(X) \to \mathbb R_{>0}$ (surjective) induces $\mathcal T_{\rho_H} \subseteq \mathcal P(\mathcal P(X))$.
		%--------------------------------

		%--------------------------------
		\item Given $A$ as a set, the hamming metric $\rho: A^n \times A^n \to \mathbb R_{> 0}$ (non-surjective) with $n \in \mathbb N$ induces $\mathcal T_\rho \subseteq \mathcal P(X)$. $\rho[A^n \times A^n] = \mathbb N_{\le n}$.
		%--------------------------------
	\end{enumerate}
	%--------------------------------

	These topology is not induced by any metric.
	%--------------------------------
	\begin{enumerate}
		%--------------------------------
		\item The indiscrete topology $\mathcal T = \{\emptyset, X\}$ on $X$ is not induced by any metric space. Suppose it was, then there would be a metric $\rho$ such that for all $x \in X$, for all $\varepsilon > 0$, $B(x, \varepsilon) \in \mathcal T$. But, clearly, for those $\varepsilon \in (0, \phi X)$, $B(x, \varepsilon) \notin \mathcal T$.
		%--------------------------------

		%--------------------------------
		\item Let $X = \{1,2,3\}$ with topology
		$$
		\mathcal T = \big\{ \{1,2\}, \{3\} \big\}.
		$$
		These is no such metic $\rho$ induces $\mathcal T$ for same reason.
		%--------------------------------
	\end{enumerate}
	%--------------------------------
\end{note}
%--------------------------------

%--------------------------------
\begin{definition}
	Let $X$ be a set and $\mathcal T, \mathcal T'$ be topologies on $X$. If $\mathcal T \subseteq \mathcal T'$, then we say that $\mathcal T'$ is \textit{finer} than $\mathcal T$, or $\mathcal T$ is \textit{coarser} than $\mathcal T'$.
\end{definition}
%--------------------------------

%--------------------------------
\begin{note} \
	\begin{enumerate}
		%--------------------------------
		\item Given $X$ as a set, for all topology $\mathcal T$ on $X$, $\mathcal T \subseteq \mathcal T_\mathrm{disc}$ and $\mathcal T \supseteq \mathcal T_\mathrm{indisc}$, Thus, $\mathcal T_\mathrm{disc}$ is the finest topology on $X$, and $\mathcal T_\text{indisc}$ is the coarsest.
		%--------------------------------

		%--------------------------------
		\item $\rho_p$ and $\rho_\text{disc}$ induced same topology on $\mathbb Z$. But on $\mathbb Q$, $\mathcal T_{\rho_p}$ is coarser than $\mathcal T_{\rho_\text{disc}}$.
		%--------------------------------
	\end{enumerate}
\end{note}
%--------------------------------

%--------------------------------
\begin{definition}
	Given $(X, \mathcal T)$ as a topological space, a set $V \subseteq X$ is said to be \textit{closed} in $(X, \mathcal T)$ iff $X \setminus V \in \mathcal T$.
\end{definition}
%--------------------------------

%--------------------------------
\begin{definition} \
	\begin{enumerate}
		\item In the discrete topology on $X$, all subsets are closed. Because for all $U \in \mathcal T_\text{disc}$, $X \setminus U \in \mathcal T_\text{disc}$.
		\item In the indiscrete topology on $X$, only $\emptyset$ and $X$ is closed.
	\end{enumerate}
\end{definition}
%--------------------------------

%--------------------------------
\begin{lemma}
	Let $X = (X, \mathcal T)$ be a topological space, and let
	$$
	\mathcal C = \left\{ V \subseteq X : V = X \setminus U , \; U \in \mathcal T\right\}.
	$$
	\begin{enumerate}[(i)]
		\item $\mathcal C$ is closed under arbitrary intersection;
		\item $\mathcal C$ is closed under finite intersection;
		\item $\emptyset, X \in \mathcal C$.
	\end{enumerate}
\end{lemma}
%--------------------------------

%--------------------------------
\begin{proof} \
	\begin{enumerate}[(i).]
		\item By De Morgan's laws,
		$$
		\begin{aligned}
			V = X \setminus \bigcup_{i \in I} U_i = \bigcap_{i \in I} (X \setminus U_i).
		\end{aligned}
		$$
		So, if $U_i \in \mathcal T$, then $V \in \mathcal C$.

		\item By De Morgan's laws,
			$$
			V = X \setminus \bigcap_{i = 1}^n U_i = \bigcup_{i = 1}^n (X \setminus U_i).
			$$

		\item
			$$
			\emptyset = X \setminus X, \; X = X \setminus \emptyset.
			$$
	\end{enumerate}
\end{proof}
%--------------------------------

%--------------------------------
\begin{definition}
	Let $(X, \mathcal T)$ be a topological space, and let $x \in X$. An \textit{open neighbourhood} of $x$ is a set $N_x \in \mathcal T$ with $x \in N_x$. A \textit{neighbourhood} of $x$ is any $N_x' \supseteq N_x$.
\end{definition}
%--------------------------------

%--------------------------------
\begin{note}
	Given $(X, \mathcal T)$ as a topological space. If $A \in \mathcal T$,
	$$
	A = \bigcup_{x \in A} B, \quad B \in x, \text{ and } B \in \mathcal T.
	$$
	If $\mathcal T = \mathcal T_\rho$ for some metric $\rho$ on $X$, then $A \in \mathcal T$ implies
	$$
	A= \bigcup_{x \in A} B(x, \varepsilon)
	$$
	for some $\varepsilon > 0$.
\end{note}
%--------------------------------

%--------------------------------
\begin{lemma}
	Let $(X, \mathcal T)$ be a topological space and $U \subseteq X$. Then $U \in \mathcal T$ iff for all $x \in U$, there is a neighbourhood $N_x' \subseteq U$.
\end{lemma}
%--------------------------------

%--------------------------------
\begin{proof}
	If $U \ni x$ and $U \in \mathcal T$, then $U$ is an open neighbourhood of $x$, naturally, it is a neighbourhood of $x$.

	For only if, clearly, if for all $x \in U$, there is a neighbourhood $N_x' \subseteq U$, then, by definition, there is $N_x \subseteq N_x'$ with $N_x \in \mathcal T$. Now we have $x \in N_x \subseteq N_x' \subseteq U$, then,
	$$
	U = \bigcup_{x \in U} N_x.
	$$
	By definition, $\mathcal T$ is closed under arbitrary union, thus $U$ is open.
\end{proof}
%--------------------------------

%


\subsection{Frobenius number}

Given a set of nautral numbers, the Frobenius number is the biggest number which can't be made as linear combination of the set.


\section{Adam, Momentum etc}
\subsubsection{Momentum Mini-Batch}
Then we are going to introduce the momentum method to accelerate the gradient method.
{\bf Without special statement, the general setup for those next algorithms are the same to Mini-Batch}

\begin{algorithm}[H]
\caption{Momentum Mini-Batch}
\label{alg:mom}
{\bf Input}: momentum coefficient $\alpha$(we often choose as: 0.5, 0.9, 0.999) \\
Compute the gradient on $B_{i_t}$:
$$
g_t = \nabla_{w} \frac{1}{m} \sum_{i \in B_{i_t}} f_i(w_{t}).
$$
Compute the momentum:
\begin{equation}
v_t = \alpha v_{t-1} - \eta_t g_t \quad (v_0 = 0).
\end{equation}
Update $w$:
\begin{equation}
w_{t+1} = w_t + v_t.
\end{equation}
\end{algorithm}

\subsubsection{Nesterov Momentum Mini-Batch Method}
We may also use the Nesterov accelerated skill and get the Nesterov momentum Mini-Batch method.
\begin{algorithm}[H]
\caption{Nesterov Momentum Mini-Batch}
\label{alg:Nesterov}
{\bf Input}: momentum coefficient $\alpha$\\
Compute the gradient on $B_{i_t}$:
$$
g_t = \nabla_{w} \frac{1}{m} \sum_{i \in B_{i_t}} f_i(w_{t} + \alpha v_{t-1}) \quad (v_0 = 0).
$$
Compute the momentum:
\begin{equation}
v_t = \alpha v_{t-1} - \eta_t g_t.
\end{equation}
Update $w$:
\begin{equation}
w_{t+1} = w_t + v_t.
\end{equation}
\end{algorithm}

\subsection{Adaptive Learning Rate Mini-batch Based Method}
The delta-bar-delta algorithm (Jacobs, 1988) is an early heuristic approach to adapting individual learning rates for model parameters during training. The approach is based on a simple idea: if the partial derivative of the loss, with respect to a given model parameter, remains the same sign, then the learning rate should increase. If the partial derivative with respect to that parameter changes sign, then the learning rate should decrease. Of course, this kind of rule can only be applied to full batch optimization. More recently, a number of incremental (or mini-batch-based) methods have been introduced that adapt the learning rates of model parameters.

\subsubsection{AdaGrad}
The AdaGrad algorithm, individually adapts the learning rates of all model parameters by scaling them inversely proportional to the square root of the sum of all of their historical squared values (Duchi et al., 2011).
\begin{algorithm}[H]
\caption{AdaGrad}
\label{alg:AdaGrad}
{\bf Input}: small constant $\delta$(perhaps $10^{-7}$, for numerical stability)\\
Compute the gradient on $B_{i_t}$:
$$
g_t = \nabla_{w} \frac{1}{m} \sum_{i \in B_{i_t}} f_i(w_{t}).
$$
Accumulate squared gradient:
\begin{equation}
r_t = r_{t-1} + g_t \otimes g_t. \quad (r_0 = 0),
\end{equation}
with $\otimes$ means multiplication element-wise.  \\
Compute Update:
\begin{equation}
\Delta w_t = -\frac{\eta_t}{\delta + \sqrt{r_t}} \otimes g_t,
\end{equation}
with division and square root applied element-wise. \\
Update $w$:
\begin{equation}
w_{t+1} = w_t + \Delta w_t.
\end{equation}
\end{algorithm}

\subsubsection{RMSProp}
The RMSProp algorithm (Hinton, 2012) modifies AdaGrad to perform better in the non-convex setting by changing the gradient accumulation into an exponentially weighted moving average.

\begin{algorithm}[H]
\caption{RMSProp}
\label{alg:RMSProp}
{\bf Input}: decay rate $\rho$, small constant $\delta$(usually $10^{-6}$)\\
Compute the gradient on $B_{i_t}$:
\begin{equation}
g_t = \nabla_{w} \frac{1}{m} \sum_{i \in B_{i_t}} f_i(w_{t}).
\end{equation}
Accumulate squared gradient:
\begin{equation}
r_t = \rho r_{t-1} + (1-\rho)g_t \otimes g_t. \quad (r_0 = 0),
\end{equation}
Compute update:
\begin{equation}
\Delta w_t = -\frac{\eta_t}{\sqrt{\delta + {r_t}}} \otimes g_t
\end{equation}
Update $w$:
\begin{equation}
w_{t+1} = w_t + \Delta w_t.
\end{equation}
\end{algorithm}

And if we combine RMSProp  with Nesterov momentum, we can get:

\begin{algorithm}[H]
\caption{RMSProp with Nesterov momentum}
\label{alg:RMSPropNesterov}
{\bf Input}: decay rate $\rho$, momentum coefficient $\alpha$\\
Compute the gradient on $B_{i_t}$:
\begin{equation}
g_t = \nabla_{w} \frac{1}{m} \sum_{i \in B_{i_t}} f_i(w_{t} + \alpha v_{t-1}), \quad (v_0 = 0).
\end{equation}
Accumulate squared gradient:
\begin{equation}
r_t = \rho r_{t-1} + (1-\rho)g_t \otimes g_t. \quad (r_0 = 0),
\end{equation}
Compute velocity update:
\begin{equation}
v_t = \alpha v_{t-1} -\frac{\eta_t}{\sqrt{{r_t}}} \otimes g_t
\end{equation}
Update $w$:
\begin{equation}
w_{t+1} = w_t + v_t.
\end{equation}
\end{algorithm}

\subsubsection{Adam}
Adam (Kingma and Ba, 2014) is yet another adaptive learning rate optimization algorithm, and its name ``Adam'' derives from the phrase \emph{Adaptive Moment Estimation}, which adapt both learning rate $\eta$ and subgradient $g$.
Use moving average of both $g$ and $g^2$ (here $g^2$ indicates the element-wise square $g \bigodot g$) to compute first order moment vector
\begin{equation}
m_t  =\beta_1 m_{t-1} + (1-\beta_1) g_t=(1-\beta_1)\sum_{i=1}^{t}\beta_1^{t-i} g_i ,
\end{equation}
and second order moment vector
\begin{equation}
v_t  =\beta_2 v_{t-1} + (1-\beta_2) g_t^2=(1-\beta_2)\sum_{i=1}^{t}\beta_2^{t-i} g^2_i.
\end{equation}
Here $\beta_1 = 0.9$, $\beta_2 = 0.99$.

Since
\begin{align}
\mathbb{E} \left[ m_t \right] & =\mathbb{E} \left[ g_t \right] \cdot (1-\beta_1^t) +\zeta\\
\mathbb{E} \left[ v_t \right] & =\mathbb{E} \left[ g^2_t \right] \cdot (1-\beta_2^t) +\zeta.
\end{align}
Here $\zeta=0$ if the true moment is stationary.
So, we use a bias-corrected version here
\begin{align}
\tilde{m}_t=\frac{m_t}{1-\beta_1^t} \\
\tilde{v}_t=\frac{v_t}{1-\beta_2^t}.
\end{align}

The iterate can be written as
\begin{equation}\label{Adam}
w_{t+1}=w_t-\frac{\eta}{\sqrt{\tilde{v}_t}+\delta} \tilde{m}_t.
\end{equation}

\begin{algorithm}[H]
\caption{Adam}
\label{alg:Adam}
{\bf Input}: small constant $\delta$ (can be $10^{-8}$), decay rate for moment estimates $\rho_1$ and $\rho_2$ (suggested defaults: 0.9 and 0.999 respectively)\\
Compute the gradient on $B_{i_t}$:
\begin{equation}
g_t = \nabla_{w} \frac{1}{m} \sum_{i \in B_{i_t}} f_i(w_{t}).
\end{equation}
Update biased first moment estimate:
\begin{equation}
m_t = \rho_1 m_{t-1} + (1-\rho_1)g_t, \quad (m_0 = 0).
\end{equation}
Update biased second moment estimate:
\begin{equation}
v_t = \rho_2 v_{t-1} + (1-\rho_2)g_t \otimes g_t, \quad (v_0 = 0).
\end{equation}
Correct bias in first moment:
\begin{equation}
\tilde m_t = \frac{m_t}{1 - \rho_1^{t}}.
\end{equation}
Correct bias in second moment:
\begin{equation}
\tilde v_t = \frac{v_t}{1 - \rho_2^{t}}.
\end{equation}
Compute update:
\begin{equation}
\Delta w_t =  -\frac{\eta_t}{\sqrt{\tilde v_t} + \delta} \otimes \tilde m_t
\end{equation}
Update $w$:
\begin{equation}
w_{t+1} = w_t + \Delta w_t.
\end{equation}
\end{algorithm}

\subsection{For linear problems}
Now we apply Adam for the following SPD linear system
\begin{equation}
	A u-b=0
\end{equation}
where $A \in \mathbb{R}^{n \times n}$ is \emph{symmetric positive definite} matrix and $b \in \mathbb{R}^n$.
It is equal to solve the following minimization problem
\begin{equation}
	\min_u\ \left\{f(u):= \frac{1}{2} u^T Au - b^T u\right\}.
\end{equation}

Assume that we didn't take stochastic technique into consideration, then
\begin{equation}
g_k=A u_k-b.
\end{equation}
Compute first order moment vector
\begin{align}
m_k & = \beta_1 m_{k-1} +(1-\beta_1) g_k\\
&= \beta_1 m_{k-1} + (1-\beta_1) (A u_k - b)
\end{align}
The update can be written as
\begin{equation}
u_{k+1}=u_k-\eta P^{-1}_k R_k
\end{equation}
where
\begin{align}
R_k & =\frac{1-\beta_1}{1-\beta_1^k} \sum_{i=1}^{k} \beta_1^{k-i} (A u_i -b)\\
& = \frac{(1-\beta_1) \beta_1^{(k-1)}}{1-\beta_1^k} (Au_1 -b) + \frac{(1-\beta_1) \beta_1^{(k-2)}}{1-\beta_1^k} (Au_2 -b) +\cdots \\
& +\frac{(1-\beta_1) \beta_1^{0}}{1-\beta_1^k} (Au_k -b) \\
& =c_1 (Au_1-b) + c_2(Au_2-b) + \cdots + c_k (Au_k-b).
\end{align}
Here
\begin{equation}
c_i = \frac{(1-\beta_1) \beta_1^{(k-i)}}{1-\beta_1^k}
\end{equation}
is increase geometrical sequence.
And we can easily draw the conclusion that
\begin{equation}
\sum_{i=1}^{k} c_i =1.
\end{equation}
So $R_k$ is weighted average of $\left\{u_1, u_2, \dots, u_k\right\}$.

$P_k$ is a diagonal matrix
\begin{align}
P_k & =\text{diag} \left\{
\sqrt{ \frac{1-\beta_2}{1-\beta_2^k} \sum_{i=1}^{k} \beta_2^{k-i} (A u_i -b)^2 }+\epsilon
\right\}\\
& = \text{diag} \left\{ p_1, p_2, \cdots, p_n \right\},
\end{align}
where $(Au_i -b)^2$ is element-wise product.
$p_i^2$ is weighted average of $(A u_i-b)^2$.

To simplify our analysis, we will use $A u_k - b$ instead of $R_k$, as well as
\begin{equation}
	P_k=\text{diag} \left\{ |A u_k-b| \right\}
\end{equation}

Assume $u^*$ is the solution of $Au-b=0$.
So
\begin{equation}
u^* = u^* + \eta P^{-1} (A u^* -b)
\end{equation}
And
\begin{equation}
u^* -u_k = (I- \eta P_k^{-1} A) (u^* - u_k)
\end{equation}
We need estimate the upper bound of $\|I-\eta P^{-1}_k A\|$.

In other way, we will find the relation of ADAM and SDA.

\subsection{Choosing the Right Optimization Algorithm}
Schaul et al. (2014) presented a valuable comparison of a large number of optimization algorithms across a wide range of learning tasks. While the results suggest that the family of algorithms with adaptive learning rates (represented by RMSProp and Adam) performed fairly robustly, no single best algorithm has emerged.

Currently, the most popular optimization algorithms actively in use include SGD, SGD with momentum, RMSProp, RMSProp with momentum, AdaDelta and Adam. The choice of which algorithm to use, at this point, seems to depend largely on the user's familiarity with the algorithm (for ease of hyperparameter tuning).

%\section{Incremental Methods}
%%\subsection{Problem setting}
%Now we consider those kind of problem:
%\begin{problem}Find ${x}^{*} \in \mathcal{X}$  s.t:
%	\begin{equation}\label{equ:sum-opt}
%	{x}^* = \mathop{\arg\min}_{{x} \in \mathcal{X} \subseteq \mathbb{R}^{n}} \frac{1}{N}\sum_{i=1}^N f_i(x).
%	\end{equation}
%	Here $\mathcal{X}$ is a convex compact subset. (This condition is used
%	to prove the gradient is bounded, i.e. $\|\nabla f_i(x)\| \le M$.)
%\end{problem}


\columnbreak

\section{Maximum Entropy Inference}

% ===
\iffalse
    Max. entropy via $\partial\mathcal L / \partial p(x) {=} 0$.
    \quad $\mathcal L$ given by
    \\
    $\sup_{p(X)} \left\{ -\sum_x p(x) \log p(x) \right\}$,\;
    s.t. $\sum_x p(x) = 1$,\;
    $p(x) \geq 0$,\;
    $\sum_x p(x) r_j(x) = \mu_j, 1\leq j\leq m$.
    \\
    $\implies$
    \emph{Gibbs dist.:}
    $p(x) {=} \frac{ \exp(-\sum_j \lambda_j r_j(x)) }{ \sum_{x'} \exp(-\sum_j \lambda_j r_j(x')) }$
\fi

Sample $c \sim p(\cdot\mid X)$\enspace s.t.\enspace $H[p(\cdot\mid x)]$ is maximal, $\E[C\mid X]{R(C,X)} = \mu$ and $\sum_c p(c\mid X) = 1$.
\\
$\implies$
\emph{Gibbs dist.:}\enspace
$\highlight*{p(c\mid X) = \frac{1}{Z(X)} \exp(-\beta R(c,X))}$
%$p(c\vert X) = \frac{ \exp(-\beta R(c,X)) }{ \sum_{\tilde c\in \mathcal C} \exp(-\beta R(\tilde c,X)) }$%$\substack{\\\\ \eqqcolon Z(X)}$
\\
\textbf{Free energy:}\enspace $F(X) \coloneqq - \frac1\beta \log Z(X)$ % = -\frac1\beta \log \sum_c \eu^{-\beta R(c,X)}
\\
$\iff \highlight*{p(c\mid X)} = \exp\paren*{-\beta [R(c,X) - F(X)]}$
\\
$\implies$ \textbf{entropy:}\enspace $H[c\mid X] = \beta \underbracket[.7pt][.7pt]{\E[C\mid X]{R(C,X)}}_{=\mu} - \beta F(X)$

\emph{ME:}\enspace
\highlight*{$\max H[c\mid X] \iff \max Z(X) \iff \min F(X)$}
\begin{itemize}
    \item Exp. generalisation costs:\enspace
        $\mathbb E_{X''} \cancelto{\mathbb E_C}{\mathbb E_{X'} \mathbb E}_{C\mid X'} \brack{ R(c,X'') }$
    \item Min. out-of-sample descr. length per deg. of freedom\\
        $\min\limits_{p(\cdot\mid\cdot)} \mathbb E_{X',X''} \E*[C\mid X']{-\log \frac{p(c\mid X'')}{p(c)}}$\hfill $\color{gray} p(c) = \E[X]{p(c\mid X)}$\\
        \enspace $\overset{\text{Jensen}}{\geq} \min_{p(\cdot\mid\cdot)} \E*[X',X'']{ -\log \E[C\mid X']{p(c\mid X'')} } - H[c]$\\
        \enspace $\overset{\phantom{\text{Jensen}}}{=} \max_{p(\cdot\mid\cdot)} \E[X',X'']{\eu^{H[c]} \cdot \kappa(X',X'')}$
\end{itemize}

%\emph{Goal:}
%$p(c\mid X) \propto \exp(-\frac1T \mathcal R(c,X))$, \quad $T>0$

%\todo{Below to be reviewed \\\hfill $\to$ \texttt{slt21\_lecture03\_notes.pdf}}

\emph{PA:}\enspace
\highlight*{$T^\ast = \arg\max_T \kappa(X', X'')$}
\begin{itemize}
    \item PA-kernel:\enskip $\kappa (X', X'') \coloneqq \sum_c p(c\mid X') p(c\mid X'')$
    \item combined:\enskip $p(c\mid X', X'') \propto p(c\mid X') p(c\mid X'')$
\end{itemize}

% ===


The main idea of this project is to extract some characteristic audio data from
the signal without having to resort to memory and computationally expensive
fourier transforms. The 2KiB SRAM of the Arduino is the biggest botleneck here
for doing any processing. In order to do this processing, we came up with
multiple optimizations.

\subsection{Optimizations}

\subsubsection{Cross Correlations}
Instead of doing a FFT on the data, we will extract a few characteristic
frequencies components by correlating the signal with a set of frequencies. The
expression for crosscorrelation of 2 discrete signals \(x,y\) is given by

\begin{equation}
    R_{xy}[k] = \sum_{i} x[i]y[i-k]
\end{equation}

Along with some normalization. The harmonics form a linearly independent set and
return zero-correlation when the product is integrated over several time periods,
i.e.

\begin{equation}
    \frac{1}{\pi} \int_0^{2\pi}sin(n_1x)sin(n_2x) = \delta_{nn'}
\end{equation}

Where \(\delta\) is the usual Kronecker delta. We can normalize the correlation
with the autocorrelation at 0 to get a number between -1 and 1 that
characterized the coefficients.

\begin{equation}
    c_{xy} = \frac{R_{xy}[0]}{\sqrt{R_{xx}[0]R_{yy}[0]} }
\end{equation}

However, the above expression does not account for the phase shift \(\phi\)
between the harmonics which reduces the correlation by a factor of
\(cos(\phi)\)Therefore we modify check the signal with phase shifted test
harmonics by modifying the expression to:

\begin{equation}
    c_{xy} = \frac{\mathnormal{max}\{R_{xy}[k]\}}{\sqrt{R_{xx}[0]R_{yy}[0]} }
\end{equation}

This ensures that the loss due to potential phase shift is avoided by getting an
estimate of the phase. Further, these correlations avoid the complex
multiplications required in FFT calculations while giving more characteristic
data.

\subsubsection{Memory Management}
We store the signal using an array of 4 bit numbers, i.e. numbers from -7 to 7.
This has multiple benefits, such as being able to outright store more numbers,
and more subtly, perform calculations in a smaller time period as the Arduino
processor is based around an 8-bit bus. Instead of requiring multiple clock
cycles to process 16 / 32 bit datatypes like \texttt{float} and \texttt{int}, we
can process the signals much faster.

\subsubsection{Frequency Space Pruning}
Beginning with a Dirac comb in frequency space, we prune a lattice tree of the
frequency power set via depth first binary search. This bypasses the
computationally expensive cross correlation calculations for a lot of (wrong)
frequencies.


\subsection{\acrlong{mdp}}%
\label{sub:mdp}

Optimization problem solved using \gls{mcts} algorithm are modelised as a \gls{mdp}.
it modelises the problem as a set of states and actions that can be applies or played on a state.
A state contains all the information necessary to understand the situation meaning that the past state are irrelevant.
In this section, we will give a vulgarize explanation of what a game is and a more definition of a \gls{mdp}.

\subsubsection{Sequential problem modelization}%
\label{ssub:sequential_problem_modelization}

Let's consider a single player basic game called the \gls{lmp}.
Its rules are simple:

\begin{itemize}
    \item Each turn the player can choose \textit{left}  or \textit{right}.
    \item A player choosing \textit{left} gain one point.
    \item A player choosing \textit{right} gain no point.
    \item The game ends after \(N\) turn.
\end{itemize}

The goal of the game is to have as much point as possible at the end.
It is a very simple problem but is perfect to explaine the key concept.
A state of the game can be represented by two elements: the score of the player and the number of turn that passed.
So if we define the state \(s=(6, 10)\) it means that the player has 6 points and has played 10 turns.
Past states are irrelevant to describe \(s\), it contains all information necessary to describe the situation.
Some states are called terminal because they correspond to state where the game has ended.
Here, all the states where the player has played \(N\) turns are final or terminal.

The actions the player can choose are \textit{left} and \textit{right}.
If the player applies \textit{left}  to \(s\) then, we will obtain a new state \(s'=(7, 11)\).
The transition from \(s\) to \(s'\) is always the same if we apply \textit{left} to \(s\).

The goal being to obtain the most point as possible, a player will learn to always pick \textit{left}.
This is called a policy or, more conviniently, a strategy: it dictates the action according to a situation.
In other, if I give \(s\) to a player following the policy to always go right, he will choose \textit{right} and will go to state \(s''=(6, 11)\).

This simply explains the key concepts of a \gls{mdp}.
You have states on which you apply actions.
Applying certain actions on certain states will give you a new state.
And states have what is called a reward, which is the number of point the player scored.

\subsubsection{Definition}%
\label{ssub:definition}

Now for a more formal definition, a \gls{mdp} is composed of:

\begin{itemize}
    \item \(S\) a set of state.
    \item \(A\) a set of actions
        \begin{itemize}
            \item \(A_{s} \subseteq A\) are the legal actions or actions that can be applied to \(s\)
            \item If \(A_{s} = \emptyset\) then it means that \(s\) is a final or terminal state
        \end{itemize}
    \item \(P_{a}(s, s') = P(s_{t+1} = s' | s_{t} = s, a_{t} = a)\) is the probability that when applying action \(a\) to state \(s\), we obtain state \(s'\)
    \item \(R_{a}(s, s')\) is the reward obtained when transitioning from \(s\) to \(s'\) by applying action \(a\)
\end{itemize}

When optimizing a \gls{mdp}, we search for the best policy, that means the policy that will give the best expected reward possible.
A policy \(\pi(s)\) is a distribution of probability over all the elements of \(A_s\).
A policy that maximizes the reward is called an optimal policy and is often noted \(\pi^{*}\).

A \gls{mdp} represents a stochastic game if the probabilities given by \(P_{a}(s, s')\) are not equal to 1.
That is to say, if the transitions between the states are uncertain, therefore random, the modelized problem is stochastic.


\documentclass[twocolumns]{IEEEtran}
\usepackage{algorithm}
\usepackage{algpseudocode}
\usepackage{graphicx}
\usepackage{caption}
\usepackage{url}
\captionsetup{justification=centering}

\algnewcommand\algorithmicforeach{\textbf{for each}}
\algdef{S}[FOR]{ForEach}[1]{\algorithmicforeach\ #1\ \algorithmicdo}

\author{Erdal Sidal Dogan, Alp Gokcek \\ MEF University \\ \today}
\title{Dijkstra's Shortest Path Algorithm Implementation \& Visualization}

\begin{document}
	\maketitle
	\begin{abstract}
		Dijkstra's Shortest Path Algorithm \cite{dijkstra} is an algorithm which finds the shortest path between two nodes in a graph. It is widely used and adopted for many different applications, such as computer networks, road maps, social platforms etc. In this project we implemented algorithm in Python Language and visualized the shortest path.
	\end{abstract}
	\begin{IEEEkeywords}
		Graph, Node, Vertex, Edge
	\end{IEEEkeywords}
	\section{Introduction}
	\textit{Dijkstra's Algorithm} calculates the shortest path from one desired node of the graph to every other nodes. There are varios implementations of the algorithm, main difference between them is how they store the verticies of the graph, denoted with $Q$. They can be stored in regular arrays, linked lists, adjacency lists or tree structures.\cite{524471} Running time of the algorithm can be represented with number of edges $|E|$ and number of verticies $|V|$ using big-o notation. Running time mainly depends on these structures, storing $Q$ in arrays it is $\mathcal{O}(|V|^2)$. In this project we utilized the \textit{priority queue} structure while implementing. It is more efficient way to implement comparing to arrays and adjacency lists etc. The \textit{priority queue} implementation requires $\mathcal{O}((|E| + |V|) \log{|V|})$ time in the worst case.
	\begin{algorithm}
		\caption{Using a priority queue \cite{wiki} \hfill Cost}
		\begin{algorithmic}[1]
			\Function {Dijkstra} {\textit{Graph, source}}
				\ForEach {vertex \textbf{v} in \textit{Graph}}	\hfill v + 1
					\State $dist[v] \leftarrow$ INFINITY	\hfill v
					\State $prev[v] \leftarrow$ UNDEFINED	\hfill v
					\State add $v$ to $Q$	\hfill v
				\EndFor
				\State $dist[source] \leftarrow$ 0	\hfill 1

				\While {$Q$ is not empty}	\hfill v + 1
					\State $u \leftarrow$ vertex in $Q$ with min $dist[u]$	\hfill v
					\State remove $u$ from $Q$	\hfill v
					\ForEach {neighbor $v$ of $u$}	\hfill $\sum_{i=0}^{v}(v-i+2)$
						\State $alt \leftarrow dist[u] + length(u, v)$ $\sum_{i=0}^{v}(v-i+1)$
						\If{alt $<$ dist[v]}	\hfill $\sum_{i=0}^{v}(v-i+2)$
							\State $dist[v] \leftarrow alt$	\hfill $\sum_{i=0}^{v}(v-i+2)$
							\State $prev[v] \leftarrow u$	\hfill $\sum_{i=0}^{v}(v-i+2)$
						\EndIf
					\EndFor
				\EndWhile \\
				\Return dist[], prev[]
			\EndFunction
		\end{algorithmic}
	\end{algorithm}

	\section{The Program}

	The program consists a simple and intuitive Graphical User Interface. On the top there are text fields where the user can type de number of nodes, start node and the destination node respectively. Below, there is the path from source to destination and time it took to travel between these nodes. Rest of the application window displays the nodes and emphasizes the shortest path between two given nodes in red color edges.

	\begin{figure}[h]
		\centering
		\includegraphics[scale=.2]{main_window.png}
	\end{figure}
	\captionof{figure}{Main Window of the program for values selected as; N=10, S=1, D=6}

	\section{Time Measurements}

	After the implementation, we measured the time it takes to calculate shortest path from 1 to $N$ in a graph with $N$ number of nodes. We observed a correlation between theory and the practice.
	Theoritical results are calculated by the $\mathcal{O}((|E| + |V|) \log{|V|})$ which stated in \textit{Introduction} section previously. Time it takes to run the program depends on the computer it ran on. During our demonstration we used the same machine with same configuration in order to get most accurate results. For different machines the expectation is to get ratios that are closer to ratio of our results. Results can be seen in detail from the figure below; \\

	\begin{figure}
		\centering
		\includegraphics[scale=.4]{matlab/time_2.eps}
		\caption{$|V|$ vs. time (in seconds)}
	\end{figure}

	Note that theoretical time (showed in green line) doesn't correspond to any value of time, it is put there to emphasize the overall similarities such as curvature and trajectory etc. between actual results and calculated results.

	\section{Conclusion \& Discussion}
	In this project we implemented \textit{Dijkstra's Algorithm} using Python. We developed an GUI in order to visualize the graph and the shortest path. After the implementation, we compared our results with the theoritical results. This project helped us to develop a deeper insight about the graphs and the shortest path algorithms.

	\bibliographystyle{IEEEtran}
	\bibliography{references}

\end{document}


\subsection{Floating Point and Testing}
Nearly all computational research is done using the floating-point arithmetic supplied by the vendor. These days this is normally assumed to conform to the IEEE (binary) floating point system \cite{IEEE2008}, which specifies the results of a sequence of floating-point operations. This actually \emph{does} simplify the developers' life (compared to the days of negotiating hexadecimal-based IBM formats etc.), but does \emph{not} mean that there are no problems with the floating point.

\begin{enumerate}
\item Floating-point may not produce the results the na\"\i{}ve expect:
\begin{equation}\label{eq:plus1}
\left(1+10^{20}\right)-10^{20}=10^{20}-10^{20}=0,
\end{equation}
not the $1$ one might expect. Of course,
\begin{equation}\label{eq:plus2}
1+\left(10^{20}-10^{20}\right)=1+0=1.
\end{equation}
\item \cite{IEEE2008} does specify the result of a sequence of floating-point operations, but the user may not fully specify the sequence! In particular, in most programming languages,
\begin{equation}\label{eq:plus3}
1+10^{20}-10^{20}
\end{equation}
is ambiguous as to whether it is (\ref{eq:plus1}) or  (\ref{eq:plus2}), and therefore the compiler is free to produce 1 or 0. In practice, of course, the code will not be (\ref{eq:plus3}) but \verb!a+b+c!, and indeed \verb!a! etc. will probably be array elements, or expressions themselves. A slight change in \verb!a! etc., or indeed in the surrounding program, can change which order the compiler chooses to do the additions in, and, as we have seen, change the result.
\par
This ambiguity is multiplied if our language allows us to write vector or array operations, or if we use parallel operations like MPIs' \verb+Reduce+\footnote{See section \ref{test:parallel}.}.
\end{enumerate}
This means that it is futile to expect a program to be as deterministic as we would na\"\i{}vely expect. Even a recompilation of an untouched source program can arrange operations differently, and re-running an untouched parallel program can perform additions in a different order. Hence exact reproducibility is impossible, and there isn't necessarily a ``right answer''.
\begin{quote}
So how do I tell the difference between such legitimate variations and a bug?
\end{quote}
This is the key question, and the worrying answer is that only an expert can truly know. However, we can give some suggestions.
\begin{enumerate}
\item Consider \emph{relative errors} rather than absolute ones, i.e., rather than asking whether $|a-b|$ is small, ask whether $\frac{|a-b|}{\max(|a|,|b|)}$ is small.
\end{enumerate}

See \cite{Barba2016b}.


\input{SingleAssignmentSetup.tex}
\input{../WeekTitles.tex}
\begin{document}

\begin{center}
\subsection*{MNTC P01 - Week \#7 - \WeekTitleSeven}
\end{center}

\begin{enumerate}

\subsection*{Verifying Solutions}
% ******************************
\item
\begin{Question} %9.1 - 1
  Show that $\ds y = \frac{2}{3}\text{e}^{x} + \text{e}^{-2x}$ is a solution of the differential equation $y' + 2y = 2\text{e}^{x}$. \\
\end{Question}

% 1 - Solution
\begin{Solution}
\begin{equation*}
y = \frac{2}{3}\text{e}^{x} + \text{e}^{-2x} \Rightarrow
y' = \frac{2}{3}\text{e}^{x} - 2\text{e}^{-2x}
\end{equation*}
To show that $y$ is a solution of the differential equation, we will substitute the expressions for $y$ and $y'$ in the left-hand side of the equation and show that the left-hand side of the equation and show that the left-hand side is equal to the right-hand side.

\begin{align*}
	\text{LHS} &= y' + 2y = \frac{2}{3}\text{e}^{x} - 2\text{e}^{-2x} + 2(\frac{2}{3}\text{e}^{x} + \text{e}^{-2x}) \\
	&= \frac{2}{3}\text{e}^{x} - 2\text{e}^{-2x} +
	\frac{4}{3}\text{e}^{x} + 2\text{e}^{-2x} =
	\frac{6}{3}\text{e}^{x} = 2\text{e}^{x} \\
	&= \text{RHS}
\end{align*}
\end{Solution}
%%%%%%%%%%%%%%%%%%

%9.1 - 3
\item
\begin{Question}
\begin{enumerate}[(a)]
\item For what values of $r$ does the function $y = \text{e}^{rx}$ satisfy the differential equation $2y'' + y' - y = 0$?

\item If $r_1$ and $r_2$ are the values of $r$ that you found in part (a), show that every member of the family of functions $y = a\text{e}^{r_{1}x} + b\text{e}^{r_{2}x}$ is also a solution.
\end{enumerate}
\end{Question}

\begin{Solution}
\begin{enumerate}[(a)]
\item \begin{equation*}
		y = \text{e}^{rx} \Rightarrow
		y' = r\text{e}^{rx} \Rightarrow
		y'' = r^2\text{e}^{rx}
	\end{equation*}
	Substituting these expressions into the differential equation
	$2y'' + y' - y = 0$, we get
	\begin{align*}
		&2r^2\text{e}^{rx} + r\text{e}^{rx} - \text{e}^{rx} = 0 \\
		\Rightarrow \quad &(2r^2 + r - 1)\text{e}^{rx} = 0 \\
		\Rightarrow \quad &(2r - 1)(r + 1) = 0
	\end{align*}
	(since $\text{e}^{rx}$ is never zero) $r = \frac{1}{2}$ or -1.

\item Let $r_1 = \frac{1}{2}$ and $r_2 = -1$, so we need to show that every member of the family of functions $y = a\text{e}^{x/2} + b\text{e}^{-x}$ is a solution of the differential equation $2y'' + y' - y = 0$.\\
\begin{align*}
	&y = a\text{e}^{x/2} + b\text{e}^{-x} \\
	\Rightarrow \quad &y' = \frac{1}{2}a\text{e}^{x/2} - b\text{e}^{-x}\\
	\Rightarrow \quad &y'' = \frac{1}{4}a\text{e}^{x/2} + b\text{e}^{-x}
\end{align*}
\begin{align*}
	\text{LHS} &= 2y'' + y' - y \\
	&= 2(\frac{1}{4}a\text{e}^{x/2} + b\text{e}^{-x}) + (\frac{1}{2}a\text{e}^{x/2} - b\text{e}^{-x}) \\ & \hspace{4mm}- (a\text{e}^{x/2} + b\text{e}^{-x}) \\
	&= \frac{1}{2}a\text{e}^{x/2} + 2b\text{e}^{-x} + \frac{1}{2}a\text{e}^{x/2} - b\text{e}^{-x}  \\
	& \hspace{4mm}- a\text{e}^{x/2} - b\text{e}^{-x} \\
	&= (\frac{1}{2}a + \frac{1}{2}a - a)\text{e}^{x/2} + (2b - b -b)\text{e}^{-x} \\
	&= 0 \\ &= \text{RHS}
\end{align*}
\end{enumerate}
\end{Solution}
%%%%%%%%%%%%%%%%%%

%9.1 - 4
\item
\begin{Question}
\begin{enumerate}[(a)]
\item For what values of $k$ does the function $y = \cos(kt)$ satisfy
  the differential equation $4y'' = -25y$?
\item For those values of $k$, verify that every member of the vamily of functions $y = A\sin kt + B\cos kt$ is also a solution.
\end{enumerate}
\end{Question}

\begin{Solution}
\begin{enumerate}[(a)]
\item \begin{equation*}
		y = \cos kt \ \Rightarrow \ y' = -k \sin kt \ \Rightarrow \ y'' = -k^2 \cos kt
	\end{equation*}
	Substituting expressions into the differential equation $4y'' = -25y$, we get
	\begin{align*}
		&4(-k^2\cos kt) = -25(\cos kt) \\
		\Rightarrow \quad &(25 - 4k^2) \cos kt = 0 (\text{for all } t)\\
		\Rightarrow \quad &25 - 4k^2 = 0 \\
		\Rightarrow \quad &k^2 = \frac{25}{4} \ \Rightarrow \ k = \pm \frac{5}{2}
	\end{align*}

\item \begin{align*}
		&y = A\sin kt + B \cos kt \\
		\Rightarrow \quad &y' = Ak\cos kt - Bk \sin kt \\
		\Rightarrow \quad &y'' = -Ak^2 \sin kt - Bk^2 \cos kt
	\end{align*}
	The given differential equation $4y'' = -25y$ is equivalent to $4y'' + 25y = 0$.  Thus,
\begin{align*}
	\text{LHS} &= 4y'' + 25y \\
	&= 4(-Ak^2 \sin kt - Bk^2 \cos kt) \\
	& \quad + 25(A\sin kt + B\cos kt) \\
	&= -4Ak^2 \sin kt - 4Bk^2 \cos kt \\
	& \quad+ 25A\sin kt + 25B \cos kt \\
	&= (25 - 4k^2)A\sin kt + (25 - 4k^2)B\cos kt \\
	&= 0 \hspace{4mm} \text{since } k^2 = \frac{25}{4}
\end{align*}
\end{enumerate}
\end{Solution}
%%%%%%%%%%%%%%%%%%

%9.1 - 7
\item
\begin{Question}
  Consider the differential equation $\ds \frac{dy}{dx} = -y^2$.
\begin{enumerate}[(a)]
\item If you were asked whether the solutions to this equation would
  {\em increase} or {\em decrease} as $x$ increased, what could you
  say based on only the equation itself?
\item Verify that all members of the family $y = 1/(x + C)$ are
  solutions of the equation in part (a).
\item Can you think of a (very simple) solution of the differential
  equation $y' = -y^2$ that is {\em not} a member of the family in
  part (b)?
\item Find the solution to the initial-value problem
\begin{equation*}
	y' = -y^2 \hspace{6mm} y(0) = 0.5
\end{equation*}
\end{enumerate}
\end{Question}
\begin{Solution}
\begin{enumerate}[(a)]
\item Since the derivative of $y' = -y^2$ is always negative (or 0 if
  $y = 0$), the function $y$ must be {\bf decreasing} (or maybe
  horizontal) on any interval on which it is defined.
\item We sub in the proposed solution into the original equation.  To do this, we will need the derivative of $y$:  $\ds y = \frac{1}{x + C} \Rightarrow y' = -\frac{1}{(x + C)^2}$.\\
  $\text{LHS} = y' = -\frac{1}{(x + C)^2} = -\left(\frac{1}{x + C}
  \right)^2 = -y^2 = \text{RHS}$ Therefore, any function of the form
  $y(x) = \frac{1}{x+C}$ {\bf is} a solution to $y' = -y^2$.
\item $y=0$ is a simple solution to $y'=-y^2$ that is not a member of
  the family in part (b).  We can confirm this by subbing $y=0$ into
  the DE and checking the LHS equals the RHS.  If $y=0$, then $y'= 0$
  as well, so $\text{LHS} = y' = 0 = -y^2 = RHS$.
\item We already know that the solutions will be of the form
  $y(x) = \frac{1}{x+C}$; we just need to sub in the initial value to
  solve for $C$.

  If $y(x) = \frac{1}{x + C}$, then
  $y(0) = \frac{1}{0 + C} = \frac{1}{C}$.  Since $y(0) = 0.5$,
  $\frac{1}{C} \frac{1}{2} \Rightarrow C = 2$, so $y =\frac{1}{x + 2}$
\end{enumerate}
\end{Solution}
%%%%%%%%%%%%%%%%%%

\hrulefill

\subsection*{Numerical ODE Solutions With MATLAB}

%*******************************
\item
\begin{Question}
  Create a plot for the solution to the differential equation
  $y' - \frac{y^2}{x^3} = 0$ if y(2) = 1.  Include a large enough
  \verb#xspan# to see the long-term behaviour.
\end{Question}

\begin{Solution}
  For this first example of use MATLAB to build a numerical solution
  to a DE, we will show the full listing of a script that generates a
  solution to the given differential equation.  In later solutions, we
  will only include the key lines for the MATLAB script.

  Notes:
  \begin{itemize}
  \item We set \verb#xspan# to start at 2 in the line
    \verb#xspan = [2, 30]#.  This is used because the solution MATLAB
    is generating will start at the coordinates $x_0$ = first element
    of \verb#xspan#, and $y_0$ = \verb#y0# in the code, and our
    initial condition is $x = 2$, $y=1$.
  \item We find the second value in the time span with some trial and
    error.  Any value larger than 15 or 20 would be sufficient to show
    the long-term trend in the solution.
  \end{itemize}
\lstinputlisting[showstringspaces=false]{MATLAB/W07DE01.m}

Link to the MATLAB code: \\
\href{http://www.mast.queensu.ca/~apsc171/MNTCP01/PracticeProblems/MATLAB/W07DE01.m}{W07DE01.m}

Here is the graph of the solution.

\includegraphics[width = 0.5\linewidth]{graphics/Week07_DESolutions/W07DE01}

\end{Solution}

%*******************************
\item
\begin{Question}
Create a plot for the solution to the differential equation
  $(2y - 4)y' - 3x^2 = 4x - 4$, if y(1) = 3.

\end{Question}

\begin{Solution}
  To generate a first-order DE solution in MATLAB, the differential
  equation must be written first in the form $y' = \ldots$.
  \begin{align*}
    (2y - 4)y' - 3x^2 & = 4x - 4 \\
    (2y - 4)y' & =  3x^2 +4x - 4 \\
    y' & =  \frac{(3x^2 +4x - 4)}{(2y - 4)} \\
  \end{align*}

Link to the MATLAB code: \\
\href{http://www.mast.queensu.ca/~apsc171/MNTCP01/PracticeProblems/MATLAB/W07DE02.m}{W07DE02.m}

Here is the graph of the solution.

\includegraphics[width = 0.5\linewidth]{graphics/Week07_DESolutions/W07DE02}
\end{Solution}

%*******************************
\item
\begin{Question}
Create a plot for the solution to the differential equation
  $y' = e^{-y}(2t - 4)$ if $y(0) = 5$

\end{Question}

\begin{Solution}
  This DE is already in the form $y' = \ldots$, so we can input it
  into MATLAB as-is.  Note that the independent variable in this
  example is $t$, so we will use that in MATLAB instead of the
  variable $x$.

Link to the MATLAB code: \\
\href{http://www.mast.queensu.ca/~apsc171/MNTCP01/PracticeProblems/MATLAB/W07DE03.m}{W07DE03.m}

Here is the graph of the solution.

\includegraphics[width = 0.5\linewidth]{graphics/Week07_DESolutions/W07DE03}

\end{Solution}

%*******************************
\item
\begin{Question}
Create a plot for the solution to the differential equation
  $ty' - 2y = t^5 \sin (2t) - t^3 + 4t^4$, if
  $y(\pi) = \frac{3}{2}\pi^4$

\end{Question}

\begin{Solution}
  To generate a first-order DE solution in MATLAB, the differential
  equation must be written first in the form $y' = \ldots$.
  \begin{align*}
    ty' - 2y & = t^5 \sin (2t) - t^3 + 4t^4 \\
    ty' & = 2y + t^5 \sin(2t) - t^3 + 4t^4 \\
    y' & = \frac{1}{t} (2y + t^5 \sin(2t) - t^3 + 4t^4) \\
  \end{align*}

Link to the MATLAB code: \\
\href{http://www.mast.queensu.ca/~apsc171/MNTCP01/PracticeProblems/MATLAB/W07DE04.m}{W07DE04.m}

Here is the graph of the solution.

\includegraphics[width = 0.5\linewidth]{graphics/Week07_DESolutions/W07DE04}

Note that in this example, because of the $\sin(2t)$ introducing an
oscillation in the system, the solution won't look at simple as some
of the other examples.
\end{Solution}

%*******************************
\item
\begin{Question}
  Create a plot for the solution to the differential equation
  $ty' + 2y = t^2 - t + 1$, if $y(1) = 0.5$.

\end{Question}

\begin{Solution}
  To generate a first-order DE solution in MATLAB, the differential
  equation must be written first in the form $y' = \ldots$.
  \begin{align*}
    ty' + 2y = t^2 - t + 1 \\
    ty' = -2y + t^2 - t + 1 \\
    y' = \frac{1}{t} (-2y + t^2 - t + 1) \\
  \end{align*}

Link to the MATLAB code: \\
\href{http://www.mast.queensu.ca/~apsc171/MNTCP01/PracticeProblems/MATLAB/W07DE05.m}{W07DE05.m}

Here is the graph of the solution.

\includegraphics[width = 0.5\linewidth]{graphics/Week07_DESolutions/W07DE05}

\end{Solution}

%*******************************
\item
\begin{Question}
  Create a plot for the solution to the differential equation
  $2xy^2 + 4 = 2(3 - x^2y)y'$ if $y(5) = 8$.

\end{Question}

\begin{Solution}
  To generate a first-order DE solution in MATLAB, the differential
  equation must be written first in the form $y' = \ldots$.  We start
  by switching both sides of the equation to put $y'$ on the left.
  \begin{align*}
    2(3 - x^2y)y' & = 2xy^2 + 4   \\
    y' & = \frac{ 2xy^2 + 4}{2 (3-x^2 y)}
  \end{align*}

Link to the MATLAB code: \\
\href{http://www.mast.queensu.ca/~apsc171/MNTCP01/PracticeProblems/MATLAB/W07DE06.m}{W07DE06.m}

Here is the graph of the solution.

\includegraphics[width = 0.5\linewidth]{graphics/Week07_DESolutions/W07DE06}

\end{Solution}

\end{enumerate}

\end{document}


%!TEX root = ceres-solver.tex
\chapter{Hello World!}
\label{chapter:tutorial:helloworld}
To get started, let us consider the problem of finding the minimum of the function
\begin{equation}
 \frac{1}{2}(10 -x)^2.
\end{equation}
This is a trivial problem, whose minimum is easy to see is located at $x = 10$, but it is a good place to start to illustrate the basics of solving a problem with Ceres\footnote{Full working code for this and other examples in this manual can be found in the \texttt{examples} directory. Code for this example can be found in \texttt{examples/quadratic.cc}}.

Let us write this problem as a non-linear least squares problem by defining the scalar residual function $f_1(x) = 10 - x$. Then $F(x) = [f_1(x)]$ is a residual vector with exactly one component.

When solving a problem with Ceres, the first thing to do is to define a subclass of \texttt{CostFunction}. It is responsible for computing the value of the residual function and its derivative (also known as the Jacobian) with respect to $x$.
\begin{minted}[mathescape]{c++}
class SimpleCostFunction
  : public ceres::SizedCostFunction<1 /* number of residuals */,
                                    1 /* size of first parameter */> {
 public:
  virtual ~SimpleCostFunction() {}
  virtual bool Evaluate(double const* const* parameters,
                        double* residuals,
                        double** jacobians) const {
    const double x = parameters[0][0];
    residuals[0] = 10 - x;  // $f(x) = 10 - x$
    // Compute the Jacobian if asked for.
    if (jacobians != NULL && jacobians[0] != NULL) {
      jacobians[0][0] = -1;
    }
    return true;
  }
};
\end{minted}
\texttt{SimpleCostFunction} is provided with an input array of parameters, an output array for residuals and an optional output array for Jacobians. In our example, there is just one parameter and one residual and this is known at compile time, therefore we can save some code and instead of inheriting from \texttt{CostFunction}, we can instaed inherit from the templated \texttt{SizedCostFunction} class.

The \texttt{jacobians} array is optional, \texttt{Evaluate} is expected to check when it is non-null, and if it is the case then fill it with the values of the derivative of the residual function. In this case since the residual function is linear, the Jacobian is constant.

Once we have a way of computing the residual vector, it is now time to construct a Non-linear least squares problem using it and have Ceres solve it.
\begin{minted}{c++}
int main(int argc, char** argv) {
  double x = 5.0;
  ceres::Problem problem;

  // The problem object takes ownership of the newly allocated
  // SimpleCostFunction and uses it to optimize the value of x.
  problem.AddResidualBlock(new SimpleCostFunction, NULL, &x);

  // Run the solver!
  Solver::Options options;
  options.max_num_iterations = 10;
  options.linear_solver_type = ceres::DENSE_QR;
  options.minimizer_progress_to_stdout = true;
  Solver::Summary summary;
  Solve(options, &problem, &summary);
  std::cout << summary.BriefReport() << "\n";
  std::cout << "x : 5.0 -> " << x << "\n";
  return 0;
}
\end{minted}

Compiling and running this program gives us
\begin{minted}{bash}
0: f: 1.250000e+01 d: 0.00e+00 g: 5.00e+00 h: 0.00e+00 rho: 0.00e+00 mu: 1.00e-04 li:  0
1: f: 1.249750e-07 d: 1.25e+01 g: 5.00e-04 h: 5.00e+00 rho: 1.00e+00 mu: 3.33e-05 li:  1
2: f: 1.388518e-16 d: 1.25e-07 g: 1.67e-08 h: 5.00e-04 rho: 1.00e+00 mu: 1.11e-05 li:  1
Ceres Solver Report: Iterations: 2, Initial cost: 1.250000e+01,  \
Final cost: 1.388518e-16, Termination: PARAMETER_TOLERANCE.
x : 5 -> 10
\end{minted}

Starting from a $x=5$, the solver in two iterations goes to 10~\footnote{Actually the solver ran for three iterations, and it was by looking at the value returned by the linear solver in the third iteration, it observed that the update to the parameter block was too small and declared convergence. Ceres only prints out the display at the end of an iteration, and terminates as soon as it detects convergence, which is why you only see two iterations here and not three.}. The careful reader will note that this is a linear problem and one linear solve should be enough to get the optimal value.  The default configuration of the solver is aimed at non-linear problems, and for reasons of simplicity we did not change it in this example. It is indeed possible to obtain the solution to this problem using Ceres in one iteration. Also note that the solver did get very close to the optimal function value of 0 in the very first iteration. We will discuss these issues in greater detail when we talk about convergence and parameter settings for Ceres.


%!TEX root = ../thesis.tex
% ******************************* Thesis Appendix B ********************************

\input{style.tex}

\chapter{Convolution}
\label{appendix.convolution}
Say we want to measure an observable $Q$, for example, for a specific occupied number of sites(bonds) in site(bond) percolation. But doing this we ignore the fact that we would have chose to fix a probability $p$ first and starting occupying the lattice sequentially then after we have had visited all the sites we would not have $p=n/(L^2)$ for site or $p=n/(2 L^2)$ for bond percolation each time. Therefore it would lead to some error. We can solve this dilemma by the following process.
The trick \cite{Hu1992, Gould2006} is to measure $Q$ for fixed numbers of occupied sites (or bonds) $n$ in the range of interest. Let us refer to the ensemble of states of a percolation system with exactly $n$ occupied sites or bonds as a "microcanonical percolation ensemble," the number $n$ playing the role of the energy in thermal statistical mechanics. The more normal case in which only the occupation probability $p$ is fixed is then the "canonical ensemble" for the problem. (If one imagines the occupied sites or bonds as representing particles instead, then the two ensembles would be "canonical" and "grand canonical," respectively. Taking site
percolation as an example, the probability of there being
exactly $n$ occupied sites on the lattice for a canonical percolation ensemble is given by the binomial distribution
	\begin{equation}
		B(N,n,p) = {N\choose n} p^n (1-p)^{N-n}
	\end{equation}
	The same expression applies for bond percolation, but
	with $N$ replaced by $M$ , the total number of bonds.	Thus, if we can measure our observable within the microcanonical ensemble for all values of $n$, giving a set
	of measurements ${Q_n}$, then the value in the canonical
	ensemble will be given by
	\begin{equation}
		Q(p)
		= \sum_{n=0}^{N} B(N,n,p) Q_n
		= \sum_{n=0}^{N} {N\choose n} p^n(1-p)^{N-n} Q_n
	\end{equation}
	Thus we need only measure $Q_n$ for all values of $n$.
	\cite{Newman2001}.

\section{Algorithm}
	But using this formula to calculate $Q(p)$ is quite expensive since the number of sites(bonds) can be quite large  direct evaluation of the binomial coefficients using factorials is not possible. We use alternative way to measure $Q(p)$, which is basically does the same thing but in an efficient way.

	Instead, therefore, we recommend the following method of evaluation. The binomial distribution, Eq. (1), has its largest value for given $N$ and $p$ when $n = n_{max} = p N$ . We arbitrarily set this value to $1$. Now we calculate $B(N, n, p)$ iteratively for all other $n$ from

	\begin{equation*}
	B(N,n,p) =
	\begin{cases}
		B(N, n-1, p) \frac{N-n+1}{n} \frac{p}{1-p}  \text{ if } n > n_{max}\\
		B(N, n+1, p) \frac{n+1}{N-n} \frac{1-p}{p}  \text{ if } n < n_{max}
	\end{cases}
	\end{equation*}
	Then we calculate the normalization coefficient $C = \sum_{n} B(N, n, p)$ and divide all the $B(N, n, p)$ by it, to correctly normalize the distribution 	\cite{Newman2000, Newman2001}.

\section{Code}
	In order to use convolution the following function can be used. It is very efficient and uses OpenMP for parallel run, meaning, all the cores of the processors performs the same task in a divide-and-conquer manner. This function takes an array as input and returns the convoluted version as output.
	\begin{lstlisting}[style=CStyle]
	vector<double> convolution(vector<double>& data_in) {
	size_t N = data_in.size();
	std::vector<double> _forward_factor(N);
	std::vector<double> _backward_factor(N);
	for (size_t i=0; i < N; ++i)
	{
	_forward_factor[i]  = (double) (N - i + 1) / i;
	_backward_factor[i] = (double) (i + 1) / (N - i);
	}
	vector<double> data_out(N);
	long step = N / 1000;
	#pragma omp parallel for schedule(dynamic)
	for (long j=0; j <N; ++j) // start from j=1
	{
	double prob     = (double) j / N;
	double factor   = 0;
	double binom    = 0;
	double prev     = 0;
	double bn_tot   = 1; // normalization factor
	double sum      = data_in[j];
	// forward iteraion part
	factor = prob / (1-prob);
	prev   = 1;
	for (long i=j+1; i<N; ++i)
	{
	binom     = prev * _forward_factor[i] * factor;
	bn_tot += binom;
	sum      += data_in[i] * binom;
	prev      = binom;
	}
	// backward iteration part
	factor = (1-prob)/prob;
	prev   = 1;
	for (long i=j-1; i>=0; --i)
	{
	binom     = prev * _backward_factor[i] * factor;
	bn_tot += binom;
	sum      += data_in[i] * binom;
	prev      = binom;
	}
	// normalizing data
	data_out[j] = sum / bn_tot;
	cout << bn_tot << endl;
	}
	return data_out;
	}
	\end{lstlisting}
Complete code for convolution is available at
\url{https://github.com/sha314/Convolution}.\\

The program linked above works using command line arguments. For example to perform convolution on a file "data.txt" which contains two columns out of which only second column needed to be convoluted use the following command.

\begin{lstlisting}[language=bash]
./convolute -f data.txt -a 0 -b 1
\end{lstlisting}

\begin{lstlisting}[style=Instruction]
Usage:
convolution [-f <STRING>] [-a <INT>,<INT>,...[:[<STRING>,<STRING>,...]]] [-b <INT>,<INT>,...[:[<STRING>,<STRING>,...]]] [-h] [-t <INT>] [-i <STRING>]
perform convolution based on provided options.
Options                      Description
-a,                        columns that we want in the output file without performing convolution.
No default value.
-b,                        columns that we want in the output file with performing convolution.
No default value.
-c,                        If provided the header and comment from the input file will be written
without modification to the output file. Header is the first line of the
input file.
-d                         Delimeter to use. Default value is ' '.
-f                         name of the input file that we want to convolute. No default value.
-i                         Info to write as comment in the output file
-o                         name of the output file. If not provided the string '_convoluted.txt' will be
appended to the input file.
-p, --precision            Floating point precision when writing in the data file. Default value is 10
-s                         Number of rows to skip from the input file. Default value is 0.
-t                         to test the performance of the convolution program. No default value.
--threads              Explicitly specify number of thread to use. Default is the max number of thread
allowed by the system.
-h, --help                 display this help and exit
-v, --version              output version information and exit
-w                         If provided input b data will be written to the output file.
The INT argument is an integer.
The STRING argument is a string of characters.
A line that begins with '#' is considered a commented line.
Exit status:
0  if OK,
1  if minor problems (e.g., cannot access subdirectory),
2  if serious trouble (e.g., cannot access command-line argument).
\end{lstlisting}


\documentclass{article}
\usepackage{amsmath, amsthm, amsfonts, amssymb, verbatim, hyperref}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{lemma}[theorem]{Lemma}

\begin{document}

\section{Hensel lifts}
Let $p$ be a prime and suppose we have a polynomial factorization
\[
u = \bar v \cdot \bar w \mod p
\] with \(p\)-prime, where \(\bar v\) and \(\bar w\) are relatively prime monic polynomials in the variable \(x\). Set
\[
\deg \bar v = m\quad \deg \bar  w = n.
\]
Our goal is to construct monic polynomials \(v, w\) of the same degrees as \(\bar v,\bar  w\) with
\[
u = v \cdot w \mod p^k.
\]
Since factorization $\mod p$ is unique, it follows that \(v =  \bar v \mod p \) and \(w = \bar w \mod p\). Our secondary goal is to show that, \(\mod p^{k}\), these polynomials are unique. Proceed by induction. Suppose we have already found \(v, w\) as above and are looking for \(v', w'\) with
\[
u = v'\cdot w' \mod p^{k+1}.
\]
Over $\mathbb Z$, we have that

\[
u = vw + p^k z
\]
for some polynomial \(z\). Since \(u,v,w\) are monic, the leading monomials of \(u\) and \(vw\) are $x^{m+n}$ and so $\deg z = m + n - 1$. Since  \(v, w\) are unique \(\mod p^{k}\), it follows that \(v', w'\) must be of the form
\[
v' = v + p^{k} a \quad w' = w+ p^k b
\]
with \(\deg a < \deg v= m\) and \(\deg b < \deg w = n\).
\begin{equation}\label{eqHenselLift0}
\begin{array}{rcl}
v' w'& =& \left(v+p^k a\right)\left(w + p^k b\right) \\
&=&vw + p^k(aw + bv) + p^{2k} ab\\
&=&u + p^{k}z + p^{k}(aw+bv) + p^{2k}ab\\
&=&u + p^{k}\left( z + aw + bv\right) + p^{2k}ab
\end{array}
\end{equation}
It follows that
\begin{equation}\label{eqHenselLift1}
z + aw + bv = 0 \mod p
\end{equation}
Introduce notation for the coefficients of the polynomials in \eqref{eqHenselLift1}:
\begin{equation}
\begin{array}{rcl}
a &=& a_{1} x^{m-1} + \dots + a_m\\
b &=& b_{1} x^{n-1} + \dots + b_m \\
z &=& z_{1}x^{m + n - 1} + \dots +z_{m+n}\\
w &=& x^{m} + w_{1} \cdot x^{m - 1} + \dots + w_{m}\\
v &=& x^{n} + v_{1} \cdot x^{n - 1} + \dots + v_{n}\\
\end{array}
\end{equation}
Then \eqref{eqHenselLift1} is a linear system in the \(m+n\) variables \(a_1, \dots, a_m, b_1, \dots, b_n\) with $\deg z + 1 = m + n $ equations. More precisely, the matrix form of the equation \eqref{eqHenselLift1} is given by the Sylvester matrix:
\begin{equation}\label{eqHanselLift2Sylvester}
\left( \begin{array}{ccccccccccc}
w_0    & 0       & \cdots & 0       & v_0    & 0       & \cdots & 0      \\
w_1    & w_0     & \cdots & 0       & v_1    & v_0     & \cdots & 0      \\
w_2    & w_1     & \ddots & 0       & v_2    & v_1     & \ddots & 0      \\
\vdots &\vdots   & \ddots & w_0     & \vdots &\vdots   & \ddots & v_0    \\
w_m    & w_{m-1} & \cdots & \vdots  & v_n    & v_{n-1} & \cdots & \vdots  \\
0      & w_m     & \ddots & \vdots  & 0      & v_n     & \ddots & \vdots  \\
\vdots & \vdots  & \ddots & w_{m-1} & \vdots & \vdots  & \ddots & v_{n-1} \\
0      & 0       & \cdots & w_m     & 0      & 0       & \cdots & v_n

\end{array}\right) \begin{pmatrix}
a_1 \\
a_2\\
\vdots\\
a_m\\
b_1\\
b_2\\
\vdots \\
b_n
\end{pmatrix} = \begin{pmatrix}
z_1\\
z_2\\
\vdots
\\
z_m\\
z_{m+1} \\
z_{m+2} \\
\vdots \\
z_{m+n}

\end{pmatrix} \mod p,
\end{equation}
where for convenience we have set $v_0 = w_0=1$. The determinant of the matrix above, called the resultant, is known to equal \(\text{res}(v,w) = \displaystyle w_0^mv_0^n\prod_{i, j} \left(\nu_i-\mu_j \right)\), where $\nu_i, \mu_j$ are the roots of $v$ and $w$ over the algebraic closure of $\mathbb Z / p\mathbb Z$. In our starting factorization $u=\bar v\cdot \bar w $, the factors $\bar v, \bar w$ are relatively prime and so have no common roots and we have that $\text{res}(\bar v, \bar w)\neq 0$. At the same time we established that $v =\bar v \mod p$ and $w=\bar w\mod p $ and so $\text{res} (\bar v, \bar w) = \text{res}(v,w) \neq 0$. Since the determinant of \eqref{eqHanselLift2Sylvester} is non-zero, \eqref{eqHanselLift2Sylvester} has a unique solution. This shows both the existence and uniqueness of $v',w'$. This in turn concludes our inductive step.
\subsection{Hensel lift for more than two factors}
We want to extend the Hensel lift from the previous section to the case of more than two factors. Let
\[
u = \bar v_1 \dots \bar v_l \mod p
\]
be a factorization with \(p\)-prime. The considerations of the two factor case carry over to the multi-factor one. TODO(tmilev): spell out the notation. Equation \eqref{eqHenselLift0} becomes
\[
\begin{array}{rcl}
v_1'\dots v_k'& = & u + p^k \left(z+ a_1 v_2\dots v_l + v_1 a_2\dots v_l +\dots + v_1\dots v_{l-1} a_l   \right) +p^{2k} s
\end{array}
\]
for some polynomial $s$. Let $t_j$ be the polynomial obtained by removing the $j^{th}$ multiplicand from $v_1\dots v_l$, i.e., let
\[t_j=\frac{v_1\dots v_l}{v_j}. \]

Then equation \eqref{eqHanselLift2Sylvester} carries over directly. Its matrix becomes
\[
\left( \begin{array}{ccccccccccc}
t_{1,0}       & 0      & \cdots & 0              & t_{2,0}       & 0      & \cdots & 0              & \dots \\
\vdots        & \ddots & \ddots & \vdots         & \vdots        & \ddots & \ddots & \vdots         & \dots \\
\vdots        & \ddots & \ddots & 0              & \vdots        & \ddots & \ddots & 0              & \dots \\
t_{1,\deg t_1}& \ddots & \ddots & t_{1,0}        & t_{2,\deg t_2}& \ddots & \ddots & t_{2,0}        & \dots \\
0             & \ddots & \ddots & \vdots         & 0             & \ddots & \ddots & \vdots         & \dots \\
\vdots        & \ddots & \ddots & \vdots         & \vdots        & \ddots & \ddots & \vdots         & \dots \\
0             & \cdots & 0      & t_{1,\deg t_1} & 0             & \cdots & 0      & t_{2,\deg t_2} & \dots \\
\end{array}\right)\]

\end{document}


\section{Probability interpretation of logistic regression}%Jonathan will input his notes from camera scaned files.
Logistic regression models the probabilities for classification problems with two possible outcomes. It's an extension of the linear regression model for classification problems.
\begin{itemize}
\item Input: d-dimension feature vector $x\in R^{d}$
\item Output: a class or label $l \in\{1, \ldots, k\}$ (how to classify)
\item Example 1: Image classfication:\newline
Given input image $x \in R^{n \times n}$, predict a label for the image. e.g. cat/ dog;
MNIST: $\{0, \ldots, 9\}$; CIFAR-10, $\{0, \ldots, 9\}$
\item Example 2: Binary Classfication:\newline
Given some medical data $x\in R^{d}$(features, like heart pressure, resting heart rate, family history, etc), we try to predict incidence of heart disease. The label will be binary $\{0, 1\}$, called binary classification.
\end{itemize}

\subsection{Logistic Regression Model}
Given input feature vector $x\in R^{n}$, the model predicts a probability distribution over the labels $\{0, \ldots, k\}$.
$$
\mbox{affine linear map: }Ax+b:\quad R^{n} \rightarrow R^{k}
$$
with $A\in R^{k \times n}$, $x\in R^{n}$, $b\in R^{k}$.
 $$
 \mbox{softmax} \left( Ax+b\right) : \quad R^{n} \rightarrow R^{k} \xrightarrow{softmax}P\left( \{1, \ldots, k\}\right)
 $$
Softmax:
\begin{enumerate}
\item Input: $y \in \mathbb{R}^{k}=\left(\begin{array}{l}y_{1} \\ \vdots \\ y_{k}\end{array}\right)$
\item Output: Distribution $p(j)=\frac{e^{y_{j}}}{\sum_{i=1}^{n} e^{y_{i}}}$
\item Take exponential and normalize
\end{enumerate}

{\bf Logistic Regression Model}: Given input (feature/data) $x \in R^{n}$, we return the distribution softmax  $\left( Ax+b\right) $ where A, b are parameters.
\begin{itemize}
	\item Example 1: Divide data into two classes: parameters are $\vec{a}\in \mathbb{R}^{n}, b \in \mathbb{R}$
	$$
	A\in \mathbb{R}^{1\times n},\quad Ax+b\in \mathbb{R}.
	$$
	The probability that the data $x$ belongs to class 1 is
	$$
	P(1)=\frac{e^{\vec{a}\cdot x+b}}{e^{\vec{a} \cdot x+b}+1},
	$$
	and the probability that the data $x$ belongs to class 2 is
	$$
	P(2)=\frac{1}{e^{\vec{a} \cdot x+ b}+1}.
	$$
	If $\vec{a} \cdot x+ b=0$, $P(1)=	P(2)={1\over 2}$. We don't know how to classify the data lying on the line $\vec{a} \cdot x+ b=0$ as shown in the figure below.
   \begin{figure}[ht!]
  	\centering
  	\includegraphics[width=.55\textwidth]{../figures/probabilityLR1.png}
  \end{figure}
  Note that
$$
\frac{p(1)}{p(2)}=e^{\vec{a} \cdot x+b},\quad \log \left(\frac{p(1)}{p(2)}\right)=\vec{a} \cdot x+b.
$$
By the above equation, $\vec{a}$ means which feature is important.  \newline
Logarithm of the odds: $ \log \left(\frac{p(1)}{p(2)}\right)$. \newline
Assumption:  $\log \left(\frac{p(1)}{p(2)}\right)$ is linear in the feature vector
\end{itemize}

\subsection{Learning the parameters  $\vec{a}, b$ from data }
Data: feature vectors $x$ and corresponding labels $l$. Given data
$$
\left\{\left(x_{1}, l_{1}\right), \ldots,\left(x_{n}, l_{1}\right)\right\}=D
$$
How can we estimate A, b?

\section{Maximamum Likelihood}
If parameters $A$ and $b$ are known, for any data $x_1\in \mathbb{R}^n$, model gives softmax  $\left( Ax_1+b\right) $ \newline
$$
\frac{1}{\sum_{i=1}^{k} e^{\vec{a_{i}}\cdot x_{i}+b_{i}}}
\left(\begin{array}{c}e^{\vec{a}_{1}\cdot x_{1} +b_{1}} \\ \vdots \\ e^{a_{k} \cdot x_{1}+b_{k}}\end{array}\right)
=
\left(\begin{array}{c}
p(1)
 \\ \vdots \\
p(k)
 \end{array}\right)
$$
This means that the probability that the model assigns to $l_{1}$ is
$$
p(l)=\frac{1}{\sum_{i=1}^{k} e^{a_{i}\cdot x_{i}+b_{i}}} \cdot e^{\vec{a}_{l} \cdot x_{1}+b_{l}}
$$
Instead of considering this probability, we consider its negtive logarithm: \newline
$$
-\log p(l)=\log \left(\sum_{i=1}^{k} e^{a_{i} \cdot x_{i}+b_{i}}\right)-\left(\vec{a}_{l} \cdot x_{1}+b_{l}\right)
$$
Since data points are independent, so we take product of the probability which leads to the logistic regression loss.\newline
Logistic Regression Loss:
$$
L_{D}(A, b)=\sum_{(x, l) \in D}\left[\log \left(\sum_{i=1}^{k} a_{i} \cdot x+b_{i}\right)-\left(\vec{a}_{l} \cdot x+b_{l}\right)\right]
$$
We want to maximize the probability that the model assigns to $l$, that means we need to minimize $-\log p(l)$. So we need to find
$$
(A, b)=\min _{A, b} L_{D}(A, b).
$$

\section{Basic Statistical Learning Theory}
\begin{itemize}
	\item Goal: Estimate an unknown probability distribution $D$ on a set $X$ from samples $(i,i,d)$ $x_{1}, \ldots, x_{n} \in X$
	\item Introduce a family of distributions $P_{\theta}$ for $\theta \in \Theta$ and try to choose $\theta$ to "match" the samples.
	\begin{itemize}
		\item Maximum Likelihood Estimate: Choose $\theta$ to maximize the probability of the samples.
		\item Example: Let $X=R$, have some samples $x_{1}, \ldots, x_{n}$ drawn from a distribution D, say $P_{\theta}$ is a Gaussian with variance 1, centered at $\theta \in \mathbb{R}=\Theta$, i.e. density  $p_{\theta}(x)=\frac{1}{\sqrt{2 \pi}} e^{-(x-\theta)^{2} / 2}$

		   \begin{figure}[ht!]
		  	\centering
		  	\includegraphics[width=.55\textwidth]{../figures/probabilityLR2.png}
		  \end{figure}
		\item Use the samples to find the center $\theta$.
	\end{itemize}
\end{itemize}

\subsection{Maximum Likelihood Estimate(MLE)}
\begin{itemize}
	\item Given $\theta \in \Theta (=\mathbb{R}$ for this example), what is the probability of the data $\left\{x_{j}\right\}_{j=1}^{n}$?
	\begin{itemize}
		\item Samles independent: Likelihood function(as a function of $\sigma$)
		$$
		P_{\theta}\left(\left\{x_{j}\right\}_{j=1}^{n}\right)=\prod_{j=1}^{n} p_{\theta}\left(x_{j}\right)
		=\frac{1}{(\sqrt{2 \pi})^{n}}\prod_{j=1}^{n} e^{-\left(x_{j}-\theta\right)^{2} / 2}
		=\frac{1}{(2 \pi)^{n/ 2}} e^{-\sum_{j=1}^{n}\left(x_{j}-\theta\right)^{2} / 2}
		$$
		\item MLE: Choose $\theta$ to maximize this!
		\item Often it's useful to consider log likelihood function $\log \left(P_{\theta}\left(\left\{x_{j}\right\}_{j=1}^{n}\right)\right) $
		$$
		\begin{aligned} \theta^{*}= \operatorname{argmax} _{\theta \in \Theta}\log \left(P_{\theta}\left(\left\{x_{j}\right\}_{j=1}^{n}\right)\right) \\
		\left(\operatorname{argmin} _{\theta \in \Theta}-\log \left(P_{\theta}\left(\left\{x_{j}\right\}_{j=1}^{n}\right)\right)\right) \end{aligned}
		$$
	\end{itemize}
     \item For this example:
     $$
     \log \left(P_{\theta}\left(\left\{x_{j}\right\}_{j=1}^{n}\right)\right)=-\log (2 \pi) \cdot\left(\frac{n}{2}\right)-\sum_{j=1}^{n} \frac{\left(x_{j}-\theta\right)^{2}}{2}
     $$
     $$
     \theta^{*}=\operatorname{argmin}_{\theta \in \mathbb{R}} \sum_{j=1}^{n} \frac{\left(x_{j}-\theta\right)^{2}}{2}.
     $$
     $$
      \theta^{*}=\frac{1}{n} \sum_{j=1}^{n} x_{j}.
      $$

     	   \begin{figure}[ht!]
       	\centering
       	\includegraphics[width=.55\textwidth]{../figures/probabilityLR3.png}
       \end{figure}
\end{itemize}

\section{Classfication/ Logistic Regression}
\begin{itemize}
	\item For Classification: X is feature space and Y is label space
	$$
	\widetilde{X}=X \times Y
	$$
	\item We have samples $\left\{\left(x_{j}, y_{j}\right)\right\}_{j=1}^{n}$
	\item Suppose that D is an unknown distribution on $\widetilde{X}$, but we're only trying to estimate $D_{Y|X}$ as a function of X. (Given the feature $X$, what is the possible label.)
	\item Introduce parameters $\theta\in \Theta$, we define $p(y|x, \theta)$ (called the model)
	\item Now choose $\theta$ to match the data $\left\{\left(x_{j}, y_{j}\right)\right\}_{j=1}^{n}$
	\item MLE:
	\begin{align*}
	\theta^{*}&=\operatorname{argmax}_{\theta \in\Theta} p\left(\left\{y_{i}\right\}_{j=1}^{n} |\left\{x_{j}\right\}_{j=1}^{n}, \theta\right)\\
	&=\operatorname{argmax}_{\theta \in\Theta}  \prod_{j=1}^{n} p\left(y_{j} | x_{j}, \theta\right)   \\
	&=\operatorname{argmin}_{\theta \in\Theta}  \sum_{j=1}^{n}-\log \left(p\left(y_{j} | x_{j}, \theta\right)\right)
	\end{align*}
	\item Example: Logistic Regression:
	\begin{itemize}
		\item Model
		$$
		p(y | x, \theta)=p(y | x, w, b)
		$$
		with $W\in \mathbb{R}^{d\times d}$ and $b\in \mathbb{R}^k$ (parameters for affine linear map)
		\item d: dimension of features, i.e. number of pixels
		\item k: number of classes
	\end{itemize}
$$
p(y | x, w, b)=\mbox{ softmax} \left( W x+b\right)=\frac{1}{\sum_{i=1}^{k} e^{w_{i}\cdot x+b_{i}}}\left(\begin{array}{c}e^{w_{1}\cdot x +b_{1}}  \\ \vdots \\ e^{w_{k} \cdot x +b_{k}}\end{array}\right)
$$
\item Need to calculate:
\begin{equation}\label{eq:pp}
\begin{split}
&-\log \left(p\left(y_{i} | x_{i}, w, b\right)\right)\\
&=-\log \left(\frac{1}{e^{w  x_i+b} \cdot \mathbb{I}} e^{w x_i+b} \cdot y_{i}\right)\\
&=\log \left(e^{w x_i  +b} \cdot 1\right)-\log \left(e^{w x_i+b} \cdot y_{i}\right)
\end{split}
\end{equation}
with $y_i=(0,\cdots, 0,1,0,\cdots, 0)$(only the $i$-th entry is 1, all others are 0) and all the entries of $\mathbb{I}$  are 1.
$$
(w, b)^{*}=\underset{w,b}{\operatorname{argmin}} \sum_{i=1}^{n} \log \left(e^{wx_{i}+b} \cdot \mathbb{I}\right)-\log \left(e^{wx_{i}+b} \cdot y_{i}\right)
$$
\end{itemize}

\section{Bayesian Approach to Machine Learning}
\subsection{Goal}
\begin{itemize}
	\item Goal: Estimate an unknown distribution on X from data $\left\{x_{j}\right\}_{j=1}^{n}$
	\begin{itemize}
		\item Build a model
	    \begin{itemize}
	    	\item Set of parameters $\Theta$
	    	\item Family of distribution on X,
		$$p(x|\theta)\quad \mbox{with}\quad \theta \in \Theta $$
	    	\item Prior distribution on the parameters $\Theta$,
		$$
		q(\theta).
		$$
	    \end{itemize}
    \item Use Bayes' Law
    $$
    p(\theta|x) p(x)=p(x \  and \   \theta )=p(x | \theta) p(\theta)
    $$
    \item Recall if $A_{1} , A_{2}$ are events:
    $$
    p\left(A_{1} | A_{2}\right)=\frac{P\left(A_{1} \cap A_{2}\right)}{P\left(A_{2}\right)}
    $$
    $$
    p(\theta | x)=\frac{p(x | \theta) q(\theta)}{p(x)}
    $$
    $$
    p(\theta |x) \sim p(x | \theta) q(\theta)
    $$
    where $p(\theta |x)$ is the posteriori distribution, $q(\theta)$ is the prior distribution and $p(x | \theta)$ is the likelihood function.
	\end{itemize}
\item Stant with: prior $q(\theta)$
\item Add data $x$, replace $q$ with the posterior
 $$
 p(\theta | x) \sim p(x | \theta) q(\theta)
 $$
\item More data: multiply by likelihood function and then normalize
\item Left with a posterior distribution
     \begin{itemize}
	\item Sample from posterion distribution to approximate
	$$
	P_{pred}(x)=\int_{\Theta} p(x | \theta) p(\theta) d \theta
	$$
	\item Choose $\theta $ to maximize posterior distribution
	\begin{align*}
	\theta^{*}&=\arg \max _{\theta \in \Theta} p(\theta|x)\\
 &=\arg \min _{\theta \in \Theta}-\log p(\theta|x)\\
&=\arg \min _{\theta \in \Theta}-\log (p(x | \theta))-\log (q(\theta))+\log (p(x))\\
&=\arg \min _{\theta \in \Theta}-\log (p(x | \theta))-\log (q(\theta))
\end{align*}
where $-\log (p(x | \theta))$ is the negative log likelihood and $ -\log (q(\theta))$ is the regularization coming from prior.
    \end{itemize}
\subsection{Example: Image  Classification/ Logistic Regression}
\item Images $x \in X=\mathbb{R}^{d}$
\item Labels $y \in Y=\left\{e_{1}, \ldots, e_{k}\right\}$ with $k$ is the dimension of labels
\item Data $\left\{(x_{j},y_{j})\right\}_{j=1}^{n}$
\item Model $\theta=(W, b) \in \mathbb{R}^{k \times d} \times \mathbb{R}^{k}$. By \eqref{eq:pp},
$$
p(y | x, \theta)=\frac{e^{Wx+b} \cdot y}{e^{Wx+b} \cdot \mathbb{I}}
$$
\item Prior distribution: suppose it is a Gaussian,
$$
q(W, b)=C e^{-\alpha\left(\left\|W\right\|_{2}^{2}+ \left\|b\right\|_{2}^{2}\right)}
$$

     	%   \begin{figure}[ht!]
%  	\centering
%  	\includegraphics[width=.35\textwidth]{../figures/probability1.png}
%  \end{figure}

\begin{itemize}
	\item Calculate the posterior: here $q(W,b)=p(W, b|x)$,
	$$
	p(W, b | x, y)=\frac{p(y | x, W, b) p(W, b|x) }{p(y | x)}
	$$
	$$
	p(y | x)=\int_{\Theta} p(y|x, W, b) q(W, b) d \theta
	$$
	\begin{align*}
	(W, b)^{*}&=\underset{W,b}{\arg \min }-\log \left( p\left(W, b |\left\{x_{j},y_{j}\right\}_{j=1}^{n}\right)\right) \\
&=\underset {W,b}\arg  \min -\log \left( p\left(\left\lbrace y_{j}\right\rbrace _{j=1}^{n} | \left\lbrace x_{j}\right\rbrace _{j=1}^{n}, W, b\right)\right)-\log (q(W, b))\\
&=\underset {W,b}\arg  \min	-\log \left(\prod_{i=1}^{n} p\left(y_{j} | x_{j}, W, b\right)\right)-\log (q(W, b))\\
&=\underset{W, b}{\operatorname{argmin}} \sum_{j=1}^{n} \log \left(e^{Wx+b} \cdot \mathbb{I}\right)-\log \left(e^{Wx_j+b} \cdot y_{j}\right)+\alpha\left(\|W\|_{2}^{2}+\| b\|_{2}^{2}\right).
	\end{align*}

\end{itemize}

\end{itemize}

%\includepdf{HandWrittenNotes/LR1.pdf}
%\includepdf{HandWrittenNotes/LR2.pdf}
%\includepdf{HandWrittenNotes/LR3.pdf}
%\includepdf{HandWrittenNotes/BasicSLT1.pdf}
%\includepdf{HandWrittenNotes/BasicSLT2.pdf}
%\includepdf{HandWrittenNotes/BasicSLT3.pdf}
%\includepdf{HandWrittenNotes/BasicSLT4.pdf}
%\includepdf{HandWrittenNotes/Bayesian2.pdf}
%\includepdf{HandWrittenNotes/Bayesian1.pdf}
%\includepdf{HandWrittenNotes/Bayesian3.pdf}
%\includepdf{HandWrittenNotes/Bayesian4.pdf}


\lab{Markov Chains}{Markov Chains}
\objective{Learn about Markov chains.}
\label{lab:EigSolve}

\section*{Markov chains}
A Markov chain is a collection of states with specified probabilities for transitioning from one state to another.
Markov chains are characterized by the fact that future behavior of the system depends only on its current state.

This lab is a very brief introduction to Markov chains. To learn more, see [TODO: ref something].

\subsection*{An example}
Suppose Fredo the frog is jumping between the three lily pads 1, 2, and 3.
These three pads are the \emph{possible states} of the system, and the lily pad on which Fredo is presently sitting is the \emph{current state}.
If Fredo is on lily pad 1 and jumps, there is a 25\% chance that it will land back on lily pad 1, a 25\% chance that it will land on lily pad 2, and a 50\% chance that it will land on lily pad 3.
These probabilities are the \emph{transition probabilities}.
Figure \ref{fig:markov1} is a \emph{transition diagram} that depicts all transition probabilities.

\begin{figure}
\begin{tikzpicture}[normalcircle/.style={draw, circle, minimum size=1cm, fill=shadecolor, thick, node distance=1.5cm} ]

\node[normalcircle](circle1)[]{};
\node[draw=none](one)[]{1};

\node[draw=none, node distance=2.5cm](dummy)
	[below of=circle1]{};

\node[normalcircle](circle2)[left of=dummy]{};
\node[draw=none, node distance= 1.5cm](two)[left
	of=dummy]{2};
\node[normalcircle](circle3)[right of=dummy]{};
\node[draw=none, node distance= 1.5cm](three)
	[right of=dummy]{3};

\foreach \s /\t in {circle2/circle1, circle1/circle3, circle3/circle2}
	{\path[draw,bend left=20, thick, ->, >=stealth'] (\s)edge(\t);}
\foreach \s /\t in {circle1/circle2, circle3/circle1, circle2/circle3}
	{\path[draw,bend left=20, thick, ->, >=stealth'] (\s)edge(\t);}

\draw[thick,->, >=stealth'](-1.9,-2.8) arc (325:40:.4 and .5);
\draw[thick,->, >=stealth'](1.9,-2.2) arc (500:215:.4 and .5);
\draw[thick,->, >=stealth'](-.4,.3) arc (220:-35:.5 and .4);

\node[draw=none, node distance=1.6cm](dummy2)
	[below of=circle1]{};
\node[draw= none, node distance=.3cm](midvalues)
	[above left of=dummy2]{$\frac{1}{4}$};
\node[draw= none, node distance=.3cm](midvalues2)
	[above right of=dummy2]{$\frac{1}{2}$};
\node[draw= none, node distance=.25cm](midvalues3)
	[below of=dummy2]{$\frac{1}{3}$};
\node[draw=none, node distance=1.6cm](outsidevalue)
	[above left of=midvalues3]{$\frac{1}{2}$};
\node[draw=none, node distance=1.6cm](outsidevalue2)
	[above right of=midvalues3]{$\frac{1}{2}$};
\node[draw=none, node distance=1.35cm](outsidevalue3)
	[below of=midvalues3]{$\frac{1}{2}$};
\node[draw=none, node distance=1.25cm](circlevalue)
	[above of=circle1]{$\frac{1}{4}$};
\node[draw=none, node distance=2.8cm](circlevalue2)
	[left of=dummy]{$\frac{1}{6}$};
\node[draw=none, node distance=2.8cm](circlevalue3)
	[right of=dummy]{$0$};

\end{tikzpicture}
\caption{Transition diagram for Fredo the Frog.}
\label{fig:markov1}
\end{figure}

We can convert our transition diagram into a \emph{transition matrix} The $(i,j)$-entry of the transition matrix is the probability that Fredo jumps from lily pad $j$ to lily pad $i$.
Fredo's transition matrix is
\[
A = \begin{pmatrix}
1/4 & 1/2 & 1/2\\
1/4 & 1/6 & 1/2\\
1/2 & 1/3 & 0
\end{pmatrix}.
\]

At any time, the chances that Fredo is on each lily pad is encoded by a \emph{state distribution vector} $\x = (x_1, x_2, x_3)^T$, where $x_i$ is the probability that Fredo is on lily pad $i$.
For $\x$ to be a state distribution vector, we require $x_i\geq 0$ and $\|\x\|_1=1$ (recall that the 1-norm of a column vector is the sum of the magnitudes of its entries).
Then $A\x$ will be another state distribution vector, which tells us the probability that Fredo is on each lily pad after one jump.

Thus, we can use Fredo's transition matrix to find where he will be after $k$ jumps. In fact, the $(i,j)$-entry of $A^k$ is the probability that Fredo goes from lily pad $j$ to lily pad $i$ in $k$ jumps.
In our case,

\[
A^2 \approx \begin{pmatrix}
0.4375 & 0.3750 & 0.3750\\
0.3542 & 0.3194 & 0.2083\\
0.2083 & 0.3056 & 0.4167
\end{pmatrix}.
\]
Therefore, if Fredo starts on lily pad 1, there is a 43.75\% chance it will still be on lily pad 1 after two jumps.
Maybe Fredo jumped from 1 to 1 to 1, denoted $1 \rightarrow 1 \rightarrow 1$, or perhaps it jumped to one of the other lily pads and then back again, that is, either $1 \rightarrow 2 \rightarrow 1$ or $1 \rightarrow 3 \rightarrow 1$.

In addition, there is a 35.42\% chance Fredo will be on lily pad 2 and a 20.83\% chance that it will be on lily pad 3.
We can type our transition matrix into Python and see where Fredo is likely to be after any number of jumps.

\begin{lstlisting}
# The 1.'s in the numerator force floating point division.
>>> A = np.array([[1./4,1./2,1./2],[1./4,1./6,1./2],[1./2,1./3,0]])
>>> np.linalg.matrix_power(A,10)
array([[ 0.40000057,  0.39999962,  0.39999962],
       [ 0.30002369,  0.29999268,  0.29997574],
       [ 0.29997574,  0.3000077 ,  0.30002464]])
\end{lstlisting}

In fact, as we take higher and higher powers of $A$, it appears that
\[
\lim_{k \rightarrow \infty} A^k = \begin{pmatrix}
0.4 & 0.4 & 0.4\\
0.3 & 0.3 & 0.3\\
0.3 & 0.3 & 0.3
\end{pmatrix}.
\]
This means that Fredo's state distribution approaches $(0.4, 0.3, 0.3)^T$ after many jumps, regardless of his initial state distribution.

Moreover, suppose Fredo's initial state distribution is $(0.4, 0.3, 0.3)^T$.
We can use Python to compute his state distribution after 1 jump.
\begin{lstlisting}
>>> A.dot(np.array([0.4,0.3,0.3]))
array([ 0.4,  0.3,  0.3])
\end{lstlisting}
Fredo's state distribution after a jump stays ``fixed.'' We call the vector $(0.4, 0.3, 0.3)^T$ a \emph{stable fixed point} for Fredo.

\subsection*{General Markov chains}
Let us generalize this example. A Markov chain is a collection of states with the probabilities that we will move from one state to another.
These transition probabilities are encoded in a transition matrix, whose $(i-j)$-entry is the probability of moving from state $j$ to state $i$.
Each column of such a matrix will necessarily have entries that sum to 1.

%When we have a Markov chain, we want to know what the state distribution is after our system has been running for some time. We are especially interested if the state distributions converge to something that is independent of the initial state, as was the case for Fredo. That is, we want to know about stable fixed points.

Let $A$ be the transition matrix of a Markov chain.
Then a state distribution vector $\x$ is a \emph{stable fixed point} if $A\x=\x$.
So $\x$ is a stable fixed point if and only if $\x$ is a positive unit eigenvector of $A$ corresponding to the eigenvalue 1.

Every Markov chain has at least one stable fixed point.
If in addition we assume some power $A^k$ of $A$ has all positive (nonzero) entries, then the stable fixed point is unique.
In this case, $A^k$ will converge to a matrix whose columns are all equal to the unique stable fixed point.

Note that Fredo's transition matrix does not have positive entries, but its square does.
So Fredo has the unique stable fixed point $(0.4, 0.3, 0.3)^T$.

\subsection*{Finding stable fixed points}
Calculating stable fixed points is an important problem in Markov chain analysis.
Suppose a Markov chain has a transition matrix $A$ such that $A^k$ has strictly positive entries for some $k$.
Then the \emph{Perron-Frobenius theorem} says that 1 is the unique eigenvalue of $A$ of largest magnitude, and the corresponding eigenvector is unique.
This means that we can use the power method to find the unique stable fixed point of $A$.

Let us look at what the power method is doing in the example of Fredo the frog.
We will use the 1-norm.

Suppose we know Fredo starts on lily pad 1. Then we begin with the state distribution vector
\[
\x_0 = \begin{bmatrix}
1\\
0\\
0
\end{bmatrix}
\]
because we know for certainty (100\%) that Fredo is in the first state.
The next iteration of the power method is $\x_1=A\x_0/\|A\x_0\|_1$. But since $\|A\x_0\|_1=1$, this is just
\[
\x_1 = A \x_0 = \begin{bmatrix}
0.25\\
0.25\\
0.50
\end{bmatrix},
\]
which is exactly Fredo's state distribution after 1 jump.
After two jumps, Fredo's state distribution is
\[
\x_2 = A \x_1 = A^2 \x_0 = \begin{bmatrix}
0.4375\\
0.3542\\
0.2083
\end{bmatrix},
\]
which is also the second iteration of the power method.
After a large number of jumps, we have
\[
\x_n = A \x_{n-1} = \dots = A^n \x_0 \approx \begin{bmatrix}
0.4\\
0.3\\
0.3
\end{bmatrix}.
\]
Thus, the limiting vector of the power method is exactly the unique stable fixed point of the Markov chain.

\subsection*{A final example}
Consider the Markov chain with transition matrix
\[
A = \begin{pmatrix}
0.5 & 0.3 & 0.4\\
0.2 & 0.2 & 0.3\\
0.3 & 0.5 & 0.3
\end{pmatrix}.
\]

Because all entries of $A$ are positive, the Markov chain has a unique stable fixed point.
We could find this fixed point with the power method as outlined above, or we can do it by computing eigenvalues and eigenvectors in Python, as shown below.
\begin{lstlisting}
>>> from scipy import linalg as la
>>> A = np.array([[.5,.3,.4],[.2,.2,.3],[.3,.5,.3]])
>>> evals, evecs = la.eig(A)
>>> evals
array([ 1.        ,  0.14142136, -0.14142136])
\end{lstlisting}
We are interested in the eigenvalue 1, which is the first one outputted in this case.
The corresponding eigenvector is the first column of \li{evecs}; let us call it \li{x}.
\begin{lstlisting}
>>> x = evecs[:,0]
\end{lstlisting}
Now, the one-norm of \li{x} will probably not be 1. To make it 1, we divide by the one-norm of \li{x}.
\begin{lstlisting}
>>>x = x/np.sum(x)
\end{lstlisting}
Finally, let us check that \li{x} is a stable fixed point of $A$. There are two things to check.
\begin{lstlisting}
# Check Ax = x
>>> np.allclose(A.dot(x),  x)
True

# Check ||x||_1 = 1
>>> np.sum(x)==1
True
\end{lstlisting}

\begin{problem}
Write a function that accepts as input a transition matrix, a vector representing the initial state, and a number of iterations \li{niter}. Your function should
\begin{enumerate}
\item Assume that the input matrix has a unique stable fixed point.
\item Calculate the unique stable fixed point by computing eigenvectors and eigenvalues in Python.
\item Return the current state of the Markov chain after \li{niter} iterations and the stable fixed point.
\end{enumerate}
\label{prob:markov}
\end{problem}

\begin{problem}
Suppose a basketball player's success at shooting free throws can be described with the following Markov chain
\[
A = \begin{pmatrix}.75&.50\\.25&.50\end{pmatrix}
\]
where the first state corresponds to success and the second state to failure. Use the function you wrote in Problem \ref{prob:markov} to answer the following questions.
\begin{enumerate}
\item If the player makes his first free throw, what is the probability that he also makes his third one?
(That is, what is the probability that this system starts in state 1 and is still in state 1 after 3 steps?)
\item What is the player's average free throw percentage?
(This is equal to the success-component of the stable fixed point.)
\end{enumerate}
\label{prob:markov_freethrow}
\end{problem}

\begin{comment}
\begin{problem}
Consider the Markov process given by the transition diagram in Figure \ref{fig:markov2}.
\begin{figure}[H]
\includegraphics[width=\textwidth]{markov2}
\caption{Transition diagram}
\label{fig:markov2}
\end{figure}

\begin{enumerate}
\item Find the transition matrix.
\item If the Markov process is in state 1 initially, find the probability that it is in state 2 after two transitions.
\item Find the stable fixed point if it exists.
\end{enumerate}
\label{prob:markov_stablept}
\end{problem}
\end{comment}