\documentclass[12pt]{article}

% essential math based packages
\usepackage{amsfonts}	% commonly used fonts and symbols in mathematics
\usepackage{amsmath}	% additional set of math tools on top of LaTeX
\usepackage{amssymb}	% common math symbols
\usepackage{amsthm}	% theorem based setup following AMS standards

\usepackage[T1]{fontenc}

\usepackage{multicol}

\usepackage{hyperref}
\hypersetup{
    colorlinks=true,
    linkcolor=blue,
    filecolor=magenta,
    urlcolor=cyan,
}

\begin{document}

% remove indentations for entire document
\setlength\parindent{0pt}

\section{Math enviornments}
% Introduciont to math enviornments
%------------------------------------%
% short cut symbols
\subsection*{Inline elements}
\begin{enumerate}
	\item using \verb!$...$!: $a+b=c$
	\item using \verb!\[...\]!: \(a/b=c\)
	\item using environment \verb!\begin{math}...\end{math}!):
		\begin{math}
		a - b = c
		\end{math}
\end{enumerate}
Takeaway: Although there are said to be issues, or incompatibilities with using \verb!$...$!, most examples and working code use this shorthand for inline math. If for whatever reason you do find issues, then use \verb!\(...\)!.

\subsection*{Blocked elements}
\begin{enumerate}
	\item using \verb!$$...$$!: $$ \frac{a}{b}=c $$
	\item using \verb!\[...\]!: \[ \int^a_b = c\]
	\item using enviornment \verb!\begin{displaymath}...\end{displaymath}!:
		\begin{displaymath}
		\dfrac{\partial a}{\partial	b} = c
		\end{displaymath}
\end{enumerate}
Takeaway: Much like for inline math, \verb!$$...$$! is commonly used. However, if you do find issues (which rarely does), then use \verb!\[...\]!.

\subsection*{Alignments \& numberings}
\begin{enumerate}
	\item Numbered equations
	\begin{equation}
		\text{KE} = 1/2mv^2
	\end{equation}

	\item No numbered equations ({\*}trick, similar to section numberings)
	\begin{equation*}
		\text{PE} = \int_{\text{ref}}^{x} F \operatorname{d}\!\overrightarrow{x}
	\end{equation*}

	\item Numbered equations (not aligned)
	\begin{gather}
		\exp^{ix} = \cos{x} + i\sin(x) \\
		\exp^{i\pi} + 1 = 0
	\end{gather}

	\item Numbered and aligned
	\begin{align}
		\nabla \cdot \vec{D} &= \rho_v \\
		\nabla \cdot \vec{B} &= 0 \\
		\nabla \times \vec{E} &= - \frac{\partial B}{\partial t} \\
		\nabla \times \vec{B} &= \mu_{0}\vec{J} +
		\mu_{0}\epsilon_{0}\frac{\partial E}{\partial t}
	\end{align}

	\item Controlling numbering and alignment
	\begin{align}
		&\nabla \cdot \vec{D} = \rho_v \nonumber \\
		&\nabla \cdot \vec{B} = 0 \\
		&\nabla \times \vec{E} = - \frac{\partial B}{\partial t} \\
		&\nabla \times \vec{B} = \mu_{0}\vec{J} +
		\mu_{0}\epsilon_{0}\frac{\partial E}{\partial t} \nonumber
	\end{align}
\end{enumerate}
Takeaway: All environments can use the \* trick to suppress numbering, or \verb!\nonumber! can do this specifically per line. Although not shown in this demonstration, if equations get too long, or multiple equations should be given 1 equation number (such as an \textit{if/else} statement), use the \verb!\begin{split}! or \verb!\begin{multiline}! environments.

%------------------------------------%

\section{Symbols}

\subsection*{Greek symbols}
Note that greek symbols that can be represented by english letters such as \verb!\Alpha! and \verb!\Chi! do not exists, as their symbols $A$ and $X$ are indistinguishable from using letters \verb!$A$! and \verb!$X$!. However, some packages override this behavior, so please check what math packages you import.

$$\alpha, A, \beta, B, \gamma, \Gamma, \delta, \Delta ...\, \mu, \nu $$

\subsection*{Equation symbols}
You have control over all types of symbols relevant to mathematical, and even graphical representation. To get an extensive list please look \href{https://en.wikibooks.org/wiki/LaTeX/Mathematics#List_of_mathematical_symbols}{here}.

\subsection*{Formatting mathematical symbols}
Some equations need more than a simple definition or symbol. Some symbols can be compounded to make more complex statements. For example
$$
\overrightarrow{\sum_{i=\iiint}^{j={\widehat{AAA}}}}
$$
A more comprehensive discussion on this topic, and how to customize the look can be found \href{https://en.wikibooks.org/wiki/LaTeX/Mathematics#Formatting_mathematics_symbols}{here}.

\section{Spacing}
Horizontal spacing is dictated by the document class font size (e.g. 11pt, 12pt, etc.) and is measured by \textit{em} which is roughly proportional to the horizontal width of a capital M. To artificially create 1 em width is to use \verb!\quad!. See: \\

\indent A{\quad}B\\
\indent AMB (... a little bit more than M)\\

Knowing this, there are many commands such as \verb!\,! \verb!\:! that create fractions of \verb!\quad!. The variety of commands for horizontal spacing in normal and math mode can be found \href{https://en.wikibooks.org/wiki/LaTeX/Mathematics#Controlling_horizontal_spacing}{here}. There is also a discussion on which \href{https://tex.stackexchange.com/questions/41476/lengths-and-when-to-use-them}{spacing is appropriate}.

\end{document}

test-files/3 - include math and symbols/math-symbols.tex


\chapter{Assignment}
\section{MCQ}
\begin{enumerate}
	\item One of the possible solutions of the differential equation
	$y \sqrt{\left(1+x^{2}\right)} d y+x \sqrt{\left(1+y^{2}\right)} d x=0$ (where $c$ is some constant) is
	 \begin{tasks}(2)
		\task[\textbf{a.}]$\left(\sqrt{1+y^{2}}\right)\left(\sqrt{1+x^{2}}\right)=c$
		\task[\textbf{b.}]$\frac{\sqrt{1+y^{2}}}{\sqrt{1+x^{2}}}=c$
		\task[\textbf{c.}]$\sqrt{1+y^{2}}+\sqrt{1+x^{2}}=c$
		\task[\textbf{d.}]  $\sqrt{1+y^{2}}-\sqrt{1+x^{2}}=c$
	\end{tasks}
	\item The solution of the differential equation $x \frac{d y}{d x}+\cot y=0$, subject to the initial condition $y=\frac{\pi}{4}$ at $x=\sqrt{2}$ is
	 \begin{tasks}(2)
		\task[\textbf{a.}]$x=2 \cos y$
		\task[\textbf{b.}]$x=2 \sec y$
		\task[\textbf{c.}]$x=2 \sin y$
		\task[\textbf{d.}]  $x=2 \operatorname{cosecy}$
	\end{tasks}
	\item The solutions to the differential equation $\frac{d y}{d x}=-\frac{x}{y+1}$ are a family of
	 \begin{tasks}(2)
		\task[\textbf{a.}]Circles with different radii
		\task[\textbf{b.}]Circles with different centres
		\task[\textbf{c.}]Straight lines with different slopes
		\task[\textbf{d.}] Straight lines with different intercepts on the $y$-axis
	\end{tasks}
	\item The solution of the differential equation $x y \frac{d y}{d x}=3 y^{2}+x^{2}$ with the initial condition $y=2$ when $x=1$ is
	 \begin{tasks}(2)
		\task[\textbf{a.}]$2 y^{2}+x^{2}=9 x^{6}$
		\task[\textbf{b.}] $y^{2}+2 x^{2}=9 x^{6}$
		\task[\textbf{c.}] $2 y^{2}+x^{2}=8 x^{6}$
		\task[\textbf{d.}] $y^{2}+2 x^{2}=8 x^{6}$
	\end{tasks}
	\item The solution of the differential equation
	$(x+2 y)(d x-d y)=d x+d y$ (where $a$ is some constant) is
	 \begin{tasks}(2)
		\task[\textbf{a.}]$3 x+3 y+a=2 \log (3 x+6 y-1)$
		\task[\textbf{b.}] $3 x-3 y+a=2 \log (3 x+6 y-1)$
		\task[\textbf{c.}]$3 x-3 y+a=2 \log (3 x-6 y-1)$
		\task[\textbf{d.}] $3 x+3 y+a=2 \log (3 x-6 y-1)$
	\end{tasks}
	\item For the differential equation $\frac{d y}{d x}+3 y=e^{2 x}$, the possible solution is:
	 \begin{tasks}(2)
		\task[\textbf{a.}]$y=c_{1} e^{2 x}+c_{2} e^{3 x}$
		\task[\textbf{b.}]$y=c_{1} e^{-2 x}+c_{2} e^{3 x}$
		\task[\textbf{c.}]$y=c_{1} e^{2 x}+c_{2} e^{-3 x}$
		\task[\textbf{d.}] $y=c_{1} e^{-2 x}+c_{2} e^{-3 x}$
	\end{tasks}
	\item The solution of the differential equation for $y: x \frac{d y}{d x}+y=x^{4}$, subject to the initial condition $y=1$ at $x=1$ is
	 \begin{tasks}(2)
		\task[\textbf{a.}]$y=5 x^{4}-4$
		\task[\textbf{b.}]$y=\frac{x^{4}}{5}+\frac{4 x}{5}$
		\task[\textbf{c.}] $y=\frac{x^{4}}{5}+\frac{1}{5 x}$
		\task[\textbf{d.}] $y=\frac{x^{4}}{5}+\frac{4}{5 x}$
	\end{tasks}
	\item Which one of the following curves gives the solution of the differential equation $K_{1} \frac{d x}{d t}+K_{2} x=K_{3}$, where $K_{1}, K_{2}$ and $K_{3}$ are positive constants with initial conditions $x=0$ at $t=0$ ?
	 \begin{tasks}(2)
		\task[\textbf{a.}]
		\begin{figure}[H]
			\centering
			\includegraphics[height=2.5cm,width=4cm]{DE -assignment-04}
		\end{figure}
		\task[\textbf{b.}]
		\begin{figure}[H]
			\centering
			\includegraphics[height=2.5cm,width=4cm]{DE -assignment-01}
		\end{figure}
		\task[\textbf{c.}]
			\begin{figure}[H]
			\centering
			\includegraphics[height=2.5cm,width=4cm]{DE -assignment-02}
		\end{figure}
		\task[\textbf{d.}]
			\begin{figure}[H]
			\centering
			\includegraphics[height=2.5cm,width=4cm]{DE -assignment-03}
		\end{figure}
	\end{tasks}
	\item The solution of the differential equation $t \frac{d y}{d t}+y \log y=t y e^{t}$
	 \begin{tasks}(2)
		\task[\textbf{a.}]$\log y=(t+1) e^{t}+c$
		\task[\textbf{b.}]$\log y=(t-1) e^{t}+c$
		\task[\textbf{c.}]$t \log y=(t+1) e^{t}+c$
		\task[\textbf{d.}] $t \log y=(t-1) e^{t}+c$
	\end{tasks}
	\item The solution of the differential equation
	$\left(1+y^{2}\right) d x=\left(\tan ^{-1} y-x\right) d y$ (where $c$ is some constant) is
	 \begin{tasks}(2)
		\task[\textbf{a.}] $x=\left(\tan ^{-1} y+2\right)+c e^{-\tan ^{-1} y}$
		\task[\textbf{b.}] $x=\left(\tan ^{-1} y-2\right)+c e^{-\tan ^{-1} y}$
		\task[\textbf{c.}] $x=\left(\tan ^{-1} y+1\right)+c e^{-\tan ^{-1} y}$
		\task[\textbf{d.}]  $x=\left(\tan ^{-1} y-1\right)+c e^{-\tan ^{-1}y}$
	\end{tasks}
	\item The solution of the differential equation
	$y \log y \frac{d x}{d y}+x-\log y=0$ (where $c$ is some constant) is
	 \begin{tasks}(2)
		\task[\textbf{a.}]$x \log y=\frac{1}{2}(\log y)^{2}+c$
		\task[\textbf{b.}] $x \log y=2(\log y)^{2}+c$
		\task[\textbf{c.}]$x \log y=\frac{1}{2}(\log y)^{3}+c$
		\task[\textbf{d.}]  $y \log x=\frac{1}{2}(\log y)^{2}+c$
	\end{tasks}
	\item The solution of the differential equation
	$\left(e^{y}+2\right) \sin x d x-e^{y} \cos x d y=0$ (where $c$ is some constant) is
	 \begin{tasks}(2)
		\task[\textbf{a.}]$\left(e^{y}+2\right) \sin x=c$
		\task[\textbf{b.}] $\left(e^{y}+2\right) \cos x=c$
		\task[\textbf{c.}] $\left(e^{y}+2\right) \operatorname{cosec} x=c$
		\task[\textbf{d.}] $\left(e^{y}+2\right) \sec x=c$
	\end{tasks}
	\item For the differential equation $\left(y^{4}+2 y\right) d x+\left(x y^{3}+2 y^{4}-4 x\right) d y=0$ one of the possible solution (where $C$ is some constant) is:
	 \begin{tasks}(2)
		\task[\textbf{a.}]$x+\frac{2}{y^{2}}+y^{2}=c$
		\task[\textbf{b.}] $x\left(y+\frac{2}{y^{2}}\right)=c$
		\task[\textbf{c.}]$x\left(y+\frac{2}{y^{2}}\right)+y^{2}=c$
		\task[\textbf{d.}] $x\left(1+\frac{2}{y^{2}}\right)+x^{2}=c$
	\end{tasks}
	\item The solution of the differential equation for $y(t): \frac{d^{2} y}{d t^{2}}-\frac{3 d y}{d t}+2 y=e^{3 t}$ is
	 \begin{tasks}(2)
		\task[\textbf{a.}] $c_{1} e^{t}+c_{2} e^{2 t}+\frac{1}{2} e^{3 t}$
		\task[\textbf{b.}]$c_{1} e^{-t}+c_{2} e^{2 t}+\frac{1}{2} e^{3 t}$
		\task[\textbf{c.}]$c_{1} e^{t}+c_{2} e^{2 t}+e^{3 t}$
		\task[\textbf{d.}] $c_{1} e^{-t}+c_{2} e^{-2 t}+e^{3 t}$
	\end{tasks}
	\item The solution of the differential equation for $\frac{d^{2} y}{d x^{2}}+2 \frac{d y}{d x}+101 y=10.4 e^{x}$, subject to the initial conditions $y(0)=1.1$ and $\left.\frac{d y}{d x}\right|_{x=0}=-0.9$, is
	 \begin{tasks}(2)
		\task[\textbf{a.}]$e^{-x} \cos 10 x+0.1 e^{x}$
		\task[\textbf{b.}]$e^{-x} \sin 10 x+0.1 e^{x}$
		\task[\textbf{c.}]$e^{x} \cos 10 x+0.1 e^{-x}$
		\task[\textbf{d.}] $e^{x} \sin 10 x+0.1 e^{-x}$
	\end{tasks}
	\item The solution of the differential equation $\frac{d^{2} y}{d t^{2}}+4 y=\cos 2 t$ is:
	 \begin{tasks}(2)
		\task[\textbf{a.}] $A \cos 2 t+B \sin 2 t$
		\task[\textbf{b.}] $\frac{t \sin 2 t}{4}$
		\task[\textbf{c.}]$A \cos 2 t+B \sin 2 t+\frac{t \sin 2 t}{4}$
		\task[\textbf{d.}] $A \cos 2 t+B \sin 2 t+\frac{t^{2} \sin 2 t}{4}$
	\end{tasks}
	\item The solution of the differential equation for $y(t): \frac{d^{2} y}{d t^{2}}-y=2 \cosh (t)$, subject to the initial conditions $y(0)=0$ and $\left.\frac{d y}{d t}\right|_{t=0}=0$, is
	 \begin{tasks}(2)
		\task[\textbf{a.}]$\frac{1}{2} \cosh (t)+t \sin h(t)$
		\task[\textbf{b.}]$-\sin h(t)+t \cosh (t)$
		\task[\textbf{c.}] $t \cosh (t)$
		\task[\textbf{d.}] $t \sin h(t)$
	\end{tasks}
	\item The solution of the differential equation for $y(t): \frac{d^{2} y}{d t^{2}}-y=2 \sinh (t)$, subject to the initial conditions $y(0)=0$ and $\left.\frac{d y}{d t}\right|_{t=0}=0$, is
	 \begin{tasks}(2)
		\task[\textbf{a.}] $\frac{1}{2} \cosh (t)+t \sin h(t)$
		\task[\textbf{b.}]$-\sin h(t)+t \cosh (t)$
		\task[\textbf{c.}]$t \cosh (t)$
		\task[\textbf{d.}] $t \sin h(t)$
	\end{tasks}
\end{enumerate}
\section{NAT}
\begin{enumerate}
	\item The solution of the differential equation $\frac{d x}{d t}=x^{2}$ with the initial condition $x(0)=1$ will blow up as $t$ tends to..........
	\item The value of $\lambda \ldots .$ for which the differential equation $\left(x y^{2}+\lambda x^{2} y\right) d x+(x+y) x^{2} d y=0$ is exact
	\item The particular integral of the differential equation $\left(D^{2}+D+1\right) y=\cos 2 x$ is $\frac{1}{\alpha}(2 \sin 2 x-3 \cos 2 x) .$ Then the value of $\alpha$ is.....
	\item The particular integral of the differential equation $\frac{d^{2} y}{d x^{2}}+2 \frac{d y}{d x}+2 y=\sin x$ is $\frac{1}{5}(\sin x-\alpha \cos x) .$ Then the value of $\alpha$ is........
	\item The particular integral of the differential equation $\left(D^{2}-5 D+6\right) y=e^{t} \cos 2 t$ is $-\frac{e^{t}}{20}(\alpha \sin 2 t+\cos 2 t)$. Then the value of $\alpha$ is........
	\item The particular integral of the differential equation $\left(D^{2}-4 D+4\right) y=x^{3} e^{2 x}$ is $e^{2 x} \frac{x^{\alpha}}{20}$. Then the value of $\alpha$ is........
	\item The particular integral of the differential equation $\left(D^{2}+5 D+4\right) y=3-2 x$ is $\frac{1}{8}[\alpha-4 x]$.
	Then the value of $\alpha$ is........
	\item The maximum value of the solution $y(t) \ldots \ldots \ldots . .$ of the differential equation for $y(t)+\ddot{y}(t)=0$, subject to the initial conditions $\dot{y}(0)=1$ and $y(0)=1$ for $t \geq 0$, is
	\item If the characteristic equation $\frac{d^{2} y}{d x^{2}}+2 \alpha \frac{d x}{d t}+y=0$ has two equal roots, then the value of $\alpha \ldots \ldots \ldots$ is
	\item Consider the differential equation $\frac{d^{2} x}{d t^{2}}+3 \frac{d x}{d t}+2 x=0 .$ Given $x(0)=20$ and $x(1)=\frac{10}{e}$, where $e=2.718$, the value of $x(2)$ is
\end{enumerate}

CSIR- Mathematical Physics/chapter/Assignments/Assignment- Differential Equations.tex


\section{A basic machine learning problem: image classification}
Classification problem is a very important part of machine learning. Its goal is to determine which class a new data belongs to based on a set of training data whose class is known. For example, in mail management, classifying an e-mail as ``spam" or ``non-spam" is a typical binary classification problem, and the classification of credit rating of credit card customers by banks belong to multiple classification problems. More specifically, in order to understand image classification problem, let us pose the following question:

\begin{itemize}
	\item Given a set of images of cat, dog and rabbit, how does a machine classify the three different classes?
\end{itemize}

\begin{figure}[H]
	\begin{center}
		\includegraphics[width=.4\textwidth, height=.2\textheight]{figures/cat-dog-1.png}
	\end{center}
\end{figure}

\break

First of all, we introduce how an image is represented on computer. Mathematically, {gray-scale image can be considered as a matrix  in $ \mathbb{R}^{n_0\times n_0}$ shown below, where the left one is the image from human vision and the right one is the matrix represented in computer. Each entry in this $ \mathbb{R}^{n_0\times n_0}$ corresponds to the value of a pixel belong to $[0,255]$.
	\begin{center}
		\includegraphics[width=.4\textwidth, height=.2\textheight]{6DL/figures/gray-1.png}
	\end{center}

	A color image can be taken as 3D tensor (matrix with $3$ channels (RGB)) in $ \mathbb{R}^{n_0\times n_0 \times 3}$
	\begin{center}
		\includegraphics[width=.4\textwidth, height=.2\textheight]{6DL/figures/corlor-1.png}
	\end{center}
	\break

	Then, let us think about the image classification problem of cat, dog and rabbit. Each image is a big vector of pixel values, for example
	$$ d=1280\times720\times 3  (\text{width} \times \text{height} \times \text{RGB channel}) \approx 3\text{M},$$
	which can be considered as a point $x \in \mathbb{R}^d$. The question becomes: given 3 different sets of points (cat, dog and rabbit) in $\mathbb{R}^d$, how does a machine classify them?
	\begin{figure}[H]
		\begin{center}
			\includegraphics[width=.3\textwidth, height=.3\textwidth]{figures/cat-dog-2.png}   \quad \quad  \quad \quad
			\includegraphics[width=.3\textwidth, height=.3\textwidth]{figures/cat-dog-3.png}
		\end{center}
	\end{figure}

	\noindent To answer this question, we consider a mathematical problem: Find $f(\cdot; \bm \theta): \mathbb{R}^d \to \mathbb{R}^3$ such that:
	$$
	f(\includegraphics[width=.07\textwidth]{figures/cat.png}; \Theta)
	\approx \begin{pmatrix}
	1\\ 0 \\ 0
	\end{pmatrix}
	,\quad
	f(\includegraphics[width=.07\textwidth]{figures/dog.png}; \Theta)
	\approx \begin{pmatrix}
	0\\ 1 \\ 0
	\end{pmatrix}
	,\quad
	f(\includegraphics[width=.07\textwidth]{figures/rabbit.png};
	\Theta) \approx
	\begin{pmatrix}
	0\\ 0 \\ 1
	\end{pmatrix}
	.
	$$
	$f(\cdot; \bm \theta)$ maps a given image to a 3-dimensional vector, which is a probability distribution  $\begin{pmatrix}
	p_1\\ p_2 \\ p_3
	\end{pmatrix}$
	, where $p_1, p_2, p_3$ are probabilities of the given image being cat, dog, rabbit, respectively. For example

	$$
	f(\includegraphics[width=.07\textwidth]{figures/cat.png}; \bm \theta)
	=
	\begin{pmatrix}
	0.7\\ 0.2 \\ 0.1
	\end{pmatrix}
	\quad \Longrightarrow \quad \includegraphics[width=.07\textwidth]{figures/cat.png}= {\rm cat}.
	$$
	$f$ is a classifier that can be used to classify what a given image is.

\break
\section{Some popular data sets in image classification}
In this subsection, we will introduce some popular and standard data sets
in image classification. The most popular four data sets, MNIST, CIFAR-10, CIFAR-100 and ImageNet are shown in Table \ref{popular_dataset}.

\begin{table}[H]
	\centering
	\begin{tabular}{|c|c|c|c|c|c|}
		\hline
		% after \\: \hline or \cline{col1-col2} \cline{col3-col4} ...
		dataset &  training (N) & test (M)   & classes (k) & channels (c)& input size (d)\\\hline
		MNIST	&	60K	&	10K	&	10	& Greyscale & 28*28  \\\hline
		CIFAR-10	&	50K 	&	10K	&	10 & RGB & 32*32   \\\hline
		CIFAR-100	&	50K 	&	10K	&	100 & RGB & 32*32   \\\hline
		ImageNet	&	1.2M 	&	50K	&	1000 &	RGB & 224*224  \\\hline
	\end{tabular}
	\caption{Basic descriptions about popular datasets }
	\label{popular_dataset}
\end{table}

\break
\subsection{MNIST (Modified National Institute of Standards and Technology Database)}
MNIST\cite{lecun1998mnist} is a database for handwritten digits. It is a simple database for people who want to try learning techniques and pattern recognition methods on real-world data while spending minimal efforts on preprocessing and formatting. In order to use MNIST for the classification problem, the following setup is often used:
\begin{itemize}
	\item Training set : $N = 60,000$;
	\item Test set : $M = 10,000$;
	\item Image size : $d =28*28*1=784$;
	\item Classes: $ k = 10$;
\end{itemize}
\begin{figure}[H]
	\begin{center}
		\includegraphics[height=.2\textheight]{mnist_short.png}
		\caption{Some images in MNIST.}
	\end{center}
\end{figure}
The following example shows  an image of handwritten digits is represented mathematically
\break
$$
x=\adjustbox{valign=c}{\includegraphics[height=.03\textheight]{mnist2_1.jpg}}
=
\begin{pmatrix}
  x_1\\
x_2\\
\vdots\\
x_{784}
\end{pmatrix}
\in \mathbb R^{784}.
$$
MNIST has 10 classes denoted by $\{A_k\}_{k=1}^{10}$ as shown in the following, where $A_{k}$ is the set of handwritten digits $k$ for $k=1,2,3,...,9$ and $A_{10}$ is the set of handwritten digits $0$
\begin{equation}
  \label{A2}
A_2=
\left\{
\adjustbox{valign=c}{\includegraphics[height=.03\textheight]{mnist2_1.jpg}},
\adjustbox{valign=c}{\includegraphics[height=.03\textheight]{mnist2_2.jpg}},\cdots
%\adjustbox{valign=c}{\includegraphics[height=.03\textheight]{mnist2_3.jpg}}, \cdots
\right\},~
%\subset \mathbb R^{784}
A_{9}=
\left\{
\adjustbox{valign=c}{\includegraphics[height=.03\textheight]{mnist9_1.jpg}},
\adjustbox{valign=c}{\includegraphics[height=.03\textheight]{mnist9_2.jpg}},\cdots
%\adjustbox{valign=c}{\includegraphics[height=.03\textheight]{mnist9_3.jpg}}, \cdots
\right\},~
A_{10}=
\left\{
\adjustbox{valign=c}{\includegraphics[height=.03\textheight]{mnist0_1.jpg}},
\adjustbox{valign=c}{\includegraphics[height=.03\textheight]{mnist0_2.jpg}}, \cdots
%\adjustbox{valign=c}{\includegraphics[height=.03\textheight]{mnist0_3.jpg}}, \cdots
\right\}
\subset \mathbb R^{784}.
\end{equation}

\break
\subsection{CIFAR}
\paragraph{CIFAR-10}
CIFAR-10\cite{krizhevsky2009learning} is a set of images that can be used to teach a computer how to recognize objects. It contains 60,000 32x32 color images in 10 different classes,  with 6000 images per class. The 10 different classes represent airplanes, cars, birds, cats, deer, dogs, frogs, horses, ships, and trucks. In order to use CIFAR-10 for the classification problem, people usually use the following setup:
\begin{itemize}
	\item Training set : $N = 50,000$
	\item Test set : $M = 10,000$
	\item Image size : $d  = 32*32*3$
	\item Classes: $k = 10$
\end{itemize}
\begin{figure}[H]
	\begin{center}
		\includegraphics[height=.26\textheight]{cifar10.jpg}
		\includegraphics[height=0.26\textheight]{cifar100.png}
		\label{Fig: CIFAR-10}
		\caption{Left: Some images in CIFAR-10. Right: Some images in CIFAR-100.}
	\end{center}
\end{figure}

\break

\paragraph{CIFAR-100}
CIFAR-100\cite{krizhevsky2009learning} is just like the CIFAR-10, except it has 100 classes containing 600 images each. There are 500 training images and 100 testing images per class. The 100 classes in the CIFAR-100 are grouped into 20 superclasses and each superclass has 5 classes. Each image comes with a ``fine" label (the class to which it belongs) and a ``coarse" label (the superclass to which it belongs). In order to use CIFAR-100 for the classification problem, people usually use the following setup:

\begin{itemize}
	\item Training set : $N = 50,000$
	\item Test set : $M = 10,000$
	\item Image size : $d = 32*32*3$
	\item Classes: $k = 100$
	\end{itemize}

\subsection{ImageNet}
The ImageNet\cite{deng2009imagenet} project is a large visual database designed for use in visual object recognition software research. More than 1 million images have been hand labeled to indicate what objects are pictured. It is organized according to the WordNet hierarchy (currently only the nouns), in which each node of the hierarchy is depicted by hundreds and thousands of images.
Since 2010, the ImageNet runs an annual software contest, the ImageNet Large Scale Visual Recognition Challenge (ILSVRC), where software programs compete to correctly classify and detect objects and scenes. In order to use ImageNet for the classification problem, we list the setup of ILSVRC2012 in the following
\begin{itemize}
	\item Training set : $N = 1,200,000$
	\item Test set : $M = 50,000$
	\item Image size : $d = 224*224*3$
	\item Classes: $k = 1,000$
\end{itemize}
\begin{figure}[H]
	\begin{center}
\includegraphics[height=0.22\textheight]{imagenet-example.png}
		\caption{Some images in ImageNet.}
	\end{center}
\end{figure}
\break

\endinput


\section{Discretized Hamiltonian}\label{sec:hamiltonian}

We consider a cubic finite volume (FV) of linear size $L$ with periodic boundary conditions and lattice spacing $\epsilon$ so that $N=L/\epsilon$ is an even integer that counts the number of sites in one spatial direction.

The contact interaction Hamiltonian \eqref{hamiltonian} is implemented on the lattice as an entirely local operator, vanishing everywhere except at the origin where it is of strength $C$---the interaction is not smeared.
The Hamiltonian is given by
\begin{equation}
    \left\langle \vec{r}' \middle| H \middle| \vec{r} \right\rangle
    \rightarrow
    H_{\vec{r}',\vec{r}}^\dispersion
    =
    \frac{1}{2\mu} K_{\vec{r}',\vec{r}}^\dispersion + \frac{1}{\epsilon^D} C^\dispersion \delta_{\vec{r}',\vec{r}} \delta_{\vec{r},\vec{0}}
\end{equation}
where $K$ is a discretized Laplacian, implementing the momentum squared.
The $\dispersion$ symbol indicates that quantities depend on the lattice spacing $\epsilon$ and the explicit implementation of discretization effects like derivatives.

To ensure we control the discretization effects in generality, we study a variety of kinetic operators $K_{xy}^\dispersion$.
An often-used set of finite-difference kinetic operators are constructed from the one-dimensional finite-difference Laplacian that reaches $\nstep$ nearest neighbors,
\begin{equation}
    \Laplacian^\dispersion_{r'r} = \frac{1}{\epsilon^2}\sum_{s=-\nstep}^{\nstep} \gamma_{\abs{s}}^{(\nstep)} \delta_{r',r+\epsilon s}^{(L)}
\end{equation}
where the $(L)$ index of the Kronecker delta indicates that the spatial indices are understood modulo the periodic boundary conditions of the lattice.
In $D$ dimensions we simply take on-axis finite differences, so that the Laplacian is a $(1+2\nstep D)$-point stencil
\begin{equation}\label{eq:kinetic}
    K_{\vec{r}',\vec{r}}^\dispersion(\epsilon)
    =
    - \sum_{d=1}^D \Laplacian_{r_d'r_d^{}}^\dispersion
    \, .
\end{equation}

In the Fourier transformed space, momentum space, the one-dimensional Laplacian may be written
\begin{equation}
    \label{eq:laplacian}
    -\Laplacian_{r'r^{}}^\dispersion
    \overset{\text{F.T.}}{\longleftrightarrow}
    \Laplacian^\dispersion_{p'p}
    =
    \frac{1}{\epsilon^2}
    \delta_{p'p}
    \sum_{s=0}^{\nstep} \gamma_s^{(\nstep)} \cos(s p \epsilon)
    \, , \qquad p = \frac{2 \pi}{L} n \,
\end{equation}
where $n$ is an integer.
In $D$ dimensions we just sum the same expression over the different components of momentum.
Note that this is a specialization, in the sense that it contains no off-axis differencing (in position space) or products of different components (in momentum space).
However, since the numerical formalism we will describe is valid for every $\nstep$, we believe it holds for every possible kinetic operator.

\begin{figure}
    \input{figure/dispersion.pgf}
    \caption{We show the continuum dispersion relation of energy as a function of momentum for different one-dimensional $\nstep$ derivatives.  For a finite number of lattice points $N$, the allowed momenta are evenly-spaced in steps of $2\pi/N$.
    As additional steps are incorporated into the finite difference, the dispersion relation more and more faithfully reproduces the desired $p^2$~behavior of $\nstep=\infty$.
    }
    \label{fig:dispersion relation}
\end{figure}

The coefficients $\gamma_{s}^{(\nstep)}$ are determined by requiring the dispersion relation be as quadratic as possible,
\begin{equation}
    \label{eq:gamma determination}
    \Laplacian^\dispersion_{p'p}
    \overset{!}{=}
    \delta_{p'p} \;
    p^2 \left[
        1 + \order{(\epsilon p)^{2\nstep}}
    \right].
\end{equation}
Additionally, we study a nonlocal operator with $\nstep=\infty$ which, in momentum space, can be implemented by multiplying by $p^2$ directly,
\begin{equation}
    \lim\limits_{n_s \to \infty}
    \Laplacian^\dispersion_{p'p}
    =
    \delta_{p'p} p^2,
\end{equation}
including at the edge of the Brillouin zone, the Laplacian implementation of the ungauged SLAC derivative.
Including the edge of the Brillouin zone does not introduce a discontinuity at the boundary, nor does including the corners pose any problem.
In addition to the $\nstep=\infty$ operator, we also call this kinetic operator the \emph{exact-$p^2$} operator.
The resulting dispersion relations are presented in \Figref{dispersion relation} for a variety of $\nstep$s and in \Appref{coefficients} we collect the required $\gamma$ coefficients.
In \Refs{Endres:2011er,Endres:2012cw} the exact dispersion relation is cut off by a LEGO sphere in momentum space (see equation (6) and the discussion after (9) in those references, respectively).
The formalism we develop here takes into account the implemented dispersion relation and thus is in principle extendable to these cut off operators, though the analytic results are harder to extract and we do not discuss such operators further.

The Hamiltonian in momentum space reads
\begin{equation}
    \label{eq:p space hamiltonian}
    \left\langle \vec{p}' \middle| H \middle| \vec{p} \right\rangle
    \rightarrow
    H_{\vec{p}',\vec{p}}^\dispersion
    =
    \frac{4 \pi^2}{2\mu L^2} \tilde K_{\vec n \vec n}^{N}
    +\frac{1}{L^D}C^\dispersion
\end{equation}
where $\vec{p} = 2\pi \vec{n}/L$ for a $D$-plet of integers $\vec{n} \in (-N/2, +N/2]^D$, and the coefficients $\gamma_{s}^{(\nstep)}$ are determined as described above.
Furthermore, we replaced the lattice-spacing-dependent kinetic Hamiltonian with the $N$-dependent
\begin{equation}\label{eq:normalized-kinetic-hamitlonian}
	\tilde K_{\vec n \vec n}^{N}
	= \frac{L^2}{4\pi^2} K_{\vec p \vec p}^{\dispersion} \bigg|_{\vec p=\frac{2\pi \vec n}{L}}
	= \frac{N^2}{4\pi^2}
    \sum_{i=1}^{D}\sum_{s=0}^{\nstep} \gamma_s^{(\nstep)} \cos\left(\frac{2 \pi s n_i}{N}\right)
\end{equation}
which goes to $n^2$ in the continuum limit $N\goesto\infty$.

Although the non-interacting energy levels are no longer proportional to $n^2$ at generic \nstep, $n^2$ is still a useful classification for states, as long as it is understood simply as the magnitude of the lattice momentum---describing shells---rather than as a proxy for energy.

\subsection{Reduction to  \texorpdfstring{\Aoneg}{A-one-g}}

Because we are interested in contact interactions, infinite-volume arguments suggest that only the s-wave will feel the interaction; such arguments translate to the lattice relatively cleanly.
Since the s-wave is most like \Aoneg we will focus on the spectrum in that irreducible representation of the cubic symmetry group $O_h$ in three dimensions, of the symmetry group of the square $D_{4h}$ in two dimensions, or $Z_2$ in one dimension, where an \Aoneg restriction amounts to focusing on parity-even states.

With a projection operator to the \Aoneg sector $P_{\Aoneg}$ we can raise the energy of all the other states an arbitrary amount $\alpha$ by supplementing the Hamiltonian
\begin{equation}
    H(\alpha) = H + \alpha (\one - P_{\Aoneg}) \, ,
\end{equation}
Because $P_{\Aoneg}$ commutes with $H$, $H$ and $H(\alpha)$ have the same spectrum within the $\Aoneg$ irrep.
If $\alpha$ is much larger than the expected energies of the Hamiltonian, the \Aoneg states remain low-lying and all other states are shifted to much higher energies.
Then, exact diagonalization for low-lying eigenvalues of $H(\alpha)$ provides an easier extraction of \Aoneg eigenenergies.

Because of the simplicity of \Aoneg we can also easily construct the Hamiltonian directly in that sector (a construction for general $O_h$ irreps was recently given in \Ref{Li:2019qvh}).
In momentum space we can label plane wave states by a vector on integers $\vec{n}$.
In the \Aoneg basis we can use one plane wave label and understand that we intend a normalized unweighted average of every plane wave state.
That is,
\begin{equation}
    \ket{\Aoneg\; \vec{n}} = \frac{1}{\sqrt{\normalization}} \sum_{g \in O_h} \ket{ g\vec{n}}
\end{equation}
where $g$ is an element of the group $O_h$, the sum is over all inequivalent states, and $\normalization$ the normalization.
When $\vec{n}$ is large we should be careful not to double-count states that live right on the edge of the Brillouin zone.
The states may be labeled by symmetry-inequivalent vectors with components all as large as $N/2$.
As a simple example, in three dimensions the $N/2(1,1,1)$ plane wave state in one corner of the Brillouin zone is invariant under all the $O_h$ operations modulo periodicity in momentum space, so $\normalization=1$ for that state.

Formulated in this basis, the kinetic energy operator remains diagonal and proportional to $n^2$ when $N\goesto\infty$.
Reading off the momentum-state potential matrix element from \eqref{p space hamiltonian}, the contact interaction is given by
\begin{equation}
    \braMket{\Aoneg\; \vec{n}'}{V}{\Aoneg\; \vec{n}}
    =
    \sum_{g'g \in O_h}
        \frac{1}{\sqrt{\normalization'}\sqrt{\normalization}} \braMket{g'\vec{n}'}{V}{g\vec{n}}
    =
    \frac{C^\dispersion}{L^D} \sqrt{\normalization'\normalization},
\end{equation}
so that every \Aoneg state talks to every other.
So, the Hamiltonian is in this sector is
\begin{equation}
    H_{\vec{n}'\vec{n}}^\dispersion = \frac{4 \pi^2}{2\mu L^2} \tilde K_{\vec n \vec n}^{N} + \frac{C^\dispersion}{L^D} \sqrt{\normalization'\normalization}
\end{equation}
and we divide by $4\pi^2/\mu L^2$ to make everything dimensionless.

We have implemented both this \Aoneg-only Hamiltonian and the general Hamiltonian with an energy penalty for non-\Aoneg states and verified that the spectra match where expected to as much precision as desired.

For a given $N$ multiple momenta inequivalent under the $O_h$ symmetry may have the same $n^2$.
For example, when $N\geq5$ there are two $n^2=9$ shells corresponding to $n=(2,2,1)$ and $n=(3,0,0)$, which lives on the edge of the Brillouin zone for $N=5$.
When $\nstep=\infty$ the corresponding non-interacting eigenstates are degenerate, while with imperfect dispersion relations the degeneracy is, generically, lifted.
For the contact interaction and $\nstep=\infty$, one linear combination of these \Aoneg states overlaps the $S$-wave and has a nontrivial finite-spacing finite-volume energy, and the other overlaps a higher partial wave and has $x^{\dispersion}=2\mu E^{\dispersion}L^2/4\pi^2=9$ to machine precision, sitting right on a pole of the \Luscher zeta function~\eqref{spherical S}.
In contrast, when $N=4$ there is no $n=(3,0,0)$ state, and the $(2,2,1)$ state is itself an eigenstate.
When $N$ is very large sometimes there are multiple eigenstates that have no support for the delta function---$n^2=41, 50, 54\ldots$ have two non-interacting states, while $n^2=81, 89, 101\ldots$ have three non-interacting states, and $n^2=146$ is the first shell with four non-interacting states, for example.
After diagonalizing, we exclude these non-interacting \Aoneg states from our analysis.
We do not discuss these non-interacting states further and omit them from figures without comment.


\section{Periodic Motion}

A periodic wave in time is characterized by the time to complete one
cycle called the period $T$, then linear and angular frequencies obtain
\begin{align*}
  f &= 1/T  & \omega &= 2\pi f = 2\pi/T
\end{align*}
A restoring cause that is proportional to displacement
\begin{align*}
  F &= -kx  & \tau &= \kappa \theta
\end{align*}
will yield simple harmonic motion with a solution
\begin{align*}
  x &= A \cos(\omega t + \phi)  &  \theta &= A_{\theta} \cos(\omega t +
  \phi)\\
  \omega &= \sqrt{k/m}  & \omega &= \sqrt{\kappa/I}.
\end{align*}
Pendulums with small amplitudes approximate simple harmonic motion
\begin{align*}
  k &= mg/\ell  & \kappa &= mg\ell\\
  \omega &= \sqrt{g/l}  & \omega &= \sqrt{mg\ell/I}
\end{align*}


\section{Detective problem: From which spaces could these have come?}

Given finitely many points and some pairs of distances between them. Usual suspects
are some kinds of spaces from which they have come.

\vspace{0.15in}

\textbf{Problem}: Decide if they could have come from the given space.

\vspace{0.1in}

\noindent \textbf{Model Spaces}:

\vspace{0.4in}

\noindent \textbf{Examples}:

\vspace{0.1in}

(0) 1, 2 or 3 pts with a metric can always come from $\mathbb{R}^2$

\vspace{0.08in}

(0.5) $X = \{A_1, A_2, A_3, A_4\}$ with

\[ d(A_i, A_j) = \left\{
  \begin{array}{l l}
    1 & \quad \text{if $i=j$}\\
    0 & \quad \text{otherwise}
  \end{array} \right.\]

\qquad can come from $\mathbb{R}^3$ but not $\mathbb{R}^2$

\vspace{0.08in}

(1) $X = \{O, A_1, A_2, A_3\}$ with
    $$d(O, A_i) = 1 \qquad \qquad \forall i$$
    $$d(A_i, A_j) = 2 \qquad \text{if}\ \ i\neq j$$

\qquad cannot come from $\mathbb{R}^n$. It can come from a
tree with $O$ as center and $A_i$ as nodes.

\vspace{0.1in}

\textit{Proof}: Assume $X \subset \mathbb{R}^n$. Since $d(A_i, O) + d(O, A_j)= d(A_i, A_j)\ \
\forall i \in \{1, 2, 3\}$.\ $O$ must be a midpoint $A_1A_2$, $A_2A_3$, $A_3A_1$
simultaneously which is impossible.

\vspace{0.2in}

\noindent \textbf{More General Arguments}:

\vspace{0.1in}

Given points $O, A_1, A_2, A_3$ and pairwise distances between them, we want
obstructions to find these point in $\mathbb{R}^n$ with these distances.

\begin{enumerate}
    \item If $O, A_1, A_2\ \&\ A_3 \in \mathbb{R}^n$ with given distances.
    \begin{itemize}
        \item Determine angles from $\angle A_i O A_j$ from distances using cosine rule.
        \item Total angle at $O$ must be $\leq 2\pi$
    \end{itemize}

    \item Consider $\mathbb{R}^n$ as vector space with origin as $O$ and let
    $v_i = \overrightarrow{OA_i}$.
    $$\left | \left \langle v_i, v_i \right \rangle \right | = \left \| v_i
    \right \| ^2 = (d(O, A_i))^2 \ \ \text{and} \ \ \left \| v_i - v_j \right \|^2 = (d(A_i, A_j))^2 $$

    Hence we can calculate $\left \| a_1v_1 + a_2v_2 + a_3v_3 \right \|^2, \
    \forall\ a_1, a_2, a_3 \in \mathbb{R} $

    \textit{Obstruction}: These should all be positive or alternatively
    $V = [\left \langle v_i, v_j \right \rangle]$ should be positive semi$-$definite.

\end{enumerate}


\chapter{March}

\section{Perceptron algorithm} \index{Perceptron algorithm}
Suppose we have input data $(x_1, y_1), (x_2, y_2), ..., (x_n, y_n) \in \mathbb{R}^p \times \{-1, 1\}$,
and if the data points are separable, the perceptron algorithm works as following:

\begin{minted}[frame=lines, framesep=2mm,tabsize=4]{cpp}
w = 0
while some (x, y) is misclassified:
    w = w + yx
\end{minted}

\begin{remark}
In the separable case, perceptron algorithm guarantees to converge.
\end{remark}

\myheader{Multi-class perceptron}
\begin{minted}[frame=lines, framesep=2mm,tabsize=4]{cpp}
w_1 = w_2 = ... = w_k = 0
while some (x, y) is misclassified:
    for correct label y: w_y = w_y + x
    for incorrect label y*: w_(y*)  = w_(y*) - x
\end{minted}

\section{Kernel function} \index{Kernel function}
Following the perceptron algorithm, suppose $\phi$ is a function that maps $x$ to another feature space, such as $\phi(x) = (1, x_1, x_2, ..., x_1^2, x_2^2,..., x_1 x_2,...)$, which is a quadratic embedding.
In this case we can also run perceptron algorithm in the new feature space.

\begin{minted}[frame=lines, framesep=2mm,tabsize=4]{cpp}
w = 0
while y*(w * \phi(x)) < 0:
w = w + y\phi(x)
\end{minted}

A problem is that every time we need to calculate $\phi(x)$, which may be of high dimensions. To solve this problem, we observe that in fact we don't need to access $\phi(x)$ at all to make a decision, instead we
can write $w$ as following:
\myequ{
    w = a_1 \phi(x_1) + a_2 \phi(x_2) + ... + a_n \phi(x_n)
}
then $w\cdot\phi(x)$ is a weighted sum of $\phi(x)\cdot\phi(x_i)$. In addition, we also observe that
\myequ{
    \phi(x) \cdot \phi(z) = (1 + x\cdot z)^2
}
That is, we don't need to calculate $\phi(x)$.

\myheader{kernel function} From above we know that we don't care about the embedding $\phi(x)$, we only
care about the similarity between a pair of data points. Therefore, the kernel function is defined as following:
\vspace{0.5cm}
\begin{definition}[Kernel function]
    A function $k$: $\mathbb{R}^p \times \mathbb{R}^p \rightarrow \mathbb{R}$ is a valid kernel if it corresponds to some embedding, that is, there exists $\phi$ defined on $\mathbb{R}^p$ such that
    \myequ{
        k(x,z) = \phi(x) \cdot \phi(z)
        }
\end{definition}

This is equivalent to require that for any finite subset $\{x_1, x_2, ..., x_m\} \subset \mathbb{R}^p$,
the $m \times m$ similarity matrix
\myequ{
    K_{ij} = k(x_i, x_j)
    }
is \textit{positive semidefinite}. Proof:
\myequ{
    Z^T K Z = Z^T (X^T X) Z = (XZ)^T (XZ) \geq 0
    }

\myheader{RBF kernel}
RBF kernel or Gaussian kernel is defined as
\myequ{
    k(x,z) = e^{-||x-z||^2 / 2\sigma^2}
    }

\myheader{string kernel}
For each substring $s$, we define feature:
\myequ{
    \phi_s(x) &= \# \text{ of times substring $s$ appears in $x$} \\
    \phi(x) &= (\phi_s(x): \text{ all strings } s)
    }

\section{$k$-means Clustering}\index{$k$-means Clustering}
\myheader{$k$-means} Minimize average squared distance between points and their nearest representatives.
The input is data points $x_1, x_2, ..., x_n$, and integer $k$, and the output is centers
$\mu_1, \mu_2, ..., \mu_k$.

\myheader{Lloyd's $k$-means algorithm}
\begin{minted}[frame=lines, framesep=2mm,tabsize=4]{cpp}
Initialize centers u_1, u_2, ... u_k in some manner.
Repeat until convergence:
    assign each point to its nearest center
    update each u_j to the mean of points assigned to it
\end{minted}

\myheader{How to initialize centers?} $k$-means++: start with extra centers, then prune later.

\begin{lstlisting}[
style=liststy,
]
Pick a data point x at random as the first center
Let $C = \{x\}$
    Repeat until the desired number of centers is attained:
    pick a data point $x$ at random from the following distributions:
        $Pr(x) \propto dist(x, C)^2$, where $dist(x, C) = min_{z\in C}||x-z||$
    Add $x$ to $C$
\end{lstlisting}

\myheader{Streaming and online computation} If there are too much data to fit in memory, or the data
is continuously collected, we have to update the model gradually.

\myheader{The good and the bad} Good: fast and easy, effective in quantization. Bad: geared towards data in which the clusters are spherical, and of roughly the same results.

\section{Mixtures of Gaussians} \index{Mixtures of Gaussians}
Each of $k$ clusters is specified by a Guassian distribution $P_j = N(\mu_i, \sum_i)$ and a mixing weight
$\pi_j$. The overall distribution is a mixture of all Gaussians:
\myequ{
    Pr(x) = \pi_1 P_1(x) + \pi_2 P_2(x) + ... + \pi_k P_k(x)
}
We need to determine all the parameters including $\pi, \mu, \sum$. We apply \textbf{EM} algorithm to solve this problem
(see Figure~\ref{fig:em_mar}).
\begin{figure}[H]
    \centering{
        \includegraphics[width=0.9\textwidth]{./images/mar/gmm_em.PNG}
    }
    \caption{EM algorithm for GMM clustering.}
    \label{fig:em_mar}
\end{figure}

\section{Hierarchical clustering} \index{Hierarchical clustering}
Clustering is of multi-scale, and often there is no single right answer. Hierarchical
clustering avoids these problems.
\begin{lstlisting}[
style=liststy,
]
Start with each point on its own
Repeat until there is just one cluster:
    Merge the two clusters with the $closest$ pair of points
Discard singleton clusters
\end{lstlisting}

\myheader{Linkage method} The problem is how we measure the distance
between two cluster of points.
\begin{enumerate}
    \item Single linkage: $dist(C, C') = min_{x \in C, x' \in C'} ||x - x'||$
    \item Complete linkage: $dist(C, C') = max_{x \in C, x' \in C'} ||x - x'||$
    \item Average linkage:
        \begin{enumerate}
            \item average pairwise distance between all pair of points in the two clusters
            \item distance between cluster centers
            \item Ward's method
        \end{enumerate}
\end{enumerate}

\section{Boosting}
\myheader{Adaboost} See Figure~\ref{fig:adaboost_mar}.
\begin{figure}[H]
    \centering{
        \includegraphics[width=0.9\textwidth]{./images/mar/adaboost.PNG}
    }
    \caption{Adaboost algorithms.}
    \label{fig:adaboost_mar}
\end{figure}

\myheader{Bagging} See Figure~\ref{fig:bagging_mar}
\begin{figure}[H]
    \centering{
        \includegraphics[width=0.9\textwidth]{./images/mar/bagging.PNG}
    }
    \caption{Bagging.}
    \label{fig:bagging_mar}
\end{figure}

\myheader{Random forest} See Figure~\ref{fig:random_forest_mar}
\begin{figure}[H]
    \centering{
        \includegraphics[width=0.9\textwidth]{./images/mar/random_forest.PNG}
    }
    \caption{Random forests.}
    \label{fig:random_forest_mar}
\end{figure}

\section{Informative projections} \index{Informative projections}

\myheader{Project to multiple directions} Suppose we want to project $x \in \mathbb{R}^p$
into the $k$-dimensional subspace spanned by $u_1, u_2, ..., u_k \in \mathbb{R}^p$,
and suppose all $u_i$ are orthonormal (each has length one, and they are perpendicular to
each other). Then the projection is:
\myequ{
    (x \cdot u_1)u_1 + (x \cdot u_2)u_2 + ... + (x \cdot u_k)u_k = UU^Tx
}

\myheader{Best single direction} Suppose we want to map our data $x_1, x_2, ..., x_n \in
\mathbb{R}^p$ into just one dimension $x \mapsto u \cdot x$, what is the best direction $u$.

The best direction $u$ should be the one that maximize the variance after projection. Let $X$
be the data matrix, where each column is a data point, and $\sum$ be the covariance matrix of $X$.
Suppose the mean of $X$ is $\mu \in \mathbb{R}^p$, then
\myequ{
    \mathbb{E}(u^T X) & = u^T \mathbb{E}(X) = u^T \mu \\
    var(u^T X) & = \mathbb{E}(u^T X - u^T \mu) = \mathbb{E} (u^T (X - \mu) (X - \mu)^T u)  \\
                & = u^T \mathbb{E}(X - \mu)(X - \mu)^T u = u^T \sum u
}

\begin{remark}
    $u^T\sum u$ is maximized by setting $u$ to the first \textbf{eigenvector} of $\sum$. The maximum
    value is the corresponding eigenvalue.
\end{remark}

\myheader{Best $k$-dimensional projection} Let $\sum$ be the $p\times p$ covariance matrix of $X$.
and $\lambda_1 \geq  \lambda_2 \geq ... \geq \lambda_p$ are the eigenvalues, and $u_1, u_2, ...
u_p$ are the corresponding eigenvectors. Then the best $k$-dimensional projection directions are
$u_1, u_2, ..., u_k$.

\myheader{Spectral decomposition} See Figure~\ref{fig:spec_decomp_mar}.
\begin{figure}[H]
    \centering{
        \includegraphics[width=0.9\textwidth]{./images/mar/spectral_decomp.PNG}
    }
    \caption{Spectral decomposition.}
    \label{fig:spec_decomp_mar}
\end{figure}

\myheader{Singular value decomposition (SVD)} See Figure~\ref{fig:svd_mar}.
    Where $u_i, \sigma_i, v_i$ comes from? We know that:
    \begin{itemize}
        \item $Mv_i = \sigma_i u_i, M^T u_i = \sigma_i v_i$
        \item $M M^T u_i = \sigma_i^2 u_i, M^T M v_i = \sigma_i^2 v_i$
    \end{itemize}
    Therefore, $v_i$ is the eigenvectors of $M^T M$, and $u_i$ is the eigenvectors
    of $M M^T$. $MM^T$ and $M^TM$ has the same eigenvalues $\sigma_i^2$.  Note
    that all $\sigma_i$ are \textbf{non-negative}.

\begin{figure}[H]
    \centering{
        \includegraphics[width=0.9\textwidth]{./images/mar/SVD.PNG}
    }
    \caption{Singular value decomposition.}
    \label{fig:svd_mar}
\end{figure}

\section{Positive definite matrix }
\vspace{0.5cm}
\begin{definition}[Positive definite matrix]
    A square $p \times p$ symmetric matrix $A$ is positive definite if for all
    nonzero $x \in \mathbb{R}^p$,
    \myequ{
        x^T A x > 0
        }
\end{definition}

\myheader{Properties of positive definite matrix}
\begin{enumerate}
    \item The $r \times r$ submatrix $A_r$ (start from top left element) is also positive
    semidefinite.
    \item The $p$ eigenvalues of A $\lambda_1, \lambda_2, ..., \lambda_p$ are positive.
    Conversely, if all the eigenvalues of a matrix $B$ are
    positive, then $B$ is positive definite.

    \item There exist a unique decomposition of $A = LL^T$, where $L$ is a lower triangular
    matrix. This is called \textit{Cholesky Decomposition}.
    \item There exists a unique decomposition of $A = VDV^T$.

\end{enumerate}

\section{Beyond projections} \index{Beyond projections}
Sometimes data in a high-dimensional space $\mathbb{R}^p$ in fact
lies close to a $k$-dimensional manifold, for $k\ll p$.

\myheader{ISOMAP algorithm}
Given data $x_1, x_2, ..., x_n$,
\begin{enumerate}
\item estimate \textit{geodesic distances} between the data points, that is, distance along the manifold.
\item embed these points in Euclidean space so as to match these distances.
\end{enumerate}

\myheader{Geodesic distances}
To estimate geodesic distances:
\begin{enumerate}
   \item Construct neighborhood graph, connect nodes whenever two nodes are close together.
   \item Compute distance in this graph (shortest-path algorithm).
\end{enumerate}

\myheader{Distance-preserving embeddings} Problem definition:
\begin{lstlisting}[
style=liststy,]
Input:an $n\times n$ matrix of pairwise distances $D$, where $D_ij$ is the distance between
points $i$ and $j$.
Output: an embedding $z_1, z_2, ..., z_n \in \mathbb{R}^k$ that realizes these distances as closely as
possible.
\end{lstlisting}

\myheader{Gram matrix}
Gram matrix on a set of vectors $z_1, z_2, ..., z_n$ is the matrix $B$ where
$B_{ij} = z_i \cdot z_j$.

\myheader{Classical multidimensional scaling} See Figure~\ref{fig:cms_mar}

\begin{figure}[H]
    \centering{
        \includegraphics[width=0.9\textwidth]{./images/mar/cms.PNG}
    }
    \caption{Classical multidimensional scaling.}
    \label{fig:cms_mar}
\end{figure}

\section{Jacobian matrix} \index{Jacobian matrix}
In vector calculus, the Jacobian matrix is the matrix of all first-order partial derivatives of a vector-valued function.
\myequ{
    J  = \frac{d \mathbf{f}}{d \mathbf{x}} &= \begin{bmatrix}
        \frac{\partial \mathbf{f}}{\partial x_1} &
         \frac{\partial \mathbf{f}}{\partial x_2} &
         \dots & \frac{\partial \mathbf{f}}{\partial x_n}
    \end{bmatrix}    \\
    & = \begin{bmatrix}
        \frac{\partial f_1}{\partial x_1} &
        \dots &
        \frac{\partial f_1}{\partial x_n} \\
         \vdots & \ddots & \vdots \\
         \frac{\partial f_m}{\partial x_1}  &
         \dots &
         \frac{\partial f_m}{\partial x_n}
    \end{bmatrix}
}

\section{Gaussian-Newton algorithm} \index{Gaussian-Newton algorithm}
The Gaussian-Newton algorithm is used to solve non-linear least square problems. It is a modification
of Newton's method for finding a minimum of a function. Unlike Newton's method, the Gaussian-Newton algorithm
can only be used to minimize a sum of squared function values, but it has the advantage that the second
derivatives, which can be challenging to compute, are not required.

Given $m$ functions $r = (r_1, r_2, \dots, r_m)$ (often called residuals) of $n$ variables $\mathbf{\beta} = (\beta_1, \dots, \beta_n)$ with $m \geq n$, the Gauss–Newton algorithm iteratively finds the value of the variables which minimizes the sum of squares:
\myequ{
  S(\beta) = \sum\limits_{i=1}^{m} r_i^2(\beta)
}
Start with an initial guess $\mathbf{\beta}^{(0)}$, the method proceeds by the iterations:
\myequ{
    \mathbf{\beta}^{(s+1)} = \mathbf{\beta}^{(s)}    - (J_r^T J_r)^{-1} J_r^T r(\mathbf{\beta}^{(s)})
}
where
\myequ{
    (J_r)_{ij} = \frac{\partial r_i(\beta^{(s)})}{\partial \beta_j}
}
If $m = n$, the iteration simplifies to
\myequ{
    \mathbf{\beta}^{(s+1)} = \mathbf{\beta}^{(s)}    - (J_r)^{-1} r(\mathbf{\beta}^{(s)})
}

\section{Functional programming: introductions} \index{Functional programming: introductions}

\myheader{Why FP?}
We want software to be \textit{readable, reusable, modifiable, predictable and checkable}. Functional
programming could satisfy these requirements.

There is no assignment, mutation, or loop in FP.

\section{$\lambda$-calculus}\index{$\lambda$-calculus}
Lambda calculus (also written as $\lambda$-calculus) is a formal system in mathematical logic for expressing computation
 based on function abstraction and application using variable binding and substitution.

\myheader{Syntax}
Three kinds of expressions:
\begin{lstlisting}[
style=liststy,]
e ::= x
    | $\lambda$ x. e
    | $\lambda$ e_1 e_2
\end{lstlisting}
called \textit{variables}, \textit{functions}($\lambda$-abstraction), and \textit{application}.
Functions $\lambda x. e$ takes $x$ as an input and output $e$.

Application associates to the left: $x y z$ means $(x y) z$. However, abstraction extends as far right
as possible:
$\lambda x. x \lambda y. x y z \Rightarrow \lambda x. (x \lambda y. ((x y) z))$

Identity function: $I = \lambda x. x$

\myheader{Scope of identifier (variable)}
Scope of a variable is the part of program where the variable is \textit{accessible}.

$\lambda x. E$ binds variable $x$ in $E$:
\begin{enumerate}
    \item $x$ is the newly introduced varialbe.
    \item $E$ is the scope of $x$.
    \item $x$ is bound in $\lambda x. E$
\end{enumerate}

$y$ is \textit{free} if it occurs not bound in $E$:
\begin{enumerate}
    \item $Free(x) = \{x\}$
    \item $Free(E_1 E_2) = Free(E_1) \cup Free(E_2)$
    \item $Free(\lambda x. E) = Free(E) - \{x\}$
\end{enumerate}

$\alpha$-renaming:
\begin{enumerate}
    \item Allows bound variable names to be changed.
    \item $\lambda x. x = \lambda y. y$
\end{enumerate}

substitution:
\begin{enumerate}
    \item $[E'/x]E$: use $E'$ to substitute all $x$ bounded in $E$
    \item $[y (\lambda x. x)/x] \lambda y. (\lambda x. x) y x
           \equiv$
           $[y (\lambda v. v)/x] \lambda y. (\lambda u. u) z x $ (renaming)
           $\equiv \lambda z. (\lambda u. u) z (y (\lambda v. v))$ (substitution)
\end{enumerate}

$\beta$-reduction:
\begin{enumerate}
    \item $\beta$-reduction captures the idea of function application.
    \item $(\lambda x. e) e' \rightarrow [e'/x]e$
    \item $((\lambda n. n\times2) 7) \rightarrow 7\times 2$
\end{enumerate}

local variable:
\begin{lstlisting}[
style=liststy,]
let x = $e_1$ in $e_2$
$\equiv (\lambda x. e_2) e_1$
\end{lstlisting}

boolean:
\begin{lstlisting}[
style=liststy,]
true $\equiv$ $\lambda x. \lambda y. x$
false $\equiv$ $\lambda x. \lambda y. y$
if $E_1$ then $E_2$ else $E_3$ $\equiv$ $E_1 E_2 E_3$
if true then u else v $\rightarrow$ $(\lambda x. \lambda y. x) u\ v  \rightarrow (\lambda  y. u) \rightarrow u$
\end{lstlisting}

not:
\begin{lstlisting}[
style=liststy,]
function takes b:
    return function x, y:
        return (if b then y else x)

not $\equiv \lambda b. (\lambda x. \lambda y. \  b\ y\ x)$
not true $\rightarrow \lambda x. \lambda y. \ true \ y \ x \rightarrow \lambda x. \lambda y. \ y \rightarrow false$
\end{lstlisting}

or:
\begin{lstlisting}[
style=liststy,]
function takes $b_1, b_2$
    return function takes x, y:
        return (if $b_1$ then x else (if $b_2$ then x else y))
or $\equiv \lambda b_1 . \lambda b_2 . (\lambda x. \lambda y.\ b_1\ x (b_2 \ x \ y))$
\end{lstlisting}

records:
\begin{lstlisting}[
style=liststy,]
pair = function takes a bool
       return the left or right element
mkpair $e_1 \ e_2 \equiv \lambda b. \ b\ e_1\ e_2$
fst p $\equiv$ p true
snd p $\equiv$ p false
\end{lstlisting}

natural numbers:
\begin{lstlisting}[
style=liststy,]
natural number: iterate a number of times over some function
$n$ = function that takes function $f$, starting value $s$
      returns: $f$ applied to $s$ $n$ times
$0 \equiv \lambda f. \lambda s. \ s$
$1 \equiv \lambda f. \lambda s. \ f \ s$
$2 \equiv \lambda f. \lambda s. \ f (f \ s)$
(n f s) = apply f to s n times
\end{lstlisting}

operations on natural numbers:
\begin{lstlisting}[
style=liststy,]
iszero n $\Leftrightarrow$ n ($\lambda$ b. false) true
iszero $\Leftrightarrow$ $\lambda $ n. ($\lambda$ b. false) true
succ n  $\Leftrightarrow$ $\lambda$f. $\lambda$s. f (n f s)
add a b $\Leftrightarrow$ a succ b
multi a b $\Leftrightarrow$ a (add b) 0
\end{lstlisting}

\section{Haskell: basics}\index{Haskell: basics}
Haskell is a standardized, general-purpose purely functional programming language, with non-strict semantics and strong static typing.

\myheader{GHC system}
\begin{lstlisting}[
style=liststy,
language=Haskell,]
:load foo.hs
:type expression
:info variable
\end{lstlisting}

\myheader{Basic types}
\begin{haskellcode}
32   :: Integer
4.2  :: Double
'a'  :: Char
True :: Bool

-- function types
pos :: Integer -> Bool
pos x = (x > 0)

-- multiple argument function types
-- function takes args of A1, A2, A3, gives out B
A1 -> A2 -> A3 -> B

-- tuples, elements do not have to be of the same type
(A1, ..., An)
-- pattern matching extracts values from tuple
pat :: (Int, Int, Int) -> Int
pat (x, y, z) = x * (y + z)

-- Lists, elements have to be of the same type
[1, 3, 5, 7]
-- construct lists
'a' : ['b', 'c'] = ['a', 'b', 'c']
cons2 x y zs = x : y : zs

-- type
-- Not a new type, just shorthand
type XY = (Double, Double)
type Circle = (Double, Double, Double)
-- data creates new types
data CircleT = Circle (Double, Double, Double)
data Shape =
    | Rectangle Side Side
    | Ellipse Radius Radius
    | RtTriangle Side Side
    | Polygon [Vertex]
type Side = Double
type Radius = Double
type Vertex = (Double, Double)
\end{haskellcode}

\myheader{Input and output}

\begin{haskellcode}
-- action: value describing an effect on world
IO a -- type of an action that returns an a

-- takes input string, return action that writes string to stdout
putStr :: String -> IO ()

-- only one way to execute action: make it the value of name main
main :: IO ()
main = putStr "hello world\n"

-- do many actions with 'do'
do putStr "Hello"
   putStr "World"
   putStr "\n"

-- input
getLine :: IO String
main:: IO()
main = do putStr "What is your name?"
       n <- getLine -- assignment
       putStrLn ("Happy New Year " ++ n)
\end{haskellcode}

\section{Haskell: higher-order functions}\index{Haskell: higher-order functions}
In all functional languages, functions are first-class values, meaning, that they can be treated just as you would any other data. That is, you can pass functions around to in any manner that you can pass any other data around in.

\myheader{Functions are data}
\begin{haskellcode}
plus1 :: Int -> Int
plus1 x = x + 1

minus1 :: Int -> Int
minus1 x = x - 1

funp :: (Int -> Int, Int -> Int)
funp = (plus1, minus1)
\end{haskellcode}

\myheader{Take functions as input and output}
\begin{haskellcode}
-- functions as input
doTwice :: (t -> t) -> t -> t
doTwice f x = f (f x)

-- functions as output
plusn :: Int -> (Int -> Int)
plusn n = f
          where f x = x + n

-- partially apply functions
plus :: Int -> Int -> Int
plus a b = a + b
plus5 :: Int -> Int
plus5 = plus 5 -- plus5 1000 outputs 1005
\end{haskellcode}

\myheader{Anonymous functions}
We will see many situations where a particular function is only used once, and hence,
there is no need to explicitly name it. Haskell provides a mechanism to create such anonymous functions.

\begin{haskellcode}
\x -> x + 1
(\x -> x + 1) 100 -- 101
doTwice (\x -> x + 1) 100 -- 102
\end{haskellcode}

\myheader{Infix operations}
Haskell allows you to use any function as an infix operator,
simply by wrapping it inside backticks.

To further improve readability, Haskell allows you to use partially applied infix operators, ie infix operators with only a single argument. These are called \textit{sections}.
\begin{haskellcode}
2 `plus` 4 -- 6
doTwice (+1) 0 -- 2
doTwice (1+) 0 -- 2
doTwice (1:) [2..5] -- [1, 1, 2, 3, 4, 5]
\end{haskellcode}

\myheader{Polymorphism}
\textit{doTwice} is polymorphic in that it works with different kinds of values,
e.g. functions that increment integers and concatenate strings. This is vital for \textit{abstraction}.

Polymorphic functions which can operate on different kinds values are often associated with polymorphic
data structures which can contain different kinds of values.
These are also represented by types containing type variables.

\begin{haskellcode}
foo :: a -> (a -> b) -> b
foo x f = f x

x |> f = f x

do1 :: (a -> b) -> a -> b
do1 f x = x |> f

do2 :: (a -> a) -> a -> a
do2 f x = x |> f |> f
\end{haskellcode}

\myheader{Bottling Computation Patterns With Polymorphic Higher-Order Functions}

\begin{haskellcode}
toUpperString :: String -> String
toUpperString [] = []
toUpperString (c:cs) = toUpper c : toUpperString cs

-- map
map :: (a -> b) -> [a] -> [b]
map f [] = []
map f (x:xs) = (f x) : (map f xs)

toUpperString = map toUpper

-- foldr
foldr op base [] = base
foldr op base (x:xs) = x `op` (foldr op base xs)

-- foldr on actions
fuseActions :: [IO ()] -> IO ()
fuseActions []        = return ()
fuseActions (a1:acts) = do a1
                           fuseActions acts

fuseActions :: [IO ()] -> IO ()
fuseActions = foldr (>>) (return ())
\end{haskellcode}

\section{Higher-order programming} \index{Higher-order programming}

\myheader{Recursive types}
\begin{haskellcode}
data Shape  = Rectangle Double Double
            | Polygon [(Double, Double)]

data IntTree = ILeaf Int
             | INode IntTree IntTree
\end{haskellcode}

\myheader{Parameterized types}

\begin{haskellcode}
-- a is a type
data List a = Empty
            | OneAndMore a (List a)

data Tree a = Leaf a
            | Node (Tree a) (Tree a)

type IntList = List Int
type CharList = List Char
type DoubleList = List Double
\end{haskellcode}

\myheader{Kinds}
In the area of mathematical logic and computer science known as type theory,
a kind is the type of a type constructor or, less commonly, the type of a higher-order type operator.

The \textit{Tree a} corresponds to trees of values of type a.
If a is the type parameter, then what is Tree ?
A function that takes a type a as input and returns a type Tree a as output!
But wait, if List is a function then what is its type? A \textit{kind} is the type of a type.

\begin{haskellcode}
:kind Int -- *
:kind Char -- *
:kind Bool -- * -> *
:kind (->) -- * -> * -> *
\end{haskellcode}

\begin{enumerate}
    \item *: the kind of all data types seen as nullary type constructors,
    and also called proper types in this context.
    \item * -> * : the kind of a unary type constructor, e.g. list type constructor.
    \item * -> * -> * : the kind of a binary type constructor (via currying), e.g. of
             a pair type constructor  and also that of a function type constructor.
    \item (* -> *) -> *: the kind of a higher-order type operator from unary type constructors to proper types.
\end{enumerate}

\section{Haskell: typeclass}\index{Haskell: typeclass}
We want the operator such as $+$ to work for a bunch of different data types
such as integers, doubles. So what should be the type of $+$?

\begin{haskellcode}
-- too anemic
(+) :: Integer -> Integer -> Integer
-- too aggressive, it doesn’t make sense to add two functions
(+) :: a -> a -> a
\end{haskellcode}

Haskell solves this problem with an insanely slick mechanism called typeclasses.

\myheader{Qualified types}
\begin{haskellcode}
-- truth
(+) :: (Num a) => a -> a -> a
\end{haskellcode}
We call the above a qualified type. Read it as, $+$ takes in two a values and returns an a value
for any type a that is a Num or is an instance of Num.

The name Num can be thought of as a predicate over types.
Some types satisfy the Num predicate. Examples include Integer, Double etc, and any values of those types can be passed to +.
Other types do not satisfy the predicate. Examples include Char, String, functions etc, and
so values of those types cannot be passed to +.

\myheader{Typeclass}
A \textit{typeclass} is a collection of operations (functions) that must exist for the underlying type.

A typeclass defines some behavior (like comparing for equality, comparing for ordering, enumeration) and then types that can behave in that way are made instances of that typeclass. The behavior of typeclasses is achieved by defining functions or just type declarations that we then implement. So when we say that a type is an instance of a typeclass, we mean that we can use the functions that the typeclass defines with that type.

\begin{haskellcode}
-- typeclass Eq
class  Eq a  where
  (==) :: a -> a -> Bool
  (/=) :: a -> a -> Bool

-- typeclass show
class  Show a  where
  show :: a -> String
\end{haskellcode}
A type a can be an instance of Eq as long as there are two functions
that determine if two a values are respectively equal or disequal.
Similarly, the typeclass Show captures the requirements that make a particular datatype be viewable,

\myheader{Automatic derivation}
Haskell allows us automatically derive functions for certain key type classes, namely those in the standard library.
\begin{haskellcode}
data Showable = A' | B' | C' deriving (Eq, Show)

class (Eq a, Show a) => Num a where
    (+) :: a -> a -> a
    (*) :: a -> a -> a
    (-) :: a -> a -> a
    negate :: a -> a
    abs :: a -> a
    signum :: a -> a
    fromInteger :: Integer -> a
-- A type T can only be deemed an instance of Num if
--     1. The type is also an instance of Eq and Show, and
--     2. There are functions for adding, multiplying, etc
\end{haskellcode}

\myheader{Explicit signatures}
While Haskell is pretty good about inferring types in general,
there are cases when the use of type classes requires explicit annotations.
\begin{haskellcode}
-- Read is a build typeclass, parse a string and turn it into an a
read:: (Read a) => String -> a

-- read "2" -- error: it doesn’t know what to convert the string to
-- (read "2") :: Int -- correct
-- (read "2") :: Double -- correct
\end{haskellcode}

\section{Haskell: monads}\index{Haskell: monads}
The function programming community divides into two camps:
\begin{enumerate}
    \item "Pure" languages, such as Haskell, are based directly upon the
    mathematical notion of a function as a mapping from arguments to
    results.

    \item "Impure" languages, such as ML, are based upon the extension of this notion with a range of possible effects, such as exceptions and assignments.
\end{enumerate}
Pure languages are easier to reason about and may benefit from lazy
evaluation, while impure languages may be more efficient
and can lead to shorter programs.

\subsection{Abstracting programming patterns}
Monads are an example of the idea of abstracting out a common
programming pattern as a definition.

\begin{haskellcode}
inc :: [Int] -> [Int]
inc []     =  []
inc (n:ns) =  n+1 : inc ns

sqr :: [Int] -> [Int]
sqr []     =  []
sqr (n:ns) =  n^2 : sqr ns

-- above functions have the same programming pattern,
-- namely mapping the empty list to itself, and a non-empty
-- list to some function applied to each element in the list

-- abstract this pattern gives us the map function
map         :: (a -> b) -> [a] -> [b]
map f []     = []
map f (x:xs) = f x : map f xs

inc = map (+1)
sqr = map (^2)
\end{haskellcode}

\myheader{Maybe}
The \texttt{Maybe} type encapsulates an optional value.
A value of type \texttt{Maybe a} either
contains a value of type \texttt{a} (represented as \text{Just a}),
or it is empty
(represented as \texttt{Nothing}).
Using \texttt{Maybe} is a good way to deal with errors or
exceptional cases without resorting to drastic measures such as error.

\begin{haskellcode}
data Maybe a = Just a | Nothing

foo :: (a -> b) -> Maybe a -> Maybe b
foo f z = case z of
            Just x  -> Just (f x)
            Nothing -> Nothing
\end{haskellcode}

\myheader{Functor typeclass}
Functor typeclass is basically for things that can be mapped over.
\begin{haskellcode}
class Functor f where
    fmap :: (a -> b) -> f a -> f b

-- map is just a map that works on lists
instance Functor [] where
    fmap = map

-- Maybe is also a functor
instance Functor Maybe where
    fmap f (Just x) = Just (f x)
    fmap f Nothing = Nothing

-- make Tree type instance of Functor
instance Functor Tree where
    fmap f EmptyTree = EmptyTree
    fmap f (Node x left right) = Node (f x)
           (fmap f left) (fmap f right)

-- IO is an instance of Functor
instance Functor IO where
    fmap f action = do
        results <- action
        return (f result)

-- <-: bind that result to a name
result <- getLine

-- return: a function that makes an I/O action that doesn't do
--         anything but only presents something as its result

\end{haskellcode}

\myheader{Generalize map to many arguments}
We can generalize map to many arguments.
\begin{haskellcode}
lift1 :: (a -> b) -> [a] -> [b]
lift2 :: (a1 -> a2 -> b) -> [a1] -> [a2] -> [b]
lift3 :: (a1 -> a2 -> a3 -> b) -> [a1] -> [a2] -> [a3] -> [b]
\end{haskellcode}

There is a typeclass called \texttt{Applicative} that corresponds to
the type constructors that you can \texttt{lift2} or \texttt{lift3} over.

\begin{haskellcode}
liftA  :: Applicative t => (a -> b) -> t a -> t b

liftA2 :: Applicative t
       => (a1 -> a2 -> b)
       -> t a1
       -> t a2
       -> t b

liftA3 :: Applicative t
       => (a1 -> a2 -> a3 -> b)
       -> t a1
       -> t a2
       -> t a3
       -> t b
\end{haskellcode}

\myheader{Sequencing operator}
Here we introduce a new sequencing operator that we write as
\texttt{>>=}, and read as \textit{then}.

\begin{haskellcode}
(>>=)   :: Maybe a -> (a -> Maybe b) -> Maybe b
m >>= f =  case m of
             Nothing -> Nothing
             Just x  -> f x

-- evaluate each of the expression m1, m2, ..., mn in turn,
-- and combine their results x1, x2, ..., xn by applying
-- the function f
m1 >>= \x1 ->
  m2 >>= \x2 ->
  ...
    mn >>= \xn ->
      f x1 x2 ... xn
\end{haskellcode}

\section{To add}


\documentclass[10pt,a4paper]{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{makeidx}
\usepackage{graphicx}
\usepackage[all]{xy}

\newtheorem{defi}{Definition}
\newtheorem{lemm}{Lemma}
\newcommand{\bws}{\paragraph{Proof}}
\author{moi}
\begin{document}
\section{Groups and heaps}
We intend to discuss the concept of groups and heaps which are closely related. As convention, we use
$$\underbrace{X \times \ldots \times X}_{n-\mathrm{times}} = X^n,\ \forall n \in \mathbb{N}.$$
\subsection{Definitions}
For the sake of completeness, we are going to repeat the definition for semi groups, monoids and groups.
\begin{defi}
A set $S$ with a binary map $m : S^2 \longrightarrow S$ is called a semi group if $\mathrm{im}(m) \subset S$ and the following diagram
$$\xymatrix{S^3 \ar[r]^{id \times m}\ar[d]_{m \times id} & S^2\ar[d]^m\\
S^2 \ar[r]_m &S\\
}$$
commutes. We call $m$ the operations or sometimes incorrectly the multiplication and denote a semi group as a pair $(S,m)$. A subset $T \subset S$ is called a sub semi group if both conditions hold for the restriction $m \mid_{T^2} : T^2 \longrightarrow T$.
\end{defi}
The first condition is called closeness - the latter is called the associativity.
\begin{defi}
We call a semi group $(M,m)$ a monoid if $M$ has a sub semi group, usually denoted by $\{\ast\}$, with exactly one element such that
$$\xymatrix{
\{\ast\} \times G
}$$
This distinguished element $e \in \{\ast\}$ is called the neutral element and induces a map
$$
\end{defi}
\begin{defi}
Let $X$ be a set. We call a pair $(X, h)$, with $h : X^3 \longrightarrow X$ a terniar operation, a heap if the following diagrams commute:
$$\begin{array}{c}
\xymatrix{
&&&X^5 \ar[rrrd]^{id_{X^2} \times h}\ar[llld]_{h \times id_{X^2}} \ar[dd]_{id_{X} \times \tau_3 \times id_X}&&& \\
X^3\ar[rrrddd]_{h}&&&&&&X^3\ar[lllddd]^{h}\\
&&&X^5 \ar[d]_{id_X \times h \times id_X}&&&\\
&&&X^3\ar[d]_h&&&\\
&&&X&&&\\
}\\
\mathrm{para-associativity}\\
\end{array}$$$$
\begin{array}{c}
\xymatrix{
&&&X^2\ar[llld]_{\Delta \times id_X \circ \tau} \ar[ldd]_{\Delta \times id_X \circ \tau} \ar[ddd]^{\pi_1}_{\pi_2 \circ \tau}\ar[rdd]^{id_X \times \Delta}\ar[rrrd]^{id_X \times \Delta} &&&\\
X^3\ar[d]_{\tau_3}&&&&&& X^3\ar[d]^{\tau_3}\\
X^3\ar[rrrd]_h&&X^3\ar[rd]_h&&X^3\ar[ld]^{h}& &X^3\ar[llld]^{h}\\
&&&X&&&\\
}\\
\mathrm{identity,}\\
\end{array}
$$
where
$$\begin{array}{rrclcrrcl}
\Delta : &X& \longrightarrow &X^2,&&\tau_3 : &X^3& \longrightarrow& X^3\\
&x&\longmapsto&(x,x)&&&(x,y,z)&\longmapsto&(z,y,x)\\
&&&\\
\pi_1:&X^2&\longrightarrow&X&&\pi_2:&X^2&\longrightarrow&X\\
&(x,y)&\longmapsto&x&&
&(x,y)&\longmapsto&y\\
&&&\\
\tau :& X^2 &\longrightarrow&X^2\\
&(x,y)&\longmapsto&(y,x)\\
\end{array}$$
\end{defi}
Firstly, we obviously have $\tau_3 = (id \times \tau)\circ(\tau \times id) \circ (id \times \tau)$ and $\pi_2 \tau = \pi_1$. Secondly, the projections $\pi_i$ are tighed to the left or right side of the diagram. Furthermore, we remark that heaps can be thought of as groups with no unique neutral element, i.e. we forgot to a assign a neutral element. One can easily check that each group $(G, m, e, inv)$ is a heap via
$$h = m \circ (id_G \times m)\circ (id_G \times inv \times id_G) = m \circ (m \times id_G)\circ(id_G \times inv \times id_G).$$
Conversely, each heap $(G, h)$ has a unique group structure modulo a distinguished element $e \in G$ being the neutral element. We subsequently get the group morphisms:
$$m = h\mid_{G \times \{e\} \times G} : G^2 \longrightarrow G,\ (g,g') \longmapsto h(g,e,g'),$$
$$e : \{\ast\} \longrightarrow G,\ \ast \longmapsto e.$$
To be more precise, the first statement shall be read via the identification induced by the map
$$id_G \times f_e \times id_G : G^3 \longrightarrow G^2,\ (g, g'', g') \longmapsto (g, e, g').$$
The equivalence classes are
$$\mathcal{G} := \left\{[(g,g')] \subset G^3: (g,g'',g') \in [(g,g')] \forall g'' \in G\right\}$$
We claim
\begin{lemm}
The set of equivalence classes $\mathcal{G}$ is isomorphic to $G^2$.
\end{lemm}
\bws This is obvious if we consider
$$\iota_{G^2} : G^2 \longrightarrow G^3,\ (g,g') \longmapsto (g,e,g')$$
is a monomorphism and therefore we have $\mathrm{im} \pi \iota_{G^2} \simeq \mathcal{G}$.\\
The inverse is given via
$$m(g,g') = e \Leftrightarrow g' \in m\mid_{\{g\}\times G}^{-1}(e) = \{g'' \in G: m(g,g'') = e\}$$
%$$G \times \{e\} \times G/
\end{document}


\section{Displacement interpolation}
\label{iso_section:interpolation}
\paragraph{}
Another difference between it with conventional FEM or SBFEM lies in the post processing.
After solving the partial differential equation numerically, the displacements on the nodes will be one of the output in
    the traditional method.
However, similar to what is discussed in \ref{iso_section:surface_traction}, NURBS curves are defined by the control points
    that are not geometrically located on the curves.
As a consequence, not only the input such as surface traction need to be translated into a NURBS-like representation, the
    output such as the displacements will be the dummy values on the control points as well, or ``control displacements''
    $\left\{ u_c \right\}$.
Dislike that in the traditional method, the ``control displacements'' do not have any physical meaning. It can only be used
    to interpolate the real displacements within its span.

\begin{equation}
    \left\{ u \right\}=
    \sum_{i=0}^n
    R(u) \left\{u^{(N)}\right\}
\label{iso_eq:displacement_interpolation}
\end{equation}


\documentclass[10pt]{article}
\usepackage[usenames]{color} %used for font color
\usepackage{amssymb} %maths
\usepackage{amsmath} %maths
\usepackage[utf8]{inputenc} %useful to type directly diacritic characters

\usepackage{geometry}
 \geometry{
 a4paper,
 total={210mm,297mm},
 left=20mm,
 right=20mm,
 top=15mm,
 bottom=15mm,
 }

\begin{document}
\noindent Considering a simpler Langevin dynamics under the foloowing equations
\begin{align*}
  \begin{cases}
    \dot{x}&=v \\
    \dot{v}&=\frac{F(x)}{m}-\gamma v+\frac{\xi(t)}{m}
  \end{cases}
\end{align*}
where $(x,v)$ are position and velocity. $F(x)=-\partial_x U(x)$ which is the position-dependent force. The $m$, $\gamma$ are mass and friction coefficient. The $\xi(t)$ represents randomness defines the Wiener noise in the Ito convention:
\begin{align*}
\langle\xi(t)\xi(t')\rangle=\frac{2m\gamma}{\beta}\langle dW_tdW_{t'}\rangle=\frac{2m\gamma}{\beta}\delta(t-t')
\end{align*}
This equation describes a dynamics equivalent to an evolution of probability density f(x,v;t) through Fokker-Planck equation:
\begin{align*}
\frac{\partial f(x,v;t)}{\partial t}=-\hat{L}f(x,v;t)
\end{align*}
where
\begin{align*}
\hat{L}\equiv\frac{F(x)}{m}\frac{\partial}{\partial v}+v\frac{\partial}{\partial x}-\gamma\left(\frac{\partial}{\partial v}v+\frac{1}{m\beta}\frac{\partial^2}{\partial v^2}\right)
\end{align*}
The probabaility density f evolves through
\begin{align*}
f(x,v, t+\Delta t)=e^{-\Delta t\hat{L}}f(x,v;t)
\end{align*}
The operator $\hat{L}$ is separated into three parts, and the interation is approximated through Lie-Trotter formula,
\begin{align*}
e^{-\Delta t\hat{L}}&\approx e^{-\Delta t\hat{L}_x}e^{-\Delta t\hat{L}_v}e^{-\Delta t\hat{L}_{\gamma}} \\
&\approx e^{-(\Delta t/2)\hat{L}_{\gamma}}e^{-(\Delta t/2)\hat{L}_v}e^{-(\Delta t/2)\hat{L}_x}e^{-(\Delta t/2)\hat{L}_x}e^{-(\Delta t/2)\hat{L}_v}e^{-(\Delta t/2)\hat{L}_{\gamma}}
\end{align*}
where
\begin{align*}
\hat{L}_x&=v\frac{\partial}{\partial x} \\
\hat{L}_v&=\frac{F(x)}{m}\frac{\partial}{\partial v} \\
\hat{L}_{\gamma}&=-\gamma\left(\frac{\partial}{\partial v}v+\frac{1}{m\beta}\frac{\partial^2}{\partial v^2}\right)
\end{align*}
The integration can then be taken into two parts.
\subsubsection*{I. $\hat{L}_{\gamma}$ - Diffusion in momentum space}
This first part is a diffusion in momentum space
\begin{align*}
f(v^{+}\equiv v(t^{+}), t^{+})=e^{-(\Delta t/2)\hat{L}_{\gamma}}f(v,t)
\end{align*}
which corresponds to the evolution of $f$
\begin{align*}
\frac{\partial f}{\partial t}=-\frac{\hat{L}_{\gamma}}{2}f=\frac{\gamma}{2}\left[\frac{\partial }{\partial v}(vf)+\frac{1}{m\beta}\frac{\partial^2 f}{\partial v^2}\right]
\end{align*}
which has a solution [``The Fokker-Planck Equation Second Edition", H. Risken] of
\begin{align*}
f(v^{+}, t^{+}|v,t)\propto\exp\left[-\frac{m\beta}{2} \frac{\left(v^{+}-v e^{-\frac{\gamma}{2}\Delta t}\right)^2}{1-e^{-\gamma\Delta t}} \right]=\exp\left[ -\frac{1}{2}\left(\frac{v^{+}-\mu_v}{\sigma_v}\right)^2 \right]
\end{align*}
This step can then be performed by sampling from a Normal distribution $N(\mu_v, \sigma_v)$,
\begin{align*}
v(t^{+})&\sim N(\mu_v, \sigma_v) \\
&=\mu_v+\sigma_v N(0,1) \\
&=v(t)e^{-\frac{\gamma}{2}\Delta t}+\sqrt{ \frac{1}{m\beta}\left(1-e^{-\gamma\Delta t}\right) } N(0,1) \\
&=c_1 v(t)+c_2 N(0,1)
\end{align*}
\subsubsection*{I. $(\hat{L}_x, \hat{L}_v)$ - Deterministic canonical transformation in phase space}
Both $\hat{L}_v$ and $\hat{L}_x$ are deterministic canonical transformation in phase space.  Applying $f(v',t')=f(v(t+\Delta t), t+\Delta t)=e^{-(\Delta t/2)\hat{L}_v}f(v(t),t)$, and observing that by doing the transformation $p=t+\frac{F(x)}{2m}v$ and $q=\frac{F(x)}{2m}t-v$, we have
\begin{align*}
\frac{\partial f}{\partial t}+\frac{F(x)}{2m}\frac{\partial f}{\partial v}=\left[1+\left(\frac{F(x)}{2m}\right)^2\right] \left.\frac{\partial f}{\partial p}\right|_q=0
\end{align*}
The solution is then $f(v,t)=f(q)=f(\frac{F(x)}{2m}t-v)$. With the boundary condition $v(0)=v$, the solution must be a delta function
\begin{align*}
f(v,t)&=f(\frac{F(x)}{2m}t-v)=\delta(\frac{F(x)}{2m}t-v) \\
\Delta v&=\frac{F(x)}{2m}\Delta t
\end{align*}
Similarly, the operation of $\hat{L}_x$ during $\Delta t/2$ is equivalent to $\Delta x=v(\Delta t/2)$.
\end{document}