\section{Basic Notions}
\subsection{Definitions}
\begin{definition}
    A subset $U\subset\mathbb C$ is open if $\forall u\in U,\exists r>0,D_r(u)\subset U$.
\end{definition}
If we identify $\mathbb C$ with $\mathbb R^2$ in the obivous way, and give $\mathbb R^2$ the usual topology, then $U\subset\mathbb C$ is open iff it is open in $\mathbb R^2$.\\
We shall be interested in functions $f:U\to\mathbb C$ where $U$ is open.
\begin{definition}
    Let $U\subset\mathbb C$ be open and $f:U\to\mathbb C$.
    We say $\lim_{z\to c}f(z)=A$ if $\forall\epsilon>0,\exists\delta>0$,
    $$0<|z-c|<\delta\implies |f(z)-f(c)|<\epsilon$$
    $f$ is continuous at $c\in U$ if $\lim_{z\to c}f(z)=f(c)$.
    $f$ is continuous in $U$ if it is continuous everywhere in $U$.
\end{definition}
We can always write $f(x+iy)=u(x,y)+iv(x,y)$ where $u,v:\mathbb U\to\mathbb R$ where $U$ is open in $\mathbb R^2$.
It is easy to see that $\mathbb C$ inherits the continuity condition in $\mathbb R^2$ since it inherits the topology from it.
So we have that the continuity of $f$ is equivalent to that of $u,v$.
\begin{definition}
    Let $f:U\to\mathbb C$ be as before, and $w\in U$.
    We say $f$ is differentiable at $w$ if the limit
    $$\lim_{z\to w}\frac{f(z)-f(w)}{z-w}$$
    exists.
    If it exists, we say $f$ is differentiable at $w$ and has derivative equals the limit.\\
    If $f$ is differentiable everywhere in an open neighbourhood of $w$, then we say $f$ is holomorphic at $w$.
    \footnote{Some authors use the word `analytic'}\\
    We say $f$ is holomorphic on $U$ if it is holomorphic everywhere on $U$.
    (Or equivalently it is differentiable everywhere.)
\end{definition}
Rules that can be obtained from real differentiation by first principle mostly extends to complex differentiation.
For example, polynomials are differentiable everywhere, and rational functions are differentiable in the subset (which is open as the complement of a finite subset) where they are defined.
\subsection{The Cauchy-Riemann Equation}
Although complex differentiation exhibits similar definition as real differentiation, they behave very differently.
A natural question is, is the differentiability of $f$ behave the same as the differentiability of $u,v$?
The answer is no.
\begin{theorem}[Cauchy-Riemann Equation]
    $f:U\to\mathbb C$ is differentiable at $w=c+id$ iff the functions $u,v$ are differentiable at $(c,d)$ and they satisfies $u_x=v_y,u_y=-v_x$.
\end{theorem}
\begin{proof}
    By definition, $f$ is differentiable at $w$ with derivative $f^\prime(w)=p+iq$ if and only if
    $$\lim_{z\to w}\frac{f(z)-f(w)-f^\prime(z)(z-w)}{|z-w|}=0$$
    which is equivalent to the case where we seperate the real and imaginary part, which is just to say
    $$
    \begin{cases}
        \lim_{(x,y)\to(c,d)}\frac{u(x,y)-u(c,d)-(p(x-c)-q(y-d))}{\|(x-c,y-d)\|}=0\\
        \lim_{(x,y)\to(c,d)}\frac{v(x,y)-v(c,d)-(p(x-c)+q(y-d))}{\|(x-c,y-d)\|}=0
    \end{cases}$$
    which happens iff $Du(c,d)=(p,-q)$ and $Dv(c,d)=(p,q)$.
    The theorem follows.
\end{proof}
\begin{remark}
    Just because $u,v$ has partial derivatives satisfying Cauchy-Riemann equation does not guarantee the total differentiability.
    A deeper question is what if we require them to be so on an open set, but that goes beyond the scope of the course.
\end{remark}
From the theorem, we have obtained an expression of the complex derivative $f^\prime=u_x+iv_x$.
If we just want to show that the differentiability of $f$ at $c+id$ implies the existence of partials of $u,v$ satisfying Cauchy-Riemann equations, then we can just proceed by taking the limit of $z-w\to 0$ from both axes.
\begin{example}
    $f(z)=\bar{z}$ is nowhere differentiable since it does not satisfy the first C-R equation.
\end{example}
\begin{remark}
    Complex differentiability is much more restrictive than real ones.
    Of course, we will exhibit examples to justify it.
\end{remark}
\begin{theorem}[Liouville Theorem]\label{holo_bdd_const}
    If $f:\mathbb C\to \mathbb C$ is holomorphic and bounded, then $f$ is constant.
\end{theorem}
\begin{theorem}\label{infinite_holo}
    Let $U$ be an open set in $\mathbb C$, then if $f:U\to\mathbb C$ is holomorphic, so is $f^\prime$.
\end{theorem}
\begin{theorem}\label{uniform_holo}
    A sequence of holomorphic functions $f_n:U\to\mathbb C$ on the same open domain, and $f_n\to f$ uniformly, then $f$ is holomorphic.
\end{theorem}
By Theorem \ref{infinite_holo}, holomorphic functions are infinite differentiable, so any order of partial derivatives of $u,v$ exists, so we can differentiate the Cauchy-Riemann equation one more time to get $u_{xx}+u_{yy}=0$, so $u$ is a harmonic function, similarly $v$ is harmonic as well.
So the real and imaginary part of a holomorphic function are harmonic.
\begin{corollary}
    Let $f=u+iv:U\to\mathbb C$, suppose that $u,v$ has continuous partial derivatives in $U$ and $u,v$ satisfy Cauchy-Riemann Equation, then $f$ is holomorphic.
\end{corollary}
\begin{proof}
    Immediate from what we have got and a result from Analysis and Topology.
\end{proof}
\begin{remark}
    Once we know Theorem \ref{infinite_holo}, the converse of the preceding corollary follows.
\end{remark}
\begin{definition}
    A curve is a continuous map $\gamma:[a,b]\to\mathbb C$.
    It is called $C^1$ if $\gamma^\prime$ exists and is continuous on $[a,b]$.\\
    An open subset $U\subset\mathbb C$ is path connected if for any $z,w\in U$, there is a curve $\gamma:[a,b]\to U$ such that $\gamma(a)=z,\gamma(b)=w$.\\
    A non-empty open path-connected subset of $\mathbb C$ is called a domain.
\end{definition}
\begin{corollary}
    Let $U$ be a domain and $f:U\to\mathbb C$ is homomorphic. If $f^\prime\equiv 0$, then $f$ is constant.
\end{corollary}
\begin{proof}
    Proved in Analysis and Topology.
\end{proof}
\subsection{Power Series}
\begin{theorem}
    For any sequence of complex numbers $(c_n)$, there is some $R\in [0,\infty]$ such that the power series
    $$\sum_{n=0}^\infty c_n(z-a)^n$$
    such that it converges absolutely for $|z-a|<R$ and diverges for $|z-a|>R$.\\
    If $R\in\mathbb R_{>0}$, and if $0<r<R$, then the convergence is uniform on $B_r(a)$ (or $D_r(a)$, they are equivalent anyways).
\end{theorem}
\begin{definition}
    Such an $R$ is called the radius of convergence.
\end{definition}
\begin{proposition}
    $R=\frac{1}{\lambda}$ where $\lambda=\limsup_{n\to\infty}\sqrt[n]{|c_n|}$.
\end{proposition}
\begin{theorem}
    Define $f$ on the disk $D_R(a)$ by
    $$f(z)=\sum_{n=0}^\infty c_n(z-a)^n$$
    where $R>0$ is the radius of convergence of the series.
    Then\\
    1. $f$ is holomorphic on this disk,\\
    2. And
    $$\sum_{n=0}^\infty (n+1)c_{n+1}(z-a)^n$$
    is convergent on $D_R(a)$ and is the derivative of $f$.\\
    3. $f$ has derivatives of all orders on $D_R(a)$ and $f^{(n)}(a)=c_nn!$.\\
    4. If $f$ vanished in some open disk $D_r(a)$ with $0<r<R$, then it vanishes on the whole of $D_R(a)$.
\end{theorem}
\begin{proof}
    3 follows from 2 and 4 follows from 3.\\
    WLOG $a=0$.
    To prove 1 and 2, let $|z|<R$ and choose $\rho$ such that $|x|<\rho<R$, then we have
    $$\lim_{n\to\infty}\frac{n|c_n||z|^{n-1}}{|c_n|\rho^n}=\lim_{n\to\infty}\frac{n}{\rho}\left( \frac{|z|}{\rho} \right)^n\to 0$$
    so the derived series also converges on $D_R(0)$.\\
    To show that $f$ is holomorphic with derivative equals the derived series, we fix some $w\in D_R(0)$.
    Note that $f$ is differentiable at $w$ with derivative being the derived series $\sigma$ if and only if the function
    $$g(h)=
    \begin{cases}
        \frac{f(z)-f(w)}{z-w}\text{, if $z\neq w$}
        \sigma\text{, if $z=w$}
    \end{cases}$$
    is continuous at $w$.
    But $g(z)=\sum_{n=0}^\infty h_n(z)$ where
    $$h_n(z)=\begin{cases}
        c_n(z^{n-1}+z^{n-2}w+\cdots+w^{n-1})\text{, for $z\neq w$}\\
        nc_nw^{n-1}\text{, for $z=w$}
    \end{cases},n>0;h_0(z)=0$$
    Note that each $h_n$ is continuous.
    So the continuity of $g$ at $w$ follows from the uniform convergence of the series.
    Again we take $0<|w|<r<R$, so for any point $z\in D_r(0)$, we have
    $$|h_n(z)|=|c_n||z^{n-1}+z^{n-2}w+\cdots+w^{n-1}|\le n|c_n||r|^{n-1}$$
    which proves the uniform convergence.
    So 1 and 2 are proved.
\end{proof}
\begin{definition}
    The exponential function is defined as
    $$e^z=\exp(z)=\sum_{k=0}^\infty\frac{z^k}{k!}$$
\end{definition}
\begin{definition}
    An entire function is a function that is holomorphic on all of $\mathbb C$.
\end{definition}
\begin{proposition}
    1. $\exp$ is entire with derivative equals to itself.\\
    2. $\exp(z+w)=\exp(z)\exp(w)$.\\
    3. $\exp(z)\neq 0$.\\
    4. $\exp(z)=1\iff z\in2\pi i\mathbb Z$.\\
    5. $\exp:\mathbb C\to\mathbb C\setminus\{0\}$ is surjective.
\end{proposition}
\begin{proof}
    Trivial.
\end{proof}
\begin{definition}
    Given $z\in\mathbb C$, we say $w\in\mathbb C$ is a logarithm of $z$ of $e^w=z$.
\end{definition}
Note that $z$ has a logarithm iff $z\neq 0$.
If $w_1,w_2$ are two logrithms of $z$, then we immediately have $w_1-w_2\in 2\pi i\mathbb Z$.
\begin{definition}
    Let $U\subset\mathbb C^\star=\mathbb C\setminus\{0\}$.
    A branch of the logarithm on $U$ is a continuous function $\lambda:U\to\mathbb C$ such that $\exp\circ\lambda=\operatorname{id}$.
\end{definition}
\begin{remark}
    1. If such $\lambda$ does exixt, then $\lambda$ is holomorphic on $U$ by simply computation.\\
    2. From the definition, it follows that $|z|=e^{\operatorname{Re}\lambda(z)}$, so it is immediate that any branch $\lambda$ has $\operatorname{Re}\lambda(z)=\log|z|$.
\end{remark}
\begin{definition}
    The principal branch of logarithm is the function $\operatorname{Log}:U_1=\mathbb C\setminus\{x\in\mathbb R:x\le 0\}\to\mathbb C$ given by $\operatorname{Log}(z)=\log|z|+i\arg z$ where the value of $\arg$ is taken in the open interval $(-\pi,\pi)$.
\end{definition}
Obviously in this open set it is a inverse of $\exp$.
To see the continuity, observe that the map $z\mapsto z/|z|$ is continuous and the map $\theta\mapsto e^{i\theta}$ is a homeomorphism $(-\pi,\pi)\to S\setminus\{-1\}$
\begin{proposition}
    1. $\operatorname{Log}$ is holomorphic on $U_1$ with derivative $1/z$.\\
    2. For $|z|<1$,
    $$\operatorname{Log}(z)=\sum_{n=1}^\infty\frac{(-1)^{n+1}z^n}{n}$$
\end{proposition}
\begin{proof}
    Trivial.
\end{proof}
\begin{remark}
    There is no way to extend $\operatorname{Log}$ to the whole of $\mathbb C^\star$ and remain a branch since
    $$\lim_{\theta\to \pi^+}e^{i\theta}\neq\lim_{\theta\to \pi^-}e^{i\theta}$$
    In fact there is no branch that can be such extended.
\end{remark}
Using $\exp$ and $\operatorname{Log}$, we can construct the familiar functions we had in real analysis, like the trigonometrics, hypertrigonometrics, etc., in the way we know (like $\cos(z)=(e^{iz}+e^{-iz})/2$, etc).
We can also define, for $z\notin \mathbb R_{\le 0}$, $z^\alpha=\exp(\alpha\operatorname{Log}(z))$.\\
Let $f:U\to\mathbb C$ be a holomorphic function.
A number $w\in U$ having $f^\prime(w)$ is nice in the sense that $f$ is invertible there by Inverse Function Theorem with $(f^{-1})^\prime(f(w))=1/f^\prime(w)$.
One can check that $f^{-1}$ also satisfies the Cauchy-Riemann equation, hence is also holomorphic.\\
It is also nice in a geometric sense as $f$ preserves angles at that point.
Let $\gamma_1,\gamma_2:[-1,1]\to\mathbb C$ be $C^1$ curves with $\gamma_1(0)=\gamma_2(0)=w$ and $\gamma_1^\prime(0)\gamma_2^\prime(0)\neq 0$, then since $(f\circ \gamma_1)^\prime(0)=f^\prime(w)\gamma^\prime_1(0)$.
Similar for $\gamma_2$.
Since $f^\prime(w)\neq 0$, we have
$$\frac{\gamma^\prime_1(0)}{\gamma_2^\prime(0)}=\frac{(f\circ\gamma_1)^\prime(0)}{(f\circ\gamma_2)^\prime(0)}$$
So $f$ preserves angles.
\begin{definition}[Conformal Equivalence]
    Let $U$ be an open set.
    A holomorphic function $f:U\to\mathbb C$ is called conformal at $w\in U$ if $f^\prime(w)\neq 0$.\\
    If for domains $D,D'$ there is a holomorpic bijection $f:D\to\tilde{D}$ which is conformal at every point, then we say $D,D'$ are conformal equivalent.
\end{definition}
\begin{example}
    1. Any Mobius map $f:\mathbb C\cup\{\infty\}\to\mathbb C\cup\{\infty\}$ is a conformal equivalence.\\
    2. The map $z\mapsto z^n$ from $\{\mathbb z\in\mathbb C^\star:0\le\arg z\le \pi/n\}$ to the upper half plane $\mathbb H=\{z\in\mathbb C:\operatorname{Im}z>0\}$ is conformal.\\
    3. The exponential $\exp:\{z\in\mathbb C:-\pi<\operatorname{Im}(z)<\pi\}\to\mathbb C\setminus\mathbb R_{\le 0}$.\\
    4. Consider $g:z\mapsto (z-i)/(z+i)$ with $g:\mathbb H\to D_1(0)$.
    It is also conformal.
\end{example}
\begin{theorem}[Riemann Mapping]
    Let $D$ be a domain bounded by a simple closed curve, then it is conformally equivalent to the unit disk.\\
    More generally, any simply connected domain that is not the whole complex plane is conformally equivalent to the unit disk.
\end{theorem}


% https://www.mathworks.com/examples/matlab/mw/matlab-ex13677222-solve-nonstiff-odes

\PassOptionsToPackage{hyphens}{url}% allow line breaks at hyphens in urls
\documentclass[12pt]{matlatex}
\usepackage{examples}
\usepackage{pgfplots}
\usepackage{caption}

\begin{document}

\section*{Numerical integration of coupled ODEs}

This example is based that given in the Mathworks web page%
\ \url{https://www.mathworks.com/examples/matlab/mw/matlab-ex13677222-solve-nonstiff-odes}.
It uses the Matlab function {\tt\small ODE45} to integrate a coupled pair of ordinary differential equations -- the van der Pol equation with $\mu=1$.

\begin{matlab}
   [t,y] = ode45(@vdp1,[0 20],[2; 0]);

   % Plot of the solution
   plot(t,y(:,1),'-o',t,y(:,2),'-o')
   xlabel('Time t')
   ylabel('Solution y')
   legend('y_1','y_2')

   print(gcf,'example_02_fig.png','-dpng');

   dlmwrite ('example_02.txt',[t';y(:,1)';y(:,2)']','delimiter',' ','precision','% .8e');

   function dydt = vdp1(t,y)
   %VDP1  Evaluate the van der Pol ODEs for mu = 1
   %
   %   See also ODE113, ODE23, ODE45.

   %   Jacek Kierzenka and Lawrence F. Shampine
   %   Copyright 1984-2014 The MathWorks, Inc.

   dydt = [y(2); (1-y(1)^2)*y(2)-y(1)];
   end
\end{matlab}

\clearpage

\begin{figure}
   \centering
   \IfFileExists{example_02_fig.png}%
   {\includegraphics[width=0.75\textwidth]{example_02_fig.png}}{Failed to create png plot.}
   \caption{Solution of van der Pol equation ($\mu = 1$) using ODE45.}
\end{figure}

\clearpage

\pgfplotsset{compat=newest}
\pgfplotsset{width=0.60\textwidth,height=0.40\textwidth}

\begin{center}
   \begin{tikzpicture}
      \begin{axis}
         [xmin= 0.0, xmax=20.0,
          ymin=-3.0, ymax=3.0,
          xlabel=$\text{Time }t$, ylabel=$\text{Solution }y$,
          grid=major, grid style={dashed,gray!30},
          legend entries = {$y_1$, $y_2$}]
          \addplot[blue, mark=o]   table [x index=0, y index=1]{example_02.txt};
          \addplot[red, mark=o]    table [x index=0, y index=2]{example_02.txt};
      \end{axis}
   \end{tikzpicture}
   \captionof{figure}{Solution of van der Pol equation ($\mu = 1$) using ODE45.}
\end{center}

\vfill

\begin{latex}
   \begin{tikzpicture} % requires \usepackage{pgfplots}
      \begin{axis}
         [xmin= 0.0, xmax=20.0,
          ymin=-3.0, ymax=3.0,
          xlabel=$\text{Time }t$, ylabel=$\text{Solution }y$,
          grid=major, grid style={dashed,gray!30},
          legend entries = {$y_1$, $y_2$}]
          \addplot[blue, mark=o]   table [x index=0, y index=1]{example_02.txt};
          \addplot[red, mark=o]    table [x index=0, y index=2]{example_02.txt};
      \end{axis}
   \end{tikzpicture}
   \captionof{figure}{Solution of van der Pol equation ($\mu = 1$) using ODE45.} % requires \usepackage{caption}
\end{latex}

\end{document}


\part{Classification Algorithms}
\paragraph{Classification}
\begin{itemize}
	\item Input: a database $D = {x_1, x_2, \dots, x_n}$ of tuples, a set of classes $C = {C_1, C_2, \dots, C_m}$
	\item Output: a \textbf{mapping} $f: D\rightarrow C$, where each $x_i$ is assigned to a class.
\end{itemize}
\section{Generalized Linear Models}
\subsection{Component of GLM}
\begin{itemize}
	\item Random component: identifies \textbf{dependent variable $\mu$} and \textbf{its probability function}
	\item Systematc component: identifies the set of \textbf{explanatory variables -- predictors} ($X_1, \dots, X_k$)
	\item Link function: a \textbf{linear} function that links the dependent variable and all explanatory variables.
	$$g(\mu) = \beta_0 + \beta_1 X_1 + \beta_2 X_2 + \dots + \beta_k X_k$$
\end{itemize}
\subsection{Common Link Functions}
\begin{itemize}
	\item \textbf{Identity Link: linear regression}
	$$g(\mu) = \mu = X\beta$$
	\item \textbf{Logit Link: logistic regression}
	$$g(\mu) = \ln(\frac{\mu}{1 - \mu}) = X\beta $$
	\item \textbf{Log Link: Poisson regression}
	$$g(\mu) = \log(\mu) =  X\beta$$
\end{itemize}
\section{Logistic Regression: Binary Classification}
\begin{itemize}
	\item Idea:
	\begin{itemize}
		\item Gauss-Markov assumptions need to be fulfilled to implement an OLS-Estimator

		$\rightarrow$ more \textbf{generalized models} needed to relax the assumptions for OLS

		\item Predicting \textbf{categorical dependent variables}: a \textbf{classification} problem.
		\item Limitation from linear regression in classification:
		\begin{itemize}
			\item prediction values $\hat{y}$ should range within [0, 1], linear regression model prediction \textbf{exceeds this range}.
			\item \textbf{violation of Homoscedasticity}: residuals $e_i$ doesn't have constant variance since the true Y only takes two values(0/1). The distribution of the residuals is no longer a normal distribution.
			\item \textbf{violation of No Autocorrelation}: overall residuals of the model follows a systematic pattern, it's positive on one side and negative on the other side $\rightarrow$ Autocorrelation
		\end{itemize}
	\end{itemize}
\end{itemize}

\subsection{The Logistic Regression Model}
The \textbf{binary} logistic regression model described in \textbf{log odds/logit}:
$$\text{log odds} = \ln(\frac{p(X)}{1 - p(X)}) $$
$$\ln(\frac{p(X)}{1 - p(X)}) = \beta_0 + \beta_1 X + \varepsilon$$
\begin{itemize}
	\item Modeling Input: $X_i$, 0/1

		  Output: a Logit-model
	\item p(X): probability that Y = 1 given X.
	\item \textbf{log odds / logit}: range ($-\infty, +\infty$), the log-ratio of Y = 1 to Y = 0 given X
\end{itemize}

The logistic regression model described in \textbf{odds}:
$$odds = \frac{p(X)}{1 - p(X)}$$
$$\frac{p(X)}{1 - p(X)} = e^{\beta_0 + \beta_1 X}$$
\begin{itemize}
	\item \textbf{odds}: range [$0, +\infty$), the ratio of Y = 1 to Y = 0 given X
	\begin{itemize}
		\item p(X) = 0.5, odds = 1
		\item p(X) $< 0.5, \rightarrow 0$, odds $\rightarrow 0$
		\item p(X) $> 0.5, \rightarrow 1$, odds $\rightarrow +\infty$
	\end{itemize}
\end{itemize}

\subsection{The Logistic Function}
$$Pr[Y|X] = p(X) = \dfrac{e^{\beta_0 + \beta_1 X}}{1 + e^{\beta_0 + \beta_1 X}}$$
\begin{itemize}
	\item Prediction Input: $X_i$, $\beta_i$

	\textbf{Output: $p(X)$}

	\item Range p(X): [0,1]
	\begin{itemize}
		\item $\beta_0 + \beta_1 X = 0$:  p(X) = 0.5
		\item $\beta_0 + \beta_1 X  \uparrow$:  p(X) $\rightarrow 1$
		\item $\beta_0 + \beta_1 X  \downarrow$:  p(X) $\rightarrow 0$
	\end{itemize}
	\item $\beta_0$: regression constant, moves the curve \textbf{left/right}
	\item $\beta_1$: regression slope, defines \textbf{steepness} of the curve. $\beta_1 \uparrow$, steepness $\uparrow$
	\item This can be reformed into the logistic regression model.
	\item Comparison Linear Model \& Logistic Model(the logistic function):
	\begin{figure}[H]
		\centering
		\includegraphics[width=0.6\textwidth]{logit.png}
	\end{figure}
\end{itemize}
Comparison p(X) \& odds \& log-odds:
\begin{figure}[H]
	\centering
	\includegraphics[width=0.25\textwidth]{logodds_odds.png}
\end{figure}

\subsection{Multiple Logistic Regression Model}
$$\ln(\frac{p(X)}{1 - p(X)}) = \beta_0 + \beta_1 X_1 + \beta_2 X_2 + \dots + \beta_k X_k$$
\begin{itemize}
	\item Interpretation of Coefficients $\beta_j$:
	\begin{itemize}
		\item \textbf{while keeping all other $x_j$ constant}, if $x_{ij}$ increase by 1, the \textbf{log-odds} will increase/decrease by $\beta_j$, or the \textbf{odds} will increase/decrease by $\mathbf{e^{\beta_j}}$.
	\end{itemize}
	\item Test of Multicollinearity: VIF / correlation coefficient
	\item Test of irrelevant variables: Wald-Test (significance of coefficients)
\end{itemize}

\subsection{Estimation of Coefficients: Maximum Likelihood Estimator}
\subsubsection{Intro}
The probability of one data point $x_i$ can be modeled as \textbf{Bernoulli trial}:
$$p^{y_i}(1-p)^{1-y_i}$$
The \textbf{likelihood function}:  the \textbf{joint probability} of observing the dependent variable values of random samples. $\rightarrow$ the product of all Bernoulli trials
$$L = \Pi_{i=1} p^{y_i}(1-p)^{1-y_i}$$
The logistic function can also be described as a \textbf{sigmoid function} in form $S(x) = \frac{e^x}{1 + e^x} = \sigma(x)$
$$P(X) = \sigma(\beta_0 + \beta_1 X)$$
\subsubsection{The Likelihood Function and Maximum Likelihood Estimator}
The \textbf{likelihood function for Logistic Regression Model}:
\begin{align*}
	L &= \Pi_{i=1} p^{y_i}(1-p)^{1-y_i} \\
	  &= \Pi_{i=1} \sigma(\beta_0 + \beta_1 X)^{y_i} \cdot (1 - \sigma(\beta_0 + \beta_1 X))^{1-y_i}
\end{align*}

The \textbf{Maximum Likelihood Estimator}: \textbf{maximizes the joint probability} of observing the set of dependent variables of the random samples.
\\ \ \\
Process:
\begin{itemize}
	\item take the log: $$LL = \ln(L) = \Sigma_{i=1} ( y_{i} \ln(p) + (1 - y_i)\ln(1-p))$$
	\item \textbf{maximize} LL:
	\begin{align*}
		\beta = \arg\max_{\beta}(LL) &= \arg\max_{\beta} [\Sigma_{i=1} ( y_{i} \ln(p) + (1 - y_i)\ln(1-p))] \\
									 &= \arg\max_{\beta} [\Sigma_{i=1} ( y_{i} \ln(\sigma(\beta_0+\beta_1X)) + (1 - y_i)\ln(1-\sigma(\beta_0 + \beta_1X)))]
	\end{align*}
	\item Method: \textbf{Gradient Ascent}
	\begin{figure}[H]
		\centering
		\includegraphics[width=0.65\textwidth]{gradientascent.png}
	\end{figure}
	\begin{itemize}
		\item initial start point
		\item select a step size $\alpha$
		\item compute \textbf{partial derivatives} and \textbf{maximizes the function} $f(x^{(k)} + \alpha \bigtriangledown f(x^{(k)}))$
	\end{itemize}
	\item \textbf{Gradient}(partial derivatives) of the LL-Function: \textbf{chain rule}

	 with $z = \beta_0+\beta_1X$

	$$\frac{\partial LL}{\beta_j} = \Sigma_{i=1} \frac{\partial LL}{\partial p} \cdot \frac{\partial p}{\partial z} \cdot \frac{\partial z}{\partial \beta_j}$$
	\begin{align*}
		\frac{\partial LL}{\partial p} &= \frac{y_i}{p} - \frac{1 - y_i}{1 - p} \\
		 \frac{\partial p}{\partial z} &= \sigma(z) \cdot(1 - \sigma(z)) \\
		 \frac{\partial z}{\partial \beta_0} &= 1 \text{, } \frac{\partial z}{\partial \beta_j} = x_j\\
	\end{align*}
	\begin{align*}
		\frac{\partial LL}{\beta_0} &= \left[ \frac{y_i}{p} - \frac{1 - y_i}{1 - p}\right]  \sigma(z) \cdot(1 - \sigma(z)) = \left[ y_i - \sigma(X\beta) \right] \\
		\frac{\partial LL}{\beta_j} &= \left[ \frac{y_i}{p} - \frac{1 - y_i}{1 - p}\right]  \sigma(z) \cdot(1 - \sigma(z))\cdot x_j = \left[ y_i - \sigma(X\beta) \right] x_j
	\end{align*}
\end{itemize}
\subsection{Quality Metrics of the Model}
\begin{itemize}
	\item \textbf{Null Model}: all predictors $x_i$ has no impact. The model is explained only by the intercept.
	\item \textbf{Fitted Model}: the model is explained by p predictors and 1 intercept.
\end{itemize}

\subsubsection{Null Deviance}
It measures how well the response is explained by \textbf{only the intercept (no predictors)}.
$$\text{null deviance} = -2 \ln(L(null))$$

\subsubsection{Residual Deviance}
$$\text{residual deviance} = -2 \ln(L(fitted))$$
\begin{itemize}
	\item residual deviance $\downarrow$, model quality $\uparrow$
	\item difference between null and residual deviance $\uparrow$, model quality $\uparrow$
\end{itemize}

\subsubsection{AIC}
Additional penalizing term to get a balance between the \textbf{goodness of fit} and \textbf{simplicity of model}.

$$AIC = \text{residual deviance} + 2 \cdot \#\text{parameters in model}$$

\begin{itemize}
	\item AIC $\downarrow$, model quality $\uparrow$
\end{itemize}

\subsubsection{McFadden $\mathbf{R^2}$} the ratio of improvement from null model to fitted model.
$$R_{McFadden}^2 = 1 - \frac{LL(fitted)}{LL(null)} = 1 - \frac{\text{residual deviance}}{\text{null deviance}}$$
\begin{itemize}
	\item model quality $\uparrow$ , LL(fitted) $\ll$ LL(null), $R^2 \rightarrow 1$
	\item model quality $\downarrow$ , LL(fitted) $\approx$ LL(null), $R^2 \rightarrow 0$
	\item rule of thumb: > 0.2 acceptable, > 0.4 ok.
\end{itemize}

\subsubsection{Likelihood Ratio Test}
Does the fitted model explain significantly more variance than null model?
\begin{itemize}
	\item $H_0$: The fitted model explains \textbf{no more variance} than null model

	$H_1$: The fitted model explains \textbf{significantly more variance}.
	\item test statistic:
	$$D = -2 \ln (\frac{L(null)}{L(fitted)})$$
	\item Distribution: $\chi^2$-Distribution
\end{itemize}

\subsubsection{Significance Test of Coefficients: Wald Test}
\begin{itemize}
	\item $H_0$: $\beta_i = 0$

	$H_1$: $\beta_i \neq 0$
\end{itemize}

\subsubsection{Error Rates}
comparing the predicted classification and the actual classification:
\begin{itemize}
	\item percentage of correct Yes
	\item percentage of correct No
	\item overall percentage of correct predictions
\end{itemize}

\subsection{Model Interpretation}
\subsubsection{R-Result Interpretation}
\begin{figure}[H]
	\centering
	\includegraphics[width=0.75\textwidth]{logit_r.png}
\end{figure}
\begin{itemize}
	\item significance of coefficients: significant if $p < \alpha$ in $\alpha$-level
	\item model: difference between null and residual deviance, AIC
\end{itemize}

\subsubsection{Interpretation of Coefficients}
Effect of change in $x_{ij}$ in \textbf{one unit}:
\begin{figure}[H]
	\centering
	\includegraphics[width=0.9\textwidth]{logit_interpretation.png}
\end{figure}
\begin{itemize}
	\item $\beta_j >0$:
	\begin{itemize}
		\item If $x_{ij}$ increase by 1, then the \textbf{log odds} will \textbf{increase} by $\beta_j$, or the \textbf{odds} will \textbf{increase} by $\mathbf{e^{\beta_j}}$.
		\item If $x_{ij}$ increase by 1, then the \textbf{log odd ratio} is $\beta_j$, or the \textbf{odd ratio} is $e^{\beta_j}$.
	\end{itemize}
	\item $\beta_j < 0$:
	 \begin{itemize}
	 	\item If $x_{ij}$ increase by 1, then the \textbf{change of log odds} will \textbf{decrease} by $\beta_j$, or the \textbf{change of odds} will \textbf{decrease by a factor} of $\mathbf{e^{\beta_j}}$.
	 	\item If $x_{ij}$ increase by 1, then the \textbf{log odd ratio} is $\beta_j$, or the \textbf{odd ratio} is $e^{\beta_j}$.
	 \end{itemize}
\end{itemize}

\section{Poisson Regression: Binary Classification for Count Data}
\begin{itemize}
	\item Idea:
	\begin{itemize}
		\item \textbf{count variables (non-negative integers)} as dependent variables
		\item limitation of linear regression / OLS-Estimator
		\begin{itemize}
			\item linear model \textbf{predicts negative values}
			\item count data is often \textbf{highly skewed}: \#crimes committed -- most are 0.

			$\rightarrow$ violates normality assumption (residuals follows normal distribution) of OLS-Estimator
		\end{itemize}
	\end{itemize}
	\item Assumption: observed count follows a \textbf{Poisson distribution}.
	$$Pr(y|\mu) = \dfrac{e^{-\mu} \mu^y}{y!}$$
	\begin{itemize}
		\item $\mu$: expected count and expected variance $E(Y) = Var(Y) = \mu$
		\item y: observed count
	\end{itemize}

	\item Limitation:
	\begin{itemize}
		\item \textbf{Overdispersion}: $E(Y) = Var(Y) = \mu$ not met in real data.

		$\rightarrow$ underestimation of standard errors, potential overconfidence in result.

		$\rightarrow$ Alternative: negative binomial regression

		\item \textbf{Zero-inflation}: highly skewed observed data/predictions in 0.

		this can't be changed even with negative binomial regression.
	\end{itemize}

\end{itemize}
\subsection{Poisson Regression Model}
$$ln(\mu(x)) = \beta_0 + \beta_1 X_1 + \dots \beta_j X_i$$
\subsection{Estimation of Coefficients: Maximum Likelihood Estimator}
The random component(dependent variable) is:
$$Pr(Y|X) = p(X) = \dfrac{e^{-\mu} \mu^y}{y!} = \dfrac{e^{\beta xy} e^{-e^{\beta x}}}{y!}$$

The \textbf{likelihood function}:
$$L(\beta|X,Y) = \Pi_{i=1} p = \Pi_{i=1} \dfrac{e^{\beta x_iy_i} e^{-e^{\beta x}}}{y_i!}$$

The Maximum Likelihood Estimator:
$$\log L(\beta | X,Y) = \Sigma_{i=1} (\beta x_iy_i - e^{\beta x_i} - \log(y_i!))$$
\begin{itemize}
	\item Method: \textbf{Gradient Ascent}
\end{itemize}
\subsection{Model Interpretation}
R-Result interpretation is same as logistic regression.
\subsubsection{Interpretation of Coefficients}
Effect of change in $X_{ij}$ in \textbf{one unit}:
\begin{figure}[H]
	\centering
	\includegraphics[width=0.65\textwidth]{poisson_coeff.png}
\end{figure}
\begin{itemize}
	\item $\beta_j > 0$: If $x_{ij}$ increases by 1, then the \textbf{log-incidence rate} will \textbf{increase} by $\beta_j$, or the \textbf{incidence rate} will \textbf{increase by a factor} of $e^{\beta_j}$.
	\item $\beta_j < 0$: If $x_{ij}$ increases by 1, then the \textbf{log-incidence rate} will \textbf{decrease} by $\beta_j$, or the \textbf{incidence rate} will \textbf{decrease by a factor} of $e^{\beta_j}$.
\end{itemize}


\section{Parametric and Nonparametric Models}

A statistical model $\mathfrak{F}$ is a family of functions. \textit{Parametric models} can be parametrized by a finite number of parameters. For example, the Gaussian Normal distribution $\mathscr{\mu,\sigma^2}$ has the two parameters $\mu$ and $\sigma^2$. The general form is:

\begin{equation}
\mathfrak{F} = \{f(x;\theta): \theta \in \Theta \}
\end{equation}

Where $\theta$ is a vector of parameters and $\Theta$ is the parameter space. Elements of $\theta$ that are not of interest are called \textit{nuisance parameters}. \textit{Nonparametric models} cannot be described by a finite number of parameters. An example is an interpolating spline.

% fundamental concepts
\section{Fundamental Concepts in Inference}

% point estimations
\subsection{Point Estimators}
Point estimation refers to providing a single best guess of some quantity of interest. The point estimator is denoted with a had, i.e. $\hat{\theta}$. The point estimator for some quantity based on $n$ datapoints:

\begin{equation}
\hat{\theta}_n = g(X_1,X_2,...,X_n)
\end{equation}

% bias
\subsection{Bias}
A point estimator $\hat{\theta}$ has bias:

\begin{equation}
\mathrm{bias}(\hat{\theta}_n) = \mathbb{E}_\theta (\hat{\theta}) - \theta
\end{equation}

Where $\theta$ is the "true" value $\theta_n \rightarrow \theta$ as $n \rightarrow \infty$. $\mathbb{E}_{\theta}(r(X)) = \int r(x) f(x;\theta) \mathrm{d}x$.

% consistency
\subsection{Consistency}

A point estimator $\hat{\theta}_n$ is consistent if $\hat{\theta}_n \xrightarrow{P}\theta$. That is, it converges in probability to $\theta$. $\hat{\theta}_n$ is consistent with both bias and standard error approach $0$ as $n\rightarrow \infty$.

\subsection{Sampling Distribution, Standard Error}

The distribution of $\hat{\theta}_n$ is the \textit{sampling distribution}. The standard deviation of the distribution of $\hat{\theta}_n$ is the \textit{standard error}, denoted $\mathrm{se}$.

\begin{equation}
\mathrm{se}(\hat{\theta}_n) = \sqrt{\mathbb{V}(\hat{\theta}_n)}
\end{equation}

The standard error might depend on the unknown population CDF, in which case it is estimated. The point estimator for the standard error is then $\hat{\mathrm{se}}$.

\subsubsection{Example: Bernoulli Distribution}

Let $X_1,X_2,X_3 \sim \mathrm{Bernoulli}(p)$. The point estimator for $p$ based on $n$ datapoints is $\hat{p}_n = \frac{1}{n}\sum_{i=1}^n X_i$. Then the expected value of the point estimator $\mathbb{E}(\hat{p}_n) = \frac{1}{n}\sum_{i=1}^n \mathbb{E}(X_i) = p$, so that $\hat{p}_n$ is unbiased. The standard error is $\mathrm{se} = \sqrt{\mathbb{V}(\hat{p}_n)} = \sqrt{\frac{p(1-p)}{n}}$. The estimated standard error is $\hat{\mathrm{se}} = \sqrt{\frac{\hat{p}(1-\hat{p})}{n}}$.

% MSE
\subsection{Mean Squared Error}
The quality of a point estimator is often measured using the \textit{mean squared error}:

\begin{equation}
\mathrm{MSE} = \mathbb{E}_{\theta}(\hat{\theta}_n - \theta)^2 = \mathrm{bias}^2(\hat{\theta}_n) + \mathbb{V}_{\theta}(\hat{\theta}_n)
\end{equation}

% Asymptotically Normal
\subsection{Asymptotically Normal Estimators}
An asymptotically normal estimator satisfies:

\begin{equation}
\frac{\hat{\theta}_n - \theta}{\mathrm{se}} \xrightarrow{dist} \mathscr{N}(0,1)
\end{equation}

% confidence sets
\subsection{Confidence Sets}

A confidence set $C_n$ is the subset of parameters $\theta$ that has a greater than $1-\alpha$ probability of containing the true value of $\theta$.

\begin{equation}
\mathbb{P}_{\theta}(\theta \in C_n)\geq 1-\alpha\ \mathrm{\ for\ all\ }\theta\in\Theta
\end{equation}

\subsubsection{Normal-Based Confidence Intervals}

If $\hat{\theta}_n \approx \mathscr{N}(\theta,\hat{\mathrm{se}}^2)$ and $z_{\frac{\alpha}{2}} = \Phi^{-1}(1-\frac{\alpha}{2})$ the value of the standard normally distributed random variable $Z$ at which $\mathbb{P}(-\frac{\alpha}{2} < Z < \frac{\alpha}{2}) = 1-\alpha$, then, transforming backwards, $\mathbb{P}(\hat{\theta}_n - z_{\frac{\alpha}{2}}\hat{\mathrm{se}} < \theta < \hat{\theta}_n + z_{\frac{\alpha}{2}}\hat{\mathrm{se}} ) = 1-\alpha$. Hence, the confidence interval for a normally distributed point estimator $\hat{\theta}_n$ is:

\begin{equation}
C_n = (\hat{\theta}_n - z_{\frac{\alpha}{2}}\hat{\mathrm{se}},\hat{\theta}_n + z_{\frac{\alpha}{2}}\hat{\mathrm{se}})
\end{equation}

For a 95\% confidence interval $\alpha=0.05$ and $z_{\frac{\alpha}{2}} = 1.96 \approx 2$, so that the confidence interval is approximately $\hat{\theta}_n \pm 2 \hat{\mathrm{se}}$.

\subsubsection{Pointwise and Uniform Asymptotic Confidence Intervals}

A \textit{pointwise asymptotic} confidence interval requires:

\begin{equation}
\liminf_{n\rightarrow \infty} \mathbb{P}_{\theta}(\theta \in C_n) \geq 1-\alpha,	\ \ \forall \theta \in \Theta
\end{equation}

A \textit{uniform asymptotic} confidence interval requires:

\begin{equation}
\lim_{n\rightarrow \infty}\inf_{\theta \in \Theta} \mathbb{P}_{\theta}(\theta \in C_n) \geq 1-\alpha
\end{equation}

\subsection{Pivots, Pivotal Quantities}
Pivotal quantities are functions of the sample that are independent of the distribution parameters of the sample. For example, if $X=(X_1,X_2,...,X_n)$ is a random sample from a distribution with parameters $\theta$, then the random variable $g(X,\theta)$ is a pivot if it has the same distribution regardless of the choice of $\theta$. An example is the $z$-score $z = \frac{x-\mu}{\sigma}$. The $z$-score may require population parameters to be known (in this case $\mu$,$\sigma$) but it's distribution is independent of them.


% !TeX root = ../main.tex

\chapter{Theory}\label{theory}

\section{SVM}
A \acrfull{svm} is a machine learning algorithm which tries to separate binary-class training data. The \gls{svm} achieves this by searching for a hyperplane or a set of hyperplanes to separate the points of training data in a multidimensional space. There might be several possible hyperplanes that sufficiently separate the data but a \gls{svm} searches for the hyperplane with the biggest perpendicular distance between the hyperplane and the borders of both classes that should be separated. The space between the hyperplane and the closest of the training data points is defined as the margin. If the algorithm finds the maximal margin it has also found the optimal hyperplane. See Figure \ref{fig:svmHyperplaneSeparation} for an example of possible hyperplane separations.

\begin{figure}[ht]
	\centering
	\includegraphics[scale=0.5]{figures/theorySVM_hyperplanes}
	\caption{Separation of two classes by hyperplanes in 2 dimensional space. $H_1$ does not separate, $H_2$ and $H_4$ separate but the margin is very slim, $H_3$ separates with a much better margin.}
	\label{fig:svmHyperplaneSeparation}
\end{figure}

\subsection*{Formal definition}
Since a \Gls{svm} is a binary classifier a data point $x$ has to either belong into class A or class B. Let the training dataset of $n$ points be represented as $x_0,\dots,x_n$ and the target values be $y_0,\dots,y_n \in {-1,+1}$. The values of $y_n$ should be
\begin{equation}
	y_i=
		\begin{cases}
		+1 & \text{if } x_i \in \text{class } {\color{red}A} \\
		-1 & \text{if } x_i \in \text{class } {\color{blue}B}
		\end{cases}.
\end{equation}
The optimal hyperplane should separate all vectors $x_i$ with a value of $y_i=1$ {(class ${\color{red}A}$)} and those with a value of $y_i=-1$ {(class ${\color{blue}B}$)} so that the distance between those two groups is maximal \cite{Thome2012}. Let the hyperplane be
\begin{equation}
	D(x)=W \bullet x+b
\end{equation}

where $W$ is a vector normal to the hyperplane (see Figure \ref{fig:svmHyperplaneDistance}) and $b$ is the bias \cite{Boser1992}. $D(x)$ has to be calculated so that
\begin{equation}
	x \in
	\begin{cases}
	\text{class } {\color{red}A} & \text{if } D(x) > 0 \\
	\text{class } {\color{blue}B} & \text{if } D(x) < 0
	\end{cases}
\end{equation}
is true. The distance between a point $X$ and the hyperplane $D(x)$ is given as
\begin{equation}
\frac{D(X)}{\|W\|}
\end{equation}
which is illustrated in Figure \ref{fig:svmHyperplaneDistance} \cite{Thome2012}. Assuming that the data is linear separable {(as in Figure \ref{fig:svmHyperplaneSeparation})} we can then select two hyperplanes ${\color{red}D(x)>0}$ and ${\color{blue}D(x)<0}$ so that there are no points in between them and try to maximize the distance between those two hyperplanes {(in Figure \ref{fig:svmHyperplaneSeparation} $H_2$ and $H_4$)} which is then given as
\begin{equation}
\frac{2}{\|W\|}.
\end{equation}
To maximize the distance we must therefore minimize $\|W\|$ but with the constraint {(\ref{eq:svmHyperplaneCondition})} that no data gets between the two hyperplanes {(hard-margin classification)}. Both constraints
\begin{equation}
	\begin{split}
	x_i \bullet w+b \geq +1 \quad \text{if } {\color{red}y_i=+1}\\
	x_i \bullet w+b \leq -1 \quad \text{ if } {\color{blue}y_i=-1}
	\end{split}
\label{eq:svmHyperplaneCondition}
\end{equation}
can be combined to $y_i(x_i\bullet w + b)\geq 1\quad \forall i, i=1\dots n$. The solution for a maximum margin can therefore be found by solving
\begin{equation}
\begin{split}
\operatorname*{arg\,min}_{w, b}	\left\{\frac{1}{2}\|W\|^2\right\} \text{subject to} \\
y_i(x_i\bullet w + b)-1\geq 0 \quad \forall i, i=1\dots n
\end{split}
\end{equation}
with the Lagrangian multiplier method \cite{Thome2012}.

If the data is not linear separable we have to relax the constraints from \ref{eq:svmHyperplaneCondition} and allow some points in the margin {(soft-margin-classification)}.

Classification of unknown data is now simply done through a calculation of the decision function $D(x)$
\begin{equation}
\text{prediction}
	\begin{cases}
	x \in \text{class } {\color{red}A} & \text{if } D(x) > 0 \\
	x \in \text{class } {\color{blue}B} & \text{if } \text{otherwise}
	\end{cases}
\end{equation}

\begin{figure}[ht]
	\centering
	\includegraphics[scale=0.4]{figures/theorySVM_distanceToW}
	\caption{Visualization of optimal margin and hyperplane in 2 dimensional space. The distance of Point $X$ to $D(x)$ is $\frac{D(X)}{\|W\|}$. Source Bosner et al. \cite{Boser1992}.}
	\label{fig:svmHyperplaneDistance}
\end{figure}

However, there are still cases where the data can not be separated by a linear function as shown in Figure \ref{fig:svmKernelTrick}. In this case the data is mapped into another higher dimensional feature space {(Kernel trick)} so that it becomes linear separable again.

\begin{figure}
	\centering
	\subfloat[Data not separable]{\includegraphics[width=55mm]{figures/theorySVM_kernelTrick1}}
	\subfloat[Data separable]{\includegraphics[width=55mm]{figures/theorySVM_kernelTrick2}}
	\subfloat[Data separable with hyperplane]{\includegraphics[width=55mm]{figures/theorySVM_kernelTrick3}}
	\caption{Illustration of Kernel trick. Data in {(a)} is not separable but if the data gets mapped into a higher dimensional space {(b)} a linear hyperplane can be fitted {(c)}. Figures plotted with code from \texttt{http://www.eric-kim.net/eric-kim-net/posts/1/kernel\_trick.html}}
	\label{fig:svmKernelTrick}
\end{figure}

\subsection*{Multiclass SVMs}
\label{subsec:svmMulticlass}
In case of a multiclass \gls{svm} problem the most popular approach is to use several different binary class \glspl{svm} instead of only one multiclass \gls{svm} \cite{Hsu2002, Duan2005, Thome2012}. There are two common solutions for the multiclass problem: One-VS-Rest and One-VS-One.

A One-VS-Rest multiclassifier with $n$ classes consists of $n$ distinct \glspl{svm}. Each \gls{svm} is trained on one of the classes as class A and the others as class B. Classification is done in a "winner-takes-it-all" fashion meaning that the \gls{svm} with the highest output function assigns the class label.

In the One-VS-One approach $n\frac{n-1}{2}$ \glspl{svm} are constructed. Each class is paired in a \gls{svm} with each other class. Classification is done in a "majority-vote" fashion so each \gls{svm} votes for its class and the class with the most votes is assigned as the class label.

\section{K-Nearest neighbors}
The \gls{knn} algorithm is a lazy classification algorithm which stores the complete training data in the learning phase \cite{Keller1985}. In the classification phase it searches for the $k$ points in the training data that are nearest to the point that should be classified. The label that gets the majority of votes from the $k$ closest points {(neighbors)} is the result of the classification. See Figure \ref{fig:knn3} for a three class example and $k=3$.

The most popular metric to get the distance between the neighbors is the Euclidean distance however the accuracy can greatly be improved by using a learned distance metric such as the Mahanalobis distance which tries to maximize the margin between classes \cite{Weinberger2005}.

\begin{figure}[ht]
	\centering
	\includegraphics[scale=0.45]{figures/theoryKNN_3}
	\caption{Example for a knn classification with $k=3$. The three nearest elements to the purple pentagon {(point that should be classified)} are marked with a square. Since two of the three neighbors are triangles the new point should also be a triangle.}
	\label{fig:knn3}
\end{figure}

\section{Convolutional Neural Networks}
\begin{figure}[ht]
	\centering
	\includegraphics[scale=0.35]{figures/theoryCNN_model}
	\caption{Sample architecture of a \gls{cnn}. Source: Christodoulidis and Anthimopoulos \cite{Christodoulidis2015}}
	\label{fig:cnnModel}
\end{figure}
A \acrfull{cnn} is a multi-layer-feed-forward neural network that emulates the processes in the visual cortex by employing feature detection with small convolutional filters. \glspl{cnn} generally outperform other gradient based learning techniques \cite{LeCun1998} and are currently state-of-the-art \cite{Russakovsky2015} for image recognition.

\glspl{cnn} usually consist of a succession of trainable convolutional and dense layers and pooling layers. Normally the first layers comprise of several convolutional and pooling layers for feature extraction. For the classifications of features extracted in the previous layers, the network has at least one dense layer of neurons before the last layer. The last layer is usually the output layer and for the standard classification task the number of neurons is equal to the number of classes to classify so that the output of a neuron is the probability of a specific class at the same time. Figure \ref{fig:cnnModel} shows an example of a \gls{cnn} architecture with two convolutional and pooling layers and a dense, fully connected layer before the output layer.

\subsection*{Convolutional layers}
\begin{figure}[ht]
	\centering
	\includegraphics[scale=0.45]{figures/theoryCNN_filter}
	\caption{96 convolutional filters of size $3\times 11\times 11$ in the first layer. Filters trained on \gls{ilsvrc}-2010. Source: Krizhevsky et al. \cite{Krizhevsky2012}}
	\label{fig:cnnFilter}
\end{figure}
Convolutional layers contain multiple convolutional filters or kernels. They convolve the image to extract image features. Each filter convolves the output of the previous layer. In case of the first layer the filters operate directly on the input image. In later layers the collect more complex features from previous layers. Convolution happens by calculating the dot-product of the input matrix with the convolutional filter. Filters in \glspl{cnn} usually overlap which helps to get a better translation invariance since each pixel is convolved multiple times by the same filter. The main goal of convolution in \glspl{cnn} is to get distinctive image features. These convolutional filters are not handcrafted like filters in typical feature detectors. They learn the best filters by learning large quantities of data and change over the time and are ultimately able to extract features that are characteristic for the problem. Often these kernels extract simple features like horizontal, vertical or diagonal edges {(figure \ref{fig:cnnFilter})}. For food prominent first layer kernels typically includes many color kernels that specialize in extracting a range of similar colors \cite{Christodoulidis2015}. The basic idea behind convolutional filters is that many features like edges or corners appear in multiple locations in images. By using the filters as a sliding window one filter is able to detect the feature in any position.

Therefore the main befit of using convolutional filters is the reduction of learnable weights. Filters are very small {(7x7 as the maximum size for a current \gls{cnn} \cite{Szegedy2014})} but are applied on the whole image in a sliding window approach. This means that a single convolutional filter shares weights for the image which significantly improves learning because it reduces the number of parameters to learn \cite{LeCun1998}. GoogLeNet, a 22-layer \gls{cnn}, uses \gls{rgb} input images with a size of 224x224 \cite{Szegedy2014}. A normal Neural Net without convolutions would not be able to handle this input size.

\subsection*{Pooling layers}
Once a feature is detected the exact position becomes less important. Only the spatial relation to other features remains valuable. Knowing the exact position of a feature may even be harmful for generalization because the model becomes less invariant to position \cite{LeCun1998}. One way to solve this problem and reduce the number of weights to learn is subsampling. In \glspl{cnn} this normally occurs in pooling layers. The most common type pooling layer is currently max-pooling. This filter basically divides the input into $z\times z$ blocks and extracts the maximum value of each block. A popular max-pooling layer is a $z=2$, stride $s=2$ layer. This layer downsamples the input by two. Recently, overlapping max-pooling layers have become popular \cite{Szegedy2014}. They do not downsample as much as non-overlapping pooling layers {(in fact $z=3$ and $s=2$ outputs the same dimensions as the input)} but they seem to slightly decrease the error \cite{Krizhevsky2012}.

\subsection*{Overfitting and Dropout}
\label{subsec:overfittingDropout}
Although \glspl{cnn} need fewer parameters to train high dimensional inputs than conventional neural nets, they also have the tendency to overfitt. Overfitting is a common problem in machine learning that occurs if models are very complex. An overfitted model is optimized across the training data and describes or "fits" this data so well that it can not explain unseen data \cite{Falkenauer1998}. Figure \textbf{XX} shows an overifitting neural network. After epoch \textbf{XX} the error on the training set continues to decrease but the error on the validation set increases which means that although the net gets better at predicting the training data it gets worse for predicting unseen data.

Dropout is a novel way for neural nets to reduce the problem of overfitting by randomly dropping out nodes from the model. This forces the net to generalize better \cite{Srivastava2014}.

\subsection*{Learning rate and momentum}
\label{subsec:learningMomentum}
Neural networks try to find the global minimum of their cost function by adjusting the network weights. If there is only one minimum this is fairly easy as the network only has to adjust the weights towards the downward gradient. Real world scenarios however, are much more complex and include many local minima as well.

Learning rate and momentum are measurements of how weights are adjusted in backpropagation in search of the global minima. The learning rate denotes the magnitude of the weight change. That means that networks with a high learning rate advance faster along the gradient. Momentum is a term that multiplies a fraction of the previous weight update to the weight adjustment. By applying momentum, the steps the network takes on the gradient towards the minimum get bigger with each iteration. This helps to increase the speed of learning and may also prevent the network get stuck on local minima and saddle points because of the "momentum" the network simply "steps over" these points.

\section{Colour features}
\label{sec:theory_colour}
Todo {(maybe)}

\section{Feature detectors and descriptors}
A common approach in image classification is the use of feature detectors and feature descriptors. A feature detector detects interest points or keypoints in images. Ideally those keypoints are invariant to image transformations like rotation, scale or illumination changes so that the points can be found even if the image is rotated or taken from another perspective. In most cases keypoints are corners as corners be can be localized quite easily. Knowing that a corner exist however does not help much with recognition. An algorithm also needs to describe the characteristics of an interest point so that it is possible to compare different interest points. The part of describing a keypoint is done by keypoint descriptors.

\subsection[HOG]{Histograms of oriented gradients (HOG)}
\begin{figure}
	\centering
	\includegraphics[scale=0.45]{data/images/theory/theoryHOG_VisualizationGradients}
	\caption{Visualization of HOG descriptor.}
	\label{fig:hogVisualization}
\end{figure}

\acrfull{hog} is a popular image descriptor for human detection by Dalal and Triggs in 2005 \cite{Dalal2005}. The algorithm makes use of **TODO: WORD MISSING** by describing the distribution local gradients. This approach is believed to be based on biological processes of neurons in the primary visual cortex \cite{Lowe2004}.

The first step is to divide the image into small blocks with a size of 16x16 pixels and a 50\% overlap for better results. Each block is then subdivided by four 8x8 pixel cells. For each of these cells, gradient intensities are computed by applying the 1-D centered point derivative convolution mask horizontally and vertically:
\begin{equation}
	D_x=
	\begin{bmatrix}
	-1 & 0 & 1
	\end{bmatrix}
	\quad
	D_y=
	\begin{bmatrix}
	-1 \\ 0 \\ 1
	\end{bmatrix}
\end{equation}
The magnitude of the gradients is
\begin{equation}
	m=\sqrt{(I\ast D_x)^2+(I\ast D_y)^2}
\end{equation}
where $I$ is the image cell and the gradient orientation is given as
\begin{equation}
	\theta = \arctan \left[\ \frac{(I\ast D_y)}{(I\ast D_x)}\right]\text{.}
\end{equation}
To achieve a better illumination invariance like shadowing it is useful to contrast-normalize each cell.

To get a feature vector for each cell a 9-bin histogram of the gradient orientations {(0\degree - 180\degree)} is calculated. The gradient orientations $\theta$ are then scaled by the corresponding magnitude $m$. In Figure \ref{fig:hogVisualization} there is a visualization of gradient histograms. The gradient histogram in the center row on the right, is a single horizontal line meaning that the gradients in this cell are all horizontal which is logical because the original image in this cell is a horizontal edge.

To get the feature vector for the whole image all histograms are concatenated to form a large gradient histogram.

\gls{hog} is not rotation invariant which does not matter for human detection as humans tend to always have the same upright orientation. However for food rotation invariance is quite important.

\subsection[LBP]{Local Binary Pattern (LBP)}
\begin{figure}[ht]
	\centering
	\includegraphics[width=\linewidth]{figures/theoryLBP_coding}
	\caption{Example for a LBP coding of the center pixel with value 6 (a). The neighboring pixels are thresholded against the center value (b) and multiplied by values in (c) with the result in (d). Source: Ojala et al. \cite{Ojala1999}}
	\label{fig:lbpCoding}
\end{figure}
\acrfull{lbp} is a simple texture descriptor which was originally proposed by Ojala et al. in 1994 \cite{Ojala1994}. For the calculation of the \gls{lbp} descriptor the 8 neighbors in a 3x3 block for each pixels are taken into account {(Fig \ref{fig:lbpCoding}a)}. The neighbors of the center pixel are then thresholded by comparing the value {(grayscale intensity)} of the pixel in the center to the intensities of the neighboring pixels. Pixels with a higher or equal value are getting labeled as "1" and "0" otherwise {(Fig \ref{fig:lbpCoding}b)}. Each position in the 3x3 block is then assigned a weight of $2^n$ where $n$ is ascending from left-to-right, top-to-bottom {(Fig \ref{fig:lbpCoding}c)}. In recent publications these weights are labeled clockwise ascending. The binary values are than multiplied by the corresponding weights {(Fig \ref{fig:lbpCoding}d)}. The values are then summed up and assigned as the new value for this "texture unit". The values can then be inserted into a histogram to form a 256-dim feature vector since there are 256 possible LBP values {($2^8=256$)}. Local binary patterns are invariant to grayscale changes because \gls{lbp} describes spatial texture structure but not contrast intensities \cite{Ojala1999}.

In the following years \gls{lbp} has been adapted and extended with additional contrast information \cite{Ojala1999} or different neighborhood sizes to achieve rotation invariance with uniform \glspl{lbp} \cite{Ojala2002}.

\subsection[SIFT]{Scale invariant feature transform (SIFT)}
\label{subsec:sift}
\acrfull{sift} is a very popular local image feature detector, descriptor and matcher by Lowe from 1999 \cite{Lowe1999}. It is invariant to uniform image scaling, translation and rotation and partially invariant to affine distortions, 3D viewpoint and illumination changes. It is also invariant to noise, partial occlusion and clutter \cite{Lowe1999}.

\subsubsection*{Keypoint detection}
%\begin{figure}
%	\centering
%	\begin{minipage}{.5\textwidth}
%		\centering
%		\includegraphics[width=0.45\linewidth]{data/images/theory/theorySIFT_gaussianScales}
%		\captionof{figure}{Image with different Gaussian scales.}
%		\label{fig:gaussianScales}
%	\end{minipage}%
%	\begin{minipage}{.5\textwidth}
%		\centering
%		\includegraphics[width=0.45\linewidth]{data/images/theory/theorySift_dog}
%		\captionof{figure}{Example of a Difference of Gaussians with $\sigma=1$ and $\sigma=3$.}
%		\label{fig:dog}
%	\end{minipage}
%\end{figure}

\begin{figure}
	\centering
	\begin{minipage}{.5\textwidth}
		\centering
		\includegraphics[width=0.45\linewidth]{figures/theorySIFT_scaleSpaceFiltering}
		\captionof{figure}{Sequence of Gaussian smoothings of a 1D graph with $\sigma$ increasing from bottom to top. Source Witkins \cite{Witkin1983}}
		\label{fig:scaleSpaceFiltering}
	\end{minipage}%
	\begin{minipage}{.5\textwidth}
			\centering
			\includegraphics[width=0.45\linewidth]{figures/theorySIFT_interestPointDetection}
			\captionof{figure}{Interest points are detected by searching for minima and maxima. The value of the center pixel {(marked with X)} is compared to its 26 neighbors {(marked with circles)} on the current and adjacent scales. Source: Lowe \cite{Lowe2004}}
			\label{fig:interestPointDetection}
	\end{minipage}
\end{figure}

The basic idea of \gls{sift}'s keypoint extraction follows Witkins scale-space filtering \cite{Witkin1983}. Witkins found out that by applying Gaussian smoothings at different scales of $\sigma$ on a 1D-graph he could find robust edges depending on the scale of $\sigma$ {(Figure \ref{fig:gaussianScales})}. Since this process can also be applied to 2D images scale-space filtering is a good way to find scale invariant interest points {(also called \gls{log})} {(See figure \ref{fig:gaussianScales} for an example)}. \gls{sift} however uses a much more computation efficient approximation of \gls{log} called \acrfull{dog} \cite{Lowe2004}. \gls{dog} can be computed by subtracting two adjacent scales from each other. Figure \ref{fig:dogScaleFiltering} shows the process of creating \glspl{dog} at different scales. Figure \ref{fig:dog} shows the result of a \gls{dog} operation on an image. \gls{sift} samples 3 scales per octave. After each octave the image size is halved and \gls{dog} is applied again on the smaller image {(Figure \ref{fig:dogScaleFiltering})}.

\begin{figure}[ht]
	\centering
	\includegraphics[scale=0.3]{figures/theorySIFT_dogScaleFiltering}
	\caption{For each octaves the image is blurred with Gaussian smoothings at rising scales. A \gls{dog} is formed by subtracting two adjacent scales. After each octave the image size is halved and the process repeated. Source: Lowe \cite{Lowe2004}}
	\label{fig:dogScaleFiltering}
\end{figure}

To check if a pixel qualifies as an interest point at a certain scale, \gls{sift} compares the values of the 3x3 neighboring pixels. If the pixel has the maximum or minimum value \gls{sift} then compares the neighbors on the scales above and below. If the pixel has indeed the lowest or highest value across all 27 points it is taken as a potential interest point {(figure \ref{fig:interestPointDetection})}. The cost of this operation is reasonably low because most of the pixels will be eliminated after the first few comparisons \cite{Lowe2004}. To filter weak interest points, \gls{sift} tries to eliminate low contrast points and points on edges since edges are not very robust features. After the elimination of weak points, \gls{sift} computes a 32-bin gradient orientation histogram in the interest point neighborhood. Orientations are weighted by the magnitude of the gradient. Each bin represents 10\degree of the 360 possible orientations. The bin with the highest peak is then selected as the dominant orientation for this keypoint. Bins with at least 80\% of the highest bin value are selected to form a additional new keypoint.

\subsubsection*{Keypoint description}
\begin{figure}[ht]
	\centering
	\includegraphics[scale=0.2]{figures/theorySIFT_descriptor}
	\caption{Creation of the \gls{sift} descriptor. Around the keypoint a 16x16 region {(this image is only half the size)} is taken and gradient orientations are calculated. The region forms 16 4x4 blocks with 8-bin gradient histograms. Source Lowe \cite{Lowe2004}}
	\label{fig:siftDescriptor}
\end{figure}
The \gls{sift} keypoint descriptor relies on gradient orientation histograms. They are more stable features than just raw intensity values and less sensitive to 3D rotation. To calculate a descriptor for a keypoint, a neighborhood region of 16x16 pixels is selected oriented along the dominant orientation of the keypoint. The 16x16 region is then divided into 4x4 blocks and for each of these blocks a 8-bin gradient orientation histogram is calculated {(similar to the keypoint extraction)}. This results in a 128 dimensional feature vector for each keypoint {($16\text{ blocks}*8\text{ bins per histogram}=128$)} \cite{Lowe2004}. Figure \ref{fig:siftDescriptor} shows the process of the descriptor creation.

\subsection[SURF]{Speeded up robust features (SURF)}
\label{subsec:surf}
\gls{surf} is a feature detector and descriptor which was proposed in 2006 \cite{Bay2006} and later revised in 2008 by Bay et al. \cite{Bay2008}. It was developed with the goal to provide a faster alternative for the popular \gls{sift} algorithm without trade offs in recognition performance.

\subsubsection*{Keypoint detection}
\begin{figure}[ht]
	\centering
	\includegraphics[scale=0.2]{figures/theorySURF_boxFilters}
	\caption{Left half: discretized and cropped Gaussian second order partial derivates in $y$-direction and $xy$-direction. Right half: corresponding box filter approximations. Gray ares are equal to zero. Source Bay et al. \cite{Bay2008}}
	\label{fig:surfBoxFilters}
\end{figure}
Like \gls{sift}, \gls{surf} also uses scale spaces to detect interest points. \gls{surf} however relies on an approximation of the Hessian matrix

The first step is to compute the integral image. In integral images pixel values are the sum of all pixels within a square between the pixel and the origin. These representations are efficient to compute and allow to find the sum of pixel values for any sized box within the image with only four arithmetic operations. Throughout the algorithm this benefit is leveraged to speed up detection and description.

They approximate the Hessian matrix by using box filters {(in the right of figure \ref{fig:surfBoxFilters})}. By using integral images in combination with box filters the process of scale space creation can be sped up significantly. The benefit of box filters contrary to Gaussians which have to be applied iteratively is, that box filters can be applied directly at the original image which makes parallelization possible. In addition \gls{surf} does not downscale after each octave. It scales the image up which has the advantage of not causing aliasing. To localize interest points across scales \gls{surf} applies non-maximal suppression on a 3x3x3 neighborhood.

\subsubsection*{Keypoint description}
To achieve rotation invariance \gls{surf} computes a dominant orientation for each interest point from a circular 6$s$ neighborhood by calculating Haar-wavelet responses in $x$ {(horizontal wavelet response)} and $y$ {(vertical wavelet response)} directions with $s$ being the scale of the interest point. Again the combination of integral images and wavelets helps to calculate Haar-wavelets efficiently. The dominant orientation is calculated by the use of a sliding rotating window with an angle of $\frac{\pi}{3}$. The wavelet responses are then represented as vectors weighted with a Gaussian of $\sigma=2.5s$. Horizontal and vertical vectors are added and the highest sum is the dominant interest point orientation.

To describe a keypoint, \gls{surf} forms a square neighborhood region oriented along the dominant orientation with a size of 20$s$ {(so keypoints at higher scales include a bigger neighborhood)}. The region is then divided into 4x4 smaller sub-regions. In each of these sub-regions \gls{surf} computes features at 5x5 evenly spaced sample points. Contrary to \gls{sift}, \gls{surf} does not utilize orientation histograms but again Haar-wavelet responses in horizontal $d_x$ and vertical $d_y$ direction relative to the dominant keypoint orientation. $d_x$ and $d_y$ are then summed up over each subregion. Those two and the sum of absolute response values $|d_x|$ and $|d_y|$ from the four dimensional feature vector for each subregion. Therefore each keypoint has a 64 dimensional feature vector taking into account that there are 16 subregions each with a four dimensional vector. To achieve additional invariance to contrast and illumination changes the feature vector is turned into a unit vector.

\subsection[ORB]{Oriented FAST and Rotated BRIEF (ORB)}
\gls{orb} is a combination of the corner detection algorithm \gls{fast} \cite{Rosten} and the binary descriptor \gls{brief} \cite{Calonder2010}. \gls{orb} was introduced by Rublee and Bradski in 2011 \cite{Rublee2011} as replacement for \gls{sift} for low-power devices and real time performance.

\subsubsection*{FAST keypoint detection}
\begin{figure}[ht]
	\centering
	\includegraphics[scale=0.3]{figures/theoryFAST_corners}
	\caption{FAST corner detection. Interest points are selected if at least 12 adjoining pixel values that are either above or below the center pixels intensity {(dashed line)}. Source: Rosten and Drummond \cite{Rosten}.}
	\label{fig:fastCorners}
\end{figure}
\gls{fast} is a corner detection algorithm by Rosten and Drummond from 2006 that does not rely on Gaussians or scale spaces. \gls{fast} compares 16 pixel intensities arranged on a circle with radius 9 around a possible corner point in the center. It is an interest point has at least 12 adjoining pixel values that are either above or below the center pixels intensity {(figure \ref{fig:fastCorners})}. \gls{fast} however, is prone to have many responses along edges which are weaker features so \gls{brief} tries to filter these edge responses by using a Harris corner measure. To achieve scale invariance \gls{brief} applies \gls{fast} on different image pyramid scale spaces. To also make the keypoints rotation invariant \gls{brief} calculates the intensity centroid which assumes that a corner's maximum intesity is offset from the actual center of the corner. This makes it possible to calculate the orientation by creating the vector from the intensity centroid to the actual corner point.

\subsubsection*{BRIEF keypoint description}
\gls{brief} is a binary descriptor from 2010 by Calonder et al. \cite{Calonder2010}. Unlike \gls{sift} or gls{surf} a binary descriptor creates the descriptor values by comparing intensity values of pixels in the neighborhood of the keypoint. If the intensity of pixel $a$ is lower than the intensity of pixel $b$ than the test yields "1" and "0" otherwise. To create the feature vector \gls{brief}  performs 256 binary tests and concatenates the results. The pixels that are compared are chosen by a Gaussian distribution around the center of the keypoint. Prior to these tests the image is smoothed. Additional in-plane rotation invariance is achieved by the proposed "steered" \gls{brief}. Steered \gls{brief} rotates the matrix of binary tests by a rotation matrix along the keypoint orientation. A lookup table with precomputed \gls{orb} features is computed with a 12\degree interval. However due to the very rotation by steered \gls{brief}, the binary tests loose most of their variance so the expressiveness of the descriptor suffers. To prevent this, Rublee and Bradski proposed an algorithm to "learn" the optimal location of the pixel-pairs that are compared for the binary tests.

\subsection[CenSurE]{Center surround extremas (CenSurE)}
\begin{figure}
	\centering
	\includegraphics[scale=0.2]{figures/theoryCenSurE_filters}
	\caption{Center surround filters. From left to right: decreasing accuracy and computational cost.  Source Agrawal et al. \cite{Agrawal2008}}
	\label{fig:censureFilters}
\end{figure}
The \gls{censure} feature detector was proposed by Agrawal et al. in 2008 as a framework of several bi-level filters for interest point detection \cite{Agrawal2008}. \gls{censure} is scale and rotation invariant and similar to the way \gls{sift} and \gls{surf} perform interest point detection. Contrary to the aforementioned keypoint detectors, \gls{censure} does not up- or downscale the image in scale space and therefore does not inherit the disadvantages of up- and downscaling like inaccurate keypoint locations.

To detect corners \gls{censure} employs approximations of \gls{log} with center-surround discrete bi-level filters. The closest approximation of \gls{log} is a circular bi-level filter as shown in figure \ref{fig:censureFilters}a. Bi-level means that the filter only contains two values namely $1$ or $-1$. The circular filter is rotation invariant due to its symmetry but it is also expensive to compute. Therefore \gls{censure} uses octagons and boxes as approximations of the circular filter. Like \gls{surf}, \gls{censure} also leverages integral images for box filters and applies them on five scales. Since a major shortcoming of box filters is the lack of rotation invariance Agrawal et al. also proposed an octagon shaped filter which resembles the circle filter much more closely. Integral images however do not work with diagonal edges and thus do not work with octagons. To make computation still possible Agrawal et al. introduced "slanted" integral images which are able to construct any trapezoidal area in the same amount of time as rectangular integral images.

To detect interest points, \gls{censure} performs non-maximal suppression over scale space across a 3x3x3 neighborhood. Weak interest points are filtered by threshold and edges are filtered the Harris corner measure.

%\subsection{DAISY}

\section{Bag of Words}
\acrfull{bow} or bag of features, bag of visual words or bag of keypoints is a simple, yet powerful image feature quantization method. The original idea of a \gls{bow} approach for images came from texture classification \cite{Leung2001}. Leung and Malik found out that by quantification of small texture patches {(textons)} in histograms they were able to accurately classify different textures. In 2004 Csurka et al. refined this approach and applied it to general image classification \cite{Csurka2004}.

The idea behind a \gls{bow} is very simple: During learning the \gls{bow} clusters image descriptors using $k$-means clustering. The clustered representation of the image descriptors is called vocabulary. The vocabulary size is $k$, so using $K=1000$ for the k-mean clustering results in a vocabulary of 1000 image features. Choosing $k$ is a trade-off between speed and accuracy and has to be determined during experiments. In an ideal world, a \gls{bow} clusters image features together that are unique and expressive so that we would have a vocabulary of bread, salad, meat and noodle features.

During classification the \gls{bow} gets a set of image descriptors and matches these features with the previously learned vocabulary. The output of this process is a histogram of vocabulary occurrences. Referring to the ideal world example, the \gls{bow} would get a set of features including many that look like bread, some that look like meat and a few that are similar to the salad texton. The output would be a histogram with a large value for the bread bin and some smaller values for the salad and meat bins. Giving this occurrence vector to a trained SVM would reveal that the image might be a Hamburger.

%\subsection{Matchers}

%\subsubsection*{Brute force}

%\subsubsection*{Flann Based}

%\subsubsection*{knnMatch}

%\section{Size and Volume estimation}


\chapter{}

\define{1} Let \(f(\cdot)\) be a density.
\begin{itemize}
    \item The class \(\left\{f(\cdot-\mu\}_{\mu \in \mathbb{R}}\right.\) is called the location family with standard density \(f(\cdot)\), and \(\mu\) is called the location parameter for the family.
    \item The class \(\{(1 / \sigma) f(\cdot / \sigma)\}_{\sigma \in(0, \infty)}\) is called the scale family with standard density \(f(\cdot)\), and \(\sigma\) is called the scale parameter for the family.
    \item By extension, the class \(\{(1 / \sigma) f((\cdot-\mu) / \sigma)\}_{(\mu, \sigma) \in \mathbb{R} \times(0, \infty)}\) is called the location-scale family with standard density \(f(\cdot)\).
\end{itemize}

\begin{ex}
    \begin{enumerate}[(a)]
        \item Let \(f(\cdot)\) be a density, let \((\mu, \sigma) \in \mathbb{R} \times(0, \infty)\). Show that \(X\) is a random variable with density
        \[
        \frac{1}{\sigma} f\left(\frac{\cdot-\mu}{\sigma}\right)
        \]
        if and only if there exists a random variable \(Z\) with density \(f(\cdot)\) such that \(X=\sigma Z+\mu\).
        \item Let \(X \sim\) Cauchy \((0,1)\), and recall that its density is given by
        \[
        f(x)=\frac{1}{\pi} \frac{1}{1+x^{2}}, \quad x \in \mathbb{R} .
        \]
        Consider the location-scale family generated by \(f\). We know that \(\mathbb{E}|X|=\infty\), and thus the parameters \(\mu\) and \(\sigma^{2}\) cannot be interpreted as the mean and the variance of \(X\). Notwithstanding this, show that
        \begin{enumerate}[(i)]
            \item \(\mathbb{P}(X \geq \mu)=\mathbb{P}(X \leq \mu)=\frac{1}{2} ;\)
            \item \(\mathbb{P}(X \geq \mu+\sigma)=\mathbb{P}(X \leq \mu-\sigma)=\frac{1}{4}\).
        \end{enumerate}  In other words, \(\mu-\sigma, \mu\) and \(\mu+\sigma\) are the first, second and third quartiles of \(f\) (hint: consider first the case \(\mu=0\) and \(\sigma=1\)).
        \item We say a family of c.d.f.s \(\{F(x \mid \theta)\}_{\theta \in \Theta}\) is stochastically increasing in \(\theta\) when \[\theta_{1}>\theta_{2}\Rightarrow F\left(x \mid \theta_{1}\right)\text{ is stochastically greater than }F\left(x \mid \theta_{2}\right), \]i.e., \(F\left(x \mid \theta_{1}\right) \leq F\left(x \mid \theta_{2}\right), x \in \mathbb{R}\), and \(F\left(x_{0} \mid \theta_{1}\right)<F\left(x_{0} \mid \theta_{2}\right)\) for some \(x_{0}\). Show that
        \begin{enumerate}[(i)]
            \item a location family is stochastically increasing in its location parameter;
            \item a scale family is stochastically increasing in its scale parameter if its support is \([0, \infty)\).
        \end{enumerate}
    \end{enumerate}
\end{ex}

\begin{solution}
    \begin{enumerate}
        \item
        $\Rightarrow:$ If $X\sim\frac{1}{\sigma}f\left(\frac{\cdot-\mu}{\sigma}\right)$, Let $Z=\frac{X-\mu}{\sigma}$,
        \begin{align*}
            P(Z\leqslant z)&=P\left(\frac{X-\mu}{\sigma}\leqslant z\right)\\
            &=P\left(X\leqslant\sigma z+\mu\right).
        \end{align*}
        \[
            F_Z(z)=F_X(\sigma z+\mu),
        \]
        \begin{align*}
            f_Z(z)&=\sigma f_X(\sigma z+\mu)\\
            &=\sigma \frac{1}{\sigma}f_X\left(\frac{\sigma z +\mu-\mu}{\sigma}\right)\\
            &=f_X(z)
        \end{align*}
        Which means $Z$ has density $f(\cdot)$.

        $\Leftarrow$: If $Z\sim f(z)$, $X=\sigma Z+\mu$,
        \begin{align*}
            P(X\leqslant x)&=P(\sigma Z+\mu\leqslant x)\\
            &=P(Z\leqslant\frac{x-\mu}{\sigma})\\
            &=F_Z\left(\frac{x-\mu}{\sigma}\right),
        \end{align*}
        So, \[
            f_X(x)=\frac{1}{\sigma}f_Z\left(\frac{x-\mu}{\sigma}\right).
        \]
        \item \begin{enumerate}[(i)]
            \item Because for a density function $f(\cdot)$, $\int f(x) \der x=1$, and Cauchy distribution is symmetric about the y-axis. So,
            \[
                P(X\leqslant0)=P(X\geqslant0)=1/2.
            \]
            \item \[
                P(X\leqslant1)=P(X\geqslant1)=\int_1^\infty\frac{1}{\pi}\frac{1}{1+x^2}\der x=\frac{1}{\pi}\arctan(x)\big|_1^\infty=1/4.
            \]
        \end{enumerate}
        \item \begin{enumerate}[(i)]
            \item For a location family: $F(x|\mu)=F(x-\mu)$. If $\mu_1>\mu_2$, then $x-\mu_1<x-\mu_2$,
            \[
                F(x|\mu_1)<F(x|\mu_2).
            \]
            So, it is stochastically increasing.
            \item For a location family: $F(x|\sigma)=1/\sigma F(x/\sigma)$. If $\sigma_1>\sigma_2$, then $1/\sigma_1<1/\sigma_2$,
            \[
                F(x|\sigma_1)<F(x|\sigma_2).
            \]
            So, it is stochastically increasing, too.
        \end{enumerate}
    \end{enumerate}
\end{solution}

\begin{ex}
    We saw the following result in class (an adaptation of Theorem 1.6.4 in Bickel and Doksum \((2002)\), p. 60\()\).

    \thm{1} Suppose that \(\{f(\boldsymbol{x} \mid \eta)\}_{\eta \in \mathcal{E}}\) is a canonical exponential family generated by \(\left(T_{k \times 1}, h\right)\), where \(\mathcal{E} \subseteq \mathbb{R}^{k}, \mathcal{E} \neq \emptyset\), is open. Then, the following are equivalent.
    \begin{enumerate}[(i)]
        \item For any \(\eta \in \mathcal{E}\), the random variables \(1, T_{1}(\boldsymbol{X}), \ldots, T_{k}(\boldsymbol{X})\) are not a.s. linearly dependent, i.e.
        \[
            \left(\mathbf{a}, a_{k+1}\right) \equiv\left(a_{1}, \ldots, a_{k}, a_{k+1}\right) \neq \mathbf{0} \Rightarrow \mathbb{P}_{\eta}\left(\langle\mathbf{a}, T(\boldsymbol{X})\rangle=a_{k+1}\right)<1,
        \]
        where \(T(\boldsymbol{X}):=\left(T_{1}(\boldsymbol{X}), \ldots, T_{k}(\boldsymbol{X})\right)^T\);
        \item \(\eta \in \mathcal{E}\) is identifiable;
        \item for any \(\eta \in \mathcal{E}, \operatorname{Var}_{\eta}(T)\) is symmetric positive definite;
        \item \(\mathcal{E} \ni \eta \mapsto A^{\prime}(\eta)\) is an injective mapping;
        \item \(\mathcal{E} \ni \eta \mapsto A(\eta)\) is a strictly convex mapping.
    \end{enumerate}
    We showed in class that \((i) \Leftrightarrow(i i) \Leftrightarrow(i i i)\). Finish the theorem.

    \rmk{1} Notice that, for a generic function \(f, f\) being strictly convex does not imply that \(f^{\prime \prime}(x)\) is a positive definite matrix.

    \emph{\bfseries Suggested route:} \((v) \Rightarrow(i v),(i i i) \Rightarrow(v),(i v) \Rightarrow(i i)\). The latter two claims are simpler than the former. So, for \((v) \Rightarrow(i v)\), consider the following definition and propositions.

    \define{2} Let \(C\) be a convex set, and consider \(F: C \rightarrow \mathbb{R}^{k}\). We say \(F\) is monotone on \(C\) if
    \[
    \left\langle F(x)-F\left(x^{\prime}\right), x-x^{\prime}\right\rangle \geq 0, \quad x \neq x^{\prime},
    \]
    and strictly monotone if strict inequality holds. Now prove the next two propositions.

    \prop{1} Let \(C\) be a convex set, and consider \(F: C \rightarrow \mathbb{R}^{k}\). If \(F\) is strictly monotone on \(C\), then \(F\) is injective.

    \prop{2} Let \(C\) be an open, convex set, and consider \(f: C \rightarrow \mathbb{R}\). If \(f\) is strictly convex and differentiable on \(C\), then \(f^{\prime}\) is strictly monotone.

    On a related note, it can be shown that the converse to Proposition \(2.2\) is also true.
\end{ex}

\begin{solution}
    \emph{First, prove propositions 2.1: }

    If $F$ is not injective, then $\exists\, x_1\neq x_2$, s.t. $F(x_1)=F(x_2)$. So, $F$ is not strictly monotone. And we get a contradiction, i.e. $F$ must be injective.

    \emph{Propositions 2.2 is the definition of convex function. }

    \emph{$v\to iv$: }

    Because $\eta$ is open set, and $f:\,\eta\to A(\eta)$ is strictly convex, then $A'(\eta)$ is strictly monotone, and is an injective mapping.

    \emph{$iii\to v$: }

    $Var_\eta(T)>0$, i.e. $A''(\eta)>0$, then $A(\eta)$ is strictly convex.

    \emph{$iv\to ii$: }

    If $\exists\,\eta_1\neq\eta_2$, s.t. $f(x|\eta_1)=f(x|\eta_2)$. Then
    \[
        A'(\eta_1)=E_{\eta_1}(T(X))=E_{\eta_2}(T(X))=A'(\eta_2).
    \]
    So, we ge the contradiction. i.e. $\eta$ is identifiable.
\end{solution}

\begin{ex}
    Let \(X \sim \mathcal{N}\left(\mu, \sigma^{2}\right)\). Consider any \(g \in C^{1}(\mathbb{R})\) such that \(\mathbb{E}_{\eta}\left|g^{\prime}(X)\right|<\infty\).
    \begin{enumerate}[(a)]
        \item (Stein's identity) Prove Stein's identity, i.e., that
        \[
        \mathbb{E}_{\eta}\left(\left(\frac{h^{\prime}(X)}{h(X)}+\sum_{i=1}^{k} \eta_{i} T_{i}^{\prime}(X)\right) g(X)\right)=-\mathbb{E}_{\eta} g^{\prime}(X)
        \]
        in canonical parametrization.

        Hint: consider first the case \(\mu=0\) and \(\sigma^{2}=1\). Let \(\phi(\cdot)\) be the density function. Now use the fact that \(\phi^{\prime}(z)=-z \phi(z)\) to rewrite \(\phi(x)\) as an integral from \(x\) to \(\infty\) and as another integral from \(-\infty\) to \(x\) in the expression
        \[
            -\mathbb{E} g^{\prime}(X)=-\mathbb{E} g^{\prime}(X)\left(1_{[0, \infty)}(X)+1_{(-\infty, 0)}(X)\right).
        \]
        \item By choosing polynomial \(g(\cdot)\) functions, use Stein's identity to calculate the third and fourth moments of the \(\mathcal{N}\left(\mu, \sigma^{2}\right)\) distribution.

        \rmk{2} Stein's identity is one aspect of the so-named Stein's method for establishing limits in distribution and their rates of convergence.
    \end{enumerate}
\end{ex}

\begin{solution}
    \begin{enumerate}[(a)]
        \item First, $\mu=0$, $\sigma^2=1$. Then
        \[
            \phi(z)=\frac{1}{\sqrt{2\pi}}\exp\left(-\frac{z^2}{2}\right).
        \]
        \[
            \phi'(z)=-z\frac{1}{\sqrt{2\pi}}\exp\left(-\frac{z^2}{2}\right)=-z\phi(z).
        \]
        Then
        \[
            \begin{aligned}
                E[g'(Z)]&=\int_{-\infty}^\infty g'(z)\phi(z)\der z\\
                &=\int_{-\infty}^0 g'(z)\phi(z)\der z + \int_{0}^\infty g'(z)\phi(z)\der z\\
                &=\int_{-\infty}^0 g'(z)\phi(z)\der z + \int_{0}^\infty g'(z)\int_z^\infty -\phi'(t)\der t \der z\\
                &=\int_{-\infty}^0 g'(z)\phi(z)\der z + \int_{0}^\infty g'(z)\int_z^\infty t\phi(t)\der t \der z\\
                &=\int_{-\infty}^0 g'(z)\phi(z)\der z + \int_{0}^\infty\int_z^\infty g'(z) t\phi(t)\der t \der z\\
                &=\int_{-\infty}^0 g'(z)\phi(z)\der z + \int_{0}^\infty\int_0^t g'(z) t\phi(t)\der z \der t\\
                &=\int_{-\infty}^0 g'(z)\phi(z)\der z + \int_{0}^\infty (g(t)-g(0))t\phi(t)\der t\\
                &=\int_{-\infty}^0 (g(t)-g(0))t\phi(t)\der t + \int_{0}^\infty (g(t)-g(0))t\phi(t)\der t\\
                &=E[z(g(z)-g(0))]=E[zg(z)-zg(0)]=E[zg(z)].
            \end{aligned}
        \]
        Let $g(x)=\phi\left(\frac{x-\mu}{\sigma}\right)$, then
        \[
            E[g'(x)]=E\left[\frac{1}{\sigma}\phi'\left(\frac{x-\mu}{\sigma}\right)\right]=\frac{1}{\sigma}E\left[\frac{x-\mu}{\sigma}\phi\left(\frac{x-\mu}{\sigma}\right)\right]=\frac{1}{\sigma}E\left[\frac{x-\mu}{\sigma}g(x)\right]
        \]
        For normal distribution:
        \[
            \begin{aligned}
            \phi(x)&=\frac{1}{\sqrt{2\pi}\sigma}\exp\left(-\frac{(x-\mu)^2}{2\sigma^2}\right)\\
            &=\frac{1}{\sqrt{2\pi}\sigma}\exp\left(-\frac{1}{2\sigma^2}x^2+\frac{\mu}{\sigma^2}x-\frac{\mu^2}{2\sigma^2}\right)
            \end{aligned}
        \]
        $h'(x)=0$, and $T'_1(x)=x$, $T'_2(x)=1$, $\eta=(-1/\sigma^2, \mu/\sigma^2)$.
        \[
            \begin{aligned}
                &\quad E\left(\left(\frac{h^{\prime}(X)}{h(X)}+\sum_{i=1}^{k} \eta_{i} T_{i}^{\prime}(X)\right) g(X)\right)\\
                &=E\left(\left(-\frac{1}{\sigma^2}x+\frac{\mu}{\sigma^2}\right) g(X)\right)\\
                &=E\left(\frac{\mu-x}{\sigma^2} g(X)\right)\\
                &=-\frac{1}{\sigma} E\left(\frac{x-\mu}{\sigma} g(X)\right)\\
                &=-E(g'(x)).
            \end{aligned}
        \]
        \item We can know that \[
            \frac{1}{\sigma} E\left(\frac{x-\mu}{\sigma} g(X)\right)=E(g'(x)),
        \]
        then let $g(x)=x^2$,
        \[
            E\left(\frac{x-\mu}{\sigma^2}x^2\right)=2\mu, \Rightarrow E(x^3)=3\mu\sigma^2+\mu^3.
        \]
        Let $g(x)=x^3$, then
        \[
            E\left(\frac{x-\mu}{\sigma^2}x^3\right)=3(\sigma^2+\mu^2), \Rightarrow E(x^4)=6\mu^2\sigma^2+\mu^4+3\sigma^4.
        \]
    \end{enumerate}
\end{solution}

\begin{ex}
    Prove that any (measurable) injective function of a minimal sufficient statistic is a minimal sufficient statistic.
\end{ex}

\begin{solution}
    Let $T(X)$ be a minimal sufficient statistic, $T^*(X)=f(T(X))$, $g(T^*(X))=T(X)$. Because $T(X)$ is minimal, there is another sufficient statistic $G(X)$ and a measurable function $h$, s.t.
    \[
        h(G(X))=T(X),
    \]
    then
    \[
        (f\circ h)(G(X))=f(T(X))=T^*(X).
    \]
    So, $T^*(X)$ is also minimal.
\end{solution}

\begin{ex}
    Prove that, for any two statistics \(T, T^{*}\),
    \[
    T(\mathbf{X})=T(\mathbf{Y}) \Leftrightarrow T^{*}(\mathbf{X})=T^{*}(\mathbf{Y}) \quad \mathcal{P}-a.s.
    \]
    if and only if there are functions \(f\) and \(g\) such that
    \[
    T(\mathbf{X})=f\left[T^{*}(\mathbf{X})\right], \quad T^{*}(\mathbf{X})=g[T(\mathbf{X})] \quad \mathcal{P}-a.s.
    \]
\end{ex}

\begin{solution}
    $\Leftarrow: $

    If $T^*(X)=T^*(Y)\Leftrightarrow T(X)=T(Y)$, then
    \[
        T^*(X)=T^*(Y)\rightarrow f(T^*(X))=f(T^*(Y))\rightarrow T(X)=T(Y).
    \]

    If $T(X)=T(Y)$, then
    \[
        T(X)=T(Y)\rightarrow g(T(X))=g(T(Y))\rightarrow T^*(X)=T^*(Y).
    \]

    $\Rightarrow: $
\end{solution}

\begin{ex}
    Establish the following claims.
    \begin{enumerate}[(a)]
        \item Let \(T\) and \(T^{*}\) be two statistics. Then, there are measurable functions \(f, g\) such that
        \[
            T=f\left(T^{*}\right)\text{ and }T^{*}=g(T)
        \]
        if and only if \(\sigma(T)=\sigma\left(T^{*}\right)\)
        (i.e., the \(\sigma\)-algebras generated by \(T\) and \(T^{*}\) are identical).

        Hint: recall, from probability theory, that if \(\mathbf{X}\) and \(\mathbf{Y}\) are two random vectors such that \(\mathbf{Y}\) is \(\sigma(\mathbf{X})\)-measurable, then there exists a measurable function \(\varphi\) such that \(\mathbf{Y}=\varphi(\mathbf{X})\).
        \item Conclude that if we drop the condition " \(\mathcal{P}\)-a.s." in the definition of equivalent statistics, the slightly stronger notion of equivalence we obtain is identical to the equality of the \(\sigma\)-algebras generated by the statistics in question.
    \end{enumerate}
\end{ex}

\begin{solution}
    \begin{enumerate}[(a)]
        \item We know that $T$ is $\sigma(T^*)-$measurable and $T^*$ is also $\sigma(T)-$measurable. So, $\sigma(T)=\sigma(T^*)$.

        On the other hand, because $\sigma(T)=\sigma(T^*)$, there are $f$ and $g$, s.t. $T=f(T^*)$, $T^*=g(T)$.
        \item
    \end{enumerate}
\end{solution}

\begin{ex}
    Let \(\mathcal{P}=\{f(\mathbf{x} \mid \theta)\}_{\theta \in \Theta}\) be a parametric family.
    \begin{enumerate}[(a)]
        \item Prove that \(\mathbf{X}\) is sufficient for \(\theta\).
        \item In addition, suppose \(\mathcal{P}\) corresponds to an i.i.d. sample. Prove that \(\left(X_{(1)}, \ldots, X_{(n)}\right)^T\) is sufficient for \(\theta\).
    \end{enumerate}
\end{ex}

\begin{solution}
    \begin{enumerate}[(a)]
        \item $\prod_{i=1}^nf(x_i|\theta)$, $h(X)=1$, $g(T(X), \theta)=g(X,\theta)=\prod_{i=1}^nf(x_i|\theta)$. So, $X$ is sufficient.
        \item $\prod_{i=1}^nf(x_{(i)}|\theta)$, $h(X)=1$, $g(T(X), \theta)=g((X_{(1)}, \cdots,X_{(n)})^T,\theta)=\prod_{i=1}^nf(x_{(i)}|\theta)$. So, $(X_{(1)}, \cdots,X_{(n)})^T$ is sufficient.
    \end{enumerate}
\end{solution}

\begin{ex}
    Let \(f\) be a positive integrable function over \((0, \infty)\), and let \(p(x \mid \theta)\) be the probability density function over \((0, \theta)\) the density defined by
    \[
        p(x \mid \theta)=\left\{\begin{array}{cl}
        c(\theta) f(x), & 0<x<\theta \\
        0, & \text { otherwise }
        \end{array}\right.
    \]
    If \(X_{1}, \ldots, X_{n}\) are i.i.d. with density \(p(\cdot \mid \theta)\), show that \(X_{(n)}\) is sufficient for \(\theta\).
\end{ex}

\begin{solution}
    \[
        \prod_{i=1}^n c(\theta)f(x_i)\mathbf{1}_{x_i<\theta}=c^n(\theta)\prod_{i=1}^nf(x_i)\mathbf{1}_{x_{(n)}<\theta},
    \]
    $h(x)=\prod_{i=1}^nf(x_i)$, $g(T(X),\theta)=c^n(\theta)\mathbf{1}_{x_{(n)}<\theta}$. So, $X_{(n)}$ is sufficient.
\end{solution}

\begin{ex}
    Let \(f\) be a positive integrable function over \(\mathbb{R}\), and let \(p(x \mid \xi, \eta)\) be the probability density function defined by
    \[
        p(x \mid \xi, \eta)=\left\{\begin{array}{cl}
        c(\xi, \eta) f(x), & \xi<x<\eta \\
        0, & \text { otherwise }
        \end{array}\right.
    \]
    If \(X_{1}, \cdots, X_{n}\) are i.i.d. with density \(p(\cdot \mid \xi, \eta)\), show that \(\left(X_{(1)}, X_{(n)}\right)\) is sufficient for \((\xi, \eta)\).
\end{ex}

\begin{solution}
    \[
        \prod_{i=1}^np(x_i|\xi, \eta)=\prod_{i=1}^nc(\xi, \eta) f(x_i)\mathbf{1}_{\xi<X_{(1)}, X_{(n)}<\eta},
    \]
    $h(x)=\prod_{i=1}^nf(x_i)$, $g(T(X),\xi,\eta)=c^n(\xi,\eta)\mathbf{1}_{\xi<X_{(1)}, X_{(n)}<\eta}$. So, $X_{(1)}, X_{(n)}$ is sufficient for $\xi,\eta$.
\end{solution}

\begin{ex}
    \begin{enumerate}[(a)]
        \item Prove that, if \(\mathbf{x}=\left(x_{1}, \ldots, x_{n}\right)\) and \(\mathbf{y}=\left(y_{1}, \ldots, y_{n}\right)\) have the same elementary symmetric functions
        \[
            \sum_{i} x_{i}=\sum_{i} y_{i}, \quad \sum_{i \neq j} x_{i} x_{j}=\sum_{i \neq j} y_{i} y_{j}, \quad \ldots, \quad \prod_{i} x_{i}=\prod_{i} y_{i}
        \]
        then \(\mathbf{y}\) is a permutation of \(\mathbf{x}\).
        \item Consider a family \(\mathcal{P}=\{f(\cdot \mid \theta)\}_{\theta \in \Theta}\), where \(f(\cdot \mid \theta)\) is an a.c. density for an i.i.d. sample for all \(\theta\), and set
        \[
            T(\mathbf{X})=\left(X_{(1)}, \ldots, X_{(n)}\right)^T .
        \]
        Now consider \(U=\left(U_{1}(X), \ldots, U_{n}(X)\right)^T\), where
        \[
            U_{1}(X)=\sum_{i} X_{i}, \quad U_{2}(X)=\sum_{i \neq j} X_{i} X_{j}, \quad \ldots, \quad U_{n}(X)=\prod_{i} X_{i} .
        \]
        Conclude that \(U\) and \(T\) are equivalent.
    \end{enumerate}
\end{ex}

\begin{solution}
    \begin{enumerate}[(a)]
        \item We know that elementary symmetric function $e_i(\cdot)$ is the coefficients for the expansion of the following polynomial:
        \[
            \prod_{i=1}^n(\lambda-X_i)=\sum_{j=0}^n(-1)^je_j(X_1,\cdots,X_n)\lambda^{n-j},
        \]
        $e_0(\cdot)=1$. So, we can get that
        \[
            \sum_{j=0}^n(-1)^je_j(X_1,\cdots,X_n)\lambda^{n-j}=\sum_{j=0}^n(-1)^je_j(Y_1,\cdots,Y_n)\lambda^{n-j}
        \]
        \[
            \Rightarrow \prod_{i=1}^n(\lambda-X_i)=\prod_{i=1}^n(\lambda-Y_i).
        \]
        From the multiplicative exchange law, we know that \(\mathbf{y}\) is a permutation of \(\mathbf{x}\).
        \item
        \[
            f_1(U(X))=\sum_iX_i=\sum_iX_{(i)}=g_1(T(X)),
        \]
        \[
            f_2(U(X))=\sum_{i<j}X_iX_j=\sum_{i<j}X_{(i)}X_{(j)}=g_2(T(X)),
        \]
        \[
            \cdots\cdots
        \]
        \[
            f_n(U(X))=\prod_{i}X_i=\prod_{i}X_{(i)}=g_n(T(X)).
        \]
        So, $T(X)=(g_1^{-1}\circ f_1,\cdots,g_n^{-1}\circ f_n)(U(X))$, $U(X)=(f_1^{-1}\circ g_1,\cdots,f_n^{-1}\circ g_n)(T(X))$, they are equivalent.
    \end{enumerate}
\end{solution}

\begin{ex}
    Show that the order statistics are minimal sufficient for the location family
    \[
        p(\mathbf{x} \mid \theta)=f\left(x_{1}-\theta\right) \ldots f\left(x_{n}-\theta\right)
    \]
    when
    \[
        f(x)=\frac{1}{\pi} \frac{1}{1+x^{2}},
    \]
    i.e., Cauchy \((0,1)\).
\end{ex}

\begin{solution}
    \[
        P(X|\theta)=\prod_{i=1}^nf(x_i-\theta)=\pi^{-n}\prod_{i=1}^n(1+(x_i-\theta)^2)^{-1},
    \]
    If $x\neq y$,
    \[
        \frac{P(X|\theta)}{P(Y|\theta)}=\frac{\prod_{i=1}^n(1+(x_i-\theta)^2)^{-1}}{\prod_{i=1}^n(1+(y_i-\theta)^2)^{-1}},
    \]
    this ratio must be dependent on $\theta$. So, when $x$ is a permutation of $y$, it is independent on $\theta$. And in this case, $x_{(1)}=y_{(1)}, \cdots, y_{(n)}=y_{(n)}$. Then, $(X_{(1)}, \cdots, X_{(n)})$ is minimal sufficient statistic.
\end{solution}

\begin{ex}
    Consider the curved exponential family \(\{n \text { i.i.d. } \mathcal{N}(\theta, \theta)\}_{\theta>0}\). Show that the sufficient statistic
    \[
        T(\mathbf{X})=\left(\sum_{i=1}^{n} X_{i}, \sum_{i=1}^{n} X_{i}^{2}\right)^T
    \]
    is not minimal.
\end{ex}

\begin{solution}
    \[
        \begin{aligned}
            \prod_{i=1}^{n} f\left(x_{i} \mid \theta\right) &=\prod_{i=1}^{n} \frac{1}{\sqrt{2 \pi \theta}} \exp^{-\frac{\left(x_{i}-\theta\right)^{2}}{2 \theta}} \\ &=(2 \pi \theta)^{-\frac{n}{2}} \prod_{i=1}^{n} \exp \left\{-\frac{1}{2 \theta}\left(x_{i}^{2}-2 \theta x_{i}+\theta^{2}\right)\right\} \\ &=(2 \pi \theta)^{-\frac{n}{2}} \exp \left\{-\frac{1}{2 \theta}\left(\sum_{i=1}^{n} x_{i}^{2}-2 \theta \sum_{i=1}^{n} x_{i}+n \theta^{2}\right)\right\} \\ &=(2 \pi \theta)^{-\frac{n}{2}} \exp \left\{\sum_{i=1}^{n} x_{i}\right\} \exp \left\{-\frac{1}{2 \theta} \sum_{i=1}^{n} x_{i}^{2}\right\} \exp \left\{-\frac{n \theta}{2}\right\},
        \end{aligned}
    \]
    So, $T^*(X)=\sum_{i=1}^nx_i^2$ is another sufficient statistic. Let $f(x_1,x_2)=x_2$, then
    \[
        f(T(X))=\sum_{i=1}^nx_i^2 = T^*(X).
    \]
    But we can not find a function s.t. $g(T^*(X))=T(X)$. Thus, $T(X)$ is not minimal.
\end{solution}

\begin{ex}
    Let \(\{f(\mathbf{x} \mid \theta)\}_{\theta \in \mathbb{R}}\) the parametric family corresponding to i.i.d. random variables with joint density function
    \[
        \begin{aligned}
            f(\mathbf{x} \mid \theta)&=C \exp \left\{-\sum_{i=1}^{n}\left(x_{i}-\theta\right)^{4}\right\}\\
            &=C \exp \left(-n \theta^{4}\right) \exp \left\{4 \theta^{3} \sum_{i=1}^{n} x_{i}-6 \theta^{2} \sum_{i=1}^{n} x_{i}^{2}+4 \theta \sum_{i=1}^{n} x_{i}^{3}-\sum_{i=1}^{n} x_{i}^{4}\right\}
        \end{aligned}
    \]
    for some \(C>0\). Express the parameter space and show that
    \[
        T(\mathbf{X})=\left(\sum_{i=1}^{n} X_{i}, \sum_{i=1}^{n} X_{i}^{2}, \sum_{i=1}^{n} X_{i}^{3}\right)^T
    \]
    is a minimal sufficient statistic.
\end{ex}

\begin{solution}
    \begin{align}
        \begin{aligned}
        \frac{f(\mathrm{x} \mid \theta)}{f(\mathrm{y} \mid \theta)}=& \exp \left\{4 \theta^{3}\left(\sum_{i=1}^{n} x_{i}-\sum_{i=1}^{n} y_{i}\right)-6 \theta^{2}\left(\sum_{i=1}^{n} x_{i}^{2}-\sum_{i=1}^{n} y_{i}^{2}\right)\right.\\
        &\left.+4 \theta\left(\sum_{i=1}^{n} x_{i}^{3}-\sum_{i=1}^{n} y_{i}^{3}\right)-\left(\sum_{i=1}^{n} x_{i}^{4}-\sum_{i=1}^{n} y_{i}^{4}\right)\right\},
        \end{aligned}
    \end{align}
    if we want $\frac{f(\mathrm{x} \mid \theta)}{f(\mathrm{y} \mid \theta)}$ is independent on $\theta$, then we have
    \[
        \left\{\begin{array} { l }
            { \sum _ { i = 1 } ^ { n } x _ { i } - \sum _ { i = 1 } ^ { n } y _ { i } = 0 } \\
            { \sum _ { i = 1 } ^ { n } x _ { i } ^ { 2 } - \sum _ { i = 1 } ^ { n } y _ { i } ^ { 2 } = 0 } \\
            { \sum _ { i = 1 } ^ { n } x _ { i } ^ { 3 } - \sum _ { i = 1 } ^ { n } y _ { i } ^ { 3 } = 0 }
        \end{array}\right. \quad \Leftrightarrow \quad \left\{\begin{array}{l}
            \sum_{i=1}^{n} x_{i}=\sum_{i=1}^{n} y_{i} \\
            \sum_{i=1}^{n} x_{i}^{2}=\sum_{i=1}^{n} y_{i}^{2} \\
            \sum_{i=1}^{n} x_{i}^{3}=\sum_{i=1}^{n} y_{i}^{3}
        \end{array}\right.
    \]
    So, $T(X)$ must be minimal.
\end{solution}

\begin{ex}
    Consider an i.i.d. Poisson family with parameter \(\lambda \in(0, \infty)\), i.e.,
    \[
        f(\mathbf{x} \mid \lambda)=e^{-n \lambda} e^{(\log \lambda) \sum_{i=1}^{n} x_{i}}\left(\prod_{i=1}^{n} x_{i} !\right)^{-1}
    \]
    \begin{enumerate}[(a)]
        \item Conclude that \(\{f(\mathbf{x} \mid \lambda)\}_{\lambda>0}\) is generated by \((T(\mathbf{x}), h(\mathbf{x}))=\left(\sum_{i=1}^{n} x_{i},\left(\prod_{i=1}^{n} x_{i} !\right)^{-1}\right)\), and that \(\mathcal{E}=\mathbb{R}\). Also conclude that \(T(\mathbf{X})\) is minimal sufficient for \(\lambda\).
        \item Now fix \(\alpha \in \mathbb{R}\) and let \(T_{1}(\mathbf{X})=\alpha T(\mathbf{X}), T_{2}(\mathbf{X})=(1-\alpha) T(\mathbf{X})\). Conclude that \(\{f(\mathbf{x} \mid \lambda)\}_{\lambda>0}\) is generated by \(\left(\left(T_{1}(\mathbf{x}), T_{2}(\mathbf{x})\right) ;\left(\prod_{i=1}^{n} x_{i} !\right)^{-1}\right)\).
        \item Show that the natural parameter space \(\mathcal{E}^{\prime}\) generated by \(\left(T_{1}(\mathbf{X}), T_{2}(\mathbf{X})\right)\) is \(\mathbb{R}^{2}\). Conclude that \(\left(T_{1}(\mathbf{X}), T_{2}(\mathbf{X})\right)\) is minimal sufficient for \(\lambda\).
        \item Does (c) contradict the minimal sufficiency of \(T(\mathbf{X})\)?
    \end{enumerate}
\end{ex}

\begin{solution}
    \begin{enumerate}[(a)]
        \item \[
                f(x|\lambda)=\frac{1}{(\prod_{i=1}^nx_i!)^{-1}}\exp\left(\log{\lambda} \sum_{i=1}^nx_i-n\lambda\right).
            \]
            So, it is generated by $\langle \sum_{i=1}^nx, (\prod_{i=1}^nx_i!)^{-1} \rangle$. And
            \[
                \frac{f(x|\theta)}{f(y|\theta)} = \frac{\prod_{i=1}^ny_i!}{\prod_{i=1}^nx_i!}\exp\left(\log\lambda\left(\sum_{i=1}^n x_i-\sum_{i=1}^n y_i\right)\right).
            \]
            So, $\sum_{i=1}^n x_i=\sum_{i=1}^n y_i$. $T(x)=\sum_{i=1}^n x_i$ is minimal statistic.
        \item \[
            f(x|\lambda)=\frac{1}{(\prod_{i=1}^nx_i!)^{-1}}\exp\left(\log{\lambda} (\alpha T_1(X)+(1-\alpha)T_2(X)) - n\lambda\right).
        \]
        So ,let $\eta=(\alpha\log\lambda, (1-\alpha)\log\lambda)$, then we get the same results.
        \item $\alpha\in\mathbb{R}$, $\log\lambda\in\mathbb{R}$. So, $\eta\in\mathbb{R}^2$. Similarly, $(T_1(X),T_2(X))$ is also minimal statistic.
        \item No, $T(X)$ is minimal for $\lambda$; $(T_1(X), T_2(X))$ is minimal for $(\lambda, \alpha)$.
    \end{enumerate}

\end{solution}


% !TEX root = Main.tex
\section{Convolutional Neural Networks}
\textbf{Neurons}: $F_\sigma(\mathbf{x};\mathbf{w}) = \sigma(w_0 + \sum_{i=1}^M{x_iw_i})$.\\
\textbf{Activation}: tanh, ReLu, $s(x)=\frac{1}{1+e^{-x}}$, $s'(x)=s(x)(1-s(x))$\\
\textbf{Output}: linear regression: $\hat{\mathbf{y}} = \mathbf{W}^L\mathbf{x}^{L-1}$\\
binary classification (logistic):\\
$\hat{y_1} = \text{P}[Y=1|\mathbf{x}] = \frac{1}{1 + \exp[-\langle \mathbf{w}_1^L,\mathbf{x}^{L-1}\rangle]}$\\
multiclass (soft-max):\\
$\hat{y_k} = \text{P}[Y=k|\mathbf{x}]= \frac{\exp[\langle \mathbf{w}_k^L,\mathbf{x}^{L-1}\rangle]}{\sum_{m=1}^{K}{\exp[\langle \mathbf{w}_m^L, \mathbf{x}^{L-1}\rangle]}}$.\\
\textbf{Loss function} squared loss: $\frac{1}{2}(y - \hat{y})^2$\\
cross-entropy loss: $-y \log \hat{y} - (1-y)\log(1-\hat{y})$.\\
\textbf{Units and Layers}: layer-to-layer fwd. prop. notation: $\mathbf{x}^{l} = \sigma^{l}\left(\mathbb{W}^{\left(l\right)}\mathbf{x}^{\left(l-1\right)}\right)$. L-layer network: $\mathbf{y}=\sigma^{\left(L\right)}\left(\mathbf{W}^(L)\sigma^{(L-1)}\left(\cdots\left(\sigma^{(1)}\left(\mathbf{W}^{(1)}\mathbf{x}\right)\cdots\right)\right)\right)$

\subsection*{Backpropagation}
Layer-to-layer Jacobian: $\mathbf{x}$ = prev. layer activation, $\mathbf{x^+}$ = next layer activation. Jacobian matrix $\mathbf{J}$ = $J_{ij}$ of mapping $\mathbf{x}\rightarrow\mathbf{x^+}$, $\mathbf{x_i^+} = \sigma(\mathbf{w}_i^\top\mathbf{x})$, $J_{ij} = \frac{\partial \mathbf{x_i^+}}{\partial \mathbf{x}_j} = w_{ij}\cdot\sigma'(\mathbf{w}_i^\top\mathbf{x})$.\\
Across multiple layers:\\
$\frac{\partial\mathbf{x}^{(l)}}{\partial\mathbf{x}^{(l-n)}} = \mathbf{J}^{(l)}\cdot\frac{\partial\mathbf{x}^{(l-1)}}{\partial\mathbf{x}^{(l-n)}}=\mathbf{J}^{(l)}\cdot\mathbf{J}^{(l-1)}\cdots\mathbf{J}^{(l-n+1)}$ and then back prop. $ \nabla_{x^{(l)}}^\top\ell=\nabla_{y}^\top\ell\cdot\mathbf{J}^{(L)}\cdots\mathbf{J}^{(l+1)}$

\subsection*{Neural Networks for Images}
Translation invariance of images $\rightarrow$ neurons compute same fct, shift invariant filters; weights defined as filter masks, e.g. convolution: $F_{n,m}(\mathbf{x};\mathbf{w}) = \sigma(b + \sum_{k=-2}^2\sum_{l=-2}^{2}{w_{k,l}x_{n+k,m+l}})$. To reduce dimension of convolution, use \{max, avg\}-pooling