\section{Divide and Conquer}

%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Integer multiplication (Problem 2.15)}
  \begin{exampleblock}{Integer Multiplication}
    Multiplying two $n$-bit integers in $o(n^2)$ time. {\small (Assuming $n = 2^k$.)}
  \end{exampleblock}

  \vspace{0.50cm}

  \centerline{Column multiplication in $\Theta(n^2)$}

  \begin{block}{Elementray operations:}
	\begin{itemize}
	  \item $n$-bit + $n$-bit: $O(n)$
	  \item $1$-bit $\times$ $1$-bit: $O(1)$
	  \item $n$-bit shifted by $1$-bit: $O(1)$
	\end{itemize}
  \end{block}
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Integer multiplication (Problem 2.15)}
  \begin{block}{Simple divide and conquer:}
	\begin{align*}
	  x & = x_L : x_R = 2^{n/2} x_L + x_R  \\
	  y & = y_L : y_R = 2^{n/2} y_L + y_R
	\end{align*}

	\begin{align*}
	  xy & = (2^{n/2} x_L + x_R) (2^{n/2} y_L + y_R) \\
		 & = 2^{n} x_L y_L + 2^{n/2} (x_L y_R + x_R y_L) + x_R y_R
	\end{align*}

	\[
	  T(n) = 4T(n/2) + \Theta(n) = \Theta(n^2)
	\]
  \end{block}
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Integer multiplication (Problem 2.15)}
  \begin{block}{A little history:}
  \begin{itemize}
    \item Kolmogorov (1952) conjecture: $\Omega(n^2)$
    \item Kolmogorov (1960) seminar
    \item Karatsuba (\emph{within a week}): $\Theta(n^{1.59})$
    \item ``The Complexity of Computations'' by Karatsuba, 1995
  \end{itemize}
  \end{block}
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Integer multiplication (Problem 2.15)}
  \begin{block}{Karatsuba algorithm:}
    \[
      T(n) = 3T(n/2) + \Theta(n) = \Theta(n^{\log_{2}{3}}) = \Theta(n^{1.59})
    \]

	\pause

	\[
      xy = 2^{n} x_L y_L + 2^{n/2} (x_L y_R + x_R y_L) + x_R y_R
	\]

    \[
      \underbrace{(x_L + x_R) (y_L + y_R)}_{P_0} = \underbrace{x_L y_L}_{P_1} +
      (x_L y_R + x_R y_L) + \underbrace{x_R y_R}_{P_2}
    \]

    \[
      xy = 2^{n} P_1 + 2^{n/2} (P_0 - P_1 - P_2) + P_2
    \]
  \end{block}
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Matrix multiplication (Problem 2.16)}
  \begin{exampleblock}{Matrix multiplication}
    Multiplying two $n \times n$ matrices in $o(n^3)$ time. {\small (Assuming $n = 2^k$.)}
    \[ Z = X \times Y \]
  \end{exampleblock}

  \vspace{0.50cm}
  \begin{columns}
    \column{0.40\textwidth}
	  \[ Z_{ij} \]

	  \[ T(n) = \Theta(n^2 \cdot n) = \Theta(n^3) \]
    \column{0.60\textwidth}
	  \begin{block}{Elementrary operations:}
	    \begin{itemize}
	      \item integer addition: $O(1)$
	      \item integer multiplication: $O(1)$
	    \end{itemize}
	  \end{block}
  \end{columns}
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Matrix multiplication (Problem 2.16)}
  \begin{displaymath}
	X = \begin{bmatrix} A & B \\ C & D \end{bmatrix}, \quad
	Y = \begin{bmatrix} E & F \\ G & H \end{bmatrix} \qquad (A \ldots H \in
	\mathbb{R}^{n/2} \times \mathbb{R}^{n/2})
  \end{displaymath}

  \[
    XY = \begin{bmatrix} AE + BG & AF + BH \\ CE + DG & CF + DH \end{bmatrix}
  \]

  \[
    T(n) = 8T(n/2) + \Theta(n^2) = \Theta(n^3)
  \]
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Matrix multiplication (Problem 2.16)}
  \begin{block}{Strassen algorithm:}
    \[
      T(n) = 7T(n/2) + \Theta(n^2) = \Theta(n^{\lg {7}}) = \Theta(n^{2.808})
    \]
  \end{block}

  \[
	XY = \begin{bmatrix} P_5 + P_4 - P_2 + P_6 & P_1 + P_2 \\ P_3 + P_4 & P_1 + P_5 - P_3 - P_7 \end{bmatrix}
  \]

  \begin{columns}
	\column{0.40\textwidth}
	{\footnotesize
	  \begin{align*}
		P_1 &= A(F - H) \\
		P_2 &= (A + B)H \\
		P_3 &= (C + D)E \\
		P_4 &= D(G - E) \\
		P_5 &= (A + D)(E + H) \\
		P_6 &= (B - D)(G + H) \\
		P_7 &= (A - C)(E + F)
	  \end{align*}
	}
	\column{0.60\textwidth}
	  \pause
	  \begin{itemize}
		\item Strassen (1969): $\Theta(n^{2.808})$ \\
		  ``Gaussian Elimination is Not Optimal''
		  \pause
		\item (2014): $\Theta(n^{2.373})$
		  \pause
		\item Known lower bound: $\Omega(n^{2})$
	  \end{itemize}
  \end{columns}
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{1-D DP}
  \begin{exampleblock}{Maximal sum subarray (Problem 1.3.5)}
    \begin{itemize}
      \item array $A[1 \cdots n], a_{i} >=< 0$
      \item to find (the sum of) an MS in $A$
    \end{itemize}

    \[
      A[-2,1 ,-3,4,-1,2,1,-5,4] \Rightarrow [4,-1,2,1]
    \]
  \end{exampleblock}

  \pause
  \begin{alertblock}{Trial and error.}
    \begin{itemize}
      \item try subproblem $\text{MSS}[i]$: the sum of the MS (\text{MS}[i]) in $A[1 \cdots i]$
      \item goal: $\text{mss} = \text{MSS}[n]$
      \item question: Is $a_{i} \in \text{MS}[i]$?
      \item recurrence:
	\[
	  \text{MSS}[i] = \max \set{\text{MSS}[i-1], \textcolor{red}{???}}
	\]
    \end{itemize}
  \end{alertblock}
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{1-D DP}
  \begin{block}{Solution.}
    \begin{itemize}
      \item subproblem $\text{MSS}[i]$: the sum of the MS \textcolor{red}{\it ending with} $a_{i}$ or 0
      \item goal: $\text{mss} = \max_{1 \le i \le n} \text{MSS}[i]$
      \item<2-> question: where does the $\text{MS}[i]$ start?
      \item<2-> recurrence:
		\[
		  \text{MSS}[i] = \max \set{\text{MSS}[i-1] + a_{i}, 0} \text{ \textcolor{red}{(prove it!)}}
		\]
      \item<3-> initialization: $\text{MSS}[0] = 0$
    \end{itemize}

    % \begin{displaymath}
    %   \text{MSS}[i] = \left\{ \begin{array}{ll}
    %     0 & i = 0 \\
    %     \max \set{\text{MSS}[i-1] + a_{i}, 0} & i > 0
    %   \end{array} \right.
    % \end{displaymath}
  \end{block}
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}[fragile]{1-D DP}
  \begin{block}{Code.}
    \begin{verbatim}
      MSS[0] = 0
      For i = 1 to n
        MSS[i] = max{MSS[i-1] + A[i], 0}
      return max_{i = 1 to n} MSS[i]
    \end{verbatim}
  \end{block}

  \pause

  \begin{block}{Simpler code.}
    \begin{verbatim}
      mss = 0
      MSS = 0
      For i = 1 to n
        MSS = max{MSS + A[i], 0}
        mss = max{mss, MSS}
      return mss
    \end{verbatim}
  \end{block}
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Pancake sorting (Problem 1.3.1)}
  \fignocaption{width = 0.25\textwidth}{figs/pancake-sorting.png}

  \pause
  \centerline{How to bring the biggest pancake to the bottom?}

  \pause
  \[
	T(n) = 2n-3
  \]

  \pause
  \begin{alertblock}{Reference}
	\begin{itemize}
	  \item $T(n) \le \frac{5n+5}{3}$, 1979: ``Sorting by Perfix Reversals'' \pause by Bill Gates \emph{et al.}
		\pause
	  \item $T(n) \le \frac{18n}{11}$, 2009
	\end{itemize}
  \end{alertblock}
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{\# Big V's (Problem 1.3.8)}
  \centerline{How many Big V's are there at most?}

  \pause
  \vspace{0.50cm}
  \centerline{``Does A follow B?''}

  \pause
  \vspace{0.50cm}
  \centerline{Don't forget to check it!}
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Bolts and nuts (Problem 2.10)}
  \fignocaption{width = 0.30\textwidth}{figs/bolts-nuts.jpg}

  \pause
  \centerline{Using quicksort}

  \pause
  \[
	A(n) = O(n \log n)
  \]

  \pause
  \begin{alertblock}{Reference}
	In the worst case:
	\begin{itemize}
	  \item ``Matching Nuts and Bolts'' by Alon \emph{et al.}, $\Theta(n \log^4 n)$
	  \pause
	  \item ``Matching Nuts and Bolts Optimality'' by Bradford, 1995, $\Theta(n \log n)$
	\end{itemize}
  \end{alertblock}

\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Bolts and nuts (Problem 2.10)}
  \fignocaption{width = 0.30\textwidth}{figs/bolts-nuts.jpg}

  \[
	\Omega(n \log n)
  \]

  \pause
  \centerline{At least as hard as the sorting problem.}

  \pause
  \[
	3^{H} \ge L \textcolor{red}{\ge n!} \Rightarrow H \ge \log (n!) \Rightarrow H = \Omega(n \log n)
  \]
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{$K$-sorted (Problem 2.9)}
  \[
	1,\;2,\;4,\;3;\quad 7,\;6,\;8,\;5;\quad 10,\;11,\;9,\;12;\quad 15,\;13,\;16,\;14
  \]

  \pause
  \vspace{0.20cm}

  \begin{center}
	$1\text{-sorted}$? \\ \pause
	$2\text{-sorted}$? \\ \pause
	$n\text{-sorted}$?
  \end{center}

  \pause
  \[
	1\text{-sorted} \to 2\text{-sorted} \to 4\text{-sorted} \to \cdots \to n\text{-sorted}
  \]

  \pause
  \centerline{Quicksort (with median) stops after the $\log k$ recursions.}

  \pause
  \[
	O(n \log k)
  \]
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{$K$-sorted (Problem 2.9)}
  \[
    \Omega(n \log k)
  \]

  \pause
  \[
	L = \frac{n!}{\left( (\frac{n}{k})! \right)^{k}}
  \]

  \pause
  \[
	H \ge \log \left(\frac{n!}{\left( (\frac{n}{k})! \right)^{k}} \right)
  \]
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Dutch national flag problem (Problem 2.5)}
  \begin{columns}
	\column{0.50\textwidth}
	  \fignocaption{width = 0.50\textwidth}{figs/dutch-flag.png}{\centerline{The Dutch national flag}}
	\column{0.50\textwidth}
	  \fignocaption{width = 0.35\textwidth}{figs/dijkstra.jpg}{\centerline{Edsger W. Dijkstra}}
  \end{columns}

  \vspace{0.50cm}
  \centerline{Red balls \emph{before} White balls \emph{before} Blue balls}

  \pause
  \[
	\textsc{Color}(i) \quad \textsc{Swap}(i,j)
  \]
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Dutch national flag problem (Problem 2.5)}
  Loop invariant:

  \begin{equation*}
	\begin{cases}
	  \text{White} & r \le i < w \\
	  \text{Red} & 0 \le i < r \\
	  \text{Blue} & b \le i < n
	\end{cases}
  \end{equation*}

  \pause
  \[
	\text{Init: } r = 0; \; w = 0; \; b = n - 1\;
  \]

  \vspace{0.30cm}
  \begin{description}
	\item[Red:] \uncover<3->{$\textsc{Swap}(r,w); \; r \gets r + 1; \; w \gets w + 1;$}
	\item[White:] \uncover<4->{$w \gets w + 1;$}
	\item[Blue:] \uncover<5->{$\textsc{Swap}(b-1, w); \; b \gets b - 1;$}
  \end{description}
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Repeated elements (Problem 2.12)}
  \begin{itemize}
	\item $R[1 \dots n]$
	\item $\text{check}(R[i], R[j])$
	\item $\# > \frac{n}{13}$
  \end{itemize}

  \pause
  \vspace{0.30cm}

  \[
	\# > \frac{n}{k}
  \]

  \begin{center}
    an $O(n \log k)$ algorithm \\
    the lower bound $\Omega(n \log k)$
  \end{center}

  \begin{alertblock}{Reference}
	``Finding Repeated Elements'' by Misra \& Gries, 1982
  \end{alertblock}
\end{frame}
%%%%%%%%%%%%%%%%%%%%
% \begin{frame}{VLSI layout}
%   \begin{exampleblock}{Problem (Area-Efficient VLSI Layout)}
%     Embedding a complete binary tree with $n$ leaves into a grid with minimum
%     area.
%     \begin{itemize}
%       \item VLSI: Very Large Scale Integration
%       \item complete binary tree circuit of $\#layer = 3,5,7,\ldots$
% %       \item $n$ leaves (\textcolor{red}{why only leaves?})
%       \item vertex on grid; no crossing edges
%       \item area = width $\times$ height
%     \end{itemize}
%   \end{exampleblock}
% \end{frame}
% %%%%%%%%%%%%%%%%%%%%
% \begin{frame}{VLSI layout}
%   Na\"{\i}ve embedding:
%    \[ H(n) = H(\frac{n}{2}) + \Theta(1) = \Theta(\lg n) \]
%    \[ W(n) = 2W(\frac{n}{2}) + \Theta(1) = \Theta(n) \]
%    \[ A(n) = \Theta(n \lg n) \]
% \end{frame}
% %%%%%%%%%%%%%%%%%%%%
% \begin{frame}{VLSI layout}
%   Smart (H-Layout) embedding:
%   \[ \Box \times \Box = n? \; 1 \times n;\; \frac{n}{\lg n} \times \lg n;\;
%   \textcolor{blue}{\sqrt{n} \times \sqrt{n}}
%   \]
%   Goal: $H(n) = \Theta(\sqrt{n}); W(n) = \Theta(\sqrt{n}); A(n) = \Theta(n).$
%   \pause
%   \[
% 	H(n) = \Box H(\frac{n}{\Box}) + O(\Box); H(n) = 2 H(\frac{n}{4}) +
% 	O(n^{\frac{1}{2} - \epsilon})
%   \]
%   \pause
%   \[ H(n) = 2H(\frac{n}{4}) + \Theta(1) \]
%   \pause
%   \centering \textcolor{blue}{Here it is: H-Layout}
% \end{frame}
% %%%%%%%%%%%%%%%%%%%%
% \begin{frame}{VLSI layout}
%   \begin{figure}[htp]
% 	\begin{center}
% 	  \includegraphics[width = 0.80\textwidth]{figs/vlsi}
% 	\end{center}
%   \end{figure}
% \end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}[noframenumbering]
  \fignocaption{width = 0.50\textwidth}{figs/thankyou.jpg}
\end{frame}
%%%%%%%%%%%%%%%%%%%%
% \begin{frame}{Counting inversions (Problem 2.11)}
% \end{frame}
%%%%%%%%%%%%%%%%%%%%
% \begin{frame}{Maxima-finding (Problem 2.14)}
%   \centerline{Wrong recursions!}
%   \pause
%   \vspace{0.30cm}
%
%   \centerline{3D?}
%   \pause
%   \vspace{0.30cm}
%
%   \centerline{Lower bound $\Omega(n \log n)$!}
% \end{frame}
%%%%%%%%%%%%%%%%%%%%


\section{Romberg Integration}
{%
\flushleft\includegraphics{figures/4_5_Romberg-algo-1.eps}\\
\flushleft\includegraphics{figures/4_5_Romberg-algo-2.eps}
}%


\section*{Section A : Multiple Choice Questions [10 points] (Dan)}

\begin{enumerate}
	\item {\textbf{[2 points]} Suppose that you are using ridge regression to estimate the relationship between your data and a value of interest. Your estimate for $\hat{\beta}$ is given by:
	$$\hat{\beta} = \argmin_\beta \norm{y - X \beta}_2^2 + \lambda \norm{\beta}_2^2$$
	Where $\lambda$ is a hyper-parameter that governs how much to penalize model complexity. Which of the following would be valid ways to use a 10-fold cross validation scheme? Select all that apply.
	}
	\begin{enumerate}[label=\Alph*)]
			\item {
				 For different values of $\lambda$, train on 9 of the folds and estimate the risk on the 10th fold. Select $\lambda$ by using the value that has the lowest risk on this 10th fold. Repeat this procedure 10 times for each possible hold out and use the mean of the risks for the estimators selected at each round as an estimate of the true risk of the model.
			}
			\item {
				 For different values of $\lambda$, train on 8 of the folds and estimate the risk on the 9th fold. Select $\lambda$ by using the value that has the lowest risk on this 9th fold. Then estimate the risk using this $\lambda$ on the 10th fold. Repeat this procedure 45 (10 choose 2) times and use the mean of the risks for the estimators selected each round as an estimate of the true risk of the model.
			}
			\item {
				 For different values of $\lambda$, train on 1 of the folds, and estimate the risk on this same fold. Select $\lambda$ by using the value that has the lowest risk on this fold. Repeat this procedure 10 times (once for each fold), and use the mean of the risks for the estimators as an estimate of the true risk of the model.
			}
			\item {
				 Select a value of $\lambda$ before doing your experiment. Train on 9 of the folds and estimate the risk on the 10th fold. Repeat this procedure 10 times (once for each fold), and use the mean of the risks for the estimators as an estimate of the true risk of the model.
			}
	\end{enumerate}
	\textbf{Answer:}\underline{D}\\
	To estimate the true risk from a CV, a fix $\lambda$ should be selected first.

	\item {\textbf{[2 points]} For a k-Nearest Neighbor classifier and the data set below, which class will the test point (marked by the black circle) be  classified as for each of the following values of $k$? \\
	\begin{minipage}[!ht]{0.5\textwidth}
			\centering
			\includegraphics[width=1.0\textwidth]{knn_test_point.png}
			\label{fig:knn_test_point}
		\end{minipage}
	\begin{minipage}[!ht]{0.25\textwidth}
		\begin{enumerate}[label=(\roman*)]
			\item $k = 1$ \textbf{Answer:}\underline{A}
			\item $k = 2$ \textbf{Answer:}\underline{A}
			\item $k = 7$ \textbf{Answer:}\underline{B}
		\end{enumerate}
	\end{minipage}
	\begin{minipage}[!ht]{0.25\textwidth}
		\begin{enumerate}[label=\Alph*)]
			\item Red
			\item Green
			\item Blue
		\end{enumerate}
	\end{minipage}
	}

	\item {\textbf{[2 points]} Which of the following statements is true about the k-Nearest Neighbor classifier? Select all that apply.
		\begin{enumerate}[label=\Alph*)]
			\item{ As the value of $k$ increases, the variance of the model increases }
			\item{ As the value of $k$ increases, the bias of the model increases }
			\item{ As the value of $k$ increases, the model complexity increases }
			\item{ As the value of $k$ increases, the number of parameters in the model increases }
			\item{ As the number of training data points increases, the memory requirements of the model increase }
		\end{enumerate}
	}
	\textbf{Answer:}\underline{B, E}\\
	\begin{itemize}
		\item For A: the variance of the model will decrease
		\item For C: the model complexity will decrease
		\item For D: the number of parameters in the model will not change
	\end{itemize}

	\item {\textbf{[2 points]} What is the maximum training error for a decision tree on an arbitrary data set with $k$ discrete output classes?

		\begin{enumerate}[label=\Alph*)]
			\item $\frac{1}{k}$
			\item $1$
			\item $\frac{1}{\log_2{k}}$
			\item $\frac{k - 1}{k}$
		\end{enumerate}

	}
	\textbf{Answer}:\underline{D}\\
	This may happen when $P(Y)=P(Y|X_i),~\forall i$, and all data will be in a leaf node with a class label whose prior $P(Y)$ is the highest. To maximize the training error, the minimum training accuracy will be $1/k$; therefore, the maximum training error is $(k-1)/k$

	\item {\textbf{[2 points]} Consider the following data set. Which methods will classify all data points in this set correctly? Select all that apply. \\
		\begin{minipage}[!ht]{0.5\textwidth}
			\centering
			\includegraphics[width=1.0\textwidth]{mcq_data.png}
			\label{fig:svm_quad_a}
		\end{minipage}
		\begin{minipage}[!ht]{0.5\textwidth}
			\begin{enumerate}[label=\Alph*)]
				\item Soft-Margin SVM (with no kernel)
				\item SVM with a quadratic kernel, when the coefficient on the penalty for slack variables $C = 0$
				\item SVM with a quadratic kernel, when the coefficient on the penalty for slack variables $C \rightarrow \infty$
				\item Logistic regression (no kernel)
				\item 3-NN
			\end{enumerate}
		\end{minipage}
	}
	\textbf{Answer}:\underline{C}\\
	\begin{itemize}
		\item For A: The data is not linear separable. Two outlier red points will be misclassified.
		\item For B: Because $C=0$, there will be no classification at all.
		\item For C: $C\rightarrow \infty$ means hard-margin SVM is used. The quadratic kernel is used, and the data is non-linear separable.
		\item For D: Logistic is linear classifier.
		\item For E: The two outlier red points will be misclassified.
	\end{itemize}

\end{enumerate}

\newpage


\subsection{Tan}

The \(\tan(\theta )\) function is defined as:

\(\tan(\theta ):=\dfrac{\sin(\theta )}{\cos(\theta )}\)

\subsubsection{Behaviour around \(0\)}

\(\sin(0)=0\)

\(\cos(0)=1\)

\(\tan(0):=\dfrac{\sin(0)}{\cos(0)}\)

\(\tan(0)=\dfrac{0}{1}\)

\(\tan(0)=0\)

\subsubsection{Behaviour around \(\cos(\theta )=0\)}

\(\tan(\theta )=\dfrac{\sin(\theta )}{\cos(\theta )}\)

So \(\tan (\theta )\) is undefined where \(\cos(\theta )=0\).

This happens where:

\(\theta=\dfrac{\tau }{4}+\dfrac{1}{2}n\tau \)

\(\theta=\dfrac{1}{4}\tau (1+2n)\)

Where \(n\in \mathbb{Z}\).

\subsubsection{Derivatives}

\(\tan(\theta )=\dfrac{\sin(\theta )}{\cos(\theta )}\)

\(\dfrac{\delta }{\delta \theta } \tan(\theta )=\dfrac{\delta }{\delta \theta }\dfrac{\sin(\theta )}{\cos(\theta )}\)

\(\dfrac{\delta }{\delta \theta } \tan(\theta )=\dfrac{\cos(\theta )}{\cos(\theta )}+ \dfrac{\sin^2(\theta )}{\cos^n(\theta )}\)

\(\dfrac{\delta }{\delta \theta } \tan(\theta )=1+\tan^2(\theta )\)

Note this is always positive. This means:

\(\lim_{\cos(\theta )\rightarrow 0^+}=-\infty\)

\(\lim_{\cos(\theta )\rightarrow 0^-}=\infty\)


\section*{Generative Models}
\subsection*{Variational Autoencoder (VAE)}
Sample random vector $\mathbf{z} \sim \mathcal{N}(0,\mathbf{I})$. Transform through (deterministic) DNN $F_\theta : \mathbb{R}^m \to \mathbb{R}^n$. Note: expectations $\mathbb{E}_x[f(x)] = \mathbb{E}_z [f(F_\theta(z))]$ (law of the unconscious statistician). Infeasible, would need to find inv. Jacobian determinant\\
$D_{KL}(P\|Q)=\sum_i P(i)\log\frac{P(i)}{Q(i)}=\mathbb{E}_i [\frac{\log P_i}{\log Q_i}]$ \\
More general: $p_\theta (\mathbf{x}|\mathbf{z}) $ instead of $F_\theta$: ELBO \\
$\log p_\theta(x^{(i)})
% = \mathbb{E}_{z \sim q_\phi (z|x)}[\log p_\theta (x^{(i)})]
= \mathbb{E}_{z \sim q_\phi (z|x)} \left[\log \frac{p_\theta(z) \cdot p_\theta(x^{(i)}|z)}{p_\theta(z|x^{(i)}}\frac{q_\Phi(z|x^{(i)})}{q_\Phi(z|x^{(i)})} \right] \allowbreak
= \mathbb{E}_z[\log p_\theta (x^{(i)}|z)] - D_{KL}(q_\Phi(z|x^{(i)}) || p_\theta(z))$ \\
$ + D_{KL}(q_\Phi(z|x^{(i)}) || p_\theta(z|x^{(i)})$ (drop last part)\\
1st: reconstr. quality, 2nd: posterior close to prior. \\
Update: $\nabla_\theta \mathbb{E}_{q_\phi}[\log p_\theta(x|z)] = \mathbb{E}[\nabla_\theta \log p_\theta(x|z)] \allowbreak \approx \frac 1 L \sum_{r=1}^L \nabla_\theta \log p_\theta(x|z^{(r)}), z^{(r)} \sim^{iid} q_\theta (\cdot | x) $\\
\textbf{Reinforce trick:}\\
$\nabla_\theta\mathbb{E}_{q_\phi}[\mathcal{L}(\mathbf{x}, \mathbf{z})] = \mathbb{E}_{q_\phi}[\mathcal{L}(\mathbf{x}, \mathbf{z}) \nabla_\theta\log q_\theta(\mathbf{z};\mathbf{x})]$ \\
\textbf{Re-parameterization trick:} use variational distribution  $q_\phi(\mathbf{z};\mathbf{x})=g_\phi(\zeta;\mathbf{x})$ for $\zeta$ simple \\
(e.g. $\zeta \sim \mathcal{N}(\mathbf{0},\mathbf{I}), \mathbf{z}=\mathbf{\mu} + \mathbf{U}\zeta \Rightarrow \mathbf{z}\sim \mathbf{N}(\mathbf{\mu}, \mathbf{UU}^T)$)\\
\textbf{Stochastic Backprop:} for $\zeta^{(r)} \sim^{iid} \text{simple}$ \\
$\mathbb{E}_{q_\phi}[\nabla_\phi\mathcal{L}(\mathbf{x}, \mathbf{z})] \approx \frac 1 L \sum_{r=1}^L [\nabla_\phi \mathcal{L}(\mathbf{x}, g_\phi(\zeta^{(r)})] $
\subsection*{Generative Adversarial Network (GAN)}
$\min_G \max_D V(D,G) = \mathbb{E}_{\mathbf{x}\sim p_{data}(\mathbf{x})}[\log D(\mathbf{x}) ]$ \\
$+ \mathbb{E}_{\mathbf{z}\sim p_{\mathbf{z}}(\mathbf{\mathbf{z}})}[\log (1 - D(G(\mathbf{x})) ] \\
$ $\theta^* := \argmin_{\theta\in \Theta} \{ \sup_{\phi \in \Phi} l(\theta, \phi)\}$
\textbf{SGD:}  $\theta^{t+1} = \theta^t - \eta \nabla_\theta l(\theta^t, \phi^t)$ ; $\phi^{t+1} = \phi^t + \eta \nabla_\phi l(\theta^{t+1}, \phi^t)$
\subsection*{Autoregressive Models}
Generate output one variable at a time: $p(x_1, ..., x_m) = \prod_{t=1}^m p(x_t|x_{1:t-1})$ \\\textbf{PixelCNN:} uses exactly that over a window to predict the next pixel (slow process).


% % % % % % Notation % % % % %

\chapter{Notation}
\label{apx:notation}

In this text we use the following notation conventions:
\begin{description}
\item[$x$] A column vector, or scalar, as implied by the text.
\item[$x'$] We use $'$ for the transpose operation.
For a $1\times p$ row vector $x$, then $x'$ is a $p \times 1$ column vector.
\item[$:=$] An assignment, or definition. $A:=a$ means that $A$ is defined to be $a$.
\item[$\prod_{i=1}^{n}$] The product operator: $\prod_{i=1}^{n} x_i:= x_1 \times \dots \times x_n$.
\item[$\coprod_{i=1}^{n}$] The coproduct operator: $\coprod_{i=1}^{n} x_i:= 1-(1-x_1) \times \dots \times (1-x_n)$.
\item[$\#\set{A}$] The count operator. Returns the number of elements of the set $A$. Also known as the \emph{cardinality}.
\item[$\Phi(t)$] The standard Gaussian CDF at $t$: $\Phi(t):= P(Z<t)$.
\item[$\phi(t)$] The standard Gaussian density at $t$: $\phi(t):= \frac{\partial}{\partial t}\Phi(t)$.
\item[$\Phi(x)$] The structure function of a system in reliability analysis.
\item[$\x_n \rightsquigarrow \dist$] Convergence in distribution: for large enough $n$, then $\x_n$ is distributed like $\dist$.
\item[$\conv$] The convolution operator: $f \conv g=(f \conv g)(t)=\int f(s) g(t-s) ds$.
\item[$f^{\conv n}$] The convolution power: $n$ convolutions of $f$ with itself.
\item[$\sigma_{(j)}(A)$] The $j$'th largest singular value of matrix $A$.
\item[$\sigma_{max}(A)$] The largest singular value of matrix $A$.
\item[$\lambda_{max}(A)$] The largest eigenvalue of a symmetric positive definite matrix $A$.
\end{description}


\section[Applications]{The homology of $\Omega S^n$, and the Serre exact sequence}
%Leray originally invented his spectral sequences for locally compact things,
%but $\Omega S^n$ for instance isn't locally compact.
The goal of this section is to describe a computation of the homology of
$\Omega S^n$ via the Serre spectral sequence, as well as describe a
``degenerate'' case of the Serre spectral sequence.

\subsection*{The homology of $\Omega S^n$}\label{loops-sn}
Let us first consider the case $n=1$. The space $\Omega S^1$ is the base of a
fibration $\Omega S^1 \to PS^1 \to S^1$. Comparing this to the fibration $\Z\to
\RR\to S^1$, we find that $\Omega S^1 \simeq \Z$. Equivalently, this follows
from the discussion in \S \ref{loops-bg} and the observation that $S^1 \simeq
K(\Z,1)$.

Having settled that case, let us now consider the case $n>1$. Again, there is a
fibration $\Omega S^n\to PS^n \to S^n$. In general, if $F\to E\to B$ is a
fibration and the space $F$ has torsion-free homology, we can (via the
universal coefficients theorem) rewrite the $E^2$-page:
$$E^2_{s,t} = H_s(B;H_t(F)) \simeq H_s(B)\otimes H_t(F).$$
Since $S^n$ has torsion-free homology, the Serre spectral sequence (see \S
\ref{serre-sseq}) runs:
$$E^2_{s,t} = H_s(S^n) \otimes H_t(\Omega S^n) \Rightarrow H_{\ast}(PS^n) =
\Z.$$
Since $H_s(S^n)$ is concentrated in degrees $0$ and $n$, we learn that
$E^2$-page is concentrated in columns $s=0,n$. For instance, if $n=4$, then the
$E^2$-page (without the differentials drawn in) looks like:
\sseqset{classes={draw=none}}
\begin{sseqdata}[name = {loops-Sn}, x range={0}{5}, y range={0}{5},
    homological Serre grading, differentials={->}, y axis gap=0.8cm, x label =
    {$H_\ast(S^4)$}, y label = {$H_\ast(\Omega S^4)$}]
    \foreach \y in {0,...,6} {\class["H_{\y}(\Omega S^n)"](0,\y)}
    \foreach \y in {0,...,6} {\class["H_{\y}(\Omega S^n)"](4,\y)}
    \foreach \y in {0,...,2} {\d["d^4"]4(4,\y)}
\end{sseqdata}
\begin{center}
    \printpage[name={loops-Sn},page=0,no differentials]
\end{center}
We know that $H_0(\Omega S^n) = \Z$. Since the target has homology concentrated
in degree $0$, we know that $E^2_{n,0}$ has to be killed. The only possibility
is that it is hit by a differential, or that it supports a nonzero
differential.

There are not very many possibilities for differentials in this spectral
sequence. In fact, up until the $E^n$-page, there are no differentials (either
the target or source of the differential is zero), so $E^2\simeq E^3 \simeq
\cdots \simeq E^n$. On the $E^n$-page, there is only one possibility for a
differential: $d^n:E^2_{n,0} \to E^n_{0,n-1}$. This differential has to be a
monomorphism because if it had anything in its kernel, that will be left over
in the position. In our example above (with $n=4$), we have
\begin{center}
    \printpage[name={loops-Sn},page=4]
\end{center}

However, we still do not know the group $E^n_{0,n-1}$. If it is bigger than
$\Z$, then $d^n$ is not surjective. There can be no other differentials on the
$E^r$-page for $r\geq n+1$ (because of sparsity), so the $d^n$ differential is
our last hope in killing everything in degree $(0,n-1)$.  This means that $d^n$
is an epimorphism. We find that $E^n_{0,n-1} = H_{n-1}(\Omega S^n) \simeq \Z$,
and that $d^n$ is an isomorphism.

We have now discovered that $H_{n-1}(\Omega S^n) \simeq \Z$ --- but there is a
lot more left in the $E^2$-page! For instance, we still have a $\Z$ in
$E^n_{n,n-1}$. Because $H^\ast(PS^n)$ is concentrated in degree $0$, this, too,
must die! We are in exactly the same situation as before, so the same arguments
show that the differential $d^n:E^n_{n,n-1}\to E^n_{0,2(n-1)}$ has to be an
isomorphism. Iterating this argument, we find:
\begin{equation*}
    H_q(\Omega S^n) \simeq \begin{cases}
        \Z & \text{if }(n-1)|q\geq 0\\
        0 & \text{else}
    \end{cases}
\end{equation*}
This is a great example of how useful spectral sequences can be.
\begin{remark}
    The loops $\Omega X$ is an associative $H$-space. Thus, as is the case for
    any $H$-space, the homology $H_\ast(\Omega X; R)$ is a graded associative
    algebra. Recall that the suspension functor $\Sigma$ is the left adjoint to
    the loops functor $\Omega$, so there is a unit map $A\to \Omega \Sigma A$.
    This in turn begets a map $\widetilde{H}_\ast(A)\to H_\ast(\Omega \Sigma A)$.

    Recall that the universal tensor algebra
    $\mathrm{Tens}(\widetilde{H}_\ast(A))$ is the free associative algebra on
    $\widetilde{H}_\ast(A)$. Explicitly:
    $$\mathrm{Tens}(\widetilde{H}_\ast(A)) = \bigoplus_{n\geq
    0}\widetilde{H}_\ast(A)^{\otimes n}.$$
    In particular, by the universal property of
    $\mathrm{Tens}(\widetilde{H}_\ast(A))$, we get a map
    $\alpha:\mathrm{Tens}(\widetilde{H}_\ast(A))\to H_\ast(\Omega \Sigma A)$.
    \begin{theorem}[Bott-Samelson]\label{bott-samelson}
	The map $\alpha$ is an isomorphism if $R$ is a PID and $H_\ast(A)$ is
	torsion-free.
    \end{theorem}
    For instance, if $A = S^{n-1}$ then $\Omega S^n = \Omega \Sigma A$. Theorem
    \ref{bott-samelson} then shows that
    $$
    H_\ast(\Omega S^n) = \mathrm{Tens}(\widetilde{H}_\ast(S^{n-1})) = \langle
    1, x, x^2, x^3, \cdots\rangle,
    $$
    where $|x| = n-1$. It is a mistake to call this ``polynomial'', since if
    $n$ is even, $x$ is an odd class (in particular, $x$ squares to zero by the
    Koszul sign rule).
\end{remark}
%Note that $H_\ast(\Omega S^n)$ acts on this spectral sequence.
%Then the $d^r$'s are linear (module homomorphisms).

Theorem \ref{bott-samelson} suggests thinking of $\Omega \Sigma A$ as the
``free associative algebra'' on $A$. Let us make this idea more precise.
\begin{remark}
    The space $\Omega A$ is homotopy equivalent to a topological monoid
    $\Omega_M A$, called the \emph{Moore loops} on $A$. This means that
    $\Omega_M A$ has a \emph{strict} unit and is \emph{strictly} associative
    (i.e., not just up to homotopy). Concretely,
    $$\Omega_M A := \{(\ell,\omega):\ell\in\RR_{\geq 0}, \omega:[0,\ell]\to A,
    \omega(0) = \ast = \omega(\ell)\},$$
    topologized as a subspace of the product. There is an identity class
    $1\in\Omega_M A$, given by $1 = (0,c_\ast)$ where $c_\ast$ is the constant
    loop at the basepoint $\ast$. The addition on this space is just given by
    concatenatation. In particular, the lengths get added; this overcomes the
    obstruction to $\Omega A$ not being strictly associative, so the Moore
    loops $\Omega_M A$ are indeed strictly associative.
    If the basepoint is nondegenerate, it is not hard to see that the inclusion
    $\Omega A\hookrightarrow \Omega_M A$ is a homotopy equivalence.

    Given the space $A$, we can form the free monoid $\mathrm{FreeMon}(A)$. The
    elements of this space are just formal sequences of elements of $A$
    (with topology coming from the product topology), and the multiplication is
    given by juxtaposition. Let us adjoin the element $1 = \ast$. As with all
    free constructions, there is a map $A\to\mathrm{FreeMon}(A)$ which is
    universal in the sense that any map $A\to M$ to a monoid factors through
    $\mathrm{FreeMon}(A)$.

    The unit $A\to\Omega\Sigma A$ is a map from $A$ to a monoid, so we get a
    monoid map $\beta:\mathrm{FreeMon}(A)\to\Omega\Sigma A$.
    \begin{theorem}[James]
	The map $\beta:\mathrm{FreeMon}(A)\to \Omega\Sigma A$ is a weak
	equivalence if $A$ is path-connected.
    \end{theorem}
    The free monoid looks very much like the tensor product, as the following
    theorem of James shows.
    \begin{theorem}[James]\label{james-splitting}
	Let $J(A) = \mathrm{FreeMon}(A)$. There is a splitting:
	$$
	\Sigma J(A) \simeq_w \Sigma \left(\bigvee_{n\geq 0}A^{\wedge n}\right).
	$$
    \end{theorem}
    Applying homology to the splitting of Theorem \ref{james-splitting} shows
    that:
    $$
    \wt{H}_\ast(J(A)) \simeq \bigoplus_{n\geq 0} \wt{H}_\ast(A^{\wedge n}).
    $$
    Assume that our coefficients are in a PID, and that $\wt{H}_\ast(A)$ is
    torsion-free; then this is just $\bigoplus_{n\geq 0}\wt{H}_\ast(A)^{\otimes
    n}$. In particular, we recover our computation of $H_\ast(\Omega S^n)$ from
    these general facts.
\end{remark}
\subsection{The Serre exact sequence}
Suppose $\pi:E\to B$ is a fibration over a path-connected base. Assume that
$\widetilde{H}_s(B) = 0$ for $s<p$ where $p\geq 1$. Let $\ast\in B$ be a chosen
basepoint. Denote by $F$ the fiber $\pi^{-1}(\ast)$. Assume $\widetilde{H}_t(F)
= 0$ for $t<q$, where $q\geq 1$.  We would like to use the Serre spectral
sequence to understand $H_\ast(E)$. As always, we will assume that $\pi_1(B)$
acts trivially on $H_\ast(F)$.

Recall that the Serre spectral sequence runs
$$
E^2_{s,t} = H_s(B;H_t(F)) \Rightarrow H_{s+t}(E).
$$
Our assumptions imply that $E^2_{0,0}=\Z$, and $E^2_{0,t} = 0$ for $t<q$.
Moreover, $E^2_{s,0} = 0$ for $s<p$. In particular, $E^2_{0,q+t} = H_{q+t}(F)$
and $E^2_{p+k,0} = H_{p+k}(B)$ --- the rest of the spectral sequence is
mysterious.

By sparsity, the first possible differential is $d^{p}:H_p(B) \to H_{p-1}(F)$, and
$d^{p+q}:H_{p+1}(B)\to H_{p}(F)$. In the mysterious zone, there are
differentials that hit $E^2_{p,q}$.

Again by sparsity, the only differential is $d^s:E^s_{s,0}\to E^s_{0,s-1}$ for
$s<p+q-1$. This is called a \emph{transgression}. It is the last possible
differential which has a chance at being nonzero. This means that the cokernel
of $d^s$ is $E^\infty_{0,s-1}$. There is also a map $E^\infty_{s,0}\to
E^s_{s,0}$. We obtain a mysterious composite
\begin{equation}\label{part1-serre-lexseq}
    0\to E^\infty_{s,0}\to E^s_{s,0} \simeq H_s(B) \xrightarrow{d^s}
    E^s_{0,s-1}\simeq H_{s-1}(F) \to E^\infty_{0,s-1}\to 0.
\end{equation}

Let $n<p+q-1$. Recall that $F_s H_n(E) =
\img(H_\ast(\pi^{-1}(\mathrm{sk}_s(B)))\to H_\ast(E))$, so $F_0 H_n(E) =
E^\infty_{0,n}$. Here, we are using the fact that $F_{-1} H_\ast(E) = 0$. In
particular, there is a map $E^\infty_{0,n}\to H_n(E)$. By our hypotheses, there
is only one other potentially nonzero filtration in this range of dimensions,
so we have a short exact sequence:
\begin{equation}\label{part2-serre-lexseq}
    0\to F_0 H_n(E) = E^\infty_{0,n} \to H_n(E) \to E^\infty_{n,0} \to 0
\end{equation}
Splicing the short exact sequences \eqref{part1-serre-lexseq} and
\eqref{part2-serre-lexseq}, we obtain a long exact sequence:
$$
H_{p+q-1}(F)\to \cdots\to H_n(F)\to H_n(E)\to H_n(B) \xrightarrow{\text{transgression}} H_{n-1}(F) \to H_{n-1}(E)\to \cdots
$$
This is called the \emph{Serre exact sequence}. In this range of dimensions,
homology behaves like homotopy.


\documentclass[12pt,a4paper]{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{amssymb}
\author{Karthik A}
\title{Machine Learning -- Course summary notes and guidelines}
\begin{document}

\maketitle
\part{Algorithms}
\section{Supervised Learning}
\subsection{Regression}
Cost function:
$$J(\theta) = \frac{1}{2m}\sum_{i=1}^m(y^{(i)}-h_\theta(x^{(i)}))^2  + \frac {\lambda}{2m}\sum_{j=1}^{n}\theta_{j}^2$$

Gradient descent:
Simultaneous update: $\{$

$\theta_j := \theta_j - \frac {\alpha}{m}\frac {\partial }{\partial \theta_j}J(\theta) - \alpha \frac{\lambda}{m}\theta_j$

$\}$

Simultaneous update: $\{$

$\theta_j := \theta_j - \frac {\alpha}{m} \sum_{i=1}^{m} (h_\theta(x^{(i)})-y)x^{(i)}$

$\}$

For small $\alpha$, $J(\theta)$ should reduce with every iteration
If polynomial regression, ensure feature scaling and mean normalization

Replace each value with $(x-\mu)/s$ where $s=(x_{max}- x_{min})$ or s=Std Deviation

\subsection{Logistic Regression}

Logistic and cost function
$$h_\theta(x) = \frac{1}{(1+e^-\theta^T X)}$$
$$J(\theta) = -\frac{1}{m}\sum_{i=1}^{m}[y^{(i)}log(h_\theta(x^{(i)}))+(1-y^{(i)})log(1-h_\theta(x^{(i)}))]$$

Simultaneous update: $\{$

$\theta_j := \theta_j - \frac {\alpha}{m} \sum_{i=1}^{m} (h_\theta(x^{(i)})-y)x^{(i)}$

$\}$

\subsection{Support Vector Machines (SVM)}
\begin{itemize}
\item if n is small (10k) and m is intermedeate (50k) use SVM. If n is small and m is large -- use logistic regression. Neural network is good for all but difficult to train
\item In SVM $C=1/\lambda$
\item Without kernel or linear regression SVM is logistic regression
\end{itemize}

\subsection{Neural Network}
Back propogation - neural network terminology for minimizing our cost funtion
$\delta = a -y$, where a is the activation value\\
Layer L = Output layer\\
Layer 1 = Input layer\\
$s_j$ = units in layer j\\
$a_i^{j}$ = activation of unit i in layer j\\
$\Theta^{(j)}$ =matrix of weights controlling function mapping from layer j to layer j+1\\
$a^{j+1} = g(\Theta^TX)$\\
If $s_j$ units in layer j, $s_{j+1}$ units in layer $(j+1)$, then dimensions of $\Theta^{j}$ will be $s_{j+1}\times(s_{j}+1)$

\section{Unsupervised Learning}
\subsection{K Means and Dimensionality reduction}

How to choose K in Kmeans
\begin{enumerate}
\item $J(\theta)$ vs K -- see if it elbows out
\item No good elbows, use the downstream purpose. For example, shirt sizes
\end{enumerate}

Dimensionality reduction
\begin{itemize}
\item Data compression
\item Visualization
\end{itemize}

\[
[U S V] = svd(\Sigma)    \;where , \Sigma  = (1/m)\sum (X*X^T)
\]

Determining number of dimensions k explaining variation of a target say 95\%
\[Explained\;Variation = 1 - \sum_{i=1}^k s_{ii}/\sum_{i=1}^n s_{ii}\]

\subsection{Anomaly detection}
Common applications : Fraud detection, Manufacturing, monitoring machines in a large cluster

Differences between Anomaly detection and clustering
\begin{enumerate}
\item y=1 is very small , typically 0--20
\item y=0 is a large number
\item Different types of anamolies exist which are not known upfront
\end{enumerate}

Some tips for anomaly detection
\begin{itemize}
\item Add a few failed samples to CV and test, do a $F_1$ score and evaluate the $\epsilon$ values
\item When features are not gaussian distributed, use the $x^{(1/10)}$ or $x^{(1/2)}$  or $log(x)$ etc and make gaussian
\item When encountering common problem that p(x) is similar to regular in muti dimensions try,
\subitem --Introducing features such as \\$CPU Load/network traffic$ or \\$(CPU load)^2/network traffic$
\subitem --Use multivariate gaussian - automatically captures the correlation between features but is computationally expensive (also m \textgreater n else $\Sigma$ is not inversible)
\end{itemize}

\section{Recommend-er systems and COllaborative FIltering}

$n_u$ = number of users (columns)(j)
$n_m$ = number of movies (i)
$r(i,j) = 1$ if user j has rated movie i

X = hypothetical or real vector such as Genre - action, comedy each being a column $x^1$ or $x^2$ or $x^n$
$m^j$ = number of movies rated by user j

In COFI , X is also not known
\begin{itemize}
\item min $\theta$  = $\sum_i \sum _n$  and get the double gradient over X and $\theta$
\item Initialize with small random values
\item Feature scaling is not required as the rating scale is always same across uses but mean normalization helps faster convergence

\end{itemize}

\part{Practical suggestions for machine learning}
\section{Model selection and refinement}

\subsection{Over--fitting problem}
\begin{itemize}
\item Reduce the number of features
\item Use the model selection algorithm described later
\item Use regularization to reduce the magnitude of theta ($\lambda$ high)
\end{itemize}

\subsection{Model selection}
60\%--20\%--20\% is a broad split for the train, cross validation and test sets

\begin{enumerate}
\item Get  the polynomial degree -- Find $J(\theta)$ for each increasing polynomial feature. Test on CV to fix the polynomial features. Check on test
\item If underfit reduce $\lambda$, if overfit , increase $\lambda$ \-- check the error on the CV set to fix the lambda value
\item To determine whether more or less data,  review the learning curves i.e plot the $J_{cv}$ and $J_{train}$ against increasing number of examples
\subitem ---- if $J_{cv}$ and $J_{train}$ are converging with less samples and the value is high, then it is High Bias problem and adding more data to the problem will not help
\subitem ---- if $J_{cv}$ and $J_{train}$ are different, then adding more samples will help them converge \-- high variance problem
\item Error Analysis: General guideline, start with a simple algorithm and then increase the number of features. Manually examine the errors on examples in the cross validation set and try to spot a trend where most of the errors were made. We can find some feaures that may be useful
\end{enumerate}

\subsection{Types of actions possible}
\begin{enumerate}
\item Collect more data -- High variance
\item Try smaller set of features -- High Variance
\item Try additional features -- High Bias
\item Try polynomial features -- High Bias
\item Increasing $\lambda$ -- High Variance
\item Decreasing $\lambda$ -- High Bias
\end{enumerate}

\section{Pipeline Analysis}
Example of pipeline for machine learning is given below:
Image $\rightarrow$ text detection $\rightarrow$ Character segmentation $\rightarrow$ Character recognition

\begin{itemize}
\item Sliding windows - step size or stride
\item 1D sliding window for char segmentation
\item Get lots of data and artificial data
\end{itemize}

\section{Rules for getting more data}
\begin{itemize}
\item Make sure you have a low bias classifier before starting data collection eg. add more features or hidden units in a neural network until bias is low
\item Estimate data collection effort
\item Ceiling analysis - which part of the pipeline must I improve - Look at the accuracy in the test set by feeding the $y_{actual}$ to the next part of the pipeline and see how it improves - delta accuracy should be used to determine where to put the effort
\end{itemize}

\section{Learning with large data sets}
\begin{itemize}
\item \emph{It is not who has the best algorithm that wins but who has the most data. The problem with large data sets is that gradient descent is computationally expensive for each iteration}
\item Plot the learning curve and if high bias and J is ok, no need to do on the full data set (m) as it will converge before then -- adding more samples has no benefit
\item If high variance use map reduce -- i.e split the summation into separate machines

\end{itemize}

\subsection{Stochastic gradient}
\begin{itemize}
\item Stochastic gradient - Do gradient descent with m=1 after shuffling the data set
\item Mini batch - do gradient descent in batches b \textless m
\item Batch gradient descent - full m
\end{itemize}

Plot J over number of iterations and average over the batch sizes
\begin{enumerate}
\item Gets smoother with smaller $\alpha$
\item Gets smoother with larger b
\item Clarifies with larger b if there is a lot of zig-zag in the chart
\item If J increases with b, use smaller $\alpha$
\item interesting but not often used is reducing $\alpha$ with iterations = $\alpha = const/(const+iterations)$

\end{enumerate}

Online learning is extreme example where the data is taken evaluated and dropped immediately

$P(y=1|r;\theta)$ - predicted CTR (Click Through Rate)

\end{document}


\section{Bin smoothers}
A bin smoother, also known as a regressogram, mimics a categorical
smoother by partitioning the predicted value into disjoint and
exhaustive regions, then averaging the response in each
region. Formally, we choose cut-points $c_0 < \dots <c_K$ where $c_0 =
-\infty$ and $c_K = \infty$, and define
\[
R_k = \{i; c_k \leq x_i < c_{k+1} \}; k=0,\dots,K
\]
the indexes of the data points in each region. Then $S[\by|\bx]$ is
given by
\[
s(x_0) = \ave_{i \in R_k}\{ y_i \} \mbox{ if } x_0 \in R_k
\]

Notice that the bin smoother will have discontinuities.

\begin{figure}[htp]
\caption{\label{f2.3} CD4 cell count since seroconversion for HIV
  infected men.}
\centerline{\epsfig{figure=Plots/plot-02-03.ps,angle=270,width=.8\textwidth}}
\end{figure}


\section*{Objective function}
The goal of the problem is to minimize the total amount of penalties generated by near conflicting exams, weighted on the number of enrolled students.
\[
\min{\sum_{i=1}^n \sum_{j=1}^n {\frac{ns_{ij}}{|S|} (p_{1} y_{ij1} + p_{2} y_{ij2} + p_{3} y_{ij3} + p_{4} y_{ij4} + p_{5} y_{ij5})}}
\]
\[
\Rightarrow \min{\frac{1}{|S|} \sum_{i=1}^n \sum_{j=1}^n {ns_{ij} \sum_ {k=1}^5 {p_{k} y_{ijk}}}}
\]


\subsection{Vertex Bends}
\label{sec:Vertex-Bends}

\begin{figure}[!htp]
\centering
\begin{verbatim}
                   e1
             p1 o--------o p0
                        a \
                           \ e2
                            \
                             o p2
\end{verbatim}
\caption{Polyline vertex neighborhood.
\label{fig:Polyline-vertex-neighborhood}}
\end{figure}

In this section, I disuss measures of curvature
based on the bending at each vertex in a polyline.

%------------------------------------------------------------------

\subsubsection{Cosine}
\label{sec:polyline-vertex-cosine}

Consider the non-boundary vertex
at point $\p_0 \in \Reals^3$ in figure \ref{fig:Polyline-vertex-neighborhood}.
$\v$ has degree $2$;
the incident edges are labeled $\e_1$ and $e_2$;
and the neighboring vertices are at points $\p_1 \in \Reals^3$
and $\p_2 \in \Reals^3$.
The shape of the neighborhood is determined by
$\p = (\p_0, \p_1, \p_2) \in \Reals^9$
The unsigned angle between edges $\e_1$ and $\e_2$ is $\alpha(\p)$.

One measure of the amount of bending is the cosine of $\alpha$:
\begin{equation}
\cos(\alpha(\p)) =
{{(\p_1 - \p_0)} \over {\| \p_1 - \p_0 \|} }
\bullet
{{(\p_2 - \p_0)} \over {\| \p_2 - \p_0 \|} }
\end{equation}

A little calculus shows that the partial gradients are:
\begin{eqnarray}
\label{eq:polyline-vertex-cosine-gradient}
\Gf{\p_0}{\cos(\alpha(\p))}
& = &
-
\left[
{{(\p_1 - \p_0) \perp  (\p_2 - \p_0)}
\over
{\| \p_1 - \p_0 \| \| \p_2 - \p_0 \|} }
+
{{(\p_2 - \p_0) \perp  (\p_1 - \p_0)}
\over
{\| \p_1 - \p_0 \| \| \p_2 - \p_0 \|} }
\right]
\\
\Gf{\p_1}{\cos(\alpha(\p))}
& = &
{{(\p_2 - \p_0) \perp  (\p_1 - \p_0)}
\over
{\| \p_1 - \p_0 \| \| \p_2 - \p_0 \|} }
\nonumber
\\
\Gf{\p_2}{\cos(\alpha(\p))}
& = &
{{(\p_1 - \p_0) \perp  (\p_2 - \p_0)}
\over
{\| \p_1 - \p_0 \| \| \p_2 - \p_0 \|} }
\nonumber
\end{eqnarray}

%------------------------------------------------------------------

\subsubsection{Squared Cosine}
\label{sec:polyline-vertex-squared-cosine}

For any positive measure of bending,
minimizing the sum of squared bends,
rather than simply the sum of bends,
will tend to produce a more even distribution of curvature.
Since $\cos(\alpha)$ ranges over $[-1,1]$,
we use $\left( \frac{1 + \cos(\alpha)}{2} \right)^2$.

The gradient is:
\begin{equation}
\Gf{\p}{\left( {{1 + \cos(\alpha(\p))} \over 2} \right)^2}
=
\left( 1 + \cos(\alpha(\p)) \right)
\Gf{\p}{\cos\left(\alpha(\p)\right)}
\end{equation}

$\Gf{\p}{\cos\left(\alpha(\p)\right)}$ is given
in equation \ref{eq:polyline-vertex-cosine-gradient}.

%------------------------------------------------------------------

\subsubsection{Angle}
\label{sec:polyline-vertex-angle}

The angle $\alpha$ is $\pi$ for a straight polyline,
and $0$ for a maximally bent vertex.
Therefore we may choose to minimize the sum of negative angles:
\begin{equation}
-\alpha(\p) =
-\cos^{-1} \left(
{{(\p_1 - \p_0)} \over {\| \p_1 - \p_0 \|} }
\bullet
{{(\p_2 - \p_0)} \over {\| \p_2 - \p_0 \|} }
\right)
\end{equation}

The gradient is:
\begin{equation}
\Gf{\p}{-\alpha(\p)}
=
{{1} \over {\sqrt{ 1 - \cos(\alpha(\p))^2}}}
\Gf{\p}{\cos\left(\alpha(\p)\right)}
\end{equation}

$\Gf{\p}{\cos\left(\alpha(\p)\right)}$ is given
in equation \ref{eq:polyline-vertex-cosine-gradient}.

%------------------------------------------------------------------

\subsubsection{Squared Angle}
\label{sec:polyline-vertex-squared-angle}

Since $-\alpha$ ranges over $[-\pi,0]$
we square $\pi - \alpha$.

The gradient is:
\begin{equation}
\Gf{\p}{\left( \pi - \alpha(\p) \right)^2}
=
{{2 \left( \pi - \alpha(\p) \right)}
\over
{\sqrt{ 1 - \cos(\alpha(\p))^2}}}
\Gf{\p}{\cos\left(\alpha(\p)\right)}
\end{equation}

$\Gf{\p}{\cos \left( \alpha(\p) \right)}$ is given
in equation \ref{eq:polyline-vertex-cosine-gradient}.


%================================
\section{Quotient Spaces}
%================================

%--------------------------------
\begin{definition}
	[quotient topology]
	\label{def: quotient topology}
	Let $(X, \mathcal T)$ be a topological space and let $\sim$ be an equivalence relation on $X$. The \textit{quotient topology} is a topology on $\mathcal P(X/ \sim)$; it is defined as
	$$
	\mathcal T_{X / \sim} = \left\{ U \in \mathcal P(X/\sim) : \{ x \in X: [x] \in U \} \in \mathcal T_X \right\}.
	$$
\end{definition}
%--------------------------------


\documentclass{article}
\usepackage{graphics} % Required for the inclusion of images
\usepackage{graphicx}
\usepackage{caption}
\usepackage{amsmath,amsfonts}
\usepackage{subfigure}
\usepackage{listings}
\title{Math Cookbook}
\author{Junyan Su}
\date{\today}
\begin{document}
\maketitle

\section{Linear Algebra}
\textbf{Sherman–Morrison formula:}
\begin{equation}
\left(A+u v^{\top}\right)^{-1}=A^{-1}-\frac{A^{-1} u v^{\top} A^{-1}}{1+v^{\top} A^{-1} u}
\end{equation}
\begin{equation}
(A+U C V)^{-1}=A^{-1}-A^{-1} U\left(C^{-1}+V A^{-1} U\right)^{-1} V A^{-1}
\end{equation}

\section{Caculus}
\noindent\textbf{Leibniz integral rule:}
\begin{equation}
\frac{d}{d x}\left(\int_{a(x)}^{b(x)} f(x, t) d t\right)=f(x, b(x)) \cdot \frac{d}{d x} b(x)-f(x, a(x)) \cdot \frac{d}{d x} a(x)+\int_{a(x)}^{b(x)} \frac{\partial}{\partial x} f(x, t) d t
\end{equation}
\textbf{Grönwall's inequality} from two useful references~\cite{jonesFundamental1964,mitrinovicInequalities2012}.
\begin{align}
	f(x) &\geq a(x)+b(x) \int_{\underline{x}}^{x} f(u) d u, x \in[\underline{x}, \bar{x}] \\
	\implies f(x) &\geq a(x)+b(x) \int_{\underline{x}}^{x} a(u) \exp \left(\int_{u}^{x} b(s) d s\right) d u, \quad x \in[\underline{x}, \bar{x}]
\end{align}

\noindent\textbf{Grönwall's inequality in differential form:}
\begin{align}
	f'(x) &\leq b(x) f(x) \\
	\implies f(x) &\leq f(\underline{x}) \exp\left( \int_{\underline{x}}^{x} b(s) ds \right)
\end{align}

\bibliographystyle{plain}
\bibliography{ref}

\end{document}