\documentclass[
  aspectratio=1610, %default screen size (4:3); other values: 1610, 169, 149, 54, 32
% handout,
%  smaller, %uncomment this to fit more text onto the slides (similar to the CD suggestions)
  intlimits %put limits of integrals above and below the integral symbol
]{beamer}

%\usepackage[ngerman]{babel}
%\usepackage[english]{babel}
%\usepackage[UKenglish]{babel}

%my favourite packages
%\usepackage[latin1]{inputenc} %type in accented characters easily, i. e. ä instead of \"a
%\usepackage{siunitx} %easy input & nice formatting of phys. quantities (number + SI unit)
%\usepackage{latexsym}
%\usepackage{amssymb}
%\usepackage{movie15}
%\usepackage{animate}
%\usepackage{ragged2e}\RaggedRight
\usepackage{ifthen}
\usepackage{mathtools}
\DeclareMathOperator*{\argmin}{arg\,min}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%% some optional settings %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%uncomment this if you want the title page text aligned flushleft; CD rules seem to
%demand this, although it looks awful
%\def\titleflushleft{}

%uncomment this if you want frame titles centered
%\def\frametitlecentered{}

%uncomment this if you don't like the wallclock
%do not want clock, does not work with okular
\def\noclock{}

%uncomment this if you don't like the navigation buttons
%\def\nonavigation{}

%uncomment this if you want Arial as the main text font (very ugly),
%no matching math font, using CM-Bright for math text
%\def\arial{}

%uncomment this if you want Helvetica as the main text font (very ugly),
%Helvetica is very similar to Arial; again, no matching math font, using CM-Bright for
%math text
%\def\helvetica{}


\usepackage{helmholtzai}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\hypersetup{%
  breaklinks,colorlinks,
  %pdfpagemode=FullScreen,
  pdfhighlight=/P,
  linkcolor=hgfdarkblue,
  linkcolor=hgfdarkblue,
  anchorcolor=hgfdarkblue,
  citecolor=hgfdarkblue,
  filecolor=hgfdarkblue,
  menucolor=hgfdarkblue,
  runcolor=hgfdarkblue,
  urlcolor=hgfdarkblue
}

% \usepackage[T1]{fontenc}
% \usepackage[utf8]{inputenc}
% \usepackage{sistyle}
% \usepackage[english]{isodate}
% \usepackage[version=3]{ageschem}
%\usepackage{picins}
% \usepackage{url}
% \usepackage{tabularx}
% \usepackage{multimedia}

%\usepackage{etex}
\usepackage{tikz}
\usetikzlibrary{arrows.meta,arrows,decorations.pathmorphing,backgrounds,positioning,fit,petri,calc,shapes.geometric,scopes,fit,spy}
\usepackage{pgfplots}
\pgfplotsset{,compat=1.11}
\tikzset{/pgf/number format/1000 sep={\,}}
%\usepackage[version=3]{mhchem}
\usepackage{calc}
\usepackage[fixed]{fontawesome5}

%\usepackage[export]{adjustbox}

%\usepackage{enotez}
%\def\enmarkstyle{\tiny}

% \usepackage{python}

% \usepackage{dashrule}

%\usepackage{appendixnumberbeamer}

%\input{UserFunkt.tex}
%\SIstyle{USA}

\title{%
 How did we learn
}
\subtitle{DeepLearning540 - Lesson 06
}
%[short version of authors to be used in footlines]{long version in the title page}
\author{{<NAME>}}

\date{3rd~March~2021}

\institute{%
 \iflanguage{ngerman}{%
  Zentralabteilung Informationsdienste und Computing, Helmholtz-Zentrum Dresden-Rossendorf
 }{%
  Department of Information Services and Computing, Helmholtz-Zentrum Dresden-Rossendorf
 }%
}



%%%%%%%%%%%% document content starts here  %%%%%%%%%%%%%%%%%%%%%%%%%%%
%\AtBeginSubsection

\definecolor{darkgreen}{RGB}{0,130,0}

% \makeatletter
% \let\old@section\section
% \let\section\relax
% \newcommand\section{%
%  \@ifstar\mySectionStar\mySectionNoStar%
% }%
% \newcommand\mySectionNoStar[1]{%
%   \old@section{#1}%
%   \noautobookmark%add excerpt from toc to beginning of section
%   \frame{\frametitle{#1}\tableofcontents[currentsection]}%
%   \autobookmark
% }%
% \newcommand\mySectionStar[1]{%
%   \old@section*{#1}%
% }%
% \makeatother

\renewcommand\footnotesize{\tiny}
\newlength\mytextheight

\newlength\tmplength

\renewcommand\emph{\textbf}

\input{figures/ann.tex}
\input{figures/mlp.tex}

\renewcommand{\arraystretch}{1.2}

% required for pygment syntax-highlighting
\usepackage{fancyvrb}
\input{code/pygments.tex}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%

\begin{document}

\maketitle

\begin{frame}
 \frametitle{Recap: Fitting the Penguin Dataset}


 \begin{block}{inputs}
   \begin{itemize}
   \item 3 features \newline
     (\texttt{flipper\_length\_mm}, \texttt{bill\_depth\_mm}, \texttt{bill\_length\_mm})
   \item 3 inputs means $\vec{x} \in \mathbb{R}^{3}$
   \end{itemize}
 \end{block}

 \begin{block}{outputs}
   \begin{itemize}
   \item probability for being of \texttt{Adelie} or \texttt{not Adelie}
   \item using one-hot-encoding
   \item 2 outputs means  $\vec{y} \in \mathbb{R}^{2}$
   \end{itemize}
 \end{block}


 \begin{alertblock}{Note}
 \centering
   \textbf{classification with supervised learning!}
 \end{alertblock}


\end{frame}


\begin{frame}
 \frametitle{Code that needs (some) explanation}

 \input{code/keras_model_compile.tex}

\end{frame}

\begin{frame}
  \frametitle{supervised learning in one slide\footnote{credits to Uwe Schmidt}}
  \vfill
  \begin{exampleblock}{}
   \begin{itemize}
   \item given $m$ input pairs in a dataset $\mathcal{D} = \{\langle \vec{x}_i, y_i\rangle \dots \}$ with $x \in \mathbb{R}^n$, $y \in \mathbb{R}$
   \item we would like to train a model $f$ with parameters $\vartheta$ such that
     \begin{equation*}
     \vec{y} = f(\vec{x}, \vartheta)
   \end{equation*}
   \item we optimize a loss function $\mathcal{L}$ in such a fashion that
     \begin{equation*}
     \vartheta \approx \argmin_{\vartheta} \mathcal{L}( \vec{y}^{\,true}, f(\vec{x}, \vartheta) )
   \end{equation*}
   \item during optimisation with gradient descent, parameters $\vartheta$ at step $s+1$ are given by

     \begin{equation*}
\vartheta_{s+1} = \vartheta_{s} - \eta \nabla_{\vartheta}\mathcal{L}( \vec{y}^{\,true}, f(\vec{x}, \vartheta_{s}))
\end{equation*}

   \end{itemize}
 \end{exampleblock}
  \vfill

\end{frame}


 \begin{frame}
 \frametitle{Weight Update Rule}

 \begin{equation*}
\vartheta_{s+1} = \vartheta_{s} - \eta \nabla_{\vartheta}\mathcal{L}( \vec{y}^{\,true}, f(\vec{x}, \vartheta_{s}))
\end{equation*}

\begin{exampleblock}{gradient descent}
  \begin{center}
    free parameter $\eta$ is known as the step size or learning rate
  \end{center}
\end{exampleblock}

\begin{columns}
  \begin{column}{.45\textwidth}
    \begin{alertblock}{Classic Gradient Descent}
    \begin{itemize}
    \item using the full training set
    \item problem: does not scale with size of dataset
    \end{itemize}
  \end{alertblock}
\end{column}

\begin{column}{.45\textwidth}

  \begin{alertblock}{``Online'' Gradient Descent}
    \begin{itemize}
    \item using the one datum at a time
    \item problem: expensive to compute, strongly depends on $\eta$
    \end{itemize}
  \end{alertblock}

  \end{column}
\end{columns}

\end{frame}

 \begin{frame}
   \frametitle{Stochastic gradient descent}
   \vspace{-1em}
\begin{columns}
  \begin{column}{.475\textwidth}
    \begin{equation*}
\vartheta_{s+1} = \vartheta_{s} - \frac{\eta}{k} \sum_{b=1}^{k} \nabla_{\vartheta}\mathcal{L}_b( \vec{y}^{\,true}_b, f(\vec{x}_b, \vartheta_{s}))
\end{equation*}

\begin{block}{mini-batch based stochastic gradient descent}

  \begin{itemize}
  \item randomly split training set in mini batches, e.g. $b=32$
  \item completing one batch is referred to as a \texttt{step}
  \item completing the entire dataset is referred to as an \texttt{epoch}
  \item after each epoch: batches are randomized again
  \end{itemize}
\end{block}
\end{column}

\begin{column}{.475\textwidth}
  \centering
  \textbf{loss landscape}
  \footnotesize
  \includegraphics[width=.9\textwidth]{figures/stepsize}\\
  from \href{https://cs231n.github.io/optimization-1/}{Stanford CS231n}
  \end{column}
\end{columns}
\end{frame}

\begin{frame}
 \frametitle{Loss function: categorical cross-entropy}

 \begin{columns}
 \begin{column}{.49\textwidth}
 \begin{exampleblock}{in general}
   \begin{align*}
     H(p, q) &= - \sum_{x \in \mathcal{X}} p(x) \log q(x)\\
             &= H(p) + D_{KL}(p || q)
 \end{align*}
   \begin{itemize}
   \item from information theory
   \item defined above between two discrete probability distributions $p$ and $q$
   \item relates to the Kullback-Leibler divergence
   \end{itemize}
 \end{exampleblock}
\end{column}

 \begin{column}{.49\textwidth}
 \begin{exampleblock}{for binary classification}
   \begin{itemize}
   \item probability $p$ refers to the true label
   \item probability $q$ refers to the predicted label (as output of the softmax layer)
 \end{itemize}

   \begin{align*}
 \mathcal{L}( \vec{y}^{\,true}, f(\vec{x}, \vartheta)) &= - \sum_{x \in \mathcal{X}} p(x) \log q(x) \\
                                             &= - p_{y=1} \log q_{\hat{y}=1} \\
                                               &\quad - (1-p_{y=1}) \log (1-q_{\hat{y}=1})
   \end{align*}

\end{exampleblock}

\end{column}
\end{columns}


\end{frame}


 \begin{frame}
 \frametitle{Softmax Activation}
 \begin{equation*}
\sigma(x_i) = \frac{\exp{x_i}}{\sum_{j=1}^{n} \exp{x_j}}
\end{equation*}
\begin{itemize}
\item mapping of input $x \in \mathbb{R}^n$ to output $\mathbb{R}^n$
\item common activation layer for classification networks in last layer
\item outputs are $(0,1)$
\item converts input to probability distribution over predicted output classes
\end{itemize}
\end{frame}

\renewcommand{\inText}[1]{
 $I_{#1}$
}
\renewcommand{\hidText}[1]{
 {$F_{relu}(H_{#1})$}%
}
\renewcommand{\outText}[1]{
 $F_{smax}(O_{#1})$%
}
\renewcommand{\synapseIHText}[2]{%
 {\ifthenelse{\equal{#1}{1}}{}{\small$W^{(I)}_{#1#2}$}}}
\renewcommand{\synapseHOText}[2]{\small$W^{(H)}_{#1#2}$}

 \begin{frame}
 \frametitle{A Simple MLP}

 \begin{columns}
   \begin{column}{.7\textwidth}
     \tikzANNwBottomIns{neuron/.append style={minimum size=2em}}
     {\node[below=.6em of hid 2] {%
         $+\vec{b_H}$};
       \node[below=.6em of out 1] {%
         $+\vec{b_O}$};
     }
   \end{column}
   \begin{column}{.28\textwidth}

     \begin{exampleblock}{But how to compute the gradient?}
       \vspace{-1em}
       \Large
       \begin{equation*}
         \nabla_{\vartheta}\mathcal{L} ( \vec{y}^{\,true}, f(\vec{x}, \vartheta) )
       \end{equation*}
       \small
       \centering
     ($\vartheta \in \{W^{(i)}, b_i\}$)
   \end{exampleblock}

   \end{column}
 \end{columns}

\end{frame}

\begin{frame}
  \frametitle{Backpropagation in 1D}
  \centering
  \vspace{-5em}
 \begin{tikzpicture}[shorten >=1pt,node distance=3cm,auto]

   \node[ultra thick, draw, circle] (input) {Input $x$};
   \node[ultra thick, draw, ellipse] (hidden) [right of=input] {$F(w \cdot x + b)$};
   \node[ultra thick, draw, circle] (output) [right of=hidden] {Output $o$};
   \node[ultra thick, draw, rectangle, minimum width=2.5cm] (loss) [right of=output] {$\mathcal{L}(y, \hat{y})$};

   \path[->] (input) edge node {w} (hidden)
   (hidden) edge (output)
   (output) edge node {$\hat{y}$} (loss);

   \node (ytrue) [right of=loss, node distance=2cm] {$y$};
   \path[->] (ytrue) edge (loss);

 \end{tikzpicture}
 \vfill

\end{frame}

\begin{frame}
  \frametitle{Backpropagation in 1D, plugging in numbers}
  \centering
  \vspace{-5em}
 \begin{tikzpicture}[shorten >=1pt,node distance=3cm,auto]

   \node[dashed,draw, rectangle, text width=1.25cm, align=right] (fixed) [left of=input, node distance=2cm] {$x=2$ $w=.5$ $b=1$};

   \node[ultra thick, draw, circle] (input) {Input $x$};
   \node[ultra thick, draw, ellipse] (hidden) [right of=input] {$F(w \cdot x + b)$};
   \node[ultra thick, draw, circle] (output) [right of=hidden] {Output $o$};
   \node[ultra thick, draw, rectangle, minimum width=2.5cm] (loss) [right of=output] {$\mathcal{L}(y, \hat{y})$};

   \path[->] (input) edge node {w} (hidden)
   (hidden) edge (output)
   (output) edge node {$\hat{y}$} (loss);

   \node (ytrue) [right of=loss, node distance=2cm] {$y$};
   \path[->] (ytrue) edge (loss);

 \end{tikzpicture}
 \vfill

\end{frame}

\renewcommand{\inText}[1]{
 $I_{#1}$
}
\renewcommand{\hidText}[1]{
 {$H_{#1}$}%
}
\renewcommand{\outText}[1]{
 $O_{#1}$%
}
\renewcommand{\synapseIHText}[2]{%
 {\ifthenelse{\equal{#1}{1}}{}{\small$W^{(I)}_{#1#2}$}}}
\renewcommand{\synapseHOText}[2]{\small$W^{(H)}_{#1#2}$}

\begin{frame}
 \frametitle{For a single weight}

 \centering
     \tikzANNwBottomIns{neuron/.append style={minimum size=2em}}
     { \node (lastmid) at($(out 0)!.5!(out 1)$) {};
       \node[draw,circle, minimum width=1.75em] (yhatlabel) [right of=lastmid, node distance=1.5cm] {$\hat{y}$};
       \path[-] (out 0) edge (yhatlabel)
                 (out 1) edge (yhatlabel);
      %  \node[below=.6em of out 1] {%
      %    $+\vec{b_O}$};
     }

     \begin{equation*}
       \frac{\partial \mathcal{L}}{\partial W^{I}_{00}} = \frac{\partial \mathcal{L}}{\partial \hat{y}} %
       \frac{\partial \hat{y}}{\partial O_{0}}%
       \frac{\partial O_{0}}{\partial W_{00}^{H}}%
       \frac{\partial W_{00}^{H}}{\partial W_{00}^{I}} + %
       \frac{\partial \mathcal{L}}{\partial \hat{y}} %
       \frac{\partial \hat{y}}{\partial O_{1}}%
       \frac{\partial O_{1}}{\partial W_{01}^{H}}%
       \frac{\partial W_{01}^{H}}{\partial W_{00}^{I}}
     \end{equation*}


\end{frame}

 \begin{frame}
     \frametitle{Take Aways}

     \vfill
     \begin{itemize}\Large
     \item (mini-batch) stochastic gradient descent (SGD) typically default optimizers for deep learning
     \item weight update rule governs how to update model parameters
     \item loss function defines optimisation landscape
     \item gradient for weight update obtained by backpropagation (chain rule)
     \end{itemize}
     \vfill

   \end{frame}


   \begin{frame}
     \frametitle{Further Reading}

     \vfill
     \begin{itemize}
     \item Excellent overview of optimisation algorithms \href{https://ruder.io/optimizing-gradient-descent/}{ruder.io/optimizing-gradient-descent}
     \item a classic resource getting introduced to gradient descent at \href{https://cs231n.github.io/optimization-1/}{Stanford CS231n} course

     \item cross-entropy \href{https://towardsdatascience.com/data-science-interview-deep-dive-cross-entropy-loss-b10355eb4ace}{well explained for the statistically inclined}
     \item \href{https://mml-book.github.io/}{Mathematics for Machine Learning book} by Deisenroth, <NAME>, and <NAME> is an excellent resource to learn about mathematical tools used for ML

     \item some of the material is based on and inspired by Sebastian Raschka's lecture \href{L6.2 Understanding Automatic Differentiation via Computation Graphs}{https://youtu.be/oY6-i2Ybin4}

     \end{itemize}
     \vfill
   \end{frame}

% \begin{frame}
%  \frametitle{Artificial Neural Networks---Activation functions}
%  \begin{columns}
%   \begin{column}{\linewidth/5*3}
%    \begin{tikzpicture}
%      \begin{axis} [
%        grid=both,minor tick num=0,
%       legend pos=north west,
%       legend/.style={font=\small},
%       xmin=-2,xmax=2,
%       ymin=-.1,ymax=1.1,
%       xlabel=$x$,
%       width=\linewidth,
%      ]
%      \addplot[mark=none,blue] {1/(1+exp(-x))};
%      \addlegendentry{$\sigma(x)$}
%      % \only<2>{
%       \addplot[mark=none,domain=-2:0] {0};
%       \addplot[mark=none,domain=0:2] {x};
%       \addlegendentry{$R(x)$}
%      %}
%     \end{axis}
%    \end{tikzpicture}
%   \end{column}
%   \begin{column}{\linewidth/5*2}
%    \begin{itemize}
%     \item sigmoid
%      \[
%       \sigma(x) = \frac{1}{1+\mathrm{e}^{-x}}
%      \]
%     \item ReLU
%      \[
%       R(x) = \begin{cases}
%        0 & x \leq 0 \\
%        x & x > 0\\
%       \end{cases}
%     \]

%    \end{itemize}
%   \end{column}
%  \end{columns}
% \end{frame}


\end{document}
