<filename>slides/evaluation/slides-evaluation-intro.tex
\input{../../style/preamble}
\input{../../latex-math/basic-math.tex}
\input{../../latex-math/basic-ml.tex}

\newcommand{\titlefigure}{figure_man/eval_fig_title_intro.png}
\newcommand{\learninggoals}{
\item Understand the goal of performance estimation
\item Understand the difference between outer and inner loss
\item Know the definition of generalization error}


\title{Introduction to Machine Learning}
% \author{<NAME>, <NAME>, <NAME>, <NAME>}
\institute{\href{https://compstat-lmu.github.io/lecture_i2ml/}{compstat-lmu.github.io/lecture\_i2ml}}
\date{}

\begin{document}

\lecturechapter{Evaluation: Introduction and Remarks}
\lecture{Introduction to Machine Learning}

% \begin{vbframe}{Goals of performance evaluation}
% 
% 
% \textbf{Model evaluation}\\
% <<out.width = "0.2\\textwidth", fig.height=5>>=
% set.seed(31415)
% 
% x = 1:5
% y = 2 + 0.5 * x + rnorm(length(x), 0, 1.5)
% dat = data.frame(x = x, y = y)
% model = lm(y ~ x)
% dat$y_hat <- predict(model)
% 
% ggplot(data = dat, aes(x = x, y = y)) +
%     geom_abline(intercept = coef(model)[1], slope = coef(model)[2]) +
%     geom_segment(aes(xend = x, yend = y_hat), color = "red") +
%     geom_point() +
%     theme_void()
% @
%      
%      
% 
% \textbf{Algorithm evaluation}\\
% \begin{center}
%     { \Huge \checkmark }
% \end{center}
% 
% 
% \textbf{Model selection}
% \begin{center}
%     \begin{tikzpicture}
%     \draw [thick] (0,0) rectangle (.5,.5);
%     \draw [fill=green, thick] (0.5,0) rectangle (1,1);
%     \draw [thick] (1,0) rectangle (1.5,.75);
%     \end{tikzpicture}
% \end{center}
% 
% \textbf{Learning curve estimation}
% <<fig.height=2, fig.width=5, out.width="0.2\\textwidth">>=
% # rsrc data from rsrc/learning_curve.R
% load("rsrc/learning_curve.RData")
% 
% opt = res.rpart
% opt$mmce = opt$mmce - mean(opt$mmce) + 0.08
% 
% ggplot(data = opt, aes(x = percentage, y = mmce)) + 
%     geom_line(aes(colour = measure), size = 3) + 
%     theme_void() + theme(legend.position = "none")
% @
%     
% \end{vbframe}
% 
% 
% 
% \begin{vbframe}{Goals of performance evaluation}
% 
% \begin{itemize}
%   \item \textbf{Model evaluation:}
%     Estimate \emph{generalization} error of a model on new (unseen) data, drawn from the same data generating process that training data came from.
%   \item \textbf{Algorithm evaluation:}
%     Estimate \emph{generalization} error of a learning algorithm, trained on a data set
%     of a certain size, on new (unseen) data, all drawn from the same data generating process.
%   \item \textbf{Model selection:}
%     Select the best model from a set of potential candidate models (e.g., different model classes, different
%     hyperparameter settings, different features)
%   \item \textbf{Learning curve estimation:}
%     How does the generalization error scale when an algorithm is trained on training sets of different sizes?
% \end{itemize}
% 
% All goals are related, i.e., reliable estimation of (predictive) performance is the foundation for all of them.
% 
% \end{vbframe}

% \begin{vbframe}{Performance Estimation}
% \begin{itemize}
% \item Goal: Estimate performance on new data 
% \item For now, we start by assuming to have a fixed model, already fitted on some data.
% \item We also assume to have some reasonable data for testing available. 
%  \item ML performance evaluation provides clear and simple protocols for reliable model validation. 
%    These protocols are often much simpler than classical statistical model diagnosis and rely only on few assumptions
%  \item Most important assumption: Data we use is realistic and i.i.d.  
% \item ML evaluation is still hard enough and offers LOTS of options to cheat yourself and especially your clients, mistakes can happen on many levels. 
% \end{itemize}
% \end{vbframe}


\begin{vbframe}{Performance Evaluation}
\begin{center}
How well does my model perform...\\

\lz

\begin{tikzpicture}
    \draw [fill=purple, thick] (0,0) rectangle (.5,.5);
    \draw [fill=green, thick] (0.5,0) rectangle (1,1);
    \draw [fill=orange, thick] (1,0) rectangle (1.5,.75);
\end{tikzpicture}

\lz
... on data from the same data-generating process?\\
\lz
In practice:\\
\lz
... on current data (training data)?\\
... on new data (test data)?\\
... based on a certain measure/metric?\\
...

\end{center}
\end{vbframe}


% \begin{vbframe}{Performance Evaluation: Assumption}
% (for now)
% \begin{itemize}
% \item We have a fixed model, already fitted on some data.
% \item We have appropriate test data. 
% \item Data is realistic and i.i.d.
% \end{itemize}
% \end{vbframe}


\begin{vbframe}{Performance Evaluation}
ML performance evaluation provides clear and simple protocols for reliable model
validation. 

\begin{itemize}
\item Often simpler than classical statistical model diagnosis 
\item Relies only on few assumptions
\item Still hard enough and offers \textbf{lots} of options to cheat / make mistakes 
\end{itemize}
\end{vbframe}

\begin{vbframe}{Performance Measures}

We measure performance using a statistical estimator for the 
\textbf{generalization error} (GE).

\lz
$\text{GE} = $ expected loss of a fixed model

\lz
$\hat{\text{GE}} = $ average loss 

\lz \lz

Example: Mean squared error (L2 loss)
\[
\hat{\text{GE}} = MSE = \meanin (\yi - \yih)^2
\]

\end{vbframe}

% \begin{vbframe}{Performance Estimation}
%  Clearly, we are looking for a statistical estimator - for the generalization error! 
%  Different levels of randomness are involved: 
% \begin{itemize}
%  \item Even if we are evaluating on a fixed test data set, this is only a sample and not full reality. 
%    The sample can be too small, then our estimator will be of high variance; 
%    the sample could not be from the distribution of interest, then our estimator will be biased.
%  \item The same holds true for our model - it was only fitted on a sample. This creates another source of randomness, the training
%    data sample. This is true, even if the fitting algorithm is deterministic. 
% \item In ML, many learning algorithms are stochastic: Think random forest, or stochastic gradient descent. This is a third source of randomness.
% \end{itemize}
% \end{vbframe}

\begin{vbframe}{Measures: Inner vs. Outer Loss}

\begin{center}
\lz
Inner loss = loss used in learning\\
\lz
Outer loss = loss used in evaluation\\
= evaluation measure
\lz
%% CC-0 https://publicdomainvectors.org/en/free-clipart/Grumpy-student-vector-image/76859.html

\includegraphics[width=0.3\textwidth]{figure_man/student.pdf}\\

\end{center}

\end{vbframe}



% \begin{vbframe}{Measures: Inner vs. Outer Loss}
% 
% \begin{itemize}
% \item To judge the performance of a model on a given data set, we might want to produce a quantitative 
% measure of the performance on that set. 
% \item Usually we define a function that measures the quality of a prediction per observation.
% \item We then aggregate over the complete set - often by some form of averaging.
% \end{itemize}
% 
% \lz
% 
% Don't we already know this? Sounds like loss functions and risk estimation? 
% It nearly is the same.
% \end{vbframe}

% \begin{vbframe}{Measures: Inner Loss}
% \begin{itemize}
% \item We already covered this, its associated empirical risk is optimized during model fitting
% \item The keyword above is \textbf{optimization}, some functions are much tamer to handle than others, 
%   smoothness and differentiability are often required for efficient optimization
% \item For this reason, we often choose something that's easier to handle numerically
% \item Another pretty practical reason might be: Our toolkit of choice only implements the optimization of a certain inner loss; 
%   changing the outer loss to something custom is simple, changing the inner often is not
% \end{itemize}
% \end{vbframe}
% 
% \begin{vbframe}{Measures: Outer Loss}
% \begin{itemize}
% \item Performance measure to assess the model
% \item Should be carefully considered and selected
% \item There are no objectively better or worse measures - YOU have to select depending on your application, domain
%   and what you want
% \item Except for \enquote{should be reasonable and reflect what I want} there are no huge requirements for an outer measure, it is a function which takes a vector of prediction and a vector of labels and evaluates them
% \item Think about what will happen after a (wrong) prediction of your model, that should help to design an outer loss
% \item Yes, in model selection (later), we optimize it, but we use special techniques there anyway that can deal with arbitrary measures 
% \end{itemize}
% \end{vbframe}


\begin{vbframe}{Measures: Inner vs. Outer Loss}
\lz
Optimally: inner loss = outer loss\\[.5em]
Not always possible:\\ some losses are hard to optimize / no loss is specified directly\\

\lz

\underline{Example}:\\[.5em]
\begin{tabular}{ll}
Logistic Regression & $\rightarrow$ minimize binomial loss \\
kNN & $\rightarrow$ no explicit loss minimization\\
\end{tabular}
\begin{itemize}
  \item When evaluating the models we might be interested in (cost-weighted) classification error 
  \item Or some of the more advanced measures from ROC analysis like AUC
\end{itemize}

% Nowadays, one can also optimize many of the harder losses directly, but this is less standard, less
% often implemented and we will not cover this here.

\end{vbframe}

\endlecture

\end{document}
