<filename>slides/evaluation/slides-evaluation-test.tex<gh_stars>10-100
\input{../../style/preamble}
\input{../../latex-math/basic-math.tex}
\input{../../latex-math/basic-ml.tex}


\newcommand{\titlefigure}{figure/eval_test_3}
\newcommand{\learninggoals}{
\item Understand the definition of test error
\item Understand how overfitting can be seen in the test error}
\usepackage{../../style/lmu-lecture}


\title{Introduction to Machine Learning}
% \author{<NAME>, <NAME>, <NAME>, <NAME>}
\institute{\href{https://compstat-lmu.github.io/lecture_i2ml/}{compstat-lmu.github.io/lecture\_i2ml}}
\date{}


\begin{document}



% This file loads R packages, configures knitr options and sets preamble.Rnw as parent file
% IF YOU MODIFY THIS, PLZ ALSO MODIFY setup.Rmd ACCORDINGLY...








% Defines macros and environments

%! includes: evaluation-train

\lecturechapter{Evaluation: Test Error}
\lecture{Introduction to Machine Learning}


% \begin{vbframe}{Test Error and Hold-Out Splitting}
% To measure performance, let’s simulate how our model will be applied on new, unseen data.\\
% \lz
% $\rightarrow$ Predict only on data not used during training and measure performance there.\\[.5em]
% $\rightarrow$ For a given set $D$, we have to preserve some data for testing that we cannot use for training.
% \end{vbframe}


\begin{vbframe}{Test Error}

% FIGURE SOURCE: https://docs.google.com/drawings/d/1q7WN1_YKHedIPNySiZBEraLtTkHRX12Ej6M6ISbfMD0/edit?usp=sharing
\includegraphics[width=0.8\textwidth, trim=270 0 0 0, clip]{figure_man/test_error.pdf}

\end{vbframe}



\begin{vbframe}{Test Error and Hold-Out Splitting}
\begin{itemize}
  \item Split data into 2 parts, e.g., 2/3 for training, 1/3 for testing
  \item Evaluate on data not used for model building
\end{itemize}

% FIGURE SOURCE: https://docs.google.com/drawings/d/1q7WN1_YKHedIPNySiZBEraLtTkHRX12Ej6M6ISbfMD0/edit?usp=sharing
\includegraphics[width=\textwidth]{figure_man/test_error.pdf}

\end{vbframe}

\begin{vbframe}{Test Error}

Let's consider the following example:\\
Sample data from sinusoidal function
$0.5 + 0.4 \cdot \sin (2 \pi x) + \epsilon$\\
\lz
\begin{knitrout}\scriptsize
\definecolor{shadecolor}{rgb}{0.969, 0.969, 0.969}\color{fgcolor}

{\centering \includegraphics[width=0.8\textwidth]{figure/eval_test_1} 

}



\end{knitrout}
Try to approximate with a $d^{th}$-degree polynomial:
\[ \fxt = \theta_0 + \theta_1 x + \cdots + \theta_d x^d = \sum_{j = 0}^{d} \theta_j x^j\text{.} \]
\end{vbframe}


\begin{vbframe}{Test Error}
\begin{knitrout}\scriptsize
\definecolor{shadecolor}{rgb}{0.969, 0.969, 0.969}\color{fgcolor}

{\centering \includegraphics[width=0.95\textwidth]{figure/eval_test_2} 

}



\end{knitrout}

\begin{itemize}
\item d=1: MSE = 0.038: Clear underfitting
\item d=3: MSE = 0.002: Pretty OK
\item d=9: MSE = 0.046: Clear overfitting
\end{itemize}

\end{vbframe}


\begin{vbframe}{Test Error}
Plot evaluation measure for all polynomial degrees:
\begin{knitrout}\scriptsize
\definecolor{shadecolor}{rgb}{0.969, 0.969, 0.969}\color{fgcolor}

{\centering \includegraphics[width=0.9\textwidth]{figure/eval_test_3} 

}



\end{knitrout}

Increase model complexity (tendentially)
\begin{itemize}
\item decrease in training error\\
\item U-shape in test error\\ 
(first underfit, then overfit, sweet-spot in the middle)
\end{itemize}
\end{vbframe}

\begin{vbframe}{Test error Problems}
\begin{itemize}
  \item Test data has to be i.i.d. compared to training data.
  \item Bias-variance of hold-out:\\
  \begin{itemize}
    \item The smaller the training set, the worse the model $\rightarrow$ biased estimate.\\
    \item The smaller the test set, the higher the variance of the estimate.
  \end{itemize}   
  \item If the size of our initial, complete data set $\D$ is limited,
  single train-test splits can be problematic.
\end{itemize}
\end{vbframe}

\begin{vbframe}{Test error Problems}
A major point of confusion:
\begin{itemize}
\item In ML we are in a weird situation. We are usually given one data set. At the end of our model selection and evaluation process
we will likely fit one model on exactly that complete data set. As training error evaluation does not work,
we have nothing left to evaluate exactly that model.
\item Hold-out splitting (and resampling) are tools to estimate the future
performance. All of the models produced during that phase of evaluation are intermediate results.
% \item Keep that already in mind now, it will help to avoid confusion when we move on to cross-validation and nested cross-validation.
\end{itemize}
\end{vbframe}



% \begin{vbframe}{Test error Problems}
% Let's produce repeated 2/3 training, 1/3 testing splits on two ML tasks:\\ 
% a) iris ($n = 150, n_\text{test} \approx 50$) \\
% b) sonar ($n = 208, n_\text{test} \approx 70$).
% 
% \lz
% Plots below show the strong stochastic fluctuation of test errors\\
% (50 repeats).
% 
% <<echo=FALSE, fig.width = "\\textwidth", fig.height = 3, fig.align="center">>=
% my_ss = function(task) {
%   lrn = makeLearner("classif.naiveBayes")
%   r1 = subsample(lrn, task, iters = 50, show.info = FALSE)
%   return(r1$measures.test[,2])
% }
% err1 = my_ss(iris.task)
% err2 = my_ss(sonar.task)
% d = rbind(
%   data.frame(task = "iris", mce = err1),
%   data.frame(task = "sonar", mce = err2)
% )
% 
% ggplot(d, aes(x = mce)) + geom_histogram() + 
%     facet_wrap(vars(task), scales = "free_x") +
%     xlab("MCE")
% @
% \end{vbframe}
% 
% 

% 
% 
% % \begin{vbframe}{Training vs. test error}
% %   \vspace{-0.25cm}
% %   \begin{blocki}{The training error}
% %   \vspace{-0.25cm}
% %     \item is an over-optimistic (biased) estimator as the performance is measured on the same data the learned model was trained for
% %     \item decreases with smaller training set size as it is easier for the model to learn the underlying structure in the training set perfectly
% %     \item decreases with increasing model complexity as the model is able to learn more complex structures
% %   \end{blocki}
% %   \vspace{-0.25cm}
% %   \begin{blocki}{The test error}
% %   \vspace{-0.25cm}
% %   \item will typically decrease when the training set increases as the model generalizes better with more data (more data to learn)
% %   \item will have higher variance with decreasing test set size
% %   \item will have higher variance with increasing model complexity
% %   \end{blocki}
% % \end{vbframe}
% 
% 
% % \framebreak
% %
% % Visualize the perfomance estimator - and the MSE of the estimator - in relation to the true error rate.
% 
% % \begin{vbframe}{Bias-Variance of Hold-Out}
% % \begin{itemize}
% % \item If the size of our initial, complete data set $\D$ is limited,
% %   single train-test splits can be problematic.
% % \item The smaller our single test set is, the higher the variance
% %   of our estimated performance error (e.g., if we test on one observation, in the extreme case).
% %   But note that by just making the test set smaller, we do not introduce any bias,
% %   as we simply average losses on i.i.d. observations from $\Pxy$.
% % \item The smaller our training set becomes, the more pessimistic bias we introduce into the model.
% %   Note that if $|D| = n$, our aim is to estimate the performance of a model fitted
% %   on $n$ observations (as this is what we will do in the end). If we fit on less data during
% %   evaluation, our model will learn less, and perform worse. Very small training sets will also
% %   increase variance a bit.
% % \end{itemize}
% % \end{vbframe}
% 
% \begin{vbframe}{Bias-Variance of Hold-Out - Experiment}
% \begin{itemize}
% \item Data: simulate spiral data (sd = 0.1) 
% \item Learner: CART 
% \item Goal: estimate real performance of a model with $|\Dtrain| = 500$
% \item Get the "true" estimator by repeatedly sampling 500 observations from the simulator, fit the learner, then evaluate on $10^5$ observations
% \item Sample $\D$ with $|\D| = 500$ and analyze different split-rate $s \in \{0.05, 0.1, ..., 0.95\}$ for training with $|\Dtrain| = s \cdot 500$
% \item Estimate performance on $\Dtest$ with $|\Dtest| = 500 \cdot (1 - s)$
% \item Repeat the experiment for each split rate 50 times
% \end{itemize}
% 
% \framebreak
% 
% <<out.width="0.8\\textwidth", fig.height = 3, fig.width = 5>>=
% load("rsrc/holdout-biasvar.RData")
% ggd1 = reshape2::melt(res)
% colnames(ggd1) = c("split", "rep", "ssiter", "mmce")
% ggd1$split = as.factor(ggd1$split)
% ggd1$mse = (ggd1$mmce -  realperf)^2
% ggd1$type = "holdout"
% ggd1$ssiter = NULL
% mse1 = plyr::ddply(ggd1, "split", plyr::summarize, mse = mean(mse))
% mse1$type = "holdout"
% 
% ggd2 = plyr::ddply(ggd1, c("split", "rep"), plyr::summarize, mmce = mean(mmce))
% ggd2$mse = (ggd2$mmce -  realperf)^2
% ggd2$type = "subsampling"
% mse2 = plyr::ddply(ggd2, "split", plyr::summarize, mse = mean(mse))
% mse2$type = "subsampling"
% 
% ggd = rbind(ggd1, ggd2)
% gmse = rbind(mse1, mse2)
% ggd$type = as.factor(ggd$type)
% # ggd$split = as.numeric(as.character(ggd$split))
% gmse$split = as.numeric(as.character(gmse$split))
% gmse$type = as.factor(gmse$type)
% 
% ggplot(ggd[ggd$type == "holdout", ], aes(x = split, y = mmce)) +
%     geom_boxplot() + geom_hline(yintercept = realperf) + 
%     theme(axis.text.x = element_text(angle = 270, vjust = 0.5, hjust = 0)) +
%     ylab("MCE")
% 
% @
% 
% \begin{itemize}
% \item Pessimistic bias for small training sets
% \item Increased variance when test sets become smaller
% \end{itemize}
% 
% % \framebreak
% % 
% % \begin{itemize}
% %   \item We now plot the mean quadratic error between the true performance (line in 1st plot) and the hold-out values in each boxplot
% %   \item The split rate with the lowest MSE value produces the best estimator, which is pretty much 2/3 data for training
% %   \item NB: This is a single experiment and not a scientific study, but this rule-of-thump is also validated in larger studies
% % \end{itemize}
% % 
% % <<out.width="0.7\\textwidth", out.height="4cm">>=
% % ggplot(gmse[gmse$type == "holdout", ], aes(x = split, y = mse, col = type)) +
% %     geom_line() + scale_y_log10() + scale_x_continuous(breaks = gmse$split)
% %     theme(axis.text.x = element_text(angle = 270, vjust = 0.5, hjust = 0)) 
% % @
% 
% \end{vbframe}
\endlecture
\end{document}
