% !TEX root = ../../main.tex
\chapter{Reconstructing Neutrino $p_z$}
\label{ch:neutrinopz}

Along the beam axis, $\hat{z}$, an unknown fraction of energy from the event escapes. This restricts the ability to constrain missing energy to the transverse $\hat{x}-\hat{y}$ plane, as described in \Ch{\ref{ch:objreco}}.
For the event topology of this analysis, $WV\to l\nu J$, the neutrino escapes the detector leaving virtually no energy deposit; thus, the missing energy in the transverse plane, $E_{\mathrm T}^{\rm miss}$, is attributed to the transverse momentum of the neutrino, $p_{\mathrm{T}, \nu}$. The invariant mass of the $WV$ system is given below in \Eqn{\ref{eq:wv_m}}.
\begin{equation}
m_{WV}^2 = \left(E_{J} + E_{l} + E_{\nu}\right)^2 - \left(\vec{p}_{J} + \vec{p}_{l} + \vec{p}_{\nu}\right)^2
\label{eq:wv_m}
\end{equation}

Since only the transverse momentum of the neutrino is known, however, \Eqn{\ref{eq:wv_mt}} for the transverse mass, $m_{\mathrm{T}}$, must be used.
\begin{equation}
m_{\mathrm{T}, WV}^2 = \left(E_{\mathrm{T}, J} + E_{\mathrm{T}, l} + E_{\mathrm{T}, \nu}\right)^2 - \left(\vec{p}_{\mathrm{T}, J} + \vec{p}_{\mathrm{T}, l} + \vec{p}_{\mathrm{T}, \nu}\right)^2
\label{eq:wv_mt}
\end{equation}

The transverse mass, $m_{\mathrm{T}, WV}$, will have a dependence on the angle between the daughter particles, and will have worse resolution than the invariant mass. It would be more desirable to be able to use \Eqn{\ref{eq:wv_m}} to provide a better signal to background ratio.

It is therefore advantageous to reconstruct the $\hat{z}$ component of the neutrino momentum vector, $p_{z,\nu}$, with the constraint of the $W$ boson mass, $m_W$ (80.385\,\GeV). Using energy-momentum 4-vectors, $p^{\mu}=(E, \vec{p})$, for the $W$ boson, lepton, and neutrino, it is possible to derive a quadratic expression for $p_{z,\nu}$ in terms of $m_W$, $\vec{p}_l$, and $p_{\mathrm{T}, \nu}$.
\begin{eqnarray}
    m_W^2 &=&p_W^{\mu}p_{W,\mu} = (p^{\mu}_l+p^{\mu}_{\nu})(p_{l,\mu}+p_{\nu, \mu}) \nonumber \\
    &=& m_l^2 + m_{\nu}^2 + 2(\Cline[red]{E_l}E_{\nu}-\vec{p}_l\cdot\vec{p}_{\nu}) \label{eq:n_first} \\
    &=& 2\Cline[red]{\sqrt{p_{\mathrm{T},l}^2+p_{z,l}^2}}\sqrt{p_{\mathrm{T},\nu}^2+p_{z, \nu}^2}\nonumber\\
    &&-2\vec{p}_{\mathrm{T},l}\cdot \vec{p}_{\mathrm{T},\nu}-2p_{z,l}p_{z,\nu} \label{eq:n_el}\\
\left(\Cline[green]{m_W^2+2\vec{p}_{\mathrm{T},l}\cdot \vec{p}_{\mathrm{T},\nu}}+2p_{z,l}p_{z,\nu}\right)^2    &=&4(p_{\mathrm{T},l}^2+p_{z,l}^2)(p_{\mathrm{T},\nu}^2+p_{z, \nu}^2) \label{eq:n_c} \\
 \Cline[green]{C}^2 + 4\Cline[green]{C}p_{z,l}p_{z,\nu}+\Cline[blue]{4p_{z,l}^2p_{z,\nu}^2}&=&   4\left[p_{\mathrm{T},l}^2p_{z,\nu}^2+p_{\mathrm{T},\nu}^2(p_{\mathrm{T},l}^2+p_{z,l}^2)+\Cline[blue]{p_{z,l}^2p_{z,\nu}^2}\right] \label{eq:n_cancel}\\
   0&=& \left[4p_{\mathrm{T},l}^2\right]p_{z,\nu}^2 - \left[4Cp_{z,l}\right]p_{z,\nu}+ \nonumber \\
   &&\left[4p_{\mathrm{T},\nu}^2(p_{\mathrm{T},l}^2+p_{z,l}^2)-C^2\right] \label{eq:n_final}
\end{eqnarray}

In \Eqn{\ref{eq:n_el}}, the limit where the neutrino mass is negligible is taken. It is also prudent to take the limit where the lepton momentum is much larger than the mass, $m_l$, which drops out as well in \Eqn{\ref{eq:n_el}}.
The lepton energy, $E_l$, underlined in red in \Eqn{\ref{eq:n_first}} is then approximated to be equal to the magnitude of the lepton momentum, $\vec{p_l}$, underlined in red in \Eqn{\ref{eq:n_el}}. The radical in \Eqn{\ref{eq:n_el}} is removed in \Eqn{\ref{eq:n_c}} by isolating it and squaring both sides. The green underlined quantity in \Eqn{\ref{eq:n_c}} is redefined as a constant, $C$, in  \Eqn{\ref{eq:n_cancel}}. In \Eqn{\ref{eq:n_cancel}}, the blue underlined quantities cancel out, leading to the result in \Eqn{\ref{eq:n_final}} which is a quadratic equation for $p_{z,\nu}$.

There can be up to two solutions in this case for $p_{z,\nu}$, which are handled as follows:
\begin{enumerate}
	\item If a solution is complex, remove the imaginary part
	\item If two unique solutions exist, compare the magnitudes and keep the smallest one
\end{enumerate}

The effect of these choices are minimal, and presented in \Fig{\ref{fig:neutrinoPz}}. In \Fig{\ref{fig:mass_vs_mt}}, the reconstructed invariant mass (using the $p_{z,\nu}$ defined above) is compared to the transverse mass. It is clear that the invariant mass reproduces the signal mass with a sharper peak, smaller width, and more accurate mean value.

\begin{figure}[hp]
 \begin{center}
\subfloat[]{\includegraphics[width=0.49\linewidth]{figures/Appendix/h_pz_low_high}\label{fig:nz_lowhigh}}
\subfloat[]{\includegraphics[width=0.49\linewidth]{figures/Appendix/h_real_complex}\label{fig:nz_realcomplex}}
\end{center}
  \caption[Neutrino $p_{z}$ solution comparisons]{The invariant mass is constructed for choice of $p_{z, \nu}$ \protect\subref{fig:nz_lowhigh} lowest (blue) vs highest (red) solution and \protect\subref{fig:nz_realcomplex} real (blue) vs complex (red) solution. The ratio (blue/red) is shown for each choice. Truth invariant mass (green) is shown as a reference.  HVT $Z'$ benchmark signal at $m=2.0\,\TeV$ is used.}
  \label{fig:neutrinoPz}
\end{figure}

\begin{figure}[hp]
  \begin{center}
\includegraphics[width=0.6\linewidth]{figures/Appendix/h_m_mt}
     \end{center}
  \caption[Comparison between reconstructed invariant mass and reconstructed transverse mass]{The reconstructed invariant mass, $m_{WV}$ (blue), and the reconstructed transverse mass, $m_{\mathrm{T}, WV}$ (red) are shown for the HVT $Z'$ benchmark signal at $m=2.0\,\TeV$. Truth invariant mass (green) is shown as a reference. There is a significant improvement in the reconstructed mass: the peak is more centered and sharper, while the width is smaller.}
  \label{fig:mass_vs_mt}
\end{figure}


\mychapter{1}{Lesson 1} %180926

\section{Secret communication}

The typical setting for the problem of secret communication is depicted in figure \ref{fig:secrecy}. The parties Alice and Bob want to share data in a private fashion, thus preventing a third party Eve from \emph{eavesdropping}.\footnote{That is, getting a hold of the information shared between the parties} The objects of interest here are:
\begin{itemize}
    \item The data to be shared, or \emph{message} $m$;
    \item Some secret information, shared between and known only to Alice and Bob, that is used to \emph{encrypt} the message: the \emph{encryption key} or just \emph{key} $k$;
    \item The result of encrypting a message $m$ using the key $k$: the ciphertext $c$.
\end{itemize}

\begin{figure}[ht]
    \centering
    \begin{tikzpicture}
        \draw
            (0, 0) node (a) [box, fill = white] {Alice}
            (5, 0) node (b) [box, fill = white] {Bob}
            (2.5, -1) node (e) [box, fill = white] {Eve}
        ;

        \draw[-Stealth] (a) -- node [midway, above] {$c$} (b);
        \draw[-Stealth] (0, -1) node [below] {$k$} -- (a);
        \draw[-Stealth] (5, -1) node [below] {$k$} -- (b);
        \draw[-Stealth] (-1, 0) node [left] {$m$} -- (a);
        \draw[-Stealth] (b) -- (6, 0) node [right] {$m$};
        \draw[-Stealth] (2, -0.1) -- (3, -0.1) (2.5, -0.1) -- (e);

    \end{tikzpicture}

    \caption{A depiction of the problem of secret communication}
    \label{fig:secrecy}
\end{figure}

To complete the picture, the two parties Alice and Bob employ a \emph{cryptographic secrecy scheme}, or \emph{encryption scheme} to convert the message in an encrypted form, and vice versa. It has the form $\Xi = (\Enc, \Dec)$, where:
\begin{itemize}
    \item $\Enc \in \K \times \M \to \C$ is a machine that, given a message $m$ in $\M$ and a key $k$ in $\K$, returns a ciphertext $c$ in $\C$, which is an \emph{encrypted} form of the meesage;
    \item $\Dec \in \K \times \C \to \M$ is a machine that restores the message $m$ encrypted in the given ciphertext $c$ by using the key $k$, effectively \emph{decrypting} the message.
\end{itemize}

For the time being, assume that both \Enc{} and \Dec{} work as normal functions. A fundamental requirement of encryption schemes is that they always completely preserve the message after a whole round of encryption and decryption:
\[
    \forall m \in \M \; \forall k \in \K \quad \Dec(k, \Enc(k, m)) = m
\]

A first definition that formalizes secrecy of an encryption scheme comes from Shannon:

\begin{definition}[Perfect secrecy]
    Given an encryption scheme $\Xi: (\Enc, \Dec)$, let $M$ be a generic random variable over the message space $\M$, and $K$ be a uniform random variable over the key space $\K$:
    \begin{align*}
        M \in&\ \mathcal{R}and(\M) \\
        K \in&\ \unifdist(\K)
    \end{align*}
    Then let $\Enc(K, M)$ be the resulting ciphertext from encrypting $M$ using $K$. The scheme $\Xi$ is deemed \emph{perfectly secret} iff such ciphertext is effectively useless in retrieving any information about the original message:
    \[
        \forall m \in \M \; \forall c \in \C \quad \Pr[M = m] = \Pr[M = m \knowing \Enc(K, M) = c]
    \]
\end{definition}

This definition can be rephrased in different ways, bringing more details to light:

\begin{proposition}
    The following statements are equivalent:
    \begin{enumerate}
        \item \label{def:ps1} $\Pr[M = m] = \Pr[M = m \knowing \Enc(K, M) = c]$
        \item \label{def:ps2} $M \indep \Enc(K, M)$
        \item \label{def:ps3} $\forall m_1, m_2 \in \M \, \forall c \in \C \quad \Pr[\Enc(K, m_1) = c] = \Pr[\Enc(K, m_2) = c]$
    \end{enumerate}
\end{proposition}

\begin{proof}
    The proof is structured as a cyclic implication between the three proposed definitions:

    \begin{itemize}
        \item $(\ref{def:ps1}) \implies (\ref{def:ps2})$: Start from one side of the independency definition, and work through the other:
        \begin{align*}
             & \Pr[M = m \wedge \Enc(K, M) = c]                         & \\
            =& \Pr[M = m \knowing \Enc(K, M) = c] \Pr[\Enc(K, M) = c]   & \text{(Conditional prob.)} \\
            =& \Pr[M = m] \Pr[\Enc(K, M) = c]                           & \text{(Using \ref{def:ps1})} \\
        \end{align*}
        This proves that $M$ and $\Enc(K, M)$ are indeed independent random variables.

        \item $(\ref{def:ps2}) \implies (\ref{def:ps3})$: Let $M$ be a generic random variable over $\M$. Then:
        \begin{align*}
             & \Pr[\Enc(K, m_1) = c]                                & \\
            =& \Pr[\Enc(K, M) = c \knowing M = m_1]                 & \text{(Conditioning $m$)} \\
            =& \Pr[\Enc(K, M) = c \wedge M = m_1] \Pr[M = m_1]^{-1} & \text{(Conditional prob.)} \\
            =& \Pr[\Enc(K, M) = c]                                  & \text{(Using \ref{def:ps2})} \\
            =& \Pr[\Enc(K, m_2) = c]                                & \mathllap{\text{(Same steps reversed, where $m_1 \mapsto m_2$)}} \\
        \end{align*}

        \item $(\ref{def:ps3}) \implies (\ref{def:ps1})$:
        \begin{align*}
             & \Pr[\Enc(K, M) = c]                                          & \\
            =& \sum_{m \in \M}\Pr[\Enc(K, M) = c \wedge M = m]              & \text{(Total prob.)} \\
            =& \sum_{m \in \M}\Pr[\Enc(K, M) = c \knowing M = m] \Pr[M = m] & \text{(Conditional prob.)} \\
            =& \sum_{m \in \M}\Pr[\Enc(K, m) = c] \Pr[M = m]                & \text{(Condition collapse)} \\
            =& \Pr[\Enc(K, m_0) = c] \sum_{m \in \M} \Pr[M = m]             & \mathllap{\text{(Using \ref{def:ps3} with arbitrary $m_0$})} \\
            =& \Pr[\Enc(K, m_0) = c]                                        & \text{(Total prob.)} \\
            =& \Pr[\Enc(K, M) = c \knowing M = m_0]                         & \text{(Conditioning $m_0$)}
        \end{align*}

        By applying Bayes' theorem, the above result can be turned into the first definition of perfect secrecy:
        \begin{align*}
                    & \Pr[C = c] = \Pr[C = c \knowing M = m]                                        & \\
            \implies& \Pr[C = c] = \Pr[M = m \knowing C = c] \cdot \frac{\Pr[C = c]}{\Pr[M = m]}    & \text{(Bayes' theorem)} \\
            \implies& \Pr[M = m] = \Pr[M = m \knowing C = c]                                        &
        \end{align*}
        where $C = \Enc(K, M)$.
    \end{itemize}
\end{proof}

About definition \ref{def:ps3}, it could be insightful to remark that, for any message $m$ and ciphertext $c$:
\begin{align*}
        & \Pr[\Enc(K, m) = c] \\
       =& \Pr[\Enc(K, M) = c \knowing M = m] \\
    \neq& \Pr[\Enc(K, M) = c]
\end{align*}

which is the difference between \emph{choosing} a specific message $m$, and picking it at random, according to $M$' distribution.

\subsection{One-time Pad}

The One-time Pad, or \otp{} in short, is a simple encryption scheme that leverages the involutory property of the \textsc{xor} operation. Let all the spaces $\K = \M = \C = \binary^l$ be binary strings of some length $l$, and define this scheme $\otp = (\Enc, \Dec)$ as such:
\begin{itemize}
    \item $\Enc(k, m) = k \oplus m$
    \item $\Dec(k, c) = k \oplus c$
    \item[--] Correctness: $\Dec(k, \Enc(k, m)) = \Dec(k, k \oplus m) = k \oplus k \oplus m = m$
\end{itemize}

\begin{theorem}
    \otp{} is perfectly secret.
\end{theorem}
\begin{proof}
    The proof makes use of definition \ref{def:ps3} of perfect secrecy. Let $K$ be a uniformly random key; then for any binary strings $m_1$, $m_2$ and $c$:
    \begin{align*}
        & \Pr[\Enc(K, m_1) = c]     & \\
        =& \Pr[K \oplus m_1 = c]    & \\
        =& \Pr[K = c \oplus m_1]    & \\
        =& |\K|^{-1}                & \text{(K is uniform)} \\
        =& \Pr[\Enc(K, m_2) = c]    & \text{(Same steps reversed, where $m_1 \mapsto m_2$)}
    \end{align*}
\end{proof}

%TODO OLD NOTE: Observation: k is truly random, but fixed, compare with Pi_\oplus's encryption routine: from a security standpoint, nothing changes! actually, \Pi_\oplus may be weaker brute-force wise than OTP!

At this point, some important observations are in order:
\begin{enumerate}
    \item As the scheme's name suggests, keys are useful just for one encryption. In fact, if the same key is used for two different encryptions, an attacker may exploit the \textsc{xor}'s idempotency to extract valuable information from both ciphertexts:
    \[
        c_1 = k \oplus m_1 \wedge c_2 = k \oplus m_2 \implies c_1 \oplus c_2 = m_1 \oplus m_2
    \]
    \item The key and the message's lengths must always match ($|k| = |m|$);

    % AP210226: Do not introduce malleability at this level; rather, cite OTP instead when malleability is actually introduced
    %\footnote{This vulnerability of applying a function on a ciphertext, and expecting as a result the image of the original message by the same function, is called \emph{malleability}, and is explored further in the notes.}
    %\footnote{An encryption algorithm is "malleable" if it is possible to transform a ciphertext into another ciphertext which decrypts to a related plaintext. That is, given an encryption $c$ of a plaintext $m$, it is possible to generate another valid ciphertext $c'$, for a known $Enc$, $Dec$, without necessarily knowing or learning $m$.}

\end{enumerate}

Combined with the fact that keys must be preemptively shared in a secret fashion, these caveats make for an impractical encryption scheme. The last point can actually be generalized for any scheme that is perfectly secret:

\begin{theorem}
    For an encryption scheme to be perfectly secret, there must be more distinct keys than distinct messages:
    \[
        \Xi \textrm{ is perfectly secret} \implies |\K| \geq |\M|
    \]
\end{theorem}
\begin{proof}

    Let $M$ be a generic random variable over the messages, then fix a ciphertext $c$ that has some positive probability to be the result of $M$'s encryption. This means that there are some key-message pairs that result in such ciphertext by means of encryption:
    \[
        \exists (k, m) \in \K \times \M \quad \Enc(k, m) = c
    \]
    Let $S$ be the set collecting all possible message ``sources'' that do encrypt into $c$:
    \[
        S = \{m \in \M : \exists k \in \K \quad \Enc(k, m) = c\}
    \]

    By using $\Xi$'s correctness, this definition can be twisted into using \Dec:
    \[
        S = \{\Dec(k, c) \in \M : k \in \K\}
    \]

    This reveals in a clearer way that there cannot be more sources than keys,\footnote{Remember that \Enc{} and \Dec{} are still considered as mathematical functions, despite them actually being machines} therefore $|S| \leq |\K|$. By also assuming that $|\K| < |\M|$, it follows that $|S| < |\M|$ too; of consequence is that there are some messages that cannot be possibly encrypted into $c$:
    \[
        x \in \M \setminus S \implies \forall k \in \K \quad \Pr[\Enc(k, x) = c] = 0
    \]

    Let $x$ be a message in $\M \setminus S$. Since $M$ is nonexclusive:
    \[
        \Pr[M = x] > 0
    \]
    However, $x$ cannot be encrypted into $c$ by definition, which also means:
    \[
        \Pr[M = x \knowing \Enc(K, M) = c] = 0
    \]
    Therefore:
    \[
        0 = \Pr[M = x \knowing \Enc(K, M) = c] \neq \Pr[M = m] > 0
    \]
    which violates perfect secrecy.
\end{proof}


\documentclass[a4paper,10pt,oneside]{article}
% \setcounter{secnumdepth}{-1}

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{courier}
\usepackage{graphicx}
\usepackage{xcolor}
\usepackage{color}
\usepackage{pdflscape}
\usepackage[utf8]{inputenc}
\usepackage{listings}
\usepackage[inline]{enumitem}
\usepackage{verbatim}
\usepackage{pxfonts}
\usepackage{algorithm2e}
\usepackage{algpseudocode}

\usepackage[top=2cm, bottom=1.5cm, left=1cm, right=1cm]{geometry}
\usepackage{multicol}
\usepackage{fancyhdr}

\pagestyle{fancy}

\renewcommand{\sectionmark}[1]{\markboth{#1}{}}
\renewcommand{\subsectionmark}[1]{\markright{#1}}

\fancyhf{}
\rhead{\leftmark, \thepage}

\lhead{University of Brasilia}
\cfoot{\thepage}

\usepackage{titlesec}
\titlespacing*{\section}
{0pt}{2ex}{1ex}

%\definecolor{dkgray}{rgb}{0.4,0.4,0.4}
\definecolor{gray}{rgb}{0.6,0.6,0.6}
\definecolor{dkgreen}{rgb}{0,0.6,0}
%\definecolor{gray}{rgb}{0.5,0.5,0.5}
\definecolor{mauve}{rgb}{0.58,0,0.82}

\lstset{
	language=c++,
	tabsize=4,
	aboveskip=.1em,
	belowskip=.1em,
	showstringspaces=false,
	basicstyle={\small\ttfamily},
	columns=fullflexible,
	numberstyle=\tiny\color{gray},
	% keywordstyle=\color{blue},
	commentstyle=\color{dkgreen},
	stringstyle=\color{mauve},
	numbers=none,
	breaklines=true,
	breakindent=1.1em,
	breakatwhitespace=false,
	commentstyle=\color{gray},
}

\newcommand\includes[2]{
   \subsection{#1}
   \lstinputlisting{#2}
}
\newcommand\includess[2]{
   \subsubsection{#1}
   \lstinputlisting{#2}
}

\setlength{\columnseprule}{1pt}

\date{}
\title{Rock Lee do Pagode Namora D+}
%\title{ICPC Team Reference}
\author{University of Brasilia}

\begin{document}
\maketitle
\begin{multicols}{2}
\tableofcontents
\newpage
\thispagestyle{fancy}

\lstinputlisting[language=bash]{vimrc}

\lstinputlisting[language=bash]{bashrc}

\section{Data Structures}
\includes{Merge Sort Tree}{code/ed/merge_sort_tree.cpp}
\includes{Wavelet Tree}{code/ed/wavelet_tree.cpp}
\includes{Order Set}{code/ed/order_set.cpp}
\includes{Hash table}{code/ed/hash_table.cpp}
\includes{Convex Hull Trick Simple}{code/ed/cht_simple.cpp}
% \includes{Convex Hull Trick}{code/ed/cht.cpp}
\includes{Convex Hull Trick}{code/ed/LineContainer.cpp}
\includes{Min queue}{code/ed/minq.cpp}
\includes{Sparse Table}{code/ed/sparse_table.cpp}
\includes{Treap}{code/ed/treap.cpp}
\includes{ColorUpdate}{code/misc/ColorUpdate.cpp}
\includes{Heavy Light Decomposition}{code/ed/hld.cpp}
\includes{Iterative Segtree}{code/ed/segtree.cpp}
\includes{Recursive Segtree + lazy}{code/ed/segtree_rec.cpp}
\includes{LiChao's Segtree}{code/ed/lichao.cpp}
\includes{Palindromic tree}{code/ed/eertree.cpp}

\section{Math}
\includes{Extended Euclidean Algorithm}{code/math/euclides.cpp}
\includes{Chinese Remainder Theorem}{code/math/crt.cpp}
\includes{Preffix inverse}{code/math/inv.cpp}
\includes{Pollard Rho}{code/math/pollard_rho.cpp}
\includes{Miller Rabin}{code/math/miller_rabin.cpp}
\includes{Totiente}{code/math/tot.cpp}
\includes{Primitive root}{code/math/primitive_root.cpp}
\includes{Mobius Function}{code/math/mobius.cpp}
\includes{Mulmod TOP}{code/math/mod.cpp}
\includes{Matrix Determinant}{code/math/det.cpp}
\includes{Simplex Method}{code/math/simplex.cpp}
\includes{FFT}{code/fft.cpp}
\includes{FFT Tourist}{code/fft_tourist.cpp}
\includes{NTT}{code/ntt.cpp}
\includes{Gauss}{code/gauss.cpp}
\includes{Gauss Xor}{code/gauss_xor.cpp}
\includes{Simpson}{code/math/simpson.cpp}
\includes{Modular Arithmetic}{code/math/mod_arithmetic.cpp}
\includes{Matrix}{code/math/matrix.cpp}

\section{Graphs}
\includes{Dinic}{code/graph/dinic.cpp}
\includes{Push relabel}{code/graph/pushrelabel.cpp}
\includes{Min Cost Max Flow}{code/graph/mcmf.cpp}
\includes{Blossom Algorithm for General Matching}{code/graph/blossom.cpp}
% \includes{Blossom Algorithm for Weighted General Matching}{code/graph/weight_blossom.cpp}
\includes{Small to Large}{code/graph/stl.cpp}
\includes{Centroid Decomposition}{code/graph/centroid_decomp.cpp}
\includes{Kosaraju}{code/graph/kosaraju.cpp}
\includes{Tarjan}{code/graph/tarjan.cpp}
\includes{Max Clique}{code/graph/maxcliq.cpp}
\includes{Dominator Tree}{code/graph/dominator_tree.cpp}
\includes{Min Cost Matching}{code/graph/hungarian_mcm.cpp}

\section{Strings}
\includes{Aho Corasick}{code/string/aho_corasick.cpp}
\includes{Suffix Array}{code/string/suffix_array.cpp}
\includes{Adamant Suffix Tree}{code/string/adamant_suffix_tree.cpp}
\includes{Z Algorithm}{code/string/z_algo.cpp}
\includes{Prefix function/KMP}{code/string/pf.cpp}
\includes{Min rotation}{code/string/min_rot.cpp}
\includes{Manacher}{code/string/all_palindrome.cpp}
\includes{Suffix Automaton}{code/string/suffix_automaton.cpp}
\includes{Suffix Tree}{code/ed/suffix_tree.cpp}

\section{Geometry}
\includes{2D basics}{code/geometry/2D.cpp}
\includes{Circle line intersection}{code/geometry/circle_line_intersection.cpp}
\includes{Half plane intersection}{code/geometry/halfplane.cpp}
\includes{Detect empty Half plane intersection}{code/geometry/detect_halfplane.cpp}

\subsection{Circle Circle intersection}
Assume that the first circle is centered at the origin and second at $(x2, y2)$. Find circle line intersection of first circle and line $Ax + By + C = 0$, where $A = -2x_2$, $B = -2y_2$, $C = x_2^2 + y_2^2 + r_1^2 - r_2^2$.

Be aware of corner case with two circles centered at the same point.
\includes{Tangents of two circles}{code/geometry/tangents.cpp}
\includes{Convex Hull}{code/geometry/convexhull.cpp}
\includes{Check point inside polygon}{code/geometry/in_poly.cpp}
\includes{Check point inside polygon without lower/upper hull}{code/geometry/in_poly2.cpp}
\includes{Minkowski sum}{code/geometry/mink.cpp}
\subsection{Geo Notes}
\subsubsection{Center of mass}
\textbf{System of points(2D/3D):} Mass weighted average of points. \\
\textbf{Frame(2D/3D):} Get middle point of each segment solve as previously. \\
\textbf{Triangle:} Average of vertices. \\
\textbf{2D Polygon:} Compute \textbf{signed} area and center of mass of triangle $((0, 0), p_i, p_{i+1})$. Then solve as system of points.\\
\textbf{Polyhedron surface:} Solve each face as a 2D polygon(be aware of (0, 0)) then replace each face with its center of mass and solve as system of points. \\
\textbf{Tetrahedron(Triangular pyramid):} As triangles, its the average of points. \\
\textbf{Polyhedron:} Can be done as 2D polygon, but with tetrahedralization intead of triangulation.

\subsubsection{Pick's Theorem}
Given a polygon without self-intersections and all its vertices on integer coordinates in some 2D grid. Let $A$ be its area, $I$ the number of points with interger coordinates stricly inside the polygon and $B$ the number of points with interger coordinates in the border of the polygon. The following formula holds: $A = I + \frac{B}{2} - 1$.

\section{Miscellaneous}
\includes{LIS}{code/misc/lis.cpp}
\includes{DSU rollback}{code/misc/bipar.cpp}
\includes{Buildings}{code/misc/burn.cpp}
\includes{Rand}{code/misc/rand.cpp}
\includes{Klondike}{code/misc/klondike.cpp}
\includes{Hilbert Order}{code/misc/hilbert_order.cpp}
\includes{Modular Factorial}{code/misc/factmod.cpp}
\includes{Enumeration all submasks of a bitmask}{code/misc/submasks.cpp}
\includes{Slope Trick}{code/misc/slope.cpp}
% \includes{Fast IO}{code/misc/fastio.cpp}
% \includes{Big int}{code/misc/bigint.cpp}
\includes{Knapsack Bounded with Cost}{code/misc/knapsack_bounded_cost.cpp}
\includes{LCA \textless O(nlgn), O(1)\textgreater}{code/misc/lca.cpp}
\includes{Buffered reader}{code/misc/buffered_reader.cpp}
\includes{Modular summation}{code/misc/sum_mod.cpp}
\includes{Edge coloring CPP}{code/misc/edge_coloring.cpp}

\subsection{Burnside's Lemma}
Let $(G, \oplus)$ be a finite group that acts on a set $X$. It should hold that $e_g*x=x$ and $g_1 *(g_2 * x) = (g_1 \oplus g_2) * x$, $\forall x \in X, g_1, g_2 \in G$. For each $g \in G$ let $X^g = \{x \in X \mid g*x = x \}$. The number of orbits its given by:

$\mid X / G\mid~= \frac{1}{|G|} \sum_{g \in G}{|X^g|}$

\subsection{Wilson's Theorem}
$(n-1)! = -1 \mod n \iff n\text{ is prime}$

\subsection{Fibonacci}
\begin{itemize}
\item $F_{n-1}F_{n+1} - F_n^2 = (-1)^n$
\item $F_{n+k} = F_kF_{n+1} + F_{k-1}F_n$
\item $GCD(F_n, F_m) = F_{GCD(n, m)}$
\item $F_n = \frac{(\frac{1+\sqrt{5}}{2})^n - (\frac{1-\sqrt{5}}{2})^n}{\sqrt{5}}$
\end{itemize}

\subsection{Lucas's Theorem}
For non-negative integers $m$ and $n$ and a prime $p$, the following congruence holds:

$\displaystyle \binom{m}{n} \equiv \prod_{i = 0}^{k} \binom{m_i}{n_i} \pmod p$

where $m_i$ is the i-th digit of $m$ in base $p$. ${\displaystyle {\tbinom {a}{b}}=0}$ if $a < b$.

\subsection{Kirchhoff's Theorem}
Laplacian matrix is $L = D - A$, where $D$ is a diagonal matrix with vertex degrees on the diagonals and $A$ is adjacency matrix.

The number of spanning trees is any cofactor of L. i-th cofactor is determinant of the matrix gotten by removing i-th row and column of L.

\subsubsection{Multigraphs}
In $D[i][i]$ all loops are excluded. $A[i][j]$ = number of edges from $i$ to $j$.

\subsubsection{Directed multigraphs}
$D[i][i]$ = indegree of i minus the number of loops at i. $A[i][j]$ = number of edges from $i$ to $j$.

The number of oriented spanning trees rooted at a vertex i is the determinant of the matrix gotten by removing the ith row and column of L.

\subsection{Matroid}
Let $X$ set of objects, $I \subseteq 2^X$ set of independents sets such that:
\begin{enumerate}
\item $\emptyset \in I$
\item $A \in I, B \subseteq A \implies B \in I$
\item Exchange axiom, $A \in I, B \in I, |B| > |A| \implies \exists x \in B \setminus A : A \cup \{x\} \in I$
\item $A \subseteq X$ and $I$ and $I'$ are maximal independent subsets of A then $|I| = |I'|$
\end{enumerate}
Then $(X, I)$ is a matroid. The combinatorial optimization problem associated with it is: Given a weight $w(e) \geq 0 ~\forall e \in X$, find an independet subset that has the largest possible total weight.

\includess{Matroid intersection}{code/matroid.cpp}

Where path(e) = [e] if label[e] = MARK2, path(label[e]) + [e] otherwise.

\subsubsection{Matroid Union}
Given $k$ matroids over the same set of objects $(X, I_1)$, $(X, I_2)$, \dots, $(X, I_k)$ find $A_1 \in I_1$, $A_2 \in I_2$, \dots, $A_k \in I_k$ such that $i \not= j, A_i \cap A_j = \emptyset$ and $|\bigcup\limits_{i=1}^{k} A_i|$ is maximum. Matroid union can be reduced to matroid intersection as follows.

Let $X' = X \times \{1, 2, \dots, k\}$, ie, $k$ copies of each element of $X$ with different colors. $M1 = (X', Q)$ where $B \in Q \iff \forall ~1 \le i \le k, ~\{x\mid (x, i) \in B\} \in I_i$, ie, for each color, $B$ is independent. $M2 = (X', W)$ where $B \in W \iff i \not= j \implies \lnot((x, i) \in B \land (x, j) \in B)$, ie, each element is picked by at most one color.

Intersection of $M1$ and $M2$ is the answer for the combinatorial problem of matroid union.

\subsection{Notes}
When we repeat something and each time we have probability $p$ to succeed then the expected number or tries is $\frac{1}{p}$, till we succeed.

\textbf{Small to large}

\textbf{Trick in statement} If $k$ sets are given you should note that the amount of different set sizes is $O(\sqrt{s})$ where $s$ is total size of those sets. And no more than $\sqrt{s}$ sets have size greater than $\sqrt{s}$. For example, a path to the root in Aho-Corasick through suffix links will have at most $O(\sqrt{s})$ vertices.

\textbf{gcd on subsegment}, we have at most $\log(a_i)$ different values in $\{\gcd(a_j, a_{j+1}, ..., a_i)$ for $j < i\}$.

\textbf{From static set to expandable}. To insert, create a new set with the new element. While there are two sets with same size, merge them. There will be at most $\log(n)$ disjoints sets.

\textbf{Matrix exponentiation optimization}. Save binary power of $A_{nxn}$ and answer $q$ queries $b = A^mx$ in $O((n^3 + qn^2)log(m))$.

\textbf{Ternary search on integers into binary search}, comparing f(mid) and f(mid+1), binary search on derivative

\textbf{Dynamic offline set} For each element we will wind segment of time $[a, b]$ such that element is present in the set during this whole segment. Now we can come up with recursive procedure which handles $[l, r]$ time segment considering that all elements such that $[l, r] \subset [a, b]$ are already included into the set. Now, keeping this invariant we recursively go into $[l, m]$ and $[m+1, r]$ subsegments. Finally when we come into segment of length 1.

$a > b \implies a \mod b < \frac{a}{2}$

\textbf{Convex Hull}. The expected number of points in the convex hull of a random set of points is $O(log(n))$. The number of points in a convex hull with points coordinates limited by $L$ is $O(L^{2/3})$.

\textbf{Tree path query}. Sometimes the linear query is fast enough. Just do adamant's hld sorting subtrees by their size and remap vertices indexes.

\end{multicols}

\end{document}


\section{Optimal sorting}
\label{sec:opt_sort}
\index{sorting}

Sorting is the process of rearranging a series of objects, called
\emph{keys}\index{sorting!key}, to conform with a predefined
order. According to \cite{Knuth_1998}, the first sorting algorithms
were invented and automated as tabulating machines in the late
nineteenth century, in order to support the establishment of the
census of the United States of America.

\paragraph{Permutations}
\label{par:permutations}
\index{permutation}

We saw \vpageref{par:mean_sort} that the average cost of a
comparison\hyp{}based sorting algorithm is defined as the arithmetic
mean of the costs of sorting all permutations of a given length. A
permutation of \((1,2,\dots,n)\) is another tuple
\((a_1,a_2,\dots,a_n)\) such that \(a_i \in \{1,\dots,n\}\) and \(a_i
\neq a_j\) for all~\(i \neq j\). For example, all the permutations
of~\((1,2,3)\) are
\begin{equation*}
(1,2,3) \quad (1,3,2) \quad (2,1,3) \quad (2,3,1) \quad (3,1,2) \quad
(3,2,1).
\end{equation*}
Given all the permutations of~\((1,2,\dots,n-1)\), let us build
inductively all the permutations of
\((1,2,\dots,n)\). If~\((a_1,a_2,\dots,a_{n-1})\) is a permutation of
\((1,2,\dots,n-1)\), then we can construct \(n\)~permutations
of~\((1,2,\dots,n)\) by inserting~\(n\) at all possible places
in~\((a_1,a_2,\dots,a_{n-1})\):
\begin{equation*}
(\boldsymbol{n},a_1,a_2,\dots,a_{n-1})\quad
(a_1,\boldsymbol{n},a_2,\dots,a_{n-1})\quad \ldots \quad
(a_1,a_2,\dots,a_{n-1},\boldsymbol{n}).
\end{equation*}
For example, it is obvious that all the permutations of \((1,2)\) are
\((1,2)\) and \((2,1)\). The method leads from \((1,2)\) to
\((\boldsymbol{3},1,2)\), \((1,\boldsymbol{3},2)\) and
\((1,2,\boldsymbol{3})\); and from \((2,1)\) to
\((\boldsymbol{3},2,1)\), \((2,\boldsymbol{3},1)\) and
\((2,1,\boldsymbol{3})\). If we name~\(p_n\) the number of
permutations on \(n\)~elements, we draw from this the recurrence \(p_n
= n \cdot p_{n-1}\), which, with the additional obvious \(p_1 = 1\),
leads to \({p_n = n!}\), for all \({n > 0}\), exactly as expected. If
the \(n\)~objects to permute are not \((1,2,\dots,n)\) but, for
example, \((\textsf{b},\textsf{d},\textsf{a},\textsf{c})\), simply
associate each of them to their index in the tuple, for example,
\textsf{b}~is represented by~\(1\), \textsf{d}~by~\(2\),
\textsf{a}~by~\(3\) and \textsf{c}~by~\(4\), so the tuple is then
associated to \((1,2,3,4)\) and, for instance, the permutation
\((4,1,2,3)\) means~\((\textsf{c},\textsf{b},\textsf{d},\textsf{a})\).

\paragraph{Factorial}\index{factorial}

We encountered the factorial function in the introduction and here
again. There is a simple derivation enabling the characterisation of
its asymptotic growth, proposed by
\cite{GrahamKnuthPatashnik_1994}. We start by squaring the factorial
and regrouping the factors as follows:
\begin{equation*}
n!^2 = (1 \cdot 2 \cdot \ldots \cdot n) (n \cdot \ldots \cdot 2 \cdot
1) = \prod_{k=1}^{n}{k(n+1-k)}.
\end{equation*}
The parabola \(P(k) := k(n+1-k) = -k^2 + (n+1)k\) reaches its maximum
where its derivative is zero: \(P'(k_{\max}) = 0 \Leftrightarrow
k_{\max}=(n+1)/2\). The corresponding ordinate is \(P(k_{\max}) =
((n+1)/2)^2 = k_{\max}^2\). When \(k\)~ranges from~\(1\) to~\(n\), the
minimal ordinate, \(n\), is reached at absissas \(1\)~and~\(n\), as
shown in \fig~\vref{fig:parabola}.
\begin{figure}
\centering
\includegraphics[bb=55 563 205 730]{parabola}
\caption{Parabola \(P(k) := k(n+1-k)\)\label{fig:parabola}}
\end{figure}
Hence, \(1 \leqslant k \leqslant k_{\max}\) implies
\begin{equation*}
P(1) \leqslant P(k) \leqslant P(k_{\max}),\quad \text{that is,} \quad
n \leqslant k(n+1-k) \leqslant \left(\frac{n+1}{2}\right)^2.
\end{equation*}
Multiplying the sides by varying~\(k\) over the discrete interval
\([1..n]\) yields
\begin{gather*}
n^n = \prod_{k=1}^{n}{n} \leqslant n!^2
\leqslant
\prod_{k=1}^{n}{\left(\!\frac{n+1}{2}\!\right)^2} \!\!=
\left(\!\frac{n+1}{2}\!\right)^{2n}
\!\!\!\Rightarrow\!
n^{n/2} \leqslant n! \leqslant \left(\!\frac{n+1}{2}\!\right)^n.
\end{gather*}
It is clear now that \(n!\)~is \emph{exponential}, so it
asymptotically outgrows any polynomial. Concretely, a function whose
cost is proportional to a factorial is useless even for small
inputs. For the cases where an equivalence is preferred, Stirling's
formula\index{Stirling's formula} states that
\begin{equation}
n! \sim n^n e^{-n} \sqrt{2\pi n}.\label{eq:Stirling}
\end{equation}

\paragraph{Enumerating all permutations}

Let us write a program computing all the permutations of a given
stack. We define the function~\fun{perm/1}\index{perm@\fun{perm/1}}
such that \(\fun{perm}(s)\) is the stack of all permutations of the
items in stack~\(s\). We implement the inductive method presented
above, which worked by inserting a new object into all possible places
of a shorter permutation.
\begin{equation*}
\fun{perm}(\el)         \xrightarrow{\smash{\alpha}} \el;\quad
\fun{perm}([x])         \xrightarrow{\smash{\beta}} [[x]];\quad
\fun{perm}(\cons{x}{s}) \xrightarrow{\gamma}
                          \fun{dist}(x,\fun{perm}(s)).
\end{equation*}
The function~\fun{dist/2}\index{dist@\fun{dist/2}} (\emph{distribute})
is such that \(\fun{dist}(x,s)\) is the stack of all stacks obtained
by inserting the item~\(x\) at all different places in
stack~\(s\). Because such an insertion into a permutation of
length~\(n\) yields a permutation of length~\(n+1\), we must join the
new permutations to the previously found others of same length:
\begin{equation*}
\fun{dist}(x,\el)         \xrightarrow{\smash{\delta}} \el;\quad
\fun{dist}(x,\cons{p}{t}) \xrightarrow{\smash{\epsilon}}
                            \fun{cat}(\fun{ins}(x,p),\fun{dist}(x,t)).
\end{equation*}
The call~\(\fun{ins}(x,p)\)\index{ins@\fun{ins/2}} computes the stack
of permutations resulting from inserting~\(x\) at all places in the
permutation~\(p\). We thus derive
\begin{equation*}
\fun{ins}(x,\el) \xrightarrow{\smash{\zeta}} [[x]];\quad
\fun{ins}(x,\cons{j}{s}) \xrightarrow{\smash{\eta}}
 \cons{\cons{x,j}{s}}{\fun{push}(j,\fun{ins}(x,s))}.
\end{equation*}
where the function~\fun{push/2}\index{push@\fun{push/2}} (not to be
confused with the function of same name and arity in
section~\ref{sec:persistence}) is such that any
call~\(\fun{push}(x,t)\) pushes item~\(x\) on all the permutations of
the stack of permutations~\(t\). The order is left unchanged:
\begin{equation*}
\fun{push}(x,\el) \xrightarrow{\smash{\theta}} \el;\quad
\fun{push}(x,\cons{p}{t}) \xrightarrow{\smash{\iota}}
 \cons{\cons{x}{p}}{\fun{push}(x,t)}.
\end{equation*}
Now we can compute all the permutations of \((4,1,2,3)\) or
\((\fun{c}, \fun{b}, \fun{d}, \fun{a})\) by calling
\(\fun{perm}([4,1,2,3])\) or \(\fun{perm}([\fun{c}, \fun{b}, \fun{d},
  \fun{a}])\). Note that, after computing the permutations of
length~\(n+1\), the permutations of length~\(n\) are not needed
anymore, which would allow an implementation to reclaim the
corresponding memory for further uses (a process called \emph{garbage
  collection}\index{garbage collection}). As far as the costs are
concerned, the definition of \fun{push/2} yields
\begin{equation*}
\C{\fun{push}}{0} \eqn{\theta} 1;\qquad
\C{\fun{push}}{n+1} \eqn{\iota} 1 + \C{\fun{push}}{n},\quad
\text{with \(n \geqslant 0\)}.
\end{equation*}
We easily deduce \(\C{\fun{push}}{n} = n +
1\).\index{push@$\C{\fun{push}}{n}$} We know that the result of
\(\fun{ins}(x,p)\) is a stack of length~\(n+1\) if~\(p\) is a
permutation of \(n\)~objects into which we insert one more
object~\(x\). Hence, the definition of \fun{ins/2} leads to
\begin{equation*}
\C{\fun{ins}}{0}   \eqn{\zeta} 1;\quad
\C{\fun{ins}}{k+1} \eqn{\eta} 1 + \C{\fun{push}}{k+1} +
\C{\fun{ins}}{k} = 3 + k + \C{\fun{ins}}{k}, \quad \text{with \(k
                   \geqslant 0\)},
\end{equation*}
where~\(\C{\fun{ins}}{k}\) is the cost of \(\fun{ins}(x,p)\)
with~\(p\) of length~\(k\). By summing for all~\(k\) from
\(0\)~to~\(n-1\), for~\(n>0\), on both sides we draw
\begin{equation*}
\sum_{k=0}^{n-1}{\C{\fun{ins}}{k+1}}
  = \sum_{k=0}^{n-1}{3} + \sum_{k=0}^{n-1}{k}
     + \sum_{k=0}^{n-1}{\C{\fun{ins}}{k}}.
\end{equation*}
By cancelling identical terms in the sums (telescoping)
\(\sum_{k=0}^{n-1}{\C{\fun{ins}}{k+1}}\) and
\(\sum_{k=0}^{n-1}{\C{\fun{ins}}{k}}\), we draw\index{ins@$\C{\fun{ins}}{n}$}
\begin{equation*}
\C{\fun{ins}}{n}
  = 3n + \tfrac{1}{2}n(n-1) + \C{\fun{ins}}{0}
  = \tfrac{1}{2}{n^2} + \tfrac{5}{2}{n} + 1.
\end{equation*}
This last equation is actually valid even if \(n =
0\). Let~\(\C{\fun{dist}}{n!}\) be the cost for distributing an item
among \(n!\)~permutations of length~\(n\). The definition of
\fun{dist/2}\index{dist@\fun{dist/2}} shows that it repeats calls to
\fun{cat/2}\index{cat@\fun{cat/2}} and
\fun{ins/2}\index{ins@\fun{ins/2}} whose arguments are always of
length \(n+1\)~and~\(n\), respectively, because all processed
permutations here have the same length. We deduce, for~\(k \geqslant
0\), that
\begin{equation*}
\C{\fun{dist}}{0} \eqn{\delta} 1;\quad
\C{\fun{dist}}{k+1}
  \eqn{\epsilon} 1 + \C{\fun{cat}}{n+1} + \C{\fun{ins}}{n}
                    + \C{\fun{dist}}{k}
  = \tfrac{1}{2}{n^2} + \tfrac{7}{2}{n} + 4 + \C{\fun{dist}}{k},
\end{equation*}
since we already know that \(\C{\fun{cat}}{n} = n +
1\)\index{cat@$\C{\fun{cat}}{n}$} and the value
of~\(\C{\fun{ins}}{n}\). By summing both sides of the last equation
for all~\(k\) from \(0\)~to~\(n!-1\), we can eliminate most of the
terms and find a non\hyp{}recursive definition of
\(\C{\fun{dist}}{n!}\)\index{dist@$\C{\fun{dist}}{n}$}:
\begin{align*}
\sum_{k=0}^{n!-1}{\C{\fun{dist}}{k+1}}
   &= \sum_{k=0}^{n!-1}{\left(\frac{1}{2}{n^2}
      + \frac{7}{2}{n} + 4\right)}
      + \sum_{k=0}^{n!-1}{\C{\fun{dist}}{k}},\\
\C{\fun{dist}}{n!}
  &= \left(\frac{1}{2}{n^2}+\frac{7}{2}{n}+4\right)n! +
\C{\fun{dist}}{0} = \frac{1}{2}(n^2 + 7n + 8)n! + 1.
\end{align*}
Let us finally compute the cost of \(\fun{perm}(s)\), noted
\(\C{\fun{perm}}{k}\)\index{perm@$\C{\fun{perm}}{n}$|(}, where \(k\)~is
the length of the stack~\(s\). From rules \clause{\alpha}
to~\clause{\gamma}, we deduce the following equations, where \(k >
0\):
\begin{align*}
\C{\fun{perm}}{0}   &\eqn{\alpha} 1;\qquad
\C{\fun{perm}}{1}   \eqn{\beta} 1;\\
\C{\fun{perm}}{k+1}
  &\eqn{\gamma} 1 + \C{\fun{perm}}{k} + \C{\fun{dist}}{k!}
   = \tfrac{1}{2}(k^2 + 7k + 8)k! + 2 + \C{\fun{perm}}{k}.
\intertext{Again, summing both sides, most of the terms cancel out:}
\sum_{k=1}^{n-1}{\C{\fun{perm}}{k+1}}
  &= \frac{1}{2}\sum_{k=1}^{n-1}{(k^2+7k+8)k!} + \sum_{k=1}^{n-1}{2}
     + \sum_{k=1}^{n-1}{\C{\fun{perm}}{k}},\\
\C{\fun{perm}}{n}
  &= \frac{1}{2}\sum_{k=1}^{n-1}{(k^2 + 7k + 8)k!}
     + 2(n-1) + \C{\fun{perm}}{1}\\
  &= \frac{1}{2}\sum_{k=1}^{n-1}{((k+2)(k+1)+6+4k)k!} + 2n - 1\\
  &= \frac{1}{2}\sum_{k=1}^{n-1}{(k+2)(k+1)k!}
     + 3 \sum_{k=1}^{n-1}{k!} + 2 \sum_{k=1}^{n-1}{kk!} + 2n - 1\\
  &= \frac{1}{2}\sum_{k=1}^{n-1}{(k+2)!}
     + 3 \sum_{k=1}^{n-1}{k!} + 2 \sum_{k=1}^{n-1}{kk!} + 2n - 1\\
  &= \frac{1}{2}\sum_{k=3}^{n+1}{k!}
     + 3 \sum_{k=1}^{n-1}{k!} + 2 \sum_{k=1}^{n-1}{kk!} + 2n - 1\\
%  &= \left(\frac{1}{2}((n+1)! + n! - 2! - 1!)
%     + \frac{1}{2}\sum_{k=1}^{n-1}{k!}\right)
%     + 3 \sum_{k=1}^{n-1}{k!}\\
%  &\phantom{= x} + 2 \sum_{k=1}^{n-1}{kk!} + 2n - 1.\\
  &= \frac{1}{2}{(n+2)n!} + \frac{7}{2}\sum_{k=1}^{n-1}{k!}
     + 2 \sum_{k=1}^{n-1}{kk!} + 2n - \frac{5}{2}.
\end{align*}
This last equation is actually valid even if~\(n = 1\). One sum has a
simple closed form:
\begin{equation*}
\sum_{k=1}^{n-1}{kk!} = \sum_{k=1}^{n-1}((k+1)! - k!) =
\sum_{k=2}^{n}{k!} - \sum_{k=1}^{n-1}{k!} = n! - 1.
\end{equation*}
Resuming our previous derivation,
\begin{align*}
\C{\fun{perm}}{n}
  &= \frac{1}{2}{nn!} + n! + \frac{7}{2}\sum_{k=1}^{n-1}{k!}
     + 2(n! - 1) + 2n - \frac{5}{2}\\
  &= \frac{1}{2}{nn!} + 3n!
     + 2n - \frac{9}{2} + \frac{7}{2}\sum_{k=1}^{n-1}{k!},\quad
     \text{with \(n > 0\)}.
\end{align*}
The remaining sum is called the \emph{left factorial}
\citep{Kurepa_1971}\index{factorial!left $\sim$} and is usually
defined as
\begin{equation*}
!n := \sum_{k=1}^{n-1}{k!},\quad \text{with \(n > 0\)}.
\end{equation*}
Unfortunately, no closed expression of the left factorial is
known. This is actually a common situation when determining the cost
of relatively complex functions. The best course of action is then to
study the asymptotic approximation of the cost. Obviously, \(n!
\leqslant !(n+1)\). Also,
\begin{equation*}
!(n+1) - n! \leqslant (n-2) \cdot (n-2)! + (n-1)! \leqslant
(n-1) \cdot (n-2)! + (n-1)! = 2 (n-1)!
\end{equation*}
Therefore,
\begin{equation*}
1 \leqslant \frac{!(n+1)}{n!} \leqslant \frac{n! + 2(n-1)!}{n!} = 1 +
\frac{2}{n} \;\Rightarrow\; !n \sim (n-1)!
\end{equation*}
Also, \((n+1)! = (n+1)n!\), so \((n+1)!/(nn!) = 1 + 1/n\), hence \(nn!
\sim (n+1)!\). Consequently,
\begin{equation*}
\abovedisplayshortskip=0pt
\abovedisplayskip=0pt
%\belowdisplayskip=0pt
\C{\fun{perm}}{n} \sim \frac{1}{2}(n+1)! + 3n! + \frac{7}{2}(n-1)!
+ 2n - \frac{9}{2} \sim \frac{1}{2}(n+1)!
\end{equation*}
This is an unbearably slow program, as expected. We should not hope to
compute \(\C{\fun{perm}}{11}\) \index{perm@$\C{\fun{perm}}{n}$|)}
easily and there is no way to improve significantly the cost because
the number of permutations it computes is inherently exponential, so
it would even suffice to spend only one function call per permutation
to obtain an exponential cost. In other words, the memory necessary to
hold the result has a size which is exponential in the size of the
input, therefore, the cost is at least exponential, because at least
one function call is necessary to allocate some memory. For a deep
study on the enumeration of all permutations, refer to the survey of
\cite{Knuth_2011}.

\paragraph{Permutations and sorting}

Permutations\index{permutation|(} are worth studying in detail because
of their intimate relationship with sorting. A permutation can be
thought of as scrambling originally ordered keys and a sorting
permutation puts them back to their place. A slightly different
notation for permutations is helpful here, one which shows the indexes
together with the keys. For example, instead of writing \(\pi_1 =
(2,4,1,5,3)\), we write
\begin{equation*}
\pi_1 =
\begin{pmatrix}
1 & 2 & 3 & 4 & 5\\
2 & 4 & 1 & 5 & 3
\end{pmatrix}.
\end{equation*}
The first line is made of the ordered indexes and the second line
contains the keys. In general, a permutation \(\pi =
(a_1,a_2,\dots,a_n)\) is equivalent to
\begin{equation*}
\pi =
\begin{pmatrix}
     1 &      2 & \dots &     n\\
\pi(1) & \pi(2) & \dots & \pi(n)
\end{pmatrix},
\end{equation*}
where~\(a_i = \pi(i)\), for all~\(i\) from \(1\)~to~\(n\). The
following permutation~\(\pi_s\) sorts the keys of~\(\pi_1\):
\begin{equation*}
\pi_s =
\begin{pmatrix}
1 & 2 & 3 & 4 & 5\\
3 & 1 & 5 & 2 & 4
\end{pmatrix}.
\end{equation*}
To see why, we define the composition\index{permutation!composition}
of two permutations\index{permutation} \(\pi_a\)~and~\(\pi_b\):
\begin{equation*}
\pi_b \circ \pi_a :=
\begin{pmatrix}
              1 &               2 & \dots & n\\
\pi_b(\pi_a(1)) & \pi_b(\pi_a(2)) & \dots & \pi_b(\pi_a(n))
\end{pmatrix}.
\end{equation*}
Then \(\pi_s~\circ~\pi_1 = \mathcal{I}\), where the \emph{identity
  permutation}~\(\mathcal{I}\)\index{permutation!identity} is such
that \(\mathcal{I}(k) = k\), for all indexes~\(k\). In other words,
\(\pi_s = \pi_1^{-1}\), that is, \emph{sorting a permutation consists
  in building its inverse}\index{permutation!inverse}:
\begin{equation*}
\begin{pmatrix}
1 & 2 & 3 & 4 & 5\\
3 & 1 & 5 & 2 & 4
\end{pmatrix}
\circ
\begin{pmatrix}
1 & 2 & 3 & 4 & 5\\
2 & 4 & 1 & 5 & 3
\end{pmatrix}
=
\begin{pmatrix}
1 & 2 & 3 & 4 & 5\\
1 & 2 & 3 & 4 & 5
\end{pmatrix}.
\end{equation*}

An alternative representation of permutations and their
composition\index{permutation!composition} is based on considering
them as bijections from an interval onto itself, denoted by
\emph{bipartite graphs}\index{graph!bipartite
  $\sim$}\index{permutation!bigraph}, also called
\emph{bigraphs}\index{bigraph|see{graph, bipartite}}. Such
graphs are made of two disjoint, ordered sets of vertices of same
cardinal, the indexes and the keys, and the edges always go from an
index to a key, without sharing the vertices with other edges. For
example, permutation~\(\pi_1\) is shown in \fig~\vref{fig:perm1}
\begin{figure}
\centering
\subfloat[Bigraph of \(\pi_1 = (2,4,1,5,3)\)\label{fig:perm1}]{%
\includegraphics[bb=79 662 206 721]{perm1}
}
\qquad\qquad
\subfloat[Bigraph of \(\pi_1^{-1} = (3,1,5,2,4)\)\label{fig:perm3}]{%
\includegraphics[bb=77 662 208 721]{perm3}
}
\caption{Permutation \(\pi_1\) and its inverse \(\pi_1^{-1}\)
\label{fig:pi_1}\index{permutation!inverse}}
\end{figure}
and its inverse \(\pi_1^{-1}\) is displayed in
\fig~\vref{fig:perm3}. The composition\index{permutation!composition}
of~\(\pi_1^{-1}\) and~\(\pi_1\) is then obtained by identifying the
key vertices of~\(\pi_1\) with the index vertices of~\(\pi_1^{-1}\),
as shown in \fig~\vref{fig:perm4}.
\begin{figure}[t]
\centering
\subfloat[\(\pi_1^{-1} \circ \pi_1\)\label{fig:perm4}]{%
\includegraphics[bb=81 634 204 721]{perm4}
}%
\qquad\qquad
\subfloat[\(\mathcal{I} = \pi_1^{-1} \circ \pi_1\)\label{fig:perm2}]{%
\includegraphics[bb=81 634 204 721]{perm2}
}
\caption{Applying \(\pi_1\) to \(\pi_1^{-1}\).}
\end{figure}
The identity permutation\index{permutation!identity} is obtained by
replacing two adjacent edges by their transitive
closure\index{transitive closure} and erasing the intermediate
vertices, as shown in \fig~\ref{fig:perm2}
page~\pageref{fig:perm2}. Note that a permutation may equal its
inverse, like
\begin{equation*}
\pi_3 =
\begin{pmatrix}
1 & 2 & 3 & 4 & 5\\
3 & 4 & 1 & 2 & 5
\end{pmatrix}.
\end{equation*}
In \fig~\ref{fig:involution}
\begin{figure}
\centering
\subfloat[\(\pi_3 \circ \pi_3\)\label{fig:inv1}]{%
\includegraphics[bb=81 634 204 721]{inv1}
}
\qquad\qquad
\subfloat[\(\pi_3 \circ \pi_3\)]{%
\includegraphics[bb=81 634 204 721]{inv2}
}
\caption{Involution \(\pi_3\) sorts itself\label{fig:involution}}
\end{figure}
is shown that \(\pi_3~\circ~\pi_3 = \pi_3\), so \(\pi_3\) is an
\emph{involution}\index{permutation!involution}\index{permutation!inverse}.

Studying permutations and their basic properties helps understanding
sorting algorithms, particularly their average cost. They also provide
a way to quantify disorder. Given \((1,3,5,2,4)\), we can see that
only the pairs of keys \((3,2)\), \((5,2)\) and~\((5,4)\) are out of
order. In general, given \((a_1, a_2, \dots, a_n)\), the pairs
\((a_i,a_j)\) such that~\(i < j\) and~\(a_i > a_j\) are called
\emph{inversions}\index{permutation!inversion}. The more inversions,
the greater the disorder. As expected, the identity
permutation\index{permutation!identity} has no inversions and the
previously studied permutation \(\pi_1 = (2,4,1,5,3)\) has~\(4\). When
considering permutations as represented by
bigraphs\index{graph!bipartite $\sim$}\index{permutation!bigraph}, an
inversion\index{permutation!inversion} corresponds to an intersection
of two edges, more precisely, it is the pair made of the keys pointed
at by two arrows. Therefore, the number of
inversions\index{permutation!inversion} is the number of edge
crossings, so, for instance, \(\pi_1^{-1}\)~has \(4\)~inversions. In
fact, \emph{the inverse\index{permutation!inverse} of a permutation
  has the same number of inversions\index{permutation!inversion} as
  the permutation\index{permutation} itself}. This can be clearly seen
when comparing the bigraphs\index{permutation!bigraph} of~\(\pi_1\)
and~\(\pi_1^{-1}\) in \fig~\vref{fig:pi_1}: in order to deduce the
bigraph of~\(\pi_1^{-1}\) from the one corresponding to~\(\pi_1\), let
us reverse each edge, that is, the direction in which the arrows are
pointing, then swap the indexes and the keys, that is, exchange the
two lines of vertices. Alternatively, we can imagine that we fold down
the paper along the key line, then look through and reverse the
arrows. Anyhow, the crossings are invariant. The horizontal symmetry
is obvious in \figs~\ref{fig:perm4} and~\ref{fig:inv1}.

\paragraph{Minimax}
\label{par:minimax}

After analysing the cost of a sorting algorithm based on comparisons,
we will need to know how close it is to an optimal sorting
algorithm\index{sorting!optimality|(}. The first theoretical problem
we examine is that of the best worst case, so called the
\emph{minimax}\index{cost!minimax}\index{sorting!minimax|(}. \Fig~\vref{fig:cmp_tree}
features the tree of all possible comparisons \index{tree!comparison
  $\sim$|see{sorting}} for sorting three numbers.
\begin{figure}[t]
\centering
\includegraphics[bb=71 636 327 723]{cmp_tree}
\caption{A comparison tree for sorting three keys
\label{fig:cmp_tree}}
\end{figure}
The \emph{external nodes}\index{binary tree!external
  node}\label{def:external_node} are all the
permutations\index{permutation|)} of \((a_1,a_2,a_3)\). The
\emph{internal nodes}\index{binary tree!internal node} are comparisons
between two keys \(a_i\)~and~\(a_j\), noted \(a_i?a_j\). Note that
leaves, in this setting, are internal nodes with two external nodes as
children.
\begin{wrapfigure}[8]{r}[0pt]{0pt}
% [8] vertical lines
% {r} mandatory right placement
% [0pt] of margin overhang
\centering
\includegraphics[bb=71 638 217 723]{red_cmp}
\caption{Useless \(a_1 > a_3\)\label{fig:red_cmp}}
\end{wrapfigure}
If \(a_i < a_j\), then this property holds everywhere in the left
subtree, otherwise \(a_i > a_j\) holds in the right subtree. This tree
is one possible amongst many: it corresponds to an algorithm which
starts by comparing \(a_1\)~and~\(a_2\) and there are, of course, many
other strategies. But it does not contain redundant comparisons: if a
path from the root to a leaf includes \(a_i < a_j\) and \(a_j < a_k\),
we do not expect the useless \(a_i < a_k\). \Fig~\vref{fig:red_cmp}
shows an excerpt of a comparison tree with such a useless
comparison. The special external node~\(\bot\) corresponds to no
permutation because the comparison \(a_1 < a_3\) cannot fail as it is
implied by transitivity of the previous comparisons on the path from
the root.
\begin{center}
  \emph{A comparison tree for \(n\)~keys without redundancy has
    \(n!\)~external nodes.}
\end{center}
Because we are investigating minimum\hyp{}comparison sorting, we shall
consider henceforth optimal comparison trees with \(n!\)~external
nodes. Furthermore, amongst them, we want to determine the trees such
that the maximum number of comparisons is minimum.

An \emph{external path}\index{binary tree!external path} is a path
from the root to an external node. Let the \emph{height}\index{binary
  tree!height} of a tree be the length (that is, the number of edges)
of its longest external path. In \fig~\ref{fig:cmp_tree}, the height
is~\(3\) and there are~\(4\) external paths of length~\(3\), like
\((a_1 < a_2) \rightarrow (a_2 > a_3) \rightarrow (a_1 > a_3)
\rightarrow (a_3,a_1,a_2)\).

The maximality constraint means that we must consider the height of
the comparison tree because the number of internal nodes (comparisons)
along the maximum external paths is an upper bound for the number of
comparisons needed for sorting \emph{all} the
permutations\index{permutation}\index{sorting}.

The minimality constraint in the problem statement above then
signifies that \emph{we want a lower bound on the height of a
  comparison tree with \(n!\)~external nodes.}

% Wrapping figure better declared before a paragraph
%
\setlength{\intextsep}{0pt}
\begin{wrapfigure}[]{r}[0pt]{0pt}
% {r} mandatory right placement (better because of a list)
\centering
\includegraphics{comp_tree}
\caption{Perfect binary tree of height~\(3\)\label{fig:comp_tree}}
\end{wrapfigure}
A \emph{perfect binary tree}\index{binary tree!perfect $\sim$} is a
binary tree whose internal nodes have children which are either two
internal nodes or two external nodes. If such a tree has height~\(h\),
then it has \(2^h\)~external nodes. For instance,
\fig~\vref{fig:comp_tree} shows the case where the height~\(h\)
is~\(3\) and there are indeed \(2^h=8\) external nodes, figured as
squares. Since, by definition, minimum\hyp{}comparison trees have
\(n!\)~external nodes and height~\(S(n)\), they must contain fewer
external nodes than a perfect binary tree of identical height, that
is, \(n! \leqslant 2^{S(n)}\), therefore
\begin{equation*}
S(n) \geqslant \ceiling{\lg n!},
\end{equation*}
where \(\ceiling{x}\) (\textsl{ceiling of~\(x\)})
\index{ceiling@$\ceiling{x}$|see{ceiling function}} \index{ceiling
  function} is the least integer greater than or equal to~\(x\). To
deduce a good lower bound on~\(S(n)\), we need the following theorem.
\setlength{\intextsep}{12pt}
\begin{thm}[Sum and integral]
\label{thm:integral_bounds}
Let \(f\colon [a,b] \rightarrow \mathbb{R}\) be an integrable,
monotonically increasing function. Then
\begin{equation}
\abovedisplayskip=0pt
\sum_{k=a}^{b-1}f(k) \leqslant \int_{a}^{b}{\!\!f(x)} \,dx
                   \leqslant \sum_{k=a+1}^{b}{\!\!\!f(k)}.
\tag*{\(\blacksquare\)}
\end{equation}
\end{thm}
\noindent Let us take \(a:=1\), \(b:=n\) and \(f(x) := \lg x\). The
theorem implies
\begin{equation*}
n\lg n - \frac{n}{\ln 2} + \frac{1}{\ln 2}
= \int_{1}^{n}{\!\!\lg x} \,dx \leqslant \sum_{k=2}^{n}{\lg k}
= \lg n! \leqslant S(n).
\end{equation*}
A more powerful but more complex approach in real analysis, known as
Euler-Maclaurin summation, yields Stirling's formula
\citep[chap.~4]{SedgewickFlajolet_1996}\index{Stirling's formula},
which is a very precise lower bound for \(\lg n!\):
\begin{equation}
\Big(n + \frac{1}{2}\Big)\lg n - \frac{n}{\ln 2} + \lg\sqrt{2\pi}
< \lg n! \leqslant S(n).
\label{ineq:S_lower}
\end{equation}

\paragraph{Minimean}
\label{par:opt_sort:minimean}
\index{cost!minimean|(}
\index{sorting!minimean|(}
\index{binary tree!external path length|(}

We investigate here the best mean case, or \emph{minimean}. Let us
call the sum of the lengths of all the external paths the
\emph{external path length of the
  tree}\label{sorting__external_path_length}\index{binary
  tree!external path length}. Then, the average number of comparisons
is the mean external path length. In \fig~\vref{fig:cmp_tree}, this is
\((2+3+3+3+3+2)/3!=8/3\). Our problem here therefore consists in
determining the shape of the optimal comparison trees of minimum
external path length.

These trees have their external nodes on one or two successive levels
and are thus \emph{almost perfect}.\index{binary tree!almost perfect
  $\sim$} Let us consider a binary tree where this is not the case, so
the topmost external nodes are located at level~\(l\) and the
bottommost at level~\(L\), with \(L \geqslant l + 2\). If we exchange
a leaf at level~\(L-1\) with an external node at level~\(l\), the
external path length is decreased by \((l+2L) - (2(l+1) + (L-1)) = L -
l - 1 \geqslant 1\). Repeating these exchanges yields the expected
shape.

The external paths are made up of \(p\)~paths ending at the
penultimate level~\(h-1\) and \(q\)~paths ending at the bottommost
level~\(h\). (The root has level~\(0\).) Let us find two equations
whose solutions are \(p\)~and~\(q\).
\begin{itemize}

  \item From the minimax problem\index{sorting!minimax|)}, we know
    that an optimal comparison tree for \(n\)~keys has \(n!\)~external
    nodes: \(p+q=n!\).

  \item If we replace the external nodes at level~\(h-1\) by leaves,
    the level~\(h\) becomes full with~\(2^h\) external nodes:
    \(2p+q=2^h\).

\end{itemize}
We now have two linear equations satisfied by~\(p\) and~\(q\), whose
solutions are \(p=2^h-n!\) and \(q=2n!-2^h\). Furthermore, we can now
express the minimal external path length as follows: \((h-1)p + hq =
(h+1)n! - 2^h\). Finally, we need to determine the height~\(h\) of the
tree in terms of~\(n!\). This can be done by remarking that, by
contruction, the last level may be incomplete, so \(0 < q \leqslant
2^h\), or, equivalently, \(h = \ceiling{\lg n!}\). We conclude that
the minimum external path length is
\begin{equation*}
(\ceiling{\lg n!} + 1) n! - 2^{\ceiling{\lg n!}}.
\end{equation*}
Let \(M(n)\) \label{def:Mn} be the minimum average number of
comparisons of an optimal sorting algorithm. We have
\begin{equation*}
M(n) = \ceiling{\lg n!} + 1 - \frac{1}{n!}2^{\ceiling{\lg n!}}.
\end{equation*}
We have \(\ceiling{\lg n!} = \lg n! + x\), with \(0 \leqslant x < 1\),
therefore, if we set the function \(\theta(x) := 1 + x - 2^x\), we
draw
\begin{equation*}
M(n) = \lg n! + \theta(x).
\end{equation*}
We have \(\max_{0\leqslant x < 1}\theta(x) = 1 - (1 + \ln\ln 2)/\!\ln 2
\simeq 0.08607\), therefore,
\begin{equation}
\lg n! \leqslant M(n) < \lg n! + 0.09.
\label{ineq:Mn}
\end{equation}
\index{sorting!optimality|)}
\index{cost!minimean|)}
\index{sorting!minimean|)}
\index{binary tree!external path length|)}


\documentclass{beamer}

\usepackage{verbatim}
\usepackage{fancyvrb}
\usepackage{amsmath}
\usepackage{mathtools}
\usepackage{booktabs}
\usepackage{amssymb}
\usepackage{graphicx}
\usepackage{calc}
\usepackage{color}
\usepackage{multicol}
\usepackage{wrapfig}
\usepackage{natbib}
\usepackage[ruled,vlined]{algorithm2e}
\usepackage{animate}
\usepackage{mathtools}
\usepackage{listings}

\usepackage{cmbright}
\fontencoding{OT1}\fontfamily{cmbr}\selectfont %to load ot1cmbr.fd
\DeclareFontShape{OT1}{cmbr}{bx}{n}{% change bx definition
<->cmbrbx10%
}{}
\normalfont % back to normalfont

% two col: two columns
\newenvironment{twocol}[4]{
\begin{columns}[c]
\column{#1\textwidth}
#3
\column{#2\textwidth}
#4
\end{columns}
}

\makeatletter
\setbeamertemplate{theorem begin}
{%
\begin{\inserttheoremblockenv}
  {}{\usebeamerfont*{block title}\usebeamercolor[fg]{block title}%
  \inserttheoremname
  %\inserttheoremnumber
  \ifx \inserttheoremaddition \empty \else\ (\inserttheoremaddition)\fi
  \inserttheorempunctuation}
  \normalfont
  }
  \setbeamertemplate{theorem end}{\end{\inserttheoremblockenv}}
\makeatother

\newcommand{\E}{\mathrm{E}}
\newcommand{\Var}{\mathrm{Var}}
\newcommand{\Cov}{\mathrm{Cov}}
\newcommand{\sd}{\mathrm{sd}}
\newcommand{\s}{\mathrm{s}}
\newcommand{\Corr}{\mathrm{Corr}}
\newcommand{\rank}{\mathrm{rank}}
\newcommand{\trace}{\mathrm{trace}}
\newcommand{\nullspace}{\mathrm{null}}
\newcommand{\myspan}{\mathrm{span}}
\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator*{\softmax}{softmax}

\definecolor{darkgreen}{rgb}{0,0.5,0}

\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{exe}{Exercise}
\newtheorem{notation}{Notation}
\newtheorem{remark}{Remark}

\definecolor{darkgreen}{rgb}{0,0.5,0}

\title{Model Selection and Validation}
\author{Zhenisbek Assylbekov}
\institute{Department of Mathematics}
\date{Regression Analysis}

\AtBeginSection[]
{
  \begin{frame}<beamer>
    \tableofcontents[currentsection]
  \end{frame}
}

\begin{document}

\begin{frame}
  \titlepage
\end{frame}

\section{Manual model selection}

\begin{frame}[fragile]{Salary example}{\url{https://github.com/zh3nis/MATH440/tree/main/chp09/salary.R}}
Model annual salary (in \$1000) as function of \begin{itemize}
    \item\pause \texttt{age} (in years),
    \item\pause \texttt{educ}ation (years of post-high-school education), and
    \item\pause \texttt{pol}itical affiliation (\texttt{pol} = D for
Democrat, \texttt{pol} = R for Republican, and \texttt{pol} = O for other).
\end{itemize}

\begin{footnotesize}
\pause\begin{verbatim}
> salary_data = read.table("path/to/salary.txt", header=FALSE)
> colnames(salary_data) = c('salary', 'age', 'educ', 'pol')
> head(salary_data)
  salary age educ pol
1     38  25    4   D
2     45  27    4   R
3     28  26    4   O
4     55  39    4   D
5     74  42    4   R
6     43  41    4   O
\end{verbatim}
\end{footnotesize}
\end{frame}

\begin{frame}[fragile]{Salary example}{\url{https://github.com/zh3nis/MATH440/tree/main/chp09/salary.R}}
\begin{footnotesize}
\begin{verbatim}
Coefficients:
            Estimate Std. Error t value Pr(>|t|)
(Intercept)  17.0313     7.3459   2.318  0.03735 *
age           0.8983     0.1968   4.565  0.00053 ***
educ          1.5039     1.1841   1.270  0.22632
polO        -16.5404     4.8807  -3.389  0.00484 **
polR          9.1587     4.8482   1.889  0.08139 .
---

Residual standard error: 8.209 on 13 degrees of freedom
Multiple R-squared:  0.8374,	Adjusted R-squared:  0.7873
\end{verbatim}
\end{footnotesize}

\begin{itemize}
\item We can also test quadratic effects and interactions.
\item From the initial fit, \texttt{educ} is not needed with \texttt{age} and \texttt{pol} in
the model. Let's refit:
\end{itemize}

\end{frame}

\begin{frame}[fragile]
\frametitle{Droc \texttt{educ}?}
\begin{footnotesize}
\begin{verbatim}
> m2 = update(m1, . ~ . - educ)
> anova(m1, m2)
Analysis of Variance Table

Model 1: salary ~ age + educ + pol
Model 2: salary ~ age + pol
  Res.Df    RSS Df Sum of Sq      F Pr(>F)
1     13 876.03
2     14 984.72 -1    -108.7 1.6131 0.2263

> summary(m2)
lm(formula = salary ~ age + pol, data = salary_data)

Coefficients:
            Estimate Std. Error t value Pr(>|t|)
(Intercept)  21.5172     6.5806   3.270  0.00559 **
age           1.0345     0.1686   6.136 2.58e-05 ***
polO        -16.7414     4.9838  -3.359  0.00468 **
polR          8.6379     4.9354   1.750  0.10196
Residual standard error: 8.387 on 14 degrees of freedom
Multiple R-squared:  0.8172,	Adjusted R-squared:  0.778
\end{verbatim}
\end{footnotesize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Add 2nd order terms?}
\begin{footnotesize}
\begin{verbatim}
> salary_data$age2 = salary_data$age^2
> m3 = update(m2, . ~ . + age2 + age*pol)
> summary(m3)

Call:
lm(formula = salary ~ age + pol + age2 + age:pol, data = salary_data)

Coefficients:
             Estimate Std. Error t value Pr(>|t|)
(Intercept) -20.94355   20.34047  -1.030  0.32528
age           3.17751    0.93793   3.388  0.00606 **
polO        -16.90846   22.83050  -0.741  0.47444
polR         -1.18699   21.44129  -0.055  0.95684
age2         -0.02514    0.01255  -2.004  0.07037 .
age:polO      0.05101    0.65536   0.078  0.93936
age:polR      0.28944    0.61956   0.467  0.64950
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 7.434 on 11 degrees of freedom
Multiple R-squared:  0.8871,	Adjusted R-squared:  0.8256
\end{verbatim}
\end{footnotesize}
\end{frame}

\begin{frame}[fragile]{Drop 2nd order terms?}
\begin{footnotesize}
\begin{verbatim}
> anova(m3, m2)
Analysis of Variance Table

Model 1: salary ~ age + pol + age2 + age:pol
Model 2: salary ~ age + pol
  Res.Df    RSS Df Sum of Sq      F Pr(>F)
1     11 607.88
2     14 984.72 -3   -376.84 2.2731 0.1369
\end{verbatim}
\end{footnotesize}
\pause We don't need \textit{all} the 2nd order terms ($p=0.137$), although there’s some indication in the table of regression effects that age$^2$ might be needed.
\pause\begin{footnotesize}
\begin{verbatim}
> anova(m4, m2)
Analysis of Variance Table

Model 1: salary ~ age + pol + age2
Model 2: salary ~ age + pol
  Res.Df    RSS Df Sum of Sq      F  Pr(>F)
1     13 645.09
2     14 984.72 -1   -339.64 6.8444 0.02134 *
\end{verbatim}
\end{footnotesize}
\end{frame}

\begin{frame}[fragile]{Final model}
Our final model is
$$
\text{salary}_i = \beta_0+\beta_1\cdot\text{age}+\beta_{2}\cdot\mathbb{I}[\text{pol}=\text{O}]+\beta_{3}\cdot\mathbb{I}[\text{pol}=\text{R}]+\beta_{4}\cdot\text{age}^2+\epsilon_i
$$
\begin{footnotesize}
\begin{verbatim}
> summary(m4)

Call:
lm(formula = salary ~ age + pol + age2, data = salary_data)

Coefficients:
              Estimate Std. Error t value Pr(>|t|)
(Intercept) -24.751745  18.529243  -1.336  0.20452
age           3.272388   0.867049   3.774  0.00232 **
polO        -15.891696   4.198662  -3.785  0.00227 **
polR          9.260234   4.152253   2.230  0.04399 *
age2         -0.024576   0.009394  -2.616  0.02134 *
---

Residual standard error: 7.044 on 13 degrees of freedom
Multiple R-squared:  0.8802,	Adjusted R-squared:  0.8434
\end{verbatim}
\end{footnotesize}
\end{frame}

\section{Caution regarding scatterplots}

\begin{frame}{Scatterplots}
\begin{itemize}
\item Scatterplots show the \textit{marginal} relationship between $Y$ and
each of the $x_1, \ldots, x_k$. \pause They \textit{cannot} show you anything about the joint relationship among the $Y, x_1, \ldots, x_k$.
\item \pause Nonlinear relationship between $Y$ and $x_j$ ($j = 1, \ldots, k$) \textit{marginally} may or may not be present in the joint relationship.
\item \pause Actually, any strong relationship between $Y$ and $x_j$ marginally
doesn't mean that $x_j$ will be needed in the presence of other
variables.
\item \pause Seeing no marginal relationship between Y and $x_j$ does not
mean that $x_j$ is not needed in a model including other
predictors.
\end{itemize}
\end{frame}

\begin{frame}{No relationship?}
Here $Y$ vs. $x_1$ and $Y$ vs. $x_2$ shows nothing. There seems to be some multicollinearity though.
\vspace{10pt}

\centerline{\includegraphics[scale=0.3]{plots/scat}}
\end{frame}

\begin{frame}{\texttt{proc reg} output}
\centerline{$x_1$ important marginally? $Y_i=\beta_0+\beta_1 x_{i1}+\epsilon_i$}

\includegraphics[scale=0.35]{plots/x1}

\centerline{$x_2$ important marginally? $Y_i=\beta_0+\beta_2 x_{i2}+\epsilon_i$}

\includegraphics[scale=0.35]{plots/x2}

\centerline{$x_1$, $x_2$ important jointly? $Y_i=\beta_0+\beta_1 x_{i1}+\beta_2 x_{i2}+\epsilon_i$}

\includegraphics[scale=0.35]{plots/x12}
\end{frame}

\begin{frame}{Nonlinear relationship?}
Marginally, $x_1$ and $x_2$ have highly nonlinear relationships with $Y$. Should we transform?

\centerline{\includegraphics[scale=0.3]{plots/nonlinear}}
\end{frame}

\begin{frame}[fragile]
\frametitle{Regression output}
Let's try fitting a simple main effects model without any transformation.
$$
Y_i=\beta_0 + \beta_1 x_{i1} + \beta_2 x_{i2} + \epsilon_i
$$

\begin{footnotesize}
\begin{verbatim}
                   Parameter     Standard
Variable   DF       Estimate        Error      t Value     Pr > |t|
Intercept   1    -0.00036626      0.00130        -0.28       0.7791
x1          1        1.00022   0.00059936      1668.80       <.0001
x2          1        1.00009   0.00060998      1639.54       <.0001
\end{verbatim}
\end{footnotesize}

both $x_1$ and $x_2$ are important, but does the model fit okay?
\end{frame}

\begin{frame}{Model fit is okay}
\centerline{\includegraphics[scale=0.25]{plots/diag}}
\vspace{10pt}

Look at $Y_i$ vs. $\hat{Y}_i$ and $R^2$!
\end{frame}

\begin{frame}{No pattern here, either}
\centerline{\includegraphics[scale=0.25]{plots/yvsy}}
\end{frame}

\section{Model Building and Types of Studies}

\begin{frame}{9.1 Model building overview (pp.~343--349)}

Book outlines four steps in data analysis:
\begin{enumerate}
\item Data collection and preparation
\item \pause Reduction of explanatory variables. Mass screening for ``good'' predictors.
\item \pause Model refinement and selection.
\item \pause Model validation.
\end{enumerate}
\vspace{20pt}

\pause The way the data was collected may affect steps 2 \& 3.\\~\\
\pause Model validation should not be confused with model diagnostics (residual analysis).
\end{frame}

\begin{frame}{Data collection strategies}
\begin{itemize}
    \item \textbf{Controlled experiments}: subjects (experimental units) assigned to $x$-levels by experimenter
    \begin{itemize}
        \item<2-> \textbf{Purely controlled experiments}: researcher  uses only predictors that were assigned to units
        \item<3-> \textbf{Controlled experiments with covariates} (uncontrolled variables): researcher has additional predictors associated with units
    \end{itemize}
    \item<4-> \textbf{Observational studies}: subjects have $x$-levels associated with them (not assigned by researcher)
    \begin{itemize}
        \item<5-> \textbf{Confirmatory studies}: it is \textit{hypothesized} that new (primary) predictors are associated with $Y$; \pause while there are predictors, \textit{known} to be associated with $Y$ (called risk factors).
        \item<6-> \textbf{Exploratory studies}: it is hypothesized that some or all of potential predictors are associated with $Y$.
    \end{itemize}
\end{itemize}
\end{frame}

\begin{frame}{Reduction of Explanatory Variables}
\begin{itemize}
    \item Controlled experiments
    \begin{itemize}
        \item\pause Purely controlled experiments: rarely any need or desire to reduce number of predictors
        \item\pause Controlled experiments with covariates: remove any covariates that do not reduce the error variance
    \end{itemize}
    \item\pause Observational Studies
    \begin{itemize}
        \item\pause Confirmatory Studies: Must keep in all risk factors to compare with previous research, should keep all primary variables as well
        \item\pause Exploratory Studies: Often have many potential predictors (and polynomials and interactions). \pause Want to fit parsimonious model that explains much of the variation in $Y$, while keeping model as basic as possible. \pause Caution: one shall not make decisions based on single variable $t$-tests.
    \end{itemize}
\end{itemize}

\end{frame}

\begin{frame}{9.2 Surgical unit example}
\begin{itemize}
\item First steps often involve plots:
\begin{itemize}
\item Plots to indicate correct functional form of predictors and/or response.
\item Plots to indicate possible interaction.
\item Exploration of correlation among predictors (maybe).
\item Often a first-order model is a good starting point.
\end{itemize}
\item Once a reasonable set of potential predictors is identified, formal model selection begins.
\item If the number of predictors is large, say $k\ge10$, we can use (automated) stepwise procedures to reduce the number of variables (and models) under consideration.
\end{itemize}
\end{frame}

\begin{frame}{Surgical unit example: predictors}
A hospital surgical unit was interested in predicting survival in patients undergoing a particular type of liver operation. A random selection of 108 patients was available for analysis.
\begin{align*}
X_1\qquad&\text{blood clotting score}\\
X_2\qquad&\text{prognostic index}\\
X_3\qquad&\text{enzyme function test score}\\
X_4\qquad&\text{liver function test score}\\
\ldots
\end{align*}
\centerline{\includegraphics[scale=0.25]{plots/dataset}}
\end{frame}

\begin{frame}{Surgical unit example: residual plots}
\centerline{\includegraphics[scale=0.27]{plots/res-y}}
\end{frame}

\begin{frame}{Surgical unit example: scatterplot matrix}
\centerline{\includegraphics[scale=0.27]{plots/scat-mat}}
\end{frame}

\section{Model Selection}

\begin{frame}{9.3: Model selection}
Once we reduce the set of potential predictors to a reasonable
number, we can examine all possible models and choose the ``best''
according to some criterion.
\vspace{10pt}

\pause Say we have $k$ predictors $x_1$, \ldots, $x_k$ and we want to find a good subset of predictors that predict the data well. There are several
useful criteria to help choose a subset of predictors.
\end{frame}

\begin{frame}{Adjusted-$R^2$, $R_a^2$}
\begin{small}
%``Regular'' $R^2$ measures how well the model predicts the data that built it. It is possible to have a model with $R^2=1$ (predicts the data that built it perfectly), but has \textit{lousy out-of-sample prediction}.
%\vspace{10pt}

The \textit{adjusted} $R^2$, denoted $R_a^2$ ``fixes'' $R^2$ to provide a measure of how good the  model will predict data not used to build the model. For a candidate model with $p-1$ predictors
$$
R^2_a=1-\frac{\text{SSE}/(n-p)}{\text{SSTO}/(n-1)}\pause\quad\left(=1-\frac{\text{MSE}}{S^2_Y}\right).
$$
\begin{itemize}
\item<2-> Equivalent to choosing the model with the \textit{smallest} $\text{MSE}$.
\item<3-> If irrelevant variables are added, $R_a^2$ may decrease unlike ``regular'' $R^2$ ($R^2_a$ can be negative!).
\item<4-> $R_a^2$ penalizes model for being too complex.
\item<5-> Problem: $R_a^2$ is greater for a ``bigger'' model whenever the
F-statistic for comparing bigger to smaller is greater than 1 (\textbf{show this}). \onslide<6->{We usually want F-statistic to be a \textit{lot} bigger than 1 before
adding in new predictors}\onslide<7->{\quad $\Rightarrow$\quad \textit{too liberal}.}
\end{itemize}
\end{small}
\end{frame}

\begin{frame}{Akaike Information Criterion}
In general, \textbf{Akaike Information Criterion (AIC)} is
$$
\text{AIC}=-2\ln L(\hat{\boldsymbol\theta})+2p
$$
for a model with parameters $\boldsymbol\theta\in\mathbb{R}^p$ and likelihood function $L$.

\pause\begin{exe}
Show that for the MLR with normal errors,
$$
\text{AIC} = n\ln(\text{SSE})+2p + C
$$
where $C$ is a constant that does not depend on SSE and $p$.
\end{exe}
\begin{itemize}
\item\pause $2p$ is ``penalty'' term for adding predictors.
\item\pause Like $R_a^2$, AIC favors models with small SSE, but penalizes models with too many variables $p$.
\item\pause $\Rightarrow$ Between two models, we prefer the one with lower AIC.
\end{itemize}
\end{frame}

\begin{frame}{Bayesian
Information Criterion}
In general, \textbf{Bayesian
Information Criterion (BIC)} is
$$
\text{BIC}=-2\ln L(\hat{\boldsymbol\theta})+p\ln(n)
$$
\pause\begin{exe}
Show that for the MLR with normal errors,
$$
\text{BIC} = n\ln(\text{SSE}) + p\log(n) + C
$$
\end{exe}
\pause\begin{itemize}
\item BIC is similar to AIC, but for $n\ge8$, the BIC ``penalty term'' is
more severe.
\end{itemize}
\end{frame}

\begin{frame}{Mallow's $C_p$}
Full model with $k$ predictors and Reduced model with $p - 1$ predictors. \pause \textbf{Mallow's $C_p$} is
$$
C_p=\frac{\text{SSE}(\text{Reduced})}{\text{MSE}(\text{Full})}-n+2p
$$
\begin{itemize}
\item\pause Estimates $\frac{\E[\hat{Y}_i-\E[Y_i]]^2}{\sigma^2}$, where $\hat{Y}_i$ is from the Reduced model (pp. 357--359).
\item\pause Recall that $\E[\text{MSE}(\text{Full})]=\sigma^2$
\item\pause If the Reduced model is unbiased, i.e. $\E[\text{MSE}(\text{Reduced})]=\sigma^2$, \pause then $\E[\text{SSE}(\text{Reduced})]=(n-p)\sigma^2$, \pause and
$$
C_p\approx\frac{(n-p)\sigma^2}{\sigma^2}-n+2p=p
$$
\item\pause The Full model always has $C_{k+1} = k + 1$.
\end{itemize}
\end{frame}

\begin{frame}{Mallow's $C_p$}
If $C_p \approx p$ then the reduced model predicts as well as the full model. If $C_p < p$ then the reduced model is estimated to
be less \textit{biased} than the full model.

\begin{figure}
    \centering
    \includegraphics[height=.4\textheight]{plots/mallows_cp.png}
    \caption{A $C_p$ plot}
\end{figure}

\pause In practice, just choose model with smallest $C_p$.
\end{frame}

\begin{frame}{Which criteria to use?}
$R_a^2$, AIC, BIC, and $C_p$ may give different ``best'' models, or they
may agree. Ultimate goal is to find model that balances:
\begin{itemize}
\item\pause A good fit to the data.
\item\pause Low bias.
\item\pause Parsimony.
\end{itemize}
All else being equal, the simpler model is often easier to interpret
and work with. Christensen (1996) recommends $C_p$ and notes the
similarity between $C_p$ and AIC.
\end{frame}

\begin{frame}{Methods for ``automatically'' picking variables}
\begin{itemize}
\item Any regression textbook will caution against not thinking
about the data at all and simply using automated procedures.
\item\pause Automated procedures \textit{cannot} assess a good functional form for a predictor, \textit{cannot} think about which interactions might be important, etc.
\item\pause Anyway, automated procedures are widely used and \textit{can} produce good models. \pause But they can also produce models that are \textit{substantially inferior} to other models built from the same predictors using scientific input and common sense.
\end{itemize}
\end{frame}

\begin{frame}[fragile]{Example: cruise ships}{\url{https://github.com/zh3nis/MATH440/blob/main/chp09/cruise.R}}
A cruise ship company wishes to model the \texttt{crew size} needed for a ship using predictors such as: \texttt{age}, \texttt{tonnage}, \texttt{passengers}, \texttt{length}, \texttt{cabins} and passenger density (\texttt{passdens}).

\pause
\begin{tiny}
\begin{verbatim}
> cruise <- read.fwf("http://www.stat.ufl.edu/~winner/data/cruise_ship.dat", ...)
> head(cruise)
         ship      cline age tonnage passengers length cabins passdens  crew
1 Journey        Azamara   6  30.277       6.94   5.94   3.55    42.64  3.55
2 Quest          Azamara   6  30.277       6.94   5.94   3.55    42.64  3.55
3 Celebration   Carnival  26  47.262      14.86   7.22   7.43    31.80  6.70
4 Conquest      Carnival  11 110.000      29.74   9.53  14.88    36.99 19.10
5 Destiny       Carnival  17 101.353      26.42   8.92  13.21    38.36 10.00
6 Ecstasy       Carnival  22  70.367      20.52   8.55  10.20    34.29  9.20
\end{verbatim}
\end{tiny}

\pause Without concerning ourselves with potential interactions we will look at simple additive models.
\end{frame}

\begin{frame}[fragile]{Cruise ships --- Full model}
\begin{tiny}
\begin{verbatim}
> fit0 = lm(crew ~ age + tonnage + passengers + length + cabins + passdens, data=cruise)
> summary(fit0)

Call:
lm(formula = crew ~ age + tonnage + passengers + length + cabins +
    passdens, data = cruise)

Residuals:
    Min      1Q  Median      3Q     Max
-1.7700 -0.4881 -0.0938  0.4454  7.0077

Coefficients:
              Estimate Std. Error t value Pr(>|t|)
(Intercept) -0.5213400  1.0570350  -0.493  0.62258
age         -0.0125449  0.0141975  -0.884  0.37832
tonnage      0.0132410  0.0118928   1.113  0.26732
passengers  -0.1497640  0.0475886  -3.147  0.00199 **
length       0.4034785  0.1144548   3.525  0.00056 ***
cabins       0.8016337  0.0892227   8.985 9.84e-16 ***
passdens    -0.0006577  0.0158098  -0.042  0.96687
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.9819 on 151 degrees of freedom
Multiple R-squared:  0.9245,	Adjusted R-squared:  0.9215
F-statistic:   308 on 6 and 151 DF,  p-value: < 2.2e-16
\end{verbatim}
\end{tiny}
\end{frame}

\begin{frame}[fragile]
\frametitle{Best subsets}
\begin{tiny}
\begin{verbatim}
> library(leaps)
> allcruise <- regsubsets(crew ~ age + tonnage + passengers + length + cabins + passdens,
+                         nbest=4, data=cruise)
> all_output <- summary(allcruise)
> with(all_output, round(cbind(which, rsq, adjr2, cp, bic), 3))
  (Intercept) age tonnage passengers length cabins passdens   rsq adjr2      cp      bic
1           1   0       0          0      0      1        0 0.904 0.903  37.772 -360.238
1           1   0       1          0      0      0        0 0.860 0.859 125.086 -300.954
1           1   0       0          1      0      0        0 0.838 0.837 170.523 -277.122
1           1   0       0          0      1      0        0 0.803 0.801 240.675 -246.201
2           1   0       0          0      1      1        0 0.916 0.915  15.952 -376.131
2           1   0       0          0      0      1        1 0.912 0.911  24.261 -368.502
2           1   0       1          0      0      1        0 0.911 0.909  26.792 -366.249
2           1   0       0          1      0      1        0 0.908 0.907  32.443 -361.332
3           1   0       0          1      1      1        0 0.922 0.921   5.857 -382.878
3           1   0       0          0      1      1        1 0.919 0.918  11.341 -377.413
3           1   0       1          1      0      1        0 0.918 0.916  14.023 -374.808
3           1   1       0          0      1      1        0 0.917 0.915  15.909 -373.002
4           1   0       1          1      1      1        0 0.924 0.922   3.847 -381.933
4           1   1       0          1      1      1        0 0.923 0.921   5.084 -380.652
4           1   0       0          1      1      1        1 0.923 0.921   5.197 -380.534
4           1   0       1          0      1      1        1 0.919 0.917  13.056 -372.631
5           1   1       1          1      1      1        0 0.924 0.922   5.002 -377.752
5           1   0       1          1      1      1        1 0.924 0.922   5.781 -376.939
5           1   1       0          1      1      1        1 0.924 0.921   6.240 -376.462
5           1   1       1          0      1      1        1 0.920 0.917  14.904 -367.717
6           1   1       1          1      1      1        1 0.924 0.921   7.000 -372.692
\end{verbatim}
\end{tiny}
\pause A good model choice might be the model with 4 predictors: tonnage, passengers, length, and cabins, whose $R_a^2 = 0.922$, $C_p = 3.847$, and
$\text{BIC}= −381.933$. %Also we note that this model's AIC is lower than that of the full model.
\end{frame}

\begin{frame}[fragile]{AIC full vs reduced}
\begin{verbatim}
> fit3 <- update(fit0, . ~ . - age - passdens)
> AIC(fit3)
[1] 448.3229
> AIC(fit0)
[1] 451.4394
\end{verbatim}
\end{frame}

\begin{frame}{9.4 Automated variable search}
As discussed, it is possible to have a large set of predictor variables (including
interactions). The goal is to fit a ``parsimoneous'' model that explains as
much variation in the response as possible with a relatively small set of
predictors.\\~\\

\pause There are 3 automated procedures
\begin{itemize}
    \item\pause Backward Elimination (Top down approach)
    \item\pause Forward Selection (Bottom up approach)
    \item\pause Stepwise Regression (Combines Forward/Backward)
\end{itemize}

\pause We will explore these procedures using two different elimination/selection
criteria. \pause One that uses t-test and p-value and another that uses the AIC
value.
\end{frame}

\begin{frame}{Backward elimination}
\begin{enumerate}
\item Select a significance level to \textit{stay} in the model (e.g. $\alpha_s = 0.20$, generally
.05 is too low, causing too many variables to be removed).
\item\pause Fit the full model with all possible predictors.
\item\pause Consider the predictor with lowest t-statistic (highest p-value).
\begin{itemize}
    \item\pause If p-value $> \alpha_s$, remove the predictor and fit model without this
variable (must re-fit model here because partial regression coefficients change).
    \item\pause If p-value $\le \alpha_s$, stop and keep current model.
\end{itemize}
\item\pause Continue until all predictors have p-values $\le\alpha_s$.
\end{enumerate}
\end{frame}

\begin{frame}{Forward selection}
\begin{enumerate}
    \item Select a significance level to \textit{enter} the model (e.g. $\alpha_e = 0.20$, generally .05 is too low, causing too few variables to be entered).
\item\pause Fit all simple regression models.
\item\pause Consider the predictor with the highest t-statistic (lowest p-value).
\begin{itemize}
    \item\pause If p-value $\le\alpha_e$, keep this variable and fit all two variable models
that include this predictor.
    \item\pause If p-value $>\alpha_e$, stop and keep previous model.
\end{itemize}
\item\pause Continue until no new predictors have p-values $\le\alpha_e$.
\end{enumerate}
\end{frame}

\begin{frame}{Stepwise regression}
\begin{enumerate}
\item Select $\alpha_s$ and $\alpha_e$, ($\alpha_e < \alpha_s$).
\item\pause  Start like Forward Selection (bottom up process) where new variables
must have p-value $\le\alpha_e$ to enter.
\item\pause Re-test all ``old variables'' that have already been entered, must have
p-value $\le\alpha_s$ to stay in model.
\item\pause Continue until no new variables can be entered and no old variables
need to be removed.
\end{enumerate}
\end{frame}

\begin{frame}[fragile]{Backward and Forward for cruise ships}
\begin{scriptsize}
\begin{verbatim}
> library(olsrr)
> ols_step_backward_p(fit0)

                          Elimination Summary
------------------------------------------------------------------------
        Variable                  Adj.
Step    Removed     R-Square    R-Square     C(p)       AIC        RMSE
------------------------------------------------------------------------
   1    passdens      0.9245       0.922    5.0017    449.4412    0.9786
   2    age            0.924      0.9221    3.8468    448.3229    0.9782
------------------------------------------------------------------------

> ols_step_forward_p(fit0)

                             Selection Summary
---------------------------------------------------------------------------
        Variable                    Adj.
Step     Entered      R-Square    R-Square     C(p)        AIC        RMSE
---------------------------------------------------------------------------
   1    cabins          0.9041      0.9034    37.7724    479.2060    1.0886
   2    length          0.9160      0.9149    15.9524    460.2507    1.0221
   3    passengers      0.9220      0.9205     5.8566    450.4411    0.9878
   4    tonnage         0.9240      0.9221     3.8468    448.3229    0.9782
---------------------------------------------------------------------------
\end{verbatim}
\end{scriptsize}
\end{frame}

\begin{frame}[fragile]{Stepwise procedure for cruise ships}
\begin{tiny}
\begin{verbatim}
> ols_step_both_p(fit0)

                              Stepwise Selection Summary
---------------------------------------------------------------------------------------
                       Added/                   Adj.
Step     Variable     Removed     R-Square    R-Square     C(p)        AIC        RMSE
---------------------------------------------------------------------------------------
   1      cabins      addition       0.904       0.903    37.7720    479.2060    1.0886
   2      length      addition       0.916       0.915    15.9520    460.2507    1.0221
   3    passengers    addition       0.922       0.921     5.8570    450.4411    0.9878
   4     tonnage      addition       0.924       0.922     3.8470    448.3229    0.9782
---------------------------------------------------------------------------------------
\end{verbatim}
\end{tiny}
\end{frame}

\section{Model Validation (briefly)}

\begin{frame}{PRESS$_p$ criterion}
$$
\text{PRESS}_p = \sum_{i=1}^n(Y_i-\hat{Y}_{i(i)})^2\quad \left(=\sum_{i=1}^n\left[\frac{e_i}{1-h_{ii}}\right]^2\right),
$$
where $\hat{Y}_{i(i)}$ is the fitted value at $\mathbf{x}_i$ with the $(\mathbf{x}_i, Y_i)$ omitted.
\begin{itemize}
\item\pause This is leave-one-out prediction error. The smaller, the better.
\item\pause Having $\text{PRESS}_p \approx \text{SSE}_p$ supports the \textit{validity} of the model
with $p$ predictors (p.~374).
\end{itemize}
\end{frame}

\begin{frame}{Caveats for automated procedures}
\begin{itemize}
\item There is no ``best'' way to search for good models.
\item\pause There may be \textit{several} ``good'' models.
\item\pause If you use the same data to \textit{estimate} the model and \textit{choose}
the model, the regression effects are \textit{biased}!
\textit This leads to the idea of data splitting; one portion of the data
is the \textit{training data} and the other portion is the \textit{validation set}
(Section 9.6, p.~372). $\text{PRESS}_p$ can also be used.
\end{itemize}
\end{frame}

\end{document}

Slides/09 Model Selection and Validation/main.tex


\documentclass[UTF8]{article}

\usepackage{amssymb}
\usepackage{float}
\usepackage[T1]{fontenc}
\usepackage[left=1.5in, right=1.5in, top=0.8in, bottom=1in]{geometry}
\usepackage[nice]{nicefrac}
\usepackage{listings}
\usepackage{mathtools}
\usepackage{xcolor}

\setlength\parindent{0pt}
\setlength{\parskip}{\baselineskip}
\widowpenalties 1 10000

\definecolor{blue}{rgb}{0,0,0.5}
\definecolor{red}{rgb}{0.6,0,0}

\lstset{
    language=Python,
    basicstyle=\ttfamily\small,
    columns=fullflexible,
    frame=L,
    numbers=left,
    emph={exhaustive, babygiant, _map, pollard, pohlighellman},
    otherkeywords={True,False},
    emphstyle=\color{red},
    keywordstyle=\color{blue},
    showstringspaces=false
}

\renewcommand{\figurename}{Python Implementation}

\begin{document}

\title{
    Discrete Logarithm Algorithms \\
    \large Description and Implementation
}
\author{Davide Bergamaschi}
\date{2019}
\maketitle

\begin{abstract}
\noindent In this document we present the cryptographic problem of discrete logarithm and a number of algorithms to solve it, along with our implementation of said algorithms in the Python programming language. In particular, the following algorithms will be examined: exhaustive search, Baby-Step Giant-Step, Pollard's Rho, Pohlig-Hellman.
\end{abstract}

\section{The Discrete Logarithm Problem}

Let $G$ be a cyclic group of order $n$, let $\alpha$ be a generator of G and let $\beta$ belong to G. The discrete logarithm of $\beta$ to the base $\alpha$ ($\log_{\alpha}\beta$) is the integer value $x$ such that $0 \leq x \leq n - 1$ and $\alpha^x=\beta$.

\section{Solving Discrete Logarithms}

Calculating discrete logarithms is in general a computationally hard problem. In the following section we present a selection algorithms that perform this task with different levels of sophistication.

Each algorithm is implemented by a Python function which takes $\alpha$, $\beta$ and $n$ as its first arguments, and returns either $\log_{\alpha}\beta$ or $None$ if the logarithm cannot be found due to input data not respecting preconditions (e.g. if $\alpha$ is not a generator) or in case of failure of a randomized algorithm.

\newpage

\subsection{Exhaustive Search}

This naive method consists in computing $\alpha^0, \alpha^1, \ldots, \alpha^{n - 1}$ until $\beta$ is eventually obtained.

\begin{figure}[H]
    \centering
    \caption{Exhaustive Search}
    \begin{tabular}{c}
        \lstinputlisting{lst1-exhaustive.py}
    \end{tabular}
\end{figure}

The time complexity is trivially given by $O(n)$ multiplications.

\subsection{Baby-Step Giant-Step}

This method relies on the fact that the logarithm $x$ can be written as $x = i m + j$, where $m$ can be conveniently chosen as $\lceil \sqrt{n} \rceil$. By doing so, one obtains $\beta = \alpha^{x} = \alpha^{i m + j} = \alpha^{i m} \alpha^{j}$, which is true if and only if $\beta (\alpha^{-m})^i = \alpha^{j}$.

The algorithm hence proceeds in the following way. For $0 \leq j < m$, entries $(a^j, j)$ are computed and stored in a hash table (hashed on the first component).

Then, for $0 \leq i < \lceil n / m \rceil$, $\beta (\alpha^{-m})^i$ is computed. At each iteration the algorithm checks if the obtained value is present in the hash table. Upon finding a match with a value $j$, the logarithm x is obtained as $x = i m + j$.

\begin{figure}[H]
    \centering
    \caption{Baby-Step Giant-Step}
    \begin{tabular}{c}
        \lstinputlisting{lst2-babygiant.py}
    \end{tabular}
\end{figure}

In the first part, if \verb|m| has not been specified, it is calculated as $\lceil \sqrt{n} \rceil$. The lookup table is then constructed accordingly.

In each iteration of the last part, $\beta (\alpha^{-m})^i$ is computed by multiplying the \verb|candidate| variable, initially set to $\beta$, by the precomputed value $\alpha^{-m}$.

The algorithm performs $O(m)$ group multiplications while constructing the table, and $O(n/m)$ multiplications and lookups in the last part. If $m$ is set to $\sqrt{n}$, the time complexity becomes $O(\sqrt{n})$. The space complexity is instead given by the $m$ group elements stored by the algorithm.

\subsection{Pollard's Rho Algorithm for Logarithms}

This randomized algorithm explores a sequence of group elements $\gamma_i = \alpha^{a_i} \beta^{b_i}$ looking for a cycle using Floyd's algorithm.

Detecting a cycle means discovering two elements $\gamma_c = \alpha^{a}\beta^{b}$ and $\gamma_{2c} = \alpha^{A} \beta^{B}$ in the sequence, such that $\gamma_c = \gamma_{2c}$.

It follows that $\alpha^{a} \beta^{b} = \alpha^{A} \beta^{B}$, and so $\beta^{b - B} = \alpha^{a - A}$. By taking the logarithm to the base of $\alpha$ of both sides of the equation one obtains the following equation:
$$(b - B) \log_\alpha{\beta} \equiv (a - A) \pmod{n},$$
which can be solved to find $x=\log_\alpha{\beta}$, but only if $b \not\equiv B \pmod{n}$.

Let us discuss how the sequence of group elements is generated. To increase the likelihood of obtaining a cycle in a small number of samples, G is partitioned into three disjoint subsets $S_0$, $S_1$ and $S_2$ of approximately equal size and chosen in a sufficiently ``random'' manner.

The map for group elements $f: G \rightarrow G$ is then defined as:
$$
f(\gamma) =
\begin{cases}
    \beta \gamma  & \enspace \text{if } \gamma \in S_0 \\
    \gamma^2      & \enspace \text{if } \gamma \in S_1 \\
    \alpha \gamma & \enspace \text{if } \gamma \in S_2
\end{cases},
$$

while the map for $a_i$ coefficients $g: G \times \mathbb{N} \rightarrow \mathbb{N}$ and the map for $b_i$ coefficients $h: G \times \mathbb{N} \rightarrow \mathbb{N}$ are consequently defined as:
$$
g(\gamma, a) =
\begin{cases}
    a                & \enspace \text{if } \gamma \in S_0 \\
    (2a)    \bmod{n} & \enspace \text{if } \gamma \in S_1 \\
    (a + 1) \bmod{n} & \enspace \text{if } \gamma \in S_2
\end{cases},
\qquad
h(\gamma, b) =
\begin{cases}
    (b + 1) \bmod{n} & \enspace \text{if } \gamma \in S_0 \\
    (2b)    \bmod{n} & \enspace \text{if } \gamma \in S_1 \\
    b                & \enspace \text{if } \gamma \in S_2
\end{cases}.
$$

The algorithm begins by iterating over the sequence induced by the above maps, starting from two random coefficients $a_0$ and $b_0$ (and with $x_0 = \alpha^{a_0} \beta^{b_0}$); in particular, at each step $i$ it computes:
\begin{align*}
    \gamma_{i}  & = f(\gamma_{i-1})  & a_{i}  & = g(\gamma_{i-1}, a_{i-1})   & b_{i}  & = h(\gamma_{i-1}, a_{i-1})   \\
    \gamma_{2i} & = f(\gamma_{2i-2}) & a_{2i} & = g(\gamma_{2i-2}, a_{2i-2}) & b_{2i} & = h(\gamma_{2i-2}, b_{2i-2}),
\end{align*}
and compares the obtained $\gamma_{i}$ and $\gamma_{2i}$ values.

When the algorithm finds $\gamma_{i} = \gamma_{2i}$, that is when a cycle is detected, if ($b_{i} - b_{2i}) \bmod{n} \not= 0$ the algorithm returns the logarithm:
$$x = (b_{i} - b_{2i})^{-1} (a_{2i} - a_{i}) \bmod{n},$$
otherwise it terminates with failure.

\begin{figure}[H]
    \centering
    \caption{Pollard's Rho Algorithm for Logarithms}
    \begin{tabular}{c}
        \lstinputlisting{lst3-pollard.py}
    \end{tabular}
\end{figure}

The \verb|s_map| argument is a function which takes a group element as input and returns the index of the partition to which the element belongs.

For what concerns asymptotic complexity, if it can be assumed that the map $f$ behaves like a random function, the expected running time will be of $O(\sqrt{n})$ group operations. On the other hand, the required storage is negligible.

\subsection{Pohlig-Hellman Algorithm}

This method leverages the prime factorization of the group order: $n = p_1^{e_1} \ldots p_r^{e_r}$.

For $1 \leq i \leq r$, the algorithm calculates $x_i = x \bmod{p_i^{e_i}}$, obtaining a set of congruences that can ultimately be solved with the Chinese Remainder Theorem to find $x$.

In particular each $x_i$ can be seen as a truncated $p_i$-ary representation of $x$:
$$x_i = x \bmod{p_i^{e_i}} = l_0 + l_1 p_i + l_2 p_i^2 + \ldots + l_{e_i - 1} p_i^{e_i - 1},$$
which allows for a further reduction of the magnitude of the calculations.

Let us examine how the digits are calculated.

From the definition of modulus, there exists some integer $h$ for which $x=x_i+hp_i^{e_i}$. Since \mbox{$\beta=\alpha^x$}, one can derive:
\begin{align*}
    \beta^{n/p_i} & = (\alpha^x)^{n/p_i} \\
                  & = (\alpha^{n/p_i})^x \\
                  & = (\alpha^{n/p_i})^{x_i+hp_i^{e_i}} \\
                  & = (\alpha^{n/p_i})^{l_0 + l_1 p_i + l_2 p_i^2 + \ldots + l_{e_i - 1} p_i^{e_i - 1}+hp_i^{e_i}} \\
                  & = (\alpha^{n/p_i})^{l_0 + K p_i} \quad \text{\small{(for some integer $K$)}}  \\
                  & = (\alpha^{n/p_i})^{l_0} (\alpha^{n/p_i})^{K p_i} \\
                  & = (\alpha^{n/p_i})^{l_0} \quad \text{\small{(since $\alpha^{n/p_i}$ is of order $p_i$)}},
\end{align*}
from which $l_0$ can be found using any other discrete logarithm technique.

Then, starting from $\beta\alpha^{-l_0} = \alpha^{l_1 p_i + l_2 p_i^2 + \ldots + l_{e_i - 1} p_i^{e_i - 1}+hp_i^{e_i}}$ and by raising both sides to the power of $\nicefrac{n}{p_i^2}$, one can obtain $(\alpha^{n/p_i})^{l_1}=(\beta\alpha^{-l_0})^{n/p_i^2}$ and ultimately $l_1$ in a similar way.

The remaining digits can be computed analogously.

The calculations performed for each $p_i^{e_i}$ in the first part of the algorithm are in conclusion the following:
\begin{align*}
    l_0         &              = \log_{\alpha^{n / p_i}}(\beta)^{n / p_i} \\
    l_1         &              = \log_{\alpha^{n / p_i}}(\beta\alpha^{-l_0})^{n / p_i^2} \\
    l_2         &              = \log_{\alpha^{n / p_i}}(\beta\alpha^{-l_0 - l_1 \, p_i})^{n / p_i^3} \\
                & \vdotswithin{=} \\
    l_{e_i - 1} &              = \log_{\alpha^{n / p_i}}(\beta\alpha^{-l_0 - l_1 \, p_i - \, \ldots \, - l_{e_i - 2} \ p_i^{e_i - 2}})^{n / p_i^{e_i}}.
\end{align*}

Determining all the $x_i$ values yields a set of congruences of the following type:
\begin{align*}
    x & \equiv x_1 \pmod{p_1^{e_1}} \\
      & \vdotswithin{\equiv}        \\
    x & \equiv x_r \pmod{p_r^{e_r}}.
\end{align*}

Since the moduli are pairwise coprime by construction, the Chinese Remainder Theorem guarantees the existence of a solution $x$, which can be found with any suitable computational method (e.g. Gauss's algorithm).

\begin{figure}[H]
    \centering
    \caption{Pohlig-Hellman Algorithm}
    \begin{tabular}{c}
        \lstinputlisting{lst4-pohlighellman.py}
    \end{tabular}
\end{figure}

The prime factorization of $n$ is passed as fourth argument in the form of a Python dictionary, with keys being the prime factors and values are respectively their exponents.

Notice that during the computation of an $x_i$ value, at each iteration $j$ in the internal loop the implementation keeps track of the necessary powers of $p_i$ ($p_i^{j-1}$, $p_i^j$ and $p_i^{j+1}$), so that only one multiplication is enough to calculate them at each step.

The running time is given by $O(\sum_{i=1}^{r}e_i(\lg n + \sqrt{p_i}))$ group multiplications.

\end{document}