\documentclass[a4paper]{article}
\usepackage{graphicx,subcaption}
\usepackage{amsmath,amsfonts}
\title{Notes on Optimisation}
\author{G.A. Jarrad}
\begin{document}
\maketitle
\numberwithin{equation}{section}
\numberwithin{figure}{section}
\numberwithin{table}{section}
\section{Introduction}\label{sec:intro}
blah, blah, blah

\section{Recursive Least Square Error Estimation}
Consider using least square error optimisation to approximately fit the linear coefficient model
\begin{eqnarray}
   y & = & \vec{a}\cdot\vec{x}+\varepsilon
\label{eq:y_ax}
\end{eqnarray}
to the data $\left\{(\vec{x}_t,y_t)\;|\;t=1,2,\ldots,n\right\}$.
The square error is given by
\begin{eqnarray}
   S_n & = & \sum_{t=1}^n(y_t-\vec{a}\cdot\vec{x}_t)^2
\end{eqnarray}
and hence
\begin{eqnarray}
   \frac{\partial S_n}{\partial\vec{a}} & = & -2\sum_{t=1}^n(y_t-\vec{a}\cdot\vec{x}_t)\vec{x}_t\,.
\end{eqnarray}
The turning point $\hat{\vec{a}}_n$ then occurs when
\begin{eqnarray}
   \sum_{t=1}^n y_t\vec{x}_t = \sum_{t=1}^n\vec{x}_t\vec{x}_t^{T}\hat{\vec{a}}_n\,,
\label{eq:yx_xxa_n}
\end{eqnarray}
and thus $S_n$ is maximised when
\begin{eqnarray}
   \hat{\vec{a}}_n = \left(\sum_{t=1}^n\vec{x}_t\vec{x}_t^{T}\right)^{-1}\sum_{t=1}^n y_t\vec{x}_t\,.
\end{eqnarray}

Suppose now that another data point $(y_{n+1},\vec{x}_{n+1})$ is given. Then the new least square error estimate
is given by
\begin{eqnarray}
   \hat{\vec{a}}_{n+1} = \left(\sum_{t=1}^{n+1}\vec{x}_t\vec{x}_t^{T}\right)^{-1}\sum_{t=1}^{n+1} y_t\vec{x}_t\,,
\label{eq:yx_xxa_np1}
\end{eqnarray}
which appears to require yet another matrix inverse. However, this secondary inverse can avoided by using recursive estimation
of the form
\begin{eqnarray}
   P_{n+1} & = & \left(\sum_{t=1}^{n+1}\vec{x}_t\vec{x}_t^{T}\right)^{-1}
\nonumber\\
  & = & \left(\sum_{t=1}^{n}\vec{x}_t\vec{x}_t^{T}+\vec{x}_{n+1}\vec{x}_{n+1}^{T}\right)^{-1}
\nonumber\\
  & = & \left(P_n^{-1}+\vec{x}_{n+1}\vec{x}_{n+1}^{T}\right)^{-1}
\nonumber\\
  & = & P_n-\frac{P_n\vec{x}_{n+1}\vec{x}_{n+1}^{T}P_n}{1+\vec{x}_{n+1}^{T}P_n\vec{x}_{n+1}}\,,
\end{eqnarray}
from the Woodbury matrix identity
\begin{eqnarray}
    \left(A+UCV \right)^{-1} = A^{-1} - A^{-1}U \left(C^{-1}+VA^{-1}U \right)^{-1} VA^{-1}\,.
\end{eqnarray}
Consequently, observe that
\begin{eqnarray}
   \sum_{t=1}^{n+1} y_t\vec{x}_t & = & \sum_{t=1}^{n} y_t\vec{x}_t+y_{n+1}\vec{x}_{n+1}
\nonumber\\
& = & \sum_{t=1}^n\vec{x}_t\vec{x}_t^{T}\hat{\vec{a}}_n+y_{n+1}\vec{x}_{n+1}
\nonumber\\
& = & \sum_{t=1}^{n+1}\vec{x}_t\vec{x}_t^{T}\hat{\vec{a}}_n-\vec{x}_{n+1}\vec{x}_{n+1}^{T}\hat{\vec{a}}_n
         +y_{n+1}\vec{x}_{n+1}
\nonumber\\
& = & P_{n+1}^{-1}\hat{\vec{a}}_n+\vec{x}_{n+1}(y_{n+1}-\hat{\vec{a}}_n\cdot\vec{x}_{n+1})
\end{eqnarray}
from equation~\eqref{eq:yx_xxa_n}, and hence the estimate $\hat{\vec{a}}_{n+1}$ is given recursively by
\begin{eqnarray}
   \hat{\vec{a}}_{n+1} = \hat{\vec{a}}_{n} + P_{n+1} \vec{x}_{n+1}(y_{n+1}-\hat{\vec{a}}_n\cdot\vec{x}_{n+1})
\end{eqnarray}
from equation~\eqref{eq:yx_xxa_np1}. Observe that this recursive estimate takes the form of a predictor--corrector
update, since $\hat{\vec{a}}_n\cdot\vec{x}_{n+1}$ is the expected value of
$y_{n+1}$ given $\hat{\vec{a}}_n$, from model~\eqref{eq:y_ax}.

As a slight alternative to the above approach to re-estimation,
 suppose now instead that we are using a sliding data window of fixed width $n$, e.g.\ for time series analysis.
Then the $k$-th parameter estimate $\hat{\vec{a}}_{n}^{(k)}$ is given by
\begin{eqnarray}
   \hat{\vec{a}}_{n}^{(k)} = \left(\sum_{t=k}^{n+k-1}\vec{x}_t\vec{x}_t^{T}\right)^{-1}\sum_{t=k}^{n+k-1} y_t\vec{x}_t\,.
\label{eq:yx_xxa_nk_alt}
\end{eqnarray}
Hence, observe that
\begin{eqnarray}
   P_{n}^{(k+1)} & = & \left(\sum_{t=k+1}^{n+k}\vec{x}_t\vec{x}_t^{T}\right)^{-1}
\nonumber\\
  & = & \left(\sum_{t=k}^{n+k-1}\vec{x}_t\vec{x}_t^{T}-\vec{x}_k\vec{x}_k^T+\vec{x}_{n+k}\vec{x}_{n+k}^{T}\right)^{-1}
\nonumber\\
  & = & \left({Q_n^{(k)}}^{-1}+\vec{x}_{n+k}\vec{x}_{n+k}^{T}\right)^{-1}
\nonumber\\
  & = & Q_n^{(k)}-\frac{Q_n^{(k)}\vec{x}_{n+k}\vec{x}_{n+k}^{T}Q_n^{(k)}}{1+\vec{x}_{n+k}^{T}Q_n^{(k)}\vec{x}_{n+k}}\,,
\end{eqnarray}
where
\begin{eqnarray}
   Q_{n}^{(k)} & = & \left(\sum_{t=k}^{n+k-1}\vec{x}_t\vec{x}_t^{T}-\vec{x}_k\vec{x}_k^T\right)^{-1}
\nonumber\\
  & = & \left({P_n^{(k)}}^{-1}-\vec{x}_{k}\vec{x}_{k}^{T}\right)^{-1}
\nonumber\\
  & = & P_n^{(k)}+\frac{P_n^{(k)}\vec{x}_{k}\vec{x}_{k}^{T}P_n^{(k)}}{1-\vec{x}_{k}^{T}P_n^{(k)}\vec{x}_{k}}\,.
\end{eqnarray}
Consequently,
\begin{eqnarray}
   \sum_{t=k+1}^{n+k} y_t\vec{x}_t & = & \sum_{t=k}^{n+k-1} y_t\vec{x}_t-y_k\vec{x}_k+y_{n+k}\vec{x}_{n+k}
\nonumber\\
& = & \sum_{t=k}^{n+k-1}\vec{x}_t\vec{x}_t^{T}\hat{\vec{a}}_n^{(k)}-y_k\vec{x}_k+y_{n+k}\vec{x}_{n+k}
\nonumber\\
& = & \sum_{t=k+1}^{n+k}\vec{x}_t\vec{x}_t^{T}\hat{\vec{a}}_n^{(k)}
+\vec{x}_k\vec{x}_k^{T}\hat{\vec{a}}_n^{(k)}-\vec{x}_{n+k}\vec{x}_{n+k}^{T}\hat{\vec{a}}_n^{(k)}
\nonumber\\
&&        \hspace*{10mm}{}-y_k\vec{x}_k+y_{n+k}\vec{x}_{n+k}
\nonumber\\
& = & {P_{n}^{(k+1)}}^{-1}\hat{\vec{a}}_n^{(k)}
-\vec{x}_{k}(y_{k}-\hat{\vec{a}}_n^{(k)}\cdot\vec{x}_{k})
\nonumber\\
&& \hspace*{10mm}{}+\vec{x}_{n+k}(y_{n+k}-\hat{\vec{a}}_n^{(k)}\cdot\vec{x}_{n+k})\,,
\end{eqnarray}
and hence
\begin{eqnarray}
   \hat{\vec{a}}_{n}^{(k+1)} & = & \hat{\vec{a}}_{n}^{(k)}
-P_n^{(k+1)}\vec{x}_{k}(y_{k}-\hat{\vec{a}}_n^{(k)}\cdot\vec{x}_{k})
\nonumber\\
&&\hspace*{6mm}{}+P_n^{(k+1)}\vec{x}_{n+k}(y_{n+k}-\hat{\vec{a}}_n^{(k)}\cdot\vec{x}_{n+k})
\,,
\label{eq:yx_xxa_nkp1_alt}
\end{eqnarray}
from equation~\eqref{eq:yx_xxa_nk_alt}.

\end{document}


\lab{The QR Decomposition}{The QR Decomposition}
\label{lab:QRdecomp}
\objective{The QR decomposition is a fundamentally important matrix factorization.
It is straightforward to implement, is numerically stable, and provides the basis of several important algorithms.
In this lab we explore several ways to produce the QR decomposition and implement a few immediate applications.
% \\ \indent We restrict our discussion to real matrices.
% However, the results and algorithms presented here can be extended to complex matrices by replacing ``transpose'' with ``hermitian conjugate'' and ``symmetric matrix'' with ``hermitian matrix.''
}

The QR decomposition of a matrix $A$ is a factorization $A=QR$, where $Q$ is  has orthonormal columns and $R$ is upper triangular.
Every $m \times n$ matrix $A$ of rank $n \le m$ has a QR decomposition, with two main forms.
%
\begin{itemize}
    \item \textbf{Reduced QR}: $Q$ is $m \times n$, $R$ is $n \times n$, and the columns $\{\q_j\}_{j=1}^n$ of $Q$ form an orthonormal basis for the column space of $A$.
    \item \textbf{Full QR}: $Q$ is $m \times m$ and $R$ is $m \times n$.
    In this case, the columns $\{\q_j\}_{j=1}^m$ of $Q$ form an orthonormal basis for all of $\mathbb{F}^m$, and the last $m - n$ rows of $R$ only contain zeros.
    If $m = n$, this is the same as the reduced factorization.
\end{itemize}
We distinguish between these two forms by writing $\widehat{Q}$ and $\widehat{R}$ for the reduced decomposition and $Q$ and $R$ for the full decomposition.
%
\begin{align*} % Reduced and full QR decompositions.
\begin{array}{ccc}
\textcolor{red}{\widehat{Q}\ (m \times n)} & \textcolor{blue}{\widehat{R}\ (n \times n)} & \\
\left[\begin{array}{cccccccc}
\arrayrulecolor{red}
\cline{2-4}
& \lvl{}     &        & \rvl{}     &          &        &      & \\
& \lvl{}     &        & \rvl{}     &          &        &      & \\
& \lvl{}     &        & \rvl{}     &          &        &      & \\
& \lvl{\q_1} & \cdots & \rvl{\q_n} & \q_{n+1} & \cdots & \q_m & \\
& \lvl{}     &        & \rvl{}     &          &        &      & \\
& \lvl{}     &        & \rvl{}     &          &        &      & \\
& \lvl{}     &        & \rvl{}     &          &        &      & \\
\cline{2-4}
\end{array}\right]
&
\left[\begin{array}{ccccc}
\arrayrulecolor{blue}
\cline{2-4}
& \lvl{r_{11}} & \cdots & \rvl{r_{1n}} & \\
& \lvl{}       & \ddots & \rvl{\vdots} & \\
& \lvl{}       &        & \rvl{r_{nn}} & \\
\cline{2-4}
& 0            & \cdots & 0            & \\
& \vdots       &        & \vdots       & \\
& 0            & \cdots & 0            & \\
\end{array}\right]
& =\ A\ (m \times n)
\\
Q\ (m \times m) & R\ (m \times n) & \\
\end{array}
\end{align*}

\section*{QR via Gram-Schmidt} % ==============================================

The \emph{classical Gram-Schmidt algorithm} takes a linearly independent set of vectors and constructs an orthonormal set of vectors with the same span.
Applying Gram-Schmidt to the columns of $A$, which are linearly independent since $A$ has rank $n$, results in the columns of $Q$.

Let $\{\x_j\}_{j=1}^n$ be the columns of $A$.
Define
%
\begin{align*}
\q_1 = \frac{\x_1}{\|\x_1\|},
&&
\q_{k} = \frac{\x_k - \p_{k-1}}{\|\x_k - \p_{k-1}\|},\quad k=2,\ \ldots,\ n,
\\ \\
\p_0 = \0,
&&
\p_{k-1} = \sum_{j=1}^{k-1} \langle \q_j, \x_k\rangle \q_j,\quad k=2,\ \ldots,\ n.
\end{align*}

Each $\p_{k-1}$ is the projection of $\x_k$ onto the span of $\{\q_j\}_{j=1}^{k-1}$, so $\q_k' = \x_k - \p_{k-1}$ is the residual vector of the projection.
Thus $\q_k'$ is orthogonal to each of the vectors in $\{\q_j\}_{j=1}^{k-1}$.
Therefore, normalizing each $\q_k'$ produces an orthonormal set $\{\q_j\}_{j=1}^n$.

To construct the reduced QR decomposition, let $\widehat{Q}$ be the matrix with columns $\{\q_j\}_{j=1}^n$, and let $\widehat{R}$ be the upper triangular matrix with entries
%
\begin{align*}
r_{kk} = \|\x_k-\p_{k-1}\|,
&&
r_{jk} = \langle \q_j, \x_k\rangle = \q_j\trp\x_k,\ j < k.
\end{align*}
%
This clever choice of entries for $\widehat{R}$ reverses the Gram-Schmidt process and ensures that $\widehat{Q}\widehat{R} = A$.

\begin{comment}
To construct the full QR decomposition, choose $m - n$ vectors $\{\x_j\}_{j=n+1}^m$ such that the entire set of original vectors $\{\x_j\}_{j=1}^m$ is linearly independent, then continue the Gram-Schmidt process to produce the additional columns of $Q$.
Appending $m - n$ rows of zeros to $\widehat{R}$ results in $R$.
\end{comment}

\subsection*{Modified Gram-Schmidt} % -----------------------------------------

If the columns of $A$ are close to being linearly dependent, the classical Gram-Schmidt algorithm often produces a set of vectors $\{\q_j\}_{j=1}^n$ that are not even close to orthonormal due to rounding errors.
The \emph{modified Gram-Schmidt algorithm} is a slight variant of the classical algorithm which more consistently produces a set of vectors that are ``very close'' to orthonormal.

Let $\q_1$ be the normalization of $\x_1$ as before.
Instead of making just $\x_2$ orthogonal to $\q_1$, make \textbf{each} of the vectors $\{\x_j\}_{j=2}^n$ orthogonal to $\q_1$:
\[\x_k = \x_k - \langle \q_1,\x_{k} \rangle \q_1,\quad k=2,\ \ldots,\ n.\]
Next, define $\q_2 = \frac{\x_2}{\|\x_2\|}$.
Proceed by making each of $\{\x_j\}_{j=3}^n$ orthogonal to $\q_2$:
\[\x_k = \x_k - \langle \q_2,\x_{k} \rangle \q_2,\quad k=3,\ldots,n.\]
Since each of these new vectors is a linear combination of vectors orthogonal to $\q_1$, they are orthogonal to $\q_1$ as well.
Continuing this process results in the desired orthonormal set $\{\q_j\}_{j=1}^n$.
The entire modified Gram-Schmidt algorithm is described below. % in Algorithm \ref{Alg:modified-Gram-Schmidt}.

\begin{algorithm}[H]
\begin{algorithmic}[1]
\Procedure{Modified Gram-Schmidt}{$A$}
    \State $m, n \gets \shape{A}$
        \Comment{Store the dimensions of $A$.}
    \State $Q \gets \makecopy{A}$
        \Comment{Make a copy of $A$ with \li{np.copy()}.}
    \State $R \gets \zeros{n}{n}$
        \Comment{An $n\times n$ array of all zeros.}
    \For{$i=0\ldots n-1$}
        \State $R_{i,i} \gets \|Q_{:,i}\|$\label{step:mgs-normalize}
            % \Comment{(Hint: use \li{scipy.linalg.norm()}).}
        \State $Q_{:,i} \gets Q_{:,i}/R_{i,i}$\label{step:mgs-mult1}
            \Comment{Normalize the $i$th column of $Q$.}
        \For{$j=i+1\ldots n-1$}
            \State $R_{i,j} \gets Q_{:,j}\trp  Q_{:,i}$\label{step:mgs-mult2}
            \State $Q_{:,j} \gets Q_{:,j}-R_{i,j}Q_{:,i}$\label{step:mgs-mult3}
                \Comment{Orthogonalize the $j$th column of $Q$.}
        \EndFor
    \EndFor
    \State \pseudoli{return} $Q, R$
\EndProcedure
\end{algorithmic}
\caption{}
\label{Alg:modified-Gram-Schmidt}
\end{algorithm}

\begin{problem} % QR via Modified Gram Schmidt.
Write a function that accepts an $m \times n$ matrix $A$ of rank $n$.
Use Algorithm \ref{Alg:modified-Gram-Schmidt} to compute the reduced QR decomposition of $A$.

Consider the following tips for implementing the algorithm.
\begin{itemize}

% \item In Python, the operation \li{a = a + b} can also be written as \li{a += b}.

\item Use \li{scipy.linalg.norm()} to compute the norm of the vector in step \ref{step:mgs-normalize}.

\item Note that steps \ref{step:mgs-mult1} and \ref{step:mgs-mult3} employ scalar multiplication or division, while step \ref{step:mgs-mult2} uses vector multiplication.

\end{itemize}

To test your function, generate test cases with NumPy's \li{np.random} module.
Verify that $R$ is upper triangular, $Q$ is orthonormal, and $QR = A$.
You may also want to compare your results to SciPy's QR factorization routine, \li{scpiy.linalg.qr()}.

\begin{lstlisting}
>>> import numpy as np
>>> from scipy import linalg as la

# Generate a random matrix and get its reduced QR decomposition via SciPy.
>>> A = np.random.random((6,4))
>>> Q,R = la.qr(A, mode="economic") # Use mode="economic" for reduced QR.
>>> print(A.shape, Q.shape, R.shape)
(6,4) (6,4) (4,4)

# Verify that R is upper triangular, Q is orthonormal, and QR = A.
>>> np.allclose(np.triu(R), R)
<<True>>
>>> np.allclose(Q.T @ Q, np.identity(4))
<<True>>
>>> np.allclose(Q @ R, A)
<<True>>
\end{lstlisting}
\label{prob:qr-via-mgs}
\end{problem}

\section*{Consequences of the QR Decomposition} % =============================

The special structures of $Q$ and $R$ immediately provide some simple applications.

\subsection*{Determinants} % --------------------------------------------------

Let $A$ be $n \times n$.
Then $Q$ and $R$ are both $n \times n$ as well.%
\footnote{An $n \times n$ orthonormal matrix is sometimes called \emph{unitary} in other texts.}
Since $Q$ is orthonormal and $R$ is upper-triangular,
\[\det(Q) = \pm 1 \qquad\text{and}\qquad \det(R) = \prod_{i=1}^n r_{i,i}.\]
Then since $\det(AB) = \det(A)\det(B)$,
\begin{equation}
\label{eq:qr-determinant}
\abs{\det(A)} = \abs{\det(QR)} = \abs{\det(Q)\det(R)} = \abs{\det(Q)}\abs{\det(R)} = \abs{\prod_{i=1}^n r_{i,i}}.
\end{equation}

\begin{problem} % Use the QR decomposition to calculate |det(A)|.
Write a function that accepts an invertible matrix $A$.
Use the QR decomposition of $A$ and (\ref{eq:qr-determinant}) to calculate $\abs{\det(A)}$.
You may use your QR decomposition algorithm from Problem \ref{prob:qr-via-mgs} or SciPy's QR routine.
Can you implement this function in a single line?
\\(Hint: \li{np.diag()} and \li{np.prod()} may be useful.)

Check your answer against \li{la.det()}, which calculates the determinant.
\end{problem}

\subsection*{Linear Systems} % ------------------------------------------------

The LU decomposition is usually the matrix factorization of choice to solve the linear system $A\x = \b$ because the triangular structures of $L$ and $U$ facilitate forward and backward substitution.
However, the QR decomposition avoids the potential numerical issues that come with Gaussian elimination.

Since $Q$ is orthonormal, $Q^{-1}= Q\trp$.
Therefore, solving $A\x = \b$ is equivalent to solving the system $R\x = Q\trp\b$.
Since $R$ is upper-triangular, $R\x = Q\trp\b$ can be solved quickly with back substitution.%
\footnote{See the Linear Systems lab for details on back substitution.}

\begin{problem} % Use the QR decomposition to solve Ax = b quickly.
Write a function that accepts an invertible $n \times n$ matrix $A$ and a vector $\b$ of length $n$.
Use the QR decomposition to solve $A\x = \b$ in the following steps:
\begin{enumerate}
    \item Compute $Q$ and $R$.
    \item Calculate $\y = Q\trp\b$.
    \item Use back substitution to solve $R\x = \y$ for $\x$.
\end{enumerate}
\end{problem}

\section*{QR via Householder} % ===============================================

The Gram-Schmidt algorithm orthonormalizes $A$ using a series of transformations that are stored in an upper triangular matrix.
Another way to compute the QR decomposition is to take the opposite approach: triangularize $A$ through a series of orthonormal transformations.
Orthonormal transformations are numerically stable, meaning that they are less susceptible to rounding errors.
In fact, this approach is usually faster and more accurate than Gram-Schmidt methods.

The idea is for the $k$th orthonormal transformation $Q_k$ to map the $k$th column of $A$ to the span of $\{\e_j\}_{j=1}^k$, where the $\e_j$ are the standard basis vectors in $\mathbb{R}^m$.
In addition, to preserve the work of the previous transformations, $Q_k$ should not modify any entries of $A$ that are above or to the left of the $k$th diagonal term of $A$.
For a $4 \times 3$ matrix $A$, the process can be visualized as follows.
%
\begin{align*}
Q_3 Q_2 Q_1
\left[\begin{array}{ccccc}
\cline{2-4}
& \lvl{*} & * & \rvl{*} & \\
& \lvl{*} & * & \rvl{*}& \\
& \lvl{*} & * & \rvl{*} & \\
& \lvl{*} & * & \rvl{*} & \\ \cline{2-4}
\end{array}\right]
= Q_3 Q_2
\left[\begin{array}{ccc}
     *  & * &      * \\
\cline{2-3}
\rvl{0} & * & \rvl{*} \\
\rvl{0} & * & \rvl{*} \\
\rvl{0} & * & \rvl{*} \\
\cline{2-3}
\end{array}\right]
= Q_3
\left[\begin{array}{ccc}
* & * & * \\
0 & * & * \\
\cline{3-3}
0 & \rvl{0} & \rvl{*} \\
0 & \rvl{0} & \rvl{*} \\
\cline{3-3}
\end{array}\right]
=
\left[\begin{array}{ccc}
* & * & * \\
0 & * & * \\
0 & 0 & * \\
0 & 0 & 0 \\
\end{array}\right]
\end{align*}

Thus $Q_3 Q_2 Q_1 A = R$, so that $A = Q_1\trp Q_2\trp Q_3\trp R$ since each $Q_k$ is orthonormal.
Furthermore, the product of square orthonormal matrices is orthonormal, so setting $Q = Q_1\trp Q_2\trp Q_3\trp$ yields the full QR decomposition.

How to correctly construct each $Q_k$ isn't immediately obvious.
The ingenious solution lies in one of the basic types of linear transformations: reflections.

\subsection*{Householder Transformations} % -----------------------------------

The \emph{orthogonal complement} of a nonzero vector $\v \in \mathbb{R}^n$ is the set of all vectors $\x \in \mathbb{R}^n$ that are orthogonal to $\v$, denoted $\v^\perp = \{ \x \in \mathbb{R}^n \mid \langle \x, \v \rangle = 0 \}$.
A \emph{Householder transformation} is a linear transformation that reflects a vector $\x$ across the orthogonal complement $\v^\perp$ for some specified $\v$.

The matrix representation of the Householder transformation corresponding to $\v$ is given by $H_{\v} = I - 2\frac{\v\v\trp}{\v\trp\v}$.
Since $H_{\v}\trp H_{\v} = I$, Householder transformations are orthonormal.

\begin{figure}[H]
\centering
\begin{tikzpicture}
    \draw[-,dashed, gray](-2,-1.333)--(3, 2);               % hyperplane
    \draw[->, gray, >=stealth,ultra thick](0,0)--(.8,-1.2); % v
    \draw[->, >=stealth, thick](0,0)--(2.815, .777);        % H x
    \draw[->, >=stealth, thick](0,0)--(1.8,2.3);            % x
    \node[draw=none](v)at(.65,-.6){$\v$};
    \node[draw=none](x)at(.75,1.5){$\x$};
    \node[draw=none](Hx)at(3, .5){$H_{\v}\x$};
    \node[draw=none](H)at(-1.5,-.6){$\v^\perp$};
    % \node[draw=none](bullet)at(2.31, 1.53){\textbullet};
    % \node[draw=none](vandx)at(4.0,1.5){$\x -
    % \left \langle \dfrac{\v}{\|\v\|}, \x \right \rangle \dfrac{\v}{\|\v\|}$};
\end{tikzpicture}
\caption{The vector $\v$ defines the orthogonal complement $\v^\perp$, which in this case is a line.
Applying the Householder transformation $H_{\mathbf{v}}$ to $\x$ reflects $\x$ across $\v^\perp$.}
\label{fig:Householder_reflector}
\end{figure}

\subsection*{Householder Triangularization} % ---------------------------------

The \emph{Householder algorithm} uses Householder transformations for the orthonormal transformations in the QR decomposition process described on the previous page.
The goal in choosing $Q_k$ is to send $\x_k$, the $k$th column of $A$, to the span of $\{\e_j\}_{j=1}^k$.
In other words, if $Q_k\x_k = \y_k$, the last $m - k$ entries of $\y_k$ should be $0$, i.e.,
\begin{align*}
Q_k\x_k = Q_k
\left[\begin{array}{c}
z_1 \\ \vdots \\ z_k \\ z_{k+1} \\ \vdots \\ z_m
\end{array}\right]
=
\left[\begin{array}{c}
y_1 \\ \vdots \\ y_k \\ 0 \\ \vdots \\ 0
\end{array}\right]
= \y_k.
\end{align*}

To begin, decompose $\x_k$ into $\x_k = \x_k' + \x_k''$, where $\x_k'$ and $\x_k''$ are of the form
\[
\x_k' = \left[z_1\quad \cdots\quad z_{k-1}\quad 0\quad \cdots\quad 0\right]\trp,
\qquad\qquad
\x_k'' = \left[0\quad \cdots\quad 0\quad z_k\quad \cdots\quad z_m\right]\trp.
\]
Because $\x_k'$ represents elements of $A$ that lie above the diagonal, only $\x_k''$ needs to be altered by the reflection.

The two vectors $\x_k''\ \pm\ \|\x_k''\|\e_k$ both yield Householder transformations that send $\x_k''$ to the span of $\e_k$ (see Figure \ref{fig:householder-two-possible-reflectors}).
Between the two, the one that reflects $\x_k''$ further is more numerically stable.
This reflection corresponds to \[\v_k = \x_k'' + \sign(z_k)\|\x_k''\| \e_k,\] where $z_k$ is the first nonzero component of $\x_k''$ (the $k$th component of $\x_k$).

\begin{figure}[H] % Two possible reflections into the span of e_1.
\begin{tikzpicture}

\draw[-, dashed, gray](-3,-1)--(3,1);
\draw[-, dashed, gray](-.8,2.4)--(.6,-1.8);
\draw[-, gray, thick](-4,0)--(4,0);
\draw[->, thick, >=stealth'](0,0)--(2,1.6);
\draw[<->, thick, >=stealth'](-2.6,0)--(2.6,0);
\draw[->, gray,  ultra thick, >=stealth](0,0)--(1.5,.5);
\draw[->, gray, ultra thick, >=stealth](0,0)--(-.5,1.5);

\node[draw=none, node distance=3.5cm]
    (dummy)at(2.5,.2){};
\node[draw=none, node distance=.5cm](Hvx)
    [below of=dummy]{$H_{\v_1}\x$};
\node[draw=none, node distance=2cm](x)
    [above left of=Hvx]{$\x$};
\node[draw=none](v1)[left of=x]{$\v_1$};
\node[draw=none, node distance=.55cm](v2)[below of=x]{$\v_2$};
\node[draw=none, node distance=4cm](hvx2)[left of=dummy]{$H_{\v_2}\x$};

\end{tikzpicture}
\caption{There are two reflections that map $\x$ into the span of $\e_1$, defined by the vectors $\v_1$ and $\v_2$.
In this illustration, $H_{\v_2}$ is the more stable transformation since it reflects $\x$ further than $H_{\v_1}$.}
\label{fig:householder-two-possible-reflectors}
\end{figure}

After choosing $\v_k$, set $\u_k = \frac{\v_k}{\|\v_k\|}$.
Then $H_{\v_k} = I - 2\frac{\v_k\v_k\trp}{\|\v_k\|^2} = I - 2\u_k\u_k\trp$, and hence $Q_k$ is given by the block matrix
\begin{equation*}
Q_k =
\left[\begin{array}{cc}
I_{k-1} & \0 \\
\0      & H_{\v_k} \\
\end{array}\right] =
\left[\begin{array}{cc}
I_{k-1} & \0 \\
\0      & I_{m-k+1} - 2 \u_k\u_k\trp \\
\end{array}\right].
\end{equation*}

Here $I_{p}$ denotes the $p \times p$ identity matrix, and thus each $Q_k$ is $m \times m$.

It is apparent from its form that $Q_k$ does not affect the first $k-1$ rows and columns of any matrix that it acts on.
Then by starting with $R = A$ and $Q = I$, at each step of the algorithm we need only multiply the entries in the lower right $(m-k+1) \times (m-k+1)$ submatrices of $R$ and $Q$ by $I-2\u_k\u_k\trp$.
This completes the Householder algorithm, detailed below.

\begin{algorithm}[H] % QR Decomposition via Householder Reflections.
\begin{algorithmic}[1]
\Procedure{Householder}{$A$}
    \State $m, n \gets \shape{A}$
    \State $R \gets \makecopy{A}$
    \State $Q \gets I_{m}$
        \Comment{The $m\times m$ identity matrix.}
    \For{$k=0\ldots n-1$}
        \State $\u \gets \makecopy{R_{k:,k}}$
        \State $u_0 \gets u_0 + \sign(u_0)\|\u\|$
            \Comment{$u_0$ is the first entry of $\u$.}\label{step:HH-sign}
        \State $\u \gets \u/\|\u\|$
            \Comment{Normalize $\u$.}
        \State $R_{k:,k:} \gets R_{k:,k:} - 2\u\left(\u\trp R_{k:,k:}\right)$
            \Comment{Apply the reflection to $R$.}\label{step:HH-outer1}
        \State $Q_{k:,:} \gets Q_{k:,:} - 2\u\left(\u\trp Q_{k:,:}\right)$
            \Comment{Apply the reflection to $Q$.}\label{step:HH-outer2}
    \EndFor
    \State \pseudoli{return} $Q\trp, R$
\EndProcedure
\end{algorithmic}
\caption{}
\label{Alg:QR-via-Householder}
\end{algorithm}

\begin{problem} % QR Decomposition via Householder Triangularization
Write a function that accepts as input a $m \times n$ matrix $A$ of rank $n$.
Use Algorithm \ref{Alg:QR-via-Householder} to compute the full QR decomposition of $A$.

Consider the following implementation details.
\begin{itemize}

\item NumPy's \li{np.sign()} is an easy way to implement the $\sign()$ operation in step \ref{step:HH-sign}.
However, \li{np.sign(0)} returns $0$, which will cause a problem in the rare case that $u_0 = 0$ (which is possible if the top left entry of $A$ is $0$ to begin with).
The following code defines a function that returns the sign of a single number, counting $0$ as positive.

\begin{lstlisting}
sign = lambda x: 1 if x >= 0 else -1
\end{lstlisting}

\item In steps \ref{step:HH-outer1} and \ref{step:HH-outer2}, the multiplication of $\u$ and $(\u\trp X)$ is an \emph{outer product} ($\x\y\trp$ instead of the usual $\x\trp\y$).
Use \li{np.outer()} instead of \li{np.dot()} to handle this correctly.

\end{itemize}

Use NumPy and SciPy to generate test cases and validate your function.

\begin{lstlisting}
>>> A = np.random.random((5, 3))
>>> Q,R = la.qr(A)                  # Get the full QR decomposition.
>>> print(A.shape, Q.shape, R.shape)
(5,3) (5,5) (5,3)
>>> np.allclose(Q @ R, A)
<<True>>
\end{lstlisting}
\label{prob:qr-via-hessenberg}
\end{problem}

\begin{comment}
\begin{info}
Householder QR factorization is more numerically stable than modified Gram-Schmidt.
However, modified Gram-Schmidt is still useful for some types of iterative methods because it finds the orthonormal basis one vector at a time instead of all at once.
\end{info}
\end{comment}

\section*{Upper Hessenberg Form} % ============================================

An \emph{upper Hessenberg matrix} is a square matrix that is nearly upper triangular, with zeros below the first subdiagonal.
Every $n \times n$ matrix $A$ can be written $A = QHQ\trp$ where $Q$ is orthonormal and $H$, called the \emph{Hessenberg form} of $A$, is an upper Hessenberg matrix.
Putting a matrix in upper Hessenberg form is an important first step to computing its eigenvalues numerically.

This algorithm also uses Householder transformations.
To find orthogonal $Q$ and upper Hessenberg $H$ such that $A = QHQ\trp$, it suffices to find such matrices that satisfy $Q\trp A Q = H$.
Thus, the strategy is to multiply $A$ on the left and right by a series of orthonormal matrices until it is in Hessenberg form.

Using the same $Q_k$ as in the $k$th step of the Householder algorithm introduces $n - k$ zeros in the $k$th column of $A$, but multiplying $Q_k A$ on the right by $Q_k\trp$ destroys all of those zeros.
Instead, choose a $Q_1$ that fixes $\e_1$ and reflects the first column of $A$ into the span of $\e_1$ and $\e_2$.
The product $Q_1A$ then leaves the first row of $A$ alone, and the product $(Q_1A)Q_1\trp$ leaves the first column of $(Q_1A)$ alone.
\[
\begin{array}{ccccc}
\left[\begin{array}{ccccc}
* & * & * & * & * \\
* & * & * & * & * \\
* & * & * & * & * \\
* & * & * & * & * \\
* & * & * & * & *
\end{array}\right]
&\underrightarrow{Q_1}&
\left[\begin{array}{ccccc}
* & * & * & * & * \\
* & * & * & * & * \\
0 & * & * & * & * \\
0 & * & * & * & * \\
0 & * & * & * & *
\end{array}\right]
&\underrightarrow{Q_1\trp }&
\left[\begin{array}{ccccc}
* & * & * & * & * \\
* & * & * & * & * \\
0 & * & * & * & * \\
0 & * & * & * & * \\
0 & * & * & * & *
\end{array}\right]
\\
A & & Q_1A & & (Q_1 A) Q_1\trp
\end{array}
\]
Continuing the process results in the upper Hessenberg form of $A$.
%
\begin{equation*}
Q_3 Q_2 Q_1 A Q_1\trp Q_2 \trp Q_3\trp =
\left[\begin{array}{ccccc}
* & * & * & * & * \\
* & * & * & * & * \\
0 & * & * & * & * \\
0 & 0 & * & * & * \\
0 & 0 & 0 & * & *
\end{array}\right]
\end{equation*}

This implies that $A = Q_1\trp Q_2\trp Q_3\trp H Q_3 Q_2 Q_1$, so setting $Q = Q_1\trp Q_2\trp Q_3\trp$ results in the desired factorization $A = QHQ\trp$.

\subsection*{Constructing the Reflections} % ----------------------------------

Constructing the $Q_k$ uses the same approach as in the Householder algorithm, but shifted down one element.
Let $\x_k = \y_k' + \y_k''$ where $\y_k'$ and $\y_k''$ are of the form
\[
\y_k' = \left[z_1\quad \cdots\quad z_k\quad 0\quad \cdots\quad 0\right]\trp,
\qquad\qquad
\y_k'' = \left[0\quad\cdots\quad 0\quad z_{k+1}\quad\cdots\quad z_m\right]\trp.
\]
Because $\y_k'$ represents elements of $A$ that lie above the first subdiagonal, only $\y_k''$ needs to be altered.
This suggests using the reflection
\begin{gather*}
Q_k =
\left[\begin{array}{cc}
I_k & \0 \\
\0      & H_{\v_k} \\
\end{array}\right] =
\left[\begin{array}{cc}
I_k & \0 \\
\0      & I_{m-k} - 2 \u_k\u_k\trp \\
\end{array}\right],\textrm{ where}
\\
\v_k = \y_k'' + \sign(z_k)\|\y_k''\| \e_k,
\qquad\qquad
\u_k = \frac{\v_k}{\|\v_k\|}.
\end{gather*}

The complete algorithm is given below.
Note how similar it is to Algorithm \ref{Alg:QR-via-Householder}.
% Although the Hessenberg form exists for any square matrix, this algorithm only works for full-rank square matrices.

\begin{algorithm}[H]
\begin{algorithmic}[1]
\Procedure{Hessenberg}{$A$}
    \State $m, n \gets \shape{A}$
    \State $H \gets \makecopy{A}$
    \State $Q \gets I_{m}$
    \For{$k=0 \ldots n-3$}
        \State $\u \gets \makecopy{H_{k+1:,k}}$
        \State $u_0 \gets u_0 + \sign(u_0)\|\u\|$
        \State $\u \gets \u/\|\u\|$
        \State $H_{k+1:,k:} \gets H_{k+1:,k:} - 2\u(\u\trp H_{k+1:,k:})$
            \Comment{Apply $Q_k$ to $H$.}
        \State $H_{:,k+1:} \gets H_{:,k+1:} - 2(H_{:,k+1:} \u) \u\trp$
            \Comment{Apply $Q_k\trp$ to $H$.}
        \State $Q_{k+1:,:} \gets Q_{k+1:,:} - 2\u(\u\trp Q_{k+1:,:})$
            \Comment{Apply $Q_k$ to $Q$.}
    \EndFor
    \State \pseudoli{return} $H, Q\trp$
\EndProcedure
\end{algorithmic}
\caption{}
\label{Alg:Upper-Hessenberg-Form}
\end{algorithm}

\begin{problem} % Reduce a matrix to upper Hessenberg form.
Write a function that accepts a nonsingular $n \times n$ matrix $A$.
Use Algorithm \ref{Alg:Upper-Hessenberg-Form} to compute the upper Hessenberg $H$ and orthogonal $Q$ satisfying $A = QHQ\trp$.

Compare your results to \li{scipy.linalg.hessenberg()}.

\begin{lstlisting}
# Generate a random matrix and get its upper Hessenberg form via SciPy.
>>> A = np.random.random((8,8))
>>> H, Q = la.hessenberg(A, calc_q=True)

# Verify that H has all zeros below the first subdiagonal and QHQ^T = A.
>>> np.allclose(np.triu(H, -1), H)
<<True>>
>>> np.allclose(Q @ H @ Q.T, A)
<<True>>
\end{lstlisting}
\label{prob:hessenberg-via-householder}
\end{problem}

\newpage

\section*{Additional Material} % ==============================================

\subsection*{Complex QR Decomposition} % --------------------------------------

The QR decomposition also exists for matrices with complex entries.
The standard inner product in $\mathbb{R}^m$ is $\langle \x,\y \rangle = \x\trp \y$, but the (more general) standard inner product in $\mathbb{C}^m$ is $\langle \x,\y \rangle = \x\hrm \y$.
The $\hrm$ stands for the \emph{Hermitian conjugate}, the conjugate of the transpose.
Making a few small adjustments in the implementations of Algorithms \ref{Alg:modified-Gram-Schmidt} and \ref{Alg:QR-via-Householder} accounts for using the complex inner product.

\begin{enumerate}

\item Replace any transpose operations with the conjugate of the transpose.

\begin{lstlisting}
>>> A = np.reshape(np.arange(4) + 1j*np.arange(4), (2,2))
>>> print(A)
[[ 0.+0.j  1.+1.j]
 [ 2.+2.j  3.+3.j]]

>>> print(A.T)                      # Regular transpose.
[[ 0.+0.j  2.+2.j]
 [ 1.+1.j  3.+3.j]]

>>> print(A.conj().T)               # Hermitian conjugate.
[[ 0.-0.j  2.-2.j]
 [ 1.-1.j  3.-3.j]]
\end{lstlisting}

\item Conjugate the first entry of vector or matrix multiplication before multiplying with \li{np.dot()}.

\begin{lstlisting}
>>> x = np.arange(2) + 1j*np.arange(2)
>>> print(x)
[ 0.+0.j  1.+1.j]

>>> np.dot(x, x)                    # Standard real inner product.
2j

>>> np.dot(x.conj(), y)             # Standard complex inner product.
(2 + 0j)
\end{lstlisting}

\item In the complex plane, there are infinitely many reflections that map a vector $\x$ into the span of $\e_k$, not just the two displayed in Figure \ref{fig:householder-two-possible-reflectors}.
Using $\sign(z_k)$ to choose one is still a valid method, but it requires updating the \li{sign()} function so that it can handle complex numbers.

\begin{lstlisting}
sign = lambda x: 1 if np.real(x) >= 0 else -1
\end{lstlisting}

\end{enumerate}

\subsection*{QR with Pivoting} % ----------------------------------------------

The LU decomposition can be improved by employing Gaussian elimination with partial pivoting, where the rows of $A$ are strategically permuted at each iteration.
The QR factorization can be similarly improved by permuting the columns of $A$ at each iteration.
The result is the factorization $AP = QR$, where $P$ is a permutation matrix that encodes the column swaps.
To compute the pivoted QR decomposition with \li{scipy.linalg.qr()}, set the keyword \li{pivoting} to \li{True}.

\begin{lstlisting}
# Get the decomposition AP = QR for a random matrix A.
>>> A = np.random.random((8,10))
>>> Q,R,P = la.qr(A, pivoting=True)

# P is returned as a 1-D array that encodes column ordering,
# so A can be reconstructed with fancy indexing.
>>> np.allclose(Q @ R, A[:,P])
<<True>>
\end{lstlisting}

\begin{comment}
\subsection*{Stability of the QR Decomposition in SciPy} % --------------------

SciPy's QR factorization routine uses Householder reflections.
Though this routine is numerically stable, there are still potential numerical problems.
Consider the following example.

\begin{lstlisting}
# Generate a random orthonormal matrix and a random upper-triangular matrix.
>>> Q, _ = la.qr(np.random.normal(size=(500,500)))
>>> R  = np.triu(np.random.normal(size=(500,500)))

# Calculate A = QR, noting that Q and R are the EXACT QR decomposition of A.
>>> A = Q @ R

# Use SciPy to rediscover the QR decomposition of A.
>>> Q1, R1 = la.qr(A)

# Compare the true Q and R to the computed Q1 and R1.
>>> print(la.norm(Q1-Q, <<ord>>=np.inf) / la.norm(Q, <<ord>>=np.inf))
1.21169651649

>>> print(la.norm(R1-R, <<ord>>=np.inf) / la.norm(R, <<ord>>=np.inf))
1.72312747194
\end{lstlisting}

This is terrible!
This algorithm works in $16$ decimal points of precision, but $Q_1$ and $R_1$ are accurate to $0$ decimal points.
These errors in $Q_1$ and $R_1$ are called \emph{forward errors}.
Even so, miraculously, the product $Q_1 R_1$ is very close $A$.

\begin{lstlisting}
>>> A1 = Q1 @ R1
>>> la.norm(A1 - A, <<ord>>=np.inf) / la.norm(A, <<ord>>=np.inf)
3.353843164496928e-15
\end{lstlisting}

The error in $A_1$, called the \emph{backward error}, is very small.
This shows that the errors in $Q_1$ and $R_1$ are somehow ``correlated,'' so that they cancel out in the product.
In fact, the large errors in $Q_1$ and $R_1$ are not because the algorithm is bad, but rather because $A$ was \emph{poorly conditioned} to begin with.
Conditioning and stability will be discussed more in depth later in the curriculum.
\end{comment}

\subsection*{QR via Givens} % -------------------------------------------------

The Householder algorithm uses reflections to triangularize $A$.
However, $A$ can also be made upper triangular using rotations.
To illustrate the idea, recall that the matrix for a counterclockwise rotation of $\theta$ radians is given by
\[
R_\theta =
\left[\begin{array}{cc}
\cos\theta & -\sin\theta \\
\sin\theta &  \cos\theta
\end{array}\right].
\]

This transformation is orthonormal.
Given $\x = \left[a, b\right]\trp$, if $\theta$ is the angle between $\x$ and $\e_1$, then $R_{-\theta}$ maps $\x$ to the span of $\e_1$.

\begin{figure}[H]
\centering
\begin{tikzpicture}
    \draw[-, thick](-.5,0)--(3,0);
    \draw[-, thick](0,-.5)--(0,1.5);
    \draw[->, thick, >=stealth'](0,0)--(2.5,1);
    \draw[-,thick](2.5,-.2)--(2.5,.2);
    \draw[-, thick](-.2,1)--(.2,1);
    \node[draw=none](point_b)at(2.5,-.4){$a$};
    \node[draw=none](point_a)at(-.4,1){$b$};
    \draw[-, thick] (.5,.2)arc [start angle=60, end angle=-20, radius=4.5pt];
    \node[draw=none](theta)at(.85, .18){$\theta$};
\end{tikzpicture}
\caption{Rotating clockwise by $\theta$ sends the vector $\left[a,b\right]\trp$ to the span of $\e_1$.}
\label{fig:Givens-rotation}
\end{figure}

In terms of $a$ and $b$, $\cos \theta =  \frac{a}{\sqrt{a^2+b^2}}$ and $\sin\theta = \frac{b}{\sqrt{a^2+b^2}}$.
Therefore,
\[
R_{-\theta}\x
=
\left[\begin{array}{cc}
\cos\theta  & \sin\theta \\
-\sin\theta & \cos\theta \\
\end{array}\right]
\left[\begin{array}{c} a \\ b \end{array}\right]
=
\left[\begin{array}{cc}
 \frac{a}{\sqrt{a^2+b^2}} & \frac{b}{\sqrt{a^2+b^2}} \\ \\
-\frac{b}{\sqrt{a^2+b^2}} & \frac{a}{\sqrt{a^2+b^2}} \\
\end{array}\right]
\left[\begin{array}{c} a \\ b \end{array}\right]
=
\left[\begin{array}{c} \sqrt{a^2+b^2} \\ 0 \end{array}\right].
\]

The matrix $R_{\theta}$ above is an example of a $2 \times 2$ \emph{Givens rotation matrix}.
In general, the Givens matrix $G(i,j,\theta)$ represents the orthonormal transformation that rotates the 2-dimensional span of $\e_i$ and $\e_j$ by $\theta$ radians.
The matrix representation of this transformation is a generalization of $R_\theta$.
\[
G(i,j,\theta) =
\left[\begin{array}{ccccc}
I & 0 & 0 & 0 & 0 \\
0 & c & 0 & -s & 0 \\
0 & 0 & I & 0 & 0 \\
0 & s & 0 & c & 0 \\
0 & 0 & 0 & 0 & I
\end{array}\right]
\]

Here $I$ represents the identity matrix, $c=\cos \theta$, and $s=\sin \theta$.
The $c$'s appear on the $i$th and $j$th diagonal entries.

\subsubsection*{Givens Triangularization} % - - - - - - - - - - - - - - - - - -

As demonstrated, $\theta$ can be chosen such that $G(i,j,\theta)$ rotates a vector so that its $j$th-component is 0.
Such a transformation will only affect the $i$th and $j$th entries of any vector it acts on (and thus the $i$th and $j$th rows of any matrix it acts on).

To compute the QR decomposition of $A$, iterate through the subdiagonal entries of $A$ in the order depicted by Figure \ref{fig:Givens-iteration-order}.
Zero out the $ij$th entry with a rotation in the plane spanned by $\e_{i-1}$ and $\e_i$, represented by the Givens matrix $G(i-1,i,\theta)$.

\begin{figure}[H]
\centering
\begin{tikzpicture}[xscale=.7, yscale=.7]
    \draw[->, gray, thick](0,0)--(0,5.5);
    \draw[->, gray, thick](1,0)--(1,4.5);
    \draw[->, gray, thick](2,0)--(2,3.5);
    \draw[->, gray, thick](3,0)--(3,2.5);
    \draw[->, gray, thick](4,0)--(4,1.5);
    \node[draw=none] at(0,-1){\large 1};
    \node[draw=none] at(1,-1){\large 2};
    \node[draw=none] at(2,-1){\large 3};
    \node[draw=none] at(3,-1){\large 4};
    \node[draw=none] at(4,-1){\large 5};

    \draw[-, ultra thick] (0,6)--(4,2);

    \draw[-, thick](-1,-.5)--(-1,6.5);
    \draw[-, thick](-1, -.5)--(-.5,-.5);
    \draw[-, thick](-1,6.5)--(-.5,6.5);

    \draw[-, thick](5,-.5)--(5,6.5);
    \draw[-, thick](5,-.5)--(4.5,-.5);
    \draw[-, thick](5,6.5)--(4.5,6.5);
\end{tikzpicture}
\caption{The order in which to zero out subdiagonal entries in the Givens triangularization algorithm.
The heavy black line is the main diagonal of the matrix.
Entries should be zeroed out from bottom to top in each column, beginning with the leftmost column.}
\label{fig:Givens-iteration-order}
\end{figure}

On a $2 \times 3$ matrix, the process can be visualized as follows.
\[
\left[\begin{array}{cc}
* & * \\
* & * \\
* & * \\
\end{array}\right]
\underrightarrow{G(2,3,\theta_1)}
\left[\begin{array}{cccc}
&      *  &      *  & \\
\cline{2-3}
& \lvl{*} & \rvl{*} & \\
& \lvl{0} & \rvl{*} & \\
\cline{2-3}
\end{array}\right]
\underrightarrow{G(1,2,\theta_2)}
\left[\begin{array}{cccc}
\cline{2-3}
& \lvl{*} & \rvl{*} & \\
& \lvl{0} & \rvl{*} & \\
\cline{2-3}
&      0  &      *  & \\
\end{array}\right]
\underrightarrow{G(2,3,\theta_3)}
\left[\begin{array}{ccc}
     * &       *  & \\
\cline{2-2}
\rvl{0} & \rvl{*} & \\
\rvl{0} & \rvl{0} & \\
\cline{2-2}
\end{array}\right]
\]
At each stage, the boxed entries are those modified by the previous transformation.
The final transformation $G(2,3,\theta_3)$ operates on the bottom two rows, but since the first two entries are zero, they are unaffected.

Assuming that at the $ij$th stage of the algorithm $a_{ij}$ is nonzero, Algorithm \ref{Alg:QR-via-Givens} computes the Givens triangularization of a matrix.
Notice that the algorithm does not actually form the entire matrices $G(i,j,\theta)$; instead, it modifies only those entries of the matrix that are affected by the transformation.

\begin{algorithm}[H]
\begin{algorithmic}[1]
\Procedure{Givens Triangularization}{$A$}
\State $m, n \gets \shape{A}$
\State $R \gets \makecopy{A}$
\State $Q \gets I_{m}$
\For{$j=0\ldots n-1$}
    \For{$i=m-1\ldots j+1$}
      \State $a, b \gets R_{i-1,j}, R_{i,j}$
      \State $G \gets [[a, b],[-b,a]]/\sqrt{a^2+b^2}$
      \State $R_{i-1:i+1,j:} \gets GR_{i-1:i+1, j:}$
      \State $Q_{i-1:i+1,:} \gets GQ_{i-1:i+1,:}$
    \EndFor
\EndFor
\State \pseudoli{return} $Q\trp , R$
\EndProcedure
\end{algorithmic}
\caption{}
\label{Alg:QR-via-Givens}
\end{algorithm}

\begin{comment}
\begin{info} % Advantages and disadvantages of Givens
Like Householder transformations, Givens rotations are orthonormal, and therefore numerically stable.
In addition, they affect only a small part of the array at each iteration, making them ideal for some problems.

In general, however, Algorithm \ref{Alg:QR-via-Givens} requires more floating point operations than Algorithm \ref{Alg:QR-via-Householder}.
On the other hand, the Givens algorithm can be \emph{parallelized}, meaning that multiple processors can be used to simultaneously carry out different iterations of the algorithm.
\end{info}
\end{comment}

\subsubsection*{QR of a Hessenberg Matrix via Givens} % - - - - - - - - - - - -

The Givens algorithm is particularly efficient for computing the QR decomposition of a matrix that is already in upper Hessenberg form, since only the first subdiagonal needs to be zeroed out.
Algorithm \ref{Alg:QR-of-Hessenberg-via-Givens} details this process.

\begin{algorithm}[H]
\begin{algorithmic}[1]
\Procedure{Givens Triangularization of Hessenberg}{$H$}
\State $m, n \gets \shape{H}$
\State $R \gets \makecopy{H}$
\State $Q \gets I_{m}$
\For{$j=0\ldots \min\{n-1, m-1\}$}
    \State $i = j+1$
    \State $a, b \gets R_{i-1,j}, R_{i,j}$
    \State $G \gets [[a, b],[-b,a]]/\sqrt{a^2+b^2}$
    \State $R_{i-1:i+1,j:} \gets GR_{i-1:i+1, j:}$
    \State $Q_{i-1:i+1,:i+1} \gets GQ_{i-1:i+1,:i+1}$
\EndFor
\State \pseudoli{return} $Q\trp , R$
\EndProcedure
\end{algorithmic}
\caption{}
\label{Alg:QR-of-Hessenberg-via-Givens}
\end{algorithm}

\begin{info} % Hessenberg of Symmetric -> Tridiagonal.
When $A$ is symmetric, its upper Hessenberg form is a \emph{tridiagonal} matrix, meaning its only nonzero entries are on the main diagonal, the first subdiagonal, and the first superdiagonal.
This is because the $Q_k$'s zero out everything below the first subdiagonal of $A$ and the $Q_k\trp$'s zero out everything to the right of the first superdiagonal.
Tridiagonal matrices make computations fast, so computing the Hessenberg form of a symmetric matrix is very useful.
\end{info}


\documentclass{article}

\usepackage{amsmath}
\usepackage{graphicx}
\usepackage[colorlinks]{hyperref}

\newcommand{\vx}[0]{\mathbf{x}}

\author{Mika Illouz}
\title{EigenDog's \texttt{dawg}}

\begin{document}

\maketitle

EigenDog's implementation of {\bf Gradient Boosting Trees}, called
\texttt{dawg} follows the mathematical description in Friedman's 1999
paper \cite{GBM}.

Following Friedman's notation, our training set is represented by
$(y_i, \vx_i)$ where $i = 1 \ldots n$: We have $n$ records, each
record $i$ consists of an outcome (aka response) $y_i$ and feature
vector $\vx_i$.  In the general \texttt{dawg} formulation the objective is to
minimize the loss function $Q$:
%
\begin{equation}
\label{objective}
Q = \sum_{i=1}^{n} L(y_i, F(\vx_i))
\end{equation}
%
In this context context, $y \in \{ -1, 1 \}$ and
%
\[
L(y,F) = \log ( 1 + \exp( -2 y F ) )
\]
$L$ is know as the {\bf deviance} or {\bf cross-entropy.}  $F$ is the
object we want to construct.  The probability of classifications are:
\begin{equation}
\begin{split}
p(y =1|\vx) &= \frac{1}{1 + \exp(-2 F(\vx) )} \\
p(y =-1|\vx) &= 1 - p(y=1|\vx) = \frac{1}{1 + \exp(2 F(\vx) )} \\
\end{split}
\end{equation}

Some intuition about function $L(y,F)$:
first, $\log( 1 + \exp (\bullet) )$ is always positive, since
$\exp(\bullet) \ge 0$.  When $y=1$ and $F \rightarrow \infty$, the
quantity $\exp ( -2 y F ) \rightarrow 0$ and $L(y, F) \rightarrow 0$.
So, to drive $L(y, F)$ closer to zero (which is its floor) when
$y_i=1$, we want to increase $F$.  Similarly, when $y=-1$, decreasing
$F \rightarrow -\infty$ drives $L(y, F)$ closer to zero.  In either
case, increasing the magnitude of $F$ in the same direction as $y$
reduces $L(y,F)$.  Similarly, when $y$ and $F$ have opposite signs and
$| F | \rightarrow \infty$ then $L(y,F) \rightarrow \infty$.

In \texttt{dawg}, the function $F(\vx)$ is a sum of $K+1$ decision trees:
\[
F(\vx) = \sum_{k=0}^K F_k(\vx)
\]
More precisely, each $F_k(\vx)$ is a binary decision trees, so that
each of its non-leaf nodes has two child trees, called the {\bf left}
and {\bf right} sub-trees.  A decision tree is a piecewise constant
function, where a leaf node represents one of the constant pieces, and
a non-leaf node represent a decision over the value of a single
feature $\vx^j$.  (Notation: whereas $\vx_i$ is the $i$-th row of the
training set, $\vx^j$ is the $j$-th feature/column).

The ``boosting'' in \texttt{dawg} refers to the fact that in each iteration $k$,
a new decision tree is constructed that improves upon the previously
constructed function defined by the sequence of trees $F_0, \ldots,
F_{k-1}$.  The essential (and inductive) question in \texttt{dawg} is: having
constructed these trees, what ought the {\em next} tree be?

\paragraph{Feature Splitting}

To build the next tree, we must partition the population of records
into two subsets.  To build deeper trees, we must recursively
partition each subset.  With each recursion, we must answer the
question: were we to stop recursion here -- and therefore build a pair
of leaf nodes -- what would be their constant values?  We denote the
leaf's constant value as $\gamma$, with $\gamma_L$ being the left
value, and $\gamma_R$ being the right one.  Over a subset of records
$S$, we want to choose a value for $\gamma$ so that the loss function
in equation [\ref{objective}] is minimized.  For each record $i$ in
$S$, $\gamma$ would be added to sum of the all previously constructed
trees $F_k(\vx_i)$.  So our goal is to find a $\gamma^*$:
%
\[
\gamma^* = \arg \min_\gamma \sum_{i \in S} L
  \bigg[
    y_i, \bigg\{ \sum_{k=0}^K F_k(\vx_i) \bigg\} + \gamma
  \bigg]
\]
To simplify the presentation, we'll define $F_i^K \equiv \sum_{k=0}^K
F_k(\vx_i)$, which is the value of the record $i$ evaluated over the
sum of all the decision trees constructed thus far:
%
\[
\gamma^* = \arg \min_\gamma \sum_{i \in S} L
  \bigg[ y_i, F_i^K + \gamma \bigg]
\]
%
Finding the best $\gamma$, $\gamma^*$, is univariate minimization
problem because $y_i$ and $F_i^K$ are all fixed.  Furthermore, it
turns out that we don't need the optimal $\gamma^*$, just a decent
approximation $\hat \gamma \approx \gamma^*$.  In his paper, Friedman
declares (without giving proof) that this approximation is:
\[
\hat \gamma =
  \frac { \sum_{i \in S} z_i } { \sum_{i \in S} | z_i | ( 2 - | z_i | ) }
\]
where
\[
z_i \equiv \frac{ 2 y_i } { 1 + \exp [ 2 y_i F_i^K ] }
\]
are called the {\em pseudo-response}.

\paragraph{Proof}

The proof is as follows: we model $Q(\gamma)$ using a local quadratic
approximation defined by the Taylor series expansion.  The textbook
definition of a Taylor series expansion centered at zero (actually
known as the Maclaurin series) is:
\[
f(x) = f(0) + f'(0)x + f''(0)x^2/2! + \cdots
\]
Applying this to $Q(\gamma)$, we have:
\begin{equation*}
Q(\gamma) = Q(0) + Q'(0)\gamma + Q''(0)\gamma^2/2! + \cdots \\
\end{equation*}
Then dropping the higher-order terms, we have the quadratic
approximation:
\begin{equation}
Q(\gamma) \approx Q(0) + Q'(0)\gamma + Q''(0)\gamma^2/2! \\
\label{Q-quad}
\end{equation}

Since we are no longer minimizing $Q(\gamma)$ exactly, but rather an
approximation of that function, the resulting $\hat \gamma$ will be
also be an approximation of the true minimizer $\gamma^*$.  The
minimizer of $Q(\gamma)$ must satisfy $Q'(\hat \gamma)=0$, and, taking
this first derivative results in the condition $Q'(0) + Q''(0)\hat
\gamma = 0$ or
\[
\hat \gamma = -\frac{ Q'(0) }{ Q''(0) }
\]
This approximation might initially seem crude. However, one must view
this in the context of the broader \texttt{dawg} algorithm, in which each new
decision tree addresses the shortcomings of the previous ones.
Therefore, a crude selection of $\gamma$ in one branch of one tree is
corrected in the tree-building iterations that follow it.  In fact,
since the \texttt{dawg} model improves with each new tree, we expect the leaf
values necessary to make further improvements to shrink in magnitude.
In other words, as $k \rightarrow \infty$, $\gamma \rightarrow 0$.

We now derive expressions for $Q'(0)$ and $Q''(0)$:

\begin{equation*}
\begin{split}
Q'(\gamma) &= \frac {\partial Q}{\partial \gamma} \\
&= \frac{\partial}{\partial \gamma} \sum_{i \in S} L \bigg[ y_i, F_i^K + \gamma \bigg] \\
&= \frac{\partial}{\partial \gamma} \sum_{i \in S} \log \big( 1 + \exp [ -2 y_i (F_i^K + \gamma) ] \big) \\
&= \sum_{i \in S} \frac{\partial}{\partial \gamma} \log \big( 1 + \exp [ -2 y_i (F_i^K + \gamma) ] \big) \\
&= \sum_{i \in S} \big( 1 + \exp [ -2 y_i (F_i^K + \gamma) ] \big)^{-1} \frac{\partial}{\partial \gamma} \big( 1 + \exp [ -2 y_i (F_i^K + \gamma) ] \big) \\
&= \sum_{i \in S} \big( 1 + \exp [ -2 y_i (F_i^K + \gamma) ] \big)^{-1}  \exp [ -2 y_i (F_i^K + \gamma) ] \big( -2 y_i \big) \\
&= \sum_{i \in S} \frac{  \big( -2 y_i \big) \exp [ -2 y_i (F_i^K + \gamma) ] } { 1 + \exp [ -2 y_i (F_i^K + \gamma) ] } \\
&= \sum_{i \in S} \frac{  -2 y_i \exp [ -2 y_i (F_i^K + \gamma) ] } { 1 + \exp [ -2 y_i (F_i^K + \gamma) ] } \times
   \frac { \exp [ 2 y_i (F_i^K + \gamma) ] } { \exp [ 2 y_i (F_i^K + \gamma) ] } \\
&= \sum_{i \in S} \frac{ -2 y_i } { \exp [ 2 y_i (F_i^K + \gamma) ] + 1 } \\
\end{split}
\end{equation*}

Now the second derivative:

\begin{equation*}
\begin{split}
Q''(\gamma) &= \frac {\partial Q'}{\partial \gamma} \\
&= \frac {\partial}{\partial \gamma} \sum_{i \in S} \frac{ -2 y_i } { \exp [ 2 y_i (F_i^K + \gamma) ] + 1 } \\
&= \sum_{i \in S} (-2 y_i ) \frac {\partial}{\partial \gamma} \bigg( \exp [ 2 y_i (F_i^K + \gamma) ] + 1 \bigg)^{-1} \\
&= \sum_{i \in S} (-2 y_i ) (-1) \bigg( \exp [ 2 y_i (F_i^K + \gamma) ] + 1 \bigg)^{-2} \frac {\partial}{\partial \gamma} \bigg( \exp [ 2 y_i (F_i^K + \gamma) ] + 1 \bigg) \\
&= \sum_{i \in S} (-2 y_i ) (-1) \bigg( \exp [ 2 y_i (F_i^K + \gamma) ] + 1 \bigg)^{-2} \exp [ 2 y_i (F_i^K + \gamma) ] ( 2 y_i ) \\
&= \sum_{i \in S} \frac{ (2 y_i)^2 } { \bigg( \exp [ 2 y_i (F_i^K + \gamma) ] + 1 \bigg)^2 } \times \exp [ 2 y_i (F_i^K + \gamma) ] \\
\end{split}
\end{equation*}

Since we are interested in $Q'(0)$, $Q''(0)$, we now simplify with
$\gamma=0$:
\begin{equation*}
\begin{split}
Q'(0) &= \sum_{i \in S} \frac{ -2 y_i } { \exp [ 2 y_i F_i^K ] + 1 } = - \sum_{i \in S} \frac{ 2 y_i } { \exp [ 2 y_i F_i^K ] + 1 } \\
Q''(0) &= \sum_{i \in S} \frac{ (2 y_i)^2 } { \bigg( \exp [ 2 y_i F_i^K ] + 1 \bigg)^2 } \times \exp [ 2 y_i F_i^K ] \\
\end{split}
\end{equation*}
Then, conveniently defining
\begin{equation*}
z_i \equiv \frac{ 2 y_i } { 1 + \exp [ 2 y_i F_i^K ] }
\end{equation*}
we get
\begin{equation*}
\begin{split}
Q'(0) &= - \sum_{i \in S} z_i \\
Q''(0) &= \sum_{i \in S} z_i^2 \exp [ 2 y_i F_i^K ] \\
\end{split}
\end{equation*}
Note from the definition of $z_i$ that $2 y_i = z_i ( 1 + \exp [ 2 y_i
  F_i^K ] )$ and therefore $\exp [ 2 y_i F_i^K ] = 2y_i/z_i - 1 =
(2y_i - z_i)/z_i$.  So,
\begin{equation*}
Q''(0) = \sum_{i \in S} z_i^2 \frac{ 2 y_i - z_i }{ z_i } = \sum_{i \in S} z_i ( 2 y_i - z_i ) = \sum_{i \in S} ( 2 y_i z_i - z_i^2 )
\end{equation*}
Also note that
\begin{equation*}
y_i z_i = y_i \frac{ 2 y_i } { 1 + \exp [ 2 y_i F_i^K ] } = \frac{ 2 y_i^2 } { 1 + \exp [ 2 y_i F_i^K ] }
\end{equation*}
However, since $y_i$ is always either $1$ or $-1$, $y_i^2 = 1$, and so
$y_i z_i = | z_i |$.  Substituting this into $Q''(0)$, we have
\begin{equation*}
Q''(0) = \sum_{i \in S} ( 2 | z_i | - z_i^2 ) = \sum_{i \in S} ( 2 | z_i | - | z_i | ^2 ) = \sum_{i \in S} | z_i | ( 2 - | z_i | )
\end{equation*}
Finally, we can confirm Friedman's approximation for the optimal $\gamma$:
\begin{equation*}
\hat \gamma
  = - \frac{ Q'(0) }{ Q''(0) }
  = - \frac{ - \sum_{i \in S} z_i }{ \sum_{i \in S} | z_i | ( 2 - | z_i | ) }
  = \frac{ \sum_{i \in S} z_i }{ \sum_{i \in S} | z_i | ( 2 - | z_i | ) }
\end{equation*}

\paragraph{Optimal Loss}

I'll define $N$ and $D$ denoting numerator and denominator:
\begin{equation*}
\begin{split}
N(S) &= \sum_{i \in S} z_i \\
D(S) &= \sum_{i \in S} | z_i | ( 2 - | z_i | ) \\
\end{split}
\end{equation*}
and then $\hat \gamma = N(S)/D(S)$.

The non-leaf nodes in the tree we are constructing are labeled with
the identity of a feature $x^j$ and a rule for deciding whether
evaluation should proceed to the left or right sub-tree.  For features
that are ordered, this decision takes the form of $x^j \le \alpha$
(where $\alpha =$ is also known as the {\bf split}.  A split
partitions a population of records into two subsets $S_L$ and its
complement $S_R$.  For each subset, we can compute $\gamma_L$ and
$gamma_R$ resp., which would reveal the leaf values for a branch of
the decision tree.  we could also determine what the loss of
population of records would be under the split:
\[
Q(\hat \gamma) = Q_L( \hat \gamma_L) + Q_R( \hat \gamma_R ) =
\sum_{i \in S_L} L [ y_i, F_i^K + \gamma_L ] +
\sum_{i \in S_R} L [ y_i, F_i^K + \gamma_R ]
\]
A feature with $M$ distinct values has $M-1$ candidate splits.  By
evaluating $Q(\hat \gamma)$ for each the candidate splits (and
correspondingly, the partition $S_L$ and $S_R$ that would result from
the split), we can identify a split associated with the minimum loss
$Q$.  Similarly, determining the optimum split and corresponding loss
of each feature in the training set, we can identify the feature
associated with the minimum loss.  Exhaustively determining $\gamma$
and the resulting loss for each candidate split and for each feature
is the key computation in \texttt{dawg}.

Computing $Q(\hat \gamma)$ exactly is expensive; fortunately our local
quadratic approximation \ref{Q-quad} is already available:
\begin{equation*}
\begin{split}
Q(\hat \gamma) &\approx
  Q(0) +
  \hat \gamma ( - N ) +
  \frac{1}{2} \hat \gamma^2 ( D ) \\
&= Q(0) + \frac{N}{D} (-N) +   \frac{1}{2} \big( \frac{N}{D} \big)^2 ( D ) \\
&= Q(0) - \frac{N^2}{D} + \frac{1}{2} \frac{N^2}{D} \\
&= Q(0) - \frac{1}{2} \frac{N^2}{D}
\end{split}
\end{equation*}

\paragraph{Ordinal Features}

For an ordered feature, we can efficiently compute $\hat \gamma$ and
loss $Q(\hat \gamma)$ for all possible splits.  For example, the 11-th
feature $\vx^{11}$ of a training set consisting of 7 records is
$\vx^{11} = [5,1000,21,5,1000,5,5]^T$.  $\vx^{11}$ has three distinct
values $\{5,21,1000\}$, and correspondingly, two possible splits:
$x^{11} \le 5$ and $x^{11} \le 21$.  We want to determine which of
these splits would result in lower loss.  Assuming $Q(0)$ is evaluated
identically for all features, We now need to compute $N_{\{i| x^{11}
  \le 5\}}$, $N_{\{i| x^{11} \le 21\}}$, $D_{\{i| x^{11} \le 5\}}$,
$D_{\{i| x^{11} \le 21\}}$.

\paragraph{Deep Trees}

A decision tree consisting of one root node and two leaves is called a
{\bf stump}.  Typically, we want to grow deeper decision trees, as
this will better capture interaction-effects among features.  To build
sub-trees, we simply capture the splitting feature and value and apply
the same tree-building logic on the partition of records defined by
the split.  When only terminate the tree with leaves once the depth of
the tree has reached a user-specified limit.

We must maintain the partition (of partition of ...) over which a best
split is to be searched.  For trees whose depth is greater than one
(non-stumps), the ancestor split along with its feature data is
necessary in order to assign a record to a descendant partition.

\paragraph{Categorical Features}

Categorical features are defined by some finite number of labels.  For
example, the feature {\em nationality} might have three categories
(aka class labels) \texttt{French}, \texttt{American}, and
\texttt{Panamanian}.  The categories of such a feature have no natural
ordering.  A decision rule for a categorical feature $x^j$ takes the
form $x^j \in C_L$ where $C_L$ is a subset of the categories of that
feature.  For example, \texttt{if $x \in$ \{French, American\} go left
  else if $x \in$ \{Panamanian\} go right else raise NoSuchCategory}.
In this example, $C_L =$ \texttt{\{French, American\}} and $C_R =$
\texttt{\{Panamanian\}}.  For a categorical feature with $B$
categories, there are $2^B - 2$ ways to partition these into two sets.
A naive approach would be to evaluate every partition, and pick the
one associated with the smallest loss.  However, for even a modest
$B$, this can be quite expensive.  Fortunately, a heuristic which
imposes an order on the categories exists; with this ordering,
computing $\hat \gamma$'s and associated loss is nearly identical to
that of an ordinal features.

In the context of non-boosted trees, the idea is use the response $y$
determine the order -- after all, we want to use the feature to
discriminate between $y=1$ and $y=-1$.  Stated differently, in an
ideal partition, the response on one side will be uniformly $1$ and on
the other uniformly $-1$.

In the context boosted trees, we do not care about the response $y_i
\in {-1,1}$ but rather the pseudo-response $z_i \in \Re$, because the
latter reflects that which has not yet been explained by previous
trees.  Why does Friedman call
\[
z_i \equiv \frac{ 2 y_i } { 1 + \exp [ 2 y_i F_i^K ] }
\]
the pseudo-response?  First note that $z_i$ is some $y_i$ scaled by
the positive quantity $2/( 1 + \exp [ 2 y_i F_i^K ] )$.  we stated
earlier that when $y_i$ and $F_i^K$ have the same sign, then the
component of the loss $L(y_i,F_i^K)$ due to record $i$ is low (by
design) and that record is considered to be well explained.  In that
case, as $|F_i^K| \rightarrow \infty$, $\exp [ 2 y_i F_i^K ]
\rightarrow \infty$, and $z_i \rightarrow 0$.  Conversely for poorly
explained records, $y_i$ and $F_i^K$ have opposite signs, $\exp [ 2
  y_i F_i^K ] \rightarrow 0$ and $z_i \rightarrow 2 y_i$.

By ordering the categories according to the average of $z_i$ in each
category, that is
\[
\frac{1}{\text{card}(S)} N \equiv \frac{1}{\text{card}(S)} \sum_{i \in S} z_i
\]
We can find the split among the categories which clusters the
unexplained responses of each type ${-1,1}$.

\paragraph{Shrinkage and Stochastic Boosting}

In his follow-on paper \cite{SGB}, Friedman improved tree-boosting in
two ways.  The first idea is to train each tree (produced in a single
iteration) on a different, random subset of the training set ({\bf
stochastic}).  The second idea is to shrink the leaf values by some
scalar $\alpha < 1$.

The stochastic idea is inspired by the common practice in statistics
of setting aside data for model validation.  Predictive modeling
algorithms can easily {\bf overfit}, by which we mean that they pay
too much attention to the records in the training set, and in the
process degrade their ability to perform well on data that they have
not seen.  So, using the same set of records for both model assessment
and development often leads to a very optimistic view of a model's
performance.  A validation set consisting of records that are hidden
during model development can provide a more realistic view of the
model's performance.  In tree boosting, training on a different random
subset of records is intended to mitigate overfitting.  Experience
shows that it has the intended effect.

The second idea is to shrink in magnitude the value of each tree (that
is, replace the $\hat \gamma$'s that are at the leaf of every tree
with a scaled version, $\alpha \hat \gamma$ where $0 < \alpha < 1$.
This is inspired by common practice in optimization algorithms, where
a {\bf step}, intended to bring the algorithm closer to the solution,
is scaled back.  In \texttt{dawg}, a step is a tree.  The impact of
scaling each tree with the shrinkage factor $\alpha$ (also known as
the {\bf learning rate}) is to (a) slow down the algorithm (b) give
opportunities for a wider set of features to be included in the model,
resulting in a superior model (c) promote convergence.

It has been shown, both in academic papers, and by applying
\texttt{dawg} to various data sets, that shrinkage and stochastic
boosting used in combination leads to higher quality models.  These
runtime costs are significant, and the code necessary to implement
these features is more complex.

The point about convergence deserves some elaboration.  It is possible
that for some learning rate $\alpha \le 1$ the loss $Q \rightarrow
\infty$.  This happens typically because $L(y_i, F_i^K) \rightarrow
\infty$ for some record $i$ which was held out during the last
iteration $K$.  \texttt{dawg} detects to this situation by restarting
learning with a rate half the original one.

\paragraph{Algorithm Termination}

Although shrinkage and stochastic boosting reduce the tendency for
\texttt{dawg} to overfit, they do not eliminate that tendency.  In an
ideal world without overfitting, \texttt{dawg}'s loss function would
simply stop decreasing with each iteration $k$, defining a convenient
termination point.  However, with enough diverse features, it seems
possible to drive the loss arbitrarily close to zero.  This raises the
question: for a given learning rate, what is the optimal number of
trees (iterations) ?  To answer this question, \texttt{dawg} performs
k-fold cross-validation: it trains internally on $(k-1)$ folds, and
monitors the loss on the remaining ``validation'' fold.  Once the loss
on this fold starts increasing, \texttt{dawg} stops learning.
\texttt{dawg} constructs $k$ such sub-models, and averages them in a
final model.

During the validation phase, we can speak of several sets of records.
The training set is the union of the validation set and the working
set.  In each iteration of the validation phase, the working set is
randomly partitioned in two.  One partition is used for building a
tree, and the other partition is unused.  The partition of records
which are used are further portioned as splits are discovered.

\begin{thebibliography}{5}
\bibitem{GBM}Friedman, Jerome H.  Greedy Function Approximation: A
  Gradient Boosting machine, 1999;
  \url{//http://www-stat.stanford.edu/~jhf/ftp/trebst.pdf}
\bibitem{SGB}Friedman, Jerome H.  Stochastic Gradient Boosting, 1999;
  \url{http://www-stat.stanford.edu/~jhf/ftp/stobst.pdf}
\bibitem{ARFF} \url{http://www.cs.waikato.ac.nz/~ml/weka/arff.html}
\end{thebibliography}

\end{document}


\section*{Exercises}

\begin{ex}
  Find $-7\begin{mymatrix}{r}
    6 \\
    0 \\
    4 \\
    -1
  \end{mymatrix}
  + 6\begin{mymatrix}{r}
    -13 \\
    -1 \\
    1 \\
    6
  \end{mymatrix}$.
  % \begin{sol}
  % \end{sol}
\end{ex}

\begin{ex}
  Decide whether
  \begin{equation*}
    \vect{v}= \begin{mymatrix}{r}
      4 \\
      4 \\
      -3
    \end{mymatrix}
  \end{equation*}
  is a linear combination of the vectors
  \begin{equation*}
    \vect{u}_1 = \begin{mymatrix}{r}
      3 \\
      1 \\
      -1
    \end{mymatrix}
    \quad\mbox{and}\quad
    \vect{u}_2 =
    \begin{mymatrix}{r}
      2 \\
      -2 \\
      1
    \end{mymatrix}.
  \end{equation*}
  If yes, find the coefficients.

  \begin{sol}
    \begin{equation*}
      \begin{mymatrix}{r}
        4 \\
        4 \\
        -3
      \end{mymatrix}
      =
      2
      \begin{mymatrix}{r}
        3 \\
        1 \\
        -1
      \end{mymatrix}
      -
      \begin{mymatrix}{r}
        2 \\
        -2 \\
        1
      \end{mymatrix}.
    \end{equation*}
  \end{sol}
\end{ex}

\begin{ex}
  Decide whether
  \begin{equation*}
    \vect{v}= \begin{mymatrix}{r}
      4 \\
      4 \\
      4
    \end{mymatrix}
  \end{equation*}
  is a linear combination of the vectors
  \begin{equation*}
    \vect{u}_1 = \begin{mymatrix}{r}
      3 \\
      1 \\
      -1
    \end{mymatrix},
    \quad
    \vect{u}_2 =
    \begin{mymatrix}{r}
      8 \\
      0 \\
      -1
    \end{mymatrix}
    \quad\mbox{and}\quad
    \vect{u}_3 =
    \begin{mymatrix}{r}
      2 \\
      -2 \\
      1
    \end{mymatrix}.
  \end{equation*}
  If yes, find the coefficients.

  \begin{sol}
    The system
    \begin{equation*}
      \begin{mymatrix}{r}
        4 \\
        4 \\
        4
      \end{mymatrix}
      =
      a_1
      \begin{mymatrix}{r}
        3 \\
        1 \\
        -1
      \end{mymatrix}
      + a_2 \begin{mymatrix}{r}
        8 \\
        0 \\
        -1
      \end{mymatrix}
      + a_3
      \begin{mymatrix}{r}
        2 \\
        -2 \\
        1
      \end{mymatrix}
    \end{equation*}
    has no solution.
  \end{sol}
\end{ex}


%!TEX root = ../notes.tex
\section{February 25, 2022}
\recall Midterm discrete log problem where
\[x\to x^e\mod p\]
and a solution we gave was take $(x^e)^p\equiv x$ where $de\equiv 1\mod (p-1)$.

What if we didn't take this mod $p$, but instead took it mod $pq$.

We can take the analog of Fermat's Little Theorem mod $pq$, where
\[a^{(p-1)(q-1)}\equiv 1\mod pq\]

What if we did the same thing, instead of taking inverse mod $p-1$, we took it mod $(p-1)(q-1)$ to extract $e^\mathrm{th}$ roots if we know $(p-1)(q-1)$.

We know $p$ and $q$, we can easily figure out $(p-1)(q-1)$. Also, if we know $(p-1)(q-1)$, we also know
\[pq-p-q+1\]
We know that $pq = x$ and $p+q = y$, then $pq$ are roots of $t^2-yt+x=0$.

\subsection{RSA Public-Key Cryptography}
Alice generates
\begin{align*}
    p, q                  & \to \text{Two large prime numbers} \\
    e                     & \to \text{``Public exponent''}     \\
    N = pq                & \to \text{``Public modulo''}       \\
    \boxed{(e, N)}        & \to \text{Public key}              \\
    (d, N)                & \to \text{Private key}             \\
    \text{where }d\cdot e & \equiv 1\mod(p-1)(q-1)
\end{align*}

Bob has some message $m$ he wishes to send to Alice. Bob sends $m^e\mod N$ and sends it to Alice.

After receiving this message, Alice can recover $(m^e)^d\equiv m\mod N$.

$p, q$ are private, but $pq$ is private. The security of RSA rests on multiplication being easy, but factorization being hard ($pq$ is hard to factorize into $p$ and $q$).

We might implement such an algorithm like so:
\begin{lstlisting}[language=Python]
from crypto import gcd, ext_gcd, pow_mod
N = p * q
d = ext_gcd(e, (p-1) * (q-1))[0] % ((p-1) * (q-1))
m = 1234567891234786951234010239847123748
# 0 < m < N is True
c = pow_mod(m, e, N)
pow_mod(c, d, N) # => m
\end{lstlisting}

If we were Alice and Bob, we \emph{still} have one step to go! We need to find ourselves big prime numbers $p, q$ (finding $e$ is easy, we can just use our \textsf{gcd} algorithm). How do we do that?

\subsection{Primality Testing}
\textsf{Prime hunting!}

Prime numbers are reasonably common. So generating a large prime number amounts to generating a number, checking if it's prime, and repeating until we get a prime.

This reduces to the problem of checking if a number is prime. Given an $n$, is it a prime?

\begin{lstlisting}
from crypto import pow_mod
n = 123874610239487102893741890237023
pow_mod(2, n-1, n)
\end{lstlisting}
gives us a basic primality check. Fermat's Little Theorem says that if $n$ is prime, then $2^{p-1}\equiv 1\mod n$.

\begin{definition}[Witnesses]
    We say that $a$ is a \ul{witness} for the compositeness of $n$ if
    \[a^{n-1}\not\equiv 1\mod n\]
    and $gcd(a, n) = 1$.
\end{definition}

We have a problem! Fermat's Little Theorem is not an if-and-only-if. We can have numbers that pass this test for almost every base. Take $n = 3\cdot 11\cdot 17 = 561$.

\begin{lstlisting}[language=Python]
from crypto import pow_mod, gcd
for a in range(n):
    if gcd(a, n) == 1:
        print(pow_mod(a, n-1, n))
\end{lstlisting}
gives $1$ for \emph{everything}\footnote{oh no!}

\begin{definition}[Carmichael Number]
    A \ul{Carmichael number} is a composite number with \emph{no} witness of compositeness.
\end{definition}
\begin{example}
    $561$ is a Carmichael Number.
\end{example}

We now \emph{almost} have a way of checking for primality, but it doesn't always quite work.

Since taking to the power of $n-1$ is a group homomorphism, then Lagrange's Theorem states that if there is a witness, then there are \emph{a lot} of witnesses.

\begin{proposition}\label{prop:aq-1-mod-p}
    Let $p$ be an odd prime. Write
    \[p-1 = 2^k\cdot q\quad\text{with $q$-odd}\]
    Then either
    \[a^{q}\equiv 1\mod p\]
    or one of $a^q, a^{2q}, a^{4q},\dots, a^{2^{k-1}q}$ is $\equiv -1\mod p$.
\end{proposition}
\begin{proof}
    We look at this sequence
    \[a^q, a^{2q}, a^{4q},\dots, \underbrace{a^{2^iq}}_{\not\equiv 1\pmod p}, \underbrace{a^{2^{i+1}q}}_{\equiv 1\pmod p}, \dots, a^{2^{k-1}q}, a^{2^kq}\]
    We have that $a^{2^kq}\equiv 1\mod p$ by Fermat's Little Theorem. There's some point where we `become' congruent to $1\mod p$. If the first one is one, then we have the first case. Otherwise we have $a^q\not\equiv 1$ mod $p$, then we repeatedly square until we get to $1\mod p$. Then \emph{right before} we turned to $1\mod p$, we would have had $-1\mod p$. In our example above, this is $a^{2^iq}$.
\end{proof}


\chapter{Introduction}

\section{Motivation}
An example of computing a result over a data type is computing the \textbf{maximum path sum} over a binary tree. Given a \inlinehaskell{BinTree}, starting at the root node, find a path from the root node to the leaves that leads to the maximum total.

The implementation of the computation is done in \texttt{Haskell}. And the definition of the binary tree and example tree is:
\begin{haskell}
data BinTree = Leaf Int
             | Node BinTree Int BinTree
\end{haskell}

\begin{haskell}
exampleTree :: BinTree
exampleTree = Node (Node (Leaf 8) 7 (Leaf 9)) 3 (Node (Leaf 5) 4 (Leaf 2))
\end{haskell}

\begin{figure}[H]
    \centering
    \includegraphics[width=.4\textwidth]{introduction/Tree.pdf}
    \caption{Graphic visualization of the \inlinehaskell{exampleTree}}
\end{figure}

An implementation of computing the max path sum over data type \inlinehaskell{BinTree} can be seen below and has a complexity of $\mathcal{O}(N)$, where $N$ is the number of nodes in the tree.
\begin{haskell}
maxPathSum :: BinTree -> Int
maxPathSum (Leaf x)     = x
maxPathSum (Node l x r) = x + max (maxPathSum l, maxPathSum r)
\end{haskell}

And computing the max path sum over the \inlinehaskell{exampleTree} results in 19.

\begin{figure}[H]
\begin{minipage}[c]{0.35\textwidth}
\begin{haskell}
maxPathSum exampleTree !$\equiv$! 19
\end{haskell}
\end{minipage}
\hspace{0.1\textwidth}
\begin{minipage}[c]{0.55\textwidth}
\includegraphics[width=0.7\textwidth]{introduction/SolvedTree.pdf}
\end{minipage}
\end{figure}

To reduce the number of recomputations, first, we need to compare the structure of the data type for equality. When comparing the data type structure for equality, if they are equal, the previously computed result can be reused. Otherwise, the result needs to be recomputed. To compare the structure of the data type efficiently, we introduce the use of hash functions. Generating a hash every time a comparison takes place would be inefficient because part of the structure does not change, which leads to the same hash. Thus, every substructure in the data type stores the hash of its structure. Using hash functions and storing them to compare the structures of the data type for equality is performed in constant time ($\mathcal{O}(1)$).

In the example, a new data type is introduced: the \inlinehaskell{MerkleTree}. This data type is the same as the \inlinehaskell{BinTree}, but the constructors also contain a \inlinehaskell{Hash}. To create a \inlinehaskell{MerkleTree}, we traverse through a \inlinehaskell{BinTree} and hash the structure and store it. Creating a \inlinehaskell{MerkleTree} has a time complexity of $\mathcal{O}(N)$.

\begin{haskell}
data MerkleTree = LeafH Hash Int
                | NodeH Hash MerkleTree Int MerkleTree

merkle :: BinTree -> MerkleTree
merkle (Leaf x)     = LeafH (hash ["Leaf", x]) x
merkle (Node l x r) = NodeH (hash ["Node", x, hl, hr]) l' x r'
  where
    hl = getHash l'
    hr = getHash r'
    l' = merkle l
    r' = merkle r
\end{haskell}

\begin{figure}[H]
    \centering
    \includegraphics[width=.4\textwidth]{introduction/MerkleTree.pdf}
    \caption{The \inlinehaskell{MerkleTree} of \inlinehaskell{exampleTree}}
\end{figure}

To compute the max path sum and its intermediate results, we can use the precomputed hashes of the \inlinehaskell{MerkleTree} to efficiently generate a \inlinehaskell{Map Hash Int}. The complexity of computing the max path sum and the intermediate results is $\mathcal{O}(N)$.

The example implementation of computing the max path sum and its intermediate results can be seen below. However, this implementation uses a \texttt{union} (\inlinehaskell{<>}) to combine the \inlinehaskell{Map}'s which has a complexity of $\mathcal{O}(m*log(n/m + 1)), m <= n$\cite{HaskellDataUnion}. This can be implemented more efficiently by giving the \inlinehaskell{Map} to the left side and then to the right side of the Tree, which makes it a constant operation. Except, this would make the code more complex. So, for clarity, the example is implemented with a union.

\begin{haskell}
maxPathSumInc :: MerkleTree -> (Int, Map Hash Int)
maxPathSumInc (LeafH h x)     = (x, insert h x empty)
maxPathSumInc (NodeH h l x r) = (y, insert h y (ml <> mr))
  where
    y = x + max (xl, xr)
    (xl, ml) = maxPathSumInc l
    (xr, mr) = maxPathSumInc r
\end{haskell}
\vspace{15pt}
\begin{figure}[H]
    \begin{minipage}[c]{0.55\textwidth}
        \centering
        \includegraphics[width=.7\textwidth]{introduction/MerkleTree.pdf}
    \end{minipage}
    \hspace{0.1\textwidth}
    \begin{minipage}[c]{0.35\textwidth}
        \centering
        \begin{tabular}{|l|r|}
            \hline
            \textbf{Hash Nodes} & \textbf{Max Sum} \\
            \hline
            \# 6dd & 19 \\
            \hline
            \# 5df & 16 \\
            \hline
            \# fa0 & 8 \\
            \hline
            \# 8d0 & 9 \\
            \hline
            \# f3b & 9 \\
            \hline
            \# 84b & 5 \\
            \hline
            \# 1ad & 2 \\
            \hline
        \end{tabular}
    \end{minipage}
    \caption{The \inlinehaskell{MerkleTree} with intermediate results}
\end{figure}

When there is a change in the \inlinehaskell{BinTree}, only the hashes of the change itself and its parents need to be recomputed. The recomputation of the hash has a time complexity with an upper bound of $\mathcal{O}(N)$ and an average of $\Theta(M \log{N})$ where $M$ is the number of changed nodes. The upper bound is $\mathcal{O}(N)$, because if $N = M \Rightarrow \mathcal{O}(N \log{N}) > \mathcal{O}(N)$, but when every node in the tree is changed, then the original function can be used. Meaning that this implementation only works efficiently with small changes to the data type.

\begin{figure}[H]
    \centering
    \includegraphics[width=.4\textwidth]{introduction/ChangedMerkleTree.pdf}
    \caption{Changed Merkle Tree}
\end{figure}

Then to compute the max path sum over the Changed Tree, the previously computed \inlinehaskell{Map} can be used to reduce the amount of recomputation. This function also has the same complexity as the change Merkle tree, with an upper bound of $\mathcal{O}(N)$ and an average of $\Theta(M \log{N})$.

\begin{haskell}
maxPathSumMap :: Map Hash Int -> MerkleTree -> (Int, Map Hash Int)
maxPathSumMap m (LeafH h x) = case lookup m h of
  Just y  -> (y, m)
  Nothing -> (x, insert h x m)
maxPathSumMap m (NodeH h l x r) = case lookup m h of
  Just y  -> (y, m)
  Nothing -> (y, insert h y (ml <> mr))
    where
      y = x + max (xl, xr)
      (xl, ml) = maxPathSumMap m l
      (xr, mr) = maxPathSumMap m r
\end{haskell}

In Table \ref{table:function-compl-inc} and \ref{table:function-compl} the function complexities can be seen of the functions used to compute incrementally and complete. As can be seen, the upper bound complexities of the functions for incremental computation are the same as for the complete computation. However, when there are small changes to the data type the incremental update of the result has lower complexity ($\Theta(M \log{N})$) than the complete recomputation complexity ($\Theta(N)$), which makes it more efficient.

\begin{table}[H]
    \centering
    \begin{tabular}{|l|r|r|}
        \hline
        \textbf{Function} & \textbf{Average} & \textbf{Upper bound} \\
        \hline
        \inlinehaskell{merkle} & $\Theta(N)$ & $\mathcal{O}(N)$ \\
        \hline
        \inlinehaskell{maxPathSumInc} & $\Theta(N)$ & $\mathcal{O}(N)$ \\
        \hline
        Change Merkle Tree & $\Theta(M \log{N})$  & $\mathcal{O}(N)$ \\
        \hline
        \inlinehaskell{maxPathSumMap} & $\Theta(M \log{N})$  & $\mathcal{O}(N)$ \\
        \hline
    \end{tabular}
    \caption{Functions used for incremental computation}
    \label{table:function-compl-inc}
\end{table}

\begin{table}[H]
    \centering
    \begin{tabular}{|l|r|r|}
        \hline
        \textbf{Function} & \textbf{Average} & \textbf{Upper bound} \\
        \hline
        \inlinehaskell{maxPathSum} & $\mathcal{O}(N)$ & $\mathcal{O}(N)$ \\
        \hline
    \end{tabular}
    \caption{Functions used for complete computation}
    \label{table:function-compl}
\end{table}

The problem with this implementation is that it only works for the \inlinehaskell{BinTree} datatype. The goal would be to create a generic function, where only the \inlinehaskell{maxPathSum} would be defined and the result would automatically contain the intermediate results. A generic definition could look like this:

\begin{haskell}
cataMerkle :: (f a -> a) -> Map Hash a -> Fix (f :*: K Hash) -> (a, Map Hash a)
\end{haskell}

Using the \inlinehaskell{cataMerkle} function would lead to only needing to implement the \inlinehaskell{maxPathSum} function and the intermediate results are then automatically stored.

\section{Research Questions}
\todo[inline]{Write the research questions}

\section{Contributions}
\todo[inline]{Write the contributions}


\sec{Laplace Transforms}
\subsection{The Laplace Transform}
\begin{defn}[The Laplace Transform]
	Let $f:(0, \infty)\to\mathbb{R}$ be a good enough function.\\
	The Laplace transform of $f$ is another function denoted as $\mathcal{L}(f)$ or $\mathcal{L}f$ or simply, $F.$ It is defined as follows:
	\begin{equation} \label{eq:lap}
		(\mathcal{L}f)(s) := \int_{0}^{\infty} e^{-st}f(t) dt.
	\end{equation}
\end{defn}

\begin{mdframed}[style=boxstyle, frametitle={Remarks}]
	\begin{enumerate}[leftmargin=*]
		\item It is customary to use uppercase letters for the Laplace transform and use the variable $s$ as the argument of the transform. Thus, we write $F(s), X(s), Y(s),$ et cetera for the Laplace transforms of $f(t), x(t), y(t),$ et cetera.
		\item We have not stated what ``good enough'' means. Also we have not stated the domain of $\mathcal{L}f.$ It is precisely whenever the integral in (\ref{eq:lap}) exists. More details about sufficient conditions can be found in the notes linked.
		\item Notations will be abused a lot and we'll write things like $\mathcal{L}(f) = F(s)$ or $\mathcal{L}(f(t)) = F(s).$ (If you don't realise why this is abuse of notation, then ignorance is bliss and continue.)
	\end{enumerate}
\end{mdframed}

\begin{mdframed}[style=boxstyle2, frametitle={A remark}]
	Note that in general, the function $F(s)$ need not exist on the whole real line. Often, there exists some $a \in \mathbb{R}$ such that $F(s)$ exists for $s > a.$
\end{mdframed}

\newpage

\subsection{Properties}
The advantage of studying Laplace transforms will be seen when we see its many different properties.\\
We have the following property that's easy to verify:
\begin{mdframed}[style=boxstyle, frametitle={Linearity of Laplace Transforms}]
	\begin{equation*}
		\mathcal{L}(af + bg) = a\mathcal{L}(f) + b\mathcal{L}(g).
	\end{equation*}
	Of course, $a$ and $b$ are real numbers.
\end{mdframed}

Recall from calculus that changing the value of a function at finitely many points has no effect on the integral. Thus, it is possible for two different functions to have the same Laplace transform. However, we do have the following theorem if we demand continuity.
\newpage
\begin{thm}[Equality of Laplace transforms]
	Let $f, g:[0, \infty) \to \mathbb{R}$ be continuous functions such that
	\[\mathcal{L}f = \mathcal{L}g.\]
	In this case, we have
	\[f = g.\]
\end{thm}

\begin{mdframed}[style=boxstyle, frametitle={A trick}]
	One useful trick is differentiate with respect to the parameter. For example, consider:
	\begin{align*}
		& \mathcal{L}(e^{at})(s) = \dfrac{1}{s - a}\\
		& \text{Differentiating with respect to }a:\\
		& \mathcal{L}(te^{at})(s) = \dfrac{1}{(s - a)^2}.
	\end{align*}
	Note that the interchanging of $\dfrac{\partial}{\partial a}$ and $\mathcal{L}$ needs justification. (Which we do not provide.)
\end{mdframed}

\begin{thm}[Derivatives of Laplace]
	\begin{equation*}
		\mathcal{L}(tf(t)) = -\dfrac{d}{ds}F(s).
	\end{equation*}
	In general,
	\begin{equation*}
		\mathcal{L}(t^nf(t)) = (-1)^n\dfrac{d^n}{ds^n}F(s).
	\end{equation*}
\end{thm}

\begin{thm}[Laplace of derivatives]
	\begin{align*}
		\mathcal{L}(f'(t)) &= sF(s) - f(0),\\
		\mathcal{L}(f''(t)) &= s^2F(s) - sf(0) - f'(0),\\
		\mathcal{L}(f^{(n)}(t)) &= s^nF(s) - \sum_{k=0}^{n-1} s^{n-1-k}f^{(k)}(0).
	\end{align*}
\end{thm}

\begin{thm}[First Shift Theorem]
	If
	\begin{equation*}
		\mathcal{L}(f(t)) = F(s),
	\end{equation*}
	then
	\begin{equation*}
		\mathcal{L}(e^{at}f(t)) = F(s - a).
	\end{equation*}
\end{thm}

\begin{defn}[Heaviside step function]
	The \defin{Heaviside unit step function} $u:\mathbb{R}\to\{0, 1\}$ is defined as
	\begin{align*}
		u(t) := \begin{cases}
			0 & \text{if } t < 0\\
			1 & \text{if } t \ge 0
		\end{cases}
	\end{align*}
	For $c \in \mathbb{R},$ the function $u_c(t)$ is defined as $u(t - c).$
\end{defn}

\begin{thm}[Second Shift Theorem]
	Suppose $\mathcal{L}f = F(s)$ for $s > a \ge 0.$ \\
	If $c > 0,$ then we have
	\begin{equation*}
		\mathcal{L}(u_c(t)f(t - c)) = e^{-cs}F(s),
	\end{equation*}
	for $s > a.$
\end{thm}

\begin{thm}[Laplace transform of periodic functions]
	Let $p > 0$ be such that $f(t + p) = f(t).$
	\begin{equation*}
		\mathcal{L}f = \dfrac{1}{1 - e^{-ps}}\int_{0}^{p} e^{-st}f(t) dt.
	\end{equation*}
\end{thm}

\subsection{Laplace table}
\begin{center}
	\bgroup
	\def\arraystretch{2}
	\begin{tabular}{|c|c||c|c|}
	\hline
	$f(t)$ & $F(s)$ & $f(t)$ & $F(s)$ \\
	\hline
	\hline
	$t$ & $1/s^2$ & $t^a$ & $\dfrac{\Gamma(a + 1)}{s^{a + 1}}$\\
	$u_c(t)$ & $e^{-cs}/s$ & $e^{at}$ & $\dfrac{1}{s - a}$\\
	$\sin(\omega t)$ & $\dfrac{\omega}{s^2 + \omega^2}$ & $\cos(\omega t)$ & $\dfrac{s}{s^2 + \omega^2}$\\
	$t\sin(\omega t)$ & $\dfrac{2\omega s}{(s^2 + \omega^2)^2}$ & $t\cos(\omega t)$ & $\dfrac{s^2 - \omega^2}{(s^2 + \omega^2)^2}$\\
	$e^{at}\sin(\omega t)$ & $\dfrac{\omega}{(s - a)^2 + \omega^2}$ & $e^{at}\cos(\omega t)$ & $\dfrac{s - a}{(s - a)^2 + \omega^2}$\\
	$\sinh(\omega t)$ & $\dfrac{\omega}{s^2 - \omega^2}$ & $\cosh(\omega t)$ & $\dfrac{s}{s^2 - \omega^2}$\\
	$e^{at}\sinh(\omega t)$ & $\dfrac{\omega}{(s - a)^2 - \omega^2}$ & $e^{at}\cosh(\omega t)$ & $\dfrac{s - a}{(s - a)^2 - \omega^2}$\\
	\hline
	\end{tabular}
	\egroup
\end{center}

\subsection{The convolution}
\begin{defn}[Convolution]
	The \defin{convolution} of functions $f$ and $g$ is another function $f*g$ defined as
	\begin{equation*}
		(f*g)(t) = \int_{0}^{t} f(t - \tau)g(\tau) d\tau.
	\end{equation*}
\end{defn}

\begin{mdframed}[style=boxstyle, frametitle={Properties}]
	\begin{enumerate}[leftmargin=*]
		\item $f*g = g*f.$
		\item $f*(g_1 + g_2) = f*g_1 + f*g_2.$
		\item $(f * g) * h = f * (g * h).$
		\item $f * 0 = 0 * f = 0.$ (Here, $0$ denotes the zero \emph{function}.)
	\end{enumerate}
\end{mdframed}

\begin{mdframed}[style=boxstyle, frametitle={Caution}]
	$f*1 = f$ is \textbf{not} true in general.\\
	For instance, $\sin t * 1 = 1 - \cos t.$
\end{mdframed}

\begin{thm}[Convolution of Laplace]
	Suppose $\mathcal{L}f$ and $\mathcal{L}g$ exist for all $s > a \ge 0.$ Then,
	\begin{align*}
		\mathcal{L}(f * g) = \mathcal{L}(f)\mathcal{L}(g) \qquad \text{for } s > a.
	\end{align*}
\end{thm}