\documentclass[11pt,letterpaper]{article}
\usepackage[hmargin=0.7in,vmargin=1in,landscape]{geometry}
\usepackage[T1]{fontenc}
\usepackage{url}
\usepackage{tabularx,array,varwidth}

\setlength{\parindent}{0pt}

\begin{document}
{\LARGE Big-O Cheat Sheet} \\
Generated \today. \\
Brandon Amos <\url{http://bamos.github.io}>

\newcolumntype{M}{>{\begin{varwidth}{4.5cm}}l<{\end{varwidth}}}
\section{Searching}
\begin{tabularx}{\textwidth}{ MXXXXX }
Algorithm&Data Structure&\multicolumn{2}{l}{Time Complexity}&Space Complexity\\
\hline
&&Average&\multicolumn{2}{l}{Worst}\\
\hline
Depth First Search (DFS) & Graph of $|V|$ vertices and $|E|$ edges & - & $O\left(|E| + |V|\right)$ & $O\left(|V|\right)$\\
Breadth First Search (BFS) & Graph of $|V|$ vertices and $|E|$ edges & - & $O\left(|E| + |V|\right)$ & $O\left(|V|\right)$\\
Binary search & Sorted array of n elements & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(1\right)$\\
Linear (Brute Force) & Array & $O\left(n\right)$ & $O\left(n\right)$ & $O\left(1\right)$\\
Shortest path by Dijkstra, using a Min-heap as priority queue & Graph with $|V|$ vertices and $|E|$ edges & $O\left((|V| + |E|) \log |V|\right)$ & $O\left((|V| + |E|) \log |V|\right)$ & $O\left(|V|\right)$\\
Shortest path by Dijkstra, using an unsorted array as priority queue & Graph with $|V|$ vertices and $|E|$ edges & $O\left(|V|^2\right)$ & $O\left(|V|^2\right)$ & $O\left(|V|\right)$\\
Shortest path by Bellman-Ford & Graph with $|V|$ vertices and $|E|$ edges & $O\left(|V||E|\right)$ & $O\left(|V||E|\right)$ & $O\left(|V|\right)$\\

\end{tabularx}
\section{Sorting}
\begin{tabularx}{\textwidth}{ MXXXXXX }
Algorithm&Data Structure&\multicolumn{3}{l}{Time Complexity}&Worst Case Auxiliary Space Complexity\\
\hline
&&Best&Average&\multicolumn{2}{l}{Worst}\\
\hline
Quicksort & Array & $O\left(n \log(n)\right)$ & $O\left(n \log(n)\right)$ & $O\left(n^2\right)$ & $O\left(n\right)$\\
Mergesort & Array & $O\left(n \log(n)\right)$ & $O\left(n \log(n)\right)$ & $O\left(n \log(n)\right)$ & $O\left(n\right)$\\
Heapsort & Array & $O\left(n \log(n)\right)$ & $O\left(n \log(n)\right)$ & $O\left(n \log(n)\right)$ & $O\left(1\right)$\\
Bubble Sort & Array & $O\left(n\right)$ & $O\left(n^2\right)$ & $O\left(n^2\right)$ & $O\left(1\right)$\\
Insertion Sort & Array & $O\left(n\right)$ & $O\left(n^2\right)$ & $O\left(n^2\right)$ & $O\left(1\right)$\\
Select Sort & Array & $O\left(n^2\right)$ & $O\left(n^2\right)$ & $O\left(n^2\right)$ & $O\left(1\right)$\\
Bucket Sort & Array & $O\left(n+k\right)$ & $O\left(n+k\right)$ & $O\left(n^2\right)$ & $O\left(nk\right)$\\
Radix Sort & Array & $O\left(nk\right)$ & $O\left(nk\right)$ & $O\left(nk\right)$ & $O\left(n+k\right)$\\

\end{tabularx}
\section{Data Structures}
\begin{tabularx}{\textwidth}{ MXXXXXXXXXX }
Data Structure&\multicolumn{8}{l}{Time Complexity}&Space Complexity\\
\hline
&\multicolumn{4}{l}{Average}&\multicolumn{5}{l}{Worst}\\
\hline
&Indexing&Search&Insertion&Deletion&Indexing&Search&Insertion&Deletion&\\
\hline
Basic Array & $O\left(1\right)$ & $O\left(n\right)$ & - & - & $O\left(1\right)$ & $O\left(n\right)$ & - & - & $O\left(n\right)$\\
Dynamic Array & $O\left(1\right)$ & $O\left(n\right)$ & $O\left(n\right)$ & $O\left(n\right)$ & $O\left(1\right)$ & $O\left(n\right)$ & $O\left(n\right)$ & $O\left(n\right)$ & $O\left(n\right)$\\
Singly-Linked List & $O\left(n\right)$ & $O\left(n\right)$ & $O\left(1\right)$ & $O\left(1\right)$ & $O\left(n\right)$ & $O\left(n\right)$ & $O\left(1\right)$ & $O\left(1\right)$ & $O\left(n\right)$\\
Doubly-Linked List & $O\left(n\right)$ & $O\left(n\right)$ & $O\left(1\right)$ & $O\left(1\right)$ & $O\left(n\right)$ & $O\left(n\right)$ & $O\left(1\right)$ & $O\left(1\right)$ & $O\left(n\right)$\\
Skip List & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(n\right)$ & $O\left(n\right)$ & $O\left(n\right)$ & $O\left(n\right)$ & $O\left(n \log(n)\right)$\\
Hash Table & - & $O\left(1\right)$ & $O\left(1\right)$ & $O\left(1\right)$ & - & $O\left(n\right)$ & $O\left(n\right)$ & $O\left(n\right)$ & $O\left(n\right)$\\
Binary Search Tree & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(n\right)$ & $O\left(n\right)$ & $O\left(n\right)$ & $O\left(n\right)$ & $O\left(n\right)$\\
Cartresian Tree & - & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & - & $O\left(n\right)$ & $O\left(n\right)$ & $O\left(n\right)$ & $O\left(n\right)$\\
B-Tree & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(n\right)$\\
Red-Black Tree & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(n\right)$\\
Splay Tree & - & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & - & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(n\right)$\\
AVL Tree & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(\log(n)\right)$ & $O\left(n\right)$\\

\end{tabularx}

\end{document}


% !TEX root = Main.tex
\section{Optimization}
To show that an objective is not convex, compute Hessian and show that it's not p.s.d. (det(H) < 0).

\subsection*{Alternating Least Squares}
Optimising L(U,V) is not convex, but keeping one of them fixed and optimising the other is.
$f(U,v_j)=\sum_{(i,j)\in I} (a_{i,j} - \langle u_i, v_j \rangle)^2 + \lambda \sum_i \lVert\mathbf{u_i}\rVert^2 +(...)$\\
$f(u_i,V)=\sum_{(i,j)\in I} (a_{i,j} - \langle u_i, v_j \rangle)^2 + (...) + \lambda \sum_j \lVert\mathbf{v_j}\rVert^2$\\
$u_i = (\sum_{i,j \in \mathcal{I}} v_jv_j^T + \lambda I_k)^{-1} \sum_{i,j \in \mathcal{I}} a_{ij}v_j$
Derived by setting up Lagrangian and setting its partial derivative w.r.t $u_i$ to 0.

\subsection*{Gradient Descent (or Deepest Descent)}
\textbf{Gradient}: $\nabla f(\mathbf{x}) := \left( \frac{\partial f(\mathbf{x})}{\partial \mathbf{x}_1}, \ldots, \frac{\partial f(\mathbf{x})}{\partial \mathbf{x}_D} \right)^\top$

\begin{inparaenum}[\color{red}1.]
	\item init: $\mathbf{x}^{(0)} \in \mathbb{R}^D$
	\item for $t = 0 \ \text{to} \ \mathit{maxIter}$:
	\item $\mathbf{x}^{(t+1)} = \mathbf{x}^{(t)} - \gamma \nabla f(\mathbf{x}^{(t)})$, usually $\gamma \approx \frac{1}{t}$
\end{inparaenum}

\subsection*{Stochastic Gradient Descent (SGD)}
Assume \textbf{Additive Objective}: $f(x) = \frac{1}{N}\sum_{n=1}^{N}f_n(x)$\\

\begin{inparaenum}[\color{red}1.]
	\item init: $\mathbf{x}^{(0)} \in \mathbb{R}^D$
	\item for $t = 0 \ \text{to} \ \mathit{maxIter}$:
	\item sample $n \in_{u.a.r.} \{1, \ldots, N\}$
	\item $\mathbf{x}^{(t+1)} = \mathbf{x}^{(t)} - \gamma \nabla f_n(\mathbf{x}^{(t)})$, typically  $\gamma \approx \frac{1}{t}$.
\end{inparaenum}

\subsection*{Projected Gradient Descent (Constrained Opt.)}
minimize $f(x)$, $x \in Q$ (constraint).\\
\textbf{Project} $x$ onto $Q$: $P_Q(\mathbf{x}) = \argmin_{y \in Q} \|\mathbf{y} - \mathbf{x}\|$,\\
\textbf{Update}: $\mathbf{x}^{(t+1)} = P_Q[\mathbf{x}^{(t)} - \gamma \nabla f(\mathbf{x}^{(t)})]$,\\
$\mathbf{x}^{(t+1)}$ is unique if $Q$ convex.

\subsection*{Lagrangian Multipliers}
Minimize  $f(\mathbf{x})$ s.t. $g_i(\mathbf{x}) \leq 0,\ i = 1, .., m$ (\textbf{inequality constr.}) and $h_i(\mathbf{x}) = \mathbf{a}_i^\top \mathbf{x} - b_i = 0,\ i = 1, .., p$ (\textbf{equality constraint}).
\textbf{Lagrangian:} $L(\mathbf{x}, \boldsymbol{\lambda}, \boldsymbol{\nu}) := f(\mathbf{x}) + \sum_{i=1}^m \lambda_i g_i(\mathbf{x}) + \sum_{i=1}^p \nu_i h_i(\mathbf{x})$\\

\subsection*{Convex Optimization}
Def.: $\{(x,t)|x \in dom f, f(x) \leq t\}$, $f : \mathbb{R}^D \rightarrow \mathbb{R}$ is convex, if $dom\ f$ is a convex set, and if $\forall \mathbf{x}, \mathbf{y} \in dom\ f$, and for $0 \leq \alpha \leq 1$: $f(\alpha \mathbf{x} + (1 - \alpha)\mathbf{y}) \leq \alpha f(\mathbf{x}) + (1-\alpha)f(\mathbf{y})$. local=global min, \textbf{Convergence}: $f(\mathbf{x}^{(t)}) - f(\mathbf{x}^*) \le \frac{c}{t}$.
\textbf{Subgradient} $g \in \mathbb{R}^D$ of $f$ at $\mathbf{x}$: $f(\mathbf{y}) \geq f(\mathbf{x}) + g^\top(\mathbf{y}-\mathbf{x}) \ \forall \mathbf{y}$

\subsection*{Convex Relaxation}
Replace non-convex rank constraints by convex norm constraints.
$rank(B) \geq ||B||_*$. Repeat: $B^* = shrink_\tau(A) = arg min_B(\frac{1}{2}||A - B||_F^2 + \tau ||B||_*)$, with $B^*=UD_\tau V^T, D_\tau = diag(max(0,\sigma_i - \tau))$. Then $B_{t+1} = B_t + \eta_t \Pi(A - shrink_\tau(B_t))$, where $\Pi$ cuts off $x < 0$. Exact reconstruction of rank k matrix $A^*$ w.h.p, if it is strongly incoherent, if $\mathcal{I} \in O(n)$, spread of $\sigma_i$ large enough.


\subsection{Average Treatment Effects (ATE, ATET, ATEUT)}

\subsubsection{Average Treatment Effect (ATE)}

\(ATE=E[y_{i1}-y_{i0}]\)

\subsubsection{Average Treatment Effect on the Treated (ATET)}

\(ATE=E[y_{i1}-y_{i0}|D_i=1]\)

\(ATE=E[y_{i1}|D_i=1]-E[y_{i0}|D_i=1]\)

\subsubsection{Average Treatment Effect on the Untreated (ATEUT)}


\lab{The Drazin Inverse}{The Drazin Inverse}
\label{lab:drazin_inverse}
% TODO: include more on the Moore-Penrose pseudoinverse and rename the lab "Pseudoinverses"?

\objective{The Drazin inverse of a matrix is a pseudoinverse which preserves certain spectral properties of the matrix.
In this lab, we compute the Drazin inverse using the Schur decomposition, then use it to compute the effective resistance of a graph and perform link prediction.}

% TODO: compute Ind(A), effective resistance explanation, algorithm for computing drazin

\section*{Definition of the Drazin Inverse} % =================================

The \emph{index} of an $n \times n$ matrix $A$ is the smallest nonnegative integer $k$ for which $\mathscr{N}(A^k) = \mathscr{N}(A^{k+1})$.
The \emph{Drazin inverse} $A^D$ of $A$ is the unique $n \times n$ matrix satisfying the following properties.
\begin{itemize}
\item $ AA^{D} =  A^{D}A$
\item $A^{k+1}A^D = A^k$
\item $A^DAA^D = A^D$
\end{itemize}
Note that if $A$ is \emph{invertible}, in which case $k=0$, then $A^D = A^{-1}$.
On the other hand, if $A$ is \emph{nilpotent}, meaning $A^j = \0$ for some nonnegative integer $j$, then $A^D$ is the zero matrix.

\begin{problem} % Check for the Drazin inverse.
Write a function that accepts an $n \times n$ matrix $A$, the index $k$ of $A$, and an $n \times n$ matrix $A^D$.
Use the criteria described above to determine whether or not $A^D$ is the Drazin inverse of $A$.
Return \li{True} if $A^D$ satisfies all three conditions; otherwise, return \li{False}.

Use the following matrices as test cases for your function.
\[
A = \left[\begin{array}{cccc}
1 & 3 & 0 & 0 \\
0 & 1 & 3 & 0 \\
0 & 0 & 1 & 3 \\
0 & 0 & 0 & 0
\end{array}\right],
\quad
A^D = \left[\begin{array}{cccc}
1 & -3 & 9 & 81 \\
0 & 1 & -3 & -18 \\
0 & 0 & 1 & 3 \\
0 & 0 & 0 & 0
\end{array}\right],
\quad
k = 1
\]
%
\[
B = \left[\begin{array}{ccc}
 1 &  1 &  3 \\
 5 &  2 &  6 \\
-2 & -1 & -3
\end{array}\right],
\quad B^D = \left[\begin{array}{ccc}
0 & 0 & 0 \\
0 & 0 & 0 \\
0 & 0 & 0
\end{array}\right],
\quad
k = 3
\]
(Hint: \li{np.allclose()} and \li{np.linalg.matrix_power()} may be useful).
\label{prob:test-drazin}
\end{problem}

\subsection*{Computing the Drazin Inverse} % ----------------------------------

The Drazin inverse is often defined theoretically in terms of the eigenprojections of a matrix.
However, eigenprojections are often costly or unstable to calculate, so we resort to a different method to calculate the Drazin inverse.

To begin, suppose that the $n \times n$ matrix $A$ can be written in the form
\begin{equation}
A = S^{-1}
\left[\begin{array}{cc}
M & \0 \\
\0 & N \\
\end{array}\right] S,
\label{eq:nilpotent-sort-decomposition}
\end{equation}
where $S$ is a change of basis matrix, $N$ is nilpotent, and $M$ is the restriction of $A$ onto the range of $I - P_0$, where $P_0$ is the 0-eigenprojection in the spectral decomposition.
Then the Drazin inverse can be calculated as
\begin{equation}
A^D = S^{-1}
\left[\begin{array}{cc}
M^{-1} & \0 \\
\0 & \0 \\
\end{array}\right] S.
\label{eq:nilpotent-sort-compute-drazin}
\end{equation}

Next, the \emph{Schur decomposition} of $A$ is given by
\begin{align}
A = QTQ^{-1},
\end{align}
where $Q$ is orthonormal and $T$ is upper triangular.
Since $T$ is similar to $A$, the eigenvalues of $A$ are listed along the diagonal of $T$.
Then if $A$ is singular, at least one diagonal entry of $T$ must be $0$.
The columns that contain the $0$ eigenvalues of $A$ form the nilpotent matrix $N$ in (\ref{eq:nilpotent-sort-decomposition}).
To compute $M$ and $N$, we sort the Schur decomposition so that the $0$ eigenvalues are listed last along the diagonal of $T$, and then we can use (\ref{eq:nilpotent-sort-compute-drazin}) to compute $A^D$.

SciPy's \li{la.schur()} is a routine for computing the Schur decomposition of a matrix, but it does not automatically sort it by eigenvalue.
However, sorting can be accomplished by specifying the \li{sort} keyword argument.

\begin{lstlisting}
>>> from scipy import linalg as la

# The standard Schur decomposition.
>>> A = np.array([[0,0,2],[-3,2,6],[0,0,1]])
>>> T,Z = la.schur(A)
>>> T                       # The eigenvalues (2, 0, and 1) are not sorted.
array([[ 2., -3.,  6.],
       [ 0.,  0.,  2.],
       [ 0.,  0.,  1.]])

# Specify a sorting function to get the desired result.
>>> f = lambda x: abs(x) > 0
>>> T1,Z1,k = la.schur(A, sort=f)
>>> T1
array([[ 2.        ,  0.        ,  6.70820393],
       [ 0.        ,  1.        ,  2.        ],
       [ 0.        ,  0.        ,  0.        ]])
>>> k                       # k is the number of columns satisfying the sort,
2                           # which is the number of nonzero eigenvalues.
\end{lstlisting}

The procedure for finding the Drazin inverse using the Schur decomposition and (\ref{eq:nilpotent-sort-decomposition}) is given in Algorithm \ref{Alg:Drazin-Inverse}.
Due to possible floating point arithmetic errors, consider all eigenvalues smaller than a certain tolerance to be $0$.

\begin{algorithm}[H]
\begin{algorithmic}[1]
\Procedure{Drazin}{$A$, tol}
    \State $(n,n) \gets \shape{A}$
    \State $Q_1,S,k_1 \gets $ \text{schur($A, |x| > $ tol)}
	   \Comment{Sort the Schur decomposition.}
    \State $Q_2,T,k_2 \gets $ \text{schur($A, |x| \leq $ tol)}
    \State $U \gets [S_{:,:k_1}\ |\ T_{:,:n - k_1}]$
        \Comment{Concatenate part of $S$ and $T$ column-wise.}
    \State $U^{-1} \gets \text{inverse(U)}$
    \State $V \gets U^{-1}AU$
    \State $Z \gets \0_{n\times n}$
        \Comment{The $n\times n$ zero matrix \textbf{as floats}, not ints.}
    \If {$k_1 \neq 0$}
        \State $M^{-1} \gets $ \text{inverse($V_{:k_1,:k_1}$)}
        \State $Z_{:k_1,:k_1} \gets M^{-1}$
    \EndIf
    \State \pseudoli{return} $UZU^{-1}$
\EndProcedure
\end{algorithmic}
\caption{}
\label{Alg:Drazin-Inverse}
\end{algorithm}

\begin{problem} % Compute the Drazin Inverse.
Write a function that accepts an $n \times n$ matrix $A$ and a tolerance for rounding eigenvalues to zero.
Use Algorithm \ref{Alg:Drazin-Inverse} to compute the Drazin inverse $A^D$.
Use your function from Problem \ref{prob:test-drazin} to verify your implementation.
\end{problem}

\begin{warn} % ill-conditioning warning.
Because the algorithm for the Drazin inverse requires calculation of the inverse of a matrix, it is unstable when that matrix has a high condition number.
If the algorithm does not find the correct Drazin inverse, check the condition number of $V$ from Algorithm \ref{Alg:Drazin-Inverse}
%For the matrix where it breaks, the conditions number is 32960398578565896.0000
\end{warn}

\begin{info}
The Drazin inverse is called a \emph{pseudoinverse} because $A^D = A^{-1}$ for invertible $A$, and for noninvertible $A$, $A^D$ always exists and acts similarly to an inverse.
There are other matrix pseudoinverses that preserve different qualities of $A$, including the \emph{Moore-Penrose pseudoinverse} $A^\dagger$, which can be thought of as the least squares approximation to $A^{-1}$.
\end{info}

\section*{Applications of the Drazin Inverse} % ===============================

\subsection*{Effective Resistance} % ------------------------------------------

The \emph{effective resistance} between two nodes in a undirected graph is a measure of how connected those nodes are.
The concept originates from the study of circuits to measure the resistance between two points on the circuit.
A \emph{resistor} is a device in a circuit which limits or regulates the flow of electricity.
Two points that have more resistors between them have more resistance, while those with fewer resistors between them have less resistance.
The entire circuit can be represented by a graph where the nodes are the points of interest and the number of edges connecting two nodes indicates the number of resistors between the corresponding points.
See Figure \ref{fig:resistors} for an example.
% To apply this concept to a graph, imagine that each edge of the graph is replaced by a resistor with a resistance of 1 unit (See Figure \ref{fig:resistors})

\begin{center}
\begin{figure}[H]
\begin{tikzpicture}[circuit ee IEC,set resistor graphic=var resistor IEC graphic,scale=.75]
%Set the coordinates and labels
\foreach \x/\y/\z/\t in {-3.5/2/.5/a,0/2/.5/c,3.5/2/.5/e,
						 -3.5/-2/-.5/b,0/-2/-.5/d,3.5/-2/-.5/f}{
	\coordinate (\t) at (\x,\y);
	\filldraw (\t) circle (2pt) node[text depth=.25ex,text height=1.5ex,font=\Large] at (\x,\y+\z) {$\t$};}
%Draw the lines with resistors
\foreach \p/\q in {a/c,a/d,b/d,c/d,c/f,d/f,e/f} \draw[every resistor/.style={circuit symbol size=width 4.5 height 0.75}] (\p) to [resistor] (\q);
\end{tikzpicture}
\caption{A graph with a resistor on each edge.}
\label{fig:resistors}
\end{figure}
\end{center}

In electromagnetism, there are rules for manually calculating the effective resistance between two nodes for relatively simple graphs.
However, this is infeasible for large or complicated graphs.
Instead, we can use the Drazin inverse to calculate effective resistance for any graph.

First, create the \emph{adjacency matrix}\footnote{See Problem 1 of Image Segmentation for a refresher on adjacency matrices and the Laplacian.} of the graph, the matrix where the $(ij)$th entry is the number of connections from node $i$ to node $j$.
Next, calculate the Laplacian $L$ of the adjacency matrix.
% TODO: must at least briefly state what the Laplacian is.
Then if $R_{ij}$ be the effective resistance from node $i$ to node $j$,
\begin{equation}
R_{ij} = \begin{cases}
(\widetilde{L}^j)^D_{ii} & \mbox{if $i \neq j$} \\
0 & \mbox{if $i = j$,}
\end{cases}
\label{eq:effective-resistance-with-Drazin}
\end{equation}
where $\widetilde{L}^j$ is the Laplacian with the $j$th row of the Laplacian replaced by the $j$th row of the identity matrix, and $(\widetilde{L}^j)^D$ is its Drazin inverse.

\begin{problem} % Compute effective resistance.
Write a function that accepts the $n \times n$ adjacency matrix of an undirected graph.
Use (\ref{eq:effective-resistance-with-Drazin}) to compute the effective resistance from each node to every other node.
Return an $n \times n$ matrix where the $(ij)$th entry is the effective resistance from node $i$ to node $j$.
Keep the following in mind: % TODO: Rephrase this somehow...
\begin{itemize}
\item The resulting matrix should be symmetric.
\item The effective resistance from a node to itself is $0$.
\item Consider creating the matrix column by column instead of entry by entry. Every time you compute the Drazin inverse, the whole diagonal of the matrix can be used.
\end{itemize}
Test your function using the graphs and values from Figure \ref{fig:eff-res}.
% Write each graph as an adjacency matrix, run the function and verify that the effective resistance from $a$ to $b$ is as given in the Figure.
\label{prob:effective-resistance}
\end{problem}

%%%%%% IMAGE %%%%%%%%
\begin{center}
\begin{figure}[H]
\begin{tikzpicture}

%%%3 Line segments
\begin{scope}
%Points, labels a and b
\foreach \x/\t in {-1.5/a,-.5/,.5/,1.5/b} \filldraw (\x,0) circle (2pt) node[anchor=north,yshift=-.25cm,text depth=.25ex,text height=1.5ex] {$\t$};
%Line and label R
\draw[thick] (-1.5,0) -- (1.5,0);
\node[draw=none] at (3,0) {$R_{ab}=3$};
\end{scope}

%%%Single line segment
\begin{scope}[xshift=6.5cm]
%Points, labels a and b
\foreach \x/\t in {-.5/a,.5/b} \filldraw (\x,0) circle (2pt) node[anchor=north,yshift=-.25cm,text depth=.25ex,text height=1.5ex] {$\t$};
%Line and laebl R
\draw[thick] (-.5,0) -- (.5,0);
\node[draw=none] at (2.5,0) {$R_{ab}=1$};
\end{scope}

%%%Triangle
\begin{scope}[yshift=-2.5cm]
%Points, labels a and b
\foreach \x/\y/\t in {-.5/0/a,0/.75/,.5/0/b} \filldraw (\x,\y) circle (2pt) node[anchor=north,yshift=-.25cm,text depth=.25ex,text height=1.5ex] {$\t$};
%Triangle and label R
\draw[thick] (-.5,0) -- (0,.75) -- (.5,0) -- cycle;
\node[draw=none] at (3,0) {$R_{ab}=\frac{2}{3}$};
\end{scope}

%%%Line segment and circle
\begin{scope}[xshift=6.5cm,yshift=-2.5cm]
%Points, labels a and b
\foreach \x/\t/\p/\y in {-.5/a/left/-.4,.5/b/right/.4} \filldraw (\x,0) circle (2pt) node[text depth=.25ex,text height=1.5ex] at (\x+\y,0) {$\t$};
%Line, circle, and laebl R
\draw[thick] (-.5,0) -- (.5,0);
\draw[thick] (0,0) circle (.5cm);
\node[draw=none] at (2.5,0) {$R_{ab}=\frac{1}{3}$};
\end{scope}

%%%Circle
\begin{scope}[yshift=-5cm]
%Points, labels a and b
\foreach \x/\t/\p/\y in {-.5/a/left/-.4,.5/b/right/.4} \filldraw (\x,0) circle (2pt) node[text depth=.25ex,text height=1.5ex] at (\x+\y,0) {$\t$};
%Circle and label R
\draw[thick] (0,0) circle (.5cm);
\node[draw=none] at (3,0) {$R_{ab}=\frac{1}{2}$};
\end{scope}

%%%Cross circle
\begin{scope}[xshift=6.5cm,yshift=-5cm]
%Points, labels a and b
\foreach \x/\t/\p/\y in {-.5/a/left/-.4,.5/b/right/.4} \filldraw (\x,0) circle (2pt) node[text depth=.25ex,text height=1.5ex] at (\x+\y,0) {$\t$};
%Circle, cross elipse, and label R
\draw[thick] (0,0) circle (.5cm);
\draw[thick] (0,0) ellipse (.5cm and .1cm);
\node[draw=none] at (2.5,0) {$R_{ab}=\frac{1}{4}$};
\end{scope}

\end{tikzpicture}
\caption{The effective resistance between two points for several simple graphs.
Nodes that are farther apart have a larger effective resistance, while nodes that are nearer or better connected have a smaller effective resistance.}
\label{fig:eff-res}
\end{figure}
\end{center}
%%%%%% END IMAGE %%%%%%%%

\subsection*{Link Prediction} % -----------------------------------------------

\emph{Link prediction} is the problem of predicting the likelihood of a future association between two unconnected nodes in a graph.
Link prediction has application in many fields, but the canonical example is friend suggestions on Facebook.
The Facebook network can be represented by a large graph where each user is a node, and two nodes have an edge connecting them if they are ``friends.''
Facebook aims to predict who you would like to become friends with in the future, based on who you are friends with now, as well as discover which friends you may have in real life that you have not yet connected with online.
To do this, Facebook must have some way to measure how closely two users are connected.

We will compute link prediction using effective resistance as a metric.
Effective resistance measures how closely two nodes are connected, and nodes that are closely connected at present are more likely to be connected in the future.
Given an undirected graph, the next link should connect the two unconnected nodes with the least effective resistance between them.

\begin{problem}
Write a class called \li{LinkPredictor} for performing link prediction.
Implement the \li{\_\_init\_\_()} method so that it accepts the name of a \li{csv} file containing information about a social network.
Each row of the file should contain the names of two nodes which are connected by an (undirected) edge.

Store each of the names of the nodes of the graph as an ordered list.
Next, create the adjacency matrix for the network where the $i$th row and column of the matrix correspond to the $i$th member of the list of node names.
Finally, use your function from Problem \ref{prob:effective-resistance} to compute the effective resistance matrix.
Save the list of names, the adjacency matrix, and the effective resistance matrix as attributes.
\end{problem}

\begin{problem}
Implement the following methods in the \li{LinkPredictor} class:

\begin{enumerate}
\item \li{predict\_link()}: Accept a parameter \li{node} which is either \li{None} or a string representing a node in the network. If \li{node} is \li{None}, return a tuple with the names of the nodes between which the next link should occur. However, if \li{node} is a string, return the name of the node which should be connected to \li{node} next out of all other nodes in the network. If \li{node} is not in the network, raise a \li{ValueError}. Take the following into consideration:

\begin{enumerate}
\item You want to find the two nodes which have the smallest effective resistance between them which are not yet connected.
Use information from the adjacency matrix to zero out all entries of the effective resistance matrix that represent connected nodes. The ``\li{*}" operator multiplies arrays component-wise, which may be helpful.

\item Find the next link by finding the minimum value of the array that is nonzero.
Your array may be the whole matrix or just a column if you are only considering links for a certain node.
This can be accomplished by passing \li{np.<<min>>()} a masked version of your matrix to exclude entries that are $0$.

\item NumPy's \li{np.where()} is useful for finding the minimum value in an array:

\begin{lstlisting}
>>> A = np.random.randint(-9,9,(3,3))
>>> A
array([[ 6, -8, -9],
       [-2,  1, -1],
       [ 4,  0, -3]])

# Find the minimum value in the array.
>>> minval = np.<<min>>(A)
>>> minval
-9

# Find the location of the minimum value.
>>> loc = np.where(A==minval)
>>> loc
(array([0], dtype=int64), array([2], dtype=int64))
\end{lstlisting}
\end{enumerate}

\item \li{add\_link()}: Take as input two names of nodes, and add a link between them. If either name is not in the network, raise a \li{ValueError}. Add the link by updating the adjacency matrix and the effective resistance matrix.
\end{enumerate}

Figure \ref{fig:social-network} visualizes the data in \texttt{social\_network.csv}.
Use this graph to verify that your class is suggesting plausible new links.
You should observe the following:
\begin{itemize}
\item In the entire network, Emily and Oliver are most likely to become friends next.
\item Melanie is predicted to become friends with Carol next.
\item Alan is expected to become friends with Sonia, then with Piers, and then with Abigail.
\end{itemize}
\end{problem}

\begin{figure}[H]
\includegraphics[width=.7\linewidth]{network.png}

\caption{The social network contained in \texttt{social\_network.csv}.
Adapted from data by Wayne. W Zachary (see \url{https://en.wikipedia.org/wiki/Zachary\%27s\_karate\_club}).}
\label{fig:social-network}

\begin{multicols}{4}
\begin{enumerate}
    \item Piers
    \item Abigail
    \item Oliver
    \item Stephanie
    \item Carol
    \item Melanie
    \item Stephen
    \item Sally
    \item Penelope
    \item Alan
    \item Trevor
    \item Jake
    \item Mary
    \item Anna
    \item Ruth
    \item Evan
    \item Connor
    \item John
    \item Max
    \item Eric
    \item Theresa
    \item Paul
    \item Alexander
    \item Colin
    \item Jake
    \item Jane
    \item Brandon
    \item Thomas
    \item Christopher
    \item Charles
    \item Madeleine
    \item Tracey
    \item Sonia
    \item Emily
\end{enumerate}
\end{multicols}
\end{figure}


\subsection{Non-Uniform Rational B-Spline (NURBS)}
\label{lr_sec:NURBS}
\paragraph{}
In the proposed approach, the geometry and the unknown fields are represented by the NURBS.
In this section, a brief overview is given.
For more detailed description and implementation aspects, interested readers can refer to \citep{Pie1997,NGUYEN201589}.
\paragraph{}
% basis function
NURBS are the superset of B-spline functions.
B-spline is short for basis spline and is a generalization of Bézier curves.
A spline function is a piecewise polynomial function of degree $p$ and the points of intersection of such functions are called knots.
The number of knots must be equal to or greater than $p + 1$.
One of the salient features of spline functions is that the functions are continuous at the knots, however, the continuity of the functions can be altered by repeating the knots.
The B-spline functions are parametric functions of the form $F(\eta)$ in which the parameter $\eta$ lies in the parametric space.
The key ingredients in the construction of B-spline functions are: the knot vector (a non decreasing sequence of parameter values, $\eta_i \leq \eta_{i+1}$ , $i = 0,1,\dots,m -1$) and the degree of the curve p. The $i$th B-spline basis function of degree $p$, denoted by $_{i,p}$ is defined as \citep{Pie1997}:
\begin{equation}
    \begin{aligned}
        N_{i,0}(\eta) &=
            \begin{cases}
                1   & \eta \in [\eta_i, \eta_{i+1}]   \\
                0   & \eta \in (-\infty, \eta_i) \cup (\eta_{i+1}, \infty )
            \end{cases}\\
        N_{i,p}(\eta) &=
            \frac{\eta - \eta_i}{\eta_{i+p}-\eta_i}     N_{i,p-1}(\eta) -
            \frac{\eta_{i+p+1}-\eta}{\eta_{i+p+1} - \eta_{i+1}}     N_{i+1,p-1}(\eta)
    \end{aligned}
    \label{lr_nurbs_basis}
\end{equation}
%
The first derivative of the B-spline basis function can be computed recursively from lower order basis functions as:
\begin{equation}
    \frac{d}{d\eta} N_{i,p}(\eta) =
        \frac{p}{\eta_{i+p} - \eta_i} N_{i,p-1}(\eta) -
        \frac{p}{\eta_{i+p+1} - \eta_{i+1}} N_{i+1,p-1}(\eta)
\end{equation}
%
The B-spline basis functions has the following properties:
\begin{enumerate}
    \item Non-negativity
    \item Partition of unity, $\sum_i N_{i,p}=1$
    \item Interpolatory at the end points. The last point requires special treatment when imposing non-homogeneous Dirichlet boundary conditions \citep{NGUYEN201589}.
\end{enumerate}

\paragraph{}
% curves
Moreover, the spline function has limited support.
Given $n + 1$ control points $(P_0 ,P_1,\dots,P_n )$ and a knot vector
    $\Xi$ = $\left\{
        \eta_0 ,\eta_1 ,\dots,\eta_m
    \right\}$, the piecewise polynomial B-spline curve of degree p is defined as:
\begin{equation}
    C(\eta) = \sum_{i=0}^n P_i N_{i,p} (\eta)
\end{equation}
%
where $P_i$ are the control points.
A B-spline curve has the following information: $n+1$ control points, $m+1$ knots and a degree $p$.
It is noted that $n$,$m$ and $p$ must satisfy $m = n + p + 1$.
The B-spline functions also provide a variety of refinement algorithms, which are essential when employing B-spline functions to discretize the unknown fields.
The analogous $h$ and $p$ refinement can be done by the process of `knot insertion' and `order elevation'.
Another unique feature of the B-spline basis function is that, it is possible to combine the knot insertion and the degree elevation, commonly referred to as ‘k-refinement’ in the literature \citep{Hug2005b}.
Here we briefly discuss the knot insertion and the degree elevation.
For more details, interested readers are referred to \citep{Pie1997,Hug2005b} and references therein.

\subsubsection{Knot insertion}
\label{lr_sec:nurbs_knot_ins}
\paragraph{}
Consider a B-spline basis functions defined on
$\Xi = \left\{
    \eta_0 ,\eta_1,\dots,\eta_m
    \right\}$,
let $\overline{\eta} \in [\eta_k ,\eta_{k+1} )$, and insert $\overline{\eta}$ into $\Xi$ to form a new knot vector
$\overline{\Xi} = \left\{
    \eta_0 ,\dots, \overline{\eta}_k = \eta_k ,
    \overline{\eta}_{k+1} = \overline{\eta},
    \overline{\eta}_{k+2} = \overline{\eta}_{k+1} ,
    \dots,\eta_{m+1} = \eta_m
    \right\}$.
Simultaneously, the size of the control points is increased by one. Thus $C(\eta)$ has a representation on $\overline{\Xi}$ of the form
\begin{equation}
    \mathbf{C}(\eta) = \sum_{i=0}^{n+1}
                        \overline{N}_{i,p} (\eta)
                        \mathbf{Q}_i
\end{equation}
Where $\mathbf{Q}_i$ is:
\begin{equation}
    \mathbf{Q}_i = \alpha_i \mathbf{P}_i +
                    (1-\alpha_i) \mathbf{P}_{i-1}
\end{equation}
where
\begin{equation}
    \alpha_i =  \begin{cases}
                    1       & i \leq k-p \\
                    \frac{ \overline{\eta} -\eta_i }{ \eta_{i+p} - \eta_i } & k-p \leq i \leq k \\
                    0 & i \geq k+1
                \end{cases}
\end{equation}

\subsubsection{Order elevation}
\label{lr_sec:nurbs_order_ele}
\paragraph{}
Let $
\mathbf{C}(\eta) =  \sum_{i=0}^n
                    N_{i,p}(\eta)
                    \mathbf{P}_i
$
be a $p$th-degree B-spline curve on the knot vector $\Xi$.
As a piecewise polynomial curve with $p+1$ order, $\mathbf{C}(\eta)$ is expected to be expressed in higher order basis functions.
In other words, another set of control points $Q_i$ and knot vector $\Xi$ should exists such that
\begin{equation}
    \mathbf{C}(\eta) =  \sum_{i=0}^{\overline{n}}
                        N_{i, p+1}(\eta)
                        \mathbf{Q}_i
\end{equation}
The procedure to elevate the order of a B-spline is listed as follows \citep{Pie1997}:
\begin{enumerate}
    \item Extract each Bézier segment from the curve
    \item Elevate the order of each Bézier segment
    \item Remove unnecessary knots separating the ($i-1$)th and $i$th segments
\end{enumerate}
When elevating a $p$th Bézier curve, a new set of control points can be determined from:
\begin{equation}
    \mathbf{Q}_i =  (1-\alpha_i) \mathbf{P}_i +
                    \alpha_i \mathbf{P}_{i-1}
\end{equation}
where $\alpha_i=\frac{i}{p+i}$, $i=0$, $\dots$, $p+1$.
Fig.~\ref{lr_fig:nurbs_knotins} and Fig.~\ref{lr_fig:nurbs_orderele} show an example of basis function when performing a knot insertion and order elevation, respectively.

\begin{figure}[h!]
    \centering
    \input{literature/images/lr_nurbs_knotins.tikz}
    \caption{B-spline functoins: knot insertion}
    \label{lr_fig:nurbs_knotins}
\end{figure}

\begin{figure}[h!]
    \centering
    \input{literature/images/lr_nurbs_orderele.tikz}
    \caption{B-spline functoins: order elevation}
    \label{lr_fig:nurbs_orderele}
\end{figure}

Besides, it is found that if the order is elevated to $q$ and only then inserted a unique knot value, the basis would have $q-1$ continuous derivatives at the knot we inserted and this process is called $k$-refinement \citep{Hug2005b}.
Despite the flexibility offered by the B-splines, they lack the ability to exactly represent some shapes such as circle and ellipsoids.
To improve this, non-uniform rational B-splines (NURBS) are formed through rational functions of B-splines.
The NURBS thus form the superset of B-splines.
The key ingredients in the construction of NURBS basis functions are: the knot vector (a non decreasing sequence of parameter values, $\eta_i \leq \eta_{i+1}$, $i=0,1$,$\dots$, $m-1$), the degree of the curve $p$ and the weight associated to a control point, $w$.
A $p$th degree NURBS basis function is defined as follows:
\begin{equation}
    R(\eta) =   \frac{ N_{i,p}(\eta) w_i }{W(\eta)}
            =   \frac{ N_{i,p}(\eta) w_i }{
                    \sum_{i=0}^{n} N_{i,p}(\eta)w_i
                }
\label{iso_eq:rational_basis_function}
\end{equation}
where $w_i$ are the weights for the $i$th basis function $N_{i,p}(\eta)$.
Fig.~\ref{lr_fig:nurbs_rational_basis} shows the third order NURBS for an open knot vector.

\begin{figure}
    \centering
    \scalebox{0.5}{
        \includegraphics{literature/images/lr_nurbs_rational_basis.png}
    }
    \caption[3rd NURBS basis function]{
        3rd NURBS basis function for an open knot vector $\Xi$ = \{ -1, -1, -1, -1, -1/3, -1/3, -1/3, 0, 1/3, 1, 1, 1, 1 \}.
        Note that the functions are only interpolatory at the end points
    }
    \label{lr_fig:nurbs_rational_basis}
\end{figure}

The first derivative of a NURBS basis function is computed using the quotient rule and is given by:
\begin{equation}
    \frac{d}{d\eta}R_{i,p}(\eta) =  w_i\frac{
        N^\prime_{i,p}(\eta)W(\eta) - N_{i,p}(\eta) W^\prime(\eta)
    }{W(\eta)^2}
\end{equation}


\chapter{m-Reductions}
In the proofs of \Cref{theorem:halting,theorem:undefined-functions}, to prove
that a set $S$ is undecidable we proved that $S$ is decidable iff an undecidable
set $K$ should be also decidable, which lead to the conclusion that $S$ is
undecidable. This was done using ``reduction'' argument. We constructed a total
computable function $f : \N \to \N$ such that $f(x) \in S$ iff $x \in K$ for all
$x \in \N$. In
this chapter we will study this method in more details.

\begin{definition}
  Let $A, B \subseteq \N$. We say that $A$ is \emph{$m$-reducible} to
  $B$\footnote{%
    The letter ``m'' here stands for ``many-to-one''; however, Sipser's
    ``Introduction to the Theory of Computation'' suggests to call such
    reductions ``mapping reductions'' giving another life for the letter $m$ in
    this notation.
  }
  ($A \le_m B$) if there is a total computable function $f : \N \to \N$ such
  that $x \in A$ iff $f(x) \in B$ for all $x \in \N$. We say that $f$
  \emph{$m$-reduces} $A$ to $B$.
\end{definition}

This notion allows us to translate properties from one set to another.
\begin{theorem}
\label{theorem:m-reduction-complexity-preservation}
  Let $A, B \subseteq \N$ such that $A \le_m B$.
  \begin{itemize}
    \item If $B$ is decidable, then $A$ is decidable.
    \item If $B$ is enumerable, then $A$ is enumerable.
  \end{itemize}
\end{theorem}

\begin{exercise}
  Let $A, B, C \subseteq \N$. Show that
  \begin{itemize}
    \item $A \le_m A$, and
    \item if $A \le_m B$ and $B \le_m C$, then $A \le_m C$.
  \end{itemize}
\end{exercise}

Notice that the sets $\emptyset$ and $\N$ behave differently than other
decidable sets with respect to $\le_m$.
\begin{remark}
  Let $A, B \subseteq \N$.
  \begin{itemize}
    \item $A \le_m \emptyset$ iff $A = \emptyset$,
    \item $A \le_m \N$ iff $A = \N$, and
    \item $A \le_m B$ provided that $A$ and $B$ are dicidable and $B \not\in
      \set{\emptyset, \N}$.
  \end{itemize}
\end{remark}

However, in case of enumerable sets, the situation is not that simple.
\begin{exercise}
  Show that there are enumerable sets $A, B \subseteq \N$ such that $A \not\le_m
  B$.
\end{exercise}

In other words, enumerable sets form layers of sets increasing with repspect to
$\le_m$. So the questions is whether there is the last layer or not, the
following theorem give an affirmative answer to this question.
\begin{theorem}
  In the class of enumerable sets, there are sets maximal with respect to
  $m$-reducibility; i.e., there is an enumerable set $B$ sucht that $A \le_m B$
  for any enumerable set $A$.
\end{theorem}
\begin{proof}
  Let $W$ be a enumerable universal set for the set of all enumerable subsets of
  $\N$ (it exists by \Cref{theorem:universal-set-enumerable}).
  We claim that the set $B = \set[(n, x) \in W]{\pair{n}{x}}$ satisfies the
  requirement of the theorem. Indeed, let $A$ be enumerable set. Then there is
  $n \in \N$ such that $W_n = A$. Hence, it is easy to see that $f(x) \in B$ iff
  $n \in A$, where $f(x) = \pair{n}{x}$.
\end{proof}

\begin{definition}
  An enumerable set $B$ maximal with respect to $m$-reducibility is called
  \emph{$m$-complete for the class of enumerable sets}.
\end{definition}

\begin{theorem}
\label{theorem:m-complete-enumerable}
  Let $U$ be a G\"odel universal function.
  Then $\set[U(x, x) \text{ terminates}]{x \in \N}$ is $m$-complete for the
  class of enumerable sets.
\end{theorem}
\begin{proof}
  Let $K \subseteq \N$ be a enumerable set. Let us consider a computable
  function $V : \N^2 \to \N$s such that $V(n, x) = 1$ if $n \in K$ and undefined
  otherwise. Since $U$ is G\"odel universal function, there is a total
  computable $s : \N \to \N$ such that $V_n = U_{s(n)}$. Hence,  $U_{s(n)}$
  decides $\N$ if $n \in K$ and $U_{s(n)}$ decides $\emptyset$ if $K \not\in K$.
  Therefore $s(n) \in D$ iff $n \in K$.
\end{proof}

\begin{chapterendexercises}
  \exercise Prove that the set of all programs that halt on the input $0$ is
    $m$-complete for the class of enumerable sets.
  \exercise Prove that the set of all programs that halt on at least one input
  is $m$-complete for the class of enumerable sets.
\end{chapterendexercises}


%
% Chapter 2.3
%

\section*{2.3 Differentiation Formulae}

\begin{enumerate}
    \item \(\frac{d}{dx}(c)=0\) \quad where \(c\) is a constant
    \item \(\frac{d}{dx}(x)=1\)
    \item \(\frac{d}{dx}(x^n)=nx^{n-1}\) \quad if \(n\) is a positive integer
    \item \(\frac{d}{dx}[cf(x)]=c\frac{d}{dx}f(x)\) \quad where \(c\) is a constant and \(f(x)\) is differentiable
    \item \(\frac{d}{dx}[f(x)+g(x)]=\frac{d}{dx}f(x)+\frac{d}{dx}g(x)\) \quad if \(f(x)\) and \(g(x)\) are differentiable
    \item \(\frac{d}{dx}[f(x)-g(x)]=\frac{d}{dx}f(x)-\frac{d}{dx}g(x)\) \quad if \(f(x)\) and \(g(x)\) are differentiable
    \item \(\frac{d}{dx}[f(x)g(x)]=f(x)\frac{d}{dx}[g(x)]+g(x)\frac{d}{dx}[f(x)]\) \quad if \(f\) and \(g\) are differentiable
    \item \(\frac{d}{dx}[\frac{f(x)}{g(x)}]=\frac{g(x)\frac{d}{dx}[f(x)]-f(x)\frac{d}{dx}[g(x)]}{g{(x)}^2}\) \quad if \(f\) and \(g\) are differentiable
\end{enumerate}
\begin{enumerate}
    \item The derivative of a constant is 0.
    \item The derivative of \(x\) is 1.
    \item The general power rule
    \item The derivative of a constant times a function is the constant times the derivative of the function.
    \item The derivative of a sum of functions is the sum of the derivatives.
    \item The derivative of the difference is the difference of the derivatives.
    \item The product rule
    \item The quotient rule
\end{enumerate}


\chapter{Determinant}
The goal of this chapter is to give the basis-free
definition of the determinant:
that is, we're going to define $\det T$
for $T \colon V \to V$ without making reference to the encoding for $T$.
This will make it obvious the determinant of a matrix
does not depend on the choice of basis,
and that several properties are vacuously true
(e.g.\ that the determinant is multiplicative).

The determinant is only defined for finite-dimensional
vector spaces, so if you want you can restrict
your attention to finite-dimensional vector spaces for this chapter.
On the other hand we do not need the
ground field to be algebraically closed.

\section{Wedge product}
\prototype{$\Lambda^2(\RR^2)$ gives parallelograms.}
We're now going to define something called the wedge product.
It will look at first like the tensor product $V \otimes V$,
but we'll have one extra relation.

For simplicity, I'll first define the wedge product $\Lambda^2(V)$.
But we will later replace $2$ with any $n$.

\begin{definition}
	Let $V$ be a $k$-vector space.
	The $2$-wedge product $\Lambda^2(V)$ is the abelian group
	generated by elements of the form $v \wedge w$ (where $v,w \in V$),
	subject to the same relations
	\begin{align*}
		(v_1 + v_2) \wedge w &= v_1 \wedge w + v_2 \wedge w \\
		v \wedge (w_1 + w_2) &= v \wedge w_1 + v \wedge w_2 \\
		(c \cdot v) \wedge w &= v \wedge (c \cdot w)
	\end{align*}
	plus two additional relations:
	\[ v \wedge v = 0 \quad\text{and}\quad
		v \wedge w = - w \wedge v. \]
	As a vector space, its action is given by
	$c \cdot (v \wedge w) = (c \cdot v) \wedge w = v \wedge (c \cdot w)$.
\end{definition}
\begin{exercise}
	Show that the condition $v \wedge w = - (w \wedge v)$
	is actually extraneous:
	you can derive it from the fact that $v \wedge v = 0$.
	(Hint: expand $(v + w) \wedge (v + w) = 0$.)
\end{exercise}

This looks almost exactly the same as the definition for a tensor product,
with two subtle differences.
The first is that we only have $V$ now, rather than $V$ and $W$
as with the tensor product\footnote{So maybe the wedge product
	might be more accurately called the ``wedge power''!}
Secondly, there is a new \emph{mysterious} relation
\[ v \wedge v = 0 \implies v \wedge w = - (w \wedge v). \]
What's that doing there?
It seems kind of weird.

I'll give you a hint.
\begin{example}
	[Wedge product explicit computation]
	Let $V = \RR^2$, and let $v = ae_1 + be_2$, $w = ce_1 + de_2$.
	Now let's compute $v \wedge w$ in $\Lambda^2(V)$.
	\begin{align*}
		v \wedge w &= (ae_1 + be_2) \wedge (ce_1 + de_2) \\
		&= ac (e_1 \wedge e_1) + bd (e_2 \wedge e_2)
		+ ad (e_1 \wedge e_2) + bc (e_2 \wedge e_1) \\
		&= ad (e_1 \wedge e_2) + bc (e_2 \wedge e_1) \\
		&= (ad-bc) (e_1 \wedge e_2).
	\end{align*}
\end{example}

What is $ad-bc$? You might already recognize it:
\begin{itemize}
	\ii You might know that the area of the parallelogram
	formed by $v$ and $w$ is $ad-bc$.
	\ii You might recognize it as the determinant of
	$ \begin{bmatrix} a & c \\ b & d \end{bmatrix}$.
	In fact, you might even know that the determinant
	is meant to interpret hypervolumes.
\end{itemize}

\begin{center}
\begin{asy}
	pair v = (4,1);
	pair w = (2,3);
	dot("$0$", origin, dir(225));
	dot("$v = ae_1 + be_2$", v, dir(-45), red);
	dot("$w = ce_1 + de_2$", w, dir(135), red);
	dot("$v+w$", v+w, dir(45));
	label("$ad-bc$", (v+w)/2, blue);
	fill(origin--v--(v+w)--w--cycle, opacity(0.1)+lightcyan);
	draw(origin--v, EndArrow, Margins);
	draw(origin--w, EndArrow, Margins);
	draw(v--(v+w), EndArrow, Margins);
	draw(w--(v+w), EndArrow, Margins);
\end{asy}
\end{center}

This is absolutely no coincidence.
The wedge product is designed to interpret signed areas.
That is, $v \wedge w$ is meant to interpret the area of the parallelogram
formed by $v$ and $w$.
You can see why the condition $(cv) \wedge w = v \wedge (cw)$ would make sense now.
And now of course you know why $v \wedge v$ ought to be zero:
it's an area zero parallelogram!

The \textbf{miracle of wedge products} is that the only additional condition
we need to add to the tensor product axioms is that $v \wedge w = -(w \wedge v)$.
Then suddenly, the wedge will do all our work of interpreting volumes for us.

In analog to earlier:
\begin{proposition}
	[Basis of $\Lambda^2(V)$]
	Let $V$ be a vector space
	with basis $e_1$, \dots, $e_n$.
	Then a basis of $\Lambda^2(V)$ is
	\[ e_i \wedge e_j \]
	where $i < j$.
	Hence $\Lambda^2(V)$ has dimension $\binom n2$.
\end{proposition}
\begin{proof}
	Surprisingly slippery, and also omitted.
	(You can derive it from the corresponding theorem on tensor products.)
\end{proof}

Now I have the courage to define a multi-dimensional wedge product.
It's just the same thing with more wedges.
\begin{definition}
	Let $V$ be a vector space and $m$ a positive integer.
	The space $\Lambda^m(V)$ is generated by wedges of the form
	\[ v_1 \wedge v_2 \wedge \dots \wedge v_m \]
	subject to relations
	\begin{align*}
		\dots \wedge (v_1+v_2) \wedge \dots
			&= (\dots \wedge v_1 \wedge \dots)
			 + (\dots \wedge v_2 \wedge \dots) \\
		\dots \wedge (cv_1) \wedge v_2 \wedge \dots
			&= \dots \wedge v_1 \wedge (cv_2) \wedge \dots  \\
		\dots \wedge v \wedge v \wedge \dots &= 0 \\
		\dots \wedge v \wedge w \wedge \dots &=
			- (\dots \wedge w \wedge v \wedge \dots)
	\end{align*}
	As a vector space
	\[ c \cdot (v_1 \wedge v_2 \wedge \dots \wedge v_m)
	 = (cv_1) \wedge v_2 \wedge \dots \wedge v_m
	 = v_1 \wedge (cv_2) \wedge \dots \wedge v_m
	 = \dots .
	\]
\end{definition}
This definition is pretty wordy, but in English the three conditions say
\begin{itemize}
	\ii We should be able to add products like before,
	\ii You can put constants onto any of the $m$ components
	(as is directly pointed out in the ``vector space'' action), and
	\ii Switching any two \emph{adjacent} wedges negates the whole wedge.
\end{itemize}
So this is the natural generalization of $\Lambda^2(V)$.
You can convince yourself that any element of the form
\[ \dots \wedge v \wedge \dots \wedge v \wedge \dots \]
should still be zero.

Just like $e_1 \wedge e_2$ was a basis earlier, we can find the basis
for general $m$ and $n$.
\begin{proposition}[Basis of the wedge product]
	Let $V$ be a vector space with basis $e_1, \dots, e_n$.
	A basis for $\Lambda^m(V)$ consists of the elements
	\[ e_{i_1} \wedge e_{i_2} \wedge \dots \wedge e_{i_m} \]
	where
	\[ 1 \le i_1 < i_2 < \dots < i_m \le n. \]
	Hence $\Lambda^m(V)$ has dimension $\binom nm$.
\end{proposition}
\begin{proof}[Sketch of proof]
	We knew earlier that $e_{i_1} \otimes \dots \otimes e_{i_m}$
	was a basis for the tensor product.
	Here we have the additional property that (a)
	if two basis elements re-appear then the whole thing becomes zero,
	thus we should assume the $i$'s are all distinct;
	and (b) we can shuffle around elements,
	and so we arbitrarily decide to put the basis elements
	in increasing order.
\end{proof}

\section{The determinant}
\prototype{$(ae_1+be_2)\wedge(ce_1+de_2) = (ad-bc)(e_1\wedge e_2)$.}
Now we're ready to define the determinant.
Suppose $T \colon V \to V$ is a square matrix.
We claim that the map $\Lambda^m(V) \to \Lambda^m(V)$ given on wedges by
\[ v_1 \wedge v_2 \wedge \dots \wedge v_m
	\mapsto T(v_1) \wedge T(v_2) \wedge \dots \wedge T(v_m) \]
and extending linearly to all of $\Lambda^m(V)$ is a linear map.
(You can check this yourself if you like.)
We call that map $\Lambda^m(T)$.
\begin{example}
	[Example of $\Lambda^m(T)$]
	In $V = \RR^4$ with standard basis $e_1$, $e_2$, $e_3$, $e_4$,
	let $T(e_1) = e_2$, $T(e_2) = 2e_3$, $T(e_3) = e_3$ and $T(e_4) = 2e_2 + e_3$.
	Then, for example, $\Lambda^2(T)$ sends
	\begin{align*}
		e_1 \wedge e_2 + e_3 \wedge e_4
		&\mapsto T(e_1) \wedge T(e_2) + T(e_3) \wedge T(e_4) \\
		&= e_2 \wedge 2e_3 + e_3 \wedge (2e_2 + e_3) \\
		&= 2(e_2 \wedge e_3 + e_3 \wedge e_2) \\
		&= 0.
	\end{align*}
\end{example}

Now here's something interesting.
Suppose $V$ has dimension $n$, and let $m=n$.
Then $\Lambda^n(V)$ has dimension $\binom nn = 1$ --- it's a one dimensional space!
Hence $\Lambda^n(V) \cong k$.

So $\Lambda^n(T)$ can be thought of as a linear map from $k$ to $k$.
But we know that \emph{a linear map from $k$ to $k$ is just multiplication by a constant}.
Hence $\Lambda^n(T)$ is multiplication by some constant.
\begin{definition}
	Let $T \colon V \to V$, where $V$ is an $n$-dimensional vector space.
	Then $\Lambda^n(T)$ is multiplication by a constant $c$;
	we define the \vocab{determinant} of $T$ as $c = \det T$.
\end{definition}

\begin{example}[The determinant of a $2 \times 2$ matrix]
	Let $V = \RR^2$ again with basis $e_1$ and $e_2$.
	Let
	\[ T = \begin{bmatrix}
			a & c \\ b & d
		\end{bmatrix}.
	\]
	In other words, $T(e_1) = ae_1 + be_2$
	and $T(e_2) = ce_1 + de_2$.

	Now let's consider $\Lambda^2(V)$.
	It has a basis $e_1 \wedge e_2$.
	Now $\Lambda^2(T)$ sends it to
	\[ e_1 \wedge e_2 \xmapsto{\Lambda^2(T)}
		T(e_1) \wedge T(e_2) =
		(ae_1 + be_2) \wedge (ce_1 + de_2)
		= (ad-bc)(e_1 \wedge e_2).
	\]
	So $\Lambda^2(T) : \Lambda^2(V) \to \Lambda^2(V)$
	is multiplication by $\det T = ad-bc$,
	because it sent $e_1 \wedge e_2$ to
	$(ad-bc)(e_1 \wedge e_2)$.
\end{example}
And that is the definition of a determinant.
Once again, since we defined it in terms of $\Lambda^n(T)$,
this definition is totally independent of the choice of basis.
In other words, the determinant can be defined based on $T \colon V \to V$ alone
without any reference to matrices.

\begin{ques}
	Why does $\Lambda^n(S \circ T) = \Lambda^n(S) \circ \Lambda^n(T)$?
\end{ques}
In this way, we also get \[ \det(S \circ T) = \det(S) \det(T) \] for free.

More generally if we replace $2$ by $n$,
an write out the result of expanding
\[ \left( a_{11}e_1 + a_{21}e_2 + \dots \right) \wedge \dots \wedge
	\left( a_{1n}e_1 + a_{2n}e_2 + \dots + a_{nn} e_n \right) \]
then you will get the formula
\[ \det(A) = \sum_{\sigma \in S_n} \opname{sgn}(\sigma)
	a_{1, \sigma(1)} a_{2, \sigma(2)} \dots a_{n, \sigma(n)} \]
called the \vocab{Leibniz formula} for determinants.
American high school students will recognize it;
this is (unfortunately) taught as the definition of the determinant,
rather than a corollary of the better definition using wedge products.

\begin{exercise}
	Verify that expanding the wedge product
	yields the Leibniz formula for $n=3$.
\end{exercise}

\section{Characteristic polynomials, and Cayley-Hamilton}
Let's connect with the theory of eigenvalues.
Take a map $T \colon V \to V$, where $V$ is $n$-dimensional
over an algebraically closed field,
and suppose its eigenvalues
are $\lambda_1$, $\lambda_2$, \dots, $\lambda_n$ (with repetition).
Then the \vocab{characteristic polynomial} is given by
\[
	p_T(X) = (X-\lambda_1)(X-\lambda_2) \dots (X-\lambda_n).
\]
Note that if we've written $T$ in Jordan form, that is,
\[
	T = \begin{bmatrix}
		\lambda_1 & \ast & 0 & \dots & 0 \\
		0 & \lambda_2 & \ast & \dots & 0 \\
		0 & 0 & \lambda_3 & \dots & 0 \\
		\vdots & \vdots & \vdots & \ddots & \vdots \\
		0 & 0 & 0 & \dots & \lambda_n
	\end{bmatrix}
\]
(here each $\ast$ is either $0$ or $1$),
then we can hack together the definition
\[
	p_T(X) \defeq
	\det \left( X \cdot \id_n - T \right)
	= \det \begin{bmatrix}
		X - \lambda_1 & \ast & 0 & \dots & 0 \\
		0 & X - \lambda_2 & \ast & \dots & 0 \\
		0 & 0 & X - \lambda_3 & \dots & 0 \\
		\vdots & \vdots & \vdots & \ddots & \vdots \\
		0 & 0 & 0 & \dots & X - \lambda_n
	\end{bmatrix}.
\]
The latter definition is what you'll see in most
linear algebra books because it lets you define the characteristic polynomial
without mentioning the word ``eigenvalue''
(i.e.\ entirely in terms of arrays of numbers).
I'll admit it does have the merit that it means that given any matrix,
it's easy to compute the characteristic polynomial and hence
compute the eigenvalues;
but I still think the definition should be done in terms of
eigenvalues to begin with.
For instance the determinant definition obscures the following theorem,
which is actually a complete triviality.
\begin{theorem}[Cayley-Hamilton]
	Let $T \colon V \to V$ be a map of finite-dimensional
	vector spaces over an algebraically closed field.
	Then for any $T \colon V \to V$,
	the map $p_T(T)$ is the zero map.
\end{theorem}
Here, by $p_T(T)$ we mean that if
\[ p_T(X) = X^n + c_{n-1} X^{n-1} + \dots + c_0 \]
then \[ p_T(T) = T^n + c_{n-1} T^{n-1} + \dots + c_1 T +  c_0 I \]
is the zero map,
where $T^k$ denotes $T$ applied $k$ times.
We saw this concept already when we proved
that $T$ had at least one nonzero eigenvector.

\begin{example}[Example of Cayley-Hamilton using determinant definition]
	Suppose $T = \begin{bmatrix} 1 & 2 \\ 3 & 4 \end{bmatrix}$.
	Using the determinant definition of characteristic polynomial,
	we find that $p_T(X) = (X-1)(X-4)-(-2)(-3) = X^2 - 5X - 2$.
	Indeed, you can verify that
	\[ T^2 - 5T - 2
		= \begin{bmatrix}
			7 & 10 \\
			15 & 22
		\end{bmatrix}
		- 5 \cdot \begin{bmatrix}
			1 & 2 \\
			3 & 4
		\end{bmatrix}
		- 2 \cdot \begin{bmatrix}
			1 & 0 \\
			0 & 1
		\end{bmatrix}
		= \begin{bmatrix}
			0 & 0 \\
			0 & 0
		\end{bmatrix}.
	\]
\end{example}
If you define $p_T$ without the word eigenvalue,
and adopt the evil view that matrices are arrays of numbers,
then this looks like a complete miracle.
(Indeed, just look at the terrible proofs on Wikipedia.)

But if you use the abstract viewpoint of $T$ as a linear map,
then the theorem is almost obvious:
\begin{proof}[Proof of Cayley-Hamilton]
	Suppose we write $V$ in Jordan normal form as
	\[ V = J_1 \oplus \dots \oplus J_m \]
	where $J_i$ has eigenvalue $\lambda_i$ and dimension $d_i$.
	By definition,
	\[ p_T(T) = (T - \lambda_1)^{d_1} (T - \lambda_2)^{d_2}
		\dots (T - \lambda_m)^{d_m}. \]
	By definition, $(T - \lambda_1)^{d_1}$ is the zero map on $J_1$.
	So $p_T(T)$ is zero on $J_1$.
	Similarly it's zero on each of the other $J_i$'s --- end of story.
\end{proof}
\begin{remark}
	[Tensoring up]
	The Cayley-Hamilton theorem holds without the hypothesis that
	$k$ is algebraically closed:
	because for example any real matrix can be regarded
	as a matrix with complex coefficients
	(a trick we've mentioned before).
	I'll briefly hint at how you can use tensor products to formalize this idea.

	Let's take the space $V = \RR^3$, with basis $e_1$, $e_2$, $e_3$.
	Thus objects in $V$ are of the form $r_1 e_1 + r_2 e_2 + r_3 e_3$
	where $r_1$, $r_2$, $r_3$ are real numbers.
	We want to consider essentially the same vector space,
	but with complex coefficients $z_i$ rather than real coefficients $r_i$.

	So here's what we do: view $\CC$ as a $\RR$-vector space
	(with basis $\{1,i\}$, say)
	and consider the \vocab{complexification}
	\[ V_\CC \defeq \CC \otimes_\RR V. \]
	Then you can check that our elements are actually of the form
	\[ z_1 \otimes e_1 + z_2 \otimes e_2 + z_3 \otimes e_3. \]
	Here, the tensor product is over $\RR$,
	so we have $z \otimes re_i = (zr) \otimes e_i$ for $r \in \RR$.
	Then $V_{\CC}$ can be thought as a three-dimensional vector space over $\CC$,
	with basis $1 \otimes e_i$ for $i \in \{1,2,3\}$.
	In this way, the tensor product lets us formalize the idea
	that we ``fuse on'' complex coefficients.

	If $T \colon V \to W$ is a map, then $T_\CC \colon V_\CC \to W_\CC$
	is just the map $z \otimes v \mapsto z \otimes T(v)$.
	You'll see this written sometimes as $T_\CC = \id \otimes T$.
	One can then apply theorems to $T_\CC$
	and try to deduce the corresponding results on $T$.
\end{remark}

\section\problemhead
\begin{problem}[Column operations]
	Show that for any real numbers $x_{ij}$ (here $1 \le i,j \le n$) we have
	\[
		\det
		\begin{bmatrix}
			x_{11} & x_{12} & \dots & x_{1n} \\
			x_{21} & x_{22} & \dots & x_{2n} \\
			\vdots & \vdots & \ddots & \vdots \\
			x_{n1} & x_{n2} & \dots & x_{nn} \\
		\end{bmatrix}
		=
		\det
		\begin{bmatrix}
			x_{11} + cx_{12} & x_{12} & \dots & x_{1n} \\
			x_{21} + cx_{22} & x_{22} & \dots & x_{2n} \\
			\vdots & \vdots & \ddots & \vdots \\
			x_{n1} + cx_{n2} & x_{n2} & \dots & x_{nn} \\
		\end{bmatrix}.
	\]
	\begin{hint}
		The point is that
		\[
			(v_1+cv_2) \wedge v_2 \dots \wedge v_n
			= v_1 \wedge v_2 \dots \wedge v_n
			+ c(v_2 \wedge v_2 \dots \wedge v_n)
		\]
		and the latter term is zero.
	\end{hint}
\end{problem}
\begin{problem}
	[Determinant is product of eigenvalues]
	Let $V$ be an $n$-dimensional vector space
	over an algebraically closed field $k$.
	Let $T \colon V \to V$ be a linear map with
	eigenvalues $\lambda_1$, $\lambda_2$, \dots, $\lambda_n$
	(counted with algebraic multiplicity).
	Show that $\det T = \lambda_1 \dots \lambda_n$.
	\begin{hint}
		You can either do this by writing $T$ in matrix form,
		or you can use the wedge definition of $\det T$
		with the basis given by Jordan form.
	\end{hint}
\end{problem}

\begin{problem}
	[Exponential matrix]
	Let $X$ be an $n \times n$ matrix with complex coefficients.
	We define the exponential map by
	\[ \exp(X) = 1 + X + \frac{X^2}{2!} + \frac{X^3}{3!} + \dots \]
	(take it for granted that this converges to some $n \times n$ matrix).
	Prove that
	\[ \det(\exp(X)) = e^{\Tr X}. \]
	\begin{hint}
		This is actually immediate by taking any basis
		in which $X$ is upper-triangular!
	\end{hint}
\end{problem}

\begin{problem}
	[Extension to \Cref{prob:equal_dimension}]
	Let $T \colon V \to V$ be a map of finite-dimensional vector spaces.
	Prove that $T$ is an isomorphism
	if and only if $\det T \ne 0$.
	\begin{hint}
		You don't need eigenvalues (though they could work also).
		In one direction, recall that (by \Cref{prob:equal_dimension})
		we can replace ``isomorphism'' by ``injective''.
		In the other, if $T$ is an isomorphism,
		let $S$ be the inverse map and look at $\det(S \circ T)$.
	\end{hint}
	\begin{sol}
		Recall that (by \Cref{prob:equal_dimension})
		we can replace ``isomorphism'' by ``injective''.

		If $T(v) = 0$ for any nonzero $v$,
		then by taking a basis for which $e_1 = v$,
		we find $\Lambda^n(T)$ will map $e_1 \wedge \dots$
		to $0 \wedge T(e_2) \wedge \dots = 0$,
		hence is the zero map, so $\det T = 0$.

		Conversely, if $T$ is an isomorphism,
		we let $S$ denote the inverse map.
		Then $1 = \det(\id) = \det(S \circ T) = \det S \det T$,
		so $\det T \ne 0$.
	\end{sol}
\end{problem}

\begin{problem}
	[Based on Sweden 2010]
	\gim
	A herd of $1000$ cows of nonzero weight is given.
	Prove that we can remove one cow such that the
	remaining $999$ cows cannot be split
	into two halves of equal weights.
	\begin{hint}
		Consider $1000 \times 1000$ matrix $M$
		with entries $0$ on diagonal and $\pm 1$ off-diagonal.
		Mod $2$.
	\end{hint}
	\begin{sol}
		We proceed by contradiction.
		Let $v$ be a vector of length $1000$
		whose entries are weight of cows.
		Assume the existence of a matrix $M$ such that $Mv = 0$,
		with entries $0$ on diagonal and $\pm 1$ off-diagonal.
		But $\det M \pmod 2$ is equal to the number of derangements
		of $\{1, \dots, 1000\}$, which is odd.
		Thus $\det M$ is odd and in particular not zero,
		so $M$ is invertible.
		Thus $Mv = 0 \implies v = 0$, contradiction.
	\end{sol}
\end{problem}

\begin{problem}
	[Putnam 2015]
	\yod
	Define $S$ to be the set of real matrices
	$\left(\begin{smallmatrix} a & b \\ c & d \end{smallmatrix}\right)$
	such that $a$, $b$, $c$, $d$ form
	an arithmetic progression in that order.
	Find all $M \in S$ such that for some integer $k > 1$, $M^k \in S$.
	\begin{hint}
		There is a family of solutions other than just $a=b=c=d$.

		One can solve the problem using Cayley-Hamilton.
		A more ``bare-hands'' approach is to
		show the matrix is invertible (unless $a=b=c=d$)
		and then diagonalize the matrix as
		$
		M =
		\begin{bmatrix} s & -q \\ -r & p \end{bmatrix}
		\begin{bmatrix} \lambda_1 & 0 \\ 0 & \lambda_2 \end{bmatrix}
		\begin{bmatrix} p & q \\ r & s \end{bmatrix}
		=
		\begin{bmatrix}
			ps\lambda_1 - qr\lambda_2 & qs(\lambda_1-\lambda_2) \\
			pr(\lambda_2-\lambda_1) & ps\lambda_2 - qr\lambda_1
		\end{bmatrix}
		$.
	\end{hint}
	\begin{sol}
		The answer is
		\[ \begin{bmatrix} t&t\\t&t \end{bmatrix}
			\quad\text{and}\quad
			\begin{bmatrix} -3t&-t\\t&3t \end{bmatrix} \]
		for $t \in \RR$.
		These work by taking $k=3$.

		Now to see these are the only ones, consider an arithmetic matrix
		\[ M = \begin{bmatrix} a & a+e \\ a+2e & a+3e \end{bmatrix}. \]
		with $e \neq 0$.
		Its characteristic polynomial is $t^2 - (2a+3e)t - 2e^2$,
		with discriminant $(2a+3e)^2 + 8e^2$,
		so it has two distinct real roots; moreover, since $-2e^2 \le 0$
		either one of the roots is zero or they are of opposite signs.
		Now we can diagonalize $M$ by writing
		\[
			M =
			\begin{bmatrix} s & -q \\ -r & p \end{bmatrix}
			\begin{bmatrix} \lambda_1 & 0 \\ 0 & \lambda_2 \end{bmatrix}
			\begin{bmatrix} p & q \\ r & s \end{bmatrix}
			=
			\begin{bmatrix}
				ps\lambda_1 - qr\lambda_2 & qs(\lambda_1-\lambda_2) \\
				pr(\lambda_2-\lambda_1) & ps\lambda_2 - qr\lambda_1
			\end{bmatrix}
		\]
		where $ps-qr=1$. By using the fact the diagonal entries have sum equalling
		the off-diagonal entries, we obtain that
		\[ (ps-qr)(\lambda_1+\lambda_2) = (qs-pr)(\lambda_1-\lambda_2)
			\implies qs-pr = \frac{\lambda_1+\lambda_2}{\lambda_1-\lambda_2}. \]
		Now if $M^k \in S$ too then the same calculation gives
		\[ qs-pr = \frac{\lambda_1^k+\lambda_2^k}{\lambda_1^k-\lambda_2^k}. \]
		Let $x = \lambda_1/\lambda_2 < 0$ (since $-2e^2 < 0$). We appropriately get
		\[ \frac{x+1}{x-1} = \frac{x^k+1}{x^k-1}
			\implies \frac{2}{x-1} = \frac{2}{x^k-1}
			\implies x = x^k \implies x = -1 \text{ or } x = 0 \]
		and $k$ odd. If $x=0$ we get $e=0$ and if $x=-1$ we get $2a+3e=0$,
		which gives the curve of solutions that we claimed.

		A slicker approach is by Cayley-Hamilton.
		Assume that $e \neq 0$, so $M$ has two distinct real eigenvalues as above.
		We have $M^k = cM + d\id$ for some constants $c$ and $d$
		(since $M$ satisfies some quadratic polynomial).
		Since $M \in S$, $M^k \in S$ we obtain $d=0$.
		Thus $M^k = cM$, so it follows the eigenvalues of $M$ are negatives of each other.
		That means $\Tr M = 0$, and the rest is clear.
	\end{sol}
\end{problem}

%\begin{problem}[USAMO 2008, edited]
%	At a certain mathematical conference,
%	every two mathematicians are either friends or strangers.
%	At mealtime, every participant eats in one of two large dining rooms.
%	Each mathematician insists upon eating in a room which contains an
%	even number of his or her friends.
%	Assuming that such a split exists, prove that the number of ways
%	that the mathematicians may be split between the two rooms is a power of two.
%	% http://www.artofproblemsolving.com/Forum/viewtopic.php?p=3338962#p3338962
%\end{problem}

\begin{problem}
	\yod
	Let $V$ be a finite-dimensional vector space over $k$ and $T : V \to V$.
	Show that
	\[
		\det(a \cdot \id_V - T) =
		\sum_{n=0}^{\dim V} a^{\dim V-n} \cdot (-1)^n
		\Tr\left( \Lambda^n(T) \right)
	\]
	where the trace is taken by viewing $\Lambda^n(T) : \Lambda^n(V) \to \Lambda^n(V)$.
	\begin{hint}
		Take bases, and do a fairly long calculation.
	\end{hint}
	\begin{sol}
		\newcommand{\Fix}{\opname{Fix}}
		\newcommand{\NoFix}{\opname{NoFix}}
		Pick a basis $e_1, \dots, e_n$ of $V$.
		Let $T$ have matrix $(x_{ij})$, and let $m = \dim V$.
		Let $\delta_{ij}$ be the Kronecker delta.
		Also, let $\Fix(\sigma)$ denote the fixed points of a permutation $\sigma$
		and let $\NoFix(\sigma)$ denote the non-fixed points.

		Expanding then gives
		\begin{align*}
			&\qquad \det (a \cdot \id - T) \\
			&= \sum_{\sigma \in S_m} \left( \sign(\sigma)
			\cdot \prod_{i=1}^m \left( a \cdot \delta_{i \sigma(i)} - x_{i \sigma(i)} \right)\right) \\
			% ------------------------
			&=
			\sum_{s=0}^m
			\sum_{1 \le i_1 < \dots < i_s \le m}
			\sum_{\substack{\sigma \in S_m \\ \sigma \text{ fixes } i_k}}
			\left( \sign(\sigma)
			\cdot \prod_{i=1}^m \left( a \cdot \delta_{i \sigma(i)} - x_{i \sigma(i)} \right)\right) \\
			% ------------------------
			&=
			\sum_{s=0}^m
			\sum_{1 \le i_1 < \dots < i_s \le m}
			\sum_{\substack{\sigma \in S_m \\ \sigma \text{ fixes } (i_k)}}
			\left( \sign(\sigma)
			\cdot \prod_{i \notin (i_k)} -x_{i \sigma(i)}
			\prod_{i \in (i_k)}^n \left( a \cdot - x_{ii}
			\right)\right) \\
			% -----------------------
			&=
			\sum_{\sigma \in S_m}
			\left( \sign(\sigma)
			\cdot \prod_{i \in \NoFix(\sigma)} -x_{i \sigma(i)}
			\prod_{i \in \Fix{\sigma}} \left( a - x_{ii}
			\right)\right) \\
			% -----------------------
			&=
			\sum_{\sigma \in S_m}
			\left( \sign(\sigma)
			\cdot \left( \prod_{i \in \NoFix(\sigma)} -x_{i \sigma(i)} \right)
			\left( \sum_{t=0}^{\left\lvert \Fix(\sigma) \right\rvert}
			a^{\left\lvert \Fix(\sigma) \right\rvert - t} \cdot \sum_{i_1 < \dots < i_t \in \Fix(\sigma)}
			\prod_{k=1}^t -x_{i_k i_k} \right)
			\right) \\
			% -----------------------
			&=
			\sum_{\sigma \in S_m}
			\left( \sign(\sigma)
			\left( \sum_{t=0}^{\left\lvert \Fix(\sigma) \right\rvert}
			a^{m-t-\left\lvert \Fix(\sigma) \right\rvert}
			\sum_{\substack{X \subseteq \{1, \dots, m\} \\ \NoFix(\sigma) \subseteq X \\ X \text{ has exactly $t$ fixed}}} \prod_{i \in X} -x_{i \sigma(i)}
			\right) \right) \\
			% -----------------------
			&=
			\sum_{n=0}^m
			a^{m-n}
			\left(
			\sum_{\sigma \in S_m}
			\sign(\sigma)
			\sum_{\substack{X \subseteq \{1, \dots, m\} \\ \NoFix(\sigma) \subseteq X \\ \left\lvert X \right\rvert = n} }
			\prod_{i \in X} -x_{i \sigma(i)}
			\right) \\
			% -----------------------
			&= \sum_{n=0}^m
			a^{m-n} (-1)^n
			\left(
			\sum_{\substack{X \subseteq \{1, \dots, m\} \\ \left\lvert X \right\rvert = n} }
			\sum_{\substack{\sigma \in S_m \\ \NoFix(\sigma) \subseteq X}}
			\sign(\sigma) \prod_{i \in X} x_{i \sigma(i)}
			\right).
		\end{align*}

		Hence it's the same to show that
		\[
			\sum_{\substack{X \subseteq \{1, \dots, m\} \\ \left\lvert X \right\rvert = n} }
			\sum_{\substack{\sigma \in S_m \\ \NoFix(\sigma) \subseteq X}}
			\sign(\sigma) \prod_{i \in X} x_{i \sigma(i)}
			= \Tr_{\Lambda^n(V)} \left( \Lambda^n(T) \right)
		\]
		holds for every $n$.

		We can expand the definition of trace as using basis elements as
		\begin{align*}
			\Tr\left( \Lambda^n(T) \right)
			&= \sum_{1 \le i_1 < \dots < i_n \le m}
			\left( \bigwedge_{k=1}^n e_{i_k} \right)^\vee
			\left( \Lambda^n(T) \left( \bigwedge_{k=1}^n e_{i_k} \right) \right) \\
			&= \sum_{1 \le i_1 < \dots < i_n \le m}
			\left( \bigwedge_{k=1}^n e_{i_k} \right)^\vee
			\left(  \bigwedge_{k=1}^n T(e_{i_k}) \right) \\
			&= \sum_{1 \le i_1 < \dots < i_n \le m}
			\left( \bigwedge_{k=1}^n e_{i_k} \right)^\vee
			\left(  \bigwedge_{k=1}^n
			\left( \sum_{j=1}^m x_{i_k j} e_j \right)
			\right) \\
			&= \sum_{1 \le i_1 < \dots < i_n \le m}
			\sum_{\pi \in S_n} \sign(\pi) \prod_{k=1}^n x_{i_{\pi(k)}k} \\
			&= \sum_{\substack{X \subseteq \{1,\dots,m\} \\ \left\lvert X \right\rvert = n}}
			\sum_{\pi \in S_X} \sign(\pi) \prod_{i \in X} x_{t \pi(t)}
		\end{align*}
		Hence it remains to show that the permutations over $X$
		are in bijection with the permutations over $S_m$ which fix $\{1, \dots, m\} - X$,
		which is clear, and moreover, the signs clearly coincide.
	\end{sol}
\end{problem}


\documentclass[12pt]{article}
\usepackage[pdftex,pagebackref,colorlinks=true,pdfpagemode=none,urlcolor=blue,linkcolor=blue,citecolor=blue,pdfstartview=FitH]{hyperref}

\usepackage{amsmath,amsfonts}
\usepackage{graphicx}
\usepackage{color}
\usepackage{hyperref}
\usepackage{minted}
\usemintedstyle{bw}

\newcommand{\homedir}{\string~}

\setlength{\oddsidemargin}{0pt}
\setlength{\evensidemargin}{0pt}
\setlength{\textwidth}{6.0in}
\setlength{\topmargin}{0in}
\setlength{\textheight}{8.5in}

\setlength{\parindent}{0in}
\setlength{\parskip}{5px}

\input{\homedir/Develop/python/gly.fish/latex2wp/macrosblog.tex}

\title{Inverse CDF Sampling}
\author{Troy Stribling}

\begin{document}

% ------------------------------------------------------------------------------------------------
% Introduction
% ------------------------------------------------------------------------------------------------
\iftex
\maketitle
\section{Introduction}
\fi

Inverse \href{https://en.wikipedia.org/wiki/Cumulative_distribution_function}{CDF} sampling is a method for obtaining samples
from both discrete and continuous probability distributions
that requires the CDF to be invertible. The method assumes values of the CDF are Uniform random variables on [0, 1].
CDF values are generated and used as input into the inverted CDF to obtain samples with the distribution defined by the CDF.

% ------------------------------------------------------------------------------------------------
% Sampling discrete distributions
% ------------------------------------------------------------------------------------------------
\ifblog

Sampling Discrete Distributions

\fi
\iftex
\section{Sampling Discrete Distributions}
\fi

A discrete probability distribution consisting of a finite set of $N$ probability values is defined by,
$\{p_i\}_N = \{p_1, p_2,\ldots,p_N\}$ with $p_i \geq 0, \forall i$ and $\sum_{i=1}^N{p_i} = 1.$ The CDF specifies the probability
that $i \leq n$ and is given by,
\begin{equation}
\label{eq:discrete_cdf}
P(i \leq n)=P(n)=\sum_{i=1}^n{p_i},
\end{equation}
where $P(N)=1.$

For a given generated CDF value, $U$, Equation (\ref{eq:discrete_cdf}) can always be inverted by evaluating it for each $n$ and
searching for the value of $n$ that satisfies, $P(n) \geq U.$ It can be seen that the generated samples will have
distribution $\{p_i\}_N$ since the intervals $P(n)-P(n-1) = p_n$ are Uniformly sampled.

Consider the example distribution,

\begin{equation}
\left\{\frac{1}{12}, \frac{1}{12}, \frac{1}{6}, \frac{1}{6}, \frac{1}{12}, \frac{5}{12} \right\}
\label{eq:discrete}
\end{equation}

It is shown in the following plot with its CDF. Note that the CDF is a monotonically increasing function.

\ifblog
\image{width = 600}{https://gly.fish/wp-content/uploads/posts/inverse-cdf-sampling/discrete_cdf.png}{\homedir/Develop/python/gly.fish/assets/posts/inverse_cdf_sampling/discrete_cdf.png}
\fi
\iftex
\image{width = 400}{https://gly.fish/wp-content/uploads/posts/inverse-cdf-sampling/discrete_cdf.png}{\homedir/Develop/python/gly.fish/assets/posts/inverse_cdf_sampling/discrete_cdf.png}
\fi

A sampler using the Inverse CDF method on the distribution $\{p_i\}_N$ implemented in Python is shown below.
The program first stores the CDF computed from each of the sums $P(n)$ in an array. Next,
CDF samples using $U \sim \textbf{Uniform}(0, 1)$ are generated. Finally, for each sampled CDF value, $U$, the array containing $P(n)$
is scanned for the value of $n$ where $P(n) \geq U$. The resulting values of $n$ will have the distribution $\{p_i\}_N$.

% Sampler code examples
\ifblog

import numpy

n = 10000
df = numpy.array([1/12, 1/12, 1/6, 1/6, 1/12, 5/12])
cdf = numpy.cumsum(df)

samples = [numpy.flatnonzero(cdf >= u)[0] for u in numpy.random.rand(n)]

\fi

\iftex
\pagebreak
\begin{minted}[mathescape, frame=lines, framesep=2mm, fontsize=\footnotesize]{python}
import numpy

n = 10000
df = numpy.array([1/12, 1/12, 1/6, 1/6, 1/12, 5/12])
cdf = numpy.cumsum(df)

samples = [numpy.flatnonzero(cdf >= u)[0] for u in numpy.random.rand(n)]
\end{minted}
\fi

The figure below favorably compares generated samples and distribution (\ref{eq:discrete}),
\ifblog
\image{width = 600}{https://gly.fish/wp-content/uploads/posts/inverse-cdf-sampling/discrete_sampled_distribution.png}{\homedir/Develop/python/gly.fish/assets/posts/inverse_cdf_sampling/discrete_sampled_distribution.png}
\fi
\iftex
\image{width = 400}{https://gly.fish/wp-content/uploads/posts/inverse-cdf-sampling/discrete_sampled_distribution.png}{\homedir/Develop/python/gly.fish/assets/posts/inverse_cdf_sampling/discrete_sampled_distribution.png}
\fi

% multinomial sampling
\ifblog
It is also possible to directly sample $\{p_i\}_N$ using the multinomial sampler from
numpy,

import numpy

n = 10000
df = numpy.array([1/12, 1/12, 1/6, 1/6, 1/12, 5/12])
samples = numpy.random.multinomial(n, df, size=1)/n

\fi

\iftex
It is also possible to directly sample $\{p_n\}$ using the \mintinline{python}{multinomial} sampler from
\mintinline{python}{numpy},

\begin{minted}[mathescape, frame=lines, framesep=2mm,fontsize=\footnotesize]{python}
import numpy

n = 10000
df = numpy.array([1/12, 1/12, 1/6, 1/6, 1/12, 5/12])
samples = numpy.random.multinomial(n, df, size=1)/n
\end{minted}
\fi

The number of operations required for generating samples using Inverse CDF sampling from a discrete
distribution will scale $O(N_{samples}N)$ where $N_{samples}$ is the desired number of samples
and $N$ is the number of terms in the discrete distribution.

% ------------------------------------------------------------------------------------------------
% Sampling continuous distributions
% ------------------------------------------------------------------------------------------------
\ifblog

Sampling Continuous Distributions

\fi
\iftex
\section{Sampling Continuous Distributions}
\fi

A continuous probability distribution is defined by the  \href{https://en.wikipedia.org/wiki/Probability_density_function}{PDF},
$f_X(x)$, where $f_X(x) \geq 0, \forall x$ and $\int f_X(x) dx = 1.$ The CDF is a monotonically increasing function
that specifies the probability that $X \leq x$, namely,
\begin{equation}
\label{eq:continuous_cdf}
P(X \leq x) = F_X(x) = \int^{x} f_X(w) dw.
\end{equation}

% ------------------------------------------------------------------------------------------------
% Proof continuous distribution
% ------------------------------------------------------------------------------------------------
\ifblog

Proof the Inverse CD Sampling Works

\fi
\iftex
\subsection{Proof that Inverse CD Sampling Works}
\fi

To prove that Inverse CDF sampling works for continuous distributions it must be shown that,

\begin{equation}
\label{eq:continuous_proof}
P[F_X^{-1}(U) \leq x] = F_X(x),
\end{equation}

where $F_X^{-1}(x)$ is the inverse of $F_X(x)$ and $U \sim \textbf{Uniform}(0, 1)$.

A more general result needed to complete
this proof is obtained using a change of variable on a CDF. If $Y=G(X)$ is a monotonically increasing invertible function
of $X$ then,

\begin{equation}
\label{eq:CDF_invariance}
P(X \leq x) = P(Y \leq y) = P[G(X) \leq G(x)].
\end{equation}

To prove this note that $G(x)$ is monotonically increasing so the ordering of values is preserved,

$$ X \le x \implies G(X) \le G(x).$$

Consequently, the order of the integration limits is maintained by the transformation. Further, since $G(x)$ is invertible,
$x = G^{-1}(y)$ and $dx = \frac{dG^{-1}}{dy} dy$, so

$$
\begin{aligned}
P(X \leq x) & = \int^{x} f_X(w) dw \\
& = \int^{y} f_X(G^{-1}(z)) \frac{dG^{-1}}{dz} dz \\
& = \int^{y} f_Y(z) dz \\
& = P(Y \leq y) \\
& = P[G(X) \leq G(x)],
\end{aligned}
$$

where,

$$ f_Y(y) = f_X(G^{-1}(y)) \frac{dG^{-1}}{dy} $$

The desired proof of Equation (\ref{eq:continuous_proof}) follows from Equation (\ref{eq:CDF_invariance}) by noting that
$U \sim \textbf{Uniform}(0, 1)$ so $f_U(u) = 1$,

$$
\begin{aligned}
P[F_X^{-1}(U) \leq x] & = P[F_X(F_X^{-1}(U)) \leq F_X(x)] \\
& = P[U \leq F_X(x)] \\
& = \int_{0}^{F_X(x)} f_U(w) dw \\
& = \int_{0}^{F_X(x)} dw \\
& = F_X(x).
\end{aligned}
$$

% ------------------------------------------------------------------------------------------------
% Example continuous distribution
% ------------------------------------------------------------------------------------------------
\ifblog

Example

\fi
\iftex
\subsection{Example}
\fi

Consider the \href{https://en.wikipedia.org/wiki/Weibull_distribution}{Weibull Distribution}, with density

$$
f_X(x; k, \lambda) =
\begin{cases}
\frac{k}{\lambda}\left(\frac{x}{\lambda} \right)^{k-1} e^{\left(\frac{-x}{\lambda}\right)^k} & x \geq 0 \\
0 & x < 0,
\end{cases}
$$

where $k$ is the shape parameter and $\lambda$ the scale parameter. The CDF is given by,

$$
F_X(x; k, \lambda) =
\begin{cases}
1-e^{\left(\frac{-x}{\lambda}\right)^k
} & x \geq 0 \\
0 & x < 0.
\end{cases}
$$

The CDF can be inverted to yield,

$$
F_X^{-1}(u; k, \lambda) =
\begin{cases}
\lambda\ln\left(\frac{1}{1-u}\right)^{\frac{1}{k}} & 0 \leq u \leq 1 \\
0 & u < 0 \text{ or } u > 1.
\end{cases}
$$

In the example described here it will be assumed that $k=5.0$ and $\lambda=1.0$. The following plot shows the PDF and CDF using these values.
\ifblog
\image{width = 600}{https://gly.fish/wp-content/uploads/posts/inverse-cdf-sampling/weibull_cdf.png}{\homedir/Develop/python/gly.fish/assets/posts/inverse_cdf_sampling/weibull_cdf.png}
\fi
\iftex
\image{width = 400}{https://gly.fish/wp-content/uploads/posts/inverse-cdf-sampling/weibull_cdf.png}{\homedir/Develop/python/gly.fish/assets/posts/inverse_cdf_sampling/weibull_cdf.png}
\fi

The sampler implementation for the continuous case is simpler than for the discrete case. Just as in the discrete case
CDF samples with distribution $U \sim \textbf{Uniform}(0, 1)$ are generated. The desired samples with
the Weibull distribution are then computed using the CDF inverse. Below an implementation of the sampler in Python is listed.

\ifblog

import numpy

k = 5.0
λ = 1.0
nsamples = 100000

cdf_inv = lambda u: λ * (numpy.log(1.0/(1.0 - u)))**(1.0/k)
samples = [cdf_inv(u) for u in numpy.random.rand(nsamples)]

\fi

\iftex
\begin{minted}[mathescape, frame=lines, framesep=2mm,fontsize=\footnotesize]{python}
import numpy

k = 5.0
l = 1.0
nsamples = 100000

cdf_inv = lambda u: l * (numpy.log(1.0/(1.0 - u)))**(1.0/k)
samples = [cdf_inv(u) for u in numpy.random.rand(nsamples)]
\end{minted}
\fi

The following plot compares a histogram of the samples generated by the sampler above. The fit is quite good. The subtle
asymmetry of the Weibull distribution is captured.

\ifblog
\image{width = 600}{https://gly.fish/wp-content/uploads/posts/inverse-cdf-sampling/weibull_sampled_distribution.png}{\homedir/Develop/python/gly.fish/assets/posts/inverse_cdf_sampling/weibull_sampled_distribution.png}
\fi
\iftex
\image{width = 400}{https://gly.fish/wp-content/uploads/posts/inverse-cdf-sampling/weibull_sampled_distribution.png}{\homedir/Develop/python/gly.fish/assets/posts/inverse_cdf_sampling/weibull_sampled_distribution.png}
\fi

A measure of convergence of the samples to the target distribution can be obtained by comparing the cumulative
moments of the distribution computed from the samples with the value computed analytically. For the Weibull distribution
the first and second moments are given by,

$$
\begin{aligned}
\mu & = \lambda\Gamma\left(1+\frac{1}{k}\right) \\
\sigma^2 & = \lambda^2\left[\Gamma\left(1+\frac{2}{k}\right)-\left(\Gamma\left(1+\frac{1}{k}\right)\right)^2\right],
\end{aligned}
$$

where $\Gamma(x)$ is the \href{https://en.wikipedia.org/wiki/Gamma_function}{Gamma function}. The following plots
perform this comparison. The first shows the convergence of $\mu$ and the second the convergence of
$\sigma$. Within only 1000 samples both $\mu$ and $\sigma$ computed from samples is comparable to the analytic value.

\ifblog
\image{width = 600}{https://gly.fish/wp-content/uploads/posts/inverse-cdf-sampling/weibull_sampled_mean_convergence.png}{\homedir/Develop/python/gly.fish/assets/posts/inverse_cdf_sampling/weibull_sampled_mean_convergence.png}
\fi
\iftex
\image{width = 400}{https://gly.fish/wp-content/uploads/posts/inverse-cdf-sampling/weibull_sampled_mean_convergence.png}{\homedir/Develop/python/gly.fish/assets/posts/inverse_cdf_sampling/weibull_sampled_mean_convergence.png}
\fi

\ifblog
\image{width = 600}{https://gly.fish/wp-content/uploads/posts/inverse-cdf-sampling/weibull_sampled_std_convergence.png}{\homedir/Develop/python/gly.fish/assets/posts/inverse_cdf_sampling/weibull_sampled_std_convergence.png}
\fi
\iftex
\image{width = 400}{https://gly.fish/wp-content/uploads/posts/inverse-cdf-sampling/weibull_sampled_std_convergence.png}{\homedir/Develop/python/gly.fish/assets/posts/inverse_cdf_sampling/weibull_sampled_std_convergence.png}
\fi

%%------------------------------------------------------------------------------------------------
% Performance continuous distribution
% ------------------------------------------------------------------------------------------------
\ifblog

Performance

\fi
\iftex
\subsection{Performance}
\fi

Any continuous distribution, $f_X(x)$, can be approximated by the discrete distribution,
$\left\{f_X(x_i)\Delta x_i \right\}_N$ for $i=1,2,3,\ldots,N$, where $\Delta x_i=(x_{max}-x_{min})/(N-1)$ and
$x_i = x_{min}+(i-1)\Delta x_i$. This method has a couple of drawbacks compared to using Inverse CDF sampling on the
continuous distribution. First, a bounded range for the samples must be assumed when in general the range
of the samples can be unbounded. The Inverse CDF method can sample an unbounded range. Second, the performance for
sampling a discrete distribution scales $O(N_{samples}N)$ while sampling the continuous distribution scales
$O(N_{samples})$.

\end{document}


\section{Equation}

\subsection{Multi-line Formula}

When writing about PDEs, we usually need to write the equations and their initial and/or boundary conditions in one equation, i.e., use only one number for the equation and IC/BC\@. We will show how to write multi-line formula with the example of heat equation and its solution. The main idea of writing multi-line formula are using \verb|eqnarray| and \verb|array| environment.
Heat Equation.
\begin{equation}
    \left\{\begin{array}{l}
        \dfrac{\partial u}{\partial t} = a^2 \dfrac{\partial^2 u}{\partial t^2}, t
        > 0, - \infty < x < + \infty \\
        u |_{t = 0} = \varphi (x) |
    \end{array}\right.
\end{equation}
Solution
\begin{eqnarray}
    u (t, x) & = & \mathcal{F}^{- 1} \left[ \hat{\varphi} (\lambda) e^{_{- a^2
                        \lambda^2 t}} \right] \nonumber\\
    & = & \mathcal{F}^{- 1} [\hat{\varphi} (\lambda)] \ast \mathcal{F}^{- 1}
    \left[ e^{_{- a^2 \lambda^2 t}} \right] \nonumber\\
    & = & \varphi (x) \ast \dfrac{1}{2 a \sqrt{\pi t}} \exp \left( -
    \dfrac{x^2}{4 a^2 t} \right) \nonumber\\
    & = & \dfrac{1}{2 a \sqrt{\pi t}} \int_{- \infty}^{+ \infty} \varphi (\xi)
    \exp \left( - \dfrac{(x - \xi)^2}{4 a^2 t} \right) \mathrm{d} \xi
\end{eqnarray}

\subsection{Matrix and determinant}

Matrix and determinant are common in dynamics and continuum mechanics. Here we use the relationship of coordinates vector as an example. The main idea of writing matrix and determinant is \verb|array| and \verb|&| symbol.
\begin{equation}
    \textbf{e}_i = \sum_{i = 1}^n \dfrac{1}{H_i} \dfrac{\partial r_i}{\partial
        x_i} \textbf{i}_i
\end{equation}
Denote \(\textbf{e} = \{ \textbf{e}_1, \textbf{e}_2, \ldots,
\textbf{e}_n \}\), \(\textbf{i} = \{ \textbf{i}_1, \textbf{i}_2, \ldots,
\textbf{i}_n \}\), then
\begin{equation}
    \textbf{e} = A \textbf{i}
\end{equation}
where
\begin{eqnarray}
    A & = & \left(\begin{array}{cccc}
        \dfrac{1}{H_1} \dfrac{\partial r_1}{\partial x_1} & \dfrac{1}{H_1}
        \dfrac{\partial r_2}{\partial x_1}                & \cdots         & \dfrac{1}{H_1} \dfrac{\partial
            r_n}{\partial x_1}                                                                                \\
        \dfrac{1}{H_2} \dfrac{\partial r_1}{\partial x_2} & \dfrac{1}{H_2}
        \dfrac{\partial r_2}{\partial x_2}                &                &                                                 \\
        \vdots                                            &                & \ddots                         &                \\
        \dfrac{1}{H_n} \dfrac{\partial r_1}{\partial x_n} &                &                                & \dfrac{1}{H_n}
        \dfrac{\partial r_n}{\partial x_n}
    \end{array}\right)
\end{eqnarray}
and
\begin{eqnarray}
    \det A & = & \left|\begin{array}{cccc}
        \dfrac{1}{H_1} \dfrac{\partial r_1}{\partial x_1} & \dfrac{1}{H_1}
        \dfrac{\partial r_2}{\partial x_1}                & \cdots         & \dfrac{1}{H_1} \dfrac{\partial
            r_n}{\partial x_1}                                                                                \\
        \dfrac{1}{H_2} \dfrac{\partial r_1}{\partial x_2} & \dfrac{1}{H_2}
        \dfrac{\partial r_2}{\partial x_2}                &                &                                                 \\
        \vdots                                            &                & \ddots                         &                \\
        \dfrac{1}{H_n} \dfrac{\partial r_1}{\partial x_n} &                &                                & \dfrac{1}{H_n}
        \dfrac{\partial r_n}{\partial x_n}
    \end{array}\right|
\end{eqnarray}

\subsection{Differential Symbol}

Integral is common in equations and formula. The \verb|d| symbol is very tricky. It should not be italic and should be separated from the integrand by a space. We can use the command \verb|\dif| in package \verb|commath| to write \verb|d| correctly.

\begin{eqnarray}
    \int_0^1 x dx = \frac{1}{2}x^2 \Bigg|_0^1 = \frac{1}{2}\\
    \int_0^1 x \mathrm{d}x = \frac{1}{2}x^2 \Bigg|_0^1 = \frac{1}{2}\\
    \int_0^1 x \dif x = \frac{1}{2}x^2 \Bigg|_0^1 = \frac{1}{2}
\end{eqnarray}

\subsection{Tensor Scripts}

Here are two different approach to write subscript and superscript for a tensor or operator symbol. For general purpose
\begin{equation}
    \prescript{a}{b}{C}_{d}^{c}
\end{equation}
, or use package \verb|Tensor|
\begin{eqnarray}
    M\indices{^a_b^{cd}_e}\\
    \tensor[^a_b^c_d]{M}{^a_b^c_d}
\end{eqnarray}


\section{PI in the class}
\subsection{}

\begin{frame}
\frametitleTC{Transfer function}
\framesubtitleTC{}
\myPause
 \begin{itemize}[<+-| alert@+>]
 \item Let us reload our systems theory knowledge:
       \begin{displaymath}
        \left\{\begin{array}{rcl}
         u_P(k) &=& K_P \, e(k) \\
         u_I(k) &=& u_I(k-1)+ K_I e(k) \\
         u(k)   &=& u_P(k)+u_I(k)
        \end{array}\right. \quad
        {\Rightarrow} \quad
        \left\{\begin{array}{rcl}
         zu_I(k) &=& u_I(k) + zK_I e(k) \\
         u(k)    &=& u_I(k) + K_P \, e(k)
        \end{array}\right.
       \end{displaymath}
 \item Hence
       \begin{displaymath}
        u(k) = \left( \frac{z}{z-1}K_I + K_P \right) e(k)
       \end{displaymath}
 \item and finally
       \begin{displaymath}
        C_{PI}(z) = \frac{u(k)}{e(k)} = \frac{(K_P+K_I)z-K_P}{z-1},
       \end{displaymath}
       with one zero and one pole in $z=1$, the latter produced by the\\
       integral action.
 \end{itemize}
\end{frame}

\begin{frame}
\frametitleTC{Transfer function}
\framesubtitleTC{}
\myPause
 \begin{itemize}[<+-| alert@+>]
 \item Reformulating we get
       \begin{displaymath}
        C_{PI}(z) = \frac{(K_P+K_I)z-K_P}{z-1}
                  = \left( K_P+K_I \right) \frac{z-\frac{K_P}{K_P+K_I}}{z-1}
       \end{displaymath}
 \item that is, the compact expression
       \begin{displaymath}
        C_{PI}(z) = K \frac{z-\zeta}{z-1}, \qquad
        K         = K_P+K_I, \quad
        \zeta     = \frac{K_P}{K_P+K_I}.
       \end{displaymath}
       that is well suited for a cancellation-based tuning policy like direct\\
       synthesis.
 \end{itemize}
\end{frame}

\begin{frame}
\frametitleTC{When is PI control advisable?}
\framesubtitleTC{}
\myPause
 \begin{itemize}[<+-| alert@+>]
 \item When the process is \TC{dominantly} first-order, i.e., when its step response is well approximated
       by $\mu(1-p^k)$.
 \item In that case we know that $P(z)=\mu/(z-p)$ is a good model, and if we want e.g.
       $G_{yw}^{\circ}(z)=(1-\alpha)/(z-\alpha)$ we can first determine a desired loop transfer function
       \begin{displaymath}
        L^{\circ}(z) = \frac{G_{yw}^{\circ}(z)}{1-G_{yw}^{\circ}(z)}
                     = \frac{\frac{1-\alpha}{z-\alpha}}{1-\frac{1-\alpha}{z-\alpha}}
                     = \frac{1-\alpha}{z-1}
       \end{displaymath}
 \item Then we determine the controller (which we know to be structurally a PI) as
       \begin{displaymath}
        C(z) = \frac{L^{\circ}(z)}{P(z)} = \frac{1-\alpha}{z-1}\,\frac{z-p}{\mu}
       \end{displaymath}
 \item[] readily obtaining its parameters
       \begin{displaymath}
        \zeta = p, \quad
        K     = \frac{1-\alpha}{\mu}.
       \end{displaymath}
 \end{itemize}
\end{frame}


\section*{Exercises}

\begin{ex} The following are the cylindrical coordinates of points, $(
r,\theta,z)$. Find the Cartesian and spherical coordinates of each point.

\begin{enumerate}
\item $\paren{5,\frac{5\pi }{6},-3} $

\item $\paren{3,\frac{\pi }{3},4} $

\item $\paren{4,\frac{2\pi }{3},1} $

\item $\paren{2,\frac{3\pi }{4},-2} $

\item $\paren{3,\frac{3\pi }{2},-1} $

\item $\paren{8,\frac{11\pi }{6},-11} $
\end{enumerate}
%\begin{sol}
%\end{sol}
\end{ex}

\begin{ex} The following are the Cartesian coordinates of points, $(
x,y,z)$. Find the cylindrical and spherical coordinates of these
points.

\begin{enumerate}
\item $\paren{\frac{5}{2}\sqrt{2},\frac{5}{2}\sqrt{2},-3} $

\item $\paren{\frac{3}{2},\frac{3}{2}\sqrt{3},2} $

\item $\paren{-\frac{5}{2}\sqrt{2},\frac{5}{2}\sqrt{2},11} $

\item $\paren{-\frac{5}{2},\frac{5}{2}\sqrt{3},23} $

\item $(-\sqrt{3},-1,-5) $

\item $\paren{\frac{3}{2},-\frac{3}{2}\sqrt{3},-7} $

\item $(\sqrt{2},\sqrt{6},2\sqrt{2}) $

\item $\paren{-\frac{1}{2}\sqrt{3},\frac{3}{2},1} $

\item $\paren{-\frac{3}{4}\sqrt{2},\frac{3}{4}\sqrt{2},-\frac{3}{2}\sqrt{3}} $

\item $(-\sqrt{3},1,2\sqrt{3}) $

\item $\paren{-\frac{1}{4}\sqrt{2},\frac{1}{4}\sqrt{6},-\frac{1}{2}\sqrt{2}} $
\end{enumerate}
%\begin{sol}
%\end{sol}
\end{ex}

\begin{ex}
The following are spherical coordinates of points in the form $(
\rho,\phi,\theta)$. Find the Cartesian and cylindrical
coordinates of each point.

\begin{enumerate}
\item $\paren{4,\frac{\pi }{4},\frac{5\pi }{6}} $

\item $\paren{2,\frac{\pi }{3},\frac{2\pi }{3}} $

\item $\paren{3,\frac{5\pi }{6},\frac{3\pi }{2}} $

\item $\paren{4,\frac{\pi }{2},\frac{7\pi }{4}} $

\item $\paren{4,\frac{2\pi }{3},\frac{\pi }{6}} $

\item $\paren{4,\frac{3\pi }{4},\frac{5\pi }{3}} $
\end{enumerate}
%\begin{sol}
%\end{sol}
\end{ex}

\begin{ex} Describe the surface $\phi =\pi
/4$ in Cartesian coordinates, where $\phi $ is the polar angle in spherical coordinates.
%\begin{sol}
%\end{sol}
\end{ex}

\begin{ex} Describe the surface $\theta =\pi /4$ in spherical coordinates, where $\theta $ is
the angle measured from the positive $x$ axis.
%\begin{sol}
%\end{sol}
\end{ex}

\begin{ex} Describe the surface $r=5$ in Cartesian coordinates, where
$r$ is one of the cylindrical coordinates.
%\begin{sol}
%\end{sol}
\end{ex}

\begin{ex} Describe the surface $\rho =4$ in Cartesian coordinates,
where $\rho $ is the distance to the origin.
%\begin{sol}
%\end{sol}
\end{ex}

\begin{ex} Give the cone described by $z=\sqrt{x^{2}+y^{2}}$ in cylindrical coordinates and
in spherical coordinates.
%\begin{sol}
%\end{sol}
\end{ex}

\begin{ex} The following are described in Cartesian coordinates. Rewrite them in terms of spherical coordinates.

\begin{enumerate}
\item $z=x^{2}+y^{2}$.

\item $x^{2}-y^{2}=1$.

\item $z^{2}+x^{2}+y^{2}=6$.

\item $z=\sqrt{x^{2}+y^{2}}$.

\item $y=x$.

\item $z=x$.
\end{enumerate}
%\begin{sol}
%\end{sol}
\end{ex}

\begin{ex} The following are described in Cartesian coordinates. Rewrite them in terms of cylindrical coordinates.

\begin{enumerate}
\item $z=x^{2}+y^{2}$.

\item $x^{2}-y^{2}=1$.

\item $z^{2}+x^{2}+y^{2}=6$.

\item $z=\sqrt{x^{2}+y^{2}}$.

\item $y=x$.

\item $z=x$.
\end{enumerate}
%\begin{sol}
%\end{sol}
\end{ex}