\section{Operator}

% Normal
\subsection{Normal}

\begin{theorem}\label{eigenvectorforadjointoperator}
    If $T$ has eigenvector, then $T^*$ has eigenvector.
\end{theorem}
\begin{proof}
    $0 = \innerproduct{0}{x} = \innerproduct{(T - \lambda I)(v)}{x} = \innerproduct{v}{(T - \lambda I)^* (x)} = \innerproduct{v}{(T^* - \overline{\lambda} I)(x)}$. Since $v \neq 0$ is reciprocal to the range of $T^* - \overline{\lambda} I$, $v \notin \rangespace{T^* - \overline{\lambda} I}$, so $\nullspace{T^* - \overline{\lambda} I} \neq \{ 0 \}$.
\end{proof}

\begin{theorem}[\cindex{Schur}]\label{schurincomplexfield}
    Suppose the characteristic polynomial of $T$ splits. Then there exists an orthonormal basis $\beta$ for $V$ that the $\coordinate{T}_\beta$ is upper trianglar. Note:
    \begin{enumerate}
        \item $\beta$ does \emph{not} need to be eigenvectors of $T$.
        \item It works in $\mathcal{R}$ as long as $T$ splits.
    \end{enumerate}
\end{theorem}
\begin{proof}
    Use induction. Since $T$ splits, it has a eigenvector. By \thmref{eigenvectorforadjointoperator} $T^*$ has eigenvector, and make it a unit eigenvector $z$. Let $W = \text{span}\{z\}$. Then prove $W^\bot$ is $T$-invariant: for $\forall y \in W^\bot$ and $x = cz \in W$:
    \begin{equation*}
        \begin{aligned}
            \innerproduct{T(y)}{x} = \innerproduct{T(y)}{cz} = \innerproduct{y}{T^*(cz)} = \innerproduct{y}{cT^*(z)} = \innerproduct{y}{c \lambda z} = \overline{c\lambda} \innerproduct{y}{z} = 0
        \end{aligned}
    \end{equation*}
    According to induction, $\dimension{W^\bot} = n - 1$ and there exists an orthonormal basis $\gamma$ that $\coordinate{T_{W^\bot}}_\gamma$ is upper triangular. Take $\gamma \cup \set{z}$.
\end{proof}

\begin{theorem}
    If $\beta$ is an orthonormal basis and $\coordinate{T}_\beta$ is a diagonal matrix, $\coordinate{T^*}_\beta = \left(\coordinate{T}_\beta\right)^*$ is also a diagonal matrix.
\end{theorem}

\begin{theorem}
    If an operator $T$ has orthogonal eigenvectors $\beta$ that are basis of the inner product space, then $\coordinate{T}_\beta$ is a diagonal matrix.
\end{theorem}

\begin{definition}
    $T$ is \cindex{normal} if $T T^* = T^* T$. A square matrix $A$ is \cindex{normal} if $AA^* = A^* A$.
\end{definition}

\begin{theorem}
    $T$ is normal if and only of $\coordinate{T}_\beta$ is normal under orthonormal basis $\beta$.
\end{theorem}

\begin{theorem}\label{propertyofnormaloperator}
    Properties of normal operator $T$ on $V$:
    \begin{enumerate}
        \item $\forall x \in V$, $\norm{T(x)} = \norm{T^*(x)}$
        \item $\forall c \in F$, $T - cI$ is normal.
        \item If $x$ is a eigenvector of eigenvalue $\lambda$ for $T$, $T^*(x) = \overline{\lambda} x$, so $x$ is also an eigenvector of eigenvalue $\overline{\lambda}$ for $T^*$.
        \item If $x_1$ and $x_2$ are for eigenvalues $\lambda_1$ and $\lambda_2$, $\innerproduct{x_1}{x_2} = 0$
    \end{enumerate}
\end{theorem}
\begin{proof}
    \begin{equation*}
        \norm{T(x)}^2 = \innerproduct{T(x)}{T(x)} = \innerproduct{T^* T (x)}{x} = \innerproduct{TT^*(x)}{x} = \innerproduct{T^*(x)}{T^*(x)} = \norm{T^*(x)^2}
    \end{equation*}

    \begin{equation*}
        0 = \norm{(T - \lambda I)(x)} = \norm{(T - \lambda I)^*(x)} = \norm{(T^* - \overline{\lambda} I)(x) }
    \end{equation*}

    \begin{equation*}
        \lambda_1 \innerproduct{x_1}{x_2} = \innerproduct{\lambda x_1}{x_2} = \innerproduct{T(x_1)}{x_2} = \innerproduct{x_1}{T^*(x_2)} = \innerproduct{x_1}{\overline{\lambda_2} x_2} = \lambda_2 \innerproduct{x_1}{x_2}
    \end{equation*}
    So $(\lambda_1 - \lambda_2) \innerproduct{x_1}{x_2} = 0$. Since $\lambda_1 \neq \lambda_2$, $\innerproduct{x_1}{x_2} = 0$
\end{proof}

\begin{theorem}
    If $T$ is normal, $\nullspace{T} = \nullspace{T^*}$ and $\rangespace{T} = \rangespace{T^*}$. So being normal will refine \thmref{nullandreciprocaladjoint}.
\end{theorem}
\begin{proof}
    If $x \in \nullspace{T}$, $\norm{T(x)} = \norm{T^*} = 0$, so $T^*(x) = 0$ and $x \in \nullspace{T^*}$.
\end{proof}

\begin{theorem}
    In $\mathcal{C}$, let $V$ be finite dimensional inner product space. $T$ is normal if and only if there exists an orthonormal basis for $V$ consisting of eigenvectors of $T$.
\end{theorem}
\begin{proof}
    in $C$ the polynomial always splits. According to \thmref{schurincomplexfield} there exists a orthonormal basis $\beta = \{v_1, v_2, \dots, v_n\}$ that $\coordinate{T}_\beta = A$ is upper triangular. $v_1$ is an eigenvector because $T(v_1)=A_{1,1} v_1$. Assuming $v_1, v_2, \dots, v_{k-1}$ are eigenvector of $T$, we prove that $v_k$ is also an eigenvector of $T$. Because $A$ is upper triangular,
    \begin{equation*}
        T(v_k) = A_{1,k} v_1 + A_{2,k} v_2 + \dots + A_{j,k} v_j + \dots + A_{k,k} v_k
    \end{equation*}
    Because $\forall j < k$, $A_{j,k} = \innerproduct{T(v_k}{v_j} = \innerproduct{v_k}{T^*(v_j)} = \innerproduct{v_k}{\overline{\lambda} v_j} = \lambda_j \innerproduct{v_k}{v_j} = 0$, we have $T(v_k) = A_{k,k} v_k$, so $v_k$ is an eigenvector of $T$.

    btw, it does not work in infinite dimensional complex inner product space.
\end{proof}

% self-adjoint
\subsection{Hermitian}

\begin{definition}
    $T$ is \cindex{self-adjoint} (\cindex{Hermitian}) if $T = T^*$, or $A = A^*$. For real matrix, it means $A$ is symmetric.
\end{definition}

\begin{theorem}
    Let $T$ be a linear operator on complex inner product space. Then $T$ is self-adjoint if and only if $\forall x \in V$,$\innerproduct{T(x)}{x} \in \mathcal{R}$.
\end{theorem}
\begin{proof}
    If $T$ is self-adjoint, $\overline{\innerproduct{T(x)}{x}} = \innerproduct{x}{T(x)} = \innerproduct{T^*(x)}{x} = \innerproduct{T(x)}{x}$. So $\innerproduct{T(x)}{x} \in \mathcal{R}$.

    If $\innerproduct{T(x)}{x} \in \mathcal{R}$, $\innerproduct{T(x)}{x} = \overline{\innerproduct{T(x)}{x}} = \innerproduct{x}{T(x)} = \innerproduct{T^*(x)}{x}$. So $\forall x \in V$, $\innerproduct{(T - T^*)(x)}{x} = 0$. According to Theorem (\ref{zerotforalltx}), $T - T^* = 0$.
\end{proof}

\begin{theorem}
    Let $T$ be a self-adjoint operator on finite dimensional inner product space $V$. Then:
    \begin{enumerate}
        \item every eigenvalue is real.
        \item If $V$ is a real inner product space, the characteristic polynomial for $T$ splits.
    \end{enumerate}
\end{theorem}
\begin{proof}
    Because $T$ is self-adjoint, $T$ is also normal. So according to \thmref{propertyofnormaloperator} if $\lambda$ is an eigenvalue of $T$,  $\overline{\lambda}$ is an eigenvalue of $T^*$. So:
    \begin{equation*}
        \lambda x = T(x) = T^*(x) = \overline{\lambda} x
    \end{equation*}
    So $\lambda = \overline{\lambda}$, and $\lambda$ is real.

    For a orthonormal basis $\beta$, $A = \coordinate{T}_\beta$ is self-adjoint because $A^* = (\coordinate{T}_\beta)^* = [T^*]_\beta = \coordinate{T}_\beta = A$. Define $L_A(x) = Ax$ in $\mathcal{C}^n$. Here we create a function in $\mathcal{C}^n$ from a function in $\mathcal{R}^n$. Let $\gamma$ be the standard basis for $\mathcal{C}$ which is orthonormal. $[L_A]_\gamma = A$ is self-adjoint, so $L_A$ is self-adjoint in $\mathcal{C}^n$. The characteristic polynomial of $L_A$ splits. Since $L_A$ is self-adjoint, all eigenvalues are real, so the polynomial split in $\mathcal{R}$. But $L_A$, $A$ and $T$ has the same characteristic polynomial.
\end{proof}

\begin{theorem}\label{selfadjointmatrixhasorthonormalbasis}
    Let $T$ be a linear operator on finite dimensional real inner product space. $T$ is self-adjoint if and only if there exists an orthonormal basis $\beta$ for $V$ consisting of eigenvectors of $T$.
\end{theorem}
\begin{proof}
    By \thmref{schurincomplexfield} there exists orthonormal basis $\beta$ for $V$ that $A = \coordinate{T}_\beta$ is upper triangular. Because $A^* = (\coordinate{T}_\beta)^* = [T^*]_\beta = \coordinate{T}_\beta = A$, $A$ is diagonal matrix.
\end{proof}

\begin{theorem}
    For the orthonormal basis of eigenvector $T$ problem we have:
    \begin{enumerate}
        \item If $T$ splits, we have orthonormal basis that make $T$ upper triangular in $\mathcal{R}$ or $\mathcal{C}$. This basis may not be eigenvectors, or $T$ may not have eigenvectors.
        \item $T$ is complex normal.
        \item $T$ is real symmetric.
    \end{enumerate}
\end{theorem}

\begin{theorem}\label{zerotforalltxforselfadjoint}
    Let $T$ be self-adjoint operator. If $\forall x \in V$,$\innerproduct{T(x)}{x} = 0$. Then $T = 0$.\footnote{Self-adjoint is not needed of $V=\mathcal{C}$. See \thmref{zerotforalltx} on page \pageref{zerotforalltx}.}
\end{theorem}
\begin{proof}
    Choose orthonormal basis $\beta$ that consist of eigenvector of $T$. For $x\in \beta$, $T(x) =\lambda x$. So
    \begin{equation*}
        0 = \innerproduct{x}{T(x)} = \innerproduct{x}{\lambda x} = \overline{\lambda} \innerproduct{x}{x}
    \end{equation*}
    Hence $\overline{\lambda} = 0$ and $\forall x \in \beta,  T(x) = 0$.
\end{proof}

\subsection{Positive Operator}

\begin{definition}
    An operator $T$ is called \cindex{positive operator} if $T$ is self-adjoint and $\forall x \in V$:
    \begin{equation}
        \innerproduct{Tx}{x} \geq 0
    \end{equation}
\end{definition}

\begin{definition}
    An Operator $R$ is called a \cindex{square root} of an operator $T$ if
    \begin{equation}
        R^2 = T
    \end{equation}
\end{definition}

\begin{theorem}\label{positiveoperatorproperty}
    All the following are equivalent:
    \begin{enumerate}
        \item \label{positiveoperatorproperty1} $T$ is positive.
        \item \label{positiveoperatorproperty2} $T$ is self-adjoint and all eigenvalue of $T$ are non-negative.
        \item \label{positiveoperatorproperty3} $T$ has positive square root.
        \item \label{positiveoperatorproperty4} $T$ has self-adjoint square root.
        \item \label{positiveoperatorproperty5} $\exists R: T = R^* R$
    \end{enumerate}
\end{theorem}
\begin{proof}
    For \ref{positiveoperatorproperty2}, if $T$ is positive, $0 \leq \innerproduct{Tv}{v} = \innerproduct{\lambda v}{v} = \lambda \innerproduct{v}{v}$, so $\lambda \geq 0$.

    For \ref{positiveoperatorproperty3}, if $T$ is self-adjoint, by \thmref{selfadjointmatrixhasorthonormalbasis} there are orthonormal basis $\beta=\set{v_i}$ with eigenvalue $\lambda_i$. Define $R(v_i) = \sqrt{\lambda_i} v_i$. Then $\forall v_i \in \beta,  R^2(v_i) = T(v_i)$.

    For \ref{positiveoperatorproperty1}, $\innerproduct{Tv}{v} = \innerproduct{R^*Rv}{v} = \innerproduct{Rv}{Rv} \geq 0$.
\end{proof}

\begin{theorem}
    A positive operator has a unique positive square root.
\end{theorem}

\begin{definition}
    If $T$ is a positive operator, $\sqrt{T}$ is its positive square root.
\end{definition}

% unitary and orthogonal operator
\subsection{Isometry}

\begin{definition}
    Let $T$ be a linear operator on finite dimensional inner product space $V$ over $F$. If $\forall x \in V$, $\norm{T(x)} = \norm{x}$, we call $T$ \cindex{unitary operator} if $F = \mathcal{C}$ or \cindex{orthogonal operator} if $F=\mathcal{R}$. Unitary and orthogonal are also called \cindex{isometry}.
\end{definition}

\begin{definition}
    A square matrix $A$ is called \cindex{unitary matrix} if $AA^* = A^*A = I$ and \cindex{orthogonal matrix} if $AA^\top = A^\top A = I$.
\end{definition}

\begin{theorem}\label{unitaryproperty}
    Let $T$ be an linear operator. Then the following are equivalent:
    \begin{enumerate}
        \item $TT^* = T^* T = I$.\label{unitaryisnormal}
        \item $\innerproduct{T(x)}{T(y)} = \innerproduct{x}{y}$.
        \item If $\beta$ is an orthonormal basis for $V$. Then $T(\beta)$ is an orthonormal basis.
        \item $\norm{T(x)} = \norm{x}$.
    \end{enumerate}

    So unitary or orthogonal operator preserve inner product and norm.
\end{theorem}
\begin{proof}
    $\innerproduct{x}{y} = \innerproduct{T^* T x}{y} = \innerproduct{T(x)}{T(y)}$.

    If $\beta = \{v_1,v_2,\dots,v_n \}$ is an orthonormal basis. $\innerproduct{T(v_i)}{T(v_j)} = \innerproduct{v_i}{v_j} = 0$.

    If $\beta$ and $T(\beta)$ are both orthonormal basis, expand $\norm{T(x)}$ and $\norm{x}$ to prove they are equal.

    $\innerproduct{x}{x} = \norm{x}^2 = \norm{T(x)}^2 = \innerproduct{T(x)}{T(x)} = \innerproduct{x}{T^*Tx}$. So $\forall x \in V, \innerproduct{x}{(I - T^*T)(x)} = 0$. $I - T^*T$ is normal, so according to \thmref{zerotforalltxforselfadjoint}, $I - T^* T = 0$.
\end{proof}

\begin{theorem}
    Unitary operator is normal.
\end{theorem}
\begin{proof}
    See \thmref{unitaryproperty} property (\ref{unitaryisnormal}).
\end{proof}

\begin{theorem}
    Let $T$ be a linear operator on \emph{real} inner product space $V$. $V$ has an orthonormal basis of eigenvectors of $T$ with absolute value of all eigenvalues equal to $1$ if and only if $T$ is self-adjoint and orthogonal.
\end{theorem}
\begin{proof}
    If $T$ is self-adjoint, there is orthonormal basis $\beta$ of eigenvectors. If $T$ is orthogonal, $\forall v_i \in \beta$, $\absolutevalue{\lambda_i} \times \norm{v_i} = \norm{\lambda_i v_i} = \norm{T(v_i)} = \norm{v_i}$, so $\absolutevalue{\lambda_i} = 1$.

    If $V$ has orthonormal basis $\beta$ of eigenvectors, $T$ is self-adjoint. $\forall v_i \in \beta$, we have $TT^* (v_i) = T(\lambda_i v_i ) = \lambda_i T(v_i) = \lambda_i^2 v_i$. If $\absolutevalue{\lambda_i} = 1$, $TT^* = I$.
\end{proof}

\begin{theorem}
    Let $T$ be a linear operator on \emph{complex} inner product space $V$. $V$ has an orthonormal basis of eigenvectors of $T$ with absolute value of all eigenvalues equal to  $1$ if and only if $T$ is unitary.
\end{theorem}
\begin{proof}
    If $T$ is unitary, it is normal, so there is orthonormal basis $\beta$ of eigenvectors. If $T$ is unitary, $\forall v_i \in \beta$, $\absolutevalue{\lambda_i} \times \norm{v_i} = \norm{\lambda_i v_i} = \norm{T(v_i)} = \norm{v_i}$, so $\absolutevalue{\lambda_i} = 1$.

    If $V$ has orthonormal basis $\beta$ of eigenvectors, $T$ is normal. If $\absolutevalue{\lambda_i} = 1$, $\forall v_i \in \beta$, $\absolutevalue{\lambda_i} \times \norm{v_i} = \norm{\lambda_i v_i} = \norm{T(v_i)} = \norm{v_i}$, so $\norm{T(v_i)} = \norm{v_i}$ and it is unitary.
\end{proof}

\begin{theorem}
    $T$ is isometry if $\coordinate{T}_\beta$ is isometry for a orthonormal basis $\beta$ of $V$.
\end{theorem}

\begin{definition}
    $A$ is \cindex{unitarily equivalent} or \cindex{orthogonally equivalent} to $D$ if and only if there exists a unitary or orthogonal matrix $P$ that $A = P^* D P$.
\end{definition}

\begin{theorem}
    Let $A$ be a complex square matrix. $A$ is normal if and only if it is unitarily equivalent to a diagonal matrix.
\end{theorem}

\begin{theorem}
    Let $A$ be a real square matrix. $A$ is symmetric if and only if it is orthogonally equivalent to a diagonal matrix.
\end{theorem}

% rigid motion
\subsection{Rigid motion}

\begin{definition}
    Let $V$ be real inner product space. $f: V \rightarrow V$ is a \cindex{rigid motion} if
    \begin{equation}
        \norm{f(x) - f(y)} = \norm{x - y}
    \end{equation}
\end{definition}

\begin{definition}
    Let $V$ be real inner product space. $g: V \rightarrow V$ is a \cindex{translation} by $v_0 \in V$ if
    \begin{equation}
        \exists v_0 \forall x \in V \left( g(x) = x + v_0 \right)
    \end{equation}
\end{definition}

\begin{theorem}
    A translation is a rigid motion. And a composite of rigid motion is rigid motion.
\end{theorem}

\begin{theorem}
    Let $f$ be a rigid motion. Then there exists a unique orthogonal operator $T$ and unique translation $g$ that $f = g \circ T$.
\end{theorem}
\begin{proof}
    Define $T(x) = f(x) - f(0)$. $T$ is a composite of rigid motion, so it is a rigid motion. Therefore $\norm{T(x)} = \norm{f(x) - f(0)} = \norm{x - 0} = \norm{x}$. Since
    \begin{equation*}
        \begin{aligned}
            \norm{T(x) - T(y)}^2 &= \norm{x}^2 - 2 \innerproduct{T(x)}{T(y)} + \norm{y}^2 \\
            \norm{x - y}^2 &= \norm{x}^2 - 2 \innerproduct{x}{y} + \norm{y}^2 \\
            \norm{T(x) - T(y)}^2 &= \norm{x - y}^2
        \end{aligned}
    \end{equation*}
    We have $\innerproduct{T(x)}{T(y)} = \innerproduct{x}{y}$.

    Then $\norm{T(ax + y) - aT(x) - T(y)}^2 = 0$ after expansion, $T$ is linear. So $T$ is an orthogonal operator. So we have unique $T$ and $g$ that
    \begin{equation}
        \begin{aligned}
            T(x) &= f(x) &- f(0) \\
            g(x) &= x &+ f(0)
        \end{aligned}
    \end{equation}
\end{proof}

\begin{theorem}
    Let $T$ be an orthogonal operator on $R^2$, and let $A = \coordinate{T}_\beta$ where $\beta$ is the standard basis of $R^2$. Then one of the following is satisfied:
    \begin{enumerate}
        \item $T$ is a rotation, so $\determinate{T} = 1$.
        \item $T$ is a reflection about a line through the origin, so $\determinate{T} = -1$.
    \end{enumerate}
\end{theorem}
\begin{proof}
    Because $T$ is orthogonal, $T(\beta) = \set{T(e_1), T(e_2)}$ is an orthonormal basis of $R^1$. Since $T(e_1)$ is an unit vector, it has the form $T(e_1) = (\cos{\theta}, \sin{\theta})$. Since $T(e_2)$ is orthogonal to $T(e_1)$, it has the form $T(e_2) = (-\sin{\theta}, \cos{\theta})$ or $T(e_2) = (\sin{\theta}, -\cos{\theta})$.
\end{proof}

\begin{theorem}
    For expression $f(x,y) = a x^2 + 2b xy + c y^2$, let $A = \begin{pmatrix}
        a & b \\
        b & c
    \end{pmatrix}$ and $X = \begin{pmatrix}
        x \\
        y
    \end{pmatrix}$, the formula is $f(X) = X^\top A X = \innerproduct{AX}{X}$. Since $A$ is symmetric, there is an orthogonal matrix $P$ and diagonal matrix $D$ that $A = P^\top D P$. Define $X_0 = \begin{pmatrix}
        x_0 \\
        y_0
    \end{pmatrix}$ that $X = PX_0$. We have $f(X) = X^\top A X = (P X_0)^\top A (P X_0) = X_0^\top D X_0 = \lambda_1 x_1^2 + \lambda_2 x_2^2$. So the $xy$ term could be removed by rotation.
\end{theorem}

% spectral theorem
\subsection{Spectral Theorem}

\begin{definition}
    Let $V = W_1 \oplus W_2$. $T$ is a \cindex{projection} on $W_1$ along $W_2$ if $\forall x = x_1 + x_2$ that $x_1 \in W_1$ and $x_2 \in W_2$, $T(x) = x_1$.
\end{definition}

\begin{theorem}
    $T$ is a projection if and only if $T^2 = T$.
\end{theorem}

\begin{definition}
    $T$ is an \cindex{orthogonal projection} if $\rangespace{T}^\bot = \nullspace{T}$ and $\rangespace{T}= \nullspace{T}^\bot$\footnote{In finite dimensional space $V$, $\rangespace{T}^\bot = \nullspace{T} \leftrightarrow \rangespace{T}= \nullspace{T}^\bot$}.
\end{definition}

\begin{theorem}
    $T$ is an orthogonal projection if and only if $T$ has an adjoint $T^*$ that $T^2 = T = T^*$.
\end{theorem}
\begin{proof}
    $T^2 = T$ because $T$ is a projection. Let $x=x_1+x+2$ and $y=y_1+y_2$ where $x_1,y_1 \in \rangespace{T}$ and $x_2,y_2 \in \nullspace{T}$. So
    \begin{equation*}
        \begin{aligned}
            \innerproduct{x}{T(y)} &= \innerproduct{x_1 + x_2}{y_1} = \innerproduct{x_1}{y_1} \\
            \innerproduct{T(x)}{y} &= \innerproduct{x_1}{y_1 + y_2} = \innerproduct{x_1}{y_1}
        \end{aligned}
    \end{equation*}
    So $T = T^*$ and $T^2 = T = T^*$.

    For the reverse side, prove that $\rangespace{T}^\bot = \nullspace{T}$ and $\rangespace{T}= \nullspace{T}^\bot$.
\end{proof}

\begin{theorem}[\cindex{Spectral Theorem}]
    Let $T$ be real symmetric or complex normal with distinct eigenvalue $\lambda_i$ and its corresponding eigenspace $W_i$. Let $T_i$ be the orthogonal projection on $W_i$. We have:
    \begin{enumerate}
        \item $T_i T_j = \delta_{ij} T_i$
        \item $\displaystyle I = \sum_{i=1}^k T_i$
        \item $\displaystyle T = \sum_{i=1}^k \lambda_i T_i$
    \end{enumerate}

    $\lambda_i$ is the \cindex{spectrum} of $T$. $I$ is the resolution of the identity operator induced by T. $\displaystyle T = \sum_{i=1}^k \lambda_i T_i$ is the \cindex{spectral decomposition} of $T$.
\end{theorem}
\begin{proof}
    Let $\displaystyle x= \sum_{i=1}^k x_i$ where $x_i \in W_i$. Then
    \begin{equation*}
        T(x) = \sum_{i=1}^k T(x_i) = \sum_{i=1}^k \lambda_i x_i= \sum_{i=1}^k \lambda_i T_i (x_i) = \sum_{i=1}^k \lambda_i T_i (x) = \left(\sum_{i=1}^k \lambda_i T_i \right) x
    \end{equation*}
\end{proof}

\begin{theorem}
    Let $F=\mathcal{C}$. $T$ is normal if and only if $\exists g \in P$, $T^* = g(T)$.
\end{theorem}
\begin{proof}
    Let $\displaystyle T = \sum_{i=1}^k \lambda_i T_i$ be the spectral decomposition of $T$. Take the adjoint of both side and we have
    \begin{equation}
        T^* = \sum_{i=1}^k \overline{\lambda_i} T_i^*
    \end{equation}
    According to Lagrange formula\footnote{Theorem (\ref{lagrangeinterpolationformula}) on page \pageref{lagrangeinterpolationformula}.} , $\exists g$, $g(\lambda_i) = \overline{\lambda_i}$. So $g(T) = T^*$. The reverse is easy to prove.
\end{proof}

\begin{theorem}
    Let $F=\mathcal{C}$. $T$ is unitary if and only if $T$ is normal and $\absolutevalue{\lambda} = 1$ for all eigenvalue $\lambda$ of $T$.
\end{theorem}
\begin{proof}
    Let $\displaystyle T = \sum_{i=1}^k \lambda_i T_i$ be the spectral decomposition of $T$. We have
    \begin{equation*}
    TT^* = \left( \sum_{i=1}^k \lambda_i T_i  \right) \times \left( \sum_{i=1}^k \overline{\lambda_i} T_i \right) = \sum_{i=1}^k \absolutevalue{\lambda_i}^2 T_i^2 = \sum_{i=1}^k \absolutevalue{\lambda_i}^2 T_i = \sum_{i=1}^k T_i  = I
    \end{equation*}
\end{proof}

\begin{theorem}
    Let $F=\mathcal{C}$ and $T$ normal. $T$ is self-adjoint if and only if every eigenvalue of $T$ is real.
\end{theorem}
\begin{proof}
    $\displaystyle T^* = \sum_{i=1}^k \overline{\lambda_i} T_i = \sum_{i=1}^k \lambda_i T_i = T$, so $\overline{\lambda_i} = \lambda_i$.
\end{proof}

% single value decomposition
\subsection{Single Value Decomposition}

\begin{theorem}
    Let $T:V \rightarrow W$ be a linear transformation with rank $r$. Then there exists orthonormal basis $\beta = \{v_1, v_2, \dots, v_n \}$ for $V$ and $\gamma = \{ u_1, u_2, \dots, u_m \}$ for $W$ and positive scalars \cindex{singular values} $\sigma_1 \geq \sigma_2 \geq \dots \geq \sigma_r$ such that
    \begin{equation}
        T(v_i) = \begin{cases}
            \sigma_i u_i & \text{if } 1 \leq i \leq r \\
            0 & \text{if } i > r
        \end{cases}
    \end{equation}

    Conversely, for $1 \leq i \leq n$, $v_i$ is an eigenvector of $T^*T$ with corresponding eigenvalue $\sigma_i^2$ if $1 \leq i \leq r$ and $0$ if $i > r$.
\end{theorem}
\begin{proof}
    $T^*T$ has rank $r$ according to \thmref{rankofadjoint}, and positive semidefinite by \thmref{positiveoperatorproperty}. So there is an orthonormal basis $v_i$ for $V$ consisting of eigenvectors of $T^*T$ with corresponding eigenvalues $\lambda_i$ where $\lambda_1 \geq \lambda_2 \geq \dots \geq \lambda_r > 0$ and $\lambda_i = 0$ for $i > r$. For $1 \leq i \leq r$, define $\sigma_i = \sqrt{\lambda_i}$ and $u_i = \dfrac{1}{\sigma_i} T(v_i)$. We have:
    \begin{equation*}
    \innerproduct{u_i}{u_j} = \innerproduct{\frac{1}{\sigma_i} T(v_i)}{\frac{1}{\sigma_j} T(v_j)} = \frac{1}{\sigma_i \sigma_j} \innerproduct{T^*T(v_i)}{v_j} = \frac{1}{\sigma_i \sigma_j} \innerproduct{\lambda_i v_i}{v_j} = \frac{\sigma_i^2}{\sigma_i \sigma_j} \innerproduct{v_i}{v_j} = \delta_{ij}
    \end{equation*}

    So $\{u_1, u_2, \dots, u_r \}$ are orthogonal. Because the choice of $\sqrt{\lambda_i}$, they are unitary and therefore orthonormal. Extend it to an orthonormal basis $\{u_1, u_2, \dots, u_m \}$.
\end{proof}

\begin{definition}
    The \cindex{singular values} of $A$ is the singular value of $L_A$.
\end{definition}

\begin{theorem}[Singular Value Decomposition Theorem]
    Let $A_{m \times n}$ be of rank $r$ with positive singular values $\sigma_1 \geq \sigma_2 \geq \dots \geq \sigma_r$, and let $\Sigma_{m \times n}$ be
    \begin{equation}
        \Sigma_{ij} = \begin{cases}
            \sigma_i & \text{if } i = j \leq r \\
            0
        \end{cases}
    \end{equation}
    Then there exists \cindex{singular value decomposition} that with $U_{m \times m}$ and $V_{n \times n}$, we have
    \begin{equation}
        A = U \Sigma V^*
    \end{equation}

    The process to find singular value decomposition is:
    \begin{enumerate}
        \item find singular value of $A$ by calculating the eigenvalue of $A^*A$.
        \item sort the singular value from big to small.
        \item for non-zero singular value $\sigma_i$, put $\sqrt{\sigma_i}$ to the $i$-th diagonal of $\Sigma$.
        \item form $U$ of normalized eigenvector of $A^*A$.
        \item for non-zero singular value $\sigma_i$, calculate orthonormal vector $u_i = \dfrac{1}{\sigma_i} L_A(v_i)$.
        \item expand the $u_i$ to orthonormal basis and form $V$.
    \end{enumerate}
\end{theorem}

% Polar Decomposition
\subsection{Polar Decomposition}

\begin{theorem}[Polar Decomposition]
    Any square matrix $A$, there exists a \cindex{Polar Decomposition} using unitary matrix $W$ and a positive semidefinite matrix $P$ that
    \begin{equation}
        A = WP
    \end{equation}
    If $A$ is invertible, the Polar Decomposition is unique.
\end{theorem}
\begin{proof}
    Use singular value decomposition on $A$ and we get $A = U \Sigma V^* = U V^* V \Sigma V^* = ( U V^*) ( V \Sigma V^*) =  WP$.
    So let $W = U V^*$ and $P = V \Sigma V^*$.
\end{proof}

% Pseudoinverse
\subsection{Pseudoinverse}

\begin{definition}
    Let $T: V \rightarrow W$ be a linear transformation. Let $L: \nullspace{T}^\bot \rightarrow \rangespace{T}$ be a linear transformation that $\forall x \in \nullspace{T}^\bot$, $L(x) = T(x)$. The \cindex{pseudoinverse} (or \cindex{Moore-Penrose generalised inverse}) of $T$ is a unique linear transformation from $W$ to $V$ that
    \begin{equation}
        T^\dag (y) = \begin{cases}
            L^{-1}(y) & \text{for } y \in \rangespace{T} \\
            0 & \text{for } y \in \rangespace{T}^\bot
        \end{cases}
    \end{equation}
    Let $\set{v_1, v_2,\dots, v_r}$ be a basis for $\nullspace{T}^\bot$, $\set{v_{r+1}, v_{r+2}, \dots, v_{n}}$ be a basis for $\nullspace{T}$, $\set{u_1, u_2, \dots, u_r}$ be basis for $\rangespace{T}$, $\set{u_{r_1}, u_{r+2}, \dots, u_m}$ be a basis for $\rangespace{T}^\bot$, then:
    \begin{equation*}
        T^\dag (u_i) = \begin{cases}
            \dfrac{1}{\sigma_i} v_i & \text{if } 1 \leq i \leq r \\
            0
        \end{cases}
    \end{equation*}
    So although not all $T$ has inverse, the restriction $\left. T \right|_{\nullspace{T}^\bot}$ could have proper inverse.
\end{definition}

\begin{theorem}
    Let $A_{m \times n}$ be a square matrix of rank $r$ with singular value decomposition $A = U \Sigma V^*$ and non-zero singular values $\sigma_1 \geq \sigma_2 \geq \dots \geq \sigma_r$. Let $\Sigma^\dag_{m \times n}$  be a matrix that
    \begin{equation}
        \Sigma^\dag_{ij} = \begin{cases}
            \frac{1}{\sigma_i} & \text{if } i = j \leq r \\
            0
        \end{cases}
    \end{equation}

    Then $A^\dag = V \Sigma^\dag U^*$ is a singular value decomposition of $A$.
\end{theorem}

\begin{theorem}
    Let $T: V \rightarrow W$ be a linear transformation, then
    \begin{enumerate}
        \item $T^\dag T$ is the orthogonal projection of $V$ on $\nullspace{T}^\bot$.
        \item $TT^\dag$ is the orthogonal projection of $W$ on $\rangespace{T}$.
    \end{enumerate}
\end{theorem}
\begin{proof}
    Define $L: \nullspace{T}^\bot \rightarrow W$ by $L(x) = T(x)$. If $x \in \nullspace{T}^\bot$, then $T^\dag T (x) = L^{-1} L (x) = x$. If $x \in \nullspace{T}$, then $T^\dag T (x) = T^\dag (0) = 0$.
\end{proof}

\begin{theorem}
    For a system of linear equations $Ax = b$. If $z = A^\dag b$, then
    \begin{enumerate}
        \item If $Ax=b$ is consistent, then $z$ is the unique solution with minimal norm.
        \item If $Ax=b$ is inconsistent, then $z$ is the best approximation: $\forall y$, $\norm{Ax-b} \leq \norm{Ay-b}$. Also if $Az = Ay$, then $\norm{z} \leq \norm{y}$.
    \end{enumerate}

    $A^\dag b$ is the optimal solution discussed in section \ref{consistentandinconsistentequation} on page \pageref{consistentandinconsistentequation}.
\end{theorem}
\begin{proof}
    Let $z = A^\dag b$. If the equation is consistent, then $b \in \rangespace{T}$, then $Az = AA^\dag b = TT^\dag (b) = b$ because $TT^\dag$ is a orthogonal projection, so $z$ is a solution to the linear system.

    If $y$ is any solution, then $T^\dag T (y) = A^\dag A y = A^\dag b = z$. So $z$ is a orthogonal projection of $y$ on $\nullspace{T}^\bot$. So $\norm{z} \leq \norm{y}$.

    If the equation is inconsistent, then $Az = AA^\dag b$ is the orthogonal projection of $b$ on $\rangespace{T}$, so $Az$ is the nearest vector to $b$.
\end{proof}

% conditioning
\subsection{Conditioning}

\begin{definition}
    For $Ax=b$, if a small change to $A$ and $b$ cause small change to $x$, the property is called \cindex{well-conditioned}. Otherwise the system is \cindex{ill-conditioned}.
\end{definition}

\begin{definition}
    The \cindex{relative change} in $b$ is $\dfrac{\norm{\dif{b}}}{\norm{b}}$ with $\norm{\cdot}$ be the standard norm on $\mathcal{C}^n$.
\end{definition}

\begin{definition}
    The \cindex{Euclidean norm} of square matrix $A$ is
    \begin{equation}
        \norm{A} = \max_{x \neq 0} \frac{\norm{Ax}}{\norm{x}}
    \end{equation}
\end{definition}

\begin{definition}
    Let $B$ be a self-adjoint matrix. The \cindex{Rayleigh quotient} for $x \neq 0$ is $R(x) = \dfrac{\innerproduct{Bx}{x}}{\norm{x}^2}$
\end{definition}

\begin{theorem}
For a self-adjoint matrix $B$, the $\displaystyle \max_{x \neq 0} R(x)$ is the largest eigenvalue of $B$ and $\displaystyle \min_{x \neq 0} R(x)$ is the smallest eigenvalue of $B$.
\end{theorem}
\begin{proof}
    Choose the orthonormal basis $v_i$ of $B$ such that $Bv_i = \lambda_i v_i$ where $\lambda_1 \geq \lambda_2 \geq \lambda_n$. $\forall x \in F^n$, $\exists a_i$ that $\displaystyle x = \sum_{i=1}^n a_i v_i$. So
    \begin{equation*}
        R(x) = \frac{\innerproduct{Bx}{x}}{\norm{x}^2} = \frac{\innerproduct{\sum_{i=1}^n a_i \lambda_i v_i}{\sum_{j=1}^n a_j v_j}}{\norm{x}^2} = \frac{\sum_{i=1}^n \lambda_i \absolutevalue{a_i}^2}{\norm{x}^2} \leq \frac{\lambda_1 \sum_{i=1}^n \absolutevalue{a_i}^2}{\norm{x}^2} = \frac{\lambda_1 \norm{x}^2}{\norm{x}^2} = \lambda_1
    \end{equation*}
\end{proof}

\begin{theorem}
    $\norm{A} = \sqrt{\lambda}$ where $\lambda$ is the largest eigenvalue of $A^* A$.
\end{theorem}

\begin{theorem}
    $\lambda$ is an eigenvalue of $A^* A$ if and only if $\lambda$ is an eigenvalue of $AA^*$.
\end{theorem}

\begin{theorem}
    Let $A$ be invertible matrix. Then $\norm{A^{-1}} = \dfrac{1}{\sqrt{\lambda}}$ where $\lambda$ is the smallest eigenvalue of $A^*A$.
\end{theorem}

\begin{definition}
    $\norm{A} \times \norm{A^{-1}}$ is the \cindex{condition number} of $A$ and denoted as $\text{cond}(A)$.
\end{definition}

\begin{theorem}
    For system $Ax=b$ where $A$ is invertible and $b \neq 0$, we have:
    \begin{enumerate}
        \item For any norm $\norm{\cdot}$, we have $\dfrac{1}{\text{cond}(A)} \dfrac{\norm{\dif b}}{\norm{b}} \leq \dfrac{\norm{\dif x}}{\norm{x}} \leq \text{cond}(A) \dfrac{\norm{\dif b}}{\norm{b}}$.
        \item If $\norm{\cdot}$ is the Euclidean norm, then $\text{cond}(A) = \sqrt{\dfrac{\lambda_1}{\lambda_n}}$ where $\lambda_1$ and $\lambda_n$ are the largest and smallest eigenvalue of $A^*A$.
    \end{enumerate}

    So when $\text{cond} (b) \geq 1$. If $\text{cond}(b)$ is close to $1$, the relative error in $x$ is small when relative error of $b$ is small. However when $\text{cond}(b)$ is large, the relative error in $x$ could be large or small.

    $\text{cond}(x)$ is seldom calculated because when calculating $A^{-1}$ in computer, there are rounding errors which is related to $\text{cond}(A)$.
\end{theorem}


\documentclass[12pt]{cdblatex}
\usepackage[final]{pdfpages}

\begin{document}

% =======================================================================================
\section*{The ADM evolution equations.}

The vacuum ADM equations, exactly as written in the following Cadabra code, are as follows.

\IfFileExists{adm-eqtns.cdbtex}{}{Where is {\tt adm-eqtns.cdbtex}?}

\begin{dgroup*}
   \begin{dmath*} \cdb{dotgab.lhs} = \Cdb*{dotgab.101} \end{dmath*}
   \begin{dmath*} \cdb{dotKab.lhs} = \Cdb*{dotKab.101} \end{dmath*}
   \begin{dmath*} \cdb{dotN.lhs}   = \Cdb*{dotN.101}   \end{dmath*}
   \begin{dmath*} \cdb{Ham.lhs}    = \Cdb*{Ham.101}    \end{dmath*}
   \begin{dmath*} \cdb{Mom.lhs}    = \Cdb*{Mom.101}    \end{dmath*}
\end{dgroup*}

% Note that the covariant derivative for the 3-metric is usually denoted by the vertical bar

Cadabra's job was to express $R_{ab}$, $R$, $N_{ab}$ and $D_c$ in terms of the ADM
variables and their partial derivatives. It's all plain sailing from here, so cutting to
the chase, here are the results.

\begin{dgroup*}
   \begin{dmath*} \cdb{Rab.lhs} = \Cdb*{Rab.112} \end{dmath*}
   \begin{dmath*} \cdb{R.lhs}   = \Cdb*{R.110} \end{dmath*}
   \begin{dmath*} \cdb{Nab.lhs} = \Cdb*{Nab.102} \end{dmath*}
   \begin{dmath*} \cdb{Mom.lhs} = \Cdb*{Mom.110}   \end{dmath*}
\end{dgroup*}

\clearpage

% =======================================================================================
\section*{The ADM evolution equations. The big picture.}

\begin{cadabra}
   from shared import *
   import cdblib

   jsonfile = 'adm-eqtns.json'
   cdblib.create (jsonfile)

   # ------------------------------------------------------------------------------------
   # generic rules for covariant derivs

   deriv1 := A?_{; m}   -> \partial_{m}{A?}.                                              # cdb(deriv1.lhs,deriv1)
   deriv2 := A?_{; m n} -> \partial_{m}{A?_{; n}} - \Gamma^{c}_{m n} A?_{; c}.            # cdb(deriv2.lhs,deriv2)

   substitute (deriv2, deriv1)                                                            # cdb (deriv2.101,deriv2)

   deriv3 := A?_{m n ; p} -> \partial_{p}{A?_{m n}} - \Gamma^{c}_{m p} A?_{c n}
                                                    - \Gamma^{c}_{n p} A?_{m c}.          # cdb(deriv3.lhs,deriv3)

   # ------------------------------------------------------------------------------------
   # partial derivs of g_{ab} in terms of partial of g^{ab}

   defDgab := {g^{a e} g^{b f} \partial_{c}{g_{e f}} -> - \partial_{c}{g^{a b}},
               g^{e a} g^{b f} \partial_{c}{g_{e f}} -> - \partial_{c}{g^{a b}},
               g^{a e} g^{f b} \partial_{c}{g_{e f}} -> - \partial_{c}{g^{a b}},
               g^{e a} g^{f b} \partial_{c}{g_{e f}} -> - \partial_{c}{g^{a b}}}.         # cdb (defDgab.lhs,defDgab)

   # ------------------------------------------------------------------------------------
   # standard defintions

   defGamma := \Gamma^{a}_{b c} ->
               (1/2) g^{a e} (   \partial_{b}{g_{e c}}
                               + \partial_{c}{g_{b e}}
                               - \partial_{e}{g_{b c}}).                                  # cdb (defGamma.lhs,defGamma)

   defRabcd := R^{a}_{b c d} ->
               \partial_{c}{\Gamma^{a}_{b d}} + \Gamma^{a}_{e c} \Gamma^{e}_{b d}
             - \partial_{d}{\Gamma^{a}_{b c}} - \Gamma^{a}_{e d} \Gamma^{e}_{b c}.        # cdb (defRabcd.lhs,defRabcd)

   defRab := R_{a b} -> R^{c}_{a c b}.                                                    # cdb (defRab.lhs,defRab)

   # ------------------------------------------------------------------------------------
   # Ricci tensor

   Rab := R_{a b}.                                         # cdb (Rab.lhs,Rab)

   substitute     (Rab, defRab)                            # cdb (Rab.101,Rab)
   substitute     (Rab, defRabcd)                          # cdb (Rab.102,Rab)
   substitute     (Rab, defGamma)                          # cdb (Rab.103,Rab)
   product_rule   (Rab)                                    # cdb (Rab.104,Rab)
   distribute     (Rab)                                    # cdb (Rab.105,Rab)

   Rab = product_sort (Rab)                                # cdb (Rab.106,Rab)

   rename_dummies (Rab)                                    # cdb (Rab.107,Rab)
   canonicalise   (Rab)                                    # cdb (Rab.108,Rab)
   substitute     (Rab, defDgab)                           # cdb (Rab.109,Rab)

   Rab = product_sort (Rab)                                # cdb (Rab.110,Rab)

   rename_dummies (Rab)                                    # cdb (Rab.111,Rab)
   canonicalise   (Rab)                                    # cdb (Rab.112,Rab)

   defRab := R_{a b} -> @(Rab).

   # ------------------------------------------------------------------------------------
   # Ricci scalar

   Rscalar := R.                                           # cdb (R.lhs,Rscalar)
   Rscalar := g^{a b} R_{a b}.                             # cdb (R.101,Rscalar)

   substitute     (Rscalar, defRab)                        # cdb (R.102,Rscalar)
   distribute     (Rscalar)                                # cdb (R.103,Rscalar)

   Rscalar = product_sort (Rscalar)                        # cdb (R.104,Rscalar)

   rename_dummies (Rscalar)                                # cdb (R.105,Rscalar)
   canonicalise   (Rscalar)                                # cdb (R.106,Rscalar)
   substitute     (Rscalar, defDgab)                       # cdb (R.107,Rscalar)

   Rscalar = product_sort (Rscalar)                        # cdb (R.108,Rscalar)

   rename_dummies (Rscalar)                                # cdb (R.109,Rscalar)
   canonicalise   (Rscalar)                                # cdb (R.110,Rscalar)

   defRscalar := R -> @(Rscalar).

   # ------------------------------------------------------------------------------------
   # Hessian

   Nab := N_{; a b}.                                       # cdb (Nab.lhs,Nab)

   substitute (Nab, deriv2)                                # cdb (Nab.101,Nab)
   substitute (Nab, defGamma)                              # cdb (Nab.102,Nab)

   defHess := N_{; a b} -> @(Nab).                         # cdb (Hess.lhs,defHess)

   # ------------------------------------------------------------------------------------
   # ADM evolution equations

   DgabDt := \partial_{t}{g_{a b}}.                        # cdb (dotgab.lhs,DgabDt)
   DKabDt := \partial_{t}{K_{a b}}.                        # cdb (dotKab.lhs,DKabDt)
   DNDt   := \partial_{t}{N}.                              # cdb (dotN.lhs,DNDt)

   DgabDt := -2 N K_{a b}.                                                        # cdb (dotgab.101,DgabDt)
   DKabDt := -N_{; a b} + N (R_{a b} + trK K_{a b} - 2 K_{a c} K_{b d} g^{c d}).  # cdb (dotKab.101,DKabDt)
   # DNDt := -2 N trK.     # 1+log
   # DNDt := -N*N trK.     # Harmonic
   DNDt := 0.                                              # cdb (dotN.101,DNDt)  # Static

   substitute (DKabDt,defHess)                             # cdb (dotKab.102,DKabDt)
   distribute (DKabDt)                                     # cdb (dotKab.103,DKabDt)

   # ------------------------------------------------------------------------------------
   # The Hamiltonian contsraint

   defHam := Ham     -> R + K_{a b} g^{a b} K_{c d} g^{c d} - K_{a b} K_{c d} g^{a c} g^{b d}.
   Ham    := Ham.                                          # cdb (Ham.lhs,Ham)
   substitute     (Ham, defHam)                            # cdb (Ham.101,Ham)

   canonicalise   (Ham)                                    # cdb (Ham.102,Ham)

   # ------------------------------------------------------------------------------------
   # The momentum contsraint

   defMom := Mom_{c} -> g^{a b} K_{a c ; b} - \partial_{c}{g^{a b} K_{a b}}.
   Mom    := Mom_{c}.                                      # cdb (Mom.lhs,Mom)
   substitute     (Mom, defMom)                            # cdb (Mom.101,Mom)

   substitute     (Mom, deriv3)                            # cdb (Mom.102,Mom)
   product_rule   (Mom)                                    # cdb (Mom.103,Mom)
   distribute     (Mom)                                    # cdb (Mom.104,Mom)
   substitute     (Mom, defGamma)                          # cdb (Mom.105,Mom)
   distribute     (Mom)                                    # cdb (Mom.106,Mom)
   substitute     (Mom, defDgab)                           # cdb (Mom.107,Mom)

   Mom = product_sort (Mom)                                # cdb (Mom.108,Mom)

   rename_dummies (Mom)                                    # cdb (Mom.109,Mom)
   canonicalise   (Mom)                                    # cdb (Mom.110,Mom)

   cdblib.put ('Rscalar', Rscalar, jsonfile)
   cdblib.put ('Rab',     Rab,     jsonfile)
   cdblib.put ('Nab',     Nab,     jsonfile)
   cdblib.put ('DgabDt',  DgabDt,  jsonfile)
   cdblib.put ('DKabDt',  DKabDt,  jsonfile)
   cdblib.put ('DNDt',    DNDt,    jsonfile)
   cdblib.put ('Ham',     Ham,     jsonfile)
   cdblib.put ('Mom',     Mom,     jsonfile)

\end{cadabra}

\clearpage

% =======================================================================================
\section*{The Hessian of the lapse.}

% \begin{dgroup*}[spread=5pt]
%    \begin{dmath*}
%       \cdb{Nab.lhs}
%          = \Cdb*{Nab.101}
%    \end{dmath*}
%    \begin{dmath*}
%       \phantom{\cdb{Nab.lhs}}
%          = \Cdb*{Nab.102}
%    \end{dmath*}
% \end{dgroup*}

\begin{align*}
   \cdb{Nab.lhs}
      &= \Cdb{Nab.101}\\
      &= \Cdb{Nab.102}
\end{align*}

% =======================================================================================
\section*{The Ricci curvature.}

\begin{dgroup*}[spread=5pt]
   \begin{dmath*}
      \cdb{Rab.lhs}
         = \Cdb*{Rab.101}
         = \Cdb*{Rab.102}
         = \Cdb*{Rab.103}
         = \Cdb*{Rab.104}
         = \Cdb*{Rab.105}
         = \Cdb*{Rab.106}
   \end{dmath*}
\end{dgroup*}

\clearpage

\begin{dgroup*}[spread=5pt]
   \begin{dmath*}
      \cdb{Rab.lhs}
         = \Cdb*{Rab.107}
         = \Cdb*{Rab.108}
         = \Cdb*{Rab.109}
         = \Cdb*{Rab.110}
         = \Cdb*{Rab.111}
         = \Cdb*{Rab.112}
   \end{dmath*}
\end{dgroup*}

\clearpage

% =======================================================================================
\section*{The Ricci scalar.}

\begin{dgroup*}[spread=5pt]
   \begin{dmath*}
      \cdb{R.lhs}
         = \Cdb*{R.101}
         = \Cdb*{R.102}
         = \Cdb*{R.103}
         = \Cdb*{R.104}
         = \Cdb*{R.105}
         = \Cdb*{R.106}
         = \Cdb*{R.107}
   \end{dmath*}
\end{dgroup*}

\clearpage

\begin{dgroup*}[spread=5pt]
   \begin{dmath*}
      \cdb{R.lhs}
          = \Cdb*[\hskip 2cm\hfill]{R.108}
          = \Cdb*[\hskip 2cm\hfill]{R.109}
          = \Cdb*{R.110}
   \end{dmath*}
\end{dgroup*}

\clearpage

% =======================================================================================
\section*{The ADM constraints.}

\begin{dgroup*}[spread=5pt]
   \begin{dmath*}
      \cdb{Ham.lhs}
         = \Cdb*{Ham.101}
         = \Cdb*{Ham.102}
   \end{dmath*}
\end{dgroup*}

\begin{dgroup*}[spread=5pt]
   \begin{dmath*}
      \cdb{Mom.lhs}
         = \Cdb*{Mom.101}
         = \Cdb*{Mom.102}
         = \Cdb*{Mom.103}
         = \Cdb*{Mom.104}
         = \Cdb*{Mom.105}
         = \Cdb*{Mom.106}
         = \Cdb*{Mom.107}
         = \Cdb*{Mom.108}
         = \Cdb*{Mom.109}
         = \Cdb*{Mom.110}
   \end{dmath*}
\end{dgroup*}

\end{document}


\documentclass{article}

\usepackage{fancyhdr}
\usepackage{extramarks}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amsfonts}
\usepackage[plain]{algorithm}
\usepackage{algpseudocode}
\usepackage{matlab-prettifier}
\usepackage{graphicx}
\usepackage[export]{adjustbox}
%
% Basic Document Settings
%
\lstMakeShortInline[style=Matlab-editor]"

\topmargin=-1in
\evensidemargin=0in
\oddsidemargin=0in
\textwidth=6.5in
\textheight=9.0in
\headsep=0.25in

\linespread{1.1}

\rhead{\firstxmark}
\lfoot{\lastxmark}
\cfoot{\thepage}

\renewcommand\headrulewidth{0.4pt}
\renewcommand\footrulewidth{0.4pt}

%
% Homework Details
%   - Title
%   - Due date
%   - Class
%   - Section/Time
%   - Instructor
%   - Author
%

\newcommand{\hmwkTitle}{AMATH 482 Homework 4: Extended Yale Faces B Database – Eigenfaces \& Music Genre Identification}
\newcommand{\hmwkDueDate}{March 8, 2019}
\newcommand{\hmwkClassInstructor}{Professor Nathan Kutz}
\newcommand{\hmwkAuthorName}{\textbf{Skyler Hallinan}}

%
% Title Page
%

\title{
    \textmd{\textbf{\text{ } \hmwkTitle}}\\
}

\author{\hmwkAuthorName}
\date{}

\begin{document}
\maketitle

\section*{\fontsize{19}{15}\selectfont Abstract}
		In part one, we demonstrated the reduction of images by doing an SVD analysis on cropped and uncropped sets, and reconstructed them with lower dimensions. In part two, we trained various supervised machine learning algorithms by using the singular value decomposition of short segments of fourier transformed audio. We showed that some models had remarkably high accuracy, and that SVD was not always beneficial.
\section*{\fontsize{19}{15}\selectfont Introduction and Overview}
	Often, we have too many features in our data. In this homework, we explored the feature reduction in images by using the Singular Value Decomposition on a matrix of images. We determined how many "significant" features were needed to reconstruct the faces after applying this SVD transform, and plotted the images again after projecting the images to a new orthonormal basis and reducing the features. \\ \\
	Machine learning extracts meaningful featuers from data, and bins data into distinct patterns that can be later used for decision making; it can learn from and make predictions on the data. There is both supervised and unsupervised machine learning. No labels are given in unsupervised learning, and the model must organize and cluster data by itself, based on observe patterns, and try to go to a lower rank subspace.  Supervised machine learning is a great tool to classify and predict new data after training a model with sample data and labels. In all of these machine learning algorithms, the main goal is to construct a low-rank feature space of a given dataset; this can be either done automatically with unsupverised learning, or manually, with PCA modes generated by SVD, where only a certain $r$ number of features are needed to represent the data. In this homework, we explored the Naive Bayes, Support Vector Machine, and Random Forests algorithm to classify our data using this supervised machine learning. We gave three test sets of songs, and saw how they performed.
\section*{\fontsize{19}{15}\selectfont Theoretical Background}
	The Singular Value Decomposition is used to reformat a matrix into the following:
	\begin{align*}
\mathbf { A } = \mathbf { U } \boldsymbol { \Sigma } \mathbf { V } ^ { * } \text{ where} \\
\begin{array} { l } { \mathbf { U } \in \mathbb { C } ^ { m \times m } \text { is unitary } } \\ { \mathbf { V } \in \mathbb { C } ^ { n \times n } \text { is unitary } } \\ { \boldsymbol { \Sigma } \in \mathbb { R } ^ { m \times n } \text { is diagonal } } \end{array}
	\end{align*}
	The diagonal values of $\sigma$ are nonnegative and ordered from largest to smallest. We can also compute the SVD with:
\begin{align*}
\mathbf { A } ^ { T } \mathbf { A } & = \left( \mathbf { U } \boldsymbol { \Sigma } \mathbf { V } ^ { * } \right) ^ { T } \left( \mathbf { U } \boldsymbol { \Sigma } \mathbf { V } ^ { * } \right) \\ & = \mathbf { V } \boldsymbol { \Sigma } \mathbf { U } ^ { * } \mathbf { U } \boldsymbol { \Sigma } \mathbf { V } ^ { * } \\ & = \mathbf { V } \boldsymbol { \Sigma } ^ { 2 } \mathbf { V } ^ { * }  \\ \\
\mathbf { A } \mathbf { A } ^ { T } & = \left( \mathbf { U } \boldsymbol { \Sigma } \mathbf { V } ^ { * } \right) \left( \mathbf { U } \boldsymbol { \Sigma } \mathbf { V } ^ { * } \right) ^ { T } \\ & = \mathbf { U \Sigma V } ^ { * } \mathbf { V } \boldsymbol { \Sigma } \mathbf { U } ^ { * } \\ & = \mathbf { U } \boldsymbol { \Sigma } ^ { 2 } \mathbf { U } ^ { * } \\ \\
\mathbf { A } ^ { T } \mathbf { A V } & = \mathbf { V } \Sigma ^ { 2 } \\ \mathbf { A } \mathbf { A } ^ { T } \mathbf { U } & = \mathbf { U } \boldsymbol { \Sigma } ^ { 2 }
\end{align*}
Computing the normalized eigenvectors for the final two equations gives orthonormal basis vectors for $U$ and $V$. In addition, the square root of the eigenvalues of these equations produces the singular values $\sigma_j$. The SVD enables for every matrix to be diagonal if the proper bases for the domain and the range are used. \\ \\
	One of the primary applications of the SVD is for Principal Component Analysis (PCA). The PCA allows for large, complex, and even somewhat random sets of data to be reduced to lower dimensions of dynamics without any knowledge of underlying behavior.
	The covariance matrix will show correlations between all variables in the system; strong statistically dependent variables can be classified as redundant. Since we are looking for the covariance of a specific matrix, we have $\mathbf { C } _ { \mathbf { X } } = \frac { 1 } { n - 1 } \mathbf { X X } ^ { T }$, where $C_X$ is a square $m \times m$. This covariance matrix is key to understanding redundancies in data; large values correspond to redundancy. However, large diagonal terms also correspond to large variances, which suggest strong fluctuations in specific variables, and help identify important components. Large variances are thus dynamics of interest, while small variances are non-interesting. Thus, our goal is to view this $C_X$ matrix ordered from largest to smallest, with off-diagonal values of 0 (diagonilization). \\ \\
	The SVD does this. Each singular direction in the SVD captures as much energy as possible, which is measured by the singular values $\theta_j$.  We know that SVD can diagnonalize any matrix via using the appropriate bases $U$ and $V$ as shown above. We define the transformed variable $Y = U*X$ where $U$ is the unitary transformation associated with the SVD ($\mathbf { X } = \mathbf { U } \boldsymbol { \Sigma } \mathbf { V } ^ { * }$). We then calculate the variance in Y:
 \begin{align*}
C_Y = \frac { 1 } { n - 1 } \mathbf { Y } \mathbf { Y } ^ { T }\\
= \frac { 1 } { n - 1 } \left( \mathbf { U } ^ { * } \mathbf { X } \right) \left( \mathbf { U } ^ { * } \mathbf { X } \right) ^ { T } \\
= \frac { 1 } { n - 1 } \mathbf { U } ^ { * } \left( \mathbf { X } \mathbf { X } ^ { T } \right) \mathbf { U } \\
= \frac { 1 } { n - 1 } \mathbf { U } ^ { * } \mathbf { U } \boldsymbol { \Sigma } ^ { 2 } \mathbf { U } \mathbf { U } ^ { * } \\
C_Y= \frac { 1 } { n - 1 } \mathbf { \Sigma } ^ { 2 }
\end{align*}

	We can apply the SVD to a matrix of images in order to isolate the highest singular components and reduce the number of features in the data. The resulting matrices will give us information on the most important features within all the images in the matrix, and will give us a new orthonormal set that we can project the images onto. We will also be able to look at the energy components of this by looking at the singular values and dividing them by the maximum value.\\ \\
\textbf{Part 2}
We have seen that the fourier transform is an integral transform that decomposes a time defined function into its frequency components, and is defined by
	\begin{equation} \label{eq:2a}
		F(k) = \frac{1}{\sqrt2\pi} \int_{-\infty}^{\infty} e^{-ikx} f(x) dx
	\end{equation}
		\begin{equation} \label{eq:2b}
		f(x) = \frac{1}{\sqrt2\pi} \int_{-\infty}^{\infty} e^{ikx} F(k) dk
	\end{equation}
	The fast fourier transform, which is accessible in things like MATLAB, assumes that the user is working on a $2\pi$ periodic domain. \\ \\
A downside to the fourier transform is that in changing the domain to the frequency domain, it completely removes all time data from the equation. However, since we will be using the fourier transform to look at the frequency content of different audio samples, we do not care about this. In addition, the fourier transform gives complex outputs, so it will be necessary to take the absolute value before proceeding forward and giving information to the models. \\ \\

Naive Bayes classifier is a conditional probability model; given a vector $x$ with some $n$ features, it assigns an instance probability $P_{C_k}$ where $C_k$ is some class or label in the system. 	Using Bayes Theorem, we can write the probability as $p\left(C_{k} | \mathbf{x}\right)=\frac{p\left(C_{k}\right) p(\mathbf{x} | C_{k})}{p(\mathbf{x})}$. Under assumptions of indepdence, we see that Naive Bayes has conditional distribution over some class $C$ of $p\left(C_{k} | x_{1}, \ldots, x_{n}\right)=\frac{1}{Z} p\left(C_{k}\right) \prod_{i=1}^{n} p\left(x_{i} | C_{k}\right)$. \\ \\
Linear SVM is based on constructing a hyperplane $\mathbf{w} \cdot \mathbf{x}+b=0$, where vector w and constant b paramterize the hyperplane. SVM optimizes the decision line that makes the fewest labeling errors for the data, and optimizes the largest margin between the data. $w$ and $b$ must be optimized in a principle way, wihch is by stating a loss function:	$\ell\left(\mathbf{y}_{j}, \overline{\mathbf{y}}_{j}\right)=\left\{\begin{aligned} 0 & \text { if data is correctly labeled } \\+1 & \text { if data is incorrectly labeled } \end{aligned}\right.$. This can be simplified down to $\underset{\mathbf{w}, b}{\operatorname{argmin}} \sum_{j=1}^{m} H\left(\mathbf{y}_{j}, \overline{\mathbf{y}}_{j}\right)+\frac{1}{2}\|\mathbf{w}\|^{2} \quad$ subject to $\min _{j}\left|\mathbf{x}_{j} \cdot \mathbf{w}\right|=1$. There are also kernel methods for SVD, which solves the curse of dimensinoality, when there are too many additional features. We define the kernel function as $K\left(\mathbf{x}_{j}, \mathbf{x}\right)=\Phi\left(\mathbf{x}_{j}\right) \cdot \Phi(\mathbf{x})$, and we get a new optimization problem $\underset{\boldsymbol{\alpha}, b}{\operatorname{argmin}} \sum_{j=1}^{m} H\left(\mathbf{y}_{j}, \overline{\mathbf{y}}_{j}\right)+\frac{1}{2}\left\|\sum_{j=1}^{m} \alpha_{j} \Phi\left(\mathbf{x}_{j}\right)\right\|^{2} \quad$ subject to $\min _{j}\left|\mathbf{x}_{j} \cdot \mathbf{w}\right|=1$. This allows us to represent Taylor series expansions of observables in a compact way. The radial basis function and polynomial kernel are commonly used. \\ \\
Random forests are an example of decision tree learning. The decision tree is a hierarchical construct that optimally splits data for classification and regression. The model scans through each feature and compares the prediction accuracy across different features. It branches when it reaches a feature giving the best segmentation.
\section*{\fontsize{19}{15}\selectfont Algorithm Development}
	\textbf{Part 1} \\
	We first loaded in the images for both cropped and uncropped. We used "dir" and iteratively went through all the folders, loading in each image, using "double" to convert our uint8 data matrices into workable formats. We reshaped each image using $reshape$ and placed it in a matrix, so that we had 1 row representing an image. \\ \\
We then applied the "svd" to this matrix. We plotted the energies of the singular values and looked at how many modes could represent a high percentage of energy (99.9\%) in the system. Once we found a value, we reconstructed the matrix by subsetting the "u", "s", and "v" matrices by this rank $r$, then multiplied "u*s*v.'". Each column of this corresponded to a reconstruction of an image with lower dimension. We plotted out the first 9 of these reconstructed faces. \\ \\
\textbf{Part 2} \\
	We first selected our audio. For test case one, we needed to choose audio from three different artists/bands that were from different genres. We chose to use audio from Billy Joel (Rock), Travis Scott (Rap / Hip Hop), and Liszt (Classical Piano), who have music from three diverse and different categories. We downloaded five pieces/songs from each of the artists, and loaded them into "MATLAB" as a vector using "[mus, Fs] = audioread(file)". \\ \\
The procedure for one song was as follows: The songs were in stereo, but for the purpose of this analysis, we converted them to mono. To do this, if both channels were nonzero, we averaged their value, but if at least one channel was 0, we took the max of the two channels instead. This resulted in a 1D vector representing music. We then resampled this data by taking the 10th data point from each of these vectors, effectively reducing our data size by one order of magnitude. We then removed leading and trailing 0s from the audio using $find$ and conditional statements. We split these into 5 second segments by subsetting from an index to the index plus five times the sampling rate ("Fs"), then reshaped these into a matrix, where each column of the matrix represented a 5 second clip of the mono version of the song, using "reshape". Finally, we created a label vector that corresponded to the number of 5 second clips from this artist, with the label name being the artist's name. We repeated this for all songs and all artists, iteratively adding them to a large dataset of 5 second clips, along with labels. This resulted in a large data matrix where each column was a vector representation of a 5 second clip from a song, and a large 1D array of labels. \\ \\
Next, we fourier-transformed each song (each column of the data) using the fast fourier transform "fft". We took the absolute value of this so that some of our models could use the data (like the SVM). We then took the SVD of this data, using "svd" and getting the "u","s", and "v" matrices. We used "diag" to isolate the singular values to a 1D array from the "s" matrix, and calculated the number of modes at which 95\% of energy was captured. We then truncated our "u" matrix to be this length which remove redundant and less important information. We multiplied the transpose of u by our original data matrix. With this transformed dataset, we used "randperm" to generate a random set of data to extract for training, and a random set to extract for testing. We used roughly 8 times the number of clips for training than we did for testing. \\ \\
We then trained supervised machine learning methods using this training data, and the accompanying labels. We used Naive Bayes, Random Forests, and Support Vector Machine, using "fitcnb",  "fitctree", "fitceoc" respectively. We created a cross validated model and non cross-validated model for each of these three. We then assessed the inaccuracy using "loss" with an input of the test data as well as the correct labels, which returned a scalar from 0 to 1 representing the inaccuracy of the model. We used "kFoldloss" with the same test data to assess the inaccuracy of the cross-validated data. We subtracted 1 minus these inaccuracies to obtain the accuracies of these models. We then plotted these to compare how each model performed.
	We repeated the above procedure for test case two, where we distinguished between three bands of the same genre. For this case, we used artists Drake, Travis Scott, and Migos. Finally, we repeated this for case three, where we distinguished between three genres in general. For this case, we used Pop, Rock, and Motown.
\section*{\fontsize{19}{15}\selectfont Computational Results}
	\textbf{Part 1: Faces} \\
\begin{figure}[H]
\begin{center}
\includegraphics[width = 8cm]{var1}
\includegraphics[width = 8cm]{var2}
\caption{ Plots of energy for cropped (left) and uncropped (right)}
\end{center}
\end{figure}

\begin{figure}[H]
\begin{center}
\includegraphics[width = 8cm]{crop}
\includegraphics[width = 8cm]{uncrop}
\caption{ Plots of energy for cropped (left) and uncropped (right)}
\end{center}
\end{figure}
We looked at the energy diagrams for both sets of images to reconstruct them with a lower rank. For the uncropped images, we found that a rank of 108 was sufficient enough to convey 99.9\% of the energy. For the cropped images, we found this value to be 160. We were able to come up with clear reconstructions of the data, as shown in the above images. We see that the rank of the face space is 108 and 160 respectively for uncropped and cropped. In our SVD decomposition, our U is the eigenface space. This means that our U*S is our weighted eigen face; they are the "modes" or the basis vectors in n-dimensional space. The S matrix tells us how strong each of these vectors explains the variance; how important each of these principal components are to our data. Finally, V gives the coefficients for each of the data point's projection into the new basis; the ith column of the V matrix represents the coefficients of linear combinations in which we add U*S to get the ith face back. We see that compared to the cropped faces, the uncropped faces do not need as many singular modes to reconstruct the data at a high energy level. \\ \\
	\textbf{Part 2: Music Classification} \\ \\
\begin{figure}[H]
\begin{center}
\includegraphics[width = 16cm]{acc1}
\caption{\label{fig:scaled_diss}  Plots of accuracy of supervised learning algorithms (test 1)}
\end{center}
\end{figure}
\begin{figure}[H]
\begin{center}
\includegraphics[width = 18cm]{use1}
\caption{\label{fig:scaled_diss} Plots of energy (right) and accuracy with variable mode number (left)  (test 1) }
\end{center}
\end{figure}
	\textbf{Test 1: Band Classification}
	We saw that we were able to capture 95\% of energy with 217 modes, as shown in Figure 4, so we used this number for our truncation of our "u" matrix.. We then compared the prediction accuracy of each of the models, as shown in Figure 3. We see that Naive Bayes performed remarkably well on non cross validated and cross validated data. For SVD, it had 88.73\% and 84.14\% accuracy respectively, and for non SVD, 95.16\% and 94.26\% percent accuracy. We see that with SVM, with SVD, it had low classifcation accuracies of non cross validated and cross validated data, both under 45\%, while for the non SVD data in these categories, it had a 97.06\% classification and 54.08\% classification. We see that there is a large descrepancy in SVM's accuracy of this non SVD data; it has high classification accuracy for non-cross validated, but not for cross-validated. For RF, we observed somewhat high accuracies for the SVD data (0.7760 and 0.6692 for n-cv and cv), and very high accuracies for both non-cross validated and cross validated non SVD data, both being above 91\%. Figure 4 shows that for SVD data, Naive  Bayes had consistently high accuracy with variable number of modes, followed by random forests and SVM. However, all three models were above the 50\% mark, which shows that they are all competent. \\ \\
\textbf{Test 2: The Case for Seattle}
We saw that we were able to capture 95\% of energy with 180 modes, as shown in Figure 6. Figure 5 shows the relative accuracies for each of our models. For NB, we saw that it did not perform well on the SVD data, with accuracies near or slightly below 0.5, for non-crossvalidated and cross-validated cases. However, for the non SVD data, it performed well for both cross-validated and non-cross validated, with accuracies of 87.26\% and 84.64\% respectively.  We saw similar results with RF; low accuracy (under 0.5) for SVD data, and high (>75\%) accuracy for non SVD data for both cross-validated and non-cross validated. For our SVM, it did not perform well for 3/4 of our tests, with classification under 50\%. However, for the non SVD data with no cross-validation, it had an astonishing 99.01\% accuracy. This was the highest we observed, and was especially impressive due to the general trend of low accuracies for this test case. In addition, Figure 6 shows the huge variability in accuracy in all 6 SVD classification methods by changing the number of modes. Since we were classifying within the same genre, we expect this, because the model should theoretically be worse since there are not as many differences. This is why we see so low classification rates, as well as a huge variability with number of modes. \\ \\
\begin{figure}[H]
\begin{center}
\includegraphics[width = 16cm]{acc2}
\caption{\label{fig:scaled_diss}  Plots of accuracy of supervised learning algorithms (test 2)}
\end{center}
\end{figure}
\begin{figure}[H]
\begin{center}
\includegraphics[width = 18cm]{use2}
\caption{\label{fig:scaled_diss} Plots of energy (right) and accuracy with variable mode number (left) (test 2)}
\end{center}
\end{figure}
\textbf{Test 3: Genre Classification}
We found that at 239 modes we were able to represent 95\% of the energy, as shown in Figure 8. In terms of accuracy, all three models (NB, SVM, RF) were not very good at classifying the SVD data, for both non cross-validated cases cross-validated and. For SVM and RF, they had under 50 \% classification for these categories. NB was slightly better with 63.21\% and 64.48\% classification for those categories respectively. Once again, our models were much better at classifying non SVD data: NB was solid with two classification accuracies above 83\% for cross validated and non cross validated, while random forests was slightly worse, with two values above 73\%. Once again, SVM showed strange behavior with the non SVD data: It had remarkbly high classification accuracy of 92.37 \% for the non-cross validated data, and a much lower 45.56\% classification percentage for the cross-validated data. Figure 5 of the accuracies of models on SVD data showed that NB (no cross validation) and NB (cross validation) had consistently high accuracies, depending on the number of nodes, while some models did not perform well at any number, such as both SVM models. Overall, although this graph was less sporadic than Figure 6 for test case 2, there is still some variability, especially compared to test case 1. This shows that it is harder to distinguish between genres when different artists are used, as there is more variability within each genre, since different artists are used. In addition, one reason for the reduced accuracy of this model may be that the genres could have been slightly more similar to eachohter (Rock, Pop, and Motown) than they were in test case 1 (Rap, Classical, Rock). \\ \\
Overall, we found that SVM of non SVD, non cross-validated was consistently accurate for all three test cases. Naive Bayes also had consistently high accuracy for non SVD, in both non cross-validated and cross-validated data, but not as high. We also observe that the SVD classification had much lower accuracy than non SVD data. It makes sense that the non SVD classification had higher accuracy rates, because we are lowering the dimension and feature space with SVD, so we might lose information and accuracy in our model to gain speed (and decrease in computation time). Finally, we note that we had our worst total results in test 2, where we had to distinguish between artists in the same genre. This makes sense because their music will be similar.
\begin{figure}[H]
\begin{center}
\includegraphics[width = 16cm]{acc3}
\caption{\label{fig:scaled_diss}  Plots of accuracy of supervised learning algorithms (test 3)}
\end{center}
\end{figure}
\begin{figure}[H]
\begin{center}
\includegraphics[width = 18cm]{use3}
\caption{\label{fig:scaled_diss} Plots of energy (right) and accuracy with variable mode number (left) (test 3)}
\end{center}
\end{figure}

\section*{\fontsize{19}{15}\selectfont Summary and Conclusions}
In part 1, we were able to perform an SVD across faces and lower the rank after identifying the number of modes that conveyed a lot of the energy. We were then able to reproduce these faces. In part 2, we were able to develop classification models for audio identification using supervised machine learning models. We found that our models performed best when the genres were very different, and the worst when they were the most similar, which is to be expected. In addition, SVM was our best model overall, and non SVD datasets had better classification accuracies than SVD ones.
\pagebreak

\section*{\fontsize{19}{15}\selectfont Appendix A}
\subsection*{MATLAB functions used and implementation}
"audioread(file)": Loads a audio file as one or two vectors (depending on if the file is mono or stereo). We used this to load in our music for classification in part 2. \\
"diag(X)" : Returns a column vector of the diagonal values of a matrix. We used this to generate our variance numbers from SVD. \\ \\
"find(X)" : Returns a vector of non-zero indeces in a matrix. We used this in our conditional matrix to find locations in the image of bright spots, which corresponded to moving object. We then used in "ind2sub" to find direct coordinates.  \\ \\
"fitcnb(data, labels)" : Fits a Naive Bayes Model across the rows of "data", using "labels" to create a model. Can specify cross validation with (..., "crossval", "on") We used this for our music classifcation. \\ \\
"fitcecoc(data, labels)" : Fits a multi-labeled Support Vector Machine with "data", using "labels" to create a model. Can specify cross validation with (..., "crossval", "on") We used this for our music classifcation. \\ \\
"fitctree(data, labels)" : Fits a Random Forests model with "data", using "labels" to create a model. Can specify cross validation with (..., "crossval", "on") We used this for our music classifcation. \\ \\
"floor(x)" : Returns the rounded down integer of a double. We used this to calculate 5 second segments of our sample audio and reshape it. \\ \\
"imread(file)": Reads in an image file as a 3D or 2D matrix (based on if the images is color or grayscale). We used this to load in the Yale Faces for part 1. \\ \\
"kFoldloss(Mdl, data, labels)" : Returns a number between 0 to 1 representing the loss (inaccuracy) of a given cross-volidated model "Mdl", where predicted labels are compared to actual ones ("labels"). We used this to assess the accuracy of our cross validated models. \\ \\
"loss(Mdl, data, labels)" : Returns a number between 0 to 1 representing the loss (inaccuracy) of a given model "Mdl", where predicted labels are compared to actual ones ("labels"). We used this to assess the accuracy of our various models. \\ \\
"max(A)" : If "A" is a vector, then it returns the maximum value of this vector. We used this for normalization of data. \\ \\
"randperm(limit, n)" : Returns a random permutation from 1 to "limit" of size "n". We used this to randomly select training and testing data for model classification. \\ \\
"reshape(A,sz)" : Reshapes the array A to the size "sz". We used this to resize our image matrices into a vector for part 1, and to resize our audio into 5 second segments in part 2. \\ \\
"size(X)" : Returns the dimensions of the matrix "X". We used this to calculate size for audio. \\ \\
"[u,s,v] = svd(X)" : Returns the $U$, $S$, and $V$ matrices corresponded with the singular value decomposition of "X". We used this for the eigen faces in part 1, and for music classification in part 2. \\ \\

\section*{\fontsize{19}{15}\selectfont Appendix B}
\subsection*{MATLAB code}
\begin{lstlisting}[style=Matlab-editor]
clear all; close all; clc;

close all; clear all; clc;
%% Importing all images from directories (uncropped)
storagestan = [];
Dcy = dir("yalefaces");
for k = 3:length(Dcy)
    data = double(imread(strcat("yalefaces\", Dcy(k).name)));
    storagestan = [storagestan; reshape(data,1,243*320)];
end
%% Importing all images from directories (Cropped)

storagecrop = [];
Dcy = dir("CroppedYale");
for k = 1:length(Dcy)
   curD = Dcy(k).name;
   Dtemp = dir(strcat("CroppedYale\",curD, "\*.pgm"));

   for j = 1:length(Dtemp)
       data = double(imread(strcat("CroppedYale\",curD, "\", Dtemp(j).name)));
       storagecrop = [storagecrop; reshape(data,1,192*168)];
   end
end
%% Computation (cropped)
tic
[u,s,v] = svd(storagecrop', 'econ');
toc

%% Plotting and reconstrunction (cropped)
sig = diag(s);
lambda = sig.^2;

subplot(1,2,1)
plot(cumsum(lambda/sum(lambda)), 'bo')
refline(0,0.99)
legend("Energy Captured", "99% of Energy Captured", "Location", "Southeast");
ylabel("Cumulative Energy Captured by Diagonal Variances "); xlabel("Diagonal Variances");
title("Cumulative Energy Captured (Cropped)", "Fontsize", 14);
subplot(1,2,2)
plot(lambda/sum(lambda), 'ro')
set(gca, 'YScale', 'log')
ylim([10e-7, 1]); ylabel("Log of Energy Captured by Each Diagonal Variance"); xlabel("Diagonal Variances");
title("Log Of Energy Captured (Cropped)", "Fontsize", 14);
%% Eigen Faces
reconstruct = u(:,1:160) *s(1:160,:)*v(1:160,:)';
for z = 1:9
    curimg = reshape(reconstruct(:,z), [192,168]);
    subplot(3,3,z);
    pcolor(flip(curimg)), shading interp, colormap(gray)
end

%% NEW PART
tic
[u1,s1,v1] = svd(storagestan', 'econ');
toc

sig1 = diag(s);
lambda1 = sig1.^2;

reconstruct1 = u1(:,1:108) *s1(1:108,:)*v1(1:108,:)';
for z = 1:9
    curimg = reshape(reconstruct1(:,z), [243,320]);
    subplot(3,3,z);
    pcolor(flip(curimg)), shading interp, colormap(gray)
end

sig1 = diag(s1);
lambda1 = sig1.^2;

subplot(1,2,1)
plot(cumsum(lambda1/sum(lambda1)), 'bo')
refline(0,0.99)
legend("Energy Captured", "99% of Energy Captured", "Location", "Southeast");
ylabel("Cumulative Energy Captured by Diagonal Variances"); xlabel("Diagonal Variances");
title("Cumulative Energy Captured  (Uncropped)", "Fontsize", 14);
subplot(1,2,2)
plot(lambda1/sum(lambda1), 'ro')
set(gca, 'YScale', 'log')
ylim([10e-7, 1]); ylabel("Log of Energy Captured by Each Diagonal Variance"); xlabel("Diagonal Variances");
title("Log Of Energy Captured (Uncropped)", "Fontsize", 14);

close all; clear all; clc;

%% Import audio files
alldata = [];
Alabels = [];
for str = {"travi", "billy", "liszt"}
    for i = 1:5
        [song, Fs] = audioread(strcat("Music\",str{1}, num2str(i),".mp3"));
        Fs = Fs/10;

        %Lower sampling rate
        song = song(1:10:end,:);

        monosong = [];
        %Convert from stereo to mono
        for z = 1:length(song)
            if (song(z,1) == 0 || song(z,2) == 0)
                monosong(z,1) = max(song(z,:));
            else
                monosong(z,1) = (song(z,1) + song(z,2))/2;
            end
        end

        %Remove leading and trailing 0s
        monosong = monosong(find(monosong,1,'first'):find(monosong,1,'last'));

        %5 second intervals
        chunk_size = Fs*5;

        endpoint = floor(length(monosong) / chunk_size);
        monosong = monosong(1:(chunk_size * endpoint),1);

        data = reshape(monosong, [chunk_size, endpoint]);
        alldata = [alldata, data];
        Alabels = [Alabels; repmat(str{1}, size(data,2) ,1)];
    end
end
Alabels = Alabels.';
traindata = alldata;
trainlabels = Alabels;
%% Rearrange and apply fft
[u,s,v] = svd(abs(fft(alldata)), 'econ');
trainlabels = Alabels;

sig = diag(s);
lambda = sig.^2;

utrunc = u(:, 1:217);
traindata = utrunc'*alldata;

n = floor(size(traindata, 2)/(8));
c = randperm(size(traindata,2), n);

testdata = traindata(:,c);
testlabels = trainlabels(:,c);

traindata(:,c) = [];
trainlabels(:,c) = [];

nosvdtraindata = abs(fft(alldata));
nosvdtestdata = nosvdtraindata(:,c);
nosvdtraindata(:,c) = [];
%% Modeling
nbMdl = fitcnb(traindata.', trainlabels);
nbL = loss(nbMdl, testdata.', testlabels)

%Cross validation model
nbcvMdl = fitcnb(traindata.', trainlabels, 'crossval', 'on');
nbcvL = kfoldLoss(nbcvMdl,'LossFun','ClassifErr')

svmMdl = fitcecoc(traindata.', trainlabels);
svmL = loss(svmMdl, testdata.', testlabels)

%Cross validation model
svmcvMdl = fitcecoc(traindata.', trainlabels, 'crossval', 'on');
svmcvL = kfoldLoss(svmcvMdl,'LossFun','ClassifErr')

rfMdl = fitctree(traindata.', trainlabels);
rfL = loss(rfMdl, testdata.', testlabels)

%Cross validation model
rfcvMdl = fitctree(traindata.', trainlabels, 'crossval', 'on');
rfcvL = kfoldLoss(rfcvMdl,'LossFun','ClassifErr')

%% Modeling (no SVD)
nosvdnbMdl = fitcnb(nosvdtraindata.', trainlabels);
nosvdnbL = loss(nosvdnbMdl, nosvdtestdata.', testlabels)

%Cross validation model
nosvdnbcvMdl = fitcnb(nosvdtraindata.', trainlabels, 'crossval', 'on');
nosvdnbcvL = kfoldLoss(nosvdnbcvMdl,'LossFun','ClassifErr')

nosvdsvmMdl = fitcecoc(nosvdtraindata.', trainlabels);
nosvdsvmL = loss(nosvdsvmMdl, nosvdtestdata.', testlabels)

%Cross validation model
nosvdsvmcvMdl = fitcecoc(traindata.', trainlabels, 'crossval', 'on');
nosvdsvmcvL = kfoldLoss(nosvdsvmcvMdl,'LossFun','ClassifErr')

nosvdrfMdl = fitctree(nosvdtraindata.', trainlabels);
nosvdrfL = loss(nosvdrfMdl, nosvdtestdata.', testlabels)

%Cross validation model
nosvdrfcvMdl = fitctree(nosvdtraindata.', trainlabels, 'crossval', 'on');
nosvdrfcvL = kfoldLoss(nosvdrfcvMdl,'LossFun','ClassifErr')

%% Plotting
close all;

bar(1-[nbL nbcvL nosvdnbL nosvdnbcvL; svmL svmcvL nosvdsvmL nosvdsvmcvL; rfL rfcvL nosvdrfL nosvdrfcvL])
xticklabels(["Naive Bayes", "Support Vector Machine", "Random Forests"]);
legend("SVD, No Cross Validation", "SVD, Cross Validation", "No SVD, No Cross Validation", "No SVD, Cross Validation",...
    "Location", "south");
ylabel("Classification Accuracy (Scale of 0 to 1)"); xlabel("Supervised Learning Algorithms");
title("Test 1: Accuracy of Trained Models of Three Bands in Different Genres (217 Modes)", 'FontSize', 20);
grid on

%% Loop
tic
classify= []
for z = 1:25:size(u,2)
    trunc = u(:, 1:z);
    traindata = utrunc'*alldata;
    trainlabels = Alabels;

    n = floor(size(traindata, 2)/(8));
    c = randperm(size(traindata,2), n);

    testdata = traindata(:,c);
    testlabels = trainlabels(:,c);
    traindata(:,c) = [];
    trainlabels(:,c) = [];

    nbMdl = fitcnb(traindata.', trainlabels);
    nbcvMdl = fitcnb(traindata.', trainlabels, 'crossval', 'on');

    svmMdl = fitcecoc(traindata.', trainlabels);
    svmcvMdl = fitcecoc(traindata.', trainlabels, 'crossval', 'on');

    rfMdl = fitctree(traindata.', trainlabels);
    rfcvMdl = fitctree(traindata.', trainlabels, 'crossval', 'on');

    classify= [classify; loss(nbMdl, testdata.', testlabels) , kfoldLoss(nbcvMdl,'LossFun','ClassifErr'), ...
                       loss(svmMdl, testdata.', testlabels), kfoldLoss(svmcvMdl,'LossFun','ClassifErr'), ...
                        loss(rfMdl, testdata.', testlabels), kfoldLoss(rfcvMdl,'LossFun','ClassifErr')];

end
toc

%% Plot bigger stuff
close all;

subplot(1,2,1)
plot(1:25:size(u,2), 1-classify, "Linewidth", 2)
title("Test 1: Model Accuracy vs Number of Modes", "Fontsize", 20);
xlabel("Number of Modes"); ylabel("Accuracy (Scale of 0 to 1)");
refline(0,0.5)
legend("Naive Bayes", "Naive Bayes (Cross Validation)", "SVM", "SVM (Cross Validation)", ...
    "Random Forests", "Random Forests (Cross Validation)","50% Accuracy Mark", "Location", "best");

subplot(1,4,3)
plot(cumsum(lambda/sum(lambda)), 'bo')
refline(0,0.95)
legend("Energy Captured", "95% of Energy Captured", "Location", "Southeast");
ylabel("Cumulative Energy Captured by Diagonal Variances"); xlabel("Diagonal Variances");
title("Cumulative Energy Captured", "Fontsize", 14);
subplot(1,4,4)
plot(lambda/sum(lambda), 'ro')
set(gca, 'YScale', 'log')
ylim([10e-5, 1]); ylabel("Log of Energy Captured by Each Diagonal Variance"); xlabel("Diagonal Variances");
title("Log Of Energy Captured", "Fontsize", 14);

close all; clear all; clc;

%% Import audio files
alldata = [];
Alabels = [];
for str = {"drake", "travi", "migos"}
    for i = 1:5
        [song, Fs] = audioread(strcat("Music\",str{1}, num2str(i),".mp3"));
        Fs = Fs/10;

        %Lower sampling rate
        song = song(1:10:end,:);

        monosong = [];
        %Convert from stereo to mono
        for z = 1:length(song)
            if (song(z,1) == 0 || song(z,2) == 0)
                monosong(z,1) = max(song(z,:));
            else
                monosong(z,1) = (song(z,1) + song(z,2))/2;
            end
        end

        %Remove leading and trailing 0s
        monosong = monosong(find(monosong,1,'first'):find(monosong,1,'last'));

        %5 second intervals
        chunk_size = Fs*5;

        endpoint = floor(length(monosong) / chunk_size);
        monosong = monosong(1:(chunk_size * endpoint),1);

        data = reshape(monosong, [chunk_size, endpoint]);
        alldata = [alldata, data];
        Alabels = [Alabels; repmat(str{1}, size(data,2) ,1)];
    end
end
Alabels = Alabels.';
traindata = alldata;
trainlabels = Alabels;
%% Rearrange and apply fft
[u,s,v] = svd(abs(fft(alldata)), 'econ');
trainlabels = Alabels;

sig = diag(s);
lambda = sig.^2;

utrunc = u(:, 1:180);
traindata = utrunc'*alldata;

n = floor(size(traindata, 2)/(8));
c = randperm(size(traindata,2), n);

testdata = traindata(:,c);
testlabels = trainlabels(:,c);

traindata(:,c) = [];
trainlabels(:,c) = [];

nosvdtraindata = abs(fft(alldata));
nosvdtestdata = nosvdtraindata(:,c);
nosvdtraindata(:,c) = [];
%% Modeling
nbMdl = fitcnb(traindata.', trainlabels);
nbL = loss(nbMdl, testdata.', testlabels)

%Cross validation model
nbcvMdl = fitcnb(traindata.', trainlabels, 'crossval', 'on');
nbcvL = kfoldLoss(nbcvMdl,'LossFun','ClassifErr')

svmMdl = fitcecoc(traindata.', trainlabels);
svmL = loss(svmMdl, testdata.', testlabels)

%Cross validation model
svmcvMdl = fitcecoc(traindata.', trainlabels, 'crossval', 'on');
svmcvL = kfoldLoss(svmcvMdl,'LossFun','ClassifErr')

rfMdl = fitctree(traindata.', trainlabels);
rfL = loss(rfMdl, testdata.', testlabels)

%Cross validation model
rfcvMdl = fitctree(traindata.', trainlabels, 'crossval', 'on');
rfcvL = kfoldLoss(rfcvMdl,'LossFun','ClassifErr')

%% Modeling (no SVD)
nosvdnbMdl = fitcnb(nosvdtraindata.', trainlabels);
nosvdnbL = loss(nosvdnbMdl, nosvdtestdata.', testlabels)

%Cross validation model
nosvdnbcvMdl = fitcnb(nosvdtraindata.', trainlabels, 'crossval', 'on');
nosvdnbcvL = kfoldLoss(nosvdnbcvMdl,'LossFun','ClassifErr')

nosvdsvmMdl = fitcecoc(nosvdtraindata.', trainlabels);
nosvdsvmL = loss(nosvdsvmMdl, nosvdtestdata.', testlabels)

%Cross validation model
nosvdsvmcvMdl = fitcecoc(traindata.', trainlabels, 'crossval', 'on');
nosvdsvmcvL = kfoldLoss(nosvdsvmcvMdl,'LossFun','ClassifErr')

nosvdrfMdl = fitctree(nosvdtraindata.', trainlabels);
nosvdrfL = loss(nosvdrfMdl, nosvdtestdata.', testlabels)

%Cross validation model
nosvdrfcvMdl = fitctree(nosvdtraindata.', trainlabels, 'crossval', 'on');
nosvdrfcvL = kfoldLoss(nosvdrfcvMdl,'LossFun','ClassifErr')

%% Plotting
close all;

bar(1-[nbL nbcvL nosvdnbL nosvdnbcvL; svmL svmcvL nosvdsvmL nosvdsvmcvL; rfL rfcvL nosvdrfL nosvdrfcvL])
xticklabels(["Naive Bayes", "Support Vector Machine", "Random Forests"]);
legend("SVD, No Cross Validation", "SVD, Cross Validation", "No SVD, No Cross Validation", "No SVD, Cross Validation");
ylabel("Classification Accuracy (Scale of 0 to 1)"); xlabel("Supervised Learning Algorithms");
title("Test 2: Accuracy of Trained Models of Three Bands in Same Genre (180 Modes)", 'FontSize', 20);
grid on
%% Loop
tic
classify= []
for z = 1:25:size(u,2)
    trunc = u(:, 1:z);
    traindata = utrunc'*alldata;
    trainlabels = Alabels;

    n = floor(size(traindata, 2)/(8));
    c = randperm(size(traindata,2), n);

    testdata = traindata(:,c);
    testlabels = trainlabels(:,c);
    traindata(:,c) = [];
    trainlabels(:,c) = [];

    nbMdl = fitcnb(traindata.', trainlabels);
    nbcvMdl = fitcnb(traindata.', trainlabels, 'crossval', 'on');

    svmMdl = fitcecoc(traindata.', trainlabels);
    svmcvMdl = fitcecoc(traindata.', trainlabels, 'crossval', 'on');

    rfMdl = fitctree(traindata.', trainlabels);
    rfcvMdl = fitctree(traindata.', trainlabels, 'crossval', 'on');

    classify= [classify; loss(nbMdl, testdata.', testlabels) , kfoldLoss(nbcvMdl,'LossFun','ClassifErr'), ...
                       loss(svmMdl, testdata.', testlabels), kfoldLoss(svmcvMdl,'LossFun','ClassifErr'), ...
                        loss(rfMdl, testdata.', testlabels), kfoldLoss(rfcvMdl,'LossFun','ClassifErr')];

end
toc

%% Plot bigger stuff
close all;

subplot(1,2,1)
plot(1:25:size(u,2), 1-classify, "Linewidth", 2)
title("Test 2: Model Accuracy vs Number of Modes", "Fontsize", 20);
xlabel("Number of Modes"); ylabel("Accuracy (Scale of 0 to 1)");
refline(0,0.5)
legend("Naive Bayes", "Naive Bayes (Cross Validation)", "SVM", "SVM (Cross Validation)", ...
    "Random Forests", "Random Forests (Cross Validation)","50% Accuracy Mark", "Location", "Southwest");

subplot(1,4,3)
plot(cumsum(lambda/sum(lambda)), 'bo')
refline(0,0.95)
legend("Energy Captured", "95% of Energy Captured", "Location", "Southeast");
ylabel("Cumulative Energy Captured by Diagonal Variances"); xlabel("Diagonal Variances");
title("Cumulative Energy Captured", "Fontsize", 14);
subplot(1,4,4)
plot(lambda/sum(lambda), 'ro')
set(gca, 'YScale', 'log')
ylim([10e-5, 1]); ylabel("Log of Energy Captured by Each Diagonal Variance"); xlabel("Diagonal Variances");
title("Log Of Energy Captured", "Fontsize", 14);

close all; clear all; clc;

%% Import audio files
alldata = [];
Alabels = [];
for str = {"pop", "rock", "motown"}
    for i = 1:5
        [song, Fs] = audioread(strcat("Music\",str{1}, num2str(i),".mp3"));
        Fs = Fs/10;

        %Lower sampling rate
        song = song(1:10:end,:);

        monosong = [];
        %Convert from stereo to mono
        for z = 1:length(song)
            if (song(z,1) == 0 || song(z,2) == 0)
                monosong(z,1) = max(song(z,:));
            else
                monosong(z,1) = (song(z,1) + song(z,2))/2;
            end
        end

        %Remove leading and trailing 0s
        monosong = monosong(find(monosong,1,'first'):find(monosong,1,'last'));

        %5 second intervals
        chunk_size = Fs*5;

        endpoint = floor(length(monosong) / chunk_size);
        monosong = monosong(1:(chunk_size * endpoint),1);

        data = reshape(monosong, [chunk_size, endpoint]);
        alldata = [alldata, data];
        Alabels = [Alabels; repmat(str{1}, size(data,2) ,1)];
    end
end
Alabels = Alabels.';
traindata = alldata;
trainlabels = Alabels;
%% Rearrange and apply fft
[u,s,v] = svd(abs(fft(alldata)), 'econ');
trainlabels = Alabels;

sig = diag(s);
lambda = sig.^2;

utrunc = u(:, 1:239);
traindata = utrunc'*alldata;

n = floor(size(traindata, 2)/(8));
c = randperm(size(traindata,2), n);

testdata = traindata(:,c);
testlabels = trainlabels(:,c);

traindata(:,c) = [];
trainlabels(:,c) = [];

nosvdtraindata = abs(fft(alldata));
nosvdtestdata = nosvdtraindata(:,c);
nosvdtraindata(:,c) = [];
%% Modeling
nbMdl = fitcnb(traindata.', trainlabels);
nbL = loss(nbMdl, testdata.', testlabels)

%Cross validation model
nbcvMdl = fitcnb(traindata.', trainlabels, 'crossval', 'on');
nbcvL = kfoldLoss(nbcvMdl,'LossFun','ClassifErr')

svmMdl = fitcecoc(traindata.', trainlabels);
svmL = loss(svmMdl, testdata.', testlabels)

%Cross validation model
svmcvMdl = fitcecoc(traindata.', trainlabels, 'crossval', 'on');
svmcvL = kfoldLoss(svmcvMdl,'LossFun','ClassifErr')

rfMdl = fitctree(traindata.', trainlabels);
rfL = loss(rfMdl, testdata.', testlabels)

%Cross validation model
rfcvMdl = fitctree(traindata.', trainlabels, 'crossval', 'on');
rfcvL = kfoldLoss(rfcvMdl,'LossFun','ClassifErr')

%% Modeling (no SVD)
nosvdnbMdl = fitcnb(nosvdtraindata.', trainlabels);
nosvdnbL = loss(nosvdnbMdl, nosvdtestdata.', testlabels)

%Cross validation model
nosvdnbcvMdl = fitcnb(nosvdtraindata.', trainlabels, 'crossval', 'on');
nosvdnbcvL = kfoldLoss(nosvdnbcvMdl,'LossFun','ClassifErr')

nosvdsvmMdl = fitcecoc(nosvdtraindata.', trainlabels);
nosvdsvmL = loss(nosvdsvmMdl, nosvdtestdata.', testlabels)

%Cross validation model
nosvdsvmcvMdl = fitcecoc(traindata.', trainlabels, 'crossval', 'on');
nosvdsvmcvL = kfoldLoss(nosvdsvmcvMdl,'LossFun','ClassifErr')

nosvdrfMdl = fitctree(nosvdtraindata.', trainlabels);
nosvdrfL = loss(nosvdrfMdl, nosvdtestdata.', testlabels)

%Cross validation model
nosvdrfcvMdl = fitctree(nosvdtraindata.', trainlabels, 'crossval', 'on');
nosvdrfcvL = kfoldLoss(nosvdrfcvMdl,'LossFun','ClassifErr')

%% Plotting
close all;

bar(1-[nbL nbcvL nosvdnbL nosvdnbcvL; svmL svmcvL nosvdsvmL nosvdsvmcvL; rfL rfcvL nosvdrfL nosvdrfcvL])
xticklabels(["Naive Bayes", "Support Vector Machine", "Random Forests"]);
legend("SVD, No Cross Validation", "SVD, Cross Validation", "No SVD, No Cross Validation", "No SVD, Cross Validation");
ylabel("Classification Accuracy (Scale of 0 to 1)"); xlabel("Supervised Learning Algorithms");
title("Test 3: Accuracy of Trained Models of Three Genres (239 Modes)", 'FontSize', 20);
grid on
%% Loop
tic
classify= []
for z = 1:25:size(u,2)
    trunc = u(:, 1:z);
    traindata = utrunc'*alldata;
    trainlabels = Alabels;

    n = floor(size(traindata, 2)/(8));
    c = randperm(size(traindata,2), n);

    testdata = traindata(:,c);
    testlabels = trainlabels(:,c);
    traindata(:,c) = [];
    trainlabels(:,c) = [];

    nbMdl = fitcnb(traindata.', trainlabels);
    nbcvMdl = fitcnb(traindata.', trainlabels, 'crossval', 'on');

    svmMdl = fitcecoc(traindata.', trainlabels);
    svmcvMdl = fitcecoc(traindata.', trainlabels, 'crossval', 'on');

    rfMdl = fitctree(traindata.', trainlabels);
    rfcvMdl = fitctree(traindata.', trainlabels, 'crossval', 'on');

    classify= [classify; loss(nbMdl, testdata.', testlabels) , kfoldLoss(nbcvMdl,'LossFun','ClassifErr'), ...
                       loss(svmMdl, testdata.', testlabels), kfoldLoss(svmcvMdl,'LossFun','ClassifErr'), ...
                        loss(rfMdl, testdata.', testlabels), kfoldLoss(rfcvMdl,'LossFun','ClassifErr')];

end
toc

%% Plot bigger stuff
close all;

subplot(1,2,1)
plot(1:25:size(u,2), 1-classify, "Linewidth", 2)
title("Test 3: Model Accuracy vs Number of Modes", "Fontsize", 20);
xlabel("Number of Modes"); ylabel("Accuracy (Scale of 0 to 1)");
refline(0,0.5)
legend("Naive Bayes", "Naive Bayes (Cross Validation)", "SVM", "SVM (Cross Validation)", ...
    "Random Forests", "Random Forests (Cross Validation)","50% Accuracy Mark", "Location", "Southwest");

subplot(1,4,3)
plot(cumsum(lambda/sum(lambda)), 'bo')
refline(0,0.95)
legend("Energy Captured", "95% of Energy Captured", "Location", "Southeast");
ylabel("Cumulative Energy Captured by Diagonal Variances"); xlabel("Diagonal Variances");
title("Cumulative Energy Captured", "Fontsize", 14);
subplot(1,4,4)
plot(lambda/sum(lambda), 'ro')
set(gca, 'YScale', 'log')
ylim([10e-5, 1]); ylabel("Log of Energy Captured by Each Diagonal Variance"); xlabel("Diagonal Variances");
title("Log Of Energy Captured", "Fontsize", 14);

\end{lstlisting}
\end{document}