\documentclass{article}
\usepackage[utf8]{inputenc,hyperref}
\usepackage{amsmath,amsfonts,amssymb,amsthm,verbatim}
\newcommand{\R}{\mathbb{R}}

\title{MM Tensor Decomp progress}
\author{Jason Yang}
\date{Oct 2021 -}

\begin{document}

\maketitle

\subsection*{Notation}
\begin{itemize}
    \item $A\times B$: The outer product of two multidimensional arrays $A,B$, defined as $(A\times B)_{a_0,\dots,a_{p-1},b_0,\dots,b_{q-1}}=A_{a_0,\dots,a_{p-1}}B_{b_0,\dots,b_{q-1}}$ for all indices $a_0,\dots,a_{p-1}$ and $b_0,\dots,b_{q-1}$;
    \item $E(n)_{i,j}$: The $n\times n$ matrix with a 1 at cell $(i,j)$ and 0s everywhere else; if it is clear from context what $n$ is, the abbreviation $E_{i,j}$ may be used;
    \item $\delta_{i,j}=1$ if $i=j$, 0 otherwise (the Kronecker delta);
    \item Commas between indices may be omitted for brevity;
    \item $[N]$ denotes the set $\{0,1,\dots,N-1\}$;
    \item $S_n$ denotes the set of all permutations over $n$ objects;
\end{itemize}

\section{Goal}

For a natural number $n$, the \textit{$<n,n,n>$ matrix multiplication tensor} is defined as
\[\mathcal{M}(n)=\sum_{0\le i,j,k<n} E_{ij}\times E_{jk}\times E_{ki};\]

Thus, $\mathcal{M}(n)_{abcdef}=\delta_{fa}\delta_{bc}\delta_{de}$.

We want to find the minimum $R$ and three lists of matrices $A,B,C$ such that
\[\mathcal{M}(n)=\sum_{r=0}^{R-1}A_r\times B_r\times C_r.\]

$R$ is known as the rank of $\mathcal{M}(n)$, and the list $[(A_r,B_r,C_r)]_{r=0\dots R-1}$ of triplets of matrices satisfying the equation is known as an \textit{$R$-rank decomposition} of $\mathcal{M}(n)$. Note that this equation is equivalent to

\[\forall abcdef,\ \mathcal{M}(n)_{abcdef}=\sum_{r=0}^{R-1}A_{rab}B_{rcd}C_{ref}.\]

The motivation for this problem is that an $R$-rank decomposition of $\mathcal{M}(n)$ can be directly translated into an algorithm for multiplying two $n\times n$ matrices while using only $R$ multiplications between elements of the matrices; we exclude multiplications of elements with fixed scalars. Such an algorithm can then be translated into a $O(N^{\log_n R})$ divide-and-conquer algorithm for multiplying two $N\times N$ matrices for arbitrarily large $N$, which for $R<n^3$ is asymptotically faster than naive matrix multiplication, which has many applications across numerous fields, such as graph algorithms and machine learning.

\section{Transformations of Decompositions}
Given an $R$-rank decomposition $[(A_r,B_r,C_r)]_{r=0\dots R-1}$ of $\mathcal{M}(n)$, we wish to analyze how we can transform it into other valid $R$-rank decompositions.

There are two trivial transformations we can do: one is to permute the triples of the decomposition: $[(A_{\sigma(r)},B_{\sigma(r)},C_{\sigma(r)})]_{r=0\dots R-1}$ for any permutation $\sigma$; the other is to scale elements of each triple with scalars whose product is 1: $[(a_rA_r,b_rB_r,c_rC_r)]_{r=0\dots R-1}$ for any $a_r,b_r,c_r$ such that $a_r b_r c_r=1$ for all $r$.

There is another transformation, called the \textit{trace transformation}, which relies on special properties of $\mathcal{M}(n)$: \[[(\alpha A_r\beta^{-1},\beta B_r\gamma^{-1},\gamma C_r\alpha^{-1})]_{r=0\dots R-1},\] for invertible $n\times n$ matrices $\alpha,\beta,\gamma$. The validity of this transformation can be checked with some algebra.

\subsection{Families of Decompositions}
To find nontrivial decomposition transformations, we can first consider the family of all transformations of the form $[(f_A(A_r),f_B(B_r),f_C(C_r))]_{r=0\dots R-1}$, where $f_A,f_B,f_C:\R^{n\times n}\rightarrow \R^{n\times n}$ are linear transformations over matrices, i.e. each element of the output matrix is a linear combination of all the elements of the input matrix. Thus, we can represent $f_A,f_B,f_C$ with tensors $\alpha,\beta,\gamma$ such that $f_A(A)=\sum_{uv}\alpha_{abuv}A_{uv}$ and likewise for $f_B,\beta$ and $f_C,\gamma$.

Plugging our transformed decomposition into the equation for decompositions of $\mathcal{M}(n)$ and applying some algebraic manipulation gives

\[\forall abcdef,\ \mathcal{M}(n)_{abcdef}=\sum_{r=0}^{R-1}(\sum_{uv}\alpha_{abuv}A_{uv})(\sum_{wx}\beta_{cdwx}B_{rwx})(\sum_{yz}\gamma_{efyz}C_{ryz})\]
\[=\sum_{r=0}^{R-1}\sum_{uvwxyz}\alpha_{abuv}A_{uv}\beta_{cdwx}B_{rwx}\gamma_{efyz}C_{ryz}\]
\[=\sum_{r=0}^{R-1}\sum_{uvwxyz}A_{uv}B_{rwx}C_{ryz}\alpha_{abuv}\beta_{cdwx}\gamma_{efyz}\]
\[=\sum_{uvwxyz}(\sum_{r=0}^{R-1}A_{uv}B_{rwx}C_{ryz})\alpha_{abuv}\beta_{cdwx}\gamma_{efyz}\]
\[=\sum_{uvwxyz}\mathcal{M}(n)_{uvwxyz}\alpha_{abuv}\beta_{cdwx}\gamma_{efyz}\]
\[=\sum_{ijk}\alpha_{abij}\beta_{cdjk}\gamma_{efki}.\]

Thus, any triplet of tensors $\alpha,\beta,\gamma$ satisfying $\mathcal{M}(n)_{abcdef}=\sum_{ijk}\alpha_{abij}\beta_{cdjk}\gamma_{efki}\ \forall abcdef$ is a valid transformation over $R$-rank decompositions.

\subsubsection{Rank 1 (incomplete)}
Suppose we restrict each matrix $\alpha_{ab},\beta_{cd},\gamma_{ef}$ to be rank-1 matrices, i.e. there are tensors $\alpha^0,\alpha^1,\beta^0,\beta^1,\gamma^0,\gamma^1$ such that $\alpha_{abij}=\alpha^0_{abi}\alpha^1_{abj},\beta_{cdjk}=\beta^0_{cdj}\beta^1_{cdk},\gamma_{efki}=\gamma^0_{efk}\gamma^1_{efi}$. Plugging this into the previous equation gives
\[\forall abcdef,\ \mathcal{M}(n)_{abcdef}=\sum_{ijk}\alpha^0_{abi}\alpha^1_{abj}\beta^0_{cdj}\beta^1_{cdk}\gamma^0_{efk}\gamma^1_{efi}\]
\[=\sum_{ijk}\gamma^1_{efi}\alpha^0_{abi}\alpha^1_{abj}\beta^0_{cdj}\beta^1_{cdk}\gamma^0_{efk}\]
\[=(\sum_i\gamma^1_{efi}\alpha^0_{abi})(\sum_j\alpha^1_{abj}\beta^0_{cdj})(\sum_k\beta^1_{cdk}\gamma^0_{efk})\]
\[=(\gamma^1_{ef}\cdot\alpha^0_{ab})(\alpha^1_{ab}\cdot\beta^0_{cd})(\beta^1_{cd}\cdot\gamma^0_{ef})\]
\[=\delta_{fa}\delta_{bc}\delta_{de}.\]

Since this equation must be true for all $abcdef$, we can immediately deduce that $\gamma^1_{ei}\cdot\alpha^0_{ib}\ne 0,\alpha^1_{aj}\cdot\beta^0_{jd}\ne 0$, and $\beta^1_{ck}\cdot\gamma^0_{kf}\ne 0$; otherwise if $\gamma^1_{ei}\cdot\alpha^0_{ib}=0$ for some $i$, then when setting $(abcdef)$ to some $(ijjkki)$ will lead to $(\gamma^1_{ef}\cdot\alpha^0_{ab})(\alpha^1_{ab}\cdot\beta^0_{cd})(\beta^1_{cd}\cdot\gamma^0_{ef})=0\ne \delta_{fa}\delta_{bc}\delta_{de}=1$, and a similar argument applies to the other two inequalities.

We can then further determine that $\gamma^1_{ef}\cdot\alpha^0_{ab}=0$ for all $a\ne b$, since $(\gamma^1_{ef}\cdot\alpha^0_{ab})(\alpha^1_{ab}\cdot\beta^0_{cd})(\beta^1_{cd}\cdot\gamma^0_{ef})=0$ when $(abcdef)$ is set to some $(ajjkkf)$, since then $\alpha^1_{ab}\cdot\beta^0_{cd}$ and $\beta^1_{cd}\cdot\gamma^0_{ef}$ would both be equal to 1. Using similar arguments, $\alpha^1_{ab}\cdot\beta^0_{cd}=0$ for all $c\ne d$ and $\beta^1_{cd}\cdot\gamma^0_{ef}=0$ for all $d\ne e$.

Thus, $[\gamma^1_{ef}\cdot\alpha^0_{ab}]_{f,a}$, $[\alpha^1_{ab}\cdot\beta^0_{cd}]_{b,c}$, and $[\beta^1_{cd}\cdot\gamma^0_{ef}]_{d,e}$ are all diagonal matrices.

\section{Searching for Symmetric Decompositions}
From this point onward we will focus on the case $n=3$. We will use two symmetries of $\mathcal{M}(n)$ and force all the decompositions we search for to have these same symmetries:

\[\mathcal{M}(n)_{a,b,c,d,e,f}=\mathcal{M}(n)_{c,d,e,f,a,b}=\mathcal{M}(n)_{e,f,a,b,c,d}\ \forall a,b,c,d,e,f;\]

\[\mathcal{M}(n)_{a,b,c,d,e,f}=\mathcal{M}(n)_{\sigma(a),\sigma(b),\sigma(c),\sigma(d),\sigma(e),\sigma(f)} \forall \sigma\in S_n.\]

We say two decompositions $[[A_{r,t}]_{t\in[3]}]_{r\in[R]}$ and $[[B_{r,t}]_{t\in[3]}]_{r\in[R]}$ are \textit{equal} if there exists a permutation $\sigma\in S_R$ s.t. $\forall r, [A_{\sigma(r),t}]_{t\in[3]}=[B_{r,t}]_{t\in[3]}$.

We want all decompositions we search for to be equal under all the symmetries described above. Formally, for functions
$h_{\sigma}(A):=[A_{\sigma(r),\sigma(c)}]_{r,c\in[n]}$ and $T_{s,\sigma}([A_t]_{t\in[3]}):=[h_{\sigma}(A_{(t+s)\mod 3})]_{t\in[3]}$, we want decomposition $[[A_{r,t}]_{t\in[3]}]_{r\in[R]}$ to be equal to decomposition $[T_{s,\sigma}([A_{r,t}]_{t\in[3]})]_{r\in[R]}$ for all $s,\sigma$.

\subsection{First attempt}
Consider decompositions of the form $\mathcal{D}=[[P_r]_{t\in[3]}]_{r\in[R_p]}\ ||\ [T_{s,\sigma}([A,B,C])]_{\forall s,\sigma}$ for $P,A,B,C\in\{-1,0,1\}^{3\times 3}$, $P$ satisfying $P=h_\sigma(P)\ \forall\sigma$, and $R_p\le 5$. Note that the second part of the decomposition consists of 18 matrix triples.

We will also define the \textit{error} $e(A,B)$ between two tensors $A,B$ of equal shape to be the number of coordinates $c$ s.t. $A_c\ne B_c$.

By brute force search, the minimum possible $e(\mathcal{M}(3),\mathcal{D})$ is 9. One possible solution is $R_p=0,
A=\begin{bmatrix}
-1 & 0 & 1\\
0 & 0 & 0\\
-1 & 0 & 0\\
\end{bmatrix},
B=\begin{bmatrix}
0 & 0 & 1\\
0 & 0 & 0\\
0 & 0 & 0\\
\end{bmatrix},
C=\begin{bmatrix}
0 & 0 & 0\\
0 & 0 & 0\\
-1 & 0 & 0\\
\end{bmatrix}$.

\section{Symmetry Reduction (old)}
We will use two symmetries of $\mathcal{M}(n)$ to force $A,B,C$ to have these same symmetries, in order to reduce the search space. The first is \textit{cyclic symmetry}, which can be easily verified:

\[\mathcal{M}(n)_{a,b,c,d,e,f}=\mathcal{M}(n)_{c,d,e,f,a,b}=\mathcal{M}(n)_{e,f,a,b,c,d}\ \forall a,b,c,d,e,f.\]

We can force our decomposition of $\mathcal{M}(n)$ to be cyclically symmetric by requiring that for all triplets $(A_r,B_r,C_r)$ in our decomposition, the triplets $(B_r,C_r,A_r)$ and $(C_r,A_r,B_r)$ also appear in the decomposition.

A special case occurs if $(A_r,B_r,C_r)=(\alpha W,\beta W,\gamma W)$ for some matrix $W$ and scalars $\alpha,\beta,\gamma$; in this case $(B_r,C_r,A_r)$ and $(C_r,A_r,B_r)$ do not need to appear as they can be merged with $(A_r,B_r,C_r)$ to yield a single triplet $(3\alpha\beta\gamma W,W,W)$ (note that $(\alpha W,\beta W,\gamma W)$ is equivalent to $(\alpha\beta\gamma W,W,W)$ since the outer product of the matrices in each of the triplets yields the same tensor). We can then adjust the triplet $(3\alpha\beta\gamma W,W,W)$ to $(W',W',W')$ where
$W'=(3\alpha\beta\gamma)^{\frac{1}{3}}W$, so that the new triplet's matrices are all the same. Thus, every cyclically symmetric decomposition can be written as the form
\[A=W||X||Y||Z,B=W||Y||Z||X,C=W||Z||X||Y\]
where $W,X,Y,Z$ are lists of matrices and $||$ is list concatenation.

Note that Strassen's 7-rank decomposition of $\mathcal{M}(2)$ is cyclically symmetric. Also, Grey Ballard found a cyclically symmetric 23-rank decomposition of $\mathcal{M}(3)$ (\url{http://perso.ens-lyon.fr/bora.ucar/tensors-cse17/ballard-talk.pdf}). Although the rank is the same as the current record, none of the previously discovered 23-rank decompositions were cyclically symmetric.

The second symmetry we will use is \textit{reversal symmetry}:
\[\mathcal{M}(n)_{a,b,c,d,e,f}=\mathcal{M}(n)_{n-1-a,n-1-b,n-1-c,n-1-d,n-1-e,n-1-f}\ \forall a,b,c,d,e,f.\]

We will in fact use an extra-restrictive decomposition of $\mathcal{M}(n)$ when combining both symmetries, namely we will only consider decompositions of the form
\[A=W||X||Y||Z||\overline{X}||\overline{Y}||\overline{Z},B=W||Y||Z||X||\overline{Y}||\overline{Z}||\overline{X},C=W||Z||X||Y||\overline{Z}||\overline{X}||\overline{Y},\]

where $\overline{X}$ denotes the list of matrices $X$ but with each matrix $A$ altered to the matrix $A'=[A_{n-1-i,n-1-j}]_{i,j}$ (i.e. $A$ flipped both horizontally and vertically), and where $W$ satisfies the property $W=\overline{W}$. Strassen's decomposition also satisfies this restrictive form.

\subsection{Progress}
Let $R=|W|$ and $S=|X|=|Y|=|Z|$ represent the length of $W$ and the length of each of $X,Y,Z$ respectively. Our tensor that we want to decompose is $\mathcal{M}(3)$, and our decomposition of interest is
\[A=W||X||Y||Z||\overline{X}||\overline{Y}||\overline{Z},B=W||Y||Z||X||\overline{Y}||\overline{Z}||\overline{X},C=W||Z||X||Y||\overline{Z}||\overline{X}||\overline{Y}.\]
The rank of this decomposition is $R+3S$.

Below is our search progress:
\begin{itemize}
    \item $R\le 5,S\le 2$, each matrix in $A,B,C$ has at most 2 nonzero elements: no decompositions found
\end{itemize}

\end{document}


\subsection{Differentiable functions}

\subsubsection{Introduction}

A differentiable function is one where the differential is defined at all points on the real line.

All differentiable functions are continuous. Not all continuous functions are differentiable.

\subsubsection{Differentiability class}

We can describe a function with its differentiability class. If a function can be differentiated \(n\) times and these differentials are all continous, then the function is class \(C^n\).

\subsubsection{Smooth functions}

If a function can be differentiated infinitely many times to produce continous functions, it is \(C^{\infty }\), or smooth.


\chapter{Vector Space}

\section{Matrix}
\begin{remind}
A \textbf{field} is a \textit{good} algebraic structure, which has the addition and the multiplication. Formally, a field $(F,~+,~\cdot)$, or simply $F$, is a pair of a set and two operations which is from $F\times F$ to $F$ satisfying the following:
\begin{itemize}
\item $(F,~ +)$ is an \textbf{abelian group}, that is, $+$ is commutative, associative, and there is an additional identity 0 and the inverse element $-a$ of $a$ for all $a \in F.$
\item $(F^{\times },~\cdot)$ is also an abelian group, that is, $\cdot$ is commutative, associative, and there is an multiplicational identity 1 and the inverse element $a^{-1}$ of $a$ for all $a \in F^{\times},$ where $F^{\times} = F - \{0\}.$
\item + and $\cdot$ are \textit{compatible}, which means $\cdot$ is distributing over $+.$
\end{itemize}
We simply write $a-b := a+(-b)$ and $a/b=ab^{-1}.$

A \textbf{matrix} over a field $F$ is a rectangular arrangement of \textit{scalars}, elements of the field $F$. The space of $m$ by $n$ matrices is denoted as $\mathfrak M_{m,n}(F).$
\end{remind}
\begin{ex}\leavevmode
\begin{itemize}
\item $\mathbb Q$, $\mathbb R$, $\mathbb C$ are well-known(?) fields.
\item $\mathfrak M_{m,n}(F) \approx F^{mn},$ without the product.
\item Matrices do not form a field.
\end{itemize}
\end{ex}

\subsection{Transpose and Trace}
\begin{defn}
For every $m$ by $n$ matrix $A \in \mathfrak M _{m,n}(F),$ the \textbf{transpose} of $A$ is defined as follows: $$ A^{\mathsf T} = (a_{ji})_{n,m}.$$ For every $n$ by $n$ \textit{square} matrix $A \in \mathfrak M_{n,n}(F),$ the \textbf{trace} of $A$ is defined as follows: $$\operatorname{tr}A = \sum_{i=1}^n a_{ii}.$$
\end{defn}

\begin{prop}[Linearity of transpose and trace]
For every pair of $m$ by $n$ matrices $A$ and $B \in \mathfrak M_{m,n}(F)$ and every pair of scalars $a,~b \in F$, $$(aA + bB)^{\mathsf T} = a A^{\mathsf T} + bB^{\mathsf T}.$$ For every pair of $n$ by $n$ square matrices $A$ and $B \in \mathfrak M_{n,n}(F)$ and every pair of scalars $a,~b \in F$, $$\operatorname{tr}(aA + bB) = a \operatorname{tr}A + b \operatorname{tr}B.$$
\end{prop}
\begin{proof}
ㅎㅎ.
\end{proof}

\begin{prop}[Behaviour of transpose and trace]
$$(AB)^\mathsf T = B^\mathsf T A^\mathsf T, \qquad \operatorname{tr}A = \operatorname{tr} A^\mathsf T, \qquad \operatorname{tr}(AB) = \operatorname{tr}(BA).$$
\end{prop}
\begin{proof}
Try it!
\end{proof}

\subsection{Inverse Matrix}
\begin{defn}
For a \textit{square} matrix $A\in\mathfrak M_{n,n}(F),$ if there is another square matrix $B\in\mathfrak M_{n,n}(F)$ such that $$AB = I = BA,$$ then we call $B=A^{-1}$ the(?) \textbf{inverse matrix} of $A$.
\end{defn}

\begin{prop}[Uniqueness of inverse matrix] The inverse matrix of a matrix $A$ is unique (if exists). This justifies the occurrence of \textbf{`the'} above.
\end{prop}
\begin{proof}
Let those be $A^{-1}$ and ${\tilde A }^{-1},$ then $${\tilde A }^{-1} = (A^{-1}A) {\tilde A }^{-1} = A^{-1}(A {\tilde A }^{-1}) = A^{-1}.$$
\end{proof}

\begin{quest} If $AB = I$ for two square matrices $A$ and $B$, what can we say about the invertibility of them? We will solve this problem using a `function,' which is from and to some vector spaces, defined below.
\end{quest}

\newpage

\section{Vector Space}
\begin{defn}
A \textbf{vector space} $V$ over $F$, or simply an \textbf{\textit{F}-vector space} $V$, is a \textit{good} algebraic structure, which has the addition $+:~V\times V \to V$ and the $F$-scalar multiplication $\mathrm{SM}_F:~F\times V \to V$. Formally, a field $(V,~F,~+,~\mathrm{SM}_F)$, or simply $V$, is a pair of a set, a field and two operations satisfying the following:
\begin{itemize}
\item $(V,~ +)$ is an abelian group.
\item $\mathrm{SM}_F$ and $\cdot_F$ are compatible: $(ab)v = a(bv)$ for every $a, ~b \in F$ and $v\in V,$ and $1v = v$ for all $v\in V.$
\item $+$s and $\mathrm{SM}_F$ are compatible: $(a+b)v = av + bv,$ $a(v+w) = av + aw$ for every $a,~b\in F$ and $v,~w \in V.$
\end{itemize}
We simply write $v-w := v+(-w).$
\end{defn}
\begin{ex} The following structures are examples of vector space.
\begin{itemize}
\item \textbf{\{0\}} is a vector space over \textit{arbitrary field}, and is called the \textbf{trivial space.}
\item $\mathbb R^n$ and $\mathbb C^n$ are vector spaces. In fact, for any field $F$, $F^n$ is an $F$-vector space, trivially.
\item Hence, a matrix space $\mathfrak M_{m,n}$ of $m$ by $n$ matrices over $F$ is a vector space since is \textit{the same with} $F^{mn}$, and the polynomial space of $n$-th degree $\mathbf{P}_n [t] = \{\sum_{i=0}^n a_i t^i: ~ a_i \in F\}$ is also a vector space, \textit{the same with} $F^{n+1}.$
\item (Field extension) If there are two fields which one is a subfield of another, namely $E \ge F$, then $E$ is a $F$-vector space, with its addition and multiplication (as a scalar multiplication.)
\item (\textbf{dual space!}) A \textbf{linear functional} $f$ on $V$ is a \textit{linear} map from $V$ to $F$, that is, $$f(av+b) = af(v)+b$$ for every $a,~b\in F$ and $v\in V.$ The \textbf{dual space} of $V$ is the space of linear functionals on $V$, and it forms a $F$-vector space, with the following operations: $$(f+g)(v) = f(v) + g(v),\qquad (af)(v) = af(v).$$ The dual space is one of the most interesting things not only in linear algebra, but in abstract algebra, or even in \textit{any} branches which use the term \textit{dual} (e.g., in projective geometry).
\end{itemize}
\end{ex}

\begin{prop}
\leavevmode

\begin{itemize}

\item $0v = 0$ and $(-1)v = -v$.
\begin{proof} $0v=(0+0)v=0v+0v$ implies $0v=0$ and $v+(-v)=0=0v=(1+(-1))v=1v+(-1)v=v+(-1)v$ implies $-v=(-1)v.$
\end{proof}
\item Every \textbf{linear combination} $\sum_i a_i v_i$ of vectors $v_i \in V$ is in $V.$
\begin{proof} Easy induction on the number of summands(terms).
\end{proof}
\end{itemize}
\end{prop}

\begin{defn}
A subset $W\subseteq V$ is called a \textbf{subspace} of $V$ if it forms a vector space itself with the \textit{inherited} operations from $V$. We denote it $W\le V.$
\end{defn}
\begin{prop}\label[prop]{p:subspace}
$$W\le V~~~\Longleftrightarrow~~~ \forall c\in F,~ \forall v,~w\in W,~~~ cv+w \in W.$$
\end{prop}
\begin{proof}
($\Leftarrow$) The other axioms of vector space is satisfied by the fact that the operations are inherited by $V$, and hence it suffices to show that the operations are closed. First, let $c=-1$ and $v=w.$ Then we have $(-1)w + w = 0 \in W$ by the proposition above. Then, the addition is closed if we let $c=1$; and so is the scalar multiplication if we let $w=0$, which is in $W$, as we proved. Hence $W$ forms a vector space.

($\Rightarrow$) $cv+w$ is a linear combination. ㅋㅋ.
\end{proof}
\begin{ex}
\leavevmode
\begin{itemize}
\item Removing a coordinate(\textbf{projection}): $$F^2 = \left\{(a,b):~a,b\in F\right\} \le \left\{(a,b,c):~a,b,c\in F\right\} = F^3.$$
\item \textbf{Symmetric}, \textbf{alternating} matrices over $F$: $$\mathrm{Sym}_n(F) = \{A \in \mathfrak M_{n,n}(F):~A=A^{\mathsf T}\} \le M_{n,n}(F),$$
$$\mathrm{Alt}_n(F) = \{A \in \mathfrak M_{n,n}(F):~A=-A^{\mathsf T}\} \le M_{n,n}(F).$$
\item \textbf{Hermitian} matrices: $$\mathrm{Her}_n = \{A \in \mathfrak M_{n,n}(\mathbb C):~A=\overline{A^{\mathsf T}}\} \le M_{n,n}(\mathbb C).$$
\item The solution space of a system of homogeneous linear (differential) equations.
\end{itemize}
\end{ex}

\subsection{Spanned subspace}

\begin{defn} For a subset $S$ of a vector space $V$, the \textbf{subspace spanned by \textit{S}} is the following set: $$\left< S \right> = \left\{ \sum_{\textrm{finite}} a_i v_i: ~~ a_i\in F,~v_i \in S \right\},$$ where $\sum_{\textrm{finite}} a_i v_i$ means $a_i \ne 0$ for \textit{only} some finitely many indices $i$, and $a_i = 0$ for others; i.e., we only add finitely many vectors.\footnote[2]{We take finitely many vectors since it is not sufficient to define `convergence' of an infinite series with just axioms of vector space.}

And we call a element of $\left< S \right>$ a \textbf{linear combination} of $S$.
\end{defn}

\begin{ex}
Let $\mathbb R^{\infty}$ be the space of all sequences which are \textbf{eventually zero}, that is, there are only \textit{finitely many} nonzero terms. Then,$$ \mathcal E^{\infty} = \left\{\mathbf{e}_i =\left(0,~\cdots,~0,~\underset {i-\textrm{th}} 1,~0,~\cdots\right): ~~~ i\in \mathbb {N} \right\}$$ spans $\mathbb R^\infty.$ Hence, for example, a sequence $$\left(1,~1,~1,~\cdots\right)$$is not a linear combination of $\mathcal E^{\infty}.$
\end{ex}
\begin{prop}
For every subset $S \in V,$ $\left <S\right>$ is a subspace of $V$.
\end{prop}
\begin{proof}
Use \cref{p:subspace}.
\end{proof}

\begin{prop}
For every subset $S \in V,$ $\left <S\right>$ is \textit{the smallest} subspace of $V$ which contains $S$, namely, $$\left<S\right> = \bigcap_{\substack{S \subseteq W \\ W:~\text{vector space}}} W.$$
\end{prop}
\begin{proof}
($\subseteq$) Since $W$ is a vector space containing $S$, it must contain other linear combinations of $S$ also. Therefore $\left< S \right> \subseteq W$ for every $W$ satisfying the condition whence$$\left<S\right> \subseteq \bigcap_{\substack{S \subseteq W \\ W:~\text{vector space}}} W.$$

($\supseteq$) $\left<S\right>$ is a vector space containing $S$.
\end{proof}

\subsection{Basis}

\begin{defn} A subset $S = \{v_i:~i\in I\}$ is \textbf{linearly independent} if every vector of $S$ cannot be represented by a linear combination other vectors; i.e., $$\forall i\in I,~\forall a_j\in F,\quad v_i \ne \sum_{\substack{j\ne i \\ \textrm{finite}}} a_j v_j.$$ Equivalently, if every linear combination whose coefficients are not all zero is non-zero, the subset is linearly independent. If not, $S$ is \textbf{linearly dependent}.
\end{defn}

\begin{defn} A \textbf{(Hamel) basis} $\mathfrak B$ of $V$ is a subset of $V$ which satisfies the followings: $$\left<\mathfrak B \right>=V$$ and $$\mathfrak B\textrm{ is linearly independent.}$$
We usually fix the order of elements of $\mathfrak B$, which is called an \textbf{ordered basis}. Hereafter, \textit{every basis is an ordered basis.} For example, a basis $\{(1,0),~(0,1)\}$ and $\{(0,1),~(1,0)\}$ are different bases.
\end{defn}

\begin{ex}\leavevmode
\begin{itemize}
\item (\textbf{standard basis}) $$\mathcal E = \left\{ \mathbf e_i = \left(0,~\cdots,~0,~\underset {i-\textrm{th}} 1,~0,~\cdots,~0\right): ~~~ 1\le i \le n \right\}$$ is a basis for $F^n$, for arbitrary field $F$.
\item $\{ (1,0), (1,1)\}$ is a basis for $\mathbb R^2$ (over the field $\mathbb R$).
\end{itemize}
\end{ex}

\subsection{Dimension}
\begin{defn} `The' \textbf{dimension} $\operatorname{dim} V$ of given vector space $V$ is the \textbf{cardinality}(the number of elements of given set for finite set) of a basis.
\end{defn}

\begin{defn} A \textbf{finite dimensional vector space} is a vector space whose bases are all finite.\footnote[2]{It is the best way for defining finite dimensional vector space since the dimension is \textit{not} well-defined yet.}
\end{defn}

We consider \textit{finite dimensional vector spaces only} unless there is an additory description.

\begin{lemma}\label[lemma]{l:span} Let $\mathfrak B = \{v_i:~ 1\le i \le n\}$ be a basis, and $\mathfrak C = \{w_j:~ 1\le j \le m\}$ span $V$. Then $m\ge n$.
\end{lemma}
\begin{proof}
If there is a vector $v$ of $\mathfrak B$ which is not in $\mathfrak C$, without loss of generality, rename it $v_1 \ne w_i$. Then $\mathfrak C \cup \{v_1\}$ is linearly dependent: since $\mathfrak C$ spans $V$, there is a linear combination of $\mathfrak B$ represents $-v_1$ whence $\mathfrak C \cup \{v_1\}$ is linearly dependent. Hence there is at least one vector $w_t$ which is represented by a linear combination of others. Then let $\mathfrak C_1 = \mathfrak C \cup \{v_1\} - \{w_t\}.$

Repeat this process. It is possible up to $n$-th stage since $\mathfrak B$ is linearly independent: if there is a linear dependence, it must contain a $w$-vector. Hence we obtain $$\mathfrak C_n = \{v_1,~\cdots,~v_n,~w_{r_1},~\cdots,~w_{r_{m-n}} \}$$ and it is linearly dependent.
\end{proof}

\begin{theorem}[uniqueness of dimension] Let $\mathfrak B$ and $\mathfrak C$ be two bases of finite dimensional vector space $V$. Then $|\mathfrak B| = |\mathfrak C|.$
\end{theorem}
\begin{proof}
$|\mathfrak B| \ge  |\mathfrak C|$ and
$|\mathfrak C| \ge  |\mathfrak B|$ by \cref{l:span}.
\end{proof}

Well... How about the existence? The existence of the dimension needs the existence of the basis of $V$.

\begin{theorem}[existence of basis] Every vector space has a basis, if \textbf{AC}(Axiom of Choice) assumed. In addition, it is equivalent to \textbf{AC}.
\end{theorem}
\begin{proof}
$\exists \mathfrak B \Longleftrightarrow \mathbf{ZL} \Longleftrightarrow \mathbf{AC}.$ See a set theory textbook.
\end{proof}

\subsection{Basis extension}
We can \textit{extend} a basis of smaller space to a larger space.

\begin{theorem}[basis extension] Let $W\le V$ be two vector spaces and $\mathfrak C$ be a basis for $W$. Then there is a basis $\mathfrak B$ of $V$ which contains $\mathfrak C.$
\end{theorem}
\begin{proof}
Induction on \textbf{\textit{n$-$m}}, where $n=\operatorname{dim}V$ and $m=\operatorname{dim}W$. If $n-m=0$, just let $\mathfrak B = \mathfrak C.$ (Why?) Now, assuming there is a vector in $V - W$, take a vector $v$ in $V-W$. Then $v$ is linearly independent with $\mathfrak B$ (that is, $\tilde {\mathfrak B} = \mathfrak B \cup \{v\}$ is linearly independent) and hence $\tilde W = \left<\tilde {\mathfrak B }\right>$ is a vector space which $\tilde W \le V.$ Since $n-m$ decreases, the induction proceeds.
\end{proof}

\subsection{Sum and direct sum}
\begin{defn}
For a set $\{S_i\}_{i\in I}$ of sets with a common addition, we define the \textbf{sum} of $\{S_i\}$ as follows: \begin{align*}\sum_{i \in I}S_i &= \left\{ \sum_{\textrm{finite}} s_i:~~~ s_i \in S_i\right\} \\&= \left\{ \sum_{i \in I} s_i:~~~ s_i \in S_i\textrm{ and all }s_i = 0\textrm{ but for finitely many }i\right\}.\end{align*}
\end{defn}
\begin{defn}
For a set $\{W_i\}_{i\in I}$ of \textit{subspaces} of $V$ which is mutually disjoint: $$W_i \cap W_j = \{ 0 \},\qquad\textrm{for }i\ne j,$$ we define the \textbf{direct sum} of $\{W_i\}$ just the sum of them: $$\bigoplus_{i\in I}  W_i = \sum_{i \in I} W_i.$$

If the set is not mutually disjoint, even if it is mutually disjoint itself, we \textit{make it be} mutually disjoint: isolate the vectors with giving different coordinates for each vector space. For example, if $V\oplus W \ne \{0\}$, $$V\oplus W :\approx \{(v,~w):~~~v\in V,~ w\in W\}.$$
Trivially the direct sum of some vector spaces is a vector space.
\end{defn}

\begin{ex}
\leavevmode
\begin{itemize}
\item $\mathbb R \oplus \mathbb R \approx \mathbb R^2,$
\item $\mathbb R \oplus \mathbb R \oplus \cdots \approx \mathbb R^\infty.$
\end{itemize}
\end{ex}

\begin{add}
\leavevmode
\begin{itemize}
\item For finite spaces, the direct sum of them is \textit{the same}(isomorphic) with the cartesian product.
\item The direct sum can be represented by a commutative diagram: \begin{center}
\leavevmode
\xy
\xymatrix {
X \\
& V\oplus W \ar@{.>}[ul]^{f} & V
\ar[l]^{i_1}
\ar@/_/[ull]_{f_1} \\
& W
\ar[u]_{i_2}
\ar@/^/[uul]^{f_2} &
}
\endxy
\end{center} where the cartesian product is represented by:
\begin{center}
\leavevmode
\xy
\xymatrix{
X \ar@/_/[ddr]_{f_2} \ar@{.>}[dr]_{f} \ar@/^/[drr]^{f_1} \\
& X \times Y \ar[d]^{p_2} \ar[r]_{p_1}
& X \\
& Y  & }
\endxy
\end{center}

\end{itemize}
\end{add}

\newpage

\section{Linear Transformation}
\begin{defn}
Let $V$ and $W$ be two $F$-vector spaces, then a map $f:~V\to W$ is \textbf{linear} if $$f(cv+dw) = cf(v)+df(w)$$ for every $c,~d \in F$ and $v,~w\in V.$

In another viewpoint, a linear map is a \textbf{vector space homomorphism} since it \textit{preserves} the operations of vector spaces.
\end{defn}
\begin{ex}
\leavevmode
\begin{itemize}
\item A map $f:~F\to F, ~~~ x \mapsto ax$ is a linear map from and to $F$. It is why maps of this kind are called \textit{linear}.
\item Producting a matrix is a linear map. For a matrix $A\in\mathfrak M_{m,n}(F),$ $$L_A:~F^m \to F^n,~~~ X\mapsto AX$$ is a linear map from $F^m$ to $F^n.$ We will show that every linear map (from and to finite dimensional vector spaces) can be represented in this way, i.e., \textit{matrices and linear transformations are the same things.}
\item Another familiar linear maps are differentiation and integration. Let $\mathcal C^{n}$ be the space of function from and to $\mathbb R$ which is $n$-th differentiable and has continuous $n$-th derivative. Then for $n\in\mathbb N$, the \textbf{differentiation operator} $$D:~\mathcal C^{n}\to \mathcal C^{n-1}, \qquad f \mapsto f'$$ is a linear transformation since $(cf+dg)'=cf'+dg'.$ Similarly, the \textbf{integration operator} $$J:~\mathcal C^{n\ge 0} \to \mathcal C^{n+1},\qquad f \mapsto \int_0^x f ~\mathrm dx$$ is linear. (Here, $F$ is not a field but the antiderivative of $f.$) Similarly, partial differentiation operators are linear.
\item Transpose and trace are linear.
\end{itemize}
\end{ex}

\begin{theorem}
If $f$ is linear, values of $f$ at the basis elements determine $f$. Equivalently, there is an one-to-one correspondence between $f$ and $f(\mathbf e_i)$'s.
\end{theorem}
\begin{proof}
\leavevmode
\begin{center}
\leavevmode
\xy
\xymatrix @C=4pc {
  f(v) \ar@/^/^{v=\mathbf e_i} [r]
& f(\mathbf e_i) \ar@/^/^{v=\sum_i a_i \mathbf e_i} [l]
}
\endxy
\end{center}
\end{proof}

\begin{prop}
Let denote the space of all linear transformations from $V$ to $W$ as $\mathfrak L(V,W)$. Then $\mathfrak L(V,W)$ is a vector space dimension of $mn$.
\end{prop}

\begin{defn}
For an $F$-vector space $V$, a linear map $f:~V\to F$ is called a \textbf{linear functional}. And it forms a vector space, which is called the \textbf{dual space} of $V$. $$V^{*} = \left\{ f:~V\to F~|~ f~\textrm{linear}\right\}.$$ Since $V^*$ is an $F$-vector space, we can define the \textbf{double dual} $V^{**}$ of $V$ as follows: $$V^{**} = (V^*)^* = \left\{ \alpha:~V^*\to F~|~ \alpha~\textrm{linear}\right\}.$$
An example of elements of $V^{**}$ is \textbf{evaluation}: $$\alpha_a(f) = f(a),\qquad f\in V^*.$$
\end{defn}

\begin{defn}
For a linear map $f:~V\to W$, the \textbf{kernel} or the \textbf{null space} of $f$ is $$\ker  f = f^{-1}(0) = \left\{v\in V: ~ f(v) = 0 \right\},$$ where $0$ is the zero vector of $W.$ The \textbf{image} of $f$ is just $\operatorname{im}f = f(V).$
\end{defn}
\begin{prop}
The kernel and the image of a linear map form subspaces of $V$, respectively.
\end{prop}
\begin{theorem}[dimension theorem] \label[thm]{t:dimthm} For a linear map $f:~V \to W,$ $$\operatorname{dim} \ker  f + \operatorname{dim} \operatorname{im} f = \operatorname{dim}V.$$
\end{theorem}
\begin{proof}
Proof by basis extension. Let $\mathfrak B=\{v_i:~1\le i \le n\}$ be a basis of $\ker f.$ Then there is a basis $\mathfrak C = \mathfrak B \cup \{w_j:~1\le j \le m\}$ of $V$ which contains $\mathfrak B$, and we will show that $f(\mathfrak C - \mathfrak B)$ is a basis of $\operatorname{im}f$.

For an arbitrary vector $v = \sum_{i} a_i v_i + \sum_j b_j w_j$ of $V$, $$f(v) = f\left(\sum_i a_i v_i + \sum_j b_j w_j \right) = \sum_j b_j f(w_j)$$ since $v$'s are in the kernel. Since $v$'s and $w$'s are linearly independent, so are $f(w)$'s. Hence $f(w)$'s form a basis of $\operatorname{im}f$.
\end{proof}
We call $\operatorname{dim}\ker f$ the \textbf{nullity} of $f$ and denote it as $\operatorname{null}f$.

\begin{defn}
\leavevmode
\begin{itemize}
\item A \textbf{monomorphism} is an injective homomorphism.
\item An \textbf{epimorphism} is an surjective homomorphism.
\item An \textbf{isomorphism} is an bijective homomorphism.
\item An \textbf{automorphism} is an bijective homomorphism from and to itself.
\end{itemize}
\end{defn}

\begin{theorem}[vector space version of pigeonhole principle] Let $f:~V \to W$ is linear, and suppose $\operatorname{dim}V = \operatorname{dim}W = n < \infty.$ Then the followings hold:
\begin{itemize}
\item if $f$ is a monomorphism, then it is an isomorphism;
\item if $f$ is an epimorphism, then it is an isomorphism.
\end{itemize}
\end{theorem}

\begin{figure}
\centering
\includegraphics[width=3cm]{dove}
\caption{A pigeon.}
\end{figure}

\begin{proof}
Let $f$ be a monomorphism; suppose that $f$ is not surjective. Then there is a vector $w \in W$ such that $\forall v\in V,~w\ne f(v).$ Since $f(0)=0$, other vectors in $V$ are not mapped to 0 and hence $\ker f = \{0\}.$ And we get $\operatorname{dim}\operatorname{im}f = n$ from $\operatorname{dim}\ker  f = 0$. Hence $\operatorname{im}f = W.$

Changing im and ker proves the rest part of the theorem.
\end{proof}

Now we can prove the question in \textbf{Section 1.2}.

\begin{theorem}\label[thm]{t:matinv}
For two square matrices $A$ and $B\in\mathfrak M_{n,n}(F)$, if $AB=I$, then $A=B^{-1}.$
\end{theorem}
\begin{proof}
$AB=I$ implies that $B$ is left-invertible, which is equivalent to that $L_B$ is a monomorphism. Since $L_B:~F^n\to F^n$, $L_B$ is an isomorphism whence $B$ is invertible: $$B^{-1} = \begin{pmatrix} | & & | \\ L_B^{-1}\mathbf e_1 &\cdots & L_B^{-1}\mathbf e_n \\ | & & | \end{pmatrix}. $$ Multiplying $B^{-1}$ right in the both sides of $AB=I$, we obtain $A=B^{-1}.$ Similarly, $A$ is invertible and $A^{-1} = B.$
\end{proof}

\subsection{Rank}
\begin{defn}
For a matrix $A\in\mathfrak M_{m,n}(F),$ the \textbf{row space} is a space which is generated by the row vectors of $A.$ Similarly, the \textbf{column space} is a space which is generated by the column vectors of $A.$ Then the \textbf{row}(\textit{column}) \textbf{rank} is the dimension of the row(\textit{column}) space.
\end{defn}
\begin{ex}
We can know the row rank and the column rank in the (R, C)-REF of the matrix; one can do elementary row operations: \begin{align*}
A &= \begin{pmatrix}1&0&1\\5&6&7\\0&3&1\end{pmatrix}\\
&\sim_r \begin{pmatrix}1&0&1\\0&6&2\\0&3&1\end{pmatrix}\\
&\sim_r \begin{pmatrix}1&0&1\\0&6&2\\0&0&0\end{pmatrix}\\
&\sim_r \begin{pmatrix}1&0&1\\0&1&\frac 1 3 \\0&0&0\end{pmatrix};
\end{align*}
hence the row space is $\{(a,b,a+b/3)^\mathsf T:~a,b\in R\}$ whence the row rank is 2; while the column space is $\{(a+c,b+c/3,0)^\mathsf T = (\tilde a,\tilde b,0)^\mathsf T:~\tilde a,\tilde b\in R\}$ whence the column rank is also 2. Otherwise one can do elementary column operations: \begin{align*}
A &= \begin{pmatrix}1&0&1\\5&6&7\\0&3&1\end{pmatrix}\\
&\sim_c \begin{pmatrix}1&0&0\\5&6&2\\0&3&1\end{pmatrix}\\
&\sim_c \begin{pmatrix}1&0&0\\5&6&0\\0&3&0\end{pmatrix}\\
&\sim_c \begin{pmatrix}1&0&0\\0&1&0\\-\frac 5 2&\frac1 2&0\end{pmatrix};
\end{align*}
which makes the same result.
\end{ex}

Are the row rank and the column rank the same? The answer is...

\begin{lemma}For a matrix $A\in\mathfrak M_{m,n}(F)$ over an \emph{ordered field} $F$, $$\operatorname{col~rk}A = \operatorname{col~rk}A^\mathsf T.$$
\end{lemma}
\begin{proof}
It is suffices to show that $\operatorname{col~rk}A \le \operatorname{col~rk}A^\mathsf T,$ since it implies $$\operatorname{col~rk}A^\mathsf T \le \operatorname{col~rk}(A^\mathsf T )^\mathsf T  = \operatorname{col~rk}A$$which completes the proof of lemma.

We will show that $Av = 0$ if and only if $(A^\mathsf T A)v = 0$ whence $$\operatorname{col~rk}A = \operatorname{col~rk}(A^\mathsf T A) \le \operatorname{col~rk}A^\mathsf T$$; the last inequality follows because each column of $A^\mathsf T A$ is a linear combination of the columns of $A^\mathsf T$. First, $Av=0 \implies A^\mathsf T Av = 0$ trivially. Conversely, $$A^\mathsf T Av = 0 \implies v^\mathsf T A^\mathsf T A v = 0 \implies (Av)^\mathsf T Av = 0 \implies Av = 0,$$ by positive-definiteness of the dot product. (More generalized version of proof uses the orthogonal complement, or even \textbf{Erdős-Kaplansky Theorem}(?). See http://math.stackexchange.com/questions/2315/is-the-rank-of-a-matrix-the-same-of-its-transpose-if-yes-how-can-i-prove-it)\end{proof}
\begin{theorem}[rank theorem] The row rank and the column rank are the same, and we call it the \textbf{rank} of the given matrix.
\end{theorem}
\begin{proof}[Proof 1]
Count the number of \textit{leading 1} in RREF.
\end{proof}
\begin{proof}[Proof 2 assuming that $F$ is an ordered field]
It is trivial that the row rank of $A$ equals the column rank of $A^\mathsf T.$ Since the column rank of $A^\mathsf T$ is the same with of $A$, the proof completed.
\end{proof}
\begin{theorem}[rank-nullity theorem]
Let $A\in\mathfrak M_{m,n}(F)$ be a matrix, then $$\operatorname{rank}A + \operatorname{dim}\ker L_A = m.$$ We call $\operatorname{dim}\ker L_A$ the \textbf{nullity} of $A$, and denote $\operatorname{null}A$. Hence $$\operatorname{rank}A + \operatorname{null}A = m = \operatorname{dim}\operatorname{dom}L_A.$$
\end{theorem}
\begin{proof}
We know that $A\mathbf e_i$ is $i$-th column of $A$. Hence the column space of $A$ is just the image of $L_A:~F^m \to F^n$, hence $\operatorname{col~rk} A = \operatorname{dim}\operatorname{im}L_A.$ By the dimension theorem(\cref{t:dimthm}), $$\operatorname{dim}\ker L_A  + \operatorname{col~rk} A = \operatorname{dim} F^m = m.$$\end{proof}

\begin{defn}
Given an $m$ by $n$ matrix $A$ of rank $r$, a \textbf{rank decomposition} of $A$ is a representation by a product $A=PQ$ of two matrices $P\in\mathfrak M_{m,r}(F)$ and $Q\in\mathfrak M_{r,n}(F).$
\end{defn}
\begin{theorem}
A rank decomposition of a matrix exists, but not uniquely.
\end{theorem}
\begin{proof}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%5
\end{proof}

\subsection{Matrix representation and similarity}
\begin{defn}
For a finite dimensional vector space $V$ and a basis $\mathfrak B = \{v_i\}_{i\in I}$ of $V$, every vector $v$ in V can be represented as a linear combination of $\mathfrak B$ \textit{uniquely}, namely $$v = \sum a_i v_i;$$ and we call the row vector $$[v]_{\mathfrak B} = \begin{pmatrix}a_1 \\ \vdots \\ a_n\end{pmatrix}$$ the \textbf{coordinate} vector.
\end{defn}

\begin{theorem}
For a $F$-vector space homomorphism $f:~V\to W$ and the bases $\mathfrak B$ and $\mathfrak C$ of V and W, respectively, there is a unique matrix $[f]^{\mathfrak B}_{\mathfrak C} \in \mathfrak M_{\operatorname{dim}V,~\operatorname{dim}W}(F)$ such that $$[f]^{\mathfrak B}_{\mathfrak C} [v]_{\mathfrak B} = [fv]_{\mathfrak C}.$$
\end{theorem}
\begin{proof}
Let $n=\operatorname{dim}V$, $m=\operatorname{dim}W$, $\mathfrak B = \{ v_i\}_{i=1}^{n}$ and $\mathfrak C = \{ w_j\}_{j=1}^{m}$, then
\begin{align*}
\left[f(v)\right]_\mathfrak C &= \left[f\left(\sum_i a_i v_i \right) \right]_\mathfrak C
  \\ &= \left[\sum_i a_i f\left( v_i \right)\right]_\mathfrak C
  \\ &= \left[\sum_i a_i \sum_j b_{ij} w_j\right]_\mathfrak C
  \\ &= \left[\sum_j \left(\sum_i a_i b_{ij}\right) w_j\right]_\mathfrak C
  \\ &= \begin{pmatrix}\sum_i a_i b_{i1} \\ \vdots \\ \sum_i a_i b_{im}\end{pmatrix}
  \\ &= \begin{pmatrix}b_{11} & \cdots & b_{n1} \\ \vdots & \ddots & \vdots \\ b_{1m} & \cdots & b_{nm} \\ \end{pmatrix}\begin{pmatrix}a_1 \\ \vdots \\ a_n\end{pmatrix}
  \\ &= [f]^{\mathfrak B}_{\mathfrak C} [v]_\mathfrak B,
\end{align*} where $[f(v_i)]_\mathfrak C = [b_{i1} ~ \cdots ~ b_{im}]^\mathsf T$ whence $$[f]^\mathfrak B _ \mathfrak C = \bigg( [f(v_1)]_\mathfrak C ~~~ \cdots ~~~ [f(v_n)]_\mathfrak C \bigg).$$ Uniqueness follows from the uniqueness of the coordinate representation.
\end{proof}
\begin{prop}[composition and product] \label[prop]{p:comprod} Let $V \xrightarrow{f} W \xrightarrow{g} U$ be two homomorphisms. Then
$$[g\circ f]^\mathfrak B _ \mathfrak D = [g]^\mathfrak C _ \mathfrak D [f]^\mathfrak B _ \mathfrak C.$$
\end{prop}
\begin{proof}
Easy.
\end{proof}
Therefore, if the bases are fixed, there is a \textit{one-to-one correspondence} between the space of matrices and the space of linear transformations; where the operations are preserved under the correspondence, as follows: $$f+g \quad \longleftrightarrow \quad [f] + [g]$$ and $$f\circ g \quad \longleftrightarrow \quad [f][g].$$ We just say that, \begin{center}``the matrices are the same thing as the linear transformations.''\end{center}

\subsection{Basis transition}
\begin{theorem} Let $\mathfrak B$ be a basis of $F^n$ and let $A$ is an $n$ by $n$ square matrix. Then $A\mathfrak B = \{Av_i:~v_i\in\mathfrak B\}$ is a basis of $F^n$ if and only if $A$ is invertible.
\end{theorem}
\begin{proof}
Denote $\mathfrak B= \{v_i:~1\le i \le n\}$ like a column vector, although $F^n$ is not a field, namely, $$\mathfrak B = \begin{pmatrix} v_1\\ v_2 \\ \vdots \\ v_n \end{pmatrix}.$$ And define $A\mathfrak B$ as componentwise product: $$A\mathfrak B = \begin{pmatrix} Av_1\\ Av_2 \\ \vdots \\A v_n \end{pmatrix}.$$

($\Rightarrow$) If $Y$ is a basis, then $\exists C \in \mathfrak M_{n,n}(F),\quad C(A\mathfrak B) = \mathfrak B$ since $A\mathfrak B$ spans $F^n.$ $(CA)\mathfrak B = \mathfrak B$ implies that $(CA)v_i = v_i$ for every basis element $v_i \in \mathfrak B$ whence $CA = I$ and it implies that $A$ is invertible.

($\Leftarrow$) Let $v=\sum_i a_i v_i$ and $$R = \begin{pmatrix}a_1&\cdots&a_n\end{pmatrix}$$ be the coefficient matrix. Then $$v = \sum a_i v_i = R\mathfrak B = (R A^{-1})(A\mathfrak B) = \tilde R (A\mathfrak B)$$ whence $A\mathfrak B$ is a basis of $F^n$.
\end{proof}
\begin{theorem} \label[thm]{t:trninv}
If $\mathfrak B$ and $\tilde{\mathfrak B}$ are bases of V and $\mathfrak C$ and $\tilde{\mathfrak C}$ are bases of W, then for linear $f:~V\to W,$ $$[\operatorname{id}_W]_{\tilde{\mathfrak C }} ^{ \mathfrak C} [f]^{\mathfrak B}_{\mathfrak C} [\operatorname{id}_V]^{\tilde{\mathfrak B}} _{ \mathfrak B} = [f]^{\tilde{\mathfrak B}}_{\tilde{\mathfrak C}}.$$ Furthermore, $[\operatorname{id}]_\bullet^\bullet$ are all invertible.
\end{theorem}
\begin{proof}
For the first assertion, just use \cref{p:comprod}. For the second one, since $$[\operatorname{id}]_{\mathfrak B}^{\tilde{\mathfrak B}}[\operatorname{id}]^{\mathfrak B}_{\tilde{\mathfrak B}} = [\operatorname{id}]_{\mathfrak B}^{\mathfrak B} = I_{\operatorname{dim}\bullet},$$they are invertible by \cref{t:matinv}.

\end{proof}
\begin{prop}
For two bases $\mathfrak B$ and $\tilde{\mathfrak B}$ of $V$, the \textbf{transition matrix} $[\operatorname{id}_V]^\mathfrak B _{\tilde{ \mathfrak B}}$ is invertible. Conversely, for a basis $\mathfrak B$ of $V$ and an invertible square matrix $U$ whose the number of row is the dimension of the space $V$, there is a basis $\tilde{\mathfrak B}$ of $V$ such that $$U = [\operatorname{id}_V]^\mathfrak B _{\tilde{ \mathfrak B}}.$$
\end{prop}
\begin{proof}
The first assertion was proved in \cref{t:trninv}. For the second, let $$U = \begin{pmatrix}a_{11}&\cdots &a_{1n} \\ \vdots &\ddots &\vdots \\a_{n1}&\cdots&a_{nn} \end{pmatrix}$$ and $\mathfrak B = \{v_i\}$. We want another basis $\tilde{\mathfrak B} = \{w_i\}$ which satisfies $$U = [\operatorname{id}_V]^\mathfrak B _{\tilde{ \mathfrak B}} = \begin{pmatrix} | & & | \\ [v_1]_{\tilde{ \mathfrak B}} & \cdots & [v_n]_{\tilde{ \mathfrak B}} \\ | && | \end{pmatrix},$$ that is, $$v_i = \sum_{j} a_{ij}w_j, \qquad \begin{pmatrix} v_1 \\ \vdots \\ v_n \end{pmatrix} = \begin{pmatrix}a_{11}&\cdots &a_{1n} \\ \vdots &\ddots &\vdots \\a_{n1}&\cdots&a_{nn} \end{pmatrix}\begin{pmatrix} w_1 \\ \vdots \\ w_n \end{pmatrix} = U \begin{pmatrix} w_1 \\ \vdots \\ w_n \end{pmatrix} \qquad $$ Hence we have $$\tilde{\mathfrak B} = U^{-1}\mathfrak B.$$
\end{proof}
\subsection{Similarity}
\begin{defn}[similarity]
For two square matrices $A$ and $\tilde A$, we say those are \textbf{similar} if there is an invertible matrix $U$ such that $$\tilde A = U^{-1}AU,$$and denote $A\sim \tilde A$.
\end{defn}
\begin{prop}
Similarity relation is an \textbf{equivalence relation}, that is, satisfies the following three properties: \begin{itemize}
\item (Reflexivity) $A \sim A,$
\item (Symmetricity) $A \sim B \Longrightarrow B \sim A,$
\item (Transitivity) $A\sim B \sim C \implies A\sim C.$
\end{itemize}
\end{prop}

\begin{ex}[similarity]
For two square matrices $A$ and $\tilde A$, we say those are \textbf{similar} if there is an invertible matrix $U$ such that $$\tilde A = U^{-1}AU,$$and denote $A\sim \tilde A$.
\end{ex}
\begin{prop}
[similarity]
For a linear operator $f:~V\to V$ and the bases $\mathfrak B$, $\tilde{\mathfrak B}$ of $V$, two matrix representations of $f$ are similar, that is, $$[f]^{\mathfrak B} _{\mathfrak B} \sim [f]^{\tilde{\mathfrak B}} _{\tilde{\mathfrak B}}.$$
\end{prop}
\begin{proof}
$$[\operatorname{id}_V] _{\tilde{\mathfrak B}} ^{\mathfrak B} [f]^{\mathfrak B} _{\mathfrak B} [\operatorname{id}_V] ^{\tilde{\mathfrak B}} _{\mathfrak B} = [f]^{\tilde{\mathfrak B}} _{\tilde{\mathfrak B}}.$$
\end{proof}


To fulfill our study we provide an opportunity to verify its results by means of Wolfram Mathematica language.

\subsection{Mathematica commands} \label{subsec:mathematica-commands}
Proceeding to the repository~\cite{PK22Source} reader is able to find there a folder named \texttt{mathematica}
that contains the files
\begin{itemize}
    \item \texttt{OnTheBinomialTheoremAndDiscreteConvolution.m} is a package file with definitions
    \item \texttt{OnTheBinomialTheoremAndDiscreteConvolution.nb} is a notebook file with examples.
\end{itemize}
The following commands may be used to reproduce the results of this manuscript:
\begin{itemize}
    \item \texttt{A[m, r]} returns the real coefficient $\coeffA{m}{r}$ defined by~\eqref{eq:def_coeff_a}.
    \item \texttt{PrintTriangleOfA[rows]} prints the table of coefficients $\coeffA{m}{r}$. \\
    Command \texttt{PrintTriangleOfA[7]} reproduces the table (\ref{tab:table_of_coefficients_a}).
    \item \texttt{PolynomialL[m, n, k]} returns the polynomial $\polynomialL{m}{n}{k}$ defined by~\eqref{eq:def_polynomial_l}.
    \item \texttt{PolynomialP[m, x, b]} returns the polynomial $\polynomialP{m}{b}{x}$ defined by~\eqref{eq:def_polynomial_p}.
    \item \texttt{Expand[PolynomialP[m, x + y, x + y]]} verifies the Lemma~\ref{lemma_polynomial_p_and_odd_power}.
    \item \texttt{PolynomialH[m, t, j]} returns the polynomial $\coeffH{m}{t}{j}$ defined by~\eqref{eq:def_coeff_h}.
    \item \texttt{PolynomialX[m, t, k]} returns the polynomial $\polynomialX{m}{t}{k}$ defined by~\eqref{eq:def_coeff_x}.
    \item \texttt{Expand[BinomialTheoremAndDiscreteConvolutionTest[m, x + y]]} verifies the Corollary~\ref{cor_bin_exp_and_macaulay_conv}.
    \item \texttt{Expand[BinomialTheoremAndDiscreteConvolutionStrictTest[m, x + y]]} verifies the Corollary~\ref{cor_bin_exp_and_macaulay_conv_strict}.
    \item \texttt{DiscreteConvolutionPowerIdentityParametricTest[m, x, a]} verifies an equation~\eqref{eq:parametric-identity}.
    Usage \texttt{Column[Table[DiscreteConvolutionPowerIdentityParametricTest[1, x, 1], {x, 3, 20}], Left]}.
    \item \texttt{DiscreteConvolutionPowerIdentityStrictParametricTest[m, x, a]} verifies an equation~\eqref{eq:parametric-identity-strict}.
    Usage \texttt{Column[Table[DiscreteConvolutionPowerIdentityStrictParametricTest[1, x, 1], {x, 3, 20}], Left]}.
    \item \texttt{Expand[PolynomialIdentityOfP[1, n, b]]} validates an identity
    \[\polynomialP{m}{b}{x} = \sum_{r=0}^{m} \coeffA{m}{r} \sum_{j=0}^{r} (-1)^{j} x^{r-j} \binom{r}{j} \sum_{k=0}^{b-1} k^{r+j}\]
    \item \texttt{PolynomialIdentityInvolvingX[m, x, b]} validates an identity~\eqref{eq:p_all_forms}
    \[\polynomialP{m}{b}{x} = \sum_{r=0}^{m} (-1)^{m-r} \polynomialX{m}{r}{b} \cdot x^r\]
    \item \texttt{PolynomialIdentityInvolvingH[m, n, b]} validates an identity~\eqref{eq:p_all_forms}.
    \[\polynomialP{m}{b}{x} =\sum_{r=0}^{m} \sum_{\ell=1}^{2m-r+1} (-1)^{2m-r} \coeffH{m}{r}{\ell} \cdot b^\ell \cdot x^r\]
\end{itemize}

\subsection{Examples} \label{subsec:examples}
For example, given $m=1$ we have the following values of $\polynomialL{1}{x}{k}$
\begin{table}[H]
    \begin{tabular}{c|cccccccc}
        $x/k$ & 0 & 1  & 2  & 3  & 4  & 5  & 6  & 7 \\[3px]
        \hline
        0     & 1 &    &    &    &    &    &    &   \\
        1     & 1 & 1  &    &    &    &    &    &   \\
        2     & 1 & 7  & 1  &    &    &    &    &   \\
        3     & 1 & 13 & 13 & 1  &    &    &    &   \\
        4     & 1 & 19 & 25 & 19 & 1  &    &    &   \\
        5     & 1 & 25 & 37 & 37 & 25 & 1  &    &   \\
        6     & 1 & 31 & 49 & 55 & 49 & 31 & 1  &   \\
        7     & 1 & 37 & 61 & 73 & 73 & 61 & 37 & 1
    \end{tabular}
    \caption{Values of $\polynomialL{1}{x}{k}$. See OEIS entry: \href{https://oeis.org/A300656}{\texttt{A300656}}.}
    \label{tab:tab_3}
\end{table}
Table~\ref{tab:tab_3} can be reproduced using Mathematica command
\begin{center}
    \texttt{PrintTriangleOfPolynomialL[1, 7]}
\end{center}
defined in the~\cite{PK22Source}.
From Table~\ref{tab:tab_3} it is seen that
\begin{equation*}
    \begin{split}
        \polynomialP{1}{0}{0} &= 0 = 0^3 \\
        \polynomialP{1}{1}{1} &= 1 = 1^3 \\
        \polynomialP{1}{2}{2} &= 1+7 = 2^3 \\
        \polynomialP{1}{3}{3} &= 1+13+13 = 3^3 \\
        \polynomialP{1}{4}{4} &= 1+19+25+19 = 4^3 \\
        \polynomialP{1}{5}{5} &= 1+25+37+37+25 = 5^3
    \end{split}
\end{equation*}
Another case, given $m=2$ we have the following values of $\polynomialL{2}{x}{k}$
\begin{table}[H]
    \begin{tabular}{c|cccccccc}
        $x/k$ & 0 & 1    & 2    & 3    & 4    & 5    & 6    & 7 \\ [3px]
        \hline
        0     & 1 &      &      &      &      &      &      &   \\
        1     & 1 & 1    &      &      &      &      &      &   \\
        2     & 1 & 31   & 1    &      &      &      &      &   \\
        3     & 1 & 121  & 121  & 1    &      &      &      &   \\
        4     & 1 & 271  & 481  & 271  & 1    &      &      &   \\
        5     & 1 & 481  & 1081 & 1081 & 481  & 1    &      &   \\
        6     & 1 & 751  & 1921 & 2431 & 1921 & 751  & 1    &   \\
        7     & 1 & 1081 & 3001 & 4321 & 4321 & 3001 & 1081 & 1
    \end{tabular}
    \caption{Values of $\polynomialL{2}{x}{k}$. See OEIS entry: \href{https://oeis.org/A300656}{\texttt{A300656}}.}
    \label{tab:tab_4}
\end{table}
Table~\ref{tab:tab_4} can be reproduced using Mathematica command
\begin{center}
    \texttt{PrintTriangleOfPolynomialL[2, 7]}
\end{center}
defined in the~\cite{PK22Source}.
Again, an odd-power identity~\ref{lemma_polynomial_p_and_odd_power} holds
\begin{equation*}
    \begin{split}
        \polynomialP{2}{0}{0} &= 0 = 0^5 \\
        \polynomialP{2}{1}{1} &= 1 = 1^5 \\
        \polynomialP{2}{2}{2} &= 1+31 = 2^5 \\
        \polynomialP{2}{3}{3} &= 1+121+121 = 3^5 \\
        \polynomialP{2}{4}{4} &= 1+271+481+271 = 4^5 \\
        \polynomialP{2}{5}{5} &= 1+481+1081+1081+481 = 5^5
    \end{split}
\end{equation*}


%\chapter{Image Classification and Convolutional Neural Networks}\label{chap:introCNN}
In this chapter, we will give some basic introduction for supervised
learning especially for image classification problems.
Then, we will introduce and define the data and feature space
with related mappings such as convolution and pooling.
In addition, we use the  piecewise (bi-)linear functions on multilevel grids to
interpret the multi-resolution images in the process of CNN. At last,
we define some classical CNN models mathematically with those notation.

\section{Supervised learning on image classification}\label{sec:MLbasics}
We consider a basic machine learning problem for classifying a
collection of images into $\kappa$ distinctive classes.  As an
example, we consider a two-dimensional image which is usually
represented by a tensor
$$
f\in \mathbb  \FC := \mathbb{R}^{m\times n\times c}.
$$
Here
\begin{equation}
\label{data-c}
c=
\left\{
\begin{array}[rl]{rl}
1 & \mbox{for black-white image}\\
3 & \mbox{for color image}.
\end{array}
\right.
\end{equation}

A typical supervised machine learning problem begins with a data set (training data)
$$
D := \{(f_i, y_i)\}_{i=1}^N,
$$
with
$$
\{f_i\}_{i=1}^N \subset \FC,
$$
and $y_i \in \mathbb{R}^{\kappa}$ is the label for data $f_i$, with
$[y_i]_j$ as the probability for $f_i$ in classes $j$.

Roughly speaking, a supervised learning problem can be thought as data fitting
problem in a high dimensional space $\FC$.
Namely, we need to find a mapping
$$
\classmap:  \mathbb R^{m\times n\times c}\mapsto \mathbb R^\kappa
$$
such that, for a given $f\in  \FC$,
\begin{equation}\label{eq:idealouput}
\classmap(f)\approx e_i\in \mathbb R^\kappa
\end{equation}
if $f$ is in class $i$, for $1\le i\le \kappa$.
For the general setting above, we use a probatilistic model for understanding the
output $\classmap(f) \in \mathbb{R}^{\kappa}$ as a discrete
distribution on $\{1, \cdots,\kappa\}$, with $[\classmap (f)]_i$ as the probability
for $f$ in the class $i$, namely
\begin{equation}
\label{distrib}
0 \le [\classmap(f)]_i \le 1,\quad
\sum_{i=1}^\kappa  [\classmap(f)]_i=1.
\end{equation}
At last, we finish our model with a simple strategy to choose
\begin{equation}\label{eq:maxchoose}
\mathop{\arg\max}_{i}\{[\classmap(f)]_i~:~ i = 1:\kappa\},
\end{equation}
as the label for a test data $f$, which ideally is close to
\eqref{eq:idealouput}.  The remaining key issue is the construction of
the classification mapping $\classmap$.

%\subsection{Model construction}
%In many CNN type models for image classification, there are usually
%three major steps to construct $\classmap$.
The main step in the construction of $\classmap$ is to
construct a nonlinear mapping
\begin{equation}
\label{linearize}
\linearize: \FC \mapsto V_J
\end{equation}
with
%\begin{equation}
%  \label{F}
%\FC= \mathbb R^{m\times n\times c}
%\end{equation}
\begin{equation}
\label{VJ}
V_J = \mathbb R^{m_J\times n_J\times c_J}.
\end{equation}
To be consistent with the notation for CNN which will be described below,
here the subscript $J$ refers to the number of
coarsening girds in CNN.
Roughly speaking, the map $\linearize$ plays two roles.  The first role
is to make a dimension reduction, namely
$$
m_Jn_Jc_J\ll  mnc.
$$
The second role is to map a complicated set of data into a set of data
that are linearly separable. As a result, the simple logistic regression
procedure can be applied.

The first step in a logistic regression it to introduce a linear mapping:
\begin{equation*}
\Theta: \FC \to\mathbb{R}^{\kappa} ,
\end{equation*}as
\begin{equation}\label{thetamap}
\Theta(x)=Wx+b,
\end{equation}
where $W=(w_{ij})\in\mathbb{R}^{(m_J \times n_J \times c_J)\times \kappa}$,
$b\in\mathbb{R}^{\kappa}$.

We then use the soft-max function.
\begin{equation}
\label{softmax}
[S(z)]_i=[{\rm Solftmax}(z)]_i= \frac{e^{z_i}}{\sum_{j} e^{z_j}},
\end{equation}
to obtain a logistic regression model
\begin{equation}
\label{eq:log_reg}
S \circ \Theta: \mathbb R^{m_J\times n_J\times c_J}\mapsto \mathbb R^\kappa.
\end{equation}
%	This makes $S\circ \linearize(f) \in \mathbb{R}^\kappa$ as a distribution on
%	$\{1,\cdots,\kappa\}$.

By combining the nonlinear mapping $\classmap$ in \eqref{linearize}
and the logistic regression \eqref{eq:log_reg}, we obtain the following classifier:
\begin{equation}
\label{classifier}
\classmap=  S\circ \Theta\circ \linearize.
\end{equation}

%\subsection{Loss function and training}
Given the model \eqref{classifier},
we finish the training phase with solving the next optimization
problem:
\begin{equation}
\label{eq:abstractloss}
\min \sum_{j=1}^Nl(\classmap(f_j),y_j),
\end{equation}
where
Here $l(\classmap(f_j),y_j)$ is a  loss function that measures the
predicted result $\classmap(f_j)$ and the real label $y_j$.
In logistic regression,
the following cross-entropy loss function is often used
\begin{equation}\label{eq:cross-entropy}
l(\classmap(f), y) = \sum_{i=1}^\kappa -[y]_i \log [\classmap(f)]_i.
\end{equation}

\section{Data space, feature space and relevant mappings}\label{sec:spaces}
Given a data
\begin{equation}
\label{data-f}
f \in \mathbb{R}^{m \times n \times c},
\quad \text{or}\quad [f]_i \in
\mathbb{R}^{m\times n}, \quad i = 1:c,
\end{equation}
where $m\times n$ is called the spatial dimension and $c$ is the
channel dimension.

For the given data $f$ in \eqref{data-f}, we look for some
feature vector, denoted by $u$,  associated with $f$:
\begin{equation}
\label{u}
u \in \mathbb{R}^{m \times n \times h}.
\end{equation}
We make an assumption that the data $f$ and feature $u$ are related by a mapping
(which can be either linear or nonlinear)
\begin{equation}
\label{u}
A:  \mathbb{R}^{m \times n \times h}\mapsto \mathbb{R}^{m \times n \times c},
\end{equation}
so that
\begin{equation}
\label{Auf}
A(u)=f.
\end{equation}
A mapping
\begin{equation*}
B : \mathbb{R}^{m\times n\times c} \mapsto \mathbb{R}^{m \times n\times h},
\end{equation*}
is called a feature extractor if $B \approx A^{-1}$ and
\begin{equation}\label{vBf}
v = B(f),
\end{equation}
is such that $v \approx u$.

The data-feature relationship \eqref{Auf} or \eqref{vBf} is not
unique.   Different relationships give rise to different features.
We can view the data-feature relationship given in \eqref{Auf}
as a model that we propose.  Here the mapping $A$, which can be either
linear or nonlinear, is unknown and needs to be trained.

%\begin{remark}
We point out  that the data space and feature space may have different
numbers of channels.
%\end{remark}

\subsection{A special linear mapping: convolution}
%Firstly, as mentioned before, an image type
%data with multichannel can be understand as
%vector valued functions
%$$
%f \in \mathbb{R}^{m \times n \times c}, \quad \text{or}\quad [f]_i \in
%\mathbb{R}^{m\times n}, \quad i = 1:c,
%$$
One important class of linear mapping is the so called convolution:
%We first consider $\theta$ a convolution operator (with stride $1$)
%and padding:
$$
\theta: \mathbb{R}^{m\times n\times c} \mapsto \mathbb{R}^{m\times n \times h},
$$
that can be defined by
\begin{equation}\label{conv-1}
[\theta(f)]_{t} = \sum_i^{c}K_{i,t} \ast [f]_i + b_t
\bm{1}  \in \mathbb{R}^{m\times n}, \quad t = 1:h,
\end{equation}
where $\bm{1}  \in \mathbb{R}^{m\times n} $ is a
$m\times n$ matrix with all elements being $1$,
and for $g \in \mathbb{R}^{m\times n}$
\begin{equation}\label{con1}
[K \ast g]_{i,j} = \sum_{p,q=-k}^k K_{k+1+p,k+1+q} g_{i + p, j + q}, \quad i=1:m, j = 1:n.
\end{equation}
The coefficients in \eqref{con1} constitute  a kernel matrix
\begin{equation}
K \in \mathbb{R}^{(2k+1) \times (2k+1)},
\end{equation}
where $k$ is often taken as small integers.
Here padding means how choose $ X_{i+ p, j + q}$
when $(i+ p, j + q)$ is out of $1:m$ or $1:n$.
Those next three choices are often used
\begin{equation}\label{eq:padding}
f_{i + p, j + q} = \begin{cases}
0,  \quad &\text{zero padding}, \\
f_{(i + p)\pmod{m}, (s + q)\pmod{n}},  \quad &\text{periodic padding}, \\
f_{|i-1 +p|, |j -1  +q|},  \quad &\text{reflected padding}, \\
\end{cases}
\end{equation}
if
\begin{equation}
i + p \notin \{1, 2, \dots, m\} ~\text{or} ~  j+ q \notin \{1, 2, \dots, n\}.
\end{equation}
Here $ d \pmod m \in \{1, \cdots, m\} $  means the remainder when $d$ is divided by $m$.

%Then we remark that, if we take convolution without pooling, there is a
%shift with
%$$
%i+ p  \rightarrow i + k+ p,
%$$
%and also for $j$ such that all index $(i + k+ p, j + k+ p)$ is during
%$1:n$.

If we formally write
\begin{equation}
f=
\begin{pmatrix}
f_1\\
\vdots\\
f_c
\end{pmatrix}
\end{equation}
We can then write the operation \eqref{conv-1} as
\begin{equation}
\label{eq:4}
\theta(f)=K\ast f+b
\end{equation}
where
$$
K=(K_{ij})\in \mathbb R^{[(2k+1)\times(2k+1)]\times h\times c}
$$
and
$$
\bm{b}=\bm{1}_{m\times n}\otimes b.
$$

The operation \eqref{eq:4} is also called a convolution with stride 1. More generally,
given an integer $s\ge1$, a convolution with stride $s$ for $f \in \mathbb{R}^{m\times
	n}$ is defined as:
\begin{equation}\label{stride}
[K \ast_s f]_{i,j} = \sum_{p,q=-k}^k K_{p,q} f_{s(i-1)+1 + p, s(j-1)+1 + q},
\quad i = 1: \lceil  \frac{m}{s}\rceil , j = 1: \lceil  \frac{n}{s}\rceil.
\end{equation}
Here $ \lceil  \frac{m}{s}\rceil$ denotes the smallest integer that greater than $\frac{m}{s}$.
In CNN, we often take $s=2$.

\subsection{Some linear and nonlinear mappings and extractors}
A data-feature map $A$ and feacture extractor $B$ can be either
linear or nonlinear.   The nonlinearity can be obtained from
appropriate application of an activation function
\begin{equation}
\label{act}
\sigma: \mathbb{R} \to \mathbb{R} .
\end{equation}
In this paper, we mainly consider a special activation function, known
as the {\it rectified linear unit} (ReLU), which is defined by
\begin{equation}
\label{relu}
\sigma(x)= {\rm ReLU}(x) :=\max(0,x), \quad x\in\mathbb{R}.
\end{equation}
By applying the function to each component, we can extend this
\begin{equation}
\label{vector-act}
\sigma:\mathbb R^{m\times n\times c}\mapsto \mathbb R^{m\times n\times c}.
\end{equation}

A linear data-feature mapping can simply given by a convolution as in \eqref{con1}:
\begin{equation}
\label{linearA}
A(f)=\xi\ast f
\end{equation}
A nonlinear mapping can be given by compositions of convolution and
activation functions:
\begin{equation}
\label{nonlinearA}
A=\xi\circ\sigma\circ\eta ,
\end{equation}

and
\begin{equation}
\label{extractor}
B=\sigma\circ \gamma \circ\sigma  .
\end{equation}
Here $\xi$, $\eta$ and $\gamma$ are all
appropriate convolution mappings.

\subsection{An iterative feacture extraction scheme}
One key idea in this paper is that we use a simple iterative
process to approximately solve \eqref{Auf} using \eqref{vBf}. Namely,
for $i=1:\nu$
\begin{equation}\label{eq:smoothB}
u^{i} = u^{i-1} + B^{i}(f- A(u^{i-1}))
\end{equation}
for an appropriately chosen $u^0$.  We refer to \cite{xu1992iterative}
for more discussion on iterative scheme in the form of \eqref{eq:smoothB}.

\section{Piecewise (bi-)linear functions on multilevel grids}\label{sec:functions}
An image can be viewed as a function on a grid.  Images
with different resolutions can then be viewed as functions on grids of
different sizes.  The use of such multiple-grids is a main technique
used in the standard multigrid method for solving discretized partial
differential equations \cite{xu1992iterative, xu2002method},
and it can also be interpreted as a main ingredient used in
convolutional neural networks (CNN).

Without loss of generality, for simplicity, we assume that the initial
grid, $\mathcal T$, is of size
$$
m=2^{s}+1, n=2^{t}+1
$$
for some integers $s, t\ge 1$.
Starting from $\mathcal T_1=\mathcal T$,  we consider a sequence of
coarse grids (as depicted in Fig.~\ref{mgrid} with $J=4$):
\begin{equation}
\label{grids}
\mathcal T_1, \mathcal T_2, \ldots, \mathcal T_J
\end{equation}
such that ${\mathcal T}_\ell$ consist of $m_\ell\times n_\ell$ grid
points, with
\begin{equation}
\label{mn-ell}
m_\ell=2^{s-\ell+1}+1, n_\ell=2^{t-\ell+1}+1.
\end{equation}
\begin{figure}[!htbp]\label{mgrid}
	\begin{center}
		\includegraphics[width=0.15\textwidth]{grid2.png} \quad
		\includegraphics[width=0.15\textwidth]{grid1.png} \quad
		\includegraphics[width=0.15\textwidth]{grid0.png} \quad
		\includegraphics[width=0.15\textwidth]{grid.png}
	\end{center}
	$$
	\mathcal T_1\hskip1in \mathcal T_2\hskip1in \mathcal T_3\hskip1in\mathcal T_4
	$$
	\caption{multilevel grids for piecewise linear functions}
\end{figure}

The grid points of these grids can be given by
$$
x_i^{\ell}=i h_{1,\ell}, y_j^{\ell}=j h_{2,\ell},  i=1, \ldots, m_\ell,
j=1, \ldots, n_\ell.
$$
Here $h_{1,\ell} = 2^{-s + \ell -1}a$ and $h_{2,\ell} = 2^{-t + \ell - 1}b$,
for some $a,b >0$. The above geometric coordinates $(x_i^\ell, y_i^\ell)$
are usually not used in image precess literatures, but they are relevant
in the context of multigrid method for numerical solution of PDEs.
We now consider piecewise linear functions on the sequence of grids
\eqref{grids} and we obtain a nested sequence of linear vector spaces
\begin{equation}
\label{Vk}
\mathcal V_1\supset\mathcal V_2\supset\ldots\supset \mathcal
V_J.
\end{equation}
Here each $\mathcal V_\ell$ consists of all piecewise bilinear (or linear)
functions with respect to the grid \eqref{grids} and \eqref{mn-ell}.
Each $\mathcal V_\ell $ has a set of basis functions:
$\phi_{ij}^\ell\in \mathcal V_\ell$ satisfying:
$$
\phi_{ij}^\ell(x_p,y_q)=\delta_{(i,j), (p,q)} =
\begin{cases}
1 \quad &\text{if} \quad (p,q) = (i,j), \\
0 \quad &{\text{if}} \quad (p,q)\neq (i,j).
\end{cases}
$$
Thus, for each $v \in \mathcal V_{\ell}$, we have
\begin{equation}\label{expand}
v(x,y)=\sum_{i,j}v^\ell_{ij}\phi_{ij}^\ell(x,y).
\end{equation}
\subsection{Prolongation}
Given a piecewise (bi-)linear function $\bm v\in\mathcal V_{\ell+1}$, the
nodal values of $\bm v$ on $m_{\ell+1}\times n_{\ell+1}$ grids point
constitute a tensor
$$
v^{\ell+1}\in \mathbb R^{m_{\ell+1}\times n_{\ell+1}}.
$$
We note that $\bm v\in\mathcal V_{\ell}$ thanks to \eqref{Vk} and the
nodal values of $\bm v$ on $\mathcal T_\ell$
constitute a tensor
$$
v^{\ell}\in \mathbb R^{m_{\ell}\times n_{\ell}}.
$$
Using the property of piecewise (bi-)linear functions, it is easy to see that
\begin{equation}
\label{eq:13}
v^{\ell}=\bar P_{\ell+1}^\ell v^{\ell+1}
\end{equation}
where
\begin{equation}
\label{mg-prolong}
\bar P_{\ell+1}^\ell: \mathbb R^{m_{\ell+1}\times n_{\ell+1}}\mapsto  \mathbb R^{m_{\ell}\times n_{\ell}}
\end{equation}
which is called a prolongation in multigrid terminology.  More
specifically,
\begin{equation}
\label{eq:7}
v^{\ell}_{2i-1,2j-1}=  v^{\ell+1}_{i,j},
\end{equation}
with
\begin{equation}
\label{eq:9}
v^{\ell}_{2i-1, 2j} = \frac{1}{2}(v^{\ell+1}_{i,j} + v^{\ell+1}_{i,j+1}), \quad
v^{\ell}_{2i, 2j-1} = \frac{1}{2}(v^{\ell+1}_{i,j} + v^{\ell+1}_{i+1,j}),
\end{equation}
and
\begin{equation}
%\begin{tiny}
%{\scriptsize
v^{\ell}_{2i, 2j}  =
\begin{cases}
\frac{1}{4}(v^{\ell+1}_{i,j} + v^{\ell+1}_{i+1,j} + v^{\ell+1}_{i,j+1} + v^{\ell+1}_{i+1,j+1}),  &\text{if $v^\ell$ is piecewise bilinear }, \\
\frac{1}{2}( v^{\ell+1}_{i+1,j} + v^{\ell+1}_{i,j+1}), &\text{if $v^\ell$ is piecewise linear }.
\end{cases}
%\end{tiny}
%}
\end{equation}
%when
%\begin{description}
%\item[$v^\ell$ is piecewise bilinear]
%\begin{equation}
%\label{eq:pro-bilinear}
%v^{\ell}_{2i, 2j} = \frac{1}{4}(v^{\ell+1}_{i,j} + v^{\ell+1}_{i+1,j} + v^{\ell+1}_{i,j+1} + v^{\ell+1}_{i+1,j+1}),
%\end{equation}
%\item[$v^\ell$ is piecewise linear]
%\begin{equation}
%\label{eq:pro-linear}
%v^{\ell}_{2i, 2j} = \frac{1}{2}( v^{\ell+1}_{i+1,j} + v^{\ell+1}_{i,j+1}).
%\end{equation}
%\end{description}

\subsection{Pooling, restriction and interpolation}\label{sec:cnn-restriction}
The prolongation given by \eqref{mg-prolong} can be used to transfer
feature from a coarse grid to a fine grid.
On the other hand, we also
need a mapping, known as restriction,  that transfer data from fine grid to corse grid:
\begin{equation}
\label{mg-restrict}
\bar R_\ell^{\ell+1}: \mathbb R^{m_{\ell}\times n_{\ell}}  \mapsto  \mathbb R^{m_{\ell+1}\times n_{\ell+1}}.
\end{equation}
%This operation can often be obtained by two different steps.
%First, we carry out a convolution operation $\tilde R: \mathbb R^{n_\ell\times n_\ell}\mapsto
%\mathbb R^{n_\ell\times n_\ell}$:
%\begin{equation}
%\label{linear-restriction}
%\begin{aligned}
%(\tilde Rr)_{i,j}&=r_{i,j} +{1\over 2}(r_{i,j-1} + r_{i,j+1} + r_{i-1,j} + r_{i+1,j} + r_{i+1,j-1} + r_{i-1,j+1}) ,
%\end{aligned}
%\end{equation}
%Then, we define
%\begin{equation}
%\label{linear-restriction-stride}
%(Rr)_{i,j}=(\tilde R r)_{2i-1,2j-1}, \quad 1\le i, j \le n_{\ell+1}.
%\end{equation}
%%Using the terminology from deep learning \cite{goodfellow2017deep},
%As the definition in above section,
%we note that \eqref{linear-restriction} and \eqref{linear-restriction-stride} can be written as a
%convolution with a $3\times3$ kernel with stride 2:
%\begin{equation}\label{eq:restriction}
%Rr=K\ast_2 r,\quad K=\left ( \begin{array}{ccc}
%0 &\frac{1}{2}&\frac{1}{2}\\
%\frac{1}{2}& 1&\frac{1}{2}\\
%\frac{1}{2}&\frac{1}{2}& 0
%\end{array}\right ).
%\end{equation}
In multigrid for solving discretized partial differential equation,
the restriction is often taken to be transpose of the prolongation
given by \eqref{mg-prolong}:
\begin{equation}
\label{mg-RP}
\bar R_\ell^{\ell+1} = [\bar P_{\ell+1}^\ell]^T.
\end{equation}
\begin{lemma}
	If $\tilde P_{\ell+1}^\ell$ takes the form of prolongation
	in multigrid methods for linear finite element functions
	on the above grids, then $\tilde R_{\ell}^{\ell+1}$ is
	a convolution with stride $2$ and a $3\times3$
	kernel as:
	\begin{equation}\label{restriction}
	R_\ell^{\ell+1} f=K_R\ast_2 f,
	\end{equation}
	where, if $\mathcal V_\ell$ is piecewise bilinears,
	\begin{equation}\label{bi-restrict}
	K_R=
	\begin{pmatrix}
	\frac{1}{4} &\frac{1}{2}&\frac{1}{4}\\
	\frac{1}{2}& 1&\frac{1}{2}\\
	\frac{1}{4}&\frac{1}{2}&  \frac{1}{4}
	\end{pmatrix},
	\end{equation}
	or, if $\mathcal V_\ell$ is piecewise linears,
	\begin{equation}
	\label{linear-restrict}
	K_R=
	\begin{pmatrix}
	0 &\frac{1}{2}&\frac{1}{2}\\
	\frac{1}{2}& 1&\frac{1}{2}\\
	\frac{1}{2}&\frac{1}{2}&  0
	\end{pmatrix}.
	\end{equation}
\end{lemma}
In addition, all these convolutions are applied with zero padding as in \eqref{eq:padding}
, which is consistent with the Neumann boundary condition for applying FEM to
numerical PDEs. More details will be discussed in \S \ref{sec:cnn-restriction}.

In the deep learning literature, the restriction such as
\eqref{mg-restrict} is often known as pooling operation.  One popular
pooling is a convolution with stride $s$, with some small integer $s>1$.

Some other fixed (or untrained) poolings are also often used.
One popular pooling is the so called average pooling $R_{avr}$ which
can be a convolution with stride $2$ or bigger using the  kernel $K$
in the form of
\begin{equation}
\label{average-K}
K=
\frac{1}{9}\begin{pmatrix}
1 & 1 & 1 \\
1 & 1 & 1 \\
1 & 1 & 1
\end{pmatrix}.
\end{equation}
Nonlinear pooling operator is also used, for the example the $(2k+1)
\times (2k+1)$ max-pooling operator with stride $s$ as follows:
\begin{equation}
[{R}_{\rm max}(f)]_{i,j} = \max_{-k\le p ,q\le k} \{f_{s(i-1)+1 + p, s(j-1)+1 +q} \}.
\end{equation}

Another approach to the construction of restriction of pooling can be
obtained by using interpolation.
Given
$$
v^\ell\in \mathbb \mathbb R^{m_{\ell}\times n_{\ell}},
$$
let $\bm v\in \mathcal V_\ell$ be the function whose nodal values are
precisely give by $v^\ell$ as in \eqref{expand}.
Any reasonable linear operator
\begin{equation}
\label{eq:11}
\Pi:  \mathcal V_\ell\mapsto \mathcal V_{\ell+1},
\end{equation}
such as:
nodal value interpolation,
Scott-Zhang interpolation and
$L^2$ projection \cite{xu2019FEM},
would give rise to a mapping
\begin{equation}
\label{mg-Pi}
\Pi_\ell^{\ell+1}: \mathbb R^{m_{\ell}\times n_{\ell}}  \mapsto
\mathbb R^{m_{\ell+1}\times n_{\ell+1}},
\end{equation}
such that
$$
v^{\ell+1}=\Pi_\ell^{\ell+1} v^\ell.
$$
As situations permit, we can use these a priori given restrictions to replace
unknown pooling operators to reduce the number of parameters.


\section{Derivation of Elasticity Equation}

For completeness we start our discussing of the governing equations
with a derivation of the elasticity equation. Consider domain $\Omega$
bounded by boundary $\Gamma$. Applying a Lagrangian description of the
conservation of momentum gives
\begin{equation}
\label{eqn:momentum:vec}
\frac{\partial}{\partial t}\int_{\Omega}\rho(\vec{x})\frac{\partial\vec{u}}{\partial t}\, d\Omega=\int_{\Omega}\vec{f}(\vec{x},t)\, d\ + \int_{\Gamma}\vec{\tau}(\vec{x},t)\, d\Gamma.
\end{equation}
The traction vector field is related to the stress tensor through
\begin{equation}
\vec{\tau}(\vec{x},t) = \tensor{\sigma}(\vec{u}) \cdot \vec{n},
\end{equation}
where $\vec{n}$ is the outward normal vector to $\Gamma$. Substituting
into equation \vref{eqn:momentum:vec} yields
\begin{equation}
\frac{\partial}{\partial t}\int_{\Omega}\rho(\vec{x})\frac{\partial\vec{u}}{\partial t}\, d\Omega=\int_{\Omega}\vec{f}(\vec{x},t)\, d\Omega+\int_{\Gamma}\tensor{\sigma}(\vec{u})\cdot\vec{n}\, d\Gamma.
\end{equation}
Applying the divergence theorem,
\begin{equation}
\int_{\Omega}\tensor{\nabla}\cdot\vec{a}\: d\Omega=\int_{\Gamma}\vec{a}\cdot\vec{n}\: d\Gamma,
\end{equation}
to the boundary integral results in
\begin{equation}
\frac{\partial}{\partial t}\int_{\Omega}\rho(\vec{x})\frac{\partial\vec{u}}{\partial t}\, d\Omega=\int_{\Omega}\vec{f}(\vec{x},t)\, d\Omega+\int_{\Omega}\tensor{\nabla}\cdot\tensor{\sigma}(\vec{u})\, d\Omega,
\end{equation}
which we can rewrite as
\begin{equation}
\int_{\Omega}\left(\rho(\vec{x})\frac{\partial^{2}\vec{u}}{\partial t^{2}}-\vec{f}(\vec{x},t)-\tensor{\nabla}\cdot\tensor{\sigma}(\vec{u})\right)\, d\Omega=\vec{0}.
\end{equation}
Because the domain $\Omega$ is arbitrary, the integrand must be the zero
vector at every location in the domain, so that we end up with
\begin{gather}
\rho(\vec{x})\frac{\partial^{2}\vec{u}}{\partial t^{2}}-\vec{f}(\vec{x},t)-\tensor{\nabla}\cdot\tensor{\sigma}=\vec{0}\text{ in }\Omega,\\
\tensor{\sigma}(\vec{u})\cdot\vec{n}=\vec{\tau}(\vec{x},t)\text{ on }\Gamma_{\tau}\text{,}\\
\vec{u}=\vec{u}_0(\vec{x},t)\text{ on }\Gamma_{u},\text{ and}\\
\vec{u}^{+}-\vec{u}^{-}=\vec{d}\text{ on }\Gamma_{f}.
\end{gather}
We specify tractions, $\vec{\tau}$, on boundary $\Gamma_{f}$, displacements,
$\vec{u^{o}}$, on boundary $\Gamma_{u}$, and slip, $\vec{d}$,
on fault interface $\Gamma_{f}$.
%(we will consider the case of fault constitutive models in Section \vref{sec:fault}).


\section{Differentiation}
\begin{definition}
    Fix $m,n\in\mathbb N$, let $L(\mathbb R^m,\mathbb R^n)$ be the set of linear maps to $\mathbb R^m$ to $\mathbb R^n$.
    Note that this space is isomorphic to $\mathbb R^{mn}$, both algebraicly and topologically, as we have the metric
    $$\forall T\in L(\mathbb R^m,\mathbb R^n),\|T\|=\sqrt{\sum_{i=1}^m\sum_{j=1}^n|T_{ij}|^2}=\sqrt{\sum_{i=1}^m\|T{e_i}\|^2}$$
\end{definition}
\begin{lemma}\label{mat_norm}
    (a) Given a linear map $T$, for every $x\in\mathbb R^m$, we have $\forall x\in\mathbb R^m,\|Tx\|\le \|T\|\|x\|$.
    So $T$ is Lipschitz hence continuous.\\
    (b) For $S\in L(\mathbb R^m,\mathbb R^n),T\in L(\mathbb R^n,\mathbb R^p)$, $\|TS\|\le \|T\|\|S\|$.
\end{lemma}
\begin{proof}
    (a) If $x=\sum_ix_ie_i$, then
    $$\|Tx\|=\left\|\sum_{i=1}^mx_iTe_i\right\|\le \sum_{i=1}^m|x_i|\|Te_i\|\le \sqrt{\sum_{i=1}^m|x_i|^2}\sqrt{\sum_{i=1}^m\|Te_i\|^2}=\|x\|\|T\|$$
    (b) We have
    $$\|TS\|=\sqrt{\sum_{i=1}^m\|TSe_i\|^2}\le \sqrt{\sum_{i=1}^m\|T\|^2\|Se_i\|^2}=\|T\|\|S\|$$
    As desired.
\end{proof}
Recall that a function $f:\mathbb R\to\mathbb R$ is differentiable at $a$ if $\lim_{h\to 0}(f(a+h)-f(a))/h$ exists.
So let $\epsilon(h)=(f(a+h)-f(a))/h-f^\prime(a)$, then $f(a+h)=f(a)+f^\prime(a)h+\epsilon(h)h$ and $\epsilon\to 0$ as $h\to 0$.
We can think of this as $\epsilon(0)=0$ and $\epsilon$ is continuous at $0$.
So we want to use it to define differentiation in higher dimensions.
\begin{definition}
    Given $m,n\in\mathbb N$ and an open set $U\subset\mathbb R^m$, a function $f:U\to\mathbb R^n$ and $a\in U$.
    We say $f$ is differentiable at $a$ if there is a linear map $T:\mathbb R^m\to\mathbb R^n$ and a function $\epsilon:\{h\in\mathbb R^m:a+h\in U\}\to\mathbb R^n$ such that
    $$f(a+h)=f(a)+T(h)h+\epsilon(h)\|h\|$$
    where $\epsilon\to 0$ as $h\to 0$. (Or $\epsilon(0)=0$ and $\epsilon$ is continuous ar $0$).
\end{definition}
\begin{remark}
    $$\epsilon(h)=
    \begin{cases}
        0\text{, if $h=0$}\\
        \frac{f(a+h)-f(a)-T(h)}{\|h\|}\text{, if $h\neq 0$ and $a+h\in U$}
    \end{cases}$$
\end{remark}
Since $U$ is open, $\exists r>0, D_r(a)\subset U$, so $D_r(a)\subset \operatorname{Dom}\epsilon$.
Note also that our condition on $\epsilon$ is also equivalent to say $\epsilon(h)\|h\|=o(\|h\|)$ as $h\to 0$.\\
Next, we observe that $T$ (if it exists) is unique.
Indeed, if both $T,S$ satisfies our condition, then $(S(h)-T(h))/\|h\|\to 0$ as $h\to 0$, so by choosing $h=x/n$ for $n\in\mathbb N$ we have $S=T$.
\begin{definition}
    This unique $T$ is called the derivative of $f$ at $a$, denoted by $f^\prime(a)$ or $Df(a)$ or $Df|_a$, so
    $$f(a+h)=f(a)+f^\prime(a)(h)+o(\|h\|)$$
\end{definition}
\begin{definition}
    We say $f$ is differentiable at $U$ if it is differentiable at $a$ for every $a\in U$.
    So the derivative of $f$ on $U$ is the map $f^\prime:U\to L(\mathbb R^m,\mathbb R^n)$.
\end{definition}
\begin{remark}
    When $m=1$, $T$ is a linear map $\mathbb R\to\mathbb R^n$, so for $T\in L(\mathbb R^m,\mathbb R^n)$, so b y setting $v=T(1)$, so $\forall x\in\mathbb R,T(x)=xv$.
    Indeed, $L(\mathbb R,\mathbb R^n)\cong\mathbb R^n$ by $T\mapsto T(1)$.
    Hence for open $U\subset\mathbb R,f:U\to\mathbb R^n,a\in U$, we have $f$ is differentiable at $a$ if and only if there is some $v\in\mathbb R^n$ with $f(a+h)=f(a)+hv+o(h)$.
    So $v=f^\prime(a)$.
\end{remark}
\begin{example}
    1. Every constant function is differentiable as we can take $f^\prime(a)\equiv 0\in L(\mathbb R^m,\mathbb R^n)$.
    2. Every linear map $f$ is differentiable by taking $f'(a)=f\in L(\mathbb R^m,\mathbb R^n)$. for every $a$.\\
    3. Any bilinear map $f:\mathbb R^m\times\mathbb R^n\to\mathbb R^p$ is differentiable.
    Indeed, we have
    $$f((a,b)+(h,k))=f(a+b,h+k)=f(a,b)+f(a,k)+f(h,b)+f(h,k)$$
    Note that $f(a,k)+f(h,b)$ is linear in $(h,k)$, therefore it remains to checl $f(h,k)=o(\|h\|)$.
    We see
    \begin{align*}
        \|f(h,k)\|&=\left\|f\left( \sum_{i=1}^mh_ie_i, \sum_{j=1}^nk_je_j\right)\right\|\\
        &\le \sum_{i,j}|h_i||k_j|\|f(e_i,e_j)\|\\
        &\le \|(h,k)\|^2\sum_{i,j}\|f(e_i,e_j)\|\\
        &=O(\|(h,k)\|^2)=o(\|(h,k)\|)
    \end{align*}
    4. Take $f:\mathbb R^n\to\mathbb R$ by $f(x)=\|x\|^2$, so
    $$f(a+h)=\|a+h\|^2=\|a\|^2+2\langle a,h\rangle+\|h\|^2=f(a)+2\langle a,h\rangle+o(\|h\|)$$
    So we can have $f^\prime(a)(h)=2\langle a,h\rangle$.\\
    5. Let $M_n\cong L(\mathbb R^n,\mathbb R^n)$ be the collection of all $n\times n$ real matrices.
    Consider $f:M_n\to M_n, A\mapsto A^2$, so
    $$f(A+H)=A^2+AH+HA+H^2=f(A)+AH+HA+o(\|H\|)$$
    due to Lemma \ref{mat_norm}.
    So $f^\prime(A)(H)=AH+HA$.
\end{example}
\begin{proposition}
    Differentiablility implies continuity.
\end{proposition}
\begin{proof}
    Write $f(a+h)=f(a)+f^\prime(a)(h)+\epsilon(h)\|h\|$ where $\epsilon(0)=0$ and $\epsilon$ is continuous at $0$.
    $f^\prime(a)$ is continuous by Lemma \ref{mat_norm} (which implies every linear map is continuous), so the RHS is continuous in $h$, therefore $h\mapsto f(a+h)$ is continuous at $h=0$, hence $f$ is continuous at $a$.
\end{proof}
\begin{proposition}[Chain Rule]
    Consider open $U\in\mathbb R^m,V\in\mathbb R^n$ and functions $f:U\to\mathbb R^n,g:V\to\mathbb R^m$ and $f(U)\subset V$.
    If $f$ is differentiable at $a$ and $g$ is differentiable at $f(a)$, then $g\circ f$ is differentiable at $a$ and $(g\circ f)^\prime(a)=g^\prime(f(a))\circ f^\prime(a)$.
\end{proposition}
\begin{proof}
    Let $b=f(a)$ and $S=f^\prime(a),T=g^\prime(b)$, then
    $$
    \begin{cases}
        f(a+h)=f(a)+S(h)+\epsilon(h)\|h\|\\
        g(b+k)=g(b)+T(k)+\delta(k)\|k\|
    \end{cases}
    $$
    Where $\epsilon(0)=0,\delta(0)=0$ and both of them are continuous at $0$.
    So
    $$(g\circ f)(a)=g(b+S(h)+\epsilon(h)\|h\|)$$
    Let $k(h)=S(h)+\epsilon(h)\|h\|$, so it equals
    \begin{align*}
        g(b)+T(k)+\delta(k)\|k\|&=(g\circ f)(a)+T\circ S(h)\\
        &+\|h\|T(\delta(h))+\delta(k(h))\|S(h)+\delta(h)\|h\|\|
    \end{align*}
    Due to continuity of $\epsilon,\delta$ are continuous at $0$, $\|h\|T(\delta(h))=o(\|h\|)$ and $T(\epsilon(0))=0$, so this term is fine.\\
    Also, $\delta(k(0))=0$ and $\delta\circ k$ is continuous at $0$.
    In addition,
    $$0\le \frac{\|S(h)+\delta(h)\|h\|\|}{\|h\|}\le\frac{\|S(h)\|+\|\epsilon(h)\|\|h\|}{\|h\|}\le \|S\|+\|h\|$$
    by Lemma \ref{mat_norm}.
    So
    $$\lim_{h\to 0}\frac{\delta(k(h))\|S(h)+\delta(h)\|h\|\|}{\|h\|}=0\implies \delta(k(h))\|S(h)+\delta(h)\|h\|\|=o(\|h\|)$$
    Hence $g\circ f$ is differentiable and its derivative is $T\circ S=g^\prime (b)\circ f^\prime(a)=g^\prime (f(a))\circ f^\prime(a)$.
\end{proof}
\begin{proposition}\label{component_diff}
    $f:U\to\mathbb R^n$ ($U\in\mathbb R^m$ is open) is differentiable if and only if each components $f_j=\pi_j\circ f$ is differentiable at $a$.
    Also,
    $$f^\prime(a)(h)=\sum_{j=1}^nf_j^\prime(a)(h)e_j'$$
\end{proposition}
\begin{proof}
    Note that $\pi_j(x)=\langle x,e_j'\rangle$ is linear hence differentiable, thus by chain rule the $\implies$ direction is done.
    For $\impliedby$, we have for every $j$,
    $$f_j(a+h)=f_j(a)+f^\prime(a)(h)+\epsilon_j(h)\|h\|$$
    So
    \begin{align*}
        f(a+h)&=\sum_{j=1}^nf_j(a+h)e_j'\\
        &=\sum_{j=1}^n(f_j(a)+f^\prime(a)(h)+\epsilon_j(h)\|h\|)e_j'\\
        &=f(a)+\left( \sum_{j=1}^nf_j^\prime(a)(h)e_j' \right)+\left( \sum_{j=1}^n\epsilon_j(h)e_j' \right)\|h\|
    \end{align*}
    Since $\epsilon(h)=\sum_{j=1}^n\epsilon_j(h)e_j'$ has $\epsilon(0)=0$ and is continuous at $0$, hence the result.
\end{proof}
\begin{proposition}
    Let $f,g:U\to\mathbb R^n$ where $U\subset\mathbb R^m$ is open and $\phi:U\to\mathbb R$ is differentiable at $a\in U$, then so are $f+g$ and $\phi f:x\mapsto \phi(x)f(x)$, and
    $$(f+g)^\prime(a)=f^\prime(a)+g^\prime(a)$$
    $$(\phi f)^\prime(a)(h)=\phi^\prime(a)(h)f(a)+\phi(a)f^\prime(a)(h)$$
\end{proposition}
\begin{proof}
    We have
    $$f(a+h)=f(a)+f^\prime(a)(h)+\epsilon(h)\|h\|$$
    $$g(a+h)=g(a)+g^\prime(a)(h)+\delta(h)\|h\|$$
    $$\phi(a+h)=\phi(a)+\phi^\prime(a)(h)+\eta(h)\|h\|$$
    Hence
    $$(f+g)(a+h)=(f+g)(a)+(f^\prime(a)+g^\prime(a))(h)+(\epsilon(h)+\delta(h))\|h\|$$
    We can do the same thing for products as well which will provide a proof, but we shall give a different proof.
    Let $F:U\to R\times R^n=R^{n+1}$ by $f(x)=(\phi(x),f(x))$ and $G:R\times R^n\to\mathbb R^n$ by $(a,x)\mapsto ax$.
    $F$ is differentiable by Proposition \ref{component_diff} and $G$ is differentiable since it is bilinear, therefore $\phi f=G\circ F$ is differentiable and we can obtain the form of the derivative from the chain rule which is the formula as claimed.
\end{proof}
\begin{definition}
    Let $U\subset\mathbb R^m$ be open and $f:U\to\mathbb R^n$.
    Fix $a\in U$ and a direction (nonzero vector) $u\in\mathbb R^m\setminus\{0\}$.
    The limit
    $$\lim_{t\to 0}\frac{f(a+tu)-f(a)}{t}$$
    if exists, is called the directional derivative of $f$ at $a$ to direction $u$ and is denoted by $D_uf(a)$.
\end{definition}
\begin{remark}
    1. $f(a+tu)=f(a)+tD_uf(a)+o(t)$.\\
    2. Let $\gamma(t)=a+tu$, then $(f\circ\gamma)^\prime(0)=D_uf(0)$.
\end{remark}
In the special case where $u=e_i$, we write $D_if(a)$ to denote $D_{e_i}f(a)$ and it is called the $i^{th}$ partial derivative of $f$ at $a$.
\begin{proposition}
    If $f$ is differentiable at $a$, then all $D_uf(a)$ exists and we have $D_uf(a)=f^\prime(a)(u)$, so for $h=\sum_ih_ie_i$, we have
    $$f^\prime(a)(h)=\sum_ih_iD_if(a)$$
\end{proposition}
\begin{proof}
    We have
    $$f(a+h)=f(a)+f^\prime(a)(h)+\epsilon(h)\|h\|$$
    Then
    $$\frac{f(a+tu)-f(a)}{t}=f^\prime(a)(u)+\epsilon(tu)\frac{\|t\|}{t}\to f^\prime(a)(u)$$
    As $t\to 0$.
    The rest follows.
\end{proof}
\begin{remark}
    1. Assume $f$ is differentiable at $a$, then the matrix of $f^\prime(a)$ is exactly represented by $(f^\prime(a))_{ji}=D_if_j(a)=(\partial f_j/\partial x_i)(a)$.
    This is called the Jacobian of $f$ at $a$, denoted by $Jf(a)$.\\
    2. If all partial derivatives exists, so does $D_uf_j(a),\forall j$, and we have $D_uf_j(a)=\pi_j(D_uf(a))$
    So $D_u\pi_j=\pi_jD_u$.\\
    3. The converse of the proposition fails in general.
\end{remark}
\begin{theorem}
    If all partial derivatives of $f$ exists at $a\in U$.
    Assume $\exists r>0,D_r(a)\in U$ and $D_if$ exists in $D_r(a)$ and is continuous at $a$ for all $i$, then $f$ is differentiable at $a$.
\end{theorem}
\begin{proof}
    WLOG $n=1$ by Proposition \ref{component_diff} and the second remark above.
    We shall prove the case for $m=2$, and the general case is similar.
    Let $a=(a_1,a_2)$ and consider $h=(h_1,h_2)\in D_r(0)$.
    Certainly we want the derivative to equal $h_1D_1f(a_1,a_2)+h_2D_2f(a_1,a_2)$, so we will try to prove
    $$f(a_1+h_1,a_2+h_2)-f(a_1,a_2)-h_1D_1f(a_1,a_2)-h_2D_2f(a_1,a_2)=o(\|h\|)$$
    Note that we can write it out in two parts
    \begin{align*}
        &f(a_1+h_1,a_2+h_2)-f(a_1+h_1,a_2)-h_2D_2f(a_1,a_2)\\
        &+f(a_1+h_1,a_2)-f(a_1,a_2)-h_1D_1f(a_1,a_2)
    \end{align*}
    We have $f(a_1+h_1,a_2)-f(a_1,a_2)-h_1D_1f(a_1,a_2)=o(h_1)=o(\|h\|)$ as $h\to 0$.
    As for the first part, let $\phi(t)=f(a_1+h_1,a_2+t)$ for $t\in [-|h_2|,|h_2|]$, so we have
    $$f(a_1+h_1,a_2+h_2)-f(a_1+h_1,a_2)-h_2D_2f(a_1,a_2)=\phi(h_2)-\phi(0)-h_2D_2f(a_1,a_2)$$
    Note that $\phi$ is continuous and is differentiable in $(-|h_2|,|h_2|)$.
    Indeed we have $\phi^\prime(t)=D_2f(a_1+h_2,a_2+t)$.
    By MVT, there is some $\theta(h_1,h_2)\in (0,1)$ such that $\phi(h_2)-\phi(0)=\phi^\prime(\theta h_2)h_2$.
    Hence
    \begin{align*}
        \phi(h_2)-\phi(0)-h_2D_2f(a_1,a_2)&=h_2(D_2f(a_1+h_1,a_2+\theta h_2)-D_2f(a_1,a_2))\\
        &=o(h_2)=o(\|h\|)
    \end{align*}
    as $h\to 0$.
    So the theorem is proved.
\end{proof}
\begin{theorem}[Mean Value Inequality]\label{mean_val_ineq}
    Consider an open $U\subset \mathbb R^m$ and a function $f:U\to\mathbb R^n$.
    Assume $f$ is differentiable on $U$ and we are given $a,b\in U$ such that $[a,b]=\{(1-t)a+tb:t\in [0,1]\}\subset U$ and $\exists M>0$ such that $\forall z\in [a,b],\|f^\prime(z)\|\le M$, then
    $$\|f(b)-f(a)\|\le M\|b-a\|$$
\end{theorem}
\begin{proof}
    Let $v=f(b)-f(a)$.
    Consider $\phi:[0,1]\to\mathbb R$ defined by $\phi(t)=\langle f((1-t)a+tb),v\rangle$.
    Then $\phi(1)-\phi(0)=\|f(b)-f(a)\|^2$ and $\phi$ is differentiable with $\phi^\prime(t)=\langle f^\prime((1-t)a+tb)(b-a),v\rangle$.
    By MVT, $\exists\theta\in (0,1)$ with $\phi(1)-\phi(0)=\phi^\prime(\theta)$, so we have
    \begin{align*}
        \|f(b)-f(a)\|^2&=\langle f^\prime((1-\theta)a+\theta b)(b-a),v\rangle\\
        &\le \|f^\prime((1-\theta)a+\theta b)(b-a)\|\|v\|\\
        &\le \|f^\prime((1-\theta)a+\theta b)\|\|b-a\|\|v\|\\
        &\le M\|b-a\|\|f(b)-f(a)\|
    \end{align*}
    The theorem follows.
\end{proof}
\begin{corollary}
    Let $U\subset\mathbb R^m$ be open and connected, and $f:U\to\mathbb R^n$ be differentiable such that $f^\prime\equiv 0$, then $f$ is constant.
\end{corollary}
\begin{proof}
    We can show that it is locally constant by the preceding theorem, thus it is constant everywhere by connectedness.
\end{proof}
\begin{remark}
    Suppose we have open $U\subset\mathbb R^m,V\subset\mathbb R^n$ and $f:U\to V$ is a bijection such that $f$ is differentiable at $a\in U$ and $f^{-1}$ at $f(a)\in V$.
    Let $S=f^\prime(a), T=(f^{-1})^\prime(f(a))$, then $ST=I_n$ and $TS=I_m$.
    $\operatorname{rank}(I_n)=\operatorname{rank}(ST)=\operatorname{rank}(TS)=\operatorname{rank}(I_m)$, hence $n=m$.
\end{remark}
\begin{theorem}[Inverse Function Theorem]
    We have an open set $U\subset\mathbb R^n$, a $C^1$ (continuously differentiable) function $f:U\to\mathbb R^n$ and a point $a\in U$ such that $f^\prime(a)$ is invertible, then there exists open set $V\subset U,W\subset\mathbb R^n$ open such that $a\in V,f(a)\in W,f|_{V}:V\to W$ is a bijection with a $C^1$ inverse $g:W\to V$ and $\forall y\in W,g^\prime(y)=(f^\prime(g(y)))^{-1}$.
\end{theorem}
\begin{proof}
    Step 1: WLOG $a=f(a)=0$ and $f^\prime(a)=I$.
    We can do this because we can consider $h:U-a=\{x-a:x\in U\}\to\mathbb R^n$ by $h(x)=(f^\prime(a))^{-1}(f(x+a)-f(a))$.
    Now we can fix $r>0$ such that $D_r(0)\subset U$ and $\forall x\in D_r(0),\|f^\prime(x)-I\|\le 1/2$ by continuity.\\
    Step 2: $\forall x,y\in D_r(0)$, $\|f(x)-f(y)\|\ge \|x-y\|/2$, so $f$ is injective.
    To prove this, consider $h(x)=x-f(x)$, then $h^\prime=I-f^\prime(x)$, so $\forall x\in D_r(0),\|h^\prime(x)\|\le 1/2$.
    By Theorem \ref{mean_val_ineq}, $\|h(x)-h(y)\|\le \|x-y\|/2$, so
    $$\frac{\|x-y\|}{2}\ge \|h(x)-h(y)\|\ge \|x-y\|-\|f(x)-f(y)\|$$
    Step 3: For $0\le s\le r/2$, $D_s(0)\subset f(B_{2s}(0))\subset f(D_r(0))$.
    Fix $y\in D_s(0)$, then consider $h:B_{2s}(0)\to\mathbb R^n$ by $x\mapsto y-f(x)+x$.
    We have $h^\prime(x)=-f^\prime(x)+I$, so $\forall x\in B_{2s}(0),\|h^\prime(x)\|\le 1/2$, so by Theorem \ref{mean_val_ineq}, $h$ is $1/2$-Lipschitz.
    Note also that for $x\in B_{2r}(0)$, then $\|h(x)\|=\|h(x)-h(0)+y\|\le \|x\|/2+\|y\|\le 2s$, so by Theorem \ref{banach} there is some $x\in B_{2r}(0)$ such that $h(x)=x$, which means that $y=f(x)$.\\
    Step 4: Fix $0<s<r/2$, then let $W=D_s(0)$ and $V=f^{-1}(D_s(0))\cap D_r(0)$, so $V$ is open and $f(V)=W$ by step 3 and $f$ is injective by step 2, so $f|_V:V\to W$ is a bijection.
    Let $f^{-1}=g:W\to V$ be the inverse, then given $a,b\in W$, let $x=g(a),y=g(b)$, then $x,y\in V\subset D_r(0)$, so by step 2, $\|f(x)-f(y)\|\ge 1/2\|x-y\|$, so $g$ is $1/2$-Lipschitz hence continuous.
    Note that if $g$ is differentiable then we have $I=(f\circ g)^\prime(y)$ for any $y$, hence by Chain Rule, $g^\prime(y)=(f^\prime(g(y)))^{-1}$.
    So we want to show that $g$ has this as derivative.
    Indeed, fix $b\in W$ and $a=g(b),T=f^\prime(a)$, we have $f(a+h)=f(a)+T(h)+\epsilon(h)\|h\|$.
    Fix $\delta>0$ such that $D_\delta(b)\in W$ and $k\in D_\delta(0)$, by setting $h=h(k)=g(b+k)-g(b)$ we have $k=f(a+h)-f(a)=T(h)+\epsilon(h)\|h\|$, so $h=T^{-1}(k)-T^{-1}(\epsilon(h))\|h\|$, hence
    $$g(b+k)=g(b)+h=g(b)+T^{-1}(k)-T^{-1}(\epsilon(h))\|h\|=g(b)+T^{-1}(k)+o(\|k\|)$$
    Hence $g$ is differentiable at $b$ with derivative $T^{-1}=(f^\prime(g(b)))^{-1}$ which is continuous.
\end{proof}
\begin{definition}
    We have open set $U\subset\mathbb R^m$, then a function $f:U\to\mathbb R^n,a\in U$ have $f$ twice differentiable at $a$ if there is some open $V$ with $a\in V\subset U$ such that $f$ is differentiable in $V$ and the derivative $f^\prime:V\to L(\mathbb R^m,\mathbb R^n)$ is differentiable.
    $f^{\prime\prime}(a)=(f^\prime)^\prime(a)$ is called the second derivative of $f$.
\end{definition}
So we have $f^{\prime\prime}\in L(\mathbb R^m,L(\mathbb R^m,\mathbb R^n))$ where we have
$$f^\prime(a+h)=f^\prime(a)+f^{\prime\prime}(a)(h)+\epsilon(h)\|h\|$$
where $\epsilon\to 0$ as $h\to 0$.
Note that $\epsilon(h)\in L(\mathbb R^m,\mathbb R^n)$.
So $f^\prime(a+h)(k)=f^\prime(a)(k)+f^{\prime\prime}(a)(h)(k)+\epsilon(h)(k)\|h\|$
for each fixed $k\in\mathbb R^m$.
Note also that $L(\mathbb R^m,L(\mathbb R^m,\mathbb R^n))\cong \operatorname{Bil}(\mathbb R^m\times\mathbb R^m,\mathbb R^n)$ by the correspondence $T\mapsto B$ with $B(h,k)=T(h)(k)$.
So we can think of the second derivative at $a$ as a bilinear map $\mathbb R^m\times\mathbb R^m\to\mathbb R^n$.\\
In summary, $f$ is twice differentiable at $a$ if and only if there is a bilinear map $B:\mathbb R^m\times\mathbb R^m\to\mathbb R^n$ such that for any $k\in\mathbb R^m$ we have
$$f^\prime(a+h)(k)=f^\prime(a)(h)+B(h,k)+o(\|h\|)$$
$B$ here is then the second derivative.
\begin{example}
    For $f:M_n\to M_n$ by $A\mapsto A^3$.
    It is differentiable and $f^\prime(A)(H)=HA^2+AHA+A^2H$.
    Then to find second derivative by
    \begin{align*}
        f^\prime(A+H)(K)&=K(A+H)^2+(A+H)K(A+H)+(A+H)^2K\\
        &=f^\prime(A)(K)\\
        &+KAH+KHA+AKH+HKA+AHK+HAK\\
        &+o(\|H\|^2)
    \end{align*}
    So the seond derivative is the bilinear map $f^{\prime\prime}(A)=B(H,K)=KAH+KHA+AKH+HKA+AHK+HAK$.
\end{example}
Assume that $f$ has second derivative at $a$ under the usual setup, then
$$f^\prime(a+h)(k)=f^\prime(a)(k)+f^{\prime\prime}(a)(h,k)+o(\|h\|)$$
So fix $u,v\in\mathbb R^m\setminus\{0\}$, by putting $k=v$ we have
$$D_vf(x+h)=D_vf(a)+f^{\prime\prime}(h,v)+o(\|h\|)$$
So $D_vf$ is differentiable, therefore we can write
$$D_uD_vf(a)=f^{\prime\prime}(a)(u,v)$$
\begin{theorem}
    Let $U\subset\mathbb R^m$ be open and $f:U\to\mathbb R^n$ be second differentiable on $U$ with $f^{\prime\prime}$ continuous at $a$ for some $a\in U$, then $f^{\prime\prime}(a)$ is a symmetric form, that is, for any $0\neq u,v\in\mathbb R^m$, $D_uD_vf(a)=D_vD_uf(a)$.
\end{theorem}
\begin{proof}
    WLOG $n=1$ since $(f_j)^{\prime\prime}=f^{\prime\prime}_j$.
    Define
    \begin{align*}
        \phi(s,t)&=f(a+su+tv)-f(a+su)-(f(a+tv)-f(a))\\
        &=f(a+su+tv)-f(a+tv)-(f(a+su)-f(a))
    \end{align*}
    Fix $s,t$, consider $\Psi(x)=f(a+xu+tv)-f(a+xu)$, so $\phi(s,t)=\Psi(s)-\Psi(0)=s\Psi^\prime(\alpha s)$ where $\alpha=\alpha(s,t)\in (0,1)$.
    So $\phi(s,t)=(D_uf(a+\alpha su+tv)-D_uf(a+\alpha su))s$
    Consider $\psi(y)=D_uf(a+\alpha su+yv)$, so $\phi(s,t)=s(\psi(t)-\psi(0))=st\psi^\prime(\beta t),\beta=\beta(s,t)\in (0,1)$.
    So
    \begin{align*}
        \frac{\phi(s,t)}{st}&=D_vD_uf(a+\alpha su+\beta tv)\\
        &=f^{\prime\prime}(a+\alpha su+\beta tv)(v,u)\to f^{\prime\prime}(a)(v,u)
    \end{align*}
    Repeat the process in the other order to get
    $$\frac{\phi(s,t)}{st}\to f^{\prime\prime}(a)(u,v)$$
    So they are equal.
\end{proof}
$U\subset\mathbb R^m$ open, $f:U\to\mathbb\mathbb R$, we say $f$ has a local maximum at $a\in U$ if $\exists r>0,\forall b\in D_r(a),f(b)\le f(a)$.
Similarly we can define local minimums.
\begin{definition}
    We say $f$ has a stationary point at $a$ if $f^\prime(a)=0$.
\end{definition}
It is immediate that $f$ has stationary points in local maxima/minima.
\begin{theorem}
    Let $U\subset\mathbb R^m$ be open and $f:U\to\mathbb R$ be twice differentiable in $U$ and $f^{\prime\prime}$ is continuous at $a$ and $f^\prime(a)=0$.
    Then if the symmetric form $f^{\prime\prime}$ is positive definite at $a$ then $f$ has a local minimum, and if it is negative definite then it has a local maximum.
\end{theorem}
\begin{proof}
    It is a non-examinable fact that
    $$f(a+h)=f(a)+f^\prime(a)(h)+1/2f^{\prime\prime}(a)(h,h)+\epsilon(h)o(\|h\|^2)$$
    Recall that $f^{\prime\prime}(a)$ is (real) diagonalizable.
    So there is a base $\{u_k\}$ such that
    $$f^{\prime\prime}(u_i,u_j)=\begin{cases}
        0\text{, if $i\neq j$}\\
        \lambda_i\text{, if $i=j$}
    \end{cases}$$
    Assume that $f^{\prime\prime}$ is positive definite, it means that $f^{\prime\prime}(a)(h,k)>0$ for any $(h,k)\neq (0,0)$.\\
    So $\lambda_i=f^{\prime\prime}(a)(u_i,u_i)>0$, hence $\mu=\min\{\lambda_i:1\le i\le m\}>0$.
    For $h\in\mathbb R^m$, we have
    $$f^{\prime\prime}(a)(h,h)=\sum_{i,j}h_ih_jf^{\prime\prime}(a)(u_i,u_j)=\sum_ih_i^2\lambda_i\ge\mu\|h\|^2$$
    Hence $f(a+h)-f(a)\ge \mu\|h\|^2/4\ge 0$, therefore $f$ has a local minimum at $a$.
    Similar for negative definite case.
\end{proof}


\subsection{Arithmetic}

\noindent
Normally Eigenmath uses integer and rational number arithmetic.

\begin{Verbatim}[formatcom=\color{blue}]
1/2 + 1/3
\end{Verbatim}

\noindent
$\displaystyle \tfrac{5}{6}$

\bigskip
\noindent
A floating point value causes Eigenmath to switch to floating point arithmetic.

\begin{Verbatim}[formatcom=\color{blue}]
1/2 + 1/3.0
\end{Verbatim}

\noindent
$\displaystyle 0.833333$

\bigskip
\noindent
An integer or rational number result can be converted to a floating
point value by entering {\it float}.

\begin{Verbatim}[formatcom=\color{blue}]
212^17
\end{Verbatim}

\noindent
$\displaystyle 3529471145760275132301897342055866171392$

\begin{Verbatim}[formatcom=\color{blue}]
float
\end{Verbatim}

\noindent
$\displaystyle 3.52947\times10^{39}$

\bigskip
\noindent
The following example shows how to enter a floating point value
using scientific notation.

\begin{Verbatim}[formatcom=\color{blue}]
epsilon = 1.0 10^(-6)
epsilon
\end{Verbatim}

\noindent
$\displaystyle \varepsilon=1.0\times10^{-6}$