\section{Deep finite neuron functions, adaptivity and spectral
  accuracy}
\label{sec:deep-fnm}
In this section, we will study deep finite neural functions through
the framework of  deep neural networks and then discuss its adaptive
and spectral accuracy properties.

\subsection{Deep finite neuron functions}
Given $d, \ell\in\mathbb{N}^+$,
$
n_1,\dots,n_{\ell}\in\mathbb{N} \mbox{ with }n_0=d, n_{\ell+1}=1,
$
\begin{equation}\label{thetamap}
\theta^i(x)=\omega_i\cdot x + b_i,\quad \omega_i\in \mathbb{R}^{n_{i+1}\times n_i},\ b\in \mathbb{R}^{n_{i+1}},
\end{equation}
and the activation function ${\rm ReLU}^k$, define
a  deep finite neuron function $u(x)$ from $\mathbb{R}^d$ to $\mathbb{R}$  as follows:
\begin{align*}
f^0(x)   &=\theta^0(x) \\
f^{i}(x) &= [  \theta^{i} \circ \sigma ](f^{i-1}(x)) \quad i = 1:\ell \\
f(x) &= f^\ell(x). \\
\end{align*}
The following more concise notation is often used in computer science literature:
\begin{equation}
\label{compress-dnn}
f(x) = \theta^{\ell}\circ \sigma \circ \theta^{\ell-1} \circ \sigma \cdots \circ \theta^1 \circ \sigma \circ \theta^0(x),
\end{equation}
here $\theta^i: \mathbb{R}^{n_{i}}\to\mathbb{R}^{n_{i+1}}$ are linear
functions as defined in \eqref{thetamap}.  Such a deep neutral network
has $(\ell+1)$-layer DNN, namely $\ell$-hidden layers. The size of
this deep neutral network is $n_1+\cdots+n_{\ell}$.

Based on these notation and connections, define deep finite neuron
functions with activation function $\sigma={\rm ReLU}^k$ by
\begin{equation}
\label{NNL}
\Sigma^k_{n_1,n_2,\ldots, n_\ell}=\bigg\{ f^{\ell}(x) = \theta^\ell (x^{\ell}),
 \mbox{ with } W^i\in \mathbb R^{n_{i+1}\times
	n_{i}}, b^i\in\mathbb R^{n_i}, i=0:\ell, n_0=d, n_{\ell+1}=1\bigg\}
\end{equation}
Generally, we can define the $\ell$-hidden layer neural network as:
\begin{equation}
\Sigma^k_{\ell,n}:= \bigcup_{n_1, n_2, \cdots, n_{\ell}\ge 1}\Sigma^k_{n_1,n_2,\ldots, n_\ell}.
\end{equation}
For $\ell=2$, functions in $\Sigma^k_{1,n}$ consist of piecewise polynomials of degree $k^2$ on a finite neuron grids whose boundaries are level sets of quadratic polynomials, see Fig \ref{fig:3}.
\begin{figure}[!ht]
\begin{center}
\includegraphics[width=.3\textwidth]{6DL/figures/2t20t20-1.jpg}
\includegraphics[width=.3\textwidth]{6DL/figures/2t40t40-1.jpg}
\includegraphics[width=.28\textwidth]{6DL/figures/2t60t60-1.jpg}
\caption{Hidden finite neuron grids with $\ell=2$}
\label{fig:3}
\end{center}
\end{figure}

\subsection{Reproduction of polynomials and spectral accuracy}
One interesting property of the ReLU$^k$-DNN is that it reproduces
polynomials of degree $k$.
\begin{lemma}
Given $k\ge 2$, $q\ge 2$, there exist $\ell\ge 1$, $n_1, \cdots, n_\ell$ such that
$$
\mathbb{P}_q\subset \Sigma^k_{\ell,n},
$$
where $\mathbb{P}_q$ is the set of all polynomials with degree not larger than $q$.
\end{lemma}
\begin{proof}
	Here we provide a sketch of the proof. We first notice that
	$$
	x^k = {\rm ReLU}^k(x) + (-1)^k{\rm ReLU}^k(-x) \in \Sigma^k_{2}, \quad \forall x \in \mathbb{R},
	$$
	for any $k\ge2$. In addition, there exists $a_i, b_i, c_i \in \mathbb{R}$ such that
	$$
	x^2 = \sum_{i=1}^{k+1} c_i(a_i x + b_i)^k  \in \Sigma^k_{2(k+1)}, \forall x \in \mathbb{R}.
	$$
	Thus, we have
	$$
	x\times y = \frac{(x+y)^2 - (x-y)^2}{4} \in \Sigma^k_{2, 4(k+1)}, \forall x,y \in \mathbb{R}.
	$$
	This shows that
	\begin{equation}\label{key}
	\begin{split}
	x_1^{\alpha_1}x_2^{\alpha_2}\cdots x_d^{\alpha_d}
	&= x_1 \times x_1^{\alpha_1-1}x_2^{\alpha_2}\cdots x_d^{\alpha_d} \\
	&\vdots \\
	&= x_1\times \left(x_1 \times \left(x_1 \cdots \times \left(x_d\times (x_d \times x_d)\right)\cdots\right)\right).
	\end{split}
	\end{equation}
	Thereby, any monomial $x_1^{\alpha_1}x_2^{\alpha_2}\cdots x_d^{\alpha_d}$ with degree $|\alpha| = \sum_{i=1}^d \alpha_i$ can be represented by a deep ${\rm ReLU}^k$ neural network.
\end{proof}
For a more detailed proof of the above result, we refer to \cite{li2019better}.

\begin{theorem}\label{thm:spectral}
Let  ${\rm ReLU}^k$ be the activation function, and $\dnn_\ell^k(N)$ be the DNN model with $\ell$ hidden layers.
There exists some $\ell$ such that
\begin{equation}\label{eq:spectral}
\inf_{v_n\in \Sigma^k_{\ell,n}}\|u-v_n \|_{H^m(\Omega)} \lesssim
\inf_{v_n\in \mathbb{P}_{k^\ell}} \|u-v_n\|_{H^m(\Omega)},
\end{equation}
\end{theorem}
Estimate \eqref{eq:spectral} indicates that the deep finite neuron function may provide spectral approximate accuracy.

\subsection{Reproduction of linear finite element functions and adaptivity}
The deep neural network with ReLU activation function have
been much studied in the literature and most widely used in practice.
One interesting fact is that ReLU-DNN is simply piecewise linear
functions.  More specifically, recall the results in Section \ref{sec:reluFEM}.

\begin{lemma}
Assume that ${\cal T}_h$ is a simplicial finite element grid of $N$ elements, in which any
union of simplexes that share a same vertex is convex, any linear finite element function on this grid
can be written as a ReLU-DNN with at most $\mathcal
  O(d)$ hidden layers. The number of neurons is at most
  $\mathcal{O}(\kappa^dN)$ for some constant $\kappa\ge 2$ depending
  on the shape-regularity of $\mathcal T_h$.  The number of non-zero
  parameters is at most $\mathcal{O} (d\kappa^dN)$.
\end{lemma}

The above result indicate that the deep finite neuron functions can
reproduce any linear finite element functions.  Given the adaptive
feature and capability of finite element methods, we see that the
finite neuron method can be at least as adaptive as finite element method.

\section{Elliptic boundary value problems of order $2m$}\label{sec:model}
Let $\Omega\subset \mathbb{R}^d$ be a bounded domain with a
sufficiently smooth boundary $\partial\Omega$.  For any integer $m\ge
1$, we consider the following model $2m$-th order partial differential
equation with certain boundary conditions:
%\begin{equation}\label{equ:2mpde}(-\Delta)^m u +u = f \qquad \mbox{in }\Omega.\end{equation}
\begin{equation} \label{2mPDE}
\left\{
  \begin{array}{rccl}\displaystyle
Lu &=& f &\mbox{in }\Omega, \\
B^k(u) &= &0 & \mbox{on }\partial\Omega \quad(0\le k\le m-1),
  \end{array}
\right.
\end{equation}
where $L$ is a partial differential operator as follows
\begin{equation}\label{Lu}
Lu= \sum_{|\alpha|=m}(-1)^m\partial^\alpha (a_\alpha(x)\,\partial^\alpha\,u) +a_0(x)u,
 \end{equation}
and ${\boldsymbol{\alpha}}$ denotes $n$-dimensional multi-index ${\boldsymbol{\alpha}}
= (\alpha_1, \cdots, \alpha_n)$ with
$$
|{\boldsymbol{\alpha}}| = \sum_{i=1}^n \alpha_i, \quad
\partial^{\boldsymbol{\alpha}} = \frac{\partial^{|{\boldsymbol{\alpha}}|}}{\partial x_1^{\alpha_1}
\cdots \partial x_n^{\alpha_n}}.
$$
For simplicity, we assume that $a_\alpha$ are strictly positive and smooth functions on
$\Omega$ for  $|\alpha|=m$ and $\alpha=0$, namely, $\exists \alpha_0>0$, such that
\begin{equation}\label{ass:1}
a_\alpha(x), a_0(x)\ge \alpha_0,\,\, \forall x\in\Omega,\,\,|\alpha|=m.
 \end{equation}
Given a nonnegative integer
$k$ and a bounded domain $\Omega\subset \mathbb{R}^d$, let
$$
H^k(\Omega):=\left\{v\in L^2(\Omega), \partial^\alpha v\in L^2(\Omega), |\alpha|\le k\right\}
$$
be standard Sobolev spaces with norm and seminorm given respectively by
$$
 \|v\|_k:=\left(\sum_{|\alpha|\le k} \|\partial^\alpha v\|_0^2\right)^{1/2}, \quad  |v|_k:=\left(\sum_{|\alpha|= k} \|\partial^\alpha v\|_0^2\right)^{1/2}.
$$
For $k=0$, $H^0(\Omega)$ is the standard $L^2(\Omega)$ space
with the inner product denoted by $(\cdot, \cdot)$.  Similarly, for
any subset $K\subset \Omega$, $L^2(K)$ inner product is denoted by
$(\cdot, \cdot)_{0,K}$.  We note that, by well-known property of Sobolev space,  the assumption \eqref{ass:1} implies that
\begin{equation}
  \label{avv}
a(v,v)\gtrsim \|v\|^2_{m,\Omega}, \forall v\in H^m(\Omega).
\end{equation}

The boundary value problem \eqref{2mPDE} can be cast into an
equivalent optimization or a variational problem as described below for some approximate subspace $V\subset H^m(\Omega)$.
\begin{description}
\item[Minimization Problem M:] Find $u\in V$ such that
\begin{equation}
\label{minJv}
J(u)=\min_{v\in V} J(v),
\end{equation}
\noindent or
\item[Variational Problem V:]  Find  $u\in V$ such that
\begin{equation}\label{m-vari}
a(u,v) = \langle f, v\rangle \quad \forall v \in V.
\end{equation}
\end{description}
The bilinear form $a(\cdot,\cdot)$ in \eqref{m-vari}, the objective
functional $J(\cdot)$ in \eqref{minJv} and the functional space $V$
depend on the type of boundary condition in \eqref{2mPDE}.

One popular type of boundary conditions are Dirichlet boundary
condition when $B^k=B_D^k$ are given by the following Dirichlet type
trace operators
\begin{equation}\label{BD}
B_D^k(u):=\left.\frac{\partial^k u}{\partial
    \nu^k}\right|_{\partial\Omega}\quad (0\le k\le m-1),
\end{equation}
with $\nu$ being the outward unit normal vector of $\partial\Omega$. The Dirichlet boundary
value problem as follows:
\begin{equation} \label{m-BD}
\left\{
  \begin{array}{rccl}\displaystyle
Lu &=& f &\mbox{in }\Omega, \\
B_{D}^k(u) &= &0 & \mbox{on }\partial\Omega \quad(0\le k\le m-1).
  \end{array}
\right.
\end{equation}

For the aforementioned Dirichlet boundary condition, the elliptic
boundary value problem \eqref{2mPDE} is equivalent to \eqref{minJv} or
\eqref{m-vari} with $V=H^m_0(\Omega)$ and
\begin{equation}
  \label{auv}
a(u,v) := \sum_{|\alpha | = m}(a_\alpha\partial^{\alpha}u, \partial^{\alpha}v)_{0,\Omega} +(a_0u,v)\quad \forall
u, v \in V,
\end{equation}
and
\begin{equation}
  \label{Jv}
J(v)=\frac12 a(v,v) -\int_{\Omega} fv dx.
\end{equation}

We next discuss about the pure Neumann boundary conditions
for general PDE operator \eqref{Lu} when $m\ge 2$.  We first begin our discussion
with the following simple result.
\begin{lemma} \label{lem:BDBNdual}
For each $k=0,1,\ldots,m-1$, there exists a bounded linear differential operator of order $2m-k-1$:
\begin{equation}
    \label{BN}
B_N^k: H^{2m}(\Omega)\mapsto L^2(\partial\Omega)
  \end{equation}
such that the following identity holds
$$
(Lu,v)=a(u,v)-\sum_{k=0}^{m-1}\langle B_N^k(u),B_D^k(v)\rangle _{0,\partial\Omega}.
$$
Namely
\begin{equation}
\label{BDBNdual}
\sum_{|\alpha|=m}(-1)^m\left(\partial^\alpha
  (a_\alpha\,\partial^\alpha\,u),\,v\right) _{0,\Omega}
=\sum_{|{\boldsymbol{\alpha}}|=m}\left(a_\alpha\partial^{\boldsymbol{\alpha}}\,u, \partial^{\boldsymbol{\alpha}} v\right) _{0,\Omega}
-\sum_{k=0}^{m-1}\langle B_N^k(u),B_D^k(v)\rangle _{0,\partial\Omega}
\end{equation}
for all $u\in H^{2m}(\Omega), v\in  H^{m}(\Omega)$. Furthermore,
\begin{equation}\label{equ:regassum}
\sum_{k=0}^{m-1}\|B_D^k(u)\|_{L^2(\partial\Omega)}+\sum_{k=0}^{m-1}\|B_N^k(u)\|_{L^2(\partial\Omega)}\lesssim \|u\|_{2m, \Omega}.
\end{equation}
\end{lemma}
Lemma \ref{lem:BDBNdual} can be proved by induction with
respect to $m$.  We refer to \cite{lions2012non} (Chapter 2) and \cite{chen2020nonconforming} for a proof on a similar identity.

In general the explicit expression of $B_N^k$ can be quite
complicated.  Let us get some idea by looking at some simple examples
with the following special operator:
\begin{equation}
  \label{Delta-m}
Lu=(-\Delta)^m u+u,
\end{equation}
and
\begin{equation}
a(u,v)= \sum_{|\alpha | = m}(a_\alpha\partial^{\alpha}u, \partial^{\alpha}v)_{0,\Omega} +(a_0u,v)\quad \forall
u, v \in V.
\end{equation}

\begin{itemize}
\item For $m=1$, it is easy to see that $B_N^0 u=\frac{\partial u}{\partial
    \nu}|_{\partial\Omega}$.
\item For $m=2$ and $d=2$, see \cite{chien1980variational}:
$$
B_N^0 u= \frac{\partial}{\partial \nu}\left(\Delta
  u+\frac{\partial^2u}{\partial
    \tau^2}\right)-\frac{\partial}{\partial
  \tau}\left({\kappa_\tau}\frac{\partial u}{\partial \tau}\right)|_{\partial\Omega}~~\hbox{and}~~B_N^1 u=\frac{\partial^2u}{\partial \nu^2}|_{\partial\Omega},
$$
   with $\tau$ being the
anti-clockwise unit tangential vector, and $\kappa_\tau$ the curvature
of $\partial\Omega$.
\end{itemize}

We are now in a position to state that the pure Neumann boundary value
problems for PDE operator \eqref{Lu} as follows.
\begin{equation} \label{m-BN}
\left\{
  \begin{array}{rccl}
Lu &=& f &\mbox{in }\Omega, \\
B_{N}^k(u) &= &0 & \mbox{on }\partial\Omega \quad(0\le k\le m-1).
  \end{array}
\right.
\end{equation}
Combining the trace theorem for $H^m(\Omega)$, see \cite{adams2003sobolev}, and Lemma
\eqref{lem:BDBNdual},  it is easy to see that
\eqref{minJv} is equivalent to
\eqref{m-BN}
with $V=H^m(\Omega)$.

For a given parameter $\delta>0$, we next consider the following
problem with mixed boundary condition:
\begin{equation} \label{equ:delta}
\left\{
\begin{aligned}
Lu_{\delta} &= f \qquad \mbox{in }\Omega, \\
B_D^k(u_{\delta})+\delta B_N^k(u_\delta)  &= 0, \ \ 0\le k\le m-1.
\end{aligned}
\right.
\end{equation}
It is easy to see that \eqref{equ:delta} is equivalent to the following problem: Find $u_\delta\in H^m(\Omega)$, such that
\begin{equation}\label{equ:varpdelta}
J_{\delta}(u_{\delta})=\min_{v\in H^m(\Omega)} J_{\delta}(v).
\end{equation}
where
\begin{equation}\label{J-delta}
J_\delta(v)={1\over 2}a_\delta(v,v)-(f,v)
\end{equation}
and
\begin{equation}\label{a-delta}
a_\delta(u,v)=a(u,v)+\delta^{-1}\sum_{k=0}^{m-1}\langle B_D^k(u), B_D^k(v)\rangle_{0,\partial\Omega}.
\end{equation}
In summary, we have
\begin{lemma}
The following equivalences hold:
\begin{enumerate}
\item $u$ solves for \eqref{m-BD} or \eqref{m-BN} if and only if $u$ solves
\begin{equation}\label{m-mini}
J(u)=\min_{v\in V} J(v)
\end{equation}
 with $V=H^m_0(\Omega)$ or  $V=H^m(\Omega)$,
\item $u_\delta$ solves for \eqref{equ:delta}  if and only if $u_\delta$ solves
$$
\displaystyle J_\delta(u_\delta)=\min_{v\in V} J_\delta(v)
$$
with $V=H^m(\Omega)$.
\end{enumerate}
\end{lemma}

\begin{lemma} \label{lem:JvJu}
Assume that $u\in V$ be solution of \eqref{m-BD} or \eqref{m-BN} and $u_\delta\in V$ be the solution of \eqref{equ:varpdelta}, then
the following identities hold:
\begin{equation}\label{vuidentity1}
\|v-u\|_{a}^2=J(v)-J(u)\quad \forall v\in V,
\end{equation}
and
\begin{equation}\label{vuidentity2}
\|v-u_\delta\|_{a,\delta}^2=J_{\delta}(v)-J_{\delta}(u_{\delta})\quad \forall v\in V.
\end{equation}
Here
\begin{equation}
  \label{a-norm}
\|v\|_{a}^2=a(v,v),\quad \|v\|_{a,\delta}^2=a_\delta(v,v).
\end{equation}
\end{lemma}
\begin{proof}
Let $u$ be the solution of \eqref{minJv}. Given $v\in V$, consider the quadratic function of $t$:
$$
g(t)=J(u+t(v-u)).
$$
It is easy to see that
$$
0=\arg\min_{t}g(t), \quad g'(0)=0,
$$
and
$$
J(v)-J(u)=g(1)-g(0)=g'(0)+{1\over2}g''(0)=\|v-u\|_{a}^2.
$$
This completes the proof of \eqref{vuidentity1}. The proof of \eqref{vuidentity2} is similar.
\end{proof}

\begin{lemma}\label{Nitchtrick}
  Let $u$ be the solution of \eqref{m-BD} and $u_\delta$ be the solution of \eqref{equ:delta}. Then
  \begin{align}\label{diff:uudelta}
\|u-u_\delta\|_{a,\delta} \lesssim\sqrt{\delta}  \|u\|_{2m,\Omega}.
\end{align}
\end{lemma}
\begin{proof}
Let $w=u-u_{\delta}$ and we have

\begin{equation} \label{equ:diff}
\left\{
\begin{aligned}
Lw &= 0 \qquad \mbox{in }\Omega, \\
B_D^k(w)+\delta B_N^k(w)  &= \delta B_N^k(u),\ \ 0\le k\le m-1.
\end{aligned}
\right.
\end{equation}
By Lemma \ref{lem:BDBNdual}, and \eqref{equ:diff},  we have
\begin{align}
0&=(Lw, w)
\\
&=\sum_{|\alpha|=m}(a_\alpha\partial^\alpha w, \partial^\alpha w)-\sum_{k=0}^{m-1}\int_{\partial \Omega}B_N^k(w)B_D^k(w) ds+(a_0w, w)
\\
&=\sum_{|\alpha|=m}(a_\alpha\partial^\alpha w, \partial^\alpha w)+ \sum_{k=0}^{m-1} \int_{\partial \Omega}(\delta^{-1}B_D^k(w)-  B_N^k(u) )B_D^k(w) ds+(a_0w, w),
\end{align}
implying
\begin{align}
&a(w, w)+ \delta^{-1}\sum_{k=0}^{m-1}\int_{\partial \Omega}B_D^k(w)^2 ds
=
\sum_{k=0}^{m-1}\int_{\partial \Omega}B_N^k(u)B_D^k(w) ds.
\end{align}
By Cauchy inequality, we have
\begin{align}
a(w,w)+\delta^{-1}\sum_{k=0}^{m-1}\|B_D^k(w)\|^2_{L^2(\partial \Omega)}
\le& \sum_{k=0}^{m-1}\|B_N^k(u) \|_{L^2(\partial \Omega)}\|B_D^k(w)\|_{L^2(\partial \Omega)}
\\
\le& 2\delta  \sum_{k=0}^{m-1}\|B_N^k(u) \|^2_{L^2(\partial \Omega)}+\frac12 \delta^{-1} \sum_{k=0}^{m-1} \|B_D^k(w)\|^2_{L^2(\partial \Omega)},
\end{align}
which implies
\begin{align}
a(w,w)+\frac12 \delta^{-1}\sum_{k=0}^{m-1}\|B_D^k(w)\|^2_{L^2(\partial \Omega)}\le 2\delta  \sum_{k=0}^{m-1}\|B_N^k(u) \|^2_{L^2(\partial \Omega)}.
\end{align}
By the definition of $\|\cdot\|_{a,\delta}$ and noting that $w=u-u_\delta$, we have
\begin{align}\label{diff:uudelta1}
\|u-u_\delta\|_{a,\delta}^2\le 4\delta  \sum_{k=0}^{m-1}\|B_N^k(u) \|^2_{L^2(\partial \Omega)}.
\end{align}
Combing this with \eqref{equ:regassum}, then completes the proof.
\end{proof}

\begin{lemma}\label{Regularity}
For any $s\ge -m$ and  $f\in H^s(\Omega)$, the solution $u$
  of \eqref{m-BD} or \eqref{m-BN} satisfies $u\in H^{2m+s}(\Omega)$ and
  \begin{equation}
    \label{regularity}
\|u\|_{2m+s,\Omega}\lesssim     \|f\|_{s,\Omega}.
  \end{equation}
\end{lemma}
We refer to \cite{lions2012non} (Chapter 2, Theorem 5.1 therein) for a detailed proof.

Following from \eqref{regularity} and \eqref{embend}, we have
\begin{lemma} For any $s\ge -m$,  $\epsilon >0$,
 and  $f\in H^s(\Omega)$, the solution $u$
  of \eqref{m-BD} or \eqref{m-BN} satisfies
  \begin{equation}
  \label{Barron-regularity}
\|u\|_{B^{m+1}(\Omega)}\le   \|f\|_{-m+\frac{d}{2}+1+\epsilon,\Omega}.
\end{equation}
\end{lemma}

\section{The finite neuron method and error estimates}\label{sec:FNM}
Let $V_n\subset V$ be a subset of $V$ defined by \eqref{VkN} which may not be a linear
subspace. Consider the the discrete problem of \eqref{m-mini}:
\begin{equation}
  \label{m-mini-VN}
\mbox{Find $u_n\in V_n$ such that } J(u_n)=\min_{v_n\in V_n}J(v_n).
\end{equation}
It is easy to see that the solution to \eqref{m-mini-VN} always exists (for deep
neural network functions as defined below), but may not be unique.

\begin{theorem}\label{thm:best} Let $u\in V$ and $u_n\in V_n$ be solutions to
  \eqref{m-BN} and \eqref{m-mini-VN} respectively.   Then
  \begin{equation}
    \label{best-approx}
\|u-u_n\|_a=\inf_{v_n\in V_n}\|u-v_n\|_a.
  \end{equation}
\end{theorem}
\begin{proof}
By  Lemma~\ref{lem:JvJu}, we have
$$
\|u_n-u\|_a^2=J(u_n)-J(u)\le J(v_n)-J(u)=\|v_n-u\|_a^2,\quad\forall v\in V_n.
$$
The proof is completed.
\end{proof}
We obtain the following result.
\begin{theorem}
Let $u\in V$ and $u_n\in V_n$ be solutions to
\eqref{m-BN} and \eqref{m-mini-VN} respectively.  Then for arbitrary $\epsilon>0$, we have
\begin{equation}\label{error:N}
\|u-u_n\|_a \lesssim
(\|f\|_{L^2(\Omega)}+\|f\|_{\mathcal{K}_1(\mathbb{D})})
\begin{cases}
n^{-{1\over 2}-{2(k-m)+1\over 2d}}&m< k,
\\
n^{m-k-1}& m\ge k.
\end{cases}
\end{equation}
\end{theorem}
By \eqref{best-approx},  and the embedding of spectral Barron space  into Sobolev space, namely Lemma \ref{smoothness-lemma}, the regularity result \eqref{regularity}, we get the proof.

Next we consider the discrete problem of \eqref{equ:varpdelta}:
\begin{equation}
\label{min:uN}
\mbox{Find $u_n\in V_n$ such that } J_\delta(u_n)=\min_{v_n\in V_n}J_\delta(v_n).
\end{equation}

\begin{lemma}
For any given number $\delta$, let $u_\delta$ be the solution of \eqref{equ:delta} and
  $u_n$ be the solution of \eqref{min:uN}, respectively. We have
  \begin{equation}
  \label{eq:1}
\|u_n-u_\delta\|_{a,\delta} \lesssim (1+ \delta^{-\frac12})  \inf_{v_n\in
  V_n} \|v_n-u_\delta\|_{m,\Omega}.
  \end{equation}
\end{lemma}
\begin{proof}
First of all, by Lemma~\ref{lem:JvJu} and the variational property, it holds that
\begin{align}\label{eq:min}
\|u_n-u_\delta\|_{a,\delta}^2=J_{\delta}(u_n)-J_{\delta}(u_{\delta})\le J_{\delta}(v_n)-J_{\delta}(u_{\delta})
=\|v_n-u_\delta\|_{a,\delta}^2, \quad \forall\,v_n\in V_n.
\end{align}
Further, for any $v_n\in V_n$, by the definition of $\|\cdot\|_{a,\delta}$ and trace inequality, we have
$$
\|v_n-u_\delta\|_{a,\delta}\lesssim  \|v_n-u_\delta\|_{m,\Omega}+\delta^{-\frac12} \|v_n-u_\delta\|_{0,\partial\Omega}\lesssim (1+\delta^{-\frac12})\|v_n-u_\delta\|_{m,\Omega}.
$$
This completes the proof.
\end{proof}

\begin{theorem}\label{thm:bestD}
  Let $u$ be the solution of \eqref{m-BD} and
  $u_n$ be the solution of \eqref{min:uN}, respectively. We have
  \begin{equation}
  \label{eq:13}
\|u-u_n\|_{a,\delta} \lesssim (1+ \delta^{-\frac12})  \inf_{v_n\in
  V_n} \|v_n-u\|_{m,\Omega}
+\sqrt{\delta}  \|f\|_{L^2(\Omega)}.
  \end{equation}
\end{theorem}
\begin{proof}
First, by triangle inequality and \eqref{eq:min}, for any $v_n\in V_n$, we have
\begin{align}
\|u_n-u\|_{a,\delta}&\le  \|u_n-u_\delta\|_{a,\delta}+\|u_\delta-u\|_{a,\delta}\\
&\le \|v_n-u_\delta\|_{a,\delta}+\|u_\delta-u\|_{a,\delta}\\
&\le  \|v_n-u\|_{a,\delta}+2\|u_\delta-u\|_{a,\delta} .
\end{align}
Then by the definition of $\|\cdot\|_{a,\delta}$, trace inequality and \eqref{diff:uudelta}, for any $v_n\in V_n$, we have
 \begin{equation}
\|u_n-u\|_{a,\delta} \lesssim (1+ \delta^{-\frac12}) \|v_n-u\|_{m,\Omega}
+\sqrt{\delta}  \|f\|_{L^2(\Omega)}.
\end{equation}
This completes the proof.
\end{proof}

\begin{theorem}
 Let $u$ be the solution of \eqref{m-BD} and
  $u_n$ be the solution of \eqref{min:uN} with $\delta\sim n^{-1/2 -1/{d}}$, respectively. Then
\begin{equation}
\label{error:D}
%\|u-u_n\|_{m,\Omega}\lesssim N^{-{1\over 4}-{1\over    2d}}(\|f\|_{-m+1+\frac{d}{2}+\epsilon}+\|f\|_{L^2(\Omega)}).
\|u-u_n\|_a \lesssim
(\|f\|_{L^2(\Omega)}+\|f\|_{-k+\frac{d}{2}+1+\epsilon})
\begin{cases}
n^{-{1\over 4}-{1\over {2d}}}&m< k,
\\
n^{-{1\over 4}}& m=k.
\end{cases}
\end{equation}
\end{theorem}
 \begin{proof} Let us only consider the case that $k>m$.
By Theorem \ref{lem:stratifiedapprox},
$$
\inf_{v_n\in V_n}\|u-v_n\|_{m,\Omega}\lesssim n^{-\frac12 -{1\over d}}\|u\|_{ B^{m+1}(\Omega)}.
$$
Thus, by \eqref{eq:1}
\begin{multline}
\label{eq:11}
\|u-u_n\|_{m,\Omega}\lesssim \|u_n-u\|_{a,\delta} \lesssim \delta^{-\frac12 } n^{-\frac12 -{1\over d}}\|u\|_{B^{m+1, q}(\Omega)}+\delta^{\frac12}\|f\|_{L^2(\Omega)}
\\
\le
(\delta^{-\frac12}  n^{-\frac12 -{1\over d}}+\delta^{\frac12})(\|u\|_{ B^{m+1}(\Omega)}+\|f\|_{L^2(\Omega)}),\ \ \forall\,\delta>0.
  \end{multline}
Set $\delta\sim n^{-1/2 -1/{d}}$, and it follows that
\begin{equation}
\|u-u_n\|_{m,\Omega}\lesssim n^{-{1\over 4}-{1\over 2d}}(\|u\|_{B^{m+1}(\Omega)}+\|f\|_{L^2(\Omega)}).
\end{equation}
Now by the embedding of spectral Barron space to Sobolev space, namely
Lemma \ref{smoothness-lemma}, the regularity result
\eqref{regularity}, the proof is completed.
 \end{proof}

 \begin{remark}
Although the analysis in this section is mainly for activation
function ReLU$^k$, similar results are valid for other
activations.  For example, Theorem \ref{thm:best} and Theorem
\ref{thm:bestD} naturally hold for any activation functions.
 \end{remark}

\begin{remark}
We also note that analysis in this section naturally
generalize to problems other than \eqref{2mPDE}.  For example, Theorem
\ref{thm:best} can be generalized to the case when $J(\cdot)$ is
replaced by a general strictly convex functional.
\end{remark}
%

\begin{remark}
 We note that \eqref{min:uN} was studied in \cite{weinan2018deep} for
 $m=1$ and $k=3$.  Convergence analysis for \eqref{m-mini-VN} and
 \eqref{min:uN} seems to be new in this paper.  For other convergence
 analysis of DNN for numerical PDE, we refer to
 \cite{shin2020convergence} and
 \cite{mishra2020enhancing,mishra2020estimates} for convergence
 analysis of PINN (Physics Informed Neural Network).
\end{remark}

\input{6DL/quadrature}

\section{Summary and discussions} \label{sec:Summary}

In this chapter, we consider a very special class of neural network
function based on ReLU$^k$ as activation function.  This function
class consists of piecewise polynomials which closely resemble finite
element functions.  By considering elliptic boundary value problems of $2m$-th
order in any dimensions, it is still unknown how to construct
$H^m$-conforming finite element space in general in the classic finite
element setting.  In contrast, it is rather straightforward to
construct $H^m$-conforming piecewise polynomials using neural
networks, known as the finite neuron method,  and we further proved
that the finite neuron method provides good approximation
properties.

It is still a subject of debate and of further investigation whether
it is practically efficient to use artificial neural network for
numerical solution of partial differential equations.  One major
challenge for this type of method is that the resulting optimization
problem is hard to solve, as we shall discuss below.

\subsection{Solution of the non-convex optimization problem}
\eqref{m-mini-VN} or \eqref{min:uN} is a highly nonlinear and
non-convex optimization problem with respect to parameters defining
the functions in $V_n$, see \eqref{VkN}. How to solve this type of
optimization problem efficiently is a topic of intensive research in
deep learning. For example,
stochastic gradient method is used in \cite{weinan2018deep} to solve
\eqref{min:uN} for $m=1$ and $k=3$.  Multi-scale deep neural network
(MscaleDNN) \cite{liu2020multi} and phase shift DNN (PhaseDNN)
\cite{cai2019phase} are developed to convert the high frequency
solution to a low frequency one before training. Randomized Newton's
method is developed to train the neural network from a nonlinear
computation point of view \cite{chen2019randomized}.  More refined
algorithms still need to be developed to solve \eqref{m-mini-VN} or
\eqref{min:uN} with high accuracy so that the convergence order,
\eqref{error:N} or \eqref{error:D}, of the finite neuron
method can not be achieved.

In order to approximate solutions to \eqref{model-problem}, we propose to solve the following optimization problem
\begin{equation}\label{discrete-formulation}
 u_{n,M,N} = \arg\min_{v\in B_M(\mathbb{D})} J_N(v),
\end{equation}
where
\begin{equation}
 B_M(\mathbb{D}) = \{u\in \mathcal{K}_1(\mathbb{D}):~\|u\|_{\mathcal{K}_1(\mathbb{D})} \leq M\}.
\end{equation}
Some result that says under which circumstances to have an estimate of $\|u\|_{\mathcal{K}_{1}}$ can be found in \cite{xu2020finite} (see Lemma 5.6) and \cite{lu2021priori} (See Theorem 2.5 and Theorem 2.6).
We note that the minimizer above is achieved as long as the $\sup_{d\in \mathbb{D}} \|d\|_{H^m(\Omega)} < \infty$. For in this case the optimization is over a bounded set and $J_N$ only depends upon the values and derivatives at finitely many points. Thus the optimization is effectively over a bounded set in a finite dimensional space, which is compact by the Heine-Borel Theorem.

For the case of mixed boundary conditions \eqref{equ:varpdelta} we sample $N$ points $x_1,...,x_N\in \Omega$ uniformly at random and also sample $N_0$ points $y_1,...,y_{N_0}\in \partial \Omega$ uniformly at random from the boundary. We then approximate the integrals in \eqref{equ:varpdelta} by
\begin{equation}\label{discrete-energy-dirichlet}
\begin{aligned}
 J_{N,\delta}(u) &= \frac{1}{2N}\sum_{i=1}^N\sum_{|\alpha| = m} a_\alpha(x_i)(\partial^\alpha u(x_i))^2 + \frac{1}{2N}\sum_{i=1}^Na_0(x_i)u(x_i)^2 - \frac{1}{N}\sum_{i=1}^Nf(x_i)u(x_i) \\
 &+ \frac{\delta^{-1}}{N_0}\sum_{i=1}^{N_0} \sum_{k=0}^{m-1} \left|\frac{\partial^k}{\partial \nu^k}u(y_i)\right|^2.
 \end{aligned}
\end{equation}
Forming the loss function \eqref{discrete-energy-dirichlet} requires being able to calculate normal derivatives of functions $u$ and to sample points uniformly from the boundary. This is possible for many important domains, such as the sphere or cube. We note that if the above sampling is not possible, then it suffices to be able to sample from a non-uniform distribution on $\Omega$ and $\partial \Omega$ with mass function $\rho$ if we weight each of the sums above by $\rho^{-1}$. We then propose to solve the following optimization problem to approximate solutions to \eqref{equ:varpdelta}
\begin{equation}\label{discrete-formulation-dirichlet}
 u_{n,M,N,\delta} = \arg\min_{v\in B_M(\mathbb{D})} J_{N,\delta}(v).
\end{equation}

 The optimization problems \eqref{discrete-formulation} and \eqref{discrete-formulation-dirichlet} can be efficiently approximately solved using greedy algorithms \cite{zhang2003sequential,lee1996efficient,jones1992simple,li1999mixture,barron2008approximation}. Specifically, the algorithm we use is the following
\begin{equation}\label{greedy-algorithm}
 u_0 = 0,~g_k = \arg\max_{g\in \mathbb{D}} \langle \nabla J_N(u_{k-1}), g\rangle,~u_k = (1-s_k)u_{k-1} - Ms_kg_k,
\end{equation}
where $s_k = \min\left(1,\frac{2}{k}\right)$. Importantly, we have that $u_k\in \Sigma_{k,M}(\mathbb{D})$, so that this algorithm produces neural networks with finite width. Note that this algorithm requires the computation of an $\arg\min$ over the dictionary $\mathbb{D}$, which may be a complicated step for certain dictionaries. We assume in what follows that this step can be efficiently calculated, which we argue is a reasonable assumption in practice for a wide range of dictionaries. For instance, when using the dictionary $\mathbb{P}_k^d$ corresponding to ReLU$^k$, this corresponds to an optimization over a compact $d$-dimensional set. Solving this problem for shallow neural network training was considered in \cite{lee1996efficient} and numerical experiments demonstrating the application to PDEs can be found in \cite{hao2021efficient}.

The algorithm \eqref{greedy-algorithm} was first introduced in \cite{jones1992simple} for the quadratic least-squares objective. It was analyzed in the context of neural networks for least squares fitting in \cite{lee1996efficient} and for density estimation \cite{li1999mixture}, and for general activation functions in \cite{zhang2003sequential}. Further extensions, such as the orthogonal greedy algorithm, relaxed greedy algorithm, and pure greedy algorithm, which however do not ensure that their iterates have bounded $\mathcal{K}_1(\mathbb{D})$-norm, are studied in \cite{devore1996some,barron2008approximation,temlyakov2008greedy,livshits2009lower,livshitz2003two,sil2004rate}. In what follows, we provide a convergence analysis which is applicable to our problem of interest.

More generally, we assume in our convergence analysis that the argmax in \eqref{greedy-algorithm} is not solved exactly, but rather is approximated in the following sense
\begin{equation}\label{argmax-approximation}
 \langle \nabla J_N(u_{k-1}), g_k\rangle \geq \frac{1}{R}\max_{g\in \mathbb{D}} \langle \nabla J_N(u_{k-1}), g\rangle
 \end{equation}
for some $R > 1$. This is a more tractable problem for most dictionaries. However, we only consider the case of $R > 1$ in the convergence analysis given in this section. In later sections, we assume for simplicity that $R = 1$ in the analysis of the quadrature error.

For our analysis we will need that the objectives $J_N$ and $J_{N,\delta}$ are convex and $K$-smooth with respect to a Hilbert space norm $H$. Recall that a function $L:H\rightarrow \mathbb{R}$ is $K$-smooth if
\begin{equation}
 J_N(g) \leq J_N(f) + \langle\nabla J_N(f), g-f\rangle + \frac{K}{2}\|g-f\|_H^2.
\end{equation}
In the following analysis, we will let $H$ be the discrete $H^k$ norm corresponding to the sample points $x_1,...,x_N$ and $y_1,...,y_{N_0}$, i.e. $H$ is given by
\begin{equation}\label{discrete-hm-inner-product}
 \langle u,v\rangle_H = \frac{1}{N}\sum_{i=1}^N\sum_{|\alpha| = m}(\partial^\alpha u(x_i))(\partial^\alpha v(x_i)) + \frac{1}{N}\sum_{i=1}^N u(x_i)v(x_i) + \frac{1}{N_0}\sum_{i=1}^{N_0} \sum_{k=0}^{m-1} \left|\frac{\partial^k}{\partial \nu^k}u(y_i)\right|^2.
\end{equation}
With respect to the norm $H$ above, the smoothness parameter $K$ in the pure Neumann case is the maximum of the coefficient functions $a_\alpha$ and $a_0$. In the case of mixed boundary conditions \eqref{discrete-energy-dirichlet} the smoothness parameter is bounded by $K \leq \max(\|a_{\alpha}\|_{L^\infty}, \|a_{0}\|_{L^\infty}) + \delta^{-1}$.

We have the following convergence result for the algorithm \eqref{greedy-algorithm}.
\begin{theorem}\label{relaxed-boosting-theorem}
 Suppose that the dictionary $\mathbb{D}$ is symmetric and satisfies $\sup_{d\in \mathbb{D}}\|d\|_H \leq C < \infty$. Let the iterates $u_n$ be given by the relaxed greedy algorithm \eqref{greedy-algorithm} with $s_k = \max\left(1,\frac{2}{k}\right)$, where the loss function $L$ is convex and $K$-smooth (on the Hilbert space $H$). Suppose that the $\arg\max$ in \eqref{greedy-algorithm} is approximated up to a factor $R$ as in \eqref{argmax-approximation}. Then we have $\|u_n\|_{\mathcal{K}_1(\mathbb{D})} \leq M$ and
 \begin{equation}
  L(u_n) - \inf_{\|v\|_{\mathcal{K}_1(\mathbb{D})} \leq R^{-1}M} L(v) \leq \frac{32(CM)^2K}{n}.
 \end{equation}

\end{theorem}
In particular, if $M \geq \|u\|_{\mathcal{K}_1(\mathbb{D})}$ where $u = \arg\min_v L(v)$ is the global minimizer, then the objective converges to the optimal value in the above theorem.

We note that applying this theorem to the squared error loss $L(f) = \frac{1}{2}\|f - f^*\|_H^2$ implies an approximation rate of
\begin{equation}\label{approximation-rate}
 \inf_{f_n\in \Sigma_{n,M}} \|f_n - f^*\|_H \lesssim Mn^{-\frac{1}{2}},
\end{equation}
for any $f^*\in B_M(\mathbb{D})$. This is essentially the method of proof used in \cite{jones1992simple}. In fact, for special dictionaries $\mathbb{D}$, the above rate can be improved. For instance, when $\mathbb{D} = \mathbb{P}_k^d$ the optimal rate was determined in \cite{siegel2021optimal} to be $n^{-1-\frac{2(k-m) + 1}{d}}$. Further, for more general activation functions it was shown that these rates can also be moderately improved in \cite{siegel2020approximation}.

\begin{proof}
 Since $u_0 = 0$ and $u_k$ is a convex combination of $u_{k-1}$ and $-Mg_k$, we see by induction that $\|u_k\|_{\mathcal{K}_1(\mathbb{D})} \leq M$.

 The $K$-smoothness of the objective $L$ implies that
 \begin{equation}
  L(u_k) \leq L(u_{k-1}) + \langle \nabla L(u_{k-1}), u_k - u_{k-1}\rangle + \frac{K}{2}\|u_k - u_{k-1}\|_H^2.
 \end{equation}
 Using the iteration \eqref{greedy-algorithm}, we see that $u_k - u_{k-1} = -s_ku_{k-1}-Ms_kg_k$. Plugging this into the above equation, we get
 \begin{equation}
  L(u_k) \leq L(u_{k-1}) - s_k\langle \nabla L(u_{k-1}), u_{k-1} + Mg_k\rangle + \frac{Ks_k^2}{2}\|u_{k-1} + Mg_k\|_H^2.
 \end{equation}
 Since the dictionary elements $g_k$ satisfy $\|g_k\|_H \leq C$ and $\|u_{k-1}\|_{\mathcal{K}_1(\mathbb{D})} \leq M$, we see that $\|u_{k-1}\|_H \leq CM$ as well. Plugging this into the previous equation implies the bound
 \begin{equation}\label{eq-479}
  L(u_k) \leq L(u_{k-1}) - s_k\langle \nabla L(u_{k-1}), u_{k-1} + Mg_k\rangle + 2(CM)^2Ks_k^2.
 \end{equation}
 Now let $z$ with $\|z\|_{\mathcal{K}_1(\mathbb{D})} \leq R^{-1}M$ be arbitrary. Then also $\|-z\|_{\mathcal{K}_1(\mathbb{D})} \leq R^{-1}M$ and the $\arg\max$ characterization of $g_k$ \eqref{argmax-approximation}  implies that
 \begin{equation}
  \langle \nabla L(u_{k-1}), -z\rangle \leq \langle \nabla L(u_{k-1}), Mg_k\rangle.
 \end{equation}
 Using this in equation \eqref{eq-479} gives
 \begin{equation}
  L(u_k) \leq L(u_{k-1}) - s_n\langle \nabla L(u_{k-1}), u_{k-1} - z\rangle + 2(CM)^2Ks_k^2.
 \end{equation}
 The convexity of $L$ means that $L(u_{k-1}) - L(z) \leq \langle \nabla L(u_{k-1}), u_{k-1} - z\rangle$. Using this and subtracting $L(z)$ from both sides of the above equation gives
 \begin{equation}
  L(u_k) - L(z) \leq (1 - s_k)(L(u_{k-1}) - L(z)) + 2(CM)^2Ks_k^2.
 \end{equation}
 Expanding the above recursion (using that $s_k\leq 1$), we get that
 \begin{equation}
  L(u_n) - L(z) \leq \left(\prod_{k=1}^n(1-s_k)\right)(L(u_0) - L(z)) + 2(CM)^2K\sum_{i=1}^n\left(\prod_{k=i+1}^n(1-s_k)\right)s_i^2.
 \end{equation}
 Using the choice $s_k = \max\left(1,\frac{2}{k}\right)$, for which $s_1 = 1$, we get
 \begin{equation}\label{eq-268}
  L(u_n) - L(z) \leq 2(CM)^2K\sum_{i=1}^n\left(\prod_{k=i+1}^n(1-s_k)\right)s_i^2.
 \end{equation}
 Finally, we bound the product $\displaystyle\prod_{k=i+1}^n(1-s_k)$ using that $\log(1+x) \leq x$ as
 \begin{equation}
  \log\left(\prod_{k=i+1}^n(1-s_k)\right) \leq -\sum_{k=i+1}^ns_k = -\sum_{k=i+1}^n\frac{2}{k} \leq -\int_{i+1}^{n+1}\frac{2}{x}dx \leq 2(\log(i+1) - \log(n+1)),
 \end{equation}
 for $i \geq 1$. Thus, $\prod_{k=i+1}^n(1-s_k) \leq \frac{(i+1)^2}{(n+1)^2}$. Using this in equation \eqref{eq-268}, we get
 \begin{equation}
  L(u_n) - L(z) \leq 2(CM)^2K\sum_{i=1}^n \frac{(i+1)^2}{(n+1)^2}s_i^2 \leq 8(CM)^2K\frac{1}{(n+1)^2}\sum_{i=1}^n \frac{(i+1)^2}{i^2}.
 \end{equation}
 Crudely bounding $\frac{(i+1)^2}{i^2} \leq 4$ for $i \geq 1$, we get
\begin{equation}
  L(u_n) - L(z) \leq 32(CM)^2K\frac{n}{(n+1)^2} \leq \frac{32M^2K}{n},
 \end{equation}
 Taking the infemum over $z$ with $\|z\|_{\mathcal{K}_1(\mathbb{D})} \leq R^{-1}M$ gives the result.
\end{proof}

\subsection{Competitions between locality and global smoothness}
One insight gained from the studies in this chapter is that the
challenges in constructing classic $H^m$-finite element subspace seems
to lie in the competitions between local d.o.f.  (degree of freedom)
and global smoothness.  In the classic finite element, one requires to
define d.o.f. on each element and then glue the local d.o.f. together
to obtain a globally $H^m$-smooth function. This process has proven to
be very difficult to realize in general when $m\ge 2$. But, if we relax the
locality, as in Powell-Sabine
element~\cite{powell1977piecewise}, we can use piecewise polynomials
of lower degree to construct globally smooth function. The neural
network approach studied in this paper can be considered as a global
construction without any use of a grid in the first place (even though
an implicitly defined grid exists). As a result, it is quite easy to
construct globally smooth functions that are piecewise polynomials. It
is quite remarkable that such a global construction
leads to function class that has very good approximation
properties. This is an attractive property of the function classes from
the artificial neural network.  One feasible question to ask if it is
possible to develop finite element construction technique that are
more global than the classic finite element but more  local than the
finite neuron method, which may be an interesting topic for
further research.

 \begin{center}
 \begin{table}[!ht]
\begin{tabular}{|c|c|c|c|}
\hline
Local D.O.F.  &Slightly more global &$\cdots$ &global \\
\hline
General grid& Special grid& $\cdots$ &No grid\\
\includegraphics[width=0.15\textwidth]{6DL/figures/unstructureGrid.pdf} &\includegraphics[width=0.22\textwidth]{6DL/figures/PowellSabin2.png} &$\cdots$ & \\
Conjecture: $k=(m-1)2^d+1$  &  Powell-Sabin \cite{powell1977piecewise}&  $\cdots$ & ReLU$^m$-DNN \\
True: $d=1,m\ge 1$ &    $k=2$ &   $\cdots$ & $k=m$     \\
$d=2,m=2$ (Still open)  & $d=2,m=2$ & $?$&  any $d$ and $m$ \\
\hline
\end{tabular}
\caption{Competitions between locality and global smoothness.}
\label{compare}
\end{table}
\end{center}
Observation: More global d.o.f. lead to easier
construction of conforming elements for high order PDEs.

\subsection{Piecewise $P_m$ for $H^m(\Omega)$: from finite element to
  finite neuron method} \label{sec:concluding} As it is noted above,
in the classic finite element setting, it is challenging to construct
$H^m$-conforming finite element spaces for any $m, d\ge 1$.  But if we
relax the conformity, as shown in \cite{wang2013minimal}, it is
possible to give a universal construction of convergent
$H^m$-nonconforming finite element consisting of piecewise polynomial
of degree $m$.  In the finite neuron method setting, by relaxing the
constraints from the a priori given finite element grid, the
construction of $H^m$-conforming piecewise polynomials of degree $m$
becomes straightforward.  In fact, the finite neuron method can be
considered as mesh-less method, or even, vertex-less method although
there is a hidden grid for any finite neuron function.  This raises a
question if it is possible to develop some "in-between" method that have
the advantages of both the classic finite element method and the
finite neuron method.

\subsection{Adaptivity and spectral accuracy}
One of the important properties in the traditional finite element
method is its ability to locally adapt the finite element grids to
provide accurate approximation of PDE solution that may have local
singularities (such as corner singularities and interface
singularities).  In contrast, the traditional spectral method (using
high order polynomials) can provide very high order accuracy for
solutions that are globally smooth.  The finite neuron method analyzed
in this paper seems to possess both the adaptivity feature as in the
traditional finite element method and also the global spectral
accuracy as in the traditional spectral methods.  Adaptivity feature
of the finite neuron method is expected since, as shown in
\S~\ref{sec:deep-fnm},  the deep finite neuron method can recover locally
adaptive finite element spaces for $m=1$.  Spectral feature of the
finite neuron method is illustrated in Theorem~\ref{thm:spectral}.
As a result, tt is conceivable that the finite neuron method may have both the
local and also global adaptive feature, or perhaps even adaptive
features in all different scales.  Nevertheless, such highly adaptive
features of the finite neuron method come with a potentially big
price, namely the solution of a nonlinear and non-convex optimization
problems.

\subsection{Comparison with PINN}
One important class of methods that is related to the FNM analyzed in
this paper is the the method of physical-informed neural networks
(PINN) introduced in \cite{raissi2019physics}.  By minimizing
certain norms of PDE residual together with penalizations of boundary
conditions and other relevant quantities, PINN is a very general
approach that can be directly applied to a wide range of problems.  In
comparison, FNM can only be applied to some special class of problems
that admit some special physical laws such as principle of energy
minimization or principle of least action, see \cite{feynmanfeynman}.
%\footnote{ Chapter 19
%  of Volume II, Feynman R, Leighton R, and Sands M. The Feynman
%  Lectures on Physics . 3 volumes 1964, 1966. Library of Congress
%  Catalog Card No. 63-20717. ISBN 0-201-02115-3 (1970 paperback
%  three-volume set); ISBN 0-201-50064-7 (1989 commemorative hardcover
%  three-volume set); ISBN 0-8053-9045-6 (2006 the definitive edition
%  (2nd printing); hardcover)}.
Because of the special physical law associated with our underlying
minimization problems, the Neumann boundary conditions are naturally
enforced in the minimization problem and, unlike in the PINN method,
no penalization is needed to enforce such type of boundary conditions.

\subsection{On the sharpness of the error estimates}

We  note our error estimate \eqref{error:D} for Dirichlet boundary
condition is not as good as the one \eqref{error:N} for Neumann
boundary conditions.  This is undesirable and may not be optimal.  In
comparison, Nitsche trick does not suffer a loss of accuracy when used
in traditional finite element method.

\subsection{Neural splines in multi-dimensions}
The spline functions described in Section~\ref{sec:Bsplines} are widely
used in scientific and enginnering computing, but their generalization
multiple dimension are non-trivial, especially when $\Omega$ has
curved boundary. In \cite{hu2015minimal}, using the tensor product,
the authors extended the 1D spline to multi-dimensions on rectangular
grids.  Some others involve rational functions such as NURBS
\cite{cottrell2009isogeometric}. But the generalization of neural networks to multi-dimension is straightforward and also the
resulting (nonlinear) space has very good approximate properties.  It
is conceivable that the neural network extension of B-spline to
multiple dimensions which are locally polynomials and globally smooth,
may find useful applications in computer aid design (CAD) and
isogeometric analysis \cite{cottrell2009isogeometric}. This is a
potentially an interesting research direction.


\section{Eigenvalues and Singular Values}
\subsection{}
\begin{definition}
  Let $A$ be an $nxn$ matrix. A scalar $\lambda$ is an eigenvalue of $A$ if
  \[
  A\vec{v}=\lambda\vec{v}
  .\]
  For some nonzero vector $\vec{v}\cdot\vec{v}$ is called the eigen vector corresponding to lambda.
\end{definition}
\begin{eg}
  Consider $A=\SmallMatrix{2&1\\1&2}$. Let's find the eigenvalues of this matrix.
    \begin{align*}
    0 &= det\left( \begin{pmatrix}2&1\\1&2\end{pmatrix}-\begin{pmatrix}\lambda&0\\0&\lambda\end{pmatrix}\right) \\
      &=det\begin{pmatrix} 2-\lambda&1\\1&2-\lambda \end{pmatrix} \\
      &=(2-\lambda)^2-1\\
    0&=(3-\lambda)(1-\lambda).
    \end{align*}
  Our eigenvalues are 3 and 1. Remember that $(A-\lambda I)\vec{v}=\vec{0}$. Consider $\lambda = 3$
  \begin{align*}
    \begin{pmatrix} -1&1\\1&-1 \end{pmatrix} \begin{pmatrix} x\\y \end{pmatrix} =\begin{pmatrix} 0\\0 \end{pmatrix} \\
    -x+y=0\\
    x=y\\
    \begin{pmatrix} x\\x \end{pmatrix} =x\begin{pmatrix} 1\\1 \end{pmatrix}
  .\end{align*}
  Now consider $\lambda = 1$.
  \begin{align*}
    A(\lambda I)\vec{v}=\vec{0}\\
    (A-\lambda I)\vec{v}\\
    \begin{pmatrix} 1&1\\1&1 \end{pmatrix} \begin{pmatrix} x\\y \end{pmatrix} =\begin{pmatrix} 0\\0 \end{pmatrix} \\
    x+y=0\\
    y=-x\\
    \begin{pmatrix} x\\-x \end{pmatrix} =x\begin{pmatrix} 1\\-1 \end{pmatrix}
  .\end{align*}
\end{eg}

\begin{eg}
  Consider $A=\SmallMatrix{1&1&2\\0&1&2\\0&0&3}\begin{pmatrix}1&1&2\\0&1&2\\0&0&3\end{pmatrix}$. For eigenvalues, $0=det(A-\lambda I)$.
  \begin{align*}
    0 = det \begin{pmatrix} 1-\lambda&1&2\\0&1-\lambda&2\\0&0&3-\lambda \end{pmatrix} \\
    0=(1-\lambda)(1-\lambda)(3-\lambda)
  .\end{align*}
  This makes our eigenvalues 1 and 3. For $\lambda=3$,
  \begin{align*}
    \begin{pmatrix} -2&1&2\\0&-2&2\\0&0&0 \end{pmatrix} \begin{pmatrix} x\\y\\z \end{pmatrix} &=\begin{pmatrix} 0\\0\\0 \end{pmatrix} \\
    -2y+2z&=0\\
    y&=0\\
    -2x+z+2z&=0\\
    -2x+3z&=0\\
    3z&=2x\\
    y&=z=\frac{2}{3}
  .\end{align*}
  This would make our eigenvector
  \[
  \begin{pmatrix} x\\\frac{2}{3}x\\\frac{2}{3}x \end{pmatrix} =x\begin{pmatrix} 1\\\frac{2}{3}\\\frac{2}{3} \end{pmatrix}
  .\]
\end{eg}
\begin{definition}
  Given an eigenvalue $\lambda$ of $A$, the corresponding eigenvectors form a subspace denoted $v_\lambda$. Note that $v_\lambda=ker(A-\lambda I)$
\end{definition}
\begin{note}
  $\lambda=0$ is an eigenvalue of $A$ if and only if the $ker(A-\lambda I)=ker(A)=v_0\neq \{0\}$. This is true if and only if $A$ is singular ($det(A)=0$).
\end{note}
\begin{eg}
  Consider $\begin{pmatrix} 1&1\\1&1 \end{pmatrix} $. The determinant of
  \[
    \begin{pmatrix} 1-\lambda&1\\1&1-\lambda \end{pmatrix}
  \]
  is equal to zero. We can see that when $\lambda =0$, the determinant is zero.
\end{eg}
\begin{prop}
  If $A$ is a real matrix and $\lambda+i\mu$ is an eigenvalue of $A$ with eigenvector $\vec{v}=\vec{x}+i\vec{y}$, then $\lambda-i\mu$ is an eigenvalue of $A$ with eigenvector $\vec{x}-i\vec{y}$.
\end{prop}
\begin{eg}
  Consider the matrix $A=\begin{bmatrix} 0&-1\\1&0 \end{bmatrix} $.
  \begin{align*}
    | A-\lambda I| = 0\\
    \left| \begin{matrix} -\lambda&-1\\1&-\lambda \end{matrix} \right| =0\\
    \lambda^2+1=0\\
    \lambda^2 = -1\\
    \lambda = \pm i = 0\pm i\\
    (A-\lambda I) \vec{v} = 0\\
    \begin{pmatrix} -i&-1\\1&-i \end{pmatrix} \begin{pmatrix} x\\y \end{pmatrix} =\begin{pmatrix} 0\\0 \end{pmatrix} \\
    -ix-y=0\\
    -ix=y
  .\end{align*}
  We now know that if we let $x$ be anything, and $y=-ix$, then we have the eigenvector. We can rewrite it as
  \begin{align*}
    \begin{pmatrix} x\\-ix \end{pmatrix} =\boxed{} + i\boxed{}\\
    =\begin{pmatrix} x\\0 \end{pmatrix}+\begin{pmatrix} 0\\-ix \end{pmatrix}\\
    =x\begin{pmatrix} 1\\0 \end{pmatrix}+x\begin{pmatrix} 0\\-i \end{pmatrix}  \\
    =x\begin{pmatrix} 1\\0 \end{pmatrix} + i\begin{pmatrix} 0\\-1 \end{pmatrix}
  .\end{align*}
  Now we also know that if $\lambda  = -i$ then the eigenvector is
  \[
  \begin{pmatrix} 1\\0 \end{pmatrix} -i\begin{pmatrix} 0\\1 \end{pmatrix}
  .\]
\end{eg}
\lecture{1}{Monday March 29, 2021}{8.2, 8.3}
If $A$ is an $n\times n$ matrix with real entries, then
\[
  det(A-\lambda I) = p(\lambda)= \text{The characteristic polynomial}
.\]
\begin{note}
  If $A$ is $2\times 2$, then $P_A(\lambda)=\lambda^2-Tr(A)+det(A)$
\end{note}
Suppose that $A=\begin{bmatrix} a&b\\c&d \end{bmatrix}$. The trace ($Tr(A)$) is $=a+d$, and the determinant if $A$ is $ad-bc$, so
\begin{align*}
  p_a(\lambda)=\left| \begin{matrix} a-\lambda&b\\c&d-\lambda \end{matrix} \right|
  =(a-\lambda)(d-\lambda)-bc\\
  =ad-d\lambda-a\lambda+\lambda^2-bc\\
  =\lambda^2-(a+d)\lambda+(ad-bc)
.\end{align*}
Remember, the trace is $a+d$, and the determinant is $ad-bc$. Recall that $A$ is real, so $P_A(\lambda)$ is real. If we set $P_A(\lambda)=0$. By the fundamental theorem of algebra, $P_A(\lambda)$ factors into linear factors over $\C$. Consider the equation
\[
x^{10}-7x^{9}+8x^2+\frac{1}{2}=0
.\]
This factors into 10 different roots. So if $A$ is $n\times n$, $P_A(\lambda)$ has at most $n$ roots in $\C$
\begin{theorem}
  Let $A$ be $n\times n$ with real entries. Then $A$ has at most $n$ eigenvalues. If $a+bi$ is an eigenvalue, then so is $a-bi$
\end{theorem}
\begin{eg}
  The Jordan Block Matrix. Let's look at $J_{2,3}=\begin{bmatrix} 2&1&0\\0&2&1\\0&0&2 \end{bmatrix} $. Let's find the eigenvalues:
  \begin{align*}
    | J_{2,3}-\lambda I| = 0\\
    \left| \begin{matrix} 2-\lambda&1&0\\0&2-\lambda&1\\0&0&2-\lambda \end{matrix} \right| =0\\
    (2-\lambda)(2-\lambda)(1-\lambda)=0\\
    \lambda=2
  .\end{align*}
  Now let's find the eigenvector(s):
  \begin{align*}
    (a-lambda I)\vec{v}=\vec{0}\\
    \begin{pmatrix} 0&1&0\\0&0&1\\0&0&0 \end{pmatrix} \begin{pmatrix} x\\y\\z \end{pmatrix} =\begin{pmatrix} 0\\0\\0 \end{pmatrix} \\
    y=0\\
    z=0\\
    \begin{pmatrix} x\\0\\0 \end{pmatrix} =x\begin{pmatrix} 1\\0\\0 \end{pmatrix}
  .\end{align*}
\end{eg}
\begin{theorem}
  If $A$ is square, then $P_A(\lambda)=P_{A^{T}}(\lambda)$. So, $A$ and $A^{T}$ have the same eigenvalues. Probably not the same eigenvectors.
\end{theorem}
\begin{theorem}
  Let $A$ be $n\times n$. The sum of the eigenvalues of $A$ is equal to the $Tr(A)$, and the product of the eigenvalues of $A$ is equal to $det(A)$.
\end{theorem}
\begin{eg}
  $J_{2,3}=\begin{pmatrix} 2&1&0\\0&2&1\\0&0&2 \end{pmatrix} $. The Trace of $J_{2,3}=2+2+2$ (adding the diagonals). The determinant of $J_{2,3}=2*2*2$ (multiplication of the diagonals).
  \[
    P_{J_{2,3}}(\lambda)=(2-\lambda)(2-\lambda)(2-\lambda)
  .\]
\end{eg}
\subsection{}
\begin{prop}
  If $\lambda_1,\ldots,\lambda_k$ are distinct eigenvalues of $A,$ then the corresponding eigenvectors are linearly independent.
\end{prop}
\begin{eg}
  Let's let $A=\begin{bmatrix} 1&1&0\\0&2&-1\\0&0&3 \end{bmatrix} $. Let's find the eigenvalues of $A$.
  \begin{align*}
    0=P_A(\lambda) &= \left| \begin{matrix} 1-\lambda&1&0\\0&2-\lambda&-1\\0&0&3-\lambda \end{matrix} \right| \\
    0&=(1-\lambda)(2-\lambda)(3-\lambda)\\
    \lambda&=1,2,3
  .\end{align*}
  For $\lambda=1$
  \begin{align*}
    (A-\lambda I)\vec{v}&=\vec{0}\\
    \begin{pmatrix} 0&1&0\\0&1&-1\\0&0&2 \end{pmatrix} \begin{pmatrix} x\\y\\z \end{pmatrix} &=\begin{pmatrix} 0\\0\\0 \end{pmatrix} \\
    y&=0\\
    y-z&=0\\
    z&=0\\
    \begin{pmatrix} x\\0\\0 \end{pmatrix} &=x\begin{pmatrix} 1\\0\\0 \end{pmatrix}
  .\end{align*}
  Now for $\lambda=2$
  \begin{align*}
    \begin{pmatrix} -1&1&0\\0&0&-1\\0&0&1 \end{pmatrix} \begin{pmatrix} x\\y\\z \end{pmatrix} &=\begin{pmatrix} 0\\0\\0 \end{pmatrix} \\
    -x+y&=0\\
    x&=y
    -z&=0\\
    \begin{pmatrix} x\\x\\0 \end{pmatrix} &=x\begin{pmatrix} 1\\1\\0 \end{pmatrix}
  .\end{align*}
  Now for $\lambda=3$
  \begin{align*}
    \begin{pmatrix} -2&1&0\\0&-1&-1\\0&0&0 \end{pmatrix} \begin{pmatrix} x\\y\\z \end{pmatrix} &=\begin{pmatrix} 0\\0\\0 \end{pmatrix} \\
    -2x+y&=0\to y=2x\\
    -y-z&=0\to z=-y\\
    z&=-2x\\
    \begin{pmatrix} x\\2x\\-2x \end{pmatrix} &=x\begin{pmatrix} 1\\2\\-2 \end{pmatrix}
  .\end{align*}
  We now have the following eigenvectors
  \[
  \begin{pmatrix} 1\\0\\0 \end{pmatrix} ,\begin{pmatrix} 1\\1\\0 \end{pmatrix} ,\begin{pmatrix} 1\\2\\-2 \end{pmatrix}
  .\]
  These are linearly independent so we have a basis for $\R^3$
\end{eg}
\begin{theorem}
  If $A$ is $n\times n$ and $a$ has $n$ distinct real (/complex) eigenvalues, then the corresponding eigenvectors form a basis for $\R^b(\C^n)$.
\end{theorem}
Now\ldots vector spaces. Vectors have two main operations,
\begin{align*}
  \vec{v}+\vec{w}\\
  c\cdot v
.\end{align*}
Let $V, W$ be vector spaces over $\R$. We know that $\R^3\begin{pmatrix} a\\0\\0 \end{pmatrix}$. In reality, $V, W$ are the same, but we must show that they are the same. We would need a function that preserves the operations from $V$ to $W$. A map (function) $f$ from $V$ to $W$ should have the following
\begin{align*}
  f(\vec{v_1}+\vec{v_2})&=f(\vec{v_1})+f(\vec{v_2})\\
  f(r\cdot \vec{v})&=rf(\vec{v})
.\end{align*}
\lecture{2}{Wednesday, March 31st}{8.3}
Recall that $\lambda=0$ is an eigenvalue of $A$, if and only if
\[
  (A-0\cdot I)\vec{v}=\vec{0}
\] has a $\vec{v}\neq \vec{0}$ solution if and only if
\[
  ker(A)\neq\{0\}
\] if and only if  $A^{-1}$ does not exist.
\begin{theorem}
  If $\lambda_1,\ldots,\lambda_k$ are distinct eigenvalues of $A$, then the corresponding eigenvectors are linearly independent.
\end{theorem}
\begin{eg}
  Consider $A=\begin{pmatrix} 2&1\\1&2 \end{pmatrix} $. The eigenvalues for this matrix are $\lambda=3,1$. The eigenvector for $\lambda=3$ is $\begin{pmatrix} 1\\1 \end{pmatrix} $, and the eigenvector for $\lambda=1$ is $\begin{pmatrix} -1\\1 \end{pmatrix} $.
\end{eg}
\begin{eg}
  Consider $A=\begin{pmatrix} 1&1&2\\0&1&2\\0&0&3 \end{pmatrix} $. The characteristic polynomial is $P(\lambda)=(1-\lambda)^2(3-\lambda)$. For $ \lambda = 1$, the eigenvector is $\begin{pmatrix} 1\\0\\0 \end{pmatrix} $, and for $\lambda = 3$, the eigenvector is $\begin{pmatrix} 3\\2\\2 \end{pmatrix} $
\end{eg}
\begin{theorem}
  If $A $ is $n\times n$ and $A$ has $n$ distinct real (or complex) eigenvalues, then the corresponding eigenvectors $\vec{v_1},\ldots,\vec{v_n}$ form a basis for $\R^{n}$.
\end{theorem}
  Consider $A=\begin{pmatrix} 1&-1\\2&4 \end{pmatrix} $. $A$ defines a map from $\R^{2}$ to $\R^2$. Our map is
  \begin{align*}
    L: \begin{pmatrix} x\\y \end{pmatrix}\to A\begin{pmatrix} x\\y \end{pmatrix}\\
    L\left( r\vec{v} \right) =rL(\vec{v})
  .\end{align*}
  $L$ is a linear transformation.

\begin{eg}
  Consider $A=\begin{pmatrix} 1&-1\\2&4 \end{pmatrix} $. What happens if we multiply $A$ by $\begin{pmatrix} 2\\1 \end{pmatrix} $?
  \begin{align*}
    A\begin{pmatrix} 2\\1 \end{pmatrix} =\begin{pmatrix} 1&-1\\2&4 \end{pmatrix} \begin{pmatrix} 2\\1 \end{pmatrix} \\
    =\begin{pmatrix} 1\\8 \end{pmatrix}
  .\end{align*}
  Let's think of $\begin{pmatrix} 2\\1 \end{pmatrix} $ as $2e_1+1e_2$. This would make our equation \[
  2\begin{pmatrix} 1\\0 \end{pmatrix} +\begin{pmatrix} 0\\1 \end{pmatrix}
  .\]
  Let's also think about $\begin{pmatrix} 1\\8 \end{pmatrix} $ as \[
  1\begin{pmatrix} 1\\0 \end{pmatrix} +8\begin{pmatrix} 0\\1 \end{pmatrix}
  .\]
  We can change the basis to
  \begin{align*}
    v_1=\begin{pmatrix} 1\\-1 \end{pmatrix} && v_2=\begin{pmatrix} 1\\-2 \end{pmatrix}
  .\end{align*}
\end{eg}
\lecture{3}{Monday March 29, 2021}{8.2, 8.3}

\begin{figure}[h]
  \centering
  \includegraphics[width=0.8\textwidth]{resource/images/8-2Figure1.png}
  \caption{Example Line image}
  \label{fig:}
\end{figure}

A linear transformation consists of the following:
\begin{align*}
  L(v_1+v_2)&=L(v_1)+L(v_2)\\
  L(r\cdot v)&=rL(v)
.\end{align*}
  Consider $A = \begin{pmatrix} 1&-1\\2&4 \end{pmatrix} $. If everything is in the standard basis then
  \begin{align*}
    \begin{pmatrix} 1&-1\\2&4 \end{pmatrix} \begin{pmatrix} -1\\4 \end{pmatrix} =\begin{pmatrix} -5\\14 \end{pmatrix} \\
    -1\begin{pmatrix} 1\\0 \end{pmatrix} + 4\begin{pmatrix} 0\\1 \end{pmatrix} \to -5\begin{pmatrix} 1\\0 \end{pmatrix} +14\begin{pmatrix} 0\\1 \end{pmatrix}
  .\end{align*}
  It's harder to figure out what the vector is with the new basis from the picture, but the transformation has a nice description of \[
    B=\begin{pmatrix} 2&0\\0&3 \end{pmatrix}
  .\]
  If we take $B$ and hit it with the new coefficients:
  \begin{align*}
    \begin{pmatrix} 2&0\\0&3 \end{pmatrix} \begin{pmatrix} 2\\-3 \end{pmatrix} =\begin{pmatrix} 4\\-9 \end{pmatrix} \\
    2\begin{pmatrix} 1\\-1 \end{pmatrix} + -3\begin{pmatrix} 1\\-2 \end{pmatrix} \to 4\begin{pmatrix} 1\\-1 \end{pmatrix} +-9\begin{pmatrix} 1\\-2 \end{pmatrix}
  .\end{align*}
    The left hand side is equal to $\begin{pmatrix} -1\\4 \end{pmatrix}$ and the right hand side is equal to $\begin{pmatrix} -5\\14 \end{pmatrix}$ If we take $\begin{pmatrix} 2&0\\0&3 \end{pmatrix} \begin{pmatrix} 1\\0 \end{pmatrix} =\begin{pmatrix} 2\\0 \end{pmatrix} $, which looks like an eigenvector. The first basis that we had was $\left\{\begin{pmatrix} 1\\0 \end{pmatrix}, \begin{pmatrix} 0\\1 \end{pmatrix}  \right\}$, and the second basis was $\left\{\begin{pmatrix} 1\\-1 \end{pmatrix}, \begin{pmatrix} 1\\-2 \end{pmatrix}  \right\}$. So we took basis 1 and did a linear transofrmation to get basis 2 like the following
    \[
    \begin{pmatrix} 1&-1\\2&4 \end{pmatrix} \to \begin{pmatrix} 2&0\\0&3 \end{pmatrix}
    .\] If $A$ is in the standard basis. Then $B=S^{-1}AS$, where $S=\begin{pmatrix} \vec{v_1},\vec{v_2},\ldots,\vec{v_{n}} \end{pmatrix} $, where $\{\vec{v_1},\ldots,\vec{v_n}\}$ is the new basis.
\begin{eg}
  Let $A=\begin{pmatrix} 1&-1\\2&4 \end{pmatrix} $.
  \begin{align*}
    S=\begin{pmatrix} 1&1\\-1&-2 \end{pmatrix} \\
    S^{-1}=\frac{1}{-2--1}\begin{pmatrix} -2&-1\\1&1 \end{pmatrix} \\
    =\begin{pmatrix} 2&1\\-1&-1 \end{pmatrix} \\
    B=S^{-1}AS=\begin{pmatrix} 2&1\\-1&-1 \end{pmatrix} \begin{pmatrix} 1&-1\\2&4 \end{pmatrix} \begin{pmatrix} 1&1\\-1&-2 \end{pmatrix} \\
    = \begin{pmatrix} 2&1\\-1&-1 \end{pmatrix} \begin{pmatrix} 2&3\\-2&-6 \end{pmatrix} \\
    =\begin{pmatrix} 2&0\\0&3 \end{pmatrix}
  .\end{align*}
\end{eg}
\begin{definition}
  A square matrix $A$ is diagonalizable if there is a matrix $S$ and a diagonal matrix $\Lambda$ such that
  \begin{equation*}
    \Lambda = S^{-1}AS
  \end{equation*}
\end{definition}
\begin{theorem}
  $A$ is diagonalizable if and only if $A$ has $n$ linearly independent eigenvectors $\vec{v_1},\ldots,\vec{v_n}$. In this case, \[
  \Lambda = S^{-1}AS
  .\] where $S=\begin{bmatrix} \vec{v_1},\ldots,\vec{v_n} \end{bmatrix} $, and $\Lambda=\begin{bmatrix} \lambda_1&0\\0&\lambda_n \end{bmatrix} $, where $\lambda_i$ is the eigenvalue for $\vec{v_i}$
\end{theorem}
If we are looking for the linearly independent solutions we need to do $A-\lambda I=0$. From here we would get lambda values and create eigenvectors using $A-\lambda I\vec{v}=0$
\begin{eg}
  Let $A=\begin{pmatrix} 0&-1&-1\\1&2&1\\1&1&2 \end{pmatrix} $, which is our standard matrix. Once we find our \[
  \Lambda = \begin{pmatrix} 1&0&0\\0&1&0\\0&0&2 \end{pmatrix}
  .\] Our basis is \[
  \begin{pmatrix} -1\\1\\0 \end{pmatrix} \begin{pmatrix} -1\\0\\1 \end{pmatrix} \begin{pmatrix} -1\\1\\1 \end{pmatrix}
  .\] Let's take $A$ and hit it with coefficients in the standard basis. Let's try \[
  A\cdot\begin{pmatrix} -3\\2\\2 \end{pmatrix}
  .\] We need to think about the matrix on the right hand side as $-3\begin{pmatrix} 1\\0\\0 \end{pmatrix} +2\begin{pmatrix} 0\\1\\0 \end{pmatrix} +2\begin{pmatrix} 0\\0\\1 \end{pmatrix} $, so we can do the following:
  \begin{align*}
    \begin{pmatrix} 0&-1&-1\\1&2&1\\1&1&2 \end{pmatrix} \begin{pmatrix} -3\\2\\2 \end{pmatrix} =\begin{pmatrix} -4\\3\\3 \end{pmatrix}
  .\end{align*}
  We know that our divided up equation from before (for the right matrix) is getting mapped to $4\begin{pmatrix} 1\\0\\0 \end{pmatrix} +3\begin{pmatrix} 0\\1\\0 \end{pmatrix} +3\begin{pmatrix} 0\\0\\1 \end{pmatrix} $. We know that our $\Lambda$ must take $\begin{pmatrix} -3\\2\\2 \end{pmatrix} $, but we need to rewrite it in our new basis.\[
  1\begin{pmatrix} -1\\1\\0 \end{pmatrix} +1\begin{pmatrix} -1\\0\\1 \end{pmatrix} +1\begin{pmatrix} -1\\1\\1 \end{pmatrix}
  .\]
  If we multiply this by our $\lambda$, we can figure out out map. \[
    \begin{pmatrix} 1&0&0\\0&1&0\\0&0&2 \end{pmatrix} \begin{pmatrix} 1\\1\\1 \end{pmatrix} =\begin{pmatrix} 1\\1\\2 \end{pmatrix}
  .\]
  Our equation gets mapped to \[
  1\begin{pmatrix} -1\\1\\0 \end{pmatrix} +1\begin{pmatrix} -1\\0\\1 \end{pmatrix} +2\begin{pmatrix} -1\\1\\1 \end{pmatrix}
  .\]
  This means that our vector is the same from before because we get $\begin{pmatrix} -4\\3\\3 \end{pmatrix} $.\newline
  $Av_1\to w_1$, while $\Lambda v_2\to w_2$.
\end{eg}

\lecture{4}{Monday April 12, 2021}{8.5}

Let $A$ be an $m\times n$ matrix. $A$ defines a function from $\R^{n}$ to $\R^{n}$
\begin{align*}
  L:\vec{x}\to A\vec{x}\\
  L(\vec{x}+\vec{y})=L(\vec{x})+L(\vec{y})\\
  L(c\vec{x})=cL(\vec{x})
.\end{align*}
This fixes the standard basis $\begin{pmatrix} 1\\0\\ \vdots\\0 \end{pmatrix},\begin{pmatrix} 0\\1\\ \vdots \\0 \end{pmatrix}, \begin{pmatrix} 0\\0\\ \vdots \\0 \end{pmatrix}   $ and a different basis. Consider $\begin{pmatrix} 2\\1\\5 \end{pmatrix} = 2\begin{pmatrix} 1\\0\\-1 \end{pmatrix} + \begin{pmatrix} 0\\1\\0 \end{pmatrix} +5\begin{pmatrix} 0\\0\\1 \end{pmatrix}  $. A is mapping coefficients in the standard basis to coefficients in the standard basis.

\lecture{5}{Wednesday April 14, 2021}{}

If we have an $n\times n$ matrix with $n$ distinct eigenvalues, we have $n$ linearly independent eigenvectors. \[
\Lambda=S^{-1}AS
.\]
If the matrix is real symmetric, then we can get a real eigenvector basis so we can diagonalize like \[
S\Lambda S^{-1}=A
.\]
If we do $A\cdot A$, then we get
\begin{align*}
  S\Lambda S^{-1}S\Lambda S^{-1}\\
  =S \Lambda S^{-1}\\
  A^{k}=S \Lambda ^{k}S^{-1}
.\end{align*}
Let's look at an example where this is not going to work.
\begin{eg}
  Let $M=\begin{bmatrix} 1&1&1\\0&1&0\\0&0&1 \end{bmatrix} $. This matrix only has one eigenvalue of $\lambda=1$. Let's find the eigenvectors.
  \begin{align*}
    \begin{bmatrix} 0&1&1\\0&0&0\\0&0&0 \end{bmatrix} \begin{bmatrix} x\\y\\z \end{bmatrix} =\begin{bmatrix} 0\\0\\0 \end{bmatrix} \\
    y+z=0\to z=-y\\
    \begin{bmatrix} x\\y\\-y \end{bmatrix} =x\begin{bmatrix} 1\\0\\0 \end{bmatrix} +y\begin{bmatrix} 0\\1\\-1 \end{bmatrix}
  .\end{align*}
\end{eg}
Now supposed instead, if we let $M=\begin{bmatrix} 1&0&0\\0&1&0\\0&0&1 \end{bmatrix} $. This matrix must have 3 linearly independent eigenvectors.
\begin{align*}
  |M- \lambda I|=0\\
  \left| \begin{matrix} 1-\lambda &0&0\\0&1-\lambda&0\\0&0&1-\lambda \end{matrix} \right| =0\\
  (M-\lambda I)\vec{v} = 0\\
  \begin{bmatrix} 0&0&0\\0&0&0\\0&0&0 \end{bmatrix} \begin{bmatrix} x\\y\\z \end{bmatrix} =\begin{bmatrix} 0\\0\\0 \end{bmatrix} \\
  \begin{bmatrix} x\\y\\z \end{bmatrix} =x\begin{bmatrix} 1\\0\\0 \end{bmatrix} +y\begin{bmatrix} 0\\1\\0 \end{bmatrix} +z\begin{bmatrix} 0\\0\\1 \end{bmatrix}
.\end{align*}

\begin{theorem}
  Let $A=A^{T}$ be an $n\times n$ real matrix. Let $\vec{v_1},\ldots,\vec{v_n}$ be an eigenvector basis such that $\vec{v_1},\ldots,\vec{v_r}$ correspond to nonzero eigenvalues and $\vec{v_{r+1}},\ldots,\vec{v_n}$ correspond to the zero eigenvalue. Then $r=rank(A)$, $\vec{v_1},\ldots,\vec{v_r}$ form an orthogonal basis for $IM(A)=coimg(A)$, and $\vec{v_{r+1}},\ldots,\vec{v_n}$ form an orthogonal basis for the $ker(A)=coker(A)$.
\end{theorem}
\begin{theorem}
  The spectral theorem. Let $A$ be a real symmetric matrix. Then there exists an orthogonal matrix $Q$ such that $A=Q\Lambda Q^{-1}=Q\Lambda Q^{T}$, where $\Lambda $ is a real diagonal matrix. The eigenvalues of $A$ appear on the diagonal of $\Lambda $, while the columns of $Q$ are orthonormal eigenvectors of $A$.
\end{theorem}
\begin{eg}
  Let $A=\begin{bmatrix} 1&-1&1\\-1&1&0\\1&01 \end{bmatrix} $. Our eigenvalues are $1+\sqrt{2},1,1-\sqrt{2}  $, which means our eigenvectors are \[
  \begin{bmatrix} \sqrt{2} \\-1\\1 \end{bmatrix} ,\begin{bmatrix} 0\\1\\1 \end{bmatrix} ,\begin{bmatrix} -\sqrt{2} \\-1\\1 \end{bmatrix}
  .\]
  The orthonormal basis of the eigenvectors is
  \begin{align*}
    \sqrt{(\sqrt{2} )^2+(-1)^2+1^2}\\
    =\sqrt{6}\\
    \begin{bmatrix} \frac{\sqrt{2} }{2}\\-\frac{1}{2}\\\frac{1}{2} \end{bmatrix} ,\begin{bmatrix} 0\\\frac{1}{\sqrt{2} }\\\frac{1}{\sqrt{2} } \end{bmatrix} ,\begin{bmatrix} \frac{1}{2}\\\frac{1}{2}\\\frac{1}{2} \end{bmatrix}
  .\end{align*}
  Now if we let $S=\begin{bmatrix} \sqrt{2} &0&-\sqrt{2}\\-1&1&-1\\1&1&1  \end{bmatrix} $ and we do \[
  S^{-1}AS= \Lambda=\begin{bmatrix} 1+\sqrt{2}&0&0\\0&1&0\\0&0&1-\sqrt{2}   \end{bmatrix}
  .\]
\end{eg}

\lecture{6}{Friday April 16}{8.6}


% Copyright 2018 Melvin Eloy Irizarry-Gelpí
\chapter{W Binions}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
A W binion has the form
\begin{equation}
    a_{0} + a_{1} W
\end{equation}
These follow from a parabolic Cayley-Dickson construct on the reals. Since you need two real numbers for a W binion, you can represent a W binion as a pair.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Multiplication}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
The multiplication operation is non-trivial. Since the W binions are a parabolic Cayley-Dickson construct, you use the parabolic multiplication operation. If
\begin{align}
    x &= \begin{pmatrix}
        x_{0} & x_{1}
    \end{pmatrix} & y &= \begin{pmatrix}
        y_{0} & y_{1}
    \end{pmatrix}
\end{align}
then
\begin{equation}
    z = x \wedge y = \begin{pmatrix}
        x_{0} y_{0} & y_{1} x_{0} + y_{0} x_{1}
    \end{pmatrix}
\end{equation}
Note that this multiplication operation is commutative. Note that
\begin{equation}
    W = \begin{pmatrix}
        0 & 1
    \end{pmatrix}
\end{equation}
which leads to
\begin{equation}
    W \wedge W = \begin{pmatrix}
        0 & 0
    \end{pmatrix}
\end{equation}
That is, $W$ is nilpotent.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conjugate Operations}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
There are many conjugate operations acting on W binions.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Asterisk Conjugation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
If
\begin{equation}
    z = \begin{pmatrix}
        z_{0} & z_{1}
    \end{pmatrix}
\end{equation}
then the asterisk conjugation is
\begin{equation}
    {\ast z} = \begin{pmatrix}
        z_{0} & -z_{1}
    \end{pmatrix}
\end{equation}
Note that this is an involution.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Quadrance}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
If
\begin{equation}
    z = \begin{pmatrix}
        z_{0} & z_{1}
    \end{pmatrix}
\end{equation}
then the quadrance is given by
\begin{equation}
    \Vert z \Vert^{2} = (z_{0})^{2}
\end{equation}
Note that the quadrance takes non-negative values. However, you can have zero-quadrance from a non-trivial W binion. Any W binion of the form
\begin{equation}
    \begin{pmatrix}
        0 & z_{1}
    \end{pmatrix}
\end{equation}
has zero quadrance.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Cloak Conjugation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
If
\begin{equation}
    z = \begin{pmatrix}
        z_{0} & z_{1}
    \end{pmatrix}
\end{equation}
then the cloak conjugation is
\begin{equation}
    {\diamond z} = \begin{pmatrix}
        -z_{0} & z_{1}
    \end{pmatrix}
\end{equation}
Note that this is an involution. Cloak conjugation is equivalent to negation of the asterisk conjugation.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Dagger Conjugation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
If
\begin{equation}
    z = \begin{pmatrix}
        z_{0} & z_{1}
    \end{pmatrix}
\end{equation}
then the dagger conjugation is
\begin{equation}
    {\dagger z} = \begin{pmatrix}
        z_{0} & -z_{1}
    \end{pmatrix}
\end{equation}
Note that this is equivalent to the asterisk conjugation.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Hodge Star}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
If
\begin{equation}
    z = \begin{pmatrix}
        z_{0} & z_{1}
    \end{pmatrix}
\end{equation}
then the Hodge star is
\begin{equation}
    {\star z} = \begin{pmatrix}
        z_{1} & z_{0}
    \end{pmatrix}
\end{equation}
Note that this is an involution.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Zero-Divisors}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Since multiplication of non-zero W binions can lead to zero, there are non-trivial zero-divisors. For example, if
\begin{align}
    x &= \begin{pmatrix}
        0 & x_{1}
    \end{pmatrix} & y &= \begin{pmatrix}
        0 & y_{1}
    \end{pmatrix}
\end{align}
then
\begin{equation}
    z = x \wedge y = \begin{pmatrix}
        0 & 0
    \end{pmatrix}
\end{equation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Decompositions}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
With so many involutions you can have many decomposition.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Asterisk Decomposition}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Since the asterisk conjugation is an involution, you can have the following decomposition:
\begin{equation}
    z = \frac{1}{2} \left( z + {\ast z} \right) + \frac{1}{2} \left( z - {\ast z} \right)
\end{equation}
The first term is the self-asterisk part of $z$ and the second term is the anti-self-asterisk part of $z$. If
\begin{equation}
    z = \begin{pmatrix}
        z_{0} & z_{1}
    \end{pmatrix}
\end{equation}
then the \textbf{self-asterisk part} of $z$ is given by
\begin{equation}
    \frac{1}{2} \left( z + {\ast z} \right) = \begin{pmatrix}
        z_{0} & 0
    \end{pmatrix}
\end{equation}
and the \textbf{anti-self-asterisk part} of $z$ is given by
\begin{equation}
    \frac{1}{2} \left( z - {\ast z} \right) = \begin{pmatrix}
        0 & z_{1}
    \end{pmatrix}
\end{equation}
Note that the anti-self-asterisk part of $z$ is always nilpotent.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Hodge Star Decomposition}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Since the Hodge star operation is an involution, you can have the following decomposition:
\begin{equation}
    z = \frac{1}{2} \left( z + {\star z} \right) + \frac{1}{2} \left( z - {\star z} \right)
\end{equation}
The first term is the self-star part of $z$ and the second term is the anti-self-star part of $z$. If
\begin{equation}
    z = \begin{pmatrix}
        z_{0} & z_{1}
    \end{pmatrix}
\end{equation}
then the \textbf{self-star part} of $z$ is given by
\begin{equation}
    \frac{1}{2} \left( z + {\star z} \right) = \frac{1}{2} (z_{0} + z_{1}) \begin{pmatrix}
        1 & 1
    \end{pmatrix}
\end{equation}
and the \textbf{anti-self-star part} of $z$ is given by
\begin{equation}
    \frac{1}{2} \left( z - {\star z} \right) = \frac{1}{2} (z_{0} - z_{1}) \begin{pmatrix}
        1 & {-1}
    \end{pmatrix}
\end{equation}
Note that the self-star and anti-self-star parts are proportional to W binions that are idempotent. If
\begin{align}
    P &\equiv \frac{1}{2} \begin{pmatrix}
        1 & 1
    \end{pmatrix} & Q &\equiv \frac{1}{2} \begin{pmatrix}
        1 & {-1}
    \end{pmatrix}
\end{align}
then
\begin{align}
    P \wedge P &= P & Q \wedge Q &= Q
\end{align}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Differential Operators}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
With two real variables $x$ and $y$, you can define a W binion variable $z$ via
\begin{equation}
    z = \begin{pmatrix}
        x & y
    \end{pmatrix}
\end{equation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{M\"{o}bius Transformations}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
...
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Cross-Ratio}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
...


\section{Application: Resistor networks}

The tools of linear algebra can be used to study the application of
resistor networks. An example of an electrical circuit is below.

\begin{center}
  \scalebox{0.8}{
    \begin{circuitikz}[american] \draw
      (0,0) to [battery1, v^= $18\volt$~~] (0,4)
      (0,0) to [R = $2 \ohm$] (4,0)
      to [R = $4 \ohm$] (4,4)
      (0,4) to [R =$2 \ohm$] (4,4)
      (2,2) node[scale=4]{$\circlearrowleft$}
      (2,2) node{$I_1$}
      ;
    \end{circuitikz}
  }
\end{center}

\noindent
The jagged lines (\begin{circuitikz}[baseline=-0.5ex] \draw (0,0) to
  [R] (2,0); \end {circuitikz}) denote resistors and the numbers next
to them give their resistance%
\index{resistance} in ohms, written as $\ohm$. The voltage%
\index{voltage} source (\begin{circuitikz}[baseline=-0.5ex] \draw
  (0,0) to [/tikz/circuitikz/bipoles/length=0.75cm, battery1]
  (1,0); \end {circuitikz}) causes the current%
\index{current} to flow in the direction from the longer of the two
lines toward the shorter\footnote{By {\em current}, we always mean the
  {\em conventional current}, which flows from plus to minus. It is
  the opposite of the electron flow, which goes from minus to plus.}.
Voltage is measured in volts, written as $\volt$.  The current for a
circuit is labelled $I_k$, and is measured in amperes, written as
$\amp$.

In the above figure, the current $I_1$ has been labelled with an arrow
in the counterclockwise direction. This is an entirely arbitrary
decision and we could have chosen to label the current in the
counterclockwise direction.  With our choice of direction here, we
define a positive current to flow in the counterclockwise direction
and a negative current to flow in the clockwise direction.

The goal of this section is to use the values of resistors and voltage
sources in a circuit to determine the current. An essential theorem
for this application is Kirchhoff's law%
\index{Kirchhoff's law}.

\begin{theorem}{Kirchhoff's law}{kirchhoff-law}
  The sum of the resistance ($R$) times the amperes ($I$) in the
  counterclockwise direction around a loop equals the sum of the
  voltage sources ($V$) in the same direction around the loop.
\end{theorem}

Kirchhoff's law allows us to set up a system of linear equations and
solve for any unknown variables. When setting up this system, it is
important to trace the circuit in the counterclockwise direction. If a
resistor or voltage source is crossed against this direction, the
related term must be given a negative sign.

We will explore this in the next example where we determine the value
of the current in the initial diagram.

\begin{example}{Solving for current}{solve-current1}
  Applying Kirchhoff's Law to the diagram below, determine the value for $I_1$.

  \begin{center}
    \scalebox{0.8}{
      \begin{circuitikz}[american,scale=0.8] \draw
        (0,0) to [battery1, v^= $18\volt$~~] (0,4)
        (0,0) to [R = $2 \ohm$] (4,0)
        to [R = $4 \ohm$] (4,4)
        (0,4) to [R =$2 \ohm$] (4,4)
        (2,2) node[scale=4]{$\circlearrowleft$}
        (2,2) node{$I_1$}
        ;
      \end{circuitikz}
    }
  \end{center}

\end{example}

\begin{solution}
  Begin in the bottom left corner, and trace the circuit in the
  counterclockwise direction. At the first resistor, multiplying
  resistance and current gives $2I_1$. Continuing in this way through
  all three resistors gives $2I_1 + 4I_1 + 2 I_1$. This must equal the
  voltage source in the same direction. Notice that the direction of
  the voltage source matches the counterclockwise direction specified,
  so the voltage is positive.

  Therefore the equation and solution are given by
  \begin{eqnarray*}
    2I_1 + 4I_1 + 2 I_1 &=& 18, \\
    8I_1 &=& 18, \\
    I_1 &=& \frac{9}{4} \amp.
  \end{eqnarray*}
  Since the answer is positive, this confirms that the current flows
  counterclockwise.
\end{solution}

\begin{example}{Solving for current}{solve-current2}
  Applying Kirchhoff's Law to the diagram below, determine the value for $I_1$.

  \begin{center}
    \scalebox{0.8}{
      \begin{circuitikz}[american,scale=0.8] \draw
        (0,0) to [R = $4\ohm$] (0,4)
        (2,4) to [battery1, v_= \raisebox{1ex}{$27\volt$}] (0,4)
        (2,4) to [R = $3\ohm$] (4,4)
        to [R = $1\ohm$] (4,0)
        to [R =$6\ohm$] (0,0)
        (2,2) node[scale=4]{$\circlearrowleft$}
        (2,2) node{$I_1$}
        ;
      \end{circuitikz}
    }
  \end{center}

\end{example}

\begin{solution}
  Begin in the top left corner this time, and trace the circuit in the
  counterclockwise direction. At the first resistor, multiplying
  resistance and current gives $4I_1$. Continuing in this way through
  the four resistors gives $4I_1 + 6I_1 + 1 I_1 + 3I_1$. This must
  equal the voltage source in the same direction. Notice that the
  direction of the voltage source is opposite to the counterclockwise
  direction, so the voltage is negative.

  Therefore the equation and solution are given by
  \begin{eqnarray*}
    4I_1 + 6I_1 + 1 I_1 + 3I_1&=& -27, \\
    14I_1 &=& -27, \\
    I_1 &=& -\frac{27}{14} \amp.
  \end{eqnarray*}
  Since the answer is negative, this tells us that the current flows
  clockwise.
\end{solution}

A more complicated example follows. Two of the circuits below may be
familiar; they were examined in the examples above. However as they
are now part of a larger system of circuits, the answers will differ.

\begin{example}{Unknown currents}{unknown-current}
  The diagram below consists of four circuits. The current ($I_k$) in
  the four circuits is denoted by $I_1$, $I_2$, $I_3$, $I_4$. Using
  Kirchhoff's Law, write an equation for each circuit and solve for
  each current.

  \begin{center}
    \scalebox{0.8}{
      \begin{circuitikz}[american, scale=0.7] \draw
        (0,0) to [battery1, v^= $18\volt$~~] (0,4)
        (0,0) to [R = $2 \ohm$] (4,0)
        to [R = $4 \ohm$] (4,4)
        (0,4) to [R =$2 \ohm$] (4,4)
        (6,4) to [battery1, v_= \raisebox{1ex}{$27\volt$}] (4,4)
        (6,4) to [R = $3 \ohm$] (8,4)
        to [R = $1 \ohm$] (8,0)
        (4,0) to [R = $6 \ohm$] (8,0)
        to [R = $2 \ohm$] (8,-4)
        to [R = $3 \ohm$] (4,-4)
        to [R = $1 \ohm$] (4,0)
        (4,-4)to [R = $5 \ohm$] (0,-4)
        (0,0) to [battery1, v_= $23\volt$~~] (0,-4)
        (2,2) node[scale=3]{$\circlearrowleft$}
        (2,2) node{$I_2$}
        (6,2) node[scale=3]{$\circlearrowleft$}
        (6,2) node{$I_3$}
        (6,-2) node[scale=3]{$\circlearrowleft$}
        (6,-2) node{$I_4$}
        (2,-2) node[scale=3]{$\circlearrowleft$}
        (2,-2) node{$I_1$}
        ;
      \end{circuitikz}
    }
  \end{center}
\end{example}

\begin{solution}
  Starting with the top left circuit, multiply the resistance by the
  current and sum the resulting products. Specifically, consider the
  resistor labelled $2 \ohm$ that is part of the circuits of $I_1$ and
  $I_2$. Notice that current $I_2$ runs through this in a positive
  (counterclockwise) direction, and $I_1$ runs through in the opposite
  (negative) direction. The product of resistance and current is then
  $2 (I_2 - I_1) = 2I_2 - 2I_1$.  Continue in this way for each
  resistor, and set the sum of the products equal to the voltage
  source to write the equation:
  \begin{equation*}
    2I_2-2I_1+4I_2-4I_3+2I_2=18.
  \end{equation*}
  The above process is used on each of the other three circuits, and
  the resulting equations are:

  \noindent
  Upper right circuit:
  \begin{equation*}
    4I_3 - 4I_2 + 6I_3 - 6I_4 + I_3 + 3I_3 = -27.
  \end{equation*}
  Lower right circuit:
  \begin{equation*}
    3I_4 + 2I_4 + 6I_4 - 6I_3 + I_4 - I_1 = 0.
  \end{equation*}
  Lower left circuit:
  \begin{equation*}
    5I_1+I_1-I_4+2I_1-2I_2=-23.
  \end{equation*}

  Notice that the voltage for the upper right and lower left circuits
  are negative due to the clockwise direction they indicate. The
  resulting system has four equations in four variables.  Simplifying
  and rearranging with variables in order, we have:
  \begin{equation*}
    \begin{array}{r@{~}c@{~}l}
      -2I_1+8I_2-4I_3&=&18, \\
      - 4I_2 + 14I_3 - 6I_4 &=& -27, \\
      -I_1 - 6I_3 + 12I_4 &=& 0, \\
      8I_1-2I_2 - I_4 &=& -23.
    \end{array}
  \end{equation*}
  The augmented matrix is
  \begin{equation*}
    \begin{mymatrix}{rrrr|r}
      -2 & 8 & -4 & 0 & 18 \\
      0 & -4 & 14 & -6 & -27 \\
      -1 & 0 & -6 & 12 & 0 \\
      8 & -2 & 0 & -1 & -23
    \end{mymatrix}.
  \end{equation*}
  The solution to this system of equations is
  \begin{eqnarray*}
    I_1 &=& -3 \amp, \\
    I_2 &=& \frac{1}{4} \amp, \\
    I_3 &=& -\frac{5}{2} \amp, \\
    I_4 &=& -\frac{3}{2} \amp.
  \end{eqnarray*}
  This tells us that currents $I_1, I_3$, and $I_4$ travel clockwise
  while $I_2$ travels counterclockwise.
\end{solution}


\subsection{Harmonic Functions}
By the open mapping theorem, a non-constant function $u:D\to\mathbb R$ where $D$ is a domain cannot be analytic.
However, it can be harmonic
\begin{definition}
    Let $D\subset\mathbb C$ be a domain.
    A smooth function $u:D\to\mathbb R$ is harmonic if
    $$\nabla^2 u=\frac{\partial^2u}{\partial x^2}+\frac{\partial^2u}{\partial y^2}=0$$
\end{definition}
\begin{lemma}\label{harmonic_disk}
    Consider a disk $D\subset\mathbb C$, a function $u:D\to\mathbb R$ is harmonic iff $u=\operatorname{Re}(f)$ for an analytic $f$ on $D$.
\end{lemma}
\begin{proof}
    The ``if'' direction is trivial by the Cauchy-Riemann Equations.
    The ``only if'' direction is exercise.
\end{proof}
\begin{definition}
    Let $R$ be a Riemann surface.
    A function $u\to\mathbb R$ is harmonic if for any chart $(\phi,U)$ on $R$ the composition
    $$u\circ\phi^{-1}:U\to\mathbb R$$
    is harmonic.
\end{definition}
\begin{lemma}
    A real function $u$ on $R$ is harmonic iff for any $p\in R$ there exists one chart $(\phi,U)$ on $R$ such that $u\circ\phi^{-1}$ is harmonic.
\end{lemma}
\begin{proof}
    The ``only if'' direction is trivial.
    For the ``if'' direction, we know from Lemma \ref{harmonic_disk} that for any $p\in R$, there is a chart $(\phi,U)$ with $p\in U$ such that $u\circ\phi^{-1}=\operatorname{Re}f$ for some analytic $f$ on a disk contained in $\phi(U)$.
    Hence for any chart $(\psi,V)$ and any $p\in V$, let $(\phi,U)$ be as above,
    $$u\circ\psi^{-1}=(u\circ\phi^{-1})\circ(\phi\circ\psi^{-1})=\operatorname{Re}(f\circ(\phi\circ\psi^{-1}))$$
    is harmonic near $p$.
    Hence $u$ is harmonic.
\end{proof}
\begin{proposition}[Identity Principle for Harmonic Functions]
    Let $u,v$ be harmonic functions on a Riemann surface $R$.
    Then the set $\{p\in R:u(p)=v(p)\}$ is either $R$ or discrete.
\end{proposition}
\begin{proof}
    Exercise.
\end{proof}
\begin{theorem}[Open Mapping Theorem for Harmonic Functions]
    Any non-constant harmonic function $u$ on a Riemann surface $R$ is an open map.
\end{theorem}
\begin{proof}
    Let $W\subset R$ be open and $p\in W$.
    For small enough $U\ni p$ that is contained in $W$ there is a chart $\phi:U\to\mathbb C$ and analytic $f$ such that $u\circ\phi^{-1}=\operatorname{Re}f$.
    The theorem then follows from the identity principle and open mapping theorem for analytic functions.
\end{proof}
\begin{corollary}
    If $R$ is a compact Riemann surface, all harmonic functions on $R$ are constant.
\end{corollary}