\section{The Physical System and Modeling}
\begin{frame}\frametitle{On Models and Modeling}
  \begin{itemize}
	\item Mathematical models are an abstraction of reality.
	\item There are three types of responses.
	\begin{enumerate}
		\item Type I\\
		$\ y'=\alpha y$
		\item Type II\\
		$\ y' = \frac{\alpha y}{y+ \beta}$
		\item Type III\\
		$\ y'= \frac {\alpha y^2}{\beta + y^2}$
	\end{enumerate}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Type I Response}
   Type I: $y'=\alpha y$\\ Applied to phenomena including chemical reaction kinetics.
   \includegraphics[scale=.4]{Rplot.png}
\end{frame}

\begin{frame}
  \frametitle{Type II Response}

    Type II: $y'=\frac{\alpha y}{y+L}$\\Models population growth until carrying capacity is reached.
   \includegraphics[scale=0.4]{Rplot01.png}

   % Type II: $y'=\frac{\alpha y}{y+\beta}$\\Models population growth until carrying capacity is reached.
  % \includegraphics[natwidth=162bp, natheight=227bp,width=280bp]{Rplot02.png}
\end{frame}

\begin{frame}
  \frametitle{Type III Response}
    Type III: $y'=y(1-y)$. Used to model neuronal action potentials.
    \includegraphics[scale=.4]{Rplot02.png}
\end{frame}
\begin{frame}
    Type III: $y'=\frac{\alpha y^2}{\beta+y^2}$. Used to model neuronal action potentials.
    \includegraphics[scale=.4]{Rplot01.png}
\end{frame}

\begin{frame}
\frametitle{Coral Reef Dynamics}
\hspace{1.5em}Coral reefs serve as a complex aquatic ecosystem physically supported by calcium carbonate secretions of the colorful stone coral. To model this ecosystem, one approach taken by Li-Wang-Zhang-Hastings is to describe the relationships between macroalgae, algal turfs, and corals \cite{Hastings}.
\end{frame}

\begin{frame}
\frametitle{Coral Reef Dynamics}
\includegraphics[natwidth=162bp,natheight=227bp,width=280bp]{./coral-reef-triangle.png}
\end{frame}
\begin{frame}\frametitle{Coral Reef Dynamics}
$$\begin{cases}\begin{array}{rl}
\frac{dM}{dt}\hspace{-.8em}&=aMC - \frac{gM}{M+T}+\gamma M T\\
\frac{dC}{dt}\hspace{-.8em}&=rTC - dC - aMC\\
\frac{dT}{dt}\hspace{-.8em}&=\frac{gM}{M + T} - \gamma MT - rTC + dC
\end{array}\end{cases},$$ where \begin{itemize}\itemsep0pt
\item $r$ is the rate corals overgrow upon algal turfs\\
\item $d$ is the mortality rate of corals\\
\item $a$ is the rate that macroalgae overgrow upon corals\\
\item $\gamma$ is the rate that macroalgae spread over algal turfs\\
\item $g$ is the indiscriminate grazing rate of parrotfish.
\end{itemize}
\end{frame}

\begin{frame}\frametitle{Coral Reef Dynamics}
\hspace{1.57em}Note that our scope is limited to regions entirely covered by coral, macroalgae, and algal turf, as $\frac{dT}{dt}=-\frac{dM}{dt}-\frac{dC}{dt}$ implies $M+C+T=constant$, say $constant = 1$.  We reduce our system to, $$\begin{cases}
\begin{array}{rl}
\frac{dM}{dt}&= aMC-\frac{gM}{M+T} + \gamma MT\\
\frac{dC}{dt}&=rTC-dC-aMC
\end{array} \end{cases}.$$
\end{frame}

\begin{frame}
  \frametitle{Nondimensionalization}
  \hspace{1.57em}To simplify systems of equations we scale problems using the technique of nondimensionalization: set {\begin{itemize}\itemsep0pt\item $M(t)=\overline{M}\HAT{M}(s)$;\\\item $C(t)=\overline{C}\HAT{C}(s)$;\\\item $T(t)=\overline{T}\HAT{T}(s)$, and;\\\item $t=\tau s$. \end{itemize} From this we get: \begin{itemize}\itemsep0pt \item$\frac{dM}{dt}=\overline{M}\HAT{M}'\frac{ds}{dt}$;\\\item $\frac{dC}{dt}=\overline{C}\HAT{C}'\frac{ds}{dt}$;\\\item $\frac{dT}{dt}=\overline{T}\HAT{T}'\frac{ds}{dt}$, and;\\ \item $\frac{ds}{dt}=\frac{1}{\tau}.$\end{itemize} }
  \end{frame}
 \begin{frame}
  \frametitle{Nondimensionalization}Our system then becomes: $$\begin{cases}
\begin{array}{rl}
\frac{\overline{M}}{\tau}\frac{d\HAT{M}}{dt}&= a\overline{M}\overline{C}\HAT{M}\HAT{C}-\frac{g\overline{M}\HAT{M}}{\overline{M}\HAT{M}+\overline{T}\HAT{T}} + \gamma \overline{M}\HAT{M}\overline{T}\HAT{T}\\
\frac{\overline{C}}{\tau}\frac{d\HAT{C}}{dt}&=r\overline{T}\overline{C}\HAT{T}\HAT{C}-d\overline{C}\HAT{C}-a\overline{M}\overline{C}\HAT{M}\HAT{C}
\end{array} \end{cases}$$
\end{frame}

{\center
\begin{frame}\frametitle{Nondimensionalization}Set $\overline{T}=\frac{r}{\gamma}$, $\overline{C}=\frac{r^2}{a\gamma}$, $\overline{M}=1$, $\tau=\frac{\gamma}{r^2}$. Then, $$\begin{cases}
\begin{array}{rl}
\frac{d\HAT{M}}{dt}&= \HAT{M}(a\tau\overline{C}\HAT{C}-\frac{g\tau}{\overline{M}\HAT{M}+\overline{T}\HAT{T}} + \gamma \tau\overline{T}\HAT{T})\\
\frac{d\HAT{C}}{dt}&=\HAT{C}(r\tau\overline{T}\HAT{T}-d\tau-a\tau\overline{M}\overline{C}\HAT{M})
\end{array} \end{cases}$$
\end{frame}
}

\begin{frame}\frametitle{Nondimensionalization}$$\begin{cases}
\begin{array}{rl}
\frac{d\HAT{M}}{dt}&= \HAT{M}(\HAT{C}+\HAT{\gamma}\HAT{T}+\frac{\HAT{g}\gamma^2}{\gamma\HAT{M}+ \HAT{T}}), \\
\frac{d\HAT{C}}{dt}&=\HAT{C}(\HAT{T}-\HAT{M}-\HAT{d}\HAT{\gamma}),
\end{array} \end{cases}$$ where $\HAT{d}=\frac{d}{r}$, $\HAT{\gamma} =\frac{\gamma}{r}$, $\HAT{g}=\frac{g}{\gamma^2}$,  Recall, \begin{itemize}\itemsep0pt
\item $r$ is the rate corals overgrow upon algal turfs\\
\item $d$ is the mortality rate of corals\\
\item $a$ is the rate that macroalgae overgrow upon corals\\
\item $\gamma$ is the rate that macroalgae spread over algal turfs\\
\item $g$ is the indiscriminate grazing rate of parrotfish.
\end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Blood Coagulation}
  Blood coagulation in response to an injury occurs in three stages:\begin{enumerate}
  \item Initiation Stage\\
  \item Amplification Stage\\
  \item Propagation Stage
  \end{enumerate} \end{frame}
  \begin{frame}\frametitle{Initiation stage}
 \hspace{1.57em}Injury exposes tissue factor to the site of injury thereby initiating the coagulation process. TF + factor VII = TF-VIIa complex activates factors IX and X. X activates thrombin before plasma protease inhibitor (TFPI) binds to and inactivates TF, Xa, and finally TF-VIIa which terminates this stage.
 \end{frame}
 \begin{frame}\frametitle{Amplification Stage}
 \hspace{1.57em}Thrombin activates platelets and factor V and VIII which drives the formation of a coagulation complex (IXa-VIIIa-Ca-PL, Xa-Va-Ca-PL) serving as an initial plug of the injury. TFPI inactivates thrombin terminating this stage.
 \end{frame}\begin{frame}\frametitle{Propagation Stage}
 \hspace{1.57em}Increasing platelet production of thrombin leads to thrombin's cleavage of the initial plug's fibrinogen into fibrin to which the final plug owes its strength.
\end{frame}

\begin{comment}
\begin{frame}\frametitle{Clotting Factor Models}
\begin{itemize}
\item 1989 Khamin-Semenov Model - Treated Coagulation as a three stage process as previously described.\\
\item 2002 Xu Model extends the K-S Model to be more applicable to bleeding pathologies.\\
\item 2004 Qiao et. al. Model focuses on the role of activated protein C (factor XIV)
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{1989 Khamin-Semenov Model}
$$\begin{cases}\begin{array}{rl}
\frac{dVIIa}{dt}\hspace{-.8em}&=\alpha k_1- H_4[VIIa]\\
\frac{dXa}{dt}\hspace{-.8em}&=k_2[VIIa]-H_2[Xa]\\
\frac{dVa}{dt}\hspace{-.8em}&=k_3[IIa]-H_3[Va]\\
\frac{dIIa}{dt}\hspace{-.8em}&=k_4[Xa]\frac{[Va]}{k_a+[Va]}-H_4[IIa]
\end{array}\end{cases},$$ where $\{k_i\}$ are kinetic coefficients and $\{H_i\}$ are dissipation constants. Xu notes that this model lacks the role that platelets play in this model.
\end{frame}
\end{comment}

\begin{frame}\frametitle{Coagulation Cascade}
\includegraphics[scale=.4]{./coag-cascade}
\end{frame}

\begin{frame}
\frametitle{2002 Xu Model \cite{Xu}}\vspace{-1.5em}
$$\begin{cases}\begin{array}{rl}
\frac{d(TF-VIIa)}{dt}\hspace{-.8em}&=[TF-VIIa]_0-h'_1[TF-VIIa]([TFPI][Xa]k_{14})\\
\frac{dXa}{dt}\hspace{-.8em}&=k_2[TF-VIIa]+k_{21}[IXa]\left(\frac{VIIIa}{VIIIa+d_2}\right)\left(c_0+\frac{f(IIa)}{1+f(IIa)}\right)\\&\hspace{1em}-h'_{21}[TFPI][Xa]-h'_{22}[AT][Xa]\\
\frac{dIXa}{dt}\hspace{-.8em}&=k_3[TF-VIIa]-h'_3[AT][IXa]\\
\frac{dVa}{dt}\hspace{-.8em}&=k'_5[IIa][V]-h'_5[Va]\\
\frac{dVIIIa}{dt}\hspace{-.8em}&=k'_6[IIa][VIII]-h'_6[VIIIa]\\
\frac{dIIa}{dt}\hspace{-.8em}&=k_4[Xa]\frac{[Va]}{[Va]+d_1}\left(c_0+\frac{f(IIa)}{1+f(IIa)}\right)-h'_4[IIa]
\end{array}\end{cases},$$ where $\{k'_i\}$ are kinetic coefficients and $\{h'_i\}$ are dissipation constants.

\hspace{1.57em}The role that platelets have in this model is realized by the term $c=c_0+\frac{[IIa]}{1+[IIa]}$, the proportion of platelets that have be activated by thrombin.% (Xu asserts this is a strict, twice differentiable function of $[IIa]$ whose codomain is $[0,1]$).
\end{frame}
\begin{comment}
\begin{frame}
\frametitle{2004 Qiao et. al. Model}\vspace{-1.5em}
$$\begin{cases}\begin{array}{rl}
\frac{d[IXa]}{dt}\hspace{-.8em}&=k_1\beta-h_1[IX_a]\\
\frac{d[VIIIa]}{dt}\hspace{-.8em}&=k_2[IIa]+k_3[X_a]-k_4[APC]\frac{[VIIIa]}{b_1+[VIIIa]}-h_2[VIIIa]\\
\frac{d[Xa]}{dt}\hspace{-.8em}&=k_5[IXa]\frac{[VIIIa]}{b_2+[VIIIa]}-h_3[Xa]\\
\frac{d[Va]}{dt}\hspace{-.8em}&=k_6[IIa]-k_7[APC]\frac{Va}{b_3+Va}-h_4[V_a]\\
\frac{d[APC]}{dt}\hspace{-.8em}&=k_8[IIa]-h_5[APC]\\
\frac{d[IIa]}{dt}\hspace{-.8em}&=k_9[Xa]\frac{[Va]}{b_4+[Va]}-h_6[IIa]
\end{array}\end{cases},$$ where $\{k'_i\}$ are kinetic coefficients and $\{h'_i\}$ are dissipation constants.
\end{frame}

\begin{frame}\frametitle{Moiseyev and Friends' Random Fibrin Walk}
Basis for the model is the 1998 Keener-Snead model: $$\frac{\partial n}{\partial t}+V\frac{\partial n}{\partial x}=F(1-n)-Gn,$$ where \begin{itemize}\item$n(x,t)dx$ is the fraction of connected links in the interval $[x,x+dx]$ at time $t$;\item $F(x)$ is the rate of link formation; \item $G(x)$ rate of lin dissociation; \item $V$ is the velocity field representing a shear flow.\end{itemize}
\end{frame}
\end{comment}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "Presentation1"
%%% End:


%%%%%%%%%%%%%%%%%%%%%%% Minimum Spanning Tree %%%%%%%%%%%%%%%%%%%%%%%

\section{Minimum Spanning Tree}

\begin{frame}
  \frametitle{Minimum Spanning Tree}

    \begin{itemize}

    \item (\textcolor{purple}{Evaluate Prim's MST algorithm implemented with min-heap} ([\textcolor{blue}{$P_{417}$ 8.9}]):) \\
        \begin{enumerate}
          \item Find the asymptotic order of the number of comparisons of edge weights in the worst case.
          \item Find the asymptotic order of the number of comparisons of edge weights on a \textcolor{blue}{bounded-degree} family.
          \item Find the asymptotic order of the number of comparisons of edge weights on a \textcolor{blue}{planar graph}.
        \end{enumerate}

    \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{Minimum Spanning Tree}

    \[
      T(n,m) = O \big (nT(getMin) + nT(deleteMin) + mT(decreaseKey) \big ). (\lbrack \textcolor{blue}{P_{395}} \rbrack)
    \]

    \pause

    \begin{itemize}
      \item $getMin: O(1)$.  \\  \emph{no comparison of edge weights}.
      \item $deleteMin: O(\log n)$.  \\  \textcolor{red}{NOT}: $O(\log m)$.
      \item $decreaseKey: O(\log n)$.  \\ \textcolor{red}{NOT}: $O(n + \log n)$ where $O(n)$ for search ([\textcolor{blue}{$P_{296}$}]).
      \item ``Else if newWgt less than fringeWgt for w'' ([\textcolor{blue}{$P_{395}$}]) : $O(m).$
      \item \textcolor{blue}{In total,}
      \[
      T(n,m) = O(n \log n + m \log n + 2m) = O((n+m) \log n).
      \]
    \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{Minimum Spanning Tree}

  \begin{itemize}
      \item
        $m \le \frac{nk} {2}, \Rightarrow T(n,m) = O((n+m) \log n) = O((n+ \frac{nk}{2}) \log n) = O(n \log n).$

        \pause

      \item
        First, we should know the relation between $m$ and $n$.
        \begin{equation}
          n - m + f = 2. \label{eq: EulerExtension}
        \end{equation}

        If $deg(f_i) \ge l$,
        \begin{equation}
          2m = \sum_{i=1}^f deg(R_i) \ge lf. \label{eq: Handshaking}
        \end{equation}
        \begin{equation}
          m \le \frac{l}{l-2}(n-2)
        \end{equation}

        \pause

        If $G$ is tree, $m=n-1$; Else, $l \ge 3$.
        \begin{equation}
          \textcolor{blue}{m \le \frac{l}{l-2}(n-2) \le 3(n-2) = 3n - 6}
        \end{equation}
        \[
          T(n,m) = O((n+m) \log n) = O((n + 3n) \log n) = O(n \log n).
        \]
  \end{itemize}

\end{frame}


\subsection{Linear finite element in 2D as a 2-depth ReLU DNN}\label{sec:2DFEMandDNN}
In this section, we will present our unexpected discovery pertaining to the
expressive power of ReLU DNNs, which derived from our investigation into ReLU DNNs
in relation to hierarchical basis methods.

We have
$$
s(x) = x^2, \quad m(x,y) = xy,
$$
and
$I_{\ell}$ means the interpolation on 1D and $\Pi_\ell$ means interpolation
on $[0,1]^2$ with mesh size $2^{-\ell}$.

In terms of the approximation of ReLU DNNs for $x^2$,
the most critical point is that
\begin{equation}\label{key}
	(I_{\ell} - I_{\ell-1})s(x) = -h_\ell^2\sum_{i\in \mathcal I_\ell}\phi_{\ell,i}(x) = -h_\ell^2g_\ell(x) \in {\mathcal N}_\ell^3
\end{equation}
for all $x\in \mathbb{R}$. Thus, a key question arises:
\begin{quote}
	Is there composition structure for $(\Pi_\ell - \Pi_{\ell-1})m (x, y)$?
\end{quote}

We have the following
corollary for the explicit formula for $(\Pi_\ell- \Pi_{\ell-1})m$ on $[-1,1]^2$.
\begin{corollary}
	For any $(x,y) \in [-1,1]^2$, we have
	\begin{equation}\label{eq:Pi-Pi}
		\begin{aligned}
			&(\Pi_\ell- \Pi_{\ell-1})m (x,y) \\
			&= m_{\ell+2}(x,y) - m_{\ell+1}(x,y)  \\
			&= 2h^2_{\ell+1}\left(g_{\ell+1}\left(\frac{|x|}{2}\right)+g_{\ell+1}\left(\frac{|y|}{2}\right)-g_{\ell+1}\left(\frac{|x+y|}{2}\right)\right).
		\end{aligned}
	\end{equation}
	This means that $\psi_\ell \in \widehat{{\mathcal N}}_{3(\ell+2)}^3$ where $\psi_\ell = (\Pi_\ell- \Pi_{\ell-1})m$. Furthermore,
	it is worth noting that we can actually have $\psi_\ell \in {\mathcal N}_{\ell+2}^9$ since $g_{\ell+1}(\frac{|x+y|}{2}) \in {\mathcal N}_{\ell+2}^3$.
\end{corollary}

By choosing a suitable scale, we have $\|h_{\ell}^{-2}(\Pi_\ell- \Pi_{\ell-1})m\|_{L^{\infty}([-1,1]^2)} = 1$.
Fig.~\ref{fig:T2T3} shows function graphs of $h_2^{-2}(\Pi_2 - \Pi_{1})m$ and $h_{3}^{-2}(\Pi_3- \Pi_{2})m$ on $[0,1]^2$.
\begin{figure}[h!]
	\centering
	\begin{minipage}[t]{0.43\textwidth}
		\centering
		\includegraphics[width=\textwidth]{6DL/figures/4basis}
	\end{minipage}
	\begin{minipage}[t]{0.43\textwidth}
		\centering
		\includegraphics[width=\textwidth]{6DL/figures/16basis}
	\end{minipage}
	\caption{Functions of $h_{\ell}^{-2}(\Pi_\ell- \Pi_{\ell-1})m$ for $\ell = 2,3$ on $[0,1]^2$.}
	\label{fig:T2T3}
\end{figure}

These graphs and meshes of $(\Pi_\ell - \Pi_{\ell-1})m(x,y)$ give rise to a natural discussion about
the representation theorem of ReLU DNN for the basis function of the 2D linear finite element.
By taking $\ell=1$ with a suitable scale, we have
\begin{equation}\label{eq:varphi_1}
	4(\Pi_1- \Pi_{0})m (x,y)
	= \frac{1}{2}\left(g_{2}\left(\frac{x}{2}\right)+g_2\left(\frac{y}{2}\right)-g_2\left(\frac{x+y}{2}\right)\right)
\end{equation}
for all $ (x,y) \in [0,1]^2$.
Here, $4(\Pi_1- \Pi_{0})m (x,y)$ equals the basis function $\varphi(x,y)$ for the
linear finite element (see left-hand graph of Fig.~\ref{fig:varphi_1})
on the uniform mesh on $[0,1]^2$ with mesh size $h_1=\frac{1}{2}$ (see right-hand graph of Fig.~\ref{fig:varphi_1}).
\begin{figure}[h!]
	\begin{minipage}[t]{0.43\linewidth}
		\centering
		\includegraphics[width=\textwidth]{6DL/figures/varphi}
	\end{minipage}
	\begin{minipage}[t]{0.45\linewidth}
		\centering
		\includegraphics[scale=.5]{6DL/figures/2D_Basis}
		%		\begin{tikzpicture}[scale = 2 ]
		%		%\fill[green] (0,0) -- (0,2) -- (2,0) -- cycle;
		%		\draw[gray, step = 1cm] (0, 0) grid (2, 2);
		%		\node[anchor = north] at (0, 0) {$(0,0)$};
		%		%			\node[anchor = north] at (1,0) {$6$};
		%		\node[anchor = north] at (2,0) {$(0,1)$};
		%		%			\node[anchor = east] at (0,1) {$(\frac{1}{2},\frac{1}{2})$};
		%		\node[anchor = south west] at (1,1) {$(\frac{1}{2},\frac{1}{2})$};
		%		\node[anchor = south] at (0,2) {$(1,0)$};
		%		\node[anchor = south] at (2,2) {$(1,1)$};
		%		\draw(0,2) -- (2,0);
		%		\draw (0,1) -- (1,0);
		%		\draw (1,2) -- (2,1);
		%		\end{tikzpicture}
	\end{minipage}
	\caption{left: $\varphi(x,y)$ on $[0,1]^2$; right:  mesh for $\ell=1$ on $[0,1]^2$.}
	\label{fig:varphi_1}
\end{figure}
Without loss of generality, we can define $\varphi(x,y)$ globally as
\begin{equation}\label{key}
	\varphi(x,y) = \begin{cases}
		4(\Pi_1- \Pi_{0})m (x,y), \quad &(x,y) \in [0,1]^2, \\
		0, \quad &\text{others}.
	\end{cases}
\end{equation}
However, this representation,
\begin{equation}\label{eq:varphi_1-1}
	\varphi(x,y) = \frac{1}{2}\left(g_{2}\left(\frac{x}{2}\right)+g_2\left(\frac{y}{2}\right)-g_2\left(\frac{x+y}{2}\right)\right),
\end{equation}
holds only for $(x,y) \in [0,1]^2$.
Related to this observation is the simple argument that
\begin{equation}\label{key}
	\frac{1}{2}\left(g_{2}\left(\frac{x}{2}\right)+g_2\left(\frac{y}{2}\right)-g_2\left(\frac{x+y}{2}\right)\right) = \frac{1}{2} \neq 0
\end{equation}
if $(x,y) = (\frac{3}{2}, \frac{3}{2})$.
On the other hand, on the assumption that the identity in \eqref{eq:varphi_1-1} holds for all $(x,y) \in \mathbb{R}^2$, we can rewrite $g_2(x)$ as
\begin{equation}\label{eq:defg2_2}
	g_2(x) = \sum_{i=0}^4 \alpha_i {\rm ReLU}\left(x-\frac{i}{4}\right),
\end{equation}
for all $x\in \mathbb{R}$ where $(\alpha_0,\alpha_1,\cdots, \alpha_4) = (4, -8, 8, -8, 4)$ based on the relations between the linear finite element functions and ReLU DNNs on 1D~\cite{he2020relu}.
This means that $g_2 \in {\mathcal N}_{1}^5$ and $\varphi(x,y)$ can be represented by a ReLU DNN with only one hidden layer on the entire space $\mathbb{R}^2$.
However, this representation is contradictory to the theorem in~\cite{he2020relu} that the locally supported basis function of the 2D linear finite element cannot be represented globally by ReLU DNN  with just one hidden layer.

Given the global representation of $\varphi(x,y)$, \cite{he2019finite} constructs
a ReLU DNN with four hidden layers to reproduce $\varphi(x,y)$ explicitly.
Although \cite{arora2018understanding,he2020relu} show that
a two-hidden-layer ReLU DNN should be able to represent $\varphi(x,y)$ globally on $\mathbb{R}^2$,
the structure of network will be extremely complicated, as well as it requires a
large number of neurons. Thus, to find a concise formula to
reproduce $\varphi(x,y)$ on $\mathbb{R}^2$ directly becomes the focus of the inquiry.
Based on the discovery in \eqref{eq:varphi_1-1} and the properties of ReLU DNNs,
we can construct a ReLU DNN function with two hidden layers to represent $\varphi(x,y)$.
To simplify the statement of that result, let us first denote the following
${\rm ReLU1}(x)$ function:
\begin{equation}\label{eq:defReLU1}
	{\rm ReLU1}(x)  := {\rm ReLU}(x) - {\rm ReLU}(x-1) \in {\mathcal N}_1^2.
\end{equation}

\begin{lemma}
	The basis function $\varphi(x,y)$ is in  ${\mathcal N}_2^{15}$, more precisely, we have
	\begin{equation}\label{eq:b-DNN2}
		%\widehat b(x,y) :=
		{\varphi}(x,y) = \frac{1}{2}\left(g_{2}\left(\frac{{\rm ReLU1}(x)}{2}\right)+g_2\left(\frac{{\rm ReLU1}(y)}{2}\right)-g_2\left(\frac{{\rm ReLU1}(x)+{\rm ReLU1}(y)}{2}\right)\right)
	\end{equation}
	for all $(x,y) \in \mathbb{R}^2$.
\end{lemma}
\begin{proof}
	According to the definition of $g_2(x)$ in \eqref{func:glinduction} (or \eqref{eq:defg2_2}) and ${\rm ReLU1}(x)$ in \eqref{eq:defReLU1},
	we have $ \varphi(x,y) \in {\mathcal N}_2^{15}$ and
	\begin{equation}\label{key}
		\varphi(x,y) = 0
	\end{equation}
	for any $(x,y) \notin [0,1]^2$ as ${\rm ReLU1}(x)$ and ${\rm ReLU1}(y)$ will equal to $0$ or $1$.
	Then, \eqref{eq:b-DNN2} holds, given that ${\rm ReLU1}(x) = x$ and ${\rm ReLU1}(y) = y$ for $(x,y) \in [0,1]^2$.
\end{proof}

By employing the above lemma, we can construct the following theorem pertaining to the representation
of linear finite element functions by using ReLU DNNs with only two hidden layers on
two-dimensional space.
\begin{theorem}\label{thm:FEM-DNN}
	Assume $u_h$ is a two-dimensional linear finite element function, which can be written as
	\begin{equation}\label{key}
		u_h(x,y) = \sum_{i=1}^N \mu_i \varphi(T_i(x,y)),
	\end{equation}
	where $T_i : \mathbb{R}^2 \mapsto \mathbb{R}^2$ is an affine mapping and $N$ denotes
	the number of degree of freedom. Then, $u_h(x)$
	can be reproduced globally by a ReLU DNN with only two hidden layers and $15N$ neurons
	at most for each layer, i.e., $u_h(x,y)  \in {\mathcal N}_2^{15N}$.
\end{theorem}
According to~\cite{arora2018understanding}, any d-dimensional continuous piecewise
linear function can be represented by ReLU DNNs with at most $\lceil \log_2(d+1) \rceil$ hidden layers.
However, the representation theory in \cite{arora2018understanding} requires an extremely complicated
construction by induction with a tremendously high number of neurons. On the other hand, \cite{he2020relu}
shows that ReLU DNNs with only one hidden layer cannot be used to represent general linear
finite element functions even on two-dimensional space.
Thus, the focus inevitably turns to finding
a concise and explicit representation of linear finite element functions using ReLU DNNs with
only two hidden layers on two-dimensional space.
Here, Theorem~\ref{thm:FEM-DNN} provides the answer for any two-dimensional linear finite element functions on a uniform mesh or meshes, which can be obtained by adding affine mappings.


\lab{Polynomial Interpolation}{Polynomial Interpolation}
\label{lab:Polynomial Interpolation}
\objective{Learn and compare three methods of polynomial interpolation: standard Lagrange interpolation, Barycentric Lagrange interpolation and Chebyshev interpolation.
Explore Runge's phenomenon and how the choice of interpolating points affect the results.
Use polynomial interpolation to study air polution by approximating graphs of particulates in air.}

\section*{Polynomial Interpolation}
Polynomial interpolation is the method of finding a polynomial that matches a function at specific points in its range.
More precisely, if $f(x)$ is a function on the interval $[a,b]$ and $p(x)$ is a polynomial then $p(x)$ interpolates the function $f(x)$ at the points $x_0,x_1,\dots ,x_n$ if $p(x_j)=f(x_j)$ for all $j=0,1,\dots,n$.
In this lab most of the discussion is focused on using interpolation as a means of approximating functions or data, however, polynomial interpolation is useful in a much wider array of applications.

Given a function $f(x)$ and a set of unique points $\{x_i\}_{i=0}^n$, it can be shown that there exists a unique interpolating polynomial $p(x)$.
That is, there is one and only one polynomial of degree $n$ that interpolates $f(x)$ through those points.
This uniqueness property is why, for the remainder of this lab, an interpolating polynomial is referred to as \emph{the} interpolating polynomial.
One approach to finding the unique interpolating polynomial of degree $n$ is Lagrange interpolation.

\begin{figure}
\captionsetup[subfigure]{justification=centering}
\captionsetup{justification=centering}
\centering
\begin{subfigure}{.5\textwidth}
    \centering
    \includegraphics[width=\linewidth]{figures/bad_interp1.pdf}
    \caption{Interpolation using 5 equally spaced points.}
    \label{fig:bad1}
\end{subfigure}%
\begin{subfigure}{.5\textwidth}
    \centering
    \includegraphics[width=\linewidth]{figures/bad_interp2.pdf}
    \caption{Interpolation using 11 equally spaced points.}
    \label{fig:bad2}
\end{subfigure}
\caption{Interpolations of Runge's function $f(x)=\frac{1}{1+25x^2}$ with equally spaced interpolating points.}
\label{fig:badinterp}
\end{figure}

\subsection*{Lagrange interpolation} %------------------------------------------------------------------------------------------------------------------------------------------------
Given a set $\{x_i\}_{i=1}^n$ of $n$ points to interpolate, a family of $n$ basis functions with the following property is constructed:
\[
L_j(x_i) = \begin{cases} 0 &\mbox{ if } i \neq j\\ 1 &\mbox{ if } i =j \\ \end{cases}.
\]

The Lagrange form of this family of basis functions is
\begin{equation}
\label{equa:lagrange}
L_j(x) = \frac{\displaystyle\prod_{k=1, k \neq j}^n (x-x_k)}{\displaystyle\prod_{k=1, k \neq j}^n (x_j-x_k)}
\end{equation}
Each of these Lagrange basis functions is a polynomial of degree $n-1$ and has the necessary properties as given above.

\begin{problem}
Define a function \li{lagrange()} that will be used to construct and evaluate an interpolating polynomial on a domain of $x$ values.
The function should accept two NumPy arrays of length $n$ which contain the $x$ and $y$ values of the interpolating points as well as a NumPy array of values of length $m$ at which the interpolating polynomial will be evaluated.

Within \li{lagrange()}, write a subroutine that will evaluate each of the $n$ Lagrange basis functions at every point in the domain.
It may be helpful to follow these steps:
\begin{enumerate}
\item Compute the denominator of each $L_j$ (as in Equation \ref{equa:lagrange}) .
\item Using the previous step, evaluate $L_j$ at all points in the computational domain (this will give you $m$ values for each $L_j$.)
\item Combine the results into an  $n \times m$ NumPy array, consisting of each of the $n$ $L_j$ evaluated at each of the $m$ points in the domain.
\end{enumerate}

You may find the functions \li{np.product()} and \li{np.delete()} to be useful while writing this method.
\label{prob:lagrange_basis}
\end{problem}

\emph{Lagrange interpolation} is completed by combining the Lagrange basis functions with the y-values of the function to be interpolated $y_i=f(x_i)$ in the following manner:
\begin{equation}
\label{equa:poly}
p(x) = \sum_{j=1}^n y_j L_j(x)
\end{equation}
This will create the unique interpolating polynomial.

Since polynomials are typically represented in their expanded form with coefficients on each of the terms, it may seem like the best option when working with polynomials would be to use Sympy, or NumPy's
\li{poly1d} class to compute the coefficients of the interpolating polynomial individually.
This is rarely the best approach, however, since expanding out the large polynomials that are required can quickly lead to instability (especially when using large
numbers of interpolating points).
Instead, it is usually best just to leave the polynomials in unexpanded form (which is still a polynomial, just not a pretty-looking one), and compute values of the polynomial directly from this unexpanded form.

\begin{lstlisting}
# Evaluate the polynomial (x-2)(x+1) at 10 points without expanding the expression.
>>> pts = np.arange(10)
>>> (pts - 2) * (pts + 1)
array([ 2,  0,  0,  2,  6, 12, 20, 30, 42, 56])
\end{lstlisting}
In the given example, there would have been no instability if the expression had actually been expanded but in the case of a large polynomial, stability issues can dominate the computation.
Although the coefficients of the interpolating polynomials will not be explicitly computed in this lab, polynomials are still being used, albeit in a different form.

\begin{problem}

Complete the implementation of \li{lagrange()}.

Evaluate the interpolating polynomial at each point in the domain by combining the $y$ values of the interpolation points and the evaluated Lagrange basis functions from Problem \ref{prob:lagrange_basis} as in Equation \ref{equa:poly}.
Return the final array of length $m$ that consists of the interpolating polynomial evaluated at each point in the domain.

You can test your function by plotting Runge's function $f(x)=\frac{1}{1+25x^2}$ and your interpolating polynomial on the same plot for different values of $n$ equally spaced interpolating values then comparing
your plot to the plots given in Figure \ref{fig:badinterp}.
\label{prob:lagrange}
\end{problem}

The Lagrange form of polynomial interpolation is useful in some theoretical contexts and is easier to understand than other methods, however, it has some serious drawbacks that prevent it from being a
useful method of interpolation.
First, Lagrange interpolation is $O(n^2)$ where other interpolation methods are $O(n^2)$ (or faster) at startup but only $O(n)$ at run-time,
Second, Lagrange interpolation is an unstable algorithm which causes it to return innacurate answers when larger numbers of interpolating points are used.
Thus, while useful in some situations, Lagrange interpolation is not desirable in most instances.

\subsection*{Barycentric Lagrange interpolation}
Barycentric Lagrange interpolation is simple variant of Lagrange interpolation that performs much better than plain Lagrange interpolation.
It is essentially just a rearrangement of the order of operations in Lagrange multiplication which results in vastly improved perfomance, both in speed and stability.

Barycentric Lagrange interpolation relies on the observation that each basis function $L_j$ can be rewritten as
\[
L_j(x) = \frac{w(x)}{(x-x_j)}w_j
\]
where
\[
w(x) = \prod_{j=1}^n (x-x_j)
\]
and
\[
w_j = \frac{1}{\prod_{k=1, k \neq j}^n (x_j-x_k)}.
\]
The $w_j$'s are known as the \emph{barycentric weights}.

Using the previous equations, the interpolating polynomial can be rewritten
\[
p(x) = w(x) \sum_{j=1}^n \frac{w_j y_j}{x-x_j}
\]
which is the \emph{first barycentric form}.
The computation of $w(x)$ can be avoided by first noting that
\[
1 = w(x) \sum_{j=1}^n \frac{w_j}{x-x_j}
\]
which allows the interpolating polynomial to be rewriten as
\[
p(x) = \frac{\displaystyle\sum_{j=1}^n \frac{w_j y_j}{x-x_j}}{\displaystyle\sum_{j=1}^n \frac{w_j}{x-x_j}}
\]
This form of the Lagrange interpolant is known as the \emph{second barycentric form} which is the form used in Barycentric Lagrange interpolation.
So far, the changes made to Lagrange interpolation have resulted in an algorithm that is $O(n)$ once the barycentric weights ($w_j$) are known.
The following adjustments will improve the algorithm so that it is numerically stable and later discussions will allow for the quick addition of new interpolating points after startup.

The second barycentric form makes it clear that any factors that are common to the $w_k$ can be ignored (since they will show up in both the numerator and denominator).
This allows for an important improvement to the formula that will prevent overflow error in the arithmetic.
When computing the barycentric weights, each element of the product $\prod_{k=1, k \neq j}^n (x_j-x_k)$ should be multiplied by $C^{-1}$, where $4C$ is the width of the interval being interpolated
(C is known as the \emph{capacity} of the interval).
In effect, this scales each barycentric weight by $C^{1-n}$ which helps to prevent overflow during computation.
Thus, the new barycentric weights are given by
\[
w_j = \frac{1}{\prod_{k=1, k \neq j}^n \left[(x_j-x_k) / C\right]}.
\]
Once again, this change is possible since the extra factor $C^{1-n}$ is cancelled out in the final product.
This process is summed up in the following code:

\begin{lstlisting}
# Given a NumPy array xint of interpolating x-values, calculate the weights.
>>> n = len(xint)                   # Number of interpolating points.
>>> w = np.ones(n)                  # Array for storing barycentric weights.
# Calculate the capacity of the interval.
>>> C = (np.<<max>>(xint) - np.<<min>>(xint)) / 4

>>> shuffle = np.random.permutation(n-1)
>>> for j in range(n):
>>>     temp = (xint[j] - np.delete(xint, j)) / C
>>>     temp = temp[shuffle]        # Randomize order of product.
>>>     w[j] /= np.product(temp)
\end{lstlisting}

The order of \li{temp} was randomized so that the arithmetic does not overflow due to poor ordering (if standard ordering is used, overflow errors can be encountered since all
of the points of similar magnitude are multiplied together at once).
When these two fixes are combined, the Barycentric Algorithm becomes numerically stable.

\begin{problem}
\label{prob:barycentric}
Create a class that performs Barycentric Lagrange interpolation.
The constructor of your class should accept two NumPy arrays which contain the $x$ and $y$ values of the interpolation points.
Store these arrays as attributes.
In the constructor, compute the corresponding barycentric weights and store the resulting array as a class attribute.
Be sure that the relative ordering of the arrays remains unchanged.

Implement the \li{__call__()} method so that it accepts a NumPy array of values at which to evaluate the interpolating polynomial and returns an array of the evaluated points.
Your class can be tested in the same way as the Lagrange function written in Problem \ref{prob:lagrange}
\end{problem}

\begin{warn}
As currently explained and implemented, the Barycentric class from Problem \ref{prob:barycentric} will fail when a point to be evaluated exactly matches one of the $x$-values of the interpolating points.
This happens because a divide by zero error is encountered in the final step of the algorithm.
The fix for this, although not required here, is quite easy: keep track of any problem points and replace the final computed value with the corresponding y-value (since this is a point that is exactly interpolated).
If you do not implement this fix, just be sure not to pass in any points that exactly match your interpolating values.
\end{warn}

Another advantage of the barycentric method is that it allows for the addition of new interpolating points in $O(n)$ time.
Given a set of existing barycentric weights $\{ w_j\}_{j=1}^n$ and a new interpolating point $x_i$, the new barycentric weight is given by
\[
w_i = \frac{1}{\prod_{k=1}^n (x_i-x_k)}.
\]
In addition to calculating the new barycentric weight, all existing weights should be updated as follows $w_j=\frac{w_j}{x_j-x_i}$.

\begin{problem}
\label{prob:add weights}
Include a method in the class written in Problem \ref{prob:barycentric} that allows for the addition of new interpolating points by updating the barycentric weights.
Your function should accept two NumPy arrays which contain the $x$ and $y$ values of the new interpolation points.
Update and store the old weights then extend the class attribute arrays that store the weights, and the $x$ and $y$ values of the interpolation points with the new data.
When updating all class attributes, make sure to maintain the same relative order.
\end{problem}

The implementation outlined here calls for the y-values of the interpolating points to be known during startup, however, these values are not needed until run-time
This allows the y-values to be changed without having to recompute the barycentric weights.
This is an additional advantage of Barycentric Lagrange interpolation.

\subsection*{Scipy's Barycentric Lagrange class}
Scipy includes a Barycentric interpolator class.
This class includes the same functionality as the class described in Problems \ref{prob:barycentric} and \ref{prob:add weights} in addition to the ability to update the y-values of the interpolation points.
The following code will produce a figure similar to Figure \ref{fig:bad2}.
\begin{lstlisting}
>>> from scipy.interpolate import BarycentricInterpolator

>>> f = lambda x: 1/(1+25 * x**2)   # Function to be interpolated.
# Obtain the Chebyshev extremal points on [-1,1].
>>> n = 11
>>> pts = np.linspace(-1, 1, n)
>>> domain = np.linspace(-1, 1, 200)

>>> poly = BarycentricInterpolator(pts[:-1])
>>> poly.add_xi(pts[-1])          # Oops, forgot one of the points.
>>> poly.set_yi(f(pts))           # Set the y values.

>>> plt.plot(domain, f(domain))
>>> plt.plot(domain, poly.<<eval>>(domain))
\end{lstlisting}

\begin{comment} # Potential timing problem comparing student written barycentr, lagrange and Scipy's Lagrange.  Superceded by later problem which also includes Chebyshev.
\begin{problem}
Compare the runtime and error of your Lagrange and Barycentric methods with Scipy's Barycentric class by writing two functions, one comparing error and the other run-time.
Both functions should accept a callable function to interpolate and an integer designating how many interpolating points should be used.
All computations should be made on 200 equally spaced points on the interval $[-1, 1]$ with the interpolating points coming from the Chebyshev roots.
Both functions should print the runtime or error of the interpolation for all three methods with an accompanying description (so that it is obvious which time/error corresponds to which method).
When calculating error, use \li{np.linalg.norm()} with \li{ord=np.inf}.
\end{problem}
\end{comment}

\begin{figure}[H]
\captionsetup[subfigure]{justification=centering}
\captionsetup{justification=centering}
\centering
\begin{subfigure}{.5\textwidth}
    \centering
    \includegraphics[width=\linewidth]{figures/good_interp1.pdf}
    \caption{Polynomial using 5 Chebyshev roots.}
    \label{fig:good1}
\end{subfigure}%
\begin{subfigure}{.5\textwidth}
    \centering
    \includegraphics[width=\linewidth]{figures/good_interp2.pdf}
    \caption{Polynomial using 11 Chebyshev roots.}
    \label{fig:good2}
\end{subfigure}
\caption{Example of overcoming Runge's phenomenon by using Chebyshev nodes for interpolating values.
Plots made using Runge's function $f(x)=\frac{1}{1+25x^2}$.
Compare with Figure \ref{fig:badinterp}}
\label{fig:goodinterp}
\end{figure}

\section*{Chebyshev Interpolation}
\subsection*{Chebyshev Nodes}
As has been mentioned previously, the Barycentric version of Lagrange interpolation is a stable process that does not accumulate large errors, even with extreme inputs.
However, polynomial interpolation itself is, in general, an ill-conditioned problem.
Thus, even small changes in the interpolating values can give drastically different interpolating polynomials.
In fact, poorly chosen interpolating points can result in a very bad approximation of a function.
As more points are added, this approximation can worsen.
This increase in error is called \emph{Runge's phenomenon}.

The set of equally spaced points is an example of a set of points that may seem like a reasonable choice for interpolation but in reality produce very poor results.
Figure \ref{fig:badinterp} gives an example of this using Runge's function.
As the number of interpolating points increases, the quality of the approximation deteriorates, especially near the endpoints.

Although polynomial interpolation has a great deal of potential error, a good set of interpolating points can result in fast convergence to the original function as the number of interpolating points is increased.
One such set of points is the Chebyshev extremal points which are related to the Chebyshev polynomials (to be discussed shortly).
%The $n$ Chebyshev roots on the interval $[a,b]$ are given by the formula $z_j=\frac{1}{2}(a+b + (b-a)\cos(\frac{\pi}{n}(j+\frac{1}{2})))$ for $j=0,1,\dots,n-1$.
The $n+1$ Chebyshev extremal points on the interval $[a,b]$ are given by the formula $y_j=\frac{1}{2}(a+b + (b-a)\cos(\frac{j\pi}{n}))$ for $j=0,1,\dots,n$.
These points are shown in Figure \ref{fig:extrema}.
One important feature of these points is that they are clustered near the endpoints of the interval, this is key to preventing Runge's phenomenon.

\begin{problem} % Compare error.
% Compare the error of interpolation using equally spaced points and interpolation using the Chebyshev extrema by writing a function that performs the following experiment.
Write a function that defines a domain $\x$ of $400$ equally spaced points on the interval $[-1, 1]$.
For $n=2^2,2^3,\ldots,2^8$, repeat the following experiment.
\begin{enumerate}
    \item Interpolate Runge's function $f(x) = 1/(1+25x^2)$ with $n$ equally spaced points over $[-1,1]$ using SciPy's \li{BarycentricInterpolator} class, resulting in an approximating function $\tilde{f}$.
    Compute the absolute error $\|f(\x) - \tilde{f}(\x)\|_\infty$ of the approximation using \li{la.norm()} with \li{<<ord>>=np.inf}.
    \item Interpolate Runge's function with $n+1$ Chebyshev extremal points, also via SciPy, and compute the absolute error.
    % Note that the definition for Chebyshev extremal points uses $n+1$ interpolating values.
\end{enumerate}
Plot the errors of each method against the number of interpolating points $n$ in a log-log plot.

To verify that your figure make sense, try plotting the interpolating polynomials with the original function for a few of the larger values of $n$.
\end{problem}

\begin{figure}[H]
\centering
\includegraphics{figures/extrema.pdf}
\caption{The Chebyshev extremal points.  The $n$ points where the Chebyshev polynomial of degree $n$ reaches its local extrema.
These points are also the projection onto the x-axis of $n$ equally spaced points around the unit circle.}
\label{fig:extrema}
\end{figure}

\subsection*{Chebyshev Polynomials}
The Chebyshev roots and Chebyshev extremal points are closely related to a set of polynomials known as the Chebyshev polynomials.
The first two Chebyshev polynomials are defined as $T_0(x)=1$ and $T_1(x)=x$.
The remaining polynomials are defined by the recursive algorithm $T_{n+1}(x)=2xT_n(x)-T_{n-1}(x)$.
The Chebyshev polynomials form a complete basis for the polynomials in $\mathbb{R}$ which means that for any polynomial $p(x)$,  there exists a set of unique coefficients $\{a_k\}_{k=0}^n$
such that
\[
p(x) = \sum_{k=0}^n a_kT_k.
\]

Finding the Chebyshev representation of an interpolating polynomial is a slow process (dominated by matrix multiplication or solving a linear system), but when the interpolating values are the
Chebyshev extrema, there exists a fast algorithm for computing the Chebyshev coefficients of the interpolating polynomial.
This algorithm is based on the Fast Fourier transform which has temporal complexity $O(n\log n)$.
Given the $n+1$ Chebyshev extremal points $y_j=\cos(\frac{j\pi}{n})$ for $j=0,1,\dots,n$ and a function $f$, the unique n-degree interpolating polynomial $p(x)$ is given by
\[
p(x)=\sum_{k=0}^na_kT_k
\]
where
\[
a_k = \gamma_k \Re \left[ DFT(f(y_0), f(y_1),\dots, f(y_{2n-1}))\right]_k.
\]
Note that although this formulation includes $y_j$ for $j>n$, there are really only $n+1$ distinct values being used since $y_{n-k}=y_{n+k}$.
Also, $\Re$ denotes the real part of the Fourier transform and $\gamma_k$ is defined as
\[
\gamma_k =
\begin{cases}
1 & k\in \{0,n\} \\
2 & \text{otherwise.}
\end{cases}
\]

\begin{problem}
Write a function that accepts a function $f$ and an integer $n$.
Compute the $n+1$ Chebyshev coefficients for the degree $n$ interpolating polynomial of $f$ using the Fourier transform (\li{np.real()} and \li{np.fft.fft()} will be helpful).
When using NumPy's \li{fft()} function, multiply every entry of the resulting array by the scaling factor $\frac{1}{2n}$ to match the derivation given above.

Validate your function with \li{np.polynomial.chebyshev.poly2cheb()}.
The results should be exact for polynomials.
\begin{lstlisting}
# Define f(x) = -3 + 2x^2 - x^3 + x^4 by its (ascending) coefficients.
>>> f = lambda x: -3 + 2*x**2 - x**3 + x**4
>>> pcoeffs = [-3, 0, 2, -1, 1]
>>> ccoeffs = np.polynomial.chebyshev.poly2cheb(pcoeffs)

# The following callable objects are equivalent to f().
>>> fpoly = np.polynomial.Polynomial(pcoeffs)
>>> fcheb = np.polynomial.Chebyshev(ccoeffs)
\end{lstlisting}
\end{problem}

\begin{comment}
Once the coefficients of the Chebyshev polynomial have been computed, there exists a fast algorithm (discussed in the Additional Material section) for evaluating the polynomials.
NumPy's \li{polynomial.chebyshev} module contains a method for doing this.
\begin{lstlisting}
>>> from NumPy.polynomial.chebyshev import chebval

>>> domain = np.linspace(-1, 1, 5)
>>> f = lambda x: x**4              # Function to interpolate.
>>> coeffs = chebyshv_coeffs(f, 4)  # Function from Problem 5.
>>> print(coeffs)
[  3.75000000e-01  -5.88784672e-17   5.00000000e-01   5.88784672e-17
   1.25000000e-01]

>>> chebval(domain, coeffs)         # Evaluate at the points in domain.
[ 1.      0.0625  0.      0.0625  1.    ]
\end{lstlisting}

NumPy's \li{polynomial.chebyshev} module has many useful functions for working with Chebyshev polynomials.
Some of the methods include fast implementations of polynomial operations such as addition, division, multiplication, and integration.
\end{comment}

\section*{Lagrange vs. Chebyshev}

As was previously stated, Barycentric Lagrange interpolation is $O(n^2)$ at startup and $O(n)$ at runtime while Chebyshev interpolation is $O(n\log n)$.
This improved speed is one of the greatest advantages of Chebyshev interpolation.
Chebyshev interpolation is also more accurate than Barycentric interpolation, even when using the same points.
Despite these significant advantages in accuracy and temporal complexity, Barycentric Lagrange interpolation has one very important advantage over Chebyshev interpolation: Barycentric interpolation can be used on any set of interpolating points while Chebyshev is restricted to the Chebyshev nodes.
In general, because of their better accuracy, the Chebyshev nodes are more desirable for interpolation, but there are situations when the Chebyshev nodes are not available or when specific points are needed in an interpolation.
In these cases, Chebyshev interpolation is not possible and Barycentric Lagrange interpolation must be used.

\begin{comment} % Too confusing and too much of a repeat of a previous problem.
\begin{problem}
Investigate the claims made in the previous section by writing two different functions, one to compare error and the other to compare speed.
This function should accomplish this by doing the following:
\begin{itemize}
\item Interpolate Runge's function on the interval $[-1,1]$ using your Lagrange function, your Barycentric Lagrange class, Scipy's Barycentric class and Chebyshev interpolation.
\item Use the Chebyshev extremal points for the interpolating values.
\item Print the error or time required to run the function for each of the methods for $n=[500,1000,1500]$ interpolating values.
Be sure to clearly label the method and value of $n$ being used for each value printed.
\item Always evaluate at 200 equally spaced points in the domain.
\end{itemize}
Use \li{scipy.linalg.norm()} with \li{ord=np.inf} and the Chebyshev extremal points.
Note that if \li{inf} appears in any of your compuatations \li{scipy.linalg.norm()} will raise an error, in this case print the error as \li{inf} (you may need to use a \li{try} \li{except} block for some of the methods).
\end{problem}
\end{comment}

\section*{Utah Air Quality}

The Utah Department of Environmental Quality has air quality stations throughout the state of Utah that measure the concentration of particles found in the air.
One particulate of particular interest is $PM_{2.5}$ which is a set of extremely fine particles known to cause tissue damage to the lungs.
The file \texttt{airdata.npy} has the hourly concentration of $PM_{2.5}$ in micrograms per cubic meter for a particular measuring station in Salt Lake County for the year 2016.
The given data presents a fairly smooth function which can be reasonably approximated by an interpolating polynomial.
Although Chebyshev interpolation would be preferable (because of its superior speed and accuracy), it is not possible in this case because the data is not continous and the information at the Chebyshev nodes
is not known.
In order to get the best possible interpolation, it is still preferable to use points close to the Chebyshev extrema with Barycentric interpolation.
The following code will take the $n+1$ Chebyshev extrema and find the closest match in the non-continuous data found in the variable \li{data} then calculate the barycentric weights.

\begin{lstlisting}
>>> fx = lambda a, b, n: .5*(a+b + (b-a) * np.cos(np.arange(n+1) * np.pi / n))
>>> a, b = 0, 366 - 1/24
>>> domain = np.linspace(0, b, 8784)
>>> points = fx(a, b, n)
>>> temp = np.<<abs>>(points - domain.reshape(8784, 1))
>>> temp2 = np.argmin(temp, axis=0)

>>> poly = barycentric(domain[temp2], data[temp2])
\end{lstlisting}

\begin{problem}
Write a function that interpolates the given data along the whole interval at the closest approximations to the $n+1$ Chebyshev extremal nodes.
The function should accept $n$, perform the Barycentric interpolation then plot the original data and the approximating polynomial on the same domain on two separate subplots.
Your interpolating polynomial should give a fairly good approximation starting at around 50 points.
Note that beyond about 200 points, the given code will break down since it will attempt to return multiple of the same points causing a divide by 0 error.
If you did not perform the fix suggested in the \texttt{ACHTUNG} box, make sure not to pass in any points that exactly match the interpolating values.
\end{problem}

\newpage

\section*{Additional Material}

The \emph{Clenshaw Algorithm} is a fast algorithm commonly used to evaluate a polynomial given its representation in Chebyshev coefficients.
This algorithm is based on the recursive relation between Chebyshev polynomials and is the algorithm used by NumPy's \li{polynomial.chebyshev} module.

\begin{algorithm}
\begin{algorithmic}[1]
\Procedure{ClenshawRecursion}{$x, a$}
	\State $u_{n+1} \gets 0$
	\State $u_{n} \gets 0$
	\State $k \gets n-1$
	\While{$k \geq 1$}
		\State $u_k \gets 2 x u_{k+1} - u_{k+2} + a_k$
		\State $k \gets k-1$
	\EndWhile
	\State \pseudoli{return} $a_0 + x u_1 -u_2$
\EndProcedure
\end{algorithmic}
\caption{Accepts an array $x$ of points at which to evaluate the polynomial and an array $a=[a_0,a_1,\dots,a_{n-1}]$ of Chebyshev coefficients.}
\label{alg:clenshaw_recursion}
\end{algorithm}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Some material from the old Chebyshev Polynomial lab that expands on the material contained in the current lab. Some of it may be useful for future iterations of this lab.

\begin{comment} %This section gives a more complete introduction to Chebyshev polynomials including their orthogonality.
This sort of interpolation is also related to what are called the Chebyshev polynomials.
Chebyshev Polynomials can be used very nicely to approximate functions on $[ -1, 1 ]$ and, with proper scaling, can be used to approximate reasonably well-behaved functions on any interval.
The $k$'th Chebyshev polynomial is commonly written $T_k \left( x \right)$.
One way to define the Chebyshev Polynomials is that the $k$'th Chebyshev polynomial on the unit circle is the real part of the function $z^k$ on the unit circle.
More precisely, let $z(x) = t + i \sqrt{1 - x^2}$, then the $k$'th Chebyshev polynomial on $[-1, 1]$ is the real part of the polynomial $x^k$.
We can write this real part explicitly as
\[T_k \left( x \right) = \frac{1}{2} \left( z^k + z^{-k} \right) = \cos \left( k \cos^{-1} \left( x \right) \right)\]
Notice that, as this function has been defined, the Chebyshev polynomial of order $k$ on an interval takes values of $1$ and $-1$ at the $k+1$ chebyshev points corresponding to interpolation with a polynomial of order $k$ on that interval.
The first few Chebyshev polynomials are shown in Figure \ref{fig:cheb_polys}.

These polynomials are orthogonal with respect to the integral inner product
\[\langle f, g \rangle = \int_{-1}^1 \frac{1}{\sqrt{1 - x^2}} f\left( x \right) g\left( x \right) dx\]
This inner product allows us to write many functions as an infinite series of Chebyshev polynomials.
In particular, we can write any polynomial as a finite series of Chebyshev polynomials.
As it turns out, it is relatively easy to compute the Chebyshev series representation of an interpolating polynomial for a function on a given interval.
These approximations converge to the function we are approximating nearly as well as the partial sums of the infinite series formed using the inner product above, so, in practice, we will use them instead.

It can be shown that the Chebyshev polynomials follow the following recurrence relation: $ T_{k+1} \left( x \right) = 2 x T_k \left( x \right) - T_{k-1}$.
$T_0 \left( x \right)$ is equal $1$ and $T_1 \left( x \right)$ is equal to $x$ (still working on the interval $[-1, 1]$.)
\end{comment}

\begin{comment} %This section includes a more mathematical introduction to the connection between Fourier series and Chebyshev Polynomials.
\section*{Chebyshev Polynomials and Fourier Series}
In looking at the plots of the Chebyshev polynomials you may have noticed that the Chebyshev polynomials take alternating values of $1$ and $-1$ at the Chebyshev points on the interval $[-1,1]$ similar to how the functions $\cos\left(k x \right)$ change between $-1$ and $1$ on equispaced points in the interval $[ 0 , \pi ]$.
Recall that, as mentioned above, $T_k \left( x \right) = \cos \left( k \cos^{-1} \left( x \right) \right)$.
For discrete samples, this means that the Chebyshev coefficients for the interpolating polynomial through a function at the Chebyshev nodes has the same coefficients as the discrete cosine series would at the points $\cos^{-1} x$ where $x$ is a chebyshev node.
In practice this means we can compute an array of samples of a function at the Chebyshev nodes of the interval $[-1, 1]$, then compute the discrete cosine transform to get the corresponding coefficients to the Chebyshev interpolation.
This is very helpful since the discrete cosine transform is a close relative of the discrete Fourier transform.
\li{scipy.fftpack} and \li{pyfftw} both include discrete cosine transforms as \li{scipy.fftpack.dct} and \li{pyfftw.interfaces.scipy_fftpack.dct} respectively.
The discrete cosine transform is defined a little differently than the coefficients we need here.
There are several different versions of the discrete cosine transform.
Using the inverse discrete cosine transform of type 1 on an array $y$ of length $N$ will give us an array $x$ of values such that
\[y_k = z_0 + 2 \sum_{n=1}^{N-2} z_n \cos\left(\frac{\pi k n}{N-1}\right) + \left(-1\right)^k z_{N-1}\]
where $z = \frac{x}{2 \left(N - 1\right)}$.
To obtain the Chebyshev coefficients, we desire to find the vector $a$ of coefficients $a_n$ such that
\[y_k = \sum_{n=0}^{N-1} a_n \cos\left(\frac{\pi k n}{N-1}\right)\]
This means that, given the vector $x$, we can obtain $a$ by dividing all the terms of $x$ by $N-1$, and then dividing the first and last entries of $x$ by 2.
This means that we can obtain the coefficients of the Chebyshev interpolant using the following steps:

\begin{itemize}

\item Sample the function you desire to approximate at the Chebyshev nodes.
Make sure you use the sampled values in the order that corresponds to the chebyshev nodes when sorted in \emph{descending} order and not ascending order.

\item Compute the inverse discrete cosine transform of the data.
Use the option \li{type=1} to tell either scipy or pyfftw the version of the discrete cosine transform you want to use.

\item Divide all the coefficients by $\left( N - 1 \right)$ where $N$ is the number of nodes used.

\item Divide the first and last coefficient by $2$.

\end{itemize}

\begin{warn}
Often it is customary in computation to use values in ascending order.
When you compute the Chebyshev coefficients for the interpolatng polynomial through a set of points you will need the samples from the points sorted in descending order, not ascending order.
\end{warn}
\end{comment}

\begin{comment} % Potential problem and discussion exploring convergence of polynomial interpolation.
\begin{problem}
\label{prob:cheb_interpolations}
We will briefly consider the rate of convergence of these polynomial approximations.
Compute the coefficients for the interpolating Chebyshev series for the function $\cos x$ on the interval $[-1, 1]$.
This approximation converges very rapidly, so the first $20$ terms or so should be more than enough.
How many of these coefficients have absolute value greater than $10^{-14}$?
How close does your series approximate the actual function?

Now compute the coefficients for a degree $100000$ polynomial approximating the function
\[\sin \left( \frac{1}{x} \right) \sin \left( \frac{1}{\sin \left( \frac{1}{x} \right)} \right) \]
on the interval $[-1, 1]$.
How large are the last $10$ coefficients in the series?
Use NumPy's Chebyshev class to plot this function at $100001$ equispaced points on the interval $[-1, 1]$.
Plot it with the original function.
Compare the two.
How close are they?
Notice how the interpolating polynomial is able to approximate the original function about as well as the discrete sample of the original function.
\end{problem}

Problem \ref{prob:cheb_interpolations} also illustrates an interesting principle.
These series converge more quickly when a function is infinitely differentiable everywhere in the complex plane (the proper term for a function like this is "entire").
For functions that do not satisfy this property, it can be shown (roughly speaking) that the rate of convergence depends on how many times differentiable the function is on the interval of interpolation.
For a less well-behaved function like the one considered in the second half of Problem \ref{prob:cheb_interpolations} the coefficients converge much more slowly.
The last $10$ coefficients for the interpolant of degree $2^{23} - 1$ are still
\begin{lstlisting}
[9.52973125e-08,  -1.89451973e-09,  -7.42182166e-08,
 1.89319137e-09,   5.26564839e-08,  -1.89451836e-09,
 -3.13050802e-08,   1.89319005e-09,   1.03700608e-08,
 -9.47258778e-10]
\end{lstlisting}
\end{comment}


\subsection{Density estimation through direct sampling}

I THINK THE STUFF HERE IS LIMITATIONS TO REJECTION SAMPLING??

DIRECT SAMPLING IS DOING PHYSICAL SAMPLES, MANUALLY PICKING BALLS FROM URL ETC?

There is distribution \(P(x)\) which we want to know more about.

If the function was closed, we could estimate it by using values of \(x\).

\subsection{Limitations of direct sampling}

However if the function does not have such a form, we cannot do that.

We can't plug in values, because the function is complex.

Sometimes we may know a function of the form:

\(f(x)=cP(x)\)

That is, a multiple of the function.

This can happen from Bayes' theorem:

\(P(y|x)=\dfrac{P(x|y)P(y)}{P(x)}\)

We may be able to estimate \(P(x|y)\) and \(P(y)\), but not \(P(x)\)

This means be have

\(P(y|x)=cP(x|y)P(y)\)


\chapter{Extending FLID}
\label{sec:models}

\section{Beyond Diversity: FLDC}

Diversity is an important property in the context of summarization \citep{tschiatschek16learning}, however it is not the only one that should be considered. Coherence is another desired property of summaries, for example in structured summaries \citep{Yan:2011:ETS:2009916.2010016}. Balancing coherence and diversity is considered a challenge and maxiziming diversity alone may lead to disconnected summaries, while focusing on coherence can hinder the breadth of the resulting summaries \citep{Shahaf2012}.

To better understand why coherence is important, consider the construction of a spatial summary for photos taken in a city. A model based on spatial diversity would create sets containing photos taking at distant locations in the city, however another possibility is to have summaries that contain photos that were taken close to some starting point, in this case the desired property is not spatial diversity but rather coherence.

Therefore, a natural extension to the FLID model is to add the capacity of modeling coherence along with diversity. We propose the addition of a log-supermodular term analogous to the log-submodular term \eqref{eq:diversity} in order to model the complementarity between elements.

This supermodular term introduces a representation of the elements using $K$-dimensional vectors $\mathbf{w}^{K} \in \mathbb{R}^{K}_{\geq 0}$, where each dimension $1 \leq c \leq K$ captures a concept that relates to set coherence and $w_{i,c}$ quantifies the contribution of each element $i$ to each coherence concept $c$. The coherence of a set $S$ with respect to each concept $c$ is quantified as $\sum_{i \in S}{w^{e}_{i,c}} - \max_{i \in S}{w^{e}_{i,c}}$ which is a supermodular term mirroring the submodular term in FLID. Summing over all the concepts results in the proposed coherence term. Hereby, define $\mathbf{W}^{e} \in \mathbb{R}^{|V| \times K}_{\geq 0}$ as the matrix where the $i$-th row is the representation $\mathbf{w}^{e}$ of element $i$.

Adding the coherence term to FLID results in the extended model, which we refer to as the Facility Location Diversity and Coherence (FLDC) model. Its probability distribution is defined as,

\begin{equation}
  \tag{FLDC}
  P(S) = \frac{1}{Z}\exp{\left(\sum_{i \in S}{u_{i}} + \mathrm{div}(S) + \sum_{c=1}^{K}{\left(\sum_{i \in S}{w^{e}_{i,c}} - \max_{i \in S}{w^{e}_{i,c}}\right)}\right)},
  \label{eq:fldc}
\end{equation}

where $\mathrm{div}(S)$ is the diversity term in Equation \eqref{eq:diversity}.

For the FLDC model, the partition function can be computed in $\mathcal{O}(|V|^{L+K+1})$ time. This complexity can be derived using a similar algorithm as the one presented by \citet{tschiatschek16learning} for FLID, the key observation is that the algorithm they present only requires the ordering of weights to determine eligible elements for a set given  the maximum weights for each dimension. This can be easily extended to include the ordering for the coherence weights as well. This requires evaluating $K$ more dimensions which results in the added term to the exponent of the complexity expression, which makes the computation of the partition function more expensive than in the FLID case.

\subsection{Learning FLDC Models}

As with FLID, the parameters for an FLDC model can be estimated using NCE. For FLDC the vector $\boldsymbol{\theta}$ is composed of $[\mathbf{u}, \mathbf{W}^{b}, \mathbf{W}^{e}, \hat{Z}]$ and its gradient is the same as in equations \eqref{eq:gradient-flid-1}-\eqref{eq:gradient-flid-4} with the addition of a gradient with respect to the new parameter, $\mathbf{W}^{e}$, which is given by,

\begin{equation}
\left(\nabla_{\mathbf{W}^{e}}\log{\hat{P}_{d}(S; \hat{Z}, \mathbf{u}, \mathbf{W}^{e}, \mathbf{W}^{e})}\right)_{i,c} = \begin{cases}
1 & \text{if}\ i \in S\ \text{and}\ i \neq \argmax_{j \in S}{w^{e}_{j,c}} \\
0 & \text{otherwise}
\end{cases}, \label{eq:sub-fldc}\\
\end{equation}

where $\left(\nabla_{\mathbf{W}^{e}}\cdot \right)_{i,c}$ represents the $(i,c)$-th entry in the gradient with respect to $\mathbf{W}^{e}$.

This also affects the overall complexity of the learning algorithm. Computing the gradient takes $\mathcal{O}(L+K)|S|$ for FLDC at each step, therefore the running time of a full pass over the data and noise samples takes $\mathcal{O}(|\mathcal{D}\cup\mathcal{N}|\kappa(L+K))$. This is not significantly more expensive than learning FLID because the complexity is still linear in the data and noise.

\subsection{Example: Two Non-overlapping Clusters}
\label{sec:fldc-toy}

As an example of the extended model, consider the distribution presented in Table \ref{tab:fldc-toy-probs} for $V = \{1,2,3,4\}$, representing a set of two non-overlapping clusters with 2 elements each. Intuitively, this can be modeled by considering that there is diversity between elements in different clusters while inside a cluster there is a coherence component that ties them together.

Concretely, the weight matrices $\mathbf{W}^{b}, \mathbf{W}^{e}$ in Figure \ref{fig:fldc-toy-mixed-weights} illustrate one possible instance of the model. The corresponding utility vector is $\mathbf{u} = \overrightarrow{0}$, because there is no indication that an individual element is favored over the pairs. This model is easily interpretable and accurately realizes the distribution.

\begin{table}
  \centering
  \caption{Probability distribution for Example \ref{sec:fldc-toy}.}
  \begin{tabular}{@{}ll@{}}
    \toprule
    $S$ & $P(S)$  \\
    \midrule
    $\{1,2\}$ & $0.5$ \\
    $\{3,4\}$ & $0.5$ \\
    Otherwise & $0.0$ \\
    \bottomrule
  \end{tabular}
  \label{tab:fldc-toy-probs}
\end{table}

\begin{figure}
  \centering
  \includegraphics[width=\textwidth]{fldc_toy_example_mixed_weights}
  \caption{Diversity and coherence weights for FLDC model in Example \ref{sec:fldc-toy}. The dotted line divides the clusters.}
  \label{fig:fldc-toy-mixed-weights}
\end{figure}

\section{Generalizing through Features: FFLDC}

An important characteristic of the FLID model, and by extension the FLDC one, is that it is agnostic to the type of elements in the ground set, this allows its application to a wide range of problems without prior knowledge. However the downside is that the model has no capability to make use of information about the elements to improve the modeling of the data. Moreover, if a new element is added to the set there is no way to generalize the existing knowledge about similar elements.

Considering these issues, we propose a further extension to the model. First, an element ($i \in V$) is represented as a vector $\mathbf{x}_{i} \in \mathbb{R}^{M}$ where each entry is a feature, e.g. for venues one feature could be its aggregated rating while another feature indicates whether it is indoors or outdoors. The representations of all elements are represented in matrix $\mathbf{X} \in \mathbb{R}^{|V| \times M}$ where each row $i$ corresponds to the representation $\mathbf{x}_{i}$ of element $i$.

Then, the utility vector $\mathbf{u}$ and the weight matrices $\mathbf{W}^{b}, \mathbf{W}^{e}$ are factorized,

\begin{align}
  \mathbf{u} &= \mathbf{Xa},   \label{eq:ffldc-factorization-1} \\
  \mathbf{W}^{b} &= \mathbf{XB},\ \text{and}  \label{eq:ffldc-factorization-2} \\
  \mathbf{W}^{e} &= \mathbf{XE},
  \label{eq:ffldc-factorization-3}
\end{align}

where $\mathbf{a} \in \mathbb{R}^{M}$ represents the contribution of each feature to the total utility of an item, and $\mathbf{B} \in \mathbb{R}^{M \times L}$ and $\mathbf{E} \in \mathbb{R}^{M \times K}$ encode the contribution of each feature to each latent diversity and coherence dimension, respectively. The intuition behind this factorization is that the information about the items can enhance the latent representations. For example if $M < |V|$ and the features are sufficient to represent the items, i.e. each item can be uniquely identified from its features, it is expected that learning requires less computation because of the reduced number of parameters in contrast to learning an equivalent FLDC model.

We refer to the extended model as the Featurized Facility Location Diversity and Coherence (FFLDC) model and its probability distribution is defined as:

\begin{align}
  \tag{FFLDC} \label{eq:ffldc}
  & P(S) &=& \frac{1}{Z}\exp{\left(\sum_{i \in S}{\sum_{k=1}^{M}x_{i,k}a_{k}} + fdiv(S) + fcoh(S)\right)}, \\
  \text{where} & & \\
  & fdiv(S) &=& \sum_{d=1}^{L}{\left(\max_{i \in S}{\sum_{k=1}^{M}x_{i,k}b_{k,d}} - \sum_{i \in S}{\sum_{k=1}^{M}x_{i,k}b_{k,d}}\right)}\ \text{and} \\
  & fcoh(S) &=& \sum_{c=1}^{K}{\left(\sum_{i \in S}{\sum_{k=1}^{M}x_{i,k}e_{k,c}} - \max_{i \in S}{\sum_{k=1}^{M}x_{i,k}e_{k,c}}\right)}
\end{align}

Where $a_{k}$ is the $k$-th entry of $\mathbf{a}$, $b_{k,d}$ is the $(k,d)$-th entry of $\mathbf{B}$, $e_{k,d}$ is the $(k,d)$-th entry of $\mathbf{E}$ and $x_{k,c}$ is the $(k,c)$-th entry of $X$.

\begin{remark}
  If $\mathbf{X} = \mathcal{I}$, then FFLDC is equivalent to FLDC, with $\mathbf{u} = \mathbf{a}$, $\mathbf{W}^{b} = \mathbf{B}$ and $\mathbf{W}^{e} = \mathbf{E}$.
\end{remark}

The use of features also allows the application of the model to previously unseen elements, hence solving the aforementioned problem of generalization. This is possible because the model is defined on the space of features and not directly on items, to evaluate the probability of a set containing a previously unseen element $j$ only its representation $\mathbf{x}_{j}$ is required.

The computation of the partition function for this model can be performed using the fact that any FFLDC model can be transformed into an equivalent FLDC model through the factorizations in equations \eqref{eq:ffldc-factorization-1}-\eqref{eq:ffldc-factorization-3}. This transformation requires $\mathcal{O}(|V|M(L+K))$ time for the matrix multiplications, therefore the overall complexity of calculating the partition function for an FFLDC model is $\mathcal{O}(|V|M(L+K) + |V|^{L+K+1})$. This expression can be simplified to $\mathcal{O}(|V|^{L+K+1})$ because the exponential term is significantly larger assuming sensible values for $M$. Given that the complexity of computing the partition function for FFLDC is equivalent to the FLDC model, it is also intractable in practice.

\subsection{Learning FFLDC Models}

For FFLDC, the parameter vector $\boldsymbol{\theta}$ is different from the previous models, it is composed of $[\mathbf{a}, \mathbf{B}, \mathbf{C}, \hat{Z}]$ and the gradient of the NCE objective $g(\boldsymbol{\theta})$ is given by,

\begin{align}
  \left(\nabla_{\mathbf{a}}\log{\hat{P}_{d}(S; \hat{Z}, \mathbf{a}, \mathbf{B}, \mathbf{E})}\right)_{m} &= \sum_{i \in S} x_{i,m} \label{eq:sub-a}\\
  \left(\nabla_{\mathbf{B}}\log{\hat{P}_{d}(S; \hat{Z}, \mathbf{a}, \mathbf{B}, \mathbf{E})}\right)_{m,d} &= x_{i^{d},m} - \sum_{i \in S} x_{i,m} \label{eq:sub-b} \\
  \left(\nabla_{\mathbf{E}}\log{\hat{P}_{d}(S; \hat{Z}, \mathbf{a}, \mathbf{B}, \mathbf{E})}\right)_{m,c} &= x_{i^{c},m} - \sum_{i \in S} x_{i,m}, \label{eq:sub-e}
\end{align}

where $i^{d} = \argmax_{i \in S}{\sum_{k=1}^{M}{x_{i,k}b_{k,d}}}$ and $i^{c} = \argmax_{i \in S}{\sum_{k=1}^{M}x_{i,k}e_{k,c}}$. $\left(\nabla_{\mathbf{a}}\cdot \right)_{m}$ represents the $m$-th entry of the sub-gradient with respect to $a$, $\left(\nabla_{\mathbf{B}}\cdot\right)_{m,d}$ represents the $(m,d)$-th entry of the sub-gradient with respect to $B$ and $\left(\nabla_{\mathbf{E}}\cdot\right)_{m,c}$ represents the $(m,c)$-th entry of the sub-gradient with respect to $E$.

\begin{remark}
  For $\mathbf{X} = \mathcal{I}$, the gradients in \eqref{eq:sub-a}-\eqref{eq:sub-e} are equivalent to the FLDC sub-gradients. Because $x_{i,m} = 1$ only when $i = m$.
\end{remark}

Regarding time complexity, the FFLDC model requires more operations for the gradient due to the inclusion of features. Concretely, computing the gradient for a set $S$ takes $\mathcal{O}(|S|M(L+K))$ which makes the running time of a single pass over data and noise samples $\mathcal{O}(|\mathcal{D}\cup\mathcal{N}|M(L+K)\kappa)$. Even though this is larger than the complexity of FLID or FLDC, it is still linear in the size of the data and noise samples.

\subsection{Example: Rated Locations}
\label{sec:ffldc-toy}

A small town has 6 popular locations, each of them has been rated from 0 to 5, where 0 represents a bad review and 5 an excellent review. Collected data shows that a typical visitor is more likely to visit and take photos at places with high ratings, however this is not the only factor driving their behavior. For example, an average visitor does not visit two places that serve food on the same day instead the data shows that visitors usually go to one place that serves food and then to an outdoor one, or they just visit one that has both characteristics. This data is summarized in Table \ref{tab:ffldc-toy-probs}, the task is to model this behavior using the FFLDC model.

\begin{table}
  \centering
  \caption{Synthetic data for Example \ref{sec:ffldc-toy}.}
  \begin{tabular}{@{}ll@{}}
    \toprule
    Locations visited $(S)$ & $P(S)$  \\
    \midrule
    $\{1,3\}$ & $0.30$ \\
    $\{3,4\}$ & $0.25$ \\
    $\{3,6\}$ & $0.15$ \\
    $\{2\}$ & $0.10$ \\
    $\{1\}$ & $0.06$ \\
    $\{3\}, \{4\}$ & $0.04$ \\
    $\{5\}, \{6\}$ & $0.03$ \\
    Otherwise & $0.00$ \\
    \bottomrule
  \end{tabular}
  \label{tab:ffldc-toy-probs}
\end{table}

In this example there is knowledge about the items and what features are relevant for the data. These features are summarized in equation \eqref{eq:ffldc-toy-feats}. The first column corresponds to the aforementioned rating, the second and third are binary features indicating whether the location is an outdoor one and whether it serves food, respectively.

\begin{equation}
  \mathbf{X} = \left(
    \begin{array}{ccc}
      4 & 1 & 0 \\
      4 & 1 & 1 \\
      3 & 0 & 1 \\
      3 & 1 & 0 \\
      2 & 1 & 1 \\
      2 & 1 & 0  \\
     \end{array}
  \right)
  \label{eq:ffldc-toy-feats}
\end{equation}

A possiblity is an FFLDC model that encourages diversity on the second and third feature, i.e. models the idea that visitors do not go to more than one place that serves food or is outdoor, while assigning a positive utility and coherence value to the first feature, i.e. modeling the preference for places with high ratings. One such model is presented in Figure \ref{fig:ffldc-toy-all-weights}, this approximates distribution from Table \ref{tab:ffldc-toy-probs} and illustrates the type of model that is useful in this example.

\begin{figure}
  \centering
  \includegraphics[width=\textwidth]{ffldc_toy_example}
  \caption{FFLDC sample model for Example \ref{sec:ffldc-toy}.}
  \label{fig:ffldc-toy-all-weights}
\end{figure}

If a new location $j$ is considered then it is possible to apply the model and estimate the probability of sets that include it. For example, consider a seventh location with the following features $\mathbf{x}_{7} = [5, 1, 0]$, using the same model parameters a new distribution can be computed without changing the model parameters, as it would be the case with FLID or FLDC. The updated distribution is shown in Table \ref{tab:ffldc-toy-probs-2} and it can be considered a sensible distribution given the problem description and the high rating of the new location, hence showing the model's capacity to generalize.

\begin{table}
  \centering
  \caption{Synthetic data for Example \ref{sec:ffldc-toy} after adding a new item.}
  \begin{tabular}{@{}ll@{}}
    \toprule
    Locations visited $(S)$ & $P(S)$  \\
    \midrule
    $\{3,7\}$ & $0.24$ \\
    $\{1,3\}$ & $0.21$ \\
    $\{3,4\}$ & $0.19$ \\
    $\{3,6\}$ & $0.10$ \\
    $\{7\}$ & $0.06$ \\
    $\{1\}, \{2\}$ & $0.04$ \\
    $\{3\}, \{4\}$ & $0.03$ \\
    $\{5\}, \{6\}$ & $0.02$ \\
    Otherwise & $0.00$ \\
    \bottomrule
  \end{tabular}
  \label{tab:ffldc-toy-probs-2}
\end{table}


\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage[letterpaper, total={7.5in, 9in}]{geometry}
\usepackage{amsmath}
\usepackage{amsfonts}

\title{Applied Partial Differential Equations}
\author{Grant Smith}
\date{Spring 2022}

\begin{document}

\maketitle

\section{Classifying PDEs}

\begin{itemize}
    \item 2) Classify the following operators as linear or nonlinear:
    \begin{enumerate}
        \item $\mathcal{L} u = u_x + xu_y$ -- linear
        \item $\mathcal{L} u = u_x + uu_y$ -- nonlinear
        \item $\mathcal{L} u = u_x + \left(u_y\right)^2$ -- nonlinear
        \item $\mathcal{L} u = u_x + u_y + 1$ -- nonlinear
        \item $\mathcal{L} u =  \sqrt{1+x^2}\left(cos(y)\right)u_x+u_{yxy}-\left[arctan(x/y)\right]u $ -- linear
    \end{enumerate}
    \item 3) For each of the following equations, state the order and whether it is nonlinear, linear inhomogeneous, or linear homogeneous. Provide reasons.

        \begin{tabular}{|| c| c| c| c|| }
        \hline
        Equation & Order & Classification & Reason \\
        \hline \hline
 $u_t - u_{xx} + 1 = 0$ & 2 & linear inhomogeneous & inspection \\ \hline
 $u_t - u_{xx} + xu = 0$ & 2 & linear homogeneous & inspection \\ \hline
 $u_t - u_{xxt} + uu_x = 0$ & 3 & linear homogeneous & inspection \\ \hline
 $u_{tt} - u_{xx} + x^2 = 0$ & 2 & linear inhomogeneous & inspection \\ \hline
 $iu_t - u_{xx} + u/x = 0$ & 2 & linear inhomogeneous & inspection \\ \hline
 $u_x(1+u_x^2)^{-1/2} + u_y(1+u_y^2)^{-1/2} = 0$ & 1 & nonlinear & The derivative terms are squared \\ \hline
 $u_x - e^yu_{y} = 0$ & 1 & linear homogeneous & inspection \\ \hline
 $u_t + u_{xxxx} + \sqrt{1+u} = 0$ & 4 & nonlinear & u is under the square root \\
\hline
\end{tabular}

\item 4) Show that the difference of two solutions of an inhomogenous linear equation $\mathcal{L} u = g$ with the same $g$ is a solution of the homogenous equation $\mathcal{L} u =0$

\begin{enumerate}
    \item let solution 1 be $u_1$, and let solution 2 be $u_2$
    \item the question asks about the difference of the two solutions. Thus, $\mathcal{L} \left( u_1 - u_2 \right)$
    \item because $\mathcal{L}$ is linear, $\mathcal{L} \left( u_1 - u_2 \right)$ = $\mathcal{L}  u_1 - \mathcal{L}  u_2 = g - g = 0$
    \item thus, $u_1 - u_2$ is a solution to $\mathcal{L} u =0$

\end{enumerate}

\end{itemize}
\newpage
\section{First Order Linear PDEs}

\subsection{Deriving the Method of Characteristics}

The problem we will solve is of the form
$$ a(x,y) \frac{\partial u}{\partial x}(x,y) + b(x,y) \frac{\partial u}{\partial y}(x,y) = c(x,y)$$

It's solution is:

$$\frac{\partial x}{\partial s}(s,t) = a^*(s,t)$$
$$\frac{\partial y}{\partial s}(s,t) = b^*(s,t)$$
$$ \frac{\partial u^*}{\partial s}(s,t) = c^*(s,t)$$

Which we will derive now. First, let's define some coordinate transforms.  These will help us get x and y from s and t

$$x(s,t); y(s,t)$$

And we will define our transforms to be bijective (or nonzero Jacobians), so we can go the other way and get s and t from x and y

$$s(x,y); t(x,y)$$

This is enforced by:

$$ x(s(x,y),t(x,y)) = x $$
$$ y(s(x,y),t(x,y)) = y$$
$$s(x(s,t),y(s,t)) = s $$
$$ t(x(s,t),y(s,t)) = t$$

We also have a transformed u that we get by using these transform equations:

$$u^*(s,t):= u(x(s,t),y(s,t))$$

And for posterity and clarity's sake, we show that we can get u back from the transformed u:

$$u(x,y) = u^{**}(x,y) = u^*(x(s(x,y),t(x,y)),y(s(x,y),t(x,y)))$$

We can also transform the whole first equation:

$$a^*(s,t) \frac{\partial u}{\partial x}^*(s,t) + b^*(s,t) \frac{\partial u}{\partial y}^*(s,t) = c^*(s,t)$$

Here, we rewrite $u^*$ for convenience

$$u^*(s,t):= u(x(s,t),y(s,t))$$

We could differentiate $u^*$ with respect to either of its arguments. Let's just choose the first one for now, $s$.

$$\frac{\partial u^*}{\partial s}(s,t) = \frac{\partial u}{\partial x}(x(s,t),y(s,t)) \frac{\partial x}{\partial s}(s,t) + \frac{\partial u}{\partial y}(x(s,t),y(s,t)) \frac{\partial y}{\partial s}(s,t)$$

And we can do the coordinate transform on the partials of $u$ to get:

$$\frac{\partial u^*}{\partial s}(s,t) = \frac{\partial u}{\partial x}^*(s,t) \frac{\partial x}{\partial s}(s,t) + \frac{\partial u}{\partial y}^*(s,t) \frac{\partial y}{\partial s}(s,t)$$

Which we can pattern match against the transformed version of the original equation (rewritten here for convenience)

$$a^*(s,t) \frac{\partial u}{\partial x}^*(s,t) + b^*(s,t) \frac{\partial u}{\partial y}^*(s,t) = c^*(s,t)$$

 or

$$c^*(s,t)= \frac{\partial u}{\partial x}^*(s,t)a^*(s,t) +  \frac{\partial u}{\partial y}^*(s,t)b^*(s,t)$$

and get:

$$\frac{\partial x}{\partial s}(s,t) = a^*(s,t)$$
$$\frac{\partial y}{\partial s}(s,t) = b^*(s,t)$$
$$ \frac{\partial u^*}{\partial s}(s,t) = c^*(s,t)$$

\newpage

\subsection{Problems}

\begin{itemize}
    \item 1) Solve: $2u_t + 3u_x = 0$
    \begin{enumerate}
        \item The form of this equation hints at a directional derivative. So if we change the coordinates to one which runs along this direction, then we have a nice form of differential equation.
        \item Change the coordinates:
        $$u\left(x(c,k),t(c,k)\right) = u^*(c,k)$$
        \item Now, pick one of the variables, $c$ or $k$, and partially differentiate the above equation with respect to that variable. For example, we'll pick $c$.
        $$\frac{\partial u}{\partial x} \frac{\partial x}{\partial c} + \frac{\partial u}{\partial t} \frac{\partial t}{\partial c} = \frac{\partial u^*}{\partial c}$$
        \item By pattern matching against the initial equation, we have the following three equations:
            $$\frac{\partial x}{\partial c} = 3 ; \frac{\partial t}{\partial c} = 2; \frac{\partial u^*}{\partial c} = 0$$
        \item By solving all three, we get the following:
            $$x = 3c + f_1(k) ; t = 2c + f_2(k); u^* = f(k)$$
        \item Given that the relationships between $x, t, c, k$ are just coordinate transforms, we can choose $f_1$ and $f_2$ to be suitable. We choose $f_1 = k$ and $f_2 = 0$, which gives us the following:
        $$x = 3c + k ; t = 2c; u^* = f(k)$$
        This works because we know that as $c$ and $k$ vary throughout the whole plane, the corresponding $x$ and $y$ will also trace the whole plane. Also, these are convenient choices because the initial condition has $t=0$, so having $t$ be a very simple function (i.e. not dependent upon $k$) will be convenient.
        \item We can rearrange to get:
        $$c = \frac{t}{2} ; k = x - \frac{3t}{2}$$
        \item Looking back at step 2, we can now write:
        $$u\left(x(c,k),t(c,k)\right) = u^*(c,k) = f(k)$$
        And if we transform the coordinates back using our equation for $k$ in step 7, we have:
        $$u(x,t) = f (x-\frac{3t}{2})$$
        \item Now we use the initial condition that $$u(x,0) = sin(x) = f(x)$$
        \item Now we know that $$u(x,t) = sin(x-\frac{3t}{2})$$
        \item And this also works if we plug it into the original equation.

    \end{enumerate}

    \newpage
    \item 2) Solve $3u_y + u_{xy} = 0$
    \begin{enumerate}
        \item Assuming differentiability, we can change the order of integration, so $3u_y + u_{yx} = 0$
        \item Let $v = u_y$. This gives $3v + v_x = 0$
        \item Rearranging gives:
        $$\frac{\partial v}{\partial x} = -3v$$
        \item separating and integrating:
        $$\frac{-1}{3}\frac{\partial v}{v} =\partial x$$
        $$\frac{-1}{3}\ln\left| v\right| = x + f^{***}(y)$$
        $$\ln\left| v\right| = -3x + f^{**}(y)$$
        $$\left| v\right| = e^{-3x}f^*(y)$$
        And assuming that $f^*$ can be positive or negative, we can drop the absolute value:
        $$v = e^{-3x}f^*(y)$$
        \item Remembering that $v$ is not necessarily $u$,
        $$\frac{\partial u}{\partial y} = u_y = v = e^{-3x}f^*(y)$$
        $$\partial u = e^{-3x}f^*(y) \partial y$$
        $$u  = e^{-3x}f(y) + g(x)$$
        \item Which satisfies the original PDE, which can be verified by substitution.
        \item I suppose I haven't proven that this is the most general form of the equation, but I don't really know how to prove that. I know that this is linear, so any linear combination of this function will also work, but all linear combinations are already captured in $f$ and $g$.

    \end{enumerate}
    \newpage
    \item 7) Solve the equation
    $$yu_x + xu_y = 0; u(0,y) = e^{-y^2}$$
    And find the region of the x-y plane in which the solution is uniquely determined.
    \begin{enumerate}
        \item Note that this is the directional derivative of u.  Thus, we change coordinates using the transform $T$:
        $$T(u) = u^*(t,k) = u(x(t,k),y(t,k))$$
        \item Now take the partial derivative of both sides with respect to $t$ or $k$. Either should work. We'll arbitrarily choose $t$.
        $$\frac{\partial u}{\partial x}\frac{\partial x}{\partial t} + \frac{\partial u}{\partial y}\frac{\partial y}{\partial t} = \frac{\partial u^*}{\partial t}$$
        \item Now pattern match against the original equation to get the following PDEs:
        $$\frac{\partial x}{\partial t} = y;\frac{\partial y}{\partial t} = x;\frac{\partial u^*}{\partial t} = 0$$

        \item Now we will focus on the other two PDEs. Please forgive this poor mathematics, but I will integrate them and get the following:
        $$x = yt + f_1(k); y = xt + f_2(k); u^* = f(k)$$
        Which, using the third equation, now extends the equality from step 1 to:
        $$T(u) = u^*(t,k) = u(x(t,k),y(t,k)) = f(k)$$
        \item Let's pause for a moment to discuss the region of the x-y plane in which the solution is uniquely determined. The equation gets information along the line $x=0$. This means that any characteristic curve that does not pass through this line will not be uniquely determined. Given the shape of the characteristic curves determined in step 4, there is an "X" shape in the x-y plane formed by $y=x$ and $y=-x$ that determines the areas of determination. The areas of the x-y plane that are between these two lines are not determined, but the area above and below both of these lines is determined. Thus, it makes a sort of bow-tie of non-determinedness, and an hourglass of determinedness.
        \item Having discussed the determinedness, I also will mention that I'm not sure how to choose $f_1$ and $f_2$ such that they sweep the entire x-y plane. I seem to only be able to get half of it without making a non-injective function. Given this fact along with the fact that we only will be able to define a solution in the hourglass region anyway, I will choose $f_1$ and $f_2$ to sweep that region, and I will ignore the bow-tie region. Thus, we can choose:
        $$ f_1(k) = 0; f_2(k) = k$$
        So
        $$x = yt; y = xt + k$$
        And
        $$t = \frac{x}{y}; k = y - \frac{x^2}{y}$$
        \item Rearranging and rewriting the transforms gives:
        $$T(u) = u^*(t,k) = u(x(t,k),y(t,k)) = u(yt, xt + k) = f(k)$$
        $$T(u^*) = u(x,y) = u^*(t(x,y),k(x,y)) = u^*(\frac{x}{y},y - \frac{x^2}{y}) = f(y - \frac{x^2}{y})$$
        \item And using the initial condition gives:
        $$u(0,y) = e^{-y^2} = f(y)$$
        So
        $$u(x,y) = e^{-\left(y - \frac{x^2}{y}\right)^2}$$
        Which satisfies the initial condition, but it does not satisfy the original equation, so I'm doing something wrong. But after graphing it, it appears that this solution works in the hourglass region, and the only area in which it looks off is in the bow-tie, so possibly it does work.

    \end{enumerate}

    \newpage
    \item 8)
    Solve the following:
    $$au_x + bu_y + cu = 0$$
    \begin{enumerate}
        \item First, move $cu$ to the other side:
        $$au_x + bu_y =  - cu$$
        \item Now note that this is the directional derivative of u.  Thus, we change coordinates using the transform $T$:
        $$T(u) = u^*(t,k) = u(x(t,k),y(t,k))$$
        \item Now take the partial derivative of both sides with respect to $t$ or $k$. Either should work. We'll arbitrarily choose $t$.
        $$\frac{\partial u}{\partial x}\frac{\partial x}{\partial t} + \frac{\partial u}{\partial y}\frac{\partial y}{\partial t} = \frac{\partial u^*}{\partial t}$$
        \item Now pattern match against the original equation to get the following PDEs:
        $$\frac{\partial x}{\partial t} = a;\frac{\partial y}{\partial t} = b;\frac{\partial u^*}{\partial t} = -cu$$
        \item Which, when integrated, give:
        $$x = at + f_1(k);y = bt + f_2(k);\frac{\partial u^*}{\partial t} = -cu$$
        In which we have not yet simplified equation 3
        \item We can simplify the $f_1$ and $f_2$ because we don't need that much generality. Simply letting $f_1 = 0$ and $f_2 = k$ will suffice. Thus:
        $$x = at ;y = bt + k;\frac{\partial u^*}{\partial t} = -cu$$
        \item and we can solve for $t$ and $k$ in terms of $y$ and $x$ as a way to invert transform $T$:
        $$t = \frac{x}{a}; k = y - \frac{bx}{a}$$
        Which gives us:
        $$T(u) = u^*(t,k) = u(at,bt + k)$$
        $$T^{-1}(u^*) = u(x,y) = u^*(\frac{x}{a},y - \frac{bx}{a})$$
        \item Now we need to solve the PDE we left above:
        $$\frac{\partial u^*}{\partial t} = -cu$$
    \end{enumerate}
    \newpage
    \item 13) Use the coordinate method to solve the following equation:
    $$u_x + 2u_y + (2x-y)u = 2x^2 + 3xy -2y^2$$
    \begin{enumerate}
        \item Using the methods from the previous problem, we can get:
        $$x = t ;y = 2t + k;\frac{\partial u^*}{\partial t} = 2x^2 + 3xy - 2y^2 - (2x-y)u$$
        $$t = x; k = y - 2x$$
        $$T(u) = u^*(t,k) = u(t,2t + k)$$
        $$T^{-1}(u^*) = u(x,y) = u^*(x,y - 2x)$$
        \item and I am stuck again
    \end{enumerate}
    \newpage
    \item 1.5-5: Solve the following PDE and discuss behavior at boundary conditions given below.
    $$\forall x,y; u_x(x,y) + yu_y(x,y) = 0$$
    Transform coordinates:
    $$\forall s,t; u_x^*(s,t) + y(s,t)u_y^*(s,t) = 0$$
    which gives the following equations:
    $$\frac{\partial x}{\partial s} = 1 \rightarrow x(s,t) = s + f_1(t)$$
    $$\frac{\partial y}{\partial s} = y(s,t) \rightarrow \frac{1}{y(s,t)}\partial y = \partial s \rightarrow y(s,t) = f_2(t)e^s$$
    $$\frac{\partial u^*}{\partial s} = 0 \rightarrow u^*(s,t) = f(t)$$
    And we know that $x(s,t)$ can trace out the whole line without $f_1(t)$, so we will drop $f_1(t)$.  Also, we know that $y(s,t)$ traces out the whole plane if $f_2(t) = t$, so these three equations simplify to:
    $$x(s,t) = s; y(s,t) = te^s; u^*(s,t)=f(t)$$
    We can also invert the coordinate transforms and get:
    $$s(x,y) = x; t(x,y) = \frac{y}{e^x}$$
    And so now we can invert the transform to get $u(x,y)$ from $u^*(s,t)$:
    $$u(x,y) = f(\frac{y}{e^x})$$
    Now we consider the boundary condition $u(x,0) = x$
    $$u(x,0) = f(\frac{0}{e^x}) = f(0) \neq x$$
    We have found a contradiction because $f(0)$ is constant, and $x$ is not.
    Now we consider the boundary condition $u(x,0) = 1$
    $$u(x,0) = f(\frac{0}{e^x}) = f(0) = 1$$
    Thus, we have found a requirement that $f(0) = 1$, but there are no other requirements on $f$, thus, there are still infinitely many possibilities for the function. The moral of this problem is that we're specifying our boundary along our characteristics instead of against them.
    \newpage
    \item 1.5-6: Solve the following PDE
    $$\forall x,y; u_x(x,y) + 2xy^2u_y(x,y) = 0$$
    First, divide the whole equation by $x$. This highlights a problem at $x = 0$
    $$\forall x,y; \frac{1}{x}u_x(x,y) + 2y^2u_y(x,y) = 0$$
    Now we transform coordinates:
    $$\forall s,t; \frac{1}{x(s,t)}u_x^*(s,t) + 2y(s,t)^2u_y^*(s,t) = 0$$
    And we have three smaller PDEs and their solutions:
    $$\frac{\partial x}{\partial s} = \frac{1}{x(s,t)} \rightarrow x(s,t)^2 = 2s + f_1(t) $$
    $$\frac{\partial y}{\partial s} = 2y(s,t)^2 \rightarrow y(s,t) = \frac{-1}{2s + f_2(t)}$$
    $$\frac{\partial u^*}{\partial s} = 0 \rightarrow u^*(s,t) = F(t)$$
    The next issue is how to choose $f_1$ and $f_2$. If we isolate the $s$ in the $x$ equation, we can substitute it in for the $y$ equation to get:
    $$y(s,t) = \frac{-1}{x(s,t)^2 + f(t)}$$
    Which if we simply let $f$ be the negative identity function the characteristics still trace out the whole plane, and we get:
    $$y(s,t) = \frac{-1}{x(s,t)^2 - t} \rightarrow t = \frac{1}{y}+x^2$$
    Which means if we invert the transform on $u^*$, we get:
    $$u(x,y) = F(\frac{1}{y}+x^2)$$
    Which works if we substitute it back into the original equation.

\end{itemize}
\newpage
\section{The Wave Equation}

For the following wave equation:

$$u_{tt} = c^2u_{xx}$$

with the initial conditions:

$$u(x,0) = \phi(x)$$
$$u_t(x,0) = \psi(x)$$

is
$$u(x,t) = \frac{1}{2}\left(\phi(x + ct) + \phi(x-ct)\right) + \frac{1}{2c} \int_{x-ct}^{x+ct} \psi(s) \,ds $$

\subsection{Example Problems}
\begin{itemize}

    \item 2.1-1:
    Solve $$u_{tt} = c^2u_{xx} ; u(x,0) = e^x ; u_t(x,0) = \sin{x}$$
    $$u(x,t) = \frac{1}{2}\left(e^{x + ct} + e^{x - ct}\right) + \frac{1}{2c}\int_{x-ct}^{x+ct} \sin{s} \,ds$$
$$u(x,t) =  \frac{1}{2}\left(e^{x + ct} + e^{x - ct}\right) + \frac{1}{2c} [\cos{(x-ct)}-\cos{(x+ct)}] $$
    \item 3.1-2:
    Solve $$u_{tt} = c^2u_{xx} ; u(x,0) = \log{1 + x^2} ; u_t(x,0) = 4 + x$$
    $$u(x,t) = \frac{1}{2}\left[\log{\left(1 + (x + ct)^2\right)}+\log{\left(1 + (x - ct)^2\right)}\right] + \frac{1}{2c}\int_{x-ct}^{x+ct} (4+s) \,ds$$
    $$u(x,t) = \frac{1}{2}\left[\log{\left(1 + (x + ct)^2\right)}+\log{\left(1 + (x - ct)^2\right)}\right] + 4t + xt$$
\end{itemize}

\newpage
\section{Fourier Series and Boundary Conditions}

I think one of the most important things you can know that will help you when doing fourier series is that fourier series converge to the average of the limit on the left and right. That might seem innocuous, but it will help. Here's how.  We sort of learned two different types or uses of fourier series:

\begin{itemize}
    \item We learned that that the boundary values determine whether you want to use sines or cosines in your expansion.
    \item And we also separately learned that when we're computing fourier series to approximate a function, we often extend the function and find its period etc etc to find the fourier coefficients.
\end{itemize}
  But what I noticed with the help of the above fact is how to unify these two views of the fourier series. What we can really do is use the boundary conditions to choose how to extend the function, and then that's the function we want to approximate. This is helpful because it unifies the two seemingly separate issues of using a fourier series to approximate a function vs choosing a certain type of fourier series to meet the requirements of a boundary condition. Just extend your function to meet your boundary condition, then the correct choice of sines and cosines will fall out.

\end{document}