\section{Uniform and Non-Uniform Hierarchical B-Splines}
\label{sec:31standardBSplines}

\minitoc{83mm}{7}

\noindent
In this section, we mainly follow the presentation of
\multicite{Pflueger10Spatially,Valentin14Hierarchische,Valentin16Hierarchical}
to define hierarchical B-splines
starting from the well-known nodal B-spline basis
\multicite{Hoellig03Finite,Hoellig13Approximation,Quak16About}.
Note that thanks to the groundwork laid in \cref{chap:20sparseGrids},
especially \thmref{lemma:tensorProductLinearIndependence} and
\thmref{prop:splittingUVToMV},
it suffices to study the univariate case
of all bases that will be defined in the rest of this thesis.
The multivariate case is treated canonically by tensor products.

\subsection{Uniform Hierarchical B-Splines}
\label{sec:311uniform}

\paragraph{Cardinal B-splines}

The \term{cardinal B-spline}
$\cardbspl{p}\colon \real \to \real$ of \term{degree} $p \in \natz$
is defined by
\begin{equation}
  \label{eq:cardinalBSpline}
  \cardbspl{p}(x)
  \ceq
  \begin{cases}
    \displaystyle\int_0^1 \cardbspl{p-1}(x - y) \diff{}y,&p \ge 1,\\
    \charfun{\hopint{0, 1}}(x),&p = 0,
  \end{cases}
\end{equation}
where $\charfun{\hopint{0, 1}}$ is the characteristic function of
the half-open unit interval $\hopint{0, 1}$
(see \cite{Hoellig13Approximation}).
The cardinal B-spline $\cardbspl{p}$ has the following properties
\cite{Hoellig03Finite},
which are shown in \cref{fig:cardinalBSplineProps}:

\begin{figure}
  \includegraphics{cardinalBSplineProps_1}%
  \caption[%
    Properties of cardinal B-splines%
  ]{%
    Eight properties of cardinal B-splines using the quadratic case
    $p = 2$ as an example.\\
    \lefthphantom{1.}{5.}\,
    $\cardbspl{p}$ is compactly supported on $\clint{0, p+1}$.\\
    \lefthphantom{2.}{5.}\,
    $\cardbspl{p}$ is symmetric and $0 \le \cardbspl{p} \le 1$.\\
    \lefthphantom{3.}{5.}\,
    $\cardbspl{p}$ is a weighted combination of
    $\cardbspl{p-1}$ \emph{\textcolor{C0}{(blue)}} and
    $\cardbspl{p-1}({\cdot} - 1)$ \emph{\textcolor{C1}{(red)}.}\\
    %where the coefficients \emph{(dashed)} are linear in $x$.\\
    \lefthphantom{4.}{5.}\,
    $\cardbspl{p}$ is a piecewise polynomial of degree $p$.\\
    \lefthphantom{5.}{5.}\,
    $\deriv{x}{\cardbspl{p}}$ \emph{(dashed)}
    is the difference of
    $\cardbspl{p-1}$ \emph{\textcolor{C0}{(blue)}} and
    $\cardbspl{p-1}({\cdot} - 1)$ \emph{\textcolor{C1}{(red)}.}\\
    \lefthphantom{6.}{5.}\,
    $\cardbspl{p}$ has unit integral.\\
    \lefthphantom{7.}{5.}\,
    $\cardbspl{p}$ is the convolution of
    $\cardbspl{p-1}$ \emph{\textcolor{C0}{(blue)}} and $\cardbspl{0}$.\\
    \lefthphantom{8.}{5.}\,
    Hat function \emph{\textcolor{C0}{(blue)}} and
    Gaussian function \emph{\textcolor{C1}{(red)}}
    are special cases of $\cardbspl{p}$.%
  }%
  \label{fig:cardinalBSplineProps}%
\end{figure}

\begin{enumerate}
  \item
  \emph{Compact support:}
  The support of $\cardbspl{p}$ is given by $\supp \cardbspl{p} = \clint{0, p + 1}$.

  \item
  \emph{Bounds and symmetry:}
  The cardinal B-spline $\cardbspl{p}$ is non-negative and bounded from above by one.
  It is symmetric with respect to $x = \tfrac{p+1}{2}$, i.e.,
  $\cardbspl{p}(x) = \cardbspl{p}(p + 1 - x)$.

  \item
  \emph{Recursion:}
  The cardinal B-spline $\cardbspl{p}$ ($p \ge 1$)
  satisfies the following recurrence relation
  (which can be used as an alternative definition):
  \begin{equation}
    \cardbspl{p}(x)
    = \frac{x}{p} \cardbspl{p-1}(x) + \frac{p+1-x}{p} \cardbspl{p-1}(x-1).
  \end{equation}

  \item
  \emph{Spline:}
  On every \term{knot interval} $\hopint{k, k+1}$ for $k = 0, \dotsc, p$,
  $\cardbspl{p}$ is a polynomial of degree~$p$, i.e.,
  $\cardbspl{p}$ is a spline of degree $p$ (piecewise polynomial).

  \item
  \emph{Derivative:}
  At the \term{knots} $k = 0, \dotsc, p + 1$,
  $\cardbspl{p}$ is $(p - 1)$ times continuously differentiable (if $p \ge 1$).
  The derivative can be computed by differentiating
  \eqref{eq:cardinalBSpline}:
  \begin{equation}
    \label{eq:cardinalBSplineDerivative}
    \deriv{x}{\cardbspl{p}}(x)
    = \cardbspl{p-1}(x) - \cardbspl{p-1}(x-1),\quad
    x \in \real.
  \end{equation}

  \item
  \emph{Integral:}
  The B-spline $\cardbspl{p}$ has unit integral, i.e.,
  $\int_{\real} \cardbspl{p}(x) \dx = 1$.
  %\vspace{-0.5em}
  %\begin{equation}
  %  \int_{\real} \cardbspl{p}(x) \dx = 1.
  %\end{equation}

  \item
  \emph{Convolution:}
  The integral in the definition of $\cardbspl{p}$
  is the convolution $\cardbspl{p-1} \convolution \cardbspl{0}$
  of the B-spline $\cardbspl{p-1}$
  of degree $p - 1$ with the B-spline $\cardbspl{0}$ of degree zero.

  \item
  \emph{Generalization:}
  As a special case, $\cardbspl{1}$ is a hat function,
  interpolating the data
  $\{(k, \kronecker{k}{1}) \mid k \in \integer\}$.
  For $p \to \infty$, the normalized cardinal B-splines converge
  pointwise to the standard Gaussian function
  $\cardbspl{\infty}(x) \ceq (2\pi)^{-1/2} \exp(-x^2/2)$ \cite{Unser92Asymptotic}:%
  \footnote{%
    This can also be seen as a consequence of the central limit theorem
    applied to uniformly distributed random variables.
    The pointwise convergence of the probability density functions
    can be proven from the convergence
    in distribution using a converse to Scheffé's theorem
    \cite{Boos85Converse}.%
  }
  \vspace{-0.5em}
  \begin{equation}
    \lim_{p \to \infty}
    c^p \cardbspl{p}(c^p x + \tfrac{p+1}{2})
    = \cardbspl{\infty}(x),\quad
    c^p \ceq \sqrt{\frac{p+1}{12}},\quad
    x \in \real.
  \end{equation}
\end{enumerate}

The cardinal B-splines of the first degrees are shown in
\cref{fig:cardinalBSpline}.
Due to the convolution property,
cardinal B-splines of degree $p \ge 2$ are ``smoothed versions''
of the hat function.
This is shown in the flip book animation in the bottom left corner
of the even-numbered pages of this thesis.

\begin{figure}
  \includegraphics{cardinalBSpline_1}%
  \caption[%
    Cardinal B-splines%
  ]{%
    Cardinal B-splines $\cardbspl{p}$ up to quintic degree $p = 5$.%
  }%
  \label{fig:cardinalBSpline}%
\end{figure}

\paragraph{Definition of uniform hierarchical B-splines}

\usenotation{Ëp}
As for the hat functions in \cref{chap:20sparseGrids},
we can use the cardinal B-spline $\cardbspl{p}$ as a ``parent function'' to
define the uniform hierarchical B-spline
$\bspl{l,i}{p}\colon \clint{0, 1} \to \real$ of level~$l \in \natz$ and index
$i \in \hiset{l}$ via an affine parameter transformation
\cite{Pflueger10Spatially}:
\begin{equation}
  \label{eq:uniformHierarchicalBSplineUV}
  \bspl{l,i}{p}(x)
  \ceq \cardbspl{p}(\tfrac{x}{\ms{l}} + \tfrac{p+1}{2} - i).
\end{equation}
The support of $\bspl{l,i}{p}$ is given
by $\supp \bspl{l,i}{p} = \clint{0, 1} \cap \clint{\gp{l,i-(p+1)/2}, \gp{l,i+(p+1)/2}}$.
The hat function basis $\bspl{l,i}{1}$ defined in
\eqref{eq:hatFunctionUV} is a special case of
\eqref{eq:uniformHierarchicalBSplineUV} for $p = 1$,
which allows us to use the same notation $\bspl{l,i}{p}$ for both.
Note that due to the \term{translation invariance} of $\bspl{l,i}{p}$
(i.e., the basis functions are the same up to scaling and translation),
it suffices to precompute and implement the polynomial pieces of $\cardbspl{p}$
to enable evaluations of all hierarchical B-splines
$\bspl{l,i}{p}$ ($l \in \natz$, $i \in \hiset{l}$).

\begin{figure}
  \subcaptionbox{%
    Nodal B-splines $\bspl{l,i}{p}$ ($i \in \hiset{l}$) and grid points $\gp{l,i}$
    \emph{(dots).}%
  }[67mm]{%
    \includegraphics{hierarchicalBasis_4}%
  }%
  \hfill%
  \begin{tikzpicture}
    \draw[decorate,decoration={brace,aspect=0.153}] (0,-8.5) -- (0,0);
    \node[anchor=east,inner sep=0mm] at (-0.15,-7.208) {$= \bigoplus$};
  \end{tikzpicture}%
  \hfill%
  \subcaptionbox{%
    Hierarchical B-splines $\bspl{l',i'}{p}$ ($l' \le l$, $i' \in \hiset{l'}$)
    and grid points $\gp{l',i'}$
    \emph{(dots).}%
  }[69mm]{%
    \includegraphics{hierarchicalBasis_5}%
  }%
  \caption[%
    Nodal and hierarchical B-splines%
  ]{%
    Univariate nodal and hierarchical cubic B-splines ($p = 3$)
    \vspace{0.1em}%
    up to level $l = 3$.
    When restricting all functions to $\rspldomain{l}{p}$
    \emph{(thick black line),}
    \vspace{-0.2em}%
    the nodal space $\restrictspace{\nsbspl{l}{p}}{\rspldomain{l}{p}}$
    decomposes into the direct sum of the hierarchical subspaces
    $\restrictspace{\hsbspl{l'}{p}}{\rspldomain{l}{p}}$ ($l' \le l$).%
  }%
  \label{fig:hierarchicalBSpline}%
\end{figure}

\paragraph{Odd and even degrees}

In this thesis, we will only allow odd degrees $p = 1, 3, 5, \dotsc$
for hierarchical B-splines.
Many theoretical considerations fail for even degrees.
The basic reason is that for odd degrees, the knots of
$\bspl{l,i}{p}$ coincide with the grid points \cite{Valentin14Hierarchische}
\begin{equation}
  \gp{l,i-(p+1)/2},\quad
  \dotsc,\quad
  \gp{l,i},\quad
  \dotsc,\quad
  \gp{l,i+(p+1)/2}.
\end{equation}
For even degrees $p$, the knots of $\bspl{l,i}{p}$ lie exactly in
the middle between two subsequent grid points:
\begin{equation}
  \gp{l,i-p/2} - \frac{\ms{l}}{2},\quad
  \dotsc,\quad
  \gp{l,i} - \frac{\ms{l}}{2},\quad
  \gp{l,i} + \frac{\ms{l}}{2},\quad
  \dotsc,\quad
  \gp{l,i+p/2} + \frac{\ms{l}}{2}.
\end{equation}
This fact has many adverse implications on the hierarchical basis.
The most crucial implication is
that for even degrees $p$,
the hierarchical splitting \eqref{eq:hierSplittingUV} does not hold.
Furthermore,
we would not be able to define non-uniform hierarchical B-splines as
simple as for odd degrees and
fundamental splines would not be defined at all
(as we will see in \cref{sec:443fundamentalSplines}).
Additionally,
odd degrees include the hat function case~($p = 1$) and the
most commonly applied cubic degree~($p = 3$).
Therefore,
it is reasonable to restrict ourselves to odd degrees
for the rest of the thesis.

\subsection{Non-Uniform B-Splines and Proof of the Hierarchical Splitting}
\label{sec:312proofHierarchicalSplitting}

\paragraph{Non-uniform B-splines and spline space}

With the hierarchical B-splines $\bspl{l,i}{p}$, we can define
the nodal spaces $\nsbspl{l}{p}$ and hierarchical subspaces $\hsbspl{l}{p}$
as in \cref{chap:20sparseGrids}.
However, in order for the hierarchical splitting \eqref{eq:hierSplittingUV}
to be correct, we have to prove that the conditions of
\thmref{lemma:hierSplittingUV} are satisfied.
To investigate how the nodal space $\nsbspl{l}{p}$ looks like,
we introduce the notion of non-uniform B-splines.

\begin{definition}[non-uniform B-splines]
  \label{def:nonUniformBSpline}
  Let $m, p \in \natz$ and $\knotseq = (\knot{0}, \dotsc, \knot{m+p})$ be an
  increasing sequence of real numbers \term{(knot sequence).}
  For $k = 0, \dotsc, m - 1$,
  the \term{(non-uniform) B-spline} $\nonunifbspl{k,\knotseq}{p}$ of degree $p$
  with knots~$\knotseq$ and index $k$ is defined by the
  Cox--de~Boor recurrence
  \multicite{Cox72Numerical,Boor72Calculating,Hoellig13Approximation}
  \begin{equation}
    \nonunifbspl{k,\knotseq}{p}(x)
    \ceq
    \begin{cases}
      \dfrac{x - \knot{k}}{\knot{k+p} - \knot{k}} \nonunifbspl{k,\knotseq}{p-1}(x) +
      \dfrac{\knot{k+p+1} - x}{\knot{k+p+1} - \knot{k+1}}
      \nonunifbspl{k+1,\knotseq}{p-1}(x),&p \ge 1,\\
      \charfun{\hopint{\knot{k}, \knot{k+1}}}(x),&p = 0.
    \end{cases}
    \hspace*{-4mm}
  \end{equation}
\end{definition}
Note that when choosing $\knotseq = (0, 1, \dotsc, p + 1)$ and
$k = 0$, we obtain the cardinal B-spline~$\cardbspl{p}$.
\Cref{def:nonUniformBSpline} can be used to characterize
the nodal space $\nsbspl{l}{p}$:

\begin{proposition}[spline space]
  \label{prop:splineSpace}
  Let $\knotseq = (\knot{0}, \dotsc, \knot{m+p})$ be a knot sequence.
  Then, the B-splines $\nonunifbspl{k,\knotseq}{p}$ ($k = 0, \dotsc, m - 1$)
  form a basis of the \term{spline space}
  \begin{equation}
    \nonunifsplspace{\knotseq}{p}
    \ceq \spn\{\nonunifbspl{k,\knotseq}{p} \mid k = 0, \dotsc, m - 1\}.
  \end{equation}
  $\nonunifsplspace{\knotseq}{p}$ contains exactly those functions that are continuous
  on $\spldomain{\knotseq}{p} \ceq \clint{\knot{p}, \knot{m}}$,
  polynomials of degree $\le p$ on every knot interval
  $\hopint{\knot{k}, \knot{k+1}}$ in
  $\spldomain{\knotseq}{p}$
  ($k = p, \dotsc, m - 1$) and at least $(p - 1)$ times
  continuously differentiable at every knot $\knot{k}$ in the interior of
  $\spldomain{\knotseq}{p}$ ($k = p + 1, \dotsc, m - 1$).
\end{proposition}

\begin{proof}
  See \cite{Hoellig13Approximation}.
\end{proof}

This proposition gives the reason for the letter ``B'' in ``B-splines,''
which stands for ``basis'' (of the space of splines) \cite{Schoenberg67Spline}.
One example of a knot sequence and the corresponding B-splines is
given in \cref{fig:splineSpaceGeneral}.
The key observation is that B-splines of a knot sequence $\knotseq$
do not form a basis of the spline space on the union
$\clint{\knot{0}, \knot{m+p}}$ of the B-spline supports.
Instead, they form a basis of the spline space
on a proper sub-interval $\spldomain{\knotseq}{p}$.
Intuitively, for every point in $\spldomain{\knotseq}{p}$ that is not a knot,
exactly $p + 1$ B-splines must be \term{relevant} (i.e., non-zero)
to uniquely span the spline space,
as on every knot interval, the spline is a polynomial of degree $\le p$
and therefore, there must be $p + 1$ degrees of freedom.
Outside $\spldomain{\knotseq}{p}$, there are too few relevant B-splines
to span the spline space.
This fact,
which is shown in \cref{fig:splineSpaceUniform},
forces us to restrict the nodal space and the hierarchical subspaces
to~$\spldomain{\knotseq}{p}$:

\begin{figure}
  \includegraphics{splineSpace_1}%
  \caption[%
    Non-uniform B-splines with knot sequence and interpolation domain%
  ]{%
    Knot sequence $\knotseq = (\knot{0}, \dotsc, \knot{m+p})$
    \vphantom{$\nonunifbspl{0,\knotseq}{p}$}%
    with the corresponding $m = 7$ non-uniform cubic B-splines
    $\nonunifbspl{k,\knotseq}{p}$ ($k = 0, \dotsc, m - 1$, $p = 3$).
    On $\spldomain{\knotseq}{p}$ \emph{(thick line, delimited by dashed lines),}
    which starts with the last knot interval of the first B-spline
    $\nonunifbspl{0,\knotseq}{p}$
    and ends with the first knot interval of the last B-spline
    $\nonunifbspl{m-1,\knotseq}{p}$,
    the B-splines span the spline space $\nonunifsplspace{\knotseq}{p}$.
    Elements of this space are splines $\spl\colon \spldomain{\knotseq}{p} \to \real$
    \emph{(black line),}
    which are linear combinations
    $\spl = \sum_{k=0}^{m-1} \interpcoeff{k} \nonunifbspl{k,\knotseq}{p}$
    of the B-splines.%
  }%
  \label{fig:splineSpaceGeneral}%
\end{figure}

\begin{figure}
  \includegraphics{splineSpace_2}%
  \caption[%
    Uniform nodal B-splines and knot sequence%
  ]{%
    Uniform knot sequence $\nodalknotseq{l}{p}$ \emph{(ticks on horizontal axis)}
    and corresponding nodal cubic B-splines ($p = 3$) of level $l = 3$.
    In the domain $\clint{0, 1}$ \emph{(delimited by dashed lines),}
    the grid points $\fgset{l}$ \emph{\textcolor{mittelblau}{(blue dots)}}
    coincide with the B-spline knots.
    The spline interpolation domain
    $\rspldomain{l}{p} \ceq \spldomain{\nodalknotseq{l}{p}}{p}$
    \emph{(thick line)}
    is only a proper subset of~$\clint{0, 1}$.%
  }%
  \label{fig:splineSpaceUniform}%
\end{figure}

\begin{corollary}[nodal B-spline space]
  \label{cor:nodalBSplineSpace}
  The restricted nodal B-splines $\restrictfcn{\bspl{l,i}{p}}{\rspldomain{l}{p}}$
  ($i = 0, \dotsc, 2^l$)
  of level $l \in \natz$ are
  a basis of the spline space $\nonunifsplspace{\nodalknotseq{l}{p}}{p}$,
  where
  \begin{subequations}
    \begin{gather}
      \nodalknot{l,k}{p}
      \ceq (k - \tfrac{p+1}{2}) \ms{l},\quad
      k = 0, \dotsc, m + p,\quad
      m \ceq 2^l + 1,\\
      \rspldomain{l}{p} \ceq [\tfrac{p-1}{2} \ms{l},\;
      1 - \tfrac{p-1}{2} \ms{l}],
    \end{gather}
  \end{subequations}
  and consequently
  \begin{equation}
    \restrictspace{\nsbspl{l}{p}}{\rspldomain{l}{p}}
    = \restrictedsplspace{l}{p}
    \ceq \nonunifsplspace{\nodalknotseq{l}{p}}{p}.
  \end{equation}
\end{corollary}

\begin{proof}
  We have $\bspl{l,i}{p} = \nonunifbspl{i,\nodalknotseq{l}{p}}{p}$ for
  $i = 0, \dotsc, m - 1$,
  as the B-splines on both sides have the same knots.
  The assertions now follow from \thmref{prop:splineSpace}.
\end{proof}

\vspace{1em}

Note that $\rspldomain{l}{p}$ might contain only a single point or even be empty,
if $p$ is too large or $l$ is too small.
However, the corresponding B-splines $\bspl{l,i}{p}$ are still linearly
independent on~$\clint{0, 1}$ (see \cite{Hoellig13Approximation}).
Similarly, the corollary also implies that the hierarchical functions
$\bspl{l,i}{p}$ of level $l$ ($i \in \hiset{l}$)
are linearly independent on $\clint{0, 1}$.

\paragraph{Hierarchical splitting for uniform B-splines}

We can use \cref{prop:splineSpace} and \cref{cor:nodalBSplineSpace}
to prove the hierarchical splitting for the uniform B-spline basis
\cite{Valentin16Hierarchical}.

\begin{lemma}[hierarchical B-splines in nodal space]
  \label{lemma:hierBSplineInNodalSpace}
  The restricted hierarchical subspaces
  $\restrictspace{\hsbspl{l'}{p}}{\rspldomain{l}{p}}$ ($l' \le l$) are
  subspaces of the restricted nodal space $\restrictspace{\nsbspl{l}{p}}{\rspldomain{l}{p}}$.
\end{lemma}

\begin{proof}
  Every function $\bspl{l',i'}{p}$ ($i' \in \hiset{l'}$) is continuous on
  $\rspldomain{l}{p}$, a polynomial of degree $\le p$ on every knot interval
  of $\nodalknotseq{l}{p}$ (due to $p$ odd),
  and at the knots themselves at least $(p - 1)$ times continuously
  differentiable.
  \Cref{prop:splineSpace} implies $\bspl{l',i'}{p} \in \restrictedsplspace{l}{p}$
  and from \cref{cor:nodalBSplineSpace}, it follows
  $\bspl{l',i'}{p} \in \restrictspace{\nsbspl{l}{p}}{\rspldomain{l}{p}}$.
  As the functions $\bspl{l',i'}{p}$ ($i' \in \hiset{l'}$) span
  $\restrictspace{\hsbspl{l'}{p}}{\rspldomain{l}{p}}$, we can conclude
  $\restrictspace{\hsbspl{l'}{p}}{\rspldomain{l}{p}} \subset \restrictspace{\nsbspl{l}{p}}{\rspldomain{l}{p}}$.
\end{proof}

\vspace{1em}

It is crucial to note that this lemma does not hold for even $p$,
as the knots of the B-splines of level $l - 1$ are not contained in the
knots of level $l$.
This implies that in general,
$\restrictspace{\hsbspl{l-1}{p}}{\rspldomain{l}{p}}$
is not contained in $\restrictspace{\nsbspl{l}{p}}{\rspldomain{l}{p}}$
for even degrees $p$.
Therefore, the hierarchical splitting equation \eqref{eq:hierSplittingUV}
does not hold.

\begin{restatable}[hierarchical B-splines are linearly independent]{%
  proposition%
}{%
  propHierBSplineLinearlyIndependent%
}
  \label{prop:hierBSplineLinearlyIndependent}
  The hierarchical B-splines
  $\bspl{l',i'}{p}$ ($l' \le l$, $i' \in \hiset{l'}$)
  are linearly independent.
\end{restatable}

\begin{proof}
  See \cref{sec:a121proofHierBSplineLinearlyIndependent}.
\end{proof}

\vspace{1em}

Although we have to restrict all functions and spaces to $\rspldomain{l}{p}$,
\thmref{lemma:hierSplittingUV} is still applicable to prove that
the hierarchical splitting equation \eqref{eq:hierSplittingUV}
is correct for hierarchical B-splines:

\begin{corollary}[hierarchical splitting for uniform B-splines]
  \label{cor:hierSplittingBSpline}
  The hierarchical splitting \eqref{eq:hierSplittingUV}
  holds for the hierarchical B-spline basis
  if restricting all functions to $\rspldomain{l}{p}$:
  \begin{equation}
    \restrictedsplspace{l}{p}
    = \restrictspace{\nsbspl{l}{p}}{\rspldomain{l}{p}}
    = \bigoplus_{l'=0}^l \restrictspace{\hsbspl{l'}{p}}{\rspldomain{l}{p}}.
  \end{equation}
\end{corollary}

\begin{proof}
  Analogously to the proof of \thmref{lemma:hierSplittingUV}
  and apply \thmref{cor:nodalBSplineSpace}.
\end{proof}

\vspace{1em}

This corollary is also visualized in \cref{fig:hierarchicalBSpline}.
We can now proceed to define
multivariate nodal spaces $\nsbspl{\*l}{p}$,
multivariate hierarchical subspaces $\hsbspl{\*l}{p}$, and
sparse grid spaces $\regsgspace[p]{n}{d}$ as in \cref{chap:20sparseGrids}.
Note that it is possible to choose different degrees $p_t$ for
different dimensions $t = 1, \dotsc, d$,
since the hierarchical splitting \eqref{eq:hierSplittingMV} does not
require the bases in each dimension to be the same.
Consequently, we can define degree-dimension-adaptive
(so-called \term{$hp$-adaptive}) sparse grids
$\regsgspace[\*p]{n}{d}$ for arbitrary odd degree vectors $\*p$.

In the course of this thesis, we will derive multiple variations
of the standard hierarchical B-spline basis.
We will not repeat formal proofs of the hierarchical splitting equation
\eqref{eq:hierSplittingUV}
(i.e., verifying the two conditions of \cref{lemma:hierSplittingUV})
for each of these bases for the sake of brevity.
The idea of the proof of \cref{prop:hierBSplineLinearlyIndependent},
which is inductively exploiting the smoothness conditions given by
B-splines of previous levels, can be transferred to similar B-spline
bases.

\subsection{Modification}
\label{sec:313modification}

\paragraph{Marsden's identity}

Similar to the piecewise linear case in \cref{sec:242modified},
Pflüger defined modified
hierarchical B-splines to obtain reasonable values on the boundary
without having to place grid points there \cite{Pflueger10Spatially}.
The main motivation is to define basis
functions $\bspl[\modified]{l,i}{p}$ that satisfy natural boundary
conditions, i.e.,
\begin{equation}
  \label{eq:naturalBoundaryConditions}
  \deriv[2]{x}{\bspl[\modified]{l,i}{p}}(x) = 0,\quad
  x \in \bndrydomain{\clint{0, 1}} = \{0, 1\}.
\end{equation}
Originally, this requirement stems from financial problems
\cite{Pflueger10Spatially}.
For the left boundary,
\eqref{eq:naturalBoundaryConditions} can be satisfied by
modifying the left-most function $\bspl{l,1}{p}$ such that
$\bspl[\modified]{l,1}{p}(x) = 2 - \tfrac{x}{\ms{l}}$ is a linear polynomial
when $x$ is ``near'' the boundary.
As in \cite{Pflueger10Spatially},
we append $\bspl{l,1}{p}$ with
B-splines $\bspl{l,i}{p}$ with index $i \le 0$ and
use \term{Marsden's identity} to compute the corresponding coefficients.
The identity enables us to explicitly compute the B-spline coefficients
of an arbitrary polynomial of degree $\le p$ by giving an explicit formula
for the B-spline coefficients of shifted monomials $({\cdot} - y)^p$.
Interestingly, the coefficients are polynomials themselves (in $y$):

\vspace*{0pt plus 0.5fill}

\begin{lemma}[Marsden's identity]
  \label{lemma:marsden}
  Let $p \in \natz$ and
  $\knotseq = (\knot{0}, \dotsc, \knot{m+p})$ be a knot sequence.
  Then, for all $x \in \spldomain{\knotseq}{p}$ and $y \in \real$,
  \begin{equation}
    \label{eq:marsden}
    (x - y)^p
    = \sum_{k=0}^{m-1}
    \;\bracket*{(\knot{k+1} - y) \dotsm (\knot{k+p} - y)}\;
    \nonunifbspl{k,\knotseq}{p}(x).
  \end{equation}
\end{lemma}

\begin{proof}
  See \cite{Hoellig13Approximation}.
\end{proof}

\vspace*{0pt plus 1fill}

\paragraph{Modified hierarchical B-splines}

By extending the sum in Marsden's identity to $i \in \integer$ and
comparing the coefficients of $y^p$ and $y^{p-1}$ of both sides,
we obtain representations for the first two monomials:
\begin{equation}
  1
  = \sum_{i \in \integer} \bspl{l,i}{p}(x),\quad
  x
  = \sum_{i \in \integer} \gp{l,i} \bspl{l,i}{p}(x),\quad
  x \in \real.
\end{equation}
This can be used to derive a definition for $\bspl[\modified]{l,i}{p}$:
\begin{equation}
  \label{eq:modifiedBSplineConstruction}
  2 - \frac{x}{\ms{l}}
  = 2 \sum_{i \in \integer} \bspl{l,i}{p}(x)
  - \frac{1}{\ms{l}} \sum_{i \in \integer} \gp{l,i} \bspl{l,i}{p}(x)
  = \sum_{i \in \integer} (2 - i) \bspl{l,i}{p}(x),\quad
  x \in \real.
\end{equation}
Note that only the B-splines with $i \ge 1 - \tfrac{p+1}{2}$
are relevant for the unit interval
as all other B-splines vanish in $\clint{0, 1}$.
Pflüger omits summands with $i > 1$ as he only wants to modify
$\bspl{l,1}{p}$ left of its grid point $\gp{l,1}$ \cite{Pflueger10Spatially}.
The right-most function $\bspl[\modified]{l,2^l-1}{p}$ can be derived
analogously by mirroring $\bspl[\modified]{l,1}{p}$ at $x = \tfrac{1}{2}$.
\pagebreak%
For~$l = 1$, again the ``constant one'' function is taken for the definition
of modified hierarchical B-splines (see \cref{fig:modifiedBSpline}):
{%
  \setlength{\abovedisplayskip}{9pt}%
  \setlength{\belowdisplayskip}{9pt}%
  \begin{equation}
    \label{eq:modifiedBSplineDefinition}
    \bspl[\modified]{l,i}{p}(x)
    \ceq
    \begin{cases}
      1,&
      l = 1,\quad i = 1,\\
      \displaystyle\sum_{i'=1-(p+1)/2}^1\!\!\!\! (2 - i') \bspl{l,i'}{p}(x),&
      l \ge 2,\quad i = 1,\\
      \bspl{l,i}{p}(x),&
      l \ge 2,\quad i \in \hiset{l} \setminus \{1, 2^l - 1\},\\
      \bspl[\modified]{l,1}{p}(1 - x),&
      l \ge 2,\quad i = 2^l - 1.
    \end{cases}
  \end{equation}%
}%
By \thmref{prop:splineSpace},
this definition implies that
$\bspl{l,1}{p}(x) = 2 - \tfrac{x}{\ms{l}}$ ($l \ge 2$)
is only valid for $x \in \clint{0, \tfrac{5-p}{2} \ms{l}}$, i.e.,
the second derivative at $x = 0$ vanishes only for $p \le 5$.
For higher degrees, it is non-zero, albeit very small
in its absolute value.
To enforce natural boundary conditions
for higher than quintic degrees,
the upper bound of $i'$ in the sum in \eqref{eq:modifiedBSplineDefinition}
must be extended to $\tfrac{p+1}{2}$.
In addition, more than just the left-most and the right-most inner
hierarchical B-spline must be modified for $p \ge 5$,
as the size of the supports of $\bspl{l,i}{p}$ increases
for growing $p$.

\begin{figure}
  \subcaptionbox{%
    Construction of the modified
    \vphantom{$\bspl[\modified]{l',i'}{p}$}\vspace{-0.2em}%
    hierarchical cubic B-spline
    $\bspl[\modified]{l',1}{p}$ (\emph{dashed,} $l' \ge 2$)
    \vphantom{$\bspl[\modified]{l',i'}{p}$}\vspace{-0.2em}%
    as a linear combination of neighboring B-splines $\bspl{l',i'}{p}$.%
    \vphantom{$\bspl[\modified]{l',i'}{p}$}%
  }[72mm]{%
    \includegraphics{modifiedBSpline_1}%
  }%
  \hfill%
  \subcaptionbox{%
    Modified hierarchical cubic B-splines
    \vphantom{$\bspl[\modified]{l',i'}{p}$}\vspace{-0.2em}%
    $\bspl[\modified]{l',i'}{p}$
    ($l' = 1, \dotsc, l$, $i' \in \hiset{l'}$) and
    \vphantom{$\bspl[\modified]{l',i'}{p}$}\vspace{-0.2em}%
    grid points $\gp{l',i'}$
    \emph{(dots).}%
    \vphantom{$\bspl[\modified]{l',i'}{p}$}%
  }[72mm]{%
    \includegraphics{hierarchicalBasis_6}%
  }%
  \caption[%
    Modified hierarchical B-splines%
  ]{%
    Construction of modified hierarchical cubic B-splines ($p = 3$) and
    the resulting basis up to level $l = 3$.%
  }%
  \label{fig:modifiedBSpline}%
\end{figure}

\subsection{Non-Uniform Hierarchical B-Splines}
\label{sec:314nonUniform}

Sparse grid spaces and their corresponding grid point sets,
as we have defined them in \cref{chap:20sparseGrids},
are completely independent of the actual location of the grid points
$\gp{l,i}$.
Therefore, it is possible to use different distributions for the grid points
than the standard equidistant choice of $\gp{l,i} = i \cdot \ms{l}$,
if the basis functions are altered accordingly
\cite{Valentin14Hierarchische}.
\usenotation{Ëcc}
The so-called Chebyshev points $\ccgp{l,i}$ and the
resulting Clenshaw--Curtis grids and B-splines will serve as an example.

\paragraph{Chebyshev points and Clenshaw--Curtis grids}

The \term{Chebyshev points} $\ccgp{l,i}$ of level $l \in \natz$
are defined as
\begin{equation}
  \ccgp{l,i}
  \ceq \frac{1 - \cos(\pi \gp{l,i})}{2},\quad
  i = 0, \dotsc, 2^l,
\end{equation}
see \cite{Xu16Chebyshev} for example.
Chebyshev points are the locations of the extrema%
\footnote{%
  The literature sometimes uses the name ``Chebyshev points'' for
  the roots of $T_{2^l}$, which are closely connected with the extrema.
  One way to distinguish them is to call the extrema
  ``Chebyshev--Lobatto points'' and the roots
  ``Chebyshev--Gauss points'' \cite{Xu16Chebyshev}.%
}
of the Chebyshev polynomials $T_{2^l}$, defined as
$T_{2^l}\colon \clint{0, 1} \to \real$,
$T_{2^l}(x) \ceq \cos(2^l \arccos(2x - 1))$.
They are geometrically obtained
by dividing a semicircle into $2^l$ equally-sized
segments and subsequently orthogonally projecting the
segment endpoints onto the diameter
(see \cref{fig:pointSplittingChebyshev}).
Analytically, they are determined by minimizing
the interpolation error term in polynomial interpolation.
The most practical use of Chebyshev points is in
polynomial interpolation to avoid Runge's phenomenon and in
numerical integration (quadrature), resulting in the
so-called Clenshaw--Curtis quadrature rules.

\begin{SCfigure}
  \includegraphics{pointSplitting_2}%
  \caption[%
    Decomposition of the set of univariate Clenshaw--Curtis grid points%
  ]{%
    The set of Chebyshev points $\fgset[\cc]{l}$ of level
    $l = 4$ \emph{(top)}
    decomposes into hierarchical grids of level $l' \le l$
    (compare with \cref{fig:pointSplittingUniform}).
    The Chebyshev points are constructed as
    the orthogonal projection of the
    endpoints of $2^l$ equally-sized segments
    of a semicircle onto its diameter \emph{\textcolor{C8}{(gray, top)}.}%
  }%
  \label{fig:pointSplittingChebyshev}%
\end{SCfigure}

In some settings, it may be beneficial to use full or sparse grids consisting
of Chebyshev points, which are then called \term{Clenshaw--Curtis grids,}
instead of uniform grids.
Besides the already mentioned advantages for polynomials and
quadrature, Clenshaw--Curtis grids can help to reduce interpolation
errors in a neighborhood of the boundary of the domain due to the increased
grid point density near the boundary
(at the cost of a lower grid point density in the interior).
If we want to use Chebyshev points as grid points for sparse grids,
we have to employ an appropriate basis to ensure that interpolation
is still possible.

\paragraph{Hierarchical Clenshaw--Curtis B-splines}

The \term{hierarchical Clenshaw--Curtis B-spline}
$\bspl[\cc]{l,i}{p}$ of level $l \in \natz$ and index
$i \in \hiset{l}$ is defined as \cite{Valentin14Hierarchische}
\begin{equation}
  \bspl[\cc]{l,i}{p}
  \ceq \nonunifbspl{i,\nodalknotseq[\cc]{l}{p}}{p},
\end{equation}
where
\begin{subequations}
  \label{eq:clenshawCurtisBSplineKnots}
  \begin{gather}
    \nodalknot[\cc]{l,k}{p}
    \ceq
    \begin{cases}
      (k - \frac{p+1}{2}) \cdot \ccgp{l,1},&
      k = 0,\, \dotsc,\, \frac{p-1}{2},\\
      \ccgp{l,k-(p+1)/2},&
      k = \frac{p+1}{2},\, \dotsc,\, 2^l + \frac{p+1}{2},\\
      1 + (k - 2^l - \frac{p+1}{2}) \cdot \ccgp{l,1},&
      k = 2^l + \frac{p+3}{2},\, \dotsc,\, 2^l + p + 1,
    \end{cases}\\
    k = 0, \dotsc, m + p,\quad
    m \ceq 2^l + 1.
  \end{gather}
\end{subequations}
For the construction of the knot sequence $\nodalknotseq[\cc]{l}{p}$,
the Chebyshev points $\ccgp{l,i}$
are equidistantly extended onto the real line $\real$.
As for the standard hierarchical B-spline basis,
it is now straightforward to define nodal spaces
$\nsbspl[\cc]{l}{p}$
and hierarchical subspaces $\hsbspl[\cc]{l}{p}$ as well as
sparse grid spaces $\regsgspace[p,\cc]{n}{d}$ and
grid point sets $\regsgset[\cc]{n}{d}$
using tensor products of Clenshaw--Curtis B-splines
and Cartesian products of Chebyshev point sets.
The one-dimensional cubic Clenshaw--Curtis basis and a
sparse Clenshaw--Curtis grid are shown in \cref{fig:clenshawCurtis}.

\begin{figure}
  \subcaptionbox{%
    Hierarchical cubic Clenshaw--Curtis B-splines
    $\bspl[\cc]{l',i'}{p}$
    ($l' \le l$, $i' \in \hiset{l'}$, $p = 3$) and
    \vspace{-0.2em}%
    modified Clenshaw--Curtis B-splines
     $\bspl[\cc,\modified]{l',i'}{p}$
    \emph{(dashed).}%
  }[72mm]{%
    \includegraphics{hierarchicalBasis_7}%
  }%
  \hfill%
  \subcaptionbox{%
    Sparse Clenshaw--Curtis grid $\regsgset[\cc]{n}{d}$
    of level $n = 4$ in $d = 2$ dimensions.%
  }[72mm]{%
    \includegraphics{sg_7}%
  }%
  \caption[%
    Clenshaw--Curtis B-splines and sparse grids%
  ]{%
    Clenshaw--Curtis B-splines and sparse grids.%
  }%
  \label{fig:clenshawCurtis}%
\end{figure}

Note that Clenshaw--Curtis B-splines
$\bspl[\cc]{l,i}{p}$ are not translation-invariant,
in contrast to uniform B-splines $\bspl{l,i}{p}$.
As a result, both implementation effort and computation time for evaluation
are significantly higher for Clenshaw--Curtis B-splines than
for uniform B-splines,
as the Clenshaw--Curtis B-splines cannot be precomputed.

\paragraph{Modification and generalizations}

We define \term{modified hierarchical Clenshaw--Curtis B-splines}
$\bspl[\cc,\modified]{l,i}{p}$ using the
same method as in \cref{eq:modifiedBSplineDefinition}.
Here, the second derivative of $\bspl[\cc,\modified]{l,1}{p}$
does not vanish at $x = 0$ even for degrees $p \le 5$,
as the formula \eqref{eq:modifiedBSplineConstruction}
derived from \thmref{lemma:marsden} assumes uniform knots.
However, since most of the B-spline knots in the summation formula
lie outside $\clint{0, 1}$ and are thus uniform according
to \cref{eq:clenshawCurtisBSplineKnots},
the absolute deviation of the second derivative from zero is small.
Again, to enforce natural boundary conditions,
we can recompute the coefficients
of the components $\bspl[\cc]{l,i'}{p}$
dynamically with Marsden's identity using the correct Chebyshev knots
in \cref{eq:marsden}.

Note that our framework permits arbitrary grid point distributions
$\gp{l,i}^\ast$,
as long as two requirements are met:
First, their number should grow exponentially
(i.e., there are $2^l + 1$ points $\gp{l,i}^\ast$ in each level $l \in \natz$),
and second, they should be nested
(i.e., \cref{eq:rewriteGridPoint} holds).
Appropriate non-uniform B-spline bases can be defined analogously
to Clenshaw--Curtis B-splines.


\subsection{Noncompactness measures}\label{subsec:noncompactness_measures}

\begin{definition}\label{def:noncompactness_measures}\mcite[def. 7.1]{Deimling1985}
  Let \( (X, \rho) \) be a \hyperref[def:metric_space]{metric space} and let \( \mscrB \) be the family of \hyperref[def:metric_space/bounded_set]{bounded sets} in \( X \). We define the following functions
  \begin{thmenum}
    \thmitem{def:noncompactness_measures/sets} The \term{Kuratowski measure of noncompactness},
    \begin{balign*}
       & \alpha: \mscrB \to [0, \infty)                                                                                \\
       & \alpha(A) \coloneqq \inf \{d > 0 \colon \exists U_1, \ldots, U_n \subseteq X: \diam {U_k} < d \T{and} A \subseteq \bigcup_{k=1}^n U_k \}
    \end{balign*}

    \thmitem{def:noncompactness_measures/balls} The \term{ball measure of noncompactness},
    \begin{balign*}
       & \beta: \mscrB \to [0, \infty)                                                                                \\
       & \beta(A) \coloneqq \inf \{r > 0 \colon \exists x_1, \ldots, x_2 \in X: A \subseteq \cup_{k=1}^n B(x_k, r) \}
    \end{balign*}
  \end{thmenum}
\end{definition}

\begin{example}\label{ex:noncompactness_measures}\mcite[exer. 7.3]{Deimling1985}
  Consider the subsets \( B_2 \subseteq B_3 \subseteq B_1 \subseteq C([0, 1]) \), defined by
  \begin{balign*}
    B_1 & \coloneqq \left\{
    x \in C([0, 1]) \colon \begin{aligned}
      0 \leq t \leq 1 \implies 0 \leq x(t) \leq 1 \\
      x(0) = 0, x(1) = 1                          \\
    \end{aligned}
    \right\}
    \\
    B_2 & \coloneqq \left\{
    x \in B_1 \colon \begin{aligned}
      0 \leq t \leq \frac 1 2 \implies 0 \leq x(t) \leq \frac 1 2 \\
      \frac 1 2 \leq t \leq 1 \implies \frac 1 2 \leq x(t) \leq 1
    \end{aligned}
    \right\}
    \\
    B_3 & \coloneqq \left\{
    x \in B_1 \colon \begin{aligned}
      0 \leq t \leq \frac 1 2 \implies 0 \leq x(t) \leq \frac 2 3 \\
      \frac 1 2 \leq t \leq 1 \implies \frac 1 3 \leq x(t) \leq 1
    \end{aligned}
    \right\}
  \end{balign*}

  Then \( \alpha(B_1) = 1, \alpha(B_2) = \frac 1 2, \alpha(B_3) = \frac 1 3 \) and \( \beta(B_1) = \beta(B_2) = \beta(B_3) = \frac 1 2 \).
\end{example}
\begin{proof}
  Since the distance between any two functions from \( B_1 \) is at most 1, we have that \( \diam B_1 = 1 \) and \( \alpha(B_1) \leq 1 \).

  Fix \( \varepsilon > 0 \). For any function \( f \in B_1 \), continuity of \( f \) gives us a radius \( \delta_f > 0 \) such that
  \begin{equation*}
    x < 2 \delta_f \implies f(x) < \varepsilon.
  \end{equation*}

  \begin{figure}
    \centering
    \text{\todo{Add diagram}}\iffalse\begin{mplibcode}
      input metapost/plotting;
      u := 4cm;
      e := 0.16; % epsilon
      d := sqrt(e); % delta

      vardef f(expr x) =
      x ** 2
      enddef;

      vardef tf(expr x) =
      if x < d / 2:
      (2 / d) * x
      elseif x < d:
      f(d) + (1 - f(d)) * ((2 / d) * (d - x))
      else:
      f(x)
      fi
      enddef;

      beginfig(1)
      drawarrow (0, 0) -- (u,  0);
      drawarrow (0, 0) -- (0, u);

      drawoptions(dashed withdots scaled 0.3);
      draw (0, e) scaled u -- (1, e) scaled u;
      label.lft("$\varepsilon$", (0, e) scaled u);

      draw (d, 0) scaled u -- (d, e) scaled u;
      label.bot("$\delta$", (d, 0) scaled u);
      drawoptions();

      draw path_of_plot(f, 0, 1, 0.01, u);
      label.rt("$f$", (0.75, 0.5) scaled u);

      drawoptions(dashed evenly);
      draw path_of_plot(tf, 0, d, 0.01, u);
      drawoptions();
      label.rt("$T_\varepsilon(f)$", (0.3, 0.7) scaled u);
      endfig;
    \end{mplibcode}\fi
    \caption{The operator \( T_\varepsilon \) adds \enquote{spikes} to functions.}\label{ex:noncompactness_measures/spike_plot}
  \end{figure}

  Define
  \begin{balign*}
    T_\varepsilon(f)(x) \coloneqq \begin{cases}
      \frac x {\delta_f},                                       & 0 \leq x < \delta_f          \\
      f(\delta_f) + [1 - f(\delta_f)] (2 - \frac x {\delta_f}), & \delta_f \leq x < 2 \delta_f \\
      f(x),                                                     & x \geq 2 \delta_f,
    \end{cases}
  \end{balign*}
  so that
  \begin{balign*}
    \norm{T_\varepsilon(f) - f}
    \geq
    T_\varepsilon(f) (\delta_f) - f(\delta_f)
    =
    1 - f(\delta_f)
    >
    1 - \varepsilon.
  \end{balign*}

  Additionally, because \( \delta_{T_\varepsilon(f)} < \delta_f \), we have that \( f(\delta_{T_\varepsilon(f)}) < \varepsilon \) and
  \begin{balign*}
    \norm{T_\varepsilon(T_\varepsilon(f)) - f}
    \geq
    T_\varepsilon(T_\varepsilon(f)) (\delta_{T_\varepsilon(f)}) - f(\delta_{T_\varepsilon(f)})
    =
    1 - f(\delta_{T_\varepsilon(f)})
    >
    1 - \varepsilon.
  \end{balign*}

  Thus, proceeding by induction, we see that for any \( m = 1, 2, \ldots \)
  \begin{equation*}
    \norm{T_\varepsilon^m(f) - f} > 1 - \varepsilon,
  \end{equation*}
  where \( T_\varepsilon^m \) denotes repeated application of \( T_\varepsilon \).

  Consider the sequence
  \begin{equation*}
    \{ T_\varepsilon^k(f) \}_{k=0}^\infty = \{ f, T_\varepsilon(f), T_\varepsilon(T_\varepsilon(f)), \ldots \}.
  \end{equation*}

  We can easily see that the distance between any two elements of the sequence, say \( T_\varepsilon^k(f) \) and \( T_\varepsilon^{k+m}(f) \), is strictly greater that \( 1 - \varepsilon \), i.e.
  \begin{balign*}
    \norm{T_\varepsilon^k(f) - T_\varepsilon^{k+m}(f)}
    =
    \norm{T_\varepsilon^k(f) - T_\varepsilon^m(T_\varepsilon^k(f))}
    >
    1 - \varepsilon.
  \end{balign*}

  Hence, \( B_1 \) cannot be covered by a finite \( (1-\varepsilon) \)-net and \( \alpha(B_1) \geq 1 - \varepsilon \). Since \( \varepsilon > 0 \) can be made arbitrarily small, this implies that \( \alpha(B_1) \geq 1 \) and, because we already have the reverse inequality, \( \alpha(B_1) = 1 \).

  In the set \( B_2 \), the maximum distance between two functions is \( \frac 1 2 \), thus \( \diam(B_2) = \frac 1 2 \) and \( \alpha(B_2) \leq \frac 1 2 \). We can then define an operator similar to \( T_\varepsilon \) that creates \enquote{spikes} of height \( \frac 1 2 \) to prove the reverse inequality, obtaining
  \begin{equation*}
    \alpha(B_2) = \frac 1 2.
  \end{equation*}

  Finally, the set \( B_3 \) has diameter \( \frac 2 3 \) and hence \( \alpha(B_3) = \frac 2 3 \).

  The ball measure for \( B_1 \) satisfies the inequalities
  \begin{equation*}
    \frac 1 2 \leq \beta(B_1) \leq 1.
  \end{equation*}

  Additionally, \( B_1 \) is strictly contained in the ball centered in the constant function \( \frac 1 2 \) with radius \( \frac 1 2 \), which implies that \( \beta(B_1) \leq \frac 1 2 \), hence \( \beta(B_1) = \frac 1 2 \).

  For \( B_2 \) we have
  \begin{equation*}
    \frac 1 4 \leq \beta(B_2) \leq \frac 1 2.
  \end{equation*}

  Assume that for some \( \varepsilon > 0 \) the set \( B_2 \) can be covered by a finite set of balls with centers \( \{ f_1, \ldots, f_n \} \subsetneq C([0, 1]) \) and radius \( \frac 1 2 - \varepsilon \).

  Because of continuity, we can find a radius \( \delta > 0 \) such that for all \( f_k, k = 1, \ldots, n \) we have
  \begin{equation*}
    x \in \left[\tfrac {1 - \delta} 2, \tfrac {1 + \delta} 2 \right] \implies \abs{f_k(x) - f_k(\tfrac 1 2)} < \varepsilon.
  \end{equation*}

  Consider the function
  \begin{balign*}
    g(x) \coloneqq \begin{cases}
      0,                                & 0 \leq x < \frac {1 - \delta} 2,                       \\
      \frac{2x + \delta - 1} {2\delta}, & \frac {1 - \delta} 2 \leq x \leq \frac {1 + \delta} 2, \\
      1,                                & \frac {1 + \delta} 2 < x \leq 1.
    \end{cases}
  \end{balign*}

  \begin{figure}
    \centering
    \text{\todo{Add diagram}}\iffalse\begin{mplibcode}
      input metapost/plotting;

      u := 6cm;
      e := 0.1; % epsilon

      vardef f_k(expr x) =
      1 - 1 / (1 + exp(2 - 3 * x))
      enddef;

      vardef g(expr x) =
      if x < (1 - d) / 2:
      0
      elseif x < (1 + d) / 2:
      (2x + d - 1) / 2d
      else:
      1
      fi
      enddef;

      f_minus := f_k(0.5) - e;
      f_plus := f_k(0.5) + e;
      d := (ln(1 / (1 - f_plus) - 1) - ln(1 / (1 - f_minus) - 1)) / 3; % delta
      d_minus := (1 - d) / 2;
      d_plus := (1 + d) / 2;

      beginfig(1)
      drawarrow (-0.1, 0) -- (u,  0);
      drawarrow (0, 0) -- (0, u);

      drawoptions(dashed withdots scaled 0.3);
      draw (0, f_minus) scaled u -- (1, f_minus) scaled u;
      label.lft("$f_k(\frac 1 2) - \varepsilon$", (0, f_minus) scaled u);

      draw (0, f_plus) scaled u -- (1, f_plus) scaled u;
      label.lft("$f_k(\frac 1 2) + \varepsilon$", (0, f_plus) scaled u);

      draw (0.5, 0) scaled u -- (0.5, 1) scaled u;
      label.bot("$\frac 1 2$", (0.5, 0) scaled u);

      draw (d_minus, 0) scaled u -- (d_minus, 1) scaled u;
      label.bot("$\frac {1 - \delta} 2$", (d_minus, 0) scaled u);

      draw (d_plus, 0) scaled u -- (d_plus, 1) scaled u;
      label.bot("$\frac {1 + \delta} 2$", (d_plus, 0) scaled u);
      drawoptions();

      drawoptions(dashed evenly);
      draw path_of_plot(f_k, -0.1, 1, 0.01, u);
      label.rt("$f_k$", (1, f_k(1)) scaled u);
      drawoptions();

      draw path_of_plot(g, 0, 1, 0.005, u);
      label.lft("$g$", (d_minus, 0.1) scaled u);
      endfig;
    \end{mplibcode}\fi
    \caption{The function \( g \) always has points that are far enough from all \( f_k, k = 1, \ldots, n \).}\label{ex:noncompactness_measures/sigmoid_plot}
  \end{figure}

  If \( f_k(\tfrac 1 2) \geq \frac 1 2 \), then \( f_k(\tfrac {1 - \delta} 2) > \tfrac 1 2 - \varepsilon \) and
  \begin{equation*}
    \norm{f_k - g} \geq f_k(\tfrac {1 - \delta} 2) - g(\tfrac {1 - \delta} 2) = f_k(\tfrac {1 - \delta} 2) > \tfrac 1 2 - \varepsilon.
  \end{equation*}

  Analogously, if \( f_k(\tfrac 1 2) < \frac 1 2 \), then \( f_k(\tfrac {1 + \delta} 2) < \tfrac 1 2 + \varepsilon \) and
  \begin{equation*}
    \norm{g - f_k} \geq g(\tfrac {1 + \delta} 2) - f_k(\tfrac {1 + \delta} 2) = 1 - f_k(\tfrac {1 + \delta} 2) > \tfrac 1 2 - \varepsilon.
  \end{equation*}

  Thus, for every \( k = 1, \ldots, n \) we have
  \begin{equation*}
    \norm{g - f_k} > \frac 1 2 - \varepsilon,
  \end{equation*}
  i.e. \( g \) in not contained in a ball of radius \( \frac 1 2 - \varepsilon \) around any of the centers \( f_1, \ldots, f_n \).

  Hence, \( \beta(B_2) \geq \frac 1 2 \), which implies \( \beta(B_2) = \frac 1 2 \). Because of the inclusion \( B_2 \subsetneq B_3 \subsetneq B_1 \), we have
  \begin{equation*}
    \frac 1 2 = \beta(B_2) \leq \beta(B_3) \leq \beta(B_1) = \frac 1 2,
  \end{equation*}
  hence \( \beta(B_3) = \frac 1 2 \).
\end{proof}

\begin{theorem}[Kuratowski's noncompactness lemma]\label{thm:noncompact_kuratowskis_lemma}\mcite[exer. 7.4]{Deimling1985}
  Let \( X \) be a Banach space and \( \{ A_n \}_n \) be a decreasing sequence of nonempty closed subsets such that \( \alpha(A_n) \to 0 \). Then \( A \coloneqq \bigcap_n A_n \) is nonempty and compact.
\end{theorem}
\begin{proof}
  The set \( A \) is compact because it is closed as the intersection of closed sets and \( \alpha(A) \leq \alpha(A_n) \to 0 \), hence \( \alpha(A) = 0 \).

  It remains to show that \( A \) is nonempty.
  Choose any sequence \( \{ x_n \}_n \) where \( x_n \in A_n \). Since any finite set is compact, we have that for any \( k \geq 1 \)
  \begin{balign*}
    \alpha(\{ x_n \}_{n \geq 1})
    =
    \max\{ \alpha(\{ x_n \}_{n < k}), \alpha(\{ x_n \}_{n \geq k}) \}
    =
    \alpha(\{ x_n \}_{n \geq k})
    \leq
    \alpha(A_k) \to 0,
  \end{balign*}
  hence the set \( \{ x_n \colon n \geq 1 \} \) is compact and thus sequentially compact. We can choose a convergent subsequence \( \{ x_{n_k} \}_k \) of \( \{ x_n \}_n \) whose limit lies in every \( A_n \) (since they are closed) and, consequently, in their intersection \( A \). So \( A \) is nonempty.
\end{proof}


% As a sample LaTeX document, this is an actual assignment
% written in LaTeX with my template for MATH 417,
% Honors Real Variables (Measure Theory) at University of Alberta.
% This source has been released with permission with the instructor,
% Professor John C. Bowman as the solutions are available at
% https://www.math.ualberta.ca/~bowman/m417/assign3.pdf

% ------- END DISCLAIMER -------------

% https://www.math.ualberta.ca/~bowman/m417/assign3.pdf
\documentclass{article}
\usepackage{import}

% Default settings, which adds XY matrix, 1.5 inch margins,
% slightly more space between paragraphs and the \SCSHAPE current template.
\newcommand{\DEFAULTSETTINGS}{}

% Change this directory if you put your template files in a different directory.
\newcommand{\basedir}{../../}

% If you are using Crowdmark or require new page for each problem, uncomment this.
% \newcommand{\CROWDMARK}{}

\import{\basedir/base/}{mathPreamble}

\newcommand{\assignmentnum}{3}
\newcommand{\coursename}{MATH 417}

% Your name, email, Student ID.
% Please don't submit assignments under the name
% "Jamie van Lindenberg" :(
\newcommand{\studentname}{Jamie van Lindenberg}
\newcommand{\studentid}{1234567}
\newcommand{\studentemail}{rassamee@ualberta.ca}
\newcommand{\coursesec}{Q1}

\begin{document}

\integratedtitle

\problemsec

What is the wrong in the following argument that \textit{The set $(0,1)$ is countable}?
Express the open set $(0,1)$ as a union of open intervals $I_x$ centered on each real $x\in (0,1)$:
\[
    \bigcup_{x\in (0,1)} I_x=(0,1).
\]
Each interval $I_x$ contains a rational which we can use as a label for that interval.
Since there are only countably many rational numbers, there are countably many labels.
Hence the above union actually contains only a countable number of intervals, each centered at a real number.
There are therefore only a countable number of real numbers.

\begin{solution}
    Solution removed.
\end{solution}

\problemsec

Let $S\subset \real^d$ such that
\[
    m^*(E\cap S) + m^*(E\cap S^c) = m(E).
\]
for all elementary set $E$.
Show that $S$ is Lebesgue measurable.

\begin{solution}
    Solution removed.
\end{solution}

\problemsec

Suppose $S_n\subset \real^d$ for $n\in \naturals$ are Lebesgue measurable sets that converge pointwise to a set $S$.

\subsection{Part (a)}

Show that $S$ is Lebesgue measurable.

\begin{solution}
    Solution removed.
\end{solution}

\subsection{Part (b)}
\textit{Dominated Convergence}.
Suppose that $S_n$ are all contained inside another Lebesgue measurable set $F$ of finite measure.
Show that $m(S_n)\to m(S)$ as $n\to \infty$.

\begin{solution}
    Solution removed.
\end{solution}

\subsection{Part (c)}

Give a counterexample to show that the dominated convergence theorem fails if the $S_n$ are not contained in a set of finite measure, even if we assume that $m(S_n)$ are all uniformly bounded.

\begin{solution}
    Solution removed.
\end{solution}

\problemsec

Show that an unsigned function $f:\real^d\to[0,\infty]$ is a simple function if and only if it is measurable and takes on finitely many values.

\begin{solution}
    Solution removed.
\end{solution}

\problemsec

Let $d,d'\in \naturals$.

\subsection{Part (a)}

If $A\subset \real^d$ and $B\subset \real^d$, show that
\[
    m^{d+d'*}(A\times B)\le m^{d*}(A)m^{d'*}(B).
\]
where $m^{d*}$ denotes the $d$-dimensional Outer Lebesgue Measure.

\begin{solution}
    Solution removed.
\end{solution}

\subsection{Part (b)}
Let $A\subset \real^d$ and $B\subset \real^{d'}$ Lebesgue measurable sets (not necessarily of finite measure).
Show that $A\times B$ is Lebesgue Measurable with
\[
    m(A\times B)=m(A)m(B).
\]

\begin{solution}
    Solution removed.
\end{solution}

\problemsec

If $f:\real^d\to [0,\infty]$ is Lebesgue measurable, show that the Lebesgue measure of
\[
    A_f \defeq \set{(x,y)\in \real^{d+1} : 0\le y \le f(x)}
\]
exists and equals $\int_{\real^d} f$.

\begin{solution}
    Solution removed.
\end{solution}
\end{document}


\chapter{Motivation for Numerical Methods}
Many partial differential equations do not have exact closed-form solutions for all choices of initial conditions\footnote{An example is the Navier-Stokes equation which is thought to describe the motion of an incompressible viscous fluid.}. Irregular boundary conditions can also make finding an analytic solution difficult for many partial differentail equation. In these cases, finding an approximate solution with a numerical method can be helpful either for physical purposes, engineering purposes or for mathematical investigations of the behavior of solutions to these partial differential equations.
There are also cases where the partial differential equations have explicitly known exact solutions, but the formulae used to express the exact solutions require a large number of computations to evaluate them\footnote{An example is the sine-Gordon equation.}. In this case we are interested in making numerical approximations that result in accurate and cost-efficient solutions.

Numerical methods allows us to use a computer to calculate approximate solutions to partial differential equations. The accuracy of the solution will depend on which numerical method is used and usually more accurate numerical methods tend to be more complicated than less accurate methods. We will therefore start with some simple numerical methods to familiarize ourselves with how numerical methods work.
We encourage the reader to take a full course on the numerical solution of partial differential equations as well as reading the references to find out about numerical techniques not discussed here.


\section{Introduction}
\label{sec:intro}

We consider a dataset that reports the new daily COVID-19 cases for each country, worldwide.
Our goal is to compute:
\begin{enumerate}
    \item The seven-days moving average of new cases, for each day and for each country
    \item The percentage increase of the seven-days average w.r.t.\ the day before, for each day and for each country
    \item The top-10 countries with the highest percentage increase, for each day
\end{enumerate}

\noindent
If data is provided with a coarser grain (e.g.\ weekly), the increment is spread evenly between the week when computing the average.
We implemented the solution with the \emph{Apache Spark SQL API}, since it provides:
\begin{itemize}
    \item an easy way to load CSV datasets as tables
    \item built-in windowing functions to aggregate data based on time
    \item efficient and transparent partitioning of the dataset between the worker nodes
    \item a concise and easy-to-read API, that reduces the amount of code to write and makes it way clearer that what we would have come up with using MPI
\end{itemize}

\noindent
We also considered using the Structured Streaming API, but ultimately we discarded it. This is our rationale:
\begin{itemize}
    \item The \emph{Structured Streaming API} is suitable for scenarios where the data has to be analyzed online, for example if we were to produce our reports during the pandemic with data coming in each day. In this case we could also use watermarking to wait for late data. However, this approach makes sense only if the data is actually generated periodically, otherwise the effort spent to simulate a periodic data source from file is totally unjustified.
    \item The \emph{SQL API} is more indicated for offline analysis, where the whole dataset is already available before the computation. This is exactly our case, so we chose this approach.
\end{itemize}


\chapter{Quantum Computing and Type Theory }
Programming languages are often recognized as Procedural and Descriptive corresponding to performing a given n-step logic in a program. Quantum Programs are verified throughly using Formal methods. A qubits working semantics are completely writtten as mathematical abstractions and then verified with some given initial conditions. Doing QC\footnote{Quantum computing} things this way is helpful as we are still not capable enough to have a practical quantum computer to try out things on. As in case of \textbf{Stack Verification} of classical computing systems, complete verification for quantum computers is also done. Some of the most recent works include \textit{Steve Zdancewic's} paper\cite{qwire} that discusses the \textbf{QWire} as a core language for Quantum circuits, an analogue to VHDL and HDL in classical computing. He has verified its correctness too in CoQ\cite{COQ}. He describes Phantom Types as the type system he used for Qwire.

\textbf{EpiQC}\cite{epiqc} is a multi-institutional paradigm foucsed on reducing the current gap between existing theoretical algorithms and practical quantum computing architectures. They have a very nice video series\cite{epiqc_video} that nicely explains the application of program verification in Quantum computing. They refer to Technology-Aware Programming Environment as an upgrade to traditional optimization and abstraction techniques to one that is more practical to use. Major focus is upon developing better tools for robustness and scalability.

\begin{itemize}
\item{
	\textbf{Procedural Implementations} include verifying step-by-step correctness of program semantics. \textbf{QCL} as described earlier is a programming language that has Procedural behaviour inherited from classical languages. It is now being verifed .
}
\item{
	\textbf{Descriptive Implementations} involves verifying the program logic only. For example, a sum function should return summation of input parameters correctly. The procedural verification aspect of same would mean to check the register allocation and machine cycle logic verification.
}
\end{itemize}

\section{Phantom Types}
Phantom Types \cite{phantom_types} is the  lightweight dependent type system used by Qwire. Most of the mathematical work involves matrices and using phantom types have increased the functional expressivity of the same. The complete implementation cna be found here


\chapter{Analytical Forces Induced by Dislocations on Linear Rectangular Surface Elements}
\label{c:lin_rect}
%
\section{Forces Exerted by a Dislocation Line Segment on Linear Rectangular Surface Elements}
\label{s:f_lin_rect}
%
Coupling discrete dislocation dynamics to finite element method requires the traction field $ \tns{\sigma^{\infty}}(\vec{x}) \cdot \vec{n}(\vec{x}) $ to be distributed among the set of relevant discrete nodes of a finite element or boundary element model. This is usually achieved as follows,
\begin{align}\label{eq:ddd_fem_force}
  \vec{F}^{(m)} = \int_{S^{e}} \left[\tns{\sigma^{\infty}}(\vec{x}) \cdot \vec{n}(\vec{x})\right] N_{m}(\vec{x})\; \rvar{d}S_{e}\,,
\end{align}
where $ \rvar{d}S_{e} $ is the infinitesimal surface element with surface area $ S_{e} $. $ N_{m}(\vec{x}) $ are so-called shape functions (interpolation functions) that distribute  the traction field among the surface element's nodes.

The problematic singularity associated with the classical Volterra dislocation is avoided by using the non-singular formulation of \citet{a_non-singular_continuum_theory_of_dislocations}. The stress field of a dislocation in a homogenous infinite linear elastic domain can be calculated as a contour integral along the loop \cite{eigenstrain},
\begin{align}\label{eq:field_stress}
  \sigma_{ij}^{\infty}(\vec{x}) = C_{ijkl} \oint \epsilon_{lnh} C_{pqmn} \dfrac{\partial G_{kp}(\vec{x} - \vec{x'})}{\partial x_{q}} b_{m} \rvar{d}x'_{h}\,,
\end{align}
where $ C_{ijkl} $ is the elastic stiffness matrix, $ \epsilon_{lnh} $ the permutation operator, $\vec{b}$ the Burgers vector, $ \vec{x'} $ the coordinate that spans the dislocation, and $ G_{kp}(\vec{x} - \vec{x'}) $ is Green's function of elasticity \cite{eigenstrain}. $ G_{kp}(\vec{x} - \vec{x'}) $ is defined as the displacement component in the $ x_{k} $ direction at point $ \vec{x} $ due to a force applied in the $ x_{p} $ direction at point $ \vec{x'} $. The traditional singularity comes from taking the Burgers vector distribution as a delta function. \citet{a_non-singular_continuum_theory_of_dislocations} proposed an alternative definition of $ G_{kp}(\vec{x} - \vec{x'}) $ which has a wider isotropic spread mainly localised in a radius $ a $ around the dislocation core,
\begin{align}\label{eq:elastic_green_func}
  G_{ij}(\vec{x} - \vec{x'}) = \dfrac{1}{8\pi \mu}\left[ \delta_{ij} \partial_{pp} - \dfrac{1}{2(1-\nu)} \partial_{ij} \right] R_{a}\,,
\end{align}
where $ \mu $, $ \nu $ are the isotropic shear modulus and Poisson's ratio respectively, $ \delta_{ij} $ is the Kronecker Delta, $ \partial_{x_{1} \ldots\, x_{n}} \equiv \dfrac{\partial^{n}}{\partial x_{1} \ldots\, \partial x_{n}}$. $ R_{a} $ is the defined as,
\begin{subequations}
  \begin{align}\label{eq:ra}
    R_{a}(\vec{x}) & = R(\vec{x}) * w(\vec{x}) = \int R(\vec{x} - \vec{x'}) w(\vec{x'}) \rvar{d}^{3}\vec{x'}\nn
                   & = \sqrt{R^{2} + a^{2}}                                                                     \\
    w(\vec{x})     & = \dfrac{15 a^{4}}{8\pi (R^{2} + a^{2})^{7/2}}\,,
  \end{align}
\end{subequations}
where $ w(\vec{x}) $ is the isotropic Burgers vector distribution derived in the appendix of \cite{a_non-singular_continuum_theory_of_dislocations}, $ \vec{x} = (x, y, z) $ and $ R(\vec{x}) = \sqrt{x^{2} + y^{2} + z^{2}} $.

For two dislocation nodes (1, 2) connected by straight line segments \cref{eq:field_stress} becomes,
\begin{align}\label{eq:two_node_stress_field}
  \tns{\sigma}^{(12)}(\vec{x}) =
  \begin{split}
    &-\dfrac{\mu}{8\pi} \int\limits_{\vec{x_{1}}}^{\vec{x_{2}}} \left( \dfrac{2}{R_{a}^{3}} + \dfrac{3 a^{2}}{R_{a}^{5}} \right) \left[ \left( \vec{R} \times \vec{b} \right) \otimes \rvar{d}\vec{x'} + \rvar{d}\vec{x'} \otimes \left( \vec{R} \times \vec{b} \right) \right]\\
    %
    &+ \dfrac{\mu}{4\pi (1 - \nu)} \int\limits_{\vec{x_{1}}}^{\vec{x_{2}}} \left( \dfrac{1}{R_{a}^{3}} + \dfrac{3 a^{2}}{R_{a}^{5}} \right) \left[ \left(\vec{R} \times \vec{b}\right)\cdot \rvar{d}\vec{x'} \right] \mtx{I}_{2}\\
    %
    &- \dfrac{\mu}{4\pi (1 - \nu)} \int\limits_{\vec{x_{1}}}^{\vec{x_{2}}} \dfrac{1}{R_{a}^{3}} \left[ \left(\vec{b} \times \rvar{d}\vec{x'}\right) \otimes \vec{R} + \vec{R} \otimes \left(\vec{b} \times \rvar{d}\vec{x'}\right) \right]\\
    &+ \dfrac{\mu}{4\pi (1 - \nu)} \int\limits_{\vec{x_{1}}}^{\vec{x_{2}}} \dfrac{3}{R_{a}^{5}} \left[ \left( \vec{R} \times \vec{b} \right) \cdot \rvar{d}\vec{x'} \right]\vec{R} \otimes \vec{R}
  \end{split}\,.
\end{align}
\begin{figure}
  \centering
  \includegraphics[width=\linewidth]{force_calc_linear_rectangle.pdf}
  \caption[Diagram of the analytical force calculation on linear rectangular surface elements.]{Diagram of the line integral method used to find analytical expressions for the forces exerted by dislocation lines on linear rectangular surface elements \cite{analytic_tractions}.
    \textit{1}) For any given point $ \vec{x} $ on the surface element and any given point $ \vec{x'}$ on the dislocation line segment, define distance $ R_{a} $.
    \textit{2}) Integrate from $ x_{1} \to x_{2} $ along line direction $ \vec{t} $.
    \textit{3}) Integrate from $ r_{1} \to r_{2} $ along vector $ \vec{p} $.
    \textit{4}) Integrate from $ s_{1} \to s_{2} $ along vector $ \vec{q} $.}
  \label{f:flrs}
\end{figure}
\subsection{Resolving Singularities when Dislocation Line Segments are Parallel to Surface Elements}
\label{ss:par_dln_se}
%
\begin{figure}
  \centering
  \includegraphics[width=\linewidth]{ftot_rotation_lin_rect.pdf}
  \caption[Avoiding singularities by rotating dislocation line segments.]{Effects of rotating a single dislocation line segment on the forces exerted by it on a linear rectangular surface element. The specific values of this function are not known \emph{a priori}, all that is known is that it must be periodic ($ T = 2\pi$) and have finite maximum and minimum values. The singularity is avoided by perturbing the angle $ \theta = 0 \to \theta = \pm \epsilon\,, \epsilon \gtrsim 0 $.}
  \label{f:rflrs}
\end{figure}
%
\savearabiccounter


\documentclass[a4paper]{article}

\usepackage[utf8]{inputenc}
\usepackage{amsmath}

\title{A dynamical Reed-Frost model for Covid-19 household contamination}

\begin{document}

\maketitle

\section*{Model description}

We consider a population divided into \emph{households}, with individuals
interacting both within and outside of their respective households. We will
focus on modeling the spread within the household, while taking the possibility
of community-acquired infection into account.

For a given household with \(m\) individuals, consider at each time \(t\) the
set \(S_t\) of susceptible individuals and the set \(I_t\) of infectious
individuals. The number of such individuals will be noted \(s_t\) and \(i_t\)
respectively. Given the long incubation time of Covid-19 and the possibility of
presymptomatic transmission, we will also consider the set \(E_t\) of
\emph{exposed} individuals, who have yet to develop symptoms. They are not
infectious in their current state (pre-infectious incubation). In accordance
with recent evidence about viral shedding in Covid-19, we will divide the set of
infectious individuals into pre-symptomatic and post-symptomatic individuals:
\[
	I_t=I_t^p\cup I_t^s.
\]

The level of infectiousness of an infectious individual \(j\) will be noted
\(h_j(t)\). We will assume that infectiousness is constant during the
presymptomatic phase \([t_p,t_s)\), then decreases with time in a geometric
fashion during the symptomatic phase \([t_s,\infty)\), such
that, if \(j\) is infectious, \begin{equation}
	h_j(t)  =
		\begin{cases}
		h_0 & \text{if } t_p \le t < t_s,\\
		h_0\gamma^{t-t_s}& \text{if } t\ge t_s.
		\end{cases}
\end{equation}
Within the household, individuals are mixing completely. Hence, at time \(t\), a
susceptible individual gets exposed to all infectious individuals. The
collective force of infection is then the sum of the contributions of both
presymptomatic and post-symptomatic infectious individuals:
\begin{equation}
	H_t = H_t^p + H_t^s = h_0 |I^p_t| + \sum_{j\in I_t^s} h_j(t),
\end{equation}
and a susceptible individual becomes exposed (is infected with the virus) with
probability
\begin{equation}
	1-Q_h(t) = 1- \exp(-\beta H_t).
\end{equation}
The term \(Q_h(t)\) is sometimes dubbed \emph{household escape probability}.

In addition to infections within the household, each susceptible individual can
also be infected in the community. If \(1-A\) is the probability of that event,
which is assumed to be independent from within-household infection, then total
infection probability at time \(t\) is

\[
	P(t)=1-Q(t) = 1-A\exp(-\beta H_t).
\]

This process of infections happens at each timestep, independently for each
susceptible individual, so that at time \(t+1\), the number of new infections is
binomially distributed with parameters \(s_t\) and \(P(t)\). We will study the
Markovian process
\begin{equation}
	X_t=(s_t,e_t,i^p_t,i_t^s,H_t^s),
\end{equation}
whose transitions can be summed up as follow:
\begin{align*}
	e_{t+1} & = e_t + \Delta E_t - \Delta I^p_t, \\
	i_{t+1}^p & = i_{t}^p +\Delta I^p_t - \Delta I_t^s, \\
	i_{t+1}^s & = i_t^s +\Delta I_t^s, \\
	H_{t+1}^s & = \gamma H_t^s + h_0 \Delta I_t^s, \\
	s_{t+1} & = s_t - \Delta E_t,
\end{align*}
where
\begin{align*}
	\Delta E_t & \sim \mathsf{Bin}(s_t,P(t)),\\
	\Delta I^p_t & \sim \mathsf{Bin}(e_t,p_I),\\
	\Delta I^s_t & \sim \mathsf{Bin}(i^p_t,p_S)
\end{align*}
are the number of newly exposed, newly presymptomatic and symptomatic
individuals, respectively.

\section*{Observation model}

We assume that the data contain the number \(s_0\) of individuals in each
household, along with symptom onset times of symptomatic individuals. In our
context, we will assume that symptomatic infectious individuals are detected
with probability \(F\) (thus aggregating subclinical or entirely asymptomatic
disease and biases due to self-reporting) and we will write
\[
	Y_t \sim \mathsf{Bin}(\Delta I_t^s,F)
\]
for the observed number of newly-infected, symptomatic individuals.

This, and the Markov property of the hidden process \(X\) makes inference
through classical HMM algorithms theoretically possible, such as particle MCMC
\cite{Endo2019}, or iterated filtering \cite{Ionides2015}. As a sanity check,
it should also be possible to fit the classical Reed-Frost model to the final
size data \((s_0,i_\infty)\) as in \cite{Cauchemez2014} to estimate some of the
same parameters as in this model.

\section*{Parameters}

\begin{itemize}
\item Rate of decrease of infectiousness \(\gamma >0\).
\item Initial infectiousness \(h_0 >0\).
\item Transmission parameter \(\beta >0\).
\item Community escape probability \(A\).
\item Rate of asymptomatic infection \(F\).
\item Probability of an exposed individual becoming infectious per time unit
\(p_I\).

\end{itemize}

\section*{Possible extensions}

There are a number of important aspects of Covid-19 infection that we would like
to address with these data. This requires refining the model to take individual
covariates into account, such as age (which should be modelled as a categorical
variable), severity of disease or the frequency of outside contacts.

For now, we assume that infectious individuals have the same transmission
characteristics, but this could very well not be the case. The initial
infectiousness \(h_0\) could be drawn from a common distribution,
such as a Gamma distribution, which allows for varying levels of heterogeneity
(see \cite{Fraser2011}). Also, we did not include the age structure of
households, which will have to be done at some point since transmissibility
between age groups is a crucial knowledge gap at this time.

The impact of asymptomatic infections should also be modelled carefully, since
it is not known at this time whether they can be infectious or not. Following
\cite{Fraser2011}, we could distinguish between uninfectious and infectious
asymptomatic disease.

A very important factor is the presence of non-Covid-19 infections in the
community, which can lead to symptoms and outbreaks similar to Covid-19. This
represents a major bias in the data. This can be resolved in several ways: the
simplest one is to independently estimate the proportion of influenza-like
illnesses who are in fact due to Covid-19 disease (for instance, by using the
rate of positive PCR tests as a proxy), and adjust the parameters accordingly. A
more sophisticated approach would be to finely distinguish between symptoms,
according to their predictive value for Covid-19.

\bibliographystyle{plain}
\bibliography{library.bib}

\end{document}


\chapter{Wave and optics}

Differential equation? Diffusion? Oscillation? Wave behavior?

This chapter has two purposes:
introduce differential equations to the reader,
and prepare the reader for quantum mechanics.

\section{Wave}

% https://en.wikipedia.org/wiki/Wave
("Wave", Wikipedia):
A \emph{wave} is an oscillation accompanied by a transfer of energy?
A wave is a disturbance that transfers energy through matter or space?

\section{Detecting waves}

We can detect a wave by a diffraction slit.
If it's a wave, it diffracts.

We assume the converse: if it diffracts, it's very likely a wave.

\emph{Undulation} is an old term for \emph{wave}.

\section{Oscillation of a loaded spring}

If a spring is loaded, pulled, and released, then it will oscillate.

The equation of motion can be derived from Hooke's law of spring restoring force (\S\ref{sec:hooke-s-law}).

Wave equation

Second-order differential equation

Water wave

D'Alembert's waves?

How do we describe oscillation?
Periodic motion?
Harmonic motion?

How do we describe a diffusion?
How do we derive the wave equation from the diffusion equation?

% https://en.wikipedia.org/wiki/Fick%27s_laws_of_diffusion
% https://en.wikipedia.org/wiki/Continuity_equation
% https://en.wikipedia.org/wiki/Diffusion_equation#Derivation
Diffusion equation is derived from continuity equation and Fick's laws of diffusion.
Assume homogeneous (made of the same thing everywhere)
isotropic (behaving the same everywhere) medium?
Diffusion:
The rate of diffusion is proportional to the gradient?
\[
    f(x,t+h) - f(x,t) = c \cdot \frac{[f(x - h, t) - f(x,t)] + [f(x + h, t) - f(x,t)]}{2}
\]
Divide both sides by \(h\)
\[
    D_t f(x,t) = c \cdot D_x f(x,t)
\]
\[
    \frac{\partial f}{\partial t} = c \cdot \frac{\partial f}{\partial x}
\]
\[
    \frac{\partial f}{\partial t} = \vec{c} \cdot \nabla f
\]
???

How do we describe waves?

How do we describe waves on a string?
Pulse on a string?
Pulse on a chain of springs?
Replace the springs with more smaller springs?

% https://en.wikipedia.org/wiki/D%27Alembert%27s_formula

A function \(f\) has \emph{period} \(p\) iff \(f(x+p) = f(x)\) for all \(x\).

Let \(f(x,t)\) be the \emph{amplitude} of the wave at position \(x\) and time \(t\).

Let the oscillator be at position \(0\).

Let \(g\) be an unknown function.

Flow:
\(f(x,t + dt) - f(x,t) = g(c,f(x,t),f(x-dx,t),f(x+dx,t))\)

\(f(x,t + dt) - f(x,t) = [f(x-dx,t)-f(x,t)] + [f(x+dx,t)-f(x,t)]\)

\section{Velocities}

Propagation velocity

Phase velocity

Group velocity

\section{Light wave}

\section{Fermat's principle of least time}

Light takes the path that takes the least time.

\section{Snell\textendash{}Descartes law of refraction}

% https://en.wikipedia.org/wiki/Snell%27s_law
% https://en.wikipedia.org/wiki/Snell%27s_law#History
\begin{equation}
    \frac{\sin \theta_1}{\sin \theta_2} = \frac{v_1}{v_2} = \frac{\lambda_1}{\lambda_2} = \frac{n_2}{n_1}
\end{equation}

Descartes 1637 \emph{Dioptrics},

Huygens 1678: Huygens\textendash{}Fresnel principle.

Snell's law can be derived from Fermat's principle?

Snell's law can be derived from Huygens\textendash{}Fresnel principle?

% https://en.wikipedia.org/wiki/Huygens%E2%80%93Fresnel_principle

\section{Optics}

\emph{Wavenumber} is?
\emph{Wavelength} is?
\emph{Frequency} is?
\emph{Phase speed} is?
\emph{Group velocity} is?

\emph{Dispersion relation} is?

\emph{Doppler effect} is

Expanding universe?

\section{Reflection}
\section{Diffraction}
\section{Diffusion}
\section{Dispersion}
\section{Interference}
\section{Superposition}
\section{Fresnel spot}
\section{Transversal wave}
\section{Longitudinal wave}
\section{Young's double-slit experiment}
\section{Isochronic oscillation of a pendulum}

\section{Camera obscura}

\section{Newton's 1672 prism splits white light into colors?}

\section{Young's 1803 double-slit experiment}


\documentclass[11pt,onecolumn]{article}
\usepackage{amsmath,amssymb,graphicx,epsfig,cite,algorithm,algorithmic,epstopdf}
\usepackage[utf8]{inputenc}
\usepackage{xfrac, nicefrac}
\usepackage{multirow}
\usepackage{appendix}

\begin{document}
\title{Monte Carlo Methods for Mortals}
\author{Santhosh Nadig}
\maketitle
\pagebreak
\section{Introduction}
Here's a scenario: assume that you are playing solitaire (the card game) and wondering what's the probability of a successful outcome? Well, this was a question that occurred to Stan Ulam (the Polish mathematician) who was recovering from illness and playing solitaire to while away his time. He tried to solve it using combinatorics and failed. Finally, he thought of simulating a very large number of games (with cards shuffled at random, of course) and just counting the number of successes. He, and his friend, Von Neumann, who knew a thing or two about computers and programming, set out do this calculation and eventually used this method for other scientific purposes.

This is the central idea of MC. Use randomness to "simulate" a large number of experiments. Infer / calculate a quantity of interest based on the outcome of the experiments. More the number of experiments, the better!

What are Monte Carlo (MC) methods? An (over)simplified view is that they are tools for:
\begin{enumerate}
 \item Estimating (multi-dimensional) integrals
 \item Estimating probabilities
 \item ... etc.
\end{enumerate}

First, a note about estimating integrals. The usual deterministic methods of numerical integration (such as trapizoidal method) suffer from what's known as the "curse of dimensionality". That is, for a fixed error the number of function evaluations (of the integrand) grows exponentially with the "dimension of integration". In general, calculations that grow exponentially are frowned upon by engineers and scientists. At the risk of giving away the story-line, it sufficies to say that MC methods do not suffer from the aforementioned "curse." I will (probably) give examples of this later on.

\section{Estimating Probabilities}
\subsection{Random Variables}
What is a random variable? As someone said, it is neither random nor variable. For example, let $X$ denote the resistance of a resistor (assume fixed temperature and other conditions, just for argument's sake). $X$ is an {\em unknown constant} (or, only known to the supreme fascist, SF). We do have, however, a series of measurements $x_1, x_2, \dots $ from an ohm-meter. These measurements are called realizations/observations/samples of the random variable. So, now we have a random variable $X$ and its realizations (or samples) $x_1, x_2, \dots $ and so on.

The expected value (mean, for ordinary mortals) of a random variable, $X$, is defined as
\[
E[X] = \int x~ p(x)~ dx,
\]
where $p(x)$ is the probability density function (pdf). Assume that $p(x)$ represents a Normal distribution (or Gaussian distribution) with mean $m$ and standard deviation $\sigma$. Thus, the probability density function $p(x)$ is given by
\[
 p(x) = \frac{1}{\sqrt{2\pi \sigma^2}} \exp\biggl({-(x-m)^2/(2\sigma^2)}\biggr).
\]
What it means is that if you plot a histogram of  $x_1, x_2, \dots $, it resembles the bell shape centered around  $m$ and its width $\propto \sigma$.

Assume that the unknown expected value $m$ is to be estimated given $N$ realizations $x_1,\dots,x_N$ of $X$. We could estimate $m$ (and hence, the integral) simply as a weighted sum of the realizations [it can proven that this is indeed the maximum likelihood estimate of $m$, but I will skip the derivation].
\[
E[X] = m =  \int x~ p(x)~ dx \approx \frac{1}{N} \sum_{i=1}^N x_i.
\]
The estimate gets better (closer to $m$) as the number of samples, N, increases.

The above idea might not be a revelation. But, let us assume that we have an integral of the kind:
\begin{align}
\int  g(x) \cdot f(x)~ dx ,
\label{eq:gxfx}
\end{align}

where $f(x)$ is a pdf (i.e., $f(x) \ge 0$ for all $x$ and $\int f(x)~dx = 1$.
For instance, if $g(x) = x$, we obtain the expected value; And, if $g(x) = x^2$, we obtain the
"variance", etc. These quantities are very useful - for example, the expected
 value represents the best estimate of the random variable X and the variance
 gives how confident we are of the estimate.

{\bf I1: MC method says:}
\[
\int g(x) \cdot f(x) ~ dx \approx \frac{1}{N} \sum_i g(x'_i)
\]
where $x'_1, x'_2,\dots$ are "sampled" from the pdf $f(x)$.

\subsection{Sampling}
What do we mean by ``sampling'' from a distribution? It means drawing realizations from a given pdf. Let's consider an example:
Suppose we have a "bent coin" which has a probability of coming up heads = 0.7 and coming up tails = 0.3. A sample is simply the result of a toss. Now, if we sampled long enough; that is, if we tossed the bent coin many times and observed the results (realizations) and plotted a histogram (normalized), then, we'd see that almost 70\% of the coin tosses came up heads and the remaining 30\% or so came up tails. A Python script to simulate such a coin is shown below:

\begin{verbatim}
"""
Simlate a biased coin
"""

import numpy as np

class biased_coin:
    """ biased_ is a class that implements a biased coin with
        probability of heads = p, probability of tails = 1-p """
    def __init__(self, p):
        self.p = p

    def toss(self):
        """ generate a single toss of the biased coin """
        if np.random.rand() > self.p:
            return 'T'
        else:
            return 'H'

    def get_sequence(self,num_tosses):

        seq = [self.toss() for i in range(num_tosses)]
        return seq

def main():
    p = 0.7 # probability of heads
    c = biased_coin(0.7);
    tosses = c.get_sequence(100);
    num_heads = tosses.count('H')
    num_tails = tosses.count('T')
    print ("simulated 100 tosses of a biased coin with p = " + str(p))
    print ("obtained " + str(num_heads) + " heads and " + str(num_tails) + " tails")

if __name__ == '__main__':
    main()

----- Sample Output -----
simulated 100 tosses of a biased coin with p = 0.7
obtained 71 heads and 29 tails

\end{verbatim}

A key observation in the above program is the following: we first sample from a uniform distribution (a random number that has the same probability of assuming any value between 0 and 1.0). This is provided by the function {\texttt{np.random.rand()}}. This random variable is then used to determine whether the appropriate result of the coin toss shall be heads or tails. If the random number is $> p (= 0.7)$, then the outcome is tails; otherwise heads. So, about 70\% of the coin tosses result in heads as expected.

{\bf I2: Random variables from a uniform distribution are ``translated'' to another desired distribution}.

This translation is a key feature of MC methods. There are a plethora of methods that achieve this (e.g., inverse transform sampling, rejection sampling, etc etc). But, the basic idea is that random variables from one distribution is used to generate random variables of another (desired) distribution.

The above example had a discrete pdf (a probabilty mass function (pmf)). If we assume a continuous pdf (such as a Normal distribution), then, the histogram of the samples would resemble a ``bell curve''.

\subsection{Recap}
Just a quick recap; the first idea ({\bf I1}) says that integrals of the kind (\ref{eq:gxfx}) can be approximated by averaging $g(x)$ s evaluated at $x$s drawn from the pdf $f(x)$. The second idea ({\bf I2}) says that samples from one distribution (such as uniform distribution, which is provided as a standard library in most OSes) can be used to generate samples from another distribution.

\subsection{Conditional Probabilty}
In this section, we explore how to generate samples from a pdf as a function of other pdfs. I will use the term probability density function (which, if you remember, is applicable to a continuous random variable) instead of probability mass function (of a discrete random variable) just for convinience \footnote{see appendix \ref{app:a}}. I know that this violates mathematical rigour, but it is worth the sacrifice (in my opinion).
\subsubsection{Task 1}
Assume that we are given samples from a pdf $p(x)$ and we know the conditional pdf $p(y|x)$. Our task is to generate samples from the pdf $p(y)$. So, say that we have samples $x_1, x_2, \dots, x_N$ from the pdf:
\begin{align}
 p(x) = \begin{cases}
         0.6, & x = 0. \\
         0.4, & x = 1.
        \end{cases}
\end{align}
We are also given the conditional probabilities:
\begin{align}
 p(y|x = 0) &= \begin{cases}
         0.1, & y = 0. \\
         0.9, & y = 1.
        \end{cases} \\
 p(y|x = 1) &= \begin{cases}
         0.7, & y = 0. \\
         0.3, & y = 1.
        \end{cases}
\end{align}
\footnote{Digression: Althought this is an artificial toy example, it bears resemblance to a binary assymetric channel where $y$ can be thought of as the received signal.}
Now, the first task is to draw samples from the distribution $p(y)$. One way to approach this task is to estimate $p(x)$ (call it $\hat p(x)$) using the samples $x_1,\dots,x_N$ and then apply a simple formula to calculate $p(y)$ as:
\begin{align}
 p(y = 0) &= p(y = 0|x = 0) \cdot \hat p(x = 0) + p(y = 0|x = 1)\cdot \hat p(x = 1) \\
 p(y = 1) &= p(y = 1|x = 0) \cdot \hat p(x = 0) + p(y = 1|x = 1)\cdot \hat p(x = 1)
\end{align}
and then, sample from $p(y)$ as before. Another approach is to do the following:
For each $x_i$, simulate $M$ samples of $y$s (i.e., $y_1, y_2, \dots, y_M$) using the conditional probability $p(y|x_i)$. Thus, we end up with $M\cdot N$ samples of $y$ which are essentially distributed as $p(y)$.
The following code sample illustrates the procedure (with $N = 10000$ and $M = 10$).
\begin{verbatim}
  x = sample(x_labels, p_x, 10000)
  y = np.array([])
  for i in range(len(x)):
      q = p_y_x[x[i],]
      y = np.append(y, sample(y_labels, q, 10))

  cnt_y = np.array([(y == 0).sum(), (y == 1).sum()])
  p_y = cnt_y/len(y)
  print('p(y) is: [%1.2f, %1.2f]'%(p_y[0], p_y[1]))

----- Sample Output -----
p(y) is: [0.34, 0.66]

\end{verbatim}

\subsubsection{Task 2}
For the next task, let us suppose the following:
\begin{enumerate}
  \item The pdf $p(x)$ (a.k.a prior) is known
 \item The conditional pdfs $p(y|x)$ (a.k.a likelihood) is known
 \item We have a $y_i$, which is either a 0 or 1 in our case (a.k.a measurement); and
 \item \bf Our job is to find the best estimate of $x$ that might have generated the given $y_i$. That is, we need to estimate the posterior probability $p(x|y)$.
\end{enumerate}

In theory, this can be achieved by applying Bayes' theorem
\begin{align}
 \textbf{posterior} &= \frac{\textbf{prior} \cdot \textbf{likelihood}}{\textbf{evidence}} \\
 p(x = x_j| y = y_i) &= \frac{p(y = y_i | x = x_j) p(x = x_j)}{\sum_{j} p(y = y_i | x = x_j) p(x = x_j) }
\end{align}
Say, $y_i = 0$ and we would like to determine $p(x|y=0)$. Using the above theorem, it turns out to be:
\begin{align}
 p(x = 0|y = 0) &= \frac{p(y = 0 | x = 0) p(x = 0)}{p(y = 0 | x = 0) p(x = 0) + p(y = 0 | x = 1) p(x = 1) } = 0.176  \nonumber \\
 p(x = 1|y = 0) &= (1 - 0.176) = 0.823.
\end{align}

One approach to solve this problem the ``Monte Carlo'' way is to do the following:
\begin{enumerate}
 \item Generate ``candidate'' samples of $x$ (say, $x_1,\dots,x_N$) from the known pdf $p(x)$.
 \item For each of the $x_i$s generated, generate a ``measurement'' (call it $\hat y$) according to the pdf (likelihood) $p(y|x)$.
 \item If $\hat y$ is the same as $y$, save that $x_i$, else discard the $x_i$.
\end{enumerate}
Once the above steps are complete, we would have a new (but reduced) set of $x_i$s (say, $x_1, \dots, x_M$) which would be distributed according to the pdf $p(x|y)$.
The following code sample illustrates the procedure.

\begin{verbatim}
 x = sample(x_labels, p_x, 10000)
    x_new = np.array([])
    for i in range(len(x)):
        q = p_y_x[x[i],]
        y = sample(y_labels, q, 1)
        if y == 0:
            x_new = np.append(x_new, x[i])

    #done! now, check the distribution
    cnt_x_new = np.array([(x_new == 0).sum(), (x_new == 1).sum()])
    p_x_y0 = cnt_x_new / len(x_new)
    print('p(x|y=0) is: [%1.2f, %1.2f]'%(p_x_y0[0], p_x_y0[1]))
    print('INFO: len(x) = %d, len(x_new) = %d'%(len(x), len(x_new)))

----- Sample Output -----
p(x|y=0) is: [0.18, 0.82]
INFO: len(x) = 10000, len(x_new) = 3375

\end{verbatim}

\subsection{Recap}
We have seen one way of generating samples from conditional probabilities (or sampling from the posterior probability). The method in the example we used discarded samples from a set of candidates (generated using the prior pdf) to generate another set of samples which were distributed according to the posterior pdf. If this method is applied recursively, one may end up with too few (or even no) samples. However, there are ways to overcome this problem.

\begin{appendices}
\section{Probability Density Function and Probability Mass Function}
\label{app:a}
It is easy to be misled\footnote{I, for one was...} to interpret the probability density function (pdf) of a continuous random variable as an analog of the probability mass function (pmf) of a discrete random variable. That is to say, that the ``y'' axis of the pdf represents probability. This is absolutely wrong! This confusion may arise, for example, when one compares the expression for the expected value of a discrete random variable $y$ and that of a continuous random variable $x$
\begin{align}
E[x] = \int x p(x) dx \\
E[y] = \sum_i y_i p(y_i)
\end{align}
In the case of a continuous random variable $p(y_i)$ represents an actual probability. That is, $p(y_i) = Pr(Y = y_i)$; whereas for a continuous random variable $p(x)$ is the probability density (and not an actual probability). That is:
\begin{align}
Pr(a \le X \le b) = \int_{a}^b p(x) dx.
\end{align}
From the above expression it also follows that the probability of a continous random variable taking on exactly one particular value is  0.
\end{appendices}
\end{document}


\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{hyperref}
\renewcommand{\vec}[1]{\boldsymbol{#1}}

\title{Variational optimization for the minimization of functions on the binary domain}
\date{\today}
\author{Antti Ajanki}

\begin{document}

\maketitle

We consider the task of minimizing a non-linear scalar function
$f(\vec{x})$, where the input is an $N$ dimensional binary vector,
$\vec{x} \in \{0, 1\}^N$.

\section{Variational optimization}

The minimum of a function is always less than or equal to its
expected value over any distribution $p(x | \theta)$:
\[
\min f(x) \leq E\left[f(x)\right]_{p(x | \theta)}
\]

The bound can be made tight if $p(x | \theta)$ is so flexible that all
the probability mass can be concentrated on the true minimum
$\text{argmin} f(x)$.

The idea of variational optimization~\cite{staines2012} is to consider
the upper bound $U(\theta) = E\left[f(x)\right]_{p(x | \theta)}$ as a
function of $\theta$ and minimize that. This converts the task of
minimizing a function $f(x)$ of binary variables to minimization of a
function $U(\theta)$ of a continuous variable. Any method for
continuous optimization can be applied for find a local minimum of
$U(\theta)$.

\section{Stochastic gradient descent}

Because $\vec{x}$ is a binary vector, it is natural to choose to take
the expectation over a separable Bernoulli distribution:
\[
p(\vec{x} | \vec{\theta}) = \prod_i p_i(x_i | \theta_i) = \prod_i
\theta_i^{x_i} (1 - \theta_i)^{1-x_i}
\]

Let's use the stochastic gradient descent to find the local minimum of
the upper bound $U(\vec{\theta})$. First we need the partial
derivates:
\begin{align*}
\frac{\partial U(\vec{\theta})}{\partial \theta_j}  &=
\frac{\partial}{\partial \theta_j} E
\left[f(\vec{x})\right]_{p(\vec{x} | \vec{\theta})}\\
&= \int f(\vec{x}) \frac{\partial}{\partial \theta_j} p(\vec{x} |
\vec{\theta})  d\vec{x}\\
&= \int f(\vec{x})
\frac{\partial}{\partial \theta_j} \theta_j^{x_j} (1-\theta_j)^{1 - x_j}
\prod_{i \neq j} p_i(x_i | \theta_i) d\vec{x}\\
&= \int f(\vec{x})
\left( \theta_j^{x_j} (x_j - 1) (1-\theta_j)^{1 - x_j - 1} + x_j
\theta_j^{x_j - 1} (1-\theta_j)^{1 - x_j} \right)
\prod_{i \neq j} p_i(x_i | \theta_i) d\vec{x}\\
&= \int f(\vec{x}) \theta_j^{x_j}
(1-\theta_j)^{1-x_j} \left( \frac{x_j - 1}{1 - \theta_j} +
\frac{x_j}{\theta_j} \right) \prod_{i \neq j} p_i(x_i | \theta_i) d\vec{x}\\
&= \int f(\vec{x}) \left( \frac{x_j - 1}{1 - \theta_j} +
\frac{x_j}{\theta_j} \right) \prod_i p(\vec{x} | \vec{\theta})
d\vec{x}\\
&= E\left[ f(\vec{x}) \left( \frac{x_j - 1}{1 - \theta_j} +
\frac{x_j}{\theta_j} \right) \right]_{p(\vec{x} | \vec{\theta})}
\end{align*}

David Barber proposed approximating the last expectation by sampling~\cite{barber2017}:
\[
\frac{\partial U(\vec{\theta})}{\partial \theta_j} \approx \frac{1}{K}
\sum_{k=1}^K f\left( \vec{x}^{(k)} \right) \left( \frac{x_j^{(k)} -
  1}{1 - \theta_j} + \frac{x_j^{(k)}}{\theta_j} \right),
\]
where $\vec{x}^{(1)}$ through $\vec{x}^{(K)}$ are samples from
$p(\vec{x}| \vec{\theta})$.

Now that we have a way to approximate the gradient $\nabla
U(\vec{\theta})$, we can apply the stochastic gradient descent to
iteratively search for the minimum. The $\vec{\theta}$ is updated by
taking small steps in the direction of the negative gradient:
\[
\vec{\theta}^{\text{new}} = \vec{\theta} - \frac{\eta}{K} \sum_{k=1}^K
f\left( \vec{x}^{(k)} \right) \left( \frac{x_j^{(k)} - 1}{1 -
  \theta_j} + \frac{x_j^{(k)}}{\theta_j} \right),
\]
where $\eta$ is the learning rate. Next, new $\vec{x}$ samples are drawn
from $p(\vec{x} | \vec{\theta}^{\text{new}})$ and the iteration is
repeated until convergence.

\begin{thebibliography}{9}
\bibitem{staines2012}
  Joe Staines, David Barber:
  \textit{Variational Optimization},
  \url{https://arxiv.org/abs/1212.4507v2}, 2012.
\bibitem{barber2017}
  David Barber:
  \textit{Evolutionary Optimization as a Variational Method},
  \url{https://davidbarber.github.io/blog/2017/04/03/variational-optimisation/},
  Apr 3, 2017.
\end{thebibliography}

\end{document}


\lesson{2}{Nov 15 2021 Mon (12:15:15)}{Polynomial Transformations}{Unit 3}

\begin{definition}[Theorems]
    A theorem is a proven statement based on experimentation. I will go over these three \bf{Theorems}:

    \begin{itemize}
        \item Fundamental Theorem of Algebra
        \item Factor Theorem
        \item Remainder Theorem
    \end{itemize}
\end{definition}

\begin{theorem}[Fundamental Theorem of Algebra]
    The Fundamental Theorem of Algebra generally states that the degree of a polynomial is equivalent to the number of zeros (both real and complex) of a function.

    By the \bf{Fundamental Theorem of Algebra}, the polynomial function $f(x) = x^2 - 3x - 28$ has two zeros since the degree of the function is two. To determine these zeros, replace the function notation of $f(x)$ with $0$ and solve by factoring.

    To factor, let's use the \bf{Quadratic Equation}:

    \begin{align}
        x    &= \frac{-b \pm \sqrt{b^2 - 4ac}}{2a} \\
        f(x) &= x^2 - 3x - 28 \\
        a    &= 1, b = -3, c = -28 \\
        x    &= \frac{-(-3) \pm \sqrt{(-3)^2 - 4 \times 1(-28)}}{2 \times 1} \\
        x    &= \frac{3 + 11}{2} \rm{ AND } x = \frac{3 - 11}{2} \\
        x    &= 7 \rm{ AND } -4 \rightarrow (x + 4)(x - 7)
    \end{align}

    The zero product property tells us that for these factors to result in a product of $0$, one or both of them must equal $0$.

    \begin{minipage}[c]{0.5\textwidth}
        \centering
        \begin{tabular}{cccccc}
            & 0 & = & x & + & 4 \\
            - & 4 & = &   & - & 4 \\
            \hline
            & x & = & & - & 4 \\
        \end{tabular}
    \end{minipage}

    \hfill
    \begin{minipage}[c]{0.5\textwidth}
        \centering
        \begin{tabular}{cccccc}
            & 0 & = & x & - & 7 \\
            + & 7 & = &   & + & 7 \\
            \hline
            & x & = & & + & 7 \\
        \end{tabular}
    \end{minipage}

    The \bf{Zeros} of $f(x) = x^2 - 3x - 28$ are: $x_1 = -4, x_2 = 7$.
\end{theorem}

\begin{marginfigure}
    \centering
    \incfig{fundamental-theorem-of-algebra}
    \sidecaption{$f(x) = x^2 - 3x - 28$ Graphed.}
    \label{fig:fundamental-theorem-of-algebra}
\end{marginfigure}

\begin{theorem}[Factor Theorem]
    The \bf{Factor Theorem} states that a first degree binomial is a factor of a polynomial function if the remainder, when the polynomial is divided by the binomial, is $0$.

    To determine whether $x - 5$ is a factor of the function $f(x) = -4x^3 + 21x^2 - 25$, set up a division problem whereby $-4x^3 + 21x^2 - 25$ is divided by $x - 5$.

    \[ \polylongdiv{-4x^3 + 21x^2 - 25}{x - 5} \]

    When the function $f(x) = -4x^3 + 21x^2 - 25$ is divided by the binomial $x - 5$, the remainder is $0$. So, $x - 5$ is a factor of the function $f(x) = -4x^3 + 21x^2 - 25$.
\end{theorem}

\begin{theorem}[Remainder Theorem]
    The \bf{Remainder Theorem} states that when the opposite of the constant from the binomial divisor is substituted into a function for $x$, the result is the remainder.

    When the polynomial function $f(x) = x^4 + 11x^3 + 26x^2 + 15x - 17$ is divided by $x + 8$ using division, the remainder is the last integer on the bottom row.

    \[ \polylongdiv{x^4 + 11x^3 + 26x^2 + 15x - 17}{x + 8} \]

    When the opposite constant in the divisor is substituted into the function, the result will be the same as the remainder in the division process.

    \begin{align}
        f(x)  &= x^4 + 11x^3 + 26x^2 + 15x - 17 \\
        f(-8) &= (-8)^4 + 11(-8)^3 + 26(-8)^2 + 15(-8) - 17 \\
              &= 4096 + 11(-512) + 26(64) + 15(-8) - 17 \\
              &= 4096 - 5632 + 1664 - 120 - 17 \\
              &= -9 \leftarrow \rm{Remainder}
    \end{align}
\end{theorem}

\subsubsection*{Using these Theorems}

Given a polynomial function $f(x)$ and a number $a$, if $(x - a)$ is a factor of $f(x)$, then $a$ is a $0$ of the polynomial.

The binomial $(x - a)$ can be proved as a factor of $f(x)$ by:

\begin{itemize}
    \item Using \bf{Long Division} with $(x - a)$ as the divisor.
    \item Using factoring methods when appropriate (grouping, completing the square, \ldots).
\end{itemize}

If a is a $0$ of the polynomial function $f(x)$, then:

\begin{itemize}
    \item The graph of $f(x)$ crosses the $x$ axis at $(a,0)$.
    \item Substituting $a$ into $(x - a)$ will equal $0$
    \item Substituting $a$ into $f(x)$ will result in $f(a) = 0$.
\end{itemize}

\newpage