\section{Some Properties of Probability Measure and More Counting}
\subsection{Elementary Results}
Let $(\Omega,\mathscr F,\mathbb P)$ be a probability space.
\begin{proposition}[Countable Subadditivity]
    Let $(A_n)_{n\in\mathbb N}\in\mathscr F$, then
    $$\mathbb P\left( \bigcup_{n\in\mathbb N}A_n \right)\le\sum_{n=1}^\infty \mathbb P(A_n)$$
\end{proposition}
\begin{proof}
    Consider $B_1=A_1$ and $B_n=A_n\setminus (A_1\cup A_2\cup\cdots\cup A_{n-1})$.
    The result follows.
\end{proof}
$\mathbb P$ is upward continuous.
\begin{proposition}
    Suppose $(A_n)_n\in\mathbb N\in\mathscr F$ is increasing, then
    $$\mathbb P(A_n)\to \mathbb P\left( \bigcup_{n\in\mathbb N} A_n\right)$$
    as $n\to\infty$.
\end{proposition}
\begin{proof}
    Consider $B_1=A_1$ and $B_n=A_n\setminus (A_1\cup A_2\cup\cdots\cup A_{n-1})$.
    Note that $\mathbb P(A_n)=\mathbb P(B_1)+\cdots+\mathbb P(B_n)$, so
    $$\mathbb P(A_n)=\sum_{k=1}^n\mathbb P(B_k)=\mathbb P\left(\bigcup_{k=1}^nB_n\right)=\mathbb P\left( \bigcup_{k=1}^nA_n \right)\to\mathbb P\left( \bigcup_{n\in\mathbb N} A_n\right)$$
    As desired.
\end{proof}
By taking complement, we know that $\mathbb P$ is also downward continuous.
\begin{proposition}[Inclusion-Exclusion Principle]
    Let $A_1,A_2,\ldots,A_n$ be events, then we have
    $$\mathbb P(A_1\cup\cdots\cup A_n)=\sum_{k=1}^n(-1)^{k+1}\sum_{1\le i_1<\cdots<i_k\le n}\mathbb P(A_{i_1}\cap\cdots\cap A_{i_k})$$
\end{proposition}
\begin{proof}
    Induction.
\end{proof}
For the probability on finite sample space with equally likely outcomes, the principle reduced to the known inclusion-exclusion principle on finite sets.
\begin{corollary}
    Let $A_1,A_2,\ldots,A_n$ be sets, then we have
    $$|A_1\cup\cdots\cup A_n|=\sum_{k=1}^n(-1)^{k+1}\sum_{1\le i_1<\cdots<i_k\le n}|A_{i_1}\cap\cdots\cap A_{i_k}|$$
\end{corollary}
\begin{proof}
    Follows directly.
\end{proof}
\begin{proposition}[Bonferroni Inequality]
    Truncating the inclusion-exclusion formula after the $n^{th}$ term gives an overestimate if $n$ is odd and underestimate if $n$ is even.
\end{proposition}
\begin{proof}
    Trivial.
\end{proof}
\subsection{Counting by Inclusion-Exclusion}
Inclusion-Exclusion allows us to do some more counting.\\
We want to count the number of surjections $\{1,2,\ldots,n\}\to\{1,2,\ldots,m\}$ for $m\le n$.
Take $\Omega=\{f:\{1,2,\ldots,n\}\to\{1,2,\ldots,m\}\}$ and $A$ be the set of surjections.
Let $A_i=\{f\in\Omega:i\notin \operatorname{Im}f\}$, so $A=A_1^c\cap\cdots\cap A_m^c=(A_1\cup\cdots\cup A_m)^c$.
Now since we have $|A_{i_1}\cap\cdots\cap A_{i_k}|=(m-k)^n$, hence by Inclusion-Exclusion, we have
$$|A|=\sum_{k=0}^m(-1)^k\binom{m}{k}(m-k)^n$$
(Note that a complement has been taken.)\\
We now want to count the number of derangements of $\{1,2,\ldots,n\}$.
Let $\Omega=S_n$ and $A$ be the set of derangements.
Let $A_i=\{\sigma\in\Omega:\sigma(i)=i\}$, so again $A=(A_1\cup\cdots\cup A_n)^c$.
But we have $|A_{i_1}\cap\cdots\cap A_{i_k}|=(n-k)!$, so assume an equally likely probability measure $\mathbb P$, we then have
$$\mathbb P(A^c)=\sum_{k=1}^n(-1)^{k+1}\frac{n}{k}\frac{(n-k)!}{n!}\implies \mathbb P(A)=\sum_{k=0}^n(-1)^k\frac{1}{k!}\to \frac{1}{e}$$
So asymptopically $|A|\sim n!/e$.
\subsection{Independence}
\begin{definition}
    $A,B\in\mathscr F$ are said to be independent if $\mathbb P(A)\mathbb P(B)=\mathbb (A\cap B)$.\\
    A countable sequence $(A_n)_{n\in\mathbb N}$ of events is independent if $\forall k\ge 2$ and for any set of indices $i_1,i_2,\ldots,i_k$ we have
    $$\mathbb P(A_{i_1}\cap\cdots\cap A_{i_k})=\mathbb P(A_{i_1})\cdots\mathbb P(A_{i_k})$$
\end{definition}
\begin{remark}
    1. Pairwise independence does not imply independence.
    Say we flip a fair coin twice, then $\Omega=\{(0,0),(0,1),(1,0),(1,1)\}$ and $A=\{(0,0),(0,1)\},B=\{(0,0),(1,0)\},C=\{(0,1),(1,0)\}$, then they are pairwisely independent but not independent.\\
    2. If $A$ is independent of $B$, then $A$ is also independence of $B^c$.
\end{remark}
\begin{definition}
    Let $A,B$ be events such that $\mathbb P(B)>0$.
    The conditional probability is defined as
    $$\mathbb P(A|B)=\frac{\mathbb P(A\cap B)}{\mathbb P(B)}$$
\end{definition}
In particular, $\mathbb P(A|B)=\mathbb P(A)$ iff $A,B$ are independent.
\begin{proposition}
    Let $(A_n)_{n\in\mathbb N}$ be a sequence of disjoint events, then
    $$\mathbb P\left( \bigcup_{n\in\mathbb N}A_n\middle|B \right)=\sum_{n\in\mathbb N}\mathbb P(A_n|B)$$
\end{proposition}
\begin{proof}
    Trivial.
\end{proof}
So conditional probability measure is also a probability measure.
\begin{theorem}[Law of Total Probability]
    Suppose $(B_n)_{n\in\mathbb N}$ is a disjoint sequence of events such that $\mathbb P(B_n)>0$ for all $n$ and $\bigcup B_n=\Omega$.
    Then
    $$\mathbb P(A)=\sum_{n\in\mathbb N}\mathbb P(A|B_n)\mathbb P(B_n)$$
\end{theorem}
\begin{proof}
    Obvious.
\end{proof}
\begin{theorem}[Bayes' Formula]
    Let $(B_n)_{n\in\mathbb N}$ be as above and $\mathbb P(A)>0$, then
    $$\mathbb P(B_n|A)=\frac{\mathbb P(A|B_n)\mathbb P(B_n)}{\sum_{k}\mathbb P(A|B_k)\mathbb P(B_k)}$$
\end{theorem}
\begin{example}[False Positives for a Rare Condition]
    Consider a rare medical condition $A$ which affects $0.1\%$ of the population and a medical test which is positive for $98\%$ of the people affected and $1\%$ of those unaffected by the condition.
    Take a random person, then we want to know the probablity that he has condition $A$ given that he was tested positive.
    Let $A$ be the event of individuals suffering from the condition and $P$ be the individuals being tested positive.
    Hence
    $$\mathbb P(A|P)=\frac{\mathbb P(P|A)\mathbb P(A)}{\mathbb P(P|A^c)\mathbb P(A^c)+\mathbb P(P|A)\mathbb P(A)}\approx 8.9\%$$
    This may seems counterintuitive since this probability seems low, but the thing is that $\mathbb P(P|A^c)>>\mathbb P(P|A)$ since the medical condition is so rare.
\end{example}
\begin{example}[Extra Knowledge Changes Probability]
    Consider the following statements:\\
    (a) I have $2$ childrens, the elder of whom is a boy.\\
    (b) I have $2$ childrens, one of whom is a boy.\\
    (c) I have $2$ childrens, one of whom is a boy who is born on a Tuesday.\\
    We assume equally likely distributions.
    Let $GB$ denotes that the younger is a girl and the elder is a boy.
    Similar for $BG,BB,GG$.
    $$\mathbb P(BB|a)=\mathbb P(BB|BG\cup BB)=\frac{1}{2}$$
    $$\mathbb P(BB|b)=\mathbb P(BB|BG\cup GB\cup BB)=\frac{1}{3}$$
    Write $TN$ be that there are two boy, the younger is born on Tuesday but the elder is not, similar for ($2$-)combinations of $T,N,G$.
    \begin{align*}
        \mathbb P(BB|c)&=\mathbb P(TT\cup TN\cup NT|TT\cup TN\cup NT\cup TG\cup GT)\\
        &=\frac{\mathbb P(TT\cup TN\cup NT)}{\mathbb P(TT\cup TN\cup NT\cup GT\cup TG)}\\
        &=\frac{13}{27}
    \end{align*}
\end{example}
\begin{example}[Simpson's Paradox]
    There are $50$ men and $50$ women applying to a college.
    \begin{center}
        \begin{tabular}{c|c|c|c}
            \hline
            All applicants&Admitted&Rejected&Success Rate\\
            \hline
            State&25&25&50\%\\
            Indep&28&22&56\%\\
            \hline
            Men Only&Admitted&Rejected&Success Rate\\
            \hline
            State&15&22&41\%\\
            Indep&5&8&38\%\\
            \hline
            Women Only&Admitted&Rejected&Success Rate\\
            \hline
            State&10&3&77\%\\
            Indep&23&14&68\%
        \end{tabular}
    \end{center}
    So both men and women in state schools have higher acceptance rate, but the overall acceptance rate of state schools is still lower than that in independent schools.
    This is because the overall acceptance rate for women in this set of data is larger and there are much more of them in independent schools than state schools.
    Basically it is because $A/B>a/b$ and $C/D>c/d$ does not imply $(A+C)/(B+D)>(a+c)/(b+d)$.
\end{example}


% declare document class and geometry
\documentclass[12pt]{article} % use larger type; default would be 10pt
\usepackage[margin=1in]{geometry} % handle page geometry

\input{../header2.tex}

\title{Phys 220A -- Classical Mechanics -- Lec09}
\author{UCLA, Fall 2014}
\date{\formatdate{30}{10}{2014}} % Activate to display a given date or no date (if empty),
         % otherwise the current date is printed

\begin{document}
\setlength{\unitlength}{1mm}
\maketitle

\section{Proof of Liouville's theorem}

If we imagine we have $N$ coordinates $x^i$ and we consider an infinitesimal volume element then we can consider a change in coordinates that if we define new coordinates as a function of the old coordinates we have
\begin{equation}
\widetilde{x}^i = f^i(x^j), \qquad \dif^n \widetilde{x} = J \, \dif^n x.
\end{equation}
where $J$ is the Jacobian,
\begin{equation}
J = \det \pd{\widetilde{x}^i}{{x^j}}.
\end{equation}
For an infinitesimal change, what does this mean? It's close to an identity so our new coordinates can be written
\begin{equation}
\widetilde{x}^i = x^i + \epsilon v^i (x^j)
\end{equation}
for some function $V^i(x^j)$. Then the Jacobian is just
\begin{equation}
J = \det(\delta^i_j + \epsilon \pd{v^i}{{x^j}}).
\end{equation}
One can show using linear algebra that when we expanding this the term of order $\epsilon$ is simply the trace. So up to first order
\begin{equation}
J = 1 + \epsilon \tr \left( \dpd{v^i}{{x^j}} \right)
	= 1 + \epsilon \sum_i \dpd{v^i}{{x^i}}.
\end{equation}
So now our volume element is
\begin{equation}
\dif^n \widetilde{x} = \dif^n x \, J = \dif^n x \left( 1 + \epsilon \sum_i \dpd{v^i}{{x^i}} \right).
\end{equation}
The volume element does not change if the vector field $v_i$ is divergence free, i.e.
\begin{equation}
\sum_i \pd{v^i}{{x^i}} = 0.
\end{equation}
Now we'll apply this to the Hamilton's equations. Now instead of $x^i$ we really have a set of coordinates $\set{q_i, p_i}$ where both position and momentum are coordinates. And then we have our Hamilton's equations. How do we now view those? The equations are time evolution equations but now we map it to an infinitesimal time, i.e.
\begin{equation}
q_i(t + \dif{t}) = \widetilde{q_i}, \qquad
p_i(t + \dif{t}) = \widetilde{p_i}.
\end{equation}
Now hamilton's equations become
\begin{equation}
\widetilde{q_i} = q_i + \pd{H}{p_i} \dif{t}, \qquad
\widetilde{p_i} = p_i - \pd{H}{q_i} \dif{t}.
\end{equation}
Our $\epsilon$ here is an infinitesimal time step $\dif{t}$, our $V$ becomes
\begin{equation}
V^k = \set{ \dpd{H}{p_i}, - \dpd{H}{q_i} },
\end{equation}
and our volume element is
\begin{equation}
\dif^nx \rightarrow \dif^N q \, \dif^N p
\end{equation}
Applying the general formula
\begin{equation}
\dif^N \widetilde{q} \, \dif^N \widetilde{p} =\dif^N q \, \dif^N p \left[ 1 + \dif{t} \sum_{i=1}^N \left( \dpd{}{q_i} \dpd{H}{p_i} - \dpd{}{p_i} \dpd{H}{q_i} \right) \right],
\end{equation}
the second term is zero,
\begin{equation}
\dif^N \widetilde{q} \, \dif^N \widetilde{p} =\dif^N q \, \dif^N p.
\end{equation}
Under Hamiltonian time evolution, the volume of phase space is invariant. The flow in phase space is incompressible. Note however that volume is not conserved in certain cases, for example when there is friction.

\section{Phase space density}

From here we can start to see a route to classical statistical mechanics. Suppose we have some phase space density
\begin{equation}
\rho(p_i, q_i, t)
\end{equation}
which depends on all $2N$ coordinates in phase space. We can normalize so that it is a probability distribution,
\begin{equation}
\int \dif^n p \, \dif^n q \, \rho(p_i q_i, t) = 1,
\end{equation}
but we won't go into that and instead save it for later. An example will be the classical Boltzmann distribution
\begin{equation}
\rho \propto \exp \left(-\frac{H(p, q, t)}{kt} \right).
\end{equation}
The volume element stays invariant so basically you then realize that we can say what the time evolution of this distribution. So we can take the time derivative and think about the time evolution from Liouville theorem and find that
\begin{equation}
\d{}{t} \rho(p_i, q_i, t) = 0.
\end{equation}
Working around with this a bit, we find
\begin{align}
0 &= \dpd{\rho}{t} + \dpd{\rho}{p_i} \dot{p}_i + \dpd{\rho}{q_i} \dot{q}_i \\
	&= \dpd{\rho}{t} + \sum_i \left( -\dpd{\rho}{p_i} \dpd{H}{q_i} + \dpd{\rho}{q_i} \dpd{H}{p_i} \right),
\end{align}
thus we have
\begin{equation}
\pd{\rho}{t} = \sum_i \left( \dpd{\rho}{q_i} \dpd{H}{p_i} - \dpd{\rho}{p_i} \dpd{H}{q_i} \right).
\end{equation}
So this gives us a continuity equation for $\rho$.

\section{Poincare Recurrence Theorem}

What can we say about staying at point in phase space and evolving in time. Basically we can pick a small neighborhood about the point and if you wait long enough, the point will come back to that neighborhood and come arbitrary close. So we'll have Hamiltonian evolution. Also we have to say that this is in a bounded phase space.

\begin{theorem}
Under Hamiltonian evolution in a system with bounded phases space $M$ (i.e. the allowed region in phase space has finite volume). Given an initial point $P = \set{p_i(t_0), q_i(t_0)}$ and a neighborhood $U$ of $P$, there is a point $P'$ in $U$ which returns to $U$ in finite time.
\end{theorem}

\begin{proof}
Consider a map that evolves in time points in phase space
\begin{eqn}
g_t : \set{p_i(0), q_i(0)} \mapsto \set{p_i(t), q_i(t)},
\end{eqn}
which maps phase space to phase space. Consider taking the map many times on itself, i.e.
\[
U \rightarrow g_t(U) \rightarrow g_t^2 (U) \rightarrow g_t^3 (U) \rightarrow \dots.
\]
Now, there must be some intersecting $g^k(U)$ and $g^\ell(U)$, since otherwise we would have
\begin{eqn}
\opname{Vol} M > \sum_{n=0}^\infty \opname{Vol} g^n (U) = \infty \times \opname{Vol} U.
\end{eqn}
Then taking the inverse map $g^{-\ell}$ of $g^k (U) \cap g^\ell (U) \neq \varnothing$, we have
\begin{eqn}
g^{k-\ell} (U) \cap U \neq \varnothing.
\end{eqn}
Thus, calling $n = k - \ell$, if
\begin{eqn}
y \in g^n(U) \cap U,
\end{eqn}
then
\begin{eqn}
y = g^n (x) \in U
\end{eqn}
for some $x \in U$.
\end{proof}

\section{Poisson Brackets}

\begin{itemize}
\item Geometrical structure on phase space (symmetric structure)
\item New higher level understanding of symmetry (Algebra of charges)
\item Makes classical mechanics almost look like QM
\end{itemize}
Recall that time dependence of a ``phase space observable'' $A(p, q, t)$ can be written
\begin{align}
\dod{A}{t} &= \dpd{A}{t} + \sum_i\left( \dpd{A}{p_i} \dod{p_i}{t} + \dpd{A}{q_i} \dpd{q_i}{t} \right) \\
	&= \dpd{A}{t} + \sum_i \left( \dpd{A}{q_i} \dpd{H}{p_i} - \dpd{A}{p_i} \dpd{H}{q_i} \right) \\
	&= \dpd{A}{t} + \{A, H\}
\end{align}
where $\{A, B\}$ is the Poisson bracket
\begin{eqn}
\{A,B\} \equiv \sum_i \left( \dpd{A}{q_i} \dpd{B}{p_i} - \dpd{A}{p_i} \dpd{B}{q_i} \right).
\end{eqn}

Let's look at the algebraic properties. We have antisymmetry
\begin{equation}
\{A, B\} = - \{B, A\},
\end{equation}
linearity across real numbers
\begin{equation}
\{\alpha A + \beta B, C\} = \alpha\{ A, C\} + \beta\{B, C\},
\end{equation}
Leibnitz rule
\begin{equation}
\{AB, C\} = A\{B, C\} + \{ A, C\} B,
\end{equation}
and the Jacobi identity
\begin{equation}
\{A, \{B, C\}\} + \{B, \{C, A\}\} + \{C, \{A, B\}\} = 0.
\end{equation}

These are the properties that also hold for the commutator in quantum mechanics and the Lie bracket for Lie algebras and Lie derivatives for manifolds. But In quantum mechanics, the objects you put into the commutator are not $c$-numbers but operators, so that order matters, i.e.
\begin{equation}
\{AB, C\} = A\{B, C\}+ \{A, C\} B
\end{equation}
is NOT the same as
\begin{equation}
\{AB, C\} = A\{B, C\}+ B\{A, C\}
\end{equation}
for Lie brackets and commutators.

We can now rewrite Hamilton's equations as
\begin{equation}
\dot{q_i} = \{q_i, H\}, \qquad
\dot{p_i} = \{ p_i, H\}.
\end{equation}
(Note that conservation laws for some quantity $A$ assume that $A$ does not have explicit time dependence.)

If $A$ and $B$ are conserved then so is $\{A, B\}$. So if we have
\begin{equation}
\{A, H\} = 0, \qquad
\{B, H\} = 0
\end{equation}
Then the quantity
\begin{equation}
C = \{A, B\}
\end{equation}
is also conserved. Using the Jacobi identity
\begin{equation}
\{H, C\} = \{H, \{A, B\}\} = -\{A, \{B, H\}\} + \{B, \{A, H\}\} = 0,
\end{equation}
so we see that $C$ is also conserved.

For example, angular momentum components $L_1$ and $L_2$ are conserved. Then $\{L_1, L_2\}$ is conserved

[incomplete. continued in next lecture?]

\end{document}


%
% Chapter 1.4
%

\section*{1.4 Tangent and Velocity}

A \textbf{tangent} is a line touching a curve. A tangent line has the same direction as the curve at the point of contact. A secant line, on the other hand, cuts through the curve more than once.
\\\\
To find the tangent at a point on a curve (in slope-intercept form: \(y=mx+b\)), we must find the slope \(m\) at that point and use it to find \(b\) (the \(y\)-intercept).

\subsection*{Average Velocity}

$$\text{average velocity} = \frac{\text{change in position}}{\text{elapsed time}}$$


\section{Statistical Analysis}
\label{sec:statisticalAnalysis}
One of the most assessed ways to investigate the impact of the intrinsic variation of the input space is through the computation of
statistical moments and linear correlation among variables/parameters/FOMs.

As shown in Section~\ref{sec:forwardSamplingStrategies}, RAVEN employs several different sampling methodologies to explore the response of a model subject to uncertainties. In order to correctly compute the statistical moments a weight-based approach is used. Each \textit{Sampler} in RAVEN associate to each ``sample'' (i.e.
realization in the input/uncertain space) a \textbf{weight}  to represent the \textit{importance} of the particular
combination of input values from a statistical point of view (e.g., reliability weights). These weights are used in subsequential
steps in order to compute the previously listed statistical moments and correlation metrics.
\\In the following subsections, the formulation of these statistical moments is reported.
\subsection{Expected Value}
The expected value represents one of the most fundamental metrics in probability theory: it represents a measurement of the center of the distribution (mean) of the random variable.
From a practical point of view, the expected value of a discrete random variable is the probability-weighted average of all possible values of the subjected variable. Formally, the expected value of a random variable $X$:
\begin{equation}
\begin{matrix}
\mathbb{E}(X) = \mu = \sum_{x \in \chi} x  pdf_{X}(x) & \text{if  $X$  discrete} \\
\\
\mathbb{E}(X) = \mu = \int_{x \in \chi} x pdf_{X}(x) & \, \text{if $X$ continuous}
\end{matrix}
\end{equation}
In RAVEN, the expected value (i.e. first central moment) is computed as follows:
\begin{equation}
\begin{matrix}
\mathbb{E}(X) = \mu \approx \overline{x} = \frac{1}{n} \sum_{i=1}^{n}  x_{i} & \text{if  random sampling} \\
\\
\mathbb{E}(X) = \mu \approx \overline{x} = \frac{1}{V_{1}} \sum_{i=1}^{n} w_{i}  x_{i}  & \, \text{otherwise}
\end{matrix}
\end{equation}
where:
\begin{itemize}
  \item $w_{i}$ is the weight associated with the sample $i$
  \item $n$ are the total number of samples
  \item $V_{1} = \sum_{i=1}^{n} w_{i}$.
\end{itemize}
\subsection{Standard Deviation and Variance}
The variance ($\sigma^{2}$) and standard deviation ($\sigma$) of $X$ are both measures of the spread of the distribution of the random variable about the
mean. Simplistically, the variance measures how far a set of realizations of a random variable are spread out.
The standard deviation is the square root of the variance. The standard deviation has the same unit of the original data, and hence is comparable to deviations from the mean.
\\Formally:
\begin{equation}
  \begin{matrix}
  \sigma^{2}(X)= \mathbb{E}\left(\left[X - \mathbb{E}(X)\right]^{2}\right) = \int_{x \in \chi} (x - \mu)^2 pdf(x) dx  & \,\text{if $X$ continuous} \\
  \sigma^{2}(X)= \mathbb{E}\left(\left[X - \mathbb{E}(X)\right]^{2}\right)  = \sum_{x \in \chi} (x - \mu)^2 pdf(x)  & \text{if  $X$ discrete}
  \\
  \\
  \sigma(X)= \mathbb{E}\left(\left[X - \mathbb{E}(X)\right]\right)  = \sqrt{\sigma^{2}(X)}
  \end{matrix}
\end{equation}
In RAVEN, variance (i.e., second central moment) and standard deviation are computed as follows:
\begin{equation}
\begin{matrix}
\mathbb{E}\left(\left[X - \mathbb{E}(X)\right]^{2}\right)  \approx  m_{2} = \frac{1}{n} \sum_{i=1}^{n}  (x_{i} - \overline{x})^{2} & \text{if  random sampling} \\
\\
\\
\mathbb{E}\left(\left[X - \mathbb{E}(X)\right]^{2}\right)  \approx m_{2}  = \frac{1}{V_{1}} \sum_{i=1}^{n} w_{i}  (x_{i} - \overline{x})^{2}  & \, \text{otherwise}
\\
\\
\mathbb{E}\left(\left[X - \mathbb{E}(X)\right]^{2}\right)  \approx s  =  \sqrt{m_{2}}
\end{matrix}
\end{equation}
where:
\begin{itemize}
  \item $w_{i}$ is the weight associated with the sample $i$
  \item $n$ are the total number of samples
  \item $V_{1} = \sum_{i=1}^{n} w_{i}$.
\end{itemize}
RAVEN performs an additional correction of variance to obtain an unbiased estimation  with respect to the sample-size~\cite{RimoldiniUnbiased}:
\begin{equation}
\begin{matrix}
\mathbb{E}\left(\left[X - \mathbb{E}(X)\right]^{2}\right)  \approx M_{2} = \displaystyle \frac{n}{n-1}m_{2} & & \text{if random sampling}
\\
\mathbb{E}\left(\left[X - \mathbb{E}(X)\right]^{2}\right)  \approx M_{2} = \frac{V_{1}^{2}}{V_{1}^{2} - V_{2}}m_{2} &  text{otherwise}
\end{matrix}
\end{equation}
\begin{equation}
S = \sqrt{M_{2}}
\end{equation}
where:
\begin{itemize}
  \item $w_{i}$ is the weight associated with the sample $i$
  \item $n$ are the total number of samples
  \item $V_{1} = \sum_{i=1}^{n} w_{i}^{1}$.
  \item $V_{2} = \sum_{i=1}^{n} w_{i}^{2}$.
\end{itemize}
It is important to notice that $S$ is not an unbiased estimator.

\subsection{Skewness}
The Skewness is a measure of the asymmetry of the distribution of a
real-valued random variable about its mean. Negative skewness
indicates that the tail on the left side of the distribution is longer or fatter
than the right side.  Positive skewness indicates that the tail on the right
side is longer or fatter than the left side. From a practical point of view, the
skewness is useful to identify distortion  of the random variable with respect to
the Normal distribution function.
\\Formally,
\begin{equation}
\gamma_{1} = \mathbb{E} \left [ \left ( \frac{X-\mu}{\sigma} \right )^{3} \right ] = \frac{ \mathbb{E}\left [ \left ( X-\mu \right )^{3} \right ]}{\left ( \mathbb{E}\left [ \left ( X-\mu \right )^{2} \right ] \right )^{3/2}}
\end{equation}
In RAVEN, the skewness is computed as follows:
\begin{equation}
\begin{matrix}
\mathbb{E} \left [ \left ( \frac{X-\mu}{\sigma} \right )^{3} \right ]  \approx \frac{m_{3}}{m_{2}^{3/2}} = \frac{  \frac{1}{n} \sum_{i=1}^{n}  (x_{i} - \overline{x})^{3} }{\left ( \frac{1}{n} \sum_{i=1}^{n}  (x_{i} - \overline{x})^{2} \right )^{3/2}} & \text{if random sampling}
\\
\\
\mathbb{E} \left [ \left ( \frac{X-\mu}{\sigma} \right )^{3} \right ]  \approx \frac{m_{3}}{m_{2}^{3/2}} = \frac{  \frac{1}{V_{1}} \sum_{i=1}^{n} w_{i} \times (x_{i} - \overline{x})^{3} }{\left ( \frac{1}{V_{1}} \sum_{i=1}^{n}  w_{i} \times (x_{i} - \overline{x})^{2} \right )^{3/2}} &  \, \text{otherwise}
\end{matrix}
\end{equation}
where:
\begin{itemize}
  \item $w_{i}$ is the weight associated with the sample $i$
  \item $n$ are the total number of samples
  \item $V_{1} = \sum_{i=1}^{n} w_{i}$.
\end{itemize}
RAVEN performs an additional correction of skewness to obtain an unbiased estimation  with respect to the sample-size~\cite{RimoldiniUnbiased}:
\begin{equation}
\begin{matrix}
\mathbb{E} \left [ \left ( \frac{X-\mu}{\sigma} \right )^{3} \right ]  \approx \frac{M_{3}}{M_{2}^{3/2}}  = \displaystyle \frac{n^{2}}{(n-1)(n-2)}m_{3}\times \frac{1}{\left ( \displaystyle \frac{n}{n-1}m_{2}  \right )^{3/2}} & \text{if random sampling}
\\
\\
\mathbb{E} \left [ \left ( \frac{X-\mu}{\sigma} \right )^{3} \right ]  \approx \frac{M_{3}}{M_{2}^{3/2}}  = \displaystyle \frac{V_{1}^{3}}{V_{1}^{3}-3V_{1}V_{2}+2V_{3}}m_{3} \times \frac{1}{\left ( \displaystyle \frac{V_{1}^{2}}{V_{1}^{2}-V_{2}}m_{2}  \right )^{3/2}} &  \,  \text{otherwise}
\end{matrix}
\end{equation}
where:
\begin{itemize}
  \item $w_{i}$ is the weight associated with the sample $i$
  \item $n$ are the total number of samples
  \item $V_{1} = \sum_{i=1}^{n} w_{i}^{1}$
  \item $V_{2} = \sum_{i=1}^{n} w_{i}^{2}$
  \item $V_{3} = \sum_{i=1}^{n} w_{i}^{3}$.
\end{itemize}

\subsection{Excess Kurtosis}
The  Kurtosis~\cite{Abramowitz}  is the degree of peakedness of a distribution of a real-valued random variable. In a similar way to the concept of skewness, kurtosis describes the shape of the distribution. The Kurtosis is defined in order to
obtain a value of $0$ for a Normal distribution. If it is greater than zero, it indicates that the distribution is high peaked; If it is smaller
that zero, it testifies that the distribution is flat-topped.
\\Formally, the Kurtosis can be expressed as follows:
\begin{equation}
\gamma_{2} = \frac{ \mathbb{E}\left [ \left ( X-\mu \right )^{4} \right ]}{\left ( \mathbb{E}\left [ \left ( X-\mu \right )^{2} \right ] \right )^{2}}
\end{equation}
In RAVEN, the kurtosis (excess) is computed as follows:
\begin{equation}
\begin{matrix}
\frac{ \mathbb{E}\left [ \left ( X-\mu \right )^{4} \right ]}{\left ( \mathbb{E}\left [ \left ( X-\mu \right )^{2} \right ] \right )^{2}}   \approx \frac{m_{4}-3m_{2}^{2}}{m_{2}^{2}} = \displaystyle  \frac{  \frac{1}{n} \sum_{i=1}^{n}  (x_{i} - \overline{x})^{4} -3\left ( \frac{1}{n} \sum_{i=1}^{n}  (x_{i} - \overline{x})^{2} \right )^{2}}{\left ( \frac{1}{n} \sum_{i=1}^{n}  (x_{i} - \overline{x})^{2} \right )^{2}} & \text{if random sampling}
\\
\\
\frac{ \mathbb{E}\left [ \left ( X-\mu \right )^{4} \right ]}{\left ( \mathbb{E}\left [ \left ( X-\mu \right )^{2} \right ] \right )^{2}}   \approx \frac{m_{4}-3m_{2}^{2}}{m_{2}^{2}} = \displaystyle  \frac{  \frac{1}{V_{1}} \sum_{i=1}^{n} w_{i} \times (x_{i} - \overline{x})^{4} -3\left ( \frac{1}{V_{1}} \sum_{i=1}^{n}  w_{i} \times (x_{i} - \overline{x})^{2} \right )^{2}}{\left ( \frac{1}{V_{1}} \sum_{i=1}^{n}  w_{i} \times (x_{i} - \overline{x})^{2} \right )^{2}} &   \text{otherwise}
\end{matrix}
\end{equation}
where:
\begin{itemize}
  \item $w_{i}$ is the weight associated with the sample $i$
  \item $n$ are the total number of samples
  \item $V_{1} = \sum_{i=1}^{n} w_{i}$.
\end{itemize}
RAVEN performs an additional correction of kurtosis (excess) to obtain an unbiased estimation  with respect to the sample-size~\cite{RimoldiniUnbiased}:
\begin{equation}
\begin{split}
\begin{matrix}
\frac{ \mathbb{E}\left [ \left ( X-\mu \right )^{4} \right ]}{\left ( \mathbb{E}\left [ \left ( X-\mu \right )^{2} \right ] \right )^{2}}   \approx \frac{M_{4}-3M_{2}^{2}}{M_{2}^{2}}  = \displaystyle \frac{n^{2}(n+1)}{(n-1)(n-2)(n-3)}m_{4}-\frac{3n^{2}}{(n-2)(n-3)}m_{2}^{2} & \text{if random sampling}
\\
\\
\frac{ \mathbb{E}\left [ \left ( X-\mu \right )^{4} \right ]}{\left ( \mathbb{E}\left [ \left ( X-\mu \right )^{2} \right ] \right )^{2}}    \approx \frac{M_{4}-3M_{2}^{2}}{M_{2}^{2}}  = \displaystyle  \frac{V_{1}^{2}(V_{1}^{4}-4V_{1}V_{3}+3V_{2}^{2})}{(V_{1}^{2}-V_{2})(V_{1}^{4}-6V_{1}^{2}V_{2}+8V_{1}V_{3}+3V_{2}^{2}-6V_{4})}m_{4}
- \\
\displaystyle \frac{3V_{1}^{2}(V_{1}^{4}-2V_{1}^{2}V_{2}+4V_{1}V_{3}-3V_{2}^{2})}{(V_{1}^{2}-V_{2})(V_{1}^{4}-6V_{1}^{2}V_{2}+8V_{1}V_{3}+3V_{2}^{2}-6V_{4})}m_{2}^{2} & \text{otherwise}
\end{matrix}
\end{split}
\end{equation}
where:
\begin{itemize}
  \item $w_{i}$ is the weight associated with the sample $i$
  \item $n$ are the total number of samples
  \item $V_{1} = \sum_{i=1}^{n} w_{i}^{1}$
  \item $V_{2} = \sum_{i=1}^{n} w_{i}^{2}$
  \item $V_{3} = \sum_{i=1}^{n} w_{i}^{3}$
  \item $V_{4} = \sum_{i=1}^{n} w_{i}^{4}$.
\end{itemize}

\subsection{Median}
The median of the distribution of a real-valued random variable is the number separating the higher half from the lower half of all
the possible values. The median of a finite list of numbers can be found by arranging all the observations from lowest value to highest value and picking the middle value.
\\Formally, the median $m$ can be cast as the number that satisfy the following relation:
\begin{equation}
  P(X\leq m) = P(X \geq m) = \int_{-\infty}^{m} pdf(x) dx=\frac{1}{2}
\end{equation}

\subsection{Percentile}
A percentile (or a centile) is a measure indicating the value below which a given percentage of observations in a group of observations fall.

\subsection{Covariance and Correlation Matrices}
Simplistically, the Covariance is a measure of how much two random variables variate together. In other words, It represents a
measurement of the correlation, in terms of variance,  among different variables. If the greater values of one variable mainly
correspond with the greater values of the other variable, and the same holds for the lesser values (i.e., the variables tend to show
similar behavior) the covariance is positive. In the opposite case, when the greater values of one variable mainly correspond to the
lesser values of the other (i.e., the variables tend to show opposite behavior) the covariance is negative.
Formally, the Covariance can be expressed as
\begin{equation}
 \boldsymbol{\Sigma}(\boldsymbol{X},\boldsymbol{Y})  = \mathbb{E} \left [ \left ( \boldsymbol{X}- \mathbb{E}\left [ \boldsymbol{X} \right ] \right ) \left ( \boldsymbol{Y}- \mathbb{E}\left [ \boldsymbol{Y} \right ] \right )^{T}\right ]
\end{equation}
Based on the previous equation, in RAVEN each entry of the Covariance matrix is computed as follows:
\begin{equation}
\begin{matrix}
 \mathbb{E} \left [ \left ( X- \mathbb{E}\left [ X \right ] \right ) \left ( Y- \mathbb{E}\left [ Y \right ] \right )\right ] \approx
 \frac{1}{n}\sum_{i=1}^{n} (x_{i} - \mu_{x})(y_{i} -  \mu_{y})  & \text{if random sampling}
\\
\\
 \mathbb{E} \left [ \left ( X- \mathbb{E}\left [ X \right ] \right ) \left ( Y- \mathbb{E}\left [ Y \right ] \right )\right ] \approx
\frac{1}{V_{1}} \sum_{i=1}^{n} w_{i} \times (x_{i} -  \mu_{x})(y_{i} -  \mu_{y}) &   \text{otherwise}
\end{matrix}
\end{equation}
where:
\begin{itemize}
  \item $w_{i}$ is the weight associated with the sample $i$
  \item $n$ are the total number of samples
  \item $V_{1} = \sum_{i=1}^{n} w_{i}$.
\end{itemize}
The correlation matrix (Pearson product-moment correlation coefficient matrix) can be obtained through the Covariance matrix, as follows:
\begin{equation}
\boldsymbol{\Gamma}(\boldsymbol{X},\boldsymbol{Y}) = \frac{\boldsymbol{\Sigma}(\boldsymbol{X},\boldsymbol{Y})}{\sigma_{x} \sigma_{y}}
\end{equation}
As it can be seen, The correlation between $X$ and $Y$ is the
covariance of the corresponding standard scores.

\subsection{Variance-Dependent Sensitivity Matrix}
The variance dependent sensitivity matrix is the matrix of the sensitivity
coefficients that show the relationship of the individual uncertainty
component to the standard deviation of the reported value for a test
item.
\\ Formally:
\begin{equation}
\boldsymbol{\Lambda}= \boldsymbol{\Sigma}(\boldsymbol{X},\boldsymbol{Y})  vc^{-1}(\boldsymbol{Y})
\end{equation}
where:
\begin{itemize}
  \item $vc^{-1}(\boldsymbol{Y})$ is the inverse of the covariance of the
  input space.
\end{itemize}


%!TEX root = Calculus_I.tex
\chapter{Key Concept: The Definite Integral}

\section{How Do We Measure Distance Traveled?}
The \textbf{\textit{distance}} formula is commonly referred to as:
%
\begin{equation}
\text{Distance} = \text{Velocity} \cdot \text{Time}
\end{equation}
%
Recall that \textbf{\textit{velocity}} has both magnitude \textit{and} direction. Thus, it is important to know what direction indicates positive \textbf{\textit{velocity}}. This section will estimate \textbf{\textit{distance}} when \textbf{\textit{velocity}} is time-varying.

\vspace{0.1in}
When estimating the \textbf{\textit{distance}} traveled, it is important to know how often the \textbf{\textit{velocity}} measurements are taken. For example, consider this table with measurements taken every 2 seconds:
%
\begin{table}
\begin{center}
\begin{tabular}{ccccccc}
\hline
Time (sec) & 0 & 2 & 4 & 6 & 8 & 10\\
\hline
Velocity $\left(\frac{\text{ft}}{\text{s}}\right)$ & 20 & 30 & 38 & 44 & 38 & 50\\
\hline
\end{tabular}
\end{center}
\end{table}
%
Higher frequency \textbf{\textit{velocity}} measurements result in less \textbf{\textit{distance}} estimation error, but an estimate can still be made. This results in:
%
\begin{equation}
20\cdot2 + 30\cdot2 + 38\cdot2 + 44\cdot2 + 48\cdot2 = 360 \text{feet}
\end{equation}
%
This serves as a lower limit because we know the car has moved at these speeds when it was measured. An upper limit considers the fact that the car instantaneously accelerated to the maximum velocity from the next measurement, yielding:
%
\begin{equation}
30\cdot2 + 38\cdot2 + 44\cdot2 + 48\cdot2 + 50\cdot2 = 420 \text{feet}
\end{equation}
%
Thus, we can conclude that:
%
\begin{equation}
360 \leq \text{Total Distance Traveled} \leq 420 \text{feet}
\end{equation}
%
A smaller difference between the upper and lower estimates can be obtained by increasing the measurement frequency. Each time interval between measurements can be represented by a rectangle on the \textit{Time-Velocity}-axes. As the time intervals get smaller, the rectangles become thinner. The \textbf{\textit{limit}} of this is as the time interval approaches zero, the rectangles are infinitesimally thin, and the distance error between the lower and upper estimates approaches zero. It will be shown later that, in the \textbf{\textit{distance}} and \textbf{\textit{velocity}} relationship, the \textbf{\textbf{area under the curve}} of the \textit{Time-Velocity}-axes is equivalent to the \textbf{\textit{total distance}} traveled, if \textbf{\textit{velocity}} is strictly positive.

\vspace{0.1in}
If \textbf{\textit{velocity}} is ever negative, then the object is traveling back towards the starting position. Thus, its \textbf{\textit{distance}} from the starting position is decreasing, but the \textbf{\textit{total distance}} traveled is increasing.

\vspace{0.1in}
In the general case, let $v = f(t)$ be a non-negative \textbf{\textit{velocity}} function, $t \geq 0$. One may wish to determine the \textbf{\textit{distance}} traveled between times $a$ and $b$. Measurements are taken at evenly spaced times, $t_0$, $t_1$, ... $t_n$. If $a = t_0$ and $b = t_n$, then the time interval between any two measurements is given by:
%
\begin{equation}
\Delta t = \frac{b - a}{n}
\end{equation}
%
For each time interval, $t_i$, the \textbf{\textit{distance}} traveled is given by:
%
\begin{equation}
\text{Distance} = f\left(t_i\right)\Delta t
\end{equation}
%
Summing all of the distances between each subsequent time interval between $a$ and $b$ yields:
%
\begin{equation}
\label{eq:LHSum}
\text{Distance} \approx \sum_{i=0}^{n-1} f\left(t_i\right)\Delta t
\end{equation}
%
This is a \textbf{\textit{Left-Hand Sum}} because it includes all velocities from the left-side of the rectangular intervals. The \textbf{\textit{Right-Hand Sum}} can be written as:
%
\begin{equation}
\label{eq:RHSum}
\textbf{Distance} \approx \sum_{i=1}^n f\left(t_i\right)\Delta t
\end{equation}
%
If $f$ is an \textbf{\textit{increasing function}}, then the \textbf{\textit{Left-Hand Sum}} underestimates the \textbf{\textit{total distance}} and \textbf{\textit{Right-Hand Sum}} overestimates it. Conversely, if $f$ is \textbf{\textit{decreasing}}, then the \textbf{\textit{Left-Hand Sum}} overestimates the \textbf{\textit{total distance}} and \textbf{\textit{Right-Hand Sum}} underestimates it. For a \textbf{\textit{monotonically increasing}} or \textbf{\textit{monotonically decreasing}} function, the accuracy of the estimates is given by:
%
\begin{equation}
\text{Error} = |f(b) - f(a)| \cdot \Delta t
\end{equation}
%

\begin{center}
\section*{\small Examples}
Coming soon$!^{\text{TM}}$
\end{center}

\section{The Definite Integral}
The \textbf{\textit{Definite Integral}} is defined by taking the \textbf{\textit{limit}} of the \textbf{\textit{Left-Hand Sum}} or \textbf{\textit{Right-Hand Sum}} as the parameter $n$ approaches $+\infty$, provided the function $f(x)$ is \textbf{\textit{continuous}} on $[a, b]$. This can be written as:
%
\begin{equation}
\int_a^b f(x)dx
\end{equation}
%
The summations represented by Equations (\ref{eq:LHSum}) and (\ref{eq:RHSum}) are referred to as \textbf{\textit{Riemann Sums}}. The \textbf{\textit{integrand}} is the function being integrated, $f(x)$, and the \textbf{\textit{limits of integration}} are the endpoints of the interval, $a$ and $b$.

\vspace{0.1in}
More specifically, Equations (\ref{eq:LHSum}) and (\ref{eq:RHSum}) are special cases of \textbf{\textit{Riemann Sums}}. The general form of the \textbf{\textit{Riemann Sum}} for a function, $f(x), x \in [a, b]$, is given by:
%
\begin{equation}
\sum_{i=1}^n f\left(c_i\right) \Delta t_i
\end{equation}
%
where $a = t_0 < t_1 <$ ... $t_n = b$ and, for $i = 1, 2$ ..., $n$, $\Delta t_i = t_i - t_{i-1}$, and $t_{i-1} \leq c_i \leq t_i$.
\vspace{0.1in}
The \textbf{\textit{Definite Integral}} approximates the area under the curve down to the $x$-axis by summing the areas of $n$ rectangles in the \textbf{\textit{Riemann Sum}}. When the \textbf{\textit{integrand}} is negative, the distance to the $x$-axis is above the curve. Because as positive sign convention is used, the resulting area above the curve found through  \textbf{\textit{integration}} is negative. This is what causes \textbf{\textit{integrations}} such as $\int_0^{2\pi} \sin x$ $dx$ to equal zero.

\begin{center}
\section*{\small Examples}
Coming soon$!^{\text{TM}}$
\end{center}

\section{The Fundamental Theorem and Interpretations}
The \textbf{\textit{Fundamental Theorem of Calculus}} is written as:

\vspace{0.2in}
If $f$ is \textbf{\textit{continuous}} on $[a, b]$, and $f(x) = F\prime(x)$, then:
%
\begin{equation}
\int_a^b f(x)dx = F(b) - F(a)
\end{equation}
%
Thus, if a function $f$ is equal to the \textbf{\textit{rate of change}} of a quantity, then the \textbf{\textit{definite integral}} results in the total change.

\vspace{0.1in}
The \textbf{\textit{integral}} can also be used to approximate the average value of a function, $f$, over a given interval, $[a, b]$:
%
\begin{equation}
\text{Average Value of } f = \frac{1}{b-a}\int_a^bf(x)dx
\end{equation}
%
\vspace{0.1in}
Lastly, the \textbf{\textit{Fundamental Theorem of Calculus}} can be used to compute \textbf{\textit{definite integrals}} exactly.

\begin{center}
\section*{\small Examples}
Coming soon$!^{\text{TM}}$
\end{center}

\section{Theorems About Definite Integrals}
So far, we have only considered the \textbf{\textit{Definite Integral}} when $a < b$. Recall that:
%
\begin{equation}
\int_a^b f(x)dx = \lim_{n \rightarrow \infty} \sum_{i=1}^n f\left(x_i\right) \Delta x
\end{equation}
%
Then, provided $f(x)$ is \textbf{\textit{continuous}}, for any numbers, $a$, $b$, and $c$:
%
\begin{enumerate}
\item $\int_b^a f(x)$ $dx = - \int_a^b f(x)$ $dx$\\
\item $\int_a^c f(x)$ $dx + \int_c^b f(x)$ $dx = \int_a^b f(x)$ $dx$
\end{enumerate}
%
The first result can be derived from the definition of $\Delta x$, namely:
%
\begin{equation}
\Delta x = \frac{(a - b)}{n} = -\frac{(b - a)}{n}
\end{equation}
%
The second result is true because of the definition of $\Delta x$ and that the upper limit of integration of the first is equal to the lower limit of integration for the second, $c$.

\vspace{0.1in}
We can also evaluate properties of \textbf{\textit{integrals}} for multiple functions. Suppose $f$ and $g$ are both continuous functions, and $c$ is an arbitrary constant. Then:
%
\begin{enumerate}
\item $\int_a^b \left(f(x) \pm g(x)\right)$ $dx = \int_a^b f(x)$ $dx \pm \int_a^b g(x)$ $dx$\\
\item $\int_a^b c \cdot f(x)$ $dx = c\int_a^b f(x)$ $dx$
\end{enumerate}
%
These properties hold because of the \textbf{\textit{Principle of Superposition}} and that $c$ is simply a scaling factor.

\vspace{0.1in}
The area between curves can also be calculated, provided $f(x)$ lies above $g(x)$ for $a \leq x \leq b$:
%
\begin{equation}
\text{Area between $f$ and $g$} = \int_a^b\left(f(x) - g(x)\right)dx
\end{equation}
%
\vspace{0.1in}
Symmetry can also be used to aid in the evaluation of \textbf{\textit{integrals}}. For \textbf{\textit{Even Functions}}:
%
\begin{equation}
\int_{-a}^a f(x)dx = 2\int_0^a f(x)dx
\end{equation}
%
and for \textbf{\textit{Odd Functions}}:
%
\begin{equation}
\int_{-a}^a f(x)dx = 0
\end{equation}
%
This is because of the definition of \textbf{\textit{Even}} and \textbf{\textit{Odd Functions}}.

\begin{center}
\section*{\small Examples}
Coming soon$!^{\text{TM}}$
\end{center}


\section{Introduction to Ordinary Differential Equations}

Differential equations come from real-world problems and problems in applied mathematics. When mathematics is applied to real-world problems, it is often the case that finding a relation between a function and its rate of change is easier than finding a formula for the function itself; it is this relation between an unknown function and its derivatives that produces a differential equation.\newline
To give a very simple example, a biologist studying the growth of a population with size at time $t$ given by the function $P(t)$, might make the very simple, but logical, assumption that a population grows at a rate directly proportional to its size. In mathematical notation, the equation for $P(t)$ could then be written as:

\[
  \frac{dp}{dt} = rP(t)
\]

Where the constant of proportionality, $r$ would probably be determined experimentally by biologists working in the field. Equations used for modeling population growth can be much more complicated than this, sometimes involving scores of interacting populations with different properties.

\subsection{Basic Terminology}

  \begin{definition}
    A differential equation is any equation involving an unknown function and one or more if its derivatives.
  \end{definition}

  The following are examples of differential equations:

    \begin{enumerate}
      \item $P'(t)=rP(t)(1-P(t)/N)-H$ harvested population growth
      \item $\frac{d^2x}{d\tau^2}+0.9\frac{dx}{d\tau}+2x=0$ spring mass equation
      \item $I''(t)+4I(t)=sin(\omega t) $ RCL circuit showing beats
      \item $y''(t) + \mu(y^2(t)-1)y'(t)+y(t)$ ban der Pol equation
      \item $\frac{\partial^2}{\partial x^2}u(x,y)+\frac{\partial^2}{\partial y^2}u(x,y) = 0$ Laplace's equation
    \end{enumerate}

  \subsubsection{Ordinary vs. Partial Differential Equations}

    Differential equations fall into two very broad categories, called ordinary differential equations and partial differential equations. If the unknown function in the equation is a function of only one variable, the equation is called an ordinary differential equation. If the unknown function in the equation depends on more than one independent variable, the equation is called a partial differential equation, and in this case, the derivatives appearing in the equation will be partial derivatives.

  \subsubsection{Independent Variables, Dependent Variables, and Parameters}

   Three different types of quantities can appear in a differential equation. The unknown function, for which the equation is to be solved, is called the dependent variable, and when considering ordinary differential equations, the dependent variable is a function of a single independent variable. In addition to the independent and dependent variables, a third type of variable, called a parameter, may appear in the equation. A parameter is a quantity that remains fixed in any specification of the problem, but can very from problem to problem.

  \subsubsection{Order of a Differential Equation}

    Another important way in which differential equations are classified is in terms of their order.

    \begin{definition}
      The order of a differential equation is the order of the highest derivative of the unknown function that appears in the equation.
    \end{definition}

    The differential equation 1 is a first-order equation and the others are all second-order. Even though equation 5 is a partial differential equation, it is still said to be of second order since no derivatives of order higher than two appear in the equation.

  \subsubsection{What is a solution}

    Given a differential equation, what is a solution? We must realize that we are looking for a function, and therefore it needs to be defined on some interval of its independent variable.

    \begin{definition}
      An analytic solution of a differential equation is a sufficiently differentiable function that, if substituted into the equation, together with the necessary derivatives, makes the equation an identity (a true statement for all values of the independent variable) over some interval of the independent variables.
    \end{definition}

    \begin{problem}
      Show that the function $p(t)=e^{-2t}$ is a solution to the differential equation:

      \[
        x'' + 3x' + 3x = 0
      \]

      Solution. To show that it is a solution, compute the first and second derivatives of $p(t)$:

      \begin{align*}
        p'(t) &=- 2e^{ - 2t}\\
        p''(t) &= 4e^{ - 2t}
      \end{align*}

      When the three functions $p(t)$, $p'(t)$, and $p''(t)$ are substituted into the differential equation in place of $x$, $x'$, and $x''$, it becomes:

      \begin{align*}
        (4e^{ - 2t}) + 3( - 2e^{ - 2t}) + 2(e^{ - 2t})&\equiv 0\\
        (4 - 6 + 2)(e^{ - 2t})&\equiv0\\
        (0)(e^{ - 2t})&\equiv0
      \end{align*}

      which is an identity (in the independent variable $t$ for all real values of $t$).
      When showing that both sides of an equation are identical for all values of the variables, we will use the equivalence sign $\equiv$.
    \end{problem}

    \begin{problem}
      Show that the function $\phi(t)=(1-t^2)^{1/2}\equiv\sqrt{1-t^2}$ is a solution of the differential equation $x'=-t/x$.

      Solution. First, notice that $\phi(t)$ is not even defined outside the interval $-1\le t\le 1$. In the interval $-1<t<1$, $\phi(t)$ can be differentiated by the chain rule (for powers of functions):

      \[
        \phi'(t) =\frac{1}{2}(1 - t^2)^{ - \frac{1}{2}}( - 2t) =- \frac{t}{(1 - t^2)^{\frac{1}{2}}}
      \]

      The right-hand side of the equation $x'=-t/x$, with $phi(x)$ substituted for $x$, is

      \[
        - \frac{t}{\phi (t)} =- \frac{t}{(1 - t^2)^{\frac{1}{2}}}
      \]

      which is identically equal to $\phi(t)$ wherever $\phi$ and $\phi'$ are both defined. Therefore, $\phi(t)$ is a solution to the differential equation $x'=-t/x$ on the interval (-1, 1).
    \end{problem}

\subsection{Systems of Differential Equations}

  \begin{problem}
    Show that the functions $x(t)=e^{-t},y(t)=-4e^{-t}$ form a solution of the system of differential equations

    \[
      x'(t) = 3x + y\newline
      y'(t) =- 4x - 2y
    \]

    Solution. The derivatives that we need are $x'(t)=-e^-t$ and $y'(t)=-(-4e^{-t})=4e^{-t}$. Then substitution into the second equation gives:

    \begin{align*}
      3x+y=(2e^{-t})+(-4e^{-t})=(3-4)e^{-t}=-e^{-t}\equiv x'(t),\\
      -4x-2y=-4(e^{-t})-2(-4e^{-t})=(-4+8)e^{-t}=4e^{-t}\equiv y'(t);
    \end{align*}

    therefore, the given functions of x and y form a solution for the system.
  \end{problem}

\subsection{Families of Solutions, Initial-Value Problems}
  In this section the solutions of some very simple differential equations will be examined in order to give us an understanding of the terms $n$-parameter family of solutions and general solution of a differential equation. We will also be shown how to use certain types of information to pick one particular solution out of a set of solutions.

  While we do not yet have any formal methods for solving differential equations, there are some very simple equations that can be solved by inspection. One of these is:

  \[
    x' = x
  \]

  This first-order differential equation asks you to find a function $x(t)$ which is equal to its own derivative at every value of $t$.

  \begin{definition}
    A first-order differential equation with one initial condition specified is called an initial-value problem, usually abbreviated as an IVP. The solution of an IVP will be called a particular solution of the differential equation.
  \end{definition}

  \begin{problem}
    Solve the IVP $x'=x, x(0)=\frac{1}{2}$.

    Solution. Since we just found that the general solution of $x'=x$ is $x(t)=Ce^t$, we only need to use the initial condition to determine the value of C. This will pick out one particular curve in the family. Substituting $t=0$ and $x(0)=\frac{1}{2}$ into the general solution,

    \[
      x(0)=Ce^0=C=\frac{1}{2}.
    \]

    With $C=\frac{1}{2}$, the solution of the IVP is $x(t)=\frac{1}{2}e^t$. This particular solution is the dotted curve shown in Figure 1.1, with the initial point $(0,\frac{1}{2})$ circled.
  \end{problem}


\chapter{Binomial Coefficients}
\label{chapter:binomials}
\marginurl{%
  Permutations and Binomial Coefficients:\\\noindent
  Introduction to Combinatorics \#4
}{youtu.be/HLClazoqqzg}
This chapter studies the following question:
``how many ways to take $k$ objects out of a box with $n$ objects''.
We assume that the objects are taken one by one; note that there are four modes
for this question.
\begin{enumerate}
  \item we return objects to the box after we take them and the order in which
    we take them matters,
  \item we are \emph{do not} return the objects and the order in which
    we take them matters,
  \item we return objects to the box after we take them and the order in which
    we take them \emph{does not} matter,
  \item we are \emph{do not} return the objects and the order in which
    we take them \emph{does not} matter.
\end{enumerate}

The Table~\ref{table:selections} summarizes the results we are going
to prove.
\begin{table}[h!]
  \centering
  \begin{tabular}{lll}
    \toprule
    Object's name & Parameters & Formula \\
    \midrule
    \multirow{2}{*}{Functions} & we return objects and & \multirow{2}{*}{$n^k$} \\
                               & the order \emph{is} important & \\
    \multirow{2}{*}{Injections} & we \emph{do not} return objects and &
      \multirow{2}{*}{$\numberOfPermutations[k]{n}$} \\
                                & the order \emph{is} important & \\
    \multirow{2}{*}{Subsets} & we \emph{do not} return objects and & \multirow{2}{*}{$\binom{n}{k}$} \\
                             & the order \emph{is not} important & \\
    \multirow{2}{*}{Multisets} & we return objects and & \multirow{2}{*}{$\binom{n + k - 1}{k}$} \\
                               & the order \emph{is not} important & \\
    \bottomrule
  \end{tabular}
  \caption{Formulas for the numbers of ways to take $k$ objects out of a box
  with $n$ objects}
  \label{table:selections}
\end{table}

\section{Counting Functions}
Note that if we number objects using numbers from $1$ to $n$, then in the first
mode the answer is the same as the number of functions from $[n]$ to $[k]$ since
we need to just choose which objects is selected on the $i$th step for
$i \in \range{k}$.

Let us solve a more general question; assume we have two finite sets $X$ and
$Y$: how many functions exist from $X$ to $Y$?

\begin{theorem}
\label{theorem:number-of-functions}
  Let $X$ and $Y$ be some finite sets. $\functions{X}{Y}$ represents the set of all
  functions from $X$ to $Y$. Then $\cardinality{\functions{X}{Y}} =
  \cardinality{Y}^{\cardinality{X}}$.
\end{theorem}
\nomenclature[S]{$\functions{A}{B}$}{denotes the set of functions from $A$ to $B$}
\begin{proof}
  For simplicity we prove the statement in the case when $X = \range{n}$. Fix some
  finite set $Y$. We prove the statement using induction by $n$. The base case
  for $n = 1$ is obvious, since there are $\cardinality{Y}$ different functions
  from $\range{1}$ to $Y$. Let us prove the induction step, by the induction
  hypothesis, $\cardinality{\functions{\range{n - 1}}{Y}} =
  \cardinality{Y}^{n - 1}$. Note that
  \begin{multline*}
    \cardinality{\functions{\range{n}}{Y}} =
      \cardinality{\set[f \in Y^{[ n - 1]}, y \in Y]{(f, y)}} = \\
    \cardinality{\functions{\range{n - 1}}{Y} \times Y} =
      \cardinality{\functions{\range{n - 1}}{Y}} \cdot \cardinality{Y} =
      \cardinality{Y}^n.
  \end{multline*}
\end{proof}

\begin{corollary}
  There are $n^k$ ways to select $k$ objects out of $n$ if the order matters
  and we return objects to the box after we pick them.
\end{corollary}

\begin{exercise}
  Finish the proof of Theorem~\ref{theorem:number-of-functions} by proving that
  the statement holds for any set $X$.
\end{exercise}

However, what if we need to find size of a subset of $Y^X$ satisfying some
constraint? For example, we may try to find the size of the set
\[
  \injections{X}{Y} = \set[f\text{ is an injection}]{f \in \functions{X}{Y}}.
\]
\nomenclature[S]{$\injections{A}{B}$}{denotes the set of injections from $A$ to $B$}
First, let us try to do this informally. Assume that $X = \range{n}$ and
$\cardinality{Y} = m$,
to define $f \in \injections{X}{Y}$ we need to choose images of $1$, $2$, \dots,
$n$. There are $m$ possible ways to select an image of $1$, $m - 1$ ways to
define $f(2)$ since we cannot use the value selected for $1$ etc. Hence,
$\cardinality{\injections{X}{Y}} = m (m - 1) \dots (m - n + 1)$ (we denote this
number as $\numberOfPermutations[n]{m}$\footnote{%
  In combinatorics the notation $\numberOfPermutations[n]{m}$ is widely used and
  is also called the Pochhammer symbol. However, Knuth's $m^{\underline{n}}$
  notation is increasingly popular. It is also important to note that sometimes
  $\numberOfPermutations[n]{m}$ denotes $m (m + 1) \dots (m + n - 1)$. Moreover,
  Leo August Pochhammer, the author of the Pochhammer symbol, used the
  Pochhammer symbol to denote the binomial coefficients.
}).
\nomenclature[C]{$\numberOfPermutations[n]{m}$}{denotes the number of ways to choose a
subset of $n$ elements from a fixed set of $m$ elements}

\begin{theorem}
\label{theorem:number-of-injections}
  Let $X$ and $Y$ be some finite sets. Then $\cardinality{\injections{X}{Y}} =
  \numberOfPermutations[\cardinality{X}]{\cardinality{Y}}$.
\end{theorem}
\begin{proof}
  Let us prove this statement for $X = \range{n}$. We prove this using induction by
  $n$. The base case, for $n = 1$, is clear. Now we need to prove the induction
  step from $n$ to $n + 1$. By the induction hypothesis, for any $m$, the
  number of injections from $\range{n}$ to $Y$ is equal to
  $\numberOfPermutations[n]{\cardinality{Y}}$.

  Fix some $m$ and some set $Y$ of cardinality $m$. Note that
  \[
    \cardinality{\injections{X}{Y}} =
    \cardinality{
      \set[v \not\in \Im f]{(f, v) \in \injections{\range{n - 1}}{Y} \times \range{m}}
    }.
  \]
  It is easy to see that $\cardinality{\set[v \not\in \Im f]{(f, v)}} = m - n + 1$
  for any $f \in \injections{\range{n - 1}}{Y}$ and
  \[
    \set[v \not\in \Im f]{(f, v) \in \injections{\range{n - 1}}{Y} \times
      \range{m}} =
    \bigcup_{f \in \injections{\range{n - 1}}{Y}} \set[v \not\in \Im f]{(f, v)}.
  \]
  As a result, $\cardinality{\injections{X}{Y}} =
    \numberOfPermutations[n - 1]{m} \cdot (m - n + 1) =
    \numberOfPermutations[n]{m}$.
\end{proof}

The special case of this result is that there are
$n \cdot (n - 1) \cdot  \dots \cdot  1$ different
permutations of $\range{n}$ (recall that the number is denoted by
$\numberOfPermutations{n}$).

\begin{exercise}
  Finish the proof of Theorem~\ref{theorem:number-of-injections} by proving that
  the statement holds for any finite set $X$.
\end{exercise}

\begin{corollary}
  There are $\numberOfPermutations[k]{n}$ ways to select $k$ objects out of $n$ if the order matters
  and we do not return objects to the box after we pick them.
\end{corollary}

\section{Counting Subsets}
In this section we study the version of the question when we do not return the
objects back to the box; i.e., we cannot select an object twice.

Recall that we denoted the set of all subsets of $X$ by $2^X$. The reason for
this notation is that $\cardinality{\subsets{X}} = 2^{\cardinality{X}}$. A quite
famous example of a subset of this set is the set
\[
  \subsets[n]{X} = \set[\cardinality{A} = n]{A \subseteq X}.
\]
\nomenclature[S]{$\subsets[k]{A}$}{denotes the set of subsets of $A$ of
cardinality $k$}
In other words, it is the set of all possible ways to select $n$ elements from
$X$. Size of the set $\subsets[n]{\range{m}}$ we denote by $\binom{m}{n}$ and
call it a binomial coefficient.
\nomenclature[C]{$\binom{m}{n}$}{denotes the number of ways to choose an
unordered subset of $n$ elements from a fixed set of $m$ elements}
\begin{exercise}
  Show that for any two finite sets $X$ and $Y$, if $\cardinality{X} =
  \cardinality{Y}$, then
  $\cardinality{\subsets[k]{X}} = \cardinality{\subsets[k]{Y}}$.
\end{exercise}

Note that by any ordered selection of $n$ object out of $m$, one may construct
an unordered selection of $n$ objects out of $m$, and each unordered selection
is counted $n!$.
\begin{theorem}
\label{theorem:binomial-coefficients-explicit}
  For any $n > k \ge 0$,
  $\binom{n}{k} = \frac{\numberOfPermutations[k]{n}}{k!} = \frac{n!}{k!  (n - k)!}$.
\end{theorem}

\begin{exercise}
  Show that $\binom{n}{k} = \binom{n}{n - k}$ for any $n > k$.
\end{exercise}

The formula in the Theorem~\ref{theorem:binomial-coefficients-explicit} allows
to find the values of binomial coefficients, however, it is not very convenient
since $n!$ is growing very fast. Thus the following theorem provides a much more
efficient way to compute the values of binomial coefficients.

\begin{theorem}[Pascal's rule]
\label{theorem:pascals-rule}
  For $n > k \ge 1$, $\binom{n}{k} = \binom{n - 1}{k - 1} + \binom{n - 1}{k}$.
\end{theorem}
\begin{proof}
  The first, algebraic, proof of this theorem is quite simple, we just notice
  that
  \begin{multline*}
    \binom{n - 1}{k - 1} + \binom{n - 1}{k} =
      \frac{(n - 1)!}{(k - 1)!(n - k)!} + \frac{(n - 1)!}{k!(n - k - 1)!} = \\
      \frac{(n - 1)!}{(k - 1)! (n - k - 1)!}
        \left(
          \frac{1}{n - k} + \frac{1}{k}
        \right) = \frac{n!}{k! (n - k)!} = \binom{n}{k}.
  \end{multline*}

  However, this proof does not explain \emph{why} the statement is true.
  So we consider an alternative proof, which informally can be explained as
  follows. Assume we need to choose $k$ objects out of $n$. There are two
  possible ways:
  \begin{itemize}
    \item we may select $n$ and choose $k - 1$ objects from the rest,
    \item or we may decide to not select $n$ choose $k$ objects from the rest.
  \end{itemize}
  In the first case we have $\binom{n - 1}{k - 1}$ ways to select objects and
  in the second case we have $\binom{n - 1}{k}$ ways to select objects.

  Let us prove the statement a bit more formally. Note that
  \begin{multline*}
    \subsets[k]{\range{n}} =
      \set[\cardinality{A} = k \text{ and } n \in A]{A \subseteq \range{n}} \cup \\
    \set[\cardinality{A} = k \text{ and } n \notin A]{A \subseteq \range{n}}.
  \end{multline*}
  Since these sets are disjoint and
  $\set[\cardinality{A} = k \text{ and } n \notin A]{A \subseteq \range{n}} =
  \subsets[k]{\range{n - 1]}}$, we get the following equality
  \[
    \binom{n}{k} =
    \cardinality{\set[\cardinality{A} = k \text{ and } n \in A]{A \subseteq \range{n}}} +
    \binom{n - 1}{k}.
  \]

  Hence, to finish the proof we need to explain that
  \[
    \cardinality{\set[\cardinality{A} = k \text{ and } n \in A]{A \subseteq \range{n}}} =
    \binom{n - 1}{k - 1}.
  \]
  To prove this statement we construct a bijection
  \[
    f : \set[\cardinality{A} = k \text{ and } n \in A]{A \subseteq \range{n}} \to
      \subsets[k]{\range{n - 1}}
  \]
  such that $f(A) = A \setminus \set{n}$.
  It is clear that this is a bijection. Thus, we prove the statement.
\end{proof}

A mnemonic rule for the Pascal's rule is to use Pascal's triangle.
\footnote[][-8cm]{
  The pattern of numbers that forms Pascal's triangle was known well before
  Pascal's time.  Halayudha, around 975 explained obscure references to
  Meru-prastaara, the Staircase of Mount Meru, giving the first surviving
  description of the arrangement of these numbers into a triangle.

  The Persian mathematician Al-Karaji (953–1029) wrote a now lost book which
  contained the first description of Pascal's triangle. It was later repeated by
  the Persian poet-astronomer-mathematician Omar Khayyám (1048–1131); thus the
  triangle is also referred to as the Khayyam triangle in Iran.

  Pascal's triangle was known in China in the early 11th century through the
  work of the Chinese mathematician Jia Xian (1010–1070). In the 13th century,
  Yang Hui (1238–1298) presented the triangle and hence it is still called Yang
  Hui's triangle in China.

  Pascal's Traité du triangle arithmétique (Treatise on Arithmetical Triangle)
  was published in 1655. In this, Pascal collected several results then known
  about the triangle, and employed them to solve problems in probability theory.
  The triangle was later named after Pascal by Pierre Raymond de Montmort (1708)
  who called it ``Table de M. Pascal pour les combinaisons'' (French: Table of
  Mr. Pascal for combinations) and Abraham de Moivre (1730) who called it
  ``Triangulum Arithmeticum PASCALIANUM'' (Latin: Pascal's Arithmetic Triangle),
  which became the modern Western name.
}
\begin{figure}
  \centering
  \begin{tabular}{lccccccccc}
    &    &    &    &  1\\\noalign{\smallskip\smallskip}
    &    &    &  1 &    &  1\\\noalign{\smallskip\smallskip}
    &    &  1 &    &  2 &    &  1\\\noalign{\smallskip\smallskip}
    &  1 &    &  3 &    &  3 &    &  1\\\noalign{\smallskip\smallskip}
    1 &    &  4 &    &  6 &    &  4 &    &  1\\\noalign{\smallskip\smallskip}
  \end{tabular}
\end{figure}
In this diagram the $k$th entry of the $n$th row
(entries and rows have numbers starting from $0$) is equal to $\binom{n}{k}$.
Thus the rule for the triangle is very simple, the value of an entry is equal
to $1$ if it is the first or the last in the row or it is equal to the sum
of the two entries to the left and right on the row above.

\begin{exercise}
  Show that $\binom{n}{k} = \binom{n}{n - k}$ for any integers $n > k \ge 0$
\end{exercise}

\subsection{Binomial Theorem}
Now we are ready to prove the theorem which gave the name to binomial
coefficients.
\begin{theorem}[Binomial theorem]
\label{theorem:binomial}
  For any real numbers $x$ and $y$,
    \[
      \sum_{k = 0}^n \binom{n}{k} x^k y^{n - k} = (x + y)^n.
    \]
\end{theorem}
\begin{proof}
  Informally, the explanation of the equality is as follows.
  If we consider the product
  \[
    \underbrace{(x + y) \cdot (x + y) \cdot
      \ldots \cdot (x + y)}_{n \text{ times}},
  \]
  then for every $k$ there are exactly $\binom{n}{k}$ possibilities to obtain
  $x^k y^{n - k}$. Indeed, to obtain $x^k y^{n - k}$ we need to choose $x$ from
  $n$ possibilities (corresponding to the multiplier $x + y$) exactly $k$ times.

  A formal proof uses the induction by $n$. The base case is
  true, since $\sum_{k = 0}^1 \binom{1}{k} x^k y^{1 - k} = x + y =
  (x + y)^1$. Assume that
  \[
    \sum_{k = 0}^n \binom{n}{k} x^k y^{n - k} = (x + y)^n,
  \]
  we wish to prove that
  \[
    \sum_{k = 0}^{n + 1} \binom{n + 1}{k} x^k y^{n + 1 - k} =
      (x + y)^{n + 1}.
  \]
  Note that
  \begin{multline*}
    (x + y)^{n + 1} = (x + y)
      \left(
        \sum_{k = 0}^n \binom{n}{k} x^k y^{n - k}
      \right) = \\
    \sum_{k = 0}^n \binom{n}{k} x^{k + 1} y^{n - k} +
      \sum_{k = 0}^n \binom{n}{k} x^{k} y^{n + 1 - k} = \\
    \sum_{k = 1}^{n + 1} \binom{n}{k - 1} x^k y^{n + 1 - k} +
      \sum_{k = 0}^n \binom{n}{k} x^k y^{n + 1 - k} = \\
    \sum_{k = 0}^{n + 1}
      \left(
        \binom{n}{k - 1} + \binom{n}{k}
      \right)
      x^k y^{n + 1 - k} =
    \sum_{k = 0}^{n + 1} \binom{n + 1}{k} x^k y^{n + 1 - k}.
  \end{multline*}
\end{proof}

Finally, we need to answer the question in the mode, when the order does not
matter and we do not return the objects to the box. The answer to this question
is clearly equal to the number of multisets of $\range{n}$ containing $k$
objects.
\begin{theorem}
\label{theorem:multisets}
  The number of $k$-element multisets whose elements all belong to $\range{n}$ is
  $\binom{n + k - 1}{k}$.
\end{theorem}

\begin{exercise}
  Prove Theorem~\ref{theorem:multisets}
\end{exercise}

Using \Cref{theorem:binomial} one may prove several important equalities of sums
of binomial coefficients.
\begin{corollary}
\label{corollary:binommials-equality}
  Let $n \in \N$. Then
  \begin{enumerate}
    \item $\sum_{k = 0}^n (-1)^k \binom{n}{k} = 0$ and
    \item $\sum_{k = 0}^n k \binom{n}{k} = n 2^{n - 1}$.
  \end{enumerate}
\end{corollary}
\begin{proof}
  \begin{enumerate}
    \item Let $x = -1$. We may notice that, by \Cref{theorem:binomial},
      \[
        \sum_{k = 0}^n (-1)^k \binom{n}{k} =
        \sum_{k = 0}^n 1^{n - k} x^k \binom{n}{k} = (1 - 1)^n = 0.
      \]
    \item This equality is a bit more tricky. Let $x = 1$. Note that
      \[
        \sum_{k = 0}^n k \binom{n}{k} =
	\sum_{k = 0}^n k x^{k - 1} 1^{n - k} \binom{n}{k} =
        \frac{d \sum_{k = 0}^n x^k \binom{n}{k}}{dx} = n (1 + x)^{n - 1}.
      \]
      As a result, $\sum_{k = 0}^n k \binom{n}{k} = n 2^n$.
  \end{enumerate}
\end{proof}

Using the idea of the second equality, we can give an alternative --- more
explicit --- proof of \Cref{claim:guess-one-out-of-many}.
\begin{proof}[Proof of \Cref{claim:guess-one-out-of-many}]
\label{proof-claim:guess-one-out-of-many}
  Let $x = 1 / 2$. Then the value we would like to compute is
  $\sum_{k = 1}^n k x^k$. Note that
  \begin{multline*}
    \sum_{k = 1}^n k x^k = x \sum_{k = 1}^n k x^{k - 1} =
    x \frac{d \sum_{k = 0}^n x^k}{dx} = \\
    x \frac{d}{dx} \frac{1 - x^{n + 1}}{1 - x} =
    x \frac{-(n + 1) x^n (1 - x) + (1 - x^{n + 1})}{(1 - x)^2} =  \\
    2\left(
      1 - \frac{1}{2^{n + 1}} - (n + 1) \frac{1}{2^{n + 1}}
    \right) = 2 - \frac{n + 2}{2^n}.
  \end{multline*}
\end{proof}

\subsection{Counting Groups of Subsets}
In this section we study a generalization of the question we study in the
previous sections: ``How many ways to select $\ell$ groups made of $k_1$, $k_2$,
\dots, $k_\ell$ objects, respectively, out of $n$''. We denote this number by
$\binom{n}{k_1 \ k_2 \ \dots \ k_\ell \  (n - m)}$, where
$m = k_1 + \dots + k_\ell$.

Clearly selecting these objects is the same as selecting $k_1$ objects out of
$n$, after that selecting $k_2$ objects out of $n - k_1$ etc. As a result,
\begin{multline*}
  \binom{n}{k_1 \ k_2 \ \dots \ k_\ell \ (n - m)} = \\
  \frac{n!}{k_1! (n - k_1)!} \cdot \frac{(n - k_1)!}{k_2! (n - k_1 - k_2)!}
  \cdot \ldots \cdot
  \frac{
    (n - k_1 - k_2 - \dots - k_{\ell - 1})!
  }{
    k_\ell! (n - k_1 - k_2 - \dots - k_\ell)!
  } = \\
  \frac{n!}{k_1! k_2! \dots k_\ell! (n - k_1 - k_2 - \dots - k_\ell)!}.
\end{multline*}

Similarly to the Binomial theorem, we can prove the following.
\begin{theorem}[Multinomial theorem]
\label{theorem:multinomial-coefficients}
  For any real numbers $x_1$, $x_2$, \dots, $x_\ell$ and integer $n$,
  \[
    (x_1 + x_2 + \dots + x_\ell)^n =
    \sum_{k_1, k_2, \dots, k_\ell ~:~ k_1 + k_2 + \dots + k_\ell = n}
      \binom{n}{k_1 \ k_2 \ \dots \ k_\ell} \prod_{i = 1}^n x_i^{k_i}.
  \]
\end{theorem}

\begin{exercise}
  Prove Theorem~\ref{theorem:multinomial-coefficients}.
\end{exercise}

\section{Double Counting}
\marginurl{%
  Double Counting:\\\noindent
  Introduction to Combinatorics \#4
}{youtu.be/OrzMP8nuuho}
The method that was used to prove Theorem~\ref{theorem:pascals-rule} can be
generalized to a method that is called \emph{double counting principle}.
The double counting principle states the following “obvious” fact: if the
size of a set is counted in two different ways, the answers are the same.

Using this principle we may prove the following theorem.
\begin{theorem}[Vandermonde's identity]
  For any integers $n, m > k$,
  $\sum_{i = 0}^k \binom{n}{i} \binom{m}{k - i} = \binom{n + m}{k}$.
\end{theorem}
\begin{proof}
  The idea is as follows, let us imagine that we have $n$ parrots and $m$ crows,
  and we need to find how many ways to select $k$ birds.
  It is easy to see that it is equal to $\binom{n + m}{k}$. At the same
  if we need to select $i$ parrots there are $\binom{n}{i}\binom{m}{k - i}$
  ways to do this. Thus the number is also equal to
  $\sum_{i = 0}^k \binom{n}{i}\binom{m}{k - i}$.
\end{proof}

However, the method can be used in a more sophisticated way.
\begin{lemma}[Handshaking Lemma]
\label{lemma:handshaking}
  Suppose some number of people meet at a party and some shake hands. Assume
  that no person shakes his or her own hand and furthermore no two people shake
  hands more than once.

  The number of guests who shake hands an odd number of times is even.
\end{lemma}
\begin{proof}
  Let $1$, \dots, $n$ be the people at the party. We apply double counting to
  the set of ordered pairs $(i, j)$ for which $i$ and $j$ shake hands with each
  other at the party. Let $d_i$ be the number of times that $i$ shakes hands,
  and $e$ be the total number of handshakes that occur. On one hand, the number
  of pairs is $\sum_{i = 1}^n d_i$, since for each $i$ the number of choices of
  $j$ is equal to $d_i$. On the other hand, each handshake gives rise to two
  pairs $(i, j)$ and $(j, i)$; so the total is $2e$.
  Thus $\sum_{i = 1}^n d_i = 2e$. But, if the sum of $n$ numbers is even, then
  evenly many of the numbers are odd.
  (Because if we add an odd number of odd numbers and any number of even
  numbers, the sum will be always odd).
\end{proof}

\begin{chapterendexercises}
  \exercise Let $m$ be some integer. Show that product of $m$ consecutive
    integers is divisible by $m!$.
    \begin{solution}
     In other words we need to show that
     $\frac{n \cdot (n + 1) \cdot \dots \cdot (n + m - 1)}{m!}$ is an integer,
     for any integer $n$. But one may notice that
     $\frac{n \cdot (n + 1) \cdot \dots \cdot (n + m - 1)}{m!} =
      \binom{n + m - 1}{m}$ which is an integer.
    \end{solution}
  \exercise Show that
    $\numberOfPermutations[n]{x + y} = \sum_{k = 0}^n \binom{n}{k}
      \numberOfPermutations[k]{x} \numberOfPermutations[n - k]{y}$.
  \exercise Show that
    $\sum_{k = 0}^n \binom{n}{k}^2 = \binom{2n}{n}$.
  \exercise Show that $\sum_{m = k}^n \binom{m}{k} =
    \binom{n + 1}{k + 1}$.
    \hint{Note that the formula on the right corresponds to the number of ways
      to select $k + 1$ elements out of $n + 1$; $m$ in the summation on the
      left denotes the maximum of this selected set minus one.}
  \exercise Using the previous formula, find the formulas for the following
    expressions:
    \begin{enumerate*}
      \item $\sum_{k = 0}^n k$,
      \item $\sum_{k = 0}^n k^2$, and
      \item $\sum_{k = 0}^n k^3$.
    \end{enumerate*}
  \exercise Prove the second statement from \Cref{corollary:binommials-equality}
	using double counting.
  \exercise Show that
    $\sum_{k = 2}^n k (k - 1) \binom{n}{k} = n (n - 1) 2^{n - 2}$ for any
    positive integer $n > 2$.
    \begin{solution}
      There are several possible solutions. We present two of them: the first
      one is similar to \Cref{corollary:binommials-equality}, and the second is
      based on the combinatorual meaning of the equality.
      \begin{enumerate}
        \item Note that $k (k - 1) \binom{n}{k} =
          k (k - 1) \frac{n!}{k! (n - k)!} = n (n - 1) \binom{n - 2}{k - 2}$.
          Thus
          \begin{multline*}
            \sum_{k = 2}^n k (k - 1) \binom{n}{k} =
            \sum_{k = 2}^n n (n - 1) \binom{n - 2}{k - 2} =  \\
              n (n - 1) \sum_{k = 0}^{n - 2} \binom{n - 2}{k} =
              n (n - 1) 2^{n - 2}.
          \end{multline*}
        \item Another solution is the following. Imagine that we have $n$ people in a
          group and we need to choose a subgroup of them, a head of this subgroup, and
          a vice-head of this group.

          It is easy to see that we have $n$ ways to select the head, $n - 1$
          ways to select the vice-head, and $2^{n - 2}$ ways
          to select the rest of the subgroup. On the other hand, if we know that the
          subgroup has $i$ members, then there are $\binom{n}{i}$ ways to select the
          subgroup, $i$ ways to select the head, and $i - 1$ ways to select the
          vice-head, thus there are $\sum_{k = 2}^n k (k - 1) \binom{n}{k}$ ways to
          select the subgroup, its leader, and its vice-leader. As a result,
          $n (n - 1) 2^{n - 2} = \sum_{k = 2}^n k (k - 1) \binom{n}{k}$.
      \end{enumerate}
    \end{solution}

  \exercise Using the binomial theorem, explain the following equalities:
    \begin{enumerate*}
      \item $\sum_{k = 0}^{n} \binom{2n}{2k} =
        \sum_{k = 0}^{n - 1} \binom{2n}{2k + 1}$,
        and
      \item $\sum_{k = 0}^{n} \binom{2n + 1}{2k} =
        \sum_{k = 0}^n \binom{2n + 1}{2k + 1}$.
    \end{enumerate*}

  \exercise[recommended] Show that $\sum_{k = 0}^n \binom{m + k}{k} =
    \binom{m + n + 1}{n}$.
  \exercise Show that $\sum_{k = 0}^n \binom{n - k}{k} = f_{n + 1}$,
    where $f_1 = 1$, $f_2 = 1$, and $f_{n + 2} = f_{n + 1} + f_n$ for $n > 0$.
  \exercise Show that $\binom{n}{m} \binom{m}{k} =
    \binom{n}{k} \binom{n - k}{m - k}$.
  \exercise[recommended] Show that
    $(a + 1)^p \equiv a^p + 1 \pmod{p}$.
    \hint{Use the binomial theorem.}
  \exercise[recommended] We say that a function
    $f : \set{0, 1}^n \to \set{0, 1}$ depends on the $i$th argument iff for some
    $a_1, \dots, a_{i - 1}, a_{i + 1}, \dots, a_n \in \set{0, 1}$
    \[
      f(a_1, \dots, a_{i - 1}, 0, a_{i + 1}, \dots, a_n) \neq
      f(a_1, \dots, a_{i - 1}, 1, a_{i + 1}, \dots, a_n).
    \]
    We also say that the function $f$ depends on all the arguments iff for all
    $i \in \range{n}$ it depends on $i$th argument.

    Find the number of functions $f : \set{0, 1}^n \to \set{0, 1}$ depending on all
    arguments.

    \begin{solution}
      Let $S \subseteq \range{n}$ and $F_i$ be the set of function from $\set{0,
      1}^n$ to $\set{0, 1}$ not depending on the $i$th input. It is easy to see
      that there are $2^{2^{n - 1}}$ elements in $F_i$ and moreover, there are
      $2^{2^{n - \cardinality{S}}}$ elements in the set $\cap_{i \in S} F_i$.
      Thus by the inclusion-exclusion principle, there are
      \[
        \sum_{S \subseteq [n] ~:~ S \neq \emptyset}
          (-1)^{\cardinality{S} + 1} 2^{2^{n - \cardinality{S}}} =
        \sum_{k = 1}^n (-1)^k 2^{2^{n - k}} \binom{n}{k}
      \]
      functions not depending on at least one argument.
      As a result, the answer is $2^{2^n} -
      \sum_{k = 1}^n (-1)^k 2^{2^{n - k}} \binom{n}{k}$.
    \end{solution}
  \exercise Find the largest coefficient of $(x_1 + x_2 + \dots + x_k)^k$.
  \exercise Prove that,
    without using Theorem~\ref{theorem:multinomial-coefficients},
    \[
      \sum_{k_1, k_2, k_3 ~:~ k_1 + k_2 + k_3 = n} \binom{n}{k_1 \ k_2 \ k_3} =
      3^n.
    \]
  \exercise How many different words one can get by reordering the letters of
    the word ``combinatorics'' (these words may be meaningless)?
    \begin{solution}
      Note that the word ``combinatorics'' has $13$ letters, two of them are
      ``o'', two of them are ``c'', two of them are ``i'', and all other are
      different. Hence, if we assume for a second that all the letters are
      different, we can get $13!$ different words. But note that we can exchange
      ``o''s and it does not change the word, we can also exchange ``c''s etc.
      Hence, we need to divide $13!$ by $2 \cdot 2 \cdot 2$. As a result, the
      answer is $\frac{13!}{8}$.
    \end{solution}
\end{chapterendexercises}


\chapter{Gaussian processes}

\section{Introduction}
In supervised learning, we observe some inputs $\vec{x}_i$ and some outputs $y_i$. We assume that $y_i =f(\vec{x}_i)$, for some unknown function $f$, possibly corrupted by noise. The optimal approach is to infer a \emph{distribution over functions} given the data, $p(f|\mathcal{D})$, and then to use this to make predictions given new inputs, i.e., to compute
\begin{equation}
p(y|\vec{x},\mathcal{D})=\int p(y|f,\vec{x})p(f|\mathcal{D})\mathrm{d}f
\end{equation}

Up until now, we have focussed on parametric representations for the function $f$, so that instead of inferring $p(f|\mathcal{D})$, we infer $p(\vec{\theta}|\mathcal{D})$. In this chapter, we discuss a way to perform Bayesian inference over functions themselves.

Our approach will be based on \textbf{Gaussian processes} or \textbf{GP}s. A GP defines a prior over functions, which can be converted into a posterior over functions once we have seen some data.

It turns out that, in the regression setting, all these computations can be done in closed form, in $O(N^3)$ time. (We discuss faster approximations in Section \ref{sec:Approximation-methods-for-large-datasets}.) In the classification setting, we must use approximations, such as the Gaussian approximation, since the posterior is no longer exactly Gaussian.

GPs can be thought of as a Bayesian alternative to the kernel methods we discussed in Chapter \ref{chap:Kernels}, including L1VM, RVM and SVM.

\section{GPs for regression}
Let the prior on the regression function be a GP, denoted by
\begin{equation}
f(\vec{x}) \sim GP(m(\vec{x}),\kappa(\vec{x},\vec{x}'))
\end{equation}
where $m(\vec{x}$ is the mean function and $\kappa(\vec{x},\vec{x}')$ is the kernel or covariance function, i.e.,
\begin{align}
m(\vec{x} & = \mathbb{E}[f(\vec{x})] \\
\kappa(\vec{x},\vec{x}') & = \mathbb{E}[(f(\vec{x})-m(\vec{x}))(f(\vec{x})-m(\vec{x}))^T]
\end{align}
where $\kappa$ is a positive definite kernel.

\section{GPs meet GLMs}

\section{Connection with other methods}

\section{GP latent variable model}

\section{Approximation methods for large datasets}
\label{sec:Approximation-methods-for-large-datasets}


\section{Residual systematics}
We measure the mean density of the data as a function imaging parameters, but limited to the mock footprint. This enables a null test based on the $\chi^{2}$ statistics to evaluate the performance of systematics mitigation. The null hypothesis is that the mean density in each bin of a particular imaging systematic $s_{k}$ should be equal to the global average density $\overline{n}_{tot}$, or the ratio should be one. Then, we have
\begin{equation}\label{eq:chi2nnbar}
    \chi^{2} = \sum_{s}\sum_{k} (1/\sigma^{2})[\frac{\overline{n}(s_{k})}{\overline{n}_{tot}}-1]^{2},
\end{equation}
where index k runs over the different bins, and index s runs over the different imaging systematics (see eqs. \ref{eq:nnbar_stat} and \ref{eq:error_jack}). The total number of terms is the number of bins multiplied by the number of the imaging maps, 20x18=360. We construct the distribution of the $\chi^{2}$ statistics by measuring the same quantity with the simulated data sets. Fig. \ref{fig:chi2pdf} represents the distributions of the $\chi^{2}$ for the null mocks (solid), contaminated mocks (dashed), and contaminated mocks after mitigation (dot-dashed). The vertical lines show the $\chi^{2}$ values of the data. The $\chi^{2}$ values observed in the data are 9567.1 (before treatment), 2066.7 (linear treatment), 1212.0 (quadratic treatment), 767.3 (Neural Network treatment), and 744.4 (Neural Network plain treatment). We also compute the observed $\chi^{2}$ value in the data after applying masking on the galactic extinction. With the NN-based treatment, we obtain 766.6 and 714.4 respectively for masking out EBV > 0.15 and EBV > 0.12.

The mean and standard deviation for the distribution of $\chi^{2}$  values observed in the null mocks are 487.1 and 49.2, respectively. The same quantities in the contaminated mocks are 2819.0 and 293.0, while in the contaminated mocks after Neural Network mitigation are 472.0 and 65.9. The mean $\chi^{2}$ observed in the contaminated mocks is much smaller than the value observed in the data (2819.0 < 9567.1). This indicates that the contamination introduced in the mocks is not as strong as in the real data. Future work will incorporate much more sophisticated systematics.

We perform a hypothesis testing given the $\chi^{2}$ value observed in the data after neural network mitigation (767.3) and the distribution of $\chi^{2}$ values in the null mocks.
If the terms in Eq. \ref{eq:chi2nnbar} were independent and normal deviates, we would have expected that $\chi^{2}$ follow a Chi-squared distribution with the mean being around 360. Nevertheless, we assume that the $\chi^{2}$ values observed in the mocks follow a Chi-squared distribution with the degree of freedom (dof) being equal to 487. We set the threshold of $\alpha$=0.05 and perform a one-tail test. For dof=487,
\begin{equation}
    P(\chi^{2}>593.4 | dof=487) = 0.05
\end{equation}
Given that the observed $\chi^{2}$ value in the data (after treatment) 767.3 is larger than 593.4, we can reject the null hypothesis that the data is completely clean. Therefore, this data set cannot be used for cosmology. This means that the 40\% excess in the residual squared could have been removed if we had achieved a 20\% better modeling of the target density or kept the errorbars under the control by 20\%.

The residual systematic effects are going to be due to known systematics that we had the template for since this test was based on the mean density vs the imaging systematics. Fig. \ref{fig:chi2breakdown} shows the contribution of each systematic to Eq. \ref{eq:chi2nnbar} for the $\chi^{2}$ values observed in the data after linear (light blue), quadratic (red), and neural network plain (dark blue) treatments. Further investigations on the templates for the depth-g, skymag-g, skymag-z, EBV, lnHI, and MJD-z along with the resolution used for pixelization or the cost function are needed, however the suggested analysis is beyond the scope of this paper. Also, our test does not show that there is no unknown systematic effects. We suggest that further analyses on the moments of density contrast ($\delta$) incorporating more realistic mock datasets could provide an insight on unknown systematics.

\begin{figure}
    \centering
    \includegraphics[width=0.45\textwidth]{fig25-crop.pdf}
    \caption{\textit{Left}: $\chi^{2}$ distribution for the null mocks (solid black), contaminated mocks (dashed black), and contaminated mocks with NN mitigation (dot-dashed black). The vertical dotted lines overlay the $\chi^{2}$ values for the data with NN correction (blue), quadratic correction (purple), and linear correction (dark red). The $\chi^{2}$ statistics before treatment is shown on the right (red).}
    \label{fig:chi2pdf}
\end{figure}

\begin{figure}
    \centering
    \includegraphics[width=0.45\textwidth]{fig26-crop.pdf}
    \caption{ The breakdown of $\chi^{2}$ values observed in the data after linear (light purple), quadratic (dark purple), neural network with feature selection (red), and neural network plain (light blue) treatments. We also plot the 95-th percentile of the same quantity observed in the null mocks by a solid orange curve (note that $\chi^{2}(\textbf{s)}$ is not a continuous quantity). Given the 5\% threshold, we can argue that there exists known residual systematics against EBV, depth-g, skymag-gz, and MJD-z.}
    \label{fig:chi2breakdown}
\end{figure}


%---------------------------Distortion---------------------------
\section{Distortion}

The distortion is a measure of how well-behaved the mapping from
parameter space to world coordinates is.
The parameter space is defined using a ``master'' tetrahedron
with vertices
\[
\begin{array}{lcrcrcrc}
 \vec P_0 &= (& -1&,& -\frac{ \sqrt{3}}{3}&,& -\frac{2\sqrt{6}}{9}&)\\
 \vec P_1 &= (&  1&,& -\frac{ \sqrt{3}}{3}&,& -\frac{2\sqrt{6}}{9}&)\\
 \vec P_2 &= (&  0&,&  \frac{2\sqrt{3}}{3}&,& -\frac{2\sqrt{6}}{9}&)\\
 \vec P_3 &= (&  0&,&                    0&,&  \frac{4\sqrt{6}}{9}&)
\end{array}
\]
and volume $V_m$.
The behavior of the map is measured by sampling the determinant of the
Jacobian at Gauss points $G = \{g_k\}$.
The minimum of these is then used to scale the ratio of the
``master'' tetrahedron to the tetrahedron of interest:
\[
q = \frac{\min_k\{\det(J_{g_k})\} V_m}{V}
\]

Note that if $V < DBL\_MIN$, we set $q = DBL\_MAX$.
This metric is currently unsupported.

\tetmetrictable{distortion}%
{$1$}%                          Dimension
{$[0.5,1]$}%                    Acceptable range
{$[0,1]$}%                      Normal range
{$[-DBL\_MAX,DBL\_MAX]$}%       Full range
{$0$}%                          Equilateral tet
{Adapted from \cite{ideas:xx}}% Citation
{v\_tet\_distortion}%                            Verdict function name


\appendix
\appendixpage
\section{Terms That Need More Explanation Then A Footnote}
\subsection{Potential} \label{sec:potential}
Potential is the energy change that occurs when the position of an object changes \cite{potential}. There are many potentials, like electric potential, gravitational potential and elastic
potential. Let me explain the concept with an example. Say you are walking on a set of stairs in the upwards direction. As your muscles move to bring you one step upwards, energy that is used
by your muscles is converted into gravitational potential. Now imagine you turn around and go downwards instead. Notice how that is easier? That is due to the gravitational potential being
converted back into energy so your muscles have to deliver less energy to get you down. The potential is usually tied to a force, like the gravitational force.

\subsection{Laplacian Operator} \label{sec:laplace}
The Laplacian operator ($\nabla^2$, sometimes also seen as $\Delta$) has two definitions, one for a vector field and one for a scalar field. The two concepts are not indpendent, a vector field
is composed of scalar fields \cite{vectorscalarfields}. Let us define a vector field first. A vector field is a function whose domain and range are a subset of the Eucledian $\mathbb{R}^3$ space.
A scalar field is then a function consisting out of several real variables (meaning that the variables can only take real numbers as valid values). So for instance the circle equation
$x^2 + y^2 = r^2$ is a scalar field as $x, y$ and $r$ are only allowed to take real numbers as their values.

With the vector and scalar fields defined, let us take a look at the Laplacian operator. For a scalar field $\phi$ the laplacian operator is defined as the divergence of the gradient of $\phi$
\cite{laplacian}. But what are the divergence and gradient? The gradient is defined in \autoref{eq:gradient} and the divergence is defined in \autoref{eq:divergence}. Here $\phi$ is a vector
with components $x, y, z$ and $\Phi$ is a vector field with components $x, y, z$. $\Phi_1, \Phi_2$ and $\Phi_3$ refer to the functions that result in the corresponding $x, y$ and $z$ values
\cite{vectorscalarfields}. Also, $i, j$ and $k$ are the basis vectors of $\mathbb{R^3}$, and the multiplication of each term with their basis vector results in $\Phi_1, \Phi_2$ and $\Phi_3$
respectively. If we then combine the two we get the Laplacian operator, as in \autoref{eq:laplacian scalar}.

\begin{subequations}
    \begin{equation}
        \text{grad } \phi = \nabla \phi = \frac{\delta \phi}{\delta x}i + \frac{\delta \phi}{\delta y}j + \frac{\delta \phi}{\delta z}k
        \label{eq:gradient}
    \end{equation}
    \begin{equation}
        \text{div} \Phi = \nabla \cdot \Phi = \frac{\delta \Phi_1}{\delta x} + \frac{\delta \Phi_2}{\delta y} + \frac{\delta \Phi_3}{\delta z}
        \label{eq:divergence}
    \end{equation}
    \begin{equation}
        \nabla^2 \phi = \nabla \cdot \nabla \phi = \frac{\delta^2 \phi}{\delta x^2} + \frac{\delta^2 \phi}{\delta y^2} + \frac{\delta^2 \phi}{\delta z^2}
        \label{eq:laplacian scalar}
    \end{equation}
\end{subequations}

For a vector field $\Phi$ the Laplacian operator is defined as in \autoref{eq:laplacian vector}. Which essential boils down to taking the Laplacian operator of each function and multiply it by
the basis vector.

\begin{equation}
    \nabla^2 \Phi = (\nabla^2 \Phi_1)i + (\nabla^2 \Phi_2)j + (\nabla^2 \Phi_3)k
    \label{eq:laplacian vector}
\end{equation}

\subsection{Interpolation} \label{sec:interpolation}
Interpolation is a form of estimation, where one has a set of data points and desires to know the values of other data points that are not in the original set of data points\cite{interpolation}.
Based on the original data points, it is estimated what the values of the new data points will be. There are various forms of interpolation like linear interpolation, polynomial interpolation
and spline interpolation. The CLAuDE model uses linear interpolation which is specified in \autoref{eq:interpolation}. Here $z$ is the point inbetween the known data points $x$ and $y$.
$\lambda$ is the factor that tells us how close $z$ is to $y$ in the interval $[0, 1]$. If $z$ is very close to $y$, $\lambda$ will have the value on the larger end of the interval, like 0.9.
Whereas if $z$ is close to $x$ then $\lambda$ will have a value on the lower end of the interval, like 0.1.

\begin{equation}
    z = (1 - \lambda)x + \lambda y
    \label{eq:interpolation}
\end{equation}


\mychapter{12}{Lesson 12} %181107
\section{Hashing}

Remember one solution to domain-extension for \prf{}s, as a composition of a \prf{} $F$ with an almost universal hash function $H$. Hash functions compress their arguments to some ``fingerprint'', which is assumed to be unique. However, since this compression in this context inherently introduces information loss, it is not guaranteed that every message gets its own unique fingerprint; indeed, there will be some instances where two messages yield the same hash value, or in other words, the hashes \emph{collide}. It is desirable for a hash function to be \emph{resistant} to these events, meaning that it is hard to reproduce such collisions.

\begin{definition}
    A hash function family $H$ is deemed \emph{collision-resistant}, denoted as $H \in \crh$ iff the probability of finding a collision is negligible, even when knowing the seed $s$. Formally:
    \[
        \forall \adversary \in \ppt \implies \Pr(\cryptog{crh}(\lambda) = 1) \in \negl(\lambda)
    \]

    \begin{cryptogame}
        {crh}
        {The \emph{collision-resistance} game}
        {crh}

        \receive{$s \pickUAR \binary^\lambda$}{$s$}{}

        \send{}{$x, y$}{\textsc{Output 1 iff} $h_s(x) = h_s(y)$}

    \end{cryptogame}
\end{definition}

A note: before, we were dealing with unbounded adversaries, and the key was hidden. Now the key is public, but the adversary must be efficient.

\begin{exercise}
    Let $\Pi$ be a \ufcma{} authentication scheme over the message space $\binary^n$. Show that $\Pi' = (\Tag', \Ver') : \Tag'_{k, s}(m) = \Tag_k(h_s(m))$ is \ufcma-secure over $\binary^l$, where $l \in \poly(n)$, as long as $h_s$ itself is \crh.
\end{exercise}

\subsection{Merkle-Damg\r{a}rd construction}

A construct for starters has been defined by Ralph Merkle and Ivan Damg\r{a}rd, which revolves around the use of a \crh{} function $h_s \in \binary^{n + c} \to \binary^n$, where $c$ is the size of one ``block'' of a message. For now, consider the messages to be of an arbitrarily fixed length $l$, and let $c$ be 1.  The steps to follow are:

\begin{enumerate}
    \item Let $\textsc{iv} \in \binary^b$ be an \emph{initialization vector}
    \item Initialize $t_0 := \textsc{iv}$
    \item For each bit $b_i$ of the message to hash $m$:
    \begin{enumerate}
        \item Compute $t_i := h_s(b_i, t_{i - 1})$
    \end{enumerate}
    \item Return $\phi = t_n$
\end{enumerate}

% AP190203: To review, it is somewhat confused
%First step: Compress the original message (assuming fixed size) by one bit

%Let $H$ be a one-bit shrinking function. Then, it can be used to construct a hash function H' that splits an arbitrary-size message into fixed-size blocks, apply H onto them, and return a digest of fixed length. This is exemplified by the diagram in figure \ref{fig:mdbase}

\begin{figure}
    \centering

    \tikzstyle{int}   = [draw, minimum size=2em]
    \tikzstyle{empty} = [minimum size=2em]
    \tikzstyle{init}  = [pin edge={to-,thin,black}]

    \begin{tikzpicture}[node distance = 1.9cm, auto, >=latex']

        \node (a) [empty] {$\textsc{iv}$};
        \node (r) [int, pin={[init]above:$b_1$}] [right of=a] {$h_s$};
        \node (d) [int, pin={[init]above:$b_2$}] [right of=r] {$h_s$};
        \node (e) [int, pin={[init]above:$b_3$}] [right of=d] {$h_s$};
        \node (f) [empty] [right of=e] {$...$};
        \node (g) [int, pin={[init]above:$b_n$}] [right of=f] {$h_s$};
        \node (h) [empty, right of=g] {$\phi$};

        \path[->] (a) edge (r);
        \path[->] (r) edge node {$t_1$} (d);
        \path[->] (d) edge node {$t_2$} (e);
        \path[->] (e) edge node {$t_3$} (f);
        \path[->] (f) edge node {$t_{n-1}$} (g);
        \path[->] (g) edge (h);

    \end{tikzpicture}
    \caption{Basic outline of a Merkle-Damg\r{a}rd construction}
    \label{fig:mdbase}
\end{figure}

Figure \ref{fig:mdbase} depicts a general view of the algorithm. Let it be denoted as another hash function $h'_s \in \binary^l \to \binary^n$.

\begin{theorem}
    The construction $H'$ obtained by Merkle-Damg\r{a}rd is a \textsc{crh} function.
\end{theorem}

% AP190204: Tentative
\begin{proof}
    Assume $H'$ can be broken efficiently by a distinguisher $\distinguisher^{\crh}$, meaning that finding two distinct block sequences that give the same hash is easy. Consider the reduction to $H$'s \crh-ness in figure \ref{cryptoredux:mdcrh}

    \begin{cryptoredux}
        {mdcrh}
        {Breaking the underlying \crh-ness of $H$}
        {h-crh}
        {h'-crh}[3]

        \receive{}{$s$}{}
        \invoke{}{$s$}{}
        \return{}{$x, y$}{}

        \cseqdelay

        % AP190911: need a better way to convey meaning here
        \send{\shortstack[r]{
            \textsc{Find} $j :$ \\
            $h'_{s, (j - 1)}(x) \neq h'_{s, (j - 1)}(y)$ \\
            $\wedge\: h'_{s, (j)}(x) = h'_{s, (j)}(y)$
        }}{$h'_{s, (j - 1)}(x), h'_{s, (j - 1)}(y)$}{}

    \end{cryptoredux}

    Ignore same blocks: find the largest j such that:
    \[
        (b_j, x_{j - 1}) \neq (b'_j, y_{j - 1}) \wedge h_s(b_j, x_{j - 1}) = h_s(b'_j, y_{j - 1})
    \]
    this implies the rest of the message is equal, then the resulting final hash will be equal, thus for $j > 0$ we have a collision.

\end{proof}

%AP190911: Not clear at all
\subsubsection{Domain extension}

In order to adapt the MD-construct to messages of variable length, some sort of ``strengthening'' is required:

% Also called MD-strengthening
\begin{lemma}[Length padding]
    Let $H_s \in \binary^{n + l} \to \binary^n$, then:
    %AP190911: This needs to be rewritten correctly
    \[
        H'_s = H_s(\left<l'\right>, H_s(x_{l'}, \dots , H_s(x_1, 0^n) \dots)), |l'|, |x_i| \in \binary^c
    \]
\end{lemma}

This is an example of padding which encodes the message length in itself.

\begin{theorem}
    The strengthened construct is collision-resistant for variable-length messages.
\end{theorem}

\begin{proof}
    Hint: similar as above, case by case
\end{proof}

\subsubsection{Merkle trees}

This is an alternative construction to the ``linear'' approach used beforehand: it starts from the message's blocks acting as leaves of a complete binary tree, and using a halving compression function on each node from the bottom up, and outputting the final tag as the image of the root invocation. It also has nice properties, such as easy verification of the presence of a single specific block in the message.

\subsection{Compression functions}

The compression functions are the central point of these kinds of ``fingerprinting'' tag schemes. They are, essentially, hash functions with a smaller codomain than its domain, thereby forcing the existence of collisions.

Let ($\textsf{Gen} : 0 \mapsto (\pk, \sk), f, g)$ be a \pke{} scheme, where the functions $f$ and $g$ are keyed \prp{}s. A \emph{claw} is a couple of values $(x, x')$ such that:
\[
    f(\pk, x) = g(\pk, x')
\]

\begin{cryptogame}
    {cfp}
    {The game of claw-free permutations}
    {cfp}

    \receive{$(\pk, \sk) \pickUAR \textsf{Gen}$}{$\pk$}{}
    \send{}{$x, x'$}{\textsc{Output 1 iff} $f(\pk, x) = g(\pk, x')$}

\end{cryptogame}

\begin{theorem}
    Assuming $\mathcal{F}$ is claw-free, then $h_\pk$ is \crh{} from $n + l$ bits to $n$.
\end{theorem}

\subsubsection{Davies-Meyer construct}

% AP190911: How does AES come in here?!
\begin{definition}
    $x_{i + 1} = E_k(x_i) \oplus x_i$, maps n+$\lambda$ to n. E is AES
\end{definition}


\documentclass{article}

\usepackage{mathrsfs,amssymb,amsmath}

\begin{document}

\section{Show that the particular point topology is $T_0$ but not $T_1$}

Consider the particular point topology of a set $X$ cardinality $\ge 3$ with particular point $x$, that is all sets containing $x$ are open as is the empty set, but no other sets.

Consider two arbitrary "other" points $x_1$ and $x_2$.

To show $X$ is $T_0$ consider the two cases:

Case 1: $x$ and $x_1$. $x$ has an open neighborhood $\{x\}$ that does not contain $x_1$.

Case 2: $x_1$ and $x_2$. $x_1$ has an open neighborhood $\{x, x_1\}$ that does not contain $x_2$.

This is sufficient to prove the statement for all pairs of points, since $x_1$ and $x_2$ were chosen arbitrarily, and every pair of points falls into one of these cases.

To show $X$ is not $T_1$, consider the case $x$ and $x_1$. Every open set containing $x_1$ will also contain $x$ (since all non-empty open sets contain $x$)

\section{Show that the particular point topology is not Hausdorf}

Obviously if a space is not $T_1$ it can't be $T_2$.

A more illustrative proof starts with the fact limits in Hausdorf are unique. So we must construct a sequence that converges to more than one point.

Consider any sequence $s_n$ that is eventually $x$. This sequence has $x$ as a limit, but it also has every other point in the space. In the case of $x_1$, every neighborhood contains of $\{x, x_1\}$, which $s_n$ eventually never leaves.

\end{document}