%
% Copyright 2018 Parakram Majumdar
%
% Licensed under the Apache License, Version 2.0 (the "License");
% you may not use this file except in compliance with the License.
% You may obtain a copy of the License at
%
%     http://www.apache.org/licenses/LICENSE-2.0
%
% Unless required by applicable law or agreed to in writing, software
% distributed under the License is distributed on an "AS IS" BASIS,
% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
% See the License for the specific language governing permissions and
% limitations under the License.
%

\documentclass{article}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{tikz}
\usetikzlibrary{shapes.geometric, arrows,positioning}
\title{Multi Layered Perceptron}
\author{Parakram Majumdar}

\begin{document}
  \maketitle

\newcommand{\R}{\mathbb{R}}
\newcommand{\Prob}[2]{\mathbb{#1}\left[ #2 \right]}
\newcommand{\sigmoid}[1]{\text{sigmoid}\left(#1\right)}
\newcommand{\map}[2]{\text{map}\left(#1,#2\right)}

\section{Introduction}
  We are given a training set $T$ of $t$ points in $\R^d$,
  a set $L$ of $l$ labels,
  and a \emph{classification} $C: T \rightarrow L$.

  Machine learning is interested in learning classifiers of the form
  $f: \R^d \rightarrow L$ that can closely approximate $C$.

\section{Single Perceptron, Single Point, Single Label}
  Having truly one label is an uninteresting case,
  as basically all points would have the same label,
  and there is nothing to ``learn''.
  Instead we assume two labels, $L_1$ and $L_2$.
  We then train a classifier for $L_1$,
  and assign to $L_2$ whatever is not in $L_1$.

  Given a single point $p \in \R^d$,
  a single perceptron $c$
  with \emph{weight} vector $w_c \in \R^d$,
  and a \emph{bias} $b_c \in \R$,
  for classifying a single Label $L_1$,
  accepts $p$ with the probability
  \[ \hat y_{p,L_1} = \Prob{\hat P}{p \in L_1} =  \sigmoid{w_c \cdot p + b_c}\]
  where
  \[ \sigmoid{x} = \frac{e^x}{e^{x} + 1} \]
  If the actual classification of the training point
  is given by $y_{p,L_1} = \Prob{I}{p \in L_1}$
  then training criterion is to minimize
  the cross entropy:
  \begin{eqnarray*}
    \mathbb{H} & = & \Prob{H}{\mathbb{P}, \mathbb{\hat P}} \\
               & = & \Prob{E}{\log{ \frac{1}{\hat y}}} \\
               & = & \sum_{L_i \in L}{\Prob{H}{y_{p, L_i}, \hat y_{p, L_i}}}\\
               & = & -\sum_{L_i \in L}{y_{p, L_i}\log{\hat y_{p, L_i}}}\\
               & = & -y_{p,L_1}\log(\hat y_{p,L_1}) - (1 - y_{p,L_1})\log(1 - \hat y_{p,L_1})
   \end{eqnarray*}

\section{Multiple Points}
  Given a set of $t$ points
  represented as a 2 dimensional tensor $T \in \R^{t\times d}$,
  the perceptron $c$ can then be \emph{applied}
  to all the points in $T$ to get
  \[ \hat y_{T,c} = \map{T \cdot w_c + b_c\overrightarrow{u_t}}{\sigmoid \cdot} \]
  where
  \begin{itemize}
    \item $\map{\overrightarrow x}{f}$ is
            the element-wise application of $f$ on $\overrightarrow x$
    \item $\overrightarrow{u_n} = [1, 1, 1, ... n \text{ times}]$
  \end{itemize}
  The $i^{th}$ element of $\hat y_{T, c}$ then gives us
  the probability of the $i^{th}$ point being in $L_1$.
  \[ \hat y_{T, c}[i] = \Prob{\hat P}{T[i] \in L_1} = \hat y_{T[i], c}\]
  And therefore, if the actual classification
  of all training points is given by
  \[ y_{T,c} = \map{T}{\Prob{I}{\cdot \in L_1}} \]
  then the training criterion is to minimize the cross entropy
  \begin{eqnarray*}
    \mathbb{H} & = & \sum_{i=1}^t \Prob{H}{y_{T, c}[i], \hat y_{T, c}[i]} \\
               & = & \text{sum}(\text{map}(y_{T, c},
                                            \hat y_{T, c},
                                            \Prob{H}{\cdot, \cdot}
                                           )
                               )
  \end{eqnarray*}
  where
  \begin{itemize}
    \item $\text{sum}(x)$ gives the sum of all the elements
          in the tensor $x$
    \item $\text{map}(x, y, f(\cdot, \cdot))$
          is the element-wise application of
          the binary function $f$ to the elements of $x$ and $y$.
          In other words,
          \[ \text{map}(x, y, f(\cdot, \cdot))[i] = f(x[i], y[i]) \]
  \end{itemize}

\section{Multiple Labels}
  Classifying a point across multiple labels may be achieved
  using multiple perceptrons.
  However, if we simply use the multiple perceptrons independently,
  we end up with unconstrained probabilities
  for a point belonging to each label.
  Instead, we would ideally want the probabilities to add up to 1,
  since a point can only belong to one label.

  Suppose we have $l$ labels, and corresponding $l$ perceptrons.
  Each perceptron produces an output $x_i \in \R^t$,
  where $t$ is the number of points.
  Then the probability of point $T_j$
  belonging to label $L_k$
  is
  \[ \hat y_{T_j, L_k} =
        \frac{e^{x_k[j]}}
             {\sum_{m = 1}^{l} e^{x_m[j]} }
  \]

  To simplify this notation, we introduce the softmax function:
  \begin{eqnarray*}
    \text{softmax}(\overrightarrow z)
      & = & \text{softmax}(z_1, z_2, ... z_n)
    \\
      & = & \left[ \frac{e^{z_1}}{\sum_{i=1}^n e^{z_i}},
                   \frac{e^{z_2}}{\sum_{i=1}^n e^{z_i}},
                   ...,
                   \frac{e^{z_n}}{\sum_{i=1}^n e^{z_i}}
            \right]
    \\
    \text{softmax}(\overrightarrow z; j)
      & = & \frac{e^{z_j}}{\sum_{i=1}^n e^{z_i}}
  \end{eqnarray*}

  And concisely represent
  all the probabilities as:
  \[\hat y_{T, L} = \text{map}(x_1, x_2, ..., x_l, \text{softmax})\]

  Therefore, the training criteria is to minimize the cross-entropy:
  \[ \mathbb{H}(\hat y_{T, L}, y_{T, L})
       = \sum_{T_i \in T, L_j \in L} \mathbb{H}(\hat y_{T_i, L_j},
                                                y_{T_i, L_j})
  \]

\section{Multiple Layers of Perceptrons}
  Multiple layers of perceptrons,
  with non-linear activation functions, are necessary
  for classifying data that is not linearly separable.
  An activation function is applied to the output of a neuron
  before feeding it to the next layer.

\section{Sample 2-Layer Perceptron}
  Let the input data be organized
  into a 2-dimensional structure
  $T \in \R^{t \times d}$,
  where $t$ is the number of points
  and $d$ is the dimensionality of each point.

  We feed these points to the first layer
  consisting of, say, $m$ perceptrons.
  Thus, we have $m$ weight vectors:
  \[ w_{1, 1}, w_{1, 2}, ..., w_{1, m} \in \R^d \]
  and $m$ biases:
  \[ b_{1, 1}, b_{1, 2}, ..., b_{1, m} \in \R \]
  to give the unactivated output
  $p_{1, i} \in \R^{t}$
  of the $i^{th}$ perceptron:
  \[ p_{1, i} = T \cdot w_{1, i} + b_{1, i}\overrightarrow u_{t} \]

  These unactivated outputs $p_{1, \cdot}$ are passed
  through a sigmoid activation function
  to get the final activated outputs
  $z_{1, \cdot} \in R^{t}$
  of this layer:
  \[ z_{1, i} = \text{map}(p_{1, i}, \sigmoid{\cdot})
  \]

  These outputs $z_{1, \cdot}$ then become
  the inputs for the next and final layer.
  For convenience, we define
  $z_1 \in \R^{t \times m}$
  as the output vector of the first layer,
  created by concatenating all the individual outputs.

  Since se have $l$ labels for classification,
  the final layer should have $l$ perceptrons.
  Thus, we have $l$ weight vectors:
  \[ w_{2, 1}, w_{2, 2}, ..., w_{2, l} \in R^m \]
  and $l$ biases:
  \[ b_{2, 1}, b_{2, 2}, ..., b_{2, l} \in R \]
  to give the unactivated outputs $p_{2, \cdot} \in \R^t$:
  \[ p_{2, i} = z_1 \cdot w_{2, i} + b_{2, i} \overrightarrow u_{t} \]

  We leave the outputs unactivated for this final layer,
  and concatenate the outputs to get $p_2 \in {R^{t \times l}}$
  Thus, the probabilities $\hat y \in \R^{t \times l}$
  of the points belonging to classes is given by:
  \[ \hat y = \text{map}(p_2, \text{softmax}(\cdot)) \]

  If the real probabilities are given by $y \in R^{t \times l}$
  then the training criterion is to minimize the cross entropy:
  \[
    \mathbb{H}
    = \Prob{H}{\hat y, y}
    = \sum_{i \in T, j \in L} \Prob{H}{\hat y_{i, j}, y_{i, j}}
  \]

  \tikzstyle{io} = [circle,
                    text centered,
                    draw=black]
  \tikzstyle{perceptron} = [rectangle,
                            text centered,
                            draw=black,
                            minimum width=1.5cm,
                            minimum height=.8cm,
                            fill=blue!20]
  \tikzstyle{weight} = [rectangle,
                        text centered,
                        draw=black,
                        fill=yellow!30,
                        rounded corners]
  \tikzstyle{arrow} = [thick,->,>=stealth]

  \begin{tikzpicture}[node distance=2cm]
  \node (input) [io, fill=green!20] {$T$};
  \node (inputlabel) [above of=input] {Input};

  \node (p12) [perceptron, right=1cm of input] {$z_{12}$};
  \node (p11) [perceptron, above=2cm of p12] {$z_{11}$};
  \node (p13) [perceptron, below=2cm of p12] {$z_{13}$};
  \node (w11) [weight, below=.5cm of p11] {$w_{11}, b_{11}$};
  \node (w12) [weight, below=.5cm of p12] {$w_{12}, b_{12}$};
  \node (w13) [weight, below=.5cm of p13] {$w_{13}, b_{13}$};
  \node (p1label) [above=0.2cm of p11] {Layer 1};

  \node (p22) [perceptron, right=1cm of p12] {$p_{22}$};
  \node (p21) [perceptron, above=2cm of p22] {$p_{21}$};
  \node (p23) [perceptron, below=2cm of p22] {$p_{23}$};
  \node (w21) [weight, below=.5cm of p21] {$w_{21}, b_{21}$};
  \node (w22) [weight, below=.5cm of p22] {$w_{22}, b_{22}$};
  \node (w23) [weight, below=.5cm of p23] {$w_{23}, b_{23}$};
  \node (p2label) [above=0.2cm of p21] {Layer 2};

  \node (output) [perceptron, right=1cm of p22] {$\hat y$};
  \node (softmaxlabel) [above of=output] {Prediction};

  \node (error) [io, right=1cm of output, fill=red!20] {$\mathbb{H}$};
  \node (errorlabel) [above of=error] {Error};
  \node (y) [io, below=.5cm of error, fill=green!20] {$y$};

  \draw [arrow] (input) -- (p11.west);
  \draw [arrow] (input) -- (p12.west);
  \draw [arrow] (input) -- (p13.west);

  \draw [arrow] (p11.east) -- (p21.west);
  \draw [arrow] (p11.east) -- (p22.west);
  \draw [arrow] (p11.east) -- (p23.west);

  \draw [arrow] (p12.east) -- (p21.west);
  \draw [arrow] (p12.east) -- (p22.west);
  \draw [arrow] (p12.east) -- (p23.west);

  \draw [arrow] (p13.east) -- (p21.west);
  \draw [arrow] (p13.east) -- (p22.west);
  \draw [arrow] (p13.east) -- (p23.west);

  \draw [arrow] (p21.east) -- (output);
  \draw [arrow] (p22.east) -- (output);
  \draw [arrow] (p23.east) -- (output);

  \draw [arrow] (w11) -- (p11);
  \draw [arrow] (w12) -- (p12);
  \draw [arrow] (w13) -- (p13);
  \draw [arrow] (w21) -- (p21);
  \draw [arrow] (w22) -- (p22);
  \draw [arrow] (w23) -- (p23);

  \draw [arrow] (output) -- (error);
  \draw [arrow] (y) -- (error);
  \end{tikzpicture}

\end{document}


\subsection{Geometric description of the cross product}

The following is the geometric description of the cross
product. Recall that the dot product of two vectors results in a
scalar. In contrast, the cross product results in a vector, as the
cross product gives a direction as well as a magnitude.%
\index{cross product!geometric description}

\begin{definition}{Geometric definition of cross product}{cross-product-geometric}
  Let\/ $\vect{u}$ and $\vect{v}$ be two vectors in $\R^3$. Their
  \textbf{cross product}%
  \index{cross product}, written $\vect{u}\times \vect{v}$, is the
  vector defined by the following three rules.%
  \index{cross product!geometric description}

  \begin{enumerate}
  \item Its length is $\norm{\vect{u}\times \vect{v}} =\norm{\vect{u}} \norm{
      \vect{v}} \sin \theta$,
    where $\theta$ is the included angle between $\vect{u}$ and $\vect{v}$.

  \item It is orthogonal to both $\vect{u}$ and $\vect{v}$.

  \item The vectors $\vect{u}$, $\vect{v}$, and $\vect{u}\times
    \vect{v}$, in that order, form a right-handed system.
  \end{enumerate}
\end{definition}

We note that the length of the cross product,
$\norm{\vect{u}\times\vect{v}}$, given by the formula
$\norm{\vect{u}}\norm{\vect{v}}\sin\theta$, is the area of the
parallelogram determined by $\vect{u}$ and $\vect{v}$, as shown in the
following picture.%
\index{cross product!area of parallelogram}%
\index{area!of parallelogram}%
\index{parallelogram!area of}%
\begin{center}
  \begin{tikzpicture}[scale=1.3]
    \filldraw[fill=green!20,draw=green!50!black] (0,0) -- (0:8mm) arc (0:45:8mm) -- cycle;
    \draw[->,thick,red] (0,0) -- node[left] {$\vect{v}$} (2,2);
    \draw[->,thick,blue] (0,0) -- node[below] {$\vect{u}$} (3,0);
    \draw[thick] (2,2)--(5,2);
    \draw[thick] (3,0)--(5,2);
    \draw[dashed] (2,2)--(2,0);
    \node[green!30!black] at (22.5:5.5mm) {$\theta$};
    \node[right] at (2,1){$\norm{\vect{v}} \sin(\theta)$};
  \end{tikzpicture}
\end{center}


\section{Function extensionality}
\label{chap:funext}

\subsection{Equivalent forms of function extensionality}

\begin{axiom}[Function Extensionality]\label{axiom:funext}
  \index{function extensionality}\index{identity type!of a Pi-type@{of a $\Pi$-type}}\index{extensionality principle!for functions}%
  For any type family $B$ over $A$, and any two dependent functions $f,g:\prd{x:A}B(x)$, the canonical map\index{htpy-eq@{$\htpyeq$}}\index{htpy-eq@{$\htpyeq$}!is an equivalence}\index{is an equivalence!htpy-eq@{$\htpyeq$}}
  \begin{equation*}
    \htpyeq:(f=g)\to (f\htpy g)
  \end{equation*}
  that sends $\refl{f}$ to $\reflhtpy_{f}$ is an equivalence. We will write $\eqhtpy$\index{eq-htpy@{$\eqhtpy$}} for its inverse.
\end{axiom}

In other words, the axiom of function extensionality asserts that for any two dependent functions $f,g:\prd{x:A}B(x)$, the type of identifications $f=g$ is equivalent to the type of homotopies $f\htpy g$ from $f$ to $g$. By the fundamental theorem of identity types (\cref{thm:id_fundamental})\index{fundamental theorem of identity types} there are three equivalent ways of asserting function extensionality. In the following theorem we state one further equivalent condition.

\begin{thm}\label{thm:funext_wkfunext}
The following are equivalent:\index{function extensionality}
\begin{enumerate}
\item The axiom of function extensionality.
\item For any type family $B$ over $A$ and any dependent function $f:\prd{x:A}B(x)$, the total space
  \begin{equation*}
    \sm{g:\prd{x:A}B(x)}f\htpy g
  \end{equation*}
  is contractible.
\item
  The principle of \define{homotopy induction}\index{homotopy induction}\index{induction principle!for homotopies}:
  for any type family $B$ over $A$, any dependent function $f:\prd{x:A}B(x)$, and any family of types $P(g,H)$ indexed by $g:\prd{x:A}B(x)$ and $H:f\htpy g$, the evaluation function
\begin{equation*}
\Big(\prd{g:\prd{x:A}B(x)}{H:f\htpy g}P(g,H)\Big)\to P(f,\reflhtpy_f)
\end{equation*}
given by $s\mapsto s(f,\reflhtpy_f)$ has a section.
\item The \define{weak function extensionality principle}\index{weak function extensionality} holds: For every type family $B$ over $A$ one has\index{contractible!weak function extensionality}
\begin{equation*}
\Big(\prd{x:A}\iscontr(B(x))\Big)\to\iscontr\Big(\prd{x:A}B(x)\Big).
\end{equation*}
\end{enumerate}
\end{thm}

\begin{proof}
The fact that function extensionality is equivalent to (ii) and (iii) follows directly from \cref{thm:id_fundamental}.

To show that function extensionality implies weak function extensionality, suppose that each $B(a)$ is contractible with center of contraction $c(a)$ and contraction $C_a:\prd{y:B(a)}c(a)=y$. Then we take $c\defeq \lam{a}c(a)$ to be the center of contraction of $\prd{x:A}B(x)$. To construct the contraction we have to define a term of type
\begin{equation*}
\prd{f:\prd{x:A}B(x)}c=f.
\end{equation*}
Let $f:\prd{x:A}B(x)$. By function extensionality we have a map $(c\htpy f)\to (c=f)$, so it suffices to construct a term of type $c\htpy f$. Here we take $\lam{a}C_a(f(a))$. This completes the proof that function extensionality implies weak function extensionality.

In the remaining part of the proof, we will show that weak function extensionality implies that the type
\begin{equation*}
\sm{g:\prd{x:A}B(x)}f\htpy g
\end{equation*}
is contractible for any $f:\prd{x:A}B(x)$. In order to do this, we first note that we have a section-retraction pair
\begin{equation*}
  \begin{tikzcd}[column sep=small]
    \Big(\sm{g:\prd{x:A}B(x)}f\htpy g\Big) \arrow[r,"i"] & \Big(\prd{x:A}\sm{b:B(x)}f(x)=b\Big) \arrow[r,"r"] & \Big(\sm{g:\prd{x:A}B(x)}f\htpy g\Big).
  \end{tikzcd}
\end{equation*}
Here we have the functions
\begin{align*}
i & \defeq \lam{(g,H)}{x}(g(x),H(x)) \\
r & \defeq \lam{p}\pairr{\lam{x}\proj 1(p(x)),\lam{x}\proj 2(p(x))}.
\end{align*}
Their composite is homotopic to the identity function by the computation rule for $\Sigma$-types and the $\eta$-rule for $\Pi$-types:
\begin{align*}
r(i(g,H)) & \jdeq r(\lam{x}\pairr{g(x),H(x)}) \\
& \jdeq \pairr{\lam{x}g(x),\lam{x}H(x)} \\
& \jdeq \pairr{g,H}.
\end{align*}
Now we observe that the type $\prd{x:A}\sm{b:B(x)}f(x)=b$ is a product of contractible types, so it is contractible by our assumption of the weak function extensionality principle. The claim therefore follows, since retracts of contractible types are contractible by \cref{ex:contr_retr}\index{contractible!retract of}.
\end{proof}

For the remainder of this chapter we will assume that the function extensionality axiom holds\index{axiom!function extensionality}. In \cref{thm:funext-univalence} we will derive function extensionality from the univalence axiom.

As a first application of the function extensionality axiom we generalize the weak function extensionality axiom to $k$-types.

\begin{thm}\label{thm:trunc_pi}\index{k-type@{$k$-type}}
For any type family $B$ over $A$ one has\index{truncated type!closed under Pi@{closed under $\Pi$}}
\begin{equation*}
\Big(\prd{x:A}\istrunc{k}(B(x))\Big)\to \istrunc{k}\Big(\prd{x:A}B(x)\Big).
\end{equation*}
\end{thm}

\begin{proof}
The theorem is proven by induction on $k\geq -2$. The base case is just the weak function extensionality principle\index{weak function extensionality}, which was shown to follow from function extensionality in \cref{thm:funext_wkfunext}.

For the inductive hypothesis, assume that the $k$-types are closed under dependent function types. Assume that $B$ is a family of $(k+1)$-types. By function extensionality, the type $f=g$ is equivalent to $f\htpy g$ for any two dependent functions $f,g:\prd{x:A}B(x)$. Now observe that $f\htpy g$ is a dependent product of $k$-types, and therefore it is an $k$-type by our inductive hypotheses. Therefore, it follows by \cref{thm:ktype_eqv} that $f=g$ is an $k$-type, and hence that $\prd{x:A}B(x)$ is an $(k+1)$-type.
\end{proof}

\begin{cor}\label{cor:funtype_trunc}\index{truncated type!closed under exponentials}
Suppose $B$ is a $k$-type. Then $A\to B$ is also a $k$-type, for any type $A$.
\end{cor}

\subsection{The type theoretic principle of choice}

The type theoretic principle of choice\index{type theoretic choice} asserts that $\Pi$ distributes over $\Sigma$\index{distributivity!of Pi over Sigma@{of $\Pi$ over $\Sigma$}}. More precisely, it asserts that the canonical map\index{choice@{$\choice$}}
\begin{equation*}
  \choice:\Big(\prd{x:A}\sm{y:B(x)}C(x,y)\Big)\to \Big(\sm{f:\prd{x:A}B(x)}\prd{x:A}C(x,f(x))\Big)
\end{equation*}
given by $\lam{h}(\proj 1(h(x)),\proj 2(h(x)))$, is an equivalence. In order to see this as a principle of choice, one can view the left hand side as the type of functions $h$ that pick for every $x:A$ a term $y:B(x)$ equipped with a term of type $C(x,y)$. The function $\choice$ then constructs a dependent function $f:\prd{x:A}B(x)$ equipped with a term of type $\prd{x:A}C(x,f(x))$. In this section we show that the map $\choice$ is an equivalence, and we use this to characterize the identity of any dependent function type $\prd{x:A}B(x)$ in terms of any characterization of the identity types of the individual types $B(x)$.

\begin{thm}\label{thm:choice}
Consider a family of types $C(x,y)$ indexed by $x:A$ and $y:B(x)$. Then the map
\begin{equation*}
  \choice:\Big(\prd{x:A}\sm{y:B(x)}C(x,y)\Big)\to \Big(\sm{f:\prd{x:A}B(x)}\prd{x:A}C(x,f(x))\Big)
\end{equation*}
given by $\lam{h}(\proj 1(h(x)),\proj 2(h(x)))$ is an equivalence.
\end{thm}

\begin{proof}
  We define the map\index{choice-1@{$\choice^{-1}$}}
  \begin{equation*}
    \choice^{-1}:\Big(\sm{f:\prd{x:A}B(x)}\prd{x:A}C(x,f(x))\Big)\to\Big(\prd{x:A}\sm{y:B(x)}C(x,y)\Big)
  \end{equation*}
  by $\lam{(f,g)}{x}(f(x),g(x))$. Then we have to construct homotopies
  \begin{equation*}
    \choice\circ\choice^{-1}\htpy\idfunc,\qquad\text{and}\qquad
    \choice^{-1}\circ\choice\htpy\idfunc.
  \end{equation*}
  For the first homotopy it suffices to construct an identification
  \begin{equation*}
    \choice(\choice^{-1}(f,g))=(f,g)
  \end{equation*}
  for any $f:\prd{x:A}B(x)$ and any $g:\prd{x:A}C(x,f(x))$. We compute the left-hand side as follows:
  \begin{align*}
    \choice(\choice^{-1}(f,g))
    & \jdeq \choice(\lam{x}(f(x),g(x))) \\
    & \jdeq (\lam{x}f(x),\lam{x}g(x)).
  \end{align*}
  By the $\eta$-rule it follows that $f\jdeq \lam{x}f(x)$ and $g\jdeq\lam{x}g(x)$. Therefore we have the identification
  \begin{equation*}
    \refl{(f,g)}:\choice(\choice^{-1}(f,g))=(f,g).
  \end{equation*}
  This completes the construction of the first homotopy.

  For the second homotopy we have to construct an identification
  \begin{equation*}
    \choice^{-1}(\choice(h))=h
  \end{equation*}
  for any $h:\prd{x:A}\sm{y:B(x)}C(x,y)$. We compute the left-hand side as follows:
  \begin{align*}
    \choice^{-1}(\choice(h))
    & \jdeq \choice^{-1}(\lam{x}\proj 1(h(x)),(\lam{x}\proj 2(h(x)))) \\
    & \jdeq \lam{x}(\proj 1(h(x)),\proj 2(h(x)))
  \end{align*}
  However, it is \emph{not} the case that $(\proj 1(h(x)),\proj 2(h(x)))\jdeq h(x)$ for any $h:\prd{x:A}\sm{y:B(x)}C(x,y)$. Nevertheless, we have the identification
  \begin{equation*}
    \eqpair(\refl{},\refl{}):(\proj 1(h(x)),\proj 2(h(x)))= h(x).
  \end{equation*}
  Therefore we obtain the required homotopy by function extensionality:
  \begin{equation*}
    \lam{h}\eqhtpy(\lam{x}\eqpair(\refl{\proj 1(h(x))},\refl{\proj 2(h(x))})):\choice^{-1}\circ\choice\htpy\idfunc.\qedhere
  \end{equation*}
\end{proof}

\begin{cor}
For type $A$ and any type family $C$ over $B$, the map
\begin{equation*}
\Big(\sm{f:A\to B} \prd{x:A}C(f(x))\Big)\to\Big(A\to\sm{y:B}C(x)\Big)
\end{equation*}
given by $\lam{(f,g)}{x}(f(x),g(x))$ is an equivalence.
\end{cor}

\begin{rmk}
  The type theoretic choice principle can be used to derive the binomial theorem\index{binomial theorem}. We give an informal argument of how this goes. Recall that the binomial theorem asserts that
  \begin{equation*}
    (n+m)^k=\sum_{l=0}^k\binom{k}{l}n^l m^{k-l}
  \end{equation*}
  for any three natural numbers $k,m,n$.

  Consider the types $A\defeq\Fin(k)$\index{Fin@{$\Fin$}}, $B\defeq\Fin(n)$ and $C\defeq\Fin(m)$. Then we can define the type family $P:\bool\to\UU$ given by
  \begin{align*}
    P(\btrue) & \defeq B \\
    P(\bfalse) & \defeq C.
  \end{align*}
  Now, the type theoretic principle of choice gives us an equivalence
  \begin{equation*}
    \Big(\prd{x:A}\sm{t:\bool}P(t)\Big)\simeq \Big(\sm{f:A\to\bool}\prd{x:A}P(f(x))\Big).
  \end{equation*}
  Now we note that the type $(f(x)=1)+(f(x)=0)$ is contractible for any $f:A\to\bool$ and $x:A$. Therefore we have equivalences
  \begin{align*}
    \sm{f:A\to\bool}\prd{x:A}P(f(x)) & \simeq
    \sm{f:A\to\bool}\prd{x:A}{t:(f(x)=1)+(f(x)=0)}P(f(x)) \\
    & \simeq \sm{f:A\to\bool}(\fib{f}{1}\to B)\times (\fib{f}{0}\to C)
  \end{align*}
  Now we note that, because there are $\binom{k}{l}$\index{binomial coefficient} ways to choose a subset of $l$ elements of $A$, there are
  \begin{equation*}
    \sum_{l=0}^k\binom{k}{l}n^l m^{k-l}
  \end{equation*}
  elements in the above type.
\end{rmk}

\subsection{Universal properties}
The function extensionality principle allows us to prove \emph{universal properties}. Universal properties are characterizations of all maps out of or into a given type, so they are very important. Among other applications, universal properties characterize a type up to equivalence. In the following theorem we prove the universal property of dependent pair types.

\begin{thm}\index{universal property!Sigma-types@{$\Sigma$-types}}\index{Sigma-type@{$\Sigma$-type}!universal property}
Let $B$ be a type family over $A$, and let $X$ be a type. Then the map\index{ev-pair@{$\evpair$}}
\begin{equation*}
\evpair:\Big(\Big(\sm{x:A}B(x)\Big)\to X\Big)\to \Big(\prd{x:A}(B(x)\to X)\Big)
\end{equation*}
given by $f\mapsto\lam{a}{b}f(a,b)$ is an equivalence.
\end{thm}

\begin{proof}
The map in the converse direction is simply
\begin{equation*}
\ind{\Sigma} : \Big(\prd{x:A}(B(x)\to X)\Big)\to \Big(\Big(\sm{x:A}B(x)\Big)\to X\Big).
\end{equation*}
By the computation rules for $\Sigma$-types we have
\begin{equation*}
\lam{f}\refl{f}:\evpair\circ\ind{\Sigma}\htpy\idfunc
\end{equation*}

To show that $\ind{\Sigma}\circ\evpair\htpy\idfunc$ we will also apply function extensionality. Thus, it suffices to show that $\ind{\Sigma}(\lam{x}{y}f((x,y)))=f$. We apply function extensionality again, so it suffices to show that
\begin{equation*}
\prd{t:\sm{x:A}B(x)}\ind{\Sigma}\big(\lam{x}{y}f((x,y))\big)(t)=f(t).
\end{equation*}
We obtain this homotopy by another application of $\Sigma$-induction.
\end{proof}

\begin{cor}\label{cor:times_up_out}\index{universal property!cartesian product}\index{cartesian product!universal property}
Let $A$, $B$, and $X$ be types. Then the map\index{ev-pair@{$\evpair$}}
\begin{equation*}
\evpair: (A\times B \to X)\to (A\to (B\to X))
\end{equation*}
given by $f\mapsto\lam{a}{b}f((a,b))$ is an equivalence.
\end{cor}

The universal property of identity types is sometimes called the \emph{type theoretical Yoneda lemma}\index{Yoneda lemma (type theoretical)}: families of maps out of the identity type are uniquely determined by their action on the reflexivity identification.

\begin{thm}\label{thm:yoneda}\index{universal property!identity type}\index{identity type!universal property}
Let $B$ be a type family over $A$, and let $a:A$. Then the map\index{ev-refl@{$\evrefl$}}
\begin{equation*}
\evrefl:\Big(\prd{x:A} (a=x)\to B(x)\Big)\to B(a)
\end{equation*}
given by $\lam{f} f(a,\refl{a})$ is an equivalence.
\end{thm}

\begin{proof}
The inverse $\varphi$ is defined by path induction, taking $b:B(a)$ to the function $f$ satisfying $f(a,\refl{a})\jdeq b$. It is immediate that $\evrefl\circ\varphi\htpy \idfunc$.

To see that $\varphi\circ \evrefl\htpy\idfunc$, let $f:\prd{x:A}(a=x)\to B(x)$. To show that $\varphi(f(a,\refl{a}))=f$ we use function extensionality (twice), so it suffices to show that
\begin{equation*}
\prd{x:A}{p:a=x} \varphi(f(a,\refl{a}),x,p)=f(x,p).
\end{equation*}
This follows by path induction on $p$, since $\varphi(f(a,\refl{a}),a,\refl{a})\jdeq f(a,\refl{a})$.
\end{proof}

\subsection{Composing with equivalences}

We show in this section that a map $f:A\to B$ is an equivalence if and only if for any type $X$ the precomposition map\index{precomposition map}
\begin{equation*}
\blank\circ f: (B\to X)\to (A\to X)
\end{equation*}
is an equivalence. Moreover, we will show in \cref{ex:equiv_precomp} that the `dependent version' of this statement also holds: a map $f:A\to B$ is an equivalence if and only if for any type family $P$ over $B$, the precomposition map
\begin{equation*}
\blank\circ f: \Big(\prd{y:B}P(y)\Big)\to\Big(\prd{x:A}P(f(x))\Big)
\end{equation*}
is an equivalence.

\begin{thm}\label{ex:equiv_precomp}\index{equivalence!precomposition}
For any map $f:A\to B$, the following are equivalent:
\begin{enumerate}
\item $f$ is an equivalence.
\item For any type family $P$ over $B$ the map
\begin{equation*}
\Big(\prd{y:B}P(y)\Big)\to\Big(\prd{x:A}P(f(x))\Big)
\end{equation*}
given by $h\mapsto h\circ f$ is an equivalence.
\item For any type $X$ the map
\begin{equation*}
(B\to X)\to (A\to X)
\end{equation*}
given by $g\mapsto g\circ f$ is an equivalence.
\end{enumerate}
\end{thm}

\begin{proof}
To show that (i) implies (ii), we first recall from \cref{lem:coherently-invertible} that any equivalence is also coherently invertible\index{coherently invertible}. Therefore $f$ comes equipped with
\begin{align*}
g & : B \to A\\
G & : f\circ g \htpy \idfunc[B] \\
H & : g\circ f \htpy \idfunc[A] \\
K & : G\cdot f \htpy f\cdot H.
\end{align*}
Then we define the inverse of $\blank\circ f$ to be the map
\begin{equation*}
\varphi:\Big(\prd{x:A}P(f(x))\Big)\to\Big(\prd{y:B}P(y)\Big)
\end{equation*}
given by $h\mapsto \lam{y}\tr_P(G(y),h(g(y)))$.

To see that $\varphi$ is a section of $\blank\circ f$, let $h:\prd{x:A}P(f(x))$. By function extensionality it suffices to construct a homotopy $\varphi(h)\circ f\htpy h$. In other words, we have to show that
\begin{equation*}
\tr_P(G(f(x)),h(g(f(x)))=h(x)
\end{equation*}
for any $x:A$. Now we use the additional homotopy $K$ from our assumption that $f$ is coherently invertible. Since we have $K(x):G(f(x))=\ap{f}{H(x)}$ it suffices to show that
\begin{equation*}
\tr_P(\ap{f}{H(x)},hgf(x))=h(x).
\end{equation*}
A simple path-induction argument yields that
\begin{equation*}
\tr_P(\ap{f}{p})\htpy \tr_{P\circ f}(p)
\end{equation*}
for any path $p:x=y$ in $A$, so it suffices to construct an identification
\begin{equation*}
\tr_{P\circ f}(H(x),hgf(x))=h(x).
\end{equation*}
We have such an identification by $\apd{h}{H(x)}$.

To see that $\varphi$ is a retraction of $\blank\circ f$, let $h:\prd{y:B}P(y)$. By function extensionality it suffices to construct a homotopy $\varphi(h\circ f)\htpy h$. In other words, we have to show that
\begin{equation*}
\tr_P(G(y),hfg(y))=h(y)
\end{equation*}
for any $y:B$. We have such an identification by $\apd{h}{G(y)}$. This completes the proof that (i) implies (ii).

Note that (iii) is an immediate consequence of (ii), since we can just choose $P$ to be the constant family $X$.

It remains to show that (iii) implies (i). Suppose that
\begin{equation*}
\blank\circ f:(B\to X)\to (A\to X)
\end{equation*}
is an equivalence for every type $X$. Then its fibers are contractible by \cref{thm:contr_equiv}. In particular, choosing $X\jdeq A$ we see that the fiber
\begin{equation*}
\fib{\blank\circ f}{\idfunc[A]}\jdeq \sm{h:B\to A}h\circ f=\idfunc[A]
\end{equation*}
is contractible. Thus we obtain a function $h:B\to A$ and a homotopy $H:h\circ f\htpy\idfunc[A]$ showing that $h$ is a retraction of $f$. We will show that $h$ is also a section of $f$. To see this, we use that the fiber
\begin{equation*}
\fib{\blank\circ f}{f}\jdeq \sm{i:B\to B} i\circ f=f
\end{equation*}
is contractible (choosing $X\jdeq B$).
Of course we have $(\idfunc[B],\refl{f})$ in this fiber. However we claim that there also is an identification $p:(f\circ h)\circ f=f$, showing that $(f\circ h,p)$ is in this fiber, because
\begin{align*}
(f\circ h)\circ f & \jdeq f\circ (h\circ f) \\
& = f\circ \idfunc[A] \\
& \jdeq f
\end{align*}
Now we conclude by the contractibility of the fiber that $(\idfunc[B],\refl{f})=(f\circ h,p)$. In particular we obtain that $\idfunc[B]=f\circ h$, showing that $h$ is a section of $f$.
\end{proof}

\begin{exercises}
\exercise Show that the functions\index{inv-htpy@{$\invhtpy$}!is an equivalence}\index{concat-htpy@{$\concathtpy$}!is a family of equivalences}\index{concat-htpy'@{$\concathtpy'$}!is a family of equivalences}\index{is an equivalence!inv-htpy@{$\invhtpy$}}\index{is an equivalence!concat-htpy(H)@{$\concathtpy(H)$}}\index{is an equivalence!concat-htpy'(K)@{$\concathtpy'(K)$}}
\begin{align*}
\invhtpy & : (f \htpy g) \to (g \htpy f) \\
\concathtpy(H) & : (g \htpy h) \to (f \htpy h) \\
\concathtpy'(K) & : (f \htpy g) \to (f \htpy h)
\end{align*}
are equivalences for every $f,g,h : \prd{x:A}B(x)$. Here, $\concathtpy'(K)$ is the function defined by $H\mapsto \ct{H}{K}$.
\exercise \label{ex:isprop_istrunc}
\begin{subexenum}
\item Show that for any type $A$ the type $\iscontr(A)$ is a proposition\index{is-cont(A)r@{$\iscontr(A)$}!is a proposition}\index{is contractible!is a property}. %There's an easy proof using double singleton induction. This is a nice application of weak funext.
\item Show that for any type $A$ and any $k\geq-2$, the type $\istrunc{k}(A)$ is a proposition.\index{istrunc@{$\istrunc{k}$}!is a proposition}
\end{subexenum}
\exercise \label{lem:postcomp_equiv}
Let $f:X\to Y$ be a map. Show that the following are equivalent:
\begin{enumerate}
\item $f$ is an equivalence.\index{equivalence!postcomposition}
\item The map $f\circ\blank : X^A\to Y^A$ is an equivalence for every type $A$.
\end{enumerate}
\exercise \label{ex:isprop_isequiv}Let $f:A\to B$ be a function.
\begin{subexenum}
\item Show that if $f$ is an equivalence, then the type $\sm{g:B\to A}f\circ g\htpy \idfunc$ of sections of $f$ is contractible.
\item Show that if $f$ is an equivalence, then the type $\sm{h:B\to A}h\circ f\htpy \idfunc$ of retractions of $f$ is contractible.
\item Show that $\isequiv(f)$ is a proposition.\index{is-equiv(f)@{$\isequiv(f)$}!is a proposition}
\item Use \cref{ex:prop_equiv,ex:isprop_istrunc} to show that $\isequiv(f)\eqvsym \iscontr(f)$.\index{is-equiv(f)@{$\isequiv(f)$}!is-equiv(f) is-contr(f)@{$\isequiv(f)\eqvsym\iscontr(f)$}}
\end{subexenum}
Conclude that $\eqv{A}{B}$ is a subtype of $A\to B$, and in particular that the map $\proj 1 : (\eqv{A}{B})\to (A\to B)$ is an embedding.
\exercise \label{ex:prop_equiv}
\begin{subexenum}
\item \label{ex:equiv-bi-implication}Let $P$ and $Q$ be propositions. Show that\index{bi-implication}
\begin{equation*}
\eqv{(P\leftrightarrow Q)}{(\eqv{P}{Q})}.
\end{equation*}
\item Show that $P$ is a proposition if and only if $P\to P$ is contractible.
\end{subexenum}
\exercise Show that $\pathsplit(f)$\index{path-split!is a proposition} and $\iscohinvertible(f)$\index{coherently invertible!is a proposition} are propositions for any map $f:A\to B$. Conclude that we have equivalences\index{is-equiv(f)@{$\isequiv(f)$}!is-equiv(f) path-split(f)@{$\isequiv(f)\eqvsym\pathsplit(f)$}}\index{is-equiv(f)@{$\isequiv(f)$}!is-equiv(f) is-coh-invertible(f)@{$\isequiv(f)\eqvsym\iscohinvertible(f)$}}
  \begin{equation*}
    \isequiv(f) \eqvsym \pathsplit(f) \eqvsym \iscohinvertible(f).
  \end{equation*}
%\item Let $B$ and $C$ be type families over $A$, suppose that $p:\id{a}{a'}$ in $A$, and consider two functions $f:B(a)\to C(a)$ and $g:B(a')\to C(a')$.
%\begin{subexenum}
%\item Show that the square
%\begin{equation*}
%\begin{tikzcd}
%B(a) \arrow[r,"f"] \arrow[d,swap,"\tr_B(p)"] & C(a) \arrow[d,"\tr_C(p)"] \\
%B(a') \arrow[r,swap,"g"] & C(a')
%\end{tikzcd}
%\end{equation*}
%commutes for every homotopy $H:\tr_{B(x)\to C(x)}(p,f)\htpy g$. In other words, construct a function of type
%\begin{equation*}
%\Big(\tr_{B(x)\to C(x)}(p,f)\htpy g\Big)\to \Big(\tr_C(p)\circ f\htpy g\circ \tr_B(p)\Big)
%\end{equation*}
%\item Show that this map is an equivalence.
%\end{subexenum}
\exercise \label{ex:idfunc_autohtpy}Construct for any type $A$ an equivalence\index{has-inverse(f)@{$\hasinverse(f)$}!has-inverse(id) id htpy id@{$\hasinverse(\idfunc)\simeq (\idfunc\htpy\idfunc)$}}
\begin{equation*}
\eqv{\hasinverse(\idfunc[A])}{\Big(\idfunc[A]\htpy\idfunc[A]\Big)}.
\end{equation*}
Note: We will use this fact in \cref{ex:is_invertible_id_S1} to show that there
are types for which $\isinvertible(\idfunc[A])\not\eqvsym\isequiv(\idfunc[A])$.
\exercise
\begin{subexenum}
\item Show that the type\index{universal property!empty type}\index{empty type!universal property}
\begin{equation*}
\prd{t:\emptyt}P(t)
\end{equation*}
is contractible for any $P:\emptyt\to \UU$.
\item Show that for any type $X$ the following are equivalent:
  \begin{enumerate}
  \item the unique map $\emptyt \to X$ is an equivalence.
  \item The type $Y^X$ is contractible for any type $Y$.
  \end{enumerate}
\end{subexenum}
\exercise Consider two types $A$ and $B$.\index{universal property!coproduct}\index{coproduct!universal property}
\begin{subexenum}
\item Show that the map\index{ev-inl-inr@{$\evinlinr$}}
\begin{equation*}
  \evinlinr: \Big(\prd{t:A+B}P(t)\Big) \to
  \Big(\prd{x:A}P(\inl(x))\Big)\times\Big(\prd{y:B}P(\inr(y))\Big)
\end{equation*}
given by $f\mapsto (f\circ\inl,f\circ\inr)$ is an equivalence.
\item Show that the following are equivalent for any type $X$ equipped with maps $i:A\to X$ and $j:B\to X$:
  \begin{enumerate}
  \item The map $\indcoprod(i,j) :A+B\to X$ is an equivalence.
  \item For any type $Y$, the map
    \begin{equation*}
      \lam{f}(f\circ i,f\circ j):(X\to Y)\to (A\to Y)\times (B \to Y)
    \end{equation*}
    is an equivalence.
  \end{enumerate}
\end{subexenum}
\exercise
\begin{subexenum}
\item Show that the map\index{universal property!unit type}\index{unit type!universal property}\index{ev-pt@{$\evpt$}}
\begin{equation*}
\Big(\prd{t:\unit}P(t)\Big)\to P(\ttt)
\end{equation*}
given by $\lam{f}f(\ttt)$ is an equivalence.
\item Consider a type $X$ equipped with a point $x:X$. Show that the following are equivalent:
\begin{enumerate}
\item The map $\indunit(x):\unit\to X$ is an equivalence (i.e., $X$ is contractible).
\item For any type $Y$ the map
\begin{equation*}
\lam{f}f(x) : (X\to Y)\to Y
\end{equation*}
is an equivalence.
\end{enumerate}
\end{subexenum}
\exercise \label{ex:sec_retr}Consider a commuting triangle
\begin{equation*}
\begin{tikzcd}[column sep=tiny]
A \arrow[rr,"h"] \arrow[dr,swap,"f"] & & B \arrow[dl,"g"] \\
& X
\end{tikzcd}
\end{equation*}
with $H:f\htpy g\circ h$.
\begin{subexenum}
\item Show that if $h$ has a section, then $\sections(g)$ is a retract of $\sections(f)$.
\item Show that if $g$ has a retraction, then $\retractions(h)$ is a retract of $\sections(f)$.
\end{subexenum}
\exercise \label{ex:equiv_pi} Consider a family $f_i:A_i\to B_i$ of $k$-truncated maps, indexed by $i:I$. Show that the map
  \begin{equation*}
    \lam{h}{i}f_i(h(i)):\Big(\prd{i:I}A_i\Big)\to\Big(\prd{i:I}B_i\Big)
  \end{equation*}
  is again $k$-truncated. Conclude that if each $f_i$ is an equivalence, then so is the above map.
  \exercise Consider a map $f:X\to Y$. Show that the following are equivalent:
  \begin{enumerate}
  \item The map $f$ is $k$-truncated.
  \item For every type $A$, the postcomposition function
    \begin{equation*}
      f\circ\blank : (A\to X)\to (A\to Y)
    \end{equation*}
    is $k$-truncated.
  \end{enumerate}
  In particular it follows that $f$ is an embedding if and only if $f\circ\blank$ is an embedding.

  Hint: Show that the square
  \begin{equation*}
    \begin{tikzcd}[column sep=huge]
      (f=g) \arrow[r,"\apfunc{m}"] \arrow[d,swap,"\htpyeq"] & (m\circ f=m\circ g) \arrow[d,"\htpyeq"] \\
      (f\htpy g) \arrow[r,swap,"H\mapsto m\cdot H"] & (m\circ f\htpy m\circ g)
    \end{tikzcd}
  \end{equation*}
  commutes, and apply \cref{ex:equiv_pi}.
  \exercise \label{ex:pi-fib}Consider a function $f:A\to B$, and let $P$ be a family of types over $B$. Show that the map
  \begin{equation*}
    \Big(\prd{b:B}\fib{f}{b}\to P(b)\Big)\to\Big(\prd{a:A}P(f(a))\Big)
  \end{equation*}
  given by $h\mapsto h_{f(a)}(a,\refl{f(a)})$ is an equivalence.
  \exercise \label{ex:triangle_fib}Consider a diagram of the form
  \begin{equation*}
    \begin{tikzcd}[column sep=tiny]
      A \arrow[dr,swap,"f"] & & B \arrow[dl,"g"] \\
      & X
    \end{tikzcd}
  \end{equation*}
  \begin{subexenum}
  \item Show that the type $\sm{h:A\to B} f\htpy g\circ h$ is equivalent to the type of families of maps
    \begin{equation*}
      \prd{x:X}\fib{f}{x}\to\fib{g}{x}.
    \end{equation*}
  \item Show that the type $\sm{h:\eqv{A}{B}} f\htpy g\circ h$ is equivalent to the type of families of equivalences
    \begin{equation*}
      \prd{x:X}\fib{f}{x}\eqvsym\fib{g}{x}.
    \end{equation*}
  \end{subexenum}
  \exercise \label{ex:sq_fib}Consider a diagram of the form
  \begin{equation*}
    \begin{tikzcd}
      A \arrow[d,swap,"f"] & B \arrow[d,"g"] \\
      X \arrow[r,swap,"h"] & Y.
    \end{tikzcd}
  \end{equation*}
  Show that the type $\sm{i:A\to B}h\circ f\htpy g\circ i$ is equivalent to the type of families of maps
  \begin{equation*}
    \prd{x:X}\fib{f}{x}\to\fib{g}{h(x)}.
  \end{equation*}
  % Note: In \cref{thm:pb_fibequiv_complete} we will characterize the type of families of equivalences $\prd{x:X}\fib{f}{x}\simeq\fib{g}{x}$.
  % \item Show that the type $\sm{i:\eqv{A}{B}}h\circ f\htpy g\circ i$ is equivalent to the type of families of equivalences
  %   \begin{equation*}
  %     \prd{x:X}\fib{f}{x}\eqvsym\fib{g}{h(x)}.
  %   \end{equation*}
  \exercise \label{ex:iso_equiv}Let $A$ and $B$ be sets. Show that type type $\eqv{A}{B}$ of equivalences from $A$ to $B$ is equivalent to the type $A\cong B$ of \define{isomorphisms}\index{isomorphism}\index{set!isomorphism} from $A$ to $B$, i.e., the type of quadruples $(f,g,H,K)$ consisting of
  \begin{align*}
    f & : A\to B \\
    g & : B\to A \\
    H & : f\circ g = \idfunc[B] \\
    K & : g\circ f = \idfunc[A].
  \end{align*}
  \exercise \label{ex:pi_sec}Let $B$ be a type family over $A$, and consider the postcomposition function
  \begin{equation*}
    \proj1 \circ \blank : \Big(\sm{x:A} B(x)\Big)^A \to A^A.
  \end{equation*}
  Construct equivalences
  \begin{equation*}
    \Big(\prd{x:A}B(x)\Big) \eqvsym \sections(\proj 1) \eqvsym \fib{\proj 1 \circ\blank}{\idfunc[A]}.
  \end{equation*}
  \exercise Construct equivalences
  \begin{align*}
    \Fin(n^m) & \simeq (\Fin(m)\to\Fin(n)) \\
    \Fin(n!) & \simeq (\Fin(n)\simeq\Fin(n)).
  \end{align*}
\end{exercises}


\section*{Exercises}

\begin{ex} Find the determinants of the following matrices.
  \begin{equation*}
    (a)~\begin{mymatrix}{rr}
      1 & 3 \\
      0 & 2
    \end{mymatrix}
    \quad
    (b)~\begin{mymatrix}{rr}
      0 & 3 \\
      0 & 2
    \end{mymatrix}
    \quad
    (c)~\begin{mymatrix}{rr}
      4 & 3 \\
      6 & 2
    \end{mymatrix}
    \quad
    (d)~\begin{mymatrix}{rr}
      -3 & 4 \\
      -1 & 2
    \end{mymatrix}
  \end{equation*}
\end{ex}

\begin{ex}
  Find the following determinants.
  \begin{equation*}
    (a)~\begin{absmatrix}{rrr}
      1 & 2 & 3 \\
      4 & 5 & 6 \\
      7 & 8 & 9 \\
    \end{absmatrix}
    \quad
    (b)~\begin{absmatrix}{rrr}
      1  & 0 & 2 \\
      2  & 5 & 3 \\
      -1 & 0 & 0 \\
    \end{absmatrix}
    \quad
    (c)~\begin{absmatrix}{rrr}
      3 &  4 & 1 \\
      0 & -1 & 1 \\
      1 &  2 & 1 \\
    \end{absmatrix}
    \quad
    (d)~\begin{absmatrix}{rrr}
      0  & -2 &  1 \\
      4  & 1  & -3 \\
      -1 & 3  &  1 \\
    \end{absmatrix}
  \end{equation*}
\end{ex}


%% Magic command to compile root document
% !TEX root = ../../thesis.tex

%% Reset glossary to show long gls names
\glsresetall

%% Set path to look for the images
\graphicspath{{./Sections/Basics/Resources/}}

So far we have explained how \glspl{ann} works assuming that we feed them with vectors of fixed length. Even though we could take a multichannel image and transform it into a vector, in practice this would be computationally very expensive. For instance, assuming that we have a 3 channel image of size 224 by 224, this would result into an input vector of length $3 \cdot 224 \cdot 224=150'528$. Then, if the first layer of our network has 100 units, this would mean more than 15 millions of parameters only for the first layer. Furthermore, the transformation of our image into a vector would mean a loss of spatial information. This means that the \gls{ann} would not be able to capture or use the spatial relationship between pixels and shapes within the image.

A \gls{cnn} is a type of \gls{ann} widely used to analyze data in the form of images. The intuition behind a \gls{cnn} is that instead of just looking at an image and trying to predict the target value directly, first learn some \hl{features} within the image, and then make the predict base on this features.
To achieve this, \glspl{cnn} mainly use \hl{convolution} and \hl{pooling} layers.

\subsubsection{Convolution layer}

The only difference a

A convolution layer is very similar to a regular layer described in section \ref{sec:basics:ANN}. Basically, they only differ in the way the layer input is multiplied by the the layer weights.
Recall that in a regular layer, the input of a unit is the dot product between the layer input and its corresponding weight vector (i.e., $z=\bs{w}^T\bs{x}$).
This means that for each element in the input vector $\bs{x}$, there is a corresponding element in the weight vector $\bs{w}$. However, for a convolution layer this is not the case.
Convolution layers are based on the shared-weight architecture of the convolution \hl{kernels} or \hl{filters} that slide along the input and returns a translation known as \hl{feature maps} \cite{zhang1988shift}. This means that the \hl{kernels} weights will be used for multiple elements of the layer input. Figure \ref{fig:basics:conv_layer} shows the convolution process with a 2 by 2 kernel over a RGB image (3 channels) of size 4 by 4. Each entrance of the returned feature map $z_i$ is the dot product between the kernel weights $\bs{w}$ and the $\bs{x}_i-th$ chunk of the image.

\begin{figure}[!ht]
  \centering
  \includegraphics[width=\linewidth]{Diagrams/Conv_layer.png}
  \caption{Convolution process steps. In red, green and blue the input image, in orange the convolution kernel (size 2 by 2 and stride of 1) and in gray the convolution output (feature map).}
  \label{fig:basics:conv_layer}
\end{figure}

Mathematically this looks as follow

\begin{equation}
  z_i = \bs{w}^T \bs{x}_i + b
\end{equation}

where $\bs{w}\in\mathbb{R}^{2 \times 2 \times 3}$, $\bs{x_i}\in\mathbb{R}^{2 \times 2 \times 3}$ and $b\in\mathbb{R}$ is the bias (not shown in the images).

Like the kernel size, the number of pixels we shift the kernel each time along side the input (\hl{Stride}) is also a hyperparameter of convolution layers. IN figure \ref{fig:basics:conv_layer}, the stride size is 1.

Figure \ref{fig:basics:conv_layer} also shows that size (width and height) of the returned feature map is smaller than the input image. If we want to keep the input and output size the same (\hl{Same convolution}), then we must add zeros at the edges of the input features (zero-padding). This is shown on figure \ref{fig:basics:conv_layer_pad}.

\begin{figure}[!ht]
  \centering
  \includegraphics[width=0.7\linewidth]{Diagrams/Conv_layer_pad.png}
  \caption{Convolution with padding. In blue a single-channel input features, in orange the convolution kernel (size 3 by 3 and stride of 1) and in gray the convolution output (feature map).}
  \label{fig:basics:conv_layer_pad}
\end{figure}

So far we have seen that a convolution projects a multi-channel input feature (image) into a single-channel feature map. Therefore, if we want our output feature map to have $n$ channels, then our convolution must have $n$ different kernels.

Normally, a non-linear activation function is applied to the output of convolution layers (and normally also after batch normalization) to enable the \gls{cnn} to learn non-linear relations.

\subsubsection{Pooling layer}

Unlike convolution layers, the goal of Pooling layers is to reduce the feature image (height and width, but not depth) rather than learn features.
However, Pooling layers work in a similar way to convolution layers in the way that they also slide a kernel along the input. However, in this case the kernel works independently on each feature map (that is, each channel) and has no weights to learn.
This means that the pooling layers maintain the same number of input and output channels.
There are several ways to do this downsampling, but the most common are Max Polling and Average Polling. As the name suggests, Average pooling shrinks the feature image by averaging sections of it, while Max pooling takes the maximum value. Figure \ref{fig:basics:pooling} shows an example of a max and average pooling layer on a single-channel feature image using a 2 by 2 kernel and a stride of 2.

\begin{figure}[!ht]
  \centering
  \includegraphics[width=0.7\linewidth]{Diagrams/Pooling.png}
  \caption{Max and average pooling with a 2 by 2 kernel and stride 2. The color denotes the kernel position.}
  \label{fig:basics:pooling}
\end{figure}

Normally, Pooling layers are applied over the output of the activation functions.

\subsubsection{Global Average Pooling layer}

As we mentioned at the beginning of this section, the idea of a \gls{cnn} is to first learn the features within the input images and then make a prediction based on these features. To do this, the \hl{Global Average Pooling layer} transforms the channels of the last feature map into a vector (by averaging each of its channels), so that this can be used as input in a regular \gls{ann} to make the final prediction. Figure \ref{fig:basics:global_avg_pool} shows an example of this, when it is applied into a feature map with 7 channels.

\begin{figure}[!ht]
  \centering
  \includegraphics[width=0.7\linewidth]{Diagrams/Global_avg_pooling.png}
  \caption{Global Average Pooling layer.}
  \label{fig:basics:global_avg_pool}
\end{figure}

\subsubsection{Inception module}

Recall that a convolution layer is meant to learn features from a 3D object with 2 spatial dimensions (width and height) and a channel dimension. This means that each kernel in the convolution needs to learn simultaneously cross-channel and spatial correlations.
The intuition behind the \hl{Inception module} is to improve this process by separating this two tasks, so that the cross-channel correlations and the spatial correlations can be learned separately and independently \cite{chollet2017xception}.

A normal inception model looks at the cross-channel correlations first through a set of 3 or 4 \hl{pointwise convolutions}\footnote{A \hl{pointwise convolution} is a convolution with 1 by 1 kernels and stride 1.}, and then learns the spacial information in the downsampled feature image (in depth, not height and width), by means of regular convolution (usually with 3 by 3 or 5 by 5 kernels). Figure \ref{fig:basics:inception_module} shows a diagram of an Inveption V3 module.

\begin{figure}[!ht]
  \centering
  \includegraphics[width=0.6\linewidth]{Inception_module.png}
  \caption{A regular Inception module (Inveption V3). Image source \cite{chollet2017xception}.}
  \label{fig:basics:inception_module}
\end{figure}

François Chollet \cite{chollet2017xception}, used the inception module as reference to propose the \hl{depthwise separable convolution}, which is something between a normal convolution and a normal convolution combined/followed by a pointwise convolution.
Figure \ref{fig:basics:extreme_inception_module} shows an \hl{extreme} version of the inception module shown in figure \ref{fig:basics:inception_module}. The \hl{depthwise separable convolution} is very similar to the one shown in figure \ref{fig:basics:extreme_inception_module}, the only difference is that the pointwise convolution is applied before the 3 by 3 convolutions instead of after.

\begin{figure}[!ht]
  \centering
  \includegraphics[width=0.6\linewidth]{Extreme_Inception_module.png}
  \caption{An extreme version of our Inception module. Image source \cite{chollet2017xception}.}
  \label{fig:basics:extreme_inception_module}
\end{figure}

Even though the \hl{depthwise separable convolution} is a simplified version of the inception module, the idea and motivation behind it is the same. The \hl{depthwise separable convolution}, and the residual block, are the main components of the \hl{Xception} architecture \cite{chollet2017xception}.


\section{Robust PCA}
\begin{compactitem}
	\item Idea: Approximate $\mathbf{X}$ with $\mathbf{L} + \mathbf{S}$, $\mathbf{L}$ is low-rank, $\mathbf{S}$ is sparse.
	\item $\min_{\mathbf{L},\mathbf{S}}\mathsf{rank}(\mathbf{L}) + \mu \lVert \mathbf{S}\rVert_0$, s. t. $\mathbf{L} + \mathbf{S} = \mathbf{X}$. As non-convex, change to $\min_{\mathbf{L},\mathbf{S}} \|\mathbf{L}\|_\star + \lambda \lVert\mathbf{S}\rVert_1$ (\emph{not} the same in general) Deal w/ missing values: subj. to $L_{ij} + S_{ij} = X_{ij}, \forall(i,j) \in \Omega_{observed}$
	\item Perfect reconstruction is \emph{not} possible if $\mathbf{S}$ is low-rank, $\mathbf{L}$ is sparse, or $\mathbf{X}$ is low-rank \textit{and} sparse. Formally coherence: $\|\mathbf{U}^\top \mathbf{e}_i\|^2 \leq \frac{\nu r}{n}$, $\|\mathbf{V}^\top \mathbf{e}_i\|^2 \leq \frac{\nu r}{n}$, $\|\mathbf{UV}^\top\|^2_{ij} \leq \frac{\nu r}{n^2}$ : $\mathbf{L}=\mathbf{U}\mathbf{D}\mathbf{V}^\top$
\end{compactitem}

\subsection*{Dual Ascent (Gradient Method for Dual Problem)}
$\boldsymbol{\lambda}^{t+1} = \boldsymbol{\lambda}^{t} + \eta \nabla D(\boldsymbol{\lambda}^t)$,
$ \nabla D (\boldsymbol{\lambda}) = \mathbf{A}\mathbf{x}^*-\mathbf{b}$ for $\mathbf{x}^* \in \arg\min_\mathbf{x} \mathcal{L}(\mathbf{x},\boldsymbol{\lambda})$
\textbf{Dual Decomposition for Dual Ascent}:
\\$\mathbf{x}_i^{t+1} := \arg\min_{\mathbf{x}_i} \mathcal{L}_i(\mathbf{x}_i,\lambda^t)$;
$\boldsymbol{\lambda}^{t+1} := \boldsymbol{\lambda}^t + \eta^t \left( \sum_{i=1}^{N} \mathbf{A}_i \mathbf{x}_i^{t+1} -\mathbf{b} \right) $

\subsection*{Alternating Direction Method of Multipliers (ADMM)}
$\min_{\mathbf{x}_1, \mathbf{x}_2} f_1(\mathbf{x}_1) + f_2(\mathbf{x}_2)$ s. t. $\mathbf{A}_1 \mathbf{x}_1 + \mathbf{A}_2 \mathbf{x}_2 = \mathbf{b}$, $f_1, f_2$ convex
\begin{inparaitem}[\color{red}\textbullet]
	\item Augmented Lagrangian: $L_p(\mathbf{x}_1, \mathbf{x}_2, \boldsymbol{\nu}) = f_1(\mathbf{x}_1) + f_2(\mathbf{x}_2) + \boldsymbol{\nu}^\top (\mathbf{A}_1 \mathbf{x}_1 + \mathbf{A}_2 \mathbf{x}_2 - \mathbf{b}) + \frac{p}{2}\| \mathbf{A}_1 \mathbf{x}_1 + \mathbf{A}_2 \mathbf{x}_2 - \mathbf{b} \|_2^2$
	\item ADMM: $\mathbf{x}_1^{(t+1)} := \argmin_{\mathbf{x}_1} L_p(\mathbf{x}_1, \mathbf{x}_2^{(t)}, \boldsymbol{\nu}^{(t)})$, $\mathbf{x}_2^{(t+1)} := \argmin_{\mathbf{x}_2} L_p(\mathbf{x}_1^{(t+1)}, \mathbf{x}_2, \boldsymbol{\nu}^{(t)})$, $\boldsymbol{\nu}^{(t+1)} := \boldsymbol{\nu}^{(t)} + p(\mathbf{A}_1 \mathbf{x}_1^{(t+1)} + \mathbf{A}_2 \mathbf{x}_2^{(t+1)} - \mathbf{b})$
  \item ADMM for RPCA: $f_1(\mathbf{L}) = \|\mathbf{L}\|_\star$, $f_2(\mathbf{S}) = \lambda \| \mathbf{S} \|_1$, $\mathbf{A}_1 \mathbf{x}_1 + \mathbf{A}_2 \mathbf{x}_2 = \mathbf{b} \text{ becomes } \mathbf{L} + \mathbf{S} = \mathbf{X}$, therefore $L_p(\mathbf{L}, \mathbf{S}, \boldsymbol{\nu}) = \|\mathbf{L}\|_* + \nu \|\mathbf{S}\|_1 + \left< \nu, \mathrm{vec}(\mathbf{L}+\mathbf{S}-\mathbf{X}) \right> + \frac{P}{2} \| \mathbf{L}+ \mathbf{S} - \mathbf{X} \|_F^2$
  %updates: $\mathbf{L}^{t+1} = \mathcal{D}_{\rho^{-1}}(X-S-\rho^{-1}\text{mat}(\lambda))$
\end{inparaitem}


\section{MAP Operator}
The operator MAP applies a uniform evaluation pattern
to all members of a composite structure: a matrix, a list or the arguments
of an operator expression. The evaluation pattern can be a
unary procedure, an operator, or an algebraic expression with
one free variable. MAP is used with the syntax:
\begin{verbatim}
  MAP(EXPRN1:algebraic,EXPRN2:expression)
\end{verbatim}
{\tt EXPRN2} is a list, a matrix or an operator expression.

{\tt EXPRN1} is
\begin{itemize}
\item the name of an operator for a single argument: the operator
 is evaluated once with each element of {\tt EXPRN1} as its single argument,
\item an algebraic expression with exactly one free variable, that is
a variable preceded by the tilde symbol: the expression
 is evaluated for each element of {\tt EXPRN1} where the element is
 substituted for the free variable,
\item a replacement rule of the form
\begin{verbatim}
  VAR => EXPRN3
\end{verbatim}
  where {\tt VAR} is a variable and {\tt EXPRN3} is an expression
  which contains {\tt VAR}.
  Here {\tt EXPRN3} is evaluated for each element of {\tt EXPRN1} where
  the element is substituted for  {\tt VAR}. {\tt VAR} may be
  optionally preceded by a tilde.
\end{itemize}
The rule form for {\tt EXPRN2} is needed when more than
one free variable occurs.

{\it Examples:}
\begin{verbatim}
     % collect absolute values
   map(abs,{1,-2,a,-a});

                      ->

   {1,2,abs(a),abs(a)}

     % integrate a matrix
   map(int(~w,x), mat((x^2,x^5),(x^4,x^5)));

                      ->

        [  3     6 ]
        [ x     x  ]
        [----  ----]
        [ 3     6  ]
        [          ]
        [  5     6 ]
        [ x     x  ]
        [----  ----]
        [ 5     6  ]

     % multiply an equation
   map(~w*6 , x^2/3 = y^3/2 -1);

                      ->

         2     3
      2*x =3*(y  - 2)

\end{verbatim}
{\tt MAP} can be applied in nested mode:
\begin{verbatim}
   map(sub(x=y,~q)-sub(x=0,~q),map(mat(int(~w,x),(x^2-y,x^3))));
                      ->

       [  2             4 ]
       [ y *(y - 3)    y  ]
       [------------  ----]
       [     3         4  ]

    % The following example needs the rule form because there are two
    % free variables in the rightmost expression.

   map(~w=>map(int(~r,x),w),{mat((x^2,x^4)),mat((x^3,x^5))});

                      ->
       {

        [  3     5 ]
        [ x     x  ]
        [----  ----]
        [ 3     5  ]

        ,

        [  4     6 ]
        [ x     x  ]
        [----  ----]
        [ 4     6  ]

        }
\end{verbatim}

\section{SELECT Operator}
The operator SELECT extracts from a list
or from the arguments of an n--ary operator elements corresponding
to a boolean predicate. The predicate pattern can be a
unary procedure, an operator or an algebraic expression with
one free variable. SELECT is used with the syntax:
\begin{verbatim}
  SELECT(EXPRN1:expression,EXPRN2:list)
\end{verbatim}

{\tt EXPRN1} is
\begin{itemize}
\item the name of an operator for a single argument: the operator
 is evaluated once with each element of {\tt EXPRN2} as its single argument,
\item an algebraic expression with exactly one free variable, that is
a variable preceded by the tilde symbol at least once: the expression
 is evaluated for each element of {\tt EXPRN2} where the element is
 substituted for the free variable,
\item a replacement rule of the form
\begin{verbatim}
  VAR => EXPRN3
\end{verbatim}
  where {\tt VAR} is a variable and {\tt EXPRN3} is an expression
  which contains {\tt VAR}.
  Here {\tt EXPRN3} is evaluated for each element of {\tt EXPRN2} where
  the element is substituted for  {\tt VAR}. {\tt VAR} may be
  optionally preceded by a tilde.
\end{itemize}
The rule form for {\tt EXPRN1} is needed when more than
one free variable occurs. The result of the operation is built
from those elements of {\tt EXPRN2} which let {\tt EXPRN1} evaluate
to a value different from {\tt 0} or {\tt nil}, using the
leading operator of  {\tt EXPRN2}.

Examples:

\begin{verbatim}
  select( ~w>0, {1,-1,2,-3,3})
                      ->
                    {1,2,3}

   % select the tersm with even powers of y
  select(evenp deg(~w,y), part((x+y)^5,0):=list);

                      ->
          5     3  2      4
        {x ,10*x *y ,5*x*y }

   % select elements of a sum directly
  select(evenp deg(~w,x), 2x^2+3x^3+4x^4);

                      ->
          2    4
        2x  +4x
\end{verbatim}


\section{The univalence axiom}

\subsection{Equivalent forms of the univalence axiom}

The univalence axiom characterizes the identity type of the universe. Roughly speaking, it asserts that equivalent types are equal. It is considered to be an \emph{extensionality principle}\index{extensionality principle!for types} for types.

\begin{axiom}[Univalence]\label{axiom:univalence}
  The \define{univalence axiom}\index{univalence axiom} on a universe $\UU$ is the statement that for any $A:\UU$ the family of maps\index{equiv-eq@{$\equiveq$}}
\begin{equation*}
\equiveq : \prd{B:\UU} (\id{A}{B})\to(\eqv{A}{B}).
\end{equation*}
that sends $\refl{A}$ to the identity equivalence $\idfunc:\eqv{A}{A}$ is a family of equivalences.\index{identity type!of a universe} A universe satisfying the univalence axiom is referred to as a \define{univalent universe}\index{univalent universe}. If $\UU$ is a univalent universe we will write
$\eqequiv$\index{eq-equiv@{$\eqequiv$}}
for the inverse of $\equiveq$.
\end{axiom}

The following theorem is a special case of the fundamental theorem of identity types (\cref{thm:id_fundamental})\index{fundamental theorem of identity types}. Subsequently we will assume that any type is contained in a univalent universe.\index{axiom!univalence}

\begin{thm}\label{thm:univalence}
The following are equivalent:
\begin{enumerate}
\item The univalence axiom holds.
\item The type
\begin{equation*}
\sm{B:\UU}\eqv{A}{B}
\end{equation*}
is contractible for each $A:\UU$.
\item The principle of \define{equivalence induction}\index{equivalence induction}\index{induction principle!for equivalences} holds: for every $A:\UU$ and for every type family
\begin{equation*}
P:\prd{B:\UU} (\eqv{A}{B})\to \UU,
\end{equation*}
the map
\begin{equation*}
\Big(\prd{B:\UU}{e:\eqv{A}{B}}P(B,e)\Big)\to P(A,\idfunc[A])
\end{equation*}
given by $f\mapsto f(A,\idfunc[A])$ has a section.
\end{enumerate}
\end{thm}

\subsection{Univalence implies function extensionality}
One of the first applications of the univalence axiom was Voevodsky's theorem that the univalence axiom on a universe $\UU$ implies function extensionality for types in $\UU$. The proof uses the fact that weak function extensionality implies function extensionality.

We will also make use of the following lemma. Note that this statement was also part of \cref{lem:postcomp_equiv}. That exercise is solved using function extensionality. Since our present goal is to derive function extensionality from the univalence axiom, we cannot make use of that exercise.

\begin{lem}\label{lem:postcomp-equiv}
  For any equivalence $e:\eqv{X}{Y}$ in a univalent universe $\UU$, and any type $A$, the post-composition map
  \begin{equation*}
    e\circ\blank : (A \to X) \to (A\to Y)
  \end{equation*}
  is an equivalence.
\end{lem}

\begin{proof}
  The statement is obvious for the identity equivalence $\idfunc : \eqv{X}{X}$. Therefore the claim follows by equivalence induction, which is by \cref{thm:univalence} one of the equivalent forms of the univalence axiom.
\end{proof}

\begin{thm}\label{thm:funext-univalence}\index{univalence axiom!implies function extensionality}
  For any universe $\UU$, the univalence axiom on $\UU$ implies function extensionality on $\UU$.
\end{thm}

\begin{proof}
  Note that by \cref{thm:funext_wkfunext}\index{weak function extensionality} it suffices to show that univalence implies weak function extensionality, where we note that \cref{thm:funext_wkfunext} also holds when it is restricted to small types.

Suppose that $B:A\to \UU$ is a family of contractible types. Our goal is to show that the product $\prd{x:A}B(x)$ is contractible.
Since each $B(x)$ is contractible, the projection map $\proj 1:\big(\sm{x:A}B(x)\big)\to A$ is an equivalence by \cref{ex:proj_fiber}.

Now it follows by \cref{lem:postcomp-equiv} that $\proj1\circ\blank$ is an equivalence. Consequently, it follows from \cref{thm:contr_equiv} that the fibers of
\begin{equation*}
\proj 1\circ\blank : \Big(A\to \sm{x:A}B(x)\Big)\to (A\to A)
\end{equation*}
are contractible. In particular, the fiber at $\idfunc[A]$ is contractible. Therefore it suffices to show that $\prd{x:A}B(x)$ is a retract of $\sm{f:A\to\sm{x:A}B(x)}\proj 1\circ f=\idfunc[A]$. In other words, we will construct
\begin{equation*}
\begin{tikzcd}
\Big(\prd{x:A}B(x)\Big) \arrow[r,"i"] & \Big(\sm{f:A\to\sm{x:A}B(x)}\proj 1\circ f=\idfunc[A]\Big) \arrow[r,"r"] & \Big(\prd{x:A}B(x)\Big),
\end{tikzcd}
\end{equation*}
and a homotopy $r\circ i\htpy \idfunc$.

We define the function $i$ by
\begin{equation*}
  i(f) \defeq (\lam{x}(x,f(x)),\refl{\idfunc}).
\end{equation*}
To see that this definition is correct, we need to know that
\begin{equation*}
  \lam{x}\proj 1(x,f(x))\jdeq \idfunc.
\end{equation*}
This is indeed the case, by the $\eta$-rule\index{eta-rule@{$\eta$-rule}} for $\Pi$-types.

Next, we define the function $r$. Let $h:A\to \sm{x:A}B(x)$, and let $p:\proj 1 \circ h = \idfunc$. Then we have the homotopy $H\defeq\htpyeq(p):\proj 1 \circ h \htpy \idfunc$. Then we have $\proj 2(h(x)):B(\proj 1(h(x)))$ and we have the identification $H(x):\proj 1(h(x))=x$. Therefore we define $r$ by
\begin{equation*}
  r((h,p),x)\defeq \tr_B(H(x),\proj 2(h(x))).
\end{equation*}

We note that if $p\jdeq \refl{\idfunc}$, then $H(x)\jdeq\refl{x}$. In this case we have the judgmental equality $r((h,\refl{}),x)\jdeq\proj 2 (h(x))$. Thus we see that $r\circ i\jdeq \idfunc$ by another application of the $\eta$-rule for $\Pi$-types.
\end{proof}

\subsection{Propositional extensionality and posets}

\begin{thm}\label{thm:propositional-extensionality}
  Propositions satisfy \define{propositional extensionality}\index{propositional extensionality}\index{extensionality principle!for propositions}:
  for any two propositions $P$ and $Q$, the canonical map\index{bi-implication}\index{iff-eq@{$\iffeq$}}
  \begin{equation*}
    \iffeq:(P=Q)\to (P\leftrightarrow Q)
  \end{equation*}
  that sends $\refl{P}$ to $(\idfunc,\idfunc)$ is an equivalence. It follows that the type $\prop$ of propositions in $\UU$ is a set.\index{Prop@{$\prop$}!is a set}
\end{thm}

Note that for any $P:\prop$, we usually also write $P$ for the underlying type of the proposition $P$. If we would be more formal about it we would have to write $\proj 1(P)$ for the underlying type, since $\prop$ is the $\Sigma$-type $\sm{X:\UU}\isprop(X)$. In the following proof it is clearer if we use the more formal notation $\proj 1(P)$ for the underlying type of a proposition $P$.

\begin{proof}
  We note that the identity type $P=Q$ is an identity type in $\prop$. However, since $\isprop(X)$ is a proposition for any type $X$, it follows that the map
  \begin{equation*}
    \apfunc{\proj 1} : (P = Q) \to (\proj 1(P) = \proj 1(Q))
  \end{equation*}
  is an equivalence. Now we observe that we have a commuting square
  \begin{equation*}
    \begin{tikzcd}[column sep=huge]
      (P=Q) \arrow[d,swap,"\apfunc{\proj 1}"] \arrow[r] & (P\leftrightarrow Q) \\
      (\proj 1(P)=\proj 1(Q)) \arrow[r,swap,"\equiveq"] & (\proj 1(P)\simeq\proj 1(Q)) \arrow[u,swap,"\simeq"]
    \end{tikzcd}
  \end{equation*}
  Since the left, bottom, and right map are equivalences, it follows that the top map is an equivalence.
\end{proof}

\begin{defn}
  A \define{partially ordered set (poset)}\index{partially ordered set|see poset}\index{poset} is a set $P$ equipped with a relation
  \begin{equation*}
    \blank\leq\blank : P \to (P \to \prop)
  \end{equation*}
  that is \define{reflexive}\index{reflexive!poset} (for every $x:P$ we have $x\leq x$), \define{transitive}\index{transitive!poset} (for every $x,y,z:P$ such that $x\leq y$ and $y\leq z$ we have $x\leq z$), and \define{anti-symmetric}\index{anti-symmetric!poset} (for every $x,y:P$ such that $x\leq y$ and $y\leq x$ we have $x=y$).
\end{defn}

\begin{rmk}
  The condition that $X$ is a set can be omitted from the definition of a poset. Indeed, if $X$ is any type that comes equipped with a $\prop$-valued ordering relation $\leq$ that is reflexive and anti-symmetric, then $X$ is a set by \cref{lem:prop_to_id}.
\end{rmk}

\begin{eg}
  The type $\prop$ is a poset\index{Prop@{$\prop$}!is a poset}, where the ordering relation is given by implication: $P$ is less than $Q$ if $P\to Q$. The fact that $P\to Q$ is a proposition is a special case of \cref{cor:funtype_trunc}. The relation $P\to Q$ is reflexive by the identity function, and transitive by function composition. Moreover, the relation $P\to Q$ is anti-symmetric by \cref{thm:propositional-extensionality}\index{propositional extensionality}.
\end{eg}

\begin{eg}
  The type of natural numbers\index{natural numbers!is a poset with leq@{is a poset with $\leq$}}\index{natural numbers!is a poset with divisibility}\index{poset!N with leq@{$\N$ with $\leq$}}\index{poset!N with divisibility@{$\N$ with divisibility}} comes equipped with at least two important poset structures. The first is given by the usual ordering relation $\leq$, and the second is given by the relation $d\mid n$ that $d$ divides $n$.
\end{eg}

\begin{thm}
  For any poset $P$ and any type $X$, the set $P^X$ is a poset.\index{poset!closed under exponentials} In particular the type of subtypes of any type is a poset.\index{poset!type of subtypes}\index{subtype!poset}
\end{thm}

\begin{proof}
  Let $P$ be a poset with ordering $\leq$, and let $X$ be a type. Then $P^X$ is a set by \cref{cor:funtype_trunc}. For any $f,g:X\to P$ we define
  \begin{equation*}
    (f\leq g) \defeq \prd{x:X}f(x)\leq g(x).
  \end{equation*}
  Reflexivity and transitivity follow immediately from reflexivity and transitivity of the original relation. Moreover, by the anti-symmetry of the original relation it follows that
  \begin{equation*}
    (f\leq g)\times (g\leq f) \to (f\htpy g).
  \end{equation*}
  Therefore we obtain an identification $f=g$ by function extensionality. The last claim follows immediately from the fact that a subtype of $X$ is a map $X\to\prop$, and the fact that $\prop$ is a poset.
\end{proof}

\begin{exercises}
\exercise \label{ex:istrunc_UUtrunc}
\begin{subexenum}
\item Use the univalence axiom to show that the type $\sm{A:\UU}\iscontr(A)$ of all contractible types in $\UU$ is contractible.\index{universe!of contractible types}
\item Use \cref{cor:emb_into_ktype,cor:funtype_trunc,ex:isprop_isequiv} to show that if $A$ and $B$ are $(k+1)$-types, then the type $\eqv{A}{B}$ is also a $(k+1)$-type.\index{A simeq B@{$\eqv{A}{B}$}!truncatedness}
\item Use univalence to show that the universe of $k$-types\index{universe!of k-types@{of $k$-types}}\index{U leq k@{$\UU^{\leq k}$}}\index{k-type@{$k$-type}!universe of k-types@{universe of $k$-types}}\index{truncated type!universe of k-types@{universe of $k$-types}}
\begin{equation*}
\UU^{\leq k}\defeq \sm{X:\UU}\istrunc{k}(X)
\end{equation*}
is a $(k+1)$-type, for any $k\geq -2$.
\item Show that $\UU^{\leq-1}$ is not a proposition.\index{universe!of propositions}
\item Show that $\eqv{(\eqv{\bool}{\bool})}{\bool}$, and conclude by the univalence axiom that the universe of sets\index{universe!of sets} $\UU^{\leq 0}$ is not a set.
\end{subexenum}
\exercise Use the univalence axiom to show that the type $\sm{P:\prop}P$ is contractible.
\exercise Let $A$ and $B$ be small types.
\begin{subexenum}
\item Construct an equivalence
\begin{equation*}
\eqv{(A\to (B\to\UU))}{\Big(\sm{S:\UU} (S\to A)\times (S\to B)\Big)}
\end{equation*}
\item We say that a relation $R:A\to (B\to\UU)$ is \define{functional}\index{relation!functional} if it comes equipped with a term of type\index{is-function(R)@{$\isfunction(R)$}}
\begin{equation*}
\isfunction(R) \defeq \prd{x:A}\iscontr\Big(\sm{y:B}R(x,y)\Big)
\end{equation*}
For any function $f:A\to B$, show that the \define{graph}\index{graph!of a function} of $f$
\begin{equation*}
\graph_f:A\to (B\to \UU)
\end{equation*}
given by $\graph_f(a,b)\defeq (f(a)=b)$ is a functional relation from $A$ to $B$.
\item Construct an equivalence
\begin{equation*}
\eqv{\Big(\sm{R:A\to (B\to\UU)}\isfunction(R)\Big)}{(A\to B)}
\end{equation*}
\item Given a relation $R:A\to (B\to \UU)$ we define the \define{opposite relation}\index{relation!opposite relation}\index{opposite relation}\index{op R@{$\opp{R}$}}
\begin{equation*}
\opp{R} : B\to (A\to\UU)
\end{equation*}
by $\opp{R}(y,x)\defeq R(x,y)$. Construct an equivalence\index{A simeq B@{$\eqv{A}{B}$}!as relation}
\begin{equation*}
\eqv{\Big(\sm{R:A\to (B\to \UU)}\isfunction(R)\times\isfunction(\opp{R})\Big)}{(\eqv{A}{B})}.
\end{equation*}
\end{subexenum}
\exercise
  \begin{subexenum}
  \item Show that $\isdecidable(P)$ is a proposition\index{is-decidable@{$\isdecidable$}!is a proposition}, for any proposition $P$.
  \item Show that $\classicalprop$%
    \index{classical-Prop@{$\classicalprop$}!classical-Prop bool@{$\classicalprop\eqvsym\bool$}} is equivalent to $\bool$.
  \end{subexenum}
\exercise Recall that $\UU_\ast$ is the universe of pointed types\index{UU*@{$\UU_\ast$}}.
  \begin{subexenum}
  \item For any $(A,a)$ and $(B,b)$ in $\UU_\ast$, write $(A,a)\simeq_\ast(B,b)$ for the type of \define{pointed equivalences}\index{pointed equivalence}\index{equivalence!pointed equivalence} from $A$ to $B$, i.e.,
    \begin{equation*}
      (A,a)\simeq_\ast (B,b)\defeq \sm{e:A\simeq B}e(a)=b.
    \end{equation*}
    Show that the canonical map\index{UU*@{$\UU_\ast$}!identity type}\index{identity type!of UU*@{of $\UU_\ast$}}
    \begin{equation*}
      \big((A,a)=(B,b)\big)\to \Big((A,a)\simeq (B,b)\Big)
    \end{equation*}
    sending $\refl{(A,a)}$ to the pair $(\idfunc,\refl{a})$, is an equivalence.
  \item Construct for any pointed type $(X,x_0)$ an equivalence
    \begin{equation*}
      \Big(\sm{P:X\to \UU}P(x_0)\Big)\simeq \sm{(A,a_0):\UU_\ast}(A,a_0)\to_\ast(X,x_0).
    \end{equation*}
  \end{subexenum}
\exercise Show that any subuniverse\index{subuniverse!closed under equivalences} is closed under equivalences, i.e., show that there is a map
  \begin{equation*}
    (X\simeq Y) \to (P(X)\to P(Y))
  \end{equation*}
  for any subuniverse $P:\UU\to\prop$, and any $X,Y:\UU$.
  \exercise Show that the universe inclusions
  \begin{equation*}
    \UU \to \UU^+\qquad\text{and}\qquad \UU\to \UU\sqcup\VV
  \end{equation*}
  defined in \cref{rmk:universe-constructions}, are embeddings.
\end{exercises}


\documentclass{article}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{mathtools}
\usepackage{graphicx}

\begin{document}

\title{Discretization of Common Filters}
\author{K. Loux}
\date{\today}
\maketitle

\section{Introduction}
This document contains difference equation implementations for transfer functions of common filters. Continuous-time transfer functions can be discretized using the relationship $z = e^{sT}$, where $T$ is the sampling period, $s$ is imaginary frequency and $z$ is discretized imaginary frequency.

In order for this relationship to be helpful, we need to solve for $s$, then replace all instances of $s$ in our continuous-time transfer functions with the result.  Solving for $s$, however, results in a relationship involving a natural logarithm.  We make a first-order approximation of the logarithm instead.

\begin{equation}
s = \frac{1}{T} \ln \left( z \right) \approx \frac{2}{T} \frac{1 - z^{-1}}{1 + z^{-1}}
\end{equation}

To go from a discrete-time transfer function to a difference equation, variables that are multiplied by $z^{-n}$ are replaced with the values of those variables from $n$ time steps in the past.

We use the variable $u$ to represent input and $y$ to represent output from the filters.  We give all coefficients in the form of $a_0$ through $a_n$ and $b_0$ through $b_m$ where $a$ coefficients are applied to input quantities and $b$ coefficients are applied to output quantities.  All coefficients can be scaled by $1 / b_0$, so then $b_0 = 1$ and the first $b$ coefficient required to be stored will be $b_1$.  These can be used in a difference equation as shown below.

\begin{equation}
y_k = \frac{a_0 u_k + a_1 u_{k - 1} + ... + a_n u_{k - n} - b_1 y_{k - 1} - ... - b_m y_{k - m}}{b_0}
\end{equation}

\section{First Order Low-Pass Filter}

Continuous-time transfer function:

\begin{equation}
\frac{Y}{U} = \frac{\omega}{s + \omega}
\end{equation}

Difference equation coefficients:

\begin{align*}
a_0 &= \omega T \\
a_1 &= a_0 \\
b_0 &= \omega T + 2 \\
b_1 &= \omega T - 2
\end{align*}

\section{Second Order Low-Pass Filter}

Continuous-time transfer function:

\begin{equation}
\frac{Y}{U} = \frac{\omega^2}{s^2 + 2 \zeta \omega s + \omega^2}
\end{equation}

Difference equation coefficients:

\begin{align*}
a_0 &= \omega^2 T^2 \\
a_1 &= 2 a_0 \\
a_2 &= a_0 \\
b_0 &= \omega^2 T^2 + 4 + 4 \zeta \omega T \\
b_1 &= 2 \omega^2 T^2 - 8 \\
b_2 &= \omega^2 T^2 + 4 - 4 \zeta \omega T
\end{align*}

\section{First Order High-Pass Filter}

Continuous-time transfer function:

\begin{equation}
\frac{Y}{U} = \frac{s}{s + \omega}
\end{equation}

Difference equation coefficients:

\begin{align*}
a_0 &= 2 \\
a_1 &= -2 \\
b_0 &= \omega T + 2 \\
b_1 &= \omega T - 2
\end{align*}

\end{document}


\section{Determinants and row operations}

\begin{outcome}
  \begin{enumerate}
  \item Determine the effect of a row operation on the determinant of
    a matrix.
  \item Use row operations to calculate a determinant.
  \end{enumerate}
\end{outcome}

Recall that there are three kinds of elementary row operations%
\index{matrix!row operation}%
\index{matrix!elementary row operation}%
\index{row operation}%
\index{elementary row operation} on matrices:
\begin{enumerate}
\item Switch two rows.
\item Multiply a row by a non-zero number.
\item Add a multiple of one row to another row.
\end{enumerate}
The following theorem examines the effect of these row operations on
the determinant of a matrix.

\begin{theorem}{Effect of row operations on the determinant}{determinant-row-operations}
  Let $A$ be an $n\times n$-matrix.
  \begin{enumerate}
  \item If $B$ is obtained from $A$ by switching two rows, then
    \begin{equation*}
      \det(B) = -\det(A).
    \end{equation*}
  \item If $B$ is obtained from $A$ by multiplying one row by a
    non-zero scalar $k$, then
    \begin{equation*}
      \det(B) = k\det(A).
    \end{equation*}
  \item If $B$ is obtained from $A$ by adding a multiple of one row to
    another row, then
    \begin{equation*}
      \det(B) = \det(A).
    \end{equation*}
  \end{enumerate}
\end{theorem}

Notice that the second part of this theorem is true when we multiply
{\em one} row of the matrix by $k$.  If we were to multiply {\em two}
rows of $A$ by $k$ to obtain $B$, we would have
$\det(B) = k^2 \det(A)$.

\begin{example}{Using row operations to calculate a determinant}{determinant-row-operations1}
  Use row operations to calculate the following determinant:
  \begin{equation*}
    \begin{absmatrix}{rrr}
      1 & 5 & 5 \\
      0 & 0 & -3 \\
      0 & 2 & 7 \\
    \end{absmatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  If we switch the second and third rows, we obtain a triangular
  matrix, of which the determinant is easy to compute. By
  Theorem~\ref{thm:determinant-row-operations}, switching two rows
  negates the determinant. We therefore have:
  \begin{equation*}
    \begin{absmatrix}{rrr}
      1 & 5 & 5 \\
      0 & 0 & -3 \\
      0 & 2 & 7 \\
    \end{absmatrix}
    ~=~
    -\begin{absmatrix}{rrr}
      1 & 5 & 5 \\
      0 & 2 & 7 \\
      0 & 0 & -3 \\
    \end{absmatrix}
    ~=~ -(1\cdot 2\cdot(-3)) ~=~ 6.
  \end{equation*}
\end{solution}

\begin{example}{Using row operations to calculate a determinant}{determinant-row-operations2}
  Use row operations to calculate the following determinant:
  \begin{equation*}
    \begin{absmatrix}{rrr}
      1 & 4 & -2 \\
      1 & 8 & 1 \\
      2 & 4 & -9 \\
    \end{absmatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  We can use elementary row operations to reduce this matrix to
  triangular form:
  \begin{equation*}
    \begin{mymatrix}{rrr}
      1 & 4 & -2 \\
      1 & 8 & 1 \\
      2 & 4 & -9 \\
    \end{mymatrix}
    \stackrel{R_2\rowop R_2-R_1}{\roweq}
    \begin{mymatrix}{rrr}
      1 & 4 & -2 \\
      0 & 4 & 3 \\
      2 & 4 & -9 \\
    \end{mymatrix}
    \stackrel{R_3\rowop R_3-2R_1}{\roweq}
    \begin{mymatrix}{rrr}
      1 & 4 & -2 \\
      0 & 4 & 3 \\
      0 & -4 & -5 \\
    \end{mymatrix}
    \stackrel{R_3\rowop R_3+R_2}{\roweq}
    \begin{mymatrix}{rrr}
      1 & 4 & -2 \\
      0 & 4 & 3 \\
      0 & 0 & -2 \\
    \end{mymatrix}.
  \end{equation*}
  Each of the row operations is of the form ``add a multiple of one
  row to another row'', and therefore does not change the
  determinant. We therefore have:
  \begin{equation*}
    \begin{absmatrix}{rrr}
      1 & 4 & -2 \\
      1 & 8 & 1 \\
      2 & 4 & -9 \\
    \end{absmatrix}
    ~=~
    \begin{absmatrix}{rrr}
      1 & 4 & -2 \\
      0 & 4 & 3 \\
      2 & 4 & -9 \\
    \end{absmatrix}
    ~=~
    \begin{absmatrix}{rrr}
      1 & 4 & -2 \\
      0 & 4 & 3 \\
      0 & -4 & -5 \\
    \end{absmatrix}
    ~=~
    \begin{absmatrix}{rrr}
      1 & 4 & -2 \\
      0 & 4 & 3 \\
      0 & 0 & -2 \\
    \end{absmatrix}
    ~=~ 1\cdot 4\cdot(-2) = -8.
  \end{equation*}
\end{solution}

In general, we can convert any square matrix to triangular form using
elementary row operations. In fact, it is always possible to do so
using only elementary operations of the first and third kind (swap two
rows or add a multiple of one row to another). This gives us a very
efficient way to compute determinants. If the matrices are large, this
method is much more efficient than the cofactor method.

\begin{example}{Using row operations to calculate a determinant}{determinant-row-operations3}
  Use elementary row operations of the first and third kind to calculate the
  following determinant:
  \begin{equation*}
    \begin{absmatrix}{rrrr}
      0 & 2 & 1 & 4 \\
      2 & 2 & -4 & -1 \\
      1 & 1 & -2 & -1 \\
      1 & 3 & 2 & 5 \\
    \end{absmatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  We use elementary row operations to reduce the matrix to triangular
  form:
  \begin{equation*}
    \begin{array}{ccccc}
      \begin{mymatrix}{rrrr}
        0 & 2 & 1 & 4 \\
        2 & 2 & -4 & -1 \\
        1 & 1 & -2 & -1 \\
        1 & 3 & 2 & 5 \\
      \end{mymatrix}
      &\stackrel{R_1\rowswap R_3}{\roweq}&
      \begin{mymatrix}{rrrr}
        1 & 1 & -2 & -1 \\
        2 & 2 & -4 & -1 \\
        0 & 2 & 1 & 4 \\
        1 & 3 & 2 & 5 \\
      \end{mymatrix}
      &\stackrel{R_2\rowop R_2-2R_1}{\roweq}&
      \begin{mymatrix}{rrrr}
        1 & 1 & -2 & -1 \\
        0 & 0 & 0 & 1 \\
        0 & 2 & 1 & 4 \\
        1 & 3 & 2 & 5 \\
      \end{mymatrix}
      \\\\[-1ex]
      &\stackrel{R_4\rowop R_4-R_1}{\roweq}&
      \begin{mymatrix}{rrrr}
        1 & 1 & -2 & -1 \\
        0 & 0 & 0 & 1 \\
        0 & 2 & 1 & 4 \\
        0 & 2 & 4 & 6 \\
      \end{mymatrix}
      &\stackrel{R_4\rowop R_4-R_3}{\roweq}&
      \begin{mymatrix}{rrrr}
        1 & 1 & -2 & -1 \\
        0 & 0 & 0 & 1 \\
        0 & 2 & 1 & 4 \\
        0 & 0 & 3 & 2 \\
      \end{mymatrix}
      \\\\[-1ex]
      &\stackrel{R_2\rowswap R_3}{\roweq}&
      \begin{mymatrix}{rrrr}
        1 & 1 & -2 & -1 \\
        0 & 2 & 1 & 4 \\
        0 & 0 & 0 & 1 \\
        0 & 0 & 3 & 2 \\
      \end{mymatrix}
      &\stackrel{R_3\rowswap R_4}{\roweq}&
      \begin{mymatrix}{rrrr}
        1 & 1 & -2 & -1 \\
        0 & 2 & 1 & 4 \\
        0 & 0 & 3 & 2 \\
        0 & 0 & 0 & 1 \\
      \end{mymatrix}
    \end{array}
  \end{equation*}
  By Theorem~\ref{thm:determinant-row-operations}, the determinant
  changes signs each time we swap two rows. The determinant is
  unchanged when we add a multiple of one row to another. Therefore,
  we have
  \begin{equation*}
    \begin{array}{ccccccc}
      \begin{absmatrix}{rrrr}
        0 & 2 & 1 & 4 \\
        2 & 2 & -4 & -1 \\
        1 & 1 & -2 & -1 \\
        1 & 3 & 2 & 5 \\
      \end{absmatrix}
      &=&
      - \begin{absmatrix}{rrrr}
        1 & 1 & -2 & -1 \\
        2 & 2 & -4 & -1 \\
        0 & 2 & 1 & 4 \\
        1 & 3 & 2 & 5 \\
      \end{absmatrix}
      &=&
      - \begin{absmatrix}{rrrr}
        1 & 1 & -2 & -1 \\
        0 & 0 & 0 & 1 \\
        0 & 2 & 1 & 4 \\
        1 & 3 & 2 & 5 \\
      \end{absmatrix}
      \\\\[-1ex]
      &=&
      - \begin{absmatrix}{rrrr}
        1 & 1 & -2 & -1 \\
        0 & 0 & 0 & 1 \\
        0 & 2 & 1 & 4 \\
        0 & 2 & 4 & 6 \\
      \end{absmatrix}
      &=&
      - \begin{absmatrix}{rrrr}
        1 & 1 & -2 & -1 \\
        0 & 0 & 0 & 1 \\
        0 & 2 & 1 & 4 \\
        0 & 0 & 3 & 2 \\
      \end{absmatrix}
      \\\\[-1ex]
      &=&
      + \begin{absmatrix}{rrrr}
        1 & 1 & -2 & -1 \\
        0 & 2 & 1 & 4 \\
        0 & 0 & 0 & 1 \\
        0 & 0 & 3 & 2 \\
      \end{absmatrix}
      &=&
      - \begin{absmatrix}{rrrr}
        1 & 1 & -2 & -1 \\
        0 & 2 & 1 & 4 \\
        0 & 0 & 3 & 2 \\
        0 & 0 & 0 & 1 \\
      \end{absmatrix}
      &=& -6.
    \end{array}
  \end{equation*}
  In practice, the last calculation could have been done in a single
  step. All we had to do is count the number of swap operations we
  performed during the row operations. If there is an odd number of
  swap operations, the sign of the determinant changes; otherwise, it
  stays the same.
\end{solution}


\section{Common Vector-Valued Functions}
\input{./vectorValuedFunctions/circles}
\input{./vectorValuedFunctions/helices}

\noindent
Since VVFs are essentially multiple single-input single-output functions packaged together, the domain of a VVF is the domain on which all components are defined.\\

\noindent
For example, if $\vec{r}(t) = \langle \tan{t},6t,\ln{\left(16-t^2\right)} \rangle$,
\begin{itemize}
	\item $\tan{t}$ is defined for all real numbers not equivalent to $\pm\pi/2$ radians.
	\item $6t$ is defined for all real numbers.
	\item $\ln{\left(16-t^2\right)}$ is defined for $t\in\left(-4,4\right)$.
\end{itemize}
The intersection of these domains is $\left(-4, -\pi/2\right) \cup \left(-\pi/2, \pi/2\right) \cup \left(\pi/2, 4\right)$, which is the domain of $\vec{r}(t)$.


\documentclass[notitlepage]{problem-solving}

\author{Matt McCarthy}
\title{Fibonacci Generating Function}
\date{June 2016}

\addbibresource{fibonacci.bib}
\nocite{*}

\begin{document}

\maketitle

\begin{problem*}
	Let $(F_n)_{n\in\NN}$ denote the sequence of Fibonacci numbers.
	Find the closed form of $F_n$.
\end{problem*}

\section{Background}

Lets begin with the definition of the Fibonacci numbers.
\begin{definition}
	The Fibonacci numbers $(F_n)_{n\in\NN}$ are defined by $F_1=F_2=1$, and $F_{n+2}=F_{n+1}+F_n$.
\end{definition}

From the Fibonacci numbers we get the golden ratio, $\varphi$, and its conjugate, $\overline{\varphi}$ where
\[
	\varphi = \frac{1+\sqrt{5}}{2} \text{ and } \overline{\varphi} = \frac{1-\sqrt{5}}{2}.
\]

While the Fibonacci recurrence is simple, its difficult to tell from the definition what the $n$th Fibonacci number is without computing all of the terms before it.
Since the Fibonacci's are defined recursively, we can use \textit{generating functions} to extract a closed form for $F_n$ relatively easily.

\begin{definition}
	Let $(a_n)_{n\in\NN}$ be a $\RR$-valued sequence.
	Then the generating function for $a_n$ is the power series
	\[
		G(x) = \sum_{n\in\NN} a_nx^n.
	\]
\end{definition}

After we create the generating function, we generally want to find its closed form (that is, a representation without using limits).
Then, we use that new representation to find a closed form for the terms of the sequence (in this case, a non-recursive representation).

\section{Solution}

\begin{problem*}
	Let $(F_n)_{n\in\NN}$ denote the sequence of Fibonacci numbers.
	Find the closed form of $F_n$.
\end{problem*}

We begin by setting up a generating function, $F(x)$, for $F_n$.
\[
	F(x) := \sum_{n=1}^\infty F_n x^n.
\]
The next step is to use the recurrence, however $F_1$ and $F_2$ need to be handled separately since they are not defined recursively.
Thus,
\[
	F(x) = F_1 x + F_2 x^2 + \sum_{n=3}^\infty F_n x^n.
\]
Now we apply the definition of the Fibonacci numbers yielding,
\[
	F(x) = x + x^2 + \sum_{n=3}^\infty (F_{n-1} + F_{n-2}) x^n.
\]
By rearranging we get,
\[
	F(x) = x + x^2 +\paren{\sum_{n=3}^\infty F_{n-1} x^n} + \paren{\sum_{n=3}^\infty F_{n-2} x^n}.
\]
Now, we factor out some power of $x$ from each sum, yielding
\[
	F(x) = x + x^2 +x\paren{\sum_{n=3}^\infty F_{n-1} x^{n-1}} + x^2\paren{\sum_{n=3}^\infty F_{n-2} x^{n-2}}.
\]
We now reindex the sums and do some manipulation so that we get the following.
\[
	F(x) = x + x^2 +x\paren{F_1x-F_1x+\sum_{n=2}^\infty F_{n} x^n} + x^2\paren{\sum_{n=1}^\infty F_{n} x^n}
\]
Some more manipulation yields,
\[
	F(x) = x + x^2 -x^2 +x\paren{\sum_{n=1}^\infty F_n x^n} + x^2 \paren{\sum_{n=1}^\infty F_n x^n}.
\]
Note, that we see the original definition of $F(x)$ in this new form, therefore
\[
	F(x) = x + xF(x) + x^2 F(x).
\]
Solving for $F(x)$ yields,
\[
	F(x) = \frac{x}{1-x-x^2} = \frac{x}{-(x^2+x-1)}.
\]

The next step is to find the closed form of $F_n$.
We know that $-\varphi$ and $-\overline{\varphi}$ are roots of  the polynomial in the denominator, therefore
\[
	F(x) = \frac{x}{-(x+\varphi)(x+\overline{\varphi})}.
\]
Using $\varphi\overline{\varphi} = -1$ yields,
\[
	F(x)=\frac{x}{(1-\varphi x)(1-\overline{\varphi}x)}.
\]
Next, we perform partial fraction decomposition of $F(x)$, which gives us
\[
	F(x) = \frac{1}{\sqrt{5}} \paren{\frac{1}{1-\varphi x}-\frac{1}{1-\overline{\varphi}x}}.
\]
Using the formula for the power series for $(1-x)^{-1}$ yields,
\[
	F(x) = \sum_{k=1}^\infty \frac{\varphi^n-\overline{\varphi}^n}{\sqrt{5}} x^n = \sum_{k=1}^\infty F_n x^n.
\]
By uniqueness of power series, we deduce that
\[
	F_n = \frac{\varphi^n-\overline{\varphi}^n}{\sqrt{5}}.
\]

\printbibliography

\end{document}