% !TEX root = hott_intro.tex

\section{The hierarchy of homotopical complexity}
\sectionmark{Homotopical complexity}\label{chap:hierarchy}
%Not all types have interesting higher groupoid structure. For example, we will see below that two natural numbers can only be equal in at most one way. Voevodsky articulated a useful notion to detect the homotopical complexity of types, which allows us to distinguish between contractible types (also called \emph{$(-2)$-types}), \emph{propositions} (also called \emph{$(-1)$-types}), \emph{sets} (\emph{$0$-types}), and \emph{$k$-types} for higher $k$.

%We will see [later] that there are types that are not $k$-types for any $k$.

\subsection{Propositions and subtypes}

\begin{defn}
A type $A$ is said to be a \define{proposition} if there is a term of type
\begin{equation*}
\isprop(A)\defeq\prd{x,y:A}\iscontr(x=y).
\end{equation*}
We define $\prop$ to be the type of all small propositions, i.e.,
\begin{equation*}
  \prop\defeq\sm{X:\UU}\isprop(A).
\end{equation*}
\end{defn}

\begin{eg}\label{eg:prop_contr}
Any contractible type is a proposition by \cref{ex:prop_contr}. However, propositions do not need to be inhabited: the empty type is also a proposition, since
\begin{equation*}
\prd{x,y:\emptyt}\iscontr(x=y)
\end{equation*}
follows from the induction principle of the empty type.
\end{eg}

In the following lemma we prove that in order to show that a type $A$ is a proposition, it suffices to show that any two terms of $A$ are equal. In other words, propositions are types with \define{proof irrelevance}.

\begin{thm}\label{lem:isprop_eq}
  Let $A$ be a type. Then the following are equivalent:
  \begin{enumerate}
  \item The type $A$ is a proposition.
  \item Any two terms of type $A$ can be identified, i.e., there is a dependent function
    \begin{equation*}
      \prd{x,y:A}\id{x}{y}.
    \end{equation*}
  \item The type $A$ is contractible as soon as it is inhabited, i.e., there is a function
    \begin{equation*}
      A \to \iscontr(A).
    \end{equation*}
  \item The map $\mathsf{const}_\ttt : A\to\unit$ is an embedding.
  \end{enumerate}
\end{thm}

\begin{proof}
  To show that (i) implies (ii), let $A$ be a proposition. Then its identity types are contractible, so the center of contraction of $\id{x}{y}$ is identification $\id{x}{y}$, for each $x,y:A$.

  To show that (ii) implies (iii), suppose that $A$ comes equipped with $p:\prd{x,y:A}\id{x}{y}$. Then for any $x:A$ the dependent function $p(x):\prd{y:A}\id{x}{y}$ is a contraction of $A$. Thus we obtain the function
  \begin{equation*}
    \lam{x}(x,p(x)):A\to\iscontr(A).
  \end{equation*}

  To show that (iii) implies (iv), suppose that $A\to\iscontr(A)$ and let $x,y:A$. We have to show that
  \begin{equation*}
    \apfunc{\mathsf{const}_\ttt}:(x=y)\to (\ttt=\ttt)
  \end{equation*}
  is an equivalence. Since we have $x:A$ it follows that $A$ is contractible. Since the unit type is contractible it follows that $\mathsf{const}_\ttt$ is an equivalence. Therefore we conclude by \cref{cor:emb_equiv} that it is an embedding.

  To show that (iv) implies (i), note that if $A\to\unit$ is an embedding, then the identity types of $A$ are equivalent to contractible types and therefore they must be contractible.
\end{proof}

In the following lemma we show that propositions are closed under equivalences.

\begin{lem}\label{lem:prop_equiv}
Let $A$ and $B$ be types, and let $e:\eqv{A}{B}$. Then we have
\begin{equation*}
\isprop(A)\leftrightarrow\isprop(B).
\end{equation*}
\end{lem}

\begin{proof}
We will show that $\isprop(B)$ implies $\isprop(A)$. This suffices, because the converse follows from the fact that $e^{-1}:B\to A$ is also an equivalence.

Since $e$ is assumed to be an equivalence, it follows by \cref{cor:emb_equiv} that
\begin{equation*}
\apfunc{e} : (x=y)\to (e(x)=e(y))
\end{equation*}
is an equivalence for any $x,y:A$. If $B$ is a proposition, then in particular the type $e(x)=e(y)$ is contractible for any $x,y:A$, so the claim follows from \cref{thm:contr_equiv}.
\end{proof}

  In set theory, a set $y$ is said to be a subset of a set $x$, if any element of $y$ is an element of $x$, i.e., if the condition
  \begin{equation*}
    \forall_z (z\in y)\to (z\in x)
  \end{equation*}
  holds. We have already noted that type theory is different from set theory in that terms in type theory come equipped with a \emph{unique} type. Moreover, in set theory the proposition $x\in y$ is well-formed for any two sets $x$ and $y$, whereas in type theory the judgment $a:A$ is only well-formed if it is derived using the postulated inference rules. Because of these differences we must find a different way to talk about subtypes.

  Note that in set theory there is a correspondence between the subsets of a set $x$, and the \emph{predicates} on $x$. A predicate on $x$ is just a proposition $P(z)$ that varies over the elements $z\in x$. Indeed, if $y$ is a subset of $x$, then the corresponding predicate is the proposition $z\in y$. Conversely, if $P$ is a predicate on $x$, then we obtain the subset
  \begin{equation*}
    \{z\in x\mid P(z)\}
  \end{equation*}
  of $x$. Now we have the right idea of subtypes in type theory: they are families of propositions.

\begin{defn}
A type family $B$ over $A$ is said to be a \define{subtype} of $A$ if for each $x:A$ the type $B(x)$ is a proposition.
\end{defn}

We will show in \cref{thm:subtype} that a type family $B$ over $A$ is a subtype of $A$ if and only if the projection map $\proj 1:\big(\sm{x:A}B(x)\big)\to A$ is an embedding.

\begin{comment}
\begin{samepage}
\begin{thm}\label{thm:subtype}
Let $B$ be a type family over $A$. The following are equivalent:
\begin{enumerate}
\item The family $B$ over $A$ is a \define{subtype} of $A$, in the sense that for each $x:A$ the type $B(x)$ is a proposition.
\item The projection map
\begin{equation*}
\proj 1 : \Big(\sm{x:A}B(x)\Big)\to A
\end{equation*}
is an embedding.
\end{enumerate}
\end{thm}
\end{samepage}

\begin{proof}
First assume that $B(x)$ is a proposition for each $x:A$. Our goal is to show that
\begin{equation*}
\apfunc{\proj 1} : (\id{s}{t})\to (\id{\proj 1(s)}{\proj 1(t)})
\end{equation*}
is an equivalence for every $s,t:\sm{x:A}B(x)$. By $\Sigma$-induction on $s$ and \cref{thm:id_fundamental} it suffices to show that the type
\begin{equation*}
\sm{t:\sm{x:A}B(x)} \id{a}{\proj 1(t)}
\end{equation*}
is contractible, for any $a:A$ and $b:B(a)$.
For the center of contraction we take $\pairr{\pairr{a,b},\refl{a}}$.
The contraction is constructed by applying $\Sigma$-induction twice, by which it suffices to construct a term of type
\begin{equation*}
\prd{x:A}{y:B(x)}{p:\id{a}{x}} \pairr{\pairr{a,b},\refl{a}}=\pairr{\pairr{x,y},p}.
\end{equation*}
This term is constructed by path induction on $p$, so it suffices to construct a term of type
\begin{equation*}
\prd{y:B(a)} \pairr{\pairr{a,b},\refl{a}}=\pairr{\pairr{a,y},\refl{a}}
\end{equation*}
However, the proposition $B(a)$ is contractible by \cref{cor:contr_prop}, since we have $b:B(a)$. Therefore we may proceed by singleton induction, so it suffices to construct an identification of type
\begin{equation*}
\pairr{\pairr{a,b},\refl{a}}=\pairr{\pairr{a,b},\refl{a}},
\end{equation*}
which we have by reflexivity. This completes the proof that if each $B(x)$ is a proposition, then the projection map $\proj 1 : \big(\sm{x:A}B(x)\big)\to A$ is an embedding.

For the converse, assume that the projection map is an embedding, and let $x:A$. Our goal is to show that $B(x)$ is a proposition. By \cref{lem:isprop_eq} it suffices to show that
\begin{equation*}
\prd{x:A}{y,z:B(x)} \id{y}{z}
\end{equation*}
Let $y,z:B(x)$. By our assumption that the projection map is an embedding we have an equivalence
\begin{equation*}
\eqv{(\id{\pairr{x,y}}{\pairr{x,z}})}{(\id{x}{x})}
\end{equation*}
In particular, we obtain an identification $p:\id{\pairr{x,y}}{\pairr{x,z}}$ which comes equipped with an identification $q:\ap{\proj 1}{p}=\refl{x}$. Now it follows that
\begin{equation*}
\begin{tikzcd}[column sep=huge]
y \arrow[r,equals,"\apfunc{\mathsf{tr}_B(\blank,y)}(q)"] & \mathsf{tr}_B(p,y) \arrow[r,equals,"\apd{\proj 2}{p}"] & z,
\end{tikzcd}
\end{equation*}
where $\apdfunc{\proj 2}$ is the \emph{dependent} action on paths of the dependent function $\proj 2:\prd{t:\sm{x:A}B(x)} B(\proj 1(t))$, constructed in \cref{defn:apd}.
\end{proof}

\begin{cor}
Let $f:A\to B$ be a map. The following are equivalent:
\begin{enumerate}
\item For each $y:B$, the fiber $\fib{f}{y}$ is a proposition.
\item $f$ is an embedding.
\end{enumerate}
\end{cor}

\begin{proof}
By \cref{ex:fib_replacement} there is a commuting triangle
\begin{equation*}
\begin{tikzcd}[column sep=large]
A \arrow[rr,"\lam{a}\pairr{f(a),\pairr{a,\refl{f(a)}}}"] \arrow[dr,swap,"f"] & & \sm{y:B}\fib{f}{y} \arrow[dl,"\proj 1"] \\
& B
\end{tikzcd}
\end{equation*}
in which the top map is an equivalence. Thus it follows from \cref{ex:emb_triangle} that $f$ is an embedding if and only if $\proj 1:\big(\sm{y:B}\fib{f}{y}\big)\to B$ is an embedding. Now the claim follows from \cref{thm:subtype}.
\end{proof}
\end{comment}

\subsection{Sets}

\begin{defn}
A type $A$ is said to be a \define{set} if there is a term of type
\begin{equation*}
\isset(A)\defeq \prd{x,y:A}\isprop(\id{x}{y}).
\end{equation*}
\end{defn}

\begin{lem}
A type $A$ is a set if and only if it satisfies \define{axiom K}, which asserts that
\begin{equation*}
\prd{x:A}{p:\id{x}{x}}\id{\refl{x}}{p}.
\end{equation*}
\end{lem}

\begin{proof}
If $A$ is a set, then $\id{x}{x}$ is a proposition, so any two of its elements are equal.
This implies axiom $K$.

For the converse, if $A$ satisfies axiom $K$, then for any $p,q:\id{x}{y}$ we have $\id{\ct{p}{q^{-1}}}{\refl{x}}$, and hence $\id{p}{q}$. This shows that $\id{x}{y}$ is a proposition, and hence that $A$ is a set.
\end{proof}

\begin{thm}\label{lem:prop_to_id}
Let $A$ be a type, and let $R:A\to A\to\UU$ be a binary relation on $A$ satisfying
\begin{enumerate}
\item Each $R(x,y)$ is a proposition,
\item $R$ is reflexive, as witnessed by $\rho:\prd{x:A}R(x,x)$,
\item There is a map
  \begin{equation*}
    R(x,y)\to (x=y)
  \end{equation*}
  for each $x,y:A$.
\end{enumerate}
Then any family of maps
\begin{equation*}
\prd{x,y:A}(\id{x}{y})\to R(x,y)
\end{equation*}
is a family of equivalences. Consequently, the type $A$ is a set.
\end{thm}

\begin{proof}
Let $f:\prd{x,y:A}R(x,y)\to(\id{x}{y})$.
Since $R$ is assumed to be reflexive, we also have a family of maps
\begin{equation*}
\mathsf{path\usc{}ind}_x(\rho(x)):\prd{y:A}(\id{x}{y})\to R(x,y).
\end{equation*}
Since each $R(x,y)$ is assumed to be a proposition, it therefore follows that each $R(x,y)$ is a retract of $\id{x}{y}$. Therefore it follows that $\sm{y:A}R(x,y)$ is a retract of $\sm{y:A}x=y$, which is contractible. We conclude that $\sm{y:A}R(x,y)$ is contractible, and therefore that any family of maps
\begin{equation*}
  \prd{y:A}(x=y)\to R(x,y)
\end{equation*}
is a family of equivalences.

Now it also follows that $A$ is a set, since its identity types are equivalent to propositions, and therefore they are propositions by \cref{lem:prop_equiv}.
\end{proof}

\begin{defn}
  A map $f:A\to B$ is said to be \define{injective} if for any $x,y:A$ there is a map
  \begin{equation*}
    (f(x)=f(y))\to (x=y).
  \end{equation*}
\end{defn}

\begin{cor}
  Any injective map into a set is an embedding.
\end{cor}

\begin{proof}
  Let $f:A\to B$ be an injective map between sets. Now consider the relation
  \begin{equation*}
    R(x,y)\defeq (f(x)=f(y)).
  \end{equation*}
  Note that $R$ is reflexive, and that $R(x,y)$ is a proposition for each $x,y:A$. Moreover, by the assumption that $f$ is injective, we have
  \begin{equation*}
    R(x,y)\to (x=y)
  \end{equation*}
  for any $x,y:A$. Therefore we are in the situation of \cref{lem:prop_to_id}, so it follows that the map $\apfunc{f} : (x=y)\to (f(x)=f(y))$ is an equivalence.
\end{proof}

\begin{thm}\label{thm:eq_nat}
The type of natural numbers is a set.
\end{thm}

\begin{proof}
We will apply \cref{lem:prop_to_id}. Note that the observational equality $\mathsf{Eq}_\N:\N\to(\N\to\UU)$ on $\N$ (\cref{defn:obs_nat}) is a reflexive relation by \cref{ex:obs_nat_eqrel}, and moreover that $\mathsf{Eq}_\N(n,m)$ is a proposition for every $n,m:\N$ (proof by double induction).
Therefore it suffices to show that
\begin{equation*}
\prd{m,n:\nat}\mathsf{Eq}_\N(m,n)\to (\id{m}{n}).
\end{equation*}
This follows from the fact that observational equality is the \emph{least} reflexive relation, which was shown in \cref{ex:obs_nat_least}.
\end{proof}

\begin{comment}
\begin{thm}[Hedberg]\label{thm:dec_eq}
Any type with decidable equality is a set.
\end{thm}

\begin{proof}
Let $A$ be a type, and let $d:\prd{x,y:A}(\id{x}{y})+\neg(\id{x}{y})$ be the witness that $A$ has decidable equality.
We first construct a reflexive binary relation $E:A\to A\to\type$ such that each $E(x,y)$ is a proposition.
For every $x,y:A$, we first define a type family $E'(x,y):((\id{x}{y})+\neg(\id{x}{y}))\to\type$ by
\begin{align*}
E'(x,y,\inl(p)) & \defeq \unit \\
E'(x,y,\inr(p)) & \defeq \emptyt.
\end{align*}
Note that $E'(x,y,q)$ is a proposition for each $x,y:A$ and $q:(\id{x}{y})+\neg(\id{x}{y})$.
Now we set $E(x,y)\defeq E'(x,y,d(x,y))$. Then $E$ is clearly reflexive, and a family of propositions.
Therefore it remains to show that $E$ implies identity.

Since $E$ is defined as an instance of $E'$, it suffices to construct a term of type
\begin{equation*}
\prd{x,y:A}{q:(\id{x}{y})+\neg(\id{x}{y})} E'(q)\to (\id{x}{y}).
\end{equation*}
By induction of disjoint sums, it suffices to construct terms of types
\begin{align*}
& \prd{x,y:A}{p:\id{x}{y}} \unit\to (\id{x}{y}) \\
& \prd{x,y:A}{p:\neg(\id{x}{y})} \emptyt\to (\id{x}{y}).
\end{align*}
In the first case, we take $\lam{x}{y}{p}{t}p$, and the second case is by induction on the empty type.
\end{proof}
\end{comment}

\subsection{General truncation levels}
\begin{defn}
We define $\istrunc{} : \Z_{\geq-2}\to\UU\to\UU$ by induction on $k:\Z_{\geq -2}$, taking
\begin{align*}
\istrunc{-2}(A) & \defeq \iscontr(A) \\
\istrunc{k+1}(A) & \defeq \prd{x,y:A}\istrunc{k}(\id{x}{y}).\qedhere
\end{align*}
For any type $A$, we say that $A$ is \define{$k$-truncated}, or a \define{$k$-type}, if there is a term of type $\istrunc{k}(A)$. We say that a map $f:A\to B$ is $k$-truncated if its fibers are $k$-truncated.
\end{defn}

%For the rest of this section, let $k:\Z_{\geq-2}$.

\begin{thm}\label{thm:istrunc_next}
If $A$ is a $k$-type, then $A$ is also a $(k+1)$-type.
\end{thm}

\begin{proof}
We have seen in \cref{eg:prop_contr} that contractible types are propositions. This proves the base case.
For the inductive step, note that if any $k$-type is also a $(k+1)$-type, then any $(k+1)$-type is a $(k+2)$-type, since its identity types are $k$-types and therefore $(k+1)$-types.
\end{proof}

\begin{thm}\label{thm:ktype_eqv}
If $e:\eqv{A}{B}$ is an equivalence, and $B$ is a $k$-type, then so is $A$.
\end{thm}

\begin{proof}
We have seen in \cref{ex:contr_equiv} that if $B$ is contractible and $e:\eqv{A}{B}$ is an equivalence, then $A$ is also contractible. This proves the base case.

For the inductive step, assume that the $k$-types are stable under equivalences, and consider $e:\eqv{A}{B}$ where $B$ is a $(k+1)$-type. In \cref{cor:emb_equiv} we have seen that
\begin{equation*}
\apfunc{e}:(\id{x}{y})\to(\id{e(x)}{e(y)})
\end{equation*}
is an equivalence for any $x,y$. Note that $\id{e(x)}{e(y)}$ is a $k$-type, so by the induction hypothesis it follows that $\id{x}{y}$ is a $k$-type. This proves that $A$ is a $(k+1)$-type.
\end{proof}

\begin{cor}\label{cor:emb_into_ktype}
If $f:A\to B$ is an embedding, and $B$ is a $(k+1)$-type, then so is $A$.
\end{cor}

\begin{proof}
By the assumption that $f$ is an embedding, the action on paths
\begin{equation*}
\apfunc{f}:(\id{x}{y})\to (\id{f(x)}{f(y)})
\end{equation*}
is an equivalence for every $x,y:A$. Since $B$ is assumed to be a $(k+1)$-type, it follows that $f(x)=f(y)$ is a $k$-type for every $x,y:A$. Therefore we conclude by \cref{thm:ktype_eqv} that $\id{x}{y}$ is a $k$-type for every $x,y:A$. In other words, $A$ is a $(k+1)$-type.
\end{proof}

In the following definition we generalize the notion of contractible map.

\begin{defn}
We say that a map $f:A\to B$ is \define{$k$-truncated} if for each $y:B$ the fiber $\fib{f}{y}$ is $k$-truncated.
\end{defn}

\begin{thm}
Let $B$ be a type family over $A$. Then the following are equivalent:
\begin{enumerate}
\item For each $x:A$ the type $B(x)$ is $k$-truncated.
\item The projection map
\begin{equation*}
\proj 1 : \Big(\sm{x:A}B(x)\Big)\to A
\end{equation*}
is $k$-truncated.
\end{enumerate}
\end{thm}

\begin{proof}
By \cref{ex:proj_fiber} we obtain equivalences
\begin{equation*}
\eqv{\fib{\proj 1}{x}}{B(x)}
\end{equation*}
for every $x:A$. Therefore the claim follows from \cref{thm:ktype_eqv}.
\end{proof}

\begin{thm}\label{thm:trunc_ap}
Let $f:A\to B$ be a map. The following are equivalent:
\begin{enumerate}
\item The map $f$ is $(k+1)$-truncated.
\item For each $x,y:A$, the map
\begin{equation*}
\apfunc{f} : (x=y)\to (f(x)=f(y))
\end{equation*}
is $k$-truncated.
\end{enumerate}
\end{thm}

\begin{proof}
First we show that for any $s,t:\fib{f}{b}$ there is an equivalence
\begin{equation*}
\eqv{(s=t)}{\fib{\apfunc{f}}{\ct{\proj 2(s)}{\proj 2(t)^{-1}}}}
\end{equation*}
We do this by $\Sigma$-induction on $s$ and $t$, and then we calculate using \cref{ex:trans_ap} and basic manipulations of identifications that
\begin{align*}
(\pairr{x,p}=\pairr{y,q}) & \eqvsym \sm{r:x=y} \mathsf{tr}_{f(\blank)=b}(r,p)=q \\
& \eqvsym \sm{r:x=y} \ct{\ap{f}{r}^{-1}}{p}=q \\
& \eqvsym \sm{r:x=y} \ap{f}{r}=\ct{p}{q^{-1}} \\
& \jdeq \fib{\apfunc{f}}{\ct{p}{q^{-1}}}.
\end{align*}
By these equivalences, it follows that if $\apfunc{f}$ is $k$-truncated, then for each $s,t:\fib{f}{b}$ the identity type $s=t$ is equivalent to a $k$-truncated type, and therefore we obtain by \cref{thm:ktype_eqv} that $f$ is $(k+1)$-truncated.

For the converse, note that we have equivalences
\begin{align*}
\fib{\apfunc{f}}{p} & \eqvsym ((x,p)=(y,\refl{f(y)})).
\end{align*}
It follows that if $f$ is $(k+1)$-truncated, then the identity type $(x,p)=(y,\refl{f(y)})$ in $\fib{f}{f(y)}$ is $k$-truncated for any $p:f(x)=f(y)$. We conclude by \cref{thm:ktype_eqv} that the fiber $\fib{\apfunc{f}}{p}$ is $k$-truncated.
\end{proof}

\begin{cor}\label{cor:prop_emb}
A map is an embedding if and only if its fibers are propositions.
\end{cor}

\begin{cor}\label{thm:subtype}
A type family $B$ over $A$ is a subtype if and only if the projection map
\begin{equation*}
\proj 1 : \Big(\sm{x:A}B(x)\Big)\to A
\end{equation*}
is an embedding.
\end{cor}

\begin{thm}
Let $f:\prd{x:A}B(x)\to C(x)$ be a family of maps. Then the following are equivalent:
\begin{enumerate}
\item For each $x:A$ the map $f(x)$ is $k$-truncated.
\item The induced map
\begin{equation*}
\total{f}:\Big(\sm{x:A}B(x)\Big)\to\Big(\sm{x:A}C(x)\Big)
\end{equation*}
is $k$-truncated.
\end{enumerate}
\end{thm}

\begin{proof}
This follows directly from \cref{lem:fib_total,thm:ktype_eqv}.
\end{proof}

\begin{comment}
\begin{proof}
By \cref{ex:contr_retr} it follows that if $A$ is a retract of a contractible type, then $A$ is contractible.
For the inductive step, suppose that the $k$-types are closed under retracts, and consider a section-retraction pair
\begin{equation*}
\begin{tikzcd}
A \arrow[r,"i"] & B \arrow[r,"r"] & A,
\end{tikzcd}
\end{equation*}
with $H:r\circ i\htpy \idfunc$, where $B$ is a $(k+1)$-type.
By the induction hypothesis it suffices to show that for any $x,y:A$, the function $\apfunc{i}:(\id{x}{y})\to (\id{i(x)}{i(y)})$ has a retraction.
The retraction $\varphi:(\id{i(x)}{i(y)})\to(\id{x}{y})$ is defined as
\begin{equation*}
\varphi \defeq \lam{q} \ct{H(x)^{-1}}{\ap{r}{q}}{H(y)}
\end{equation*}
To see that $\varphi(\ap{i}{p})=p$, we have to show that the square
\begin{equation*}
\begin{tikzcd}
r(i(x)) \arrow[d,equals,swap,"\ap{r}{q}"] \arrow[r,equals,"H(x)"] & x \arrow[d,equals,"p"] \\
r(i(y)) \arrow[r,equals,swap,"H(y)"] & y
\end{tikzcd}
\end{equation*}
commutes. This square commutes by the naturality of homotopies, proven in \cref{ex:htpy_nat}.
\end{proof}
\end{comment}

\begin{exercises}
\item
  \begin{subexenum}
  \item Show that $\succN:\N\to\N$ is an embedding.
  \item Show that $n\mapsto m+n$ is an embedding, for each $m:\N$. Moreover, conclude that there is an equivalence
    \begin{equation*}
      \fib{\mathsf{add}_\N(m)}{n}\simeq (m\leq n).
    \end{equation*}
  \item Show that $n\mapsto mn$ is an embedding, for each $m>0$ in $\N$. Conclude that the divisibility relation
    \begin{equation*}
      d\mid n
    \end{equation*}
    is a proposition for each $d,n:\N$ such that $d>0$.
  \end{subexenum}
\item \label{ex:diagonal}Let $A$ be a type, and let the \define{diagonal} of $A$ be the map $\delta_A:A\to A\times A$ given by $\lam{x}(x,x)$.
\begin{subexenum}
\item Show that
\begin{equation*}
{\isequiv(\delta_A)}\leftrightarrow{\isprop(A)}.
\end{equation*}
\item Construct an equivalence $\eqv{\fib{\delta_A}{(x,y)}}{(x=y)}$ for any $x,y:A$.
\item Show that $A$ is $(k+1)$-truncated if and only if $\delta_A:A\to A\times A$ is $k$-truncated.
\end{subexenum}
\item \label{ex:istrunc_sigma}
\begin{subexenum}
\item Let $B$ be a type family over $A$. Show that if $A$ is a $k$-type, and $B(x)$ is a $k$-type for each $x:A$, then so is $\sm{x:A}B(x)$. Conclude that for any two $k$-types $A$ and $B$, the type $A\times B$ is also a $k$-type. Hint: for the base case, use \cref{ex:contr_in_sigma,ex:contr_equiv}.
\item Show that for any $k$-type $A$, the identity types of $A$ are also $k$-types.
\item Show that any maps $f:A\to B$ between $k$-types $A$ and $B$
is a $k$-truncated map.
\item Use \cref{ex:proj_fiber} to show that for any type family $B:A\to \UU$, if $A$ and $\sm{x:A}B(x)$ are $k$-types, then so is $B(x)$ for each $x:A$.
\end{subexenum}
\item \label{ex:eq_bool}Show that $\bool$ is a set by applying \cref{lem:prop_to_id} with the observational equality on $\bool$ defined in \cref{ex:obs_bool}.
\item \label{ex:set_coprod}Show that for any two $(k+2)$-types $A$ and $B$, the disjoint sum $A+B$ is again a $(k+2)$-type. Conclude that $\mathbb{Z}$ is a set.
\item Use \cref{ex:contr_retr,ex:retr_id} to show that if $A$ is a retract of a $k$-type $B$, then $A$ is also a $k$-type.
\item Show that a type $A$ is a $(k+1)$-type if and only if the map $\mathsf{const}_x:\unit\to A$ is $k$-truncated for every $x:A$.
\item Consider a commuting triangle
\begin{equation*}
\begin{tikzcd}[column sep=tiny]
A \arrow[rr,"h"] \arrow[dr,swap,"f"] & & B \arrow[dl,"g"] \\
& X
\end{tikzcd}
\end{equation*}
with $H: f \htpy g \circ h$, and suppose that $g$ is $k$-truncated. Show that $f$ is $k$-truncated if and only if $h$ is $k$-truncated.
% Suppose that $h$ is $k$-truncated and surjective. Show that $f$ is $(k+1)$-truncated if and only if $g$ is $(k+1)$-truncated.
\end{exercises}


\chapter{Introduction}
\label{chap:intro}

\textbf{by Bastian Boll} \\

There is a constant need to generate subtitles for video. For large video quantities such as on sites like YouTube, machine learning systems have been employed to perform this task \cite{youtubeFAQ}.

In this work, we aim to solve a related problem: given an audio file $A$ and a finished transcript $\mathcal{T}$, we try to compute time alignment information. Let $\mathcal{T}$ be an $l$-tuple of words and $A$ be a spoken-word audio signal with known length such that every word $w$ in $\mathcal{T}$ occurs in $A$ at time $t_w$ . We try to find the mapping
\begin{align*}
	f&\colon\textrm{Transcripts}_l\to\R^l\\
	\mathcal{T}&\mapsto f(\mathcal{T})\textrm{ such that }(f(\mathcal{T}))_w=t_w
\end{align*}
Because the transcript is given, this problem is easier to solve than general speech-to-text. Specifically, a respective model does not need to solve the tasks addressed by the language model which is typically present in a speech-to-text system \cite{anusuya2010speech}.

However, a good solution for the above problem is still a useful tool in generating subtitles as the laborious process of aligning a transcript to the video could be automated.

\section{The dataset}

\textbf{by Bastian Boll} \\

We use a dataset consisting of 2436 \href{https://www.ted.com}{TED-talks}. Part of the metadata for these talks can be found in \href{https://www.kaggle.com/rounakbanik/ted-talks}{a kaggle dataset}. Additional data, such as subtitles and audio files can be aquired from the TED website. Both the subtitles and the audio are permitted to be used for the described purpose by their \href{https://www.ted.com/about/our-organization/our-policies-terms/ted-talks-usage-policy}{usage policy and license}.\\
The dataset contains approximately 560 hours of high quality English spoken audio recordings. Speakers come from a very diverse international pool. The used language is relatively erudite. Both subtitles and transcripts for each talk are of consistently good quality.\\
This dataset was chosen to be representative of the type of data which can be cheaply acquired but still contains structural information with regard to the problem at hand. This specifically represents a tradeoff between data accuracy and available data quantity. Most acoustic modelling approaches do not predict words directly. Instead, sequences of phonemes or even sub-phonetic states (as in e.g. \cite{maas2015building}) are predicted from short intervals of an audio signal to be used in conjunction with a language model. Using much more coarsely labeled data from sources such as the one we chose prohibits the use of single phonemes or sub-phonetic states as labels. It is therefore to be expected that any model constructed to work with this data is presented with a fairly hard classification task. On the other hand, one can try and make up for this challenge by being able to source form an extremely large pool of available data. In chapter \ref{chap:algorithm} we also describe how pre-existing knowledge such as word order can be effectively employed to reduce the accuracy requirements for the language model.

\section{Implementation}

\textbf{by Bastian Boll} \\

All presented methods are implemented in Python 3.6 making prevalent use of \href{https://www.tensorflow.org/}{Tensorflow 1.4} and \href{http://www.numpy.org/}{numpy}. The source code is available at \url{https://github.com/bbboll/ml_subtitle_align}. The \emph{Readme.md} provides additional information on how to get started.


\chapter{Gradient Boosting \label{chapter:gradientboosting}}

For several years, no one knew why AdaBoost (Chapter~\ref{chapter:boosting}) worked. Then, in 2001, Jerome Friedman published a paper on \textbf{gradient boosting}, which generalized the boosting idea to many problem classes. In the gradient boosting framework, of which AdaBoost is a subclass, there are three components:

\begin{enumerate}
\item A loss function to be optimized.
\item Weak learners to make predictions.
\item An additive model that adds the contributions of different weak learners to minimize the loss function.
\end{enumerate}

Given a predefined loss function and a method for building new weak learners, gradient boosting provides a principled way to create new learners and add them to the combined model. It can be used to boost learners for a wide array of different objective functions.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Gradient Boosting for Classification}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Gradient Boosting for Regression}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Gradient Boosting for Survival Analysis}


\documentclass{article}
% Packages
\usepackage{booktabs}
\usepackage[table]{xcolor}
\usepackage[fixamsmath]{mathtools}
    \newcommand{\vecw}{\mathbf{w}}
    \newcommand{\vecx}{\mathbf{x}}
    \newcommand{\vecy}{\mathbf{y}}
    \newcommand{\vecz}{\mathbf{z}}
    \newcommand{\vecp}{\mathbf{p}}
    \newcommand{\rvx}{\mathbf{X}}
    \newcommand{\matx}{\mathbf{X}}
    \newcommand{\matd}{\mathbf{D}}
    \newcommand{\dd}{\mathrm{d}}
    \newcommand{\diff}[2]{\frac{\dd #1}{\dd #2}}
    \newcommand{\partialdd}[2]{\frac{\partial #1}{\partial #2}}
\usepackage{unicode-math}
    \unimathsetup{math-style=ISO, bold-style=ISO, mathrm=sym}
    \setsansfont{FiraGO}[BoldFont=* SemiBold, Numbers=Monospaced]
    \setmathfont{Fira Math}
\usepackage{hyperref}
    \hypersetup{
        pdfauthor={lenovo},
        pdfcreator={Microsoft® Word 2017},
        pdfproducer={Microsoft® Word 2017},
    }
\usepackage{tikz}
    \usetikzlibrary{ducks,external}
\usepackage{fancyhdr}
    \pagestyle{fancy}
    \fancyhead{}
        \renewcommand{\headrulewidth}{0pt}
    \fancyfoot[C]{%
        \tikzset{external/export=false}%
        \shuffleducks
        \begin{tikzpicture}[scale=0.5]
            \duck[signpost=\scalebox{0.6}{\thepage},\randomhead]
        \end{tikzpicture}
    }
\usepackage{geometry}
    \geometry{margin=1in}
\usepackage{tcolorbox}
    \tcbuselibrary{listings,breakable}
    \newtcblisting{PyListing}[1]{%
        listing options={language=Python,numbers=left,breaklines,
            numberstyle=\tiny\color{red!75!black},firstnumber=last,
            basicstyle=\small,keywordstyle=\color{blue!85!black},
            commentstyle=\color{white!75!black}\textit},
        title=\texttt{#1},listing only,breakable,
        left=6mm,right=6mm,top=2mm,bottom=2mm,
        colback=violet!5!white,colframe=violet!75!black}
    \newtcblisting{Output}{%
        title=\texttt{Output},listing only,breakable,
        left=6mm,right=6mm,top=2mm,bottom=2mm,
        colback=blue!5!white,colframe=blue!75!black}
    \tcbuselibrary{xparse}
        \DeclareTotalTCBox{\verbbox}{ O{orange} v !O{} }{
            fontupper=\ttfamily,nobeforeafter,tcbox raise base,%
            arc=0pt,outer arc=0pt,top=0pt,bottom=0pt,left=0mm,%
            right=0mm,leftrule=0pt,rightrule=0pt,toprule=0.3mm,%
            bottomrule=0.3mm,boxsep=0.5mm,bottomrule=0.3mm,boxsep=0.5mm,%
            colback=#1!10!white,colframe=#1!50!black,#3}{#2}%
    \DeclareTotalTCBox{\commandbox}{ s v }{
            verbatim,colupper=white,colback=black!75!white,colframe=black}{
                \IfBooleanTF{#1}{\textcolor{red}{\ttfamily\bfseries >> }}{}%
                    \lstinline[language=sh,morekeywords={python,python2,python3},keywordstyle=\color{blue!35!white}\bfseries]^#2^}%
% Information
\title{Project 1}
\author{Iydon Leong}
\date{April 3, 2019}
\begin{document}
\maketitle

% Question 1
\section{Question 1}\label{S:1}
According to the question, we can write equation~\eqref{E:1-1}.
\begin{equation}\label{E:1-1}
    \begin{dcases}
        p_1(x;\vecw) = P(y=1|\rvx=\vecx) = \frac{\exp(\vecw^T\vecx)}{1+\exp(\vecw^Tx)} \\
        p_0(x;\vecw) = P(y=0|\rvx=\vecx) = \frac{1}{1+\exp(\vecw^Tx)}
    \end{dcases}
\end{equation}

Therefore, we can conclude that
\begin{equation*}
    p_{t}(x;\vecw) = P(y=t|\rvx=\vecx) = \frac{\exp(t\vecw^T\vecx)}{1+\exp(\vecw^Tx)},\quad\text{where $t\in\{0,1\}$.}
\end{equation*}

Then the log-likelihood function can be rewritten as equation~\eqref{E:1-2}.
\begin{equation}\label{E:1-2}
    \begin{aligned}
        l(\vecw) &= \sum_{i=1}^n\log\left[\frac{\exp(y_i\vecw^T\vecx_i)}{1+\exp(\vecw^T\vecx_i)}\right] \\
                 &= \sum_{i=1}^n\left[y_i\vecw^T\vecx_i-\log\left(1+e^{\vecw^T\vecx_i}\right)\right]
    \end{aligned}
\end{equation}

% Question 2
\section{Question 2}\label{S:2}
Since $\vecw=(w_0,w_1,\ldots,w_d)^T$ and $\vecx=(x_0,x_1,\ldots,x_d)^T$, then we have equation~\eqref{E:2-1}.
\begin{equation}\label{E:2-1}
    l(w_j) = \sum_{i=1}^n\left[y_iw_jx_{ij}-\log\left(1+e^{w_jx_{ij}}\right)\right],\quad j=1,2,\ldots,d.
\end{equation}

To find the maxima of each log-likelihood function, we just let $\partialdd{l(w_j)}{w_j}=0$, where $j=1,2,\ldots,d$. That is,
\begin{equation}\label{E:2-2}
    \begin{aligned}
        0 &= \partialdd{l(w_j)}{w_j} \\
          &= \sum_{i=1}^n\left[y_ix_{ij}-\frac{x_{ij}e^{w_jx_{ij}}}{1+e^{w_jx_{ij}}}\right] \\
          &= \sum_{i=1}^nx_{ij}\left(y_i-p(\vecx_i;\vecw)\right) \\
          &= \vecx_{\cdot j}^T\left(\vecy-p(\vecx_i;\vecw)\right).
    \end{aligned}
\end{equation}

Moreover, we can get the vector form
\begin{equation}\label{E:2-3}
    \partialdd{l(\vecw)}{\vecw} = \matx^T\left(\vecy-p(\matx;\vecw)\right).
\end{equation}

% Question 3
\section{Question 3}\label{S:3}
From equation~\eqref{E:2-2}, we have the specific forms of $\partialdd{l(w_j)}{w_j}$, where $j=1,2,\ldots,d$. In addition, we can derive the expression of $\partialdd{^2l(w_j)}{w_j^2}$.
\begin{equation}\label{E:3-1}
    \begin{aligned}
        \partialdd{^2l(w_j)}{w_j^2} &= \partialdd{}{w_j}\partialdd{l(w_j)}{w_j} \\
                                    &= \sum_{i=1}^n-\frac{x_{ij}x_{ij}e^{w_jx_{ij}}\left(1+e^{w_jx_{ij}}\right)-x_{ij}e^{w_jx_{ij}}x_{ij}e^{w_jx_{ij}}}{\left(1+e^{w_jx_{ij}}\right)^2} \\
                                    &= \sum_{i=1}^n-\frac{x_{ij}^2e^{w_jx_{ij}}}{\left(1+e^{w_jx_{ij}}\right)^2}
    \end{aligned}
\end{equation}

We can rewrite equation~\eqref{E:3-1} with vector form.
\begin{equation}\label{E:3-2}
    \begin{aligned}
        \partialdd{^2l(\vecw)}{\vecw\partial\vecw^T} &= -\sum_{i=1}^n\vecx_i\vecx_i^Tp\left(\vecx_i;\vecw\right)\left(1-p(\vecx_i;\vecw)\right) \\
                                                     &= \matx^T\Lambda\matx
    \end{aligned}
\end{equation}
where $\Lambda$ is a $n\times n$ diagonal matrix with diagonal entries $p\left(\vecx_i;\vecw\right)\left(1-p(\vecx_i;\vecw)\right)$.

% Question 4
\section{Question 4}\label{S:4}
\begin{equation}\label{E:4-1}
    \vecw^{(k)} = \arg\min_{\vecw}(\vecz-\matx\vecw)^T\matd(\vecz-\matx\vecw)
\end{equation}

First of all, we should know the size of matrix and vector to avoid incompatible operation. This can be checked through table~\ref{T:4}.
\begin{table}[htb]
    \centering
    \begin{tabular}{cm{7em}cm{15em}}
        \toprule
        Symbols   & Visual Size                         & Size        & Description \\ \midrule
        \rowcolor[HTML]{EFEFEF}
        $\vecp$   & \textcolor{violet!50!white}{\rule{1em}{7em}} & $n\times 1$ & the column vector of $p(\vecx_i;\vecw^{(k-1)})$ \\
        $\vecw$   & \textcolor{violet!50!white}{\rule{1em}{3em}} & $d\times 1$ & weight \\
        \rowcolor[HTML]{EFEFEF}
        $\vecx_i$ & \textcolor{violet!50!white}{\rule{1em}{3em}} & $d\times 1$ & the column vector sample $x_i$ \\
        $\vecy$   & \textcolor{violet!50!white}{\rule{1em}{7em}} & $n\times 1$ & the column vector of $y_i$ \\
        \rowcolor[HTML]{EFEFEF}
        $\vecz$   & \textcolor{violet!50!white}{\rule{1em}{7em}} & $n\times 1$ & $\matx\vecw^{(k-1)}+\matd^{-1}(\vecy-\vecp)$ \\
        $\matd$   & \textcolor{violet!50!white}{\rule{7em}{7em}} & $n\times n$ & matrix with diagonal entries as $p(\vecx_i;\vecw^{(k-1)})(1-p(\vecx_i;\vecw^{(k-1)}))$ \\
        \rowcolor[HTML]{EFEFEF}
        $\matx$   & \textcolor{violet!50!white}{\rule{3em}{7em}} & $n\times d$ & matrix with row entries as $\vecx_i$ \\ \bottomrule
    \end{tabular}
    \caption{Visual Size of Matrics and Vectors}\label{T:4}
\end{table}

Inspired by the least-square algorithm, we then differentiate the evaluation function at equation~\eqref{E:4-2}.
\begin{equation}\label{E:4-2}
    \begin{aligned}
        \partialdd{(\vecz-\matx\vecw)^T\matd(\vecz-\matx\vecw)}{\vecw} &= -2\matx^T\matd(\vecz-\matx\vecw) \\
                                                                       &= 2\matx^T\matd\matx\vecw-2\matx^T\matd\vecz \\
                                                                       &= 0
    \end{aligned}
\end{equation}
where $\vecz=\matx\vecw^{(k-1)}+\matd^{-1}(\vecy-\vecp)$. Obviously, we can get the equation~\eqref{E:4-3} by solving $(\matx^T\matd\matx)\vecw = \matx^T\matd\vecz$.
\begin{equation}\label{E:4-3}
    \begin{aligned}
        \vecw &= (\matx^T\matd\matx)^{-1}\matx^T\matd\vecz \\
              &= (\matx^T\matd\matx)^{-1}\matx^T\matd\left(\matx\vecw^{(k-1)}+\matd^{-1}(\vecy-\vecp)\right) \\
              &= \vecw^{(k-1)}+(\matx^T\matd\matx)^{-1}\matx^T(\vecy-\vecp)
    \end{aligned}
\end{equation}

Back to question~\ref{S:2} and question~\ref{S:3}, or exactly, equation~\eqref{E:2-3} and equation~\eqref{E:3-2}.
\begin{equation}\label{E:4-4}
    \vecw^{(k)} = \vecw^{(k-1)} - \left[\left(\partialdd{^2l(\vecw)}{\vecw\partial\vecw^T}\right)^{-1}\partialdd{l(\vecw)}{\vecw}\right]_{\vecw=\vecw^{(k-1)}}
\end{equation}

That is, the new notations, equation~\eqref{E:4-4} in Newton-Raphson method can be rewritten as equation~\eqref{E:4-1}, where $\vecz=\matx\vecw^{(k-1)}+\matd^{-1}(\vecy-\vecp)$. This is the so-called iteratively reweighted least square(RLS) algorithm.

\clearpage

% Question 5-7
\section{Question 5--7}\label{S:5-7}
There are two \verbbox{Python} programs, one implements the IRLS algorithm, the other compute and compare the results between my own program and the sklearn program. The result is shown at the end.
\begin{PyListing}{logistic\_regression.py}
from numpy import array, diag, divide, exp, matrix, power, zeros
from numpy.linalg import inv, norm

class logistic_regression_IRLS(object):
    """
    Maximum likelihood approach for logistic regression
    """
    def __init__(self):
        self.w = None

    def fit(self, X, y, w0=None, erf=None, maxit=None):
        self.__newton_raphson(X, y, w0, erf, maxit)

    def predict(self, X):
        y = X * self.w
        return array((y>0).astype(int).T)[0]

    def __newton_raphson(self, X, y, w0=None, erf=None, maxit=None):
        """
        Args:
            X:   nxd matrix.
            y:   nx1 column vector.
            w0:  dx1 column vector.
            erf: error function with two parameters.
                 erf = lambda old,new: ...
        """
        # Pre-process
        X   = matrix(X)
        n,d = X.shape
        y   = matrix(y).reshape(n, 1)
        if w0==None:
            w0 = zeros((d, 1))
        if erf==None:
            erf = lambda old,new: norm(old-new,2)<1e-6
        if maxit==None:
            maxit = 64
        w1 = zeros((d, 1))
        ex = zeros((n, 1))
        # Iterations
        for i in range(maxit):
            ex = exp(X*w0).T
            D  = diag(array(ex/power(1+ex,2))[0])
            w1 = w0 + inv(X.T*D*X)*X.T*(y-(ex/(1+ex)).T)
            if erf(w0, w1):
                print("Convergence @ the %d iteration(s)."%i)
                break
            w0 = w1
        self.w = w1
\end{PyListing}

\begin{PyListing}{main.py}
import numpy as np

from logistic_regression import logistic_regression_IRLS
from sklearn.metrics import confusion_matrix, accuracy_score, precision_score, recall_score

from sklearn.linear_model import LogisticRegression as Regression

def read_dat(path):
    # raw text
    encode = "utf-8"
    with open(path, "r", encoding=encode) as f:
        while True:
            content = f.readline()
            if content[0] != "@":
                break
        content += f.read()
    # analysis
    data = [line.split(", ") for line in content.splitlines()]
    one_hot = {"Absent":0, "Present":1}
    for i,k in enumerate(data):
        data[i][4] = one_hot.get(k[4], -1)
    data = np.array(data, dtype=float)
    # input and output
    return data[:,:-1], data[:,-1].astype(int)

def evaluation(y_true, y_pred):
    # data
    c = confusion_matrix(y_true, y_pred)
    a = accuracy_score(y_true, y_pred)
    p = precision_score(y_true, y_pred)
    r = recall_score(y_true, y_pred)
    # output
    print("Accuracy classification score:  %.3f"%a)
    print("Precision classification score: %.3f"%p)
    print("Recal classification score:     %.3f"%r)
    print("Confusion matrix:\n%s"%c)

for i in range(5):
    train = "saheart-5-%dtra.dat"%(i+1)
    test  = "saheart-5-%dtst.dat"%(i+1)
    X_train,y_train = read_dat(train)
    X_test,y_test   = read_dat(test)

    model1 = logistic_regression_IRLS()
    model1.fit(X_train, y_train)

    model2 = Regression(solver="newton-cg")
    model2.fit(X_train, y_train)

    y_pred1 = model1.predict(X_test)
    y_pred2 = model2.predict(X_test)

    print(train, test)
    print("-"*37)
    print("IRIS Algorithm" + "-"*23)
    evaluation(y_test, y_pred1)
    print("LogisticRegression" + "-"*19)
    evaluation(y_test, y_pred2)
    print("-"*37, "\n"*3)
\end{PyListing}

\twocolumn
\begin{Output}
saheart-5-1tra.dat saheart-5-1tst.dat
-------------------------------------
IRIS Algorithm-----------------------
Accuracy classification score:  0.731
Precision classification score: 0.684
Recal classification score:     0.406
Confusion matrix:
[[55  6]
 [19 13]]
LogisticRegression-------------------
Accuracy classification score:  0.731
Precision classification score: 0.640
Recal classification score:     0.500
Confusion matrix:
[[52  9]
 [16 16]]
-------------------------------------

saheart-5-2tra.dat saheart-5-2tst.dat
-------------------------------------
IRIS Algorithm-----------------------
Accuracy classification score:  0.774
Precision classification score: 0.739
Recal classification score:     0.531
Confusion matrix:
[[55  6]
 [15 17]]
LogisticRegression-------------------
Accuracy classification score:  0.763
Precision classification score: 0.727
Recal classification score:     0.500
Confusion matrix:
[[55  6]
 [16 16]]
-------------------------------------

saheart-5-3tra.dat saheart-5-3tst.dat
-------------------------------------
IRIS Algorithm-----------------------
Accuracy classification score:  0.739
Precision classification score: 0.682
Recal classification score:     0.469
Confusion matrix:
[[53  7]
 [17 15]]
LogisticRegression-------------------
Accuracy classification score:  0.728
Precision classification score: 0.667
Recal classification score:     0.438
Confusion matrix:
[[53  7]
 [18 14]]
-------------------------------------

saheart-5-4tra.dat saheart-5-4tst.dat
-------------------------------------
IRIS Algorithm-----------------------
Accuracy classification score:  0.707
Precision classification score: 0.586
Recal classification score:     0.531
Confusion matrix:
[[48 12]
 [15 17]]
LogisticRegression-------------------
Accuracy classification score:  0.739
Precision classification score: 0.633
Recal classification score:     0.594
Confusion matrix:
[[49 11]
 [13 19]]
-------------------------------------

saheart-5-5tra.dat saheart-5-5tst.dat
-------------------------------------
IRIS Algorithm-----------------------
Accuracy classification score:  0.609
Precision classification score: 0.429
Recal classification score:     0.375
Confusion matrix:
[[44 16]
 [20 12]]
LogisticRegression-------------------
Accuracy classification score:  0.685
Precision classification score: 0.552
Recal classification score:     0.500
Confusion matrix:
[[47 13]
 [16 16]]
-------------------------------------
\end{Output}

% Question 8
\section{Question 8}
Yes, I can improve my results for the following reasons exist.
\begin{enumerate}
    \item The data I use does not have regularization.
    \item The diagonal matrices can use sparse matrix algorithm to calculate its inverse and multiplication.
    \item Python is flexible but slow, we can use some compiled language to speed up computing.
\end{enumerate}

\end{document}


\documentclass[a4paper,man,natbib]{apa6}
\usepackage[english]{babel}

\usepackage[cache=false]{minted}
\usemintedstyle{vs}
\usepackage{xcolor}
\definecolor{bg}{rgb}{.95,.95,.95}

\graphicspath{ {./images/} }
\usepackage{graphicx}
\usepackage{caption}

\usepackage{setspace}
%\usepackage{titlesec}
%\titleformat{\subsection}[runin]% runin puts it in the same paragraph
%	{\normalfont\bfseries}% formatting commands to apply to the whole heading
%	{\thesubsection}% the label and number
%	{0.5em}% space between label/number and subsection title
%	{}% formatting commands applied just to subsection title
%	[]% punctuation or other commands following subsection title
% End Packages %

\title{Advanced Statistical Methods Homework 3}
\shorttitle{DAT 530 HW3}
\author{Brandon Hosley}
\date{\today}
\affiliation{University of Illinois - Springfield}
%\abstract{}

\begin{document}
\maketitle
\singlespacing

\section{Introduction to Statistical Learning \\ Chapter 3: Problem 15}
This problem involves the \textbf{\textcolor{red}{Boston}} data set,
which we saw in the lab for this chapter.
We will now try to predict per capita crime rate
using the other variables in this data set.
In other words, per capita crime rate is the response,
and the other variables are the predictors.

\begin{minted}[bgcolor=bg]{r}
# Load Boston data set
library(MASS)
mount(Boston)
\end{minted}

\subsection{(a)}
\emph{
	For each predictor,
	fit a simple linear regression model to predict the response.
	Describe your results.
	In which of the models is there a statistically significant association
	between the predictor and the response?
	Create some plots to back up your assertions.}
\includegraphics[width=\linewidth]{LinearMatrix}
\begin{minted}[bgcolor=bg]{r}
par(mfrow=c(2,7))
plot(crim,  crim); abline(lm(crim~crim))
plot(zn,    crim); abline(lm(crim~zn))
plot(indus, crim); abline(lm(crim~indus))
plot(chas,  crim); abline(lm(crim~chas))
plot(nox,   crim); abline(lm(crim~nox))
plot(rm,    crim); abline(lm(crim~rm))
plot(age,   crim); abline(lm(crim~age))
plot(dis,   crim); abline(lm(crim~dis))
plot(rad,   crim); abline(lm(crim~rad))
plot(tax,   crim); abline(lm(crim~tax))
plot(ptratio, crim); abline(lm(crim~ptratio))
plot(black, crim); abline(lm(crim~black))
plot(lstat, crim); abline(lm(crim~lstat))
plot(medv,  crim); abline(lm(crim~medv))
\end{minted}

It appears that only certain predictors have slopes not near-zero or near-infinite:
zn, indus, age, rad, lstat, medv.

\subsection{(b)}
\emph{
	Fit a multiple regression model to predict the
	response using all of the predictors.
	Describe your results.
	For which predictors can we reject the null hypothesis
	$H_0 : \beta_j = 0$?
}
\begin{minted}[bgcolor=bg]{r}
summary(lm(crim~., data=Boston))
\end{minted}
Zn, dis, rad, black, and medv are the predictors that allow rejection of null hypothesis with a fairly high confidence.

\subsection{(c)}
\emph{
	How do your results from (a) compare to your results from (b)?
	Create a plot displaying the univariate regression coefficients
	from (a) on the x-axis, and the multiple regression coefficients
	from (b) on the y-axis. That is, each predictor is displayed as a
	single point in the plot. Its coefficient in a simple linear regression model is shown on the x-axis, and its coefficient estimate
	in the multiple linear regression model is shown on the y-axis.
} \\
\begin{minted}[bgcolor=bg]{r}
predictors <- names(Boston)
y <- coefficients(lm(crim~., data=Boston))
x <- vector(mode="numeric", length=1)
x <- append(x,coefficients(lm(crim~zn))[[2]])
x <- append(x,coefficients(lm(crim~indus))[[2]])
x <- append(x,coefficients(lm(crim~chas))[[2]])
x <- append(x,coefficients(lm(crim~nox))[[2]])
x <- append(x,coefficients(lm(crim~rm))[[2]])
x <- append(x,coefficients(lm(crim~age))[[2]])
x <- append(x,coefficients(lm(crim~dis))[[2]])
x <- append(x,coefficients(lm(crim~rad))[[2]])
x <- append(x,coefficients(lm(crim~tax))[[2]])
x <- append(x,coefficients(lm(crim~ptratio))[[2]])
x <- append(x,coefficients(lm(crim~black))[[2]])
x <- append(x,coefficients(lm(crim~lstat))[[2]])
x <- append(x,coefficients(lm(crim~medv))[[2]])

df <- data.frame(predictors,x,y)

library(ggplot2)
library(ggrepel)
p <- ggplot(df, aes(x,y))
p <- p + geom_point()
p <- p + geom_text_repel(aes(x,y,label=predictors))
p <- p + coord_cartesian(xlim=c(-3,2),ylim=c(-1,0.75))
p <- p + geom_abline(intercept=0,slope=1)
p
\end{minted}
\includegraphics[width=\linewidth]{c-ratio}
This figure shows that for all values other than nox, the coefficient ratios are close to 1. For values above the plotted line the predictors are given a higher coefficient in the multivariate regression model than they would receive on their own.

\subsection{(d)}
\emph{
	Is there evidence of non-linear association between any of the
	predictors and the response? To answer this question, for each
	predictor $X$, fit a model of the form }\\
	$Y = \beta_0 + \beta_1X + \beta_2X^2 + \beta_3X^3 + \epsilon$.

\begin{minted}[bgcolor=bg]{r}
lm(crim~poly( zn,    3, raw=TRUE))
lm(crim~poly( chas,  3, raw=TRUE))
lm(crim~poly( indus, 3, raw=TRUE))
lm(crim~poly( nox,   3, raw=TRUE))
lm(crim~poly( rm,    3, raw=TRUE))
lm(crim~poly( age,   3, raw=TRUE))
lm(crim~poly( dis,   3, raw=TRUE))
lm(crim~poly( rad,   3, raw=TRUE))
lm(crim~poly( tax,   3, raw=TRUE))
lm(crim~poly(ptratio,3, raw=TRUE))
lm(crim~poly( black, 3, raw=TRUE))
lm(crim~poly( lstat, 3, raw=TRUE))
lm(crim~poly( medv,  3, raw=TRUE))
\end{minted}

From this information we can determine that certain predictors bear a non-linear relationship to the crime statistic. \\
dis, medv, nox, ptratio, rm all have significant coefficients associated with their $X^2$ and $X^3$ values.

\end{document}

2020-Fall Advanced Statistical Methods/HW 03/HW 03.2 - Hosley.tex


\documentclass[inequalities.tex]{subfile}

\begin{document}
	\section{IrMO}\label{sec:irmo}

		\begin{problem}[$2002$ Team Selection Test, problem $6$]
			Let $x_{1},\ldots,x_{n}$ be positive real numbers such that $x_{1}^{2}+\ldots+x_{n}^{2}=n$, $\lambda$ be a real number such that $0\leq \lambda\leq 1$ and $s$ be a positive real number such that $\sum_{i=1}^{n}x_{i}\geq s$. Prove that at least
				\begin{align*}
					\left\lceil\dfrac{s^{2}(1-\lambda)^{2}}{n}\right\rceil
				\end{align*}
			of these numbers are larger than $\frac{\lambda s}{n}$.
		\end{problem}

		\begin{problem}[$2003$ Round $2$, problem $1$]
			Let $x,y,z$ be real numbers such that $xyz=-1$. Prove that
				\begin{align*}
					x^{4}+y^{4}+z^{4}+3(x+y+z)
						& \geq \dfrac{x^{2}}{y}+\dfrac{y^{2}}{z}+\dfrac{z^{2}}{x}+\dfrac{y^{2}}{x}+\dfrac{z^{2}}{x}+\dfrac{x^{2}}{z}
				\end{align*}
		\end{problem}

		\begin{problem}[$2005$ Team Selection Test, problem $1$]
			Let $a_{1},\ldots,a_{n}$ be real numbers and
				\begin{align*}
					\dfrac{a_{1}+\ldots+a_{n}}{n}
						& = m\\
					\dfrac{a_{1}^{2}+\ldots+a_{n}^{2}}{n}
						& = 1
				\end{align*}
			If there is an $i$ such that $a_{i}\leq m$, prove that
				\begin{align*}
					n-i
						& \geq n(m-a_{i})^{2}
				\end{align*}
		\end{problem}

		\begin{problem}[$2006$ Team Selection Test, problem $4$]
			Let $x_{1},\ldots,x_{n}$ be real numbers. Prove that
				\begin{align*}
					\sum_{i,j=1}^{n}|x_{i}+x_{j}|
						& \geq n\sum_{i=1}^{n}|x_{i}|
				\end{align*}
		\end{problem}

		\begin{problem}[$2008$ Team Selection Test, problem $5$]
			Let $a,b,c$ be positive real numbers such that $ab+bc+ca=1$. Prove that
				\begin{align*}
					\sqrt{a^{3}+a}+\sqrt{b^{3}+b}+\sqrt{c^{3}+c}
						& \geq 2\sqrt{a+b+c}
				\end{align*}
		\end{problem}

		\begin{problem}[$2009$ Team Selection Test, problem $3$]
			Let $a,b,c$ be positive real numbers such that $a+b+c=3$. Prove that
				\begin{align*}
					\dfrac{1}{2+a^{2}+b^{2}}+\dfrac{1}{2+b^{2}+c^{2}}+\dfrac{1}{2+c^{2}+a^{2}}
						& \leq \dfrac{3}{4}
				\end{align*}
		\end{problem}

		\begin{problem}[$2010$ Third Round, problem $6$]
			We call a sequence of real numbers $a_{10},\ldots,a_{1389}$ \textit{concave} if $2a_{i}\geq a_{i-1}+a_{i+1}$ for $0<i<1389$. Find the largest real number $c$ such that
				\begin{align*}
					\sum_{i=0}^{1389}ia_{i}
						& \geq c\sum_{i=0}^{1389}a_{i}^{2}
				\end{align*}
			for all concave sequence of non-negative real numbers $(a_{n})$.
		\end{problem}

		\begin{problem}[$2011$ Third Round, problem $10$]
			Find the smallest real number $k$ such that
				\begin{align*}
					\sqrt{(a^{2}+1)(b^{2}+1)(c^{2}+1)}+\sqrt{(b^{2}+1)(c^{2}+1)(d^{2}+1)}+\sqrt{(c^{2}+1)(d^{2}+1)(a^{2}+1)}+\sqrt{(d^{2}+1)(a^{2}+1)(b^{2}+1)}
						& \geq 2(ab+bc+cd+da+ac+bd)-k
				\end{align*}
			for all real numbers $a,b,c,d$.
		\end{problem}

		\begin{problem}[$2012$ Third Round, problem $10$]
			Let $a,b,c$ be positive real numbers such that $ab+bc+ca=1$. Show that
				\begin{align*}
					\sqrt{3}(\sqrt{a}+\sqrt{b}+\sqrt{c})
						& \leq \dfrac{a\sqrt{a}}{bc}+\dfrac{b\sqrt{b}}{ca}+\dfrac{c\sqrt{c}}{ab}
				\end{align*}
		\end{problem}

		\begin{problem}[$2013$ Team Selection Test, problem $11$]
			Let $a,b,c$ be the sides of a triangle such that $a\geq b\geq c$. Prove that
				\begin{align*}
					\sqrt{a(a+b-\sqrt{ab})}+\sqrt{b(b+c-\sqrt{bc})}+\sqrt{c(c+a-\sqrt{ca})}
						& \geq a+b+c
				\end{align*}
		\end{problem}

		\begin{problem}[$2014$ Round $2$, problem $3$]
			Let $x,y,z$ be non-negative real numbers such that $x^{2}+y^{2}+z^{2}=2(xy+yz+zx)$. Prove that
				\begin{align*}
					\dfrac{x+y+z}{3}
						& \sqrt[3]{2xyz}
				\end{align*}
		\end{problem}

		\begin{problem}[$2014$ Team Selection Test, problem $5$]
			Let $n$ be a positive integer and $x_{1},\ldots,x_{n+1}$ be positive real numbers such that $x_{1}\cdots x_{n}=1$. Prove that
				\begin{align*}
					\sqrt[x_{1}]{n}+\ldots+\sqrt[x_{n+1}]{n}
						& \geq n^{\sqrt[n]{x_{1}}}+\ldots+n^{\sqrt[n]{x_{n+1}}}
				\end{align*}
		\end{problem}

		\begin{problem}[$2014$ Team Selection Test $2$, problem $5$]
			Let $x,y,z$ be positive real numbers such that
				\begin{align*}
					x^{2}+y^{2}+z^{2}
						& = x^{2}y^{2}+y^{2}z^{2}+z^{2}x^{2}
				\end{align*}
			Prove that
				\begin{align*}
					\left((x-y)(y-z)(z-x)\right)^{2}
						& \leq 2\left((x^{2}-y^{2})^{2}+(y^{2}-z^{2})^{2}+(z^{2}-x^{2})^{2}\right)
				\end{align*}
		\end{problem}

		\begin{problem}[$2015$ Team Selection Test, problem $6$]
			If $a,b,c$ are positive real numbers such that $a+b+c=abc$, prove that
				\begin{align*}
					\dfrac{abc}{3\sqrt{2}}\left(\sum_{cyc}\dfrac{\sqrt{a^{3}+b^{3}}}{ab+1}\right)
						& \geq \sum_{cyc}\dfrac{a}{a^{2}+1}
				\end{align*}
		\end{problem}

		\begin{problem}[$2016$ Team Selection Test, problem $1$]
			Let $a,b,c,d$ be positive real numbers such that
				\begin{align*}
					\dfrac{1}{a+1}+\dfrac{1}{b+1}+\dfrac{1}{c+1}+\dfrac{1}{d+1}
						& = 2
				\end{align*}
			Prove that
				\begin{align*}
					\sqrt{\dfrac{a^{2}+1}{2}}+\sqrt{\dfrac{a^{2}+1}{2}}+\sqrt{\dfrac{c^{2}+1}{2}}+\sqrt{\dfrac{d^{2}+1}{2}}
						& \geq 3(\sqrt{a}+\sqrt{b}+\sqrt{c}+\sqrt{d})-8
				\end{align*}
		\end{problem}

		\begin{problem}[$2016$ Round $2$, problem $1$]
			Let $a,b,c$ be positive real numbers such that $c\geq b\geq a$. Prove that
				\begin{align*}
					\dfrac{(c-a)^{2}}{6c}
						& \leq \dfrac{a+b+c}{3}-\dfrac{3}{\dfrac{1}{a}+\dfrac{1}{b}+\dfrac{1}{c}}
				\end{align*}
		\end{problem}

		\begin{problem}[$2017$ Round $2$, problem $4$]
			Let $x,y$ be two positive real numbers such that $x^{4}-y^{4}=x-y$. Prove that
				\begin{align*}
					\dfrac{x-y}{x^{6}-y^{6}}
						& \leq \dfrac{4}{3}(x+y)
				\end{align*}
		\end{problem}

		\begin{problem}[$2017$ Team Selection Test, problem $1$]
			Let $a,b,c,d$ be positive real numbers such that $a+b+c+d=2$. Prove that
				\begin{align*}
					\dfrac{(a+c)^{2}}{ad+bc}+\dfrac{(b+d)^{2}}{ac+bd}+4
						& \geq 4\left(\dfrac{a+b+1}{c+d+1}+\dfrac{c+d+1}{a+b+1}\right)
				\end{align*}
		\end{problem}

		\begin{problem}[$2018$ Team Selection Test, problem $2$]
			Find the smallest real number $k$ such that
				\begin{align*}
					\left(\dfrac{2a}{a-b}\right)^{2}+\left(\dfrac{2b}{b-c}\right)^{2}+\left(\dfrac{2c}{c-a}\right)^{2}+k
						& 4\left(\dfrac{2a}{a-b}+\dfrac{2b}{b-c}+\dfrac{2c}{c-a}\right)
				\end{align*}
			for all real numbers $a,b,c$.
		\end{problem}

		\begin{problem}[$2020$ Second Round, problem $2$]
			Let $x,y,z$ be positive real numbers such that $x+y+z=1399$. Find
				\begin{align*}
					\max\left(\lfloor x\rfloor y,\lfloor y\rfloor z,\lfloor z\rfloor x\right)
				\end{align*}
		\end{problem}
\end{document}


\section{Thursday, 18 October 2018}

\subsection{Steps}
\begin{itemize}
\item We get the columns that ends with ``MEAN'' because that columns represent the average values. In that way the dataset is more representative. After that, we work on the visualization.
\item We have done a \texttt{PCA} attempt but the \texttt{PCA} fails because we haven't deleted the version column and there are rows where the version number is not a float, it's a string like ``x.x.x'', so we eliminate the version column.
The covariance is pretty high, about $0.85$.

\begin{figure}[!htb]
\centering
\includegraphics[width=0.5\textwidth]{../../reports/figures/PCA_AccelerometerStat.png}
\caption{Principal Component Analysis}
\label{fig:pca}
\end{figure}

\item Next, we start clustering with k-means. We analyse different groups in order to interpret the plots. We distribute the job: Ivan analyse the magnetic field, Benjamin the gyroscope and Dídimo the accelerometer.
\item If we do k-means with the complete dataset, the computer is out of memory, that is the reason why we only consider the values of 5 days.
\item We do k-means in a range of 2 to 50 clusters to know what is the ideal number of clusters. In order to get it, we use the \textit{silhouette} method and we analyse which is the best coefficient. When we already know the ideal number of clusters, we plot the graphic, showing the different groups.

\begin{figure}[!htb]
\centering
\includegraphics[width=0.5\textwidth]{../../reports/figures/KMeans_AccelerometerStat.png}
\caption{K-means clustering algorithm with best number of clusters}
\label{fig:k-means}
\end{figure}
\end{itemize}


\chapter{Assignment-Probability}
\begin{enumerate}
	\item Consider a random walker on a square lattice. At each step the walker moves to a nearest neighbour site with equal probability for each of the four sites. The walker starts at the origin and takes 3 steps. The probability that during this walk no site is visited more than once is
	 \begin{tasks}(4)
		\task[\textbf{a.}]$\frac{9}{16}$
		\task[\textbf{b.}]$\frac{27}{64}$
		\task[\textbf{c.}]$\frac{3}{8}$
		\task[\textbf{d.}]$\frac{4}{9}$
	\end{tasks}
	\item There are two baskets. Baskets I contains 3 red and 4 blue balls while Basket II contains 4 red and 3 blue balls. A ball is transferred from box I to box II and then a ball from box II is drawn, what is the probability that it is blue?
	 \begin{tasks}(4)
		\task[\textbf{a.}]$\frac{17}{56}$
		\task[\textbf{b.}]$\frac{19}{56}$
		\task[\textbf{c.}]$\frac{21}{56}$
		\task[\textbf{d.}] $\frac{25}{56}$
	\end{tasks}
	\item If the distribution function of $x$ is $f(x)=x e^{-x / \lambda}$ over the interval $0<x<\infty$, the mean value of $x$ is
	 \begin{tasks}(4)
		\task[\textbf{a.}]$\lambda$
		\task[\textbf{b.}]$2 \lambda$
		\task[\textbf{c.}] $\frac{\lambda}{2}$
		\task[\textbf{d.}] 0
	\end{tasks}
	\item If $x(t)$ denote the position of a Brownian particle at time $t$, given that its position coincide with the point $x=0$ at $t=0$. The particle is to jump a distance $l$ of constant magnitude with equally likely either positive or negative direction along $x$ axis. If the particle take 30 jumps then what is probability that particle have 10 more jump towards right.
	 \begin{tasks}(4)
		\task[\textbf{a.}]\color{red}{?}
		\task[\textbf{b.}]\color{red}{?}
		\task[\textbf{c.}]\color{red}{?}
		\task[\textbf{d.}]\color{red}{?}
	\end{tasks}
	\item In a square lattice as shown in the figure, a particle starts from origin. If the particle can move only in $x$ or $y$ direction and the probability of each direction is $\frac{1}{4}$, the probability that particle will return to origin in four steps is
	\begin{figure}[H]
		\centering
		\includegraphics[height=3.5cm,width=4cm]{P -assignment-01}
	\end{figure}
	 \begin{tasks}(4)
		\task[\textbf{a.}]$\frac{1}{4}$
		\task[\textbf{b.}] $\frac{1}{8}$
		\task[\textbf{c.}]$\frac{1}{16}$
		\task[\textbf{d.}] $\frac{3}{16}$
	\end{tasks}
	\item If the density function of a random variable $X$ is $f(x)= \begin{cases}k x^{2}, & 1 \leq x \leq 5 \\ 0, & \text { otherwise }\end{cases}$
	Then the probability that the random variables assumes value between 2 to 4 is
	 \begin{tasks}(4)
		\task[\textbf{a.}]$\frac{15}{31}$
		\task[\textbf{b.}] $\frac{14}{31}$
		\task[\textbf{c.}]$\frac{17}{31}$
		\task[\textbf{d.}] $\frac{18}{31}$
	\end{tasks}
	\item For a continuous random variable $x$ probability distribution function
	$$
	f(x)= \begin{cases}c x^{2} & \text { for }|x| \leq 1 \\ 0 & \text { for }|x|>1\end{cases}
	$$
	Then the value of $c$ and $\operatorname{var}(x)$ are respectively
	 \begin{tasks}(4)
		\task[\textbf{a.}]$\frac{3}{2}, \frac{3}{5}$
		\task[\textbf{b.}] $\frac{3}{2}, \frac{5}{3}$
		\task[\textbf{c.}]$\frac{2}{3}, \frac{3}{5}$
		\task[\textbf{d.}] $\frac{2}{3}, \frac{5}{3}$
	\end{tasks}
	\item Consider the following two experiments:
	Experiment I: A pair of die is thrown 6 times. Let $P_{1}$ be the probability of getting a doublet on 5 throws (a doublet on a throw means same number on both die).

	Experiment II: A coin is tossed 6 times. Let $P_{2}$ be the probability of getting a head on 4 throws.
	The ratio of $P_{1}$ and $P_{2}$ is
	 \begin{tasks}(4)
		\task[\textbf{a.}]$\frac{1}{2^{5}}$
		\task[\textbf{b.}]$\frac{2}{3^{5}}$
		\task[\textbf{c.}]$\frac{1}{2^{6}}$
		\task[\textbf{d.}]$\frac{2}{3^{6}}$
	\end{tasks}

\end{enumerate}


\documentclass{beamer}

\input{../../shared_slides.tex}
\usepackage{changepage}
% https://openai.com/blog/deep-double-descent/

\title{(Sub)-gradient method}
\date{\today}

\begin{document}
\maketitle
\frame{\tableofcontents}

\section{Intro}%

\begin{frame}
  \frametitle{Spoiler: Smooth vs.\ nonsmooth}
  We consider the convex optimization problem
  \begin{equation}
    \min_x \, f(x)
  \end{equation}
  \vspace{-0.5cm}
  \begin{block}{}
    \begin{equation}
      x_{k+1} = x_k - \alpha g_k
    \end{equation}
  \end{block}
  \begin{adjustwidth}{-1.5em}{-1.5em}
    \begin{minipage}{0.52\textwidth}
      \begin{block}{}
        \begin{itemize}
          \item If $f$ is \textcolor{blue}{smooth} we take $g_k = \nabla f(x_k)$ $\rightarrow$ \textbf{Gradient Descent}.
          \item stepsize can be constant $1/L$ (smoothness constant)
          \item convergence rate $f(x_k)-f^* = \mathcal{O}(1/k)$
        \end{itemize}
      \end{block}
    \end{minipage}
    \hfill
    \begin{minipage}{0.52\textwidth}
      \begin{block}{}
        \begin{itemize}
          \item If \textcolor{blue}{not} we take $g_k$ a \textit{subgradient} $\rightarrow$ \textbf{Subgradient method}.
          \item stepsize has to be chosen \emph{small or decreasing} $\approx 1/\sqrt{k}$
          \item convergence rate is \emph{worse} $f(x_k)-f^* = \mathcal{O}(1/\sqrt{k})$
        \end{itemize}
      \end{block}
    \end{minipage}
  \end{adjustwidth}
\end{frame}

\begin{frame}
  \frametitle{Intuition behind GD}
  \begin{itemize}
    \item derivative (gradient) points in the direction of steepest ascent
          $\rightarrow$ GD is also called \textbf{steepest descent}
    \item GD update is equivalent to
          \begin{equation}
            x_{k+1} = \argmin_{x\in \R^d} \Big\{ \underbrace{f(x_k) + \langle \nabla f(x_k), x-x_k \rangle }_{\text{linearization of $f$}}+ \frac{1}{2 \alpha} \Vert x-x_k \Vert^2 \Big\}
          \end{equation}
          \begin{itemize}
            \item solves a linear model of $f$
            \item minimizing unconstrained linear models is no good
            \item so we add a ``proximity term''
          \end{itemize}
  \end{itemize}
\end{frame}

\section{Subgradient theory}%

\begin{frame}
  \frametitle{Subgradients}
  What if $f$ is not differentiable?
  \begin{definition}
    $g \in \R^d$ is a \textcolor{blue}{subgradient} of $f$ at $x$ if
    \begin{equation}
      f(y) \ge f(x) + g^T (y-x)
    \end{equation}
  \end{definition}
  \begin{figure}[ht]
    \centering
    \includegraphics[width=\textwidth,height=\textheight,keepaspectratio]{subgradients}
    % \caption{\label{fig:label} }
  \end{figure}
\end{frame}

\begin{frame}
  \frametitle{Subgradients II}
  \begin{definition}
    The \textcolor{blue}{subdifferential} $\partial f(x)$ is the \emph{set of all subgradients} of $f$ at $x$.
  \end{definition}
  Example
  \begin{figure}[ht]
    \centering
    \includegraphics[width=0.5\textwidth,keepaspectratio]{subdifferential}
    % \caption{\label{fig:label} }
  \end{figure}
  Subgradient condition at $x=0$ is $f(y)\ge f(0) + g(y-0) = gy$.
  \textcolor{purple}{What is $\partial f(0)$?}
\end{frame}

\begin{frame}
  \frametitle{Subgradients III}
  \begin{lemma}%
    If $f$ is differentiable at $x$ then $\partial f(x) \subset \{\nabla f(x)\}$
  \end{lemma}
  So either one subgradient or none.\\
  \begin{minipage}{0.48\textwidth}
    \begin{figure}[ht]
      \centering
      \includegraphics[width=\textwidth,keepaspectratio]{graph_above_tangent}
      % \caption{\label{fig:label} }
    \end{figure}
  \end{minipage}
  \hfill
  \begin{minipage}{0.48\textwidth}
    \begin{figure}[ht]
      \centering
      \includegraphics[width=\textwidth,keepaspectratio]{differentiable_function}
      % \caption{\label{fig:label} }
    \end{figure}
  \end{minipage}
\end{frame}

\begin{frame}
  \frametitle{Subgradient characterization of convexity}
  \begin{lemma}%
    A function $f: \R^d \to \R$ is convex \textit{if and only if} $\partial f(x)$ is not empty for all $x$.
  \end{lemma}
  \begin{figure}[ht]
    \centering
    \includegraphics[width=\textwidth,keepaspectratio]{subgradients}
    \caption{Subgradients at every point.}
  \end{figure}
\end{frame}

\begin{frame}
  \frametitle{Lipschitz = bounded subgradients}
  \begin{definition}
    We call $f$ $L$-Lipschitz (continuous) if
    \begin{equation}
      \Vert f(x)-f(y) \Vert \le L \Vert x-y \Vert.
    \end{equation}
  \end{definition}
  \begin{lemma}%
    Let $f$ be convex. Then the following two are equivalent.
    \begin{enumerate}
      \item All \textbf{subgradients are uniformly bounded}.
            \begin{equation}
              \Vert g \Vert \le L \quad \forall x, \forall g \in \partial f(x)
            \end{equation}
      \item $f$ is $L$-Lipschitz
    \end{enumerate}
  \end{lemma}
\end{frame}

\begin{frame}
  \frametitle{Subgradient optimality condition}
  \begin{lemma}%
    Let \textcolor{blue}{$0 \in \partial f(\bar{x})$}, then $\bar{x}$ is a \textcolor{blue}{global minimum}.
  \end{lemma}
  \begin{proof}
    By the definition of subgradients, $g=0 \in \partial f(\bar{x})$ gives
    \begin{equation}
      f(y) \ge f(\bar{x}) + g^T(y-\bar{x}) = f(\bar{x}).
    \end{equation}
  \end{proof}
\end{frame}

\section{Convergence subgradient method}%
\label{sec:}

\begin{frame}
  \frametitle{Convergence statement: Subgradient method}
  \textcolor{gray}{We assume there exists minimizer $x^*$ and we write $f^*=f(x^*)$.}
  \begin{theorem}
    $f$ is convex, subgradients are bounded $\Vert g(x) \Vert \le G$ for all $g(x)\in \partial f(x)$. Then,
    \begin{equation}
      f(\bar{x}_k) - f^* \le \frac{\Vert x_0-x^*\Vert G}{\sqrt{k}}
    \end{equation}
    for the \textbf{averaged} iterates $\bar{x}_k = \frac{1}{k} \sum_{i=0}^{k-1} x_i $
  \end{theorem}
  \begin{itemize}
    \item Also holds for the \textbf{``best''} iterate.
    \item \textcolor{blue}{Dimension independent!} (no $d$)
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Proof}
  \vspace{-0.5cm}
  \begin{equation}
    \begin{aligned}
      \Vert x_{k+1} - x^* \Vert^2 &\le \Vert x_k - \alpha g_k - x^* \Vert^2 \\
      &= \Vert x_k-x^* \Vert^2 + 2 \alpha \langle g_k, x^*-x_k \rangle + \alpha^2 \Vert g_k \Vert^2.
    \end{aligned}
  \end{equation}
  Using the \textbf{subgradient inequality} $\langle g_k , x^* -x_k \rangle \le f(x^*) - f(x_k)$
  % \begin{equation}
  %   \langle g_k , x^* -x_k \rangle \le f(x^*) - f(x_k)
  % \end{equation}
  \begin{equation}
    \Vert x_{k+1} - x^* \Vert^2 \le \Vert x_k-x^* \Vert^2 + 2 \alpha(f(x^*) - f(x_k)) + \alpha^2 \Vert g_k \Vert^2.
  \end{equation}
  Summing up (telescoping) yields
  \begin{equation}
    \label{eq:telescoping}
    2\sum_{i=0}^{k-1}  \alpha(f(x_i) - f(x^*)) + \Vert x_{k} - x^* \Vert^2 \le \Vert x_0-x^* \Vert^2 +  \alpha^2 \sum_{i=0}^{k-1} \Vert g_k \Vert^2.
  \end{equation}
  Via the \emph{bounded subgradient} assumption
  \begin{equation}
    2\sum_{i=0}^{k-1}  \alpha(f(x_i) - f(x^*)) \le \Vert x_0-x^* \Vert^2 +  \alpha^2 k G^2.
  \end{equation}
\end{frame}

\begin{frame}
  \frametitle{Proof [contd]}
  We divide by $2\alpha$ and $k$
  \begin{equation}
    \frac{1}{k}\sum_{i=0}^{k-1}  f(x_i) - f^*  \le \frac{1}{2 \alpha k} \Vert x_0-x^* \Vert^2 +  \frac{\alpha G^2}{2}
  \end{equation}
  Using Jensens inequality (convexity with more than $2$ points)
  \begin{equation}
    \sum_{i=0}^{k-1} \frac{1}{k} f(x_i) \ge \sum_{i} f \left( \frac{1}{k}\sum_{i=0}^{k-1}x_i \right)
  \end{equation}
  we obtain
  \begin{equation}
    f(\bar{x}_k) - f^*  \le \frac{1}{2 \alpha k} \Vert x_0-x^* \Vert^2 + \frac{\alpha G^2}{2}.
  \end{equation}

\end{frame}

\begin{frame}
  \frametitle{How to choose the stepsize?}
  We have
 \begin{equation}
    f(\bar{x}_k) - f^*  \le \frac{1}{2 \alpha k} \Vert x_0-x^* \Vert^2 + \frac{\alpha G^2}{2}.
  \end{equation}
  Choose $\alpha$ such that \textit{RHS is minimized}, i.e.\
  \begin{equation}
    \alpha = \frac{\Vert x_0-x^*\Vert }{G \sqrt{k}},
  \end{equation}
  which gives
  % Clearly $\alpha_i = \ell_2 \textbackslash \ell_1$ leads convergence, for example $1/i$.
  % However, $\alpha = \mathcal{O}(1/\sqrt{i})$ gives
  % \begin{equation}
  %   \sum \alpha_i = (\frac{1}{\sqrt{1}} + \frac{1}{\sqrt{2}} + \cdots +  \frac{1}{\sqrt{k}}) > \sqrt{k}
  %   \sum \alpha^2 = (\frac{1}{1} + \frac{1}{2} + \cdots + \frac{1}{k}) \approx \log(k)
  % \end{equation}

  \begin{equation}
    f(\bar{x}_k) - f^* \le \frac{\Vert x_0-x^*\Vert G }{\sqrt{k}}. \qed
  \end{equation}
  When ignoring constants (and focusing on the rate) we sometimes write
  \begin{equation}
    \mathcal{O}\left(\frac{1}{\sqrt{k}}\right).
  \end{equation}
\end{frame}

\begin{frame}
  \frametitle{Complexity}
  For convex Lipschitz functions we require \textcolor{blue}{$\mathcal{O}(\epsilon^{-2})$} iterations. For $D:= \Vert x_0 -x^* \Vert$
  \begin{equation}
    f(\bar{x}_k) - f^* \le \frac{D G}{\sqrt{k}}
  \end{equation}
  \textbf{Q:} How many iterations to get
  \begin{equation}
    f(\bar{x}_k) - f^* \le \epsilon ?
  \end{equation}
  \textbf{A:} We get this if
  \begin{equation}
    \frac{D G}{\sqrt{k}} \le \epsilon
  \end{equation}
  Equivalently
  \begin{equation}
    k \ge \frac{D^2 G^2}{\epsilon^2}.
  \end{equation}
\end{frame}

\begin{frame}
  \frametitle{Polyak stepsize}
  Let's revisit the convergence proof of the subgradient method
  \begin{equation}
    \begin{aligned}
      \Vert x_{k+1} - x^* \Vert^2 &\le \Vert x_k - \alpha_k g_k - x^* \Vert^2 \\
      &= \Vert x_k-x^* \Vert^2 + 2 \alpha \langle g_k, x^*-x_k \rangle + \alpha^2 \Vert g_k \Vert^2\\
      &\le \Vert x_k-x^* \Vert^2 + 2 \alpha (f^* - f(x_k))+ \alpha^2 \Vert g_k \Vert^2.
    \end{aligned}
  \end{equation}
  Can we pick $\alpha$ such that the RHS is minimized?
  \begin{equation}
    \min_\alpha \, \alpha^2 \Vert g_k \Vert^2 + 2 \alpha (f^* - f(x_k))
  \end{equation}
  gives
  \begin{equation}
    \alpha_k^* = \frac{f(x_k)-f^*}{\Vert g_k \Vert^2}
  \end{equation}
  \begin{equation}
      \Vert x_{k+1} - x^* \Vert^2 = \Vert x_k-x^* \Vert^2 - {\left( \frac{f(x_k)-f^*}{\Vert g_k \Vert} \right)}^2
  \end{equation}
\end{frame}

\begin{frame}
  \frametitle{Polyak stepsize [contd]}
  \begin{itemize}
    \item Requires us to know the optimal objective function value
    \item can be the case in certain setting:
          separable data, feasibility problems
    \item modern deep learning interpolation setting
  \end{itemize}
  \begin{figure}[ht]
    \centering
    \includegraphics[width=0.7\textwidth]{modern-regime}
    \caption{from openai.com}
  \end{figure}
\end{frame}

\section{Smooth case}%
\label{sec:}

\begin{frame}
  \frametitle{Can we do better if the function is smooth?}
  \begin{definition}
    We call a function $L$-\textcolor{blue}{smooth} if
    \begin{equation}
      f(y) \le f(x) + \langle \nabla f(x), y-x \rangle + \frac{L}{2} \Vert y-x \Vert^2.
    \end{equation}
  \end{definition}
  \begin{center}
    \textit{Can be upper bounded by a quadratic.}
  \end{center}
  \begin{lemma}%
    If the gradient of $f$ is $L$-Lipschitz
    \begin{equation}
      \Vert \nabla f(x)- \nabla f(y) \Vert \le L \Vert x-y \Vert.
    \end{equation}
    then it is also $L$-smooth.
  \end{lemma}
  Note: Definition does not require convexity.
\end{frame}

\begin{frame}
  \frametitle{Smoothness}
  If $f$ is convex we get upper and lower bound:

  \begin{figure}[ht]
    \centering
    \includegraphics[width=\textwidth,keepaspectratio]{smoothness}
    % \caption{\label{fig:label} }
  \end{figure}
\end{frame}

\begin{frame}
  \frametitle{Smooth vs. Lipschitz}
  \begin{itemize}
    \item Bounded (sub)gradients $\Leftrightarrow$ Lipschitz continuity of $f$
    \item Smoothness $\Leftrightarrow$ Lipschitz continuity of $\nabla f$ (if convex)
  \end{itemize}

  \begin{lemma}%
    Let $f$ be convex and differentiable, then the following are \textit{equivalent}
    \begin{enumerate}
      \item $f$ is smooth with parameter $L$
      \item $\nabla f$ is $L$-Lipschitz
    \end{enumerate}
  \end{lemma}
\end{frame}

\begin{frame}
  \frametitle{Sufficient decrease}
  \begin{lemma}%
    If $f$ is $L$-smooth with stepsize $\alpha = 1/L$, then gradient descent satisfies
    \begin{equation}
      f(x_{k+1}) \le f(x_k) - \frac{1}{2L} \Vert \nabla f(x_k) \Vert^2
    \end{equation}
  \end{lemma}
  \begin{proof}
    \begin{equation}
      \begin{aligned}
        f(x_{k+1}) &\le f(x_k) + \langle \nabla f(x_k), x_{k+1}-x_k \rangle + \frac{L}{2}\Vert x_{k+1}-x_k \Vert^2 \\
        &= f(x_k) - \alpha \Vert \nabla f(x_k) \Vert^2 + \frac{\alpha^2 L}{2} \Vert \nabla f(x_k) \Vert^2 \\
        &= f(x_k) - \left(\frac{1}{L} - \frac{1}{2L}\right) \Vert \nabla f(x_k) \Vert^2
      \end{aligned}
    \end{equation}
  \end{proof}
\end{frame}

\begin{frame}
  \frametitle{Smooth convex functions}
  \begin{theorem}
    Let $f: \R^d \to \R$ be convex and $L$-smooth and the stepsize $\alpha=1/L$, then
    \emph{gradient descent} yields
    \begin{equation}
      f(x_k)-f^* \le \frac{L}{2k}\Vert x_0-x^* \Vert^2.
    \end{equation}
  \end{theorem}
  \begin{itemize}
    \item holds for last iterate
    \item independet of dimension $d$
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Complexity of gradient method}
  Denote $D^2 := \Vert x_1- x^* \Vert^2$
  \begin{equation}
    \text{iteration: } k \ge \frac{D^2 L}{2 \epsilon} \Rightarrow \text{error} \le \frac{L D^2}{2 k} \le \epsilon
  \end{equation}
  Given error $\epsilon=0.01$ results in
  \begin{itemize}
    \item $50 \cdot D^2 L$ iterations for \textit{smooth} case
      \item $10 000 \cdot D^2 G^2$ for nonsmooth but Lipschitz
  \end{itemize}

  What if we don't know $L$?
\end{frame}

\begin{frame}
  \frametitle{Proof of $\mathcal{O}(\epsilon^{-1})$ for smooth functions}
    Subgradient analysis gave us
    \begin{equation}
      \sum_{i=0}^{k-1} (f(x_i) - f^*) \le \frac{1}{2 \alpha}\Vert x_0-x^* \Vert^2 +  \frac{\alpha}{2} \sum_{i=0}^{k-1}  \Vert g_k \Vert^2,
    \end{equation}
    see~\eqref{eq:telescoping}. This time we use \textcolor{blue}{sufficient decrease} to bound gradient norm
    \begin{equation}
      \frac{1}{2 L} \sum_{i=0}^{k-1} \Vert \nabla f(x_k) \Vert^2 \le \sum_{i=0}^{k-1} (f(x_i) - f(x_{i+1})) = f(x_0) - f(x_k)
    \end{equation}
    Combining the above two (with $\alpha=1/L$)
    \begin{equation}
      \begin{aligned}
        \sum_{i=0}^{k-1} (f(x_i) - f^*)  &\le \frac{L}{2} \Vert x_0-x^* \Vert^2 +  \frac{1}{2L} \sum_{i=0}^{k-1}  \Vert g_k \Vert^2 \\
        &\le \frac{L}{2} \Vert x_0-x^* \Vert^2 + f(x_0) - f(x_k)
      \end{aligned}
    \end{equation}
\end{frame}

\begin{frame}
  \frametitle{Proof II}
  By rewriting:
    \begin{equation}
      \sum_{\textcolor{red}{i=1}}^{\textcolor{red}{k}} (f(x_i) - f^*) \le \frac{L}{2} \Vert x_0-x^* \Vert^2
    \end{equation}
    As last iterate is the best (sufficient decrease):
    \begin{equation}
      f(x_k) - f^* \le \frac{1}{k} \sum_{i=1}^{k} f(x_i) - f^* \le \frac{L}{2 k} \Vert x_0 - x^* \Vert^2 \qed
    \end{equation}
\end{frame}

\end{document}


\documentclass[12pt,timesnewroman,letterpaper]{article}
\input{ramesh_abbreviations}
\usepackage{times}
\usepackage{mathrsfs}
\usepackage{relsize}
\usepackage{tabto}
\usepackage{amsmath}
\usepackage{helvet}
\usepackage{courier}
\usepackage{fancyheadings}
\pagestyle{fancy}
\usepackage{pmc}
\usepackage{graphicx}
\usepackage{hyperref}
\hypersetup{
    colorlinks=true,
    urlcolor=magenta,
}
\setlength\textwidth{6.5in}
\setlength\textheight{8.5in}
\begin{document}
\title{Logistic Regression in Q}
\author{ Tara Mirmira and Ramesh Subramonian }
\maketitle
\thispagestyle{fancy}
\lfoot{{\small Data Analytics Team}}
\cfoot{}
\rfoot{{\small \thepage}}

\section{Objective}

We have observations that fall into one of two classes: $0$ or $1$. Given a new observation, we want to predict if the observation will be in class $0$ or class $1$. \\

\noindent We would like to use a linear model to make our predictions, meaning we would like to express the class of an observation as a linear combination of the explanatory variables. We consider the simplest linear model, the linear probability model, in the next section.

\section{Linear Probability Model}

Consider the model $y_i = \alpha + \beta x_i + \epsilon_i$

\begin{itemize}
    \item $y_i$ is the class associated with $x_i$, and is what we want to predict
    \item $\epsilon_i$ are independent and identically distributed with mean of $0$ and a variance of $\sigma^2$
    \item $E(y_i) = \alpha + \beta x_i$
    \item $E(y_i) = 0 * P(y_i = 1) + 1*P(y_i = 0) = P(y_i = 1) = \pi_i$
    \item $\pi_i = \alpha + \beta_i$
\end{itemize}

\noindent The problem is $\epsilon_i$ can only take on the values $0$ or $1$ so $Var(\epsilon_i) = (1 - \pi_i)^2 \pi_i + -\pi_i^2(1 - \pi_i) = \pi_i(1-\pi_i)$ which is not constant and non constant variance causes all sorts of problems for linear modelling.\\

\noindent This simple model does not work so we will present a more complex linear model in the next section.

\section{Alternate Model to Linear Probability Model - Linear Model Using Link Function}

General overview:
\begin{itemize}
    \item Let $\pi_i$ be the probability that observation $i$ is in class $1$.
    \item Find a function $G$ such that $\pi_i = G(\alpha + \beta_i)$
    \item We want $G$ to be invertible so we can say $G^{-1}(\pi_i) = \alpha + \beta x_i$
    \item For logistic regression, we will use the function $G(z) = \frac{1}{1+e^{-z}}$
    \item $\frac{1}{1+e^{-z}}$ is bounded between $0$ and $1$ for all values of $z$.
\end{itemize}

\noindent Using the above link function:
\begin{itemize}

\item $P(y_i = 1) = \pi_i = G(\alpha + \beta_i) = \dfrac{1}{1 + e^{-(\alpha + \beta x_i)}}  = \dfrac{1}{1 + e^{-(\alpha + \beta x_i)}} * \dfrac{e^{\alpha + \beta x_i}}{e^{\alpha + \beta x_i}} = \dfrac{e^{\alpha + \beta x_i}}{e^{\alpha + \beta x_i} + 1}$
\item $P(y_i = 0) = 1 - P(y_i = 1) = 1 - \dfrac{e^{\alpha + \beta x_i}}{e^{\alpha + \beta x_i} + 1} = \dfrac{1}{1 + e^{\alpha + \beta x_i}}$

\end{itemize}

\noindent \textbf{The odds ratio}: the ratio of the probability of the observation being in one category versus the probability of being in the other category
\begin{itemize}

\item odds ratio = $\dfrac{\pi}{1-\pi} = \dfrac{e^{\alpha + \beta x_i}}{1} = e^{\alpha + \beta x_i} = e^{\alpha}e^{\beta x_i}$ which means that when we increase $x_i$ by one unit, the odds increase by $e^{\beta}$

\item $\log(\text{odds ratio}) = \log\big(\dfrac{\pi}{1-\pi}\big) = \log(e^{\alpha + \beta x_i}) = \alpha + \beta x_i$
\end{itemize}

\noindent From the above derivation, we see that we have found the \textbf{link function}:\\\\
$\log\big(\dfrac{\pi_i}{1-\pi_i}\big) = \alpha + \beta x_i$
\\\\
The log of the odds ratio is linear in the ${x_i} 's$. The link function connects $E(y_i) = \pi_i$ to a linear function of the explanatory variables $x_i$.

\section{Fitting a Model to the Data Using Maximum Likelihood}
General overview:
\begin{itemize}
    \item We will use maximum likelihood to estimate $\alpha$ and $\beta$. We want to find $\alpha$ and $\beta$ that makes the given data most likely to occur and use these values to predict future values.
    \item The response variables are the classes $0$ or $1$, which means the response variables are $Bernoulli(\pi_i)$ variables.
\end{itemize}

\noindent Maximum Likelihood:
\\
\\
Recall a few probabilities:
\begin{itemize}
    \item $P(i^{th} \text{ observation } = 1) = \pi_i$
    \item $P(i^{th} \text{ observation } = 0) = 1 - \pi_i$
    \item $P(i^th \text{ observation } = y_i) = \pi_i^{y_i}$ if $y_i = 1$
    \item $P(i^th \text{ observation } = y_i) = (1-\pi_i)^{1-y_i}$ if $y_i = 0$
    \item Using the above two bullet points, $P(i^th \text{ observation } = y_i) =\\ \pi_i^{y_i}(1-\pi_i)^{1-y_i}$
\end{itemize}

$$
\text{Likelihood } = \mathscr{L}(y_1 \ldots y_n) = \prod_{i=1}^{n} p(y_i) = \prod_{i=1}^n P(i^{th} \text{ observation } = y_i) = \prod_{i=1}^n \pi_i^{y_i}(1-\pi_i)^{1-y_i} =
$$

$$\prod_{i=n}^n \big(\dfrac{\pi_i}{1-\pi_i}\big)^{y_i}(1-\pi_i) = \prod_{i=1}^n (e^{\alpha + \beta x_i})^{y_i}\big(\dfrac{1}{1 + e^{\alpha + \beta x_i}}\big)$$

\noindent Log Likelihood:

$$\log \mathscr{L} = \sum_{i=1}^n y_i(\alpha + \beta x_i) - \log(1 + e^{\alpha + \beta x_i})$$

\noindent We want to find $\alpha$ and $\beta$ that maximize the likelihood of observing the data. To maximize the likelihood (equivalent to maximizing the log likelihood), the next steps are to take the derivatives of the likelihood (or log likelihood) with respect to $\alpha$ and $\beta$, set the equations to $0$, and solve for $\alpha$ and $\beta$.\\\\

\noindent Take the derivative with respect to $\alpha$, set to $0$ and solve:

$$
\dfrac{\partial}{\partial \alpha} \log \mathscr{L} = \sum \big(y_i - \dfrac{1}{1 + e^{\alpha + \beta x_i}}e^{\alpha + \beta x_i}\big) = 0
$$

$$
\sum y_i = \sum \dfrac{e^{\hat{\alpha} + \hat{\beta}x_i}}{1 + e^{\hat{\alpha} + \hat{\beta}x_i}}
$$

\noindent Take the derivative with respect to $\beta$, set to $0$ and solve:

$$
\dfrac{\partial}{\partial \beta} \log \mathscr{L} = \sum \big(y_i x_i - \dfrac{1}{1 + e^{\alpha + \beta x_i}}e^{\alpha + \beta x_i}x_i\big) = 0
$$

$$
\sum y_i x_i = \sum x_i \dfrac{e^{\hat{\alpha} + \hat{\beta} x_i}}{1 + e^{\hat{\alpha} + \hat{\beta} x_i}} \quad \quad (1)
$$

\noindent If we use the design matrix
$$
\boldsymbol{X} = \begin{bmatrix}
    \boldsymbol{1}       & \boldsymbol{x_1} & \dots & \boldsymbol{x_p}
\end{bmatrix}
$$
with $p$ explanatory variables, and the response vector $\boldsymbol{y}$, the equation $(1)$ becomes

$$
\boldsymbol{X^t y} = \boldsymbol{X^t \hat{\pi}}
$$
\\
where $\hat{\pi}$ is the vector of estimated $\pi_i$ values for the observations with values $(1, x_{1,i}, \ldots, x_{p,i})$. $x_{j,i}$ are the observed data values and the $1$ value is added so that an intercept value $\alpha$ is included in the linear model.
\\\\
This equation is nonlinear so instead of solving directly for $\boldsymbol{\hat{\pi}}$, we will use an iterative method called Newton-Raphson.

\section{Newton-Raphson}

The Newton-Raphson method can be used to approximate the roots of a real-valued function. \\\\
Steps:
\begin{enumerate}
    \item Start with an initial guess $x_0$ for the root
    \item Iterative step: $x_{i+1} = x_i - \dfrac{f(x_i)}{f'(x_i)}$. Note that $(x_{i+1}, 0)$ is the intersection of the $x-axis$ and the line tangent to $f$ at $x_i$
    \item Repeat the iterative step until $f(x_i)$ is sufficiently close to $0$. Then the value $x_i$ is the approximation for the root.
\end{enumerate}

\noindent The diagram at the top of the next page provides a visual for the above steps.\\

\begin{figure}[hbtp]
        \centering
        \includegraphics[width=4.5in]{NewtonRaphson.png}
        \caption{Newton Raphson Diagram}
        \label{Newton Raphson}
    \end{figure}

\noindent Next, we will show how to apply the Newton-Raphson method to find the estimate  $\boldsymbol{\hat{\beta}}$ for $\boldsymbol{\beta}$. Note that we have switched to vector notation so instead of estimating $\alpha$ and $\beta$, we want to estimate $\boldsymbol{\beta} =
\begin{bmatrix}
    \alpha \\
    \beta_1 \\
    \vdots \\
    \beta_p
\end{bmatrix}$ where $\beta_i$ is the coefficient for the explanatory variable $x_i$ and $\alpha$ is the intercept. \\

\noindent Suppose we want to find the maximum likelihood estimate $\hat{\theta_n}$ for the parameter $\theta$. Let the derivative of the log likelihood function be the function $\ell'$. We want to find the roots of this function, which means we want to find $\hat{\theta_n}$ such that $\ell'(\hat{\theta_n}) = 0$, which means $\hat{\theta_n}$ is the maximum likelihood estimate for $\theta$
\\\\
Recall the general Taylor Series expansion formula:
$$
f(x) = f(a) + \dfrac{f'(a)}{1!}(x - a) + \dfrac{f''(a)}{2!}(x-a)^2 + \ldots
$$
where $f$ is infinitely differentiable at $a$.
\\\\\\
Using a one term Taylor Series expansion, we get that:
$$
\ell'(\hat{\theta_n}) = \ell'(\theta_0) + \ell''(\theta_0)*(\hat{\theta_n} - \theta_0)
$$
\\
It can be shown that $\ell''(\theta_0)$ can be estimated by $E[\ell''(\theta_0)] = -I(\theta_0)$ where $I$ is the Fisher information. \\\\

Now we get:
$$
\ell'(\hat{\theta_n}) = \ell'(\theta_0) - I(\theta_0)*(\hat{\theta_n} - \theta_0)
$$
\\
Rearranging terms we get:
$$
\hat{\theta_n} = \theta_0 + \ell'(\theta_0)[I(\theta_0)]^{-1}
$$

\noindent Now, we can implement Newton-Raphson in the following manner:

\begin{enumerate}
    \item Start with the initial value $\theta_0$
    \item $\theta_{k+1} = \theta_k + \ell'(\theta_k)[I(\theta_k)]^{-1}$
    \item Iterate until $\theta_k \approx \theta_{k+1}$ or until $\ell'(\theta_{k+1}) \approx 0$
\end{enumerate}

\noindent For the case of logistic regression, the parameter $\theta = \boldsymbol{\beta}$\\

\noindent Although not discussed here, it can be shown that for Bernoulli data $I(\boldsymbol{\beta}_{(k)}) = \boldsymbol{X}^t \boldsymbol{W}_{(k)} \boldsymbol{X}$.\\

\noindent $\boldsymbol{W}_{(k)}$ is an $N \times N$ diagonal matrix with the $i^{th}$ diagonal element is $\pi_i(1-\pi_i)$ where $\pi_i$ is the fitted probability for the $i^{th}$ observation using $\boldsymbol{\beta}_{(k)}$\\

\noindent The following is the Newton-Raphson implementation for logistic regression:
\begin{enumerate}
    \item Start with the initial value $\boldsymbol{\beta}_0 = \begin{bmatrix}
    0 \\
    \vdots \\
    0
    \end{bmatrix}$
    \item $\boldsymbol{\hat{\beta}}_{(k+1)} = \boldsymbol{\hat{\beta}}_{k} + (\boldsymbol{X}^t \boldsymbol{W}_{(k)} \boldsymbol{X})^{-1}[\boldsymbol{X}^t\boldsymbol{y} - \boldsymbol{X}^t\boldsymbol{\hat{\pi}}] = \boldsymbol{\hat{\beta}}_{k} + (\boldsymbol{X}^t \boldsymbol{W}_{(k)} \boldsymbol{X})^{-1}\boldsymbol{X}^t(\boldsymbol{y - \hat{\pi}})$
    \\
    Matrix inversion can be costly. (See \href{https://www.johndcook.com/blog/2010/01/19/dont-invert-that-matrix/}{Don't invert that matrix} for more details). We will manipulate the above equation so the iteration can be performed without needing to invert any matrices.
    \\\\
    $\boldsymbol{\hat{\beta}}_{(k+1)} - \boldsymbol{\hat{\beta}}_{k} = (\boldsymbol{X}^t \boldsymbol{W}_{(k)} \boldsymbol{X})^{-1}\boldsymbol{X}^t(\boldsymbol{y - \hat{\pi}})$
    \\\\
    $(\boldsymbol{X}^t \boldsymbol{W}_{(k)} \boldsymbol{X})(\boldsymbol{\hat{\beta}}_{(k+1)} - \boldsymbol{\hat{\beta}}_{k}) = \boldsymbol{X}^t(\boldsymbol{y - \hat{\pi}})$
    \\\\
    Use a matrix solver to solve for $\boldsymbol{x}$ in $\boldsymbol{Ax} = \boldsymbol{b}$ where
    \begin{itemize}
    \item $\boldsymbol{A} = \boldsymbol{X}^t \boldsymbol{W}_{(k)} \boldsymbol{X}$
    \item $\boldsymbol{x} = \boldsymbol{\hat{\beta}}_{(k+1)} - \boldsymbol{\hat{\beta}}_{k}$
    \item $\boldsymbol{b} = \boldsymbol{X}^t(\boldsymbol{y - \hat{\pi}})$
    \end{itemize}

    Note that $\boldsymbol{\beta}_{(k+1)} = \boldsymbol{x} + \boldsymbol{\hat{\beta}}_{k}$ and $\boldsymbol{\hat{\pi}} = \frac{e^{\boldsymbol{X\beta_{(k)}}}}{1 + e^{\boldsymbol{X\beta_{(k)}}}}$

    \item Iterate until $\boldsymbol{\hat{\beta}}$ has converged. At convergence, $(\boldsymbol{X}^t \boldsymbol{W}_{(k)} \boldsymbol{X})^{-1}$ is $\approx 0$
\end{enumerate}

\section{How to Predict the Class of a New Observation}

Recall the link function:
$$
\log \big( \dfrac{\pi_i}{1-\pi_i} \big) = \alpha + \beta x_i
$$

\noindent We can rewrite this in vector notation as the following:
$$
\log \big( \dfrac{\pi_i}{1-\pi_i} \big) = \boldsymbol{x_i}^t \boldsymbol{\hat{\beta}}
$$

where $\boldsymbol{x_i} = \begin{bmatrix}
    x_{1,i} \\
    \vdots \\
    x_{p,i}
    \end{bmatrix}$ is the vector that contains the data values for each explanatory variable $x_i$ and $\boldsymbol{\hat{\beta}}$ is the vector of coefficients found by the Newton-Raphson calculation.\\

\noindent Given the link function:
$$
\pi_i = \dfrac{e^{\alpha + \beta x_i}}{e^{\alpha + \beta x_i} + 1}
$$

\noindent In vector notation, this is:
$$
\pi_i = \dfrac{e^{\boldsymbol{x_i}^t \boldsymbol{\hat{\beta}}}}{e^{\boldsymbol{x_i}^t \boldsymbol{\hat{\beta}}} + 1}
$$

\noindent Given a new vector of observations $\boldsymbol{z}$, we can plug $\boldsymbol{z}$ in for $\boldsymbol{x_i}$ in the above formula and solve for $\pi_i$.

A basic classification procedure is the following:

\begin{itemize}
    \item If $\pi_i \geq 0.5$, we classify the new observation as class $1$
    \item If $\pi_i < 0.5$, we classify the new observation as class $0$
\end{itemize}

\noindent The above classification procedure can be modified as desired.

\begin{thebibliography}{9}

\bibitem{notes citation}
Lecture notes from Professor Deborah Nolan, UC Berkeley, STAT 151A, Spring 2017

\bibitem{stats textbook}
Trevor Hastie, Robert Tibshirani, and Jerome Friedman.
\textit{The Elements of Statistical Learning 2nd Edition}.
Springer, New York, NY, 2009.

\bibitem{image citation}
Image citation: Ruye Wang - Newton-Raphson method (univariate)
\\\texttt{http://fourier.eng.hmc.edu/e176/lectures/NM/node20.html}

\bibitem{don't invert that matrix}
``Don't invert that matrix" citation: John D. Cook
\\\texttt{https://www.johndcook.com/blog/2010/01/19/dont-invert-that-matrix/}
\end{thebibliography}

\section{Data Structures}

\bi
\item \(X\) is an \(N \times M\) matrix containing the input data concatenated with the value $1$.
  \(X_{i, j}\) is value of \(j^{th}\) attribute of \(i^{th}\)
  instance. \(X\) is stored as \(M\) columns, where \(X_j\) is
  observations for attribute \(j\)

\item \(y\) is an \(N \times 1\) classification vector. \(y_i\) is
  classification of instance \(i\) and can be 1 or 0.
\item \(\beta\) is an \(M \times 1\) coefficient vector. \(\beta_j\)
  is coefficient for attribute \(j\).
\item \(\beta^{\mathrm new}\) is an \(M \times 1\) vector, which are the new
  coefficients that we solve for in each iteration
\item \(A\) is an \(M \times M\) matrix. Since it is symmetric, we can skip
  computing the lower diagonal elements.
\item \(W = X^TWX\) is a diagonal \(N \times N\) matrix. Since the
  off-diagonal elements are zero, we can represent is as an \(N \times
  1\) vector.   When used as a vector, we will use lower case \(w\). When used
  as a matrix, we will use upper case \(W\). Note that \(W_{i, i} = w_i\) and
  that \( i \neq j \Rightarrow W_{i,j} = 0\)
\item \(b\) is an \(M \times 1\) vector, \(X^T W ( X \beta + W^{-1}(y - p) )\)
  \ei

With these elements, the Newton Raphson iteration becomes the following:
\\
$(X^TWX)\times (\beta^{new} - \beta) = X^T(y-p)$

where
\be
\item \(W\) is symmetric and positive definite
\item Because \(W\) is symmetric and positive definite, \(A =
  X^T W X\) is at least positive semi-definite.
\item If the attributes are linearly independent, then \(A\) will actually be
  positive definite; else, the dependent attributes should be removed prior to
  starting the computation.

\framebox{{\bf ANDREW Any easy way to do the above? }}

\ee

\section{computations}

Step by step computations in Table~\ref{step_by_step_calc}.
\begin{table}[ht]
\centering
\begin{tabular}{|l|l|l|l|} \hline \hline
  {\bf Name} & {\bf Description} & {\bf Type} & {\bf Code} \\ \hline \hline
  \(Xbeta\) & \(X \beta\) & \((N \times M) \times (M \times 1)\)  &  \\
  & & \(= N \times 1\) & \(Xbeta = \mathrm{mv\_mul} (X, beta)\) \\ \hline
  \(p\) & & \(N \times 1\) & \( p = \mathrm{logit}(t1) = e^{t1}/(1 + e^{t1})\) \\ \hline
  \(ysubp\) &  \(y - p\) & \(N \times 1\) & \( t_2 = \mathrm{vvsub}(y, p)\) \\ \hline
  \(w\) & & \(N \times 1\) & \( w = \mathrm{logit2}(t1) = e^{t1}/(1 + e^{t1})^2\) \\ \hline
  \(b\) & \(X^T (y-p)\) & \((M\times N) \times (N \times 1)\)
  & \(\forall j_{j=1}^{j=M} b_j = \) \\
        & & \( = M \times 1 \) & \(\mathrm{sum}(\mathrm{vvmul}(X_i, ysubp))\) \\ \hline
  \(A\) & \(X^T W X\) & \((M \times N) \times (N \times N) \times (N \times M)\)
  & \(\forall_{j=1}^{j=M} \forall_{k=j}^{k=M} A_{j, k} = \) \\
  & & \(= (M \times M)\) & \(\mathrm{sumprod2}(X_j, w, X_k)\) \\ \hline
  \hline

\hline
\end{tabular}
\caption{Listing of individual steps and intermediate values}
\label{step_by_step_calc}
\end{table}

\begin{table}[hb]
\centering
\begin{tabular}{|l|l|l|l|l|} \hline \hline
  {\bf Name} & {\bf Input Type} & {\bf Output Type} & {\bf Return Value} \\ \hline \hline
  logit & Vector \(x\) & Vector \(y\) & \(y = \frac{e^x}{1 + 1^x}\) \\ \hline
  logit2 & Vector \(x\) & Vector \(y\) & \(y = \frac{e^x}{(1 + 1^x)^2}\) \\ \hline
  vvadd & Vector \(x\), Vector \(y\) & Vector \(z\) & \(z = x + y \)  \\ \hline
  vvsub & Vector \(x\), Vector \(y\) & Vector \(z\) & \(z = x - y \)  \\ \hline
  vvmul & Vector \(x\), Vector \(y\) & Vector \(z\) & \(z = x \times y \)  \\ \hline
  vvdiv & Vector \(x\), Vector \(y\) & Vector \(z\) & \(z = x / y \)  \\ \hline
  vsmul & Vector \(x\), Scalar \(y\) & Vector \(z\) & \(\forall i:~z_i = x_ \times y \)  \\ \hline
  sumprod & Vector \(x\), Vector \(y\) & Scalar \(z\) & \(z = \sum_i (x_i \times y_i)\) \\ \hline
  sumprod2 & Vector \(x\), Vector \(y\), Vector \(z\) & Scalar \(w\) &  \(w = \sum_i (x_i \times y_i \times z_i)\) \\ \hline
\hline
\end{tabular}
\caption{Necessary Operators}
\label{tbl_custom_ops}
\end{table}

\section{Details}

\subsection{Notes}

\be
\item The calculation of \(A\) is simplified by the fact that the off-diagonal
  elements of \(w\) are 0 and that it is a symmetric matrix. See last row of
  Table~\ref{step_by_step_calc}
\ee
\subsection{Clarifications needed}

\be
\item Initial guess for \(\beta\)
\ee

\section{Putting it all together}
The Q code will look like the following.

\begin{verbatim}
t1 = mvmul(X, beta)
p = logit(t1)
w = logit2(t1)
t2 = vvsub(y, p)
for j in 1 to M do
  b[j] = sumprod(Xj, t2)
end
for j in 1 to M do
  for k in j to M do
    A[j][k] = sumprod2(Xj, w, Xk)
  end
end
\end{verbatim}

\end{document}