\section[The Riemann-Stieltjes Integral]{\hyperlink{toc}{The Riemann-Stieltjes Integral}}

\subsection{Definition of the Integral}
\begin{definition}{Partition}{6.1a}
    A \textbf{partition} of $[a, b] \subset \RR$ is a set $\set{x_0, x_1, \ldots, x_n}$ (for some $n \in \NN$) such that:
    \begin{align*}
        a = x_0 \leq x_1 \leq x_2 \leq \ldots \leq x_{n-1} \leq x_n = b
    \end{align*}
    We can then write:
    \begin{align*}
        \Delta x_i = x_i - x_{i-1}
    \end{align*}
\end{definition}

\begin{figure}[htbp]
    \centering
    \begin{tikzpicture}[scale=2]
        \draw[very thick, latex-latex] (-2, 0) -- (2, 0);
        \node[] at (-1.5, 0) {$[$};
        \node[below] at (-1.5, -0.15) {$x_0$};
        \node[below] at (-1.5, -0.45) {$a$};
        \node[] at (1.5, 0) {$]$};
        \node[below] at (1.5, -0.15) {$x_4$};
        \node[below] at (1.5, -0.4) {$b$};
        \draw[] (-1, 0) -- (-1, -0.15);
        \node[below] at (-1, -0.15) {$x_1$};
        \draw[] (-0.2, 0) -- (-0.2, -0.15);
        \node[below] at (-0.2, -0.15) {$x_2$};
        \draw[] (0.4, 0) -- (0.4, -0.15);
        \node[below] at (0.4, -0.15) {$x_3$};
    \end{tikzpicture}
    \caption{Visualization of a partition $\set{x_0, x_1, x_2, x_3, x_4}$ of $[a, b]$. Note that the points in the partitions need not be equally spaced.}
    \label{fig27}
\end{figure}

\setcounter{rudin}{0}
\begin{definition}{Upper and Lower Sums}{6.1b}
    Given $f: [a, b] \mapsto \RR$ and a partition $P$ of $[a, b]$ let:
    \begin{align*}
        M_i &= \sup\set{f(x): x_{i-1} \leq x \leq x_i}
        \\ m_i &= \inf\set{f(x): x_{i-1} \leq x \leq x_i}
    \end{align*}
    Then, we can define the \textbf{upper} and \textbf{lower sums}:
    \begin{align*}
        U(P, f) &= \sum_{i=1}^n M_i \Delta x_i
        \\ L(P, f) &= \sum_{i=1}^n m_i \Delta x_i
    \end{align*}
\end{definition}

\begin{figure}[htbp]
    \centering
    \begin{tikzpicture}[scale=2]
        \draw[very thick, latex-latex] (-2, 0) -- (2, 0);
        \node[] at (-1.5, 0) {$[$};
        \node[below] at (-1.5, -0.15) {$x_0$};
        \node[below] at (-1.5, -0.45) {$a$};
        \node[] at (1.5, 0) {$]$};
        \node[below] at (1.5, -0.15) {$x_4$};
        \node[below] at (1.5, -0.4) {$b$};
        \draw[] (-1, 0) -- (-1, -0.15);
        \node[below] at (-1, -0.15) {$x_1$};
        \draw[] (-0.2, 0) -- (-0.2, -0.15);
        \node[below] at (-0.2, -0.15) {$x_2$};
        \draw[] (0.4, 0) -- (0.4, -0.15);
        \node[below] at (0.4, -0.15) {$x_3$};
        \draw[] (-1, 0) sin (-1.5, 1);
        \draw[] (-1, 0) sin (-0.5, -1);
        \draw[] (0, 0) sin (-0.5, -1);
        \draw[] (0, 0) sin (0.5, 1);
        \draw[] (1, 0) sin (0.5, 1);
        \draw[] (1, 0) sin (1.5, -1);
        \node[circle, fill = red, minimum size = 0.2cm, inner sep = 0pt, label={[right, text=red]:$M_1$}] at (-1.5, 1) {};
        \node[circle, fill = purple, minimum size = 0.2cm, inner sep = 0pt, label={[right, text=blue]:$m_1$}] at (-1, 0) {};
        \node[right, text = red] at (-0.8, 0.06) {$/M_2$};
        \node[circle, fill = blue, minimum size = 0.2cm, inner sep = 0pt, label={[above, text=blue]:$m_2$}] at (-0.5, -1) {};
        \node[circle, fill = blue, minimum size = 0.2cm, inner sep = 0pt, label={[right, text=blue]:$m_3$}] at (-0.2, -0.6) {};
        \node[circle, fill = red, minimum size = 0.2cm, inner sep = 0pt, label={[left, text=red]:$M_3$}] at (0.4, 0.94) {};
        \node[circle, fill = red, minimum size = 0.2cm, inner sep = 0pt, label={[right, text=red]:$M_4$}] at (0.5, 1) {};
        \node[circle, fill = blue, minimum size = 0.2cm, inner sep = 0pt, label={[above, text=blue]:$m_4$}] at (1.5, -1) {};
        \end{tikzpicture}
    \caption{Example of a function $f$, a partition $P$ of $[a, b]$, and the $M_i, m_i$s for this choice of partition.}
    \label{fig28}
\end{figure}

\noindent By construction, it should be evident that $L(P, f) \leq U(P, f)$ for all $P, f$.

A natural question that arises from the form of the above expression is whether these are Riemann sums or not. Recall from first year calculus that we would choose the left endpoint, right endpoint, or some other arbitrary choice of a point in the subinterval. Here, we in a sense use a ``special case'' of the supremum/infimum. We will see that this choice is musch easier to use in proofs due to monotonicity properties. Namely, if we have a partition and add another point, then $U(P, f)$ can only decrease, and $L(P, f)$ can only increase (we will see this in a theorem soon)!

\setcounter{rudin}{0}
\begin{definition}{Upper/Lower Integrals and Riemann Integrability}{6.1c}
    We define the \textbf{upper Riemann integral} to be:
    \begin{align*}
        \uint{a}{b} f dx = \inf_P U(P, f).
    \end{align*}
    and the \textbf{lower Riemann integral} to be:
    \begin{align*}
        \lint{a}{b} f dx= \sup_P L(P, f).
    \end{align*}
    Here, the infimum/supremum is taken over all partitions $P$. We say that $f$ is \textbf{Riemann integrable} on $[a, b]$, and write $f \in \R[a, b]$ if:
    \begin{align*}
        \uint{a}{b} f dx= \lint{a}{b} f dx
    \end{align*}
    which we can write as:
    \begin{align*}
        \int_{a}^{b} f dx \text{ or } \int_{a}^{b} f(x) dx
    \end{align*}
\end{definition}
\noindent Note that the choice of variable in the above definition is totally arbitrary.

Also, note that while $f$ is not required to be continuous in the above definition, it is required to be bounded; else, $M_i$ and $m_i$ may not exist. Since $f$ is bounded, $U(P, f)$, $L(P, f)$ are bounded for all $P, f$ and hence we have a set of real numbers for which we may consider the supremum/infimium of by the LUB/GLB property of the reals. Since the upper/lower sums lie in a bounded interval, there is no questions about whether the lower/upper integrals exist. The question becomes whether they are equal or not. Before getting into further discussion on this topic, we discuss a bound:

\setcounter{rudin}{0}
\begin{theorem}{ML Bounds}{6.1}
    Let $m = \inf\set{f(x): a \leq x \leq b}$ and $M = \sup\set{f(x): a \leq x \leq b}$ (which exist by the boundedness of $f$). Then,
    \begin{align*}
        m(b - a) \leq L(P, f) \leq U(P, f) \leq M(b - a)
    \end{align*}
    For any choice of partition $P$.
\end{theorem}
\begin{nproof}
    For any $i$, we have that:
    \begin{align*}
        m \leq m_i \leq M_i \leq M
    \end{align*}
    Therefore:
    \begin{align*}
        \sum_{i=1}^n m \Delta x_i \leq \sum_{i=1}^n m_i \Delta x_i \leq \sum_{i=1}^n M_i \Delta x_i \leq \sum_{i=1}^n M \Delta x_i
    \end{align*}
    So we conclude that:
    \begin{align*}
        m(b - a) \leq L(P, f) \leq U(P, f) \leq M(b - a)
    \end{align*} \qed
\end{nproof}

\noindent Now that we have established the Riemann integral, a natural question is how can we extend this notion. In order to do so, we will use a montonically increasing function $\alpha: [a, b] \mapsto \RR$ (that is, $\alpha(x) \leq \alpha(y)$ for all $x \leq y$). Note that $\alpha$ need not be continuous. Indeed, compared to the Riemann integral where $\alpha(x) = x$ and was continuous, in this general setting, $\alpha$ is allowed to have jumps. This allows for certain benefits, as we will soon discuss. However, we note that $\alpha$ can only have a finite number of jumps.

\begin{ntheorem}{ 4.30}{}
    Let $\alpha: [a, b] \mapsto \RR$ be monotonic. Then, it can only have finitely many discontinities.
\end{ntheorem}
\begin{nproof}
    Assign a rational number $r(x)$ to each of the discontinuities of $\alpha$. Then, we have that:
    \begin{align*}
        \lim_{x \rightarrow r(x)^-} \alpha(x) < \alpha(x) < \lim_{x \rightarrow r(x)^+} \alpha(x)
    \end{align*}
    Since $x_1 < x_2$ implies $\lim_{x \rightarrow r(x_1)^+} \alpha(x) \leq lim_{x \rightarrow r(x_2)^-} \alpha(x)$, we have that $r(x_1) \neq r(x_2)$ if $x_1 \neq x_2$. We therefore have established a function $r$ from the set of discontinuities of $\alpha$ to the rationals. As the rationals are countable, the set of discontiuities of $\alpha$ are also countable. \qed
\end{nproof}

\noindent With this established, we now define the generalized Riemann-Stieltjes integral.

\begin{definition}{Riemann-Stieltjes Integral}{6.2}
    Let $\alpha: [a, b] \mapsto \RR$ be increasing, and given a partition $P$ of $[a, b]$, define:
    \begin{align*}
        \Delta \alpha_i = \alpha(x_i) - \alpha(x_{i-1}) (\geq 0)
    \end{align*}
    For bounded $f: [a, b] \mapsto \RR$, Let:
    \begin{align*}
        U(P, f, \alpha) &= \sum_{i=1}^n M_i \Delta \alpha_i
        \\ L(P, f, \alpha) &= \sum_{i=1}^n m_i \Delta \alpha_i
    \end{align*}
    We then take the infimum/supremum over partitions $P$ to get:
    \begin{align*}
        \uint{a}{b} f d\alpha &= \inf_P U(P, f, \alpha)
        \\ \lint{a}{b}f d\alpha &= \sup_P L(P, f, \alpha)
    \end{align*}
    If equal, we write their value as:
    \begin{align*}
        \int_a^b f d\alpha \text{ or } \int_a^b f(x)d\alpha(x)
    \end{align*}
    and we write that $f \in \R_\alpha[a, b]$. In the case where $\alpha(x) = x$, we recover the Riemann integral.
\end{definition}

\noindent Why is this definition useful? What does it accomplish for us that the original Riemann integral does not? We consider a physically motivating example. Suppose we have a thin wire with varying mass density $\rho(x)$. If we wanted to calculate the mass density of the wire, we would integrat the density $\rho(x)$ over the length of the wire. Now, suppose our wire consists of steel of continuously varying mass density, as well as beads/point masses placed on certain locations of the wire. The Riemann integral cannot handle these point masses, but the Riemann-Stieltjes integral can deal with this case if we use an $\alpha$ with discontinuities in it. Hence, the Riemann-Stieltjes integral allows us to handle cases where we both have continuous and discrete masses to integrate over. It acts as a bridge between Riemann and Lebesgue integration (the latter of which will be the subject of a later course in measure theory).

We now will answer the question: ``for what choices of $f, \alpha$ is $f$ Riemann-Stieltjes integrable?''

\subsection{Criterion for Integrability}

\begin{definition}{Refinements and Common Refinemnet}{6.3}
    $P^*$ is a \textbf{refinement} of $P$ if $P \subset P^*$ and $P, P^*$ are partitions. The common refinemnet of $P_1, P_2$ is $P^* = P_1 \cup P_2$.
\end{definition}

\begin{theorem}{}{6.4}
    If $P^*$ is a refinemne tof $P$, then:
    \begin{align*}
        L(P, f, \alpha) \leq L(P^*, f, \alpha) \leq U(P^*, f, \alpha) \leq U(P, f, \alpha)
    \end{align*}
\end{theorem}

\noindent As a remark, when we take infimums/supremums over partitions $P$ to obtain the upper/lower Riemann-Stieltjes integals, we are taking refinements.

Also, note that the above theorem does \textit{not} apply to (right-hand, left-hand, midpoint, arbitrary) Riemann sums, and is a consequence of the choice of upper/lower sums with supremums/infimums taken over the subintervals.

\begin{nproof}
    It suffices to consider $P^*$ with a single extra point $x_{i - 1} < x^* < x_i$, and then the genral case follows by induction.

    For the case where $\alpha(x) = x$, the refinement adds $(m^* - m_i)(x^* - x_{i-1}) \geq 0$.

    For the general case, we have that:
    \begin{align*}
        L(P^*, f, \alpha) - L(P, f, \alpha) &= \left(m^*(\alpha(x^*) - \alpha(x_{i-1})) + m_i(x^* - x_{i-1}) \right)- m_i(\alpha_{x_i} - \alpha_{x_{i-1}})
        \\ &= \left(m^*(\alpha(x^*) - \alpha(x_{i-1})) + m_i(x^* - x_{i-1}) \right)
        \\ &- m_i\left[(\alpha(x^*) - \alpha(x_{i-1})) + (\alpha(x_i) - \alpha(x^*))\right]
        \\ &= (m^* - m_i)(\alpha(x^*) - \alpha(x_i))
    \end{align*}
    $\alpha$ is montonically increasing, so $x^* \geq x_i$ implies that the second term is positive. Furthermore, $m^* \geq m_i$ as $\inf\set{f(x): x \in [x_{i-1}, x^*]} \geq \inf\set{f(x): x \in [x_{i-1}, x_i]}$. It follows that:
    \begin{align*}
        L(P^*, f, \alpha) - L(P, f, \alpha) \geq 0
    \end{align*}
    and the proof for $U(P^*, f, \alpha) - U(P, f, \alpha) \leq 0$ follows analogously. \qed
\end{nproof}
\begin{figure}[htbp]
    \centering
    \begin{tikzpicture}[scale=1.5]
        \draw[latex-latex, very thick] (-1.5, 0) -- (1.5, 0);
        \draw[] (-1, 0) sin (-0.5, 1);
        \draw[] (0, 0) sin (-0.5, 1);
        \draw[] (0, 0) sin (0.5, -1);
        \draw[] (1, 0) sin (0.5, -1);
        \draw[] (-1, 0) -- (-1, -0.15);
        \draw[] (1, 0) -- (1, -0.15);
        \node[below] at (-1, -0.15) {$x_{i-1}$};
        \node[below] at (1, -0.15) {$x_i$};
        \draw[] (-0.25, 0) -- (-0.25, -0.15);
        \node[below] at (-0.25, -0.12) {$x^*$};
        %\node[circle, fill = red, minimum size = 0.2cm, inner sep = 0pt, label={[right, text=red]:$M_{i\text{original}}/M_{i\text{new}}$}] at (-0.5, 1) {};
        \node[circle, fill = blue, minimum size = 0.2cm, inner sep = 0pt, label={[above, text=blue]:$m_{i}$}] at (0.5, -1) {};
        %\node[circle, fill = red, minimum size = 0.2cm, inner sep = 0pt, label={[right, text=red]:$M_{i+1\text{new}}$}] at (-0.25, 0.7) {};
        \node[circle, fill = blue, minimum size = 0.2cm, inner sep = 0pt, label={[above, text=blue]:$m^*$}] at (-1, 0) {};
    \end{tikzpicture}

    \caption{Visualization of the effect of adding an extra point $x^*$ to the partition $P$. We can see that this has the net effect of increasing $L(P, f, \alpha)$ as $m^* \geq m_i$.}
    \label{fig29}
\end{figure}

\newpage
Note that as a point of notation, if $f, \alpha$ are fixed, we sometimes can write $L(P), U(P)$ in place of $L(P, f, \alpha)$ and $U(P, f, \alpha)$. Sometimes where the context is clear, it is also common to write $\R(\alpha)$ in place of $\R_\alpha[a, b]$.

\begin{theorem}{}{6.5}
    $\lint{a}{b} f d\alpha \leq \uint{a}{b} f d\alpha$.
\end{theorem}
\begin{nproof}
    For partitions $P_1, P_2$, let $P^* = P_1 \cup P_2$ be the common refinement. By Theorem \ref{thm:6.4}, we have that:
    \begin{align*}
        L(P_1) \leq L(P^*) \leq U(P^*) \leq U(P_2)
    \end{align*}
    And in particular, $L(P_1) \leq U(P_2)$. Therefore, for any fixed $P_2$, $U(P_2)$ is an upper bound on the set of all lower sums. As the supremum is the least upper bound, we have that:
    \begin{align*}
        \sup_{P_1} L(P_1) \leq U(P_2)
    \end{align*}
    Therefore, as $\sup_{P_1} L(P_1)$ is a lower bound on the set of all upper sums, and the infimum is the greatest lower bound, we have that:
    \begin{align*}
        \sup_{P_1} L(P_1) \leq \inf_{P_2} U(P_2)
    \end{align*}
    So therefore:
    \begin{align*}
        \lint{a}{b} f d\alpha \leq \uint{a}{b} f d\alpha
    \end{align*}
    as claimed. \qed
\end{nproof}

\begin{theorem}{\texorpdfstring{$\e$}{e}-Criterion for Integrability}{6.6}
    $f \in \R_\alpha[a, b]$ if and only if for all $\e > 0$, there exists a partition $P_\e$ of $[a, b]$ such that $U(P_\e) - L(P_\e) < \e$.
\end{theorem}
\begin{nproof}
    $\boxed{\implies}$ By hypothesis, $\sup_P L(P) = \int_a^b fd\alpha = \inf_P U(P)$. Let $\e > 0$. Then by the property of sup/inf, there exist $P_1, P_2$ such that:
    \begin{align*}
        \int_a^b f d\alpha - L(P_1) &< \frac{\e}{2}
        \\ U(P_2) - \int_a^b f d\alpha &< \frac{\e}{2}
    \end{align*}
    Adding the first inequality to the second, we get:
    \begin{align*}
        U(P_2) - L(P_1) < \e
    \end{align*}
    Letting $P^* = P_1\cup P_2$, by Theorem \ref{thm:6.4} we have that:
    \begin{align*}
        U(P^*) - L(P^*) \leq U(P_2) - L(P_1) < \e
    \end{align*}
    which proves the claim.

    $\boxed{\impliedby}$ Let $\e > 0$ Then by Theorem \ref{thm:6.5} we have that:
    \begin{align*}
        0 \leq \uint{a}{b} f d\alpha - \lint{a}{b} f d\alpha
    \end{align*}
    and furthermore:
    \begin{align*}
        0 \leq \uint{a}{b} f d\alpha - \lint{a}{b} f d\alpha \leq U(P_\e) - L(P_\e) < \e
    \end{align*}
    Where the second inequality is true for any choice of partition. $\e$ is arbitrary, so we conclude that:
    \begin{align*}
        \uint{a}{b} f d\alpha - \lint{a}{b} f d\alpha = 0
    \end{align*}
    And therefore $\uint{a}{b} f d\alpha = \lint{a}{b} f d\alpha$, and $f \in \R_\alpha[a, b]$. \qed
\end{nproof}

\noindent Theorem 6.7 is a little technical, so we shall skip it for now.

\stepcounter{rudin}
\begin{theorem}{Continuity implies integrability}{6.8}
    If $f$ is continuous on $[a, b]$, then $f \in \R_\alpha[a, b]$.
\end{theorem}
\noindent Note in the above theorem that we make no assumptions on $\alpha$, only that (of course) it is monotonic.
\begin{nproof}
    By definition, $U(P) - L(P) = \sum_{i=1}^n (M_i - m_i)\Delta \alpha_i$. The idea will be to choose small intervals to make these differences small. Since $[a, b]$ is compact, $f$ is uniformly continuous by Theorem \ref{thm:4.19}. So, for all $\eta > 0$, there exists $\delta > 0$ such that $\abs{f(x) - f(t)} < \eta$ if $\abs{x - t} < \delta$. Thus, if $P$ is constructed such that $\Delta x_i < \delta$ then $M_i - m_i < \eta$. We then have that:
    \begin{align*}
        U(P) - L(P) \leq \sum_{i=1}^n \eta \Delta \alpha_i = \eta \sum_{i=1}^n (\alpha(x_{i}) - \alpha(x_{i-1})) = \eta\left(\alpha(b) - \alpha(a)\right)
    \end{align*}
    Where in the last equality we use the fact that we have a telescoping sum. Given $\e > 0$, we choose $\eta < \frac{\e}{\alpha(b) - \alpha(a)}$. With this choice of partition with $\Delta x_i < \delta = \delta(\eta)$, we have that:
    \begin{align*}
        U(P) - L(P) < \e
    \end{align*}
    and we conclude that $f \in \R_\alpha[a, b]$ by Theorem \ref{thm:6.6}. \qed
\end{nproof}
\noindent A natural question is ``what $f$s are Riemann integrable in general?'' The answer turns out to be ``if $f$ is continuous almost everywhere''. This sounds handwavy, but has a precise definition; although it is not covered in this course, one can refer to Rudin 11.33(b) for details.

\begin{theorem}{}{6.9}
    If $f$ is monotone on $[a, b]$ and $\alpha$ is continuous on $[a, b]$m then $f \in \R_\alpha[a, b]$.
\end{theorem}
\noindent In the proof of Theorem \ref{thm:6.8}, we used the continuity of $f$ to bound the maximum/minimum on each subinterval. Here, $f$ is no longer continuous, so we cannot control the maximum/minimum. Instead, we will use the continuity of $\alpha$ to control the size of the $\Delta \alpha$s.
\begin{nproof}
    Given $n \in \NN$, choose $P$ such that $\Delta \alpha_i = \frac{\alpha(b) - \alpha(a)}{n}$ for all $i \in {1, \ldots, n}$. Note that such a choice is possible by the continuity of $\alpha$ and the IVT (Theorem \ref{thm:4.23}). We then have that:
    \begin{align*}
        U(P) - L(P) = \sum_{i=1}^n (M_i - m_i)\Delta \alpha_i = \frac{\alpha(b) - \alpha(a)}{n}\sum_{i=1}^n (M_i - m_i)
    \end{align*}
    Suppose (WLOG) that $f$ is an increasing function. Then, $M_i = f(x_i)$ and $m_i = f(x_{i-1})$ due to the monotone increasing property. Hence:
    \begin{align*}
        U(P) - L(P) = \frac{\alpha(b) - \alpha(a)}{n}\sum_{i=1}^n f(x_i) - f(x_{i-1}) = \frac{\alpha(b) - \alpha(a)}{n}\left[f(b) - f(a)\right]
    \end{align*}
    Where in the last equality we use the fact that the sum telescopes. If $\alpha(b) = \alpha(a)$, then $U(P) - L(P) = 0$ and the claim immediately follows. If $\alpha(b) > \alpha(a)$, then the claim follows by choosing $n > \frac{1}{\e}\frac{f(b) - f(a)}{\alpha(b) - \alpha(a)}$ (that is, selecting a partition $P$ with sufficiently large $n$), from which it follows that:
    \begin{align*}
        U(P) - L(P) < \e
    \end{align*}
    and hence $f \in \R_\alpha[a, b]$ by Theorem \ref{thm:6.6}. \qed
\end{nproof}
\begin{figure}[htbp]
    \centering
    \begin{tikzpicture}[scale=2]
        \draw[-latex, very thick] (0, 0) -- (2, 0);
        \draw[-latex, very thick] (0, 0) -- (0, 2);
        \foreach \i in {0.5,0.7,...,1.7}{
            \draw[] (0, \i) -- (-0.15, \i);
        }
        \filldraw[] (0.5, 0.5) circle (1pt);
        \filldraw[] (1.7, 1.7) circle (1pt);
        \draw[] (0.5, 0) -- (0.5, -0.15);
        \draw[] (1.7, 0) -- (1.7, -0.15);
        \node[below] at (0.5, -0.16) {$a$};
        \node[below] at (1.7, -0.13) {$b$};
        \node[left] at (-0.15, 0.5) {$\alpha(a)$};
        \node[left] at (-0.15, 1.7) {$\alpha(b)$};
        \draw[dashed] (0, 0.5) -- (0.5, 0.5);
        \draw[dashed] (0.5, 0) -- (0.5, 0.5);
        \draw[dashed] (0, 0.7) -- (0.86, 0.7);
        \draw[dashed] (0.86, 0) -- (0.86, 0.7);
        \draw[dashed] (0, 0.9) -- (0.95, 0.9);
        \draw[dashed] (0.95, 0) -- (0.95, 0.9);
        \draw[dashed] (0, 1.1) -- (1.15, 1.1);
        \draw[dashed] (1.15, 0) -- (1.15, 1.1);
        \draw[dashed] (0, 1.3) -- (1.3, 1.3);
        \draw[dashed] (1.3, 0) -- (1.3, 1.3);
        \draw[dashed] (0, 1.5) -- (1.36, 1.5);
        \draw[dashed] (1.36, 0) -- (1.36, 1.5);
        \draw[dashed] (0, 1.7) -- (1.7, 1.7);
        \draw[dashed] (1.7, 0) -- (1.7, 1.7);
        \draw [] (0.5 , 0.5) to [ curve through ={(0.8, 0.6)..(1, 1)..(1.25, 1.2)..(1.4, 1.6)}] (1.7, 1.7);
    \end{tikzpicture}

    \caption{Visualization of the idea of the proof of Theorem \ref{thm:6.9}. We chop up $[\alpha(a), \alpha(b)]$ into equal sized subintervals.}
    \label{fig30}
\end{figure}

\begin{theorem}{}{6.10}
    Suppose $f: [a, b] \mapsto \RR$ is bounded and has finitely many discontinuities. Suppose $\alpha$ is continuous at every point where $f$ is not. Then, $f \in \R_\alpha[a, b]$.
\end{theorem}

\begin{nproof}
    As has become standard, we will be applying Theorem \ref{thm:6.6}. We have that:
    \begin{align*}
        U(P) - L(P) = \sum_{i=1}^n (M_i - m_i)\Delta \alpha_i
    \end{align*}
    Let $\e > 0$, and let $E = \set{e_1, \ldots e_k}$ be the set of points where $f$ is not continuous. $\alpha$ is continuous at each $e_j$ by hypothesis. Therefore, there exists disjoint intervals $(u_j, v_j)$ that cover points in $E$, such that $u_j < e_j < v_j$ and $\alpha(v_j) - \alpha(v_j) < \e$ (by the continuity of $\alpha$). Let $K = [a, b] \cap \left(\bigcup_{j=1}^k (u_j, v_j)\right)^c$. $K$ is a compact set as it is a finite union of closed intervals. $f$ is continuous on $K$ by hypothesis, so $f$ is uniformly conitnuous on $K$ by Theorem \ref{thm:4.19}. Hence, there exists $\delta > 0$ such that for $s, t \in K$, $\abs{s - t} < \delta \implies \abs{f(s) - f(t)} < \e$. We then form a partition $P = \set{x_1, \ldots, x_n}$ to consist of $\set{u_1, v_1, \ldots, u_k, v_k}$ and additional points in $K$ with $\delta x_i < \delta$. For such $i$, $f$ is continuous on the subinterval and hence $M_i - m_i < \e$. For the other intervals $[u_j, v_j]$, we have that $M_j - m_j \leq 2M$ where $M = \sup\set{\abs{f(x)}: x \in [a, b]}$ and that $\Delta \alpha_j < \e$. Therefore, we have that:
    \begin{align*}
        0 \leq U(P) - L(P) &= \sum_{i=1}^n (M_i - m_i)\Delta \alpha_i
        \\ &\leq 2M\e + \e\left(\alpha(b) - \alpha(a)\right)
    \end{align*}
    Where the first term comes from the $[u_j, v_j]$s and the second term is the maximum possible value from the subintervals of $K$. By choosing $\e$ small enough, the RHS is small as desired for some choice of $P$. Hence, $U(P) - L(P) < \e$ for some $P$ and hence $f \in \R_\alpha[a, b]$. \qed
\end{nproof}
\noindent A natural question that arises is ``what if $f$ and $\alpha$ are discontinuous at the same point?'' In this case, we can construct functions such that $f \notin \R_\alpha[a, b]$ (see HW1Q5).
\begin{figure}[htbp]
    \centering
    \begin{tikzpicture}[scale=2]
        \draw[latex-latex, very thick] (-2, 0) -- (2, 0);
        \node[] at (-1.5, 0) {$[$};
        \node[] at (1.5, 0) {$]$};
        \node[] at (-1.2, 0) {$($};
        \filldraw[] (-1, 0) circle (0.75pt);
        \node[] at (-0.8, 0) {$)$};
        \node[] at (0.2, 0) {$($};
        \filldraw[] (0.5, 0) circle (0.75pt);
        \node[] at (0.8, 0) {$)$};
        \foreach \i in {-1.45, -1.4, ..., -1.21} {
            \draw[blue] (\i, 0.05) -- (\i, -0.05);
        }
        \foreach \i in {-0.75, -.7, ..., 0.19} {
            \draw[blue] (\i, 0.05) -- (\i, -0.05);
        }
        \foreach \i in {0.85, 0.9, ..., 1.49} {
            \draw[blue] (\i, 0.05) -- (\i, -0.05);
        }
        \node[below] at (-1.5, -0.1) {$a$};
        \node[below] at (-1.2, -0.1) {$u_1$};
        \node[below] at (-1, -0.1) {$e_1$};
        \node[below] at (-0.8, -0.1) {$v_1$};
        \node[below] at (0.2, -0.1) {$u_2$};
        \node[below] at (0.5, -0.1) {$e_2$};
        \node[below] at (0.8, -0.1) {$v_2$};
        \node[below] at (1.5, -0.08) {$b$};
    \end{tikzpicture}
    \caption{Visualization of how $[a, b]$ gets split up in the proof of Theorem \ref{thm:6.10}. $K$ consists of the union of the closed intervals marked in blue.}
    \label{fig31}
\end{figure}

\begin{theorem}{}{6.11}
    If $f \in \R_\alpha[a, b]$, $m \leq f(x) \leq M$ for all $x \in [a, b]$, and $\phi$ is continuous on $[m, M]$, then $\phi \circ f \in \R_\alpha[a, b]$.
\end{theorem}
\begin{nproof}
    Not covered in lecture, see Rudin. \qed
\end{nproof}

\noindent As an example of the above theorem, suppose we have that $f \in \R(\alpha)$. Then, we have that $f^2 \in \R(\alpha)$ and $\abs{f} \in \R(\alpha)$ as $\phi(x) = x^2$ and $\phi(x) = \abs{x}$ are both continuous functions.

\subsection{Properties of the Integral}
\begin{theorem}{Linearity and Related Properties}{6.12}
\begin{enumerate}
    \item Suppose $f_1 \in \R_\alpha[a, b]$ and $f_2 \in \R_\alpha[a, b]$. Then, $f_1 + f_2 \in \R_\alpha[a, b]$ and $cf_1 \in \R_{[a,b]}(\alpha)$ for all $c \in \RR$. Furthermore:
    \begin{align*}
        \int_a^b(f_1 + f_2) d\alpha = \int_a^b f_1 d\alpha + \int_a^b f_2d\alpha \text{ and } \int_a^b cf_1 d\alpha = c\int_a^b f_1d\alpha
    \end{align*}
    \item Suppose $f_1, f_2 \in \R_\alpha[a, b]$ and $f_1(x) \leq f_2(x)$ for all $x \in [a, b]$. Then,
    \begin{align*}
        \int_a^b f_1 d\alpha \leq \int_a^b f_2d\alpha
    \end{align*}
    \item If $f \in \R_\alpha[a, b]$ and $c \in (a, b)$, then $f \in \R_\alpha[a, c] \cap \R_\alpha[c, b]$ and furthermore:
    \begin{align*}
        \int_a^b f d\alpha = \int_a^c f d\alpha + \int_c^b f d\alpha
    \end{align*}
    \item If $f \in \R_\alpha[a, b]$ and $\abs{f(x)} \leq M$ for all $x \in [a, b]$, then:
    \begin{align*}
        \abs{\int_a^b f d\alpha} \leq M(\alpha(b) - \alpha(a))
    \end{align*}
    \item If $f \in \R_{\alpha_1}[a, b]$ and $f \in \R_{\alpha_2}[a, b]$, then $f \in \R_{\alpha_1 + \alpha_2}[a, b]$ and $f \in \R_{c\alpha_1}[a, b]$ for all $c \geq 0$. Furthermore:
    \begin{align*}
        \int_a^b f d\alpha_1 + \int_a^b f d\alpha_2 = \int_a^b f d(\alpha_1 + \alpha_2) \text{ and } \int_a^b f d(c\alpha_1) = c\int_a^b f d\alpha_1
    \end{align*}

\end{enumerate}
\end{theorem}
\begin{nproof}
    \begin{enumerate}
        \item See Rudin and HW2Q1.
        \item We have that $L(P, f_1) \leq L(P, f_2)$ for all partitions $P$ as $\inf\set{f_1(x): x \in [x_{i-1}, x_i]} \leq \inf\set{f_2(x): x \in [x_{i-1}, x_i]}$ for all $i$ (as $f_1(x) \leq f_2(x)$). Therefore, we have that:
        \begin{align*}
            \int_a^b f_1 d\alpha = \sup_P L(P, f_1) \leq \sup_P L(P, f_2) = \int_a^b f d\alpha.
        \end{align*}
        \item Exercise (see HW2Q1 for the $\uint{a}{b}$ case)
        \item Consider any partition $P$ of $[a, b]$. Let $M = \sup\set{\abs{f(x): x \in [a, b]}}$. We then have that (using the ML bound from Theorem \ref{thm:6.1}):
        \begin{align*}
            -M[\alpha(b) - \alpha(a)] = \sum_{i=1}^n(-M)\Delta \alpha_i \leq \sum_{i=1}^n M_i \Delta \alpha_i = L(P, f, \alpha) \leq \int_a^b fd\alpha
            \\ \leq U(P, f, \alpha) = \sum_{i=1}^n M_i \Delta \alpha_i \leq \sum_{i=1}^n M\Delta \alpha_i = M[\alpha(b) - \alpha(a)].
        \end{align*}
        \item We prove the first (additivity) statement. Let $\e > 0$. Choose $P_i, i \in \set{1, 2}$ (By Theorem \ref{thm:6.6}) such that \begin{align*}
            U(P_i, f, \alpha_i) - L(P_i, f, \alpha_i) < \frac{\e}{2}
        \end{align*}
        Let $P = P_1 \cup P_2$. Then, by Theorem \ref{thm:6.4}, we have that:
        \begin{align*}
            U(P^*, f, \alpha_i) - L(P^*, f, \alpha_i) < \frac{\e}{2}
        \end{align*}
        Adding the two expressions (for $i = 1, 2$) together, we have:
        \begin{align*}
            U(P^*, f, \alpha_1 + \alpha_2) - L(P^*, f, \alpha_1 + \alpha_2) < \e
        \end{align*}
        So $f \in \R_{\alpha_1 + \alpha_2}[a, b]$ by Theorem \ref{thm:6.6}. Furthermore, We have that:
        \begin{align*}
            \int_a^b f d(\alpha_1 + \alpha_2) \leq U(P^*, f, \alpha_1 + \alpha_2) = U(P^*, f, \alpha_1) + U(P^*, f, \alpha_2) < \int_a^b f d\alpha_1 + \int_a^b fd\alpha_2 + \e
        \end{align*}
        where we apply the inequality of $U(P^*, f, \alpha_i) < \int_a^b f d\alpha_i + \frac{\e}{2}$ in the last line. Similarly, we have that:
        \begin{align*}
            \int_a^b fd(\alpha_1 + \alpha_2) \geq L(P^*, f, \alpha_1 + \alpha_2) = L(P^*, f, \alpha_1) + L(P, f, \alpha_2) > \int_a^b fd\alpha_1 + \int_a^b fd\alpha_2 - \e
        \end{align*}
        Since $\e$ is arbitrary, we therefore conclude that:
        \begin{align*}
            \int_a^b f d(\alpha_1 + \alpha_2) = \int_a^b fd\alpha_1 + \int_a^b fd\alpha_2
        \end{align*} \qed
    \end{enumerate}
\end{nproof}

\begin{theorem}{}{6.13}
    \begin{enumerate}
        \item Suppose $f, g \in \R(\alpha)$. Then, $fg \in \R(\alpha)$.
        \item Suppose $f \in \R(\alpha)$. Then, $\abs{f} \in \R(\alpha)$ and $\abs{\int_a^b fd\alpha} \leq \int_a^b \abs{f}d\alpha$.
    \end{enumerate}
\end{theorem}
\begin{nproof}
    \begin{enumerate}
        \item By Theorem \ref{thm:6.12}(a), $f \pm g \in \R(\alpha)$, so by Theorem \ref{thm:6.11}, $(f \pm g)^2 \in \R(\alpha)$. Since $(f+g)^2 - (f-g)^2 = 4fg$, we then have that:
        \begin{align*}
            \frac{(f+g)^2 + (f-g)^2}{4} = fg \in \R(\alpha)
        \end{align*}
        \item By Theorem \ref{thm:6.11}, $\abs{f} \in \R(\alpha)$ letting $\phi(t) = \abs{t}$. Let $c = \sgn\left(\int_a^b f d\alpha\right) \in \set{-1, 0, 1}$. Then, we have that:
        \begin{align*}
            \abs{\int_a^b fd\alpha} = c\int_a^b d\alpha = \int_a^b cfd\alpha \leq \int_a^b \abs{f}d\alpha
        \end{align*}
        Where we use Theorem \ref{thm:6.12}(a) in the second equality, and Theorem \ref{thm:6.12}(b) in the last inequality (as $cf \leq \abs{f}$). \qed
    \end{enumerate}
\end{nproof}

\setcounter{rudin}{14}

\begin{theorem}{}{6.15}
    Suppose $f$ is bounded on $[a, b]$, $s \in (a, b)$, and $f$ is continuous at $s$. Let:
    \begin{align*}
        \alpha(x) = \begin{cases}
            0 & x \leq s
            \\ 1 & x > s
        \end{cases}
    \end{align*}
    Then, $\int_a^b fd\alpha = f(s)$ (and in particular, $f \in \R(\alpha)$).
\end{theorem}

\noindent This result is interesting, and some remarks on the nature of this theorem are in order. Firstly, by Theorem 4.29 (not covered in Lecture, see Rudin), since $\alpha$ is monotonically increasing, we have that $\lim_{t \rightarrow x^+}\alpha(t) = \alpha(x^+)$ and $\lim_{t \rightarrow x^-}\alpha(t) = \alpha(x^-)$ exist at every $x \in (a, b)$ and $\alpha(x^-) \leq \alpha(x) \leq \alpha(x^+)$. Secondly, note that Rudin defines $\alpha$ in the above Theorem to be left contiuous, but in probability theory, it is conventional to use a right continuous (i.e. $\alpha(x) = \alpha(x^+)$ for all $x \in (a, b)$). We leave it as an exercise to prove the theorem for the case of a right continuous $\alpha$ (the strategy and result are identitcal).

\begin{figure}[htbp]
    \centering
    \begin{tikzpicture}[scale=2]
        \draw[-latex, very thick] (0, 0) -- (2, 0);
        \draw[-latex, very thick] (0, 0) -- (0, 2);
        \draw [] (0.25 , 0) to [ curve through ={(0.5, 0.3)}] (1, 0.5);
        \draw[] (1, 1.5) to [curve through = {(1.4, 1.7)}] (1.9, 1.8);
        \draw[fill = white] (1, 0.5) circle (1pt);
        \filldraw[] (1, 1) circle (1pt);
        \draw[fill = white] (1, 1.5) circle (1pt);
        \draw[] (1, 0) -- (1, -0.15);
        \node[below] at (1, -0.15) {$x$};
        \draw[] (0, 0.5) -- (-0.15, 0.5);
        \draw[] (0, 1) -- (-0.15, 1);
        \draw[] (0, 1.5) -- (-0.15, 1.5);
        \node[left] at (-0.15, 0.5) {$\alpha(x^-)$};
        \node[left] at (-0.15, 1) {$\alpha(x)$};
        \node[left] at (-0.15, 1.5) {$\alpha(x^+)$};
        %\draw[] (0.5, 0) -- (0.5, -0.15);
        %\draw[] (1.7, 0) -- (1.7, -0.15);
        %\node[below] at (0.5, -0.16) {$a$};
        %\node[below] at (1.7, -0.13) {$b$};
    \end{tikzpicture}

    \caption{Visualization of Theorem 4.29. For any strictly increasing function, we have that $\alpha(x^-) \leq \alpha(x) \leq \alpha(x^+)$ at a discontinuity $x$.}
    \label{fig32}
\end{figure}
Any physicists looking at the result of the Theorem will note that it looks very much like the ``Dirac Delta'' function; indeed, this choice of $\alpha$ is making rigorous the notion of a $\delta$ function where:
\begin{align*}
    \int_a^b f(x)\delta(x - s) = f(s)
\end{align*}
We leave it as a homework exercise (HW2Q2) to prove that no such function can actually exist. This step-function method is one way to make the $\delta$ function well-defined; other methods include taking the limit of a bell curve, or bringing in the theory of distributions (the latter which is most definitely outside the scope of this course).

Finally, we note that this example really shows off something that the Riemann integral cannot reproduce; the incorporation of ``discrete'' points.

\begin{nproof}
    Choose $P = \set{x_0, x_1 = s, x_2, x_3 = b}$. We then have that:
    \begin{align*}
        U(P) &= \sum_{i=1}^3M_i\Delta \alpha_i = M_2\Delta \alpha_2 = M_2 = \sup\set{f(x): x \in [x_1, x_2]}
        \\ L(P) &= \sum_{i=1}^3 m_i \Delta \alpha_i = m_2\Delta \alpha_2 = m_2
    \end{align*}
    So, we have that $m_2 \leq \int_a^b f d\alpha \leq M_2$ (assuming the integral exists). Taking the limit as $x_2 \rightarrow s^+$, by the continuity of $f$ at $s$ we have that $M_2 \rightarrow f(s)$ and $m_2 \rightarrow f(s)$. Therefore, $\int_a^b f d\alpha$ exists and equals $f(s)$. \qed
\end{nproof}
\noindent Note that the above proof works exactly the same way if $s = a$. If $s = b$, then because we take $\alpha$ to be left continuous, $\alpha = 0$ everywhere on $[a, b]$ and the integral just equals zero.

\setcounter{rudin}{13}
\begin{definition}{Unit Step Function}{6.14}
    We define the (left continuous) \textbf{unit step function} as:
    \begin{align*}
        I(x) = \begin{cases}
            0 & x \leq 0
            \\ 1 & x > 0
        \end{cases}
    \end{align*}
    In Theorem \ref{thm:6.15}, we have that $\alpha(x) = I(x - s)$.
\end{definition}
\begin{figure}[htbp]
    \centering
    \begin{tikzpicture}[scale=2]
        \draw[-latex, very thick] (0, 0) -- (0, 2);
        \draw[latex-latex, very thick] (-2, 0) -- (2, 0);
        \draw[<-, thick, blue] (-1.5, 0) -- (0, 0);
        \draw[->, thick, blue] (0, 1) -- (1.5, 1);
        \filldraw[fill = blue, draw = blue] (0, 0) circle (1pt);
        \filldraw[fill = white, draw = blue] (0, 1) circle (1pt);
    \end{tikzpicture}

    \caption{Plot of the (left continuous) unit step function. To obtain the $\alpha$ used in Theorem \ref{thm:6.15}, move the step to $x = s$ instead of $x = 0$.}
    \label{fig33}
\end{figure}

\stepcounter{rudin}
\begin{theorem}{}{3.16}
    Let $c_n \geq 0$ be such that $\sum_{n=1}^\infty c_n < \infty$. Take $s_n \in (a, b)$ distinct (that is, $s_n \neq s_m$ if $n \neq m$). Define $\alpha(x) = \sum_{n=1}^\infty c_n I(x - s_n)$ (which converges/exists by comparison, as $\sum c_n$ converges). Let $f$ be continuous on $[a, b]$. Then,
    \begin{align*}
        \int_a^b f d\alpha = \sum_{n=1}^\infty c_nf(s_n)
    \end{align*}
\end{theorem}
\noindent Note that the resultant series in the theorem above is convergent/well defined as $f$ is bouded on $[a, b]$ and hence $\sum_{n=1}^\infty c_nf(s_n) \leq M\sum_{n=1}^\infty c_n$ where $M = \max\set{f(x)}$. Hence the series converges by comparison. If all the $c_n$s are zero except for one, this is just Theorem \ref{thm:6.15}. Note that the above result holds for a finite sum as well (we would use induction to prove it in this case).

\begin{nproof}
    Let $R_N = \sum_a^b fd\alpha - \sum_{n=1}^N c_nf(s_n)$. We show that $R_n \rightarrow 0$ as $N \rightarrow \infty$ (i.e. given $\e > 0$, we show there exists $N_0$ such that $\abs{R_N} < \e$ if $N \geq N_0$). We define:
    \begin{align*}
        \alpha_1(x) = \sum_{n=1}^N c_nI(x - s_n) \quad \alpha_2(x) = \sum_{n=N+1}^\infty c_n I(x - s_n)
    \end{align*}
    Then, by Theorem \ref{thm:6.12}(e) we have that:
    \begin{align*}
        \int_a^b fd\alpha = \int_a^b f d\alpha_1 + \int_a^b f d\alpha_2
    \end{align*}
    Then, we have that:
    \begin{align*}
        \int_a^b d\alpha_1 &= \sum_{i=1}^N \int_a^b f(x)d(c_nI(x - s_n))
        \\ &= \sum_{n=1}^N c_n\int_a^b f(x)d(I(x - s_n))
        \\ &= \sum_{n=1}^Nc_nf(s_n)
    \end{align*}
    Wherein the first two equalities we again apply Theorem \ref{thm:6.12}(e) and in the last equality we use Theorem \ref{thm:6.15}. We therefore obtain that $R_N = \int_a^b d\alpha_2$. We then have that:
    \begin{align*}
        \abs{R_n} \leq M\left[\alpha_2(b) - \alpha_2(a)\right] = M\sum_{n=N+1}^{\infty} c_n
    \end{align*}
    Then by the convergence of $\sum c_n$, we can choose $N_0$ such that $\sum_{n=N_0+1}^\infty c_n < \frac{\e}{M}$. We therefore have that $R_N < \e$ if $N \geq N_0$, proving the claim. \qed
\end{nproof}

\begin{theorem}{}{6.17}
    Suppose:
    \begin{enumerate}[(i)]
        \item $\abs{f(x)} \leq M$ for all $x \in [a, b]$.
        \item $\alpha$ is continuous and increasing on $[a, b]$ and differentable on $(a, b)$.
        \item $\alpha' \in \R[a, b]$.
    \end{enumerate}
    Then, $f \in \R_\alpha[a, b]$ if and only if $f\alpha' \in \R[a, b]$ and in this case:
    \begin{align*}
        \int_a^b fd\alpha = \int_a^b f(x)\alpha'(x)dx
    \end{align*}
\end{theorem}

\begin{nproof}
    It suffices to show that:
    \begin{align*}
        \uint{a}{b} f d\alpha = \uint{a}{b} f\alpha'dx \text{ and } \lint{a}{b}fd\alpha = \lint{a}{b}f\alpha' dx
    \end{align*}
    We prove the first equality and the second equality follows analogously. Let $\e > 0$. Since $\alpha' \in \R$, there exists a partition $P$ such that $U(P, \alpha') - L(P, \alpha') < \e$ (this also holds for any refinement of $P$). Letting $A_i = \sup \alpha', a_i = \inf \alpha' on [x_{i-1}, x_i]$ we have that:
    \begin{align*}
        \sum_{i=1}^n(A_i - a_i) \Delta x_i < \e
    \end{align*}
    By the Mean Value Theorem (Theorem \ref{thm:5.10}) we have that there exists $t_i \in [x_{i-1}, x_i]$ such that $\Delta \alpha_i = \alpha'(t_i)\Delta x_i$. We also have that for all $s_i \in [x_{i-1}, x_i]$, $\abs{\alpha'(s_i) - \alpha'(t_i)} \leq A_i - a_i$ ($\alpha'$ can only vary as much as the maximum minus the minimum on the interval). We therefore have that:
    \begin{align*}
        \sum_{i=1}^n \abs{\alpha'(s_i) - \alpha'(t_i)}\Delta x_i \leq \sum_{i=1}^n A_i - a_i\Delta x_i < \e
    \end{align*}
    Recall that we skipped Theorem 6.7, but it might be worth comparing this result to Theorem 6.7(c). Let $M = \sup{f(x): x \in [a, b]}$. Now, for any choice of $s_i \in [x_{i-1}, x_i]$, we have that:
    \begin{align*}
        \abs{\sum_{i=1}^n f(s_i)\Delta\alpha_i - \sum_{i=1}^n \alpha'(s_i)\Delta x_i} \leq \sum_{i=1}^m M\abs{\alpha'(t_i) - \alpha'(s_i)}\Delta x_i < M\e \quad (*)
    \end{align*}
    Therefore:
    \begin{align*}
        \sum_{i=1}^n f(s_i) \Delta \alpha_i \leq \sum_{i=1}^n f(s_i)\alpha'(s_i)\Delta x_i + M\e \leq U(P, f\alpha') + M\e
    \end{align*}
    We now take the supremum over each $[s_{i-1}, s_i]$. Since $s_i$ is arbitrary, we have that $\uint{a}{b}fd\alpha \leq U(P, f, \alpha) \leq U(P, f\alpha') + M\e$, and hence $\uint{a}{b}f d\alpha \leq \uint{a}{b}f\alpha' dx + M\e$. $(*)$ gives $\uint{a}{b}f\alpha' dx \leq \uint{a}{b}fd\alpha + M\e$. Since $\e$ is arbitrary, we have that $\uint{a}{b}fd\alpha \leq \uint{a}{b}f\alpha'dx$ and $\uint{a}{b}f\alpha'dx \leq \uint{a}{b}fd\alpha$, so $\uint{a}{b}f\alpha'dx = \uint{a}{b}fd\alpha$. \qed
\end{nproof}
\noindent This theorem tells us something that looks very familiar from Calculus... namely, a change of variables! Recall the COV formula from first year:
\begin{align*}
    \int_a^b f(x)dx = \int_{\phi^{-1}(a)}^{\phi^{-1}(b)} f(\phi(y))\phi'(y)dy
\end{align*}
Where $x = \phi(y), y = \phi^{-1}(y)$. We now prove this notion rigorously.

\setcounter{rudin}{18}
\begin{theorem}{Change of Variable}{6.19}
    Let $f: [a, b] \mapsto \RR$. Suppose $\phi: [A, B] \mapsto [a, b]$ is continuous and strictly increasing. Suppose $\alpha$ is increasing on $[a, b]$ and $f \in \R_{\alpha}[a, b]$. Let $g = f \circ \phi: [A, B] \mapsto \RR$ and $\beta = \alpha \circ \phi: [A, B] \mapsto \RR$. Then, we have that $g \in \R_\beta[A, B]$, and:
    \begin{align*}
        \int_A^B g d\beta = \int_a^b fd\alpha
    \end{align*}
\end{theorem}
\noindent As an example, consider the integral $\int_a^b \sin x^2dx$ for $0 \leq a < b$. We then have that $f(x) = \sin x^2$ and $\alpha(x) = x$. We make the substitution $x^2 = y$, so $\phi(x) = \sqrt{x}$ and $\phi^{-1}(y) = y^2$. Then, A = $\phi^{-1}(a) = a^2$, $B = \phi^{-1}(b) = b^2$. We then have that $g(y) = f \circ \phi(y) = f(\sqrt{y}) = \sin(y)$, and $\beta(y) = \alpha \circ \phi(y) = \alpha(\sqrt{y}) = \sqrt{y}$. We then have that:
\begin{align*}
    \int_a^b \sin x^2 dx = \int_{a^2}^{b^2} \sin y d\beta = \int_{a^2}^{b^2} \frac{\sin y}{2\sqrt{y}}dy
\end{align*}
Where the first equality follows by Theorem \ref{thm:6.19} and the second equality follows by Theorem \ref{thm:6.17}.

\begin{figure}[htbp]
    \centering
    \begin{tikzpicture}[scale=2]
        \draw[-latex, very thick] (0, 0) -- (2, 0);
        \draw[-latex, very thick] (0, 0) -- (0, 2);
        \draw[] (0, 0.5) -- (-0.15, 0.5);
        \draw[] (0, 0.7) -- (-0.15, 0.7);
        \draw[] (0, 1.5) -- (-0.15, 1.5);
        \draw[] (0, 1.7) -- (-0.15, 1.7);
        \node[] at (0.7, 1.2) {$x = \phi(y)$};
        \filldraw[] (0.5, 0.5) circle (1pt);
        \filldraw[] (1.7, 1.7) circle (1pt);
        \draw[] (0.5, 0) -- (0.5, -0.15);
        \draw[] (1.7, 0) -- (1.7, -0.15);
        \node[below] at (0.5, -0.16) {$a$};
        \node[below] at (1.7, -0.13) {$b$};
        \node[below] at (0.86, -0.15) {$x_1$};
        \node[below] at (1.36, -0.15) {$x_2$};
        \draw[] (0.86, 0) -- (0.86, -0.15);
        \draw[] (1.36, 0) -- (1.36, -0.15);
        \node[left] at (-0.15, 0.5) {$A$};
        \node[left] at (-0.15, 1.7) {$B$};
        \node[left] at (-0.15, 0.7) {$y_1$};
        \node[left] at (-0.15, 1.5) {$y_2$};
        \draw[dashed] (0, 0.5) -- (0.5, 0.5);
        \draw[dashed] (0.5, 0) -- (0.5, 0.5);
        \draw[dashed] (0, 0.7) -- (0.86, 0.7);
        \draw[dashed] (0.86, 0) -- (0.86, 0.7);
        \draw[dashed] (0, 1.5) -- (1.36, 1.5);
        \draw[dashed] (1.36, 0) -- (1.36, 1.5);
        \draw[dashed] (0, 1.7) -- (1.7, 1.7);
        \draw[dashed] (1.7, 0) -- (1.7, 1.7);
        \draw [] (0.5 , 0.5) to [ curve through ={(0.8, 0.6)..(1, 1)..(1.25, 1.2)..(1.4, 1.6)}] (1.7, 1.7);
    \end{tikzpicture}
    \caption{Plot of a (monotonically increasing and continuous) function $\phi$ and a demonstration of how it puts partitions of $[a, b]$ and $[A, B]$ in one-to-one correspondence. This gives the intuition for the proof of Theorem \ref{thm:6.19}.}
    \label{fig34}
\end{figure}

\begin{nproof}
    We make the observation that partitions $P$ of $[a, b]$ and parititons $Q$ of $[A, B]$ can be put in 1-1 correspondence via $x_i = \phi(y_i)$. Additionally, we make the observation that the set of $g$ values on $[y_{i-1}, y_i]$ is equal to the set of $f$ values on $[x{i-1}, x_i]$. Finally, we observe that $\alpha(x_i) = (\alpha\circ\phi)(y_i) = \beta(y_i)$. With these three observations, we have that:
    \begin{align*}
        U(P, f, \alpha) = U(Q, g, \beta) \text{ and } L(P, f, \alpha) = U(Q, g, \beta)
    \end{align*}
    Let $\e > 0$. Since $f \in \R_{\alpha}[a, b]$, there exists a $P$ such that $U(P, f, \alpha) - L(P, f, \alpha) < \e$ by Theorem \ref{thm:6.6}. For this partition, we have that $U(Q, g, \beta) - L(Q, g, \beta) < \e$ for the corresponding partition $Q$. Hence, $g \in \R_\beta[A, B]$. Finally, we have that:
    \begin{align*}
        \int_A^B gd\beta = \inf_Q U(Q, g, \beta) = \inf_P U(P, f, \alpha) = \int_a^b fd\alpha
    \end{align*}
    \qed
\end{nproof}

\subsection{The Fundamental Theorem of Calculus}

\begin{theorem}{Fundamental Theorem of Calculus I}{6.20}
    Let $f \in \R[a, b]$ and for $x \in [a, b]$, define $F(x) = \int_a^x f(t)dt$. Then, $F$ is continuous on $[a, b]$. If $f$ is continuous at $x_0 \in [a, b]$ then $F'(x_0)$ exists and $F'(x_0) = f(x_0)$.
\end{theorem}
\begin{figure}[htbp]
    \centering
    \begin{tikzpicture}[scale=2]
        \draw[-latex, very thick] (0, 0) -- (2, 0);
        \draw[-latex, very thick] (0, 0) -- (0, 2);
        \filldraw[] (0.5, 0.5) circle (1pt);
        \filldraw[] (1.7, 1.7) circle (1pt);
        \draw[] (0.5, 0) -- (0.5, -0.15);
        \draw[] (1.7, 0) -- (1.7, -0.15);
        \draw[] (1, 0) -- (1, -0.15);
        \node[below] at (0.5, -0.16) {$a$};
        \node[below] at (1.7, -0.13) {$b$};
        \node[below] at (1, -0.15) {$x$};
        \draw [] (0.5 , 0.5) -- (1.7, 1.7);
        \filldraw[opacity = 0.5, blue] (0.5, 0.5) -- (1, 1) -- (1, 0) -- (0.5, 0) -- (0.5, 0.5);
        \node[text = blue] at (1.2, 0.5) {$F(x)$};
        \node[above] at (1.2, 1.3) {$f(x)$};
    \end{tikzpicture}

    \caption{The Fundamental Theorem of Calculus/Theorem \ref{thm:6.20} gives us a way to relate a curve ($f$) with the culmulative area beneath it ($F$) by means of the derivative.}
    \label{fig35}
\end{figure}

To get intuition for Theorem \ref{thm:6.20}, we think about what happens when we ``zoom in'' to a function $f$. Over an interval $(x_0, x_0 + h)$, if $x_0$ is continuous at $x_0$ and $h$ is small, then $f$ will be roughly constant over the interval. Hence, $F(x_0 + h) - F(x_0) \approx f(x_0)h$. In the limit of $h \rightarrow 0$, this approximation becomes exact.

\begin{figure}[htbp]
    \centering
    \begin{tikzpicture}[scale=2]
        \draw[latex-latex, very thick] (0, 0) -- (2, 0);
        \draw [<->, thick] (0.3, 1) -- (1.7, 1.05);
        \draw[] (0.5, 0) -- (0.5, -0.15);
        \draw[] (1.5, 0) -- (1.5, -0.15);
        \node[below] at (0.5, -0.18) {$x_0$};
        \node[below] at (1.46, -0.15) {$x_0 + h$};
        \filldraw[opacity = 0.5, blue] (0.5, 0) -- (1.5, 0) -- (1.5, 1.04) -- (0.5, 1.004) -- (0.5, 0);

    \end{tikzpicture}

    \caption{Visual Intuition for Theorem \ref{thm:6.20} and its proof. As $f$ is continuous at $x_0$, for small $h$, $f$ is roughly constant on $(x_0, x_0 + h)$. Hence, the increase in culmulative area below the curve is roughly $f(x_0)\cdot h$ (height of $f(x_0)$ times width $h$ of the approximate rectangle).}
    \label{fig36}
\end{figure}

\begin{nproof}
    We first show the continuity of $F$. Choose $M$ such that $\abs{f(t)} \leq M$ for all $t \in [a, b]$. For $a \leq x < y \leq b$, we then have that:
    \begin{align*}
        \abs{F(x) - F(y)} = \abs{\int_a^xf(t)dt - \int_a^yf(t)dt} = \abs{\int_x^y f(t)dt} \leq M(y - x)
    \end{align*}
    Where we use Theorem \ref{thm:6.12}(c) for the second equality and Theorem \ref{thm:6.12}(d) for the inequality at the end. Let $\e > 0$. If $\abs{x - y} < \delta = \frac{\e}{M}$, then $\abs{F(x) - F(y)} < \e$ so $F$ is continuous.

    We next show the differentiability of $F$.We wish to show that:
    \begin{align*}
        \lim_{h \rightarrow 0} \frac{F(x_0 + h) - F(x_0)}{h} = f(x_0)
    \end{align*}
    We show the case for $h > 0$, that is, taking the limit of $h \rightarrow 0^+$. We have that:
    \begin{align*}
        \frac{1}{h}\left[F(x_0 + h) - F(x_0)\right] - f(x_0) = \frac{1}{h}\int_{x_0}^{x_0 + h}f(t)dt - f(x_0) = \frac{1}{h}\int_{x_0}^{x_0 + h}\left[f(t) - f(x_0)\right]dt
    \end{align*}
    Where in the last line we use the observation that $\frac{1}{h}\int_{x_0}^{x_0 + h}f(x_0)dt = f(x_0)$. Since $f$ is continuous at $x_0$, for any $\e > 0$, there exists $\delta > 0$ such that:
    \begin{align*}
        \abs{t - x_0} < \delta \implies \abs{f(t) - f(x_0)}  <\e
    \end{align*}
    Thus, if $h < \delta$, then:
    \begin{align*}
        \abs{ \frac{1}{h}\left[F(x_0 + h) - F(x_0)\right] - f(x_0)} \leq \frac{1}{h}\int_{x_0}^{x_0 + h}\abs{f(t) - f(x_0)}dt < \frac{1}{h}\e h = \e
    \end{align*}
    So we conclude that:
    \begin{align*}
        \lim_{h \rightarrow 0^+} \frac{F(x_0 + h) - F(x_0)}{h} = F'(x_0) = f(x_0)
    \end{align*}
    \qed
\end{nproof}

\begin{theorem}{Fundamental Theorem of Calculus II}{6.21}
    If $f \in \R[a, b]$ and there exists $F$ on $[a, b]$ such that $F' = f$, then:
    \begin{align*}
        \int_a^b f(x)dx = F(b) - F(a)
    \end{align*}
\end{theorem}

\begin{nproof}
    For any partition $P$, we have that
    \begin{align*}
        F(b) - F(a) &= \sum_{i=1}^n \left[F(x_i) - F(x_{i-1})\right]
        \\ &= \sum_{i=1}^n F'(t_i) \Delta x_i
        \\ &= \sum_{i=1}^n f(t_i) \Delta x_i
    \end{align*}
    Where the first equality follows by the fact that the sum telescopes, the second equality follows from the Mean Value Theorem/Theorem \ref{thm:5.10} (By the continuity of $F$, there exists $t_i \in [x_{i-1}, x_i]$ such that $F(x_i) - F(x_{i-1}) = F'(t_i)(x_i - x_{i-1})$) and the third equality follows by Theorem \ref{thm:6.20}. Now, we ahve that $f(t_i) \in [m_i, M_i]$ for each $i$, so we therefore have that:
    \begin{align*}
        \sum_{i=1}^n m_i \Delta x_i \leq \sum_{i=1}^n f(t_i)\Delta x_i \leq \sum_{i=1}^n M_i \Delta x_i
    \end{align*}
    Hence:
    \begin{align*}
        F(b) - F(a) \in [L(P, f), U(P, f)]
    \end{align*}
    Additionally, by definition of the integral we have that:
    \begin{align*}
        \int_a^b f(x)dx \in [L(P, f), U(P, f)]
    \end{align*}
    Let $\e > 0$. Choose $P$ such that $U(P, f) - L(P, f) < \e$. Then, we have that:
    \begin{align*}
        \abs{F(b) - F(a) - \int_a^b f(x)dx} < \e
    \end{align*}
    Since $\e$ is arbitrary, we have that:
    \begin{align*}
        F(b) - F(a) = \int_a^b f(x)dx
    \end{align*} \qed
\end{nproof}

\begin{theorem}{Integration By Parts}{6.22}
    If $F, G$ are differentiable on $[a, b]$ with $F' = f \in \R[a, b]$ and $G' = g \in \R[a, b]$, then:
    \begin{align*}
        \int_a^b F(x)g(x)dx = F(b)G(b) - F(a)G(a) - \int_a^b f(x)G(x)dx
    \end{align*}
\end{theorem}
\noindent Note that the above integration by parts formula generalizes to different $\alpha$ (not just $\alpha(x) = x$. The proof of this is left as an exercise (Rudin Chapter 6 Problem 17).
\begin{nproof}
    Let $H(x) = F(x)G(x)$. Then, $H'(x) = F'(x)G(x) + F(x)G'(x) = f(x)G(x) + F(x)g(x) \in \R[a, b]$ by the product rule (Theorem \ref{thm:5.3}) and the FTC I (Theorem \ref{thm:6.20}). Therefore, we have that:
    \begin{align*}
        H(b) - H(a) = \int_a^b H'(x)dx = \int_a^b f(x)G(x)dx + \int_a^b F(x)g(x)dx
    \end{align*}
    Where the first equality follows by FTC II (Theorem \ref{thm:6.21}). We have that $H(b) - H(a) = F(b)G(b) - F(a)G(a)$, so rearranging the above expression, we find that:
    \begin{align*}
        \int_a^b F(x)g(x)dx = F(b)G(b) - F(a)G(a) - \int_a^b f(x)G(x)dx
    \end{align*}
    Which is the desired expression. \qed
\end{nproof}

\begin{ndef}{: Improper Integrals}{}
    If $f \in \R[a, b]$ for all $b > a$, then we define:
    \begin{align*}
        \int_a^\infty f(x)dx = \lim_{b \rightarrow \infty}\int_a^b f(x)dx
    \end{align*}
    if the integral exists in $\RR$. Then, we say that the \textbf{improper integral} $\int_a^\infty f(x)dx$ converges.
\end{ndef}
\begin{ndef}{: Absolute Convergence of Integrals}{}
    If $\int_a^\infty \abs{f(x)}dx$ converges, then we say that $\int_a^\infty f(x)dx$ \textbf{converges absolutely}.
\end{ndef}

\noindent To finish off this chapter, we will work through a comprehensive problem together; we prove that $\int_0^\infty \sin t^2 dt$ converges but not absolutely.

\begin{proof}
    The proof that the integral does not converge absolutely is left as a homework exercise (HW3Q1). We will prove that the integral converges here. Let $p_n = \int_0^n \sin t^2dt$ where $n \in \NN$. We will show that:
    \begin{enumerate}[(i)]
        \item $\set{p_n}$ is a Cauchy sequence and hence has a limit in $\RR$.
        \item This is enough to show that the improper integral converges.
    \end{enumerate}
    We start by showing (i), namely that $\set{p_n}$ is Cauchy. For $x < y$, we have that:
    \begin{align*}
        \int_x^y \sin t^2 dt = \int_{x^2}^{y^2} \sin u \frac{1}{2\sqrt{u}}du
    \end{align*}
    by a change of variable (Theorem \ref{thm:6.19}). Now performing an integration by parts (Theorem \ref{thm:6.22}) with $F(u) = \frac{1}{2\sqrt{u}}$ and $G'(u)du = \sin u du$, we have that $F'(u) = -\frac{1}{4u^{3/2}}$ and $G(u) = -\cos u$, so:
    \begin{align*}
        \int_x^y \sin t^2 dt = \int_{x^2}^{y^2} \sin u \frac{1}{2\sqrt{u}}du = \left.-\frac{\cos u}{2\sqrt{u}}\right|_{x^2}^{y^2} - \int_{x^2}^{y^2} \frac{\cos u}{4u^{3/2}} du
    \end{align*}
    Hence:
    \begin{align*}
        \int_x^y \sin t^2 dt = \frac{\cos x^2}{2x} - \frac{\cos y^2}{2y} - \int_{x^2}^{y^2} \frac{\cos u}{4u^{3/2}}du
    \end{align*}
    Suppose $n \geq m$. Then, we have that:
    \begin{align*}
        \abs{p_n - p_m} &= \abs{\int_m^n \sin t^2 dt}
        \\ &= \abs{\frac{\cos m^2}{2m} - \frac{\cos n^2}{2n} - \int_{m^2}^{n^2} \frac{\cos u}{4 u^{3/2}}du }
    \end{align*}
    Using the fact that $\abs{\cos(x)} \leq 1$ and applying the Triangle inequality, we have that:
    \begin{align*}
        \abs{p_n - p_m} \leq \frac{1}{2m} + \frac{1}{2n} - \int_{m^2}^{n^2} \frac{1}{4u^{3/2}}du = \frac{1}{2m} + \frac{1}{2n} + \left. \frac{-1}{2u^{1/2}}\right|_{m^2}^{n^2} = \frac{1}{2m} + \frac{1}{2n} - \frac{1}{2n} + \frac{1}{2m} = \frac{1}{m} \quad (*)
    \end{align*}
    Let $\e > 0$. Choose $N_0$ such that $\frac{1}{N_0} < \e$. Then, for $m \geq n \geq N_0$ we have that:
    \begin{align*}
        \abs{p_n - p_m} \leq \frac{1}{m} \leq \frac{1}{N_0} < \e
    \end{align*}
    and we conclude that $\set{p_n}$ is Cauchy. Since $\RR$ is complete, $p_n \rightarrow p$ for some $p \in \RR$. To finish the proof, we show (ii); that this is sufficient. For $b \geq N_0$, choose $N \geq N_0$ such that $b \in [N, N+1)$. Then, we have that:
    \begin{align*}
        \int_{0}^b \sin t^2 dt = p_N + \int_{N}^b \sin t^2 dt \leq p_N + \frac{1}{N}
    \end{align*}
    Where the inequality follows by $(*)$. We therefore have that:
    \begin{align*}
        \abs{p - \int_0^b \sin t^2 dt} \leq \abs{p - p_N} + \abs{\int_{N}^b \sin t^2 dt} < \e + \frac{1}{N} \leq \e + \frac{1}{N_0} < \e + \e  = 2\e
    \end{align*}
    Since $\e$ is arbitrary, we conclude that $\lim_{b \rightarrow \infty} \int_0^b \sin t^2 dt = p$ and hence the improper integral converges.
\end{proof}


\subsection{Fermat's little theorem}


\section{Experiments}

The following experiments refer to \emph{linearly} and \emph{nonlinearly} separable generated datasets of 100 training examples. All the training times refer to running on a laptop with an Intel i7-6700HQ (8) @ 3.500GHz and 31.2 GB of memory.

The Python source code is available at: \href{https://github.com/dmeoli/optiml}{\texttt{github.com/dmeoli/optiml}}.

\subsection{Support Vector Classifier}

Below experiments are about the SVC for which has been tested different values for the regularization hyperparameter $C$, i.e., from \emph{soft} to \emph{hard margin}, and in case of nonlinearly separable data also different \emph{kernel functions} mentioned above.

The experiments about SVCs are available at: \\ \href{https://github.com/dmeoli/optiml/blob/master/notebooks/optimization/CM_SVC_report_experiments.ipynb}{\texttt{github.com/dmeoli/optiml/blob/master/notebooks/optimization/CM\_SVC\_report\_experiments.ipynb}}.

\subsubsection{Hinge loss}

\paragraph{Primal formulation}

% https://arxiv.org/pdf/1008.4000.pdf
% https://hal.archives-ouvertes.fr/hal-00708243v1/file/appendixMNBCI.pdf
% https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=6580579&casa_token=5p51fnC0_mgAAAAA:CEwEO0ZeqzWDMlS_QG2fsfk4L7xlwUr6LzLu-Gw-nYD9CPEAhMCIMeyDDOTh7iATszKN3V4rvQ

The experiments results shown in~\ref{primal_l1_svc_cv_results} referred to \emph{Stochastic Gradient Descent} algorithm are obtained with $\alpha$, i.e., the \emph{learning rate} or \emph{step size}, setted to $1/L$ where $\displaystyle L = \frac{C}{n} \sum_{i=1}^n \| x_i \|^2$ and $\beta$, i.e., the \emph{momentum}, equal to 0.5. The optimization process is stopped if after 5 iterations the function value does not improve by at least $1\mathrm{e}{-8}$.

\input{experiments/primal_l1_svc}

\begin{figure}[H]
	\centering
	\includegraphics[scale=0.55]{img/l1_svc_loss_history}
	\caption{SGD convergence for the primal formulation of the $\protect \mathcal{L}_1$-SVC}
	\label{fig:l1_svc_loss_history}
\end{figure}

\paragraph{Linear dual formulations}

The experiments results shown in~\ref{linear_lagrangian_dual_l1_svc_cv_results} are obtained with $\alpha$, i.e., the \emph{learning rate} or \emph{step size}, setted to 1 for the \emph{AdaGrad} algorithm. Note that the \emph{unreg\_bias} and \emph{reg\_bias} duals refers to the \emph{Lagrangian dual} formulations~\eqref{eq:l1_svc_aug_lagrangian_dual} and~\eqref{eq:l1_svc_bcqp_aug_lagrangian_dual} respectively with $\rho$ equals to 1. The optimization process is stopped if the primal-dual weight vector does not change by at least $1\mathrm{e}{-6}$  between two consecutive iterations.

\input{experiments/linear_dual_l1_svc}

\input{experiments/linear_lagrangian_dual_l1_svc}

\paragraph{Nonlinear dual formulations}

The experiments results shown in~\ref{nonlinear_dual_l1_svc_cv_results} and~\ref{nonlinear_lagrangian_dual_l1_svc_cv_results} are obtained with \emph{d} and \emph{r} hyperparameters equal to 3 and 1 respectively for the \emph{polynomial} kernel; \emph{gamma} is setted to \emph{`scale`} for both \emph{polynomial} and \emph{gaussian} kernels. The experiments results shown in~\ref{nonlinear_lagrangian_dual_l1_svc_cv_results} are obtained with $\alpha$, i.e., the \emph{learning rate} or \emph{step size}, setted to 1 for the \emph{AdaGrad} algorithm. Note that the \emph{unreg\_bias} and \emph{reg\_bias} duals refers to the \emph{Lagrangian dual} formulations~\eqref{eq:l1_svc_aug_lagrangian_dual} and~\eqref{eq:l1_svc_bcqp_aug_lagrangian_dual} respectively with $\rho$ equals to 1. The optimization process is stopped if the primal-dual weight vector does not change by at least $1\mathrm{e}{-6}$  between two consecutive iterations.

\input{experiments/nonlinear_dual_l1_svc}

\input{experiments/nonlinear_lagrangian_dual_l1_svc}

\begin{figure}[H]
	\centering
	\includegraphics[scale=0.55]{img/lagrangian_dual_l1_svc_loss_history}
	\caption{AdaGrad convergence for the Lagrangian dual formulation of the $\protect \mathcal{L}_1$-SVC}
	\label{fig:lagrangian_dual_l1_svc_loss_history}
\end{figure}

\pagebreak

\subsubsection{Squared hinge loss}

\paragraph{Primal formulation}

% https://arxiv.org/pdf/1008.4000.pdf
% https://hal.archives-ouvertes.fr/hal-00708243v1/file/appendixMNBCI.pdf
% https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=6580579&casa_token=5p51fnC0_mgAAAAA:CEwEO0ZeqzWDMlS_QG2fsfk4L7xlwUr6LzLu-Gw-nYD9CPEAhMCIMeyDDOTh7iATszKN3V4rvQ

The experiments results shown in~\ref{primal_l2_svc_cv_results} referred to \emph{Stochastic Gradient Descent} algorithm are obtained with $\alpha$, i.e., the \emph{learning rate} or \emph{step size}, setted to $1/L$ where $\displaystyle L = \frac{C}{n} \sum_{i=1}^n \| x_i \|^2$ and $\beta$, i.e., the \emph{momentum}, equal to 0.5. The optimization process is stopped if after 5 iterations the function value does not improve by at least $1\mathrm{e}{-8}$.

\input{experiments/primal_l2_svc}

\begin{figure}[H]
	\centering
	\includegraphics[scale=0.55]{img/l2_svc_loss_history}
	\caption{SGD convergence for the primal formulation of the $\protect \mathcal{L}_2$-SVC}
	\label{fig:l2_svc_loss_history}
\end{figure}

\pagebreak

\paragraph{Linear dual formulations}

The experiments results shown in~\ref{linear_lagrangian_dual_l2_svc_cv_results} are obtained with $\alpha$, i.e., the \emph{learning rate} or \emph{step size}, setted to 1 for the \emph{AdaGrad} algorithm. Note that the \emph{unreg\_bias} and \emph{reg\_bias} duals refers to the \emph{Lagrangian dual} formulations~\eqref{eq:l2_svc_aug_lagrangian_dual} and~\eqref{eq:l2_svc_lb_aug_lagrangian_dual} respectively with $\rho$ equals to 1. The optimization process is stopped if the primal-dual weight vector does not change by at least $1\mathrm{e}{-6}$  between two consecutive iterations.

\input{experiments/linear_lagrangian_dual_l2_svc}

\paragraph{Nonlinear dual formulations}

The experiments results shown in~\ref{nonlinear_lagrangian_dual_l2_svc_cv_results} are obtained with \emph{d} and \emph{r} hyperparameters equal to 3 and 1 respectively for the \emph{polynomial} kernel; \emph{gamma} is setted to \emph{`scale`} for both \emph{polynomial} and \emph{gaussian} kernels. The experiments results shown in~\ref{nonlinear_lagrangian_dual_l1_svc_cv_results} are obtained with $\alpha$, i.e., the \emph{learning rate} or \emph{step size}, setted to 1 for the \emph{AdaGrad} algorithm. Note that the \emph{unreg\_bias} and \emph{reg\_bias} duals refers to the \emph{Lagrangian dual} formulations~\eqref{eq:l2_svc_aug_lagrangian_dual} and~\eqref{eq:l2_svc_lb_aug_lagrangian_dual} respectively with $\rho$ equals to 1. The optimization process is stopped if the primal-dual weight vector does not change by at least $1\mathrm{e}{-6}$  between two consecutive iterations.

\input{experiments/nonlinear_lagrangian_dual_l2_svc}

\begin{figure}[H]
	\centering
	\includegraphics[scale=0.55]{img/lagrangian_dual_l2_svc_loss_history}
	\caption{AdaGrad convergence for the Lagrangian dual formulation of the $\protect \mathcal{L}_2$-SVC}
	\label{fig:lagrangian_dual_l2_svc_loss_history}
\end{figure}

\pagebreak

\subsection{Support Vector Regression}

Below experiments are about the SVR for which has been tested different values for regularization hyperparameter $C$, i.e., from \emph{soft} to \emph{hard margin}, the $\epsilon$ penalty value and in case of nonlinearly separable data also different \emph{kernel functions} mentioned above.

The experiments about SVRs are available at: \\ \href{https://github.com/dmeoli/optiml/blob/master/notebooks/optimization/CM_SVR_report_experiments.ipynb}{\texttt{github.com/dmeoli/optiml/blob/master/notebooks/optimization/CM\_SVR\_report\_experiments.ipynb}}.

\subsubsection{Epsilon-insensitive loss}

\paragraph{Primal formulation}

% https://arxiv.org/pdf/1008.4000.pdf
% https://hal.archives-ouvertes.fr/hal-00708243v1/file/appendixMNBCI.pdf
% https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=6580579&casa_token=5p51fnC0_mgAAAAA:CEwEO0ZeqzWDMlS_QG2fsfk4L7xlwUr6LzLu-Gw-nYD9CPEAhMCIMeyDDOTh7iATszKN3V4rvQ

The experiments results shown in~\ref{primal_l1_svr_cv_results} referred to \emph{Stochastic Gradient Descent} algorithm are obtained with $\alpha$, i.e., the \emph{learning rate} or \emph{step size}, setted to $1/L$ where $\displaystyle L = \frac{C}{n} \sum_{i=1}^n \| x_i \|^2$ and $\beta$, i.e., the \emph{momentum}, equal to 0.2. The optimization process is stopped if after 5 iterations the function value does not improve by at least $1\mathrm{e}{-8}$.

\input{experiments/primal_l1_svr}

\begin{figure}[H]
	\centering
	\includegraphics[scale=0.5]{img/l1_svr_loss_history}
	\caption{SGD convergence for the primal formulation of the $\protect \mathcal{L}_1$-SVR}
	\label{fig:l1_svr_loss_history}
\end{figure}

\pagebreak

\paragraph{Linear dual formulations}

The experiments results shown in~\ref{linear_lagrangian_dual_l1_svr_cv_results} are obtained with $\alpha$, i.e., the \emph{learning rate} or \emph{step size}, setted to 1 for the \emph{AdaGrad} algorithm. Note that the \emph{unreg\_bias} and \emph{reg\_bias} duals refers to the \emph{Lagrangian dual} formulations~\eqref{eq:l1_svr_aug_lagrangian_dual} and~\eqref{eq:l1_svr_bcqp_aug_lagrangian_dual} respectively with $\rho$ equals to 1. The optimization process is stopped if the primal-dual weight vector does not change by at least $1\mathrm{e}{-6}$  between two consecutive iterations.

\input{experiments/linear_dual_l1_svr}

\input{experiments/linear_lagrangian_dual_l1_svr}

\begin{figure}[H]
	\centering
	\includegraphics[scale=0.55]{img/linear_lagrangian_dual_l1_svr_loss_history}
	\caption{AdaGrad convergence for the Lagrangian dual formulation of the linear $\protect \mathcal{L}_1$-SVR}
	\label{fig:linear_lagrangian_dual_l1_svr_loss_history}
\end{figure}

\paragraph{Nonlinear dual formulations}

The experiments results shown in~\ref{nonlinear_dual_l1_svr_cv_results} and~\ref{nonlinear_lagrangian_dual_l1_svr_cv_results} are obtained with \emph{gamma} setted to \emph{`scale`} for both \emph{gaussian} and \emph{laplacian} kernels. The experiments results shown in~\ref{nonlinear_lagrangian_dual_l1_svc_cv_results} are obtained with $\alpha$, i.e., the \emph{learning rate} or \emph{step size}, setted to 1 for the \emph{AdaGrad} algorithm. Note that the \emph{unreg\_bias} and \emph{reg\_bias} duals refers to the \emph{Lagrangian dual} formulations~\eqref{eq:l1_svr_aug_lagrangian_dual} and~\eqref{eq:l1_svr_bcqp_aug_lagrangian_dual} respectively with $\rho$ equals to 1. The optimization process is stopped if the primal-dual weight vector does not change by at least $1\mathrm{e}{-6}$  between two consecutive iterations.

\input{experiments/nonlinear_dual_l1_svr}

\input{experiments/nonlinear_lagrangian_dual_l1_svr}

\begin{figure}[H]
	\centering
	\includegraphics[scale=0.55]{img/laplacian_lagrangian_dual_l1_svr_loss_history}
	\caption{AdaGrad convergence for the Lagrangian dual formulation of the laplacian $\protect \mathcal{L}_1$-SVR}
	\label{fig:laplacian_lagrangian_dual_l1_svr_loss_history}
\end{figure}

\begin{figure}[H]
	\centering
	\includegraphics[scale=0.55]{img/gaussian_lagrangian_dual_l1_svr_loss_history}
	\caption{AdaGrad convergence for the Lagrangian dual formulation of the gaussian $\protect \mathcal{L}_1$-SVR}
	\label{fig:gaussian_lagrangian_dual_l1_svr_loss_history}
\end{figure}

\pagebreak

\subsubsection{Squared epsilon-insensitive loss}

\paragraph{Primal formulation}

% https://arxiv.org/pdf/1008.4000.pdf
% https://hal.archives-ouvertes.fr/hal-00708243v1/file/appendixMNBCI.pdf
% https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=6580579&casa_token=5p51fnC0_mgAAAAA:CEwEO0ZeqzWDMlS_QG2fsfk4L7xlwUr6LzLu-Gw-nYD9CPEAhMCIMeyDDOTh7iATszKN3V4rvQ

The experiments results shown in~\ref{primal_l2_svr_cv_results} referred to \emph{Stochastic Gradient Descent} algorithm are obtained with $\alpha$, i.e., the \emph{learning rate} or \emph{step size}, setted to $1/L$ where $\displaystyle L = \frac{C}{n} \sum_{i=1}^n \| x_i \|^2$ and $\beta$, i.e., the \emph{momentum}, equal to 0.2. The optimization process is stopped if after 5 iterations the function value does not improve by at least $1\mathrm{e}{-8}$.

\input{experiments/primal_l2_svr}

\begin{figure}[H]
	\centering
	\includegraphics[scale=0.5]{img/l2_svr_loss_history}
	\caption{SGD convergence for the primal formulation of the $\protect \mathcal{L}_2$-SVR}
	\label{fig:l2_svr_loss_history}
\end{figure}

\pagebreak

\paragraph{Linear dual formulations}

The experiments results shown in~\ref{linear_lagrangian_dual_l2_svr_cv_results} are obtained with $\alpha$, i.e., the \emph{learning rate} or \emph{step size}, setted to 1 for the \emph{AdaGrad} algorithm. Note that the \emph{unreg\_bias} and \emph{reg\_bias} duals refers to the \emph{Lagrangian dual} formulations~\eqref{eq:l2_svr_aug_lagrangian_dual} and~\eqref{eq:l2_svr_lb_aug_lagrangian_dual} respectively with $\rho$ equals to 1. The optimization process is stopped if the primal-dual weight vector does not change by at least $1\mathrm{e}{-6}$  between two consecutive iterations.

\input{experiments/linear_lagrangian_dual_l2_svr}

\begin{figure}[H]
	\centering
	\includegraphics[scale=0.55]{img/linear_lagrangian_dual_l2_svr_loss_history}
	\caption{AdaGrad convergence for the Lagrangian dual formulation of the linear $\protect \mathcal{L}_2$-SVR}
	\label{fig:linear_lagrangian_dual_l2_svr_loss_history}
\end{figure}

\paragraph{Nonlinear dual formulations}

The experiments results shown in~\ref{nonlinear_lagrangian_dual_l2_svr_cv_results} are obtained with \emph{gamma} setted to \emph{`scale`} for both \emph{gaussian} and \emph{laplacian} kernels. The experiments results shown in~\ref{nonlinear_lagrangian_dual_l2_svr_cv_results} are obtained with $\alpha$, i.e., the \emph{learning rate} or \emph{step size}, setted to 1 for the \emph{AdaGrad} algorithm. Note that the \emph{unreg\_bias} and \emph{reg\_bias} duals refers to the \emph{Lagrangian dual} formulations~\eqref{eq:l2_svr_aug_lagrangian_dual} and~\eqref{eq:l2_svr_lb_aug_lagrangian_dual} respectively with $\rho$ equals to 1. The optimization process is stopped if the primal-dual weight vector does not change by at least $1\mathrm{e}{-6}$  between two consecutive iterations.

\input{experiments/nonlinear_lagrangian_dual_l2_svr}

\begin{figure}[H]
	\centering
	\includegraphics[scale=0.55]{img/laplacian_lagrangian_dual_l2_svr_loss_history}
	\caption{AdaGrad convergence for the Lagrangian dual formulation of the laplacian $\protect \mathcal{L}_2$-SVR}
	\label{fig:laplacian_lagrangian_dual_l2_svr_loss_history}
\end{figure}

\begin{figure}[H]
	\centering
	\includegraphics[scale=0.55]{img/gaussian_lagrangian_dual_l2_svr_loss_history}
	\caption{AdaGrad convergence for the Lagrangian dual formulation of the gaussian $\protect \mathcal{L}_2$-SVR}
	\label{fig:gaussian_lagrangian_dual_l2_svr_loss_history}
\end{figure}


\section{Decomposition Attack using Impossible Monomials}
\SecLabel{monoattack}

\newcommand\func{f}

In this section I describe how large enough classes of impossible monomials can be used to mount a recovery attack on the last round's Feistel function.

The high level idea is the following. Consider a 5-round $2n$-bit Feistel Network with bijective Feistel functions, i.e. let $S^5 \in \bij{5}{n-1}$. Let $S^4 \in \bij{4}{n-1}$ be the Feistel Network consisting of the first 4 rounds of $S^5$ and let $\func\colon \field{n} \to \field{n}$ be the Feistel function used in the last round.
From Theorem~\Ref{thm:low-bij}, for any $k$, any $(n,k)$-monomial is not present in the ANFs of the right output branch of $S^4$. However, in the following 5-th round the output of the last Feistel function is xored into this branch and becomes the left output branch of $S^5$. This result now may or may not contain the $(n,k)$-monomials. By observing the presence of such monomials in the ANFs of the left branch of $S^5$, we can deduce some information about the last Feistel function in the form of linear equations. If the number of impossible monomials is large enough, an equivalent of the Feistel function $f$ can be recovered. For an illustration see \FigRef{impmono-explanation}, where the 5-th round of a Feistel Network with 3-bit branches is shown. $a_u$ denotes the ANF coefficient of a monomial that is impossible in the right branch of a 4-round Feistel Network.

\FigTex{impmono-explanation.tex}

More formally, observe that by the Feistel structure
$$(\LB \circ S^5) \oplus (f \circ \RB \circ S^5) = \RB \circ S^4.$$
Consider an arbitrary coordinate position $i, 1 \le i \le n$. For any monomial $x^u$ that is impossible in $\RB \circ S^4$ (e.g. any $(n,k)$-monomial),
$$
\coef{u}{\inprod{e_i, \LB \circ S^5}} \oplus
\coef{u}{\inprod{e_i, \func \circ \RB \circ S^5}} =
0.
$$
By decomposing $\func$ through its ANF,
$$
\coef{u}{\inprod{e_i, \LB \circ S^5}} =
\bigoplus_{v \in \field{n}} \coef{u}{
    \coef{v}{\inprod{e_i, \func}}
    (\RB \circ S^5)^v}.
$$
Since $S^5$ is known, this can be considered as a linear equation on the unknown ANF coefficients of $\inprod{e_i, \func}$. In total, there are $2^n-1$ equations (from $2^n$ impossible $(n,k)$-monomials, except the $(n,n)$-monomial) and $2^n-1$ unknowns (the constant is excluded). More equations can be obtained by considering the other classes of impossible monomials from Theorem~\Ref{thm:low-bij}. Therefore, it can be expected that the system will have (close to) full rank with high probability and the solution will be unique. The algorithm of the attack is given in Algorithm~\Ref{alg:recovery}.

\begin{algorithm}
    \caption{Feistel Function Recovery Attack}
    \Label{alg:recovery}

    \begin{algorithmic}[1]
        \Require the full codebook of a function $S \in \nbij{r}{d}, S\colon \field{2n} \to \field{2n}$; a set $U \subseteq \field{2n}\}$.

        \Ensure a function $f\colon \field{n} \to \field{n}, \deg{f} \le d$ if exists, such that
        \Statex for all $u \in U$ and for all $i,1 \le i \le n$,~ $\coef{u}{\inprod{e_i, \RB \circ R_f \circ S}} = 0$.

        \State $V \gets \pset{v \in \field{n} \mid 1 \le \wt(v) \le d}$

        \State $M \gets$ a $|U|\times|V|$ matrix indexed by $U$ and $V$
        \ForAll{$i \in \seg{1}{n}$}
            \State $b^i \gets$ a $|U|$-bit vector indexed by $U$
        \EndFor
        \ForAll{$u \in U$}
            \ForAll{$v \in V$}
                \State $M_{u,v} \gets \bigoplus_{x \preceq u}
                    (\RB \circ S(x))^v$
            \EndFor
            \ForAll{$i \in \seg{1}{n}$}
                \State $b^i_u \gets \bigoplus_{x \preceq u} \inprod{e_i, \LB \circ S(x)}$
            \EndFor
        \EndFor

        \ForAll{$i \in \{1, \ldots, n\}$}
            \State $a \gets$ a solution of $M\times a = b^i$
            \State $f_i \gets \pround{x \mapsto \oplus_{v \in V} a_v x^v}$
        \EndFor
        \State \Return $f = (f_1, \ldots, f_n)$
    \end{algorithmic}
\end{algorithm}

% ===============================================================

\subsection{On the Assumptions}

In the decomposition attack it is assumed that the equation system will have full or close to full rank. Then the correct Feistel function $f$ will be among one of the few system's solutions.

One reason for a possible rank deficiency is that the low-degree monomials in the ANF of the Feistel function $f$ may not generate the high-degree 4-round impossible monomials. In such case the equations generated from the high-degree 4-round impossible monomials provide no information about the low-degree monomials of $f$. In particular, Theorem~\Ref{thm:bij} proves that linear monomials of $f$ can not generate $(n,n-1)$ monomials when composed with the right output branch of a 5-round Feistel Network.

To the best of my knowledge, there are no known ways to prove even a possibility of presence of any highest-degree monomials in, for example, a 5-round Feistel Network. Indeed, in general, proving lower bounds on the algebraic degree is a very difficult problem.

% ===============================================================

\subsection{Instantiations}

The attack is not restricted to the case of a 5-round Feistel Network with bijective functions. The requirement is to have enough impossible monomials, which can be obtained from Theorems~\Ref{thm:low-nbij},\Ref{thm:low-bij} or by another analysis methods. In practice, a cryptanalyst can generate random instances of the analyzed structure and empirically determine all impossible monomial classes with high probability. This analysis will not dominate the complexity.

The described 5-round attack corresponds to the case of the type-II distinguisher. It exploits a large amount of impossible monomials in a 4-round network, i.e. the one that has the type-I distinguisher. I propose a conjecture on the generalization of this rule.

\begin{conjecture}
\Label{conj:impmono}
Let $r$ be the maximum number of rounds such that all $S \in \nbij{r}{d}$ have the type-II distinguisher. Then the impossible monomial attack succeeds with high probability on all $S \in \nbij{r}{d}$, i.e. it outputs a negligible number of candidates for the last Feistel function, and the correct one is always among them.
\end{conjecture}

\paragraph{Experiment.} I have implemented the attack in~Sage~\cite{sage} and performed a few experiments on small values of the branch size $n$. For all $n \in \pset{3,4,5,6,7}$ and $d \in \pset{2,3,n-1,n}$, I generated 100 random instances of $\nbij{r}{d}$ (and $\bij{r}{n-1}$) for maximum $r$ such that the structure has the type-II distinguisher. Then for the first $r-1$ rounds I empirically evaluated all impossible monomial classes. Using these classes, I generated the equation system of the impossible monomial attack for each of the 100 instances. I computed the average rank of the system and the system's dimension, i.e. the number of unknowns. In addition, I verified that the actual last round Feistel function satisfies the equations. The results of the experiment are given in \TabRef{impmonorank}.

\FigTex{impmonorank.tex}

The results show that the rank is close to the maximum on average. It means that there are only a few solutions on average and the impossible monomial attack succeeds in the analyzed cases. The rank deficiency is larger for cases with $n=3$ and decreases fast with the growth of $n$. Furthermore, the results confirm the conjecture on the analyzed cases.

% ===============================================================

\subsection{Relation with Integral Attack from~\cite{LeoFeistel}}

An integral distinguisher was already used to mount a Feistel function recovery attack by~Biryukov~\etal{}~\cite{LeoFeistel}. They show that, for a 5-round Feistel Network, a 4-round integral distinguisher provides a linear equation on the \emph{values} of the last Feistel function.
The impossible monomial attack described in this section considers the same equation system but in the \emph{monomial basis}. That is, the unknown variables are the monomial coefficients in the ANFs of the coordinates of the Feistel function. Since the ANF coefficients can be computed by summing the function over particular sets, this is a change of basis (in other words, the \Mobius{} transform is linear).
The advantage of the monomial basis is that an upper bound on the degree of the Feistel functions can be used to decrease the number of unknowns.

\Todo{Implementation of full decomposition? maybe a subsection here. Experiments (how large n?) Link to published source code.}


\documentclass{beamer}
\usetheme{Copenhagen}

\title{Introduction to Python}
\usepackage{listings}
\subtitle{Scientific Computing with Python}
\date{2nd December 2016}

\begin{document}

\lstset{language=Python}

\newcommand{\snippet}{\lstinline}

\begin{frame}
  \titlepage
\end{frame}

\section{Options}

\begin{frame}[fragile]{Choices choices}
\begin{itemize}
\item Python has {\em a lot} of options for scientific computing
\item Today is a small selection.
\pause
\item Today: numpy, matplotlib, (no sympy - sorry)
\pause
\item Missed out: Pandas, SciPy, PyGSL, ScientificPython, GmPy\ldots
\item Please \bf{read the docs!}
\end{itemize}
\end{frame}

\section{numpy}

\begin{frame}[fragile]{Introduction to numpy}
\begin{itemize}
\item What is it?
\begin{block}{}
Numpy is an extension to Python adding support for large, multi-dimensional arrays and matrices, along with
a large library of high-level mathematical functions to operate on these arrays.
\end{block}
\pause
\item More importantly - numpy is {\bf fast}.
\item numpy is written in C and so is as fast as e.g. matlab.
\pause
\item Designed for ease of use with matlab - similar functions and can accept .mat files.
\end{itemize}
\end{frame}

\subsection{numpy Classes and Functions}
\begin{frame}[fragile]{ndarray}
\begin{itemize}
\item The basic building block is the \snippet{ndarray} type.
\pause
\item Multidimensional array. Can use it just like a vector, matrix or higher dimensional analogues.
\pause
\item Don't use the \snippet{matrix} class as it will give you odd bugs!
\pause
\item Arrays have a \snippet{shape} field that returns a vector of dimensions.
\begin{block}{}
\begin{lstlisting}{frame=single}
import numpy as np
x = np.random.rand(3) # x is 1d array
print x.shape # 3
\end{lstlisting}
\end{block}
\end{itemize}
\end{frame}

\begin{frame}[fragile]{ndarray}
\begin{itemize}
\item Many similar functions to matlab.
\begin{block}{}
\begin{lstlisting}{frame=single}
import numpy as np
x = np.random.rand(3) # shape is 3
y = np.eye((3,3)) # identity matrix
z = np.zeros((4,4,4)) # note two sets of brackets
\end{lstlisting}
\end{block}
\end{itemize}
\end{frame}

\begin{frame}[fragile]{ndarray}
\begin{itemize}
\item All Python arithmetic operators are {\em elementwise}.
\item i.e. \snippet{*},\snippet{+},\snippet{**} etc.
\pause
\item Use \snippet{numpy.dot} for matrix-type multiplication.
\pause
\item If shapes don't match well then numpy tries to broadcast - read the docs!
\pause
\item You can also change the shape with \snippet{.reshape}.
\pause
\item Demo!
\end{itemize}
\end{frame}

\begin{frame}[fragile]{ndarray}
\begin{itemize}
\item Remember that numpy is still just a Python library.
\pause
\item Everything you know still applies
\pause
\item e.g. slicing - slightly nicer syntax for numpy
\begin{block}{}
\begin{lstlisting}{frame=single}
import numpy as np
x = np.arange(6).reshape(4,4)
y = x[3,:] # y is 4th row of x
\end{lstlisting}
\end{block}
\end{itemize}
\end{frame}

\begin{frame}[fragile]{Statistics Functionality}
\begin{itemize}
\item numpy is smart - if you find yourself writing a for loop always try and rewrite in terms numpy functions.
\pause
\item Numpy is designed so that everything that works in one-dimension should work in any dimension.
\item A lot of basic stats is built in: mean, standard deviation etc.
\pause
\item Example: linear regression
\end{itemize}
\end{frame}

\section{Matplotlib}
\begin{frame}[fragile]{What is matplotlib}
\begin{itemize}
\item What is it?
\begin{block}{}
matplotlib is a python 2D plotting library which produces publication quality figures in a variety of hardcopy formats and interactive environments across platforms.
\end{block}
\pause
\item Makes things very simple!
\pause
\item Line plots, scatter plots, histograms, contour plots, 3d plots etc.
\pause
\item Integrates well with numpy
\end{itemize}
\end{frame}

\begin{frame}[fragile]{Basic idea}
\begin{itemize}
\item There is always a 'current figure' and 'current plot'.
\item You build up the plot as you go before using \snippet{.show()} or \snippet{.savefig("filename")}.
\pause
\item Can have subplots, labelled axes etc.
\pause
\item Best seen through example!
\end{itemize}
\end{frame}

\end{document}