\subsection{Real differentiability}\label{subsec:real_differentiability}

\begin{proposition}\label{thm:real_valued_differentiability}
  Let \( U \subseteq \BbbR^n \) be an open set. A real-valued function \( f: U \to \BbbR \) is differentiable at \( x \) in the direction \( h \) if and only if \( \varphi(t) = f(x + th) \) is right-differentiable at \( 0 \).
\end{proposition}
\begin{proof}
  \begin{equation*}
    f_+'(x)(h) \coloneqq \lim_{t \downarrow 0} \frac {f(x + th) - f(x)} t = \varphi_+'(0)(1).
  \end{equation*}
\end{proof}

\begin{example}[Weierstrass' nowhere differentiable function]\label{ex:weierstrass_nowhere_differentiable_function}\mcite[\textnumero 271]{Фихтенгольц1968Том2}
  Let \( a \in (0, 1) \) and \( b \) is a positive odd integer such that
  \begin{equation*}
    ab > 1 + \frac 3 2 \pi.
  \end{equation*}

  Define the function
  \begin{equation*}
    f(x) \coloneqq \sum_{k=0}^\infty a^k \cos(b^k \pi x).
  \end{equation*}

  \begin{figure}
    \centering
    \includegraphics{output/ex__weierstrass_nowhere_differentiable_function.pdf}
    \caption
    {
      Plot of the third partial sum of the Weierstrass function with \( a = 0.9 \) and \( b = 7 \) from \( -\sfrac \pi 8 \) to \( \sfrac \pi 8 \).
    }
    \label{fig:ex:weierstrass_nowhere_differentiable_function/plot}
  \end{figure}

  Since \( \cos \) is bounded for real arguments and \( a \in (0, 1) \), each term is uniformly bounded by \( 1 \) and by \fullref{thm:weierstrass_series_criterion}, \( f \) is continuous. However, it is not \hyperref[def:differentiability]{differentiable} at any point. The proof of the latter is involved and will not be given here.
\end{example}


\section{Theoretical Justifications of AMP}

\begin{frame}{AMP Finds Flatter Local Minima}

\begin{columns}
\column{0.68\textwidth}

\begin{block}{Locally Gaussian Assumption of Empirical Risk}
\vspace{-0.5em}
\begin{equation*}
\mathcal{L}_\mathrm{ERM}\approx\gamma(\boldsymbol{\theta};\boldsymbol{\mu},\boldsymbol{\kappa},A,C)
\end{equation*}
\vspace{-1.5em}

\textit{where $\gamma(\boldsymbol{\theta};\boldsymbol{\mu},\boldsymbol{\kappa},A,C)$ is minimized when $\boldsymbol{\theta}=\boldsymbol{\mu}$ and the minimum value is $\gamma^\ast(\boldsymbol{\mu},\boldsymbol{\kappa},A,C)=C-A$.}
\end{block}

\begin{theorem}[stated informally]
The minimum value of the AMP loss is
\vspace{-0.5em}
\begin{equation*}
\gamma_\mathrm{AMP}^\ast(\boldsymbol{\mu},\boldsymbol{\kappa},A,C)=C-A\exp\left(-\frac{\epsilon^2}{2\sigma^2}\right)
\end{equation*}
\vspace{-1.5em}

where $\sigma^2$ is the smallest eigenvalue of $\boldsymbol{\kappa}$.
\end{theorem}

\column{0.32\textwidth}
\begin{figure}
\includegraphics[width=.7\textwidth]{figs/surface.png}
\end{figure}
\vspace{-0.5em}
\begin{figure}
\includegraphics[width=.7\textwidth]{figs/gaussian.pdf}
\caption{The minimum values of $\gamma$ and $\gamma_\mathrm{AMP}$.}
\end{figure}
\end{columns}
\vspace{1em}
\end{frame}

\begin{frame}{AMP Regularizes Gradient Norm}

\begin{theorem}[stated informally]
Let $N=1$. The AMP training is equivalent to ERM training with an additional term:
\begin{equation*}
\widetilde{\mathcal{J}}_\mathrm{ERM}(\boldsymbol{\theta}):=\mathcal{J}_\mathrm{ERM}(\boldsymbol{\theta})+\Omega(\boldsymbol{\theta})
\end{equation*}
where
\begin{equation*}
\Omega(\boldsymbol{\theta}):=\begin{cases}
\zeta\Vert\nabla_{\boldsymbol{\theta}}\mathcal{J}_\mathrm{ERM}(\boldsymbol{\theta})\Vert_2^2,&\Vert\zeta\nabla_{\boldsymbol{\theta}}\mathcal{J}_\mathrm{ERM}(\boldsymbol{\theta})\Vert_2\le\epsilon\\
\epsilon\Vert\nabla_{\boldsymbol{\theta}}\mathcal{J}_\mathrm{ERM}(\boldsymbol{\theta})\Vert_2,&\Vert\zeta\nabla_{\boldsymbol{\theta}}\mathcal{J}_\mathrm{ERM}(\boldsymbol{\theta})\Vert_2>\epsilon
\end{cases}
\end{equation*}
\end{theorem}

% Thus, the AMP training algorithm effectively tries to find the local minima of empirical risk that not only have low values, but also have small gradient norm near the minima. Note that a minimum with smaller gradient norms around it is a flatter minimum.

\end{frame}


%---------------------------Condition Number-----------------------------
\section{Condition\label{s:tri-condition}}

The condition number of the weighted Jacobian matrix is
\[
  q = \frac{
    \left(\vec L_2\cdot\vec L_2 + \vec L_1\cdot\vec L_1 + \vec L_1\cdot\vec L_2 \right)}%
    {2A\sqrt{3}}.
\]

Note that when $A = 0$, we set $q = DBL\_MAX$.
In theory the condition number is invariant to which node it is computed at,
but floating point truncation error can contribute to differences between
values computed for each node.
\verd\ always uses the first vertex.

\trimetrictable{condition}%
{$1$}%                                                Dimension
{$[1,1.3]$}%                                          Acceptable range
{$[1,DBL\_MAX]$}%                                     Normal range
{$[1,DBL\_MAX]$}%                                     Full range
{$1$}%                                                Unit equilateral triangle value
{\cite{knu:00,knu:03}}%                               Reference(s)
{v\_tri\_condition}%                            Verdict function name


\section{Proper Methodology}
The problem is formulated as a normal form game where the players are the two hospitals.
Each hospital is given \( N_A \) and \( N_B \) number of strategies where \( N_A \)
and \( N_B \) are the total capacities of the hospitals.
In other words, depending on the capacity of each hospital, they may choose to stop
receiving patients from arriving ambulances whenever they reach a certain capacity
threshold.
The goal of this problem is to satisfy the ED regulations which state that 95\%
of the patients should see a specialist within 4 hours of their arrival to the hospital.
The mean of the random variable \( W_q \) is the average waiting time in the queue
for hospital i.

\begin{equation}
     W_q(\lambda_i, \mu_i, \hat{c_i}) = \frac{1}{\hat{c_i} \mu_i}
     \frac{(\hat{c_i} \rho_i) ^ {\hat{c_i}}}{\hat{c_i}! (1 - \rho_i) ^ 2}P_0,
     \quad i \in \{A,B\}
\end{equation}

Thus, the utilities of the two players should be the proportion of people that fall
within the 4 hours target.
This is also equivalent to the probability of the waiting time of an individual
to be less than or equal to 4 hours.

\begin{equation}
    P(W_q(\lambda_i, \mu_i, \hat{c_i}) \leq 4), \quad i \in \{A,B\}
\end{equation}

Therefore, a sensible goal for each player should be to minimise that probability,
but the actual target of the hospitals is to satisfy 95\% of those patients within
the 4-hour time limit. Therefore, the goal should be to get that probability as
close to 0.95 as possible.
Thus each player should aim to minimise:

\begin{equation}
    |0.95 - P(W_q(\lambda_i, \mu_i, \hat{c_i}) \leq 4)|, \quad i \in \{A,B\}
\end{equation}

The classic formulation of a normal form game looks into the maximisation of each
player's payoff.
Consequently the utilities can be altered such that the goal of each player is to
maximise:

\begin{align}\label{Utilities}
    U_{\hat{c_A}, \hat{c_B}} ^ {A} = 1 - |0.95 - P(W_q(\lambda_A, \mu_A, \hat{c_A}) \leq 4)| \\
    U_{\hat{c_A}, \hat{c_B}} ^ {B} = 1 - |0.95 - P(W_q(\lambda_B, \mu_B, \hat{c_B}) \leq 4)|
\end{align}

Finally, the problem can be expressed as a normal form game with two players where
each player/hospital has \( N_A \) and \( N_B \) strategies respectively.
The two \( N_A \times N_B \) payoff matrices for the utilities of the two hospitals
can be defined as:

\begin{table}[h]
    \centering
    \begin{minipage}{.5\linewidth}
        A =
        \begin{tabular}{|l|l|l|l|}
            \hline
            \( U_{1,1}^A \) & \( U_{1,2}^A \) & \dots & \( U_{1,C_2}^A \) \\ \hline
            \( U_{2,1}^A \) & \( U_{2,2}^A \) & \dots & \( U_{2,C_2}^A \) \\ \hline
            \vdots & \vdots & \( \ddots \) & \vdots \\ \hline
            \( U_{C_1,1}^A \) & \( U_{C_1,2}^A \) & \dots & \( U_{C_1,C_2}^A \) \\ \hline
        \end{tabular}
    \end{minipage}%
    \begin{minipage}{.5\linewidth}
        B =
        \begin{tabular}{|l|l|l|l|}
            \hline
            \( U_{1,1}^B \) & \( U_{1,2}^B \) & \dots & \( U_{1,C_2}^B \) \\ \hline
            \( U_{2,1}^B \) & \( U_{2,2}^B \) & \dots & \( U_{2,C_2}^B \) \\ \hline
            \vdots & \vdots & \( \ddots \) & \vdots \\ \hline
            \( U_{C_1,1}^B \) & \( U_{C_1,2}^B \) & \dots & \( U_{C_1,C_2}^B \) \\ \hline
        \end{tabular}
    \end{minipage}
\end{table}
Once the certain strategies of the game have been selected the ambulance service
can decide what would be the optimal way to distribute patients.
However, the way the ambulance service distributes patients can affect the utilities
of the game. So how would one solve this kind of problem?

\subsection{Solution}
As mentioned before the problem requires the construction of two queuing models
that will be needed for the formulation of the normal form game.
Based on those utilities the ambulance service will then decide the percentage of
patients that will distribute to each hospital.

First and foremost, the model consists of several parameters that are unknown and
are assumed to be fixed.
The model will be run multiple times for various values of these parameters.

\begin{table}[h]
    \centering
    \begin{tabular}{|l|l|}
        \hline
        \( \Lambda_2 \) & Number of patients that need to be distributed \\ \hline
        \( \lambda_{1,i} \) & Arrival rate of other patients that enter hospital i \\ \hline
        \( \mu_i \) & Service rate of hospital i \\ \hline
        \( N_i \) & Total capacity of hospital i \\ \hline
    \end{tabular}
    \caption{Fixed Parameters}
\end{table}

Having established the fixed parameters of the model, the hospitals' utilities need
to be calculated.
In order to do so a backwards induction approach will be used.
The EMS aims to distribute the patients such that the mean waiting time of patients
is minimal.
This can be further interpreted as when the mean waiting time of hospital A equals
the mean waiting time of hospital B.
Thus, the minimal mean waiting time can be found for the values of \( p_A \) and
\( p_B \) that solve the following equation:

\begin{equation}\label{Equal_Wait}
    W_q(\lambda_A, \mu_A, \hat{c_A}) = W_q(\lambda_B, \mu_B, \hat{c_B})
\end{equation}

Equation (\ref{Equal_Wait}) needs to be solved for all values of \( c_i \in \{1,2,
\dots C_A\} \) and \( c_j \in \{1,2, \dots C_B\} \).
Then, for every \( c_i \) and \( c_j \) the utility equation (\ref{Utilities}) has
to be calculated for both hospitals.
In order to solve it though, one must first estimate the probability
\( P[(W_q)_{\{A, B\}}] \leq 4] \).
That is the probability that the waiting time in the queue for one of the hospitals
is less than 4 hours.
For a multi-server system, the distribution of the waiting time can be given by
equation \ref{Dist_Wait}.
The above expression returns the probability that the waiting time in the queue
is less than some time T.

\begin{equation}\label{Dist_Wait}
    P(W_q > T) = \frac{(\frac{\lambda}{\mu})^c P_0}{c!(1-\frac{\lambda}{c \mu})}
    (e^{-(c \mu - \lambda)T})
\end{equation}

Consequently when incorporating equation (\ref{Dist_Wait}) into (\ref{Utilities})
a newer utility equation can be acquired:

\begin{equation}\label{Utilities2}
    U_{\hat{c_i}, \hat{c_j}} ^ {\{A, B\}} = 1 - \left| \left[
        \frac{(\frac{\lambda}{\mu})^c P_0}{c!(1-\frac{\lambda}{c \mu})}
        \left( e^{-(c \mu - \lambda)T} \right) \right] - 0.05 \right|
\end{equation}

\begin{table}[h]
    \centering
    \begin{minipage}{.5\linewidth}
        A =
        \begin{tabular}{|l|l|l|l|}
            \hline
            \( U_{1,1}^A \) & \( U_{1,2}^A \) & \dots & \( U_{1,C_2}^A \) \\ \hline
            \( U_{2,1}^A \) & \( U_{2,2}^A \) & \dots & \( U_{2,C_2}^A \) \\ \hline
            \vdots & \vdots & \( \ddots \) & \vdots \\ \hline
            \( U_{C_1,1}^A \) & \( U_{C_1,2}^A \) & \dots & \( U_{C_1,C_2}^A \) \\ \hline
        \end{tabular}
    \end{minipage}%
    \begin{minipage}{.5\linewidth}
        B =
        \begin{tabular}{|l|l|l|l|}
            \hline
            \( U_{1,1}^B \) & \( U_{1,2}^B \) & \dots & \( U_{1,C_2}^B \) \\ \hline
            \( U_{2,1}^B \) & \( U_{2,2}^B \) & \dots & \( U_{2,C_2}^B \) \\ \hline
            \vdots & \vdots & \( \ddots \) & \vdots \\ \hline
            \( U_{C_1,1}^B \) & \( U_{C_1,2}^B \) & \dots & \( U_{C_1,C_2}^B \) \\ \hline
        \end{tabular}
    \end{minipage}
\end{table}


\subsection{Addition of sine and cosine}

\subsubsection{Adding waves with same frequency}

We know that:

\(a\sin(bx+c)=a\sin(bx)\cos(c)+a\sin(c)\cos(bx)\)

So:

\(a\sin(bx+c)+d\sin(bx+e)=a\sin(bx)\cos(c)+a\sin(c)\cos(bx)+ d\sin(bx)\cos(e)+d\sin(e)\cos(bx)\)

We know that:

\(\sin(\theta )=\dfrac{e^{i\theta }-e^{-i\theta }}{2i}\)

So:

\(a\sin(bx+c)+d\sin(bx+f)=a\dfrac{e^{i(bx+c)}-e^{-i(bx+c)}}{2i}+d\dfrac{e^{i(bx+f)}-e^{-i(bx+f)}}{2i}\)

\(a\sin(bx+c)+d\sin(bx+f)=\dfrac{a(e^{i(bx+c)}-e^{-i(bx+c)})+d(e^{i(bx+f)}-e^{-i(bx+f)})}{2i}\)

\(a\sin(bx+c)+d\sin(bx+f)=\dfrac{a(e^{ibx}e^{ic}-e^{-ibx}e^{-ic})+d(e^{ibx}e^{if}-e^{-ibx}e^{-if)})}{2i}\)

\(a\sin(bx+c)+d\sin(bx+f)=\dfrac{(e^{ibx}(ae^{ic}+de^{if})-e^{-ibx}(ae^{-c}+d^{-if})}{2i}\)

\(a_i\sin(b_ix+c_i)+a_j\sin(b_jx+c_j)=a_i\sin(b_ix+c_i)+a_j\sin(b_ix+c_j)\)

\(a_i\sin(b_ix+c_i)+a_j\sin(b_jx+c_j)=a_i\sin(b_ix)\cos(c_i)+a_i\sin(c_i)\cos(b_ix)+a_j\sin(b_ix)\cos(c_j)+a_j\sin(c_j)\cos(b_ix)\)


\lab{Conjugate-Gradient}{Conjugate-Gradient}
\objective{Learn about the Conjugate-Gradient Algorithm and its Uses}

\section*{Descent Algorithms and the Conjugate-Gradient Method}
There are many possibilities for solving a linear system of equations, each method with its own set of pros and cons. In this lab, we
will explore the \emph{Conjugate-Gradient algorithm}, which is a method for solving large systems of equations where other methods,
such as Cholesky factorization and simple Gaussian elimination, are unsuitable. This algorithm, however, works equally well for
optimizing convex quadratic functions, and it can even be extended to more general classes of optimization problems.

The type of linear system that Conjugate-Gradient can solve involves a matrix with special structure.
Given a symmetric positive-definite $n\times n$ matrix $Q$ and an $n$-vector $b$, we wish to find the $n$-vector $x$ satisfying
\[
Qx = b.
\]
A unique solution exists because positive-definiteness implies invertibility.
For our purposes here, it is useful to recast this problem as an equivalent optimization problem:
\[
\min_{x} f(x) := \frac{1}{2}x^TQx - b^Tx + c.
\]
Note that $\nabla f(x) = Qx - b$, so that minimizing $f(x)$ is the same as solving
\[
0 = \nabla f(x) = Qx - b,
\]
which is our original linear system.

So how do we go about minimizing the quadratic objective function $f$? Line Search methods belonging to the class called
\emph{descent algorithms} use the following strategy: start with an initial guess $x_0$, identify a direction from
this particular point along which the objective function decreases (called a \emph{descent direction}), and perform a line search to
find a new point $x_1$ satisfying $f(x_1) < f(x_0)$. Continue iteratively to produce a sequence of points $\{x_0, x_1, x_2, \ldots \}$
that (hopefully) converges to the true minimizer.

One obvious candidate for the descent direction from some point $x_i$ is simply
$-\nabla f(x_i)$, since this vector points in the direction of steepest decrease. This procedure is known as the Method of Steepest
Descent. Steepest descent, however, can be very inefficient for certain problems: depending on the geometry
of the objective function, the sequence of points can ``zig-zag" back and forth without making appreciable progress toward the true
minimizer. In contrast, the Conjugate-Gradient algorithm ensures that the true global minimizer is reached in at most $n$ steps. See
Figure \ref{basis:steepVsConj} for an illustration of this contrast.
To understand why this is the case and how the algorithm chooses each descent direction, we next discuss the idea of vector conjugacy.

\begin{figure}
\centering
\includegraphics[width=\textwidth]{steepVsConj.pdf}
\caption{Paths traced by Steepest Descent (blue) and Conjugate-Gradient (green). Notice the
zig-zagging nature of the Steepest Descent path, as opposed to the direct Conjugate-Gradient path,
which finds the minimizer in 2 steps.}
\label{basis:steepVsConj}
\end{figure}

\section*{Conjugacy}
Consider again our symmetric positive definite $n \times n$ matrix $Q$. Two vectors $x, y \in \mathbb{R}^n$ are said to be \emph{conjugate}
with respect to $Q$ if $x^TQy = 0$. A set of vectors $\{x_0, x_1, \ldots, x_m\}$ is said to be conjugate if each pair of vectors are conjugate
to each other. Note that if $Q = I$, then conjugacy is the same as orthogonality. Thus, the notion of conjugacy is in some ways a generalization
of orthogonality. It turns out that a conjugate set of vectors is linearly independent, and a conjugate basis--which can be constructed in a manner
analogous to the Gram-Schmidt orthogonalization process--can be used to diagonalize the matrix $Q$.
These are some of the theoretical reasons behind the effectiveness of the Conjugate-Gradient algorithm.
%Take this section out? Add more to it?

\section*{The Algorithm}
If we are given a set of $n$ $Q$-conjugate vectors, we can simply choose these as our direction vectors and follow the basic descent algorithm.
Convergence to the minimizer in at most $n$ steps is guaranteed because each iteration in the algorithm minimizes the objective function
over an expanding affine subspace of dimension equal to the iteration number. Thus, at the $n$-th iteration, we have minimized the function over all of
$\mathbb{R}^n$.

Unfortunately, we are not often given a set of conjugate vectors in advance, so how do we produce such a set? As mentioned earlier, a
Gram-Schmidt process could be used, and the set of eigenvectors also works, but both of these options are computationally expensive.
Built into the algorithm is a way to determine a new conjugate direction based only on the previous direction, which means less memory usage and
faster computation. We have stated the details of Conjugate-Gradient in Algorithm \ref{alg:conjgrad}.
\begin{algorithm}
\begin{algorithmic}[1]
\Procedure{Conjugate-Gradient Algorithm}{}
    \State \textrm{Choose initial point } $x_0$.
    \State $r_0 \gets Qx_0 - b, d_0 \gets -r_0, k \gets 0$.
    \While{$r_k \neq 0$}
        \State $\alpha_k \gets \frac{r_k^Tr_k}{d_k^TQd_k}$.
        \State $x_{k+1} \gets x_k + \alpha_kd_k$.
        \State $r_{k+1} \gets r_k + \alpha_kQd_k$.
        \State $\beta_{k+1} \gets \frac{r_{k+1}^Tr_{k+1}}{r_k^Tr_k}$.
        \State $d_{k+1} \gets -r_{k+1} + \beta_{k+1}d_k$.
        \State $k \gets k+1$.
    \EndWhile
\EndProcedure
\end{algorithmic}
\caption{Conjugate-Gradient Algorithm}
\label{alg:conjgrad}
\end{algorithm}

Note that the points $x_i$ are the successive approximations to the minimizer, the vectors $d_i$ are the conjugate descent
directions, and the vectors $r_i$, which actually correspond to the steepest descent directions, are used in determining the conjugate directions.
The constants $\alpha_i$ and $\beta_i$ are used, respectively, in the line search, and in ensuring the $Q$-conjugacy of the descent directions.

The most numerically expensive computation in the algorithm is matrix-vector multiplication.
Notice, however, that each iteration of the algorithm only requires one distinct matrix-vector multiplication, $Qd_k$. The rest of the
operations are simply vector-vector multiplication, addition, and scalar multiplication. This makes for a very fast algorithm.
As noted earlier, Conjugate-Gradient is especially preferred when $Q$ is large and sparse. In this case, it may be possible to
design a specialized sub-routine that performs matrix-vector multiplication by $Q$, by taking advantage of its sparseness. Doing so may
lead to further speed-ups in the overall algorithm.

We now have an algorithm that can solve certain $n \times n$ linear systems and minimize quadratic functions on $\mathbb{R}^n$ in at most $n$ steps,
and sometimes fewer, depending on the spectrum of the matrix $Q$. Further improvements on convergence may be obtained by preconditioning the matrix,
but we do not go into detail here.

\begin{problem}
Implement the basic Conjugate-Gradient algorithm presented above.
Write a function \li{conjugateGradient()} that accepts a vector $b$, an initial
guess $x_0$, a symmetric positive-definite matrix $Q$, and a default tolerance of .0001 as inputs.
Continue the algorithm until $\|r_k\|$ is less than the tolerance.
Return the solution $x^*$ to the linear system $Qx = b$.
\end{problem}

\section*{Example}
We now work through an example that demonstrates the usage of the Conjugate-Gradient algorithm. We assume that we have already written
the specified function in the above problem.

We must first generate a symmetric positive-definite matrix $Q$. This can be done by generating a random matrix $A$ and setting $Q = A^TA$.
So long as $A$ is of full column rank, the matrix $Q$ will be symmetric positive-definite.
\begin{lstlisting}
>>> import numpy as np
>>> from scipy import linalg as la

>>> # initialize the desired dimension of the space
>>> n = 10

>>> # generate Q, b
>>> A = np.random.random((n,n))
>>> Q = A.T.dot(A)
>>> b = np.random.random(n)
\end{lstlisting}
At this point, check to make sure that $Q$ is nonsingular by examining its determinant (use \li{scipy.linalg.det()}).
Provided that the determinant is nonzero, we proceed by writing a function that performs matrix-vector multiplication by $Q$ (we
will not take advantage of sparseness just now), randomly selecting a starting point (Conjugate-Gradient is not sensitive to the location of
the starting point), obtaining the answer using our function, and checking it with the answer obtained by \li{scipy.linalg.solve()}.
\begin{lstlisting}
>>> # generate random starting point
>>> x0 = np.random.random(n)

>>> # find the solution
>>> x = conjugateGradient(b, x0, mult)

>>> # compare to the answer obtained by SciPy
>>> print np.allclose(x, la.solve(Q,b))
\end{lstlisting}
The output of the print statement should be \li{True}.

Time the performance of your algorithm and of \li{scipy.linalg.solve()} on inputs of size 100.

\section*{Application: Least Squares and Linear Regression}
The Conjugate-Gradient method can be used to solve linear least squares problems, which are ubiquitous in applied science.
Recall that a least squares problem can be formulated as an optimization problem:
\[
\min_x \|Ax - b\|_2,
\]
where $A$ is an $m \times n$ matrix with full column rank, $x \in \mathbb{R}^n$, and $b \in \mathbb{R}^m$. The solution can
be calculated analytically, and is given by
\[
x^* = (A^TA)^{-1}A^Tb,
\]
or in other words, the minimizer solves the linear system
\[
A^TAx = A^Tb.
\]
Since $A$ has full column rank, we know that $A^TA$ is an $n \times n$ matrix of rank $n$, which means it is invertible. We can
therefore conclude that $A^TA$ is symmetric positive-definite, so we may use Conjugate-Gradient to solve the linear system
and obtain the least squares solution.

Linear least squares is the mathematical underpinning of linear regression, which is a very common technique in many scientific fields.
In a typical linear regression problem, we have a set of real-valued data points \{$y_1,\ldots, y_m\}$, where each
$y_i$ is paired with a corresponding set of predictor variables $\{x_{i,1}, x_{i,2}, \ldots, x_{i,n}\}$ with $n < m$.
The linear regression model posits that
\[
y_i = \beta_0 + \beta_1x_{i,1} + \beta_2x_{i,2} + \cdots + \beta_nx_{i,n} + \epsilon_i
\]
for $i = 1, 2, \ldots, m$. The real numbers $\beta_0,\ldots,\beta_n$ are known as the parameters of the model, and the
$\epsilon_i$ are independent normally-distributed error terms. Our task is to calculate the parameters that best fit the data.
This can be accomplished by posing the problem in terms of linear least squares: Define
\[
b = [y_1, \ldots, y_m]^T,
\]
\[
A =
\begin{bmatrix}
1 & x_{1,1} & x_{1,2} & \cdots & x_{1,n}\\
1 & x_{2,1} & x_{2,2} & \cdots & x_{2,n}\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
1 & x_{m,1} & x_{m,2} & \cdots & x_{m,n}
\end{bmatrix},
\]
and
\[
x = [\beta_0, \beta_1,\ldots, \beta_n]^T.
\]
Now use Conjugate-Gradient to solve the system
\[
A^TAx = A^Tb.
\]
The solution $x^* = [\beta_0^*, \beta_1^*, \ldots, \beta_n^*]^T$ gives the parameters that best fit the data.
These values can be understood as defining the hyperplane that best fits the data. See Figure \ref{fig:linregression}.

\begin{figure}
\centering
\includegraphics[width=\textwidth]{linregression.pdf}
\caption{Solving the Linear Regression problem results in a best-fit hyperplane.}
\label{fig:linregression}
\end{figure}

\begin{problem}
Using your Conjugate-Gradient function, solve the linear regression problem specified by the data contained in the file
\texttt{linregression.txt}. This is a whitespace-delimited text file formatted so that the $i$-th row consists of
$y_i, x_{i,1}, \ldots, x_{i,n}$. Use the function \li{numpy.loadtxt()} to load in the data. Report your solution.
\end{problem}

\section*{Non-linear Conjugate-Gradient Algorithms}
The algorithm presented above is only valid for certain linear systems and quadratic functions, but the basic strategy may be adapted
to minimize more general convex or non-linear functions. There are multiple ways to modify the algorithm, and they all involve getting
rid of $Q$, since there is no such $Q$ for non-quadratic functions. Generally speaking, we need to find new formulas for $\alpha_k$,
$r_k$, and $\beta_k$.

The scalar $\alpha_k$ is simply the result of performing a line-search in the given direction $d_k$, so we may define
\[
\alpha_k = \underset{x}{\arg\min}f(x_k + \alpha d_k).
\]
The vector $r_k$ in the original algorithm was really just the gradient of the objective
function, and so we may define
\[
r_k = \nabla f(x_k).
\]
There are various ways to define the constants $\beta_k$ in this more general setting, and the
right choice will depend on the nature of the objective function. A well-known formula, due to Fletcher and Reeves, is
\[
\beta_{k+1} = \frac{\nabla f_{k+1}^T \nabla f_{k+1}}{\nabla f_{k}^T \nabla f_{k}}.
\]

Making these adjustments is not difficult, but we will opt instead to use built-in functions in Python. In particular,
the SciPy module \li{scipy.optimize} provides a function \li{fmin_cg()}, which uses a non-linear Conjugate-Gradient method
to minimize general functions. Using this function is easy -- we only need to pass to it the objective function and an
initial guess.

\section*{Application: Logistic Regression}
Logistic regression is an important technique in statistical analysis and classification. The core problem in
logistic regression involves an optimization that we can tackle using nonlinear Conjugate-Gradient.

As in linear regression, we have a set of data points $y_i$ together with predictor variables
$x_{i,1}, x_{i,2}, \ldots, x_{i,n}$ for $i = 1, \ldots, m$. However, the $y_i$ are binary data points --
that is, they are either $0$ or $1$. Furthermore, instead of having a linear relationship between the
data points and the response variables, we assume the following probabilistic relationship:
\[
\mathbb{P}(y_i = 1 \, | \, x_{i,1}, \ldots, x_{i,n}) = p_i,
\]
where
\[
p_i = \frac{1}{1+\exp(-(\beta_0 + \beta_1x_{i,1} + \cdots + \beta_nx_{i,n}))}.
\]
The parameters of the model are the real numbers $\beta_0, \beta_1,\ldots, \beta_n$.
Observe that we have $p_i \in (0, 1)$ regardless of the values of the predictor variables and parameters.

The probability of observing the data points $y_i$ under this model, assuming they are independent, is given by
the expression
\[
\prod_{i=1}^m p_i^{y_i}(1-p_i)^{1-y_i}.
\]
We seek to choose the parameters $\beta_0, \ldots, \beta_n$ that maximize this probability.
To this end, define the \emph{likelihood function} $L:\mathbb{R}^{n+1} \rightarrow \mathbb{R}$ by
\[
L(\beta_0, \ldots, \beta_n) = \prod_{i=1}^m p_i^{y_i}(1-p_i)^{1-y_i}.
\]
We can now state our core problem as follows:
\[
\max_{(\beta_0,\ldots,\beta_n)}L(\beta_0, \ldots, \beta_n).
\]

Maximizing this function can be problematic for numerical reasons. By taking the logarithm of the likelihood,
we have a more suitable objective function whose maximizer agrees with that of the original likelihood function,
since the logarithm is strictly monotone increasing. Thus, we define the \emph{log-likelihood function}
$l : \mathbb{R}^{n+1} \rightarrow \mathbb{R}$ by $l = \log \circ L$.

Finally, we multiply by $-1$ to turn our problem into minimization. The final statement of the problem is:
\[
\min_{(\beta_0,\ldots,\beta_n)}-l(\beta_0, \ldots, \beta_n).
\]
A few lines of calculation reveal that
\begin{align*}
l(\beta_0,\ldots,\beta_n) = &-\sum_{i=1}^{m}\log(1+\exp(-(\beta_0 + \beta_1x_{i,1} + \cdots +\beta_nx_{i,n}))) +\\
 &\sum_{i=1}^m y_i(\beta_0 + \beta_1x_{i,1} + \cdots + \beta_nx_{i,n}).
\end{align*}
The values for the parameters that we obtain are known collectively as the \emph{maximum likelihood estimate}.

Let's work through a simple example. We will deal with just one predictor variable, and therefore two parameters.
The data is given in Table \ref{table:data}.
This is obviously just toy data with no meaning, but one can think of the $y_i$ data points as indicating, for example, the
presence of absence of a particular disease in subject $i$, with $x_i$ being the subject's weight, or age, or something
of the sort.

\begin{table}
  \caption{Data for Logistic Regression Example}
  \centering
  \begin{tabular}{c c}
    \hline\hline
    $y$ & $x$\\
    \hline
    0 & 1 \\
    0 & 2 \\
    0 & 3 \\
    0 & 4 \\
    1 & 5 \\
    0 & 6 \\
    1 & 7 \\
    0 & 8 \\
    1 & 9 \\
    1 & 10\\
    \hline
  \end{tabular}
  \label{table:data}
\end{table}

In the code below we initialize our data.
\begin{lstlisting}
>>> y = np.array([0, 0, 0, 0, 1, 0, 1, 0, 1, 1])
>>> x = np.ones((10, 2))
>>> x[:,1] = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
\end{lstlisting}
Although we have just one predictor variable, we initialized \li{x} with two columns,
the first of which consists entirely of ones, and the second of which contains the values of the
predictor variable. This extra column of ones corresponds to the parameter $\beta_0$, which, as you
will note, is not multiplied by any of the predictor variables in the log-likelihood function.

We next need to write a Python function that returns the value of our objective function for any value of the parameters,
$(\beta_0, \beta_1)$.
\begin{lstlisting}
>>> def objective(b):
...     #Return -1*l(b[0], b[1]), where l is the log likelihood.
...    return (np.log(1+np.exp(x.dot(b))) - y*(x.dot(b))).sum()
\end{lstlisting}

Finally, we minimize the objective function using \li{fmin_cg()}.
\begin{lstlisting}
>>> guess = np.array([1., 1.])
>>> b = fmin_cg(objective, guess)
Optimization terminated successfully.
         Current function value: 4.310122
         Iterations: 13
         Function evaluations: 128
         Gradient evaluations: 32
>>> print b
[-4.35776886  0.66220658]
\end{lstlisting}

We can visualize our answer by plotting the data together with the function
\[
\phi(x) = \frac{1}{1 + \exp(-\beta_0 - \beta_1x)},
\]
using the values $\beta_0, \beta_1$ that we obtained from the minimization.
\begin{lstlisting}
>>> dom = np.linspace(0, 11, 100)
>>> plt.plot(x, y, 'o')
>>> plt.plot(dom, 1./(1+np.exp(-b[0]-b[1]*dom)))
>>> plt.show()
\end{lstlisting}

Using this procedure, we obtain the plot in Figure \ref{conj:logreg}. Note that the graph of $\phi$, known as a \emph{sigmoidal curve},
gives the probability of $y$ taking the value $1$ at a particular value of $x$. Observe that as $x$ increases,
this probability approaches $1$. This is reflected in the data.
\begin{figure}
\centering
\includegraphics[width=\textwidth]{logreg.pdf}
\caption{Data from the logistic regression example together with the calculated
sigmoidal curve.}
\label{conj:logreg}
\end{figure}

\begin{problem}
Following along with the example given above, find the maximum likelihood estimate of the parameters for
the logistic regression data in the file \texttt{logregression.txt}. This is a whitespace-delimited text file
formatted so that the $i$-th row consists of $y_i, x_{i,1}, x_{i,2}, x_{i,3}.$ Since there are three
predictor variables, there are four parameters in the model. Report the calculated values.

You should be able to use much of the code above unchanged. In particular, the function \li{objective()} does
not need any changes. You simply need to set your variables \li{y} and \li{x} appropriately, and
choose a new initial guess (an array of length four). Note that \li{x} should be an $m \times 4$ array whose first
column consists entirely of ones, whose second column contains the values in the second column of the data file,
and so forth.
\end{problem}

Logistic regression can become a bit more tricky when some of the predictor variables take on binary or categorical
values. In such situations, the data requires a bit of pre-processing before running the minimization.

The values of the parameters that we obtain can be useful in analyzing relationships
between the predictor variables and the $y_i$ data points. They can also be used to classify or predict values of new
data points.

%should I give the Polak-Ribi\`{e}, Fletcher-Reeves, and Hestenes-Stiefel formulas? Maybe in the exercises? Should I
%include a discussion of when each choice is appropriate?


\subsection{Hyperbolic Paraboloids}
\noindent
Hyperbolic paraboloids have the form
\begin{equation*}
	z = x^2 - y^2
\end{equation*}
They are not radially symmetric and look like a saddle or Pringle's chip.

[INSERT IMAGE]


\lab{Plotting With matplotlib and Mayavi}{Plotting}
\objective{Introduce some of the basic plotting functions available in matplotlib and Mayavi.}
\label{lab:Matplotlib_and_Mayavi}

\section*{2-D plotting with \li{matplotlib}}
The Python library \li{matplotlib} will be our primary tool for creating 2-D graphs in this text. This lab introduces the basic features of \li{matplotlib}; for more information, visit the documentation at \url{http://matplotlib.org}.

To begin, import \li{pyplot} from \li{matplotlib}.
\begin{lstlisting}
from matplotlib import pyplot as plt
\end{lstlisting}

\subsection*{Line plots}
The function call \li{plt.plot(x, y)} takes two 1-D NumPy arrays, \li{x} and \li{y}, and plots the points \li{(x[i], y[i])}, connecting them with straight lines.

For example, this code plots $y=e^x$ on the interval $[-2,3]$. The output is in Figure \ref{fig:exp_plot}.
\begin{lstlisting}
import numpy as np
# Create an array of 501 evenly spaced points in [-2,3].
x = np.linspace(-2, 3, 501)
y = np.exp(x)
plt.plot(x, y)
plt.show()
\end{lstlisting}

\begin{figure}
\centering
\begin{subfigure}[t]{.49\textwidth}
\centering
\includegraphics[width=\textwidth]{exp_plot.pdf}
\caption{A graph of $e^x$.}
\label{fig:exp_plot}
\end{subfigure}
\begin{subfigure}[t]{.49\textwidth}
\centering
\includegraphics[width=\textwidth]{statemachine.pdf}
\caption{A graph of ten lines with randomly generated $y$ values.}
\label{fig:statemachine}
\end{subfigure}
\caption{Plots created with \li{plt.plot()}.}
\label{fig:lineplots}
\end{figure}

All calls to \li{plt} functions will modify the same figure until you call \li{plt.show()}.
The function \li{plt.show()} displays the current figure and resets the system, so that the next call to a \li{plt} function modifies a new figure.

We can take advantage of this system to plot multiple lines on the same axes.
For example, the following code plots ten lines, each with random values at integers from 1 to 10.
The output is in Figure \ref{fig:statemachine}.
\begin{lstlisting}
x = np.linspace(1, 10, 10)

# Create a 10x10 array of uniformly distributed values in [0,1).
y = np.random.rand(10, 10)

# Plot each row of y
for row in y:
    plt.plot(x, row)
plt.show()
\end{lstlisting}

Alternatively, we can produce the same graph with a single call to \li{plt.plot()}.
\begin{lstlisting}
plt.plot(x, y[0], x, y[1], x, y[2], x, y[3], x, y[4], x, y[5], x, y[6], x, y[7], x, y[8], x, y[9])
plt.show()
\end{lstlisting}

\begin{comment}
\begin{problem} Plot the function $\sin(x)$ from $0$ to $2\pi$ with a red dashed line.
Then plot the function $\cos(x)$ on the same domain with a blue dotted line.
Implement both with a single call to the \li{plot()} function.
Information on how to do this can be found in Appendix \ref{mpltables}.
\end{problem}
\end{comment}

\begin{comment}
There are also many functions that we may use to set different values in
the plotting environment. A few examples are shown in Table
\ref{mpl:useful_functions}.

\begin{table}
\begin{center}
\begin{tabular}{|l|p{6cm}|p{4cm}|}

    \hline

    Function & Description & Usage\\

    \hline

    \li{annotate} & adds a commentary at a given point on the plot &
    annotate('text',(x,y))\\

    \li{arrow} & draws an arrow from a given point on the plot &
    arrow(x,y,dx,dy)\\

    \li{axhline} & draws a horizontal line at y from xmin to xmax &
    axhline(y=0, xmin=0, xmax=1)\\

    \li{axvline} & draws a vertical line at x from ymin to ymax &
    axvline(x=0, ymin=0, ymax=1)\\

    \li{axhspan} & draws a rectangle from xmin to xmax and ymin to ymax,
    if no xmin and xmax are given it goes across the plot &
    axhspan(ymin, ymax, xmin=0, xmax=1)\\

    \li{axvspan} & draws a rectangle from ymin to ymax and xmin to xmax,
    if no ymin and ymax are given it goes across the entire plot &
    axvspan(xmin, xmax, ymin=0, ymin=1)\\

    \li{figlegend} & place a legend in the plot & figlegend(handles,
    labels, loc)\\

    \li{grid} & add gridlines & grid()\\

    \li{text} & add text at a given position on the plot &
    text(x,y,'text')\\

    \li{title} & add a title to the plot & title('text')\\

    \li{xlim} & set the x limits, returns current limits if no arguments
    are given & xlim(xmin,xmax)\\

    \li{ylim} & set the y limits, returns current limits if no arguments
    are given & ylim(ymin,ymax)\\

    \li{xticks} & set the location of the tick marks on the x axis,
    returns current locations if no arguments are given & xticks(x)\\

    \li{yticks} & set the location of the tick marks on the y axis,
    returns current locations if no arguments are given & yticks(y)\\

    \li{xlabel} & add a label to the x axis & xlabel('text')\\

    \li{ylabel} & add a label to the y axis & ylabel('text')\\

    \hline

    \end{tabular}
    \end{center}
    \caption{Some Functions to Set Plotting Options}
    \label{mpl:useful_functions}
    \end{table}

	\end{comment}

\begin{problem}\label{prob:lineplot}
Plot the curve $1/(x-1)$ on [-2,6].
Do the following to your plot:

\begin{enumerate}
\item The function \li{plt.plot()} will make the curve look continuous at $x=1$.
Plot the two sides of the curve separately (still with a single call to \li{plt.plot()}) so that the graph looks discontinuous at $x=1$.
\item Plot the curve with a line that is magenta, dashed, and thick (see Appendix \ref{mpltables}).
\item Change the range of the $y$-axis to be $[-6, 6]$.
\end{enumerate}
Your final plot should look like Figure \ref{fig:problem2}.

\begin{figure}[H]
\includegraphics[width=.7\textwidth]{soln2.pdf}
\caption{Correct output for Problem \ref{prob:lineplot}.}
\label{fig:problem2}
\end{figure}
\end{problem}

\begin{comment}
\begin{problem} Plot the curve $\sin(x)\frac{1}{x+1}$ from $0$ to $10$.

Use blue shading under the curve when it is positive and red when it is negative (you may want to consider using the \li{fill_between} command.
Make the line dotted.
Label the x-axis ``x-axis'', the y-axis ``y-axis'',and the plot ``My Plot''.
Enable the grid lines.

Finally, use the \li{scatter} command to include a scatter plot of half of the value of the function at each of its maxima and minima in the range.
Display these points as upward-pointing triangles.
Don't forget to make sure the x limits of the plot are still 0 and 10.

\emph{Helpful Hint}: Since you are working with arrays of discrete values, you will want to find the index values where your $x$ and $y$ values are closest to the actual maxima and minima. As you work, consider the following:
\begin{itemize}
\item How would you manually find maxima and minima of a function?
\item How could you do something similar with your $x$ and $y$ arrays?
\end{itemize}
Your plot should look like the figure below.

\begin{figure}[H]
\includegraphics[width=\textwidth]{soln3.pdf}
\label{fig:problem3}
\end{figure}
\end{problem}
\end{comment}

\subsection*{Heatmaps}
A function from $\mathbb{R}^2$ to $\mathbb{R}$ is usually plotted as a surface in $\mathbb{R}^3$.
A \emph{heatmap} visualizes such a function in only two dimensions by assigning the output of the function to a color (instead of a height).
For example, Figure \ref{fig:pcmexample} uses a heatmap to graph $f(x,y) = \sin(x)\sin(y)$ on $[-6,6] \times [-6,6]$.

The plot in Figure \ref{fig:pcmexample} was created with the function \li{plt.pcolormesh()}.
To draw Figure \ref{fig:pcmexample}, we must first create a grid of points at which to evaluate $f$.
We do this with the function \li{np.meshgrid()}, which is explained in Figure \ref{fig:meshgrid}.

\begin{figure}
\begin{tikzpicture}[>=stealth', shorten <= .1cm,shorten >=.1cm, dot/.style=
	{circle,fill=black,minimum size=3pt,inner sep=0pt, outer sep=-1pt} ]

\foreach \x/\y in {0/0, 0/2, 0/4, 2/0, 2/2, 2/4, 4/0, 4/2, 4/4}
	\node[draw, dot]at(\x,\y){};
\foreach \x/\y in {0/0, 0/1, 0/2, 1/0, 1/1, 1/2, 2/0, 2/1, 2/2}
	\node[draw=none]at(\x*2-.5, \y*2+.3){(\x,\y)};

\foreach \x/\y in {0/0, 0/1, 0/2, 1/0, 1/1, 1/2, 2/0, 2/1, 2/2}
	\node[draw=none]at(\x*.75+7, \y*.75+.1){\y};
\foreach \x/\y in {0/0, 0/1, 0/2, 1/0, 1/1, 1/2, 2/0, 2/1, 2/2}
	\node[draw=none]at(\x*.75+7, \y*-.75+3.9){\x};

\draw[-, thick](6.7,-.25)--(6.7,1.95);
\draw[-, thick](8.8,-.25)--(8.8,1.95);
\draw[-, thick](6.7,2.05)--(6.7,4.25);
\draw[-, thick](8.8,2.05)--(8.8,4.25);
\draw[-, thick](8.8,4.14)--(8.7,4.14);
\draw[-, thick](8.8,2.16)--(8.7,2.16);
\draw[-, thick](6.7,4.14)--(6.8,4.14);
\draw[-, thick](6.7,2.16)--(6.8,2.16);
\draw[-, thick](8.8,1.84)--(8.7,1.84);
\draw[-, thick](8.8,-.135)--(8.7,-.135);
\draw[-, thick](6.8,1.84)--(6.7,1.84);
\draw[-, thick](6.8,-.135)--(6.7,-.135);

\node[draw=none](X)at(6.3,.9){\texttt{Y}=};
\node[draw=none](y)at(6.3,3.15){\texttt{X}=};

\node[draw=none](point1)at(-.3, -.4){\texttt{x}=\big[0,};
\node[draw=none, node distance=2.35cm](point2)
	[right of=point1]{1,};
\node[draw=none, node distance=2cm](point3)
	[right of=point2]{2\big]};
\node[draw=none, rotate=270](point4)at(4.4,4.25)
	{\texttt{y}=\big[0,};
\node[draw=none, rotate=270, node distance=2.35cm](point5)
	[right of=point4]{1,};
\node[draw=none, rotate=270, node distance=2cm](point6)
	[right of=point5]{2\big]};

\end{tikzpicture}

\caption{This figure illustrates the function call \li{np.meshgrid(x, y)}, which returns the arrays \li{X} and \li{Y}.
The returned arrays give the $x$- and $y$-coordinates of the points in the grid formed by \li{x} and \li{y}.}
\label{fig:meshgrid}
\end{figure}

\begin{lstlisting}
x = np.linspace(-6, 6, 401)
y = np.linspace(-6, 6, 401)
X, Y = np.meshgrid(x, y)
\end{lstlisting}
The arrays $X$ and $Y$ satisfy \li{(X[i,j], Y[i,j]) = (x[i],y[j])}.

Now we can evaluate $f(x,y)$ at each point in the grid and plot the result.
\begin{lstlisting}
f = np.sin(X) * np.sin(Y)
plt.pcolormesh(X, Y, f)
plt.pcolorbar()    # Show scale
plt.show()
\end{lstlisting}
This plot is shown in Figure \ref{fig:pcmexample}
\begin{figure}
\includegraphics[width=.7\textwidth]{sinxsiny.png}
\caption{A heatmap of $f(x,y)=\sin\left(x\right)\times\sin\left(y\right)$ drawn by \li{plt.pcolormesh()}.}
\label{fig:pcmexample}
\end{figure}

\begin{comment}
\begin{problem} Use plt.pcolormesh to plot the absolute value of the function $x^3 +2x^2 -x +3$ on the complex plane with 0 $\leq$ \li{x} $\leq$ 2 and 0 $\leq$ \li{y} $\leq$ 2.
\emph{Helpful Hint}: First create your domain arrays, then convert these to a single array of complex variables to evaluate the function.

Your plot should look like Figure \ref{fig:pcolormesh}.

\begin{figure}[H]
\includegraphics[width=\textwidth]{pcolor2.png}
\caption{Another example of a colorplot.}
\label{fig:pcolormesh}
\end{figure}
\end{problem}
\end{comment}

\begin{problem}\label{prob:heatmap}
\leavevmode
\begin{enumerate}
\item Plot the function $f(x,y) = \sin(x)\sin(y)/(xy)$ on $[-2\pi,2\pi] \times [-2\pi,2\pi]$.
Include the scale bar in your plot.
\item Change the color scheme of your plot with the keyword argument \li{cmap='seismic'} in the call to \li{plt.pcolormesh()}.
You can see a list of all possible color schemes at \url{http://matplotlib.org/examples/color/colormaps_reference.html}.
\item Change the limits on the $x$- and $y$-axes so that the plot is only over the domain $[-2\pi,2\pi] \times [-2\pi,2\pi]$.
\item Fix the aspect ratio of your plot so that it is a square using the line \li{plt.gca().set_aspect('equal')}.
\end{enumerate}
Your finished plot should look like Figure \ref{fig:heatmapProb}.

\begin{figure}[H]
\includegraphics[width=.7\textwidth]{pcolor2.png}
\caption{Correct output for Problem \ref{prob:heatmap}.}
\label{fig:heatmapProb}
\end{figure}
\end{problem}

\begin{comment}

Matplotlib can also be used for 3D plotting. The following is an example
of how to use matplotlib to plot the function $z=\sin(x)\sin(y)$ with
both x and y ranging from -6 to 6. The resulting plot is shown in Figure
\ref{mpl:3dplot}. If you change the number of sample points used you
will notice that graphs look much nicer with large numbers of sample
points, but it will also take much longer for your computer to render
the image.

\begin{lstlisting} from mpl_toolkits.mplot3d import Axes3D from
matplotlib import pyplot as plt import numpy as np fig = plt.figure() ax
= fig.gca(projection='3d') x = np.linspace(-6, 6, 301) y =
np.linspace(-6, 6, 301) X, Y = np.meshgrid(x, y) Z = np.sin(X) *
np.sin(Y) ax.plot_surface(X, Y, Z) plt.show() \end{lstlisting}

\begin{figure} \includegraphics[width=\textwidth]{3dplot.pdf} \caption{A
3D plot of $\sin\left(x\right)\times\sin\left(y\right)$.}
\label{mpl:3dplot} \end{figure}

\begin{problem} Plot the function \begin{equation*}
\frac{\cos\left(\sqrt{x^2 + y^2}\right)}{\frac{x^2 + y^2}{10} + 1}
\end{equation*} on $[-10, 10] \times [-10, 10]$. \end{problem}

Matplotlib also allows us to make interactive graphs as follows:

% This example is largely based on one of the examples in the matplotlib
% docs. I have simplified it and changed the way the libraries are
% imported, but we could do a citation anyway.
%
\begin{lstlisting} import numpy as np from matplotlib import pyplot as
plt from matplotlib import widgets as wg ax = plt.subplot(111)
plt.subplots_adjust(bottom=.25) t = np.arange(0., 1., .001) a0 = 5. f0 =
3. s = a0 * np.sin(2 * np.pi * f0 * t) l = plt.plot(t, s)[0]
plt.axis([0, 1, -10, 10]) axfreq = plt.axes([.25, .05, .65, .03]) axamp
= plt.axes([.25, .1, .65, .03]) sfreq = wg.Slider(axfreq, 'Freq', .1,
30., valinit=f0) samp = wg.Slider(axamp, 'Amp', .1, 10., valinit=a0) def
update(val): amp = samp.val freq = sfreq.val l.set_ydata(amp * np.sin(2
* np.pi * freq * t)) plt.draw() sfreq.on_changed(update)
samp.on_changed(update) plt.show() \end{lstlisting} The resulting plot
is shown in Figure \ref{mpl:interact}.

\begin{figure} \includegraphics[width=\textwidth]{interact.pdf}
\caption{A snapshot of an interactive plot made using Matplotlib.}
\label{mpl:interact} \end{figure}

\begin{problem} Modify the code above to add a third slider to
manipulate the phase of the wave shown. Have it range from 0 to $2\pi$
and set the default value to zero. \end{problem}

\end{comment}

\subsection*{Other plots}
A \emph{histogram} is a way to visualize a 1-D data set, or list of values.
A histogram is created by dividing up the range of the values into a finite number of intervals, or \emph{bins}.
Then, the number of values in each bin is added up.
Graphically, these totals are represented by bars whose length is equal to the number of values in the bin.

For example, suppose we randomly choose 20 integers between 1 and 10.
This code creates a histogram depicting how many of each number we chose.
Its output is in Figure \ref{fig:histogram}.

\begin{lstlisting}
x = np.random.randint(1, 11, 20)
plt.hist(x, bins=10, range=[.5, 10.5])
plt.show()
\end{lstlisting}

In this example our data set \li{x} consists of only integer values.
We created 10 bins in the range [.5, 10.5] so that each bin contains exactly one integer.

The function \li{plt.hist()} also returns some arrays, the first of which is a list of the total number of values in each bin.

We could also use a \emph{scatter plot} to visualize the random integers \li{x} that we generated in the previous example.
The \li{matplotlib} function call \li{plt.scatter(x,y)} draws a scatter plot from two 1-D arrays \li{x} and \li{y} by plotting the points \li{(x[i], y[i])}.
As an example, the code below produces Figure \ref{fig:scatter}.

\begin{lstlisting}
t = np.linspace(1,20,20)
# The argument 's' specifies the marker size
plt.scatter(t, x, s=100)
plt.show()
\end{lstlisting}

\begin{figure}
\centering
\begin{subfigure}[t]{.49\textwidth}
\centering
\includegraphics[width=\textwidth]{histogram.pdf}
\caption{A histogram.}
\label{fig:histogram}
\end{subfigure}
\begin{subfigure}[t]{.49\textwidth}
\centering
\includegraphics[width=\textwidth]{scatter.pdf}
\caption{A scatter plot.}
\label{fig:scatter}
\end{subfigure}
\caption{A scatter plot produced with \li{plt.scatter()} and a histogram produced with \li{plt.hist()}.}
\label{fig:otherplots}
\end{figure}

\subsection*{Subplots}
\emph{Subplots} are nonoverlapping plots arranged in a grid within a single figure (see Figure \ref{fig:subplots}).
In \li{matplotlib}, specify which subplot you wish to modify with with command \li{plt.subplot(numrows, numcols, fignum)}.
Here, \li{numrows} is the number of rows of subplots in the figure, \li{numcols} is the number of columns of subplots in the figure, and \li{fignum} is the index of the subplot you wish to modify.
This index starts at 1 and increments across rows first.

The following code draws Figure \ref{fig:subplots}.
\begin{lstlisting}
x = np.linspace(-np.pi, np.pi, 400)
y1 = np.sin(x)
y2 = np.cos(x)

# Draw the first subplot
plt.subplot(2, 1, 1)
plt.plot(x, y1)

# Draw the second subplot
plt.subplot(2, 1, 2)
plt.plot(x, y2)

plt.show()
\end{lstlisting}

\begin{figure}
\includegraphics[width=.7\textwidth]{subplots.pdf}
\caption{The graphs of $\sin(x)$ and $\cos(x)$ as subplots in a single figure.}
\label{fig:subplots}
\end{figure}

\begin{problem}\label{prob:subplot}
Follow these steps to generate a plot similar to that in Figure \ref{fig:subplotProb}.
\begin{enumerate}
\item Generate 50 random numbers in the interval $[0,1)$ with the function \li{np.random.rand()}.
\item Create a plot with two subplots.
In the first subplot, draw a histogram of your data with 5 equally-sized bins.
\item In the second subplot,
\begin{enumerate}
\item Draw a scatter plot of your data.
Use the integers 1-50 for the $x$-coordinates and use your data for the $y$-coordinates.
\item Plot a red horizontal line whose height is equal to the mean of your data.
\end{enumerate}
\end{enumerate}

\begin{figure}[H]
\includegraphics[width=.7\textwidth]{subplotProb.pdf}
\caption{Correct output for Problem \ref{prob:subplot}.}
\label{fig:subplotProb}
\end{figure}

\end{problem}

\begin{comment}
\begin{problem}
Make a plot with 4 subplots.
In the subplots place graphs of $e^x$, $sin(x)$, $cos(x)$, and $x^2$.
Plot each graph over the interval $(-\pi,\pi)$.
Title each graph accordingly and title the entire figure ``My Different Plots."
\end{problem}
\end{comment}

\begin{comment}
%Table \ref{mpl:basics}
\begin{table}
\begin{center}
\begin{tabular}{|l|p{7cm}|p{3cm}|}

    \hline

    Function & Description & Usage\\

    \hline

    \li{bar} & makes a bar graph & bar(left,height)\\

    \li{barh} & makes a horizontal bar graph & barh(bottom,width)\\

    \li{fill} & plots lines with shading under the curve & fill(x,y)\\

    \li{fill\_between} & plots lines with shading between two given y
    values & fill\_ between(x,y1, y2=0)\\

    \li{hist} & plots a histogram from data & hist(data)\\

    \li{pie} & make a pie chart & pie(x)\\

    \li{plot} & plots lines and data on standard axes & plot(x,y)\\

    \li{polar} & plots lines and data on polar axes & polar(theta,r)\\

    \li{loglog} & plots lines and data on logarithmic x and y axes &
    loglog(x,y)\\

    \li{scatter} & plots data, has more options for scatter plots than
    the plot function & scatter(x,y)\\

    \li{semilogx} & plots lines and data with a log scaled x axis &
    semilogx(x,y)\\

    \li{semilogy} & plots lines and data with a log scaled y axis &
    semilogy(x,y)\\

    \li{specgram} & make a spectogram from data & specgram(x)\\

    \li{spy} & plot the sparsity pattern of a 2D array & spy(Z)\\

    \li{triplot} & plot triangulation between given points &
    triplot(x,y)\\

    \hline

    \end{tabular}
    \end{center}
    \caption{Some basic functions in Matplotlib.}
    \label{mpl:basics}
    \end{table}

    \end{comment}

\section*{3-D plotting with Mayavi (Optional)}

Although \li{matplotlib} is capable of creating 3-D plots, Mayavi does it better.
We will use Mayavi for all 3-D plots in these labs.
For information beyond what is found in this tutorial, see \url{http://docs.enthought.com/mayavi/mayavi/}.
To get started, import \li{mlab} from \li{mayavi}.
\begin{lstlisting}
from mayavi import mlab
\end{lstlisting}

The module \li{mlab} has many plotting functions.
We will introduce a few in this lab; for more see \url{http://docs.enthought.com/mayavi/mayavi/auto/mlab_helper_functions.html}.
You can also browse examples at \url{http://docs.enthought.com/mayavi/mayavi/auto/examples.html}.

\begin{comment}
\begin{table}
\begin{center}
\begin{tabular}
{|c|l|}
\hline
Function & Description \\
\hline
\li{barchart} & Produces 3D histogram-like plots\\
\li{contour3d} & Plots level surfaces of functions of three variables\\
\li{flow} & Creates a trajectory of particles following the flow of a vector field\\
\li{imshow} & Use a colormap to view a 2D array as an image\\
\li{mesh} & Plot a surface using \li{(x,y,z)} coordinates supplied as three 2D arrays\\
\li{plot3d} & Draws lines between points\\
\li{points3d} & Plots glyphs (like points) at the coordinates supplied\\
\li{quiver3d} & Generate 3D vector fields\\
\li{surf} & Plot a surface with a 2D array as elevation data\\
\hline
\end{tabular}
\end{center}
\caption{Some plotting functions in \li{mlab}.}
\label{table:mlab_functions}
\end{table}

All these functions can be ``tested" (which provides an example figure for the function) with the command:
\begin{lstlisting}
mlab.test_<plotting function>()
mlab.show()
\end{lstlisting}
Note that pressing tab after typing \li{mlab.test_} will provide a list of available commands and that, like matplotlib, we use the \li{show()} command to actually view our plot.

For example, to test the \li{fancy_mesh} function, we run the following:
\begin{lstlisting}
mlab.test_fancy_mesh()
mlab.show()
\end{lstlisting}

The result should resemble Figure \ref{fig:fancymesh}.

\begin{figure}
\includegraphics[width=\textwidth]{fancymesh.png}
\caption{An example of the fancy mesh plotting function}
\label{fig:fancymesh}
\end{figure}

In this lab, we will focus our attention on introducing the \li{mesh}, \li{plot3d}, and \li{points3d} functions.

Like matplotlib, \li{mlab} takes a set of data points and plots it.
However, instead of passing just \li{x} and \li{y} coordinates as we did in matplotlib, we also send \li{z} coordinates to produce a 3D plot.

\end{comment}

\subsection*{Lines}
The function call \li{mlab.plot3d(x,y,z)} plots the points \li{(x[i], y[i], z[i])} and connects them with straight lines.
The code below produces the flower in Figure \ref{fig:plot3d}

\begin{lstlisting}
num = np.pi/1000
pts = np.arange(0, 2*np.pi + num, num)
x = np.cos(pts) * (1 + np.cos(pts*6))
y = np.sin(pts) * (1 + np.cos(pts*6))
z = np.sin(pts*6/11)
mlab.plot3d(x, y, z)
mlab.show()
\end{lstlisting}

\begin{figure}
\includegraphics[width=.7\textwidth]{plot3d.png}
\caption{Sample output of \li{mlab.plot3d()}.}
\label{fig:plot3d}
\end{figure}

\subsection*{Points}
The function call \li{mlab.points3d(x,y,z)} plots the points \li{(x[i], y[i], z[i])}, but does not connect them.
In the code below, the optional input array \li{s} defines a scalar for each point that modifies the color and size of the point.
The output is in Figure \ref{fig:points3d}.

\begin{lstlisting}
pts = np.linspace(0, 4 * np.pi, 30)
x = np.sin(2 * pts)
y = np.cos(pts)
z = np.cos(2 * pts)
s = 2+np.sin(pts)
# Adjust the keyword argument 'scale_factor' so all points are visible
mlab.points3d(x, y, z, s, scale_factor=.15)
mlab.show()
\end{lstlisting}

\begin{figure}
\includegraphics[width=.7\textwidth]{points3d.png}
\caption{Sample output of \li{mlab.points3d()}.}
\label{fig:points3d}
\end{figure}

\subsection*{Surfaces}
You can draw a surface in mayavi with \li{mlab.surf()}.
This function accepts three arrays, \li{X}, \li{Y}, and \li{Z}, where \li{X} and \li{Y} determine a grid of points similar to the output of \li{np.meshgrid()}.
The array \li{Z} gives the height of the surface at each of these points.

The output of \li{np.meshgrid()} is the transpose of what \li{mlab.surf()} expects.
To avoid confusion, use the function \li{np.mgrid()} instead.
This function uses the slicing syntax \li{[start:stop:step]}
The code below produces the hyperbolic paraboloid in Figure \ref{fig:surf_example}.

\begin{lstlisting}
X, Y = np.mgrid[-4:4:0.025, -4:4:0.025]
Z = X**2/4-Y**2/4
mlab.surf(X, Y, Z, colormap='RdYlGn')
mlab.show()
\end{lstlisting}

\begin{figure}
\includegraphics[width=.7\textwidth]{mesh_example.png}
\caption{Sample output of \li{mlab.surf()}.}
\label{fig:surf_example}
\end{figure}

The plotting functions in Mayavi allow you to specify the color of a plot, either as a solid color or with a varying colormap.
For example, the plot in Figure \ref{fig:surf_example} uses the colormap \li{'RdYlGn'}.
For a list of all colormaps in Mayavi, see \url{http://docs.enthought.com/mayavi/mayavi/mlab_changing_object_looks.html}.

%TODO: this problem is copied from https://github.com/enthought/mayavi/blob/master/examples/mayavi/mlab/canyon.py.
% we either need to cite this (and thus give the solution) or find a different problem.
\begin{comment}
\begin{problem}\label{prob:grand_canyon}
Provided Grand Canyon topological radar data from NASA, do the following.

\begin{itemize}
\item Reshape the data to be 3601x3601.
\item Cast the data type as \li{float32}.
\item Slice the data, taking the first 1000 rows and columns 900-1900.
\item There is some missing data, so set the minimum of your data equal to the minimum of the the positive data points.
\item Preset the figure using the following commands: \li{mlab.figure(size=(400,320)}, \li{bgcolor = (.16, .28, .46))}
\item Now plot with \li{mlab.surf}, using the colormap \li{gist_earth}, with a \li{warp_scale=.2}, \li{vmin=1200}, and \li{vmax=1610}.
\item Take a smaller view of the canyon using \li{mlab.view(-5.9, 83, 570, [5.3, 20, 238])}.
\end{itemize}

This should produce Figure \ref{fig:GrandCanyon}

\begin{figure}[H]
\includegraphics[width=.7\textwidth]{GrandCanyon.png}
\caption{Correct output for Problem \ref{prob:grand_canyon}.}
\label{fig:GrandCanyon}
\end{figure}

\end{problem}
\end{comment}


\chapter{Conclusion}

\section{Solution Transfer}

This work has described a method for conservative interpolation
between curved meshes. The transfer process conserves globally
to machine precision since we can use exact quadratures for all
integrals. The primary source of error comes from solving the linear
system with the mass matrix for the target mesh. This allows
less restrictive usage of mesh adaptivity, which can make computations
more efficient. Additionally, having a global transfer algorithm
allows for remeshing to be done less frequently.

The algorithm breaks down into three core subproblems: B\'{e}zier triangle
intersection, an advancing front for intersecting elements and
integration on curved polygons. The inherently local nature of the
advancing front allows the algorithm to be parallelized via domain
decomposition with little data shared between processes. By
restricting integration to the intersection of elements from the
target and donor meshes, the algorithm can accurately transfer
both continuous and discontinuous fields.

\subsection{Future Work}

As mentioned in the preceding chapters, there are several research
directions possible to build upon the solution transfer algorithm.
The usage of Green's theorem nicely extends to \(\reals^3\) via
Stoke's theorem, but the B\'{e}zier triangle intersection algorithm
is specific to \(\reals^2\). The equivalent B\'{e}zier tetrahedron
intersection algorithm is significantly more challenging.

The restriction to shape functions from the global coordinates basis
is a symptom of the method and not of the inherent problem. The
pre-image basis has several appealing properties, for example
this basis can be precomputed on \(\utri\). The problem of a
valid tessellation of a curved polygon warrants more exploration.
Such a tessellation algorithm would enable usage of the pre-image
basis.

The usage of the global coordinates basis does have some benefits.
In particular, the product of shape functions from different meshes
is still a polynomial in \(\reals^2\). This means that we could
compute the coefficients of \(F = \phi_0 \phi_1\) directly and
use them to evaluate the antiderivatives \(H\) and \(V\) rather
than using the fundamental theorem of calculus. Even if this
did not save any computation, it may still be preferred over
the FTC approach because it would remove the usage of quadrature
points outside of the domain \(\mathcal{P}\).

\section{Ill-conditioned B\'{e}zier Curve Intersection}

The B\'{e}zier triangle intersection subproblem can be solved
partially by intersecting B\'{e}zier curves (i.e. the edges).
However, the probability of ``almost tangent'' or ill-conditioned
B\'{e}zier curve intersections increases to unity as the mesh
size \(h\) goes to \(0\).

Turning our focus to ill-conditioned B\'{e}zier curve intersections,
we designed a modified Newton's method that computed the residual
as if in extended precision. Building on existing work, we describe
a compensated de Casteljau algorithm that allows the evaluation of
a polynomial in Bernstein form as accurate as if the computations
were done in \(K\)-times the working precision. The residual
\(F(s, t) = b_0(s) - b_1(t)\) depends on four evaluations of
a polynomial in Bernstein form, so \(F\) can be computed more precisely
by using the compensated evaluation algorithm.


\section{Integration by Parts}{}{}\label{sec:Parts}
%We have already seen that recognizing the product rule can be useful,
%when we noticed that
%$$\int \sec^3u+\sec u \tan^2u\,du=\sec u \tan u.$$
%As with substitution, we do not have to rely on insight or cleverness
%to discover such antiderivatives; there is a technique that will often
%help to uncover the product rule.
%
%Start with the product rule:
%$${d\over dx}f(x)g(x)=f'(x)g(x)+f(x)g'(x).$$
%We can rewrite this as
%$$f(x)g(x)=\int f'(x)g(x)\,dx +\int f(x)g'(x)\,dx,$$
%and then
%$$\int f(x)g'(x)\,dx=f(x)g(x)-\int f'(x)g(x)\,dx.$$
%This may not seem particularly useful at first glance, but it turns
%out that in many cases we have an integral of the form
%$$\int f(x)g'(x)\,dx$$
%but that
%$$\int f'(x)g(x)\,dx$$
%is easier. This technique for turning one integral into another is
%called {\dfont integration by parts},
%and is usually written in more compact form. If we let $u=f(x)$ and
%$v=g(x)$ then $du=f'(x)\,dx$ and $dv=g'(x)\,dx$ and
%$$\int u\,dv = uv-\int v\,du.$$
%To use this technique we need to identify likely candidates for
%$u=f(x)$ and $dv=g'(x)\,dx$.
%
%\begin{example}{Product of a Linear Function and Logarithm}{Product of a Linear Function and Logarithm}\label{Product of a Linear Function and Logarithm}
%Evaluate $\ds\int x\ln x\,dx$.
%\end{example}
%
%\begin{solution}
%Let $u=\ln x$ so $du=1/x\,dx$. Then
%we must let $dv=x\,dx$ so $\ds v=x^2/2$ and
%$$
% \int x\ln x\,dx={x^2\ln x\over 2}-\int {x^2\over2}{1\over x}\,dx=
% {x^2\ln x\over 2}-\int {x\over2}\,dx={x^2\ln x\over 2}-{x^2\over4}+C.
%$$\vskip-10pt
%\end{solution}
%
%\begin{example}{Product of a Linear Function and Trigonometric Function}{Product of a Linear Function and Trigonometric Function}\label{Product of a Linear Function and Trigonometric Function}
%Evaluate $\ds\int x\sin x\,dx$.
%\end{example}
%
%\begin{solution}
%Let $u=x$ so $du=dx$. Then
%we must let $dv=\sin x\,dx$ so $v=-\cos x$ and
%$$\int x\sin x\,dx=-x\cos x-\int -\cos x\,dx=
%-x\cos x+\int \cos x\,dx=-x\cos x+\sin x+C.$$\vskip-10pt
%\end{solution}

Here's a simple integral that we can't yet evaluate:
$$\int x\cos x \,dx.$$
It's a simple matter to take the derivative of the integrand using the Product Rule, but there is no Product Rule for integrals.  However, this section introduces \textit{Integration by Parts}, a method of integration that is based on the Product Rule for derivatives. It will enable us to evaluate this integral.

The Product Rule says that if $u$ and $v$ are functions of $x$, then  $(uv)' = u\primeskip'v + uv\primeskip'$.  For simplicity, we've written $u$ for $u(x)$ and $v$ for $v(x)$.  Suppose we integrate both sides with respect to $x$.  This gives
$$\int (uv)'\,dx = \int (u\primeskip'v+uv\primeskip')\,dx.$$
By the Fundamental Theorem of Calculus, the left side integrates to $uv$.  The right side can be broken up into two integrals, and we have
$$\int (uv)'\,dx = \int u\primeskip'v\,dx + \int uv\primeskip'\,dx.$$
rearranging we get
$$\int uv\primeskip'\,dx = \int (uv)'\,dx - \int u\primeskip'v\,dx.$$
which gives
$$\int uv\primeskip'\,dx = uv - \int u\primeskip'v\,dx.$$
Using differential notation, we can write $du = u\primeskip'(x)dx$ and $dv=v\primeskip'(x)dx$ and the expression above can be written as follows:
$$\int u\,dv = uv - \int v\,du.$$
This is the Integration by Parts formula. For reference purposes, we state this in a theorem.

\begin{theorem}{Integration by Parts}{IBP}
{Let $u$ and $v$ be differentiable functions of $x$ on an interval $I$ containing $a$ and $b$. Then
	$$\int u\ dv = uv - \int v\ du,$$ and \index{integration!by parts}
	$$\int_{x=a}^{x=b} u\ dv = uv\Big|_a^b - \int_{x=a}^{x=b}v\ du.$$
}
\end{theorem}

Let's try an example to understand our new technique.\\

\begin{example}{Integrating using Integration by Parts}{ex_ibp1}
	{
	Evaluate $\ds\int x\cos{x}\ dx$.}
\end{example}

\begin{solution}
{The key to Integration by Parts is to identify part of the integrand as ``$u$'' and part as ``$dv$.'' Regular practice will help one make good identifications, and later we will introduce some principles that help. For now, let  $u=x$ and $dv=\cos{x}\ dx$.

It is generally useful to make a small table of these values as done below. Right now we only know $u$ and $dv$ as shown on the left of Figure \ref{fig:ibp1}; on the right we fill in the rest of what we need. If $u = x$, then $du = dx$. Since $dv = \cos x\ dx$, $v$ is an antiderivative of $\cos x$. We choose $v = \sin x$.\\

	\noindent\begin{minipage}{\textwidth}
		\noindent\begin{minipage}[t]{.45\textwidth}
			%\centering
			\vskip-10pt
			\begin{align*}
			u&= x & v&=\text{?}\\
			du&= \text{?} & dv&=\cos x\ dx
			\end{align*}
		\end{minipage}\begin{minipage}[t]{.1\textwidth}\centering\vskip15pt$\Rightarrow$\end{minipage}
		\begin{minipage}[t]{.45\textwidth}
			\vskip-10pt
			\begin{align*}
			u&= x & v&=\sin x\\
			du&= dx & dv&=\cos x\ dx
			\end{align*}
		\end{minipage}
		\captionsetup{type=figure}%
		\caption{Setting up Integration by Parts.}\label{fig:ibp1}
	\end{minipage}\\
	\vskip\baselineskip

	%  On the right side of the formula we can see that we need $du$ and $v$.  We get $du$ by taking the derivative of $u$, and we get $du=(1)\,dx$, or simply $du=dx$.  We get $v$ by finding an antiderivative of $dv$.  Here we get $v=\sin x $.
	Now substitute all of this into the Integration by Parts formula, giving
	$$\int x\cos x\,dx = x\sin x - \int \sin x \,dx.$$
	We can then integrate $\sin x$ to get $-\cos x + C$ and overall our answer is
	$$\int x\cos x\ dx = x\sin x + \cos x + C.$$
	Note how the antiderivative contains a product, $x\sin x$. This product is what makes Integration by Parts necessary.
}
\end{solution}

The example above demonstrates how Integration by Parts works in general.  We try to identify $u$ and $dv$ in the integral we are given, and the key is that we usually want to choose $u$ and $dv$ so that $du$ is simpler than $u$ and $v$ is hopefully not too much more complicated than $dv$.  This will mean that the integral on the right side of the Integration by Parts formula, $\int v\,du$ will be simpler to integrate than the original integral $\int u\,dv$.

In the example above, we chose $u=x$ and $dv=\cos x\,dx$.  Then $du=dx$ was simpler than $u$ and $v=\sin x$ is no more complicated than $dv$.  Therefore, instead of integrating $x\cos x \,dx$, we could integrate $\sin x\,dx$, which we knew how to do.

A useful mnemonic for helping to determine $u$ is ``LIPET,'' where
\begin{center}L = \textbf{L}ogarithmic, I = \textbf{I}nverse Trig., P = \textbf{P}olynomial (algebraic), E = \textbf{E}xponential, and T = \textbf{T}rigonometric.
\end{center}

If the integrand contains both a logarithmic and an polynomial term, in general letting $u$ be the logarithmic term works best, as indicated by L coming before P in LIPET.

Note: Some texts us ``LIATE,'' where A = \textbf{A}lgebraic. This method works just as well as LIPET.

%A useful mnemonic for helping to determine $u$ is ``LIATE,'' where
%\begin{center}L = \textbf{L}ogarithmic, I = \textbf{I}nverse Trig., A = \textbf{A}lgebraic (polynomials),
%
%	T = \textbf{T}rigonometric, and E = \textbf{E}xponential.
%\end{center}
%
%If the integrand contains both a logarithmic and an algebraic term, in general letting $u$ be the logarithmic term works best, as indicated by L coming before A in LIATE.

We now consider another example.\\

\begin{example}{Integrating using Integration by Parts}{ex_ibp2}
	{
	Evaluate $\displaystyle \int x e^x\,dx$.}
\end{example}

\begin{solution}
{Using the LIPET rule, we see that the integrand contains a  \textbf{P}olynomial term ($x$) and an \textbf{E}xponential term ($e^x$). Our mnemonic suggests letting $u$ be the polynomial term, so we choose $u=x$ and $dv=e^x\,dx$.  Then $du=dx$ and $v=e^x$ as indicated by the tables below.\\

	\noindent\begin{minipage}{\textwidth}
		\noindent\begin{minipage}[t]{.45\textwidth}
			%\centering
			\vskip-10pt
			\begin{align*}
			u&= x & v&=\text{?}\\
			du&= \text{?} & dv&=e^x\ dx
			\end{align*}
		\end{minipage}\begin{minipage}[t]{.1\textwidth}\centering\vskip15pt$\Rightarrow$\end{minipage}
		\begin{minipage}[t]{.45\textwidth}
			\vskip-10pt
			\begin{align*}
			u&= x & v&=e^x\\
			du&= dx & dv&=e^x\ dx
			\end{align*}
		\end{minipage}
		\captionsetup{type=figure}%
		\caption{Setting up Integration by Parts.}\label{fig:ibp2}
	\end{minipage}\\
	\vskip\baselineskip

	We see $du$ is simpler than $u$, while there is no change in going from $dv$ to $v$.  This is good.  The Integration by Parts formula gives
	$$\int x e^x\,dx = xe^x - \int e^x\,dx.$$
	The integral on the right is simple; our final answer is
	$$\int xe^x\ dx = xe^x - e^x + C.$$
	Note again how the antiderivatives contain a product term.
}
\end{solution}

\begin{example}{Product of a Linear Function and Logarithm}{Product of a Linear Function and Logarithm}\label{Product of a Linear Function and Logarithm}
Evaluate $\ds\int x\ln x\,dx$.
\end{example}

\begin{solution}
Using the LIPET rule, we let $u=\ln x$ so $du=1/x\,dx$. Then
we must let $dv=x\,dx$ so $\ds v=x^2/2$ and
$$
 \int x\ln x\,dx={x^2\ln x\over 2}-\int {x^2\over2}{1\over x}\,dx=
 {x^2\ln x\over 2}-\int {x\over2}\,dx={x^2\ln x\over 2}-{x^2\over4}+C.
$$\vskip-10pt
\end{solution}

\begin{example}{Secant Cubed (again)}{Secant Cubed (again)}\label{Secant Cubed (again)}
Evaluate $\ds\int\sec^3 x\,dx$.
\end{example}

\begin{solution}
Of course we already know the answer
to this, but we needed to be clever to discover it. Here we'll use the
new technique to discover the antiderivative.
Let $u=\sec x$ and $\ds dv=\sec^2 x\,dx$. Then $du=\sec x\tan x$ and
$v=\tan x$ and
\begin{eqnarray*}
  \int\sec^3 x\,dx&=&\sec x\tan x-\int \tan^2x\sec x\,dx\cr
  &=&\sec x\tan x-\int (\sec^2x-1)\sec x\,dx\cr
  &=&\sec x\tan x-\int \sec^3x\,dx +\int\sec x\,dx.\cr
\end{eqnarray*}
At first this looks useless---we're right back to
$\ds \int\sec^3x\,dx$. But looking more closely:
\begin{eqnarray*}
  \int\sec^3x\,dx&=&\sec x\tan x-\int \sec^3x\,dx +\int\sec x\,dx\cr
  \int\sec^3x\,dx+\int \sec^3x\,dx&=&\sec x\tan x +\int\sec x\,dx\cr
  2\int\sec^3x\,dx&=&\sec x\tan x +\int\sec x\,dx\cr
  \int\sec^3x\,dx&=&{\sec x\tan x\over2} +{1\over2}\int\sec x\,dx\cr
  &=&{\sec x\tan x\over2} +{\ln|\sec x+\tan x|\over2}+C.
\end{eqnarray*}\vskip-10pt
\end{solution}

\subsection*{Tabular Method}

\begin{example}{Product of a Polynomial and Trigonometric Function}{Product of a Polynomial and Trigonometric Function}\label{Product of a Polynomial and Trigonometric Function}
Evaluate $\ds\int x^2\sin x\,dx$.
\end{example}

\begin{solution}
Let $u=x^2$, $dv=\sin x\,dx$; then $du=2x\,dx$ and $v=-\cos x$.
Now $$\ds \int x^2\sin x\,dx=-x^2\cos x+\int 2x\cos x\,dx.$$
This is
better than the original integral, but we need to do integration by
parts again. Let $u=2x$, $dv=\cos x\,dx$; then
$du=2$ and $v=\sin x$, and
\begin{eqnarray*}
  \int x^2\sin x\,dx&=&-x^2\cos x+\int 2x\cos x\,dx\cr
  &=&-x^2\cos x+ 2x\sin x - \int 2\sin x\,dx\cr
  &=&-x^2\cos x+ 2x\sin x + 2\cos x + C.
\end{eqnarray*}\vskip-10pt
\end{solution}

Such repeated use of integration by parts is fairly common, but it can
be a bit tedious to accomplish, and it is easy to make
errors, especially sign errors involving the subtraction in the
formula. There is a nice tabular method to accomplish the calculation
that minimizes the chance for error and speeds up the whole
process. We illustrate with the previous example. Here is the
table:
$$\begin{array}{|c|c|c|}
\hline
\mbox{sign}& u& dv\\\hline
\mbox{+}& x^2& \sin x\\\hline
\mbox{-}& 2x& -\cos x\\\hline
\mbox{+}& 2& -\sin x\\\hline
\mbox{-}& 0& \cos x\\\hline
\end{array}$$
To form this table, we start with $u$ at the top of the second
column and repeatedly compute the derivative; starting with $dv$ at
the top of the third column, we repeatedly compute the
antiderivative. In the first column, we place a ``$-$'' in every
second row. To form the
second table we combine the first and second columns by
ignoring the boundary; if you do this by hand, you may simply start
with two columns and add a ``$-$'' to every second row.

Alternatively, we can use the following table:
$$\begin{array}{|c|c|}
\hline
u& dv\\\hline
x^2& \sin x\\\hline
-2x& -\cos x\\\hline
2& -\sin x\\\hline
0& \cos x\\\hline
\end{array}$$
To compute with this second table we begin at the top. Multiply the
first entry in column $u$ by the second entry in column $dv$ to get
$\ds -x^2\cos x$, and add this to the integral of the product of the
second entry in column $u$ and second entry in column $dv$.  This
gives:
$$-x^2\cos x+\int 2x\cos x\,dx,$$
or exactly the result of the first application of integration by
parts.  Since this integral is not yet easy, we return to the table.
Now we multiply twice on the diagonal, $\ds (x^2)(-\cos x)$ and
$(-2x)(-\sin x)$ and then once straight across, $(2)(-\sin x)$, and
combine these as
$$-x^2\cos x+2x\sin x-\int 2\sin x\,dx,$$
giving the same result as the second application of integration by
parts. While this integral is easy, we may return yet once more to the
table. Now multiply three times on the diagonal to get $\ds
(x^2)(-\cos x)$, $(-2x)(-\sin x)$, and $(2)(\cos x)$, and once
straight across, $(0)(\cos x)$. We combine these as before to get
$$
  -x^2\cos x+2x\sin x +2\cos x+\int 0\,dx=
  -x^2\cos x+2x\sin x +2\cos x+C.
$$
Typically we would fill in the table one line at a time, until the
``straight across'' multiplication gives an easy integral. If we can
see that the $u$ column will eventually become zero, we can instead
fill in the whole table; computing the products as indicated will then
give the entire integral, including the ``$+C\,$'', as above.

\begin{example}{}{}
Do Example \ref{exa:ex_ibp2} again, using the tabular method.
\end{example}

\begin{solution}
Recognising that $ x $	has a derivative that vanishes, and $ e^x $ is easy to repeatedly integrate, we construct the table with $ u=x:$
$$\begin{array}{|c|c|}
\hline
u& dv\\\hline
x& e^x\\\hline
-1& e^x\\\hline
+0& e^x\\\hline
\end{array}$$
This gives
$$\int xe^x\ dx = xe^x - e^x + C.$$
which agrees with the result in Example \ref{exa:ex_ibp2}.
\end{solution}

\subsection*{Some Classic Examples of IBP}

\begin{example}{Solving for the unknown integral}{ex_ibp4}
	{
	Evaluate $\displaystyle \int e^x\cos x \,dx$.}
\end{example}

\begin{solution}
{This is a classic problem.  Our mnemonic (LIPET) suggests letting $u$ be the exponential factor, sowe choose $u=e^x$ and hence $dv = \cos x\,dx$.  Then $du=e^x\,dx$ and $v=\sin x$ as shown below.\\
	\noindent\begin{minipage}{\textwidth}
		\noindent\begin{minipage}[t]{.45\textwidth}
			%\centering
			\vskip-10pt
			\begin{align*}
			u&= e^x & v&=\text{?}\\
			du&= \text{?} & dv&=\cos x\ dx
			\end{align*}
		\end{minipage}\begin{minipage}[t]{.1\textwidth}\centering\vskip15pt$\Rightarrow$\end{minipage}
		\begin{minipage}[t]{.45\textwidth}
			\vskip-10pt
			\begin{align*}
			u&= e^x& v&=\sin x\\
			du&= e^x\ dx & dv&=\cos x\ dx
			\end{align*}
		\end{minipage}
		\captionsetup{type=figure}%
		\caption{Setting up Integration by Parts.}\label{fig:ibp4}
	\end{minipage}\\
	\vskip\baselineskip
	Notice that $du$ is no simpler than $u$, going against our general rule (but bear with us). The Integration by Parts formula yields
	$$\int e^x\cos x\ dx = e^x\sin x - \int e^x\sin x\,dx.$$
	The integral on the right is not much different than the one we started with, so it seems like we have gotten nowhere. Let's  keep working and apply Integration by Parts to the new integral, using $u=e^x$ and $dv = \sin x\,dx$. This leads us to the following:\\ %Then we get $du=e^x\,dx$ and $v=-\cos x$.

	\noindent\begin{minipage}{\textwidth}
		\noindent\begin{minipage}[t]{.45\textwidth}
			%\centering
			\vskip-10pt
			\begin{align*}
			u&= e^x & v&=\text{?}\\
			du&= \text{?} & dv&=\sin x\ dx
			\end{align*}
		\end{minipage}\begin{minipage}[t]{.1\textwidth}\centering\vskip15pt$\Rightarrow$\end{minipage}
		\begin{minipage}[t]{.45\textwidth}
			\vskip-10pt
			\begin{align*}
			u&= e^x& v&=-\cos x\\
			du&= e^x\ dx & dv&=\sin x\ dx
			\end{align*}
		\end{minipage}
		\captionsetup{type=figure}%
		\caption{Setting up Integration by Parts (again).}\label{fig:ibp4a}
	\end{minipage}\\
	\vskip\baselineskip

	The Integration by Parts formula then gives:
	\begin{align*}
	\int e^x\cos x\,dx &= e^x\sin x - \left(-e^x\cos x - \int -e^x\cos x\,dx\right)\\
	&= e^x\sin x+ e^x\cos x - \int e^x\cos x\ dx.
	\end{align*}
	It seems we are back right where we started, as the right hand side contains $\int e^x\cos x\,dx$.  But this is actually a good thing.

	Add $\ds\int e^x\cos x\ dx$ to both sides. This gives
	\begin{align*}
	2\int e^x\cos x\ dx & = e^x\sin x + e^x\cos x \\
	\intertext{Now divide both sides by 2:}
	\int e^x\cos x\ dx & = \frac{1}{2}\big(e^x\sin x + e^x\cos x\big).
	\end{align*}

	Simplifying a little and adding the constant of integration, our answer is thus
	$$\int e^x\cos x\ dx = \frac12e^x\left(\sin x + \cos x\right)+C.$$
	\vskip-15pt
}
\end{solution}

\begin{example}{IBP: the antiderivative of $\ln x$}{ex_ibp5}
{Evaluate $\displaystyle \int \ln x\,dx$.}
\end{example}

\begin{solution}
{One may have noticed that we have rules for integrating the familiar trigonometric functions and $e^x$, but we have not yet given a rule for integrating $\ln x$.  That is because $\ln x$ can't easily be integrated with any of the rules we have learned up to this point.  But we can find its antiderivative by a clever application of Integration by Parts.  Set $u=\ln x$ and $dv=dx$.  This is a good, sneaky trick to learn as it can help in other situations. This determines $du=(1/x)\,dx$ and $v=x$ as shown below.\\

	\noindent\begin{minipage}{\textwidth}
		\noindent\begin{minipage}[t]{.45\textwidth}
			%\centering
			\vskip-10pt
			\begin{align*}
			u&= \ln x & v&=\text{?}\\
			du&= \text{?} & dv&=dx
			\end{align*}
		\end{minipage}\begin{minipage}[t]{.1\textwidth}\centering\vskip15pt$\Rightarrow$\end{minipage}
		\begin{minipage}[t]{.45\textwidth}
			\vskip-10pt
			\begin{align*}
			u&= \ln x& v&=x\\
			du&= 1/x\ dx & dv&=dx
			\end{align*}
		\end{minipage}
		\captionsetup{type=figure}%
		\caption{Setting up Integration by Parts.}\label{fig:ibp5}
	\end{minipage}\\
	\vskip\baselineskip
	Putting this all together in the Integration by Parts formula, things work out very nicely:
	$$\int \ln x\,dx = x\ln x - \int x\,\frac1x\,dx.$$
	The new integral simplifies to $\int 1\,dx$, which is about as simple as things get.  Its integral is $x+C$ and our answer is
	$$\int \ln x\ dx = x\ln{x} - x + C.$$
}\\
\end{solution}

\begin{example}{Integrating using Int. by Parts: antiderivative of $\arctan x$}{ex_ibp6}
{Evaluate $\displaystyle \int \arctan x  \,dx$.}
\end{example}

\begin{solution}
{The same sneaky trick we used above works here.  Let $u=\arctan x$ and $dv=dx$.  Then $du=1/(1+x^2)\,dx$ and $v=x$.  The Integration by Parts formula gives
	$$\int \arctan x \,dx = x\arctan x - \int \frac x{1+x^2}\,dx.$$
	The integral on the right can be solved by substitution.  Taking $u=1+x^2$, we get $du=2x\,dx$.  The integral then becomes
	$$\int \arctan x \,dx = x\arctan x - \frac12\int \frac 1{u}\,du.$$
	The integral on the right evaluates to $\ln|u|+C$, which becomes $\ln(1+x^2)+C$.  Therefore, the answer is
	$$\int \arctan x\ dx = x\arctan x - \ln(1+x^2) + C.$$
}\\
\end{solution}

\subsection*{ Substitution Before Integration}

When taking derivatives, it was common to employ multiple rules (such as using both the Quotient and the Chain Rules). It should then come as no surprise that some integrals are best evaluated by combining integration techniques. In particular, here we illustrate making an ``unusual'' substitution first before using Integration by Parts.\\

\begin{example}{Integration by Parts after substitution}{ex_ibp8}
	{
	Evaluate $\ds \int \cos(\ln x)\ dx$.}
\end{example}

\begin{solution}
{The integrand contains a composition of functions, leading us to think Substitution would be beneficial. Letting $u=\ln x$, we have $du = 1/x\ dx$. This seems problematic, as we do not have a $1/x$ in the integrand. But consider:
	$$du = \frac 1x\ dx \Rightarrow x\cdot du = dx.$$
	Since $u = \ln x$, we can use inverse functions and conclude that $x = e^u$. Therefore we have that
	\begin{align*}
	dx &= x\cdot du \\
	&= e^u\ du.
	\end{align*}
	We can thus replace $\ln x$ with $u$ and $dx$ with $e^u\ du$. Thus we rewrite our integral as
	$$\int \cos(\ln x)\ dx = \int e^u\cos u \ du.$$
	We evaluated this integral in Example \ref{exa:ex_ibp4}. Using the result there, we have:
	\begin{align*}
	\int \cos(\ln x)\ dx &= \int e^u\cos u \ du \\
	&= \frac12e^u\big(\sin u + \cos u\big) + C \\
	&= \frac12e^{\ln x} \big(\sin(\ln x) + \cos (\ln x)\big)+C\\
	&= \frac12x \big(\sin(\ln x) + \cos (\ln x)\big)+C.
	\end{align*}
	\vskip-\baselineskip
}
\end{solution}

\subsection*{Definite Integrals and Integration By Parts}

So far we have focused only on evaluating indefinite integrals. Of course, we can use Integration by Parts to evaluate definite integrals as well, as Theorem \ref{thm:IBP} states. We do so in the next example.\\

\begin{example}{Definite integration using Integration by Parts}{ex_ibp7}
	{
	Evaluate $\displaystyle \int_1^2 x^2 \ln x \,dx$.}
\end{example}

\begin{solution}
{%Once again, our mnemonic suggests we let $u=\ln x$.  %(We could let $u = x^2$ and $dv = \ln x\ dx$, as we now know the antiderivatives of $\ln x$. However, letting $u = \ln x$ makes our next integral much simpler as it removes the logarithm from the integral entirely.)
	Our mnemonic suggests letting $u=\ln x$, hence $dv =x^2\,dx$.
	%So we have $u=\ln x$ and $dv=x^2\,dx$.
	We then get $du = (1/x)\,dx$ and $v=x^3/3$ as shown below.\\

	\noindent\begin{minipage}{\textwidth}
		\noindent\begin{minipage}[t]{.45\textwidth}
			%\centering
			\vskip-10pt
			\begin{align*}
			u&= \ln x & v&=\text{?}\\
			du&= \text{?} & dv&=x^2\ dx
			\end{align*}
		\end{minipage}\begin{minipage}[t]{.1\textwidth}\centering\vskip15pt$\Rightarrow$\end{minipage}
		\begin{minipage}[t]{.45\textwidth}
			\vskip-10pt
			\begin{align*}
			u&= \ln x& v&=x^3/3\\
			du&= 1/x\ dx & dv&=x^2\ dx
			\end{align*}
		\end{minipage}
		\captionsetup{type=figure}%
		\caption{Setting up Integration by Parts.}\label{fig:ibp7}
	\end{minipage}\\
	\vskip\baselineskip

	The Integration by Parts formula then gives
	\begin{align*}
	\int_1^2 x^2 \ln x\,dx &= \frac{x^3}3\ln x\bigg|_1^2 - \int_1^2 \frac{x^3}{3}\,\frac 1x\,dx \\
	&=  \frac{x^3}3\ln x\bigg|_1^2 - \int_1^2 \frac{x^2}{3}\,dx \\
	&=  \frac{x^3}3\ln x\bigg|_1^2 - \frac{x^3}{9}\bigg|_1^2\\
	&=  \left(\frac{x^3}3\ln x - \frac{x^3}{9}\right)\bigg|_1^2\\
	&=	\left(\frac83\ln 2 - \frac89\right)-\left(\frac13\ln 1 - \frac19\right) \\
	&= \frac83\ln 2 - \frac79 \\
	&\approx 1.07.
	\end{align*}
	\vskip-15pt
}
\end{solution}

In general, Integration by Parts is useful for integrating certain products of functions, like $\int x e^x\,dx$ or $\int x^3\sin x\,dx$.   It is also useful for integrals involving logarithms and inverse trigonometric functions.

As stated before, integration is generally more difficult than derivation. We are developing tools for handling a large array of integrals, and experience will tell us when one tool is preferable/necessary over another. For instance, consider the three similar--looking integrals
$$\int xe^x\,dx, \qquad  \int x e^{x^2}\,dx \qquad \text{and} \qquad \int xe^{x^3}\,dx.$$

While the first is calculated easily with Integration by Parts, the second is best approached with Substitution.  Taking things one step further, the third integral has no answer in terms of elementary functions, so none of the methods we learn in calculus will get us the exact answer.

Integration by Parts is a very useful method, second only to substitution. In the following sections of this chapter, we continue to learn other integration techniques.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\Opensolutionfile{solutions}[ex]
\section*{Exercises for \ref{sec:Parts}}

\begin{enumialphparenastyle}

Find the antiderivatives.

%%%%%%%%%%
\begin{ex}
 $\ds\int x\cos x\,dx$
\begin{sol}
 $\cos x+x\sin x+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int x^2\cos x\,dx$
\begin{sol}
 $\ds x^2\sin x-2 \sin x+2x\cos x +C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int xe^x\,dx$
\begin{sol}
 $\ds (x-1)e^x +C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int xe^{x^2}\,dx$
\begin{sol}
 $\ds (1/2)e^{x^2} +C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int \sin^2 x\,dx$
\begin{sol}
 $(x/2)-\sin(2x)/4 +C=$\hfill\break$(x/2)-(\sin x\cos x)/2+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int \ln x\,dx$
\begin{sol}
 $x\ln x-x +C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int x\arctan x\,dx$
\begin{sol}
 $\ds (x^2\arctan x +\arctan x -x)/2+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int x^3\sin x\,dx$
\begin{sol}
 $\ds -x^3\cos x+3x^2\sin x+6x\cos x-6\sin x+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int x^3\cos x\,dx$
\begin{sol}
 $\ds x^3\sin x+3x^2\cos x-6x\sin x-6\cos x+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int x\sin^2 x\,dx$
\begin{sol}
 $\ds x^2/4-(\cos^2 x)/4-(x\sin x\cos x)/2+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int x\sin x\cos x\,dx$
\begin{sol}
 $\ds x/4-(x\cos^2 x)/2+(\cos x\sin x)/4+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int \arctan(\sqrt x)\,dx$
\begin{sol}
 $x\arctan(\sqrt x)+\arctan(\sqrt x)-\sqrt{x}+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int \sin(\sqrt x)\,dx$
\begin{sol}
 $2\sin(\sqrt x)-2\sqrt x\cos(\sqrt x)+C$
\end{sol}
\end{ex}
%%%%%%%%%%

%%%%%%%%%%
\begin{ex}
 $\ds\int\sec^2 x\csc^2 x\,dx$
\begin{sol}
 $\sec x\csc x-2\cot x+C$
\end{sol}
\end{ex}
%%%%%%%%%%

\end{enumialphparenastyle}


% -*- coding: utf-8 -*-
\section{Theorem}
% --------------------------------------------------- Slide --
\subsection{Theorem Code}
\label{theoremCode}
\begin{frame}{Theorem}
  There is also a group of blocks that are especially useful for presenting mathematics. For example the ‘theorem’ environment, the ‘corollary’ environment and the ‘proof’ environment.
  \begin{semiverbatim}
    \\begin\{theorem\}[Pythagoras] \newline
      $ a^2 + b^2 = c^2$ \newline
    \\end\{theorem\} \newline
    \\begin\{corollary\} \newline
      $ x + y = y + x  $ \newline
    \\end\{corollary\} \newline
    \\begin\{proof\} \newline
      $\omega +\phi = \epsilon $ \newline
    \\end\{proof\}
  \end{semiverbatim}
\end{frame}

% --------------------------------------------------- Slide --
\subsection{Theorem Blocks}
\label{theoremBlocks}
\begin{frame}{Theorem Blocks}
  \begin{theorem}[Pythagoras]
    $ a^2 + b^2 = c^2$
  \end{theorem}
  \begin{corollary}
    $ x + y = y + x  $
  \end{corollary}
  \begin{proof}
    $\omega +\phi = \epsilon $
  \end{proof}
  As definition, you can also use the custom one ``thm/cor'' which is defined in ``slides/usrdef.tex''.
\end{frame}