\documentclass[twoside]{MATH77}
\usepackage{multicol}
\usepackage[fleqn,reqno,centertags]{amsmath}
\begin{document}
\begmath 17.2 Computation Using Partial Derivative Arrays or
\hbox{Multivariate Taylor Series}

\silentfootnote{$^\copyright$1997 Calif. Inst. of Technology, \thisyear \ Math \`a la Carte, Inc.}

\subsection{Purpose}

This set of subroutines performs computations in which each variable is
represented by its value, and its first and second partial derivatives with
respect to N independent variables, evaluated at a specified value of these
variables. Such a set of numbers can alternatively be regarded as a scaled
representation of the low order coefficients of the multivariate Taylor
series of the dependent variable expanded at the evaluation point.

More specifically let $u({\bf x})$ denote a scalar valued function of an
N-component argument vector, ${\bf x}$. Let $u_0$, ${\bf g}_0$, and $H_0$ denote,
respectively, the value of $u$, the gradient vector, and the Hessian matrix,
evaluated at ${\bf x}_0$. Then $\{u_0$, ${\bf g}_0$, $H_0\}$ is the set of data these
programs carry to represent $u$ evaluated at ${\bf x}_0$. The Taylor series for $u$
through second order, expanded at ${\bf x}_0$, is
\begin{equation*}
u({\bf x}) = u_0 + {\bf g}^t_0({\bf x}-{\bf x}_0) + \frac{1}{2}({\bf
x}-{\bf x}_0)^t H_0({\bf x}-{\bf x}_0)
\end{equation*}
This package provides a way of computing values of the first and second
partial derivatives of a multivariate function that is defined by a sequence
of computational steps involving arithmetic and elementary functions,
without the need to derive and code expressions for the partial derivatives.

\subsection{Usage}

\subparagraph{Definition of a U-variable}

We shall use ${\bf x}$ as the generic name of the N-dimensional independent
variable vector with respect to which all partial derivatives are defined.
The term U-variable is used to denote a sequence of values, consisting of a
function value, values of the function's gradient vector, and optionally the
(packed) Hessian matrix, evaluated at some point. We assume the Hessian
matrix is symmetric, so only one copy of each symmetric pair of elements
needs to be stored. We store the lower triangle of the Hessian matrix by
rows. Equivalently, one could regard this as representing the upper triangle
by columns. Thus if U() is an array containing a U-variable, $\{u$, ${\bf
g}$, $H\}$, its contents are defined as follows:
\begin{alignat*}{2}
\text{U}(1) &=u\\
\text{U}(1+j) &= g_j &= &\:\partial u/\partial x_j,\ j = 1,\ ...,N\\
\hspace{-15pt}\text{U}(1+\text{N}+j+i(i-1)/2)&=\: h_{i,j}\: &= &\:
\partial ^2u/\partial x_i\partial x_j,\\
&&&\hspace{-.5in}i = 1,\ ..., N;\ j = 1,\ ...,\ i.
\end{alignat*}

The required dimension size for a U-variable is $dimu$ = N + 1 if only first
partial derivatives will be requested, and $dimu$ = (N +2)(N + 1) / 2 if both
first and second partial derivatives are to be computed. For convenience we
list the value of $dimu$ {\bf for the second derivative case} for some values
of N:

\begin{tabular}{@{}r*{15}{@{\ }r}}
N = & 1 & 2 & 3 & 4 & 5 & 6 & 7 & 8 & 9 & 10 & 11 & 12 & 13 & 14 & 15\\
$dimu$ = & 3 & 6 & 10 & 15 & 21 & 28 & 36 & 45 & 55 & 66 & 78 & 91 & 105 &
120 & 136
\end{tabular}

\subparagraph{Distinguishing property of an independent variable}

For $i$ in [1, N] the scalar component, $x_i$, of the independent variable
vector, ${\bf x}$, is distinguished by the unique property that all of its first
and second partial derivatives are zero except its first partial derivative
with respect to $x_i$, which has the value~1.

\subparagraph{The variables N, M1, and M2.}

N specifies the number of independent
variables, and implicitly specifies the storage layout of components within
a U-variable.  M1 and M2 specify the lowest and highest order of partial
derivatives which the called subroutines are to produce.  A program that
uses these subroutines must initially call SUSETN (or DUSETN)
to set N, M1, and M2.

\subparagraph{The subroutines}

The subroutines of this package are described in the following sections:

\begin{description}
\item[B.1]  SUSETN, Assigning values to N, M1, and M2.

\item[B.2]  SUGETN, Fetching values of N, M1, M2, L1, and L2.

\item[B.3]  SUSET, Assigning a value to a U-variable.

\item[B.4]  The computational subroutines, except SUREV.

\item[B.5]  SUREV, series reversion, or function inversion.

\item[B.6]  Modifications for double-precision.
\end{description}

\subsubsection{SUSETN, Assigning values to N, M1, and M2}

Subroutine SUSETN must be called prior to calling any of the other
subroutines of this package.

\paragraph{Program Prototype, Single Precision}
\begin{description}
\item[INTEGER]  \ {\bf N, M1, M2}
\end{description}

Assign values to N, M1, and M2.
$$
\fbox{{\bf CALL SUSETN(N, M1, M2)}}
$$

\paragraph{Argument Definitions}

\begin{description}

\item[N]  \ [in] N specifies the number of independent
variables, and implicitly specifies the storage layout of components within
a U-variable.

\item[M1, M2]  \ [in] M1 and M2 specify the lowest and highest order of partial derivatives
which the called subroutines are to produce.  M1 and M2 must satisfy:
 0 $\leq $ M1 $\leq $ M2 $\leq 2.$

Most commonly one will probably
choose to set (M1, M2) = (0,0), (0,1), or (0,2), and leave the setting
unchanged throughout a computation. See Section C for discussion of other
strategies for setting (M1,M2).
\end{description}

\subsubsection{SUGETN, Fetching values of N, M1, M2, L1, and L2}

\paragraph{Program Prototype, Single Precision}
\begin{description}
\item[INTEGER]  \ {\bf N, M1, M2, L1, L2}
$$
\fbox{{\bf CALL SUGETN(N, M1, M2, L1, L2)}}
$$
Values are returned in N, M1, M2, L1, and L2.
\end{description}

\paragraph{Argument Definitions}

\begin{description}
\item[N, M1, M2]  \ [out] Returns the values that were set on the
previous call to SUSETN.

\item[L1, L2]  \ [out] Returns values computed from N, M1, and M2.
L1 and L2 are the indices of the first and last locations in a
U-variable array that will be subject to change due to the settings
of N, M1, and M2.\vspace{5pt}
\begin{gather*}
L1 = \begin{cases}
1 & \text{if} ~ \text{M1} = 0 \\
2 & \text{if} ~ \text{M1} = 1 \\
N+2 & \text{if} ~ \text{M1} = 2
\end{cases}\\
L2 = \begin{cases}
1 & \text{if} ~ \text{M1} = 0 \\
N+1 & \text{if} ~ \text{M1} = 1 \\
1+N+((N*(N+1))/2) & \text{if} ~ \text{M1} = 2
\end{cases}
\end{gather*}
\end{description}

\subsubsection{SUSET, Assigning a value to a U-variable}

The integer values N, M1, and M2 must be set by a call to SUSETN
before calling SUSET.  SUSET will only set partial derivatives of orders
M1 through M2.

\paragraph{Program Prototype, Single Precision}
\begin{description}
\item[INTEGER]  \ {\bf KEY}

\item[REAL]  \ {\bf VAL, U}($\geq dimu)$\newline
[$dimu = \text{N}+1 \text{ or (N}+2)(\text{N}+1)/2$, see above.]

\end{description}

Assign values to N, M1, M2, VAL, and KEY.
$$
\fbox{{\bf CALL SUSET(VAL, KEY, U)}}
$$
Computed quantities are returned in U().

\paragraph{Argument Definitions}

\begin{description}
\item[VAL]  \ [in] Value to be assigned to the U-variable, U().

\item[KEY]  \ [in] Integer in the range, [0, N]. If KEY = 0, U() is set to
represent a variable that is constant relative to the N independent
variables, $i.e.$, all of its first and second partial derivatives are set
to zero.

If 1 $\leq $ KEY $\leq $ N, U() is set to represent the $\text{KEY}^{th}$
independent variable, $i.e.$, its $\text{KEY}^{th}$ first partial derivative is set
to~1.0. All of its other first partial derivatives and all of its second
partial derivatives are set to zero.

\item[U()]  \ [out] Array in which this subroutine will define a U-variable
having the value, VAL, and having partial derivative values as specified by
KEY. Only derivative values of orders M1 through M2 are set.
\end{description}

\subsubsection{The computational subroutines, except SUREV}

The integer values N, M1, and M2 must be set by a call to SUSETN
before calling any of these subroutines.  These subroutines will only set
partial derivatives of orders M1 through M2.  To use these subroutines
with M1 $>$ 0 read the remarks about M1 $>$ 0 in Section C.

In describing the following subroutines, U() and V() denote input
U-variables and Z() denotes an output U-variable. The variables A and I are
input variables that are constant relative to ${\bf x}.$

In most of these subroutines the output array Z() must occupy storage
locations distinct from any of the input data. Exceptions to this rule are
SUSUM, SUDIF, SUPRO, SUSUM1, SUDIF1, and SUPRO1. The subroutine SUQUO, which
computes $u/v \rightarrow z$, permits $u$ and $z$ to occupy the same
storage, but $v$ and $z$ must occupy distinct storage.

\paragraph{Program Prototype, Single Precision}

\begin{description}
\item[INTEGER]  \ {\bf I}

\item[REAL]  \ {\bf A, U}($\geq dimu)${\bf ,V}($\geq dimu)${\bf , Z}($\geq
dimu)$\\
{[$dimu = \text{N}+1 \text{ or (N}+2)(\text{N}+1)/2$, see page~1.]}
\end{description}

Assign values to I, A, U(), and V(), as appropriate.

\begin{center}
{\bf Two-argument operations with both arguments depending on ${\bf x}.$}

\fbox{%
\begin{tabular}{@{\bf \ }lr}
CALL\ SUSUM(U,\ V,\ Z) & $u+v \rightarrow z$\\
CALL\ SUDIF(U,\ V,\ Z) & $u-v \rightarrow z$\\
CALL\ SUPRO(U,\ V,\ Z) & $u\times v \rightarrow z$\\
CALL\ SUQUO(U,\ V,\ Z) & $u/v \rightarrow z$\\
CALL\ SUATN2(U,\ V,\ Z) & atan2($u,v) \rightarrow z$\\
\multicolumn{2}{c}{where for atan2: $-\pi <z \leq \pi \text{ and } \tan (z)
= u/v$}\\
\end{tabular}}
\end{center}

\pagebreak
\begin{center}
{\bf Two-argument operations with only one argument depending on $u.$}

\fbox{%
\begin{tabular}{@{\bf \ }lr}
CALL SUSUM1(A, V, Z) & $a+v \rightarrow z$\\
CALL SUDIF1(A, V, Z) & $a-v \rightarrow z$\\
CALL SUPRO1(A, V, Z) & $a\times v \rightarrow z$\\
CALL SUQUO1(A, V, Z) & $a/v \rightarrow z$\\
CALL SUPWRI(I, V, Z) & $v^i \rightarrow z$\\
\multicolumn{2}{c}{(See following note.)}\\
\end{tabular}}
\end{center}

Note: I may be positive, negative, or zero. If I = 0, SUPWRI sets Z(1) = 1.0
and all derivative values of $z$ to~0.0, regardless of the given value of $v$%
. It is an error to have $v = 0.0$ when I $<$ 0.

\begin{center}
{\bf One-argument operations with the argument depending on ${\bf x}.$}

\fbox{%
\begin{tabular}{@{\bf \ }lr}
CALL SUSQRT(U, Z) & $\sqrt u \rightarrow z$\\
CALL SUEXP(U, Z) & $\exp (u) \rightarrow z$\\
CALL SULOG(U, Z) & $\log (u) \rightarrow z$\\
CALL SUSIN(U, Z) & $\sin (u) \rightarrow z$\\
CALL SUCOS(U, Z) & $\cos (u) \rightarrow z$\\
CALL SUTAN(U, Z) & $\tan (u) \rightarrow z$\\
CALL SUASIN(U, Z) & asin$(u) \rightarrow z$\\
CALL SUACOS(U, Z) & acos$(u) \rightarrow z$\\
CALL SUATAN(U, Z) & atan$(u) \rightarrow z$\\
CALL SUSINH(U, Z) & sinh$(u) \rightarrow z$\\
CALL SUCOSH(U, Z) & cosh$(u) \rightarrow z$\\
CALL SUTANH(U, Z) & tanh$(u) \rightarrow z$\\
\end{tabular}}
\end{center}

\paragraph{Argument Definitions}

\begin{description}
\item[A]  \ [in] Floating point value that is independent of ${\bf x}.$

\item[I]  \ [in] Integer value that is independent of ${\bf x}.$

\item[U()]  \ [in] An input U-variable. Must contain defined values for
derivatives of orders~0 through M2.

\item[V()]  \ [in] An input U-variable. Must contain defined values for
derivatives of orders~0 through M2.

\item[Z()]  \ [inout] If M1 $>$ 0, this array must contain defined values of
derivatives of orders~0 through M1 $-$ 1 on entry. On return it will also contain
computed values of derivatives of orders M1 through M2. If M1 = 0, no input
values are required in Z().
\end{description}

\subsubsection{SUREV, Series Reversion or Function Inversion}

Let $u_i$, $i = 1$, ..., $n$, be $n$ functions of an $n$-vector, ${\bf t}$.
Suppose values of the $u_i$'s and their first and second partial
derivatives with respect to the components of ${\bf t}$ are known for a particular
value of ${\bf t}$. Then, if the Jacobian matrix of the transformation is
nonsingular at this point, one can regard the $t_j$'s as functions
of the $u_i$'s in some neighborhood of this point in $u$-space. In
this situation this subroutine can compute values of the first and second
partial derivatives of the $t_j$'s with respect to the $%
u_i$'s at this point.

Given:
\begin{gather}
\hspace{-15pt}t_j \text{ in TU}(1,\ j),\ j = 1,\ ...,\ n,\\
\hspace{-15pt}u_i \text{ in UT}(1,\ i),\ i = 1,\ ...,\ n,\\
\hspace{-15pt}\partial u_i/\partial t_j\text{ in UT}(1+j,\ i),\ j = 1,\
...,\ n;\ i = 1,\ ...\ n,\hspace{-1in}\\
\hspace{-15pt}\partial ^2u_i/\partial t_j\partial t_k \text{ in UT}
(1+n+k+j(j-1)/2),\ i),\\
\hspace{.5in}j= 1,\ ...,\ n;\ k = 1,\ ...,\ j;\ i = 1,\ ...,\ n.
\notag
\end{gather}
If the Jacobian matrix with elements $\partial u_i/\partial t_j$ is
nonsingular, this subroutine will compute the first and second partial
derivatives of the $t_j$'s with respect to the $u_i$'s and
store them as
\begin{gather}
\hspace{-15pt}\text{TU}(1+i, j) = \partial t_j/\partial u_i,\ i =
1,\ ...,\ n;\ j = 1,\ ...,\ n,\hspace{-1in}\\
\hspace{-15pt}\text{TU}(1+n+k+i(i-1)/2,\ j) = \partial ^2t_j/\partial
u_i\partial u_k,\\
\hspace{.5in}i =1,\ ...,\ n;\ k = 1,\ ...,\ i;\ j = 1,\ ...,\ n,\notag
\end{gather}
The integer values N, M1, and M2 must be set by a call to SUSETN
before calling SUREV.  N gives
the value of $n$ for Eqs.(1--6) above.
Require 0 $\leq $ M1 $\leq $ M2 $\leq $ 2.

If M2 = 0, SUREV returns with no action.

If M2 = 1, SUREV only computes Eq.(5), and it is not necessary to provide
the data of Eq.(4).

If M2 = 2 and M1 = 0 or~1, SUREV computes Eqs.(5--6).

If M2 = 2 and M1 = 2, it is assumed that the assignment of Eq.(5) has
already been done, and thus SUREV only computes Eq.(6).

\paragraph{Program Prototype, Single Precision}

\begin{description}
\item[INTEGER]  \ {\bf IWORK}(N){\bf , LDIM}

\item[REAL]  \ {\bf RCOND, TU}(LDIM, $\geq $N){\bf , UT}(LDIM, $\geq $N){\bf %
, WORK}($\geq 3\times \text{N}^2$) \ [LDIM $\geq dimu = \text{N}+1 \text{ or}\\
(\text{N}+2)(\{text{N}+1)/2$, see page 1.]
\end{description}

Assign values to LDIM, UT() and part of TU().

\begin{center}
\fbox{%
\begin{tabular}{@{\bf }c}
CALL SUREV( UT, TU, LDIM,\\
RCOND, IWORK, WORK)\\
\end{tabular}}
\end{center}

Computed quantities are returned in RCOND and TU().

\paragraph{Argument Definitions}

\begin{description}
\item[UT(,)]  \ [in] Must contain the data of Eqs.(2--4) on entry, except as
noted above for certain values of M1 and M2.

\item[TU(,)]  \ [inout] Must contain the data of Eq.(1) on entry, and will
have the data of Eqs.(5--6) assigned by SUREV, except as noted above for
particular values of M1 and M2.

\item[LDIM]  \ [in] Leading dimension for the arrays UT(,) and TU(,).
Require LDIM $\geq dimu$, where $dimu  = \text{N}+1 \text{ or }
(\text{N}+2)(\text{N}+1)/2$, see
page~1.

\item[RCOND]  \ [out] Estimate of the reciprocal condition number of the
Jacobian matrix with elements $\partial u_i/\partial t_j$, given in UT().
Only computed when M1 $\leq $ 1 and M2 $\geq 1$. Is in the range [0.0,~1.0].

A well conditioned problem has values near~1.0. Near zero means badly
conditioned. Equal to zero means singular, in which case the subroutine
cannot complete its computation.

\item[IWORK()]  \ [scratch] Integer work space of size at least N.

\item[WORK()]  \ [scratch] Floating-point work space of size at least $%
3\times \text{N}^2.$
\end{description}

\subsubsection{Modifications for Double Precision}

For double-precision usage, replace the initial S in the name of each
subroutine by D, and replace the REAL declarations by DOUBLE PRECISION.

\subsection{Examples and Remarks}

\subsubsection{Example}

As a demonstration problem, consider the following formulae for transforming
a set of 3-dimensional rectangular coordinates $(x,y,z)$ to spherical
coordinates $(r,\varphi ,\theta )$ and the inverse transformation formulae:
\begin{align*}
s &= \sqrt{x^2 + y^2}\\
r &= \sqrt{s^2 + z^2}\\
\varphi &= \atan2(y,\ x)\\
\theta &= \tan^{-1}(z/s)\\
x &= r\ \cos \ \varphi \ \cos \ \theta \\
y &= r\ \sin \ \varphi \ \cos \ \theta \\
z &= r\ \sin \ \theta
\end{align*}
The demonstration program, DRSUCOMP, performs this mapping from $(x,y,z)$ to
$(r,\varphi ,\theta )$ with computation of all of the first and second partial
derivatives of $r$, $\varphi $, and $\theta $ with respect to $x$, $y$, and $z$%
. As a check on the computation the program then transforms back to $(x,y,z)$%
. Results are shown in ODSUCOMP. Note that the final $(x,y,z)$ agrees with
the initial $(x,y,z)$ in the values and the first and second partial
derivatives.

To demonstrate series reversion (or function inversion) the program assigns
the values of $(r,\varphi ,\theta )$ and its first and second partial
derivatives with respect to $(x,y,z)$ to the array UT(,), and then assigns the
values of $(x,y,z)$ to TU(1,~1:3). It then uses SUREV to compute the first
and second partial derivatives of $(x,y,z)$ with respect to $(r,\varphi ,\theta
)$, storing these in TU(2:10,~1:3). This computation is checked by computing
the same quantities in a different way.

\subsubsection{Omitting derivative computation}

To save time when developing new code using this package, one may run with
(M1, M2) = (0,~0) until one is satisfied that the function evaluation is as
desired, and then increase M2 to activate the derivative computation.

\subsubsection{Setting M1 $>$ 0}

There are some algorithms, such as for optimization, in which one does not
need to compute partial derivatives at every point at which the function is
evaluated. Depending on how significant efficiency is in a particular
application, one may wish to consider methods of separating the function and
derivative computation using this package. One can compute only the function
value by setting (M1, M2) = (0,~0), or one can compute the function value
and first partial derivatives by setting (M1, M2) = (0,~1).

If one has computed the function value at an argument value, ${\bf x}$, with (M1,
M2) = (0,~0), and has kept all intermediate quantities in distinct storage
locations, then one can repeat the sequence of subroutine calls with (M1,
M2) = (1,~1) to compute the first partial derivatives at the same point,
${\bf x}$%
, without the function value being recomputed. In general, computation with
M1 $>$ 0 is only valid if all $in$ and $inout$ arrays in each subroutine call
contain values of all derivatives of orders~0 through M1 $-$ 1 computed
previously with the same ${\bf x}.$

\subsection{Functional Description}

This U-computation package is based on the ideas presented in
\cite{Wengert:1964:ASA}.  See the description of W-computation in
Chapter~17.1 for a summary of these ideas.  Whereas the W-computation
package generalizes the order of differentiation, this U-computation
package generalizes the number of independent variables.

Let $f()$ be a scalar-valued function of the scalar variable $u$, and use
primes to denote differentiation with respect to $u$. Let $u$ depend on the
N-vector ${\bf x}$ and denote partial derivatives of $u$ with respect to $x_i$ by
appropriate subscripts. This package computes first and second partial
derivatives of $f()$ using the formulae:
\begin{align*}
\partial f(u)/\partial x_i &= f^{\prime}(u) u_i\\
\partial ^2f(u)/\partial x_i\partial x_j &= f^{\prime}(u) u_{i,j} +
f^{\prime\prime}(u) u_i u_j
\end{align*}

\nocite{Griewank:1991:ADA}
\nocite{Lawson:1971:CDU}
\bibliography{math77}
\bibliographystyle{math77}

\subsection{Error Procedures and Restrictions}

N must be positive. M1 and M2 must satisfy 0\ $\leq $ M1
$\leq $ M2 $\leq $ 2. See the discussion in Section~C for cautions regarding
usage with M1 $>$ 0. These conditions on N, M1, and M2 are not tested by
the subroutines and their violation may cause unpredictable actions.

\subsubsection{Invalid arguments for derivative computation}

The user will likely be accustomed to avoiding sending invalid arguments to
the elementary functions, such as a negative argument to the square root. In
computing derivatives there are some additional singularities to avoid. Note
that the derivative is infinite at zero for the square root, and at $\pm $1
for arcsin and arccosine.

\subsubsection{Error handling}

Following is a list of error conditions the package detects and for which error
messages are issued. These errors are fatal in the sense that the requested
operation cannot be done; however, the default action is to return after
issuing an error message. The user can use the MATH77 library subroutine,
ERMSET of Chapter~19.2, to alter this action to cause a STOP if desired. Error conditions
not on this list, $e.g.$, negative argument in log, will be handled by the
usual host system error handler.

\begin{tabular}{@{}l@{\ \ }l}
\bf Error No.\\
\bf \& Program & \multicolumn{1}{c}{\bf Explanation}\\
\bf 1 SUASIN & Infinite derivative when arg = $-$1\ or +1\\
\bf 1 SUACOS & Infinite derivative when arg = $-1$\ or +1\\
\bf 2 SUSQRT & Infinite derivative when arg = 0\\
\bf 3 SUQUO1 & Zero divisor\\
\bf 4 SUPWRI & $\text{U}^{\text{M}}$ is infinite when U = 0 and M $<$ 0\\
\bf 5 SUQUO & Zero divisor\\
\bf 6 SUREV & Singular Jacobian matrix
\end{tabular}

\subsection{Supporting Information}

The source language is ANSI Fortran~77. All program units reference
COMMON blocks /UCOM1/ and /UCOM2/.

All of the double precision entry points except DUREV require files:

DUCOMP, ERFIN, ERMSG, IERM1, and IERV1.

All of the single precision entry points except SUREV require files:

SUCOMP, ERFIN, ERMSG, IERM1, and IERV1.

DUREV requires: DASUM, DAXPY, DDOT, DGECO, DGEFA, DGEI, DSCAL, DSWAP,
DUREV, ERFIN, ERMSG, and IDAMAX.

SUREV requires: SASUM, SAXPY, SDOT, SGECO, SGEFA, SGEI, SSCAL, SSWAP,
SUREV, ERFIN, ERMSG, and ISAMAX.\vspace{-5pt}
\begin{center}
\begin{tabular}{llll}
\multicolumn{4}{c}{\bf Entries}\\
DUACOS & DUASIN & DUATAN & DUATN2\\
DUCOS & DUCOSH & DUDIF & DUDIF1\\
DUEXP & DUGETN & DULOG & DUPRO\\
DUPRO1 & DUPWRI & DUQUO & DUQUO1\\
DUREV & DUSET & DUSETN & DUSIN\\
DUSINH & DUSQRT & DUSUM & DUSUM1\\
DUTAN & DUTANH & SUACOS & SUASIN\\
SUATAN & SUATN2 & SUCOS & SUCOSH\\
SUDIF & SUDIF1 & SUEXP & SUGETN\\
SULOG & SUPRO & SUPRO1 & SUPWRI\\
SUQUO & SUQUO1 & SUREV & SUSET\\
SUSETN & SUSIN & SUSINH & SUSQRT\\
SUSUM & SUSUM1 & SUTAN & SUTANH\\
\end{tabular}
\end{center}\vspace{-5pt}
Designed by C. L. Lawson, JPL, 1971. Adapted to Fortran~77 for the JPL
MATH77 library, Aug.~1987. Added SUREV/DUREV, February~1992.  Added
SUSETN, DUSETN, SUGETN, and DUGETN August~1994.

%\rule{0pt}{80pt}

\begcodenp
\lstset{language=[77]Fortran,showstringspaces=false}
\lstset{xleftmargin=.8in}

\centerline{\bf \large DRSUCOMP}\vspace{10pt}
\lstinputlisting{\codeloc{sucomp}}
\newpage

\vspace{30pt}\centerline{\bf \large ODSUCOMP}\vspace{10pt}
\lstset{language={}}
\lstinputlisting{\outputloc{sucomp}}
\end{document}


% !TEX root = Main.tex
\section{Optimization}

\subsection*{Alternating Least Squares}
$f(U,v_i)=\sum_{(i,j)\in I} (a_{i,j} - \langle u_j, v_i \rangle)^2$\\
$f(u_i,V)=\sum_{(i,j)\in I} (a_{i,j} - \langle u_j, v_i \rangle)^2$\\
Least squares problems which are convex.

\subsection*{Coordinate Descent}
1. init: $\mathbf{x}^{(0)} \in \mathbb{R}^D$\\
2. for $t = 0 \ \text{to} \ \mathit{maxIter}$:\\
3. sample $d \in_{u.a.r.} \{1, \ldots, D\}$\\
4. $u^\star = \argmin_{u \in \mathbb{R}} f(x_1^{(t)}, .., x_{d-1}^{(t)}, u, x_{d+1}^{(t)}, .., x_D^{(t)})$\\
5. $\mathbf{x}_d^{(t+1)} = u^\star$ and $\mathbf{x}_i^{(t+1)} = \mathbf{x}_i^{(t)}$ for $i \neq d$

\subsection*{Gradient Descent (or Deepest Descent)}
\textbf{Gradient}: $\nabla f(\mathbf{x}) := \left( \frac{\partial f(\mathbf{x})}{\partial \mathbf{x}_1}, \ldots, \frac{\partial f(\mathbf{x})}{\partial \mathbf{x}_D} \right)^\top$

1. init: $\mathbf{x}^{(0)} \in \mathbb{R}^D$\\
2. for $t = 0 \ \text{to} \ \mathit{maxIter}$:\\
3. $\mathbf{x}^{(t+1)} = \mathbf{x}^{(t)} - \gamma \nabla f(\mathbf{x}^{(t)})$, usually $\gamma \approx \frac{1}{t}$

\subsection*{Stochastic Gradient Descent (SGD)}
Assume \textbf{Additive Objective}:\\
$f(x) = \frac{1}{N}\sum_{n=1}^{N}f_n(x)$\\
1. init: $\mathbf{x}^{(0)} \in \mathbb{R}^D$\\
2. for $t = 0 \ \text{to} \ \mathit{maxIter}$:\\
3. sample $n \in_{u.a.r.} \{1, \ldots, N\}$\\
4. $\mathbf{x}^{(t+1)} = \mathbf{x}^{(t)} - \gamma \nabla f_n(\mathbf{x}^{(t)})$, typically  $\gamma \approx \frac{1}{t}$.

\subsection*{Projected Gradient Descent (Constrained Opt.)}
minimize $f(x)$, $x \in Q$ (constraint).\\
\textbf{Project} $x$ onto $Q$: $P_Q(\mathbf{x}) = \argmin_{y \in Q} \|\mathbf{y} - \mathbf{x}\|$,\\
\textbf{Update}: $\mathbf{x}^{(t+1)} = P_Q[\mathbf{x}^{(t)} - \gamma \nabla f(\mathbf{x}^{(t)})]$,\\
$\mathbf{x}^{(t+1)}$ is unique if $Q$ convex.

\subsection*{Lagrangian Multipliers}
Minimize  $f(\mathbf{x})$ s.t. $g_i(\mathbf{x}) \leq 0,\ i = 1, .., m$ (\textbf{inequality constr.}) and $h_i(\mathbf{x}) = \mathbf{a}_i^\top \mathbf{x} - b_i = 0,\ i = 1, .., p$ (\textbf{equality constraint})
\begin{compactdesc}
	\item[Lagrangian:] $L(\mathbf{x}, \boldsymbol{\lambda}, \boldsymbol{\nu}) := f(\mathbf{x}) + \sum_{i=1}^m \lambda_i g_i(\mathbf{x}) + \sum_{i=1}^p \nu_i h_i(\mathbf{x})$
	\item[Dual function:] $D(\boldsymbol{\lambda}, \boldsymbol{\nu}) := \inf_{\mathbf{x}} L(\mathbf{x}, \boldsymbol{\lambda}, \boldsymbol{\nu}) \in \mathbb{R}$
	\item[Dual Problem:] $\max_{\boldsymbol{\lambda}, \boldsymbol{\nu}} D(\boldsymbol{\lambda}, \boldsymbol{\nu})$ s.t. $\boldsymbol{\lambda} \geq \mathbf{0}$. Note: $\max_{\boldsymbol{\lambda}, \boldsymbol{\nu}} D(\boldsymbol{\lambda}, \boldsymbol{\nu}) \le \min_\mathbf{x}{f(\mathbf{x})}$, equality if $dom\ f$ and $f$ convex
\end{compactdesc}

\subsection*{Convex Optimization}
Def.: $\{(x,t)|x \in dom f, f(x) \leq t\}$, $f : \mathbb{R}^D \rightarrow \mathbb{R}$ is convex, if $dom\ f$ is a convex set, and if $\forall \mathbf{x}, \mathbf{y} \in dom\ f$, and for $0 \leq \alpha \leq 1$: $f(\alpha \mathbf{x} + (1 - \alpha)\mathbf{y}) \leq \alpha f(\mathbf{x}) + (1-\alpha)f(\mathbf{y})$. local=global min, \textbf{Convergence}: $f(\mathbf{x}^{(t)}) - f(\mathbf{x}^*) \le \frac{c}{t}$.
\textbf{Subgradient} $g \in \mathbb{R}^D$ of $f$ at $\mathbf{x}$: $f(\mathbf{y}) \geq f(\mathbf{x}) + g^\top(\mathbf{y}-\mathbf{x}) \ \forall \mathbf{y}$

\subsection*{Convex Relaxation}
Replace non-convex rank constraints by convex norm constraints.\\
Nuclear norm: $||A||_* = \sum_i \sigma_i$


\lab{Complexity and Sparse Matrices}{Complexity and Sparse Matrices}

\objective{Introduce the temporal and spatial complexity and explore SciPy's methods for working with sparse matrices.}
\label{lab:complexity}

\section*{Complexity}
Two major constraints on computing are time and available memory (or `space').
The \emph{temporal complexity} and \emph{spatial complexity} of an algorithm measure how much of these resources the algorithm requires.
This lab intoduces complexity using examples and intuition; for a rigorous introduction, see Volume 2 of the textbook.

\subsection*{Temporal Complexity}
One of the most important questions in scientific computing is ``How long will a computer take to execute this algorithm?"
For example, suppose an algorithm operating on a 1-D array of length $n$ requires $f(n)$ calculations, where

\begin{equation*}
f(n) = \frac{3n^3}{2} + 75n^2 + 250n + 30.
\end{equation*}

As $n$ increases, the growth of $f(n)$ is dominated by the $n^3$ term.
This gives us information about how the runtime of our algorithm increases when we increase input size.
If we double the size of the input $n$, we would expect the algorithm to need about $2^3=8$ times as many steps, which would make it run about 8 times as long.

The function $f(n)$ above is called the \emph{temporal complexity} of the algorithm.
In general, $n$ is a positive integer that somehow describes the size of the inputs to your algorithm.
For example, perhaps your algorithm accepts $n \times n$ arrays, or perhaps it accepts 1-D arrays of length $n$.
The temporal complexity of your algorithm is a function that accepts an input size $n$ and returns the number of steps the algorithm needs to execute on that input.
As such, temporal complexity is a precise way to describe how the execution time of your algorithm increases as the size of your input increases.

Often, we do not care about the exact definition of $f(n)$ so much as its behavior when $n$ gets large.
This leads to the notion of an \emph{asymptotic upper bound}.
An asymptotic upper bound is another function $g(n)$ such that, eventually, $f(n)$ is less than some constant multiple of $g(n)$.
In the above example, $n^3$ is an asymptotic upper bound for $f(n)$ (see Figure \ref{fig:asymp_upper_bound}).
If $g(n)$ is an asymptotic upper bound for $f(n)$, we say $f(n)$ is $O(g(n))$.
So in the example above, $f(n)$ is $O(n^3)$ (spoken ``Big O of n cubed'' or ``order of n cubed").

\begin{figure}
\centering
\includegraphics[width=\textwidth]{asymp_upper_bound.pdf}
\caption{When $n$ is bigger than 160, $f(n)$ is less than $2n^3$. This means that $n^3$ is an asymptotic upper bound for $f(n)$, so we say $f(n)$ is $O(n^3)$.}
\label{fig:asymp_upper_bound}
\end{figure}

For example, adding two $n \times n$ matrices is $O(n^2)$.
This is because it takes 1 step to add each pair of elements in two $n \times n$ arrays, and there are $n^2$ such pairs.
By comparison, calculating the inverse of a $n \times n$ matrix using Gaussian row reduction is $O(n^3)$.
(More efficient algorithms for matrix inversion exist as well.)

\begin{comment}
How do we determine the temporal complexity of a piece of code?
Calculating the exact temporal complexity of an algorithm is fairly difficult.
However, there are ways to heuristically evaluate the asymptotic behavior of an algorithm's temporal complexity.
For example, consider the following code.

\begin{lstlisting}
s = 0
for i in xrange(n):
    s = s + i
\end{lstlisting}

The code above is $O(n)$ because it takes approximately $n$ steps to complete.
For-loops are a good indicator of the complexity.
A double for-loop suggests $O(n^2)$ or worse.

You can also approximate the temporal complexity of a function by timing how long it takes to run on inputs of various sizes.
This is especially useful if your temporal complexity is some polynomial.
In this case, the ratios of the run times will give you clues about the temporal complexity of the algorithm.
For example, if an algorithm is $O(n^3)$, then doubling the size of the input should make the algorithm take $(2n)^3/n^3=8$ times as long.
\begin{comment}
\begin{problem}
Time the runtime of the following code for \li{n = 1000, 2000, 4000, 8000}.

\lstinputlisting[style=fromfile]{test.py}

Now write a function that takes no arguments and does the following:
\begin{enumerate}
\item Plots the four runtimes (use \li{[1000, 2000, 4000, 8000]} or an equivalent array object as your domain).
The plot should look like Figure \ref{prob1} if using the \li{plt.scatter} command.
\item Returns the average ratio between successive runtimes (one such ratio would be the runtime for $n = 2000$ divided by the runtime for $n = 1000$).
\end{enumerate}
\begin{figure}
\centering
\includegraphics[width=\textwidth]{prob1.pdf}
\caption{The plot of problem 1}
\label{prob1}
\end{figure}
\end{problem}

\begin{problem}\label{complexity_ratios}
Fill in the following table. Here, $f(n)$ is the temporal complexity of an algorithm. The quantity $f(2n)/f(n)$ tells you how much longer each algorithm will take to run when you double the size of the input.

\begin{center}
\begin{tabular}{| l |p{15mm}|p{15mm}|p{15mm}|p{15mm}|p{15mm}|}\hline
$f(n)$ & $n$ & $n^2$ & $n^3$ & $n^4$  \\ \hline
$f(2n)/f(n)$&  & & &  \\
\hline
\end{tabular}
\end{center}

\end{problem}

\begin{problem}\label{complexity_timeit}
Determine the temporal complexity of each of the following functions.
\begin{enumerate}
\item
\begin{lstlisting}
def function1(n):
    t = 0
    for i in xrange(n):
        t += sum(xrange(i))
\end{lstlisting}

\item
\begin{lstlisting}
def function2(n):
    sum(xrange(1000*n))
\end{lstlisting}
\end{enumerate}

Do this as follows:
\begin{enumerate}
\renewcommand{\theenumi}{\alph{enumi}}
\item Time how long the function takes to run on an input of size $n$ for $n=1000, 2000, 4000$, and $8000$.
\item Compute the averages of the ratios between successive runtimes from part (1).
\item Use Problem \ref{prob:complexity_ratios} to determine the temporal complexity.
\end{enumerate}
\end{problem}

\begin{problem}
Create a plot depicting the complexities you found in Problem \ref{prob:complexity_timeit} by following these steps.
\begin{enumerate}
\item For each function, plot a line using $n=1000, 2000, 4000$, and $8000$ for $x$-values and your runtimes for $y$-values. When you call the function \li{plt.plot()}, specify a value for the keyword parameter \li{label} so you can reference each line later. For example, the call \li{plt.plot( x, y, label="Function 1")} attaches the label "Function 1" to this line.
\item After plotting all your lines, call \li{plt.legend()} to draw a legend on your graph. Use the keyword argument \li{loc} to specify the location of the legend. Some possible values are \li{'upper left'} and \li{'right'}. See the documentation for more options.
\end{enumerate}
Your graph should look something like this.

[TODO: put a figure here]
\end{problem}
\end{comment}

% TODO: THIS WAS IN THE NUMPY LAB BUT REALLY SHOULD BE IN THIS LAB!!!
\begin{comment}
\begin{problem} % Problem 2: Time matrix multiplication.
For a $m\times n$ matrix $A$ with entries $a_{ij}$ and an $n\times l$ matrix $B$ with entries $b_{ij}$, the matrix product $C = AB$ is defined entrywise by the formula:
\[c_{ij} = \sum_{k=1}^N a_{ik}b_{kj}\]

The following function performs matrix multiplication using nested lists without using NumPy.

\begin{lstlisting}
def matrix_multiply(A, B):
    """Calculate the matrix product AB.
    Each parameter is a list of lists.
    """
    # Get the dimensions of the matrices and initialize the new 'matrix'.
    m, n, l = len(A), len(B), len(B[0])
    result = []

    # Calculate each entry of the new matrix.
    for i in range(m):
        for j in range(l):
            result.append(sum([A[i][k] * B[k][j] for k in xrange(n)]))
    return result
\end{lstlisting}

Write a function that times matrix multiplication with the above function, and compare it with numpy.dot().
A random matrix $1000\times 1000$ as a list of lists can be created with
\begin{lstlisting}
a = [[random() for j in xrange(1000)] for i in xrange(1000)]
\end{lstlisting}

\end{problem}

% SIMPLIFIED VERSION OF THE CODE BELOW (don't include from other file)

Iterating through this triple \li{for} loop is very expensive.
NumPy also uses loops, but it uses C loops instead of Python loops.
Compare the difference between the pure Python and the NumPy ways:

% \lstinputlisting[style=fromfile]{arr_mult.py}

Table \ref{table:square_times} documents how long\footnote{You can replicate this experiment yourself. In IPython, you can find the execution time of a line of code by prefacing it with \li{\%timeit}.
If you aren't using IPython, you will need
to use the timeit function documented here: \url{https://docs.python.org/2/library/timeit.html}.}
one computer took to square a $k \times k$ matrix in both Python (using the function \li{arr_mult}) and NumPy (using the method you found in Problem \ref{prob:simple_multiply}) for various values of $k$.
As you can see, NumPy is much faster.
One reason for this is that algorithms in NumPy are usually implemented in C or in Fortran.

\begin{table}
 \begin{tabular}{|c|l|l|} \hline Data Structure & $k\times k$ & Time (s) \\ \hline
 Python List    & $10\times10$  & 0.0002758503 \\
 \cline{2-3}    & $100\times100$    & 0.1336028576 \\
 \cline{2-3}    & $1000\times1000$ & 200.4009799957 \\
 %& $1\times1$      & 0.0000181198 \\
\hline \hline
 NumPy Array    & $10\times10$  & 0.0000109673 \\
 \cline{2-3}    & $100\times100$    & 0.0009210110 \\
 \cline{2-3}    & $1000\times1000$ & 2.1682999134 \\
 %& $1\times1$      & 0.0000298023 \\
 \hline \end{tabular}
 \caption{Time for one computer to square a $k \times k$ matrix in Python and NumPy.}
\label{table:square_times}
\end{table}
%

NumPy is optimized for fast array computations.

\end{comment}

\subsection*{Spatial Complexity}
Analogous to temporal complexity, the spatial complexity of an algorithm is a function that describes how the algorithm's memory use increases as input sizes to the algorithm increase. For example, if your algorithm needs to store an $n \times n$ matrix, its memory use will increase at least as fast as $n^2$. Spatial complexity is important because  the spatial complexity of an algorithm can affect its speed in several ways. The most important way is that when the memory usage exceeds the amount of available RAM, the machine must use either the hard disk or some other slower method of storage.

This vocabulary allows us to discuss the question at the start of this lab: ``How long will a computer take to execute this algorithm?" The amount of time a computer takes to execute an algorithm depends both on the algorithm's temporal complexity and on its spatial complexity.

\subsection*{Timing Functions}
We can empirically investigate a function's temporal and spatial complexity by timing how long it takes to run.
In IPython\footnote{If you aren't using IPython, you will need
to use the timeit function documented here: \url{https://docs.python.org/2/library/timeit.html}.}, you can time a line of code by prefacing it with the command \li{\%timeit}.
As an example, we time how long it takes to add the numbers from 1 to 1000.
\begin{lstlisting}
>>> \%timeit sum(xrange(1000))
100000 loops, best of 3: 12.2 us per loop
\end{lstlisting}
This output means that the computer ran our line 100,000 times and averaged the runtimes.
The computer repeated this experiment 3 times, and the best average was 12.2 microseconds.

\begin{problem}\label{prob:complexity_problem}
In this problem you will analyze the complexity of the following functions:

\begin{lstlisting}
# Run through a single for loop.
def func1(n):
    n = 500*n
    sum(xrange(n))
# Run through a double for loop.
def func2(n):
    n = 3*n
    t = 0
    for i in xrange(n):
        for j in xrange(i):
            t += j
# Square a matrix.
def func3(n):
    n = int(1.2*n)
    A = np.random.rand(n, n)
    np.power(A, 2)
# Invert a matrix.
from scipy import linalg as la
def func4(n):
    A = np.random.rand(n, n)
    la.inv(A)
# Find the determinant of a matrix.
from scipy import linalg as la
def func5(n):
    n = int(1.25*n)
    A = np.random.rand(n, n)
    la.det(A)
\end{lstlisting}

Plot your results in the following manner:
\begin{enumerate}
\item Time how long each function takes to run on an input of size $n$ for $n=100, 200, 400$, and $800$.
\item For each function, plot a line using $100, 200, 400$, and $800$ for the $x$-values and your runtimes for the $y$-values. Make sure your $y$-values are all in the same units! When you call the matplotlib function \li{plot}, specify a value for the keyword parameter \li{label} so that you can reference each line later. For example, the call \li{plt.plot(x, y, label='Function 1')} attaches the label ``Function 1" to the line made by \li{x} and \li{y}.
\item After plotting all your lines, call the function \li{legend} to draw a legend on your graph. Use the keyword argument \li{loc} to specify the location of the legend. Some possible values are \li{'upper left'} and \li{'right'}. See the documentation for more options. Your final graph should look something like the figure below.\footnote{From this graph, you can see that inverting a matrix is an incredibly time-consuming operation. In Lab \ref{lab:ChangeBasis} you will learn fast algorithms for solving linear systems without inverting matrices. In fact, inverting matrices is so costly in terms of temporal and spatial complexity that there is hardly ever a good reason to do it.}
\end{enumerate}
Note that we rescaled the inputs to the functions so that your lines would begin at roughly the same place on the y-axis.

\centering
\includegraphics[width=\textwidth]{complexity_problem.pdf}

\end{problem}

\section*{Sparse Matrices}
A \emph{sparse} matrix is a matrix that has relatively few nonzero elements.
Sparse matrices arise frequently in both theoretical and real-world applications.
We can take advantage of the sparse structure of these matrices to use less memory and to decrease computation time.

For example, diagonal matrices are sparse.
Storing an $n \times n$ diagonal matrix in the naive way means storing $n^2$ values in memory.
For most applications, it makes more sense to store the diagonal entries in a 1-dimensional array of $n$ values.
In addition to using less storage space, this allows for much faster matrix operations.
Using the standard algorithm to multiply a matrix by a diagonal matrix involves $n^3$ steps, but most of these are multiplying by or adding zero.
A smarter algorithm that knows all off-diagonal entries are zero can accomplish the same task much faster.

\subsection*{The Sparse Module}
The SciPy module \li{sparse} has storage methods to reduce the temporal and spatial complexity of handling sparse matrices.
When we are not using these special methods, we say we are storing the \emph{full} matrix.
We can also use \li{sparse} methods on dense matrices (matrices with mostly nonzero entries), but doing so will take longer than using the usual methods for handling full matrices.

The difference in spatial complexity occurs because a full array occupies a block of memory for each entry, so an $n \times n$ array requires $n^2$ blocks of memory.
By contrast, SciPy's \li{sparse} methods store only the nonzero entries and their locations in the array.
As long as most entries are 0 (i.e., the matrix is sparse), this decreases spatial complexity.
You minimize spatial complexity when you store a sparse matrix with the \li{sparse} module and a dense matrix as a full (or ``regular'') matrix.

SciPy has seven sparse matrix types, listed in Table \ref{table:smr}.
Each type is optimized either for storing sparse matrices whose nonzero entries follow certain patterns, or for performing certain computations.
For example, the \li{csc_matrix} and \li{csr_matrix} types are optimized for arithmetic operations.
We will introduce some other types later in this lab. For more information, see the documentation (\url{http://docs.scipy.org/doc/scipy/reference/sparse.html}).

\begin{table}
\centering
\begin{tabular}{|r|l|}
\hline
Sparse Matrix Type & Description \\
\hline
\li{bsr_matrix} & Compressed Block Sparse Row\\
\li{coo_matrix} & Coordinate\\
\li{csc_matrix} & Compressed Sparse Column\\
\li{csr_matrix} & Compressed Sparse Row\\
\li{dia_matrix} & Sparse Diagonal\\
\li{dok_matrix} & Dictionary of Keys\\
\li{lil_matrix} & Linked List\\
\hline
\end{tabular}
\caption{Sparse matrix representations in SciPy.}
\label{table:smr}
\end{table}

Let us compare a \li{sparse} matrix computation with a full matrix computation.
Note that we can convert any full matrix to a \li{sparse} matrix of any of the types listed in Table \ref{table:smr}.

\begin{lstlisting}
import numpy as np
from scipy import sparse

# Create a dense matrix (stored as a full matrix).
>>> A_full = np.random.rand(600, 600)

# Store A_full as a sparse matrix (even though it is dense).
>>> A_sparse = sparse.csc_matrix(A_full)

# Create a sparse matrix (stored as a full matrix).
>>> B_full = np.diag(np.random.rand(600))

# Store B_full as a sparse matrix.
>>> B_sparse = sparse.csc_matrix(B_full)

>>> def square(A):
        return np.power(A, 2)

>>> %timeit square(A_full)
100 loops, best of 3: 9.53 ms per loop

>>> %timeit square(A_sparse)
1 loops, best of 3: 941 ms per loop

>>> %timeit square(B_full)
100 loops, best of 3: 5.36 ms per loop

>>> %timeit square(B_sparse)
1000 loops, best of 3: 259 us per loop
\end{lstlisting}

As you can see from this example, we get the best performance when we store a sparse matrix with the \li{sparse} module and a dense matrix as a full matrix.

\begin{comment}
\begin{problem}
Create a $500\times 500$ matrix and vector of length 500, both full of random values. Use the \li{A.dot(b)} command to multiply your matrix and your vector, and time how long it takes to do so. Then convert your matrix to sparse format and again time how long it takes to multiply it by your vector using \li{A.dot(b)}.
\end{problem}
\end{comment}

\subsection*{Creating Sparse Matrices}
One way to create a sparse matrix is to create a full matrix and then convert it to a sparse matrix, as we did in the previous example.
However, you reduce spatial complexity if you never create the full matrix.
Here are two ways to create sparse matrices directly.

The first way is to use the method \li{sparse.spdiags(data, diags, m, n)}.
If \li{data} is a 1-D array and \li{diags} is a scalar, then this method creates an $m \times n$ matrix with \li{data} on the specified diagonal.
The parameter \li{diags=0} indicates the main diagonal, with lower diagonals indexed by negative numbers and upper diagonals by positive numbers.
If \li{data} is a 2-D array and \li{diags} is a list, then this method creates an $m \times n$ matrix with the rows of \li{data} on the diagonals specified by \li{diags}.
See the documentation for more information.
\begin{lstlisting}
# Create a sparse 3x3 matrix with (2, 3, 4) on the diagonal.
>>> A = sparse.spdiags([2, 3, 4], 0, 3, 3)
>>> A
<3x3 sparse matrix of type '<type 'numpy.int64'>'
	with 3 stored elements (1 diagonals) in DIAgonal format>
# Convert A to a full matrix.
>>> A.todense()
matrix([[2, 0, 0],
        [0, 3, 0],
        [0, 0, 4]])

# Create a sparse 4x4 matrix with the rows of diag_entries on the diagonals.
>>> diag_entries = np.array([[3,6,9,0],[1,4,7,10],[0,2,5,8]])
>>> B = sparse.spdiags(diag_entries, [-1, 0, 1], 4, 4)
<4x4 sparse matrix of type '<type 'numpy.int64'>'
	with 10 stored elements (3 diagonals) in DIAgonal format>
>>> B.todense()
matrix([[ 1,  2,  0,  0],
        [ 3,  4,  5,  0],
        [ 0,  6,  7,  8],
        [ 0,  0,  9, 10]])
\end{lstlisting}

The final matrix $B$ in the example above is a special kind of matrix called a \emph{banded} matrix.
A banded matrix is a sparse matrix whose only non-zero entries are on the main diagonal and some diagonals on either side.
In fact, $B$ is an example of a \emph{tri-diagonal} matrix, because its nonzero entries are confined to the three central diagonals.
Banded matrices arise naturally in many applications, including numerical methods for solving differential equations.

\begin{problem}
Write a function that takes an integer argument \li{n} and returns a sparse $n\times n$
tri-diagonal array with $2$'s along the diagonal and $-1$'s along
the two sub-diagonals above and below the diagonal.
The array should be in \li{csr_matrix} format.
\emph{Hint}: Read about the \li{format} keyword parameter of the \li{sparse.spdiags} method.

This matrix is the derivative operator in numerical analysis of differential equations.
\label{prob:sparse_tridiag}
\end{problem}

A second way to create a sparse matrix is to pre-allocate an array of zeros and then specify the nonzero entries one at a time.
The most efficient sparse matrix types for building matrices incrementally are \li{lil_matrix} and \li{dok_matrix}.
Once you are done constructing the sparse matrix, you should convert it to a form that is optimized for computations.

\begin{lstlisting}
# Initialize Z.
>>> Z = sparse.lil_matrix((400, 300))
# Specify the nonzero entries of Z.
>>> Z[1,34] = 23
>>> Z[23,32] = 56
>>> Z[2,:] = 13.2
>>> Z
<400x300 sparse matrix of type '<type 'numpy.float64'>'
	with 302 stored elements in LInked List format>

\end{lstlisting}

When the matrix \li{Z} is initialized, all its entries are assumed to be zero.
Note that at the end of \li{Z}'s construction, only 302 elements are being stored for a matrix with 120000 entries.

You may have noticed that the only way to view a matrix as a 2-D array is to convert it to a full matrix.
If your matrix is too large to do this, you can still visualize it using the \li{plt.spy()} command from matplotlib.
This function plots the locations of the non-zero entries in a matrix.
The following code outputs Figure \ref{fig:mpl_spy}.

\begin{lstlisting}
>>> from matplotlib import pyplot as plt
>>> B = np.random.rand(3, 10000)
>>> A = sparse.spdiags(B, range(-1, 2), 10000, 10000)
>>> plt.spy(A)
\end{lstlisting}

\begin{figure}
\centering
\includegraphics[width=.75\textwidth]{spy.png}
\caption{The output of the \li{spy()} command}
\label{fig:mpl_spy}
\end{figure}

%insert problem here that lets them play around with large sparse matrices in diff forms

\begin{comment}
\subsection*{Banded Matrices}
\begin{problem}
Write a function that takes an integer argument \li{n} and returns a full $n\times n$
tri-diagonal array with $2$'s along the diagonal and $-1$'s along
the two sub-diagonals above and below the diagonal.
\emph{Helpful Hint}: Use the \li{np.diagflat()} command.
\label{full_tridiag}
\end{problem}
\end{comment}

\subsection*{Manipulating Sparse Matrices}
Scipy's \li{sparse} matrices behave a little differently than NumPy arrays.
You can multiply two sparse matrices element-wise with the \li{multiply()} method of one of the sparse matrices.
\begin{lstlisting}
>>> D = sparse.spdiags([2,3,4],0,3,3)
>>> C = sparse.spdiags(np.ones((3,3)), [-1,0,1], 3, 3)
>>> (D.multiply(C)).todense()
matrix([[ 2.,  0.,  0.],
        [ 0.,  3.,  0.],
        [ 0.,  0.,  4.]])
\end{lstlisting}

On the other hand, the asterisk \li{*} performs ordinary matrix multiplication.
You can also use the \li{dot} method of one of the sparse matrices.
However, you should NOT use \li{np.dot} on sparse matrices because it may return an incorrect answer.
\begin{lstlisting}
# One correct way to mutliply sparse matrices
>>> (D.dot(C)).todense()
matrix([[ 2.,  2.,  0.],
        [ 3.,  3.,  3.],
        [ 0.,  4.,  4.]])
\end{lstlisting}

Addition and scalar multiplication are implemented as usual.
\begin{lstlisting}
>>> (D + 3*C).todense()
matrix([[ 5.,  3.,  0.],
        [ 3.,  6.,  3.],
        [ 0.,  3.,  7.]])
\end{lstlisting}

\section*{Using Sparse Matrices to Reduce Runtimes}
In addition to spatial complexity, the \li{sparse} module can reduce temporal complexity.
Consider the linear system $A x = b$, where $A$ is a $100000\times 100000$ tri-diagonal matrix.
Storing a full matrix of that size requires 10 billion double-precision floating-point numbers.
Since it takes 8 bytes to store a double, we need roughly 80GB to store the full matrix.
Lack of storage space makes this system impossible to solve for most desktop computers, but even more problematic is the temporal complexity.
Methods for directly solving a linear system are usually $O(n^3)$.
As a result, even if the computer could store an 80GB matrix in RAM, it would still take several weeks to solve the system.
%However, since we don't typically have computers with that much available RAM, most of the
%matrix would have to be stored on the hard drive, so the computation would probably take between $6$ months to a year.

The point is, even as computers increase in processing speed and memory, we can still easily construct problems that they will struggle to solve in a reasonable amount of time.
However, if we store the tri-diagonal matrix as a \li{sparse} matrix, we can solve the linear system, even with a modest computer.

Let's first compute the spatial complexity of the above system when $A$ is stored as a sparse matrix.
There are three diagonals that have roughly $100000$ non-zero entries.
That's $300000$ double-precision floating point numbers, which is about 2.4 MB, or less storage than your favorite song.
Thus, the sparse matrix will easily fit into the computer's RAM. Furthermore, the temporal complexity for solving a tri-diagonal matrix is $O(n)$.
\footnote{Because there are fast algorithms for solving a tri-diagonal linear system, you may think that there are fast algorithms for inverting a tri-diagonal matrix.
In fact this is not true, and the inverse of a sparse matrix is usually not sparse.
There is rarely a good reason to invert a matrix, sparse or dense.}
Let's see how long it takes to solve the system when $A$ and $b$ are filled with random data.

\begin{lstlisting}
>>> from scipy.sparse import linalg as sl
>>> G = np.random.rand(3, 100000)
>>> b = np.random.rand(1, 100000)
>>> A = sparse.spdiags(G,[-1,0,1],100000,100000, format='csr')
>>> def solSys():
...     return sl.spsolve(A, b)

>>> %timeit solSys()
1 loops, best of 3: 80.8 ms per loop

\end{lstlisting}

This computer solved the system in only 80.8 milliseconds.

\begin{comment}
\begin{problem}
Write a function that accepts an integer argument \li{n} as well as a keyword argument \li{sparse} whose value is either \li{True} or \li{False} (default to \li{False}).
Then do the following:
\begin{enumerate}
\item Inside of the function, use your previous solutions to generate an $n \times n$ tri-diagonal array $A$ -- either sparse or full depending on the value of the \li{sparse} argument.
\item Generate an $n \times 1$ random array $b$
\item Solve the system $Ax = b$, using either \li{scipy.sparse.linalg.spsolve} or \li{scipy.linalg.solve}
(again depending on the value of \li{sparse}) and return the solution.
\item Time the function for \li{n = 2000} using both the sparse and the full option.
\end{enumerate}
\end{problem}
\end{comment}

\begin{problem}
Write a function that accepts an integer argument $n$ and does the following:
\begin{enumerate}
\item Generates an $n \times 1$ random array $b$.
\item Solves the linear system $Ax = b$, where $A$ is the tri-diagonal array in Problem \ref{prob:sparse_tridiag} of size $n \times n$.
\end{enumerate}
\end{problem}

\begin{problem}
Write a function that accepts an integer argument \li{n} and returns $\lambda n^2$, where
$\lambda$ is the smallest eigenvalue of the sparse tri-diagonal array you built in Problem \ref{prob:sparse_tridiag}.

If \li{A} is your tri-diagonal matrix, calculate $\lambda$ using the method \li{scipy.sparse.linalg.eigs} with the command \li{sl.eigs(A.asfptype(), which = 'SM')}.
The code \li{A.asfptype()} ensures that your matrix has the right data type, and the parameter \li{which = 'SM'} tells the function to look for the smallest eigenvalues.
This command will return several of the smallest eigenvalues of \li{A}, and you will have to select the smallest of these.
Read the documentation of \li{sl.eigs} for more information.

What value does $\lambda n^2$ approach as $n$ approaches infinity?
This value is meaningful in operator theory.
\li{Hint}: This value is the square of an important number.

\end{problem}