%---------------------------Stretch---------------------------
\section{Stretch}

The stretch is the ratio of the minimum edge length to the maximum diagonal, normalized
so that a unit cube has a value of 1:
\[
  q = \sqrt{3}\frac{L_{\min}}{D_{\max}}.
\]

Note that if $D_{\max} < DBL\_MIN$, we set $q = DBL\_MAX$.

\hexmetrictable{stretch}%
{$1$}%                                        Dimension
{$[0.25,1]$}%                                 Acceptable range
{$[0,1]$}%                                    Normal range
{$[0,DBL\_MAX]$}%                             Full range
{$1$}%                                        Cube
{Adapted from \cite{fimesh:xx}}%              Citation
{v\_hex\_stretch}%                            Verdict function name


\documentclass{article}

\usepackage{amsmath}

\def\r{\mathbf{r}}
\def\q{\mathbf{q}}
\def\i{\mathbf{i}}
\def\j{\mathbf{j}}
\def\k{\mathbf{k}}

\begin{document}

\section{Coordinate systems}

Let $x,y,z$ be a coordinate system such that the camera is at the
origin, the floor is some known constant $y$, and positive $z$ is the
current direction of motion of the robot. We write
\begin{equation}
  \r = x\i + y\j + z\k
\end{equation}
where $\i,\j,\k$ can be considered unit vectors or quaternions.  These
can be conveniently represented as the complex matrices
\begin{equation}
\i = \begin{pmatrix} 0 & i  \\ i & 0  \end{pmatrix} \qquad
\j = \begin{pmatrix} 0 & -1 \\ 1 & 0  \end{pmatrix} \qquad
\k = \begin{pmatrix} i & 0  \\ 0 & -i \end{pmatrix}
\end{equation}
but other representations are also possible.

The camera is at some orientation given by the quaternion $\q$ as
\begin{equation}
\r' = \q \, \r \, \q^*
\end{equation}
such that $z'$ is the forward direction of the camera.  Let $f$ be the
focal distance.  Then the camera-plane coordinates will be
\begin{equation}
  \begin{pmatrix} X' \\ Y' \end{pmatrix} =
  f \begin{pmatrix} x'/z' \\ y'/z' \end{pmatrix}
\end{equation}
The actual camera coordinates will, however, involve a nonlinear
calibration
\begin{equation}
  X,Y = K(X',Y')
\end{equation}

Consider two snapshots, between which the robot moves by $\Delta z$.
That is, the world coordinate system moves by $-\Delta z$.  This
results in visual vectors
\begin{equation}
  \begin{pmatrix} \Delta X \\ \Delta Y \end{pmatrix}
\end{equation}

\section{Lens-calibrated case}

We assume that $K$ has been calibrated out to the identity.  Then
\begin{equation}
  \Delta\r' = - \q \, \k \, \q^* \, \Delta z
\end{equation}

Suppose first that $\q=1$.  Then
\begin{equation}
   \begin{pmatrix} \Delta X \\ \Delta Y \end{pmatrix} =
   f \begin{pmatrix} x \\ y \end{pmatrix}
   \left(1/z-1/(z-\Delta z)\right)
\end{equation}
or (since $\Delta z$ is small)
\begin{equation}
   \begin{pmatrix} \Delta X \\ \Delta Y \end{pmatrix} =
-f \, \frac{\Delta z}z \begin{pmatrix} X \\ Y \end{pmatrix}
\end{equation}
In this expression, all the visual vectors point to the origin of the
camera plane.  If we now know $f$ and $\Delta z$, we have $z$.

If $\q\neq1$ the visual vectors will still all point to a single
point, namely the vanishing point corresponding to the direction of
motion in camera coordinates.  This fact suggests a simple strategy
for calibrating the pitch and yaw of the camera: find $\q$ so as to
move the vanishing point of the visual vectors to the origin.
Determining the roll (or the $\k$ component of $\q$) is harder, but
maybe not necessary.

\end{document}


\section{Proof of Theorem~\ref{thm:less_than_total_degree}}
\label{app:proof_thm_less_than_total_degree}
To prove Theorem~\ref{thm:less_than_total_degree} we need the following lemmas.
  \begin{lemma}\label{lem:difference_to_p-1}
    For all $(a,b)\in\Z^2$ we have:
    \[
      (a - b)^{p-1} = \sum_{i=0}^{p-1} a^i b^{p-1-i} \mod p.
    \]
  \end{lemma}
  \begin{proof}
    Using the binomial theorem we obtain
    \[
      (a - b)^{p-1} = \sum_{i=0}^{p-1} \binom{p-1}{i} a^i (-b)^{p-1-i}.
    \]
    Computing the binomial coefficient modulo $p$
    \begin{align*}
      \binom{p-1}{i} & = \frac{(p-1)!}{i! (p-1-i)!} \\
      & = \frac{(p-1)(p-2)\dots(i+1)}{1 \cdot 2 \dots (p-(1+i))} \\
      &= (-1)^{p-1-i} \mod p\,,
    \end{align*}
    we prove the lemma.
  \end{proof}

  \begin{lemma}\label{lem:sum_poly}
    Let $P(X)$ be a polynomial of degree $d$ less than $p-1$.
    For any prime number $p > 2$, it holds
    \[
      \sum_{a=0}^{p-1} P(a) = 0 \mod p.
    \]
  \end{lemma}
  \begin{proof}
    Since $\sum_{a=0}^{p-1} b = 0 \mod p$ for any $b \in \F_p$, it is enough to prove that the sum $\sum_{a=0}^{p-1} a^n = 0 \mod p$ for any $0 \leq n < p-1$.
    Since the case $n=0$ is straightforward, let us assume $n>0$.
    Let $g$ be a primitive element of $\F_p$.
    Since $p > 2$, we have $g \ne 1$.
    Thus, we can rewrite the above sum as follows.
    \begin{align*}
      \sum_{i=1}^{p-1} g^{in} = \frac{g^{pn} - g^n}{g^n - 1}.
    \end{align*}
    Since $g^{pn} \equiv g^n \mod p$, the sum turns into zero modulo $p$.
  \end{proof}
  Now, we have all the ingredients to prove Theorem~\ref{thm:less_than_total_degree}.
  \begin{proof}[Proof of Theorem \ref{thm:less_than_total_degree}]
    Assume that all computations are done modulo $p$.
    Using Lemma~\ref{lem:difference_to_p-1}, we obtain that $P_{\LT_\S}(X,Y)$ is equal to
    \begin{align*}
      \sum_{a = 0}^{p-2} \left(1-\sum_{i=0}^{p-1} X^i a^{p-1-i}\right) \sum_{b=a+1}^{p-1} \left(1-\sum_{j=0}^{p-1} Y^j b^{p-1-j}\right)
    \end{align*}
    Let us expand this expression distributively.
    \begin{align*}
      \sum_{a = 0}^{p-2} \sum_{b=a+1}^{p-1} 1 - \sum_{i=0}^{p-1} X^i a^{p-1-i} - \sum_{i=0}^{p-1} Y^i b^{p-1-i} \\
      + \sum_{i=0}^{p-1} \sum_{j=0}^{p-1} X^i Y^j a^{p-1-i} b^{p-1-j}.
    \end{align*}
    Let us compute individual polynomial coefficients.
    The constant term is equal to
    \begin{align*}
      \sum_{a = 0}^{p-2} \sum_{b=a+1}^{p-1} 1 - a^{p-1} - b^{p-1} + a^{p-1} b^{p-1} \\
      = \sum_{a = 0}^{p-2} \sum_{b=a+1}^{p-1} 1 - a^{p-1} - 1 + a^{p-1} = 0\,.
    \end{align*}
    Coefficients by $X^i$ with $i > 0$ can be computed as follows
    \begin{align*}
      \sum_{a = 0}^{p-2} \sum_{b=a+1}^{p-1} \left(-a^{p-1-i}\right) + a^{p-1-i}b^{p-1} = 0\,.
    \end{align*}
    Next, we compute coefficients by $Y^i$ with $i > 0$.
    \begin{align*}
      -\sum_{a = 0}^{p-2} &\sum_{b=a+1}^{p-1} b^{p-1-i} - a^{p-1} b^{p-1-i} \\
      &= -\sum_{b=1}^{p-1} b^{p-1-i} - \sum_{a = 1}^{p-2} \sum_{b=a+1}^{p-1} b^{p-1-i} - b^{p-1-i} \\
      &= -\sum_{b=1}^{p-1} b^{p-1-i}.
    \end{align*}
    If $i = p-1$, this sum is equal to $1$.
    According to Lemma~\ref{lem:sum_poly}, it is $0$ if $i < p-1$.

    To compute coefficients by $X^i Y^j$ with $i, j > 0$, we will use Faulhaber's formula below
    \begin{align*}
      \sum_{k=1}^n k^e = \frac{1}{e+1} \sum_{i=1}^{e+1} (-1)^{\delta_{ie}} \binom{e+1}{i} B_{e+1-i} \cdot n^i\,,
    \end{align*}
    where $\delta_{ie}$ is the Kronecker delta and $B_{i}$ is the $i$th Bernoulli number.
    This implies that there exist a polynomial $P(X) \in \F_p[X]$ of degree $e+1$ such that
    \begin{align}\label{eq:faulhaber}
      \sum_{k=1}^n k^e = P(n).
    \end{align}
    Note that $P(0) = 0$.
    The coefficient by $X^i Y^j$ for some positive $i$ and $j$ is equal to
    \begin{align*}
      \sum_{a = 0}^{p-2} \sum_{b=a+1}^{p-1} a^{p-1-i} b^{p-1-j} &= \sum_{b = 1}^{p-1} b^{p-1-j} \sum_{a=0}^{b-1} a^{p-1-i}\,. \\
    \end{align*}
    According to~(\ref{eq:faulhaber}), there exist a polynomial $P_i(X)$ of degree $p-i$ such that $\sum_{a=0}^{b-1} a^{p-1-i} = P_i(b)$.
    Since $Q_{ij}(X) = X^{p-1-j} P_i(X)$ has degree $2p-1-i-j$, Lemma~\ref{lem:sum_poly} implies that if $i+j > p$, then
    \begin{align*}
      \sum_{b = 1}^{p-1} b^{p-1-j} &\sum_{a=0}^{b-1} a^{p-1-i} = \sum_{b = 1}^{p-1} b^{p-1-j} P_i(b) \\
      &= \sum_{b=1}^{p-1} Q_{ij}(b) = \sum_{b=0}^{p-1} Q_{ij}(b) - Q_{ij}(0) = 0.
    \end{align*}
    Thus, all the coefficient $X^i Y^j$ with $i + j > p$ are zero, which means that the total degree of $P_{\LT_\S}(X,Y)$ is at most $p$.

    In addition, we consider the case when $i = j$ and $i,j \le (p-1)/2$.
    Let us consider the following sum
    \begin{align*}
      \sum_{a=0}^{p-1} a^{p-1-i} \sum_{b=0}^{p-1} b^{p-1-i} = 0\,.
    \end{align*}
    We can rewrite it as follows
    \begin{align*}
      \sum_{a=0}^{p-1} & a^{p-1-i} \sum_{b=0}^{p-1} b^{p-1-i} \\
      &= 2\sum_{a=0}^{p-2} a^{p-1-i} \sum_{b=a+1}^{p-1} b^{p-1-i} + \sum_{a=0}^{p-1} a^{2(p-1-i)}\,.
    \end{align*}
    This implies that
    \begin{align*}
      \sum_{a=0}^{p-2} a^{p-1-i} \sum_{b=a+1}^{p-1} b^{p-1-i} = -\frac{1}{2}\sum_{a=0}^{p-1} a^{2(p-1-i)}\,.
    \end{align*}
    Note that the inverse of $2$ is well defined modulo $p$ since $p$ is an odd prime.
    If $i < (p-1)/2$, then Lemma~\ref{lem:sum_poly} says the sum on the right side is zero.
    Thus, the coefficient by $X^i Y^i$, which is exactly the sum on the left side, is equal to zero.
    If $i = (p-1)/2$, the above equality yields that the coefficient by $(XY)^{(p-1)/2}$ is equal to $-(p-1)/2$.
  \end{proof}

\section{Decomposition of $f(X,Y)$ for $3\leq p \leq 7$}
\label{app:decomposition-f}
Let $Z = Y(X-Y)$.
One non-scalar multiplication is needed to compute $Z$.

\textbf{p=3.}
$$f(X,Y) = 2\,.$$

Since the polynomial $f(X,Y)$ is constant, it can be computed without any homomorphic multiplication.

\textbf{p=5.}
$$f(X,Y) = 4X^2 + 4X + Z\,.$$

Two non-scalar multiplications are needed to compute $X^2$ and $Z$.

\textbf{p=7.}
$$f(X,Y) = 1 + 4X(X+1) + 6[X(X+1)]^2 + (X^2+3X)Z + 6Z^2$$

In this case, four non-scalar multiplications are needed (indicated in bold) when rewritten as follows
$$f(X,Y) = 1 + 2(\bm{X}^2+X)\bm{\cdot}[2 + 3(X^2+X)] + \bm{Z}\bm{\cdot}[(X^2+3X) + 6Z]\,.$$

\section{Proof of Theorem~\ref{th:univariate}}
\label{app:proof-lem-univariate}
   Let $Z = X-Y$.
    Thus we can rewrite $Q_{\LT_\S}(X,Y)$ as the univariate function $\chi_{\F_p^-}$, namely
    \begin{align*}
      Q_{\LT_\S}(X,Y) = \chi_{\F_p^-}(Z) = \sum_{a=-\frac{p-1}{2}}^{-1} 1 - (Z - a)^{p-1}.
    \end{align*}
    Thanks to Lemma~\ref{lem:difference_to_p-1}, we can expand $(Z-a)^{p-1}$ and obtain
    \begin{align*}
      \sum_{a=-\frac{p-1}{2}}^{-1} 1 - \sum_{i=0}^{p-1} Z^i a^{p-1-i}
      = \sum_{i=1}^{p-1} Z^i \sum_{a=-\frac{p-1}{2}}^{-1} (-a^{p-1-i}).
    \end{align*}
    If $i$ is even and $i < p-1$, then the $i$th coefficient is equal to
    \begin{align*}
      -\sum_{a=-\frac{p-1}{2}}^{-1} a^{p-1-i} &= -\sum_{a=1}^{\frac{p-1}{2}} a^{p-1-i} = -\frac{1}{2}\sum_{a=-\frac{p-1}{2}}^{\frac{p-1}{2}} a^{p-1-i}.
    \end{align*}
    This coefficient is equal to $0$ for any even $0<i < p-1$ thanks to Lemma~\ref{lem:sum_poly}.
    The $(p-1)$-th coefficient is equal to $-(p-1)/2 = (p+1)/2 \bmod p$.
    If $i$ is odd, then we can rewrite the $i$th coefficient in the following way
    \begin{align*}
      -\sum_{a=-\frac{p-1}{2}}^{-1} a^{p-1-i} &= \sum_{a=1}^{\frac{p-1}{2}} a^{p-1-i},
    \end{align*}
    which finishes the proof.

%%%Local Variables:
%%% mode: latex
%%% TeX-master: "main"
%%% End:


\section{Solution approach using a CNN}
\label{sec:ansatz}
The most successful approach to image recognition based an machine-learing algorithms is the use of \textit{Convolutional Neural Networks} (CNN).
CNNs can use multidimensional input arrays which is their main advantage for working with image data compared to \enquote{normal} fully-connected neural networks,
because the information about the position of each pixel compared to its neighbors is not lost by flattening the pixel-values in an one dimensional array.
This is done by repeatedly applying weight-matrices of size $N_\text{kernel}\times M_\text{kernel}$ called \textit{kernels} on the image, while moving the kernel from one edge of the image to the other,
whereby the steps size (referring to the number of pixel-rows/ -columns that the kernel is moved by in each step) is called \textit{stride}.
This allows CNNs to learn geometric shapes within the images it is trained on.
As we want to classify the images into the eleven flower species, we also add a small fully-connected neural network after the CNN structure, so that the final output of our
neural network are eleven scores for each image which describe how likely the image shows a flower(s) of each of the eleven species.
A subsequent final assignment to one of the species is done by choosing the species with the highest score.

Our problem represents a classification problem with $k = 11$ target classes.
For this type of problem the \textit{categorical cross-entropy} is a good loss function
\begin{align}
    CE = -\frac{1}{n} \sum_{i=1}^n \sum_{k=1}^{11} p_{i,k} \ln{y_{i,k}},
\end{align}
where $n$ is the number of images used for training, $p_{i,k}$ the target vector with $p_{i,k} = 0$ for all classes but the true class $k'$ of the image $i$ ($p_{i,k'} = 1$) and
$y_{i,k}$ the prediction vector with the predicted probabilities of the image being of class $k$.
As our goal is to correctly classify as many images as possible, the \textit{accuracy} of the classification is the most fitting performance metric
\begin{align}
    \text{accuracy} = \frac{n_\text{correctly classified}}{n}.
\end{align}

\subsection{First network and manual hyperparameter tuning}
For constructing, training and applying a neural network we use the \texttt{keras} API for Googles \texttt{TensorFlow} deep-learning library \cite{keras},
while some helper functions from the \texttt{scikit-learn} library are used \cite{scikit-learn}.
Due to our preprocessing, the shape of the input data is $200\times 200\times 3$, where the last dimension represents the three RGB color channels.

We start by trying different structures for the network to get a basic of idea of which type of network works for our problem.
This leads to a network with three convolutional layers, each followed by a MaxPooling layer to stabilize the performance of the network against slight displacement or rotation of certain shapes
in different images.
MaxPooling layers take the maximal value of a $N_\text{pool}\times N_\text{pool}$ subspace of each of the two-dimensional arrays (/image-channels) of the three-dimensional input and
return it as the single output for each of those subspaces.
After the convolutional part of the network we add two hidden fully-connected layers before the output layer (a second hidden layer improves the accuracy compared with only one hidden layer).

As \textbf{activation functions} we use the \textit{ReLU} function for all layers but the output layer, for which we use the \textit{softmax} function.
This choice of activation functions makes it possible to interpret the output scores of the network as actual probabilities of the image showing a flower of the given species.

Regarding the \textbf{number of kernels} in each convolutional layer, an increasing number of kernels seems to work best and we choose to double the number of kernels in each layer
starting with 16 kernels in the first layer.
For the \textbf{MaxPooling} layers we choose $N_\text{pool} = 2$, as any larger number decreases the image size too quickly.
The combination of an increasing number and kernels with pooling layers after each convolutional layer effectively leads to a compression of the first two dimensions (height and width of the image)
of the data while more information gets transformed into the third dimension (number of kernels (RGB channels) in the (input) layer).

For an image size of $200\times 200$ the kernel must also be square ($N_\text{kernel} = M_\text{kernel}$) and a \textbf{kernel size} of $N_\text{kernel} = 4$ seems to work well.
We tried using non-unit \textbf{stride}, but the best result is achieved by using the (default) unit stride which means moving the kernel one pixel-row/ -column at a time across the image.

An overview of the network structure we find based on these manual hyperparameter variations can be found in \autoref{fig:first_model}.

For \textbf{training} the network we use the \textit{Adam} algorithm \cite{adam} with the (default) initial learing rate of 0.001 and a batch size of 128.
We test initial learing rates of 0.01 and 0.0001 and batch sizes of 256 and 64, but these settings do perform significantly worse or slow the training down a lot.

Using the hyperparameter values described so far we achieve an accuracy of 0.753 in training epoch 17 and the loss and accuracy curves can be seen in \autoref{fig:first_curves}.
To validate the performance of the model we split another $\SI{15}{\percent}$ of the images into a validation set.
The whole data is hereby split into $\SI{70}{\percent}$ training set, $\SI{15}{\percent}$ validation set and $\SI{15}{\percent}$ test set.

\subsection{Hyperparameter tuning via sequential grid searches}
After finding a network structure which performs reasonably well, we perform four sequential grid searches.
The first search covers the number of nodes in the two hidden fully-connected layers.
\begin{align}
    N_\text{dense nodes} \in \{64, 96, 128, 160, 192, 224, 256\}
\end{align}
The best accuracy on the validation set (0.770 in training epoch 22) is achieved by using 224 nodes in both layers and the loss and accuracy curves of the three best performing models
(in regards to accuracy on the validation set) can be seen in \autoref{fig:GS1}

In the second search we use more convolutional layers and vary the number of kernels in each layer (still doubling the number from layer to layer).
\begin{align}
    N_\text{conv. layers} &\in \{3, 4, 5, 6\}, &
    \text{num}_\text{kernels} &\in \{8, 16, 24, 32, 40, 48, 64, 80, 96\}
\end{align}
For $N_\text{conv. layers} = 3, 4, 5$ we still use a MaxPooling layer after each layer, but for $N_\text{conv. layers} = 6$ we only include a MaxPooling layer every two layers and
also double the number of kernels only every two layers.
This search is done in three parts, because the first two tries result in the model with the largest $\text{num}_\text{kernels}$ performing the best.
Therefore, it is necessary to increase the range of $\text{num}_\text{kernels}$ to test.
$N_\text{conv. layers} = 5$ is also not present in the first part of the search, but in the second part it performs better than $N_\text{conv. layers} = 4$ (which performed best in the first part)
for all tested number of kernels, which is why only $N_\text{conv. layers} = 5$ is tested in the third part.
The performance of the four best models in the third part of the search can be seen in \autoref{fig:GS2_ext2}.
\begin{figure}
    \centering
    \includegraphics[width=\textwidth]{../data/performance_plots/GS2_ext2.pdf}
    \caption{The loss and accuracy curves for the three best performing models in the third part of the second grid search ($N_\text{conv. layers} = 5$).}
    \label{fig:GS2_ext2}
\end{figure}
Even though the model with $\text{num}_\text{kernels} = 96$ performs the best we choose $\text{num}_\text{kernels} = 48$ as our optimal number of kernels (in the first layer),
because it performs only slightly worse while representing a much less complex model.
On one hand less complex models are generally to be preferred over more complex ones, if they perform equally well (\textit{Occam's razor}) and on the other hand training a model
with as big a capacity as the one with $\text{num}_\text{kernels} = 96$ is so slow, that it would make further hyperparameter tuning extremely difficult, because of our limited computing resources.
In the end, the model with $N_\text{conv. layers} = 5$ and $\text{num}_\text{kernels} = 48$ achieves a validation accuracy of 0.829 in epoch 32.

The third gird search covers the kernel sizes
\begin{align}
    N_\text{kernel} \in \{2, 3, 4, 5, 6\}.
\end{align}
$N_\text{kernel} = 6$ is the biggest kernel size that can be applied on the images in the fifth convolutional layer, because the image resolution is reduced to $6\times 6$ in this layer
due to the five previous MaxPooling layers.
Considering that we use the same kernel size in all convolutional layers, this search covers the whole parameter space for this hyperparameter given the restrictions of our network structure,
as chosen so far.
The performance of the three best models can be seen in \autoref{fig:GS3} and we conclude, that our previous choice of $N_\text{kernel} = 4$ still performs the best
with a validation accuracy of 0.820 in epoch 32.
\begin{figure}
    \centering
    \includegraphics[width=\textwidth]{../data/performance_plots/GS3.pdf}
    \caption{The loss and accuracy curves for the three best performing models in the third gird search.}
    \label{fig:GS3}
\end{figure}
Even though the best model in this search ends up using the exact same hyperparameter values as the optimal model in the previous search, its accuracy is $\SI{0.9}{\percent}$ worse.
This shows, that it is not possible (or at least within the scope of this project) to make the training results completely reproducible, even though we use fixed seeds for functions using
random numbers wherever possible.
It would make sense to use \textit{cross-validation} to mitigate these random fluctuations, however we do not have the computing resources to do so effectively.

By now the structure of the convolutional part of our network has changed so much, that we deem it necessary to reexamine the structure of the fully-connected part of the network,
as it effectively receives quite different inputs compared to when we performed the first gird search.
For this reason, we once again try out different values for $N_\text{dense nodes}$ and check if two hidden layers still provide a better accuracy than one.
\begin{align}
    N_\text{dense layers} &\in \{1, 2\}, &
    N_\text{dense nodes} &\in \{64, 128, 192, 256\}
\end{align}
We omitted a single layer with $N_\text{dense nodes} = 64$ to speed up the search, because this combination performed much worse than others during the manual hyperparameter search.
The three models achieving the best validation accuracies can be seen in \autoref{fig:GS4}.
\begin{figure}
    \centering
    \includegraphics[width=\textwidth]{../data/performance_plots/GS4.pdf}
    \caption{The loss and accuracy curves for the three best performing models in the fourth gird search.}
    \label{fig:GS4}
\end{figure}
The model with a single hidden layer containing $N_\text{dense nodes} = 128$ nodes performs the best with a validation accuracy of 0.824 in epoch 44.

\subsection{Regularization}
Although the accuracy curves do not show signs of significant overfitting (diminishing validation accuracy in later training epochs), the loss curves show, that the validation loss
increases again in later epochs.
Therefore, we now try introducing some regularization to our network in an afford to further improve the networks expected performance on new data by creating redundancy within the
network and forcing it to generalize more.

After implementing one \textit{Dropout} layer between the fifth convolutional and the hidden fully-connected layer and trying different dropout rates $r_\text{dropout}$, we do not see an
improved validation accuracy compared to no dropout.
No dropout layer is implemented between the hidden layer and the output layer, as there are too few weights between those two layers.
Dropout is ineffective in small layers, because too much information gets lost, when some nodes are deactivated.
The next step is using \textit{L2-regularization} in every layer instead and testing different L2-regularization rates
\begin{align}
    r_\text{L2} \in \{0, \num{1e-6}, \num{1e-5}, \num{1e-4}, \num{1e-3}\}.
\end{align}
This shows the best result when using $r_\text{L2} = \num{1e-5}$ as can be seen in \autoref{fig:l2}.

Finally we try combining dropout and L2 regularization by fixing $r_\text{L2} = \num{1e-5}$ and testing the dropout rates
\begin{align}
    r_\text{dropout} \in \{0, 0.1, 0.2, 0.3, 0.4\}.
\end{align}
The loss and accuracy curves for the three best models can be seen in \autoref{fig:l2_dropout}.
The model with $r_\text{dropout} = 0.3$ achieves the best validation accuracy of 0.844 in epoch 57.
\begin{figure}
    \centering
    \includegraphics[width=\textwidth]{../data/performance_plots/dropout_l2.pdf}
    \caption{The loss and accuracy curves for the three best performing models during the search for a combination of L2 and dropout regularization.}
    \label{fig:l2_dropout}
\end{figure}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% UMB-CS114-2015F: Introduction to Programming in Java
% Copyright 2015 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/UMB-CS114-2015F
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\def \topDirectory {.}
\def \texDirectory {\topDirectory/src/main/tex}

\documentclass[12pt,letterpaper,twoside]{article}
\usepackage{\texDirectory/template/style/directives}
\usepackage{\texDirectory/template/style/assignment}
\input{\texDirectory/template/config}

\usepackage{amsmath}

\begin{document}

\doc{title}{Solution to Assignment 2}
\doc{date-pub}{Oct 05, 2015 at 5:30 PM}
\doc{date-due}{Oct 19, 2015 at 5:30 PM}
\doc{points}{8}

\prepare{header}

\section*{Question 1}

Given coordinates of a point and a sphere in tridimensional space, a point can either be inside, outside or on the sphere.

Write a program \texttt{PointAndSphere.java} that takes respectively coordinates of a point and center of a sphere as command line arguments and prompts user for radius of the sphere.
Your program would then determine the location of the point with respect to the sphere.
Following is an example of an accepted output format.

\begin{terminal}
$ java PointAndSphere 1 1 1 0 0 0
Radius of the sphere: 1.7
The point is outside the sphere.
$ java PointAndSphere 1 1 1 0 0 0
Radius of the sphere 1.8
The point is inside the sphere.
\end{terminal}

\subsection*{Solution}

\lstset{language=Java,tabsize=2}
\begin{lstlisting}
import java.util.Scanner;
public class PointAndSphere {
	public static void main(String[] args) {
		Scanner input = new Scanner(System.in);
		double pointX = Double.parseDouble(args[0]);
		double pointY = Double.parseDouble(args[1]);
		double pointZ = Double.parseDouble(args[2]);
		double sphereX = Double.parseDouble(args[3]);
		double sphereY = Double.parseDouble(args[4]);
		double sphereZ = Double.parseDouble(args[5]);
		System.out.printf("Radius of the Sphere: ");
		double radius = input.nextDouble();
		input.close();
		double distance = Math.sqrt(Math.pow(sphereZ - pointZ, 2) + Math.pow(sphereY - pointY, 2) + Math.pow(sphereX - pointX, 2));
		if (distance < radius)
			System.out.println("The point is inside the sphere.");
		else if (distance == radius)
			System.out.println("THe point is on the sphere.");
		else
			System.out.println("The point is outside the sphere.");
	}
}
\end{lstlisting}

\section*{Question 2}

Using Leibniz Formula, the number $\pi$ can be approximated using the series given in Equation \ref{eq1}.

\begin{equation}
\pi = 4 \left( 1 - \frac{1}{3} + \frac{1}{5} + \cdots + \frac{(-1)^{i+1}}{2i-1} \right)
\label{eq1}
\end{equation}

Write a program \texttt{PiProximator.java} that prompts user for a number $i$ and approximates the number $\pi$ using the first $i$ sentences of Equation \ref{eq1}.
Your program should also print the approximation error compared to the \texttt{Math.PI} value.
Following is the expected output of a sample run of your program.

\begin{terminal}
$ javac PiProximator.java
$ java PiProximator
Number of Sentences: 100
Approximate Value: 3.131593
Approximation Error: 0.010000
\end{terminal}

\subsection*{Solution}

\lstset{language=Java,tabsize=2}
\begin{lstlisting}
import java.util.Scanner;
public class PiProximator {
	public static void main(String[] args) {
		Scanner input = new Scanner(System.in);
		System.out.printf("Number of Sentences: ");
		int num = input.nextInt();
		input.close();
		double sum = 0;
		for (int i = 1; i <= num; i ++)
			sum += Math.pow(-1, i + 1) / (2 * i - 1);
		double pi = 4 * sum;
		System.out.printf("Approximate Value: %.6f\n", pi);
		System.out.printf("Approximation Error: %.6f\n", Math.abs(Math.PI - pi));
	}
}
\end{lstlisting}

\section*{Question 3}

Write a program \texttt{PrimePrinter.java} that prompts user for a number $x$ and prints all prime numbers less than or equal to the number $x$.
Your program should function as shown in the example given below.

\begin{terminal}
$ javac PrimePrinter.java
$ java PrimePrinter
Enter Upper Limit: 29
2 3 5 7 11 13 17 19 23 29
\end{terminal}

\subsection*{Solution}

\lstset{language=Java,tabsize=2}
\begin{lstlisting}
import java.util.Scanner;
public class PrimePrinter {
	public static void main(String[] args) {
		Scanner input = new Scanner(System.in);
		System.out.printf("Enter Upper Limit: ");
		int limit = input.nextInt();
		input.close();
		for (int num = 2; num <= limit; num++) {
			boolean isPrime = true;
			for (int i = 2; i < num; i++)
				if (num % i == 0)
					isPrime = false;
			if (isPrime == true)
				System.out.printf("%d ", num);
		}
		System.out.println();
	}
}
\end{lstlisting}

\section*{Question 4}

Write a program \texttt{MatrixFiller.java} that prompts user for a number $x$ between 1 to 9 and generates an $x \times x$ matrix with random elements from 1 to $x^2$.
Following is an expected sample run of your program.

\begin{terminal}
$ javac MatrixFiller.java
$ java MatrixFiller
Size of Matrix: 5
 05 21 07 25 11
 17 02 17 01 07
 25 14 14 08 07
 02 05 01 14 02
 02 14 14 13 07
\end{terminal}

\subsection*{Solution}

\lstset{language=Java,tabsize=2}
\begin{lstlisting}
import java.util.Scanner;
public class MatrixFiller {
	public static void main(String[] args) {
		Scanner input = new Scanner(System.in);
		System.out.printf("Size of Matrix: ");
		int matrixSize = input.nextInt();
		input.close();
		if (0 < matrixSize && matrixSize < 10)
			for (int i = 0; i < matrixSize; i++) {
				for (int j = 0; j < matrixSize; j++) {
					int num = (int) Math.floor(Math.random() * Math.pow(matrixSize, 2)) + 1;
					System.out.printf(" %02d", num);
				}
				System.out.println();
			}
	}
}
\end{lstlisting}

\newpage

\section*{Question 5}
Write a program \texttt{Pyramid.java} that prompts user for a number between 1 to 10 and displays a pyramid of the given format.

\begin{terminal}
$ javac Pyramid.java
$ java Pyramid
Number of Rows: 5
                   1
               1   2   1
           1   2   4   2   1
       1   2   4   8   4   2   1
   1   2   4   8  16   8   4   2   1
\end{terminal}

\subsection*{Solution}

\lstset{language=Java,tabsize=2}
\begin{lstlisting}
import java.util.Scanner;
public class Pyramid {
	public static void main(String[] args) {
		Scanner input = new Scanner(System.in);
		System.out.printf("Number of Rows: ");
		int numRows = input.nextInt();
		input.close();
		for (int i = 0; i < numRows; i++) {
			int numElements = i;
			int numIndents = numRows - i - 1;
			for (int j = 0; j < numIndents; j++)
				System.out.printf("    ");
			for (int j = 0; j < i; j++)
				System.out.printf("%4.0f", Math.pow(2, j));
			for (int j = i; j >= 0; j--)
				System.out.printf("%4.0f", Math.pow(2, j));
			System.out.println();
		}
	}
}
\end{lstlisting}

\end{document}


\documentclass[a4paper]{article}

\usepackage[english]{babel}
\usepackage[utf8x]{inputenc}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage[colorinlistoftodos]{todonotes}

\title{MATH 542 Homework 1}
\author{Saket Choudhary\\skchoudh@usc.edu}

\begin{document}
\maketitle
\section*{Problem 1}
\begin{align*}
\mathbf{1}_n = \begin{pmatrix} 1\\
1\\
\vdots\\
1\\
\end{pmatrix}
\end{align*}
\subsection*{Problem 1.a}
Consider $\mathbf{a}' = \begin{pmatrix}a_1 & a_2 & \dots a_n \end{pmatrix}$

Thus, $$\mathbf{a}'\mathbf{1}_n = \begin{pmatrix}a_1 & a_2 & \dots a_n \end{pmatrix} \times \begin{pmatrix} 1\\
1\\
\vdots\\
1\end{pmatrix} = a_1 + a_2 + a_3 + \dots a_n
$$
$$
1_n'a = \begin{pmatrix} 1 & 1 & \dots & 1 \end{pmatrix} \times \begin{pmatrix} a_1\\
a_2 \\
\vdots \\
a_n
\end{pmatrix} = a_1+a_2+a_3+\dots+a_n =  \mathbf{a'} \mathbf{1_n}
$$
\subsection*{Problem 1.b}
\begin{align*}
A_n I &=  \begin{pmatrix} a_{11} & a_{12} & \dots & a_{1n} \\
a_{21} & a_{22}  & \dots & a_{2n}\\
\vdots\\
a_{n1} & a_{n2} & \dots  & a_{nn}
\end{pmatrix}  \times \begin{pmatrix} 1 \\
1\\
\vdots\\
1
\end{pmatrix} = \begin{pmatrix} \sum_{j=1}^n a_{ij}\\
\sum_{j=1}^n a_{2j}\\
\vdots\\
\sum_{j=1}^n a_{nj}
\end{pmatrix}\\
&= \text{Column vector with row sums of }A
\end{align*}

\subsection*{Problem 1.c}
Row sum of $j^{th}$ column of $A = \sum_{i=1}^na_{ij}$

Column sum of $i^{th}$ row of $A= \sum_{j=1}^n a_{ij}$

\begin{align*}
1_n'A &= \begin{pmatrix} 1 & 1 & \dots 1 \end{pmatrix} \times \begin{pmatrix} a_{11} & a_{12} & \dots & a_{1n} \\
a_{21} & a_{22}  & \dots & a_{2n}\\
\vdots\\
a_{n1} & a_{n2} & \dots  & a_{nn}
\end{pmatrix}\\
&= \begin{pmatrix} \sum_{i=1}^n a_{i1} & \sum_{i=1}^n a_{i2} & \sum_{i=1}^n a_{i3}  & \dots \sum_{i=1}^n a_{in} \end{pmatrix}\\
&= \text{Row vector with elements as column sums of A}
\end{align*}

\section*{Problem 2}

\begin{align*}
a_1 &= \begin{pmatrix}
2\\
3\\
1
\end{pmatrix}\\
A_2 &= \begin{pmatrix}
1 & 2\\
2 & 0\\
0 & 1\\
\end{pmatrix}\\
b_1' &= \begin{pmatrix}
1 & 1 & 1 & 0
\end{pmatrix}\\
B_2 &= \begin{pmatrix}
2 & 1 & 1 & 2\\
2 & 3 & 1 & 2
\end{pmatrix}
\end{align*}

\begin{align*}
AB &= a_1b_1' + A_2B_2
\end{align*}

\begin{align*}
a_1b_1' &= \begin{pmatrix}
2 &   2 & 2 &   0\\
3 &   3  &  3 &   0\\
1  &  1 &   1 &   0
\end{pmatrix}
\end{align*}

Similarly,
\begin{align*}
A_2B_2 &= \begin{pmatrix}
6 &   7 &   3 &   6\\
4 &   2  &  2  &  4\\
2  &  3   & 1 &   2
\end{pmatrix}
\end{align*}

And hence $AB = a_1b_1' + A_2B_2 = \begin{pmatrix}
6 &   7 &   3 &   6\\
4  &  2  &  2  &  4\\
2  &  3 &   1 &   2\\
\end{pmatrix}$
\section*{Problem 3}
$A$ is $n \times p$
\subsection*{Problem 3.a}
$AA'$ is symmetric if $AA' = A'A$

Proof:
$(AA')' = (A')'A' = AA'$ and hence $AA'_{n \times n}$ is symmetric

Similarly, for $A'A$ consider the following:

$(A'A)' = A'A''  = A'A$ and hence $A'A_{p \times p}$ is symmetric too.

\subsection*{Problem 3.b}

$(A'A)_{ij} = \sum_{k=1}^n a_{ki}a_{kj}$ and hence any diagonal element of $A'A$ is a sum of perfect squares(the case when $i=j$ $\implies$ $(A'A)_{ii} = \sum_{k=1}^n a_{ki}^2$)

If $\sum_{k=1}^n a_{ki}^2 =0$ $\implies$ $a_{ki} =0\  \forall 1 \leq k \leq n\  \forall i$ Hence $A$ is a zero matrix.

\section*{Problem 4}
\subsection*{Problem 4.a}

$$A_2 = \begin{pmatrix} 1 & 1 \\
1 & 1\\
1 & 0\\
0 & 0
\end{pmatrix}$$

$$A_3 = \begin{pmatrix} 1 & 1 & 1\\
1 & 1 & 0\\
1 & 0 & 0\\
0 & 0 & 1
\end{pmatrix}$$

$$A_4 = \begin{pmatrix} 1 & 1 & 1\\
1 & 1 & 0\\
1 & 0 & 0\\
0 & 0 & 1
\end{pmatrix}$$
\subsection*{Problem 4.b}
Rank of $V_2$:
\begin{align*}
V_2 &= \begin{pmatrix} 1 & 1 \\
1 & 1\\
1 & 0\\
0 & 0
\end{pmatrix}\\
&\stackrel{\text{C2-C1}}{\longrightarrow}\begin{pmatrix} 1 & 0 \\
1 & 0\\
1 & -1\\
0 & 0
\end{pmatrix}\\
\end{align*}
which cannot be reduced further, hence rank = $min(3,2) = 2$

Rank of  $V_3$:

\begin{align*} A_3 &= \begin{pmatrix} 1 & 1 & 1\\
1 & 1 & 0\\
1 & 0 & 0\\
0 & 0 & 1
\end{pmatrix}\\
&\stackrel{\text{R1-R2}}{\longrightarrow} \begin{pmatrix} 0 & 0 & 1\\
1 & 1 & 0\\
1 & 0 & 0\\
0 & 0 & 1
\end{pmatrix}\\
\end{align*}

Thus rank of $V_3=3$

\subsection*{Problem 4.c}

Basis for $V_2$(maximising zeroes by doing a $C1-C2$ operation):  $\big(\begin{pmatrix} 0 & 1 & 0 \end{pmatrix}'$, $\begin{pmatrix} 1 & 0 & 0\end{pmatrix}'\big)$

Basis for $V_3$ (maximised zeroes): $\big(\begin{pmatrix}1 & 0 & 0 \end{pmatrix}',\begin{pmatrix}1 & 1 & 0 \end{pmatrix}',\begin{pmatrix}0 & 0 & 1 \end{pmatrix}'\big)$

\end{document}


\documentclass{memoir}
\usepackage{linalg}

\begin{document}
\section{Algebraic Operations on $\mathcal{L}(V,W)$}

\begin{defn}[Addition and scalar multiplication on linear maps]
Suppose $S,T \in \mathcal{L}(V,W)$ and $\lambda \in F$. The  \textbf{sum} $S+T$ and \textbf{product} $\lambda T$ are the linear maps from $V \to W$ defined by
\begin{align*}
	(S+T)(v) = Sv + Tv \\
	(\lambda T)(v) = \lambda(Tv)
\end{align*}
for all $v \in V$.
\end{defn}
Composition of maps is also a map; $S(T(V))$ is similar to "multiplying" maps.\\

Moreso, these compositions are associative: ($T_3 \circ (T_2\circ T_1) = (T_3\circ T_2) \circ T_1$). It also has the identity: $Id_W \circ T = T \circ Id_V$.\\

Finally, it is also distributive; $T \circ (S_1+S_2) = T\circ S_1 + T \circ S_2$\\

Say $T \in L(V,W)$, $T:V\to W$. Suppose that $V$ is a finite-dimensional vector space. Then $T$ is \textit{determined} by what it does to a basis of $V$ :\\

Say $v_1,\ldots,v_n$ is a basis for $V$. Suppose we know the output of $T(v_1),\ldots,T(v_n)$. Then let $v \in V$. Know $v = a_1v_1+\ldots+a_nv_n$. Then $T(v) = T(a_1v_1+\ldots+a_nv_n) = T(a_1v_1)+\ldots+T(a_nv_n) = a_1T(v_1) + \ldots + a_n T(v_n)$\\

Therefore, we can construct $T(v)$ for any $v$ from knowledge of how $T$ applies to the basis.
\color{black}
\begin{thm}[Linear maps and basis of domain]
	Suppose that $w_1,\ldots,w_n \in W$ are arbitrary vectors and $v_1,\ldots,v_n$ is a basis of $V$. Then there is \textbf{exactly} one linear map $T:V\to W$ that sends $v_i$ to $w_i$.
\end{thm}
Note that \(\left\{ w_i \right\} \) are not necessarily linearly independent or spanning!

\section{Kernels}
\begin{defn}[Kernel]
	Let $T \in L(V,W)$, $T:V\to W$. The \textbf{kernel} of $T$ is

\begin{align*}
	\textrm{ker}T := \left\{ v \in V \mid T(v) = \vec{0})W \right\}
\end{align*}
\end{defn}
Note: $T(\vec{0}_V) = \vec{0}_W$ and so  $\vec{0}_V \in \text{ker}(T) $ automatically. We also refer to the kernel by Null$(T)$ sometimes.
\begin{exmp}[Examples of kernels]
	\begin{itemize}
		\item The zero map obviously has a kernel of the whole set
		\item The kernel of $\textrm{Id}_V$ is the set $\left\{ \vec{0}_V \right\} $
		\item Differentiation has a kernel consisting of constants
	\end{itemize}
\end{exmp}
\begin{defn}[Injective]
	A map is \textbf{injective} if $T(u) = T(v) \implies u = v$

In other words, different inputs give different outputs.
\end{defn}
\begin{prop}
	Let $T \in L(V,W)$. Then $T$ is injective if and only if $\text{ker}(T) = \left\{ \vec{0}_V \right\} $
\end{prop}
\section{Range and Surjectivity}
\begin{defn}[Range/Image]
	For $T$ a function from $V\to W$, the \textbf{range} or \textbf{image} of $T$ is the subset of $W$ defined by:
	\begin{align*}
		\text{range} T = \textrm{Im}T = \{Tv \mid v \in V \} .
	\end{align*}
\end{defn}
\end{document}


\section{Cohomology, Ext, cup product}
Let $R$ be a ring, probably a PID. It's often a field, but it could be $\Z$. Let $N$ be an $R$-module. Define $S^n(X;N)=\Map(\Sin_n(X),N)$. There's a boundary map $d:S^n(X;N)\to S^{n+1}(X;N)$ that takes a cochain $f$ to the map $df$ defined by $df(\sigma)=(-1)^{n+1}f(d\sigma)$ where $\sigma\in \Sin_{n+1}(X)$. Now, $d^2=0$, so $ H^n(X;N):= H^n(S^\ast(X;N))$. This is a contravariant functor from $\mathbf{Top}$ to $\mathbf{Ab}$ and it's covariant in the coefficients.

When $n=0$, you have $0\to S^0(X;N)\xrightarrow{d} S^1(X;N)$. Thus $ H^0(X;N)=\ker d$. Well, $S^0(X;N)=\Map(X,N)$, and $d$ sends a $0$-cochain $f$ to $\sigma\mapsto\pm f(d\sigma)=\pm(f(\sigma(0))-f(\sigma(1)))$. So a function is in the kernel of $d$ if its values on the ends of any path is the same. Thus $ H^0(X;N)=\Map(\pi_0(X),N)$.

We also talked about the Kronecker pairing. This gave an evaluation $ H^n(X;N)\otimes_R H_n(X;R)\to N$. Taking the adjoint gives a map $ H^n(X;N)\xrightarrow{\beta}\Hom_R( H_n(X;R),N)$. We can try to understand cohomology in terms of homology. $\beta$'ll often be an isomorphism, but not always.
\begin{theorem}[UCT for cohomology]
There is a natural sexseq:
\begin{equation*}
0\to\Ext^1_R( H_{n-1}(X;R),N)\to H^n(X;N)\xrightarrow{\beta}\Hom_R( H_n(X;R),N)\to 0
\end{equation*}
that splits, but not naturally. This also holds for relative cohomology.
\end{theorem}
I will tell you what $\Ext$ means now. I will prove this on Friday.

The problem that arises is that $\Hom_R(-,N):\mathbf{Mod}_R\to\mathbf{Mod}_R$ is not exact. More precisely, it preserves right exact sequences, but not left exact sequences. Consider $M^\prime\xrightarrow{i} M\xrightarrow{p} M^{\prime\prime}\to 0$ an exact sequence of $R$-modules. This gives a sequence $0\to \Hom_R(M^{\prime\prime},N)\to\Hom_R(M,N)\to \Hom_R(M^{\prime},N)$. If I have $f:M^{\prime\prime}\to N$, and I compose with $p$ to get a zero map, then is $f$ zero? Well, yes, because $p$ is surjective. Now suppose I have $g:M\to N$, such that $i\circ g=0$. So it facts through the cokernel, and you get a unique factorization $M^{\prime\prime}\to N$ (unique because $M\to M^{\prime\prime}$ is surjective).

Suppose I have an injection $0\to M^\prime\to M$. Is $\Hom(M,N)\to\Hom(M^\prime,N)$ surjective? If I have some map $M^\prime\to N$ and $M^\prime\hookrightarrow N$, then does this extend to a map $M\to N$? No! For example, if you have $1:\Z/2\Z\to\Z/2\Z$ and $\Z/2\Z\hookrightarrow\Z/4\Z$, then you can't lift to a map $\Z/4\Z\to\Z/2\Z$. This works, though, if the sexseq splits.

Homological algebra now comes to the rescue! Pick a free resolution of $M$ given by $\cdots\to F_2\to F_1\to M\to F_0\to 0$. If I apply $\Hom$, I get a chain complex $0\to \Hom(F_0,N)\to \Hom(F_1,N)\to \Hom(F_2,N)\to\cdots$.
\begin{definition}
Define $\Ext_R^n(M,N)= H^n(\Hom_R(F_\bullet,N))$ as the $n$-dimensional homology of this chain complex.
\end{definition}
\begin{remark}
If $R$ is a PID, then $\Ext^n=0$ if $n>1$. If $R$ is a field, then $\Ext^n=0$ for $n>0$. Also, $\Ext$ is well-defined and functorial (by the fundamental theorem of homological algebra). Another important point is that $\Hom_R(-,N)$ takes chain homotopies to chain homotopies. This is a pretty important thing, and is something to think about for a minute. This is because if I have $M^\prime\to M$, I get an induced map $\Hom_R(M^\prime,N)\leftarrow\Hom_R(M,N)$, and this is actually an $R$-module map, and in particular, additive. And this means that the $dh-hd=f_1-f_0$ is preserved. Lastly, if $M$ is free or projective, then $\Ext^n(M,-)=0$ for $n>0$. In addition, $\Ext^0(M,N)=\Hom_R(M,N)$.
\end{remark}
Recall the trick that if I have a sexseq $0\to A\to B\to C\to 0$, and I have free resolutions $F^\prime_\bullet\to A$ and $F^{\prime\prime}_\bullet\to 0$, I can get a free resolution $F_\bullet\to B$ to get a sexseq $0\to F^\prime_\bullet\to F_\bullet\to F^{\prime\prime}_\bullet\to 0$. I can now apply $\Hom_R(-,N)$ to get a sexseq of cochain complexes, because this sequence splits in any given degree. Thus, I get a lexseq:
\begin{equation*}
\xymatrix{ & & 0\ar[dll]\\
\Hom_R(M^{\prime\prime},N)\ar[r] & \Hom_R(M,N)\ar[r] & \Hom_R(M^{\prime},N)\ar[dll]\\
\Ext^1_R(M^{\prime\prime},N)\ar[r] & \Ext^1_R(M,N)\ar[r] & \Ext^1_R(M^{\prime},N)\ar[dll]\\
\cdots & &}
\end{equation*}
In this sense, $\Ext$ is like a cohomology theory for $R$-modules.

Let us use this to make a calculation.
\begin{example}
Let $R=\Z$, and look at the sexseq $0\to \Z\xrightarrow{k}\Z\to\Z/k\Z\to 0$. Here $N$ is some abelian group. So, our lexseq will look like:
\begin{equation*}
\xymatrix{ & & 0\ar[dll]\\
\Hom_R(\Z/k\Z,N)\ar[r] & \Hom_R(\Z,N)=N\ar[r] & \Hom_R(\Z,N)=N\ar[dll]\\
\Ext^1_R(\Z/k\Z,N)\ar[r] & 0\ar[r] & 0\ar[dll]\\
\cdots & &}
\end{equation*}
The map $N\to N$ in this lexseq is multiplication by $k$. Thus $\Hom(\Z/k\Z,N)=\ker(N\xrightarrow{k}N)$. And, well, $\Ext^1_\Z(\Z/k\Z,N)=N/kN$.
\end{example}
Let's get some consequences of cohomology from the UCT. Even independent of that, we get some properties.
\subsection{Properties of cohomology}
\begin{enumerate}
\item It's homotopy invariant. This means that if $f_0\sim f_1:(X,A)\to (Y,B)$, then $ H^\ast(X,A;N)\xleftarrow{f_0^\ast,f_1^\ast} H^\ast(Y,B;N)$ are equal. I can't use the UCT to address this because the UCT only tells you that things are isomorphic (use the 5-lemma). But we did establish a chain homotopy $f_{0,\ast}\sim f_{1,\ast}:S_\ast(X,A)\to S_\ast(Y,B)$, and applying $\Hom$ still retains this chain homotopy, and hence you get the same map on cohomology.
\item Excision. If $U\subseteq A\subseteq X$ such that $\overline{U}\subseteq\mathrm{Int}(A)$, then $ H^\ast(X,A;N)\leftarrow H^\ast(X-U,A-U;N)$ is an isomorphism. This follows from the UCT (in the relative form, which is also true).
\item Mayer-Vietoris sequence. If I have $A,B\subseteq X$ such that their interiors cover $X$, then I have an lexseq:
\begin{equation*}
\xymatrix{ & & \cdots\ar[dll]\\
 H^n(X;N)\ar[r] & H^n(A;N)\oplus H^n(B;N)\ar[r] & H^n(A\cap B;N)\ar[dll]\\
 H^{n+1}(X;N)\ar[r] & \cdots & }
\end{equation*}
\end{enumerate}


\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage[english]{babel}
\usepackage{amsmath,amssymb,amsthm}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{corollary}{Corollary}[theorem]
\newtheorem{lemma}[theorem]{Lemma}
\title{Mathematics Handout - Logaritmics}
\author{Daniel Frederico Lins Leite}
\date{July 2016}
\begin{document}
\section{Introduction}
\maketitle

\[{x_n}\] is a sequence of real numbers
Th sequence converge if:
$$\exists a \in Re$$
$$\exists \epsilon \in R such that \epsilon > 0$$
$$\exists N_a \in I such that N_a > 0$$
$$\forall n > N_a, |a - x_n| < \epsilon$$

if for example exists another
$$\exists b \in Re$$ such that
$$\exists N_b \in I such that N_b > 0$$
$$\forall n > N_b, |b - x_n| < \epsilon$$

then

$$|a-b| = |a - x_n + x_n - b|$$

Given that the $$R^n$$, the space of the elements of the sequence is a normed vector space, we can apply the Triangle Inequality. So

$$|a - x_n + x_n - b| <= |a - x_n| + |x_n - b|$$

given that $$|x_n - b|$$ is a even function
$$f(x) = f(-x)$$ we have that
$$|x_n - b| = -1 * |x_n - b| = |-x_n + b| = |b - x_x|$$

so we have

$$|a - x_n| + |x_n - b| < |a - x_n| + |b - x_n| <= \epsilon + \epsilon <= 2\epsilon$$

So

$$ 0 <= |a - b| <= 2\epsilon$$

So $|a-b|$ is squeezed between this two function. Given that this is true for every $\epsilon$, if $\epsilon -> 0$
we will have that:
$$|a-b| = 0$$ and $$a=b$$

The value a is called the limit of the sequence.

The

$$S = \sum_{i=0}^{\inf} {x_i}$$ is called the series associated with se sequence.

The partial sum is

$$S_n = \sum_{i=0}^{n} {x_i}$$

So

${S_n}$ is also a sequence. In the same way it can converge and have a limit.
If the sequence converge, we say that the series converge and that the limit is

$$limit of {S_n} =  \sum_{i=0}^{\inf} {x_i}$$

If we have two convergent series:

$$S_1 = \sum_{i=0}^{\inf} {a_i}$$
$$S_2 = \sum_{i=0}^{\inf} {b_i}$$

then

$$\sum_{i=0}^{\inf} {(a_i + b_i)} = \sum_{i=0}^{\inf} {a_i} + \sum_{i=0}^{\inf} {b_i}$$

\end{document}


\section{Fundamental Concepts}

\bx{
  We will check $\cup, \cap$ in DeMorgan's laws.

  Let's use
  \begin{itemize}
    \item $A = \pbrac{1, 2, 3, 4}$
    \item $B = \pbrac{-1, 2, 3, 5}$
    \item $C = \pbrac{3, 9, 11}$
  \end{itemize}

  Check
  \begin{align*}
    A - (B \cup C) &= \pbrac{1, 2, 3, 4}  - \pbrac{-1, 2, 3, 5, 9, 11}\\
    &= \pbrac{1, 4} \\
    &= (A - B) \cap (A - C) \\
    &= \pbrac{1, 4} \cap \pbrac{1, 2, 4} = \pbrac{1, 4}
  \end{align*}

  \begin{align*}
    A - (B \cap C) &= \pbrac{1, 2, 3, 4}  - \pbrac{3}\\
    &= \pbrac{1, 2, 4} \\
    &= (A - B) \cup (A - C) \\
    &= \pbrac{1, 4} \cup \pbrac{1, 2, 4} = \pbrac{1, 2, 4}
  \end{align*}
}

\bx{
  \ea{
    \item $\implies$ is true. $\impliedby$ is not true, consider $A = \pbrac{1, 2, 3}, B = \pbrac{1, 3}, C = \pbrac{2}$.
    \item $\implies$ is true. $\impliedby$ is not true, consider $A = \pbrac{1, 2, 3}, B = \pbrac{1, 3}, C = \pbrac{2}$.
    \item True.
    \item $\implies$ is not true. Consider $A = \pbrac{1} \subset B = \pbrac{1, 2}, C = \emptyset$. $\impliedby$ is true.
    \item Not true. Consider $A = \pbrac{1}, B = \pbrac{2}$. I think $\subset$ works.
    \item Not true. Consider $A = \pbrac{1, 2}, B = \pbrac{2, 3}$. LHS is equivalent to $A$, so this should be $\supset$.
    \item True.
    \item $\supset$
    \item True.
    \item True.
    \item Not true, if $A = \emptyset$ for example, we have $(A \times B)
    \subset (C \times D) = \emptyset \subset (C \times D)$, but we can set $B$
    to whatever and this statement is still true, so we can make $B$ have an
    element that is not in $D$, and therefore $B \not\subset D$.
    \item True.
    \item $\subset$
    \item $\subset$
    \item True.
    \item I think this is true at first glance...at least $\subset$ looks good.
    \item $\supset$
  }
}

\bx{
  \ea{
    \item
    \begin{itemize}
      \item \textbf{Original}: If $x < 0$ then $x^2 - x > 0$. True.
      \item \textbf{Contrapositive}: If $x^2 - x \leq 0$ then $x \geq 0$. True.
      \item \textbf{Converse}: If $x^2 - x > 0$ then $x < 0$. False.
    \end{itemize}

    \begin{figure}[H]
      \centering
      \def\domainSize{3}
      \begin{tikzpicture}
        \begin{axis}[
          axis y line = middle,
          axis x line = middle,
        ]

        \addplot[
          domain=-\domainSize:\domainSize,
          samples=100
        ]{
          x^2 - x
        };

        \end{axis}
      \end{tikzpicture}
      \caption{Showing how to visualize where $x^2 - x > 0 $}
      \label{chap1:fig:parabola}
    \end{figure}

    \item
    \begin{itemize}
      \item \textbf{Original}: If $x > 0$ then $x^2 - x > 0$. False.
      \item \textbf{Contrapositive}: If $x^2 - x \leq 0$ then $x \leq 0$. False.
      \item \textbf{Converse}: If $x^2 - x > 0$ then $x > 0$. False.
    \end{itemize}
  }
}

\bx{
  \ea{
    \item $\exists a \in A$ such that $a^2 \not\in B$
    \item $\forall a \in A, a^2 \not\in B$
    \item $\exists a \in A$ such that $a^2 \in B$.
    \item $\exists a \not\in A$ such that $a^2 \not\in B$.
  }
}

\bx{
  \ea{
    \item True. True.
    \item False. True.
    \item True. False.
    \item True. True.
  }
}

\bx{
  \TODO too lazy
}

\bx{
  \begin{align*}
    D &= A \cap (B \cup C) \\
    E &= (A \cap B) \cup C \\
    F &= A
  \end{align*}

  For $F$, I was thinking $x \in B \implies x \in C$ means that either $x \in B$
  and $x \in C$, or $x \not\in B$ and $x$ can be anything. This sounds like $x$
  can be anything in the second case, so we have $A \cap \mathcal{U} = A$.
}

\bx{
  $A = \pbrac{0, 1}$. $\mathcal{P}(A) = \pbrac{\emptyset, \pbrac{0}, \pbrac{1}, \pbrac{0, 1}}$.

  If $A$ has one element, $\abs{\mathcal{P}(A)} = 2$. It is called the power set
  because it contains all the subsets of $A$, and that $\abs{\mathcal{P}(A)} =
  2^\abs{A}$.
}

\bx{
  \TODO: You can honestly find this everywhere online. Standard proof.
}

\bx{
  \ea{
    \item $\mathbb{Z} \times \mathbb{R}$
    \item $\mathbb{R} \times \left(0, 1\right]$
    \item No. You can do a contradiction proof with cases that the first and second set are disjoint, and then that they are not disjoint.
    \item Yes, $\pa{\mathbb{R} - \mathbb{Z}} \times \mathbb{Z}$
    \item No. The cartesian product will produce a box, while this set is a circle.
  }
}


\input{../header_function}

%---------- start document ---------- %
 \section{factor.find -- find a factor}\linkedzero{factor.find}
%
\quad All methods in this module return one of a factor of given integer.
If it failes to find a non-trivial factor, it returns \(1\).
Note that \(1\) is a factor anyway.

\param{verbose} boolean flag can be specified for verbose reports.
To receive these messages, you have to prepare a logger (see \linklibrary{logging}).

  \subsection{trialDivision -- trial division}\linkedone{factor.find}{trialDivision}
   \func{trialDivision}
   {%
     \hiki{n}{integer},\ %
     **\param{options}
   }{%
     \out{integer}
   }\\
   \spacing
   % document of basic document
   \quad Return a factor of \param{n} by trial divisions.\\
   \spacing
   % input, output document
   \quad \param{options} can be either one of the following:
   \begin{enumerate}
   \item \param{start} and \param{stop} as range parameters.
         In addition to these, \param{step} is also available.
   \item \param{iterator} as an iterator of primes.
   \end{enumerate}
   If \param{options} is not given, the function divides \param{n} by primes from \(2\) to the floor of the square root of \param{n} until a non-trivial factor is found.\\
   \quad \param{verbose} boolean flag can be specified for verbose reports.\\
%
  \subsection{pmom -- $p-1$ method}\linkedone{factor.find}{pmom}
   \func{pmom}{%
     \hiki{n}{integer},\ %
     **\param{options}
   }{%
     \out{integer}
   }\\
   \spacing
   % document of basic document
   \quad Return a factor of \param{n} by the \(p-1\) method.\\
   \spacing
   % added document
   \quad The function tries to find a non-trivial factor of \param{n}
   using Algorithm 8.8.2 (\(p-1\) first stage) of \cite{Cohen1}.
   In the case of \(n = 2^{i}\), the function will not terminate.
   Due to the nature of the method, the method may return the
   trivial factor only.\\
   \spacing
   % input, output document
   \quad \param{verbose} Boolean flag can be specified for verbose reports,
   though it is not so verbose indeed.\\
%
   \subsection{rhomethod -- $\rho$ method}\linkedone{factor.find}{rhomethod}
   \func{rhomethod}{%
     \hiki{n}{integer},\ %
     **\param{options}
   }{%
     \out{integer}
   }\\
   \spacing
   % document of basic document
   \quad Return a factor of \param{n} by Pollard's \(\rho\) method.\\
   \spacing
   % added document
   The implementation refers the explanation in \cite{Pomerance}.
   Due to the nature of the method, a factorization may return the
   trivial factor only.\\
   \spacing
   % input, output document
   \quad \param{verbose} Boolean flag can be specified for verbose reports.\\
\begin{ex}
>>> factor.find.trialDivision(1001)
7
>>> factor.find.trialDivision(1001, start=10, stop=32)
11
>>> factor.find.pmom(1001)
91
>>> import logging
>>> logging.basicConfig()
>>> factor.find.rhomethod(1001, verbose=True)
INFO:nzmath.factor.find:887 748
13
\end{ex}%Don't indent!(indent causes an error.)
\C

%---------- end document ---------- %

\input{../footer}


\chapter{Clustering}
\newpage

\section{K-Means}
\begin{itemize}
    \item Dataset: $H=\{x^{(1)},x^{(2)},\dots,x^{(n)}\}$
    \item $d$ dimensions
    \item Want to divide $H$ into disjoint subsets $\{H_1,H_2,\dots,H_c\}$
\end{itemize}

\subsection{Objective Function}
Find an assignment such that the samples in the same class are as close to each other as possible.
\[ J_c \triangleq \sum_{i=1}^c\sum_{x\in H_i} \|x-m_i\|^2 \]
where $m_i$ is the centroid of each class.

\subsection{Before K-Means}
Let
\[ J_i = \sum_{x\in H_i}\|x-m_i\|^2 \]

\subsubsection{Framework}
\begin{enumerate}
    \item Initialize
    \item Re-assign data samples to make the loss smaller.
\end{enumerate}

\subsubsection{Update Rule}
Suppose a sample $\hat{x}$ is moved from $H_i$ to $H_j$. Let $n_j$ be the number of samples in $H_j$.
\[ m^*_j = \frac{n_jm_j + \hat{x}}{n_j+1} = m_j - m_j + \frac{n_jm_j + \hat{x}}{n_j+1} = m_j + \frac{\hat{x}-m_j}{n_j+1} \]
\[ m^*_i = \frac{n_im_i - \hat{x}}{n_i-1} = m_i + \frac{m_i - \hat{x}}{n_i-1} \]

\begin{align*}
    J_j^* & = \sum_{x\in H_j}\| x-m_j^* \|^2 + \| \hat{x} - m_j^* \|^2                                                \\
          & = \sum_{x\in H_j}\|x-m_j-\frac{\hat{x}-m_j}{n_j+1}\|^2 + \| \frac{n_j}{n_j+1}(\hat{x}-m_j) \|^2           \\
          & = \sum_{x\in H_j} \| x - m_j \|^2 - \frac{2}{n_j+1}\sum_{x\in H_j}(\hat{x}-m_j)^T(x-m_j)                  \\
          & \quad + \frac{1}{(n_j+1)^2}\sum_{x\in H_j}\| \hat{x} -m_j \|^2 + \frac{n_j^2}{(n_j+1)^2}\|\hat{x}-m_j\|^2 \\
          & = J_j + \frac{n_j}{(n_j+1)^2}\|\hat{x}-m_j\|^2 + \frac{n_j^2}{(n_j+1)^2}\|\hat{x}-m_j\|^2
\end{align*}
Notice that the second term is $0$ and the $\|\hat{x}-m_j\|^2$ in the first term is constant w.r.t. $x$ in the summation.

Therefore
\[ J^*_j = J_j + \frac{n_j}{n_j+1}\|\hat{x}-m_j\|^2 \]
\[ J^*_i = J_i - \frac{n_i}{n_i-1}\|\hat{x}-m_i\|^2 \]

\subsubsection{Convergence}
Always converges, because the loss function is always decreasing and has a lower bound $0$.

But it is SLOW.

\subsection{K-Means}
\subsubsection{Procedure}
\begin{enumerate}
    \item Update centroids
    \item Re-assign points
\end{enumerate}

\subsubsection{Mathematical Framework}
Let $r_{ik} \in \{0,1\}$ denote that the $i$-th sample is classified into the $k$-th class. $\sum_kr_{ik} = 1$. Then the loss function is
\[ J = \sum_i \sum_k \| x_i - m_k \|r_{ik} \]

\subsubsection{Convergence Analysis}
In E-Step, the loss always goes down because each sample finds the nearest centroid.
\[ J_c^{(old)} = \sum_i\sum_kr_{ik}^{(old)}\|x_i-m_k\| \ge \sum_i\sum_k\|x_i-m_k\|r_{ik}^{(new)} = J_c^{(new)} \]

In M-Step,
\begin{align*}
    J_c^k(old) & = \sum_{i\in H_k}\|x_i - m_k^{(old)}\|^2                               \\
               & = \sum_{i\in H_k}\| x_i - m_k^{(new)} + m_k^{(new)} - m_k^{(old)} \|^2 \\
               & = \sum_{i\in H_k} \|x_i - m_k^{(new)}\|^2                              \\
               & \quad + \sum_{i\in H_k}\|m_k^{(old)} - m_k^{(new)}\|^2                 \\
               & \quad + \sum_{i\in H_k}(x_i-m_k^{(new)})^T(m_k^{(old)}-m_k^{(new)})
\end{align*}
The first term is $J_c^k(new)$, the second term is non-negative and the third term is zero.

\subsubsection{Remarks}
\begin{itemize}
    \item Generally K-Means has good performance.
    \item Initialization matters.
    \item The choice of $k$ matters.
\end{itemize}

\subsection{Agglomerative Clustering}
\begin{enumerate}
    \item Initialize: everypoint is its own cluster.
    \item Find the most similar pair of clusters.
    \item Merge.
\end{enumerate}

\section{Gaussian Mixture Model}
    K-Means can be easily affected by outliers.

    \subsection{Multivariate Gaussian Distribution}
        \[ p(x) = \frac{1}{(2\pi|\Sigma|)^{d/2}}\exp\left\{ -\frac{1}{2}(x-\mu)^T\Sigma^{-1}(x-\mu) \right\} \]

    \subsection{Gaussian Mixture Model}
        Suppose the data has a distribution that can be modeled by the weighted sum of $K$ Gaussian distributions. Let $\pi_k = p(x=k)$ be the likelihood of the $k$-th Gaussian.
        \[ p(x) = \sum_{k=1}^Np(x=k)p(x|k) = \sum_{k=1}^K\pi_k \mathcal{N}(x|\mu_k,\Sigma_k) \]

        So the parameters of a GMM are
        \begin{itemize}
            \item $\pi_k$
            \item $\mu_k$
            \item $\Sigma_k$
        \end{itemize}

        Given the parameters of a GMM, the probabilities can be calculated.
        \begin{itemize}
            \item Prior: $p(x=k)=\pi_k$
            \item Likelihood: $p(x|x=k)=\mathcal{N}(x|\mu_k,\Sigma_k)$
            \item Joint Distribution: $p(x,x=k)=p(x=k)p(x|x=k)=\pi_k\mathcal{N}(x|\mu_k,\Sigma_k)$
            \item Evidence: $p(x) = \sum_{k=1}^Kp(x,x=k)=\sum_{k=1}^K\pi_k\mathcal{N}(x|\mu_k,\Sigma_k)$
            \item Posterior: $p(x=k|x)=\frac{p(x,x=k)}{p(x)}=\frac{\pi_k\mathcal{N}(x|\mu_k,\Sigma_k)}{\sum_{i=1}^K\pi_i\mathcal{N}(x|\mu_i,\Sigma_i)}$
        \end{itemize}

    \subsection{Estimating Parameters}
        Clustering using a GMM is simply estimating the parameters of the model.

        Suppose we have a dataset $D = \{x^{(1)},x^{(2)},\dots,x^{(N)}\}$. We maximize the log likelihood
        \[ \mathcal{L}(\theta) = \ln p(D|\theta) = \sum_{n=1}^N\ln\left\{\sum_{k=1}^K\pi_k\mathcal{N}(x^{(n)}|\mu_k,\Sigma_k)\right\} \]

        However, traditional MLE methods do not work due to the complexity of $\mathcal{L}$. To illustrate this, we take derivative w.r.t. $\mu_k$
        \[ \frac{\partial \mathcal{L}}{\partial \mu_k} = -\sum_{n=1}^N\frac{\pi_k\mathcal{N}(x^{(n)}|\mu_k,\Sigma_k)}{\sum_j\pi_j\mathcal{N}(x^{(n)}|\mu_j,\Sigma_j)}\Sigma_k^{-1}(x_n-\mu_k) \]
        but we cannot derive a closed-form solution of $\mu_k$.

        Instead, this is an exponential function that can be solved by iterative numerical methods, let
        \[ \gamma(z_{nk}) = \frac{\pi_k\mathcal{N}(x^{(n)}|\mu_k,\Sigma_k)}{\sum_j\pi_j\mathcal{N}(x^{(n)}|\mu_j,\Sigma_j)} \]
        \[ N_k = \sum_{n=1}^N \gamma(z_{nk}) \]

        To solve for $\mu_k$, we run multiple iterations, in each iteration, we first fix $\theta = (\pi_k,\mu_k,\Sigma_k)$ and update $\gamma(z_{nk})$, and then fix $\gamma(z_{nk})$ and update $\theta$.

        \subsubsection{Estimating Mean and Covariance Matrix}
        Using previous conclusions,
        \[ \mu_k = \frac{1}{N_k}\sum_{n=1}^N\gamma(z_{nk})x^{(n)} \]
        Similarly if we solve for $\Sigma_k$,
        \[ \Sigma_k = \frac{1}{N_k}\sum_{n=1}^N\gamma(z_{nk})(x^{(n)}-\mu_k)(x^{(n)}-\mu_k)^T \]

        \subsubsection{Estimating Mixture Coefficients}
        Notice that $\sum_k\pi_k = 1$, so maximizing w.r.t. $\pi_k$ is an ECP.

        The solution is given by
        \[ \pi_k = \frac{N_k}{N} \]

\section{Spectral Clustering}
    K-Means and GMM may have poor performance when the data is non-convex.

    \subsection{Graph-based Clustering}
        We use a graph $G=(V,E)$ to represent the dataset. Each sample is a node in $V$ and each edge has a weight $w_{ij}=f(d(i,j))$.

        \begin{enumerate}
            \item Construct the graph
            \item Perform edge cuts.
        \end{enumerate}

    \subsection{Graph Cut}
        \subsubsection{Definitions and Notations}
            \begin{itemize}
                \item Degree of a node $d_i = \sum_{j\in \mathcal{N}(i)}w_{ij}$.
                \item Degree of a subgraph $A_i = \sum_{i \in A}d_i$. There are two cases.
                    \begin{itemize}
                        \item $i \in A$, $j \in A$. In this case $w_{ij}$ is counted twice.
                        \item $i \in A$, $j \notin A$. In this case $w_{ij}$ is counted once.
                    \end{itemize}
                \item Cut of a graph $Cut(A, B) = \sum_{i \in A}\sum_{j \in B}w_{ij}$.
            \end{itemize}

        We minimize
        \[ \min Cut(A_1,A_2,\dots,A_k) \]

        However, the algorithm may favor isolated clusters and outliers.

        Can be mitigated by normalizing edge cuts.
        \[ NCut(A,B) = \frac{Cut(A,B)}{d_A} + \frac{Cut(A,B)}{d_B}\]
        Works. But it's NP-hard.

    \subsection{Optimizing Graph Cuts}
        \subsubsection{More Notations}
            \begin{itemize}
                \item Degree matrix $D_{ij} = \text{$d_i$ if $i=j$ else $0$}$
                \item $L=D-W$
                \item Define an indicator vector $x$. $x_i = 1$ if $i \in A$ and $x_i=0$ if $i \notin A$, i.e. $i \in B = V-A$. $x$ is unknown and we want to find $x$.
            \end{itemize}

        We can represent $d_A$ and $d_B$ in matrix form
        \[ d_A = \sum_{i\in A}d_i = \sum_i d_ix_i = x^TDx \]
        \[ d_B = \bar{x}^TD\bar{x} \]
        where $\bar{x}$ is the bitwise NOT of $x$.

        \[ Cut(A, V-A) = \sum_{i\in A}\sum_{j\in B}w_{ij} = \sum_{i \in A}d_i - \sum_{i \in A}\sum_{j \in A}w_{ij} = x^TDx - x^TWx = x^TLx \]

        Therefore
        \begin{align*}
            \min_x &\quad x^TLx\\
            \text{s.t.} &\quad x^TDx \ge \delta
        \end{align*}

        Can be solved by Lagrangian multipliers
        \[ \mathcal{L}(x, \lambda_A, \lambda_B) = x^TLx - \lambda_A(x^TDx-\delta) \]
        \[ D^{-1}Lx = \lambda_A x \]
        So we find the eigenvector of $D^{-1}L$ and take approximations to make elements of $x$ either $0$ or $1$.

        \subsubsection{Procedure}
            \begin{enumerate}
                \item Normalize $L = D^{-1/2}LD^{-1/2}$
                \item Compute the smallest $k_1$ eigenvalues and corresponding eigenvectors.
                \item Use eigenvectors to construct $N$ by $k_1$ matrix $P$.
                \item Use $P$ as the new input of some other clustering algorithm, where each sample has $k_1$ dimension.
            \end{enumerate}

        \subsubsection{Strengths and Weaknesses}
        \begin{itemize}
            \item Can handle non-convex datasets.
            \item Runs into trouble when new data samples comes. Cannot be easily extended.
        \end{itemize}


%%=========================
%% Section 1.02: Origins of the Green's Functions and Roots of P
%%=========================

\documentclass[../dissertation.tex]{subfiles}

\begin{document}
\section{Motivation of the Green's Functions}\label{sec1:RootsOfP}

To motivate our choice of Green's function for this linear spectral
problem, suppose there exists a function $G$ satisfying
$L_\delta(G^+)(x) = \delta_0(x)$,
where $\delta_0(x)$\label{sym:dirac} denotes the Dirac delta function (not to be confused with
the parameter $\delta$). Formally, by taking the Fourier
transform\label{sym:fourier} of both
sides of $L_\delta(G^+)(x) = \delta_0(x)$, we have
\[
	1
		= \xi \, \wh G^+ - \zeta \( \wh G^+ - \wh G^- \)
		= \big[ \xi - \zeta \(1 - e^{-2\delta\xi}\) \big] \wh G^+
		= p(\xi; \zeta, \delta) \, \wh G^+,
\]
where
\begin{align*}
	p(\xi; \lambda, \delta)
		&= \xi - \zeta\(1 - e^{-2\delta \xi}\) \\
		&= \( \frac{\xi}{1 - e^{-2\delta \xi}} - \zeta \)\(1 - e^{-2\delta \xi}\) \\
		&= \big(\zeta(\xi) - \zeta(\lambda)\big)\(1 - e^{-2\delta \xi}\).
\end{align*}
However, this approach is somewhat problematic, given that the function $p$ has
roots $\xi = 0$ and $\xi = \lambda$ along the real line. So, instead defining
a single Green's function for the spectral problem \eqref{eq:JostDE} based on
taking the inverse Fourier transform of $1/p$, we instead define two different
Green's functions $G_L^+$ and $G_R^+$ based on taking two different ``Fourier
inverse like'' transforms of $1/p$ for which the respective contours of
integration avoids the roots of $p$. Specifically,
	\begin{align*}
		G_L^+(x; \lambda, \delta)
			&:=
				\frac{1}{2\pi}
				\int_{\Gamma_L}
					e^{i\xi x} \frac{1}{p(\xi; \lambda, \delta)} \,
				d\xi \\
		G_R^+(x; \lambda, \delta)
			&:=
				\frac{1}{2\pi}
				\int_{\Gamma_R}
					e^{i\xi x} \frac{1}{p(\xi; \lambda, \delta)}
				\, \mathrm{d}\xi
	\end{align*}
where the contour ${\Gamma_L}$ bypasses the roots $\xi = 0$ and $\xi = \lambda$
from below, and the contour ${\Gamma_R}$ bypasses the roots $\xi = 0$ and
$\xi = \lambda$ from above, as mentioned in the previous section.

Since $p(\xi; \lambda)$ can be rewritten as
\[
	p(\xi; \lambda)
		= \(\frac{\xi}{1-e^{-2 \xi \delta}}-\zeta(\lambda)\)\(1-e^{-2 \xi \delta}\)
		= \big( \zeta(\xi) - \zeta(\lambda) \big)\(1 - e^{-2\xi \delta}\),
\]
it is easy to check that both $\xi = 0$ and $\xi = \lambda$ are roots of $p$.
In the remainder of this section, we argue that these are the only roots of
$p$ in the strip
\[
	\mathcal R_\delta:= \left\{z\in\CC ~:~ -\frac{\pi}{\delta}\leq\im z\leq\frac{\pi}{\delta} \right\}
\]
as is defined in the Introduction of this chapter.
That is, we show that the equation
\begin{align} \label{eq1:baseEqn}
	\xi - \zeta(\lambda) + \zeta(\lambda) e^{-2 \xi \delta} = 0
\end{align}
has exactly two solutions (in $\xi$) for $\xi \in \mc R_\delta$. With a little algebraic
manipulation, equation \eqref{eq1:baseEqn} can be rewritten as
\begin{align} \label{eq1:EqnToSolve}
	\big(2 \delta \xi - 2 \delta \, \zeta(\lambda)\big) \, e^{2 \delta \xi - 2 \delta \, \zeta(\lambda)}
		= -2 \delta \, \zeta(\lambda) \, e^{-2 \delta \, \zeta(\lambda)}.
\end{align}
In order to ``solve'' \eqref{eq1:EqnToSolve}, recall that the Lambert $W$ function
(which we henceforth refer to only as $W$) is defined to be the multivalued
``inverse'' of the function $z e^z$. So, to determine the number
of solutions \eqref{eq1:EqnToSolve} has for $\xi \in \mc R_\delta$, we need to specify
which branches of $W$ are appropriate to apply to both sides of \eqref{eq1:EqnToSolve}
given our restriction on $\xi$.

The following discussion of the branches of the complex $W$\label{sym1:Wk} function is heavily
inspired by Section 4 from the 1996 R.M. Corless, G.H. Gonnet, D.E.G. Hare, D.J.
Jeffery and D.E. Knuth paper \cite{Corless1996}. As is the case with the standard
complex exponential
and logarithmic functions, to define the branches of $W$, the canonical approach
is to make a branch cut along the negative real axis of the range of $z e^z$, and
determine which curves in the range of $W$ are mapped to the branch cut ({\em i.e.}
the negative real axis).

To do so, set
\[
	\begin{aligned}
		z &:= w e^w  & \qquad w&:= W(z) \\
		  &:= x + i y & \qquad &:= t + i s.
	\end{aligned}
\]
Then, using Euler's formula to simply the equation
\[
	( x + i y) = (t + i s) e^{t + i s}
\]
and taking real and imaginary parts, we obtain the system
\[
	\left\{
		\begin{aligned}
			x&= e^t \(t \cos s - s \sin s\) \\
			y&= e^t \( t \sin s + s \cos s \).
		\end{aligned}
	\right.
\]
So, if $y=0$, then either $s = 0$ or $t = - s \cot s$. Further, $x < 0$ if and
only if $t \cos s - s \sin s < 0$. Now, since $t = - s \cot s$ has asymptotes at
$s = k \pi$ ($k \in \ZZ \sm \{0\}$), and the function $t \cos s - s \sin s$ has
no roots, the inequality $t \cos s - s \sin s < 0$ holds precisely on the intervals
\[
	\(\bigcup_{-k \in \NN} \big( (2k + 1) \pi, \, 2 k \pi\big)\)
		\cup (-\pi, \,  \pi) \cup
		\( \bigcup_{k \in \NN} \big(2 k \pi, \,  (2k + 1) \)
\]
As such, the only curves the function $z e^z$ maps to the negative real axis are
\[
	\gamma_k(s) := - s \cot s + i s, \qquad k \in \ZZ,
\]
whose respective domains are given by
\begin{align*}
	\text{domain} \, \gamma_k(s) :=
		\begin{cases}
			\big( (2k + 1) \pi, \, 2 k \pi\big), & -k \in \NN \\
			(-\pi, \,  \pi), & k = 0 \\
			\big(2 k \pi, \,  (2k + 1) \pi\big), & k \in \NN
		\end{cases}
\end{align*}
and the curve whose graph is
\[
	(-\infty, -1) \cup \{ \gamma_0(s) + i s ~:~ -\pi < s \leq 0 \}.
\]
As such, these curves form the boundary values between the ranges of
the different branches of $W$. In particular, the ranges for the
the principle branch $W_0$ and the $W_{-1}$ and $W_1$ branches are
shown below in Figure \ref{fig:BoundaryCurves}.

\begin{figure}[H]
	\centering
	\begin{tikzpicture}[scale = 0.25]
	\def\ticklength{0.35}
	\def\Pi{3.141592653589793}
	\def\ee{2.718281828459045}

	%% Draw w-axes
	\draw[->, thick] (-15, 0) -- (16, 0) node[right] {$\re w = t$};
	\draw[->, thick] (0, -10) -- (0, 11) node[above] {$\im w$ = s};

	%% Draw Branch Curves
	\draw[red,smooth, samples=100, domain=-2.94756:2.94756, variable=\s, thick]
		plot ({-\s / tan(deg(\s))}, {\s} );
	\draw[red,smooth, samples=200, domain=6.70346:8.88978, variable=\s, thick]
		plot ({-\s / tan(deg(\s))}, {\s} );
	\draw[red,smooth, samples=200, domain=-8.88978:-6.70346, variable=\s, thick]
		plot ({-\s / tan(deg(\s))}, {\s} );
	\draw[red, domain=-15:-1, variable=\s, thick]
		plot ({\s}, 0);

	%% Place Region Labels
	\node (W0) at (4, 1) {$W_0$};
	\node (W-1) at (4, -5) {$W_{-1}$};
	\node (W1) at (4, 5) {$W_{1}$};
	\node (W2) at (-7, {3*\Pi}) {$W_2$};
	\node (W-2) at (-7, {-3*\Pi}) {$W_{-2}$};

	%% Draw im axis tiks
	\foreach \i in {-3, -2, -1, 1, 2, 3} {
		\draw[thick] ({\ticklength}, {\i * \Pi}) -- ({-\ticklength}, {\i * \Pi})
			node[left] {$\i \pi$};
	}

	%% Draw re axis tiks
	\draw[thick] (-1, \ticklength) -- (-1, -\ticklength);
	\node at (-1.6, -\ticklength-0.9) {$-1$};

	\end{tikzpicture}
	\caption{Ranges for the $W_{\pm2}$, $W_{\pm1}$, and $W_0$ branches of the
		$W$ function.}
	\label{fig:BoundaryCurves}
\end{figure}

Since the map $w \mapsto w e^w$ takes $w = -1$ to $z = -e^{-1}$,
on pages 17 and 18 in \cite{Corless1996},
Corless {\em et al.} proposetaking the branch cut which defines the
principle branch of $W$ along $\{ z \in \CC ~:~ -\infty < z \leq -e^{-1} \}$,
and taking all other branch cuts along the entire negative $\re z$-axis.
They further take all branch cuts in such a way that
the branch cuts are closed ``on the top,'' as shown in Figures
\ref{fig:ClosedOnTop} and \ref{fig:OtherBranches}.

\begin{figure}[h!]
	\begin{tikzpicture}[scale=0.5]
		\def\yaxisshift{-4.6}
		\def\ybranchshift{0.05}
		\def\ticklength{0.2}
		\def\Pi{3.141592653589793}
		\def\ee{2.718281828459045}

		%% Draw z-axes
		\draw[->, thick] (-22, 0) -- (-12, 0) node[right] {$\re z = x$};
		\draw[->, thick] (-17, -5) -- (-17, 5) node[above] {$\im z = y$};
		%% Draw w-axes
		\draw[->, thick] (-5, 0) -- (5, 0)
			node[right] {$\re w = t$};
		\draw[->, thick] (0, -5) -- (0, 5) node[above] {$\im w$ = s};
		%% Draw Map Arrows
		\path[->, thick] (-11, 2.5) edge[bend left] node[auto] {$w = W(z)$} (-6, 2.5);
		\path[->, thick] (-6, -2.5) edge[bend left] node[auto] {$z = w e^w$} (-11, -2.5);

		%% Draw Branch Cut on z-axes
		\draw[very thick, red]
			(-22, {\ybranchshift}) -- ({-17-1/\ee}, {\ybranchshift});
		\draw[dashed, very thick, red]
			(-22, {-\ybranchshift}) -- ({-17-1/\ee}, {-\ybranchshift});

		%% Draw Branch Cut along negative Re w axis
		\draw[dashed, very thick, red] (-5, {\ybranchshift}) -- (-1, {\ybranchshift});
		\draw[very thick, red] (-5, {-\ybranchshift}) -- (-1, {-\ybranchshift});

		%% Draw w Branch Curves
		\draw[red,smooth, samples=100, domain=-2.65:-0.001, variable=\s, very thick]
			plot ({-\s / tan(deg(\s)) -0.05}, {\s} );

		\draw[dashed, red,smooth, samples=100, domain=0.001:2.65, variable=\s, very thick]
			plot ({-\s / tan(deg(\s)) -0.05}, {\s} );

		%% Label Regions
		\node at (2, 1) {$W_0$};
		\node at (-3, 2) {$W_1$};
		\node at (-3, -2) {$W_{-1}$};

		%% Draw im w tick marks
		\foreach \i in {-1, 1} {
			\draw[thick] ({\ticklength}, {\i * \Pi}) -- ({-\ticklength}, {\i * \Pi})
				node[left] {$\i \pi$};
		}

		%% Draw re w tick mark
		\draw[thick] (-1, \ticklength) -- (-1, -\ticklength) node[below left] {$-1$};
		% \node at (-1.6, -\ticklength-0.9) {$-1$};

		%% Draw re z tick mark
		\draw[thick] ({-17-1/\ee}, \ticklength) -- ({-17-1/\ee}, -\ticklength)
			node[below left] {$-\frac{1}{e}$};

	\end{tikzpicture}
	\caption{$W_0$ Branch Cut}
	\label{fig:ClosedOnTop}
\end{figure}

\begin{figure}[h!]
	\begin{tikzpicture}[scale=0.17]
		\def\xm{15}
		\def\ashift{1}
		\def\xmin{-\xm}
		\def\xmax{\xm+\ashift}
		\def\ym{11}
		\def\ymin{-\ym}
		\def\ymax{\ym+\ashift}
		\def\yaxisshift{-4.6}
		\def\xshift{51}
		\def\mapshift{3}
		\def\ybranchshift{0.1}
		\def\ticklength{0.5}
		\def\Pi{3.141592653589793}
		\def\ee{2.718281828459045}

		%% Draw z-axes
		\draw[->, thick] (\xmin-\xshift, 0) -- (\xmax-\xshift, 0)
			node[right] {$\re z = x$};
		\draw[->, thick] (-\xshift, \ymin) -- (-\xshift, \ymax)
			node[above] {$\im z = y$};

		%% Draw w-axes
		\draw[->, thick] (\xmin, 0) -- (\xmax, 0) node[right] {$\re w = t$};
		\draw[->, thick] (0, \ymin) -- (0, \ymax) node[above] {$\im w$ = s};

		%% Draw Map Arrows
		\path[->, thick] (\xmax-\xshift+\mapshift, \ym/2) edge[bend left]
			node[auto] {$w = W(z)$} (\xmin - \mapshift, \ym/2);
		\path[->, thick] (\xmin - \mapshift, -\ym/2) edge[bend left]
			node[auto] {$z = w e^w$} (\xmax-\xshift+\mapshift, -\ym/2);

		%% Draw Branch Cut on z-axes
		\draw[very thick, red]
			({\xmin-\xshift}, {\ybranchshift}) -- ({0-\xshift}, {\ybranchshift});
		\draw[dashed, very thick, red]
			({\xmin-\xshift}, {-\ybranchshift}) -- ({0-\xshift}, {-\ybranchshift});

		%% Draw the Branch Curves
		\draw[red,smooth, samples=200, domain=6.70346:8.88978, variable=\s, very thick]
			plot ({-\s / tan(deg(\s))}, {\s+\ybranchshift} );
		\draw[dashed, red,smooth, samples=200, domain=6.70346:8.88978, variable=\s, very thick]
			plot ({-\s / tan(deg(\s))}, {\s-\ybranchshift} );
		\draw[red,smooth, samples=200, domain=-8.88978:-6.70346, variable=\s, very thick]
			plot ({-\s / tan(deg(\s))}, {\s+\ybranchshift} );
		\draw[dashed, red,smooth, samples=200, domain=-8.88978:-6.70346, variable=\s, very thick]
			plot ({-\s / tan(deg(\s))}, {\s-\ybranchshift} );

		%% Draw im axis tiks
		\foreach \i in {-3, -2, -1, 1, 2, 3} {
			\draw[thick] ({\ticklength}, {\i * \Pi}) -- ({-\ticklength}, {\i * \Pi})
				node[left] {$\i \pi$};
		}

		%% Place Region Labels
		\node (W2) at ({-2*\xm/3}, {3*\Pi}) {$W_2$};
		\node (W-2) at ({-2*\xm/3}, {-3*\Pi}) {$W_{-2}$};

	\end{tikzpicture}
	\caption{$W_k$ ($k \ne 0$) Branch Cuts}
	\label{fig:OtherBranches}
\end{figure}

Corless {\em et al.} further argue in \cite{Corless1996} that each branch
$W_k : \CC \to \ran W_k$ is bijective, which allows us to solve equation
\eqref{eq1:EqnToSolve} and find
\[
	\xi = \frac{1}{2\delta} W_k\big( -2 \delta \zeta(\lambda) \big) + \zeta(\lambda), \qquad k \in \zeta.
\]
Consequently, the only roots of $p(\xi, \lambda)$ which could have a chance of living
in $\mc R_\delta$ are those corresponding to the $W_{-1}$, $W_0$, and $W_1$ branches.
Now, if $\xi = 0$, then
\[
	W_k\big( -2 \delta \zeta(\lambda) \big) = -2 \delta \zeta(\lambda).
\]
Since $\zeta$ is a positive, strictly increasing function with $\zeta(0) = 1/(2\delta)$,
if $\lambda < 0$, then $-1 < -2\delta \zeta(\lambda) < 0$, and if $\lambda \geq 0$, then
$-2 \delta \zeta(\lambda) \leq -1$. In other words, if $\lambda < 0$ the $\xi = 0$ root of $p$
corresponds to the $W_0$ branch, but if $\lambda \geq 0$, then it corresponds to the
$W_{-1}$ branch.

On the other hand, if $\xi = \lambda$, then
\[
	W_k\big( -2 \delta \zeta(\lambda) \big) = 2\delta \lambda -2 \delta \zeta(\lambda).
\]
Let $g(\lambda):= 2\delta \lambda -2 \delta \zeta(\lambda)$.
Note that
\begin{align*}
	\begin{cases}
		0 < \zeta'(\lambda) < \frac{1}{2}, & \text{for } \lambda < 0 \\
		\zeta'(0) = \frac{1}{2}, &  \\
		\frac{1}{2} < \zeta'(\lambda) < 1, & \text{for } \lambda > 0
	\end{cases}
\end{align*}
which implies $g$ is a strictly increasing function. Moreover, since
$\lim_{\lambda\to \infty} g(\lambda) = 0$ and $g(0) = -1$, we see that
\[
	\begin{cases}
		2\delta \lambda -2 \delta \zeta(\lambda)  < -1, & \lambda < 0 \\
		2\delta \lambda -2 \delta \zeta(\lambda) = -1, & \lambda = 0 \\
		-1 < 2\delta \lambda -2 \delta \zeta(\lambda) < 0, & \lambda > 0
	\end{cases}
\]
Therefore, if $\lambda < 0$ then the $\xi = \lambda$ zero of $p$ corresponds to the
$W_{-1}$ branch, but if $\lambda \geq 0$, it corresponds to the $W_{0}$. Moreover,
given that $-2\delta \zeta(\lambda)$ lies on the negative real axis and hence on the branch
cut used for $W_k$, $k \ne 0$, each value of
$2\delta\xi-2\delta\zeta(\lambda) = W_k\big(-2\delta\zeta(\lambda)\big)$ ($k \ne 0$) lies on the boundary
between the respective ranges of the $W$ branches. In particular, given that
the strip $\big\{ z ~:~ -2\pi \leq \im z \leq 2 \pi \big\}$ does not contain
any part of the boundary between $\ran W_{-2}$ and $\ran W_{-1}$ or any part
of the boundary between $\ran W_1$ and $\ran W_2$, this strip contains exactly
two values of $W\big(-2 \delta \zeta(\lambda) \big)$. Since
\[
	\mc R_\delta = \frac{1}{2\delta}\big\{ z ~:~ -2\pi \leq \im z \leq 2 \pi \big\}+\zeta(\lambda),
\]
the strip $\mc R\delta$ contains exactly two roots of $p$\textemdash{}precisely as claimed.

\end{document}


\chapter{The Definition of Groups}
\label{ch:the-definition-of-groups}
%% FIXME: \cite{Pinter2016}*{Chapter 3}

\begin{enumerate}[label={\Alph*.},font={\bfseries}]
\item {\bf Examples of Abelian Groups}
  \begin{enumerate}[label={\arabic*},font={\bfseries}]
  \item $\Group{\mathbb{R}, x*y=x+y+k}$
    \begin{enumerate}[label={(\roman*)}]
    \item $*$ is commutative:
      $x*y=x+y+k = y+x+k=y*x$
    \item $*$ is associative.
      \begin{alignat*}{3}
        & x(yz) &&= x(y+z+k) &&= x+y+z+2k \\
        & (xy)z &&= (x+y+k)z &&= (xy)z \\
        & x(yz) &&= (xy)z
      \end{alignat*}
    \item $\mathbb{R}$ has an identity element with respect to $*$.
      \begin{align*}
        xe &= x \\
        x+e+k &= x \\
        e &= -k \\
        (-k)x &= x \\
        -k+x+k &= x
      \end{align*}
    \item $\forall x\in\mathbb{R}(\exists x^\prime\in\mathbb{R}(x*x^\prime=-k))$
      \begin{alignat*}{3}
        xx^\prime &= -k \\
        x+x^\prime+k &= -k \\
        x^\prime &= -x-2k \\
        x^{\prime}x &= xx^\prime & \text{due to commutativity}
      \end{alignat*}
    \end{enumerate}
  \item $\Group{\mathbb{R}^*, x*y=\frac{xy}{2}}$
    \begin{enumerate}[label={(\roman*)}]
    \item $*$ is commutative:
      $x*y=\frac{xy}{2} = \frac{yx}{2}=y*x$
    \item $*$ is associative.
      \begin{alignat*}{3}
        x*(y*z) &= x*(\frac{yz}{2}) &= \frac{xyz}{4} \\
        (x*y)*z &= (\frac{xy}{2})*z &= \frac{xyz}{4}
      \end{alignat*}
    \item $\mathbb{R}^*$ has an identity element with respect to $*$.
      \begin{align*}
        x*e &= \frac{xe}{2} = \frac{ex}{2}=e*x = x \\
        e &= 2
      \end{align*}
    \item $\forall x\in\mathbb{R}(\exists x^\prime\in\mathbb{R}(x*x^\prime=2))$
      \begin{align*}
        x*x^\prime &= \frac{xx^\prime}{2} = \frac{x^{\prime}x}{2} = x^{\prime}*x = e = 2 \\
        x^\prime = \frac{4}{x}
      \end{align*}
    \end{enumerate}
  \item $\Group{\Set{x\in\mathbb{R} : x \neq -1}, x*y=x+y+xy}$
    \begin{enumerate}[label={(\roman*)}]
    \item $*$ is commutative: $x*y=x+y+xy = y+x+yx=y*x$
    \item $*$ is associative.
      \begin{alignat*}{3}
        x*(y*z) &= x*(y+z+yz) = x+(y+z+yz)+x(y+z+yz) &= x+y+z+xy+xz+yz+xyz \\
        (x*y)*z &= (x+y+xy)*z = (x+y+xy)+z+(x+y+xy)z &= x+y+z+xy+xz+yz+xyz
      \end{alignat*}
    \item $\Set{x\in\mathbb{R} : x \neq -1}$ has an identity element with respect to $*$.
      \begin{align*}
        x*e &= x+e+xe = e+x+ex = e*x = x \\
        e(x+1) &= 0 \\
        e &= 0
      \end{align*}
    \item Every element of $\Set{x\in\mathbb{R} : x \neq -1}$ has an inverse with respect to $*$.
      \begin{align*}
        x*x^\prime &= x+x^\prime+xx^\prime=x^\prime+x+x^{\prime}x = e = 0 \\
        x^{\prime}(x+1) &= -x \\
        x^\prime &= -\frac{x}{x+1}
      \end{align*}
    \end{enumerate}
  \item $\Group{\Set{x\in\mathbb{R} :-1 < x < 1}, x*y=\frac{x+y}{xy+1}}$
    \begin{enumerate}[label={(\roman*)}]
    \item $*$ is commutative: $x*y=\frac{x+y}{xy+1}=\frac{y+x}{yx+1}=y*x$
    \item $*$ is associative.
      \begin{alignat*}{4}
        x*(y*z) &= x*(\frac{y+z}{yz+1})
        &= \frac{x+(\frac{y+z}{yz+1})}{x(\frac{y+z}{yz+1})+1}
        &= \frac{xyz+x+y+z}{xy+xz+yz+1} \\
        (x*y)*z &= \frac{x+y}{xy+1}*z
        &= \frac{(\frac{x+y}{xy+1})+z}{(\frac{x+y}{xy+1})z+1}
        &= \frac{x+y+z+xyz}{xy+yz+xz+1}
      \end{alignat*}
    \item $\Set{x\in\mathbb{R} : -1 < x < 1}$ has an identity element w.r.t. $*$.
      \begin{align*}
        x*e &= \frac{x+e}{xe+1} = x \\
        x+e &= x(xe+1) \\
        e &= ex^2 \\
        e(1-x^2) &= 0 \\
        e &= 0 \\
        x*0 &= \frac{x+0}{(x\times0)+1} = x = \frac{0+x}{0x+1} = 0*x
      \end{align*}
    \item Every element of $\Set{x\in\mathbb{R} : -1 < x < 1}$ has an inverse with respect to $*$.
      \begin{align*}
        x * x^\prime &= \frac{x+x^\prime}{xx^\prime+1} = 0; \quad
        x+x^\prime = 0; \quad
        x^\prime = -x \\
        x*(-x) &= \frac{x-x}{x(-x)+1} = 0 = \frac{-x+x}{-x^2+1} = (-x)*x
      \end{align*}
    \end{enumerate}
  \end{enumerate}
\item {\bf Groups on the Set $\mathbb{R} \times \mathbb{R}$}
  \begin{enumerate}[label={\arabic*},font={\bfseries}]
  \item $(a,b) * (c,d) = (ad + bc, bd)$, on the set $\Set{(x,y)\in\mathbb{R}\times\mathbb{R} : y \ne 0}$
    \begin{enumerate}[label={(\roman*)}]
    \item $*$ is commutative.
      \begin{align*}
        (c,d) * (a,b) &= (cb+da,db) \\
        &= (ad+bc,bd) \\
        &= (a,b) * (c,d)
      \end{align*}
    \item $*$ is associative.
      \begin{align*}
        (a,b) * \left[(c,d) * (e,f)\right] &= (a,b) * (cf+de,df) \\
        &= (adf+bcf+bde,bdf) \\
        &= (ad+bc, bd) * (e,f) \\
        &= \left[(a,b) * (c,d)\right] * (e,f)
      \end{align*}
    \item $(e_1,e_2) = (0,1)$
      \begin{align*}
        (a,b) * (e_1,e_2) &= (ae_2 + be_1, be_2) \\
        &= (a,b) \\
        \\
        be_2 &= b \\
        e_2 &= 1 \\
        \\
        ae_2 + be_1 &= a \\
        a + be_1 &= a \\
        e_1 &= 0
      \end{align*}
    \item $(a^\prime,b^\prime) = \left(\frac{-a}{b^2}, \frac{1}{b}\right)$
      \begin{align*}
        (a,b) * (a^\prime,b^\prime) &= (ab^\prime + ba^\prime, bb^\prime) \\
        &= (0,1)
      \end{align*}
      \begin{align*}
        bb^\prime &= 1 \\
        b^\prime &= \frac{1}{b} \\
        \\
        ab^\prime + ba^\prime &= 0 \\
        \frac{a}{b} + ba^\prime &= 0 \\
        ba^\prime &= \frac{-a}{b} \\
        a^\prime &= \frac{-a}{b^2} \\
        \\
        (a,b) * \left(\frac{-a}{b^2}, \frac{1}{b}\right) &= \left(\frac{a}{b} + \frac{-a}{b}, b\left(\frac{1}{b}\right)\right) \\
        &= (0,1)
      \end{align*}
    \end{enumerate}
  \item $(a, b) * (c, d) = (ac, bc + d)$, on the set $\Set{(x,y)\in\mathbb{R}\times\mathbb{R} : x \ne 0}$
    \begin{enumerate}[label={(\roman*)}]
    \item $*$ is not commutative: $(c,d) * (a,b) = (ca, da + b) \ne (a,b) * (c,d)$
    \item $*$ is associative.
      \begin{align*}
        \left[(a,b) * (c,d)\right] * (e,f) &= (ac, bc+d) * (e,f) \\
        &= (ace, bce+de+f) \\
        &= (a,b) * (ce, de+f) \\
        &= (a,b) * \left[(c,d) * (e,f)\right]
      \end{align*}
    \item $(e_1,e_2) = (1,0)$
      \begin{align*}
        (a,b) * (e_1,e_2) &= (ae_1, be_1+e_2) \\
        &= (a, b) \\
        \\
        ae_1 &= a \\
        e_1 &= 1 \\
        \\
        be_1+e_2 &= b \\
        b+e_2 &= b \\
        e_2 &= 0
      \end{align*}
    \item $(a^\prime,b^\prime) = (\frac{1}{a},\frac{-b}{a})$
      \begin{align*}
        (a,b) * (a^\prime,b^\prime) &= (aa^\prime, ba^\prime + b^\prime) \\
        &= (1,0) \\
        \\
        aa^\prime &= 1 \\
        a^\prime &= \frac{1}{a} \\
        \\
        ba^\prime + b^\prime &= 0 \\
        \frac{b}{a} + b^\prime &= 0 \\
        b^\prime &= \frac{-b}{a}
        \\
        (a,b) * (\frac{1}{a}, \frac{-b}{a}) &= (\frac{a}{a}, \frac{b}{a} - \frac{b}{a}) \\
        &= (1,0)
      \end{align*}
    \end{enumerate}
  \item $(a, b) * (c, d) = (ac, bc + d)$, on the set $\Set{(x,y)\in\mathbb{R}\times\mathbb{R}}$
    \begin{enumerate}[label={(\roman*)}]
    \item $*$ is not commutative, as per 2(i).
    \item $*$ is associative, as per 2(ii).
    \item $(e_1,e_2) = (1,0)$, as per 2(iii).
    \item $a^\prime$ is not defined $\forall a\in\mathbb{R}$, notably when $a=0$.
    \end{enumerate}
  \item $(a, b) * (c, d) = (ac-bd,ad+bc)$, on the set $\Set{(x,y)\in(\mathbb{R}\times\mathbb{R})\setminus\Set{(0,0)}}$
    \begin{enumerate}[label={(\roman*)}]
    \item $*$ is commutative.
      \begin{align*}
        (c,d) * (a,b) &= (ca-db,cb+da) \\
        &= (ac-db,ad+bc) \\
        &= (a,b) * (c,d)
      \end{align*}
    \item $*$ is associative.
      \begin{align*}
        (a,b) * \left[(c,d) * (e,f)\right] &= (ac-bd,ad+bc) * (ce-df,cf+de) \\
        &= \left(a(ce-df) - b(cf+de), a(cf+de)+b(ce-df)\right) \\
        &= (ace-adf-bcf-bde, acf+ade+bce-bdf) \\
        &= \left(e(ac-bd)-f(ad+bc), f(ac-bd)+e(ad+bc)\right) \\
        &= (ac-bd,ad+bc) * (e,f) \\
        &= \left[(a,b) * (c,d)\right] * (e,f)
      \end{align*}
    \item $(e_1,e_2) = (?,?)$
      \begin{align*}
        (a,b) * (e_1,e_2) &= (ae_1-be_2,ae_2+be_1) \\
        &= (a,b) \\
        \\
        ae_2+be_1 &= b \\
        be_1 &= b - ae_2 \\
        e_1 &= 1 - \frac{ae_2}{b} \\
        \\
        ae_1-be_2 &= a \\
        -be_2 &= a-ae_1 \\
        be_2 &= ae_1-a \\
        e_2 &= \frac{ae_1-a}{b}
      \end{align*}
    \end{enumerate}
  \end{enumerate}
  \newpage
\item {\bf Groups of Subsets of a Subset}
  \begin{enumerate}[label={\arabic*},font={\bfseries}]
  \item The identity element with respect to the operation $+$ is $\emptyset$.
    \begin{align*}
      A+I &= (A-I) \cup (I-A) = A \\
      &= (A-\emptyset) \cup (I-\emptyset) \\
      \\
      I &= \emptyset
    \end{align*}
  \item $\Group{\powerset{D}, +}$ is a group, since
    $\forall A \in \powerset{D}, A\inverse = A$.
    \begin{align*}
      A+A\inverse &= \emptyset \\
      (A-A\inverse) \cup (A\inverse-A) &= \emptyset \\
      A-A\inverse &= A\inverse-A = \emptyset \\
      A\inverse &= A
    \end{align*}
  \item Let $D = \Set{a,b,c}$.
    \[
    \powerset{D} = \Set{
      \emptyset,
      \Set{a}, \Set{b}, \Set{c},
      \Set{a,b}, \Set{a,c}, \Set{b,c},
      \Set{a,b,c}
    }
    \]
    \begin{center}
      \captionof{table}{$\Group{\powerset{D}, +}$}
      \begin{tabular}{ c | c c c c c c c c}
        $+$ & $\emptyset$ & $\Set{a}$ & $\Set{b}$ & $\Set{c}$ & $\Set{a,b}$ & $\Set{a,c}$ & $\Set{b,c}$ & $\Set{a,b,c}$ \\
        \hline
        $\emptyset$ & $\emptyset$ & $\Set{a}$ & $\Set{b}$ & $\Set{c}$ & $\Set{a,b}$ & $\Set{a,c}$ & $\Set{b,c}$ & $\Set{a,b,c}$ \\
        $\Set{a}$ & $\Set{a}$ & $\emptyset$ & $\Set{a,b}$ & $\Set{a,c}$ & $\Set{b}$ & $\Set{c}$ & $\Set{a,b,c}$ & $\Set{b,c}$ \\
        $\Set{b}$ & $\Set{b}$ & $\Set{a,b}$ & $\emptyset$ & $\Set{b,c}$ & $\Set{a}$ & $\Set{a,b,c}$ & $\Set{c}$ & $\Set{a,c}$ \\
        $\Set{c}$ & $\Set{c}$ & $\Set{a,c}$ & $\Set{b,c}$ & $\emptyset$ & $\Set{a,b,c}$ & $\Set{a}$ & $\Set{b}$ & $\Set{a,b}$ \\
        $\Set{a,b}$ & $\Set{a,b}$ & $\Set{b}$ & $\Set{a}$ & $\Set{a,b,c}$ & $\emptyset$ & $\Set{b,c}$ & $\Set{a,c}$ & $\Set{c}$ \\
        $\Set{a,c}$ & $\Set{a,c}$ & $\Set{c}$ & $\Set{a,b,c}$ & $\Set{a}$ & $\Set{b,c}$ & $\emptyset$ & $\Set{a,b}$ & $\Set{b}$ \\
        $\Set{b,c}$ & $\Set{b,c}$ & $\Set{a,b,c}$ & $\Set{c}$ & $\Set{b}$ & $\Set{a,c}$ & $\Set{a,b}$ & $\emptyset$ & $\Set{a}$ \\
        $\Set{a,b,c}$ & $\Set{a,b,c}$ & $\Set{b,c}$ & $\Set{a,c}$ & $\Set{a,b}$ & $\Set{c}$ & $\Set{b}$ & $\Set{a}$ & $\emptyset$
      \end{tabular}
    \end{center}
  \end{enumerate}
\item {\bf A Checkerboard Game}
  \begin{center}
    \captionof{table}{$\Group{G, *}$}
    \begin{tabular}{ c | c c c c }
      $*$ & $I$ & $V$ & $H$ & $D$ \\
      \hline
      $I$ & $I$ & $V$ & $H$ & $D$ \\
      $V$ & $V$ & $I$ & $D$ & $H$ \\
      $H$ & $H$ & $D$ & $I$ & $V$ \\
      $D$ & $D$ & $H$ & $V$ & $I$
    \end{tabular}
  \end{center}
  As shown in the \gls{Cayley table} above, the identity element is $I$ and
  every element is its own inverse. Having shown that and granting
  associativity, $\Group{G, *}$ is a group.
\item {\bf A Coin Game}
  \begin{center}
    \captionof{table}{$\Group{G, *}$}
    \begin{tabular}{ c | c c c c c c c c }
      $*$ & $I$ & $M_1$ & $M_2$ & $M_3$ & $M_4$ & $M_5$ & $M_6$ & $M_7$ \\
      \hline
      $I$ & $I$ & $M_1$ & $M_2$ & $M_3$ & $M_4$ & $M_5$ & $M_6$ & $M_7$ \\
      $M_1$ & $M_1$ & $I$ & $M_3$ & $M_2$ & $M_5$ & $M_4$ & $M_7$ & $M_6$ \\
      $M_2$ & $M_2$ & $M_3$ & $I$ & $M_1$ & $M_6$ & $M_7$ & $M_4$ & $M_5$ \\
      $M_3$ & $M_3$ & $M_2$ & $M_1$ & $I$ & $M_7$ & $M_6$ & $M_5$ & $M_4$ \\
      $M_4$ & $M_4$ & $M_6$ & $M_5$ & $M_7$ & $I$ & $M_2$ & $M_1$ & $M_3$ \\
      $M_5$ & $M_5$ & $M_7$ & $M_4$ & $M_6$ & $M_1$ & $M_3$ & $I$ & $M_2$ \\
      $M_6$ & $M_6$ & $M_4$ & $M_7$ & $M_5$ & $M_2$ & $I$ & $M_3$ & $M_1$ \\
      $M_7$ & $M_7$ & $M_5$ & $M_6$ & $M_4$ & $M_3$ & $M_1$ & $M_2$ & $I$
    \end{tabular}
  \end{center}

  As shown in the \gls{Cayley table} above, the identity element is $I$ and
  every element is invertible. Having shown that and granting associativity,
  $\Group{G, *}$ is a group. It is not commutative, because, for example
  $M_6 * M_4 = M_2$, while $M_4 * M_6 = M_1$, so $M_6 * M_4 \ne M_4 * M_6$.
  \newpage
  \captionof{figure}{$\Group{r,s,t\ |\ r^2,s^2,t^2,(rs)^4,(st)^3,(rt)^2}$}
  \begin{figure}[h]
    \begin{tikzpicture}[line width=1pt]
      \node (I) at (0,4) { $I$ };
      \node (M3) at (4,0) { $M_3$ };
      \node (M5) at (3,3) { $M_5$ };
      \node (M6) at (1,1) { $M_6$ };
      \node (M7) at (3,1) { $M_7$ };

      \node[green] (M1) at (4,4) { $\mathbf{M_1}$ };
      \node[blue] (M2) at (0,0) { $\mathbf{M_2}$ };
      \node[red] (M4) at (1,3) { $\mathbf{M_4}$ };

      \draw[green] (I)  -- (M1)
                   (M1) -- (I)
                   (M2) -- (M3)
                   (M3) -- (M2)
                   (M4) -- (M6)
                   (M5) -- (M7)
                   (M6) -- (M4)
                   (M7) -- (M5);

      \draw[blue] (I)  -- (M2)
                  (M1) -- (M3)
                  (M2) -- (I)
                  (M3) -- (M1)
                  (M4) -- (M5)
                  (M5) -- (M4)
                  (M6) -- (M7)
                  (M7) -- (M6);

      \draw[red] (I)  -- (M4)
                 (M1) -- (M5)
                 (M2) -- (M6)
                 (M3) -- (M7)
                 (M4) -- (I)
                 (M5) -- (M1)
                 (M6) -- (M2)
                 (M7) -- (M3);
    \end{tikzpicture}
  \end{figure}
\item {\bf Groups in Binary Codes}
  \begin{enumerate}[label={\arabic*},font={\bfseries}]
  \item $\Seq{a_1,a_2,...,a_n} + \Seq{b_1,b_2,...,b_n} = \Seq{b_1,b_2,...,b_n} + \Seq{a_1,a_2,...,a_n}$,
    since the left-hand side is equivalent to $\Seq{a_1+b_1,a_2+b_2,...,a_n+b_n}$,
    which by commutativity is equivalent to $\Seq{b_1+a_1,b_2+a_2,...,b_n+a_n}$,
    which is equivalent to $\Seq{b_1,b_2,...,b_n} + \Seq{a_1,a_2,...,a_n}$.
  \item
    \begin{alignat*}{5}
      &1 + (1 + 1) &&= 1 + 0 &&= 1 &&= 0 + 1 &&= (1 + 1) + 1 \\
      &1 + (1 + 0) &&= 1 + 1 &&= 0 &&= 0 + 0 &&= (1 + 1) + 0 \\
      &1 + (0 + 1) &&= 1 + 1 &&= 0 &&= 1 + 1 &&= (1 + 0) + 1 \\
      &0 + (1 + 1) &&= 0 + 0 &&= 0 &&= 1 + 1 &&= (0 + 1) + 1 \\
      &1 + (0 + 0) &&= 1 + 0 &&= 1 &&= 1 + 0 &&= (1 + 0) + 0 \\
      &0 + (0 + 1) &&= 0 + 1 &&= 1 &&= 0 + 1 &&= (0 + 0) + 1 \\
      &0 + (1 + 0) &&= 0 + 1 &&= 1 &&= 1 + 0 &&= (0 + 1) + 0 \\
      &0 + (0 + 0) &&= 0 + 0 &&= 0 &&= 0 + 0 &&= (0 + 0) + 0
    \end{alignat*}
  \item
    \begin{align*}
      \Seq{a_1,a_2,...,a_n} + \left[\Seq{b_1,b_2,...,b_n} + \Seq{c_1,c_2,...,c_n}\right]
      &= \Seq{a_1,a_2,...,a_n} + \Seq{b_1+c_1,b_2+c_2,...,b_n+c_n} \\
      &= \Seq{a_1+b_1+c_1,a_2+b_2+c_2,...,a_n+b_n+c_n} \\
      &= \Seq{a_1+b_1,a_2+b_2,...,a_n+b_n} + \Seq{c_1,c_2,...,c_n} \\
      &= \left[\Seq{a_1,a_2,...,a_n} + \Seq{b_1,b_2,...,b_n}\right] + \Seq{c_1,c_2,...,c_n}
    \end{align*}
    \item The identity element of $\mathbb{B}^n$, that is, the identity element
      for adding words on length $n$, is $0^n$.
    \item The inverse, with respect to word addition, of any word
      $(a_1,...,a_n)$ is $(a_1,...,a_n)$.
    \item $\mathbf{a} + \mathbf{b} = \mathbf{a} + (-\mathbf{b})$,
      since $\mathbf{b} = -\mathbf{b}$.
      Thus $\mathbf{a} + \mathbf{b} = \mathbf{a} - \mathbf{b}$.
    \item
      \begin{align*}
        \mathbf{a} + \mathbf{b} &= \mathbf{c} \\
        \mathbf{a} + (-\mathbf{b}) &= \mathbf{c} \\
        \mathbf{a} - \mathbf{b} &= \mathbf{c} \\
        \mathbf{a} &= \mathbf{b} + \mathbf{c}
      \end{align*}
  \end{enumerate}
  \newpage
\item {\bf Theory of Coding: Maximum-Likelihood Decoding}
  \begin{enumerate}[label={\arabic*},font={\bfseries}]
  \item
    \begin{center}
      \captionof{table}{Parity check equations in $C_1$}
      \begin{tabular}{ c | c c c c c c c }
        $C_1$ & $a_4$ & $a_1+a_3$ & $a_4=a_1+a_3$ & $a_5$ & $a_1+a_2+a_3$ & $a_5=a_1+a_2+a_3$ \\
        \hline
        $00000$ & $0$ & $0+0$ & \checkmark & $0$ & $0+0+0$ & \checkmark \\
        $00111$ & $1$ & $0+1$ & \checkmark & $1$ & $0+0+1$ & \checkmark \\
        $01001$ & $0$ & $0+0$ & \checkmark & $1$ & $0+1+0$ & \checkmark \\
        $01110$ & $1$ & $0+1$ & \checkmark & $0$ & $0+1+1$ & \checkmark \\
        $10011$ & $1$ & $1+0$ & \checkmark & $1$ & $1+0+0$ & \checkmark \\
        $10100$ & $0$ & $1+1$ & \checkmark & $0$ & $1+0+1$ & \checkmark \\
        $11010$ & $1$ & $1+0$ & \checkmark & $0$ & $1+1+0$ & \checkmark \\
        $11101$ & $0$ & $1+1$ & \checkmark & $1$ & $1+1+1$ & \checkmark
      \end{tabular}
    \end{center}
  \item
    \begin{enumerate}[label={(\alph*)}]
    \item
      \[
      C_2 = \Set{
        000000,
        001001,
        010111,
        011110,
        100011,
        101010,
        110000,
        111101
      }
      \]
    \item
      \begin{center}
        \captionof{table}{Distance in $C_2$}
        \begin{tabular}{ c | c c c c c c c c }
          $d(\mathbf{a}, \mathbf{b})$ & 000000 & 001001 & 010111 & 011110 & 100011 & 101010 & 110000 & 111101 \\
          \hline
          000000 &   & 2 & 4 & 4 & 3 & 3 & 2 & 5 \\
          001001 & 2 &  & 4 & 4 & 3 & 3 & 4 & 3 \\
          010111 & 4 & 4 &   & 2 & 3 & 5 & 4 & 3 \\
          011110 & 4 & 4 & 2 &   & 5 & 3 & 4 & 3 \\
          100011 & 3 & 3 & 3 & 5 &   & 2 & 3 & 4 \\
          101010 & 3 & 3 & 5 & 3 & 2 &   & 3 & 4 \\
          110000 & 2 & 4 & 4 & 4 & 3 & 3 &   & 3 \\
          111101 & 5 & 3 & 3 & 3 & 4 & 4 & 3 &
        \end{tabular}
      \end{center}
      The minimum distance of the code $C_2$ is $2$.
      \item Since the minimum distance is $C_2$, one error is sure to be
        detected in any codeword of $C_2$.
    \end{enumerate}
  \item $C_3 = \Set{0000,0101,1011,1110}$ where $a_3 = a_1$ and $a_4 = a_1+a_2$.
    \begin{center}
      \captionof{table}{Distance in $C_3$}
      \begin{tabular}{ c | c c c c }
        $d(\mathbf{a}, \mathbf{b})$ & 0000 & 0101 & 1011 & 1110 \\
        \hline
        0000 & 0 & 2 & 3 & 3 \\
        0101 & 2 & 0 & 3 & 3 \\
        1011 & 3 & 3 & 0 & 2 \\
        1110 & 3 & 3 & 2 & 0
      \end{tabular}
      \end{center}
    $$\min_{\mathbf{a} \in C_3, \mathbf{a}\ne\mathbf{b}} d(\mathbf{a}, \mathbf{b}) = 2$$
  \item
    \begin{itemize}
    \item $11111 \to 11101$
    \item $00101 \to 00111$
    \item $11000 \to 11010$
    \item $10011 \to 10011$
    \item $10001 \to 10011$
    \item $10111 \to 10011\ or\ 00111$
    \end{itemize}
  \end{enumerate}
\end{enumerate}


\documentclass{article}
\usepackage{amsmath}
\usepackage[margin=1.0in]{geometry}
\usepackage{xcolor}

\begin{document}

\noindent
Does $\displaystyle \sum_{n=1}^\infty \frac{n^2+1}{n^4+7}$
diverge, converge absolutely, or converge conditionally?

\subsection*{Solution 1}

$\displaystyle \sum_{n=1}^\infty \frac1{n^2}$ is a $p$-series with $p=2$. Since $p > 1$, the series $\displaystyle \sum_{n=1}^\infty \frac1{n^2}$ converges by the $p$-series test. If we use $a_n = \frac{n^2+1}{n^4+7}$ and $b_n = \frac1{n^2}$, then
\begin{align*}
\lim_{n \to \infty} \frac{a_n}{b_n}
&= \lim_{n \to \infty} \frac{n^2+1}{n^4+7} \cdot \frac{n^2}{1}\\
&= \lim_{n \to \infty} \frac{n^4+n^2}{n^4+7} \\
&= \lim_{n \to \infty} \frac{4n^3+2n}{4n^3} \text{ using L'hopital}\\
&= \lim_{n \to \infty} \frac{12n^2+2}{12n^2} \text{ using L'hopital}\\
&= \lim_{n \to \infty} \frac{24n}{24n} \text{ using L'hopital}\\
&= \lim_{n \to \infty} 1\\
&= 1
\end{align*}
So by the Limit Comparison Test, the series $\displaystyle \sum_{n=1}^\infty \frac{n^2+1}{n^4+7}$ converges.
Since $\sum |a_n| = \sum a_n$, the series $\displaystyle \sum_{n=1}^\infty \frac{n^2+1}{n^4+7}$ converges absolutely.

\subsection*{Solution 2}

We are looking for a fixed $K > 0$ such that
\[ \frac{n^2+1}{n^4+7} \leq \frac{K}{n^2}\]
By multiplying both sides by $n^2(n^4+7)$, we have
\[ n^4+n^2 \leq K(n^4+7)\]
\[ n^4+n^2 \leq Kn^4+7K\]
and it appears we can use $K=2$, because then we'd have
\[ n^4+n^2 \leq 2n^4+14\]
\[ n^4+n^2 \leq n^4+n^4+14\]
and $n^2 \leq n^4$ (when comparing second terms).
So we are ready to present starting from true inequalities.

With the scratch work above done, since $n^2 \leq n^4$, we have
\[ n^4+n^2 \leq n^4+n^4+14\]
\[ n^4+n^2 \leq 2n^4+14\]
\[ n^4+n^2 \leq 2(n^4+7)\]
by dividing both sides by $n^2(n^4+7)$, we get
\[ \frac{n^2+1}{n^4+7} \leq \frac{2}{n^2}\]

Since $\displaystyle \sum_{n=1}^\infty \frac2{n^2} = 2\sum_{n=1}^\infty \frac1{n^2}$ converges by the $p$-test, the series
$\displaystyle \sum_{n=1}^\infty \frac{n^2+1}{n^4+7}$ converges by the Direct Comparison Test.
Since $\sum |a_n| = \sum a_n$, the series $\displaystyle \sum_{n=1}^\infty \frac{n^2+1}{n^4+7}$ converges absolutely.

\end{document}%%%%%%%%%%%%%%%%%

\begin{align*}
L&=\lim_{n \to \infty} \sqrt[n]{|a_n|}\\
&= \lim_{n \to \infty} \sqrt[n]{\left| \right|}\\
\end{align*}

\begin{align*}
L&=\lim_{n \to \infty} \left|\frac{a_{n+1}}{a_n}\right|\\
&= \lim_{n \to \infty} \left| \right|\\
\end{align*}

\begin{align*}
\lim_{n \to \infty} a_n
&= \lim_{n \to \infty} \\
\end{align*}

Since $\sum |a_n| = \sum a_n$, the series $\displaystyle \sum_{n=1}^\infty AAAAAAAAAAAAAA$ converges absolutely.

Since $|r| < 1$, the series ...  converges by the Geometric Series Test.

Since $|r| \geq 1$, the series ...  diverges by the Geometric Series Test.

The function $f(x)=\frac{}{}$ is continuous, positive, and decreasing on $[1,\infty)$.

\subsection*{Solution}