\section{Clustering \textnormal{\sffamily --- Unsupervised}}

% ===
\emph{k-Means}
%\emph{Assignement}: $C_i \!=\! \{ x\!\in\!\mathbb{R}^d \,\vert\, i\!\in\!\arg\min\limits_j \norm{x\!-\!\mu_j} \}$

$\hat{R}(\mu) = \sum_{i=1}^n \min\limits_{j\in\{1,\ldots,k\}} \norm{x_i-\mu_j}_2^2 = \sum_i d(x_i,\mu_j)$

$\hat{\mu} =  \arg\!\min\limits_\mu \hat{R}(\mu)$ {\color{gray}$\leftarrow$ non-convex, NP-hard}\vspace{3pt}

\begin{highlightbox}
	\vspace{-3pt}
	\textbf{Lloyd's heuristic}: \textit{Initialize} centers $\mu_{1:k}^{(0)}$,\\
	\textit{assign} points to closest center {\small\color{gray} $\to \arg\min$},
	\textit{update centers} to mean of each cluster, repeat
\end{highlightbox}

% ===
\emph{k-Means++:}
\enskip $\to$ for initialization $2\ldots j$

$\operatorname{P}(\textrm{pick }x_l) = \frac{1}{z} d(x_i - \mu_{1:j-1})$, \enskip $\mu_j^{(0)} \leftarrow x_i$

%- Start with random data point as center\\
%- Add centers 2 to k randomly, proportionally to squared distance to closest selected center\\
%for $j=2$ to $k$:
%$i_j$ sampled with prob.\\
%$P(i_j=i) = \frac{1}{z} \underset{1\leq l<j}{min}||x_i-\mu_l||_2^2$; $\mu_j \leftarrow x_{i_j}$

% ===
\emph{Optimal \textit{k}-value:}
\enskip $\to$ ``Elbow'' trick \textit{or}

Regularization $L(\mu) = \min\limits_k \min\limits_{\mu_k} \hat R(\mu_{1:k}) + \lambda\,k$


\section{Application: Principal component analysis}

\begin{outcome}
  \begin{enumerate}
  \item Compute the principal components of a matrix $A$.
  \item Compute the centroid of a collection of data points.
  \item Find the $k$-dimensional subspace that best approximates a
    given collection of data points.
  \item Find the $k$-dimensional affine subspace that best
    approximates a given collection of data points.
  \item Compute the total squared distance of the data points to the
    best fit subspace (or best fit affine subspace).
  \end{enumerate}
\end{outcome}

In this section, we will explore an application of the diagonalization
of symmetric matrices called \textbf{principal component analysis}.
Imagine we are given a collection of data points such as the
following:
\begin{equation}\label{eqn:subspace-fitting}
  \begin{tikzpicture}[baseline=-0.5ex]
    \draw[thin,->] (-4,0) -- (4,0);
    \draw[thin,->] (0,-4) -- (0,4);
    % \draw[thick,blue] (-4,-4/1.29) -- (4,4/1.29);
    \fill[color=red] (-2.25,-1.21) circle (0.06);
    \fill[color=red] (-1.44,-1.28) circle (0.06);
    \fill[color=red] (2.29,1.82) circle (0.06);
    \fill[color=red] (0.25,0.43) circle (0.06);
    \fill[color=red] (-1.57,-1.07) circle (0.06);
    \fill[color=red] (2.51,2.00) circle (0.06);
    \fill[color=red] (-2.53,-2.34) circle (0.06);
    \fill[color=red] (-3.04,-2.40) circle (0.06);
    \fill[color=red] (0.84,0.36) circle (0.06);
    \fill[color=red] (-3.16,-2.73) circle (0.06);
    \fill[color=red] (-2.02,-1.88) circle (0.06);
    \fill[color=red] (3.06,2.44) circle (0.06);
    \fill[color=red] (-1.40,-1.12) circle (0.06);
    \fill[color=red] (-2.09,-1.34) circle (0.06);
    \fill[color=red] (-1.23,-1.13) circle (0.06);
    \fill[color=red] (0.48,0.06) circle (0.06);
    \fill[color=red] (-1.49,-1.20) circle (0.06);
    \fill[color=red] (0.98,0.75) circle (0.06);
    \fill[color=red] (1.40,0.89) circle (0.06);
    \fill[color=red] (-0.12,0.09) circle (0.06);
    \fill[color=red] (0.88,0.40) circle (0.06);
    \fill[color=red] (-0.38,0.55) circle (0.06);
    \fill[color=red] (0.34,0.46) circle (0.06);
    \fill[color=red] (0.45,0.21) circle (0.06);
    \fill[color=red] (-0.59,-0.63) circle (0.06);
    \fill[color=red] (-0.53,-0.29) circle (0.06);
    \fill[color=red] (2.10,1.35) circle (0.06);
    \fill[color=red] (1.65,1.60) circle (0.06);
    \fill[color=red] (-3.88,-2.83) circle (0.06);
    \fill[color=red] (-0.07,-0.01) circle (0.06);
    \fill[color=red] (-0.37,-0.57) circle (0.06);
    \fill[color=red] (-0.99,-0.75) circle (0.06);
    \fill[color=red] (-2.34,-2.08) circle (0.06);
    \fill[color=red] (3.63,3.15) circle (0.06);
    \fill[color=red] (1.37,0.48) circle (0.06);
    \fill[color=red] (-0.96,-0.74) circle (0.06);
    \fill[color=red] (3.51,2.79) circle (0.06);
    \fill[color=red] (-3.33,-2.72) circle (0.06);
    \fill[color=red] (0.39,0.09) circle (0.06);
    \fill[color=red] (-1.38,-1.17) circle (0.06);
    \fill[color=red] (-0.36,-0.65) circle (0.06);
    \fill[color=red] (1.38,0.66) circle (0.06);
    \fill[color=red] (1.85,1.24) circle (0.06);
    \fill[color=red] (2.35,1.85) circle (0.06);
    \fill[color=red] (0.85,0.25) circle (0.06);
    \fill[color=red] (-0.23,0.19) circle (0.06);
    \fill[color=red] (-0.90,-0.74) circle (0.06);
    \fill[color=red] (2.50,1.31) circle (0.06);
    \fill[color=red] (-0.45,-0.71) circle (0.06);
    \fill[color=red] (0.92,0.56) circle (0.06);
    \fill[color=red] (0.97,1.21) circle (0.06);
    \fill[color=red] (-0.85,-0.74) circle (0.06);
    \fill[color=red] (-0.10,0.13) circle (0.06);
    \fill[color=red] (0.32,0.49) circle (0.06);
    \fill[color=red] (2.58,1.57) circle (0.06);
    \fill[color=red] (-0.59,-0.48) circle (0.06);
    \fill[color=red] (2.28,1.50) circle (0.06);
    \fill[color=red] (1.21,0.94) circle (0.06);
    \fill[color=red] (-0.35,-0.13) circle (0.06);
    \fill[color=red] (-1.53,-1.26) circle (0.06);
    \fill[color=red] (-2.77,-2.14) circle (0.06);
    \fill[color=red] (1.23,1.02) circle (0.06);
    \fill[color=red] (2.61,1.88) circle (0.06);
    \fill[color=red] (-0.04,-0.05) circle (0.06);
    \fill[color=red] (2.09,1.30) circle (0.06);
    \fill[color=red] (2.37,1.52) circle (0.06);
    \fill[color=red] (-2.01,-1.69) circle (0.06);
    \fill[color=red] (0.48,0.72) circle (0.06);
    \fill[color=red] (0.23,0.49) circle (0.06);
    \fill[color=red] (-1.16,-0.95) circle (0.06);
    \fill[color=red] (-0.14,-0.18) circle (0.06);
    \fill[color=red] (-1.57,-1.68) circle (0.06);
    \fill[color=red] (0.39,0.15) circle (0.06);
    \fill[color=red] (-0.24,-0.29) circle (0.06);
  \end{tikzpicture}
\end{equation}
Although these points are spread out in two dimensions, they seem to
be located pretty close to a 1-dimensional subspace. Probably the best
way to interpret this particular data set is to think of the points as
being ``essentially'' on a line, up to some small random errors.

More generally, suppose we are given a collection of data points in
$n$-dimensional space, and we are looking for a $k$-dimensional
subspace that all data points are close to.  This is an important way
to make sense of high-dimensional data. For example, it would be very
difficult to visualize data in a $100$-dimensional space. However, it
we knew that the data points lie very close to a 2-dimensional
subspace, then we could project all of the points to the subspace to
obtain a 2-dimensional image of the data.

To state the problem more precisely, let us introduce the following
notation. If $W$ is a subspace of $\R^n$ and $\vect{v}\in\R^n$ is a
vector, let us write $d(\vect{v},W)$ for the shortest distance from
$\vect{v}$ to $W$ (i.e., the distance from $\vect{v}$ to $W$ along a
line that is perpendicular to $W$). Moreover, if $W$ is a subspace of
$\R^n$ and $\vect{v}_1,\ldots,\vect{v}_m\in\R^n$ are the position
vectors of $m$ points, we define the \textbf{total squared distance}%
\index{distance!total squared distance}%
\index{squared distance}%
\index{total squared distance}%
\index{subspace fitting!total squared distance} of the points to
the subspace to be the quantity
\begin{equation*}
  D = d(\vect{v}_1,W)^2 + \ldots + d(\vect{v}_m,W)^2.
\end{equation*}
Then the problem we would like to solve can be stated as follows:

\begin{problem}{Subspace fitting problem}{subspace-fitting}
  Given vectors $\vect{v}_1,\ldots,\vect{v}_m\in\R^n$ and given an
  integer $k\leq n$, find the $k$-dimensional subspace
  $W\subseteq\R^n$ that minimizes the total squared distance, i.e.,
  such that $D$ is as small as possible%
  \index{subspace fitting}%
  \index{fitting!subspace fitting}.
\end{problem}

The following proposition gives us a method for solving the subspace
fitting problem. It turns out that the key ingredient in solving this
problem is the diagonalization of symmetric matrices. The method was
discovered by Gale Young%
\index{Young, Gale}%
\index{Gale Young} in 1937.

\begin{proposition}{Solution of the subspace fitting problem}{subspace-fitting}
  Given vectors $\vect{v}_1,\ldots,\vect{v}_m\in\R^n$ and $k\leq n$,
  the optimal solution to the subspace fitting problem can be computed
  as follows:
  \begin{enumerate}
  \item Let $A$ be the $m\times n$-matrix whose rows are
    $\vect{v}_1^T,\ldots,\vect{v}_m^T$. (Or equivalently, $A^T$ is the
    $n\times m$-matrix whose columns are
    $\vect{v}_1,\ldots,\vect{v}_m$.)
  \item Let $B=A^TA$. Then $B$ is a positive semidefinite
    $n\times n$-matrix.
  \item By Proposition~\ref{prop:characterize-positive}, all
    eigenvalues of $B$ are real and non-negative. Let
    $\eigenvar_1,\ldots,\eigenvar_n$ be the eigenvalues of $B$, listed
    according to their multiplicity and in decreasing order, i.e., so
    that $\eigenvar_1\geq\eigenvar_2\geq\ldots\geq\eigenvar_n\geq
    0$. Let $\vect{u}_1,\ldots,\vect{u}_n$ be the corresponding
    eigenvectors.
  \item Then $W=\sspan\set{\vect{u}_1,\ldots,\vect{u}_k}$ is the
    solution to the subspace fitting problem.  Moreover, the total
    squared distance of the points to this subspace is
    \begin{equation*}
      D = \eigenvar_{k+1} + \ldots + \eigenvar_n.
    \end{equation*}
  \end{enumerate}
\end{proposition}

\begin{example}{Subspace fitting in $\R^2$}{subspace-fitting-r2}
  Consider the following collection of points in $\R^2$:
  \begin{equation*}
    \set{
      \begin{mymatrix}{r}  2 \\ -3 \end{mymatrix},
      \begin{mymatrix}{r} -1 \\  0 \end{mymatrix},
      \begin{mymatrix}{r}  2 \\  3 \end{mymatrix},
      \begin{mymatrix}{r} -6 \\ -7 \end{mymatrix},
      \begin{mymatrix}{r}  6 \\ 11 \end{mymatrix},
      \begin{mymatrix}{r}  0 \\ -1 \end{mymatrix},
      \begin{mymatrix}{r}  1 \\  6 \end{mymatrix},
      \begin{mymatrix}{r} -2 \\ -3 \end{mymatrix},
      \begin{mymatrix}{r} -7 \\ -6 \end{mymatrix}
    }.
  \end{equation*}
  Find the 1-dimensional subspace that best approximates this
  collection of points. What is the total squared distance of the
  points to the subspace?
\end{example}

\begin{solution}
  We follow the steps outlined in
  Proposition~\ref{prop:subspace-fitting}.
  \begin{enumerate}
  \item We have
    \begin{equation*}
      A^T = \begin{mymatrix}{rrrrrrrrr}
        2 & -1 & 2 & -6 & 6 & 0 & 1 & -2 & -7 \\
        -3 & 0 & 3 & -7 & 11 & -1 & 6 & -3 & -6
      \end{mymatrix}.
    \end{equation*}
  \item We calculate
    \begin{equation*}
      B = A^TA = \begin{mymatrix}{rr}
        135 & 162 \\
        162 & 270
      \end{mymatrix}.
    \end{equation*}
  \item The eigenvalues of $B$ are $\eigenvar_1 = 378$ and
    $\eigenvar_2 = 27$, with corresponding eigenvectors
    \begin{equation*}
      \vect{u}_1 = \begin{mymatrix}{r} 2 \\ 3 \end{mymatrix}
      \quad\mbox{and}\quad
      \vect{u}_2 = \begin{mymatrix}{r} 3 \\ -2 \end{mymatrix}.
    \end{equation*}
  \item The desired subspace $W$ is spanned by the eigenvector
    corresponding to the largest eigenvalue, i.e.,
    $W=\sspan\set{\vect{u}_1}$. The total squared distance is
    $\eigenvar_2 = 27$.
  \end{enumerate}
  The space $W$ is shown in the following illustration, along with the
  original points:
  \begin{equation*}
    \begin{tikzpicture}[scale=0.15]
      \draw[thin,->] (-12,0) -- (12,0);
      \draw[thin,->] (0,-12) -- (0,12);
      \draw[thin] (0,10) -- (-1,10) node[left] {$10$};
      \draw[thin] (10,0) -- (10,-1) node[below] {$10$};
      \draw[thick,blue] (-8,-12) -- (8,12);
      \fill[color=red] (2,-3) circle (0.4);
      \fill[color=red] (-1, 0) circle (0.4);
      \fill[color=red] (2, 3) circle (0.4);
      \fill[color=red] (-6, -7) circle (0.4);
      \fill[color=red] (6, 11) circle (0.4);
      \fill[color=red] (0, -1) circle (0.4);
      \fill[color=red] (1, 6) circle (0.4);
      \fill[color=red] (-2, -3) circle (0.4);
      \fill[color=red] (-7, -6) circle (0.4);
    \end{tikzpicture}
  \end{equation*}
  Of course, the example was rigged to ensure that the eigenvalues are
  integers. In real life, the entries of $A$ and $B$, as well as the
  eigenvalues and components of the eigenvectors are usually arbitrary
  real numbers.
\end{solution}

\begin{example}{Subspace fitting in $\R^3$}{subspace-fitting-r3}
  Consider the following collection of points in $\R^3$:
  \begin{equation*}
    \set{
      \begin{mymatrix}{r} -7 \\ 4 \\ 5 \end{mymatrix},
      \begin{mymatrix}{r} 0 \\ 3 \\ 3 \end{mymatrix},
      \begin{mymatrix}{r} 2 \\ -5 \\ -4 \end{mymatrix},
      \begin{mymatrix}{r} 10 \\ -4 \\ 1 \end{mymatrix},
      \begin{mymatrix}{r} -2 \\ 5 \\ 4 \end{mymatrix},
      \begin{mymatrix}{r} -8 \\ -1 \\ -5 \end{mymatrix},
      \begin{mymatrix}{r} 5 \\ 4 \\ 2 \end{mymatrix},
      \begin{mymatrix}{r} -6 \\ 9 \\ 6 \end{mymatrix},
      \begin{mymatrix}{r} 9 \\ -6 \\ 3 \end{mymatrix},
      \begin{mymatrix}{r} -2 \\ -7 \\ -8 \end{mymatrix}
    }.
  \end{equation*}
  \begin{enumialphparenastyle}
    \begin{enumerate}
    \item Find the 1-dimensional subspace that best approximates this
      collection of points.
    \item Find the 2-dimensional subspace that best approximates this
      collection of points.
    \item What is the 3-dimensional subspace that best approximates this
      collection of points?
    \end{enumerate}
  \end{enumialphparenastyle}
  In each case, what is the total squared distance of the points to
  the subspace?
\end{example}

\begin{solution}
  Again, we follow the steps from
  Proposition~\ref{prop:subspace-fitting}. We can do the calculations
  for parts (a), (b), and (c) at the same time.
  \begin{enumerate}
  \item We have
    \begin{equation*}
      A^T = \begin{mymatrix}{rrrrrrrrrr}
        -7 & 0 & 2 & 10 & -2 & -8 & 5 & -6 & 9 & -2 \\
        4 & 3 & -5 & -4 & 5 & -1 & 4 & 9 & -6 & -7 \\
        5 & 3 & -4 & 1 & 4 & -5 & 2 & 6 & 3 & -8 \\
      \end{mymatrix}.
    \end{equation*}
  \item We calculate
    \begin{equation*}
      B = A^TA = \begin{mymatrix}{rrr}
        367 & -154 & 16 \\
        -154 & 274 & 170 \\
        16 & 170 & 205 \\
      \end{mymatrix}.
    \end{equation*}
  \item The eigenvalues of $B$ are $\eigenvar_1 = 513$,
    $\eigenvar_2 = 306$, and $\eigenvar_3 = 27$, with corresponding
    eigenvectors
    \begin{equation*}
      \vect{u}_1 = \begin{mymatrix}{r} -2 \\ 2 \\ 1 \end{mymatrix},
      \quad
      \vect{u}_2 = \begin{mymatrix}{r} 2 \\ 1 \\ 2 \end{mymatrix}
      \quad\mbox{and}\quad
      \vect{u}_3 = \begin{mymatrix}{r} -1 \\ -2 \\ 2 \end{mymatrix}.
    \end{equation*}
  \end{enumerate}

  For part (a), the desired 1-dimensional subspace is spanned by the
  eigenvector corresponding to the largest eigenvalue, i.e., it is
  $\sspan\set{\vect{u}_1}$. The total squared distance is
  $\eigenvar_2+\eigenvar_3 = 306 + 27 = 333$.

  For part (b), the desired 2-dimensional subspace is spanned by the
  eigenvectors corresponding to the two largest eigenvalues, i.e., it
  is $\sspan\set{\vect{u}_1,\vect{u}_2}$. The total squared distance is
  $\eigenvar_3 = 27$.

  Finally, in part (c), the desired 3-dimensional subspace is spanned
  by all three eigenvectors; it is of course $\R^3$ itself, since it
  is the only 3-dimensional subspace. The total squared distance is
  $0$, since all points lie in the subspace.
\end{solution}

The vectors $\vect{u}_1,\ldots,\vect{u}_n$ that appear in the solution
of the subspace fitting problem are called the \textbf{principal
  components} of the matrix $A$.

\begin{definition}{Principal components}{principal-components}
  Let $A$ be an $m\times n$-matrix. The \textbf{principal components}%
  \index{principal component}%
  \index{component!principal component} of $A$ are the (normalized,
  orthogonal) eigenvectors $\vect{u}_1,\ldots,\vect{u}_n$ of the
  positive semidefinite $n\times n$-matrix $A^TA$. They are usually
  listed in order of decreasing eigenvalues.
\end{definition}

The first principal component $\vect{u}_1$ gives the direction in
which the rows of $A$ show the most variability. The second principal
component $\vect{u}_2$ gives the direction in which the rows of $A$
show the most remaining variability that is orthogonal to
$\vect{u}_1$. The third principal component $\vect{u}_3$ gives the
direction of most variability that is orthogonal to $\vect{u}_1$ and
$\vect{u}_2$, and so on.

% ----------------------------------------------------------------------
\subsection*{Subspace fitting vs. curve fitting}

In the particular case where $n=2$ and $k=1$, we are looking for a
1-dimensional subspace, i.e., a line through the origin, which best
fits the given 2-dimensional data, as in the illustration
{\eqref{eqn:subspace-fitting}} above or as in
Example~\ref{exa:subspace-fitting-r2}. On its face, the subspace
fitting problem in this case seems similar to the linear curve
fitting%
\index{curve fitting} problem we solved in
Section~\ref{sec:least-squares}. However, there is a subtle but
important difference: in linear curve fitting, we were seeking to
minimize the distances of the points from the line in the
$y$-direction, whereas in subspace fitting, we are seeking to minimize
the distances of the points from the subspace in the direction
perpendicular to the subspace. The following pair of pictures
illustrates the difference:
\begin{equation*}
  \begin{tikzpicture}[scale=0.8]
    \draw[thin,->] (-3,0) -- (3,0);
    \draw[thin,->] (0,-3) -- (0,3);
    \draw[thick,blue] (-3,-3*0.75) -- (3,3*0.75);
    \fill[color=red] (0,0) circle (0.12);
    \fill[color=red] (2,0) circle (0.12);
    \fill[color=red] (2,3) circle (0.12);
    \fill[color=red] (-2,0) circle (0.12);
    \fill[color=red] (-2,-3) circle (0.12);
    \draw[<->,black!50] (2,0) -- (2,1.5);
    \draw[<->,black!50] (2,3) -- (2,1.5);
    \draw[<->,black!50] (-2,0) -- (-2,-1.5);
    \draw[<->,black!50] (-2,-3) -- (-2,-1.5);
    \path (0,-3.5) node[below] {Linear curve fitting};
    \path[black!70] (2,-2) node {\begin{tabular}{c}minimize\\vertical\\distances\end{tabular}};
  \end{tikzpicture}
  \hspace{1in}
  \begin{tikzpicture}[scale=0.8]
    \draw[thin,->] (-3,0) -- (3,0);
    \draw[thin,->] (0,-3) -- (0,3);
    \draw[thick,blue] (-3*0.92,-3) -- (3*0.92, 3);
    \fill[color=red] (0,0) circle (0.12);
    \fill[color=red] (2,0) circle (0.12);
    \fill[color=red] (2,3) circle (0.12);
    \fill[color=red] (-2,0) circle (0.12);
    \fill[color=red] (-2,-3) circle (0.12);
    \draw[<->,black!50] (2,0) -- (0.997*0.92,0.997);
    \draw[<->,black!50] (2,3) -- (2.621*0.92,2.621);
    \draw[<->,black!50] (-2,0) -- (-0.997*0.92,-0.997);
    \draw[<->,black!50] (-2,-3) -- (-2.621*0.92,-2.621);
    \path (0,-3.5) node[below] {Subspace fitting};
    \path[black!70] (2,-2) node {\begin{tabular}{c}minimize\\distances\\perpendicular\\to subspace\end{tabular}};
  \end{tikzpicture}
\end{equation*}

% ----------------------------------------------------------------------
\subsection*{Affine fitting}

So far, we have been looking to approximate a given collection of
points by a {\em subspace}, which necessarily passes through the
origin. But sometimes the points may not be near the origin, as in
this example:
\begin{equation*}
  \begin{tikzpicture}[baseline=-0.5ex, scale=0.8]
    \draw[thin,->] (-4,0) -- (4,0);
    \draw[thin,->] (0,-4) -- (0,4);
    % \draw[thick, blue] (0.91,1.76) +(-6,-6/-3.20) -- +(6,6/-3.20);
    \fill[color=red] (-0.54,2.53) circle (0.075);
    \fill[color=red] (4.26,0.53) circle (0.075);
    \fill[color=red] (-0.24,2.38) circle (0.075);
    \fill[color=red] (1.98,1.52) circle (0.075);
    \fill[color=red] (0.28,2.14) circle (0.075);
    \fill[color=red] (5.71,0.05) circle (0.075);
    \fill[color=red] (2.90,1.56) circle (0.075);
    \fill[color=red] (-3.21,2.97) circle (0.075);
    \fill[color=red] (5.44,-0.09) circle (0.075);
    \fill[color=red] (-0.37,1.87) circle (0.075);
    \fill[color=red] (-0.46,2.02) circle (0.075);
    \fill[color=red] (2.21,1.49) circle (0.075);
    \fill[color=red] (-0.91,2.23) circle (0.075);
    \fill[color=red] (0.81,1.80) circle (0.075);
    \fill[color=red] (0.47,1.91) circle (0.075);
    \fill[color=red] (0.96,2.34) circle (0.075);
    \fill[color=red] (-0.93,2.29) circle (0.075);
    \fill[color=red] (-0.35,2.09) circle (0.075);
    \fill[color=red] (1.99,1.11) circle (0.075);
    \fill[color=red] (2.34,1.31) circle (0.075);
    \fill[color=red] (-0.99,2.30) circle (0.075);
    \fill[color=red] (-2.26,2.78) circle (0.075);
    \fill[color=red] (4.07,0.84) circle (0.075);
    \fill[color=red] (0.50,2.19) circle (0.075);
    \fill[color=red] (-1.08,2.76) circle (0.075);
    \fill[color=red] (-0.10,2.07) circle (0.075);
    \fill[color=red] (1.49,1.46) circle (0.075);
    \fill[color=red] (2.05,1.08) circle (0.075);
    \fill[color=red] (4.18,0.74) circle (0.075);
    \fill[color=red] (0.29,1.98) circle (0.075);
    \fill[color=red] (-1.37,2.64) circle (0.075);
    \fill[color=red] (0.75,2.22) circle (0.075);
    \fill[color=red] (0.55,1.91) circle (0.075);
    \fill[color=red] (2.41,1.73) circle (0.075);
    \fill[color=red] (3.70,1.02) circle (0.075);
    \fill[color=red] (1.73,1.85) circle (0.075);
    \fill[color=red] (-0.03,1.99) circle (0.075);
    \fill[color=red] (2.60,1.00) circle (0.075);
    \fill[color=red] (-0.57,2.42) circle (0.075);
    \fill[color=red] (2.44,0.87) circle (0.075);
    \fill[color=red] (1.67,1.20) circle (0.075);
    \fill[color=red] (2.27,1.53) circle (0.075);
    \fill[color=red] (-1.48,2.27) circle (0.075);
    \fill[color=red] (-0.65,1.73) circle (0.075);
    \fill[color=red] (3.53,0.83) circle (0.075);
    \fill[color=red] (0.00,2.10) circle (0.075);
    \fill[color=red] (0.03,2.06) circle (0.075);
    \fill[color=red] (-0.40,2.34) circle (0.075);
    \fill[color=red] (0.48,1.91) circle (0.075);
    \fill[color=red] (-0.59,2.74) circle (0.075);
    \fill[color=red] (0.71,1.61) circle (0.075);
    \fill[color=red] (-0.04,1.88) circle (0.075);
    \fill[color=red] (0.33,1.89) circle (0.075);
    \fill[color=red] (4.66,0.67) circle (0.075);
    \fill[color=red] (3.68,1.23) circle (0.075);
    \fill[color=red] (1.06,1.51) circle (0.075);
    \fill[color=red] (-0.08,2.08) circle (0.075);
    \fill[color=red] (-0.24,1.75) circle (0.075);
    \fill[color=red] (4.36,0.91) circle (0.075);
    \fill[color=red] (1.17,1.51) circle (0.075);
    \fill[color=red] (1.25,2.07) circle (0.075);
    \fill[color=red] (2.36,1.48) circle (0.075);
    \fill[color=red] (-1.21,2.26) circle (0.075);
    \fill[color=red] (-0.13,1.83) circle (0.075);
    \fill[color=red] (1.23,1.51) circle (0.075);
    \fill[color=red] (2.30,1.40) circle (0.075);
    \fill[color=red] (-1.08,2.50) circle (0.075);
    \fill[color=red] (-1.62,2.33) circle (0.075);
    \fill[color=red] (-0.27,2.32) circle (0.075);
    \fill[color=red] (-1.75,2.52) circle (0.075);
    \fill[color=red] (3.37,1.10) circle (0.075);
    \fill[color=red] (-1.10,2.10) circle (0.075);
    \fill[color=red] (-0.54,1.85) circle (0.075);
    \fill[color=red] (2.16,1.16) circle (0.075);
    \fill[color=red] (-0.15,2.11) circle (0.075);
  \end{tikzpicture}
\end{equation*}
In this case, approximating the points by a subspace passing through
the origin does not make much sense. Instead, we should be looking for
an \textbf{affine subspace}. An affine subspace is similar to a
subspace, except it does not necessarily contain the origin.

\begin{definition}{Affine subspace}{affine-subspace}
  Let $V$ be a vector space. A subset $A\subseteq V$ is called an
  \textbf{affine subspace}%
  \index{affine subspace}%
  \index{subspace!affine} of $V$ if $A$ is either empty, or else of
  the form
  \begin{equation*}
    A = \vect{v} + W = \set{\vect{v}+\vect{w} \mid \vect{w}\in W},
  \end{equation*}
  where $\vect{v}\in V$ and $W$ is a subspace of\/ $V$.
\end{definition}

For example, in Chapter~\ref{cha:lines-and-planes}, we considered
lines and planes in $\R^n$ that pass through a given point (not
necessarily the origin). These are examples of affine subspaces of
$\R^n$. The affine subspace fitting problem is analogous to the
subspace fitting problem:

\begin{problem}{Affine subspace fitting problem}{affine-subspace-fitting}
  Given the position vectors $\vect{v}_1,\ldots,\vect{v}_m$ of $m$
  points in $\R^n$, and given an integer $k\leq n$, find the
  $k$-dimensional affine subspace $A\subseteq\R^n$ that minimizes the
  total squared distance from the points to $A$%
  \index{affine subspace fitting}%
  \index{subspace fitting!affine}%
  \index{fitting!affine subspace fitting}.
\end{problem}

It turns out that the optimal solution to the affine subspace fitting
problem can be computed by first computing the \textbf{centroid} of
the points, shifting the whole problem so that the centroid is at the
origin, and then solving an ordinary subspace fitting problem.

\begin{definition}{Centroid}{centroid}
  Given $m$ vectors $\vect{v}_1,\ldots,\vect{v}_m$, their
  \textbf{centroid}%
  \index{centroid} is the vector
  \begin{equation*}
    \centroid{\vect{v}} = \frac{1}{m}(\vect{v}_1+\ldots+\vect{v}_m).
  \end{equation*}
  It is also sometimes called the \textbf{average}%
  \index{average!of vectors} or the \textbf{center of mass}%
  \index{center of mass} of the vectors.
\end{definition}

\begin{proposition}{Solution of the affine subspace fitting problem}{affine-subspace-fitting}
  Given vectors $\vect{v}_1,\ldots,\vect{v}_m\in\R^n$ and $k\leq n$,
  the optimal solution to the affine subspace fitting problem can be
  computed as follows:
  \begin{enumerate}
  \item Compute the centroid $\centroid{\vect{v}} =
    \frac{1}{m}(\vect{v}_1+\ldots+\vect{v}_m)$ of the vectors.
  \item Let $\vect{w}_i = \vect{v}_i - \centroid{\vect{v}}$, for all
    $i=1,\ldots,n$.
  \item Compute the solution $W$ to the (ordinary) subspace fitting
    problem for $\vect{w}_1,\ldots,\vect{w}_m$, as in
    Proposition~\ref{prop:subspace-fitting}.
  \end{enumerate}
  Then the best solution to the affine subspace problem is
  $\centroid{v} + W$.
\end{proposition}

\begin{example}{Affine subspace fitting problem}{affine-subspace-fitting}
  Consider the following collection of points in $\R^2$:
  \begin{equation*}
    \set{
      \begin{mymatrix}{r} 10 \\ -6 \end{mymatrix},
      \begin{mymatrix}{r} 2 \\ 10 \end{mymatrix},
      \begin{mymatrix}{r} 5 \\ -1 \end{mymatrix},
      \begin{mymatrix}{r} 8 \\ 3 \end{mymatrix},
      \begin{mymatrix}{r} 2 \\ 5 \end{mymatrix},
      \begin{mymatrix}{r} 3 \\ 3 \end{mymatrix},
      \begin{mymatrix}{r} 4 \\ 11 \end{mymatrix},
      \begin{mymatrix}{r} 10 \\ -1 \end{mymatrix},
      \begin{mymatrix}{r} 1 \\ 12 \end{mymatrix}
    }.
  \end{equation*}
  Find the 1-dimensional affine subspace that best approximates this
  collection of points. What is the total squared distance of the
  points to the subspace?
\end{example}

\begin{solution}
  We start by computing the centroid:
  \begin{equation*}
    \centroid{\vect{v}} =
    \frac{1}{9}(\vect{v}_1+\ldots+\vect{v}_9)
    = \frac{1}{9}\begin{mymatrix}{r} 45 \\ 36 \end{mymatrix}
    = \begin{mymatrix}{r} 5 \\ 4 \end{mymatrix}.
  \end{equation*}
  Next, we shift all vectors by $-\centroid{\vect{v}}$ to get a new
  collection of vectors $\vect{w}_1,\ldots,\vect{w}_9$ centered at the
  origin.
  For example,
  \begin{eqnarray*}
    \vect{w}_1 ~~=~~ \vect{v}_1 - \centroid{\vect{v}}
    &=& \begin{mymatrix}{r} 10 \\ -6 \end{mymatrix}
    - \begin{mymatrix}{r} 5 \\ 4 \end{mymatrix}
    ~~=~~ \begin{mymatrix}{r} 5 \\ -10 \end{mymatrix},
    \\
    \vect{w}_2 ~~=~~ \vect{v}_2 - \centroid{\vect{v}}
    &=& \begin{mymatrix}{r} 2 \\ 10 \end{mymatrix}
    - \begin{mymatrix}{r} 5 \\ 4 \end{mymatrix}
    ~~=~~ \begin{mymatrix}{r} -3 \\ 6 \end{mymatrix},
  \end{eqnarray*}
  and so on. We get
  \begin{equation*}
    \set{\vect{w}_1,\ldots,\vect{w}_9} =
    \set{
      \begin{mymatrix}{r} 5 \\ -10 \end{mymatrix},
      \begin{mymatrix}{r} -3 \\ 6 \end{mymatrix},
      \begin{mymatrix}{r} 0 \\ -5 \end{mymatrix},
      \begin{mymatrix}{r} 3 \\ -1 \end{mymatrix},
      \begin{mymatrix}{r} -3 \\ 1 \end{mymatrix},
      \begin{mymatrix}{r} -2 \\ -1 \end{mymatrix},
      \begin{mymatrix}{r} -1 \\ 7 \end{mymatrix},
      \begin{mymatrix}{r} 5 \\ -5 \end{mymatrix},
      \begin{mymatrix}{r} -4 \\ 8 \end{mymatrix}
    }.
  \end{equation*}
  Next we, we proceed as in Proposition~\ref{prop:subspace-fitting} to
  find the best subspace fitting $\vect{w}_1,\ldots,\vect{w}_9$. We
  have
  \begin{equation*}
    A^T = \begin{mymatrix}{rrrrrrrrr}
      5 & -3 & 0 & 3 & -3 & -2 & -1 & 5 & -4 \\
      -10 & 6 & -5 & -1 & 1 & -1 & 7 & -5 & 8 \\
    \end{mymatrix}
  \end{equation*}
  and
  \begin{equation*}
    B = A^TA = \begin{mymatrix}{rr}
      98 & -136 \\
      -136 & 302 \\
    \end{mymatrix}.
  \end{equation*}
  The eigenvalues of $B$ are $\eigenvar_1 = 370$ and $\eigenvar_2 =
  30$, with corresponding eigenvectors
  \begin{equation*}
    \vect{u}_1 = \begin{mymatrix}{r} 1 \\ -2 \end{mymatrix}
    \quad\mbox{and}\quad
    \vect{u}_2 = \begin{mymatrix}{r} 2 \\ 1 \end{mymatrix}
  \end{equation*}
  Thus, the best-fitting 1-dimensional subspace for
  $\vect{w}_1,\ldots,\vect{w}_9$ is $W = \sspan\set{\vect{u}_1}$, and
  the best-fitting 1-dimensional affine subspace for
  $\vect{v}_1,\ldots,\vect{v}_9$ is
  \begin{equation*}
    \centroid{\vect{v}} + W
    = \set{\centroid{\vect{v}} + \vect{w} \mid \vect{w}\in W}
    = \set{\left.\begin{mymatrix}{r} 5 \\ 4 \end{mymatrix} +
        t\begin{mymatrix}{r} 1 \\ -2 \end{mymatrix} ~\right\vert~ t\in\R}.
  \end{equation*}
  Note that this is the equation of a line passing through the
  centroid $\centroid{\vect{v}}$, and with direction vector
  $\vect{u}_1$. The points $\vect{v}_1,\ldots,\vect{v}_9$, their
  centroid, and the affine subspace $\centroid{\vect{v}} + W$ are
  shown in the following illustration:
  \begin{equation*}
    \begin{tikzpicture}[scale=0.2]
      \draw[thin,->] (-12,0) -- (12,0);
      \draw[thin,->] (0,-12) -- (0,12);
      \draw[thin] (0,10) -- (-1,10) node[left] {$10$};
      \draw[thin] (10,0) -- (10,-1) node[below] {$10$};
      \draw[thick,blue] (5,4) +(-6,12) -- +(6,-12);
      \fill[color=blue] (5,4) circle (0.6);
      \draw[blue, ->] (5,4) +(8,4) node[right,yshift=4] {centroid} -- +(0.4,0.2);
      \fill[color=red] (10, -6) circle (0.3);
      \fill[color=red] (10, -1) circle (0.3);
      \fill[color=red] (5, -1) circle (0.3);
      \fill[color=red] (8, 3) circle (0.3);
      \fill[color=red] (3, 3) circle (0.3);
      \fill[color=red] (2, 5) circle (0.3);
      \fill[color=red] (2, 10) circle (0.3);
      \fill[color=red] (4, 11) circle (0.3);
      \fill[color=red] (1, 12) circle (0.3);
    \end{tikzpicture}
  \end{equation*}
  \vspace{-4ex}\par
\end{solution}

% ----------------------------------------------------------------------
\subsection*{Application to U.S. Senate voting data}

The United States Senate%
\index{senate}%
\index{U.S. Senate}%
\index{United States Senate} votes on a lot of things: motions,
resolutions, amendments, and bills, among other things. Many of these
votes are roll call votes%
\index{voting}%
\index{roll call vote}, which means that the vote of every individual
senator is recorded (as opposed to a voice vote, where only the
outcome is recorded). Roll call data for the last 3 decades is
publicly available and can be downloaded from the U.S. Senate website
at \url{https://www.senate.gov/legislative/votes.htm}.

We will now explore how to use linear algebra, and in particular
principal component analysis, to gain some useful information from the
voting records.\footnote{This example was inspired by Examples~11.2.13
  and 11.3.15 of {\em ``Coding the Matrix: Linear Algebra through
    Computer Science Applications''} by Philip N. Klein.} I have made
a spreadsheet containing the votes of 99 senators for the first 200
roll call votes of 2007. Each row in the spreadsheet corresponds to a
senator, listed in alphabetical order from Daniel Akaka of Hawaii to
Ron Wyden of Oregon. I omitted one senator who died during 2007. Each
column of the spreadsheet corresponds to a vote. For example, the
first roll call vote of 2007 was on a resolution to honour President
Gerald Ford (it passed 88 to 0). Each cell of the spreadsheet contains
the number $1$ if the senator voted ``yes'', the number $-1$ if the
senator voted ``no'', and the number $0$ if the senator did not
vote. The spreadsheet is available from
\url{https://www.mathstat.dal.ca/~selinger/linear-algebra/} under
``Supplementary materials''. Here are the first few rows and columns
of the spreadsheet.
\begin{equation*}
  \begin{array}{|l|r|r|r|r|r|r|r|r}
    \hline
    \mbox{Akaka, Daniel (D) HI} & 1 & 1 & 1 & 1 & 1 & -1 & 1 & \ldots \\\hline
    \mbox{Alexander, Lamar (R) TN} & 0 & 1 & -1 & 1 & -1 & -1 & 1 & \ldots \\\hline
    \mbox{Allard, A. (R) CO} & 1 & 1 & -1 & -1 & -1 & 1 & 1 & \ldots \\\hline
    \mbox{Baucus, Max (D) MT} & 1 & 1 & 1 & 1 & 1 & -1 & 1 & \ldots \\\hline
    \mbox{Bayh, Evan (D) IN} & 1 & 1 & 1 & -1 & 1 & -1 & 1 & \ldots \\\hline
    \mbox{Bennett, Robert (R) UT} & 1 & 1 & -1 & 1 & 1 & -1 & 1 & \ldots \\\hline
    \multicolumn{1}{|c|}{\vdots} & \vdots & \vdots & \vdots & \vdots & \vdots & \vdots & \vdots & \ddots
  \end{array}
\end{equation*}
The human mind is not very well equipped to deal with such massive
amounts of data. Rather than listing 122 motions that Senator X
supported and 78 motions that she opposed, we like to come up with
abstractions, such as Senator X is ``conservative'', ``pro choice'',
``pro business'', ``hawkish'', etc. However, the problem with
abstractions is that they do not necessarily mean anything in the real
world. In the real world, a senator's record is just a sequence of
votes.

We will represent each senator by a vector in $\R^{200}$, which
corresponds to a row of the above table. For example, to Senator
Akaka, we associate the vector
\begin{equation*}
  \begin{mymatrix}{c} 1 \\ 1 \\ 1 \\ 1 \\ 1 \\ -1 \\ 1 \\ \vdots \end{mymatrix}
  \in\R^{200}.
\end{equation*}
Thus, we can represent each senator (or more precisely, each senator's
voting record) as a point in 200-dimensional space. In this way, the
voting data can be interpreted as $99$ points in $\R^{200}$.

Unfortunately, 200 dimensions are impossible to visualize. But what if
the voting records of all the senators lie on (or at least close to) a
much-smaller-dimensional affine subspace? This is actually not an
unreasonable expectation; after all, there are probably only a handful
of issues most senators care about. For example, if a certain senator
supports gun control, he will be likely to vote a certain way on
measures that affect gun control. If another senator supports the gun
lobby, she is likely to vote the opposite way.

We can thus consider this as an instance of the affine subspace
problem: we are looking for a low-dimensional affine subspace that is
close to all $99$ points. Following the method of
Proposition~\ref{prop:affine-subspace-fitting}, we first find the
centroid of the points, and then we compute a certain
$99\times 200$-matrix $A$ and a positive semidefinite
$200\times 200$-matrix $B=A^TA$. Using software, we can find the
eigenvalues and -vectors of $B$. The first few eigenvalues (in
decreasing order) are:
\begin{equation*}
  \eigenvar_1 = 7255.65,\quad
  \eigenvar_2 = 519.16,\quad
  \eigenvar_3 = 430.60,\quad
  \eigenvar_4 = 278.05,\quad\mbox{and}\quad
  \eigenvar_5 = 230.56.
\end{equation*}
All of the remaining eigenvalues are less than $200$, and the sum of
the remaining eigenvalues is
$\eigenvar_6+\ldots+\eigenvar_{200}=3913.46$. This means that the vast
majority of the voting behavior of each senator is determined by a
single dimension, given by the eigenvector corresponding to the
eigenvalue $\eigenvar_1$. In other words, there is a $1$-dimensional
affine subspace that all $99$ points are pretty close to. If we
project each senator to this affine subspace, we get the following
picture:
\begin{equation*}
  \def\dataD#1#2#3#4#5#6#7{\fill[dem](#5,0) circle (0.09);}
  \def\dataR#1#2#3#4#5#6#7{\fill[rep](#5,0) circle (0.09);}
  \def\dataDx#1#2#3#4#5#6#7{\fill[dem](#5,0) circle (0.09);
    \draw[dem,->,shorten >=0.15cm](#5,0) +(#7,1) node[above] {\rotatebox{90}{\footnotesize #1, #4}} -- +(0,0);}
  \def\dataRx#1#2#3#4#5#6#7{\fill[rep](#5,0) circle (0.09);
    \draw[rep,->,shorten >=0.15cm](#5,0) +(#7,1) node[above] {\rotatebox{90}{\footnotesize #1, #4}} -- +(0,0);}
  \def\dataDy#1#2#3#4#5#6#7{\fill[dem](#5,0) circle (0.09);
    \draw[dem,->,shorten >=0.15cm](#5,0) +(#7,-1) node[below] {\rotatebox{90}{\footnotesize #1, #4}} -- +(0,0);}
  \def\dataRy#1#2#3#4#5#6#7{\fill[rep](#5,0) circle (0.09);
    \draw[rep,->,shorten >=0.15cm](#5,0) +(#7,-1) node[below] {\rotatebox{90}{\footnotesize #1, #4}} -- +(0,0);}
  \begin{tikzpicture}[scale=0.75]
    \draw (-12.5,0) -- (10.5,0);
    \dataRx{DeMint}{James}{R}{SC}{-11.856041}{4.315015}{0}
    \dataRy{Cornyn}{John}{R}{TX}{-11.446925}{1.720794}{0}
    \dataRx{Allard}{A.}{R}{CO}{-11.414729}{2.353678}{0}
    \dataR{Ensign}{John}{R}{NV}{-11.329050}{2.598885}{0}
    \dataR{Inhofe}{Jim}{R}{OK}{-11.161216}{4.313828}{0}
    \dataRy{Coburn}{Tom}{R}{OK}{-11.095925}{6.451745}{0}
    \dataRx{Enzi}{Michael}{R}{WY}{-10.786882}{0.794427}{0}
    \dataR{Burr}{Richard}{R}{NC}{-10.716488}{2.508468}{0}
    \dataR{Bunning}{Jim}{R}{KY}{-10.674428}{1.409311}{0}
    \dataR{Chambliss}{Saxby}{R}{GA}{-10.639465}{1.781387}{0}
    \dataR{Isakson}{Johnny}{R}{GA}{-10.548229}{0.724451}{0}
    \dataRy{Sessions}{Jeff}{R}{AL}{-10.510782}{1.437248}{0}
    \dataR{Gregg}{Judd}{R}{NH}{-10.399051}{1.990574}{0}
    \dataRx{McConnell}{Mitch}{R}{KY}{-10.375319}{-1.293286}{0}
    \dataR{Dole}{Elizabeth}{R}{NC}{-10.258572}{1.732827}{0}
    \dataRy{Kyl}{Jon}{R}{AZ}{-10.042393}{2.633156}{0}
    \dataR{Craig}{Larry}{R}{ID}{-9.996687}{-0.803843}{0}
    \dataR{Crapo}{Mike}{R}{ID}{-9.987981}{-0.982945}{0}
    \dataRx{Graham}{Lindsey}{R}{SC}{-9.942743}{2.090629}{0}
    \dataR{Sununu}{John}{R}{NH}{-9.713036}{1.627775}{0}
    \dataRy{Roberts}{Pat}{R}{KS}{-9.682832}{-1.781188}{0}
    \dataR{Martinez}{Mel}{R}{FL}{-9.602384}{1.021259}{0}
    \dataR{Thune}{John}{R}{SD}{-9.412180}{0.719407}{0}
    \dataR{Hutchison}{Kay}{R}{TX}{-9.362272}{-0.132493}{0}
    \dataR{Lott}{Trent}{R}{MS}{-9.333867}{-1.703358}{0}
    \dataRx{Hatch}{Orrin}{R}{UT}{-9.325404}{-1.647643}{0}
    \dataR{Corker}{Bob}{R}{TN}{-9.321180}{1.242018}{0}
    \dataR{Grassley}{Charles}{R}{IA}{-9.274839}{1.535256}{0}
    \dataRy{Vitter}{David}{R}{LA}{-9.268543}{3.508303}{0}
    \dataRx{Alexander}{Lamar}{R}{TN}{-8.869856}{-2.298938}{0}
    \dataRy{Shelby}{Richard}{R}{AL}{-8.861314}{0.147906}{0}
    \dataRx{Bennett}{Robert}{R}{UT}{-8.230363}{-5.265448}{0}
    \dataRy{Cochran}{Thad}{R}{MS}{-8.204309}{-3.833549}{0}
    \dataRx{Bond}{Christopher}{R}{MO}{-7.907050}{-3.678825}{0}
    \dataRy{Brownback}{Samuel}{R}{KS}{-7.841176}{0.525267}{0}
    \dataRx{Warner}{John}{R}{VA}{-7.511971}{-3.669235}{0}
    \dataRy{Murkowski}{Lisa}{R}{AK}{-6.411501}{-5.705682}{0}
    \dataR{Domenici}{Pete}{R}{NM}{-6.367823}{-4.162556}{0}
    \dataRx{Hagel}{Chuck}{R}{NE}{-6.362849}{0.617170}{0}
    \dataR{Stevens}{Ted}{R}{AK}{-6.307838}{-5.264698}{0}
    \dataRy{Lugar}{Richard}{R}{IN}{-5.943015}{-3.919419}{0}
    \dataRx{McCain}{John}{R}{AZ}{-5.888698}{2.418029}{0}
    \dataRx{Coleman}{Norm}{R}{MN}{-4.118805}{-5.477121}{0}
    \dataRy{Voinovich}{George}{R}{OH}{-3.935533}{-0.529552}{0}
    \dataRx{Smith}{Gordon}{R}{OR}{-3.444280}{-2.964483}{0}
    \dataRx{Specter}{Arlen}{R}{PA}{-2.375146}{-4.450506}{0}
    \dataRy{Collins}{Susan}{R}{ME}{-1.718529}{-4.243409}{0}
    \dataDx{Johnson}{Tim}{D}{SD}{-1.400105}{3.328882}{0}
    \dataRx{Snowe}{Olympia}{R}{ME}{0.030079}{-4.052412}{0}
    \dataDx{Nelson}{E.}{D}{NE}{3.212060}{-2.989657}{0}
    \dataDx{Pryor}{Mark}{D}{AR}{6.049862}{-2.370210}{0}
    \dataD{Landrieu}{Mary}{D}{LA}{6.120904}{0.319874}{0}
    \dataD{Bayh}{Evan}{D}{IN}{6.212828}{0.590900}{0}
    \dataDy{McCaskill}{Claire}{D}{MO}{6.236225}{2.501971}{0}
    \dataDy{Baucus}{Max}{D}{MT}{6.707413}{-1.252353}{0}
    \dataD{Tester}{Jon}{D}{MT}{6.755870}{-0.131501}{0}
    \dataD{Byrd}{Robert}{D}{WV}{6.844900}{-0.777788}{0}
    \dataDx{Lieberman}{Joe}{ID}{CT}{6.941566}{-0.879456}{0}
    \dataD{Lincoln}{Blanche}{D}{AR}{7.114470}{-2.537187}{0}
    \dataD{Dodd}{Christopher}{D}{CT}{7.150155}{0.426655}{0}
    \dataDy{Biden}{Joseph}{D}{DE}{7.191390}{1.560474}{0}
    \dataD{Dorgan}{Byron}{D}{ND}{7.226223}{-0.145256}{0}
    \dataDy{Conrad}{Kent}{D}{ND}{7.509217}{-0.834957}{0}
    \dataDx{Salazar}{Ken}{D}{CO}{7.510558}{-2.642954}{0}
    \dataD{Carper}{Thomas}{D}{DE}{7.533684}{-1.016364}{0}
    \dataD{Rockefeller}{Jay}{D}{WV}{7.697069}{0.348496}{0}
    \dataDy{Webb}{James}{D}{VA}{7.909628}{1.263548}{0}
    \dataDx{Nelson}{Bill}{D}{FL}{8.073955}{0.560473}{0}
    \dataD{Kerry}{John}{D}{MA}{8.229978}{0.783072}{0}
    \dataD{Inouye}{Daniel}{D}{HI}{8.254431}{0.128503}{0}
    \dataDy{Kennedy}{Edward}{D}{MA}{8.345720}{-0.125997}{0}
    \dataD{Cantwell}{Maria}{D}{WA}{8.453248}{0.176907}{0}
    \dataD{Feingold}{Russ}{D}{WI}{8.479149}{3.536569}{-0.15}
    \dataDx{Obama}{Barack}{D}{IL}{8.521068}{2.312086}{0}
    \dataDy{Reid}{Harry}{D}{NV}{8.689703}{0.665197}{0.1}
    \dataD{Mikulski}{Barbara}{D}{MD}{8.776657}{0.129926}{0}
    \dataD{Wyden}{Ron}{D}{OR}{8.846398}{-0.272109}{0}
    \dataD{Leahy}{Patrick}{D}{VT}{8.853166}{-0.139810}{0}
    \dataD{Harkin}{Tom}{D}{IA}{8.859139}{0.517913}{0}
    \dataD{Klobuchar}{Amy}{D}{MN}{8.865049}{-0.814168}{0}
    \dataD{Murray}{Patty}{D}{WA}{8.923771}{0.434588}{0}
    \dataD{Akaka}{Daniel}{D}{HI}{8.955400}{-1.058981}{0}
    \dataD{Brown}{Sherrod}{D}{OH}{8.961322}{1.297369}{0}
    \dataDx{Feinstein}{Dianne}{D}{CA}{9.019263}{0.316944}{-0.1}
    \dataD{Bingaman}{Jeff}{D}{NM}{9.063834}{0.227690}{0}
    \dataDy{Schumer}{Charles}{D}{NY}{9.124474}{1.151808}{0.05}
    \dataD{Cardin}{Benjamin}{D}{MD}{9.154539}{0.261731}{0}
    \dataD{Kohl}{Herb}{D}{WI}{9.168217}{0.505190}{0}
    \dataD{Casey}{Bob}{D}{PA}{9.227674}{1.165228}{0}
    \dataD{Stabenow}{Debbie}{D}{MI}{9.259294}{0.701454}{0}
    \dataD{Reed}{John}{D}{RI}{9.328214}{-0.051071}{0}
    \dataDx{Clinton}{Hillary}{D}{NY}{9.345548}{1.498053}{0.1}
    \dataDy{Sanders}{Bernard}{I}{VT}{9.366115}{1.166931}{0.3}
    \dataD{Durbin}{Richard}{D}{IL}{9.450949}{1.549572}{0}
    \dataD{Lautenberg}{Frank}{D}{NJ}{9.458477}{0.934696}{0}
    \dataD{Levin}{Carl}{D}{MI}{9.464507}{0.830443}{0}
    \dataD{Menendez}{Robert}{D}{NJ}{9.493317}{0.599649}{0}
    \dataD{Boxer}{Barbara}{D}{CA}{9.537760}{1.490709}{0}
    \dataDx{Whitehouse}{Sheldon}{D}{RI}{9.675169}{0.398093}{0.3}
    \draw[->] (0,0) +(1,-2) node[right,yshift=-2] {centroid} -- +(0,0);
    \path (-1,4.5) node {2007 U.S. Senate voting data, projection to the first principal component:};
  \end{tikzpicture}
\end{equation*}
For convenience, Republican senators have been shown in red and
Democratic and independent senators in blue. Not all senators have
been named, because in some areas they are clustered very densely. An
interpretation of the principal component then immediately suggests
itself: it appears to be the ``conservative'' vs. ``liberal'' axis. We
can use this picture to assist in answering questions such as: {\em
  ``Which party votes more uniformly?''}, {\em ``Which state are the
  most liberal Republicans from?''}, {\em ``Which state are the most
  conservative Democrats from?''}, {\em ``Was Obama really a
  radical?''}, and {\em ``Was McCain really a maverick?''}.  If we
repeat the same calculation for the 2017 senate, we get the following
picture:
\begin{equation*}
  \def\dataD#1#2#3#4#5#6#7{\fill[dem](#5,0) circle (0.104);}
  \def\dataR#1#2#3#4#5#6#7{\fill[rep](#5,0) circle (0.104);}
  \def\dataDx#1#2#3#4#5#6#7{\fill[dem](#5,0) circle (0.104);
    \draw[dem,->,shorten >=0.15cm](#5,0) +(#7,1) node[above] {\rotatebox{90}{\footnotesize #1, #4}} -- +(0,0);}
  \def\dataRx#1#2#3#4#5#6#7{\fill[rep](#5,0) circle (0.104);
    \draw[rep,->,shorten >=0.15cm](#5,0) +(#7,1) node[above] {\rotatebox{90}{\footnotesize #1, #4}} -- +(0,0);}
  \def\dataDy#1#2#3#4#5#6#7{\fill[dem](#5,0) circle (0.104);
    \draw[dem,->,shorten >=0.15cm](#5,0) +(#7,-1) node[below] {\rotatebox{90}{\footnotesize #1, #4}} -- +(0,0);}
  \def\dataRy#1#2#3#4#5#6#7{\fill[rep](#5,0) circle (0.104);
    \draw[rep,->,shorten >=0.15cm](#5,0) +(#7,-1) node[below] {\rotatebox{90}{\footnotesize #1, #4}} -- +(0,0);}
  \begin{tikzpicture}[scale=0.651]
    \draw (-12,0) -- (14.5,0);
    \dataRx{Risch}{James}{R}{ID}{-11.005580}{-0.280661}{-0.1}
    \dataR{Daines}{Steve}{R}{MT}{-10.967825}{-0.207257}{0}
    \dataR{Lankford}{James}{R}{OK}{-10.942378}{-0.196446}{0}
    \dataR{Ernst}{Joni}{R}{IA}{-10.913457}{-0.326658}{0}
    \dataR{Fischer}{Deb}{R}{NE}{-10.910381}{-0.154224}{0}
    \dataRy{Inhofe}{Jim}{R}{OK}{-10.794950}{-0.297634}{0}
    \dataR{Scott}{Tim}{R}{SC}{-10.786803}{-0.087283}{0}
    \dataR{Enzi}{Mike}{R}{WY}{-10.766418}{-0.045151}{0}
    \dataR{Johnson}{Ron}{R}{WI}{-10.755255}{-0.049739}{0}
    \dataR{Toomey}{Patrick}{R}{PA}{-10.749891}{-0.569081}{0}
    \dataR{Crapo}{Mike}{R}{ID}{-10.748843}{0.003722}{0}
    \dataR{Gardner}{Cory}{R}{CO}{-10.719932}{-0.268951}{0}
    \dataR{Kennedy}{John}{R}{LA}{-10.692143}{-0.209493}{0}
    \dataR{Flake}{Jeff}{R}{AZ}{-10.615803}{-1.106815}{0}
    \dataR{Hoeven}{John}{R}{ND}{-10.600127}{-0.072623}{0}
    \dataR{Cassidy}{Bill}{R}{LA}{-10.598818}{-0.002070}{0}
    \dataR{Sullivan}{Dan}{R}{AK}{-10.596058}{-0.382321}{0}
    \dataRx{Rubio}{Marco}{R}{FL}{-10.588703}{-0.868620}{0}
    \dataR{Barrasso}{John}{R}{WY}{-10.584780}{-0.044905}{0}
    \dataR{Perdue}{David}{R}{GA}{-10.580328}{-0.070316}{0}
    \dataR{Cotton}{Tom}{R}{AR}{-10.536647}{0.114383}{0}
    \dataR{Thune}{John}{R}{SD}{-10.530769}{-0.008701}{0}
    \dataR{Wicker}{Roger}{R}{MS}{-10.524726}{0.028472}{0}
    \dataR{Cornyn}{John}{R}{TX}{-10.513419}{0.176439}{0}
    \dataR{Roberts}{Pat}{R}{KS}{-10.513419}{0.176439}{0}
    \dataR{Rounds}{Mike}{R}{SD}{-10.513419}{0.176439}{0}
    \dataR{Shelby}{Richard}{R}{AL}{-10.501896}{0.078644}{0}
    \dataR{Grassley}{Charles}{R}{IA}{-10.458537}{-0.074556}{0}
    \dataR{Blunt}{Roy}{R}{MO}{-10.456091}{0.188385}{0}
    \dataR{Hatch}{Orrin}{R}{UT}{-10.427457}{0.182143}{0}
    \dataR{Cochran}{Thad}{R}{MS}{-10.419903}{0.171741}{0}
    \dataR{Boozman}{John}{R}{AR}{-10.416449}{0.049056}{0}
    \dataR{Sasse}{Ben}{R}{NE}{-10.384081}{-0.769970}{0}
    \dataR{Moran}{Jerry}{R}{KS}{-10.354821}{-0.478472}{0}
    \dataRy{Cruz}{Ted}{R}{TX}{-10.337513}{-0.118032}{0}
    \dataRx{McConnell}{Mitch}{R}{KY}{-10.328164}{0.019018}{0.2}
    \dataR{Burr}{Richard}{R}{NC}{-10.281722}{-0.434053}{0}
    \dataR{Lee}{Mike}{R}{UT}{-10.250033}{-1.332033}{0}
    \dataR{Corker}{Bob}{R}{TN}{-10.181143}{-0.353619}{0}
    \dataR{Tillis}{Thomas}{R}{NC}{-10.164341}{0.190998}{0}
    \dataR{Alexander}{Lamar}{R}{TN}{-10.140983}{-0.605518}{0}
    \dataR{Graham}{Lindsey}{R}{SC}{-10.118998}{-0.308972}{0}
    \dataR{Young}{Todd}{R}{IN}{-10.114119}{0.195154}{0}
    \dataR{Capito}{Shelley}{R}{WV}{-10.092079}{-0.013747}{0.2}
    \dataRy{Portman}{Rob}{R}{OH}{-9.980981}{0.307914}{0.1}
    \dataRx{Paul}{Rand}{R}{KY}{-9.083156}{-2.311221}{0}
    \dataRy{McCain}{John}{R}{AZ}{-8.941401}{-1.537122}{0}
    \dataR{Murkowski}{Lisa}{R}{AK}{-8.929094}{0.087773}{0}
    \dataRx{Heller}{Dean}{R}{NV}{-8.607091}{-1.195798}{0}
    \dataRy{Isakson}{John}{R}{GA}{-7.825936}{-2.375299}{0}
    \dataRx{Collins}{Susan}{R}{ME}{-7.218464}{1.349259}{0}
    \dataDx{Manchin}{Joe}{D}{WV}{4.244339}{4.985463}{0}
    \dataDy{Heitkamp}{Heidi}{D}{ND}{4.699457}{6.230765}{0}
    \dataDx{Donnelly}{Joe}{D}{IN}{6.294362}{5.372277}{0}
    \dataDy{King}{Angus}{I}{ME}{6.567064}{5.137923}{0}
    \dataDx{Warner}{Mark}{D}{VA}{7.805257}{5.268461}{0}
    \dataDy{McCaskill}{Claire}{D}{MO}{8.515402}{4.257321}{0}
    \dataDx{Feinstein}{Dianne}{D}{CA}{9.195030}{1.541150}{0}
    \dataDy{Tester}{Jon}{D}{MT}{9.298819}{4.338771}{0}
    \dataD{Nelson}{Bill}{D}{FL}{9.415968}{3.904212}{0}
    \dataDx{Bennet}{Michael}{D}{CO}{9.648520}{3.741099}{0}
    \dataDy{Carper}{Thomas}{D}{DE}{9.787069}{3.051711}{0}
    \dataD{Coons}{Christopher}{D}{DE}{9.967535}{2.835454}{0}
    \dataDx{Kaine}{Timothy}{D}{VA}{10.380048}{2.820041}{0}
    \dataDy{Cortez Masto}{Catherine}{D}{NV}{10.700058}{0.650442}{0}
    \dataD{Menendez}{Robert}{D}{NJ}{10.833270}{-0.138012}{0}
    \dataD{Shaheen}{Jeanne}{D}{NH}{10.932457}{2.780893}{0}
    \dataDx{Durbin}{Richard}{D}{IL}{11.134223}{0.337495}{0}
    \dataD{Hassan}{Maggie}{D}{NH}{11.175503}{2.348138}{0}
    \dataDy{Klobuchar}{Amy}{D}{MN}{11.227181}{2.483025}{0}
    \dataD{Casey}{Bob}{D}{PA}{11.259986}{2.362439}{0}
    \dataD{Murphy}{Christopher}{D}{CT}{11.262853}{1.643260}{0}
    \dataD{Stabenow}{Debbie}{D}{MI}{11.331051}{0.981505}{0}
    \dataD{Peters}{Gary}{D}{MI}{11.377813}{1.008805}{0}
    \dataD{Schatz}{Brian}{D}{HI}{11.523780}{-0.180224}{0}
    \dataDx{Cardin}{Ben}{D}{MD}{11.530545}{1.600627}{0}
    \dataD{Leahy}{Patrick}{D}{VT}{11.666722}{0.850718}{0}
    \dataD{Heinrich}{Martin}{D}{NM}{11.677727}{-0.658323}{0}
    \dataD{Brown}{Sherrod}{D}{OH}{11.710129}{0.528127}{0}
    \dataD{Udall}{Tom}{D}{NM}{11.799501}{-1.446553}{0}
    \dataD{Cantwell}{Maria}{D}{WA}{11.871698}{0.994751}{0}
    \dataD{Reed}{John}{D}{RI}{11.903017}{-0.135526}{0}
    \dataDy{Duckworth}{Tammy}{D}{IL}{11.912992}{-1.455125}{0}
    \dataD{Baldwin}{Tammy}{D}{WI}{12.006094}{-0.147581}{0}
    \dataD{Hirono}{Mazie}{D}{HI}{12.012566}{-1.222498}{0}
    \dataDx{Franken}{Al}{D}{MN}{12.048895}{0.607727}{0}
    \dataD{Murray}{Patty}{D}{WA}{12.128326}{0.434861}{0}
    \dataD{Whitehouse}{Sheldon}{D}{RI}{12.129382}{-0.638112}{0}
    \dataD{Van Hollen}{Chris}{D}{MD}{12.330587}{-0.284590}{0}
    \dataDy{Schumer}{Charles}{D}{NY}{12.392888}{-1.562009}{0}
    \dataD{Blumenthal}{Richard}{D}{CT}{12.409939}{-2.289658}{0}
    \dataD{Wyden}{Ron}{D}{OR}{12.495539}{-2.739850}{0}
    \dataD{Markey}{Edward}{D}{MA}{12.823912}{-4.803898}{0}
    \dataDx{Sanders}{Bernie}{I}{VT}{12.866844}{-7.239669}{0}
    \dataD{Merkley}{Jeff}{D}{OR}{12.972657}{-5.312050}{0}
    \dataD{Harris}{Kamala}{D}{CA}{13.017672}{-6.143469}{0}
    \dataD{Booker}{Cory}{D}{NJ}{13.163851}{-6.942933}{0}
    \dataDy{Gillibrand}{Kirsten}{D}{NY}{13.288360}{-8.552288}{0}
    \dataDx{Warren}{Elizabeth}{D}{MA}{13.328440}{-7.543715}{0}
    \draw[->] (0,0) +(1,-2) node[right,yshift=-2] {centroid} -- +(0,0);
    \path (1.25,5) node {2017 U.S. Senate voting data, projection to the first principal component:};
  \end{tikzpicture}
\end{equation*}
We can use this to help answer questions such as {\em ``Has the senate
  become more partisan between 2007 and 2017?''}.

If we instead project the data onto the first two principal
components, we get the following picture for the 2007 data:
\begin{equation*}
  \def\dataD#1#2#3#4#5#6#7{\fill[dem](#5,#6) circle (0.09);}
  \def\dataR#1#2#3#4#5#6#7{\fill[rep](#5,#6) circle (0.09);}
  \def\dataDx#1#2#3#4#5#6#7{\fill[dem](#5,#6) circle (0.09);
    \draw[dem,->,shorten >=0.15cm](#5,#6) +(#7,1) node[above] {\rotatebox{90}{\footnotesize #1, #4}} -- +(0,0);}
  \def\dataRx#1#2#3#4#5#6#7{\fill[rep](#5,#6) circle (0.09);
    \draw[rep,->,shorten >=0.15cm](#5,#6) +(#7,1) node[above] {\rotatebox{90}{\footnotesize #1, #4}} -- +(0,0);}
  \def\dataDy#1#2#3#4#5#6#7{\fill[dem](#5,#6) circle (0.09);
    \draw[dem,->,shorten >=0.15cm](#5,#6) +(#7,-1) node[below] {\rotatebox{90}{\footnotesize #1, #4}} -- +(0,0);}
  \def\dataRy#1#2#3#4#5#6#7{\fill[rep](#5,#6) circle (0.09);
    \draw[rep,->,shorten >=0.15cm](#5,#6) +(#7,-1) node[below] {\rotatebox{90}{\footnotesize #1, #4}} -- +(0,0);}
  \begin{tikzpicture}[scale=0.71]
    \draw (-12.5,0) -- (10.5,0);
    \draw (0,-6) -- (0,8);
    \dataRx{DeMint}{James}{R}{SC}{-11.856041}{4.315015}{0}
    \dataRx{Cornyn}{John}{R}{TX}{-11.446925}{1.720794}{-1.5}
    \dataRx{Allard}{A.}{R}{CO}{-11.414729}{2.353678}{-1}
    \dataR{Ensign}{John}{R}{NV}{-11.329050}{2.598885}{0}
    \dataR{Inhofe}{Jim}{R}{OK}{-11.161216}{4.313828}{0}
    \dataRx{Coburn}{Tom}{R}{OK}{-11.095925}{6.451745}{0}
    \dataR{Enzi}{Michael}{R}{WY}{-10.786882}{0.794427}{0}
    \dataR{Burr}{Richard}{R}{NC}{-10.716488}{2.508468}{0}
    \dataR{Bunning}{Jim}{R}{KY}{-10.674428}{1.409311}{0}
    \dataR{Chambliss}{Saxby}{R}{GA}{-10.639465}{1.781387}{0}
    \dataR{Isakson}{Johnny}{R}{GA}{-10.548229}{0.724451}{0}
    \dataRx{Sessions}{Jeff}{R}{AL}{-10.510782}{1.437248}{0}
    \dataR{Gregg}{Judd}{R}{NH}{-10.399051}{1.990574}{0}
    \dataRx{McConnell}{Mitch}{R}{KY}{-10.375319}{-1.293286}{-3}
    \dataR{Dole}{Elizabeth}{R}{NC}{-10.258572}{1.732827}{0}
    \dataR{Kyl}{Jon}{R}{AZ}{-10.042393}{2.633156}{0}
    \dataR{Craig}{Larry}{R}{ID}{-9.996687}{-0.803843}{0}
    \dataR{Crapo}{Mike}{R}{ID}{-9.987981}{-0.982945}{0}
    \dataRx{Graham}{Lindsey}{R}{SC}{-9.942743}{2.090629}{0}
    \dataR{Sununu}{John}{R}{NH}{-9.713036}{1.627775}{0}
    \dataR{Roberts}{Pat}{R}{KS}{-9.682832}{-1.781188}{0}
    \dataR{Martinez}{Mel}{R}{FL}{-9.602384}{1.021259}{0}
    \dataR{Thune}{John}{R}{SD}{-9.412180}{0.719407}{0}
    \dataR{Hutchison}{Kay}{R}{TX}{-9.362272}{-0.132493}{0}
    \dataR{Lott}{Trent}{R}{MS}{-9.333867}{-1.703358}{0}
    \dataR{Hatch}{Orrin}{R}{UT}{-9.325404}{-1.647643}{0}
    \dataR{Corker}{Bob}{R}{TN}{-9.321180}{1.242018}{0}
    \dataR{Grassley}{Charles}{R}{IA}{-9.274839}{1.535256}{0}
    \dataRx{Vitter}{David}{R}{LA}{-9.268543}{3.508303}{0}
    \dataR{Alexander}{Lamar}{R}{TN}{-8.869856}{-2.298938}{0}
    \dataRx{Shelby}{Richard}{R}{AL}{-8.861314}{0.147906}{0}
    \dataRx{Bennett}{Robert}{R}{UT}{-8.230363}{-5.265448}{-0.3}
    \dataRx{Cochran}{Thad}{R}{MS}{-8.204309}{-3.833549}{0}
    \dataRx{Bond}{Christopher}{R}{MO}{-7.907050}{-3.678825}{0}
    \dataRx{Brownback}{Samuel}{R}{KS}{-7.841176}{0.525267}{0}
    \dataRx{Warner}{John}{R}{VA}{-7.511971}{-3.669235}{0}
    \dataRx{Murkowski}{Lisa}{R}{AK}{-6.411501}{-5.705682}{-0.3}
    \dataR{Domenici}{Pete}{R}{NM}{-6.367823}{-4.162556}{0}
    \dataRx{Hagel}{Chuck}{R}{NE}{-6.362849}{0.617170}{0}
    \dataR{Stevens}{Ted}{R}{AK}{-6.307838}{-5.264698}{0}
    \dataRx{Lugar}{Richard}{R}{IN}{-5.943015}{-3.919419}{0}
    \dataRx{McCain}{John}{R}{AZ}{-5.888698}{2.418029}{0}
    \dataRx{Coleman}{Norm}{R}{MN}{-4.118805}{-5.477121}{0}
    \dataRx{Voinovich}{George}{R}{OH}{-3.935533}{-0.529552}{0}
    \dataRx{Smith}{Gordon}{R}{OR}{-3.444280}{-2.964483}{0}
    \dataRx{Specter}{Arlen}{R}{PA}{-2.375146}{-4.450506}{0}
    \dataRx{Collins}{Susan}{R}{ME}{-1.718529}{-4.243409}{0}
    \dataDx{Johnson}{Tim}{D}{SD}{-1.400105}{3.328882}{0}
    \dataRx{Snowe}{Olympia}{R}{ME}{0.030079}{-4.052412}{-0.5}
    \dataDx{Nelson}{E.}{D}{NE}{3.212060}{-2.989657}{0}
    \dataDx{Pryor}{Mark}{D}{AR}{6.049862}{-2.370210}{-0.3}
    \dataD{Landrieu}{Mary}{D}{LA}{6.120904}{0.319874}{0}
    \dataDx{Bayh}{Evan}{D}{IN}{6.212828}{0.590900}{-0.5}
    \dataDx{McCaskill}{Claire}{D}{MO}{6.236225}{2.501971}{0}
    \dataDx{Baucus}{Max}{D}{MT}{6.707413}{-1.252353}{-0.2}
    \dataD{Tester}{Jon}{D}{MT}{6.755870}{-0.131501}{0}
    \dataD{Byrd}{Robert}{D}{WV}{6.844900}{-0.777788}{0}
    \dataDx{Lieberman}{Joe}{ID}{CT}{6.941566}{-0.879456}{-0.05}
    \dataD{Lincoln}{Blanche}{D}{AR}{7.114470}{-2.537187}{0}
    \dataD{Dodd}{Christopher}{D}{CT}{7.150155}{0.426655}{0}
    \dataDx{Biden}{Joseph}{D}{DE}{7.191390}{1.560474}{0.2}
    \dataD{Dorgan}{Byron}{D}{ND}{7.226223}{-0.145256}{0}
    \dataD{Conrad}{Kent}{D}{ND}{7.509217}{-0.834957}{0}
    \dataDx{Salazar}{Ken}{D}{CO}{7.510558}{-2.642954}{3.3}
    \dataD{Carper}{Thomas}{D}{DE}{7.533684}{-1.016364}{0}
    \dataD{Rockefeller}{Jay}{D}{WV}{7.697069}{0.348496}{0}
    \dataD{Webb}{James}{D}{VA}{7.909628}{1.263548}{0}
    \dataD{Nelson}{Bill}{D}{FL}{8.073955}{0.560473}{0}
    \dataD{Kerry}{John}{D}{MA}{8.229978}{0.783072}{0}
    \dataD{Inouye}{Daniel}{D}{HI}{8.254431}{0.128503}{0}
    \dataD{Kennedy}{Edward}{D}{MA}{8.345720}{-0.125997}{0}
    \dataD{Cantwell}{Maria}{D}{WA}{8.453248}{0.176907}{0}
    \dataD{Feingold}{Russ}{D}{WI}{8.479149}{3.536569}{-0.15}
    \dataDx{Obama}{Barack}{D}{IL}{8.521068}{2.312086}{-0.3}
    \dataDx{Reid}{Harry}{D}{NV}{8.689703}{0.665197}{-0.8}
    \dataD{Mikulski}{Barbara}{D}{MD}{8.776657}{0.129926}{0}
    \dataD{Wyden}{Ron}{D}{OR}{8.846398}{-0.272109}{0}
    \dataD{Leahy}{Patrick}{D}{VT}{8.853166}{-0.139810}{0}
    \dataD{Harkin}{Tom}{D}{IA}{8.859139}{0.517913}{0}
    \dataD{Klobuchar}{Amy}{D}{MN}{8.865049}{-0.814168}{0}
    \dataD{Murray}{Patty}{D}{WA}{8.923771}{0.434588}{0}
    \dataD{Akaka}{Daniel}{D}{HI}{8.955400}{-1.058981}{0}
    \dataD{Brown}{Sherrod}{D}{OH}{8.961322}{1.297369}{0}
    \dataDx{Feinstein}{Dianne}{D}{CA}{9.019263}{0.316944}{-0.1}
    \dataD{Bingaman}{Jeff}{D}{NM}{9.063834}{0.227690}{0}
    \dataDx{Schumer}{Charles}{D}{NY}{9.124474}{1.151808}{0.2}
    \dataD{Cardin}{Benjamin}{D}{MD}{9.154539}{0.261731}{0}
    \dataD{Kohl}{Herb}{D}{WI}{9.168217}{0.505190}{0}
    \dataD{Casey}{Bob}{D}{PA}{9.227674}{1.165228}{0}
    \dataD{Stabenow}{Debbie}{D}{MI}{9.259294}{0.701454}{0}
    \dataD{Reed}{John}{D}{RI}{9.328214}{-0.051071}{0}
    \dataDx{Clinton}{Hillary}{D}{NY}{9.345548}{1.498053}{0.8}
    \dataDx{Sanders}{Bernard}{I}{VT}{9.366115}{1.166931}{1.2}
    \dataD{Durbin}{Richard}{D}{IL}{9.450949}{1.549572}{0}
    \dataD{Lautenberg}{Frank}{D}{NJ}{9.458477}{0.934696}{0}
    \dataD{Levin}{Carl}{D}{MI}{9.464507}{0.830443}{0}
    \dataD{Menendez}{Robert}{D}{NJ}{9.493317}{0.599649}{0}
    \dataD{Boxer}{Barbara}{D}{CA}{9.537760}{1.490709}{0}
    \dataDx{Whitehouse}{Sheldon}{D}{RI}{9.675169}{0.398093}{1.3}
    \draw[->] (0,0) +(1,2) node[right,yshift=2] {centroid} -- +(0,0);
    \path (-1,10) node {2007 U.S. Senate voting data, projection to the first two principal components:};
  \end{tikzpicture}
\end{equation*}
The picture clearly shows senators clustering in certain areas. We can
use this to help answer certain questions, for example, {\em ``How
  different was Sanders's voting record from Clinton's?''}. However,
although the 2-dimensional picture seems to reveal more detail, its
interpretation is less clear. While it seems obvious that the
horizontal axis corresponds to a conservative vs.\ liberal world view,
it is much less obvious what the political meaning of the vertical
axis is. Maybe it is related to some issue that does not typically
follow party lines, such as North vs. South, rich states vs. poor
states, pro-immigration vs. anti-immigration, and so on.  To find a
convincing interpretation of the vertical axis, further investigation
of the data would be required (such as, looking at the actual content
of the votes in question).

Finally, a word of caution. Whenever we use mathematics to try to draw
real-world conclusions from data, these conclusions should be taken
with an extra-large grain of salt. People have an outsized tendency to
trust mathematics and to take its results as infallible. We therefore
have a special responsibility not to overstate any conclusions, and to
point out potential pitfalls with the analysis. No matter how
wonderful principal complement analysis is, we must keep in mind that
what we are still only looking at a 2-dimensional projection of a
200-dimensional space. Therefore it is inevitable that lots of details
and nuances are lost. We could get a completely different picture by
looking at a different 2-dimensional projection.

To see how the data can sometimes be misleading, consider the question
{\em ``How similar is Senator Tim Johnson, Democrat of South Dakota,
  to Senators Olympia Snowe and Susan Collins of Maine?''}. In the
1-dimensional picture, it looked as if they were very similar. We
could easily rationalize this by pointing out that Johnson is the most
conservative Democrat, and Snowe and Collins are the most liberal
Republicans. However, the 2-dimensional picture reveals an interesting
nuance, which is that the voting records of Johnson is not all that
similar to that of Snowe and Collins. It is entirely possible that if
we add a third or fourth dimension to the picture, many more
additional such details will emerge. In summary, while principal
component analysis is a useful tool, it is just one tool among many,
and we always need to exercise our best judgement in drawing
conclusions from data.


\subsection{ 5.1. A More General Stokes's Theorem}

Let $V$ compact oriented submanifold of $M^n$ \\
smooth $F: M^n \to W^m$

$F(V) \subset W$ need not be a submanifold, might have self-interactions, pathologies.

\begin{equation}
        \int_{F(V)} \beta^p = \int_V F^* \beta^p  \quad \quad \quad (5.1)
\end{equation}
generalizes (3.17), def.

\[
\int_{F(V)} d\beta^{p-1} = \int_V F^* d\beta^{p-1} = \int_V dF^* \beta^{p-1} = \int_{ \partial V} F^* \beta^{p-1} = \int_{ F(\partial V)} \beta^{p-1}
\]
(\emph{question}, 2nd., 3rd. equality) \\

%\begin{proof}
Answer: recall Naturality (cf. wikipedia exterior derivative)

$\Omega^k$ contravariant smooth functor $M \mapsto^{ \Omega^k } \Lambda^k(M)$

\begin{tikzpicture}
\matrix(m)[matrix of math nodes, row sep=3em, column sep=3em, text height=1.5ex, text depth=0.25ex]
{ \Omega^k(N)    &  \Omega^k(M)  \\
\Omega^{k+1}(N) &  \Omega^{k+1}(M)  \\};
\path[->,font=\scriptsize]
(m-1-1) edge node[auto]{$F^*$} (m-1-2)
edge node[auto]{$d$} (m-2-1)
(m-1-2) edge node[auto]{$d$} (m-2-2)
(m-2-1) edge node[auto]{$F^*$} (m-2-2);
%\path[right hook->](m-1-1) edge  node[auto]{$\,$} (m-2-1);
%\path[->,font=\scriptsize](m-1-2) edge  node[auto]{$p$} (m-2-2);
%edge node[auto]{$\gamma$} (m-2-2);
%\path[->,font=\scriptsize](m-1-2) edge  node[auto]{$p$} (m-2-2);
%\path[dashed,->,font=\scriptsize]
%(m-2-1) edge node[auto]{$G$} (m-1-2);
\end{tikzpicture}

So that
\[
dF^* = F^* d
\]
%\end{proof}

 \hrulefill

define $\partial F(V) = F( \partial V)$ \\

generalized Stoke's thm.
\begin{equation}
        \int_{F(V)} d\beta^{p-1} = \int_{ \partial F(V)} \beta^{p-1} \quad \quad \quad (5.2)
\end{equation}

manifold needs only ``piecewise smooth'' boundaries.

\subsection{ 5.2. Closed Forms and Exact Forms }

$\beta^p$ closed if $d\beta = 0$ \\
 $ \beta^p$ exact if $\beta^p = d\alpha^{p-1}$, some $\alpha^{p-1}$

\begin{theorem}[5.3] Let $M^n$ with 1st Betti number 0, $b_1 = 0$, i.e. $\forall \, $ closed oriented piecewise smooth curve $C$ is the boundary of some compact oriented ``surface''.  Then $\forall \, $ closed 1-form $\beta^1$ on $M^n$ is exact.
\end{theorem}

%\begin{proof}

\hrulefill \\

        Let $x,y \in M$, $y$ fixed.  \\
        oriented $C(y,x)$ starts at $y$, ends at $x$

define
\[
f(x) \equiv \int_{C(y,x)} \beta^1
\]

If $\exists \, $ another $C^1(y,x)$, then $C- C'$ closed oriented curve.

By given, $\exists \, $ oriented compact surface $F(V)$ s.t. $\partial F(V) = C$.

\[
\int_C \beta - \int_{C'} \beta = \int_{C- C'} \beta = \oint_{ \partial F(V)} \beta = \int_{F(V)} d\beta = 0
\]
$\int_C \beta = \int_{C'} \beta$.  $f$ independent of curve.

Let $\mathbf{v}_x$ vector at $x$.

Let vector field $\mathbf{v}$ coincide with $\mathbf{v}_x$ at $x$, defined in neighborhood of curve $C(y,x)$, $v=0$ at $y$.

$\phi_t$ flow generated by $v$, $\phi_t C(y,x)$ curve joining $y$ to $\phi_i x$
\[
\begin{gathered}
        \left[ \frac{ d\phi_i x}{ dt} \right]_{t=0} = v_f \\
        df(v) = \frac{d}{dt} f\lbrace \phi_t x \rbrace_{t=0} = \left[ \frac{d}{dt} \int_{\phi_tC(y,x)} \beta \right]_{t=0} = \int_{ C(y,x) } \mathcal{L}_v \beta  =
\end{gathered}
\]

\hrulefill

%\end{proof}

\subsection{ 5.3. Complex Analysis }

\subsubsection{5.5 Finding potentials}

\paragraph{5.5(1) Product of a closed and an exact form}\ \\
Let \ieq{\kappa} be a closed k-form and \ieq{\varepsilon} be an exact form with \ieq{\d\tilde\varepsilon=\varepsilon}. Then
\beq{
	\kappa\wedge\varepsilon
	&= \kappa\wedge\d\tilde\varepsilon
	 = (-1)^k(\d(\kappa\wedge\tilde\varepsilon)-\underbrace{\d\kappa}_{=0}\wedge\,\tilde\varepsilon)
	 = \d\left((-1)^k\kappa\wedge\tilde\varepsilon\right)
}

LaTeX_and_pdfs/the geometry of physics problems/05potentials.tex


\section{An axiomatic system for FOS5}

\qquad To prove the results of Chapters 2 and 3 it was convenient to state things in terms of $\nao, \ou, \ex, \Diamond$ and $=$. But to stay connected with the formulations of the last chapter consider now the version of first-order modal logic defined using $\bot$, $\impli$, $\todo$ and $\Box$ (without equality). Let $\Li$ be the same language fixed in Chapter 5. To make things simple, we write $Fml$ instead of $\FLi$.

\qquad Since we have started working only with semantical notions, we have defined the logic FOS5 as the set of all valid sentences relative to the class of all FOS5-models. Alternatively we can study the logic FOS5 using a simple and elegant axiomatic system composed of the following axiom schemes and inference rules:\\

\textbf{A$\p$1} classical axioms of first-order logic\\

\textbf{A$\p$2} $\Box \varphi \impli \varphi$\\

\textbf{A$\p$3} $\Box \varphi \impli \Box \Box \varphi$\\

\textbf{A$\p$4} $\nao \Box \varphi \impli \Box \nao \Box \varphi$\\

\textbf{A$\p$5} $\Box (\varphi \impli \psi) \impli  (\Box \varphi \impli \Box \psi)$\\

\textbf{R$\p$1} (\textit{Modus Ponens}) $\teo \varphi$, $\teo \varphi\impli\psi$ $\Rightarrow$ $\teo \psi$ \\

\textbf{R$\p$2} (\textit{generalization})  $\teo \varphi$ $\Rightarrow$ $\teo \todo x \varphi$ \\

\textbf{R$\p$3} (\textit{necessitation})  $\teo \varphi$ $\Rightarrow$  $\teo \Box \varphi$.\\

\qquad As in the case for FOJT45 we make use of the standard notion of $\Gamma \teo \varphi$. Here the restriction on the generalization rule is the same as stated for FOJT45, and the necessitation rule is allowed only when $\Gamma = \vazio$. We write $FOS5\teo \varphi$ to denote that in this axiomatic system $\vazio\teo \varphi$.

\qquad
In the seminal paper by Kripke \cite{Kripke59} the Completeness Theorem for this logic was shown, and so the semantical and the syntactical characterization of FOS5 are equivalent. To be more precise, for every sentence $\varphi \in Fml$,

\begin{center}
$FOS5\teo \varphi$ iff  $\vSB \varphi$.
\end{center}

\section{Realization}

\begin{defn}
Let $\varphi$ be a formula of FOS5. We define the \textit{realization} of $\varphi$ in the language of FOJT45, $\varphi^{r}$, as follows:

\begin{itemize}
	\item If $\varphi$ is atomic, then $\varphi^{r}$ is $\varphi$.
	\item If $\varphi$ is $\psi\impli\theta$, then $\varphi^{r}$ is $\psi^{r}\impli\theta^{r}$
	\item If $\varphi$ is $\todo x \psi$, then $\varphi^{r}$ is $\todo x \psi^{r}$
	\item If $\varphi$ is $\Box \psi$ and $fv(\varphi) = \{x_{1}, \dots, x_{n}\}$, then $\varphi^{r}$ is $t$$:_{\{x_{1}, \dots, x_{n}\}}$$ \psi^{r}$
\end{itemize}
\end{defn}

\qquad A realization is normal if all negative occurrences of $\Box$ are assigned justification variables. It can easily be checked that for every $\varphi \in Fml$, $fv(\varphi) = fv(\varphi^{r})$.

\begin{defn}
Let $\varphi$ be a formula of FOJT45. \textit{The forgetful projection} of $\varphi$, $\varphi^{\circ}$, is defined as follows:

\begin{itemize}
	\item If $\varphi$ is atomic, then $\varphi^{\circ}$ is $\varphi$.
	\item If $\varphi$ is $\psi\impli\theta$, then $\varphi^{\circ}$ is $\psi^{\circ}\impli\theta^{\circ}$
	\item If $\varphi$ is $\todo x \psi$, then $\varphi^{\circ}$ is $\todo x \psi^{\circ}$
	\item If $\varphi$ is $t$$:_{X}$$ \psi$, then $\varphi^{\circ}$ is $\Box \todo \vec{y}\psi^{\circ}$\\ where $\vec{y} \in fv(\psi)\backslash X$.
\end{itemize}
\end{defn}

\qquad As before, it can easily be checked that for every $\varphi \in \Fj$,  $fv(\varphi) = fv(\varphi^{\circ})$.

\begin{pro}
For every constant specification $\C$ and for every $\varphi \in \Fj$,
\begin{center}
If FOJT45 $\teo_{\C} \varphi$, then FOS5 $\teo \varphi^{\circ}$.
\end{center}
\end{pro}

\begin{proof}
Induction on the theorems of FOJT45 with $\C$. In this proof only we shall use $\teo$ to denote $FOS5 \teo$. And for simplicity we are going to deal only with a representative special case of each axiom. These special cases are simpler versions of each axiom; the argument can be easily generalized.\\

($\varphi$ is an instance of \textbf{A2})\\

\qquad Suppose $\varphi$ is

\begin{center}
$t$$:_{\{x,y\}}$$\psi(x,z) \impli t$$:_{\{x\}}$$\psi(x,z)$
\end{center}
Since $y \notin fv(\psi(x,z))$,

\begin{center}
$\{z\} = fv(\psi(x,z)) \backslash \{x,y\} = fv(\psi(x,z)) \backslash \{x\}$
\end{center}
Thus, $\varphi^{\circ}$ is

\begin{center}
$\Box  \todo z \psi^{\circ}(x,z) \impli \Box  \todo z \psi^{\circ}(x,z)$.
\end{center}

Clearly, $\teo \varphi^{\circ}$.\\
\vspace{5mm}

($\varphi$ is an instance of \textbf{A3})\\

\qquad Suppose $\varphi$ is

\begin{center}
$t$$:_{\{x\}}$$\psi(x,y,z) \impli t$$:_{\{x,y\}}$$\psi(x,y,z)$
\end{center}
Then, $\varphi^{\circ}$ is

\begin{center}
$\Box  \todo y\todo z \psi^{\circ}(x,y,z) \impli \Box  \todo z \psi^{\circ}(x,y,z)$
\end{center}
By classical axioms,

\begin{center}
$\teo \todo y\todo z \psi^{\circ}(x,y,z) \impli \todo z \psi^{\circ}(x,y,z)$
\end{center}
By necessitation and the distributivity of $\Box$ over $\impli$,

\begin{center}
$\teo \Box  \todo y\todo z \psi^{\circ}(x,y,z) \impli \Box  \todo z \psi^{\circ}(x,y,z)$.
\end{center}

\vspace{5mm}

($\varphi$ is an instance of \textbf{B1})\\

\qquad Suppose $\varphi$ is

\begin{center}
$t$$:_{\{x\}}$$\psi(x,y) \impli \psi(x,y)$
\end{center}
Then, $\varphi^{\circ}$ is

\begin{center}
$\Box  \todo y\psi^{\circ}(x,y) \impli \psi^{\circ}(x,y)$
\end{center}
By \textbf{A$\p$2},

\begin{center}
$\teo \Box \todo y\psi^{\circ}(x,y) \impli \todo y\psi^{\circ}(x,y)$
\end{center}
And by classical axioms,

\begin{center}
$\teo \todo y\psi^{\circ}(x,y) \impli \psi^{\circ}(x,y)$
\end{center}
So,

\begin{center}
$\teo \Box  \todo y\psi^{\circ}(x,y) \impli \psi^{\circ}(x,y)$.
\end{center}
\vspace{5mm}

($\varphi$ is an instance of \textbf{B2})\\

Suppose $\varphi$ is

\begin{center}
$t$$:_{\{x,x\p\}}$$(\psi(x,y) \impli \theta(x\p,z)) \impli$ $(s$$:_{\{x,x\p\}}$$\psi(x,y) \impli$ $[t\cdot s]$$:_{\{x,x\p\}}$$\theta(x\p,z))$
\end{center}
Then, $\varphi^{\circ}$ is

\begin{center}
$\Box  \todo y\todo z (\psi^{\circ}(x,y) \impli \theta^{\circ}(x\p,z)) \impli (\Box \todo y \psi^{\circ}(x,y) \impli \Box \todo z \theta^{\circ}(x\p,z))$
\end{center}
By classical reasoning,

\begin{center}
$\teo \todo y\todo z (\psi^{\circ}(x,y) \impli \theta^{\circ}(x\p,z)) \impli (\todo y\todo z  \psi^{\circ}(x,y) \impli \todo y\todo z  \theta^{\circ}(x\p,z))$
\end{center}
Since $z \notin fv(\psi^{\circ}(x,y))$ and $y \notin fv(\theta^{\circ}(x\p,z))$, we have that

\begin{center}
$\teo \todo y\todo z  \psi^{\circ}(x,y) \see \todo y \psi^{\circ}(x,y)$\\
$\teo \todo y\todo z  \theta^{\circ}(x\p,z) \see \todo z \theta^{\circ}(x\p,z)$
\end{center}
Hence,

\begin{center}
$\teo \todo y\todo z (\psi^{\circ}(x,y) \impli \theta^{\circ}(x\p,z)) \impli (\todo y \psi^{\circ}(x,y) \impli \todo z \theta^{\circ}(x\p,z))$
\end{center}
By necessitation and the distributivity of $\Box$ over $\impli$,

\begin{center}
$\teo \Box  \todo y\todo z (\psi^{\circ}(x,y) \impli \theta^{\circ}(x\p,z)) \impli (\Box \todo y \psi^{\circ}(x,y) \impli \Box \todo z \theta^{\circ}(x\p,z))$.
\end{center}
\vspace{5mm}

($\varphi$ is an instance of \textbf{B3})\\

\qquad If $\varphi$ is $t$$:_{\{x\}}$$\psi(x,y) \impli$ $[t+s]$$:_{\{x\}}$$\psi(x,y)$, then $\varphi^{\circ}$ is $\Box  \todo y \psi^{\circ}(x,y) \impli \Box  \todo y \psi^{\circ}(x,y)$. Clearly, $\teo \varphi^{\circ}$. The same argument holds when $\varphi$ is $s$$:_{\{x\}}$$\psi(x,y) \impli$ $[t+s]$$:_{\{x\}}$$\psi(x,y)$.
\vspace{5mm}

($\varphi$ is an instance of \textbf{B4})\\

\qquad If $\varphi$ is $t$$:_{\{x\}}$$\psi(x,y) \impli$ $!t$$:_{\{x\}}$$t$$:_{\{x\}}$$\psi(x,y)$, then $\varphi^{\circ}$ is $\Box\todo y \psi^{\circ}(x,y) \impli \Box \Box\todo y \psi^{\circ}(x,y)$, which is an instance of axiom \textbf{A$\p$3}; hence $\teo \varphi^{\circ}$.
\vspace{5mm}

($\varphi$ is an instance of \textbf{B5})\\

\qquad If $\varphi$ is $\nao t$$:_{\{x\}}$$\psi(x,y) \impli$ $?t$$:_{\{x\}}$$\nao t$$:_{\{x\}}$$\psi(x,y)$, then $\varphi^{\circ}$ is $\nao \Box\todo y \psi^{\circ}(x,y) \impli \Box \nao \Box\todo y \psi^{\circ}(x,y)$, which is an instance of axiom \textbf{A$\p$4}; hence $\teo \varphi^{\circ}$.

\pagebreak
($\varphi$ is an instance of \textbf{B6})\\

\qquad Suppose $\varphi$ is

\begin{center}
 $t$$:_{\{y\}}$$\psi(x,y,z) \impli gen_{x}(t)$$:_{\{y\}}$$\todo x\psi(x,y,z)$
\end{center}
So $\varphi^{\circ}$ is
\begin{center}
 $\Box \todo x \todo z\psi^{\circ}(x,y,z) \impli \Box \todo z \todo x\psi^{\circ}(x,y,z)$
\end{center}
By classical reasoning,

\begin{center}
 $\teo \todo x \todo z\psi^{\circ}(x,y,z) \impli \todo z \todo x\psi^{\circ}(x,y,z)$
\end{center}
By necessitation and the distributivity of $\Box$ over $\impli$,

\begin{center}
 $\teo \Box \todo x \todo z\psi^{\circ}(x,y,z) \impli \Box \todo z \todo x\psi^{\circ}(x,y,z)$.
\end{center}

\qquad If $\varphi$ is derived by using the rules \textbf{R1} or \textbf{R2} the result easily follows from the induction hypothesis.

\qquad Suppose $\varphi$ is derived using the rule \textbf{R3}. So $\varphi$ is $c$$:$$\psi(x)$ where $\psi(x)$ is an axiom. By the argument above, $\teo \psi^{\circ}(x)$. By generalization and necessitation, $\teo \Box \todo x\psi^{\circ}(x)$, i.e., $\teo \varphi^{\circ}$.
\end{proof}

\qquad As usual in the study of justification logic, the proof of Proposition 33 is a trivial induction on the theorems of the justification logic in question (in this case FOJT45). What is a more significant result is the following:\\

\textbf{(Realization Theorem)} If FOS5 $\teo \varphi$, then FOJT45 $\teo_{\C} \varphi^{r}$ for a constant specification $\C$ and a normal realization $r$.\\

\qquad Right now we believe that the best path to try to prove this theorem is to apply all the notions and results presented in this thesis in order to adapt the proof of the Realization Theorem using semantical tools (as presented in \cite{Fitting13}, \cite{Fitting13R} and \cite{Fitting14R}) for FOJT45. But we also consider different ways. Another strategy is to study the constructive argument using nested sequent calculus (as presented in \cite{Kuz10}) and see how this argument can be used for this case.
We want to consider these two paths in future research.

\section{Justification logic and interpolation}

\qquad When studying justification logic it is natural to investigate the relationship between this logic and modal logic. The Realization Theorem gives us a tool to see this relationship. Although we have left the proof of this theorem for future work, it is worthwhile to see one easy conclusion of the Realization Theorem. To do so we need to state one definition:

\begin{itemize}
\item[] \textit{The Interpolation Theorem} holds for FOJT45 iff for every constant specification $\C$ and sentences $\varphi$ and $\psi$, if $\teo_{\C} \varphi \impli\psi$, then there is a formula $\theta$ such that $\teo_{\C} \varphi \impli \theta$, $\teo_{\C} \theta \impli \psi$ and the non-logical symbols and the justification terms that occur in $\theta$ occur both in $\varphi$ and $\psi$.
\end{itemize}

\begin{pro}
If the Realization Theorem holds between FOS5 and FOJT45, then the Interpolation Theorem fails for FOJT45.
\end{pro}

\begin{proof}
Suppose that the Interpolation Theorem holds for FOJT45. By Theorem 2 and by the Completeness Theorem for FOS5, let $\varphi$ and $\psi$ be sentences such that FOS5 $\teo \varphi \impli \psi$ and there is no interpolant between them. By the Realization Theorem, there is a normal realization $r$ such that

\begin{center}
FOJT45 $\teo_{\C} \varphi^{r} \impli \psi^{r}$
\end{center}

\qquad By hypothesis, there is a formula $\theta$ such that the non-logical symbols and the justification terms that occur in $\theta$ occur both in $\varphi^{r}$ and $\psi^{r}$. Moreover, we have that

\begin{center}
FOJT45 $\teo_{\C} \varphi^{r} \impli \theta$\\
FOJT45 $\teo_{\C} \theta \impli \psi^{r}$
\end{center}
by the forgetful projection:

\begin{center}
FOS5 $\teo (\varphi^{r} \impli \theta)^{\circ}$\\
FOS5 $\teo (\theta \impli \psi^{r})^{\circ}$
\end{center}
i.e.,

\begin{center}
FOS5 $\teo \varphi \impli \theta^{\circ}$\\
FOS5 $\teo \theta^{\circ} \impli \psi$
\end{center}

\qquad Now, since there is no interpolant between $\varphi$ and $\psi$, then there is no relation symbol occurring in $\theta^{\circ}$. Hence, $\theta^{\circ}$ is a formula such that $\bot$ is the only atomic formula that occur in $\theta^{\circ}$. Thus, either $\theta^{\circ}$ is FOS5-valid or $\theta^{\circ}$ is FOS5-unsatisfiable.

\qquad On the one hand, if $\theta^{\circ}$ is FOS5-valid, then, since $\models_{FOS5} \theta^{\circ}\impli \psi$, $\psi$ is FOS5-valid. And so, $\varphi \impli \psi$ has an interpolant, contradicting our hypothesis.

\qquad On the other hand, if $\theta^{\circ}$ is FOS5-unsatisfiable, then, since $\models_{FOS5} \varphi \impli \theta^{\circ}$, $\varphi$ is FOS5-unsatisfiable. And so, $\varphi \impli \psi$ has an interpolant, contradicting our hypothesis.
\end{proof}

\vspace{10mm}

\qquad We hope that the topics presented in this thesis fulfilled two objectives: i) give a brief introduction to first-order S5; ii) clarify the connections between first-order modal logic and first-order justification logic.

\qquad About the last objective it is important to stress that, as Proposition 34 shows, the failure of the Interpolation Theorem for FOJT45 is just a straightforward consequence of the Realization Theorem. And so to prove this theorem for FOJT45 will not be only a subject of interest for the researchers involved in justification logic, but will be a result of interest for the broader modal logic community.


\documentclass[dvisvgm,hypertex,aspectratio=169]{beamer}
\usefonttheme{serif}

%\usepackage[utf8]{inputenc}
%\usepackage[T1]{fontenc}

%\usepackage[draft]{animate}
\usepackage[final]{animate}
\usepackage{ifthen}

%\usepackage{pythontex} % <--
\usepackage{graphicx}

\usepackage{tikz}
\usepackage{pgfplots}
\usepackage{pgfplotstable}
\pgfplotsset{compat=1.16}
\usetikzlibrary{calc}
\usetikzlibrary{decorations.pathmorphing,patterns}
\usepackage{amsmath}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Define footer
\usepackage{ccicons}

\makeatletter
\setbeamertemplate{footline}
{
  \leavevmode%
  \hbox{%
  %\begin{beamercolorbox}[wd=.333333\paperwidth,ht=2.25ex,dp=1ex,center]{title in head/foot}%
    %\usebeamerfont{title in head/foot}\insertsubsection
  %\end{beamercolorbox}%
  %\begin{beamercolorbox}[wd=.333333\paperwidth,ht=2.25ex,dp=1ex,right]{date in head/foot}%
  %  \usebeamerfont{date in head/foot}\insertshortdate{}\hspace*{2em}
  %  \insertframenumber{} / \inserttotalframenumber\hspace*{2ex}
  %\end{beamercolorbox}}%
  %\vskip0pt%
  \begin{beamercolorbox}[wd=.92\paperwidth,ht=2.25ex,dp=1ex,right]{author in head/foot}%
    \usebeamerfont{author in head/foot}\insertauthor
  \end{beamercolorbox}%
  \begin{beamercolorbox}[wd=.08\paperwidth,ht=2.25ex,dp=1ex,right]{date in head/foot}%
    \ccbysa
  \end{beamercolorbox}}%
  \vskip0pt%
}
\makeatother
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\author{\href{mailto:kjartan@tec.mx}{kjartan@tec.mx}}

\begin{document}

\section{Create animation}

\begin{frame}[label=A]{Aliasing}
  \pgfplotsset{
    dirac/.style={
      mark=triangle*,
      mark options={scale=0.6},
      ycomb,
      scatter,
      visualization depends on={y/abs(y)-1 \as \sign},
      scatter/@pre marker code/.code={\scope[rotate=90*\sign,yshift=-2pt]}
    }
  }

\pgfmathsetmacro{\omegaS}{20}
\pgfmathsetmacro{\omegaN}{\omegaS*0.5}
\pgfmathsetmacro{\nframes}{\omegaS - 1}
\pgfmathsetmacro{\ndsamples}{36}
\pgfmathsetmacro{\hh}{2*pi/\omegaS}
\pgfmathsetmacro{\tend}{\hh*(\ndsamples-1)}

  \begin{center}
    \begin{animateinline}[controls, loop, palindrome]{4}
      \multiframe{\nframes}{n=1+1}{
        \pgfmathsetmacro{\ww}{\n*1.0}
        \pgfmathsetmacro{\nsamples}{\n*30}
        \pgfmathsetmacro{\nasamples}{(\nframes-\n+1)*30}
        \pgfmathsetmacro{\omegaal}{\omegaS - \ww}
        \pgfmathsetmacro{\aliasing}{\omegaN - \omegaal}

        \begin{tikzpicture}[scale=0.6,]
          \begin{axis}[
            width=14cm,
            height=3.8cm,
            title={Spectrum},
            xlabel={$\omega$ [rad/s]},
            ylabel={$|Y(i\omega)|$},
            xmin=-22,
            xmax=22,
            xtick={-\omegaS, -\omegaN, 0, \omegaN, \omegaS},
            xticklabels={$-\omega_s$, $-\omega_N$, 0, $\omega_N$, $\omega_s$},
            ytick = \empty,
            ymin = 0,
            ymax = 1.2,
            ]
            \addplot[blue!80!green, thick, dirac] coordinates {(-\ww, 1) (\ww, 1)};
            \addplot[red, thick, dashed, dirac] coordinates {(-\omegaal, 1) (\omegaal, 1)};
          \end{axis}

          \begin{axis} [
            title={Original signal},
            width = 14cm,
            height=3.8cm,
            yshift = -4cm,
            xlabel = {$t$},
            ylabel = {$y$},
            xtick = \empty,
            ytick = \empty,
            ymin=-1.1,
            ymax = 1.1,
            ]
            \addplot+[thick, blue!80!green, no marks, domain=0:\tend, samples=\nsamples, smooth] {sin(deg(\ww*x)-20)};
              \addplot+[black!80, ycomb, mark=*,  mark options={black!70}, domain=0:\tend, samples=\ndsamples] {sin(deg(\ww*x)-20)};

          \end{axis}
          \begin{axis} [
            title={Alias signal},
            width = 14cm,
            height=3.8cm,
            yshift = -8cm,
            xlabel = {$t$},
            ylabel = {$y_a$},
            xtick = \empty,
            ytick = \empty,
            ymin=-1.1,
            ymax = 1.1,
            ]
            \addplot[black!80, ycomb, mark=*, mark options={black!70}, domain=0:\tend, samples=\ndsamples] {sin(deg(\ww*x)-20)};
              \addplot[red!80!black, domain=0:\tend, no marks, samples=\nasamples, smooth] {-(\aliasing >0)*sin(deg(\omegaal*x)+20) + (\aliasing <= 0)*sin(deg(\ww*x)-20)};

            \end{axis}
            %\node at (1,1) {$\omega_1 = \ww$};
            %\node at (1,-1) {$\omega_a = \omegaal$};
            %\node at (1,-2) {$h = \hh$};
        \end{tikzpicture}
      }
    \end{animateinline}
  \end{center}
\end{frame}
\end{document}


\section{Pathfinding}

Pathfinding is a technique or a process carried by a software running in a computer from which the shortest path or route between two points is extrapolated. Pathfinding is not only important in video games, but also in other areas such as delivery, transport services and intelligent storage among others. Pathfinding algorithms are typically, although not solely, heavily based on the Dijkstra's algorithm[5] which implies what follows: Let the node starting be X. Let the distance of node Y be the distance from X to Y. Dijkstra's algorithm will assign some initial distance values and will try iteratively to improve them step by step.

\vspace{2mm}
However there are other complementary approaches using different techniques that seek to improve such algorithms.

\subsection{Techniques for Pathfinding}
Pathfinding is usually a two-step process: first, a graph generation algorithm followed by a Pathfinding algorithm to calculate the best path. Zeyad Adb Algfoor et al[10] published a brief but thorough compendium of the most common techniques developed during the last 10 years.

\subsubsection{Grids}
In Pathfinding grids are represented as graphs, a list of vertices or points connected by edges to represent the map, the navigation performance of which is based on its attributes. The most popular grid approaches are: regular grids and irregular grids.

\vspace{2mm}
Regular grids use triangles, squares and hexagons to describe tessellations of a given terrain. The most prominent of which are the following:

\begin{itemize}
	\item 2D hexagonal Grid
	\item 2D Square Grid
	\item 2D Triangular Grid
	\item 3D Cubic Grid
\end{itemize}

\vspace{2mm}
Whereas irregular grids use different techniques to define terrain topology such as:

\begin{itemize}
	\item Visibility Graphs
	\item Mesh Navigation
	\item Waypoints
\end{itemize}

\vspace{2mm}
Further information on the actual implementation of these pathfinding techniques can be found in Zeyad et al[9]'s paper.

\subsection{Using Potential Fields}

In 1985, Ossama Khatib discussed a new concept in regards a real-time obstacle avoidance for manipulators and mobile Robots. He named it Artificial Potential Fields[6]. In 2008, Johan Hagelback et al.[10] used it to create a Multi-agent Potential Field-based Bot for Real-Time Strategy Games(RTS) in which objects in the map where assigned charges (like magnetic chargers positive or negative) and such charges were used as forces to attract the agents to specific destination or to repel such agents to avoid collisions or to delimit the terrain.

\vspace{2mm}
They proposed two-different scenarios using Open real-time strategy (ORTS) engine to deploy the bot. In the first scenario the task of the bot was to recollect as many resource as possible in a limited amount of time. To do so Johan Hagelback et al. had 20 agents (workers) using a finite state machine (FSM) to define their behaviour and assigned the charges to the agents, to the resource gathering point, the base were resources had to be delivered and "sheep" moving around the map to provide moving obstacles. Even though the results shown that the bot was prone to disconnect from the game in some occasions, it beat bots developed in previous years improving the average of resources gathered.

\vspace{2mm}
The second scenario, using ORTS, was a tank game where two players had a number of tanks and bases and the goal of the game was to destroy both enemy bases and tanks. After outlining the rules for the game agents, they assigned the charges accordingly and programmed the bot to avoid situating tanks in the middle of enemy clusters. The results of such scenario, tested against other bots, were of a 98\% of victory rate over their opponents.

\subsection{Aesthetics in Pathfinding}

While most of the algorithms focuses lies on obtaining the best suitable path, such path does not always meet the expectations by using an unrealistic behaviour that differs greatly of what a humans would perform. In order to find more "human-like" behaviour in Pathfinding some authors such as Coleman, R.[7] introduce the idea of Aesthetics in it.

\vspace{2mm}
The experiment's approach was to use Mandelbrot's Fractal Dimension Analysis[8] to determine the aesthetics level of certain paths, an A* Pathfinding algorithm was used to create the path and then tweaked to reward paths that were obstacle-prone to simulate a behaviour of "stealthyness". By doing so the levels of aesthetics were higher than in those were paths were determined as best suited by the algorithm.


% !TeX root = ./forallxsol.tex

\stepcounter{chapter} % Introducing ML

\chapter{Natürliche Herleitung für die ML}

\setcounter{ProbPart}{0}

\problempart
Geben Sie Beweise für die folgenden Aussagen:
\begin{earg}
\item $\Box (A\eand B)\vdash_\mlK \Box A \eand \Box B$
\myanswer{
\[\begin{nd}
\hypo{1}{\Box (A \eand B)}
\open
\hypo{2}{\ebox}
\have{3}{A \eand B}\boxe{1}
\have{4}{A}\ae{3}
\close
\have{5}{\ebox A}\boxi{3-4}
\open
\hypo{6}{\ebox}
\have{7}{A \eand B}\boxe{1}
\have{8}{B}\ae{7}
\close
\have{9}{\ebox B}\boxi{6-8}
\have{10}{\Box A \eand \Box B}\ai{5,9}
\end{nd}\]
}
\newpage
\item $\Box A\eand\Box B\vdash_\mlK \Box( A \eand  B)$
\myanswer{
\[\begin{nd}
\hypo{1}{\Box A \eand \Box B}
\have{2}{\Box A}\ae{1}
\have{3}{\Box B}\ae{1}
\open
\hypo{4}{\ebox}
\have{5}{A}\boxe{2}
\have{6}{B}\boxe{3}
\have{7}{A\eand B}\ai{5,6}
\close
\have{8}{\Box (A\eand B)}\boxi{4-7}
\end{nd}\]
}

\item $\Box A\eor\Box B\vdash_\mlK \Box( A \eor  B)$
\myanswer{
\[\begin{nd}
\hypo{1}{\Box A \eor \Box B}
\open
\hypo{2}{\Box A}
\open
\hypo{3}{\ebox}
\have{4}{A}\boxe{2}
\have{5}{A\eor B}\oi{4}
\close
\have{6}{\ebox(A \eor B)}\boxi{3-5}
\close
\open
\hypo{7}{\Box B}
\open
\hypo{8}{\ebox}
\open
\have{9}{B}\boxe{7}
\have{10}{A\eor B}\oi{9}
\close
\have{11}{\Box (A\eor B)}\boxi{8-10}
\close
\have{12}{\Box(A\eor B)}\oe{1,2-6,7-11}
\end{nd}\]
}

\item $\Box (A \eiff B)\vdash_\mlK  \Box A \eiff \Box B$
\myanswer{
\[\begin{nd}
\hypo{1}{\Box(A\eiff B)}
\open
\hypo{2}{\Box A}
\open
\hypo{3}{\ebox}
\have{4}{A\eiff B}\boxe{1}
\have{5}{A}\boxe{2}
\have{6}{B}\be{4,5}
\close
\have{7}{\ebox B}\boxi{3-6}
\close
\open
\hypo{8}{\Box B}
\open
\hypo{9}{\ebox}
\have{10}{A\eiff B}\boxe{1}
\have{11}{B}\boxe{8}
\have{12}{A}\be{10,11}
\close
\have{13}{\ebox A}\boxi{9-12}
\close
\have{14}{\Box A \eiff \Box B}\bi{2-7, 8-13}
\end{nd}\]
}
\end{earg}

\problempart
Geben Sie Beweise für die folgenden Aussagen an, ohne Modalwechselregeln zu verwenden:
\begin{earg}
\item $\enot\Box A\vdash_\mlK  \Diamond \enot A$
\myanswer{
\[\begin{nd}
\hypo{1}{\enot\Box A}
\open
\hypo{2}{\ebox\enot\enot A}
\open
\hypo{3}{\ebox}
\have{4}{\enot\enot A}\boxe{2}
\have{5}{A}\dne{4}
\close
\have{6}{\ebox A}\boxi{3-5}
\have{7}{\ered}\ne{1,6}
\close
\have{8}{\enot\Box\enot\enot A}\ni{2-6}
\have{9}{\Diamond\enot A}\diadf{8}
\end{nd}\]
}

\item $\Diamond\enot A\vdash_\mlK \enot \Box A$
\myanswer{
\[\begin{nd}
\hypo{1}{\Diamond\enot A}
\have{2}{\enot\Box\enot \enot A}\diadf{1}
\open
\hypo{3}{\ebox A}
\open
\hypo{4}{\ebox}
\open
\hypo{5}{\enot A}
\have{6}{A}\boxe{3}
\have{7}{\ered}\ne{5,6}
\close
\have{8}{\enot\enot A}\ni{5-7}
\close
\have{9}{\ebox\enot\enot A}\boxi{4-8}
\have{10}{\ered}\ne{2,9}
\close
\have{11}{\enot\Box A}\ni{3-10}
\end{nd}\]
}
\newpage
\item $\enot\Diamond A\vdash_\mlK \Box\enot A$
\myanswer{
\[\begin{nd}
\hypo{1}{\enot\Diamond A}
\open
\hypo{2}{\enot\Box\enot A}
\have{3}{\Diamond A}\diadf{2}
\have{4}{\ered}\ne{1,3}
\close
\have{5}{\enot\enot\Box\enot A}\ni{2-4}
\have{6}{\Box \enot A}\dne{5}
\end{nd}\]
}

\item $\Box\enot A\vdash_\mlK \enot\Diamond A$
\myanswer{
\[\begin{nd}
\hypo{1}{\Box\enot A}
\open
\hypo{2}{\Diamond A}
\have{3}{\enot \Box \enot A}\diadf{2}
\have{4}{\bot}\ne{1,3}
\close
\have{5}{\enot\Diamond A}\by{$\enot$I}{2-4}
\end{nd}\]
}
\end{earg}

\problempart
In den folgenden Fällen ist es Ihnen erlaubt, die Modalwechselregeln in Ihren Beweisen zu verwenden:
\begin{earg}
\item $\Box(A\eif B), \Diamond A \vdash_\mlK  \Diamond B$
\myanswer{
\[\begin{nd}
\have{1}{\Box(A\eif B)}
\hypo{2}{\Diamond A}
\have{3}{\enot\Box\enot A}\diadf{2}
\open
\hypo{4}{\ebox\enot B}
\open
\hypo{5}{\ebox}
\have{6}{A\eif B}\boxe{1}
\have{7}{\enot B}\boxe{4}
\have{8}{\enot A}\mt{5,6}
\close
\have{9}{\ebox\enot A}\boxi{5-8}
\have{10}{\ered}\ne{3,9}
\close
\have{11}{\enot\Box\enot B}\ni{4-10}
\have{12}{\Diamond B}\diadf{11}
\end{nd}\]
}

\item $\Box A \vdash_\mlK  \enot\Diamond\enot A$
\myanswer{
\[\begin{nd}
\hypo{1}{\Box A}
\open
\hypo{2}{\Diamond\enot A}
\have{3}{\enot\Box A}\mc{2}
\have{4}{\bot}\ne{1,3}
\close
\have{5}{\enot\Diamond\enot A}\ni{2-4}
\end{nd}\]
}

\item $\enot\Diamond\enot A \vdash_\mlK  \Box A$
\myanswer{
\[\begin{nd}
\hypo{1}{\enot\Diamond\enot A}
\have{2}{\Box\enot\enot A}\mc{1}
\open
\hypo{3}{\ebox}
\have{4}{\enot\enot A}\boxe{2}
\have{5}{A}\dne{4}
\close
\have{6}{\Box A}\boxi{3-5}
\end{nd}\]
}
\end{earg}

\problempart
Geben Sie Beweise für die folgenden Aussagen:
\begin{earg}
\item $P\vdash_\mlT \Diamond P$
\myanswer{
\[\begin{nd}
\hypo{1}{P}
\open
\hypo{3}{\Box \enot P}
\have{4}{\enot P}\rt{3}
\have{5}{\ered}\ne{1,4}
\close
\have{6}{\enot\Box\enot P}\ni{3-5}
\have{7}{\Diamond P}\diadf{6}
\end{nd}\]
}

\item $\vdash_\mlT  (A\eand B)\eor(\enot \Box A\eor\enot\Box B)$
\myanswer{
\[\begin{nd}
\open
\hypo{1}{\Box A\eand\Box B}
\have{2}{\Box A}\ae{1}
\have{3}{\Box B}\ae{1}
\have{4}{A}\rt{2}
\have{5}{B}\rt{3}
\have{6}{A\eand B}\ai{4,5}
\have{7}{(A\eand B)\eor (\enot \Box A\eor \enot \Box B)}\oi{6}
\close
\open
\hypo{8}{\enot(\Box A\eand \Box B)}
\have{9}{\enot\Box A \eor \enot\Box B}\dem{8}
\have{10}{(A\eand B)\eor (\enot\Box A \eor \enot\Box B)}\oi{9}
\close
\have{11}{(A\eand B)\eor (\enot\Box A \eor \enot\Box B)}\tnd{1-7,8-10}
\end{nd}\]
}
\end{earg}
\newpage
\problempart
Geben Sie Beweise für die folgenden Aussagen:
\begin{earg}
\item $\Box(\Box A\eif B), \Box (\Box B\eif C), \Box A \vdash_\mlSfour \Box\Box C$
\myanswer{
\[\begin{nd}
\have{1}{\Box(\Box A \eif B)}
\have{2}{\Box(\Box B\eif C)}
\hypo{3}{\Box A}
\open
\hypo{4}{\ebox}
\have{5}{\Box A}\rfour{3}
\have{6}{\Box(\Box A\eif B)}\rfour{1}
\have{7}{\Box(\Box B\eif C)}\rfour{2}
\open
\hypo{8}{\ebox}
\have{9}{\Box A}\rfour{5}
\have{10}{\ebox(\ebox A \eif B)}\rfour{6}
\open
\hypo{11}{\ebox}
\have{12}{\ebox A}\rfour{9}
\have{13}{\Box A\eif B}\boxe{10}
\have{14}{B}\ce{12,13}
\close
\have{15}{\ebox B}\boxi{11-14}
\have{16}{\ebox B \eif C}\boxe{7}
\have{17}{C}\ce{15,16}
\close
\have{18}{\Box C}\boxi{8-17}
\close
\have{19}{\Box \Box C}\boxi{4-18}
\end{nd}\]
}

\item $\Box A \vdash_\mlSfour  \Box(\Box A \eor B)$
\myanswer{
\[\begin{nd}
\hypo{1}{\Box A}
\open
\hypo{2}{\ebox}
\have{3}{\Box A}\rfour{1}
\have{4}{\Box A \eor B}\oi{4}
\close
\have{5}{\Box(\Box A \eor B)}\boxi{2-4}
\end{nd}\]
}

\item $\Diamond \Diamond A \vdash_\mlSfour  \Diamond A$
\myanswer{
\[\begin{nd}
\hypo{1}{\Diamond\Diamond A}
\open
\hypo{2}{\ebox\enot A}
\have{3}{\enot\ebox\enot\ediamond A}\diadf{1}
\open
\hypo{5}{\ebox}
\have{6}{\Box\enot A}\rfour{2}
\have{7}{\enot\Diamond A}\mc{6}
\close
\have{12}{\ebox\enot\Diamond A}\boxi{5-7}
\have{13}{\ered}\ne{3,12}
\close
\have{14}{\enot\ebox\enot A}\ni{2-13}
\have{15}{\Diamond A}\diadf{14}
\end{nd}\]
}
\end{earg}
\newpage
\problempart
Geben Sie Beweise in \mlSfive{} für die folgenden Aussagen:
\begin{earg}
\item $\enot\Box\enot A, \Diamond B\vdash_\mlSfive\Box(\Diamond A \eand \Diamond B)$
\myanswer{
\[\begin{nd}
\have{1}{\enot\Box\enot A}
\hypo{2}{\Diamond B}
\have{3}{\enot\ebox\enot B}\diadf{2}
\open
\hypo{4}{\ebox}
\have{5}{\enot\ebox\enot A}\rfive{1}
\have{6}{\Diamond A}\diadf{5}
\have{7}{\enot\ebox\enot B}\rfive{3}
\have{8}{\Diamond B}\diadf{7}
\have{10}{\Diamond A \eand \Diamond B}\ai{6,8}
\close
\have{11}{\Box(\Diamond A \eand \Diamond B)}\boxi{4-10}
\end{nd}\]
}

\item $A\vdash_\mlSfive\Box\ediamond A$
\myanswer{
\[\begin{nd}
\hypo{1}{A}
\open
\hypo{2}{\ebox\enot A}
\have{3}{\enot A}\rt{2}
\have{4}{\ered}\ne{1,3}
\close
\have{5}{\enot\ebox\enot A}
\open
\hypo{6}{\Box}
\have{7}{\enot\ebox\enot A}\rfive{5}
\have{8}{\Diamond A}\diadf{7}
\close
\have{9}{\Box\Diamond A}\boxi{6-8}
\end{nd}\]
}

\item $\ediamond\ediamond A\vdash_\mlSfive\ediamond A$
\myanswer{
\[\begin{nd}
\hypo{1}{\ediamond\ediamond A}
\have{2}{\enot\ebox\enot \ediamond A}\diadf{1}
\open
\hypo{3}{\ebox}
\open
\hypo{4}{\Box\enot A}
\open
\hypo{5}{\ebox}
\open
\hypo{6}{\ediamond A}
\have{7}{\enot\ebox\enot A}\diadf{6}
\have{8}{\ebox\enot A}\rfour{4}
\have{9}{\ered}\ne{7,8}
\close
\have{10}{\enot\ediamond A}\ni{5-9}
\close
\have{11}{\ebox\enot\ediamond A}\boxi{5-10}
\have{12}{\enot\ebox\enot\ediamond A}\rfive{2}
\have{13}{\ered}\ne{11,12}
\close
\have{14}{\enot\ebox\enot A}\ni{4-13}
\have{15}{\ediamond A}\diadf{14}
\close
\have{16}{\ebox\ediamond A}\boxi{3-15}
\have{17}{\ediamond A}\rt{16}
\end{nd}\]
}
\end{earg}

\chapter{Semantik der ML}

Wir präsentieren alle Gegeninterpretationen diagrammatisch.

\setcounter{ProbPart}{0}
\
\problempart
Legen Sie Gegeninterpretationen zu den folgenden falschen Behauptungen vor:
\begin{earg}
\item $\enot P \vDash_\mlK  \enot\Diamond P$
\myanswer{
\begin{center}
\begin{tikzpicture}
\node (atom1) at (0,1) {1};
\node (atom2) at (3,1) {2};
\node (atom3) at (-0.25,0) {$\enot P$};
\node (atom4) at (3,0) {$P$};
\draw[->, thick] (atom1) -- (atom2);
\draw (-0.8,-0.6) rectangle (0.4,0.5);
\draw (2.4,-0.6) rectangle (3.6,0.5);
\end{tikzpicture}
\end{center}
}

\item $\Box(P \eor Q)\vDash_\mlK  \Box P \eor \Box Q$
\myanswer{
\begin{center}
\begin{tikzpicture}
\node (atom1) at (0,1) {1};
\node (atom2) at (3,1) {2};
\node (atom3) at (-0.25,0) {$P$};
\node (atom4) at (3,0) {$\enot P$};
\node (atom5) at (-0.25,-0.7) {$\enot Q$};
\node (atom6) at (3,-0.7) {$Q$};
\draw[->, thick] (atom1)+(-0.15,0.15) arc (-330:-30:.3);
\draw[->, thick] (atom1) -- (atom2);
\draw (-0.8,-1.2) rectangle (0.4,0.5);
\draw (2.4,-1.2) rectangle (3.6,0.5);
\end{tikzpicture}
\end{center}
}

\item $\vDash_\mlK  \enot \Box (A\eand \enot A)$
\myanswer{
\begin{center}
\begin{tikzpicture}
\node (atom1) at (0,1) {1};
\node (atom3) at (-0,0) {$\enot A$};
\draw (-0.5,-0.6) rectangle (0.6,0.5);
\end{tikzpicture}
\end{center}
}
\newpage
\item $\Box A\vDash_\mlK  A$
\myanswer{
\begin{center}
\begin{tikzpicture}
\node (atom1) at (0,1) {1};
\node (atom2) at (3,1) {2};
\node (atom3) at (-0.25,0) {$\enot A$};
\node (atom4) at (3,0) {$A$};
\draw[->, thick] (atom1) -- (atom2);
\draw (-0.8,-0.6) rectangle (0.4,0.5);
\draw (2.4,-0.6) rectangle (3.6,0.5);
\end{tikzpicture}
\end{center}
}
\end{earg}

\problempart
Legen Sie Gegeninterpretationen zu den folgenden falschen Behauptungen vor:
\begin{earg}
\item $\Box (M\eif O),\Diamond M\vDash_\mlT  O$
\myanswer{
\begin{center}
\begin{tikzpicture}
\node (atom1) at (0,1) {1};
\node (atom2) at (3,1) {2};
\node (atom3) at (-0.25,0) {$\enot M$};
\node (atom4) at (3,0) {$M$};
\node (atom5) at (-0.25,-0.7) {$\enot O$};
\node (atom6) at (3,-0.7) {$O$};
\draw[->, thick] (atom1)+(-0.15,0.15) arc (-330:-30:.3);
\draw[->, thick] (atom1) -- (atom2);
\draw[->, thick] (atom2)+(0.15,-0.15) arc (-150:150:.3);
\draw (-0.8,-1.2) rectangle (0.4,0.5);
\draw (2.4,-1.2) rectangle (3.6,0.5);
\end{tikzpicture}
\end{center}
}

\item $\Box A\vDash_\mlT  \Box \Box A$
\myanswer{
\begin{center}
\begin{tikzpicture}
\node (atom1) at (0,1) {1};
\node (atom2) at (3,1) {2};
\node (atom3) at (1.5,-0.5) {3};
\node (atom4) at (-0.25,1.9) {$A$};
\node (atom5) at (3,1.9) {$A$};
\node (atom6) at (1.4,-1.5) {$\enot A$};
\draw[->, thick] (atom1)--(atom2);
\draw[->, thick] (atom1)+(-0.15,0.15) arc (-330:-30:.3);
\draw[->, thick] (atom2)+(0.15,-0.15) arc (-150:150:.3);
\draw[->, thick] (atom2)--(atom3);
\draw[->, thick] (atom3)+(-0.15,0.15) arc (-330:-30:.3);
\draw (-0.8,1.5) rectangle (0.4,2.3);
\draw (2.4,1.5) rectangle (3.6,2.3);
\draw (0.8,-1.15) rectangle (2.2,-1.9);
\end{tikzpicture}
\end{center}
}
\end{earg}

\problempart
Legen Sie Gegeninterpretationen zu den folgenden falschen Behauptungen vor:
\begin{earg}
\item $\Diamond A\vDash_\mlSfour  \Box\Diamond A$
\myanswer{
\begin{center}
\begin{tikzpicture}
\node (atom1) at (0,1) {1};
\node (atom2) at (3,1) {2};
\node (atom3) at (-0.25,0) {$A$};
\node (atom4) at (3,0) {$\enot A$};
\draw[->, thick] (atom1)+(-0.15,0.15) arc (-330:-30:.3);
\draw[->, thick] (atom2)+(0.15,-0.15) arc (-150:150:.3);
\draw[->, thick] (atom1) -- (atom2);
\draw (-0.8,-0.6) rectangle (0.4,0.5);
\draw (2.4,-0.6) rectangle (3.6,0.5);
\end{tikzpicture}
\end{center}
}

\item $\Diamond A, \Box (\Diamond A \eif B)\vDash_\mlSfour \Box B$

\myanswer{
\begin{center}
\begin{tikzpicture}
\node (atom1) at (0,1) {1};
\node (atom2) at (3,1) {2};
\node (atom3) at (-0.25,0) {$A$};
\node (atom4) at (3,0) {$\enot A$};
\node (atom5) at (-0.25,-0.7) {$B$};
\node (atom6) at (3,-0.7) {$\enot B$};
\draw[->, thick] (atom1)+(-0.15,0.15) arc (-330:-30:.3);
\draw[->, thick] (atom2)+(0.15,-0.15) arc (-150:150:.3);
\draw[->, thick] (atom1) -- (atom2);
\draw (-0.8,-1.2) rectangle (0.4,0.5);
\draw (2.4,-1.2) rectangle (3.6,0.5);
\end{tikzpicture}
\end{center}
}
\end{earg}