\documentclass{article}

\usepackage{enumitem}
\usepackage[english]{babel}
\usepackage[utf8]{inputenc}
\usepackage{parskip}
\usepackage{graphicx}
\usepackage{mathtools}
\usepackage{mathrsfs}
\usepackage{amsmath,amsthm,hyperref}
\usepackage{amssymb}
\usepackage{amsmath}
\DeclareMathOperator{\Tr}{tr}
\usepackage{bbm}

% Margins
\usepackage[top=2.5cm, left=3cm, right=3cm, bottom=4.0cm]{geometry}
% Colour table cells
\usepackage[table]{xcolor}

% Get larger line spacing in table
\newcommand{\tablespace}{\\[1.25mm]}
\newcommand\Tstrut{\rule{0pt}{2.6ex}}         % = `top' strut
\newcommand\tstrut{\rule{0pt}{2.0ex}}         % = `top' strut
\newcommand\Bstrut{\rule[-0.9ex]{0pt}{0pt}}   % = `bottom' /

% my new commands
\newcommand\partialkj{\frac{\partial^2}{\partial\theta_k\partial\theta_j}}
\makeatletter
\newcommand*\bigcdot{\mathpalette\bigcdot@{.5}}
\newcommand*\bigcdot@[2]{\mathbin{\vcenter{\hbox{\scalebox{#2}{$\m@th#1\bullet$}}}}}
\makeatother
\newcommand{\minus}{\scalebox{0.5}[1.0]{$-$}}
\newcommand{\zero}{\scalebox{0.6}[0.75]{$^{(0)}$}}
\newcommand{\supi}[1]{\scalebox{0.6}[0.75]{$^{(#1)}$}}
\newcommand{\bigDash}{\scalebox{3.0}[1.0]{$-$}}

%%%%%%%%%%%%%%%%%
%     Title     %
%%%%%%%%%%%%%%%%%
\title{Problem Set #1: Supervised Learning}
\author{Eitan Joseph \and Caroline Wang}
\date{\today}

\begin{document}
\maketitle

%%%%%%%%%%%%%%%%%
%   Problem 1   %
%%%%%%%%%%%%%%%%%
\section{Problem 1}
\textbf{Newton’s method for computing least squares\\\\}
In this problem, we will prove that if we use Newton’s method solve the least squares optimization problem, then we only need one iteration to converge to $\theta$.
\begin{enumerate}[label=(\alph*)]
    \item Find the Hessian of the cost function $J(\theta) = \frac{1}{2}\sum_{i=1}^{m}\left(\theta^T x^{(i)}-y^{(i)}\right )^2$.\\
    \textit{answer:}
    \begin{align*}
        H_k_j = \partialkj J(\theta) = \partialkj \frac{1}{2}\sum_{i=1}^{m}\left(\theta^T x^{(i)}-y^{(i)}\right )^2 = \frac{\partial}{\partial\theta_k} \sum_{i=1}^{m}\left(\theta^T x^{(i)}-y^{(i)}\right )(x_j^{(i)}) = \sum_{i=1}^{m}x_k^{(i)}x_j^{(i)}
    \end{align*}
    The sum can be understood as $x_k^{(i)}\bigcdot x_j^{(i)}\quad \forall i$
    \begin{align*}
         H = X^TX
    \end{align*}
    \item Show that the first iteration of Newton’s method gives us $\theta^* = \left(X^TX\right)^{\minus1}X^T\Vec{y}$, the
solution to our least squares problem.\\
    \textit{answer:}
    \begin{align*}
        &&\text{The first iteration of Newton's Method is given by}
        &&\theta^* = \theta^{\zero} - H^{\minus1}\nabla_{\theta^{\zero}} J(\theta^{\zero}) \\
        &&\text{Via lecture 2 we know that}
        &&\nabla_\theta J(\theta) = X^TX\theta - X^T\Vec{y}\\
        &&\text{Which means we need to solve}
        &&\theta^* = \theta^{\zero} - H^{\minus1}\left(X^TX\theta^{\zero} - X^T\Vec{y}\right)\\
        &&\text{We can substitute our result from part (a) to get}
        &&\theta^* = \theta^{\zero} - \left(X^TX\right)^{\minus1}\left(X^TX\theta^{\zero} - X^T\Vec{y}\right)\\
        &&\text{This reduces to}
        &&\theta^* = \left(X^TX\right)^{\minus1}X^T\Vec{y}
    \end{align*}
\end{enumerate}

%%%%%%%%%%%%%%%%%
%   Problem 3   %
%%%%%%%%%%%%%%%%%
\section{Problem 3}
\textbf{Multivariate least squares\\\\}
So far in class, we have only considered cases where our target variable $y$ is a scalar value. Suppose that instead of trying to predict a single output, we have a training set with multiple outputs for each example:\\
\begin{align*}
    \{(x^{\supi{i}}y^{\supi{i}}),i=1,...,m\},\; x^{\supi{i}}\in\mathbb{R}^n,\; y^{\supi{i}}\in\mathbb{R}^p
\end{align*}
Thus for each training example, $y^{\supi{i}}$ is vector-valued, with p entries. We wish to use a linear model to predict the outputs, as in least squares, by specifying the parameter matrix $\Theta$ in
\begin{align*}
    y = \Theta^Tx \text{, where }\Theta \in \mathbb{R}^{n\times p}
\end{align*}

\begin{enumerate}[label=(\alph*)]
    \item The cost function for this case is
    \begin{align*}
        J(\Theta) = \frac{1}{2}\sum_{i=1}^{m}\sum_{j=1}^{p}\left((\Theta^T x^{\supi{i}})_j - y_j^{\supi{i}}\right)^2
    \end{align*}
    Write $J(\Theta)$ in matrix-vector notation (i.e., without using any summations).\\
    \begin{center}
    $X=$
        \begin{bmatrix}
            \bigDash && (x^{(1)})^T && \bigDash\\
            \bigDash && (x^{(2)})^T && \bigDash\\
            && \vdots &&\\
            \bigDash && (x^{(m)})^T && \bigDash\\
        \end{bmatrix}
    \end{center}
    and the $m\times p$ target matrix
    \begin{center}
    $Y=$
        \begin{bmatrix}
            \bigDash && (y^{(1)})^T && \bigDash\\
            \bigDash && (y^{(2)})^T && \bigDash\\
             && \vdots &&\\
            \bigDash && (y^{(m)})^T && \bigDash\\
        \end{bmatrix}
    \end{center}
    \textit{answer:}
    \begin{align*}
        J(\Theta) &{}= \frac{1}{2}\sum_{i=1}^{m}\sum_{j=1}^{p}\left((\Theta^T x^{\supi{i}})_j - y_j^{\supi{i}}\right)^2\\
        &{}= \frac{1}{2}\sum_{i=1}^{m}\sum_{j=1}^{p}\left(X\Theta - Y\right)^2_{i,j}\\
        &{}= \frac{1}{2}\sum_{j=1}^{p}\left(X\Theta - Y\right)^T\left(X\Theta - Y\right)_{j,j}\\
        &{}= \frac{1}{2}\Tr\left(\left(X\Theta - Y\right)^T\left(X\Theta - Y\right)\right)
    \end{align*}
    \item  Find the closed form solution for Θ which minimizes $J(\Theta)$. This is the equivalent to the normal equations for the multivariate case.\\\\
    \textit{answer:}\\
    We now have an optimization problem of the form:
    \begin{align*}
        \min_{\Theta}\;J(\Theta)
    \end{align*}
    We can solve this by setting the gradient of $J(\Theta)$ to 0 and solving for $\Theta$.
    \begin{align*}
        \nabla_{\Theta}J(\Theta) &{}= \frac{1}{2}\nabla_{\Theta}\Tr\left(\left(X\Theta - Y\right)^T\left(X\Theta - Y\right)\right)\\
        &{}=\frac{1}{2}\nabla_{\Theta}\Tr\left(\Theta^TX^TX\Theta - \Theta^TX^TY - Y^TX\Theta + Y^TY\right)\\
        % maybe make a note here that you need to use the property of trace and gradient
        &{}=\frac{1}{2}\left(\nabla_{\Theta}\Tr\Theta^TX^TX\Theta - 2\nabla_{\Theta}\Tr\Theta^TX^TY + \nabla_{\Theta}\Tr Y^TY\right)\\
        &{}=\frac{1}{2}\left(X^TX\Theta + X^TX\Theta - 2\nabla_{\Theta}\Tr\Theta^TX^TY\right)\\
        &{}=X^TX\Theta - X^TY
    \end{align*}
    Now after setting this result to 0 we get
    \begin{align*}
        &{}X^TX\Theta - X^TY = 0\\
        &{}\Theta = \left(X^TX\right)^{\minus1} X^TY
    \end{align*}
    \item Suppose instead of considering the multivariate vectors $y^{\supi{i}}$ all at once, we instead compute each variable $y^{\supi{i}}_j$ separately for each $j = 1,\dots, p$. In this case, we have a $p$ individual linear models, of the form
    \begin{align*}
        y^{\supi{i}}_j = \theta^T_jx^{\supi{i}},\;j = 1,\dots, p.
    \end{align*}
    How do the parameters from these $p$ independent least squares problems compare to the multivariate solution?\\\\
    \textit{answer:}\\
    We first realize that $\Theta$ can be written in terms of each $\theta_j$s as
    \begin{align*}
        \sum_{i=1}^p
        \begin{bmatrix}
            \theta_i^{\supi{1}}\mathbbm{1}\{i=1\} & \theta_i^{\supi{1}}\mathbbm{1}\{i=2\} & \dots & \theta_i^{\supi{1}}\mathbbm{1}\{i=p\} \\
            \theta_i^{\supi{2}}\mathbbm{1}\{i=1\} & \theta_i^{\supi{2}}\mathbbm{1}\{i=2\} & \dots & \theta_i^{\supi{2}}\mathbbm{1}\{i=p\} \\
            \vdots &\vdots &\ddots &\vdots\\
            \theta_i^{\supi{n}}\mathbbm{1}\{i=1\} &\theta_i^{\supi{n}}\mathbbm{1}\{i=2\} &\dots &\theta_i^{\supi{n}}\mathbbm{1}\{i=p\}
        \end{bmatrix}_{n\times p}
         =
        \begin{bmatrix}
            \theta_1 &\theta_2 &\dots &\theta_p
        \end{bmatrix}
    \end{align*}
    Combining this with the original result of part (b) gives us
    \begin{align*}
        \begin{bmatrix}
            \theta_1 &\theta_2 &\dots &\theta_p
        \end{bmatrix}
        &{}= \begin{bmatrix}
            \left(X^TX\right)^{\minus1} X^T\Vec{y_1} &\left(X^TX\right)^{\minus1} X^T\Vec{y_2} &\dots &\left(X^TX\right)^{\minus1} X^T\Vec{y_p}
        \end{bmatrix}\\
        &{}= \left(X^TX\right)^{\minus1} X^T
        \begin{bmatrix}
            \Vec{y_1} &\Vec{y_2} &\dots &\Vec{y_p}
        \end{bmatrix}\\
        &{}= \left(X^TX\right)^{\minus1} X^TY\\
        &{}= \Theta
    \end{align*}
    This result implies that evaluating the parameters separately yields the same result as evaluating them together.
\end{enumerate}

%%%%%%%%%%%%%%%%%
%   Problem 4   %
%%%%%%%%%%%%%%%%%
\section{Problem 4}
\textbf{Naive Bayes\\\\}
In this problem, we look at maximum likelihood parameter estimation using the naive Bayes assumption. Here, the input features $x_j, j = 1,\dots, n$ to our model are discrete, binary-valued variables, so $x_j \in \{0, 1\}.$ We call $x = [x_1 x_2 \dots x_n]^T$ to be the input vector. For each training example, our output targets are a single binary-value $y \in \{0, 1\}$. Our model is then parameterized by $\phi_{j|y=0} = p(x_j = 1|y = 0)$, $\phi_{j|y=1} = p(x_j = 1|y = 1)$, and
$\phi_y = p(y = 1)$. We model the joint distribution of (x, y) according to
\begin{align*}
    p(y) \quad=&{}\quad (\phi_y)^y(1-\phi_y)^{1-y}\\
    p(x|y=0) \quad=&{}\quad \prod_{j=1}^n p(x_j|y=0)\\
    =&{}\quad \prod_{j=1}^n(\phi_{j|y=0})^{x_j}(1-\phi_{j|y=0})^{1-x_j}\\
    p(x|y=1) \quad=&{}\quad \prod_{j=1}^n p(x_j|y=1)\\
    =&{}\quad \prod_{j=1}^n(\phi_{j|y=1})^{x_j}(1-\phi_{j|y=1})^{1-x_j}\\
\end{align*}
\begin{enumerate}[label=(\alph*)]
    \item Find the joint likelihood function $\ell(\varphi) = \log\prod_{i=1}^mp(x^{\supi{i}}, y^{\supi{i}};\varphi)$ in terms of the model parameters given above. Here, $\varphi$ represents the entire set of parameters $\{\phi_y, \phi_{j|y=0}, \phi_{j|y=1} | j = 1,\dots,n\}$.\\\\
    \textit{answer:}
    \begin{align*}
        \ell(\varphi) =&{} \log\prod_{i=1}^mp(x^{\supi{i}}, y^{\supi{i}};\varphi)\\
        =&{}\log\prod_{i=1}^m\left(\prod_{j=1}^{n_i}p(x_j^{\supi{i}}|y^{\supi{i}};\varphi)\right)p(y^{\supi{i}};\varphi)\\
        =&{}\sum_{i=1}^m\left(\log p(y^{\supi{i}};\varphi) + \log\prod_{j=1}^{n_i}p(x_j^{\supi{i}}|y^{\supi{i}};\varphi)\right)\\
         =&{}\sum_{i=1}^m\left(\log p(y^{\supi{i}};\varphi) + \sum_{j=1}^{n_i}\log p(x_j^{\supi{i}}|y^{\supi{i}};\varphi)\right)\\
         =&{}\sum_{i=1}^m\left(\log\left((\phi_y)^{y^{\supi{i}}}(1-\phi_y)^{1-y^{\supi{i}}}\right) + \sum_{j=1}^{n_i}\log\left( (\phi_{j|y})^{x_j}(1-\phi_{j|y})^{1-x_j}\right)\right)\\
         =&{}\sum_{i=1}^m\left(y^{\supi{i}}\log(\phi_y) + (1-y^{\supi{i}})\log(1-\phi_y) + \sum_{j=1}^{n_i}x_j\log (\phi_{j|y})+(1-x_j)\log(1-\phi_{j|y})\right)
    \end{align*}
    \item Show that the parameters which maximize the likelihood function are the same as those given in the lecture notes.\\\\
    \textit{answer:}\\\\
    To find the maximum $\phi_y$ that maximizes the likelihood function, we will set the gradient with respect to $\phi_y$ of the result above to zero.
    \begin{align*}
       &{}\nabla_{\phi_y} \sum_{i=1}^m\left(y^{\supi{i}}\log(\phi_y) + (1-y^{\supi{i}})\log(1-\phi_y) + \sum_{j=1}^{n_i}x_j\log (\phi_{j|y})+(1-x_j)\log(1-\phi_{j|y})\right) \overset{set}= 0\\
       =&{}\quad \nabla_{\phi_y} \sum_{i=1}^my^{\supi{i}}\log(\phi_y) + (1-y^{\supi{i}})\log(1-\phi_y)\\
       =&{}\quad \sum_{i=1}^m\frac{y^{\supi{i}}}{\phi_y} - \frac{1-y^{\supi{i}}}{1-\phi_y}\\
       =&{}\quad\sum_{i=1}^my^{\supi{i}}(1-\phi_y) - \phi_y(1-y^{\supi{i}})\\
       =&{}\quad\sum_{i=1}^my^{\supi{i}} - \phi_y = 0\\
       \implies&{}\quad\sum_{i=1}^my^{\supi{i}} = \sum_{i=1}^m\phi_y\\
       \implies&{}\quad\sum_{i=1}^my^{\supi{i}} = m\phi_y\\
       \implies&{}\quad\phi_y = \frac{\sum_{i=1}^m\mathbbm{1}\{y^{\supi{i}} = 1\}}{m}
    \end{align*}
    Similarly for $\phi_{j|y}$ we can do the same thing and solve
    \begin{align*}
       &{}\nabla_{\phi_{j|y}} \sum_{i=1}^m\left(y^{\supi{i}}\log(\phi_y) + (1-y^{\supi{i}})\log(1-\phi_y) + \sum_{j=1}^{n_i}x_j\log (\phi_{j|y})+(1-x_j)\log(1-\phi_{j|y})\right) \overset{set}= 0\\
       =&{}\quad \sum_{i=1}^m\frac{x_j^{\supi{i}}}{\phi_{j|y}} - \frac{1-x_j^{\supi{i}}}{1-\phi_{j|y}} = 0\\
       =&{}\quad \sum_{i=1}^m x_j^{\supi{i}}(1-\phi_{j|y}) - \phi_{j|y}(1-x_j^{\supi{i}}) = 0\\
       =&{}\quad \sum_{i=1}^m x_j^{\supi{i}} - \phi_{j|y} = 0
    \end{align*}
    We can now separate this into two cases.\\\\
    $\phi_{j|y=0}$:
    \begin{align*}
        &{}\sum_{i=1}^m (x_j^{\supi{i}} - \phi_{j|y=0})\mathbbm{1}\{y^{\supi{i}} = 0\} = 0\\
        \implies&{}\sum_{i=1}^m x_j^{\supi{i}}\mathbbm{1}\{y^{\supi{i}} = 0\} = \sum_{i=1}^m \phi_{j|y=0}\mathbbm{1}\{y^{\supi{i}} = 0\}\\
        \implies&{}\phi_{j|y=0} =  \frac{\sum_{i=1}^m x_j^{\supi{i}}\mathbbm{1}\{y^{\supi{i}} = 0\}}{ \sum_{i=1}^m\mathbbm{1}\{y^{\supi{i}} = 0\}}\\
        \implies&{}\phi_{j|y=0} =  \frac{\sum_{i=1}^m\mathbbm{1}\{x_j^{\supi{i}} = 0 \wedge y^{\supi{i}} = 0\}}{ \sum_{i=1}^m\mathbbm{1}\{y^{\supi{i}} = 0\}}
    \end{align*}
    $\phi_{j|y=1}$:
    \begin{align*}
        &{}\sum_{i=1}^m (x_j^{\supi{i}} - \phi_{j|y=1})\mathbbm{1}\{y^{\supi{i}} = 1\} = 0\\
        \implies&{}\sum_{i=1}^m x_j^{\supi{i}}\mathbbm{1}\{y^{\supi{i}} = 1\} = \sum_{i=1}^m \phi_{j|y=1}\mathbbm{1}\{y^{\supi{i}} = 1\}\\
        \implies&{}\phi_{j|y=1} =  \frac{\sum_{i=1}^m x_j^{\supi{i}}\mathbbm{1}\{y^{\supi{i}} = 1\}}{ \sum_{i=1}^m\mathbbm{1}\{y^{\supi{i}} = 1\}}\\
        \implies&{}\phi_{j|y=1} =  \frac{\sum_{i=1}^m\mathbbm{1}\{x_j^{\supi{i}} = 1 \wedge y^{\supi{i}} = 1\}}{ \sum_{i=1}^m\mathbbm{1}\{y^{\supi{i}} = 1\}}
    \end{align*}

    %commenting out this question
    \iffalse
    \item Consider making a prediction on some new data point $x$ using the most likely class estimate generated by the naive Bayes algorithm. Show that the hypothesis returned by Naive Bayes is a linear classifier—i.e., if $p(y = 0|x)$ and $p(y = 1|x)$ are the class probabilities returned by naive Bayes, show that there exists some $\theta\inR^{n+1}$ such that
    \begin{align*}
        p(y = 1|x) \geq p(y = 0|x) \iff \theta^{T} \begin{bmatrix}
            1\\
            x
        \end{bmatrix} \geq 0
    \end{align*}
    \fi

\end{enumerate}
\section{Problem 5}
\textbf{Exponential family and the geometric distribution}\\
\begin{enumerate}[label=(\alph*)]
    \item Consider the geometric distribution parameterized by $\phi$:\begin{align*}
        p(y;\phi)=(1-\phi)^{y-1}\phi, \quad j=1,2,3\dots
    \end{align*}Show that the geometric distribution is in the exponential family, and give $b(y)$, $\eta$, $T(y)$ and $a(\eta)$.\\\\
    \textit{answer:}\\\\
    Recall that to be a member of the exponential family distribution, a distribution's PDF must be in the form $p(y;\phi ) = b(y)\exp[T(y)\cdot \eta-a(\eta)]$. \\\\Here we have
    \begin{align*}
        p(y;\phi)=&(1-\phi)^{y-1}\phi\\
        =&\exp[\log(1-\phi)^{y-1} + \log(\phi)]\\
        =&\exp[(y-1)\log(1-\phi) + \log(\phi)]\\
        =&\exp[y\log (1-\phi)-\log(1-\phi)+\log(\phi)]
    \end{align*}Which can be decomposed as\begin{align*}
        &b(y)=1\\
        &\eta = \log (1-\phi)\\
        &T(y) = y\\
        &a(\eta)=-\eta +\log (1-e^\eta)
    \end{align*}
    \item Consider performing regression using a GLM model with a geometric response variable. What is the canonical response function for the family? You may use the fact that the mean of a geometric distribution is given by $1/\phi$.\\\\
    \textit{answer}:\begin{align*}
        g(\eta) = E[y;\phi ] = \frac{1}{\phi } = \frac{1}{1-e^\eta}
    \end{align*}
    \item For a training set $\{(x^{(i)}, y^{(i)}); i=1,\dots m\}$, let the log-likelihood of an example be $\log p(y^{(i)}|x^{(i)};\theta)$. By taking the derivative of the log-likelihood with respect to, derive the stochastic gradient ascent rule for learning using a GLM model with geometric responses y and the canonical response function.\\\\
    \textit{answer:}\\\\
    The log-likelihood of $\theta$ with respect to a training example $(x^{(i)}, y^{(i)})$ is defined as $l_i(\theta) = \log p(y\supi{i} | x\supi{i}; \theta)$. We use the GLM assumption that $\eta = \theta^Tx$. Therefore, we obtain \begin{align*}
        l_i(\theta ) =& \log [\exp(\theta^Tx\supi{i}\cdot y\supi{i}-\theta^Tx\supi{i} +\log(1-e^{\theta^Tx\supi{i}}))]\\
        =&\log [\exp(\theta^Tx\supi{i}\cdot y\supi{i}-\log(e^{\theta^Tx\supi{i}}) +\log(1-e^{\theta^Tx\supi{i}}))]\\
        =&\log \left[\exp\left(\theta^Tx\supi{i}\cdot y\supi{i}-\log\left(\frac{e^{\theta^Tx\supi{i}}}{1-e^{\theta^Tx\supi{i}}}\right)\right) \right]\\
        =&\log \left[\exp\left(\theta^Tx\supi{i}\cdot y\supi{i}-\log\left(\frac{1}{e^{-\theta^Tx\supi{i}}-1}\right)\right) \right]\\
        =&\theta^Tx\supi{i}\cdot y\supi{i} +\log \left(e^{-\theta^Tx\supi{i}}-1\right)
    \end{align*}Then we want to take the gradient respect to $\theta_j$\begin{align*}
        \nabla_{\theta_j} =& x\supi{i}_j\cdot y\supi{i}+\frac{e^{-\theta^Tx\supi{i}}}{e^{-\theta^Tx\supi{i}}-1}(-x\supi{i}_j)\\
        =& x\supi{i}_j \left(y\supi{i}-\frac{e^{-\theta^Tx\supi{i}}}{e^{-\theta^Tx\supi{i}}-1}\right)\\
        =&x\supi{i}_j \left(y\supi{i}-\frac{1}{1-e^{\theta^Tx\supi{i}}}\right)
    \end{align*}Finally, we can derive the stochastic gradient ascent update rule \begin{align*}
        \theta_j \coloneqq \theta_j + \alpha\left(x\supi{i}_j \left(y\supi{i}-\frac{1}{1-e^{\theta^Tx\supi{i}}}\right)\right)
    \end{align*}
\end{enumerate}
\end{document}

Supervised Learning/Problem Set 1 - Supervised Learning.tex


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Copyright (c) 2003-2018 by The University of Queensland
% http://www.uq.edu.au
%
% Primary Business: Queensland, Australia
% Licensed under the Apache License, version 2.0
% http://www.apache.org/licenses/LICENSE-2.0
%
% Development until 2012 by Earth Systems Science Computational Center (ESSCC)
% Development 2012-2013 by School of Earth Sciences
% Development from 2014 by Centre for Geoscience Computing (GeoComp)
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Some Remarks on Lumping}
\label{LUMPING}

Explicit time integration schemes (two examples are discussed later in this
section), require very small time steps in order to maintain numerical stability.
Unfortunately, these small time increments often result in a prohibitive
computational cost.
In order to minimise these costs, a technique termed lumping can be utilised.
Lumping is applied to the coefficient matrix, reducing it to a simple diagonal
matrix. This can significantly improve the computational speed, because the
solution updates are simplified to a simple component-by-component
vector-vector product. However, some care is required when making radical
approximations such as these. In this section, two commonly applied lumping
techniques are discussed, namely row sum lumping
\index{linear solver!row sum lumping}\index{row sum lumping}and HRZ
lumping\index{linear solver!HRZ lumping}\index{HRZ lumping}.

\subsection{Scalar wave equation}
One example where lumping can be applied to a hyperbolic problem, is
the scalar wave equation
\begin{eqnarray} \label{LUMPING WAVE}
u_{,tt}=c^2 u_{,ii} \; .
\end{eqnarray}
In this example, both of the lumping schemes are tested against the reference solution
\begin{eqnarray} \label{LUMPING WAVE TEST}
u=sin(5 \pi (x_0-c*t) )
\end{eqnarray}
over the 2D unit square. Note that $u_{,i}n_i=0$ on faces $x_1=0$ and $x_1=1$.
Thus, on the faces $x_0=0$ and $x_0=1$ the solution is constrained.

To solve this problem the explicit Verlet scheme\index{Verlet scheme} was used
with a constant time step size $dt$ given by
\begin{eqnarray} \label{LUMPING WAVE VALET}
u^{(n)}=2u^{(n-1)}-u^{(n-2)} + dt^2 a^{(n)}
\end{eqnarray}
for all $n=2,3,\ldots$ where the upper index ${(n)}$ refers to values at
time $t^{(n)}=t^{(n-1)}+h$ and $a^{(n)}$ is the solution of
\begin{eqnarray} \label{LUMPING WAVE VALET 2}
a^{(n)}=c^2 u^{(n-1)}_{,ii} \; .
\end{eqnarray}
This equation can be interpreted as a PDE for the unknown value $a^{(n)}$,
which must be solved at each time-step.
In the notation of equation~\ref{LINEARPDE.SINGLE.1} we thus set $D=1$ and
$X=-c^2 u^{(n-1)}_{,i}$. Furthermore, in order to maintain stability,
the time step size needs to fulfill the Courant-Friedrichs-Lewy condition
(CFL condition).
\index{Courant condition}
\index{explicit scheme!Courant condition}
For this example, the CFL condition takes the form
\begin{eqnarray} \label{LUMPING WAVE CFL}
dt = f \cdot \frac{dx}{c} .
\end{eqnarray}
where $dx$ is the mesh size and $f$ is a safety factor. In this example,
we use $f=\frac{1}{6}$.

Figure~\ref{FIG LUMPING VALET A} depicts a temporal comparison between four
alternative solution algorithms: the exact solution; using a full mass matrix;
using HRZ lumping; and row sum lumping. The domain utilised rectangular order 1
elements (element size is $0.01$) with observations taken at the point
$(\frac{1}{2},\frac{1}{2})$.
All four solutions appear to be identical for this example. This is not the case
for order $2$ elements, as illustrated in Figure~\ref{FIG LUMPING VALET B}.
For the order $2$ elements, the row sum lumping has become unstable. Row sum
lumping is unstable in this case because for order $2$ elements, a row sum can
result in a value of zero. HRZ lumping does not display the same problems, but
rather exhibits behaviour similar to the full mass matrix solution. When using
both the HRZ lumping and full mass matrix, the wave-front is slightly delayed
when compared with the analytical solution.

\begin{figure}[ht]
\centerline{\includegraphics[width=7cm]{lumping_valet_a_1}}
\caption{Amplitude at point $(\frac{1}{2},\frac{1}{2})$ using the acceleration formulation~\ref{LUMPING WAVE VALET 2} of the
Velet scheme with order $1$ elements, element size $dx=0.01$, and $c=1$.}
\label{FIG LUMPING VALET A}
\end{figure}

\begin{figure}[ht]
\centerline{\includegraphics[width=7cm]{lumping_valet_a_2}}
\caption{Amplitude at point $(\frac{1}{2},\frac{1}{2})$ using the acceleration formulation~\ref{LUMPING WAVE VALET 2} of the
Velet scheme with order $2$ elements, element size $0.01$, and $c=1$.}
\label{FIG LUMPING VALET B}
\end{figure}

\begin{figure}[ht]
\centerline{\includegraphics[width=7cm]{lumping_valet_u_1}}
\caption{Amplitude at point $(\frac{1}{2},\frac{1}{2})$ using the displacement formulation~\ref{LUMPING WAVE VALET 3} of the
Velet scheme with order $1$ elements, element size $0.01$ and $c=1$.}
\label{FIG LUMPING VALET C}
\end{figure}

Alternatively, one can directly solve for $u^{(n)}$ by inserting
equation~\ref{LUMPING WAVE VALET} into equation~\ref{LUMPING WAVE VALET 2}:
\begin{eqnarray} \label{LUMPING WAVE VALET 3}
u^{(n)}=2u^{(n-1)}-u^{(n-2)} + (dt\cdot c)^2 u^{(n-1)}_{,ii} \; .
\end{eqnarray}
This can also be interpreted as a PDE that must be solved at each time-step, but
for the unknown $u^{(n)}$.
As per equation~\ref{LINEARPDE.SINGLE.1} we set the general form coefficients to:
$D=1$; $Y=2u^{(n-1)}-u^{(n-2)}$; and $X=-(h\cdot c)^2 u^{(n-1)}_{,i}$.
For the full mass matrix, the acceleration ~\ref{LUMPING WAVE VALET 2} and displacement formulations ~\ref{LUMPING WAVE VALET 3}
are identical.

The displacement solution is depicted in Figure~\ref{FIG LUMPING VALET C}. The
domain utilised order $1$ elements (for order $2$, both
lumping methods are unstable). The solutions for the exact and the full mass
matrix approximation are almost identical while the lumping solutions, whilst
identical to each other, exhibit a considerably faster wave-front propagation
and a decaying amplitude.

\subsection{Advection equation}
Consider now, a second example that demonstrates the advection equation
\begin{eqnarray} \label{LUMPING ADVECTIVE}
u_{,t}=(v_i u)_i \; .
\end{eqnarray}
where $v_i$ is a given velocity field. To simplify this example, set $v_i=(1,0)$ and
\begin{equation} \label{LUMPING ADVECTIVE TEST}
u(x,t)=
\left\{
   \begin{array}{cl}
   1 &  x_0 < t \\
   0 &  x_0 \ge t
   \end{array}
\right\}.
\end{equation}
The solution scheme implemented, is the two-step Taylor-Galerkin scheme
\index{Taylor-Galerkin scheme}
(which is in this case equivalent to SUPG\index{SUPG}):
the incremental formulation is given as
\begin{eqnarray} \label{LUMPING SUPG 1}
du^{(n-\frac{1}{2})} = \frac{dt}{2} (v_i u^{(n-1)})_i \\
du^{(n)} = dt (v_i (u^{(n-1)}+du^{(n-\frac{1}{2})}) )_i \\
u^{(n)} = u^{(n)} + du^{(n)}
\end{eqnarray}
This can be reformulated to calculate $u^{(n)}$ directly:
\begin{eqnarray} \label{LUMPING SUPG 2}
u^{(n-\frac{1}{2})} = u^{(n-1)} + \frac{dt}{2} (v_i u^{(n-1)})_i \\
u^{(n)} =  u^{(n-1)} + dt (v_i u^{(n-\frac{1}{2})} )_i
\end{eqnarray}
In some cases it may be possible to combine the two equations to calculate
$u^{(n)}$ without the intermediate step. This approach is not discussed, because
 it is inflexible when a greater number of terms (e.g. a diffusion term) are
 added to the right hand side.

The advection problem is thus similar to the wave propagation problem, because
the time step also needs to satisfy the CFL condition
\index{Courant condition}\index{explicit scheme!Courant condition}. For the
advection problem, this takes the form
\begin{eqnarray} \label{LUMPING ADVECTION CFL}
dt = f \cdot \frac{dx}{\|v\|} .
\end{eqnarray}
where $dx$ is the mesh size and $f$ is a safty factor.
For this example, we again use $f=\frac{1}{6}$.

Figures~\ref{FIG LUMPING SUPG INC A} and~\ref{FIG LUMPING SUPG INC B} illustrate
the four incremental formulation solutions: the true solution; the exact mass matrix;
the HRZ lumping; and the row sum lumping. Observe, that for the order $1$ elements
case, there is little deviation from the exact solution before the wave front,
whilst there is a significant degree of osciallation after the wave-front has
passed. For the order $2$ elements example, all of the numerical techniques fail.

\begin{figure}[ht]
\centerline{\includegraphics[width=7cm]{lumping_SUPG_du_1}}
\caption{Amplitude at point $(\frac{1}{2},\frac{1}{2})$ using the incremental formulation~\ref{LUMPING SUPG 1} of the
Taylor-Galerkin scheme with order $1$ elements, element size $dx=0.01$, $v=(1,0)$.}
\label{FIG LUMPING SUPG INC A}
\end{figure}

\begin{figure}[ht]
\centerline{\includegraphics[width=7cm]{lumping_SUPG_du_2}}
\caption{Amplitude at point $(\frac{1}{2},\frac{1}{2})$ using the incremental formulation~\ref{LUMPING SUPG 1} of the
Taylor-Galerkin scheme  with order $2$ elements, element size $0.01$, $v=(1,0)$.}
\label{FIG LUMPING SUPG INC B}
\end{figure}

Figure~\ref{FIG LUMPING SUPG A} depicts the results from the direct formulation
of the advection problem for an order $1$ mesh. Generally, the results have
improved when compared with the incremental formulation. The full mass matrix
still introduces some osciallation both before and after the arrival of the
wave-front at the observation point. The two lumping solutions are identical, and
have introduced additional smoothing to the solution. There are no oscillatory
effects when using lumping for this example. In Figure~\ref{FIG LUMPING SUPG Ab}
the mesh or element size has been reduced from 0.01 to 0.002 units. As predicted
by the CFL condition, this significantly improves the results when lumping is
applied. However, when utilising the full mass matrix, a smaller mesh size will
result in post wave-front oscilations which are higher frequency and slower to
decay.

Figure~\ref{FIG LUMPING SUPG B} illustrates the results when utilising elements
of order $2$. The full mass matrix and HRZ lumping formulations are unable to
correctly model the exact solution. Only the row sum lumping was capable of
producing a smooth and sensical result.

\begin{figure}[ht]
\centerline{\includegraphics[width=7cm]{lumping_SUPG_u_1}}
\caption{Amplitude at point $(\frac{1}{2},\frac{1}{2})$ using the direct formulation~\ref{LUMPING SUPG 2} of the
Taylor-Galerkin scheme using order $1$ elements, element size $dx=0.01$, $v=(1,0)$.}
\label{FIG LUMPING SUPG A}
\end{figure}

\begin{figure}[ht]
\centerline{\includegraphics[width=7cm]{lumping_SUPG_u_1b}}
\caption{Amplitude at point $(\frac{1}{2},\frac{1}{2})$ using the direct formulation~\ref{LUMPING SUPG 2} of the
Taylor-Galerkin scheme using order $1$ elements, element size $dx=0.002$, $v=(1,0)$.}
\label{FIG LUMPING SUPG Ab}
\end{figure}

\begin{figure}[ht]
\centerline{\includegraphics[width=7cm]{lumping_SUPG_u_2}}
\caption{Amplitude at point $(\frac{1}{2},\frac{1}{2})$ using the direct formulation~\ref{LUMPING SUPG 2} of the
Taylor-Galerkin scheme  using order $2$ elements, element size $0.01$, $v=(1,0)$.}
\label{FIG LUMPING SUPG B}
\end{figure}

\subsection{Summary}
The examples in this section have demonstrated the capabilities and limitations
of both HRZ and row sum lumping with comparisons to the exact and full mass
matrix solutions. Wave propagation type problems that utilise lumping, produce
results simular the full mass matrix at significantly
lower computation cost. An acceleration based formulation, with HRZ lumping
should be implemented for such problems, and can be applied to both order $1$ and
 order $2$ elements.

In transport type problems, it is essential that row sum lumping is used to
achieve a smooth solution. Additionally, it is not recommended that second order
elements be used in advection type problems.


\section{Established Vortex models}
Several vortex models are already commonly used to describe the behavior of
axial wake vortices. Among these models are, in order of introduction, the
Rankine, Lamb-Oseen, Burnham-Hallock, and Proctor models \cite{ahmad2014}. The
simplest and oldest model, the Rankine vortex, approximates the azimuthal
velocity of the vortex within the core zone as a rigid body of rotating fluid,
with that component of velocity decreasing as dictated by an inviscid potential
flow model, predicting that the azimuthal velocity decreases with the inverse
of radius beyond the central core boundary \cite{rankine1869}. The azimuthal
velocity profile of a Rankine vortex is given by

\begin{equation}
v_{\theta}(r) = \frac{\Gamma_0}{2 \pi r_{core}} \frac{r}{r_{core}}
	\text{ for } r \leq r_{core}
	\label{eq:rankine1}
\end{equation}

\begin{equation}
v_{\theta}(r) = \frac{\Gamma_0}{2 \pi r}
	\text{ for } r > r_{core}
	\label{eq:rankine2}
\end{equation}

\noindent
where $\Gamma_0$ is the circulation and $r_{core}$ is the core radius,
which is defined as the location of the maximum azimuthal velocity,
$v_{\theta, max}$. The azimuthal velocity $v_{\theta}$, is proportional to
radius $r$, in the core zone, but proportional to $1/r$ in the region outside
the core zone. This shear stress discontinuity makes the model unsuitable in
the vicinity of the core region of the vortex. The model is therefore
unsuitable for the study of unsteady flow or turbulence, and the rigid body
assumption within the core has been shown experimentally to be invalid.

Much later, the Lamb-Oseen vortex model was developed from an
analytical solution to the Navier-Stokes equations \cite{lamb1932}. In this
model, a potential
line vortex with an infinite velocity at the centerline was introduced
instantaneously as a point, then subjected to viscous decay. This is an
unsteady solution for the azimuthal
velocity profile at any time after the introduction of the discontinuity, given
as

\begin{equation}
v_{\theta}(r,t) = \frac{\Gamma_0}{2 \pi r_{core}}[1 -
							exp(-(r / r_{core})^2)]
	\label{eq:lamb1}
\end{equation}

\noindent
where the core radius dilates according to

\begin{equation}
r_{core}(t) = \sqrt{4 \nu t}
	\label{eq:lamb2}
\end{equation}

\noindent
Utilizing the kinematic viscosity, $\nu$. Alternatively, the azimuthal velocity
profile can be expressed in terms of the maximum azimuthal velocity,
$v_{\theta, max}$, as

\begin{equation}
v_{\theta}(r,t) = v_{\theta, max} (1 + \frac{1}{2 \alpha}) \frac{r_{max}}{r}
[1 - exp(- \alpha \frac{r^2}{r_{max}^2})]
\label{eq:lamb3}
\end{equation}

\noindent
where the core radius dilates according to

\begin{equation}
r_{core}(t) = \sqrt{\alpha 4 \nu t}
\label{eq:lamb4}
\end{equation}

\noindent
In the case where $\alpha = 1.25643$ \cite{davenport1996}. Like the Rankine
vortex, a
Lamb-Oseen vortex does not allow for unsteady flow. In addition, the predicted
centerline velocity at $r=0$ is undefined. It has utility in estimating average
velocity profiles, and has been used to initialize large eddy
simulations \cite{hennemann2011}.

A model which is widely used in the study of wake vortices, and demonstrates
great suitability for modeling the velocity profiles of large experimental
vortices is the Burnham-Hallock model \cite{burnham1982}. This
empirically-based model has been independently discovered by many
\cite{burnam2013}, and describes the azimuthal velocity profile as

\begin{equation}
v_{\theta}(r) = \frac{\Gamma_0}{2 \pi r_{core}} \Big(\frac{r^2}{r^2 +
r_{core}^2} \Big)
\label{eq:burnham-hallock}
\end{equation}

\noindent
Similar to the Lamb-Oseen vortex, the Burnham-Hallock model has been widely
used to initialize large eddy simulations and model aircraft response to wake
encounters \cite{ahmad2014}.

\section{Vortex Model with non-equilibrium Pressure}

Non-equilibrium pressure influences on an incompressible, Newtonian fluid can
be introduced through the application of the Hamilton variational principle.
\cite{zuckerwar2006} employed this approach in an attempt to clarify confusion
associated with the volume or bulk viscosity coefficient for simple fluids.
They were concerned primarily with its relevance to high-speed compressible
flows. When Hamilton's principle was subjected to conservation of mass,
conservation of reacting species, and material entropy constraints, the
procedure resulted in
two dissipative coefficients in the Navier Stokes equation; a traditional
volume viscosity term and a term that was proportional to the material time
derivative of the pressure gradient \cite{zuckerwar2009}. The resulting
modified Navier Stokes equation can be expressed as

\begin{equation}
\rho \frac{Dv}{Dt} = - \nabla[P - \eta_p] -
\rho \nabla \Omega +
\nabla[(\eta_v - \frac{2}{3} \mu)\nabla \cdot \pmb{\text{v}})] +
\nabla \times (\mu \nabla \times \pmb{\text{v}}) +
2[\nabla \cdot (\mu \nabla)] \pmb{\text{v}}
\label{eq:modified_ns1}
\end{equation}

\noindent
where $\eta_p$ is the pressure relaxation coefficient, $\eta_v$ is the volume
viscosity, $P$ is pressure, $\rho$ is density of the fluid, and $\mu$ is the
dynamic viscosity. If each of the thermophysical parameters are considered
to be constant, and body forces are
neglected, the resulting conservation of momentum equation simplifies to

\begin{equation}
\rho \frac{Dv}{Dt} = - \nabla P + \eta_p \nabla \frac{DP}{Dt} -
(\eta_v + \frac{4}{3}\nu) \nabla (\nabla \cdot\pmb{\text{v}})
- \mu \nabla \times (\nabla \times \cdot\pmb{\text{v}})
\label{eq:modified_ns2}
\end{equation}

\noindent
Alternatively, Equation \ref{eq:modified_ns2} can be expressed in index
notation as

\begin{equation}
\rho \frac{Dv_i}{Dt} = -\frac{\partial P}{\partial x_i} +
\eta_p \frac{D}{Dt} \Bigg( \frac{\partial P}{\partial x_i} \Bigg) +
\mu \frac{\partial^2 v_i}{\partial x_{k}^2} +
\eta_p \Bigg[\frac{\partial v_k}{\partial x_i} \frac{\partial P}{\partial x_k} -
\frac{\eta_v + (\frac{1}{3}\mu)}{\eta_p} \frac{\partial}{\partial
x_i} \Bigg( \frac{1}{\rho} \frac{D\rho}{Dt} \Bigg) \Bigg]
\label{eq:modified_ns3}
\end{equation}

The term in brackets has been examined and appears to be related to sound
production
in incompressible flows \cite{ash1998}, but in any scenario, it should be
negligibly small
when multiplied by the pressure relaxation coefficient. Thus, Equation
\ref{eq:modified_ns3} can be dramatically simplified \cite{ash2011}.

\begin{equation}
\rho \frac{Dv_i}{Dt} = -\frac{\partial P}{\partial x_i} +
\eta_p \frac{D}{Dt} \Bigg( \frac{\partial P}{\partial x_i} \Bigg) +
\mu \frac{\partial^2 v_i}{\partial x_{k}^2} +
\label{eq:modified_ns4}
\end{equation}

When applied to a steady, incompressible axial vortex with zero axial velocity,
the azimuthal velocity profile is described by

\begin{equation}
v_\theta(r) = \frac{\Gamma_0}{\pi} \frac{\sqrt{2}}{R_\Gamma \sqrt{\nu \eta_p}}
\frac{(r / r_{core})}{(r/r_{core})^2 + 1}
\label{eq:ash_vortex_model}
\end{equation}

\noindent
where $R_\Gamma$ is the circulation based Reynolds number. The maximum
azimuthal velocity occurs where $r = r_{core}$, and can be represented as

\begin{equation}
v_{\theta, max} = \frac{\Gamma_0}{\pi} \frac{1}{R_\Gamma \sqrt{2 \nu
\eta_p}}
\label{eq:ash_vthetamax}
\end{equation}

\noindent
Therefore, in terms of azimuthal velocity, Equation \ref{eq:ash_vortex_model}
becomes

\begin{equation}
v_\theta(r) = 2 v_{\theta, max}\frac{(r / r_{core})}{(r/r_{core})^2 + 1}
\label{eq:ash_vortex_model2}
\end{equation}

Equation \ref{eq:ash_vortex_model2} is the same form as the
empirically based Burnham-Hallock model, but is an exact solution to the
modified Navier Stokes equations. As the pressure relaxation coefficient
tends to zero, the solution becomes a potential vortex similar to the
Lamb-Oseen model. For very large pressure relaxation coefficients, the velocity
distribution becomes similar to that of a rigid body rotation as in the inner
portion of the Rankine model.

\section{Turbulent Structure of Axial Vortices}

Understanding the turbulent structure of an axial vortex is critical to
understanding the mechanisms controlling vortex decay. Axial wake vortices
typically
start with axial velocity deficits at their centers and strong rotational
momentum about their cores. Turbulence is responsible for the observed rapid
decay of the axial velocity deficit in the core region, but its influence on
the relatively slow decay of the rotational velocity profile is  unclear.
Vortices have been shown to quickly absorb large
scale turbulent structures, and break them up into smaller ones
\cite{ragab1994,beninati2005}. This activity
is associated with the exchange of momentum between the vortex core and the
rotating surroundings, strengthening the near rigid body rotation of the core
region and relaminarizing subsequently the inner vortex region
\cite{bandyopadhyay1991}. Organized motions surrounding the core
cause intermittent ejections of fluid with low momentum and high vorticity, and
entrainment and laminarization of highly turbulent fluid into the core
\cite{bandyopadhyay1991}. Direct Numerical Simulation (DNS) has shown
intense turbulent structures in the free stream to impact the longevity of a
vortex formed within it, with large scale motions causing long wave instability
and slight periodic movements of the vortex core (wobbling), and small scale
motions enhancing the diffusion of vorticity
between the core region and the free stream \cite{risso1997}. It is
unsurprising that the turbulent structure of an axial vortex likely depends
upon local atmospheric conditions, and thus the ambient environment can
influence the rate at which a vortex decays. This has a significant impact on
aviation safety \cite{ash1998}.

The vortex model developed by Ash, Zardadkhan
and Zuckerwar comes from an exact solution to the modified Navier Stokes
equations, employing a simple turbulent eddy viscosity assumption
\cite{ash2011}.
Since that model merely changes the ratio of the effective
viscosity to the pressure relaxation coefficient, the resulting axial velocity
profile continues to be identical with the empirical Burnham-Hallock model.
Specific assumptions for the turbulent viscosity model and Reynolds stresses
will be examined in the present study.


\chapter{Periodicity, Reciprocal Lattice, Floquet Modes}
\label{chap:periodicity}

This chapter discusses the direct and reciprocal lattice vectors, and
defines the Floquet modes that can exist in the dielectric regions.

\section{The Direct Lattice}
We consider a structure with discrete translational invariance in two space
dimensions.  The periodicity is characterized by the {\em direct lattice
  vectors} $\s_1$ and $\s_2$, a pair of real vectors satisfying
\begin{equation}
  \s_1 \bdot \z = \s_2 \bdot \z = 0, \quad A \equiv \z \bdot \s_1 \cross \s_2 > 0.
\end{equation}
The structure is invariant to a translation consisting of any integer number of
shifts in the $\s_1$ or $\s_2$ directions.  Such periodicity
is exhibited by idealized models of frequency selective surfaces
(FSSs) and phased arrays, for example.
This periodicity gives rise to the concept of the {direct lattice},
the set of points $\vecrho_{mn} = \x x_{mn} + \y y_{mn}$ satisfying
\begin{equation}
  \vecrho_{mn} = m \s_1 + n \s_2, \quad \text{for $m$ and $n$ any integers.}
\end{equation}
A periodic structure and its direct lattice is shown in
Figure~\ref{fig:direct}.
\begin{figure}[htbp]
  \begin{center}
        \fbox{%
      \psset{unit=0.005in}
      \pspicture*(-1,-10)(660,410)
                                % Paint it on a white, opaque background:
      \psframe*[linecolor=white,fillcolor=white,fillstyle=solid](-1,-10)(660,410)
      \multips(-60.62,-105.0)(60.62,105.0){5}%
      {%
        \multips(-242.48,0)(121.24,0){8}%
        {%
          \pspolygon(73.68,18.0)(108.18,18)(130.77,52.5)%
          (108.18,87.0)(73.68,87.0)(51.096,52.5)%
          %
          \psline[linestyle=dashed,linewidth=0.5pt](0,0)(121.24,0)
          \psline[linestyle=dashed,linewidth=0.5pt](0,0)(60.62,105.0)
          \qdisk(0,0){1.5pt}
        }%
      }
    \rput(181.86,105){%
      \psline[linewidth=1pt]{->}(0,0)(121.24,0)
      \psline[linewidth=1pt]{->}(0,0)(60.62,105.0)
      \rput(90,-13){$\s_1$}
      \rput[l](10,90){$\s_2$}
      }
    \endpspicture
      }
    \caption{A frequency selective surface consisting of a thin metal plate
        with hexagonal perforations, and the associated direct
        lattice. The location selected for the lattice origin is arbitrary.}
    \label{fig:direct}
  \end{center}
\end{figure}

\section{Periodic Boundary Conditions and the Unit Cell}
\label{sec:pbcuc}
We now assume that an electromagnetic excitation of some type is
applied to the structure.  In the case of a FSS, the excitation takes
the form of an incident plane wave.  In the case of a phased array,
the excitation may be an incident plane wave, or perhaps a set of
incoming waveguide modes in each of the excitation ports of the
radiating elements.  Denote the spatial variation of the excitation by
the function $V(\r)$.  We insist that the function $V$ satisfy the
following quasi-periodicity condition:
\begin{equation}
  \label{eq:floquetbc}
  V(\r + m\s_1 + n\s_2) = V(\r) e^{-j(m\psi_1 + n\psi_2)}, \quad
  \text{for any integers $m$ and $n$}
\end{equation}
where $\psi_1$ and $\psi_2$ are given real numbers, which we will
refer to as the ``unit cell incremental phase shifts''.  By the
translational invariance of Maxwell's equations and given the discrete
translational invariance of the structure, it is clear that all
electromagnetic fields, charges, etc., resulting from the given
excitation must also satisfy \eqref{eq:floquetbc}, which we refer to
as the ``Floquet boundary condition.''

Since the fields throughout the structure satisfy
Equation~\eqref{eq:floquetbc}, it suffices to restrict consideration
to a single unit cell $U$, defined\footnote{The definition of a unit
  cell is not unique.  The present definition is most useful for our
  purposes.}
as the set of points $\r$
satisfying
\begin{equation}
  \label{eq:unitcell}
  U = \{\r: \; \r = \xi_1 \s_1 + \xi_2 \s_2 + \z z, \quad 0 \leq \xi_1 , \xi_2
  \leq 1\},
\end{equation}
where $\xi_1$ and $\xi_2$ are the so-called ``normalized area
coordinates,'' each constrained to the interval $[0,1]$.
We seek a set of modes that can propagate in the unit cell, subject to
an appropriate set of boundary conditions to be stated below.  Let
$E(\r)$ be some rectangular component of electric or magnetic field
evaluated at a point $\r = \x x + \y y + \z z = \xi_1 \s_1 + \xi_2
\s_2 + \z z$ within the unit cell.
%Let $f(\xi_1,\xi_2) =
%E(\xi_1\s_1+\xi_2\s_2+\z z) =E(\r)$.
Then the quasi-periodic boundary condition can be expressed as
\begin{subequations}
  \label{eq:cellfloquetbc}
  \begin{align}
    E(\s_1 + \xi_2 \s_2 + \z z) &= E(\xi_2 \s_2 + \z z) e^{-j\psi_1} \\
    E(\xi_1 \s_1 + \s_2 + \z z) &= E(\xi_1 \s_1 + \z z) e^{-j\psi_2}
  \end{align}
\end{subequations}
  which must hold for all $z$ and for all $\xi_1$ and $\xi_2$ in the
  interval $[0,1]$.
\subsection{Mode Potentials}

Following the formalism of Section~5.1 of \cite{coll:91}, for both TE
and TM modes we seek mode potentials $\Psi(\vecrho) = \Psi(x,y)$ that satisfy the
two-dimensional Helmholtz equation
\begin{equation}
  \label{eq:laplace2d}
  \laplace_t \Psi + k_c^2 \Psi = 0
\end{equation}
within the unit cell in addition to the boundary
conditions \eqref{eq:cellfloquetbc}.
To simplify the following derivation, let $f(\xi_1,\xi_2) =
\Psi(\xi_1\s_1 + \xi_2\s_2) = \Psi(x,y)$.
Then the boundary condition \eqref{eq:cellfloquetbc} satisfied by $\Psi$
can be expressed more simply in terms of $f$ as
\begin{subequations}
  \label{eq:cellfloquetbcf}
  \begin{align}
    f(1, \xi_2) &= f(0,\xi_2) e^{-j\psi_1} \\
    f(\xi_1, 1) &= f(\xi_1, 0) e^{-j\psi_2}
  \end{align}
\end{subequations}
Note that $f$ is periodic in $\xi_1$ and $\xi_2$ with unit period
except for the
progressive phase shifts $\psi_1$ and $\psi_2$.  This motivates us to
consider the function $f(\xi_1,\xi_2) e^{j(\xi_1\psi_1 +
  \xi_2\psi_2)}$ which is indeed periodic and can therefore be
expanded in a double Fourier series:
\begin{equation*}
  f(\xi_1,\xi_2) e^{j(\xi_1\psi_1 + \xi_2\psi_2)} =
  \sum_{m=-\infty}^{\infty} \sum_{n=-\infty}^{\infty} \!\!\!
  f_{mn} \, e^{-j(m2\pi\xi_1 + n2\pi\xi_2)}
\end{equation*}
or equivalently
\begin{equation}
  \label{eq:fourier}
  f(\xi_1,\xi_2) =
  \sum_{m=-\infty}^{\infty} \sum_{n=-\infty}^{\infty} \!\!\!
  f_{mn} \, e^{-j[\xi_1(\psi_1+m2\pi) + \xi_2(\psi_2+n2\pi)]}.
\end{equation}
We wish to write Equation~\eqref{eq:fourier} explicitly in terms of
$\vecrho = \x x + \y y$.  Recalling that $\vecrho = \xi_1\s_1 +
\xi_2\s_2$ and writing the relation in matrix form yields
\begin{equation}
  \colvec{x\\y} =
  \begin{bmatrix}
    s_{1x} & s_{2x} \\
    s_{1y} & s_{2y}
  \end{bmatrix}
    \colvec{\xi_1 \\ \xi_2}.
\end{equation}
Inverting, we obtain
\begin{align}
\colvec{\xi_1 \\ \xi_2} &=
  \frac{1}{A}
  \begin{bmatrix}
    s_{2y} & -s_{2x} \\
    -s_{1y} & s_{1x}
  \end{bmatrix}
      \colvec{x\\y} \nonumber \\
      &=
      \frac{1}{A}
      \colvec{s_{2y}x -s_{2x}y \\
        -s_{1y}x + s_{1x}y}  \nonumber \\
      &=
      \frac{1}{A}
      \colvec{\s_2 \cross \z \bdot \vecrho \\
        \z \cross \s_1 \bdot \vecrho}  \nonumber \\
      &=
      \frac{1}{2\pi}
      \colvec{
        \vecbeta_1 \bdot \vecrho \\
        \vecbeta_2 \bdot \vecrho
        }
\end{align}
where
\begin{equation}
  \label{eq:betadef}
  \vecbeta_1 = \frac{2\pi}{A} \s_2\cross\z, \quad
  \vecbeta_2 = \frac{2\pi}{A} \z \cross \s_1,
\end{equation}
are the {\em reciprocal lattice vectors} \cite{dufo:67,kitt:66}
and $A = \z \bdot \s_1 \cross \s_2$ is the area of the unit cell.
Substituting \eqref{eq:betadef} into \eqref{eq:fourier}, we obtain the
desired representation of the mode potential:
\begin{equation}
    f(\xi_1,\xi_2) = \Psi(x,y) =
  \sum_{m=-\infty}^{\infty} \sum_{n=-\infty}^{\infty}
  f_{mn} e^{-j\vecbeta_{mn} \bdot \vecrho}
\end{equation}
where
\begin{subequations}
  \begin{align}
    \vecbeta_{mn} &= \vecbeta_{00} + m\vecbeta_1 + n\vecbeta_2, \\
    \vecbeta_{00} &= \frac{\psi_1}{2\pi}\vecbeta_1 +  \frac{\psi_2}{2\pi}\vecbeta_2.
  \end{align}
\end{subequations}
We see that the mode potentials assume the form of a discrete set of plane waves for
both TE and TM modes.  The cutoff wavenumber $k_c$ of a plane wave
with transverse propagation vector $\vecbeta_{mn}$ is given by
\begin{equation}
  k_c = \beta_{mn} \equiv \sqrt{\vecbeta_{mn} \bdot \vecbeta_{mn}}.
\end{equation}
For a lossless medium a finite number of modes may satisfy $k >
\beta_{mn}$; these are the propagating modes.  The remaining modes,
comprising a denumerably infinite set, are cut-off (or evanescent).
The situation is depicted in Figure~\ref{fig:reciprocal} for the
structure of Figure~\ref{fig:direct}.
\begin{figure}[tbp]
  \begin{center}
        \fbox{%
      \psset{unit=0.006in}
      \pspicture*(20,10)(660,400)
                                % Paint it on a white, opaque background:
      \psframe*[linecolor=white,fillcolor=white,fillstyle=solid](20,10)(660,400)
      \multips(-105,303.1)(105.0,-60.62){8}%
      {%
        \multips(0,-242.48)(0,121.24){8}%
        {%
          \psline[linestyle=dashed,linewidth=0.5pt](0,0)(0,121.24)
          \psline[linestyle=dashed,linewidth=0.5pt](0,0)(105.0,-60.62)
          \psline[linewidth=1pt]{->}(0,0)(35,25)
          \qdisk(0,0){1.5pt}
        }%
      }
    \rput(315,181.86){%
      \psline[linewidth=1pt]{->}(0,0)(105.0,-60.62)
      \psline[linewidth=1pt]{->}(0,0)(0,121.24)
      \rput(75,-65){$\vecbeta_1$}
      \rput(-20,95){$\vecbeta_2$}
      \pscircle[linewidth=0.75pt,linestyle=dashed](0,0){70}
      \rput*(62,25){$\vecbeta_{0,0}$}
      }
    \endpspicture
      }
    \caption[Reciprocal lattice.]{The reciprocal lattice for the
      structure of Figure~\ref{fig:direct}.  Note that this lattice is
        a scaled and rotated (by $90^\circ$) version of the direct
        lattice.
        Modes are located at the tips of the small, offset vectors.
        Propagating modes lie within the dashed circle of radius $k$
        centered on the origin.  The offset vector $\vecbeta_{00}$
        accounts for the effects of the impressed phase shift.
        }
    \label{fig:reciprocal}
  \end{center}
\end{figure}

Following the prescription given in \cite{coll:91}, we may now write
down the explicit forms of the modal fields:

\subsection{TE modes}
\label{sec:temodes}

\subsubsection{Oblique Incidence}
We first assume that $\beta_{mn} \neq 0$, in which case
the modal fields are
\begin{subequations}
  \label{eq:TEmodes}
  \begin{align}
    \Psi_{mn}\TE(\vecrho) &= \frac{c_{mn}\TE}{k\eta\beta_{mn}}
    e^{-j\vecbeta_{mn} \bdot \vecrho} \\
    \gamma_{mn} &= \sqrt{\beta_{mn}^2 - k^2} \qquad \text{(1st quadrant)} \\
    Z_{mn}\TE &= \frac{1}{Y_{mn}\TE} = \frac{jk\eta}{\gamma_{mn}} \\
    (\x\x+\y\y) \bdot \H_{mn}\TE(\r) &= \pm \gamma_{mn} e^{\pm\gamma_{mn}z}
    \,\gradient_t \Psi_{mn}\TE \nonumber \label{eq:htransTE} \\
    &= \mp j \gamma_{mn} \Psi_{mn}\TE e^{\pm\gamma_{mn}z}  \vecbeta_{mn}
    \nonumber \\
    &= \pm c_{mn}\TE Y_{mn}\TE e^{-j\vecbeta_{mn}\bdot\vecrho \pm
      \gamma_{mn} z} \betahat_{mn}
    \\
    \z \bdot \H_{mn}\TE(\r) &= \beta_{mn}^2 \Psi_{mn}\TE
    e^{\pm\gamma_{mn} z} \nonumber  \\
    &= c_{mn}\TE \frac{\beta_{mn}}{k\eta} e^{-j\vecbeta_{mn}\bdot\vecrho \pm
      \gamma_{mn} z} \\
    \E_{mn}\TE(\r) &= \pm Z_{mn}\TE \z \cross \H_{mn}(\r) \nonumber \\
     &= c_{mn}\TE  e^{-j\vecbeta_{mn}\bdot\vecrho \pm \gamma_{mn} z}
     \z\cross\betahat_{mn} \label{eq:etransTE}
  \end{align}
\end{subequations}
where we have used  $  \betahat_{mn} = \vecbeta_{mn} / \beta_{mn}$.

\subsubsection{Normal Incidence}
In the case where $\beta_{mn} = 0$, we use the following convention:
\begin{equation}
  \betahat_{mn} = \x, \label{eq:betahatnormal}
\end{equation}
so that the final formulas in \eqref{eq:TEmodes} remain valid.

\subsection{TM modes}
\label{sec:tmmodes}

\subsubsection{Oblique Incidence}
We first assume that $\beta_{mn} \neq 0$, in which case
\begin{subequations}
  \label{eq:TMmodes}
  \begin{align}
    \Psi_{mn}\TM(\vecrho) &= \frac{\pm j c_{mn}\TM}{\gamma_{mn}\beta_{mn}}
    e^{-j\vecbeta_{mn} \bdot \vecrho} \\
    \gamma_{mn} &= \sqrt{\beta_{mn}^2 - k^2} \qquad \text{(1st quadrant)} \\
    Y_{mn}\TM &= \frac{1}{Z_{mn}\TM} = \frac{jk}{\eta\gamma_{mn}} \\
    (\x\x+\y\y) \bdot \E_{mn}\TM(\r) &= \pm \gamma_{mn} e^{\pm\gamma_{mn}z}
    \,\gradient_t \Psi_{mn}\TM \nonumber \\
    &=
    \mp j \gamma_{mn} \Psi_{mn}\TM e^{\pm\gamma_{mn}z}  \vecbeta_{mn}
    \nonumber \\
    &= c_{mn}\TM e^{-j\vecbeta_{mn}\bdot\vecrho \pm \gamma_{mn} z}
    \betahat_{mn} \\
    %
    \z \bdot \E_{mn}\TM(\r) &=
    \beta_{mn}^2 \Psi_{mn}\TM e^{\pm\gamma_{mn} z} \nonumber \\
    &= \pm j c_{mn}\TM \frac{\beta_{mn}}{\gamma_{mn}}
    e^{-j\vecbeta_{mn}\bdot\vecrho \pm \gamma_{mn} z} \\
    %
    \H_{mn}\TM(\r) &= \mp Y_{mn}\TM \z \cross \E_{mn}\TM(\r) \nonumber \\
     &= \mp c_{mn}\TM Y_{mn}\TM
     e^{-j\vecbeta_{mn}\bdot\vecrho \pm \gamma_{mn} z}
     \z\cross\betahat_{mn}
  \end{align}
\end{subequations}

\subsubsection{Normal Incidence}
In the case where $\beta_{mn} = 0$, we again employ the convention
\eqref{eq:betahatnormal}, so that the final formulas in
\eqref{eq:TMmodes}
remain valid.

\subsection{Mode Normalization}
So far we have not specified values for
the set of mode normalization constants $\{c_{mn}\TE\}$ and
$\{c_{mn}\TM\}$.  These can be specified in any convenient manner.  We
will choose a normalization that allows us to easily interpret the
incident and reflected traveling wave variables in terms of power
transported by the modes.  Consider a source-free slab of the unit cell bounded by
$z = \text{constant}$ planes, filled with homogeneous dielectric
material.  Taking account of the results of Sections~\ref{sec:temodes}
and~\ref{sec:tmmodes}, we see that the transverse components of the
most general electromagnetic field that can exist in
this region can be written as
\begin{subequations}
  \label{eq:modalexpansions}
  \begin{align}
    (\x\x+\y\y) \bdot \E(\r) &=
    \mkern -15mu \sum_{(p,m,n) \in S} \mkern -15mu
    \e_{pmn}(x,y)
    \left(
      a_{pmn} e^{-\gamma_{mn} z} + b_{pmn} e^{\gamma_{mn} z}
    \right), \\
    %
    (\x\x+\y\y) \bdot \H(\r) &=
    \mkern -15mu \sum_{(p,m,n) \in S} \mkern -15mu
    \h_{pmn}(x,y)
    \left(
      a_{pmn} e^{-\gamma_{mn} z} - b_{pmn} e^{\gamma_{mn} z}
    \right),
  \end{align}
\end{subequations}
where the summations are taken over the set of integer triples
$S = \{(p,m,n) \in \{1,2\}
\cross \Integers \cross \Integers\}$, $\Integers$ is the set of
all integers, $p=1$ corresponds to TE modes, and $p=2$ corresponds to
TM modes.  The modal fields $\e_{pmn}$ and $\h_{pmn}$ are given
explicitly by
\begin{subequations}
  \label{eq:modaldefs}
  \begin{align}
    \e_{1mn} &=
      c_{1mn}  \z \cross \betahat_{mn} e^{-j\vecbeta_{mn} \bdot \vecrho}
       \\
    %
    \e_{2mn} &=
      c_{2mn}  \betahat_{mn} e^{-j\vecbeta_{mn} \bdot \vecrho}
      \hphantom{\z \cross \mbox{}}
      \\
    %
    \h_{pmn} &= Y_{pmn} \z \cross \e_{pmn}
  \end{align}
\end{subequations}
where
\begin{subequations}
  \begin{align}
    Y_{1mn} &= Y_{mn}\TE, \quad  c_{1mn} = c_{mn}\TE \\
    Y_{2mn} &= Y_{mn}\TM, \quad  c_{2mn} = c_{mn}\TM.
  \end{align}
\end{subequations}

By virtue of the orthogonality of the Floquet modes (see
Appendix~\ref{app:orthogonal}), the
complex power P traveling down the unit cell in the $z$ direction can
be expressed as a sum of the individual complex powers transported by each mode:
\begin{align}
  P &= \iint_{U'} \E \cross \H^* \bdot \z \; \d A
  \nonumber \\
  &= \sum_{(p,m,n) \in S}
  (a_{pmn} + b_{pmn})(a_{pmn}^* - b_{pmn}^*) \iint_{U'}
  \e_{pmn} \cross \h_{pmn}^* \bdot \z \; \d A \nonumber \\
  %
  &= \sum_{(p,m,n) \in S} \mkern -16 mu
  \left[
    \abs{a_{pmn}}^2 - \abs{b_{pmn}}^2 - 2j\Imag{a_{pmn}b_{pmn}^*}
  \right]
  P_{pmn}
\end{align}
where $U'$ is the restriction of the unit cell to the plane $z=0$,
and
\begin{equation}
  P_{pmn} = \iint_{U'} \mkern -8 mu
  \e_{pmn} \cross \h_{pmn}^* \bdot \z \; \d A
  = \abs{c_{pmn}}^2
  Y_{pmn}^* A
\label{eq:Ppmn}
\end{equation}
is the complex power associated with each unit-strength positive-going
mode.  Its value depends on the choice of mode normalization constant
$c_{pmn}$, which has not yet been specified.  Note that if $P_{pmn}$
is equal to $1$, then the time-average (real) power carried down the
guide in the $+z$ direction is just $\abs{a_{pmn}}^2 -
\abs{b_{pmn}}^2$, consistent
with the usual definition of traveling wave variables \cite{bbse:69}.
Such a normalization is not possible, since in many cases
of practical interest, $P_{\mkern -4mu pmn}$ has zero real part.
Consider the case of a lossless medium
with $\beta_{mn} > k$.  Then $\gamma_{mn}$ is pure real, so that
$Y_{pmn}$ is pure imaginary.  It follows from Equation~\eqref{eq:Ppmn}
that $P_{\mkern -4mu pmn}$ is pure imaginary, since $A$ and $\abs{c_{pmn}}$ are
both pure real.  Therefore, we content ourselves with setting the
magnitude of $P_{\mkern -4mu pmn}$ equal to $P_0 \equiv \text{one watt}$:
\begin{equation}
  P_{pmn} = \frac{Y_{pmn}^*}{\abs{Y_{pmn}}} P_0.
  \label{eq:Pset}
\end{equation}
Substituting \eqref{eq:Pset} into \eqref{eq:Ppmn} determines the value
of the mode normalization constant (up to an arbitrary phase):
\begin{equation}
  \abs{c_{pmn}} = \sqrt{\frac{P_0}{A \abs{Y_{pmn}}}}.
\end{equation}
This choice results in a unitary scattering matrix for propagating
modes in lossless media.  It will be convenient for later work to
choose the phase of $c_{pmn}$ as follows:
\begin{equation}
  c_{pmn} = \sqrt{\frac{P_0}{A Y_{pmn}}}.
  \label{eq:modenormalization}
\end{equation}
where we agree to take that square root of $Y_{pmn}$ having positive
real part.


%
% CMPT 310: Artificial Intelligence - A Course Overview
% Section: Flashcard Questions
%
% Author: Jeffrey Leung
%

\section{Flashcard Questions}
	\label{sec:flashcard-questions}
\begin{easylist}

& What characteristics does a task environment consist of?
	&& Performance measure, Environment, Actuators, Sensors (PEAS)
& What does a problem formulation consist of?
	&& Initial state, goal test, successor function, and cost function

& How is uniform-cost search different from breadth-first search?
	&& Uniform-cost search uses a priority queue to follow the path of least cost
& Which search algorithm is iterative deepening search derived from?
	&& Depth-first search
& How is iterative deepening search unique?
	&& Maximum depth of depth-first search increases until the goal is found

& What is an admissible heuristic?
	&& Function which underestimates the true cost to the goal
& What is a dominant heuristic?
	&& Admissible heuristic which is greater than or equal to another admissible heuristic
& How is greedy/heuristic search different from A* search?
	&& Heuristic search does not consider the distance already travelled
& What is the equation for A* search?
	\end{easylist}
	\begin{align*}
		f(n) & = g(n) + h(n) \\
		\textrm{where }
		& f(n) = \textrm{ estimated total cost of the path through } n \textrm{ to the goal} \\
		& g(n) = \textrm{ cost so far to reach } n \\
		& h(n) = \textrm{ heuristic-estimated cost from } n \textrm{ to the goal}
	\end{align*}
	\begin{easylist}

& Which of $\alpha / \beta$ is the upper bound on the potential minimum node, and which is the lower bound on the potential maximum node?
	&& $\alpha$: Lower bound on the potential value of a maximum node
	&& $\beta$: Upper bound on the potential value of a minimum node

& When does backtracking search backtrack?
	&& When a variable has no possible valid values, given the assignments of the other variables
& What algorithm is DPLL derived from?
	&& Backtracking search
& How does WalkSAT `walk'?
	&& The value of a random variable from a false clause is flipped

& What is a normalization constant?
	&& A value which every probability is divided by, to reduce a probability function to a total probability of 1
& What is the difference between dependence and conditional dependence?
	&& Dependence: Whether knowing a variable has an effect on the probability of another variable (share a parent node in Bayesian network)
	&& Conditional dependence: Given an evidence variable(s), whether knowing a variable has an effect on the probability of another variable (share a child node in Bayesian network)
& What is the formula for the Chain Rule?
	\end{easylist}
		\begin{align*}
		P(a, b) &= P(a) P(b|a) = P(b) P(a|b) \\
		P(x_1, \dotsc, x_n) &= \prod_{i=1}^n P(x_i | x_1, \dotsc, x_{i-1})
		\end{align*}
	\begin{easylist}

& What is the formula for Bayes' Rule?
	\[
		P(B | A) = \frac{P(A | B) \times P(B)}{P(A)}
	\]

& What is rejection sampling?
	&& Sampling each variable multiple times to find approximate probabilities of all variables
& What is Gibbs sampling?
	&& Fixing evidence variables, randomly assigning values to non-evidence variables, sampling a non-evidence variable, and iterating through all non-evidence variables

& What is filtering?
	&& Finding a hidden probability given evidence of all previous probabilities (i.e. finding $x_t$ given $e_{1:t}$)
& What is smoothing?
	&& Finding previous hidden probabilities given evidence of previous and future probabilities (i.e. finding $x_t$ given $e_{1:T}$ where $1 <= t < T$)
& What does the Viterbi algorithm do?
	&& Find the most likely sequence of states ending in xt

& How do you avoid overfitting?
	&& Use relatively less training data

& What is the equation used for entropy?
	&& $H(p_1, p_2, \dotsc, p_n) = - \sum_i p_i \log_2 p_i$
& What is the equation used for reduction in uncertainty?
	&& $H(root) - \sum_{i \in \textrm{children}} \frac{\textrm{samples of } i}{\textrm{samples of root}} H(i)$
& What is the entropy of a 50/50 choice?
	&& 1
& What is the entropy of a 0/100 choice?
	&& 0
& Do you make a choice at a decision tree to increase or decrease entropy?
	&& Decrease entropy (i.e. maximize reduction of entropy)

& What is the equation for a threshold function?
	\end{easylist}
	\[
		a = \begin{dcases}
			-1 & \textrm{if } in < 0 \\
			1 & \textrm{otherwise}
		\end{dcases}
	\]
	\begin{easylist}

& What is the equation for a ReLU function?
	\end{easylist}
	\[
		a = max(0, in)
	\]
	\begin{easylist}

\end{easylist}
\clearpage


\documentstyle[fullpage]{article}

\input{tex_stuff.tex}

\begin{document}
\bibliographystyle{alpha}
\title{{\bf The Omega Calculator and Library}, {\em version 1.1.0}}

\author{
Wayne Kelly, Vadim Maslov, William Pugh, \\
Evan Rosser, Tatiana Shpeisman, Dave Wonnacott
}

\maketitle

\section{Introduction}

This document gives an overview of the Omega library and
describes the Omega Calculator, a text-based interface to
the Omega library. A separate document describes the C++ interface
to the Omega library and we are still working on a third document that describes
some of the algorithms used in implementing the Omega library.

The Omega library
manipulates integer tuple relations and sets, such as
$$\{[i,j] \rightarrow [j,j'] : 1 \leq i < j < j' \leq n\}
{\rm\ and\ } \{ [ i ,j ] : 1 \leq i < j \leq n\}$$
Tuple relations and
sets are described using Presburger
formulas\cite{presburger1,Shostak77,presburger2,presburger2a} a class of logical
formulas which can be built from affine constraints over integer
variables, the logical connectives $\neg$, $\land$ and $\lor$, and the
quantifiers $\forall$ and $\exists$.
The best known upper bound on the performance of an algorithm for
verifying Presburger formulas is $2^{2^{2^{n}}}$ \cite{presburger3},
and we have no
reason to believe that our method provides better worst-case
performance. However,  we have found it
to be reasonably efficient for our applications.

The following relation, which maps 2-tuples to 1-tuples:
$\{[i, j] \rightarrow [i] : 1 \le i,j \le 2\}$
represents the following set of mappings: $\{[1,1]\rightarrow[1],\
[1,2]\rightarrow[1],\ [2,1]\rightarrow[2],\ [2,2]\rightarrow[2]\}$.
In addition to variables in the input and output tuples, the Presburger
formulas may also contain free variables. This allows parameterized relations to be described.
For example, $n$ and $m$ are free in
$\{ [i, j] \rightarrow [i] : 1 \le i \le n \land 1 \le j \le m \}$.
The language allows new relations to be defined in terms of existing relations
by providing a number of relational operators.  The relational operations
provided include intersection, union, composition, inverse, domain, range and complement.
For example, $\{[p] \rightarrow [2p, q]\}$ {\tt compose}
$\{[i,j]\rightarrow[i]\}$ evaluates to
$\{[i,j]\rightarrow[2i,q]\}$.

Relations are simplified before being displayed to the user.  This
involves transforming them into disjunctive normal form and removing
redundant constraints and conjuncts.  During simplification, it may be determined
that the relation contains no points or contains all points, in which
case the simplified constraints will be False or True respectively.
For example, $\{[i]\rightarrow[i]\}$ {\tt intersect}
$\{[i]\rightarrow[i+1]\}$ evaluates to $\{[i]\rightarrow[i'] : {\tt
False}\}$.

The copyright notice and legal fine print for the Omega calculator and library are contained in
the README and omega.h files. Basically, you can do anything you want with
them (other than sue us); if you redistribute it
you must include a copy of our copyright notice and legal fine print.

\section{Omega Calculator invocation, syntax and semantics}

The calculator reads from the file given as the first argument, or
standard input, and prints results on standard output.
You can specify several command line flags. Using {\tt -D}{\em mk},
where $m$ is a character and $k$ is a digit, sets the debugging level
to $k$ for module $m$. Using {\tt -G}$g$ sets the maximum number
of inequalities in a conjunct to $g$. Using {\tt -E}$e$ sets the maximum number
of equalities in a conjunct to $E$. In version 1.1.0, we intend to
change our data structures so that these will not need to be specified.
There is also a limit on the maximum number of variables in a conjunct,
but this cannot be changed at run-time. It is given by
{\tt maxVars} in {\tt oc.h}. We also intend to make this go away.

The input is a
series of statements terminated by semicolons. A \# indicates that
the rest of the line is a comment.
The syntax {\tt <<}{\em filename}{\tt >>} can occur anywhere
(and indicates that the text of the file is to be included here.
The statements can have one of the
forms listed in Figure~\ref{IPIstatements}.  {\em RelExpr} is a short
form of {\em Relational Expression}.

\begin{figure}[tbh]
\begin{tabular}{l|p{4.5in}}
Syntax & Semantics \\\hline

{\tt symbolic} {\em varList} &
  Defines the variable names as symbolic
  constants that can be used in all later
  expressions. \\

{\em var} := {\em RelExpr} &
  Computes the relational expression and binds the
  variable name to the result. \\

{\em RelExpr} &
  Computes and prints the relational expression. 	\\

{\tt codegen} $\ldots$ &
	This is described in Section \ref{section:codegen}.
	\\

{\em RelExpr} {\tt subset} {\em RelExpr} &
  Prints True if the first relation is a subset of the second,
  otherwise prints False. \\
\end{tabular}
\caption{Omega Calculator statements}
\label{IPIstatements}
\end{figure}

Figures \ref{figure:example1} and \ref{figure:example2} show a sample
session with the Omega
Calculator. Lines starting with \# are input to the Omega
calculator, the other lines are output from the calculator.

\input{calculator-ex.tex}

Some relational operations may not preserve
the names of input and output variables.
If this happens, the variables get default names:
{\tt In\_}$n$ for input variables and
{\tt Out\_}$n$ for output variables,
where $n$ is a position of variables in its tuple.
When printing, primes are added to variables to distinguish
between multiple variables with the same name.
In input, primes may also be used to distinguish between multiple variables
with the same name: the primes are stripped before being passed
to the Omega library.

\section{Relations}

\subsection{Building relations}

A relation is an operand of a relational expression.
Its syntax is:
\begin{quote}
{\tt \{ [} {\em InputList} {\tt ] -> [} {\em OutputList} {\tt ] :}
{\em formula} {\tt \}}
\end{quote}

{\em InputList} and {\em OutputList} are lists of tuple elements.
A tuple element can be:

\begin{tabular}{cp{5in}}
{\em var} & The corresponding tuple variable is given this name.
	If a variable with that name is already in scope,
	an equality is added
	forcing this tuple variable to be equal to the value
	of the previous definition.
\\
{\em exp} & The tuple variable is unnamed and forced to be equal
	to the expression.
\\
{\em exp}:{\em exp} & The tuple variable is unnamed and forced to be
	greater than or equal to the first expression and less than
	or equal to the second.
\\
{\em exp}:{\em exp}:{\em int} & The tuple variable is unnamed and forced to be
	greater than or equal to the first expression and less than
	or equal to the second, and the difference between the tuple
variable and the first expression must be an integer multiple of the
	integer.
\\
{\tt *} &The tuple variable is unnamed.
\end{tabular}

The {\em formula} is optional.  If it is omitted, no constraints other than
those introduced by the input and output expressions are imposed upon
the relation's variables.  Otherwise, the formula describes additional
constraints on variables used in the relation.

\subsection{Sets}
In addition to relations, the system can represent {\em sets}.

When a relation is declared with only one tuple, as in:
\begin{quote}
{\tt \{ [} {\em SetList} {\tt ] :} {\em formula} {\tt \}}
\end{quote}
then the relation becomes a set.
The variables that are used to describe a set ({\em SetList})
are called {\em set variables} rather than input or output variables.

\subsection{Presburger formula operations}
As mentioned above, the tuples belonging to the relation are defined
by a {\em Presburger formula}.  This formula is built from constraints
using the operations described in Figure~\ref{FormulaOps}.

\begin{figure}[tbh]
\begin{tabular}{l|lll}
Name & \multicolumn{3}{l}{Notation} \\\hline

And & {\em formula} {\tt \&} {\em formula} & {\em formula} {\tt \&\&} {\em formula} & {\em formula} {\tt and} {\em formula} \\

Or  & {\em formula} {\tt |} {\em formula}  & {\em formula} {\tt ||} {\em formula}   & {\em formula} {\tt or} {\em formula} \\

Not & {\tt !} {\em formula}  &                    & {\tt not} {\em formula}\\

Exists & \multicolumn{3}{l}{{\tt exists} ($v_1,...,v_n$: {\em formula})} \\

Forall & \multicolumn{3}{l}{{\tt forall} ($v_1,...,v_n$: {\em formula})} \\

Parentheses & ({\em formula}) \\
Constraint & {\em constraint} \\
\end{tabular}
\caption{Presburger formula syntax}
\label{FormulaOps}
\end{figure}

\subsection{Constraints and arithmetic and comparison operations}

A Presburger formula is built from constraints using the operations
described in the previous subsection.  In this subsection we describe
the syntax of individual constraints.

A constraint is a series of expression lists, connected with
the arithmetic relational operators {\tt =, !=, >, >=, <, <=}.
An example is {\tt 2i + 3j <= 5k,p <= 3x-z = t}.

When an constraint contains a comma-separated list of expressions, it
indicates that the same constraints should be introduced for each of
the expressions.  The constraint {\tt a,b <= c,d > e,f} translates to
the constraints
$$a \leq c \land a \leq d \land b \leq c \land b \leq d \land
c > e \land c > f \land d > e \land d > f$$

Expressions can be of the forms (where {\em var} is a variable,
{\em integer} is an integer constant, and $e$, $e_1$, and $e_2$ are
expressions):
{\em var}, {\em integer}, $e$, {\em integer} $e$,
$e_1 + e_2$, $e_1 - e_2$,
$e_1 \ast e_2$, $- e$, $(e)$.

An important restriction is that all expressions in
the constraints must be affine functions of the variables.
For example, $2\ast x$ is legal, $x \ast 2$ is legal, but $x \ast x$ is
illegal.

\section{Relational and set operations}

A relational expression is an expression over individual relations.
The relational operations defined in the system are listed
in Figures \ref{RelOps1} and \ref{RelOps2}.
Here $r, r_1, r_2$ are relations and $s, s_1,
s_2$ are sets.

\begin{figure}[tbp]
\begin{tabular}{l|l|p{3.5in}}
Name & Syntax & Explanation \\\hline

Union &
$r = r_1$ {\tt union} $r_2$
& $x \rightarrow y \in  r$ iff $x \rightarrow y \in  r_1$
or $x \rightarrow y \in  r_1$
\\

&
$s = s_1$ {\tt union} $s_2$
& $x \in  s$ iff $x \in  s_1$
or $x \in  s_1$
\\

Intersection &
$r = r_1$ {\tt intersection} $r_2$
& $x \rightarrow y \in  r$ iff $x \rightarrow y \in  r_1$
and $x \rightarrow y \in  r_1$
\\

&
$s = s_1$ {\tt intersection} $s_2$
& $x \in  s$ iff $x \in  s_1$
and $x \in  s_1$
\\

Difference &
$r = r_1 - r_2$
& $x \rightarrow y \in  r$ iff $x \rightarrow y \in  r_1$
and $x \rightarrow y \not\in  r_1$
\\

&
$s = s_1 - s_2$
& $x \in  s$ iff $x \in  s_1$
and $x \not\in  s_1$
\\

Complement &
$r =$ {\tt complement} $r_1$
& $x \rightarrow y \in  r$ iff $x \rightarrow y \not\in  r_1$
\\

&
$s =$ {\tt complement} $s_1$
& $x \in  s$ iff $x \not\in  r_1$
\\

Composition &
$r = r_1$ {\tt compose} $r_2$ &
$x \rightarrow y \in r$ iff $\exists y \st x \rightarrow y \in r_2$
and $ y \rightarrow z \in r_1$
\\

Application &
$s = r_1 (s_2)$ &
$x \in s$ iff $\exists y \st x \rightarrow y \in r_1$
and $ y \in s_2$
\\

&
$r = r_1 (r_2)$ &
Equivalent to $r_1$ {\tt compose} $r_2$ \\

Join &
$ r = r_1 . r_2 $
& Equivalent to $r_2$ {\tt compose} $r_1$\\
& & $x \rightarrow y \in r$ iff $\exists y \st x \rightarrow y \in r_1$
and $ y \rightarrow z \in r_2$
\\

Inverse &
$r =$ {\tt inverse} $r_1$ &
$x \rightarrow y \in r $ iff $y \rightarrow x \in r_1$\\

Domain &
$s =$ {\tt domain} $r_1$
& $x \in s$ iff $\exists y \st x \rightarrow y \in r_1$\\

Range &
$s =$ {\tt range} $r_1$
& $y \in s$ iff $\exists x \st x \rightarrow y \in r_1$\\

Restrict Domain &
$r = r_1 ~\verb+\+~ s_2$
& $x \rightarrow y \in r$ iff $x \rightarrow y \in r_1$ and $x \in s_2$
\\

Restrict Range &
$r = r_1 ~/~ s_2$
& $x \rightarrow y \in r$ iff $x \rightarrow y \in r_1$ and $y \in s_2$
\\

Gist &
$r=$ {\tt gist} $r_1$ {\tt given} $r_2$ &
Computes gist of relation $r_1$ given relation $r_2$. \\

\end{tabular}
\caption{Relational operations, part 1}
\label{RelOps1}
\end{figure}

\begin{figure}[tbp]
\begin{tabular}{l|l|p{3.5in}}
Name & Syntax & Explanation \\\hline

Hull & $r=$ {\tt hull} $r_1$ & Computes an single convex region that
contains all of $r_1$.  Not as tight as the convex hull, but intended
to be fairly fast.  \\

Convex Hull &
$r=$ {\tt ConvexHull} $r_1$ &
Computes the convex hull \cite{Schrijver86,Wilde93} of $r_1$.
May be prohibitively expensive to compute
and/or induce numeric overflow of coefficients (leading
to a core dump)
\\

Affine Hull &
$r=$ {\tt AffineHull} $r_1$ &
Computes the affine hull \cite{Schrijver86,Wilde93} of $r_1$.
May be prohibitively expensive to compute
and/or induce numeric overflow of coefficients (leading
to a core dump)
\\

Linear Hull &
$r=$ {\tt LinearHull} $r_1$ &
Computes the linear hull \cite{Schrijver86,Wilde93} of $r_1$.
May be prohibitively expensive to compute
and/or induce numeric overflow of coefficients (leading
to a core dump)
\\

Conic Hull &
$r=$ {\tt ConicHull} $r_1$ &
Computes the conic or cone hull \cite{Schrijver86,Wilde93} of $r_1$.
May be prohibitively expensive to compute
and/or induce numeric overflow of coefficients (leading
to a core dump)
\\

Farkas Lemma &
$r=$ {\tt farkas} $r_1$ &
Applies Farkas's lemma \cite{Schrijver86,Wilde93} to $r_1$.
In the result, the values of the variables
correspond to legal values for coefficients
of equations implied by the convex hull of $r_1$.
Also happens to represent the convex hull of $r_N$
using a points and rays representation.
See \cite{Wilde93} for a more detailed explaination.
\\

Convex Check &
$r=$ {\tt ConvexCheck} $r_1$ &
Returns the convex hull of $r_1$ if we can easily determine
that it is equal to $r_1$, otherwise returns $r_1$.
\\

Pairwise Convex Check &
$r=$ {\tt PairwiseCheck} $r_1$ &
Checks to see if any two conjuncts in $r_1$ can be replaced
exactly by their convex hull (doing so if possible).
\\

Transitive Closure &
$r = r_1 {\tt +}$ &
Least fixed point of $r^+ \equiv r \cup (r \circ r^+)$
\\

&
$r = r_1 {\tt +}$ {\tt within} $b$ &
Least fixed point of $r^+$ for dependence relation $r$ within iteration space $b$
\\

Conic Closure &
$r = r_1 {\tt @}$ &
Gives a simple dependence relation $r$ such that
any linear schedule for  all of the dependences in
$r$ is non-negative if and only
if it is legal for all of the dependences in $r_1$.
Note that $r_1 {\tt +} \subseteq r_1 {\tt @}$.
We previously refered to this as affine closure \cite{uniform2.4}.
\\

Cross-product &
$r = r_1$ {\tt *} $r_2$ &
$x \rightarrow y \in r $ iff $x \in r_1$ and $y \in r_2$
\\

Create superset &
$r= $ {\tt supersetof} $r_1$ &
$r$ is inexact with lower bound $r_1$
\\

Create subset &
$r= $ {\tt subsetof} $r_1$ &
$r$ is inexact with upper bound $r_1$
\\

Create upper bound &
$r= $ {\tt upper\_bound} $r_1$ &
$r$ is an exact relation and is an upper bound on $r_1$ (all UNKNOWN
constraints in $r_1$ are interpreted as TRUE)
\\

Create lower bound &
$r= $ {\tt lower\_bound} $r_1$ &
$r$ is an exact relation and a lower bound on $r_1$ (all UNKNOWN
constraints in $r_1$ are interpreted as FALSE)
\\

Get an example &
$r= $ {\tt example} $r_1$ &
$r \subseteq r_1$ and all variables in $r$ are single-valued
\\

Symbolic example &
$r= $ {\tt sym\_example} $r_1$ &
$r \subseteq r_1$ and all non-symbolic variables in $r$ are single-valued

\end{tabular}
\caption{Relational operations, part 2}
\label{RelOps2}
\end{figure}

\section{Code Generation}
\label{section:codegen}
The Omega Calculator incorporates an algorithm for generating code for
multiple, overlapping iteration spaces \cite{Uniform4}.  Each
iteration space has an associated statement or block of statements.
The syntax is
\centerline{{\tt codegen} [{\em effort}] $IS_1, IS_2, \ldots ,IS_n$ [ {\tt given} {\em known}] }
Each
iteration space $IS_i$ can be specified either as a set representing
the iteration space, or as an original iteration space and a
transformation, {\em T}:{\em IS}, where {\em IS} is the original
iteration space and {\em T} is a relation defining an affine, 1-1 mapping
to a new iteration space. That is, given a point in the original
iteration space, the mapping specifies the point in the new iteration
space at which to execute that iteration.

The effort value specifies the amount of effort to be used to
eliiminate sources of overhead in the generated code. Sources of
overhead include if statements an {\tt min} and {\tt max} functions
in loop bounds. If not specified, the effort level is 0. The different effort
levels are:
\begin{description}
\item{-2}	Minimal possible effort. Loop bounds may not be finite.
\item{-1}	Forces finite bounds
\item{0}	Forces finite bounds and tries to remove {\tt if}'s
		within most deeply nested loops (at a possible cost
		of code duplication).
\item{1}	removes {\tt if}'s within most deeply nested loops
		and loops one short of being most deeply nested.
\item{2}	$\ldots$ and loops two short of being most deeply nested.
\item{$x$}	$\ldots$ and loops $x$ short of being most deeply nested.
\end{description}

The known information, if specified, is used to simplify the generated
code. The generated code will not be correct if known is not true.
Currently, the known relation needs to be a set with the same arity and the transformed
iteration space.

A discussion of program transformations using this framework is
given in \cite{Uniform2.1,Uniform2.3,Uniform3}.

The following is an example of code generation, given three original
iteration spaces and mappings.  The transformed code requires the
traditional transformations loop distribution and imperfectly nested
triangular loop interchange.   Below, the program information has been
extracted and presented to the Omega Calculator in relation form.

\vspace{0.2in}

{\bf Original code}
\begin{verbatim}
      do 30 i=2,n
10      sum(i) = 0.
        do 20 j=1,i-1
20        sum(i) = sum(i) + a(j,i)*b(j)
30      b(i) = b(i) - sum(i)
\end{verbatim}

{\bf Schedule} (for parallelism)
$$T10:\{\ [\ i\ ] \rightarrow\   [\ 0,i,0,0\ ]\ \}$$
$$T20:\{\ [\ i,j\ ] \rightarrow\ [\ 1,j,0,i\ ]\ \}$$
$$T30:\{\ [\ i\ ] \rightarrow\   [\ 1,i-1,1,0\ ]\ \}$$

{\bf Omega Calculator output:}
\input{calculator-ex2.tex}

\section{Inexact Relations}

The special constraint UNKNOWN represents one or more additional
constraints that are not known to the Omega Library.  Such constraints
can arise from uses of uninterpreted function symbols or transitive
closure (as described below), or when the user explicitly requests it.
Such relations require conservative treatment from the library (e.g.,
subtracting a conjunct containing UNKNOWN from a relation must
return that relation, since the unknown constraints might make the
conjunct unsatisfiable.)

The {\tt upper\_bound} and {\tt lower\_bound} operations can be used to
produce exact relations from inexact relations.  They are produced by
treating UNKNOWN constraints as TRUE or FALSE, respectively.

\section{Presburger Arithmetic with Uninterpreted Function Symbols}
The Omega Calculator allows certain restricted uses of
{\em uninterpreted function symbols} in a Presburger formula.
Functions may be declared in the {\tt symbolic} statement as
\begin{quote}
{\tt symbolic} {\em Function} ({\em Arity})
\end{quote}
where {\em Function} is the function name and
{\em Arity} is its number of arguments.
Functions of arity 0 are symbolic constants.

Functions may only be applied to a prefix of the input or output tuple
of a relation, or a prefix of the tuple of a set.  The function
application may list the names of the argument variables explicitly
({\em not yet supported}), or use the abbreviations {\em Function({\tt
In})}, {\em Function({\tt Out})}, and {\em Function({\tt Set})}, to
describe the application of a function to the appropriate length
prefix of the desired tuple.

Our system relies on the following observation: Consider a formula $F$
that contains references $f(i)$ and $f(j)$, where i and j are free in
$F$.  Let $F'$ be $F$ with $f_{i}$ and $f_{j}$ substituted for $f(i)$
and $f(j)$.  $F$ if satisfiable iff $((i = j) \Rightarrow (f_{i} =
f_{j})) \land F'$ is satisfiable.  For more details, see \cite{Shostak79}.
% Robert
% Shostak's paper ``A Practical Decision Procedure for Arithmetic with
% Function Symbols'' \cit(JACM, April 1979).

Presburger Arithmetic with uninterpreted function symbols is in
general undecidable, so in some circumstances we will have to produce
approximate results (as we do with the transitive closure operation)
\cite{transitiveClosure}.

The following examples show some legal uses of uninterpreted function
symbols in the Omega Calculator:

\input{calculator-ex3.tex}

\section{Reachability}

Consider a graph where each directed edge is specified as a tuple
relation.  Given a tuple set for each node representing starting
states at each node, the library can compute which nodes of the graph are
reachable from those start states, and the values the tuples can take
on.

The syntax is:
\begin{verbatim}
reachable ( [list of nodes] ) { [node:startstates] | node_i->node_j:transition] }
\end{verbatim}

For example,

\begin{verbatim}
reachable (a,b,c)
	  { a->b:{[1]->[2]},
	     b->c:{[2]->[3]},
	     a:{[1]}};
\end{verbatim}

The transitions and start states may be any expression that evaluates
to a relation.

You can also compute a tuple set containing the reachable values a t
given node; for example:

\begin{verbatim}
R := reachable of c in (a,b,c)
	                { a->b:{[1]->[2]},
	                  b->c:{[2]->[3]},
	                  a:{[1]}};
\end{verbatim}

The current implementation is very straightforward and can be very slow.

\section{Current limitations}

The transitive closure operation will not work on a relation with
uninterpreted function symbols of arity $> 0$.  Any operation that
requires the projection of input or output variables (such as
composition) may return inexact results if variables in the argument
list of a function symbol are projected.

\bibliography{para}

\end{document}


% introduction-to-linear-algebra.tex
% xelatex ./introduction-to-linear-algebra.tex
\documentclass[UTF8]{ctexart}

\title{Introduction to Linear Algebra学习笔记}
\author{Andy Yao}
\date{\today}

\begin{document}

\maketitle

\section{Introduction to Vectors}
  \subsection{Vectors and Linear Combinations}
  \subsection{Lengths and Dot Porducts}
  \subsection{Matrices}

\section{Solving Linear Equations}
  \subsection{Vectors and Linear Equations}
  \subsection{The Idea of Elimination}
  \subsection{Elimination Using Matrices}
1. $Ax = x_{1}$ times column 1 + ... + $x_{n}$ times column n. And $(Ax)_{i} = \sum_{j=1}^n a_{ij}x_{j}$.

2. Identity matrix $= I$, elimination matrix = $E_{ij}$ using $l_{ij}$, exchange matrix $= P_{ij}$.

3. Multiplying Ax = b by E21 subtracts a multiple l21 of equation 1 from equation 2. The number -l21 is the (2.1) entry
of the elimination matrix E21.

4. For the augmented matrix [A b], that elimination step gives [E21A E21b].

5. When A multiplies any matrix B, it multiplies each column of B separately.

  \subsection{Rules for Matrix Operations}
  \subsection{Inverse Matrices}
  \subsection{Elimination = Factorization: A = LU}
  \subsection{Transposes and Permutations}

\end{document}