\newpage\section{Sets}

\subsection{Lemmas}

\lem{}{\label{lemma:sets_lemma_1}
    Let $ S $ be a set with $ n $ elements, and let $ F $ be a family of
    subsets of S such that for any pair $ A, B $ in $ F $, $ A \cap B \not=
    \varnothing $. Then $ |F| \leq 2^{n-1} $ .
}

% \begin{enumerate}
%     \iref{lemma:sets_lemma_1_1}{Iran TST 2008 D3P1}{}
% \end{enumerate}

\theo{https://en.wikipedia.org/wiki/Erdos-Ko-Rado_theorem}{Erdos Ko Rado theorem}{
    Suppose that $ A $ is a family of distinct subsets of $\{ 1 , 2 \dots n
    \}$ such that each subset is of size $ r $ and each pair of subsets has a
    nonempty intersection, and suppose that $ n \geq 2r $. Then the number of
    sets in $ A $ is less than or equal to the binomial coefficient \[\binom
    {n-1}{r-1}\]
}

\lem{}{Let $ S $ be a set with $ n $ elements, and let $ F $ be a family of subsets of $ S $ such that for any pair $ A, B $ in $ F $, $ S $ is not contained by $ A \cup B $. Then $ |F| \leq 2^{n-1} $.}\label{lemma:sets_lemma_2}

\lem{Kleitman lemma}{A set family $F$ is said to be downwards closed if the following holds: if $X$ is a set in $F$, then all subsets of $X$ are also sets in $F$. Similarly, $F$ is said to be upwards closed if whenever $X$ is a set in $F$, all sets containing $X$ are also sets in $F$. Let $F_1$ and $F_2$ be downwards closed families of subsets of $S = \{1, 2, ..., n\}$, and let $F_3$ be an upwards closed family of subsets of $S$. Then we have

    \begin{align}
        |F_1 \cap F_2| &\geq \frac{|F_1| \cdot |F_2|}{2^n}\\
        |F_1 \cap F_3| &\leq \frac{|F_1| \cdot |F_3|}{2^n}
\end{align}}\label{lemma:sets_lemma_3_Kleitman}

\lem{}{Let $ S $ be a set with $ n $ elements, and let $ F $ be a family of subsets of $ S $ such that for any pair $ A, B $ in $ F $, $ A \cap B \not= \varnothing $ and $ A \cap B \not= S $. Then $ |F| \leq 2^{n-2} $.}\label{lemma:sets_lemma_4}

\solu{
    Using the sets in \autoref{lemma:sets_lemma_1} and
    \autoref{lemma:sets_lemma_1}, defining upwards and downwards sets like in
    \autoref{lemma:sets_lemma_3_Kleitman}.
}

\theo{https://www.wikiwand.com/en/Sunflower_(mathematics)}
{The Sunflower Lemma}{
    A \textbf{sunflower} with $ k $ petals and a core $ X $ is a family of sets $ S_1,
    S_2,\dots, S_k $ such that $ S_i\cap S_j = X $ for each $ i \neq j $. (The
    reason for the name is that the Venn diagram representation for such a
    family resembles a sunflower.) The sets $ S_i \setminus X $ are known as
    petals and must be nonempty, though $ X $ can be empty. \\

    Show that if $ F $ is a family of sets of cardinality $ s $, and $ |F| >
    s!(k-1)^s $, then $ F $ contains a sunflower with $ k $ petals.
}\label{problem:induction_type1_20}\label{problem:extreme_object_10}

\solu{Applying induction and considering the best case where $ |X|=0 $}

\subsection{Extremal Set Theory}

\begin{myitemize}
    \item \href{http://math.mit.edu/~cb_lee/18.318/lecture8.pdf}{MIT 18.314 Lecture-8}
\end{myitemize}

\theo{}{Mirsky Theorem}{A set $ S $ with a chain of height $ h $ can’t be partitioned into $ t $ anti-chains if $ t < h $. In other words, the minimum number of sets in any anti-chain partition of $ S $ is equal to the maximum height of the chains in $ S $. (And Vice Versa)}\label{theorem:mirsky_theorem}

\theo{}{}{In any poset, the largest cardinality of an antichain is at most the smallest cardinality of a chain-decomposition of that poset.}

\theo{}{Dilworth's Theorem}{Let $ P $ be a poset. Then there exist an antichain $ A $ and a chain decomposition $ \mathcal{C} $ of $ P $ such that $ |A| = |\mathcal{C}| $}

\theo{http://mathworld.wolfram.com/Erdos-SzekeresTheorem.html}{Erdos-Szekeres Theorem}{Any sequence of $ ab+1 $ real numbers contains either a monotonically decreasing subsequence of length $ a+1 $ or a monotonically increasing subsequence of length $ b+1 $. The more useful case is when $ a=b=n $. }

\prob{}{}{E}{Let $ n\ge 1 $ be an integer and let $ X $ be a set of $ n^2+1 $ positive integers such that in any subset of $ X $ with $ n+1 $ elements there exist two elements $ x\neq y $ such that $ x|y $. Prove that there exists a subset $ \{x_1, x_2\dots x_{n+1} \in X $ such that $ x_i|x_{i+1} $ for all $ i=1, 2, \dots n $.}

\den{Sperner Family}{
    A family of sets in which none of the sets is a strict subset of another
    is called a \textbf{Sperner family}, that is, an antichain of sets.
}

\theo{https://www.wikiwand.com/en/Lubell–Yamamoto–Meshalkin_inequality}
{LYM Inequality}{
    Let $U$ be a set of $n$ elements and $\mathcal{S}$ is a Sperner family of
    subsets of $U$. If $a_k$ is the number of $k$ element subsets of
    $\mathcal{S}$, then
    \[\impeq{\sum^{n}_{k=0}\frac{a_k}{{n \choose k}}\le 1}\]
}

\begin{prooof}
    Rewrite the inequality by
    \[\sum^{n}_{k=0} a_k k!(n-k)! \le n!\]
    The left hand side equals to
    \[\sum^{}_{S\in \mathcal{S}} \left|S\right|!
    \left(n-\left|S\right|\right)!\]
    For a $S \in \mathcal{S}$, $\left|S\right|!\left(n-\left|S\right|\right)!$
    is the number of permutations of $[n]$ for which the first
    $\left|S\right|$ terms are the elements of $S$. Since the sets of
    $\mathcal{S}$ doesn't include each other, all of these permutations are
    different. And so if we count them together, we will get at most all the
    permutatios of $[n]$, which gives us our desired result.
\end{prooof}

\theo{https://www.wikiwand.com/en/Sperner's_theorem}
{Sperner's Theorem}{
    Sperner's theorem bounds the number of sets in any Sperner family. For any
    Sperner Family $\mathcal{S}$, if the union of those sets is a set of $n$
    elements, then
    \[\left|\mathcal{S}\right|< {n \choose \left\lfloor n/2 \right\rfloor}\]
    Where equality holds iff $\mathcal{S}$ is consisted of all subsets of
    $[n]$ with $\left\lfloor n/2 \right\rfloor$ or $\left\lceil
    n/2  \right\rceil $ elements.
}

\begin{prooof}
    Let $a_k$ be the number of $k$ element subsets of $\mathcal{S}$. Then
    since ${n\choose \left\lfloor n/2\right\rfloor} \le {n\choose k}$, we have
    \[\frac{a_k}{{n \choose \left\lfloor n/2\right\rfloor}} \le
    \frac{a_k}{{n\choose k}}\]
    And since the sets doesn't contain each other, we can sum them up and by
    \autoref{theorem:LYM Inequality}, we get
    \[\sum^{n}_{k=0} \frac{a_k}{{n\choose \left\lfloor
    n/2\right\rfloor}} \le \sum^{n}_{k=0} \frac{a_k}{{n\choose k}} \le 1\]
\end{prooof}

\newpage
\subsection{Problems}

\prob{https://artofproblemsolving.com/community/c6h148835p841269}{USA TST 2005 P1}{E}{
    Let $ n $ be an integer greater than $ 1 $. For a positive integer
    $ m $ , let $ S_{m}= \{ 1,2,\ldots, mn\} $. Suppose that there exists a $
    2n $ -element set $ T $ such that

    \begin{enumerate}
        \item each element of $ T $ is an $ m $ -element subset of $ S_{m} $
        \item each pair of elements of $ T $ shares at most one common element
        \item each element of $ S_{m} $ is contained in exactly two elements of $ T $
    \end{enumerate}

Determine the maximum possible value of $ m $ in terms of $ n$.
\index[cat]{Sets!Max-set with property!USA TST 2005 P1}
\index[strat]{Double Counting!USA TST 2005 P1}
}
\label{problem:double_counting_5}

\solu{We use double counting to find the ans, after that the rest is easy.}

\prob{https://artofproblemsolving.com/community/c6h206650p1136980}{Iran TST 2008 D3P1}{E}{
    Let $S$ be a set with $n$ elements, and $F$ be a family of subsets of $S$ with
    $ 2^{n-1}$ elements, such that for each $A,B,C\in F$, $A\cap B\cap C$ is not
    empty. Prove that the intersection of all of the elements of $F$ is not empty.

    \index[cat]{Sets!Local!Iran TST 2008 D3P1}
    \index[strat]{Induction!Iran TST 2008 D3P1}
}\label{lemma:sets_lemma_1_1}\label{problem:induction_type1_18}

\solu{Using Induction with \hrf{lemma:sets_lemma_1}{this} lemma.}

\prob{https://artofproblemsolving.com/community/c6t309f6h1538018}{Romanian TST
2016 D1P2}{EM}{
Let $n$ be a positive integer, and let $S_1, S_2,\dots S_n$ be
a collection of finite non-empty sets such that
\[\sum_{1\leq i<j\leq n}{\frac{|S_i \cap S_j|}{|S_i||S_j|}} <1\]

Prove that there exist pairwise distinct elements $x_1, x_2\dots x_n$ such that $x_i$ is a member of $S_i$ for each index $i$.

\index[cat]{Sets!Local!Romanian TST 2016 D1P2}
\index[strat]{Focus!Romanian TST 2016 D1P2}
\index[strat]{Induction!Romanian TST 2016 D1P2}
}
\label{problem:induction_type1_25}\label{problem:forget_and_focus_7}

\solu{The Inductive proof reduces the problem to
    \hrf{problem:induction_type1_26}{American Mathematical Monthly problem
E2309}}

\solu{The other approach is to focus on the given weird condition, and interpolate it to something nice, like probabilistic condition.}

\prob{https://artofproblemsolving.com/community/c6h225275p1252232}{American Mathematical Monthly E2309}{EM}{If $ A_1$, $ A_2$,\dots $ A_n$ are $ n$ nonempty subsets of the set $ \left\{1,2,...,n - 1\right\}$, then prove that

\[\sum_{1\leq i < j\leq n}\frac {\left|A_i\cap A_j\right|}{\left|A_i\right|\cdot\left|A_j\right|}\geq 1\]

\index[cat]{Sets!Subsets of $[N]$!American Mathematical Monthly E2309}
}
\label{problem:induction_type1_26}

\prob{https://artofproblemsolving.com/community/c6h362426p1986928}{CGMO 2010
    P1}{E}{Let $n$ be an integer greater than two, and let $A_1,A_2, \cdots ,
    A_{2n}$ be pairwise distinct subsets of $\{1, 2, \dots n\}$. Determine the maximum
    value of
    \[\sum_{i=1}^{2n} \dfrac{|A_i \cap A_{i+1}|}{|A_i| \cdot |A_{i+1}|}\]
    Where $A_{2n+1}=A_1$ and $|X|$ denote the number of elements in $X.$

    \index[cat]{Sets!Subsets of $[N]$!CGMO 2010 P1}
}

\prob{https://artofproblemsolving.com/community/c6h17340p119108}{ISL 2002 C5}{M}{
    Let $r\geq2$ be a fixed positive integer, and let $F$ be an infinite
    family of sets, each of size $r$, no two of which are disjoint. Prove that
    there exists a set of size $r-1$ that meets each set in $F$.

\href{https://artofproblemsolving.com/community/c6h17340p7934669}{HMMT 2016
Team Round}:
    Fix positive integers $r>s$, and let $\mathcal F$ be an infinite
    family of sets, each of size $r$, no two of which share fewer than $s$
    elements. Prove that there exists a set of size $r-1$ that shares at least $s$
    elements with each set in $F$.

\index[cat]{Sets!Local!ISL 2002 C5}
\index[strat]{Induction!ISL 2002 C5}
\index[strat]{Focus!ISL 2002 C5}
\index[strat]{Adding new stuffs!ISL 2002 C5}

}\label{problem:induction_type1_31}

\solu{[Focus on a set] If we take an arbitrary set, we can say that there
exists infinitely many sets $ \in \mathbb{F} $ which includes a fixed element
from our test set. If we do this argument for $ r-1 $ times, we get a set $ X
$ of $ r-1 $ elements, and an infinte family of sets that contains $ X $
completely. At this point the problem is trivial.}

\solu{[Adding Elements]Since it's tricky to work with one family, why not introduce another family, like the second monk. \hrf{http://artofproblemsolving.com/community/c6h17340p3251745}{This} solution generalizes the problem as such.}

\prob{https://artofproblemsolving.com/community/c6h57290p352698}{ISL 1988 P10}{M}{
    Let $ N = \{1,2 \ldots, n\}, n \geq 2. $

    A collection $ F = \{A_1, \ldots,A_t\} $ of subsets
    $ A_i \subseteq N, $  $ i = 1, \ldots, t, $ is said to be
    \textbf{separating}, if for every pair $ \{x,y\} \subseteq N, $ there is a set $ A_i
    \in F $ so that $ A_i \cap \{x,y\} $ contains just one element.

    $ F $ is said to be \textbf{covering}, if every element of $ N $ is contained
    in at least one set $ A_i \in F. $

    What is the smallest value $ f(n) $ of $ t, $ so there is a set $
    F = \{A_1, \ldots, A_t\} $ which is simultaneously separating and covering.

    \index[cat]{Sets!Subsets of $[N]$!ISL 1988 P10}
    \index[strat]{Bijection!Binary Representation!ISL 1988 P10}
} \label{problem:binary_2}

\solu{[Binary Representation]Using \hrf{binary}{Binary} Representations for the elements as in or not in, we get an easy bijection.}

\prob{https://artofproblemsolving.com/community/c6h530095p3024272}{Iran TST 2013 D1P2}{E}{
    Find the maximum number of subsets from $\left \{ 1,...,n \right \}$ such
    that for any two of them like $A,B$ if $A\subset B$ then $\left | B-A
    \right |\geq 3$. (Here $\left | X \right |$ is the number of elements of
    the set $X$.)
    \index[cat]{Sets!Max-set with property!Iran TST 2013 D1P2}
    \index[strat]{Roots of unity filtering!Iran TST 2013 D1P2}
    \index[strat]{Induction!$n$ is or isn't!Iran TST 2013 D1P2}
}

\solu{By partitioning the maximum set of subsets into groups which contain the number $n$ and which don't and \href{induction}{Induction} on $n$ we can show that the maximum number of subset is
\[\frac{2^{n}-(-1)^{n}}{3}\]. }

\prob{https://artofproblemsolving.com/community/c7h64445p383300}{Putnam 2005 B4}{E}{For positive integers $ m$ and $ n$, let $f\left(m,n\right)$ denote the number of $ n$-tuples $\left(x_1,x_2,\dots,x_n\right)$ of integers such that $\left|x_1\right| + \left|x_2\right| + \cdots + \left|x_n\right|\le m$. Show that $ f\left(m,n\right) = f\left(n,m\right)$.}

\solu{Try to show \href{bijection}{Bisection} between the result and choosing $m$ or $n$ objects from $m+n$ objects or show that the result is $\binom{m+n}{n}$.}

\newpage
\subsection{Hamming Distance}

\begin{myitemize}
    \item \href{https://dgrozev.wordpress.com/2019/11/09/hamming-distance-in-olympiad-problems/} {Blogpost by dgrozev}
\end{myitemize}

\vspace{1em}

\den{Hamming Distance}{
    The Hamming distance between two equal-length strings of symbols is the
    number of positions at which the corresponding symbols are different.
}

Occasionally we might be able to use the idea of Hamming distance to
``partition'' a set into sets of equivalence classes. For example consider the
following problem:

\prob{}
{}{}{
    Suppose we have some real numbers $\left(x_{n}\right), n=1,2, \ldots, N$
    and consider the sums
    \[ \sum_{i=1}^{N} b_{i} x_{i},\quad b_{i} \in\{0,1\}\]
    We want to estimate how many among those sums hit a fixed interval
    $\Delta$ with some length.

    \index[strat]{Hamming distance}
}

\begin{solution}
    Suppose a sum $\sum_{i=1}^{N} b_{i} x_{i}$ is inside $\Delta$. \\

    Then changing any bit $b_{i}, i=1,2, \ldots, N$ we leave $\Delta$ if this
    interval is smaller than the largest $\left|x_{i}\right|, i \in[1 \ldots
    N]$.\\

    So, for any $N$ -tuple of bits $b=\left(b_{1}, b_{2}, \ldots,
    b_{N}\right)$ with corresponding sum in $\Delta$, all the other tuples of
    bits that differ from $b$ in exactly one bit have sums outside $\Delta$.\\

    In fact the set of those bits is the unit sphere around $b$ with respect
    to the Hamming distance. So no other set on this sphere has the sum in
    $\Delta$.\\

    Now, we take the set $B$ of all tuples $b=\left(b_{1}, b_{2}, \ldots,
    b_{N}\right)$ with $\sum_{i=1}^{N} b_{i} x_{i} \in \Delta$. Then for any $b
    \in B$ the unit sphere
    {\color{solC} \[S(b):=\left\{b^{\prime}: b^{\prime} \text{differs from } b
    \text{ in exactly one bit}\right\}\]}
    consists of tuples not in $B$. If it happens for $b \in B$, $S(b)$ are
    disjoint, we can estimate the number of elements in $B$ as
    \[ |B| \leq \frac{2^{N}}{N+1} \]
\end{solution}

\newpage
\prob{https://artofproblemsolving.com/community/c6h1946999p13424041}
{CIIM 2019 P3}{M}{
    Let $\{x_n\}_{n\in{}\mathbb{N}}$ be a sequence of non-zero real numbers.
    For every positive integer $m$, we define:
    \[X_m=\left\{A\subseteq{}\{0,1,...,m-1\}\left|\quad\left|\sum_{a\in{A}}
    x_a\right| >\frac{1}{m}\right.\right\}\]
    Prove that: \[\lim_{n\to\infty} \frac{\vert{}X_n\vert{}}{2^n}=1\]

    \index[strat]{Hamming distance!CIIM 2019 P3}
}

\begin{Remark}
    The idea roughly speaking is as follows.

    {\color{impC}For sufficiently large $m$ we map each $A$ having small sum $\sum_{a\in
    A}x_a$ to a family $F(A)$ of many other sets with big sums.} If those
    families do not intersect, we obtain that the number of $A$'s with small
    sums is a small portion of the all subsets.

    The vague idea is to mark some finite set of distinct elements
    $x_n$ and for a fixed set $A$ we may add or remove any marked element,
    depending if it's in $A$ or not. Thus we obtain another sets, but those
    sets cannot have small sums if the marked elements are appropriately
    taken.
\end{Remark}

\begin{solution}[dgrozev]
    To apply this idea we consider two cases.\\

    \boxed{\textbf{1.}} There are infinitely many distinct numbers among
    $(x_n)_{n\ge 0}$. \\

    Let us fix some $N\in\mathbb{N}$ and $I$ be a set of $N$ non negative
    integers (indices) such $x_i, i\in I$ are all distinct. Denote by
    $\varepsilon$ any positive real number less than $\min \left\{|x_i-x_j|,\
    |x_i|\right\}, i, j\in I$. \\

    We take any $m$ large enough such that $2/m<\varepsilon$ and $m>\max\{i\in
    I\}$. Let $A$ is any subset of $\{0,1,\dots,m\}$ satisfying

    \begin{align*}
        \left|\sum_{a\in A}x_a\right |\leq \frac{1}{m}\qquad\qquad (1)
    \end{align*}
    For any $i\in I$ we construct the set $A_i$ as
    \[A_i:=\begin{cases} A\cup {i}& \text{if } i\not\in A\\
    A\setminus\{i\}&\text{if } i\in A \end{cases} \]
    It can be easily seen
    \[\displaystyle \left|\sum_{a\in A_i}x_a\right |> \frac{1}{m}\,,\,\forall
    i\in I\]

    By $F(A)$ we denote the family $\{A_i:i\in I\}$. For any two subsets $A,B$
    of $\{0,1,\dots,m\}$ satisfying $(1)$, {\color{impC}the corresponding families $F(A),
    F(B)$ do not meet}.\\

    Indeed, if $X\in F(A)$ and $X\in F(B)$, then $X$
    differs from $A, B$ only in one element of $I$. That is, $A$
    differs from $B$ only in two elements that are in $I$. Hence,
    \[\left|\sum_{a\in A}x_a-\sum_{a\in B}x_a\right|>2/m\]
    because of the choice of $m$. It means $A$ and $B$ cannot both satisfy
    $(1)$.\\

    To recap. Let $\mathcal{A}$ be the family of all subsets $A$ of $\left\{0,
    1,\dots m-1\right\}$ satisfying $(1)$. Then for any $A\in\mathcal{A}$ all
    sets in $F(A)$ are not in $\mathcal{A}$. Further, $F(A), A\in \mathcal{A}$
    are disjoint. Since $|F(A)|=N,\forall A\in\mathcal{A}$, we have
    \[|\mathcal{A}|\leq \frac{2^m}{N+1}\]and henceh
    \[\displaystyle \frac{|X_m|}{2^m}\geq \frac{N}{N+1}\]
    Since $N$ could be chosen arbitrary large, the result follows.\\

    \boxed{\textbf{2.}} Only finite values of $(x_n)_{n\ge 0}$ are distinct.\\

    So, suppose $x_n=a$ for infinitely many $n\in \mathbb{N}$. Let $I$ be a
    finite set of such indices with $|I|=N$ and $m\in \mathbb{N}$ satisfies
    $2/m<|a|$ and $m\geq \max\{i:i\in I\}$. For any fixed subset $A\subset
    [0..m]\setminus I$ there exists at most one $k\in\mathbb{N}$ with
    $\left|\sum_{i\in A}x_i+ka\right|\leq \frac{1}{m}$. Hence the number of
    subsets $A'\subset I$ with $\left|\sum_{i\in A\cup A'}x_i+ka\right|\leq
    \frac{1}{m}$ are at most $\dbinom{N}{\lfloor N/2\rfloor}$.\\

    It means the number of sets $A\subset [0..m]$ satisfying $(1)$ are at most
    $2^{m-N}\dbinom{N}{\lfloor N/2\rfloor}$, thus
    \[\displaystyle \frac{|X_m|}{2^m}\geq 1-\frac{\lfloor N/2\rfloor}{2^N}\]
    which proves the desired result.
\end{solution}

\begin{solution}[Sperner's theorem, IMD2]
    We call a subset ``bad'' if it doesn't satisfy the inequality.\\

    Now fix an integer $N$. Let $m$ be an integer such that
    \[\frac{2}{m}< \min \left\{\left|x_1\right|, \left|x_2\right|\dots
    \left|x_N\right|\right\}\]
    WLOG assume that there are as many positives in the first $N$ terms as
    there are negatives. Let $P$ be the positive elements subset of
    $\left\{x_1, x_2\dots x_N\right\}$.\\

    Define $S = \left\{x_1, x_2, \dots x_m\right\} - P$. Then every subset of
    the first $m$ terms can be written as ${A, B}$, where $A\subseteq P,
    B\subseteq S$ and the subset is $A\cup B$. \\

    Using this notation, notice that if $A\subset A' \subseteq P$, then both
    ${A, B}$ and ${A', B}$ can't be bad, because of our choice of $m$. So for
    a fixed $B\subseteq S$, the largest family of subsets of $P$ is a
    \emph{Sperner Family} of $P$. Which by Sperner's theorem has size at most
    \[t_p = {\left|P\right| \choose \left\lfloor\left|P\right|/2\right\rfloor}\]
    So the probability of a subset of $\left\{x_1, x_2,\dots x_m\right\}$
    being bad is $\frac{t_p}{2^p}$, which tends to $0$ for larger choice of
    $N$.
\end{solution}


\documentclass[]{report}

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{graphicx}
\graphicspath{ {images/} }

\title{CSCI 567 HW\#2}
\author{Mohmmad Suhail Ansari\\USC ID: 8518586692}

\begin{document}
\maketitle

\newpage

	\paragraph{Sol. 1.1}

		The negative of log likelihood can be written as:

		\[ \mathcal{L}(\textbf{w}) = - log\bigg( \prod_{i=1}^N P(Y= y_i|\textbf{X}=x_i) \bigg) \]
		\[ \mathcal{L}(\textbf{w}) =
			\left\{
				\begin{array}{ll}
					{y_n = 1}  & - \sum_{i = 0}^{N} log(\sigma(b + \textbf{w}^T \textbf{x}_n))\\
					{y_n = 0}  & - \sum_{i = 0}^{N} log(1 - \sigma(b + \textbf{w}^T \textbf{x}_n))
				\end{array}
			\right.
		\]
		combining the two parts, we get

		\begin{equation}
			\mathcal{L}(\textbf{w}) = - \sum_{i = 0}^{N} y_n [log(\sigma(b + \textbf{w}^T \textbf{x}_n))] + (1- y_{n})[log(1 - \sigma(b + \textbf{w}^T \textbf{x}_n))]
		\end{equation}

	\paragraph{Sol. 1.2}
		First we will transform the eq(1) above by appending $1$ to $\textbf{x}$ and $b$ to $\textbf{w}$ i.e.
		\[ \textbf{x} = [1 \quad x_i \quad x_2 \quad x_3 \quad ... \quad x_D ]\]
		\[ \textbf{w} = [b \quad w_1 \quad w_2 \quad w_3 \quad ... \quad w_D ]\]

		$\therefore$ the eq(1) can be written as

		\begin{equation}
			\mathcal{L}(\textbf{w}) = - \sum_{i = 0}^{N} y_n [log(\sigma(\textbf{w}^T \textbf{x}_n))] + (1- y_{n})[log(1 - \sigma(\textbf{w}^T \textbf{x}_n))]
		\end{equation}

		Now, we know that the derivative of $\sigma(a)$ is given as
		\[ \frac{d\,\sigma(a)}{d\,a} = \frac{1}{1 + e^{-a}} \bigg( 1 - \frac{1}{1 + e^{-a}} \bigg) \]
		\[ = \sigma(a)[1 - \sigma(a)]\]
		similarly we can write the derivative of $log(\sigma(a))$ w.r.t $a$
		\[ \frac{d \, log(\sigma(a))}{d \, \sigma(a)} = 1 - \sigma(a)\]

		Now, using the above definitions and we can write the derivative of the loss function i eq(2) as
		\[ \frac{\partial{\mathcal{L}(\textbf{w})}}{\partial{\textbf{w}}} = - \sum_{i = 0}^{N} y_n ({ 1 - \sigma(\textbf{w}^T \textbf{x}_n)})\textbf{x}_n + (1-y_n){\sigma(\textbf{w}^T \textbf{x}_n)}\textbf{x}_n \]

		\begin{equation}
			\frac{\partial{\mathcal{L}(\textbf{w})}}{\partial{\textbf{w}}}= \sum_{i = 0}^{N} \{ \sigma(\textbf{w}^T \textbf{x}_n) -y_n \}\textbf{x}_n
		\end{equation}

		from eq(3) we get the error as
		\[ e_n = {\sigma(\textbf{w}^T \textbf{x}_n ) - y_n} \]
		and the stationary point as
		\[ \sum_{i = 0}^{N} \sigma(\textbf{w}^T \textbf{x}_n ) \textbf{x}_n =  \sum_{i = 0}^{N} \textbf{x}_n y_n \]

		Now, let $\eta$ be the step size, then we can write the update rule for $\textbf{w}$ as
		\begin{equation}
			\textbf{w}^{(t + 1)}  = \textbf{w}^{(t)} - \eta \sum_{i = 0}^{N} \{ \sigma(\textbf{w}^T \textbf{x}_n) - y_n\}x_n
		\end{equation}

		Yes, it would converge to a global minimum. Since, the curve is linear and the local minimum would be the global minimum itself, which we can reach through gradient descent while following along the curve.

	\paragraph{Sol. 1.3}
		For multi-class classification we are given the posterior probability as
		\begin{equation}
			 P(Y = k | \textbf{X} = \textbf{x}) = \frac{exp(\textbf{w}_k^T \textbf{x})}{ 1 + \sum_{t=1}^{K-1} exp(\textbf{w}_t^T \textbf{x})} \quad for \quad k = 1,\, ...\, {K-1}
		\end{equation}
		\begin{equation}
			P(Y = k | \textbf{X} = \textbf{x}) = \frac{1}{ 1 + \sum_{t=1}^{K-1} exp(\textbf{w}_t^T \textbf{x})} \quad for \quad k = K
		\end{equation}

		since, $\textbf{w}_K = 0$, we can simply write
		\begin{equation}
			P(Y = k | \textbf{X} = \textbf{x}) = \frac{exp(\textbf{w}_k^T \textbf{x})}{1 + \sum_1^{K-1} exp(\textbf{w}_t^T \textbf{x})}
		\end{equation}

		Now, using eq(7) we can write the negative log-likelihood as
		\[ \mathcal{L}(\textbf{w}_1\,...\,\textbf{w}_K) = - \sum_n log(P(y_n | \textbf{x}_n = \textbf{x})) = - \sum_n log( \prod_k [P(y = k | \textbf{x}_n)]) \]
		\[ \mathcal{L}(\textbf{w}_1\,...\,\textbf{w}_K) = - \sum_n \sum_k log\, P(y = k | \textbf{x}_n)) \]

		using, eq(7), we can write the negative log-likelihood function as
		\begin{equation}
			 \mathcal{L}(\textbf{w}_1\,...\,\textbf{w}_K) = - \sum_n \sum_k \big[ \textbf{w}_k^T \textbf{x}_n - log(1 + \sum_{k=1}^{K-1} exp(\textbf{w}_t^T \textbf{x}_n )) \big]
		\end{equation}

	\paragraph{So. 1.4}
		To find the maximum log-likelihood, we use eq(8) and take partial derivative w.r.t $\textbf{w}_i$, we get
		\[ \frac{\partial{\mathcal{L}(\textbf{w}_1\,...\,\textbf{w}_K)}}{\partial{\textbf{w}_i}} = - \sum_n \big[ \textbf{x}_i - \frac{exp(\textbf{w}_i^T \textbf{x}_i)\textbf{x}_i}{ 1 + \sum_{k=1}^{K-1} exp(\textbf{w}_t^T \textbf{x}_n )} \big] \]
		\[ = - \sum_n \big[ 1 - \frac{exp(\textbf{w}_i^T \textbf{x}_i)}{ 1 + \sum_{k=1}^{K-1} exp(\textbf{w}_t^T \textbf{x}_n)} \big] \textbf{x}_i \]
		\[ = - \sum_n \big[ 1 - P(y = i| \textbf{x}_i) \big] \textbf{x}_i \]

		$\therefore$ we can define the error as
		\[ e_i = 1 - P(y = i | \textbf{x}_i )\]

		let $\eta$ be the step function, then we can write the update rule as
		\[ \textbf{w}_{(i + 1)} = \textbf{w}_{(i)} - \eta \sum_n \big[ 1 - P(y = i| \textbf{x}_i) \big] \textbf{x}_i \]

	%% Linear/Gaussian Discriminant

	\paragraph{Sol. 2.1}
		From the definition of $p(x_n, y_n)$ we can write the log-likelihood as
		\[ \mathcal{L}(D) =  \sum_{n=1, y_n = 1}^N log(p_1 \frac{1}{\sqrt{2\pi}\sigma_1} exp(- \frac{(x_n - \mu_1)^2}{2\sigma_1^2}))\] \[ + \sum_{n=1, y_n = 2}^N log(p_2 \frac{1}{\sqrt{2\pi}\sigma_2} exp(- \frac{(x_n - \mu_2)^2}{2\sigma_2^2}))\]

		\[ \mathcal{L}(D) = \sum_{n=1, y_n = 1}^N log(p_1) - log(\sqrt{2\pi}\sigma_1) - \frac{(x_n - \mu_1)^2}{2\sigma_1^2} \]
		\[ + \sum_{n=1, y_n = 2}^N log(p_2) - log(\sqrt{2\pi}\sigma_2) - \frac{(x_n - \mu_2)^2}{2\sigma_2^2} \]

		Now, we know that $p_1 + p_2 = 1$. Let, $N_1$ = no. of samples where $y_n = 1$ and $N_2$ = no. of samples where $y_n = 2$
		we'll first find the estimate of $p_1$ that minimizes $-\mathcal{L}(D)$

		\[ \mathcal{L}(D) = \sum_{n=1, y_n = 1}^N log(p_1) - log(\sqrt{2\pi}\sigma_1) - \frac{(x_n - \mu_1)^2}{2\sigma_1^2} \]
		\[ + \sum_{n=1, y_n = 2}^N log(1 - p_1) - log(\sqrt{2\pi}\sigma_2) - \frac{(x_n - \mu_2)^2}{2\sigma_2^2} \]

		taking partial derivative w.r.t. $p_1$, we get

		\[ \frac{\partial{\mathcal{L}(D)}}{\partial{p_1}} = \sum_{n=1, y_n = 1}^N \frac{1}{p_1} - \sum_{n=1, y_n = 2}^N \frac{1}{1 - p_1} \]
		from definition of $N_1$ and $N_2$, we get
		\[ \frac{\partial{\mathcal{L}(D)}}{\partial{p_1}} = 0 = \frac{N_1}{p_1} + \frac{N_2}{1 -p_1} \]
		\[ \hat{p_1} = \frac{N_1}{N_1 + N_2}\]
		similarly, $\hat{p_2}$ will be
		\[ \hat{p_2} = \frac{N_2}{N_1 + N_2}\]

		To, estimate $\mu_1$, we take the partial derivative of $\mathcal{L}(D)$ w.r.t $\mu_1$, we get
		\[ \frac{\partial{\mathcal{L}(D)}}{\partial{\mu_1}} = \sum_{n=1, y_n = 1}^N (x_n - \mu_1) = 0\]
		\[ \hat{\mu_1} = \frac{1}{N_1} \sum_{n=1, y_n = 1}^N x_n\]
		similarly, estimate of $\mu_2$ will be
		\[ \hat{\mu_2} = \frac{1}{N_2} \sum_{n=1, y_n = 2}^N x_n\]

		To, estimate $\sigma_1$, we take the partial derivative of $\mathcal{L}(D)$ w.r.t $\sigma_1$, we get
		\[ \frac{\partial{\mathcal{L}(D)}}{\partial{\sigma_1}} = \sum_{n=1, y_n = 1}^N - \frac{1}{\sigma_1} + \frac{(x_n - \mu_1)^2}{\sigma_i^3} = 0\]
		which gives us
		\[ \hat{\sigma_1^2} = \frac{\sum_{n=1, y_n = 1}^N (x_n - \mu_1)^2}{N_1} \]
		similarly, estimate of $\sigma_2^2$ will be
		\[ \hat{\sigma_2^2} = \frac{\sum_{n=1, y_n = 2}^N (x_n - \mu_2)^2}{N_2} \]

	\paragraph{Sol. 2.2}
		From Baye's formula we can write that
		\[ P(y = k | x) = \frac{P(x | y = k)\, P(y = k)}{\sum_k P(x | y = k)\, P(y = k)} \]
		Now, since $P(y = 1) + P(y = 2) = 1$, then let $P(y = 1) = \pi$, $\therefore P(y = 2) = 1 - \pi$.
		We'll first find the probability $P(y = 1| x)$

		\[ P(y = 1| x) = \frac{P(x|y = 1)\, P(y = 1)}{ [P(x | y = 1)\, P(y = 1)] + [P(x | y = 2)\, P(y = 2)]} \]
		\[ = \frac{\pi \mathcal{N}(\mu_1, \Sigma)}{\pi \mathcal{N}(\mu_1, \Sigma) + (1 - \pi)\pi \mathcal{N}(\mu_2, \Sigma)} \]
		\[ = \frac{1}{1 + \frac{(1 - \pi) \mathcal{N}(\mu_2, \Sigma)}{\pi \mathcal{N}(\mu_1, \Sigma)}}\]

		Now,
		\[ \frac{(1 - \pi) \mathcal{N}(\mu_2, \Sigma)}{\pi \mathcal{N}(\mu_1, \Sigma)} = exp(log(\frac{(1 - \pi) \mathcal{N}(\mu_2, \Sigma)}{\pi \mathcal{N}(\mu_1, \Sigma)}))\]

		Solving for $log(\frac{(1 - \pi) \mathcal{N}(\mu_2, \Sigma)}{\pi \mathcal{N}(\mu_1, \Sigma)})$
		\[ log(\frac{(1 - \pi) \mathcal{N}(\mu_2, \Sigma)}{\pi \mathcal{N}(\mu_1, \Sigma)}) = \]
		\[ = - log(\frac{\pi}{1 - \pi}) + log(\frac{\mathcal{N}(\mu_2, \Sigma)}{\mathcal{N}(\mu_1, \Sigma)})\]

		\[ = -log(\frac{\pi}{1 - \pi}) -\frac{1}{2}(x- \mu_2)^T \Sigma^{-1}(x-\mu_2) + \frac{1}{2}(x- \mu_1)^T \Sigma^{-1}(x-\mu_1) \]
		\[ = -log(\frac{\pi}{1 - \pi}) - \frac{1}{2}{x^T}{\Sigma^{-1}}{x} + {\mu_2^T}{\Sigma^{-1}}{x} - \frac{1}{2}{\mu_2^T}{\Sigma^{-1}}{\mu_2} + \frac{1}{2}{x^T}{\Sigma^{-1}}{x} - {\mu_1^T}{\Sigma^{-1}}{x} + \frac{1}{2}{\mu_1^T}{\Sigma^{-1}}{\mu_1} \]
		\[ = (\mu_2 - \mu_1)^T \Sigma^{-1} x - \frac{1}{2}{\mu_2^T}{\Sigma^{-1}}{\mu_2} + \frac{1}{2}{\mu_1^T}{\Sigma^{-1}}{\mu_1} - log(\frac{\pi}{1 - \pi})\]

		\[ = - \boldsymbol{\theta}^{T} \textbf{x} + b \]
		where,
		\[ \theta = \Sigma^{-1}(\mu_1 - \mu_2)\]
		and
		\[ b = -\frac{1}{2} \mu_2^T \Sigma^{-1} \mu_2  +\frac{1}{2} \mu_1^T \Sigma^{-1} \mu_1 - log \frac{\pi}{1 - \pi}\]

	\paragraph{Sol. 3.1}
		Data Analysis\\

		\begin{center}
			\includegraphics[scale=0.2]{hist_0}\\
			\includegraphics[scale=0.2]{hist_1}\\
			\includegraphics[scale=0.2]{hist_2}\\
			\includegraphics[scale=0.2]{hist_3}\\
			\includegraphics[scale=0.2]{hist_4}\\
			\includegraphics[scale=0.2]{hist_5}\\
			\includegraphics[scale=0.2]{hist_6}\\
			\includegraphics[scale=0.2]{hist_7}\\
			\includegraphics[scale=0.2]{hist_8}\\
			\includegraphics[scale=0.2]{hist_9}\\
			\includegraphics[scale=0.2]{hist_10}\\
			\includegraphics[scale=0.2]{hist_11}\\
			\includegraphics[scale=0.2]{hist_12}\\
		\end{center}

	\paragraph{Sol. 3.2}
		The after applying Linear Regression to the training set, we get the results

		\begin{center}
			\begin{tabular}{c|c}
				Test Data & MSE \\
				\hline
				test\_data & 28.4644 \\
				training\_data &20.9512 \\
				\hline
			\end{tabular}
		\end{center}

		%\newline

		After applying Ridge regression for different $\lambda$ we get the following results
		\begin{center}
		\begin{tabular}{c|c|c}
			\hline
			   	 $\lambda$ & Test Data    &     MSE \\
			\hline
			     0.01 & test\_data  & 28.4183 \\
			     0.01 & train\_data & 20.9501 \\
			     0.1  & test\_data  & 28.4217 \\
			     0.1  & train\_data & 20.9502 \\
			     1    & test\_data  & 28.4574 \\
			     1    & train\_data & 20.9539 \\
			\hline
		\end{tabular}
		\end{center}

		\begin{center}
		\begin{tabular}{c|c}
		\hline
		   $\lambda$ &     CVE \\
		\hline
		    0.001 & 9.99383 \\
		    0.01  & 9.99273 \\
		    0.02  & 9.9915  \\
		    0.1   & 9.98177 \\
		    0.1   & 9.98177 \\
		    1     & 9.8773  \\
		    2     & 9.77169 \\
		    3     & 9.67664 \\
		    4     & 9.59175 \\
		    5     & 9.51671 \\
		    6     & 9.45125 \\
		    7     & 9.39513 \\
		    8     & 9.34814 \\
		    9     & 9.31007 \\
		   10     & 9.28075 \\
		\hline
		\end{tabular}
		\end{center}

		As we see that the cross-validation error is minimum for $\lambda = 10$, therefore we choose $\lambda = 10$ for out MSE calculation.
		For \textbf{Test Data set we get MSE = 28.98456207} and for \textbf{Training Data set we get MSE = 21.28681485}.

		\paragraph{Sol. 3.3.a}
		First, we will calculate the Pearson's Coefficient for each attribute with target values and we get
		\begin{center}
		\begin{tabular}{c | c}
			\hline
			 Feature   &   abs(r) \\
			\hline
			 CRIM      & 0.387697 \\
			 ZN        & 0.362987 \\
			 INDUS     & 0.483067 \\
			 CHAS      & 0.2036   \\
			 NOX       & 0.42483  \\
			 RM        & 0.690923 \\
			 AGE       & 0.390179 \\
			 DIS       & 0.252421 \\
			 RAD       & 0.385492 \\
			 TAX       & 0.468849 \\
			 PTRATIO   & 0.505271 \\
			 B         & 0.343434 \\
			 LSTAT     & 0.73997  \\
			\hline
			\end{tabular}
		\end{center}

		We see that the attributes with highest Pearson's Coefficients are INDUS, PTRATIO, RM, LSTAT.\\

		Now, applying \textbf{Linear Regression}
	    \begin{center}
	    \begin{tabular}{c | c}
	        \hline
	        Test Data & MSE \\
	        \hline
	        test\_data & 31.4962 \\
	        train\_data & 26.4066
	    \end{tabular}
	    \end{center}

	    Now, applying Ridge Regression for different $\lambda$ we get

	    \begin{center}
	    \begin{tabular}{c|c|c}
			\hline
			   $\lambda$ & Test Data    &     MSE \\
			\hline
			     0.01 & test\_data  & 31.496  \\
			     0.01 & train\_data & 26.4066 \\
			     0.1  & test\_data  & 31.4944 \\
			     0.1  & train\_data & 26.4066 \\
			     1    & test\_data  & 31.4806 \\
			     1    & train\_data & 26.4094 \\
			\hline
		\end{tabular}
	    \end{center}

	    For K-Fold Ridge Regression we get the following values of CVE

	    \begin{center}
	    \begin{tabular}{c | c}
			\hline
			   $\lambda$ &    CVE \\
			\hline
			    0.001 & 8.3003 \\
			    0.01  & 8.2992 \\
			    0.1   & 8.2885 \\
			    1     & 8.1855 \\
			    2     & 8.0791 \\
			    3     & 7.9811 \\
			    4     & 7.8915 \\
			    5     & 7.8099 \\
			    6     & 7.7363 \\
			    7     & 7.67   \\
			    8     & 7.6125 \\
			    9     & 7.562  \\
			   10     & 7.5189 \\
			\hline
			\end{tabular}
	    \end{center}

	  	Choosing $\lambda = 10$, we get the MSE as

		\begin{center}
	    \begin{tabular}{c|c|c}
			\hline
			   $\lambda$ & Test Data    &     MSE \\
			\hline
			     10 & test\_data  & 31.5937  \\
			     10 & train\_data & 26.6775 \\
			\hline
		\end{tabular}
	    \end{center}

	\paragraph{Sol. 3.3.b}
	    From the previous exercise we know that the feature with the highest correlation coefficient with the target is LSTAT.

	    Now, calculating residue and correlation coefficient of the rest of the attributes with the residue we get

	    \begin{center}
	    \begin{tabular}{c|c}
			\hline
			 Feature & abs(r(residue, attr\_vals)  \\
			\hline
			 CRIM    & 0.426588158091 \\
			 ZN      & 0.398133315751 \\
			 INDUS   & 0.537842703653 \\
			 CHAS    & 0.175073939283 \\
			 NOX     & 0.491958859016 \\
			 RM      & 0.702473672384 \\
			 AGE     & 0.469369575525 \\
			 DIS     & 0.331853691887 \\
			 RAD     & 0.436013562838 \\
			 TAX     & 0.514184631153 \\
			 PTRATIO & 0.496710217811 \\
			 B       & 0.374589793897 \\
			\hline
			\end{tabular}
	    \end{center}

	    We get RM as the next feature with highest correlation coefficient. \\

	    After updating the residue, we get the correlation coeff. as

	    \begin{center}
	    \begin{tabular}{c|c}
			\hline
			 Feature & abs(r(residue, attr\_vals)  \\
			\hline
			 CRIM    & 0.129554112358  \\
			 ZN      & 0.0461901567406 \\
			 INDUS   & 0.0561934824677 \\
			 CHAS    & 0.249301273706  \\
			 NOX     & 0.0109129511978 \\
			 AGE     & 0.0159396362859 \\
			 DIS     & 0.134713549718  \\
			 RAD     & 0.087477994304  \\
			 TAX     & 0.13681944445   \\
			 PTRATIO & 0.297604744231  \\
			 B       & 0.156620202695  \\
			\hline
			\end{tabular}
	    \end{center}

	    We get PTRATIO as the next feature with highest correlation coefficient. \\

	    Again,
	    \begin{center}
	    \begin{tabular}{c|c}
			\hline
			 Feature & abs(r(residue, attr\_vals)  \\
			\hline
			 CRIM    & 0.0889293951571  \\
			 ZN      & 0.0323063735902  \\
			 INDUS   & 0.00214795182009 \\
			 CHAS    & 0.2195949638     \\
			 NOX     & 0.0198942311859  \\
			 AGE     & 0.0435891823287  \\
			 DIS     & 0.170389023828   \\
			 RAD     & 0.0210281033187  \\
			 TAX     & 0.0440426778355  \\
			 B       & 0.144133379832   \\
			\hline
		\end{tabular}
	    \end{center}

	  	Finally, we select the feature CHAS.\\

	  	So, finally our selected features (in-order) are LSTAT, RM, PTRATIO, CHAS. \\

	  	Now, calculating MSE from previously fitted data, we get

	  	\begin{center}
	  	\begin{tabular}{c|c}
			\hline
			 Input Data   &     MSE \\
			\hline
			 test\_data    & 34.5988 \\
			 train\_data   & 25.106  \\
			\hline
			\end{tabular}
	  	\end{center}

	  	\textbf{Brute Force}: The best combination is B, RM, LSTAT, PTRATIO with $MSE = 30.09226$ with test data set.

	  \paragraph{Sol. 3.4}
	  	The result after calculating the MSE using Linear Regression with the augmented data is

	  	\begin{center}
	  	\begin{tabular}{c|c}
			\hline
			 Input Data   &     MSE \\
			\hline
			 test\_data    & 14.5553 \\
			 train\_data   & 5.05978 \\
			\hline
			\end{tabular}
	  	\end{center}

\end{document}


%auto-ignore
\providecommand{\MainFolder}{..}
\documentclass[\MainFolder/Text.tex]{subfiles}
\newcommand{\TestF}{\mathcal{D}}
\begin{document}
\section{Schwartz form and smooth extension to blow-up}\label{Sec:SchwFrom}
\allowdisplaybreaks

For a smooth vector bundle $\pi: E\rightarrow M$, let $\TestF(E)$ be the space of ``test sections of $E$'', and let $\TestF'(E^*)$ be the space of distributions.  Let $\langle\cdot,\cdot\rangle: \TestF'(E^*)\otimes\TestF(E)\rightarrow \R$ be the natural pairing. Proper definitions can be formulated easily based on \cite[Section~6]{HormanderI}.
\begin{Proposition}[Schwartz kernel theorem, see {\cite[Theorem~5.2.1]{HormanderI}} for the local version]\label{Prop:SchwKer}
Let $E_1$ and $E_2$ be smooth vector bundles over smooth manifolds $M_1$ and $M_2$, respectively. Then there is a one-to-one correspondence between continuous operators $\TOp: \TestF(E_1)\rightarrow\TestF'(E_2)$ and elements $\KKer_\TOp\in\TestF'(E_1\boxtimes E_2^*)$ which is given by the equation
$$ \langle \TOp s_1, \psi_2 \rangle = \langle \KKer_\TOp, s_1 \boxtimes \psi_2 \rangle\quad\text{for all }s_1\in \TestF(E_1)\text{ and }\psi_2\in \TestF(E_2^*). $$
The distributional section $\KKer_{\TOp}$ is called the \emph{Schwartz kernel} of $\TOp$.
\end{Proposition}

An $L^1_{\text{loc}}$-integrable function $k: M\times M \rightarrow \Hom(E_1,E_2)=E_1^*\boxtimes E_2$ on an oriented Riemannian manifold $M$ defines the distribution $\KKer\in \TestF'(E_1\boxtimes E_2^*)$ by
$$ \langle\KKer,s_1\boxtimes \psi_2\rangle = \int_{x_1, x_2} \langle k(x_1,x_2)s_1(x_1),\psi_2(x_2)\rangle \Vol(x_1)\Vol(x_2) $$
%$$ \langle T s_1, \psi_2 \rangle = \int_{x_1} \langle t(s_1)(x_1),\psi_2(x_1)\rangle\Vol(x_1) $$
In the case of exterior bundles, we introduce an equivalent notion of a Schwartz form. This name was proposed by Dr.~A.~Hermann in a discussion in Potsdam.

\begin{Proposition}[Schwartz kernel and Schwartz form]\label{Prop:SchwForm}
In the setting of Proposition~\ref{Prop:SchwKer}, suppose that $M_1 = M_2 = M$ is a smooth oriented Riemannian manifold and $E_1 = E_2 = \Lambda T^* M$. We consider the isomorphism of vector bundles $\Psi:\Lambda T^*M\boxtimes(\Lambda T^*M)^*\rightarrow\Lambda T^*(M\times M)$ which is for every $x_1$, $x_2\in M$ given by
\begin{align*}
 \Psi: \Lambda T^*_{x_1}M\otimes\bigl(\Lambda T_{x_2}^*M\bigr)^* &\longrightarrow \Lambda T^*_{(x_1,x_2)}(M\times M) \\
 \omega_1\otimes\xi_2&\longmapsto\omega_1\wedge\sharp\xi_2.
\end{align*}
Here, $\sharp: \Lambda T^*M \rightarrow \bigl(\Lambda T^*M \bigr)^* $ denotes the musical isomorphism with respect to the natural pointwise inner product on $\Lambda T^*M$. We obtain the isomorphism
\begin{align*}
\Psi_*:\DR'(M\times M)&\longrightarrow\TestF'(E_1^*\boxtimes E_2)\\
\DR_\TOp&\longmapsto\KKer_\TOp,
\end{align*}
where $\DR'$ denotes the space of de Rham currents.

If $\TOp$ is homogenous of degree~$\Abs{\TOp}$, then the degree of $\DR_\TOp$ satisfies
$$ \Deg(\DR_\TOp) = \dim(M) + \Abs{\TOp}. $$

If $\KKer_T$ is represented by an $L^1_{\mathrm{loc}}$-integrable section of $\Hom(E_1,E_2)\simeq E_1^*\boxtimes E_2$, then $\DR_T$ is represented by an $L^1_{\mathrm{loc}}$-integrable form, and the other way round. In this case, for all $\omega_1 \in \DR(M)$, $\omega_2\in \DR_c(M)$, we have
\begin{equation}\label{Eq:SchwarzKerForm}
\begin{aligned}
\int_{x_2} \bigl((\TOp \omega_1)(x_2)\bigr)\bigl[\omega_2(x_2)\bigr] &= \int_{x_1, x_2} \KKer_T(x_1,x_2)[\omega_1(x_1)]\wedge \Vol(x_1)\wedge \omega_2(x_2)\\
&= \int_{x_1, x_2} \DR_T(x_1,x_2)\wedge\omega_1(x_1)\wedge\omega_2(x_2).
\end{aligned}
\end{equation}
\end{Proposition}
\begin{proof}
Straightforward computations similar to Example~\ref{Ex:IdFE} below.
\end{proof}

\begin{Definition}[Schwartz form]\label{Def:SDFDF}
The current $\DR_\TOp\in \DR'(M\times M)$ from Proposition~\ref{Prop:SchwForm} is called the \emph{Schwartz form} of $\TOp$.
\end{Definition}

We will consider pseudo-differential operators $\TOp:\DR(M)\rightarrow\DR(M)$ on a Riemannian manifold $M$; this class of operators generalizes differential operators and contains generalized inverses of elliptic operators (see \cite{Hormander} for thorough treatment).

\begin{Proposition}[Schwartz form of pseudo-differential operators] \label{Prop:ASD}
Let $\TOp: \DR(M) \rightarrow \DR(M)$ be a pseudo-differential operator on a smooth oriented Riemannian manifold $M$. Then the Schwartz form $\DR_{\TOp}$ restricts to a smooth form on $M\times M\backslash\Diag$.
\end{Proposition}
\begin{proof}
Well known fact, proof based on \cite{Hormander}.
\end{proof}

\begin{Example}[Schwartz form of $\Id$]\label{Ex:IdFE}The Schwartz form of the identity $\Id: \DR(\R^n)\rightarrow \DR(\R^n)$ reads
\begin{equation}\label{Eq:SchwartzFormOfId}
\DR_{\Id}(x,y) = \delta(x-y)(\Diff{x}^1 - \Diff{y}^1)\dotsb(\Diff{x}^n-\Diff{y}^n),
\end{equation}
where $\delta$ denotes the Dirac delta function on $\R^n$ centered at $0$. In order to prove this, we start by rewriting
\begin{align*}
(\Diff{x}^1 - \Diff{y}^1) \dotsb (\Diff{x}^n - \Diff{y}^n) &=\sum_{I}(-1)^{\Abs{I}} \varepsilon(I^c I \mapsto[n])\Diff{x}^{I^c} \wedge \Diff{y}^I \\
&=\sum_{I} (-1)^{n \Abs{I}} (\Star \Diff{x}^I) \wedge \Diff{y}^I.
\end{align*}
Here, the sum is over all multiindices $I\subset \{1,\dotsc,n\}$, and we use that $\Star(\Diff{x}^I) = \varepsilon(I, I^c) \Diff{x}^{I^c}$, where $\varepsilon(I,I^c)$ denotes the sign to order $I I^c$ to $\{1,\dotsc,n\}$. Now, for any $\omega\in \DR_c(\R^n)$, which we write as $\omega(x) = \sum_K \omega_K(x)\Diff{x}^K$, we compute using $\Diff{x}^K\wedge\Star(\Diff{x}^I)=(\Diff{x}^K,\Diff{x}^I)\Vol(x) = \delta^{KI}\Vol(x)$ the following:
\begin{align*}
&\int_x \delta(x-y)  (\Diff{x}^1-\Diff{y}^1)\dotsb(\Diff{x}^n-\Diff{y}^n) \omega(x)  \\
&\qquad=\sum_{I,K} \int_x \delta(x-y)(-1)^{n\Abs{I}}  \omega_K(x)(\Star \Diff{x}^I)\Diff{y}^I\Diff{x}^K \\
&\qquad=\sum_{I,K}\int_x \delta(x-y) (-1)^{n\Abs{I} + n\Abs{K}} \delta(x-y)  \omega_K(x) \Diff{x}^K \Star(\Diff{x}^I) \Diff{y}^I \\
&\qquad=\sum_{I} \int_x \delta(x-y) \omega_I(x)\Vol(x) \Diff{y}^I \\
&\qquad=\omega(y).
\end{align*}
This shows \eqref{Eq:SchwarzKerForm}, and \eqref{Eq:SchwartzFormOfId} follows. Notice that
$$ \Restr{\DR_\Id}{\R^n\times \R^n\backslash \Diag} = 0, $$
and thus the smooth part of $\DR_\Id$ does not recover the data of the operator $\Id$.
\end{Example}

%\begin{Definition}[Laplace Green kernel and standard Hodge propagator]\label{Def:GKerHPr}
%Let $M$ be a closed oriented Riemannian manifold, $\DR(M)=\Dd\DR(M)\oplus\CoDd\DR(M)\oplus\Harm(M)$ the Hodge decomposition, $\pi: \DR(M) \rightarrow \Harm(M)$ the harmonic projection and $\Laplace= \Dd\circ\CoDd + \CoDd\circ\Dd$ the Hodge de Rham Laplacian. We define the \emph{Laplace Green operator}  $\GOp: \DR(M)\rightarrow \DR(M)$ by
%$$ .$$
%We denote its Schwartz form by $\GKer$ and call it the \emph{Laplace Green kernel.}  We define the \emph{standard Hodge homotopy} by
%$$ \StdHtp \coloneqq - \CoDd \circ \GOp : \DR^\bullet(M) \longrightarrow \DR^{\bullet - 1}(M). $$
%It satisfies $\Dd\circ\StdHtp + \StdHtp\circ\Dd = - \Id$. We denote the Schwartz form of $\StdHtp$ by $\StdPrpg$ and call it the \emph{standard Hodge propagator.}
%\end{Definition}

We consider the Green operator $\GOp$ for the Laplacian $\Laplace$ (see \cite{Warner1983}) and the standard Hodge propagator $\HtpStd$. They are both pseudo-differential operators, and it holds $\HtpStd = - \CoDd \GOp$. We will study their Schwartz forms $\GKer$ and $\PrpgStd$, which are called the \emph{Green kernel} and the \emph{standard Hodge propagator,} respectively.

\begin{Proposition}[Basic facts about $\GKer$ and $\StdPrpg$]\label{Prop:BasicFactsGP}
The Green kernel $\GKer$ represents an $L^2$-integrable form.
The standard Hodge propagator $\StdPrpg$ represent an $L^1$-integrable form.
\end{Proposition}
\begin{proof}
The fact that $\GKer$ is $L^2$ is an exercise in \cite{Warner1983}. The fact that~$\StdPrpg$ defines an $L^1$-integrable form on $M\times M$ for a compact manifold $M$ was proved in \cite{Harris2004} using the heat kernel approximation (see the next section).
\end{proof}

A consequence of Proposition~\ref{Prop:BasicFactsGP} is that the Schwartz forms $\GKer$ and $\StdPrpg$ are determined by their smooth restrictions to $M\times M\backslash \Diag$. This follows from \eqref{Eq:SchwarzKerForm} because the integral does not depend on sets of zero measure. Therefore, we will write $\GKer$, $\StdPrpg \in \DR^{n-1}(M\times M\backslash\Diag)$.

\begin{Definition}[Smooth extension to the blow-up]\label{Def:Esdas}
Let $M$ be a smooth manifold, and let $\pi: \Bl_\Diag(M\times M)\rightarrow M\times M$ be the spherical blow-up of $M\times M$ at the diagonal $\Diag$.\footnote{Another name of this construction suggested to me by Dr.~Oliver Lindblad Petersen after explaining him our setting should be ``Melrose blow-up''.} Consider the blow-up diagram
\[\begin{tikzcd}
 & \Bl_\Diag(M\times M)\arrow[two heads]{d}{\pi} \\
 M\times M\backslash \Diag \arrow[hook]{r}{\iota}\arrow[hook]{ru}{\tilde{\iota}} & M\times M,
\end{tikzcd}\]
where $\iota$ is the inclusion and $\tilde{\iota}$ its unique smooth lift --- the embedding of the interior. We say that a smooth form $\omega\in\DR(M\times M\backslash\Diag)$ \emph{extends smoothly to the blow-up} if there is a smooth form $\tilde{\omega}\in \DR(\Bl_\Diag(M\times M))$ such that
$$ \tilde{\iota}^*\tilde{\omega}=\omega. $$
\end{Definition}

Note that the extension, if it exists, is necessarily unique.

\begin{Question}
Do $\GKer$ and $\StdPrpg$ extend smoothly to the blow-up?
We expect that $\GKer$ does not and $\StdPrpg$ does.
\end{Question}
\end{document}


\documentclass[twoside]{MATH77}
\usepackage{multicol}
\usepackage[fleqn,reqno,centertags]{amsmath}
\begin{document}
\begmath 10.4 Multi-Dimensional Real Fourier Transform

\silentfootnote{$^\copyright$1997 Calif. Inst. of Technology, \thisyear \ Math \`a la Carte, Inc.}

\subsection{Purpose}

This subroutine computes Real Fourier Transforms for real data in up to six
dimensions using the fast Fourier transform. In ND dimensions, the relations
between the values $x$ and the Fourier coefficients $\xi $ have the form%
\begin{multline*}
x(j_1,j_2,...,j_{ND})=\sum_{k_1=0}^{N_1-1}\cdots\sum_{k_{ND}=0}^{N_{ND}-1}%
\xi (k_1,k_2,...,k_{ND})\\
\times W_1^{j_1k_1}\cdots W_{ND}^{j_{ND}k_{ND}},
\end{multline*}
where $N_\ell=2^{\text{M}(\ell)}$, $W_\ell=e^{2\pi i/N_\ell}$, 0 $\leq $ $j_\ell
\leq N_\ell-1$, $x$ is real and $\xi $ is complex.

\subsection{Usage}

\subsubsection{Program Prototype, Single Precision}

\begin{description}
\item[\bf REAL]  \ {\bf A}$(N_1$, $N_2$, ..., $\geq N_{ND})$\quad $%
[N_k=2^{\text{M}(k)}]$

\item[\bf REAL]  \ {\bf S}$(\geq \max (\nu _1$, $\nu _2$, ..., $\nu
_{ND})-1)$\  $[\nu _k=2^{\text{M}(k)}-^2]$

\item[\bf INTEGER]  \ {\bf M}$(\geq $ ND){\bf , ND, MS}

\item[\bf CHARACTER]  \ {\bf MODE}
\end{description}
On the initial call set MS to~0 to indicate the array S() does not yet
contain a sine table. Assign values to A(), MODE, M(), and ND.
$$
\fbox{{\bf CALL SRFT(A, MODE, M, ND, MS, S)}}
$$
On return A() will contain computed results. S() will contain the sine table
used in computing the Fourier transform. MS may have been changed.

\subsubsection{Argument Definitions}

\begin{description}
\item[A()]  [inout] If the argument MODE selects Analysis, A() contains
values $x$ on entry and the Fourier coefficients $\xi $ on exit. If MODE
selects Synthesis, A() contains the Fourier coefficients $\xi $ on entry and
the values $x$ on exit. The Functional Description below describes the way $x
$ and $\xi $ are stored in A().

\item[MODE]  [in] The character variable MODE selects Analysis or Synthesis.

'A' or 'a' selects Analysis, transforming $x$'s to $\xi ^{\prime }s.
$

'S' or 's' selects Synthesis, transforming $\xi $'s to $x^{\prime
}s.$

\item[M()]  [in] Defines $N_k=2^{\text{M}(k)}$, the number of real data points in
the $k^{th}$ dimension. Require $0\leq \text{M}(k)\leq 31$ for all $k$%
, and M($1)=0$ only if M($k)\equiv 0$ for all $k$. No action is taken with
respect to dimensions for which M($k)=0.$

\item[ND]  [in] Number of dimensions. Require 1 $\leq $ ND $\leq $ 6.

\item[MS]  [inout] Gives the state of the sine table in $S()$.  Let
$\text{MS}_{in}\text{ and MS}_{out}$ denote the values of MS on entry
and return respectively. If the sine table has not previously been
computed, set $\text{MS}_{in} = 0$ or $-$1 before the call. Otherwise
the value of $\text{MS}_{out}$ from the previous call using the same
S() array can be used as $\text{MS}_{in}$ for the current call.

Certain error conditions described in Section E cause the subroutine
to set $\text{MS}_{out} = -2$ and return.  Otherwise, with $\max _i
\{\text{M}(i)\} > 0$, the subroutine sets $\text{MS}_{out} = \max
($M(1), M(2), ..., M(ND), $\text{MS}_{in}).$

If $\text{MS}_{out} > \max (2, \text{MS}_{in}),$ the subroutine sets
NT = $2^{\text{MS}_{out}-2}$ and fills S() with NT $-$ 1 sine values.

If $\text{MS}_{in}=-1$, the subroutine returns after the above
actions, not transforming the data in A().  This is intended to allow
the use of the sine table for data alteration before a subsequent Fourier
transform, as discussed in Section G of Chapter~16.0.

\item[S()]  [inout] When the sine table has been computed, S($j)=\sin \pi
j/(2\times {\textstyle NT})$, $j=1$, 2, ..., NT $-$ 1, see MS above.
\end{description}

\subsubsection{Modifications for Double Precision}

Change SRFT to DRFT and the REAL type statements to DOUBLE PRECISION.

\subsection{Examples and Remarks}

A ``smooth'' function that approximates%
\begin{equation*}
A(i,j)=
\begin{cases}
0 & \text{if}\ |9-i|+|9-j|>4 \\
1 & \text{if}\ |9-i|+|9-j|\leq 4
\end{cases}
\ 1\leq i,\ j\leq 16.
\end{equation*}
is desired. The example at the end of the chapter does this by computing
the two dimensional transform of A, applying sigma factors (see Section
G of Chapter~16.0), and then transforming back. Results are printed only
for 1 $\leq $ $i,\ j\leq 9$ since, to within round-off limitations, A($%
9+m,9-n)={\textstyle A}(9-m,9-n)$, $1\leq 9\pm m\leq 16$ and $1\leq $ $9\pm
n\leq 16.$

\subsection{Functional Description}

The multi-dimensional real Fourier transform is done by changing it to a
problem in complex variables, doing a multi-dimensional complex Fourier
transform, and then adjusting the results to obtain the solution for the
original problem.

Taking complex conjugates in Eq.\,(1) and using the fact that $x$ is real, it
can be verified that%
\begin{equation*}
\xi (N_1-k_1,\;N_2-k_2,...,N_d-k_d)=\overline{\xi }(k_1,k_2,...,k_d),
\end{equation*}
where $N_i-k_i$ is interpreted modulo $N_i$. (Thus $N_i-k_i=0$ if $k_i$ is
0.) Using the above, it is possible to pack the nonredundant $\xi ^{\prime
}s $ in the same space as is required for $x$.

Storage of $\xi $ in A() for the case $d=2$ is illustrated in Table~1. The
rows and columns in the table correspond to rows and columns in the array
A(). Only the subscripts $k_1$, $k_2$ of the $\xi $'s are given.
The symbols $J_1$ and $J_2$ are used as abbreviations for $N_1/2$ and $N_2/2$
respectively. The coefficients with subscripts (0,0), $(J_1,0)$, $(0,J_2)$,
and $(J_1,J_2)$ are real and occupy single array elements as shown. For the
other coefficients, the subscript $``k_1,k_2"$ identifies the location of
the real part of $\xi (k_1$, $k_2)$, and an ``I'' immediately below such a
subscript gives the location of the imaginary part of $\xi (k_1$,\ $k_2).$

The first column (with the second subscript ignored) gives the storage
scheme for the case $d=1$. For $d>2$, and with $\nu =$ the smallest value of
$i(>1)$ for which $k_i\neq 0$ and $k_i\neq N_\nu /2$, the storage scheme
generalizes as follows.

If $1 < k_1 < N_1$ (A$(2k_1+1$, $k_2+1$, ..., $k_d+1),$ A$(2k_1+2$,
$k_2+1$, ..., $k_d+1))$ contains $\xi (k_1,k_2,...,k_d)$.  Else
(A(1, $k_2+1$, ..., $k_d+1),$ A(2, $k_2+1$, ..., $k_d+1))$ contains
\begin{alignat*}{2}
&\xi (0,k_2,...,k_d) &\ &1\leq k_\nu <N_\nu /2\\
&\xi (N_1/2,k_2,...,k_d)&\ \qquad &N_\nu /2<k_\nu <N_\nu
\end{alignat*}
and when $k_i$ is either 0 or $N_i/2$ for all i,
(A(1, $k_2+1$, ..., $k_d+1),$ A(2, $k_2+1$, ..., $k_d+1))$ contains $(\xi
(0,k_2,...,k_d),~\xi (N_1/2,k_2,...,k_d)).$ Note that in this last case both
$\xi $'s are real.

More details can be found in \cite{Krogh:1970:RFT}.

\bibliography{math77}
\bibliographystyle{math77}

\subsection{Error Procedures and Restrictions}

Require $0 \leq \text{M}(k)\leq 31$ and $1 \leq \text{ND} \leq $ 6.  MODE
must contain one of the allowed values.  If any of these conditions are
violated the subroutine will issue an error message using the error
processing procedures of Chapter~19.2 with a severity level of~2 to cause
execution to stop.  A return is made with $\text{MS}=-2$ instead of
stopping if the statement ``CALL\ ERMSET($-$1)'' is executed before
calling this subroutine.

If the sine table does not appear to have valid data, an error message is
printed, and the sine table and then the transform are computed.

\subsection{Supporting Information}

The source language is ANSI Fortran~77.

\begin{tabular}{@{\bf}l@{\hspace{5pt}}l}
\bf Entry & \hspace{.35in} {\bf Required Files}\vspace{2pt} \\
DRFT & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
 DFFT, DRFT, ERFIN, ERMSG, IERM1, IERV1\rule[-5pt]{0pt}{8pt}}\\
SRFT & \parbox[t]{2.7in}{\hyphenpenalty10000 \raggedright
 ERFIN, ERMSG, IERM1, IERV1, SFFT, SRFT}\\
\end{tabular}

Subroutine designed and written by: Fred T. Krogh, JPL, October~1969,
revised January~1988.

\end{multicols}

\begin{table} \centering
  \begin{tabular}{cccccccc}
     0,0 & 0,1 & $\cdots$ & 0,$J_2-1$ & 0,$J_2$ & $J_1$,$J_2+1$ & $\cdots$ & $J_1$,$N_2 -1$ \\
     $J_1$, 0 & I & $\cdots$ & I & $J_1$,$J_2$ & I & $\cdots$ & I \\
      1,0 & 1,1 & $\cdots$ & 1,$J_2-1$ & 1,$J_2$ & 1,$J_2+1$ & $\cdots$ & 1,$N_2 -1$ \\
     I & I & $\cdots$ & I & I & I & $\cdots$ & I \\
    $\cdots$ & & & & $\cdots$ & & & $\cdots$ \\
     $J_1 -1$,0 & $J_1-1$,1 & $\cdots$ & $J_1-1$,$J_2-1$ & $J_1-1$, $J_2$ & $J_1-1$,$J_2+1$ & $\cdots$ & $J_1-1$,$N_2 -1$ \\
    I & I & $\cdots$ & I & I & I & $\cdots$ & I \\
  \end{tabular}
  \caption{Storage of $\xi$ in A() for Two Dimensions}
\end{table}

\begcodenm

\lstset{language=[77]Fortran,showstringspaces=false}
\lstset{xleftmargin=.8in}

\centerline{\bf \large DRSRFT}\vspace{10pt}
\lstinputlisting{\codeloc{srft}}

\vspace{30pt}\centerline{\bf \large ODSRFT}\vspace{10pt}
\lstset{language={}}
\lstinputlisting{\outputloc{srft}}
\end{document}


\section{Modeling Coral Reef Dynamics}

\begin{frame}
\frametitle{Coral Reef Dynamics}
\includegraphics[scale=.175]{./coral-reef-triangle.png}\\ \cite{Hastings}
\end{frame}

\begin{frame}\frametitle{Coral Reef Dynamics}
The deterministic ordinary differential equation model:
$$\begin{cases}\begin{array}{rl}
\frac{dM}{dt}\hspace{-.8em}&=aMC - \frac{gM}{M+T} + \gamma M T, \\
\frac{dC}{dt}\hspace{-.8em}&=rTC - dC - aMC, \\
\frac{dT}{dt}\hspace{-.8em}&=\frac{gM}{M + T} - \gamma MT - rTC + dC.
\end{array}\end{cases}$$

where
\begin{itemize}\itemsep0pt
\item $r$ is the rate corals overgrow upon algal turfs\\
\item $d$ is the mortality rate of corals\\
\item $a$ is the rate that macroalgae overgrow upon corals\\
\item $\gamma$ is the rate that macroalgae spread over algal turfs\\
\item $g$ is the indiscriminate grazing rate of scarid
\end{itemize} We also assume $a<d<\gamma<r<2\gamma$ and $0<g<\gamma$. \vspace{1em}
\cite{Hastings}
\end{frame}

\begin{frame}\frametitle{Coral Reef Dynamics}

\hspace{1.57em}

\begin{itemize}
\item $\frac{dT}{dt}=-\frac{dM}{dt}-\frac{dC}{dt}$ implies $M+C+T$ is
  constant.
\item We assume that $M+C+T=1$.
\item This limits our scope to regions entirely covered by coral,
  macroalgae, and algal turf.
\end{itemize}

Thus, we reduce our system to,
$$\begin{cases}
\begin{array}{rl}
\frac{dM}{dt}&= aMC-\frac{gM}{1-C} + \gamma M - \gamma M^2 -\gamma M C,\\
\frac{dC}{dt}&=rC - rC^2 - rCM - dC - aMC.
\end{array}
\end{cases}$$
\end{frame}

\begin{comment}\frametitle{Equilibria and Stability}
Equilibrium point

\end{comment}

\begin{frame}\frametitle{Equilibria and Stability}
To find equilibrium points analytically we first set the derivatives equal to zero to obtain the nullclines:
$$\begin{cases}
\begin{array}{rl}
0\hspace{-.8em}&=M(aC + \gamma-\gamma M-\gamma C - \frac{g}{1-C}),\\
0\hspace{-.8em}&=C(r-Mr-Cr - d - aM).
\end{array}
\end{cases}$$
\end{frame}

\begin{frame}\frametitle{Equilibria and Stability}

  \begin{itemize}
  \item $M'=0$ is satisfied when:
    \begin{itemize}
    \item $M=0$, or
    \item
      $aC + \gamma - \gamma M - \gamma C -
      \frac{g}{1-C}=0$.
    \end{itemize}
  \item $C'=0$ is satisfied when:
    \begin{itemize}
    \item$C=0$, or
    \item $r-Mr-Cr-d-aM=0$.
    \end{itemize}
  \end{itemize}

\end{frame}

\begin{frame}\frametitle{Phase Plane}

  %Our equilibrium points are located at the intersections of the nullclines:\\
  \includegraphics[scale=.325]{./nullclines.pdf}

\end{frame}

\begin{frame}
\frametitle{Time Delay in the Coral Ecosystem}

We can identify time delays in the coral ecosystem:
\begin{itemize}
\item the delay between overfishing of herbivores and growth of
  macroalgae \cite{Dinsdale},
\item the delay between the growth of algae and the effect of algae on
  coral, \cite{Mumby}
\item the delay between the grazing of macroalgae and growth of algal
  turf \cite{Hastings}.
\end{itemize} We focus on the third phenomena.

\end{frame}

\begin{frame}\frametitle{Coral Reef Dynamics}
Scarid grazing has an impact for the macroalgae in the future:
$$\begin{cases}
  \begin{array}{rl}
    \frac{dM}{dt}\hspace{-.8em}&=aMC - \frac{gM(t-\tau)}{1-C(t-\tau)} + \gamma M (1-M-C),\\
    \frac{dC}{dt}\hspace{-.8em}&=rC(1-M-C) - dC - aMC,\\
  \end{array}
\end{cases}$$ where $\tau$ is a fixed time delay.
\end{frame}

\begin{frame}
\frametitle{Equilibria and Stability}
These are three equilibrium points of interest:
\begin{itemize}
\item $(0,0)$ (Unstable for all $\tau\geq0$)\\
\item $(0,1-\frac{g}{\gamma})$ (High Coral Cover)\\
\item $(1-\frac{d}{r},0)$ (High Algae Cover).
\end{itemize}
Notice that time delay does not affect the equilibria.
\end{frame}

\begin{frame}\frametitle{Jacobian Matrices}
We linearize our delay model:

\begin{eqnarray}
  \label{eqn:linearizedDelayModel}
  \begin{bmatrix}
    M'\\C'
  \end{bmatrix}=J_1
  \begin{bmatrix}
    M \\
    C
  \end{bmatrix} +
  J_2
  \begin{bmatrix}
    M(t-\tau) \\
    C(t-\tau)
  \end{bmatrix},
\end{eqnarray}

where

\begin{eqnarray*}
  J_1 & = & \begin{bmatrix}
    \gamma-2\gamma M^* +(a-\gamma)C^* & (a-\gamma)M^*\\
    -(a+r)C^* & r-d-(a+r)-2M^*C^*
  \end{bmatrix},  \\
  J_2 & = &
  \begin{bmatrix}
    \frac{-g}{1-C^*} & \frac{-gM^*}{(1-C^*)^2} \\
    0 & 0
  \end{bmatrix}.
\end{eqnarray*}

\end{frame}

\begin{comment}

\begin{frame}\frametitle{Putting Jacobians to Use}
{ Suppose $\begin{bmatrix} x\\y\end{bmatrix}=\overrightarrow{v_1}e^{\lambda t}$.}\\\vspace{2em}
{ Differentiating, setting equal to 1, and doing algebra stuff, we get $(J_1+J_2e^{-\lambda t} -\lambda I)\overrightarrow{v_1}=0$.}
\end{frame}
\end{comment}

\begin{frame}\frametitle{Putting Jacobians to Use}
Consider the Jacobians evaluated at the origin:

\begin{eqnarray*}
  J_1 & = &
            \begin{bmatrix}
              \gamma & 0\\
              0 & r-d
            \end{bmatrix}, \\
  J_2 & = &
            \begin{bmatrix}
              -g & 0\\
              0 & 0
            \end{bmatrix}.
\end{eqnarray*}

\end{frame}

\begin{frame}[c]\frametitle{Putting Jacobians to Use}
%Recall, $(J_1+J_2e^{-\lambda t} -\lambda I)\overrightarrow{v_1}=0$. \\\vspace{2em} Taking determinants, we find the characteristic polynomial of $(J_1+J_2e^{-\lambda t} -\lambda I)$ evaluated at $M=0$, $C=0$ to be

  The characteristic polynomial of
  $(J_1+J_2e^{-\lambda \tau})$ is,
$$(\lambda - r + d)(\lambda -\gamma + ge^{-\lambda\tau})=0.$$

\end{frame}

  \begin{frame}\frametitle{Get Them Eigenvalues}
Hence,
\begin{itemize}{\itemsep .5in}
\item $\lambda = r-d>0$ is an eigenvalue with positive real part
\item Other eigenvalues satisfy $\lambda=\gamma-ge^{-\lambda\tau}$.
\item For $\tau\geq0$, our system has two eigenvalues with positive
  real part. This implies our system is unstable at $(0,0)$ for all $\tau\geq0$.
\end{itemize}
\end{frame}

\begin{comment}
\begin{frame}\frametitle{Get Them Eigenvalues}
Now we consider positive $\tau$.
\begin{itemize}
\item Let $\lambda \tau=(\alpha+i\omega)\tau$.
\item Apply Euler's Formula: $\gamma-g(e^{-\lambda\tau})=\gamma-g(\cos(\lambda\tau)-i\sin(\lambda\tau))$.
\end{itemize}
\end{frame}\end{comment}

\begin{frame}\frametitle{Stochastic Coral Features}
The time delay model fails to account for stochastic features of the ecosystem:
\begin{itemize}
\item human activities (e.g. fishing and boating),
\item grazing habits of scarids,
\item hurricanes.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Stochastic Model}
To account for these stochastic features we add noise!
$$\begin{cases}
\begin{array}{rl}
dM\hspace{-.8em}&=(aMC - \frac{gM(t-\tau)}{1-C(t-\tau)+\gamma M T})dt+{\color{red}\beta M(1-M)dW},\\
dC\hspace{-.8em}&=(rTC  - dC - aMC)dt.\\
\end{array}
\end{cases}$$
\end{frame}


\section{Preliminaries}
All rings mentioned are commutative and have a multiplicative identity,
referred to as $1$.

We will try not to assume that the reader knows everything,
but we will assume general knowledge of algebra and some introductory commutative algebra like definitions of rings,
modules, algebras etc.

\begin{theorem}[Noether's normalization lemma]
Let $k$ be a field and $B$ a non-trivial finitely generated $k$-algebra.
Then there exists a non-negative integer $d$ and algebraically independent elements $y_1, ..., y_d$,
i.e. elements with no relations,
in $B$ such that $B$ integral over the polynomial ring $k[y_1, ..., y_d]$.
\end{theorem}
\begin{proof}
Note that the following proof is heavily based on the proof from Mumford's ``The red book''.

Since $B$ is finitely generated the proof is done by induction on the number of generators, $m$.

$m=0$:
In this case $A=k$ and we are done.

Assume $m>0$ and that the lemma holds for all $k$-algebras generated by $m-1$ or fewer elements.

Now, let $B$ be generated by $y_1, \ldots, y_{m}$.
If there is no relations on the generators,
i.e. they are algebraically independent, then
\begin{equation*}
    B \cong k[y_1, \ldots, y_{m}]
\end{equation*}
and we are done.
Hence we assume that $f$ is a relation on the $m$ generators,
i.e. a polynomial equation
\begin{equation*}
    f(y_1, \ldots y_m) = 0.
\end{equation*}
Let $r$ be an integer.
We are going to determine this later.
Set $z_1 = y_1$ and $z_i = y_i - y_1^{r^{i-1}}$ for $2\leq i \leq m$.
Note that the $z_i$'s also are generators of $B$.
Then we can rewrite the relation as
\begin{equation*}
    f(z_1, z_2+y_1^r, z_3+y_1^{r^2}, \ldots z_m+y_1^{r^{m-1}}) = 0.
\end{equation*}
For $a\in k$,
the highest term of $y_1$ appearing in $a\prod_1^m (z_i + z_1^{r^{i-1}})^{\alpha_i}$ looks like $ay_1^{\alpha_1+r\alpha_2+\ldots+r^{m-1}\alpha_m}$.
Hence, if $r$ is larger than all of the $\alpha_i$'s appearing in $f$,
then the highest term of $y_1$ appearing in $f(z_1, z_2+y_1^r, z_3+y_1^{r^2}, \ldots z_m+y_1^{r^{m-1}})$ also has the same form as the one above,
which means that $y_1$ is integral over $k[y_2, \ldots, y_m]$.

Since $y_{i}=z_{i}+y_{1}^{r^{i-1}}$ are also integral over the same ring,
the $z_i$'s generate $B$, and being an integral extension is a transitive property,
we get that B is integral over $k[y_2, \ldots, y_m]$.
Since $k[y_2, \ldots, y_m]$ is generated by $m-1$ elements,
we can apply the inductive hypothesis, and hence we are done.
\end{proof}

%Note that Noether's normalization lemma also can be formulated,
%and hence is equivalent to the statement that there exists a finite injective $k$-algebra homomorphism $k[y_1, \ldots y_d]\longrightarrow B$ for some $d\in \mathbb{N}$.

We want to apply Noether's normalization lemma to prove Zariski's lemma.
In the proof we present we use a result stating that integral ring extensions preserve Krull dimension.
We actually only use the fact that if a ring with Krull dimension zero is an integral ring extension of another ring,
then this ring also has Krull dimension zero.
The more general statement also is true,
but we only prove a more strict statement that we need.

We remind ourself what we mean by Krull dimension.
\begin{definition}[Krull dimension]
Let $A$ be a ring.
The Krull dimension of $A$ is defined as the supremum over the length of all chains of prime ideals in $A$.
\end{definition}

\begin{proposition}
Let $S$ be a ring and $T$ be an integral ring extension of $S$.
Then the Krull dimension of $S$ is lower or equal to the Krull dimension of $T$.
\end{proposition}
\begin{proof}
Let $\mathfrak{p}_0\subsetneq \cdots \subsetneq \mathfrak{p}_n$ be a chain of prime ideals in $S$.
By the Going-up theorem we can extend any chain of length less than $n$,
$\mathfrak{q}_0\subsetneq \cdots \subsetneq \mathfrak{q}_m$ such that $\mathfrak{q}_i\cap S = \mathfrak{p}_i$,
for $i=1,\ldots, m$,
to a chain of prime ideals $\mathfrak{q}_0\subsetneq \cdots \subsetneq \mathfrak{q}_n$ in $T$,
such that $\mathfrak{q}_i\cap S = \mathfrak{p}_i$ for $i=1,\ldots, n$.
Hence the Krull dimension of an integral ring extension can't get any lower than the Krull dimension of the ring it extends.
\end{proof}

Note that we here used the so called Going-up theorem.
This theorem proves exactly what is stated in the proof of the proposition,
i.e. that we can extend chains of prime ideals in integral extensions.
We will not prove this theorem,
but we refer the reader to the proof in Introduction to commutative algebra by Atiyah and MacDonald.

This immediately implies what we need,
i.e. that if a ring with Krull dimension zero is an integral ring extension of another ring,
then this ring also has Krull dimension zero.

\begin{corollary}[Zariski's lemma]
Let $k$ be a field, and $K$ be a field extension that is finitely generated as an algebra over $k$.
Then $K$ is a finite field extension,
i.e. a finite dimensional $k$-vectorspace.
\end{corollary}
\begin{proof}
By Noether's normalization lemma,
$K$ is integral over the ring $k[y_1, \ldots y_d]$,
where $y_1, \ldots y_d$ are the elements of $K$ that are algebraically independent over $k$.
Since $K$ is a field, it has Krull dimension $0$,
and since integral ring extensions preserve Krull dimension,
the polynomial ring $k[y_1, \ldots y_d]$ must have Krull dimension zero,
i.e. $d=0$. Hence $K$ is a finitely generated module over $k$,
i.e. a finite dimensional $k$-vectorspace.
\end{proof}

Now, before we tackle the actual proof of Hilbert's nullstellensatz,
we need to chose a formulation.
There are many formulations of the theorem,
all hopefully equivalent,
but we have chosen one of the more standard and iconic ones.
The formulation is based on some language from algebraic geometry,
hence we need some definitions.

\begin{definition}[Radical]
Let A be a ring and $\mathfrak{a}$ an ideal in A.
We define the radical of $\mathfrak{a}$ to be the set
\begin{equation*}
    \sqrt{\mathfrak{a}} = \{ f\in A \ | \ \exists n \in \mathbb{N} \text{ s.t. } f^n \in \mathfrak{a} \}.
\end{equation*}
We say an ideal $\mathfrak{a}$ is a radical ideal if $\mathfrak{a}=\sqrt{\mathfrak{a}}$.
\end{definition}

\begin{definition}[Zero locus]
Let $k$ be an algebraically closed field,
$A$ be the polynomial ring $A = k[t_1, \ldots, t_n]$ and let $S \subset A$ be a subset.
We define the zero locus of $S$ to be the set
\begin{equation*}
    Z(S) = \{ x\in k^n \ | \ f(x)=0, \ \forall f\in S\}.
\end{equation*}
\end{definition}
If a subset $V\subset k^n$ has the form $V=Z(S)$ for some $S\subset A$,
then we call $V$ an affine algebraic variety.

\begin{definition}[Vanishing set]
Let $V\subset k^n$. We define the vanishing set of $V$ to be the set
\begin{equation*}
    I(V) = \{ f\in A \ | \ f(x)= 0, \ \forall x\in V\}.
\end{equation*}
\end{definition}


\chapter{Implementation}
\label{implementation}

\section{Integer Linear Programming}
For prototype the ILP problem formulated in the section 3.1.1 was modeled and solved in AMPL language.

\begin{lstlisting}[caption=AMPL model]

set Tier;
set VM;

param tier_used{Tier, VM};
param vm_used{VM};
param cpu_demand{Tier};
param mem_demand{Tier};
param cpu_max{VM};
param mem_max{VM};
param use_tier_weight{Tier, VM};
param use_vm_weight{VM};
param not_use_tier_weight{Tier, VM};
param not_use_vm_weight{VM};

var tier_usage{Tier, VM} >= 0 binary;
var vm_usage{VM} >= 0 binary;
var tier_idle{Tier, VM} >= 0 binary;
var vm_idle{VM} >= 0 binary;
var cpu{Tier, VM} >= 0 integer;
var mem{Tier, VM} >= 0 integer;

minimize cost:
  sum{i in Tier, j in VM}
    (use_tier_weight[i, j] * tier_usage[i, j] +
      not_use_tier_weight[i, j] * tier_idle[i, j]) +
  sum{j in VM} (use_vm_weight[j] * vm_usage[j] +
    not_use_vm_weight[j] * vm_idle[j]);

subject to CPU_availability{j in VM}:
  sum{i in Tier}
    cpu[i, j] <= cpu_max[j] * vm_usage[j];

subject to RAM_availability{j in VM}:
  sum{i in Tier} mem[i, j] <= mem_max[j];

subject to CPU_demand{i in Tier}:
  sum{j in VM} cpu[i, j] = cpu_demand[i];

subject to RAM_demand{i in Tier}:
  sum{j in VM} mem[i, j] = mem_demand[i];

subject to CPU_activation{i in Tier, j in VM}:
  cpu_max[j] * tier_usage[i, j] >= cpu[i, j];

subject to RAM_activation{i in Tier, j in VM}:
  mem_max[j] * tier_usage[i, j] >= mem[i, j];

subject to CPU_RAM_activation{i in Tier, j in VM}:
  mem_max[j] * cpu[i, j] >= mem[i, j];

subject to RAM_CPU_activation{i in Tier, j in VM}:
  cpu_max[j] * mem[i, j] >= cpu[i, j];

subject to link_tier_idle{i in Tier, j in VM}:
  tier_idle[i, j] + tier_usage[i, j] = 1;

subject to link_vm_idle{j in VM}:
  vm_idle[j] + vm_usage[j] = 1;
\end{lstlisting}

In the executor we solve the problem using \textit{or-tools} and ILP solver by Google. The Solver component
accepts the current allocation, the topology and provides as output a new allocation. The new allocation is
the input of the Translator component which translates the new allocation to the list of actions to be run.
This list of actions is the input to the Executor component.

\section{API}
\subsection{Main Node}
\textbf{[GET] /api/allocation}
Returns the allocation JSON of known VMs. In reality, just make calls to VMs and groups answers.
\begin{lstlisting}[caption=Example GET allocation output,basicstyle=\tiny]
{
  "52.34.2.170": {
    "rubis_app_server":{
      "cpuset": [0],
      "cpu_cores": 1,
      "mem_units": 2
    },
    "pwitter_app_server":{
      "cpuset": [1],
      "cpu_cores": 1,
      "mem_units": 3
    }
  }
}
\end{lstlisting}

\vspace{5mm}
\textbf{[GET] /api/inspect}
Returns the inspect output JSON for known VMs. In reality, just make calls to VMs and groups answers.
\begin{lstlisting}[caption=Example GET inspect output,basicstyle=\tiny]
{
  "52.34.2.170": {
    "rubis_app_server":{
     "MemUnits": 2,
      "CpusetCpus": "0",
      "Memory":1073741824
    },
    "pwitter_app_server": {
      "MemUnits": 3,
      "CpusetCpus": "1",
      "Memory": 1610612736
    }
  }
}
\end{lstlisting}

\vspace{5mm}
\noindent\textbf{[GET] /api/topology} \\
Returns JSON with the current topology.
\begin{lstlisting}[caption=Example topology,basicstyle=\tiny]
{
  "infrastructure": {
    "cloud_driver": {
      "name": "aws-ec2",
      "autoscaling_group_name": "monolithic-ex-8cpu",
      "credentials": "/usr/me/utils/aws.properties"
    },
    "max_vms": 10,
    "hooks_git_repo": "https://github.com/n43jl/hooks.git"
  },
  "apps": [{
    "name": "rubis",
    "tiers": {
      "loadbalancer": {
        "name": "Front Load Balancer",
        "max_node": 1,
        "docker_image": "haproxy",
        "depends_on": ["app_server"],
        "on_dependency_scale": "reload_server_pool.sh",
        "max_rt": 0.1
      },
      "app_server": {
        "name": "Application Logic Tier",
        "docker_image": "polimi/rubis-jboss",
        "depends_on": ["db"],
        "on_node_scale": "jboss_hook.sh",
        "on_dependency_scale": "reload_connections.sh",
        "ports": ["80:8080"],
         "entrypoint_params": "-w 3 -k eventlet"
      },
      "db": {
        "name": "Data Tier",
        "max_node": 1,
        "docker_image": "mysql",
        "on_node_scale": "mysql_hook.sh",
        "max_rt": 0.2,
        "ports": ["3306:3306"]
      }
    }
  },{
    "name": "pwitter",
    "tiers": {
      "app_server": {
        "name": "Application Logic Tier",
        "docker_image": "pwitter-web",
        "ports": ["8080:5000"],
        "entrypoint_params": " /opt/jboss-4.2.2.GA/bin/run.sh --host=0.0.0.0
        --bootdir=/opt/rubis/rubis-cvs-2008-02-25/Servlets_Hibernate -c default"
      },
      "db": {
        "name": "Data Tier",
        "max_node": 1,
        "docker_image": "mysql",
        "on_node_scale": "mysql_hook.sh",
        "max_rt": 0.2,
        "ports": ["3307:3306"]
      }
    }
  }],
}
\end{lstlisting}

\vspace{5mm}
\noindent\textbf{[PUT] /api/topology} \\
Set the topology. The payload is the same as the example output of the [GET] /api/topology method.

\vspace{5mm}
\noindent\textbf{[PUT] /api/translate} \\
Returns JSON with the actions list required to execute provided in the payload plan.
\begin{lstlisting}[caption=Example of the input payload,basicstyle=\tiny]
{
  "rubis": {
    "app_server": {
      "cpu_cores": 1,
      "mem_units": 3
    }
  },
  "pwitter": {
    "app_server": {
      "cpu_cores": 1,
      "mem_units": 4
    }
  }
}
\end{lstlisting}
\begin{lstlisting}[caption=Example of the output,basicstyle=\tiny]
{
  "actions":[
    "update container \"rubis_app_server\"
     on the vm "52.34.2.170" set cpu_cores=1 and mem_units=3",
    "update container \"pwitter_app_server\"
     on the vm "52.34.2.170" set cpu_cores=1 and mem_units=4"
  ]
}
\end{lstlisting}

\subsection{Agent}
\textbf{[GET] /api/allocation} \\
Returns JSON with the current allocation of the containers on the VM. The difference with the "inspect" command is that
"inspect" is the output of "docker inspect" command, while the "allocation" is taken from the app runtime state. e.g. We
create using the API some container "Jboss". Both the "allocation" and "inspect" will return 1 container. After that we
stop / start our Executor Agent Node. The "allocation" will be empty, but the "inspect" will return 1 container.
\begin{lstlisting}[caption=Example GET allocation output,basicstyle=\tiny]
{
  "rubis_app_server":{
    "cpu_cores": 1,
    "mem_units": 2
  },
  "pwitter_app_server":{
    "cpu_cores": 1,
    "mem_units": 3
  }
}
\end{lstlisting}

\vspace{5mm}
\noindent\textbf{[GET] /api/inspect} \\
Returns JSON with the current docker containers running on the VM.
\begin{lstlisting}[caption=Example GET inspect output,basicstyle=\tiny]
{
  "rubis_app_server":{
    "MemUnits": 2,
    "CpusetCpus": "0",
    "Memory":1073741824
  },
  "pwitter_app_server": {
    "MemUnits": 3,
    "CpusetCpus": "1",
    "Memory": 1610612736
  }
}
\end{lstlisting}

\vspace{5mm}
\noindent\textbf{[GET] /api/topology} \\
The output is the same as for the main node.

\vspace{5mm}
\noindent\textbf{[PUT] /api/topology} \\
The input and the output is the same as for the main node.

\vspace{5mm}
\noindent\textbf{[PUT] /api/translate} \\
Returns JSON with the actions list required to execute provided in the payload plan.
\begin{lstlisting}[caption=Example of the input payload,basicstyle=\tiny]
{
  "rubis": {
    "app_server": {
      "cpu_cores": 1,
      "mem_units": 3
    }
  },
  "pwitter": {
    "app_server": {
      "cpu_cores": 1,
      "mem_units": 4
    }
  }
}
\end{lstlisting}
\begin{lstlisting}[caption=Example of the output,basicstyle=\tiny]
{
  "actions":[
    "update container \"rubis_app_server\" set cpu_cores=1 and mem_units=3",
    "update container \"pwitter_app_server\" set cpu_cores=1 and mem_units=4"
  ]
}
\end{lstlisting}

\vspace{5mm}
\noindent\textbf{[PUT] /api/topology} \\
Set the topology. The payload is the same as the example output of the [GET] /api/topology method.

\vspace{5mm}
\noindent\textbf{[PUT] /api/execute} \\
Returns JSON with the actions list required to execute provided in the payload plan.
\begin{lstlisting}[caption=Example of the input payload,basicstyle=\tiny]
{
  "rubis": {
    "app_server": {
      "cpu_cores": 1,
      "mem_units": 3
    }
  },
  "pwitter": {
    "app_server": {
      "cpu_cores": 1,
      "mem_units": 4
    }
  }
}
\end{lstlisting}

\vspace{5mm}
\noindent\textbf{[PUT] /api/run/tier\_hooks} \\
Returns JSON with the actions list required to execute provided in the payload plan.
\begin{lstlisting}[caption=Example of the input payload,basicstyle=\tiny]
[
  {
    "app": "rubis",
    "dependent": "app_server",
    "depends_on": ["db"],
    "allocation": "..."
  }
]
\end{lstlisting}

\vspace{5mm}
\noindent\textbf{[PUT] /api/docker/run} \\
Runs the docker container specified in the payload.
\begin{lstlisting}[caption=Example of the input payload,basicstyle=\tiny]
{
  "name":"pwitter_app_server",
  "cpu_cores": 2,
  "mem_units": 3
}
\end{lstlisting}

\noindent\textbf{[PUT] /api/docker/remove} \\
Removes the docker container specified in the payload.
\begin{lstlisting}[caption=Example of the input payload,basicstyle=\tiny]
{
  "name":"pwitter_app_server"
}
\end{lstlisting}

\noindent\textbf{[PUT] /api/docker/update} \\
Returns JSON with the actions list required to execute provided in the payload plan.
\begin{lstlisting}[caption=Example of the input payload,basicstyle=\tiny]
{
  "name":"pwitter_app_server",
  "cpu_cores": 1,
  "mem_units": 2
}
\end{lstlisting}

\section{Topology}
There are restrictions for the topology:
\begin{enumerate}
    \item Ports are specified in the form (host VM port : guest container port). All hosts ports specified for all tiers should be unique, even for different applications.
    \item The script name specified in the "on\_node\_scale" should be located in the repository specified in "hooks\_git\_repo" in the folder "scale\_hooks".
    \item The script name specified in the "on\_dependency\_scale" should be located in the repository specified in "hooks\_git\_repo" in the folder "tier\_hooks".
    \item The max\_node=1 means that the tier is not scalable. It will not be moved from one VM to another and it must have the resources demand less than it is available on one VM.
    \item For tiers with the max\_node \textgreater{} 1 is allowed to have dependencies only to tiers with max\_node=1 (n-to-1). n-to-n dependencies are not allowed.
    \item Dependencies between tiers from different applications are not allowed.
    \item Circular dependencies are not allowed
\end{enumerate}

Also tier names should be unique inside the app, and the app name should be unique too. Any tiers of any apps can be on the same VM, so uniqueness of the container name is imposed by using as container name the compound name: "{APP\_NAME}\_{TIER\_NAME}".

\section{Hooks}
To adjust to the new allocation they should be provided with input parameters.

The "on\_node\_scale" hook requires as the input parameter the previous resources and the new. So there are 4 input parameters: previous CPU cores, previous memory units, new CPU cores and new memory units.

The "on\_dependency\_scale" hook requires the IP address of the dependee tier and new allocation resources. So the input parameters are: the dependee tier name, the dependent tier name and stringified allocation JSON.

\section{Algorithms}
\subsection{Processing list of actions}
\begin{figure}[ht]
  \centering
    \includegraphics[height=550px,natwidth=219,natheight=635]{./pictures/algorithm1}
    \caption{Processing list of actions }
\end{figure}
The algorithm figure 4.1 describes the workflow in processing the list of actions for containers and processing hooks. First of all the tiers that depends on other tiers should be processed only after the dependee. To support this we build the dependency graph and process tiers without dependencies first and after remove them from the graph.

The second is the order of processing a tier. We should first create containers, after we can do update and only after that we can do remove. This is done to have more resources than it required for the tiers, otherwise the monitor component may detect that applications needs more resources.

All the hooks (scale-hooks and tier-hooks) are run only they are specified for this tier.

\subsection{Processing plan}
\begin{figure}[ht]
  \centering
    \includegraphics[height=550px,natwidth=202,natheight=714]{./pictures/algorithm2}
    \caption{Processing the plan }
\end{figure}
Processing the plan is specified in the figure 4.2.
While processing the plan, the plan "app -\textgreater{} tier -\textgreater{} demand" is flatten to the form "app\_tier -\textgreater{} demand".
If the demand is not equal to allocation than the ILP formulation is built and solved. As the result we get the new resource allocation considering VMs and containers with resources on them.
The procedure "process list of actions" is described in the figure 4.1.

\clearpage
\subsection{Translation}

The goal of the translation is to express the new allocation as the list of actions considering the current allocation. The set of possible actions considering we the container with the name "app\_tier" and the VM with the IP "127.0.0.1":
\begin{itemize}
    \item Create VM '127.0.0.1'
    \item Delete VM '127.0.0.1'
    \item Create container 'app\_tier' on the VM '127.0.0.1' with cpu=2 and mem=2.5gb
    \item \begin{sloppypar}Update container 'app\_tier' on the VM '127.0.0.1' set cpu=1 and mem=1gb\end{sloppypar}
    \item Run scale\_hooks for the container 'app\_tier' on the VM '127.0.0.1'
    \item Delete container 'app\_tier" on the VM '127.0.0.1'
    \item Run tier\_hooks for the container 'app\_tier' on the VM '127.0.0.1'
\end{itemize}

The algorithm for this translation is:
\begin{lstlisting}[caption=Translation algorithm]
FOR ALL vm IN current_allocation:
  IF vm IN new_allocation:
    FOR ALL container IN current_allocation[vm]:
      IF not container IN new_allocation[vm]:
        push_delete_container_action
  ELSE:
    push_delete_vm_action
FOR ALL vm IN new_allocation:
  IF vm IN current_allocation:
    FOR ALL container IN new_allocation[vm]:
      IF container IN old_allocation[vm]:
        push_update_container_action_if_needed
        push_run_scale_hooks_if_needed
      ELSE:
        push_create_container_action
        push_run_scale_hooks_if_needed
  ELSE:
    push_create_vm_action
    FOR ALL container IN new_allocation[vm]:
      push_create_container_action
      push_run_scale_hooks_if_needed
\end{lstlisting}

Scale-hooks run after each container update / create. Tier-hooks can be handled only considering the dependency graph. The executor adds tier-hooks to the list of actions.

\section{Monolithic vs Hierarchical approach}

\begin{figure}[ht]
  \centering
    \includegraphics[width=350px,natwidth=684,natheight=687]{./pictures/arch-hierachical}
    \caption{Hierarchical Executor}
\end{figure}

The current approach with solving a plan using ILP formulation we call the "monolithic" approach. Considering that all the implementation is run in the MAPE framework environment for the monolithic approach we have a global planner. Having a global planner has some drawbacks, as the control loop timeout should be at least the time required to run the plan (considering the VM creation take a long time for our test it was approximately 150s).

To go around large control loop timeouts we decided that MAPE components can be hierarchically spread among all the VMs and containers. So instead of having the one global planner, we want to have planner for each container. This we call the hierarchical approach.

In the hierarchical approach for each container there is the planner which produce the plan for its own container. To handle the VM constraints in CPU cores and memory, the VM has another planner that consolidates all containers plan and resolves this constraints. In the future work we are thinking to have the CPU requirement in the plan as the float number, this simplifies the constraint resolving considering each plan.

After plans are built for each VM, it hierarchically goes to the root planner and the root planer provides the plan to the master node executor. The master node executor accepts the plan, but it do not run it until the plans from each VM have arrived. This is because of the tier-hooks. To support tier-hooks we need to execute actions in the particular order (calculated by the dependency-graph), so we can not run some actions, until we know all the actions we need to run.

\section{Drivers}
There were implemented several drivers for managing virtual / cloud infrastructure: Vagrant, AWS, Docker.

As vagrant is an interface (a facade) to large list of different virtualization providers and AWS also supports very large list of vitalization (even Docker containers). This 3 cases covers a lot of different virtualization techniques.

\subsection{Vagrant driver}
Vagrant was used mostly during the implementation stage of the work to test everything locally without paying for Amazon AWS infrastructure.

The image for Vagrant is based on the "ubuntu/trusty64" image with docker and "Ecoware executor agent" installed.

All ports that are required for tiers should be preconfigured in the Vagrantfile. The application "Ecoware executor agent" uses the port 8000, which should be also preconfigured in the Vagrant file. The Vagrant configuration files can be found in the repository.

Vagrant is something like an interface or a facade to list of the virtualization providers. As providers Vagrant supports:
\begin{itemize}
    \item VirtualBox
    \item VMware
    \item Hyper-V
    \item Docker
    \item You can develop your own plugin for custom provider
\end{itemize}
Using Vagrant helps to abstract from different providers and make it easy to change providers without changing our driver.

To manage Vagrant our driver uses the python module \textit{subprocess} to call OS commands like \textit{vagrant up}

\subsection{AWS driver}
For implementing AWS driver it were tested 3 different ways to communicate with AWS:
\begin{enumerate}
    \item HTTP REST calls
    \item AWS management console
    \item Python AWS SDK boto3
\end{enumerate}
As all the code for the paper was written in the python, it was decided that using boto3 is the simplest and cleanest way.

In current implementation it was easier to use the AWS Auto Scaling Groups to create VMs. So creating VMs now is based on the "desired capacity" method for Auto Scaling Groups in AWS. So the workflow for creating VMs now is:
\begin{enumerate}
    \item Get required VM number from ILP solution (n).
    \item Set desired capacity for the Auto Scaling Group to n.
    \item Poll number of instances in the Auto Scaling Group until it equals to n.
    \item Poll the status of all instances of the Auto Scaling Group to be the "in service".
    \item Get IP addresses of all VMs.
    \item Analyse the previous allocation, the new allocation and separate existed machines with the new.
    \item Set the topology to the new VMs, on error sleep 10s and repeat.
\end{enumerate}

The "polling" in this context means to make the request to the AWS and if the result is not satisfactory, sleep 10s, and repeat the request again.

Creating the VM takes approximately 150s.

For the deletion of the VM we can not just set the desired capacity to the smaller number, because we need to delete some particular machine, but not any one from all machines. To remove particular machine we just do 2 actions:
\begin{enumerate}
    \item Detach the instance from the autoscaling group.
    \item Terminate the instance
\end{enumerate}

The AWS infrastructure supports different virtualization techniques:
\begin{itemize}
    \item HVM
    \item PV
    \item Docker containers
\end{itemize}
HVM stands for Hardware-Assisted Virtualization.
PV stands for Paravirtualization.
In spite of the fact that these virtualization techniques are different and have its own advantages and disadvantages, AWS helps us to abstract from this and use its unified API.

To configure AWS for using with AWS driver it is required to do some steps:
\begin{description}
    \item [AMI.] The AMI should contain the \textit{Ecoware Executor Agent} and docker installed. The \textit{Ecoware Executor Agent} and docker engine should be in the autostart. All the containers images that will be used for different tiers should be pulled in the docker. Also all the containers should be removed to not have the name collision.
    \item [Security Group.] Security Group should be configured considering all the ports that containers want to listen to and the \textit{Ecoware Executor Agent} default port 8000.
    \item [Launch Configuration.] Launch Configuration should be created considering the AMI we want to use, the security group, and the AWS instance type.
    \item [Auto Scaling Group.] Auto Scaling Group should be created using the Launch Configuration with the initial instance number equals to 0.
\end{description}

\subsection{Docker driver}
Besides the fact that both Vagrant and AWS support Docker containers as under layer virtualization technique (or the provider), we use docker without any facade or interface, but directly. This helps to simplify and separate the separation between fine-grained and coarse-grained virtualization.

Docker is managed as the Vagrant using OS calls with the python module \textit{subprocess}.

\begin{sloppypar}The command to run have this structure (example of the run action):\end{sloppypar}

\begin{lstlisting}[caption=Docker run template,basicstyle=\small]
docker run -itd {PORTS} {HOSTS} --cpuset-cpus={CPUS}
  -m={MEM}m --name={NAME} -v=/ecoware:/ecoware {IMAGE}
  {ENTRY_PARAMS}
\end{lstlisting}

\begin{lstlisting}[caption=Docker run example,basicstyle=\small]
docker run -itd -p 8080:5000 --add-host="db:172.31.31.123"
  --cpuset-cpus=0,1 -m=512m --name=rubis_app_server
  -v=/ecoware:/ecoware pwitter-web -w 3 -k eventlet
\end{lstlisting}

\textit{-it} option means to allocate a tty for the container process (to run it interactively).\\
\textit{-d} option means to run container in the \textit{detached} mode (background mode).\\
The variable \textit{\{PORTS\}} is built considering ports specified in the topology.\\
The variable \textit{\{HOSTS\}} is built if the tier has dependency n-to-1 or 1-to-1.\\
Variables \textit{\{CPUS\}} and \textit{\{MEM\}} is resources demand from the plan.\\
The variable \textit{\{NAME\}} is built from the topology like\\ "\{APP\_NAME\}\_\{TIER\_NAME\}".\\
\textit{-v} option means to add a data volume. The format is host\_dir:guest\_dir. The name of the directory is hardcoded, this is directory where the git repository with hooks is pulled.\\
The variable \textit{\{IMAGE\}} is taken from the topology.\\
The variable \textit{\{ENTRY\_PARAMS\}} is the entry point params also taken from the topology.

\section{Unit testing}
Some of the algorithms can be easily tested with unit-testing.

For example, the ILP algorithm is quite weight sensitive and tuning the weights can break some logic. Moreover there are many different ways how the system can satisfy the new plan and sometimes it is not clear and obvious which way is more optimal. So for the ILP solver it was written a list of the tests.

The input for the test is 3 json files: "plan.json", "result.json" and "allocation.json". So the iLP solver is tested that given the current allocation and the plan, it produces the new allocation equals to specified in the "result.json".

Any arguable ILP solution can be inspected in details and added to the tests.

Also algorithms like "building the dependency graph", "translation from allocation to actions" are tested too.

\clearpage
\section{UML}
\begin{figure}[ht]
  \centering
    \includegraphics[width=350px,natwidth=633,natheight=482]{./pictures/implementation-main}
    \caption{Class diagram for the executor on the Main Node}
\end{figure}

\begin{figure}[ht]
  \centering
    \includegraphics[width=350px,natwidth=643,natheight=190]{./pictures/implementation-agent}
    \caption{Class diagram for the executor on the Agent Node}
\end{figure}

\begin{figure}[ht]
  \centering
    \includegraphics[height=450px,natwidth=433,natheight=593]{./pictures/sequence}
    \caption{Sequence diagram}
\end{figure}

\clearpage
\section{Examples}
\subsection{The monolithic executor}
For example we have initially empty system without running VMs and containers. The topology of our applications is the same as in the listing 4.4. The executor is running locally on the port 8000: http://localhost:8000/. The autoscaling group "monolithic\_ex\_8cpu " is configured and it uses VMs with 8 CPU cores and 32gb of RAM (64 RAM units).

As the HTTP client we can consider the unix tool "curl".

First we set the topology

\texttt{curl -X PUT -d @topology.json \$endpoint}\\
where "topology.json" file contains the same json as in the listing 4.4 and \$endpoint is
\texttt{http://localhost:8000/api/topology}

We can check the new topology opening in any browser \\ \texttt{http://localhost:8000/api/topology}.

For example now we want to start only the rubis application, giving 4 CPU cores and 4gb of RAM to "app\_server" and 1 CPU core and 4gb of RAM to the "db" tier.

For this example the payload.json" should be:

\begin{lstlisting}[caption=payload.json,basicstyle=\tiny]
{
  "rubis": {
    "app_server": {
      "cpu_cores": 4,
      "mem_units": 8
    },
    "db": {
      "cpu_cores": 1,
      "mem_units": 8
    }
  }
}
\end{lstlisting}

First, we can check which actions would be run for this plan by calling endpoint for translation. This call will not run these actions.

\texttt{curl -X PUT -d @payload.json \$endpoint}\\
the \$endpoint is \texttt{http://localhost:8000/api/translate}

After that we can execute the plan by calling the execute endpoint:

\texttt{curl -X PUT -d @payload.json \$endpoint}\\
where \$endpoint is \texttt{http://localhost:8000/api/execute}

We can check new allocation by calling these endpoints from the browser: \texttt{http://localhost:8000/api/allocation} and \\ \texttt{http://localhost:8000/api/inspect}.

After the planner decides to change the allocation of the containers it should just send to /execute/ endpoint the new payload with the new resources demand.


\documentclass{article}

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{hyperref}
\usepackage{parskip}

\newcommand{\KL}[2]{\mathrm{KL}(#1~\|~#2)}

\begin{document}

Suppose we wish to estimate $p(y \mid x)$ when we observe $(x_i, y_i) \sim p(y
\mid x) p_1(x) $ but care about achieving high accuracy under some other $p(x
\mid x) p_2(x)$ (the \emph{covariate shift} problem). Suppose further that we
have some information about $y$ under the target distribution---namely that we
can sample from $p_2(y) := \sum_x p_2(x) p(y \mid x)$. How can we use this
information to find some $p_\theta(y \mid x)$ that maximizes $E_{x \sim p_2(x)} p(x
\mid y)$?

Idea: optimize
\begin{align}
  \min_{\theta,\eta} ~~ &-\sum_i \log p_\theta(y_i \mid x_i) + \lambda~\KL{p_2}{p_\eta(x)
  p_\theta(y \mid x)}
\end{align}
This can be viewed as a form of posterior regularization and specifically
\textbf{expectation regularization} as in
\url{https://www.aclweb.org/anthology/D19-1004/}.
\begin{align}
  &= -\sum_i \log p_\theta(y_i \mid x_i) + \lambda H(p_2) - \lambda E_{y \sim p_2(y)}
  [\log E_{x \sim p_\eta} p_\theta(y \mid x)] \\
  &\leq -\sum_i \log p_\theta(y_i \mid x_i) + \lambda H(p_2) -  \lambda E_{y \sim p_2(y)}
  E_{x \sim p_\eta} \log p_\theta(y \mid x) \\
  \intertext{(Jensen)}
  \min_{\theta, \eta, \phi} ~~ &\leq -\sum_i \log p_\theta(y_i \mid x_i) + \lambda H(p_2) -  \lambda E_{y \sim p_2(y)}[
  E_{x \sim q_\phi(x \mid y)} [\log p_\theta(y \mid x)] -\KL{q_\phi(x \mid y)}
  {p_\eta(x)}]
\end{align}
(ELBO)

Important special case of this model: \textbf{backtranslation}, where we assume
that $q(x \mid y)$ is the same under $p_1$ and $p_2$ and estimate it directly
from training data. From this perpsective, backtranslation can be viewed not
just as amortizing a noisy channel translation model but actually providing
robustness to the covariate shift implied by any difference between $p(y)$ on
bilingual and monolingual data.

Claim: we can get lots of ``NLP-flavored'' inductive biases in this framework
just by setting $p(y)$ uniform over an appropriate set. Example:

\paragraph{Mutual exclusivity}
Let $\pi$ be a permutation on $[n]$ and $(x_i, y_i)$ be one-hot vectors $(e_i,
e_{\pi(i)}$ for $i \in [n-1]$. Prove that a regularized logistic regression
model has $p(y \mid e_{n-1})$ uniform over classes when trained via MLE but
mostly on $e_{\pi(n-1)}$ when trained as in Equation 1.

\paragraph{Compositionality}
As above, but let each $x_i$ be a concatenation of two one-hot vectors and $y_i$
be the concatenation of their permuted values. Prove that if we hold out $k$ of
these vectors, Equation 1 still recovers the correct values but MLE doesn't.

\paragraph{Productivity}
Something about learning a weighted string transducer.

\end{document}


\subsection{Branching Properties}
$\wp_\Lambda$ has a unique pole in $\mathbb C/\Lambda$ of order $2$.
The other ramification points are at the zeros of $\wp_\Lambda^\prime$.
Recall that
$$\wp_\Lambda^\prime(z)=\sum_{\omega\in\Lambda}\frac{-2}{(z-\omega)^3}$$
which is an odd function with degree $3$ and poles exactly at the lattice points.
For any $\omega\in\Lambda$, we have $\wp_\Lambda^\prime(\omega/2)=\wp_\Lambda^\prime(\omega/2-\omega)=\wp_\Lambda^\prime(-\omega/2)=-\wp_\Lambda^\prime(\omega/2)$ as $\wp_\Lambda^\prime$ is odd, so $\omega/2$ is either a zero or a pole.
So in the period parallelogram $P$, there are at least three zeros (up to $\Lambda$) namely $\omega_1/2,\omega_2/2,(\omega_1+\omega_2)/2$.
But $\deg\wp_\Lambda^\prime=3$, so these are all the zeros and all of them are simple.
\begin{remark}
    $\wp_\Lambda$ has $4$ ramification points in $\mathbb C/\Lambda$, namely $0,\omega_1/2,\omega_2/2,(\omega_1+\omega_2)/2$ each with multiplicity $2$.
    So by the valency theorem, they must have distinct images $\infty=\wp_\Lambda(0),e_1=\wp_\Lambda(\omega_1/2),e_2=\wp_\Lambda(\omega_2/2),e_3=\wp_\Lambda((\omega_1+\omega_2)/2)$.
\end{remark}
\begin{remark}
    By plugging in, this is consistent with Riemann Hurwitz as $\mathbb C/\Lambda$ has genus $1$.
\end{remark}