\documentclass{article}
\usepackage{theorem}

\newcommand{\implies}{\Rightarrow}
\newcommand{\have}{{.~}}
\newcommand{\bstate}{{\mbox{\cal S}}}
\newcommand{\blocks}{{\mbox{\cal B}}}
\newcommand{\tabtop}{{\mbox{\sc t}}}
\newcommand{\tblocks}{{\blocks_\tabtop}}
\newcommand{\bclear}{{\mbox{\sc clear}}}
\newcommand{\bon}{{\mbox{\sc on}}}
\newcommand{\st}{~|}

\begin{document}

\section{Definitions}

\begin{itemize}
\item Let
  $$ \blocks = \{ 1, \ldots, N \} $$
represent a set of $N$ blocks in a blocks-world problem.
We represent the table top with the symbol $\tabtop$, and
define $$ \tblocks \equiv \blocks \cup \{ \tabtop \} $$

\item Call any
  $$ \bstate \subseteq \blocks \times \tblocks $$
a {\em part-state} of the blocks-world problem. The pairs of blocks in
$\bstate$ represent {\em on} relations.

\item A part-state $\bstate$ is {\em legal} iff
  \begin{itemize}
  \item No block is on two things: $$
    \forall i \in \blocks \have
      \langle i , j \rangle \in \bstate \implies
      \not \exists j' \in \tblocks \have
	j \neq j' \wedge \langle i , j' \rangle \in \bstate
  $$
  \item No two blocks are on the same block: $$
    \forall j \in \blocks \have
      \langle i , j \rangle \in \bstate \implies
      \not \exists i' \in \blocks \have
	i \neq i' \wedge \langle i' , j \rangle \in \bstate
  $$
  \end{itemize}

\item A part-state is a {\em total state} or just a {\em state}
iff it is legal and every block is on something: $$
    \forall i \in \blocks \have
      \exists j \in \tblocks \have
	\langle i , j \rangle \in \bstate
$$

\item A legal part-state $\bstate^1$ is {\em simpler} than
a legal part-state $\bstate^2$ iff every block in $\bstate^2$
is either on the same block as on $\bstate^1$ or on the
table: $$
  \forall \langle i, j \rangle \in \bstate^2 \have
    j = \tabtop ~\vee~ \langle i, j \rangle \in \bstate^1
$$
We will justify this terminology in Lemma \ref{simpler-states}
below.

\item A block $j$ is {\em clear} in a part-state $\bstate$ iff no block
is above it: $$
  \not \exists i \in \blocks \have \langle i , j \rangle \in \bstate
$$
We will define the set of blocks clear in a part-state $\bstate$ by $$
  \bclear(\bstate) = \{ j \in \bstate \st
  \not \exists i \have \langle i , j \rangle \in \bstate \}
$$

\item A block $i$ is {\em above} a block $i'$ in a
part-state $\bstate$ if there is a sequence
of blocks whose {\em on} relationships lead from $i$ to $i'$: $$
  \exists \{ j_1 , j_2 , \ldots , j_k \} \subseteq \blocks \have
    \{ \langle i , j_1 \rangle , \langle j_1 , j_2 \rangle ,
       \ldots , \langle j_k , i' \rangle \} \subseteq \bstate
$$

\item A part-state $\bstate$ is {\em realizable} iff it
is legal and no block is above itself.

\item {\em Moves} are functions $m_{ij}$ which take a realizable part-state in
which blocks $i$ and $j$ are clear to a
realizable part-state in which $i$ is on $j$.

Let $i \in \blocks$ and $j \in \tblocks$, and let $C(ij)$ be the set
of realizable part-states
in which $i$ is clear, and either $j = \tabtop$ or $j$ is clear.
Then the domain of $m_{ij}$ is
$C(ij)$, and the range is the set of all realizable part-states.

Define the partial functions
$\bon_i(\bstate)$ mapping legal part-states to
blocks such that
$\bon_i(\bstate) = j$ when $\langle i, j \rangle \in \bstate$;
$\bon_i$ is undefined elsewhere.
(In other words, $\bon_i$ says
which block block $i$ is on in a given state.)
By the definition of a legal part-state, these functions are well-defined.
We then define
$m_{ij}$ by $$
  m_{ij}(\bstate) \equiv
    \left ( \bstate - \{ \langle i, \bon_i(\bstate) \rangle \} \right )
    \cup \{ \langle i, j \rangle \}
$$
when $i$ and $j$ are clear in $\bstate$; $m_{ij}$ is undefined
elsewhere.  A move is {\em legal} iff $m_{ij}$ is defined.

\item A {\em move sequence} $M$
of length $k$ from state $\bstate^0$ to state $\bstate^1$ consists of
a sequence of pairs $$
  \langle i_1, j_1 \rangle , \langle i_2 , j_2 \rangle , \ldots ,
  \langle i_k , j_k \rangle \in \left ( \blocks \times \tblocks \right ) ^ \ast
$$
such that $$
  \left ( m_{i_k j_k} \circ \cdots \circ m_{i_2 j_2} \circ
  m_{i_1 j_1} \right ) (\bstate^0) = \bstate^1
$$
We allow ourselves the liberty of writing $M(\bstate^0) = \bstate^1$.
A move sequence $M$ is {\em legal} for a state $\bstate$ iff
$M(\bstate)$ is defined.

\item A {\em primitive-blocks-world} (PBW) {\em problem} consists of two
realizable states: an
initial state $I$ and a goal state $G$.
A {\em solution} to the problem is a move sequence $M$ such that $M(I) = G$.
We say that a solution is {\em optimal} if there is no solution
of shorter length.

\item A {\em stack} is a sequence of blocks $i_1, i_2, \ldots, i_k$
in a realizable
part-state $\bstate$ such that $$
  \{ \langle i_1, i_2 \rangle, \langle i_2, i_3 \rangle, \ldots,
  \langle i_{k-1}, i_k \rangle \} \subseteq \bstate
$$
and such that $$
  \forall i \in \{ i_1, i_2, \ldots, i_{k-1} \} \have
    i \neq \tabtop
$$
A {\em tower} is a stack such that $i_k = \tabtop$.

\end{itemize}

\section{Solution Optimization}

In the search for an algorithm giving optimal solutions for PBW
problems, we make use of the following lemmas, all of which
have been proved previously elsewhere.  All of these lemmas are
proved using essentially the same reasoning:  If $\bstate$ is a
solution, and we can construct from $\bstate$ a solution $\bstate'$ which
is the same as $\bstate$ except for deletion of one or more elements,
then $\bstate$ is not an optimal solution.

\begin{lemma}{simpler-states}{Simpler States}
\given
States $\bstate^1$ and $\bstate^1$ such that $$
\bclear(\bstate^1) \subseteq \bclear(\bstate^2)
$$
A legal move sequence $M$ starting at $\bstate^1$.
\prove
$M$ is also a legal move sequence starting at $\bstate^2$.
\proof
Consider the states just after the first move $\langle i, j \rangle$ in $M$.
\begin{eqnarray*}
  \bstate^1_1 = m_{i j}(\bstate^1) &=&
  \left ( \bstate^1 - \{ \langle i, \bon_i(\bstate^1) \rangle \} \right )
  \cup \{ \langle i, j \rangle \} \\
  \bstate^2_1 = m_{i j}(\bstate^2) &=&
  \left ( \bstate^2 - \{ \langle i, \bon_i(\bstate^2) \rangle \} \right )
  \cup \{ \langle i, j \rangle \}
\end{eqnarray*}
This first move is legal, since it is legal in $\bstate^1$ and
$\bclear(\bstate^1) \subseteq \bclear(\bstate^2)$, and the set of
clear blocks after the move has the property that
\begin{eqnarray*}
  \bclear(\bstate^1_1) &=& \bclear (
  \left ( \bstate^1 - \{ \langle i, \bon_i(\bstate^1) \rangle \} \right )
  \cup \{ \langle i, j \rangle \} ) \\
  &=&  \left ( \bstate^1 - \{ \langle i, \bon_i(\bstate^1) \rangle \} \right )
  - j
\end{eqnarray*}
\end{lemma}

\begin{lemma}{table-to-table}{Table-To-Table Moves}
\given
A PBW problem $\langle I, G \rangle$.
An optimal solution $M$ to $\langle I, G \rangle$.
\prove
No block moves from the table to the table in $M$.
\proof
Assume the contrary of the lemma, i.e., that there is
some table-to-table move in $M$.
Use this move to split $M$
$$
  M = M^1 \cdot \langle i, \tabtop \rangle \cdot M^2
$$
where $$
  \bon_i(M^1(I)) = \tabtop
$$
Let $\bstate^1 = M^1(I)$.
Applying the definitions of the previous section yields
\begin{eqnarray*}
  (\langle i, \tabtop \rangle)(\bstate^1)
  &\equiv& m_{i \tabtop}(\bstate^1) \\
  &=& \left ( \bstate^1 - \{ \langle i, \bon_i(\bstate^1) \rangle \} \right )
  \cup \{ \langle i, \tabtop \rangle \} \\
  &=& \left ( \bstate^1 - \{ \langle i, \tabtop \rangle \} \right )
  \cup \{ \langle i, \tabtop \rangle \} \\
  &=& \bstate^1
\end{eqnarray*}
This says that the state at the start of the table-to-table
move is the same as the state at the end, and thus by the
definition of a solution, the sequence obtained by deleting
this move from $M$ is also a solution.  But by the definition
of optimality $M$ is therefore not optimal, and thus we have
a condradiction.

Therefore, no such move can be in $M$.
\end{lemma}

\begin{lemma}{tabled}{Correctly Tabled Blocks}
\given
A PBW problem $\langle I, G \rangle$, such that
a block $i$ is on the table in both $I$ and $G$.
An optimal solution $M$ to $\langle I, G \rangle$.
\prove
$i$ doesn't move in $M$, i.e. $$
  \not \exists j \in \tblocks \have
    M = \ldots, \langle i, j \rangle, \ldots
$$
\proof
Assume the contrary, i.e., that $i$ moves in $M$.

We first split $M$ such that $$
  M = M^1 \cdot \langle i, j \rangle \cdot M^2 \cdot
  \langle i, \tabtop \rangle \cdot M^3
$$
where $M^1$ contains no moves of $i$, and
$M^2$ contains no moves of $i$ to $\tabtop$.
To see that this split is possible, we first note
that since by assumption $i$ moves in $M$, the first
move in the split exists.  Further, by Lemma
\ref{table-to-table}, $j \neq \tabtop$.
Let \begin{eqnarray*}
\bstate^1 &=& M^1(I) \\
\bstate^2 &=& (\langle i, j \rangle)(\bstate^1) \\
\bstate^3 &=& (M^2 \cdot \langle i, \tabtop \rangle)(\bstate^2)
\end{eqnarray*}
Since
$\bon_i(G) = \tabtop$, but $\bon_i(\bstate^2) \neq \tabtop$,
then the second move in the split exists as well.

Now consider the subproblem $\langle \bstate^1, \bstate^3$ and its known
solution $$
  \langle i, j \rangle \cdot M^2 \cdot \langle i, \tabtop \rangle
$$
We will construct a shorter solution to the problem from this
solution.

For any solution $M$ to $\langle I, G \rangle$,
let $M_i$ be the sequence of moves obtained as follows:
Consider any move $m = \langle j, i \rangle$ in $M$, and the
state $\bstate$ just before that move.  Replace $m$ with the two-move
sequence $$
  m' = \langle i, tabtop \rangle, \langle j, \bon_i(\bstate) \rangle
$$
We note that $M_i$ is still a sequence of
legal moves by Lemma \ref{simpler-states}.

\end{lemma}

\end{document}


% Created 2019-09-14 Sat 01:34
% Intended LaTeX compiler: pdflatex
\documentclass[11pt]{article}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{grffile}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{textcomp}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\author{Subhajit Sahu (2018801013)}
\date{\today}
\title{Arithmetic Lang}
\hypersetup{
 pdfauthor={Subhajit Sahu (2018801013)},
 pdftitle={Arithmetic Lang},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 26.3 (Org mode 9.1.9)},
 pdflang={English}}
\begin{document}

\maketitle
\tableofcontents

\section{Imports}
\label{sec:orgdbb9a84}

I needed to use \texttt{isPrefixOf} for doing string replacement. It is part of
module \texttt{Data.List}. In the \texttt{main} function, using \texttt{putStr} does not flush
the string to \texttt{stdout} immediately. For that, module \texttt{System.IO} nneded
to be imported.

\begin{verbatim}
import Data.List
import System.IO
\end{verbatim}

\section{Values}
\label{sec:org0f91885}

Unlike the arithmetic language interpreter (using racket), discussed in
class, this arithmetic lang needs to support both numbers and booleans.
To support that, i used a boxing type, here called \texttt{Value}. To keep the
code simple, i made it an instance of \texttt{Eq}, \texttt{Show}, \texttt{Num}, and \texttt{Fractional}.
Nummbers are called \texttt{Numv}, and booleans \texttt{Boolv}, to distinguish them from
builtin typeclass / type.

\begin{verbatim}
data Value =
  Numv  Float |
  Boolv Bool
  deriving (Eq)

instance Show Value where
  show (Numv x)  = show x
  show (Boolv x) = show x

instance Num Value where
  (Numv x) + (Numv y) = Numv $ x + y
  (Numv x) * (Numv y) = Numv $ x * y
  abs (Numv x) = Numv $ abs x
  signum (Numv x) = Numv $ signum x
  fromInteger x = Numv $ fromInteger x
  negate (Numv x) = Numv $ negate x

instance Fractional Value where
  (Numv x) / (Numv y) = Numv $ x / y
  fromRational x = Numv $ fromRational x
\end{verbatim}

\section{Abstract Syntax Tree}
\label{sec:org187dc02}

The AST is as expected, and again numbers and booleans are called differently.

\begin{verbatim}
data Ast =
  Numa   Float   |
  Boola  Bool    |
  Add    Ast Ast |
  Mul    Ast Ast |
  Sub    Ast Ast |
  Div    Ast Ast |
  Equals Ast Ast |
  IsZero Ast
  deriving (Eq, Read, Show)
\end{verbatim}

\section{Run}
\label{sec:org36d132d}

There is a \texttt{main} function which provides the REPL. It simply accepts a line
and shows the output \texttt{Value} of \texttt{run} function. Use an empty (null) line to
terminate. After displaying the "arithmetic: " prompt, it was necessary to
flush to \texttt{stdout}.

\begin{verbatim}
main = do
  putStr "arithmetic: "
  hFlush stdout
  exp <- getLine
  if null exp
    then return ()
    else do
      putStrLn (show . run $ exp)
      main
\end{verbatim}

The run function is simply parses and evaluates a string.

\begin{verbatim}
run :: String -> Value
run = eval . parse
\end{verbatim}

\section{Evaluator}
\label{sec:org48765be}

The \texttt{eval} function pattern matches with all constructors of AST, and does the
desired action. It finally returns a \texttt{Value}. Since \texttt{Value} is already an instance
of \texttt{Eq}, \texttt{Num}, \texttt{Fractional} we can directly use arithmetic operators. However,
equals operator is an exception, since it always returns a \texttt{Bool} and thus it
had to be boxed in a \texttt{Boolv}.

\begin{verbatim}
eval :: Ast -> Value
eval (Numa  x) = Numv  x
eval (Boola x) = Boolv x
eval (Add x y) = (eval x) + (eval y)
eval (Mul x y) = (eval x) * (eval y)
eval (Sub x y) = (eval x) - (eval y)
eval (Div x y) = (eval x) / (eval y)
eval (Equals x y) = Boolv $ (eval x) == (eval y)
eval (IsZero x)   = Boolv $ (eval x) == Numv 0
\end{verbatim}

\section{Parser}
\label{sec:org1edeaab}

I wanted to depend upon the \texttt{read} function to generate the AST. But a simple
\texttt{read} on the string would not work since it would read numbers as \texttt{Num} and
not \texttt{Numa} (AST needs that). Same is true for the boolean values. It would
also not work with operators.

However if the input string is transformed into a suitable format, using our
AST constructors, read should work fine. So, as you can see we split the
input into words, transform it, and rejoin it back, before feeding it
back to read, which then directly gives us the AST.

Also, since brackets and operators, or brackets and numbers are not separated
by a space, it causes problems for word splitting, so it needed to be done
beforehand.

\begin{verbatim}
parse :: String -> Ast
parse s = (read . unwords . map token . words $ bpad) :: Ast
  where bpad = replace "(" " ( " . replace ")" " ) " $ s
\end{verbatim}

Here is the token replacement strategy.

\begin{verbatim}
token :: String -> String
token "+" = "Add"
token "*" = "Mul"
token "-" = "Sub"
token "/" = "Div"
token "=" = "Equals"
token "zero?" = "IsZero"
token t
  | isFloat t  = "(Numa "  ++ t ++ ")"
  | isBool  t  = "(Boola " ++ t ++ ")"
  | otherwise  = t
\end{verbatim}

And, here are a few utility functions we are using.

\begin{verbatim}
replace :: (Eq a) => [a] -> [a] -> [a] -> [a]
replace _ _ [] = []
replace from to all@(x:xs)
  | from `isPrefixOf` all = to ++ (replace from to . drop (length from) $ all)
  | otherwise             = x : replace from to xs

isFloat :: String -> Bool
isFloat s = case (reads s) :: [(Float, String)] of
  [(_, "")] -> True
  _         -> False

isBool :: String -> Bool
isBool s = case (reads s) :: [(Bool, String)] of
  [(_, "")] -> True
  _         -> False
\end{verbatim}

\section{This is where you put it all together}
\label{sec:orgfadf544}

\begin{verbatim}
import Data.List
import System.IO

data Value =
  Numv  Float |
  Boolv Bool
  deriving (Eq)

instance Show Value where
  show (Numv x)  = show x
  show (Boolv x) = show x

instance Num Value where
  (Numv x) + (Numv y) = Numv $ x + y
  (Numv x) * (Numv y) = Numv $ x * y
  abs (Numv x) = Numv $ abs x
  signum (Numv x) = Numv $ signum x
  fromInteger x = Numv $ fromInteger x
  negate (Numv x) = Numv $ negate x

instance Fractional Value where
  (Numv x) / (Numv y) = Numv $ x / y
  fromRational x = Numv $ fromRational x

data Ast =
  Numa   Float   |
  Boola  Bool    |
  Add    Ast Ast |
  Mul    Ast Ast |
  Sub    Ast Ast |
  Div    Ast Ast |
  Equals Ast Ast |
  IsZero Ast
  deriving (Eq, Read, Show)

main = do
  putStr "arithmetic: "
  hFlush stdout
  exp <- getLine
  if null exp
    then return ()
    else do
      putStrLn (show . run $ exp)
      main

run :: String -> Value
run = eval . parse

eval :: Ast -> Value
eval (Numa  x) = Numv  x
eval (Boola x) = Boolv x
eval (Add x y) = (eval x) + (eval y)
eval (Mul x y) = (eval x) * (eval y)
eval (Sub x y) = (eval x) - (eval y)
eval (Div x y) = (eval x) / (eval y)
eval (Equals x y) = Boolv $ (eval x) == (eval y)
eval (IsZero x)   = Boolv $ (eval x) == Numv 0

parse :: String -> Ast
parse s = (read . unwords . map token . words $ bpad) :: Ast
  where bpad = replace "(" " ( " . replace ")" " ) " $ s

token :: String -> String
token "+" = "Add"
token "*" = "Mul"
token "-" = "Sub"
token "/" = "Div"
token "=" = "Equals"
token "zero?" = "IsZero"
token t
  | isFloat t  = "(Numa "  ++ t ++ ")"
  | isBool  t  = "(Boola " ++ t ++ ")"
  | otherwise  = t

replace :: (Eq a) => [a] -> [a] -> [a] -> [a]
replace _ _ [] = []
replace from to all@(x:xs)
  | from `isPrefixOf` all = to ++ (replace from to . drop (length from) $ all)
  | otherwise             = x : replace from to xs

isFloat :: String -> Bool
isFloat s = case (reads s) :: [(Float, String)] of
  [(_, "")] -> True
  _         -> False

isBool :: String -> Bool
isBool s = case (reads s) :: [(Bool, String)] of
  [(_, "")] -> True
  _         -> False
\end{verbatim}
\end{document}


\section{Sorts}
\label{sec-sort}

\subsection{Introduction}

The {\GF} logic is sorted, although you are not forced to make any
sort declarations.
If you never use any of the system's sort commands, you can use the
logic as if it were unsorted. {\GF} makes sure that everything works correctly
with appropriate default sorts
(see later in this section).

Sorts in {\GF} are specially-handled unary predicates: that is, aside from the
usual features of unary predicates, they possess some additional ones.
This choice won't be deeply justified here.
The resulting logic can, however, be proved to be
consistent and to allow more compact axiomatizations and proofs
than the unsorted one.

\subsection{Defining the sort hierarchy}
We present here the commands for the declaration of new sorts, for the
definition of the generality relations between sorts, for the definition
of a most general sort and for the declaration sort extensions.

The generality relations can be intuitively specified in this way:
stating that sort $S_1$ is {\it weakly more general} than sort $S_2$
is equivalent to
specifying an axiom of the form $\forall x(S_2(x) \imp S_1(x))$.
In this case we also say that $S_2$ is {\it weakly less general} than $S_1$.
Two sorts are said to be {\it equivalent} when they are mutually
(weakly) more general.
When $S_1$ is {\it weakly more (less) general} than $S_2$ and it is not
equivalent to it, then we say that $S_1$ is {\it strictly more (less)
general} than $S_2$.
Finally, we say that $S$ is a {\em most general sort} if it is weakly more
general than any sort. {\GF} has a default most general sort, {\tt
UNIVERSAL}, that is used when no explicit sort information is provided to
the system.

\subsection{Sorted declarations}

A sort is associated to every {\indsym} at the moment of its declaration.
In unsorted
declarations, when no sort is specified, the default most general sort
{\tt UNIVERSAL} is taken to be the sort of the symbol.

The sort information associated with a {\funsym} is a set of so-called
{\it fmaps},
that is of $n+1$-tuples of sorts, where $n$ is the arity of the \funconst.
The fmaps for $f$ specify the sort of a term whose external {\funsym} is
$f$, depending on the sort of its arguments. When a {\funsym} is declared
the $n+1$-tuple ({\tt UNIVERSAL} \ldots {\tt UNIVERSAL}) is always added to its
{\it fmaps}.

According to the sort information of symbols we give now the
definition of the useful notion of {\it sort of a term}:
we say that $t$ is a term of sort $S$
if and only if
%
\begin{itemize}
\item
  $t$ is an individual symbol, and its sort is weakly less general than $S$,
  or
\item
  $t=f(t_1,\dots,t_n)$, $f$ has an {\it fmap} of the form $(S_1,\dots,S_n,
  S_f)$, $S_f$ is weakly less general than $S$ and all $t_i$ are terms of sort
  $S_i$.
\end{itemize}


\chapter{Convergence of Random Variables}

% 1
\begin{ex}
  \begin{enumerate}[(a)]
    \item[]
    \item Note that since
          \begin{align*}
            \E{\overline{X}_n}
             & =\E{\frac{1}{n}\sum_{i=1}^n X_i}
            =\frac{1}{n}\sum_{i=1}^n \E{X_i}
            =\mu,                                 \\
            \var{\overline{X}_n}
             & =\var{\frac{1}{n}\sum_{i=1}^n X_i}
            =\frac{1}{n^2}\sum_{i=1}^n \var{X_i}
            =\frac{\sigma^2}{n},
          \end{align*}
          we have
          \begin{align*}
            \E{S_n^2}
             & =\E{\frac{1}{n-1}\sum_{i=1}^n(X_i-\overline{X}_n)^2}                                                           \\
             & =\frac{1}{n-1}\E{\sum_{i=1}^n\left[X_i^2-2X_i\overline{X}_n+\overline{X}_n^2\right]}                           \\
             & =\frac{1}{n-1}\E{\sum_{i=1}^n X_i^2-2\left(\sum_{i=1}^n X_i\right)\overline{X}_n+\sum_{i=1}^n\overline{X}_n^2} \\
             & =\frac{1}{n-1}\E{\sum_{i=1}^nX_i^2-2n\overline{X}_n^2+n\overline{X}_n^2}                                       \\
             & = \frac{n}{n-1}\left(\E{X_1^2}-\E{\overline{X}_n^2}\right)                                                     \\
             & = \frac{n}{n-1}\left(\var{X_1}+\E{X_1}^2-\var{\overline{X}_n}-\E{\overline{X}_n}^2\right)                      \\
             & = \frac{n}{n-1}\left(\sigma^2+\mu^2-\frac{\sigma^2}{n}-\mu^2 \right)                                           \\
             & =\sigma^2.
          \end{align*}
    \item We have
          \begin{align*}
            S^2
             & =\frac{1}{n-1}\sum_{i=1}^n(X_i-\overline{X}_n)^2                                  \\
             & =\frac{1}{n-1}\left(\sum_{i=1}^nX_i^2-2n\overline{X}_n^2+n\overline{X}_n^2\right) \\
             & =\frac{n}{n-1}n^{-1}\sum_{i=1}^nX_i^2-\frac{n}{n-1}\overline{X}_n^2,
          \end{align*}
          where $\frac{n}{n-1}\to 1$ as $n\to\infty$. Note that then
          $\overline{X}_n\to \mu$ in probability by the law of large numbers,
          and therefore by Theorem 5.5, part (d), $\overline{X}_n^2\to \mu^2$
          in probability. Likewise, we can apply the law of large numbers to
          conclude that $n^{-1}\sum_{i=1}^nX_i^2\xrightarrow{P} \E{X_1^2}$
          where $\E{X_1^2}=\sigma^2+\mu^2$. Combining all of these results, it
          follows that
          \[
            S^2\xrightarrow{P}\sigma^2+\mu^2-\mu^2=\sigma^2.
          \]
  \end{enumerate}
\end{ex}

\begin{ex}
  Note that
  \begin{align*}
    \E{[X_n-b]^2}
     & =\E{X_n^2-2bX+b^2}                   \\
     & =\var{X_n}+\E{X_n}^2-2b\E{X}+b^2     \\
     & =\var{X_n}+\left[\E{X_n}-b\right]^2.
  \end{align*}
  If $\var{X_n}\to 0$ and $\E{X_n}\to b$, as
  $n\to \infty$, it follows that $\E{[X_n-b]^2}\to 0$ as well and hence
  $X_n$ converges to $b$ in quadratic mean.

  Conversely, if $\E{[X_n-b]^2}\to 0$ as $n\to\infty$ it follows that
  \[
    0=\lim_{n\to\infty}\left(\var{X_n}+\left[\E{X_n}-b\right]^2\right),
  \]
  or, since both summands are non-negative, that
  \[
    \lim_{n\to\infty}\var{X_n}=0,\text{ and }
    \lim_{n\to\infty}\left[\E{X_n}-b\right]^2=0.
  \]
  If $\lim_{n\to\infty}\left[\E{X_n}-b\right]^2=0$, however, it follows that
  $\lim_{n\to\infty}\E{x_n}=b$.
\end{ex}

\begin{ex}
  Let $\var{X_i}=\sigma^2$. Let $Y_i=X_i-\mu$. Note that then
  \[
    \mu_Y=\E{Y_i}=\E{X_i-\mu}=0,
  \]
  \[
    \sigma^2_Y=\var{Y_i}=\var{X_i-\mu}=\sigma^2,
  \]
  \begin{align*}
    \E{(\overline{X}_n-\mu)^2}
     & =\E{\frac{1}{n}\left[\sum_{i=1}^n(X_i-\mu)\right]^2}            \\
     & =\frac{1}{n^2}\E{\left(\sum_{i=1}^n Y_i\right)^2}               \\
     & =\frac{1}{n^2}\E{\sum_{i=1}^n Y_i^2 +
      \sum_{j=1}^n\sum_{\substack{k=1                                  \\ j\neq k}}^n Y_jY_k} \\
     & =\frac{1}{n^2}\left[n(\sigma_Y^2+\mu_Y^2)+n(n-1)\mu_Y^2 \right] \\
     & =\frac{\sigma^2}{n},
  \end{align*}
  and $\E{(\overline{X}_n-\mu)^2}\to 0$ as $n\to\infty$. Therefore,
  $\overline{X}_n$ converges to $\mu$ in quadratic mean.
\end{ex}

\begin{ex}
  Fix an $\epsilon>0$. Note that for any random variable $X$,
  \begin{align*}
    \lim_{n\to\infty}\P{|X_n-X|>\epsilon}
     & =\lim_{n\to\infty}\left[\P{\left|\frac{1}{n}-X\right|>\epsilon}\P{X_n=\frac{1}{n}}
      +\P{|n-X|>\epsilon}\P{X_n=n}\right]                                                    \\
     & =\lim_{n\to\infty}\P{\left|\frac{1}{n}-X\right|>\epsilon}\left(1-\frac{1}{n^2}\right)
    +\lim_{n\to\infty}\P{|n-X|>\epsilon}\frac{1}{n^2}                                        \\
     & =\P{|X|>\epsilon},
  \end{align*}
  and thus, $X_n$ converges to $X$ in probability if and only if $X$ is a point
  mass at $0$.

  Recall that if $X_n$ converges in quadratic mean to $X$, by Theorem 5.4,
  $X_n$ also converges to $X$ in probability. However, we know that $X_n$
  only converges to $0$ in probability, and thus it suffices to check whether
  $X_n$ converges to $0$ in quadratic mean as well. Since
  \begin{align*}
    \lim_{n\to\infty}\E{(X_n-0)^2}
     & =\lim_{n\to\infty}\E{X_n^2}                                                              \\
     & =\lim_{n\to\infty}\left[\frac{1}{n^2}\P{X_n=\frac{1}{n}}+n^2\P{X_n=n}\right]             \\
     & =\lim_{n\to\infty}\left[\frac{1}{n^2}\left(1-\frac{1}{n^2}\right)+\frac{n^2}{n^2}\right] \\
     & =\lim_{n\to\infty}\left[1+\frac{1}{n^2}-\frac{1}{n^4}\right]                             \\
     & = 1,
  \end{align*}
  $X_n$ does not converge in quadratic mean.
\end{ex}

% 5
\begin{ex}
  First, note that if $X_i$ is a Bernoulli random variable, $X_i^2=X_i$. Next,
  recall that by Theorem 5.4, if a sequence of random variables converges to a
  distribution in quadratic mean, it converges to the same distribution in
  probability as well. Thus, it suffices to establish convergence in quadratic
  mean. However,
  \begin{align*}
    \E{\left[\frac{1}{n}\sum_{i=1}^nX_i^2 -p\right]^2}
     & =\frac{1}{n^2}\E{\left[\sum_{i=1}^n(X_i -p)\right]^2}                   \\
     & =\frac{1}{n^2}\E{\sum_{i=1}^n(X_i-p)^2 +\sum_{j=1}^n\sum_{\substack{k=1 \\ j\neq k}}^n (X_j-p)(X_k-p)} \\
     & =\frac{1}{n^2}\E{\sum_{i=1}^n(X_i-\E{X_i})^2}                           \\
     & =\frac{1}{n^2}\left[n\left(\var{X_1} \right) \right]                    \\
     & =\frac{p(1-p)}{n},
  \end{align*}
  which converges to $0$ as $n$ goes to infinity.
\end{ex}

\begin{ex}
  Let $\overline{X}$ be the average height of the $100$ random men. By the
  central limit theorem, we know that we can approximate $\overline{X}$ by $Y$,
  an $N(68, 2.6^2/100)$ distribution. Hence,
  \begin{align*}
    \P{\overline{X}\geq 72}
    \approx \P{Y\geq 72}
    = \P{\frac{Y-68}{0.26}\geq 15.385}
    =1-\Phi(15.385)
    \approx 0.
  \end{align*}

  \noindent
  \textit{Note: In the second printing of the book, the problem asks for the
    probability that the average height of the men will be at least 68 inches.
    This is trivial, since that is also the population mean. The problem is
    corrected in the errata on the author's website to 72 inches instead. }
\end{ex}

\begin{ex}
  Let $\lambda_n=1/n$ for $n=1,2,\ldots$. Let
  $X_n\sim\text{Poisson}(\lambda_n)$. Then
  \begin{enumerate}
    \item Fix an $\epsilon>0$. Then,
          \begin{align*}
            \lim_{n\to\infty}\P{|X_n|>\epsilon}
             & \leq \lim_{n\to\infty}\P{|X_n|>\min\{\epsilon, 1\}} \\
             & =1-\lim_{n\to\infty}\P{X_n=0}                       \\
             & =1-\lim_{n\to\infty}e^{-1/n}                        \\
             & =0.
          \end{align*}
    \item Let $Y_n=nX_n$. Then, for $\epsilon>0$,
          \[
            \lim_{n\to\infty}\P{|Y_n|>\epsilon}
            \leq 1-\lim_{n\to\infty}\P{Y_n=0}
            =1-\lim_{n\to\infty}e^{-1/n}
            =0.
          \]
  \end{enumerate}
\end{ex}

\begin{ex}
  Recall that the variance of a Poisson distribution is equal to its mean. By
  the central limit theorem, we can approximate
  $\frac{1}{n}\sum_{i=1}^n X_i$ by an $N(1, 1/n)$ distribution, and therefore
  can approximate $Y$ by an $N(n, n)$ distribution. Hence,
  \[
    \P{Y<90}
    =\P{Z<\frac{90-n}{\sqrt{n}}}
    =\P{Z<-1}
    \approx 0.15865.
  \]
\end{ex}

\begin{ex}
  Let $0<\epsilon<e$. Note that
  \[
    \P{X-X_n=x}=\begin{cases}
      \frac{n-1}{n} & x=0,     \\
      \frac{1}{n}   & x=e^{n},
    \end{cases}
  \]
  and that therefore
  \[
    \P{|X_n-X|>\epsilon}=1/n,
  \]
  which goes to $0$ as $n$ goes to infinity. Hence, $X_n$ converges to $X$ in
  probability. By Theorem 5.4 it follows that $X_n$ converges to $X$ in
  distribution as well.

  Finally, note that
  \[
    \E{(X-X_n)^2}=\frac{e^{2n}}{n},
  \]
  which does not converge to $0$ as $n\to\infty$. Therefore, $X_n$ does not
  converge to $X$ in quadratic mean.
\end{ex}

% 10
\begin{ex}
  Note that
  \[
    \P{|Z|>t}
    =\P{|Z|^k>t^k}
    \leq \frac{\E{|Z|^k}}{t^k}
  \]
  by applying Markov's inequality to $|Z|^k$. We proved that $\E{|Z|^k}$
  exists for any non-negative integer $k$ in Exercise 4.6, and note that we can
  extend this to all $k>0$, since
  \begin{align*}
    \E{|Z|^k}
     & =\int_0^\infty\! 2z^k \phi(z)\,\d{z}                      \\
     & \leq \int_0^1\! 2z^{\lfloor k\rfloor} \phi(z)\,\d{z}
    + \int_1^\infty\! 2z^{\lceil k\rceil} \phi(z)\,\d{z}         \\
     & \leq \int_0^\infty\! 2z^{\lfloor k\rfloor} \phi(z)\,\d{z}
    + \int_0^\infty\! 2z^{\lceil k\rceil} \phi(z)\,\d{z}         \\
     & =\E{|Z|^{\lfloor k\rfloor}} + \E{|Z|^{\lceil k\rceil}}.
  \end{align*}

  By Mill's inequality,
  \[
    \P{|Z|>t}\leq \sqrt{\frac{2}{\pi}}\frac{e^{-t^2/2}}{t}.
  \]

  Note that this bound is $O(e^{-t^2/2}/t)$ instead of $O(t^{-k})$ and that
  therefore, for $t$ sufficiently large, it provides a tighter bound.
\end{ex}

\begin{ex}
  To show convergence in distribution, note that $Y_n=\sqrt{n} X_n$ is a
  standard normal random variable, and that therefore
  \[
    F_{X_n}(x)=F_{Y_n}(\sqrt{n} x)=\Phi(\sqrt{n} x),
  \]
  and so
  \[
    \lim_{n\to\infty} F_{X_n}(x)
    =\lim_{n\to\infty} \Phi(\sqrt{n}x)
    =\begin{cases}
      0 & x < 0,   \\
      1 & x\geq 0.
    \end{cases}
  \]

  To show convergance in probability, fix an $\epsilon>0$. Then
  \begin{align*}
    \P{|X_n-X|>\epsilon}
     & \leq \P{|X_n|+|X|>\epsilon}                                      \\
     & \leq \P{|X_n|>\epsilon/2}+\P{|X|>\epsilon/2}                     \\
     & = 1 - \P{-\epsilon/2\leq X_n\leq \epsilon/2}
    + 1 - \P{-\epsilon/2\leq X\leq \epsilon/2}                          \\
     & =1-\left[\Phi\left(n\epsilon/2\right)-\Phi(-n\epsilon/2)\right]
    +1 - F(\epsilon/2) + F(-\epsilon/2)                                 \\
     & =1-\left[\Phi\left(n\epsilon/2\right)-\Phi(-n\epsilon/2)\right],
  \end{align*}
  which goes to $0$ as $n$ goes to infinity since the expression between the
  square brackets is the probability that a standard normal random variable is
  between $-\sqrt{n}\epsilon/2$ and $\sqrt{n}\epsilon/2$. Thus, $X_n$ converges
  to $X$ in probability.

\end{ex}

\begin{ex}
  Let $X,X_1,X_2,\ldots$ be random variables that are positive and integer
  valued. Let $F$ be the CDF of $X$ and $F_i$ the CDF of $X_i$. Suppose that
  $X_n\rightsquigarrow X$. Then
  \begin{align*}
    \lim_{n\to\infty}\P{X_n=k}
     & =\lim_{n\to\infty}\left[F_n(k)-F_n(k-1)\right]     \\
     & =\lim_{n\to\infty}F_n(k)-\lim_{n\to\infty}F_n(k-1) \\
     & =F(k) - F(k-1)                                     \\
     & =\P{X=k}.
  \end{align*}

  Conversely, suppose that
  \[
    \lim_{n\to\infty}\P{X_n=k}=\P{X=k},
  \]
  for all $k$. Then, for $k$ a positive integer,
  \begin{align*}
    \lim_{n\to\infty} F_n(k)
     & =\lim_{n\to\infty}\sum_{i=1}^k \P{X_n=k} \\
     & =\sum_{i=1}^k\lim_{n\to\infty} \P{X_n=k} \\
     & =\sum_{i=1}^k \P{X=k}                    \\
     & =F(k),
  \end{align*}
  and, since the random variables can only take on positive integer values, it
  follows that $X_n\rightsquigarrow X$.
\end{ex}

\begin{ex}
  Note that
  \begin{align*}
    \P{X_n\leq x}
     & =\P{n\min\{Z_1,\ldots, Z_n\}\leq x}              \\
     & =\P{\min\{Z_1,\ldots, Z_n\}\leq x/n}             \\
     & =1 - \P{Z_1>x/n}\cdots\P{Z_n> x/n}               \\
     & =1-\left(1-\int_{0}^{x/n}\!f(t)\,\d{t}\right)^n,
  \end{align*}
  where the lower limit of immigration follows from the fact that $\P{Z_i>0}=1$.
  Let $F(x)=\int_0^x\!f(t)\,\d{t}$ and note that then
  \begin{align*}
    \lim_{n\to\infty}\P{X_n\leq x}
     & =1-\lim_{n\to\infty}\left[1-F\left(\frac{x}{n}\right)\right]^n \\
     & =1-\lim_{n\to\infty}\left[1-F(0)-F'(0)\left(\frac{x}{n}\right)
      -h\left(\frac{x}{n}\right)\left(\frac{x}{n}\right)\right]^n
     & (\text{Taylor's theorem})                                      \\
     & =1-\lim_{n\to\infty}\left[\left(1-\frac{\lambda x}{n}\right)
      -h\left(\frac{x}{n}\right)\left(\frac{x}{n}\right)\right]^n     \\
     & =1-\lim_{n\to\infty}\left(1-\frac{\lambda x}{n}\right)^n,
  \end{align*}
  by binomial expansion and using the fact that the $h(x)$ error term from
  Taylor's theorem goes to $0$ as $x$ goes to $0$.

  Finally, taking the limit gives us
  \begin{align*}
    \lim_{n\to\infty}\P{X_n\leq x}=1-e^{-\lambda x},
  \end{align*}
  the CDF of an $\text{Exponential}(\lambda)$ distribution.
\end{ex}

\begin{ex}
  Let $X_1,\ldots,X_n\sim\text{Uniform(0,1)}$. Then
  \[
    \sqrt{12n}(\overline{X}_n - 1/2) \rightsquigarrow N(0,1)
  \]
  by the central limit theorem. Note that $g(x)=x^2$ is a differentiable
  function with $g'(x)=2x$. Therefore, $g'(\mu)=1\neq 0$ and thus it follows by
  the delta method,
  \[
    \sqrt{12n}\left(\overline{X}_n^2-\frac{1}{4}\right)
    \rightsquigarrow N(0, 1),
  \]
  i.e.\ that
  \[
    \overline{X}_n^2\approx N\left(\frac{1}{4}, \frac{1}{12n}\right).
  \]
\end{ex}

% 15
\begin{ex}
  Let $g(y)=y_1/y_2$ and note than then
  \[
    \nabla g(y) = \begin{pmatrix}
      1/y_2 \\ -y_1/y_2^2
    \end{pmatrix}.
  \]
  By the multivariate central limit theorem, we have
  \[
    \sqrt{n}\left[\begin{pmatrix}
        \overline{X}_{1} \\ \overline{X}_{2}
      \end{pmatrix}-\begin{pmatrix}
        \mu_1 \\ \mu_2
      \end{pmatrix}\right]\sim N(0, \Sigma),
  \]
  and therefore it follows by the multivariate delta method that
  \[
    \sqrt{n}\left(
    \overline{X}_{1}/\overline{X}_{2} - \mu_1/\mu_2
    \right)\sim N(0, \nabla^T_\mu\Sigma\nabla_\mu),
  \]
  where
  \[
    \nabla^T_\mu\Sigma\nabla_\mu
    =\begin{pmatrix}
      \mu_2^{-1} & -\mu_1\mu_2^{-2} \\
    \end{pmatrix}
    \begin{pmatrix}
      \sigma_{11} & \sigma_{12} \\
      \sigma_{12} & \sigma_{22}
    \end{pmatrix}
    \begin{pmatrix}
      \mu_2^{-1} \\ -\mu_1\mu_2^{-2}
    \end{pmatrix}
    =\mu_2^{-1}\sigma_{11}
    -2\mu_1\mu_2^{-3}\sigma_{12}
    +\mu_1^2\mu_2^{-4}\sigma_{22}.
  \]
  Therefore,
  \[
    \overline{X}_{1}/\overline{X}_{2}\approx
    N(\mu_1/\mu_2,
    \mu_2^{-1}\sigma_{11}
    -2\mu_1\mu_2^{-3}\sigma_{12}
    +\mu_1^2\mu_2^{-4}\sigma_{22}.
    ).
  \]
\end{ex}

\begin{ex}
  Let $X_1, X_2, \ldots$ be a sequence of random variables such that
  $X_n\sim N(0, 1)$ and let $Y_n=-X_n$ for all $n$. Note that since the standard
  normal distribution is symmetric about the origin, $Y_n\sim N(0, 1)$. Hence,
  both $X_n$ and $Y_n$ converge in distribution to an $N(0, 1)$ distribution,
  and thus $X + Y=N(0, 2)$, but since $X_n+Y_n=X_n+(-X_n)=0$,
  $X_n+Y_n\rightsquigarrow 0$.
\end{ex}


\section{Clustering}
\label{sec:clustering}

We compared three clustering algorithms applied to the original
substitute vectors using many-to-one accuracy on the 45-tag 24K word
test corpus.  Hierarchical agglomerative clustering with complete
linkage (HAC) starts with each instance in its own cluster and
iteratively combines the two closest groups (measured by their most
distant points) at each step \cite{manning2008introduction}.
K-medoids minimizes sum of pairwise distances between each datapoint
to the exemplar at the center of its cluster
\cite{kaufman2005finding}.  Spectral clustering\footnote{We used the
  implementation in \cite{chen2011parallel} with a symmetric sparse
  affinity matrix of 550 nearest neighbors.} uses the eigenvalues of
the graph Laplacian $L=D^{-1/2} W D^{-1/2}$ to reduce the number of
dimensions (similar to Laplacian eigenmaps) and uses simple k-means
clustering on the resulting representation \cite{ng2002spectral}.  All
three algorithms accept the distance matrix based on the KL2 distance
(see Section~\ref{sec:dist}) as input.

\begin{figure}[h]
\includegraphics[width=.5\textwidth]{clustering_graph_mono.eps}
\caption{Many-to-one score for three clustering algorithms on the
  45-tag 24K word corpus.}
\label{fig:clustering}
\end{figure}

% #cluster kmedoid spectral hac
% 2          0.20795 0.17781 0.135762
% 4          0.29413 0.33439 0.155579
% 8          0.36615 0.39550 0.159076
% 16        0.43830 0.46116 0.263988
% 32        0.43318 0.54192 0.329850
% 45        0.46245 0.59413 0.401832
% 64        0.46965 0.59151 0.438843
% 128      0.48235 0.63106 0.506703
% 256      0.53709 0.65000 0.558659
% 512      0.58464 0.66520 0.616653
% 1024     0.62985 0.67523 0.658576
% 2048     0.66107 0.68414 0.696878
% 4096     0.69842 0.71286 0.744338

Figure~\ref{fig:clustering} plots the many-to-one score versus number
of clusters for the three algorithms on the 45-tag 24K word test
corpus.  The many-to-one score naturally increases as we approach the
one cluster per word limit, however we find the evolution of the
curves informative.  At the high end (more than 2000 clusters) HAC
performs best with its conservative clusters, but its performance
degrades fast as we reduce the number of clusters because it cannot
reverse the accumulating mistakes.  At the low end (less than 16
clusters) k-medoids and spectral have similar performance.  However
for the region of interest (between 16 to 2000 clusters) spectral
clustering is clearly superior with \spectralResult\% many-to-one
accuracy at 45 clusters.


\documentclass{article}
\usepackage{latexsym}
\usepackage{amsmath}
\usepackage{amssymb}

\begin{document}
\title{Measure Theory notes}
\author{Dave Neary}

\maketitle

\section{Motivation for Lebesgue integral}

\subsection{Overview of the Riemann integral}

The Riemann integral of a continuous function $f:\mathbb{R} \rightarrow \mathbb{R}$ is defined as:

\[ \int_{a}^{b}f(x) dx = \lim_{n \rightarrow \infty}\sum_{i=1}^{n}f(x_{i}) \Delta x \]

where $x_{i} = a + (i-1)\Delta x$ and $\Delta x=\frac{b-a}{n}$

In other words, we partition the domain of the function into small slices,
and calculate the area under the curve by multiplying the width of the slices
by the value of the function at the beginning of the slice.

This works well for a certain class of functions, called Riemann-integrable
functions. These functions must satisfy the condition that the domain is $\mathbb{R}$,
and that limit above exists.

More generally, we can calculate an upper Riemann sum $U(f)$ by summing the
areas using $\sup{f(x)}$ on each partition, and a lower sum $L(f)$ by using $\inf{f(x)}$
for each interval. The function $f$ is Riemann integrable when $\lim L(f) = \lim U(x)$.

\subsection{Limitations of the Riemann integral}

It is possible to generalize the Riemann integral to two
or more dimensions, but the problem of finding an appropriate partition of the domain
means that for dimensions of the real numbers which are higher than $\mathbb{R}^2$,
the Riemann integral is limited.  In addition, we would like to consider other classes
of domains than the reals for functions - for example, probability spaces or generic
Hilbert spaces - where some alternative idea of the area under the curve (or more
generally, the volume of a set) may make sense. Another limitation of the Riemann
integral is that there are useful classes of functions for which it does not converge,
but for which a reasonable value for the integral exists.

Another limitation of the Riemann integral is that there is only a very limited set of
functions for which  it is possible to say
\[\int \sum_n f_n(x) dx = \sum_n \int f_n(x) dx \]

Namely, $f_n(x)$ must converge uniformly to $f(x)$, which is a very strong constraint.

As a result of these limitations, the idea of the Lebesgue integral is to partition
the function range instead of the domain. We then identify the subsets of the domain for
specific values of $f(x)$, and calculate their volume using a generic measure function
$\mu$. By taking finer and finer intervals of the range, we can get better and better
estimates of the volume under the function with respect to the domain and the measure.

The remainder of this document will describe the characteristics of a domain, the
constraints required for a measure, which types of functions we can integrate, and
a precise definition of the Lebesgue integral. We will also include a selection of
proofs and problems which we can use the Lebesgue integral to solve.

\section{Lebesgue Measure}

Working backwards, to define what we mean by an integrable function, we will need to
first define how to measure the volume of a subset of a domain (a measure), and to
define a measure, we must first define the types of sets which will be measurable.

\subsection{Measurable spaces}

Starting from a set $X$, a collection of subsets of $X$, $\mathcal{A}$, is called a
$\sigma$-algebra if it satisfies the following conditions:

\begin{enumerate}
		\item $X \in \mathcal{A}$
		\item For each $A \in \mathcal{A}$, $X \setminus A \in \mathcal{A}$
		\item For a countable sequence of subsets $(A_n)_{n \in \mathbb{N}} \in \mathcal{A}$,
			\[\bigcup_{n} A_n \in \mathcal{A} \]
\end{enumerate}

We will see when we define a measure why this is called a $\sigma$-algebra.

The pair $(X, \mathcal{A})$ is called a measurable space.

Given any collection of subsets $\mathcal{C}$ of subsets of $X$, we can generate a
smallest $\sigma$-algebra which contains $\mathcal{C}$. That is, there is a $\sigma$-algebra
$\mathcal{A}$ such that if $\mathcal{B}$ is a $\sigma$-algebra containing $\mathcal{C}$, then
$\mathcal{A} \subseteq \mathcal{B}$. We say that such a $\sigma$-algebra $\mathcal{A}$ is
generated by $\mathcal{C}$.

\subsubsection{Examples}

\begin{enumerate}
	\item \textbf{Exercise:} If $X=\{1,2,3,4\}$, and the $\sigma$-algebra $\mathcal{A}$ is
		generated by $\{\{1,2\},\{2,3\}\}$, what are the other members of
		$\mathcal{A}$? \\
		\textbf{Answer:} By condition 1 above, $X=\{1,2,3,4\} \in \mathcal{A}$,
		and by condition 2, since $X \in \mathcal{A}$, $X \setminus X = \emptyset
		\in \mathcal{A}$. Similarly, since $\{1,2\}$ and $\{2,3\} \in \mathcal{A}$,
		$X \setminus \{1,2\} = \{3,4\}$ and $X \setminus \{2,3\} = \{1,4\} \in
		\mathcal{A}$. By rule 3, $\{1,2\} \cup \{2,3\} = \{1,2,3\}$ and $\{2,3\} \cup
		\{3,4\} = \{2,3,4\} \in \mathcal{A}$. And by rule 2 again, $X \setminus
		\{1,2,3\} = \{4\}$ and $X \setminus \{2,3,4\} = \{1\} \in \mathcal{A}$. Back to
		rule 3, $\{4\} \cup \{1,2\} = \{1,2,4\}$ and $\{1\} \cup \{3,4\} = \{1,3,4\} \in
		\mathcal{A}$. Finally, $X \setminus \{1,2,4\} = \{3\}$ and $X \setminus
		\{1,3,4\} = \{2\} \in \mathcal{A}$. Since each of the individual elements of
		$X$ are in a subset on their own, we can now create all possible subsets
		of $X$. $\mathcal{A} = \mathcal{P}(X)$, the power set of all subsets of $X$.
	\item \textbf{Exercise:} Prove that for a countable sequence of subsets
		$(A_n)_{n \in \mathbb{N}} \in \mathcal{A}$, a $\sigma$-algebra on $X$, that
		\[\bigcap_{n} A_n \in \mathcal{A} \]
		\textbf{Answer:} Define a sequence of sets $B_n = X \setminus A_n$. Then,
		by condition 2, $B_n \in \mathcal{A}$ for all $n$. By condition 3,
		\[ B = \bigcup_n B_n \in \mathcal{A} \]
		\[ X \setminus B \in \mathcal{A} \]
		By Demorgan's laws,
		\[
			X \setminus \bigcup_n B_n = \bigcap_n (X \setminus B_n) = \bigcap_n A_n
		\]
		So $\bigcap_n A_n \in \mathcal{A}$. QED.
	\item \textbf{Exercise:} $(X, \mathcal{A})$ is a measurable space, with $Y \subset X$.
		Prove that $(Y,\mathcal{A^\prime})$ is a measurable space, where
		$\mathcal{A}^\prime = \{A \bigcap Y | A \in \mathcal{A}\}$\\
		\textbf{Answer:} Since $\emptyset \in \mathcal{A}$, $\emptyset \bigcap Y =
		\emptyset \in \mathcal{A}^\prime$. Similarly, since $X \in \mathcal{A}$,
		$X \bigcap Y = Y \in \mathcal{A}^\prime$. \\
		For any $A \in \mathcal{A}$, $X \setminus A \in \mathcal{A}$, and $(X \setminus A)
		\bigcap Y = (X \bigcap Y) \setminus (A \bigcap Y) = Y \setminus (A \bigcap Y)$.
		So if $A \bigcap Y \in \mathcal{A}^\prime$, then $Y \setminus (A \bigcap Y) \in
		\mathcal{A}^\prime$\\
		Finally, let $(A_i)_{i \in \mathbb{N}}$ be a sequence of sets in $\mathcal{A}$. Then
		\[\bigcup_{i \in \mathbb{N}} A_i \in \mathcal{A} \]
		Define a sequence $(B_i)_{i \in \mathbb{N}}$ with  $B_i = Y \bigcap A_i$ for all $i$.
		Then
		\begin{equation}
			\bigcup_{i \in \mathbb{N}} B_i = \bigcup_{i \in \mathbb{N}} (Y \bigcap A_i) \\
			= Y \bigcap \left(\bigcup_{i \in \mathbb{N}} (A_i)\right) \in \mathcal{A}^\prime
		\end{equation}
		Therefore, $(Y, \mathcal{A^\prime})$ is a measure space.
\end{enumerate}

\subsection{Measures}

The extended real numbers is the set $\overline{\mathbb{R}} = \mathbb{R} \bigcup \{-\infty, \infty\}$.

\textbf{Definition:} A measure is a function $\mu:X \rightarrow \overline{\mathbb{R}_{0}^{+}}$ on a
measurable space $(X, \mathcal{A})$ which satisfies the conditions:

\begin{enumerate}
	\item $\mu(\emptyset) = 0$
	\item if $(A_i)_{i \in \mathbb{N}} \in \mathcal{A}$ is a sequence of pairwise disjoint
		sets (that is, $A_i \bigcap A_j = \emptyset$ if $i \ne j$), then:
		\[ \mu \left( \bigcup_{i =1}^{\infty} A_i \right) = \sum_{i=1}^{\infty} \mu
		\left( A_i \right) \]
\end{enumerate}

This characteristic of being able to turn a countable union of disjoint sets into a sum is why
$\mathcal{A}$ is called a $\sigma$-algebra.

In general, we can think of the measure of a set as its volume, or (for real functions) as
the area under the curve.

A measurable space $(X, \mathcal{A})$ with a measure $\mu$ is called a measure space, and
is written $(X, \mathcal{A}, \mu)$.

We can deduce a number of lemmas from this definition:

\textbf{Lemma:} If $A \subseteq B$ and $A, B \in \mathcal{A}$, then $\mu(A) \le \mu(B)$

\textbf{Lemma:} For a sequence of sets $(A_i)_{i \in \mathbb{N}} \in \mathcal{A}$
\[ \mu \left( \bigcup_{i =1}^{\infty} A_i \right) \ge \sum_{i=1}^{\infty} \mu
                \left( A_i \right) \]

\textbf{Lemma:} For a sequence of sets $(A_i)_{i \in \mathbb{N}} \in \mathcal{A}$ with
$A_i \subseteq A_j$ if $i<j$ then
\[ \mu \left( \bigcup_{i =1}^{\infty} A_i \right)= \lim_{i \rightarrow \infty} \mu(A_i) \]

Similarly, for a sequence where $A_i \supseteq A_j$ for $i<j$,
$\mu \left( \bigcap_{i =1}^{\infty} A_i \right)= \lim_{i \rightarrow \infty} \mu(A_i)$

Some examples of measures are the trivial measure $\mu(A)=0$ for all $A \in \mathcal{A}$,
the counting measure $\mu(A) = |A|$ if A is finite, or $\infty$ if it is infinite, and the
Dirac measure $\delta_a(S) = 1$ if $a \in S$ or 0 otherwise.

\textbf{Exercise:} Prove that the trivial, counting, and Dirac functions are measures.

\begin{itemize}
	\item \textbf{Trivial measure:} if $\mu(A)=0$ for all $A$, then
		$\mu(\emptyset)=0$, and for a pairwise disjoint collection of
		sets $(A_i)_{i \in \mathbb{N}} \in \mathcal{A}$,
		\[ \mu(\left( \bigcup_{i =1}^{\infty} A_i \right) = 0 \]
		and since $\mu \left( A_i \right) = 0$ for all $i$,
		\[ \sum_{i=1}^{\infty} \mu \left( A_i \right) = 0 =
		\mu(\left( \bigcup_{i=1}^{\infty} A_i \right) \]
		Therefore, $\mu$ is a measure.
	\item \textbf{Counting measure:} $\mu(\emptyset) = |\emptyset|=0$
		For a collection of pairwise disjoint sets
		$(A_i)_{i \in \mathbb{N}} \in \mathcal{A}$, if
		\[ \mu(\left( \bigcup_{i =1}^{\infty} A_i \right) < \infty \]
		then each of $\mu(A_i)$ is finite, and there is a finite
		collection of subsets $(A_i)$ of $X$. Each element of
		$\bigcup_{i =1}^{\infty} A_i$ is also an element of exactly one $A_i$
		and each element of each $A_i$ is also an element of
		$\bigcup_{i =1}^{\infty} A_i$ by definition.
		If \[ \mu(\left( \bigcup_{i =1}^{\infty} A_i \right) = \infty \], then
		for each $a \in \bigcup_{i =1}^{\infty} A_i$, $a \in A_i$ for some $i$.
		Therefore, in either case,
		\[ \sum_{i=1}^{\infty} \mu \left( A_i \right) =
		\sum_{i=1}^{\infty} |A_i| =
		\mu(\left( \bigcup_{i =1}^{\infty} A_i \right) \]
		Therefore, the counting measure $\mu(A)=|A|$ is a measure.
	\item \textbf{Dirac measure:} For an element $a \in X$, $\delta_a(A)=0$
		if $a \notin A$
		Since $a \notin \emptyset$, $\delta_a(\emptyset) = 0$.
		Consider a pairwise disjoint collection of sets
		$(A_i)_{i \in \mathbb{N}} \in \mathcal{A}$. If
		$a \in \bigcup_{i =1}^{\infty} A_i $ then
		\[\delta_a \left(\bigcup_{i=1}^{\infty} A_i \right) = 1 \]
		and $a \in A_i$ for some $i \in \mathbb{N}$, and since
		$(A_i)$ are pairwise disjoint, $\delta_a(A_i)=1$ and
		$\delta_a(A_j)=0$ for $j \ne i$. Then
		\[ \sum_{i=1}^{\infty} \delta_a \left( A_i \right) = 1 =
                \delta_a(\left( \bigcup_{i=1}^{\infty} A_i \right) \]
		If $a \notin \bigcup_{i =1}^{\infty} A_i $ then
		$a \notin A_i$ for all $i$, and
		\[ \sum_{i=1}^{\infty} \delta_a \left( A_i \right) = 0 =
                \delta_a(\left( \bigcup_{i=1}^{\infty} A_i \right) \]
		Therefore, $\delta_a$ is a measure.
\end{itemize}

\textbf{Exercise:} Prove the lemmas above.

\subsection{Measurable functions}

Let $\left(\mathcal{X}, \mathcal{A}\right)$ and $\left(\mathcal{Y}, \mathcal{B}\right)$
be measurable spaces. A function $f:\mathcal{X} \rightarrow \mathcal{Y}$ is measurable with
respect to the $\sigma$-algebras $\mathcal{A}, \mathcal{B}$ if, for each subset
$B \in \mathcal{B}$, $f^{-1}(B) \in \mathcal{A}$ (where $f^{-1}(B)$ is the pre-image of
the set $B$ under the function $f$, $\{x \in X \text: f(x) \in B\}$).

Thinking about useful collections of sets for a measure space, for functions mapping onto
$\overline{\mathbb{R}}$, we can generate a $\sigma$-algebra from the set of open intervals
in $\mathbb{R}$, plus $\infty$.
For the open interval $A=(a,b)$. we define the measure $\lambda (A) = b-a$. This measure
is called the Lebesgue measure.

A $\sigma$-algebra $\mathcal{A}$ generated from all of the open subsets of $\mathcal{X}$
is called the Borel $\sigma$-algebra. It is a useful concept, because by choosing a Borel
$\sigma$-algebra, $\mathcal{A}$ is also a topology, and we inherit all of the useful
theorems from topology too.

\textbf{Reminder:} A topological space is a nonempty set $X$ plus a set of subsets $A$ possessing
the properties:

\begin{enumerate}
\item $X, \emptyset \in A$
\item If $O_1 \in A$ and $O_2 \in A$, then $O_1 \bigcap O_2 \in A$
\item For a sequence of sets $\left(O_i\right)_{i\in \mathbb{N}} \in A$, the countable
	union $\bigcup_{i\in \mathbb{N}} O_i \in A$
\end{enumerate}

\textbf{Exercise:}

\textbf{Exercise:}

\subsection{Lebesgue Integral}

We can now pull all of these ideas together to define the Lebesgue integral.

We define the characteristic function $\chi_E(x)$ of
the set $E$:
\[ \chi_E(x)=\left\{
\begin{array}{ll}
1 & x \in E\\
0 & x \notin E
\end{array} \right.
\]

A linear combination
\[ \phi(x) = \sum_{i=1}^{n}a_i\chi_{E_i}(x) \]
is called a simple function, if $\phi$ is measurable with respect to the $\sigma$-algebra
generated by the sets $(E_i)$, and assumes only a finite number of values
$\{a_1,a_2,...,a_n\}$.

The Lebesgue integral of a simple function
\[\phi = \int_X \left( \sum a_i \chi_{A_i}(x)\right) d\mu = \sum a_i \mu(A_i) \]
is the result of defining $A_i = \phi^{-1}(a_i)$ (that is,
$A_i = \{x:\phi(x)=a_i\}$) for each of the values $a_i$ that $\phi$ assumes. One consequence
of this definition is that $\left(A_i\right)$ is a sequence of pairwise disjoint sets.

We can define the Lebesgue integral for a measurable non-negative function $f:X \rightarrow
\overline{\mathbb{R}^{+}}$ with respect to a measure space $(X, \mathcal{A},\mu)$ as:
\[ \int_X f(x) d\mu = \sup\left\{\int_X \phi(x) d\mu: 0 \le \phi(x) \le f(x), \phi \textrm{ a
simple function} \right\} \]
In other words, we look over all of the simple functions that are less than $f$, and take
the supremum across all of them.

For functions which are not non-negative, we split $f(x)$ into
\[ g(x) = \max(f(x),0) \]
and
\[ h(x) = - \min(f(x,0)) \]

Then
\[\int_X f(x) d\mu = \int_X g(x) d\mu - \int_X h(x) d\mu \]

In other words, we split $f(x)$ into two non-negative functions, one representing the positive
part of $f$, and one representing the absolute value of the negative part of $f$, and we can
calculate the final interval by removing the negative area from the positive area.

For any continuous functions $f(x): X \rightarrow \overline{\mathbb{R}^{+}}$, we can construct
a sequence of simple functions $\{f_n(x)\}$ which converges pointwise to $f(x)$ as follows.
For $f_n(x)$, partition the range into $2^{2n}+1$ disjoint partitions $\{I_{n,i}\}$ with
\[ I_{n,i}  = \left\{
\begin{array}{ll}
	\left[\frac{i-1}{2^n},\frac{i}{2^n}\right) & 1 \le i \le 2^{2n} \\[3pt]
	\left[\frac{i-1}{2^n},\infty\right) & i=2^{2n} + 1
\end{array} \right. \]

Then define $\{A_{n,i}\}_{i \le 2^{2n} + 1} = f^{-1}(I_{n,i})$, the preimage of $I_{n,i}$.
The collection $\{I_{n,k}\}$ cover $[0,\infty)$ for all $n$. The simple function
\[ f_n(x) = \sum_{i=1}^{2^{2n+1}}\frac{(i-1)}{2^n}\chi_{A_{n,i}}(x) \]
is a sequence of increasing functions which converge pointwise to $f(x)$.

\textbf{Exercise:} Prove that the sequence $f_n(x)$ above converges pointwise to $f(x)=x^2$ for all
$x \in \overline{\mathbb{R}^{+}}$.

\section{Lebesgue Integrals and Probability Theory}

Probability distributions all share some common characteristics which allow the application of measure
theory to be useful. Given a sample space $\Omega$ of possible outcomes, and an event space
$\mathcal{A}$, which is a $\sigma$-algebra, and a probability measure $P$, a measure space
$(\Omega, \mathcal{A}, P)$ is called a probability space if:
\begin{enumerate}
\item $P(\emptyset)=0$
\item $P(\Omega)=1$
\item if $\{A_{i}\}_{i=1}^{\infty } \subseteq {\mathcal{A}}$ is a countable collection of
	pairwise disjoint sets, then:
		\[ P(\bigcup _{i=1}^{\infty }A_{i}) = \sum_{i=1}^{\infty} P(A_{i}) \]
\end{enumerate}

\end{document}


\chapter{Verification}
\label{chapter:verification}

We improve the Halide simplifier term rewriting system by ensuring its soundness in
two ways: first, we verify that each individual rule is \emph{correct}, meaning that the
rewrite preserves semantics. Then we verify that the term rewriting system is
\emph{guaranteed to terminate} on all inputs by ensuring that no sequence of
rule applications, on any input expression, can form a cycle.

\section{Correctness}
\label{sec:ruleverification}
We verify each individual rule is correct by modeling Halide
expressions in SMT2 and using the SMT solver Z3~\citep{de2008z3} to
prove that the rule's left- and right-hand sides are equivalent. Most Halide expression
semantics map cleanly to SMT2 formulas. The functions \hmax{} and
\hmin{} are defined in the usual way, and \hsel{} in
Halide is equivalent to the SMT2 operator \texttt{ite}. Division and
modulo are given the Euclidean definitions in both Halide and
SMT2~\citep{boute1992euclidean}, though division and modulo by zero is handled
differently (in Halide both evaluate to zero).
%If a variable appears in the LHS of a rule as a divisor in a
%division or modulo operation, it is assumed to be non-zero. %The Halide
%expressions do not have a true boolean type (true and false are represented by
%unsigned integers of 1 bitwidth), so expressions must be typed as either
%\texttt{Int} or \texttt{Bool} when translated into SMT2. The Halide expression
All integer and boolean operators can be coerced to vector operators on vector inputs of the appropriate type.
Halide's TRS uses two vector-constructing operators, \texttt{broadcast} and \texttt{ramp}.
\texttt{broadcast(x, l)} projects some value $x$ to a vector of length $l$; because of
the type coercion, we can simply represent \texttt{broadcast(x, l)} as the variable
\texttt{x} in SMT2. \texttt{ramp(x, s, l)} creates a vector of length $l$
whose initial entry has the value $x$ and all subsequent entries increase with
stride $s$. In SMT2, we represent this term as the symbolic expression $x + l *
s$, where $l$ must be zero or positive.

Given this modeling, for each rule, we assert any predicate guards are true, then
ask Z3 to search for a variable assignment that makes the LHS and RHS not
equivalent.  If Z3 indicates no such assignment exists, the LHS must be equivalent to
the RHS and the rule must be correct. We implemented an SMT2 printer for
Halide rewrite rules that automatically constructs an SMT2 verification problem for each rule.
Rule verification using Z3 is fully automated
and can be run for the current set of rewrite rules used in the compiler via a script.

However, for 123
rules, Z3 either timed out or returned unknown. Nearly all of these rules used
either division or modulo. We used the proof assistant Coq to manually prove the
correctness of these remaining rules. In the course of these proofs, we
discovered we were also able to relax the predicate guards of \NumPredicatesRelaxed
rules; for example, in some cases a rule
with a guard requiring some constant to be positive would be equally valid
if the constant was non-zero.

This mostly-automated approach to verification assists with changing
the language semantics. Our initial work on verification was not on
the semantics described above: division or modulo by zero was originally
considered undefined behavior. Since we had already
modeled Halide semantics in SMT2, it was easy to alter the
definitions of division and modulo and re-run the verification scripts.
We proved \NumZdivCoqProvedRules rules manually in Coq after Z3 failed to verify them;
since in the previous round all Coq proofs
included the assumption that all divisors were non-zero, in most cases
we had only to add a case to show that the rule was true when the
divisor was zero as well. In the course of reviewing
these proofs, we identified \NumZdivRelaxedPredicates rules whose
predicates included the condition that a divisor be non-zero and where
that condition could safely be lifted. We found that
\NumZdivFalseRules rules were not correct under the new semantics and
submitted a patch to amend them.

\section{Termination}
\label{sec:termination}

\modified{Under the umbrella goal of simplifying expressions, the Halide TRS uses
many strategies: it may attempt to make expressions as short as possible; it may factor out
vector operations or more expensive operations such as division; it may attempt to
canonicalize subexpressions so they can cancel or be shown equivalent. These
strategies are not necessarily aligned and may even undo each other. Crafting new rules
can thus require a detailed understanding of the ruleset and its various applications.
In this section we formalize the Halide expression simplification strategy that was
previously only encoded in the ruleset itself. In doing so, we also prove that since
each rule strictly makes progress in accordance to this strategy, the Halide TRS always terminates.}

Consider a term
rewriting system containing only one rule: $x + y \rewrites y + x$. The term
$3 + 5$ matches the LHS of the rule and is rewritten to $5 + 3$, which can again
be matched to the rule and rewritten to $3 + 5$, and so on. Termination failures in the Halide TRS have occurred in the past\footnote{See for example https://github.com/halide/Halide/pull/1525}, causing unbounded recursion and eventually a stack overflow in the compiler. This is tricky to debug, and may not always be reported by users, since the error is fairly opaque. To show that this type of error has been eliminated, we must prove that there is no expression in the Halide expression language that can be infinitely rewritten by some sequence of rules that form a cycle.

Intuitively, we can think of Halide expressions as existing in some multi-dimensional space; when an expression is rewritten by a rule, it moves from one point in that space to another. If each rule always rewrites expressions such that they move monotonically in some direction through the expression space, then no sequence of rules can form a cycle. These directions correspond to our intuition about why certain rules are useful (like the examples at the beginning of this section). We can consider each of these directions as a dimension in the expression space. If we formalize this desirable ordering and show that all rewrites from one expression to another strictly obey it, then we will have a proof of termination.

We provide this formalism and prove that the Halide term rewriting system must terminate by constructing a \emph{reduction order}, a strict order with properties that ensure that, for an order $>$ and a rule $l \rewrites r$, if $l > r$, then for any expression $e_1$ that matches $l$ and is rewritten by $l \rewrites r$ into $e_2$, it must be true that $e_1 > e_2$. Crucially, this order is evaluated over rule terms, and not over all expressions that those terms may match. We take the definition of a reduction order and the next two theorems from~\citep{baader1999term}.

\begin{theorem}\label{theorem:terminates}
A term rewriting system $R$ terminates iff there exists a reduction order $>$ that satisfies $l > r$ for all $l \rewrites r \in R$.
\end{theorem}

A reduction order is a strict order that must be well-founded, meaning that every non-empty set has a least element with regard to the order, to prevent infinitely descending chains. It must be \emph{compatible with $\Sigma$-operations}: for all expressions $s_1, s_2$, all $n \geq 0$, and all $f \in \Sigma$:
\[
s_1 > s_2 \implies f(t_1,...t_{i-1},s_1,t_{i+1},...,t_n) > f(t_1,...t_{i-1},s_2,t_{i+1},...,t_n)
\]
for all $i, 1 \leq i \leq n$ and all expressions $t_1,...t_{i-1},t_{i+1},...,t_n$. This property means that if a rewrite rule transforms a subtree in some expression $e$, the $>$ relation is preserved between the original expression $e$ and the rewritten expression $e'$. Finally, a reduction order is \emph{closed under substitution}: for all expressions $s_1, s_2$ and all substitutions $\sigma \in \mathcal{S}ub(T(\Sigma,V))$,
$s_1 > s_2 \implies \sigma(s_1) > \sigma(s_2)$. When we match some left-hand side term $l$ to some expression $e$, we are defining a substitution for each of the variables in $l$ with some subtree in $e$; we then use that substitution to rewrite $e$ to $e'$. If our order is closed under substitutions, we know that for any expression we match to $l$, the resulting rewritten expression will obey the ordering.

Choosing a single monotonic direction in which to rewrite expressions would be overly restrictive.
The Halide TRS is used both to prove expressions true and to simplify them; when using it as a prover, we want to put both sides of an equality into some normal form, but it doesn't particularly matter what that form is. When using the TRS to simplify expressions, on the other hand, reducing the size of an expression has important performance benefits. Since we need an ordering that covers the full Halide simplification strategy, we make use of the following theorem:

\begin{theorem}
The lexicographic product of two terminating relations is again terminating.
\end{theorem}

Thus, our strategy in finding a reduction order to cover the handwritten ruleset is to pick an order $>_a$ such that for all rules $l \rewrites r$, either $l >_a r$ or $l =_a r$. Then, we pick another order $>_b$ such that for all rules $l \rewrites r$ where $l =_a r$, either $l >_b r$ or $l =_b r$. We continue in this way until a sequence of orders has been found such that for their product $>_{\times}$, $l >_{\times} r$ holds for the entire ruleset.  Our final ordering consists of 13 component orders, given in full in appendix~\ref{a:sreductionorder}.

Many of our component orders are defined using measure functions that count the number of particular operations or other features in a term. We say that $s > t$ when $s$ has more vector operations than $t$, then when $s$ has more division, modulo and multiplications operations, and so on. As a sample proof sketch of this flavor of order, consider an order $s_1 >_* s_2$ that holds when the number of multiplication operations is greater in $s_1$ than in $s_2$. We represent this through a measure function $|s_1|_*$ that returns the count of multiplication operations in $s_1$; since this function maps a term to a natural number, the order is clearly well-founded. The order is also compatible with $\Sigma$-operations; we compute our measure function as follows:

\[
|f(t_1,...,t_n)|_* = \sum_i^n |t_i|_* + \begin{cases} 1 & \textrm{if } f = * \\
                                                      0 & \textrm{otherwise}
                                        \end{cases}
\]

It clearly follows that given $|s_1|_* > |s_2|_*$, it must be true that:

\[
|f(t_1,...t_{i-1},s_1,t_{i+1},...,t_n)|_* > |f(t_1,...t_{i-1},s_2,t_{i+1},...,t_n)|_*
\]

To ensure the order is closed under substitution, we need to add one more constraint. Imagine a rule $x * 2 \rewrites x + x$. Although there are fewer $*$ symbols in the righthand term than on the left, that would not be true for a substitution $\sigma = \{x \mapsto (z * z)\}$. We add a condition that for every variable present in $s_1$, it must occur either fewer or an equal number of times in $s_2$. With this constraint there is no possible substitution that increases the value of the measure function in $s_2$ that would not result in an increase by an equivalent or greater amount in $s_1$. This gives us the order:

\[
s_1 >_* s_2 \textrm{ iff } |s_1|_* > |s_2|_* \wedge \forall x \in \mathcal{V}ar(s_1) . |s_1|_x \geq |s_2|_x
\]

Most of the component orders in the full reduction order take the form above. These orders guarantee termination no matter what sequence rewrite rules are applied to an expression. However, in a few exceptional cases, we were obliged to take into account the order in which rules are applied in the Halide TRS algorithm.

For example, one existing rule is the canonicalization $(c_0 - x) + y \rewrites (y - x) + c_0$ where $c_0$ is a constant. If $y$ is also a constant, this rule forms a cycle with itself, and could not possibly obey any reduction order. Fortunately, the rule immediately before it in the TRS handles that specific case ($((c_0 - x) + c_1 \rewrites \texttt{fold}(c_0 + c_1) - x)$), so by this sort of non-local reasoning we know that $y$ is not a constant, and therefore the rule strictly decreases a measure which counts the number of constants on the right-hand side of an addition.

%The handwritten ruleset had many rules that eliminated the occurrence of a variable, such as $x + x \rewrites x * 2$. It seems natural to define an order based on the measure function $|s_1|_x$, but for a substitution $\sigma = \{x \mapsto 3\}$, $|3 + 3|_x \not > |3 * 2|_x$. However, the simplifier algorithm always attempts constant folding before any other rule, so we know that the rule $x + x \rewrites x * 2$ can only be invoked if $x$ is not a ground term.

%Similarly, we have several rules that factor out an occurrence of a variable and introduce the constant 0 into the expression. We define an order on the occurrences of the constant 0 by defining a measure function that takes the count of terminals or leaves in the expression and subtracts the count of the constant 0; if terms $s_1$ and $s_2$ have the same number of leaves, but more of the leaves of $s_2$ are the constant 0, then $s_1 > s_2$.

%\[
%s_1 >_0 s_2 \textrm { iff } |s_1|_{leaf} - |s_1|_0 > |s_2|_{leaf} - |s_2|_0 \wedge |s_1|_{leaf} = |s_2|_{leaf} \wedge \forall x \in \mathcal{V}ar(s_1) . |s_1|_x \geq |s_2|_x
%\]

% For the rule $\texttt{max}(x + y, x) \rewrites \texttt{max}(y, 0) + x$, the order will not hold for the substitution $\sigma = \{x \mapsto 0\}$. However, we know the rule $0 + x \rewrites x$ will be invoked before this one, so the rule cannot be evaluated on the expression $\texttt{max}(0 + y, 0)$.

Relying on non-local reasoning makes our order more brittle; if the simplifier algorithm were to be changed, the termination guarantee could be lost. However, we use only a small number of basic rules in this way, which are unlikely to be changed.

Besides giving a termination guarantee, the reduction order is necessary if we want to synthesize new rewrite rules. If we do not constrain newly-synthesized rules to obey a consistent reduction order with the existing human-written ones, they may form cycles with the existing rules and cause infinite recursion in the TRS. Additionally, the reduction order is the formal encoding of the types of transformations we find desirable, so the reduction order limits synthesis to rules that rewrite expressions in a useful direction; this is discussed in greater detail in chapter~\ref{chapter:synthfromscratch}.

In constructing the reduction order, we found \NumOrderingProblems rules that contradicted a desirable ordering, and submitted patches to either delete or modify them. With this amendment, the reduction order can be shown to hold over the entire Halide ruleset, and the guarantee of termination is complete. To ensure this guarantee is preserved, we built a script that automatically checks the full set of rules in the compiler to ensure they respect the reduction order.

\subsection{Evaluation of verification work}

Changes to the simplifier ruleset undergo a stringent development process: new rules are peer reviewed after they are proven on paper, and fuzz-testing is used to discover bugs. It is thus reasonable to ask whether mechanized verification can add any value. Our verification discovered \NumRulesFixed previously-unknown correctness bugs and \NumPredicatesRelaxed instances of rules whose predicates were overly restrictive. The former bugs eluded the fuzzer; the latter are deemed too hard to detect so the fuzzer does not look for them.  Some of the corrected rules are listed in more detail in table~\ref{tab:verfirstround}.

%The first use of verification took place when the TRS had not yet been merged into the Halide master branch. We ran the verification pipeline and discovered \NumRulesFixed incorrect rewrite rules, listed in Table~\ref{tab:verfirstround}. The rules that could not be checked with Z3 were proved true using the Coq proof assistant (none of the manually proved rules were found to be incorrect). While these bugs were found automatically the fixes were performed by hand, as the synthesis pipeline did not yet exist.

Furthermore, because the verification infrastructure was in place, it was possible to verify a change of division semantics without much additional effort, identifying 44 rules that were incorrect under the new semantics. This change to the semantics of Halide may not have even been attempted without the verifier. In this change, Halide defined division or modulo by zero to evaluate to zero, instead of being undefined behavior, in response to an issue discovered by Alex Reinking~\citep{reinkingthesis}. Existing tests and real uses of Halide were useless as a test of this change, as \emph{they were all carefully written to never divide by zero}. Within the TRS, this change required rechecking every rewrite rule that involves the division and modulo operators. Whereas previously each rule assumed that a denominator on the LHS could not be zero, now it was necessary to either show that the rule was still correct in the case where a denominator was zero, or constrain the rule to only trigger when the denominator was known to be non-zero. This was done by encoding the new semantics into the verifier, and reverifying all rules. Because division and modulo is involved, these rules cannot always be mechanically verified.
\NumZdivCoqProvedRules rules were reverified with a human in the loop by revisiting and modifying existing Coq proofs. The mechanical re-verification was all but push-button; the manual effort for updating the Coq proofs was non-trivial, but about half of the effort of writing the original proofs from scratch. In this process, \NumZdivFalseRules existing rules were found to be incorrect in the new semantics and fixed\footnote{\url{https://github.com/halide/Halide/pull/4439}}.
Two of them were in fact not related to division, but were instead the first discovery of the bugs injected in a patch after the initial ruleset verification.
The remaining 42 rules were modified to only trigger when the denominator was known to be non-zero, either by adding a predicate to the rule, or by exploiting the TRS’s ability to track constant bounds and alignment of subexpressions. Three examples of now-incorrect rules were:

\begin{align*}
(x/y)*y + (x\;\%\;y) \rewrites & x \\
  -1 / x \rewrites & \hsel(x < 0, 1, -1) \\
(x + y)/x \rewrites & y/x + 1
\end{align*}

The first was modified to:
\[
(x/c_0)*c_0 + (x\;\%\;c_0) \rewrites x \pred c_0 \neq 0
\]
and the other two were constrained to only trigger when the denominator is known to be non-zero via other means.

The cases discussed in Section~\ref{sub:bugfixes} all concern fixing existing problems while not introducing new ones. By giving a proof of soundness, showing that the ruleset is correct and that the rules are cycle-free, we also remove two entire classes of future bugs. For reference, over the life of the Halide project there have been 14 pull requests that fix incorrect rules, and 3 pull requests that modify rules in order to avoid cycles. Fixing a reduction order also guarantees that no new cycles can be introduced as long as new rules obey this order; without such a guide, it is possible to introduce a rule that would close a loop in some sequence of existing rule applications and cause a cycle, resulting in infinite recursion during compilation.

\begin{table*}
\caption{Rules corrected through the first round of verification.}
{\renewcommand{\arraystretch}{1.2}
\begin{tabular}{r|l|l}
& Rule & Counterexample \\
\hhline{=|=|=}
Wrong &  $\frac{x * c_0}{c_1} \rewrites \frac{x}{(c_1 / c_0)} \pred c_1 \;\%\; c_0 = 0 \wedge c_1 > 0$ & $c_0 = -1, c_1 = 2, x = 1$\\
Fixed & $\frac{x * c_0}{c_1} \rewrites \frac{x}{(c_1 / c_0)} \pred c_1 \;\%\; c_0 = 0 \wedge c_0 > 0 \wedge \frac{c_1}{c_0} \neq 0$ & \\
\hhline{=|=|=}
Wrong & $(\frac{x + c_0}{c_1})*c_1 - x \rewrites x \;\%\; c_1 \pred c_1 > 0 \wedge c_0 + 1 = c_1$ & $c_0 = 2, c_1 = 3, x = -5$\\
Fixed & $(\frac{x + c_0}{c_1})*c_1 - x \rewrites {-x} \;\%\; c_1 \pred c_1 > 0 \wedge c_0 + 1 = c_1$ & \\
\hhline{=|=|=}
Wrong & $x - (\frac{x + c_0}{c_1})*c_1 \rewrites -(x \;\%\; c_1) \pred c_1 > 0 \wedge c_0 + 1 = c_1$ & $c_0 = 2, c_1 = 3, x = -5$\\
Fixed & $x - (\frac{x + c_0}{c_1})*c_1 \rewrites ((x + c_0) \;\%\; c_1) + {-c_0} \pred c_1 > 0 \wedge c_0 + 1 = c_1$ & \\

\end{tabular}
}
\label{tab:verfirstround}
\end{table*}

\section{Related Work}

An \textit{equivalence graph} or e-graph (introduced by \citep{nelson1980techniques}, see \citep{willsey2021egg} for a recent treatment) is a data structure used to compute applications of the rules of a term rewriting system. The algorithm builds up equivalence classes by successively applying all rules to all expressions within those classes, then queries to see if two expressions are equivalent by checking if they are present in the same class. Like our algorithm, it does not backtrack, but the e-graph can require significant amounts of memory, which our algorithm avoids.

AProVE~\citep{giesl2004automated, giesl2017analyzing} is a tool that automatically generates proofs of termination for term rewriting systems (as well as programs in Java, C, Haskell, and so on). It employs a variety of techniques for doing so. It may prove a TRS terminations through direct termination proofs, or finding a reduction order that fits all rules in the TRS, as we do in our work, searching classes of orders including path orders, Knuth-Bendix orders, and polynomial orders. It may also prove termination through dependency pairs (finding all instances in which terms of RHSs can unify with rule LHSs, then showing that a weakly monotonic ordering holds over all dependency pairs) or by abstracting rules by their effect on term height and proving that rule application must cause term height to decrease. It also employs techniques to remove portions of the ruleset that have no effect on termination and for reducing the size of the ruleset to make termination proof search more efficient.
A proof of termination by term height abstraction would not be useful for synthesis, since it encodes no information about progress towards a goal state. A proof of termination through dependency pairs could do so, and since it uses weakly rather than strongly monotonic orders, it could permit rewriting strategies that our technique cannot. However, this method requires reasoning over the full ruleset rather than individual rules, so using it as a specification for synthesis would result in a significantly more complicated synthesis task. However, it would be interesting to see if AProVE can find a direct termination proof for the Halide simplifier and handwritten variable solver TRSs and if those direct termination proofs are human-interpretable. If AProVE can find reduction orders to fit, it would also be interesting to see what rulesets could be synthesized using them as an input to our synthesis pipeline.


% Womelsdorf Lab burst library user guide - Algorithms
% Written by Christopher Thomas.

\chapter{Segmentation and Parametric Representation}
\label{sect-math}

%
%
%
\section{Segmentation -- Detecting Burst Events}
\label{sect-math-segment}

``Segmentation'' is the process of deciding which portions of an input
signal contain oscillatory bursts. Historically, this has usually been done
by performing a Hilbert transform to get an analytic signal with known
magnitude and phase, and looking for magnitude excursions above some
threshold (usually $3\sigma$ with respect to the mean magnitude across all
parts of the waveform and all trials). Many variants of this approach exist,
and several other approaches have been used, but these are beyond the scope
of this document.

The wlBurst library provides an implementation of magnitude-threshold
detection, using the algorithm illustrated in Figure \ref{fig-math-algmag}.
Rather than using a fixed threshold, this implementation uses a
low-pass-filtered version of the magnitude as its ``DC'' level, and looks
for magnitude excursions with respect to that (comparing to a local rather
than global mean). The intention is that this is less sensitive to drift
in signal properties.

A typical event segmented by magnitude thresholding is shown in Figure
\ref{fig-math-evmag}. Notice that the analytic magnitude (orange envelope
in the top pane) is only well-behaved for the band-pass-filtered signal.

\figdef{\begin{center}
\includegraphics[width=6in]{figs/alg-detmag-v1.png}\end{center}}
{Segmentation by Magnitude Thresholding}
{fig-math-algmag}

\figdef{\begin{center}
\includegraphics[height=3.5in]{plots/multi-evmag-be-vsbpf-0051.png} \\
Bandpass \\
\vspace{2\baselineskip}
\includegraphics[height=3.5in]{plots/multi-evmag-be-vsraw-0051.png} \\
Wideband
\end{center}}
{Typical event segmented by magnitude thresholding.}
{fig-math-evmag}

\FloatBarrier

In addition to magnitude-threshold event detection, the wlBurst library
provides an implementation of ``frequency-stability'' event detection. This
algorithm looks at the frequency of the analytic signal (middle panes in
Figure \ref{fig-math-evmag}. The analytic frequency is defined as the
derivative of the phase of the analytic signal (which is provided directly
by the Hilbert transform).

During oscillation events, the frequency is expected to be well-defined
(staying within a narrow range with few or no excursions). Outside of
oscillation events, when the neural signal is not oscillating with a
well-defined frequency, excursions in analytic frequency are expected to
occur. These two situations are difficult to distinguish in the Hilbert
transform of the band-pass-filtered waveform, but are visually apparent
in the Hilbert transform of the wideband signal.

To perform automated segmentation, high-frequency noise with known
properties is added to the band-pass-filtered signal. The Hilbert transform
of this ``noisy'' signal is computed, and the variance of the resulting
analytic frequency signal is measured. This algorithm is illustrated
in Figure \ref{fig-math-algfreq}.

A typical event segmented by frequency stability detection is shown in
Figure \ref{fig-math-evfreq}. Notice that the wideband signal shows
out-of-band events, and that the band-pass-filtered signal with added
noise shows only in-band events and has more consistenly different variance
in the frequency signal between event and non-event regions. The ``noisy''
signal is only used for detection - parameter extraction uses the clean
band-pass-filtered signal.

\figdef{\begin{center}
\includegraphics[height=7.5in]{figs/alg-detfreq-v1.png}\end{center}}
{Segmentation by Frequency Stability}
{fig-math-algfreq}

\figdef{\begin{center}
\includegraphics[height=3.5in]{plots/multi-evfreq-al-vsraw-0026.png} \\
Wideband \\
\vspace{2\baselineskip}
\includegraphics[height=3.5in]{plots/multi-evfreq-al-vsnoisy-0026.png} \\
BPF Plus Noise
\end{center}}
{Typical event segmented by frequency stability.}
{fig-math-evfreq}

\FloatBarrier

%
%
%
\section{Feature Extraction -- Curve-Fitting Oscillatory Bursts}
\label{sect-math-param}

``Feature extraction'' or ``parameter extraction'' is the process of
building a description of a burst event that contains all desired information
about the burst. This is usually done as a separate step after segmentation
(burst detection). The model used to represent oscillatory bursts in the
wlBurst library is shown in Figure \ref{fig-math-params}, and is described
in \texttt{EVENTFORMAT.txt} (as the ``chirpramp'' model). Future versions
of the wlBurst library will explicitly support additional models; for the
time being, custom models may be used provided care is taken to not call
analysis or plotting functions that assume use of the ``chirpramp'' model.

\figdef{\begin{center}
\includegraphics[width=0.9\textwidth]{figs/synth-burst.png}
\end{center}}
{Parametric burst model (``\texttt{chirpramp}'' model).}
{fig-math-params}

``Chirpramp'' model fits are performed through several steps:

\begin{itemize}
%
\item Endpoints of the burst are assumed. These may either be the times
when the detection criterion (absolute magnitude or frequency variance)
crossed its threshold, or where the detection criterion crosses some other
lower threshold (for ``dual'' variants of the magnitude-threshold and
frequency-stability detection algorithms).
%
\item The magnitude envelope is curve-fit. This involves choosing roll-on
and roll-off times, and performing exponential and linear fits to the
analytic magnitude and choosing the best match. Choosing roll-on and roll-off
times may be done via a grid search (fast but less accurate) or by simulated
annealing (slow but more accurate).
%
\item The frequency and phase are curve-fit. This involves performing
exponential and linear fits to the analytic frequency and choosing the best
match, followed by testing possible starting phases and choosing the one that
best matches modeled burst phase and the analytic phase across the duration
of the burst.
%
\item Optionally, a simulated annealing algorithm may perturb all components
of the parametric burst to try to better match the input waveform. This is
slow but slightly improves accuracy.
%
\end{itemize}

User-specified segmentation and parameter extraction algorithms may
alternatively be used, per \linebreak
\texttt{SEGCONFIG.txt} and \texttt{PARAMCONFIG.txt}.

%
% This is the end of the file.


\documentclass[11pt, oneside]{amsart}   	% use "amsart" instead of "article" for AMSLaTeX format
\usepackage[top=1.3cm, bottom=1.3cm]{geometry}                		% See geometry.pdf to learn the layout options. There are lots.
\geometry{letterpaper}                   		% ... or a4paper or a5paper or ...
%\geometry{landscape}                		% Activate for rotated page geometry
%\usepackage[parfill]{parskip}    		% Activate to begin paragraphs with an empty line rather than an indent
\usepackage{graphicx}				% Use pdf, png, jpg, or eps§ with pdflatex; use eps in DVI mode
\usepackage{bm}							% TeX will automatically convert eps --> pdf in pdflatex
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{subcaption}
\usepackage{float}
\pagenumbering{gobble}
%SetFonts

\newcommand\indep{\protect\mathpalette{\protect\indeP}{\perp}}
 \def\indeP#1#2{\mathrel{\rlap{$#1#2$}\mkern2mu{#1#2}}}

%SetFonts

\title{MVA-DM2 Probabilistic graphical models}
\author{Hugo Cisneros}
\date{}							% Activate to display a given date or no date

\begin{document}
\maketitle
\section{Conditional independence and factorizations}
 \vfill
\textbf{1.}  The implied factorization of a distribution $p\in \mathcal{L}(G)$ is
\begin{align*}
p(x, y, z, t) = p(t|z)p(z|x, y)p(x)p(y)
\end{align*}
For two real valued independent r.v $X$ and $Y$ that follow the distribution $\mathcal{U}(0, 2)$, and $Z = X+Y$, $T = \mathbf{1}_{(Z \geq 3)}$,$\boxed{\text{the statement } X\indep Y | T \text{ is not true}}$.
Indeed, knowing that $X+Y \geq 3$ makes the two random variables not independent ($p(x| y=2, z=1) = p(x \geq 1) \neq p(x|z=1)$).
\vfill

 \textbf{2.a.} If $Z$ is a binary variable, let $p = p(z=1) = 1 - p(z=0)$. We have
 \begin{align*}
 	p(x, y) &= p(x)p(y) = p(x, y|z=1)p + p(x, y|z=0)(1-p) & (X\indep Y ) \\
	& = p(x|z=1)p(y|z=1)p + p(x|z=0)p(y|z=0)(1-p) & (X\indep Y | Z)\\
	&  = p(x)p(y)\left(\frac{1}{p} p(z=1|x)p(z=1|y) + \frac{1}{1-p}p(z=0|x)p(z=0|y)\right)
 \end{align*}
 If we write $p_x = p(z=1|x) = 1-p(z=0|x)$ and $p_y = p(z=1|y)$, the equality becomes $p^2 - (p_x + p_y)p + p_xp_y
 = 0$ which yields $ \left(p(z=1) = \right)\ p = p_x\ ( =  p(z=1|x) )$ or $p = p_y$. Therefore either $\boxed{X \indep Z \text{ or } Y \indep Z}$.
\vfill

 \textbf{2.b.} Let $\mathcal{A}$ be the finite space $\{0, 2\}$, $X_0$ and $Y_0$ two independent random variables with uniform distributions on $\mathcal{A}$ ,
 we define $X =\begin{bmatrix} X_0 \\ 0 \end{bmatrix}$, $Y = \begin{bmatrix} 0 \\ Y_0 \end{bmatrix}$ and $Z = \begin{bmatrix} X_0 \\ Y_0 \end{bmatrix}$.
 \\

 $X$, $Y$ and $Z$ are three random variables with $X \indep Y$ and $ X \indep Y | Z$
 \begin{itemize}
\item  $\mathbb{E}[XY] =  \begin{bmatrix} 0 \\ 0 \end{bmatrix} = \mathbb{E}[X]\mathbb{E}[Y]$
\item $\forall (k,l) \in \mathcal{A}\times\mathcal{A},  \mathbb{E}[X|Z=(k,l)] =  \begin{bmatrix} k \\ 0 \end{bmatrix}$ and similarly for $Y$, therefore $\mathbb{E}[XY|Z] =  \begin{bmatrix} 0 \\ 0 \end{bmatrix} = \mathbb{E}[X|Z]\mathbb{E}[Y|Z]$
 \end{itemize}
 However, $X$ and $Y$ are not independent from $Z$, since  $\mathbb{E}[XZ] = \begin{bmatrix} \mathbb{E}[X_0^2] \\ 0 \end{bmatrix}$, $\mathbb{E}[X]\mathbb{E}[Z] = \begin{bmatrix} \mathbb{E}[X_0]^2 \\ 0 \end{bmatrix}$ and $1 = \boxed{\mathbb{E}[X_0]^2 \neq \mathbb{E}[X_0^2] }= 2$.
 \vfill

 \clearpage
 \section{Distributions factorizing in a graph}
  \vfill
 \textbf{1.} Let $p\in\mathcal{L}(G)$, $p(x) = \prod_{s\in V} p(x_s|x_{\pi_s})$. Since $\{i\rightarrow j\}$ is a covered edge,  $\pi_j = \pi_i \cup \{i\}$. Therefore,
 \begin{align*}
 p(x) = p(x_i|x_{\pi_i})\cdot p(x_j|x_{\pi_i}, x_i) \cdot\prod_{s\in V-\{ i, j \}} p(x_s|x_{\pi_s})
 \end{align*}

 And

 \begin{align*}
 p(x_i|x_{\pi_i})\cdot p(x_j|x_{\pi_i}, x_i) &= \frac{p(x_i, x_{\pi_i})}{p(x_{\pi_i})} \frac{p(x_i, x_{\pi_i}, x_j)}{p(x_{\pi_i}, x_i)} =p(x_i,  x_j |x_{\pi_i})\cdot \frac{p(x_j, x_{\pi_i})}{p(x_j, x_{\pi_i})}\\[+5pt]
 & = p(x_i | x_{\pi_i}, x_j)\cdot p(x_j|x_{\pi_i})
 \end{align*}
 \\
 In $G'$, $\{i\rightarrow j\}$ has been replaced by $\{j\rightarrow i\}$, thus $\pi_i' = \pi_i \cup \{j\}$ and $\pi_j' = \pi_i $, the equation above shows that $p$ can be factorized in this new graph, hence $p\in \mathcal{L}(G')$. From the same equality above, if $p\in \mathcal{L}(G')$, also $p\in \mathcal{L}(G)$. Therefore $\boxed{ \mathcal{L}(G) =  \mathcal{L}(G')}$.
 \vfill

 \textbf{2.} Since $G$ is a directed tree, it doesn't contain any v-structure and all nodes have either 0 or 1 parent (only the root of the tree doesn't have a parent). Moreover, all cliques or $G'$ contain at most 2 elements because any clique of 3 or more elements would contain a 3-clique. That 3-clique in the directed graph must contain either a v-structure or a cycle, which is not possible in a directed tree.
 \\

 Hence if $p\in \mathcal{L}(G)$, $p(x) = \prod_i p(x_i | x_{\pi_i}) = \prod_i \Psi_i(x_i, x_{\pi_i})$, where $\Psi_i: (x_i, x_{\pi_i})\rightarrow p(x_i|x_{\pi_i})$ is a function of an element and its parent (a 2-clique of the graph), or of the root of the tree only (a 1-clique). We have $\boxed{\mathcal{L}(G) \subset \mathcal{L}(G')}$.
 \\

 For $p\in \mathcal{L}(G')$, we have for all $x$, $p(x) = \frac{1}{Z} \prod_{c\in\mathcal{C}} \psi_c(x_c)$ where $\mathcal{C}$ is the set of maximal cliques of $G'$ and $Z = \sum_x  \prod_{c\in\mathcal{C}} \psi_c(x_c)$. From the argument above, we know that all maximal cliques of $G'$ have size 2 except for the root of the corresponding directed tree. Therefore, $p(x) = \prod_i f(x_i, x_{\pi_i})$. Since the $f_i$ are defined up to a multiplicative constant, we can make them equal to a conditional probability by properly normalizing. Hence  $\boxed{\mathcal{L}(G') \subset \mathcal{L}(G)}$.

 \vfill

 \clearpage
\section{Implementation - Gaussian mixtures}
 \vfill
 \textbf{3.a.} The histogram below shows the repartition of distortions for 1000 random initializations of K-Means in a 10 by 10 square centered on the global mean of the dataset. It clearly shows that the algorithm might converge to several local minima, where distortion scores differ slightly. The different cluster centers estimated by the algorithm are represented by blue stars on next page's plot of the K-means algorithm.
\begin{figure}[h!]
\centering
 \includegraphics[width=0.7\linewidth]{Distortions.pdf}
 \caption{Histogram of distortions for 1000 random initializations of the K-means algorithm}
\end{figure}
\vfill

\textbf{3.b.} When we write $\Sigma_{j, t} = \sigma_{j, t} I$, the part of the likelihood to be maximized that depends on $\sigma$ is written
\begin{align*}
\sum_{i=1}^n \sum_{j=1}^k \tau_i^j\left(-\frac{n}{2}\log(\sigma_{j, t}) - \frac{1}{2\sigma_{j,t}} ||x_i - \mu_{j, t}||^2\right)
\end{align*}
For each j, the expression above is equal to $-\frac{n}{2}\log(\sigma_{j,t})\sum_i \tau_i^j - \frac{1}{2\sigma_{j,t}}\sum_i \tau_i^j ||x_i - \mu_{j,t}||^2$. They can be minimized separately. The gradient of this expression with respect to $\sigma_{j}$ reads:
\begin{align*}
-\frac{n}{2\sigma_{j}}\sum_i \tau_i^j + \frac{1}{2\sigma_{j}^2} \sum_i\tau_i^j ||x_i - \mu_{j}||^2 = 0 \implies \boxed{\sigma_{j, t+1} = \frac{\sum_i\tau_i^j ||x_i - \mu_{j, t+1}||^2}{n \sum_i \tau_i^j}}
\end{align*}
The other derivations ($\mu$ and $\pi$) are the same than for the complete Gaussian mixture model detailed in the course notes.
\vfill

\textbf{3.c.} From the course notes we have $\boxed{\Sigma_{j,t+1} = \dfrac{\sum_i\tau_i^j(x_i-\mu_{j,t+1})(x_i-\mu_{j,t+1})^T}{ \sum_i\tau_i^j}}$
\vfill

\textbf{3.d.} For the isotropic gaussian model, the log-likelihood on the train set is -2649.9 and -2639 for the test set. Visually, the results for clustering are close to the results of K-means, which can be understood by the fact that the distance function used in K-means is the isotropic norm L2. However, the ``equivalent'' distance function in the isotropic Gaussian mixture model is non quadratic and therefore penalizes differently points far from a cluster center depending on the covariance of the cluster.

The mixture of full gaussians allows for dealing with much more complex dataset shapes, which in the case of this dataset yields higher log-likelihoods: -2340.2 for the training set and -2431 for the test set. Estimations more closely ressemble the training data and are more accurate than the isotropic model as the test results show.
\vfill
 \clearpage
 \begin{figure}[t!]
\centering
\begin{subfigure}{.49\textwidth}
  \centering
  \includegraphics[width=\linewidth]{KMeans.pdf}

\end{subfigure}
\begin{subfigure}{.49\textwidth}
  \centering
  \includegraphics[width=\linewidth]{Isotropic.pdf}

\end{subfigure}
\\[+5pt]
\begin{subfigure}{.49\textwidth}
  \centering
  \includegraphics[width=\linewidth]{Full.pdf}

\end{subfigure}

\end{figure}

 \end{document}