\section{Lecture 5 - A \tsolver for \Idl}

\begin{frame}
  \frametitle{A \tsolver for \Idl}

  \scriptsize

  The constraint $\quad x - y \leq c \quad$ says that

  \begin{center}
    ``the distance between $x$ and $y$ is at most $c$''
  \end{center}

  This can be encoded as $(x,y;c)$

  \begin{center}
    \input{arc}
  \end{center}

  \vfill
  So, a set $\babst{\mu}$ can be encoded as a graph. Concrete example:

  \begin{columns}

    \begin{column}{.3\textwidth}
      \ra{1.3}
      $$
      \begin{array}{rcr}
	x - y & \leq & 8   \\
	y - z & \leq & -1  \\
	z - x & \leq & -6  \\
	z - w & \leq & 2   \\
	w - x & \leq & -10 \\
	w - t & \leq & 0   \\
	t - x & \leq & 3
      \end{array}
      $$
    \end{column}

    \begin{column}{.6\textwidth}
      \begin{center}
	\input{example}
      \end{center}
    \end{column}

  \end{columns}
  \vfill
  $G( V, E )$: $V = \{ x, y, z, w, t \}$, $E = \{ (x,y;8), (y,z;-1), (z,x;-6), (z,w;2), (w,x;-10), \ldots \}$

\end{frame}

\begin{frame}
  \frametitle{A \tsolver for \Idl}

  \scriptsize

    \begin{theorem}[Translation]
      \label{the:idl}
      $\babst{\mu}$ is \Idl-unsatisfiable
      \begin{center}
      iff
      \end{center}
      there is a {\bf negative cycle} in the corresponging graph $G(V,E)$
    \end{theorem}

  \vfill
  E.g.:

  \begin{columns}

    \begin{column}{.3\textwidth}
      \ra{1.3}
      $$
      \begin{array}{rcr}
	\coloneat{x - y}{1} & \coloneat{\leq}{1} & \coloneat{8}{1}   \\
	\coloneat{y - z}{1} & \coloneat{\leq}{1} & \coloneat{-1}{1}  \\
	\colfouat{z - x}{1} & \colfouat{\leq}{1} & \colfouat{-6}{1}  \\
	\coloneat{z - w}{1} & \coloneat{\leq}{1} & \coloneat{2}{1}   \\
	\coloneat{w - x}{1} & \coloneat{\leq}{1} & \coloneat{-10}{1} \\
	\colfouat{w - t}{1} & \colfouat{\leq}{1} & \colfouat{0}{1}   \\
	\colfouat{t - x}{1} & \colfouat{\leq}{1} & \colfouat{3}{1}
      \end{array}
      $$
    \end{column}

    \begin{column}{.6\textwidth}
      \begin{center}
	\begin{overlayarea}{.5\textwidth}{3cm}
	  %\only<2|handout:0>{\input{example}}
	  \input{example_hl}
	\end{overlayarea}
      \end{center}
    \end{column}

  \end{columns}

\end{frame}

\begin{frame}
  \frametitle{Lecture 5 - Exercize 4}

  \scriptsize

  Let's first recall the notion of {\bf minimality}
  \begin{center}
  A conflict $\nu^\Boo$ is {\bf minimal} if it does not contain redundant \tatoms
  \end{center}
  A \tatom $P$ in a conflict $\nu^\Boo$ is redundant if $\nu^\Boo \setminus \{ P \}$
  is still a conflict
  \vfill
  So, how do we check, in general, that a conflict $\nu^\Boo = \{ P_1, \ldots, P_n \}$ is minimal ?
  Iteratively for $i=1,\ldots,n$, we see if $\nu^\Boo \setminus \{ P_i \}$ is still a conflict.
  \vfill\pause
  In the case of difference logic every conflict is minimal {\bf by construction}. In fact $\nu^\Boo$
  is a conflict if and only if it is a {\bf cycle} with negative sum.
  \vfill
  Doing $\nu^\Boo \setminus \{ P_i \}$ is equivalent to breaking the cycle, no matter what $P_i$.
  Therefore all \tatoms are not redundant, and so conflicts are minimal.

\end{frame}

\begin{frame}
  \frametitle{Lecture 5 - Exercize 5}

  \scriptsize

  Prove
  \vfill

  \begin{exampleblock}{Observation 2}
    A set of constraints \\
    $\{\ \colone{x_1} - x_2 \leq c_1,\ x_2 - x_3 \leq c_2,\ \ldots,\ x_{n-1} - \coltwo{x_n} \leq c_{n-1}\ \}$ \\
    implies
    $\quad\quad\colone{x_1} - \coltwo{x_n} \leq c_n\quad\quad$
    iff
    $\quad\quad c_1 + c_2 + \ldots c_{n-1} \leq c_n$
  \end{exampleblock}

  \vfill
  using
  \vfill

  \begin{lemma}[Farka's Lemma for \Idl]
    $\babst{\mu}$ is unsatisfiable iff there exists a subset
    $\babst{\nu} = \{\ \coltwo{x_1} - x_2 \leq c_1,\ x_2 - x_3 \leq c_2,\ \ldots,\ x_n - \coltwo{x_1} \leq c_n\ \}$ of $\babst{\mu}$
    such that $c_1 + \ldots + c_n < 0$
  \end{lemma}

\end{frame}

\begin{frame}
  \frametitle{Lecture 5 - Exercize 5}

  \scriptsize

  \begin{exampleblock}{Observation 2}
    A set of constraints \\
    $\{\ \colone{x_1} - x_2 \leq c_1,\ x_2 - x_3 \leq c_2,\ \ldots,\ x_{n-1} - \coltwo{x_n} \leq c_{n-1}\ \}$ \\
    implies
    $\quad\quad\colone{x_1} - \coltwo{x_n} \leq c_n\quad\quad$
    iff
    $\quad\quad c_1 + c_2 + \ldots c_{n-1} \leq c_n$
  \end{exampleblock}

  \vfill
  is better formalized as
  \vfill

  \begin{center}
  $(\colone{x_1} - x_2 \leq c_1\ \wedge\ x_2 - x_3 \leq c_2\ \wedge\ \ldots\ \wedge\ x_{n-1} - \coltwo{x_n} \leq c_{n-1}) \rightarrow (\colone{x_1} - \coltwo{x_n} \leq d_n)$ \\
  is valid iff \\
  $c_1 + c_2 + \ldots c_{n-1} \leq d_n$
  \end{center}

  \begin{lemma}[Farka's Lemma for \Idl]
    $\babst{\mu}$ is unsatisfiable iff there exists a subset
    $\babst{\nu} = \{\ \coltwo{x_1} - x_2 \leq c_1,\ x_2 - x_3 \leq c_2,\ \ldots,\ x_n - \coltwo{x_1} \leq c_n\ \}$ of $\babst{\mu}$
    such that $c_1 + \ldots + c_n < 0$
  \end{lemma}

  \vfill
  is better formalized as
  \vfill

  \begin{center}
  $\colone{x_1} - x_2 \leq c_1\ \wedge\ x_2 - x_3 \leq c_2\ \wedge\ \ldots\ \wedge\ x_{n-1} - x_n \leq c_{n-1}\ \wedge\ x_n - \colone{x_1} \leq c_n$ \\
  is unsatisfiable iff \\
  $c_1 + c_2 + \ldots c_{n-1} + c_n < 0$
  \end{center}

\end{frame}

\begin{frame}
  \frametitle{Lecture 5 - Exercize 5}

  \scriptsize

  \begin{center}
  $(\colone{x_1} - x_2 \leq c_1\ \wedge\ x_2 - x_3 \leq c_2\ \wedge\ \ldots\ \wedge\ x_{n-1} - \coltwo{x_n} \leq c_{n-1}) \rightarrow (\colone{x_1} - \coltwo{x_n} \leq d_n)$ \\
  is valid iff \\
  $c_1 + c_2 + \ldots c_{n-1} \leq d_n$
  \end{center}

  \vfill
  is equivalent to (using the well-known fact: $\varphi \rightarrow \psi$ is valid iff $\varphi \wedge \neg \psi$ is unsat)
  \vfill

  \begin{center}
  $\colone{x_1} - x_2 \leq c_1\ \wedge\ x_2 - x_3 \leq c_2\ \wedge\ \ldots\ \wedge\ x_{n-1} - \coltwo{x_n} \leq c_{n-1}\ \wedge\ \neg(\colone{x_1} - \coltwo{x_n} \leq d_n)$ \\
  is unsatisfiable iff \\
  $c_1 + c_2 + \ldots c_{n-1} \leq d_n$
  \end{center}

  \vfill\pause
  which is equivalent to (using the fact $\neg (x - y \leq c)\ \Longleftrightarrow\ y - x \leq - c - 1$)
  \vfill

  \begin{center}
  $\colone{x_1} - x_2 \leq c_1\ \wedge\ x_2 - x_3 \leq c_2\ \wedge\ \ldots\ \wedge\ x_{n-1} - \coltwo{x_n} \leq c_{n-1}\ \wedge\ \coltwo{x_n} - \colone{x_1} \leq - d_n - 1$ \\
  is unsatisfiable iff \\
  $c_1 + c_2 + \ldots c_{n-1} \leq d_n$
  \end{center}

  \vfill\pause
  which is equivalent to (using the fact $c \leq d\ \Longleftrightarrow\ c - d \leq 0\ \Longleftrightarrow\ c - d - 1 < 0$)
  \vfill

  \begin{center}
  $\colone{x_1} - x_2 \leq c_1\ \wedge\ x_2 - x_3 \leq c_2\ \wedge\ \ldots\ \wedge\ x_{n-1} - x_n \leq c_{n-1} \wedge x_n - \colone{x_1} \leq - d_n - 1$ \\
  is unsatisfiable iff \\
  $c_1 + c_2 + \ldots c_{n-1} - d_n - 1 < 0$
  \end{center}

  \vfill
  which is Farka's Lemma if we set $c_n \equiv - d_n - 1$

\end{frame}


\section{Overview}\label{sec:boundedrefinementtypes:overview}

We start with a high level overview of bounded refinement types.
%
We first present a short introduction to refinement type specifications,
to make this chapter self contained.
%
Then, we introduce bounded refinements,
and show how they permit \emph{modular} higher-order specifications.
%
Finally, we describe how they are implemented via an elaboration
process that permits \emph{automatic} first-order verification.

\subsection{Preliminaries}

\mypara{Refinement Types} let us precisely specify subsets of values,
by conjoining base types with logical predicates that constrain the values.
We get decidability of type checking, by limiting these predicates to
decidable, quantifier-free, first-order logics, including the theory
of linear arithmetic, uninterpreted functions, arrays, bit-vectors
and so on.
%
For example, the refinement types
%
\begin{code}
  type Pos     = {v:Int | 0 < v}
  type IntGE x = {v:Int | x <= v}
\end{code}
%
specify subsets of @Int@ corresponding to values
that are positive or larger than some other value @x@
respectively.
%
We can use refinement types to specify contracts
like pre- and post-conditions by suitably refining the input
and output types of functions.

\mypara{Preconditions} are specified by refining input types.
We specify that the function @assert@ must
\emph{only} be called with @True@,
%
where the type @TRUE@ contains only the singleton @True@:
%
\begin{code}
  type TRUE = {v:Bool | v <=> True}

  assert         :: TRUE -> a -> a
  assert True x  = x
  assert False _ = error "Provably Dead Code"
\end{code}

\mypara{We can specify post-conditions} by refining output types.
For example, a primitive @Int@ comparison operator @leq@ can be
assigned a type that says that the output is @True@ iff the
first input is actually less than or equal to the second:
%
\begin{mcode}
  leq :: x:Int -> y:Int -> {v:Bool | v  <=> x <= y}
\end{mcode}

\mypara{Refinement Type Checking} proceeds by checking that at each
application, the types of the actual arguments are \emph{subtypes}
of those of the function inputs, in the environment (or context) in
which the call occurs.
%
Consider the function:
%
\begin{code}
  checkGE     :: a:Int -> b:IntGE a -> Int
  checkGE a b = assert cmp b
    where cmp = a `leq` b
\end{code}
%
To verify the call to @assert@ we check that
the actual parameter @cmp@ is a subtype of @TRUE@,
under the assumptions given by the input types for
@a@ and @b@.
%
Via subtyping~\cite{Vazou14} the check reduces to establishing
the validity of the \emph{verification condition}~(VC)
%
\begin{code}
  a <= b => (cmp <=> a <= b) => v = cmp => (v<=>true)
\end{code}
%
The first antecedent comes from the input type of @b@, the
second from the type of @cmp@ obtained from the output of @leq@,
the third from the \emph{actual} input passed to @assert@,
and the goal comes from the input type \emph{required} by @assert@.
%
An SMT solver \cite{Nelson81} readily establishes the validity
of the above VC, thereby verifying @checkGE@.

\begin{comment}

% \subsection{Abstract Refinements}

\mypara{First order refinements prevent modular specifications.}
Consider the function that returns the largest element of a list:
%
\begin{code}
  maximum         :: List Int -> Int
  maximum [x]     = x
  maximum (x:xs)  = max x (maximum xs)
    where max a b = if a  Pos
  posMax = maximum
\end{code}
%
%   posMax :: List Neg -> Neg
%   posMax = maximum
%
Any suitable specification would have to enumerate the
situations under which @maximum@ may be invoked
breaking modularity.

\mypara{Abstract Refinements} overcome the above modularity
problems \cite{vazou13}.
%
The main idea is that we can type @maximum@ by observing
that it returns \emph{one of} the elements in its input list.
Thus, if every element of the list enjoys some refinement @p@
then the output value is also guaranteed to satisfy @p@.
%
Concretely, we can type the function as:
%
\begin{code}
maximum :: forall
Bool>. List Int
 -> Int

\end{code}
%
where informally, @Int
@ stands for @{v:Int | p v}@,
and @p@ is an \emph{uninterpreted function} in the refinement
logic~\cite{Nelson81}.
%
The signature states that for any refinement @p@ on @Int@,
the input is a list of elements satisfying @p@
and returns as output an integer satisfying @p@.
%
In the sequel, we will drop the explicit quantification
of abstract refinements; all free abstract refinements
will be \emph{implicitly} quantified at the top-level
(as with classical type parameters.)

\paragraph{Abstract Refinements Preserve Decidability.}
Abstract refinements do not require the use of higher-order
logics. Instead, abstractly refined signatures (like @maximum@)
can be verified by viewing the abstract refinements @p@ as
uninterpreted functions that only satisfy the axioms of
congruence, namely:
%
\begin{code}
  forall x y. x = y => p x <=> p y
\end{code}
%
As the quantifier free theory of uninterpreted functions
is decidable~\cite{Nelson81}, abstract refinement type
checking remains decidable \cite{vazou13}.

\paragraph{Abstract Refinements are Automatically Instantiated} at call-sites,
via the abstract interpretation framework of Liquid Typing~\cite{vazou13}.
Each instantiation yields fresh refinement variables on
which subtyping constraints are generated; these constraints
are solved via abstract interpretation yielding the instantiations.
%
Hence, we verify @posMax@ % and @negMax@
by instantiating:
%
\begin{code}
  p |-> \ v -> 0 < v   -- at posMax
\end{code}
  % p |-> \ v -> v < 0   -- at negMax

\end{comment}

\subsection{Bounded Refinements}

Refinement types hit various expressiveness walls,
as for decidability, refinements are constraint to
first order, decidable logics.
%
Consider the following example
from~\cite{TerauchiPOPL13}.
%
@find@ takes as input a predicate @q@, a continuation
@k@ and a starting number @i@; it proceeds to compute
the smallest @Int@ (larger than @i@) that satisfies
@q@, and calls @k@ with that value.
%
@ex1@ passes @find@ a continuation that checks that the
``found'' value equals or exceeds @n@.
%
\begin{code}
  ex1 :: (Int -> Bool) -> Int -> ()
  ex1 q n = find q (checkGE n) n

  find q k i
    | q i       = k i
    | otherwise = find q k (i + 1)
\end{code}

\mypara{Verification fails} as there is no way to specify that
@k@ is only called with arguments greater than @n@.
%
First, the variable @n@ is not in scope at the function
definition so we cannot refer to it.
%
Second, we could try to say that @k@ is invoked with values
greater than or equal to @i@, which gets substituted with @n@
at the call-site. Alas, due to the currying order, @i@ too is
not in scope at the point where @k@'s type is defined so
the type for @k@ cannot depend upon @i@.

\mypara{Can Abstract Refinements Help?} Lets try to
use Abstract Refinements, from chapter~\ref{chapter:abstractrefinements},
to abstract over the refinement that @i@ enjoys, and
assign @find@ the type:
%
\begin{code}
  find :: (Int -> Bool) -> (Int
 -> a) -> Int
 -> a
\end{code}
%
which states that for any refinement @p@, the function takes
an input @i@ which satisfies @p@ and hence that the continuation
is also only invoked on a value which trivially enjoys @p@, namely @i@.
%
At the call-site in @ex1@ we can instantiate
\begin{equation}
\cc{p} \mapsto \lambda \cc{v} \rightarrow \cc{n} \leq \cc{v} \label{eq:inst:find}
\end{equation}
%
This instantiated refinement is satisfied by the parameter @n@ and is
sufficient to verify, via function subtyping, that @checkGE n@ will
only be called with values satisfying @p@, and hence larger than @n@.

\mypara{The function find is ill-typed} as the signature requires that
at the recursive call site, the value @i+1@ \emph{also}
satisfies the abstract refinement @p@.
%
While this holds for the example we have in mind~(\ref{eq:inst:find}),
it does not hold \emph{for all} @p@, as required by the type of @find@!
%
Concretely, @{v:Int | v = i + 1}@ is in general \emph{not} a subtype of
@Int
@, as the associated VC
% Concretely, the recursive call generates the VC
%
\begin{equation}
    ... \Rightarrow \cc{p i} \Rightarrow \cc{p (i+1)} \label{eq:vc:find}
\end{equation}
%
%\begin{equation}
%p\ i \Rightarrow p\ (i + 1) \label{eq:vc:find}
%\end{equation}
%
is \emph{invalid} -- the type checker thus (soundly!) rejects @find@.

\mypara{We must Bound the Quantification} of @p@ to limit
it to refinements satisfying some constraint, in this case
that @p@ is \emph{upward closed}. In the dependent setting,
where refinements may refer to program values, bounds
are naturally expressed as constraints between refinements.
% Horn clauses over refinements.
%
We define a bound, @UpClosed@
%
which states that @p@ is a refinement that is \emph{upward closed},
\ie satisfies @forall x. p x =>  p (x+1)@,
and use it to type @find@ as:
%
\begin{code}
  bound UpClosed (p :: Int -> Bool)
    = \x -> p x => p (x+1)

  find :: (UpClosed p) => (Int -> Bool)
                       -> (Int
 -> a)
                       ->  Int
 -> a
\end{code}
%
This time, the checker is able to use the bound to
verify the VC~(\ref{eq:vc:find}).
%
We do so in a way that refinements (and thus VCs) remain quantifier
free and hence, SMT decidable~(\S~\ref{sec:overview:implementation}).

\mypara{At the call} to @find@ in the body of @ex1@, we perform
the instantiation~(\ref{eq:inst:find}) which generates the
\emph{additional} VC
%
\hbox{@n <=  x => n <=  x+1@}
%
by plugging in the concrete refinements to the bound constraint.
%
The SMT checks the validity of the VC
and hence this instantiation, thereby statically
verifying @ex1@, \ie that the assertion inside
@checkGE@ cannot fail.
%

\subsection{Bounds for Higher-Order Functions}

Next, we show how bounds expand the scope of refinement typing by
letting us write precise modular specifications for various canonical
higher-order functions.

\subsubsection{Function Composition}\label{sec:compose}

First, consider @compose@. What is a modular specification
for @compose@ that would let us verify that @ex2@ enjoys
the given specification?
%
\begin{code}
  compose f g x = f (g x)

  type Plus x y = {v:Int | v = x + y}

  ex2    :: n:Int -> Plus n 2
  ex2    = incr `compose` incr

  incr   :: n:Int -> Plus n 1
  incr n = n + 1
\end{code}

\mypara{The challenge is to chain the dependencies} between the
input and output of @g@ and the input and output of @f@ to
obtain a relationship between the input and output of the
composition. We can capture the notion of chaining in a bound:
%
%% f -> p
%% g -> q
\begin{code}
  bound Chain p q r = \x y z ->
        q x y => p y z => r x z
\end{code}
%
which states that for any @x@, @y@ and @z@, if
%
(1) @x@ and @y@ are related by @q@, and
(2) @y@ and @z@ are related by @p@, then
(3) @x@ and @z@ are related by @r@.

We use @Chain@ to type @compose@ using three abstract
refinements @p@, @q@ and @r@, relating the arguments
and return values of @f@ and @g@ to their composed value.
%
(Here, @c<r x>@ abbreviates @{v:c | r x v}@).

\begin{code}
  compose :: (Chain p q r) => (y:b -> c
)
                           -> (x:a -> b<q x>)
                           -> (w:a -> c<r w>)
\end{code}

\mypara{To verify} @ex2@ we instantiate, at the call to @compose@,
%
\begin{code}
  p, q |-> \x v -> v = x + 1
     r |-> \x v -> v = x + 2
\end{code}
%
The above instantiation satisfies the bound, as shown by the validity
of the VC derived from instantiating @p@, @q@, and @r@ in @Chain@:
%
\begin{code}
  y = x + 1 => z = y + 1 => z == x + 2
\end{code}
%
and hence, we can check that @ex2@ implements its specified type.

\subsubsection{List Filtering}

Next, consider the list @filter@ function.
%
What type signature for @filter@ would let us check @positives@?
\begin{code}
  filter q (x:xs)
    | q x         = x : filter q xs
    | otherwise   = filter q xs
  filter _ []     = []

  positives       :: [Int] -> [Pos]
  positives       = filter isPos
    where isPos x = 0 < x
\end{code}
%
Such a signature would have to relate the @Bool@ returned by
@f@ with the property of the @x@ that it checks for.
%
Typed Racket's latent predicates~\cite{typedracket}
account for this idiom, but are a special construct
limited to @Bool@-valued ``type'' tests, and not
arbitrary invariants.
%
Another approach is to avoid the so-called
``Boolean Blindness'' that accompanies
@filter@ by instead using option types
and @mapMaybe@.

\mypara{We overcome blindness using a witness} bound:
%
\begin{code}
  bound Witness p w = \x b -> b => w x b => p x
\end{code}
%
which says that @w@ \emph{witnesses} the
refinement @p@. That is, for any boolean @b@ such
that @w x b@ holds, if @b@ is @True@ then @p x@ also holds.

\mypara{We can give} @filter@ a type that says that the output values
enjoy a refinement @p@ as long as the test predicate @q@ returns
a boolean witnessing @p@:
%
\begin{code}
  filter :: (Witness p w) => (x:a -> Bool<w x>)
                          -> List a
                          -> List a

\end{code}

\mypara{To verify} @positives@ we infer the following type and
instantiations for the abstract refinements @p@ and @w@ at the
call to @filter@:
%
\begin{code}
  isPos :: x:Int -> {v:Bool | v <=> 0 < x}
  p     |-> \v    -> 0 < v
  w     |-> \x b  -> b <=> 0 < x
\end{code}

\subsubsection{List Folding}

Next, consider the list fold-right function. Suppose we
wish to prove the following type for @ex3@:
%
\begin{code}
  foldr :: (a -> b -> b) -> b -> List a -> b
  foldr op b []     = b
  foldr op b (x:xs) = x `op` foldr op b xs

  ex3 :: xs:List a -> {v:Int | v == len xs}
  ex3 = foldr (\_ -> incr) 0
\end{code}
%
where @len@ is a \emph{logical} or \emph{measure}
function used to represent the number of elements of
the list in the refinement logic~\ref{sec:measures}:
%
\begin{code}
  measure len   :: List a -> Nat
    len []      = 0
    len (x:xs)  = 1 + len xs
\end{code}

\mypara{We specify induction as a bound.} Let
(1)~@inv@ be an abstract refinement relating a list @xs@
    and the result @b@ obtained by folding over it and
(2)~@step@ be an abstract refinement relating the
    inputs @x@, @b@ and output @b'@ passed to and
    obtained from the accumulator @op@ respectively.
%
We state that @inv@ is closed under @step@ as:
%
\begin{code}
  bound Inductive inv step = \x xs b b' ->
        inv xs b => step x b b' => inv (x:xs) b'
\end{code}
%

\mypara{We can give} @foldr@ a type that says that the
function \emph{outputs} a value that is built inductively
over the entire \emph{input} list:
%
\begin{code}
  foldr :: (Inductive inv step)
        => (x:a -> acc:b -> b<step x acc>)
        -> b<inv []>
        -> xs:List a
        -> b<inv xs>
\end{code}
%
That is, for any invariant @inv@ that is inductive
under @step@, if the initial value @b@ is @inv@-related
to the empty list, then the folded output is @inv@-related
to the input list @xs@.

\mypara{We verify} @ex3@ by inferring, at the call to @foldr@
%
\begin{code}
  inv  |-> \xs v   -> v  == len xs
  step |-> \x b b' -> b' == b + 1
\end{code}
%
The SMT solver validates the VC obtained by plugging the
above into the bound.
%
Instantiating the signature for @foldr@ yields precisely the
output type desired for @ex3@.

%% \nv{addition:}
Previously,~\ref{chapter:abstractrefinements} describes a way to type @foldr@
using abstract refinements that required the operator @op@
to have one extra ghost argument.
Bounds let us express induction without ghost arguments.

\subsection{Implementation}\label{sec:overview:implementation}

To implement bounded refinement typing, we must solve two
problems. Namely, how do we
%
(1)~\emph{check} and
%
(2)~\emph{use}
%
functions with bounded signatures?
%
We solve both problems via an insight inspired
by the way typeclasses are implemented in Haskell.
%
\begin{enumerate}
%
\item \emphbf{A Bound Specifies} a function type
whose inputs are unconstrained and whose output is
some value that carries the refinement corresponding
to the bound's body.
%
\item \emphbf{A Bound is Implemented} by a ghost
function that returns @True@, but is defined
in a context where the bound's constraint holds when
instantiated to the concrete refinements at the context.
%
\end{enumerate}

\mypara{We elaborate bounds into ghost functions} satisfying
the bound's type.
%
To \emph{check} bounded functions, we need to
\emph{call} the ghost function to materialize the
bound constraint at particular values of interest.
%
Dually, to \emph{use} bounded functions, we need to
\emph{create} ghost functions whose outputs are
guaranteed to satisfy the bound constraint.
%
This elaboration reduces \emph{bounded} refinement
typing to the simpler problem
of \emph{unbounded} abstract refinement typing.
%
The formalization of our elaboration is described in
\S~\ref{sec:check}.
%
Next, we illustrate the elaboration by explaining how
it addresses the problems of checking and using bounded
signatures like @compose@.

\mypara{We Translate Bounds into Function Types} called the
bound-type where the inputs are unconstrained, and the
outputs satisfy the bound's constraint.
%
For example, the bound @Chain@ used to type @compose@ in
\S~\ref{sec:compose}, corresponds to a function type, yielding
the translated type for @compose@:
%
\begin{code}
  type ChainTy p q r
    =  x:a -> y:b -> z:c -> {v:Bool | q x y => p y z => r x z}

  compose :: (ChainTy p q r)
          -> (y:b -> c
)
          -> (x:a -> b<q x>)
          -> (w:a -> c<r w>)
\end{code}

\mypara{To Check Bounded Functions} we view the bound constraints
as extra (ghost) function parameters (cf. type class dictionaries),
that satisfy the bound-type. Crucially, each expression where a
subtyping constraint would be generated (by plain refinement typing)
is wrapped with a ``call'' to the ghost to materialize the constraint
at values of interest. For example we elaborate @compose@ into:
%
\begin{code}
  compose dollarchain f g x =
    let t1 = g x
        t2 = f t1
        _  = dollarchain x t1 t2   -- materialize
    in  t2
\end{code}
%
In the elaborated version @dollarchain@ is the ghost parameter %
corresponding to the bound. As is standard \cite{LiquidPLDI08},
we perform ANF-conversion to name intermediate values, and then
wrap the function output with a call to the ghost to materialize
the bound's constraint. Consequently, the output of compose, namely
@t2@, is checked to be a subtype of the specified output type,
in an environment \emph{strengthened} with the bound's constraint
instantiated at @x@, @t1@ and @t2@. This subtyping reduces to a
quantifier free VC:
%
\begin{code}
      q x t1
  =>  p t1 t2
  => (q x t1 => p t1 t2 => r x t2)
  =>  v = t2 => r x v
\end{code}
%
whose first two antecedents are due to the types of @t1@ and @t2@
(via the output types of @g@ and @f@ respectively) and the third
comes from the call to @dollarchain@.
%
The output value @v@ has the singleton refinement that
states it equals to @t2@ and finally the VC states that the
output value @v@ must be related to the input @x@ via @r@.
%
An SMT solver validates this decidable VC easily, thereby
verifying @compose@.

Our elaboration inserts materialization calls \emph{for all}
variables (of the appropriate type) that are in scope at the
given point. This could introduce upto $n^k$ calls where $k$
is the number of parameters in the bound and $n$ the number
of variables in scope. In practice (\eg in @compose@) this
number is small (\eg 1) since we limit ourselves to variables
of the appropriate types.

%% Colin's comment: how about strictness and termination?
To preserve semantics we ensure that none of these materialization
calls can diverge, by carefully constraining the structure of
the arguments that instantiate the ghost functional parameters.

\mypara{At Uses of Bounded Functions} our elaboration uses
the bound-type to create lambdas with appropriate parameters
that just return @true@. For example, @ex2@ is elaborated to:
%
\begin{code}
  ex2 = compose (\_ _ _ -> true) incr incr
\end{code}
%
This elaboration seems too na\"ive to be true: how do we
ensure that the function actually satisfies the bound type?

Happily, that is automatically taken care of by function subtyping.
%
Recalling the translated type for @compose@, the elaborated lambda
@(\_ _ _ ->  true)@ is constrained to be a subtype of @ChainTy p q r@.
%
In particular, given the call site instantiation
%
\begin{mcode}
  p $\mapsto$ \ y z -> z == y + 1
  q $\mapsto$ \ x y -> y == x + 1
  r $\mapsto$ \ x z -> z == x + 2
\end{mcode}
%
this subtyping constraint reduces to the quantifier-free VC:
%
\begin{equation}
\inter{\Gamma}
  \Rightarrow \mathtt{true}
  \Rightarrow \cc{(z == y + 1)}
   \Rightarrow \cc{(y == x + 1)}\notag
   \Rightarrow \cc{(z == x + 2)}  \label{vc:bounded:ex2}
\end{equation}
%
%
where $\Gamma$ contains assumptions about the various binders in
scope.
%
The above VC is easily proved valid by an SMT solver, thereby
verifying the subtyping obligation defined by the bound, and hence,
that @ex2@ satisfies the given type.


% !TEX root = scombinatorics.tex
\documentclass[scombinatorics.tex]{subfiles}
\begin{document}
\chapter{Stability}
\label{sauer}

\def\medrel#1{\parbox[t]{5ex}{$\displaystyle\hfil #1$}}
\def\ceq#1#2#3{\parbox[t]{20ex}{$\displaystyle #1$}\medrel{#2}{$\displaystyle #3$}}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{The order property}\label{ladder}

The \emph{chain index\/} of $\phi(\U\,;b)_{b\in\V}$, or of $\phi(x\,;z)$ when $\U$ and $\V$ are clear, is the maximal length (that is, $n+1$) of a chain of the form

\ceq{\ssf{ch}\hfill\phi(A\,;b_0)\ \subset}{\dots}{\subset\ \phi(A\,;b_n)}

for some set $A\subseteq\U$ and some $b_0,\dots,b_n\in\V$.
Note that we allow $\phi(A\,;b_0)$ to be empty.
This choice produces a small asymmetry in the definition of ladder; see also Fact~\ref{fact_stability_dual}.

\begin{example}
  If $\phi(\U\,;b)_{b\in\V}$ consists of just one set, the ladder index is $1$.
  If it contains two distinct sets, the  chain index is at least $2$ and it is exactly $2$ if there are no more two sets, or if all sets are disjoint.\QED
\end{example}

If a maximal length does not exist, we say that $\phi(x\,;z)$ is \emph{unstable,} or that it has the \emph{order-property.}
Otherwise we say that it is \emph{stable.}

In place of requiring the existence of the chain in \ssf{ch}, we could equivalently ask for a pair of tuples $a_1,\dots,a_n\in\U$ and $b_0,\dots,b_n\in\V$ such that

\ceq{\ssf{ld}\hfill\phi(a_h\,;b_k)}
{\IFF}
{h\le k.}

We call this pair of tuples a \emph{ladder\/} of length $n+1$.
We may also say \emph{ladder index\/} instead of chain index.
Setting $A=\{a_1,\dots,a_n\}$ we easily obtain a chain from a ladder, the converse is left as an easy exercise for the reader.

\begin{exercise}
  Let $\phi(x\,;z)$ have chain index $n+1$ or more.
  Let $A\subseteq\U$ be a minimal set such that a chain as in \ssf{ch} obtains for some $b_0,\dots,b_n\in\V$.
  Prove that there is a ladder $a_1,\dots,a_n$ and $b_0,\dots,b_n$ such that $A=\{a_1,\dots,a_n\}$ and conclude that $\phi(x\,;z)$ has ladder index $n+1$.\QED
\end{exercise}

The following facts are obvious but worth noting.

\begin{fact}\label{fact_stability_dual}
  Let $\phi(x\,;z)$ have ladder index $n+1$.
  Then $\phi(x\,;z)^{\rm op}$ has ladder index $\ge n$.\QED
\end{fact}

\begin{fact}\label{fact_stability_neg}
  Let $\phi(x\,;z)$ have ladder index $n$.
  Then $\neg\phi(x\,;z)$ has ladder index $n$.\QED
\end{fact}

\begin{proof}
  If for all $h\in(n]$ and $k\in[n]$

  \ceq{\hfill\neg\phi(a_h\,;b_k)}
  {\IFF}
  {h\le k}

  then $a'_h=a_{n+1-h}$ and $b'_k=b_{n-k}$ satisfy

  \ceq{\hfill\phi(a'_h\,;b'_k)}
  {\IFF}
  {\phi(a_{n+1-h}\,;b_{n-h})}

  \ceq{}
  {\IFF}
  {n-k>n+1-h}

  \ceq{}
  {\IFF}
  {h\le k}
\end{proof}

The following definition is connected with those above, though in a less evident manner.
We write ${}^n2$ for the set of binary sequences of length $n$ or, more precisely, the set of functions $s:[n)\to[2)$.
We write $s_h$ for the value of $s$ at $h$, and $s{\restriction} h$ for the restriction of $s$ to $[h)$.
We define ${}^{<n}2=\big\{r\, :\, r\in {}^h2,\ h\in[n)\big\}$.

A \emph{branching tree\/} of hight $n$ for the formula $\phi(x\,;z)$ is a function

\ceq{\hfill\bar{a}\ :\ {}^{<n}2}{\to}{\U}
\nopagebreak[4]\par
\ceq{\hfill r}{\mapsto}{a_r},

which we may also present by writing $\bar a=\<a_r:r\in{}^{<n}2\>$, such that

\ceq{\ssf{2r}\hfill\0}
{\neq}
{\bigcap_{h=0}^n\neg^{s_h}\,\phi(a_{s\restriction h}\,;\V)}

where $\neg^i$, for $i$ a non negative integer, denotes a negation symbol repeated $i$ times.
In other words \ssf{2r} requires the existence of some $\<b_s:s\in{}^n2\>$ such that

\ceq{\hfill\phi(a_{s\restriction h}\,;b_s)}
{\IFF}
{s_h=1}\hfill for all pairs $s\in {}^n2$ and $h\in[n)$

or, with slightly different notation,

\ceq{\hfill\phi(a_r\,;b_s)}
{\IFF}
{r^\frown1\subseteq s}\hfill for all pairs $r\subset s\in {}^n2$.

It helps to represent a branching tree as follows.
For definiteness, fix $n=3$.
Consider a full binary tree of height $n+1$ and assign to each internal node (different from root and leaves) a formula as depicted below.
Then \ssf{2r} requires that all formulas in each branch $s\in2^n$ are satified by some $b_s\in\V$.
\medskip

% Set the overall layout of the tree
\tikzstyle{level 1}=[level distance=3.5cm, sibling distance=2.5cm]
\tikzstyle{level 2}=[level distance=3.5cm, sibling distance=1.2cm]
\tikzstyle{level 3}=[level distance=2.5cm, sibling distance=0.5cm]
\tikzstyle{level 4}=[level distance=0.5cm, sibling distance=0.5cm]

% Define styles for bags and leafs
\tikzstyle{bag0} = [text width=2.5ex, align=left]
\tikzstyle{bag} = [text width=7.5ex, align=right]
%\tikzstyle{end} = [circle, minimum width=3pt,fill, inner sep=0pt]

\begin{tikzpicture}[grow=right]
\node[bag0] {{$\top$}}
    child {
        node[bag] {$\phi(a_{\0};z)$}
            child {
                node[bag] {$\phi(a_1;z)$}
                    child {
                       node[bag] {\footnotesize$\phi(a_{11};z)$\rlap{ ----- $b_{111}$}}
                       edge from parent
                    }
                    child {
                       node[bag] {\footnotesize$\llap{$\neg$}\phi(a_{11};z)$\rlap{ ----- $b_{110}$}}
                       edge from parent
                    }
                 edge from parent
            }
            child {
                node[bag] {\llap{$\neg$}$\phi(a_1;z)$}
                edge from parent
                    child {
                       node[bag] {\footnotesize$\phi(a_{10};z)$\rlap{ ----- $b_{101}$}}
                       edge from parent
                    }
                    child {
                       node[bag] {\footnotesize\llap{$\neg$}$\phi(a_{10};z)$\rlap{ ----- $b_{100}$}}
                       edge from parent
                    }
                 edge from parent
            }
       edge from parent
    }
    child {
        node[bag] {\llap{$\neg$}$\phi(a_{\0};z)$}
            child {
                node[bag] {$\phi(a_0;z)$}
                    child {
                       node[bag] {\footnotesize$\phi(a_{01};z)$\rlap{ ----- $b_{011}$}}
                       edge from parent
                    }
                    child {
                       node[bag] {\footnotesize\llap{$\neg$}$\phi(a_{01};z)$\rlap{ ----- $b_{010}$}}
                       edge from parent
                    }
                 edge from parent
            }
            child {
                node[bag] {\llap{$\neg$}$\phi(a_0;z)$}
                edge from parent
                    child {
                       node[bag] {\footnotesize$\phi(a_{00};z)$\rlap{ ----- $b_{001}$}}
                       edge from parent
                    }
                    child {
                       node[bag] {\footnotesize\llap{$\neg$}$\phi(a_{00};z)$\rlap{ ----- $b_{000}$}}
                       edge from parent
                    }
                 edge from parent
            }
        edge from parent
    };
\end{tikzpicture}
\smallskip

The Shelah local \emph{2-rank\/} of $\phi(x\,;z)$ is the maximal height of a branching tree for $\phi(x\,;z)$.
If such a maximal integer does not exist, we say that the 2-rank is infinite.

A branching tree $\bar a'=\<a'_r:r\in{}^m2\>$ is a \emph{subtree\/} of $\bar a$ if there is an $\subseteq$-preserving map $f:{}^m2\to{}^n2$ such that $a'_r=a_{fr}$.

For the theorem below we need the following Ramsey-like lemma.

% \begin{lemma}\label{lem_Ramsey_on_trees}
%   Let $\bar a=\<a_r\, :\, r\in{}^{<2m}2\>$ be a branching tree for $\phi(x\,;z)$.
%   Let $k\in[2m]$ be given.
%   Then, for every $2$-coloring of $\range(\bar a)$, there is a branching tree $\bar a'=\<a'_r\, :\, r\in{}^m2\>$ such that  $\range(\bar a')$ is a monochromatic subset of $\range(\bar a)$.\QED
% \end{lemma}

% As it happens, a more general lemma is easier to prove.

\begin{lemma}\label{lem_Reamsey_tree}
  Let $\bar a=\<a_r\, :\, r\in{}^{<n}2\>$ be a branching tree for $\phi(x\,;z)$.
  Let $k\in[n]$ be given.
  Then, for every red-blue coloring of $\range(\bar a)$,  there is a monochromatic branching subtree, say $\bar a'=\<a'_r\, :\, r\in{}^{n'}2\>$, and either all nodes of $\bar a'$ are red and $n'=k$, or they are all blue and $n'=n-k$.
\end{lemma}

\begin{proof}
  Induction on $n$.
  First, note that the lemma is trivial when $k=0$ or $k=n$.
  In particular the lemma holds for $n=0$.

  Assume the lemma true for $n$ and prove it for $n+1$.
  Let  $\bar a=\<a_r\, :\, r\in{}^{n+1}2\>$ be given.
  Fix some $k\in[n+1]$.
  We want a branching tree $\bar a'$ that is either red of height $k$ or blue of height $n+1-k$.

  If we discard the trivial cases, we can assume $k\in(n]$.

  For $i=0,1$ we define $\bar a_i=\<a_{i^\frown r}\;:\;r\in {}^{<n}2\>$.

  First suppose that $a_\0$ is blue.
  If, for either $i=0$ or $i=1$, there is a red branching tree $\bar a'_i$ of height $k$, we are done.
  Otherwise, for both $i=0,1$ there is a blue branching tree $\bar a'_i$ of hight $n-k$.
  Then we graft these two trees on $a_\0$, which is also blue, and obtain the required blue tree of height $n-k+1$.

  Now suppose that $a_\0$ is red.
  If, for either $i=0$ or $i=1$, there is a blue branching tree $\bar a'_i\subseteq \bar a_i$ of height $n-(k-1)$, we are done.
  Otherwise, we graft on $a_\0$ two red trees of hight $k-1$ to obtain a red tree of height $k$.
\end{proof}

We are now ready to characterize stability via the 2-rank.
The proof is based on Hodges~\cite{hodges}.
It is a direct proof which yields an explicit bound on the 2-rank given the ladder index (it yields also the converse, but this is easy).
However, this bound seems to be far from optimal.
Shelah's original proof is model theoretic and does not gives explicit bounds.
In the next section we will introduce some of the ideas used in Shelah's proof.
% In the next section we sketch a second proof which elaborates on Shelah's original ideas.
% This proof is indirect but.

\begin{theorem}\label{thm_hodges}
  The following are equivalent
  \begin{itemize}
    \item[1.] $\phi(x\,;z)$ is stable;
    \item[2.] $\phi(x\,;z)$ has finite 2-rank.
  \end{itemize}
  Precisely, if $n_{\rm ld}$ and $n_{\rm 2r}$ are the ladder index and the 2-rank, respectively, then

  \ceq{\hfill n_{\rm ld}}{<}{2^{n_{\rm 2r}+1};}

  \ceq{\hfill n_{\rm 2r}}{<}{2^{n_{\rm ld}}+2.}
\end{theorem}

\begin{proof}[Proof (\kern.1ex\ssf{2}\kern.3ex\boldmath$\IMP$\ssf{1})]
  % \def\ceq#1#2#3{\parbox[t]{29ex}{$\displaystyle #1$}\medrel{#2}{$\displaystyle #3$}}
  % \ssf{1}$\IMP$\ssf{2}
  % We prove the contrapositive.
  % If there is a binary tree of hight $n$ then there is a ladder of length $n$.
  % This also proves the first inequality of the proposition.

  % Let $\{a_s\; :\; s\in2^{<n}\}$ and $\{b_s\; :\; s\in2^{n}\}$ satisfy \ssf{2r}.
  % Apply \ssf{2r} to sequences of the form $1^k\,0^{n-k}$ for $0\le k\le n$.

  % % Define $a'_1,\dots,a'_n$ and $b'_0,\dots,b'_n$, where $a'_h=a_{1^{h-1}}$ and $b'_k=b_{1^k0^{n-k}}$.
  % % By \ssf{2r} we have

  % \ceq{\hfill\phi(a_{1^k\,0^{n-k}\,\restriction\, h-1}\;;\,b_{1^k\,0^{n-k}})}
  % {\IFF}
  % {\big(1^k0^{n-k}\big)_{h-1}=1}

  % \ceq{}
  % {\IFF}
  % {h\le k}

  We prove the contrapositive.
  We show that if there is a ladder of length $m=2^n$, say $a_1,\dots,a_{m-1}$ and $b_0,\dots,b_{m-1}$, then there is a branching tree $\bar a'$ of height $n$.
  This also proves the first inequality above.
  In fact, if $n_{\rm ld}\ge 2^{n_{\rm 2r}+1}$, there would exist a branching tree of height $n_{\rm 2r}+1$ which is a contradiction.

  The branching tree $\bar a'=\<a'_r\, :\, r\in{}^{<n}2\>$ is defined as follows

  \quad $a'_r=a_h$\quad  where $h$ is obtained reading $r^\frown1^\frown0^{n-|r|-1}$ as an $n$-digit binary number.

  To verify \ssf{2r} we define for $s\in{}^n2$

  \quad $b'_s=b_k$\quad  where $k$ is obtained reading $s$ as an $n$-digit binary number.

  Then it is easy to verify that for all pairs $r\subset s\in{}^n2$

  \ceq{\hfill\phi(a'_r\,;b'_s)}
  {\IFF}
  {\phi(a_h\,;b_k)}\hfill where $h$ and $k$ are like above

  \ceq{}
  {\IFF}
  {h\le k}

  \ceq{}
  {\IFF}
  {r^\frown1^\frown0^{n-|r|-1}\ \le\ s}\hfill  as $n$-digit binary numbers

  \ceq{}
  {\IFF}
  {r^\frown1\ \subseteq\ s}

  \textbf{(\ssf{1}\kern.2ex\boldmath$\IMP$\ssf{2}\kern.1ex)}\
  We prove the contrapositive.
  We claim that if there is a branching tree of height $2^n+2$ then there is a ladder of length $n+1$.
  This yields also the second inequality of the theorem.
  In fact, if $n_{\rm 2r}\ge 2^{n_{\rm ld}}+2$, there would exist a ladder of length $n_{\rm ld}+1$ which is a contradiction.

  The claim is true for $n=0$, as witnesses by the ladder $a_\0$.
  Now we assume the claim is true for $n$ and prove it for $n+1$.

  Let $\<a_r\, :\, r\in{}^{<\, 2m+2}2\>$, where $m=2^n$, be a branching tree of height $2^{n+1}+2$.
  To each $b\in\V$ we associate a red-blue coloring of $\range(\bar a)$ as follows.
  A node $a\in\range(\bar a)$ is colored

  \qquad\qquad \llap{red}\qquad if $\phi(a\,;b)$ holds;

  \qquad\qquad \llap{blue}\qquad otherwise, that is, $\neg\phi(a\,;b)$.

  We consider two cases that are exhaustive by Lemma~\ref{lem_Reamsey_tree}.
  Note that we are applying the lemma only to the subtree $\<a_{1^\frown r}:r\in{}^{2m+1}2\>$.

  Case 1: for some $b$ there is a red subtree of $\<a_{1^\frown r}:r\in{}^{2m+1}2\>$ of height $m+1$.
  Let $\bar a'$ be this red tree and consider its subtree $\<a'_{0^\frown r}\,:\,r\in {}^{<\,m}2\>$.
  By induction hypothesis, there are $A\subseteq\{a'_{0^\frown r}\,:\,r\in {}^{<\,m}2\}$ and $b_0,\dots,b_n$ such that

  \ceq{(1)\hfill\phi(A\,;b_0)\ \subset}{\dots}{\subset\ \phi(A\,;b_n)}

  Let $A'=A\cup\{a'_\0\}$ then

  \ceq{(2)\hfill\phi(A'\,;b_0)\ \subset}{\dots}{\subset\ \phi(A'\,;b_n)}

  In fact, as $b_0,\dots,b_n\in\neg\phi(a'_\0\,;\V)$, this is the same chain as (1).
  Therefore, if we extend the chain on the right with $\phi(A'\,;b)=A'$, we obtain the required chain of length $n+2$.

  Case 2: for every $b$ there is a blue subtree of $\<a_{1^\frown r}:r\in{}^{2m+1}2\>$ of height $m$.
  Choose $b$ such that $\neg\phi(a_\0, b)$ and let $\bar a'$ the corresponding blue subtree.
  Apply the induction hypothesis to obtain $A\subseteq\range(\bar a')$ and $b_0,\dots,b_n$ such that (1).
  We claim that (2) above holds with $A'=A\cup\{a_\0\}$.
  In fact, $b_0,\dots,b_n\in\phi(a_\0\,;\V)$ so (2) is the chain in (1) with all sets augmented by $a_\0$.
  We can extend the chain on the left with $\phi(A'\,;b)=\0$ and obtain the required chain of length $n+2$.
\end{proof}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Approximable sets}

The notion of approximable set is a combinatorial counterpart of the model theoretical notion of externally definable set.

\begin{definition}\label{def_approx}
  We say that $\Aa\subseteq\U$ is \emph{approximable\/} if for every finite set $A\subseteq\U$ there is a $b\in\V$ such that $\phi(A\,;b)=\Aa\cap A$.
  If we also have that $\phi(\U\,;b)\subseteq\U$, then we say that $\Aa$ is approximable \emph{from below.}
\end{definition}

The following is immediate.

\begin{fact}
  The following are equivalent for every $\Aa\subseteq\U$
  \begin{itemize}
    \item[1.] $\Aa$ is approximable from below;
    \item[2.] for every finite set $A\subseteq\Aa$ there is a $b\in\V$ such that $A\subseteq\phi(\U\,;b)\subseteq\Aa$.\QED
  \end{itemize}
\end{fact}

\begin{proposition}
  Let $\phi(x\,;z)$ be a stable formula.
  Every set $\Aa\subseteq\U$ approximable by $\phi(x\,;z)$ is approximable from below by the formula

  \ceq{\hfill \psi(x\,;z_0,\dots,z_n)}{=}{\bigwedge_{i=0}^n\phi(x\,;z_i)}

  where $n+1$ is the ladder index of $\phi(x\,;z)$.
\end{proposition}

\begin{proof}
  Let $A\subseteq\Aa$ be finite.
  We prove that $A\subseteq\psi(\U\,;b_0,\dots,b_n)\subseteq\Aa$ for some $b_0,\dots,b_n$.
  To obtain the $b_k$, we construct a ladder inductively.
  Pick any $b_0$ be such that $A\subseteq\phi(\U\,;b_0)$, which we can do by approximability.
  Now, suppose that $a_1,\dots,a_k$ and $b_0,\dots,b_k$ have been defined.
  If

  \ceq{\hfill A}{\subseteq}{\bigcap_{i=0}^k\phi(\U\,;b_i)\medrel{\subseteq}\Aa}

  we set $b_{k+1}=\dots=b_n=b_k$ and stop as we already have the required parameters.
  Otherwise pick any element

  \ceq{\hfill a_{k+1}}{\in}{\bigcap_{i=0}^k\phi(\U\,;b_i)\sm\Aa}

  and some $b_{k+1}$ such that

  \ceq{\hfill A}{\subseteq}{\phi(\U\,;b_{k+1})\medrel{\subseteq}\U\sm\{a_1,\dots,a_{k+1}\}}.

  Such a parameter $b_{k+1}$ exists because $\Aa$ is approximable (apply Definition~\ref{def_approx} with $A\cup\{a_1,\dots,a_{k+1}\}$ for $A$).
  Note that at stage $n$ we have constructed a ladder of length $n$ for $\neg\phi(x\,;z)$.
  In fact for all $h\in(n]$ and $k\in[n]$ we have

  \ceq{\hfill\neg\phi(a_h\,;b_k)}
  {\IFF}
  {h\le k.}

  As $\phi(x\,;z)$ has ladder index $n+1$, by Fact~\ref{fact_stability_neg} it is not possible to further prolong this chain, hence

  \ceq{\hfill A}{\subseteq}{\bigcap_{i=0}^n\phi(\U\,;b_i)\medrel{\subseteq}\Aa}

  as required.
\end{proof}

We prove that the formula $\psi(x\,;z_0,\dots,z_n)$ in the lemma above is itself stable, though with a larger ladder index.

\begin{lemma}
  Let $\psi_i(x\,;z)$, where $i=1,\dots,m$, be formulas with ladder index $n_i$. Let

  \ceq{\hfill \phi(x\,;z)}{=}{\bigwedge_{i=1}^m\phi_i(x\,;z)}

  Then $\phi(x\,;z)$ has ladder index $<R(n_1,\dots, n_m)$, the Ramsey number for $m$-colorings.
\end{lemma}

\begin{proof}
  Suppose for a contradiction that there is a ladder $a_1,\dots,a_n$ and $b_0,\dots,b_n$ of length $n=R(n_1,\dots, n_m)$.
  Let $C_i$ contains the pairs $\{h,k\}$ such that $0\le k<h\le n$ and $\neg\phi_i(a_h\,;b_k)$.
  Then from \ssf{ld} we obtain

  \ceq{\hfill\bigcup_{i=1}^mC_i}{=}{\binom{[n]}{2}}

  By the definition of $n$, for some $i\in[m]$, there is a set $H$ of cardinality $n_i$ such that $H^{(2)}\subseteq C_i$.
  Assume $i{=}1$ for definiteness.

  Write $a'_1,\dots,a'_{n_1}$ and $b'_0,\dots,b'_{n_1}$ for the tuples obtained by restricting $a_1,\dots,a_n$ and $b_0,\dots,b_n$ to the indexes in $H$.
  These tuples witness that $\phi_1(x\,;z)$ has ladder index at least $n_1+1$, which contradicts the assumption of the lemma.
\end{proof}

To apply the proposition to the formula $\psi(x\,;z_0,\dots,z_n)$ in the lemma above, let $z=z_0,\dots,z_n$ and let $\phi_i(x\,;z)=\phi(x\,;z_i)$.
That is, $\phi_i(x\,;z)$ defines a relation between $\U$ and $\V^{n+1}$ that depends only on the $i$-th coordinate.
The ladder index of every $\phi_i(x\,;z)$ is the same as $\phi(x\,;z_0)$ as a relation between $\U$ and $\V$.

\begin{lemma}
  If $\Aa$ is approximated from below by a stable formula $\phi(x\,;z)$ then for some $b_0,\dots,b_n\in\V$
  \\[-1ex]
  \ceq{\hfill \Aa}{=}{\bigcup^n_{i=0}\phi(\U\,;b_i)}

  where $n+1$ is the ladder index of $\phi(x\,;z)$.
\end{lemma}

\begin{proof}
  Let $A\subseteq\U$ be finite.
  To obtain the $b_k$, we construct a ladder by stages.
  Pick any $b_0$ be such that $\phi(\U\,;b_0)\subseteq\Aa$.
  Now, suppose that $a_1,\dots,a_k$ and $b_0,\dots,b_k$ have been defined.
  If
  \\[-1ex]
  \ceq{\hfill\Aa}{\subseteq}{\bigcup_{i=1}^k\phi(\U\,;b_i)}

  we set $b_{k+1}=\dots=b_n=b_k$ and stop.
  As the construction guarantees the converse inclusion, we have the required parameters.
  Otherwise pick any element

  \ceq{\hfill a_{k+1}}{\in}{\Aa\sm\bigcup_{i=1}^k\phi(\U\,;b_i)}

  and some $b_{k+1}$ such that

  \ceq{\hfill \{a_1,\dots,a_{k+1}\}}{\subseteq}{\phi(\U\,;b_{k+1})\medrel{\subseteq}\Aa}.

  Such a parameter $b_{k+1}$ exists because $\Aa$ is approximable from below.
  Note that at stage $n$ we have constructed a ladder of length $n$ for $\phi(x\,;z)$.
  In fact for all $h\in(n]$ and $k\in[n]$ we have

  \ceq{\hfill\neg\phi(a_h\,;b_k)}
  {\IFF}
  {h\le k.}

  As $\phi(x\,;z)$ has ladder index $n+1$, it is not possible to prolonge this chain, hence

  \ceq{\hfill\Aa}{=}{\bigcup_{i=0}^n\phi(\U\,;b_i)}

  as required.
\end{proof}

The main theorem of this section is an immediate corollary is the three lemmas above.

\begin{theorem}\label{thm_stable_definability}
  Let $\phi(x\,;z)$ be stable.
  Then there is a formula of the form

  \ceq{\hfill\psi(x\,;\bar z)}{=}{\bigvee_{j=0}^m\bigwedge_{i=0}^n\phi(z\,;z_{i,j})}

  such that every set $\Aa\subseteq\U$ approximable by  $\phi(x\,;z)$ is definable by $\psi(x\,;\bar b)$ for some $n{\times}m$-tuple of parameters $\bar b\in\V^{n\times m}$.\QED
\end{theorem}

We conclude this section with the sketch of an argument that shows that stable formulas have finite 2-rank, that is, \ssf{1}$\IMP$\ssf{2} of Theorem~\ref{thm_hodges}.

% Assume the 2-rank is infinite and reason for a contradiction.
% Assume for the moment that the rank is $\omega$, in the sense that there is an infinite branching tree $\<a_r:r\in2^{<\omega}\>$.
% Let $\<b_s:s\in2^{\omega}\>$ be witnesses of \ssf{2r}.
% Let $Q\subseteq\{b_s:s\in2^{\omega}\}$ the set of sequences that are eventually $0$.
% Note that in the proof of Theorem~\ref{thm_stable_definability} we can require that $\bar b\in Q^{n\times m}$.
% But there are

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%\section{Stable Erd\H{o}s-Hajnal}

\end{document}


\section{Green's Theorems}
\noindent
Similar to how we had the fundamental theorem of calculus (FTC) for single-variable calculus, there are several higher-dimensional versions of the FTC.\\
Recall that the fundamental theorem of calculus is $\int_{a}^{b}{f^\prime(x)\mathrm{d}x} = f(b) - f(a)$. One way to think of this statement is “summing up the derivative on a closed interval is the same as the ‘sum’ of $f$ on the boundary.” We will see that many of these higher versions of the FTC deal with intervals and boundaries.

\input{./vectorAnalysis/greensTheoremCirculation}
\input{./vectorAnalysis/greensTheoremFlux}


\input{../../style/preamble}
\input{../../latex-math/basic-math}
\input{../../latex-math/basic-ml}
\input{../../latex-math/ml-ensembles.tex}
\input{../../latex-math/ml-trees.tex}

\newcommand{\titlefigure}{figure/boosting_classif_title.png}
\newcommand{\learninggoals}{
  \item Transfering gradient boosting for regression to binary classification problems
  \item Introducing gradient boosting for multiclass problems
}

\title{Introduction to Machine Learning}
\date{}

\begin{document}

\lecturechapter{Gradient Boosting for Classification}
\lecture{Introduction to Machine Learning}

\begin{vbframe}{Binary classification}

For $\Yspace = \{0, 1\}$, we simply have to select an appropriate loss function, so let us
use Bernoulli loss as in logistic regression:

$$ \Lxy = - y \cdot \fx + \log(1 + \exp(\fx)).$$

Then,

\vspace{-0.5cm}

\begin{align*}
\tilde{r}(f) &=-\pd{\Lxy}{\fx} \\
&= y - \frac{\exp(\fx)}{1 + \exp(\fx)} \\
&= y - \frac{1}{1 + \exp(-\fx)} = y - s(\fx).
\end{align*}

Here, $s(\fx)$ is the logistic function, applied to a scoring model.
Hence, effectively, the pseudo-residuals are $y - \pix$.

Through $\pix = s(\fx)$ we can also estimate posterior probabilities.

\framebreak
%

\begin{itemize}
\item Using this loss function, we can simply run GB as for regression.
  NB: Also here, we fit regression base learners against our numerical
  vector of pseudo-residuals with $L2$ loss.
%\item The tree extension for boosting for the binary case works exactly as in regression.
\item  We could also have used the exponential loss for classification with
  GB. It can be shown that the resulting GB algorithm is basically equivalent
    to AdaBoost. In practice there is no big difference, although Bernoulli loss
    makes a bit more sense from a theoretical (maximum likelihood) perspective.
\item It follows that GB is a generalization of AdaBoost which can also use other loss functions and be used for different ML scenarios.
\end{itemize}

\end{vbframe}

\begin{vbframe}{Example}

We now illustrate the boosting iterations for a classification example in a similar manner as we did for regression.
However, we will now look at a simulation example with 2 instead of 1 influencial features and one binary target variable.

\begin{columns}
\column{5.5cm}
\begin{itemize}
\item We used the \texttt{mlbench} dataset \texttt{circle} with $n = 50$ observations.
\item We used the Bernoulli loss to calculate pseudo-residuals and fitted in each iteration a base learner (regression tree with max. depth of 3) on them.
\item We initialized with $f^{[0]} = log(1) = 0.$
\end{itemize}
\column{4.5cm}
\begin{center}
\includegraphics[width=\textwidth]{figure_man/boosting_classif_example.png}
\end{center}

\end{columns}

\end{vbframe}

\begin{frame}{Example}
Left: Background color refers to prediction probabilites $\hat{\pi}^{[m]}$ and points to $y$ (probabilities);
Right: Background color refers to predictions of base learner $\hat{b}^{[m]}$ and points to $\tilde{r}$.

\vfill

\only<1>{
\includegraphics[width=\textwidth]{figure_man/boosting_classif_1.png}

\vfill
Iteration 1
}
\only<2>{
\includegraphics[width=\textwidth]{figure_man/boosting_classif_2.png}

\vfill
Iteration 2
}

\only<3>{
\includegraphics[width=\textwidth]{figure_man/boosting_classif_5.png}

\vfill
Iteration 5
}
\only<4>{
\includegraphics[width=\textwidth]{figure_man/boosting_classif_10.png}

\vfill
Iteration 10
}

\only<5>{
\includegraphics[width=\textwidth]{figure_man/boosting_classif_100.png}

\vfill
Iteration 100
}

\end{frame}

%\section{Interactive Playgrounds}

% \begin{vbframe}{Gradient Boosting Playground}
% \begin{center}
%
% \includegraphics[width=0.7\textwidth]{figure_man/gbm_playground.png}
%
% \href{http://arogozhnikov.github.io/2016/07/05/gradient_boosting_playground.html}{\beamergotobutton{Open in browser.}}
%
% \end{center}
% \end{vbframe}

\begin{vbframe}{mlrPlayground}
\begin{center}

\includegraphics[width=\linewidth]{figure_man/mlrplayground_welcome.png}

\href{https://compstat-lmu.shinyapps.io/mlrPlayground/}{\beamergotobutton{Open in browser.}}

\end{center}
\end{vbframe}

% \section{Gradient Boosting for Multiclass Problems}

\begin{vbframe}{Multiclass problems}

We proceed as in softmax regression and model a categorical distribution with multinomial / log loss.
For $\Yspace = \{1, \ldots, g\}$, we create $g$ discriminant functions $\fkx$, one for each class and each one being an \textbf{additive} model of base learners.

We define the $\pi_k(\xv)$ through the softmax function:
$$ \pikx = s_k(f_1(\xv), \ldots, f_g(\xv)) = \exp(\fkx) / \sum_{j=1}^g \exp(f_j(\xv)). $$

Multinomial loss $L$:
$$ L(y, f_1(\xv), \ldots f_g(\xv)) = - \sumkg \mathds{1}_{\{y = k\}} \ln \pikx. $$

Pseudo-residuals:
$$-\pd{L(y, f_1(\xv), \ldots, f_g(\xv))}{\fkx} =  \mathds{1}_{\{y = k\}} - \pikx. $$

\framebreak

\begin{algorithm}[H]
  \begin{footnotesize}
  \begin{center}
  \caption{GB for Multiclass}
    \begin{algorithmic}[1]
      \State Initialize $f_{k}^{[0]}(\xv) = 0,\ k = 1,\ldots,g$
      \For{$m = 1 \to M$}
      \State Set $\pik^{[m]}(\xv) = \frac{\exp(f_k^{[m]}(\xv))}{\sum_j \exp(f_j^{[m]}(\xv))}, k = 1,\ldots,g$
            \For{$k = 1 \to g$}
            \State For all $i$: Compute $\rmi_k = \mathds{1}_{\{\yi = k\}} - \pik^{[m]}(\xi)$
              \State Fit a regression base learner $\hat{b}^{[m]}_k$ to the pseudo-residuals $\rmi_k$.
              \State Obtain $\betamh_k$ by constant learning rate or line-search.
              \State Update $\hat{f}_k^{[m]} = \hat{f}_k^{[m-1]} + \betamh_k \hat{b}^{[m]}_k$
            \EndFor
      \EndFor
    \State Output $\hat{f}_1^{[M]}, \ldots, \hat{f}_g^{[M]}$
    \end{algorithmic}
    \end{center}
    \end{footnotesize}
\end{algorithm}

\end{vbframe}

\endlecture
\end{document}


\begin{wrapfigure}{r}{0.7\textwidth}
    %\centering
    \includegraphics[width = 0.7\textwidth]{Chapters/BackgroundKnowledgeAndRelatedWork/MultiAgentTargetDetectionBackground/Figs/DBNs/Complex2TDBN.PNG}
    \caption{A DBN used for \textbf{S}imultaneous \textbf{L}ocalisation \textbf{a}nd \textbf{M}apping (SLAM), based on \cite[p.~311]{Thrun:2005:ProbabilisticRobotics}.}
    \label{fig:2TDBNExample}
\end{wrapfigure}

DBNs are a generalization of HMMs and are used to model time series. The key difference between DBNs and HMMs is that DBNs are not limited in how they decompose the hidden state variables of a complex stochastic system into the variables that represent its constituent conditional distributions \cite{AIAMA}. Rather than use a single random variable, $X_t$ to represent the state space, a set of random variables are used \cite{Murphy1994DynamicLearning}. DBNs can be considered as a special case of a Bayesian Network with an infinite number of nodes and repeating structure which specifies the transition probabilities between one time step and the next \cite[p.~204]{KollerPGM}. The hidden state variables are represented by nodes which have an associated conditional probability distribution (CPD) which states the probability of observing a value of a random variable given its parents: $p(X_i | Parents(X_i))$, where $i$ is the index of a node in the network \cite[p.~14]{Murphy1994DynamicLearning}. Given this representation, it is possible to calculate the joint distribution, $p(X_t | X_{t-1})$, of all $N$ state variables at time $t$ given the joint distribution at $t-1$ as \cite[p.~14]{Murphy1994DynamicLearning}:
\[
p(X_t | X_{t-1}) = \prod_{i=1}^{N}p(X_t^i | Parents(X_t^i)
\]
$i$ denotes the index of each $N$ variables in the network. \citeauthor{KollerPGM} provide a technical definition of DBNs:
"\textit{A dynamic Bayesian network (DBN) is a pair $\langle B_0, B_\rightarrow \rangle$, where $B_0$ is a Bayesian network over $X^{(0)}$, representing the initial distribution over states, and $B_\rightarrow$ is a 2-TBN for the process. For any desired time span T $\geq$ 0, the distribution over $X^{(0:T)}$ is defined as a unrolled Bayesian network, where, for any $i$ = 1, . . . , $n$:
\begin{itemize}
    \item The structure and CPDs of $X^{(0)}_i$ are the same as those for $X_i$ in $B_0$,
    \item The structure and CPD of $X^{(t)}_i$ for $t > 0$ are the same as those for $X^{'}_i$ in $B_\rightarrow$.
\end{itemize}}" \cite[p.~204]{KollerPGM}. What this essentially means is that a DBN is a compact way of specifying an infinite set of Bayesian Networks (one for each $T>0$) with repeating structure.

Technically, every DBN can be represented as a HMM and visa versa, however, the number of parameters that need to be determined to represent DBNs can be significantly less than that of HMMs \cite{KollerPGM}. For example, consider the case of $n$ discrete random variables, each with support of size $m$. Specifying the full joint distribution without assuming any conditional independences between variables would require a number of probabilities exponential in the number of random variables, $m^{n}-1$, whereas specifying the same joint distribution as factored conditional distributions may require far fewer, reaching a number of probabilities directly proportional to the number of random variables, $n \times m$ in the degenerate case \cite[p.~63]{KollerPGM}.
%\subsubsection{A Note on Bayesian Networks}
%A Bayesian Network (BN) is a graphical way to represent a particular factorization of a joint distribution. For a detailed explanation, the reader is referred to \cite{KollerPGM}. To fully explain a complex system, it is often natural to model it as the joint distribution of a number of random variables. This is in general intractable \cite{KollerPGM}. In order to circumvent this, independence properties in the distribution can be exploited to provide a much more compact representation of the distribution. BNs exploit the fact that independence is a strong notion that doesn't often occur in the real-world; however conditional independence is a weaker property that is far more prevalent, which still leads to the desired compact representation. BNs are described by a directed acyclic graph (DAG), $G$. The nodes in the graph correspond to the random variables whose joint distribution is of interest, and the arcs represent conditional independences. Specifically, if one Burglary points to Alarm as in figure \ref{fig:BayesianNetwork}, it is implied that the distribution of Alarm is conditionally independent on all other nodes in the network given Burglary. This means that only local conditional probability distributions must be provided in order to specify the full joint distribution.

%\begin{figure}
%example bayesian network figure
%\begin{tikzpicture}[
%  node distance=1cm and 0cm,
%  mynode/.style={draw,ellipse,text width=2cm,align=center}
%]
%\node[mynode] (i) {Burglary};
%\node[mynode,below right=of i] (g) {Alarm};
%\node[mynode,above right=of g] (c) {Earthquake};
%\path %(ra) edge[latex-] (sp)
%(g) edge[latex-] (c)
%(g) edge[latex-] (i);
%\node[left=0.45cm of i]
%{
%\begin{tabular}{cM{3}M{3}}
%\toprule
%\multicolumn{2}{c}{Burglary} \\
%\multicolumn{1}{c}{T} & \multicolumn{1}{c}{F} \\
%\cmidrule(r){1-2}
%0.001 & 0.999 \\
%\bottomrule
%\end{tabular}
%};
%\node[right=0.45cm of c]
%{
%\begin{tabular}{cM{3}M{3}}
%\toprule
%\multicolumn{2}{c}{Earthquake} \\
%\multicolumn{1}{c}{T} & \multicolumn{1}{c}{F} \\
%\cmidrule(r){1-2}
%0.008 & 0.992 \\
%\bottomrule
%\end{tabular}
%};
%\node[below=0.5cm of g]
%{
%\begin{tabular}{ccM{2}M{2}}
%\toprule
%& & \multicolumn{2}{c}{Alarm} \\
%\multicolumn{2}{l}{Burglary Earthquake} & %\multicolumn{1}{c}{T} & \multicolumn{1}{c}{F} \\
%\cmidrule(r){1-2}\cmidrule(l){3-4}
%F & F & 0.01 & 0.99 \\
%F & T & 0.95 & 0.05 \\
%T & F & 0.8 & 0.2 \\
%T & T & 0.99 & 0.01 \\
%\bottomrule
%\end{tabular}
%};
%\end{tikzpicture}

%\caption{Simple Bayesian Network based on %\cite[P.~512]{AIAMA}}
%\label{fig:BayesianNetwork}
%\end{figure}

%\subsubsection{Dynamic Bayesian Network Technical Description}

%, which represent a general temporal probability model that describes a random system which is assumed to have a number of random variables, some of which are observable and some not \cite{AIAMA}.
%Dynamic Bayesian networks are usually used to represent the dynamics of a random system over time and can be described generally by specifying how the system transitions from its state at $t-1$ to $t$, if the first order Markov property is assumed. This thesis is concerned with the case when the first order Markov property holds.
It is often convenient to categorise the random variables in the network into state variables (assumed to be hidden), control variables and observation variables \cite{Thrun:2005:ProbabilisticRobotics}. An example of a Dynamic Bayesian Network can be seen in Figure \ref{fig:2TDBNExample}. The hidden state variables are shaded in light grey, the observation variables in light orange and the control variables in light green. The directed arcs represent "instantaneous" causation, as with regular Bayesian Networks \cite[p.~15]{Murphy1994DynamicLearning}. Note that unlike HMMs, there isn't a requirement to have a single state and observation variable across each time-step and other variables can be introduced into the model.

%The nodes in the graph correspond to the random variables whose joint distribution we would like to calculate, and the arcs represent conditional independences \cite{KollerPGM}. Specifically, if one node points to another. \par


\subsection{Structural breaks}

Testing for structural breaks with the Chow test.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Contents: TeX and LaTeX and AMS symbols for Maths
% $Id: lssym.tex 477 2011-06-18 13:47:14Z oetiker $
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{List of Mathematical Symbols}  \label{symbols}

The following tables demonstrate all the symbols normally accessible
from \emph{math mode}.

%
% Conditional Text in case the AMS Fonts are installed
%
Note that some tables show symbols only accessible after loading the \pai{amssymb}
package in the preamble of your document\footnote{The tables were derived
  from \texttt{symbols.tex} by David~Carlisle and subsequently changed
extensively as suggested by Josef~Tkadlec.}. If the \AmS{} package and
fonts are not installed on your system, have a look at
\CTANref|CTAN:pkg/amslatex|. An even more comprehensive list of
symbols can be found at \CTANref|CTAN:info/symbols/comprehensive|.

\begin{table}[!h]
\caption{Math Mode Accents.}  \label{mathacc}
\begin{symbols}{*3{cl}}
\W{\hat}{a}   & \W{\check}{a} & \W{\tilde}{a}       \\
\W{\grave}{a} & \W{\dot}{a}   & \W{\ddot}{a}        \\
\W{\bar}{a}   & \W{\vec}{a}   & \W{\widehat}{AAA}   \\
\W{\acute}{a} & \W{\breve}{a} & \W{\widetilde}{AAA} \\
\W{\mathring}{a}
\end{symbols}
\end{table}

\begin{table}[!h]
\caption{Greek Letters.} \label{greekletters}
\bigskip
There is no uppercase of some of the letters like \ci{Alpha}, \ci{Beta} and so
on, because they look the same as normal roman letters: A, B\ldots
\begin{symbols}{*4{cl}}
 \X{\alpha}     & \X{\theta}     & \X{o}          & \X{\upsilon}  \\
 \X{\beta}      & \X{\vartheta}  & \X{\pi}        & \X{\phi}      \\
 \X{\gamma}     & \X{\iota}      & \X{\varpi}     & \X{\varphi}   \\
 \X{\delta}     & \X{\kappa}     & \X{\rho}       & \X{\chi}      \\
 \X{\epsilon}   & \X{\lambda}    & \X{\varrho}    & \X{\psi}      \\
 \X{\varepsilon}& \X{\mu}        & \X{\sigma}     & \X{\omega}    \\
 \X{\zeta}      & \X{\nu}        & \X{\varsigma}  &               \\
 \X{\eta}       & \X{\xi}        & \X{\tau} & \\
 \X{\Gamma}     & \X{\Lambda}    & \X{\Sigma}     & \X{\Psi}      \\
 \X{\Delta}     & \X{\Xi}        & \X{\Upsilon}   & \X{\Omega}    \\
 \X{\Theta}     & \X{\Pi}        & \X{\Phi}
\end{symbols}
\end{table}

\begin{table}[!tbp]
\caption{Binary Relations.} \label{binaryrel}
\bigskip
You can negate the following symbols by prefixing them with a \ci{not} command.
\begin{symbols}{*3{cl}}
 \X{<}           & \X{>}           & \X{=}          \\
 \X{\leq}or \verb|\le|   & \X{\geq}or \verb|\ge|   & \X{\equiv}     \\
 \X{\ll}         & \X{\gg}         & \X{\doteq}     \\
 \X{\prec}       & \X{\succ}       & \X{\sim}       \\
 \X{\preceq}     & \X{\succeq}     & \X{\simeq}     \\
 \X{\subset}     & \X{\supset}     & \X{\approx}    \\
 \X{\subseteq}   & \X{\supseteq}   & \X{\cong}      \\
 \X{\sqsubset}$^a$ & \X{\sqsupset}$^a$ & \X{\Join}$^a$    \\
 \X{\sqsubseteq} & \X{\sqsupseteq} & \X{\bowtie}    \\
 \X{\in}         & \X{\ni}, \verb|\owns|  & \X{\propto}    \\
 \X{\vdash}      & \X{\dashv}      & \X{\models}    \\
 \X{\mid}        & \X{\parallel}   & \X{\perp}      \\
 \X{\smile}      & \X{\frown}      & \X{\asymp}     \\
 \X{:}           & \X{\notin}      & \X{\neq}or \verb|\ne|
\end{symbols}
\centerline{\footnotesize $^a$Use the \textsf{latexsym} package to access this symbol}
\end{table}

\begin{table}[!tbp]
\caption{Binary Operators.}
\begin{symbols}{*3{cl}}
 \X{+}              & \X{-}              & &                 \\
 \X{\pm}            & \X{\mp}            & \X{\triangleleft} \\
 \X{\cdot}          & \X{\div}           & \X{\triangleright}\\
 \X{\times}         & \X{\setminus}      & \X{\star}         \\
 \X{\cup}           & \X{\cap}           & \X{\ast}          \\
 \X{\sqcup}         & \X{\sqcap}         & \X{\circ}         \\
 \X{\vee}, \verb|\lor|     & \X{\wedge}, \verb|\land|  & \X{\bullet}       \\
 \X{\oplus}         & \X{\ominus}        & \X{\diamond}      \\
 \X{\odot}          & \X{\oslash}        & \X{\uplus}        \\
 \X{\otimes}        & \X{\bigcirc}       & \X{\amalg}        \\
 \X{\bigtriangleup} &\X{\bigtriangledown}& \X{\dagger}       \\
 \X{\lhd}$^a$         & \X{\rhd}$^a$         & \X{\ddagger}      \\
 \X{\unlhd}$^a$       & \X{\unrhd}$^a$       & \X{\wr}
\end{symbols}

\end{table}

\begin{table}[!tbp]
\caption{BIG Operators.}
\begin{symbols}{*4{cl}}
 \X{\sum}      & \X{\bigcup}   & \X{\bigvee}  \\
 \X{\prod}     & \X{\bigcap}   & \X{\bigwedge} \\
 \X{\coprod}   & \X{\bigsqcup} & \X{\biguplus} \\
 \X{\int}      & \X{\oint}     & \X{\bigodot} \\
 \X{\bigoplus} & \X{\bigotimes} & \\
\end{symbols}

\end{table}

\begin{table}[!tbp]
\caption{Arrows.} \label{tab:arrows}
\begin{symbols}{*2{cl}}
 \X{\leftarrow}or \verb|\gets|& \X{\longleftarrow} \\
 \X{\rightarrow}or \verb|\to|& \X{\longrightarrow} \\
 \X{\leftrightarrow}    & \X{\longleftrightarrow} \\
 \X{\Leftarrow}         & \X{\Longleftarrow}     \\
 \X{\Rightarrow}        & \X{\Longrightarrow}    \\
 \X{\Leftrightarrow}    & \X{\Longleftrightarrow}\\
 \X{\mapsto}            & \X{\longmapsto}        \\
 \X{\hookleftarrow}     & \X{\hookrightarrow}    \\
 \X{\leftharpoonup}     & \X{\rightharpoonup}    \\
 \X{\leftharpoondown}   & \X{\rightharpoondown}  \\
 \X{\rightleftharpoons} & \X{\iff}(bigger spaces) \\
 \X{\uparrow}   & \X{\downarrow} \\
 \X{\updownarrow} & \X{\Uparrow} \\
 \X{\Downarrow} &  \X{\Updownarrow} \\
 \X{\nearrow} &  \X{\searrow} \\
  \X{\swarrow} & \X{\nwarrow} \\
 \X{\leadsto}$^a$
\end{symbols}
\centerline{\footnotesize $^a$Use the \textsf{latexsym} package to access this symbol}
\end{table}

\begin{table}[!tbp]
\caption{Arrows as Accents.}  \label{arrowacc}
\begin{symbols}{*2{cl}}
\W{\overrightarrow}{AB}     & \W{\underrightarrow}{AB}     \\
\W{\overleftarrow}{AB}      & \W{\underleftarrow}{AB}      \\
\W{\overleftrightarrow}{AB} & \W{\underleftrightarrow}{AB} \\
\end{symbols}
\end{table}

\begin{table}[!tbp]
\caption{Delimiters.}\label{tab:delimiters}
\begin{symbols}{*3{cl}}
 \X{(}            & \X{)}            & \X{\uparrow} \\
 \X{[}or \verb|\lbrack|   & \X{]}or \verb|\rbrack|  & \X{\downarrow}   \\
 \X{\{}or \verb|\lbrace|  & \X{\}}or \verb|\rbrace|  & \X{\updownarrow} \\
 \X{\langle}      & \X{\rangle}      &  \X{\Uparrow} \\
 \X{|}or \verb|\vert| & \X{\|}or \verb|\Vert| & \X{\Downarrow} \\
  \X{/}            & \X{\backslash}   &   \X{\Updownarrow}  \\
 \X{\lfloor}      & \X{\rfloor}      &  \\
 \X{\rceil}       &  \X{\lceil}  &&\\
\end{symbols}
\end{table}

\begin{table}[!tbp]
\caption{Large Delimiters.}
\begin{symbols}{*3{cl}}
 \Y{\lgroup}      & \Y{\rgroup}      & \Y{\lmoustache}  \\
 \Y{\arrowvert}   & \Y{\Arrowvert}   & \Y{\bracevert} \\
 \Y{\rmoustache} \\
\end{symbols}
\end{table}

\begin{table}[!tbp]
\caption{Miscellaneous Symbols.}
\begin{symbols}{*4{cl}}
 \X{\dots}       & \X{\cdots}      & \X{\vdots}      & \X{\ddots}     \\
 \X{\hbar}       & \X{\imath}      & \X{\jmath}      & \X{\ell}       \\
 \X{\Re}         & \X{\Im}         & \X{\aleph}      & \X{\wp}        \\
 \X{\forall}     & \X{\exists}     & \X{\mho}$^a$      & \X{\partial}   \\
 \X{'}           & \X{\prime}      & \X{\emptyset}   & \X{\infty}     \\
 \X{\nabla}      & \X{\triangle}   & \X{\Box}$^a$     & \X{\Diamond}$^a$ \\
 \X{\bot}        & \X{\top}        & \X{\angle}      & \X{\surd}      \\
\X{\diamondsuit} & \X{\heartsuit}  & \X{\clubsuit}   & \X{\spadesuit} \\
 \X{\neg}or \verb|\lnot| & \X{\flat}       & \X{\natural}    & \X{\sharp}
\end{symbols}
\centerline{\footnotesize $^a$Use the \textsf{latexsym} package to access this symbol}
\end{table}

\begin{table}[!tbp]
\caption{Non-Mathematical Symbols.}
\bigskip
These symbols can also be used in text mode.
\begin{symbols}{*4{cl}}
 \SC{\dag}  &  \SC{\S}  &  \SC{\copyright} &  \SC{\textregistered}  \\
 \SC{\ddag} &  \SC{\P}  &  \SC{\pounds}    &  \SC{\%}               \\
\end{symbols}
\end{table}

\clearpage

%
%
% If the AMS Stuff is not available, we drop out right here :-)
%

\begin{table}[!tbp]
\caption{\AmS{} Delimiters.}\label{AMSD}
\bigskip
\begin{symbols}{*4{cl}}
\X{\ulcorner}&\X{\urcorner}&\X{\llcorner}&\X{\lrcorner}\\
\X{\lvert}&\X{\rvert}&\X{\lVert}&\X{\rVert}
\end{symbols}
\end{table}

\begin{table}[!tbp]
\caption{\AmS{} Greek and Hebrew.}
\begin{symbols}{*5{cl}}
\X{\digamma}     &\X{\varkappa} & \X{\beth} &\X{\gimel} & \X{\daleth}
\end{symbols}
\end{table}

\begin{table}[tbp]
  \caption{Math Alphabets.} \label{mathalpha}
\bigskip See Table~\ref{mathfonts} on \pageref{mathfonts} for other math fonts.
\begin{symbols}{@{}*3l@{}}
Example& Command &Required package\\
\hline
\rule{0pt}{1.05em}$\mathrm{ABCDE abcde 1234}$
        & \verb|\mathrm{ABCDE abcde 1234}|
        &       \\
$\mathit{ABCDE abcde 1234}$
        & \verb|\mathit{ABCDE abcde 1234}|
        &       \\
$\mathnormal{ABCDE abcde 1234}$
        & \verb|\mathnormal{ABCDE abcde 1234}|
        &  \\
$\mathcal{ABCDE abcde 1234}$
        & \verb|\mathcal{ABCDE abcde 1234}|
        &  \\
$\mathscr{ABCDE abcde 1234}$
        &\verb|\mathscr{ABCDE abcde 1234}|
        &\pai{mathrsfs}\\
$\mathfrak{ABCDE abcde 1234}$
        & \verb|\mathfrak{ABCDE abcde 1234}|
        &\pai{amsfonts}  or \textsf{amssymb}  \\
$\mathbb{ABCDE abcde 1234}$
        & \verb|\mathbb{ABCDE abcde 1234}|
        &\pai{amsfonts}  or \textsf{amssymb} \\
\end{symbols}
\end{table}

\begin{table}[!tbp]
\caption{\AmS{} Binary Operators.}
\begin{symbols}{*3{cl}}
 \X{\dotplus}        & \X{\centerdot}      &       \\
 \X{\ltimes}         & \X{\rtimes}         & \X{\divideontimes} \\
 \X{\doublecup}      & \X{\doublecap}	   & \X{\smallsetminus} \\
 \X{\veebar}         & \X{\barwedge}       & \X{\doublebarwedge}\\
 \X{\boxplus}        & \X{\boxminus}       & \X{\circleddash}   \\
 \X{\boxtimes}       & \X{\boxdot}         & \X{\circledcirc}   \\
 \X{\intercal}       & \X{\circledast}     & \X{\rightthreetimes} \\
 \X{\curlyvee}       & \X{\curlywedge}     & \X{\leftthreetimes}
\end{symbols}
\end{table}

\begin{table}[!tbp]
\caption{\AmS{} Binary Relations.}
\begin{symbols}{*3{cl}}
 \X{\lessdot}           & \X{\gtrdot}            & \X{\doteqdot} \\
 \X{\leqslant}          & \X{\geqslant}          & \X{\risingdotseq}     \\
 \X{\eqslantless}       & \X{\eqslantgtr}        & \X{\fallingdotseq}    \\
 \X{\leqq}              & \X{\geqq}              & \X{\eqcirc}           \\
 \X{\lll}or \verb|\llless| & \X{\ggg}            & \X{\circeq}  \\
 \X{\lesssim}           & \X{\gtrsim}            & \X{\triangleq}        \\
 \X{\lessapprox}        & \X{\gtrapprox}         & \X{\bumpeq}           \\
 \X{\lessgtr}           & \X{\gtrless}           & \X{\Bumpeq}           \\
 \X{\lesseqgtr}         & \X{\gtreqless}         & \X{\thicksim}         \\
 \X{\lesseqqgtr}        & \X{\gtreqqless}        & \X{\thickapprox}      \\
 \X{\preccurlyeq}       & \X{\succcurlyeq}       & \X{\approxeq}         \\
 \X{\curlyeqprec}       & \X{\curlyeqsucc}       & \X{\backsim}          \\
 \X{\precsim}           & \X{\succsim}           & \X{\backsimeq}        \\
 \X{\precapprox}        & \X{\succapprox}        & \X{\vDash}            \\
 \X{\subseteqq}         & \X{\supseteqq}         & \X{\Vdash}            \\
 \X{\shortparallel}     & \X{\Supset}            & \X{\Vvdash}           \\
 \X{\blacktriangleleft} & \X{\sqsupset}          & \X{\backepsilon}      \\
 \X{\vartriangleright}  & \X{\because}           & \X{\varpropto}        \\
 \X{\blacktriangleright}& \X{\Subset}            & \X{\between}          \\
 \X{\trianglerighteq}   & \X{\smallfrown}        & \X{\pitchfork}        \\
 \X{\vartriangleleft}   & \X{\shortmid} 	 & \X{\smallsmile} 	\\
 \X{\trianglelefteq}    & \X{\therefore} 	 & \X{\sqsubset}
\end{symbols}
\end{table}

\begin{table}[!tbp]
\caption{\AmS{} Arrows.}
\begin{symbols}{*2{cl}}
 \X{\dashleftarrow}      & \X{\dashrightarrow}     \\
 \X{\leftleftarrows}     & \X{\rightrightarrows}   \\
 \X{\leftrightarrows}    & \X{\rightleftarrows}    \\
 \X{\Lleftarrow}         & \X{\Rrightarrow}        \\
 \X{\twoheadleftarrow}   & \X{\twoheadrightarrow}  \\
 \X{\leftarrowtail}      & \X{\rightarrowtail}     \\
 \X{\leftrightharpoons}  & \X{\rightleftharpoons}  \\
 \X{\Lsh}                & \X{\Rsh}                \\
 \X{\looparrowleft}      & \X{\looparrowright}     \\
 \X{\curvearrowleft}     & \X{\curvearrowright}    \\
 \X{\circlearrowleft}    & \X{\circlearrowright}   \\
 \X{\multimap}  &  \X{\upuparrows}  \\
 \X{\downdownarrows} & \X{\upharpoonleft} \\
 \X{\upharpoonright} & \X{\downharpoonright} \\
 \X{\rightsquigarrow} & \X{\leftrightsquigarrow} \\
\end{symbols}
\end{table}

\begin{table}[!tbp]
\caption{\AmS{} Negated Binary Relations and Arrows.}\label{AMSNBR}
\begin{symbols}{*3{cl}}
 \X{\nless}           & \X{\ngtr}            & \X{\varsubsetneqq}  \\
 \X{\lneq}            & \X{\gneq}            & \X{\varsupsetneqq}  \\
 \X{\nleq}            & \X{\ngeq}            & \X{\nsubseteqq}     \\
 \X{\nleqslant}       & \X{\ngeqslant}       & \X{\nsupseteqq}     \\
 \X{\lneqq}           & \X{\gneqq}           & \X{\nmid}           \\
 \X{\lvertneqq}       & \X{\gvertneqq}       & \X{\nparallel}      \\
 \X{\nleqq}           & \X{\ngeqq}           & \X{\nshortmid}      \\
 \X{\lnsim}           & \X{\gnsim}           & \X{\nshortparallel} \\
 \X{\lnapprox}        & \X{\gnapprox}        & \X{\nsim}           \\
 \X{\nprec}           & \X{\nsucc}           & \X{\ncong}          \\
 \X{\npreceq}         & \X{\nsucceq}         & \X{\nvdash}         \\
 \X{\precneqq}        & \X{\succneqq}        & \X{\nvDash}         \\
 \X{\precnsim}        & \X{\succnsim}        & \X{\nVdash}         \\
 \X{\precnapprox}     & \X{\succnapprox}     & \X{\nVDash}         \\
 \X{\subsetneq}       & \X{\supsetneq}       & \X{\ntriangleleft}  \\
 \X{\varsubsetneq}    & \X{\varsupsetneq}    & \X{\ntriangleright} \\
 \X{\nsubseteq}       & \X{\nsupseteq}       & \X{\ntrianglelefteq}\\
 \X{\subsetneqq}      & \X{\supsetneqq}      &\X{\ntrianglerighteq}\\[0.5ex]
 \X{\nleftarrow}      & \X{\nrightarrow}     & \X{\nleftrightarrow}\\
 \X{\nLeftarrow}      & \X{\nRightarrow}     & \X{\nLeftrightarrow}

\end{symbols}
\end{table}

\begin{table}[!tbp] \label{AMSmisc}
\caption{\AmS{} Miscellaneous.}
\begin{symbols}{*3{cl}}
 \X{\hbar}             & \X{\hslash}           & \X{\Bbbk}            \\
 \X{\square}           & \X{\blacksquare}      & \X{\circledS}        \\
 \X{\vartriangle}      & \X{\blacktriangle}    & \X{\complement}      \\
 \X{\triangledown}     &\X{\blacktriangledown} & \X{\Game}            \\
 \X{\lozenge}          & \X{\blacklozenge}     & \X{\bigstar}         \\
 \X{\angle}            & \X{\measuredangle}    & \\
 \X{\diagup}           & \X{\diagdown}         & \X{\backprime}       \\
 \X{\nexists}          & \X{\Finv}             & \X{\varnothing}      \\
 \X{\eth}              & \X{\sphericalangle}   & \X{\mho}
\end{symbols}
\end{table}

\endinput

%

% Local Variables:
% TeX-master: "lshort2e"
% mode: latex
% mode: flyspell
% End:

synctex test files/less basic/2017/lshort-5.05/src/lssym.tex


% jam 2004-09-10

\section{Mesh Operations}
\label{sec:mesh-operations}

\subsection{Split}

Splitting a simplex produces a refined complex by adding a new vertex
and subdivided some of the simplices.

Suppose $\Ssimplex$ is a $d$-simplex
in the pure $n$-dimensional simplicial complex $\Kcomplex_0$.
Let $\Vvertex$ be a vertex not in $\Kcomplex_0$.
{\it Splitting $\Ssimplex$ around $\Vvertex$}
produces a refined pure $n$-dimensional simplicial complex, $\Kcomplex_1$:

$\Kcomplex_1$ contains all the  $n$-simplexes of $\Kcomplex_0$
that do not contain $\Ssimplex$.

The simplexes that do contain $\Ssimplex$ are split:
Let $\Tsimplex$ be an $n$-simplex that contains $\Ssimplex$.
Let $\Rsimplex$ be the $(n-d-1)$-simplex opposite $\Ssimplex$
in $\Tsimplex$.
(If $\Tsimplex = \Ssimplex$, then $\Rsimplex$ is the empty set.)
Let $\{ \Ffacet_0 \ldots \Ffacet_d \}$ be the
d $(d-1)$-simplex facets of $\Ssimplex$.
$\Kcomplex_1$ has
$d$ $n$-simplices formed from
$\Vvertex$, $\Rsimplex$, and
$\Ffacet_i$ where $i=0 \ldots d$.

This is also known as "placing a vertex"
(See Lee~\cite[sec.~17.2]{lee-hdcg-17-2004}).

\subsection{Collapse}

Collapsing a simplex produces a reduced complex by mergng
the vertices of the simplex into a single new one.

Suppose $\Ssimplex$ is a $d$-simplex
in the pure $n$-dimensional simplicial complex $\Kcomplex_0$.
Let $\Vvertex$ be a vertex not in $\Kcomplex_0$.
{\it Collapsing $\Ssimplex$ to $\Vvertex$}
produces a reduced pure $n$-dimensional simplicial complex, $\Kcomplex_1$:

$\Kcomplex_1$ contains all the
$n$-simplices of $\Kcomplex_0$ that share no vertices with $\Ssimplex$.
Any $n$-simplex in $\Kcomplex_0$ sharing more than one vertex
with $\Ssimplex$ is ignored.
For every $n$-simplex in $\Kcomplex_0$ sharing one vertex with $\Ssimplex$,
$\Kcomplex_1$ gets an $n$-simplex with $\Vvertex$ replacing the
shared vertex.


\subsection{Testing for cointegration with Johansen}