\documentclass[]{article}

\usepackage{amsthm} %qed
\usepackage[cmex10]{amsmath}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{graphicx}

\newtheorem{theorem}{Theorem}[section]
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\theoremstyle{definition}
\newtheorem{definition}[theorem]{Definition}

\newcommand{\BIN}{\begin{bmatrix}}
\newcommand{\BOUT}{\end{bmatrix}}

%\newcommand{\mdiff1}[1]{\frac{\partial}{\partial #1}}
\newcommand{\diff}[2]{\dfrac{\partial #1}{\partial #2}}
\newcommand{\norm}[1]{\left\| #1 \right\|}

\begin{document}

\title{\Large Substitutions}
\author{Adrien Escande}

\maketitle

\section{Introduction}
In this document, we are considering a system of linear equations that we want to pre-solve for some variables. We write this system
\begin{align}
A_{1,1} x_1 + A_{1,2} x_2 + \ldots + A_{1,k} x_k + B_1 y &= c_1 \\
A_{2,1} x_1 + A_{2,2} x_2 + \ldots + A_{2,k} x_k + B_2 y &= c_2 \\
\vdots& \nonumber\\
A_{k,1} x_1 + A_{k,2} x_2 + \ldots + A_{k,k} x_k + B_k y &= c_k \\
D_1 x_1 + D_2 x_2 + \ldots + D_k x_k + E y &= f \label{eq:other}
\end{align}
where the $x_j \in \mathbb{R}^{n_j}$, $j=1..k$ are the variables for which we want to pre-solve, $y \in \mathbb{R}^p$ are all the other possible variables, the $A_{i,j}$ are $m_i \times n_j$ matrices, the $B_i$ are $m_i \times p$, the $D_i$ are $q \times n_j$, $E$ is $q \times p$, the $c_i$ are $m_i$-vectors, and $f$ is a $q$-vector.
We suppose the system is feasible.

Without loss of generality, we can consider the $k$ first lines to form a block triangular system:
\begin{align}
A_{1,1} x_1 + A_{1,2} x_2 + \ldots + A_{1,k} x_k + B_1 y &= c_1 \label{eq:triang1}\\
A_{2,2} x_2 + \ldots + A_{2,k} x_k + B_2 y &= c_2 \label{eq:triang2}\\
\vdots& \nonumber\\
A_{k,k} x_k + B_k y &= c_k \label{eq:triangk}
\end{align}
Indeed, if $l\leq k$ variables, let's note them $x_1, \ldots x_l$, appears in exactly $l$ equations, that we can take to be the $l$ first ones, then these $l$ equations can be rewritten as one single equation $\tilde{A} \tilde{x} + \tilde{A}_{l+1} x_{l+1} + \ldots + \tilde{A}_k x_k + \tilde{B} y = \tilde{c}$, with
\begin{equation*}
	\tilde{A} = \BIN A_{1,1} & \hdots & A_{1,k} \\ \vdots & \ddots & \vdots \\ A_{k,1} & \hdots & A_{k,k} \BOUT, \
	\tilde{A_j} = \BIN A_{1,j} \\ \vdots \\ A_{l,j}\BOUT, \
	\tilde{B} = \BIN B_{1} \\ \vdots \\ B_{l}\BOUT,\
	\tilde{c} = \BIN c_{1} \\ \vdots \\ c_{l} \BOUT \mbox{and}\
	\tilde{x} = \BIN x_{1} \\ \vdots \\ x_{l} \BOUT
\end{equation*}

Let's note $r_j$ the rank of the matrix $A_{j,j}$. We require that $r_j \leq n_j$.
Our goal is to transform the system (\ref{eq:triang1}) - (\ref{eq:triangk}), (\ref{eq:other}) into an equivalent system
\begin{align}
  x_1 &= \varphi_1(y,z_1, \ldots, z_k) \\
	x_2 &= \varphi_2(y,z_2, \ldots, z_k) \\
  \vdots & \nonumber \\
	x_k &= \varphi_{k-1}(y,z_k) \\
	x_k &= \varphi_k(y) \\
	\psi(y,z_1, \ldots z_k) &= 0
\end{align}
where $z_j \in \mathbb{R}^{n_j-r_j}$, and $\psi$ and the $\varphi_i$ are linear functions.

\section{One variable substitution}
\subsection{Principle}
Consider the system
\begin{align}
	A x + B y &= c \label{eq:simple1}\\
	D x + E y &= f \label{eq:simple2}
\end{align}
where we want to pre-solve in $x$, using the first equation. We denote $r$ the rank of $A$.

Let $A^\sharp$ be a generalized inverse of $A$, $N$ a basis of $\mathrm{null}(A)$, and $S$ a basis of $\mathrm{null}(A^T)$, and define $P = (I-A^T G^T) N^{+T}$. Using the tautology $P^T x = P^T x$, we have
\begin{align*}
  \mathrm{(\ref{eq:simple1})} &\Longleftrightarrow
	\left\{\begin{array}{rcl}
	  \BIN A \\ P^T\BOUT x \hspace{-5pt}&=&\hspace{-5pt} \BIN -By + c \\ P^T x\BOUT
	\end{array}\right.\\
	&\Longleftrightarrow
	\left\{\begin{array}{rcl}
	  \BIN A^{\sharp} & N \\ S^T & 0 \BOUT \BIN A \\ P^T\BOUT x \hspace{-5pt}&=&\hspace{-5pt} \BIN A^{\sharp} & N \\ S^T & 0 \BOUT \BIN -By + c \\ P^T x\BOUT
	\end{array}\right.\\
	&\Longleftrightarrow
	\left\{\begin{array}{rcl}
	  x \hspace{-5pt}&=&\hspace{-5pt}  -A^{\sharp}By + N z + A^{\sharp}c \\
		z \hspace{-5pt}&=&\hspace{-5pt} P^T x\\
		S^T By \hspace{-5pt}&=&\hspace{-5pt} S^T c
	\end{array}\right.\\
	&\Longleftrightarrow
	\left\{\begin{array}{rcl}
	  x \hspace{-5pt}&=&\hspace{-5pt}  -A^{\sharp}By + N z + A^{\sharp}c, \quad z \in \mathbb{R}^{n-r} \\
		S^T By \hspace{-5pt}&=&\hspace{-5pt} S^T c
	\end{array}\right.
\end{align*}
The second equivalence uses the fact that the introduced matrix is invertible (see theorem~\ref{th:inverse}), the third that $A^{\sharp}A + NP^T = I$ (see theorem~\ref{th:NP}), and the last that $P$ is full rank, and so $P^T$ spans $\mathbb{R}^{n-r}$.

Noting $M = -A^\sharp B$, and $u = A^\sharp c$, we thus have
\begin{equation*}
  \left. \begin{array}{c} (\ref{eq:simple1}) \\ (\ref{eq:simple2}) \end{array} \right\}
	\Longleftrightarrow
	\left\{ \begin{array}{rcl}
	x \hspace{-5pt}&=&\hspace{-5pt} M y + N z + u \\
	(E + D M) y + D N z \hspace{-5pt}&=&\hspace{-5pt} f - D u \\
	S^T B y \hspace{-5pt}&=&\hspace{-5pt} S^T f
	\end{array} \right.
\end{equation*}

\subsection{Generic $A$}
For a generic $A$, we can compute a rank-revealing QR
\begin{equation}
	A = \BIN Q^r & Q^c \BOUT \BIN R^r & R^c \\ 0 & 0 \BOUT \BIN {\Pi^r}^T \\ {\Pi^c}^T \BOUT
\end{equation}
Then we can take $A^{\sharp} = {\Pi^r} {R^r}^{-1} {Q^r}^T$, $N = - \Pi^r {R^r}^{-1} R^c + \Pi^c$ and $S = Q^c$. This yields $P = \Pi^c$.

\section{Multiple variables}
Consider the system
\begin{align}
A_{1,1} x_1 + A_{1,2} x_2 + \ldots + A_{1,l} x_l + B_1^{(l)} y + Z_1^{(l)} z^{(l)}&= c_1^{(l)} \label{eq:triang_1}\\
A_{2,2} x_2 + \ldots + A_{2,l} x_l + B_2^{(l)} y + Z_2^{(l)} z^{(l)}&= c_2^{(l)} \label{eq:triang_2}\\
\vdots& \nonumber\\
A_{l,l} x_l + B_l^{(l)} y + Z_l^{(l)} z^{(l)}&= c_l^{(l)} \label{eq:triang_l}\\
D_1 x_1 + D_2 x_2 + \ldots + D_l x_l + E^{(l)} y + H^{(l)} z^{(l)}&= f^{(l)} \label{eq:other_}
\end{align}
for $l \leq k$, with $z \in \mathbb{R}^{n_s}$, $n_s = kn - \sum_1^k{r_i}$ and $Z_i^{(l)} \in \mathbb{R}^{m_i \times n_s}$.

We use the above section for eq.~(\ref{eq:triang_l}): let's $A_l^\sharp$, $N_l$ and $S_l$ be respectively a generalized inverse of $A_{l,l}$, a basis of $\mathrm{null}(A_{l,l})$ and a basis of $\mathrm{null}(A_{l,l}^T)$.
We define
\begin{align*}
	M_l &= -A_l^\sharp B_l^{(l)}\\
	u_l &= A_l^\sharp c_l^{(l)}
\end{align*}
Eq.~(\ref{eq:triang_l}) becomes
\begin{equation}
	\left\{ \begin{array}{rcl}
	x_l \hspace{-5pt}&=&\hspace{-5pt} M_l y -A_l^\sharp Z_l^{(l)} z^{(l)} + N_l z_l + u_l \\
	S_l^T B_l^{(l)} y + S_l^T Z_l^{(l)} z^{(l)} \hspace{-5pt}&=&\hspace{-5pt} S_l^T c_l^{(l)}
	\end{array}\right.
\end{equation}
Substituting $x_l$ in the $i^{\mathrm{th}}$ equation ($i<l$) yields
\begin{equation*}
\footnotesize
  A_{i,i} x_i + \ldots + A_{i,l-1} x_{l-1} + (B_i^{(l)} + A_{i,l} M_l)y
	+ \BIN\hspace{-1pt} Z_i^{(l)} \hspace{-7pt}-\hspace{-3pt} A_{i,l} A_l^\sharp Z_l^{(l)} &\hspace{-5pt} A_{i,l} N_l\hspace{-1pt}\BOUT\hspace{-5pt} \BIN \hspace{-1pt}z^{(l)}\hspace{-1pt} \\ z_l \BOUT \hspace{-4pt}
	= \hspace{-3pt} c_i^{(l)} \hspace{-3pt}- A_{i,l} u_l
\end{equation*}
Likewise eq.~(\ref{eq:other_}) becomes
\begin{equation*}
\small
  D_1 x_ 1+ \ldots + D_{l-1} x_{l-1} + (E^{(l)} + D_l M_l)y
	+ \BIN\hspace{-1pt} H^{(l)} \hspace{-5pt}-\hspace{-3pt} D_l A_l^\sharp Z_l^{(l)} &\hspace{-5pt} D_l N_l\hspace{-1pt}\BOUT\hspace{-5pt} \BIN \hspace{-1pt}z^{(l)}\hspace{-1pt} \\ z_l \BOUT \hspace{-4pt}
	= \hspace{-3pt} f^{(l)} \hspace{-3pt}- D_l u_l
\end{equation*}
We define
\footnote{
Note that we could also define $z^{(l-1)} = \BIN z_l \\ z^{(l)} \BOUT$ and change $Z_i^{(l-1)}$ and $H^{(l-1)}$ accordingly. We make the current choice with the implementation efficiency in mind: it is easier to grow a matrix by the bottom or the right, while ensuring the memory alignement.
}
\begin{align*}
B_i^{(l-1)} &= (B_i^{(l)} + A_{i,l} M_l) = (B_i^{(l)} - A_{i,l} A_l^\sharp B_l^{(l)})\\
z^{(l-1)} &= \BIN z^{(l)}\\ z_l \BOUT\\
Z_i^{(l-1)} &= \BIN Z_i^{(l)} - A_{i,l} A_l^\sharp Z_l^{(l)} & A_{i,l}  N_l\BOUT\\
c_i^{(l-1)} &= c_i^{(l)} - A_{i,l} u_l = c_i^{(l)} - A_{i,l} A_l^\sharp c_l^{(l)}\\
E^{(l-1)} &= E^{(l)} + D_l M_l = E^{(l)} - D_l A_l^\sharp B_l^{(l)}\\
H^{(l-1)} &= \BIN H^{(l)} - D_l A_l^\sharp Z_l^{(l)} & D_l N_l\BOUT\\
f^{(l-1)} &= f^{(l)} - D_l u_l = f^{(l)} - D_l A_l^\sharp c_l^{(l)}
\end{align*}
and the system (\ref{eq:triang_1}) - (\ref{eq:other_}) is equivalent to
\begin{align}
A_{1,1} x_1 + A_{1,2} x_2 + \ldots + A_{1,l-1} x_{l-1} + B_1^{(l-1)} y + Z_1^{(l-1)} z^{(l-1)}&= c_1^{(l-1)}  \label{eq:triang_1s}\\
A_{2,2} x_2 + \ldots + A_{2,l-1} x_{l-1} + B_2^{(l-1)} y + Z_2^{(l-1)} z^{(l-1)}&= c_2^{(l-1)}\\
\vdots& \nonumber\\
A_{l-1,l-1} x_{l-1} + B_{l-1}^{(l-1)} y + Z_{l-1}^{(l-1)} z^{(l-1)}&= c_{l-1}^{(l-1)}\\
D_1 x_1 + D_2 x_2 + \ldots + D_{l-1} x_{l-1} + E^{(l-1)} y + H^{(l-1)} z^{(l-1)}&= f^{(l-1)} \label{eq:other_s}\\
M_l y -A_l^\sharp Z_l^{(l)} z^{(l)} + N_l z_l + u_l &= x_l\\
	S_l^T B_l^{(l)} y + S_l^T Z_l^{(l)} z^{(l)} &= S_l^T c_l^{(l)}
\end{align}
Equations~(\ref{eq:triang_1s})-(\ref{eq:other_s}) have the same form as equations (\ref{eq:triang_1}) - (\ref{eq:other_}), so that we can apply the above recursively, starting with
\begin{align*}
B_i^{(k)} &= B_i\\
z^{(k)} &\mathrm{empty}\\
Z_i^{(k)} & \mathrm{empty}\\
c_i^{(k)} &= c_i\\
E^{(k)} &= E\\
H^{(k)} & \mathrm{empty} \\
f^{(k)} &= f
\end{align*}

We get
\begin{align}
  x_1 &= M_1 y - A_1^\sharp Z_1^{(1)} z^{(1)} + N_1 z_1 + u_1\\
  x_2 &= M_2 y - A_2^\sharp Z_2^{(2)} z^{(2)} + N_2 z_2 + u_2\\
  &\vdots\nonumber \\
  x_k &= M_k y - \underbrace{A_k^\sharp Z_k^{(k)} z^{(k)}}_0 + N_k z_k + u_k\\
  E^{(0)} y + H^{(0)} z^{(0)}&= f^{(0)} \\
	S_1^T B_1^{(1)} y + S_1^T Z_1^{(1)} z^{(1)} &= S_1^T c_1^{(1)}\\
	S_1^T B_1^{(2)} y + S_2^T Z_2^{(2)} z^{(2)} &= S_1^T c_2^{(2)}\\
	&\vdots\nonumber \\
	S_k^T B_k^{(k)} y + S_k^T Z_k^{(k)} z^{(k)} &= S_1^T c_k^{(k)}
\end{align}

\appendix
\section{Some properties around the generalized inverse}
In this section, $A$ is a general $m \times n$ matrix with rank $r$.

\begin{definition}
  $G \in \mathbb{R}^{n \times m}$ is a generalized inverse of $A$ if and only if $A G A = A$.
\end{definition}
The following theorem is due to [Rao 1972]:
\begin{theorem}
  Given a particular generalized inverse $G$, and an arbitrary matrix $M\in \mathbb{R}^{n \times m}$, the matrix
	\begin{equation}
		G + M - GAMAG
	\end{equation}
	is a generalized inverse of $A$, and all of the generalized inverses of $A$ are obtained this way.
\end{theorem}

Let's consider the SVD decomposition of $A$:
\begin{equation*}
  A = \BIN U_1 & U_2 \BOUT \BIN \Sigma & 0 \\ 0 & 0 \BOUT \BIN V_1^T \\ V_2^T \BOUT = U_1 S V_1^T
\end{equation*}
where $\BIN U_1 & U_2 \BOUT$ and $\BIN V_1 & V_2 \BOUT$ are orthogonal matrices, $U_1 \in \mathbb{R}^{m \times r}$, $U_2 \in \mathbb{R}^{m \times m-r}$, $V_1 \in \mathbb{R}^{n \times r}$, $V_2 \in \mathbb{R}^{n \times n-r}$ and $\Sigma \in \mathbb{R}^{r \times r}$. As a particular generalized inverse, we have the Moore-Penrose pseudo-inverse $A^+ = V_1 \Sigma^{-1} U_1^T$, and we can write any generalized inverse $G$ as
\begin{equation}
  G = V_1 \Sigma^{-1} U_1^T + M - V_1 V_1^T M U_1 U_1^T \label{eq:generalFormulation}
\end{equation}

\begin{lemma}
\label{lemma:projection}
For any generalized inverse $G$
	\begin{align*}
	V_2 V_2^T (I-GA) &= I-GA\\
	(I-AG)U_2U_2^T &= I-AG
	\end{align*}
\end{lemma}
\begin{proof}
Let $M$ be such that $G$ writes as in eq.~(\ref{eq:generalFormulation}). Then, using that $U_1^T U_1 = I$, $V_1^T V_1 = I$ and $V_1 V_1^T + V_2 V_2^T = I$, we get
\begin{equation*}
  I-GA = V_2 V_2^T (I-MA)
\end{equation*}
and thus
\begin{equation*}
  V_2 V_2^T (I-GA) = V_2 V_2^T V_2 V_2^T (I-MA) = V_2 V_2^T (I-MA) = I - GA
\end{equation*}
Similarly
\begin{equation*}
  I-AG = (I-AM)U_2 U_2^T
\end{equation*}
and
\begin{equation*}
  (I-AG)U_2 U_2^T = (I-AM)U_2 U_2^T U_2 U_2^T = (I-AM)U_2 U_2^T  = I-AG
\end{equation*}
\end{proof}

\begin{corollary}
\label{co:projection}
$(I-GA)G U_2 U_2^T = (I-GA)G$ and $V_2 V_2^T G(I-AG) = G(I-AG)$
\end{corollary}
\begin{proof}
Note that $(I-GA)G = (G - GAG) = G(I-AG)$. Then
\begin{align*}
  (I-GA)G U_2 U_2^T = G(I-AG) U_2 U_2^T = G(I-AG) = (I-GA)G \\
	V_2 V_2^T G(I-AG) = V_2 V_2^T (I-GA)G = (I-GA)G = G(I-AG)
\end{align*}
where for each line the second equality is a consequence of Lemma~\ref{lemma:projection}.
\end{proof}

We have that $\mathrm{rank}(GA) = \mathrm{rank}(AG) =  \mathrm{rank}(A) =  r$, so that the projectors $(I-GA)$ and $(I-AG)$ have rank $n-r$ and $m-r$ respectively.
Therefore, we have the rank factorizations
\begin{align*}
 \exists N,P\in \mathbb{R}^{n \times n-r}, \quad \mathrm{rank}(N) = \mathrm{rank}(P) = n-r &&\ I-GA = N P^T\\
  \exists T,S\in \mathbb{R}^{m \times m-r}, \quad \mathrm{rank}(T) = \mathrm{rank}(S) = m-r && I-AG = T S^T
\end{align*}

\begin{theorem}
\label{th:NP}
  $I-GA = N P^T$ with $N$ and $P$ full rank iff $N$ is a basis of $\mathrm{null}(A)$ and $P = (I-A^TG^T)N^{+T}$.
\end{theorem}
\begin{proof}
  Let's first prove $\Longrightarrow$.
	\newline
	$I-GA = N P^T \Longrightarrow A(I-GA) = A N P^T \Longrightarrow ANP^T=0$. Since $P^T$ is full rank, its range space is $\mathbb{R}^{n-r}$, so that $ANP^T=0 \Longrightarrow AN = 0$. $N$ is full rank so it is a basis of $\mathrm{null}(A)$.
\newline
	Since $N$ is full rank, $N^+ N = I$. Thus $I-GA = N P^T \Longrightarrow N^+(I-GA) = P^T \Longrightarrow P = (I-A^TG^T)N^{+T}$.

	Conversely, let's prove $\Longleftarrow$.
  \newline
	$V_2$ is also a basis of $\mathrm{null}(A)$, so that there is an $n-r \times n-r$ invertible matrix $X$ such that $N = V_2 X$. Therefore $N^+ = X^{-1} V_2^T$ and $N P^T = V_2 V_2^T (I-GA) = I-GA$ where the last equality comes from Lemma~\ref{lemma:projection}.
\end{proof}
A direct consequence of this theorem is that for any choice of a generalized inverse $G$ of $A$ and of a basis $N$ of $\mathrm{null}(A)$, we can find $P$ to get a rank factorization of $I-GA$.

An equivalent theorem can be made for $I-AG$:
\begin{theorem}
\label{th:TS}
  $I-AG = T S^T$ with $T$ and $S$ full rank iff $S$ is a basis of $\mathrm{null}(A^T)$ and $T = (I-AG)S^{+T}$.
\end{theorem}
\begin{proof}
	The proof follows the same lines as for the previous theorem.
\end{proof}

\begin{theorem}
\label{th:inverse}
	Let $G$ be a generalized inverse of $A$, $N$ a basis of $\mathrm{null}(A)$ and $S$ a basis of $\mathrm{null}(A^T)$. Then the matrix
	\begin{equation}
		H = \BIN G & N \\ S^T & 0 \BOUT
	\end{equation}
	is invertible and its inverse is
	\begin{equation}
	  \BIN A & T \\ P^T  & -P^TGS^{+T} \BOUT
	\end{equation}
	where $P$ and $T$ are as define in theorems \ref{th:NP} and \ref{th:TS}.
\end{theorem}
\begin{proof}
	Let's show that
	\begin{equation}
		\BIN A & T \\ P^T  & -P^TGS^+ \BOUT \BIN G & N \\ S^T & 0 \BOUT =
		\BIN AG + TS^T & AN \\ P^T G - P^TG S^{+T} S^T & P^T N \BOUT \label{eq:inverse}
	\end{equation}
	is the identity.
	\newline
	$AG + TS^T = I$ because by definition of $T$, $I-AG = T S^T$ (theorem \ref{th:TS}).
	\newline
	$AN = 0$ by definition of $N$.
	\newline
	Since $S$ can be written as $U_2 Y$ with $Y$ invertible, we have that $S^{+T} S^T = U_2 U_2^T$, so using the expression of $P$, $P^T G S^{+T} S^T = N^+(I-GA) G U_2 U_2^T = N^+(I-GA) G = P^T G$, with the second equality coming from Corollary \ref{co:projection}. Therefore $P^T G - P^TG S^{+T} S^T = P^T G - P^TG = 0$.
	\newline
	$P^T N = N^+(I-GA) N = N^+ N = I$ where the second equalities is a consequence of $AN = 0$ and the third one of $N$ being full rank.
	\newline
	The matrix given by eq.~(\ref{eq:inverse}) is a left inverse of $H$, therefore $H$ is invertible and it is its inverse.
\end{proof}
\emph{Remark}: the matrix in~(\ref{eq:inverse}) is therefore also the right inverse of $H$. This can be verified in the same way as above, using the fact that $P^T G S^{+T} = N^+ G T$.
\end{document}


\subsection{Tuning Rules According to Chien, Hrones and Reswick (Step Response Method)}

This  method is an improvement of the  Ziegler-Nichols  method  and  uses  the
system's step response characteristics $T_u$, $T_g$ and  $K_s$  (discussed  in
section \ref{sec:theory:characterisation}).

The procedure of the Chien-Hrones-Reswick method is:

\begin{enumerate}
    \item Record the step response of the plant and measure the gain $K_{S}$, the delay time $T_{u}$ and the compensation time $T_{g}$.
    \item Determine the controller parameters according to the tuning rules (see tables \ref{tab:CHR_rejection} and \ref{tab:CHR_tracking}).
\end{enumerate}

\begin{center}
    \begin{threeparttable}
        \begin{tabular}{cccc}
            \toprule
            Type & $K_p$                                    &  $T_{i}$            &  $T_{d}$ \\
            \midrule
            P    &  $0.3   \cdot\frac{T_g}{T_u \cdot K_s}$  &  -                  &  -                  \\
            PI   &  $0.6   \cdot\frac{T_g}{T_u \cdot K_s}$  &  $4 \cdot T_u$      &  -                  \\
            PID  &  $0.95  \cdot\frac{T_g}{T_u \cdot K_s}$  &  $2.4 \cdot T_u$    &  $0.42 \cdot T_u$   \\
            \bottomrule
        \end{tabular}
        \caption{Table with controller parameters according to the Chien-Hrones-Reswick method (good disturbance rejection).}
        \label{tab:CHR_rejection}
    \end{threeparttable}
\end{center}

\begin{center}
    \begin{threeparttable}
        \begin{tabular}{cccc}
            \toprule
            Type & $K_p$                                    &  $T_{i}$            &  $T_{d}$ \\
            \midrule
            P    &  $0.3   \cdot\frac{T_g}{T_u \cdot K_s}$  &  -                  &  -                  \\
            PI   &  $0.35  \cdot\frac{T_g}{T_u \cdot K_s}$  &  $1.2 \cdot T_g$    &  -                  \\
            PID  &  $0.6   \cdot\frac{T_g}{T_u \cdot K_s}$  &  $T_g$              &  $0.5 \cdot T_u$    \\
            \bottomrule
        \end{tabular}
        \caption{Table with controller parameters according to the Chien-Hrones-Reswick method (aperiodic behaviour, good tracking).}
        \label{tab:CHR_tracking}
    \end{threeparttable}
\end{center}

Based  on  the  ratio   of   $\lambda=\frac{T_{u}}{T_{g}}$,  we  can  say  how
controllability the control plant is.

\begin{center}
    \begin{threeparttable}
        \begin{tabular}{ccc}
            \toprule
            Ratio                     &  Controllability  & Difficulty \\
            \midrule
            $\lambda < 0.1$           &  very good        & small  \\
            $0.1 \leq \lambda < 0.2$  &  good             & medium  \\
            $0.2 \leq \lambda < 0.4$  &  fair             & large  \\
            $0.4 \leq \lambda < 0.8$  &  bad              & very large  \\
            $ \lambda \geq 0.2$       &  very bad         & \makecell{(special \\ measurements \\ required)}  \\
            \bottomrule
        \end{tabular}
        \caption{Table showing approximate ratios and how ``good'' they are}
        \label{tab:controllability}
    \end{threeparttable}
\end{center}


\documentclass{article}
\usepackage{amsmath}
\usepackage{graphicx}

\begin{document}
\begin{center}
\textbf{\huge{Week 2}}
\end{center}

\section{Independent random variables}
Definition: Two random variables $X_1, X_2 $ are said to be independent if,
$$ P(X_1=x_1,X_2=x_2)= P(X_1=x_1)P(X_2=x_2) \quad \forall \; x_1 \in \mathcal{X}_1\, \&\; x_2 \in \mathcal{X}_2$$

\fbox{$ P(X_1=x_1,X_2=x_2)$ means that $X_1=x_1$ \textbf{AND} $X_2=x_2$ occur.}

\subsection{Claim}
$$ \sum_{x_2 \in \mathcal{X}_2} P(X_1=x_1,X_2=x_2)=P(X_1=x_1)$$

\textbf{Proof}: Suppose $A $ \& $B$ are disjoint/mutually exclusive. Then,
$$ P(A \cap B)=0 \quad \& \quad P(A \cup B)=P(A)+P(B)$$

Now the events $(X_1=x_1)\cap (X_2=x_2)$ are disjoint for different values of $x_2\in \mathcal{X}_2$.(if $x_2 \neq x_2' \;\forall\; x_2\in \mathcal{X}_2$)

Thus,

    \begin{align*}
        \sum_{x_2 \in \mathcal{X}_2} P(X_1=x_1,X_2=x_2)& = \sum_{x_2 \in \mathcal{X}_2} P\bigl( (X_1=x_1)\cap (X_2=x_2) \bigr) \\
        &=P\left( \bigcup_{x_2 \in \mathcal{X}_2} (X_1=x_1)\cap (X_2=x_2) \right)
    \end{align*}
Now,
    \begin{align*}
        \bigcup_{x_2 \in \mathcal{X}_2} \left[(X_1=x_1)\cap (X_2=x_2) \right]
        &= (X_1=x_1)\bigcap \left[ \bigcup_{x_2 \in \mathcal{X}_2} (X_2=x_2) \right] \\
        &= (X_1=x_1)\bigcap \left[ x_2 \in \mathcal{X}_2 \right]
    \end{align*}
As $[ x_2 \in \mathcal{X}_2] $ forms the entire sample space,
\begin{align*}
    P\left( \bigcup_{x_2 \in \mathcal{X}_2} (X_1=x_1)\cap (X_2=x_2) \right) &= P\left( (X_1=x_1)\bigcap \left[ x_2 \in \mathcal{X}_2 \right] \right) \\
    &= P\left( (X_1=x_1) \bigcap \Omega \right) \\
    &= P(X_1=x_1)
\end{align*}

\section{Entropy}

Suppose $X_1 \in \mathcal{X}_1\, \&\; X_2 \in \mathcal{X}_2$ are \textbf{independant} random variables. Then,
$$ H(X_1,\;X_2)=H(X_1)+H(X_2)$$

Proof:
\begin{gather*}
    \sum_{ \substack{x_1 \in \mathcal{X}_1 \\ x_2 \in \mathcal{X}_2}} P(X_1=x_1,X_2=x_2) \log \left( \frac{1}{P(X_1=x_1,X_2=x_2)} \right) \\
    = \sum_{x_1 \in \mathcal{X}_1 }\sum_{x_2 \in \mathcal{X}_2} P(X_1=x_1,X_2=x_2) \left[  \log \frac{1}{P(X_1=x_1)} + \log \frac{1}{P(X_2=x_2)}\right] \\
    = \sum_{x_1 \in \mathcal{X}_1 } \log \frac{1}{P(X_1=x_1)} \left( \sum_{x_2 \in \mathcal{X}_2} P(X_1=x_1,X_2=x_2)\right) \\
    + \sum_{x_2 \in \mathcal{X}_2 } \log \frac{1}{P(X_2=x_2)} \left( \sum_{x_1 \in \mathcal{X}_1} P(X_1=x_1,X_2=x_2)\right)
\end{gather*}

From the previous claim,
\begin{gather*}
    H(X_1,X_2)= \sum_{x_1 \in \mathcal{X}_1}P(X_1=x_1) \log \frac{1}{P(X_1=x_1)} \\
    + \sum_{x_2 \in \mathcal{X}_2}P(X_2=x_2) \log \frac{1}{P(X_2=x_2)} \\
    =H(X_1)+H(X_2)
\end{gather*}

\subsection{Conditional probability distribution}

What if $X_1$ \& $X_2$ are not independent? Then we would use conditional probability distribution. i.e.
$$ P(X_2=x_2/X_1=x_1):= \frac{P(X_2=x_2, X_1=x_1)}{P(X_1=x_1)}, \quad P(X_1=x_1) \neq 0 $$

This definition for conditional probability satisfies the probability axioms and hence it is a valid probability measure.

\subsection{Conditional entropy}
Definition:
$$ H(X_2/X_1):= \sum_{x_1 \in \mathcal{X}_1} P(X_1=x_1) H(X_2/X_1=x_1)$$
where,
$$ H(X_2/X_1=x_1):= \sum_{x_2 \in \mathcal{X}_2}P(X_2=x_2/X_1=x_1) \log \frac{1}{P(X_2=x_2/X_1=x_1)}$$

\subsubsection{Support of a function}

When $P(X=x)=0$, it is not considered in calculating the entropy.
$$ H(X)=-\sum_{ \{ x \in \mathcal{X} : P(X=x) \neq 0\} } P(X=x)log(P(X=x))$$

Suppose that f : X → R is a real-valued function whose domain is an arbitrary set X. The set-theoretic support of f, written supp(f), is the set of points in X where f is non-zero:
$$ supp(X)=\{ x \in X : f(x) \neq 0 \}$$

Note: $P(X=x)$ can be denoted as $P_X(x)$ or $P(x)$.
\subsection{Important Results}
     $$ H(X)= -\sum_{x \in supp(P)} P(x)log(P(x))$$
     $$ H(X/Y)= \sum_{y \in supp(P_Y)} P_Y(y)H(X/Y=y)$$
     $$ H(X/Y=y)= -\sum_{x \in supp(P_{X/Y})} P_{X/Y}(x/y)log(P_{X/Y}(x/y))$$

\subsection{Chain Rule}
$$ H(X,Y)=H(X)+H(Y/X)=H(Y)+H(X/Y)$$

Proof:
\begin{multline*}
     H(X)+H(Y/X)= -\sum_{x \in supp(P)} P(x)log(P(x)) \\
      + \sum_{x \in supp(P_X)} P(x)\left(\sum_{y \in supp(P_{Y})} P(y/x)log\left(\frac{1}{P(y/x)}\right)\right)
\end{multline*}

\begin{multline*}
    \qquad \qquad \qquad \quad= -\sum_{x \in supp(P_X)}\left(\sum_{y \in supp(P_Y)} P(x,y)\right)log(P(x)) \\
    + \sum_{x \in supp(P_X)} \left(\sum_{y \in supp(P_{Y})} P(x,y) \right) log\left(\frac{1}{P(y/x)}\right)
\end{multline*}

$$ \qquad \qquad =\sum_{x \in supp(P_X)} \sum_{y \in supp(P_{Y})} P(x,y)\left(\log \frac{1}{P(x)P(y/x)} \right)  $$

$$ = - \sum_x \sum_y P(x,y)\log P(x,y)=H(x,y)$$

\subsection{Upper and lower bound of entropy}

$$ 0 \leq H(X) \leq |log \mathcal X|$$

Proof:

$ H(X)= \sum_x P(x)log \frac{1}{P(x)} \rightarrow $ always positive as $P(x)>0$ $\forall x \in supp(P_X)$.

$H(X)= 0$ when $|supp(P_X)|=1$.

\subsubsection{Jensen's inequality (Concave/convex functions)}

\includegraphics[width=\textwidth]{graph.png}
$x_1 < x_3 <x_2 $ such that $x_3= tx_1+(1-t)x_2$

If $f(x_3) \geq tf(x_1)+(1-t)f(x_2) \rightarrow $ Concave function.

If $f(tx_1+(1-t)x_2) \leq tf(x_1)+(1-t)f(x_2) \rightarrow $ Convex function.

Hence, the above graph is a convex function.

We shall use the concavity property of the log function to obtain the proof for $ H(X) \leq |log \mathcal X| $.

Proof:
$$ H(X)= \sum_{x \in supp(P)} P(x)log\frac{1}{P(x)}$$

Let us assume $\lambda_x$ is another notation for $P(x)$.

$$H(X)= \sum_{x \in supp(P)} \lambda_x log\frac{1}{P(x)}$$

This is a convex combination of $log\frac{1}{P(x)}:x \in supp(P_X)$ as $\sum_{x}P(x)=1$.

$$ H(X) \leq \log \left(\sum_{x \in supp(P_X)}\lambda_x \frac{1}{P(x)} \right) \leq \log |supp(P_x)|$$
$$ \Rightarrow H(X) \leq \log |\mathcal X |$$

\subsubsection{When is $H(X)$ is $ log|\mathcal X|$?}

When Jensen's inequality is satisfied with equality, i.e. if $f(x)$ is a straight line, but the $\log x$ curve is strictly concave.

Suppose $f(x)$ is strictly concave(or convex) \& $\lambda_1, \lambda_2 \neq 0$ \& $\lambda_1+\lambda_2=1$.

If $f(\lambda_1 x_1 + \lambda_2 x_2)=\lambda_1 f(x_1) + \lambda_2 f(x_2)$, then $x_1=x_2$.

More generally, if $\lambda_i \neq 0 $, $\sum_{i=1}^{n} \lambda_i=1$ and Jensen's inequality holds with equality, then $x_1=x_2\cdots x_n$.

Applying this to $H(X) \leq log|\mathcal X|$, from the previous proof,

$$ H(X)= \sum_{x \in supp(P)} \lambda_x log\frac{1}{P(x)} \leq \log |supp(P_x)| $$
Suppose equality holds, then by the above claim we must have:

$$ \frac{1}{P(X)}= const \quad \forall x \in supp(P_x)$$
$$ \Rightarrow const.\; c= P(x)= \frac{1}{|supp(P_x)|}$$

If $|supp(P_x)|=|\mathcal X|$, $H(X)= \log |supp(P_x)|$. Then $P(X)= \frac{1}{|\mathcal X|}\; \forall x \in \mathcal X$.

We have just proved that $H(X)= \log_2 |\mathcal X|$, this can be true only when $P_x$ is uniform. Thus:

Lemma:
$ H(X)= \log_2 |\mathcal X| $ iff $P_x$ is uniform.

\section{Relative Entropy/ Information Divergence/ Kullback-Leibler Divergence}

Suppose there is a random variable $X$ for which we have two probability distributions $p_x$ \& $q_x$.

Then the RE or ID or KL is defined as:
$$ D(p_X || q_Y):=\sum_{x \in supp(p_x)} p(x)\log \frac{p(x)}{q(x)}$$

($D(p_X || q_Y)$ is a `kind of' a distance measure between distributions p \& q.)

\subsection{Lower limit of relative entropy}
$$ D(p||q)\geq 0$$
Proof:
$$D(p || q)=-\sum_{x \in supp(p_x)} p(x)\log \frac{q(x)}{p(x)}$$
$$ \geq - \log \left(\sum_{x \in supp(p_x)} p(x) \frac{q(x)}{p(x)} \right)$$
$$ \geq - \log\left(\sum_{x \in supp(p_x)} q(x)\right)$$

$$ \geq 0 \quad as \quad \sum q(x) \leq 1$$

\subsection{When is RE equal to zero?}

By applying Jensen's inequality for equality condition.

$$ \frac{p(x)}{q(x)}= const. \; c \qquad \forall x \in supp(p_x)$$
$$ \Rightarrow p(x)=cq(x) \quad \&$$
$$ \sum_{x \in supp(p_x)} p(x)= \sum_{x \in supp(p_x)} cq(x) \forall x \in supp(p_x)$$
These togethermean that $c=1$.
$$\Rightarrow p(x)=q(x) \forall x \in supp(p_x)$$
$$ \Rightarrow D(p||q) \quad iff \quad p_x=q_x$$

\subsection{Upper and lower limits of conditional entropy}

$$ H(X/Y)= \sum_{y \in supp(P_Y)} P_Y(y) \sum_{x \in supp(P_{X/Y})}P_{X/Y}(x/y)\log \frac{1}{P_{X/Y}(x/y)}$$

$$ 0 \leq H(X/Y) \leq H(X)$$

Proof:

$H(X/Y) \geq 0$ is true because $H(X/Y=y)\geq 0$ \& $P(y) \geq 0$.

$$ H(X)-H(X/Y)= \sum_{x \in supp(P_x)}P(x)\log\frac{1}{P(x)}-\sum_{x,y}P(x,y)\log\frac{1}{P(x/y)}$$
$$ =\sum_{x,y}P(x,y)\log\frac{1}{P(x)}-\sum_{x,y}P(x,y)\log\frac{1}{P(x/y)}$$
$$ = \sum_{x,y}P(x,y)\log \left(\frac{P(x/y)}{P(x)}\right)= \sum_{x,y}P(x,y)\log \left(\frac{P(x,y)}{P(x)P(y)}\right)$$

As both $P(x,y)$ \& $P(x)P(y)$ are valid joint distribution of $x,y$. Hence,

$$ H(X)-H(X/Y)= D(P(x,y)||P(x)P(y))$$
$$ \geq 0 \qquad (as\; D(p||q)\geq 0 )$$
$$ \Rightarrow H(X/Y) \leq H(X)$$

\end{document}


\renewcommand*\chappic{img/satisfiability.pdf}
\renewcommand*\chapquote{What idiot called them logic errors rather than bool shit?}
\renewcommand*\chapquotesrc{Unknown}
\chapter{Satisfiability}
\label{ch:sat}
%
Boolean algebra allows us to describe functions over two-valued variables.
Satisfiability is the question for an assignment such that a function
evaluates to true. Satisfiability problems are solved by SAT~solvers.
We discuss the basic theory behind satisfiability. Because any computation
can be represented as satisfiability problem, we are able to verify
whether an algorithm can reach a certain state. In Chapter~\ref{ch:enc},
we will represent a differential cryptanalysis problem such that it is
solvable iff the corresponding SAT~problem is satisfiable.

\section{Basic notation and definitions}
\label{sec:sat-intro}
%
\index{Boolean function}
\begin{defi}[Boolean function]
  A \emph{Boolean function} is a mapping $h: X \to Y$ with $X = \left\{0,1\right\}^n$
  for $n \in \mathbb N_{\geq 1}$ and $Y = \left\{0,1\right\}$.
\end{defi}

\index{Assignment}
\begin{defi}[Assignment]
  A \emph{$k$-assignment} is an element of $\left\{0,1\right\}^k$.

  \noindent
  Let $f$ be some $k$-ary Boolean function.
  An \emph{assignment for function $f$} is any $k$-assignment.
\end{defi}

\index{Truth table}
\begin{defi}[Truth table]
  Let $f$ be some $k$-ary Boolean function.
  The \emph{truth table of Boolean function $f$} assigns
  truth value $0$ or $1$ to any assignment of $f$.
\end{defi}

Boolean functions are characterized by their corresponding truth table.

\begin{table}[pt]
  \centering
  \subfloat[\boolf{AND}]{%
    \begin{tabular}{cc|c}
      $x_1$ & $x_2$ & $f(x_1, x_2)$ \\
     \hline
      $1$ & $1$ & $1$ \\
      $1$ & $0$ & $0$ \\
      $0$ & $1$ & $0$ \\
      $0$ & $0$ & $0$
    \end{tabular}
  }
  ~
  \subfloat[\boolf{OR}]{%
    \begin{tabular}{cc|c}
      $x_1$ & $x_2$ & $f(x_1, x_2)$ \\
     \hline
      $1$ & $1$ & $1$ \\
      $1$ & $0$ & $1$ \\
      $0$ & $1$ & $1$ \\
      $0$ & $0$ & $0$
    \end{tabular}
  }
  ~
  \raisebox{13.6pt}{%
    \subfloat[\boolf{NOT}]{%
      \begin{tabular}{c|c}
        $x$ & $f(x)$ \\
       \hline
        $1$ & $0$ \\
        $0$ & $1$
      \end{tabular}
    }%
  }%
  \caption{Truth tables for \boolf{AND}, \boolf{OR} and \boolf{NOT}}
  \label{tab:andornot-truthtables}
\end{table}

Table~\ref{tab:andornot-truthtables} shows example truth tables for
the Boolean \boolf{AND}, \boolf{OR} and \boolf{NOT} functions.
A different definition of the three functions is given the following way:

\index{AND (Boolean function)}
\index{OR (Boolean function)}
\index{NOT (Boolean function)}
\begin{defi}
  Let \boolf{AND}, \boolf{OR} and \boolf{NOT} be three Boolean functions.
  \begin{itemize}[noitemsep,topsep=0pt]
    \item
      \boolf{AND} maps $X = \left\{0,1\right\}^2$
      to $1$ if all values of $X$ are $1$.
    \item
      \boolf{OR} maps $X = \left\{0,1\right\}^2$
      to $1$ if any value of $X$ is $1$.
    \item
      \boolf{NOT} maps $X = \left\{0,1\right\}^1$
      to $1$ if the single value of $X$ is $0$.
  \end{itemize}
  All functions return $0$ in the other case.
  Those functions are denoted $a_0 \land a_1$, $a_0 \lor a_1$
  and $\neg a_0$ respectively, for input parameters $a_0$ and $a_1$.
\end{defi}

It is interesting to observe, that any Boolean function can be represented
using only these three operators. This can be proven by complete induction
over the number of arguments $k$ of the function.

Let $k = 1$. Then we consider any possible $2$-assignment for one input
variable $x_1$ and one value of $f(x_1)$. Then four truth tables are possible
listed in Table~\ref{tab:unary-f}. The description shows the corresponding
definition of $f$ using \boolf{AND}, \boolf{OR} and \boolf{NOT} only.

Now let $g$ be some $k$-ary function. Let $(a_0, a_1, \ldots, a_k)$ be the
$k$ input arguments to $g$ and $x_1 \coloneqq g(a_0, a_1, \ldots, a_k)$.
Then we can again look at Table~\ref{tab:unary-f} to discover that 4 cases
are possible: 2 cases where the return value of our new $(k+1)$-ary function
depends on value $x_1$ and 2 cases where the return value is constant.

This completes our proof.

\begin{table}[ht]
  \centering
  \subfloat[$f: x \mapsto 1$]{%
    \begin{tabular}{cc}
      $x_1$ & $f(x_1)$ \\
     \hline
      $1$ & $1$ \\
      $0$ & $1$
    \end{tabular}
  }
  ~
  \subfloat[$f: x \mapsto x$]{%
    \begin{tabular}{cc}
      $x_1$ & $f(x_1)$ \\
     \hline
      $1$ & $1$ \\
      $0$ & $0$
    \end{tabular}
  }
  ~
  \subfloat[$f: x \mapsto \neg x$]{%
    \begin{tabular}{cc}
      $x_1$ & $f(x_1)$ \\
     \hline
      $1$ & $0$ \\
      $0$ & $1$
    \end{tabular}
  }
  ~
  \subfloat[$f: x \mapsto 0$]{%
    \begin{tabular}{cc}
      $x_1$ & $f(x_1)$ \\
     \hline
      $1$ & $0$ \\
      $0$ & $0$
    \end{tabular}
  }%
  \caption{Unary $f$ and its four possible cases}
  \label{tab:unary-f}
\end{table}

Boolean functions have an important property which is described
in the following definition:

\index{Satisfiability}
\index{Assignment}
\index{Model}
\begin{defi}
  A Boolean function $f$ is \emph{satisfiable} iff there exists at least one
  input $x \in X$ such that $f(x) = 1$.
  Every input $x \in X$ satisfying this property is called \emph{model}.
\end{defi}

The corresponding tool to determine satisfiability is defined as follows:

\index{SAT solver}
\begin{defi}
  A \emph{SAT solver} is a tool to determine satisfiability (SAT or UNSAT)
  of a Boolean function. If satisfiability is given, it returns some model.
\end{defi}

\subsection{Computational considerations}
\label{sec:sat-complexity}
%
The generic complexity of SAT determination is given by $2^n$ for $n$ Boolean variables.

Let $n$ be the number of variables of a Boolean function.
No known algorithm exists to determine satisfiability in polynomial runtime.
This means no algorithm solves the SAT problem with runtime behavior
which depends polynomially on the growth of $n$.
%
\index{Unit propagation}
However, SAT solvers can take advantage of the problem's description.
For example, consider function $f$:
\begin{align} f(x_0, x_1, x_2) &= x_0 \land (\neg x_1 \lor x_2) \label{eq:3f} \end{align}
Instead of trying all possible 8~cases for 3~Boolean variables,
we can immediately see that $x_0$ is required to be $1$.
So we don't need to test $x_0 = 0$ and can skip 4~cases.
This particular strategy is called \emph{unit propagation}.

\subsection{SAT competitions}
\label{sec:sat-competitions}
%
SAT research is heavily concerned with finding fast heuristics
determining (un)satisfiability. Biyearly,
SAT competitions~\cite{satcomp} take place to challenge
SAT solvers in a set of benchmarks. The committee evaluates the most successful
SAT solvers defined by solving the most problems within a given time frame.

In 2014, lingeling by Armin Biere has won first prize in
the Application benchmarks track and second prize in the Hard Combinatorial benchmarks
track for SAT and UNSAT instances, respectively. Its parallelized sibling plingeling
and Cube \& Conquer sibling treengeling have won prizes in parallel settings.
And in the most recent 2016 competition lingeling has won bronze in the Main track
for SAT+UNSAT instances.

In Chapter~\ref{ch:results}, we will discuss runtime results shown by (but not limited to)
those SAT solvers.

\section{The DIMACS de-facto standard}
\label{sec:sat-dimacs}
%
\index{Conjunction}
\index{Disjunction}
\index{Literal}
\index{Positive literal}
\index{Negative literal}
\index{Conjunctive Normal Form}
\begin{defi}
  A SAT problem is given in \emph{Conjunctive Normal Form} (CNF) if
  the problem is defined as conjunction of disjunctions of literals.

  A \emph{conjunction} is a sequence of Boolean functions combined using
  a logical \boolf{AND}. A \emph{disjunction} is a sequence of Boolean functions
  combined using a logical \boolf{OR}. A \emph{literal} is a Boolean variable
  (\emph{positive}) or its negation (\emph{negative}).
\end{defi}

A simple example for a SAT problem in CNF is the exclusive \boolf{OR} (\boolf{XOR}).
It takes two Boolean values $a$ and $b$ as arguments and returns true
if and only if the two arguments differ:
{
\setlength{\abovedisplayskip}{5pt}
\setlength{\belowdisplayskip}{5pt}
\setlength{\abovedisplayshortskip}{0pt}
\setlength{\belowdisplayshortskip}{0pt}
\begin{align} (a \lor b) \land (\neg a \lor \neg b) \label{eq:xor}\end{align}
}
It consists of one conjunction (denoted $\land$) of two disjunctions
(denoted $\lor$) of literals (denoted $a$ and $b$ where prefix $\neg$ represents
negation). This structure constitutes a CNF.

\index{Disjunctive Normal Form}
Analogously, we define a \emph{Disjunctive Normal Form} (DNF) as disjunction
of conjunctions of literals. The negation of a CNF is in DNF, because literals
are negated and conjunctions become disjunctions, vice versa.

\begin{theorem}
  \label{thm:all-cnf}
  Every Boolean function can be represented as CNF.
\end{theorem}

Theorem~\ref{thm:all-cnf} is easy to prove.
% Consider null-ary functions as
% induction base. A (null-ary) Boolean function is a set of associations.
% An association maps an assignment for input arguments to one output value.
% True can be represented as $a \lor \neg a$ and false can be
% represented as $a \land \neg a$ for some free Boolean variable $a$ and using
% only negation, conjunctions and disjunctions. Recognize that those representations
% are given in DNF as well as CNF. Hence all null-ary Boolean functions can be
% represented as CNF and DNF.
%
% Let $f$ be a Boolean function represented in DNF with $k$ input arguments.
% Extend every conjunction with a Boolean variable $v_{k+1}$.
Consider the truth table of an arbitrary Boolean function $f$ with $k$ input arguments
and $j$ rows of output value false. We represent $f$ as CNF.

Consider Boolean variables $b_{i,l}$ with $0 \leq i \leq j$ and $0 \leq l \leq k$.
For every row $i$ of the truth table with assignment $(r_i)$, add one disjunction to the CNF.
This disjunction contains $b_{i,l}$ if $r_{i,l}$ is false.
The disjunction contains $b_{i,l}$ if $r_{i,l}$ is true.

As far as $f$ is an arbitrary $k$-ary Boolean function, we have proven that
any Boolean function can be represented as CNF.

SAT problems are usually represented in the DIMACS de-facto standard.
Consider a SAT problem in CNF with \emph{nbclauses} clauses and
enumerate all variables from 1 to \emph{nbvars}. A DIMACS file is an ASCII text
file. Lines starting with \enquote{\texttt{c}} are skipped (comment lines).
The first remaining line has to begin with \enquote{\texttt{p cnf}} followed by
\emph{nbclauses} and \emph{nbvars} separated by spaces (header line).
All following non-comment lines are space-separated indices of Boolean variables
optionally prefixed by a hyphen. Then one line represents one clause and
must be terminated with a zero character after a space. All lines are conjuncted
to form a CNF.

Variations of the DIMACS de-facto standard also allow multiline clauses (the
zero character constitutes the end of a clause) or arbitrary whitespace instead of
spaces. Another variant terminates DIMACS files once it encounters a single
percent sign on a line. The syntactical details are individually published
on a per competition basis.

\renewcommand{\lstlistingname}{Listing}  % REMARK otherwise it's Japanese, WTF
\begin{lstlisting}[caption={CNF of the \boolf{XOR} in Display~\eqref{eq:xor}}]
p cnf 2 2
a b
-a -b
\end{lstlisting}

\section{Terminology}
\label{sec:sat-terminology}
%
Given a conjunctive structure of disjunctions, we can define terms
related to this structure. Those terms will be used in the SAT features
we present in Section~\ref{sec:features-suggested}.

\index{Clause}
\index{$k$-clause}
\index{Unit clause}
\index{Horn clause}
\index{Definite clause}
\begin{defi}
  A \emph{clause} is a disjunction of literals.
  A $k$-\emph{clause} is a clause consisting of exactly $k$ literals.
  A \emph{unit clause} is a $1$-clause.

  A \emph{Horn clause} is a clause with at most one positive literal.
  A \emph{definite clause} is a clause with exactly one positive literal.
  A \emph{goal clause} is a clause with no positive literal.
\end{defi}

\index{Negated literal}
\index{Existential literal}
\index{Used variable}
\begin{defi}
  Given a literal, its \emph{negated literal} is the literal with its sign negated.
  A literal is \emph{positive}, if its sign is positive. A literal is \emph{negative} if its sign is negative.

  An \emph{existential literal} is a literal which occurs exactly once and
  its negation does not occur. A \emph{used variable} is a variable which
  occurs at least once in the CNF.

  The \emph{literal frequency} is the number of occurences of a literal in the CNF divided by the number of clauses declared.
  Equivalently \emph{variable frequency} defines the number of variable occurences divided by the number of clauses declared.
\end{defi}

\index{Clause length}
\index{Tautological clause}
\begin{defi}
  The \emph{clause length of a clause} is the number of literals contained.
  A clause is called \emph{tautological} if a literal and its negated literal occurs in it.
\end{defi}

A few basic properties hold in terms of satisfiability. For example, existential literals
are interesting, because they can be set to true and make one clause immediately satisfied
without influencing other clauses.

\section{Basic SAT solving techniques}
\label{sec:sat-solving}
%
\index{Equisatisfiability}
\begin{defi}
  Given two CNFs $A$ and $B$, they are called \emph{equisatisfiable}
  iff $A$ is satisfiable iff $B$.
\end{defi}

\subsection{Boolean constraint propagation (BCP)}
\label{sec:sat-bcp}
%
\index{Boolean constraint propagation}
One of the most basic techniques to SAT solving is \emph{Boolean Constraint Propagation},
also called \emph{unit propagation}.
It is so fundamental that SATzilla, introduced in Section~\ref{sec:features-related},
applies it immediately before looking at SAT features.

Let $l$ be the literal of a unit clause in a CNF. Remove any clause containing
$l$ and replace any occurences of $-l$ from the CNF. It is easy to see, that
the resulting CNF is equisatisfiable, because due to the unit clause $l$ must
be true. So any clause containing $l$ is satisfied and $-l$ yields false,
where ($A \lor $ false) is equivalent to $(A)$ for any Boolean function $A$.

\subsection{Watched Literals}
\label{sec:sat-wl}
%
\index{Watched Literals}
Watched Literals are another fundamental concept in SAT solving. It is very
expensive to check satisfiability of all clauses for every assigned value
of a literal. Watched Literals is a neat technique to reduce the number of checks.

In each clause two unassigned literals are declared to be \enquote{watched}.
Structurally it is implemented the other way around:
A clauses watch list is maintained per literal.
Now as long as at least two literals are unassigned, the clause cannot become
false (recognize that a clause is false iff all literals are false).
Therefore the clause does not need to be visited as long as at least one
unassigned literal exist. This implies the following decision procedure:
\begin{itemize}
  \item If all but one literal is false, propagate the remaining literal to be true.
  \item If all literals are false, report UNSAT.
  \item If any literal becomes true, watched literals do not change.
  \item Else replace the literal on the watch list with a remaining unassigned literal.
\end{itemize}

%Consider the solver assigns a value for literal $l$. Instead of looking at
%all clauses and testing whether the clause is falsified by $l$, only clauses
%containing $l$ are checked if $l$ is one of the watched literals of the clause.
This empirical approach was established with the Chaff and zChaff SAT
solvers~\cite{moskewicz2001chaff} and has proven useful in various variants.

\subsection{Remark}
\label{sec:sat-remark}
%
The previous two techniques illustrate basic approaches, but actual SAT
solving research requires decades of development to tune individual SAT solvers.
Memory models and concurrency strategies lead to fundamentally different runtime
behaviors of SAT solvers.

As such, an initial idea to initiate an individual SAT solver specifically designed for
solving problems in differential cryptanalysis was dropped, because development time
is expected too long for a master thesis to be fruitful. As such we focused on popular
and established SAT solvers of the SAT community.

\section{SAT solvers in use}
\label{sec:sat-solvers}
%
\index{lingeling}
In this thesis, we considered several SAT solvers.
They have been selected either by their popularity
or their good results at previous SAT competitions:
\begin{itemize}
  \item MiniSat 2.2.0
  \item CryptoMiniSat versions 4.5.3 and~5
  \item treengeling, lingeling and plingeling, in versions:
    \begin{itemize}
      \item lingeling ats1
      \item lingeling ats1o1
      \item lingeling ats1o2
      \item lingeling ats1o4
      \item lingeling baz
    \end{itemize}
  \item glucose version 4.0 and glucose syrup version 4.0
\end{itemize}

This means the hash collision attacks we implemented have run with
these SAT solvers. The results are discussed in Chapter~\ref{ch:results}
and a more comprehensive list is provided in Appendix~\ref{app:runtimes}.

MiniSat is known as \enquote{Swiss army knife of SAT solving} meaning that
it includes many well-established techniques that can be built upon.
SAT competitions 2009, 2011, 2013 and 2014 included a special MiniSat
\enquote{hack track} where participants are asked to modify MiniSat to prove the
best performance with as little change to the MiniSat codebase as possible.
Even though is not one of the fastest SAT solvers today, it provides
a nice codebase to experiment with.

CryptoMiniSat is a derivative of MiniSat, which was originally modified
for cryptographic problems. It features XOR clauses meaning that
binary clauses of structure $a \oplus b$ could be added and will be resolved
using Gaussian elimination.
%Temporarily development has been given up but most recently it was added again.
Please recognize that our encoding
introduced in Section~\ref{sec:enc-algotocnf} uses equivalence to model
assignment and as such only clauses of structure $r = a \oplus b$ emerge
rendering this feature impractical to use.

Glucose was the gold winner 2011 in the SAT+UNSAT application track.
Modifications of glucose also ranked high throughout the years of SAT competition.
Glucose is a sequential SAT solver whereas glucose syrup is its parallelized
version.

Lingeling is SAT solver developed by Armin Biere. Lingeling has been the
winner of several tracks in the SAT competitions 2011 to 2016. For example,
it has won gold in the SAT+UNSAT application track in 2014.
Lingeling has two siblings: plingeling and treengeling. plingeling is a
parallelized version of lingeling. As such it executes in multiple threads
and shares units and equivalences between those instances. treengeling
is a Cube \& Conquer solver meaning it partitions the problem into many
subproblems and solves them individually.

Lingeling releases ats1o1, ats1o2 and ats1o4 are non-public, experimental
releases of lingeling.
They have been developed in private communication with Armin Biere.
Our main goal was to achieve a separation between two sets of variables.
First, all variables of the first need to be assigned in the best possible
way. Afterwards, the second set of variables is considered.
Specifically variables modelling the differences between the two hash
algorithm instances should constitute the first set as discussed in
Chapter~\ref{ch:enc}.

Lingeling~ats1o1 implements the strategy to guess difference variables first (with Boolean value false) and
usual heuristics apply for all other variables.
Our intermediate results with incomplete CNF files showed a high number of restarts.
Therefore ats1o2 disables backjumping and therefore skips decisions for important variables.
Finally ats1o4 is not expected to distinguish from ats1o2. It only provides further debugging information.

The SAT solvers have generally been run without any special options
and several times, except for
\begin{itemize}
  \item MiniSat was run with \texttt{\textendash{}\textendash{}pre=once} as it is generally recommended to run with the builtin preprocessor.
  \item Lingeling has been generally run with
    \texttt{\textendash{}\textendash{}phase=0} per default and
    \texttt{\textendash{}\textendash{}phase=-1} to prefer false as initial assignment to literals.
    However, lingeling ats1o1 implements this with a more forceful strategy.
\end{itemize}

%Testcases with lingeling have all been run 5 times with various seeds (for reference, the default seed is 0).
%Only the mean runtime value is displayed in the results in Chapter~\ref{ch:results}.

Preprocessing is a difficult topic on its own. Sometimes preprocessing can provide a speedup,
before actually solving the problem, but mostly SAT solvers implement preprocessing strategies
themselves and run them repeatedly when solving the problem. Chapter~\ref{ch:results}
presents runtime results for that issue.


\documentclass{homework}
\course{Math 5522H}
\author{Alex Li}
\input{preamble}

\begin{document}
\maketitle

\begin{inspiration}
I don't think \byline{Warren Buffett}
\end{inspiration}

\section{Terminology}

\begin{problem}
  What is a half-space?  Define convex hull.
  \end{problem}
  \begin{solution}
  A (open) half-space is a set of points in $\C$ is the set of points on one side of a line, i.e. $H=\{z: z\cdot u > b\}$ for a unit vector $u\in\C$, $b\in\Z$

   The convex hull of a set of points is the intersection of all half-spaces containing every point of the set.

   \end{solution}
   \begin{problem}
   Define the Wirtinger partial derivatives $\displaystyle\frac{\partial}{\partial z}$ and $\displaystyle\frac{\partial}{\partial \conj{z}}$.
   \end{problem}
   \begin{solution}
   For a function $f$ with domain $\C = \{x + iy|x, y\in\R\}$, we define the Wirtinger partial derivatives as follows:
   \[\pfrac{}{z} = \frac{1}{2}(\pfrac{}{x} - i\pfrac{}{y})\]
   \[\pfrac{}{\conj{z}} = \frac{1}{2}(\pfrac{}{x} + i\pfrac{}{y})\]
   \end{solution}

   \begin{problem}
     What are the Cauchy-Riemann equations?
     \end{problem}
     We can uniquely represent a complex function of $z = x + iy$ as a sum of two real functons of two variables, $u$ and $v$, to get the equation  $f(z) = u(x, y) + iv(x, y).$
     The Cauchy-Riemann equations are the following
     \begin{solution}
     \[\pfrac{u}{x} = \pfrac{v}{y}\label{cauchy-riemann-x-real}\]
     \[\pfrac{u}{y} = -\pfrac{v}{x}\label{cauchy-riemann-y-real}\]
     \end{solution}
     These two equations are neccessary for differentiability of f and, together with continuity of the partial derivatives, sufficient.

     \section{Numericals}

     \begin{problem}\label{cauchy-riemann-polar}
     What are the Cauchy-Riemann equations\ldots in polar coordinates?
     \end{problem}
     \begin{solution}
     Suppose that $f(re^{i\theta}) = u(r, \theta) + iv(r, \theta)$ is a complex differentiable  function. Let's ignore the origin since $\theta$ isn't even defined, this formula can apply everywhere else. Then differenting along the first coordinate,
     \begin{align*}
     f'(r_0e^{i\theta_0}) &=  \lim_{r\to r_0} \frac{u(r, \theta_0) + iv(r, \theta_0) - u(r_0, \theta_0) - iv(r_0, \theta_0)}{re^{i\theta_0} - r_0e^{i\theta_0}}\\
     &= \frac{1}{e^{i\theta_0}}\lim_{r\to r_0} \frac{u(r, \theta_0) - u(r_0, \theta_0)}{r - r_0} + \frac{i}{e^{i\theta_0}} \lim_{r\to r_0} \frac{v(r, \theta_0) - v(r_0, \theta_0)}{r - r_0}
     \end{align*}
     \begin{align}\label{first_rc_polar}
     f'(r_0e^{i\theta_0}) &= \frac{u_r + iv_r}{e^{i\theta}}
     \end{align}
     Next, we differentiate along the second coordinate, $\theta$.
     \begin{align*}
     f'(r_0e^{i\theta_0}) &=  \lim_{\theta\to \theta_0} \frac{u(r_0, \theta) + iv(r, \theta_0) - u(r_0, \theta_0) - iv(r_0, \theta_0)}{r_0(e^{i\theta} - e^{i\theta_0})}\\
     &= \lim_{\theta\to \theta_0} \frac{u(r_0, \theta) - u(r_0, \theta_0)}{r_0e^{i}(e^\theta - e^{\theta_0})} + i \lim_{\theta\to \theta_0} \frac{v(r, \theta_0) - v(r_0, \theta_0)}{r_0e^{i}(e^\theta - e^{\theta_0})}
     \end{align*}
     We want to compare this to the real partial derivatives in the direction $\theta$:
     \begin{gather*}
     \pfrac{u(r_0, \theta_0)}{\theta} = \lim_{\theta\to \theta_0} \frac{u(r_0, \theta) - u(r_0, \theta_0)}{\theta - \theta_0}\\
     \pfrac{v(r_0, \theta_0)}{\theta} = \lim_{\theta\to \theta_0} \frac{v(r_0, \theta) - v(r_0, \theta_0)}{\theta - \theta_0}
     \end{gather*}
     In the limit, the ratio between these two can be computed by noticing that $\lim_{\theta\to\theta_0}\frac{e^{i\theta} - e^{i\theta_0}}{i\theta - i\theta_0}$ is the derivative of $e^z$ at $i\theta_0$.
     \[\lim_{\theta \to \theta_0} \frac{\theta - \theta_0}{r_0(e^{i\theta} - e^{i\theta_0})} = r_0\lim_{\theta \to \theta_0} \frac{i\theta - i\theta_0}{e^{i\theta} - e^{i\theta_0}}\frac{1}{i} = r_0\times e^{i\theta}\times\frac{1}{i}
     \]
     Thus we see that
     \begin{align}\label{second_rc_polar}
     f'(r_0e^{i\theta_0}) = \frac{v_{\theta_0} - iu_{\theta_0}}{e^{i\theta_0}r_0}
     \end{align}
     Setting real and imaginary parts equal in \ref{first_rc_polar} and \ref{second_rc_polar},
     \begin{align*}
     r_0u_r = v_\theta \\
     r_0v_r = - u_\theta
     \end{align*}
     \end{solution}
     \begin{problem}
       For a natural number $n$ define $f : \mathbb{C} \to \mathbb{C}$ by
         \[
             f(r \cos \theta + i \, r\sin \theta) = r^n \cos \left( n\theta \right) + i \, r^n \sin \left( n\theta \right).
               \]
                 Use \ref{cauchy-riemann-polar} to verify that $f$ satisfies the
                   Cauchy-Riemann equations.
                   \end{problem}
                   \begin{solution}
                   Let $u=r^n\cos(n\theta)$, $v = r^n\sin(n\theta)$, and we compute the 4 partials.
                   \begin{align*}
                   u_r &= nr^{n-1}\cos(n\theta)\\
                   v_r &= nr^{n-1}\sin(n\theta)\\
                   u_\theta &= -nr^{n}\sin(n\theta) = -rv_r\\
                   v_\theta &= nr^{n}\cos(n\theta) = ru_r\\
                   \end{align*}
                   \end{solution}

                   \begin{problem}
                     On a certain domain, define $f(z) = \log \abs{z} + i \cdot \Arg z$
                       and verify that $f$ satisfies the Cauchy-Riemann equations.
                       \end{problem}
                       \begin{solution}
                       Let $z=r\cos{\theta} + r\sin{\theta}$ with $\theta \in (-\pi, \pi)$, so $f(z) = \Log r + i \theta$. We validate the cauchy riemann equations on the domain $\C\setminus(-\infty, 0]$ by first noting that they are differentiable on this domain, then checking them in polar coordinates:
                       \begin{align*}
                       u_r &= \frac{1}{r} &  u_\theta &= 0\\
                       v_r &= 0 & v_\theta &= 1
                       \end{align*}
                       Thus $rv_r = u_\theta$ and $ru_r = v_\theta$
                       \end{solution}

                       \begin{problem}
                       Compute $\displaystyle\frac{\partial}{\partial z} \left( z \conj{z} \right)$.
                       \end{problem}
                       \begin{solution}
                       \begin{align*}
                       \pfrac{}{z} \left( z \conj{z} \right) &= \frac{1}{2}(\pfrac{}{x}(x^2+y^2) - i\pfrac{}{y}(x^2+y^2))\\
                       &= x - iy\\
                       \end{align*}
                       If we pretend $z$ and $\conj z$ are independent and differentiate, we get the same answer:
                       \begin{align*}
                       \pfrac{}{z} \left( z \conj{z} \right) &= \conj{z} = x - iy\\
                       \end{align*}
                       \end{solution}

                       \begin{problem}\label{harmonic-conjugate}
                       Let $u(x,y) = e^y \sin x$.  Find $v(x,y)$ so that $u$ and $v$ satisfy the Cauchy-Riemann equations.  Such a function $v$ is a
                         \textbf{harmonic conjugate} of $u$.
                         \end{problem}
                         \begin{solution}
                         First, let's compute the partial derivatives of $v$ with the Cauchy-Riemann equations.
                         \[u_x = -e^y\cos{x} = v_y\]
                         \[-u_y = -e^y\sin{x} = v_x\]
                         Integrating,
                         \[v = \int v_y dy = -e^y\cos{x} + C(x)  = -e^y\cos{x} + C(y) = \int v_x dx\]
                         So $v = -e^y\cos{x} + C$ for any real number $C$.
                         \end{solution}
                         \section{Exploration}

                         \begin{problem}\label{cayley-transform}Relate the upper half-plane
                           $H = \{ z \in \mathbb{C} : \Imag z > 0 \}$ and the open disk
                             $D = \{ z \in \mathbb{C} : \abs{z} < 1 \}$ by finding $a,b,c,d \in \C$ so that the M\"obius transformation
                               \[f(z) = \frac{az + b}{cz + d}\] yields a bijection $f : H \to D$.
                               \end{problem}
                               \begin{solution}
                               Maybe it will work if the transformation sends the real line to the unit circle, with say $f(0)\mapsto -i$, $f(1)\mapsto 1$, $f(\infty)\mapsto i$. Solving for $a,b,c,d$, we get the following mobius transformation:
                               $f(z) = \frac{iz+1}{z + i}.$

                               % https://en.wikipedia.org/wiki/M%C3%B6bius_transformation#Composition_of_simple_transformations
                               We can check that we can write this mobius tranformation $f(z) = f_4\cdot f_3 \cdot f_2 \cdot f_1(x)$ where $f_1 = z + i$ (mapping $H$ to the half plane above the line with i=1), $f_2 = 1/z$ (mapping $Image f_1$ to the open circle with radius $\frac{1}{2}$ between 0 and $-i$), $f_3 = 2z$ (mapping $Image f_2$ to the circle between 0 and $-2i$) and $f_4 = z + i$ (mapping $Image f_3$ to D).

                               It's easy to see that each of the $f_i$ are bijective, so $f$ must be bijective as well.
                               \end{solution}

                               \begin{problem}\label{harmonic-necessary}Given $u : \mathbb{R}^2 \to \mathbb{R}$, is it always possible to
                                 find a function $v : \mathbb{R}^2 \to \mathbb{R}$ so that $u$ and
                                   $v$ satisfy the Cauchy-Riemann equations?
                                   \end{problem}
                                   \begin{solution}
                                   Obviously not if $u$ isn't differentiable. Let's assume that $u$ is twice differentiable with equal mixed partials. Then we can solve for $v$ like in \ref{harmonic-conjugate}:
                                   \[v = \int u_x dy = -\int u_y dx.\]
                                   Thus $v_{xy} = -u_{yy}$ and $v_{yx} = u_{xx}$, and we need $u$ to satisfy $u_{xx} = -u_{yy}$. If that's true, then integrating $u_xdy$ and $u_ydx$ should give the same answer for $v$, so $u$ will have a harmonic conjugate.
                                   \end{solution}

                                   \begin{problem}
                                     For $p \in \C[x]$, show that the convex hull of the zeroes of $p$
                                       contains the zeroes of $p'$.  This is the \textbf{Gauss-Lucas
                                           theorem}.
                                           \end{problem}
                                           \begin{solution}
                                           By the fundamental theorem of algebra, we can split the polynomial $p$ into linear factors $p = \prod_i(z - r_i).$
                                           Now notice that for functions $f_1(z), f_2(z), ... f_n(z)$, the product rule gives
                                           \begin{align}\label{log-derivative-product-to-sum}
                                           \frac{\frac{d}{dz}\prod_i f_i}{\prod_i f_i} = \frac{\sum_i f_i'\prod_{j\neq i}f_j}{\prod_i f_i} = \sum_i\frac{ f_i'}{f_i}
                                           \end{align}
                                           Using this fact, we can compute
                                           \[
                                           \frac{\frac{dp}{dz}}{p} = \frac{\frac{d}{dz}\prod_i (z - r_i)}{\frac{d}{dz}\prod_i (z - r_i)} = \sum_i \frac{\frac{d}{dz}(z-r_i)}{z - r_i} = \sum_i \frac{1}{z - r_i}
                                           \]

                                           Let's consider an arbitrary half space $H=\{z: z\cdot u > b\}$ for a unit vector $u\in\C$, $b\in\Z$ which contains none of the zeros of $p$. Then for points $z$ in this half space, $(z-r_i)\cdot u = z\cdot u - r_i \cdot u$.
                                           Here $z\cdot u > b$ since $z\in H$, and $r_i \cdot u <= b$ since $r_i\not\in H$, so $(z-r_i)\cdot u > 0$ and thus $\frac{1}{z-r_i}\cdot u > 0$.

                                           By linearity, \[0 < \sum_i \frac{1}{z - r_i} \cdot u  = \frac{\frac{dp}{dz}}{p}\cdot u\] so $\frac{dp}{dz} \neq 0$ on this half space (as $p$ is nonzero). Since the half space was choosen aribtrarily from the half spaces containing none of the zeros of $p$, there are no zeros of $p'$ outside of the convex hull of zeros of $p$.
                                           \end{solution}
                                           \begin{problem}\label{cauchy-riemann-again}
                                             Suppose the smooth function $f : \C \to \C$ satisfies the
                                               Cauchy-Riemann equations.  Does $f'$ also satisfy the Cauchy-Riemann
                                                 equations?
                                                 \end{problem}
                                                 \begin{solution}
                                                 Let $f = u(x,y) + iv(x,y).$
                                                 Taking the derivative of $f$ in the $x$ direction, $f' = u_x + iv_x,$ and so to show $f'$ satisfies the Cauchy Riemann equations that we want to show that $u_{xx}=v_{xy}$ and $u_{xy} = -v_{xx}.$ To do this, notice that since $f$ satisfies the equations, $u_{x} = v_y$ and $u_y=-v_x$. Taking the $x$ derivative of these two equations and using the equality of mixed partials we show what we wanted: namely, that $u_{xx}=v_{xy}$ and $u_{xy} = -v_{xx}.$ So $f'$ satisfies the Cauchy Riemann equations.
                                                 \end{solution}
                                                 \begin{problem}
                                                   Suppose $f : \R^2 \to \R^2$ satisfies the Cauchy-Riemann equations,
                                                     and consider curves $\gamma_1, \gamma_2 : (-1,1) \to \R^2$ passing
                                                       through the origin so that $\gamma_1(0) = \gamma_2(0) = (0,0)$.
                                                         Relate the angle between the curves $\gamma_1$ and $\gamma_2$ to the
                                                           angle between the curves $f \circ \gamma_1$ and $f \circ \gamma_2$.
                                                             (This is related to \ref{tj-versus-jt}.)
                                                             \end{problem}
                                                             \begin{solution}
                                                             To get the angle between the curves, we need to compare the derivatives between the curves at the point (0,0).

                                                             \[\frac{d}{dt} f(\gamma_1(t)) =
                                                             \begin{pmatrix}
                                                             u_x &u_y\\
                                                             v_x &v_y\\
                                                             \end{pmatrix}
                                                             \begin{pmatrix}
                                                             \partial_1\gamma_1\\
                                                             \partial_2\gamma_1\\
                                                             \end{pmatrix}
                                                             =
                                                             \begin{pmatrix}
                                                             u_x & u_y\\\
                                                             -u_y & u_x\\
                                                             \end{pmatrix}
                                                             \begin{pmatrix}
                                                             \partial_1\gamma_1\\
                                                             \partial_2\gamma_1\\
                                                             \end{pmatrix}\]
                                                             We can choose some choice of $R, \theta$ based on the values of $u_x, u_y$ so that we get
                                                             \[\frac{d}{dt} f(\gamma_1(t)) =
                                                             R
                                                             \begin{pmatrix}
                                                             \cos{\theta} & \sin{\theta}\\
                                                             -\sin{\theta} & \cos{\theta}\\
                                                             \end{pmatrix}
                                                             \begin{pmatrix}
                                                             \partial_1\gamma_1\\
                                                             \partial_2\gamma_1\\
                                                             \end{pmatrix}
                                                             \]
                                                             Thus, applying $f$ to $\gamma_1$ will give us a derivative vector pointing in the direction $\arg(\gamma_1) + \theta,$ and similarly applying $f$ to $\gamma_2$ will result in a derivative pointing in the direction $\arg(\gamma_2)+\theta$. Thus the argle between $\gamma_1$ and $\gamma_2$ at the point $(0,0)$ is equal to the angle after applying $f$.
                                                             \end{solution}
                                                             \begin{problem}
                                                               Suppose $f : \C \to \C$ is smooth and holomorphic.  Show that the
                                                                 real part of $f$ is harmonic (cf.~\ref{harmonic-function}).
                                                                 \end{problem}
                                                                 \begin{solution}
                                                                 We can solve similarly to \ref{cauchy-riemann-again}. Let $f=u+iv$, since $f$ is harmonic it satisfies the cauchy riemann equations and thus
                                                                 \begin{align*}
                                                                 u_x=\phantom{-}v_y &\implies u = \phantom{-}\int v_y dx &\implies u_{xx} = \phantom{-}v_{yx}\\
                                                                 u_y=-v_x &\implies u = -\int v_x dy &\implies u_{yy} = -v_{xy}\\
                                                                 \end{align*}
                                                                 Since $f$ is smooth, its mixed partials are equal and thus $u_{yy}=u_{xx}$.
                                                                 \end{solution}
                                                                 \begin{problem}
                                                                   Let's redo \ref{abels-theorem} in the context of complex analysis.
                                                                     Consider a sequence $(a_i)$ of complex numbers so that
                                                                       $\sum_{i=0}^\infty a_i$ converges to $L$.  Then
                                                                         $\lim_{z \to 1^{-}} \sum_{i=0}^\infty a_i z^i = L$ provided
                                                                           $z \to 1$ in a \textbf{Stolz sector}, i.e., suppose there is some
                                                                             $K$ so that $|1-z| \leq K(1-|z|)$.
                                                                             \end{problem}
                                                                             \begin{solution}
                                                                             \begin{lemma}\label{conditional-row-sum-absolute-column-sum}
                                                                             Suppose $\sum_i^\infty a_i,$ $\sum_i^\infty b_i=B,$ and $\sum_{i=0}^\infty\sum_{j=0}^\infty f(i,j) = L$ are convergent, with $f(i, j) = \begin{cases}a_ib_j & i \leq j\\0. \end{cases}$.
                                                                             Then $\sum_{j=0}^\infty\sum_{i=0}^\infty f(i, j)$ converges to L.
                                                                             \end{lemma}
                                                                             \begin{proof}
                                                                             For any $N$, we have the following
                                                                             \begin{align*}
                                                                             |\sum_{j=0}^\infty\sum_{i=0}^\infty f(i,j) - \sum_{i=0}^N\sum_{j=0}^\infty f(i,j)| &=
                                                                             |(\sum_{j=0}^\infty\sum_{i=0}^\infty a_ib_j - \sum_{i=0}^\infty\sum_{j=0}^{i-1} a_ib_j) - (\sum_{i=0}^N a_iB - \sum_{i=0}^N\sum_{j=0}^{i-1}a_ib_j|)\\
                                                                             &= |\sum_{j=0}^\infty\sum_{i=0}^\infty a_ib_j - B\sum_{i=0}^N a_i - \sum_{i=N+1}^\infty \sum_{j=0}^{i-1} a_ib_j|\\
                                                                             &\leq |\sum_{j=0}^\infty b_j\sum_{i=0}^\infty a_i - \sum_{j=0}^\infty b_j\sum_{i=0}^N a_i| +  |\sum_{i=N+1}^\infty a_i\sum_{j=0}^{i-1} b_j|\\
                                                                             &\leq |\sum_{j=0}^\infty \left(b_j\sum_{i=N+1}^\infty a_i\right)| +  |\sum_{i=N+1}^\infty a_i\sum_{j=0}^{i-1} b_j|
                                                                             \end{align*}
                                                                             Let $B'$ be the sup of the absolute value of the maximum partial sum of $b_j$. Then, choose $N$ so that $|\sum_{i=N+1}^\infty a_i| < \frac{\epsilon}{2B'}$. Plugging this into the last equation, we can conclude
                                                                             \begin{align*}
                                                                             |\sum_{j=0}^\infty\sum_{i=0}^\infty a_ib_j - \sum_{i=0}^N\sum_{j=0}^\infty a_ib_j| &\leq
                                                                             |B'\frac{\epsilon}{2B'}| +|\frac{\epsilon}{2B'}B'|   \leq \epsilon
                                                                             \end{align*}
                                                                             So the two sums are equal.
                                                                             \end{proof}
                                                                             With this we move onto the proof of the theorem. First, notice that the condition $|1-z|\leq K(1-|z|)$ implies $|z| <  1$, as if $|z| < 1$ then the RHS is negative (impossible since there is an absolute value). If $|z|=1$ then the RHS may be 0, but the only way the inequality is satisfied is if $z=1$, and that's the limit point.

                                                                             Thus $\sum a_iz^i$ converges absolutely by comparison to the geometric series $max(a_i)|z|^i$.

                                                                             We would like to show that the two values have difference zero. Subtract them, expand, and sum `along the columns instead of along the rows'. Lemma \ref{conditional-row-sum-absolute-column-sum} shows that this operation will not mess anything up.
                                                                             \begin{align*}
                                                                             \sum a_i - \sum a_iz^i &= \sum a_i(1-z^i)\\
                                                                             &= \sum_{i=0}^\infty a_i(1-z)(1 + z + \dots + z^{i-1})\\
                                                                             &= (1 - z)\sum_{j=0}^\infty z^j\sum_{i=j}^\infty a_i
                                                                             \end{align*}
                                                                             It suffices to prove that the absolute value of the difference is 0. Note that since $\sum a_i$ converges, $\forall \epsilon > 0$, we can find a number $N$ so that for all $i\geq N$, $L-\sum_{i=1}^{N-1} a_i = \sum_{i=N}^\infty a_i < \epsilon.$
                                                                             \begin{align*}
                                                                             |\sum a_i - \sum a_iz^i| &=  \left|(1 - z)\sum_{j=0}^\infty z^j\sum_{i=j}^\infty a_i\right|\\
                                                                             &\leq \left|K(1 - |z|)\right|\sum_{j=0}^{N-1} |z|^j\left|\sum_{i=j}^\infty a_n\right| + \left|K(1 - |z|)\right|\sum_{j=N}^\infty |z|^j|\epsilon|\\
                                                                             &\leq \left|K(1 - |z|^{N})(L-\sum_{i=0}^N\sum_{j=0}^N a_i)\right| +
                                                                             |K\epsilon|\\
                                                                             &\leq C_1(N)(1 - |z|^{N}) + C_2\epsilon
                                                                             \end{align*}
                                                                             For some positive constant $C_2$ and constant function of $N$ $C_1(N)$. We can make the second term go to 0 by choosing $N$ to be huge so that $\epsilon$ vanishes, and we can then independently make the first term go to zero by choosing $z$ close enough to 1 that $1-|z|^N$ is close to 0.
                                                                             \end{solution}
                                                                             \section{Prove or Disprove and Salvage if Possible}
                                                                             As always with PODASIPs, many of statements below are incorrect.  For
                                                                             full credit, you must not only salvage these false statements, but
                                                                             also explain why the statement is false (e.g., perhaps by exhibiting a
                                                                             counterexample).

                                                                             \begin{problem}
                                                                               Suppose $f : \R^2 \to \R^2$ and $g : \R^2 \to \R^2$ are smooth (but
                                                                                not necessarily holomorphic when regarded as functions from $\C$ to
                                                                                $\C$).  Then by the chain rule,
                                                                                \[
                                                                                \frac{\partial}{\partial z} \left( f \circ g \right) =
                                                                                \frac{\partial f}{\partial z} \frac{\partial g}{\partial z} + \frac{\partial f}{\partial \conj{z}} \frac{\partial g}{\partial \conj{z}}.
                                                                                \]
                                                                                and similarly
                                                                                \[
                                                                                \frac{\partial}{\partial \conj{z}} \left( f \circ g \right) =
                                                                                \frac{\partial f}{\partial \conj{z}} \frac{\partial g}{\partial \conj{z}} + \frac{\partial f}{\partial z} \frac{\partial g}{\partial z}.
                                                                                \]
                                                                                \end{problem}
                                                                                \begin{solution}
                                                                                Surely not, as this would imply that $\pfrac{f\circ g}{z} = \pfrac{f\circ g}{\conj z}$. Plus, $\pfrac{f}{z}$ doesn't make that much sense since $f$ isn't really a function of $z$ but rather $g(z)$.

                                                                                \begin{align*}
                                                                                \pfrac{f\circ g}{z} &= \frac{1}{2}(\pfrac{}{x} - i\pfrac{}{y})(f\circ g)
                                                                                \end{align*}
                                                                                Define real valued 2 variable functions $u_1, v_1, u_2, v_2$ so that $g(x,y) = u_2(x,y) + iv_2(x,y)$, $f(x, y) = u_1(x, y) + iv_1(x, y)$. Then
                                                                                \begin{align*}
                                                                                (f\circ g)(x+iy) = u_1(u_2(x, y), v_2(x, y)) + iv_1(u_2(x, y), v_2(x, y))
                                                                                \end{align*}
                                                                                Applying the chain rule, we can get the $x$ and $y$ derivative:
                                                                                \begin{align*}
                                                                                \pfrac{f\circ g}{x} &= \pfrac{f}{u_2}\pfrac{u_2}{x} + i\pfrac{f}{v_2}\pfrac{v_2}{x}\\
                                                                                \pfrac{f\circ g}{y} &= \pfrac{f}{u_2}\pfrac{u_2}{y} + i\pfrac{f}{v_2}\pfrac{v_2}{y}
                                                                                \end{align*}
                                                                                Then we can compute the Wirtinger partial derivative $\pfrac{f\circ g}{z}$
                                                                                \begin{align*}
                                                                                \pfrac{f\circ g}{z} &= \frac{1}{2}(\pfrac{f\circ g}{x} - i \pfrac{f\circ g}{y})\\
                                                                                &= \frac{1}{2}(\pfrac{f}{u_2}\pfrac{u_2}{x} + i\pfrac{f}{v_2}\pfrac{v_2}{x} -  i\pfrac{f}{u_2}\pfrac{u_2}{y} + \pfrac{f}{v_2}\pfrac{v_2}{y})\\
                                                                                &= \frac{1}{2}(\pfrac{f}{u_2}(\pfrac{u_2}{x} -i\pfrac{u_2}{y}) + \pfrac{f}{v_2}(i\pfrac{v_2}{x} + \pfrac{v_2}{y}))\\
                                                                                &= \frac{1}{2}(\pfrac{f}{u_2}(\pfrac{u_2}{x} -i\pfrac{u_2}{y}) - i\pfrac{f}{v_2}(\pfrac{v_2}{x} - i\pfrac{v_2}{y}))\\
                                                                                &= \pfrac{f}{u_2}\pfrac{u_2}{z} - i\pfrac{f}{v_2}\pfrac{v_2}{z}
                                                                                \end{align*}
                                                                                Let $w(x, y) = u_2(x, y) + iv_2(x, y).$ Then $u_2 = \frac{1}{2}(w + \conj{w})$ and $iv_2 = \frac{1}{2}(w - \conj{w}).$

                                                                                \begin{align*}
                                                                                \pfrac{f\circ g}{z} &= \frac{1}{2}((\pfrac{f}{w} +\pfrac{f}{\conj{w}})\pfrac{u_2}{z} + (\pfrac{f}{\conj{w}} - \pfrac{f}{w})\pfrac{v_2}{z}\\
                                                                                &= \frac{1}{2}(\pfrac{f}{w}(\pfrac{u_2}{z} - v\pfrac{v_2}{z}) + \pfrac{f}{\conj{w}}(\pfrac{u_2}{z} + \pfrac{v_2}{z}))\\
                                                                                &= \pfrac{f}{w}\pfrac{g}{z} + \pfrac{f}{\conj{w}}\pfrac{\bar{g}}{z}
                                                                                \end{align*}
                                                                                And by symmetry we can argue that
                                                                                \[
                                                                                \pfrac{f\circ g}{\conj{z}} =\pfrac{f}{\conj{w}}\pfrac{\conj{g}}{\conj{z}} + \pfrac{f}{w}\pfrac{g}{\conj{z}}
                                                                                \]
                                                                                \end{solution}

                                                                                \begin{problem}\label{schwarz-reflection-principle}
                                                                                If $f : \C \to \C$ is holomorphic, then $z \mapsto \overline{f(\conj z)}$ is holomorphic.
                                                                                \end{problem}
                                                                                \begin{solution}
                                                                                If $f(x+iy) := u_1(x, y) + iv_1(x, y)$ then $ u_2(x, y) + v_2(x, y) := \conj{f}(x - iy) = u_1(x, -y) - iv_1(x, -y).$ Let's check the equations:
                                                                                \begin{align*}
                                                                                \pfrac{u_2}{x} &= \pfrac{u}{x} &
                                                                                \pfrac{v_2}{x} &= -\pfrac{v}{x}\\
                                                                                \pfrac{u_2}{y} &= -\pfrac{v}{y} &
                                                                                \pfrac{v_2}{y} &= \pfrac{v}{y}\\
                                                                                \end{align*}
                                                                                Using the fact that Cauchy-Riemann equations are satisfied for $f$,
                                                                                \begin{align*}
                                                                                \pfrac{u_2}{x} &= \pfrac{v_2}{y} &
                                                                                \pfrac{u_2}{y} &= -\pfrac{v_2}{y}
                                                                                \end{align*}
                                                                                So the claim is true.
                                                                                \end{solution}

                                                                                \begin{problem}\label{cauchy-riemann-alone-not-sufficient}Suppose $u, v : \mathbb{R}^2 \to \mathbb{R}$ satisfy
                                                                                \[
                                                                                \frac{\partial u}{\partial x}(0,0)=\frac{\partial v}{\partial y}(0,0)
                                                                                \mbox{ and }
                                                                                \frac{\partial u}{\partial y}(0,0)=-\frac{\partial v}{\partial x}(0,0).
                                                                                \]
                                                                                Then $f : \C \to \C$ given by $f(x+iy) = u(x,y) + i \, v(x,y)$ is holomorphic at $0$.
                                                                                \end{problem}
                                                                                \begin{solution}
                                                                                False, define $f(z) = \begin{cases}1&\Re(z) = 0 \text{ or } \Im(z) = 0\\ 0 & \text{otherwise}\end{cases}$. It's not holomorphic at 0 since it's not real differentiable there. On the other hand, if $f$ is real differentiable and satisfies the Cauchy Riemann equations, then it must be complex differentiable.

                                                                                Choose $z = x + iy\in\C$. Then \[f(z) = (f(x + iy) - f(x)) +  (f(x) - f(0)).\] By the MVT we can find points on the line between $x$ to $x+iy$ and the line between $x$ to $0$ with the correct derivative for the functions $u$ and $v$:
                                                                                \begin{align*}
                                                                                u(z) &= xu_x(p_1) + u_y(p_2)y\\
                                                                                v(z) &= xv_x(p_3) +v_y(p_4)y
                                                                                \end{align*}
                                                                                Since the derivatives are all continuous at 0, and the $p_i$ are close to 0, making $z$ closer to zero allows us to find arbitrarily small $\epsilon_i$ so that
                                                                                \begin{align*}
                                                                                u(z) &= (u_x(0)+\epsilon_1)x + (u_y(0)+\epsilon_2)y \\
                                                                                v(z) &= (v_x(0)+\epsilon_3)x + (v_y(0)+\epsilon_4)y &= (u_y(0)+\epsilon_5)x + (u_x(0)+\epsilon_6)y
                                                                                \end{align*}
                                                                                Then we show that the function is complex differentiable by showing the following limit exists:
                                                                                \begin{align*}
                                                                                \lim_{z\to 0} \frac{f(z)}{z} &= \lim \frac{(u_x(0)+\epsilon_1)x + (u_y(0)+\epsilon_2)y + i((u_y(0)+\epsilon_5)x + (u_x(0)+\epsilon_6)y)}{x+ iy}\\
                                                                                &= \lim \frac{u_x(0)(x+iy) + u_y(0)(y-ix) + \epsilon_1x + \epsilon_2y + \epsilon_5x - \epsilon_6y}{x+ iy}\\
                                                                                &= \lim \frac{u_x(0)(x+iy) + iu_y(0)(x+iy)}{x+iy} + \lim_{z\to 0} \frac{(\epsilon_1+\epsilon_5)x}{x+iy} + \lim_{z\to 0}\frac{(\epsilon_2 - \epsilon_6)y}{x+ iy}\\
                                                                                &= u_x(0) + iu_y(0)
                                                                                \end{align*}
                                                                                Where the second limit disappears because $\epsilon$ goes to zero along with $z$.
                                                                                \end{solution}

                                                                                \begin{problem}\label{open-mapping-theorem-preview}
                                                                                There is a nonconstant holomorphic function with constant absolute value.
                                                                                \end{problem}
                                                                                \begin{solution}
                                                                                False, suppose that $f$ is holomorphic with $|f(z)| = c$. Then $f(x+iy) = u(x, y) + iv(x, y)$ and $u^2 + v^2 = c^2.$ This implies that
                                                                                \begin{align}
                                                                                \label{first_abs_omtp} 2uu_x + 2vv_x &= \frac{d}{dx}c^2 = 0\\
                                                                                \label{mid_abs_omtp}2uu_y + 2vv_y &= \frac{d}{dx}c^2 = 0\\
                                                                                \label{second_abs_omtp} 0 &= -2uv_x + 2vu_x \color{purple} \quad\text{ apply CR equations to \ref{mid_abs_omtp}}
                                                                                \end{align}
                                                                                Multiplying \ref{first_abs_omtp} by $v$ and \ref{second_abs_omtp} by $u$,
                                                                                \begin{align*}
                                                                                2uvu_x + 2v^2v_x &= -2u^2v_x + 2vuu_x\\
                                                                                2v^2v_x &= -2u^2v_x\\
                                                                                2(v^2 +u^2) v_x &= 0\\
                                                                                c^2v_x &= 0
                                                                                \end{align*}
                                                                                If $c^2=0$, then $f$ is constant (as $|f(z)|=c=0$ everywhere), and otherwise $v_x$ (and $u_x$ if we do the last step after multiplying \ref{second_abs_omtp} by $-u$ instead) are 0. Using the Cauchy Riemann equations, this implies that all of the derivatives are 0, so $f$ is constant.
                                                                                \end{solution}
                                                                                \begin{problem}\label{schwarzian-derivative}For $f : \mathbb{C} \to \mathbb{C}$, the \textbf{Schwarzian derivative} of $f$ is
                                                                                \[ (Sf)(z)=\left({\frac{f''(z)}{f'(z)}}\right)'-{\frac12}\left(\frac{f''(z)}{f'(z)}\right)^2={\frac {f'''(z)}{f'(z)}}-{\frac32}\left(\frac{f''(z)}{f'(z)}\right)^2. \]
                                                                                If $f$ is a M\"obius transformation, then $(Sf)(z) = 0$.
                                                                                \end{problem}
                                                                                \begin{solution}
                                                                                In the definition, $f$ should be holomorphic as well, so that the notation $f'(z)$ makes sense.

                                                                                Let $f=\frac{az + b}{cz + d},$ and we can try to compute $Sf$. First let's find $f'$ using the quotient rule:
                                                                                \begin{align*}
                                                                                f' &= \frac{(cz + d)a - (az + b)c}{(cz + d)^2}\\
                                                                                &= \frac{da - bc}{(cz + d)^2}
                                                                                \end{align*}
                                                                                When we find the logarithmic derivative, the constant $(da - bc)$ will disappear, making the next computation a bit better:
                                                                                \begin{align*}
                                                                                \frac{f''}{f'} &= \frac{-2(cz + d)}{(cz+d)^4} / \frac{1}{(cz+d)^2}\\
                                                                                &= \frac{-2}{cz + d}
                                                                                \end{align*}
                                                                                For the final term:
                                                                                \begin{align*}
                                                                                (\frac{f''}{f'})' = \frac{-2}{(cz + d)^2}
                                                                                \end{align*}
                                                                                Now we can plug into the equation and quickly see that the value is in fact 0.
                                                                                \begin{align*}
                                                                                Sf(z) &= \left({\frac{f''(z)}{f'(z)}}\right)'-{\frac12}\left(\frac{f''(z)}{f'(z)}\right)^2\\
                                                                                &= \frac{-2}{(cz + d)^2} - \frac{1}{2}(\frac{-2}{cz + d})^2
                                                                                &= 0
                                                                                \end{align*}
                                                                                \end{solution}
                                                                                \end{document}