%!TEX root = ../../PhD_thesis__Edouard_Leurent.tex
\graphicspath{{2-Chapters/5-Chapter/}}

\chapter{Complements on \Cref{chapter:5}}
\section{Proofs}
\label{sec:proofs}
\subsection{Proof of \Cref{prop:bellman-expectation}}
\label{sec:proof-bell-expect}
\begin{proof}
	Thanks to the introduction of the augmented spaces $\ocS, \ocA$ and dynamics $\augmentedtransition$, this proof is the same as that in classical \glspl{MOMDP}.
	\begin{align*}
	\oV^{\budgetedpolicy}(\os) &\eqdef \expectedvalue\left[ \augmentedreturn^{\budgetedpolicy} \condbar \ov{s_0} = \os\right] \\
	&=\sum_{\oa\in\ocA} \probability{\oa_0 = \oa \condbar\ov{s_0} = \os} \expectedvalue\left[ \augmentedreturn^{\budgetedpolicy} \condbar \ov{s_0} = \os, \oa_0 = \oa\right]\\
	&= \sum_{\oa\in\ocA} \budgetedpolicy(\oa | \os) \oQ^{\budgetedpolicy}(\os,\oa).
	\end{align*}

	\begin{align*}
	\oQ^{\budgetedpolicy}(\os, \oa) &\eqdef \expectedvalue\left[\sum_{t=0}^\infty \discountfactor^t \augmentedreward(\os_t, \oa_t)\condbar \ov{s_0} = \os, \ov{a_0} = \oa\right] \\
	&= \augmentedreward(\os, \oa) + \sum_{\os'\in\ocS}\probability{\os_1 = \os' \condbar\ov{s_0} = \os, \ov{a_0} = \oa}\cdot \expectedvalue\left[\sum_{t=1}^\infty \discountfactor^t \augmentedreward(\os_t, \oa_t)\condbar \ov{s_1} = \os'\right] \\
	&= \augmentedreward(\os, \oa) + \discountfactor\sum_{\os'\in\ocS}\augmentedtransition\left(\os' \condbar\os, \oa\right) \expectedvalue\left[\sum_{t=0}^\infty \discountfactor^t \augmentedreward(\os_t, \oa_t) \condbar \ov{s_0} = \os'\right] \\
	&= \augmentedreward(\os, \oa) + \discountfactor\sum_{\os'\in\ocS}\augmentedtransition\left(\os' \condbar\os, \oa\right) \oV^{\budgetedpolicy}(\os').
	\end{align*}

	\paragraph{Contraction of $\abo^{\budgetedpolicy}$.}
	Let $\budgetedpolicy\in\policies, \oQ_1, \oQ_2\in(\Real^2)^{\ocS\ocA}$.
	\begin{align*}
	\forall \os\in\ocS, \oa\in\ocA,\quad \left|\abo^{\budgetedpolicy} \oQ_1(\os,\oa) - \abo^{\budgetedpolicy} \oQ_2(\os,\oa)\right| &= \left|\discountfactor\expectedvalueover{\substack{\os'\sim\augmentedtransition(\os'|\os,\oa) \\ \oa'\sim\budgetedpolicy(\oa'|\os')}} \oQ_1(\os',\oa') - \oQ_2(\os',\oa')\right|\\
	&\leq \discountfactor\left\|\oQ_1-\oQ_2\right\|_\infty.
	\end{align*}
	Hence, $\left\|\abo^{\budgetedpolicy} \oQ_1 - \abo^{\budgetedpolicy} \oQ_2 \right\|_\infty \leq \discountfactor\left\|\oQ_1-\oQ_2\right\|_\infty$

	According to the Banach fixed point theorem \citep{Banach1922}, $\abo^{\budgetedpolicy}$ admits a unique fixed point.
	It can be easily verified that $\oQ^{\budgetedpolicy}$ is indeed this fixed point by combining the two Bellman Expectation equations~\eqref{eq:bellman_expectation}.

\end{proof}

\subsection{Proof of \Cref{thm:bellman-optimality}}
\label{sec:proof-bell-optim}

\begin{proof}
    Let $\os, \oa \in \ocA\times\ocS$. For this proof, we consider potentially non-stationary policies $\budgetedpolicy=(\rho, \budgetedpolicy')$, with $\rho\in\cM(\ocA)$, $\budgetedpolicy'\in\cM(\ocA)^\Natural$. The results will apply to the particular case of stationary optimal policies, when they exist.
    \begin{align}
        \Qr[^\star](\os, \oa) &=  \max_{\rho, \budgetedpolicy'} \Qr[^{\rho, \budgetedpolicy'}](\os', \oa') \label{eq:pthm_def}\\
        &= \max_{\rho, \budgetedpolicy'} \reward(s, a) + \discountfactor \sum_{\os'\in\ocS} \augmentedtransition(\os' | \os, \oa) \Vr[^{\rho, \budgetedpolicy'}](\os') \label{eq:pthm_exp}\\
        &= \reward(s, a) + \discountfactor \sum_{\os'\in\ocS}  \augmentedtransition(\os' | \os, \oa) \max_{\rho, \budgetedpolicy'} \sum_{\oa'\in\ocA} \rho(\oa' | \os')\Qr[^{\budgetedpolicy'}](\os', \oa') \label{eq:pthm_marg}\\
        &= \reward(s, a) + \discountfactor \sum_{\os'\in\ocS}  \augmentedtransition(\os' | \os, \oa) \max_\rho\sum_{\oa'\in\ocA}\rho(\oa' | \os')\max_{\budgetedpolicy'\in\policies_a(\os')}\Qr[^{\budgetedpolicy'}](\os', \oa') \label{eq:pthm_max}\\
        &= \reward(s, a) + \discountfactor \sum_{\os'\in\ocS}  \augmentedtransition(\os' | \os, \oa) \max_\rho\expectedvalueover{\oa'\sim\rho}\Qr[^\star](\os', \oa') \label{eq:pthm_marg_def2}
    \end{align}
    where $\budgetedpolicy = (\rho, \budgetedpolicy')\in\policies_a(\os)$ and $\budgetedpolicy'\in\policies_a(\os')$.

    This follows from:
    \begin{enumerate}
        \item[\eqref{eq:pthm_def}.] Definition of $\oQ^{\star}$.
        \item[\eqref{eq:pthm_exp}.] Bellman Expectation expansion from \Cref{prop:bellman-expectation}.
        \item[\eqref{eq:pthm_marg}.] Marginalisation on $\oa'$.
        \item[\eqref{eq:pthm_max}.] \begin{itemize}
            \item Trivially $\max_{\budgetedpolicy'\in\policies_a(\os')} \sum_{\oa'\in\ocA} \cdot \leq \sum_{\oa'\in\ocA} \max_{ \budgetedpolicy'\in\policies_a(\os)} \cdot$.
            \item Let $\ov{\budgetedpolicy}\in\argmax_{\budgetedpolicy'\in\policies_a(\os')} \Qr[^{\budgetedpolicy'}](\os', \oa')$, then:
            \begin{align*}
                \sum_{\oa'\in\ocA}\rho(\oa'|\os')\max_{\budgetedpolicy'\in\policies_a(\os')}\Qr[^{\budgetedpolicy'}](\os', \oa') &= \sum_{\oa'\in\ocA}\rho(\oa'|\os')\Qr[^{\budgetedpolicy'}](\os', \oa') \\
                &\leq  \max_{\budgetedpolicy'\in\policies_a(\os')} \sum_{\oa'\in\ocA}\rho(\oa'|\os')\Qr[^{\budgetedpolicy'}](\os', \oa').
            \end{align*}
        \end{itemize}
        \item[\eqref{eq:pthm_marg_def2}.] Definition of $\oQ^{\star}$.
    \end{enumerate}

    Moreover, the condition $\budgetedpolicy=(\rho, \budgetedpolicy')\in\policies_a(\os)$ gives
    \begin{equation*}
        \expectedvalueover{\oa'\sim\rho} \Qc[^{\star}](\os, \oa) = \expectedvalueover{\oa'\sim\rho} \Qc[^{\budgetedpolicy'}](\os, \oa) = \Vc[^{\budgetedpolicy}](\os) \leq \budget.
    \end{equation*}

    Consequently, $\budgetedpolicy_\text{greedy}(\cdot; \oQ^{\star})$ belongs to the $\argmax$ of \eqref{eq:pthm_marg_def2}, and in particular:
    \begin{equation*}
        \Qr[^{\star}](\os, \oa) = r(\os, \oa) + \discountfactor \sum_{\os'\in\ocS}  P(\os' | \os, \oa) \expectedvalueover{\oa'\sim\budgetedpolicy_\text{greedy}(\os', \oQ^{\star})} \Qr[^{\star}](\os', \oa').
    \end{equation*}

    The same reasoning can be made for $\Qc[^\star]$ by replacing $\max$ operators by $\min$, and $\policies_a$ by $\policies_r$.
\end{proof}

\subsection{Proof of \Cref{prop:greedy_optimal}}
\label{sec:proof-greedy-optim}
\begin{proof}
    Notice from the definitions of $\abo^{\star}$ and $\abo^{\budgetedpolicy}$ in \eqref{eq:bellman-optimality} and \eqref{eq:bellman_expectation_operator} that $\abo^{\star}$ and $\abo^{\budgetedpolicy_\text{greedy}(\cdot;\oQ^{\star})}$ coincide on $\oQ^{\star}$. Moreover, since $\oQ^{\star} = \abo^{\star}\oQ^{\star}$ by \Cref{thm:bellman-optimality}, we have: $\abo^{\budgetedpolicy_\text{greedy}(\cdot;\oQ^{\star})} \oQ^{\star} = \abo^{\star} \oQ^{\star} = \oQ^{\star}$.
    Hence, $\oQ^{\star}$ is a fixed point of $\abo^{\budgetedpolicy_\text{greedy}(\cdot;\oQ^{\star})}$, and by \Cref{prop:bellman-expectation} it must be equal to $\oQ^{\budgetedpolicy_\text{greedy}(\cdot;\oQ^{\star})}$

    To show the same result for $\oV^{\star}$, notice that
    \begin{equation*}
        \oV^{\budgetedpolicy_\text{greedy}(\oQ^{\star})}(\os) = \expectedvalueover{\oa\sim\budgetedpolicy_\text{greedy}(\oQ^{\star})}\oQ^{\budgetedpolicy_\text{greedy}(\oQ^{\star})}(\os,\oa) = \expectedvalueover{\oa\sim\budgetedpolicy_\text{greedy}(\oQ^{\star})}\oQ^{\star}(\os,\oa).
    \end{equation*}
    By applying the definitions of $\oQ^{\star}$ and $\budgetedpolicy_\text{greedy}$, we recover the definition of $\oV^{\star}$.
\end{proof}

\subsection{Proof of \Cref{thm:contraction}}
\label{sec:proof-contraction}
\begin{proof}
	In the trivial case $|\cA| = 1$, there exits only one policy $\budgetedpolicy$ and $\abo = \abo^\budgetedpolicy$, which is a contraction by \Cref{prop:bellman-expectation}.

	In the general case $|\cA| \geq 2$, we can build the following counter-example.

	Let $(\cS, \cA, P, R_r, R_c)$ be a \gls{BMDP}.
	For any $\epsilon > 0$, we define $\oQ_\epsilon^1$ and $\oQ_\epsilon^2$ as
	\begin{align*}
	\oQ_\epsilon^1(\os,\oa) =
	\begin{cases}
	(0, 0), & \text{if } a = a_0 \\
	\left(\frac{1}{\discount}, \epsilon\right), & \text{if } a \neq a_0
	\end{cases}\\
	\oQ_\epsilon^2(\os,\oa) =
	\begin{cases}
	(0, \epsilon), & \text{if } a = a_0 \\
	\left(\frac{1}{\discount}, 2\epsilon\right), & \text{if } a \neq a_0
	\end{cases}
	\end{align*}
	Then, $\|\oQ_1-\oQ_2\|_\infty = \epsilon$.
	$\oQ_\epsilon^1$ and $\oQ_\epsilon^2$ are represented in \Cref{fig:concavity_example}.

	\begin{figure}[tp]
		\centering
		\includegraphics[width=0.4\textwidth]{img/concavity_example.pdf}
		\caption{Representation of $\oQ_\epsilon^1$ (blue) and $\oQ_\epsilon^2$ (yellow)}
		\label{fig:concavity_example}
	\end{figure}

	But for $\oa=(a,\budgetaction)$ with $\budgetaction = \epsilon$, we have
	\begin{align*}
	\|\abo \oQ_\epsilon^1(\os, \oa) - \abo \oQ_\epsilon^2(\os, \oa)\|_\infty &= \discount\left\|\expectedvalueover{\os'\sim\augmentedtransition(\os'|\os,\oa)} \expectedvalueover{\oa'\sim\budgetedpolicy_\text{greedy}(\oQ^1_\epsilon)}\oQ^1_\epsilon(\os',\oa') - \expectedvalueover{\oa'\sim\budgetedpolicy_\text{greedy}(\oQ^2_\epsilon)}\oQ^2_\epsilon(\os',\oa')\right\|_\infty \\
	&= \discount\left\|\expectedvalueover{\os'\sim\augmentedtransition(\os'|\os,\oa)}\left(\frac{1}{\discount}, \epsilon\right) - (0, \epsilon)\right\|_\infty \\
	&= \discount\frac{1}{\discount} = 1
	\end{align*}
	Hence,
	\begin{align*}
	\|\abo \oQ_\epsilon^1 - \abo \oQ_\epsilon^2\|_\infty &\geq 1 = \frac{1}{\epsilon} \|\oQ_1-\oQ_2\|_\infty
	\end{align*}

	In particular, there does not exist $L>0$ such that
	$$\forall \oQ_1,\oQ_2\in(\Real^2)^{\ocS\ocA}, \|\abo \oQ^1 - \abo \oQ^2\|_\infty \leq L \|\oQ^1 - \oQ^2\|_\infty$$
	In other words, $\abo$ is not a contraction for $\|\cdot\|_\infty$.
\end{proof}

\subsection{Proof of \Cref{thm:contractivity-smooth}}
\label{sec:contraction-with-smooth}

\begin{remark}
	\begin{leftbar}[remarkbar]
	This proof makes use of insights detailed in the proof of \Cref{prop:bftq_pi_hull} (\Cref{sec:proof_pi_hull}), which we recommend the reader to consult first.
	\end{leftbar}
\end{remark}

\begin{proof}
	We now study the contractivity of $\abo^{\star}$ when restricted to the functions of $\cL_{\discountfactor}$ defined as follows:
    \begin{equation}
    \cL_{\discountfactor} = \left\{\begin{array}{cc}
   \oQ\in(\Real^2)^{\ocS\ocA}\text{ s.t. }\exists L<\frac{1}{\discountfactor}-1: \forall \os\in\ocS,\oa_1,\oa_2\in\ocA,   \\
   |\Qr(\os,\oa_1) - \Qr(\os,\oa_2)| \leq L|\Qc(\os,\oa_1) - \Qc(\os,\oa_2)|
    \end{array}\right\}.
    \end{equation}
    That is, for all state $\os$, the set $\oQ(\os, \ocA)$ plot in the $(\Qc,\Qr)$ plane must be the \emph{graph} of a $L$-Lipschitz function, with $L<1/\discountfactor-1$.

    We impose such structure for the following reason: the counter-example presented above prevented contraction because it was a pathological case in which the slope of $\oQ$ can be arbitrary large. As a consequence, when solving $\Qr[^\star]$ such that $\Qc[^\star]=\budget$, a vertical slice of a $\|\cdot\|_\infty$ ball around $\oQ_1$ (which must contain $\oQ_2$) can be arbitrary large as well.

    We denote $\text{Ball}(\oQ,R)$ the ball of centre $\oQ$ and radius $R$ for the $\|\cdot\|_\infty$-norm:
    \begin{equation*}
        \text{Ball}(\oQ,R) = \{\oQ'\in(R^2)^{\ocS\ocA}: \|\oQ-\oQ'\|_\infty \leq R\}.
    \end{equation*}

    We give the three main steps required to show that $\abo^{\star}$ restricted to $\cL_{\discountfactor}$ is a contraction. Given $\oQ^1, \oQ^2\in\cL_{\discountfactor}$, show that:

    \begin{enumerate}
        \item $\oQ^2\in\text{Ball}(\oQ^1,R)\implies\cF^2\in\text{Ball}(\cF^1, R), \forall\os\in\ocS$, where $\cF$ is the top frontier of the convex hull of undominated points, as defined in~\Cref{sec:proof_pi_hull}.
        \item $\oQ\in\cL_{\discountfactor} \implies \cF$ is the graph of a $L$-Lipschitz function, $\forall\os\in\ocS$.
        \item taking the slice $\Qc=\budget$ of a ball $\text{Ball}(\cF,R)$ with $\cF$ $L$-Lipschitz results in an interval on $\Qr$ of range at most $(L+1)R$
    \end{enumerate}

	These three steps will allow us to control $\Qr[^{2\star}] - \Qr[^{1\star}]$ as a function of $R = \|\oQ^2-\oQ^1\|_\infty$.

    \paragraph{Step 1}

    We want to show that if $\oQ^1$ and $\oQ^2$ are close, then $\cF^1$ are $\cF^2$ are close as well in the following sense:
    \begin{align}
        \cF^2\in\text{Ball}(\cF^1, R) &\iff d(\cF^1, \cF^2) \leq R \iff \max_{q^2\in\cF^2}\min_{q^1\in\cF^1}\|q^2-q^1\|_\infty \leq R.
        \label{eq:ball-set}
    \end{align}

    Assume $\oQ^2\in\text{Ball}(\oQ^1,R)$, we show by contradiction that $\cF^2 \in \text{Ball}(\cF^1, R)$. Indeed, assume there exists $q^1\in \cF^1$ such that $\cF^2 \cap \text{Ball}(q^1, R) = \emptyset$. Denote $q^2$ the unique point of $\cF^2$ such that $q^2_c = q^1_c$. By construction of $q^1$, we know that $\|q^1-q^2\|_\infty > R$. There are two possible cases:
    \begin{itemize}
        \item $q^2_r > q^1_r$: this also directly implies that $q^2_r > q^1_r + R$. But $q^2\in\cF^2$, so there exist $q^2_1, q^2_2\in Q^{2},\lambda\in\Real$ such that $q^2 = (1-\lambda)q^2_1 + \lambda q^2_2$. But since $\oQ^2\in \text{Ball}(\oQ^1, R)$, there also exist $q_1^1, q^1_2\in \oQ^1$ such that $\|q^1_1-q^2_1\|_\infty \leq R$ and $\|q^1_2-q^2_2\|_\infty \leq R$, and in particular $q^1_{1r}\geq q^2_{1r}-R$ and $q^1_{2r}\geq q^2_{2r}-R$. But then, the point $q^{1'}=(1-\mu)q^1_1 + \mu q^1_2$ with $\mu=(q^2_c-q^1_{1c})/(q^2_{2c}-q^1_{1c})$ verifies $q^{1'}_c = q^1_c$ and $q^{1'}_r \geq q^2_r - R > q^1_r$ which contradicts the definition of $q_1\in\cF^1$ as defined in \eqref{eq:top-frontier}.
        \item $q^2_r < q^1_r$: then the same reasoning can be applied by simply swapping the indexes 1 and 2.
    \end{itemize}

    % We start by showing this result for $\cC^2(Q^{1-})$ and $\cC^2(Q^{2-})$ as defined in \Cref{sec:proof_pi_hull}:
    % Let $\os\in\ocS$ and $q^2\in\cC^2(Q^{2-})$, $\exists\lambda\in[0,1], \oa_1,\oa_2\in\ocA: q^2 = (1-\lambda)Q^2(\os,\oa_1) + \lambda Q^2(\os,\oa_2)$. Define $q^1 = (1-\lambda)Q^1(\os,\oa_1) + \lambda Q^1(\os,\oa_2)$. Then
    % \begin{align*}
    %     \|q^2-q^1\|_\infty &= \|(1-\lambda)(Q^2(\os,\oa_1) - Q^1(\os,\oa_1)) + \lambda (Q^2(\os,\oa_2) - Q^1(\os,\oa_2))\|_\infty\\
    %     &\leq  (1-\lambda)\|Q^2(\os,\oa_1) - Q^1(\os,\oa_1)\|_\infty + \lambda \|Q^2(\os,\oa_2) - Q^1(\os,\oa_2)\|_\infty\\
    %     &\leq (1-\lambda)R+\lambda R = R
    % \end{align*}

    % It remains to show that when taking the top frontiers of the convex sets $\cC^2(Q^{1-})$ and $\cC^2(Q^{2-})$, they remain at a distance of at most $R$.

    We have shown that $\cF^2 \in \text{Ball}(\cF^1, R)$.
    This is illustrated in \Cref{fig:contraction_lips_hull}: given a function $\oQ^1$, we show the locus $\text{Ball}(\oQ_1,R)$ of $\oQ^2$. We then draw $\cF^1$ the top frontier of the convex hull of $\oQ^1$ and alongside the locus of all possible $\cF^2$, which belong to a ball $\text{Ball}(\cF^1, R)$.

    \begin{figure}[ht]
        \centering
        \includegraphics[trim=7cm 4cm 7cm 4cm, clip, width=0.7\textwidth]{img/contraction_lipschitz.pdf}
        \caption{We represent the range of possible solutions $\Qr[^{2\star}]$ for any $\oQ^2\in\text{Ball}(\oQ^1)$, given $\oQ_1\in\cL_\lambda$}
        \label{fig:contraction_lips_hull}
    \end{figure}

    \paragraph{Step 2}

    We want to show that if $\oQ\in\cL_{\discountfactor}$, $\cF$ is the graph of an $L$-Lipschitz function:
    \begin{equation}
        \label{eq:L-lip-set}
        \forall q^1,q^2\in\cF, |q_r^2-q_r^1| \leq |q_c^2-q_c^1|.
    \end{equation}

    Let $\oQ\in\cL_{\discountfactor}$ and $\os\in\ocS$, $\cF$ the corresponding top frontier of convex hull.
    For all $q^1,q^2\in\cF, \exists \lambda,\mu\in[0,1], q^{11},q^{12},q^{21},q^{22}\in \oQ(\os,\ocA)$ such that $q^1 = (1-\lambda)q^{11} + \lambda q^{12}$ and $q^2 = (1-\mu)q^{21} + \mu q^{22}$.
    Without loss of generality, we can assume $q_c^{11}\leq q_c^{12}$ and $q_c^{21}\leq q_c^{22}$. We also consider the worst case in terms of maximum $q_r$ deviation: $q_c^{12} \leq q_c^{21}$.
    Then the maximum increment $q_r^2-q_r^{1}$ is:
    \begin{align*}
        \|q^2_r-q^{1}_r\| &\leq \|q^{12}_r-q^{1}_r\| + \|q^{21}_r-q^{12}_r\| + \|q^{2}_r-q^{21}_r\| \\
        &= (1-\lambda)\|q^{12}_r-q^{11}_r\| + \|q^{21}_r-q^{12}_r\| + \mu\|q^{22}_r-q^{21}_r\| \\
        &\leq (1-\lambda)L\|q^{12}_c-q^{11}_c\| + L\|q^{21}_c-q^{12}_c\| + \mu L\|q^{22}_c-q^{21}_c\| \\
        &= L\|q^{12}_c-q^{1}_c\| + L\|q^{21}_c-q^{12}_c\| + L\|q^{2}_c-q^{21}_c\|\\
        &= L\|q^{2}_c-q^{1}_c\|.
    \end{align*}

    This can also be seen in \Cref{fig:contraction_lips_hull}: the maximum slope of the $\cF^1$ is lower than the maximum slope between two points of $\oQ^1$.

    \paragraph{Step 3}

    Let $\cF_1$ be a L-Lipschitz set as defined in \eqref{eq:L-lip-set}, and consider a ball $\text{Ball}(\cF_1,R)$ around it as defined in \eqref{eq:ball-set}.

    We want to bound the optimal reward value $\Qr[^{2\star}]$ under constraint $\Qc[^{2\star}] = \budget$ (regular case in \Cref{sec:proof_pi_hull} where the constraint is saturated), for any $\cF^2\in\text{Ball}(\cF_1,R)$. This quantity is represented as a red double-ended arrow in \Cref{fig:contraction_lips_hull}.

    Because we are only interested in what happens locally at $\Qc=\budget$, we can zoom in on \Cref{fig:contraction_lips_hull} and only consider a thin $\epsilon$-section around $\budget$. In the limit $\epsilon\rightarrow 0$, this section becomes the tangent to $\cF^1$ at $\Qc[^1]=\budget$. It is represented in \Cref{fig:contraction_lips_hull_slope}, from which we derive a geometrical proof:
    \begin{figure}[ht]
        \centering
        \includegraphics[trim=2cm 1cm 2cm 1cm, clip, width=0.7\textwidth]{img/contraction_lipschitz_slope.pdf}
        \caption{We represent a section $[\budget-\epsilon, \budget+\epsilon]$ of $\cF^1$ and $\text{Ball}(\cF^1, R)$. We want to bound the range of $\Qr[^{2\star}].$}
        \label{fig:contraction_lips_hull_slope}
    \end{figure}
    \begin{align*}
        \Delta \Qr[^{2\star}] &= b + c &\\
        & \leq La + c & \text{($\cF^1$ $L$-Lipschitz)}\\
        &= 2LR+2R = 2R(L+1).
    \end{align*}
    Hence,
    \begin{equation*}
        | \Qr[^{2\star}] - \Qr[^{1\star}]| \leq \frac{\Delta \Qr[^{2\star}]}{2} = R(L+1)
    \end{equation*}
    and $\Qc[^{1\star}] = \Qc[^{2\star}] = \budget$.
    Consequently, $ \|\oQ[^{2\star}] - \oQ[^{1\star}]\|_\infty \leq (L+1)R$.

    Finally, consider the edge case in \Cref{sec:proof_pi_hull}: the constraint is not active, and the optimal value is simply $\argmax_{q\in\cF} q^r$. In particular, since we showed that $\cF^2\in \text{Ball}(\cF^1, R)$, and since $\oQ[^{2\star}]\in \cF^2$, there exist $q^1\in \cF^1: \|\oQ[^{2\star}]-q^1\|_\infty\leq R$ and in particular $\oQ[^{1\star}]_r \geq q^1_r \geq \oQ[^{2\star}]_r - R$. Reciprocally, by the same reasoning, $\Qr[^{2\star}] \geq \Qr[^{1\star}] - R$. Hence, we have that $| \Qr[^{2\star}] - \Qr[^{1\star}]| \leq R \leq R(L+1).$

    \paragraph{Wrapping it up}

    We have shown that for any $\oQ^1,\oQ^2\in\cL_{\discountfactor}$,
    and all $\os\in\ocS$, $\cF^2\in\text{Ball}(\cF^1,\|\oQ^2-\oQ^1\|_\infty)$ and $\cF^1$ is
    the graph of a $L$-Lipschitz function with $L<1/\discountfactor - 1$.
    Moreover, the solutions of $\budgetedpolicy_\text{greedy}(\oQ^1)$ and $\budgetedpolicy_\text{greedy}(\oQ^2)$ at
    $\os$ are such that $ \|\oQ[^{2\star}] - \oQ[^{1\star}]\|_\infty \leq (L+1)\|\oQ^2-\oQ^1\|_\infty$.

    Hence, for all $\oa$,
    \begin{align*}
        \|\abo^{\star}\oQ^1(\os, \oa) - &\abo^{\star}\oQ^2(\os, \oa)\|_\infty \\
        &= \discountfactor\left\|\expectedvalueover{\os'\sim\augmentedtransition(\os'|\os,\oa)}
        \expectedvalueover{\oa'\sim\budgetedpolicy_\text{greedy}(\oQ^1)}\oQ^1(\os',\oa') -
        \expectedvalueover{\oa'\sim\budgetedpolicy_\text{greedy}(\oQ^2)}\oQ^2(\os',\oa')\right\|_\infty \\
        &= \discountfactor\left\|\oQ[^{2\star}] - \oQ[^{1\star}]\right\|_\infty \\
        &\leq \discountfactor(L+1)\|\oQ^2-\oQ^1\|_\infty.
    \end{align*}
    Taking the sup on $\ocS\ocA$,
    \begin{equation*}
        \|\abo^{\star}\oQ^1 - \abo^{\star}\oQ^2\|_\infty \leq \discountfactor(L+1)\|\oQ^1-\oQ^2\|_\infty
    \end{equation*}
    with $\discountfactor(L+1) < 1$.
    As a conclusion, $\abo^{\star}$ is a $\discountfactor(L+1)$-contraction on $\cL_{\discountfactor}$.
\end{proof}

%\subsection{\Cref{lemma:concavity}}

%\begin{proof}. Let $s,s'\in\cS, a\in\cA$.
%We first prove those results for $V_r^{\star}(s', \cdot)$

%\textbf{Non-decreasing}

%Consider $\beta_a^1 \leq \beta_a^2 \in \cB$.
%Any policy that satisfies the budget $\beta_a^1$ in $s'$ also satisfies $\beta_a^2$, so $\Pi_c(s', \beta_a^1) \subset \Pi_c(s', \beta_a^2)$. Hence, by taking the max over policies, $V_r^{\star}(s', \beta_a^1) \leq V_r^{\star}(s', \beta_a^2)$.
%Hence, $V_r^{\star}(s', \cdot)$ is non-decreasing.

%\textbf{Concave}

%By contradiction: assume that $V_r^{\star}(s', \cdot)$ is not concave, \ie there exist $\beta^1 < \beta^2\in \cB$ and $p\in(0, 1)$ such that $\beta^3 = (1-p)\beta^1 + p\beta^2$ verifies: $V_r^{\star}(s', \beta^3) < (1-p)V_r^{\star}(s', \beta^1) + pV_r^{\star}(s',\beta^2)$. By definition of $V^{\star}$, there must be $\pi_1,\pi_2\in\Pi^{\star}$ such that $V^{\star}(s', \beta^1) = V^{\pi_1}(s', \beta^1)$ and $V^{\star}(s', \beta^2) = V^{\pi_2}(s', \beta^2)$.

%Define $\pi = (1-p)(\pi_1(\cdot, \beta^1), \pi_1) + p(\pi_2(\cdot, \beta^2), \pi_2)$. By linearity of $V^\pi$ with respect to $\pi$, we have that $V_c^\pi(s', \beta^3) = (1-p)V_c^{\pi_1}(s', \beta^1) + pV_c^{\pi_2}(s', \beta^2) \leq (1-p)\beta^1 + p\beta^2 = \beta^3$ since $\pi_1, \pi_2\in\Pi^{\star}(s')\subset\Pi_a(s')$, so $\pi$ respects the budget $\beta^3$. Moreover, we also have $V_r^\pi(s', \beta^3) = (1-p)V_r^{\pi_1}(s', \beta^1) + pV_r^{\pi_2}(s', \beta^2) > V_r^{\star}(s', \beta^3)$, which contradicts the definition of $V_r^{\star}$.

%Consequently, $V_r^{\star}(s', \cdot)$ is non-decreasing and concave. By \eqref{eq:bellman_expectation_Q} we see that $Q_r^{\star}(s,a,\cdot) = R_r(s,a) + \discount\expectedvalueover{s'}V_r^{\star}(s', \cdot)$  is too.

%\end{proof}

%\subsection{\Cref{lemma:tau_concavity}}

%\subsection{\Cref{lemma:pi_hull}}

%\td

%\begin{proof}
%If the estimates $q^c_0, q^c_1$ are accurate, then by construction and linearity of the expectation, the returned mixture policy has an expected total cost of $\expectedvalueover{a, \beta_a \sim\pi_\text{greedy}}Q_c(s, a, \beta_a) = \beta$ as desired in \eqref{eq:pi_greedy_constraint}. Because the $Q_r(s,a,\cdot)$ is concave and under its tangents, this mixture must have the largest $Q_r$ possible as required in \eqref{eq:pi_greedy_reward}. The special case of a tie $q_r^0 = q_r^1$ is considered, where we do minimise $Q_c$ as required in \eqref{eq:pi_greedy_cost}.
%\end{proof}

\subsection{Proof of \Cref{prop:bftq_pi_hull}}
\label{sec:proof_pi_hull}
\begin{definition}
	\begin{leftbar}[defnbar]
	Let $A$ be a set, and $f$ a function defined on $A$. We define

	\begin{itemize}
		\item the convex hull of $A$: $\cC(A) = \{\sum_{i=1}^p \lambda_i a_i: a_i\in A, \lambda_i\in\Real^+, \sum_{i=1}^p \lambda_i = 1, p\in\Natural\}$;
		\item the convex edges of $A$: $\cC^2(A) = \{\lambda a_1 + (1-\lambda)a_2: a_1, a_2\in A, \lambda\in[0, 1]\}$;
		\item Dirac distributions of $A$: $\dirac(A) = \{\dirac(a-a_0): a_0\in A\}$;
		\item the image of $A$ by $f$: $f(A) = \{f(a): a\in A\}$.
	\end{itemize}
\end{leftbar}
\end{definition}

\begin{proof}
Let $\os=(s,\budget)\in\ocS$ and $\oQ\in(\Real^2)^{\ocS\ocA}$. We recall the definition of $\budgetedpolicy_\text{greedy}$:
    \begin{subequations}
        \begin{align}
            \budgetedpolicy_\text{greedy}(\oa|\os; \oQ) &\in \argmin_{\rho\in\policies_r^{\oQ}} \expectedvalueover{\oa\sim\rho}\Qc(\os, \oa) \tag{\ref{eq:pi_greedy_cost}}\\
            \text{where }\quad\policies_r^{\oQ} = &\argmax_{\rho\in\cM(\ocA)} \expectedvalueover{\oa\sim\rho} \Qr(\os, \oa) \tag{\ref{eq:pi_greedy_reward}}\\
            & \text{ s.t. }  \expectedvalueover{\oa\sim\rho} \Qc(\os, \oa) \leq \budget \tag{\ref{eq:pi_greedy_constraint}}
        \end{align}
    \end{subequations}

    Note that any policy in the $\argmin$ in \eqref{eq:pi_greedy_cost} is suitable to compute $\abo^{\star}$.
    We first reduce the set of candidate optimal policies.
    Consider the problem described in \eqref{eq:pi_greedy_reward},\eqref{eq:pi_greedy_constraint}: it can be seen as a single-step \gls{CMDP} problem with reward $\reward=\Qr$ and cost $\constraint=\Qc$. By \citep[Theorem 4.4][]{Beutler1985}, we know that the solutions are mixtures of two deterministic policies. Hence, we can replace $\cM(\cA)$ by $\cC^2(\dirac(\ocA))$ in \eqref{eq:pi_greedy_reward}.

    Moreover, remark that:
    \begin{align*}
        \{\expectedvalueover{\oa\sim\rho} \oQ(\os,\oa):& \rho\in \cC^2(\dirac(\ocA))\} \\
        &= \{\expectedvalueover{\oa\sim\rho} \oQ(\os,\oa): \rho=(1-\lambda)\dirac(\oa-\oa_1)+\lambda\dirac(\oa-\oa_2), \oa_1,\oa_2\in\ocA, \lambda\in[0,1]\} \\
        &= \{(1-\lambda)\oQ(\os, \oa_1)+\lambda \oQ(\os, \oa_2), \oa_1,\oa_2\in\ocA, \lambda\in[0,1]\} \\
        &= \cC^2(\oQ(\os,\ocA))\}.
    \end{align*}

    Hence, the problem \eqref{eq:pi_greedy_reward}, \eqref{eq:pi_greedy_constraint} has become:
    \begin{equation*}
        \tilde{\policies}^{\Qr} = \argmax_{(q_r, q_c)\in\cC^2(\oQ(\os, \ocA))} q_r \quad\text{ s.t. }\quad q_c \leq \budget
    \end{equation*}
    and the solution of $\budgetedpolicy_\text{greedy}$ is $q^{\star}=\argmin_{q\in\tilde{\policies}^{\Qr}} q_c$.

    The original problem in the space of actions $\ocA$ is now expressed in the space of values $\oQ(\os, \ocA)$ (which is why we use $=$ instead of $\in$ before $\argmin$ here).

    We further restrict the search space of $q^{\star}$ following two observations:
    \begin{enumerate}
        \item $q^{\star}$ belongs to the \emph{undominated} points $\cC^2(\oQ^-)$:
        \begin{align}
            \label{eq:q_minus_undominated}
            \oQ^+ &= \{(q_c, q_r): q_c > q_c^{\pm} = \min_{q^+} q_c^+\text{ s.t. }q^+\in\argmax_{q\in \oQ(\os,\ocA)} q_r\}\\
            \oQ^- &= \oQ(\os,\ocA) \setminus \oQ^+.
        \end{align}
        Denote $q^{\star}$ = $(1-\lambda) q^1 + \lambda q^2$, with $q^1, q^2\in \oQ(\os,\ocA)$. There are three possible cases:
        \begin{enumerate}
            \item $q^1, q^2 \not\in \oQ^-$. Then $q_c^{\star} = (1-\lambda) q^1_c + \lambda q^2_c > q_c^{\pm}$. But then $q_c^{\pm} < q_c^{\star} \leq \budget$ so $q^{\pm}\in\tilde{\policies}^{\Qr}$ with a strictly lower $q_c$ than $q^{\star}$, which contradicts the $\argmin$.
            \item $q^1\in \oQ^-, q^2 \not\in \oQ^-$. But then consider the mixture $q^\top = (1-\lambda) q^1 + \lambda q^\pm$. Since $q_r^{\pm} \geq q_r^{2}$ and $q_r^{\pm} < q_r^{2}$, we also have $q^\top_r \geq q_r^{\star}$ and $q^\top_c < q_c^{\star}$, which also contradicts the $\argmin$.
            \item $q^1,q^2\in \oQ^-$ is the only remaining possibility.
        \end{enumerate}
        \item $q^{\star}$ belongs to the \emph{top frontier} $\cF$:
        \begin{equation}
            \label{eq:top-frontier}
            \cF_{\oQ} = \{q\in \cC^2(\oQ^-): \not\exists q'\in \cC^2(\oQ^-): q_c=q_c'\text{ and }q_r<q_r'\}.
        \end{equation}
        Trivially, otherwise q' would be a better candidate than $q^{\star}$.
    \end{enumerate}

    Let us characterise this frontier $\cF$. It is both:
    \begin{enumerate}
        \item the \emph{graph of a non-decreasing function}: $\forall q^1, q^2\in\cF$ such that $q_c^1\leq q_c^2$ then $q_r^1\leq q_r^2$.\\
        By contradiction, if we had $q_r^1 > q_r^2$, we could define $q^\top = (1-\lambda)q^1 + \lambda q^\pm$ where $q^\pm$ is the dominant point as defined in \eqref{eq:q_minus_undominated}. By choosing $\lambda=(q^2_c-q^1_c)/(q^\pm_c-q^1_c)$ such that $q^\top_c = q_c^2$, then since $q_r^\pm \geq q_r^1 > q_r 2$ we also have $q^\top_r > q_r^2$ which contradicts $q^2\in\cF$.
        \item the \emph{graph of a concave function}: $\forall q^1, q^2, q^3\in\cF$ such that $q_c^1\leq q_c^2 \leq q_c^3$ with $\lambda$ such that $q^2_c = (1-\lambda)q^1_c + \lambda q^3_c$, then $q_r^2 \geq (1-\lambda)q_r^1 + \lambda q_r^3$.\\
        Trivially, otherwise the point $q^\top = (1-\lambda)q^1 + \lambda q^3$ would verify $q^\top_c=q^2_c$ and $q^\top_r > q^2_r$, which would contradict $q^2 \in\cF$.
    \end{enumerate}

    We denote $\cF_{\oQ} = \cF \cap \oQ$. Clearly, $q^{\star}\in\cC^2(\cF_{\oQ})$: let $q^1, q^2\in \oQ^-$ such that $q^{\star} = (1-\lambda)q^1 + \lambda q^2$. First, $q^1, q^2\in \oQ^-\subset\cC^2(\oQ^-)$. Then, by contradiction, if there existed $q^{1'}$ or $q^{2'}$ with equal $q_c$ and strictly higher $q_r$, again we could build an admissible mixture $q^{\top}=(1-\lambda)q^{1'}  + \lambda q^{2'}$ strictly better than $q^{\star}$.

    $q^{\star}$ can be written as $q^{\star} = (1-\lambda)q^1 + \lambda q^2$ with $q^1, q^2\in\cF_{\oQ}$ and, without loss of generality, $q^1_c \leq q^2_c$.

    \paragraph{Regular case}

    There exists $q^0\in\cF_{\oQ}$ such that $q^0_c \geq \budget$. Then $q^1$ and $q^2$ must flank the budget: $q_c^1 \leq \budget \leq q_c^2$. Indeed, by contradiction, if $q_c^2 \geq q_c^1 > \budget$ then $q_c^{\star} > \budget$ which contradicts $\policies_r^{\oQ}$. Conversely, if $q_c^1 \leq q_c^2 < \budget$ then $q^{\star} < \budget \leq q^0_c$, which would make $q^{\star}$ a worse candidate than $q^\top=(1-\lambda)q^{\star} + \lambda q^0$ when $\lambda$ is chosen such that $q_c^\top=\budget$, and contradict $\policies_r^{\oQ}$ again.

    Because $\cF$ is the graph of a non-decreasing function, $\lambda$ should be as high as possible, as long as the budget $q^{\star}\leq\budget$ is respected. We reach the highest $q_r^{\star}$ when $q^{\star}_c=\budget$, that is: $\lambda=(\budget-q_c^1)/(q_c^2-q_c^1)$.

    It remains to show that $q^1$ and $q^2$ are two successive points in $\cF_{\oQ}$: $\not\exists q\in\cF_{\oQ}\setminus\{q^1, q^2\}: q^1_c \leq q_c \leq q^2_c$. Otherwise, as $\cF$ is the graph of a concave function, we would have $q_r \geq (1-\mu)q_r^1 + \mu q_r^2$. $q_r$ cannot be strictly greater than $(1-\mu)q_r^1 + \mu q_r^2$ which would contradict $q^{\star}$, but it can still be equal, which means the tree points $q, q^1, q^2$ are aligned. In fact, every points aligned with $q^1$ and $q^2$ can also be used to construct mixtures resulting in $q^{\star}$, but among these solutions we can still choose $q^1$ and $q^2$ as the two points in $\cF_{\oQ}$ closest to $q^{\star}$.

    \paragraph{Edge case}

    $\forall q\in\cF_{\oQ}, q_c < \budget$. Then  $q^{\star} =  \argmax_{q\in\cF} q_r = q^\pm =  \argmax_{q\in \oQ^-} q_r$.
\end{proof}

%\begin{proof}
%First, a straightforward proof by induction shows that for all $k\in\Natural$, $Q_k$ computed at iteration $k$ of either \Cref{alg:bvi} or \Cref{alg:bftq} is concave non-decreasing with respect to $\beta_a$: the initialisation is trivial from $Q_0 = 0$, and the heredity stems from \Cref{lemma:tau_concavity}.
%\end{proof}

% \subsection{Decomposition Lemma}

% \begin{lemma}
%     For any sequence real valued functions $f_1,\ldots,f_n$ and any real number $c$, we have
%     \[
%         \begin{array}{lcl}
%             \underbrace{\max\limits_{\sum_i x_i \leq c}\sum_j f_j(x_j)}_{(a)} & \quad{}=\quad{} & \underbrace{\max\limits_{\sum_i c_i \leq c}\left(\sum_j\max\limits_{x\leq c_j} f_j(x)\right)}_{(b)}\\
%         \end{array}
%     \]
% \end{lemma}

% \begin{proof}
%     Let us first show that $(a)\leq(b)$.
%     By definition of the maximum on a set, for any $f_j$ and any $c_j$ we have
%     $\max\limits_{x\leq c_j} f_j(x) \geq f_j(c_j)$.
%     Hence, by replacing these terms in $(b)$ we get:
%       \[
%     \begin{array}{lcl}
%         \max\limits_{\sum_i c_i \leq c} \sum_j f_j(c_j) & \quad{}\leq\quad{} & \max\limits_{\sum_i c_i\leq c}\left(\sum_j \max\limits_{x_j\leq c_j} f_j(x_j)\right)\\
%     \end{array}
%     \]
%     The left hand side of this inequality is just a rewriting of $(a)$ with different dummy variables names.

%     Let us show now that $(a) \geq (b)$.
%     Let $\hat{x}_1,\ldots,\hat{x}_n, \hat{c}_1, \ldots \hat{c}_n$ be a realisation (argmax) of $(b)$.
%     By definition of $(b)$'s feasible set, we have $\sum_i\hat{c}_i \leq c$ and for any $i$: $\hat{x}_i\leq \hat{c}_i$.
%     Because $\sum_i\hat{x}_i\leq \sum_i\hat{c}_i \leq c$, the tuple $(\hat{x}_1, \ldots \hat{x}_n)$ is also a feasible value for $(a)$. And, by definition of the maximum on a set: $(a) = \max\limits_{\sum_i x_i \leq c} \sum_j f_j(x_j) \geq \sum_j f_j(\hat{x}_j) = (b)$.
% \end{proof}

%\section{Risk-Sensitive Exploration}
%\label{sec:risk-sensitive-supp}
%We recall the Risk-Sensitive Exploration in %\Cref{alg:risk-sensitive-exploration}:
%\input{source/risk-sensitive-explo-pseudo-code.tex}


\section*{Maneouver study}
\addcontentsline{toc}{section}{Maneouver study}
The higher the velocity is, the bigger the radius - this is, the greater the increase of altitude - given that the wing's flaps cannot deflect enough nor can support as much force to reduce the radius as to the same menauver for a lower velocity.

Note that the centripetal force follows:
\[
F=\frac{mV^2}{R}
\]
Where \textit{m}, \textit{V} and \textit{R} are the aircraft's mass (in kg), velocity (in m/s) and radius (in m) respectively. \\

The only angles affected are gamma $\gamma$ and mu $\mu$ if performed flawlessly. The former is the angle formed between the \textit{x} axis of the horizontal set and the \textit{x} axis of the wind set, while the latter is the one formed between the \textit{y} axis of the same sets. Note that for the whole maneuver the body set remains consistent with the wind set.\\
Figure \ref{fig:paramEvolution} shows how these change in the different phases of the Immelmann turn\\

\includegraphics[width=\linewidth]{../matlab/paramEvolution.jpg}
\captionof{figure}{Angles evolution over time}
\label{fig:paramEvolution}
\vspace{0.5cm}

Note that for $\gamma$ the evolution is perfectly defined, as the increment of said angle needs to be positive in order to increase the flight altitude. Conversely, for $\mu$ the pilot could choose to rotate in the opposite direction thus changing $\mu$ from 0 to $-\pi$ instead - note that $\pi$ and $-\pi$ are equivalent -, and the result would be virtually the same.\\
Furthermore, in order to describe a perfect semicircle the angular speed $\dot{\gamma}$ needs to remain constant. This results in the necessary condition of gamma's evolution to be a straight slope. For $\mu$, however, the pilot could also perform the roll at non-constant rotating speed without this having an impact on the maneuver performance. This would be reflected as a curve on mu's temporal evolution, instead of a straight line.\\

All these
For the following computations, some values will be needed in order to numerically solve the Ordinal Differential Equation systems. The ENAERT T-35 Pillán, a Chilean military small aircraft and its data \cite{jane1969jane} have been taken as references.\\

\begin{center}
	\includegraphics[width=0.9\linewidth]{figures/pillan}
	\vspace{0.5cm}
	\captionof{figure}{ENAERT T-35 Pillán. Extracted from \cite{defensa.com_2018}}\vspace{0.25cm}
\end{center}

The used information is:

\begin{center}
\begin{tabular}{|l|l||l|l|}\hline
	 & Value &   & Value\\ \hline \hline
	Mass & 1.300 kg& Wingspan & 8.84 m\\ \hline
	Airfoil & 63$_3$-414 & Wing area & 13.69 m$^2$ \\ \hline
	Stall s. & 31.94 m/s &  Cruise s. & 70.84 m/s\\ \hline
	Thrust &224 kW & Air den. & 1.225 kg/m$^3$ \\ \hline
	\multicolumn{4}{|c|}{$C_L=12\alpha+0.33$}\\ \hline
\end{tabular}
\end{center}

The selected initial speed is of 70 m/s as the third phase of the maneuver requires a few less meters per second in order to be performed safely.

For the totality of the maneuver, the fixated parameters are the gas control lever ($\pi$) and the elevator ($\alpha$), which are set as constants at 1.5 kN and 0.3 rad respectively. From the remaining variables, the position coordinates evolution ($\Dot{x}$ and $\Dot{z}$) are always left as dependant in order to be solved by the ODE solver. With the obtained results, the trajectory is plot - see the following section Trajectory analysis, where both of them are studied in moer depth -.

\begin{equation} \label{eq:lift}
	L=\frac{1}{2}S\rho V(t)^2 C_L(\alpha)
\end{equation}
\begin{equation} \label{eq:drag}
	D=\frac{1}{2}S\rho V(t)^2 C_D(\alpha)
\end{equation}

\subsubsection*{Cruise flight}
\addcontentsline{toc}{subsection}{Cruise flight}
For the first and last phases, both cruise flight, the remaining variable (not counting $\Dot{x}$ and $\Dot{z}$) is velocity (V) alone.\\
As fixed thrust and elevator deflection are imposed, there are virtually no temporal changes affecting velocity. This results in constant speed or cruise flight, and a simple slope regarding the x-axis position. When computing both the lift and drag forces, these must be constant too - see equations \ref{eq:lift} and \ref{eq:drag} -.\\

Velocity being constant results in both lift and drag being constant too.
\begin{center}
	\includegraphics[width=\linewidth]{../matlab/1/liftdrag.jpg}
	\vspace{0.5cm}
	\captionof{figure}{Radius dependance on V and $\alpha$t. Own elaboration.}\vspace{0.25cm}
\end{center}

In regards of the general overview, cruise flight is quite straight forward, as both the speed and vertical position remain constant, and as velocity is linealry dependant on time, the displacement of the aricraft is linear as well.
\begin{center}
	\includegraphics[width=\linewidth]{../matlab/1/1traj.jpg}
	\vspace{0.5cm}
	\captionof{figure}{Radius dependance on V and $\alpha$t. Own elaboration.}\vspace{0.25cm}
\end{center}

For the total time, the following integration from the first equation can be done:
\begin{align*}
	dt =& \frac{m}{T - D} dV\\
	\int dt= & \int \frac{m}{T - D} dV
	\intertext{Subsituting drag's definition from equation \ref{eq:drag} and operating for lineal aeordynamics,}
	= & \int_{V_0}^{V} \frac{m}{T - \frac{1}{2}\rho S V^2 (C_{D_0}+kC_L^2)} dV\\
	t_1=& \frac{m}{T - \frac{1}{2}\rho S V^2 (C_{D_0}+kC_L^2)} (V-V_0)
\end{align*}

\subsubsection*{Half loop}
\addcontentsline{toc}{subsection}{Half loop}
In order to work with the half loop part of the maneuver, some preliminary of the trajectory geometry is done.
The three force configurations during the maneuver correspond to the free body diagrams at stages 1, 2 and 3 (see Figure \ref{fig:immelmann-overview}). \vspace{0.5cm}

%\begin{minipage}{\textwidth}
	\includegraphics[width=0.3\linewidth]{figures/free-body-1.pdf} \hfill
%	\captionof{subfigure}{Angles evolution over time}
	\label{fig:free-body-1}
		\includegraphics[width=0.3\linewidth]{figures/free-body-2.pdf}\hfill
%	\captionof{subfigure}{Angles evolution over time}
	\label{fig:free-body-2}
		\includegraphics[width=0.3\linewidth]{figures/free-body-3.pdf}
%	\captionof{subfigure}{Angles evolution over time}
	\label{fig:free-body-3}
	\vspace{0.5cm}
	\captionof{figure}{Free body diagrams during the semiloop. Own elaboration.}\vspace{0.25cm}
		\label{fig:3avions}
%\end{minipage}

Note that at all times the lift behaves as the centripetal force, thus indicating that both the velocity and radius limitations to perform the Immelmann turn are inherent to the wing design and its maximum and minimum lift generation.\\
AS the motion will be circular, the normal and tangential accelerations will follow:
\begin{align*}
	a_n=&\frac{V^2}{R}=V\dot{\gamma}=\dot{\gamma}^2R&a_t=V^2R
\end{align*}
As a result, the radius can be writen as a function of the angle of attack. By operating with the third equation of system number \ref{eq:semicircle}, which is in wind system of reference, very convenient as is also polar set of axis:
\begin{align*}
	L=&m\left(\frac{V^2}{R}+g\cos(\gamma)\right)\\
	\frac{1}{2}\rho S V^2 C_L(\alpha)=&m\left(\frac{V^2}{R}+g\cos(\gamma)\right)\\
	R=&\frac{2mV^2}{2W\cos(\gamma)+\rho S V^2 C_L(\alpha)}
\end{align*}
As the plane must describe a semicircle, the radius must be constant.

\begin{center}
	\includegraphics[width=\linewidth]{../matlab/radius.png}
	\vspace{0.5cm}
	\captionof{figure}{Radius dependance on V and $\alpha$t. Own elaboration.}\vspace{0.25cm}
\end{center}

For this phase the evolution of $\gamma$ will be imposed. By using the following conditions:
\begin{itemize}
	\item Must begin at 0 and end at $\pi$ rad
	\item Must have a linear evolution troughout time - in order not to overcomplicate the problem -
	\item Must be compatible with circular accelerations, as the ideal performance of the maneuver would require the half loop to be a semicirle
\end{itemize}

From these it can clearly be deucted that the basic model for a slope will be required
\[\gamma(t)=At+B\]
And the rest of conditions define the parameters A and B such that the resulting expression is obtained:
\begin{align}
	\gamma(t)=&\frac{V_0}{R}t & &or& \gamma(t)=&\frac{V_0}{R}t + \pi-\frac{V_0}{R}tf
\end{align}
Where the finishing time of \textit{tf} would be $tf=\pi/V_0$.\\
Note that $V_0$ instead of V(t) has been employed. This is in order to ensure the linearity, although a time dependant velocity could be used too. This would lead to a concatenation of curves for the loop, and not a semicirle.\\

Given that the determined parameters are therefore the gas control lever ($\pi$) and the elevator ($\alpha$), with the same values as for the previous phase - 1.5 kN and 0.3 rad respectively -, but also $\gamma$ is, the only variable to study is the velocity. Nothe that the position coordinates evolution ($\Dot{x}$ and $\Dot{z}$) are explored in the following section: Trajectory analysis.\\

For this phase, the velocity must follow a non-linear evolution as it is dependant on both trigonometric and squared terms in the equations, see $\gamma$ and the lift and drag forces definitions in equations \ref{eq:lift} and \ref{eq:drag}. \\
As the previous shcemes in Figure \ref{fig:3avions} show, at the vertical flight after a quarter of loop, the velocity slows its rate of increase at that point. This inflection point is the consequence of the weight force switching the aircraft's axis for $\gamma=\pi/2$ thus forcing a force redistribution in order to mantain equilibrium.\\
As Figure \ref{fig:radivelo} shows, it is worth noting too that the higher the initial speed solely modifies the placement of the curve in the plot, but has no effect on its characteristics. Althoug it is not clearly shown in the plot, the time is reduced too. On the contrary, for a fixed velocity, different radii have a more noticeable impact on the performance of the semiloop. As greater radius require more time to get to the quarter loop point - for the same initial speed -, the local velocity maximum prior to the $\gamma=\pi/2$ state is higher the greater the radius. Although both the local maximum and the finoshing speed are higher, the greater the radius the longer it takes to complete the half loop.

\begin{center}
	\includegraphics[width=\linewidth]{../matlab/radivelo.jpg}
	\vspace{0.5cm}
	\captionof{figure}{Radii and velocity dependencies on the half loop. Own elaboration.}\vspace{0.25cm}
	\label{fig:radivelo}
\end{center}

Both the lift and drag are affected by this velocity oscillation, but its mangnitude is rather small when compared to the rest of the parameters'.

\begin{center}
	\includegraphics[width=\linewidth]{../matlab/2/liftdrag.jpg}
	\vspace{0.5cm}
	\captionof{figure}{Lift and drag on the half loop. Own elaboration.}\vspace{0.25cm}
	\label{fig:Ld2}
\end{center}

Lastly, both of the position coordinates evolution ($\Dot{x}$ and $\Dot{z}$) evolve as expected, this is, highly dependant on $\gamma$. As this angle ranges from 0 to $\pi$, both coordinates follow reasonable behaviours. It is also clearly shown that the speed remains fairly constant during this part of the maneuver too.

\begin{center}
	\includegraphics[width=\linewidth]{../matlab/2/trajectory.jpg}
	\vspace{0.5cm}
	\captionof{figure}{Radii and velocity dependencies on the half loop. Own elaboration.}\vspace{0.25cm}
	\label{fig:2traj}
\end{center}
For time integration in this case
\begin{align*}
	\begin{cases}
		\frac{\partial V}{\partial t}=&\frac{1}{m}\left(T - D -mg\sin\gamma\right)\\
		\frac{\partial \gamma}{\partial t}=&\frac{1}{mV}\left(L-mg\cos\gamma\right)
	\end{cases}
\end{align*}
As during this phase the angle $\gamma$ must evolve from 0 to $\pi$ rad -for simplification purposes we will work with a linear model - and, as previously stated, the circular motion accelerations have been assumed to apply, we can write:
\begin{align*}
	\gamma(t) =& \frac{V}{R}t & \frac{\partial \gamma(t)}{\partial t}=\frac{V}{R}
\end{align*}
\begin{align*}
	dt =& \frac{m}{T - D -mg\sin\gamma} dV\\
	\int dt= & \int \frac{m}{T - D -mg\sin\gamma} dV
	\intertext{Subsituting drag's definition from equation \ref{eq:drag} and operating for lineal aeordynamics,}
	= & \int \frac{m}{T - \frac{1}{2}\rho S V^2 (C_{D_0}+kC_L^2) -mg\sin\gamma} dV
	\intertext{Substituting ,}
\end{align*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Roll}
\addcontentsline{toc}{subsection}{Roll}
Similarly to the half loop, for this phase a part of the known fixed parameters; gas control lever ($\pi$) and elevator deflection ($\alpha$) more conditioning is needed. In this case, the angle is the roll angle; $\mu$. As previouslt done for $\gamma$, the conditions for $\mu$ are:

\begin{itemize}
	\item Must begin at $\pi$ (or -$\pi$) and end at 0 rad
	\item Must have a linear evolution troughout time - in order not to overcomplicate the problem -
\end{itemize}

From these it can clearly be deucted that the basic model for a slope will be required
\[\mu(t)=At+B\]
As we are missing some conditions to deremine the rate at which the aircraft should roll, the selection is based on the aircraft's structural capacities. For aerobatic aricrafts the rolls must be performed at approximately 120 knots for full aileron rolls and a rotating velocity of 7 m/s at the tip of the wing. With this data and the wingspan we can extract the expressions:
\begin{align}
	\mu(t)=&-\frac{V_{tip}}{b/2}t+\pi \\
	\intertext{or}
	\mu(t)=&-\frac{V_{tip}}{b/2}t+\pi-\frac{V_{tip}}{b/2}tf
\end{align}
Where the finishing time of \textit{tf} would be $tf=\pi \frac{b/2}{V_{tip}}$.\\
If the evolution of $\mu$ is considered, though, it can be seen that between $\pi$ and $\pi/2$ its trigonometric value for cosinus is negative. For the equation that yields the velocity - \textit{squared} velocity - this is rather problematic.\\

\begin{center}
	\includegraphics[width=\linewidth]{../matlab/3/sqrt.jpg}
	\vspace{0.5cm}
	\captionof{figure}{Unsolvable domain of part of the trajectory. Own elaboration.}\vspace{0.25cm}
\end{center}

For this reason, the computations have been done only for half of the phase, understanding that the remaining quarter of roll is symmetric.

The velocity is deeply conditioned by the values that $\mu$ acquires throughout the roll, even tending to infinity due to the previously mentioned trigonometric properties. In reality, we can assume a small variation of the total lift and drag, whoch would be oscillating between the lower values of the plots from Figure \ref{fig:ld3}.
\begin{center}
	\includegraphics[width=\linewidth]{../matlab/3/liftdrag.jpg}
	\vspace{0.5cm}
	\captionof{figure}{Unsolvable domain of part of the trajectory. Own elaboration.}\vspace{0.25cm}
	\label{fig:ld3}
\end{center}

For the general overview:
\begin{center}
	\includegraphics[width=\linewidth]{../matlab/3/3traj.jpg}
	\vspace{0.5cm}
	\captionof{figure}{Radii and velocity dependencies on the half loop. Own elaboration.}\vspace{0.25cm}
	\label{fig:3traj}
\end{center}

The total velocity and x(t) would continue simetrically; the velocity would rapidly increase again and the position would exit the inflection point to keep increasing again.\\

Temporal integration is exactly equal to the cruise phase one, as the roll does not have an effect on the temporal derivatives.
\begin{align*}
	dt =& \frac{m}{T - D} dV\\
	\int dt= & \int \frac{m}{T - D} dV
	\intertext{Subsituting drag's definition from equation \ref{eq:drag} and operating for lineal aeordynamics,}
	= & \int_{V_0}^{V} \frac{m}{T - \frac{1}{2}\rho S V^2 (C_{D_0}+kC_L^2)} dV\\
	t_1=& \frac{m}{T - \frac{1}{2}\rho S V^2 (C_{D_0}+kC_L^2)} (V-V_0)
\end{align*}

Note that this phase of the maneuver is perhaps the most idealised, as a roll is an unbalanced maneuver. Because of the aircraft stability design adverse yaw appears from the very beginning. In a nutshell,  an aircraft performing an aileron roll will actually fly along a slightly helical path, and a very light, positive g force will be maintained.

\subsection*{Trajectory analysis}
\addcontentsline{toc}{section}{Trajectory analysis}
The expected result of the trajectory is portrayed in the following image:

\begin{center}
	\includegraphics[width=\linewidth]{../matlab/trajectory.png}
	\vspace{0.5cm}
	\captionof{figure}{Expected trajectory output. Own elaboration.}\vspace{0.25cm}
\end{center}

We can consider the maneuver coordinates:
\begin{align*}
	\dot{x}_e=&V\cos\gamma&&&	x_e=&Vt\cos\gamma \\
	\dot{y}_e=&0& \rightarrow&&	y_e=&0\\
	\dot{x}_e=&-V\sin\gamma&&&z_e=&-Vt\sin\gamma
\end{align*}

From the computational results obtained from ODE solvers, the following plots represent the trajectory:
\begin{center}
	\includegraphics[width=\linewidth]{../matlab/t1.jpg}
	\vspace{0.25cm}
	\captionof{figure}{Trajectory output for cruise flight. Own elaboration.}\vspace{0.25cm}
\end{center}

\begin{center}
	\includegraphics[width=\linewidth]{../matlab/t2.jpg}
	\vspace{0.25cm}
	\captionof{figure}{Trajectory output for the semiloop. Own elaboration.}\vspace{0.25cm}
\end{center}

\begin{center}
	\includegraphics[width=\linewidth]{../matlab/t3.jpg}
	\vspace{0.25cm}
	\captionof{figure}{Trajectory output for the roll. Own elaboration.}\vspace{0.25cm}
\end{center}

The obtained results fit perfectly with the expected outuput.


\documentclass[main.tex]{subfiles}
\begin{document}

\section{Effects of GW on test masses}

\marginpar{Monday\\ 2021-5-3, \\ compiled \\ \today}

Consider two masses, separated by a distance \(L\).
The distance between them can be measured as \(L = (c/2) \Delta t_{P P''}\), where \(P P''\) is the trajectory of a light beam moving from \(P\) to \(Q\) and back to \(P\).

In flat spacetime, the separation vector between them can then be written as \(h^{i} = L_0 n^i\) for some unit vector \(n^i\).

In the presence of a GW, the length will read
%
\begin{align}
L^2
&= g_{\mu \nu }
\qty(x^{\mu }_{q'} - x^{\mu }_{p'})
\qty(x^{\nu }_{q'} - x^{\nu }_{p'})   \\
&\to g_{\mu \nu }
\qty(x^{i }_{q'} - x^{i }_{p'})
\qty(x^{j }_{q'} - x^{j }_{p'})  \\
&\to g_{\mu \nu }
x^{i }_{q'}
x^{j }_{q'}    \\
&\to \qty(\delta_{ij} + h_{ij}^{TT}) x^{i}_{q'} x^{j}_{q'}
= L_0^2 \qty(\delta_{ij} + h_{ij}^{TT}) n^i n^j
\,,
\end{align}
%
and so we can compute
%
\begin{align}
\frac{ \delta L}{L_0 } = \frac{L}{L_0 } - 1 = \sqrt{1 + h_{ij}^{TT} n^i n^j} - 1 \approx \frac{1}{2} h_{ij}^{TT} n^i n^j
\,.
\end{align}

This justifies the heuristic formula \(\delta L / L_0 \sim h\).

A more formal treatment can be given through the \textbf{geodesic deviation} formula: we can show that if \(u^{\mu }\) is the tangent vector of a family of geodesics and \(s^{\mu }\) is the displacement between geodesics, then
%
\begin{align}
u^{\mu } \nabla_{\mu } \qty( u^{\nu } \nabla_{\nu } s^{\alpha })
= R^{\alpha }_{\lambda \rho \sigma } u^{\lambda } u^{\rho } s^{\sigma }
\,,
\end{align}
%
and in the weak field limit the Riemann tensor is in the form \(\partial^2 h\).

If we plug in everything (as we did in the exercise) we get
%
\begin{align}
\dv[2]{s_\alpha }{t} = R_{\alpha 00 \mu } s^{\mu }
\qquad \text{with} \qquad
R_{\alpha 00 \mu } = \frac{1}{2} \ddot{h}_{\alpha \mu }^{TT}
\,.
\end{align}

The temporal evolution of the spatial vector then reads
%
\begin{align}
\ddot{s}^{i}(t) = \frac{1}{2} \ddot{h}^{TT}_{ij} (t) s_0^{j} + \order{h^2}
\,,
\end{align}
%
so if initially \(\dot{s}^{i} (t=0)\) and \(s^{i}(t=0) = s^{i}_0\) we get
%
\begin{align}
s^{i}(t) = s^{i}_0 + \frac{1}{2} h^{TT}_{ij} (t) s^{j}_{0}
= \qty(\delta_{ij} + \frac{1}{2} h^{TT}_{ij}(t)) s^{j}_{0}
\,.
\end{align}

A ring of particles in the \(xy\) plane is deformed by a wave travelling along the \(z\) axis: it becomes an ellipse with axes along the \(x\) and \(y\) direction for the \(h_+\) polarization,
%
\begin{align}
\frac{ \delta x^2}{r_0^2 (1 + h_+)^2} +
\frac{ \delta y^2}{r_0^2 (1 - h_+)^2} = 1
\,.
\end{align}

The effect of the cross polarization is similar but rotated by \SI{45}{\degree}.

\section{Sources of GW}

We start from a formal solution of \(\square \overline{h}_{\mu \nu }= - 16 \pi T_{\mu \nu }\): like in electromagnetism, we use Green functions,
%
\begin{align}
\overline{h}_{\mu \nu } (t, \vec{x}) = - 16 \pi \int G_R (x^{\alpha } - x^{\alpha \prime}) T_{\mu \nu } (x^{\alpha \prime}) \dd[4]{x'}
\,,
\end{align}
%
where \(\square G_R (x) = \delta^{(4)} (x)\); explicitly
%
\begin{align}
G_R(x) = - \frac{1}{4 \pi } \frac{ \delta (u - t)}{\abs{\vec{x}}}
\,.
\end{align}
%
where \(u\) is the retarded time. (to be defined\dots check)

With this, we get
%
\begin{align}
\overline{h}_{\mu \nu } (t, \vec{x}) &= 4 \int \frac{T_{\mu \nu } (u, \vec{x})}{\abs{\vec{x} - \vec{x}'}} \dd[3]{x'}
\,.
\end{align}

The assumptions we can make are the following: a mean-field approach, negligible self-gravity (this means that the quantity \(2GM / c^2R = R_S  / R \ll 1\)).

Also, in order to derive the quadrupole formula, we assume that the distance from us to the source is large compared to the scale of the source and that the velocity of the source is slow compared to \(c\).

The result we will find is that we can compute
%
\begin{align}
\overline{h}_{ij} (t, \vec{x}) = \frac{4}{r} \int \dd[3]{x'} T_{ij} \qty(t - \frac{r}{c} + \frac{\hat{n} \cdot \vec{x}}{r} , \vec{x}')
\,,
\end{align}
%
and we can further simplify the integrand by expanding in \(x' / r\).

\end{document}


\section{Maximum entropy approximation of distributions}\label{supp_maxent}

(Note: The Python code used for the calculations presented in this section can
be found in the
\href{https://www.rpgroup.caltech.edu//chann_cap/software/MaxEnt_approx_joint.html}{following
link} as an annotated Jupyter notebook)

On the one hand the solution of chemical master equations like the one in
\secref{sec_model} represent a hard mathematical challenge. As presented in
\siref{supp_param_inference} Peccoud and Ycart derived a closed-form solution
for the two-state promoter at the mRNA level \cite{Peccoud1995}. In an
impressive display of mathematical skills, Shahrezaei and Swain were able to
derive an approximate solution for the one- (not considered in this work) and
two-state promoter master equation at the protein level \cite{Shahrezaei2008}.
Nevertheless both of these solutions do not give instantaneous insights about
the distributions as they involve complicated terms such as confluent
hypergeometric functions.

On the other hand there has been a great deal of work to generate methods that
can approximate the solution of these discrete state Markovian models
\cite{Ale2013, Andreychenko2017, Frohlich2016, Schnoerr2017, Smadbeck2013}. In
particular for master equations like the one that concerns us here whose
moments can be easily computed, the moment expansion method provides a simple
method to approximate the full joint distribution of mRNA and protein
\cite{Smadbeck2013}. In this section we will explain the principles behind this
method and show the implementation for our particular case study.

\subsection{The MaxEnt principle}

The principle of maximum entropy (MaxEnt) first proposed by E. T. Jaynes in
1957 tackles the question of given limited information what is the least biased
inference one can make about a particular probability distribution
\cite{Jaynes1957}. In particular Jaynes used this principle to show the
correspondence between statistical mechanics and information theory,
demonstrating, for example, that the Boltzmann distribution is the probability
distribution that maximizes Shannon's entropy subject to a constraint that the
average energy of the system is fixed.

To illustrate the principle let us focus on a univariate distribution $P_X(x)$.
The $n^{\text{th}}$ moment of the distribution for a discrete set of possible
values of $x$ is given by
\begin{equation}
  \ee{x^n} \equiv \sum_x x^n P_X(x).
  \label{eq_mom_ref}
\end{equation}

Now assume that we have knowledge of the first $m$ moments $\bb{\ee{x}}_m = (
\ee{x}, \ee{x^2}, \ldots, \ee{x^m} )$. The question is then how can we use this
information to build an estimator $P_H(x \mid \bb{\ee{x}}_m)$ of the
distribution
such that
\begin{equation}
  \lim_{m \rightarrow \infty} P_H(x \mid \bb{\ee{x}}_m) \rightarrow P_X(x),
\end{equation}
i.e. that the more moments we add to our approximation, the more the estimator
distribution converges to the real distribution.

The MaxEnt principle tells us that our best guess for this estimator is to
build it on the base of maximizing the Shannon entropy, constrained by the
information we have about these $m$ moments. The maximization of Shannon's
entropy guarantees that we are the least committed possible to information that
we do not posses. The Shannon entropy for an univariate discrete distribution
is given by \cite{Shannon1948}
\begin{equation}
  H(x) \equiv - \sum_x P_X(x) \log P_X(x).
\end{equation}

For an optimization problem subject to constraints we make use of the method of
the Lagrange multipliers. For this we define the constraint equation
$\mathcal{L}(x)$ as
\begin{equation}
  \mathcal{L}(x) \equiv H(x) - \sum_{i=0}^m
  \left[ \lambda_i \left( \ee{x^i} - \sum_x x^i P_X(x) \right) \right],
  \label{seq_constraint_eq}
\end{equation}
where $\lambda_i$ is the Lagrange multiplier associated with the $i\th$ moment.
The inclusion of the zeroth moment is an additional constraint to guarantee the
normalization of the resulting distribution. Since $P_X(x)$ has a finite set of
discrete values, when taking the derivative of the constraint equation with
respect to $P_X(x)$, we chose a particular value of $X = x$. Therefore from the
sum over all possible $x$ values only a single term survives. With this in mind
we take the derivative of the constraint equation obtaining
\begin{equation}
  {d\mathcal{L} \over d P_X(x)} = -\log P_X(x) - 1 -
  \sum_{i=0}^m \lambda_i x^i.
\end{equation}

Equating this derivative to zero and solving for the distribution (that we now
start calling $P_H(x)$, our MaxEnt estimator) gives
\begin{equation}
  P_H(x) = \exp \left(- 1 - \sum_{i=0}^m \lambda_i x^i \right)
         ={1 \over \mathcal{Z}}
         \exp \left( - \sum_{i=1}^m \lambda_i x^i \right),
  \label{eq_maxEnt}
\end{equation}
where $\mathcal{Z}$ is the normalization constant that can be obtained by
substituting this solution into the normalization constraint. This results in
\begin{equation}
  \mathcal{Z} \equiv \exp\left( 1 + \lambda_0 \right) =
  \sum_x \exp \left( - \sum_{i=1}^m \lambda_i x^i \right).
\end{equation}

\eref{eq_maxEnt} is the general form of the MaxEnt distribution for a
univariate distribution. The computational challenge then consists in finding
numerical values for the Lagrange multipliers $\{ \lambda_i \}$ such that
$P_H(x)$ satisfies our constraints. In other words, the Lagrange multipliers
weight the contribution of each term in the exponent such that when computing
any of the moments we recover the value of our constraint. Mathematically what
this means is that $P_H(x)$ must satisfy
\begin{equation}
  \sum_x x^n P_H(x) =
  \sum_x {x^n \over \mathcal{Z}}
  \exp \left( - \sum_{i=1}^m \lambda_i x^i \right) = \ee{x^n}.
\end{equation}

As an example of how to apply the MaxEnt principle let us use the classic
problem of a six-face die. If we are only told that after a large number of die
rolls the mean value of the face is $\ee{x} = 4.5$ (note that a fair die has a
mean of $3.5$), what would the least biased guess for the distribution look
like? The MaxEnt principle tells us that our best guess would be of the form
\begin{equation}
  P_H(x) = {1 \over \mathcal{Z}} \exp \left( \lambda x \right).
\end{equation}
Using any numerical minimization package we can easily find the value of the
Lagrange multiplier $\lambda$  that satisfies our constraint.
\fref{fig_maxent_die} shows two two examples of distributions that satisfy the
constraint. Panel (A) shows a distribution consistent with the 4.5 average
where both 4 and 5 are equally likely. Nevertheless in the information we got
about the nature of the die it was never stated that some of the faces were
forbidden. In that sense the distribution is committing to information about
the process that we do not posses. Panel (B) by contrast shows the MaxEnt
distribution that satisfies this constraint. Since this distribution maximizes
Shannon's entropy it is guaranteed to be the least biased distribution given
the available information.

\begin{figure}[h!]
	\centering \includegraphics
  {../fig/si/figS15.pdf}
	\caption{\textbf{Maximum entropy distribution of six-face die.} (A)biased
  distribution consistent with the constraint $\ee{x} = 4.5$. (B) MaxEnt
  distribution also consistent with the constraint.}
  \label{fig_maxent_die}
\end{figure}

\subsubsection{The mRNA and protein joint distribution}

The MaxEnt principle can easily be extended to multivariate distributions. For
our particular case we are interested in the mRNA and protein joint
distribution
$P(m, p)$. The definition of a moment $\ee{m^x p^y}$ is a natural extension of
\eref{eq_mom_ref} of the form
\begin{equation}
  \ee{m^x p^y} = \sum_m \sum_p m^x p^y P(m, p).
\end{equation}

As a consequence the MaxEnt joint distribution $P_H(m, p)$ is of the form
\begin{equation}
  P_H(m, p) = {1 \over \mathcal{Z}}
              \exp \left( - \sum_{(x,y)} \lambda_{(x,y)} m^x p^y \right),
  \label{seq_maxEnt_joint}
\end{equation}
where $\lambda_{(x,y)}$ is the Lagrange multiplier associated with the moment
$\ee{m^x p^y}$, and again $\mathcal{Z}$ is the normalization constant given by
\begin{equation}
  \mathcal{Z} = \sum_m \sum_p
              \exp \left( - \sum_{(x, y)} \lambda_{(x, y)} m^x p^y \right).
\end{equation}
Note that the sum in the exponent is taken over all available $(x, y)$ pairs
that define the moment constraints for the distribution.

\subsection{The Bretthorst rescaling algorithm}

The determination of the Lagrange multipliers suffer from a numerical under and
overflow problem due to the difference in magnitude between the constraints.
This becomes a problem when higher moments are taken into account. The
resulting numerical values for the Lagrange multipliers end up being separated
by several orders of magnitude. For routines such as Newton-Raphson or other
minimization algorithms that can be used to find these Lagrange multipliers
these different scales become problematic.

To get around this problem we implemented a variation to the algorithm due to
G. Larry Bretthorst, E.T. Jaynes' last student. With a very simple argument we
can show that linearly rescaling the constraints, the Lagrange multipliers and
the ``rules'' for how to compute each of the moments, i.e. each of the
individual products that go into the moment calculation, should converge to the
same MaxEnt distribution. In order to see this let's consider again a
univariate distribution $P_X(x)$ that we are trying to reconstruct given the
first two moments $\ee{x}$, and $\ee{x^2}$. The MaxEnt distribution can be
written as
\begin{equation}
  P_H(x) = {1 \over \mathcal{Z}}
  \exp \left(- \lambda_1 x - \lambda_2 x^2 \right) =
  {1 \over \mathcal{Z}}
  \exp \left(- \lambda_1 x \right) \exp \left( - \lambda_2 x^2 \right).
\end{equation}
We can always rescale the terms in any way and obtain the same result. Let's
say that for some reason we want to rescale the quadratic terms by a factor
$a$. We can define a new Lagrange multiplier $\lambda_2' \equiv {\lambda_2
\over a}$ that compensates for the rescaling of the terms, obtaining
\begin{equation}
  P_H(x) = {1 \over \mathcal{Z}}
  \exp \left(- \lambda_1 x \right) \exp \left( - \lambda_2' ax^2 \right).
\end{equation}
Computationally it might be more efficient to find the numerical value of
$\lambda_2'$ rather than $\lambda_2$ maybe because it is of the same order of
magnitude as $\lambda_1$. Then we can always multiply $\lambda_2'$ by $a$ to
obtain back the constraint for our quadratic term. What this means is that that
we can always rescale the MaxEnt problem to make it numerically more stable,
then we can rescale back to obtain the value of the Lagrange multipliers. The
key to the Bretthorst algorithm lies in the selection of what rescaling factor
to choose in order to make the numerical inference more efficient.

Bretthorst's algorithm goes even further by further transforming the
constraints and the variables to make the constraints orthogonal, making the
computation much more effective. We now explain the implementation of the
algorithm for our joint distribution of interest $P(m, p)$.

\subsubsection{Algorithm implementation}

Let the $M \times N$ matrix $\bb{A}$ contain all the factors used to compute
the moments that serve as constraints, where each entry is of the form
\begin{equation}
  A_{ij} = m_i^{x_j} \cdot p_i^{y_j}.
  \label{seq_maxent_rules}
\end{equation}
In other words, recall that to obtain any moment $\ee{m^x p^y}$ we compute
\begin{equation}
  \ee{m^x p^y} = \sum_m \sum_p m^x p^y P(m, x).
\end{equation}
If we have $M$ possible $(m, p)$ pairs in our truncated sample space (because
we can't include the sample space up to infinity) $\{(m, p)_1, (m, p)_2, \ldots
(m, p)_N \}$, and we have $N$ exponent pairs $(x, y)$ corresponding to the $N$
moments used to constraint the maximum entropy distribution $\{(x, y)_1, (x,
y)_2, \ldots, (x, y)_N \}$, then matrix $\bb{A}$ contains all the possible $M$
by $N$ terms of the form described in \eref{seq_maxent_rules}. Let also
$\bb{v}$ be a vector of length $N$ containing all the constraints with each
entry of the form
\begin{equation}
  v_j = \ee{m^{x_j} p^{y_j}},
\end{equation}
i.e. the information that we have about the distribution. That means that the
constraint equation $\mathcal{L}$ to be used for this problem takes the form
\begin{equation}
  \mathcal{L} = -\sum_i P_i \ln P_i + \lambda_0 \left( 1 - \sum_i P_i \right)
  + \sum_{j>0} \lambda_j \left( v_j - \sum_i A_{ij} P_i \right),
\end{equation}
where $\lambda_0$ is the Lagrange multiplier associated with the normalization
constraint, and $\lambda_j$ is the Lagrange multiplier associated with the
$j\th$ constraint. This constraint equation is equivalent to
\eref{seq_constraint_eq}, but now all the details of how to compute the moments
are specified in matrix $\bb{A}$.

With this notation in hand we now proceed to rescale the problem. The first
step consists of rescaling the terms to compute the entries of matrix $\bb{A}$.
As mentioned before, this is the key feature of the Bretthorst algorithm; the
particular choice of rescaling factor used in the algorithm empirically
promotes that the rescaled Lagrange multipliers are of the same order of
magnitude. The rescaling takes the form
\begin{equation}
  A_{ij}' = {A_{ij} \over G_j},
\end{equation}
where $G_j$ serves to rescale the moments, providing numerical stability to the
inference problem. Bretthorst proposes an empirical rescaling that satisfies
\begin{equation}
G_j^2 = \sum_i A_{ij}^2,
\end{equation}
or in terms of our particular problem
\begin{equation}
G_j^2 = \sum_m \sum_p \left( m^{x_j} p^{y_j} \right)^2.
\end{equation}
What this indicates is that each pair $m_i^{x_j} p_i^{y_j}$ is normalized by
the square root of the sum of the all pairs of the same form squared.

Since we rescale the factors involved in computing the constraints, the
constraints must also be rescaled simply as
\begin{equation}
v_j' = \ee{m^{x_j} p^{y_j}}' = {\ee{m^{x_j} p^{y_j}} \over G_j}.
\end{equation}
The Lagrange multipliers must compensate this rescaling since at the end of the
day the probability must add up to the same value. Therefore we rescale the
$\lambda_j$ terms as
\begin{equation}
\lambda_j' = \lambda_j G_j,
\end{equation}
such that any $\lambda_j A_{ij} = \lambda_j' A_{ij}'$. If this empirical value
for the rescaling factor makes the rescaled Lagrange multipliers $\lambda_j'$
be of the same order of magnitude, this by itself would already improve the
algorithm convergence. Bretthorst proposes another linear transformation to
make the optimization routine even more efficient. For this we generate
orthogonal constraints that make Newton-Raphson and similar algorithms converge
faster. The transformation is as follows
\begin{equation}
  A_{ik}'' = \sum_j {e}_{jk} A_{ij}',
\end{equation}
for the entires of matrix $\bb{A}$, and
\begin{equation}
  v_k'' = \sum_j {e}_{jk} u_j',
\end{equation}
for entires of the constraint vector $\bb{v}$, finally
\begin{equation}
  \lambda_k'' = \sum_j {e}_{jk} \beta_j,
\end{equation}
for the Lagrange multipliers. Here ${e}_{jk}$ is the $j\th$ component
of the $k\th$ eigenvector of the matrix $\bb{E}$ with entries
\begin{equation}
  {E}_{kj} = \sum_i {A}_{ik}' {A}_{ij}'.
\end{equation}
This transformation guarantees that the matrix $\bb{A}''$ has the property
\begin{equation}
  \sum_i A_{ij}'' A_{jk}'' = \beta_j \delta_{jk},
\end{equation}
where $\beta_j$ is the $j\th$ eigenvalue of the matrix $\bb{E}$ and
$\delta_{jk}$ is the Kronecker delta function. What this means is that, as
desired, the constraints are orthogonal to each other, improving the algorithm
convergence speed.

\subsection{Predicting distributions for simple repression constructs}

Having explained the theoretical background along with the practical
difficulties and a workaround strategy proposed by Bretthorst, we implemented
the inference using the moments obtained from averaging over the variability
along the cell cycle (See \siref{supp_multi_gene}). \fref{fig_pmf_mRNA} and
\fref{fig_pmf_protein} present these inferences for both mRNA and protein
levels respectively for different values of the repressor-DNA binding energy
and repressor copy numbers per cell. From these plots we can easily appreciate
that despite the fact that the mean of each distribution changes as the
induction level changes, there is a lot of overlap between distributions. This
as a consequence means that at the single-cell level cells cannot perfectly
resolve between different inputs.

\begin{figure}[h!]
	\centering \includegraphics
  {../fig/si/figS16.pdf}
	\caption{\textbf{Maximum entropy mRNA distributions for simple repression
	constructs.} mRNA distributions for different biophysical parameters. From
	left to right the repressor-DNA affinity decreases as defined by the three
	lacI operators O1 ($-15.3 \; k_BT$), O2 ($ -13.9 \; k_BT$), and O3 ($-9.7 \;
	k_BT$). From top to bottom the mean repressor copy number per cell increases.
	The curves on each plot represent different IPTG concentrations. Each
	distribution was fitted using the first three moments of the mRNA
	distribution.}
  \label{fig_pmf_mRNA}
\end{figure}

\begin{figure}[h!]
	\centering \includegraphics
  {../fig/si/figS17.pdf}
	\caption{\textbf{Maximum entropy protein distributions for simple repression
	constructs.} Protein distributions for different biophysical parameters. From
	left to right the repressor-DNA affinity decreases as defined by the three
	lacI operators O1 ($-15.3 \; k_BT$), O2 ($ -13.9 \; k_BT$), and O3 ($-9.7 \;
	k_BT$). From top to bottom the mean repressor copy number per cell increases.
	The curves on each plot represent different IPTG concentrations. Each
	distribution was fitted using the first six moments of the protein
	distribution.}
  \label{fig_pmf_protein}
\end{figure}

\subsection{Comparison with experimental data}

Now that we have reconstructed an approximation of the probability distribution
$P(m, p)$ we can compare this with our experimental measurements. But just as
detailed in \siref{supp_theory_vs_data_mom} the single-cell microscopy
measurements are given in arbitrary units of fluorescence. Therefore we cannot
compare directly our predicted protein distributions with these values. To get
around this issue we use the fact that the fold-change in gene expression that
we defined as the ratio of the gene expression level in the presence of the
repressor and the expression level of a knockout strain is a non-dimensional
quantity. Therefore we normalize all of our single-cell measurements by the
mean fluorescence value of the $\Delta lacI$ strain with the proper background
fluorescence subtracted as explained in \siref{supp_theory_vs_data_mom} for the
noise measurements. In the case of the theoretical predictions of the protein
distribution we also normalize each protein value by the predicted mean protein
level $\ee{p}$, having now non-dimensional scales that can be directly
compared. \fref{sfig_cdf_delta} shows the experimental (color curves) and
theoretical (dark dashed line) cumulative distribution functions for the three
$\Delta lacI$ strains. As in \fref{sfig_noise_delta}, we do not expect
differences between the operators, but we explicitly plot them separately to
make sure that this is the case. We can see right away that as we would expect
given the limitations of the model to accurately predict the noise and skewness
of the distribution, the model doesn't accurately predict the data. Our model
predicts a narrower distribution compared to what we measured with single-cell
microscopy.

\begin{figure}[h!]
	\centering \includegraphics
  {../fig/si/figS18.pdf}
	\caption{\textbf{Experiment vs. theory comparison for $\Delta lacI$ strain.}
  Example fold-change empirical cumulative distribution functions (ECDF) for
  strains with no repressors and different operators. The color curves
  represent single-cell microscopy measurements while the dashed black lines
  represent the theoretical distributions as reconstructed by the maximum
  entropy principle. The theoretical distributions were fitted using the first
  six moments of the protein distribution.}
  \label{sfig_cdf_delta}
\end{figure}

The same narrower prediction applies to the regulated promoters.
\fref{sfig_cdf_reg}, shows the theory-experiment comparison of the cumulative
distribution functions for different repressor binding sites (different
figures), repressor copy numbers (rows), and inducer concentrations (columns).
In general the predictions are systematically narrower compared to the actual
experimental data.

\begin{figure}[h!]
	\centering \includegraphics
  {../fig/si/figS19.pdf}
	\caption{\textbf{Experiment vs. theory comparison for regulated promoters.}
  Example fold-change empirical cumulative distribution functions (ECDF) for
  regulated strains with the three operators (different colors) as a function
  of repressor copy numbers (rows) and inducer concentrations (columns). The
  color curves represent single-cell microscopy measurements while the dashed
  black lines represent the theoretical distributions as reconstructed by the
  maximum entropy principle. The theoretical distributions were fitted using
  the first six moments of the protein distribution}
  \label{sfig_cdf_reg}
\end{figure}