\documentclass{subfile}

\begin{document}
	\section{RNO}\label{sec:rno}

		\begin{problem}[$2019$ Team Selection Test, Day $5$, problem $1$]
			Determine the largest value of the expression
				\begin{align*}
					\sum_{1\leq i<j\leq 4}(x_{i}+x_{j})\sqrt{x_{i}{x_{j}}}
				\end{align*}
			where $x_{1},x_{2},x_{3},x_{4}$ non-negative real numbers such that $x_{1}+x_{2}+x_{3}+x_{4}=1$. Also, find the specific values where this maximum is achieved.
		\end{problem}

		\begin{problem}[$2018$ Team Selection Test, Day $1$, problem $1$]
			Let $x_{1},\ldots,x_{n}\geq -1$ be real numbers such that  $\sum_{i=1}^{n}x_{i}^{3}=0$. Find the least constant $c$ such that
				\begin{align*}
					\sum_{i=1}^{n}x_{i}^{2}
						& \leq cn
				\end{align*}
		\end{problem}

		\begin{problem}[$2014$ Team Selection Test, Day $3$, problem $3$]
			Determine the smallest positive constant $c$ such that
				\begin{align*}
					\sum_{i=1}^{n}\left(\dfrac{1}{i}\sum_{j=1}^{i}x_{j}\right)^{2}
						& \leq c\sum_{i=1}^{n}x_{i}^{2}
				\end{align*}
			for all positive integer $n$ and all positive real numbers $x_{1},\ldots,x_{n}$.
		\end{problem}

		\begin{problem}[$2012$ Team Selection Test, Day $2$, problem $4$]
			Let $k$ be a positive integer. Find the maximum value of
				\begin{align*}
					a^{3k-1}b+b^{3k-1}c+c^{3k-1}a+k^{2}a^{k}b^{k}c^{k}
				\end{align*}
			where $a,b,c$ are non-negative real numbers such that $a+b+c=3k$.
		\end{problem}

		\begin{problem}[$2011$ Team Selection Test, Day $3$, problem $2$]
			Given real numbers $x,y,z$ such that $x+y+z=0$. Show that
				\begin{align*}
					\dfrac{x(x+2)}{2x^{2}+1}+\dfrac{y(y+2)}{2y^{2}+1}+\dfrac{z(z+2)}{2z^{2}+1}
						& \geq 0
				\end{align*}
		\end{problem}

		\begin{problem}[$2011$ Team Selection Test, Day $5$, problem $2$]
			Let $n\geq2$ be an integer and $x_{1},\ldots,x_{n}$ be positive real numbers such that
				\begin{align*}
					\sum_{i=1}^{n}\dfrac{1}{x_{i}+1}
						& = 1
				\end{align*}
			and $k>1$ be a real number. Show that
				\begin{align*}
					\sum_{i=1}^{n}\dfrac{1}{x_{i}^{k}+1}
						& \geq \dfrac{n}{(n-1)^{k}+1}
				\end{align*}
			and determine the case of equality.
		\end{problem}

		\begin{problem}[$2010$ Team Selection Test $1$, problem $2$]
			Let $n$ be a positive integer and $a_{1},\ldots,a_{n}$ be positive real numbers. Prove that $f:[0,\infty)\to\mathbb{R}$ defined by
				\begin{align*}
					f(x)
						& = \dfrac{a_{1}+x}{a_{2}+x}+\dfrac{a_{2}+x}{a_{3}+x}+\ldots+\dfrac{a_{n}+x}{a_{1}+x}
				\end{align*}
			is decreasing.
		\end{problem}

		\begin{problem}[$2010$ Team Selection $2$, problem $1$]
			Let $n$ be a positive integer. Determine the maximum value of
				\begin{align*}
					\max\left\{\dfrac{x_{1}}{1+x_{1}},\ldots,\dfrac{x_{n}}{1+x_{1}+\ldots+x_{n}}\right\}
				\end{align*}
			as $x_{1},\ldots,x_{n}$ runs through all non-negative real numbers such that $x_{1}+\ldots+x_{n}=1$.
		\end{problem}

		\begin{problem}[$2010$ Team Selection Test $3$, problem $1$]
			Let $n$ be a positive integer and $x_{1},\ldots,x_{n}$ be positive real numbers such that $x_{1}\cdots x_{n}=1$. Prove that
				\begin{align*}
					\sum_{i=1}^{n}x_{i}^{n}(1+x_{i})
						& \geq \dfrac{n}{2^{n-1}}\prod_{i=1}^{n}(1+x_{i})
				\end{align*}
		\end{problem}

		\begin{problem}[$2009$ Team Selection Test, Day $4$, problem $1$]
			Let $n\geq 2$ be an integer and $x_{1},\ldots,x_{n}$ be positive integers such that $x_{1}\leq\ldots\leq x_{n}$ and $x_{1}+\ldots+x_{n}=x_{1}\cdots x_{n}$. What is the maximum possible value of $x_{1}+\ldots+x_{n}$.
		\end{problem}

		\begin{problem}[$2008$ Team Selection Test, Day $1$, problem $2$]
			Let $n\geq2$ be an integer and $a_{1},\ldots,a_{n};b_{1},\ldots,b_{n}$ be positive real numbers such that $a_{i}<b_{i}$ and
				\begin{align*}
					b_{1}+\ldots+b_{n}
						& < 1+a_{1}+\ldots+a_{n}
				\end{align*}
			Prove that there exists a real number $c$ such that
				\begin{align*}
					(a_{i}+c+k)(b_{i}+c+k)
						& > 0
				\end{align*}
			for $1\leq i\leq n$ and any integer $k$.
		\end{problem}

		\begin{problem}[$2008$ Team Selection Test, Day $2$, problem $1$]
			Let $n\geq 3$ be an odd integer. Determine the maximum value of
				\begin{align*}
					\sqrt{|x_{1}-x_{2}|}+\ldots+\sqrt{|x_{n}-x_{1}|}
				\end{align*}
			where $x_{1},\ldots,x_{n}$ are real numbers in the interval $[0,1]$.
		\end{problem}

		\begin{problem}[$2007$ Team Selection Test, Day $1$, problem $1$]
			Let $a_{1},\ldots,a_{n}$ be non-negative real numbers such that $a_{1}^{2}+\ldots+a_{n}^{2}=1$. Find the maximum value of
				\begin{align*}
					(1-a_{1})\cdots(1-a_{n})
				\end{align*}
		\end{problem}

		\begin{problem}[$2007$ Team Selection Test, Day $3$, problem $2$]
			Let $n,p\geq 4$ be integers and $x_{1},\ldots,x_{n}$ be positive real numbers such that $x_{1}+\ldots+x_{n}=n$. Prove that
				\begin{align*}
					\sum_{i=1}^{n}\dfrac{1}{x_{i}^{p}}
						& \geq \sum_{i=1}^{n}x_{i}^{p}
				\end{align*}
			is false.
		\end{problem}

		\begin{problem}[$2007$ Team Selection Test, Day $7$, problem $1$]
			Let $n\geq 2$ be an integer and $a_{1},\ldots,a_{n};b_{1},\ldots,b_{n}$ be real numbers such that
				\begin{align*}
					\sum_{i=1}^{n}a_{i}^{2}
						& = \sum_{i=1}^{n}b_{i}^{2}=1\\
					\sum_{i=1}^{n}a_{i}b_{i}
						& = 0
				\end{align*}
			Prove that
				\begin{align*}
					\left(\sum_{i=1}^{n}a_{i}\right)^{2}+\left(\sum_{i=1}^{n}b_{i}\right)^{2}
						& \leq n
				\end{align*}
		\end{problem}

		\begin{problem}[$2006$ Team Selection Test, Day $1$, problem $4$]
			Let $n$ be a positive integer and $a_{1},\ldots,a_{n}$ be real numbers such that $|a_{i}|\leq 1$ for $1\leq i\leq n$ and $a_{1}+\ldots+a_{n}=0$. Prove that there exists a positive integer $k\leq n$ such that
				\begin{align*}
					|1\cdot a_{1}+\ldots+ka_{k}|
						& \leq \dfrac{2k+1}{4}
				\end{align*}
			Also show that this is the best bound possible for $n>2$.
		\end{problem}

		\begin{problem}[$2006$ Team Selection Test, Day $2$, problem $4$]
			Let $n$ be a positive integer and $x_{1},\ldots,x_{n}$ be real numbers. Prove that
				\begin{align*}
					\sum_{1\leq i<j\leq n}|x_{i}+x_{j}|
						& \geq \dfrac{n-2}{2}\sum_{i=1}^{n}|x_{i}|
				\end{align*}
		\end{problem}

		\begin{problem}[$2005$ Team Selection Test, Day $5$, problem $2$]
			Let $n\geq 2$ be an integer and $x_{1},\ldots,x_{n}$ be positive real numbers such that $x_{1}\cdots x_{n}=1$. Find the smallest real value $\rho(n)$ such that for any $x_{1},\ldots,x_{n}$,
				\begin{align*}
					\sum_{i=1}^{n}\dfrac{1}{x_{i}}
						& \leq \sum_{i=1}^{n}x_{i}^{r}
				\end{align*}
			is true for all $r\geq\rho(n)$.
		\end{problem}

		\begin{problem}[$2004$ Team Selection Test, Day $1$, problem $1$]
			Let $a_{1},\ldots,a_{4}$ be the sides of a quadrilateral with perimeter $2s$. Prove that
				\begin{align*}
					\sum_{i=1}^{4}\dfrac{1}{a_{i}+s}
						& \leq \dfrac{2}{9}\sum_{1\leq i<j\leq 4}\dfrac{1}{\sqrt{(s-a_{i})(s-a_{j})}}
				\end{align*}
			When does equality hold?
		\end{problem}

		\begin{problem}[$2002$ Team Selection Test, Day $3$, problem $2$]
			Let $n\geq 4$ be an integer and $a_{1},\ldots,a_{n}$ be positive real numbers such that
				\begin{align*}
					a_{1}^{2}+\ldots+a_{n}^{2}
						& = 1
				\end{align*}
			Prove that
				\begin{align*}
					\dfrac{a_{1}}{a_{1}^{2}+1}+\ldots+\dfrac{a_{n}}{a_{n}^{2}+1}
						& \geq \dfrac{4}{5}\left(a_{1}\sqrt{a_{1}}+\ldots+a_{n}\sqrt{a_{n}}\right)^{2}
				\end{align*}
		\end{problem}

		\begin{problem}[$2001$ Team Selection Test, Day $1$, problem $3$]
			Let $a,b,c$ be the sides of a triangle. Prove that
				\begin{align*}
					(-a+b+c)(a-b+c)+(a-b+c)(a+b-c)+(a+b-c)(-a+b+c)
						& \leq \sqrt{abc}(\sqrt{a}+\sqrt{b}+\sqrt{c})
				\end{align*}
		\end{problem}

		\begin{problem}[$2000$ Team Selection Test, Day $1$, problem $2$]
			Let $n$ be a positive integer and $x_{1},\ldots,x_{n}$ be real numbers such that $|x_{k+1}-x_{k}|\leq 1$ for $1\leq k\leq n-1$. Prove that
				\begin{align*}
					\sum_{i=1}^{n}|x_{i}|-\left|\sum_{i=1}^{n}x_{i}\right|
						& \leq \dfrac{n^{2}-1}{4}
				\end{align*}
		\end{problem}

		\begin{problem}[$1999$ Team Selection Test, Day $2$, problem $2$]
			Let $n$ and $x_{1},\ldots,x_{n}$ be positive integers. Prove that
				\begin{align*}
					x_{1}^{2}+\ldots+x_{n}^{2}
						& \geq \dfrac{2n+1}{3}(x_{1}+\ldots+x_{n})
				\end{align*}
		\end{problem}

		\begin{problem}[$1996$ Team Selection Test, Day $2$, problem $4$]
			If $p_{1},\ldots,p_{k}$ are the distinct prime divisors of $n$, define
				\begin{align*}
					a_{n}
						& = \dfrac{1}{p_1}+\ldots+\dfrac{1}{p_{n}}
				\end{align*}
			Prove that
				\begin{align*}
					\sum_{i=2}^{n}a_{1}\cdots a_{i}
						& < 1
				\end{align*}
			for $n\geq 2$.
		\end{problem}

		\begin{problem}[$1996$ Team Selection Test, Day $3$, problem $1$]
			Let $n\geq 3$ be an integer and $x_{1},\ldots,x_{n-1}$ be non-negative integers such that
				\begin{align*}
					x_{1}+\ldots+x_{n-1}
						& = n\\
					1\cdot x_{1}+\ldots+(n-1)x_{n}
						& = 2(n-2)
				\end{align*}
			Find the minimum value of
				\begin{align*}
					\sum_{i=1}^{n}i(2n-i)x_{i}
				\end{align*}
		\end{problem}

		\begin{problem}[$1996$ Team Selection Test, Day $4$, problem $1$]
			Let $n$ be a positive integer and $x_{1},\ldots,x_{n}$ be positive real numbers such that
				\begin{align*}
					x_{n+1}
						& = x_{1}+\ldots+x_{n}
				\end{align*}
			Prove that
				\begin{align*}
					\sum_{i=1}^{n}\sqrt{x_{i}(x_{n+1}-x_{i})}
						& \leq \sqrt{\sum_{i=1}^{n}x_{n+1}(x_{n+1}-x_{i})}
				\end{align*}
		\end{problem}

		\begin{problem}[$1993$ Team Selection Test, Day $1$, problem $1$]
			Find the maximum possible constant $A$ such that
				\begin{align*}
					\dfrac{x}{\sqrt{y^{2}+z^{2}}}+\dfrac{y}{\sqrt{z^{2}+x^{2}}}+\dfrac{z}{\sqrt{x^{2}+y^{2}}}
						& \geq A
				\end{align*}
			for all positive real numbers $x,y,z$.
		\end{problem}
\end{document}


\chapter{Risk-Sensitive Policy Gradient}
\label{ch:risk_sensitive_policy_gradient}

Risk aware decision making plays a crucial role in many fields, such as finance and process control. In this chapter we discuss some policy gradient methods for the risk-sensitive formulation of a sequential decision problem. This extension presents some difficulties which have attracted the interest of many researchers in the last years. In particular, while in the risk-neutral framework the policy gradient theorem represents the keystone for all the learning algorithms, in the risk-sensitive framework the approaches for the episodic setting, the discounted reward and the average reward formulations are quite different. The goal of this chapter is to give an overview of the methods found in the literature and to try to unify them in a way similar to the risk-neutral framework. This chapter and the following represent the main contribution of this thesis to the policy gradient literature.

\section{Risk-Sensitive Framework}
In the risk-sensitive framework, in addition to maximizing the rewards, the agent also wants to control the risk necessary to achieve it. It is thus necessary to introduce a function $\Lambda: \Theta \to \R$ such that $\Lambda(\theta)$ measures the risk associated with the policy $\pi_\theta$. In an episodic framework where the system always starts from the same initial state $s_0$, the variance of the total return can be used \cite{tamar2012policy}.
\begin{definition}[Start Variance]
	The start variance is the variance of the return that can be obtained starting from the start state $s_0 \in \S$ and following policy $\pi_\theta$
	\begin{equation}
		\Lambda_{\text{start}}(\theta) = \Var[\pi_\theta]{G_0\ |\ S_0 = s_0}
			= U_{\pi_\theta} (s_0) - V_{\pi_\theta} (s_0)^2
	\end{equation}
\end{definition}
In many application one may want control only the downside risk, that is the risk of the actual return being below the expected return or the uncertainty about the magnitude of that difference. This risk may be measured by the semivariance.
\begin{definition}[Semivariance]
	\begin{equation}
		\Lambda_\textbf{down}(\theta) = \text{SVar}(\theta) = \E[\pi_\theta]{\min\left\{G_0 - \E[\pi_\theta]{G_0}, 0\right\}^2 | S_0 = s_0}
	\end{equation}
\end{definition}
The square root of this quantity is called semideviation. In a continuing environment, we might consider the long-run variance \cite{prashanth2014actor} defined in Section \ref{sec:risk_sensitive_formulation}.
\begin{definition}[Long-Run Variance]
	The long-run variance $\Lambda_\pi$ under policy $\pi$ is defined as
	\begin{equation}
		\begin{split}
			\Lambda_{\text{long-run}}(\theta) &= \lim_{T \to \infty} \frac{1}{T} \E[\pi_\theta]{
			\sum^{T-1}_{t=0} \left(R_{t+1} - \rho(\theta)\right)^2}\\
		\end{split}
	\end{equation}
\end{definition}
In order to formalize the trade-off between reward and risk from a mathematical point of view, we borrow two standard risk-sensitive performance measures from the financial literature: the mean-variance criterion \cite{markowitz1952portfolio} and the Sharpe ratio \cite{sharpe1994sharpe}.
\begin{definition}[Mean-Variance Criterion]
	The mean-variance criterion is defined as
	\begin{equation}
		J_\text{MV}(\theta) = J(\theta) - \chi \Lambda(\theta)
	\end{equation}
	where $\chi > 0$ is a constant that controls the trade-off between reward and risk.
\end{definition}

\begin{definition}[Sharpe Ratio]
	The Sharpe ratio is defined as
	\begin{equation}
		\Sh(\theta) = \frac{J(\theta)}{\sqrt{\Lambda(\theta)}}
	\end{equation}
\end{definition}
Let us remark that in the financial literature the ratio of the expected return and the semideviation is called Sortino ratio. In a risk-sensitive policy gradient algorithm, we try to approximate the optimal parameters that maximize these objective functions by updating the parameters following the gradient ascent directions. In the average-reward formulation, the ascent directions are given by
\begin{equation}
	\label{eq:gradient_mean_variance}
	\nabla_\theta J_{\text{MV}}(\theta) = \nabla_\theta \rho(\theta) - \chi \nabla_\theta \Lambda(\theta)
\end{equation}
for the mean-variance criterion, where
\begin{equation}
	\nabla_\theta \Lambda(\theta) =	\nabla_\theta \eta(\theta) - 2 \rho(\theta) \nabla_\theta \rho(\theta)
\end{equation}
and by
\begin{equation}
	\label{eq:gradient_sharpe_ratio}
	\nabla_\theta \Sh(\theta) = \frac{\eta(\theta) \nabla_\theta \rho(\theta) - \frac{1}{2} \rho(\theta) \nabla_\theta \eta(\theta)}{\Lambda(\theta) \sqrt{\Lambda(\theta)}}
\end{equation}
for the Sharpe ratio. Hence, in the risk-sensitive framework it is necessary to estimate to different gradients: $\nabla_\theta \rho(\theta)$ and $\nabla_\theta \eta(\theta)$. The equivalent expressions for the episodic setting can be obtained by replacing $\rho(\theta)$ (resp. $\eta(\theta)$) with $V_\theta(s_0)$ (resp. $U_\theta(s_0)$). The estimation of $\nabla_\theta \rho(\theta)$ (resp. $V_\theta(s_0)$) has been the focus of the last chapter. In the next sections, we discuss instead several techniques to estimate the new gradient $\eta(\theta)$ (resp. $U_\theta(s_0)$).

\section{Monte Carlo Policy Gradient}
For an episodic environment, the REINFORCE algorithm can be easily extended to the risk-sensitive framework described above \cite{tamar2012policy}. Indeed, it is sufficient to adapt its derivation for the average return to the second-moment of return
\begin{equation}
	U(\theta) = \E[H\sim p_\theta]{G(H)^2}
\end{equation}
Applying the likelihood ratio technique, we obtain
\begin{equation}
	\nabla_\theta U(\theta) = \E[H\sim p_\theta]{\nabla_\theta \log p_\theta(H) G(H)^2}
\end{equation}
Similarly to the risk-neutral framework, we can introduce a baseline without affecting the value of the gradient
\begin{equation}
	\nabla_\theta U(\theta) = \E[H\sim p_\theta]{\nabla_\theta \log p_\theta(H) (G(H)^2 - b)}
\end{equation}
In an episodic environment, we can estimate the gradient via its Monte Carlo estimate
\begin{equation}
	\label{eq:reinforce_gradient_2}
	\nabla_\theta U(\theta) \approx \frac{1}{M} \sum_{m = 0}^{M} \nabla_\theta \log p_\theta\left(h^{(m)}\right) \left[G\left(h^{(m)}\right)^2 - b\right]
\end{equation}
where $h^{(m)} = \{(s_t^{(m)}, a_t^{(m)})\}_{t = 0}^{T^{(m)}}$ are $M$ trajectories of the MDP sampled under policy $\pi_\theta$. When using a single trajectory, we obtain the following stochastic gradient estimate
\begin{equation}
	\nabla_\theta U(\theta) \approx \nabla_\theta \log p_\theta\left(h\right) \left[G\left(h\right)^2 - b\right]
\end{equation}
Again the baseline can be set so as to minimize the variance of the gradient estimate
\begin{equation}
	\label{eq:optimal_baseline_2}
	\widehat{b}_k^* = \frac{\sum^{M}_{m=1} \left[\partial_{\theta_k} \log p_\theta\left(h^{(m)}\right)\right]^2 G(h^{(m)})^2}{\sum^{M}_{m=1} \left[ \partial_{\theta_k} \log p_\theta\left(h^{(m)}\right)\right]^2}
\end{equation}
Combining this estimate with the one for the average return discussed in Section \ref{sec:MCPG} yields the risk-sensitive Monte Carlo Policy Gradient method, which is outlined in Algorithm \ref{algo:RSreinforce}.\\
In an episodic setting, as long as we can rewrite the objective function as expected values on all possible trajectories of the MDP, the likelihood ratio technique yields an analytical expression for its gradient. Hence, this approach can be generalized to more complex measures of risk commonly used in finance, such as the semivariance introduced above. Indeed, the semivariance can be rewritten as
\begin{equation}
	\text{SVar}(\theta) = \E[H \sim p_\theta]{\min\left\{G(H) - \E[H \sim p_\theta]{G(H)}, 0\right\}^2}
\end{equation}
Hence, by the likelihood ratio technique
\begin{equation}
	\nabla_\theta \text{SVar}(\theta) = \E[H \sim p_\theta]{\nabla_\theta \log p_\theta(H) \min\left\{G(H) - \E[H \sim p_\theta]{G(H)}, 0\right\}^2}
\end{equation}
From which we can easily derive a Monte Carlo estimate. Since the problems we will consider in the next chapters are not episodic. In the episodic setting, the extension of policy gradient algorithms to the risk-sensitive formulation does not present particular difficulties. For a more thorough presentation as well as some more advanced learning algorithms, we refer the interested reader to the extensive work of Tamar et Al.  \cite{tamar2013temporal}, \cite{tamar2013variance}, \cite{tamar2015policy}, \cite{chow2015risk}.
\begin{algorithm}[t]
	\caption{Risk-sensitive REINFORCE policy gradient estimate}
	\label{algo:RSreinforce}
	\begin{algorithmic}[1]
		\Require Policy parameterization $\theta$, number of trajectories $M$
		\Ensure Risk-sensitive REINFORCE policy gradient estimate
		\State Sample $M$ trajectories of the MDP following policy $\pi_\theta$
		\State Compute the optimal baseline for the return via Eq. (\ref{eq:optimal_baseline})
		\State Compute the optimal baseline for the squared return via Eq. (\ref{eq:optimal_baseline_2})
		\State Compute the gradient of the expected return via Eq. (\ref{eq:reinforce_gradient})
		\State Compute the gradient of the expected squared return via Eq. (\ref{eq:reinforce_gradient_2})
		\State Compute the risk-sensitive policy gradient via either Eq. (\ref{eq:gradient_mean_variance}) or (\ref{eq:gradient_sharpe_ratio})
	\end{algorithmic}
\end{algorithm}

\section{Policy Gradient Theorem}
In this section the policy gradient theorem is extended to the risk-sensitive framework. In the average reward formulation of the control problem, the theorem and its derivation are analogous to those for the risk-neutral framework. This will allow us to derive in a trivial way the risk-sensitive versions of all the learning algorithms seen in the previous chapter. The risk-sensitive policy gradient theorem for the average-reward formulation was first derived in \cite{prashanth2014actor} and the presentation of this section closely follows the original article.\\
On the other hand, obtaining an equivalent theorem for the discounted reward formulation is more challenging. In the article cited above, the authors prove the theorem under a very strong assumption on the dependence of rewards obtained at different time steps which is not verified in many applications, among which is the asset allocation problem that we will consider for the numerical applications. In the following sections, we will generalize this result by assuming that the reward can depend on both the initial state and the final state of the system. Furthermore, we will discuss the problems arising in the discounted setting and why it is not easy to derive an online policy gradient theorem in this case.\\
 In the original article the authors mostly considered the mean-variance criterion since all the algorithms they propose are easily adapted to the Sharpe ratio criterion. Here we take the opposite direction and present the algorithms for the Sharpe ratio, referring to their article for the mean-variance counterparts.\\
Let us consider a family of parametrized policies $\pi_\theta$, with $\theta
\in \Theta \subseteq \R^{D_\theta}$. The optimization problem then becomes
\begin{equation}
	\max_\theta \text{Sh}(\theta) = \frac{\rho(\theta)}{\sqrt{\Lambda(\theta)}}
\end{equation}
where we denoted by $\rho(\theta) = \rho_{\pi_\theta}$ and similarly for the other quantities. Using a policy gradient approach, the policy parameters are updated following the gradient ascent direction.

\subsection{Average Reward Formulation}
In the average reward formulation the gradient of the Sharpe ratio is
\begin{equation}
	\nabla_\theta \text{Sh}(\theta) = \frac{\eta(\theta) \nabla_\theta \rho(\theta) - \frac{1}{2} \rho(\theta) \nabla_\theta \eta(\theta)}{\Lambda(\theta) \sqrt{\Lambda(\theta)}}
\end{equation}
Hence, to compute the update direction, it is sufficient to estimate the various quantities appearing in this formula. For instance, the average reward $\rho(\theta)$, the average square reward $\eta(\theta)$ and the reward variance $\Lambda(\theta)$ can be approximated using exponentially weighted moving averages. On the other hand, the gradient of the average reward $\nabla_\theta \rho(\theta)$ is given by the standard policy gradient theorem \ref{thm:risk_neutral_policy_gradient}. The only term remaining is the gradient of the average square reward $\nabla_\theta \eta(\theta)$, which is provided by the risk-sensitive policy gradient theorem
\begin{theorem}[Risk-Sensitive Policy Gradient]
	Let $\pi_\theta$ be a differentiable policy. The policy gradient for the average square reward is given by
	\begin{equation}\label{eq:policy_gradient_theorem_W}
		\begin{split}
			\nabla_\theta \eta(\theta) &= \E[\substack{S\sim d_\theta \\
			A \sim \pi_\theta}]{\nabla_\theta \log \pi_\theta(S,A) W_\theta(S,A)}
		\end{split}
	\end{equation}
	where $d^\theta$ is the stationary distribution of the Markov chain induced by $\pi_\theta$.
\end{theorem}
\begin{proof}
	The proof is analogous to that of the risk-neutral version of theorem. From the basic relation between state-value function and action-value function, we have
		\begin{equation*}
			\begin{split}
				\nabla_\theta U_\theta(s) &= \nabla_\theta \int_{\A} \pi_\theta(s,a) W_\theta(s,a) da\\
					&= \int_{\A} \left[ \nabla_\theta \pi_\theta(s,a) W_\theta(s,a) + \pi_\theta(s,a) \nabla_\theta W_\theta(s,a)\right] da
			\end{split}
		\end{equation*}
		Using the Bellman expectation equation for $W_\theta$
		\begin{equation*}
			\begin{split}
				\nabla_\theta W_\theta(s,a) &= \nabla_\theta \left[ \calM(s,a) - \eta_\theta + \int_{\S} \calP(s,a,s') U_\theta(s') ds' \right]\\
				&= -\nabla_\theta \eta_\theta + \int_{\S} \calP(s,a,s') \nabla_\theta U_\theta(s') ds'
			\end{split}
		\end{equation*}
		Hence, plugging in the first equation
		\begin{equation*}
				\nabla_\theta U_\theta(s) = \int_{\A} \nabla_\theta \pi_\theta(s,a) W_\theta(s,a) da - \nabla_\theta \eta_\theta + \int_\A \pi_\theta(s,a) \int_{\S} \calP(s,a,s') \nabla_\theta U_\theta(s') ds'
		\end{equation*}
		Integrating both sides with respect to the stationary distribution $d^\theta$ and noting that, because of stationarity,
		\begin{equation*}
			\int_{\S} d^\theta(s) \int_{\A} \pi(s,a) \int_{\S} \calP(s,a,s') \nabla_\theta U(s') ds' da ds = \int_{\S} d^\theta(s) \nabla_\theta U_\theta(s) ds
		\end{equation*}
		we obtain the result
		\begin{equation*}
			\begin{split}
			\nabla_\theta \eta_\theta &= \int_{\S} d^\theta(s) \int_{\A} \nabla_\theta \pi_\theta(s,a) W_\theta(s,a) da ds\\
			&= \int_{\S} d^\theta(s) \int_{\A} \pi_\theta(s,a) \nabla_\theta \log\pi_\theta(s,a) W_\theta(s,a) da ds\\
			&= \E[\substack{S \sim d^\theta\\A \sim \pi_\theta}]{\nabla_\theta\log
					\pi_\theta(S,A) W_{\theta}(S, A)}
			\end{split}
		\end{equation*}
\end{proof}
The result is identical to that for the average reward, provided that we replace the action-value function $Q_\theta(s,a)$ by the square action-value function $W_\theta(s,a)$. As in the standard risk-neutral case, a state-dependent baseline can be introduced in the gradient without changing the result. In particular, by
using the average-adjusted square value function as baseline, we can replace the
average adjusted action-value functions with the following square advantage function
\begin{equation}
	B_\theta(s,a) = W_\theta(s,a) - U_\theta(s)
\end{equation}
Thus, the gradients can be written as
\begin{equation}\label{eq:pg_advantage_W}
	\nabla_\theta \eta(\theta) = \E[\substack{S\sim d_\pi \\
A \sim \pi}]{\nabla_\theta \log \pi_\theta(S,A) B_\theta(S,A)}
\end{equation}
From this result, following the exact same reasoning used in the risk-neutral framework, we can design a variety of risk-sensitive actor-critic algorithms, which
employ an approximation of the standard and of the square advantage functions to obtain a more accurate estimate of the objective function.

\subsection{Risk-Sensitive Actor-Critic Algorithm}
In \cite{prashanth2014actor}, starting from Eqs. (\ref{eq:actor_critic_pg}) and
(\ref{eq:pg_advantage_W}), the authors propose a TD(0) risk-sensitive actor-critic
algorithm for the average reward setting. The algorithm maintains two critics that estimate the average adjusted value-functions $V_\theta(S)$ and $U_\theta(S)$ respectively and are updated via a TD(0) temporal difference scheme. Let $\delta_n^A$ and $\delta_n^B$ be the TD errors for residual value
and square value functions
\begin{equation}
	\begin{split}
		\delta_t^A &= R_{t+1} - \widehat{\rho}_{t+1} + \widehat{V}(S_{t+1}) -
		\widehat{V}(S_t)\\
		\delta_t^B &= R_{t+1}^2 - \widehat{\eta}_{t+1} + \widehat{U}(S_{t+1}) -
		\widehat{U}(S_t)\\
	\end{split}
\end{equation}
where $\widehat{V}$, $\widehat{U}$, $\widehat{\rho}$ and $\widehat{\eta}$ are
unbiased estimate of $V_\theta$, $U_\theta$, $\rho(\theta)$ and $\eta(\theta)$
respectively. It is easy to show that $\delta_t^A$ and $\delta_t^B$ are
unbiased estimates of the advantage functions.
\begin{equation*}
	\begin{split}
		\E[\theta]{\delta_t^A|S_t = s, A_t = a} &= A_\theta(s, a)\\
		\E[\theta]{\delta_t^B|S_t = s, A_t = a} &= B_\theta(s, a)\\
	\end{split}
\end{equation*}
An estimate of the gradients can be obtained by replacing the advantage functions with the TD errors
\begin{equation}
	\begin{split}
		\nabla_\theta \rho(\theta) &\approx \nabla_\theta \log \pi_\theta(S_t, A_t) \delta_t^A\\
		\nabla_\theta \eta(\theta) &\approx \nabla_\theta \log \pi_\theta(S_t, A_t) \delta_t^B\\
	\end{split}
\end{equation}
The value functions are linearly approximated using some feature vectors
$\Phi_V: \S \to \R^{D_V}$ and $\Phi_U: \S \to \R^{D_U}$ as follows
\begin{equation}
	\begin{split}
		\widehat{V}(s) &= \psi_V^T \Phi_V(s)\\
		\widehat{U}(s) &= \psi_U^T \Phi_U(s)\\
	\end{split}
\end{equation}
Combining all these ingredients lead to the \gls{RSARAC}, the pseudocode of which is reported in Algorithm \ref{algo:RSARAC}.
Let us notice that the algorithm is a three time-scale stochastic approximation algorithm, where the learning rates, in addition to the usual Robbins-Monro conditions, should satisfy
\begin{equation*}
	\alpha_k < \beta_k < \gamma_k
\end{equation*}

\begin{algorithm}[t!]
	\caption{Risk-Sensitive Average Reward Actor-Critic algorithm}
	\label{algo:RSARAC}
	\begin{algorithmic}[]
		\Require {\\
			\begin{itemize}
				\item Initial actor parameters $\theta^0$
				\item Initial critics parameters $\psi_V^0$ and $\psi_U^0$
				\item Actor learning rate $\{\alpha_k\}$
				\item Critics learning rate $\{\beta_k\}$
				\item Averages learning rate $\{\gamma_k\}$
			\end{itemize}}
		\Ensure Approximation of the optimal policy $\pi_{\theta^*} \approx \pi_*$
		\begin{algorithmic}[1]
		\Repeat
			\State Observe tuple $<s_k, a_k, r_{k+1}, s_{k+1}>$ sampled from the MDP.
			\State Update averages
				\begin{equation*}
					\begin{split}
					\widehat{\rho}_{k+1} &= (1 - \gamma_k) \widehat{\rho}_k + 	\gamma_k r_{k+1}\\
					\widehat{\eta}_{k+1} &= (1 - \gamma_k) \widehat{\rho}_k + 	\gamma_k r_{k+1}^2\\
					\end{split}
				\end{equation*}
			\State Compute TD errors
				\begin{equation*}
					\begin{split}
						\delta_k^A &= r_{k+1} - \widehat{\rho}_{k+1} + (\psi_V^k)^T \Phi_V(s_{k+1}) - (\psi_V^k)^T \Phi_V(s_k)\\
						\delta_k^B &= r_{k+1}^2 - \widehat{\eta}_{k+1} + (\psi_U^k)^T \Phi_U(s_{k+1}) - (\psi_U^k)^T \Phi_U(s_k)\\
					\end{split}
				\end{equation*}
			\State Update critic parameters
				\begin{equation*}
					\begin{split}
						\psi_V^{k+1} &= \psi_V^k + \beta_k \delta_k^A \Phi_V(s_k)\\
						\psi_U^{k+1} &= \psi_U^k + \beta_k \delta_k^B \Phi_U(s_k)\\
					\end{split}
				\end{equation*}
			\State Update actor parameters $\theta^{k+1} = \theta^k + \alpha_k \widehat{\text{Sh}}(\theta_k) $.
			\State $k \leftarrow k + 1$
		\Until{converged}
		\end{algorithmic}
	\end{algorithmic}
\end{algorithm}

\subsection{Discounted Reward Formulation}
In the discounted reward formulation, we need to adapt the definition of the Sharpe ratio associated to a policy. Let us suppose that the system always start in the same initial state $s_0$, then we can introduce the start-state Sharpe ratio that can be achieved following policy $\pi_\theta$ as
\begin{equation}
	\text{Sh}(\theta) = \frac{V_\theta(s_0)}{\sqrt{\Lambda_\theta(s_0)}}
\end{equation}
where $V_\theta$ and $\Lambda_\theta$ are the state-value function and the variance-function introduced in Chapter \ref{ch:discrete_time_stochastic_optimal_control}.
Hence, the gradient of the Sharpe ratio is given by
\begin{equation}
	\nabla_\theta \text{Sh}(\theta) = \frac{U_\theta(s_0) \nabla_\theta V_\theta(s_0) - \frac{1}{2} V_\theta(s_0) \nabla_\theta U_\theta(s_0)}{\Lambda_\theta(s_0) \sqrt{\Lambda_\theta(s_0)}}
\end{equation}
The gradient of the state-value function $\nabla_\theta V_\theta(s_0)$ is given by the risk-neutral policy gradient theorem. Therefore, in order to approximate the gradient of the Sharpe ratio, we need to estimate the value functions $V_\theta(s_0)$, $U_\theta(s_0)$, the variance $\Lambda_\theta(s_0$), and the gradient of the square state-value function $U_\theta (s_0)$. The first three terms might be easily approximated using moving averages. On the other hand, estimating $\nabla_\theta U_\theta(s_0)$ in the same spirit of the policy gradient theorem is much more delicate.
\begin{theorem}[Risk-Sensitive Policy Gradient]
	Let $\pi_\theta$ be a differentiable policy. The policy gradient for the square state-value function is given by
	\begin{equation}
		\begin{split}
			\nabla_\theta U_\theta(s_0) =
			\mathbb{E}_{\substack{S \sim d_{\gamma^2}^\theta(s_0, \cdot)\\A \sim \pi_\theta(S,\cdot)\\S' \sim \calP(S,A,\cdot)}} [\nabla_\theta &\log	\pi_\theta(S,A) W_{\theta}(S, A)\\
			&+ 2 \gamma \calR(S,A) \nabla_\theta V_\theta(S') + 2 \gamma C_\theta(S, A) ]
		\end{split}
	\end{equation}
	where $d_{\gamma^2}^\theta(s_0, \cdot)$ is the $\gamma^2$-discounted visiting distribution over states starting from the initial state $s_0$ and following policy $\pi_\theta$
		\begin{equation}
			d_{\gamma^2}^\theta(s_0, x) = \sum_{k=0}^{\infty} \gamma^{2k} \calP_\theta^{(k)}(s_0, x)
		\end{equation}
\end{theorem}
\begin{proof}
	From the basic relation between square state-value function and the square action-value function, we have
	\begin{equation*}
		\begin{split}
			\nabla_\theta U_\theta(s) &= \nabla_\theta \int_{\A} \pi_\theta(s,a) W_\theta(s,a) da\\
				&= \int_{\A} \left[ \nabla_\theta \pi_\theta(s,a) W_\theta(s,a) + \pi_\theta(s,a) \nabla_\theta W_\theta(s,a)\right] da
		\end{split}
	\end{equation*}
	Hence, using the Bellman expectation equation for $W_\theta$
	\begin{equation*}
		\begin{split}
			\nabla_\theta W_\theta(s,a) &= \nabla_\theta \left[ \calM(s,a) + 2 \gamma \calR(s,a) T_a V_\pi(s) + 2 \gamma C_\theta(s, a) + \gamma^2 T_a U_\theta(s)\right]\\
			&= 2 \gamma \calR(s,a) \nabla_\theta T_a V_\theta(s) + 2 \gamma \nabla_\theta C_\theta(s, a) + \gamma^2 \nabla_\theta T_a U_\theta(s)\\
			&= \int_{\S} \calP(s,a,s') \left[2 \gamma \calR(s,a) \nabla_\theta V_\theta(s') + 2 \gamma \nabla_\theta C_\theta(s, a) + \nabla_\theta U_\theta(s')\right] ds'
		\end{split}
	\end{equation*}
	where we assumed to be able to exchange the gradient and the integral. Plugging in the first equation and exploiting the fact that $\int_\S \calP(s,a,s') ds' = 1$, we obtain
	\begin{equation*}
		\begin{split}
			\nabla_\theta U_\theta(s) = \int_{\A} \pi_\theta(s,a) &\int_{\S} \calP(s,a,s') [ \nabla_\theta \log \pi_\theta(s,a) W_\theta(s,a)\\
			&+ 2 \gamma \calR(s,a) \nabla_\theta V_\theta(s') + 2 \gamma \nabla_\theta C_\theta(s, a) + \nabla_\theta U_\theta(s') ] ds' da
		\end{split}
	\end{equation*}
	Unrolling $\nabla_\theta U_\theta$ infinite times and denoting by $\calP_\theta^{(k)}(s, x)$ the probability of going from state $s$ to state $x$ in $k$ steps under policy $\pi_\theta$, we obtain
	\begin{equation*}
		\begin{split}
			\nabla_\theta U_\theta(s) = \sum_{k=0}^\infty \gamma^{2k} \int_\S \calP_\theta^{(k)}(s, x) &\int_\A \pi_\theta(x,a) \int_{\S} \calP(x,a,x') [\nabla_\theta \log \pi_\theta(x,a) W_\theta(x,a)\\
						&+ 2 \gamma \calR(x,a) \nabla_\theta V_\theta(x') + 2 \gamma \nabla_\theta C_\theta(x, a)]  dx' da dx
		\end{split}
	\end{equation*}
	Defining the $\gamma^2$-discounted visiting distribution of state $x$ starting from state $s$ as
	\begin{equation*}
		d_{\gamma^2}^\theta(s, x) = \sum_{k=0}^{\infty} \gamma^{2k} \calP_\theta^{(k)}(s, x)
	\end{equation*}
	we have the result
	\begin{equation*}
		\begin{split}
			\begin{split}
				\nabla_\theta U_\theta(s) = \int_\S d_{\gamma^2}^\theta(s, x) &\int_\A \pi_\theta(x,a) \int_{\S} \calP(x,a,x') [\nabla_\theta \log \pi_\theta(x,a) W_\theta(x,a)\\
							&+ 2 \gamma \calR(x,a) \nabla_\theta V_\theta(x') + 2 \gamma \nabla_\theta C_\theta(x, a)]  dx' da dx
			\end{split}
		\end{split}
	\end{equation*}
\end{proof}
Compared to risk-sensitive policy gradient theorem in the average reward formulation, we have two additional terms: one term depending on the gradient of the state-value function at the next state and another term depending on the covariance between the one-step reward and the successive return. These terms are very difficult to approximate online in a continuing environment. Therefore, the result is of practical interest only for an episodic environment where the experiments have a finite (possibly random) lifespan. Since the application we considered does not fall in this category, we will not discuss any risk-sensitive algorithm for the discounted formulation.


\subsubsection{Merging node-trees}

%In lines \ref{algo:joina}-\ref{algo:joinb}, the pseudocode for merging node-trees only has a cost proportional to the number of cluster mergers, which is defined by the complexity of the Union-Find algorithm.
Merging node-trees is more complex than merging cluster-trees. The additional cost comes from the depth-first-searches (DFS's) related to the (re)calculation of the node parities and node delays in line \ref{algo:pdc}. \todo[inline]{argue that we only care about mergers of an odd number of odd clusters}.

\todo[inline]{define the meaning of step, link to discussion on max odd root, clarify if there is a constant missing}
Consider merging $k$ trees $\{\mathcal N_i\}_{i=1}^k$ with $k$ odd and let $\{m_i\}_{i=1}^k$ represent the number of nodes in each tree. Then the number of steps required to merge the $k$ trees is $\sum_{i=1}^km_i-\max_{i\in [1,k]}{m_i}$. From this observation, we define by $C(N)$ the total number of steps in the merge required

which can be expressed recursively by:

\begin{equation}
C(\mathcal N) = \sum_{i=1}^k C(\mathcal N_i) + \sum_{i=1}^km_i-\max_{i\in [1,k]}{m_i}
\end{equation}

In the following we show that if we let $n=\sum_{i=1}^km_i$ be the total number of nodes in the merged node tree, $C(\mathcal N)<n\log n$.
\todo[inline]{Present induction base}
Consider that the induction hypothesis holds up to some odd number $n-2$, then the following inequality holds:
\begin{equation}
C(\mathcal N) \leq \sum_{i=1}^k m_i\log m_i + \sum_{i=1}^km_i-\max_{i\in [1,k]}{m_i}
\end{equation}

\todo[inline]{use Jensen's inequality to forget about all except for two values}
Let $m\leq n/2$.
\begin{equation}
C(\mathcal N) \leq f(m)=(n-m)\log (n-m) + m\log m+ m
\end{equation}
Consider the first two derivatives of $f(m)$ with respect to $m$:
\begin{equation}
    f'(m) = -\log (n-m)+\log m + 1
\end{equation}
\begin{equation}
    f''(m) = (1/(n-m)) + 1/m
\end{equation}
The function is continuous for $0<m<n$ and the second derivative is strictly positive, which means that $f$ is convex and takes the maximum at a extreme value.

%Line \ref{line:grow} corresponds with  the DFS's on the node-tree results, it has an addional cost which is investigated in this section.

%, and (B) the DFS related to the growth of a cluster in line \ref{algo:grow}. We dub the two parts the \textbf{delay cost} and the \textbf{growth cost}, respectively. The $\Nodejoin$ operation in lines \ref{algo:joina}-\ref{algo:joinb} only has a linear addition to the cost.
% The cost of each step the DFS (\eqref{}, \eqref{} and \eqref{}) is independent of the tree size.
%The cost of the DFS's is proportional to the total number of nodes encountered in the DFS's. In the following we bound this number, which we denote by $N_\delta$.
% Even node-trees that join with multiple node-trees within the same growth iteration and the final set of even node-trees whose clusters are peeled thus do not count towards $N_\delta$.

%Let an odd node-tree be denoted by $\oset$, an even node-tree by $\eset$, and an even node-tree that counts towards the delay cost by $\bm{\eset}$. To find $N_\delta$, we analyze cluster growth by a time-reversed approach; starting from a single cluster at the end of growth, and move back in time to find its \emph{node-tree decomposition}.

%\begin{definition}\label{def:fragmentation}

%  Let the \textbf{fragmentation} $\frag_\oset$ of some odd node-tree $\pr{k}\oset$ return its two most recently joined predecessor node-trees, which are $\pr{k-1}\oset$ and $\pr{k-1}\eset$
%  \begin{equation}\label{eq:pfo}
%    \frag_\oset(\{\pr{k}\oset\}) = \{\pr{k-1}\oset, \pr{k-1}\eset \},
%  \end{equation}
%  where the prefix $k$ indicate the \textbf{generation} to which the node-tree belongs. Let $\frag_\eset$ fragment an even node-tree $\pr{k}\eset$ to some even number $\nu$ of its predecessor odd node-trees
%  \begin{equation}\label{eq:pfe}
%    \frag_\eset(\{\pr{k}\eset\}) =\{\pr{k}\oset_1,...,\pr{k}\oset_{\nu} \},
%  \end{equation}
%  which have joined to $\pr{k}\eset$ within the same growth iteration. As an even node-tree does not grow, it belongs to the same generation as its odd predecessors. Since $\frag_\oset$ and $\frag_\eset$ operate on node-trees of different parties, both can be combined as a \textbf{fragmentation step} $\frag$, which takes an odd node-tree $\pr{k}\oset$ and returns its $\nu+1$ predecessors
%  \begin{multline}\label{eq:fstep}
%    \frag(\{\pr{k}\oset\}) = \frag_\eset(\frag_\oset(\{\pr{k}\oset\})) =  \\ \{\pr{k-1}\oset_0,\pr{k-1}\oset_1,...,\pr{k-1}\oset_\nu\},
%  \end{multline}
%  where $\pr{k-1}\oset$ from \Cref{eq:pfo} is now labelled as $\pr{k-1}\oset_0$.
%\end{definition}

%Note that node-trees of the same \emph{generation} might not have been constructed in the same growth \emph{iteration}. An odd cluster grow for many iterations, while its node-tree does not change, before it merges with another node-tree of the same generation. We use the notation $\frag^{(2)}(\pr{k}\oset)$ to indicate that two fragmentation steps are applied on $\pr{k}\oset$ to obtain its predecessors of generation $k-2$. Furthermore, let $\frag_\oset^{(i)}(\nset)$ and be equivalent to $\frag_\oset(\frag^{(i-1)}(\nset))$.

%\Figure[hbt](topskip=0pt, botskip=0pt, midskip=0pt){figures/tikz/build/main-figure7.pdf}{
%  The \emph{fragmentation} of node-tree $\pr{k+1}\oset$ into its predecessor node-trees, where the prefix $k+1$ indicates its \emph{generation}. The fragmentation step $\frag(\pr{k+1}\oset)$ returns $\nu+1$ odd predecessors of generation $k$. Fragmentation step $\frag$ can be separated into $\frag_\oset(\pr{k+1}\oset)$ that returns an odd predecessor $\pr{k}\oset_0$ and even predecessor $\pr{k}\eset$, and subsequently the $\frag_\eset(\pr{k}\eset)$, which returns $\nu$ odd predecessors of generation $k$. The figure depicts a fragmentation step with $\nu=2$.\label{fig6}}

%\begin{definition}\label{def:decomposition}
%  Let the \textbf{decomposition} of a node-tree $\nset$ be a series of $\mu$ fragmentation steps on $\nset$, such that the output of $\frag^{(\mu)}$ is a set of node-trees that have no predecessors.
%\end{definition}

%Using node-tree decomposition, we aim to find the fragmentations that maximizes the number and size of $\bm{\eset}$ node-trees. The definitions of $\frag_\oset$ and $\frag_\eset$ matches the rules of \emph{Odd-Rooted Join} and \emph{Root List Replacement} such that each of the even node-trees in the output of $\frag_\oset$ is a $\bm{\eset}$ node-tree. The maximum value for $N_\delta$ can thus be computed using the decomposition of a maximally large odd node-tree $\pr{\mu}\oset$.
%\begin{equation}\label{eq:npdc}
%  N_\delta = 2\sum_{k=1}^\mu{ \sum_{ \pr{\mu-k}\eset \in \frag_\oset^{(k)}(\pr{\mu}\oset) }{ \abs{\pr{\mu-k}\eset}} }.
%\end{equation}
%The inner sum accounts for the combined size of all even node-trees during a single fragmentation step $\frag_\oset^{(k)}$. The outer sum loops over all $\mu$ fragmentation steps. Finally, the $2$ refers to the individual DFS's of the parity and delay calculations. Note that in the decomposition of $\pr{\mu}\oset$, the lowest generation of node-trees outputted by $\frag^{(\mu)}(\pr{\mu}\oset)$ is $1$.

%We will use two simplifications to \Cref{eq:npdc} to find $N_\delta$. Junction-nodes are initiated on the tangent of two node radii belonging to separate node-trees when merging into one. With increasing number of fragmentations, the total number of nodes in the fragmented set must therefore decrease. We thus change \Cref{eq:npdc} into the inequality
%\begin{equation}\label{eq:npdc2}
%  N_\delta \leq 2\sum_{k=1}^\mu{ \sum_{ \pr{\mu-k}\eset \in \frag_\oset^{(k)}(\pr{\mu}\oset) }{ \abs{\pr{\mu-k}\eset}}}.
%\end{equation}

%The existence of junction-nodes will be ignored in the remainder of this section, which does not compromise the inequality. If only syndrome-nodes exist, $\pr{k}\oset$'s size must equal the sum of sizes of its predecessors in $\frag(\pr{k}\oset)$. The size of each of the $\nu+1$ predecessors $\pr{k}\oset_i$ can be represented by the \textbf{fragmentation ratio} $\lambda$
%\begin{equation}\label{eq:ratio}
%  \pr{k}\lambda_i = \frac{\abs{\pr{k-1}\oset_i}}{\abs{\pr{k}\oset}}, \hspace{0.5cm} \sum_{i=0}^{\nu}{\pr{k}\lambda_i} = 1.
%\end{equation}
%Secondly, we assume that vertex-trees do not increase in size, such that $\abs{\nset}=\abs{\vset}$. Normally, the number of nodes in a cluster is bounded by the number of vertices $\abs{\nset}\leq \abs{\vset}$, as non-trivial vertices can be added to the node, which increases the node radius. By this assumption, the vertex-tree can only increase in size due to a merger between clusters, and nodes are effectively not allowed to increase in radius. While this is not possible during realistic cluster growth, this assumption allows us to further simplify \Cref{eq:npdc2}.

%To find the upper bound in $N_\delta$, we are now tasked to find: (a) $\nu$, the number of predecessors in $\frag_\eset$, (b) the fragmentation ratios $\{\lambda_0, ..., \lambda_\nu\}$, (c) the number of fragmentation generations $\mu$, and (d) the size of $\pr{\mu}\oset$.

%\begin{lemma}\label{lem:evenconstant}
%  For constant fragmentation ratios $\pr{k}\lambda_i = \lambda_i$ during all generations $1\leq k<\mu$, the inner sum of \Cref{eq:npdc} is constant:
%  \begin{equation*}
%    \sum_{ \pr{\mu-k}\eset \in \frag_\oset^{(k)}(\pr{\mu}\oset) }{ \abs{\pr{\mu-k}\eset}} = C \hspace{1em}\forall k.
%  \end{equation*}
%\end{lemma}
%\begin{proof}
%  For $k=1$, there is an even-parity predecessor $\pr{\mu-1}\eset$ of size
%  \begin{equation*}
%    \abs{\pr{\mu-1}\eset} = (1 - \lambda_0)\abs{\pr{\mu}\oset}.
%  \end{equation*}
%  For $k=2$, every odd node-tree in $\{\pr{\mu-1}\oset_0,... ,\pr{\mu-1}\oset_\nu\}$ is fragmented by $\frag_\oset$ to an even predecessor $\pr{\mu-2}\eset_i$ for $i \in \{0,...,\nu \}$, such that
%  \begin{equation*}
%    \sum_{i=0}^{\nu}{\abs{\pr{\mu-2}\eset_i}}  = \sum_{i=0}^{\nu}{\lambda_i(1 - \lambda_0)\abs{\pr{\mu}\oset}}= (1 - \lambda_0)\abs{\pr{\mu}\oset}.
%  \end{equation*}
%  The same is true for all generations $1\leq k<\mu$.
%\end{proof}

%\begin{theorem}\label{the:fragnumber}
%  The upper bound of $N_\delta$ is obtained for $\nu=2$, such that $\frag_\eset$ of even node-tree $\pr{k}\eset$ returns two odd predecessors.
%\end{theorem}
%\begin{proof}
%  The sum of even node-tree sizes in every generation is constant per \Cref{lem:evenconstant}. Thus, the upper bound in \Cref{eq:npdc2} is obtained by the largest possible $\mu$. As $\nu$ increases the number of odd node-trees in each $\frag^{(k)}_\oset$, the average size of these odd node-trees decreases. Since the size of a node-tree is proportional to the number of predecessor generations, we find that
%  \begin{equation*}
%    \mu \propto \frac{1}{\nu}.
%  \end{equation*}
%  Hence, the upper bound in \Cref{eq:npdc2} exists in the minimal value of $\nu$, which is $\nu = 2$.
%\end{proof}

%Using \Cref{the:fragnumber}, a fragmentation step on an odd cluster $\frag(\pr{k+1}\oset)$ now returns $\{\pr{k}\oset_0, \pr{k}\oset_1, \pr{k}\oset_2\}$, where $\pr{k}\oset_1, \pr{k}\oset_2$ are predecessors of the even node-tree $\pr{k}\eset$.

%\begin{lemma}\label{lem:chrono}
%  The node-tree size of $\pr{k}\oset_0$ must be smaller than or equal to $\pr{k}\oset_1, \pr{k}\oset_2$, such that $\lambda_1 \geq \lambda_0 \leq \lambda_2$.
%\end{lemma}
%\begin{proof}
%  The partial fragmentations must occur in the order of first \Cref{eq:pfo}, then \eqref{eq:pfe}, as \eqref{eq:pfe} requires an even node-tree that is returned by \eqref{eq:pfo}. In terms of cluster growth, the vertex-trees $\vset_1, \vset_2$, corresponding to $\pr{k}\oset_1, \pr{k}\oset_2$, must merge before the combined vertex-tree can merge with $\vset_0$, which corresponds to $\pr{k}\nset_0$. As a result of Weighted Growth, $\abs{\vset_1}$ and $\abs{\vset_2}$ must be smaller or equal to $\abs{\vset_0}$, such that
%  \begin{equation*}
%    \abs{\vset_1}\geq \vset_0 \leq\abs{\vset_2}.
%  \end{equation*}
%  If this condition is not met, the cluster of $\vset_0$ grows first and merges with either $\vset_1$ or $\vset_2$, and the causality of events is disturbed. Since we assumed $\abs{\nset}=\abs{\vset}$, this can be translated to
%  \begin{equation*}
%    \abs{\nset_1}\geq \nset_0 \leq\abs{\nset_2},
%  \end{equation*}
%  and subsequently to the fragmentation ratios.
%\end{proof}

%\begin{theorem}\label{the:ratios}
%  The upper bound for $N_\delta$ is obtained via the fragmentation ratios $\lambda_0 = \lambda_1 = \lambda_2 = \nicefrac{1}{3}$.
%\end{theorem}
%\begin{proof}
%  The ratios $\{\lambda_0, \lambda_1, \lambda_2\}$ can be found by maximizing the size of the even node-tree $\pr{k}\eset$ in each fragmentation step, which is
%  \begin{equation*}
%    \abs{\pr{k}\eset} = (\lambda_1 + \lambda_2)\abs{\pr{k-1}\oset}.
%  \end{equation*}
%  Since $ \lambda_1 \geq \lambda_0 \leq \lambda_2$ per \Cref{lem:chrono}, the largest values for $\lambda_1, \lambda_2$ possible are equal to $\lambda_0$.
%\end{proof}

%The last unknown parameters in finding the upper bound of $N_\delta$ in are $\mu$ and $\abs{\pr{\mu}\oset}$.

%\begin{theorem}\label{the:km}
%  For $\nu = 2$ and $\lambda_i = \{\nicefrac{1}{3},\nicefrac{1}{3},\nicefrac{1}{3}\}$, the maximum number of fragmentation generations is $\mu = \log_3{\abs{\pr{\mu}\oset}}$.
%\end{theorem}
%\begin{proof}
%  In every generation, all node-trees are fragmented into 3 predecessors that are $\nicefrac{1}{3}$ the size of their common successor. The series of $\mu$ fragmentation steps is thus simply $\mu$ divisions of the node-tree $\pr{\mu}\oset$ in 3 parts until all predecessors have size 1, at which point a node-tree cannot be fragmented.
%\end{proof}

%The maximum size of the odd node-tree $\pr{\mu}\oset$ is bounded by the system size $n$, the number of qubits on the lattice. Collecting \Cref{the:fragnumber,the:ratios,the:km} and filling in \Cref{eq:npdc2}, we find that

%\begin{align*}
%  \nonumber N_\delta &\leq 2\sum_{k=1}^\mu{ \sum_{ \pr{\mu-k}\eset \in \frag_\oset^{(k)}(\pr{\mu}\oset) }{ \abs{\pr{\mu-k}\eset}}  } \\
%  \nonumber         &\leq 2\sum_{k=1}^{\log_3{\abs{\pr{\mu}\oset}}} \frac{2}{3}\abs{\pr{\mu}\oset}\\
%                    &\leq \frac{4}{3}\abs{\pr{\mu}\oset}\log_3{\abs{\pr{\mu}\oset}} \\
%                    &\leq \frac{4}{3}n \log_3{n}.
%\end{align*}

%The worst-case time complexity of related to the delay cost is thus $\Omega(n\log{n})$.

%\subsection{Growth cost}\label{sec:growthcost}

%To grow a cluster represented by a node-tree $\nset$, a depth-first search (DFS) is performed on the node-tree to find all nodes with zero delay. The total cost of these DFS's is proportional to the total number of nodes encountered during these DFS's, which we dub $N_g$. Using \Cref{def:fragmentation,def:decomposition}, the cost of growth of a node-tree $\pr{\mu}\oset$ is proportional to the sum of sizes of all odd node-trees in the decomposition of $\pr{\mu}\oset$:
%\begin{equation}\label{eq:ngrow}
%  N_g = 2\sum_{k=1}^\mu{ \sum_{ \pr{\mu-k}\oset \in \frag^{(k)}(\pr{\mu}\oset) }{ \abs{\pr{\mu-k}\oset}} }.
%\end{equation}
%Again, we assume that no trivial vertices are added to a cluster or $|\nset| = |\vset|$ such that \Cref{eq:ngrow} becomes an upper bound. As a result of \emph{Odd-Rooted Join} and \emph{Root List Replacement}, the upper bound is obtained in the largest $\mu$. This is again achieved through $\nu = 2$. For every fragmentation of some odd node-tree $\pr{k-1}\oset$ into $\{\pr{k}\oset_0, \pr{k}\oset_1, \pr{k}\oset_2\}$, all three predecessors add to $N_g$ if they have grown. As a result of Weighted Growth, this is the case when $\abs{\vset_0}\approx \abs{\vset_1}\approx\abs{\vset_2}$ such that $\lambda_0 \approx \lambda_1 \approx \lambda_2\approx \nicefrac{1}{3}$. For these values of $\nu$ and $\lambda$, we can apply \Cref{the:km} for $\mu$. For $|\nset| = |\vset|$, the sum of even node-tree sizes in every $\frag_\oset^{(k)}$ is exactly $\abs{\pr{\mu}\oset}$, and we find that
%\begin{align*}
%  \nonumber N_g &\leq 2\sum_{k=1}^\mu{ \sum_{ \pr{\mu-k}\oset \in \frag^{(k)}(\pr{\mu}\oset) }{ \abs{\pr{\mu-k}\oset}}  } \\
%  \nonumber         &\leq 2\sum_{k=1}^{\log_3{\abs{\pr{\mu}\oset}}} \abs{\pr{\mu}\oset}\\
%                    &\leq 2\abs{\pr{\mu}\oset}\log_3{\abs{\pr{\mu}\oset}}\\
%                    &\leq 2 n \log_3{n},
%\end{align*}
%which again corresponds to a worst-case time complexity $\Omega(n\log{n})$.