\documentclass[12pt]{article}
\usepackage[utf8]{inputenc}
\usepackage[margin=1.0in]{geometry}
\linespread{1.3}
\usepackage{amssymb}
\usepackage{amsmath}
\begin{document}

\section*{2D Euler Fourier Transform}

Consider 2D Euler:
\[
\mathbf{u}_t+\mathbf{u}\cdot\nabla\mathbf{u}=-\nabla{p},\qquad\nabla\cdot{u}=0
\]
\noindent where $\mathbf{u}(x,t)$ is a 2D vector that is periodic in both dimensions and $p$ is pressure.

To find an expression for $p$ we take the divergence of both sides:
\begin{align*}
    \nabla\cdot(-\nabla{p})
    &=\nabla\cdot(\mathbf{u}_t+\mathbf{u}\cdot\nabla\mathbf{u})\\
    &=\nabla\cdot\mathbf{u}_t+\nabla\cdot(\mathbf{u}\cdot\nabla\mathbf{u})
\end{align*}
Since $\nabla\cdot{u}=0$, we have:
\begin{align*}
    \nabla\cdot\begin{bmatrix}-\frac{\partial{p}}{\partial{x}}\\-\frac{\partial{p}}{\partial{y}}\end{bmatrix}
    =&\nabla\cdot\begin{bmatrix}u^x\frac{\partial{u}^x}{\partial{x}}+u^y\frac{\partial{u}^x}{\partial{y}}\\
    u^x\frac{\partial{u}^y}{\partial{x}}+u^y\frac{\partial{u}^y}{\partial{y}}\end{bmatrix}\\
    -\left(\frac{\partial^2p}{\partial{x}^2}+\frac{\partial^2p}{\partial{y}^2}\right)
    =&\frac{\partial{u}^x}{\partial{x}}\frac{\partial{u}^x}{\partial{x}}+u^x\frac{\partial^2u^x}{\partial{x}^2}+\frac{\partial{u}^y}{\partial{x}}\frac{\partial{u}^x}{\partial{y}}+u^y\frac{\partial^2u^x}{\partial{x}\partial{y}}\\
    &+\frac{\partial{u}^x}{\partial{y}}\frac{\partial{u}^y}{\partial{x}}+u^x\frac{\partial^2u^y}{\partial{x}\partial{y}}+\frac{\partial{u}^y}{\partial{y}}\frac{\partial{u}^y}{\partial{y}}+u^y\frac{\partial^2u^y}{\partial{y}^2}
\end{align*}
Notice that
\[
u^x\frac{\partial^2u^x}{\partial{x}^2}+u^x\frac{\partial^2u^y}{\partial{x}\partial{y}}=u^x\frac{\partial}{\partial{x}}(\nabla\cdot{u})=0
\]
and
\[
u^y\frac{\partial^2u^x}{\partial{x}\partial{y}}+u^y\frac{\partial^2u^y}{\partial{y}^2}=u^y\frac{\partial}{\partial{y}}(\nabla\cdot{u})=0.
\]
So,
\[
    -\left(\frac{\partial^2p}{\partial{x}^2}+\frac{\partial^2p}{\partial{y}^2}\right)
    =\frac{\partial{u}^x}{\partial{x}}\frac{\partial{u}^x}{\partial{x}}+\frac{\partial{u}^y}{\partial{x}}\frac{\partial{u}^x}{\partial{y}}+\frac{\partial{u}^x}{\partial{y}}\frac{\partial{u}^y}{\partial{x}}+\frac{\partial{u}^y}{\partial{y}}\frac{\partial{u}^y}{\partial{y}}.
\]
Since $\mathbf{u}$ is periodic, we can represent it as a Fourier series
\[
\mathbf{u}(x,t)=\sum_{\mathbf{k}}\mathbf{u_k}(t)e^{i\mathbf{k\cdot{x}}}
\]
where $\mathbf{k}$ is a 2D wavevector and we sum over all possible integer-valued wavevectors. We can do the same for $p$:
\[
p=\sum_{\mathbf{k}}p_\mathbf{k}(t)e^{i\mathbf{k\cdot{x}}}
\]
Now we can write the left-hand side as
\begin{align*}
 -\left(\frac{\partial^2p}{\partial{x}^2}+\frac{\partial^2p}{\partial{y}^2}\right)=&-\left(\sum_\mathbf{k}-k_x^2p_\mathbf{k}e^{i\mathbf{k\cdot{x}}}+\sum_\mathbf{k}-k_y^2p_\mathbf{k}e^{i\mathbf{k\cdot{x}}}\right)\\
 =&\sum_\mathbf{k}(k_x^2+k_y^2)p_\mathbf{k}e^{i\mathbf{k\cdot{x}}}\\
 =&\sum_\mathbf{k}|\mathbf{k}|^2p_\mathbf{k}e^{i\mathbf{k\cdot{x}}}
\end{align*}
and the right-hand side as
\[
\left(\sum_\mathbf{p}ip_xu_\mathbf{p}^xe^{i\mathbf{p\cdot{x}}}\right)\left(\sum_\mathbf{q}iq_xu_\mathbf{q}^xe^{i\mathbf{q\cdot{x}}}\right)
+\left(\sum_\mathbf{p}ip_xu_\mathbf{p}^ye^{i\mathbf{p\cdot{x}}}\right)\left(\sum_\mathbf{q}iq_yu_\mathbf{q}^xe^{i\mathbf{q\cdot{x}}}\right)\\
\]
\[
+\left(\sum_\mathbf{p}ip_yu_\mathbf{p}^xe^{i\mathbf{p\cdot{x}}}\right)\left(\sum_\mathbf{q}iq_xu_\mathbf{q}^ye^{i\mathbf{q\cdot{x}}}\right)
+\left(\sum_\mathbf{p}ip_yu_\mathbf{p}^ye^{i\mathbf{p\cdot{x}}}\right)\left(\sum_\mathbf{q}iq_yu_\mathbf{q}^ye^{i\mathbf{q\cdot{x}}}\right)\]
\begin{align*}
&=-\sum_\mathbf{k}\sum_\mathbf{p+q=k}(p_xq_xu_\mathbf{p}^xu_\mathbf{q}^x+p_xq_yu_\mathbf{p}^yu_\mathbf{q}^x+p_yq_xu_\mathbf{p}^xu_\mathbf{q}^y+p_yq_yu_\mathbf{p}^yu_\mathbf{q}^y)e^{i\mathbf{k\cdot{x}}}\\
&=-\sum_\mathbf{k}\sum_\mathbf{p+q=k}(\mathbf{p}\cdot{u}_\mathbf{q})(\mathbf{q}\cdot{u}_\mathbf{p})e^{i\mathbf{k\cdot{x}}}
\end{align*}
Since $\nabla\cdot\mathbf{u}=0$,
\begin{align*}
0&=\sum_\mathbf{k}(ik_xu_\mathbf{k}^x+ik_yu_\mathbf{k}^y)e^{i\mathbf{k\cdot{x}}}\\
&=i\sum_\mathbf{k}(\mathbf{k}\cdot{u}_\mathbf{k})e^{i\mathbf{k\cdot{x}}}
\end{align*}
We multiply by $e^{-i\mathbf{j\cdot{x}}}$ and integrate:
\begin{align*}
0&=\int^{2\pi}_0i\sum_\mathbf{k}(\mathbf{k}\cdot{u}_\mathbf{k})e^{i(\mathbf{k-j})\cdot\mathbf{x}}d\mathbf{x}\\
&=i(\mathbf{j}\cdot{u}_\mathbf{j})
\end{align*}
Since $i(\mathbf{q}\cdot{u}_\mathbf{q})=0$ and $i\neq0$, $\mathbf{q}\cdot{u}_\mathbf{q}=0$. Therefore,  $\mathbf{p}\cdot{u}_\mathbf{q}=\mathbf{p}\cdot{u}_\mathbf{q}+\mathbf{q}\cdot{u}_\mathbf{q}=\mathbf{k}\cdot{u}_\mathbf{q}$ and, similarly, $\mathbf{q}\cdot{u}_\mathbf{p}=\mathbf{k}\cdot{u}_\mathbf{p}$. We can now rewrite our equation.
\[
\sum_\mathbf{k}|\mathbf{k}|^2p_\mathbf{k}e^{i\mathbf{k\cdot{x}}}=-\sum_\mathbf{k}\sum_\mathbf{p+q=k}(\mathbf{k}\cdot{u}_\mathbf{q})(\mathbf{k}\cdot{u}_\mathbf{p})e^{i\mathbf{k\cdot{x}}}
\]
Again, we multiply by $e^{-i\mathbf{j\cdot{x}}}$ and integrate:
\begin{align*}
\int^{2\pi}_0\sum_\mathbf{k}|\mathbf{k}|^2p_\mathbf{k}e^{i(\mathbf{k-j})\cdot\mathbf{x}}d\mathbf{x} &=-\int^{2\pi}_0\sum_\mathbf{k}\sum_\mathbf{p+q=k}(\mathbf{k}\cdot{u}_\mathbf{q})(\mathbf{k}\cdot{u}_\mathbf{p})e^{i(\mathbf{k-j})\cdot\mathbf{x}}d\mathbf{x}\\
|\mathbf{j}|^2p_\mathbf{j}&=-\sum_\mathbf{p+q=j}(\mathbf{j}\cdot{u}_\mathbf{q})(\mathbf{j}\cdot{u}_\mathbf{p})
\end{align*}
So we can express $p$ as
\[
p_\mathbf{k}=-\frac{1}{|\mathbf{k}|^2}\sum_\mathbf{p+q=k}(\mathbf{k}\cdot{u}_\mathbf{q})(\mathbf{k}\cdot{u}_\mathbf{p})
\]
If we return to the original equation, we can find a Fourier representation for each term.
\[
\mathbf{u}_t =\begin{bmatrix}\frac{\partial{u^x}}{\partial{t}}\\\frac{\partial{u^y}}{\partial{t}}\end{bmatrix}=\sum_\mathbf{k}\frac{d\mathbf{u_k}}{dt}e^{i\mathbf{k\cdot{x}}}
\]
\begin{align*}
\mathbf{u}\cdot\nabla\mathbf{u}=\begin{bmatrix}u^x\frac{\partial{u^x}}{\partial{x}}+u^y\frac{\partial{u^x}}{\partial{y}}\\u^x\frac{\partial{u^y}}{\partial{x}}+u^y\frac{\partial{u^y}}{\partial{y}}\end{bmatrix}&=\sum_\mathbf{k}\sum_\mathbf{p+q=k}\begin{bmatrix}iq_xu_\mathbf{p}^xu_\mathbf{q}^x+iq_yu_\mathbf{p}^yu_\mathbf{q}^x\\iq_xu_\mathbf{p}^xu_\mathbf{q}^y+iq_yu_\mathbf{p}^yu_\mathbf{q}^y\end{bmatrix}e^{i\mathbf{k\cdot{x}}}\\
&=i\sum_\mathbf{k}\sum_\mathbf{p+q=k}(\mathbf{q}\cdot{u}_\mathbf{p})u_\mathbf{q}e^{i\mathbf{k\cdot{x}}}\\
&=i\sum_\mathbf{k}\sum_\mathbf{p+q=k}(\mathbf{k}\cdot{u}_\mathbf{p})u_\mathbf{q}e^{i\mathbf{k\cdot{x}}}
\end{align*}
\begin{align*}
-\nabla{p}=-\begin{bmatrix}\frac{\partial{p}}{\partial{x}}\\\frac{\partial{p}}{\partial{y}}\end{bmatrix}&=-\sum_\mathbf{k}\begin{bmatrix}ik_xp_\mathbf{k}\\ik_yp_\mathbf{k}\end{bmatrix}e^{i\mathbf{k\cdot{x}}}\\
&=-i\sum_\mathbf{k}\mathbf{k}p_\mathbf{k}e^{i\mathbf{k\cdot{x}}}\\
&=i\sum_\mathbf{k}\frac{\mathbf{k}}{|\mathbf{k}|^2}\left(\sum_\mathbf{p+q=k}(\mathbf{k}\cdot{u}_\mathbf{q})(\mathbf{k}\cdot{u}_\mathbf{p})\right)e^{i\mathbf{k\cdot{x}}}
\end{align*}
We plug these in to the original equation, and then we multiply by $e^{-i\mathbf{j\cdot{x}}}$ and integrate.
\begin{align*}
&\int^{2\pi}_0\sum_\mathbf{k}\frac{d\mathbf{u_k}}{dt}e^{i(\mathbf{k-j})\cdot\mathbf{x}}d\mathbf{x}&\\
&+\int^{2\pi}_0i\sum_\mathbf{k}\sum_\mathbf{p+q=k}(\mathbf{k}\cdot{u}_\mathbf{p})u_\mathbf{q}e^{i(\mathbf{k-j})\cdot\mathbf{x}}d\mathbf{x}&=\int^{2\pi}_0i\sum_\mathbf{k}\frac{\mathbf{k}}{|\mathbf{k}|^2}\left(\sum_\mathbf{p+q=k}(\mathbf{k}\cdot{u}_\mathbf{q})(\mathbf{k}\cdot{u}_\mathbf{p})\right)e^{i(\mathbf{k-j})\cdot\mathbf{x}}d\mathbf{x}\\
\end{align*}
\[
\frac{d\mathbf{u_j}}{dt}+i\sum_\mathbf{p+q=j}(\mathbf{j}\cdot{u}_\mathbf{p})u_\mathbf{q} = i\frac{\mathbf{j}}{|\mathbf{j}|^2}\sum_\mathbf{p+q=j}(\mathbf{j}\cdot{u}_\mathbf{q})(\mathbf{j}\cdot{u}_\mathbf{p})
\]
So we have our final Fourier transform
\[
\frac{d\mathbf{u_k}}{dt}=-i\sum_\mathbf{p+q=k}\left((\mathbf{k}\cdot{u}_\mathbf{p})u_\mathbf{q}-\frac{\mathbf{k}}{|\mathbf{k}|^2}(\mathbf{k}\cdot{u}_\mathbf{q})(\mathbf{k}\cdot{u}_\mathbf{p})\right)
\]
which we can represent more clearly using a matrix $A$ as
\[
\frac{d\mathbf{u_k}}{dt}=-i\sum_\mathbf{p+q=k}\mathbf{k}\cdot{u}_\mathbf{p}A_\mathbf{k}u_\mathbf{q}
\]
where $A_\mathbf{k}=I-\frac{\mathbf{kk}^T}{|\mathbf{k}|^2}$.
\end{document}


% !TEX root = altosaar-2020-thesis.tex

% \appendix
\label{sec:appendix}
\section{Probabilistic inference, data, and posterior inference}

In machine learning, the posterior distribution of random variables $\mbz$ conditional on data or observations $\mbx$, $p(\mbz \mid \mbx)$, is a central quantity of interest. However, in applying machine learning to physics, we must specify the concepts of random variables and observations. In the Ising model, random variables are spins, and there is no straightforward concept of observed data. Study of the Boltzmann distribution of the Ising model is probabilistic inference, not posterior inference.

In the Ising model, there are no datapoints $\mbx$; there are only latent variables $\mbz$. Sampling from the model using Monte Carlo would yield only samples of system configurations, not data. In this model, the data can be defined as the empty set $\mbx = \{\}$, so the model evidence $\int p(\mbx, \mbz)d\mbz$ is simply the partition function $Z = \int p(\mbz) d\mbz$. An example of a model that resembles the Ising model that includes observed data is a Markov random field for image denoising~\citep{geman1984stochastic}. In this case, every spin or random variable is connected to a pixel of an image. The concept of data can be included in the Ising model in terms of an external magnetic field.

% \PP probability models are common to both physics and machine learning.

% Probability models are common in physics and machine learning. The Ising model is a statistical physics model of ferromagnetism, and is an example of an undirected graphical model in machine learning. A probability model assigns a probability to a configuration of the system in a certain condition, such as at a specific temperature, or conditional on observed data.

% \PP example; boltzmann

% In statistical physics, an example of a probability model is the Boltzmann distribution,
% \begin{equation}
% \label{eq:boltzmann}
% p(\mbz; \beta) = \frac{\exp(-\beta H(\mbz))}{Z}\, ,
% \end{equation}
% where $H$ is the energy function, $\mbz$ are random variables of the system, such as spins, $Z$ is the partition function, and the inverse thermodynamic temperature $\beta$ is a parameter of the probability model (denoted with a semicolon). In machine learning terms, $\mbz$ are latent variables and are unobserved. (We describe the relationship between data and latent variables in \Cref{sec:appendix}.)

% \PP the goal of probabilistic inference is to infer a partition function; a
% likelihood; a conditional distribution

% Probabilistic inference consists of computing statistical quantities associated with a probability model. Examples of probabilistic inference targets include the partition function; the probability of observing a certain configuration of the system; or, the conditional probability distribution of a probability model after observing data. The Boltzmann distribution, \Cref{eq:boltzmann} can be used to calculate physical quantities, such as specific heat, that can be compared to experimental values and understand the behavior of physical systems.

% \PP probabilistic inference is hard because of the intractable partition
% function

% Inference in a probability model is challenging. A task such as calculating a probability requires calculating a normalization constant or partition function, which includes a sum over all configurations of the system. For models with many random variables, enumerating the number of configurations is intractable. The Boltzmann distribution cannot be studied directly as the partition function cannot be computed for most systems. The partition function is a sum over an exponentially-large number of system configurations,

% \begin{align}
% Z &= \sum_{\mbz} \exp(-\beta H(\mbz))\, .
% \end{align}
% For example, in an Ising model, spins are in two states $z = \{-1, 1\}$. In an Ising model with $N$ spins, there are $2^N$ terms in the partition function, which cannot be computed in a reasonable time.

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "main"
%%% End:


\section{Gaussian Process Model}\label{sec:gpmodel}
A GP can be applied as a probabilistic model to a regression problem.  Here we use the GP model to generalise a stellar model grid to a continuous and probabilistic function that maps inputs to observable quantities.   This will allow us to predict observable quantities that are off (or even outside) the grid of stellar models with the prediction including uncertainty.
%
As mentioned in Section~\ref{sec:grid}, our model grid has four independent fundamental inputs, i.e., mass, initial metallicity, initial helium fraction, and mixing-length parameter. Adding the {\it EEP} which describes the evolutionary phase, the GP model contents five inputs.  We intend to train GP models that predict effective temperature, surface gravity, radius, surface metallicity, and stellar age. We summarise GP model inputs and outputs as below.

GP model inputs and their dynamic ranges:
\begin{itemize}
\item[] Mass ($M$ = 0.8 -- 1.2$\rm M_{\odot}$)
\item[] Equivalent Evolutionary Phase ({\it EEP} = 0 -- 1)
\item[] Initial metallicity ([Fe/H]$_{\rm init}$ =  -0.5 -- 0.5 dex)
\item[] Initial helium fraction ($Y_{\rm init}$ = 0.24 -- 0.32)
\item[] Mixing-length parameter ($\alpha_{\rm MLT}$ = 1.7 -- 2.5)
\end{itemize}

And the GR model outputs:
\begin{itemize}
\item[] Effective temperature ($T_{\rm eff}$)
\item[] Surface gravity ($\log g$)
\item[] Radius ($R$)
%\item[] The large spacing ($\Delta\nu$)
\item[] Surface metallicity ([Fe/H]$_{\rm surf}$)
\item[] Stellar age ($\tau$)
\end{itemize}

We aim to use the GP model as a non-parametric emulator, that is emulating the comparatively slow calls to models of stellar evolution.
This emulator can be described as a function approximation problem where the function is,
\begin{equation}\label{gprmodel}
{\rm Outputs} = f(M, EEP, ({\rm Fe/H})_{\rm init}, Y_{\rm init}, \alpha_{\rm MLT}).
\end{equation}
{\bf Tanda - is there one independent GP for each of the outputs?  If yes,  suggest ...

In fact, the way we have implemented the GP as function approximation means that we have used one GP for each of the outputs so that,
\begin{equation}\label{gprmodel1}
{T_{\rm eff}} = f(M, EEP, ({\rm Fe/H})_{\rm init}, Y_{\rm init}, \alpha_{\rm MLT}),
\end{equation}
and
\begin{equation}\label{gprmodel1}
{R} = g(M, EEP, ({\rm Fe/H})_{\rm init}, Y_{\rm init}, \alpha_{\rm MLT}),
\end{equation}
and so on for all outputs.
}

\subsection{Gaussian Process Application}

In our application to grids of stellar models,  a GP has a number of desirable properties.  While a GP is a stochastic process, the distribution of a GP can be considered as a distribution of functions with a continuous domain.  In fact,  the marginal likelihood considered in function space is equal to the likelihood of the data given some function values,  multiplied by the prior on those function values marginalised over all function values \cite{williams1996gaussian}.  That is to say that, the GP allows for the analytical evaluation of a fit over many different functions (perhaps an infinite number) weighted by some concept of a prior and the agreement with the data.   In addition,  while the marginal likelihood will be assessed on discrete data,  predictions can be made using linear algebra for new data in the continuous domain, but crucially again marginalised over these many different functional forms.  It is possible to see how this might be useful for generalising (or emulating or augmenting) a discrete grid of stellar models in order to obtain predictions in the continuous domain.

In this section we will look at the required mathematics to be able to implement a GP for our application to grids of stellar models.  We start with a series of definitions before dealing with the marginal likelihood and the posterior predictive distributions.

We start with a grid of stellar models containing $N$ models with a label we want to learn, for example model effective temperature, which we will denote with the general symbol $\bf y$, and a set of on-grid inputs $\bf X$ (e.g., mass,  EEP,  metallicity,  ...).  We can use a GP to make predictions of the effective temperature (labelled $y$) for additional off-grid input values given by $\bf X_{\star}$.  The vector $\bf y$ is arranged ${\bf y} = \left(y_{i}, ... ,y_{N} \right)^{T}$ where the subscript label references the stellar model.  The input labels are arranged into a $N \times D$ matrix where $D$ is the number of input dimensions (e.g., $D=3$ for mass, EEP, and metallicity) so that ${\bf X} = ({\bf x}_{1}, ..., {\bf x}_{N})^{T}$ where ${\bf x_{i}} = (x_{1, i}, ..., x_{D, i})^{T}$.  The matrix of additional inputs $\bf X_{\star}$ has the same form as $\bf X$ but size $N_{\star} \times D$.

\citet{williams1996gaussian}, from which our description below is based,  define a GP as a collection of random variables, where any finite number of which have a joint Gaussian distribution.  In general terms,  a GP may be written so that our on grid labels are random variables drawn from our GP distribution,
\begin{equation}
{\bf y}({\bf X}) \sim \mathcal{GP}\left( m({\bf X}),  {\bf \Sigma}\right),
\end{equation}
where $m({\bf X})$ is some mean function, and $\bf \Sigma$ is some covariance matrix.  The mean function controls the deterministic part of the regression and the covariance function controls the stochastic part.  The mean function defined here could be any deterministic function and we will label the additional parameters, or hyperparameters, $\phi$.  Each element of the more familiar covariance matrix is defined by the covariance function or {\it kernel function} $\bf K$ which has hyperparameters $\theta$ and is given by,
\begin{equation}
{\bf \Sigma} = {\bf K}({\bf X}, {\bf X},  \theta),
\end{equation}
or
\begin{equation}
{\bf \Sigma}_{n, m} = k({\bf X}_{n}, {\bf X}_{m},  \theta),
\end{equation}
where the inputs ${\bf X}_{n}$ and ${\bf X}_{n}$ are $D$-dimensional vectors and the output is a scalar covariance.
In addition to the covariance defined by the kernel function, we include additional white noise in the covariance matrix by adding an identity matrix $\mathcal{I}$ multiplied by a scalar value $\sigma_{w}^2$, so that,
\begin{equation}
{\bf \Sigma} = {\bf K}({\bf X}, {\bf X},  \theta) + \sigma_{w}^{2} \mathcal{I},
\end{equation}
where $\sigma_{w}^2$ is another hyperparameter to be learnt in training.

\subsubsection{The likelihood}
Conceptually we value the GP because of it's ability to marginalise over many functions $\bf f$ and return a marginal likelihood,
\begin{equation}
p({\bf y} | {\bf X}) = \int p({\bf y} | {\bf f}, {\bf X}) p({\bf f} | {\bf X}) \, {\rm d}{\bf f},
\end{equation}
noting that this function space marginal likelihood is weighted by the probability of the data given the function and the probability of the function.  This integral could be evaluated.  However, by noting that a GP is a collection of random variables, where any finite number of which have a joint Gaussian distribution, the marginal probability of our data $\bf y$ is also the joint likelihood of a multivariate normal distribution,
\begin{equation}
p({\bf y} | {\bf X}) = \mathcal{N}(m({\bf X}), {\bf \Sigma}),
\end{equation}
which can be straightforward to evaluate.  Thus the marginal likelihood is,
\begin{equation}
p({\bf y} | {\bf X}) = (2 \pi)^{k/2} {\rm det} ({\bf \Sigma})^{-0.5} \exp \left(\frac{-1}{2} ({\bf X} - m({\bf X}))^{T} \, {\bf \Sigma}^{-1} \, ({\bf X} - m({\bf X})) \right),
\end{equation}
which can be evaluated without integrating over all possible function space.   While this marginal likelihood expression is clearly more computationally feasible that the integral over functional space it not without it's limitations.  Because it is necessary to calculate the determinant and the inverse of the covariance matrix, typically applied algorithms,  make this a $\mathcal{O}(N^3)$ or $\mathcal{O}(N^2 \log N)$ operation.  This naturally limits the size of the data set for which the likelihood,  and optimisations of the likelihood, can be applied.

\subsubsection{Making predictions}
If we want to obtain predictive distributions for the output $\bf y_{\star}$ given the inputs $\bf X_{\star}$,  the joint probability distribution of $\bf y$ and $\bf y_{\star}$ is Gaussian and given by
\begin{equation}
p \left( \begin{bmatrix} {\bf y} \\ {\bf y_{\star}} \end{bmatrix} \right) = \mathcal{N} \left( \begin{bmatrix} m({\bf X}) \\ m({\bf X_{\star}}) \end{bmatrix} , \begin{bmatrix} {\bf \Sigma} & {\bf K_{\star} }\\ {\bf K_{\star}}^{T} & {\bf K_{\star \star}} \end{bmatrix}  \right),
\end{equation}
where the covariance matrices $\bf \Sigma$ and $\bf K$ are computed using the kernel function so that,

\begin{equation}
{\bf \Sigma}_{n, m} = k({\bf X}_{n}, \, {\bf X}_{m}),
\end{equation}
which is an $N \times N$ matrix.
\begin{equation}
{\bf K}_{\star \, n, m} = k({\bf X}_{n}, \, { \bf X}_{\star \, m}),
\end{equation}
which is an $N \times N_{\star}$ matrix, and finally
\begin{equation}
{\bf K}_{\star \star \, n, m} = k({\bf X}_{\star \, n},  {\bf X}_{\star \,m}),
\end{equation}
which is an $N_{\star} \times N_{\star}$ matrix.
The predictions of $\bf y_{\star}$ are again a Gaussian distribution so that,
\begin{equation}
{\bf y}_{\star} \sim \mathcal{N}(\bf \hat{y}_{\star}, \, \bf C),
\label{eq:pred}
\end{equation}
where
\begin{equation}
{\bf \hat{y}}_{\star} = m({\bf X}_{\star}) + {\bf K}_{\star}^{T} \, {\bf \Sigma}^{-1} \, ({\bf y} - m(\bf X)),
\end{equation}
and
\begin{equation}
{\bf C} = {\bf K}_{\star \star} - {\bf K}_{\star}^{T} \, {\bf \Sigma}^{-1} \, {\bf K_{\star}}.
\end{equation}

At point we can make predictions on model properties given a grid of stellar models using equation \ref{eq:pred}.  But these predictions will likely be poor unless we select sensible values for the form and hyperparameters of the mean function and covariance function.  In the following section we detail a number of kernel functions that will be tested against the data.  We will then discuss the method for determining the values of the hyperparameters to be used.

\subsection{Setting up the GP Model}\label{sec:set_up}

We adopt a tool package named \textsc{GPyTorch}, which is a GP framework developed by \citet{gardner2018gpytorch}. It is a Gaussian process library based on an open source machine-learning framework PyTorch (\url{https://pytorch.org}). The package provides significant GPU acceleration, state-of-the-art implementations of the latest algorithmic advances for scalability and flexibility, and easy integration with deep learning frameworks. Source codes and detailed introductions could be found on \url{https://gpytorch.ai}.

For training GP models, we need set up mean function, kernel function, likelihood function, loss function, and optimiser.
To find out the best option for each of above elements, we do a number of preliminary tests.
These tests are carried out with 2-demission (2D) GP models which map $M$ and {\it EEP} to the five observable outputs (Outputs = $f(M, EEP)$). Training data are selected from the primary grid with fixed [Fe/H]$_{\rm init}$ (0.0), $Y_{\rm init}$ (0.28), and $\alpha_{\rm MLT}$ (2.1). There are 41 evolutionary tracks which content 24,257 data points. We also compute 44 evolutionary tracks with the same [Fe/H]$_{\rm init}$, $Y_{\rm init}$, and $\alpha_{\rm MLT}$ but random $M$ for the purpose of validating and testing. We use the method mentioned in Section \ref{sec:selection} and sample 20,000 data points for training, 10,000 for validating, and 10,000 for testing.
%
We start with the \textsc{Simple GP Regression} example (\url{https://docs.gpytorch.ai/en/stable/examples/01_Exact_GPs/Simple_GP_Regression.html}) and develop our own method step by step as follow.

\subsubsection{Mean Function}

We first investigate the mean function. As discussed above, the data distribution is generally smooth but complex in some rejions of parameter space (e.g., the sub-giant hook).  Although a GP model can cope with different mean functions,  because of the flexibility of kernels, we find that using a constant or a linear mean function leads to a long time for training. For this reason, we apply a Neural Network mean function which is flexible enough to manage both simple and complex features. We adopt an architecture including 6 hidden layers and 128 nodes per layer. All layers apply the linear transformation to the incoming data. We apply the element-wise function (Elu) because it give relatively smooth mean functions. ({\bf Can Guy or Alex add some references for NN and Elu here? The referee may ask why the 6x128 architecture is sufficient, so should we mention Alex's paper?})

\subsubsection{Likelihood and Loss Function}

We then trained the model using the negative logarithm {base 10?} of the likelihood function above as the loss function.  Our training dataset is a theoretical model grid,  hence there is no observed uncertainty for each data point, but a tiny random uncertainty exists due to the approximations in the \textsc{MESA} numerical method. We model this using $\sigma_{w}$.  This noise model is then assumed to be a Gaussian function with a very small variance.
%
A likelihood specifies the mapping from input values $f(X)$ to observed labels $y$.
We adopt the the standard likelihood for regression which assumes a standard homoskedastic noise model whose conditional distribution is
\begin{equation}\label{eq:likelihood}
p(y|f(x)) = f + \epsilon, \epsilon \sim \mathcal{N}(0, \sigma^{2}),
\end{equation}
where $\sigma$ is a noise parameter.
%
We used a small and fixed noise parameter and run a few tests.  However, the strict noise parameter makes GP models hard to converge. When this noise parameter is set as free, it reduces to a small value anyway in the training progress because it is data-driven.  For this reason, we did not put strict constraint for or prioritise this noise parameter.  In practice, we only set up a loose upper limit ($\sigma$  < 0.1) to speed up the training. One thing should be noted that a GP model with a large noise parameter is not a proper description for the stellar grid. Because of this, we only adopt GP models with $\sigma$ $\lesssim 10^{-4}$.

\subsubsection{Optimiser}

We compared two optimisers named SGD and Adam. Here SGD refers to Stochastic Gradient Descent, and Adam is a combination of the advantages of two other extensions of stochastic gradient descent, specifically, Adaptive Gradient Algorithm and Root Mean Square Propagation.
%
The SGD optimiser in the \textsc{GPyTroch} package involves the formula given by \citet{sutskever2013importance}. The formula makes it possible to train using stochastic gradient descent with momentum thanks to a well-designed random initialisation and a particular type of slowly increasing schedule for the momentum parameter. The application of momentum in SGD could improve its efficiency and make it less likely to stuck in local minimums. On the other hand, the Adam optimiser includes the 'AMSGrad' variant developed by \citet{47409} to improve its weakness in the convergence to an optimal solution. With these new developments, the two optimisers give very similar results. We finally choose Adam because it works relatively efficiently and stable.
%
We adaptive learning rate in the training process. Our training starts with a learning rate of 0.01 and decreases by a factor of 2 when the loss value does not reduce in previous 100 iterations.

\subsubsection{Early Stopping}

We set up an Early Stopping for avoiding overfitting and also determining when to terminate a training progress.
When an optimal solution is found, the validating errors stop decreasing and then increase when overfitting occurs.
We hence track down the validating error every iteration and terminate training process when the validating error does not decrease in previous 300 iterations.

\subsubsection{Kernel Function}

With the above set up, we test different kernel functions. We involve four basic kernels as listed below:
\begin{itemize}
\item RBF: Radial Basis Function kernel (also known as squared exponential kernel)
\item RQ: Rational Quadratic Kernel (equivalent to adding together many RBF kernels with different lengthscales)
\item Mat12: Matern 1/2 kernel (equivalent to the Exponential Kernel)
\item Mat32: Matern 3/2 kernel
\end{itemize}
We apply each basic kernel and a couple of combined kernels (RBF + Mat21, RQ + Mat21, Mat32 + Mat21, RBF + Mat32, RQ + Mat32) to train and validate GP model for each output. Among these kernels, the combined kernel RBF+Mat21 gives the best fit to the training data, however, it does not give the best predictions for validating and testing data. On the other hand, the Mat32 kernel fits training data reasonably well and it predicts the best for validating and testing data.
%
Comparing between the two kernels, the RBF+Mat12 Kernel is a combination of a smooth and a spiky kernel, which has sufficient flexibility to fit features of data. However, the spiky function make it less accurate for off-grid regions.
%
The smoothness of Mat32 kernel is somewhere between a spiky (Mat21) and a smooth kernel (RBF). It does not exactly fit some sharp features but it maps the off-grid region with better continuity. We hence adopt the Mat32 kernel for the training.

\subsubsection{Training Procedure}

We lastly introduce the procedure of training a GP model. The full process includes training, validating, and testing.
In the training process, we literately optimise hyperparameters of a GP model to learn the underline function which maps inputs to outputs from on-grid evolutionary tracks (training dataset). In each iteration, the GP model is validated by comparing true and GP predicted values of off-grid tracks (validating dataset). Although the validating dataset is not directly involved in training hyperparameters, it still constructs the GP model to some extend because the optimal solution is the one that best fits the validating dataset. For this reason, the validating dataset does not give a completely independent validation for a GP model.
%
We hence have a testing process at the end for the purpose of estimating the systematic uncertainty of GP. We reserve a half of off-grid tracks as testing dataset and qualify a GP model using the testing error (true value - GP prediction).

\section{Preliminary Studies on Low-Demission Problems}\label{examples}

In this section, we present GP applications on 1-, 2-, and 3-demissions (1D, 2D, 3D) problems. We also investigate two key questions before training the 5-demissions (5D) stellar grid: testing method and strategy for large sample.

\subsection{1D Problem}

We first demonstrate an example of GP application on an 1D problem. We train a GP model to learn the evolution of effective temperature for a $\rm 1.1M_{\odot}$ track. We split the data points on this track into training and testing data by 70-to-30. We train a GP model which maps  {\it EEP} to effective temperatures. We then compare true and GP-predicted effective temperatures of testing data. As it can be seen in Figure \ref{fig:1dgp},  the GP model gives very accurate predictions: residuals are within $\pm$0.5 K. As a comparison, we fit the training data with the quadratic function and use it to predict testing data. We find that the two methods give very similar results. It indicate that GP can be an alternative of classical interpolators on the 1D problem.

\begin{figure}
	\includegraphics[width=1.0\columnwidth]{1d-gp.pdf}
    \caption{GP application on 1D problem. Models on this track are split into training and testing data by 70-to-30, Top: the evolution of effective temperature for a $\rm 1.1M_{\odot}$ track. The grey line is the evolutionary track computed with \textsc{MESA}; blue and red circles indicate predictions for the testing data from the quadratic interpolator and the GP model. Bottom: residuals of predictions in the top graph. }
    \label{fig:1dgp}
\end{figure}

\subsection{2D Problem: Investigating testing method}

We present a 2D example here.  We use the same training, validating, and testing datasets mentioned in Section \ref{sec:set_up} and train GP models which map $M$ and {\it EEP} to observables. We illustrate the GP model for effective temperature in Figure \ref{fig:2dtest}. As it shown that, GP transforms the sparse data into a continuous function and hence is able to predict values for unseen points.
%
We also notice that the kernel function in the area of $M \geq 1.05 {\rm M_{\odot}}$ and ${\it EEP} \leq 0.7$ is more complex than that for other regions because of the appearance of the 'hook'. This particular area are relatively difficult to learn and poorly predicted by the GP model.

\begin{figure}
	\includegraphics[width=1.0\columnwidth]{2d_GPmodel_function.pdf}
	\includegraphics[width=1.0\columnwidth]{2d_testing_hist_effective_T.pdf}
    \caption{Top: The 2D GP model for $T_{\rm eff}$. Bottom: probability distributions of validating errors of the GP model. }
    \label{fig:2dtest}
\end{figure}

When there is a subregion in which the GP model performs worse than other areas, the probability distribution of testing error is not quite a  Gaussian function. We examine the error distribution as shown at the bottom of Figure \ref{fig:2dtest}. Errors of most data follows a Gaussian distribution but about $10\%$ data form long tails on both sides.
%
We inspect testing errors for other output parameters and find similar results.
The cases of surface gravity and radius are similar to the effective temperature. The surface metallicity is not affected by the hook, but the GP predictions are relatively poor at the early subgiant phase in high-mass tracks. This is because high-mass tracks maintain shallow convective envelope and hence have strong diffusion effect during the main-sequence stage. At the early subgiant phase, the quick expansion of the surface convective envelope brings back the settling heavy elements to the surface, leading to a sharp raise of the surface metallicity.
%
We also find that the accuracy of age prediction drops down for very old low-mass stellar models. This is because age values vary in a relatively big dynamic range (15 - 50 Gyr) in a small fraction of data points. Poor age resolution causes poor GP predictions.
This is to say, the tail feature is common for all five outputs. This leads to a question of how to quantity the testing error.

For this case, a global error such like Root Mean Square Error (RMSE) is not ideal to quantify the error. Because it does not well reflect the tail feature. What we need here is a method that reflects the general accuracy level as well as the worst case.
%
We examine the error distribution of each output and count data points in the tails (outside the 3 times of full width at half maximum). The amount of these data are around 10\% (8 - 12\% for different outputs).
%
For the majority (90\%) of data points, which from a Gaussian function, the 68\% confidential interval is able to reflect their accuracies. For the worst 10\% of the data, we could use the 95\% and 99.7\% confidential intervals to describe the median value and the length of the tail. Thus, we define an Error Index (EI), which is the sum of 68\%, 95\%, and 99.7\% cumulative values of absolute errors, to qualify GP models. For the case in Figure \ref{fig:2dtest}, cumulative values at 68\%, 95\%, and 99.7\% are 1.1, 4.9, and 11.1K, which give an EI equals to 17.1K.

\subsection{3D Problem: Strategy for Large Data Sample}

As a further step, we apply GP on a 3D grid and investigate the strategy for large data sample.
The model grid we aim to train contents about 10,000,000 data points, which is much more than the upper limit of data size (20,000) of an \textsc{Exact GP} model. We work on this with 3-demission (3D) GP models, which map $M$, {\it EEP}, and  [Fe/H]$_{\rm init}$ to observable outputs.
%The 3D GP model can be described as Outputs $ = f(M, EEP, {\rm [Fe/H]_{init}})$.
We select training data from the primary grid with fixed $Y_{\rm init}$ (0.28), and $\alpha_{\rm MLT}$ (2.1). The training dataset contents $\sim$300,000 data points. For validating and testing purposes, we compute another 174 evolutionary tracks with the same input $Y_{\rm init}$ and $\alpha_{\rm MLT}$ but random $M$ and [Fe/H]$_{\rm init}$. Before proceeding with large sample, we train an \textsc{Exact GP} model using 20,000 training data points as a standard reference.

We first consider the Stochastic Variational GP (SVGP) approach based on the \textsc{GPyTorch ApproximateGP} module.
%We train our data based on the SVGP example on \url{https://docs.gpytorch.ai/en/v1.1.1/examples/04_Variational_and_Approximate_GPs/SVGP_Regression_CUDA.html}.
SVGP is an approximate scheme rely on the use of a series of inducing points which can be selected in the parameter space. It trains using minibatches on the training dataset and build up kernels on the inducing points. Underline principles and detailed descriptions of this approach can be found in \citet{hensman2015scalable}. The advantage of SVGP is the large capacity of sample size. However, the kernel complexities is still limited by the amount of inducing points.
%
In our tests, we find a practical issue with the SVGP approach. Because a large training sample takes off the memory, we can only use 10,000 inducing points. This is to say, the kernel complexity of the SVGP model is even simpler than the Exact GP model which uses 20,000 data points. As a result, the SVGP model does not show any improvements. For instance, the 68th, 95th, and 99.7th testing errors for $T_{\rm eff}$ are 2.0, 5.8, and 15.7 K (EI = 23.5K) for the \textsc{Exact GP} model and 2.2, 6.8, and 15.1K (error index = 24.1 K) for the SVGP one. Because the evolutionary feature are complex across multiple demissions, what we need here is increasing the kernel complexity. This requires more data points that actually construct the kernel function. For this particular case, the SVGP downgrades the complexity and hence does not improve the results.

We then investigate another approach designed for large dataset named Structured Kernel Interpolation (SKI GP). SKI GP was introduced by \citet{wilson2015kernel}. It produces kernel approximations for fast computations through kernel interpolation and is a great way to scale a GP up to very large datasets (100,000+ data points).
% We follow the example on \url{https://docs.gpytorch.ai/en/stable/examples/02_Scalable_Exact_GPs/KISSGP_Regression.html} to develop our script.
We run a few tests to train a 3D SKI GP model with 100, 000 training data. Compare with the \textsc{Exact GP} and SVGP, its testing errors for $T_{\rm eff}$ are slightly improved to 2.0, 6.1, and 14.8K (EI = 22.9K). However, the further test on the 5-demission data is not ideal: a SKI GP model using 100, 000 training data performs much worse than an \textsc{Exact GP} model with only 20,000 training data. The poor behaviour consists with what has been discussed in \citet{wilson2015kernel}. The method poorly scale to data with high dimensions, since the cost of creating the grid grows exponentially in the amount of data. We attempt to make some additional approximations with the \textsc{GpyTorch AdditiveStructureKernel} module. It makes the base kernel to act as one-dimension kernels on each data dimension and the final kernel matrix will be a sum of these 1D kernel matrices. However, the testing errors are not significantly improved.

 \begin{figure}
	\includegraphics[width=1.0\columnwidth]{3d-testing_teff-10sections.pdf}
	\includegraphics[width=1.0\columnwidth]{3d-testing_teff-hist-10sections.pdf}
    \caption{Top: Testing errors of 3D GP model for $T_{\rm eff}$ on the $M - EEP$ diagram. Dashes indicates section boundaries. Bottom: examination of the edge effects of the section scenario. Probability distributions of testing errors of all testing data and those near the boundary ($\pm$0.01 EEP) in the upper graph are compared.  As it can be seen, testing errors do not raise around the boundary. }
    \label{fig:3dtest}
\end{figure}

As mentioned above, the GPU memory captivity limits the actual number of data that induce the kernel function.
This limitation becomes critical for the high-demission case. To improve, more training data need to be involved.
%The parameter space exponentially increases with the demission and hence the GP model accuracy inevitable declines.
%
A simple way is breaking the grid into sections and train GP models for each section separately.
%The downside of this section scenario is that there will not be one GP model that maps the whole grid. However, as long as the goal of this work is augmenting a model grid but not deriving a universal function for stellar evolutions, this scenario is suitable.
%Here we test how this section scenario works for 3D GP models.
Here we divide the training dataset into 10 equal segments by {\it EEP}. We train one \textsc{Exact GP} model with 20,000 training data for each section.
%We then use the same testing dataset to quantify GP predictions for the five outputs, and
With this section scenario, the testing EIs for five output parameters are averagely improved by around 10\%. For instance, the testing EI for $T_{\rm eff}$ decreases from 23.5 to 21.6K.
% (1.7/5.0/14.9K at 68/95/99.7\%).
%

As expected, the section scenario improves the performance of GP model, but there is a major concern about the edge effect at the boundary between sections. If the GP model works significantly poorly at the section boundaries, it will be difficult to map the systematic errors across the whole parameter space. We examine potential edge effects as illustrated in Figure~\ref{fig:3dtest}. We inspect absolute testing errors on the $M-EEP$ diagram. No obvious edge effect is found. We also do a statistical comparison between error values of all data points and those around section boundaries ($\pm0.01EEP$). As shown in the bottom graph, the density distributions of two samples are very similar to each other, indicating no obvious edge effect. We adopt this section scenario to train the stellar grid in the following study.


\section{Conclusion}
In this paper, we have demonstrated a method for for synthesizing encodings
  for untyped lambda calculus terms and functions in parallel, based on an
  abstract specification of the behavior of those terms.
We call this dual specification of terms and function \textit{co-synthesis}.
Co-synthesis is made possible in the case of untyped lambda
  calculus, because there is no structural difference between terms and
  functions, only a difference in the meaning ascribed to them.
This means that evaluation of unknown terms simply involves carrying out
  regular lambda calculus evaluation with potential free variables, which
  are the expressions we are synthesizing.
Using bottom-up fair enumeration, we were able to synthesize functions and
  terms in boolean, natural number, and pair encodings, as well as
  co-synthesize the encoding for booleans.


\documentclass[letter]{article}
\renewcommand{\baselinestretch}{1.25}

\usepackage[margin=1in]{geometry}
\usepackage{physics}
\usepackage{amsmath}
\usepackage{graphicx}
%\usepackage{pythonhighlight}
\usepackage{hyperref}
\usepackage{fancyvrb}

% MATLAB Formating Code
\usepackage[numbered,framed]{matlab-prettifier}
\lstset{style=Matlab-editor,columns=fullflexible}
\renewcommand{\lstlistingname}{Script}
\newcommand{\scriptname}{\lstlistingname}

% Command for easier minimization problem def
\newcommand{\optpblm}[3][eq:default]{
	\begin{equation}\label{#1}
% Array method... more centered
%		\begin{array}{rl}
%			\text{minimize}  \hspace{0.2in} &#2 \vspace{5pt}\\
%			\text{subject to} \hspace{0.2in} &#3
%		\end{array}
% Aligned method... left aligned... idk if its better
		\begin{aligned}
			\text{minimize} \hspace{0.5in} &#2\vspace{5pt}\\
			\text{subject to \hspace{0.5in}} &#3
		\end{aligned}
	\end{equation}
}

\allowdisplaybreaks

\title{MECH 6327 - Homework 3}
\author{Jonas Wagner}
\date{2021, March 24}

\begin{document}

\maketitle

\newpage
\tableofcontents

\newpage
\section*{BV Textobook Problems}
\subsection{Problem 4.11}
\textbf{Problem:}
Formulate each problem as a LP and explained the relationship between the optimal solution of the problems and the solution of its LP.\\
\textbf{Solution:}
\subsubsection{Part a: Minimize $\norm{Ax-b}_\infty$}
Define the following minimization problem:
\optpblm{\norm{Ax-b}_\infty}{\text{math}}
From the definition of an $\infty$-norm as $$\norm{x}_\infty = \max_i \abs{x_i}$$ the following can be derived:
\optpblm{t}{\qty(Ax - b)_i\leq t, \ \forall i = 1,\dots,n\\
		- & \qty(Ax - b)_i\leq t, \ \forall i = 1,\dots,n}
Which is equivalent to the following linear program
\optpblm{t}{-\vb{1}t \leq Ax - b \leq \vb{1}t}
The resulted minimum to this equivalent problem, $t*$, is equivalent to the minimum of the original problem, $\norm{Ax^*-b}_\infty$.

From this the minimizing variable, $x^*$, can be found as: $$x^* = A^{-1} (\vb{1}^T t^* + b)$$

\newpage
\subsubsection{Part b: Minimize $\norm{Ax-b}_1$}
Define the following minimization problem:
\optpblm{\norm{Ax-b}_1}{\text{math}}
From the definition of an $1$-norm as $$\norm{x}_1 = \sum_i \abs{x_i}$$ the following can be derived:
\optpblm{t_1 + \cdots + t_n}{\qty(Ax - b)_i\leq t_i, \ \forall i = 1,\dots,n\\
-	& \qty(Ax - b)_i\leq t_i, \ \forall i = 1,\dots,n}
Which is equivalent to the following linear program
\optpblm{\vb{1}^T t}{-t \leq Ax - b \leq t}
The resulted minimum to this equivalent problem, $\vb{1}^T t$, is equivalent to the minimum of the original problem, $\norm{Ax-b}_1$.

From this the minimizing variable, $x^*$, can be found as: $$x^* = A^{-1} (t^* + b)$$

\newpage
\subsubsection{Part c: Minimize $\norm{Ax-b}_1$ subject to $\norm{x}_{\infty}\leq 1$}
Define the following minimization problem:
\optpblm{\norm{Ax-b}_1}{\norm{x}_\infty \leq 1}
From the definition of an $1$-norm as $$\norm{x}_1 = \sum_i \abs{x_i}$$ and the definition of an $\infty$-norm as $$\norm{x}_\infty = \max_i \abs{x_i}$$ the following can be derived:
\optpblm{t_1 + \cdots + t_n}{\qty(Ax - b)_i\leq t_i, \ \forall i = 1,\dots,n\\
	-& \qty(Ax - b)_i\leq t_i, \ \forall i = 1,\dots,n\\
	 & x_i \leq 1, \forall i = 1,\dots,n\\
	-& x_i \leq 1, \forall i = 1,\dots,n}
Which is equivalent to the following linear program
\optpblm{\vb{1}^T t}{-t \leq Ax - b \leq t\\
					&-\vb{1} \leq x \leq \vb{1}}

The resulted minimum to this equivalent problem, $\vb{1}^T t$, is equivalent to the minimum of the original problem, $\norm{Ax-b}_1$.

From this the minimizing variable, $x^*$, can be found as: $$x^* = A^{-1} (t^* + b)$$

\newpage
\subsubsection{Part d: Minimize $\norm{x}_1$ subject to $\norm{Ax-b}_\infty \leq 1$}
Define the following minimization problem:
\optpblm{\norm{x}_1}{\norm{Ax-b}_\infty \leq 1}
From the definition of an $1$-norm as $$\norm{x}_1 = \sum_i \abs{x_i}$$ and the definition of an $\infty$-norm as $$\norm{x}_\infty = \max_i \abs{x_i}$$ the following can be derived:
\optpblm{t_1 + \cdots + t_n}{x_i \leq t_i, \ \forall i = 1,\dots,n\\
							&-x_i \leq t_i, \ \forall i = 1,\dots,n\\
							&(Ax-b)_i \leq 1, \ \forall i = 1,\dots,n}
From this a linear program can be defined as:
\optpblm{\vb{1}^T t}{-t \leq x \leq t\\
			 		&Ax - b \leq \vb{1}}

The resulted minimum to this equivalent problem, $\vb{1}^T t$, is equivalent to the minimum of the original problem, $\norm{x}_1$.

From this the minimizing variable, $x^*$, can be found as: $$x^* = t^*$$

\newpage
\subsubsection{Part e: Minimize $\norm{Ax-b}_1 + \norm{x}_\infty$}
Define the following minimization problem:
\optpblm{\norm{Ax-b}_1 + \norm{x}_\infty}{math}
From the definition of an $1$-norm as $$\norm{x}_1 = \sum_i \abs{x_i}$$ and the definition of an $\infty$-norm as $$\norm{x}_\infty = \max_i \abs{x_i}$$ the following can be derived:
\optpblm{t_1 + \cdots + t_n + s}{
		\qty(Ax - b)_i\leq t_i, \ \forall i = 1,\dots,n\\
		-& \qty(Ax - b)_i\leq t_i, \ \forall i = 1,\dots,n\\
		&x_i \leq s, \ \forall i = 1,\dots,n\\
	-	&x_i \leq s, \ \forall i = 1,\dots,n}
This can be written as a standard linear program as:
\optpblm{\vb{1}^T t + s}{
		- t \leq Ax-b \leq t\\
		&-\vb{1}s \leq x \leq \vb{1}s}
The resulted minimum to this equivalent problem, $\vb{1}^T t + s$, is equivalent to the minimum of the original problem, $\norm{Ax-b}_1 + \norm{x}_\infty$. It should be noted that the $s$ and $\norm{x}_\infty$ are not used to find the minimization variable, but are important in weighting for solving for the minimization itself.

From this the minimizing variable, $x^*$, can be found as: $$x^* = A^{-1} (t^* + b)$$

\newpage
\subsection{Problem 4.16}
Consider the system given as
\begin{equation}\label{eq:dyn_sys_def}
	x(t+1) = A x(t) + b u(t), \ t = 0,\dots,N-1
\end{equation}
with $x(t) \in \real^n, u(t) \in \real, \forall t = 0,\dots,N-1$ and $A \in \real^{n\cross n}, b \in \real^n$, and $x(0) = 0$.\\

The minimum fuel optimal control problem is to select the minimum amount of inputs to minimize the amount of fuel used, given as
\begin{equation}\label{eq:min_fuel_problem_def}
	\begin{aligned}
		\text{minimize} \hspace{0.5in} &F = \sum_{t=1}^{N-1} f(u(t))\\
		\text{subject to \hspace{0.5in}} & x(t+1) = A x(t) + b u(t), \ t = 0,\dots,N-1\\
		& x(N) = x_{des}
	\end{aligned}
\end{equation}
with $N$ as the time-horizon, $x_{des} \in \real^n$ as the desired final state, and $f: \real \to \real$ given as
\begin{equation}\label{eq:fuel_usage_def}
	f(a) =
	\begin{cases}
		\abs{a} & \abs{a} \leq 1 \\
		2 \abs{a} - 1 & \abs{a} > 1
	\end{cases}
\end{equation}

\textbf{Problem:}
Formulate this problem as a Linear Program.\\

\textbf{Solution:}
First, \ref{eq:min_fuel_problem_def} can be rewritten in an epigraph form (with the additional assumption that $f(u(t))$ is always positive):
\optpblm[eq:min_fuel_problem_epigraph]{F_1 + \cdots + F_{N-1}}{
		f(u(t)) = F_t, \ \forall t = 1, \dots, N-1\\
		& x(t+1) = A x(t) + b u(t), \ \forall t = 0,\dots,N-1\\
		&x(N) = x_{des}}
Now looking at the nonlinear component, fuel usage as defined by \eqref{eq:fuel_usage_def}, can be equated to:
\begin{equation}
	\begin{aligned}
		\abs{a} \leq g\\
		2 \abs{a} - 1 \leq g\\
	\end{aligned}
\end{equation}
or equivalently,
\begin{equation}
	\begin{aligned}
		-g \leq a \leq g\\
		-g \leq 2a -1 \leq g
	\end{aligned}
\end{equation}

This represents an intersection of two half-spaces which is a simpliler convex restriction.\\
This can now be combined with \eqref{eq:min_fuel_problem_epigraph} to produce the linear program:
\optpblm[eq:min_fuel_problem_result]{F_1 + \cdots + F_{N-1}}{
	-F_t \leq u(t) \leq F_t, \ \forall t = 1, \dots, N-1\\
	&-F_t \leq 2u(t) -1 \leq F_t, \ \forall t = 1, \dots, N-1\\
	& x(t+1) = A x(t) + b u(t), \ \forall t = 0,\dots,N-1\\
	&x(N) = x_{des}}
Which can then be rewritten as:
\optpblm[eq:min_fuel_problem_result]{\vb{1}^T F}{
	-F \leq \vb{u} \leq F\\
	& x(t+1) = A x(t) + b u(t), \ \forall t = 0,\dots,N-1\\
	&x(N) = x_{des}
	}

\newpage
\subsection{Problem 4.28}
Consider the convex quadratic program given as
\begin{equation}\label{eq:convex_quadratic_program}
	\begin{aligned}
		\text{minimize} \ \ & \frac{1}{2} x^T P x + q^T x + r\\
		\text{subject to} \ \ & Ax \leq b
	\end{aligned}
\end{equation}
with a robust equivalent defined as
\begin{equation}\label{eq:robust_convex_quadratic_program}
	\begin{aligned}
		\text{minimize} \ \ & \sup_{P\in \mathcal{E}}\{\frac{1}{2} x^T P x + q^T x + r\}\\
		\text{subject to} \ \ & Ax \leq b
	\end{aligned}
\end{equation}
where $\mathcal{E}$ is the set of all possible matrices of $P$.

\subsubsection{Part a}
\textbf{Problem:}
Express the robust QP as a convex problem given $\mathcal{E} = \{P_1,\dots,P_k\}$ where $P_i\in S^n_+, \ \forall i=1,\dots,k$.\\

\textbf{Solution:}
As a base assumption, by definition all quadratic programs are convex. Additionally when taking a pointwise supremum of convex sets, the result is also convex.
Thus, for a supremum over the finite set of $\mathcal{E}$ it is known that a resultant convex problem can be defined.\\

First, we can redefine the problem as
\optpblm{\sup \{t_1, \dots, t_k\}}{
	\frac{1}{2} x^T P_i x + q^T x + r \leq t_i, \ i = 1, \dots, k\\
	&Ax \leq b}
Another eipigraph can then be analyzed to create the following convex optimization problem:
\optpblm{s}{
	t_i \leq s, \ i = 1, \dots, k\\
	&\frac{1}{2} x^T P_i x + q^T x + r \leq t_i, \ i = 1, \dots, k\\
	&Ax \leq b}

% no part b,c for 4.28

\newpage
\subsection{Problem 4.43}
Suppose $A: \real^n \to S^m$ is affine such that
\begin{equation}
	A(x) = A_0 + x_1 A_1 + \cdots + x_n A_n
\end{equation}
where $A_i \in S^m$. Let $\lambda_1(x) \geq \lambda_2(x) \geq \cdots \geq \lambda_m(x)$ be the eigenvalues of $A(x)$.\\

For each of the following minimization criteria, formulate the problem as an SDP.\\

\subsubsection{Part a}
\textbf{Problem:}
Minimize the maximum eigenvalue of $A$: $$\text{minimize} \ \ \lambda_1(x)$$
\textbf{Solution:}
This ca be re-written in epigraph form as:
\optpblm{t}{\lambda_1 \leq t}
or similarily as an SDP:
\optpblm{t}{A(x) \preceq t I}

%It is known that the eigenvalues of a sum of matrices is bounded below by the sum of the minimum eigenvalues of each and bounded above by the sum of the maximum eigenvalues.
%\cite{eigvalueBound}
%i.e.
%$$ \lambda(A)_m + \lambda(B)_m \leq \lambda(A+B)_m \leq \lambda(A+B)_1 \leq \lambda(A)_1 + \lambda(B)_1$$
%If this is to be expanded to the entire affine sum, $A(x)$, the objective of minimizing the eigenvalues of the weighted sum of symetric matrices can be done by minimizing the weighted sum of the largest eigenvalues of individual matrices.
%This means this problem can be redefined as:
%\optpblm{t^T x}{
%	t_i = \lambda_1(A_i), \ \forall i = 1,\dots,m\\
%	&s = \lambda_1(A_0)}
%Since $s$ will remain constant regrdless of $x$, this is equivalent to:
%\optpblm{t^T x}{
%	t_i = \lambda_1(A_i), \ \forall i = 1,\dots,m}
%
%????????????????????????????????????????????????????

\subsubsection{Part b}
\textbf{Problem:}
Minimize the spread of the eigenvalues of $A$: $$\text{minimize} \ \ \lambda_1(x) - \lambda_m(x)$$\\
\textbf{Solution:}
This can be rewritten in epigraph form as
\optpblm{t_1 - t_2}{\lambda_1 \leq t_1\\ & \lambda_m \leq t_2}
or similarly as an SDP:
\optpblm{t_1}{A(x) \preceq t_1 I\\ &A(x) \succeq t_2}

%It is known that the eigenvalues of a sum of matrices is bounded below by the sum of the minimum eigenvalues of each and bounded above by the sum of the maximum eigenvalues.
%\cite{eigvalueBound}
%i.e.
%$$ \lambda(A)_m + \lambda(B)_m \leq \lambda(A+B)_m \leq \lambda(A+B)_1 \leq \lambda(A)_1 + \lambda(B)_1$$
%If this is to be expanded to the entire affine sum, $A(x)$, the objective of minimizing the spread eigenvalues of the weighted sum of symetric matrices can be done by minimizing the weighted sum of the spread of eigenvalues of individual matrices.
%This means this problem can be redefined as:
%\optpblm{t^T x + s}{
%	t_i = (\lambda_1(A_i) - \lambda_m(A_i), \ \forall i = 1,\dots,m\\
%	&s = (\lambda_1(A_0) - \lambda_m(A_0)}
%Since $s$ remains constant regardless of $x$ this is equivelent to
%\optpblm{t^T x}{
%	t_i = (\lambda_1(A_i) - \lambda_m(A_i), \ \forall i = 1,\dots,m}
%
%
%????????????????????????????????????????????????????

%Defining the optimization problem as:
%\optpblm{\max \{\lambda\qty(A(x))\} - \min \{\lambda\qty(A(x))\}}{
%	A(x) = A_0 + x_1 A_1 + \cdots + x_n A_n}\\

%
%WHAT???

\newpage
\subsubsection{Part c}
\textbf{Problem:}
Minimize the conditional number of $A$ while remaining postive definite:
\begin{equation*}
	\begin{aligned}
		\text{minimize} \ \ &k(A(x)) = \frac{\lambda_1(x)}{\lambda_m(x)} \ \forall \ x \in \{x \ | \ A(x) \succ 0\}\\
		 \text{subject to} \ \ &A(x) \succ 0
	\end{aligned}
\end{equation*}
\textbf{Solution:}
This can be rewritten in epigraph form as
\optpblm{t_1 / t_2}{
	\lambda_1 \leq t_1\\
	& \lambda_m \leq t_2\\
	&A \succ 0
	}
or similarly as an SDP:
\optpblm{t_1 / t_2}{
	A(x) \preceq t_1 I\\
	&A(x) \succeq t_2\\
	&A \succ 0
	}

% Only Part a,b,c for 4.43

\newpage
\section{Problem 1: Open-loop optimal control with $1-$ and $\infty-$ norms.}
The following open-loop optimal regulation problem is given as:
\begin{equation}\label{eq:open-loop_opt-control_def}
	\begin{aligned}
		\text{minimze} \hspace{0.5in}
		&\norm{x_T}_p + \sum_{t = 0}^{T-1} \norm{x_t}_p + \gamma\norm{u_t}_q\\
		\text{subject to} \hspace{0.5in}
		& x_{t+1} = A x_t + B u_t, \ t = 0,\dots,T-1\\
		& \norm{x_t}_\infty \leq \bar{x}, \ t = 0,\dots,T\\
		& \norm{u_t}_\infty \leq \bar{u}, \ t = 0,\dots,T
	\end{aligned}
\end{equation}
with $x_t \in \real^n$ and $u_t \in \real^m$ as the system state and control input respectively and parameter $\gamma > 0$ governing the actuator and state regulation performance.\\

\textbf{Problem:}
Express this problem as a linear program for (i) $p=q=\infty$ and (ii) $p=q=1$. Code both in CVX and for the problem data provided. Verify the equivalence between the original optimization problem and transformed linear program obtained and plot the optimal state and input trajectories for each.\\

\textbf{Solution:}
\subsection{Linear program for $p = q = \infty$}

With $p = q = \infty$, the problem is defined as:
\optpblm{\norm{x_T}_\infty + \sum_{t = 0}^{T-1} \norm{x_t}_\infty + \gamma\norm{u_t}_\infty}{
	x_{t+1} = A x_t + B u_t, \ t = 0,\dots,T-1\\
	& \norm{x_t}_\infty \leq \bar{x}, \ t = 0,\dots,T\\
	& \norm{u_t}_\infty \leq \bar{u}, \ t = 0,\dots,T}

The epigraph of this problem can be found as
\optpblm{r_T + (r_0 + \gamma s_0) + (r_{T-1} + \gamma s_{T-1})}{
	\norm{x_t}_\infty \leq r_t, \ t = 0, \dots, T\\
	&\norm{u_i}_\infty \leq s_t, \ t = 0, \dots, T-1\\
	&x_{t+1} = A x_t + B u_t, \ t = 0,\dots,T-1\\
	& \norm{x_t}_\infty \leq \bar{x}, \ t = 0,\dots,T\\
	& \norm{u_t}_\infty \leq \bar{u}, \ t = 0,\dots,T
	}

From the definition of $\norm{x}_\infty = \max \{x\}$ and through vectorization, we can redefine this as the following linear program:
\optpblm{
	\mqty[\vb{1}^T & \gamma \vb{1}^T] \mqty[r\\s]
	=\vb{1}^T r + \gamma \vb{1}^T s}{
	x_{t+1} = A x_t + B u_t, \ t = 0,\dots,T-1\\
	&x_t \leq r_t \vb{1} \leq \bar{x} \vb{1}, \ t = 0, \dots, T\\
	&u_t \leq s_t \vb{1} \leq \bar{u} \vb{1}, \ t = 0, \dots, T-1
	}

\subsection{Linear program for $p = q = 1$}
With $p = q = 1$, the problem is defined as:
\optpblm{\norm{x_T}_1 + \sum_{t = 0}^{T-1} \norm{x_t}_1 + \gamma\norm{u_t}_1}{
	x_{t+1} = A x_t + B u_t, \ t = 0,\dots,T-1\\
	& \norm{x_t}_\infty \leq \bar{x}, \ t = 0,\dots,T\\
	& \norm{u_t}_\infty \leq \bar{u}, \ t = 0,\dots,T}

The epigraph of this problem can be found as
\optpblm{r_T + (r_0 + \gamma s_0) + (r_{T-1} + \gamma s_{T-1})}{
	\norm{x_t}_1 \leq r_t, \ t = 0, \dots, T\\
	&\norm{u_i}_1 \leq s_t, \ t = 0, \dots, T-1\\
	&x_{t+1} = A x_t + B u_t, \ t = 0,\dots,T-1\\
	& \norm{x_t}_\infty \leq \bar{x}, \ t = 0,\dots,T\\
	& \norm{u_t}_\infty \leq \bar{u}, \ t = 0,\dots,T
}

From the definition of $\norm{x}_1 = \sum_{i=0}^T x$ and through vectorization, we can redefine this as the following linear program:
\optpblm{
	\mqty[\vb{1}^T & \gamma \vb{1}^T] \mqty[r\\s]
	=\vb{1}^T r + \gamma \vb{1}^T s}{
	x_{t+1} = A x_t + B u_t, \ t = 0,\dots,T-1\\
	& \vb{1}^T x_t \leq r_t, \ t = 0,\dots,T\\
	& \vb{1}^T u_t \leq s_t, \ t = 0,\dots,T-1\\
	&x_t \leq \bar{x} \vb{1}, \ t = 0, \dots, T\\
	&u_t \leq \bar{u} \vb{1}, \ t = 0, \dots, T-1
}

\newpage
\subsection{CVX Formulation and Results:}
The code used to solve the linear programs and direct norm cvx calculations can be found in \appendixname \ref{apx:pblm1_matlab}.\\

\subsubsection{$\infty$-norm Solution}
\begin{figure}[h]
	\centering
	\includegraphics[width=1\linewidth]{fig/pblm1_inftyn_x}
	\caption{States for Open-loop control comparing methods for $\infty$-norm.}
	\label{fig:pblm1inftynx}
\end{figure}\newpage

\begin{figure}[h]
	\centering
	\includegraphics[width=1\linewidth]{fig/pblm1_inftyn_u}
	\caption{Inputs for Open-loop control comparing methods for $\infty$-norm.}
	\label{fig:pblm1inftynu}
\end{figure}\newpage

\subsubsection{1-norm Solution}
\begin{figure}[h]
	\centering
	\includegraphics[width=1\linewidth]{fig/pblm1_1n_x}
	\caption{States for Open-loop control comparing methods for 1-norm.}
	\label{fig:pblm11nx}
\end{figure}\newpage

\begin{figure}[h]
	\centering
	\includegraphics[width=1\linewidth]{fig/pblm1_1n_u}
	\caption{Inputs for Open-loop control comparing methods for 1-norm.}
	\label{fig:pblm11nu}
\end{figure}

\newpage
\section{Problem 2: Minimum time state transfer via quasiconvex optimization.}
Consider the LTI system:
\begin{equation}\label{eq:quasiconvex_opt_def}
	\begin{aligned}
		x_{t+1} &= Ax_t + B u_t, \ \forall t = 0,\dots,T\\
		\underline{u} &\leq u_t \leq \bar{u}, \ \forall t = 0,\dots,T
	\end{aligned}
\end{equation}
with $x_0$ as the initial state.\\

\textbf{Problem:}
Show that the minimum time required to transfer the system from $x_0$ to $x_{des}$, given as
\begin{equation}\label{eq:qualiconvex_problem_result}
	f(u_0,\dots,u_T) = \min \{\tau \ | \ x_t = x_{des} \ \text{for} \ \tau \leq t \leq {T+1}\}
\end{equation}
is a quasiconvex function of the control input sequence. Implement a bisection algorithm to solve the problem for the given data.\\

\textbf{Solution:}
It is evident from the definition of the minimization problem that the time required to reach the final state for each sequence of inputs is a convex optimization problem. From that, it is known that the function overall is a quasi-convex optimization problem defined as:
\optpblm{\tau}{
	x_{t+1} = Ax_t + B u_t \ \forall t = 0,\dots,T\\
	&\underline{u} \leq u_t \leq \bar{u} \ \forall t = 0,\dots,T\\
	&x(0) = x_0\\
	&x_t = x_{des} \ \forall t \in \{t \ | \ \tau \leq t \leq {T+1}\}
	}

For simplicity, we will be relaxing the final state to just reaching it at the earliest instead of remaining at rest:
$$x_\tau = x_{des}$$

A bisection algorithm can then be implemented to solve this as done using the MATLAB code shown in \appendixname \ \ref{apx:pblm2_matlab}. The result of this was a minimum value $$t = 51, \text{ or } \tau = 10.2$$

\newpage
The resulting system response and control sequence are provided as:
\begin{figure}[h]
	\centering
	\includegraphics[width=1\linewidth]{fig/pblm2}
	\caption{Results for problem 2.}
	\label{fig:pblm2}
\end{figure}

\newpage
\section{Problem 3: State feedback control design via SDP}
Feedback control problems can be formulated using a semidefinite program, such as
\begin{equation}\label{eq:feedback_control_def}
	\begin{aligned}
		\text{maximize} \hspace{0.5in}& \trace \{P\}\\
		\text{subject to} \hspace{0.5in}
		& \mqty [R + B^T PB & B^T PA\\
				 A^T PB & Q + A^T P A - P] \succeq 0\\
				 & P \succeq 0
	\end{aligned}
\end{equation}
with variable $P \in S^n$ and problem data $A\in \real^{n\cross n}, B \in\real^{n\cross m}, Q \in S^n_+, \real \in S^m_{++}$.\\

This problem is equivalent to the solution to the optimal solution to the infinite-horizon LQR problem:
\begin{equation}\label{eq:LQR_control_def}
	\begin{aligned}
		\text{minimze} \hspace{0.5in}& \sum_{t=0}^\infty x_t^T Q x_t + u_t^T R u_t\\
		\text{subject to} \hspace{0.5in}
		& x_{t+1} = Ax_t + B u_t, \ t \geq 0, \ x(t=0) = x_0
	\end{aligned}
\end{equation}
This is also equivelent to the solution the the discrete-time richotte equation (DARE) and can be solved in matlab with dare(A,B,Q,R). The solution to the feedback controller is
\begin{equation}\label{eq:LQR_control_solution}
	u_t = K x_t\\
	K = -\qty(R + B^T B)^{-1} B^T P^* A
\end{equation}

\textbf{Problem:}
Confirm the solution to the SDP given in \eqref{eq:feedback_control_def} is equivalent to the LQR problem given in \eqref{eq:LQR_control_def} for multiple randomly generated problems.

\textbf{Solution:}
CVX in MATLAB was used and the code can be found in \appendixname \ref{apx:pblm3_matlab}.The full set of results are provided in \appendixname \ref{apx:pblm3_results} for various randomly generated problems and solutions. The following is a few of P equivelent reuslts.

\begin{Verbatim}
	P_cvx =

	20.3336   10.5025   -3.3125   37.1904
	10.5025   12.9464   -1.8618   32.8376
	-3.3125   -1.8618    2.3441   -4.8914
	37.1904   32.8376   -4.8914   97.6353

	P_dare =

	20.3336   10.5025   -3.3125   37.1904
	10.5025   12.9464   -1.8618   32.8376
	-3.3125   -1.8618    2.3441   -4.8914
	37.1904   32.8376   -4.8914   97.6353
\end{Verbatim}
\begin{Verbatim}
	P_cvx =

	9.2752    0.3867    1.2843   -2.9794
	0.3867    4.4079    0.6458    1.1789
	1.2843    0.6458    1.9253    0.5430
	-2.9794    1.1789    0.5430    4.4857

	P_dare =

	9.2752    0.3867    1.2843   -2.9794
	0.3867    4.4079    0.6458    1.1789
	1.2843    0.6458    1.9253    0.5430
	-2.9794    1.1789    0.5430    4.4857
\end{Verbatim}
\begin{Verbatim}
	P_cvx =

	101.8040  -50.0081   -0.8792  117.2149
	-50.0081   28.3481    2.3037  -57.6749
	-0.8792    2.3037    4.1602    0.2983
	117.2149  -57.6749    0.2983  136.1756

	P_dare =

	101.8040  -50.0081   -0.8792  117.2149
	-50.0081   28.3481    2.3037  -57.6749
	-0.8792    2.3037    4.1602    0.2983
	117.2149  -57.6749    0.2983  136.1756
\end{Verbatim}

\newpage
\appendix
\section{MATLAB Code:}\label{apx:matlab}
All code I write in this course can be found on my GitHub repository:\\
\href{https://github.com/jonaswagner2826/MECH6337}{https://github.com/jonaswagner2826/MECH6327}
\lstinputlisting[caption={MECH6327\_HW3},label={script:HW3}]{MECH6327_HW3.m}

\newpage
\section{Problem 3 MATLAB Code:}\label{apx:pblm1_matlab}
All code I write in this course can be found on my GitHub repository:\\
\href{https://github.com/jonaswagner2826/MECH6327}{https://github.com/jonaswagner2826/MECH6327}
% MECH6313_HW3_pblm1
\lstinputlisting[caption={MECH6327\_HW3\_pblm1},label={script:HW3}]{MECH6327_HW3_pblm1.m}

\newpage
\section{Problem 3 MATLAB Code:}\label{apx:pblm2_matlab}
All code I write in this course can be found on my GitHub repository:\\
\href{https://github.com/jonaswagner2826/MECH6327}{https://github.com/jonaswagner2826/MECH6327}
% MECH6313_HW3_pblm2
\lstinputlisting[caption={MECH6327\_HW3\_pblm2},label={script:HW3}]{MECH6327_HW3_pblm2.m}

\newpage
\section{Problem 3 MATLAB Code:}\label{apx:pblm3_matlab}
All code I write in this course can be found on my GitHub repository:\\
\href{https://github.com/jonaswagner2826/MECH6327}{https://github.com/jonaswagner2826/MECH6327}
% MECH6313_HW3_pblm3
\lstinputlisting[caption={MECH6327\_HW3\_pblm3},label={script:HW3}]{MECH6327_HW3_pblm3.m}

\newpage
\section{Problem 3 MATLAB Results:}\label{apx:pblm3_results}

\begin{Verbatim}
	>> MECH6327_HW3_pblm3

	A =

	1.1002   -1.1372    1.1077    0.2641
	0.1751    0.6430    0.8205    3.1585
	1.0036   -0.0128   -0.8176    1.2266
	1.5110    0.9143   -0.1265    2.3206

	B =

	0.4145    1.2416
	0.2118   -0.1576
	0.6132   -1.3736
	-0.5278    0.8708

	Q =

	0.4766    0.4525    0.2565    0.4911
	0.4525    0.5808    0.3777    0.7950
	0.2565    0.3777    0.4440    0.4396
	0.4911    0.7950    0.4396    1.2997

	R =

	1.1591    0.8154
	0.8154    0.7716

	Calling SDPT3 4.0: 31 variables, 10 equality constraints
	For improved efficiency, SDPT3 is solving the dual problem.
	------------------------------------------------------------

	num. of constraints = 10
	dim. of sdp    var  = 10,   num. of sdp  blk  =  2
	*******************************************************************
	SDPT3: Infeasible path-following algorithms
	*******************************************************************
	version  predcorr  gam  expon  scale_data
	HKM      1      0.000   1        0
	it pstep dstep pinfeas dinfeas  gap      prim-obj      dual-obj    cputime
	-------------------------------------------------------------------
	0|0.000|0.000|8.4e+01|1.2e+01|1.5e+03| 4.731862e+01  0.000000e+00| 0:0:00| chol  1  1
	1|0.894|0.821|8.9e+00|2.3e+00|2.2e+02| 2.141617e+01  1.453677e+01| 0:0:00| chol  1  1
	2|0.833|0.841|1.5e+00|3.7e-01|5.1e+01| 2.431086e+01  1.330905e+01| 0:0:00| chol  1  1
	3|0.515|0.846|7.2e-01|5.7e-02|2.1e+01| 2.140755e+01  2.132680e+01| 0:0:00| chol  1  1
	4|0.187|0.237|5.9e-01|4.4e-02|1.9e+01| 2.434592e+01  7.419460e+01| 0:0:00| chol  1  1
	5|0.061|0.042|5.5e-01|4.2e-02|2.7e+01| 3.321322e+01  1.216786e+02| 0:0:00| chol  1  1
	6|0.104|0.026|4.9e-01|4.1e-02|5.4e+01| 6.400240e+01  6.740367e+01| 0:0:00| chol  1  1
	7|0.129|0.433|4.3e-01|2.3e-02|5.0e+01| 7.772307e+01  1.153429e+02| 0:0:00| chol  1  1
	8|1.000|0.825|1.4e-08|4.0e-03|4.5e+01| 1.616373e+02  1.181693e+02| 0:0:00| chol  1  1
	9|0.962|0.980|2.6e-07|7.9e-05|1.8e+00| 1.344089e+02  1.326294e+02| 0:0:00| chol  1  1
	10|0.965|0.977|9.4e-09|1.8e-06|5.8e-02| 1.333014e+02  1.332444e+02| 0:0:00| chol  1  1
	11|0.958|1.000|3.9e-10|1.9e-09|3.6e-03| 1.332623e+02  1.332587e+02| 0:0:00| chol  1  1
	12|0.987|1.000|7.9e-12|7.9e-11|2.8e-04| 1.332596e+02  1.332594e+02| 0:0:00| chol  1  1
	13|0.952|0.987|1.2e-11|2.6e-12|1.3e-05| 1.332595e+02  1.332595e+02| 0:0:00| chol  1  1
	14|1.000|1.000|5.2e-12|2.3e-12|1.2e-06| 1.332595e+02  1.332595e+02| 0:0:00|
	stop: max(relative gap, infeasibilities) < 1.49e-08
	-------------------------------------------------------------------
	number of iterations   = 14
	primal objective value =  1.33259457e+02
	dual   objective value =  1.33259456e+02
	gap := trace(XZ)       = 1.15e-06
	relative gap           = 4.31e-09
	actual relative gap    = 4.31e-09
	rel. primal infeas (scaled problem)   = 5.20e-12
	rel. dual     "        "       "      = 2.32e-12
	rel. primal infeas (unscaled problem) = 0.00e+00
	rel. dual     "        "       "      = 0.00e+00
	norm(X), norm(y), norm(Z) = 1.7e+02, 1.1e+02, 1.7e+03
	norm(A), norm(b), norm(C) = 3.5e+01, 3.0e+00, 3.9e+00
	Total CPU time (secs)  = 0.47
	CPU time per iteration = 0.03
	termination code       =  0
	DIMACS: 7.8e-12  0.0e+00  4.0e-12  0.0e+00  4.3e-09  4.3e-09
	-------------------------------------------------------------------

	------------------------------------------------------------
	Status: Solved
	Optimal value (cvx_optval): -133.259

	P_cvx =

	20.3336   10.5025   -3.3125   37.1904
	10.5025   12.9464   -1.8618   32.8376
	-3.3125   -1.8618    2.3441   -4.8914
	37.1904   32.8376   -4.8914   97.6353

	K_cvx =

	1.3793    2.0277   -0.1335    4.7523
	-1.0233    0.0284   -0.4975   -1.2125

	P_dare =

	20.3336   10.5025   -3.3125   37.1904
	10.5025   12.9464   -1.8618   32.8376
	-3.3125   -1.8618    2.3441   -4.8914
	37.1904   32.8376   -4.8914   97.6353

	K_dare =

	-1.3793   -2.0277    0.1335   -4.7523
	1.0233   -0.0284    0.4975    1.2125

	P_cvx =

	20.3336   10.5025   -3.3125   37.1904
	10.5025   12.9464   -1.8618   32.8376
	-3.3125   -1.8618    2.3441   -4.8914
	37.1904   32.8376   -4.8914   97.6353

	P_dare =

	20.3336   10.5025   -3.3125   37.1904
	10.5025   12.9464   -1.8618   32.8376
	-3.3125   -1.8618    2.3441   -4.8914
	37.1904   32.8376   -4.8914   97.6353

	>> MECH6327_HW3_pblm3

	A =

	-0.8568    1.3798   -0.6563    1.1284
	0.0484    0.0951   -0.1250    0.7425
	-0.6649   -0.4271   -0.5305    1.1436
	1.4527    0.5108    0.1056   -0.9147

	B =

	0.1798    1.2963
	-0.9833    1.0992
	0.3848    0.6532
	0.3257   -0.5051

	Q =

	0.3838    0.3714    0.4771    0.6771
	0.3714    1.2344    1.2453    1.0731
	0.4771    1.2453    1.4076    1.4208
	0.6771    1.0731    1.4208    2.1397

	R =

	0.5081    0.7407
	0.7407    1.1909

	Calling SDPT3 4.0: 31 variables, 10 equality constraints
	For improved efficiency, SDPT3 is solving the dual problem.
	------------------------------------------------------------

	num. of constraints = 10
	dim. of sdp    var  = 10,   num. of sdp  blk  =  2
	*******************************************************************
	SDPT3: Infeasible path-following algorithms
	*******************************************************************
	version  predcorr  gam  expon  scale_data
	HKM      1      0.000   1        0
	it pstep dstep pinfeas dinfeas  gap      prim-obj      dual-obj    cputime
	-------------------------------------------------------------------
	0|0.000|0.000|4.4e+01|5.2e+00|1.0e+03| 6.864540e+01  0.000000e+00| 0:0:00| chol  1  1
	1|0.886|0.852|5.0e+00|8.2e-01|1.7e+02| 5.707109e+01  1.265425e+01| 0:0:00| chol  1  1
	2|0.782|1.000|1.1e+00|5.5e-03|5.2e+01| 5.048488e+01  1.175262e+01| 0:0:00| chol  1  1
	3|0.752|1.000|2.7e-01|5.5e-04|1.6e+01| 2.602445e+01  1.535157e+01| 0:0:00| chol  1  1
	4|1.000|0.756|1.8e-07|1.8e-04|7.3e+00| 2.578672e+01  1.846254e+01| 0:0:00| chol  1  1
	5|0.933|1.000|1.8e-08|5.6e-06|5.8e-01| 2.048293e+01  1.990168e+01| 0:0:00| chol  1  1
	6|0.964|0.970|2.0e-09|7.1e-07|2.9e-02| 2.011379e+01  2.008495e+01| 0:0:00| chol  1  1
	7|0.965|1.000|6.3e-10|5.6e-08|1.7e-03| 2.009536e+01  2.009368e+01| 0:0:00| chol  1  1
	8|0.994|1.000|1.8e-10|1.3e-10|1.2e-04| 2.009415e+01  2.009404e+01| 0:0:00| chol  1  1
	9|0.953|0.987|6.7e-11|3.7e-11|5.3e-06| 2.009409e+01  2.009408e+01| 0:0:00| chol  1  1
	10|1.000|1.000|6.7e-15|1.3e-11|1.2e-06| 2.009408e+01  2.009408e+01| 0:0:00| chol  1  1
	11|1.000|1.000|1.7e-14|1.0e-12|1.3e-08| 2.009408e+01  2.009408e+01| 0:0:00|
	stop: max(relative gap, infeasibilities) < 1.49e-08
	-------------------------------------------------------------------
	number of iterations   = 11
	primal objective value =  2.00940824e+01
	dual   objective value =  2.00940824e+01
	gap := trace(XZ)       = 1.33e-08
	relative gap           = 3.22e-10
	actual relative gap    = 3.22e-10
	rel. primal infeas (scaled problem)   = 1.70e-14
	rel. dual     "        "       "      = 1.00e-12
	rel. primal infeas (unscaled problem) = 0.00e+00
	rel. dual     "        "       "      = 0.00e+00
	norm(X), norm(y), norm(Z) = 8.9e+00, 1.2e+01, 8.9e+01
	norm(A), norm(b), norm(C) = 1.8e+01, 3.0e+00, 5.7e+00
	Total CPU time (secs)  = 0.42
	CPU time per iteration = 0.04
	termination code       =  0
	DIMACS: 2.6e-14  0.0e+00  1.8e-12  0.0e+00  3.2e-10  3.2e-10
	-------------------------------------------------------------------

	------------------------------------------------------------
	Status: Solved
	Optimal value (cvx_optval): -20.0941

	P_cvx =

	9.2752    0.3867    1.2843   -2.9794
	0.3867    4.4079    0.6458    1.1789
	1.2843    0.6458    1.9253    0.5430
	-2.9794    1.1789    0.5430    4.4857

	K_cvx =

	0.5888   -0.3683    0.2601   -0.1295
	0.7294   -0.5856    0.4291   -0.9410

	P_dare =

	9.2752    0.3867    1.2843   -2.9794
	0.3867    4.4079    0.6458    1.1789
	1.2843    0.6458    1.9253    0.5430
	-2.9794    1.1789    0.5430    4.4857

	K_dare =

	-0.5888    0.3683   -0.2601    0.1295
	-0.7294    0.5856   -0.4291    0.9410

	P_cvx =

	9.2752    0.3867    1.2843   -2.9794
	0.3867    4.4079    0.6458    1.1789
	1.2843    0.6458    1.9253    0.5430
	-2.9794    1.1789    0.5430    4.4857

	P_dare =

	9.2752    0.3867    1.2843   -2.9794
	0.3867    4.4079    0.6458    1.1789
	1.2843    0.6458    1.9253    0.5430
	-2.9794    1.1789    0.5430    4.4857

	>> MECH6327_HW3_pblm3

	A =

	-1.5312    0.7880    1.6345   -0.9443
	0.5046    0.2982   -0.6235   -0.6712
	-0.8642   -0.1637   -1.3501    0.5767
	-0.3766    0.6067   -1.1622   -2.0858

	B =

	0.2360    0.0076
	-0.7784   -0.9376
	1.0996   -0.6816
	-0.8556   -0.2601

	Q =

	1.5071    1.3058    1.2427    1.5227
	1.3058    2.0018    1.1626    1.6402
	1.2427    1.1626    1.9380    2.0005
	1.5227    1.6402    2.0005    2.2226

	R =

	0.1582    0.2334
	0.2334    0.4393

	Calling SDPT3 4.0: 31 variables, 10 equality constraints
	For improved efficiency, SDPT3 is solving the dual problem.
	------------------------------------------------------------

	num. of constraints = 10
	dim. of sdp    var  = 10,   num. of sdp  blk  =  2
	*******************************************************************
	SDPT3: Infeasible path-following algorithms
	*******************************************************************
	version  predcorr  gam  expon  scale_data
	HKM      1      0.000   1        0
	it pstep dstep pinfeas dinfeas  gap      prim-obj      dual-obj    cputime
	-------------------------------------------------------------------
	0|0.000|0.000|4.4e+01|3.9e+00|1.0e+03| 8.266937e+01  0.000000e+00| 0:0:00| chol  1  1
	1|0.818|0.880|8.1e+00|5.1e-01|1.7e+02| 3.173665e+01  9.183288e+00| 0:0:00| chol  1  1
	2|0.761|0.758|1.9e+00|1.3e-01|6.0e+01| 2.745402e+01  1.125760e+01| 0:0:00| chol  1  1
	3|0.553|0.897|8.6e-01|1.3e-02|2.5e+01| 2.195296e+01  1.974743e+01| 0:0:00| chol  1  1
	4|0.169|0.214|7.2e-01|1.1e-02|2.1e+01| 2.472518e+01  1.029151e+02| 0:0:00| chol  1  1
	5|0.017|0.022|7.1e-01|1.0e-02|3.3e+01| 3.109723e+01  1.695860e+02| 0:0:00| chol  1  1
	6|0.061|0.035|6.6e-01|1.0e-02|7.8e+01| 8.019566e+01  2.054055e+02| 0:0:00| chol  1  1
	7|0.423|0.391|3.8e-01|6.1e-03|1.4e+02| 2.228571e+02  2.222143e+02| 0:0:00| chol  1  1
	8|1.000|0.531|8.3e-06|2.8e-03|1.1e+02| 3.445733e+02  2.448139e+02| 0:0:00| chol  2  1
	9|0.952|1.000|1.7e-06|1.7e-06|1.6e+01| 2.823759e+02  2.660703e+02| 0:0:00| chol  1  1
	10|0.950|0.991|8.3e-08|3.5e-07|1.1e+00| 2.712737e+02  2.701719e+02| 0:0:00| chol  1  1
	11|1.000|1.000|2.1e-10|1.7e-08|1.1e-01| 2.705570e+02  2.704431e+02| 0:0:00| chol  1  1
	12|0.959|0.980|1.7e-10|4.8e-10|4.0e-03| 2.704909e+02  2.704869e+02| 0:0:00| chol  1  1
	13|0.993|1.000|1.0e-10|3.4e-11|2.4e-04| 2.704880e+02  2.704878e+02| 0:0:00| chol  1  1
	14|0.954|0.989|8.0e-11|2.1e-11|1.1e-05| 2.704879e+02  2.704879e+02| 0:0:00| chol  1  1
	15|1.000|1.000|1.7e-10|1.6e-11|1.0e-06| 2.704879e+02  2.704879e+02| 0:0:00|
	stop: max(relative gap, infeasibilities) < 1.49e-08
	-------------------------------------------------------------------
	number of iterations   = 15
	primal objective value =  2.70487886e+02
	dual   objective value =  2.70487885e+02
	gap := trace(XZ)       = 1.04e-06
	relative gap           = 1.92e-09
	actual relative gap    = 1.87e-09
	rel. primal infeas (scaled problem)   = 1.72e-10
	rel. dual     "        "       "      = 1.59e-11
	rel. primal infeas (unscaled problem) = 0.00e+00
	rel. dual     "        "       "      = 0.00e+00
	norm(X), norm(y), norm(Z) = 8.9e+02, 2.2e+02, 1.4e+03
	norm(A), norm(b), norm(C) = 2.2e+01, 3.0e+00, 7.5e+00
	Total CPU time (secs)  = 0.45
	CPU time per iteration = 0.03
	termination code       =  0
	DIMACS: 2.6e-10  0.0e+00  3.7e-11  0.0e+00  1.9e-09  1.9e-09
	-------------------------------------------------------------------

	------------------------------------------------------------
	Status: Solved
	Optimal value (cvx_optval): -270.488

	P_cvx =

	101.8040  -50.0081   -0.8792  117.2149
	-50.0081   28.3481    2.3037  -57.6749
	-0.8792    2.3037    4.1602    0.2983
	117.2149  -57.6749    0.2983  136.1756

	K_cvx =

	-4.7337    3.0098    1.0219   -6.7728
	0.2527   -0.0799   -1.2428    0.2020

	P_dare =

	101.8040  -50.0081   -0.8792  117.2149
	-50.0081   28.3481    2.3037  -57.6749
	-0.8792    2.3037    4.1602    0.2983
	117.2149  -57.6749    0.2983  136.1756

	K_dare =

	4.7337   -3.0098   -1.0219    6.7728
	-0.2527    0.0799    1.2428   -0.2020

	P_cvx =

	101.8040  -50.0081   -0.8792  117.2149
	-50.0081   28.3481    2.3037  -57.6749
	-0.8792    2.3037    4.1602    0.2983
	117.2149  -57.6749    0.2983  136.1756

	P_dare =

	101.8040  -50.0081   -0.8792  117.2149
	-50.0081   28.3481    2.3037  -57.6749
	-0.8792    2.3037    4.1602    0.2983
	117.2149  -57.6749    0.2983  136.1756
\end{Verbatim}

%\newpage
%\section*{}
%\bibliographystyle{ieeetr}
%\bibliography{mybib.bib}

\end{document}


%!TEX root = ms.tex
\section{ Information criteria for fixed-X }
\label{sec:ic_fixedx}
\subsection{KL-based information criterion}
%In this section, we assume fixed-X, and provide expressions for C$_p$, FPE and AICc under general linear restrictions on $\beta$. The expressions of C$_p(k)$, FPE$(k)$ and AICc$(k)$ given in \eqref{eq:cp_subsetselection}, \eqref{eq:cptilde_subsetselection} and \eqref{eq:aicc_subsetselection}, respectively, for variable selection can be obtained as special cases of the general expressions.

Using the likelihood function \eqref{eq:loglike_fixedx} and the MLE \eqref{eq:betahat_sigmahatsq}, the expected log-likelihood can be derived as
\begin{equation*}
\begin{aligned}
\text{ErrF}_\text{KL} &=  E_{\tilde{y}} [-2 \log f( \tilde{y} | X,\hat\beta,\hat\sigma^2 )] =  n \log (2\pi \hat\sigma^2) + \frac{1}{\hat\sigma^2} E_{\tilde{y}} || \tilde{y}-X\hat\beta||_2^2 \\
%&= n \log (2\pi \sigma^2) + \frac{1}{\sigma^2} E_\tilde{y} || X\beta_0-X \beta+\epsilon||_2^2 \\
&= n \log (2\pi \hat\sigma^2) + \frac{1}{\hat\sigma^2}  (\hat\beta-\beta_0)^T X^T X (\hat\beta-\beta_0) + \frac{n\sigma_0^2}{\hat\sigma^2},
\end{aligned}
\end{equation*}
and the training error is
\begin{equation*}
\text{errF}_\text{KL} = -2\log f(y|X,\hat\beta,\hat\sigma^2) = n\log(2\pi\hat\sigma^2) + n.
\end{equation*}
In the context of variable selection, the assumption that the approximating model includes the true model is used in the derivations of AIC \citep{linhart1986model} and AICc \citep{hurvich1989regression}. This assumption can be generalized to the context of general restrictions.
\begin{assumption}
If the approximating model satisfies the restrictions $R\beta = r$, then the true model satisfies the analogous restrictions $R\beta_0 = r$; that is, the true model is at least as restrictive as the approximating model.
\label{assumption}
\end{assumption}
Under this assumption, we have the following lemma. The proofs for all of the lemmas and theorems in this paper are given in the Supplemental Material.
\begin{lemma}
  Under Assumption \ref{assumption}, $\hat\sigma^2$ and the quadratic form $(\hat \beta-\beta_0)^T X^T X (\hat \beta-\beta_0)$ are independent, and
  \begin{equation*}
  \begin{aligned}
    n \sigma_0^2 E_y\left[ \frac{1}{\hat{\sigma}^2} \right] &= n\frac{n}{n-p+m-2},\\
    E_y  \left [ (\hat \beta-\beta_0)^T X^T X (\hat \beta-\beta_0) \right ] &= \sigma_0^2 (p-m).
  \end{aligned}
  \end{equation*}
\label{thm:components_ekl_lr_fixedx}
\end{lemma}
Lemma \ref{thm:components_ekl_lr_fixedx} provides the fundamentals for calculating the expected optimism.
\begin{theorem}
Under Assumption \ref{assumption},
\begin{equation*}
E_y(\text{optF}_\text{KL}) = n \frac{n+p-m}{n-p+m-2} - n.
\end{equation*}
\label{thm:EoptF_KL}
\end{theorem}

Consequently,
\begin{equation*}
\widehat{\text{ErrF}}_\text{KL} = \text{errF}_\text{KL} + E_y(\text{optF}_\text{KL}) = n\log(\hat\sigma^2) + n \frac{n+p-m}{n-p+m-2} + n\log(2\pi)
\end{equation*}
is an unbiased estimator of the test error $E_y \left[ \text{ErrF}_\text{KL} \right]$. We follow the same tradition as in the derivations of AIC and AICc that since the term $n\log(2\pi)$ appears in $\widehat{\text{ErrF}}_\text{KL}$ for every model being compared, it is irrelevant for purposes of model selection. We therefore ignore this term and define
\begin{equation*}
 \text{AICc}(R,r) = n\log \left( \frac{\text{RSS}(R,r)}{n}  \right) + n \frac{n+p-m}{n-p+m-2},
\end{equation*}
where RSS$(R,r)= \lVert y -X\hat\beta \rVert_2^2$. For the variable selection problem, e.g. regressing on a subset of predictors with size $k$, we are restricting $p-k$ slope coefficients to be zero. By plugging $\hat\beta = \hat\beta(k)$ and $m=p-k$ into the expressions of AICc$(R,r)$, we obtain AICc$(k)$ given in \eqref{eq:aicc_subsetselection}.

\subsection{Squared error-based information criterion}
The covariance penalty \eqref{eq:EoptF_SE} is defined for any general fitting procedure. By explicitly calculating the covariance term for $\hat\mu=X\hat\beta$, we can obtain the expected optimism.
\begin{theorem}
\begin{equation*}
E_y (\text{optF}_\text{SE}) = 2 \sigma_0^2 (p-m).
\end{equation*}
\label{thm:EoptF_SE}
\end{theorem}

An immediate consequence of this is that
\begin{equation*}
\widehat{\text{ErrF}}_\text{SE} = \text{errF}_\text{SE} + E_y(\text{optF}_\text{SE}) = \text{RSS}(R,r) + 2 \sigma_0^2 (p-m)
\end{equation*}
is an unbiased estimator of $E_y(\text{ErrF}_\text{SE})$. Using the unbiased estimator of $\sigma_0^2$ given by the OLS fit based on all of the predictors, i.e. $\hat\sigma_0^2=\text{RSS}(p)/(n-p)$, we define
\begin{equation*}
\text{C}_p(R,r) = \text{RSS}(R,r) + \frac{\text{RSS}(p)}{n-p} 2(p-m).
\end{equation*}
An alternative estimate of $\sigma_0^2$ is $\text{RSS}(R,r)/(n-p+m)$, which yields
\begin{equation*}
\text{FPE}(R,r) = \text{RSS}(R,r)\frac{n+p-m}{n-p+m}.
\end{equation*}
For the variable selection problem, by substituting $m=p-k$ into the expressions of C$_p$ and FPE, we obtain the previously-noted definitions of them, i.e. C$_p$(k) and FPE(k) given in \eqref{eq:cp_subsetselection} and \eqref{eq:cptilde_subsetselection}, respectively.

\section{ Information criteria for random-X }
\label{sec:ic_randomx}
%In this section, we assume random-X. We start by proposing and deriving a novel KL-based criterion, RAICc. We further derive the criteria RC$_p$ and S$_p$ under linear restrictions on $\beta$.

\subsection{KL-based information criterion, RAICc}
We replace the unknown parameters by their MLE, and have the fitted model $f(\cdot|\hat\beta,\hat\sigma^2,\hat\Sigma)$. The KL information measures how well the fitted model predicts the new set of data $(X^{(n)},y^{(n)})$, in terms of the closeness of the distributions of $(X^{(n)},y^{(n)})$ based on the fitted model and the true model, i.e.
\begin{equation}
\text{KLR} = E_{X^{(n)},y^{(n)}} \left[ 2\log f(X^{(n)},y^{(n)}|\beta_0,\sigma_0^2,\Sigma_0) -2 \log f(X^{(n)},y^{(n)}|\hat\beta,\hat\sigma^2,\hat\Sigma) \right].
\label{eq:KLR}
\end{equation}
An equivalent form for model comparisons is the expected log-likelihood
\begin{equation*}
\begin{aligned}
&\text{ErrR}_\text{KL} =  E_{X^{(n)},y^{(n)}} \left[ -2 \log f(X^{(n)},y^{(n)}|\hat\beta,\hat\sigma^2,\hat\Sigma) \right] \\
&= \left[ n \log (2\pi \hat\sigma^2) + \frac{1}{\hat\sigma^2} E_{X^{(n)},y^{(n)}} || y^{(n)}-X^{(n)}\hat\beta||_2^2 \right ] + \left [np \log(2\pi) + n \log |\hat\Sigma| + E_{X^{(n)}} \left(\sum_{i=1}^n {x_{i}^{(n)}}^T \hat\Sigma^{-1} x_{i}^{(n)} \right) \right]\\
&= \left[ n \log (2\pi \hat\sigma^2) + \frac{n}{\hat\sigma^2}  (\hat\beta-\beta_0)^T \Sigma_0 (\hat\beta-\beta_0) + \frac{n\sigma_0^2}{\hat\sigma^2} \right ] + \left [np \log(2\pi) + n \log |\hat\Sigma| + n \text{Tr}(\hat\Sigma^{-1}\Sigma_{0})\right],
\end{aligned}
\end{equation*}
and the training error is
\begin{equation*}
\text{errR}_\text{KL} = -2\log f(X,y|\hat\beta,\hat\sigma^2,\hat\Sigma) = \left [ n \log (2\pi \hat\sigma^2) + n \right ] + \left [np \log(2\pi) + n \log |\hat\Sigma| + np \right ].
\end{equation*}

As in the fixed-X case, we assume that the true model satisfies the restrictions, i.e. $R\beta_0=r$, and we obtain the following lemma.
\begin{lemma}
Under Assumption \ref{assumption}, $\hat\sigma^2$ and $(\hat{\beta}-\beta_0)^T \Sigma_0 (\hat{\beta}-\beta_0)$ are independent conditionally on $X$, and
\begin{equation*}
\begin{aligned}
E \left[ \text{Tr}(\hat \Sigma^{-1}\Sigma_0) \right] &= \frac{np}{n-p-1},\\
%n\sigma_0^2 E_{X,y} \left[ \frac{1}{\hat\sigma^2} \right] &= n \frac{n}{n-p+m-2},\\
E_{X,y}  \left [ (\hat \beta-\beta_0)^T \Sigma_0 (\hat \beta-\beta_0) \right ] &= \sigma_0^2 \frac{p-m}{n-p+m-1}.
\end{aligned}
\end{equation*}
\label{thm:components_ekl_lr_randomx}
\end{lemma}
Lemma \ref{thm:components_ekl_lr_randomx} provides the components for calculating the expected optimism.
\begin{theorem}
Under Assumption \ref{assumption},
\begin{equation*}
E_{X,y}(\text{optR}_\text{KL}) = n \frac{n(n-1)}{(n-p+m-2)(n-p+m-1)} + n \frac{np}{n-p-1} - n(p+1).
\end{equation*}
\label{thm:EoptR_KL}
\end{theorem}

Consequently,
\begin{equation*}
\begin{aligned}
\widehat{\text{ErrR}}_\text{KL} &= \text{errR}_\text{KL} + E_{X,y} (\text{optR}_\text{KL}) \\
&=n\log \left(\hat\sigma^2\right) + n \frac{n(n-1)}{(n-p+m-2)(n-p+m-1)} + n\log(2\pi)(p+1) + n\frac{np}{n-p-1} + n\log|\hat\Sigma|
\end{aligned}
\end{equation*}
is an unbiased estimator of the test error $E_{X,y}(\text{ErrR}_\text{KL})$. Note that the last three terms are free of the restrictions and only depend on $n$, $p$ and $X$. They are the same when we compare two models with different restrictions on $\beta$, and are thus irrelevant when comparing criteria for any two such models. Therefore, for the purpose of model selection, we define
\begin{equation*}
\text{RAICc}(R,r) = n\log \left(\frac{\text{RSS}(R,r)}{n}\right) + n \frac{n(n-1)}{(n-p+m-2)(n-p+m-1)}.
\end{equation*}
An equivalent form is
\begin{equation*}
\text{RAICc}(R,r) = \text{AICc}(R,r) + \frac{n(p-m)(p-m+1)}{(n-p+m-1)(n-p+m-2)}.
\end{equation*}
For linear regression on a subset of predictors with size $k$, we are restricting $p-k$ coefficients to be zero. By substituting $m=p-k$ and $\hat\beta = \hat\beta(k)$ into the expression of $\text{RAICc}(R,r)$, we obtain the RAICc criterion for the variable selection problem, i.e.
\begin{equation*}
\text{RAICc}(k) = n\log \left(\frac{\text{RSS}(k)}{n}\right) + n \frac{n(n-1)}{(n-k-2)(n-k-1)}.
\end{equation*}

\subsection{Squared error-based information criteria}
According to \citet[formula 6 and proposition 1]{rosset2020fixed}, $E_{X,y}(\text{optR}_\text{SE})$ can be decomposed into $E_{X,y}(\text{optF}_\text{SE})$ plus an excess bias term and an excess variance term. We calculate both terms for our estimator $\hat\beta$ and obtain the following theorem.
\begin{theorem}
Under Assumption \ref{assumption},
\begin{equation*}
E_{X,y}(\text{optR}_\text{SE}) = \sigma_0^2(p-m) \left( 2+ \frac{p-m+1}{n-p+m-1} \right).
\end{equation*}
\label{thm:EoptR_SE}
\end{theorem}
An immediate consequence is that
\begin{equation*}
\widehat{\text{ErrR}}_\text{SE} = \text{errR}_\text{SE} + E_{X,y} (\text{optR}_\text{SE}) = \text{RSS}(R,r) + \sigma_0^2(p-m) \left( 2+ \frac{p-m+1}{n-p+m-1} \right)
\end{equation*}
is an unbiased estimator of $E_{X,y}(\text{ErrR}_\text{SE})$. Using the OLS fit on all of the predictors to estimate $\sigma_0^2$, we have
\begin{equation*}
\text{RC}_p(R,r) = \text{RSS}(R,r) + \frac{\text{RSS(p)}}{n-p}(p-m) \left(2+\frac{p-m+1}{n-p+m-1}\right).
\end{equation*}
An alternative estimate of $\sigma_0^2$ is $\text{RSS}(R,r)/(n-p+m)$, which yields
\begin{equation*}
\text{S}_p(R,r) = \text{RSS}(R,r)\frac{n(n-1)}{(n-p+m)(n-p+m-1)}.
\end{equation*}
For the variable selection problem, by substituting $m=p-k$ into the expressions of RC$_p$ and S$_p$, we obtain the previously-noted definitions of them, i.e. RC$_p$(k) and S$_p$(k) given in \eqref{eq:rcp_subsetselection} and \eqref{eq:sp_subsetselection}, respectively.