\section{Barron theory of approximation}
%\subsection{A DL function class}
\subsection{Approximation result for cosine as activation function}
For clarity, let us give a brief introduction of the approximation
result based on the results in Jones \cite{jones1992simple}, Barron \cite{barron1993universal} and some
modification in Xu \cite{xu2017approximation}.

Given $d$ and $n$, consider the following nonlinear space in $\mathbb
R^d$:
\begin{equation}
	\label{Vn}
	V_n=\left\{v: v(x)=\sum_{j=1}^n{a_j\over n}\cos(\omega_j\cdot x+b_j)+c,
	\mbox{ with }
	a_j, b_j, c\in\mathbb R, \omega_j\in\mathbb R^d
	\right\}.
\end{equation}
\begin{theorem} \label{jones} Given a bounded domain $B\subset\mathbb R^d$, a probability measure $\mu$ on $B$
	and a function $f:\mathbb R^d\mapsto\mathbb R$ whose Fourier
	transform $\hat f$ satisfying $\|\hat f\|_{L^1(\mathbb R^d)}<\infty$.
	Then, for any $n\ge 1$,  there exists $f_n\in V_n$  such that
	\begin{equation}
		\|f-f_n\|_{L^2(B)}\le \frac{2\|\hat f\|_{L^1(\mathbb R^d)}}{\sqrt{n}},
	\end{equation}
	where $f_n(x)=\sum_{j=1}^n{a_j\over n}\cos(\omega_j\cdot x+b_j)+c$ satisfying
	\begin{equation}
		\label{abc}
		a=  \|\hat f\|_{L^1(\mathbb R^d)}, \quad |b_j|\le \pi, \quad |c|\le \|f\|_{0,\infty,B}+\|\hat f\|_{L^1(\mathbb R^d)}.
	\end{equation}
\end{theorem}
\begin{proof}\small
	Consider the Fourier transform:
	\begin{equation}
	  \label{Fourier}
	  \hat f(\omega)=\frac{1}{(2\pi)^d}\int_{\mathbb{R}^d}e^{-i\omega\cdot x}f(x)dx
	  \quad \forall \omega \in \mathbb R^d.
	\end{equation}
	We write
	$\hat{f}(\omega)=e^{i\theta(\omega)}|\hat{f}(\omega)|$. By Fourier inversion formula,
	\begin{equation}
		\label{eqn1}
		f(x)=\int_{\mathbb{R}^d}e^{i\omega\cdot x}\hat{f}(\omega)d\omega
		=\int_{\mathbb{R}^d}e^{i(\omega\cdot x+\theta(\omega))}|\hat{f}(\omega)|d\omega.
	\end{equation}
Since $f(x)$ is real-valued, it implies that, for $x, x_B\in B$
	  \begin{equation}
		\label{key}
		\begin{aligned}
			f(x)-f(x_B)
			&={\rm Re}\int_{\mathbb{R}^d}
			(e^{i\omega\cdot x}-e^{i\omega\cdot x_B})
			\hat{f}(\omega)d\omega \\
			&={\rm Re}\int_{\mathbb{R}^d}
			(e^{i\omega\cdot x}-e^{i\omega\cdot x_B})
			e^{i\theta
				(\omega)}|\hat{f}(\omega)|d\omega \\
			&=\int_{\mathbb{R}^d}(\cos(\omega\cdot
			x+\theta(\omega))-\cos(\omega\cdot x_B+\theta(\omega)))|\hat{f}(\omega)|d\omega \\
			&=\int_{\mathbb{R}^d}(\cos(\omega\cdot(x-x_B)+\theta_B(\omega))-\cos(\theta_B(\omega)))|\hat{f}(\omega)|d\omega \\
			&=\int_{\mathbb{R}^d}g(x,\omega)|\hat{f}(\omega)|d\omega \\
			&=\|f\|_{B^m}\int_{\mathbb{R}^d}|\omega|_B^{-m}g(x,\omega)\lambda^m(\omega)d\omega
		\end{aligned}
	\end{equation}
where
$$
\theta_B(\omega)=\omega\cdot x_B+\theta(\omega),\quad|\omega|_B:=\sup\limits_{x\in B}|\omega\cdot(x-x_B)|
$$
and $g: B\times \mathbb{R}^d\rightarrow \mathbb{R}$
is given by
\begin{equation}\label{gz}
	g(x,\omega):=
	\cos(\omega\cdot (x-x_B)+\theta_B(\omega))  -\cos(\theta_B(\omega)),
\end{equation}
and
$$
\|f\|_{B^m}:=\int_{\mathbb R^d}|\omega|_B^m|\hat{f}(\omega)|d\omega,\quad\lambda^m(\omega)=\frac{|\omega|_B^m|\hat{f}(\omega)|}{\|f\|_{B^m}}.
$$
here $\lambda^m$ is a  probability distribution density function.

	Using $\lambda=\lambda^0$, we define the expectation $\mathbb E$ and
	$\mathbb{\bar E}$ for $u: \mathbb R^d\mapsto \mathbb R$ and
	$v: (\mathbb R^d)^n\equiv\mathbb R^d\times\cdots\times\mathbb R^d\mapsto \mathbb R$
	$$
	\mathbb E(u)\equiv\int_{\mathbb R^d}u(\omega)\lambda(\omega)d\omega,  \quad
	\mathbb{\bar E}(v)\equiv\int_{(\mathbb
		R^d)^n}v(\omega_1,\cdots,\omega_n)
	\lambda(\omega_1)\cdots\lambda(\omega_n)d\omega_1\cdots d\omega_n .
	$$
	Denote$\tilde f(x)=\frac{f(x)-f(x_B)}{\|f\|_{B}}$, by
	Lemma~\ref{MC} and a direct (and standard) calculation
	$$
	\mathbb{\bar E}(\tilde f(x)-\frac1n\sum_{j=1}^ng(x,\omega_j))^2
	=\mathbb{\bar E}(\mathbb{E}[g(x,\cdot)]-
	\frac1n\sum_{j=1}^n g(x,\omega_j))^2
	\le\frac{1}{n}\max_{x\in B, \omega\in \mathbb R^d}|g(x,\omega)|^2
	\le \frac{4}{n}.
	$$
	Integrating both sides on B, then by Fubini's Theorem we should have
	$$
	\mathbb{\bar E}\int_B(\tilde f(x)-\frac1n\sum_{j=1}^ng(x,\omega_j))^2 d\mu(x)
	\le \frac{4}{n}.
	$$
	Thus $\exists$ $ \omega_j^*\in \mathbb{R}^d$ such that
	$$
	\int_B  (\tilde f(x)-\frac1n\sum_{j=1}^ng(x,\omega_j^*))^2d\mu(x)\le \frac{4}{n}.
	$$
	The desired result then follows easily.
\end{proof}
To prove an $H^1$ estimate, we use the following density function
$$
\lambda_1(\omega)=\lambda^1=\frac{|\omega|_B|\hat f(\omega)|}{\int_{\mathbb R^d}|\omega|_B|\hat f(\omega)|}.
$$
Similarly, we have
$$
\tilde f(x)\equiv \frac{f(x)-f(x_B)}{\int_{\mathbb R^d}|\omega|_B|\hat f(\omega)|}
=
\int_{\mathbb{R}^d}\frac{g(x,\omega)}{|\omega|_B}\lambda_1(\omega)d\omega.
$$
It follows that
$$
\partial_k\tilde f(x)
=
\int_{\mathbb{R}^d}\frac{\partial_kg(x,\omega)}{|\omega|_B}\lambda_1(\omega)d\omega.
$$
We note that
$$
\max_{x\in B, \omega\in \mathbb R^d}
\frac{|g(x,\omega)|}{|\omega|_B}\le 1.
$$
Also, by the definition of $|\omega|_B$, we know there exists $x_\omega\in B$ such that:
$$
|\omega|_B=\sup\limits_{x\in B}|\omega\cdot(x-x_B)|\ge|\omega||x_\omega-x_B|\ge|\omega|\frac12\mathrm{dist}(x_B,\partial B).
$$
Thus
$$
\max_{x\in B, \omega\in \mathbb R^d}
\frac{|\partial_kg(x,\omega)|}{|\omega|_B}\le \frac{2}{\mathrm{dist}(x_B,\partial B)}.
$$
Setting
$$
v_n=\frac1n\sum_{j=1}^n\frac{g(x,\omega_j)}{|\omega_j|_B}.
$$
By a similar argument, we obtain
$$
\mathbb{\bar E}
\int_B\left((\tilde f(x)-v_n(x))^2
+\sum_{k=1}^d (\partial_k\tilde f(x)-\partial_kv_n(x))^2
\right) d\mu(x)
\le \frac{1}{n}[C(d,B)]^2 .
$$
where
$
C(d,B)=[\frac{4d}{{\rm{dist}^2(x_B,\partial B)}}+1]^{1/2}.
$
This implies that $\exists$ $ \omega_j^*\in \mathbb{R}^d$ such that
$$
\left\|\tilde f(\cdot)-{1\over n}\sum_{j=1}^n\frac{g(\cdot,\omega_j^*)}{|\omega_j^*|_B}\right\|_{1,B}^2
\le \frac{[C(d,B)]^2}{n}.
$$
\begin{theorem}  \label{thm:DLH1}
	Given a bounded domain $B\subset\mathbb R^d$, a probability measure $\mu$ on B
	and a function $f:\mathbb R^d\mapsto\mathbb R$ whose Fourier
	transform $\hat f$ satisfying $\|f\|_{B^1}<\infty$.
	%$\|\widehat{\nabla f}\|_{L^1(\mathbb R^d)}<\infty$.
	Then, for any $n\ge 1$,  there exists $f_n\in V_n$  such that
	\begin{equation}
		\|f-f_n\|_{H^1(B)}\le \frac{C(d,B)}{\sqrt{n}})\|f\|_{B^1}.
	\end{equation}
	where
	\begin{equation}
		\label{un}
		f_n(x)=\sum_{j=1}^na_j\cos(\omega_j\cdot x+b_j)+c \mbox{ for some }
		a_j,b_j,c\in \mathbb R, \omega_j\in\mathbb R^d.
	\end{equation}
\end{theorem}

One remarkable fact is that Theorem \ref{thm:DLH1} holds in any spatial
dimension and any bounded domain $B$ of any geometric shape.
Theorem \ref{thm:DLH1} is only interesting when $d\ge 2$.  For $d=1$,
we can prove much stronger results easily.  For example, if $u$
is sufficiently smooth, for any $m\ge 1$, we can find functions $u_n$
in the form of \eqref{un} such that
\begin{equation}
	\label{uun1}
	\|u-u_n\|_{1,B}\le \frac{C_1(m,u)}{n^{m}}.
\end{equation}
\newpage

\section{An improved analysis}
\subsection{Heaviside Function}
Define $g_i: [-1,1]\mapsto \mathbb R$ as
follows:
\begin{equation}
	\label{psi}
	g_i(t)=\frac{1}{|\omega_i|_B}[\cos(|\omega_i|_Bt+\theta_B(\omega_i))  -\cos(\theta_B(\omega_i))],
\end{equation}
In view of \eqref{gz}, we have
\begin{equation}
	\label{gpsi}
	g_i(s_i)=\frac{g(x,\omega_i)}{|\omega_i|_B}, \quad s_i=\omega_i^B\cdot(x-x_B),\quad \omega^B=\frac{\omega}{|\omega|_B}
\end{equation}
Now, we take an integer
\begin{equation}
	\label{k}
	k\ge \sqrt{n}
\end{equation}
and consider a partition of $[-1,1]$ with the following grid points
$$
t_j=jh_k, j=-k:k
$$
with
$$
h_k=\frac{1}{k}\le \frac{1}{\sqrt{n}}.
$$
We first take a piecewise constant interpolation for $g_i$ on $[0,1]$ to get
$$
g_{i,k}(t)=(\Pi_kg_i)(t)=\sum_{j=0}^{k-1}g_i(t_j) M_j(t),
$$
where
$$
M_j(t)=M_0(\frac{t-t_j}{h_k})
$$
and
\begin{equation}
	\label{cardinal}
	M_0(x)=
	\left\{
	\begin{array}{ll}
		0 & x\le0 \\
		1 & 0< x\le1    \\
		0 & x > 1
	\end{array}
	\right.
\end{equation}
We note that
$$
M_0(x)=H(x)-H(x-1)
$$
where $H$ is the Heaviside function
Thus
$$
M_0(\frac{t-t_{j}}{h_k})
=H(\frac{t-t_{j}}{h_k})-H(\frac{t-t_{j}}{h_k}-1)=H(\frac{t-t_{j}}{h_k})-H(\frac{t-t_{j+1}}{h_k})
\equiv H_{j}(t)-H_{j+1}(t).
$$
Thus, since $g_i(t_0)=0, H_k=0$, we have
\begin{equation}  \label{gi0}
	g_{i,k}(t)=\sum_{j=0}^{k-1}g_i(t_j) M_j(t)
	=\sum_{j=1}^{k-1}(g_i(t_j) - g_i(t_{j-1})) H_{j}(t), \quad t\in [0,1]
\end{equation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%
Now we consider
\begin{equation}
	h_i(t) = g_i(-t), \quad t\in [0,1].
\end{equation}
Similar to \eqref{gi0}, we have
$$
(\Pi_kh_i)(t)=\sum_{j=1}^{k-1}(h_i(t_j) - h_i(t_{j-1}))H_j(t)=
=\sum_{j=1}^{k-1}(g_i(-t_j) - g_i(-t_{j-1}))H_j(t)
$$
Namely
$$
(\Pi_k g_i)(-t)=\sum_{j=1}^{k-1}(g_i(-t_j) - g_i(-t_{j-1}))H_j(t), \quad t\in [0,1]
$$
or
\begin{equation}\label{gi1}
	(\Pi_k g_i)(t)=\sum_{j=1}^{k-1}(g_i(-t_j) - g_i(-t_{j-1}))H_j(-t), \quad t\in [-1,0]
\end{equation}
By combining \eqref{gi0} and \eqref{gi1}, we get a piecewise constant
interpolation of $g_i$ on $[-1,1]$ as follows:
\begin{eqnarray}
	g_{i,k}(t)&=&
	\sum_{j=1}^{k-1}(g_i(-t_j) - g_i(-t_{j-1}))H_j(-t)+\sum_{j=1}^{k-1}(g_i(t_j) - g_i(t_{j-1})) H_{j}(t)\nonumber \\
	&=&\sum_{j=1}^{k-1}[a_{ij}^-H_j(-t)+a_{ij}^+H_{j}(t)] \label{gih}
	\quad t\in [-1,1]
\end{eqnarray}
where
$$
a_{ij}^{\pm}=g_i(\pm t_j) - g_i(\pm t_{j-1})
$$
It is easy to see that
\begin{equation}
	|g_i(t)-g_{i,k}(t)|\le h_k, \quad t\in [-1,1].
\end{equation}

\begin{equation}
	\|\frac1n \sum_{i=1}^n \frac{g(\cdot,\omega_i)}{|\omega_i|_B}-f^*\|_{L^2(\mu,B)}\le h_k
\end{equation}
where
\begin{equation}
	\label{fstar}
	f^*(x)=
	\frac1n\sum_{i=1}^ng_{i,k}(\omega_i^B\cdot (x-x_B)).
\end{equation}
By the approximation in last section, we have
\begin{equation}
	\|\tilde f-f^*\|_{L^2(\mu,B)}\le \frac{2}{\sqrt{n}}
\end{equation}
Let us rewrite
$$
f^*(x)
=\sum_{i=1}^n\sum_{j=1}^{k-1}[\gamma_{ij}^- f_{ij}^-+ \gamma_{ij}^+f_{ij}^+]
$$
where
$$
\gamma_{ij}^{\pm}=\frac{|a^\pm_{ij}|}{nd_i},
f^\pm_{i,j}=d_i\/{\rm sign}(a^\pm_{ij})H_j(\pm \omega_i^B\cdot (x-x_B))
$$
and
$$
d_i=\sum_{j=1}^{k-1}(|a^-_{ij}|+|a^+_{ij}|)\le 2
$$
By definition
\begin{equation}
	\label{gammaij}
	\sum_{i=1}^n\sum_{j=1}^{k-1}[\gamma_{i,j}^-+\gamma_{i,j}^+]=1.
\end{equation}
With re-numeration as
$$
p_\ell=\gamma_{ij}^{\pm}, f_\ell = f_{ij}^{\pm}, 1\le \ell \le N=2n(k-1)
$$
We have
$$
f^*(x)=\sum_{\ell=1}^N p_\ell f_\ell
$$
Consider
$$
\mathcal N=\{1,2,\ldots, N\}
$$
and
$$
\bar f: \mathcal N\mapsto \mathbb R^1
$$
such that
$$
\bar  f(\ell)=f_{\ell}, \ell\in \mathcal N
$$
With the probability measure
$$
\mu(\mathcal M)=\sum_{m\in \mathcal M}p_m \quad \mathcal M\subset\mathcal N.
$$
By definition.
$$
\mathbb E(\bar  f) = f^*(x).
$$
By the basic result on expectation in Lemma \ref{MC}, we have
$$
\sum_{\ell_1,\ldots \ell_n=1}^Np_{\ell_1}\cdots p_{\ell_n}\left(f^*(x)-
{1\over n}\sum_{i=1}^n f_{\ell_i}\right)^2
=\mathbb E_{\mathcal N^n} \left(\mathbb E(\bar f)-{1\over
	n}\sum_{i=1}^n  \bar  f(\ell_i)\right)^2\\
\le\frac1n \|\bar  f\|_{\infty}^2 \le\frac4n.
$$
By taking the $L^2(\mu,B)$ on the above inequality, we get
$$
\sum_{\ell_1,\ldots \ell_n=1}^Np_{\ell_1}\cdots p_{\ell_n}\|f^*-{1\over n}\sum_{i=1}^n f_{\ell_i}\|_{L^2(\mu,B)}^2
\le\frac4n.
$$
Thus, there exisit $\ell_1^*, \ldots, \ell_n^*\in \mathcal N$ such that
$$
\|f^*-{1\over n}\sum_{i=1}^n f_{\ell_i^*}\|_{L^2(\mu,B)}^2
\le\frac4n.
$$
where
$$
f_n(x)={1\over n}\sum_{i=1}^n f_{\ell_i^*}(x).
$$
Then we have
\begin{equation}
	\|\tilde f-f_n\|^2_{L^2(\mu,B)}\le \frac{9}{n}.
\end{equation}
Consequently
\begin{equation}
	\left\|f(x)-f(x_B)-\|f\|_Bf_n\right\|^2_{L^2(\mu,B)}\le \frac{9\|f\|^2_B}{n}.
\end{equation}

\subsection{Heaviside to piecewise constant projection}
We first take a piecewise constant interpolation for $t$ on $[0,1]$ to get
$$
\Pi_k^0 g (t)=\sum_{j=0}^{k-1}g(t_j) M_j(t),
$$
where
$
	M_j(t)=M_0(\frac{t-t_j}{h_k}),
	\quad
	M_0(x)=
	\left\{
	\begin{array}{ll}
		0 & x\le0 \\
		1 & 0< x\le1    \\
		0 & x > 1
	\end{array}.
	\right.
$
We note that
$$
M_0(x)=H(x)-H(x-1)
$$
where $H$ is the Heaviside function.
Thus
$$
M_j(t)
=H(\frac{t-t_{j}}{h_k})-H(\frac{t-t_{j}}{h_k}-1)=H(\frac{t-t_{j}}{h_k})-H(\frac{t-t_{j+1}}{h_k})
\equiv H_{j}(t)-H_{j+1}(t).
$$
Thus, since $H_k=0$, we have
\begin{equation}  \label{gi0}
\Pi_k^0 g(t)=\sum_{j=0}^{k-1}g(t_j) (H_{j}(t)-H_{j+1}(t))
	=g(0) + \sum_{j=1}^{k-1}(g(t_j) - g(t_{j-1})) H_{j}(t).
\end{equation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%
Now we consider
$ h(t) = g(-t)$.
Similar to \eqref{gi0}, we have
$$
(\Pi_k^0 h)(t)=\sum_{j=1}^{k-1}(h(t_j) - h(t_{j-1}))H_j(t)
=\sum_{j=1}^{k-1}(g(-t_j) - g(-t_{j-1}))H_j(t),
$$
namely
\begin{equation}\label{gi1}
	\Pi_k^0 g(t)=\sum_{j=1}^{k-1}(g(-t_j) - g(-t_{j-1}))H_j(-t), \quad t\in [-1,0]
\end{equation}
Thus, the piecewise constant
interpolation of $g$ on $[-1,1]$ as follows:
\begin{eqnarray}
	\Pi_k^0 g(t)&=&
	\sum_{j=1}^{k-1}(g_i(-t_j) - g_i(-t_{j-1}))H_j(-t)+\sum_{j=1}^{k-1}(g_i(t_j) - g_i(t_{j-1})) H_{j}(t)\nonumber \\
	&=&\sum_{j=1}^{k-1}[a_{ij}^-H_j(-t)+a_{ij}^+H_{j}(t)]  \label{gih0}
	\quad t\in [-1,1]
\end{eqnarray}
where
$
a_{ij}^{\pm}=g_i(\pm t_j) - g_i(\pm t_{j-1}).
$

\subsection{Heaviside Function 2}
Recall $|\omega|_B=\sup_{x} |\omega \cdot (x-x_B)|$ and
$$
g(x,\omega) = \cos (\omega\cdot (x-x_B) +\theta_B(\omega)) - \cos(\theta_B(\omega)).
$$
Let $t={\omega\cdot (x-x_B)\over |\omega|_B}\in [-1, 1]$ and
$$
g(t, \omega)= \cos(|\omega|_Bt+\theta_B(\omega))  -\cos(\theta_B(\omega)).
$$
Define $g_i: [-1,1]\mapsto \mathbb R$ as
follows:
\begin{equation}
\label{psi}
g_i(t)=\frac{1}{|\omega_i|_B}g(t,\omega_i)=\frac{1}{|\omega_i|_B}\big (\cos(|\omega_i|_Bt+\theta_B(\omega_i))  -\cos(\theta_B(\omega_i))\big ).
\end{equation}
Note that
$$
g(x,\omega) = |\omega|_B g_i\big ({\omega\cdot (x-x_B)\over |\omega|_B}\big ).
$$
Thus,
$$
g_i(t)=\frac{g(x,\omega_i)}{|\omega_i|_B}.
$$

\noindent{\bf Step 1: Approximate $f$ by cosine functions}

Since
$$
f(x)=\int_{\mathbb R} g(x,\omega)|\hat f(\omega)|d\omega = \int_{\mathbb R} {g(x,\omega)\over |\omega|_B} |\omega|_B|\hat f(\omega)|d\omega
$$
Let $\lambda(\omega) = {|\omega|_B|\hat f(\omega)|\over \int_{\mathbb R} |\omega|_B|\hat f(\omega)|d\omega}$. Thus,
$$
f(x)=\int_{\mathbb R} {g(x,\omega)\over |\omega|_B}\lambda(\omega)d\omega
$$
By \eqref{MC}, there exist $\{\omega_i^\ast\}_{i=1}^n$ such that
$$
	\|\frac1n \sum_{i=1}^n \frac{g(x,\omega_i^\ast)}{|\omega_i^\ast|_B}-f(x)\|_{L^2(\mu,B)}\le \frac{1}{\sqrt{n}},
$$
namely,
\begin{equation}\label{eqmc1}
	\|\frac1n \sum_{i=1}^n g_i^\ast (t)-f(x)\|_{L^2(\mu,B)}\le \frac{1}{\sqrt{n}}
\end{equation}
with
$$\displaystyle g_i^\ast (t)=\frac{1}{|\omega_i^\ast |_B}g({\omega\cdot (x-x_B)\over |\omega|_B},\omega_i^\ast).
$$

\noindent{\bf Step 2: Approximate cosine function by Heaviside functions}

Consider a partition of $t\in [-1,1]$ with the following grid points
$$
t_j=jh_k, j=-k:k
$$
with
$
h_k=\frac{1}{k}\le \frac{1}{\sqrt{n}}.
$
Denote the piecewise constant interpolation of $g_i$ on $[-1,1]$ by $g_{i,k}$ with
\begin{equation}\label{pro:intererror}
	|g_i(t)-g_{i,k}(t)|\le h_k, \quad t\in [-1,1].
\end{equation}
By \eqref{gih0},
\begin{eqnarray}
	g_{i,k}(t)&=&
	\sum_{j=1}^{k-1}(g_i(-t_j) - g_i(-t_{j-1}))H_j(-t)+\sum_{j=1}^{k-1}(g_i(t_j) - g_i(t_{j-1})) H_{j}(t)\nonumber \\
	&=&\sum_{j=1}^{k-1}[a_{ij}^-H_j(-t)+a_{ij}^+H_{j}(t)] \label{gih}
	\quad t\in [-1,1]
\end{eqnarray}
where
$
a_{ij}^{\pm}=g_i(\pm t_j) - g_i(\pm t_{j-1}).
$
By \eqref{eqmc1},
\begin{equation} \label{eq:gast}
\begin{split}
	\|\frac1n  \sum_{i=1}^n g_{i,k}^\ast(t)-f(x)\|_{L^2(\mu,B)}\le &
	\|\frac1n  \sum_{i=1}^n (g_{i,k}^\ast(t)-g_i^\ast(t))\|_{L^2(\mu,B)} \\
	&+ \|\frac1n  \sum_{i=1}^n g_{i}^\ast(t)-f(x)\|_{L^2(\mu,B)}\le
	\frac{2}{\sqrt{n}}.
\end{split}
\end{equation}

\noindent{\bf Step 3: Approximate $f(x)$ by Heaviside functions}

Let
$$
f^*(x) = \frac1n  \sum_{i=1}^n g_{i,k}^\ast(t)
=\sum_{i=1}^n\sum_{j=1}^{k-1}[\gamma_{ij}^- f_{ij}^-+ \gamma_{ij}^+f_{ij}^+].
$$
where
$
d_i=\sum_{j=1}^{k-1}(|a^-_{ij}|+|a^+_{ij}|)\le 2,
$
$
\gamma_{ij}^{\pm}=\frac{|a^\pm_{ij}|}{nd_i}
$ and
$
f^\pm_{i,j}=d_i\/{\rm sign}(a^\pm_{ij})H_j(\pm {\omega_i^\ast\cdot (x-x_B)\over |\omega_i^\ast|_B}).
$
Since
$
\displaystyle \sum_{i=1}^n\sum_{j=1}^{k-1}[\gamma_{i,j}^-+\gamma_{i,j}^+]=1,
$
with re-numeration as
$$
p_\ell=\gamma_{ij}^{\pm}, f_\ell = f_{ij}^{\pm}, 1\le \ell \le N=2n(k-1),
$$
we have
$$
f^*(x)=\sum_{\ell=1}^N p_\ell f_\ell.
$$
This implies that $f^\ast (x)$ is the expectation of $\bar f$, which is randomly chosen from $\{f_\ell\}_{\ell=1}^N$ with probability $p_\ell$. ($x$ is fixed.)
$$
f^\ast(x) = \mathbb{E} (\bar f).
$$
By \eqref{MC}, there exist $\{\ell_i\}_{i=1}^n$ such that
$$
\|f^\ast (x) - {1\over n}\sum_{i=1} f_{\ell_i}\|_{L^2(\mu,B)}\le {1\over \sqrt{n}}.
$$
A combination of \eqref{eq:gast} and the above inequality gives
$$
\|f - {1\over n}\sum_{i=1} f_{\ell_i}|\le |f(x) - f^\ast (x)\|_{L^2(\mu,B)} + \|f^\ast (x) - {1\over n}\sum_{i=1} f_{\ell_i}\|_{L^2(\mu,B)}\le{3\over \sqrt{n}}.
$$

\iffalse
{\bf Outline of proof}
\begin{enumerate}
\item $f(x)=\int g(x,\omega)\lambda(\omega)d\omega$
\item $\displaystyle {1\over n}\sum_{i=1}^n g(x,\omega_i)\rightarrow f(x)$ with accuracy $n^{-1/2}$, with
\[
	g(x,\omega_i)= \cos(\omega_i\cdot (x-x_B)+\theta_B(\omega_i))  -\cos(\theta_B(\omega_i)),
\]
\item $$
\displaystyle f^\ast(x)={1\over n}\sum_{i=1}^n \sum_{j=1}^{k-1}[a_{ij}^-H_j(-{x-x_B\over h})+a_{ij}^+H_{j}({x-x_B\over h})] \rightarrow {1\over n}\sum_{i=1}^n g(x,\omega_i)
$$
with accuracy $k^{-1}$. Let $k=n^{1/2}$. The subscript $i$ relates to sample $\omega_i$ and $j$ relates to the partition of $x$.
\item For any function $f$, there exist $\{\omega_i^\ast\}_{i=1}^n$ which determine the value of $a_{ij}^\pm$ such that
$$
|f(x) - f^\ast(x)|\le n^{-1/2}.
$$
\item Rewrite $f^\ast$ as
$$
f^\ast (x)=\sum_{l=1}^N p_lf_l \quad \mbox{ with } N=2n(k-1)
$$
with
$
\displaystyle p_l={|a_l|\over nd_l},
$
$
\displaystyle f_l=d_l\/{\rm sign}(a_l)H_l(\omega_l^B\cdot (x-x_B)).
$

Let $\bar{f}$ be a random variable, $f_l$ with probability $p_l$. Then,
$$
f^\ast (x) = \mathbb{E}(\bar f).
$$
\end{enumerate}
\fi
\subsection{Piecewise linear function and ReLU}
The proof here is almost the same as the proof for Heaviside function in the last part.

Now we take a piecewise linear interpolation for $g_i$ on $[0,1]$, since $g_i(t_0)=0$ , we get
$$
g_{i,k}(t)=(\Pi_k^1g_i)(t)=\sum_{j=1}^{k}[g_i(t_{j})-g_i(t_{j-1})]\sigma_{j-1}(t),   \quad t\in [0,1]
$$
where
$$
\sigma_j(t)=M_0(\frac{t-t_j}{h_k})
$$
and
\begin{equation}
	\label{cardinal}
	M_0(x)=
	\left\{
	\begin{array}{ll}
		0 & x\le0 \\
		x & 0< x\le1    \\
		1 & x > 1
	\end{array}
	\right.
\end{equation}
%We note that
%$$
%M_0(x)=ReLU(x)-ReLU(x-1)
%$$

%%%%%%%%%%%%%%%%%%%%%%%%%%%
Consider
\begin{equation}
	h_i(t) = g_i(-t), \quad t\in [0,1].
\end{equation}
Similarly, we have
$$
(\Pi_k^1h_i)(t)=\sum_{j=1}^{k}(h_i(t_j) - h_i(t_{j-1}))\sigma_{j-1}(t)=
=\sum_{j=1}^{k}(g_i(-t_j) - g_i(-t_{j-1}))\sigma_{j-1}(t)
$$
Namely
$$
(\Pi_k^1 g_i)(-t)=\sum_{j=1}^{k}(g_i(-t_j) - g_i(-t_{j-1}))\sigma_{j-1}(t), \quad t\in [0,1]
$$
or
\begin{equation}\label{gi1}
	(\Pi_k^1 g_i)(t)=\sum_{j=1}^{k}(g_i(-t_j) - g_i(-t_{j-1}))\sigma_{j-1}(-t), \quad t\in [-1,0]
\end{equation}
Combine together we get a piecewise linear
interpolation of $g_i$ on $[-1,1]$ as follows:
\begin{eqnarray}
	g_{i,k}(t)&=&
	\sum_{j=1}^{k}(g_i(-t_j) - g_i(-t_{j-1}))\sigma_{j-1}(-t)+\sum_{j=1}^{k}(g_i(t_j) - g_i(t_{j-1}))\sigma_{j-1}(t)\nonumber \\
	&=&\sum_{j=1}^{k}[a_{ij}^-\sigma_{j-1}(-t)+a_{ij}^+\sigma_{j-1}(t)] \label{gih}
	\quad t\in [-1,1]
\end{eqnarray}
where
$$
a_{ij}^{\pm}=g_i(\pm t_j) - g_i(\pm t_{j-1})
$$
%It is easy to see that
%\begin{equation}
%|g_i(t)-g_{i,k}(t)|\le h_k, \quad t\in [-1,1].
%\end{equation}
%\begin{equation}
%\|\frac1n \sum_{i=1}^n \frac{g(\cdot,\omega_i)}{|\omega_i|_B}-f^*\|_{L^2(\mu,B)}\le h_k
%\end{equation}
%where
%\begin{equation}
%\label{fstar}
%f^*(x)=
%\frac1n\sum_{i=1}^ng_{i,k}(\omega_i^B\cdot (x-x_B)).
%\end{equation}
%
%By Theorem \ref{jones}, we have
%\begin{equation}
%%\|\tilde f-f^*\|_{1,B}\le \frac{C_1(d,B)}{\sqrt{n}}
%  \|\tilde f-f^*\|_{L^2(\mu,B)}\le \frac{2}{\sqrt{n}}
%\end{equation}
%here $C_1(d,B)=\sqrt{\mu(B)}[\sqrt{(d+{\rm diam}(B))}+{\rm diam}(B)(1+d)]$.

%Let us rewrite
%$$
%f^*(x)
%=\sum_{i=1}^n\sum_{j=1}^{k}[\gamma_{ij}^- f_{ij}^-+ \gamma_{ij}^+f_{ij}^+]
%$$
%where
%$$
%\gamma_{ij}^{\pm}=\frac{|a^\pm_{ij}|}{nd_i},
%f^\pm_{i,j}=d_i\/{\rm sign}(a^\pm_{ij})\sigma_{j-1}(\pm \omega_i^B\cdot (x-x_B))
%$$
%and
%$$
%d_i=\sum_{j=1}^{k}(|a^-_{ij}|+|a^+_{ij}|)\le 2
%$$
%By definition
%\begin{equation}
%\label{gammaij}
%\sum_{i=1}^n\sum_{j=1}^{k}[\gamma_{i,j}^-+\gamma_{i,j}^+]=1.
%\end{equation}
%With re-numeration as
%$$
%p_\ell=\gamma_{ij}^{\pm}, f_\ell = f_{ij}^{\pm}, 1\le \ell \le N=2nk
%$$
%We have
%$$
%f^*(x)=\sum_{\ell=1}^N p_\ell f_\ell
%$$
%Consider
%$$
%\mathcal N=\{1,2,\ldots, N\}
%$$
%and
%$$
%\bar f: \mathcal N\mapsto \mathbb R^1
%$$
%such that
%$$
%\bar f(\ell)=f_{\ell}, \ell\in \mathcal N
%$$
%With the probability measure
%$$
%\lambda(\mathcal M)=\sum_{m\in \mathcal M}p_m \quad \mathcal M\subset\mathcal N.
%$$
%By definition.
%$$
%\mathbb E(\bar f) = f^*(x).
%$$
%
%By the basic result on expectation:
%\begin{equation}
%\begin{aligned}
%&\sum_{\ell_1,\ldots \ell_n=1}^Np_{\ell_1}\cdots p_{\ell_n}\left((f^*(x)-
%{1\over n}\sum_{i=1}^n f_{\ell_i})^2\right)\\
%=&\mathbb E_{\mathcal N^n} \left(\mathbb E(\bar f)-{1\over
%	n}\sum_{i=1}^n  \bar f(\ell_i)\right)^2
%\le\frac1n \|\bar f\|_{\infty}^2 \le\frac{4}{n}.
%\end{aligned}
%\end{equation}
%
%By taking the $L^2(\mu,B)$ on the above inequality, we get
%$$
%\sum_{\ell_1,\ldots \ell_n=1}^Np_{\ell_1}\cdots p_{\ell_n}\|f^*-{1\over n}\sum_{i=1}^n f_{\ell_i}\|_{L^2(\mu,B)}^2
%\le\frac4n.
%$$
%Thus, there exisit $\ell_1^*, \ldots, \ell_n^*\in \mathcal N$ such that
%$$
%\|f^*-f_n(x)\|_{L^2(\mu,B)}^2
%\le\frac4n.
%$$
%where
%$$
%f_n(x)={1\over n}\sum_{i=1}^n f_{\ell_i^*}(x).
%$$
%Then we have
%\begin{equation}
%\|\tilde f-f_n\|^2_{L^2(\mu,B)}\le \frac{9}{n}.
%\end{equation}

%Consequently
%\begin{equation}
%\left\|f(x)-f(x_B)-\|f\|_Bf_n\right\|^2_{L^2(\mu,B)}\le \frac{9\|f\|^2_B}{n}.
%\end{equation}

Follow the procedure in last section, and notice that $\sigma(x)=ReLU(x)-ReLU(x-1)$, we obtain the following theorem.

\begin{theorem}
	For a probability measure $\mu$ on B and every function with $\|f\|_B<\infty$, there exists $\omega_1,\dots,\omega_n\in\mathbb{R}^d$ such that
	$$
	\left\|f(x)-f_n(x)\right\|^2_{L^2(\mu,B)}\le \frac{C\|f\|^2_B}{n}.
	$$
	where $f_n(x)=\sum_{i=1}^{n}a_iReLU(\omega_i x+b_i)+c$.
\end{theorem}

%\section{Barron theory of approximation}
%%\subsection{A DL function class}
%\subsection{Approximation result for cosine as activation function}
%For clarity, let us give a brief introduction of the approximation
%result based results in Barron \cite{barron1993universal} and some
%modification in Xu \cite{xu2017approximation}.
%
%Given $d$ and $n$, consider the following nonlinear space in $\mathbb
%R^d$:
%\begin{equation}
%\label{Vn}
%V_n=\left\{v: v(x)={a\over n}\sum_{j=1}^n\cos(\omega_j\cdot x+b_j)+c,
%  \mbox{ with }
%a, b_j, c\in\mathbb R, \omega_j\in\mathbb R^d
%\right\}
%\end{equation}
%\begin{theorem}  Given a bounded domain $\Omega\subset\mathbb R^d$
% and a function $u:\mathbb R^d\mapsto\mathbb R$ whose Fourier
% transform $\hat u$ satisfying $\|\hat u\|_{L^1(\mathbb R^d)}<\infty$.
%Then, for any $n\ge 1$,  there exists $u_n\in V_n$  such that
%\begin{equation}
%\|u-u_n\|_{L^2(\Omega)}\le \frac{2|\Omega|^{1\over 2}\|\hat u\|_{L^1(\mathbb R^d)}}{\sqrt{n}}.
%\end{equation}
%where $u_n(x)={a\over n}\sum_{j=1}^n\cos(\omega_j\cdot x+b_j)+c$ satisfying
%\begin{equation}
%\label{abc}
%a=  \|\hat u\|_{L^1(\mathbb R^d)}, \quad |b_j|\le \pi, \quad |c|\le \|u\|_{0,\infty,\Omega}+\|\hat u\|_{L^1(\mathbb R^d)}
%\end{equation}
%\end{theorem}
%\begin{proof}\small
%We write
%$\hat{u}(\omega)=e^{i\theta(\omega)}|\hat{u}(\omega)|$. By Fourier inversion formula,
%\begin{equation}
%  \label{eqn1}
%  u(x)=\int_{\mathbb{R}^d}e^{i\omega\cdot x}\hat{u}(\omega)d\omega
%=\int_{\mathbb{R}^d}e^{i(\omega\cdot x+\theta(\omega))}|\hat{u}(\omega)|
%\end{equation}
%Since $u(x)$ is real-valued,  the above identity implies that,  for $x, x^*\in \Omega$
%\begin{equation}    \label{eqn4.3}
%u(x)-u(x^*)
%={\rm Re}\int_{\mathbb{R}^d}
%(e^{i\omega\cdot x}-e^{i\omega\cdot x^*})
% e^{i\theta
%    (\omega)}|\hat{u}(\omega)|d\omega
%=\|\hat u\|_{L^1(\mathbb R^d)}\int_{\mathbb{R}^d}g(x,\omega)\lambda(\omega)d\omega
%  \end{equation}
%where $g: \Omega\times \mathbb{R}^d\rightarrow \mathbb{R}$
%and $\lambda:\mathbb R^d\mapsto\mathbb R$ is a  probability distribution density function:
%\begin{equation}\label{gz}
%  g(x,\omega):=
%\cos(\omega\cdot x+\theta(\omega))  -
%\cos(\omega\cdot x^*+\theta(\omega)) , \quad
%\lambda(\omega)=|\hat{u}(\omega)|/\|\hat u\|_{L^1(\mathbb R^d)}.
%\end{equation}
%Using $\lambda$, we define the expectation $\mathbb E$ and
%$\mathbb{\bar E}$ for $u: \mathbb R^d\mapsto \mathbb R$ and
%$v: (\mathbb R^d)^n\equiv\mathbb R^d\times\cdots\times\mathbb R^d\mapsto \mathbb R$
%$$
%\mathbb E(u)\equiv\int_{\mathbb R^d}u(\omega)\lambda(\omega)d\omega,  \quad
%\mathbb{\bar E}(v)\equiv\int_{(\mathbb
%  R^d)^n}v(\omega_1,\cdots,\omega_n)
%\lambda(\omega_1)\cdots\lambda(\omega_n)d\omega_1\cdots d\omega_n
%$$
%Denote$\tilde u(x)=[u(x)-u(x^*)]/\|\hat u\|_{L^1(\mathbb R^d)}$, by
%\eqref{eqn4.3} and a direct (and standard) calculation
%$$
%\mathbb{\bar E}(\tilde u(x)-\frac1n\sum_{j=1}^ng(x,\omega_j))^2
%=\mathbb{\bar E}(\mathbb{E}[g(x,\cdot)]-
%     \frac1n\sum_{j=1}^n g(x,\omega_j))^2
%\le\frac{1}{n}\max_{x\in \Omega, \omega\in \mathbb R^d}|g(x,\omega)|^2
%\le \frac{4}{n}
%$$
%By Fubini's Theorem,
%$\mathbb{\bar E}\int_\Omega(\tilde u(x)-\frac1n\sum_{j=1}^ng(x,\omega_j))^2 dx
%\le \frac{4}{n}|\Omega|.
%$
%Thus $\exists$ $ \omega_j^*\in \mathbb{R}^d$ such that
%$$
%\int_\Omega  (\tilde u(x)-\frac1n\sum_{j=1}^ng(x,\omega_j^*))^2dx\le \frac{4}{n}|\Omega|.
%$$
%The desired result then follows easily.
%\end{proof}
%To prove an $H^1$ estimate, we use the following density function
%$$
%\lambda_1(\omega)=\frac{|\omega||\hat u(\omega)|}{\|\widehat{\nabla u}\|_{L^1(\Omega)}}
%\quad
%\|\widehat{\nabla u}\|_{L^1(\Omega)}=\int_{\mathbb R^d}|\omega||\hat u(\omega)|.
%$$
%Similar to equation \eqref{eqn4.3}, we have
%$$
%\tilde f(x)\equiv \frac{u(x)-u(x^*)}{\|\widehat{\nabla u}\|_{L^1(\Omega)}}
%=
%\int_{\mathbb{R}^d}\frac{g(x,\omega)}{|\omega|}\lambda_1(\omega)d\omega
%$$
%It follows that
%$$
%\partial_k\tilde f(x)
%=
%\int_{\mathbb{R}^d}\frac{\partial_kg(x,\omega)}{|\omega|}\lambda_1(\omega)d\omega
%$$
%We note that
%$$
%\max_{x\in \Omega, \omega\in \mathbb R^d}
%\frac{|g(x,\omega)|}{|\omega|}\le {\rm diam}(\Omega),
%\quad
%\max_{x\in \Omega, \omega\in \mathbb R^d}
%\frac{|\partial_kg(x,\omega)|}{|\omega|}\le 1
%$$
%Setting
%$$
%v_n=\frac1n\sum_{j=1}^n\frac{g(x,\omega_j)}{|\omega_j|}
%$$
%By a similar argument, we obtain
%$$
%\mathbb{\bar E}
%\int_\Omega\left((\tilde u(x)-v_n(x))^2
%+\sum_{k=1}^d (\partial_k\tilde u(x)-\partial_kv_n(x))^2
%\right) dx
%\le \frac{1}{n}[C(d,\Omega)]^2
%$$
%where
%$
%C(d,\Omega)=[|\Omega| ({\rm diam}(\Omega)+d)]^{1/2}.
%$
%This implies that $\exists$ $ \omega_j^*\in \mathbb{R}^d$ such that
%$$
%\left\|\tilde u(\cdot)-{1\over n}\sum_{j=1}^n\frac{g(\cdot,\omega_j^*)}{|\omega_j^*|}\right\|_{1,\Omega}^2
%\le \frac{[C(d,\Omega)]^2}{n}
%$$
%\begin{theorem}  \label{thm:DLH1}
%Given a bounded domain $\Omega\subset\mathbb R^d$
% and a function $u:\mathbb R^d\mapsto\mathbb R$ whose Fourier
% transform $\hat u$ satisfying $\|\widehat{\nabla u}\|_{L^1(\mathbb R^d)}<\infty$.
%Then, for any $n\ge 1$,  there exists $u_n\in V_n$  such that
%\begin{equation}
%\|u-u_n\|_{H^1(\Omega)}\le \frac{C(d,\Omega)}{\sqrt{n}})\|\widehat{\nabla u}\|_{L^1(\mathbb R^d)}
%\end{equation}
%where
%\begin{equation}
%  \label{un}
%u_n(x)=\sum_{j=1}^na_j\cos(\omega_j\cdot x+b_j)+c \mbox{ for some }
%a_j,b_j,c\in \mathbb R, \omega_j\in\mathbb R^d.
%\end{equation}
%\end{theorem}
%
%One remarkable fact is that Theorem \ref{thm:DLH1} holds in any spatial
%dimension and any bounded domain $\Omega$ of any geometric shape.
%Theorem \ref{thm:DLH1} is only interesting when $d\ge 2$.  For $d=1$,
%we can prove much stronger results easily.  For example, if $u$
%is sufficiently smooth, for any $m\ge 1$, we can find functions $u_n$
%in the form of \eqref{un} such that
%\begin{equation}
%  \label{uun1}
%\|u-u_n\|_{1,\Omega}\le \frac{C_1(m,u)}{n^{m}}.
%\end{equation}
%\newpage

% \subsection{Ritz optimization problem}
%An optimized discretization based on machine learning amounts to solve a nonlinear problem even when the partial different equation is linear.   When adaptivity is considered, it is desirable to solve an optimization problem.  For example, for a standard linear partial differential equation such as
%\begin{equation}
%  \label{laplace}
%-\Delta u=f
%\end{equation}
%We will solve it by solving the optimization problem
%\begin{equation}
%\label{ritz}
%\min_{v\in V}J(v)
%\end{equation}
%where
%\begin{equation}
%\label{ritz}
%J(v)=\int_\Omega\left( {1\over 2}|\nabla v|^2 -fv\right)
%\end{equation}
%The following identity can be easily verified:
%\begin{equation}
%  \label{JvH1}
%|u-v|_1^2=2J(v)+|u|_1^2.
%\end{equation}
%This means that
%\begin{equation}
%\label{minmin}
%\arg\min_{v}|u-v|_1^2=
%\arg\min_{v}J(v)
%\end{equation}
%Using the approximation result, we get the following error estimate for the cosine based space:
%\begin{equation}
%  \label{cosError}
%\min_{v}|u-v|_1\le \frac{2}{\sqrt{n}}\int_{\mathbb R^d}|\omega||\hat f(\omega)|d\omega.
%\end{equation}
%where the Fourier transform can be computed with any extension of $f$ onto the entire space $\mathbb R^d$.
%
%Similarly, following error estimate for the ReLU based space:
%\begin{equation}
%  \label{ReLUError}
%\min_{v}|u-v|_1\le \frac{2}{\sqrt{n}}\int_{\mathbb R^d}|\omega|^2|\hat f(\omega)|d\omega.
%\end{equation}
%it would be interesting to compare the space of the
%approximation-class shown in \eqref{cosError} and \eqref{ReLUError}
%with those used in the classic adaptive finite element methods (see
%results by Nochetto et al).
%
%Recall the best error estimates from the classic adaptive finite
%element methods is of the following form:
%\begin{equation}
%  \label{AFEM-error}
%|u-u_n|_{1,\Omega}\le \frac{c}{n^{1\over d}}\|u\|_A
%\end{equation}
%for some norm $\|\cdot\|$ in approximation class $\mathcal A$.
%
%Comparing the error estimate in \eqref{AFEM-error} with those in
%\eqref{cosError} or \eqref{ReLUError}, the ML-FEM appears to be much
%better in terms of asymptotic order?
%
%In fact, according to the result by Lin, Xie and Xu, we have the
%following lower bound for linear elements for smooth solution
%discretized
%$$
%|u-u_n|_{1,\Omega}\ge \frac{c(u)}{n^{1\over d}}
%$$
%as long as $u$ is not a linear polynomial.
%
%
%For linear model \eqref{laplace}, the optimization problem
%\eqref{ritz} is a convex problem that has a unique solution.  But if
%we use a nonlinear space $V$ the optimization problem \eqref{ritz}
%become non-convex.  It is well-known that non-convex optimization
%problem is hard to solve.  Therefore, the ML-FEM is only viable if the
%non-convex optimization problem can be solved efficiently!


\title{Batch Training}

\subsection{Batch Training}

Running algorithms which require the full data set for each update
can be expensive when the data is large. In order to scale inferences,
we can do \emph{batch training}. This trains the model using
only a subsample of data at a time.

In this tutorial, we extend the
\href{http://edwardlib.org/tutorials/supervised-regression}
{supervised learning tutorial},
where the task is to infer hidden structure from
labeled examples $\{(x_n, y_n)\}$.
An interactive version with Jupyter notebook is available
\href{http://nbviewer.jupyter.org/github/blei-lab/edward/blob/master/notebooks/batch_training.ipynb}{here}.

\subsubsection{Data}

Simulate $N$ training examples and a fixed number of test examples.
Each example is a pair of inputs $\mathbf{x}_n\in\mathbb{R}^{10}$ and
outputs $y_n\in\mathbb{R}$. They have a linear dependence with
normally distributed noise.

\begin{lstlisting}[language=Python]
def build_toy_dataset(N, w):
  D = len(w)
  x = np.random.normal(0.0, 2.0, size=(N, D))
  y = np.dot(x, w) + np.random.normal(0.0, 0.05, size=N)
  return x, y

N = 10000  # size of training data
M = 128    # batch size during training
D = 10     # number of features

w_true = np.ones(D) * 5
X_train, y_train = build_toy_dataset(N, w_true)
X_test, y_test = build_toy_dataset(235, w_true)
\end{lstlisting}

We also define a helper function to select the next batch of data points
from the full set of examples. It keeps track of the current batch index
using an internal counter.

\begin{lstlisting}[language=Python]
def next_batch(X, y, size):
  """Grab a [size, :] tensor from X and [size] tensor from y."""
  global i
  try:  # initialize batch index
    i
  except:
    i = 0

  N = X.shape[0]
  diff = (i + 1) * size - N
  if diff <= 0:
    X_batch = X[(i * size):((i + 1) * size), :]
    y_batch = y[(i * size):((i + 1) * size)]
    i += 1
  else:
    X_batch = np.concatenate((X[(i * size):, :], X[:diff, :]))
    y_batch = np.concatenate((y[(i * size):], y[:diff]))
    i = 0

  return X_batch, y_batch
\end{lstlisting}

We will use \texttt{next_batch} during inference.

\subsubsection{Model}

Posit the model as Bayesian linear regression \citep{murphy2012machine}.
For a set of $N$ data points $(\mathbf{X},\mathbf{y})=\{(\mathbf{x}_n, y_n)\}$,
the model posits the following distributions:

\begin{align*}
  p(\mathbf{w})
  &=
  \text{Normal}(\mathbf{w} \mid \mathbf{0}, \sigma_w^2\mathbf{I}),
  \\[1.5ex]
  p(b)
  &=
  \text{Normal}(b \mid 0, \sigma_b^2),
  \\
  p(\mathbf{y} \mid \mathbf{w}, b, \mathbf{X})
  &=
  \prod_{n=1}^N
  \text{Normal}(y_n \mid \mathbf{x}_n^\top\mathbf{w} + b, \sigma_y^2).
\end{align*}

The latent variables are the linear model's weights $\mathbf{w}$ and
intercept $b$, also known as the bias.
Assume $\sigma_w^2,\sigma_b^2$ are known prior variances and $\sigma_y^2$ is a
known likelihood variance. The mean of the likelihood is given by a
linear transformation of the inputs $\mathbf{x}_n$.

Let's build the model in Edward, fixing $\sigma_w,\sigma_b,\sigma_y=1$.

\begin{lstlisting}[language=Python]
X = tf.placeholder(tf.float32, [None, D])
y_ph = tf.placeholder(tf.float32, [None])

w = Normal(loc=tf.zeros(D), scale=tf.ones(D))
b = Normal(loc=tf.zeros(1), scale=tf.ones(1))
y = Normal(loc=ed.dot(X, w) + b, scale=1.0)
\end{lstlisting}

Here, we define a placeholder \texttt{X}. During inference, we pass in
the value for this placeholder according to batches of data.
To enable training with batches of varying size,
we don't fix the number of rows for \texttt{X} and \texttt{y}. (Alternatively,
we could fix it to be the batch size if we're training and testing
with a fixed size.)

\subsubsection{Inference}

We now turn to inferring the posterior using variational inference.
Define the variational model to be a fully factorized normal across
the weights.
\begin{lstlisting}[language=Python]
qw = Normal(loc=tf.Variable(tf.random_normal([D])),
            scale=tf.nn.softplus(tf.Variable(tf.random_normal([D]))))
qb = Normal(loc=tf.Variable(tf.random_normal([1])),
            scale=tf.nn.softplus(tf.Variable(tf.random_normal([1]))))
\end{lstlisting}

Run variational inference with the Kullback-Leibler divergence.
We use $5$ latent variable samples for computing
black box stochastic gradients in the algorithm.
(For more details, see the
\href{/tutorials/klqp}{$\text{KL}(q\|p)$ tutorial}.)

For batch training, we iterate over the number of batches and
feed them to the respective placeholder. We set the number of
iterations to be the total number of batches for 5 epochs
(full passes over the data set).

\begin{lstlisting}[language=Python]
n_batch = int(N / M)
n_epoch = 5

inference = ed.KLqp({w: qw, b: qb}, data={y: y_ph})
inference.initialize(n_iter=n_batch * n_epoch, n_samples=5, scale={y: N / M})
tf.global_variables_initializer().run()

for _ in range(inference.n_iter):
  X_batch, y_batch = next_batch(X_train, y_train, M)
  info_dict = inference.update({X: X_batch, y_ph: y_batch})
  inference.print_progress(info_dict)
\end{lstlisting}

\begin{lstlisting}
390/390 [100%] ██████████████████████████████ Elapsed: 4s | Loss: 10481.556
\end{lstlisting}

When initializing inference, note we scale $y$ by $N/M$, so it is as if the
algorithm had seen $N/M$ as many data points per iteration.
Algorithmically, this will scale all computation regarding $y$ by
$N/M$ such as scaling the log-likelihood in a variational method's
objective. (Statistically, this avoids inference being dominated by the prior.)

The loop construction makes training very flexible. For example, we
can also try running many updates for each batch.

\begin{lstlisting}[language=Python]
n_batch = int(N / M)
n_epoch = 1

inference = ed.KLqp({w: qw, b: qb}, data={y: y_ph})
inference.initialize(n_iter=n_batch * n_epoch * 10, n_samples=5, scale={y: N / M})
tf.global_variables_initializer().run()

for _ in range(inference.n_iter // 10):
  X_batch, y_batch = next_batch(X_train, y_train, M)
  for _ in range(10):
    info_dict = inference.update({X: X_batch, y_ph: y_batch})

  inference.print_progress(info_dict)
\end{lstlisting}

\begin{lstlisting}
770/780 [ 98%] █████████████████████████████  ETA: 0s | Loss: 9760.541
\end{lstlisting}

In general, make sure that the total number of training iterations is
specified correctly when initializing \texttt{inference}. Otherwise an incorrect
number of training iterations can have unintended consequences; for example,
\texttt{ed.KLqp} uses an internal counter to appropriately decay its optimizer's
learning rate step size.

Note also that the reported \texttt{loss} value as we run the
algorithm corresponds to the computed objective given the current
batch and not the total data set. We can instead have it report
the loss over the total data set by summing \texttt{info_dict['loss']}
for each epoch.

\subsubsection{Criticism}

A standard evaluation for regression is to compare prediction accuracy on
held-out ``testing'' data. We do this by first forming the posterior predictive
distribution.
\begin{lstlisting}[language=Python]
y_post = ed.copy(y, {w: qw, b: qb})
# This is equivalent to
# y_post = Normal(loc=ed.dot(X, qw) + qb, scale=tf.ones(N))
\end{lstlisting}

With this we can evaluate various quantities using predictions from
the model (posterior predictive).
\begin{lstlisting}[language=Python]
print("Mean squared error on test data:")
print(ed.evaluate('mean_squared_error', data={X: X_test, y_post: y_test}))

print("Mean absolute error on test data:")
print(ed.evaluate('mean_absolute_error', data={X: X_test, y_post: y_test}))
\end{lstlisting}

\begin{lstlisting}
## Mean squared error on test data:
## 0.00659598
## Mean absolute error on test data:
## 0.0705906
\end{lstlisting}

The trained model makes predictions with low error
(relative to the magnitude of the output).

\subsubsection{Footnotes}

Only certain algorithms support batch training such as
\texttt{MAP}, \texttt{KLqp}, and \texttt{SGLD}. Also, above we
illustrated batch training for models with only global latent variables,
which are variables are shared across all data points.
For more complex strategies, see the
\href{http://edwardlib.org/api/inference-data-subsampling} {inference
data subsampling API}.

\subsubsection{References}\label{references}


\section{Regimes}
\label{s:regimes}

In this section we discuss the various limits of
the expression describing the Hanle precession curve.
First, we show that the commonly used results for zero magnetic field
and tunneling contacts are correctly reproduced.
Next, we discuss regimes where appropriate scaling will give non-unique Hanle fits.
In the following, we consider the case $r = r_0 = r_L$ of similar contacts.

In the limit of tunneling contacts, $R_C^0, R_C^L ≫ R_F$.
Putting $r_0, r_L → ∞$ gives $p_1 p_2 → \left( P_Σ^L \right)^2$ and
\begin{equation}
  f^∞ = \re{\frac{e^{- \left( L / λ \right) \sqrt{1 + i ω τ}}}{2 \sqrt{1 + i ω τ}}} ,
\end{equation}
which is of the same form as found in appendix B of
\cite{PhysRevB.37.5312}
(we will denote this limit with the superscript $∞$).
Fitting with this expression was found to give results equivalent
to fitting with the Hanle equation
\begin{equation}
  \label{eq:hanle_integral}
  \rNL^± = ± \sNL ∫_0^∞ \frac{e^{-t / τ}}{\sqrt{4 π D t}}
             \exp{\left[- \frac{L^2}{4 D t} \right]} \cos{ω t} \: dt .
\end{equation}
The agreement is expected as an explicit integration of \cref{eq:hanle_integral}
yields the same analytic expression with the identification
$\sNL = {p_1 p_2 D} / {W σ_G}$.
In the additional limit of zero magnetic field,
\begin{equation}
  Δ \rNL = \left( P_Σ^L \right)^2 R_N e^{- L / λ} ,
\end{equation}
which agrees with equation (6) in
\cite{PhysRevB.67.052409}.

Let $f_0$ denote $f$ at zero magnetic field,
\begin{equation}
  f_0 = \left[ 2 \left( 1 + λ / r \right) e^{L / λ} + \left( λ / r \right)^2 \sinh{L / λ} \right]^{-1} ,
\end{equation}
which agrees with equation (3) in
\cite{PhysRevB.80.214427}.

To further explore the nature of the Hanle curves,
we exploit the fact that it only depends on
the dimensionless ratios $λ / r$, $L / λ$, and $ω τ$.
The only other parameter of the conducting channel that enters the expression
is the overall scale $λ$ in $R_N$.
The expression $f$ contains three terms
which are of zeroth, first, and second order in $λ / r$.
Thus, as the contact resistance decreases,
one goes from a device dominated by the first term to one dominated by the last.
But precisely how this comes about depends on the value of $ω τ$.

For infinite contact resistance, it was pointed out that any rescaling
of $g$, $τ$ and $D$ that leaves $λ$ and $ω τ$ unchanged
leads to the same Hanle precession curves
\cite{Swartz2013}.
Our result shows that the same is also true
when the contact resistance is taken into account.
In numerical simulations, interesting features were observed
when $L / λ ≪ 1$ and $r / λ ≪ 1$
\cite{PhysRevB.86.235408}.

To compare across regimes, we first normalize the data to its value at zero magnetic field.
In devices where $λ / r ≫ 1$, the normalization factor is
\begin{equation}
  f_0 = \frac{2 e^{- L / λ}}{\left( λ / r \right)^2} .
\end{equation}
In this regime, if $D$ is not very different from the infinite contact resistance value,
then the lifetime can be large, i.e., $τ ≫ \SI{1}{\nano \second}$.
As one tunes the magnetic field $\sqrt{ω τ} ≫ 1$, for small values of the field,
and for much of the curve, we can approximate $1 + i ω τ ≈ i ω τ$.
An interesting consequence of this is that the zero of the Hanle precession curve
becomes independent of the scattering time.
Note that the product
\begin{equation}
   \frac{L}{λ} \sqrt{ω τ} = L \sqrt{\frac{D}{ω}} ,
\end{equation}
which appears in the exponential and oscillating factors below,
is independent of the lifetime.
As one further tunes the magnetic field, the Hanle curve is given by
\begin{equation}
  \label{eq:regime.1.f}
  f = \frac{\sqrt{ω τ}}{\left( λ / r \right)^2}
      e^{- \left( L / λ \right) \sqrt{ω τ / 2}}
      \sin{\left[ \frac{L}{λ} \sqrt{\frac{ω τ}{2}} + \frac{π}{4} \right]} ,
\end{equation}
as long as $λ / r ≫ \sqrt{ω τ} ≫ 1$.
In this limit, the nonlocal resistance scales as
\begin{equation}
  \label{eq:regime.1.nonlocal_resistance}
  Δ \rNL ∝ \frac{λ \sqrt{ω τ}}{\left( λ / r \right)^2} = r^2 \sqrt{\frac{ω}{D}} ,
\end{equation}
and the normalized nonlocal resistance as
\begin{equation}
  f / f_0 ∝ \sqrt{ω τ} .
\end{equation}

On further increasing the field,
$\sqrt{ω τ} ≫ λ / r ≫ 1$, we get
\begin{equation}
  \label{eq:regime.2.f}
  f = \frac{1}{2 \sqrt{ω τ}}
      e^{- \left( L / λ \right) \sqrt{ω τ / 2}}
      \cos{\left[ \frac{L}{λ} \sqrt{\frac{ω τ}{2}} + \frac{π}{4} \right]} .
\end{equation}
In this limit, the nonlocal resistance scales as
\begin{equation}
  \label{eq:regime.2.nonlocal_resistance}
  Δ \rNL ∝ \frac{λ}{\sqrt{ω τ}} = \sqrt{\frac{D}{ω}} ,
\end{equation}
and the normalized nonlocal resistance as
\begin{equation}
  \label{eq:regime.2.ratio}
  f / f_0 ∝ \frac{\left( λ / r \right)^2}{\sqrt{ω τ}} = D \sqrt{\frac{τ}{ω r^4}} .
\end{equation}

In the limits of \cref{eq:regime.1.f,eq:regime.2.f},
the zeros of the Hanle fit are independent of the lifetime
and are determined by $D$ though the condition
\begin{equation}
  L \sqrt{\frac{D}{2 ω}} + \frac{π}{4} = \frac{n π}{2} ,
\end{equation}
where $n = 0$ for \cref{eq:regime.1.f} and $n = 1$ for \cref{eq:regime.2.f}.

\begin{figure}[!b]
  \caption{
    Data in figure 4 (d) from \cite{PhysRevLett.105.167202}
    fit to \cref{eq:nonlocal_resistance}
    with the same values as in
    \cref{fig:nonlocal_resistance.fits} (d).
    Fits with lifetimes that differ by four orders of magnitude
    were obtained by using different starting values for $τ$.
    These fits are otherwise similar with the exception of the lifetime,
    demonstrating the $τ$-independent scaling in \cref{eq:regime.1.nonlocal_resistance}.
    The $χ^2$ for \cref{fig:nonlocal_resistance.fits} (d)
    is \SI{2}{\percent} less than the $χ^2$ for
    \cref{fig:nonlocal_resistance.large_lifetime}.
  }
  \label{fig:nonlocal_resistance.large_lifetime}
  \includegraphics[width=\columnwidth]{figures/plot_fits_large_lifetime}
\end{figure}

Note that fitting is insensitive to $τ$ in the limit of
\cref{eq:regime.1.nonlocal_resistance}
or \cref{eq:regime.2.nonlocal_resistance}.
As an example of this,
\cref{fig:nonlocal_resistance.large_lifetime} shows nearly identical fits
with lifetimes that differ by four orders of magnitude.
These fits were obtained by choosing large starting values for $τ$.
For \cref{fig:nonlocal_resistance.fits} (d)
and \cref{fig:nonlocal_resistance.large_lifetime},
$χ^2 ∼ \SI{7e-8}{}$, but the $χ^2$ for
\cref{fig:nonlocal_resistance.fits} (d)
is \SI{2}{\percent} less than the $χ^2$ for
\cref{fig:nonlocal_resistance.large_lifetime}.
In \cref{fig:nonlocal_resistance.fits} (d),
$λ / r ≫ \sqrt{ω τ}$ and $ω τ ∼ 1$ for most of the curve,
so the approximation $1 + i ω τ ≈ i ω τ$ does not hold.
However, \cref{fig:nonlocal_resistance.large_lifetime}
is in the limit of \cref{eq:regime.1.nonlocal_resistance}
for all points (save the origin).
Thus, in limit of small $r$, the fitted value of $τ$ is unreliable
unless one carefully controls the fitting procedure.

The evolution of the expression for the Hanle curve
is an interesting insight into the behavior of the device.
Fitting data on devices with small contact resistances
with the functional form applicable to infinite contact resistance
yields unreliable parameters.
In particular, they were numerically shown to severely underestimate the spin lifetime
\cite{PhysRevB.86.235408}.

Further analytic progress can be made if one assumes that
lifetimes as estimated with infinite contact resistance are long
enough that the approximation of $\sqrt{ω τ} ≫ λ / r ≫ 1$
is still valid for much of the data being analyzed.
For this case, at infinite contact resistance,
the normalized nonlocal resistance is given by
\begin{equation}
  \frac{f^∞}{f^∞_0} = \frac{1}{ \sqrt{ω τ}}
                      e^{- \left( L / λ \right) \sqrt{ω τ / 2}}
                      \cos{\left[ \frac{L}{λ} \sqrt{\frac{ω τ}{2}} + \frac{π}{4} \right]} .
\end{equation}
Provided $D$ remains constant,
this will yield the same curve with finite contact resistance if
\begin{equation}
  \label{eq:lifetime_scale.infinity}
  \frac{1}{τ^∞} = D^2 \frac{τ}{r^4} .
\end{equation}
In other words, if we fix $τ$ and ask what happens to the fitted value
assuming infinite contact resistance as a function of decreasing $r$,
\cref{eq:lifetime_scale.infinity} shows that it will decrease as well.
For $D$ fixed, $τ^∞ ∝ r^4$.
While the general trend is consistent with
\cite{PhysRevB.86.235408},
the quantitative agreement is limited by
the approximations made for analytic convenience.


\paragraph{Binomial coefficients}
\[ {n \choose k} = (-1)^k{k-n-1 \choose k},\ \ \sum_{k \leq n}{r+k \choose k} = {r+n+1 \choose n} \]
\[ \sum_{k=0}^n{k \choose m} = {n+1 \choose m+1} \]
\[ \sqrt{1+z} = 1 + \sum_{k=1}^{\infty}\frac{(-1)^{k-1}}{k\times2^{2k-1}}{2k-2 \choose k-1}z^k \]
\[ \sum_{k=0}^{r}{r-k \choose m}{s+k \choose n} = {r+s+1 \choose m+n+1} \]
\[ C_{n, m} = {n+m \choose m} - {n+m \choose m-1}, n \geq m \]
\[ {n \choose k} \equiv [n\& k=k] \pmod 2 \]
\[ {{n_1+\cdots+n_p}\choose m}=\sum_{k_1+\cdots+k_p=m}{n_1\choose k_1}\cdots{n_p\choose k_p}\]
\paragraph{Fibonacci numbers}
\[ F(z) = \frac{z}{1-z-z^2} \]
\[ f_n = \frac{{\phi}^n-{\hat{\phi}}^n}{\sqrt{5}}, \phi = \frac{1+\sqrt{5}}{2},
\hat{\phi} = \frac{1-\sqrt{5}}{2} \]
\[ \sum_{k=1}^nf_k = f_{n+2}-1,\ \ \sum_{k=1}^nf^2_k = f_nf_{n+1} \]
\[ \sum_{k=0}^nf_kf_{n-k} = \frac{1}{5}(n-1)f_n+\frac{2}{5}nf_{n-1} \]
\[ \frac{f_{2n}}{f_n} = f_{n-1} + f_{n+1}\]
\[ f_1+2f_2+3f_3+\cdots+nf_n=nf_{n+2}-f_{n+3}+2]\]
\[ \gcd(f_m,f_n)=f_{\gcd(m,n)}\]
\[ f^2_n + (-1)^n = f_{n+1}f_{n-1} \]
\[ f_{n+k} = f_nf_{k+1} + f_{n-1}f_k \]
\[ f_{2n+1} = f^2_n+f^2_{n+1} \]
\[ (-1)^kf_{n-k} = f_{n}f_{k-1} - f_{n-1}f_{k} \]
\[ \text{Modulo }f_n, f_{mn+r} \equiv \left\{
\begin{aligned}
&f_r,& m \bmod 4 = 0; \\
&(-1)^{r+1}f_{n-r},& m \bmod 4 = 1; \\
&(-1)^nf_r,& m \bmod 4 = 2; \\
&(-1)^{r+1+n}f_{n-r},& m \bmod 4 = 3.
\end{aligned}
\right.
\]

Period modulo a prime $p$ is a factor of $2p+2$ or $p-1$.

Only exception: $G(5)=20$.

Period modulo the power of a prime $p^k$: $G(p^k)=G(p)p^{k-1}$.

Period modulo $n=p_1^{k_1}...p_m^{k_m}$: $G(n)=lcm(G(p_1^{k_1}),...,G(p_m^{k_m}))$.

\paragraph{Lucas numbers}
\[ L_0=2,L_1=1,L_n=L_{n-1}+L_{n-2}=(\frac{1+\sqrt{5}}{2})^n+\frac{1-\sqrt{5}}{2})^n \]
\[ L(x)=\frac{2-x}{1-x-x^2} \]
\paragraph{Catlan numbers}
\begin{align*}
c_1=1,c_n=\sum_{i=0}^{n-1}c_ic_{n-1-i}=c_{n-1}\frac{4n-2}{n+1}=\frac{\binom{2n}{n}}{n+1}\\
=\binom{2n}{n}-\binom{2n}{n-1}, c(x)=\frac{1-\sqrt{1-4x}}{2x}
\end{align*}
\paragraph{Stirling cycle numbers}
Divide $n$ elements into $k$ non-empty cycles.
\[s(n,0)=0,s(n,n)=1,s(n+1,k)=s(n,k-1)-ns(n,k)\]
\[s(n,k)=(-1)^{n-k}{n \brack k}\]
\[{n+1 \brack k} = n{n \brack k} + {n \brack k-1},{n+1 \brack 2} = n!H_n\]
 \begin{align*}
x^{\underline{n}} = x(x-1)...(x-n+1) &= \sum_{k=0}^n{ {n \brack k}(-1)^{n-k}x^k }\\
x^{\overline{n}} = x(x+1)...(x+n-1) &= \sum_{k=0}^n{ {n \brack k}x^k }\\
 \end{align*}
\paragraph{Stirling subset numbers}
Divide $n$ elements into $k$ non-empty subsets.
\[ {n+1 \brace k} = k{n \brace k} + {n \brace k-1} \]
\[ x^n = \sum_{k=0}^n{ {n \brace k}x^{\underline{k}} } = \sum_{k=0}^n{ {n \brace k}(-1)^{n-k}x^{\overline{k}} } \]
\[ m!{n \brace m} = \sum_{k=0}^m{m \choose k}k^n(-1)^{m-k} \]
\[ \sum_{k=1}^nk^p = \sum_{k=0}^p{p \brace k}(n+1)^{\underline{k}} \]
For a fixed $k$, generating functions :
\[\sum_{n=0}^{\infty}{n \brace k}x^{n-k}=\prod_{r=1}^{k}\frac{1}{1-rx}\]
\paragraph{Motzkin numbers}
Draw non-intersecting chords between n points on a circle.

Pick $n$ numbers $k_1,k_2,...,k_n\in\{-1,0,1\}$ so that $\sum_i^ak_i(1\leq a\leq n)$ is non-negative and the sum of all numbers is $0$.
\[M_{n+1}=M_n+\sum_i^{n-1}M_iM_{n-1-i}=\frac{(2n+3)M_n+3nM_{n-1}}{n+3}\]
\[M_n=\sum_{i=0}^{\lfloor \frac{n}{2}\rfloor}\binom{n}{2k}Catlan(k)\]
\[M(X)=\frac{1-x-\sqrt{1-2x-3x^2}}{2x^2}\]
\paragraph{Eulerian numbers}
Permutations of the numbers $1$ to $n$ in which exactly $k$ elements are greater than the previous element.
\[ {n \bangle k} = (k+1){n-1 \bangle k} + (n-k){n-1 \bangle k-1} \]
\[ x^n = \sum_k{ {n \bangle k}{x+k \choose n} } \]
\[ {n \bangle m} = \sum_{k=0}^m{n+1 \choose k}(m+1-k)^n(-1)^k \]
\paragraph{Harmonic numbers}
Sum of the reciprocals of the first n natural numbers.
\[ \sum_{k=1}^nH_k = (n+1)H_n-n \]
\[ \sum_{k=1}^nkH_k = \frac{n(n+1)}{2}H_n - \frac{n(n-1)}{4} \]
\[ \sum_{k=1}^n{k \choose m}H_k = {n+1 \choose m+1}(H_{n+1} - \frac{1}{m+1}) \]
\paragraph{Pentagonal number theorem}
\[ \prod_{n=1}^{\infty}(1-x^n) = \sum_{n=-\infty}^{\infty}{(-1)^kx^{k(3k-1)/2}} \]
\[ p(n) = p(n-1)+p(n-2)-p(n-5)-p(n-7)+\cdots \]
\[ f(n, k) = p(n)-p(n-k)-p(n-2k)+p(n-5k)+p(n-7k)-\cdots \]
\paragraph{Bell numbers}
Divide a set that has exactly n elements.
\[ B_n=\sum_{k=1}^{n}{n\brace k},\ \ B_{n+1} = \sum_{k=0}^n{n \choose k}B_k \]
\[ B_{p^m+n} \equiv mB_n+B_{n+1} \pmod{p} \]
\[B(x)=\sum_{n=0}^{\infty}\frac{B_n}{n!}x^n=\mathrm{e}^{\mathrm{e}^x-1}\]
\paragraph{Bernoulli numbers}
\[ B_n = 1 - \sum_{k=0}^{n-1}{n \choose k}\frac{B_k}{n-k+1} \]
\[ G(x) = \sum_{k=0}^{\infty}\frac{B_k}{k!}x^k
= \frac{1}{\sum_{k=0}^{\infty}\frac{x^k}{(k+1)!}} \]
\[ \sum_{k=1}^nk^m = \frac{1}{m+1}\sum_{k=0}^m{m+1 \choose k}B_kn^{m-k+1} \]
\paragraph{Sum of powers}
\[\sum_{i=1}^ni^2=\frac{n(n+1)(2n+1)}{6},\ \ \sum_{i=1}^ni^3=(\frac{n(n+1)}{2})^2\]
\[\sum_{i=1}^ni^4=\frac{n(n+1)(2n+1)(3n^2+3n-1)}{30}\]
\[\sum_{i=1}^ni^5=\frac{n^2(n+1)^2(2n^2+2n-1)}{12}\]
\paragraph{Sum of squares}
Denote $r_k(n)$ the ways to form $n$ with $k$ squares. If :
\[n=2^{a_0}p_1^{2a_1}\cdots p_r^{2a_r}q_1{b_1}\cdots q_s{b_s}\]
where $p_i\equiv 3 \mod 4$, $q_i\equiv 1 \mod 4$, then
\[r_2(n)=\left\{\begin{aligned}
& 0 & \text{if any }a_i\text{ is a half-integer}\\
& 4\prod_1^r(b_i+1) & \text{if all }a_i\text{ are integers}\\
\end{aligned}\right.\]
$r_3(n)>0$ when and only when $n$ is not $4^a(8b+7)$.
\paragraph{Derangement}
\[D_1=0,D_2=1,D_n=n!(\frac{1}{0!}-\frac{1}{1!}+\frac{1}{2!}-\frac{1}{3!}+...+\frac{(-1)^n}{n!})\]
\[D_n=(n-1)(D_{n-1}+D_{n-2})\]
\paragraph{Tetrahedron volume}
If $U$, $V$, $W$, $u$, $v$, $w$ are lengths of edges of the tetrahedron (first three form a triangle; u opposite to U and so on)
\[ V = \frac{\sqrt{ 4u^2v^2w^2 - \sum_{cyc}{u^2(v^2+w^2-U^2)^2} + \prod_{cyc}{(v^2+w^2-U^2)} }}{12} \]


\documentclass{article}

\usepackage[letterpaper, margin=1.3cm]{geometry}
\usepackage[utf8]{inputenc}
\usepackage{siunitx}
\usepackage[fleqn]{mathtools}
\usepackage{amssymb}
\usepackage{mathrsfs}

\title{MATH 225 Assignment 1}
\author{Michael Kwok}
\begin{document}
\maketitle
\subsection*{1}
Let $\mathbf{u} \in V$

Suppose there exists $w_1, w_2 \in V$ such that $\mathbf{u} \oplus \mathbf{w_1} = \Tilde{\mathbf{z}}$ and $\mathbf{u} \oplus \mathbf{w_2} = \Tilde{\mathbf{z}}$.

1. $\mathbf{u}\oplus \mathbf{w_1} = \Tilde{\mathbf{z}}$

2. $\mathbf{u}\oplus \mathbf{w_2} = \Tilde{\mathbf{z}}$

3. $\mathbf{w_1} = - \mathbf{u}$ Statement 1 by axiom 5

4. $\mathbf{w_2} = - \mathbf{u}$ Statement 2 by axiom 5

$\therefore \mathbf{w_1} = \mathbf{w_2}$ Statement 3 and 4
\newpage
\subsection*{2a}
Axiom 1:

$\Tilde{\mathbf{z}} = (0, 0)$

$(x, y) = (0, 0)$

$0^2 + 0^2 \leq 1$

\noindent
Axiom 2:

Counter example

Let $\mathbf{u} = \left(0.5, 0.5\right)$

Let $\mathbf{v} = \left(0.5, 0.4\right)$

$\mathbf{u} + \mathbf{v} = (1, 0.9)$

$1^2 + 0.9^2 = 1.81 > 1$

$(1, 0.9) \not\in H$

It does not fulfil this axiom

$\therefore$ It is not a subspace.

\subsection*{2b}
Axiom 1:

From $\mathscr{P}_{n=0}, \Tilde{\mathbf{z}} = 0$

When $n = 0 \text{ and } a_0 = 0$, $0 \in H$

$\therefore \Tilde{\mathbf{z}} \in H$

\noindent
Axiom 2:

let $\mathbf{u} = a_0 + a_1x + \cdots + a_nx^n$

let $\mathbf{v} = b_0 + b_1x + \cdots + b_nx^n$
\begin{align*}
\mathbf{u} + \mathbf{v} &= (a_0 + a_1x + \cdots + a_nx^n) + (b_0 + b_1x + \cdots + b_nx^n)\\
&= (a_0 + b_0) + (a_1 + b_1)x + \cdots + (a_n + b_n)x^n
\end{align*}
\begin{align*}
\left(a_0 - a_1 + ... + (-1)^n a_n\right) = 0\\
\left(b_0 - b_1 + ... + (-1)^n b_n\right) = 0\\
\end{align*}
\noindent
Axiom 3:
\begin{align*}
\text{Let } c\mathbf{u} &= c\left(a_0 + a_1x + \cdots + a_n x^n\right)\\
&= ca_0 + ca_1x \cdots + ca_nx^n
\end{align*}
\begin{align*}
\left[ca_0 - ca_1 + \cdots +  c(-1)^na^n\right] &= 0\\
c\left[a_0 - a_1 + \cdots + (-1)^n a^n\right]&=0\\
\end{align*}
$\therefore$ It is a subspace

\subsection*{2c}
Use SV2 to show $W$ is a valid subspace.

Let $k_1, k_2 \in \mathbb{R}$

Let $\mathbf{u} = \begin{bmatrix}
a_1 + 2b_1 & b_1 + c_1\\
b_1 - c_1 & a_1 + 4c_1
\end{bmatrix}$

Let $\mathbf{v} = \begin{bmatrix}
a_2 + 2b_2 & b_2 + c_2\\
b_2 - c_2 & a_2 + 4c_2
\end{bmatrix}$

Show that $k_1 \mathbf{u} + k_2 \mathbf{v} \in W$
\begin{align*}
k_1 \mathbf{u} + k_2 \mathbf{v} &=
\begin{bmatrix}
k_1a_1 + 2 k_1 b_1 + k_2 a_2 + 2 k_2 b_2 & k_1 b_1 + k_1 c_1 + k_2 b_2 + k_2 c_2\\
k_1 b_1 - k_1 c_1 + k_2 b_2 - k_2 c_2 & k_1 a_1 + 4 k_1 c_1 + k_2 a_2 + 4 k_2 c_2
\end{bmatrix}\\
&= \begin{bmatrix}
(k_1 a_1 + k_2 a_2) + 2 (k_1 b_1 + k_2 b_2) & (k_1 b_1 + k_2 b_2) + (k_1 c_1  + k_2 c_2)\\
(k_1 b_1 + k_2 b_2) - (k_1 c_1 + k_2 c_2) & (k_1 a_1 + k_2 a_2) + 4 (k_1 c_1  + k_2 c_2)
\end{bmatrix}
\end{align*}
$a= k_1 a_1 + k_2 a_2,\text{ }b=k_1 b_2 + k_2 b_2,\text{ }c=k_1 c_1 + k_2 c_2$

$a, b, c \in \mathbb{R}$

$k_1 \mathbf{u} + k_2 \mathbf{v} \in W$

$\therefore W$ is a subspace of $M_{2,2}$
\newpage
\subsection*{3a}
\noindent
Let $\mathcal{B} = \left\{
\begin{bmatrix}
1 & 0\\
0 & 0
\end{bmatrix},
\begin{bmatrix}
0 & 1\\
1 & 0
\end{bmatrix},
\begin{bmatrix}
0 & 0\\
0 & 1
\end{bmatrix}\right\}$

\noindent
B1:

$a \begin{bmatrix}
1 & 0\\
0 & 0
\end{bmatrix} +
b \begin{bmatrix}
0 & 1\\
1 & 0
\end{bmatrix} +
c \begin{bmatrix}
0 & 0\\
0 & 1
\end{bmatrix} =
\begin{bmatrix}
0 & 0\\
0 & 0
\end{bmatrix} \iff a, b, c = 0$

$\therefore \mathcal{B}$ is linearly independent.

\noindent
B2:

$S_{2,2} = \left\{
\begin{bmatrix}
a & b\\
b & c
\end{bmatrix}: a, b, c \in \mathbb{R}
\right\}$

$a \begin{bmatrix}
1 & 0\\
0 & 0
\end{bmatrix} +
b \begin{bmatrix}
0 & 1\\
1 & 0
\end{bmatrix} +
c \begin{bmatrix}
0 & 0\\
0 & 1
\end{bmatrix} =
\begin{bmatrix}
a & b\\
b & c
\end{bmatrix}$

$\therefore \mathcal{B} \text{ spans } V$

\subsection*{3b}

Let $\mathcal{B} = \left\{1\right\}$

\noindent
B1:

$a \cdot 1 = 0 \iff a = 0$

\noindent
B2:

Let $b \in \left( \mathbb{R}_{>0}, \times,\wedge \right)$
\begin{align*}
    a \cdot 1 &= b\\
    a &= b
\end{align*}

For any value $b \in \left( \mathbb{R}_{>0}, \times,\wedge \right)$, the basis could be used to form a solution.

\end{document}


\chapter{Backward Funcoids}

This is a preliminary partial draft.

Fix a family $\mathfrak{A}$ of posets.

\begin{defn}
  Let $f$ be a staroid of filters $\mathfrak{F} (\mathfrak{A}_i)$ on boolean
  lattices $\mathfrak{A}_i$. \emph{Backward funcoid} for the argument $k
  \in \dom \mathfrak{A}$ of $f$ is the funcoid $\Back (f , k)$
  defined by the formula (for every $X \in \mathfrak{A}_k$)
  \[ \langle \Back (f , k) \rangle X = \setcond{ L \in \prod_{i \in
     \dom \mathfrak{A}} \mathfrak{F} (\mathfrak{A}_i) }{
     X \in \supfun{f}_k L } . \]
\end{defn}

\begin{prop}
  Backward funcoid is properly defined.
\end{prop}

\begin{proof}
  $\langle \Back (f , k) \rangle^{\ast} (X \sqcup Y) = \setcond{ L \in
  \prod \mathfrak{A} }{ X \sqcup Y \in \langle f
  \rangle_k L } = \setcond{ L \in \prod \mathfrak{A} }{
  X \in \supfun{f}_k L \vee Y \in \supfun{f}_k L
  } = \setcond{ L \in \prod \mathfrak{A} }{ X
  \in \supfun{f}_k L } \cup \setcond{ L \in \prod \mathfrak{A}
  }{ Y \in \supfun{f}_k L } = \langle
  \Back (f , k) \rangle^{\ast} X \cup \langle \Back (f , k)
  \rangle^{\ast} Y$.
\end{proof}

\begin{obvious}
Backward funcoid is co-complete.
\end{obvious}

\begin{prop}
  If $f$ is a principal staroid then $\Back (f , k)$ is a complete
  funcoid.
\end{prop}

\begin{proof}
  ??
\end{proof}

\begin{prop}
  $f$ can be restored from $\Back (f , k)$ (for every fixed $k$).
\end{prop}

\begin{proof}
  ??
\end{proof}

\begin{prop}
  $f \mapsto \Back (f , k)$ is an order isomorphism
  $\mathsf{Strd}^{\mathfrak{A}} \rightarrow \mathsf{FCD} \left(
  \mathfrak{A}_k , \mathsf{Strd}^{(\dom \mathfrak{A}) \setminus \{ k \}}
  \right)$.
\end{prop}

\begin{proof}
  ??
\end{proof}


% !TeX spellcheck = en_US

\chapter{Trees}

\section {General Trees}

A \textbf{tree} is an abstract data type that organize that elements hierarchically.

A tree is \textbf{ordered} if there is a meaningful order among the children of each node.

\section{Binary Tree}

A \textbf{binary tree} is an ordered tree with the following properties :
....... to finish

\begin{tikzpicture}[sibling distance=10em,
every node/.style = {shape=circle,
	draw, align=center,
	top color=white, bottom color=blue!20}]]
\node {10}
child { node {5} }
child { node {25}
	child { node {18}
		child { node {12} }
		child { node {20} } }
	child { node {30} } };
\end{tikzpicture}

\section{Tries (Prefix trees)}

A \textbf{trie} is a variant of a n-ary tree in which characters are stored at each node.

\cite{latexcompanion}

Doc/Notes on Data Structures and Algorithms/TeX_files/chapter_trees.tex


\section{Multivariate linear regression on median score}\label{app:regression_median_score}

A multivariate linear regression has also been fitted to model the relationship between
the features and the median score. The features included are given
by Table~\ref{table:linear_regression_on_median} alongside their corresponding \(p\) values
in the distinct tournaments and their regression coefficients.

\newcolumntype{g}{>{\columncolor{Gray}}c}
\begin{table}[h]
    \begin{center}
\resizebox{\textwidth}{!}{
\begin{tabular}{lggccggccgg}
    \toprule
    & \multicolumn{2}{g}{Standard} & \multicolumn{2}{c}{Noisy} & \multicolumn{2}{g}{Probabilistic ending} & \multicolumn{2}{c}{Noisy probabilistic ending} & \multicolumn{2}{g}{Overall}\\
    \midrule
    & \multicolumn{2}{g}{\(R\) adjusted: 0.576} & \multicolumn{2}{c}{\(R\) adjusted: 0.679} & \multicolumn{2}{g}{\(R\) adjusted: 0.816} & \multicolumn{2}{c}{\(R\) adjusted: 0.930} & \multicolumn{2}{g}{\(R\) adjusted: 0.318} \\
    {} &  Coefficient &  \(p\)-value &  Coefficient &  \(p\)-value &  Coefficient &  \(p\)-value &  Coefficient &  \(p\)-value &  Coefficient &  \(p\)-value \\
    \midrule
    $CC$ to $C$ rate   &  0.043 &  0.000 & -0.380 &  0.000 &  0.224 &  0.000 &  0.078 &    0.0 &  0.308 &    0.0 \\
    $CD$ to $C$ rate   & -0.325 &  0.000 &  0.124 &  0.000 &  0.060 &  0.000 &  0.073 &    0.0 & -0.014 &    0.0 \\
    $C_r$ / $C_{max}$  &      - &      - & -1.044 &  0.000 &      - &      - & -1.251 &    0.0 &      - &      - \\
    $C_r$ / $C_{mean}$ &  0.553 &  0.000 & -0.101 &  0.000 & -1.136 &  0.000 & -0.089 &    0.0 & -0.665 &    0.0 \\
    $C_{max}$          &  0.059 &  0.000 &      - &      - & -0.044 &  0.086 & -1.396 &    0.0 &      - &      - \\
    $C_{mean}$         &  1.837 &  0.000 &  3.078 &  0.000 &  1.506 &  0.000 &  3.645 &    0.0 &      - &      - \\
    $C_{min}$          &  0.156 &  0.000 &  1.528 &  0.000 &  0.311 &  0.000 &      - &      - &      - &      - \\
    $C_{min}$ / $C_r$  & -0.049 &  0.000 & -0.378 &  0.000 & -0.204 &  0.000 &      - &      - & -0.257 &    0.0 \\
    $DC$ to $C$ rate   & -0.204 &  0.000 &  0.074 &  0.000 &  0.066 &  0.000 &  0.066 &    0.0 &  0.007 &    0.0 \\
    $k$                & -0.000 &  0.853 & -0.000 &  0.987 &  0.000 &  0.008 &  0.000 &    0.0 &      - &      - \\
    $n$                & -0.000 &  0.000 &      - &      - &      - &      - &      - &      - &      - &      - \\
    $p_e$              &      - &      - &      - &      - &  0.025 &  0.000 & -0.095 &    0.0 &      - &      - \\
    $p_n$              &      - &      - &  0.124 &  0.000 &      - &      - &      - &      - &      - &      - \\
    SSE                & -0.294 &  0.000 & -0.319 &  0.000 &  0.055 &  0.000 &  0.010 &    0.0 & -0.015 &    0.0 \\
    constant           &  0.925 &  0.000 &  1.536 &  0.000 &  2.466 &  0.000 &  2.299 &    0.0 &  2.924 &    0.0 \\
    memory usage       &  0.010 &  0.000 & -0.004 &  0.000 &      - &      - &      - &      - &      - &      - \\
    \bottomrule
\end{tabular}}
\end{center}
\caption{Results of multivariate linear regressions with the median score as the dependent variable.
\(R\) squared is reported for each model.}
\label{table:linear_regression_on_median}
\end{table}


% Compile with: pdflatex explanation.tex

\documentclass[11pt]{article}

    \usepackage{amsmath}
    \usepackage{amssymb}
    %\usepackage{mathrsfs} % Cursive letters
    %\usepackage[stable]{footmisc} % Allows footnotes in titles
    \usepackage[margin=2.5cm]{geometry}
    %\usepackage{enumerate} % Easily change enumeration symbols
    % Doesnt run in sharelatex \usepackage{ulsy} % Proof by contradiction symbol \blitza

    \newcommand{\E}{\mathds{E}}
    \newcommand{\V}{\mathds{V}}
    \newcommand{\PR}{\mathds{P}}
    \newcommand{\QED}{$\blacksquare$}
    \newcommand{\qed}{$\square$}
    \newcommand{\QEA}{\blitza}
    \newcommand{\st}{\;\text{s.t.}\;}
    \newcommand{\maximize}[1] {\underset{#1}{\text{max}}}
    \def\s{\rule{0in}{.30in}} % Strut

\begin{document}
    \title{Explanation of the HDFE iteration with 3 FEs}
  \author{Sergio Correia}
    %\date{}

\maketitle

\section{Background}

Our objective is to obtain the OLS estimates for $\beta$ in the regression $y = X\beta + D_1\alpha_1 + D_2\alpha_2 + D_3\alpha_3 + \epsilon$, where $D_i$ represents a set of indicator or design variables, and $\alpha_i$ is the set of associated fixed effects.

Let $M$, $P$ be the usual annihilator and projection matrices in an OLS regression. When the regressors are dummy variables, $P$ and $M$ are just averages and demeaned values within each group respectively. Also, from the normal equation we know the projection matrix is orthogonal to the residuals: $Pe=0$.

Recall the two-part FWL theorem. Its first part states that we can regress $y = X\beta + D_1\alpha_1 + D_2\alpha_2 + D_3\alpha_3 + \epsilon$ by first regressing $y$ and each $X$ against the set $D_1, D_2, D_3$, and then just regressing the residuals of $y$ against the residuals of $X$. The second part states that the residuals of both regressions are also identical.

Thus, the strategy will be to apply FWL to obtain residuals of $(y,X)$ with respect to all the indicator variables, and then regress the residuals to obtain estimates for $\beta$.

Notice that estimates for each $\alpha$ can still be recovered, but they may not be identified.

\section{Fixed Point Iteration}

To regress against dummies there are two usual approaches: (i) add the dummies explictly or (ii) demean all the variables. Option (ii) is much faster but can only be done \textit{once}, so if we have two or more sets of DVs the usual approach is to apply (ii) to the fixed effect with more dimensions (i.e. categories) and (i) to all the others. However, if there are two or more sets of high-dimensional DVs, we would have to create a lot of dummies which is impractical (b/c of out of memory errors, etc.).

Instead, we'll apply a fixed-point iteration over the normal equations, as suggested by Guimaraes \& Portugal (2010) (i.e., guess a value, and iterate our estimates until the normal equations hold). Note that instead of iterating the estimates $\hat\alpha_1, \hat\alpha_2, \hat\alpha_3$, we will iterate $Z_2 := D_2\hat{\alpha_2}$ and $Z_3 := D_3\hat{\alpha_3}$. Also, note that we are not employing Guimaraes et al (2013) version for three sets of FEs, but an extension of their original work, which works significatively faster.

\section{Algorithm}

Wlog, start with $y$,

\begin{enumerate}
    \item Compute $P_1y$ and $\tilde{y} := M_1y$
    \item Initialize $Z_2^{(0)} = Z_3^{(0)} = 0$
    \item Until $Z_2$ and $Z_3$ converge, set
        \begin{enumerate}
        \item $Z_2^{(n)} = P_2 \left[ \tilde{y} + P_1 \left( Z_2^{(n-1)} + Z_3^{(n-1)} \right) - Z_3^{(n-1)} \right]$
        \item $Z_3^{(n)} = P_3 \left[ \tilde{y} + P_1 \left( Z_2^{(n)} + Z_3^{(n-1)} \right) - Z_2^{(n)} \right]$
        \end{enumerate}
    \item Then, compute $Z_1 = P_1(y - Z_2 - Z_3)$ and with that compute $y^* = y - Z_1 - Z_2 - Z_3$, which is what we desire.
\end{enumerate}

After repeating this for every variable, regress the transformed variables ($y^* = X^* \hat{\beta} + e$)to obtain the correct estimates $\hat{\beta}$. Notice that the residuals of this FWL regression are the same as those of the full regression.

If you want to obtain the fixed effects for the regression, notice that $y = X \hat{\beta} + Z_1 + Z_2 + Z_3 + e$ where both $\hat{\beta}$ and  $e$ are the same as in the transformed regression. Thus, compute $\tilde{e} := y - X \hat{\beta} = e + Z_1 + Z_2 + Z_3$ , and then apply the previous step to obtain the $Z_1, Z_2, Z_3$ fixed effects for that variable.

Notice that if we remove the third FE (i.e. set $Z_3=0$), the formula collapses to the usual two-FE version.

\section{Deriving the fixed point iteration with three fixed-effects}

\begin{enumerate}
    \item Start with the identity $y = D_1\hat\alpha_1 + D_2\hat\alpha_2 + D_3\hat\alpha_3 + e = Z_1 + Z_2 + Z_3 + e$, where $Z_i := D_i\hat\alpha_i$
    \item Premultiply by $M_1$ to obtain $\tilde{y} = M_1Z_2 + M_1Z_3 + e$, since $M_1D_1 = D_1 - P_1D_1 = D_1 - D_1 = 0$ and $M_1e = e - P_1 e = 0$ (from the normal equation).
    \item Rearrange $Z_2 + Z_3 = \tilde{y} + P_1 \left( Z_2 + Z_3 \right) - e$
    \item Premultiply by $P_2$ and notice that $P_2 e = 0$, and after rearranging,
    obtain $Z_2 = P_2 \left[ \tilde{y} + P_1 \left( Z_2 + Z_3 \right) - Z_3 \right]$
    \item Premultiply by $P_3$ and obtain the equivalent equation for the third fixed effect.
\end{enumerate}

With these formulas, and adding a little more structure (to show that the fixed point is attractive, etc.), we can turn the formulas above into a fixed point iteration that will converge linearly.

\section{Implementation details}

This fixed point iteration is very slow, but there are (at least) two venues for speeding it up.

\begin{enumerate}
    \item We can minimize the number of $P$ operations and speed them up. First, notice that sorting the dataset (an $O(n \log(n))$ operation) is not needed to obtain group means. Also, since in this case $P$ just demeans a variable within a group, we can store the output of $P_i$ as a $G_1 \times 1$ vector, instead of a $N \times 1$ vector. Finally, we can try precompute intermediate steps such as $P_2 \tilde{y}$ (which we only compute once) and $P_1Z_2$ (which we only compute once per iteration, instead of twice).
    \item We can try to extrapolate the fixed-point iteration so it converges faster. These methods are called acceleration tecniques, and of those, we use a vector-based version of Aitken's $\Delta^2$ method. In particular, we are employing method 3 as described by Macleod (1986), which has a reasonable performance, and accelerating every three iterations.
\end{enumerate}

All in all, each iteration requires only 6 $P$ operations (2 in the case of just two fixed effects), which are implemented reasonably fast. Together with the acceleration, this method becomes both fast and conservative in terms of memory usage.

\section{Extension: Algorithm beyond three sets of fixed effects}

The algorithm of part 3 can be extended to an arbitrarily large number of fixed effects. This will lead to the following steps:

Wlog, start with $y$, and let there be $G$ sets of fixed effects.

\begin{enumerate}
    \item Compute $P_1y$ and $\tilde{y} := M_1y$
    \item Precompute $P_g \tilde{y} \quad \forall g>1$
    \item Initialize $Z_g^{(0)} = 0 \quad \forall g>1$
    \item Initialize $\Sigma^{(0)} := \sum_{g=2}^G Z_g^{(0)} = 0$
    \item Until all $Z_g$ converge for all $g>1$ (this condition can also be replaced by a condition on $\Sigma$), set for $g=2, 3, \ldots, G$:
        \begin{enumerate}
        \item $Z_g^{(n)} = Z_g^{(n-1)} + P_g \left[ \tilde{y} + (P_1 - 1) \Sigma^{(n-1,g-1)} \right]$
        \item $\Sigma^{(n-1,g)} = \Sigma^{(n-1,g)} + Z_g^{(n)} - Z_g^{(n-1)}$
        \end{enumerate}
    \item Then, compute $Z_1 = P_1(y - \Sigma)$ and with that compute $y^* [ = M y = M_1 (y- \Sigma)  ] = y - Z_1 - \Sigma$.
\end{enumerate}

Notice that step 5 could be expressed more compactly just in terms of $\Sigma$, but that complicates obtaining the individual FEs.

\section{Extension: Interactions between FEs and continuous variables}

 If we interact a variable $V$ elementwise with each column of a design matrix $D$ representing a set of $K$ fixed effects, a few adjustements would be required.

 First, note that it's equivalent to estimating a system of bivariate regressions. For instance, if the $D$ matrix had three fixed effects (i.e. it's a $N \times 3$ matrix), then $P_D$ could be written as the partitioned matrix:

 \[ \left( \begin{array}{ccc}
P_1 & 0 & 0 \\
0 & P_2 & 0 \\
0 & 0 & P_3 \end{array} \right)\]

And each $P_j$ would be in turn be equal to (for the case of 3 obs. in the group $j$):
\[ \left( \begin{array}{ccc}
v_{11}^2 & v_1v_2 & v_1v_3 \\
v_2v_1 & v_{22}^2 & v_2v_3 \\
v_3v_1 & v_3v_2 & v_{33}^2 \end{array} \right)\]

(subindices are relative to each group)

Then, we can show that $U := P_{D} y$  is such that $u_i = v_i \times q_{g(i)}$, where $Q$ is a $K \times 1$ vector s.t. $q_j = \frac{\sum_{i \; \text{s.t.} \; D_{ij}=1} v_i y_i}{v_i^2}$.

In terms of code, before we obtained a $Q$ matrix where $v_i=1$, so we were basically computing group means ($q_j = \frac{\sum_{i \; \text{s.t.} \; D_{ij}=1} y_i}{N_j}$) and now we are i) weighting them and ii) multiplying by $v_i$ afterwards.

Some optimizations can be done, such as precomputing the denominators of $Q$, and doing scalar products of $v$ and $y$ so we can use the code of group means without many changes.

\section{Extension: Instrumental Variables}
Just demean \emph{all} variables w.r.t the set of fixed effects. Then, we just run the first stage and second stage regressions taking care of adjusting the standard errors.

\end{document}


%!TEX root = ../Tibt.tex

\exercise{3.18}

\subsection*{Review of PLS}
Let's review the PLS algorithm:

\begin{eqnarray}
    \mathbf{z}_m & = & \mathbf{X}^{(m - 1)} \mathbf{X}^{(m - 1)\, T} \mathbf{y} \\ \label{eq:3p18_recy}
    \hat{\mathbf{y}}_m & = & \hat{\mathbf{y}}_{m - 1} + \frac{\mathbf{z}_m \, \mathbf{z}_m^T}{||\mathbf{z}_m||^2}\, \mathbf{y}\\
    \\
    \mathbf{X}^{(m)} & = & \mathbf{X}^{(m - 1)} - \frac{\mathbf{z}_m \, \mathbf{z}_m^T}{||\mathbf{z}_m||^2} \mathbf{X}^{(m - 1)}
\end{eqnarray}
Denoting $P_m$ the euclidean projector over the subspace generated by $\mathbf{z}_m$:
\begin{equation*}
P_m  \equiv \frac{\mathbf{z}_m \, \mathbf{z}_m^T}{||\mathbf{z}_m||^2}: \quad P_m^2 = P_m, \;\; P_m \, \mathbf{z}_m = \mathbf{z}_m
\end{equation*}
One has then:
\begin{eqnarray}
    \hat{\mathbf{y}}_m & = & \left(P_1 + \ldots P_m\right) \mathbf{y}  \\
    \label{eq:3p18_recX}
    \mathbf{X}^{(m)} & = & \left(1 - P_m\right) \mathbf{X}^{(m - 1)}
\end{eqnarray}

$\mathbf{X}^{(m)}$ is orthogonal not only to $\mathbf{z}_m$, but also to
all previous $\mathbf{z}$'s:
\begin{equation*}
\mathbf{X}^{(m)\,T} \mathbf{z}_n \;\; \forall\; n \leq m
\end{equation*}
We can prove this by recursion over $m$. Observe that:
\begin{eqnarray*}
    \mathbf{X}^{(m)\,T} \mathbf{z}_n & = & \mathbf{X}^{(m - 1)\,T}
        \left(1 - P_m\right) \mathbf{z}_n
\end{eqnarray*}
For $n = m$ the rhs vanishes because $P_m \, \mathbf{z}_m = \mathbf{z}_m$,
while for $n \leq m - 1$ one has:
\begin{equation*}
P_m\, \mathbf{z_n} = \frac{\mathbf{z}_m}{||\mathbf{z}_m||^2} \mathbf{z}_m^T \mathbf{z}_n = \frac{\mathbf{z}_m}{||\mathbf{z}_m||^2} \mathbf{y}^T \mathbf{X}^{(m - 1)} \mathbf{X}^{(m - 1)\, T} \mathbf{z}_n = 0
\end{equation*}
by recursive hypothesis on the product of the last two terms.

Since $\mathbf{z}_{m + 1}$ is a linear combination of the columns of
$\mathbf{X}^{(m)}$, this also implies that the $\mathbf{z}$'s are mutually
orthogonal:
\begin{eqnarray*}
    \mathbf{z}_m^T \, \mathbf{z}_n & = & 0 \;\; \forall\; n \neq m\\
    P_m \, P_n & = & 0  \;\; \forall\; n \neq m
\end{eqnarray*}
The recurrence in \eqref{eq:3p18_recX} can therefore be 'unrolled' as:

\begin{equation}
\mathbf{X}^{(m)} = \left(1 - P_1 - \ldots - P_m\right) \mathbf{X}
\end{equation}

\subsection*{Connection with CGA: notation}
The connection with conjugate gradients algorithms (CGA) is at the level
of the regression coefficients $\beta$. As explained in the book,
all $\mathbf{X}^{(m)}$'s are linear combinations of the original $\mathbf{X}$, therefore:
\begin{equation}
\hat{\mathbf{y}}_m = \mathbf{X}\beta_m
\end{equation}
for some vector $\beta_m$. If we compare with \eqref{eq:3p18_recy},
we see that:
\begin{equation*}
\hat{\mathbf{y}}_m - \hat{\mathbf{y}}_{m - 1} =
    \mathbf{X} \left(\beta_m - \beta_{m - 1}\right) \propto
    \mathbf{z}_m
\end{equation*}
In CGA, the value of a function $L(\beta)$ is minimized by
successively moving $\beta$ along some directions $p_m$:
\begin{equation*}
\beta_m - \beta_{m - 1} \propto p_{m - 1}
\end{equation*}
Therefore, in order to establish a connection, we denote:
\begin{equation}\label{eq:e3p18_p}
\mathbf{X} p_{m - 1} \equiv \mathbf{z}_m
\end{equation}
Notice that this equation determines $p_{m - 1}$ uniquely
as long as the $\mathbf{X}$ are linearly independent.

Finally, we denote $l(\beta)$ the L2 loss and $g(\beta)$ its gradient:
\begin{eqnarray*}
    l(\beta) & \equiv & ||\mathbf{y} - \mathbf{X} \beta ||^2 \\
    g(\beta) & = & \nabla l(\beta) = - \mathbf{X}^T \left(\mathbf{y}
        - \mathbf{X} \beta \right)
\end{eqnarray*}

\subsection*{Review of CGA}

The CGA algorithm tries to minimize a quadratic form:
\begin{equation*}
f(\beta) = c^T \beta + \frac{1}{2} x^T G x, \;\;x \in \mathbb{R}^p\;\; G = G^T
\end{equation*}
by recursively finding the optimal update of $x$ along
increasingly larger subspaces of $\mathbb{R}^p$. In practice,
one starts from a set of vectors:
$$
p_0, p_1, \ldots \in \mathbb{R}^p
$$
and recursively defines:
\begin{eqnarray*}
    x_{m + 1} & = & x_{m} + \,
    \textrm{argmin}_{w \in \mathcal{P}_{m}} f(x_{m} + w) \\
    \mathcal{P}_{m} & \equiv & \textrm{Span} \left(p_0, \ldots, p_m\right)
\end{eqnarray*}
In particular, in conjugate gradient methods one takes:
\begin{eqnarray}
    \mathcal{P}_{m} & = & \textrm{Span} \left(g(x_0), \ldots, g(x_m)\right) \\
    p_0 & = & - g(x_0) \\ \label{eq:3p18_3}
    p_m & = & - g(x_m) + \gamma_{m - 1}\, p_{m - 1}
\end{eqnarray}
where $g(x)$ is the gradient of $f$, in accordance with
the notation of the previous section, and the coefficients $\gamma$ are chosen
in such a way that the $p_m$ are all mutually \textit{conjugate}:
\begin{equation*}
p_m^T G\,p_n = 0 \quad \forall \; m \neq n
\end{equation*}
The fact that the second term in \eqref{eq:3p18_3} is
sufficient to guarantee orthogonality is not trivial,
see \cite{Murray1982}, Section 4.8.3. Thanks to conjugacy,
the update $x_m \rightarrow x_{m + 1}$ consists in moving $x$
along the $p_m$ direction only:

\begin{equation}\label{eq:e3p18_update}
x_{m + 1} = x_m + \left(\textrm{argmin}_\alpha
    f(x_m + \alpha \, p_{m})\right)\, p_m
\end{equation}

\subsection*{Connection between PLS and CGA}

We now show that PLS is nothing but CGA on the regression coefficients
applied to the L2 loss as a function of the regression coefficients
$\beta$, with initial condition $\beta_0 = 0$:
\begin{eqnarray*}
    f(\beta) & = & c^T \beta + \frac{1}{2} \beta^T G \beta \\
    c^T & = & - \mathbf{y}^T \mathbf{X} \\
    G & = & \mathbf{X}^T \mathbf{X}
\end{eqnarray*}
The update directions are specified by \eqref{eq:e3p18_p}:
\begin{equation*}
\mathbf{X} p_m  =  \mathbf{z}_{m + 1} = \mathbf{X}_m \mathbf{X}_m^T \mathbf{y}
\end{equation*}
and they are indeed mutually conjugate:
\begin{equation}
p_m \, G \, p_n = p_m \mathbf{X}^T \mathbf{X} \, p_n =
    \mathbf{z}_{m + 1} ^T \mathbf{z}_{n + 1} = 0, \;\; m \neq n
\end{equation}
Moreover, the coefficient of the $\mathbf{z}_m$ term at iteration $m$ of PLS
is the result of a least squares regression of $\mathbf{y}$ against
$\mathbf{z}_m$ itself, which is the equivalent of \eqref{eq:e3p18_update}.

To complete the connection, we show that
$$p_m = -g(\beta_m) + p_{m, \parallel}$$
where $p_{m, \parallel} \in \textrm{Span} \left(p_0, \ldots, p_{m - 1}\right)$,
so that indeed $\textrm{Span} \left(p_0, \ldots, p_m\right) = \textrm{Span} \left(g_0, \ldots, g_m\right)$. We note that:
\begin{eqnarray}
    - \mathbf{X} g(\beta_m) & = & \mathbf{X} \mathbf{X}^T \left(y - \mathbf{X} \beta_m\right) \\
    & = &  \mathbf{X} \mathbf{X}^T \left(1 - P_1 - \ldots - P_m\right) \mathbf{y} \\
    & = & \mathbf{X} \left((1 - P_1- \ldots P_m) \mathbf{X}\right)^T \mathbf{y} \\
    \label{eq:e3p18_passage}
    & = & \mathbf{X} \mathbf{X}_m^T\, \mathbf{y}
\end{eqnarray}
Remember that:
\begin{equation}
\mathbf{X}_m = \mathbf{X}
    - \mathbf{z}_1 \frac{\mathbf{z}_1^T \mathbf{X}}{||\mathbf{z}_1||^2} - \ldots
    - \mathbf{z}_m \frac{\mathbf{z}_m^T \mathbf{X}}{||\mathbf{z}_m||^2}
\end{equation}
Plugging in the previous equation in \eqref{eq:e3p18_passage} we obtain:
\begin{eqnarray*}
    - \mathbf{X} g(\beta_m) & = & \mathbf{X}_m \mathbf{X}_m^T\, \mathbf{y} +
        c_1\, \mathbf{z_1} + \ldots + c_m\, \mathbf{z}_m \\
        & = & \mathbf{z}_{m + 1} + c_1\, \mathbf{z_1} + \ldots + c_m\, \mathbf{z}_m
\end{eqnarray*}
or, in other terms:
\begin{equation*}
\mathbf{X} p_m = - \mathbf{X}\, g(\beta_m) + \mathbf{X} p_{m, \parallel}
\end{equation*}
which completes our proof.


\subsection{Algorithmic}
From an algorithmic point of view, we are calculating each body depending on all the other bodies.
We can traduce that by 2 overlapping \texttt{for-loops} from $0$ to $n$ bodies.
In a \texttt{C-like} language we can express the problem this way:
\begin{lstlisting}[caption={$n$-body pseudo-code algorithm},label={alg:ncorps}]
body b1[N]; // array which contains all the bodies (at t time)
body b2[N]; // empty array
for(unsigned int iBody = 0; iBody < n; ++iBody)
	for(unsigned int jBody = 0; jBody < n; ++jBody)
		if(iBody != jBody)
			b2[iBody] = compute(b1[jBody]);
\end{lstlisting}
Alg. \ref{alg:ncorps} shows an important characteristic of the $n$-body problem class: for $n$ given bodies, the algorithmic complexity in term of computational time is approximatively $O(n^2)$.
It exists other methods to approximate and to resolve the problem in $O(n \log{n} )$ time but this is not in the range of this lab (see {\sc Barnes-Hut} simulation).


\documentclass[twoside]{MATH77}
\usepackage{multicol}
\usepackage[fleqn,reqno,centertags]{amsmath}
\begin{document}
\begmath 12.3 Table Look-up With Hermite Cubic Interpolation

\silentfootnote{$^\copyright$1997 Calif. Inst. of Technology, \thisyear \ Math \`a la Carte, Inc.}

\subsection{Purpose}

Given a table of values of a function and its first derivative, this
subroutine does table look-up and Hermite cubic interpolation to compute an
interpolated value of the function for an arbitrary argument. Optionally the
subroutine can compute values of the first, second, or third derivative of
the interpolation function.  All features here are also available,
with more generality, in Chapter~12.1.

The interpolated function will match the given data at the tabular points,
and thus will have a continuous value and first derivative. In general the
second and third derivatives will have jump discontinuities at the tabular
points. It is possible, of course, for the data to satisfy relations that
give rise to continuity of second or third derivatives in the interpolated
function. In particular, data from the cubic spline fitting programs, DC2FIT
or SC2FIT of Chapter~11.4, will produce continuous second derivatives.

\subsection{Usage}

\subsubsection{Program Prototype, Single Precision}

\begin{description}
\item[INTEGER]  \ {\bf NDERIV, NTAB}

\item[REAL]  \ {\bf SHINT, X, XTAB}($\geq $NTAB){\bf ,\newline
YTAB}($\geq $ NTAB){\bf , YPTAB}($\geq $NTAB){\bf , YVAL}
\end{description}

Assign values to X, NDERIV, NTAB, XTAB(), YTAB(), and YPTAB(). In
particular, NTAB, XTAB(), YTAB(), and YPTAB() may have the same values as on
return from a previous call to SC2FIT.

\begin{center}
\fbox{\begin{tabular}{@{\bf }c}
YVAL = SHINT(X, NDERIV, NTAB,\\
XTAB, YTAB, YPTAB)\\
\end{tabular}}
\end{center}

This reference will set YVAL to the value (or a derivative value as selected
by NDERIV) of the curve defined by NTAB, XTAB(), YTAB(), and YPTAB().

\subsubsection{Argument Definitions}

\begin{description}
\item[X]  \ [in] Abscissa at which interpolation is to be done. If X is
outside the domain of the table given in XTAB(), extrapolation will be done
using the two tabular points at the nearest end of the table.

\item[NDERIV]  \ [in] Set by the user to 0, 1, 2, or~3 to select computation
of the value, or the first, second, or third derivative of the interpolation
function.

\item[NTAB]  \ [in] Number of values in each of the arrays, XTAB(), YTAB(),
and YPTAB(). Require NTAB\ $\geq 2.$

\item[XTAB()]  \ [in] Abscissae of the given data. These values must be
either strictly increasing or strictly decreasing.

\item[YTAB(), YPTAB()]  \ [in] Values and first derivatives associated with
the abscissae given in XTAB().\vspace{-4pt}%
\begin{gather*}
\text{YTAB}(i)=f(\text{XTAB}(i)),\ i=1,...,\text{NTAB}\\
\text{YPTAB}(i)=f^{\prime }(\text{XTAB}(i)),\ i=1,...,\text{NTAB}
\end{gather*}
\item[SHINT]  \ [out] Returns the value or selected derivative value
computed by interpolation.
\end{description}

\subsubsection{Modifications for double precision}

For double-precision usage change the REAL type statements to DOUBLE
PRECISION and change the function name from SHINT to DHINT. Note
particularly that since this is a function subprogram the name DHINT
must be typed either explicitly or via an IMPLICIT statement if it is used.

\subsection{Examples and Remarks}

The program DRSHINT illustrates the use of SHINT. This program first builds
a table of values of the exponential function and its first derivative at
three points, XTAB$()=0$, 0.5, and 0.75. The program DRSHINT then uses SHINT
to interpolate for the value and first derivative at a set of 21~points, $x=0
$ to 1.0 in steps of~0.05. Output is shown in ODSHINT. The first two columns
show $x$ and $\exp (x)$. Columns headed YINT and YPINT contain the
interpolated value and the interpolated first derivative, respectively. The
exponential function was chosen for convenience in this demonstration since
at any point the true function value and all derivatives have the same
value. Note that the function values are approximated more accurately than
are the derivative values. Also note that the approximations are better on
the interval from~0.5 to~0.75 than on the longer interval from~0 to~0.5. For
$x>0.75$ extrapolation is taking place, leading to much larger errors.

Interpolation error bounds are given in Section D. Applying these bounds to
this example, ignoring the O($h^4)$ term in the bound for the first
derivative error, the error bounds are:
\begin{tabbing}
 for $f$ in [0, $0.5] = (0.00260)(1.65)(0.5)^4 = 0.000268$\\
 for $f$ in [0.5, $0.75] = (0.00260)(2.12)(0.25)^4 = 0.000022$\\
 for $f^{\prime}$ in [0, $0.5] = (0.00802)(1.65)(0.5)^3 = 0.00165$\\
 for $f^{\prime}$ in [0.5, $0.75] = (0.00802)(2.12)(0.25)^3 = 0.000266$
\end{tabbing}
Note that these bounds are consistent with the results shown in ODSHINT.

\subsection{Functional Description}

Hermite interpolation together with the error in such interpolation is
described in \cite[pp.~314--317]{Hildebrand:1956:INA}.

SHINT maintains a saved variable, LOOK, with an initial value of~1. The
look-up procedure always starts with the tabular interval indexed by the
saved value of\pagebreak[2] LOOK, unless the saved value is not in
{[1, $\text{NTAB} - 1$]}.
The look-up procedure is a linear search, either forward or backward, from the
starting value of LOOK. This is efficient for cases in which a succession of
look-ups are done for successive arguments that are not far apart relative
to the tabular spacing.

Each interpolation or extrapolation uses function values and first
derivative values at two adjacent tabular points and is exact for
cubic polynomial data.  If the interpolation abscissa, $x$, is equal
to an interior tabular abscissa, say $x_i$, then tabular data
associated with $x_{i-1}$ and $x_i$ will be used for the computation.
Extrapolation uses the first or last two tabular points, as
appropriate.

If the given data are samples from a function having at least a fourth order
derivative, and this derivative is bounded in magnitude by $M_4$ over the
interval $[x_i$, $x_{i+1}]$, of length, $h$, then the error, $E_i$, in
computing the interpolated derivative of order $i$ for any argument in this
interval is bounded as follows:
\begin{alignat*}{3}
\hspace{-8pt}
|E_0| &\leq c_0M_4h^4k,&\ \ c_0 &= 1/384\:&\approx \:&0.00260\\
\hspace{-8pt}
|E_1| &\leq c_1M_4h^3 + O(h^4),&\ \ c_1 &= \sqrt 3/216\:
&\approx \:&0.00802\\
\hspace{-8pt}
|E_2| &\leq c_2M_4h^2 + O(h^3),&\ \ c_2&= 1/12 \:&\approx \:&0.0833\\
\hspace{-8pt}
|E_3| &\leq c_3M_4h + O(h^2),&\ \ c_3 & = 0.5
\end{alignat*}

\bibliography{math77}
\bibliographystyle{math77}

\subsection{Error Procedures and Restrictions}

This subprogram requires NTAB $\geq 2$ and $0 \leq \text{NDERIV} \leq 3$.
The values in the XTAB() array must either be strictly increasing or
strictly decreasing. These conditions are not checked. Violation of these
conditions will cause unpredictable effects.

\subsection{Supporting Information}

The source language is ANSI Fortran~77.

\begin{tabular}{@{\bf}l@{\hspace{5pt}}l}
\bf Entry & \hspace{.15in} {\bf Required Files}\vspace{2pt} \\
DHINT & \parbox[t]{2.7in}{\hspace{.35in} DHINT\rule[-5pt]{0pt}{8pt}}\\
SHINT & \parbox[t]{2.7in}{\hspace{.35in} SHINT}\\
\end{tabular}

Original code designed by C. L. Lawson and R. J. Hanson, JPL, 1968.
Programmed and modified by Lawson, Hanson, T. Lang, and D. Campbell, JPL,
1968--1974. Adapted to Fortran~77 by Lawson and S. Y. Chiu, July~1987.

\begcode

\medskip\
\lstset{language=[77]Fortran,showstringspaces=false}
\lstset{xleftmargin=.8in}

\centerline{\bf \large DRSHINT}\vspace{10pt}
\lstinputlisting{\codeloc{shint}}

\vspace{30pt}\centerline{\bf \large ODSHINT}\vspace{10pt}
\lstset{language={}}
\lstinputlisting{\outputloc{shint}}
\end{document}