\documentclass[11pt]{article}

%%% The preamble loads packages, theorem styles, and macros %%%%%%%%%%%%%%
\input{CourseNotesPreamble}

\title{Math 626: High Dimensional Probability}
\author{Taught by Mark Rudelson \\ Lecture notes by Sayantan Khan}
%\email{schommerpries.chris.math@gmail.com}
%\address{Department of Mathematics \\
%Harvard University \\
%1 Oxford St. \\
%Cambridge, MA 02138}
\date{\today}

\begin{document}

\maketitle
\tableofcontents

\subsection*{About the notes and the course}

These notes only cover the first half of the course, which focused on measure concentration. The second half of the course focused on suprema of random processes, and closely followed chapters 7 and 8 of Roman Vershynin's \emph{High Dimensional Probability} \cite{vershynin2018high}.

\section{Introduction}
\label{sec:introduction}

High dimensional probability is the study of random variables taking values in $\RR^n$ for large but fixed values of $n$.
While this area has always been studied by classical probability theorists, it has also attracted attention from computer scientists, especially since the design and analysis of fast probabilistic algorithms requires tools and theorems from this field.
A classical result that arises from pure mathematics, but has several real life applications is Dvoretzky's theorem, whose statement does not involve any probability at all, and yet the proof uses concentration inequalities.
\begin{theorem}[Dvoretzky's theorem \cite{dvoredsky1961some}]
  Let $X$ be an $n$-dimensional normed vector space. Then for any $\varepsilon > 0$, there exists a constant $c(\varepsilon) > 0$, and a linear subspace $E$ of $X$ such that, $\dim(E) \geq c(\varepsilon) \log(n)$, and for all $v \in E$, the following inequality relating the ambient norm and the Euclidean norm on $E$.
  \begin{align*}
    (1-\varepsilon) \norm{v}_2 \leq \frac{\norm{v}_X}{M(X)} \leq (1+\varepsilon)\norm{x}_2
  \end{align*}
  Where $M(X)$ is a scaling factor that depends only on $X$, and $\norm{\cdot}_X$ and $\norm{\cdot}_2$ are the ambient and Euclidean norm on $E$.
\end{theorem}

\begin{proof}[Idea of proof]
  Pick a random subspace, and then show that with very high probability, the given inequality holds.
  We will prove the result in full detail later in the course.
\end{proof}

\begin{remark}
  \label{rem1}
  When the norm on $X$ is the $\ell^1$ norm, the lower bound on the dimension of $E$ can be improved to be linear in $c(\varepsilon) n$.
\end{remark}

\paragraph{A computer science application of Dvoretzky's theorem}

Consider a subset $S$ of $\ell_2^N$, given by inequalities involving the norms of elements in $\ell_2^N$.
Suppose that we are required to optimize a linear function $f$ on the set $S$.
Since $S$ is given by inequalities involve the $\ell^2$-norm, it will be and intersection of interiors of ellipsoids, and consequently, optimizing $f$ will be computationally expensive.
But we can get around the computational expense by embedding $\ell_2^N$ into $\ell_1^M$, where $M$ is $O(N)$, by Remark \ref{rem1}.
Since this embedding does not distort distances too much, we can replace $S$ with a nearby polytope, given by inequalities involving the $\ell^1$ norm instead.
Optimizing a linear function on a polytope is computationally much easier, thanks to linear programming.

\paragraph{Empirical covariance estimation}

Let $X$ be an $\RR^n$-valued random variable, and let $\E(X) = 0$.
The covariance of $X$ is $\E(X^{\top}X)$, and will be denoted by $\Sigma$.
Let $\left\{ X_1, X_2, \ldots, X_m \right\}$ be i.i.d. samples of $X$.
We define the sample covariance $\Sigma_m$ in the following manner.
\begin{align*}
  \Sigma_n = \frac{\sum_{i=1}^m X_i^{\top}X_i}{m}
\end{align*}
As $m$ tends to infinity, the sample covariance $\Sigma_m$ will approach the true covariance, as we would expect the law of large numbers to predict.
A harder, and more interesting question is to determine how many samples do we need to take to be within some threshold of the true covariance with high probability.

Just like in the scalar setting, one answers the question by proving appropriate concentration inequalities for matrix valued random variables.
Here is the most general set up: Let $\Sigma_m$ and $\Sigma$ be matrices mapping some normed space $X$ to some other normed space $Y$.
We define the distance between $\Sigma_m$ and $\Sigma$ using the operator norm.
\begin{align*}
  \norm{\Sigma_m - \Sigma}_{\mathrm{op}} &= \max_{\norm{v}_x \leq 1} \norm{(\Sigma_m-\Sigma)v}_Y \\
                               &= \max_{\norm{v}_x \leq 1} \max_{\norm{w}_{Y^{\ast}} \leq 1} w\left( (\Sigma_m - \Sigma)v \right) \\
                               &= \max_{\substack{(v, w) \in X \times Y^{\ast} \\ \norm{v} \leq 1 \\ \norm{w} \leq 1}} w((\Sigma_m - \Sigma)v)
\end{align*}
We can consider $w((\Sigma_m -\Sigma)v)$ to be a family of scalar random variables parameterized by points in $X \times Y^{\ast}$, i.e. a random process $V_u$ parameterized by $u \in X \times Y^{\ast}$.
This leads to the following two questions.
\begin{enumerate}[(i)]
\item How to bound $\E \max V_u$.
\item How to bound $\P(\left| \max V_u - \E \max V_u \right| \geq t)$.
\end{enumerate}
It turns out one can often answer (ii) without answering (i), which may seem surprising given that the most elementary concentration inequalities involve moment bounds (i.e. Markov's inequality).
The starting point in answering (ii) is understanding \emph{concentration of measure}.

\section{Concentration of measure}
\label{sec:conc-meas}

Concentration of measure was originally observed Lévy, but first used by Milman in the early 1970s.
Roughly speaking, concentration of measure is the following phenomenon: suppose $(X, d, \P)$ is a metric space endowed with a probability measure and $f: X \to \RR$ is a ``nice'' function.
Then the value of $f$ is essentially constant, i.e. there exists some constant $M(f)$ such that for small enough $\varepsilon$, the following probability bound holds.
\begin{align*}
  \P(|f(x) - M(f)| < \varepsilon) \ll 1
\end{align*}
Usually by nice, we will mean $1$-Lipschitz, although similar results hold for a more general class of functions like convex or quasi-convex functions.
We begin with the simplest version of measure concentration.

\subsection{Concentration for linear functions}
\label{sec:conc-line-funct}

Let the metric space $X$ in this setting be $\R^n$ for some fixed $n$, and let $\{X_1, \ldots, X_n \}$ be i.i.d scalar random variables.
Let $f: \R^n \to \R$ be a linear function, given by taking inner product with some vector $\mathbf{a}$.
We define $Y$ to be $\sum_{i=1}^n a_i X_i$.
We will prove concentration inequalities for $Y$ by imposing conditions on $X_i$: one such condition is requiring $X_i$ to be \emph{subgaussian}.

\subsubsection{Subgaussian random variables}
\label{sec:subg-rand-vari}

\begin{definition}[Subgaussian decay]
  A random variable $X$ is said to be $\sigma$-subgaussian if there exists a constant $C > 0$, such that for all $t > 0$, the following inequality holds.
  \begin{align*}
    \P(|X| > t) \leq C \exp\left( -\frac{1}{2} \left( \frac{t}{\sigma} \right)^2 \right)
  \end{align*}
\end{definition}
\begin{remark}
  The constant $\sigma$ is often referred to as the variance proxy of the subgaussian random variable.
\end{remark}

\begin{example}
  The following random variables are examples of subgaussian random variables.
  \begin{enumerate}[(i)]
  \item Normal random variables
  \item Bounded random variables
  \end{enumerate}
\end{example}

\begin{lemma}
  Let $X$ be a random variable. Then the following are equivalent:
  \begin{enumerate}[(i)]
  \item For all $t> 0$, $\P(|X| > t) \leq C \exp\left( -\frac{1}{2} \left( \frac{t}{\sigma} \right)^2 \right)$ (definition of subgaussian random variables).
  \item There exists $a > 0$ such that $\E(\exp(aX^2)) < \infty$ ($\psi_2$ condition).
  \item There exist $C^{\prime}$ and $b$ such that for all $\lambda \in \R$, $\E(\exp(\lambda X)) \leq C^{\prime} \exp(b \lambda^2)$ (Laplace transform condition).
  \item There exists $K$ such that for all $p \geq 1$, $\E(X^p)^{\frac{1}{p}} \leq K \sqrt{p}$ (moment condition).
  \end{enumerate}
  Moreover, if $\E(X) = 0$, then the constant $C^{\prime}$ in the Laplace transform condition can be taken to be equal to $1$.
\end{lemma}

\begin{proof}
  \begin{description}
  \item[$(i) \implies (ii)$] Using Fubini's theorem and a change of variables, we can express $\E(aX^2)$ as an integral involving tail bounds.
    \begin{align*}
      \E(aX^2) &= 1 + \int_0^{\infty} 2 a t e^{at^2} \cdot \P\left( |X| > t \right) \dd t \\
      &\leq 1 + \int_0^\infty 2Cat\exp\left( at^2 - \frac{1}{2}\left( \frac{t}{\sigma}  \right)^2 \right) \dd t
    \end{align*}
    Clearly, picking a value of $a$ smaller than $\frac{1}{2\sigma^2}$ will make the integral converge.
  \item[$(ii) \implies (iii)$] Since we want to estimate the expectation of $\exp(\lambda X)$, we multiply and divide by $\exp(aX^2)$ and complete the square.
    \begin{align*}
      \exp(\lambda X) = \exp\left(aX^2\right) \cdot \exp\left( \frac{\lambda^2}{4a} \right) \cdot \exp\left( - \left( \sqrt{a}X + \frac{\lambda}{2\sqrt{a}} \right)^2 \right)
    \end{align*}
    Note that the third term in the product is always less than $1$. We now take the expectation of the right hand side.
    \begin{align*}
      \E(\exp(\lambda X)) &= \E\left( \exp\left(aX^2\right) \cdot \exp\left( \frac{\lambda^2}{4a} \right) \cdot \exp\left( - \left( \sqrt{a}X + \frac{\lambda}{2\sqrt{a}} \right)^2 \right) \right) \\
      &\leq \exp\left( \frac{\lambda^2}{4a} \right) \E\left( \exp(aX^2) \right)
    \end{align*}
    Setting $b= \frac{1}{4a}$ and $C^{\prime} = \E(\exp(aX^2))$, we get the result.
  \item[$(iii) \implies (iv)$] We begin by getting a crude estimate for $\E(X^p)$ using the infinite series for $\exp(\lambda X)$.
    \begin{align*}
      \E(X^p) &\leq \frac{p!}{\lambda^p} \E(\exp(\lambda X)) \\
              &= \frac{C^{\prime}p! \exp(b\lambda^2)}{\lambda^p}
    \end{align*}
    Note that this inequality works for all values of $\lambda$, but to get the best inequality, we minimize the right hand side by varying $\lambda$ over $\R$.
    The minimum is attained when $\lambda = \frac{\sqrt{p}}{2b}$: plugging that into the right hand side, and taking $p$\textsuperscript{th} roots gives us the following.
    \begin{align*}
      \E(X^p)^{\frac{1}{p}} \leq C^{\prime \prime} \frac{(p!)^{\frac{1}{p}}}{\sqrt{p}}
    \end{align*}
    Here, we have absorbed all the constants into $C^{\prime \prime}$.
    Using Stirling's approximation, the numerator is bounded above by $p$, giving us the inequality we want.
  \item[$(iv) \implies (i)$] We rewrite the event $|X| > t$ in the following manner.
    \begin{align*}
      \P(|X| > t) &= \P(\exp(\lambda X^2) > \exp(\lambda t^2)) \\
                  &\leq \exp(-\lambda t^2) \E(\exp(\lambda X^2))
    \end{align*}
    Here, $\lambda$ is a positive real number that we will specify later, and the inequality comes from Markov's inequality.
    Of course, we do not a priori know that $\E(\lambda X^2)$ is finite, but we will pick a $\lambda$ small enough such that it is.
    Using Fubini's theorem, we can express $\E(\exp(\lambda X^2))$ in the following manner.
    \begin{align*}
      \E(\exp(\lambda X^2)) = 1 + \frac{\lambda \E(X^2)}{1!} + \frac{\lambda^2 \E(X^4)}{2!} + \cdots
    \end{align*}
    Using the bound on the moments of $X$ and Stirling's approximation, we get the following inequality.
    \begin{align*}
      \E(\exp(\lambda X^2)) &\leq \sum_{p=0}^{\infty} \frac{(2\lambda K^2p)^p}{p!} \\
                            &\leq \sum_{p=0}^{\infty} \frac{(2\lambda e K^2 p)^p}{\sqrt{2\pi p} p ^p}
    \end{align*}
    If we pick $\lambda$ to be small enough that $2e \lambda K^2$ is much smaller than $1$, then the infinite sum converges, and the expectation is finite.
    Setting $\frac{1}{2\sigma^2}$ to be equal to $\lambda$ gives us the result.
  \end{description}
  We now show that the constant in the Laplace transform condition can be set to be $1$ when $\E(X) = 0$.
  To do so, we recall the $\psi_2$ and the Laplace transform condition, i.e. there exist constants $a$, $C^{\prime}$ and $b$ such that the following two inequalities hold for all $\lambda \in \R$.
  \begin{align}
    \label{eq:1}
    \E(\exp(aX^2)) &< \infty \\
    \E(\exp(\lambda X)) & \leq C \exp(b\lambda^2)
  \end{align}
  Suppose now that $\lambda^2 > 2a$.
  By the Laplace transform condition, we have the following inequality.
  \begin{align*}
    \E(\exp(2aX)) &\leq C^{\prime} \exp(4ba^2) \\
                  &= \exp(4a^2 b^{\prime})
  \end{align*}
  Where $b^{\prime}$ is $b + \frac{\log(C^{\prime})}{4a^2}$.
  Since $b^{\prime}$ decreases as $a$ increases, for any $\lambda^2 > 2a$, $\E(\exp(aX^2))$ will be less than $\exp(b^{\prime} \lambda^2)$.

  Now suppose that $\lambda^2 < 2a$.
  We begin by considering the special case where $X$ is a symmetric random variable.
  By symmetry of $X$, we have the following inequality.
  \begin{align*}
    \exp(\lambda X) &= \frac{\exp(\lambda X) + \exp(-\lambda X)}{2} \\
            &\leq \exp\left( \frac{\lambda^2 X^2}{2} \right)
  \end{align*}
  Taking expectations on both sides gives us the following.
  \begin{align*}
    \E(\exp(\lambda X)) \leq \E\left(\exp\left( \frac{\lambda^2}{2} X^2 \right)\right)
  \end{align*}
  Since $\lambda^2 < 2a$, $\frac{2a}{\lambda^2}$ is greater than $1$, and we can use Hölder's inequality to bound the right hand term.
  \begin{align*}
    \E\left(\exp\left( \frac{\lambda^2}{2} X^2 \right)\right) \leq
    \left( \E\left(\exp\left(   aX^2   \right) \right) \right)^{\frac{\lambda^2}{2a}}
  \end{align*}
  Since $\E(\exp(aX^2))$ is a finite constant, the right hand side is $\exp(b^{\prime \prime} \lambda^2)$ for some constant $b^{\prime \prime}$.

  Now suppose $X$ is not symmetric.
  Let $X^{\prime}$ be an identical independent copy of $X$.
  Since $\E(X^{\prime})$ is $0$, we have the following equality.
  \begin{align*}
    \E(\exp(\lambda X)) = \E(\exp(\lambda (X - \E(X^{\prime}))))
  \end{align*}
  Since $\exp$ is a convex function, we can pull out the inner expectation, using Jensen's inequality.
  \begin{align*}
    \E(\exp(\lambda X)) \leq \exp(\lambda(X - X^{\prime}))
  \end{align*}
  Since $X - X^{\prime}$ is symmetric, the result follows from the previous part, and the proof is complete.
\end{proof}
We now explain why care so much about the constant in the Laplace transform condition.

\begin{theorem}[Hoeffding-Chernoff-Azuma inequality]
  \label{thm:hoeffding}
  Let $\{X_1, \ldots, X_n\}$ be i.i.d. subgaussian random variables with mean $0$.
  Then for any $(a_1, \ldots , a_n) \in \R^n$ and any $t > 0$, the following probability bound holds.
  \begin{align*}
    \P\left( \left| \sum_{i=1}^n a_i X_i \right| > t \right) \leq C \exp\left( -c \frac{t^2}{\norm{\mathbf{a}}_2} \right)
  \end{align*}
  Where $C$ and $c$ are some absolute constants.
\end{theorem}
\begin{proof}
  Without loss of generality, we can assume $\norm{\mathbf{a}}_2 = 1$.
  It will suffice to show that the sum of subgaussian random variables is subgaussian.
  We will show it verifying the Laplace transform condition.
  Let $\lambda \in \R$, and let $Y = \sum_{i=1}^n a_iX_i$.
  We compute $\E(\exp(\lambda Y))$.
  \begin{align*}
    \E(\exp(\lambda Y)) &= \prod_{i=1}^n \E(\exp(\lambda a_i X_i)) \\
                        &\leq \prod_{i=1}^n \E(\exp(b \lambda^2 a_i^2)) \\
                        &= \E(\exp(b\lambda^2))
  \end{align*}
  This proves the result. Note that having the Laplace transform coefficient equal to $1$ helped, because if the coefficient $C$ was greater than $1$, then we would pick up a constant of $C^n$, which would be very large for large values of $n$.
\end{proof}

To see how this inequality is used in practice, consider the simplest possible example of a subgaussian random variable, a random variable that takes values $1$ and $-1$ with probability $\frac{1}{2}$.
% A Rademacher random variable is a random variable that takes values $1$ and $-1$ with probability $\frac{1}{2}$.

Let's recall some elementary facts about Fourier analysis before stating the example.
Let $L^2([0,1])$ be the space of complex $L^2$ functions on $[0,1]$ and let $\{e_n\}_{n \in \Z}$ be the standard Fourier basis, i.e. $e_n(t) = \exp(2\pi i nt)$.
Since $\{e_n\}$ forms an orthonormal basis, any function $f \in L^2([0,1])$ can be decomposed into its Fourier series.
\begin{align*}
  f = \sum_{n \in \Z} \widehat{f}(n) e_n
\end{align*}
The Fourier coefficient $\widehat{f}(n)$ is given by $\int_0^1 f(t) \overline{e_n(t)} \dd t$.
The map sending $f$ to its Fourier coefficients is a linear isometry from $L^2([0,1])$ to $\ell^2(\Z)$.
For most sequences $\{\widehat{f}(n)\}$ in $\ell^2(\Z)$, the associated function $f$ will not be continuous, but we will show that under reasonably mild conditions on $\{\widehat{f}(n)\}$, $f$ can be made to be continuous.

Let $\varepsilon_n \in \{-1, 1\}$ for $n \in \Z$, and let $\varepsilon = \{\varepsilon_n\}_{n \in \Z}$.
For any $f \in L^2([0,1])$ and any such $\\varepsilon$, define $f_{\varepsilon}$ to be the following function.
\begin{align*}
  f_{\varepsilon} = \sum_{n \in Z} \varepsilon_n \widehat{f}(n) e_n
\end{align*}
We then have the following theorem.
\begin{theorem}
  \label{thm:continuous-l2-function}
  Let $f$ be a function in $L^2([0,1])$ whose Fourier coefficients satisfy the following inequality.
  \begin{align*}
    % \label{eq:convergence-condition-example}
    \sum_{n \in \Z} \left( \log(|n| + 1) \right)^3 \left| \widehat{f}(n) \right|^2 < \infty
  \end{align*}
  Let $\{\varepsilon_n\}$ be a sequence of i.i.d random variables taking values in $1$ and $-1$ with probability $\frac{1}{2}$.
  Then $f_{\varepsilon}$ is a continuous function with probability $1$.
\end{theorem}

To prove the theorem, we will need several lemmas.

\begin{lemma}
  \label{lem:main-technical-lemma}
  Let $N \in \N$.
  Then for any $\{a_n\}_{n=-N}^N$, the following probability bound holds.
  \begin{align*}
    \P\left( \norm{ \sum_{n=-N}^N \varepsilon_n a_n e_n }_{\infty} >  C \sqrt{\log(N)} \left( \sum_{n=-N}^N |a_n|^2 \right)^{\frac{1}{2}} \right) \leq \frac{1}{N^2}
  \end{align*}
  Here the constant $C$ is absolute, i.e. independent of $N$.
\end{lemma}
\begin{proof}
  The first step is to consider the maximum, not over all of $[0,1]$, but over the points $\left\{ \frac{j}{N^2} \right\}_{j = 0}^N$.
  It will suffice to bound the probability for any given point by $\frac{1}{N^4}$, and then use the union bound to get the desired inequality.
  Since $\varepsilon_n$ is a subgaussian random variable with mean $0$, by Hoeffding-Chernoff-Azuma inequality, we get the following bound for any $t \in [0,1]$.
  \begin{align*}
    \P\left( \norm{ \sum_{n=-N}^N \varepsilon_n a_n e_n }_{\infty} >  C \sqrt{\log(n)} \left( \sum_{n=-N}^N |a_n|^2 \right)^{\frac{1}{2}} \right) \leq C^{\prime} \exp(-cC^2 \log(n))
  \end{align*}
  We can pick a $C$ large enough so that the right hand side is bounded above by $\frac{1}{N^4}$, and that concludes the first step after we use the union bound.
  Note that in order to do this, we really needed something stronger than Chebyshev's inequality, since we needed an upper bound that can be made smaller than $\frac{1}{N^4}$.

  The next step is to extend the argument to all of $[0,1]$.
  The key trick here will be to estimate the maximum value of trigonometric polynomials away from points of the form $\frac{j}{N^2}$ using the maximum we derived in step $1$.
  Bernstein's inequality for trigonometric polynomial helps in this regard: given a trigonometric polynomial $p$ of degree $n$, the following inequality relates the $\norm{\cdot}_{\infty}$-norm of $p^{\prime}$ and $p$.
  \begin{align*}
    \norm{p^{\prime}}_{\infty} \leq n \norm{p}_{\infty}
  \end{align*}
  Let $V$ be the maximum value of the trigonometric polynomial $p = \sum_{n=-N}^N \varepsilon_n a_n e_n$ achieves on points of the form $\frac{j}{N^2}$, and let $W$ be the maximum value over all of $[0,1]$, and say it is achieved at some point $t$, and let $s$ be the closest point of the form $\frac{j}{N^2}$.
  Then, by the mean value theorem, we get the following relation between $V$ and $W$.
  \begin{align*}
    W &\leq V + \norm{p^{\prime}}_{\infty} |t-s| \\
      &\leq V + \frac{N \norm{p}_{\infty}}{N^2} \\
      &= V + \frac{W}{N}
  \end{align*}
  This means for $N > 1$, $W \leq 2V$, and this proves the lemma.
\end{proof}

\begin{proof}[Proof of Theorem \ref{thm:continuous-l2-function}]
  For $M \in \N$, define the function $f_{M, \varepsilon}$ in the following manner.
  \begin{align*}
    f_{M, \varepsilon} = \sum_{2^M \leq |n| < 2^{M+1}} \varepsilon_n \widehat{f}(n) e_n
  \end{align*}
  We now use Lemma \ref{lem:main-technical-lemma} with $N = 2^{M+1}$, $a_n = 0$ for $|n| < 2^M$, and $a_n = \widehat{f}(n)$ for $2^M \leq n < 2^{M+1}$.
  \begin{align*}
    \P\left( \norm{f_{M,\varepsilon}}_{\infty} > C \sqrt{M} \norm{f_{M,\varepsilon}}_2 \right) < \frac{1}{2^{2(M+1)}}
  \end{align*}
  % \confused{It seems something is going wrong here because the sum of probability upper bound converges too easily, which means one might be making a mistake, or strengthen the result significantly.}
  By the Borel-Cantelli lemma, for almost every $\varepsilon$, $\norm{f_{M, \varepsilon}}_{\infty}$ eventually becomes smaller than $C\sqrt{M} \norm{f_{M,\varepsilon}}_2$.

  Pick $\varepsilon$ to be one of the instances where the above described situation does happen.
  We have that $f$ is an infinite sum of continuous functions.
  \begin{align*}
    f = \varepsilon_0 \widehat{f}(0) e_0 + \sum_{M=0}^{\infty} f_{M, \varepsilon}
  \end{align*}
  This will converge to a continuous function if the sequence of partial sums is uniformly Cauchy.
  To see that is indeed the case, pick $K_1$ and $K_2$ larger than the threshold $M$ after which $\norm{f_{M, \varepsilon}} < C\sqrt{M}$.
  \begin{align*}
    \norm{f_{K_2, \varepsilon} - f_{K_1, \varepsilon}}_{\infty} &\leq \sum_{M=K_1}^{K_2} \norm{f_{M, \varepsilon}}_{\infty} \\
                                                                &\leq \sum_{M=K_1}^\infty C \sqrt{M} \norm{f_{M, \varepsilon}}_2 \\
                                                                &\leq C \left( \sum_{M=K_1}^{\infty} \left( \frac{1}{M} \right)^2 \right)^{\frac{1}{2}} \left( \sum_{M=K_1}^{\infty} M^3 \norm{f_{M, \varepsilon}}_2^2 \right)^{\frac{1}{2}} \\
                                                                &\leq \frac{C^{\prime}}{K_1} \left( \sum_{n \in \Z} (C^{\prime \prime} \log(|n| + 1))^3 \widehat{f}(n)^2 \right) \\
    &\leq \frac{C^{\prime\prime\prime}}{K_1}
  \end{align*}
  The upper bound goes to $0$ as $K_1$ goes to $\infty$, which shows the sequence is uniformly Cauchy, and thus the limit is a continuous function.
\end{proof}

We now prove a moment bound for sums of subgaussian random variables.
\begin{theorem}[Khintchine's inequality \cite{khintchine1923dyadische}]
  Let $\{X_1, \ldots, X_n\}$ be i.i.d subgaussian random variables with $\E(X_i) = 0$ and $\E(X_i^2) = 1$.
  Then for any $p \in [1, \infty)$, there exist constants $A_p$ and $B_p$ greater than $0$ such that for any vector $\mathbf{a} \in \R^n$, the following moment bound holds.
  \begin{align*}
    A_p \norm{\mathbf{a}}_2 \leq \left( \E \left| \sum_{i=1}^n a_i X_i \right|^p \right)^{\frac{1}{p}} \leq  B_p \norm{\mathbf{a}}_2
  \end{align*}
\end{theorem}
\begin{proof}
  Consider first the case where $p > 2$.
  Then, using Hölder's inequality for the convex function $x \mapsto x^{\frac{p}{2}}$, we get the following.
  \begin{align*}
\left( \E \left| \sum_{i=1}^n a_i X_i \right|^2 \right)^{\frac{1}{2}} \leq \left( \E \left| \sum_{i=1}^n a_i X_i \right|^p \right)^{\frac{1}{p}}
  \end{align*}
  This shows that for $p > 2$, we can set $A_p = 1$.
  To get $B_p$, we use the moment condition on subgaussian random variables.
  Since the sum of subgaussian random variables is subgaussian, we have that $Y = \sum_{i=1}^n a_i X_i$ is subgaussian, and thus satisfies the moment bound.
  We have seen that the absolute constant $K$ will only depend on $\norm{\mathbf{a}}_2$, giving us the upper bound.
  \begin{align*}
    \E(|Y|^p)^{\frac{1}{p}} \leq K \sqrt{p} \norm{\mathbf{a}}_2
  \end{align*}
  Setting $B_p = K\sqrt{p}$ proves the result in this case.

  For $p < 2$, it will suffice to prove it for $p = 1$, since the $p$\textsuperscript{th} moment of $|Y|$ is an increasing function of $Y$ and bounded above by $\norm{\mathbf{a}}_2$ by Hölder's inequality, which means we can set $B_p = 1$ (or the previous argument will also work, but $B_p = 1$ is better than $B_p = K\sqrt{p}$).
  Thus we just need to show the lower bound for $p = 1$.
  In this case, the inequality follows from Cauchy-Schwartz.
  \begin{align*}
    \E(|Y|^2) &\leq \sqrt{\E(|Y|) \E(|Y|^3)}
  \end{align*}
  Using Khintchine's inequality for $p=3$, we deal with $\E(|Y|^3) \leq B_3 \norm{\mathbf{a}}_2$.
  Squaring both sides, we see that $A_1 = B_3^{-3}$ works, and the proof is complete.
\end{proof}
There is a far reaching generalization of Khintchine's inequality, due to Kahane.
\begin{theorem}[Kahane inequality \cite{kahane1964sommes}]
  Let $X$ be a normed vector space, and let $\{\varepsilon_1, \ldots, \varepsilon_n\}$ be i.i.d. Rademacher random variables.
  For any $p \in [1, \infty)$, there exists $A_p$ and $B_p$ greater than $0$ such that for any $\{a_1, \ldots, a_n\}$ in $X$, the following holds.
  \begin{align*}
    A_p \left( \E \norm{\sum_{j=1}^n \varepsilon_j a_j}^2_X \right)^{\frac{1}{2}} \leq \left( \E \norm{\sum_{j=1}^n \varepsilon_j a_j}^p_X \right)^{\frac{1}{p}} \leq B_p \left( \E \norm{\sum_{j=1}^n \varepsilon_j a_j}^2_X \right)^{\frac{1}{2}}
  \end{align*}
\end{theorem}
The proof of this inequality requires more machinery than the previous result, so we'll defer the proof until we have developed the required tools.

Recall that we got strong tail decay for bounded random variables using Hoeffding-Chernoff-Azuma inequality, since they're subgaussian, but it turns out, we can do much better than that using boundedness.

\begin{theorem}[Bennett's inequality]
  \label{thm:bennett}
  Let $\{X_1, \ldots, X_n\}$ be i.i.d random variables satisfying the following properties.
  \begin{enumerate}[(i)]
  \item $\norm{X_j}_{\infty} \leq 1$.
  \item $\E(X_j) = 0$ and $\E(X_j^2) = \delta$.
  \end{enumerate}
  Then for any $\mathbf{a} \in \R^n$, we have the following tail decay estimate.
  \begin{align*}
    \P\left( \left| \sum_{j=1}^n a_j X_j \right| > t \right) \leq
    \begin{cases}
      2 \exp \left(- \frac{t^2}{2e\delta \norm{\mathbf{a}}_2^2} \right)\text{ for $t \leq t_{\ast}$} \\
      2 \exp \left( - \frac{t}{4 \norm{\mathbf{a}}_{\infty}} \cdot \log\left( \frac{t \norm{\mathbf{a}}_{\infty}}{\delta \norm{\mathbf{a}}_2^2} \right) \right)\text{ for $t > t_{\ast}$}
    \end{cases}
  \end{align*}
  Here $t_{\ast} = e\delta \norm{\mathbf{a}}_2^2$.
\end{theorem}
Before we start to prove the theorem, let's show why the described tail bound is a very natural upper bound to consider.
Consider $\mathbf{a} = (1, 1, \ldots, 1)$.
Then $\sum a_j X_j$ is approximately $\cN(0, \delta \norm{\mathbf{a}}^2_2) = \cN(0, \delta n)$.
The tail should then behave something like $\exp \left( - \frac{t^2}{2\delta n} \right)$, which is precisely the first case in the upper bound.
If $\delta$ is small, i.e. $\delta n$ is bounded above by some constant $\lambda$, then the central limit theorem asymptotic does not apply, but rather the Poisson limit theorem asymptotic applies.
\begin{align*}
  \P\left( \sum_{j=1}^n a_j X_j > t \right) &\sim \sum_{j=t}^{\infty} e^{-\lambda} \frac{\lambda^j}{j!} \\
                                            &\sim e^{-\lambda} \frac{\lambda^j}{j!} \\
                                            &\sim \exp\left( -t \log\left( \frac{t}{e\lambda} \right) \right)
\end{align*}
Contrast this with the second case of the tail in Bennett's inequality.

\begin{proof}[Proof of Theorem \ref{thm:bennett}]
  Without loss of generality, assume that $\norm{\mathbf{a}}_{\infty} = 1$.
  Set $Y = \sum_{j=1}^n a_j X_j$, and let $\lambda > 0$.
  We then estimate the Laplace transform of $Y$.
  \begin{align*}
    \E\left( \exp(\lambda Y) \right) &= \prod_{j=1}^n \E\left(\exp\left(\lambda a_j X_j \right) \right)
  \end{align*}
  We use an elementary inequality to estimate the Laplace transform.
  \begin{align*}
    e^x \leq 1 + x + \frac{x^2}{2} e^{|x|}
  \end{align*}
  Thus, we have the following.
  \begin{align*}
    \E\left( \exp\left( \lambda a_j X_j \right) \right) &\leq \E\left(1 + \lambda a_j X_j + \frac{\lambda^2 a_j^2 X_j^2}{2} \exp\left( \left| \lambda a_j X_j \right| \right)\right) \\
    &\leq 1 + 0 + \frac{\lambda^2 a_j^2 \delta}{2} e^{\lambda}
  \end{align*}
  Putting all of it back together, we have the following inequality.
  \begin{align*}
    \E\left( \exp\left( \lambda Y \right) \right) &\leq \prod_{j=1}^n \left( 1 + \frac{\lambda^2 a_j^2 \delta e^{\lambda}}{2} \right) \\
                                                  &\leq \prod_{j=1}^n \exp\left( \frac{\lambda^2 a_j^2 \delta e^{\lambda}}{2} \right) \\
                                                  &= \exp\left( \frac{\delta\lambda^2 e^{\lambda}}{2} \norm{\mathbf{a}}_2^2 \right)
  \end{align*}
  Now that we have the Laplace transform estimate, we can use it to estimate tail probabilities.
  \begin{align*}
    \P(Y > t) &= \P(\exp(\lambda Y) > e^{\lambda t}) \\
              &\leq \frac{\exp\left( \frac{\delta\lambda^2 e^{\lambda}}{2} \norm{\mathbf{a}}_2^2 \right)}{e^{\lambda t}} \\
              &= \exp\left( -\lambda t + \frac{\delta\lambda^2 e^{\lambda}}{2} \norm{\mathbf{a}}_2^2 \right)
  \end{align*}

  The next step is to optimize this inequality as we vary $\lambda$.
  We begin by optimizing in the region $\lambda \leq 1$.
  In this case, the optimum $\lambda$ is $\frac{t}{e\delta \norm{\mathbf{a}_2^2}}$.
  Plugging in this value of $\lambda$ gives the first case of the upper bound, and this is valid for $t \leq e\delta \norm{\mathbf{a}}_2^2 = t_{\ast}$.

  In the region $\lambda > 1$, we use the inequality $\lambda \leq e^{\lambda}$.
  \begin{align*}
    \P(Y > t) &\leq \exp\left( -\lambda t + \frac{\delta\lambda^2 e^{\lambda}}{2} \norm{\mathbf{a}}_2^2 \right) \\
              &\leq \exp\left( -\lambda t + \frac{\delta\lambda e^{2\lambda}}{2} \norm{\mathbf{a}}_2^2 \right)
  \end{align*}
  Choose $\lambda$ such that $\lambda \norm{\mathbf{a}}_2^2 e^{2\lambda} = t$. Plugging that value in, we get the second case.

  Finally, doing this for $-t$ gives similar bounds, and we combine the two using union bound to get the claimed result. This completes the proof.
\end{proof}

The theorems in this section illustrate how strong the condition of being subgaussian is, especially when taking sums of i.i.d copies of subgaussians.
In the next section, we will investigate another similar tail decay condition.

\subsubsection{Subexponential random variables}
\label{sec:subexp-rand-vari}

\begin{definition}[Subexponential random variable]
  A random variable $X$ is said to be $k$-subexponential if for all $t > 0$, the following holds.
  \begin{align*}
    \P\left( |X| > t \right) \leq 2 \exp\left(- \frac{t}{k} \right)
  \end{align*}
\end{definition}

Just like in the case of subgaussian random variables, we have a number of equivalent definitions of subexponential random variables.

\begin{lemma}
  Let $X$ be a random variable. Then the following conditions are equivalent.
  \begin{enumerate}[(i)]
  \item $X$ is $k$-subexponential.
  \item There exists a $b > 0$ such that $\E\left( \exp(b|X|) \right) \leq 2$ ($\psi_1$ condition).
  \item For all $p \geq 1$, $\E\left( |X|^p \right)^{\frac{1}{p}} \leq Cp$ (moment condition).
  \end{enumerate}
  Moreover, if $\E(X) = 0$, there exists $\lambda_0 > 0 $ such that for all $|\lambda| < \lambda_0$, $\E \left( \exp(\lambda X) \right) \leq \exp\left( \widetilde{C}\lambda^2 \right)$.
\end{lemma}

\begin{proof}
  % \begin{description}
  % \item[$(i) \implies (ii)$] Using the integrated tail probability expectation formula, we can express $\E\left( \exp(b|X|) \right)$ as the following integral.
  %   \begin{align*}
  %     \E\left( b|X| \right) &= 1 + \int_0^\infty b\exp(bt) \cdot \P\left( |X| > t \right) \dd t \\
  %                           &\leq 1 + \int_0^{\infty} b \exp\left( bt - \frac{t}{k} \right) \dd t
  %   \end{align*}
  %   Clearly, picking a small enough $b$ makes the right hand side converge to a number smaller than $2$.
  % \item[$(i) \implies (ii)$]
  % \end{description}
  The proofs of the three equivalences are similar in spirit to the versions for subgaussians, so we will skip the proof, and just prove the moreover part.

  Writing $\E(\exp(\lambda X))$ as an infinite sum of expectations, we get the following chain of inequalities for a small enough value of $\lambda$.
  \begin{align*}
    \E(\exp\left( \lambda X \right)) &= 1 + \E(\lambda X) + \sum_{j=2}^{\infty} \frac{\lambda^j \E(X^j)}{j!} \\
                                     &\leq 1 + 0 + \sum_{j=2}^{\infty} \frac{\left( \lambda C j \right)^j}{j!} \\
                                     &\leq 1 + \sum_{j=2}^{\infty} (\lambda C e)^j \\
                                     &= 1 + \frac{(\lambda C e)^2}{1 - \lambda C e}
  \end{align*}
  We get the first inequality from the moment condition, the second from Stirling's approximation, and the third follows from geometric convergence for $\lambda C e < 1$.
  Note now that if $\lambda C e < \frac{1}{2}$, we get the following inequalities.
  \begin{align*}
    1 + \frac{(\lambda C e)^2}{1 - \lambda C e} &\leq 1 + 2C^2e^2\lambda^2 \\
    &\leq \exp\left( 2C^2e^2 \lambda^2 \right)
  \end{align*}
  This proves the result.
\end{proof}

We can now prove a strong tail bound for sums of subexponential random variables like we did in Hoeffding-Chernoff-Azuma inequality.

\begin{theorem}[Bernstein's inequality]
  \label{thm:bernstein}
  Let $\{X_1, \ldots, X_n\}$ be i.i.d subexponential random variables with mean $0$, and let $\mathbf{a}$ be a vector in $\R^n$. Then the following holds.
  \begin{align*}
    \P\left( \left| \sum_{j=1}^n a_j X_j \right| > t \right) \leq
    2 \exp\left( -c \left( \frac{t^2}{\norm{\mathbf{a}}_2^2} \wedge  \frac{t}{\norm{\mathbf{a}}_\infty}\right) \right)
  \end{align*}
  Here $a \wedge b$ denotes the minimum of $a$ and $b$.
\end{theorem}
\begin{remark}
  The tail of a sum of i.i.d random variables behaves very much like the situation described above.
  When $t$ is small, the tail behave like a subgaussian, and when $t$ is large, the tail behaves like a subexponential random variable.
\end{remark}

\begin{proof}[Sketch of proof of Theorem \ref{thm:bernstein}]
  Like with all the other sum tail bounds, the proof of this theorem is via bounding the Laplace transform of the random variable.
  For small $\lambda$, we have the upper bound to be $\exp(\widetilde{C}\lambda^2)$ and then we optimize over $\lambda$, and for large $t$, we use Markov's inequality.
\end{proof}

\subsubsection{Applications of subgaussian and subexponential random variables}
\label{sec:appl-subg-subexp}

Before we list some of the applications, we make a remark on why the conditions for subexponential and subgaussian random variables were called $\psi_1$ and $\psi_2$ conditions respectively.
Let $\alpha$ be a number greater than $0$.
Define a function $\psi_{\alpha}$ in the following manner.
\begin{align*}
  \psi_{\alpha}(x) \coloneqq \exp(x^\alpha) - 1
\end{align*}
Using this function, we can define norms on random variables.
\begin{align*}
  \norm{X}_{\psi_{\alpha}} \coloneqq \inf \left( K > 0 \mid \E\left( \psi_{\alpha} \left( \frac{|X|}{K} \right) \right) \leq 1 \right)
\end{align*}
With this norm, the space of subexponential and subgaussian random variables form Banach spaces with respect to $\norm{\cdot}_{\psi_1}$ and $\norm{\cdot}_{\psi_2}$ respectively. These Banach spaces are often known as Orlicz spaces.

Our first application of Hoeffding-Chernoff-Azuma and Bernstein's inequality will be the Johnson-Lindenstrauss lemma\footnote{While this lemma was only a small, and rather easy, part of a hard technical paper of Johnson and Lindenstrauss, the lemma is (supposedly) one of the most cited lemmas in computer science.}.

\begin{theorem}[Johnson-Lindenstrauss lemma \cite{johnson1984extensions}]
  \label{thm:johnson-lindenstrauss}
  Let $F \subset \R^N$ be a finite set.
  Then for any $\varepsilon > 0$, there exists a linear mapping $\varphi: F \to \R^n$ with $n \leq \frac{C}{\varepsilon^2} \log(\#F)$ such that the mapping does not distort distances too much, i.e. the following inequalities hold for all $x$ and $y$ in $F$.
  \begin{align*}
    (1-\varepsilon) \norm{x-y}_2 \leq \norm{\varphi(x) - \varphi(y)}_2 \leq (1+\varepsilon) \norm{x-y}_2
  \end{align*}
\end{theorem}

  This is useful for computer scientists because it allow dimension reduction.
  Often, one has finitely many vectors in a very high dimensional space, and one only cares about their metric structure.
  This theorem allows one to reduce the ambient dimension significantly while not distorting the metric structure too much, and furthermore, the new ambient dimension is $O(\log \#F)$, whereas creating a metric graph would involve $O(\#F^2)$ computations.

While the $\ell^2$ norm is natural for geometry, in computer science applications, the $\ell^1$ norm is preferred, because of its relation to linear programming. So a natural follow up question is whether one can perform similar dimension reduction in $\ell^1$ instead.
This was open for a long time, but recently shown to be impossible (see \cite{10.1145/1089023.1089026}) in a very strong way.
It was shown that in order to have at most $\varepsilon$ distortion in the $\ell^1$ distance, the ambient dimension would be at least $C \# F$, i.e. linear in the size of the dataset, rather than logarithmic. The original proof is this fact was quite long and non-trivial, but Lee and Naor soon gave a simpler proof (see \cite{lee2004embedding}) that relied on some highly non-trivial functional analysis. Johnson and Naor also characterized Banach spaces that allow strong dimension reduction, and it turns out those spaces are quite similar to Hilbert spaces (see \cite{2008arXiv0807.1919J}).

\begin{proof}[Proof of Theorem \ref{thm:johnson-lindenstrauss}]
  Define a set $V \subset \R^n$ in the following manner.
  \begin{align*}
    V = \left\{ \frac{x-y}{\norm{x-y}_2} \mid \{x, y\} \subset F\text{ and }x \neq y \right\}
  \end{align*}
  We will work with this set $V$ instead.
  $V$ is contained in $S^N$, and has cardinality $\frac{\#F^2 - \#F}{2}$.

  Let $G$ be an $n \times N$ matrix with i.i.d subgaussian entries $g_{ij}$ satisfying the following two properties.
  \begin{align*}
    \E(g_{ij}) &= 0 \\
    \E(g_{ij}^2) &= 1
  \end{align*}
  Fix a point $v \in V$ and let $n = \frac{\theta}{\varepsilon^2} \log(\#F)$, where $\theta$ a parameter we'll pick later.
  We will prove that the following inequality holds with high probability.
  \begin{align*}
    \left| \norm{Gv}_2 - \sqrt{n} \right| \leq \varepsilon
  \end{align*}
  If the probability is high enough, we can do this for all elements of $V$ simultaneously using the union bound.

  Let $i \in \{1, \ldots, n\}$. Then we define $Y_i$ to be the $i$\textsuperscript{th} entry of $Gv$.
  \begin{align*}
    Y_i = \sum_{j=1}^N g_{ij}v_i
  \end{align*}
  We then compute the first and second moments of $Y_i$.
  \begin{align*}
    \E(Y_i) &= 0 \\
    \E(Y_i^2) &= \sum_{j=1}^{n} v_i^2 \E(g_{ij}^2) \\
            &= 1
  \end{align*}
  The random variable $Y_i$ is a linear combination of subgaussian random variable, and thus is also subgaussian. Also, as $i \neq i^{\prime}$, $Y_i$ and $Y_{i^{\prime}}$ are i.i.d.

  Let us now get estimates on $\norm{Gv}_2^2 - n$.
  \begin{align*}
    \norm{Gv}_2^2 - n &= \sum_{i=1}^n Y_i^2 - n \\
                      &= \sum_{i=1}^n (Y_i^2 - 1)
  \end{align*}
  Let $Z_i = Y_i^2 - 1$. Then $\E(Z_i) = 0$, and $Z_i$ are subexponential. This is a consequence of the fact that squares of subgaussian random variables are subexponential, which can be checked using the tail bound, or the moment condition.

  We use Bernstein's inequality to control the tail of the sum of $Z_i$.
  \begin{align*}
    \P\left( \left| \norm{Gv}_2^2 - n \right| > t \right)
    &= \P\left( \left| \sum_{i=1}^n Z_i \right| > t \right) \\
    &\leq 2\exp\left( -c \left( \frac{t^2}{n} \wedge t \right) \right)
  \end{align*}
  Set $t = \varepsilon n$, where $\varepsilon < 1$. In this regime, $t^2 < t$, which means the tail bound is $2 \exp\left( -c\frac{t^2}{n} \right)$.

  We now use the union bound to bound the probability that some $v \in V$ violates this condition.
  \begin{align*}
    \P\left( \exists\ v \in V \mid \left| \norm{Gv}_2^2 - n \right| > \varepsilon n \right)
    &\leq 2 \exp\left( -c \varepsilon^2 n \right) \cdot \#F \\
    &= 2 \exp\left( -c \varepsilon^2 \frac{\theta}{\varepsilon^2} \log(\#F) + \log(\#F) \right) \\
    &= \#F^{2(1 - c\theta)}
  \end{align*}
  A large enough $\theta$ makes the upper bound much smaller than $1$.

  Thus, with very high probability, $\left| \norm{Gv}_2^2 - n \right| \leq \varepsilon n$.
  We claim that this matrix $\varphi \coloneqq \frac{G}{\sqrt{n}}$ does the dimension reduction with distortion bounded by $\varepsilon$.
  This can be seen by expanding out the definition of $v$.
  \begin{align*}
    \varepsilon n
    &\geq \left| \norm{Gv}_2^2 - n \right| \\
    &= \left| \frac{\norm{Gx - Gy}_2^2}{\norm{x-y}_2^2} - n \right|
  \end{align*}
  Dividing both sides by $n$, we get the inequality we wanted.
  \begin{align*}
    \varepsilon \geq \left| \frac{\norm{\varphi x - \varphi y}_2^2}{\norm{x-y}_2^2} - 1 \right|
  \end{align*}
  This proves the result.
\end{proof}

\subsection{Concentration for quadratic forms}
\label{sec:conc-quadr-forms}

Let $\{X_i\}$ be i.i.d copes of a random variable, and let $A$ be a symmetric matrix associated to a quadratic form.
We define a new random variable $Y$ by evaluating the quadratic form $A$ on $\mathbf{X} = \left( X_1, \ldots, X_n \right)$.
\begin{align*}
  Y \coloneqq \langle \mathbf{X}, A\mathbf{X} \rangle
\end{align*}
Consider now the singular value decomposition of $A$, i.e. a decomposition as $UDV$, where $U$ and $V$ lie in $O(n)$ and $D$ is diagonal.
The diagonal entries of $D$ can be arranged to be non-decreasing, i.e. $s_1(A) \geq s_2(A) \geq \cdots \geq s_n(A) \geq 0$.
The diagonal entries are called the singular values of $A$.
The $j$\textsuperscript{th} singular value of $A$ is the square root of the $j$\textsuperscript{th} largest eigenvalue of $AA^{\top}$.

We also consider the Frobenius norm of the matrix $A$.
\begin{align*}
  \norm{A}_F \coloneqq \left( \sum_{i,j=1}^n a_{ij} \right)^{\frac{1}{2}}
\end{align*}
This is the inner-product norm for the following inner product on matrices.
\begin{align*}
  \langle A, B \rangle = \mathrm{tr}(AB^{\top})
\end{align*}
One can express the Frobenius norm of $A$ using the singular values.
\begin{align*}
  \norm{A}_F^2 = \sum_{i=1}^{n} s_i(A)^2
\end{align*}
Geometrically, the singular values are the lengths of the axes of the ellipsoid that is the image under $A$ of the standard sphere in $\R^n$.
Note that one can also express the operator norm in terms of the singular values: it's the $\sup$ norm on the singular values, just like the Frobenius norm is the $\ell^2$-norm on the singular values.

We can now state a concentration inequality for quadratic forms.
\begin{theorem}[Hanson-Wright inequality (\cite{hanson1971bound}  and \cite{wright1973bound})]
  \label{thm:hanson-wright}
  Let $A$ be any $n \times n$ matrix, and let $\{X_1, \ldots, X_n\}$ be i.i.d subgaussian random variables with $\E(X_i) = 0$.
  Let $\mathbf{X} = (X_1, \ldots, X_n) \in \R^n$. Then for any $t > 0$:
  \begin{align*}
    \P\left( \left| \langle \mathbf{X}, A \mathbf{X} \rangle - \E\left( \langle \mathbf{X}, A \mathbf{X} \rangle \right) \right| > t \right)
    \leq 2 \exp\left( -c \left( \frac{t^2}{\norm{A}_F^2} \wedge \frac{t}{\norm{A}_{\mathrm{op}}} \right) \right)
  \end{align*}
\end{theorem}
\begin{remark}
  Note that we can only expect a tail like we had in Bernstein's inequality, since entries of $\langle \mathbf{X}, A \mathbf{X} \rangle$ be squares of subgaussians, i.e. subexponential.
\end{remark}
\begin{remark}
  The original papers of Hanson and Wright proved a slightly weaker version of the above theorem.
  The version stated is from 2013, and appeared in \cite{rudelson2013}.
\end{remark}

\begin{proof}[Proof of Theorem \ref{thm:hanson-wright}]
  We first decompose $A$ as the sum of a diagonal matrix $A_{\mathrm{diag}}$, and a matrix $\wt{A}$ with all diagonal entries equal to $0$.
  \begin{align*}
    A = A_{\mathrm{diag}} + \wt{A}
  \end{align*}
  Since we are trying to bound the probability that $\langle \mathbf{X}, A \mathbf{X} \rangle$ deviates by more than $t$ from its mean, it will suffice to bound the probability that $\langle \mathbf{X}, A_{\mathrm{diag}}\mathbf{X} \rangle$ deviates by more than $\frac{t}{2}$ and the probability that $\langle \mathbf{X}, \wt{A} \mathbf{X} \rangle$ deviates by more than $\frac{t}{2}$.
  \begin{align*}
    \P\left( \left| \langle \mathbf{X}, A \mathbf{X} \rangle - \E\left( \langle \mathbf{X}, A \mathbf{X} \rangle \right) \right| > t \right)
    &\leq \P\left( \left| \langle \mathbf{X}, A_{\mathrm{diag}} \mathbf{X} \rangle - \E\left( \langle \mathbf{X}, A_{\mathrm{diag}} \mathbf{X} \rangle \right) \right| > \frac{t}{2} \right) \\
      &+ \P\left( \left| \langle \mathbf{X}, \wt{A} \mathbf{X} \rangle - \E\left( \langle \mathbf{X}, \wt{A} \mathbf{X} \rangle \right) \right| > \frac{t}{2} \right)
  \end{align*}
  Note that since $\wt{A}$ has zeroes on the diagonal, and the $X_i$s are independent with mean $0$, that means $\E\left( \langle \mathbf{X}, \wt{A} \mathbf{X} \rangle \right) = 0$.

  Observe that the first term is a tail bound on a sum of subexponential random variables.
  \begin{align*}
    \langle \mathbf{X}, A_{\mathrm{diag}} \mathbf{X} \rangle - \E\left( \langle \mathbf{X}, A_{\mathrm{diag}} \mathbf{X} \rangle \right)
    &= \sum_{i=1}^{n} a_{ii}\left( X_i^2 - \E(X_i^2) \right)
  \end{align*}
  Since $X_i$ are subgaussian random variables, $X_i^2 - \E(X_i^2)$ are subexponential random variables with mean $0$, which means we can apply Bernstein's inequality.
  \begin{align*}
    \P\left( \left| \langle \mathbf{X}, A_{\mathrm{diag}} \mathbf{X} \rangle - \E\left( \langle \mathbf{X}, A_{\mathrm{diag}} \mathbf{X} \rangle \right) \right| > \frac{t}{2} \right)
    \leq 2 \exp\left( -c \left( \frac{t^2}{\sum_{i=1}^n a_{ii}^2} \wedge \frac{t}{\max(a_{ii})} \right) \right)
  \end{align*}
  Note that we can bound the denominators using the appropriate matrix norm.
  \begin{align*}
    \sum_{i=1}^{n} a_{ii}^2 &\leq \norm{A}_F^2 \\
    \max(a_{ii}) &\leq \norm{A}
  \end{align*}
  Note that the upper bound for the diagonal term is of the form stated in the theorem, which means that all we need to do prove a similar or better bound for the second term, which involves $\wt{A}$.
  That boils down to finding a probability bound for the following event.
  For convenience of notation, we will now denote $\wt{A}$ by just $A$, where it is understood that $A$ is a matrix with all diagonal entries equal to $0$.
  We will also denote $\langle \mathbf{X}, A \mathbf{X} \rangle$ by $Y$.
  \begin{align*}
    \P\left( \left| Y \right| > t \right)
    \leq 2 \exp\left( -c \left( \frac{t^2}{\norm{A}_F^2} \wedge \frac{t}{\norm{A}_{\mathrm{op}}} \right) \right)
  \end{align*}
  Recall that when we proved Bernstein's inequality, we did so bounding the Laplace transform of $Y$, i.e. getting upper bounds for $\E\left( \exp\left( \lambda Y \right) \right)$.
  Since $Y$ in our case is a quadratic function in independent random variables $X_i$, estimating the Laplace transform is a little tricky, and we need to use \emph{decoupling}.

  Introduce new random variables $\{\delta_1, \ldots, \delta_n\}$ which are independent from each other, as well as all $X_i$, and are distributed like $\mathrm{Bernoulli}\left( \frac{1}{2} \right)$. We then have the following.
  \begin{align*}
    \E\left( \exp(\lambda Y) \right)
    &= \E\left(  \exp\left( 4\lambda \sum_{i,j = 1}^{n} \E_{\delta}\left( \delta_i(1- \delta_j) \right) a_{ij} X_i X_{j} \right) \right)
  \end{align*}
  Here, $\E_{\delta}$ represents taking expectation over the sample space of the $\delta_{i}$, and $\E$ represents taking the expectation over the sample space of $X_i$.
  Note that the equality holds because for $i=j$, $\delta_i(1-\delta_j) = 0$.
  We now use Jensen's inequality to pull out the $\E_{\delta}$.
  \begin{align*}
    \E\left( \exp(\lambda Y) \right)
    &\leq \E \E_{\delta}\left(  \exp\left( 4\lambda \sum_{i,j = 1}^{n} \left( \delta_i(1- \delta_j) \right) a_{ij} X_i X_{j} \right) \right)
  \end{align*}
  Let $I$ be the random subset of $\{1, \ldots, n\}$ where $i \in I$ iff $\delta_i = 1$.
  We can condition the above expectation on the value of $I$ to simplify things.
  \begin{align*}
    \E\left( \exp(\lambda Y) \right)
    &\leq \E_{\delta} \E \left( \exp\left( 4\lambda \sum_{i \in I} \sum_{j \not \in I} a_{ij} X_i X_j \right) \bigg|\ I \right)
  \end{align*}
  We can simplify the inner integral using the fact that $I$ is fixed.
  \begin{align*}
    \E \left( \exp\left( 4\lambda \sum_{i \in I} \sum_{j \not \in I} a_{ij} X_i X_j \right) \bigg|\ I \right)
    &= \prod_{j \not \in I} \E \exp\left( \left[ 4 \lambda \sum_{i \in I} a_{ij} X_i \right] \cdot X_j \right)
  \end{align*}
  We now use the fact that each $X_j$ is subgaussian with mean $0$ to bound the right hand side in the following manner.
  \begin{align}
    \label{eq:hw-1}
    \prod_{j \not \in I} \E \exp\left( \left[ 4 \lambda \sum_{i \in I} a_{ij} X_i \right] \cdot X_j \right)
    &\leq \prod_{j \not \in I} \E \left( \exp\left[ 16 C \lambda^2 \left( \sum_{i \in I} a_{ij} X_i \right)^2 \right] \right)
  \end{align}
  We now use a trick to turn the expectation of the quadratic form as in \eqref{eq:hw-1} to expectation of a bilinear form.
  Note that for a standard normal random variable $g$, one has the following explicit formula for the Laplace transform.
  \begin{align*}
    \E\left( \theta g \right) = \exp\left( \frac{\theta^2}{2} \right)
  \end{align*}
  Let $\{g_1, \ldots, g_n\}$ be i.i.d standard normal random variables that are independent of $X_{i}$ and $\delta_i$.
  We get that the right hand side of \eqref{eq:hw-1} is the following.
  \begin{align*}
    \prod_{j \not \in I} \E \left( \exp\left[ 16 C \lambda^2 \left( \sum_{i \in I} a_{ij} X_i \right)^2 \right] \right)
    = \E \left( \exp\left( C^{\prime}\lambda \sum_{j \not \in I} \sum_{i \in I} a_{ij} X_i g_j \right) \right)
  \end{align*}
  Repeating this whole process again, with $X_i$ rather than $X_j$, we end up with the following upper bound.
  \begin{align*}
    \E \left( \exp\left( 4\lambda \sum_{i \in I} \sum_{j \not \in I} a_{ij} X_i X_j \right) \bigg|\ I \right)
    &\leq \E\left( \exp\left( C^{\prime \prime} \lambda^2 \sum_{i \in I} \left( \sum_{j \not \in I} a_{ij} g_j \right)^2 \right) \bigg|\ I \right)
  \end{align*}
  Note that the upper bound is independent of $X_i$, and only depends on the Bernoulli random variables $\delta_i$ and the normal random variables $g_i$.

  We will now use that fact that $\mathbf{g} = (g_1, \ldots, g_n)$ is a rotation invariant random vector in $\R^n$ to estimate the upper bound.
  Let $P_{I}$ be the projection to the subspace spanned by $\{e_i\}_{i \in I}$, and let $B_I = P_{I}A(\mathrm{Id} - P_I)$.
  Then the innermost double sum can be expressed as a norm.
  \begin{align*}
    \sum_{i \in I} \left( \sum_{j \not \in I} a_{ij} g_j \right)^2 = \norm{B_I \mathbf{g}}_2^2
  \end{align*}
  This means we need to estimate the following conditional expectation.
  \begin{align*}
    \E\left( C^{\prime \prime} \lambda^2 \norm{B_I \mathbf{g}}_2^2 \bigg|\ I \right)
  \end{align*}
  Let $U_ID_IV_I$ be the singular value decomposition of $B_I$.
  Since $\norm{\cdot}_2$ is invariant under $O(n)$ action, and the distribution of $\mathbf{g}$ is also $O(n)$ invariant, $B_I \mathbf{g}$ has the same distribution as $D_I \mathbf{g}$.
  This means we really just need to understand the distribution of the singular values of $B_I$.
  \begin{align*}
    \E\left( \exp\left(   C^{\prime \prime} \lambda^2 \norm{B_I \mathbf{g}}_2^2 \right) \bigg|\ I \right)
    &= \E\left( \exp\left( C^{\prime \prime} \lambda^2 \norm{D_I \mathbf{g}}_2^2 \right)  \bigg|\ I \right) \\
    &= \E\left( \exp\left(  C^{\prime \prime} \lambda^2 \sum_{j=1}^n s_j^2(B_I) g_j^2 \right)  \bigg|\ I \right) \\
    &= \prod_{j=1}^n \E\left( \exp\left( C^{\prime \prime} \lambda^2 s_j^2(B_I) g_j^2 \right) \bigg|\ I \right)
  \end{align*}
  One can explicitly compute $\E(\exp(\theta g^2))$ for a normal random variable $g$ for $\theta \in \left[ 0, \frac{1}{2} \right]$.
  \begin{align*}
    \E(\exp\left( \theta g^2 \right)) = \frac{1}{\sqrt{1 - 2\theta}}
  \end{align*}
  Hence,
  \begin{align*}
    \E\left( C^{\prime \prime} \lambda^2 \norm{B_I \mathbf{g}}_2^2 \bigg|\ I \right)
    &= \prod_{j=1}^n \left( 1 - 2 C^{\prime \prime} \lambda^2 s_j^2(B_I) \right)^{-\frac{1}{2}}
  \end{align*}
  For $x \in \left[ 0 , \frac{1}{2} \right]$, $\frac{1}{\sqrt{1-x}} \leq e^x$. This lets us bound the right hand side in the following manner.
  \begin{align*}
    \prod_{j=1}^n \left( 1 - 2 C^{\prime \prime} \lambda^2 s_j^2(B_I) \right)^{-\frac{1}{2}}
    &\leq \prod_{j=1}^n \exp\left( 2 C^{\prime \prime} \lambda^2 s_j^2(B_I) \right)
  \end{align*}
  Note however that to use the simplification, we required $2 C^{\prime \prime} \lambda^2 s_j^2(B_I) \leq \frac{1}{2}$.
  If we pick a $\lambda$ smaller than $\frac{c^{\prime \prime \prime}}{s_1(B_I)}$, then the inequalities are valid.
  That ends up giving the following bound.
  \begin{align*}
    \E\left( \exp\left( C \lambda \norm{B_I \mathbf{g}}_2^2 \right) \right)
    &\leq \exp\left( \wt{C} \lambda^2 \norm{B_I}_F^2 \right)
  \end{align*}
  This is valid when $\lambda \leq \frac{c^{\prime \prime \prime}}{s_1(B_I)} = \frac{c^{\prime \prime \prime}}{\norm{B_I}}$.
  Finally, we can get rid of the dependence on $I$ using the following inequalities involving matrix norms.
  \begin{align*}
    \norm{B_I}_F &= \norm{P_IA(\mathrm{Id} - P_I)}_F \\ &\leq \norm{A}_F \\
    \norm{B_I} &= \norm{P_IA(\mathrm{Id} - P_I)} \\ &\leq \norm{A}
  \end{align*}
  We can now conclude our estimates. We get the following inequality for all $\lambda$ less than $\frac{c^{\prime \prime \prime}}{\norm{A}}$.
  \begin{align*}
    \E\left( \exp(\lambda Y) \right)
    &\leq \E_{\delta} \E\left( \exp\left( C \lambda \norm{B_I \mathbf{g}}_2^2 \right) \right) \\
    &\leq \exp\left( \wt{C} \lambda^2 \norm{B_I}_F^2 \right) \\
    &\leq \exp\left( \wt{C} \lambda^2 \norm{A}_F^2 \right)
  \end{align*}
  We are able to integrate easily over the $\delta$ sample space because the right hand side does not depend on $I$ at all.
  The rest of the proof follows exactly like the end of Bernstein's inequality, where we optimized over $\lambda$ to get the best bound via Markov's inequality.
\end{proof}

We now show a rather unexpected application of the Hanson-Wright inequality.
\begin{theorem}
 Let $A$ be an $n \times n$ matrix, and let $\mathbf{X} \in \R^n$ be a random vector with i.i.d subgaussian coordinates with mean $0$ and variance $1$.
 Then for any $\tau > 0$, the following holds.
 \begin{align*}
   \P\left( \left| \norm{A \mathbf{X}}_2 - \norm{A}_F \right| > \tau \right)
   \leq 2 \exp\left( -c \frac{\tau^2}{\norm{A}^2} \right)
 \end{align*}
\end{theorem}
\begin{remark}
  This result is surprising for two reasons: first, we will prove this using the Hanson-Wright inequality, but the tail bound in this inequality is better than the tail bound in Hanson-Wright. Second, all of our earlier concentration inequalities were concentration inequalities about the mean, but in this case $\E\left( \norm{A \mathbf{X}}_2 \right) \neq \norm{A}_F$.
\end{remark}

\begin{proof}
  Let $Y = \norm{A \mathbf{X}}_2^2 = \langle \mathbf{X}, A^{\top}A \mathbf{X} \rangle$.
  This expression as the quadratic form $A^\top A$ will let us apply the Hanson-Wright inequality. Using the Hanson-Wright inequality, we obtain the following.
  \begin{align*}
    \P\left( \left| \langle \mathbf{X}, A^{\top}A \mathbf{X} \rangle
    - \E\left( \langle \mathbf{X}, A^{\top}A \mathbf{X} \rangle \right)
    \right| > t \right)
    \leq 2 \exp\left( -c \left[ \frac{t^2}{\norm{A^{\top}A}_F^2} \wedge \frac{t}{\norm{A^{\top}A}} \right] \right)
  \end{align*}
  We can explicitly evaluate $\E \left( \langle \mathbf{X}, A^{\top}A \mathbf{X} \rangle \right)$: since the entries of $\mathbf{X}$ are i.i.d with mean $0$ and variance $1$, the expectation simplifies to $\mathrm{tr}(A^{\top}A) = \norm{A}_F^2$.
  We also have these inequalities involving products of matrices.
  \begin{align*}
    \norm{AB}_F &\leq \norm{A} \cdot \norm{B}_F \\
    \norm{AB} &\leq \norm{A} \cdot \norm{B}
  \end{align*}
  Using these two inequalities, we can simplify the inequality we got from the application of Hanson-Wright inequality.
  \begin{align*}
    \P\left( \left| \norm{A \mathbf{X}}_2^2 - \norm{A}_F^2 \right| > t \right)
    \leq 2 \exp\left( -c \left[ \frac{t^2}{\norm{A}^2 \cdot \norm{A}_F^2} \wedge \frac{t}{\norm{A}^2} \right] \right)
  \end{align*}
  Let $t = \varepsilon \cdot \norm{A}_F^2$. Then the above inequality can be rewritten as follows.
  \begin{align*}
    \P\left( \left| \frac{ \norm{A \mathbf{X}}_2^2}{ \norm{A}_F^2}  - 1 \right| > \varepsilon \right)
    \leq 2 \exp\left( -c \frac{\norm{A}_F^2}{\norm{A}^2} \left[ \varepsilon^2 \wedge \varepsilon \right] \right)
  \end{align*}
  Rewriting the inequality in this manner makes it clearer which of the terms in the upper bound is smaller.
  The analysis splits into two cases.
  \begin{description}
  \item[\textbf{Case 1} ($\varepsilon \leq 1$):] In this case, $\varepsilon^2 \leq \varepsilon$, so the tail bound simplifies in the following manner.
    \begin{align*}
    \P\left( \left| \frac{ \norm{A \mathbf{X}}_2^2}{ \norm{A}_F^2}  - 1 \right| > \varepsilon \right)
      \leq 2 \exp \left( -c \frac{\norm{A}_F^2}{\norm{A}^2} \varepsilon^2 \right)
    \end{align*}
    Observe now that for positive values of $x$, $|x^2-1| \geq |x-1|$. This means that we can replace the $\frac{\norm{A \mathbf{X}}_2^2}{\norm{A}_F^2}$ in left hand side of the above inequality by its square root.
    We do that, and set $\tau = \varepsilon \norm{A}_F$.
    \begin{align*}
      \P\left( |\norm{A \mathbf{X}}_2 - \norm{A}_F| > \tau \right)
      \leq 2 \exp \left( -c \frac{\tau^2}{\norm{A}^2} \right)
    \end{align*}
    This proves the inequality for all $\tau \in \left[ 0, \norm{A}_F \right]$.
  \item[\textbf{Case 2} ($\varepsilon > 1$):] In this case $\varepsilon \leq \varepsilon^2$, so the tail bound simplifies in the following manner.
    \begin{align*}
    \P\left( \left| \frac{ \norm{A \mathbf{X}}_2^2}{ \norm{A}_F^2}  - 1 \right| > \varepsilon \right)
      \leq 2 \exp \left( -c \frac{\norm{A}_F^2}{\norm{A}^2} \varepsilon \right)
    \end{align*}
    Note again that for positive values of $x$, $|x^2 - 1| \geq |x-1|^2$. We use this fact to make a substitution on the left hand side, and let $\tau = \sqrt{\varepsilon} \norm{A}_F$.
    \begin{align*}
      \P\left( |\norm{A \mathbf{X}}_2 - \norm{A}_F| > \tau \right)
      \leq 2 \exp  \left( -c \frac{\norm{A}_F^2}{\norm{A}^2} \varepsilon \right)
    \end{align*}
    This proves in the inequality for $\tau \in \left( \norm{A}_F, \infty \right)$.
  \end{description}
\end{proof}

\subsection{Concentration for matrix-valued random variables}
\label{sec:conc-matr-valu}

So far, we have looked at concentration inequalities for functions acting on vector-valued random variables, more specifically, linear and quadratic functions on vectors of random variables.
In this section, we will look at concentration inequalities for matrix valued random variables.
We will start by proving the matrix Bernstein inequality.
\begin{theorem}[Matrix Bernstein Inequality]
  \label{thm:matrix-bernstein}
  Let $\{X_1, \ldots, X_N\}$ be independent $n \times n$ symmetric random matrices satisfying the following conditions for some constant $K$ independent of $j$.
  \begin{align*}
    \E\left( X_j \right) &= 0 \\
    \norm{X_j}_{\infty} &\leq K
  \end{align*}
  Let $\sigma^2$ denote the following quantity.
  \begin{align*}
    \sigma^2 \coloneqq \norm{\sum_{j=1}^N \E(X_j^2)}
  \end{align*}
  Then for any $t > 0$, we have the following tail bound on the sum of $X_j$.
  \begin{align*}
    \P\left( \norm{\sum_{j=1}^N X_j} > t \right)
    \leq 2n \exp\left( -c \left( \frac{t^2}{\sigma^2} \wedge \frac{t}{K} \right) \right)
  \end{align*}
\end{theorem}
\begin{remark}
  Note that this inequality looks very similar to the original Bernstein inequality: the only difference is that the coefficient multiplied with $\exp$ in this case is $2n$, rather than $2$.
  This means that the bound is not dimension independent, and that does cause issues in practice.
  However, the bound is optimal.
\end{remark}

To prove Theorem \ref{thm:matrix-bernstein}, we will need the following linear algebraic result of Lieb.

\begin{theorem}[Lieb's concavity theorem \cite{LIEB1973267}]
  Let $H$ be an $n \times n$ symmetric matrix.
  Consider the function $f$ defined on the set of $n \times n$ symmetric positive-definite matrices.
  \begin{align*}
    f(X) \coloneqq \mathrm{tr} \left( \exp\left( H + \log(X) \right) \right)
  \end{align*}
  The function $f$ is concave, i.e. for any $t \in [0,1]$, and any $X$ and $Y$ in the space of symmetric positive-definite matrices, the following inequality holds.
  \begin{align*}
    f(tX + (1-t)Y) \geq tf(X) + (1-t)f(Y)
  \end{align*}
\end{theorem}

Using Lieb's concavity theorem and Jensen's inequality, we get the following corollary.
\begin{corollary}
  \label{cor:lieb}
  Let $H$ be an $n \times n$ symmetric matrix, and let $Y$ be a random symmetric matrix.
  Then the following inequality holds.
  \begin{align}
    \label{eq:2}
    \E \left( \mathrm{tr} \left( \exp\left( H + Y \right) \right) \right)
    \leq \mathrm{tr}\left( \exp\left( H + \log\left( \E(\exp(Y)) \right) \right) \right)
  \end{align}
\end{corollary}
Using this corollary, we can prove the matrix Bernstein inequality.

\begin{proof}[Proof of Theorem \ref{thm:matrix-bernstein}]
  We begin by giving an easy upper bound for the norm of a symmetric matrix $X$.
  \begin{align*}
    \norm{X} \leq \lambda_{\mathrm{max}}(X) + \lambda_{\mathrm{max}}(-X)
  \end{align*}
  Here, $\lambda_{\mathrm{max}}(X)$ denotes the largest eigenvalue of $X$.
  Expressed in terms of tail bounds, we get the following.
  \begin{align*}
    \P\left( \norm{\sum_{j=1}^N X_j} > t \right)
    \leq \P\left( \lambda_{\mathrm{max}} \left( \sum_{j=1}^N X_j \right) > \frac{t}{2} \right)
    + \P\left( \lambda_{\mathrm{max}} \left( \sum_{j=1}^N -X_j \right) > \frac{t}{2} \right)
  \end{align*}
  Note that it will suffice to get an upper bound for one of the terms on the right hand side, and multiply that by $2$.

  Just like in the proof of Bernstein's inequality, it will help to bound the Laplace transformation of the random variables involved.
  To do so, we need to chain several simplifying equalities and inequalities.
  The first equality we get from the fact that the largest eigenvalue of the exponential of a symmetric matrix is the exponential of the largest eigenvalue of the original matrix.
  \begin{align}
    \label{eq:3}
    \E\left( \exp\left( \lambda \lambda_{\mathrm{max}} \left( \sum_{j=1}^N X_j \right) \right) \right)
    &= \E\left( \lambda_{\mathrm{max}} \exp\left( \lambda \left( \sum_{j=1}^N X_j\right) \right) \right)
  \end{align}
  Here, $\lambda$ is an arbitrary positive number.
  Now note that $\exp$ of a symmetric matrix has only positive eigenvalues.
  This means that the largest eigenvalue is smaller than the trace.
  \begin{align}
    \label{eq:4}
    \E\left( \lambda_{\mathrm{max}} \exp\left( \lambda \left( \sum_{j=1}^N X_j\right) \right) \right)
    \leq \E\left( \mathrm{tr} \exp\left( \lambda \left( \sum_{j=1}^{N-1} X_j + X_N\right) \right) \right)
  \end{align}
  We now condition on the values of $\{X_1, \ldots, X_{N-1}\}$, which lets us treat the right hand side of \eqref{eq:4} using \eqref{eq:2}.
  \begin{align*}
    \E\left( \mathrm{tr} \exp\left( \lambda \left( \sum_{j=1}^{N-1} X_j + X_N\right) \right) \right)
    \leq \E\left( \mathrm{tr} \exp\left( \lambda \left( \sum_{j=1}^{N-1} X_j \right) + \log \E \exp\left( \lambda X_N \right) \right) \right)
  \end{align*}
  We repeat this process $N-2$ additional times, conditioning on all but the last $X_i$ to get the following inequality, combining \eqref{eq:3}, \eqref{eq:4}, and \eqref{eq:2}.
  \begin{align}
    \label{eq:5}
    \E\left( \exp\left( \lambda \lambda_{\mathrm{max}} \left( \sum_{j=1}^N X_j \right) \right) \right)
    \leq \mathrm{tr}\left( \exp\left[ \sum_{j=1}^N \log \E \exp\left( \lambda X_j \right)\right] \right)
  \end{align}

  We now deal with $\E \exp(\lambda X_j)$.
  Note that for $t \in [0, 1]$, we have the following elementary inequality.
  \begin{align*}
    \exp(t) \leq 1 + t + t^2
  \end{align*}
  We can extend this to symmetric matrices, where the inequality $A \leq B$ is understood to mean that $B-A$ is positive definite.
  \begin{align*}
    \E \exp\left( \lambda X_j \right) &\leq I + \E(\lambda X_j) + \E\left( \lambda^2 X_j^2 \right) \\
    &= I + \lambda^2 \E(X_j^2)
  \end{align*}
  Note that the above inequality holds when $\lambda$ is small enough such that the largest eigenvalue of $\lambda X_j$ is less than $1$.
  Now we use the inequality $1 + x \leq \exp(x)$ to get an upper bound of $\exp\left( \lambda^2 \E(X_j^2) \right)$.
  We combine this with \eqref{eq:5} to get the following.
  \begin{align}
    \label{eq:6}
    \E\left( \exp\left( \lambda \lambda_{\mathrm{max}} \left( \sum_{j=1}^N X_j \right) \right) \right)
    \leq \mathrm{tr} \left( \exp \left[ \sum_{j=1}^N \lambda^2 \E(X_j^2) \right] \right)
  \end{align}
  Now we again use the fact that the exponential of a symmetric matrix is positive definite to bound the trace above by $n$ times the largest eigenvalue.
  \begin{align}
    \label{eq:7}
     \E\left( \exp\left( \lambda \lambda_{\mathrm{max}} \left( \sum_{j=1}^N X_j \right) \right) \right)
    \leq n \lambda_{\mathrm{max}} \left( \exp \left[ \sum_{j=1}^N \lambda^2 \E(X_j^2) \right]\right)
  \end{align}
  Finally, we use the fact that $\lambda_{\mathrm{max}} X$ is less than or equal to $\norm{X}$ to get the upper bound in terms of matrix norm.
  \begin{align}
    \label{eq:8}
    \E\left( \exp\left( \lambda \lambda_{\mathrm{max}} \left( \sum_{j=1}^N X_j \right) \right) \right)
    &\leq n \left( \exp \left[ \lambda^2 \norm{\sum_{j=1}^N \E(X_j^2)}  \right]\right) \\
    &\leq n \left( \exp \left[ \lambda^2 \sigma^2  \right]\right)
  \end{align}

  This gives a bound on the Laplace transformation of the random variable we were trying to tail bound.
  We now use Markov's inequality to get the tail bound.
  \begin{align*}
    \P\left( \lambda_{\mathrm{max}} \left( \sum_{j=1}^N X_j\right) > \tau \right)
    \leq n \exp\left( \lambda^2 \sigma^2 - \lambda \tau \right)
  \end{align*}
  Note that this bound comes from \eqref{eq:8}, which is only valid when $\lambda \max \norm{X_j} \leq 1$.
  We optimize over the values of $\lambda$ now. This calculation is identical to the one we did for the original Bernstein inequality, and the tail bound follows.
\end{proof}

We now discuss a few corollaries of the matrix Bernstein inequality.
The first corollary pertains to matrices that need not be symmetric, or even square.

\begin{corollary}
  Let $\{X_1, \ldots, X_N\}$ be $m \times n$ matrices, satisfying the following conditions for some constant $K$.
  \begin{align*}
    \E(X_j) &= 0 \\
    \norm{X_j}_{\infty} &\leq K
  \end{align*}
  Let $\sigma^2 = \norm{\sum_{j=1}^N \E\left( X_j X_j^{\top} \right)} + \norm{\sum_{j=1}^N \E\left( X_j^{\top}X_j \right)}$.
  Then we have the following tail bound.
  \begin{align*}
    \P\left( \norm{\sum_{j=1}^N X_j} > t \right)
    \leq 2(n+m) \exp\left( -c \left( \frac{t^2}{\sigma^2} \wedge \frac{t}{K} \right) \right)
  \end{align*}
\end{corollary}
\begin{proof}
  The proof follows from Theorem \ref{thm:matrix-bernstein} after symmetrizing the rectangular matrices.
  Consider the $(n+m) \times (n+m)$ matrix $X_j^s$, constructed in the following manner.
  \begin{align*}
    X_j^s =
    \begin{pmatrix}
      0 & X_j  \\
      X_j^{\top} & 0
    \end{pmatrix}
  \end{align*}
  Applying Theorem \ref{thm:matrix-bernstein} to $\{X_1^s, \ldots, X_N^s\}$ gives us the proof.
\end{proof}

Another easy corollary is getting expectation bounds from tail bounds.
\begin{corollary}
  \label{cor:exp-bound}
  Let $\{X_1, \ldots, X_N\}$ be as in Theorem \ref{thm:matrix-bernstein}. Then we have the following expectation bound.
  \begin{align*}
    \E \norm{\sum_{j=1}^N X_j} \leq C \left( \sigma \sqrt{\log(n)} + K \log(n) \right)
  \end{align*}
\end{corollary}

We now return to the problem of covariance estimation that we mentioned in the introduction.

\paragraph{Empirical covariance estimation}

Let $X$ be an $\R^n$-valued random variable with mean $0$.
The population covariance $\Sigma$ is defined to be $\E X X^{\top}$.
Let $\{X_1, \ldots, X_N\}$ be i.i.d copies of $X$.
We define sample covariance $\Sigma_N$ to be the sample average of the covariance of each $X_j$.
\begin{align*}
  \Sigma_N \coloneqq \frac{\sum_{j=1}^N X_j X_j^{\top}}{N}
\end{align*}
By the law of large numbers, $\Sigma_N$ converges to the population covariance.
However, it's more useful to know how fast $\Sigma_N$ converges to $\Sigma$.
\begin{theorem}
  Let $X$ and $\{X_1, \ldots, X_N\}$ be random vectors as described previously.
  Suppose that the following inequality holds almost surely for some constant $S$.
  \begin{align*}
    \norm{X}_2^2 \leq S \E\norm{X}_2^2
  \end{align*}
  Let $\varepsilon \in (0,1)$, and set $N$ to be the following.
  \begin{align*}
    N = C \frac{S n \log(n)}{\varepsilon^2}
  \end{align*}
  Here, $C$ is some fixed constant, and we round $N$ to the nearest integer.
  Then we have the following concentration inequality for sample covariance.
  \begin{align*}
    \E \norm{\Sigma_N - \Sigma} \leq \varepsilon \cdot \norm{\Sigma}
  \end{align*}
\end{theorem}
\begin{proof}
  Define $Y_j$ to be $X_j X_j^{\top} - \Sigma$.
  The expectation of $Y_j$ is clearly $0$.
  We can express the left hand side of the inequality we are trying to prove in terms of $Y_j$.
  \begin{align*}
    \E \norm{\Sigma_N - \Sigma} &= \E \norm{ \frac{\sum_{j=1}^N X_jX_j^{\top}}{N} - \Sigma } \\
                                &= \frac{1}{N} \E \norm{\sum_{j=1}^N Y_j}
  \end{align*}
  By Corollary \ref{cor:exp-bound}, we can bound the above expectation.
  \begin{align}
    \label{eq:9}
    \E \norm{\sum_{j=1}^N Y_j}
    \leq C\left( \sigma \sqrt{\log(n)} + K \log(n) \right)
  \end{align}
  We need to compute what $\sigma$ and $K$ are in this context.
  Recall that $K$ was an almost sure upper bound on $\norm{Y_j}$.
  \begin{align*}
    \norm{Y_j} &= \norm{X_jX_j^{\top} - \Sigma} \\
               &\leq \norm{X_j}^2 + \norm{\Sigma}
  \end{align*}
  We now use the hypothesis we had.
  \begin{align*}
    \norm{X_j}^2 &\leq S \E\norm{X}_2^2 \\
                 &=S \E\left( \mathrm{tr}(X_j X_j^{\top}) \right) \\
                 &= S \mathrm{tr}(\Sigma) \\
                 &\leq Sn \norm{\Sigma}
  \end{align*}
  Thus, $K$ in this context in $S n \norm{\Sigma}$.

  To compute $\sigma$, we do something similar.
  \begin{align*}
    \E Y_j^2 &= \E\left( X_jX_j^{\top} - \Sigma \right)^2 \\
             &\leq \E\left( X_j X_j^{\top} \right)^2 \\
             &\leq \norm{X_j}_2^2 \cdot \E \left( X_j X_j^\top \right) \\
             &\leq Sn \norm{\Sigma} \Sigma
  \end{align*}
  The quantity $\sigma^2$ then works out to be less than $N S n \norm{\Sigma}^2$.
  Plugging these values into \eqref{eq:9} gives us the result we want.
\end{proof}

\section{Martingales}
\label{sec:martingales}

\subsection{Azuma's martingale inequality}
\label{sec:azum-mart-ineq}

We begin by recalling the definition of a martingale.
\begin{definition}[Martingale]
  Let $\cF_0 \subset \cF_1 \subset \ldots \subset \cF_n$ be a filtration of sigma algebras of the sample space $\Omega$, with $\cF_0 = \{\emptyset, \Omega\}$.
  A sequence of random variables $\{M_1, M_2, \ldots\}$ is said to be a martingale adapted with respect to the filtration $\{ cF_i\}$ if the following holds for all $i$ and $j$, where $j > i$.
  \begin{align*}
    \E\left( M_j \mid\ \cF_i \right) = M_i
  \end{align*}
\end{definition}

Using the notion of a martingale, we can state a generalization of a special case of Hoeffding's inequality.

\begin{theorem}[Azuma's martingale inequality]
  \label{thm:azuma-martingale}
  Let $\{M_j\}$ be a martingale. Assume that for all $j \geq 1$, there exist constants $d_j$ such that the following holds.
  \begin{align*}
    \norm{M_j - M_{j-1}}_{\infty} \leq d_j
  \end{align*}
  Then for any $t > 0$, we have the following concentration inequality for $M_n$.
  \begin{align*}
    \P\left( \left| M_n - \E M_n \right| > t \right)
    \leq 2 \exp\left( - \frac{t^2}{4\sum_{j=1}^n d_j^2} \right)
  \end{align*}
\end{theorem}
To see how this is a generalization of a special case of Hoeffding's inequality, let $\{B_i\}$ be i.i.d copies of a random variable with zero mean whose absolute value is bounded by $1$ almost surely.
If we define $M_n$ by $\sum_{j=1}^n d_jB_j$, then it's easy to verify that $\{M_n\}$ is a martingale satisfying the above hypotheses, and the tail bound in this case follows from Hoeffding's inequality.

The reason why the martingale variant of the inequality requires more work is that it is not the case that all martingales are sums of independent random variables.
However, it turns out that a lot of results that hold for sums of independent random variables can also be made to work for martingales.

\begin{proof}[Proof of Theorem \ref{thm:azuma-martingale}]
  We prove this concentration inequality again by estimating the Laplace transform of $M_n$.
  Let $\lambda > 0$, and define $f$ in the following manner.
  \begin{align*}
    f(\lambda) = \E(\exp(\lambda M_n))
  \end{align*}
  To estimate the Laplace transform, we split up $M_n$ in an artificial manner.
  \begin{align*}
    \E(\exp(\lambda M_n))
    &= \E(\exp(\lambda M_{n-1} + \lambda(M_n - M_{n+1})))
  \end{align*}
  Observe that since $M_{n-1}$ and $M_n - M_{n-1}$ are not necessarily independent, we cannot turn this into a product of expectations.
  But we do know that the expectation of $M_{n} - M_{n-1}$ when conditioned on $M_{n-1}$ is $0$.
  \begin{align*}
    \E(\exp(\lambda M_{n-1} + \lambda(M_n - M_{n+1})))
    &= \E\left( \exp\left( \lambda M_{n-1} \right)  \right)
      \cdot \E\left( \exp(\lambda (M_n - M_{n-1})) \mid\ M_{n-1} \right)
  \end{align*}
  We deal with the second term in the product first.
  To do, so consider the following elementary inequality, which we will state without proof.
  For all $x \in \R$, the following holds.
  \begin{align*}
    e^x \leq x + e^{x^2}
  \end{align*}
  We use this to bound the second term of the product as follows.
  \begin{align*}
    \E\left( \exp(\lambda (M_n - M_{n-1})) \mid\ M_{n-1} \right)
    &\leq \E\left( \lambda(M_n - M_{n-1}) \mid\ M_{n-1} \right) \\
      &+ \E\left( \exp\left( \lambda^2(M_n - M_{n-1})^2 \right) \mid\ M_{n-1} \right)
  \end{align*}
  Observe that the first term vanishes, and the second term is bounded above by $\exp(\lambda^2 d_j^2)$.

  To deal with the $\E\left( \exp\left( \lambda M_{n-1} \right) \right)$ term, we do the same thing to get an upper bound of $\exp(\lambda d_{j-1}^2)$.
  We repeat this process inductively, to get the following bound.
  \begin{align*}
    \E\left( \exp\left( \lambda M_n \right) \right)
    &\leq \exp\left( \lambda^2 \left( \sum_{j=1}^n d_j^2 \right) \right) \cdot \E\left( \lambda M_0 \right) \\
    &= \exp\left( \lambda^2 \left( \sum_{j=1}^n d_j^2 \right) \right) \cdot \E\left( \lambda M_n \right)
  \end{align*}
  Using Markov's inequality, and optimizing over $\lambda$, the inequality follows.
\end{proof}

An application of Azuma's martingale inequality is the \emph{average bounded differences inequality}.

\begin{corollary}[Average bounded differences inequality]
  \label{thm:abd}
  Let $\{X_1, \ldots, X_n\}$ be independent real valued random variables.
  Let $f$ be a function from $\R^n$ to $\R$ such that for any $j \in \{1, 2, \ldots, n-1\}$, and any $\{y_1, \ldots, y_{j+1}\}$ in $\R$, the following holds for some $d_{j+1} \geq 0$.
  \begin{align*}
    \left| \E f\left( y_1, \ldots, y_{j+1}, X_{j+2}, \ldots, X_m \right) -
    \E f\left( y_1, \ldots, y_j, X_{j+1}, \ldots, X_m \right)
    \right| \leq d_{j+1}
  \end{align*}
  Then for any $t > 0$, the following tail bound applies to $f(X_1, \ldots, X_n)$.
  \begin{align*}
    \P\left( \left| f(X_1, \ldots, X_n) - \E f(X_1, \ldots, X_n) \right| > t \right)
    \leq 2 \exp\left( - \frac{t^2}{\sum_{j=1}^n d_j^2} \right)
  \end{align*}
\end{corollary}

\begin{remark}
  The strength of this theorem comes from the fact that we require very little from the function $f$.
  For instance, if $f$ is merely continuous, and the $X_i$ are bounded random variables, then this inequality applies, and gives us a strong concentration result.
\end{remark}

\begin{proof}[Proof of Corollary \ref{thm:abd}]
  Let $\cF_j$ be the $\sigma$-algebra generated by $X_1$ to $X_j$.
  Define the martingale $M_j$ in the following manner.
  \begin{align*}
    M_j = \E\left( f(X_1, \ldots, X_n) \mid\ \cF_j \right)
  \end{align*}
  It is easy to check that the sequence of $M_j$'s actually forms a martingale.
  Furthermore, one can verify the condition required by the martingale in Theorem \ref{thm:azuma-martingale}.
  \begin{align*}
    \left| M_j - M_{j-1} \right|
    &= \left| E\left[ f(X_1, \ldots, X_n) \mid\ \cF_j \right] - E\left[ f(X_1, \ldots, X_n) \mid\ \cF_{j-1} \right] \right| \\
    &\leq \sup_{\left\{ y_1, \ldots y_j \right\}} \big| E\left[ f(y_1, \ldots, y_j, X_{j+1} \ldots, X_n) \mid\ \cF_j \right] \\
    &- E\left[ f(y_1, \ldots, y_{j-1}, X_j, \ldots X_n) \mid\ \cF_{j-1} \right] \big| \\
    &\leq d_j
  \end{align*}
  Using Theorem \ref{thm:azuma-martingale}, we get the claimed tail bound.
\end{proof}

\subsection{Applications of Azuma's martingale inequality}
\label{sec:appl-azum-mart}

In this section, we consider several applications of Azuma's martingale inequality and the average bounded differences inequality.

\paragraph{Random vectors in Banach spaces}

Let $\{X_1, \ldots, X_n\}$ be independent random vectors in a normed space $(Y, \norm{\cdot})$.
If for all $j \leq n$, $\norm{A_j} \leq a_j$ almost surely, then the following holds.
\begin{align*}
  \P\left( \left| \norm{ \sum_{j=1}^n X_j} - \E  \norm{ \sum_{j=1}^n X_j}  \right| > t \right)
  \leq 2 \exp \left( - \frac{t^2}{16 \sum_{j=1}^n a_j^2} \right)
\end{align*}
This follows from Azuma's martingale inequality by letting $M_j$ be the random variable obtained by taking the conditional expectation of $\norm{\sum_{j=1}^n X_j}$ conditioned on the $\sigma$-algebra generated by $\{X_1, \ldots, X_j\}$.
This sequence of real valued random variables forms a martingale, and from the fact that $\norm{X_j}$ is bounded above by $a_j$ almost surely, we get that that the martingale difference is bounded above by $2a_j$ almost surely, and the result follows.

Observe that this proof tells us absolutely nothing about $\E \norm{\sum_{j=1}^n X_j}$, but only gives a concentration inequality about the mean.
This will be the norm when proving concentration inequalities using Azuma's martingale inequality.

\paragraph{Randomized bin packing}

Another practical application is the \emph{randomized bin packing problem}.
Suppose $\{X_1, \ldots, X_n\}$ are numbers from the interval $[0,1]$, which we will call weights.
What is the minimum number $N(X_1, \ldots, X_n)$ of partitions (or bins) we need for these weights such that the sum of weights in each partition (or bin) does not exceed $1$?
This is an optimization problem of great importance to computer scientists, and a naïve algorithm to compute the minimum number of bins is takes super-exponential time as a function of $n$.

Consider now a variant of this problem where the weights $\{X_1, \ldots, X_n\}$ are i.i.d random variables.
We are interested in the minimal number of bins $N(X_1, \ldots, X_n)$ as $n$ tends to $\infty$.
One has a result in the spirit of the law of large numbers.
\begin{theorem}
  There exists a real number $\gamma \in [0,1]$, depending on the distribution of $X_i$ such that the following holds almost surely.
  \begin{align*}
    \lim_{n \to \infty} \frac{N(X_1, \ldots, X_n)}{n} = \gamma
  \end{align*}
\end{theorem}
  Let $M_j$ be the random obtained by taking the conditional expectation of $F(X_1, \ldots , X_n)$ conditioned on the $\sigma$-algebra $\cF_j$ generated by $\{X_1, \ldots, X_j\}$.
  This is a martingale, and we now verify that $\left| M_{j+1} - M_j \right|$ is at most $1$.
  \begin{align*}
    \left| M_{j+1} - M_j \right|
    &= \left| \E\left( N(X_1, \ldots, X_n) \mid \cF_{j+1} \right) - \E\left( N(X_1, \ldots, X_n) \mid \cF_{j} \right) \right|
  \end{align*}
  To see that the right hand side is at most $1$, it will suffice to condition further on the $\sigma$-algebra generated by $\{ X_{j+2}, \ldots, X_n\}$.
  After doing so, we see that the difference is bounded pointwise by $1$.
  \begin{align*}
    \left| N(x_1, \ldots, x_n) - N(x_1, \ldots, x_j, X_{j+1}, x_{j+2}, \ldots, x_n) \right| \leq 1
  \end{align*}
  The above inequality follows from the fact that changing one weight can at most decrease or increase the number of bins by $1$.
  If the weight increases, we need at most one additional bin.
  If the weight decreases, and the number of bins goes down by more than $1$, then increasing the weight again would increase the number of bins by more than $1$, which cannot happen.
  This proves the claim that the martingale difference is bounded above by $1$.
  Using Azuma's martingale inequality, we get the following concentration inequality for $N$.
  \begin{align*}
    \P\left( \left| N(X_1, \ldots, X_n) - \E N(X_1, \ldots, X_n) \right| > t \right)
    \leq 2 \exp\left( - \frac{t^2}{n} \right)
  \end{align*}
  Letting $t = \frac{\gamma n}{2}$, we get that the probability that $N(X_1, \ldots, X_n)$ is less than $\frac{\gamma n}{2}$ is very small.
  \begin{align*}
    \P\left( N(X_1, \ldots, X_n) \leq \frac{\gamma n}{2} \right) &\leq 2 \exp\left( -frac{\gamma^2 n^2}{n} \right) \\
   &= 2 \exp \left( -\gamma^2 n \right)
  \end{align*}

\paragraph{Functional and geometric concentration for the discrete cube}

Another application of Azuma's martingale inequality is functional and geometric concentration for the discrete cube.
The discrete cube $D_n$ is $\{0,1\}^n$ with the distance function $d_H$, where $d_H(x, y) \coloneqq \frac{1}{n} \sum_{i=1}^n |x_i - y_i|$.
Let $\mu$ be the uniform probability measure on $D_n$.
Let $f: D_n \to \R$ be $1$-Lipschitz.
If we consider the coordinates to be independent random variables, we see that the $1$-Lipschitz condition translates to the average difference of $f$ being less than or equal to $1$.
We then get the following concentration result for $f$.
\begin{align}
  \label{eq:func-conc}
  \mu\left( \left| f(x) - \E f(x) \right| > t \right) \leq 2 \exp \left( - nt^2 \right)
\end{align}
This inequality gives us \emph{geometric measure concentration}.
\begin{theorem}[Geometric measure concentration]
  Let $A \subset D_n$ be a set such that $\mu(A) \geq \frac{1}{2}$.
  Let $\varepsilon > 0$, and let $A_\varepsilon$ be the $\varepsilon$ neighbourhood of $A$.
  Then the complement $A_\varepsilon^C$ of $A_\varepsilon$ has very little measure.
  \begin{align*}
    \mu(A_\varepsilon^C) \leq 2 \exp \left( - \varepsilon^2 n \right)
  \end{align*}
\end{theorem}
This is quite unlike what we are used to in the case of a solid cube in $\R^n$, with Lebesgue measure.
In this case, as $n$ approaches $\infty$, the measure of $\mu(A_\varepsilon^C)$ does not approach $0$.
\begin{proof}[Proof of geometric measure concentration]
  Let $f(x) = d_H(x, A)$. Clearly, $f$ is $1$-Lipschitz and we can use \eqref{eq:func-conc}.
  We have that $\mu(A_\varepsilon^C) = \mu(f(x)>\varepsilon)$.
  To use \eqref{eq:func-conc}, we need to estimate $\E f(x)$.
  If $x \in A$, $f(x) = 0$.
  \begin{align*}
    \frac{1}{2} &\leq \mu(A) \\
                &\leq \mu\left( \left| f(x) - \E f(x) \right| \geq \E f(x) \right) \\
    &\leq 2 \exp \left( - n \left( \E f(x) \right)^2 \right)
  \end{align*}
  Thus, $\E f(x) \leq \frac{C}{\sqrt{n}}$ for some constant $C$.

  We thus get the following.
  \begin{align*}
    \mu\left( f(x) > t + \frac{C}{\sqrt{n}} \right) \leq 2 \exp(-nt^2)
  \end{align*}
  Setting $t$ to the appropriate value gives us the result.
\end{proof}

\paragraph{Random allocations}

Suppose we have $n$ balls, which we randomly and independently put into $m$ bins.
One can ask various statistical questions about the setup, e.g. number of empty bins, longest stretch of empty bins and so on.
The problem we consider is the following: let $Z$ be the number of empty bins.
We can write $Z$ as $\sum_{j=1}^m Z_j$, where $Z_j$ is the indicator for the event that the $j$\textsuperscript{th} bin is empty.
We can easily evaluate $\E Z$: this will be $\sum_{j=1}^m \E Z_j = m \left( 1 - \frac{1}{m} \right)^n \approx m \exp \left( - \frac{n}{m} \right)$.
Suppose now that $\frac{n}{m}$ is approximately a constant $r$.
We want concentration inequalities for $Z$, and $Z$ is a sum of identically distributed random variables.
However, the $Z_j$ are not independent, which means most of our results don't apply here.
To deal with this, we consider the random variables $\{X_j\}$, which is the number of the bin the $j$\textsuperscript{th} ball was put in.
Clearly, $Z$ is a function of $\{X_1, \ldots, X_n\}$: like the previous examples, we can create a martingale by taking the conditional expectations of $Z$ conditioned on the $\sigma$-algebra generated by $\{X_1, \ldots, X_j\}$.
Furthermore, the martingale differences are bounded above by $1$ as well, which means by Azuma's martingale inequality, we get the following concentration inequality.
\begin{align}
  \label{eq:weak-bound}
  \P\left( \left| Z - \E Z \right| > t \right) \leq 2 \exp \left( - \frac{t^2}{4n} \right)
\end{align}

We can do better though: note that the martingale difference bound we obtained was a pointwise bound, and the difference in expectation might be smaller.
Let $\Delta_j$ denote the martingale difference.
\begin{align*}
  \Delta_j = \E\left[ Z \mid X_1, \ldots, X_j \right] - \E\left[ Z \mid X_1, \ldots, X_{j-1} \right]
\end{align*}
Recall now that $Z = \sum_{j=1}^m Z_j$. Let $1 \leq k \leq m$.
Then given $X_1, \ldots, X_j$, if at least one of them equals $k$, $Z_k = 0$.
Otherwise $Z_k$ is a Bernoulli random variable with $\P(Z_k = 1 \mid X_1, \ldots , X_j) = \left( 1- \frac{1}{m} \right)^{n-j}$.
This lets evaluate the conditional expectations of $Z_k$.
\begin{align*}
  \E\left[ Z_k \mid X_1, \ldots , X_j \right] = \left( 1 - \frac{1}{m} \right)^{n-j} \prod_{l=1}^j \left( 1 - \mathbb{1}_k(X_l) \right)
\end{align*}
We similarly have the $j-1$\textsuperscript{th} term.
\begin{align*}
  \E\left[ Z_k \mid X_1, \ldots , X_{j-1} \right] = \left( 1 - \frac{1}{m} \right)^{n-j + 1} \prod_{l=1}^{j-1} \left( 1 - \mathbb{1}_k(X_l) \right)
\end{align*}
Let $X_j^{\prime}$ be an independent copy of $X_j$. Then we can rewrite the above expression in the following manner.
\begin{align*}
  \E\left[ Z_k \mid X_1, \ldots , X_{j-1} \right]
  &= \left( 1 - \frac{1}{m} \right)^{n-j} \E_{X_{j}^{\prime}} \left( 1 - \mathbb{1}_k(X_j^{\prime}) \right) \prod_{l=1}^{j-1} \left( 1 - \mathbb{1}_k(X_l) \right) \\
  &= \left( 1 - \frac{1}{m} \right)^{n-j} \E_{X_{j}^{\prime}} \left[ \left( 1 - \mathbb{1}_k(X_j^{\prime}) \right) \prod_{l=1}^{j-1} \left( 1 - \mathbb{1}_k(X_l) \right) \right]
\end{align*}
We can write the martingale difference $\Delta_j$ by taking the difference of the two terms we derived.
\begin{align*}
  \Delta_j &= \left( 1 - \frac{1}{m} \right)^{n-j}
  \cdot \left[
  \prod_{l=1}^j \left( 1 - \mathbb{1}_k(X_l) \right) -
\E_{X_{j}^{\prime}} \left[ \left( 1 - \mathbb{1}_k(X_j^{\prime}) \right) \prod_{l=1}^{j-1} \left( 1 - \mathbb{1}_k(X_l) \right) \right]
             \right] \\
  &= \left( 1 - \frac{1}{m} \right)^{n-j} \E_{X_{j}^{\prime}} \left[ \left( \mathbb{1}_k(X_j^{\prime}) - \mathbb{1}_k(X_j) \right) \prod_{l=1}^{j-1} \left( 1 - \mathbb{1}_k(X_l) \right) \right]
\end{align*}
Observe now that the above expression is bounded above pointwise by $a_j = \left( 1- \frac{1}{m} \right)^{n-j}$, which is better than the previous estimate we had.
We can work out what the terms of the improved concentration inequality will look like.
\begin{align*}
  \sum_{j=1}^{n} a_j^2 &= \sum_{j=1}^n \left( 1 - \frac{1}{m} \right)^{2(n-j)} \\
                       &=\frac{1 - \left( 1- \frac{1}{m} \right)^{2(n+2)}}{1- \left( 1 - \frac{1}{m} \right)^2} \\
                       &\approx \frac{ 1 - \exp\left( - \frac{2n}{m} \right)}{\frac{2}{m} - \frac{1}{m^2}} \\
  &\approx \frac{1 - \exp(-2r)}{\frac{2}{m}}
\end{align*}
We use this term in Azuma's martingale inequality.
\begin{align}
  \label{eq:strong-bound}
  \P\left( \left| Z - \E Z \right| > t \right) \leq 2 \exp \left( - \frac{t^2}{2m(1-e^{-2r})} \right)
\end{align}
Compare this to \eqref{eq:weak-bound}.
If $r$ is much bigger than $1$, then \eqref{eq:strong-bound} is much better, because the denominator is on the order of $2n$, which is better than $4n$.
If $r$ is much smaller than $1$, the denominator in \eqref{eq:strong-bound} is roughly $4n$, which is the same as \eqref{eq:weak-bound}, giving us no improvement.

The takeaway from this calculation is that the better we can estimate the martingale differences, the better concentration inequality we get.
Another takeaway is that for when $r$ is much smaller than $1$, the number of bins vastly outnumbers the number of balls, in which case, it's not hard to imagine that $Z_j$ are almost independent random variables.
Treating them as actually independent random variables, and using Hoeffding's inequality to get concentration for their sum gives us approximately the same result we got.

\subsection{Geometric and functional concentration for $\Pi_n$}
\label{sec:geom-funct-conc}

Recall the example of geometric and functional concentration on $D_n$ outlined in the previous section.
Geometric concentration refers to the phenomenon that the $\varepsilon$ neighbourhood of a set of measure greater than $\frac{1}{2}$ has almost full measure, and functional concentration refers to the fact that the value of a $1$-Lipschitz function concentrates near its mean.
It turns out that for a general metric probability space, if one has geometric concentration, one has functional concentration, and vice versa.
In this section, we will look and geometric and functional concentration on the symmetric group $\Pi_n$.
We define the metric on $\Pi_n$ in the following manner.
\begin{align*}
  d(\pi, \pi^{\prime}) = \sum_{j=1}^n \mathbb{1}_{\pi(j) \neq \pi^{\prime}(j)}(j)
\end{align*}
We let the probability measure on $\Pi_n$ be the uniform measure.
We then have functional concentration for this probability metric space.
\begin{theorem}[Maurey \cite{maurey1979construction}]
  \label{thm:maurey}
  Let $f: \Pi_n \to \R$ be a $1$-Lipschitz function. Then for all $t > 0$, the following tail bound holds.
  \begin{align*}
    \P\left( \left| f(x) - \E f(x) \right| > t \right) \leq 2 \exp\left( - \frac{t^2}{16n} \right)
  \end{align*}
\end{theorem}
One can deduce the geometric version of this concentration result as a corollary.
\begin{corollary}
  For any $\varepsilon > 0$, and for any $A \subset \Pi_n$, if $\P(A) \geq \frac{1}{2}$,
  then $\P\left( A_\varepsilon^C \right) \leq \exp\left( -c^{\prime} \frac{\varepsilon^2}{n} \right)$, for some fixed constant $c^{\prime}$.
\end{corollary}

Lots of natural functions on $\Pi_n$ are $1$-Lipschitz.
\begin{example}
  Let $f(\pi)$ be the length of the longest increasing subsequence of the sequence $\left( \pi(1), \ldots, \pi(n) \right)$.
  Clearly, if $\pi^{\prime}$ differs from $\pi$ on $k$ elements, then the longest increasing subsequence can only become shorter by at most $k$ elements, i.e. by removing those elements from the subsequence if they appeared.
  This shows that $f$ is a $1$-Lipschitz function, and one can get strong concentration inequalities for $f$.
\end{example}

\begin{proof}[Proof of Theorem \ref{thm:maurey}]
  We begin by defining a filtration in order to construct the martingale required to use Azuma's inequality.
  Let $\cF_0$ be the trivial $\sigma$-algebra, and for $1 \leq j \leq n$, let $\cF_j$ be the $\sigma$-algebra generated by the sets $C_{k_1, \ldots, k_j}$, where $\{k_1, \ldots, k_j\}$ varies over all the $j$-element subsets of $\{1, \ldots, n\}$, defined in the following manner.
  \begin{align*}
    C_{k_1, \ldots, k_j} = \left\{ \pi \in \Pi_n\mid\ \pi(i) = k_i \right\}
  \end{align*}
  By taking the conditional expectation of $f$ with respect to $\cF_j$, we get the $j$\textsuperscript{th} element of the martingale.
  We wish to estimate the following martingale difference.
  \begin{align}
    \label{eq:martingale-diff}
    \E\left[ f(\pi) \mid \cF_j \right] - \E\left[ f(\pi) \mid \cF_{j-1} \right]
  \end{align}
  We estimate this difference using by \emph{coupling} two copies of the random variable $\pi$.
  Denote $J = \left\{ \pi(1), \ldots, \pi(j-1) \right\}$.
  Let $\sigma$ be a random transposition such that $\sigma(j) = j^{\prime}$ where $j^{\prime}$ is chosen uniformly from $\{j, j+1, \ldots, n\}$.
  Let $\pi^{\prime} = \pi \circ \sigma$: if $\pi$ is uniformly distributed, then so is $\pi^{\prime}$ (this can be checked by a simple counting argument).
  The random variables $\pi$ and $\pi^{\prime}$ are now coupled: in particular, $\pi(k) = \pi^{\prime}(k)$ for $k \leq j-1$.
  We use the coupled random variables to make \eqref{eq:martingale-diff} easier to compute.
  \begin{align*}
    \E\left[ f(\pi) \mid \cF_j \right] - \E\left[ f(\pi) \mid \cF_{j-1} \right]
    &= \E\left[ f(\pi) \mid \cF_j \right] - \E\left[ f(\pi^{\prime}) \mid \cF_{j-1} \right] \\
    &= \E\left[ f(\pi) \mid \cF_j \right] - \E\E_{\sigma} \left[ f(\pi \circ \sigma) \mid \cF_{j} \right] \\
    &= \E_{\pi} \left[ f(\pi) - \E_{\sigma} f(\pi \circ \sigma) \mid \cF_j \right]
  \end{align*}
  The key equality in this chain of equalities is when $\E\left[ f(\pi^{\prime}) \mid \cF_{j-1} \right]$ turns into $\E\E_{\infty}\left[ f(\pi \circ \sigma) \mid \cF_{j} \right]$.
  In the first term, we are fixing the first $j-1$ entries, and taking the average over the remaining entries.
  In the second term, we are first fixing the first $j$ entries, computing the average over the remaining terms, and then computing the average of the average as we vary the $j$\textsuperscript{th} term.
  It's clear that these two quantities must be equal.

  Using the fact that $f$ is $1$-Lipschitz, and $\sigma$ is a transposition, we see that $f(\pi)$ and $f(\pi \circ \sigma)$ differ by at most $2$.
  Thus the martingale difference is bounded by $2$, and we can apply Azuma's inequality.
  \begin{align*}
    \P\left( \left| f(\pi) - \E f(\pi) \right| > t \right) \leq 2 \exp \left( - \frac{t^2}{16n} \right)
  \end{align*}
\end{proof}
\begin{remark}
  If $f$ is not $1$-Lipschitz, but $L$-Lipschitz, rescaling appropriately gives a similar bound, with $16L^2n$ in the denominator, rather than $16n$.
\end{remark}

\section{Log-concave random variables}
\label{sec:log-concave-random}

We will now consider log-concave random variables. These will be helpful in proving subgaussian concentration results for Gaussian random vectors in $\R^n$.

\begin{definition}[Log-concave measures and random variables]
  A Borel measure in $\R^n$ is called log-concave if for any $\lambda \in [0,1]$ and any compact sets $A$ and $B$ in $\R^n$, the following holds.
  \begin{align*}
    \log\left(  \mu\left( \lambda A + (1-\lambda)B \right) \right) \geq
    \lambda \log(\mu(A)) + (1-\lambda) \log(\mu(B))
  \end{align*}
  Here, $\lambda A + (1-\lambda)B$ is the Minkoswki sum of the sets.

  An $\R^n$-valued random variable $X$ is said to be log-concave if the pushforward measure $\mu(A) \coloneqq \P(X \in A)$ is a log-concave measure.
\end{definition}
\begin{example}
  The Lebesgue measure on $\R^n$ is a log-concave measure.
  The Gaussian random vector, i.e. a vector with i.i.d copies of the standard normal in its coordinates is a log-concave random variable.
\end{example}
We will shortly describe the entire class of log-concave random variables.
\begin{definition}[Log-concave function]
  A function $f: \R^n \to [0, \infty]$ is called log-concave if $\log(f)$ is a concave function.
\end{definition}

\begin{theorem}[Borell]
  Let $\mu$ be a locally finite Borel measure in $\R^n$. Assume that $\dim(\mathrm{supp}(\mu)) = n$.
  Then the following conditions are equivalent.
  \begin{enumerate}[(i)]
  \item $\mu$ is log-concave.
  \item $\mu$ is absolutely continuous with respect to Lebesgue measure, and the Radon-Nikodym derivative is log-concave.
  \end{enumerate}
\end{theorem}
With this theorem, it's easy to see why the examples we gave are log-concave.

We will only prove one direction of Borell's theorem, i.e. $(ii) \implies (i)$.
To do so, we need the Prékopa–Leindler inequality
\begin{theorem}[Prékopa–Leindler inequality]
  Let $\lambda \in [0,1]$ and let $f$, $g$, and $h$ be functions $\R^n \to [0, \infty]$ be $L^1$ functions such that for all $x$ and $y$ in $\R^n$, the following inequality holds.
  \begin{align*}
    h(\lambda x + (1-\lambda)y) \geq f(x)^{\lambda} \cdot g(y)^{1-\lambda}
  \end{align*}
  Then we have an associated integral inequality.
  \begin{align*}
    \int_{\R^n} h(x) \dd x \geq \left( \int_{\R^n} f(x) \dd x \right)^{\lambda} \cdot \left( \int_{\R^n} g(x) \dd x \right)^{1 - \lambda}
  \end{align*}
\end{theorem}
\begin{proof}
  We will prove this result in two steps.
  \begin{description}
  \item[Step 1] Let $n = 1$. Assume that $f$ and $g$ are non-zero. Without loss of generality, we can assume that $\int_\R f(x) \dd x = \int_{\R} g(x) \dd x = 1$. If not, scale $f$, $g$, and $h$ appropriately.
    We just need to show that $\int_{\R} h(x) \dd x \geq 1$.

    To make things simpler, assume that $f$ and $g$ are continuous $L^1$ functions to $(0, \infty)$ (we will get rid of this assumption later).
    Define the functions $\phi$ and $\psi$ from $(0,1)$ to $\R$ in the following manner.
    \begin{align*}
      \int_{-\infty}^{\phi(t)} f(x) \dd x &= t \\
      \int_{-\infty}^{\psi(t)} g(x) \dd x &= t
    \end{align*}
    The functions $\phi$ and $\psi$ are well defined since $f$ and $g$ are positive.
    Moreover, since $f$ and $g$ are continuous, we have the following by the fundamental theorem of calculus.
    \begin{align*}
      f(\phi(t)) \cdot \phi^{\prime}(t) &= 1 \\
      g(\psi(t)) \cdot \psi^{\prime}(t) &= 1
    \end{align*}

    To evaluate the integral of $h$, we perform the following change of variables.
    \begin{align*}
      \theta(t) = \lambda \phi(t) + (1- \lambda)\psi(t)
    \end{align*}
    The map $\theta$ is a differentiable bijection from $(0,1)$ to $\R$.
    \begin{align*}
      \theta^{\prime}(t) = \frac{\lambda}{f(\phi(t))} + \frac{1-\lambda}{g(\psi(t))}
    \end{align*}
    With this change of coordinates, we can evaluate the integral of $h$.
    \begin{align*}
      \int_{\R} h(x) \dd x &= \int_0^1 h(\theta(t)) \theta^{\prime}(t) \dd t \\
                           &= \int_0^1 H(\lambda \phi(t) + (1- \lambda)\psi(t))
                             \cdot \left[ \frac{\lambda}{f(\phi(t))} + \frac{1-\lambda}{g(\psi(t))} \right] \dd t
    \end{align*}
    Using the log-concavity hypothesis, we can bound the integrand below.
    \begin{align*}
      &\int_0^1 H(\lambda \phi(t) + (1- \lambda)\psi(t))
      \cdot \left[ \frac{\lambda}{f(\phi(t))} + \frac{1-\lambda}{g(\psi(t))} \right] \dd t \\
      \geq &\int_{0}^1 f(\phi(t))^{\lambda} \cdot g(\psi(t))^{1-\lambda}
      \cdot \left[ \frac{\lambda}{f(\phi(t))} + \frac{1-\lambda}{g(\psi(t))} \right] \dd t
    \end{align*}
    The arithmetic-geometric-mean inequality tells us that for any $\lambda \in (0,1)$, and any positive $a$ and $b$, the following holds.
    \begin{align*}
      \lambda a + (1-\lambda)b \geq a^{\lambda} \cdot b^{1-\lambda}
    \end{align*}
    Plugging in this inequality in the above integral inequality, we get that the integral of $h$ is bounded below by $1$.

    To get rid of the assumption that $f$ and $g$ are continuous and strictly positive, note that we can approximate $f$ and $g$ by such functions if not the case.
  \item[Step 2] Assume that the result holds for all dimensions less than $n+1$. We prove it for dimension $n+1$.
    We have functions $f$, $g$, and $h$ from $\R^{n+1}$ to $[0, \infty]$ such that for any $t$ and $s$ in $\R$, and any $x$ and $y$ in $\R^n$, the following holds.
    \begin{align*}
      h(\lambda t + (1-\lambda)s, \lambda x + (1-\lambda)y)
      \geq f(t, x)^{\lambda} \cdot g(s,y)^{1-\lambda}
    \end{align*}
    Fix $t$ and $s$, and consider the functions as functions on $\R^n$.
    These functions satisfy the log-concave condition, and we can use the induction hypothesis to claim the following.
    \begin{align*}
      \int_{\R^n} h(\lambda t + (1-\lambda)s, x) \dd x
      \geq \left( \int_{\R^n} f(t,x) \dd x \right)^{\lambda} \cdot \left( \int_{\R^n} g(s,x) \dd x \right)^{1- \lambda}
    \end{align*}
    Denote the left hand term as $H(\lambda t + (1-\lambda)s)$, the first term on the right hand side by $F(t)$, and the second term on the right hand side by $G(s)$.
    Then the functions $F$, $G$, and $H$ are functions on $\R$ satisfying the hypotheses of the theorem, and thus satisfy the following integral inequality by the base case of the induction.
    \begin{align*}
      \int_\R H(t) \dd t \geq \left( \int_{\R} F(t) \dd t \right)^{\lambda} \cdot \left( \int_{\R} G(t) \dd t \right)^{1-\lambda}
    \end{align*}
    Unwrapping the definitions of $F$, $G$, and $H$, and using Fubini's theorem, we get the claimed integral inequality for $\R^{n+1}$.
  \end{description}
\end{proof}

We will consider several consequences of the Pr\'ekopa-Leindler inequality, starting with some elementary corollaries.
\begin{corollary}
  Let $X$ and $Y$ be independent log-concave random variables in $\R^n$, and let $E$ be a subspace of $\R^n$.
  Then:
  \begin{enumerate}[(1)]
  \item The random variable $W = P_EX$ is log-concave.
  \item For all real $a$ and $b$, $Z = aX + bY$ is log-concave.
  \end{enumerate}
\end{corollary}
\begin{proof}
  \begin{enumerate}[(1)]
  \item Without loss of generality, we can assume that the subspace $E$ is the subspace where the last $n-k$ coordinates are $0$, where $k = \dim(E)$.
    Given any $x \in \R^n$, we can write it as $(w, z)$, where $w$ is the projection to $E$.
    By log-concavity, we have the following inequality involving the density $f_X$ of $X$ for all $\lambda \in (0,1)$.
    \begin{align*}
      f_X(\lambda w_1 + (1-\lambda)w_2, \lambda z_1 + (1-\lambda) z_2)
      \geq \left( f_X(w_1, z_1) \right)^{\lambda} \cdot \left( f_X(w_2, z_2) \right)^{1-\lambda}
    \end{align*}
    By Pr\'ekopa-Leindler, we get the corresponding integral inequality when we integrate over $z$ keeping $w$ fixed.
    But the integral of $f_X$ in the last $n-k$ coordinates is precisely the density of the pushforward measure.
  \item Without loss of generality, assume $a=b=\frac{1}{\sqrt{2}}$.
    Observe that the random vector $(X, Y) \in \R^{2n}$ is log-concave, since the $X$ and $Y$ are independent, and the joint density is the product of the individual densities.
    Upon taking logarithm of the joint density, we get a sum of two concave functions, which must also be concave.
    Furthermore, $aX+bY$ can be obtained by projecting $(X,Y)$ to an appropriate subspace of $\R^{2n}$.
    The result then follows from $(1)$.
  \end{enumerate}
\end{proof}

Another corollary that we get from Pr\'ekopa-Leindler is the Brunn-Minkowski inequality.
\begin{corollary}[Brunn-Minkowski inequality]
  Let $K$ and $D$ be compact subsets of $\R^n$.
  Then the following inequality involving the Lebesgue measure of the Minkowski sum $K+D$ holds.
  \begin{align*}
    \left( m_n(K+D) \right)^{\frac{1}{n}} \geq \left( m_n(K) \right)^{\frac{1}{n}} + \left( m_n(D) \right)^{\frac{1}{n}}
  \end{align*}
\end{corollary}
\begin{proof}
  Let $A$ and $B$ be compact sets in $\R^n$ such that they have measure $1$.
  Let $f = \mathbb{1}_A$, $g = \mathbb{1}_B$ and let $g = \mathbb{1}_{\lambda A + (1-\lambda)B}$.
  Then we can verify that $f$, $g$, and $h$ satisfy the hypothesis of the Pr\'ekopa-Leindler inequality.
  To see why that must be the case, observe that for any $x \in A$ and $y \in B$, $\lambda x + (1-\lambda )y$ must be in $\lambda A + (1-\lambda)B$, by the definition of the Minkowski sum.
  Integrating the functions gives us the following inequality involving the Lebesgue measures.
  \begin{align*}
    m_n(\lambda A + (1-\lambda B)) \geq \left( m_n(A) \right)^{\lambda} \cdot \left( m_n(B) \right)^{1 - \lambda}
  \end{align*}
  Note that the right hand side is actually equal to $1$, so we get the following inequality.
  \begin{align*}
    m_n(\lambda A + (1-\lambda B))^{\frac{1}{n}} &\geq \lambda \left( m_n(A) \right)^{\frac{1}{n}} + (1-\lambda)(m_n(B))^{\frac{1}{n}} \\
    &= \left( m_n( \lambda A) \right)^{\frac{1}{n}} + (m_n( (1-\lambda)B))^{\frac{1}{n}}
  \end{align*}
  Letting $K = \lambda A$ and $D = (1-\lambda)B$, we prove the result in the case when $K$ and $D$ have measure less than or equal to $1$.
  Furthermore, since the inequality is homogeneous with respect to scalar multiplication, the general case follows.
\end{proof}

Another more involved application of Pr\'ekopa-Leindler inequality is the following result due to Khatri and Šidak.
\begin{theorem}[Khatri-Šidak]
  Let $\left\{g_1, \ldots, g_n\right\}$ be (not necessarily independent) centered normal random variables.
  Then for any $\{a_1, \ldots, a_n\}$ in $\R_+$, the following holds.
  \begin{align*}
    \P\left( |g_1| < a_1, |g_2| < a_2,\cdots, |g_n| < a_n \right)
    &\geq \prod_{i=1}^n \P\left( |g_i| < a_i \right)
  \end{align*}
\end{theorem}
\begin{proof}
  Let $G$ be the standard Gaussian vector in $\R^n$: there exist vectors $\{x_1, \ldots, x_n\}$ such that $g_i = \langle G, x_i \rangle$.
  The vectors $x_i$ can be easily recovered from the correlation matrix of the $g_i$.
  We express the inequality we want to prove in terms of $G$.
  \begin{align}
    \label{eq:10}
  \begin{gathered}
    \P\left( |\langle G, x_1 \rangle| < a_1, \cdots, |\langle G, x_n \rangle| < a_n \right)
    \geq \P\left( |\langle G, x_1 \rangle| < a_1, \cdots, |\langle G, x_{n-1} \rangle| < a_{n-1} \right) \\
    \cdot \P\left( |\langle G, x_n \rangle| < a_n \right)
  \end{gathered}
  \end{align}

  Define $K \subset \R^n$ as follows.
  \begin{align*}
    K \coloneqq \left\{ y \in \R^n \mid\ |\langle y, x_i \rangle| < a_i\text{ for $1\leq i \leq n-1$} \right\}
  \end{align*}
  Let $\gamma_n$ be the standard Gaussian measure in $\R^n$.
  Then \eqref{eq:10} can be rewritten in terms of $K$ and $\gamma_n$.
  \begin{align}
    \label{eq:11}
      \gamma_n(K \cap \left\{ |\langle y, x_n \rangle| < a_n \right\})
      \geq \gamma_n(K) \cdot \gamma_n\left( \left\{ |\langle y, x_n \rangle| < a_n \right\} \right)
  \end{align}
  Observe that $K$ is centrally symmetric and convex.
  We claim that \eqref{eq:11} holds for any convex centrally symmetric set.
  We prove this claim by inducting on $n$.

  Let $n=1$: then $K$ is the interval $[-b,b]$ for some constant $b$.
  Similarly, the set $\left\{ |\langle y, x_1 \rangle| < a_1 \right\}$ is just $(-a_1, a_1)$.
  The intersections of these two centrally symmetric sets is also centrally symmetric.
  This gives the following chain of inequalities, proving the claim for $n=1$.
  \begin{align*}
    \gamma_1(K \cap \left\{ |\langle y, x_1 \rangle| < a_1 \right\})
    &= \gamma_1\left( [-(b \wedge a_1), (b \wedge a_1)] \right) \\
    &\geq \gamma_1([-b, b]) \cdot \gamma_1((-a_1, a_1))
  \end{align*}

  Assume now that \eqref{eq:11} holds in dimension $n$.
  Let $K$ be a convex centrally symmetric body in $\R^{n+1}$ and let $S = \left\{ y \mid\ |\langle y, x_{n+1} \rangle| < a_{n+1} \right\}$.
  We can also assume that $x_n$ is the $n+1$\textsuperscript{th} basis vector.
  Then we can express the left hand side of the claimed inequality in the following manner.
  \begin{align*}
    \gamma_{n+1}(K \cap S)
    &= \int_{\R} \gamma_n\left( K \cap \left\{ y \mid\ y_{n+1} = t \right\} \right)\cdot \mathbb{1}_{[-a_{n+1}, a_{n+1}]}(t) \dd \gamma_1(t)
  \end{align*}
  We can now interpret the right hand side as the expectation of $f(t)$, where $t$ is the standard Gaussian random variable.
  Doing so lets us rewrite the right hand side in the following manner.
  \begin{align}
    \label{eq:12}
    \gamma_{n+1}(K \cap S)
    &= \int_{0}^{\infty} \gamma_1\left( \gamma_n\left( K \cap \{y \mid\ y_{n+1} \in [-a_{n+1}, a_{n+1}]\} \right) > s \right) \dd s
  \end{align}
  Let $E_s$ be a set defined in the following manner.
  \begin{align*}
    E_s \coloneqq \left\{ t \in \R \mid\ \gamma_n\left( K \cap \{y \mid\ y_{n+1} = t\} \right)  > s\right\}
  \end{align*}
  Then $E_s$ is centrally symmetric, because $K$ is centrally symmetric.
  Moreover, $E_s$ is convex, i.e. $E_s = [-\theta(s), \theta(s)]$.
  That is because $\psi(t) = \gamma_n(K \cap \{y \mid\ y_{n+1} = t\})$ is an even log-concave function, since this is a projection of a log-concave function.
  Therefore, $\psi(t)$ is decreasing for $t \geq 0$.
  Observe that we can express the quantity we care about in terms of $E_s$, using \eqref{eq:12}.
  \begin{align*}
    \gamma_{n+1}(K \cap S) &= \int_0^{\infty} \gamma_1\left( [-\alpha, \alpha] \cap E_s \right) \\
                           &\geq \int_0^{\infty} \gamma_1\left( [-\alpha, \alpha] \right) \cdot \gamma_1(E_s) \\
                           &= C_{\alpha} \cdot \int_0^{\infty} \gamma_1(E_s) \\
                           &= \gamma_{n+1}(S) \cdot \gamma_n(K)
  \end{align*}
  This completes the proof of the theorem.
\end{proof}
\begin{remark}
  A general version of this inequality, where rather than splitting up the normal random variables in groups of $n-1$ and $1$, one splits them up in groups of $n-m$ and $m$ was the Gaussian correlation inequality.
  This inequality, which was a conjecture until 2014, attracted a lot of attention and resisted efforts to prove it for decades, until it was by Thomas Royen\footnote{See the \href{https://www.quantamagazine.org/statistician-proves-gaussian-correlation-inequality-20170328/}{Quanta article} that outlines this discovery that was almost neglected by the larger mathematical community.}.
\end{remark}

% We will now look at a combinatorial application of the Khatri-Šidak inequality.
% \begin{theorem}[$6\sigma$ Theorem]
%   Let $a_{ij}$ be numbers in $[-1, 1]$, as $i$ and $j$ range from $1$ to $n$.
%   Then there exist $\{\varepsilon_i\}_{1 \leq i \leq n}$ in $\{-1, 1\}$ such that the following holds for all $i$.
%   \begin{align*}
%     \left| \sum_{j=1}^n \varepsilon_j a_j \right| \leq C \sqrt{n}
%   \end{align*}
% \end{theorem}
% \begin{remark}
%   If one randomly picks the $\varepsilon_i$, and uses Hoeffding's inequality to get the appropriate concentration inequality, one only ends up with an upper bound of the form $C \sqrt{n \log(n)}$.
% \end{remark}
% We will need a few lemmas in order to prove this theorem.
% Before we state them, we set up some notation.
% Let $\mathbf{g} = \left( g_1, \ldots, g_n \right)$ be the standard Gaussian vector.
% Let $\gamma_n(A)$ denote $\P\left( g \in A \right)$.
% \begin{lemma}
%   Let $\{a_1, \ldots, a_m\}$ be vectors in $\R^n$ such that $\norm{a_j}_2 \leq \sqrt{n}$ for all $j$.
%   Define the set $K_t$
% \end{lemma}

We will now prove a Gaussian concentration inequality using the Pr\'ekopa-Leindler inequality.
Let $\mathbf{g}$ be a standard Gaussian vector in $\R^n$.
Denote $\gamma_n(A)$ to be the probability that $g \in A$ for a Borel subset $A$.
\begin{theorem}[Geometric concentration inequality]
  \label{thm:guassian-geometric}
  Let $K$ be a closed set in $\R^n$.
  For $t > 0$ let $K_t$ be the $t$-neighbourhood of $K$.
  Then for any $t > 0$, the following holds.
  \begin{align*}
    \gamma_n\left( \left( K_t \right)^c \right) \leq \frac{\exp\left( - \frac{t^2}{4} \right)}{\gamma_n(K)}
  \end{align*}
\end{theorem}
\begin{proof}
  We will choose functions $f$ and $h$ from $\R^n$ to $[0, +\infty]$ and find $g: \R^n \to [0, \infty]$ such that the triple $(f,g,h)$ satisfies the hypotheses for Pr\'ekopa-Leindler inequality.
  Set $h$ and $f$ to be the following functions.
  \begin{align*}
    h(x) &= \exp\left( - \frac{\norm{x}_2^2}{2} \right) \\
    h(x) &= \exp\left( - \frac{\norm{x}_2^2}{2} \right) \cdot \mathbb{1}_K(x)
  \end{align*}
  We want to find a $g$ that satisfies the hypotheses of Pr\'ekopa-Leindler for $\lambda = \frac{1}{2}$, i.e. the following inequality for all $x$ and $y$ in $\R^n$.
  \begin{align*}
    h\left( \frac{x+y}{2} \right) \geq \left( f(x) \right)^{\frac{1}{2}} \cdot \left( g(y) \right)^{\frac{1}{2}}
  \end{align*}
  Without loss of generality, we can focus on the values of $x$ that are contained in $K$, otherwise, the right hand side is $0$, and the inequality holds.
  Taking logarithms on both sides gives us the following inequality that $g$ must satisfy for all $x \in K$ and $y \in \R^n$.
  \begin{align*}
    - \frac{1}{2} \norm{\frac{x+y}{2}}_2^2
    \geq \frac{1}{2}\left( - \frac{\norm{x}_2^2}{2} \right) + \frac{1}{2} \log(g(y))
  \end{align*}
  Moving terms, we get the following constraint on $\phi(y) \coloneqq \log(g(y))$.
  \begin{align*}
    \phi(y) \leq \inf_{x \in K} \left[ \frac{\norm{x}_2^2}{2} - \norm{\frac{x+y}{2}}_2^2 \right]
  \end{align*}
  Defining $\phi(y)$ to be the right hand side, which makes the inequality hold trivially.
  We can simplify terms to get a formula for $\phi(y)$.
  \begin{align*}
    \phi(y) &\coloneqq \inf_{x \in K} \left[ \frac{\norm{x}_2^2}{2} - \norm{\frac{x+y}{2}}_2^2 \right] \\
            &= \inf_{x \in K} \left[ \frac{\norm{x-y}_2^2}{4} \right] - \frac{\norm{y}_2^2}{2} \\
            &= \frac{ \mathrm{dist}(y, K)^2}{4}  - \frac{\norm{y}_2^2}{2}
  \end{align*}
  This gives us the formula for $g$.
  \begin{align*}
    g(y) = \exp\left( \frac{\mathrm{dist}(y, K)^2}{4} \right) \cdot \exp\left( - \frac{\norm{y}_2^2}{2} \right)
  \end{align*}
  The Pr\'ekopa-Leindler inequality then gives us the corresponding integral inequality, which we multiply by $\left( 2 \pi \right)^{- \frac{n}{2}}$ to be able to interpret the integrands as integrals against the Gaussian probability measure.
  \begin{align*}
    \int_{\R^n} \left( 2\pi \right)^{-\frac{n}{2}} h(x) \dd x
    &\geq \left( \int_{\R^n}\left( 2\pi \right)^{-\frac{n}{2}} f(x) \dd x \right)^{\frac{1}{2}}
      \left( \int_{\R^n}\left( 2\pi \right)^{-\frac{n}{2}} g(x) \dd x \right)^{\frac{1}{2}}
  \end{align*}
  Observe that the left hand side is $1$, the first term on the right hand side is the measure of the set $K$, and the second term is the expectation of the random variable $\exp\left( \frac{\mathrm{dist}(y, K)^2}{4} \right)$.
  \begin{align*}
    1 \geq \gamma_n(K) \cdot \E \exp\left( \frac{\mathrm{dist}(y, K)^2}{4} \right)^{\frac{1}{2}}
  \end{align*}
  The claim then follows from an application of Markov's inequality.
\end{proof}
From the geometric concentration inequality, we can immediately derive the functional version.
\begin{theorem}[Functional concentration inequality]
  Let $F: \R^n \to \R$ be a $1$-Lipschitz function.
  Set $M = \mathrm{Median}(F)$, i.e. $\P(F(\mathbf{g}) \leq M) \geq \frac{1}{2}$ and $\P\left( F(\mathbf{g}) \geq M \right) \geq \frac{1}{2}$, where $\mathbf{g}$ is a standard Gaussian vector.
  Then for any $t > 0$, $F(x)$ concentrates near the median in the following manner.
  \begin{align*}
    \P\left( \left| F(x) - M \right| > t \right) \leq 4 \exp\left( - \frac{t^2}{4} \right)
  \end{align*}
\end{theorem}
\begin{proof}
  Let $K$ be the set of all points $x$ in $\R^n$ such that $F(x) \leq M$.
  We have that $\gamma_n(K) \geq \frac{1}{2}$.
  Since $F$ is $1$-Lipschitz, we have that the set of points $x$ where $F(x) \geq M+t$ is contained in $\left( K_t \right)^c$.
  Applying Theorem \ref{thm:guassian-geometric} gives us the claimed inequality.
  \begin{align*}
    \P\left( F(x) < M + t \right) &\leq \gamma_n(\left( K_t \right)^c) \\
                                  &\leq \frac{\exp\left( - \frac{t^2}{4} \right)}{\gamma_n(K)} \\
    &\leq 2 \exp\left( - \frac{t^2}{4} \right)
  \end{align*}
  Doing so for the probability that $F(x) < M - t$ and taking union bound gives us the result.
\end{proof}
\begin{remark}
  We can also obtain a subgaussian concentration around the $\E F(x)$, possibly with a worse constant on the exponent by obtaining a uniform upper bound on the distance between the median and the mean.
\end{remark}

We also get an easy corollary about the concentration of the norm of a Gaussian matrix.
\begin{corollary}
  Let $G$ be an $n \times m$ matrix with i.i.d $\mathcal{N}(0, 1)$ entries.
  Then for all $t > 0$, the norm of $G$ concentrates around its mean in the manner described below.
  \begin{align*}
    \P\left( \left| \norm{G} - \E \norm{G} \right| > t \right) \leq 2 \exp(-ct^2)
  \end{align*}
\end{corollary}
\begin{proof}
  The matrix norm is a $1$-Lipschitz function with respect the the $L^2$ norm when the matrix is considered as a vector in $\R^{n \times m}$.
  The proof then follows from the functional concentration inequality.
\end{proof}

For the above estimate to be useful in practice, one also needs to know $\E\norm{G}$.
The expected value of the matrix norm depends on $n$ and $k$ in the following manner.
\begin{align*}
  c(\sqrt{n} + \sqrt{k}) \leq \E \norm{G} \leq C (\sqrt{n} + \sqrt{k})
\end{align*}
Here, $c$ and $C$ are some absolute constants.
The lower bound is easy to obtain, and left as an exercise.
To obtain the upper bound, it will be convenient to have the framework of packings, coverings, and nets.

\paragraph{Packings, coverings, and nets}

\begin{definition}
  Let $(X, d)$ be a metric space.
  Let $\varepsilon > 0$ and let $K \subset X$.
  A set $N \subset X$ is said to be an $\varepsilon$-net if the $\varepsilon$-neighbourhood of $N$ contains $K$.
  The minimal cardinality of $N$ for a fixed $\varepsilon$, denoted $N(K, d, \varepsilon)$ is called the covering number of $K$.
\end{definition}

\begin{definition}
  A set $P \subset K$ is called an $\varepsilon$-separated set if for all $x$ and $y$ in $P$, $d(x,y) > \varepsilon$.
  The maximal cardinality of a separating set for $K$, denoted by $P(K, d, \varepsilon)$ is called the packing number of $K$.
\end{definition}

We will be interested in obtaining precise relationships between the covering and packing numbers for compact sets $K$.

\begin{lemma}
  Let $K \subset X$ in the metric space $(X, d)$.
  Then for any $\varepsilon > 0$, the packing and covering numbers are commensurate as we scale $\varepsilon$.
  \begin{align*}
    P(K, d, 2 \varepsilon) \leq N(K, d, \varepsilon) \leq P(K, d, \varepsilon)
  \end{align*}
\end{lemma}
We skip the proof of this fact, since it is fairly elementary.

We can also relate the covering number to Lebesgue measure.
\begin{theorem}[Volumetric estimate]
  Let $X = \left( \R^n, \norm{\cdot}_X \right)$ be a normed space.
  Then for any $K \subset X$, and any $\varepsilon > 0$, the following holds.
  \begin{align*}
    N(K, \norm{\cdot}_X, \varepsilon) \leq \frac{m_n\left( K+\frac{\varepsilon}{2}B_X \right)}{m_n \left( \frac{\varepsilon}{2}B_X \right)}
  \end{align*}
  Here, $m_n$ is the Lebesgue measure, $B_X$ is the unit ball, and $+$ is the Minkowski sum operation.
\end{theorem}
\begin{proof}
  We will prove the inequality for the packing number, and since the packing number is always greater than or equal to the covering number, the result will follow.
  Let $\mathcal{P}$ be an $\varepsilon$-separated set.
  For all $x$ and $x^{\prime}$ in $\mathcal{P}$, if $x \neq x^{\prime}$, then $B_X\left( x, \frac{\varepsilon}{2} \right)$ and $B_X\left( x^{\prime}, \frac{\varepsilon}{2} \right)$ are disjoint.
  The union of these balls is contained in $K + \frac{\varepsilon}{2}B_X$.
  The result then follows.
\end{proof}
One can estimate the covering number of $B_X$ from this theorem, by setting $K = B_X$: this gives us that the covering number of $B_X$ is less than $\left( 1 + \frac{2}{\varepsilon} \right)^n$.

We now go back to our problem of estimating the upper bound for the expected norm of a standard Gaussian $n \times k$ matrix.
Let $A$ be a linear operator from the normed space $X = (\R^k, \norm{\cdot}_X)$ to $Y = \left( \R^n, \norm{\cdot}_Y \right)$.
\begin{lemma}
  Let $\mathcal{N}$ be an $\varepsilon$-net for $B_X$ for some $\varepsilon \in (0,1)$.
  Then we have the following estimate for the operator norm of $A$.
  \begin{align*}
    \norm{A}_{\mathrm{op}} \leq \frac{1}{1 - \varepsilon} \max_{x \in \mathcal{N}} \norm{Ax}_Y
  \end{align*}
\end{lemma}
\begin{proof}
  Let $y$ be the point $B_X$ where $\norm{Ay}_Y$ is maximized.
  Since $\mathcal{N}$ is an $\varepsilon$-net, there exists an $x \in \mathcal{N}$ within distance $\varepsilon$ of $y$.
  \begin{align*}
    \norm{A}_{\mathrm{op}} &= \norm{Ay}_Y \\
                           &= \norm{Ax + A(y-x)}_Y \\
                           &\leq \norm{Ax}_Y + \norm{A(y-x)}_Y \\
                           &\leq \max_{x \in \mathcal{N}} \norm{Ax}_Y + \varepsilon \norm{A}_{\mathrm{op}}
  \end{align*}
  This proves the result.
\end{proof}
If both the norms are the standard Euclidean norms, we get the following corollary by applying the lemma twice.
\begin{corollary}
  If $\mathcal{N}$ is an $\varepsilon$-net for $B_X$ and $\mathcal{M}$ is a $\varepsilon$-net for $B_Y$, then the $L^2$-operator norm of $A$ is bounded above by the following quantity.
  \begin{align*}
    \norm{A} \leq \frac{1}{(1-\varepsilon)^2} \max_{x \in \mathcal{N}} \max_{y \in \mathcal{M}} \langle Ax, y \rangle
  \end{align*}
\end{corollary}

We can use this results about the $L^2$-operator norm to get lower bounds on the expectation of the norm of the standard $n \times k$ Gaussian matrix.
We first need the following lemma.
\begin{lemma}
  For any $t > 0$, we have the following tail bound on $\norm{G}$.
  \begin{align*}
    \P\left( \norm{G} > C(\sqrt{n} + \sqrt{k}) + t \right) \leq 2 \exp\left( - \frac{t^2}{4} \right)
  \end{align*}
\end{lemma}
\begin{proof}
  Let $\mathcal{N}$ be a $\frac{1}{2}$-net for $B_2^k$ and $\mathcal{M}$ be $\frac{1}{2}$-net for $B_2^n$ such that their cardinalities are bounded by the following quantities, coming from the volumetric estimates.
  \begin{align*}
    |\mathcal{N}| &\leq \left( \frac{3}{\varepsilon} \right)^k \\
    |\mathcal{M}| &\leq \left( \frac{3}{\varepsilon} \right)^n
  \end{align*}
  From the corollary, we have the following upper bound on $\norm{G}$.
  \begin{align*}
    \norm{G} \leq 4 \max_{x \in \mathcal{N}} \max_{y \in \mathcal{M}} \langle Gx, y \rangle
  \end{align*}
  To get the tail estimate for this maximum of random variables, we can find a tail estimate for each $x$ and $y$, and then use the union bound.
  \begin{align*}
    \P\left( \langle Gx, y \rangle > s \right)
    &= \P \left( \sum_{i=1}^k  \sum_{j=1}^n g_{ij} x_i y_j > s\right)
  \end{align*}
  Observe that $\sum g_{ij} x_i y_j$ is distributed like $\mathcal{N}(0, \norm{x}_2^2 \norm{y}_2^2)$.
  Since $x$ and $y$ are in the unit ball, the product of their norms is less than $1$.
  We therefore get the individual tail bound is less than $\exp\left( - \frac{s^2}{2} \right)$.
  Plugging in the volumetric estimate gives the result.
\end{proof}
\begin{remark}
  This result also works for centered subgaussian random variables.
\end{remark}

\paragraph{Milman-Dvoretzky theorem}

We now consider another application of the Gaussian concentration inequality: the Milman-Dvoretzky theorem, which is a key ingredient of the Dvoretzky theorem.
\begin{theorem}[Dvoretzky]
  \label{thm:dvo}
  Let $Y = \left( \R^n, \norm{\cdot}_Y \right)$ be a normed space.
  For any $\varepsilon > 0$, there exists a Euclidean norm $\norm{\cdot}_\varepsilon$ on $\R^n$ and a linear subspace $E$ with $\dim(E) \geq c(\varepsilon) \cdot \log(n)$ such that for all $x \in E$, the Euclidean norm agrees with the ambient norm up to an error of $\varepsilon$.
  \begin{align*}
    (1-\varepsilon) \cdot \norm{x}_\varepsilon \leq \norm{x}_Y \leq (1+\varepsilon)\cdot \norm{x}_\varepsilon
  \end{align*}
\end{theorem}

Before we present the proof, we will need to set up some notation.
Let $b(Y)$ be $\max_{x \in S^{n-1}} \norm{x}_Y$, where $Y$ is the normed space in theorem \ref{thm:dvo}.
Let $\mathbf{g}$ be the standard Gaussian vector in $\R^n$: we define the Dvoretzky dimension $\mathrm{Dv}$ of $Y$ as follows.
\begin{align*}
  \mathrm{Dv}(Y) \coloneqq \left( \frac{\E \norm{g}_Y}{b(Y)} \right)^2
\end{align*}

We compute the Dvoretzky dimension for a few well known normed spaces.
\begin{example}
  \begin{enumerate}[(1)]
  \item $Y = \ell_1^n$: In this case $b(Y) = \sqrt{n}$, and $\E\norm{g}_Y = cn$ for a fixed constant $c$.
    This means that the Dvoretzky dimension of $\ell_1^n$ is $c^2n$, i.e. it is proportional to the real dimension.
  \item $Y = \ell_2^n$: In this case $b(Y) = 1$, and $\E\norm{g}_Y$ which is smaller than $\sqrt{n}(1 - o(1))$, and thus the Dvoretzky dimension is $O(n)$.
  \item $Y = \ell_{\infty}^n$: In this case $b(Y) = 1$, and $\E\norm{g}_Y$ is bounded above and below by $\sqrt{\log(n)}$ times appropriate constants.
    This the Dvoretzky dimension is $\Omega(\log(n))$.
  \end{enumerate}
\end{example}
\begin{fact}
  The Dvoretzky dimension cannot exceed the actual dimension of the normed space.
\end{fact}
We can now state the Milman-Dvoretzky theorem.
\begin{theorem}
  \label{thm:mil-dvo}
  Let $Y = \left( \R^n, \norm{\cdot}_Y \right)$ be a normed space.
  Let $\varepsilon > 0$ and let $E \subset \R^n$ be a random linear subspace of dimension $k = \lfloor c(\varepsilon) \mathrm{Dv}(Y) \rfloor$, where $c(\varepsilon)$ is a function of $\varepsilon$.
  Then with probability $1 - \exp\left( -C \varepsilon^2 k \right)$, the following inequality holds for all $x$ in $E$.
  \begin{align*}
    (1-\varepsilon)M(Y)\norm{x}_2 \leq \norm{x}_Y \leq (1+\varepsilon)M(Y)\norm{x}_2
  \end{align*}
  Here, $M(Y)$ only depends on $Y$.
\end{theorem}

\begin{example}
  % \begin{enumerate}[(1)]
  \item Let $Y = \ell_1^n$: In this case, we can let $k = \frac{c \varepsilon^2}{\log\left( \frac{1}{\varepsilon} \right)} \cdot n$
    This may seem counter-intuitive, since random sections of polytopes in $\R^n$ end up looking very close to ellipses.
  % \end{enumerate}
\end{example}

The notes end here, but the course covered a few more measure concentration results, and then moved on to the study of random processes, and in particular, subgaussian processes.

\printbibliography

\end{document}