\documentclass[inequalities.tex]{subfile}

\begin{document}

	\section{Tangent Line Trick}\label{sec:tangent}
	The \textit{tangent line trick} is a very useful trick which has been around for quite some time. \textcite{li_2006} demonstrates some problems with this trick. Here, we will discuss this in detail and show how it can tackle problems.

	Imagine that we are given an inequality in $a_{1},\ldots,a_{n}$. The inequality can be divided into a sum of expressions which is same for $a_{1},\ldots,a_{n}$. That is it can be written as $f(a_{1})+\ldots+f(a_{n})\geq g(a_{1},\ldots,a_{n})$. Now, we typically try to use Jensen's inequality in cases like this. But in many cases,  $f$ is not convex. Even so, we may still be able to prove something like
		\begin{align*}
			f(x)
				& \geq f(\bar{a})+(x-\bar{a})f'(\bar{a})
		\end{align*}
	where $\bar{a}=\frac{a_{1}+\ldots+a_{n}}{n}$. Summing this for $x=a_{1},\ldots,a_{n}$, we have
		\begin{align*}
			f(a_{1})+\ldots+f(a_{n})
				& \geq nf(\bar{a})
		\end{align*}
	The motivation sort of comes from basic calculus. By \textit{wishful thinking}, we may be able to prove that the slope between the points $(x,f(x))$ and $(\bar{a},f(\bar{a}))$ is at least the slope of the tangent line of $f(x)$ at $x=\bar{a}$. This is where the name comes from. This trick is specially useful if you are given the quantity $a_{1}+\ldots+a_{n}$. If the expressions involved are homogeneous, then you do not even need this value. You can use homogeneity to impose conditions such as $a+b+c=1$. For example, see this problem.
		\begin{problem}
			Let $a,b,c$ be positive real numbers. Prove that
				\begin{align*}
					\dfrac{1}{a(b+c)}+\dfrac{1}{b(c+a)}+\dfrac{1}{c(a+b)}
						& \geq \dfrac{27}{2(a+b+c)^{2}}
				\end{align*}

				\begin{solution}
					The inequality is homogeneous in $a,b,c$. So without loss of generality, we can normalize the inequality assuming that $a+b+c=1$. Then $0<a,b,c<1$, $\bar{a}=1/3$ and we get the transformation
						\begin{align*}
							\dfrac{1}{a(1-a)}+\dfrac{1}{b(1-b)}+\dfrac{1}{c(1-c)}
								& \geq \dfrac{27}{2}
						\end{align*}
					We define
						\begin{align*}
							f(x)
								& = \dfrac{1}{a(1-a)}
						\end{align*}
					and attempt to prove the inequality that is required by tangent line trick. Since
						\begin{align*}
							f'(x)
								& = \dfrac{2x-1}{x^{2}(1-x)^{2}}
						\end{align*}
					we have $f(1/3)=9/2$ and $f'(1/3)=-27/4$. Thus, if we can prove the inequality
						\begin{align*}
							f(x)
								& \geq \dfrac{9}{2}-\dfrac{27}{4}\left(x-\dfrac{1}{3}\right)
						\end{align*}
					we are done since we will have
						\begin{align*}
							f(a)+f(b)+f(c)
								& \geq 3\cdot f\left(\dfrac{1}{3}\right)\\
								& = \dfrac{27}{2}
						\end{align*}
					So we should check if the tangent line inequality indeed holds.
						\begin{align*}
							\dfrac{1}{x(1-x)}
								& \geq \dfrac{9}{2}-\dfrac{27}{4}\left(x-\dfrac{1}{3}\right)\\
							\iff \dfrac{1}{x(1-x)}
								& \geq \dfrac{54-81x+27}{12}\\
								& \geq \dfrac{27(1-x)}{4}\\
							\iff x(1-x)^{2}
								& \leq \dfrac{4}{27}
						\end{align*}
					We can check if this holds. If $g(x)=x(1-x)^{2}$, then
						\begin{align*}
							g'(x)
								& = -2x(1-x)+(1-x)^{2}\\
								& = (1-x)(1-3x)
						\end{align*}
					We have $g'(x)=0$ if $x\in\{1,1/3\}$ and
						\begin{align*}
							g''(x)
								& = -3(1-x)-(1-3x)\\
								& = 6x-4
						\end{align*}
					Since $0<x<1$ and $g''(1)=2>0$, $g''(1/3)=-2<0$, we have that $g(x)$ is maximum at $x=1/3$ in the interval $(0,1)$. Thus,
						\begin{align*}
							x(1-x)^{2}
								& \leq \dfrac{1}{3}\left(1-\dfrac{1}{3}\right)^{2}\\
								& = \dfrac{27}{4}
						\end{align*}
					We are finally done.
				\end{solution}

				\begin{remark}
					Tangent line is a neat trick but it may always not be pretty. But before you go all in, you can easily check if the desired inequality follows from
						\begin{align*}
							f(a_{1})+\ldots+f(a_{n})
								& \geq nf(\bar{a})
						\end{align*}
					or not. If it does, there is a good chance that $f(x)\geq f(\bar{a})+(x-\bar{a})f'(\bar{a})$ holds as well. Another indication that this might work is that equality occurs for $a_{1}=\ldots=a_{n}=\bar{a}$. In order to avoid calculation with fractional values, we could also assume $a_{1}+\ldots+a_{n}=n$ for normalization so $\bar{a}=1$. Also, we may sometimes have to deal with $\leq$ instead of $\geq$. Let us see an example of this type below.
				\end{remark}
		\end{problem}

		\begin{problem}
			Given positive real numbers $a,b,c$ such that $a+b+c\geq 3$. Prove that
				\begin{align*}
					\dfrac{1}{a^{2}+b+c}+\dfrac{1}{a+b^{2}+c}+\dfrac{1}{a+b+c^{2}}
						& \leq 1
				\end{align*}

				\begin{solution}
					We have equality in the case $a=b=c=1$. So, we may be optimistic that tangent line trick will work here. Using $b+c\geq 3-a$,
						\begin{align*}
							\dfrac{1}{a^{2}+b+c}+\dfrac{1}{a+b^{2}+c}+\dfrac{1}{a+b+c^{2}}
								& \leq \dfrac{1}{a^{2}+3-a}+\dfrac{1}{b^{2}+3-b}+\dfrac{1}{c^{2}+3-c}
						\end{align*}
					So we are done if we can prove that
						\begin{align*}
							\dfrac{1}{a^{2}+3-a}+\dfrac{1}{b^{2}+3-b}+\dfrac{1}{c^{2}+3-c}
								& \leq 1
						\end{align*}
					Let
						\begin{align*}
							f(x)
								& = \dfrac{1}{x^{2}+3-x}
						\end{align*}
					We have that
						\begin{align*}
							f'(x)
								& = \dfrac{1-2x}{(x^{2}+3-x)^{2}}
						\end{align*}
					Since $f(1)=1/3$ and $f'(1)=-1/9$, we need to prove the tangent line inequality
						\begin{align*}
							f(x)
								& \leq \dfrac{1}{3}-\dfrac{1}{9}(x-1)\\
							\iff \dfrac{1}{x^{2}+3-x}
								& \leq \dfrac{4-x}{9}\\
							\iff 5x^{2}+3-7x-x^{3}
								& \geq 0\\
							\iff x^{3}-5x^{2}+7x-3
								& \leq 0
						\end{align*}
					We can see that $(x-3)$ is a factor of this polynomial. So we can factor it easily.
						\begin{align*}
							\iff x^{2}(x-3)-2x(x-3)+x-3
								& \leq 0\\
							\iff (x-3)(x^{2}-2x+1)
								& \leq 0\\
							\iff (3-x)(x-1)^{2}
								& \geq 0
						\end{align*}
					This is obviously true. And summing up the tangent line inequality, we get the conclusion.
				\end{solution}
		\end{problem}

		\begin{problem}[USAMO $2003$]\label{prob:usamo2003}
			Let $a,b,c$ be positive real numbers. Prove that
				\begin{align*}
					\dfrac{(2a+b+c)^{2}}{2a^{2}+(b+c)^{2}}+\dfrac{(2b+c+a)^{2}}{2b^{2}+(c+a)^{2}}+\dfrac{(2c+a+b)^{2}}{2c^{2}+(a+b)^{2}}
						& \leq 8
				\end{align*}

				\begin{solution}
					Due to homogeneity, we can assume that $0<a,b,c<1$ and $a+b+c=1$ without loss of generality. Then we are required to prove
						\begin{align*}
							\dfrac{(1+a)^{2}}{2a^{2}+(1-a)^{2}}+\dfrac{(1+b)^{2}}{2b^{2}+(1-b)^{2}}+\dfrac{(1+c)^{2}}{2c^{2}+(1-c)^{2}}
								& \leq 8
						\end{align*}
					We can see that equality occurs for $a=b=c=1/3$. Define
						\begin{align*}
							f(x)
								& = \dfrac{(1+x)^{2}}{2x^{2}+(1-x)^{2}}\\
								& = \dfrac{(1+x)^{2}}{3x^{2}-2x+1}
						\end{align*}
					The tangent line inequality in this case is
						\begin{align*}
							f(x)
								& \leq f(\bar{a})+(x-\bar{a})f'(\bar{a})
						\end{align*}
					where
						\begin{align*}
							f'(x)
								& = \dfrac{2(3x^{2}-2x+1)(1+x)-2(1+x)^{2}(3x-1)}{(3x^{2}-2x+1)^{2}}\\
								& = \dfrac{2(1+x)}{3x^{2}-2x+1}-\dfrac{2(1+x)^{2}(3x-1)}{(3x^{2}-2x+1)^{2}}
						\end{align*}
					We have $f(1/3)=8/3$ and $f'(1/3)=4$ so if we can prove that
						\begin{align*}
							f(x)
								& \leq \dfrac{8}{3}+4\left(x-\dfrac{1}{3}\right)\\
								& = \dfrac{4(3x+1)}{3}
						\end{align*}
					for $0<x<1$, then the inequality follows.
						\begin{align*}
							f(x)
								& \leq \dfrac{4(3x+1)}{3}\\
							\iff \dfrac{(1+x)^{2}}{3x^{2}-2x+1}
								& \leq \dfrac{12x+3}{3}\\
							\iff 36x^{3}-15x^{2}-2x+1
									& \geq 0\\
							\iff (3x-1)^{2}(4x+1)
								& \geq 0
						\end{align*}
					This is obviously true. So, we have the inequality.
				\end{solution}
		\end{problem}
\end{document}


 \documentclass [12pt]{article}

\usepackage {amsmath}
\usepackage {amsthm}
\usepackage {amssymb}
\usepackage {graphicx}
\usepackage {float}
\usepackage {multirow}
\usepackage {xcolor}
\usepackage {algorithmic}
\usepackage [ruled,vlined,commentsnumbered,titlenotnumbered]{algorithm2e} \usepackage {array}
\usepackage {booktabs}
\usepackage {url}
\usepackage {parskip}
\usepackage [margin=1in]{geometry}
\usepackage [T1]{fontenc}
\usepackage {cmbright}
\usepackage [many]{tcolorbox}
\usepackage [colorlinks = true,
            linkcolor = blue,
            urlcolor  = blue,
            citecolor = blue,
            anchorcolor = blue]{hyperref}
\usepackage {enumitem}
\usepackage {xparse}
\usepackage {verbatim}
\usepackage{listings}
\usepackage{xcolor}
\lstset { %
    language=C++,
    backgroundcolor=\color{black!5}, % set backgroundcolor
    basicstyle=\footnotesize,% basic font setting
}
\newtheorem{theorem}{Theorem}
\newtheorem{remark}{Remark}
\newtheorem{lemma}[theorem]{Lemma}
\theoremstyle{definition}
\newtheorem{definition}{Definition}[section]

\DeclareTColorBox {Solution}{}{breakable, title={Solution}} \DeclareTColorBox {Solution*}{}{breakable, title={Solution (provided)}} \DeclareTColorBox {Instruction}{}{boxrule=0pt, boxsep=0pt, left=0.5em, right=0.5em, top=0.5em, bottom=0.5em, arc=0pt, toprule=1pt, bottomrule=1pt} \DeclareDocumentCommand {\Expecting }{+m}{\textbf {[We are expecting:} #1\textbf {]}} \DeclareDocumentCommand {\Points }{m}{\textbf {(#1 pt.)}}

\begin {document}

\vspace {1em}
\begin {Instruction}
Adapted From Virginia Williams' lecture notes.
\end {Instruction}

{\LARGE \textbf {COMP 285 (NC A\&T, Spr `22)}\hfill \textbf {Lecture 18} }

\begin{centering}
\section*{Strongly Connected Components}
\end{centering}

 \section{Connected components in undirected graphs}

 A connected component of an undirected graph $ G = (V, E)$ is a maximal set of vertices $S \subset V$ such that for each $u \in S$ and $v \in S$, there exists a path in $G$ from vertex $u$ to vertex $v$.

\begin{definition}[Formal Definition]
Let $u \equiv v$ if and only if $G$ has a path from vertex $u$ to vertex $v$ . This is an equivalence relation (it is symmetric, reflexive, and transitive). Then, a connected component of $G$ is an equivalence class of this relation $\equiv$. Recall that the equivalence class of a vertex $u$ over a relation $\equiv$ is the set of all vertices $v$ such that $u \equiv v$.
\end{definition}

 \subsection{Algorithm to find connected components in a undirected graph}

In order to find a connected component of an undirected graph, we can just pick a vertex and start doing a search (BFS or DFS) from that vertex. All the vertices we can reach from that vertex compose a single connected component. To find all the connected components, then, we just need to go through every vertex, finding their connected components one at a time by searching the graph. Note however that we do not need to search from a vertex v if we have already found it to be part of a previous connected component. Hence, if we keep track of what vertices we have already encountered, we will only need to perform one BFS for each connected component.

\begin{proof}
When searching from a particular vertex v , we will clearly never reach any nodes outside the connected component with DFS or BFS. So we just need to prove that we will in fact reach all connected vertices. We can prove this by induction: Consider the vertices at minimum distance $i$ from vertex $v$. Call these vertices ``level $i$'' vertices. If BFS or DFS successfully reaches all vertices at level $i$, then they must reach all vertices at level $i + 1$, since each vertex at distance $i + 1$ from v must be connected to some vertex at distance $i from v$ . This is the inductive step, and for the base case, DFS or BFS will clearly reach all vertices at level $0$ (just v itself). So indeed this algorithm will find each connected component correctly.
\end{proof}

The searches in the above algorithm take total time $O(|E| + |V |),$ because each BFS or DFS call takes linear time in the number of edges and vertices for its component, and each component is only searched once, so all searches will take time linear in the total number of edges and vertices.

\begin{figure}[ht!]
\centering
\includegraphics[scale=0.5]{scc_graph.png}
\caption{The strongly connected components of a directed graph}
\label{fig:scc_graph}
\end{figure}

\section{Connectivity in directed graphs}
How can we extend the notion of connected components to directed graphs?

\begin{definition}[Strongly connected component (SCC)]
A strongly connected component in a directed graph $G = (V, E)$ is a maximal set of vertices $S \subset V$ such that each vertex $v \in S$ has a path to each other vertex $u \in S$. This is the same as the definition using equivalence classes for undirected graphs, except now $u \equiv v$ if and only if there is a path from $u$ to $v$ AND a path from $v$ to $u$.
\end{definition}

\begin{definition}[Weakly connected compnent]
Let $G = (V, E)$ be a directed graph, and let $G'$ be the undirected graph that is formed by replacing each directed edge of $G$ with an undirected edge. Then the weakly connected components of $G$ are exactly the connected components of $G'$.
\end{definition}

\section{Algorithm to find strongly connected components of a directed graph}

The algorithm we present is essentially two passes of depth-first search, plus some extremely clever additional book-keeping. The algorithm is described in a top-down fashion in Algorithms \ref{alg:1} to \ref{alg:3}. Algorithm \ref{alg:1} describes the top level of the algorithm, and Algorithm \ref{alg:2} and Algorithm \ref{alg:3} describe the subroutines DFS-Loop and DFS. Read these procedures carefully before proceeding to the next section.

\begin{algorithm}
\caption{The top level of our SCC algorithm. The $f$-values and leaders are computed in the first and second calls to DFS-Loop, respectively (see below)}
\label{alg:1}
\begin{algorithmic}
\STATE \textbf{INPUT:} A directed graph $G = (V,E)$, in adjacency list representation. Assume that the vertices $V$ are labeled $1, 2, 3, \cdots, n$.
\STATE $G^{\text{rev}} \gets$ the graph $G$ after the orientation of all arcs have been reversed.
\STATE Run the DFS-Loop subroutine on $G^{\text{rev}}$, processing vertices in any arbitrary order, to obtain a finishing time $f(v)$ for each vertex $v \in V$.
\STATE Run the DFS-Loop subrouting on $G$, processing vertices in creasing order of $f(v)$, to assign a ``leader'' to each vertex $v \in V$. The leader of a vertex $v$ will be the source vertex that the DFS that discovered $v$ started from.
\STATE The strongly connected components of $G$ correspond to vertices of $G$ that share a common leader.
\end{algorithmic}
\end{algorithm}

\begin{algorithm}
\caption{The DFS-Loop subroutine}
\label{alg:2}
\begin{algorithmic}
\STATE \textbf{INPUT:} A directed graph $G = (V,E)$, in adjacency list representation.
\STATE Let global variable $t \gets 0$ \texttt{/* This keeps track of the number of vertices that have been fully explored */}
\STATE Let global variable $s \gets $NULL \texttt{/* This keeps track of the vertex from which the last DFS call was invoked */}
\FOR{$i = n, n-1, \cdots, 1$}
    \STATE \texttt{// In the first call, vertices are labeled $1, 2, \cdots, n$ arbitrarily. In the second call, vertices are labeled by their $f(v)$-values from the first call.}
    \IF{$i$ not yet explored}
        \STATE Let $s \gets i$ \texttt{/* Set the current source $s$ to $i$ All vertices discovered from the below DFS call will have their leaders set to $s$ */}
        \STATE DFS(G, i)
    \ENDIF
\ENDFOR
\end{algorithmic}
\end{algorithm}

\begin{algorithm}
\caption{The DFS subroutine. The $f$-values only need to be computed during the first call to DFS-Loop, and the ledaer values only need to be computed during the second call to DFS-Loop.}
\label{alg:3}
\begin{algorithmic}
\STATE \textbf{INPUT:} A directed graph $G = (V,E)$, in adjacency list representation, and source vertex $i \in V$.
\STATE Mark $i$ as explored. \texttt{/* It remains explored for the entire duration of the DFS-Loop call */}
\STATE leader($i) \gets s$
\FOR{arc($i,j$) in $G$}
    \IF{$j$ noto yet explored}
        \STATE DFS($G,j$)
    \ENDIF
\ENDFOR
\STATE $t \gets t + 1$
\STATE Let $f(i) \gets t$
\end{algorithmic}
\end{algorithm}

\begin{remark}
The algorithm in Algorithm \ref{alg:1} is a bit different than the one in CLRS/Lecture! The difference is that in these notes, we first run DFS on the reversed graph, and then we run it again on the original; in CLRS, we first run DFS on the original, and then the second time on the reversed graph. Is it the case that one of these two textbooks has messed it up? In fact, it doesn't matter: the SCCs of$G$ are the same as the SCCs of $G^{\text{rev}}$ , so both algorithms find exactly the same SCC decomposition.
\end{remark}

As we've seen, each invocation of DFS-Loop can be implemented in linear time (i.e., $O(|E|+ |V |)$), so this whole algorithm will take linear time (the bookkeeping of leaders and finishing times just adds a constant number of operations per each node).

\section{An Example}

But why on earth should this algorithm work? An example should increase its plausibility (though it certainly doesn't constitute a proof of correctness). Figure \ref{fig:g_rev_scc} displays a reversed graph $G^{\text{rev}}$, with its vertices numbered arbitrarily, and the $f$-values computed in the first call to DFS-Loop. In more detail, the first DFS is initiated at node $9$. The search must proceed next to node $6$. DFS then has to make a choice between two different adjacent nodes; we have shown the $f$-values that ensue when DFS visits node $3$ before node $8$.\footnote{Different choices of which node to visit next generate different sets of $f$-values, but our proof of correctness will apply to all ways of resolving these choices}. When DFS visits node $3$ it gets stuck; at this point node $3$ is assigned a finishing time of $1$. DFS backtracks to node $6$, proceeds to node $8$, then node $2$, and then node $5$. DFS then backtracks all the way back to node $9$, resulting in nodes $5, 2, 8, 6$, and $9$ receiving the finishing times $2, 3, 4, 5$, and $6$, respectively. Execution returns to DFS-Loop, and the next (and final) call to DFS begins at node $7$.

\begin{figure}[ht!]
\includegraphics[scale=0.8]{g_rev_scc.png}
\caption{Example exuection of the strongly connected components algorithm. Nodes tare labaled arbitrarily and their finishing times are shown.}
\label{fig:g_rev_scc}
\end{figure}

\begin{figure}[ht!]
\includegraphics[scale=0.8]{scc_alg.png}
\caption{Example execution of the strongly connected components algorithm. Nodes are labeled by their finishing times and their leaders are shown.}
\label{fig:scc_alg}
\end{figure}

Figure \ref{fig:scc_alg} shows the original graph (with all arcs now unreversed), with nodes labeled withtheir finishing times. The magic of the algorithm is now evident, as the SCCs of $G$ present themselves to us in order: since we call DFS on the nodes in decreasing order of their finishing times, the first call to DFS discovers the nodes 7-9 (with leader 9); the second the nodes 1,5, and 6 (with leader 6); and the third the remaining three nodes (with leader 4).

\end{document}


\section{Sequential colimits}

\emph{Note: This chapter currently contains only the statements of the definitions and theorems, but no proofs. I hope to make a complete version available soon.}

\subsection{The universal property of sequential colimits}

Type sequences are diagrams of the following form.
\begin{equation*}
\begin{tikzcd}
A_0 \arrow[r,"f_0"] & A_1 \arrow[r,"f_1"] & A_2 \arrow[r,"f_2"] & \cdots.
\end{tikzcd}
\end{equation*}
Their formal specification is as follows.

\begin{defn}
An \define{(increasing) type sequence} $\mathcal{A}$ consists of
\begin{align*}
A & : \N\to\UU \\
f & : \prd{n:\N} A_n\to A_{n+1}.
\end{align*}
\end{defn}

In this section we will introduce the sequential colimit of a type sequence.
The sequential colimit includes each of the types $A_n$, but we also identify each $x:A_n$ with its value $f_n(x):A_{n+1}$.
Imagine that the type sequence $A_0\to A_1\to A_2\to\cdots$ defines a big telescope, with $A_0$ sliding into $A_1$, which slides into $A_2$, and so forth.

As usual, the sequential colimit is characterized by its universal property.

\begin{defn}
\begin{enumerate}
\item A \define{(sequential) cocone} on a type sequence $\mathcal{A}$ with vertex $B$ consists of
\begin{align*}
h & : \prd{n:\N} A_n\to B \\
H & : \prd{n:\N} f_n\htpy f_{n+1}\circ H_n.
\end{align*}
We write $\mathsf{cocone}(B)$ for the type of cones with vertex $X$.
\item Given a cone $(h,H)$ with vertex $B$ on a type sequence $\mathcal{A}$ we define the map
\begin{equation*}
\mathsf{cocone\usc{}map}(h,H) : (B\to C)\to \mathsf{cocone}(B)
\end{equation*}
given by $f\mapsto (f\circ h,\lam{n}{x}\mathsf{ap}_f(H_n(x)))$.
\item We say that a cone $(h,H)$ with vertex $B$ is \define{colimiting} if $\mathsf{cocone\usc{}map}(h,H)$ is an equivalence for any type $C$.
\end{enumerate}
\end{defn}

\begin{thm}\label{thm:sequential_up}
Consider a cocone $(h,H)$ with vertex $B$ for a type sequence $\mathcal{A}$. The following are equivalent:
\begin{enumerate}
\item The cocone $(h,H)$ is colimiting.
\item The cocone $(h,H)$ is inductive in the sense that for every type family $P:B\to \UU$, the map
\begin{align*}
\Big(\prd{b:B}P(b)\Big)\to {}& \sm{h:\prd{n:\N}{x:A_n}P(h_n(x))}\\
& \qquad \prd{n:\N}{x:A_n} \mathsf{tr}_P(H_n(x),h_n(x))={h_{n+1}(f_n(x))}
\end{align*}
given by
\begin{equation*}
s\mapsto (\lam{n}s\circ h_n,\lam{n}{x} \mathsf{apd}_{s}(H_n(x)))
\end{equation*}
has a section.
\item The map in (ii) is an equivalence.
\end{enumerate}
\end{thm}

\subsection{The construction of sequential colimits}

We construct sequential colimits using pushouts.

\begin{defn}
Let $\mathcal{A}\jdeq (A,f)$ be a type sequence. We define the type $A_\infty$ as a pushout
\begin{equation*}
\begin{tikzcd}[column sep=large]
\tilde{A}+\tilde{A} \arrow[r,"{[\idfunc,\sigma_{\mathcal{A}}]}"] \arrow[d,swap,"{[\idfunc,\idfunc]}"] & \tilde{A} \arrow[d,"\inr"] \\
\tilde{A} \arrow[r,swap,"\inl"] & A_\infty.
\end{tikzcd}
\end{equation*}
\end{defn}

\begin{defn}
The type $A_\infty$ comes equipped with a cocone structure consisting of
\begin{align*}
\mathsf{seq\usc{}in} & : \prd{n:\N} A_n\to A_\infty \\
\mathsf{seq\usc{}glue} & : \prd{n:\N}{x:A_n} \mathsf{in}_n(x)=\mathsf{in}_{n+1}(f_n(x)).
\end{align*}
\end{defn}

\begin{constr}
We define
\begin{align*}
\mathsf{seq\usc{}in}(n,x)\defeq \inr(n,x) \\
\mathsf{seq\usc{}glue}(n,x)\defeq \ct{\glue(\inl(n,x))^{-1}}{\glue(\inr(n,x))}.
\end{align*}
\end{constr}

\begin{thm}
Consider a type sequence $\mathcal{A}$, and write $\tilde{A}\defeq\sm{n:\N}A_n$. Moreover, consider the map
\begin{equation*}
\sigma_{\mathcal{A}}:\tilde{A}\to\tilde{A}
\end{equation*}
defined by $\sigma_{\mathcal{A}}(n,a)\defeq (n+1,f_n(a))$. Furthermore, consider a cocone $(h,H)$ with vertex $B$.
The following are equivalent:
\begin{enumerate}
\item The cocone $(h,H)$ with vertex $B$ is colimiting.
\item The defining square
\begin{equation*}
\begin{tikzcd}[column sep=large]
\tilde{A}+\tilde{A} \arrow[r,"{[\idfunc,\sigma_{\mathcal{A}}]}"] \arrow[d,swap,"{[\idfunc,\idfunc]}"] & \tilde{A} \arrow[d,"{\lam{(n,x)}h_n(x)}"] \\
\tilde{A} \arrow[r,swap,"{\lam{(n,x)}h_n(x)}"] & A_\infty,
\end{tikzcd}
\end{equation*}
of $A_\infty$ is a pushout square.
\end{enumerate}
\end{thm}

\subsection{Descent for sequential colimits}

\begin{defn}
The type of \define{descent data} on a type sequence $\mathcal{A}\jdeq (A,f)$ is defined to be
\begin{equation*}
\mathsf{Desc}(\mathcal{A}) \defeq \sm{B:\prd{n:\N}A_n\to\UU}\prd{n:\N}{x:A_n}\eqv{B_n(x)}{B_{n+1}(f_n(x))}.
\end{equation*}
\end{defn}

\begin{defn}
We define a map
\begin{equation*}
\mathsf{desc\usc{}fam} : (A_\infty\to\UU)\to\mathsf{Desc}(\mathcal{A})
\end{equation*}
by $B\mapsto (\lam{n}{x}B(\mathsf{seq\usc{}in}(n,x)),\lam{n}{x}\mathsf{tr}_B(\mathsf{seq\usc{}glue}(n,x)))$.
\end{defn}

\begin{thm}
The map
\begin{equation*}
\mathsf{desc\usc{}fam} : (A_\infty\to\UU)\to\mathsf{Desc}(\mathcal{A})
\end{equation*}
is an equivalence.
\end{thm}

\begin{defn}
A \define{cartesian transformation} of type sequences from $\mathcal{A}$ to $\mathcal{B}$ is a pair $(h,H)$ consisting of
\begin{align*}
h & : \prd{n:\N} A_n\to B_n \\
H & : \prd{n:\N} g_n\circ h_n \htpy h_{n+1}\circ f_n,
\end{align*}
such that each of the squares in the diagram
\begin{equation*}
\begin{tikzcd}
A_0 \arrow[d,swap,"h_0"] \arrow[r,"f_0"] & A_1 \arrow[d,swap,"h_1"] \arrow[r,"f_1"] & A_2 \arrow[d,swap,"h_2"] \arrow[r,"f_2"] & \cdots \\
B_0 \arrow[r,swap,"g_0"] & B_1 \arrow[r,swap,"g_1"] & B_2 \arrow[r,swap,"g_2"] & \cdots
\end{tikzcd}
\end{equation*}
is a pullback square. We define
\begin{align*}
\mathsf{cart}(\mathcal{A},\mathcal{B}) & \defeq\sm{h:\prd{n:\N}A_n\to B_n} \\
& \qquad\qquad \sm{H:\prd{n:\N}g_n\circ h_n\htpy h_{n+1}\circ f_n}\prd{n:\N}\mathsf{is\usc{}pullback}(h_n,f_n,H_n),
\end{align*}
and we write
\begin{equation*}
\mathsf{Cart}(\mathcal{B}) \defeq \sm{\mathcal{A}:\mathsf{Seq}}\mathsf{cart}(\mathcal{A},\mathcal{B}).
\end{equation*}
\end{defn}

\begin{defn}
We define a map
\begin{equation*}
\mathsf{cart\usc{}map}(\mathcal{B}) : \Big(\sm{X':\UU}X'\to X\Big)\to\mathsf{Cart}(\mathcal{B}).
\end{equation*}
which associates to any morphism $h:X'\to X$ a cartesian transformation of type sequences into $\mathcal{B}$.
\end{defn}

\begin{thm}
The operation $\mathsf{cart\usc{}map}(\mathcal{B})$ is an equivalence.
\end{thm}

\subsection{The flattening lemma for sequential colimits}

The flattening lemma for sequential colimits essentially states that sequential colimits commute with $\Sigma$.

\begin{lem}
Consider
\begin{align*}
B & : \prd{n:\N}A_n\to\UU \\
g & : \prd{n:\N}{x:A_n}\eqv{B_n(x)}{B_{n+1}(f_n(x))}.
\end{align*}
and suppose $P:A_\infty\to\UU$ is the unique family equipped with
\begin{align*}
e & : \prd{n:\N}\eqv{B_n(x)}{P(\mathsf{seq\usc{}in}(n,x))}
\end{align*}
and homotopies $H_n(x)$ witnessing that the square
\begin{equation*}
\begin{tikzcd}[column sep=7em]
B_n(x) \arrow[r,"g_n(x)"] \arrow[d,swap,"e_n(x)"] & B_{n+1}(f_n(x)) \arrow[d,"e_{n+1}(f_n(x))"] \\
P(\mathsf{seq\usc{}in}(n,x)) \arrow[r,swap,"{\mathsf{tr}_P(\mathsf{seq\usc{}glue}(n,x))}"] & P(\mathsf{seq\usc{}in}(n+1,f_n(x)))
\end{tikzcd}
\end{equation*}
commutes. Then $\sm{t:A_\infty}P(t)$ satisfies the universal property of the sequential colimit of the type sequence
\begin{equation*}
\begin{tikzcd}
\sm{x:A_0}B_0(x) \arrow[r,"{\total[f_0]{g_0}}"] & \sm{x:A_1}B_1(x) \arrow[r,"{\total[f_1]{g_1}}"] & \sm{x:A_2}B_2(x) \arrow[r,"{\total[f_2]{g_2}}"] & \cdots.
\end{tikzcd}
\end{equation*}
\end{lem}

In the following theorem we rephrase the flattening lemma in using cartesian transformations of type sequences.

\begin{thm}
Consider a commuting diagram of the form
\begin{equation*}
\begin{tikzcd}[column sep=small,row sep=small]
A_0 \arrow[rr] \arrow[dd] & & A_1 \arrow[rr] \arrow[dr] \arrow[dd] &[-.9em] &[-.9em] A_2 \arrow[dl] \arrow[dd] & & \cdots \\
& & & X \arrow[from=ulll,crossing over] \arrow[from=urrr,crossing over] \arrow[from=ur,to=urrr] \\
B_0 \arrow[rr] \arrow[drrr] & & B_1 \arrow[rr] \arrow[dr] & & B_2 \arrow[rr] \arrow[dl] & & \cdots \arrow[dlll] \\
& & & Y \arrow[from=uu,crossing over]
\end{tikzcd}
\end{equation*}
If each of the vertical squares is a pullback square, and $Y$ is the sequential colimit of the type sequence $B_n$, then $X$ is the sequential colimit of the type sequence $A_n$.
\end{thm}

\begin{cor}
Consider a commuting diagram of the form
\begin{equation*}
\begin{tikzcd}[column sep=small,row sep=small]
A_0 \arrow[rr] \arrow[dd] & & A_1 \arrow[rr] \arrow[dr] \arrow[dd] &[-.9em] &[-.9em] A_2 \arrow[dl] \arrow[dd] & & \cdots \\
& & & X \arrow[from=ulll,crossing over] \arrow[from=urrr,crossing over] \arrow[from=ur,to=urrr] \\
B_0 \arrow[rr] \arrow[drrr] & & B_1 \arrow[rr] \arrow[dr] & & B_2 \arrow[rr] \arrow[dl] & & \cdots \arrow[dlll] \\
& & & Y \arrow[from=uu,crossing over]
\end{tikzcd}
\end{equation*}
If each of the vertical squares is a pullback square, then the square
\begin{equation*}
\begin{tikzcd}
A_\infty \arrow[r] \arrow[d] & X \arrow[d] \\
B_\infty \arrow[r] & Y
\end{tikzcd}
\end{equation*}
is a pullback square.
\end{cor}

\begin{exercises}
\item \label{ex:seqcolim_shift}
Show that the sequential colimit of a type sequence
\begin{equation*}
\begin{tikzcd}
A_0 \arrow[r,"f_0"] & A_1 \arrow[r,"f_1"] & A_2 \arrow[r,"f_2"] & \cdots
\end{tikzcd}
\end{equation*}
is equivalent to the sequential colimit of its shifted type sequence
\begin{equation*}
\begin{tikzcd}
A_1 \arrow[r,"f_1"] & A_2 \arrow[r,"f_2"] & A_3 \arrow[r,"f_3"] & \cdots.
\end{tikzcd}
\end{equation*}
\item \label{ex:seqcolim_contr}Consider a type sequence
\begin{equation*}
\begin{tikzcd}
A_0 \arrow[r,"f_0"] & A_1 \arrow[r,"f_1"] & A_2 \arrow[r,"f_2"] & \cdots
\end{tikzcd}
\end{equation*}
and suppose that $f_n\htpy \mathsf{const}_{a_{n+1}}$ for some $a_n:\prd{n:\N}A_n$. Show that the sequential colimit is contractible.
\item Define the $\infty$-sphere $\sphere{\infty}$ as the sequential colimit of
\begin{equation*}
\begin{tikzcd}
\sphere{0} \arrow[r,"f_0"] & \sphere{1} \arrow[r,"f_1"] & \sphere{2} \arrow[r,"f_2"] & \cdots
\end{tikzcd}
\end{equation*}
where $f_0:\sphere{0}\to\sphere{1}$ is defined by $f_0(\bfalse)\jdeq \inl(\ttt)$ and $f_0(\btrue)\jdeq \inr(\ttt)$, and $f_{n+1}:\sphere{n+1}\to\sphere{n+2}$ is defined as $\susp(f_n)$. Use \cref{ex:seqcolim_contr} to show that $\sphere{\infty}$ is contractible.
\item Consider a type sequence
\begin{equation*}
\begin{tikzcd}
A_0 \arrow[r,"f_0"] & A_1 \arrow[r,"f_1"] & A_2 \arrow[r,"f_2"] & \cdots
\end{tikzcd}
\end{equation*}
in which $f_n:A_n\to A_{n+1}$ is weakly constant in the sense that
\begin{equation*}
\prd{x,y:A_n} f_n(x)=f_n(y)
\end{equation*}
Show that $A_\infty$ is a mere proposition.
\end{exercises}


\section{Recursive CTEs}
Recursive CTEs iteratively traverse hierarchical data of arbitrary length, and similarly to algorithms contain a base case (stop condition) and its recursive step. The number of times to iterate does not have to be constant.

\begin{lstlisting}[language=SQL]
with recursive r (i) as (
	select 1
	-- non-recursive term
	union all
	-- recursive term
	select i + 1
	from r
	where i < 5)
select * from r;
\end{lstlisting}

Recursive expressions can be used to manipulate data in a hierarchical structure. The non-recursive part defines the base case with eventual deterministic checks, while the recursive table traverses the tree upwards applying conditions.

Every recursive query can be defined by a simple evaluation algorithm:
\begin{lstlisting}[language=Matlab]
workingTable = evaluateNonRecursive()
output workingTable
while workingTable is not empty
	workingTable = evaluateRecursive(workingTable)
	output workingTable
\end{lstlisting}

It is important to notice that the working table gets fully replaced at every iteration of the second part.

It is also possible to increment counters within RCTEs, and create trees. To avoid infinite loops in cyclic structures, using \texttt{UNION} instead of \texttt{UNION ALL} allows to remove duplicates and checking before writing a new value.

To avoid loops within graphs, a NOT EXISTS condition can be added to check whether a node has already been discovered.

Best practices to deal with graph-like data involve knowing its cardinality, to be aware of the maximum distance, yet most problems are Turing complete.

Postgres only allows one mention of the recursive relation in the recursive subquery.

\section{Window functions}
Window functions are versatile tools to implement time series analysis, percentiles and cumulative sums.

A window function can be seen as a special case of aggregate, evaluated after every clause but ORDER BY, but opposite to aggregation they do not change the result: only additional columns are computed.

The term window describes the set of rows on which the window operates, returning values from it.

Example (from Postgres documentation):
\begin{lstlisting}
select product_name,
	price,
	group_name,
	avg(price) over (
			partition by group_name
			)
from products
	inner join product_groups
	using (group_id);
\end{lstlisting}
The previous query returns the product name, the price, product group name, along with the average prices of each product group.

AVG works as window function, operating on the subset of rows specified by OVER.

PARTITION BY distributes the rows of the result set into groups, each one of them subject to AVG, returning the average price.

Calculations on window functions are always performed on the result set after the other clauses but ORDER BY.

General window functions syntax:
\begin{lstlisting}
window_function(arg1, arg2,..) OVER (
	[PARTITION BY partition_expression]
	[ORDER BY sort_expression [ASC | DESC] [NULLS {FIRST | LAST }]
	[frame_clause])
\end{lstlisting}

To summarize:
\begin{itemize}
	\item The window function is an aggregate function, such as SUM or AVG;
	\item The (optional) partitioning clause divides the rows into multiple groups;
	\item The ordering clause specifies the order of internal rows;
	\item The framing clause defines a subset of rows in the current partition to which the window function is applied.
\end{itemize}

Multiple window functions can also be applied in the same query. Some other examples which ignore framing are:
\begin{itemize}
	\item Ranking:
	\begin{itemize}
		\item \texttt{rank()} and \texttt{dense\_rank()}, returning the rank of the current row with or without gaps;
		\item \texttt{row\_number()};
		\item\texttt{ ntile(n)}, distributing data over buckets;
	\end{itemize}
	\item Distribution:
	\begin{itemize}
		\item \texttt{percent\_rank()} and \texttt{cume\_dist()}, returning the relative rank of the current row or peer group (rows with equal partitioning);
	\end{itemize}
	\item Navigation:
	\begin{itemize}
		\item \texttt{lag(expr, offset, default)} and \texttt{lead(expr, offset, default)} evaluate the expression respectively on preceding and following row.
	\end{itemize}
\end{itemize}

Framing can follow only a few specifications, not too complex:
\begin{itemize}
	\item \texttt{current row};
	\item \texttt{unbounded preceding} or \texttt{unbounded following}, first and last row in the partition;
	\item \texttt{range between unbounded preceding and current row}, default frame with order specified;
	\item \texttt{range between unbounded preceding and unbounded following}, default frame with order unspecified;
	\item \texttt{aggregates()}, computing aggregates over all tuples in current frame;
	\item \texttt{first\_value()}, \texttt{last\_value()}, \texttt{nth\_value()}, evaluating an expression on determined rows of the frame.
\end{itemize}

\subsection{Segment trees}
Segment trees are an useful data structure for efficient evaluation, used for storing information related to intervals (segments).

Querying a static segment tree allows knowing which of the stored segments contains a given element, using logarithmic time operations.

The set of existing values gets divided into possible interval endpoints, sorted from left to right. All leaves in the tree correspond to the elementary intervals induced by endpoints, while internal nodes are the union of two intervals.

Any interval is stored in the canonical set for at most two nodes at the same depth, therefore the required storage is $O(n \log n)$ with $n$ intervals.

Space for leaves is linear, and aggregated functions can be calculated on intervals very quickly.

Segment trees are useful for window functions, since intervals can be seen as windows, either static or sliding. For each call of a window function, a new segment tree gets built: the time for construction is just linear, since ordering is already performed by the ORDER BY clause.

\subsection{Other aggregates}
Newer version of Postgres allow calculation of more complex statistical aggregates, such as standard deviation, correlation and linear regression.

Mode and percentiles are also supported, yet they require materialization and sorting therefore they need a special syntax and the clause \texttt{within group}.

\subsubsection{Grouping sets}
A grouping set is a set of columns by which data is grouped, for instance using aggregates. They can be unified using UNION ALL.

Since uniting requires all sets to have the same number of columns, eventual inconsistencies must be filled with NULL: this is both lengthy and nonperforming (linear scans).

GROUPING SETS, a GROUP BY option, allows to define multiple grouping sets in the same query. It is possible to group by one or multiple columns, or even nothing. Missing values are automatically filled with NULL.

ROLLUP calculates all the grouping possibilities ($2^n$) and applies them.

\section{Database clusters}
Storing a large quantity of data on a single machine can have limitations: performance is restricted to hardware, and in case of failure transactions get lost.

Using multiple machines is therefore a valid solution to handle databases: clusters are transparent and present themselves as a single system, still maintaining consistency between nodes.

\subsection{ACID vs BASE}
ACID and BASE are two standards to describe the properties of databases. One could be defined as the opposite of the other, because their usage is distinct between relational models and NoSQL.

\begin{itemize}
	\item Atomic: all operations in a transaction must either succeed or be reverted;
	\item Consistent: each transaction is always propagated to all nodes;
	\item Isolated: transactions do not contend with one another;
	\item Durable: results are permanent, even with failures.
\end{itemize}

\begin{itemize}
	\item Basically Available: ensures an operating cluster yet not all data might be immediately accessible after a failure;
	\item Soft state: data needs to be periodically refreshed, it does not have a permanent state;
	\item Eventual consistency: consistency is not achieved all the time, but will be reached at some point after a number of updates.
\end{itemize}

Both models have advantages and disadvantages, but generally BASE properties are much looser and less strict than ACID.

BASE does not provide a consistent view of the data, which can lead to confusion and cannot be used for critical systems, therefore it should only be used when performance is more important than ACID guarantees.

\subsection{CAP theorem}
The CAP theorem states that is impossible for a distributed database to simultaneously achieve more than two of the following guarantees:
\begin{enumerate}
	\item Consistency (every read receives either the most recent write or an error);
	\item Availability (every request receives a response);
	\item Partition tolerance (system operating even with dropped messages or delays).
\end{enumerate}
Partition tolerance is not really an option: some systems cannot be partitioned (servers with no replication), and for a distributed system not guaranteeing partition tolerance would imply to never drop messages and never have a dying node.

The probability of any node to fail increases exponentially as the number of total nodes increase.

When a network partition failure happens, therefore, the choice has to be between canceling the operation decreasing availability or proceed propagating it risking consistency.

If a system chooses to provide consistency, it will preserve atomic reads but refuse to respond to some requests.

If, on the other hand, availability is chosen, the system will respond to all requests, potentially returning outdated information and accepting conflicting writes.

\subsection{Transaction handling}
In distributed systems, transactions should be atomic. This means there are two options for ending one:
\begin{itemize}
	\item Commit, the transaction was successful and it can become persistent and visible to all nodes;
	\item Abort, the transaction failed or violates a constraint, therefore the system needs to be reverted to its previous state.
\end{itemize}
Since node crashes in a distributed system are independent, the protocol needs to be extended to ensure atomicity of transactions: a valid method is two-phase locking (2PL).

It is characterized by the presence of a coordinator node which allows $n$ agents of a system $A_1, A_2, \dots, A_n$ to either all persist the changes of $T$ or all discard them.

This is a commitment protocol that coordinates all processes participating in a distributed atomic transaction, achieving its purpose using logging of the states to aid recovery procedures.

The two phases of the protocol in a normal execution are:
\begin{itemize}
	\item Commit-request phase, in which a coordinator process attempts to prepare all the processes participating to the transaction to take the necessary steps and vote whether the operation can be committed;
	\item Commit phase, based on the votes, whose result gets communicated to the participants and the needed actions to commit or abort are taken.
\end{itemize}

Message flow in 2PL with 4 agents:
\begin{figure}[h]
	\includegraphics[scale=0.4]{2pc.png}
	\centering
\end{figure}

Issues of this method include the crash of nodes (both coordinator and agent) or potential lost messages.

\subsection{Replication}
Replication is a technique consisting in keeping a copy of the dataset in each machine. Redundant resources improve query throughput, can accommodate node failures and solve locality issues.

Access to a replicated entity is typically uniform with access to a single non-replicated entity, and copying should be transparent to external users.

Systems can also use a master-slave scheme, predominant in high availability clusters, where a single node is designated to process all requests.

\subsubsection{Horizontal partitioning}
Horizontal sharding involves every machine having only a chunk of the dataset. Query runtimes are improved, yet communication between shards should be minimized, due to network throughput.

Performance, in fact, relies heavily on the used interconnect, but allows nodes to have much less occupied memory so that dataset eveb bigger than the capacity of the machine can be accommodated.

Optimization of queries imply processing clauses separately, locally on each node, and aggregating results later so that the least amount of data is sent.

\subsubsection{Shard allocation}
Shards can be allocated to the cluster machines in different ways:
\begin{itemize}
	\item Every machine receives one shard;
	\item Every machine receives multiple shards.
\end{itemize}
Having more shards than machines results in better skew handling: they can take advantage of the resources available on nodes, increasing performance.

Also, there is no guarantee that the chosen hashing is uniform, and when values in between different shards are queried, it is likely that they are not all on the same machine.

Replication and sharding can be applied together, combining the benefits of both, but this leads to an increased resource consumption.
\begin{figure}[h]
	\includegraphics[scale=0.5]{replication_sharding.png}
	\centering
\end{figure}

\subsection{Joins}
Joins in a distributed system are not different than local ones in presence of a replicated environment, yet can become complicated when data is sharded across multiple machines.

\subsubsection{Co-located joins}
A co-located join is the most efficient way to join two large distributed tables: every table has a common distributed column and is sharded across machines in the same way.

All rows with the same column, therefore, are always on the same machine, even across different tables, so that relational operations can be performed within the groups.

The joins can be executed in parallel locally on the workers, returning results to the coordinator node.

\subsubsection{Distributed joins}
A distributed join involves data which is sharded across several nodes.

Data is usually filtered as much as possible before being sent, but data movement is still necessary for joining: all nodes send their part of the table to a single one for it to run operations.

This join is only performing when the number of rows is small, otherwise the node doing the join will be overwhelmed by the dataset size and unable to parallelize.

\subsubsection{Broadcast joins}
A broadcast join is applicable when only one side of the join requires a small set of rows (small table or very selective filter).

The side of the join with the small set is sent to every node in the cluster, then joined locally with the large table on the other side of the join.

\subsubsection{Considerations}
All the previous methods still needs to consider size of the tables and bandwidth: some operations might be faster reading the data from remote machines rather than on local disks, since cache size is limited.

In this case, data can be shared to multiple nodes, each of them storing a partition of the table and computing local results which are then aggregated.

Postgres supports read replicas, kept in sync for consistent reads: each application can send data either to the master or the slaves, and the master itself can also trigger replication.

However, it is not natively a distributed database: its alternative Greenplum (still based on Postgres) allows data distribution and partitioning, both of data and schema.

Distributed and partitioning schemes can be specified creating a table, adding DISTRIBUTED BY to define how data is spread, or PARTITION BY for partitioning within a segment.

Distributions may be round-robin (as evenly as possible, non-deterministic), or hashed (rows dispersed according to a hash function).

\begin{figure}[h]
	\includegraphics[scale=0.38]{greenplum.png}
	\centering
\end{figure}

MemSQL is another distributed database system based on SQL, which also allow replication, distributed and co-located joins.

A MemSQL cluster consists of aggregator and leaf nodes: aggregators handle metadata, monitoring and results of queries, while leaves act as storage layer and execute queries.

By default, MemSQL shards tables by their primary key, while manual sharding needs to be specified.

Distributed systems overall have more resources, but their synchronization might be expensive.

\section{Autonomous database systems}
Autonomous database systems are designed to remove the burden of managing DBMS from humans, focusing on physical database design and configuration/query tuning. The first attempts to program self-adaptive systems were in the 1970s, and have now evolved to learned components.

Indexing has been replacing with a neural network which predicts the location of a key, and transaction are scheduled through the machines by learned scheduling with unsupervised clustering. The most promising innovation is learned planning, deep learning algorithms which help running the query planner, estimating the best execution order of the operations.

A self driving DBMS is therefore a system that configures, manages and optimizes itself for the target database and its workload, using an objective function (throughput, latency) and some constraints. The two components of this are an agent (the one who makes decisions and learns over time) and an environment, the subject of the actions. Environments have a state used as feedback for the agent. This system can help exploring unknown configurations (and their consequences) and generalizing over applications or hardware.

\subsection{State modeling}
State modeling concerns the representation of the environment state, and has to be performed in an accurate way. The model is stochastic, non stationary and episodic (there is a fixed endpoint), and it can be represented with a Markov decision process model. The entire history of the DBMS is encoded in a state vector: its contents, design, workload and resources.

The table state contains number of tuples, attributes, cardinality and other values which altogether contribute to the choice of specific implementations (for instance indexes), but changes are constrained since the number of dimensions of the feature vectors always has to be the same. The content is approximated through a state vector, therefore there is no precise information regarding how the actual table looks like.

The knob configuration state depends on the machine, and is not really scalable. Not every configuration is applicable to servers, but one potential solution is to store hardware resources according to their percentages in respect of the available amount.

Feature vectors can be reduced (PCA), exacerbating instability, and hierarchical models help isolating components to reduce the propagation of changes.

Acquisition of data is done through targeted experiments while the system is running in production, training the model with end-to-end benchmarks of sample workloads. Instead of running the full system, micro-benchmarks can be run on a subset of components. This method is more effective and requires less iteration to converge.

To avoid slowing down the entire system, training is performed on replicas (master-slave architecture) having an agent recognize the resources and eventually propagating the best changes to the master. The application server primarily communicates with the master through reads and writes, and obtaining a complete view of all the operation is sometimes hard since not everything is sent to the replicas (failed queries, for instance).

An alternative is imitation learning: the model observes a tuned DBMS and tries to replicate those decisions. The state of the database still needs to be captured to extrapolate why changes have been applied, and training data will be sparse or noisy.

\subsection{Reward observation}
Rewards are classified as short-term and long-term, the latter more problematic since it is difficult to predict the workload trends. Transient hardware problems are hard to detect and could mask the true reward of an action, so current benchmarks have to be compared with historical data to reconsider recent events.

Many systems concern both OLTP and OLAP workloads, and changes in the objective functions may have a negative impact on either. Generally multiple policies define preference of one over th other in mixed environments.

Other common problems regard action selection policies, transparency and human interaction.

Foundations of Data Engineering/lectures/lecture4.tex


\documentclass[a4paper,man,natbib]{apa6}
\usepackage[english]{babel}
\usepackage[utf8x]{inputenc}
% Common Packages - Delete Unused Ones %
\usepackage{setspace}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{caption}
\graphicspath{ {./images/} }
% End Packages %
\title{Exercise Assignment 12}
\shorttitle{ES12}
\author{Brandon Hosley}
\date{2018 11 19}
\affiliation{Mike Davis}
%\abstract{}
\begin{document}
	\singlespacing
	\raggedbottom
	\maketitle
\subsection{1. You are stranded on a desert island and you only have NAND gates.  Construct the following using only NAND gates. For each of the following show the logic circuit with only NAND gates and the truth table. (Six Points)}
\emph{Create a NOT gate.}\\
	\begin{minipage}{2.5in}
		\includegraphics[width=\linewidth]{ES12p1not.png}
	\end{minipage}
	\begin{minipage}{2.5in}
		\begin{tabular}{ c | c }
			Input & Output \\
			\hline
			0 & 1 \\
			1 & 0 \\
		\end{tabular}
	\end{minipage} ~\\
\emph{Create an AND gate.} \\
	\begin{minipage}{2.5in}
		\includegraphics[width=\linewidth]{ES12p1and.png}
	\end{minipage}
	\begin{minipage}{2.5in}
		\begin{tabular}{ c c | c | c }
			\multicolumn{2}{c}{Input} & & Output \\
			A$_{0}$ & B$_{0}$ & A$_{1}$ & \\
			\hline
			0 & 0 & 1 & 0 \\
			0 & 1 & 1 & 0 \\
			1 & 0 & 1 & 0 \\
			1 & 1 & 0 & 1 \\
		\end{tabular}
	\end{minipage} ~\\
\emph{Create an OR gate.} \\
	\begin{minipage}{2.5in}
		\includegraphics[width=\linewidth]{ES12p1or.png}
	\end{minipage}
	\begin{minipage}{2.5in}
		\begin{tabular}{ c c | c c | c }
			\multicolumn{3}{c}{Input} & & Output \\
			A$_{0}$ & B$_{0}$ & A$_{1}$ & B$_{1}$ & \\
			\hline
			0 & 0 & 1 & 1 & 0 \\
			0 & 1 & 1 & 0 & 1 \\
			1 & 0 & 0 & 1 & 1 \\
			1 & 1 & 0 & 0 & 1 \\
		\end{tabular}
	\end{minipage} ~\\
\emph{Create a NOR gate.} \\
	\begin{minipage}{2.5in}
		\includegraphics[width=\linewidth]{ES12p1nor.png}
	\end{minipage}
	\begin{minipage}{2.5in}
		\begin{tabular}{ c c | c c c | c }
			\multicolumn{4}{c}{Input} & & Output \\
			A$_{0}$ & B$_{0}$ & A$_{1}$ & B$_{1}$ & A$_{2}$ & \\
			\hline
			0 & 0 & 1 & 1 & 0 & 1 \\
			0 & 1 & 1 & 0 & 1 & 0 \\
			1 & 0 & 0 & 1 & 1 & 0 \\
			1 & 1 & 0 & 0 & 1 & 0 \\
		\end{tabular}
	\end{minipage} ~\\
\emph{Create and XOR gate.} \\
	\begin{minipage}{2.5in}
		\includegraphics[width=\linewidth]{ES12p1xor.png}
	\end{minipage}
	\begin{minipage}{2.5in}
		\begin{tabular}{ c c | c c c | c }
			\multicolumn{4}{c}{Input} & & Output \\
			A$_{0}$ & B$_{0}$ & A$_{1}$ & A$_{2}$ & B$_{2}$ & \\
			\hline
			0 & 0 & 1 & 1 & 1 & 0 \\
			0 & 1 & 1 & 1 & 0 & 1 \\
			1 & 0 & 1 & 0 & 1 & 1 \\
			1 & 1 & 0 & 1 & 1 & 0 \\
		\end{tabular}
	\end{minipage} ~\\
\emph{Inverting the output of an XOR gate creates an XNOR gate.  Create an XNOR gate.} \\
	\begin{minipage}{2.5in}
		\includegraphics[width=\linewidth]{ES12p1xnor.png}
	\end{minipage}
	\begin{minipage}{2.5in}
		\begin{tabular}{ c c | c c c c | c }
			\multicolumn{5}{c}{Input} & & Output \\
			A$_{0}$ & B$_{0}$ & A$_{1}$ & A$_{2}$ & B$_{2}$ & A$_{3}$ & \\
			\hline
			0 & 0 & 1 & 1 & 1 & 0 & 1 \\
			0 & 1 & 1 & 1 & 0 & 1 & 0 \\
			1 & 0 & 1 & 0 & 1 & 1 & 0 \\
			1 & 1 & 0 & 1 & 1 & 0 & 1 \\
		\end{tabular}
	\end{minipage} ~\\

\subsection{2. You are stranded on a desert island and you only have NOR gates.  Construct the following using only NOR gates. For each of the following, show the logic circuit with only NOR gates and the truth table. (Six Points)}
\emph{Create a NOT gate.} \\
	\begin{minipage}{2.5in}
		\includegraphics[width=\linewidth]{ES12p2not.png}
	\end{minipage}
	\begin{minipage}{2.5in}
		\begin{tabular}{ c | c }
			Input & Output \\
			\hline
			0 & 1 \\
			1 & 0 \\
		\end{tabular}
	\end{minipage} ~\\
\emph{Create an AND gate.} \\
		\begin{minipage}{2.5in}
		\includegraphics[width=\linewidth]{ES12p2and.png}
	\end{minipage}
	\begin{minipage}{2.5in}
		\begin{tabular}{ c c | c c | c }
			\multicolumn{3}{c}{Input} & & Output \\
			A$_{0}$ & B$_{0}$ & A$_{1}$ & B$_{1}$ & \\
			\hline
			0 & 0 & 1 & 1 & 0 \\
			0 & 1 & 1 & 0 & 0 \\
			1 & 0 & 0 & 1 & 0 \\
			1 & 1 & 0 & 0 & 1 \\
		\end{tabular}
	\end{minipage} ~\\
\emph{Create an OR gate.} \\
		\begin{minipage}{2.5in}
		\includegraphics[width=\linewidth]{ES12p2or.png}
	\end{minipage}
	\begin{minipage}{2.5in}
		\begin{tabular}{ c c | c | c }
			\multicolumn{2}{c}{Input} & & Output \\
			A$_{0}$ & B$_{0}$ & A$_{1}$ & \\
			\hline
			0 & 0 & 1 & 0 \\
			0 & 1 & 0 & 1 \\
			1 & 0 & 0 & 1 \\
			1 & 1 & 0 & 1 \\
		\end{tabular}
	\end{minipage} ~\\
\emph{Create a NAND gate.} \\
		\begin{minipage}{2.5in}
		\includegraphics[width=\linewidth]{ES12p2nand.png}
	\end{minipage}
	\begin{minipage}{2.5in}
		\begin{tabular}{ c c | c c c | c }
			\multicolumn{4}{c}{Input} & & Output \\
			A$_{0}$ & B$_{0}$ & A$_{1}$ & B$_{1}$ & A$_{2}$ & \\
			\hline
			0 & 0 & 1 & 1 & 0 & 1 \\
			0 & 1 & 1 & 0 & 0 & 1 \\
			1 & 0 & 0 & 1 & 0 & 1 \\
			1 & 1 & 0 & 0 & 1 & 0 \\
		\end{tabular}
	\end{minipage} ~\\
\emph{Create and XOR gate.} \\
	\begin{minipage}{2.5in}
		\includegraphics[width=\linewidth]{ES12p2xor.png}
	\end{minipage}
	\begin{minipage}{2.5in}
		\begin{tabular}{ c c | c c c c | c }
			\multicolumn{5}{c}{Input} & & Output \\
			A$_{0}$ & B$_{0}$ & A$_{1}$ & A$_{2}$ & B$_{2}$ & A$_{3}$ & \\
			\hline
			0 & 0 & 1 & 0 & 0 & 1 & 0 \\
			0 & 1 & 0 & 1 & 0 & 0 & 1 \\
			1 & 0 & 0 & 0 & 1 & 0 & 1 \\
			1 & 1 & 0 & 0 & 0 & 1 & 0 \\
		\end{tabular}
	\end{minipage} ~\\
\emph{Inverting the output of an XOR gate creates an XNOR gate.  Create an XNOR gate.} \\
	\begin{minipage}{2.5in}
		\includegraphics[width=\linewidth]{ES12p2xnor.png}
	\end{minipage}
	\begin{minipage}{2.5in}
		\begin{tabular}{ c c | c c c | c }
			\multicolumn{4}{c}{Input} & & Output \\
			A$_{0}$ & B$_{0}$ & A$_{1}$ & A$_{2}$ & B$_{2}$ & \\
			\hline
			0 & 0 & 1 & 0 & 0 & 1 \\
			0 & 1 & 0 & 1 & 0 & 0 \\
			1 & 0 & 0 & 0 & 1 & 0 \\
			1 & 1 & 0 & 0 & 0 & 1 \\
		\end{tabular}
	\end{minipage} ~\\

\nocite{warford10}
\bibliographystyle{apacite}
\bibliography{CS} %link to relevant .bib file
\end{document}


\documentclass[12pt]{article}
\usepackage{fullpage}
\usepackage{minted}
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{hyperref}
\usepackage[utf8x]{inputenc}

\title{Array manipulations as functional programming}
\author{Jim Pivarski}
\date{\today}

\begin{document}
\maketitle

\setlength{\parskip}{0.5\baselineskip}

\section*{Introduction}

The central features of an array library like Numpy or Awkward Array simplify if we think of arrays as functions and these features as function composition. A one-dimensional array of \mintinline{python}{dtype} $d$ (e.g.\ \mintinline{python}{int32} or \mintinline{python}{float64}) can be thought of as a function from integer indexes to members of $d$. Thus,
\[ \mintinline{python}{array[i]} \]
\noindent becomes
\[ \mintinline{python}{array}: \mathbb{Z} \to d \]
\noindent because given an integer \mintinline{python}{i} $\in \mathbb{Z}$, it returns a value in $d$.  In Python, this function is the implementation of the array's \mintinline{python}{__getitem__} method.

Specified this way, this is a partial function\footnote{\url{https://en.wikipedia.org/wiki/Partial_function}}---for some integers, it raises an exception rather than returning a value in $d$. (Integers greater than or equal to the array's length or less than its negated length, if the array implements Python's negative indexing, are outside the bounds of the array and do not return a value.) It can be made into a total function by restricting the domain to $[0, n)$ where $n$ is the length of the array:
\[ \mintinline{python}{array}: [0, n) \to d. \]

We can choose $[0, n)$ as the domain and work with total functions or $\mathbb{Z}$ as the domain and work with partial functions---it is a matter of the granularity of the type system. Numpy has a single type \mintinline{python}{ndarray} for all arrays (effectively untyped), Numba has an array type that depends on the array's dimension, and C++ has a \mintinline{c++}{std::array<dtype, n>} type that depends on the exact size (\mintinline{c++}{n}) of the array, like our functional description above. As we'll see later, a consequence of this specificity is that the return value of some functions will depend on the values given to that function, a feature known as dependent types\footnote{\url{https://en.wikipedia.org/wiki/Dependent_type}}.

In this note, we'll describe arrays as total functions in a dependent type system.

\section*{Multidimensional arrays}

Numpy arrays can have arbitrarily many dimensions, referred to as the array's \mintinline{python}{shape}. The \mintinline{python}{shape} is a tuple of positive integers specifying the length of each dimension: $(n_1, n_2, \ldots, n_k)$ is a rank-$k$ tensor ($k = 1$ is a vector, $k = 2$ is a matrix, etc.).

To get values of type $d$ from a rank-$k$ array of \mintinline{python}{dtype} $d$, we must specify $k$ integers, each in a restricted domain $[0, n_i)$. In Numpy syntax, this is an implicit Python \mintinline{python}{tuple} between the square brackets:
\[ \mintinline{python}{array[i1, i2, ..., ik]} \]
\noindent In mathematical syntax, we can represent a $k$-tuple as a cartesian product,
\[ [0, n_1) \times [0, n_2) \times \ldots \times [0, n_k) \]
\noindent so the function corresponding to this array is
\[ \mintinline{python}{array}: [0, n_1) \times [0, n_2) \times \ldots \times [0, n_k) \to d. \]

A function with multiple arguments can be replaced with functions of one argument that each return a function, a process known as currying\footnote{\url{https://en.wikipedia.org/wiki/Currying}}. For example, the function above can be replaced with
\[ \mintinline{python}{array}: [0, n_1) \to [0, n_2) \to \ldots \to [0, n_k) \to d \]
\noindent by noting that
\[ \mintinline{python}{array[i1]} \]
\noindent returns an array of rank $k - 1$ and \mintinline{python}{dtype} $d$, which is a function
\[ \mintinline{python}{array[i1]}: [0, n_2) \to \ldots \to [0, n_k) \to d \]
\noindent (and so on, for each dimension). In fact, Numpy's indexing syntax illustrates this clearly:
\[ \mintinline{python}{array[i1, i2, i3] == array[i1][i2][i3]} \]
\noindent for any \mintinline{python}{i1}, \mintinline{python}{i2}, \mintinline{python}{i3} that satisfy a three-dimensional array's domain.

\section*{Record arrays}

Numpy also has record arrays\footnote{\url{https://docs.scipy.org/doc/numpy/user/basics.rec.html}} for arrays of record-like structures (e.g.\ \mintinline{c++}{struct} in C). In Numpy, the named fields and their types are considered part of the array's \mintinline{python}{dtype}, but they are accessed through the same square bracket syntax as elements of the array's \mintinline{python}{shape}:
\[ \mintinline{python}{array[i1, i2, i3][fieldname]} \]
\noindent where \mintinline{python}{fieldname} is a string, the name of one of the record's fields. (Numpy does not allow the \mintinline{python}{fieldname} to be uncurried---it must be in a different set of brackets from \mintinline{python}{i1}, \mintinline{python}{i2}, \mintinline{python}{i3}.)

Since record fields are accessed through a similar syntax, let's consider it part of the array's functional type, making no distinction between \mintinline{python}{shape} elements and field names. For a record type in which string-valued field names $s_1$, $s_2$, \ldots, $s_m$ map to \mintinline{python}{dtypes} $d_1$, $d_2$, \ldots, $d_m$, we can write
\begin{align*}
\mintinline{python}{recarray}: [0, n) \to &\ s_1 \to d_1 \\
 &\ s_2 \to d_2 \\
 &\ \ldots \\
 &\ s_m \to d_m
\end{align*}
\noindent to represent a one-dimensional record array of length $n$. This is a dependent type because the choice of field name determines the return type of the function.

A multidimensional record array can be described as
\begin{align*}
\mintinline{python}{recarray}: [0, n_1) \to [0, n_2) \to \ldots \to [0, n_k) \to &\ s_1 \to d_1 \\
 &\ s_2 \to d_2 \\
 &\ \ldots \\
 &\ s_m \to d_m
\end{align*}
\noindent or as
\begin{align*}
\mintinline{python}{recarray}: [0, n_1) \to &\ s_1 \to [0, n_2) \to \ldots \to [0, n_k) \to d_1 \\
 &\ s_2 \to [0, n_2) \to \ldots \to [0, n_k) \to d_2 \\
 &\ \ldots \\
 &\ s_m \to [0, n_2) \to \ldots \to [0, n_k) \to d_m
\end{align*}
\noindent or any other placement of the field name index within the ordered sequence of dimensional indexes. In general, the string indexes (field names) commute with the integer indexes (dimensions). This is evident in Numpy's syntax:
\begin{align*}
\mintinline{python}{recarray[i1][i2][i3][fieldname]} & \mintinline{python}{ == recarray[i1][i2][fieldname][i3]} \\
 & \mintinline{python}{ == recarray[i1][fieldname][i2][i3]} \\
 & \mintinline{python}{ == recarray[fieldname][i1][i2][i3]}
\end{align*}
\noindent It is also evident if the array is arranged as a rectilinear table, in which $[0, n_1)$, $[0, n_2)$, \ldots $[0, n_k)$ form a $k$-dimensional lattice of bounded integers and the field names are an additional dimension, indexed by a finite set of strings with no predefined order. This dimension of category labels is usually called the ``columns'' and all other dimensions are called ``rows.'' In this picture, rearranging the order of the string index and the integer indexes corresponds to selecting a column before a row, rather than a row before a column.

\section*{Vectorized functions}

Numpy uses so-called ``vectorized'' functions or ``universal'' functions (``ufuncs'') for most calculations\footnote{\url{https://docs.scipy.org/doc/numpy/reference/ufuncs.html}}. (These are not to be confused with vectorized instructions in CPU hardware, but are based on a similar idea.) Any function $f$ that maps scalar \mintinline{python}{dtypes} $d^A$ to $d^B$,
\[ f: d^A \to d^B\mbox{,} \]
\noindent can be lifted to a vectorized function that maps arrays of \mintinline{python}{dtype} $d^A$ to arrays of \mintinline{python}{dtype} $d^B$:
\[ \mbox{ufunc}(f): \left([0, n) \to d^A\right) \to \left([0, n) \to d^B\right)\mbox{.} \]
\noindent Note that the \mintinline{python}{shape} of the array, $[0, n)$ in this case, is the same for the argument type of $\mbox{ufunc}(f)$ as for its return type.

This $\mbox{ufunc}$ functor is a partial application of what would be called ``$\mbox{map}$'' in most functional languages\footnote{\url{https://en.wikipedia.org/wiki/Map_(higher-order_function)}}. The $\mbox{map}$ functor takes a function and a collection, returning a collection of the same length with the function applied to each element. The $\mbox{ufunc}$ functor only takes a function, and its result is applied to collections (arrays) later.

Since arrays are themselves functions, applying $\mbox{ufunc}(f)$ to an array is a composition\footnote{\url{https://en.wikipedia.org/wiki/Function_composition}} of the array with $f$. Thus, the following is true for any \mintinline{python}{i} $\in [0, n)$:
\begin{align*}
\underbrace{\mbox{ufunc}(f)(\mintinline{python}{array})}_{\mbox{\scriptsize array}}(\mintinline{python}{i}) = & \,f(\underbrace{\mintinline{python}{array}(\mintinline{python}{i})}_{\mbox{\scriptsize scalar}}) \\[0.5\baselineskip]
\underbrace{\mintinline{python}{numpy.vectorize(f)(array)}}_{\mbox{\scriptsize array}}\mintinline{python}{[i]} = & \,\mintinline{python}{f(}\underbrace{\mintinline{python}{array[i]}}_{\mbox{\scriptsize scalar}}\mintinline{python}{)}\mbox{.}
\end{align*}
\noindent by associativity of function composition. This composition always applies $f$ to the {\it output} of the array, never the {\it input} (function composition is not commutative).
\begin{align*}
f: & \mbox{ } d^A \to d^B \\
\mintinline{python}{array}: & \mbox{ } [0, n) \to d^A \\
\mbox{ufunc}(f)(\mintinline{python}{array}) = f \circ \mintinline{python}{array} : & \mbox{ } [0, n) \to d^B\mbox{.}
\end{align*}

Using associativity again, we should be able to compose a sequence of scalar functions $f: d^A \to d^B$, $g: d^B \to d^C$, \ldots, $h: d^Y \to d^Z$ before applying them to the array. If the scalar function can be extracted from a ufunc object, it would be possible to compose
\[ \mbox{ufunc}(f) \circ \mbox{ufunc}(g) \circ \ldots \circ \mbox{ufunc}(h) \]
\noindent into a single
\[ \mbox{ufunc}(f \circ g \circ \ldots \circ h): \left([0, n) \to d^A\right) \to \left([0, n) \to d^Z\right) \]
\noindent that can be applied to an array. This is an optimization known as loop fusion\footnote{\url{https://en.wikipedia.org/wiki/Loop_fission_and_fusion}}, and is often faster than making multiple passes over arrays and possibly allocating large temporary arrays between ufuncs. There have been proposals\footnote{\url{https://numpy.org/doc/1.14/neps/deferred-ufunc-evaluation.html}} and external libraries\footnote{\url{https://www.weld.rs/weldnumpy}} to add this feature transparently to Numpy. In principle, it could even be an explicit (user-visible) feature of the ufunc object, but to my knowledge, it has never been implemented as such.

\section*{Array slicing}

Whereas ufuncs compose scalar functions to the output of an array, slicing composes \mintinline{python}{index} arrays (which are functions) to the input of an array.

The fact that array slicing is itself composition may not be obvious because of the way that slicing is presented:
\[ \mintinline{python}{array[i:j]} \]
\noindent does not seem to be a composition of two arrays. The first point to make is that all of Numpy's slicing mechanisms---range slices (Python's \mintinline{python}{slice} operator or \mintinline{python}{start:stop:step}), \mintinline{python}{numpy.compress} with boolean arrays, and \mintinline{python}{numpy.take} with integer arrays---can be rewritten in terms of \mintinline{python}{numpy.take} with integer arrays:
\begin{itemize}
\item A range slice \mintinline{python}{start:stop:step} can be replaced with an integer sequence
\[ \mintinline{python}{range(start, stop, step)} \]
(ignoring the effect of negative \mintinline{python}{start} and \mintinline{python}{stop}).
\item A boolean array \mintinline{python}{mask} can be replaced with \mintinline{python}{numpy.nonzero(mask)}.
\item An integer array \mintinline{python}{index} is already an integer array.
\end{itemize}

As an array, \mintinline{python}{index} is a function $\mintinline{python}{index}: [0, x) \to [0, n)$ that can be composed with $\mintinline{python}{array}: [0, n) \to d$ to produce
\[ \mintinline{python}{array} \circ \mintinline{python}{index}: [0, x) \to d \]
\noindent Note that \mintinline{python}{index} is to the right (transforms the {\it input}) of \mintinline{python}{array}, whereas $\mbox{ufunc}(f)$ put $f$ to the left (transforms the {\it output}) of \mintinline{python}{array}.

Numpy uses the same syntax for this function composition, \mintinline{python}{array[index]}, as it does for function evaluation, \mintinline{python}{array[i]}, which is potentially confusing. Let's illustrate this with an extended example.

\subsection*{Example}

Consider two functions that are defined on all non-negative integers (at least).
\begin{minted}{python}
def f(x):
    return x**2 - 5*x + 10
def g(y):
    return max(0, 2*y - 10) + 3
\end{minted}

They may be transformed into arrays by sampling \mintinline{python}{f}, \mintinline{python}{g}, and $\mintinline{python}{f} \circ \mintinline{python}{g}$ at enough points to avoid edge effects from their finite domains. For \mintinline{python}{f} and \mintinline{python}{g} above, 100 points in \mintinline{python}{g} is enough to accept the entire range of \mintinline{python}{f} when \mintinline{python}{f} is sampled at 10 points.
\begin{minted}{python}
F   = numpy.array([f(i) for i in range(10)])     # F is f at 10 elements
G   = numpy.array([g(i) for i in range(100)])    # G is g at 100 elements
GoF = numpy.array([g(f(i)) for i in range(10)])  # GoF is g∘f at 10 elements
\end{minted}
\noindent Now $\mintinline{python}{F}: [0, 10) \to [4, 46)$, $\mintinline{python}{G}: [0, 100) \to [3, 191)$, and $\mintinline{python}{GoF}: [0, 10) \to [3, 85)$.

Indexing \mintinline{python}{G} by \mintinline{python}{F} can be expressed with square-bracket syntax or \mintinline{python}{numpy.take}, and it returns the same result as the sampled composition \mintinline{python}{GoF}.
\begin{minted}{python}
G[F]               # → [13,  5,  3,  3,  5, 13, 25, 41, 61, 85]
G.take(F)          # → [13,  5,  3,  3,  5, 13, 25, 41, 61, 85]
numpy.take(G, F)   # → [13,  5,  3,  3,  5, 13, 25, 41, 61, 85]

GoF                # → [13,  5,  3,  3,  5, 13, 25, 41, 61, 85]
\end{minted}
\noindent In \mintinline{python}{GoF}, the functions are composed before being transformed into arrays, and in \mintinline{python}{G[F]}, the arrays themselves are composed via integer-array indexing.

Function composition is associative, so we should be able to change the order of two integer-array indexings. To demonstrate this, introduce another array, which need not have integer \mintinline{python}{dtype}.
\begin{minted}{python}
H = numpy.arange(1000)*1.1
\end{minted}
\noindent When we compute \mintinline{python}{H} indexed by \mintinline{python}{G} indexed by \mintinline{python}{F}, it shouldn't matter whether the \mintinline{python}{H[G]} index is computed first or the \mintinline{python}{G[F]} index is computed first, and we see that this is the case.
\begin{minted}{python}
H[G][F]            # → [14.3  5.5  3.3  3.3  5.5 14.3 27.5 45.1 67.1 93.5]
H[G[F]]            # → [14.3  5.5  3.3  3.3  5.5 14.3 27.5 45.1 67.1 93.5]
\end{minted}

\subsection*{Multidimensional slicing}

If Numpy's integer-array indexing for multiple dimensions worked the same was as its range-slicing does, then the above would be trivially extensible to any number of dimensions. However, Numpy's integer-array indexing (called ``advanced indexing'')\footnote{\url{https://docs.scipy.org/doc/numpy/reference/arrays.indexing.html}} couples iteration over integer arrays supplied to each of the $k$ slots in a rank-$k$ array.

To work-around this caveat, consider rank-$k$ integer arrays in each of the $k$ slots, in which the integer array in slot $i$ has shape $(1, \ldots, n_i, \ldots, 1)$. For example, a three-dimensional slice
\[ \mintinline{python}{array[start1:stop1, start2:stop2, start3:stop3]} \]
\noindent can be simulated with integer arrays
\begin{align*}
\mintinline{python}{array[} & \mintinline{python}{numpy.arange(start1, stop1).reshape(-1, 1, 1),} \\
                            & \mintinline{python}{numpy.arange(start2, stop2).reshape(1, -1, 1),} \\
                            & \mintinline{python}{numpy.arange(start3, stop3).reshape(1, 1, -1)]}
\end{align*}
\noindent because Numpy broadcasts the three integer arrays into a common three-dimensional shape, and the symmetry of these arrays decouples their effects in each dimension.

\section*{Beyond rectilinear arrays}

Numpy is a library for contiguous, rectangular grids of numbers---within that scope, range-slicing and broadcasting\footnote{\url{https://docs.scipy.org/doc/numpy/user/basics.broadcasting.html}} can be effectively computed without modifying or copying array buffers, using stride tricks\footnote{\url{https://docs.scipy.org/doc/numpy/reference/generated/numpy.lib.stride_tricks.as_strided.html}}.

However, we often want more general data structures, so Awkward Array extends Numpy by interpreting collections of rectilinear arrays as non-rectilinear arrays. The two most important additions are
\begin{itemize}
\item records containing fields of any type and
\item arrays of unequal-length subarrays.
\end{itemize}
\noindent (Although union types, nullable types, and cross-referencing are also included in the Awkward Array library, they are less related to this note's focus on arrays as functions.)

\vfill

\section*{Non-rectilinear record types}

Numpy's record array only allows one dimension of category labels, and all arrays identified by a category label share the same \mintinline{python}{shape}. This is because the location of each of these arrays is defined by a stride. It also means that Numpy is limited to ``arrays of structs.''\footnote{\url{https://en.wikipedia.org/wiki/AoS_and_SoA}}

Suppose we want field $s_1$ of an array to have shape $[0, n_1)$ and field $s_2$ to have shape $[0, n_1) \times [0, n_2)$. That is, at each $i_1$, field $s_1$ has a scalar and field $s_2$ has an array. This can be described as the dependent type
\begin{align*}
\phantom{\mbox{ufunc}(f)(}\mintinline{python}{array}\phantom{)}: [0, n_1) \to & s_1 \to d^A \\
 & s_2 \to [0, n_2) \to d^A\mbox{.}
\end{align*}
With \mintinline{python}{i1} $\in [0, n_1)$ and \mintinline{python}{i2} $\in [0, n_2)$,
\begin{align*}
\mintinline{python}{array[i1][s1]} & \mbox{ returns } d^A\mbox{ (a scalar)} \\
\mintinline{python}{array[i1][s2]} & \mbox{ returns } [0, n_2) \to d^A\mbox{ (an array), and} \\
\mintinline{python}{array[i1][s2][i2]} & \mbox{ returns } d^A\mbox{ (a scalar).}
\end{align*}

Such an array can still be passed into ufuncs because ufuncs compose a function $f: d^A \to d^B$ to the {\it output} of an array. For example,
\begin{align*}
\mbox{ufunc}(f)(\mintinline{python}{array}): [0, n_1) \to & s_1 \to d^B \\
 & s_2 \to [0, n_2) \to d^B\mbox{.}
\end{align*}

Such an array can still be sliced because slicing composes an integer array $\mintinline{python}{index}: [0, x) -> [0, n_1)$ to the {\it input} of the array. For example,
\begin{align*}
\mintinline{python}{array}(\mintinline{python}{index}): [0, x) \to & s_1 \to d^A \\
 & s_2 \to [0, n_2) \to d^A\mbox{.}
\end{align*}

The string-index/integer-index can be {\it partially} commuted: the domain $\{s_1, s_2\}$ can be moved from the middle of this function type to the left, like so:
\begin{align*}
\mintinline{python}{array}: & s_1 \to [0, n_1) \to d^A \\
& s_2 \to [0, n_1) \to [0, n_2) \to d^A\mbox{,}
\end{align*}
\noindent but it cannot be moved to the right, where the type of each field is different.

Similarly, if we had nested records, with $s_1$ containing \mintinline{python}{dtype} $d$ and $s_2$ containing a record with fields $t_1$ and $t_2$, our options for commuting indexes would be limited to one possibility:
\begin{align*}
\mintinline{python}{array}: [0, n_1) \to & s_1 \to d & \mintinline{python}{array}: &\ s_1 \to [0, n_1) \to d \\
 & {s_2 \to\ }t_1 \to d                              & &\ {s_2 \to [0, n_1) \to\ }t_1 \to d \\
 & \phantom{s_2 \to\ }t_2 \to d                      & &\ \phantom{s_2 \to [0, n_1) \to\ }t_2 \to d\mbox{.}
\end{align*}

\vfill

\section*{Non-rectilinear shapes}

We can also consider arrays of unequal-length subarrays (``jagged'' or ``ragged'' arrays).

Like records, jagged arrays must be described by a dependent type if the function is to be defined on its whole domain. Just as each value in a record's domain, $\{s_1, s_2\}$, can return a different type, each value in a jagged array's domain can return a different type.

For example, the type of an array like \mintinline{python}{[[1.1, 2.2, 3.3], [], [4.4, 5.5]]} is
\begin{align*}
\mintinline{python}{array}: &\ 0 \to [0, 3) \to d^A \\
 &\ 1 \to [0, 0) \to d^A \\
 &\ 2 \to [0, 2) \to d^A\mbox{.}
\end{align*}
\noindent The type description grows with the length of the array---while it may be practical to fully enumerate a record's fields, it's not practical to enumerate a large jagged array's type.

Jagged arrays can be passed into ufuncs and sliced for the same reasons as non-rectilinear records: these features are composition to the {\it output} and {\it input} of the array, respectively:
\begin{align*}
\mbox{ufunc}(f)(\mintinline{python}{array}): &\ 0 \to [0, 3) \to d^B & \mintinline{python}{array}(\mintinline{python}{index}): &\ 0 \to [0, 2) \to d^A \\
 &\ 1 \to [0, 0) \to d^B                                             & &\ 1 \to [0, 3) \to d^A \\
 &\ 2 \to [0, 2) \to d^B                                             & &\ 2 \to [0, 3) \to d^A \\
 &                                                                   & &\ 3 \to [0, 0) \to d^A
\end{align*}
\noindent for $f: d^A \to d^B$ and \mintinline{python}{index = [2, 0, 0, 1]}, for example.

Non-rectilinear record types and non-rectilinear shapes can be combined, and these two generators can already produce data types as general as JSON. (Note that the explicit enumeration of dependent types for each array index allows heterogeneous lists and \mintinline{json}{null}.)

String-valued field indexes can always commute to the left through a jagged dimension, but it can only commute to the right if domains match for all elements of a jagged dimension. For example, $\{s_1, s_2\}$ can commute through both levels of the following jagged array, but only because it has the same combinations of nested shapes for both $s_1$ and $s_2$.
\begin{align*}
\mintinline{python}{a}: &\ s_1 \to\mbox{\hspace{-0.5 cm}} & 0 \to [0, 3) \to d^A & \mbox{\hspace{0.5 cm}\mintinline{python}{a}:\hspace{-0.5 cm}} &\ 0 \to &\ s_1 \to [0, 3) \to d^A & \mintinline{python}{a}: &\ 0 \to [0, 3) \to\mbox{\hspace{-0.5 cm}} &\ s_1 \to d^A \\
& &\ 1 \to [0, 0) \to d^A                                  & & &\ s_2 \to [0, 3) \to d^B                                & & & s_2 \to d^B \\
& &\ 2 \to [0, 2) \to d^A                                  & &\ 1 \to &\ s_1 \to [0, 0) \to d^B                         & &\ 1 \to [0, 0) \to\mbox{\hspace{-0.5 cm}} &\ s_1 \to d^A \\
&\ s_2 \to\mbox{\hspace{-0.5 cm}} & 0 \to [0, 3) \to d^B                         & & &\ s_2 \to [0, 0) \to d^B                                & & & s_2 \to d^B \\
& &\ 1 \to [0, 0) \to d^B                                  & &\ 2 \to &\ s_1 \to [0, 2) \to d^B                         & &\ 2 \to [0, 2) \to\mbox{\hspace{-0.5 cm}} &\ s_1 \to d^A \\
& &\ 2 \to [0, 2) \to d^B                                  & & &\ s_2 \to [0, 2) \to d^B                                & & & s_2 \to d^B
\end{align*}

\section*{Conclusions}

The reason Awkward Array can make use of Numpy's ufunc and slicing concepts, despite a much more general data model, is because arrays are functions and these two operations correspond to function composition at the {\it output} or the {\it input} of the array.

This note does not discuss the implementation details of Numpy or Awkward Array, though their scopes are well drawn by technical considerations. Numpy focuses on rectilinear arrays because stride tricks greatly optimize that domain. Awkward Array is more general, but it cannot use stride tricks on non-rectilinear data. Parts of a data structure must be physically separated in memory to allow generalized reshaping. However, the fact that slices can be explicitly composed with one another before applying them to an array (the associativity of function composition) has been very useful when dealing with separated data: slices do not need to be propagated all the way down a tree of nested structures---the meaning is preserved by lazy evaluation.

\end{document}