\chapter{Primitive groups, pairs and amalgams}
\section {Primitive Groups}
{\bf Definition 1:} $M<G$ is \emph{primitive} if $M=N_G(A), \forall A \lhd M$. $1 \ne M <G$ is
\emph{strongly $p$-embedded} if $|M \cap M^g|_p = 1$ for $g \in G \setminus M$.
$H<G$ is a $p$-local subgroup of $G$ if $M= N_G(P), P \in p(G)$.
\\
\\
{\bf Comment:} Bender classified groups with strongly embedded $2$-subgroups.
\\
\\
{\bf Theorem 1:}
Let $N \lhd G$, be abelian.  If $G/N$ is perfect then $G'$ is perfect.
\begin{quote}
\emph{Proof:}
$(G/N)= (G/N)'$ so $G= G'N$ and
$G/N \approx G'/(G' \cap N)$.  So
$G'/(G' \cap N)=
(G'/(G' \cap N))'$ and thus $G=G''N$ so $G/G'' \approx N/(G'' \cap N)$.  Since this is abelian,
$G' \subseteq G''$.
\end{quote}
{\bf Theorem 2:}
$C_M(O_p(M)) \subseteq O_p(M)$ is equivalent to $F^*(M)=O_p(M)$ and $O_{p'}(M)=1$.
\begin{quote}
\emph{Proof:}
Clear.
\end{quote}
{\bf Theorem 3:}
If $M<G$ is maximal and $N \lhd G$ then $M/N$ is a maximal subgroup of $G/N$.
\begin{quote}
\emph{Proof:}  Clear.
\end{quote}
{\bf Comment:} The forgoing theorem let's us assume a maximal subgroup contains no normal subgroup of
$G$.
\\
\\
{\bf Theorem 4:}  Let $L \lhd \lhd G$.  If $L \leq F^*(G)$ then (a)
$L= (L \cap F(G))(L \cap E(G))$; (b) $F^*(L) = F^*(G) \cap L$; (c)
$E(L) C_{E(G)}(L) = E(G)$.  $E(L) \lhd E(G)$.
\begin{quote}
\emph{Proof:}
Every component of $L$ is a component of $G$ and $F(L) \leq F(G)$.  Since $[F(G), E(G)]=1$.
\end{quote}
{\bf Theorem 5:}  The action of $G$ by right conjugation on cosets of primitive groups is primitive.
\begin{quote}
\emph{Proof:}  The action is obviously transitive and
$Mg=M \rightarrow g \in M$ which is maximal so the action is primitive.
\end{quote}
{\bf Theorem 6:}
If $M<G$ is primitive, $N \lhd G$ and $M \cap N \neq 1$ then $C_G(N) =1$.
\begin{quote}
\emph{Proof:}
$C_G(N) \lhd G$ and $C_G(N) < M \cap N$ then $C_G(N)=1$.
\end{quote}
{\bf Theorem 7:}
Let $M$ be a primitive subgroup of $G$.  No non-trivial subnormal subgroup of $G$ is contained in $M$.
$F(G) \cap M = 1$.
\begin{quote}
\emph{Proof:}
Suppose $L \lhd \lhd G, 1 \neq L \leq M$.  Pick a minimal one, $L \leq F^*(G)$.  By the previous result,
${\mathbb Z}(F^*(G)) = 1$.  So $F(G)=1$ and $L$ is a component of $G$.
$\langle L^M \rangle \lhd E(G)$ and by primitivity, $E(G) \leq M$.  So $E(G)=1$, which is a contradiction.
\end{quote}
{\bf Theorem 8:}
If $M<G$ is a primitive subgroup, $p \in \pi(M)$, $N \lhd G$.  Suppose $M \cap N =1$ and $O_p(M) \neq 1$.
(a) $p \notin \pi(N)$, (b) $\forall q \in \pi(N), \exists$ an $M$-invariant subgroup Sylow $q$-subgroup
of $N$; (c) if $|\pi(N)| \geq 2$ then $M$ is a maximal subgroup of $G$.
\begin{quote}
\emph{Proof:}
(a) Put $P= O_p(M)$ then $M= N_G(P)$ and $P \in S_p(NP)$ since $N \cap P = 1$ thus $p \notin \pi(N)$.
For (b), $PN$ acts on $\Omega= S_q(N)$. by conjugation and $N$ is a transitive normal subgroup of $PN$ so
$C_{\Omega}(P) \neq \emptyset$ and $C_N(P)$ is transitive on $C_{\Omega}(P)$.  Now $C_N(P) \leq M \cap N =1$ gives
$|C_{\Omega}(P)|=1$ and $C_{\Omega}(P)= C_{\Omega}(M)$ since $P \lhd M$.
For (c), $\exists M$-invariant $Q \in S_q(N)$.  Since $Q < N$, $M < QM < NM \leq G$.
\end{quote}
{\bf Theorem 9:}
Let $M<G$ be a primitive subgroup, $N \lhd G$: $M \cap F^*(N) \neq 1$ then
$F(G)=1$, $F^*(N)=F^*(G)= E(G)$. Every minimal normal subgroup of $G$ is contained in $M$.
\begin{quote}
\emph{Proof:}
$F^*(N) \leq F^*(G)$ and by a previous result, ${\mathbb Z}(R(G)) =1$ and so $F(G)=1$.$F^*(N)=E(N)$ and
$F^*(G)= C_{F^*(G)}(E(N))E(N)$.  Applying the result again,
$F^*(G)= E(N)= F^*(N)$.
\end{quote}
{\bf Theorem 10:}
Suppose $G$ contains a primitive maximal subgroup $M$ then one of the following holds:
(F1) $F(G)=F^*(G)$ and $M$ is a complement of $F(G)$ in $G$;
(F2) $G$ contains exactly two minimal normal subgroups $N_1, N_2$ which are non-abelian
$F^*(G)= N_1 \times N_2= E(G)$; (F3) $F^*(G)$ is a minimal non-abelian subgroup of $G$.
\begin{quote}
\emph{Proof:}
See Stellmacher, 6.6.4.
\end{quote}
{\bf Theorem 11:}
If F1 holds and $p \in \pi(M), O_p(M) \neq 1$ then all primitive maximal subgroups of $G$ are
conjugate.
\begin{quote}
\emph{Proof:}
Put $P= O_p(M), F= F^*(G)$ then $M= N_G(P)$, $FP \lhd G$ and $S_p(M) \subseteq S_p(G)$.  Let
$H$ be another primitive maximal subgroup.  $H$ is also a complement of $F$ so $|H|=|M|$.
$\exists g: P \leq P^g$.  $P= H^g \cap FP \lhd H^g$ so $H^g= N_G(P)= M$.
\end{quote}
{\bf Theorem 12:}
Suppose F2 holds.   There is an $M$-isomorphism $\alpha: N_1 \rightarrow N_2$ such that
$M \cap F^*G) = \{ x x^{\alpha}: x \in N_1 \}$.
\begin{quote}
\emph{Proof:}
Let $D= M \cap F^*(G)$ then by a previous result ($C_G(N)=1$), $ D \cap N_1 = 1 = D \cap N_2$.
Since $G= N_iM$, $F^*(G)= N_iD$ and $\forall x_1 \in N_1, \exists ! x_2 \in N_2: x_1 x_2 \in D$.
The mapping $\alpha: N_1 \rightarrow N_2, x_1 \mapsto x_2$ is an isomorphism.  The isomorphism
commutes with  elements of $M$ since $N_1, N_2, D$ are all $M$-invariant.
\end{quote}
{\bf Theorem 13:}
Let $F$ be a minimal normal subgroup of $G$, $N<G$, $G= FM$.
(a) Suppose $U \subseteq F$ and $U^M=U$, then $UM<G$.
\begin{quote}
\emph{Proof:}
(b) follows from (a).  For (a), assume the contrary, $G=UM$.  Then $U \lhd G$ and $F$ is not a minimal
normal subgroups of $G$.
\end{quote}
\section {Primitive pairs}
{\bf Definition 2:} $M<G$ is \emph{primitive} if $M=N_G(A), \forall A \lhd M$. $1 \ne M <G$ is
\emph{strongly $p$-embedded} if $|M \cap M^g|_p = 1$ for $g \in G \setminus M$.
A group $M$ is of characteristic $p$ if $C_M(O_p(M)) \leq O_p(M)$ (If
$M$ is $p$-separable then $O_{p'}(M)=1$).
\\
\\
{\bf Theorem 14:}
Let $M$ be a group of characteristic $p$.
Suppose $U \le M$ and $U \lhd \lhd M$ or $O_p(M) \le U$ then
$U$ has characteristic $p$.
\begin{quote}
\emph{Proof:}
The case $O_p(M) \leq $ is clear.  In the other case, apply $F^*(L) =F^*(G) \cap L$.
\end{quote}
{\bf Definition 3:}  $M_1, M_2$ is called a \emph{primitive pair} if $1 \ne A \lhd M_i$,
$A \le M_1 \cap M_2 \rightarrow N_{M_j}(A)= M_1 \cap M_2$.  The pair is respectively
(solvable or characteristic $p$ if each are.  The property ${\cal P}(M_1 , M_2, A)$ is
$1 \neq A \lhd M_i$, $A \leq M_1 \cap M_2$.
\\
\\
{\bf Theorem 15:}
Let $M_1, M_2$ be two different maximal $p$-local subgroups of $G$ that both have characteristic $p$.
Suppose $M_1$ and $M_2$ have a common Sylow $p$-subgroup, then $(M_1 , M_2)$ is a primitive pair of
characteristic $p$.
\begin{quote}
\emph{Proof:}
Let $1 \neq A \lhd M_i, A \leq M_i \cap M_j, i \neq j$.  By previous result,
$1 \neq O_p(A) \lhd M_i$ and the maximality of $M_i$ gives $M_i= N_G(O_p(A))$.  So
$N_{M_j}(A)= M_i \cap M_j$ has property ${\cal P}$.  Let $S$ be a common Sylow subgroup of
$M_1$ and $M_2$ then $O_p(M_1 ) O_p(M_2 ) \leq S \leq M_1 \cap M_2$.
\end{quote}
{\bf Theorem 16:}
Suppose $p \in \pi(G)$, every $p$-local has characteristic $p$ and $O_p(G)=1$ then either
(a) there is a primitive pair of characteristic $p$ or (b)
every maximal $p$-local subgroup of $G$ is strongly embedded in $G$.
\begin{quote}
\emph{Proof:}
Let $M$ be a maximal $p$-local of $G$, then $O_p(M) \neq 1$ and
$N_G(M) \leq N_G(O_p(M))=M < G$ so $M^g \neq M, g \in G-M$.  $M^g$ is a maximal $p$-local.
Among all such, choose one, $L \neq M$ with $|M \cap L|_p$ maximal.
If $|M \cap L|_p>1$ then $T \in S_p(M \cap L)$ and $U= N_G(T)$.
Since $U$ is a $p$-local, there is a maximal $p$-local $H \subseteq G$, maximal with $U \subseteq H$.
If $H \neq M$, let $T \in S_p(M)$.  If $T<S$, $T<N_S(T) \leq H \cap M$ which contradicts the maximality of
$|M \cap L|_p$.  This $T \in S_p(M)$.  $\exists S_1 \in S_p(L), g \in S_1 - T$ such that $T^g=T$ and
$M \neq M^g$.  By the previous result, $(M, M^g)$ is a primitive pair of characteristic $p$.  We've shown (a)
holds if $|M \cap L|_p = 1$ when $M, L$ are two different $p$-locals.   If $|M \cap M^g|_p=1, \forall g \in G - M$,
$M$ is strongly $p$-embedded.
\end{quote}
{\bf Bender's Little Theorem:}
Let $(M_1, M_2)$ be a primitive pair of $G$.  Suppose
$F^*(M_1) \le M_2$ and
$F^*(M_2) \le M_1$ then $\exists p: (M_1, M_2)$ has characteristic $p$.
\begin{quote}
\emph{Proof:}
See Stellmacher, 10.1.4.
\end{quote}
{\bf Theorem 17:}
Let $(M_1, M_2)$ be a primitive pair of $G$ of characteristic $p$ then $\exists i \in \{1 , 2 \}$
such that either
(1) The action of $M_i$ on $O_p(M_i/ \Phi(M_i))$ is not $p$-solvable or
(2) $W_i$ is elementary abelian and the action of $M_i$ on $W_i$ is not $p$-stable.
\begin{quote}
\emph{Proof:}
See Stellmacher, 10.1.5.
\end{quote}
{\bf Theorem 18:}
Let $(M_1, M_2)$ be a primitive pair of characteristic $p$, then $M_1$ or
$M_2$ has a non-abelian Sylow $2$-subgroup.
\begin{quote}
\emph{Proof:}
If $p \neq 2$, it follows from the previous result and $p$-stability for groups with abelian $2$-syslow subgroups.
For $p= 2$, if the Sylow $2$-groups are abelian, then $O_2(M_1)= O_2(M_2)$ and $(M_1, M_2)$ is not primitive.
\end{quote}
{\bf Theorem 19:}
Let $M$ be $p$-seperable and $A$ a $p$-subgroup of $M$ satisfying
$Phi(A) \leq O_p(M)$ and $A \nleq O_p(M)$ then $\exists x \in O_{p,p'}(M)$
such that for $L= \langle A, A^x \rangle$, (a) $x \in O^p(L) \leq O_{p,p'}(M)$,
(b) $[O^P(L),A]= O^p(L)$, and (c) $|A/(A \cap O_p(L))|=p$ and $[A \cap O_p(L), L] \leq O_p(M)$.
\begin{quote}
\emph{Proof:}
$\exists L$ with property (a).  Choose $L$ maximal among all such groups then (b) follows.
${\overline L}= L/O_p(L)$, ${\overline Q}= O_{p'}({\overline L})$ so ${\overline L} = {\overline {AQ}}$.
$A$ is an elementary abelian $p$-group and
$\Phi(A) \leq O_p(M) \cap L \leq O_p(L)$.  Let ${\cal B}$ be the set of maximal subgroups of $A$.
${\overline Q}=  \langle C_{\overline Q}({\overline U}): U \in {\cal B} \rangle$ so
$[C_{\overline Q}({\overline U}), A] \neq 1$ for some $U \in {\cal B}$ since $A$ acts non-trivially
on ${\overline Q}$.  This implies $U= A \cap O_p(L)$ and
$[U, O^p(L)] \leq O_p(L) \cap O_{pp'}(M) \leq O_p(M)$ and (c) follows.
\end{quote}
{\bf Theorem 20:}
Let $M$ be a group of characteristic $2$ that
possesses a section isomorphic to $S_4$  then $M$
possesses a section isomorphic to $S_4$ .
\begin{quote}
\emph{Proof:}
Let $M$ be a minimal counterexample.  $O_2(S_3)=1$ and $M/O_2(M) \leq N \lhd X < M$ and $X/N \approx S_3$.
$X=M$ by minimality.  Let ${\overline M}= M/N$ and $D \in S_3(M)$, ${\overline D} \approx C_3 \lhd M$.
By Frattini, $M= N_M(D) N$.  There are $2$-elements that act nontrivially on the $3$ group $D$.  Let $t \in N_M(D)$
of minimal order.  $\exists d \in D: |d|=3, d^t= d^{-1}$, $\langle d, t \rangle/\langle t^2 \rangle \approx S_3$.
Minimality of $M$ shows  $M= O_2(M) \langle d,t \rangle$ and $t^2 \in _2(M)$.
$\Phi(O_2(M))= 1$ and $C_{O_2(M)}(d)=1$.  Thus  $t^2=1$ and $\exists 1 \neq Z \in C_{O_2(M)}(t)$.
$V= \langle z, z^d, Z^{d^2} \rangle$ and $|V| \leq 8$ and $V \lhd M$.  Hence $V= C_2 \times C_2$ and
$V \langle d, t \rangle \approx S_4$ so $M$ is not a minimal counterexample.
\end{quote}
{\bf Notation:}
${\cal Q}(Z, X)= \{ A \leq X: [Z,A,A]=1 \neq [Z,A] \}$.
$q(Z,X)= 0$ if ${\cal Q}(Z,X)= \emptyset$;
$q(Z,X)= min \thinspace \{e \in {\mathbb R}: |A/C_A(Z)|^e= |Z/C_Z(A)|, A \in {\cal Q}(Z,X) \}$, otherwise.
\\
\\
{\bf Theorem 21:}
Let $M$ act faithfully on
an elementary abelian $2$-group, $V$ and let
$A$ be an elementary abelian $2$-subgroup of $M$.  Suppose
$C_M(O_{2'}(M)) \leq O_{2'}(M)$ and $|V/C_V(A)| < |A|^2$.  Then
$M$ posseses a section isomorphic to $S_3$.
\begin{quote}
\emph{Proof:}
Among all the elementary abelian $2$-subgroups that satisfy the condition, choose $A$ of minimal order.
Assume $|A|=2$, then $A \in {\cal A}_V(M)$ and the result follows from Glauberman's theorem.
If $|A|>2$,$C_M(O_{2'}(M) \leq O_{2'}(M)$ means $A$ acts non-trivially on $O_{2'}(M)$.  Let $Q \subseteq O_{2'}(M)$
be minimal.  $Q^A=Q$ and $[Q,A] \neq 1$ so $A_0= C_A(Q)$ is a maximal subgroup of $A$ and
$QA/A_0$ acts faithfully on $C_V(A_0)$.  The conclusion follows from the $|A|=2$ case if
$|C_V(A_0)/C_V(A)| \leq |A/A_0|^2=4$ which in turn follows from the minimality of $A$ since
$|V/C_V(A)| < |A|^2 \leq 4 |V/C_V(A_0)|$.
\end{quote}
{\bf Theorem 22:}
Let $M$ be a group and $V$ an elementary abelian normal $p$-subgroup of $M$; let $Z \leq V$ with
$V= \langle Z^M \rangle$ and $Z \lhd O_p(M)$.  Suppose $\exists A \leq O_p(M)$ with $[V,A,A]=1$.
Then $|A/C_A(V)|^q \leq |V/C_V(A)|$ where $q= q(Z,O_p(M))$.
\begin{quote}
\emph{Proof:}
See Stellmacher, 10.1.10.
\end{quote}
{\bf Theorem 23:}
Let $(M_1, M_2)$ be a solvable primitive pair of characteristic $2$, then $M_1$ or
$M_2$ has a section isomorphic to $S_4$.
\begin{quote}
\emph{Proof:}
See Stellmacher, 10.1.11.
\end{quote}
{\bf Theorem 24:}
Let $G$ be a group of even order, $O_2(G)=1$.  Suppose that for every $2$-local $M$ of $G$,
(1) $M$ has characteristic $2$ and is solvable and (2) $M$ does not possess a section
isomorphic to $S_4$ then every maximal $2$ local of $G$ is strongly $2$-embedded in $G$.
\begin{quote}
\emph{Proof:}
This is a direct consequence of the result preceeding Bender's theorem and the previous result.
\end{quote}
\section {Amalgam Graphs}
{\bf Definition 4:}
$P_1, P_2 \le G$, $|P_i|< \infty$.  Construct a graph $\Gamma(G, P_1, P_2)=\Gamma$
as follows:
$\Gamma$ has verticies consisting of right cosets of $P_1$ and $P_2$; the verticies
$P_i g_j$ and $P_n g_m$ are
joined by an edge if
$P_i g_j \ne P_n g_m$ and
$P_i g_j \cap P_n g_m \ne \emptyset$.  $\Delta(\alpha)$ denotes the verticies
adjacent to $\alpha$.  $G$ acts on graph by right multiplication on cosets.
$\Delta(\alpha)$ is a set of vertices adjacent to $\alpha$.
$G \rightarrow Aut(\Gamma)$.
\\
\\
{\bf Theorem 25:}
Suppose $G$ has two orbits and $P_1, P_2$ are representatives.  Every vertex stabilizer
$G_{\alpha}$ is a $G$-conjugate of $P_1$ or $P_2$ then
(a) $G$ acts transitively on $\Gamma$ and every edge stabilizer in
$G$ is a $G$-conjugate of $P_1 \cap P_2$, (b) $G_{\alpha}$ acts transitively
on $\Delta(\alpha)$, (c) $|\Delta(\alpha)|= |G_{\alpha}:G_{\alpha, \beta}|$,
(d) $P_1 \cap P_2$ is the kernel of the action on $\Gamma$.
\begin{quote}
\emph{Proof:}
(a) For $P_ix \in \Gamma$ and $g \in G$, $P_ixg=P_ix$ is equivalent to
$P_i g^{x^{-1}} = P_i$ is equivalent $g \in P_i^x$.
(b) Let $\langle P_1x, P_2y \rangle$ be an edge.  $\exists z \in P_1x \cap P_2y$.  Hence
$P_1x = P_1z$ and $P_1x = P_1z$ so $z^{-1}$ conjugates $(P_1x, P_2y)$ to $(P_1, P_2)$.  The
stabilizer of $(P_1z, P_2z)$ is in $P_1^z \cap P_2^z= (P_1 \cap P_2)^z$.
(c) By (a), we can assume $\alpha= P_1$ then $\Delta( \alpha ) = \{ P_2y : P_2 \cap P_1 \neq \emptyset \}$
$= \{ P_2y : y \in P_1 \}$ thus $P_1$ is transitive on $\Delta( \alpha )$.
(d) By (a), any normal subgroup of $G$ is contained in $P_1 \cap P_2$ fixes every vertex of $\Gamma$.
\end{quote}
{\bf Theorem 26:}
$\Gamma$ is connected iff $G= \langle P_1 , P_2 \rangle $.
\begin{quote}
\emph{Proof:}
Assume $G= \langle P_1 , P_2 \rangle$ and let $\Delta$ be a connected component of $\Gamma$ that contains
$P_1$.  Since $P_1$ and $P_2$ are adjacent, $P_2 \in \Delta$.  Since different components are disjoint,
$\Delta = \Delta^{\langle P_1, P_2 \rangle}= \Delta^G$ and thus $\Delta= \Gamma$ by (a) above.
Now assume $\Gamma$ is connected and let $G_0= \langle P_1, P_2 \rangle$ and $\Gamma_0=
\{ P_1x: x \in G_0 \} \cup \{ P_xx: x \in G_0 \} $ be the coset graph of $G_0$ with respect to
$P_1, P_2$.  $\Gamma_0$ is connected.  If $\Gamma= \Gamma_0$ then $G= G_0$.  Assume $\Gamma \neq \Gamma_0$.
Since $\Gamma$ is connected, $\exists \alpha, \beta \in \Gamma$ such that $\alpha \in \Gamma_0, \beta \in \Gamma
- \Gamma_0$.  By (c) above, $\beta$ and all other elements of $\Delta( \alpha )$ are in $\Gamma - \Gamma_0$.
Hence $\Gamma$ is not connected.  Contradiction.
\end{quote}
{\bf Theorem 27:}
Let $G= \langle P_1, P_2 \rangle $, $U \le G_{\alpha} \cap G_{\beta}$ and
$(\alpha, \beta)$ is an edge.  The (1) $N_{G_{\delta}}(U)$ acts transitively on
$\Delta(\delta), \delta \in \{ \alpha , \beta \}$ and (2) if $U \le G_{\alpha} \cap G_{\beta}$ then
$U$ acts trivially on $\Gamma$.
\begin{quote}
\emph{Proof:}
(2) together with (c) implies (1) so we may assume (1) holds.
Let $\Gamma_0= \alpha^{N_G(U)} \cup \beta^{N_G(U)}$.  $U$ fixes $\Gamma_0$.
If $\gamma \in \Gamma_0$, $\exists g \in N_G(U)$ and $\delta \in \{ \alpha, \beta \}$ such that
$\gamma= \delta^g$.  Then $\Delta( \delta^g ) = \Delta ( \gamma )$ and
$N_{G_{\gamma}}(U)= N_{G_{\delta}}(U)^g $.  By (1), $N_G(U)$ is transitive on $\Delta( \delta^g ) = \Delta( \gamma )$.
One of $\alpha^g, \beta^g$ is adjacent to $\gamma$ and $\{ \alpha^g , \beta^g \} \subseteq \Gamma_0$.
It follows that $\Delta( \gamma ) \subseteq \Gamma_0$.  By the previous result, $\Gamma$ is connected
and we must have $\Gamma= \Gamma_0$ so $U$ stabilizes every vertex in $\Gamma$.
\end{quote}
{\bf Condition} ${\cal A}$:  Let $G$ be a
finite group generated by $P_1, P_2$,
$T= P_1 \cap P_2$ satisfying: $C_{P_i}(O_2(P_i)) \le O_2(P_i)$, $T \in S_2(P_i)$,
$T_G=1$, $P_i/O_2(P_i) \approx S_3$ and $[\Omega(Z(T)), P_i] \ne 1$.
\\
\\
{\bf Theorem 28:}
Let ${\cal A}$ holds and $(\alpha, \beta)$ be an edge of $\Gamma$.
(a) $|G_{\alpha}:G_{\alpha, \beta}|=3$ and is a Sylow $2$ subgroup of $G_{\alpha}$.
$G_{\alpha} = \langle t, G_{\alpha, \beta} \rangle$ for $t \in G_{\alpha} - G_{\beta}$.
(b) $|\Delta( \alpha )| =3$ and $O_2(G_{\alpha})= \bigcap_{\delta \in \Delta( \alpha )} (G_{\alpha} \cap G_{\beta})$,
(c) $G_{\alpha}$ acts $2$-transitively on $\Delta( \alpha )$.
\begin{quote}
\emph{Proof:}
(a) follows from $P_1/O_2(P_i) \approx S_3$ and (b) and (c) follows from a previous result.
\end{quote}
{\bf Notation:}
For the remainder of the section, $Q_{\alpha}= O_2(G_{\alpha})$ and
$Z_{\alpha}= \langle \Omega(Z(T)): T \in S_2(G_{\alpha} \rangle$.
\\
\\
{\bf Theorem 29:}
Suppose ${\cal A}$ holds and $\alpha \in \Gamma$, $V \lhd G_{\alpha}$, $T \in S_2(G_{\alpha})$ and
$\Omega(Z(T)) \leq V \leq \Omega(Z(Q_{\alpha}))$ and $|V:\Omega(Z(T))|= 2$
then
$V= C_V(G_{\alpha}) \times W$, $W= [V, G_{\alpha}]$. $W= C_2 \times C_2$,
$C_{G_{\alpha}}(W)= Q_{\alpha}$.
\begin{quote}
\emph{Proof:}
Let $D \in S_3(G_{\alpha})$.  $V= C_V(D) \times W$, $W= [V,D]$.  By ${\cal A}$, since
$G_{\alpha}= DT$, $W \neq 1$ and thus $|W| \geq 4$.   Let $d \in D^{\#}$,
$|V/ \Omega(Z(T))|=2=|V/ \Omega(Z(T^d))|$.  $G_{\alpha}= \langle T, T^d \rangle$ means
$|V/C_V(G_{\alpha})| \leq 4$ so $C_V( G_{\alpha}) = C_V(D)$ and $|W|=4$.  The remainder follows from
${\cal A}$.
\end{quote}
{\bf Theorem 30:}
Let ${\cal A}$ hold and $(\alpha, \beta)$ be an edge of $\Gamma$;
(a) $Z_{\alpha} \leq \Omega(Z(Q_{\alpha}))$;
(b) $Q_{\alpha} Q_{\beta} = G_{\alpha} \cap G_{\beta} \in S_2(G)$;
(c) $C_{G_{\alpha}}(Z_{\alpha}) = Q_{\alpha}$;
(d) $Z_{\alpha}Z_{\eta} \lhd G_{\alpha}$ iff $\exists \gamma \in \Delta( \alpha) - \{ \beta \}$ such that
$Z_{\alpha}Z_{\beta} = Z_{\alpha}Z_{\gamma} $.
\begin{quote}
\emph{Proof:}
(a) Let $T \in S_2(G_{\alpha})$ then $Q_{\alpha} \leq T$ and ${\cal A}$ implies
$\Omega(Z(Y)) \leq Z(Q_{\alpha})$.
(b) By ${\cal A}$ and a previous result,
$Q_{\alpha}$ and $Q_{\beta}$ have index $2$ in $G_{\alpha} \cap G_{\beta}$ so it STS $
Q_{\alpha} \neq Q_{\beta} $.
If $Q_{\alpha} = Q_{\beta} $,
now two previous results show
$G$ acts failthfully on $\Gamma$, and this contradicts ${\cal A}$.
\end{quote}
{\bf Theorem 31:}
Let ${\cal A}$ hold and $(\alpha, \beta)$ be an edge of $\Gamma$.  The following are equivalent:
(1) the conclusion of Goldschmidt's Theorem holds; (2) $Z \leq Q_{\beta}$.
\begin{quote}
\emph{Proof:}
Assume the conclusion of Goldschmidt's theorem holds.  For $\delta \in \{ \alpha \beta \}$,
either
(i) $G_{\delta} = S_4$ and $Q_{\delta} \approx S_4 \times C_2$, or
(ii) $G_{\delta} = S_4 \times C_2$ and $Q_{\delta} \approx C_2 \times C_2 \times C_2$.  In either
case, $Z_{\delta} - Q_{\delta}$ and by the previous result $Z_{\alpha} \nleq Q_{\beta}$.
Set $T= Q_{\alpha} Q_{\beta}$ and $E= Q_{\alpha} \cap Q_{\beta}$.  The previous result gives
$T \in S_2(G_{\delta})$ and $|T/Q_{\delta}|=2$.  Thus
$ |Q_{\alpha}:E|=2= |Q_{\beta}:E| $ and
$T= Q_{\beta}Z_{\alpha}$ and
$Q_{\alpha}= E Z_{\alpha}$.
Todo, more.
\end{quote}
{\bf Definition 5:}
$\alpha, \alpha'$ is a \emph{critical pair} if $Z_{\alpha} \nleq Q_{\alpha'}$ and
$b=d(\alpha, \alpha')$.  Let $b$ be minimal.
\\
\\
{\bf Theorem 32:}
Suppose ${\cal A}$ holds and (a) ($\alpha, \alpha')$ is a critical pair;
(b) $G_{\alpha} \cap G_{\alpha+1} = Z_{\alpha'}Q_{\alpha}$, $G_{\alpha'-1} \cap G_{\alpha'} = Z_{\alpha}Q_{\alpha'}$;
(c) $R \leq Z(G_{\alpha}) \cap Z(G_{\alpha+1}) \cap Z(G_{\alpha'}) \cap Z(G_{\alpha'-1})$ and
$R = [Z(G_{\alpha}), G_{\alpha+1}) \cap G_{\alpha}]= [Z(G_{\alpha'}), G_{\alpha'-1}) \cap G_{\alpha'}] $
(d) $ Z_{\alpha} = [ Z_{\alpha} , G_{\alpha} ] \times \Omega( Z(G_{\alpha} ))] $ and
$[ Z_{\alpha} , G_{\alpha} ] = C_2 \times C_2$;
(e) $|Z_{\alpha}:\Omega(Z(Y))|=2$ for $Y \in S_2(G_{\alpha})$.
\begin{quote}
\emph{Proof:}
Minimality of $(b)$ implies
$Z_{\alpha} \leq Q_{\alpha'-1} \leq G_{\alpha'-1} \cap G_{\alpha'} $ and
$Z_{\alpha'} \leq Q_{\alpha+1} \leq G_{\alpha+1} \cap G_{\alpha} $.
$Z_{\alpha} \nleq Q_{\alpha'}$ shows that
$G_{\alpha'-1} \cap G_{\alpha'} = Z_{\alpha} Q_{\alpha} $ since $Q_{\alpha'}$ has index $2$ in
$G_{\alpha'-1} \cap G_{\alpha'}$.
$Z_{\alpha} \lhd G_{\alpha}$ and
$Z_{\alpha'} \lhd G_{\alpha'}$  so
$R \leq Z_{\alpha} \cap Z_{\alpha'} $.  By a previous result, $R \neq 1$ and so $Z_{\alpha'} \nleq Q_{\alpha}$
and $G_{\alpha+1} \cap G_{\alpha}= Z_{\alpha'} Q_{\alpha}$.  (a) and (b) follow and (c) follows from
$R \leq Z_{\alpha} \cap Z_{\alpha'} $ and a previous result.   These also show
$ |Z_{\alpha} / C_{\alpha}(Z_{\alpha'}(Z_{\alpha})= |Z_{\alpha'} / Z_{\alpha'}Z(Z_{\alpha}) =2 $
and
$C_{Z_{\alpha}}(Z_{\alpha'})= \Omega( Z(
G_{\alpha+1} \cap G_{\alpha}
)$ which gives (d) and (f) and, with a previous result (e).
\end{quote}
{\bf Theorem 33:}
Let $\alpha - 1 \in \Delta(\alpha) - \{ \alpha+1 \}$.  Suppose $(\alpha-1, \alpha'-1)$ is not a critical pair.
Then
(1) $ Z_{\alpha} Z_{\alpha+1} = Z_{\alpha} Z_{\alpha-1} $;
(2) $Q_{\alpha} \cap Q_{\beta} \lhd G_{\alpha}, \beta \in \Delta( \alpha)$;
(3) $\alpha$ and
$\alpha'$ are conjugate and $b$ is even.
\begin{quote}
\emph{Proof:}   Since $(\alpha-1, \alpha'-1)$ is not critical, $Z_{\alpha-1}, Z_{\alpha'}] \leq R \leq Z_{\alpha}$,
so $\langle Z_{\alpha'}, G_{\alpha} \cap G_{\alpha-1} \rangle \subseteq N(Z_{\alpha-1}Z_{\alpha})$.  The previous result
gives (a). (b) follows by a previous result and the fact that $G_{\alpha}$ is transisitve on $\Delta( \alpha )$.
Either
$\alpha \in (\alpha')^G$ or
$\alpha \in (\alpha'-1)^G$ , so the former holds and $b$ is even.
For (c), note $\alpha$ and $\alpha'-1$ are conjugate so
$G_{\alpha}$ and $G_{\alpha'-1}$ are conjugate.  Then (b) gives $Z_{\alpha} \leq
Q_{\alpha'-2} \cap Q_{\alpha'-1} =
Q_{\alpha'-1} \cap Q_{\alpha'} $.  This contradicts $Z_{\alpha} \nleq Q_{\alpha'}$.
\end{quote}
{\bf Theorem 34:}
Suppose $\alpha - 1 \in \Delta(\alpha) - \{ \alpha+1 \}$ such that  $(\alpha-1, \alpha'-1)$ is a critical pair
then $b=1$.
\begin{quote}
\emph{Proof:}
Put $R_1= [Z_{\alpha - 1}, Z_{\alpha' - 1}]$.  Assume $b > 1$.
$Z_{\alpha} \leq Q_{\alpha+1}$ and $Z_{\alpha'} \leq Q_{\alpha'-1}$.  $(\alpha - 1, \alpha' - 1)$ is a
critical pair so $|R_1'| =2$.
$R_1 = [Z_{\alpha - 1}, G_{\alpha - 1} \cap G_{\alpha}] \leq (Z(G_{\alpha - 1} \cap  Z(G_{\alpha}))
\cap (Z(G_{\alpha' - 2} \cap  Z(G_{\alpha' - 1}))$.  $R_1 \leq [Z_{\alpha - 1}, Z_{\alpha' - 1}]$, so $[R_1, Z_{\alpha'}] = 1$.
$\langle Z_{\alpha'} , G_{\alpha - 1} \cap G_{\alpha} \rangle = G_{\alpha}$.
\\
\\
(1) $R_1 \leq Z(G_{\alpha})$.
\\
Let $\alpha - 2 \in \Delta(\alpha - 1) \setminus \{ \alpha \}$.  Now we show \\
\\
(2) $(\alpha - 2, \alpha' - 2)$ is a critical pair.\\
If not, $Z_{\alpha + 1} Z_{\alpha} = Z_{\alpha - 1} Z_{\delta}, \delta \in \Delta(\alpha - 1)$
$Z_{\alpha + 1} Z_{\alpha} = Z_{\alpha + 1} Z_{\alpha + 2}$.  Minimality of $b$ gives
$Z_{\alpha + 1} Z_{\alpha + 2} \leq Q_{\alpha'}$ and $Z_{\alpha} \leq Q_{\alpha'}$, so $(\alpha, \alpha')$ is not a critical pair.
Now put $R_2= [Z_{\alpha - 2}, Z_{\alpha' - 2}]$.  $\alpha - 2$ and $(\alpha, \alpha')$ satisfy the hypothesis,
hence $|R_2| = 2$.\\
\\
(3) $R_2 = [Z_{\alpha - 2},  G_{\alpha - 2} \cap G_{\alpha - 1}] \leq Z(G_{\alpha - 1})$.\\
$\exists y \in G_{\alpha - 1}, x \in G_{\alpha} : (\alpha - 2)^y = \alpha$ and $(\alpha + 1)^x = \alpha - 1$.
$[Z_{\alpha}, G_{\alpha - 1} \cap G_{\alpha}] = [Z_{\alpha - 2}, G_{\alpha - 2} \cap G_{\alpha - 1}] = (R_1)^y \leq Z_{\alpha - 1}$.
$R^x = [Z_{\alpha}, G_{\alpha + 1} \cap G_{\alpha}]^x = [Z_{\alpha}, G_{\alpha} \cap G_{\alpha - 1}] = (R_2)^y \leq Z(G_{\alpha - 1})$.
It follows that\\
\\
(4) $ R \leq Z(G_{\alpha + 1})$, so \\
\\
(5) $ R \cap R_1 =1$.\\
If not, $Z_{\alpha'} \leq Q_{\alpha' - 2}$, $[R_2, Z_{\alpha'}] = 1= [R_2, G_{\alpha - 1}]$, so $R_2 =1$.
This contradicts $|R_2|=2$.\\
\\
(6) $b = 2$\\
If $b>2$,
$V_{\alpha} = \langle Z_{\beta}, \beta \in \Delta(\alpha) \rangle \lhd G_{\alpha}$.
$V_{\alpha + 1} = \langle Z_{\beta}, \beta \in \Delta(\alpha + 1) \rangle \lhd G_{\alpha + 1}$.
$V_{\alpha} \leq Q_{\alpha}$,
$V_{\alpha + 1} \leq Q_{\alpha + 1}$, and
$Z_{\alpha} = \langle \Omega(Z(G_{\alpha + 1} \cap G_{\alpha})^{G_{\alpha}} \rangle \leq V_{\alpha}$.
$Z_{\alpha + 1} \leq Q_{\alpha + 1}$, so \\
\\
(7) $Z_{\alpha} Z_{\alpha + 1} \leq V_{\alpha} \cap V_{\alpha + 1}$.\\
Since $R_1 \leq Z(G_{\alpha})$ is a $2$-transitive action of $G_{\alpha}$ on $\Delta(\alpha )$.
${V_{\alpha}}' = R_1 \leq Z(G_{\alpha})$.
We show $V_{\alpha}$ is abelian and $V_{\alpha}$ is generated by involutions.
$V_{\alpha} / R_1$ is elementary abelian so $R_1 = \Phi(V_{\alpha})$ and $R = \Phi(V_{\alpha + 1})$, put
${\overline {V_{\alpha}}} = V_{\alpha} / Z_{\alpha}$.  We get $Z_{\beta}/(Z_{\alpha} \cap Z_{\beta})= 2, \forall \beta \in \Delta(\alpha)$,
so $|{\overline {Z_{\beta}}}| = 2$.  ${\overline {V_{\alpha}}} = \langle Z_{\beta}, \beta \in \Delta(\alpha)\rangle$, so
$|{\overline {V_{\alpha}}}| \leq 8$.  Set $W= V_{\alpha} \cap V_{\alpha + 1} \lhd G_{\alpha} \cap G_{\alpha + 1}$ then
$Z_{\alpha}Z_{\alpha + 1} \leq W$ and\\
\\
(8)  $V_{\alpha} = \langle W^{G_{\alpha}} \rangle$.\\
$\Phi(W) \leq \Phi(V_{\alpha}) \cap \Phi(V_{\alpha + 1}) =R_1 \cap R =1$ and $W$ is elementary abelian
${V_{\alpha}}' \ne 1$ and $|V_{\alpha} / W| \geq 2$.
The kernel of the action of $G_{\alpha}$ on ${\overline {V_{\alpha}}}$ contains $Q_{\alpha}$ since
$[Z_{\alpha - 1} , G_{\alpha} \cap G_{\alpha - 1}] \leq R_1 \leq Z_{\alpha}$.
${\overline {V_0}} = [{\overline {V_{\alpha}}}, O^2(G_{\alpha})]$.
If ${\overline {V_0}} = 1$ $W \lhd G_{\alpha}$ and ${V_{\alpha}}' =1$ but this contradicts ${V_{\alpha}}' = R_1$ and
$|R_1| = 2$.
Now suppose ${\overline {V_0}} \ne 1$ since $|{\overline {V_{\alpha}}}| \leq 8$, \\
\\
(9) $|{\overline {V_0}}| = 4$.\\
Assume $|V_{\alpha} / W| = 2$.  Let $x \in G_{\alpha}$ st $W^x \ne W$
then $V_{\alpha} = W W^x$.
$W  \cap W^x = Z(V_{\alpha})$ and $|V_{\alpha} (W \cap W^x)| = 4$.  Let $D \in S_3(G_{\alpha})$, $D$ acts non-trivially on
$V_{\alpha} / (W \cap W^x)$ so all maximal subgroups of $V_{\alpha}$ that contain $W \cap W^x$ are $D$-conjugate.
Every $x \in V^{\#}$ is an involution, $V_{\alpha}$ is elementary abelian contradicts ${V_{\alpha}}' = R_1$.
We have shown $|V_{\alpha}/ W| \geq 4$ so  $|{\overline {V_{\alpha}}}| \leq 8$. \\
\\
(10) $|V_{\alpha}|=8$, $W = Z_{\alpha}Z_{\alpha + 1}$ and $|{\overline W}|=2$.\\
$Z_{\alpha'} \leq G_{\alpha}$, $Z_{\alpha'} \nleq Q_{\alpha}$,
we get $[{\overline {V_0}}, Z_{\alpha}'] \ne 1$.  But $b = 2$,
so $[V_{\alpha}, Z_{\alpha}] \leq [V_{\alpha}, V_{\alpha + 1}] \leq W$.
${\overline W} = [{\overline {V_0}}, Z_{\alpha'}], \langle {\overline W}^{G_{\alpha}} \rangle = {\overline {V_0}}$, which
contradicts (8), (9) and (10).
\end{quote}
{\bf Goldschmidt's Theorem:}
If ${\cal A}$ holds either (i) $P_1 \approx P_2 \approx S_4$ or
(ii) $P_1 \approx P_2 \approx C_2 \times S_4$.
\begin{quote}
\emph{Proof:}
Let $G$ be a counter example and choose $(G, P_1, P_2, T)$ with $|T|$ minimal.
$b > 1$ and $(\alpha - 1, \alpha' - 1)$ is not a critical pair $\forall \alpha - 1 \in \Delta(\alpha) \setminus \{\alpha + 1 \}$.
\\
\\
(1) $b = 0 \jmod{2}$, $X = Q_{\alpha} \cap Q_{\alpha + 1} \lhd G_{\alpha}$.
\\
$|Q_{\alpha} : X| = |Q_{\alpha + 1}| = 2$.  Let $D \in S_3(G_{\alpha})$, ${\overline {G_{\alpha}}} = G_{\alpha} / X$.
$|{\overline {G_{\alpha}}}|= 12$, ${\overline {Q_{\alpha}}} \leq {\overline {G_{\alpha}}}$ and $|{\overline {Q_{\alpha}}}|=2$ so
${\overline D} \lhd {\overline {G_{\alpha}}}$.\\
\\
We get (2a) $L \lhd G_{\alpha}$, $|G_{\alpha}:L| = 2$,\\
(2b) ${\overline L} = S_3$, \\
(2c) $S_2(L) = \langle Q_{\beta}: \beta \in \Delta(\alpha)\rangle$,\\
(2d) $O_2(L) = X = Q_{\alpha} \cap Q_{\beta}, \forall \beta \in \Delta(\alpha)$,\\
(2e) $Q_{\alpha +1} = Z_{\alpha'} O_2(L))$, (f) $C_L(O_2(L)) \leq O_2(L)$.\\
$Z_{\alpha} \leq G_{\alpha}$, $Z_{\alpha} \leq Q_{\alpha + 1} \leq O_2(L)$.
$C_L(O_2(L)) \leq C_L(Z_{\alpha}) \leq Q_{\alpha} \cap L \leq O_2(L)$.
By $A2$, $\exists t \in G_{\alpha + 1} \setminus Q_{\alpha + 1}, \alpha^t = \alpha + 2$ and $t^2 \in Q_{\alpha + 1}$
$Q_{\alpha + 1}, Q_{\alpha}^t \in S_2(L) \leq G_{\alpha}$, $L^t \leq G_{\alpha + 2}$. \\
\\
(3) $O_2(L)$ is not elementary abelian.
\\
$A_1 = O_2(L)$, $A_2=O_2(L^t)$ are elementary abelian of index $2$ in $Q_{\alpha + 1}$.
If $A_1 = A_2$, $A_1 \lhd \langle G_{\alpha}, G_{\alpha + 2}\rangle$.
$\langle G_{\alpha}, G_{\alpha} \cap  G_{\alpha + 1}, G_{\alpha + 1} \cap  G_{\alpha + 2}\rangle = \langle G_{\alpha}, G_{\alpha + 1}\rangle=G$
such that $A_1 \ne A_2$, $Q_{\alpha + 1}$ is non-abelian.
$A = A_1 \cap A_2 = Z(Q_{\alpha + 1})$, $|(Q_{\alpha + 1}/A)| = 4$.
$\langle G_{\alpha}, O^2(G_{\alpha + 1} \rangle = N_G(A_1)$, which is a contradiction.
$O^2(G_{\alpha + 1})$ acts transitively on $(Q_{\alpha + 1}/A)^{\#}$, its elements are involutions and $Q_{\alpha + 1}$ is elementary
abelian.
$G_0 = \langle L, L^t \rangle$.  Denote the largest normal subgroup of $G_0$ in $Q_{\alpha +1}$ by $Q$.
${G_0}^t=G_0, Q^t=Q$.  \\
\\
(4) We show $[Q, D] \ne 1$.\\
Suppose not put ${\tilde {G_0}} = G_0/Q$.
$Q_{\alpha + 1} \in S_2(L) \cap S_2(L^t)$.
$({\tilde {G_0}}, {\tilde L}, {\tilde {L^t}}, {\tilde {Q_{\alpha + 1}}})$ satisfy ${\cal {A_2}}$, ${\cal {A_3}}$, ${\cal {A_4}}$.
${\tilde W}= [{\tilde {Z_{\alpha}}}, {\tilde D}] \ne 1$.
$C_{\tilde L}(O_2({\tilde L})) \leq O_2({\tilde L})$, so  ${\cal {A_1}}$ and ${\cal {A_5}}$ hold.
Because of the minimality of $|T|$ and $|Q_{\alpha + 1}| < |T|$, ${\tilde L} = S_4$ or ${\tilde L} = C_2 \times S_4$.
${\tilde W} = [O_2(({\tilde L}), O^2({\tilde L})]$ is not contained in $O_2({\tilde L}^t)$ and ${\tilde W} \leq {\tilde {Z_{\alpha}}}$
and so $O_2(L) =(O_2(L) \cap O_2(L^t))Z_{\alpha}$.
Since $Z_{\alpha} \leq \Omega(Z(O_2(L)))$, $\Phi(O_2(L))= \Phi(O_2(L) \cap O_2(L^t))$ and thus
$\Phi(O_2(L)) = \Phi(O_2(L^t))$. $\Phi(O_2(L)) \lhd \langle G_{\alpha}, G_{\alpha + 2} \rangle = G$ and ${\cal {A_3}}$ gives
$\Phi(O_2(L))=1$ which contradicts (3) and (4).\\
\\
(5) Let $\beta \in \Delta(\alpha) \setminus \{\alpha \}$, then $\langle Z_{\alpha}, Z_{\gamma} \rangle$ is not normal in $L$.
\\
Put $\Delta(\beta) = \{ \alpha, \delta, \gamma \}$ and $V_{\beta}= \langle Z_{\alpha}, Z_{\beta}, Z_{\gamma} \rangle \lhd G_{\beta}$.
If $x \in Q_{\alpha} \setminus Q_{\beta}$ interchanges $\gamma$ and $\delta$ and normalized $L$.
If $\langle Z_{\alpha} , Z_{\gamma} \rangle$ is normal in $L$ and also
$\langle Z_{\alpha}, Z_{\delta} \rangle = \langle Z_{\alpha}, Z_{\gamma}^x \rangle$ is normal in $L$.  Thus $V_{\beta}$ is normal
in $L$ (which is not contained in $G_{\alpha} \cap G_{\beta}$) which contradiction!\\
\\
(6) Let $b \geq 4, \alpha - 1 \in \Delta(\alpha) \setminus \{ \alpha + 1 \}$ and
$\alpha - 2 \in \Delta(\alpha - 1) \setminus \{\alpha \}$ then $(\alpha - 2, \alpha' - 2)$ is a critical pair.
\\
Assume not.  $Z_{\alpha - 2} \leq Q_{\alpha' - 3} \cap Q_{\alpha' - 2}$.  Since
$\alpha' - 2$ is conjugate to $\alpha$,
$Z_{\alpha -2} \leq Q_{\alpha' -3} \cap Q_{\alpha' - 2} = Q_{\alpha' - 2} \cap Q_{\alpha' - 1} \leq
G_{\alpha' - 1} \cap G_{\alpha'} = Z_{\alpha}Q_{\alpha'}$.  So
$[Z_{\alpha - 2}, Z_{\alpha'}] \leq [Z_{\alpha}, Z_{\alpha'}] \leq Z_{\alpha}$ and so $Z_{\alpha - 2} Z_{\alpha} \leq Q_{\alpha} \cap Q_{\alpha - 1}$ is normalized by $Z_{\alpha'}$ is normal in $L$ which contradicts (5).\\
\\
$\alpha - 1 \in \Delta(\alpha) \setminus \{ \alpha + 1 \}, x \in L \leq G_{\alpha}$ with $(\alpha - 1 = (\alpha + 1)^x$.  Thus
$\alpha - 2 = (\alpha + 2)^x$, which is adjacent to $\alpha - 1$.  If $b \geq 4$, $(\alpha - 2, \alpha' - 2)$ is critical.  Hence
$R_2= [Z_{\alpha - 2}, Z_{\alpha' - 2}] \leq Z(G_{\alpha - 2} \cap G_{\alpha - 1}) \cap Z_{\alpha' - 2}$.  Also,
$b \geq 4$ also implies $Z_{\alpha'} \leq Q_{\alpha' - 2}$ and $[R_2, Z_{\alpha'}] = 1$ and so $[R_2 , L] = 1$ and
$R_2 \leq Z(G_{\alpha + 2} \cap G_{\alpha + 1}$ since $x \in L$.\\
Now $(\alpha', \alpha)$ is also a critical pair so $\exists \alpha' +2$ such that $d(\alpha', \alpha' + 2) = 2$ and
$(\alpha' +2, \alpha + 2)$ is also critical. So $Z_{\alpha' + 2} \leq Q_{\alpha' - 2}$ and so
$[R_2, Z_{\alpha' +2}] = 1$ since $R_2 \leq Z_{\alpha' - 2}$.  Hence $G_{\alpha + 2} \cap G_{\alpha + 3} = Q_{\alpha +2}Z_{\alpha'+2}$
is centralized by $R_2$.  But then $R_2 \leq Z(G_{\alpha + 2})$ and  $R_2 \leq Z(G_{\alpha - 2})$ after conjugation.  This contradicts
the action of $Z_{\alpha' - 2}$ on  $Z_{\alpha + 2}$.
This proves:\\
(7) $b \leq 4$
\\
Finally, $Q \leq O_2(L^t) \leq Q_{\alpha +2}] =  1$.\\
\\
Case a: $Z_{\alpha + 2}$ is not contained in $O_2(L)$.\\
$Q_{\alpha + 1} \leq O_2(L) Z_{\alpha + 2}$ and $L= \langle (Z_{\alpha + 2})^L \rangle O_2(L) = C_L(O_2(L)) O_2(L)$.
So $O^2(L) \leq C_L(Q)$ since $Q \lhd L$ but then $[Q, D] = 1$ which contradicts (4).
\\
\\
Case b: $Z_{\alpha + 2} \leq O_2(L)$
\\
$Z_{\alpha + 2} \leq Q_{\alpha}$ and (7) implies $b=4$.  (6) then implies
$Z_{\alpha + 2}$ is not contained in $Q_{\alpha - 2}= Q_{(\alpha + 2)^x}$ and $L^{tx}$ is normal of index $2$ in $G_{\alpha - 2}$.
$\langle (Z_{\alpha + 2})^{L^{tx}} \rangle \leq G_0$ has a Sylow $3$ subgroup, $D_2$ of $G_{\alpha - 2}$. But then
$Q \lhd G_0$ shows $[Q, D] = 1$ which contradicts (4).
This contradicts (4) since $D_2$ is a $G_0$ conjugate of $D \in S_3(G_{\alpha})$.
Assume $(\alpha - 2, \alpha' - 2)$ is not normal in $L$.
\end{quote}
{\bf Amalgam example 1:} $G= S_6$, $a=(12)$, $b=(12)(34)(56)$.  $P_1 = C_G(a)$, $P_2 = C_G(b)$.
$P_1 = \langle a \rangle \times \langle (34)(56) \rangle \times \langle (35)(46) \rangle$.
\\
\\
{\bf Amalgam example 2:} $G= SL_3(2)$, $|G|=168$.
$P_1 = \{\left(
\begin{array}{ccc}
a & b & c \\
0 & d & e \\
0 & 0 & f \\
\end{array}
\right) \}$.
$P_2 = \{\left(
\begin{array}{ccc}
a & b & c \\
d & e & f \\
0 & 0 & g \\
\end{array}
\right) \}$. $G=\langle P_1, P_2\rangle$. $P_1 \cong P_2 \cong S_4$.
$P_1 \cap P_2 = \{\left(
\begin{array}{ccc}
1 & a & b \\
0 & 1 & c \\
0 & 0 & 1 \\
\end{array}
\right) \}$.  If $M_1, M_2$ are a solvable primitive pair oc characteristic $2$, either $M_1$ or $M_2$ possesses a section isomorphic
to $S_4$.
\section {ONan}
{\bf Theorem:} Let $M$ be a primitive maximal subgroup of $G$ then either (1) $F^*(G)=F(G)$ [Example: $G=S_4$, $M=S_3$], (2)
$F(G)= 1$ and $F^*(G) = N_1 \times N_2$ where $N_1$ and $N_2$ are the only minimal normal subgroups
[$G$ involves $A_5$] or (3) $F(G)=1$ and $F^*(G)$ is the unique minimal subgroup pf $G$.


\documentclass{article}
\usepackage[a4paper, total={6in, 9in}]{geometry}
\usepackage{fancyhdr}
\usepackage{amsmath}
\usepackage{algorithm}
\usepackage{algpseudocode}

\newtheorem{theorem}{Theorem}
\newtheorem{lemma}{Lemma}
\newtheorem{proof}{Proof}[section]

\pagestyle{fancy}
\fancyhf{}
\rhead{DanDoge}
\lhead{Notes on solving recurrence equations}
\rfoot{Page \thepage}
\cfoot{latest version: 2018/02/26}

\title{Notes on solving recurrence equations}
\date{2018-02-26}
\author{DanDoge}

\begin{document}
\section{definition}
  \paragraph{}
    if the $n$-th term of a sequence can be expressed as a function of previous terms
    \begin{equation}
      x_n = F(x_{n - k}, ..., x_{n - 1}) + g_n,\ n > k,
    \end{equation}
    then this equation is called a $k$-th order \textit{recurrence relation}, the values $x_1, ..., x_k$ are called \textit{initial conditions}, if $g_n \equiv 0$, the recurrence equation is called \textit{homogeneous}, otherwise it's called \textit{non-homogeneous}
\section{solving first order recurrences}
  \paragraph{}
    this class of recurrences can be solved by \textit{iteration}, which is, i think, taught in high school, thus i omit it.
\section{solving second order recurrences}
  \paragraph{}
    this class of recurrences can be solved using \textit{a characterstic equation}, when we have constant coefficients.
  \paragraph{example:Fibonacci numbers}
    the Fibonacci sequence is defined by:
    \begin{equation}
      a_n = a_{n - 1} + a_{n - 2}, a_0 = 0, a_1 = 1
    \end{equation}
    we obtain a polynomial equation, which is called \textit{a characterstic equation}
    \begin{equation}
      \lambda^2 = \lambda + 1
    \end{equation}
    which has two roots, $\lambda_1 = \frac{1 - \sqrt{5}}{2}$ and $\lambda_2 = \frac{1 + \sqrt{5}}{2}$, therefore, $\lambda_1^n$ and $\lambda_2^n$ and their linear combination $c_1\lambda_1^n + c_2\lambda_2^n$ are solutions to this recurrence, then, from initia conditions, we find $c_1$ and $c_2$.
\section{multiple roots}
  \paragraph{}
    what if some of the roots of the characterstic equation are the same?
  \paragraph{example}
  \begin{equation}
    a_n = 2a_{n - 1} - a_{n - 2}
  \end{equation}
  the characterstic equation has two identical roots $\lambda_1 = \lambda_2 = 1$, so the first solution is $1^n$, to get the second solution, we consider a new equation
  \begin{equation}
    b_n = (2 + \epsilon)b_{n - 1} - (1 + \epsilon)b_{n - 2}
  \end{equation},
  if $\epsilon \to 0$, then $b_n$ approaches $a_n$, the characterstic equation for $b_n$ has two roots $1, 1 + \epsilon$, and $c_1 = 1 - \frac{1}{\epsilon}, c_2 = \frac{1}{\epsilon}$,
  thus $b_n = (1 - \frac{1}{\epsilon} + \frac{1}{\epsilon}*(1 + \epsilon)^n)$, consider when $\epsilon \to 0$, we have $a_n = 1 + n$.
  \paragraph{Theorom}
    let $\lambda$ be a root of a multiplicity p of the characterstic equation, then $\lambda^n, n\lambda^n, ..., n^{p - 1}\lambda^n$ are all solutions to the recurrence.
\section{non-homogeneous equations}
  \paragraph{Theorom}
    a recurrence of the form
    \begin{equation}
      x_n + c_1x_{n - 1} + ..+c_kx_{n - k} = b^nP(n)
    \end{equation}
    where $c_k$ and $b$ are all constants, and $P(n)$ is a polynomial of the order $d$ can be transformed into the characterstic equation
    \begin{equation}
      (r^k + c_1r^{k - 1} + ... + c_k)(r - b)^{d + 1} = 0
    \end{equation}
\section{generating functions}
  \paragraph{example}
    we are going to derive a generating function for this sequence
    \begin{equation}
      a_n - 3a_{n - 1} + 2a_{n - 2} = 0, a_0 = 0, a_1 = 1
    \end{equation}
    first, we define
    \begin{equation}
      f(x) = \sum_{k = 0}^\infty a_kx^k
    \end{equation}
    then we have
    \begin{equation}
      f(x) - 3xf(x) + 2x^2f(x) =a_0 + a_1x - 3a_0x = x
    \end{equation}
    thus,
    \begin{equation}
      f(x) = \frac{1}{1 - 2x} - \frac{1}{1 - x}
    \end{equation}
    by means of a geometric series, we have
    \begin{equation}
      f(x) = (2 - 1)x + (2^2 - 1)x^2 + ...
    \end{equation}
\end{document}


\section{Notation}\label{sec:notation}

\begin{description}
\item[Natural Numbers] The set $\mathbb{N}$ refers to the set of all natural
  numbers $\{0, 1, 2, \ldots\}$. The set $\mathbb{Q}$ refers to the set of
  rational numbers.
\item[Booleans] The set $\mathbb{B}$ denotes the set of booleans
  $\{\var{True}, \var{False}\}$.
\item[Powerset] Given a set $\type{X}$, $\powerset{\type{X}}$ is the set of all
  the subsets of $X$.
\item[Sequences] Given a set $\type{X}$, $\seqof{\type{X}}$ is the set of
  sequences having elements taken from $\type{X}$. The empty sequence is
  denoted by $\epsilon$, and given a sequence $\Lambda$, $\Lambda; \type{x}$ is
  the sequence that results from appending $\type{x} \in \type{X}$ to
  $\Lambda$.
\item[Functions] $A \to B$ denotes a \textbf{total function} from $A$ to $B$.
  Given a function $f$ we write $f~a$ for the application of $f$ to argument
  $a$.
\item[Inverse Image] Given a function $f: A \to B$ and $b\in B$, we write
  $f^{-1}~b$ for the \textbf{inverse image} of $f$ at $b$, which is defined by
  $\{a \mid\ f a =  b\}$.
\item[Maps and partial functions] $A \mapsto B$ denotes a \textbf{partial
    function} from $A$ to $B$, which can be seen as a map (dictionary) with
  keys in $A$ and values in $B$. Given a map $m \in A \mapsto B$, notation
  $a \mapsto b \in m$ is equivalent to both $m~ a = b$ and $\mathsf{a}~m = b$.
  Given a set $A$, $A \mapsto A$ represents the identity map on $A$:
  $\{a \mapsto a \mid a \in A\}$. The $\emptyset$ symbol is also used to
  represent the empty map as well.
\item[Domain and range] Given a relation $R \in \powerset{(A \times B)}$,
  $\dom~R \in \powerset{A}$ refers to the domain of $R$, and
  $\range~R \in \powerset{B}$ refers to the range of $R$. Note that (partial)
  functions (and hence maps) are also relations, so we will be using $\dom$ and
  $\range$ on functions.
\item[Domain and range operations] Given a relation
  $R \in \powerset{(A \times B)}$ we make use of the \textit{domain-restriction},
  \textit{domain-exclusion}, and \textit{range-restriction} operators, which
  are defined in \cref{fig:domain-and-range-ops}. Note that a map $A \mapsto B$
  can be seen as a relation, which means that these operators can be
  applied to maps as well.
\item[Integer ranges] Given $a, b \in \mathbb{Z}$, $[a, b]$ denotes the
  sequence $[i \mid a \leq i \leq b]$ . Ranges can have open ends: $[.., b]$
  denotes sequence $[i \mid i \leq b]$, whereas $[a, ..]$ denotes sequence
  $[i \mid a \leq i]$. Furthermore, sometimes we use $[a, b]$ to denote a set
  instead of a sequence. The context in which it is used should provide enough
  information about the specific type.
\item[Domain and range operations on sequences] We overload the $\restrictdom$,
  $\subtractdom$, and $\restrictrange$ to operate over sequences. So for
  instance given $S \in \seqof{A}$, and $R \in \seqof{(A \times B)}$:
  $S \restrictdom R$ denotes the sequence
  $[ (a, b) \mid (a, b) \in R, a \in S]$.
\item[Wildcard variables] When a variable is not needed in a term, we replace
  it by $\wcard$ to make it explicit that we do not use this variable in the
  scope of the given term.
\item[Implicit existential quantifications] Given a predicate
  $P \in X \to \mathbb{B}$, we use $P \wcard$ as a shorthand notation for
  $\exists x \cdot P~x$.

\item[Pattern matching in premises] In the inference-rules premises use
  $\var{patt} \leteq \var{exp}$ to pattern-match an expression $\var{exp} $
  with a certain pattern $\var{patt}$. For instance, we use
  $\Lambda'; x \leteq \Lambda$ to be able to deconstruct a sequence $\Lambda$
  in its last element, and prefix. If an expression does not match the given
  pattern, then the premise does not hold, and the rule cannot trigger.

\item[Ceiling] Given a number $n \in \mathbb{R}$, $\ceil{n}$ represents the
  ceiling of $n$, and $\floor{n}$ represents its floor.
\end{description}

\begin{figure}[htb]
  \begin{align*}
    \var{S} \restrictdom \var{R}
    & = \{ (a, b) \mid (a, b) \in R, ~ a \in S \}
    & \text{domain restriction}
    \\
    S \subtractdom R
    & = \{ (a, b) \mid (a, b) \in R, ~ a \notin S \}
    & \text{domain exclusion}
    \\
    R \restrictrange S
    & = \{ (a, b) \mid (a, b) \in R, ~ b \in S \}
    & \text{range restriction}
  \end{align*}
  \caption{Domain and range operations}
  \label{fig:domain-and-range-ops}
\end{figure}


% --------------------------------------------------------------
% This is all preamble stuff that you don't have to worry about.
% Head down to where it says "Start here"
% --------------------------------------------------------------

\documentclass[12pt]{article}

\usepackage[margin=1in]{geometry}
\usepackage{amsmath,amsthm,amssymb}
\usepackage{graphicx}
\usepackage{epstopdf}

\newcommand{\N}{\mathbb{N}}
\newcommand{\Z}{\mathbb{Z}}

\newenvironment{theorem}[2][Theorem]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}\hskip \labelsep {\bfseries #2.}]}{\end{trivlist}}
\newenvironment{lemma}[2][Lemma]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}\hskip \labelsep {\bfseries #2.}]}{\end{trivlist}}
\newenvironment{exercise}[2][Exercise]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}\hskip \labelsep {\bfseries #2.}]}{\end{trivlist}}
\newenvironment{reflection}[2][Reflection]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}\hskip \labelsep {\bfseries #2.}]}{\end{trivlist}}
\newenvironment{proposition}[2][Proposition]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}\hskip \labelsep {\bfseries #2.}]}{\end{trivlist}}
\newenvironment{corollary}[2][Corollary]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}\hskip \labelsep {\bfseries #2.}]}{\end{trivlist}}

\begin{document}

% --------------------------------------------------------------
%                         Start here
% --------------------------------------------------------------

%\renewcommand{\qedsymbol}{\filledbox}

\title{Experiments of different directions in optimization for small neural networks}%replace X with the appropriate number
% \author{Chenxin} %if necessary, replace with your course title
\date{\vspace{-10ex}}
\maketitle

\section{The problem and the data}

I only focus on two simple networks and two sets of one dimensional data.

Network 1 (N1) has the $1-2-1$ structure, thus has 7 parameters in total. Network 2 (N2) has the $1-1-1$ structure, thus has $4$ parameters in total. I tried linear, Relu, sigmoid as three activation functions and square loss as the loss function. However, for simplicity, here all the results are based on sigmoid activation function only. Figure~\ref{fig:n} shows the network structure.
\begin{figure}[h]
\centering
\includegraphics[width=6cm]{n1}
\includegraphics[width=6cm]{n2}
\caption{N1 (left) and N2 (right).}
\label{fig:n}
\end{figure}

Dataset one (D1) contains $4$ samples, two in each class. Dataset two (D2) contains $20$ samples, $10$ in each class. Figure~\ref{fig:d}  shows the distribution of both of them.
\begin{figure}[h]
\includegraphics[width=8cm]{data1.eps}
\includegraphics[width=8cm]{data2.eps}
\caption{D1 (left) and D2 (right). Red samples have $1$-label, while green samples have $0$-label.}
\label{fig:d}
\end{figure}

Notice that for D1, N1 should be able to achieve $100\%$ accuracy after training, but D2 is a rather ``difficult'' dataset for both these two networks.

\section{What directions that CG gives us?}

Here we focus on N1 only. We choose $200$ random points in ${\bf R}^7$ for initializing parameters. For each point, the gradient $d_{gd}$ and the results of CG steps $d_{cg}$ are computed. After line search, we are able to obtain two updated points and their objective values, i.e., $f_{gd}$ and $f_{cg}$. We record the ratio
$$ r:=\frac{f_{gd} - f_0}{f_{cg} - f_0 } $$
to measure the ``quality'' of two updates, where $f_0$ is the original objective value before taking any updates. Thus, $r>1$ means negative gradient could lead to more decreasing on objective value than a CG step, and vice versa.

The only difference between CG implemented here and traditional CG is, here if $\alpha<0$, we will terminate CG loops and return the negative curvature direction as output.

The number of maximal CG iterations is tuned from $1$ to $5$.

We apply backtrack line search in both gradient steps and CG steps. The initial step size is $10$ and multiplier on it is $0.5$ if the iteration has to go on. However, for CG steps, we need to flip the sign of step size in each line search iterations since the direction may not be a descent direction.

\begin{figure}[h]
\includegraphics[width=8cm]{CGD1.eps}
\includegraphics[width=8cm]{CGD2.eps}
\caption{Comparing negative gradient directions and CG outputs as search directions for D1 (left) and D2 (right). $r>1$ means negative gradient directions provides larger decrease on objective.}
\label{fig:CGvsGD}
\end{figure}

The figures above are two bar plots showing, by setting different maximal iterations for CG, the percentages of $r>1$ and $r<1$. The observation is that, running more than $3$ iterations of CG does not change the results at all. The reason is that CG will find a negative curvature direction in first $3$ iterations in all cases.

Another observation is, by allowing CG to take more iterations, the outputs of CG have lower chance to beat gradient direction. Therefore, I guess the reason here is those negative curvature directions cannot provide much decrease on objective.

\section{How is leftmost eigenvector of Hessian?}

Here we consider to use leftmost eigenvector as the direction to make updates. In particular, at any point, if the leftmost eigenvalue of Hessian is less than $0$, then we use left most eigenvector to do line search. If it is greater than $0$, then we use the output of CG with $5$ as maximal number of iterations. (It is worthwhile to note that for N1 and both datasets, as far as I see, all the random points have negative leftmost eigenvalue in the Hessian.)

Let $f_{le}$ denote the objective obtained from doing line search on leftmost eigenvector as a direction. After defining
$$ r:=\frac{f_{gd} - f_0}{f_{nc} - f_0 }.$$
and running the experiments, I found there are $\mathbf{26\%}$ cases which lead to $r>1$ for D1 and $\mathbf{19\%}$ cases leading to $r>1$ for D2. The results indicate that leftmost eigenvector of Hessian could be a direction that has larger chance to lead to a larger decrease in objective than gradient. This also suggest that the negative curvature directions returned by CG is not ``negative enough'' since the product $p^TA p$ (even though it is negative) can be rather close to $0$.

\begin{figure}[h]
\includegraphics[width=8cm]{ncd1.eps}
\includegraphics[width=8cm]{ncd2.eps}
\caption{Comparing negative gradient directions and leftmost eigenvectors as search directions for D1 (left) and D2 (right).  $r>1$ means negative gradient directions provides larger decrease on objective.}
\label{fig:CGvsGD}
\end{figure}

\section{Using three directions as three algorithms for training.}

In the above experiments, we compare reduction of three kinds of directions on random initial points. Here, we run each of them as an algorithm to see how they will perform on N1 and D1. In particular, each algorithm is run for $100$ iterations to see what is result it can reach.

Figure~\ref{fig:3m} is a screenshot of running each algorithms for $100$ iterations, starting with $10$ different initial points as $10$ sets of experiments. The observation is that CG can reach loss close to $0$ in most of cases, while the other two cannot. This is opposite to the observation in the previous experiments which show CG steps have lower chance to provide large decreases on objective, compared with graident steps. This is a surprising result, and I would like to explore the reason behind it.

\begin{figure}[h]
\includegraphics[width=16cm]{threeM}
\caption{}
\label{fig:3m}
\end{figure}

\section{Visualization of these directions on N2}

In all the experiments, we only use N1 as the network structure. Here we consider to use N2 to build visualization which help to understand different kinds of directions. Note that we will fix the value of two biases so that there will be only 2 weights as parameters. Figure~\ref{fig:landscape} shows an example of such visualization. The axis having the range from $0.25$ to $0.5$ represents the objective value, while the other two represent the values of two weights. The black, red and blue lines represents leftmost eigenvector, CG outputs and negative gradient directions respectively. I hope visualization like this will be helpful in future exploration.

\begin{figure}[t]
\includegraphics[width=15cm]{2w.eps}
\caption{Landscape of network N2 by fixing two biases and three kinds of directions at $100$ random points.}
\label{fig:landscape}
\end{figure}

\section{Next steps}
\begin{itemize}
\item Figure out the reason why CG performs better in Section 4 but worse in Section 2.
\item Improve current line search strategies.
\item Read the paper \cite{rr} and think about landscape of N1.
\end{itemize}

\begin{thebibliography}{9}
\bibitem{rr}
Li, Hao, Zheng Xu, Gavin Taylor, and Tom Goldstein.
"Visualizing the Loss Landscap of Neural Nets." arXiv preprint arXiv:1712.09913 (2017).
\end{thebibliography}

\end{document}


\subsection{Backpropagation}

Back propagation is a learning algorithm which aims to minimize the errors/cost function of the NN. Through this learning algorithm, the random weights and biases which were initially given to the network will be optimized to give the best output.

Output of each node is the sum of the multiplications of the output of previous nodes by certain weights. Therefore we can associate how much error is coming with every weight and how much error has been brought from each particular node from the previous layer.

To understand this better it is worth imagining the following example:
\begin{itemize}
   \item  node 1 in the output layer of the NN should be equal to 0.01
   \item  instead the NN is providing us with 0.8
\end{itemize}

In this case we should do the following:
\begin{enumerate}
   \item Calculate the error of the node (-0.79 in our example)
   \item Calculate how much error has been brought by every link to this node
\end{enumerate}

For instance if weight \(w_{11}\) is 0.6 and \(w_{21}\) is 0.4 then they are associated with an error of -0.79 multiplied by 0.6 and -0.79 multiplied by 0.4 respectively (see Figure \ref{fig:bp}).

\begin{figure}[H]
    \includegraphics[width=\linewidth]{pics/bp.jpg}
    \caption{\label{fig:bp} Backpropagation}
\end{figure}

After calculation of how much error is associated with every weight we can obtain the errors for the nodes in the proceeding layer.

For instance error term for node 1 in the hidden layer will be equal to:
\begin{itemize}
   \item the sum of errors associated with all the weights (\(w_{11}\) and \(w_{12}\) in our case) that link this node with the next layer (see Figure \ref{fig:bp}).
\end{itemize}

Once we repeat this procedure for all the nodes in all layers we can find out how much every node should be changed.

To do so in Python we just need to make multiplication of vector that contain errors by corresponding matrix of weights.

\begin{lstlisting}[language=Python]
    Find the errors associated with hidden layer output:
    h_errors = np.dot(w_h_o.T, o_errors),
    h_errors[0:10] # errors in the hidden layer - show the first 10 nodes out of 90.
\end{lstlisting}

\begin{lstlisting}
    array([[ 0.39443768],
    [-0.16865836],
    [ 0.0304721 ],
    [-0.85442941],
    [-0.19828127],
    [-0.53651297],
    [ 0.52033741],
    [-0.2781908 ],
    [-0.07071894],
    [-1.63579796]])
\end{lstlisting}

\subsection{Gradient Descent}

Gradient descent is one the most popular algorithms to optimize the neural networks. The name gradient descent is rooted in the procedure where the gradient is repeatedly evaluated to update the parameters. The objective of the gradient descent is to find weight parameters that will minimize the cost function.

To understand the concept of gradient descent we should ask ourselves the following question: What can be done to improve the weights we have assigned randomly at the beginning, so that the overall result improves?

To change the output of any node we should change the weights that connect it with the previous layer. Basically we want to find out how much error in every node changes once we change associated weights. Next we want to select the weights that would lead to a minimal error in the output layer. That can be achieved by differentiation of the cost function and search for its minimum.

Given multidimensionality of the function, which we need to differentiate, the search for its minimum can be a complicated task. This task is similar to some extent to the search of the path in the darkness from the top of a mountain to its valley. Because it is dark it is almost impossible to reach the valley immediately. The only way to achieve the goal is by exploring the neighbourhood (the radius you are able to see) and tacking small steps in the direction that leads downhill and constantly updating the path for the next steps. This process is illustrated on the Figure \ref{fig:gradient-descent}:

\begin{figure}[H]
    \includegraphics[width=\linewidth]{pics/gradient_descent.png}
    \caption{\label{fig:gradient-descent} How Gradient Descent works. Source: Giphy.com}
\end{figure}

Mathematically the differentiation process can be illustrated on the example of weights between output and hidden layers \(w_{ho}\). The same process but with corresponding values should be applied for the weights between input and hidden layers \(w_{ih}\).

As it can be seen from the formulas below the error we want to minimize \(E\) can be defined as the sum of squared differences between the target \(t_n\) and output \(o_n\) values of the NN. The sum of differences for all the nodes in the layer is relevant but when doing calculation for a particular node this sum can be omitted - only the difference between particular output \(o_o\) and target \(_o\) matters.

Target value is constant. Output value depends on weights and is obtained after applying sigmoid function to the sum of inputs (outputs of the previous layer - \(o_h\) multiplied by corresponding weights \(w_{ho}\).

\begin{equation}
\frac{\partial E}{\partial w_{ho}}=\frac{\partial}{\partial w_{ho}}\displaystyle\sum_{n=1}^{n} (t_n-o_n)^2=\frac{\partial}{\partial w_{ho}}(t_o-o_o)^2
\end{equation}

\begin{equation}
\frac{\partial E}{\partial w_{ho}}=\frac{\partial E}{\partial O_{o}}\frac{\partial O}{\partial w_{ho}}=-2(t_o-o_o)\frac{\partial O}{\partial w_{ho}}
\end{equation}

\begin{equation}
\frac{\partial E}{\partial w_{ho}}=-2(t_o-o_o)\frac{\partial}{\partial w_{ho}}sigmoid(\sum(w_{ho}o_h)
\end{equation}

The formula for derivative of the sigmoid function is formula 4. It is necessary to keep in mind that the sum to which we apply sigmoid function also depends on the change of weights \(w_{ho}\). Therefore one should follow the chain rule for derivation.

\begin{equation}
\frac{\partial}{\partial x}sigmoid(x)=sigmoid(x)(1-sigmoid(x))
\end{equation}

\begin{equation}
\frac{\partial E}{\partial w_{ho}}=-2(t_o-o_o)sigmoid(\sum_{h=1}^{h} (w_{ho}o_h)(1-sigmoid(\sum_{h=1}^{h} (w_{ho}o_h))\frac{\partial (\sum_{h=1}^{h} (w_{ho}o_h))}{\partial w_{ho}}
\end{equation}

\begin{equation}
\frac{\partial E}{\partial w_{ho}}=-2(t_o-o_o)sigmoid(\sum_{h=1}^{h} (w_{ho}o_h)(1-sigmoid(\sum_{h=1}^{h} (w_{ho}o_h))o_h
\end{equation}

The formula we derive is for one particular node. We can however apply it to all the nodes in the layer. In order to do so the only thing we need is to consider this formula in matrix notation. Thus, necessary update of weights linked to all the nodes in a layer will be calculated.

\begin{equation}
\frac{\partial E}{\partial W_{ho}}=-2*O_{error}*O_{output}*(1-O_{output})*O_h^T
\end{equation}

After solving the minimization problem we can update the weights we have assigned before.

\begin{equation}
new W_{ho}=old W_{ho}-\frac{\partial E}{\partial W_{ho}}
\end{equation}

\begin{equation}
\Delta W_{ho}=-2*(T_o-O_o)*sigmoid(O_o)*(1-sigmoid(O_o)*O_h^T
\end{equation}

In code this can be represented as follows:

\begin{lstlisting}[language=Python]
    # Update the matrix for weights between hidden and output layers:
    w_h_o += np.dot((o_errors * o_output * (1.0 - o_output)), np.transpose(h_output)) # 2 can be omitted as being related to the learning rate.
    # Update the matrix for weights between input and hidden layers:
    w_i_h += np.dot((h_errors * h_output * (1.0 - h_output)), np.transpose(input))"
\end{lstlisting}

\subsection{Learning Rate}

Now, there is something else, we should add in the weights updating procedure. If we completely change our weights with every new observation - our model learns to predict only the last input. Instead of updating weights 100 \% every time we can change them only partially - this way every new observation will bring some new knowledge while the previous one will still be in memory even though updated to certain extent. The bigger the learning rate the more importance has the last observation, the smaller it is the more important is all the previous knowledge. The smaller the steps - the more accurate will be the prediction. At the same time it might take more time to learn.

\begin{figure}[H]
    \centering
    \includegraphics[scale=0.5]{pics/learning_rate.png}
    \caption{\label{fig:bp} Learning Rate. Source: Business Analytics and Data Science Course by Professor S. Lessmann, Chapter 5: Artificial Neural Networks}
\end{figure}

Below is the code for weight's update procedure with learning rate included.

\begin{lstlisting}[language=Python]
    # Define the learning rate:
    l_r = 0.3

    # Update the weights for the links between the hidden and output layers:
    w_h_o += l_r * np.dot((o_errors * o_output * (1.0 - o_output)), np.transpose(h_output))
    # Update the weights for the links between the input and hidden layers:
    w_i_h += l_r * np.dot((h_errors * h_output * (1.0 - h_output)), np.transpose(input))
\end{lstlisting}

\subsection{Training}

So far we have been working with one particular observation. Let's put all the steps done before in a for-loop, so that we can perform them for all observations in our training set. More observations will allow the NN to learn from more information. Every time a new observation is feedforwarded, the error term backpropagated and the cost function minimized, the matrices of weights become more capable to label yet unknown observations.

\begin{lstlisting}[language=Python]
    for i in data:
        observation = i.split(',')
        input = np.array((np.asfarray(observation[1:])/255.0*0.99) + 0.01, ndmin=2).T
        target = np.array(np.zeros(o_n) + 0.01, ndmin=2).T
        target[int(observation[0])] = 0.99

        h_input = np.dot(w_i_h, input)
        h_output = sigmoid(h_input)
        o_input = np.dot(w_h_o, h_output)
        o_output = sigmoid(o_input)

        o_errors = target - o_output
        h_errors = np.dot(w_h_o.T, o_errors)

        w_h_o += l_r * np.dot((o_errors * o_output * (1.0 - o_output)), np.transpose(h_output))
        w_i_h += l_r * np.dot((h_errors * h_output * (1.0 - h_output)), np.transpose(input))

        pass
\end{lstlisting}

\subsection{Second evaluation of the results}

Once we have trained the model with 100 observations we can test it with new data it has never seen. After loading the test set we can first work with a particular observation to get an intuition about how good our NN can solve considered classification problem.

\begin{lstlisting}[language=Python]
    # Load the mnist test data CSV file:
    raw_data_test = open(\"data/mnist_test.csv\", 'r')
    data_test = raw_data_test.readlines()
    raw_data_test.close()
\end{lstlisting}

\begin{lstlisting}[language=Python]
    # Check a particular observation:
    observation = data_test[0].split(',')
    # Print the label:
    print(observation[0])
    # Image the number:
    image = np.asfarray(observation[1:]).reshape((28,28))
    mpp.imshow(image, cmap='Blues', interpolation='None')
\end{lstlisting}

\begin{lstlisting}
    7
\end{lstlisting}

\begin{figure}[H]
    \centering
   \includegraphics[scale=0.5]{pics/7.png}
   \caption{\label{fig:number7} Plot of the handwritten number}
\end{figure}

\begin{lstlisting}[language=Python]
    # Use this observation as an input and run NN with it:
    input = np.array((np.asfarray(observation[1:])/255.0*0.99) + 0.01, ndmin=2).T
    h_input = np.dot(w_i_h, input)
    h_output = sigmoid(h_input)
    o_input = np.dot(w_h_o, h_output)
    o_output = sigmoid(o_input)

    o_output
\end{lstlisting}

\begin{lstlisting}
   array([[ 0.05086044],
          [ 0.01692228],
          [ 0.03306648],
          [ 0.01855151],
          [ 0.17733202],
          [ 0.01942656],
          [ 0.01083799],
          [ 0.33279056],
          [ 0.13214786],
          [ 0.05988345]])
\end{lstlisting}

\begin{lstlisting}[language=Python]
    # Get the prediction of NN for this test observation:
    label = np.argmax(o_output)
    label
\end{lstlisting}

\begin{lstlisting}
    7
\end{lstlisting}

After working with a particular observation from the test set we can label all of them and evaluate the accuracy of our NN.

\begin{lstlisting}[language=Python]
    # Test the neural network using all test dataset:

    score = [] # create a list in which the predictions of the network will we saved.

    # Go through all the observations in the test data set:
    for i in data_test:
        observation = i.split(',')
        expected = int(observation[0])
        input = np.array((np.asfarray(observation[1:])/255.0*0.99) + 0.01, ndmin=2).T

        h_input = np.dot(w_i_h, input)
        h_output = sigmoid(h_input)
        o_input = np.dot(w_h_o, h_output)
        o_output = sigmoid(o_input)

        label = np.argmax(o_output)

        if (label == expected):
            score.append(1)
        else:
            score.append(0)
            pass

        pass
\end{lstlisting}

\begin{lstlisting}[language=Python]
    # Calculate the performance score, the fraction of correct answers:
    score_array = np.asarray(score)
    print ("performance = ", score_array.sum() / score_array.size)
    \end{lstlisting}

\begin{lstlisting}
    performance =  0.3959
\end{lstlisting}

  It is several times better than naive, which would be 0.1 (given that we have 10 levels of the categorical variable we have to classify). Can we do better?

\subsection{Further Improvements}

\subsubsection{Training with several epochs}

One way to improve the results of the NN is to train it more. For instance we can feedforward the same 100 observations more than once. Despite the fact that these are the same observations, longer training allows NN to accumulate more knowledge. Keep in mind that due to the presence of a learning rate NN receives only part of the information that is available and useful to predict particular observation. Seeing the same observations several times leads to smaller loss of the data.

So let's introduce one extra parameter called epochs and create a loop around the number of epochs. The rest of the code we see below is the same as before.

\begin{lstlisting}[language=Python]
    epochs = 5
\end{lstlisting}

\begin{lstlisting}[language=Python]
    # The "big loop" with epochs:
    for e in range(epochs):
        for i in data:
            observation = i.split(',')
            input = np.array((np.asfarray(observation[1:])/255.0*0.99) + 0.01, ndmin=2).T
            target = np.array(np.zeros(o_n) + 0.01, ndmin=2).T
            target[int(observation[0])] = 0.99

            h_input = np.dot(w_i_h, input)
            h_output = sigmoid(h_input)
            o_input = np.dot(w_h_o, h_output)
            o_output = sigmoid(o_input)

            o_errors = target - o_output
            h_errors = np.dot(w_h_o.T, o_errors)
            w_h_o += l_r * np.dot((o_errors * o_output * (1.0 - o_output)), np.transpose(h_output))
            w_i_h += l_r * np.dot((h_errors * h_output * (1.0 - h_output)), np.transpose(input))

            pass
        pass

    # test
    score = []

    for i in data_test:
        observation = i.split(',')
        correct_label = int(observation[0])
        input = np.array((np.asfarray(observation[1:])/255.0*0.99) + 0.01, ndmin=2).T

        h_input = np.dot(w_i_h, input)
        h_output = sigmoid(h_input)
        o_input = np.dot(w_h_o, h_output)
        o_output = sigmoid(o_input)

        label = np.argmax(o_output)
        if (label == correct_label):
            score.append(1)
        else:
            score.append(0)
            pass

        pass

    # calculate accuracy
    score_array = np.asarray(score)
    print ("performance = ", score_array.sum() / score_array.size)
\end{lstlisting}

\begin{lstlisting}
    performance =  0.959
\end{lstlisting}

\subsubsection{Training with other Learning Rate}

The smaller the learning rate the more capable the network to optimize the weights in a more accurate way. At the same time one should keep in mind that small l\_r also means additional loss of information extracted from each particular observation. Hence, there should be many training observations available in order to make the trade-off between accuracy and usage of available data reasonable. Given that we have more epochs now, it is interesting to try a smaller learning rate.

\begin{lstlisting}[language=Python]
    l_r = 0.1

    # run the "big loop" with epochs again to get measure accuracy for new settings.
\end{lstlisting}

\subsubsection{A more complicated structure}

As you may remember in the beginning we have assigned the number of nodes in the hidden layer based on some rule of thumb assumptions. Now we can test if the NN will perform better if we increase the number of hidden nodes.

\begin{lstlisting}[language=Python]
    h_n = 150

    # Determine the weights for a bigger matrices
    w_i_h = np.random.normal(0.0, pow(h_n, -0.5), (h_n, i_n))
    w_h_o = np.random.normal(0.0, pow(o_n, -0.5), (o_n, h_n))

    # run the "big loop" with epochs again to get measure accuracy for new settings.
\end{lstlisting}

It is always possible to train neural networks where the number of neurons is larger. But, with a smaller number of neurons the neural network has much better generalization abilities.

\textit{Overfitting.} To many nodes is one of the reasons that leads to a problem when the neural network is over trained which would mean that it will fail to recognize patterns which were never used in the training.

With a smaller number of neurons, it is more complicated to train the network to very small errors, but it may produce much better approximations for new patterns. The most common mistake made by many researchers is that in order to speed up the training process and to reduce the training errors, they use neural networks with a larger number of neurons than required. Such networks could perform poorly for new patterns not seen previously by the NN.

\subsubsection{Other training set}

One other source of improvement is providing the NN with a relatively big dataset for training. Everything that was done before was implemented with just 100 observations. Let's see if our results improve if we increase our training dataset to 60 000 observations. As we have more data now we will reduce the number of epochs and keep having low learning rate.

\begin{lstlisting}[language=Python]
    # Load the data
    raw_data = open("data/mnist_train.csv", 'r')
    data = raw_data.readlines()
    raw_data.close()

    # Settings
    epochs = 2
    l_r = 0.1
    h_n = 90
    w_i_h = np.random.normal(0.0, pow(h_n, -0.5), (h_n, i_n))
    w_h_o = np.random.normal(0.0, pow(o_n, -0.5), (o_n, h_n))

    # run the "big loop" with epochs again to get measure accuracy for new settings.
\end{lstlisting}

The result we achieve with a big training set is already pretty impressive. In more than 90\% of cases our NN is able to solve the classification problem properly. And we should remember that it was implemented from scratch using only basic linear algebra packages. Let's see in the following section if we can do better or if we can simplify the process using specialized packages to build neural networks.


\section{Analysis of Bubble Sort}

%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Time Complexity of Bubble Sort}
  \begin{itemize}
	\setlength{\itemsep}{10pt}
	\item Finiteness is NOT enough $\implies$ Quantifying finiteness
	  \pause
	\item Time on real computers varies $\implies$ \#Ops on our model:
	  \vspace{5pt}
	  \pause
	  \begin{description}
		\setlength{\itemsep}{5pt}
		% \item[$|P|:$] \#Passes \hfill (the ``{\bf for}'' loops)
		\item[$|C|:$] \#Comparisons \hfill ({\bf if} $a_{i} > a_{i+1}$)
		\item[$|S|:$] \#Swaps  \hfill ($\textsc{Swap}(a_{i}, a_{i+1})$)
	  \end{description}
	  % \pause
	  % \[
	  %   |C| \ge |S|
	  % \]
	  % \vspace{-0.50cm}
	  \pause
	\item Different inputs $\implies$ $|C|$ and $|S|$ vary:
	  \vspace{5pt}
	  \pause
	  \begin{itemize}
		\item Best-case, worst-case, and average-case analysis
	  \end{itemize}
\end{itemize}
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Best-case and Worst-case Analysis}
  \uncover<2->{
  \[
	\text{\textcolor{blue}{Best-case:} } 1\quad 2\; \cdots\; n\quad
  \]}

  \begin{displaymath}
	\begin{array}{lll}
	  & \hfill \textcolor{blue}{\frac{\text{Best-case:}}{\uncover<2->{\text{non-decreasingly sorted}}}} \hfill
	  & \hfill \textcolor{red}{\frac{\text{Worst-case:}}{\uncover<4->{\text{non-increasingly sorted}}}} \\[10pt]
	  % |P| & = (\uncover<3->{\min: 1,} & \uncover<5->{\max: n}); \\[6pt]
	  |C| & = (\uncover<3->{\min: n-1,} & \uncover<5->{\max: \frac{n^2 - n}{2}}); \\[6pt]
	  |S| & = (\uncover<3->{\min: 0,} & \uncover<5->{\max: \frac{n^2 - n}{2}}).
	\end{array}
  \end{displaymath}

  \uncover<4->{
	\[
	  \text{\textcolor{red}{Worst-case: }} n\quad n-1\; \cdots\; 1
	\]}
  % \uncover<5->{
  %   \[
  %     \#\text{inversions} = (n-1) + (n-2) + \cdots + 1 = \frac{n^2 - n}{2}.
  %   \]
  % }
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{$|S|:$ \#Swaps (Average-case Analysis)\footnote{An exercise: what is $|C|$ (\#Comparisions) in average?}}
  Assumptions on inputs:
  \begin{enumerate}
	\item The input is a random permutation (``average input'')
	  \pause
	\item All numbers are different (for simplicity)
  \end{enumerate}

  \pause

  \begin{center}
	\fbox{\textcolor{blue}{Lemma:} $\textsc{Swap}(a_{i}, a_{i+1}) \implies -1 \text{ inversion}$}
  \end{center}

  \[
	|S| = \textcolor{red}{\mathbb{E}(\#\text{inversions})}
  \]
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{$|S|:$ \#Swaps (Average Analysis)}
  \begin{align*}
	% \onslide<1->{I_{ij} &: \text{indicator of inversion } (a_i, a_j) \\[3pt]}
	\onslide<1->{I_{ij} &= \left\{\begin{array}{lr}
		1 & (a_i, a_j) \text{ is an inversion}\\
		0 & \text{ o.w.}
	  \end{array}\right.\\[3pt]}
	\onslide<1->{X &= \sum_{1 \le i < n} \sum_{i < j \le n} I_{ij}\qquad\quad (\text{\#inversions}) \\[3pt]}
	\onslide<2->{\mathbb{E}(X) &= \mathbb{E}(\sum_{i} \sum_{j>i} I_{ij})}\onslide<3->{= \sum_{i} \sum_{j>i} \mathbb{E}(I_{ij})\quad (\text{linearity of expectation})\\[3pt]}
	\onslide<4->{\mathbb{E}(I_{ij}) &= \mathbb{P}\set{I_{ij} = 1}} \onslide<5->{{}={} \frac{1}{2}\qquad (a_i \neq a_j; \text{half: } a_i < a_j, \text{half: } a_i > a_j) \\[3pt]}
	\onslide<6->{\mathbb{E}(X) &= \sum_{i} \sum_{i<j} \frac{1}{2} = \binom{n}{2} \cdot \frac{1}{2} = \frac{n(n-1)}{4} \textcolor{red}{{}={} O(n^2)}}
  \end{align*}
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Faster Algorithms}
  \begin{quote}
	It took a good deal of work to analyze the bubble sort;
	and although [\dots],
	the results are disappointing
	since they tell us that \textcolor{red}{the bubble sort isn't really very good at all}.\\
	\hfill --- Donald E. Knuth
  \end{quote}

  \pause

  \begin{center}
	faster: $O(n^2) \to O(n \lg n)$?\\[5pt] \pause
	\dots and faster: $O(n \lg n) \to O(n)$?
  \end{center}

  \pause
  \fignocaption{width = 0.18\textwidth}{figs/stay-tuned.jpg}
\end{frame}
%%%%%%%%%%%%%%%%%%%%
\begin{frame}[noframenumbering]
  \fignocaption{width = 0.20\textwidth}{figs/qa.png}
  \vspace{-0.8cm}
  \begin{center}
    \textcolor{blue}{\bf \large hengxin0912@gmail.com}
  \end{center}
  \vspace{-0.5cm}
  \fignocaption{width = 0.50\textwidth}{figs/thankyou.jpg}
\end{frame}
%%%%%%%%%%%%%%%%%%%%


\section{Path Tracer}
\label{path_tracer}
This section presents a closer look at the general path tracing process and highlights key implementation details of the evaluated CPU path tracer.
\subsection{Pipeline}
\begin{figure}[H]
    \centering
    \includegraphics[width=400pt]{images/ray_trcing_pipeline.pdf}
    \caption{Visualization of path tracing process as a pipeline.}
    \label{fig:ray_pipeline}
\end{figure}
The graphics rendering process if often organized into a graphics pipeline\cite{sugerman09gramps}, so I generalized the path tracing process in a similar fashion (figure \ref{fig:ray_pipeline}). In the first stage of that pipeline the acceleration structure is created, which will be discussed further in section \ref{phr}. Given a static scene, this acceleration structure can be reused for multiple frames. Dynamic scenes require additional care though, as the acceleration structure either needs to be refit or rebuild on scene changes. The implemented uses a hybrid approach further elaborated in section \ref{phr_in_interactive}. The next stages are responsible for ray traversal and intersection testing. First, primary rays are generated from the camera's origin towards each pixel. Then, each ray has to traverse the acceleration structure to find the closest hit. Depending on whether or not this intersection has been found, either the closest hit shader, or the miss shader is called. Each shader can contribute to the color of the pixel, but only the closest hit shader spawns secondary rays. Note that my implementation uses implicit light sources instead of explicitly casting shadow rays. Secondary rays are fed back to the ray traversal stage and after reaching the maximum depth, the closest hit shader exits without creating any additional rays. The steps in these stages are embarrassingly parallel, however, because every ray is independent of the others, \acrshort{simd} instructions cannot be utilized. Consequently, each pixel is processed concurrently using multithreading.
\subsection{Intersection Tests}
A ray can be expressed using the parametric form
\begin{equation} \label{eq:1}
    R(t)=O+t\textbf{d}
\end{equation}
where point $O$ defines the origin of the ray and vector \textbf{d} defines the direction along which the ray travels in a straight line. The path tracer supports two types of primitives, spheres and triangles.
\subsubsection{Sphere}
Spheres are commonly used in ray tracing because of their simple and efficient intersection algorithm\cite{haines2019}. Given a sphere with center $C$ and radius $r$, all points $P$ at the surface of the sphere can be described by the equation:
\begin{equation} \label{eq:2}
    (P-G)\cdot(P-G)=r^2
\end{equation}
By substituting point $P$ in equation \ref{eq:2} by the equation of a ray (equation \ref{eq:1}) the intersection points between sphere and ray can be calculated. Simplifying the resulting equation leads to
\[
    (\textbf{d}\cdot\textbf{d})t^2+2(\textbf{f}\cdot\textbf{d})t+\textbf{f}\cdot\textbf{f}-r^2 = at^2+b^t+c = 0
\]
which is a quadratic function. Consequently, it can be solved using
\[
    t_{0,1}=\frac{-b\pm\sqrt{b^2-4ac}}{2a}
\]
with discriminant $\Delta=b^2-4ac$.
\begin{figure}[H]
    \centering
    \includegraphics[width=150pt]{images/ray_sphere_intersection.pdf}
    \caption{Ray-sphere intersection test showing all three cases for discriminant $\Delta$.}
    \label{fig:sphere_intersec}
\end{figure}
As seen in figure \ref{fig:sphere_intersec}, if $\Delta<0$, the ray misses the sphere. If $\Delta=0$ the ray touches the sphere in one point and otherwise there are two values for $t$ that correspond to different intersection points. Inserting these $t$-values into equation \ref{eq:1} allows the calculation of intersection points $P_{0,1} = R(t_{1,2}) = O+t_{0,1}$\textbf{d}. The normal $n$ at a given intersection point is simply the vector from the spheres center $C$ to the intersection point $P$, i.e the normalized normal is calculated using $n=||P-C||$.

\subsubsection{Triangle}
Triangles have a long tradition in computer graphics, mainly because most geometry can be represented, or at least approximated, using them. Additionally, with 3 vertecies they can never be non-planar. While many ray-triangle intersection algorithms exist, the M{\"o}ller-Trumbore algorithm\cite{moeller97triangle} is still considered to be relatively fast and is often used as a comparison for other algorithms. Consequently, it is also used in this thesis.

Using barycentric coordinates, a point $P$ on a triangle can be parameterized and expressed through two scalar values:
\[
    P(u,v) = (1-u-v)V_0 + uV_1 + vV_2
\]
where $V_0$, $V_1$ and $V_2$ are the vertices of given triangle and $(u,v)$ are the barycentric coordinates with $u\geq0$,  $v\geq0$ and $u+v\leq1$. Barycentric coordinates do not change when the triangle is transformed, which the M{\"o}ller-Trumbore algorithm exploits. Geometrically speaking, the algorithm translates the triangle to the origin and transforms it into a unit triangle in $y$ and $z$, with the ray direction aligned with $x$. This can be expressed as a system of linear equations:
\[
    \left[ \begin{array}{rrr}-\textbf{d}&(V_1-V_0)&(V_2-V_0)\end{array} \right]
    \left[ \begin{array}{r}t\\u\\v\end{array} \right] = O - V_0
\]
\begin{figure}[H]
    \centering
    \includegraphics[width=400pt]{images/ray_triangle_intersec.pdf}
    \caption{Geometric representation of the transformations utilized in the M{\"o}ller-Trumbore algorithm.}
    \label{fig:triangle_intersec}
\end{figure}
Using Cramer's rule\cite{brunetti14cramersRule}, this system can be solved to obtain the distance $t$ from ray origin to intersection point and the corresponding barycentric coordinates $(u,v)$. Distance $t$ is again plugged into the ray equation to find the world coordinates of intersection point $P$ and the barycentric coordinates are used to interpolate the normal at $P$, given the vertex normals at $V_0$, $V_1$ and $V_2$.

\subsubsection{Axis-aligned bounding box}
Finally, bounding volume hierarchy traversal depends on a ray-bounding box intersection test. Axis-aligned bounding boxes are represented by the bounds $B^{min}$ and $B^{max}$, which define two planes for each dimension. The intersection distances $t$ between a given ray $R(t) = O+t\textbf{d}$ and a dimension $k$ can be computed fairly simple:
\[
    t^{min}_k = \frac{B_{k}^{min} - O_{k}}{\textbf{d}_{k}}
\]
\[
    t^{max}_k = \frac{B_{k}^{max} - O_{k}}{\textbf{d}_{k}}
\]
Whether or not a ray intersects a box, can be determined by a few simple value comparisons, as seen in figure  \ref{fig:aabb_intersec}. The algorithm is optimized further by pre-computing sign and inverse direction for all rays and adding an early exit once it is clear the ray missed.
\begin{figure}[H]
    \centering
    \includegraphics[width=400pt]{images/aabb_intersec.pdf}
    \caption{2D example of ray-AABB intersection. The top ray misses, while the bottom one hits the box.}
    \label{fig:aabb_intersec}
\end{figure}
\clearpage
\subsection{Materials}
Each primitive has an associated material. A material has an albedo specifying how reflective it is, i.e. how much color is contributed to the traced path, and an amount of emitted light. The scatter function differs between materials and is used to generate secondary rays. Note that it is also possible for materials to not scatter at all, which is used for light sources.

Diffuse materials scatter rays in random directions. To achieve true Lambertian reflectance\cite{weik01lambert}, random points are picked on the surface of a unit sphere and added to the normal at intersection point. This results in a distribution of $cos(\Phi)$, where $\Phi$ is the angle from the normal.

A smooth reflective material\cite{Greve2004ReflectionsAR} does not scatter rays in a random direction, so the resulting rays point purely in the reflection direction $\textbf{d}_r$ (figure \ref{fig:reflect_refract}):
\[
    \textbf{d}_r = \textbf{d} - 2\textbf{n}(\textbf{d}\cdot \textbf{n})
\]
where $\textbf{n}$ is the normalized normal at intersection point and $\textbf{d}$ is the direction of the incoming ray. For diffuse reflection a random vector is added to the reflected ray, similar to the above mentioned diffuse scattering.

Refractive materials\cite{Greve2004ReflectionsAR} utilize Snell's law to represent dielectrics like glass objects. Snell's law can be written as
\[
    sin \theta' = \frac{\eta}{\eta'}sin \theta
\]
where $\theta$ and $\theta'$ are angles from the normal and $\eta$ and $\eta'$ are refractive indices. The refracted direction $\textbf{d}_t$, as shown in figure \ref{fig:reflect_refract}, can be split up into a parallel and a perpendicular part:
\[
    \textbf{d}_t=\textbf{d}_{\bot}+\textbf{d}_{\parallel}
\]
Solving for $\textbf{d}_{\parallel}'$ and $\textbf{d}_{\bot}'$ leads to:
\[
     \textbf{d}_{\bot}' = \frac{\eta}{\eta'}(\textbf{d}+(-\textbf{d}\cdot\textbf{n}) \textbf{n})
\]
\[
    \textbf{d}_{\parallel}' = - \sqrt{1-|\textbf{d}_{\bot}'|^{2}\textbf{n}}
\]
Adding both parts together, leads to the refracted direction $\textbf{d}_t$
\begin{figure}[H]
    \centering
    \includegraphics[width=200pt]{images/reflection_refraction.pdf}
    \caption{Figure showing how a ray's direction (green) is reflected (red) and refracted (blue).}
    \label{fig:reflect_refract}
\end{figure}
\subsection{Traversal}
\label{traversal}
To find the nearest intersection, bounding volume hierarchies are traversed in a top-down manner. Usually, this is done using a stack\cite{meister21survey} to store nodes that might contain an intersection. First, the root is pushed to the stack. While the stack is not empty, nodes are popped and checked for an intersection with their bounding box. In case the bounding volume is hit, either the nodes children are pushed onto the stack in the case of an interior node, or all primitives are tested when dealing with a leaf node. If no intersection with the bounding box is found or the distance to the intersection is bigger than previously found intersections, then the node can be discarded. Once the stack is empty, the closest intersection found is returned.

However, this method proved to be less efficient than using a recursive procedure as described in algorithm \ref{alg:traversal}. Given that the implementation is written for the CPU, this implicitly uses the CPU stack, which results in an equivalent execution as described above, albeit without explicitly managing a stack data structure.
\begin{algorithm}
\caption{Pseudocode of recursive BVH traversal}
\label{alg:traversal}
    \tcp{Start traversal at root}

    \SetKwFunction{FTraverse}{Traverse}
    \FTraverse{root}\;
    \;
    \SetKwProg{Fn}{Function}{:}{}
    \Fn{\FTraverse{node}}{
        \If{node.aabb not intersected} {
            \Return
        }
        \If{node is leaf} {
            \ForEach{primitive}{
                test for ray-primitive intersection
            }
        }
        \Else{
            \ForEach{child}{
                \FTraverse{child}
            }
        }
    }
\end{algorithm}
\cleardoublepage


%!TEX root = ../main.tex

%=======================================================================  numbers
\section{Numbers}
\label{sec:numbers}

	In the beginning, we must define the main players in the world of math: numbers.

	\subsection{Definitions}
	\label{numbers:definitions}

		Numbers are the basic objects we use to count, measure, quantify, and calculate things.
		Mathematicians like to classify the different kinds of number-like objects into categories called \emph{sets}:					\index{set}
		\begin{itemize}
			\item  The natural numbers: $\mathbb{N} = \{0,1,2,3,4,5,6,7, \ldots \, \}$
			\item  The integers: $\mathbb{Z} = \{\ldots, -3,-2,-1,0,1,2,3 , \ldots  \, \}$
			\item  The rational numbers: $\mathbb{Q} = \{\frac{5}{3}, \frac{22}{7}, 1.5, 0.125,  -7, \, \ldots \, \}$
			\item  The real numbers: $\mathbb{R} = \{-1,0,1, \sqrt{2}, e,\pi, \;  4.94\ldots, \; \ldots \, \}$
			\item  The complex numbers: $\mathbb{C} = \{ -1, 0, 1, i,  1+i, 2+3i,  \ldots \, \}$
		\end{itemize}
		%
		These categories of numbers should be somewhat familiar to you.
		Think of them as neat classification labels for everything that you would normally call a number.
	 	Each group in the above list is a \emph{set}.
		A set is a collection of items of the same kind.
		Each collection has a name and a precise definition for which items belong in that collection.
		Note also that each of the sets in the list contains all the sets above it,												\index{set!subset}
		as illustrated in Figure~\ref{fig:nested_sets}.
		For now, we don't need to go into the details of sets and set notation,
		but we do need to be aware of the different sets of numbers.

		\begin{figure}[H]
			\centering
			\includegraphics[width=0.87\textwidth]{figures/math/nested_sets.pdf}
			\caption{	An illustration of the nested containment structure of the different number sets.
					The set of natural numbers is contained in the set of integers,
					which in turn is contained in the set of rational numbers.
					The set of rational numbers is contained in the set of real numbers,
					which is contained in the set of complex numbers.}
			\label{fig:nested_sets}
		\end{figure}

		\noindent
		Why do we need so many different sets of numbers?
		Each set of numbers is associated with more and more advanced mathematical problems.

		The simplest numbers are the natural numbers $\mathbb{N}$,
		which are sufficient for all your math needs if all you're going to do is \emph{count} things.
		How many goats? Five goats here and six goats there so the total is 11 goats.
		The sum of any two natural numbers is also a natural number.

		As soon as you start using \emph{subtraction} (the inverse operation of addition),
		you start running into negative numbers,
		which are numbers outside the set of natural numbers.
		If the only mathematical operations you will ever use are \emph{addition} and \emph{subtraction},
		then the set of integers $\mathbb{Z} = \{ \ldots, -2, -1, 0, 1, 2, \ldots \}$ will be sufficient.
		Think about it.
		Any integer plus or minus any other integer is still an integer.

		You can do a lot of interesting math with integers.
		There is an entire field in math called \emph{number theory} that deals with integers.
		However, to restrict yourself solely to integers is somewhat limiting---a rotisserie
		menu that offers $\frac{1}{2}$ of a chicken would be totally confusing.

		If you want to use division in your mathematical calculations,														\index{rational}
		you'll need the rationals~$\mathbb{Q}$.
		The set of rational numbers corresponds to all numbers that can be expressed as \emph{fractions} of the form $\frac{m}{n}$		\index{fraction}
		where $m$ and $n$ are integers, and $n \neq 0$.
		You can add, subtract, multiply, and divide rational numbers, and the result will always be a rational number.
		However, even the rationals are not enough for all of math!

		In geometry, we can obtain \emph{irrational} quantities like $\sqrt{2}$ (the diagonal of a square with side 1)
		and $\pi$ (the ratio between a circle's circumference and its diameter).
		There are no integers $x$ and $y$ such that $\sqrt{2}=\frac{x}{y}$,
		therefore we say that $\sqrt{2}$ is \emph{irrational} (not in the set $\mathbb{Q}$).
		An irrational number has an infinitely long decimal expansion that doesn't repeat.
		For example, $\pi = 3.141592653589793\ldots$ where the dots indicate
		that the decimal expansion of $\pi$ continues all the way to infinity.

		Combining the irrational numbers with the rationals gives us all the useful numbers,
		which we call the set of real numbers $\mathbb{R}$.
		The set $\mathbb{R}$ contains the integers,
		the rational numbers $\mathbb{Q}$,
		as well as irrational numbers like $\sqrt{2}=1.4142135\ldots$.
		By using the reals you can compute pretty much anything you want.
		From here on in the text, when I say \emph{number},
		I mean an element of the set of real numbers $\mathbb{R}$.

		The only thing you can't do with the reals is to take the square root of a negative number---you 							\index{complex number}
		need the complex numbers $\mathbb{C}$ for that.
		We defer the discussion on $\mathbb{C}$ until the end of Chapter~\ref{chapter:vectors}.

	\subsection{Exercises}
	\label{numbers:exercises}

		\input{problems/chapter1_numbers_exercises.tex}


%!TEX root = ../notes.tex

\section{February 7, 2022}
\subsection{Orders mod \texorpdfstring{$p$}{p}}
\recall If $a\not\equiv 0\pmod{p}$, then we have $a^{p-1}\equiv 1\pmod{p}$, which was \cref{theorem:flt}, Fermat's Little Theorem.

\begin{definition}[Order of $a$ mod $p$]
    The \ul{order} of $a \pmod{p}$ is the smallest positive $k$ such that
    \begin{equation*}
        a^k\equiv 1\pmod{p}
    \end{equation*}
    This is not to be confused with \cref{defn:order-of-prime-factor} which is the power of $p$ in the prime factorization of $a$. This is the order of $a$ in the multiplicative group $\ZZ/p\ZZ$.
\end{definition}

\begin{proposition}
    let $a\in (\ZZ/p\ZZ)^\times$ be of order $k$. If $a^n\equiv 1\pmod{p}$, then $k\mid n$.

    In particular, $k\mid p-1$ by \cref{theorem:flt}, Fermat's Little Theorem.
\end{proposition}
\begin{proof}
    We write $n = k\cdot q + r$ such that $0\leq r < k$ ($\ZZ$ is a Euclidean domain)
    \begin{align*}
        1 & \equiv a^n\equiv a^{kq+r} \equiv (a^k)^q\cdot a^r\equiv a^r
    \end{align*}
    Since $k$ is the minimal positive number such that $a^k\equiv 1$, then this forces $r = 0$. Then $k\mid n$.
\end{proof}

\begin{theorem}[Primitive Root Theorem]
    Let $p$ be prime. Then there is a $g$ such that
    \[(\ZZ/p\ZZ)^\times = \{1, g, g^2, \dots, g^{p-2}\}.\]
    We call $g$ a \ul{primitive root} or \ul{generator}.
\end{theorem}
\begin{example}
    $p = 5$, $(\ZZ/5\ZZ)^\times = \{1, 2, 3, 4\}$.

    1? No: $\{1, 1^2, 1^3\} = \{1\}$

    2? Yes: $\{1, 2, 2^2, 2^3\} = \{1, 2, 4, 3\}$

    3? Yes: $\{1, 3, 3^2, 3^3\} = \{1, 3, 4, 2\}$

    4? No: $\{1, 4, 4^2, 4^3\} = \{1, 4\}$
\end{example}

\begin{remark}
    In general, the number of primitive roots is $\varphi(p-1)$. (Take the group of exponents and solve for power).
\end{remark}

\subsection{Discrete Logarithm Problem}

We go on to discuss a fundamental property about exponentiation mod $p$. Let's fix some $p$ and primitive root $g$.

Given some $a$, we can compute $g^a$ efficiently
\begin{align*}
    a & \longrightarrow g^a \qquad \text{This is \ul{easy}}             \\
    a & \overset{?}{\longleftarrow} g^a \qquad \text{This is \ul{hard}}
\end{align*}

Note that
\[g^a\equiv g^b \Leftrightarrow g^{a-b}\equiv 1 \Leftrightarrow p-1\mid a-b\]
so $a$ is determined mod $p-1$.

\begin{definition}[Discrete Logarithm]
    The \ul{discrete logarithm} of $g^a$ is $a$.
\end{definition}

This is known as the ``Discrete Logarithm Problem'' (DLP), which is concerned with how we can compute discrete logarithms.

This idea is fundamental to computer security! The real-world analogue is if you go to the bank after hours and deposit a check or cash into the deposit slot. It is relatively easy for one to deposit an item but hard for someone who doesn't work at the bank\footnote{Say, possessing a \emph{key} or \emph{password}.} to access that item.

\subsection{Cryptographic Systems}
\subsubsection{Symmetric Cryptography}
We have $3$ people, \emph{Alice}, \emph{Bob}, and \emph{Eve}.

Bob has a message $m$ which he wants to send to Alice. However, everything he sends to Alice can (and is) intercepted by Eve. He wants to encrypt this message $m$ he sends to Alice.

We say that a message $m\in \mathcal{M}$ in the space of possible messages. We have secret key $k\in \mathcal{K}$ that can encrypt $m$ into ciphertext $c\in \mathcal{C}$ in the space of ciphertexts.
\[\left\{\begin{array}{c}
        \text{Message $m\in \mathcal{M}$} \\
        \text{Secret key $k\in\mathcal{K}$}
    \end{array}\right\} \rightsquigarrow \text{Ciphertext $c\in\mathcal{C}$} \longrightarrow \text{Alice} \rightsquigarrow m\]

If we fix $k$, we have
\begin{align*}
    e_k(m) & = e(k, m) \\
    d_k(c) & = d(k, c)
\end{align*}
be our encryption and decryption functions. We usually take $m$ to be a number, and we can encode letters to numbers (0-255) using ASCII.

In Python, this is implemented using functions like \textsf{ord} (character to encoding) and \textsf{chr} (encoding to character).

We'll just talk about transmitting numbers since we can convert freely between them and text.

\textbf{Q: What do we want out of our cryptosystem?}
\begin{enumerate}
    \setcounter{enumi}{-1}
    \item The system is secure even if Eve knows the design. (Assume Eve knows the encryption and decryption functions, but so long as she doesn't know the key).
    \item $e$, the encryption function, is easy to compute.
    \item $d$, the decryption function, is similarly easy to compute.
    \item Given $c_1, c_2, \dots$ a collection of ciphertexts, encrypted with the \emph{same} key $k$, it's hard to compute any message $m_i$.
    \item Given $(m_1, c_1), \dots (m_n, c_n)$ some collection of messages and their encryptions, it remains difficult to compute $d_k(c)$ for $c\not\in \{c_1, \dots, c_n\}$. This is called a \textit{``\ul{chosen plaintext attack}''}.
\end{enumerate}