\input{../header_function}

%---------- start document ---------- %
 \section{algorithm -- basic number theoretic algorithms}\linkedzero{algorithm}
%
  \subsection{digital\_method -- univariate polynomial evaluation}\linkedone{algorithm}{digital\_method}
   \func{digital\_method}
   {%
     \hiki{coefficients}{list},\ %
     \hiki{val}{object},\ %
     \hiki{add}{function},\ %
     \hiki{mul}{function},\ %
     \hiki{act}{function},\ %
     \hiki{power}{function},\ %
     \hiki{zero}{object},\ %
     \hiki{one}{object}
   }{%
     \out{object}%
   }\\
   \spacing
   % document of basic document
   \quad Evaluate a univariate polynomial corresponding to \param{coefficients} at \param{val}.\\
   \spacing
   % added document
   %\spacing
   % input, output document
   \quad  If the polynomial corresponding to \param{coefficients} is of $R$-coefficients for some ring $R$,
   then \param{val} should be in an $R$-algebra $D$.\\
   \param{coefficients} should be a descending ordered list of tuples $(d,\ c)$,
   where $d$ is an integer which expresses the degree and $c$ is an element of $R$ which expresses the coefficient.
   All operations 'add', 'mul', 'act', 'power', 'zero', 'one' should be explicitly given, where:\\
      'add' means addition $(D \times D \to D)$,
      'mul' multiplication $(D \times D \to D)$,
      'act' action of $R$ $(R \times D \to D)$,
      'power' powering $(D \times \mathbf{Z} \to D)$,
      'zero' the additive unit (an constant) in $D$ and
      'one', the multiplicative unit (an constant) in $D$.
%
  \subsection{digital\_method\_func -- function of univariate polynomial evaluation}\linkedone{algorithm}{digital\_method\_func}
   \func{digital\_method}
   {%
     \hiki{add}{function},\ %
     \hiki{mul}{function},\ %
     \hiki{act}{function},\ %
     \hiki{power}{function},\ %
     \hiki{zero}{object},\ %
     \hiki{one}{object}
   }{%
     \out{function}%
   }\\
   \spacing
   % document of basic document
   \quad Return a function which evaluates polynomial corresponding to 'coefficients' at 'val'
from an iterator 'coefficients' and an object 'val'.\\
   \spacing
   % added document
   %\spacing
   % input, output document
   \quad  All operations 'add', 'mul', 'act', 'power', 'zero', 'one' should be inputted
   in a manner similar to \linkingone{algorithm}{digital\_method}.\\
%
  \subsection{rl\_binary\_powering -- right-left powering}\linkedone{algorithm}{rl\_binary\_powering}
   \func{rl\_binary\_powering}
   {%
     \hiki{element}{object},\ %
     \hiki{index}{integer},\ %
     \hiki{mul}{function},\ %
     \hikiopt{square}{function}{None},\ %
     \hikiopt{one}{object}{None},\ %
   }{%
     \out{object}%
   }\\
   \spacing
   % document of basic document
   \quad Return \param{element} to the \param{index} power by using right-left binary method.\\
   \spacing
   % added document
   %\spacing
   % input, output document
   \quad  \param{index} should be a non-negative integer.
   If \param{square} is None, \param{square} is defined by using \param{mul}.\\
%
  \subsection{lr\_binary\_powering -- left-right powering}\linkedone{algorithm}{lr\_binary\_powering}
   \func{lr\_binary\_powering}
   {%
     \hiki{element}{object},\ %
     \hiki{index}{integer},\ %
     \hiki{mul}{function},\ %
     \hikiopt{square}{function}{None},\ %
     \hikiopt{one}{object}{None},\ %
   }{%
     \out{object}%
   }\\
   \spacing
   % document of basic document
   \quad Return \param{element} to the \param{index} power by using left-right binary method.\\
   \spacing
   % added document
   %\spacing
   % input, output document
   \quad  \param{index} should be a non-negative integer.
   If \param{square} is None, \param{square} is defined by using \param{mul}.\\
%
  \subsection{window\_powering -- window powering}\linkedone{algorithm}{window\_powering}
   \func{window\_powering}
   {%
     \hiki{element}{object},\ %
     \hiki{index}{integer},\ %
     \hiki{mul}{function},\ %
     \hikiopt{square}{function}{None},\ %
     \hikiopt{one}{object}{None},\ %
   }{%
     \out{object}%
   }\\
   \spacing
   % document of basic document
   \quad Return \param{element} to the \param{index} power by using small-window method.\\
   \spacing
   % added document
   \quad The window size is selected by average analystic optimization.\\
   \spacing
   % input, output document
   \quad  \param{index} should be a non-negative integer.
   If \param{square} is None, \param{square} is defined by using \param{mul}.\\
%
  \subsection{powering\_func -- function of powering}\linkedone{algorithm}{powering\_func}
   \func{powering\_func}
   {%
     \hiki{mul}{function},\ %
     \hikiopt{square}{function}{None},\ %
     \hikiopt{one}{object}{None},\ %
     \hikiopt{type}{integer}{0}
   }{%
     \out{function}%
   }\\
   \spacing
   % document of basic document
   \quad Return a function which computes 'element' to the 'index' power from an object 'element' and an integer 'index'.\\
   \spacing
   % added document
   %\spacing
   % input, output document
   \quad If \param{square} is None, \param{square} is defined by using \param{mul}.
   \param{type} should be an integer which means one of the following:\\
   0;\ \linkingone{algorithm}{rl\_binary\_powering}\\
   1;\ \linkingone{algorithm}{lr\_binary\_powering}\\
   2;\ \linkingone{algorithm}{window\_powering}\\
\\
%
\begin{ex}
>>> d_func = algorithm.digital_method_func(
... lambda a,b:a+b, lambda a,b:a*b, lambda i,a:i*a, lambda a,i:a**i,
... matrix.zeroMatrix(3,0), matrix.identityMatrix(3,1)
... )
>>> coefficients = [(2,1), (1,2), (0,1)] # X^2+2*X+I
>>> A = matrix.SquareMatrix(3, [1,2,3]+[4,5,6]+[7,8,9])
>>> d_func(coefficients, A) # A**2+2*A+I
[33L, 40L, 48L]+[74L, 92L, 108L]+[116L, 142L, 169L]
>>> p_func = algorithm.powering_func(lambda a,b:a*b, type=2)
>>> p_func(A, 10) # A**10 by window method
[132476037840L, 162775103256L, 193074168672L]+[300005963406L, 368621393481L,
 437236823556L]+[467535888972L, 574467683706L, 681399478440L]

\end{ex}%Don't indent!(indent causes an error.)
\C

%---------- end document ---------- %

\input{../footer}


\documentclass{article}

\usepackage[numbers, compress]{natbib}

\usepackage[utf8]{inputenc} % allow utf-8 input
\usepackage[T1]{fontenc}    % use 8-bit T1 fonts
\usepackage{hyperref}       % hyperlinks
\usepackage{url}            % simple URL typesetting
\usepackage{booktabs}       % professional-quality tables
\usepackage{amsfonts}       % blackboard math symbols
\usepackage{nicefrac}       % compact symbols for 1/2, etc.
\usepackage{microtype}      % microtypography

\usepackage{amsthm,amsmath,amssymb}
\usepackage{macros}
\usepackage{subcaption}
\usepackage[textfont=small, labelfont=small]{caption}
\usepackage{graphicx}
\DeclareGraphicsExtensions{.pdf,.png,.jpg,.eps}

\usepackage{algorithm}
\usepackage{algorithmic}

\usepackage[color=yellow]{todonotes}
\usepackage{booktabs}
\usepackage[inline]{enumitem}
\usepackage{verbatim}

\usepackage[margin=1in]{geometry}

\usepackage{setspace}

\title{Information form filtering and smoothing for Gaussian linear dynamical systems}

\author{Scott W. Linderman
  \and
  Matthew J. Johnson
}

\begin{document}

\singlespacing
\maketitle
\onehalfspacing

The \emph{information form} of the Gaussian density of~$x \in \reals^n$ is defined as,
\begin{align}
  p(x \given J, h)
  &=  \exp \left \{ -\frac{1}{2} x^\trans J x + h^\trans x - \log Z \right \},
\end{align}
where
\begin{align}
  \log Z = \frac{1}{2} h^\trans J^{-1} h - \frac{1}{2}\log |J| +\frac{n}{2} \log 2\pi.
\end{align}
The standard formulation is recovered by the transformations,  $\Sigma = J^{-1}$, and
$\mu = J^{-1} h$. The advantage of working in the information form is that
it corresponds to the natural parameterization of the Gaussian distribution, and
mean field variational inference is considerably easier with this form.

In order to perform Kalman filtering and smoothing, we must be able to perform
two operations: \emph{conditioning} and \emph{marginalization}.

\paragraph{Conditioning}
If,
\begin{align}
  p(x) &= \distNormal(x \given J, h) \\
  p(y \given x) &\propto \distNormal(x \given J_{\sf obs}, h_{\sf obs})
\end{align}
then,
\begin{align}
  p(x \given y) &= \distNormal(x \given J + J_{\sf obs}, h + h_{\sf obs}).
\end{align}

\paragraph{Marginalization}
If,
\begin{align}
  \begin{bmatrix} x_1 \\ x_2  \end{bmatrix}
  &\sim
  \distNormal \left(
  \begin{bmatrix}
    J_{11}        & J_{12} \\
    J_{12}^\trans & J_{22}
  \end{bmatrix},
  \begin{bmatrix} h_1 \\ h_2 \end{bmatrix}
  \right),
\end{align}
then,
\begin{align}
  x_2 &\sim \distNormal(
        J_{22} - J_{21}J_{11}^{-1} J_{21}^\trans, \;
        h_2 - J_{21} J_{11}^{-1} h_1)
\end{align}

\begin{proof}
  We use the following integral identity,
\begin{align}
  \int \exp \left \{ - \frac{1}{2} x^\trans J x + h^\trans x\right\} \mathrm{d} x
  &= \exp \left\{ \frac{1}{2} h^\trans J^{-1} h - \frac{1}{2} \log |J| + \frac{n}{2} \log 2 \pi \right\}.
\end{align}
For~$x = [x_1, x_2]^\trans$, we marginalize out~$x_1$ via,
\begin{align}
  \int \exp \left \{ - \frac{1}{2}
  \begin{bmatrix} x_1^\trans & x_2^\trans \end{bmatrix}
  \begin{bmatrix} J_{11} & J_{12} \\ J_{21} & J_{22} \end{bmatrix}
  \begin{bmatrix} x_1 \\  x_2 \end{bmatrix}
  + \begin{bmatrix} h_1^\trans & h_2^\trans \end{bmatrix}
  \begin{bmatrix} x_1 \\  x_2 \end{bmatrix}
  - \log Z
  \right \}
  \mathrm{d} x_1.
\end{align}
Rearranging terms, we have,
\begin{align}
  &\exp \left \{ -\frac{1}{2} x_2^\trans J_{22} x_2 + h_2^\trans x_2 - \log Z \right \}
  \int \exp \left \{- \frac{1}{2} x_1^\trans J_{11} x_1 + (h_1 - J_{12}x_2)^\trans x_1 \right \} \mathrm{d}x_1 \\
  &= \exp \left \{ -\frac{1}{2} x_2^\trans J_{22} x_2 + h_2^\trans x_2 - \log Z \right \}
  \exp \left \{ \frac{1}{2} (h_1 - J_{12}x_2)^\trans J_{11}^{-1} (h_1 - J_{12} x_2) - \frac{1}{2} \log |J_{11}| + \frac{n}{2} \log 2 \pi \right \} \\
  &= \exp \left \{ -\frac{1}{2} x_2^\trans (J_{22} - J_{21}J_{11}^{-1} J_{12}) x_2
  + (h_2 - J_{21} J_{11}^{-1} h_1)^\trans x_2 - \log Z \right \} \nonumber \\
  &\qquad \qquad \times
  \exp \left \{ \frac{1}{2} h_1^\trans J_{11}^{-1} h_1 - \frac{1}{2} \log |J_{11}| + \frac{n}{2} \log 2 \pi \right \}.
\end{align}
We recognize this as a Gaussian potential on~$x_2$ of the form,
\begin{align}
  p(x_2) &= \exp \left \{-\frac{1}{2} x_2^\trans \widetilde{J}_2 x_2 + \widetilde{h}_2^\trans x_2 - \log \widetilde{Z}_2 \right \} \\
  \widetilde{J}_2 &= J_{22} - J_{21}J_{11}^{-1} J_{12} \\
  \widetilde{h}_2 &= h_2 - J_{21} J_{11}^{-1} h_1 \\
  \log \widetilde{Z}_2 &= \log Z -\frac{1}{2} h_1^\trans J_{11}^{-1} h_1 + \frac{1}{2} \log |J_{11}| - \frac{n}{2} \log 2 \pi.
\end{align}
\end{proof}

\section*{Filtering, Sampling, and Smoothing}
By interleaving these two steps we can filter, sample, and smooth the latent states
in a linear dynamical system. Take the model,
\begin{align}
  x_1 &\sim \distNormal(\mu_1, Q_1) \\
  x_{t+1} &\sim \distNormal(A_t x_t + B_t u_t, Q_t) \\
  y_t &\sim \distNormal(C_t x_t + D_t u_t, R_t).
\end{align}
In information form, the initial distribution is,
\begin{align}
  % initi
  x_1 &\sim \distNormal(J=Q_1^{-1}, h = Q_1^{-1} \mu_1).
\end{align}
The dynamics are given by,
\begin{align}
  % dynamics
  p(x_{t+1} \given x_t) & \propto
  \distNormal \left(
  \begin{bmatrix} x_t \\ x_{t+1}  \end{bmatrix}
  \, \bigg| \,
  \begin{bmatrix}
    J_{11}        & J_{12} \\
    J_{12}^\trans & J_{22}
  \end{bmatrix},
  \begin{bmatrix} h_{1} \\ h_2 \end{bmatrix}
  \right),
\end{align}
with,
\begin{align}
  J_{11} &= A_t^\trans Q_t^{-1} A_t, \quad
  J_{12} = -A_t^\trans Q_t^{-1} \quad
  J_{22} = Q_t^{-1} \quad
  h_1 = -u_t^\trans B_t^\trans Q_t^{-1} A_t \quad
  h_2 = u_t^\trans B_t Q_t^{-1}.
\end{align}
Finally, the observations are given by,
\begin{align}
  p(y_t \given x_t)
  &\propto \distNormal(x_t \given J_{\sf obs}, h_{\sf obs})
\end{align}
with
\begin{align}
  J_{\sf obs} = C_t^\trans R_t^{-1} C_t \quad
  h_{\sf obs} = (y_t - D_t u_t)^\trans R_t^{-1} C_t
\end{align}

\subsection*{Filtering}
We seek the conditional distribution,~$p(x_t \given y_{1:t})$, which
will be Gaussian. We begin with the initial distribution,
\begin{align}
  p(x_1) &= \distNormal(x_1 \given J_{1 | 0}, h_{1 | 0}).
\end{align}
Assume, inductively, that~$x_t \given y_{1:t-1} \sim \distNormal(J_{t|t-1}, h_{t|t-1})$. Conditioning on the~$t$-th observation yields,
Conditioned on the first observation,
\begin{align}
  p(x_t \given y_{1:t}) &= \distNormal(x_t \given J_{t|t}, h_{t|t}), \\
  J_{t|t} &= J_{t|t-1} + J_{\sf obs} \\
  h_{t|t} &= h_{t|t-1} + h_{\sf obs}.
\end{align}
Then, we predict the next latent state by writing the joint distribution of~$x_t$ and~$x_{t+1}$ and marginalizing out~$x_t$.
\begin{align}
  p(x_{t+1} \given y_{1:t}) &= p(x_t \given y_{1:t}) \, p(x_{t+1} \given x_t) \\
  &= \distNormal(x_t \given J_{t+1|t}, h_{t+1|t}) \\
  J_{t+1|t} &= J_{22} - J_{21} (J_{t|t} + J_{11})^{-1}J_{21}^\trans \\
  h_{t+1|t} &= h_2 - J_{21} (J_{t|t} + J_{11})^{-1} (h_{t|t} + h_1)
\end{align}.
This completes one iteration and provides the input to the next. To start the recursion, we initialize,
\begin{align}
  J_{1|0} = \Sigma_{\sf init}^{-1}, \quad
  h_{1|0} = \Sigma_{\sf init}^{-1} \, \mu_{\sf init}.
\end{align}

\subsection*{Marginal Likelihood}
This filtering algorithm corresponds to message passing
in a chain-structured Gaussian graphical model. To compute
the marginal likelihood,~$p(y_{1:T})$, we observe that it
is the normalization constant of the graphical model,
\begin{align}
  p(x_{1:T}) &=
  \frac{1}{Z} \prod_{t=1}^T \psi(x_t) \prod_{t=1}^{T-1} \psi(x_t, x_{t+1}), & & \\
  \psi(x_1) &= p(x_1) \, p(y_1 \given x_1)  & & \\
  \psi(x_t) &= p(y_t \given x_t)  & \text{for } t&=2 \ldots T \\
  \psi(x_t, x_{t+1}) &= p(x_{t+1} \given x_t)  & \text{for } t&=1 \ldots T-1.
\end{align}
To compute the normalizaion constant~$Z$, we recursively eliminate nodes
via message passing.

\paragraph{Base case.} Let us work backward from the final step in which
we are left with a graph with a single, unnormalized Gaussian potential.
\begin{align}
  p(x_T) &= \frac{1}{Z} \psi(x_T)
  = \frac{1}{Z} \exp \left \{-\frac{1}{2} x_T^\trans J_{T} x_T + h_{T}^\trans x_T - \log Z_{T} \right \}
\end{align}
To compute the normalizing constant,~$Z$, we integrate over~$x_T$ and use
the normalizing constant for Gaussian distributions:
\begin{align}
  Z &= \int \exp \left \{-\frac{1}{2} x_T^\trans J_{T} x_T + h_{T}^\trans x_T - \log Z_{T} \right \} \mathrm{d} x_T \\
  &= \exp \left \{ -\log Z_{T} \right \}
  \int \exp \left \{-\frac{1}{2} x_T^\trans J_{T} x_T + h_{T}^\trans x_T \right \} \mathrm{d} x_T \\
  &= \exp \left \{ -\log Z_{T} +\frac{1}{2} h_{T}^\trans J_{T}^{-1} h_{T} - \frac{1}{2} \log |J_{T}| +\frac{n}{2} \log 2 \pi \right \}
\end{align}

\paragraph{Condition Step.} In the second to last step, we have two potentials,
one from the dynamics induced by the preceding step and one from the observation:
\begin{align}
  p(x_T) &= \frac{1}{Z} \psi_{\sf dyn}(x_T) \, \psi_{\sf obs}(x_T) \\
  &= \exp \left \{-\frac{1}{2} x_T^\trans J_{T|T-1} x_T + h_{T|T-1}^\trans x_T - \log Z_{T|T-1} \right \}
  \exp \left \{-\frac{1}{2} x_T^\trans J_{\sf obs} x_T + h_{\sf obs}^\trans x_T - \log Z_{\sf obs} \right \}
\end{align}
We reduce this to the base case by simply summing the natural parameters and
log normalizers,
\begin{align}
  J_{T} &= J_{T|T-1} + J_{\sf obs} \\
  h_{T} &= h_{T|T-1} + h_{\sf obs} \\
  \log Z_{T} &= \log Z_{T|T-1} + \log Z_{\sf obs}.
\end{align}

\paragraph{Predict Step.} Now consider a model with two latent states.
\begin{align}
  p(x_{T-1}, x_T) &= \frac{1}{Z} \psi(x_{T-1}) \, \psi_{\sf dyn}(x_{T-1}, x_T) \, \psi_{\sf obs}(x_T)
\end{align}
We will eliminate the variable~$x_{T-1}$ by marginalizing, or integrating it out.
\begin{align}
  p(x_T) &= \frac{1}{Z} \psi_{\sf obs}(x_T) \int \psi(x_{T-1}) \, \psi_{\sf dyn}(x_{T-1}, x_T) \mathrm{d}x_{T-1}
\end{align}
The integrand is of the form,
\begin{align}
  &\psi(x_{T-1}) \, \psi_{\sf dyn}(x_{T-1}, x_T)
  \\
  &= \exp \left \{ -\frac{1}{2} x_{T-1}^\trans J_{T-1} x_{T-1} + h_{T-1}^\trans x_{T-1} - \log Z_{T-1} \right \} \\
  &\qquad \times
  \exp \left \{ - \frac{1}{2}
  \begin{bmatrix} x_{T-1}^\trans & x_T^\trans \end{bmatrix}
  \begin{bmatrix} J_{11} & J_{12} \\ J_{21} & J_{22} \end{bmatrix}
  \begin{bmatrix} x_{T-1} \\  x_T \end{bmatrix}
  + \begin{bmatrix} h_1^\trans & h_2^\trans \end{bmatrix}
  \begin{bmatrix} x_{T-1} \\  x_T \end{bmatrix}
  - \log Z_{\sf dyn}
  \right \}
  \\
  &=
  \exp \left \{ - \frac{1}{2}
  \begin{bmatrix} x_{T-1}^\trans & x_T^\trans \end{bmatrix}
  \begin{bmatrix} J_{11} + J_{T-1} & J_{12} \\ J_{21} & J_{22} \end{bmatrix}
  \begin{bmatrix} x_{T-1} \\  x_T \end{bmatrix}
  + \begin{bmatrix} (h_1 + h_{T-1})^\trans & h_2^\trans \end{bmatrix}
  \begin{bmatrix} x_{T-1} \\  x_T \end{bmatrix}
  - (\log Z_{T-1} + \log Z_{\sf dyn})
  \right \}
\end{align}
Appealing to the marginalization propositions above, this implies,
\begin{align}
  \int \psi(x_{T-1}) \, \psi_{\sf dyn}(x_{T-1}, x_T) \, \mathrm{d} x_{T-1}
  &= \exp \left \{-\frac{1}{2} x_T^\trans J_{T|T-1} x_T + h_{T|T-1}^\trans x_T
  - \log Z_{T|T-1} \right \},
\end{align}
where
\begin{multline}
  \log Z_{T|T-1} = \\
  \log Z_{T-1} + \log Z_{\sf dyn}
  -\frac{1}{2} (h_1 + h_{T-1}^\trans) (J_{11} + J_{T-1})^{-1} (h_1 + h_{T-1}^\trans)
  + \frac{1}{2} \log |J_{11} + J_{T-1}| - \frac{n}{2} \log 2 \pi.
\end{multline}
Thus, the log normalizer passed into the condition step is an accumulation of
log normalizers from previous time steps ($\log Z_{T-1}$), the log normalizer
of the dynamics potential ($\log Z_{\sf dyn}$), and a term that comes from
marginalizing out the local variable~$x_{T-1}$. Once we have computed
~$\log Z_{T|T-1}$, it is passed into the condition step and then into the final
integration to compute the marginal likelihood.

This process of predicting and conditioning is recursively applied, marginalizing
out variables one at a time, starting with~$x_1$ and ending with~$x_T$. At the
end of this procedure, we are left with the marginal likelihood of the observations.

\subsection*{Standard Form Marginal likelihood}
The marginal likelihood of the observed data is given by,
\begin{align}
  \log p(y_{1:T} \given u_{1:T})
  &= \sum_{t=1}^T \log p(y_t \given y_{1:t-1}, u_{1:t}) \\
  &= \sum_{t=1}^T \log \distNormal(y_t \given C \mu_{t | t-1} + D u_t,  S_t),
\end{align}
where
\begin{align}
  S_t &= C \Sigma_{t | t-1} C^\trans + R_t \\
  \mu_{t | t-1} &= J_{t | t-1}^{-1} h_{t|t-1} \\
  \Sigma_{t | t-1} &= J_{t|t-1}^{-1}
\end{align}
Expanding the above, we have,
\begin{align}
  \log p(y_{1:T} \given u_{1:T})
  &= \sum_{t=1}^T \left[ -\frac{N}{2} \log 2 \pi - \frac{1}{2} \log \big| S_t  \big|
     - \frac{1}{2} (y_t - C \mu_{t | t-1} - D u_t)^\trans S_t^{-1}
     (y_t - C \mu_{t | t-1} - D u_t) \right]
\end{align}

\subsection*{Backward Sampling}
Having computed~$J_{t|t}$ and~$h_{t|t}$, we the proceed backward in time to draw a joint sample of the latent states.
Given~$J_{t|t}$,~$h_{t|t}$, and~$x_{t+1}$, we have,
\begin{align}
  p(x_{t} \given y_{1:t}, x_{t+1}) &\propto p(x_t \given y_{1:T}) \, p(x_{t+1} \given x_t) \\
  &\propto \distNormal(x_t \given J_{t|t}, h_{t|t})  \;
    \distNormal(x_t \given J_{11}, \, h_1 - x_{t+1}^\trans J_{21} ) \\
  &\propto     \distNormal(x_t \given J_{t|t} + J_{11}, \; h_{t|t} + h_1 - x_{t+1}^\trans J_{21} )
\end{align}
We sample~$x_t$ from this conditional, then use it to sample~$x_{t-1}$, and repeat until we reach~$x_1$.

\subsection*{Rauch-Tung-Striebel Smoothing}
Next we seek the conditional distribution given all the data,~$p(x_t \given y_{1:T})$.
This will again be Gaussian, and we will call its parameters~$J_{t|T}$ and~$h_{t|T}$.
Assume, inductively, that we have computed~$J_{t+1|T}$ and~$h_{t+1|T}$. We show how to
compute the parameters for time~$t$.

From the Markov properties of the model and the conditional distribution derived above, we have,
\begin{align}
  p(x_t \given x_{t+1}, y_{1:T})
  &= \distNormal(x_t \given J_{t|t} + J_{11}, \; h_{t|t} + h_1 - J_{12} x_{t+1}).
\end{align}
Expanding, taking care to note that~$x_{t+1}$ appears in the normalizing constant, yields,
\begin{multline}
  p(x_t \given x_{t+1}, y_{1:T})
  = \exp \bigg \{-\frac{1}{2} x_t^\trans (J_{t|t} + J_{11}) x_t + (h_{t|t} + h_1)^\trans x_t - x_{t+1}^\trans J_{12} x_t
  \\
-\frac{1}{2} x_{t+1}^\trans J_{12}^\trans (J_{t|t} + J_{11})^{-1} J_{12} x_{t+1}
+(h_{t|t} + h_1)^\trans (J_{t|t} + J_{11})^{-1} J_{12} x_{t+1} \\
- \frac{1}{2} (h_{t|t} + h_1)^\trans (J_{t|t} + J_{11})^{-1} (h_{t|t} + h_1)
    \bigg \}
\end{multline}

Now consider the joint distribution of~$x_t$ and~$x_{t+1}$ given all the data,
\begin{align}
  p(x_t, x_{t+1} \given y_{1:T})
  &= p(x_t \given x_{t+1}, y_{1:T}) p(x_{t+1} \given y_{1:T}) \\
  &\propto \distNormal \left(
  \begin{bmatrix} x_t \\ x_{t+1}  \end{bmatrix}
  \, \bigg| \,
  \begin{bmatrix}
    \widetilde{J}_{11}        & \widetilde{J}_{12} \\
    \widetilde{J}_{12}^\trans & \widetilde{J}_{22}
  \end{bmatrix},
  \begin{bmatrix}
    \widetilde{h}_1 \\
    \widetilde{h}_2
  \end{bmatrix}
  \right),
\end{align}
with,
\begin{align}
  \widetilde{J}_{11} &= J_{t|t} + J_{11} \\
  \widetilde{J}_{12} &= J_{12} \\
  \widetilde{J}_{22} &= J_{t+1|T} + J_{12}^\trans (J_{t|t} + J_{11})^{-1} J_{12} \\
  \widetilde{h}_1 &= h_{t|t} + h_1 \\
  \widetilde{h}_2 &= h_{t+1|T} + (h_{t|t} + h_1)^\trans (J_{t|t} + J_{11})^{-1} J_{12}.
\end{align}
Recall that,
\begin{align}
  J_{t+1|t} &= J_{22} - J_{21} (J_{t|t} + J_{11})^{-1}J_{21}^\trans \\
  h_{t+1|t} &= h_2 - J_{21} (J_{t|t} + J_{11})^{-1} (h_{t|t} + h_1).
\end{align}
Thus,
\begin{align}
  \widetilde{J}_{22} &= J_{t+1|T} - J_{t+1|t} + J_{22} \\
  \widetilde{h}_{2} &= h_{t+1|T} - h_{t+1|t} + h_2.
\end{align}

Finally, marginalize,
\begin{align}
  p(x_t \given y_{1:T})
  &= \distNormal(x_t \given
    \widetilde{J}_{11} - \widetilde{J}_{12} \widetilde{J}_{22}^{-1} \widetilde{J}_{12}^\trans, \;
    \widetilde{h}_{1} - \widetilde{J}_{12} \widetilde{J}_{22}^{-1} \widetilde{h}_2) \\
  &= \distNormal(x_t \given J_{t|T}, h_{t|T}).
\end{align}
Substituting the simplified forms above yields,
\begin{align}
  J_{t|T} &= J_{t|t} + J_{11} - J_{12} (J_{t+1|T} - J_{t+1|t} + J_{22})^{-1} J_{12}^\trans \\
  h_{t|T} &= h_{t|t} + h_1 - J_{12} (J_{t+1|T} - J_{t+1|t} + J_{22})^{-1} (h_{t+1|T} - h_{t+1|t} +h_2).
\end{align}

\appendix

\subsection*{Working out marginal likelihood in a simple example}
\begin{align}
  p(x) &= \frac{1}{Z} \distNormal(x \given 0, 1) \, \distNormal(1 \given x, \sigma^2) \\
  &= \frac{1}{Z}
  \exp \left \{ -\frac{1}{2} x^2 -\frac{1}{2} \log 2 \pi \right \}
  \exp \left \{ -\frac{1}{2} \frac{(x-1)^2}{\sigma^2}  -\frac{1}{2} \log 2 \pi -\frac{1}{2} \log \sigma^2 \right \} \\
  &= \frac{1}{Z} \exp \left \{ -\frac{1}{2} (x^2 + \frac{x^2}{\sigma^2}) -\frac{1}{2} \frac{-2x}{\sigma^2} -\frac{1}{2}\frac{1}{\sigma^2}  - \log 2 \pi  -\frac{1}{2} \log \sigma^2 \right \} \\
  &= \frac{1}{Z} \exp \left \{ -\frac{1}{2} (1 + \frac{1}{\sigma^2}) x^2 + \frac{x}{\sigma^2} -\frac{1}{2\sigma^2}  - \log 2 \pi  -\frac{1}{2} \log \sigma^2 \right \}
\end{align}
This implies that the normalization constant is,
\begin{align}
  Z &= \exp \left \{  - \log 2 \pi -\frac{1}{2\sigma^2} + \frac{1}{2} (\frac{1}{\sigma^2}(1+\frac{1}{\sigma^2})^{-1} \frac{1}{\sigma^2}) - \frac{1}{2} \log |1+\frac{1}{\sigma^2}| + \frac{1}{2} \log 2 \pi -\frac{1}{2} \log \sigma^2\right \} \\
  &= \exp \left \{ -\frac{1}{2} \log 2 \pi - \frac{1}{2\sigma^2} + \frac{1}{2\sigma^2} \frac{1}{1+\sigma^2} - \frac{1}{2} \log(1+\sigma^2) + \frac{1}{2}\log \sigma^2 -\frac{1}{2} \log \sigma^2\right \}\\
  &= \exp \left \{ -\frac{1}{2} \log 2 \pi - \frac{1}{2}  \frac{1}{1+\sigma^2} - \frac{1}{2} \log(1+\sigma^2) \right \}
\end{align}
From the standard marginal likelihood, we know that this is like,
\begin{align}
  \log p(y=1) &= \log \distNormal(1 \given 0, 1 + \sigma^2) \\
  &= -\frac{1}{2} \log 2 \pi -\frac{1}{2} \log (1+\sigma^2) - \frac{1}{2} \frac{1^2}{1+\sigma^2} \\
%  &= -\frac{1}{2} \log 2 \pi -\frac{1}{2} \log 2 - \frac{1}{4}
\end{align}
Thankfully, all checks out!

\subsection*{Working out marginal likelihood in an input example}
\begin{align}
  p(x) &= \frac{1}{Z} \distNormal(x \given 0, 1) \, \distNormal(1 \given x + d, 1) \\
  &= \frac{1}{Z}
  \exp \left \{ -\frac{1}{2} x^2 -\frac{1}{2} \log 2 \pi \right \}
  \exp \left \{ -\frac{1}{2} (x+d-1)^2  -\frac{1}{2} \log 2 \pi \right \} \\
  &= \frac{1}{Z} \exp \left \{ -\frac{1}{2} (x^2 + x^2) -\frac{1}{2} 2x(d-1) -\frac{1}{2}(d-1)^2  - \log 2 \pi  \right \} \\
  &= \frac{1}{Z} \exp \left \{ -\frac{1}{2} 2x^2 + x(1-d) -\frac{1}{2}(d-1)^2  - \log 2 \pi  \right \}
\end{align}
This implies that the normalization constant is,
\begin{align}
  Z &= \exp \left \{  - \log 2 \pi -\frac{1}{2}(d-1)^2  + \frac{1}{2} \frac{(1-d)^2}{2} - \frac{1}{2} \log |2| + \frac{1}{2} \log 2 \pi \right \} \\
    &= \exp \left \{  - \frac{1}{2}\log 2 \pi -\frac{1}{4}(d-1)^2   - \frac{1}{2} \log |2| \right \}
\end{align}
From the standard marginal likelihood, we know that this is like,
\begin{align}
  \log p(y=1) &= \log \distNormal(1 \given d, 2) \\
  &= -\frac{1}{2} \log 2 \pi -\frac{1}{2} \log 2 - \frac{1}{2} \frac{(1-d)^2}{2} \\
%  &= -\frac{1}{2} \log 2 \pi -\frac{1}{2} \log 2 - \frac{1}{4}
\end{align}
Thankfully, all checks out!

\end{document}


\documentclass{article}
\usepackage[top=2cm,bottom=2cm,left=2cm,right=2cm]{geometry}
\usepackage{amsmath, amssymb, amsthm}
\usepackage{tikz}
\usepackage{hyperref}

\usepackage{datetime}
\newdateformat{petsa}{\the\day\ \shortmonthname[\the\month] \the\year}

\theoremstyle{plain}
\newtheorem{definition}{Definition}[section]
\newtheorem{theorem}{Theorem}[section]
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{conjecture}[theorem]{Conjecture}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{exercise}{Exercise}[section]

\title{Fractal Tangency}
\author{poypoyan}
\date{\petsa\today}

\begin{document}

\maketitle

\begin{abstract}
Fractals are fascinating objects, but are still considered as pathological because most of the time, analytical techniques do not work on those. For example, the well-known ``inscribed square problem" is still unsolved for most fractal-like curves \cite{square}. In this short note we attempt to add something else to say to fractals by carefully distinguishing them from `smooth' curves. For now, this study is restricted to the continuum, but we provide some ideas for generalization to other sets.
\end{abstract}

\section{Intuition}
For a start, paths to a point are enough to determine fractality. For example, we can say that a surface is `fractal' if a path passing through a point inside that surface is fractal. When restricted to continuous curves in the continuum ($\mathbb{R}^n$), fractals can be thought of as exactly the nowhere differentiable functions. We somewhat follow that line of thought, but please remember a definition of the derivative: that it is the slope of the \textit{tangent line}. This means that differentiable curves will eventually look like a straight line when `zoomed in' to a point.

On the other hand, fractals are often viewed as `self-similar' (although it should not always be \cite{3b1b}!). Now observe that a straight line looks like itself when zoomed in. Thus, straight line is self-similar. In a sense, the straight line is to fractals as 0 is to positive natural numbers!

Since the derivative already captures \textit{tangency}, we can modify this to define \textit{fractal tangency}, and distinguish fractals from `smooth' curves. The following sections just develop a formalization to this idea.

\section{Definitions}
Let $C_n$ (for $n \ge 1$) be the set of all continuous functions $f:[0,1]\rightarrow\mathbb{R}^n$ such that $f(0)=(0)^n$ and $f(x) \ne (0)^n, \forall x \ne 0$, where $(0)^n$ is the origin on $\mathbb{R}^n$. Aside from the Cartesian coordinates, we make use of a variant of \textit{Hyper-spherical coordinates} (we shorten it as \textit{HSC} here). Let the set of HSCs be $S_n = \mathbb{R} \times [0, \pi]^{n-1}$ ($S_1 = \mathbb{R}$).

\begin{definition}\label{hsc}
Let $T_n: S_n \rightarrow \mathbb{R}^n$ (again, for $n \ge 1$) be the transformation from Hyper-spherical coordinates back to Cartesian coordinates such that:
\begin{itemize}
	\item $T_1(r) = r$.
	\item $T_{n+1}(r, \theta_1, \ldots, \theta_n) = ( \sin{\theta_n} \cdot T_n(r, \theta_1, \ldots, \theta_{n-1}), |r| \cos{\theta_n})$.
\end{itemize}
\end{definition}
\begin{exercise}[aka author is lazy]
Prove that $T_n$ is bijective \textit{except at origin} either by explicit construction of $T_n^{-1}$, or by showing surjection and injection. Please utilize induction if possible.
\end{exercise}
\begin{definition}
\textbf{Quasi-multiplication on HSC} is the binary operation $\cdot: S_n \times S_n \rightarrow S_n$ such that $$(r, \theta_1, \ldots, \theta_{n-1}) \cdot (s, \phi_1, \ldots, \phi_{n-1}) = (rs, \pi - |\pi - (\theta_1 + \phi_1)|, \ldots, \pi - |\pi - (\theta_{n-1} + \phi_{n-1})|).$$
\end{definition}
\begin{definition}
\textbf{Quasi-division on HSC} is the binary operation $/: S_n \times S_n \rightarrow S_n$ such that $$(r, \theta_1, \ldots, \theta_{n-1}) / (s, \phi_1, \ldots, \phi_{n-1}) = (r/s, |\theta_1 - \phi_1|, \ldots, |\theta_{n-1} - \phi_{n-1}|).$$
\end{definition}
Notice the operations on angles. The idea is that if the sum/difference of angles is outside of $[0,\pi]$, we put it back to $[0,\pi]$ by `remeasuring' it from angle $0$.

\section{Equivalence Relation}
Now we can define the main concept to distinguish the rough from the smooth:
\begin{definition}\label{equiv}
Two continuous paths to the origin $f, g \in C_n$ have the `same tangency at origin' (we notate as $f \sim_n g$) if there exists a `hyper-rotation' $r = (1, \theta_1, \ldots, \theta_{n-1}) \in S_n$ such that $$\lim_{x\rightarrow 0^{+}} \frac{r \cdot T_n^{-1}(f(x))}{T_n^{-1}(g(x))} = (1, (0)^{n-1}).$$
\end{definition}
\begin{theorem}
$\sim_n$ is an equivalence relation on $C_n$.
\end{theorem}
\begin{proof}
For reflexivity, the $r$ for $f \sim_n f$ would be $(1, (0)^{n-1})$. For symmetry, given the $r$ for $f \sim_n g$, the $r'$ for $g \sim_n f$ would be $(1, (0)^{n-1})/r$. Lastly for transitivity, given the $r$ for $f \sim_n g$ and $r'$ for $g \sim_n h$, the $r''$ for $f \sim_n h$ would be $r \cdot r'$.
\end{proof}
\begin{corollary}
$\sim_n$ partitions $C_n$ into the set of equivalence classes $C_n / \sim_n$.
\end{corollary}
One such class is the class $L_n$ of `smooth' paths to a point, or more informatively, the class of paths with the \textit{tangent line} at origin.
\begin{definition}
For a continuous path to the origin $f \in C_n$, $f \in L_n$ iff $f \sim_n (x, (0)^{n-1})$.
\end{definition}
Now we can finally distinguish fractals, and define the title of this note:
\begin{definition}
A continuous path to the origin $f \in C_n$  has \textbf{fractal tangency} at origin iff $f \not\in L_n$.
\end{definition}

\section{Other Ideas}
Here are some other ideas/comments for further study of the concepts outlined in previous sections:
\begin{itemize}
	\item The limit in Definition \ref{equiv} can be simplified to $$\lim_{x \rightarrow 0^+}\frac{T_n^{-1}(f(x))}{T_n^{-1}(g(x))} = (1, \theta_1, \ldots, \theta_{n-1}).$$ This suggests a weakening of the equivalence relation: the limit should now be $(r, \theta_1, \ldots, \theta_{n-1})$ for any $r \ne 0$. However, at the time of writing, we do not have a good `geometric' interpretation for this.
	\item Of course there are fractals beyond $C_n$ like the well-known Sierpiński carpet! The first idea is that paths on \textit{sets} can be defined as $f:[0,1] \rightarrow 2$. Fortunately there are no notions of continuity, HSC, and hyper-rotation anymore. Then the next idea is to modify the $\epsilon$ part of the $\epsilon$-$\delta$ definition: instead of $|f(x)-L|<\epsilon$, we simply have $f(x) \oplus L = 0$, where $\oplus$ is the XOR operation. Now the only perfectly `smooth' paths are $f_0(x)=0$ and $f_1(x)=1$, for all $x\in[0,1]$. The last idea is to define a suitable (quasi-)multiplication and (quasi-)division: maybe the XOR operation can also be used for that. Hopefully we can now define the analogue for Definition \ref{equiv}. These ideas should generalize to other sets aside from $\mathbb{R}^n$ and $2$.
\end{itemize}

\bibliographystyle{plain}
\bibliography{fractal}
\end{document}


\documentclass{article}

\usepackage[hidelinks]{hyperref}

\title{Math 15 Project - An algorithmic approach to Nonhomogenious equation solving}
\date{2021-06-04}
\author{Elias Schablowski}

\begin{document}
\pagenumbering{gobble}
\maketitle
\newpage
\tableofcontents
\pagebreak
\pagenumbering{arabic}
\section{Abstract}
\paragraph{Differential Equations} are a cornerstone of the physical world. Many processes can be measured and modeled most accurately with ODE.
\paragraph{Non-homogenious Differential Equations} are a family of equations which is more difficult to solve due to their dependence on two variables. It is also a very broad range of equations, for example, this category includes forced oscilators, double pendulums, and more.
\paragraph{Methods to solve} There are multiple methods to solve non-homogenious equations. Each with their own advantages and drawbacks. The most basic is undetermined coefficients.
\section{Theory}

\subsection{Undetermined Coefficients}
\paragraph{The Idea} behind undetermined coefficients is to take the sum of all linearly independent differentials. This is so that the differentials can cancel each other out when differentiated and thus form the the final equation.
$$y_p=\sum_{n=0}^Na_n f^{(n)}$$ Where N is the number of linearly independent differentials and $a_n$ is a scaling factor.
\paragraph{The advantage} of undetermined coefficients is that it is capable of turning a differential equation into a system of equations in O(N) time.
\paragraph{The disadvantage} of undetermined coefficients is the requirement for a finite set of linearly independent derivatives. This excludes many common trigonometric functions such as $tan(x)$ and $\frac{a}{x^n}$.

\subsection{Variation of Parameters}
\paragraph{The Idea} behind variation of parameters is to modify the the complementary solution in such a way as to create the forcing function.
\paragraph{The advantage} of variation of parameters is that it can solve a wide array of problems.
\paragraph{The disadvantages} of variation of parameters is that it often does not result in a system of equations with a single solution, and often requires the addition of additional constraints, therefore not lending itself to a computerized approach without bias. Furthermore, it is also requires the solutions to the complementary equation to be know, which further adds to the complexity.

\subsection{The Laplace Transform}
\paragraph{The Idea} behind using the laplace transform is replacing a problematic function with one that is simpler to solve with, and can be transformed back without information loss (1 to 1 transformation).
\paragraph{Why the Laplace Transform?} The Laplace Transform has multiple properties that make it an ideal candedate to solve ODEs and PDEs. These properties are linearity and uniqeness. Furthermore, many properties also resemble those of differentiation.
\paragraph{The disadvantage} of the Laplace Transform is that computing the laplace transform requires solving an itegral which is complex.

\section{Algorithm}
\subsection{Inputs}
\paragraph{The Equation} is inputed as two expression trees, with one representiong the ODE/PDE and the other representing the function.
\paragraph{The Preferred Method} is an optional parameter which specifies the preferred method to solve the ODE/PDE.
\subsection{Selection of Method}
This process is one of elimination, namely finding whether the faster methods work.
\subsubsection{Undetermined Coefficients}
The method in which you can determine whether the ODE can be solved using Undetermined Coefficients is to determine whether the function has a finite amount liearly independent derivatives.
The most efficient method is to determine whether the function has a finite amount of derivaties is to check whether the function has either a function with an infinete amount linearly independent derivatives, division by a variable, or exponentiation with a variable base and exponent
as per \hyperref[sec:derivative_proofs]{Proofs}.
\subsubsection{Variation of Parameters}
This method is not used, due to the proof of aplicability is more expensive than the efficiency gains of the Laplace Transform.
\subsubsection{The Laplace Transform}
This method is used when Undetermined Coefficients cannot be used.
\subsection{Undetermined Coefficients}
\begin{enumerate}
    \item Find All linearly independent derivatives
    \item Sum all linearly independent derivatives together
    \item Remove all constant coefficients and shifts.
    \item Inject variables into all critical locations (coefficients of functions and variables as well as shifts)
    \item Plug this new function into the ODE
    \item Solve the equation for the values of the injected variables \footnote{In my implementation, this was done using a system of equations (which limits this implementation to polynomials)}
    \item Substitute the found values in for the injected variables
\end{enumerate}

\subsection{Laplace Transform}
\begin{enumerate}
    \item Compute the Laplace Transform of the ODE.
    \item Query for the initial conditions.
    \item Solve for $X(s)$\footnote{I used Wolfram Alpa to do this step due to the complexities of Symbolic solving.}
    \item Compute the Inverse Laplace Transform
\end{enumerate}
\footnotetext{See the implementation (as well as the tex document for this paper on \hyperlink{https://github.com/eschablowski/}{})}
\newpage
\section{Proofs}
\subsection{Finiteness of Derivates}
\label{sec:derivative_proofs}
\paragraph{Note: } I made these proof, as I didn't find any general method of checking for infine linearly independent derivaties.
\subsubsection{Addition}
\paragraph {Let} $f$ and $g$ be any function.
\paragraph {Let} $y = f(x) + g(x)$
\paragraph{Let} $N_f$ be the number of linearly independent derivatives of $f$, $N_g$ be the number of linearly independent derivatives of $g$.
$$y^{(n)}=f^{(n)}(x)+g^{(n)}(x)$$
\paragraph{Let} $L_f = \sum_{n=0}^{N_f} f^{(n)}$ and $L_g = \sum_{n=0}^{N_g} g^{(n)}$
\paragraph{Since} $L_f$ and $L_g$ are the sums of linearly independent derivatives of $f$ and $g$ respectively,
then the sum linearly independent derivatives of $y$ is $L_g + L_f$.
\paragraph{Therefore} the linearly independent derivaties of $y$ is fine as long as $N_g \neq \infty$ and $N_f \neq \infty$

\subsubsection{Multiplication}
\paragraph {Let} $f$ and $g$ be any function.
\paragraph {Let} $y = f(x) * g(x)$
\paragraph{Let} $N_f$ be the number of linearly independent derivatives of $f$, $N_g$ be the number of linearly independent derivatives of $g$, and $N$ be $N_f$ if $N_f > N_g$ or $N_g$ otherwise.
$$y^{(n)}(x)=\sum_{i=1}^n f^{(i-n)}(x) * g^{(n-i)}(x)$$
\paragraph{Therefore,} if $f$ and $g$ have finite linearly independent derivatives, $y$ must also have a finine number of linearly independent derivatives.

\pagebreak
\section{Further items for Consideration}
\subsection{The Gaver/Stehfest Algorithm}
The Gaver/Stehfest Algorithm is a method for approximating the value of the inverse Laplace Transform. \citation{stehfest_1970}
This can be useful for solving ODEs for program internals and/or creating a function from ODEs.
The reason why it was not used in this program was due to the fact that the goal of the program was to compute $y_p$ for output, not for computing $y(x)$ at points of $x$.

\subsection{Further proofs}
The following rules of differentiation still require proofs for their finiteness from their arguments:
\begin{enumerate}
    \item Chain rule
    \item Division (probably most difficult)
    \item exponentiation
\end{enumerate}
\subsection{Improvements of the program}
The following improvements can still be made to the program:
\begin{enumerate}
    \item Add a CLI.
    \item Implement a better symbolic equation solver.
\end{enumerate}
\pagebreak\nocite{*} % nocite is used since tex is removing the \cite commands for some reason.
\bibliography{paper}
\bibliographystyle{ieeetr}
\end{document}


% Created 2021-09-20 Mon 08:08
% Intended LaTeX compiler: pdflatex
\documentclass[presentation,aspectratio=169]{beamer}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{grffile}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{textcomp}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\usepackage{amssymb}
\usepackage{circuitikz}
\usepackage{khpreamble}
\usepgfplotslibrary{groupplots}
\usetheme{default}
\author{Kjartan Halvorsen}
\date{\today}
\title{The DC motor}
\hypersetup{
 pdfauthor={Kjartan Halvorsen},
 pdftitle={The DC motor},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 26.3 (Org mode 9.4.6)},
 pdflang={English}}
\begin{document}

\maketitle

\section{Intro}
\label{sec:org1ac59b1}

\section{The DC motor}
\label{sec:org49f77d8}
\begin{frame}[label={sec:orgf50b351}]{Force acting on an electric conductor in a magnetic field}
\begin{center}
\includegraphics[width=0.4\linewidth]{../../figures/HD-fig1_14.png}
\includegraphics[width=0.53\linewidth]{../../figures/HD-fig1_15.png}
\footnotesize Source: Hughes and Drury ''Electric motors and drives''
\end{center}

\[F=k_mI=(Bl_m)I,\]
\end{frame}

\begin{frame}[label={sec:org479f6d4}]{Rotating motor}
\begin{center}
\includegraphics[width=0.4\linewidth]{../../figures/HD-fig3_1.png}
\includegraphics[width=0.53\linewidth]{../../figures/HD-fig3_2.png}
{\footnotesize Source: Hughes and Drury ''Electric motors and drives''}
\end{center}
\end{frame}

\begin{frame}[label={sec:org255a431}]{Magnetic force and electro-motive force}
\begin{block}{The magnetic force on a current-carrying conductor}
\[ F(t) = k_f i(t) \quad\Leftrightarrow\quad T(t) = k_f r i(t) = k_m i(t)\]
\end{block}

\begin{block}{Voltage generated in a conductor moving in a magnetic field}
\[ e(t) = k_v v(t) \quad\Leftrightarrow\quad e(t) = k_v r \omega(t) = k_e \omega(t)\]
\(e(t)\) is called  \emph{Back electro-motive force (Back e.m.f.)}.

\alert{In the SI-system units} \(k_m = k_e = k\).
\end{block}
\end{frame}

\begin{frame}[label={sec:orgf3a2f0f}]{Equivalent circuit}
Consider a DC motor with separate excitation
\begin{center}
  \begin{circuitikz}
    \draw (4,1) node[elmech](motor){M};
    \draw (motor.north) to[R=$R$] (4,4) to[L=$L$] (0,4)
    to[american voltage source, label=$u$] (0,0) -| (motor.south);
    \draw[thick,->>](motor.right)--++(1,0)node[midway,above]{$\omega$};

    \node[] at (2, -0.8 cm) {\(L \frac{d}{dt}i(t) +  Ri(t) + k\omega(t) = u\)};

    \node[] at (2, 4.5 cm) {Armature};

    \begin{scope}[xshift=8cm]
    \draw (0,1) to (4,1) to[R=$R_f$] (4,3) to[L=$L_f$] (0,3)
    to[american voltage source, label=$V_f$] (0,1);
    \node[] at (2, 4.5 cm) {Field};
    \end{scope}
  \end{circuitikz}
\end{center}

\begin{center}
Newton: \(J\frac{d}{dt}\omega(t) = ki(t) - T_l(t)\)
\end{center}
\end{frame}

\begin{frame}[label={sec:org25f647d}]{Modeling the DC motor}
\begin{center}
  \begin{circuitikz}[yscale = 0.5]
    \draw (4,2) node[elmech](motor){M};
    \draw (motor.north) to[short] (4,4) to[R=$R$] (2,4) to[L=$L$] (0,4)
    to[american voltage source, label=$u$] (0,0) -| (motor.south);
    \draw[thick,->>](motor.right)--++(1,0)node[midway,above]{$\omega$};

    \node[] at (9, 4 cm) {\(L \frac{d}{dt}i(t) +  Ri(t) + k\omega(t) = u\)};
    \node[] at (9, 2 cm) {\(\frac{d}{dt}i(t) = \frac{1}{L} \Big(-Ri(t) - k\omega(t) + u\Big)\)};
    \end{circuitikz}
    \end{center}
    \begin{center}
    \begin{circuitikz}[yscale = 1]
  \begin{scope}[xshift=8cm, yshift=-1cm,
  block/.style={rectangle, draw, minimum width=12mm, minimum height=10mm},
  amp/.style = {regular polygon, regular polygon sides=3,
        draw, fill=white, text width=1em,
        inner sep=1pt, outer sep=0mm,
        shape border rotate=-90},
	summ/.style = {circle, draw, inner sep = 1pt},]
   \node[block,] (int) at (0,0) {$\int$};
   \node[amp, left of=int, node distance=30mm] (oneoverL) {$\frac{1}{L}$};
   \draw[->] (oneoverL) -- node[above] {$\frac{d}{dt}i(t)$} (int);
   \node[summ, left of=oneoverL, node distance=20mm] (sum) {\small $\Sigma$};
   \node[coordinate, left of=sum, node distance=35mm] (Vin) {};
   \draw[->] (Vin) -- node[above, very near start] {$u$} node[coordinate, pos=0.6] (mp) {} (sum);
   \node[amp, above of=mp, node distance=15mm] (mkonst) {$-k$};
   \draw[->] (int) -- node[coordinate] (fb) {} node[above, near end] {$i(t)$} ++(25mm, 0);
   \draw[->] (mkonst) ++(-2cm, 0) -- node[above, near start] {$\omega(t)$} (mkonst);
   \draw[->] (mkonst) -| (sum);
   \draw[->] (sum) -- (oneoverL);
   \node[amp, below of =int, node distance=16mm, rotate=180] (RR) {\rotatebox{-180}{$-R$}};
   \draw[->] (fb) |- (RR) -| (sum);

   \end{scope}
  \end{circuitikz}
  \end{center}
\end{frame}

\begin{frame}[label={sec:org82727b0}]{Transmission}
\end{frame}

\begin{frame}[label={sec:org96d9455}]{Transmission}
  \begin{center}
  \begin{circuitikz}[xscale = 0.8]
\begin{scope}[xshift=8cm, yshift=-1cm,
block/.style={rectangle, draw, minimum width=12mm, minimum height=10mm},
amp/.style = {regular polygon, regular polygon sides=3,
      draw, fill=white, text width=1em,
      inner sep=1pt, outer sep=0mm,
      shape border rotate=-90},
      summ/.style = {circle, draw, inner sep = 1pt},]
 \node[block,] (int) at (0,0) {$\int$};
 \node[amp, left of=int, node distance=25mm] (oneoverL) {$\frac{1}{L}$};
 \draw[->] (oneoverL) -- node[above] {$\frac{d}{dt}i$} (int);
 \node[summ, left of=oneoverL, node distance=15mm] (sum) {\small $\Sigma$};
 \node[coordinate, left of=sum, node distance=45mm] (Vin) {};
 \draw[->] (Vin) -- node[above, very near start] {$u$} node[coordinate, pos=0.8] (mp) {} (sum);
 \node[amp, above of=mp, node distance=15mm] (mkonst) {$-k$};
 \node[amp, left of=mkonst, node distance=20mm] (gr) {$g_r$};
 \node[amp, right of=int, node distance=25mm] (mk2) {$k$};
 \node[amp, right of=mk2, node distance=20mm] (gr2) {$g_r$};
 \draw[->] (int) -- node[coordinate, pos=0.5] (meas) {} node[above] {$i$} (mk2);
 \draw[->] (mk2) -- node[above] {$T_m$} (gr2);
 \draw[->] (gr2) -- node[above] {$T_e$} ++(15mm, 0);
 \draw[->] (gr) ++(-2cm, 0) -- node[above, near start] {$\omega_a$} (gr);
 \draw[->] (gr) -- node[above, ] {$\omega_m$} (mkonst);
 \draw[->] (mkonst) -| (sum);
 \draw[->] (sum) -- (oneoverL);
 \node[amp, below of =int, node distance=16mm, rotate=180] (fb) {\rotatebox{-180}{$-R$}};
 \draw[->] (meas) |- (fb);
 \draw[->] (fb) -| (sum);
 \end{scope}
\end{circuitikz}
\end{center}

Ignoring losses in the transmission:
\[\underbrace{T_m\omega_m}_{\text{Power in}} = \underbrace{T_e\omega_a}_{\text{Power out}}\]
\end{frame}

\begin{frame}[label={sec:org7308dfa}]{DC motor driving a load}
  \begin{center}
  \begin{tikzpicture}[scale = 0.5,
block/.style={rectangle, draw, minimum width=12mm, minimum height=10mm},
amp/.style = {regular polygon, regular polygon sides=3,
      draw, fill=white, text width=1em,
      inner sep=1pt, outer sep=0mm,
      shape border rotate=-90},
      summ/.style = {circle, draw, inner sep = 1pt},]
 \node[block,] (dc) at (0,0) {$\frac{1/R}{s\frac{L}{R} + 1}$};
 \node[summ, left of=dc, node distance=20mm] (sum) {\small $\Sigma$};
 \node[coordinate, left of=sum, node distance=20mm] (Vin) {};
 \draw[->] (Vin) -- node[above, very near start] {$u$} node[coordinate, pos=0.8] (mp) {} (sum);
 \node[block, below of=sum, node distance=20mm] (mkonst) {$g_rk$};
 \node[block, right of=dc, node distance=25mm] (mk2) {$g_rk$};
 \node[summ, right of=mk2, node distance=20mm] (sumtorque) {\small $\Sigma$};
 \node[block, right of=sumtorque, node distance=20mm] (load) {$\frac{1}{Js}$};
 %\node[block, right of=load, node distance=25mm] (int) {$\frac{1}{s}$};
 \node[coordinate, right of = load, node distance=25mm] (output) {};
 \node[coordinate, above of = sumtorque, node distance=20mm] (tload) {};
 \draw[->] (sum) -- (dc);
 \draw[->] (dc) -- node[coordinate, pos=0.5] (meas) {} node[above] {$i$} (mk2);
 \draw[->] (mk2) -- node[above] {$T_a$} (sumtorque);
 \draw[->] (sumtorque) -- node[above] {} (load);
 \draw[->] (load)  -- node[coordinate,] (meas) node[above,] {$\omega_a$} (output);
 \draw[->] (mkonst) -- node[pos=0.9, left,] {$-$} (sum);
 %\draw[->] (int) --  node[very near end, above] {$\theta_a$} (output);
 \draw[->] (meas) |- (mkonst);
 \draw[->] (tload) -- node[right, very near start] {$T_l$} node [right, pos=0.9] {$-$} (sumtorque);
\end{tikzpicture}
\end{center}
\end{frame}

\begin{frame}[label={sec:org231c594}]{DC motor driving a load}
Assuming the inductance to be negligable.

  \begin{center}
  \begin{tikzpicture}[scale = 0.5,
block/.style={rectangle, draw, minimum width=12mm, minimum height=10mm},
amp/.style = {regular polygon, regular polygon sides=3,
      draw, fill=white, text width=1em,
      inner sep=1pt, outer sep=0mm,
      shape border rotate=-90},
      summ/.style = {circle, draw, inner sep = 1pt},]
 \node[block,] (dc) at (0,0) {$\frac{1}{R}$};
 \node[summ, left of=dc, node distance=20mm] (sum) {\small $\Sigma$};
 \node[coordinate, left of=sum, node distance=20mm] (Vin) {};
 \draw[->] (Vin) -- node[above, very near start] {$u$} node[coordinate, pos=0.8] (mp) {} (sum);
 \node[block, below of=sum, node distance=20mm] (mkonst) {$g_rk$};
 \node[block, right of=dc, node distance=25mm] (mk2) {$g_rk$};
 \node[summ, right of=mk2, node distance=20mm] (sumtorque) {\small $\Sigma$};
 \node[block, right of=sumtorque, node distance=20mm] (load) {$\frac{1}{Js}$};
 %\node[block, right of=load, node distance=25mm] (int) {$\frac{1}{s}$};
 \node[coordinate, right of = load, node distance=25mm] (output) {};
 \node[coordinate, above of = sumtorque, node distance=20mm] (tload) {};
 \draw[->] (sum) -- (dc);
 \draw[->] (dc) -- node[coordinate, pos=0.5] (meas) {} node[above] {$i$} (mk2);
 \draw[->] (mk2) -- node[above] {$T_a$} (sumtorque);
 \draw[->] (sumtorque) -- node[above] {} (load);
 \draw[->] (load)  -- node[coordinate,] (meas) node[above,] {$\omega_a$} (output);
 \draw[->] (mkonst) -- node[pos=0.9, left,] {$-$} (sum);
 %\draw[->] (int) --  node[very near end, above] {$\theta_a$} (output);
 \draw[->] (meas) |- (mkonst);
 \draw[->] (tload) -- node[right, very near start] {$T_l$} node [right, pos=0.9] {$-$} (sumtorque);
\end{tikzpicture}
\end{center}

\alert{Activity} What is the transfer function from the voltage input \(u(t)\) to the angular velocity \(\omega_a(t)\)?
\end{frame}

\begin{frame}[label={sec:orgacbd599}]{DC motor driving a load}
  \begin{center}
  \begin{tikzpicture}[scale = 0.5,
block/.style={rectangle, draw, minimum width=12mm, minimum height=10mm},
amp/.style = {regular polygon, regular polygon sides=3,
      draw, fill=white, text width=1em,
      inner sep=1pt, outer sep=0mm,
      shape border rotate=-90},
      summ/.style = {circle, draw, inner sep = 1pt},]
 \node[block,] (dc) at (0,0) {$\frac{k_1}{s\tau + 1}$};
 \node[summ, right of=dc, node distance=20mm] (sum) {\small $\Sigma$};
 \node[block, above of=sum, node distance=20mm] (loadtrf)  {$-\frac{k_2}{s\tau + 1}$};
 \node[coordinate, left of=dc, node distance=20mm] (Vin) {};
 \node[coordinate, above of=loadtrf, node distance=20mm] (tload) {};
 \node[coordinate, right of=sum, node distance=30mm] (output) {};

 \draw[->] (Vin) -- node[above, very near start] {$u$} node[coordinate, pos=0.8] (mp) {} (dc);
 \draw[->] (tload) -- node[right, very near start] {$T_l$} node [right, pos=0.9] {} (loadtrf);
 \draw[->] (dc) to (sum);
 \draw[->] (loadtrf) to (sum);
 \draw[->] (sum) -- node[above, near end,] {$\omega_a$} (output);

\end{tikzpicture}
\end{center}

\[ \tau=\frac{JR}{(g_rk)^2}, \quad k_1= \frac{1}{g_r k}, \quad k_2 = \textcolor{white}{\frac{R}{(g_r k)^2}} \]
\end{frame}
\end{document}


%================================================
\section{NIST Digits Dataset}
%================================================
	%---------------------------------------------
	\subsection{Problem and Approach}
	%----------------------------------------------
	\begin{frame}
		\frametitle{NIST Data Classification}

		\begin{itemize}
			\item Small dataset (1797 points with 64 features, 10 labels, balanced).
			\item 64 features: $8\times 8$ images with pixels in the range $[0,16]$.
			\item 10 classes: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9
			\item Simple and computationally able to run locally, which makes it ideal for proof-of-concept on short timelines where image analysis is needed quickly.
			\item Analsis Tools Considered:
				\begin{itemize}
					\item PCA Clustering (qualitative)
					\item K Nearest Neighbor Classifier Accuracy (quantitative)
					\item K Nearest Neighbor Confusion Matrix (quantitative)
					\item Persistence Diagrams (qualitative)
					\item Bottleneck Distance Matrix (quantitative)
				\end{itemize}
		\end{itemize}

		\end{frame}

	%----------------------------------------------
	\subsection{Results}
	%----------------------------------------------

		\begin{frame}
		\frametitle{PCA Clustering}

		\begin{figure}
				\centering
				\includegraphics[scale=0.5]{images/digits_PCA.png}
				\caption{PCA Clustering of the NIST Digits Dataset. Visually, we can see that there is some clustering, but the data has a pretty high degree of similarity even after PCA.}
		\end{figure}

		\end{frame}

		\begin{frame}
			\frametitle{K Nearest Neighbors Classifier}

			\begin{itemize}
				\item Pre-processing:
				\begin{itemize}
					\item We split the data into a 30\% train and 70\% test sets and keep the data balanced.
					\item We also shuffle the data.
					\item We scale the data such that it is within $(0,1)$ using a min-max scaler.
				\end{itemize}
				\item The function class we consider is trivial since it is dependent only on the training data given.
				\item We can view this as a weighting function that weights the $k$-nearest points to the input with $\frac{1}{k}$ and all other points $0$.
				\item Classification Accuracy: 76 percent
				\item Training Accuracy: 83 percent
			\end{itemize}
		\end{frame}

		\begin{frame}
			\frametitle{Confusion Matrix for Digits Data}

			\begin{figure}
				\centering
				\includegraphics[scale=0.5]{images/confusion_matrix_digits.png}
				\caption{Confusion Matrix of Digit Data. We see many interesting artifacts: 0's, 4's, and 6's are classified nicely, 1's have some confusion with 7's (vice versa), 2's with 5's, 3's with 9's, 5's with 8's, 8's with 2's and 5's, and 9's with 1's and 3's.}
		\end{figure}
		\end{frame}

		\begin{frame}
		\frametitle{Persistence Images of Digits: 0's}

		\begin{figure}
				\centering
				\includegraphics[scale=0.5]{images/PI_H0_0.png}
				\caption{Persistence Image for 0.}
		\end{figure}
		\end{frame}

		\begin{frame}
		\frametitle{Persistence Images of Digits: 0's}

		\begin{figure}
				\centering
				\includegraphics[scale=0.5]{images/PI_H0_5.png}
				\caption{Persistence Image for 5.}
		\end{figure}
		\end{frame}

				\begin{frame}
		\frametitle{Persistence Images of Digits: 8's}

		\begin{figure}
				\centering
				\includegraphics[scale=0.4]{images/PI_H0_8.png}
				\caption{Persistence Image for 8.}
		\end{figure}
		\end{frame}

				\begin{frame}
		\frametitle{Persistence Diagram of Digits: 0's, 5's, \& 8's}

		\begin{figure}
				\centering
				\includegraphics[scale=0.35]{images/pd058.png}
				\caption{Persistence Image for 0, 5, 8. Notice the birth/death are similar for 5 and 8 versus 0.}
		\end{figure}
		\end{frame}

		\begin{frame}
		\frametitle{Pairwise Bottleneck Distances}

		\begin{itemize}
			\item The bottleneck distances:
				\begin{itemize}
					\item With themselves: 0 (control)
					\item 0, 5: 2.56941795
					\item 0, 8: 2.21272278
					\item 5, 8: 2.11571217
				\end{itemize}
		\end{itemize}

		A downside to the Bottleneck Distance as a measure is that quantitatively it can be hard to distinguish for simple data what a good distance is.
		\end{frame}


\section{Instance Generation}

The instances are generated randomly \cite{bib:instances-CVRP}, \cite{bib:constrained-knapsack}, \cite{bib:grasp-and-tabu}. For that, first the graph is generated, and then the weight of each vertex is chosen. The knapsack capacity is selected so that, on average, X percent of the vertices fit in it. The following subsections analyze each of those aspects.

Consider the parameters:
\begin{enumerate}
    \item $n$: number of vertices;
    \item $K$: average number of branches;
    \item $L$: maximum number of leaf vertices;
    \item $H$: the maximum value of an entry of the weight of each vertex;
    \item $m$: fraction of the average number of elements that fit in the knapsack;
\end{enumerate}

\subsection{How to Generate the Precedences}

The process of generating the precedences is specified in \algref{algorith:find-trees}, which uses \algref{algorith:generate-precedences}. The \figref{fig:precedence-generation} has an example of such procedure. The following parameters are used to control the generation:

\begin{algorithm}[ht!]
    \caption{Find-Trees}
    \label{algorith:find-trees}
    \begin{algorithmic}[1]
        \Require{
            $\vertices$: vertices in the 2D plane,
            $K$: average number of branches,
            $L$: maximum number of leaf vertices
        }
        \State{$k \gets $ random number from 1 to $K$}
        \State{$\tuple{R, \mathcal{V}} \gets $ find $k$ clusters in $V$}
            \Comment{$R$: a set of centers}\\
            \Comment{$\mathcal{V}$: a set with each element being the set vertices of each cluster}
        \State{$\mathcal{T} \gets \emptyset$}
        \For{each pair $r \in R $ and $V' \in \mathcal{V}$}
            \If{$\abs{V'} \leqslant L$}
                \State{$T \gets $ tree with $r$ as the root node and $V'$ as the leaves}
            \Else
                \State{$T \gets $ tree with $r$ as the root node of the subtree Find-Trees($V', K, L$)}
            \EndIf
            \State{$\mathcal{T} \gets \mathcal{T} \cup \Set{T}$}
        \EndFor
        \\\Return{$\mathcal{T}$}
    \end{algorithmic}
\end{algorithm}

\begin{algorithm}[ht]
    \caption{Generate-Precedences}
    \label{algorith:generate-precedences}
    \begin{algorithmic}[1]
        \Require{
            $n$: number of vertices,
            $K$: average number of branches,
            $L$: maximum number of leaf vertices
        }
        \State{$V \gets $ generate $n$ points in the 2D plane randomly}
        \State{$\mathcal{T} \gets $ Find-Trees($V, K, L$)}
        \\\Return{$\mathcal{T}$}
    \end{algorithmic}
\end{algorithm}

\begin{figure}[ht]
    \centering
    \includegraphics[width=0.5\textwidth]{images/precedence_construction.jpg}
    \caption{Precedence generation. The root nodes are the green, red and lemon. Red has four leaf vertices. Green has two branches, the pink with one leaf and the purple with two leaves. Lemon has one leaf and one branch with two leaves.}
    \label{fig:precedence-generation}
\end{figure}

\subsection{How to Generate the Weights}

Generate the weights randomly in the interval $\interval{0}{H}$.

\subsection{How to Generate the Knapsack Capacity}

Generate each entry of the knapsack capacity $\maximumWeight$ randomly in the interval $\interval{0}{m \cdot n \cdot H}$.