<filename>research-drafts/LispMeetup-Jun11-2020.tex
\documentclass{beamer}

\usepackage{beamerthemeshadow}
\usepackage{color}
\usepackage{verbatim}
\usepackage{tikz}
\usepackage[normalem]{ulem}
\usepackage[timeinterval=10]{tdclock}
%\usepackage{wrapfig}

\mode<presentation>
{
 \usetheme{Warsaw} %%% Change later
\usecolortheme{dove}

%gets rid of bottom navigation bars
%\setbeamertemplate{footline}[page number]{}

% a version with clock in the footline
\setbeamertemplate{footline}
{%
\begin{beamercolorbox}[wd=1.0\paperwidth,ht=2.25ex,dp=1ex,right]{date in head/foot}%
           \usebeamerfont{date in head/foot} \msdark{\insertshortdate{\tdtime}\hspace*{2em}
           \insertframenumber{} / \inserttotalframenumber\hspace*{2ex}} 
        \end{beamercolorbox}%
}

  \setbeamercovered{transparent}
  % or whatever (possibly just delete it)
}


\usepackage{graphicx}

\usepackage{amssymb,soul}

\usepackage{hyperref}

\definecolor{myblue}{rgb}{0, 0, 0.9}

\definecolor{mygreen}{rgb}{0,0.6,0}

\definecolor{mymagenta}{rgb}{0.9, 0, 0.9}

\definecolor{myred}{rgb}{0.9,0,0}

\definecolor{mygray}{rgb}{0.8,0.8,0.8}

\definecolor{mydark}{rgb}{0.4, 0.4, 0.4}

\definecolor{myblack}{rgb}{0,0,0}

\newcommand{\msblue}[1]{{\color{myblue} #1}}

\newcommand{\msmagenta}[1]{{\color{mymagenta} #1}}

\newcommand{\msred}[1]{{\color{myred} #1}}

\newcommand{\msgreen}[1]{{\color{mygreen} #1}}

\newcommand{\msgray}[1]{{\color{mygray} #1}}

\newcommand{\msdark}[1]{{\color{mydark} #1}}

\newcommand{\msblack}[1]{{\color{myblack} #1}}

\newcommand{\relu}{\mbox{\bf ReLU}}

\begin{document}

\title{Dataflow Matrix Machines and V-values}
\author{\bf Mishka (<NAME>)}
\date[]  
{Boston Lisp meetup: June 11, 2020\\[2ex]
Work done in the last few  years
in research collaborations with <NAME>, <NAME>, <NAME>.\\[2ex]
I am looking for new collaborators.
}

\begin{frame}
  \initclock
  \titlepage
\end{frame}

\section{Dataflow matrix machines (DMMs)}

\begin{frame}

  \frametitle{\msmagenta{Dataflow matrix machines (DMMs)}}

DMMs: a novel class of {\bf neural machines}.\\[2ex]

They use arbitrary {\bf linear streams} instead of streams of numbers.\\[2ex]


\begin{itemize}

\item {\bf Self-modification:} can fluently modify their own weights, connectivity patterns, and size;\\[2ex]

\item Highly expressive linear streams of V-values\\ (vector-like values based on nested dictionaries);\\[2ex]

\item Expressive enough to serve as a programming platform: functional reactive programming with\\ {\bf continuously deformable programs}.

\end{itemize}

\end{frame}




\begin{frame}

\frametitle{\msmagenta{Linear streams}}

The key feature of {\bf DMMs} compared to {\bf RNNs}:
         they use\\ {\bf linear streams} instead of streams of numbers.\\[2ex]

The following streams all support the pattern of alternating\\ linear and non-linear computations:\\[2ex]

\begin{itemize}

\item Streams of numbers\\[2ex]

\item Streams of vectors from fixed vector space $V$\\[2ex]

\item Linear streams: such streams that the notion of\\ {\bf linear combination of several streams} is defined.\\[3ex]

\end{itemize}






\end{frame}




\begin{frame}

\frametitle{Kinds of linear streams}

 Some examples:\\[1ex]


\begin{itemize}


\item Every vector space $V$ gives rise to the corresponding kind of linear streams
(streams of vectors from that space)\\[2ex]

\item Every measurable space $X$ gives rise to the space of\\ {\bf streams of
probabilistic samples} drawn from $X$\\ and decorated with +/- signs\\
(linear combination is defined by a stochastic procedure)\\[2ex]

\item Streams of images of a particular size (that is, animations)\\[2ex]

\item Streams of matrices; streams of multidimensional arrays\\[2ex]

\item \msmagenta{Streams of V-values based on nested maps (instead of S-expressions)}\\[2ex]



\end{itemize}

\end{frame}












\section{V-values and variadic neurons}







\subsection{Vectors based on nested maps}


\begin{frame}

  \frametitle{\msmagenta{V-values: vector space based on nested dictionaries}}


V-values play the role of Lisp S-expressions in this formalism.\\[3ex]


We want a vector space.\\[3ex]

Take prefix trees with numerical leaves\\ implemented as nested dictionaries.\\[3ex]

We call them {\bf V-values}
(``vector-like values").\\[3ex]


\end{frame}







\begin{frame}

  \frametitle{\msmagenta{Example of a V-value: different ways to view it}}

\begin{columns}[T]
\begin{column}{0.5\textwidth}

\begin{tikzpicture}
  \clip (-3.0, -4.5) rectangle (3.0, 0.0);

  \draw[->](0, 0) -- (-1.8, -1)  node [below] {3.5};

  \draw[->](0, 0) -- (0, -1) node [midway,  below right] {:foo};

    \draw[->](0, -1) -- (0.5, -2) node[midway, below right] {\ :bar};

      \draw[->](0.5, -2) -- (0.5, -3) node [below] {7};

    \draw[->](0, -1) -- (-0.5, -2) node [below] {2};

  \draw[->](0, 0) -- (1.8, -1) node [midway, below  right] {\ \ \ \ \ \ :baz};

      \draw[->](1.8, -1) -- (1.8, -2) node [midway, below  right] {:foo};

      \draw[->](1.8, -2) -- (1.8, -3) node [midway, below  right] {:bar};      

      \draw[->](1.8, -3) -- (1.8, -4) node [below] {-4};
 

  \filldraw (1.8, -1) circle [radius=0.7pt]
                (1.8, -2) circle [radius=0.7pt]
                (1.8, -3) circle [radius=0.7pt]
                (0, -1) circle [radius=0.7pt]
                (0.5, -2) circle [radius=0.7pt];
  
 

\end{tikzpicture}
\end{column}
\begin{column}{0.5\textwidth}

\begin{itemize}

{\footnotesize

 \item 3.5 $\cdot$ ($\epsilon$) + 2 $\cdot$ (:foo) + \\ 7 $\cdot$ (:foo :bar) - 4 $\cdot$ (:baz :foo :bar)

 \item ($\leadsto$ 3.5) + (:foo $\leadsto$  2) +\\ (:foo $\leadsto$ :bar $\leadsto$ 7) +\\
          (:baz $\leadsto$ :foo $\leadsto$ :bar $\leadsto$ -4)


\msmagenta{
 \item {\tiny  scalar 3.5 + sparse 1D array \{{\tt d1[:foo]= 2}\} +\\[1.5ex] sparse 2D
matrix \{{\tt d2[:foo, :bar]= 7}\} +\\  sparse 3D array \{{\tt d3[:baz, :foo, :bar]= -4}\}}
}

 \item {\tt \{:number 3.5,\\ :foo \{:number 2, :bar 7\}, :baz \{:foo \{:bar -4\}\}\}}
          ({\tt :number} $\not\in L$)

}

\end{itemize}

\end{column}
\end{columns}

\end{frame}






\subsection{Variable arity}


\begin{frame}

  \frametitle{Dataflow matrix machines (our current implementation) based on
 streams of  V-values
 and
 variadic neurons}

$x_{f, n_f, i}^{t+1} = \sum_{g \in F} \sum_{n_g \in L} \sum_{o \in L} \msblue{w_{f, n_f, i;\, g, n_g, o}^t} * y_{g, n_g, o}^t$ {\small \msblue{(down movement)}}\\[2ex]

$y_{f, n_f}^{t+1} = f(x_{f, n_f}^{t+1})$ {\small \msgreen{(up movement)}}

\begin{tikzpicture}
  %\draw (-3.5, -2.0) rectangle (7.0, 2.0);
  \clip (-3.5, -2.0) rectangle (7.0, 2.0);

  \filldraw (-3.2,0) circle [radius=0.5pt]
                (-3.0,0)  circle [radius=0.5pt]
                (-3.4, 0) circle [radius=0.5pt]; 

\msgreen{\draw [->] (-2.6, -1.5) node[right] {\msmagenta{$x_{f, n_f}$}} -- (-2.6, 1.5) node [midway, above right] {$f$} node[right] {\msmagenta{$y_{f, n_f}$}};}

  \filldraw (0,0) circle [radius=0.5pt]
                (-0.2,0)  circle [radius=0.5pt]
                (0.2, 0) circle [radius=0.5pt];

\msgreen{\draw [->] (0.6, -1.5) node[right] {\msmagenta{$x_{g, n_g}$}} -- (0.6, 1.5) node [midway, above right] {$g$} node[right] {\msmagenta{$y_{g, n_g}$}};}

  \filldraw (3.2,0) circle [radius=0.5pt]
                (3.0,0)  circle [radius=0.5pt]
                (3.4, 0) circle [radius=0.5pt]; 


 \msblue{ \draw [->, very thick] (1.1, 1.1) .. controls (5.5, 4.3) and (5.5, -4.0) .. (1.1, -0.8)  node [midway, right] {{\bf W}};}

  \foreach \y in {-1.0, 1.0}
    {

      \draw [densely dotted] (0.45, \y+0.15) ellipse [x radius=100pt, y radius=4pt];

     \foreach \x in {-1.0, 2.2}
       {

        \foreach \d in {-0.4, -0.15, 0.1}
           {
               \msmagenta{\draw(\x-0.15, \y + 0.45) -- (\x+\d, \y+0.15); 
               \draw (\x+\d, \y + 0.15) -- (\x+\d-0.08, \y-0.15);
               \draw (\x+\d, \y + 0.15) -- (\x+\d+0.08, \y-0.15);
               \filldraw (\x+0.5*\d-0.05, \y-0.25) circle [radius=0.2pt];
               \filldraw (\x+0.5*\d+0.5, \y+0.15) circle [radius=0.2pt];}
           }
       } 
     }
 
\end{tikzpicture}

\end{frame}

\section{Programming patterns and self-referential facilities}




\begin{frame}

  \frametitle{\msmagenta{DMMs: programming with powerful neurons}}

Powerful variadic neurons and streams of V-values $\Rightarrow$ a much more expressive formalism
 than networks based on streams of numbers.\\[2ex]

Many tasks can be accomplished by \msmagenta{compact DMM networks},\\
 where {\bf single neurons function as layers or modules}.

\hrulefill

\begin{itemize}

\item Accumulators and memory

\item Multiplicative constructions and ``fuzzy if"

\item Sparse vectors

\item Data structures

\item Self-referential facilities

\end{itemize}






\end{frame}

\begin{frame}

  \frametitle{\msmagenta{Accumulators and memory}}

\begin{tikzpicture}
   \clip (-2.0, -2.0) rectangle (7.0, 2.0);




  \filldraw (0,0) circle [radius=0.5pt]
                (-0.2,0)  circle [radius=0.5pt]
                (0.2, 0) circle [radius=0.5pt];

\msgreen{\draw [->] (0.6, -1.5) node[right] {\msblack{$x_{{\tt plus}, {\tt :my\mbox{\footnotesize -}neuron}}$}} -- 
                    (0.6, 1.5) node [midway, above right] {\tt plus} node[right]
                    {\msblack{$y_{{\tt plus}, {\tt :my\mbox{\footnotesize -}neuron}}$}};}

  \filldraw (3.2,0) circle [radius=0.5pt]
                (3.0,0)  circle [radius=0.5pt]
                (3.4, 0) circle [radius=0.5pt]; 


 \msblue{\draw [->] (1.9, 1.1) .. controls (5.5, 4.3) and (5.5, -4.0) .. (2.15, -0.85)  node [midway, left] {{1.0}};

  \draw [->] (1.7, -0.2) -- (1.5, -0.9);
  \filldraw (1.5, -0.25) circle [radius=0.2pt];
  \filldraw (1.6, -0.25) circle [radius=0.2pt];
  \filldraw (1.4, -0.25) circle [radius=0.2pt];
  \draw [->] (1.3, -0.2) node [left] {{\tiny updates from other neurons}} -- (1.5, -0.8);}


  \foreach \y in {-1.0}
    {

      \draw [densely dotted] (1.0, \y+0.15) ellipse [x radius=80pt, y radius=5pt];

     \foreach \x in {2.0}
       {

        \foreach \d in {-0.4}
           {
               \draw(\x-0.15, \y + 0.45) -- (\x+\d, \y+0.15) node [left] {\scriptsize{\tt :delta}}; 
               \draw (\x+\d, \y + 0.15) -- (\x+\d-0.08, \y-0.15);
               \draw (\x+\d, \y + 0.15) -- (\x+\d+0.08, \y-0.15);
               \filldraw (\x+0.5*\d-0.05, \y-0.25) circle [radius=0.2pt];
               %\filldraw (\x+0.5*\d+0.5, \y+0.15) circle [radius=0.2pt];
           }
       \foreach \d in {0.1}
           {
               \draw(\x-0.15, \y + 0.45) -- (\x+\d, \y+0.15) node [right] {\scriptsize{\tt :accum}}; 
               \draw (\x+\d, \y + 0.15) -- (\x+\d-0.08, \y-0.15);
               \draw (\x+\d, \y + 0.15) -- (\x+\d+0.08, \y-0.15);
               \filldraw (\x+0.5*\d-0.05, \y-0.25) circle [radius=0.2pt];
               %\filldraw (\x+0.5*\d+0.5, \y+0.15) circle [radius=0.2pt];
           }
        \foreach \d in {-0.4, -0.15, 0.1}
           {
              \filldraw (\x+0.5*\d-0.05, \y-0.25) circle [radius=0.2pt];
           }
       } 
     }

  \foreach \y in {0.9}
    {

      \draw [densely dotted] (1.0, \y+0.15) ellipse [x radius=80pt, y radius=5pt];

     \foreach \x in {2.0}
       {

        \foreach \d in {-0.15}
           {
               \draw(\x-0.15, \y + 0.45) -- (\x+\d, \y+0.15) node [right] {\scriptsize{\tt :single}}; 
               \draw (\x+\d, \y + 0.15) -- (\x+\d-0.08, \y-0.15);
               \draw (\x+\d, \y + 0.15) -- (\x+\d+0.08, \y-0.15);
 
               %\filldraw (\x+0.5*\d+0.5, \y+0.15) circle [radius=0.2pt];
           }
        \foreach \d in {-0.4, -0.15, 0.1}
           {
              \filldraw (\x+0.5*\d-0.05, \y-0.25) circle [radius=0.2pt];
           }
       } 
     }
 
\end{tikzpicture}

In this implementation, activation function \msgreen{\footnotesize \tt plus} adds V-values from 
{\footnotesize\tt :accum} and {\footnotesize\tt :delta}
together and places the result into {\footnotesize\tt :single}.

\end{frame}


\subsection{Sparse vectors}



\begin{frame}

  \frametitle{Sparse vectors of high or infinite dimension}

Example:  a neuron accumulating count of words in a given text.\\[2ex]

The dictionary mapping words to their respective counts is an infinite-dimensional vector
with a finite number of non-zero elements.\\[2ex]

\msmagenta{\Large $\bullet$} Don't need a neuron for each coordinate of our vector space.\\[2ex]

 \msmagenta{\Large $\bullet$} Don't have an obligation to reduce dimension by embedding.

\end{frame}





\subsection{Data structures}





\begin{frame}

  \frametitle{Streams of immutable \msmagenta{data structures}}

One can represent {\bf lists, matrices, graphs,} and so on\\ via nested dictionaries.\\[3ex]


It is natural to use streams of immutable V-values in the implementations of DMMs.\\[3ex]

The DMM architecture is friendly towards algorithms working with immutable data structures in
the spirit of functional programming.\\[3ex]

But more imperative styles can be accommodated as well.

\end{frame}





\subsection{Self-referential facilities}





\begin{frame}

  \frametitle{\msmagenta{Self-referential facilities (easy in DMMs)}}

It is easy to represent the network matrix {\bf W} as a V-value.\\[3ex]

Emit the stream
of network matrices from neuron {\tt Self}.\\[3ex]

Use the most recent V-value from that stream
as the network matrix {\bf W} during the
next ``down movement".\\[3ex]

This mechanism allows a DMM network to 
{\bf modify its own weights, topology, and size}
while it is running.

\end{frame}


\begin{frame}

  \frametitle{\msmagenta{Self-referential facilities (easy in DMMs)}}

Our current implementation: {\tt Self} connected as an accumulator.\\[2ex]
It accumulates the value of the network matrix and
accepts additive updates from other neurons in the network.\\[2ex]

\begin{tikzpicture}
   \clip (-2.0, -2.0) rectangle (7.0, 2.0);




  \filldraw (0,0) circle [radius=0.5pt]
                (-0.2,0)  circle [radius=0.5pt]
                (0.2, 0) circle [radius=0.5pt];

  \msgreen{\draw [->] (0.6, -1.5) node[right] {\msblack{$x_{{\tt plus}, {\bf :Self}}$}} -- 
                    (0.6, 1.5) node [midway, above right] {{\tt plus}} node[right] {\msblack{$y_{{\tt plus}, {\bf :Self}}$}};}

  \filldraw (3.2,0) circle [radius=0.5pt]
                (3.0,0)  circle [radius=0.5pt]
                (3.4, 0) circle [radius=0.5pt]; 


  \msblue{\draw [->] (1.9, 1.1) .. controls (5.5, 4.3) and (5.5, -4.0) .. (2.15, -0.85)  node [midway, left] {{1.0}};

  \draw [->] (1.7, -0.2) -- (1.5, -0.9);
  \filldraw (1.5, -0.25) circle [radius=0.2pt];
  \filldraw (1.6, -0.25) circle [radius=0.2pt];
  \filldraw (1.4, -0.25) circle [radius=0.2pt];
  \draw [->] (1.3, -0.2) node [left] {{\tiny updates from other neurons}} -- (1.5, -0.8);}


  \foreach \y in {-1.0}
    {

      \draw [densely dotted] (1.0, \y+0.15) ellipse [x radius=80pt, y radius=5pt];

     \foreach \x in {2.0}
       {

        \foreach \d in {-0.4}
           {
               \draw(\x-0.15, \y + 0.45) -- (\x+\d, \y+0.15) node [left] {\scriptsize{\tt :delta}}; 
               \draw (\x+\d, \y + 0.15) -- (\x+\d-0.08, \y-0.15);
               \draw (\x+\d, \y + 0.15) -- (\x+\d+0.08, \y-0.15);
               \filldraw (\x+0.5*\d-0.05, \y-0.25) circle [radius=0.2pt];
               %\filldraw (\x+0.5*\d+0.5, \y+0.15) circle [radius=0.2pt];
           }
       \foreach \d in {0.1}
           {
               \draw(\x-0.15, \y + 0.45) -- (\x+\d, \y+0.15) node [right] {\scriptsize{\tt :accum}}; 
               \draw (\x+\d, \y + 0.15) -- (\x+\d-0.08, \y-0.15);
               \draw (\x+\d, \y + 0.15) -- (\x+\d+0.08, \y-0.15);
               \filldraw (\x+0.5*\d-0.05, \y-0.25) circle [radius=0.2pt];
               %\filldraw (\x+0.5*\d+0.5, \y+0.15) circle [radius=0.2pt];
           }
        \foreach \d in {-0.4, -0.15, 0.1}
           {
              \filldraw (\x+0.5*\d-0.05, \y-0.25) circle [radius=0.2pt];
           }
       } 
     }

  \foreach \y in {0.9}
    {

      \draw [densely dotted] (1.0, \y+0.15) ellipse [x radius=80pt, y radius=5pt];

     \foreach \x in {2.0}
       {

        \foreach \d in {-0.15}
           {
               \draw(\x-0.15, \y + 0.45) -- (\x+\d, \y+0.15) node [right] {\scriptsize{\bf :NetworkMatrix}}; 
               \draw (\x+\d, \y + 0.15) -- (\x+\d-0.08, \y-0.15);
               \draw (\x+\d, \y + 0.15) -- (\x+\d+0.08, \y-0.15);
 
               %\filldraw (\x+0.5*\d+0.5, \y+0.15) circle [radius=0.2pt];
           }
        \foreach \d in {-0.4, -0.15, 0.1}
           {
              \filldraw (\x+0.5*\d-0.05, \y-0.25) circle [radius=0.2pt];
           }
       } 
     }
 
\end{tikzpicture}

The most recent V-value at the {\footnotesize\tt :NetworkMatrix} output of\\ $y_{{\tt plus}, {\bf :Self}}$ neuron is used as \msblue{\bf W}.


\end{frame}



\begin{frame}

  \frametitle{References}

Reference paper: \href{https://arxiv.org/abs/1712.07447}{\tt\footnotesize  https://arxiv.org/abs/1712.07447}\\[2ex]

Open source experimental engine (Clojure):

\href{https://github.com/jsa-aerial/DMM}{\tt\footnotesize https://github.com/jsa-aerial/DMM}\\[2ex]

GitHub pages: \href{https://anhinga.github.io}{\tt\footnotesize  https://anhinga.github.io}\\[2ex]

Mishka on GitHub: \href{https://github.com/anhinga}{\tt\footnotesize https://github.com/anhinga}\\[2ex]

e-mail: {\tt\footnotesize <EMAIL>}\\[2ex]

I am currently trying to implement DMMs\\
and machine learning with DMMs\\ in {\bf Julia} and {\bf Julia Flux}.


\end{frame}
\end{document}
