\input{../header_function}

%---------- start document ---------- %
 \section{module -- module/ideal with HNF}\linkedzero{module}
 \begin{itemize}
   \item {\bf Classes}
   \begin{itemize}
     \item \linkingone{module}{Submodule}
     \item \linkingone{module}{Module}
     \item \linkingone{module}{Ideal}
     \item \linkingone{module}{Ideal\_with\_generator}
   \end{itemize}
 \end{itemize}

\C

 \subsection{Submodule -- submodule as matrix representation}\linkedone{module}{Submodule}
 \initialize
  \func{Submodule}{\hiki{row}{integer},\ \hiki{column}{integer},\ \hikiopt{compo}{compo}{0},\ \hikiopt{coeff\_ring}{CommutativeRing}{0},\ \hikiopt{ishnf}{True/False}{None}}{\out{Submodule}}\\
  \spacing
  % document of basic document
  \quad Create a submodule with matrix representation.\\
  \spacing
  % added document
  \quad Submodule is subclass of \linkingone{matrix}{RingMatrix}.\\
  We assume that \param{coeff\_ring} is a PID (principal ideal domain).
  Then, we have the HNF(hermite normal form) corresponding to a matrices.\\
  \spacing
  %
  % input, output document
  If \param{ishnf} is True, we assume that the input matrix is a HNF.
  \begin{at}
   \item[ishnf] If the matrix is a HNF, then \param{ishnf} should be True, otherwise False.
  \end{at}
  \method
  \subsubsection{getGenerators -- generator of module}\linkedtwo{module}{Submodule}{getGenerators}
   \func{getGenerators}{\param{self}}{\out{list}}\\
   \spacing
   % document of basic document
   \quad Return a (current) generator of the module \param{self}.\\
   \spacing
   % added document
   %
   % input, output document
   \quad Return the list of vectors consisting of a generator.\\
 \subsubsection{isSubmodule -- Check whether submodule of self}\linkedtwo{module}{Submodule}{isSubmodule}
   \func{isSubmodule}{\param{self},\ \hiki{other}{Submodule}}{\out{True/False}}\\
   \spacing
   % document of basic document
   \quad Return True if the submodule instance is a submodule of the \param{other}, or False otherwise.\\
   \spacing
   % added document
   %
   %\spacing
   % input, output document
   %
   \subsubsection{isEqual -- Check whether self and other are same module}\linkedtwo{module}{Submodule}{isEqual}
   \func{isEqual}{\param{self},\ \hiki{other}{Submodule}}{\out{True/False}}\\
   \spacing
   % document of basic document
   \quad Return True if the submodule instance is \param{other} as module, or False otherwise.\\.\\
   \spacing
   % added document
   \quad You should use the method for equality test of module, not matrix.
   For equality test of matrix simply, use \param{self}$==$\param{other}.
   \\
   % input, output document
   %
   \subsubsection{isContain -- Check whether other is in self}\linkedtwo{module}{Submodule}{isContains}
   \func{isContains}{\param{self},\ \hiki{other}{vector.Vector}}{\out{True/False}}\\
   \spacing
   % document of basic document
   \quad Determine whether \param{other} is in \param{self} or not.\\.\\
   \spacing
   % added document
   \quad If you want to represent \param{other} as linear combination with the HNF generator of \param{self}, use \linkingtwo{module}{Submodule}{represent\_element}.
   \\
   % input, output document
   %
   \subsubsection{toHNF - change to HNF}\linkedtwo{matrix}{Submodule}{toHNF}
   \func{toHNF}{\param{self}}{\out{(None)}}\\
   \spacing
   % document of basic document
   \quad Rewrite \param{self} to HNF (hermite normal form), and set True to its \param{ishnf}.\\
   \spacing
   % added document
   \quad Note that HNF do not always give basis of \param{self}.(i.e. HNF may be redundant.)\\
   \spacing
   % input, output document
   %\quad
  \subsubsection{sumOfSubmodules - sum as submodule}\linkedtwo{matrix}{Submodule}{sumOfSubmodules}
   \func{sumOfSubmodules}{\param{self},\ \hiki{other}{Submodule}}{\out{Submodule}}\\
   \spacing
   % document of basic document
   \quad Return a module which is sum of two subspaces.\\
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   %\quad
  \subsubsection{intersectionOfSubmodules - intersection as submodule}\linkedtwo{matrix}{Submodule}{intersectionOfSubmodules}
   \func{intersectionOfSubmodules}{\param{self},\ \hiki{other}{Submodule}}{\out{Submodule}}\\
   \spacing
   % document of basic document
   \quad Return a module which is intersection of two subspaces.\\
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   %\quad
 \subsubsection{represent\_element -- represent element as linear combination}\linkedtwo{module}{Submodule}{represent\_element}
   \func{represent\_element}{\param{self},\ \hiki{other}{vector.Vector}}{\out{vector.Vector/False}}\\
   \spacing
   % document of basic document
   \quad Represent \param{other} as a linear combination with HNF generators.\\
   \spacing
   % added document
   \quad If \param{other} not in \param{self}, return False.
   Note that this method calls \linkingtwo{module}{Submodule}{toHNF}.\\
   \spacing
   % input, output document
   \quad The method returns coefficients as an instance of \linkingone{vector}{Vector}.
 \subsubsection{linear\_combination -- compute linear combination}\linkedtwo{module}{Submodule}{linear\_combination}
   \func{linear\_combination}{\param{self},\ \hiki{coeff}{list}}{\out{vector.Vector}}\\
   \spacing
   % document of basic document
   \quad For given $\mathbf{Z}$-coefficients \param{coeff},
         return a vector corresponding to a linear combination of (current) basis.\\
   \spacing
   % added document
   %
   % input, output document
   \quad \param{coeff} must be a list of instances in \linkingone{ring}{RingElement} whose size is the column of \param{self}.
\begin{ex}
>>> A = module.Submodule(4, 3, [1,2,3]+[4,5,6]+[7,8,9]+[10,11,12])
>>> A.toHNF()
>>> print A
9 1
6 1
3 1
0 1
>>> A.getGenerator
[Vector([9L, 6L, 3L, 0L]), Vector([1L, 1L, 1L, 1L])]
>>> V = vector.Vector([10,7,4,1])
>>> A.represent_element(V)
Vector([1L, 1L])
>>> V == A.linear_combination([1,1])
True
>>> B = module.Submodule(4, 1, [1,2,3,4])
>>> C = module.Submodule(4, 2, [2,-4]+[4,-3]+[6,-2]+[8,-1])
>>> print B.intersectionOfSubmodules(C)
2
4
6
8
\end{ex}%Don't indent!
\C
  \subsection{fromMatrix(class function) - create submodule}\linkedtwo{module}{Submodule}{fromMatrix}
  \func{fromMatrix}{\param{cls},\ \hiki{mat}{RingMatrix},\ \hikiopt{ishnf}{True/False}{None}}{\out{Submodule}}\\
   \spacing
   % document of basic document
   \quad Create a Submodule instance from a matrix instance \param{mat}, whose class can be any of subclasses of Matrix.\\
   \spacing
   % added document
   \quad Please use this method if you want a Submodule instance for sure.\\
   \spacing
   % input, output document
   %
\C

\subsection{Module - module over a number field}\linkedone{module}{Module}
 \initialize
  \func{Module}{\hiki{pair\_mat\_repr}{list/matrix},\ \hiki{number\_field}{algfield.NumberField},\ \hikiopt{base}{list/matrix.SquareMatrix}{None},\ \hikiopt{ishnf}{bool}{False}}{\out{Module}}\\
  \spacing
  % document of basic document
  \quad Create a new module object over a number field.\\
  \spacing
  % added document
  \quad A module is a finitely generated sub $\mathbf{Z}$-module.
  Note that we do not assume rank of a module is deg(number\_field).\\
  We represent a module as generators respect to base module over $\mathbf{Z}[\theta]$, where $\theta$ is a solution of \param{number\_field}.\linkingtwo{algfield}{BasicAlgNumber}{polynomial}.\\
  \spacing
  % input, output document
  \quad \param{pair\_mat\_repr} should be one of the following form:
  \begin{itemize}
  \item $[M,\ d]$,
   where $M$ is a list of integral tuple/vectors whose size is the degree of \param{number\_field} and
         $d$ is a denominator.
  \item $[M,\ d]$,
   where $M$ is an integral matrix whose the number of row is the degree of \param{number\_field} and
         $d$ is a denominator.
  \item a rational matrix whose the number of row is the degree of \param{number\_field}.
  \end{itemize}
  Also, \param{base} should be one of the following form:
  \begin{itemize}
  \item \linkedone{module}{base} a list of rational tuple/vectors whose size is the degree of \param{number\_field}
  \item a square non-singular rational matrix whose size is the degree of \param{number\_field}
  \end{itemize}
  The module is internally represented as $\frac{1}{d}M$ with respect to \linkingtwo{module}{Module}{base},
  where $d$ is \linkingtwo{module}{Module}{denominator} and $M$ is \linkingtwo{module}{Module}{mat\_repr}.
  If \param{ishnf} is True, we assume that \param{mat\_repr} is a HNF.\\
  \begin{at}
  \item[mat\_repr]\linkedtwo{module}{Module}{mat\_repr}: an instance of \linkingone{module}{Submodule} $M$ whose size is the degree of \param{number\_field}
  \item[denominator]\linkedtwo{module}{Module}{denominator}: an integer $d$
  \item[base]\linkedtwo{module}{Module}{base}: a square non-singular rational matrix whose size is the degree of \param{number\_field}
  \item[number\_field]\linkedtwo{module}{Module}{number\_field}: the number field over which the module is defined
  \end{at}
\begin{op}
    \verb+M==N+ & Return whether \param{M} and \param{N} are equal or not as module.\\
    \verb+c in M+ & Check whether some element of \param{M} equals \param{c}.\\
    \verb|M+N| & Return the sum of \param{M} and \param{N} as module.\\
    \verb+M*N+ & Return the product of \param{M} and \param{N} as the ideal computation. \\
               &  \param{N} must be module or scalar(i.e. an element of \linkingtwo{module}{Module}{number\_field}).\\
               &  If you want to compute the intersection of $M$ and $N$, see \linkingtwo{module}{Module}{intersect}.\\
    \verb+M**c+ & Return \param{M} to \param{c} based on the ideal multiplication.\\
    \verb+repr(M)+ & Return the repr string of the module \param{M}.\\
    \verb+str(M)+ & Return the str string of the module \param{M}.\\
  \end{op}
\begin{ex}
>>> F = algfield.NumberField([2,0,1])
>>> M_1 = module.Module([matrix.RingMatrix(2,2,[1,0]+[0,2]), 2], F)
>>> M_2 = module.Module([matrix.RingMatrix(2,2,[2,0]+[0,5]), 3], F)
>>> print M_1
([1, 0]+[0, 2], 2)
 over
([1L, 0L]+[0L, 1L], NumberField([2, 0, 1]))
>>> print M_1 + M_2
([1L, 0L]+[0L, 2L], 6)
 over
([Rational(1, 1), Rational(0, 1)]+[Rational(0, 1), Rational(1, 1)],
NumberField([2, 0, 1]))
>>> print M_1 * 2
([1L, 0L]+[0L, 2L], 1L)
 over
([Rational(1, 1), Rational(0, 1)]+[Rational(0, 1), Rational(1, 1)],
NumberField([2, 0, 1]))
>>> print M_1 * M_2
([2L, 0L]+[0L, 1L], 6L)
 over
([Rational(1, 1), Rational(0, 1)]+[Rational(0, 1), Rational(1, 1)],
NumberField([2, 0, 1]))
>>> print M_1 ** 2
([1L, 0L]+[0L, 2L], 4L)
 over
([Rational(1, 1), Rational(0, 1)]+[Rational(0, 1), Rational(1, 1)],
NumberField([2, 0, 1]))
\end{ex}%Don't indent!
\method
  \subsubsection{toHNF - change to hermite normal form(HNF)}\linkedtwo{module}{Module}{toHNF}
   \func{toHNF}{\param{self}}{\out{(None)}}\\
   \spacing
   % document of basic document
   \quad Change \param{self}.\linkingtwo{module}{Module}{mat\_repr} to the hermite normal form(HNF).\\
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   %\quad
  \subsubsection{copy - create copy}\linkedtwo{module}{Module}{copy}
   \func{copy}{\param{self}}{\out{Module}}\\
   \spacing
   % document of basic document
   \quad Create copy of \param{self}.\\
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   %\quad
  \subsubsection{intersect - intersection}\linkedtwo{module}{Module}{intersect}
   \func{intersect}{\param{self},\ \hiki{other}{Module}}{\out{Module}}\\
   \spacing
   % document of basic document
   \quad Return intersection of \param{self} and \param{other}.\\
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   %\quad
  \subsubsection{issubmodule - Check submodule}\linkedtwo{module}{Module}{issubmodule}
   \func{submodule}{\param{self},\ \hiki{other}{Module}}{\out{True/False}}\\
   \spacing
   % document of basic document
   \quad Check \param{self} is submodule of \param{other}.\\
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   %\quad
  \subsubsection{issupermodule - Check supermodule}\linkedtwo{module}{Module}{issupermodule}
   \func{supermodule}{\param{self},\ \hiki{other}{Module}}{\out{True/False}}\\
   \spacing
   % document of basic document
   \quad Check \param{self} is supermodule of \param{other}.\\
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   %\quad
  \subsubsection{represent\_element - Represent as linear combination}\linkedtwo{module}{Module}{represent\_element}
   \func{represent\_element}{\param{self},\ \hiki{other}{algfield.BasicAlgNumber}}{\out{list/False}}\\
   \spacing
   % document of basic document
   \quad Represent \param{other} as a linear combination with generators of \param{self}.
        If \param{other} is not in \param{self}, return False.\\
   \spacing
   % added document
   \quad Note that we do not assume \param{self}.\linkingtwo{module}{Module}{mat\_repr} is HNF.
   \spacing
   % input, output document
   \quad
   The output is a list of integers if \param{other} is in \param{self}.\\
   \spacing
  \subsubsection{change\_base\_module - Change base}\linkedtwo{module}{Module}{change\_base\_module}
   \func{change\_base\_module}{\param{self},\ \hiki{other\_base}{list/matrix.RingSquareMatrix}}{\out{Module}}\\
   \spacing
   % document of basic document
   \quad Return the module which is equal to \param{self} respect to \param{other\_base}.\\
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   \quad
   \param{other\_base} follows the form \linkingone{module}{base}.\\
  \subsubsection{index - size of module}\linkedtwo{module}{Module}{index}
   \func{index}{\param{self}}{\out{rational.Rational}}\\
   \spacing
   % document of basic document
   \quad Return the order of a residue group over \param{self}.\linkingtwo{module}{Module}{base}.
         That is, return $[M:N]$ if $N \subset M$ or ${[N:M]}^{-1}$ if $M subset N$,
         where $M$ is the module \param{self} and $N$ is the module corresponding to \param{self}.\linkingtwo{module}{Module}{base}. \\
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   %\quad
  \subsubsection{smallest\_rational - a $\mathbf{Z}$-generator in the rational field}\linkedtwo{module}{Module}{smallest\_rational}
   \func{smallest\_rational}{\param{self}}{\out{rational.Rational}}\\
   \spacing
   % document of basic document
   \quad Return the $\mathbf{Z}$-generator of intersection of the module \param{self} and the rational field.\\
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   %\quad
 \begin{ex}
>>> F = algfield.NumberField([1,0,2])
>>> M_1=module.Module([matrix.RingMatrix(2,2,[1,0]+[0,2]), 2], F)
>>> M_2=module.Module([matrix.RingMatrix(2,2,[2,0]+[0,5]), 3], F)
>>> print M_1.intersect(M_2)
([2L, 0L]+[0L, 5L], 1L)
 over
([Rational(1, 1), Rational(0, 1)]+[Rational(0, 1), Rational(1, 1)],
 NumberField([2, 0, 1]))
>>> M_1.represent_element( F.createElement( [[2,4], 1] ) )
[4L, 4L]
>>> print M_1.change_base_module( matrix.FieldSquareMatrix(2, 2, [1,0]+[0,1]) / 2 )
([1L, 0L]+[0L, 2L], 1L)
 over
([Rational(1, 2), Rational(0, 1)]+[Rational(0, 1), Rational(1, 2)],
 NumberField([2, 0, 1]))
>>> M_2.index()
Rational(10, 9)
>>> M_2.smallest_rational()
Rational(2, 3)
\end{ex}%Don't indent!
\C

\subsection{Ideal - ideal over a number field}\linkedone{module}{Ideal}
 \initialize
  \func{Ideal}{\hiki{pair\_mat\_repr}{list/matrix},\ \hiki{number\_field}{algfield.NumberField},\ \hikiopt{base}{list/matrix.SquareMatrix}{None},\ \hikiopt{ishnf}{bool}{False}}{\out{Ideal}}\\
  \spacing
  % document of basic document
  \quad Create a new ideal object over a number field.\\
  \spacing
  % added document
  \quad Ideal is subclass of \linkingone{module}{Module}.\\
  \spacing
  % input, output document
  \quad Refer to initialization of \linkingone{module}{Module}.\\
%\begin{ex}
%\end{ex}%Don't indent!
\C
\method

\subsubsection{inverse -- inverse}\linkedtwo{module}{Ideal}{inverse}
   \func{inverse}{\param{self}}{\out{Ideal}}\\
   \spacing
   % document of basic document
   \quad Return the inverse ideal of \param{self}.\\
   \spacing
   % added document
   \quad This method calls \param{self}.\linkingtwo{module}{Module}{number\_field}.\linkingtwo{algfield}{NumberField}{integer\_ring}.\\
   \spacing
   % input, output document
   %\quad \\
 \subsubsection{issubideal -- Check subideal}\linkedtwo{module}{Ideal}{issubideal}
   \func{issubideal}{\param{self},\ \hiki{other}{Ideal}}{\out{Ideal}}\\
   \spacing
   % document of basic document
   \quad Check \param{self} is subideal of \param{other}.\\
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   %\quad \\
  \subsubsection{issuperideal -- Check superideal}\linkedtwo{module}{Ideal}{issuoerideal}
   \func{issuperideal}{\param{self},\ \hiki{other}{Ideal}}{\out{Ideal}}\\
   \spacing
   % document of basic document
   \quad Check \param{self} is superideal of \param{other}.\\
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   %\quad \\
  \subsubsection{gcd -- greatest common divisor}\linkedtwo{module}{Ideal}{gcd}
   \func{gcd}{\param{self},\ \hiki{other}{Ideal}}{\out{Ideal}}\\
   \spacing
   % document of basic document
   \quad Return the greatest common divisor(gcd) of \param{self} and \param{other} as ideal.\\
   \spacing
   % added document
   \quad This method simply executes \param{self}$+$\param{other}.\\
   \spacing
   % input, output document
   %\quad \\
  \subsubsection{lcm -- least common multiplier}\linkedtwo{module}{Ideal}{lcm}
   \func{lcm}{\param{self},\ \hiki{other}{Ideal}}{\out{Ideal}}\\
   \spacing
   % document of basic document
   \quad Return the least common multiplier(lcm) of \param{self} and \param{other} as ideal.\\
   \spacing
   % added document
   \quad This method simply calls the method \linkingtwo{module}{Module}{intersect}.\\
   \spacing
   % input, output document
   %\quad \\
%  \subsubsection{twoElementRepresentation -- Represent as two element}\linkedtwo{module}{Ideal}{twoElementRepresentation}
%   \func{twoElementRepresentation}{\param{self}}{\out{Ideal}}\\
%   \spacing
%   % document of basic document
%   \quad Return the ideal which is \param{self} represented with only two elements.\\
%   \spacing
%   % added document
%   %\quad \\
%   %\spacing
%   % input, output document
%   %\quad \\
  \subsubsection{norm -- norm}\linkedtwo{module}{Ideal}{norm}
   \func{norm}{\param{self}}{\out{rational.Rational}}\\
   \spacing
   % document of basic document
   \quad Return the norm of \param{self}.\\
   \spacing
   % added document
   \quad This method calls \param{self}.\linkingtwo{module}{Module}{number\_field}.\linkingtwo{algfield}{NumberField}{integer\_ring}.\\
   \spacing
   % input, output document
   %\quad \\
  \subsubsection{isIntegral -- Check integral}\linkedtwo{module}{Ideal}{isIntegral}
   \func{isIntegral}{\param{self}}{\out{True/False}}\\
   \spacing
   % document of basic document
   \quad Determine whether \param{self} is an integral ideal or not.\\
   \spacing
   % added document
   %\quad \\
   %\spacing
   % input, output document
   %\quad \\
%  \subsubsection{isPrime -- Check primality}\linkedtwo{module}{Ideal}{isPrime}
%   \func{isPrime}{\param{self}}{\out{True/False}}\\
%   \spacing
%   % document of basic document
%   \quad Determine whether \param{self} is a prime ideal or not.\\
%   \spacing
%   % added document
%   %\quad \\
%   %\spacing
%   % input, output document
%   %\quad \\
\begin{ex}
>>> M = module.Ideal([matrix.RingMatrix(2, 2, [1,0]+[0,2]), 2], F)
>>> print M.inverse()
([-2L, 0L]+[0L, 2L], 1L)
 over
([Rational(1, 1), Rational(0, 1)]+[Rational(0, 1), Rational(1, 1)],
 NumberField([2, 0, 1]))
>>> print M * M.inverse()
([1L, 0L]+[0L, 1L], 1L)
 over
([Rational(1, 1), Rational(0, 1)]+[Rational(0, 1), Rational(1, 1)],
 NumberField([2, 0, 1]))
>>> M.norm()
Rational(1, 2)
>>> M.isIntegral()
False
\end{ex}%Don't indent!
\C

\subsection{Ideal\_with\_generator - ideal with generator}\linkedone{module}{Ideal\_with\_generator}
 \initialize
  \func{Ideal\_with\_generator}{\hiki{generator}{list}}{\out{Ideal\_with\_generator}}\\
  \spacing
  % document of basic document
  \quad Create a new ideal given as a generator.\\
  \spacing
  % added document
  %\quad
  %\spacing
  % input, output document
  \quad \param{generator} is a list of instances in \linkingone{algfield}{BasicAlgNumber}, which represent generators, over a same number field.
  \begin{at}
  \item[generator]\linkedtwo{module}{Ideal\_with\_generator}{generator}: generators of the ideal
  \item[number\_field]\linkedtwo{module}{Ideal\_with\_generator}{number\_field}: the number field over which generators are defined
  \end{at}
\begin{op}
    \verb+M==N+ & Return whether \param{M} and \param{N} are equal or not as module.\\
    \verb+c in M+ & Check whether some element of \param{M} equals \param{c}.\\
    \verb|M+N| & Return the sum of \param{M} and \param{N} as ideal with generators.\\
    \verb+M*N+ & Return the product of \param{M} and \param{N} as ideal with generators. \\
    \verb+M**c+ & Return \param{M} to \param{c} based on the ideal multiplication.\\
    \verb+repr(M)+ & Return the repr string of the ideal \param{M}.\\
    \verb+str(M)+ & Return the str string of the ideal \param{M}.\\
  \end{op}
\begin{ex}
>>> F = algfield.NumberField([2,0,1])
>>> M_1 = module.Ideal_with_generator([
 F.createElement([[1,0], 2]), F.createElement([[0,1], 1])
])
>>> M_2 = module.Ideal_with_generator([
 F.createElement([[2,0], 3]), F.createElement([[0,5], 3])
])
>>> print M_1
[BasicAlgNumber([[1, 0], 2], [2, 0, 1]), BasicAlgNumber([[0, 1], 1], [2, 0, 1])]
>>> print M_1 + M_2
[BasicAlgNumber([[1, 0], 2], [2, 0, 1]), BasicAlgNumber([[0, 1], 1], [2, 0, 1]),
 BasicAlgNumber([[2, 0], 3], [2, 0, 1]), BasicAlgNumber([[0, 5], 3], [2, 0, 1])]
>>> print M_1 * M_2
[BasicAlgNumber([[1L, 0L], 3L], [2, 0, 1]), BasicAlgNumber([[0L, 5L], 6], [2, 0, 1]),
BasicAlgNumber([[0L, 2L], 3], [2, 0, 1]), BasicAlgNumber([[-10L, 0L], 3], [2, 0, 1])]
>>> print M_1 ** 2
[BasicAlgNumber([[1L, 0L], 4], [2, 0, 1]), BasicAlgNumber([[0L, 1L], 2], [2, 0, 1]),
BasicAlgNumber([[0L, 1L], 2], [2, 0, 1]), BasicAlgNumber([[-2L, 0L], 1], [2, 0, 1])]
\end{ex}%Don't indent!
\method
  \subsubsection{copy - create copy}\linkedtwo{module}{Ideal\_with\_generator}{copy}
   \func{copy}{\param{self}}{\out{Ideal\_with\_generator}}\\
   \spacing
   % document of basic document
   \quad Create copy of \param{self}.\\
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   %\quad
  \subsubsection{to\_HNFRepresentation - change to ideal with HNF}\linkedtwo{module}{Ideal\_with\_generator}{to\_HNFRepresentation}
   \func{to\_HNFRepresentation}{\param{self}}{\out{Ideal}}\\
   \spacing
   % document of basic document
   \quad Transform \param{self} to the corresponding ideal as HNF(hermite normal form) representation.\\
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   %\quad
  \subsubsection{twoElementRepresentation - Represent with two element}\linkedtwo{module}{Ideal\_with\_generator}{twoElementRepresentation}
   \func{twoElementRepresentation}{\param{self}}{\out{Ideal\_with\_generator}}\\
   \spacing
   % document of basic document
   \quad Transform \param{self} to the corresponding ideal as HNF(hermite normal form) representation.\\
   \spacing
   % added document
   \quad  If \param{self} is not a prime ideal, this method is not efficient.\\
   \spacing
   % input, output document
   %\quad
  \subsubsection{smallest\_rational - a $\mathbf{Z}$-generator in the rational field}\linkedtwo{module}{Ideal\_with\_generator}{smallest\_rational}
   \func{smallest\_rational}{\param{self}}{\out{rational.Rational}}\\
   \spacing
   % document of basic document
   \quad Return the $\mathbf{Z}$-generator of intersection of the module \param{self} and the rational field.\\
   \spacing
   % added document
   \quad This method calls \linkingtwo{module}{Ideal\_with\_generator}{to\_HNFRepresentation}.\\
   \spacing
   % input, output document
   %\quad
\subsubsection{inverse -- inverse}\linkedtwo{module}{Ideal\_with\_generator}{inverse}
   \func{inverse}{\param{self}}{\out{Ideal}}\\
   \spacing
   % document of basic document
   \quad Return the inverse ideal of \param{self}.\\
   \spacing
   % added document
   \quad This method calls \linkingtwo{module}{Ideal\_with\_generator}{to\_HNFRepresentation}.\\
   \spacing
   % input, output document
   %\quad \\
  \subsubsection{norm -- norm}\linkedtwo{module}{Ideal\_with\_generator}{norm}
   \func{norm}{\param{self}}{\out{rational.Rational}}\\
   \spacing
   % document of basic document
   \quad Return the norm of \param{self}.\\
   \spacing
   % added document
   \quad This method calls \linkingtwo{module}{Ideal\_with\_generator}{to\_HNFRepresentation}.\\
   \spacing
   % input, output document
   %\quad \\
  \subsubsection{intersect - intersection}\linkedtwo{module}{Ideal\_with\_generator}{intersection}
   \func{intersect}{\param{self},\ \hiki{other}{Ideal\_with\_generator}}{\out{Ideal}}\\
   \spacing
   % document of basic document
   \quad Return intersection of \param{self} and \param{other}.\\
   \spacing
   % added document
   \quad This method calls \linkingtwo{module}{Ideal\_with\_generator}{to\_HNFRepresentation}.\\
   \spacing
   % input, output document
   %\quad
 \subsubsection{issubideal -- Check subideal}\linkedtwo{module}{Ideal\_with\_generator}{issubideal}
   \func{issubideal}{\param{self},\ \hiki{other}{Ideal\_with\_generator}}{\out{Ideal}}\\
   \spacing
   % document of basic document
   \quad Check \param{self} is subideal of \param{other}.\\
   \spacing
   % added document
   \quad This method calls \linkingtwo{module}{Ideal\_with\_generator}{to\_HNFRepresentation}.\\
   \spacing
   % input, output document
   %\quad \\
  \subsubsection{issuperideal -- Check superideal}\linkedtwo{module}{Ideal\_with\_generator}{issuoerideal}
   \func{issuperideal}{\param{self},\ \hiki{other}{Ideal\_with\_generator}}{\out{Ideal}}\\
   \spacing
   % document of basic document
   \quad This method calls \linkingtwo{module}{Ideal\_with\_generator}{to\_HNFRepresentation}.\\
   \spacing
   % added document
   %\quad
   %\spacing
   % input, output document
   %\quad \\
\begin{ex}
>>> M = module.Ideal_with_generator([
F.createElement([[2,0], 3]), F.createElement([[0,2], 3]), F.createElement([[1,0], 3])
])
>>> print M.to_HNFRepresentation()
([2L, 0L, 0L, -4L, 1L, 0L]+[0L, 2L, 2L, 0L, 0L, 1L], 3L)
 over
([1L, 0L]+[0L, 1L], NumberField([2, 0, 1]))
>>> print M.twoElementRepresentation()
[BasicAlgNumber([[1L, 0], 3], [2, 0, 1]), BasicAlgNumber([[3, 2], 3], [2, 0, 1])]
>>> M.norm()
Rational(1, 9)
\end{ex}%Don't indent!
\C

%---------- end document ---------- %

\input{../footer}


\chapter{Finite element discretisation}
\label{sec:discretization}

In this chapter we introduce a mixed finite element discretisation for the steady-state incompressible MHD problem \eqref{eq:mhd}, \eqref{eq:bc} that models electrically conductive fluids under the influence of a magnetic field.  Following the setting in \cite{schotzau2004mixed}, we use curl-conforming elements for the magnetic field and conforming continuous elements for the velocity field. The resulting discretisation is verified though a series of numerical experiments which appear later in Chapter \ref{chap:results}. For simplicity, we initially only discuss in detail homogeneous Dirichlet boundary conditions, that is
\begin{equation} \label{eq:homogeneousBC}
    \uu{u} = \uu{0} \quad \mbox{and} \quad \uu{n}\times \uu{b} = \uu{0}.
\end{equation}
Inhomogeneous conditions as in \eqref{eq:bc}  are  discussed in Section~\ref{sec:bcig}.

\section{Variational formulation}
\label{sec:variation}

\RE{Suppose that the domain $\Omega$ is a Lipschitz domain of $\mathbb{R}^d$ for $d=2,3$.} To express the problem \eqref{eq:mhd}, \eqref{eq:bc} in weak form we follow \cite{schotzau2004mixed} and denote the $L^2$-inner product on $L^2(\Omega)^d$ by $(\cdot,\cdot)_\Omega$, for $d = 2,3$. We introduce the standard Sobolev spaces
\begin{equation} \label{eq:FuncSpace}
 \left. \begin{aligned}
\hspace{-1.5mm}\uu{V}&=H_0^1(\Omega)^d=\left\{\uu{u}\in H^1(\Omega)^d\,:\,\text{$\uu{u}=\uu{0}$ on $\partial\Omega$}\right\},\\
\hspace{-1.5mm}Q&=L^2_0(\Omega)=\{p\in L^2(\Omega)\,:\,(p\,,1)_\Omega=0\},\\
\hspace{-1.5mm}\uu{C}&=H_0({\rm curl};\Omega) = \left\{\uu{b}\in L^2(\Omega)^d\,:\,\nabla\times\uu{b}\in L^2(\Omega)^{\bar{d}}, \
\text{$\uu{n}\times\uu{b}=\uu{0}$ on $\partial\Omega$}\right\},\\
\hspace{-1.5mm}S&=H^1_0(\Omega)=\{r\in H^1(\Omega)\,:\,r=0\ \mbox{on $\partial\Omega$}\}.
 \end{aligned}
 \right.
 \qquad \text{}
\end{equation}
where $\bar{d}={2d-3}$ is used to cover the 2D and 3D cases. We write $\|\cdot\|_{L^2(\Omega)}$, $\|\cdot\|_{H^1(\Omega)}$ and $\|\cdot\|_{H(\rm{curl};\Omega)}$ for the associated natural norms. More precisely, for  vector fields $\uu{u},\uu{b}$ and a scalar function $r$ the norms are defined as follows:
\begin{equation} \nonumber
 \left. \begin{aligned}
    \|\uu{u}\|_{L^2 (\Omega)} &= \left({\int_{\Omega} \uu{u}\cdot\uu{u}\;dx}\right)^{\frac{1}{2}},\\
   \|\uu{u}\|_{H^1(\Omega)} &=  \left(\|\uu{u}\|_{L^2(\Omega)}^2 + \|\nabla  \uu{u}\|_{L^2(\Omega)}^2 \right)^{\frac{1}{2}},\\
   \|\uu{b}\|_{H(\rm{curl},\Omega)} &=  \left(\|\uu{b}\|_{L^2(\Omega)}^2 + \|\nabla \times \uu{b}\|_{L^2(\Omega)}^2 \right)^{\frac{1}{2}}, \\
    \|r\|_{L^2 (\Omega)} &= \left({\int_{\Omega} r^2\;dx}\right)^{\frac{1}{2}},\\
    \|r\|_{H^1(\Omega)} &=  \left(\|r\|_{L^2(\Omega)}^2 + \|\nabla  r\|_{L^2(\Omega)}^2 \right)^{\frac{1}{2}},\\
 \end{aligned}
 \right.
 \qquad \text{}
\end{equation}
where $\|\nabla  \uu{u}\|_{L^2(\Omega)}^2$ is given by:
$$\|\nabla  \uu{u}\|_{L^2(\Omega)}^2 = \left(\int_{\Omega} \sum^d_{i,j=1}(\nabla \uu{u})_{ij}(\nabla \uu{u})_{ij} \, dx\right)^{\frac12}.$$
The weak formulation of the incompressible MHD system (\ref{eq:mhd}), (\ref{eq:bc}) consists in finding~$(\uu{u},p,\uu{b},r)\in \uu{V} \times Q\times \uu{C} \times S$ such that
\begin{subequations}
\label{eq:weak}
\begin{eqnarray}
\label{eq:weak1} A(\uu{u},\uu{v}) + O(\uu{u};\uu{u},\uu{v})
+C(\uu{b};\uu{v},\uu{b})
+B(\uu{v}, p) & =& (\uu{f}, \uu{v})_{\Omega},\\[.1cm]
\label{eq:weak2}
B(\uu{u},q)&=&0, \\[.1cm]
\label{eq:weak3}
M(\uu{b},\uu{c})-C(\uu{b};\uu{u},\uu{c})+D(\uu{c},r)&=& (\uu{g},\uu{c})_\Omega, \\[.1cm]
\label{eq:weak4} D(\uu{b},s)&=&0,
\end{eqnarray}
\end{subequations}
for all $(\uu{v},q,\uu{c},s)\in \uu{V} \times Q\times \uu{C}\times
S$. The individual variational forms are given by
\begin{equation} \label{eq:forms}
 \left. \begin{aligned}
&A(\uu{u},\uu{v})=  \int_\Omega \nu \, \nabla\uu{u}:
\nabla\uu{v}\,d\uu{x},&\\  & O(\uu{w};\uu{u},\uu{v}) = \int_\Omega
(\uu{w}\cdot\nabla)\uu{u} \cdot\uu{v} \, d\uu{x},
\\[.1cm]
&  B(\uu{u},q) = -\int_\Omega\,(\nabla\cdot\uu{u}) \,q \,d\uu{x},
&\\  &
 M(\uu{b},\uu{c})= \int_\Omega\, \kappa\nu_m
(\nabla\times\uu{b})\cdot(\nabla\times\uu{c})\,d\uu{x},\\[0.1cm]
& D(\uu{b},s) = \int_\Omega\, \uu{b} \cdot \nabla s\,
d\uu{x}, & \\ &
C(\uu{d};\uu{v},\uu{b}) =  \int_\Omega \kappa\, (\uu{v}\times\uu{d})\cdot
(\nabla\times\uu{b})\, d\uu{x},
 \end{aligned}
 \right.
 \quad\text{}
\end{equation}
where  $\nabla \uu{u}:\nabla \uu{v}$ is  defined as
$$\nabla \uu{u}:\nabla \uu{v} = \sum^d_{i,j=1}(\nabla \uu{u})_{ij}(\nabla \uu{v})_{ij}.$$ In \cite{schotzau2004mixed} it has been shown that this formulation of the problem is discretely energy-stable and has a unique solution for small data (i.e. for small $\nu$, $\nu_m$, $\kappa$ and forcing terms $\uu{f}$ and $\uu{g}$ with small $L^2$-norms).

\section{Mixed finite element discretisation}

Consider the domain $\Omega$ to be divided up into a regular and quasi-uniform mesh ${\mathcal T}_h=\{K\}$ consisting of triangles ($d = 2$) or tetrahedra ($d = 3$)  with mesh size $h$. Based on the function spaces defined in \eqref{eq:FuncSpace}, our finite element approximation will be sought in the finite dimensional spaces given by:
\begin{equation}
\label{eq:FiniteSpace}
\begin{split}
\uu{V}_h &=  \{\, \uu{u}\in H^1( \Omega)\, :\, \uu{u}|_K \in {\mathcal P}_{k}(K)^d, \, K \in{\mathcal T}_h \, \},\\[.1cm]
Q_h&=  \{\, p\in L^2(\Omega) \cap H^1(\Omega)\,:\, p|_K \in {\mathcal P}_{k-1}(K), \, K \in{\mathcal T}_h \,\},\\[.1cm]
\uu{C}_h &=  \{\, \uu{b}\in H_0({\rm curl}; \Omega) \,:\, \uu{b}|_K \in {\mathcal P}_{k-1}(K)^d \oplus \uu{R}_k(K), \, K \in{\mathcal T}_h \,\},\\[.1cm]
S_h&=  \{\, r\in H_0^1(\Omega) \,:\, r|_K \in {\mathcal P}_{k}(K), \, K \in {\mathcal T}_h \, \},
\end{split}
\end{equation}
for $k\geq 2$. We define ${\mathcal P}_{k}(K)$ as the space of polynomials of total degree at most $k$ on $K$ and $ \uu{R}_k(K)$ as the space of homogeneous vector polynomials of total degree $k$ on $K$ that are orthogonal to the position vector $\uu{x}$. Here we note that we are using ${\mathcal P_k}/{\mathcal P_{k-1}}$ Taylor-Hood elements for the fluid unknowns $(\uu{u},p)$ \cite{taylor1973numerical}. For the magnetic variables $(\uu{b},r)$ we use the curl-conforming \nedelec element pair     of the first kind \cite{nedelec1980mixed}. These choices of finite elements spaces $\uu{V}_h, \, \uu{C}_h, \, Q_h$ and $S_h$ imply that  we have conforming subspaces to our Sobolev spaces $\uu{V}, \, \uu{C}, \,Q$ and $S$, respectively. Then the finite element solution to \eqref{eq:weak} consists in finding $(\uu{u}_h,p_h,\uu{b}_h,r_h)\in \uu{V}_h\times Q_h\times \uu{C}_h\times S_h$ such that
\begin{subequations}
\label{eq:VariationForm}
\begin{eqnarray}
\label{eq:bn1} \hspace{-15mm} A(\uu{u}_h,\uu{v}) + \tilde{O}(\uu{u}_h;\uu{u}_h,\uu{v}) +C(\uu{b}_h;\uu{v},\uu{b}_h) +B(\uu{v}, p_h) & = & ( \uu{f},\uu{v}),\\[.1cm]
\label{eq:bn2}
B(\uu{u}_h,q)&=& 0, \\[.1cm]
\label{eq:bn3} M(\uu{b}_h,\uu{c})-C(\uu{b}_h;\uu{u}_h,\uu{c})+ D(\uu{c},r_h)&=& (\uu{g},\uu{c}),\\[.1cm]
\label{eq:bn4} D(\uu{b}_h,s)&=&0,
\end{eqnarray}
\end{subequations}
for all $(\uu{v},q,\uu{c},s)\in \uu{V}_h\times Q_h \times \uu{C}_h\times S_h$.

The forms $A, M, B, D$ and $C$ stay the same as on the continuous level. However, for the convection term $\tilde{O}(\cdot;\cdot,\cdot)$ we modify the form $O(\uu{w};\uu{u},\uu{v})$ in a standard fashion to ensure the energy-stability property
\begin{equation} \label{eq:convection}
    \tilde{O}(\uu{w};\uu{u},\uu{u}) = 0, \quad \forall \uu{w},\uu{u} \in  \uu{V}_h.
\end{equation}
To do so we integrate by parts the convection form $O(\uu{w};\uu{u},\uu{u})$  to obtain
\begin{equation} \nonumber
 \left. \begin{aligned}
     \int_\Omega (\uu{w}\cdot\nabla)\uu{u} \cdot\uu{u} \, d\uu{x} =& -\frac{1}{2}\int_{\Omega} \nabla \cdot \uu{w} \uu{u} \cdot \uu{u} \, d\uu{x}
     +\frac{1}{2}\int_{\partial \Omega} \uu{w}\cdot \uu{n} |\uu{u}|^2\, ds,
 \end{aligned}
 \right.
 \qquad \text{}
\end{equation}
recalling that $\uu{n}$ is the unit outward normal on $\partial \Omega$. Therefore, we choose the modified convection form $\tilde{O}(\uu{w};\uu{u},\uu{v})$ as
$$\tilde{O}(\uu{w};\uu{u},\uu{v}) =  \int_\Omega (\uu{w}\cdot\nabla)\uu{u} \cdot\uu{v} \, d\uu{x} +\frac{1}{2}\int_{\Omega} \nabla \cdot \uu{w} \uu{u} \cdot \uu{v}\, d\uu{x}-\frac{1}{2}\int_{\partial \Omega} \uu{w}\cdot \uu{n} \uu{u} \cdot \uu{v}\, ds.$$
By construction, property \eqref{eq:convection} is now satisfied. Note also that for homogeneous boundary conditions as assumed in \eqref{eq:homogeneousBC}, the boundary integral term in $\tilde{O}$ can be omitted.

Again in \cite{schotzau2004mixed} it has been shown that this variational formulation of a MHD problem is discretely energy-stable and has a unique solution for small data. Also, optimal order error estimates in the mesh size $h$ have been derived for small data using the stability property \eqref{eq:convection}. Namely, for sufficiently smooth solutions, we have the error bound
$$\|\uu{u}-\uu{u}_h\|_{H^1(\Omega)}+\|\uu{b}-\uu{b}_h\|_{H(\rm{curl};\Omega)}+\|p-p_h\|_{L^2(\Omega)}+\|r-r_h\|_{H^1(\Omega)} \leq C h^k,$$
for a constant $C>0$ independent of the mesh size. In addition, the $L^2$-norm error for the velocity field is of order $\mathcal{O}(h^{k+1})$ (as $\uu{V}_h$ consists of a full polynomial space on each element). However, we cannot expect $L^2$-norm errors of  order $\mathcal{O}(h^{k+1})$ for the magnetic field (as $\uu{C}_h$ does not consist of a full polynomial space on each element).

\subsection{Matrix representation}

The variational formulation \eqref{eq:VariationForm} now can be converted into a matrix representation. To do this, we introduce the basis function for the finite element spaces in \eqref{eq:FiniteSpace}:
\begin{alignat}2
\label{eq:bases1}
\uu{V}_h & = \mbox{span}\langle  \uu{\psi}_j \rangle _{j=1}^{n_u}, & \qquad &
Q_h  = \mbox{span} \langle  \alpha_i \rangle _{i=1}^{m_u},\\[0.1cm]
 \uu{C}_h& =\mbox{span}\langle \uu{\phi}_j \rangle _{j=1}^{n_b}, & \qquad & S_h = \mbox{span} \langle \beta_i
\rangle_{i=1}^{m_b}.
\end{alignat}
The aim now is to find the coefficient vectors $u = (u_1, \ldots , u_{n_u}) \in \mathbb{R}^{n_u}$, $p = (p_1, \ldots , p_{m_u}) \in \mathbb{R}^{m_u}$, $b = (b_1, \ldots , b_{n_b}) \in \mathbb{R}^{n_b}$, and $r = (r_1, \ldots , r_{m_b}) \in \mathbb{R}^{m_b}$ of the finite element functions $(\uu{u}_h, p_h,\uu{b}_h, r_h)$ in terms of the chosen bases. As usual, this is done by writing the bilinear forms in \eqref{eq:VariationForm} in terms of the following stiffness matrices and load vectors:
\begin{alignat*}2
A_{i,j} &= A(\uu{\psi}_j,\uu{\psi}_i), &\quad  &1 \leq i,j \leq n_u,\\[0.1cm]
B_{i,j} &= B(\uu{\psi}_j,\alpha_i), &\quad &1 \leq i \leq m_u, \ 1 \leq j \leq n_u,\\[.1cm]
D_{i,j} &= D(\uu{\phi}_j,\beta_i),  & & 1 \leq i \leq m_b,\ 1 \leq j \leq n_b,\\[.1cm]
M_{i,j}&= M(\uu{\phi}_j,\uu{\phi}_i), &\qquad & 1 \leq i,j \leq n_b,\\[.1cm]
f_i &= (\uu{f},\uu{\psi}_i)_\Omega, & & 1\leq i\leq n_u,\\[.1cm]
g_i &= (\uu{g},\uu{\phi}_i)_\Omega, & & 1\leq i \leq n_b.
\end{alignat*}
For the two non-linear forms, $\tilde{O}$ and $C$, we define the corresponding stiffness matrices with respect to given finite element functions $\uu{w}_h \in \uu{V}_h$ and $\uu{d}_h\in \uu{C}_h$ in the first argument and their associated coefficient vectors $w$ and $d$ as
\begin{alignat*}2
O(w)_{i,j} &=\tilde{O}(\uu{w}_h;\uu{\psi}_j,\uu{\psi}_i), &\quad  &1 \leq i,j \leq n_u,\\[.1cm]
C(d)_{i,j} &= C(\uu{d}_h;\uu{\psi}_j,\uu{\phi}_i), & & 1\leq i \leq n_b,\ 1 \leq j \leq n_u.
\end{alignat*}

Thus, the numerical solution to \eqref{eq:mhd} consists in solving the non-linear system
\begin{equation}
\label{eq:matrix-system}
\left(
\begin{array}{cccc}
A+O(u) & B^T & C^T(b) & 0\\
B & 0 & 0 & 0\\
-C(b) & 0 & M & D^T \\
0 & 0 & D & 0
\end{array}
\right)
\,
\left(
\begin{array}{c}
u\\
p\\
b\\
r
\end{array}
\right) =
\left(
\begin{array}{c} f\\0\\g\\0
\end{array}
\right),
\end{equation}
where the vectors  $u\in\mathbb{R}^{n_u}$, $p\in\mathbb{R}^{m_u}$,  $b\in\mathbb{R}^{n_b}$, and $r\in\mathbb{R}^{m_b}$ are the unknown coefficients of the finite element functions.

\section{Picard iteration (P)}
\label{sec:nonlinear}
The discrete system \eqref{eq:matrix-system} is non-linear, and therefore appling a non-linear solver to this problem is necessary. A common choice to deal with the non-linearity within the incompressible Navier-Stokes equations in isolation is to perform Oseen or Picard iterations \cite{elman2005finite}. This involves linearising around the current velocity and solving for updates.

% One (at least theoretical) advantage of the discrete Oseen system is that it is provably energy-stable (for the skew-symmetrized form which we use) for all values of nu. That is, the matrix is not symmetric but positive definite, which might be an advantage for preconditioning. For small data (i.e. for large \nu) the fixed-point iteration is a contraction.

\RE{For simplicity we only consider the linearly convergent Picard iterations. Since we have modified the convection form to be discretely energy-stable as in \eqref{eq:convection}, an advantage of this approach is that the discrete convection-diffusion operator is real positive. Thus, with small data the fixed-point/Picard iteration is a contraction. A more efficient non-linear solver is Newton's method which converges quadratically near the solution. However, applying Newton's method is more involved, as it requires construction and solving linear systems associated with a Jacobian as well as finding an initial guess sufficiently close to the solution.
% A common approach to ensure convergence with Newton's method is to perform a few Picard iterations before starting the Newton scheme.
% In Section~5.2 we mention the possibility of using Newton's method as an area of future work.
We leave the implementation of Newton's method or other non-linear solvers as an area of possible future work (Section~5.2).
}

% \RE{For simplicity we only consider the linearly convergent Picard iterations. An example of a more efficient non-linear solver is Newton's method which converges quadratically. However, the main difficulty with Newton's method is that convergence is only local and hence the initial guess needs to be sufficiently accurate to obtain convergence. Since we have modified the convection form to be descretely energy-stable \eqref{eq:convection} then an advantage of the discrete Oseen systems is that the matrix is positive definite. Thus, this provides an advantage for preconditioning and with small data the fixed-point/Picard iterations is a contraction.
% % A common approach to ensure convergence with Newton's method is to perform a few Picard iterations before starting the Newton scheme.
% % In Section~5.2 we mention the possibility of using Newton's method as an area of future work.
% We leave the implementation of Newton's method or other non-linear solvers as an area of possible future work (Section~5.2).
% }

\RE{We adapt the fixed-point/Picard iterations to an approach for the full MHD system, where we linearise around the current velocity and magnetic fields.} Given a current iterate $(\uu{u}_h,p_h,\uu{b}_h,r_h)$  we solve for updates $(\delta \uu{u}_h,\delta p_h,\delta \uu{b}_h,\delta r_h)$ and introduce the next iterate by setting:
\begin{equation}\nonumber
\begin{array}{cc}
% \label{eq:updates}
\uu{u}_h& \hspace{-3mm} \rightarrow \uu{u}_h +\delta \uu{u}_h, \quad p_h \rightarrow p_h +\delta p_h,\\
\uu{b}_h& \hspace{-3mm}  \rightarrow \uu{b}_h +\delta \uu{b}_h, \quad r_h \rightarrow r_h +\delta r_h.
\end{array}
\end{equation}
In variational form, the updates $(\delta \uu{u}_h,\delta p_h,\delta \uu{b}_h,\delta r_h)\in \uu{V}_h\times Q_h \times \uu{C}_h\times S_h$ are found by solving the Picard system (P):
\begin{equation} \nonumber
% \label{eq:picard}
\begin{split}
A(\delta\uu{u}_h, \uu{v}) +\tilde{O}(\uu{u};\delta\uu{u}_h,\uu{v})+ C(\uu{b}_h;\uu{v},\delta \uu{u}_h) + B(\uu{v}, \delta p_h) & = R_u(\uu{u}_h,\uu{b}_h,p_h;\uu{v}),\\[.1cm]
B(\delta\uu{u}_h,q)&= R_p(\uu{u}_h;q), \\[.1cm]
M(\delta \uu{b}_h,\uu{c})+
D(\uu{c},\delta r_h)-C(\uu{b}_h;\delta \uu{u}_h,\uu{v})&= R_b(\uu{u}_h,\uu{b}_h,r_h;\uu{c}),\\[.1cm]
D(\delta \uu{b}_h,s)&= R_r(\uu{b}_h;s),
\end{split}
\end{equation}
for all $(\uu{v},q,\uu{c},s)\in \uu{V}_h\times Q_h \times \uu{C}_h\times S_h$. Note that this system is linearised around $(\uu{u}_h,\uu{b}_h)$. The right-hand side linear forms correspond to the residual at the current iteration $(\uu{u}_h,p_h,\uu{b}_h,r_h)$ defined by:
\begin{align*}
 R_u(\uu{u}_h,\uu{b}_h,p_h;\uu{v})&=(\uu{f}, \uu{v})_\Omega-A(\uu{u}_h,\uu{v})
-  \tilde{O}(\uu{u}_h;\uu{u}_h,\uu{v}) \\  & \hspace{4.2mm}- C(\uu{b}_h;\uu{v},\uu{b}_h)-B(\uu{v},p_h),\\[.1cm]
R_p(\uu{u}_h;q)&=-B(\uu{u}_h,q),\\[.1cm]
 R_b(\uu{u}_h,\uu{b}_h,r_h;\uu{c})&=(\uu{g,c})_\Omega -M(\uu{b}_h,\uu{c})
+ C(\uu{b}_h;\uu{u}_h,\uu{c})-D(\uu{c},r_h),\\[.1cm]
R_r(\uu{b}_h;s)&=-D(\uu{b}_h,s),
\end{align*}
for all $(\uu{v},q,\uu{c},s)\in \uu{V}_h\times Q_h \times \uu{C}_h\times S_h$.

In \cite{schotzau2004mixed} it is shown that for small data the Picard iteration (P) will converge to the exact solution for any initial guess.

To formulate the variational form of the Picard iteration (P) in matrix form, let $({u},p,{b},r)$ be the coefficient vectors associated with $(\uu{u}_h,p_h,\uu{b}_h,r_h)$ and $(\delta{u},\delta p,\delta{b},\delta r)$ be the coefficient vectors of $(\delta \uu{u}_h,\delta p_h,\delta \uu{b}_h,\delta r_h)$. Then it can readily seen that the Picard iteration (P) amounts to solving the matrix system
\begin{equation}
\label{eq:mhd_saddle}
%\mathcal{K} x \equiv
\left(
\begin{array}{cccc}
A+O(u) & B^T & C(b)^T & 0\\
B & 0 & 0 & 0 \\
-C(b) & 0 & M & D^T\\
0 & 0 & D & 0
\end{array}
\right)
\,
\left(
\begin{array}{c}
\delta u\\
\delta p\\
\delta b\\
\delta r
\end{array}
\right)  =
\begin{pmatrix}
r_u \\
r_p\\
r_b\\
r_r
\end{pmatrix},
\end{equation}
with
\begin{equation} \label{eq:rhsupdate}
\begin{array}{rl}
r_u &= f- Au -O(u) u - C(b)^T b- B^T p,\\
r_p &=-B u,\\
r_b &=g-Mu+C(b)b-D^T r,\\
r_r &=-D b.
\end{array}
\end{equation}
At each non-linear iteration, the right hand side vectors and matrices $O(u)$ and $C(b)$ in \eqref{eq:rhsupdate} and \eqref{eq:mhd_saddle} respectively must be assembled with the solution coefficient vectors $({u},p,{b},r)$ of the current iterate. Here, the matrix $A$  is symmetric positive definite (SPD), $O(u)$ is non-symmetric and $-C(b)$, $C(b)^T$ appear in a skew symmetric fashion. We also note that $M$ is symmetric positive semidefinite (SPSD) with nullity $m_b$ corresponding to the dimension of the scalar space $S_h$ giving rise to the discrete gradients, see \cite{greif2007preconditioners}.

\section{Decoupled iterations}
\label{sec:FEMdecouple}

The full MHD system \eqref{eq:mhd}, \eqref{eq:bc} is a coupled system consisting of the incompressible Navier-Stokes and Maxwell's equations, coupled through the non-linear skew symmetric coupling term $C(b)$. In addition, the convection term $O(u)$ is non-linear as well. These two terms make the numerical solution challenging. Therefore, if one or both of these terms is small then it may be possible to iterate explicitly. In particular if the coupling term, $C(b)$, is small then we may completely decouple the system into an incompressible Navier-Stokes problem and a Maxwell problem. The two resulting decoupling schemes are what we call Magnetic and Complete Decoupling and are both described below. Note that unlike the Picard iteration, there is no small data guarantee that such iterations based on these decoupling schemes will converge; although we see convergence for reasonable values of the non-dimensional parameters.

\subsection{Magnetic Decoupling (MD)}
\label{sec:FEMmd}

Consider first the situation where there is  weak coupling within the system, that is when $C(b)$ is small. Then it may be possible to drop these terms to completely decouple the system into the two subproblems, the incompressibleNavier-Stokes and Maxwell's equations. We will call this approach Magnetic Decoupling (MD).
% For a given solution $(\uu{u}_h,p_h,\uu{b}_h,r_h)$, neglecting the coupling terms in \eqref{eq:picard} results in solving for the updates $(\delta \uu{u}_h,\delta p_h,\delta \uu{b}_h,\delta r_h) \in \uu{V}_h \times Q_h \times \uu{C}_h \times S_h$  such that
% \begin{equation}
% \label{eq:picard_explicit_MD}
% \begin{split}
% A(\delta\uu{u}_h, \uu{v}) +O(\uu{u};\delta\uu{u}_h,\uu{v})+ B(\uu{v}, \delta p_h) & = R_u(\uu{u}_h,\uu{b}_h,p_h;\uu{v})\\[.1cm]
% B(\delta\uu{u}_h,q)&= R_p(\uu{u}_h;q), \\[.1cm]
% M(\delta \uu{b}_h,\uu{c})+
% D(\uu{c},\delta r_h)&= R_b(\uu{u}_h,\uu{b}_h,r_h;\uu{c}),\\[.1cm]
% D(\delta \uu{b}_h,s)&=R_r(\uu{b}_h;s),
% \end{split}
% \end{equation}
% where again $(\uu{v},q,\uu{c},s)\in\uu{V}_h\times Q_h\times\uu{C}_h\times S_h$ and $R_u$, $R_p$, $R_b$ and $R_r$ which are defined in section \ref{sec:nonlinear}. Again, let $({u},p,{b},r)$ be the coefficient vectors of $(\uu{u}_h,p_h,\uu{b}_h,r_h)$ and $(\delta{u},\delta p,\delta{b},\delta r)$ be the coefficient vectors of $(\delta \uu{u}_h,\delta p_h,\delta \uu{b}_h,\delta r_h)$, then this amounts to solving the linear system:
Then the system\eqref{eq:mhd_saddle} simplifies to
\begin{equation}
\label{eq:matrix_MD}
%\mathcal{K} x \equiv
\left(
\begin{array}{cccc}
A+O(u) & B^T & 0 & 0\\
B & 0 & 0 & 0 \\
0 & 0 & M & D^T\\
0 & 0 & D & 0
\end{array}
\right)
\,
\left(
\begin{array}{c}
\delta u\\
\delta p\\
\delta b\\
\delta r
\end{array}
\right)  =
\begin{pmatrix}
r_u \\
r_p\\
r_b\\
r_r
\end{pmatrix},
\end{equation}
with
\begin{align*}
r_u &= f- Au -O(u)u - C(b)^T b- B^T p,\\[0.1cm]
r_p &=-B u,\\[0.1cm]
r_b &=g-Mu+C(b)b-D^T r,\\[0.1cm]
r_r &=-D b.
\end{align*}
We iterate in the same fashion as the Picard iteration with the simpler matrix \eqref{eq:matrix_MD}. From \eqref{eq:matrix_MD} we can see that the system is now completely decoupled. This enable us to  solve each individual subproblem separately and possibly in parallel.

\subsection{Complete Decoupling (CD)}
\label{sec:FEMcd}

For the second decoupling scheme, we again consider there to be weak coupling of the system but we also consider that the fluid equations are diffusion-dominated and hence can exclude the convection terms.
% This is the simplest technique as it removes all non-linear terms. Again, for a given solution $(\uu{u}_h,p_h,\uu{b}_h,r_h)$, removing the coupling and convection terms in \eqref{eq:picard} results in solving for the updates $(\delta \uu{u}_h,\delta p_h,\delta \uu{b}_h,\delta r_h) \in \uu{V}_h \times Q_h \times \uu{C}_h \times S_h$  such that
% \begin{equation}
% \label{eq:picard_explicit_CD}
% \begin{split}
% A_h(\delta\uu{u}_h, \uu{v}) + B(\uu{v}, \delta p_h) & = R_u(\uu{u}_h,\uu{b}_h.p_h;\uu{v})\\[.1cm]
% B(\delta\uu{u}_h,q)&= R_p(\uu{u}_h;q), \\[.1cm]
% M(\delta \uu{b}_h,\uu{c})+
% D(\uu{c},\delta r_h)&= R_b(\uu{u}_h,\uu{b}_h,r_h;\uu{c}),\\[.1cm]
% D(\delta \uu{b}_h,s)&=R_r(\uu{b}_h;s),
% \end{split}
% \end{equation}
% where $(\uu{v},q,\uu{c},s)\in\uu{V}_h\times Q_h\times\uu{C}_h\times S_h$.  Taking $({u},p,{b},r)$ as the coefficient vectors of $(\uu{u}_h,p_h,\uu{b}_h,r_h)$ and $(\delta{u},\delta p,\delta{b},\delta r)$ be the coefficient vectors of $(\delta \uu{u}_h,\delta p_h,\delta \uu{b}_h,\delta r_h)$, then the proposed decoupled linear system is
This amounts to the system
\begin{equation}
\label{eq:matrix_CD}
%\mathcal{K} x \equiv
\left(
\begin{array}{cccc}
A & B^T & 0 & 0\\
B & 0 & 0 & 0 \\
0 & 0 & M & D^T\\
0 & 0 & D & 0
\end{array}
\right)
\,
\left(
\begin{array}{c}
\delta u\\
\delta p\\
\delta b\\
\delta r
\end{array}
\right)  =
\begin{pmatrix}
r_u \\
r_p\\
r_b\\
r_r
\end{pmatrix},
\end{equation}
with
\begin{align*}
r_u &= f- Au -O(u)u - C(b)^T b- B^T p,\\[0.1cm]
r_p &=-B u,\\[0.1cm]
r_b &=g-Mu+C(b)b-D^T r,\\[0.1cm]
r_r &=-D b.
\end{align*}
Again, we perform iterations in the same fashion as the Picard iteration. This is the simplest technique as it removes all non-linear terms in the iteration matrix and hence leaves the linear Stokes problem in the upper $(1,1)$ block matrix.

\section{Inhomogeneous Dirichlet boundary conditions and initial guess}
\label{sec:bcig}

% In this section, we described the formulation of the MHD system \eqref{eq:mhd}, \eqref{eq:homogeneousBC}, that is with homogeneous Dirichlet boundary conditions. In general, the problems we numerically test have inhomogeneous Dirichlet boundary conditions \eqref{eq:bc}.

When considering inhomogeneous Dirichlet boundary conditions as in \eqref{eq:bc}, we still solve \eqref{eq:mhd_saddle}, \eqref{eq:matrix_MD} and \eqref{eq:matrix_CD} for the solution updates with homogeneous Dirichlet boundary conditions. Therefore, in this approach we must incorporate the inhomogeneous Dirichlet boundary conditions only within the initial guess.

% To start the non-linear iteration schemes given in Section \ref{sec:FEMdecouple} we require an initial guess.

To form a suitable initial guess, we solve the decoupled Stokes problem with the inhomogeneous boundary condition (\ref{eq:bc}a):
\begin{equation} \label{eq:StokesInitial}
\left(
\begin{array}{cc}
A & B^T \\
B & 0 \\
\end{array}
\right)
\left(
\begin{array}{c}
u \\
p \\
\end{array}
\right)=\left(
\begin{array}{c}
f \\
0 \\
\end{array}
\right),
\end{equation}
and then the non-symmetric Maxwell problem the with inhomogeneous boundary conditions (\ref{eq:bc}b), (\ref{eq:bc}c):
\begin{equation} \label{eq:MaxwellInitial}
\left(
\begin{array}{cc}
M -C(u) & D^T \\
D & 0 \\
\end{array}
\right)
\left(
\begin{array}{c}
b \\
r \\
\end{array}
\right)=\left(
\begin{array}{c}
g \\
0 \\
\end{array}
\right).
\end{equation}
Here the term $C(u)$ corresponds the coupling term using $u$ (the initial guess for the velocity field). We expect the inclusion of the coupling term to increase the accuracy of the initial guess because additional information of the problem is used.

% When iteratively solving these two systems, the convergence tolerance for the initial guess is important.  Since the updates will have homogeneous boundary conditions then the initial guess needs to incorporate the inhomogeneous boundary conditions. Consider for example the fluid equations: if the discrete Stokes problem is solved approximately to some tolerance, then the we approximately solve the boundary equations of the form:
% $$1\times u_B = u_D,$$
% where $u_B$ is the coefficients of the solution on the boundary and $u_D$ is a sufficiently accurate boundary data interpolation. That is that the initial guess only starts with boundary values of the same order of accuracy as the initial solve.

The inhomogeneous Dirichlet boundary conditions are incorporated in a standard fashion by suitably modifying the matrix system. The outcome of this procedure is that the boundary data interpolation is only performed for the initial guess. Hence, the iterative solves for the initial guess must be run to a sufficient accuracy to ensure the accuracy of the discrete boundary conditions.

% If the subsequent solution updates (which have homogeneous Dirichlet boundary conditions) are solver accurately, the inaccuracy in the boundary values will remain. For similar reasons, low accuracy of the solution updates will lead to perturbation in the boundary conditions and thus in the whole solution.

% Suppose you solve for the initial guesses $(u_0,p_0,b_0,r_0,)$ one would get an error associated with the boundary $(\delta u_B,\delta p_B,\delta b_B,\delta r_B)$ so that
% \begin{equation} \nonumber
%  \left. \begin{aligned}
%         u_0 &= u+\delta u_B, & \quad&
%         p_0 = p+\delta p_B,\\
%         b_0 &= b+\delta b_B,  &\quad &
%         r_0 = r+\delta r_B.\\
%  \end{aligned}
%  \right.
%  \qquad \text{}
% \end{equation}
% When $(u_0,p_0,b_0,r_0,)$ as the initial guess and then solving for the updates (which have homogeneous Dirichlet boundary conditions) there will still be some error associated with the boundaries  which are $(\delta u_B,\delta p_B,\delta b_B,\delta r_B)$. This therefore means that if the Stokes and Maxwell solves are not done accurately enough then when solving for the updates errors will remain within the boundary conditions. Thus the error norms for the full MHD system will be capped by the accuracy of the Stokes and Maxwell solves for the initial guess.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% To start the Picard iterations given in Section \ref{sec:nonlinear} we require an initial guess. To form the initial guess, we solve the decoupled Stokes problem
% $$
% \left(
% \begin{array}{cc}
% A & B^T \\
% B & 0 \\
% \end{array}
% \right)
% \left(
% \begin{array}{c}
% u \\
% p \\
% \end{array}
% \right)=\left(
% \begin{array}{c}
% f \\
% 0 \\
% \end{array}
% \right),
% $$
% then the non-symmetric Maxwell problem
% $$\left(
% \begin{array}{cc}
% M -C & D^T \\
% D & 0 \\
% \end{array}
% \right)
% \left(
% \begin{array}{c}
% b \\
% r \\
% \end{array}
% \right)=\left(
% \begin{array}{c}
% g \\
% 0 \\
% \end{array}
% \right).$$
% Here the term $C$ corresponds the the coupling term using $u$ (the initial guess for the velocity field). We use the coupling term within the non-symmetric Maxwell problem as we have the initial guess for the velocity field and incorporating this into the solve for the magnetic and multiplier fields initial guess will hopefully increase the accuracy of the initial guess.

% When solving these two systems, the convergence tolerance which we use is important. In particular, inaccurate solutions cause problems with non-homogeneous boundary conditions. This is because if the matrix system with the boundary conditions applied is not solve to a sufficient accuracy then there will be errors in the boundary data. When we solved for the updates we enforce homogeneous Dirichlet boundary conditions (i.e., zero boundary conditions) on the velocity ($u$), magnetic ($b$) and multiplier ($r$) fields. This therefore means that if the Stokes and Maxwell solves are not done accurately enough then when solving for the updates errors will remain within the boundary conditions. Thus the error norms for the full MHD system will be capped by the accuracy of the Stokes and Maxwell solves for the initial guess.

\section{Summary}

In this chapter we reviewed a mixed finite element approximation to the full MHD system given in \eqref{eq:mhd} and \eqref{eq:bc}. We followed the mixed approach outlined in \cite{schotzau2004mixed} and expressed the MHD system in the matrix form \eqref{eq:mhd_saddle}. Using the Picard iteration  \eqref{eq:mhd_saddle} we introduced two possible decoupling schemes ((MD) and (CD)) which may be simpler to solve for. The performance of the resulting three non-linear iteration schemes depends on the values of the parameters $\kappa$, $\nu$ and $\nu_m$. The next chapter will discuss possible preconditioning approaches for these systems.

In the sequel, we shall omit the dependence of $O(u)$ and $C(b)$ on $u$ and~$b$, respectively, and simply  write $O$ and $C$.


\chapter{Sample chapter}

\section{Math}

\subsection{Symbols}

\begin{itemize}
  \item Caligraphic letters: $\mathcal{A}$
  \item Mathbb letters: $\mathbb{A}$
  \item Mathfrak letters: $\mathfrak{A}$
  \item Math Sans serif letters: $\mathsf{A}$
  \item Math bold letters: $\mathbf{A}$
  \item Math bold italic letters: $\mathbi{A}$
\end{itemize}

You can alias some commands to easily use these math symbols by \lstinline|\newcommand| or \lstinline|\renewcommand|.

\subsection{Equations}

\begin{equation}
  E^2 = m^2 + p^2\label{eq:mass-energy}
\end{equation}

You can use \lstinline|\cref{}| to automatically setup the cross reference name; instead, you can always use \lstinline|\ref{}| to customize the appearence of the cross reference.

\cref{eq:mass-energy} or Equation (\ref{eq:mass-energy}) gives the mass-energy relationship.

\subsection{Theorem}

\begin{definition}
  LCL is orange juice.
\end{definition}

\begin{proof}
  They are both orange.
\end{proof}

Avaiable theorem environments are listed below:

algorithm, assumption, axiom, conclusion, condition, corollary, definition, example, lemma, proof, property, proposition, remark, theorem.

\section{Figure}

\begin{figure}[H]
  \begin{tikzpicture}
    \draw (0,0) -- (1, 0) -- (1, 1) -- cycle;
  \end{tikzpicture}
  \caption{An example tikz picture with long caption: \blindtext\\\blindtext}
  \label{fig:tikz example}
\end{figure}

An example image is shown in \cref{fig:tikz example} or Figure (\ref{fig:tikz example}).

\section{Table}

\begin{table}[H]
  \caption{Test result on different platforms}
  \label{tab:environment}
  \centering
  \begin{tabular}{lll}
    \toprule
    OS & TeX & Test \\
    \midrule
    Windows 10    & \hologo{TeX}\,Live 2021    & Pass \\
    Windows 10    & \hologo{MiKTeX}            & Pass \\
    Windows 10    & \hologo{TeX}\,Live 2020    & \lstinline|cref| problem  \\
    macOS 10.15   & \hologo{TeX}\,Live 2021    & Pass \\
    Ubuntu 20.04  & \hologo{TeX}\,Live 2021    & Pass \\
    Ubuntu 20.04  & \hologo{MiKTeX}            & Pass \\
    Termux        & \hologo{TeX}\,Live 2021    & Pass \\
    Overleaf      & \hologo{TeX}\,Live 2021    & Pass \\
    \bottomrule
  \end{tabular}
\end{table}

\section{Code}

\subsection{Inline code}
Use \lstinline$\lstinline|<code>|$ to print code snippets. The \lstinline$||$ marks delimit
the code and can be replaced by any character not in the code;
\textit{e.g.}   \lstinline|\lstinline$<code>$| gives the same result.

\subsection{Code environment}
The code to draw the \cref{fig:tikz example} is listed below:
\begin{lstlisting}[caption={\hologo{LaTeX} code for inserting a figure}]
\begin{figure}[htb]
  \begin{tikzpicture}
    \draw (0,0) -- (1, 0) -- (1, 1) -- cycle;
  \end{tikzpicture}
  \caption{An example picture with long caption: \blindtext\\\blindtext}
  \label{fig:tikz example} % this is a comment
\end{figure}
\end{lstlisting}

\section{Manage citations}
\label{chap:bibliography}

Use \lstinline|biber| as \hologo{BibTeX} backend.

\subsection{Add and manage}

Entries are stored in \lstinline|mythesis.bib|. For other sources, modify following commands in \lstinline|mythesis.tex|:
\begin{lstlisting}[language=TeX]
\addbibresource{mythesis.bib}
\end{lstlisting}

\subsection{Citation style}

As described in the sample page from ECE department, the style is set to \lstinline|ieee|. You can modify the style in the \lstinline|hkustthesis.cls| file as you wish.


\section{Algorithms}
\subsection{Mo.cc}
\lstinputlisting[style=cpp]{../codebook/algorithms/Mo.cc}
\section{Dynamic Programming}
\subsection{Convex\_Hull\_Trick.cc}
\lstinputlisting[style=cpp]{../codebook/dp/Convex_Hull_Trick.cc}
\subsection{Convex\_Hull\_Trick\_Dynamic.cc}
\lstinputlisting[style=cpp]{../codebook/dp/Convex_Hull_Trick_Dynamic.cc}
\subsection{Divide\_And\_Conquer.cc}
\lstinputlisting[style=cpp]{../codebook/dp/Divide_And_Conquer.cc}
\subsection{Knuth.cc}
\lstinputlisting[style=cpp]{../codebook/dp/Knuth.cc}
\section{Data Structures}
\subsection{BIT.cc}
\lstinputlisting[style=cpp]{../codebook/datastructures/BIT.cc}
\subsection{BIT\_Range.cc}
\lstinputlisting[style=cpp]{../codebook/datastructures/BIT_Range.cc}
\subsection{Treap.cc}
\lstinputlisting[style=cpp]{../codebook/datastructures/Treap.cc}
\subsection{Treap\_Implicit.cc}
\lstinputlisting[style=cpp]{../codebook/datastructures/Treap_Implicit.cc}
\subsection{Splay.cc}
\lstinputlisting[style=cpp]{../codebook/datastructures/Splay.cc}
\subsection{Splay\_Implicit.cc}
\lstinputlisting[style=cpp]{../codebook/datastructures/Splay_Implicit.cc}
\subsection{Link\_Cut\_Tree.cc}
\lstinputlisting[style=cpp]{../codebook/datastructures/Link_Cut_Tree.cc}
\subsection{Persistent\_Segment\_Tree.cc}
\lstinputlisting[style=cpp]{../codebook/datastructures/Persistent_Segment_Tree.cc}
\section{Geometry}
\subsection{Convex\_Hull.cc}
\lstinputlisting[style=cpp]{../codebook/geometry/Convex_Hull.cc}
\subsection{Delaunay.cc}
\lstinputlisting[style=cpp]{../codebook/geometry/Delaunay.cc}
\section{Graph Theory}
\subsection{Eulerian.cc}
\lstinputlisting[style=cpp]{../codebook/graph/Eulerian.cc}
\subsection{SCC.cc}
\lstinputlisting[style=cpp]{../codebook/graph/SCC.cc}
\subsection{Biconnected\_Components.cc}
\lstinputlisting[style=cpp]{../codebook/graph/Biconnected_Components.cc}
\subsection{Max\_Flow.cc}
\lstinputlisting[style=cpp]{../codebook/graph/Max_Flow.cc}
\subsection{Max\_Flow\_Min\_Cost.cc}
\lstinputlisting[style=cpp]{../codebook/graph/Max_Flow_Min_Cost.cc}
\subsection{Max\_Matching.cc}
\lstinputlisting[style=cpp]{../codebook/graph/Max_Matching.cc}
\subsection{Min\_Cut.cc}
\lstinputlisting[style=cpp]{../codebook/graph/Min_Cut.cc}
\subsection{LCA.cc}
\lstinputlisting[style=cpp]{../codebook/graph/LCA.cc}
\subsection{HLD.cc}
\lstinputlisting[style=cpp]{../codebook/graph/HLD.cc}
\section{Mathematics}
\subsection{General.cc}
\lstinputlisting[style=cpp]{../codebook/math/General.cc}
\subsection{Miller\_Rabin.cc}
\lstinputlisting[style=cpp]{../codebook/math/Miller_Rabin.cc}
\subsection{Euclid.cc}
\lstinputlisting[style=cpp]{../codebook/math/Euclid.cc}
\subsection{Combinatorics.cc}
\lstinputlisting[style=cpp]{../codebook/math/Combinatorics.cc}
\subsection{Gauss\_Jordon.cc}
\lstinputlisting[style=cpp]{../codebook/math/Gauss_Jordon.cc}
\subsection{Matrix.cc}
\lstinputlisting[style=cpp]{../codebook/math/Matrix.cc}
\subsection{FFT.cc}
\lstinputlisting[style=cpp]{../codebook/math/FFT.cc}
\section{String}
\subsection{Manacher's.cc}
\lstinputlisting[style=cpp]{../codebook/string/Manacher's.cc}
\subsection{KMP.cc}
\lstinputlisting[style=cpp]{../codebook/string/KMP.cc}
\subsection{Rabin\_Karp.cc}
\lstinputlisting[style=cpp]{../codebook/string/Rabin_Karp.cc}
\subsection{Z\_Algorithm.cc}
\lstinputlisting[style=cpp]{../codebook/string/Z_Algorithm.cc}
\subsection{Suffix\_Array.cc}
\lstinputlisting[style=cpp]{../codebook/string/Suffix_Array.cc}
\subsection{Suffix\_Tree.cc}
\lstinputlisting[style=cpp]{../codebook/string/Suffix_Tree.cc}
\section{Misc}
\subsection{.vimrc}
\lstinputlisting[style=txt]{../codebook/misc/.vimrc}


\chapter{Neural Networks}
This chapter will focus on neural networks and how they were utilized for parts of this
report. First, some theoretical background surrounding neural networks in general, and then
more specifically, convolutional neural networks, is given in sections \ref{sec:NNwork}
and \ref{sec:CNN}.
Afterwards, the chapter goes more into how neural networks were tried and used during this project. Data collection and augmentation is described in sections \ref{sec:NNdata} and \ref{sec:NNaugment}. Further sections, \ref{sec:NNclassification} \& \ref{sec:NNtransfer}, describe how this data was used to train neural networks, with the latter section explaining an alternative to training a model from scratch.
\section{How neural networks work}
\label{sec:NNwork}
There exists many different machine learning techniques out there today. Due to its high
effectiveness and relevance, for this report we are going to focus on the highly popular
method of artificial neural networks.
A variant, convolutional neural networks, is a proven method for working well with
images and is therefore highly relevant for this project.

\subsection{Artificial neuron}
An artificial neuron is the simplest form of the neural network. It has a set of inputs and an output.
The artificial neuron first sums up all the input values, $x$, multiplied with the weight value, $w$.
After that it passes that sum through an activation function. This activation function can be everything from a simple $f(x)=x$ to the more complex sigmoid function, depending on the need. More on this in section \ref{subsec:activationfunctions}.

\begin{figure}[hbtp]
\begin{center}
\includegraphics[width = 0.75\textwidth]{./Images/perceptron.jpg}
\caption{An illustration of an artificial neuron, the simplest version of a neural network.}
\end{center}
\end{figure}

A bias also exists in every node which is not based on any input. The bias function in the artificial neuron is similar to what the $m$ in $y = kx + m$ does. It gives the function the ability to move up and down in the graph for more possibilities of splitting the data set. The bias is usually disregarded when illustrating the artificial neuron.

The artificial neuron only has the ability to draw a single line and thus is only able to split simple data sets.

The output of an artificial neuron is described by the following formula:

\[ y = b + \displaystyle\sum_{i=1}^{n} x_i \cdot w_i \]

\[y=f(b+\sum_{i=1}^n x_i\cdot w_i)\]

where $b$ is the bias value, $x$ is the input, $w$ is the weight for that input, $n$ is the number of inputs and $f(*)$ is the activation function.

\subsection{Activation functions}
\label{subsec:activationfunctions}
The activation function,  $ \phi (v_{i}) $ , takes the sum of all the inputs from a node as input and passes them through a function before giving an output. This is beneficial when for instance the output should be kept in a range between 0 and 1, or perhaps when negative values don't make sense.
Usually, activation functions are attributed to layers instead of individual nodes.

A few commonly used functions are \textbf{ReLu}, \textbf{Tanh}, \textbf{Sigmoid} and \textbf{Softmax}.

\textbf{ReLu} is described as
\[f(x) = max(0, x)\]
and is a good option when negative values should be ignored or don't make sense.
It is also a good choice avoiding the net to become computationally heavy,
since it only outputs the same input value but sets all negative values to zero.
That is usually a good choice in larger networks with many neurons which can become
slow.

\textbf{Tanh} is the tangens hyperbolicus function and is used to output values either as -1 or 1 like a binary operator. Unlike a binary operator, tanh's derivative is always defined which makes back propagation possible.
Back propagation is described in section \ref{subSec:optimizers}

\textbf{Sigmoid} is described as
\[\frac{e^x}{1+e^x}\]
and works like the Tanh function but keeps the values between 0 and 1 and sets y(0) = 0.5 instead.

\textbf{Softmax} is a little bit more complex. It takes a vector $v$ of dimension $n$ and turns it into a vector $\sigma(v)$ of the same dimension where $\displaystyle\sum_{i=1}^{n} \sigma_i(v) = 1 $ and each element value is between zero and one.

Each element in the array is described as below
\[ \sigma_i(v) = \frac{e^{v_i}}{\displaystyle\sum_{j=1}^{n} e^{v_k}} \]

This output is good for describing the probability for each element to be correct and is therefore commonly used
in the output layer.

\subsection{Calculating the loss}
When training the model, it will at first make a guess to what the right answer or value is based on the random initial weigh values. In the beginning, the model is usually mostly wrong and it is then important to know how wrong it was and in what direction it should go.

For this, the model uses a loss function to calculate that error. For different applications the loss function can be different.
One way to calculate the error, $E$, is to simply take the predicted value, $y_p$, and subtract it with the correct value, $y_c$.

\[E = y_p - y_c\]

Sometimes the sign of the error is not important. Then the absolute value can
be calculated instead. $E = |y_p - y_c| $

Another approach is the \textbf{mean squared error} which squares the difference and takes the mean value over a few predictions. $n$ is the number of predictions.
\[E(n) = \frac{1}{n} \cdot \displaystyle\sum_{}^{n} (y_p - y_c)^{2} \]

\subsection{Optimizers}
\label{subSec:optimizers}
Once the error has been calculated, the network will make changes to itself to improve the performance by decreasing the error value. This is called backpropagation.
The principle behind backpropagation is to go backwards in the network from the output node and changing the weight values, $ \omega $.
The optimizers decide how the weights should change. \\

A popular method for minimizing the error is to use \textbf{Gradient Decent} which works
by, step by step, move in the negative direction that minimize the error.
The direction is determined by taking the derivative of the error function.
The new weight value is calculated accordingly. $\eta$ is a chosen parameter called the learning rate.

\[ \Delta \omega_{ik} = -\eta \frac{\delta E}{\delta \omega_{ik}} \]

\begin{center}
where the error function is
\end{center}

\[E = \frac{1}{N} \displaystyle\sum_{i=1}^{N} E(n) \]

\begin{center}
where N is the total number of neurons. This leads to
\end{center}

\[ \hat{\Delta\omega_{ik}} = \frac{1}{N}  \displaystyle\sum_{i=1}^{N} \Delta\omega_{ik}(n) \]

$ E(n) $ can be, for example, mean squared error mentioned above.
The most common form of gradient decent is called \textbf{Stochastic gradient decent}, SGD.
The difference is that SGD only evaluates on a small number of nodes/patterns, $P$ and updates all the weights from that observation.

\[ \hat{\Delta\omega_{ik}} = \frac{1}{P}  \displaystyle\sum_{i=1}^{P} \Delta\omega_{ik}(p) \]

\textbf{Adam}\cite{adamoptimizer}, short for Adaptive moment estimation, is another optimizer which is quite popular.
The reason is because it has been proven to be a very effective algorithm in many different use cases.

Adam is a kind of combination of using \textbf{RMSPROP}\cite{rmsprop} and SGD with momentum.
Without going into too much detail, SGD with momentum keeps track of which direction the the improvement is in and keeps the improvement going in that same way. The optimisation goes faster when the previous direction is the same and slower when they are different. Almost like a rolling ball that gains speed as it is rolling down hill.
RMSPROP does something similar and uses a running average of each weight value and the previous values importance can be controlled with a parameter. It also only uses the sign of the direction and not its value. Also RMSPROP has individual weights.

Adam keeps a running average of both the past gradients and the squared past gradients.
The full mathematical description of Adam is given by the formula below where
$t$ is a time iteration index.

\[ \omega_i (t+1) = \omega_i(t) - \eta \frac{m_i}{\sqrt{v_i} + \epsilon} \]

where

\[ m_i (t+1) = \beta_1m_i(t) + (1 - \beta_1) \frac{ \delta E(t) }{\delta\omega_i} \]

\[ v_i (t+1) = \beta_2v_i(t) + (1 - \beta_2) (\frac{ \delta E(t) }{\delta\omega_i})^2 \]

and $ \eta, \beta_1 $ and $ \beta_2 $ are adjustable parameters and $ \epsilon $ a small value to keep the equation from dividing by zero.\\

There are a lot of other optimizers such as \textbf{Adagrad}, \textbf{Adadelta},
\textbf{Nadam} \cite{optimizers}. They all have their own advantage and use-cases.
We won't go into further details about them.

\subsection{Layers}

\begin{figure}[hbtp]
\begin{center}
\includegraphics[width = 1.0\textwidth]{./Images/fully_connected.jpg}
\caption{Two fully connected layers. One with 3 neurons and one with 2 neurons.}
\label{fig:layers}
\end{center}
\end{figure}

With more layers and more neurons the networks parameters and complexity begins to grow. So does also the training time, size of the model and the cost for doing predictions.
Thus, these networks are capable of describing much more complex data sets. But as a result of that, the risk of overfitting, the act of describing the training data set too well so that new data sets will not get recognized, becomes much greater.
That is why a complex network is not always wanted.
In figure \ref{fig:layers} an example is given of how a network with many layers might look.

\subsection{Overfitting}
\label{overfitting}
When training a neural net, one needs to be careful not to train the model too much or overfitting is likely to happen.
Overfitting is when a model gets really good at predicting the data that it is training on but fails to predict accurately on new data.
This is because it starts to pick up too much on detail or in some instances even noise.
For this reason it fails to pick up the general trends, which is more valuable.
An illustration is given i figure \ref{fig:overfitting}.

\begin{figure}[hbtp]
\begin{center}
\includegraphics[width = 1.0\textwidth]{./Images/overfitting.jpg}
\caption{Overfitting of a dataset. On the left is a generalized trained model and on the right is an overtrained model. Image taken from OReilly.com \cite{overfitting}.}
\label{fig:overfitting}
\end{center}
\end{figure}

There are a few methods that can help handle the problem of overfitting. They usually involve trying to limit the size of a small amount of weights.
The hypothesis behind this is that if a weight is much larger than the rest it also has much more influence over the final prediction. That way, a small detail in the data can have much more influence than the general trend.

One way to do this is to cut random connections between layers during epochs, usually by specifying a certain amount that is going to be cut.
This way, the model is not relying on a small number of nodes to make the correct predictions. This method is called \textbf{dropout} \cite{dropout}.

Another way is to introduce random noise on a layer during training. This works because if a node with a large weight receives noise, it will be heavily amplified and probably give a false prediction.
When doing this, Gaussian noise is usually implemented, which is basically random noise with a gaussian distribution.

To force the model to keep weights small, one way is to add a penalty to the loss for every weight based on its size.
This is called weight regularisation and is widely used and exists in two forms, L1 and L2.
L1 simply adds the weights size multiplied with an L1 term that is chosen by the designer.
The other, L2, is to do the same but in this case square that value. This has the effect of making values over 1 even bigger and values less than 1 smaller.
In that way, it doesn't affect the loss as much as L1 when not being overtrained. L2 is also called weight decay and is the more common method of the two.

Another way to keep the values small is to normalize the input to every layer and set the mean to 0 and variance to 1. In Tensorflow, this can be done with the BatchNormalization layer \cite{batchnormalization}.

One important thing to point out is that all these methods mentioned so far are only active during training and is not doing anything when making real predictions.

If lots of training data exists, the ensemble technique could be a good way to go. This technique divides the training data into smaller sets and trains a model for every data set.
The final prediction is then an average of the predictions of all the models. This technique works because if the model is highly overtrained in a certain direction, chances are that the other networks will drown out this result by the many other models.

It is kind of like when someone has an off pitch in a big choir. If there are many other singers, this off-pitch will not be noticed very much.

The final technique is called early stopping and is based on validating the model after every
epoch and stopping the training when the validation loss, or some other criteria, is not
improving anymore. A graph showing when to do early stopping is given in figure \ref{fig:earlystopping}.

\begin{figure}[hbtp]
\begin{center}
\includegraphics[width = 0.75\textwidth]{./Images/early_stop.jpg}
\caption{A graph showing when to do an early stop. The graph shows the total loss over trained epochs. Notice that the validation starts to increase again somewhere after epoch 20.}
\label{fig:earlystopping}
\end{center}
\end{figure}

 \section{Convolutional Neural Networks}
 \label{sec:CNN}
Convolutional neural networks, CNN, have been used for quite some time when it comes to deep learning and their capabilities are rather astounding, as proved by Krizhevsky et al \cite{NIPS2012_4824}.  They use layers which perform convolutions, hence it's name.  The input to a convolutional network are either 2D or 3D tensors, where the 3D alternative usually has color channels along the third dimension. Use of convolutional neural networks in image recognition  can be practical for several reasons, e.g., it can tell of spatial relationships in an image. This section has been written with reference to V. Dumolin et al \cite{convArit}. Figure \ref{fig:cnn} shows an example of how a CNN may look.

 \begin{figure}[hbtp]
\begin{center}
\includegraphics[width = 0.9\textwidth]{./Images/convNetwork.jpg}
\caption{Image shows the flow of a convolutional neural network from left to right. Several convolutions as well as pooling functions are done in order on the input image. The last feature map is then flattened to fit into the hidden layers, to then be classified with a softmax function. Image taken from \cite{cnnImage}.}
\label{fig:cnn}
\end{center}
\end{figure}

A convolutional layer performs the mathematical operation \textit{convolution} on the input tensor, which can be presented as an image. A convolutional filter, or kernel, of size $k \cdot l$ slides across the input. The kernel consist of weights in each element. These weights are found during the training process of the network. As the kernel is slides across the image, the dot product of the weights of the kernel and the pixels the the kernel is above, is calculated as the output. Afterwards, a new image containing all of the produced dot products is formed.

Padding on the input can be used to account for values when the kernel is outside the input. Use of padding have an impact on the output size after the convolutional layer. Stride can also be used, which tells how much the kernel translates along an axis;  increased stride leads to subsampling. Since changes of parameters in one axis do not affect outcome in another axis it simplifies explaining CNNs by having the parameter values being the same along both axes. The more convolutional layers used in a network, the more complex shapes are detected. First layers may detect edges and corners, whereas, later layers may find features representing, for example, a car or a dog. A common addition to the layers is an activation function, commonly ReLU (which is described in section \ref{subsec:activationfunctions}). This gives faster training by only allowing for positive weights to pass the layer.

 The common types of padding is \textit{no zero}, \textit{same} and \textit{full}; examples seen in figure \ref{fig:padding}. No Zero padding involves having no padding outside of the input. This means that the kernel never goes outside of the actual image, once a side of the kernel hits the side of the input it jumps down to the next line (if stride is 1).

 Same padding is used when one desires the output size of the layer to be the same as the input size. This is achieved by having a padding $p$ be $ p = \floor{\frac{k}{2}} $ for any odd kernel size $k$.

 With full padding one actually makes the output size larger than the input, by fully utilizing every possible combination of the kernel and the input image. This is accomplished  by having the padding $p $ be $p = k - 1$ for any kernel size $k$.

 \begin{figure}[hbtp]
\begin{center}
\includegraphics[width = 0.3\textwidth]{./Images/noPad.png}
\includegraphics[width = 0.3\textwidth]{./Images/samePad.png}
\includegraphics[width = 0.3\textwidth]{./Images/fullPad.png}
\caption{Different examples of padding. Grid in green is the output, grid in blue is the input, and area in shadow is the current area where the kernel is currently at. Leftmost image shows a no zero padding being used. Here the kernel size is $k = 3$, input size is $i = 4$ and the output size is  $o = 2$, i.e., smaller than input.  In the middle image same padding is used; output size is the same as input size $o = i = 5$. In the rightmost image full padding is used. Here a bigger output size than the input size is produced, $o = 7, i = 5$. Images taken from \cite{convArit}}
\label{fig:padding}
\end{center}
\end{figure}

 Another useful feature used in a lot of convolutional neural networks is pooling. Pooling layers are somewhat similar to convolutional layers in that they use a sliding window which performs an operation on the contents of the window and outputs a new value. However, the difference is that they use other functions instead of linear addition. Two common pooling functions are \textit{max pooling}, where the output is the largest value within the window, and \textit{average pooling}, where the output is the average of the components within the window.  Pooling is commonly done do reduce the size of the input. Figure \ref{fig:pooling} shows example of pooling being used.

 \begin{figure}[hbtp]
\begin{center}
\includegraphics[width = 0.45\textwidth]{./Images/avgPool.png}
\includegraphics[width = 0.45\textwidth]{./Images/maxPool.png}
\caption{Examples of two types of pooling. Blue grid represents input, green grid represents the output, and the shaded area is where the sliding window is located. The left image shows the use of average pooling, while the right one shows the use of max pooling.
Images taken from \cite{convArit}}
\label{fig:pooling}
\end{center}
\end{figure}

When a classification is to be done in the CNN, the shape needs to be shifted to an array. This is done after the last pooling layer or convolutional layer. This array can then be passed into a hidden layer for classification.

\section{Collecting data}
\label{sec:NNdata}
When working with machine learning, big sets of data is often required for a good resulting model.
A problem with this is that it can be tricky to obtain such a large data set because it usually also requires
labels with that data to be manually put in. These labels will be used by the neural network to, during training, check if
the predicted value is correct or not.

In this project the data is sets of images of different furniture parts. These images did not exist anywhere online, so they had to be collected by taking lots of photos. The photos contained the part in different background and from different angles. Table \ref{table:imageclassnumbers} shows how many images of each class we had. The backgrounds were mostly of typical office surroundings, although there were exceptions (example is shown in figure \ref{fig:exampleCutout}). All the photos were then resized to 256x256 pixels.
The reason for choosing a square size is because when rotating them
(this will be useful later on when augmenting data in section \ref{sec:NNaugment})
the images will not have any black bars on the sides nor be stretched.

\begin{table}[h]
\centering
\begin{tabular}{ |c|c| }
 \hline
 Class &  Number of images  \\
 \hline
 Leg piece & 203 \\
 \hline
 Bridge piece & 210 \\
  \hline
 Seat & 216 \\
 \hline
\end{tabular}
\caption{Table showing how many images of each class we had.}
\label{table:imageclassnumbers}
\end{table}

For the first furniture Nolmyra, there were only 3 unique parts (excluding screws and similar parts). Therefore the model was trained to recognize 4 things; the different parts and an unknown object. The unknown label was because the object detection algorithm could detect other objects and it is then unwanted that those objects be mistaken for furniture parts. This could of course also be done by setting a threshold on the confidence value, i.e., if the model gives a confidence value under 0.8 it discards it as being an unknown object. The problem with this type is that it is much harder to distinguish parts from unknown objects. This is due to that the best model is the one that can give a close to 100\% confidence value on every prediction.

For this reason, photos of random objects were also added to the data set. Most of these photos were taken by ourselves, but some were also collected from the web. The images were placed in folders with a specific item which meant labelling them became much simpler. Just put all the images containing a certain part in the same folder.

Training a model usually requires hundreds of thousands or even millions of photos. That much data is hard to get and would take a lot of time to obtain Going around the office to snap that many photos is almost unthinkable.
However, there are other options which will be explained in the next section.

\begin{figure}[hbtp]
\begin{center}
\includegraphics[width = 0.3\textwidth]{./Images/image_31.jpg}
\includegraphics[width = 0.3\textwidth]{./Images/image_133.jpg}
\includegraphics[width = 0.3\textwidth]{./Images/image_136.jpg}
\caption{Images showing different samples of training data used for training the model. From left to right: seat, bridge piece, leg piece.}
\label{fig:exampleData}
\end{center}
\end{figure}

\section{Augmenting data}
\label{sec:NNaugment}
Instead of training on only original images, some images can be created from other images by for example rotating, flipping, changing brightness, saturation, contrast etc.  This will essentially be an image of the same object, but the data will look different, thus giving the model more relevant data to train on, see figure \ref{fig:exampleArtificial}.
When doing this it is important to keep in mind that the augmented data should be relevant to real situations.
Creating data with only the blue color band when the real situations are only in daylight makes no sense.

\begin{figure}[hbtp]
\begin{center}
\includegraphics[width = 0.3\textwidth]{./Images/image_15-1.jpg}
\includegraphics[width = 0.3\textwidth]{./Images/image_15-3.jpg}
\includegraphics[width = 0.3\textwidth]{./Images/image_15-4.jpg}
\includegraphics[width = 0.3\textwidth]{./Images/image_15-9.jpg}
\includegraphics[width = 0.3\textwidth]{./Images/image_15-11.jpg}
\includegraphics[width = 0.3\textwidth]{./Images/image_15-12.jpg}
\caption{Images showing some resulting images from augmentation. Top left image shows the original image. Top middle image shows the result after rotating $180$ degrees. Top right image shows result after rotating $-90$ degrees. Bottom left image show result after  reducing the contrast with by $30\%$. Bottom middle image shows result after reducing color balance by $70\%$. Bottom right image shows result after increasing color balance by $60\%$.}
\label{fig:exampleArtificial}
\end{center}
\end{figure}

Furthermore, images of objects of interest can be cut out and pasted into random environments to create even more data, see figure \ref{fig:exampleCutout}.
The idea of this is to try and make the model understand that the focus should be on the furniture part and the background environment is irrelevant. While doing this, even though the parts could be cut into totally random environments we tried to focus on pasting them into relevant spaces like office floors or carpets.
In our project, this gave a good result as it increased the test accuracy by 8\%.

\begin{figure}[hbtp]
\begin{center}
\includegraphics[width = 0.3\textwidth]{./Images/image_13.jpg}
\includegraphics[width = 0.3\textwidth]{./Images/image_86.jpg}
\includegraphics[width = 0.3\textwidth]{./Images/image_107.jpg}
\caption{Images showing how cutouts of images were utilized. Leftmost image shows a cutout of one of the furniture parts. The other two images show the result after they were pasted in to different environments to artificially increase training data.}
\label{fig:exampleCutout}
\end{center}
\end{figure}

Doing this by hand can still be very time consuming, so automating this process as far as possible is recommended.

\section{Image classification}
\label{sec:NNclassification}
Designing a neural network for the image classification is not the easiest task and much
work consists of trying things and making intelligent guesses.
As stated in the section about data collection we had 4 different classes to classify,
which means that a test accuracy of over 25\% would be significant. We were aiming for
somewhere above or around 90\% with a relatively small amount of data.

We started out with many convolutional layers layered with pooling layers and a few dense
layers at the end. The final layer had four nodes and softmax activation functions to give a
classification probability between four classes.
The activation functions of the other layers were set to ReLu since they are the fastest
and preferred for larger nets. Tanh was also tried but with no success.

SGD with momentum as an optimizer was tested but was changed to Adam
quite early on since it had better performance.
Only linear optimizers were considered because non-linear optimizers take a lot of
computing power and are usually only recommended for small networks.

We went back and forth with trying different mixes of kernel sizes, amounts of convolutional
layers, insertion of pooling layers on different places, stride sizes, amount of dense layers
and how many nodes they had.

When training the models, overfitting was a big problem from the start. The tested methods
for combating this was dropouts with different values, L2 weight regularisation,
batch normalization and reducing the number of weights.

It was hard to reduce the number of weights as the evaluation usually converged around 25\%
whenever it was tried.
When using the dropout method a value of 50\% with a minimum of 64 nodes in the layer
reduced overfitting most effectively. That, with a combination of weight regularisation and batch
normalization gave good results. Figure \ref{fig:imageclassification} shows how overfitting was avoided.

Gaussian Noise gave very good results, but was unfortunately not possible to be
converted from the model format generated in keras, to the mlmodel format that is required for Xcode.

The most important thing to mention is the difference the amount of photos we had and how they
were taken. Adding more photos made a huge difference to the test evaluation score.
After having added 50 more images per class with a library of 600 images the test score increased
from \textbf{80.55\%} to \textbf{83.45\%} in accuracy. \\

When finally testing the model we made a mistake by not separating the training data and test data.
Since we shuffled our data before splitting it up into two parts, some augmented data got into the
test set. The model had already trained on the original image and it was similar enough to
give unrealistic good results of 99.52\%. This was finally fixed by splitting up the test and train data
into separate folders. Figure \ref{fig:augmentedtestdata} shows the printed graph from one of those
training sessions.

\begin{figure}[!hbtp]
\begin{center}
\includegraphics[width = 0.5\textwidth]{./Images/augmentedtestdata}
\caption{Test and train accuracy over a 100 epochs. The performance seems much better than it actually is.}
\label{fig:augmentedtestdata}
\end{center}
\end{figure}

\begin{figure}[!hbtp]
\begin{center}
\includegraphics[width = 0.45\textwidth]{./Images/imageclassification1}
\includegraphics[width = 0.45\textwidth]{./Images/imageclassification2}
\includegraphics[width = 0.45\textwidth]{./Images/imageclassification3}
\caption{Images shows the result from introducing weight regularisation and dropout to a model. The graphs show the training loss (green) and evaluation loss (red). The upper left image shows the training without any dropout or weight regularisation. The upper right image shows the training with dropout implemented. The bottom image shows the training with dropout and weight regularisation implemented.}
\label{fig:imageclassification}
\end{center}
\end{figure}

As shown in the code in appendix B.1., early stopping was also finally implemented with a callback to save the
highest performing model at the end of the process.

The network that proved to be the best performing network (\textbf{92.25\%} accuracy) had the configuration shown in appendix B.2.

A model with many convolutional layers and a few dense layers at the end gave the best
results. A larger network could have been implemented and tests as well but
since the model size would grow more and training time would increase drastically,
we didn't increase it further.

\section{Transfer Learning}
\label{sec:NNtransfer}
% Write about how we went about in order to insert Transfer Learning into the mix.
After trial of designing the network from scratch, it was decided to try and make use of
pre-trained networks and then retrain them for another purpose. Models trained on
imageNet \cite{imageNet} were chosen since that source domain is similar to this target
domain. The models with their respective weights were loaded, without their fully
connected layer. The top layer's weights were then frozen at different stages, and new
fully connected layers were added on top, and new classifiers were trained.
Several different models were tried, including ResNet50, InceptionV3 and VGG16,
which where all integrated in Keras to start with.

The code shown in appendix B.3. shows how to load the VGG16 network which has weights that has been pretrained on imageNet. The fully connected layer is then removed and instead a custom top layer is added to be trained.

The best result  that was achieved for each of the models using this method is listed in
 table \ref{table:transferLearning}. It shows that using InceptionV3 model with pretrained
 weights gave the best result. However, a few factors, such as, where the models were
 frozen, how adding and removing layers, has most likely affected the results. As can be
 seen from using ResNet, the best model achieved 25\% accuracy, which is the same as making a guess, seeing as there are only four possible classes. These models could, and would most like have been, improved if it was not decided to take a new approach to the problem, which is described further in section \ref{sec:ODresults}.

\begin{table}[h]
\centering
\begin{tabular}{ |c|c| }
 \hline
 Pretrained model used &  Best achieved accuracy  \\
 \hline
 VGG16 & 70.8\%. \\
 \hline
 ResNet & 25.0\% \\
  \hline
 InceptionV3 & 72.9\% \\
 \hline
\end{tabular}
\caption{Results from doing transfer learning .}
\label{table:transferLearning}
\end{table}

\newpage