\section{DT LTI -- state space}

\subsection{}

\begin{frame}
\frametitleTC{The DT LTI class}
\framesubtitleTC{Preliminaries}
\begin{itemize}[<+-| alert@+>]
\item For the LTI class, that we see here almost exclusively in the DT context, there exists a very strong theory.
\item The same is not true for more general (e.g., nonlinear) system classes.
\item Therefore, control problems of the type addressed here, are cast in the LTI framework wherever possible.
\item \vfill This motivates the importance of the DT LTI class, which we now\\
      come to examine.
\end{itemize}
\end{frame}

\begin{frame}
\frametitleTC{State-space representation of a DT LTI system}
\framesubtitleTC{Definition --- we stick to the SISO case for simplicity}
\myPause
\begin{itemize}[<+-| alert@+>]
\item The \TC{State-Space (SS)} representation of a DT LTI dynamic system is the quadruplet $(A,b,c,d)$ in
      \begin{displaymath}
       \left\{
        \begin{array}{rlll}
         x(k) &= A x(k-1) + b u(k-1) & & \text{\TC{(state equation)}}  \\
         y(k) &= c x(k)   + d u(k)   & & \text{\TC{(output equation)}}
        \end{array}
       \right.
      \end{displaymath}
\item $u(k)$ and $y(k)$ are real scalars;
\item $x(k) \in \Re^n$, where $n$ is the system's order;
\item the real \TC{dynamic matrix} $A$ is $n \times n$;
\item $b$ is a real column vector ($n \times 1$);
\item $c$ is a real row vector ($1 \times n$);
\item $d$ is a real scalar.
\end{itemize}
\end{frame}

\begin{frame}
\frametitleTC{Motion}
\framesubtitleTC{}
\myPause
\begin{itemize}[<+-| alert@+>]
\item Given $x(0)$ and $u(k)$, $k \geq 0$, we get
      {\small
      \begin{displaymath}
       \begin{array}{rclcl}
        x(1) &=& Ax(0)+bu(0) \\
        x(2) &=& Ax(1)+bu(1) &=& A^2x(0)+Abu(0)+bu(1) \\
        x(3) &=& Ax(2)+bu(2) &=& A^3x(0)+A^2bu(0)+Abu(1)+bu(2) \\
             & & \cdots\\
       \end{array}
      \end{displaymath}
      }
\item This readily generalises to the \TC{Lagrange state formula}
      \begin{displaymath}
       x(k) = A^k x(0) +\sum\limits_{h=0}^{k-1} A^{k-h-1}bu(h).
      \end{displaymath}
\end{itemize}
\end{frame}

\begin{frame}
\frametitleTC{Free and induced state motion}
\framesubtitleTC{}
\myPause
\begin{itemize}[<+-| alert@+>]
\item In LTI systems, the state motion $x(k)$ is the \TC{sum} of a \TC{free motion} $x_F(k)$ and an
      \TC{induced motion} $x_I(k)$, i.e.,
      \begin{displaymath}
       x(k) = x_F(k) + x_I(k),
      \end{displaymath} \myPause
      where
      \begin{displaymath}
       x_F(k) = A^k x(0), \quad
       x_I(k) = \sum\limits_{h=0}^{k-1} A^{k-h-1}bu(h).
      \end{displaymath} \myPause
\item $x_F(k)$ depends linearly on $x(0)$ and not on $u$,
\item while $x_I(k)$ depends linearly only on $u(k)$ and not on $x(0)$.
\end{itemize}
\end{frame}

\begin{frame}
\frametitleTC{Free and induced output motion}
\framesubtitleTC{}
\myPause
\begin{itemize}[<+-| alert@+>]
\item In LTI systems, also the state motion $y(k)$ is the \TC{sum} of a \TC{free motion} $y_F(k)$ and an
      \TC{induced motion} $y_I(k)$, i.e.,
      \begin{displaymath}
       y(k) = y_F(k) + y_I(k),
      \end{displaymath} \myPause
      where
      \begin{displaymath}
       y_F(k) = cA^k x(0), \quad
       y_I(k) = c\sum\limits_{h=0}^{k-1} A^{k-h-1}bu(h) +du(k).
      \end{displaymath} \myPause
\item again, $y_F(k)$ depends linearly on $x(0)$ and not on $u$,
\item while $y_I(k)$ depends linearly only on $u(k)$ and not on $x(0)$.
\end{itemize}
\end{frame}

\begin{frame}
\frametitleTC{Equilibrium}
\framesubtitleTC{}
\myPause
\begin{itemize}[<+-| alert@+>]
\item For an LTI system, equilibrium states are found by solving
      \begin{displaymath}
       \overline{x} = A\overline{x}+b\overline{u}.
      \end{displaymath}
\item Thus, if $A$ has no unity eigenvalues, there exists the one equilibrium
      \begin{displaymath}
       \overline{x} = (I-A)^{-1}b\overline{u},
      \end{displaymath}
\item while in the opposite case, either there is no equilibrium, or there\\
      are infinite ones.
\end{itemize}
\end{frame}

\begin{frame}
\frametitleTC{Equilibrium}
\framesubtitleTC{Peculiarities of L(TI) systems}
\myPause
\begin{itemize}[<+-| alert@+>]
\item Contrary to nonlinear systems, they cannot have a finite number of equilibria, different from zero and one.
\item If some $\overline{u}$ produces zero, one or infinite equilibria, the same is true for any other $\overline{u}$,
      contrary again to the nonlinear case.
\item For each equilibrium state, there surely exists the one equilibrium output
      \begin{displaymath}
       \overline{y}=c\overline{x}+d\overline{u}.
      \end{displaymath}
\end{itemize}
\end{frame}

\begin{frame}
\frametitleTC{Stability of an equilibrium}
\framesubtitleTC{}
\myPause
\begin{itemize}[<+-| alert@+>]
\item Let $(\overline{x},\overline{u})$ be an equilibrium for an LTI system.
\item The Lagrange state formula leads to write
      \begin{displaymath}
       \overline{x} = A^k \overline{x} +\sum\limits_{h=0}^{k-1} A^{k-h-1}b\overline{u}.
      \end{displaymath}.
\item Consider now the  perturbed motion $x_{\Delta}(k)$ produced by $\overline{u}$ and
      $x(0)=\overline{x}+\Delta\overline{x}$; the same formula yields
      \begin{displaymath}
       x_{\Delta}(k) = A^k (\overline{x}+\Delta\overline{x}) +\sum\limits_{h=0}^{k-1} A^{k-h-1}b\overline{u}.
      \end{displaymath}
\item Subtracting, therefore,
      \begin{displaymath}
       x_{\Delta}(k)-\overline{x} = A^k\Delta\overline{x}
      \end{displaymath}
\end{itemize}
\end{frame}

\begin{frame}
\frametitleTC{Stability of an equilibrium}
\framesubtitleTC{and in the L(TI) case, of a system}
\myPause
\begin{itemize}[<+-| alert@+>]
\item The way $x_{\Delta}(k)$ moves with respect to $\overline{x}$ does not depend on $\overline{x}$.
\item That is, contrary the nonlinear case, there cannot be equilibria with different stability characteristics.
\item In the L(TI) class, stability is a property of the \emph{system}, not of the individual equilibria.
\item Moreover,
      \begin{itemize}[<+-| alert@+>]
      \item for $k\rightarrow\infty$, $||x_{\Delta}(k)-\overline{x}|| \rightarrow 0 \, \forall x(0)$  iff $A^k$
            converges to a zero matrix,
      \item the same norm generally diverges if at least one element of $A^k$ does,
      \item and if $A^k$ neither converges to zero nor diverges, the same happens\\
            to $||x_{\Delta}(k)-\overline{x}||$.
      \end{itemize}
\item Thus, in the LTI case, the stability of a system only depends on its\\
      dynamic matrix $A$.
\end{itemize}
\end{frame}

\begin{frame}[label={pag:stab-eivals}]
\frametitleTC{Stability and eigenvalues of $A$}
\framesubtitleTC{}
\myPause
\begin{itemize}[<+-| alert@+>]
\item The following can be proven.
      \begin{itemize}[<+-| alert@+>]
      \item An LTI system is asymptotically stable iff all the eigenvalues of $A$ have magnitude less  than one
            (or, equivalently, lie in the open \TC{unit circle} of the complex plane).
      \item The same system is unstable if (but not only if) at least one eigenvalue of $A$ has magnitude greater
            than one.
      \item If all the eigenvalues of $A$ have magnitude less than or equal to one, and there exists at least one
            with unity magnitude, the system can be either unstable\\
            or stable, but not asymptotically.
      \end{itemize}
\end{itemize}
\end{frame}

\begin{frame}
\frametitleTC{Properties of asymptotically stable systems}
\framesubtitleTC{also in a view to control}
\myPause
\begin{itemize}[<+-| alert@+>]
\item An asymptotically stable system has one and only one equilibrium for each constant input.
\item The state and output free motions of an asymptotically stable system converge to zero (norm)
      for $k\rightarrow\infty$.
\item As a consequence, asymptotically stable systems ``forget their initial condition''...
\item ...which is a definitely desired property for a \TC{controlled} system.
\end{itemize}
\end{frame}


\section{XAFS Fourier transforms}

\begin{frame} \frametitle{XAFS Fourier Transforms}

\begin{cenpage}{130mm}
Fourier Transforms are an important part of XAFS Analysis:

\begin{postitbox}{75mm}  $\displaystyle{ \chi(R) = {\rm FT}[\chi(k)] =
  \frac{1}{\sqrt{2\pi}}
  \int_{-\infty}^{\infty} { dk \, e^{i2kR} \, k^w \, \chi(k) \, \Omega(k) }} $
\end{postitbox}

\begin{itemize}
\item $\Omega(k)$ is the Window Function
\item $w$ is the $k$-weighting
\end{itemize}

\pause
\vmm
 We really use a discrete version and Fast Fourier Transform

\[
\chi(R_m) = \frac{i \delta k}{\sqrt{\pi N_{\rm fft}}}
\sum_{n=1}^{N_{\rm fft}}
e^{2\pi i n m/N_{\rm fft}} \,  k_n^w \, \chi(k_n)\, \Omega(k_n)
\]

\pause
\begin{itemize}
\item {\chik} is put on a uniform $k$-grid with spacing of  $\delta k=0.05
  {\rm\,  \AA}^{-1}$.
\item {\chik} is filled with zeros past the real data range.

\item $N_{\rm fft} = 2048$: {\chik} can go to $102.4 {\rm\, \AA}^{-1}$
  ($\sim 40\rm \, keV$) past the edge.

\item {\chir} is on a $R$-grid with spacing $\sim0.031\,\rm\AA$, and can
  go to $31.4 \,\rm\AA$.
\end{itemize}
\end{cenpage}

\end{frame}

\begin{frame} \frametitle{Fourier Transforms: Basic Properties}

  \begin{cenpage}{130mm}
Fourier Transform of a sine wave:

  \begin{tabular}{ccc}
    \begin{minipage}{55mm}
      \includegraphics[width=55mm]{figs/reduction/sine_k0}
    \end{minipage}
    & $\Rightarrow $ &
    \begin{minipage}{55mm}
      \includegraphics[width=55mm]{figs/reduction/sine_r0}
    \end{minipage}\\
    \begin{minipage}{55mm}
      \includegraphics<2->[width=55mm]{figs/reduction/sine_k}
    \end{minipage}
    & {\onslide+<2-> $\Rightarrow $}  {\onslide<1> {\vspace{40mm}}}  &
    \begin{minipage}{55mm}
      \includegraphics<2->[width=55mm]{figs/reduction/sine_r}
    \end{minipage}\\
  \end{tabular}
\end{cenpage}
\end{frame}

\begin{frame} \frametitle{Fourier Transforms: Basic Properties(2)}

  \begin{cenpage}{130mm}
Fourier Transforms are complex:

  \begin{tabular}{ccc}
    \begin{minipage}{55mm}
      \includegraphics[width=55mm]{figs/reduction/sine_k}
    \end{minipage}
    & $\Rightarrow $ &
    \begin{minipage}{55mm}
      \includegraphics[width=55mm]{figs/reduction/sine_r2}
    \end{minipage}\\
    \noalign{\smallskip}
    \multicolumn{3}{l}{\onslide+<2->Waves with slightly different frequencies can cancel
      each other out, causing ``beats''  }  \\
    \noalign{\smallskip}

    \begin{minipage}{55mm}
      \includegraphics<2->[width=55mm]{figs/reduction/beat_k}
    \end{minipage}
    & {\onslide+<2-> $\Rightarrow $}  {\onslide<1> {\vspace{40mm}}}  &
    \begin{minipage}{55mm}
      \includegraphics<2->[width=55mm]{figs/reduction/beat_r}
    \end{minipage}\\

  \end{tabular}
\end{cenpage}
\end{frame}

\begin{frame} \frametitle{Fourier Transform Window Types}
\begin{cenpage}{130mm}
  \begin{tabular}{ll}
    \begin{minipage}{80mm}
      \includegraphics[width=80mm]{figs/reduction/ftwin_zoo}
    \end{minipage}
    &
    \begin{minipage}{50mm} \setlength{\baselineskip}{10pt}
      \hspace{-3mm}{\Red{Typical Window Functions}}
      \vspace{0.5mm}

      A Window Function:
      \begin{itemize}
      \item goes from 0 to 1 and back to 0
      \item $dk$ gives the width of the Window ``sill''
      \end{itemize}

      \vmm
      {\Red{Most important rule:}}

      \vmm

      Pick a window type and stick with it.

      \vmm

      Kaiser-Bessel and Hanning are the most commonly used, and recommended.

      \vmm

    \end{minipage}
  \end{tabular}

\vmm Personal Recommendation: \vmm

Kaiser-Bessel Window, $dk=4$.
\end{cenpage}
\end{frame}

\begin{frame} \frametitle{Fourier Transform Window Types}
\begin{cenpage}{130mm}

  \begin{columns}

    \begin{column}{70mm}
      \includegraphics[width=60mm]{figs/reduction/ftwin_anat}

      \vmm
      {\onslide+<2-> {
          \includegraphics[width=60mm]{figs/reduction/ftwin_sills}
        }}

    \end{column}

    \begin{column}{55mm}

      \hspace{-3mm}{\Red{Fourier Window Function}}  \vspace{0.5mm}

      The meaning of $k_{\rm min}$, $k_{\rm max}$, and $dk$.

      \vspace{10mm}

      {\onslide+<2-> {

      \hspace{-3mm}{\Red{Parzen, Hanning, Welch}}

      \vspace{0.5mm}

      Details of the different Window ``sills''.

      all with $k_{\rm  min}=2\rm\,\AA^{-1}$  and $dk=3\rm\,\AA^{-1} $.

      \vspace{3mm}
    }}
    \end{column}
  \end{columns}

\end{cenpage}
\end{frame}

\begin{frame} \frametitle{Fourier Transform Window and real data}

  \begin{cenpage}{130mm}
The effect of $dk$ (for Hanning Window) and different Window Function:

  \begin{tabular}{ll}
    \begin{minipage}{60mm}
      \includegraphics[width=55mm]{figs/reduction/ftwin_kdk}
    \end{minipage}
    &
    \begin{minipage}{60mm}
      \includegraphics[width=55mm]{figs/reduction/ftwin_rdk}
    \end{minipage}\\
    \begin{minipage}{60mm}
      \includegraphics<2->[width=55mm]{figs/reduction/ftwin_wins}
    \end{minipage}
    &
    {\onslide<1> {\vspace{40mm}}}     {\onslide+<2->
      \begin{minipage}{60mm}
        Changing  $dk$ and  Window functions
        gives relatively small changes to {\chir}, most noticeable in the
       ``ringing'' of small peaks.
      \end{minipage}
    }
  \end{tabular}
\end{cenpage}
\end{frame}

\begin{frame} \frametitle{Fourier Transform Window and $k$-weight }

\begin{cenpage}{130mm}
 $\displaystyle{ \chi(R) =   \frac{1}{\sqrt{2\pi}}
  \int_{-\infty}^{\infty} { dk \, e^{i2kR} \, k^{w} \, \chi(k) \, \Omega(k) }} $

  \vmm

Changing $w$, the $k$-weighting has a significant impact:

  \vmm

  \begin{tabular}{ll}
    \begin{minipage}{65mm}
      \includegraphics[width=65mm]{figs/reduction/ftwin_kw}
    \end{minipage}
    &
    \begin{minipage}{42mm}

      Fe-Fe scattering dominates with higher $w$.

      \vmm

      \vmm low $w$ emphasizes low-$k$, and low-Z scatterers.

      \vmm high $w$ emphasizes high-$k$, and  high-Z scatterers.

      \vmm
    \end{minipage}
  \end{tabular}

  \vmm This is important when trying to determine the $Z$ of a scatterer.

  \vmm Again, $w=2$ and $w=3$ are most common, and recommended.
\end{cenpage}
\end{frame}

\begin{frame} \frametitle{Fourier Transform Window and $k_{\rm min}$ }

  \begin{cenpage}{130mm}
  $k_{\rm min}$ and
  $k_{\rm max}$ are important too.

  \begin{itemize}
  \item $k_{\rm max}$ should be the end of useful data.
  \item With $k$-weight = 2, 3, it is not too important to avoid ``very low $k$''.
  \end{itemize}

  \vmm

  \begin{tabular}{ll}
    \begin{minipage}{65mm}
      \includegraphics[width=65mm]{figs/reduction/ftwin_kmin}
    \end{minipage}
    &
    \begin{minipage}{42mm}

      Conventional wisdom:  keep $k_{\rm min} > 2\rm\,\AA^{-1}$

      \vmm
      But: don't make it too big.

      \vmm

    \end{minipage}
  \end{tabular}

\begin{center}
  \begin{postitbox}{85mm}
    Use Kaiser-Bessel with $dk=4,k_{\rm min}=2 \rm\,\AA^{-1}$

    \vmm

    Use $k$-weight=2, or 3. \hspace{10mm} Don't obsess too much.
  \end{postitbox}
\end{center}
\end{cenpage}
\end{frame}


\title{Model Criticism}

\subsection{Model Criticism}

We can never validate whether a model is true. In practice, ``all
models are wrong'' \citep{box1976science}. However, we can try to
uncover where the model goes wrong. Model criticism helps justify the
model as an approximation or point to good directions for revising the
model.

Model criticism typically analyzes the posterior predictive
distribution,
\begin{align*}
  p(\mathbf{x}_\text{new} \mid \mathbf{x})
  &=
  \int
  p(\mathbf{x}_\text{new} \mid \mathbf{z})
  p(\mathbf{z} \mid \mathbf{x})
  \text{d} \mathbf{z}.
\end{align*}
The model's posterior predictive can be used to generate new data
given past observations and can also make predictions on new data
given past observations.
It is formed by calculating the likelihood of the new data, averaged
over every set of latent variables according to the posterior
distribution.

A helpful utility function to form the posterior predictive is
\texttt{copy}. For example, assume the model defines a likelihood
\texttt{x} connected to a prior
\texttt{z}. The posterior predictive distribution is
\begin{lstlisting}[language=Python]
x_post = ed.copy(x, {z: qz})
\end{lstlisting}
Here, we copy the likelihood node \texttt{x} in the graph and replace dependence
on the prior \texttt{z} with dependence on the inferred posterior \texttt{qz}.
We describe several techniques for model criticism.

\subsubsection{Point Evaluation}

A point evaluation is a scalar-valued metric for assessing
trained models \citep{winkler1994evaluating,gneiting2007strictly}.
For example, we can assess models for classification
by predicting the label for each observation in the data and comparing
it to their true labels. Edward implements a variety of metrics, such
as classification error and mean absolute error.

The \texttt{ed.evaluate()} method takes as input a set of metrics to
evaluate, and a data dictionary. As with inference, the data dictionary binds the
observed random variables in the model to realizations: in this case,
it is the posterior predictive random variable of outputs \texttt{y\_post} to
\texttt{y\_train} and a placeholder for inputs \texttt{x} to
\texttt{x\_train}.
\begin{lstlisting}[language=Python]
ed.evaluate('categorical_accuracy', data={y_post: y_train, x: x_train})
ed.evaluate('mean_absolute_error', data={y_post: y_train, x: x_train})
\end{lstlisting}

Point evaluation also applies to unsupervised tasks. For example, we
can evaluate the likelihood of observing the data.
\begin{lstlisting}[language=Python]
ed.evaluate('log_likelihood', data={x_post: x_train})
\end{lstlisting}

It is common practice to criticize models with data held-out from
training. To do this, we must first perform inference over any local
latent variables of the held-out data, fixing the global variables; we
demonstrate this below. Then we make predictions on the held-out data.

\begin{lstlisting}[language=Python]
from edward.models import Categorical

# create local posterior factors for test data, assuming test data
# has N_test many data points
qz_test = Categorical(logits=tf.Variable(tf.zeros[N_test, K]))

# run local inference conditional on global factors
inference_test = ed.Inference({z: qz_test}, data={x: x_test, beta: qbeta})
inference_test.run()

# build posterior predictive on test data
x_post = ed.copy(x, {z: qz_test, beta: qbeta}})
ed.evaluate('log_likelihood', data={x_post: x_test})
\end{lstlisting}

Point evaluations are formally known as scoring rules
in decision theory. Scoring rules are useful for model comparison, model
selection, and model averaging.

See the \href{/api/criticism}{criticism API} for further details.
An example of point evaluation is in the
\href{/tutorials/supervised-regression}{supervised learning
(regression)} tutorial.

\subsubsection{Posterior predictive checks}

Posterior predictive checks (PPCs)
analyze the degree to which data generated from the model deviate from
data generated from the true distribution. They can be used either
numerically to quantify this degree, or graphically to visualize this
degree. PPCs can be thought of as a probabilistic generalization of
point evaluation
\citep{box1980sampling,rubin1984bayesianly,meng1994posterior,gelman1996posterior}.

The simplest PPC works by applying a test statistic on new data
generated from the posterior predictive, such as
$T(\mathbf{x}_\text{new}) = \max(\mathbf{x}_\text{new})$.  Applying
$T(\mathbf{x}_\text{new})$ to new data over many data replications
induces a distribution. We compare this distribution to the test
statistic on the real data $T(\mathbf{x})$.

\includegraphics{/images/ppc.png}

In the figure, $T(\mathbf{x})$ falls in a low probability region of
this reference distribution: if the model were true, the probability
of observing the test statistic is very low. This indicates that the
model fits the data poorly according to this check; this suggests an
area of improvement for the model.

More generally, the test statistic can be a function of the
model's latent variables $T(\mathbf{x}, \mathbf{z})$, known as a
discrepancy function.  Examples of discrepancy functions are the
metrics used for point evaluation. We can now interpret the
point evaluation as a special case of PPCs: it simply calculates
$T(\mathbf{x}, \mathbf{z})$ over the real data and without a reference
distribution in mind. A reference distribution allows us to make
probabilistic statements about the point, in reference to an overall
distribution.

The \texttt{ed.ppc()} method provides a scaffold for studying various
discrepancy functions. It takes as input a user-defined discrepancy
function, and a data dictionary.
\begin{lstlisting}[language=Python]
ed.ppc(lambda xs, zs: tf.reduce_mean(xs[x_post]), data={x_post: x_train})
\end{lstlisting}
The discrepancy can also take latent variables as input, which we pass
into the PPC.
\begin{lstlisting}[language=Python]
ed.ppc(lambda xs, zs: tf.maximum(zs[z]),
       data={y_post: y_train, x_ph: x_train},
       latent_vars={z: qz, beta: qbeta})
\end{lstlisting}

See the \href{/api/criticism}{criticism API} for further details.

PPCs are an excellent tool for revising models---simplifying or
expanding the current model as one examines its fit to data.
They are inspired by classical hypothesis testing; these methods
criticize models under the frequentist perspective of large sample
assessment.

PPCs can also be applied to tasks such as hypothesis testing, model
comparison, model selection, and model averaging.  It's important to
note that while PPCs can be applied as a form of Bayesian hypothesis
testing, hypothesis testing is generally not recommended: binary
decision making from a single test is not as common a use case as one
might believe. We recommend performing many PPCs to get a holistic
understanding of the model fit.

\subsubsection{References}\label{references}


\chapter{Fourier Transform}
\begin{enumerate}
	\item  $\left. \right. $
	\begin{answer}
		\begin{align*}
		F(k)&=\frac{1}{\sqrt{2 \pi}} \int_{-\infty}^{+\infty} f(x) e^{i k x} d x\\
		\because g(x)&=f(x+a)\\
		\therefore G(k)&=\frac{1}{\sqrt{2 \pi}} \int_{-\infty}^{+\infty} f(x+a) e^{i k x} d x\\
		\text{Let }x+a&=y, d x=d y\\
		\therefore G(k)&=\frac{1}{\sqrt{2 \pi}} \int_{-\infty}^{+\infty} f(y) e^{i k(y-a)} d y=e^{-i k a} \frac{1}{\sqrt{2 \pi}} \int_{-\infty}^{+\infty} f(x) e^{i k x} d x=e^{-i a k} F(k)
	\end{align*}
		So the correct answer is \textbf{Option (c)}
	\end{answer}
	\item  $\left. \right. $
	\begin{answer}
		\begin{align*}
		 f_{c}(\omega) &=\sqrt{\frac{2}{\pi}} \int_{0}^{\infty} f(x) \cos \omega x d x \\ &=\sqrt{\frac{2}{\pi}} \int_{0}^{1} \cos \omega x d x+\sqrt{\frac{2}{\pi}} \int_{1}^{2}-\cos \omega x d x \\ &=\sqrt{\frac{2}{\pi}}\left[\left(\frac{\sin \omega x}{\omega}\right)_{0}^{1}-\left(\frac{\sin \omega x}{\omega}\right)_{1}^{2}\right] \\ &=\sqrt{\frac{2}{\pi}}\left(\frac{2 \sin \omega-\sin 2 \omega}{\omega}\right)
		\end{align*}
		So the correct answer is \textbf{Option (b)}
	\end{answer}
	\item  $\left. \right. $
	\begin{answer}
		\begin{align*}
		\mathrm{n}: g(k)&=\frac{1}{\sqrt{2 \pi}} \int_{-\infty}^{+\infty} \frac{n}{\sqrt{\pi}} e^{-n^{2} x^{2}} e^{i k x} d x=\frac{n}{\sqrt{2} \cdot \pi} \int_{-\infty}^{+\infty} e^{-n^{2} x^{2}+i i x} d x=\frac{n}{\sqrt{2} \cdot \pi} e^{-k^{2} / 4 n^{2}} \int_{-\infty}^{+\infty} e^{-n^{2}\left(x-\frac{i k}{2 n^{2}}\right)^{2}} d x\\
		&=\frac{n}{\sqrt{2} \cdot \pi} e^{-k^{2} / 4 n^{2}} \times 2 \times \frac{1}{2} \cdot \frac{\sqrt{\pi}}{n}=\frac{1}{\sqrt{2 \pi}} e^{-k^{2} / 4 n^{2}}
		\intertext{ Now inverse fourier transform of $g(k)$ is}
		\delta_{n}(x)&=\frac{1}{\sqrt{2 \pi}} \int_{-\infty}^{+\infty} \frac{e^{-k^{2} / 4 n^{2}}}{\sqrt{2 \pi}} \cdot e^{-i k x} d k\\
		\delta(x)&=\lim _{n \rightarrow \infty} \delta_{n}(x)=\lim _{n \rightarrow \infty} \frac{1}{2 \pi} \int_{-\infty}^{+\infty} e^{-k^{2} / 4 n^{2}} \cdot e^{-i k x} d k=\frac{1}{2 \pi} \int_{-\infty}^{+\infty} e^{-i k x} d k\\
		\therefore \delta(x)&=\frac{1}{2 \pi} \int_{-\infty}^{+\infty} e^{-i k x} d k
		\end{align*}
		So the correct answer is \textbf{Option (c)}
	\end{answer}
	\item  $\left. \right. $
	\begin{answer}
		\begin{align*}
		\text{(i) }g_{c}(\omega)=\sqrt{\frac{2}{\pi}} \int_{0}^{\omega} \delta(t-x) \cos \omega t d t=\sqrt{\frac{2}{\pi}} \cos \omega x
		\end{align*}
		So the correct answer is \textbf{Option (a)}
	\end{answer}
		\item  $\left. \right. $
	\begin{answer}
		\begin{align*}
		f(t)&=\delta(t-x)=\sqrt{\frac{2}{\pi}} \int_{0}^{+\infty} \sqrt{\frac{2}{\pi}} \cos \omega x \cos \omega t d \omega\\
		\Rightarrow \delta(t-x)&=\frac{2}{\pi} \int_{0}^{\infty} \cos \omega t \cos \omega x d \omega
		\end{align*}
		So the correct answer is \textbf{Option (b)}
	\end{answer}
		\item  $\left. \right. $
	\begin{answer}
		\begin{align*}
		\intertext{Fourier Transform of $f(t)$ is}
		g(\omega)&=\frac{1}{\sqrt{2 \pi}} \int_{0}^{\infty} e^{-\Gamma t / 2 \hbar} \cdot e^{-i E_{0} / / \hbar} \cdot e^{i a t} d t=\frac{1}{\sqrt{2 \pi}} \int_{0}^{\infty} \exp \left\{\frac{-i t}{\hbar}\left(E_{0}-i \Gamma / 2-\hbar \omega\right)\right\} d t\\
		&=\frac{1}{\sqrt{2 \pi}}\left[\frac{e^{\left\{\frac{-i t}{\hbar}\left(E_{0}-i / / 2-\hbar \omega\right)\right\}}}{\frac{-i}{\hbar}\left(E_{0}-i \Gamma / 2-\hbar \omega\right)}\right]_{0}^{\infty}=\frac{1}{\sqrt{2 \pi}}\left[\frac{0-1}{\frac{-i}{\hbar}\left(E_{0}-i \Gamma / 2-\hbar \omega\right)}\right]\\
		&=\frac{1}{\sqrt{2 \pi} i(E o-i \Gamma / 2-h \omega)}
		\intertext{Inverse transform of $g(\omega)$ is}
		f(t)&=\frac{1}{2 \pi} \int_{-\infty}^{+\infty} \frac{\hbar e^{-i \omega x}}{i\left(E_{0}-i \Gamma / 2-\hbar \omega\right)} d \omega=\frac{\hbar}{2 \pi i} \int_{-\infty}^{+\infty} \frac{e^{-i \omega t} d \omega}{\left(E_{0}-i \Gamma / 2-\hbar \omega\right)}\\
		\therefore &\frac{\hbar}{2 \pi i} \int_{-\infty}^{+\infty} \frac{e^{-i \omega t} d \omega}{\left(E_{0}-i \Gamma / 2-\hbar \omega\right)}=f(t)= \begin{cases}\exp (-\Gamma t / 2 \hbar) \exp \left(\frac{-i E_{0} t}{\hbar}\right), & t>0 \\ 0 & t<0\end{cases}
		\end{align*}
		So the correct answer is \textbf{Option (c)}
	\end{answer}
	\item  $\left. \right. $
	\begin{answer}
		\begin{align*}
		F\left\{t e^{-r^{2}}\right\}&=-\frac{1}{2} F\left\{\frac{d\left(e^{-r^{2}}\right)}{d t}\right\}=-\frac{1}{2}(j 2 \pi f) F\left\{e^{-t^{2}}\right\}=-j \pi f e^{-\pi^{2} f^{2}}
		\end{align*}
		So the correct answer is \textbf{Option (b)}
	\end{answer}
		\item  $\left. \right. $
	\begin{answer}
		\begin{align*}
		\intertext{This is sine function}
		f(x)&=A \sin x \Rightarrow F(k)=j \frac{A}{2}\left[\delta\left(k-k_{0}\right)-\delta\left(k+k_{0}\right)\right]
		\end{align*}
		So the correct answer is \textbf{Option (a)}
	\end{answer}
		\item  $\left. \right. $
	\begin{answer}
		\begin{align*}
		H(f)&=\int_{-\infty}^{+\infty} h(t) e^{-j 2 \pi f t} d t=\int_{0}^{+\infty} \beta e^{-\alpha t} e^{-j 2 \pi f t} d t=\beta \int_{0}^{+\infty} e^{-(\alpha+j 2 \pi f) t} d t\\
		\Rightarrow H(f)&=\left.\frac{-\beta}{\alpha+j 2 \pi f} e^{-(\alpha+j 2 \pi f) t}\right|_{0} ^{\infty}=\frac{\beta}{\alpha+j 2 \pi f}=\frac{\beta \alpha}{\alpha^{2}+(2 \pi f)^{2}}-j \frac{2 \pi f \beta}{\alpha^{2}+(2 \pi f)^{2}}\\
		\Rightarrow H(f)&=\frac{\beta}{\sqrt{\alpha^{2}+(2 \pi f)^{2}}} e^{j \tan ^{-1}[-2 \pi f / \alpha]}
		\end{align*}
		So the correct answer is \textbf{Option (d)}
	\end{answer}
		\item  $\left. \right. $
	\begin{answer}
		\begin{align*}
		f(x)&=\left\{\begin{array}{rl}-1 & -1<x<0 \\ 1 & 0<x<1 \\ 0 & \text { otherwise }\end{array}\right.\\
		F[f(x)]&=\frac{1}{\sqrt{2 \pi}}\left[\int_{-1}^{0}-e^{-i \omega x} d x+\int_{0}^{1} e^{-i \omega x} d x\right]\\
		&=\frac{1}{\sqrt{2 \pi}}\left[-\left(\frac{e^{-i \omega x}}{-i \omega}\right)_{-1}^{0}+\left(\frac{e^{-i \omega x}}{-i \omega}\right)_{0}^{1}\right]=\frac{1}{\sqrt{2 \pi}}\left[\frac{-1+e^{i \omega}}{-i \omega}+\frac{e^{-i \omega}-1}{-i \omega}\right]\\
		&=\frac{1}{\sqrt{2 \pi}}\left[\frac{2 \cos \omega-2}{-i \omega}\right]=\frac{i}{\omega} \sqrt{\frac{2}{\pi}}(\cos \omega-1)
		\end{align*}
		So the correct answer is \textbf{Option (b)}
	\end{answer}
		\item  $\left. \right. $
		\begin{answer}
			\begin{align*}
			f(x)&= \begin{cases}x, & 0<x<a \\ 0, & \text { otherwise }\end{cases}\\
			F[f(x)]&=\frac{1}{\sqrt{2 \pi}} \int_{0}^{a} x e^{-i \omega x} d x=\frac{1}{\sqrt{2 \pi}}\left[\left(\frac{x e^{-i e x}}{-i \omega}\right)_{0}^{a}-\left(\frac{e^{-i \omega x}}{(-i \omega)^{2}}\right)_{0}^{a}\right]\\&=\frac{1}{\sqrt{2 \pi}}\left[\frac{a e^{-i \omega a}}{-i \omega}-\frac{e^{-i \omega a}-1}{(-i \omega)^{2}}\right]=\frac{1}{\sqrt{2 \pi}}\left[\frac{i a \omega e^{-i e a}}{\omega^{2}}+\frac{e^{-i e a}-1}{\omega^{2}}\right]\\
			&=\frac{1}{\sqrt{2 \pi} \cdot \omega^{2}}\left[e^{-i \omega x}(1+i a \omega)-1\right]
			\end{align*}
			So the correct answer is \textbf{Option (a)}
		\end{answer}
\end{enumerate}

CSIR- Mathematical Physics/chapter/Assignments/Assignment-Fourier Transform-solutions.tex


\section{Application of isogeometric SBFEM to linear elastic fracture mechanics}
\label{iso_section:fracture}
An attractive feature of the SBFEM is that no a priori knowledge of the asymptotic solution is required to accurately handle the stress singularity at a crack tip as shown in Fig.~\ref{lr_fig:sbfem_intro}.
When modeling a cracked structure, a subdomain surrounding the crack tip is selected and the scaling center is placed at the crack tip.
The boundary of the subdomain is divided into line elements.
When the scaling center is placed at the crack tip, the solution for the stress field in Eq.\ref{lr_eq:sbfem_stress_field} is expressed, by using Eq.\ref{lr_eq:sbfem_transform}, as
    \begin{equation}
        \sigma(r,\eta) = \sum_i^n
            c_i r^{-(\lambda_i+1)} \left(
                r_\eta^{\lambda_i+1} (\eta)
                \boldsymbol{\psi}_{\sigma_i}(\eta)
            \right)
        \label{iso_eq:isosbfem_fracture_stress_field}
    \end{equation}
where $\psi_{\sigma_i}(\eta)$ is the $i$th stress mode, i.e. the $i$th column of the matrix $\psi_\sigma(\eta)$.
Like the well-known William expansion \citep{Williams1957109}, Eq.~\ref{iso_eq:isosbfem_fracture_stress_field} is a power series of the radial coordinate $r$.
The radial variation of each term of the series is expressed analytically by the power function $r^{-\lambda_i + 1}$.
At discrete points along the boundary, the angular coordinates (see Eq.~\ref{lr_eq:sbfem_transform}) are arranged as a vector $\theta(\eta)$ and the stress modes $\psi_{\sigma_i}(\eta)$ are computed.
$\psi_{\sigma_i}(\eta)$ and $\theta(\eta)$ from a parametric equation of the angular variation of stresses.
The singular stress and the T-stress terms can be easily identified by the value of the exponent $-(\lambda_i+1)$.
When the real part of the exponent $-(\lambda_i+1)$ of a term is negative, the stresses of this term at the crack tip, i.e. $\xi=0$, tend to infinity.
When the exponent $-(\lambda_i+1)$ of a term is equal to 0, the stresses of this term are constant and contribute to the T-stress.
\paragraph{}
In the case of a crack in a homogeneous material or on a material interface, two singular terms exist in the solution.
Denoting the singular stress modes as $\mathrm{I}$ and $\mathrm{II}$, the singular stress $\sigma^{s}(\xi,\eta)$ (superscript s for singular stresses) are obtained from Eq.~\ref{iso_eq:isosbfem_fracture_stress_field}:
\begin{equation}
    \sigma^s(r,\eta)
    =\sum_{i,\mathrm{I},\mathrm{II}}
    c_i r^{-(\lambda_i + 1)}
    (
        r_\eta^{(\lambda_i+1)}
        \psi_{\sigma_i}(\eta)
    )
\label{iso_eq:isosbfem_singular_stress}
\end{equation}
Note that the singular stress terms are separated from other terms and the stress singularity is represented analytically.
This allows the evaluation of the stress intensity factors by directly matching their definition with the singular stress.
For convenience, the point on the boundary along the crack from $\theta=0$ is considered.
The distance from the crack tip on the boundary is denoted as $L_0 = r_\eta(\theta=0)$.
From Eq.~\ref{iso_eq:isosbfem_singular_stress}, the values of the singular stresses at this point are equal to:
\begin{equation}
    \sigma^s (L_0, \theta = 0)
    =\sum_{i,\mathrm{I},\mathrm{II}}
    c_i \psi_{\sigma_i}(\theta=0)
    \label{iso_eq:isosbfem_singular_stress_2}
\end{equation}
where $\psi_{\sigma_i}(\theta=0)$ is the value of the stress modes at $\theta=0$.
It is obtained by interpolating $\psi_{\sigma_i}(\eta)$ at the discrete points of $\theta(\eta)$.
The stress intensity factors can be computed directly from their definition using the stresses in Eq.~\ref{iso_eq:isosbfem_singular_stress_2}.
For a crack in a homogeneous medium, the classical definition of stress intensity factors $K_{\mathrm{I}}$ and $K_{\mathrm{II}}$ for mode $\mathrm{I}$ and $\mathrm{II}$ are expressed as:
\begin{equation}
    \begin{Bmatrix}
        K_{\mathrm{I}}\\
        K_{\mathrm{II}}
    \end{Bmatrix}
    =\sqrt{2\pi r}
    \begin{Bmatrix}
        \sigma_{\theta\theta}^s(r,\theta=0) \\
        \tau_{r\theta}^s(r,\theta=0)
    \end{Bmatrix}
    \label{iso_eq:isosbfem_crack_homo}
\end{equation}
Formulating Eq.~\ref{iso_eq:isosbfem_crack_homo} at $r=L_0$ results in
\begin{equation}
    \begin{Bmatrix}
        K_{\mathrm{I}}\\
        K_{\mathrm{II}}
    \end{Bmatrix}
    =\sqrt{2\pi L_0}
    \begin{Bmatrix}
        \sigma_{\theta\theta}^s(L_0,\theta=0) \\
        \tau_{r\theta}^s(L_0,\theta=0)
    \end{Bmatrix}
\label{iso_eq:isosbfem_crack_homo_at_L}
\end{equation}
The stress intensity factors are then determined by substituting the stress components $\sigma_{\theta\theta}^s(L_0,\theta=0)$ and $\tau_{r\theta}^s(L_0,\theta=0)$ obtained from Eq.~\ref{iso_eq:isosbfem_singular_stress_2} into Eq.~\ref{iso_eq:isosbfem_crack_homo_at_L}.
For a subdomain containing a crack tip, two of the eigenvalues are equal to 1.
They represent the T-stress term and the rotational rigid body motion term, which does not contribute to the stresses.
They are separated from other terms in Eq.~\ref{iso_eq:isosbfem_singular_stress_2} and expressed as (superscript T for the T-stress)
\begin{equation}
    \sigma^T(\eta) =
    \sum_{ i=T_{ \mathrm{I} }, T_{ \mathrm{II} } }
    c_i \psi_{\sigma_i} (\eta)
\end{equation}
The T-stress along the crack front($\theta$=0) is determined by interpolating the angular variation of the two stress modes $(\theta(\eta), \psi_{\sigma_i} (\eta))$.


\documentclass{report}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{hyperref}
\usepackage{graphicx}
\usepackage{float}
\usepackage{listings}
\begin{document}
\section{LinearRegression without regularization}
\subsection{Abstract}
Hello, everyone. I'm btobab.\\
When I learned NLP, I found I completely can't understand formula derivation, so I decided to learn the theoretical derivation of machine learning from the beginning.\\
This chapter is mainly divided into two parts:
\begin{itemize}
	\item[] The first part will derive the closed-form solution of the linear regression(least squares method) from three perspectives of matrix, geometry, probability, and provide reference code.
	\item[] The second part will derive the closed-formula solution of least squares method with regularization from two perspectives of matrix and probability, and construct a complete linear regressioin class, and implement the closed-solution method and gradient descent method with code at the same time.
\end{itemize}
\subsection{Introduction}
Linear regression model is a model that use linear function to fit the relationship between one or more independent variables and the dependent variable ($y$).\\\\
The target variable ($y$) is a continuous numerical type, such as: housing price, number of people and rainfall. The regression model is to find a mapping function between input variables and output variables.\\\\
The learning of regression task equals to function fitting: use a function curve to make it fit the known data well and predict unknown data.\\\\
Regression task is divided into two processes: model learning and prediction. Construct a model based on given training dataset and predict corresponding output based on new input data.
\subsection{Algorithm}
\subsubsection{Matrix perspective}
Note:in general, the vectors we are discussing are column vectors. Therefore, in order to ensure the shape of the matrix during the derivation process, a large number of transposition characters are used\\\\
given dataset $\mathcal{D}=\{(x_1,y_1),(x_2,y_2)...(x_n, y_n)\}$\\\\
in which $x_i\in \mathcal{R}^p, y_i\in \mathcal{R}, i=1,2,...,n$

$$
X=(x_1,x_2,...,x_n)^T=\begin{pmatrix}
x_{11}&x_{12}&...&X_{1p}\\
x_{21}&x_{22}&...&x_{2p}\\
.&.&.&.&\\
.&.&.&.&\\
.&.&.&.&\\
x_{n1}&x_{n2}&...&x_{np}\\
\end{pmatrix}_{np}
$$
$$
Y=(y1,y2,...,y_n)_{n1}^T
$$
It is the model we constrct: $f(w)=w^Tx+w_0 x_0.$\\\\
Generally set $x_0=1$,and $b=w_0 x_0$, $b$ is bias, $w$ is weight. below, for the convenience of derivation, we merge $w_0$ into $w$ and $x_0$ into $x$.\\\\
so the model is updated to $f(w)=w^Tx$
The loss function of least squares method is:
$$
L(w)=\sum_{i=1}^{i=n}\|y_i-w^T x_i\|_2^2
$$
$$
=\begin{pmatrix}
y_1-w^Tx_1&y_2-w^Tx_2&...&y_n-w^Tx_n
\end{pmatrix}
\begin{pmatrix}
y_1-w^Tx_1\\
y_2-w^Tx_2\\
.\\
.\\
.\\
y_n-w^Tx_n\\
\end{pmatrix}
$$
$$
=(Y^T-w^TX^T)(Y^T-w^TX^T)^T
$$
$$
=(Y^T-w^TX^T)(Y-Xw)
$$
$$
=Y^TY-w^TX^TY-Y^TXw+w^TX^TXw
$$
Note the second and third terms are transposed to each other, and observe its matrix shape: $(1,p)(p,n)(n,1)=(1,1)$\\
Knowning that these two terms are scalars, and the transposition of a scalar is itself, so the two can be conbined, get:
$$
L(w)=Y^TY-2w^TX^TY+w^TX^TXw
$$
so $\hat{w}=argmin(L(w))$
below, to find out the minimum of $L(w)$, we need to diffentiate $L(w)$\\\\
Note there are three terms in the formual. The first term has nothing to do with $w$ and can be removed. Then the remaining two terms involve matrix derivation.\\\\
Regarding to matrix derivation, author recommends three articles by a blogger(more detailed and rigorous than textbook, each formual has proof)
\begin{itemize}
	\item \href{https://zhuanlan.zhihu.com/p/263777564}{essence}
	\item \href{https://zhuanlan.zhihu.com/p/273729929}{basics}
	\item \href{https://zhuanlan.zhihu.com/p/288541909}{advanced}
\end{itemize}
The following is the derivative solution process of the above two terms.\\\\
Because $X,Y$ are constant matrices, the derivative can be obtained directly. However, since it is the derivative of $w$, the result must be transposed.
$$
\frac{d(2w^TX^TY)}{dw}=2X^TY
$$
Below, let's solve the third term.
$$
d(w^TX^TXw)=tr(d(w^TX^TXw))=tr(X^TXd(w^Tw))
$$
$$
=tr(X^TX(d(w^T)w+w^Td(w)))=tr(X^TXw(dw)^T)+tr(X^TXw^Tdw)
$$
$$
=tr(w^TX^TXdw)+tr(X^TXw^Tdw)=tr(2X^TXw^Tdw)
$$
so

$$
\frac{d(w^TX^TXw)}{dw}=2wX^TX
$$
From a geometric perspective, we regard $X$ as a $p$ dimensional vector.\\\\
The first dimension of $X$ is $(x_{11},x_{21},...,x_{n1})$, the p-th dimension of $X$ is $(x_{1p},x_{2p},...,x_{np})$\\\\
and here $Y$ is regarded as a one-dimensional vector.\\\\
so $\frac{dL(w)}{dw}=2X^TXw-2X^TY$\\\\
set the derivative equal to 0 to get the closed-form solution of the least squares method:
$$
\hat{w}=(X^TX)^{-1}X^TY
$$
\subsubsection{Geometry perspective}
$$
X=(x_1,x_2,...,x_n)^T=\begin{pmatrix}
x_{11}&x_{12}&...&X_{1p}\\
x_{21}&x_{22}&...&x_{2p}\\
.&.&.&.&\\
.&.&.&.&\\
.&.&.&.&\\
x_{n1}&x_{n2}&...&x_{np}\\
\end{pmatrix}_{np}
$$
$$
Y=(y1,y2,...,y_n)_{n1}^T
$$
Now we assume  $p=2$ because it is easier to draw. The diagram is as follows(I really drew it for a long time and pitifully asked for a star.
\begin{figure}
\includegraphics[width=0.7 \textwidth]{/Users/btobab/TeX-Projects/figures/1}
\caption{figure 1}
\end{figure}
Change the model to $f(w)=Xw$, which means zoom $X$ with weight $w$\\\\
The geometric meaning of the least squares method is to find a $w$, so that the distance between vector $Y-Xw$ and space $w$ is the smallest. Of course the case of the smallest distance is perpendicular to space $X$.\\\\
so we get a formula: $X^T(Y-Xw)=0$\\\\
then get the solution of $w$:
$$
X^TXw=X^TY
$$
$$
\hat{w}=(X^TX)^{-1}X^TY
$$
we can see that the solved $w$ is the same as the result of matrix perspective.
\subsubsection{Probability perspective}
As we known, in reality, it is hard to fit a distribution with a straight line. True data must have some randomness, that is, noise.\\\\
so we assume noise $\epsilon\backsim N(0,\sigma^2)$\\
so $y=f(w)+\epsilon=w^Tx+\epsilon$\\
so $y|x;w\backsim N(w^Tx,\sigma^2)$\\
Bring it into the probability density function of gaussian distribution:
$$
p(y|x;w)=\frac{1}{\sqrt{2\pi}\sigma}e^{-\frac{(y-w^Tx)^2}{2\sigma^2}}
$$
Then use $MLE$ (maximum likelihood estimate)\\\\
Note: so-called MLE is to get the relative frequency via a large number of samples to approximate probability\\\\
Let's assume a function: $\zeta(w)=\log{p(Y|X;w)}$

Since $n$ data are independent, we can change the probability to a form of continuous mutiplication:

$\zeta(w)=\log{\Pi_{i=1}^np(y_i|x_i;w)}=\Sigma_{i=1}^n \log{p(y_i|x_i;w)}$

bring the probability density function of gaussian distribution into the formula:

$\zeta(w)=\Sigma_{i=1}^n(\log{\frac{1}{\sqrt{2\pi}\sigma}}-\frac{(y-w^Tx)^2}{2\sigma^2})$

since the former term has nothing to do with $w$, it can be ignored.

so:
$$
\hat{w}=argmax{\zeta(w)}
$$
$$
=argmax{ \Sigma_{i=1}^n -\frac{(y-w^Tx)^2}{2\sigma^2}}
$$
$$
=argmin{ \Sigma_{i=1}^n (y-w^Tx)^2}
$$
The conclusion obtained by using maximum likelihood estimation is the definition of least squares method.\\\\
This also shows that least squares method hides a assumption that noise is gaussian distribution.
\subsection{Implement}
\begin{lstlisting}[language={python}]
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt

# num of samples
n = 1000
# noise
epsilon = 1
X = np.expand_dims(np.linspace(0,100,1000), axis=-1)
w = np.asarray([5.2])
Y = X  w
# apply noise to X
X += np.random.normal(scale=epsilon, size=(X.shape))
X_T = X.transpose()
w_hat = np.matmul(np.linalg.pinv((np.matmul(X_T, X))), np.matmul(X_T, Y))
print(w_hat)
plt.scatter(X, Y, s=3, c="y")
Y_hat = X  w_hat
plt.plot(X, Y_hat)
plt.show()
\end{lstlisting}
\newpage
\section{LinearRegression with regularization}
\subsection{Algorithm}
\subsubsection{Matrix perspective}
Firstly, given a new loss functioin with regularization:
$$
\zeta(w)=\Sigma_{i=1}^{n}||y_i-w^T  x_i||^2 + \lambda  ||w||^2
$$
then the derivation of loss function from matrix perspective without regularization is referenced:
$$
\zeta(w)=Y^TY-2w^TX^TY+w^TX^TX w+\lambda  ||w||^2
$$
so$\hat{w}=argmax(\zeta(w))$\\\\
differentiate $\zeta(w)$:
$$
\frac{\partial \zeta(w)}{\partial w}=2X^TXw-2X^T Y+2\lambda  w
$$
set the derivative equal to 0 to get the closed-form solution of least squares method with regularization:
$$
\hat{w}=(X^TX+\lambda  I)^{-1} X^TY
$$
$I$ is identity matrix
\subsubsection{Probability perspective}
assume noise $\epsilon \backsim N(0,\sigma_1^2) \ w \backsim N(0,\sigma_2^2)$

since $y=w^T x + \epsilon$

we get $y|w \backsim N(w^T x,\sigma_1^2)$
\\Next we use MAP(Maximum a posteriori estimate):

according to Bayes theorem:
$$
P(w|Y)=\frac{P(Y|w) P(w)}{P(Y)}
$$
$P(w)$ is a priori probability, $P(Y|w)$ is a likelihood probability, $P(Y)$ is normalized probability, priori probability is mutiplied by the likelihood probability and normalized to obtain the posteriori probability $P(w|Y)$.
actually $P(Y)$ is constant, so:
$$
\hat{w}=argmax(P(w|Y))=argmax(P(Y|w) P(w))=argmax(log(P(Y|w) P(w)))
$$
since samples are independent, we can change the probability to a form of continuous mutipication.
$$
=argmax(log(\prod_{i=1}^n P(y_i|w) P(w)))=argmax(\sum_{i=1}^n log(P(y_i|w)+ log(P(w))))
$$
bring it into probability density function of gaussian distribution to get:
$$
\hat{w}=argmax(\sum_{i=1}^nlog(\frac{1}{\sqrt{2\pi} \sigma_1})-\frac{(y_i-w^T x_i)^2}{2\sigma_1^2}+log(\frac{1}{\sqrt{2 \pi} \sigma_2})-\frac{w^2}{2\sigma_2^2})
$$
since both $\sigma_1$ and $\sigma_2$ are hyperparameters, they can be omitted.

so:
$$
\hat{w}=argmin(\sum_{i=1}^n \frac{(y_i-w^T x_i)^2}{2\sigma_1^2}+\frac{w^2}{2\sigma_2^2})
$$
$$
=argmin(\sum_{i=1}^n (y_i-w^T x_i)^2+\frac{\sigma_1^2}{\sigma_2^2} w^2)
$$
we can see that the result derived via $MAP$ is the definition of the least squares method with regularization.
\subsection{Implement}
\begin{lstlisting}[language={python}]
import os
os.chdir("../")
from models.linear_models import LinearRegression
import numpy as np
import matplotlib.pyplot as plt
X_ = np.expand_dims(np.linspace(0, 10, 1000), axis=-1)
X = np.c_[X_, np.ones(1000)]
w = np.asarray([5.2, 1])
Y = X.dot(w)
X = np.r_[X, np.asarray([[11, 1], [12, 1], [13, 1]])]
Y = np.r_[Y, np.asarray([100, 110, 120])]

model = LinearRegression(l2_ratio=1e1, epoch_num=1000, lr=1e-2, batch_size=100, if_standard=False)
model.fit(X[:, :-1], Y)
print(model.get_params())
model.draw(X[:, :-1], Y)
\end{lstlisting}
\end{document}