\documentclass[10pt]{article}
\usepackage[margin=1in]{geometry}
\usepackage[all]{xy}

\usepackage[english]{babel}

\usepackage[nottoc]{tocbibind}

\usepackage{amsmath,amsthm,amssymb,color,latexsym}
\usepackage{geometry}
\geometry{letterpaper}
\usepackage{graphicx}
\usepackage{physics}

\usepackage{listings}
\usepackage[table,xcdraw]{xcolor}

\usepackage[export]{adjustbox}

\definecolor{codegreen}{rgb}{0,0.6,0}
\definecolor{codegray}{rgb}{0.5,0.5,0.5}
\definecolor{codepurple}{rgb}{0.58,0,0.82}
\definecolor{backcolour}{rgb}{0.95,0.95,0.92}
\usepackage{multirow}
\usepackage{float}

\lstdefinestyle{mystyle}{
  backgroundcolor=\color{backcolour},   commentstyle=\color{codegreen},
  keywordstyle=\color{magenta},
  numberstyle=\tiny\color{codegray},
  stringstyle=\color{codepurple},
  basicstyle=\ttfamily\footnotesize,
  breakatwhitespace=false,
  breaklines=true,
  captionpos=b,
  keepspaces=true,
  numbers=left,
  numbersep=5pt,
  showspaces=false,
  showstringspaces=false,
  showtabs=false,
  tabsize=2
}

\lstset{style=mystyle}

\usepackage{graphicx}
\graphicspath{ {./images/} }

\newtheorem{problem}{Problem}

\newenvironment{solution}[1][\it{Solution}]{\textbf{#1. } }{$\square$}

\begin{document}
\noindent CS 6140: Machine Learning Spring 2021\hfill Homework Assignment \#4\\
Sakshi Suman\hfill suman.sak@northeastern.edu

\hrulefill

\begin{problem}

Consider a logistic regression problem where $\mathcal{X}=\mathbb{R}^{d}$ and $\mathcal{Y}=\{-1,+1\}$. Derive the weight update rule that maximizes the conditional likelihood assuming that a data set $\mathcal{D}=\left\{\left(x_{i}, y_{i}\right)\right\}_{i=1}^{n}$ is given.

\end{problem}
\begin{solution}

To frame the learning problem as parameter estimation, let the data set $\mathcal{D}=\left\{\left(\mathbf{x}_{i}, y_{i}\right)\right\}_{i=1}^{n}$ is an i.i.d. sample from a fixed but unknown probability distribution $p(\mathbf{x}, y) .$ Even more specifically, we can assume that the data generating process randomly draws a data point $\mathrm{x},$ a realization of the random vector $\left(X_{0}=1, X_{1}, \ldots, X_{d}\right),$ according to $p(\mathbf{x})$ and then sets its class label $Y$ according to the Bernoulli distribution

\begin{equation*}
\scalebox{1.4}{
$p(y \mid \mathbf{x}) =
        \begin{cases}
            \left(\frac{1}{1+e^{-\omega^{\top} \mathbf{x}}}\right)^{\frac{1+y}{2}} &\text{ for } y=1 \\
			  \left(1-\frac{1}{1+e^{-\omega^{\top} \mathbf{x}}}\right)^{\frac{1-y}{2}} &\text{ for } y=-1
        \end{cases}$
        }
    \end{equation*}

where $\omega=\left(\omega_{0}, \omega_{1}, \ldots, \omega_{d}\right)$ is a set of unknown coefficients we want to recover (or learn) from the observed data $\mathcal{D}$. Based on the principles of parameter estimation, we can estimate $\omega$ by maximizing the conditional likelihood of the observed class labels $\mathbf{y}=\left(y_{1}, y_{2}, \ldots, y_{n}\right)$ given the inputs $\mathbf{X}=\left(\mathbf{x}_{1}^{\top}, \mathbf{x}_{2}^{\top}, \ldots, \mathbf{x}_{n}^{\top}\right)$
We shall first write the conditional likelihood function $p(\mathbf{y} \mid \mathbf{X}, \mathbf{w}),$ or simply $l(\mathbf{w}),$ as
$$
l(\mathbf{w})=\prod_{i=1}^{n} p\left(y_{i} \mid \mathbf{x}_{i}, \mathbf{w}\right)
$$

The parameter vector that maximizes the likelihood is,

$$
\begin{aligned}
\mathbf{w}_{\mathrm{ML}} &=\underset{\mathbf{w}}{\arg \max }\{l(\mathbf{w})\} \\
&=\underset{\mathbf{w}}{\arg \max }\left\{\prod_{i=1}^{n} p\left(y_{i} \mid \mathbf{x}_{i}, \mathbf{w}\right)\right\} .
\end{aligned}
$$

The likelihood function is
$$
l(\mathbf{w})=\prod_{i=1}^{n}\left(\frac{1}{1+e^{-\mathbf{w}^{\top} \mathbf{x}_{i}}}\right)^{\frac{1+y_{i}}{2}} \cdot\left(1-\frac{1}{1+e^{-\mathbf{w}^{\top} \mathbf{x}_{i}}}\right)^{\frac{1-y_{i}}{2}}
$$

As $\log(x)$ is a strictly increasing function, maximizing likelihood is equivalent to maximizing the log-likelihood function $l l(\mathbf{w})=\log (l(\mathbf{w}))$
$$
l l(\mathbf{w})=\sum_{i=1}^{n}\left(\left(\frac{1+y_{i}}{2}\right) \cdot \log \left(\frac{1}{1+e^{-\mathbf{w}^{\top} \mathbf{x}_{i}}}\right)+\left(\frac{1-y_{i}}{2}\right) \cdot \log \left(1-\frac{1}{1+e^{-\mathbf{w}^{\top} \mathbf{x}_{i}}}\right)\right)
$$

The negative of the log-likelihood i.e., cross-entropy minimization is equivalent to the maximization of likelihood.

$$
ll(\mathbf{w})=\sum_{i=1}^{n}\left(\left(\frac{y_{i}-1}{2}\right) \mathbf{w}^{\top} \mathbf{x}_{i}+\log \left(\frac{1}{1+e^{-\mathbf{w}^{\top} \mathbf{x}_{i}}}\right)\right)
$$

There is no closed-form solution to $\nabla ll(\mathbf{w})=\mathbf{0}$. Thus, we have to proceed with iterative optimization methods. Hence the goal is to calculate the gradient $(\nabla ll(\mathbf{w}))$ and Hessian $\left(H_{l l(\mathbf{w})}\right)$ in order to specify the update rule described by Newton-Raphson's method, as a function of inputs $\mathbf{X}$, class labels $\mathbf{y},$ and the current parameter vector. We can calculate the first and second partial derivatives of $ll(\mathbf{w})$ as follows

$$
\begin{aligned}
\frac{\partial ll(\mathbf{w})}{\partial w_{j}} &=\sum_{i=1}^{n}\left(\left(\frac{y_{i}-1}{2}\right) \cdot x_{i j}-\frac{1}{1+e^{-\mathbf{w}^{\top} \mathbf{x}_{i}}} \cdot e^{-\mathbf{w}^{\top} \mathbf{x}_{i}} \cdot\left(-x_{i j}\right)\right) \\
&=\sum_{i=1}^{n} x_{i j} \cdot\left(\left(\frac{y_{i}-1}{2}\right)+\frac{e^{-\mathbf{w}^{\top} \mathbf{x}_{i}}}{1+e^{-\mathbf{w}^{\top} \mathbf{x}_{i}}}\right) \\
&=\sum_{i=1}^{n} x_{i j} \cdot\left(\left(\frac{y_{i} + 1}{2}\right)-\frac{1}{1+e^{-\mathbf{w}^{\top} \mathbf{x}_{i}}}\right) \\
&=\mathbf{f}_{j}^{\top}\left(\left(\frac{\mathbf{y}+\mathbf{1}}{2}\right)-\mathbf{p}\right)
\end{aligned}
$$

where $\mathbf{f}_{j}$ is the $j$ -th column (feature) of data matrix $\mathbf{X}, \mathbf{y}$ is an $n$-dimensional column vector of class labels, $\mathbf{1}$ is a vector containing $n$ 1s and $\mathbf{p}$ is an $n$-dimensional column vector of (estimated) posterior probabilities $p_{i}=P\left(Y_{i}=1 \mid \mathbf{x}_{i}, \mathbf{w}\right),$ for $i=1, \ldots, n$. Considering partial derivatives for every component of $\mathbf{w}$, we have
$$
\nabla ll(\mathbf{w})=\mathbf{X}^{T}\left(\left(\frac{\mathbf{y} + \mathbf{1}}{2}\right)-\mathbf{p}\right)
$$

The second partial derivative of the log-likelihood function can be found as,
$$
\begin{aligned}
\frac{\partial^{2} l l(\mathbf{w})}{\partial w_{j} \partial w_{k}} &=\sum_{i=1}^{n} x_{ij} \cdot \frac{e^{-\mathbf{w}^{\top} \mathbf{x}_{i}}}{\left(1+e^{\left.-\mathbf{w}^{\top} \mathbf{x}_{i}\right)^{2}}\right.} \cdot\left(-x_{i k}\right) \\
&=-\sum_{i=1}^{n} x_{i j} \cdot \frac{1}{1+e^{-\mathbf{w}^{\top} \mathbf{x}_{i}}} \cdot\left(1-\frac{1}{1+e^{-\mathbf{w}^{\top} \mathbf{x}_{i}}}\right) \cdot x_{i k} \\
&=-\mathbf{f}_{j}^{\top} \mathbf{P}(\mathbf{I}-\mathbf{P}) \mathbf{f}_{k},
\end{aligned}
$$

where $\mathbf{P}$ is an $n \times n$ diagonal matrix with $P_{i i}=p_{i}=P\left(Y_{i}=1 \mid \mathbf{x}_{i}, \mathbf{w}\right)$ and $\mathbf{I}$ is an $n \times n$ identity matrix. The Hessian matrix $H_{l l(\mathbf{w})}$ can now be calculated as
$$
H_{ll(\mathbf{w})}=-\mathbf{X}^{T} \mathbf{P}(\mathbf{I}-\mathbf{P}) \mathbf{X}
$$

The weight update rule of Newton-Raphson's method is as follows:
$$
\mathbf{w}^{(t+1)}=\mathbf{w}^{(t)}+\left(\mathbf{X}^{\top} \mathbf{P}^{(t)}\left(\mathbf{I}-\mathbf{P}^{(t)}\right) \mathbf{X}\right)^{-1} \mathbf{X}^{\top}\left(\left(\frac{\mathbf{y}+\mathbf{1}}{2}\right)-\mathbf{p}^{(t)}\right)
$$

\end{solution}

\begin{problem}
Consider a logistic regression problem with its initial solution obtained through the OLS regression; i.e., $\mathbf{w}^{(0)}=\left(\mathbf{X}^{T} \mathbf{X}\right)^{-1} \mathbf{X}^{T} \mathbf{y},$ in the context of the code provided in class (week $\left.6\right) .$ Recall that $\mathbf{x}$ was drawn from a mixture of two Gaussian distributions with $\operatorname{dim}\{\mathbf{x}\}=2$ (before adding a column of ones) and that $y \in\{0,1\}$. You probably noticed that the initial separation line is consistently closer to the data points of class 0.\\\\
a) (10 points) Why was this the case? Draw a picture (if possible) to support your argument.\\
b) (5 points) Devise a better initial solution by modifying the standard formula $\mathbf{w}^{(0)}=\left(\mathbf{X}^{T} \mathbf{X}\right)^{-1} \mathbf{X}^{T} \mathbf{y}$.\\
c) (5 points) Now again consider the case where $y \in\{-1,+1\}$. What is the form of the modified solution from part (b) in this case?
\end{problem}
\begin{solution}

\subsection*{a)}

Let us add a component $x_{0}=1$ to each input $\left(x_{1}, \ldots, x_{d}\right)$. This extends the input space to $\mathcal{X}=\mathbb{R}^{d+1}$ but, fortunately, it also leads us to a simplified notation in which the decision boundary in $\mathbb{R}^{d}$ can be written as $\mathbf{w}^{\top} \mathbf{x}=0$, where $\mathbf{w}=\left(w_{0}, w_{1}, \ldots, w_{d}\right)$ is a set of weights and $\mathbf{x}=\left(x_{0}=1, x_{1}, \ldots, x_{d}\right)$ is any element of the input space. The actual inputs are $d$-dimensional.

The plane, $w_0 + w_1x_1 + w_2x_2 + \ldots + w_dx_d = 0$ and the plane, $y = 0$ have a surface which is a $d - 1$ dimensional plane (if $d = 2$ then it is called a line) as the intersection in $d + 1$ dimensions where $d$ dimensions are coming from $\mathcal{X}$ and the extra 1-dimension comes from $y$.

This surface intersects the $d$ dimensions coming from $\mathcal{X}$ with intercepts,

$\left(x_1=-\frac{w_0}{w_1}, x_2=0, \ldots , x_d = 0, y = 0\right), \left(x_1=0, x_2=-\frac{w_0}{w_2}, \ldots , x_d = 0, y = 0\right), \ldots,$

$ \left(x_1=0, x_2=0, \ldots , x_d = -\frac{w_0}{w_d}, y = 0\right)$ .

In class demo, $d = 2$. The data is obtained from a Gaussian Distribution with mean for class $0$ at $(1, 2)$ and mean for class $1$ at $(6, 4)$. The above plane becomes, $w_0 + w_1x_1 + w_2x_2 = 0$ and $y = 0$. The intersection of these planes is a line in the $x_1x_2$ plane. This is highlighted in red as shown below:

\begin{center}
\includegraphics[width=16cm, keepaspectratio]{Initial_Separation_Line_Class}
\end{center}

\begin{center}
\includegraphics[width=16cm, keepaspectratio]{Problem2}
\end{center}

\subsection*{b)}

However, the above solution can be devised better with a good understanding of the two classes. Instead of taking the separation plane/line as intersection of $\mathbf{w}^{\top} \mathbf{x}=0$ and $y = 0$ (initial separation line 1) or as intersection of $\mathbf{w}^{\top} \mathbf{x}=0$ and $y = 1$ (initial separation line 2), we could take a value that favour both classes equally, i.e., which in this case is intersection of $\mathbf{w}^{\top} \mathbf{x}=0$ and $y = 0.5$ (optimal separation line) as it is in between $y = 0$ and $y = 1$ and more importantly equidistant from the two planes in euclidean space. Intersection with plane such as $y = 0.3$ (suboptimal separation line) would be sub-optimal. This gives a modified initial solution as,

$$
\mathbf{w}^{(0)}=\left(\mathbf{X}^{T} \mathbf{X}\right)^{-1} \mathbf{X}^{T} (\mathbf{y} - 0.5)
$$
where the value $0.5$ above is subtracted from every class label.
\\

The modified line of intersection can be seen as below:

\begin{center}
\includegraphics[width=16cm, keepaspectratio]{Final_Separation_Line_Class}
\end{center}

This definitely looks like a better initial solution than the previous one.

c)

In this case, $y \in \{-1, + 1\}$, the mean of the possible class labels is $0$. This means that the solution used in the actual code given on class website should be fine. Hence, we can use,

$$
\mathbf{w}^{(0)}=\left(\mathbf{X}^{T} \mathbf{X}\right)^{-1} \mathbf{X}^{T} \mathbf{y}
$$

\end{solution}

\begin{problem}
Problem 3. (40 points) Consider two classification concepts given in Figure 1 , where $x \in \mathcal{X}=[-6,6] \times$ $[-4,4], y \in \mathcal{Y}=\{-1,+1\}$ and $p(y \mid x) \in\{0,1\}$ is defined in the drawing.

\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{Problem3}
\caption{Two concepts where examples that fall within any of the three $3 \times 3$ (panel A) or $1 \times 1$ (panel B) squares are labeled positive and the remaining examples (outside each of the squares but within $\mathcal{X}$ ) are labeled negative. The position of the point $x=\left(x_{1}, x_{2}\right)$ in the upper left-hand corner for each square is shown in the picture. Consider horizontal axis to be $x_{1}$ and vertical axis as $x_{2}$.}
\end{figure}

Your experiments in this question will rely on generating a data set of size $n \in\{250,1000,10000\}$ drawn from a uniform distribution in $\mathcal{X}$ and labeled according to the rules from Figure $1 ;$ e.g., $P(Y=1 \mid x)=1$ if $x$ that was randomly drawn is inside any of the three squares in either of the two panels, and $P(Y=1 \mid x)=0$ otherwise. The goal of the following two problems will be to train and evaluate classifiers created from the data generated in this way. You can use any library you want in this assignment and do programming in Python, MATLAB, $R$ or $C / C++$. Your code should be easy to run for each question and sub-question below so that we can replicate your results to the maximum extent possible.

Consider single-output feed-forward neural networks with one or two hidden layers such that the number of hidden neurons in each layer is $h_{1} \in\{1,4,12\}$ and $h_{2} \in\{0,3\},$ respectively, with $h_{2}=0$ meaning that there is no second hidden layer. Consider one of the standard objective functions as your optimization criterion and use early stopping and regularization as needed. Consider a hyperbolic tangent activation function in each neuron and the output but you are free to experiment with others if you'd like to. For each of the architectures, defined by a parameter combination $\left(h_{1}, h_{2}\right),$ evaluate the performance of each model using classification accuracy, balanced accuracy, and area under the ROC curve as your two performance criteria. To evaluate the performance of your models use cross-validation. However, to evaluate the performance of performance evaluation, generate another very large data set on a fine grid in $\mathcal{X} .$ Then use the predictions from your trained model on all these points to determine the "true" performance. You can threshold your predictions in the middle of your prediction range (i.e., at 0.5 if you are predicting between 0 and 1) to determine binary predictions of your models and to then compare those with true class labels you generated on the fine grid.

Provide meaningful comments about all aspects of this exercise (performance results for different network architectures, accuracy of cross-validation, run time, etc.). The comments should not just re-state the results but rather capture trends and give reasoning as to why certain behavior was observed.
\end{problem}

\begin{solution}

For code, please refer to \textbf{NeuralNet.ipynb}.

\subsection*{Results}

\begin{table}[H]
\tiny
\begin{tabular}{|l|l|l|l|l|l|l|l|l|l|l|l|l|l|l|}
\hline
\rowcolor[HTML]{ECF4FF}
\cellcolor[HTML]{ECF4FF}                        & \cellcolor[HTML]{ECF4FF}                     & \cellcolor[HTML]{ECF4FF}                     & \multicolumn{4}{l|}{\cellcolor[HTML]{ECF4FF}Balanced Accuracy}                                                                    & \multicolumn{4}{l|}{\cellcolor[HTML]{ECF4FF}Classification Accuracy}                                                              & \multicolumn{4}{l|}{\cellcolor[HTML]{ECF4FF}Area Under the ROC Curve}                                                             \\ \cline{4-15}
\cellcolor[HTML]{ECF4FF}                        & \cellcolor[HTML]{ECF4FF}                     & \cellcolor[HTML]{ECF4FF}                     & \multicolumn{2}{l|}{\cellcolor[HTML]{F8FF00}Uniform}            & \multicolumn{2}{l|}{\cellcolor[HTML]{FFFFC7}Fine}               & \multicolumn{2}{l|}{\cellcolor[HTML]{F8FF00}Uniform}            & \multicolumn{2}{l|}{\cellcolor[HTML]{FFFFC7}Fine}               & \multicolumn{2}{l|}{\cellcolor[HTML]{F8FF00}Uniform}            & \multicolumn{2}{l|}{\cellcolor[HTML]{FFFFC7}Fine}               \\ \cline{4-15}
\multirow{-3}{*}{\cellcolor[HTML]{ECF4FF}$n$}     & \multirow{-3}{*}{\cellcolor[HTML]{ECF4FF}$h_1$} & \multirow{-3}{*}{\cellcolor[HTML]{ECF4FF}$h_2$} & \cellcolor[HTML]{329A9D}Case A & \cellcolor[HTML]{96FFFB}Case B & \cellcolor[HTML]{329A9D}Case A & \cellcolor[HTML]{96FFFB}Case B & \cellcolor[HTML]{329A9D}Case A & \cellcolor[HTML]{96FFFB}Case B & \cellcolor[HTML]{329A9D}Case A & \cellcolor[HTML]{96FFFB}Case B & \cellcolor[HTML]{329A9D}Case A & \cellcolor[HTML]{96FFFB}Case B & \cellcolor[HTML]{329A9D}Case A & \cellcolor[HTML]{96FFFB}Case B \\ \hline
\rowcolor[HTML]{C0C0C0}
\cellcolor[HTML]{9AFF99}                        & \cellcolor[HTML]{FE0000}                     & \cellcolor[HTML]{FFCE93}0                    & 0.5                            & 0.5                            & 0.5                            & 0.5                            & \cellcolor[HTML]{EFEFEF}0.72   & \cellcolor[HTML]{EFEFEF}0.964  & \cellcolor[HTML]{EFEFEF}0.7188 & \cellcolor[HTML]{EFEFEF}0.9688 & 0.5645                         & 0.4226                         & 0.5992                         & 0.5025                         \\ \cline{3-15}
\rowcolor[HTML]{EFEFEF}
\cellcolor[HTML]{9AFF99}                        & \multirow{-2}{*}{\cellcolor[HTML]{FE0000}1}  & \cellcolor[HTML]{F8A102}3                    & 0.5                            & 0.5                            & 0.5468                         & 0.5                            & \cellcolor[HTML]{C0C0C0}0.72   & \cellcolor[HTML]{C0C0C0}0.964  & \cellcolor[HTML]{C0C0C0}0.7164 & \cellcolor[HTML]{C0C0C0}0.9688 & 0.4991                         & 0.4676                         & 0.688                          & 0.5117                         \\ \cline{2-15}
\rowcolor[HTML]{C0C0C0}
\cellcolor[HTML]{9AFF99}                        & \cellcolor[HTML]{FD6864}                     & \cellcolor[HTML]{FFCE93}0                    & 0.5714                         & 0.5                            & 0.5891                         & 0.5                            & \cellcolor[HTML]{EFEFEF}0.716  & \cellcolor[HTML]{EFEFEF}0.964  & \cellcolor[HTML]{EFEFEF}0.7516 & \cellcolor[HTML]{EFEFEF}0.9688 & 0.7624                         & 0.447                          & 0.7611                         & 0.6907                         \\ \cline{3-15}
\rowcolor[HTML]{EFEFEF}
\cellcolor[HTML]{9AFF99}                        & \multirow{-2}{*}{\cellcolor[HTML]{FD6864}4}  & \cellcolor[HTML]{F8A102}3                    & 0.7403                         & 0.5                            & 0.7262                         & 0.517                          & \cellcolor[HTML]{C0C0C0}0.84   & \cellcolor[HTML]{C0C0C0}0.964  & \cellcolor[HTML]{C0C0C0}0.8296 & \cellcolor[HTML]{C0C0C0}0.9679 & 0.876                          & 0.5582                         & 0.8869                         & 0.7826                         \\ \cline{2-15}
\rowcolor[HTML]{C0C0C0}
\cellcolor[HTML]{9AFF99}                        & \cellcolor[HTML]{FFCCC9}                     & \cellcolor[HTML]{FFCE93}0                    & 0.8258                         & 0.598                          & 0.8421                         & 0.4994                         & \cellcolor[HTML]{EFEFEF}0.888  & \cellcolor[HTML]{EFEFEF}0.964  & \cellcolor[HTML]{EFEFEF}0.8868 & \cellcolor[HTML]{EFEFEF}0.9676 & 0.9592                         & 0.8097                         & 0.9498                         & 0.7132                         \\ \cline{3-15}
\rowcolor[HTML]{EFEFEF}
\multirow{-6}{*}{\cellcolor[HTML]{9AFF99}250}   & \multirow{-2}{*}{\cellcolor[HTML]{FFCCC9}12} & \cellcolor[HTML]{F8A102}3                    & 0.925                          & 0.5939                         & 0.8756                         & 0.5378                         & \cellcolor[HTML]{C0C0C0}0.936  & \cellcolor[HTML]{C0C0C0}0.956  & \cellcolor[HTML]{C0C0C0}0.9047 & \cellcolor[HTML]{C0C0C0}0.9651 & 0.9806                         & 0.6856                         & 0.9465                         & 0.7522                         \\ \hline
\rowcolor[HTML]{C0C0C0}
\cellcolor[HTML]{32CB00}                        & \cellcolor[HTML]{FE0000}                     & \cellcolor[HTML]{FFCE93}0                    & 0.5                            & 0.5                            & 0.5                            & 0.5                            & \cellcolor[HTML]{EFEFEF}0.73   & \cellcolor[HTML]{EFEFEF}0.973  & \cellcolor[HTML]{EFEFEF}0.7188 & \cellcolor[HTML]{EFEFEF}0.9688 & 0.5784                         & 0.3753                         & 0.4915                         & 0.4966                         \\ \cline{3-15}
\rowcolor[HTML]{EFEFEF}
\cellcolor[HTML]{32CB00}                        & \multirow{-2}{*}{\cellcolor[HTML]{FE0000}1}  & \cellcolor[HTML]{F8A102}3                    & 0.5                            & 0.5                            & 0.597                          & 0.5                            & \cellcolor[HTML]{C0C0C0}0.73   & \cellcolor[HTML]{C0C0C0}0.973  & \cellcolor[HTML]{C0C0C0}0.7539 & \cellcolor[HTML]{C0C0C0}0.9688 & 0.6095                         & 0.563                          & 0.7428                         & 0.5052                         \\ \cline{2-15}
\rowcolor[HTML]{C0C0C0}
\cellcolor[HTML]{32CB00}                        & \cellcolor[HTML]{FD6864}                     & \cellcolor[HTML]{FFCE93}0                    & 0.633                          & 0.5                            & 0.6404                         & 0.5                            & \cellcolor[HTML]{EFEFEF}0.779  & \cellcolor[HTML]{EFEFEF}0.973  & \cellcolor[HTML]{EFEFEF}0.7832 & \cellcolor[HTML]{EFEFEF}0.9688 & 0.78                           & 0.4325                         & 0.8282                         & 0.6443                         \\ \cline{3-15}
\rowcolor[HTML]{EFEFEF}
\cellcolor[HTML]{32CB00}                        & \multirow{-2}{*}{\cellcolor[HTML]{FD6864}4}  & \cellcolor[HTML]{F8A102}3                    & 0.7083                         & 0.5                            & 0.7846                         & 0.5                            & \cellcolor[HTML]{C0C0C0}0.813  & \cellcolor[HTML]{C0C0C0}0.973  & \cellcolor[HTML]{C0C0C0}0.8505 & \cellcolor[HTML]{C0C0C0}0.9688 & 0.8764                         & 0.4511                         & 0.9223                         & 0.4485                         \\ \cline{2-15}
\rowcolor[HTML]{C0C0C0}
\cellcolor[HTML]{32CB00}                        & \cellcolor[HTML]{FFCCC9}                     & \cellcolor[HTML]{FFCE93}0                    & 0.7919                         & 0.5                            & 0.8056                         & 0.5                            & \cellcolor[HTML]{EFEFEF}0.859  & \cellcolor[HTML]{EFEFEF}0.973  & \cellcolor[HTML]{EFEFEF}0.8734 & \cellcolor[HTML]{EFEFEF}0.9688 & 0.9246                         & 0.8589                         & 0.9528                         & 0.8417                         \\ \cline{3-15}
\rowcolor[HTML]{EFEFEF}
\multirow{-6}{*}{\cellcolor[HTML]{32CB00}1000}  & \multirow{-2}{*}{\cellcolor[HTML]{FFCCC9}12} & \cellcolor[HTML]{F8A102}3                    & 0.9233                         & 0.5662                         & 0.8846                         & 0.5                            & \cellcolor[HTML]{C0C0C0}0.948  & \cellcolor[HTML]{C0C0C0}0.974  & \cellcolor[HTML]{C0C0C0}0.9206 & \cellcolor[HTML]{C0C0C0}0.9688 & 0.9857                         & 0.5164                         & 0.9708                         & 0.504                          \\ \hline
\rowcolor[HTML]{C0C0C0}
\cellcolor[HTML]{009901}                        & \cellcolor[HTML]{FE0000}                     & \cellcolor[HTML]{FFCE93}0                    & 0.5                            & 0.5                            & 0.5                            & 0.5                            & \cellcolor[HTML]{EFEFEF}0.7184 & \cellcolor[HTML]{EFEFEF}0.9699 & \cellcolor[HTML]{EFEFEF}0.7188 & \cellcolor[HTML]{EFEFEF}0.9688 & 0.5895                         & 0.4685                         & 0.4982                         & 0.5156                         \\ \cline{3-15}
\rowcolor[HTML]{EFEFEF}
\cellcolor[HTML]{009901}                        & \multirow{-2}{*}{\cellcolor[HTML]{FE0000}1}  & \cellcolor[HTML]{F8A102}3                    & 0.5                            & 0.5                            & 0.5                            & 0.5                            & \cellcolor[HTML]{C0C0C0}0.7184 & \cellcolor[HTML]{C0C0C0}0.9699 & \cellcolor[HTML]{C0C0C0}0.7188 & \cellcolor[HTML]{C0C0C0}0.9688 & 0.6229                         & 0.5404                         & 0.6598                         & 0.5215                         \\ \cline{2-15}
\rowcolor[HTML]{C0C0C0}
\cellcolor[HTML]{009901}                        & \cellcolor[HTML]{FD6864}                     & \cellcolor[HTML]{FFCE93}0                    & 0.6286                         & 0.5                            & 0.6506                         & 0.5                            & \cellcolor[HTML]{EFEFEF}0.7853 & \cellcolor[HTML]{EFEFEF}0.9699 & \cellcolor[HTML]{EFEFEF}0.7771 & \cellcolor[HTML]{EFEFEF}0.9688 & 0.6932                         & 0.5407                         & 0.8395                         & 0.613                          \\ \cline{3-15}
\rowcolor[HTML]{EFEFEF}
\cellcolor[HTML]{009901}                        & \multirow{-2}{*}{\cellcolor[HTML]{FD6864}4}  & \cellcolor[HTML]{F8A102}3                    & 0.755                          & 0.5                            & 0.8019                         & 0.5                            & \cellcolor[HTML]{C0C0C0}0.8325 & \cellcolor[HTML]{C0C0C0}0.9699 & \cellcolor[HTML]{C0C0C0}0.8523 & \cellcolor[HTML]{C0C0C0}0.9688 & 0.8894                         & 0.5129                         & 0.9048                         & 0.7529                         \\ \cline{2-15}
\rowcolor[HTML]{C0C0C0}
\cellcolor[HTML]{009901}                        & \cellcolor[HTML]{FFCCC9}                     & \cellcolor[HTML]{FFCE93}0                    & 0.8703                         & 0.5085                         & 0.8175                         & 0.5                            & \cellcolor[HTML]{EFEFEF}0.9031 & \cellcolor[HTML]{EFEFEF}0.9704 & \cellcolor[HTML]{EFEFEF}0.8709 & \cellcolor[HTML]{EFEFEF}0.9688 & 0.9601                         & 0.8795                         & 0.9412                         & 0.7865                         \\ \cline{3-15}
\rowcolor[HTML]{EFEFEF}
\multirow{-6}{*}{\cellcolor[HTML]{009901}10000} & \multirow{-2}{*}{\cellcolor[HTML]{FFCCC9}12} & \cellcolor[HTML]{F8A102}3                    & 0.9354                         & 0.5                            & 0.9271                         & 0.6089                         & \cellcolor[HTML]{C0C0C0}0.9514 & \cellcolor[HTML]{C0C0C0}0.9699 & \cellcolor[HTML]{C0C0C0}0.9455 & \cellcolor[HTML]{C0C0C0}0.9741 & 0.9879                         & 0.5582                         & 0.9857                         & 0.8204                         \\ \hline
\end{tabular}
\end{table}

\subsection*{Setup}

A single-output feed-forward neural network with one/two hidden layers is modelled such that the number of hidden neurons in each hidden layer is $h_1 \in \{1, 4, 12\}$ and $h_2 \in \{0, 3\}$, respectively, with $h_2 = 0$ meaning that there is no second hidden layer. I used binary cross-entropy standard objective function as my optimization criterion. I ran the model for 1000 epochs (configurable hyperparameter). I used Adam optimizer in order to converge to a local optimum. There is no other early stopping criterion in my training. I prefer to use regularization only for $y$. I map $y = -1 \text{ to } y = 0$ and $y = 1 \text{ to } y = 1$. There is no regularization for $X$ as the data set values are mostly comparable and picked from a uniform distribution from $[-6, 6] \text{ and } [-4, 4]$ respectively. The expected mean is any way zero and the variance is low. I also don't see any threat of outliers. I didn't use any special initialization for weights. I went with default initialization provided by PyTorch which is choosing weights from \textbf{Uniform Distribution} from $\left(-\frac{1}{\sqrt{d}}, \frac{1}{\sqrt{d}}\right)$, where $d$ is the number of features. I used a hyperbolic tangent ($tanh$) activation function in each activation. The output layer is estimated using sigmoid function and the threshold I used to classify negatives and positives is 0.5. For each of the architectures, defined by a parameter combination $(h_1, h_2)$, I evaluated the performance of each model using balanced accuracy, classification accuracy, and area under the ROC curve. These are the outputs of my function $get\_metrics$. To evaluate the performance of my model, I used $k$-fold stratified cross-validation (with $k = 5$) by splitting the sample into $80-20$ ($80$ for training and $20$ for testing). To evaluate the performance of performance evaluation, I generated another very large dataset on a fine grid in X. The points in this grid are spaced at intervals of $0.1$ both horizontally and vertically. I calculated the predictions from my trained model on all these points to determine the true performance. In order to calculate the classification accuracy and balanced accuracy, I used the thresholded predictions and to calculate the area under the curve I used soft predictions.

In the above table, the columns are appropriately named. 'Uniform' refers to values obtained from uniform distribution and 'Fine' refers to data from fine grid (spaced at $0.1$). 'Case A' refers to relatively balanced case and 'Case B' refers to highly imbalanced case. The coloring of columns $n$, $h_1$, and $h_2$ are mapped as per the expected performance of the model.

\subsection*{Plots}

\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/1}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/2}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/3}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/4}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/5}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/6}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/7}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/8}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/9}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/10}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/11}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/12}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/13}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/14}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/15}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/16}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/17}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/18}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/19}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/20}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/21}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/22}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/23}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/24}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/25}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/26}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/27}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/28}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/29}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/30}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/31}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/32}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/33}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/34}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/35}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/36}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/37}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/38}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/39}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/40}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/41}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/42}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/43}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/44}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/45}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/46}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/47}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/48}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/49}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/50}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/51}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/52}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/53}
\end{figure}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{./3/54}
\end{figure}

\subsection*{How to read Plots}

Each of the 18 combinations of $(n, h_1, h_2)$ have 3 plots above. So, total number of plots are 54. The first plot out of the $3$ contains $2$ subplots, one containing the predictions of my network in balanced and the other contained predictions of my model in imbalanced scenarios. Each of the subplot includes outputs from all $5$ networks, $1$ each from every iteration of $5$-fold Cross Validation. The second plot contains $5$ different subplots on fine grid $1$ each for every iteration of $5$-fold Cross Validation for balanced scenario. The third plot contains $5$ different subplots on fine grid $1$ each for every iteration of $5$-fold Cross Validation for imbalanced scenario. To evaluate the performance of my performance evaluation, the plots $2$ and $3$ were drawn. The points are labeled as red dot markers for negative predictions and green cross markers for positive predictions.

\subsection*{Observations and Reasoning}

\begin{enumerate}
  \item First observation is that we can observe for the balanced scenario, Case A, for a particular $n$, the balanced accuracy, classification accuracy increases as we goes down the table. Which means the model is learning well as we increase the number of activations.
  \item Area under curve is different in almost every case because, I used soft-predictions to calculate its value. It is expected to be different because the model is trained with different configuration of activations each time. Also, it is a stochastic process and due to entropy, the value depends on the system on which the program is running. The general trend however is that it also increases as we go down the table, for a particular $n$. However there can be exceptions especially when the data is less and number of activations are less. For example, $\left(n, h_1, h_2\right) = \left(250, 1, 0\right)$ and $\left(n, h_1, h_2\right) = \left(250, 1, 3\right)$.
  \item In imbalanced class scenario (since we are choosing data from a uniform distribution), sometimes it might happen that we get points belonging to only one class as we are using $k$-fold cross validation with $k = 5$. In this case, balanced accuracy and classification accuracy would be $1.0$. However, the area under the ROC curve would be $np.nan$. When we take average of all the $k=5$ values, we still get a $np.nan$. That is why we sometimes see a $np.nan$ for area under the ROC curve in Case B. In such cases, the balanced accuracy is definitely above $0.5$ as one of the $k=5$ entries is a $1.0$.
  \item In imbalanced scenario, Case B, we can see that classification accuracy is high while the balanced accuracy is around $0.5$. Balanced accuracy is defined as the average of sensitivity (true positive rate) and specificity (true negative rate). Classification accuracy is defined as the proportion of correct predictions. This is why we should not use classification accuracy as a measure in case of imbalanced classes. We should rather prefer to use other performance metrics such as balanced accuracy or Matthews correlation coefficient. The results show that there is no learning in this case.
  \item It can be noticed from the above table that in most of the cases, it turns out that when number of activations in layer 1/layer 2 are less, the model is not learning the decision boundary properly. In case of imbalanced data, the configuration (values) of $h_1$ and $h_2$ we took in this problem is very less to learn the boundary. In case of balanced data, when the values of $h_1$ and $h_2$ are less, it doesn't learn the boundary properly. When they are relatively high $(h_1, h_2) = (12, 3)$ it is learning the boundary much better. I tried to increase the values even more $(h_1, h_2) = (100, 10)$ and it learns even better. However, in case of imbalanced classes, a configuration of $(h_1, h_2) = (1000, 20)$ also doesn't seem to learn the boundary well. This might mean that a neural network is not the right model for this problem or we can probably try adding more activations and layers.
  \item Also, the cross-validation is also 20 percent. Since the probability of getting a positive label in imbalanced data is very less it can also happen that very few positives are there in the training data (especially when $n$ is small) and the test data points are also not predicted correctly.
  \item The above conclusions are also inline with the low values for area under the ROC curve obtained in various scenarios. For example, the auc increases are we increase number of activations and decreases as the imbalance of classes increase.
  \item The results are also inline with the plots. However, results only provide the average case since were are use $k$-fold Cross Validation here with $k = 5$. If we want to know for each iteration of cross-validation, the performance of our model, plots are good tools. We can say that when number of activations are less, the learning is not so great. Still in some cases, there is learning in at least one scenario out of the $5$ iterations of cross validations. For example, the case when $\left(h_1, h_2\right) = \left(4, 0\right)$. As we increase the number of activations, the results are getting better for each iteration and the final output looks much better. Few other facts like when $\left(h_1, h_2\right) = \left(1, 0\right)$, the separation plane is sometimes like a plane because the situation here is close to (but not exactly) logistic regression.

Overall, it was a very fun exercise and I enjoyed it the most. I hope I covered almost all points. Kindly contact me if there is any discrepancy during evaluation.
\end{enumerate}

\end{solution}

\begin{problem}
Matrix factorization with applications.

a) (10 points) Derive the optimization steps for the ALS algorithm by finding formula $\# 1$ and formula $\# 2$ in the pseudocode listed above.

b) (20 points) Consider now that some values in $\mathbf{X}$ are missing (e.g., the rows of $\mathbf{X}$ are users and the columns of $\mathbf{X}$ are movie ratings, when available) and that we are interested in carrying out matrix completion using matrix factorization presented above. We would like to use the ALS algorithm, but the problem is that we must exclude all missing values from optimization. Derive now a modified ALS algorithm (formulas $\# 1$ and $\# 2$) to adapt it for matrix completion. Hint: consider adding an indicator matrix $\mathbf{W}$ to the optimization process, where $w_{ij}=1$ if $x_{ij}$ is available and $w_{ij}=0$ otherwise.

c) (20 points) Consider now a MovieLens database available at
$$
\text{http://grouplens.org/datasets/movielens/}
$$
and find a data set most appropriate to evaluate your algorithm from the previous step; e.g., one of the $100 \mathrm{k}$ data sets. Now, implement the ALS algorithm on your data set and evaluate it using the mean-squared-error as the criterion of success. You can randomly remove $25 \%$ of the ratings, train a recommendation system, and then test it on the test set. You will have to make certain decisions yourselves, such as initialization of $\mathbf{U}$ and $\mathbf{V}$, convergence criterion, or picking $k$ and $\lambda$.

d) (10 points) Describe your full experimentation process (e.g., how did you vary $k$ ) and observations from (c). Additionally, can you provide some reasoning as to what $k$ is and what matrices $\mathbf{U}$ and $\mathbf{V}$ are?

e) (10 points) Compare your method against the baseline that fills in every missing movie rating value $x_{ij}$ as an average over all users who have rated the movie $j$. Discuss your empirical findings.\end{problem}

\begin{solution}
\subsection*{a)}

Consider an $n \times d$ real-valued data matrix $\mathbf{X}=\left(\mathbf{x}_{1}^{T}, \mathbf{x}_{2}^{T}, \ldots, \mathbf{x}_{n}^{T}\right) = \left(\mathbf{x}_{1}, \mathbf{x}_{2}, \ldots, \mathbf{x}_{d}\right)$. We use the letter $i$ to denote the rows and $j$ to denote the columns. Let us approximate this matrix using the following factorization
$$
\hat{\mathbf{X}}=\mathbf{U} \mathbf{V}^{T}
$$
where $\mathbf{U}=\left(\mathbf{u}_{1}^{T}, \mathbf{u}_{2}^{T}, \ldots, \mathbf{u}_{n}^{T}\right)$ is an $n \times k$ and $\mathbf{V}=\left(\mathbf{v}_{1}^{T}, \mathbf{v}_{2}^{T}, \ldots, \mathbf{v}_{d}^{T}\right)$ is a $d \times k$ matrix of real numbers,
and where $k<n, d$ is a parameter to be explored and determined. The value $x_{ij}$ in $\mathbf{X}$ can be approximated by $\mathbf{u}_{i}^{T} \mathbf{v}_{j}$ and that $\mathbf{x}_{i}^{T},$ the $i$-th row of $\mathbf{X},$ can be approximated by $\mathbf{u}_{i}^{T} \mathbf{V}^{T},$ giving $\hat{\mathbf{x}}_{i}=\mathbf{V} \mathbf{u}_{i}$,  and $\mathbf{x}_{j},$ the $j$-th column of $\mathbf{X},$ can be approximated by $ \mathbf{U}\mathbf{v}_{j},$ giving $\hat{\mathbf{x}}_{j}=\mathbf{U} \mathbf{v}_{j}$. The matrix factorization process can be formulated as the following minimization
$$
\min _{\mathbf{U}, \mathbf{V}} \sum_{i, j}\left(x_{i j}-\mathbf{u}_{i}^{T} \mathbf{v}_{j}\right)^{2}+\lambda\left(\sum_{i}\left\|\mathbf{u}_{i}\right\|^{2}+\sum_{j}\left\|\mathbf{v}_{j}\right\|^{2}\right)
$$
which minimizes the sum-of-squared-errors between real values $x_{i j}$ and reconstructed values $\hat{x}_{i j}=\mathbf{u}_{i}^{T} \mathbf{v}_{j}$. The regularization parameter $\lambda \geq 0$ is user-selected. This problem can be directly solved using gradient descent, but we will attempt a slightly different approach. To do this, we can see that for a fixed $\mathbf{V}$ we can find optimal vectors $\mathbf{u}_{i}$ by minimizing,
$$
L_{1} = \left\|\mathbf{V} \mathbf{u}_{i}-\mathbf{x}_{i}\right\|^{2}+\lambda \cdot\left\|\mathbf{u}_{i}\right\|^{2}
$$
which can be solved in a closed form using OLS regression for every $i$. Taking the derivative and equating to $0$, we get,

\begin{align*}
	\pdv{L_1}{\mathbf{u}_i} &= \pdv{}{\mathbf{u}_i}\left(\mathbf{u}_i^{T}\mathbf{V}^{T}\mathbf{V}\mathbf{u}_i - \mathbf{u}_i^{T}\mathbf{V}^{T}\mathbf{x}_{i} - \mathbf{x}_{i}^{T}\mathbf{V}\mathbf{u}_i + \mathbf{x}_{i}^{T}\mathbf{x}_{i} + \lambda\mathbf{u}_i^{T}\mathbf{u}_i\right)\\
	&= 2\mathbf{V}^{T}\mathbf{V}\mathbf{u}_i - \mathbf{V}^{T}\mathbf{x}_i - \mathbf{V}^{T}\mathbf{x}_i + 0 + 2\lambda\mathbf{u}_i\\
	&= 2\left(\left(\mathbf{V}^{T}\mathbf{V}+\lambda\mathbf{I}\right)\mathbf{u}_i - \mathbf{V}^{T}\mathbf{x}_i\right)\\
	&= 0\\
\end{align*}
$$
	\implies \boxed{\mathbf{u}_i = \left(\mathbf{V}^{T}\mathbf{V}+\lambda\mathbf{I}\right)^{-1}\mathbf{V}^{T}\mathbf{x}_i}
$$

We can equivalently express the optimization for vectors $\mathbf{v}_{j}$ and find the solution for every $j$. It will be like minimizing,
$$
L_{2} = \left\|\mathbf{U} \mathbf{v}_{j}-\mathbf{x}_{j}\right\|^{2}+\lambda \cdot\left\|\mathbf{v}_{j}\right\|^{2}
$$
which can be solved in a closed form using OLS regression for every $j$. Please note that $\mathbf{x}_j$ here is the column vector of matrix $\mathbf{X}$. Taking the derivative and equating to $0$, we get,

\begin{align*}
	\pdv{L_2}{\mathbf{v}_j} &= \pdv{}{\mathbf{v}_j}\left(\mathbf{v}_j^{T}\mathbf{U}^{T}\mathbf{U}\mathbf{v}_j - \mathbf{v}_j^{T}\mathbf{U}^{T}\mathbf{x}_{j} - \mathbf{x}_{j}^{T}\mathbf{U}\mathbf{v}_j + \mathbf{x}_{j}^{T}\mathbf{x}_{j} + \lambda\mathbf{v}_j^{T}\mathbf{v}_j\right)\\
	&= 2\mathbf{U}^{T}\mathbf{U}\mathbf{v}_j - \mathbf{U}^{T}\mathbf{x}_j - \mathbf{U}^{T}\mathbf{x}_j + 0 + 2\lambda\mathbf{v}_j\\
	&= 2\left(\left(\mathbf{U}^{T}\mathbf{U}+\lambda\mathbf{I}\right)\mathbf{v}_j - \mathbf{U}^{T}\mathbf{x}_j\right)\\
	&= 0\\
\end{align*}
$$
	\implies \boxed{\mathbf{v}_j = \left(\mathbf{U}^{T}\mathbf{U}+\lambda\mathbf{I}\right)^{-1}\mathbf{U}^{T}\mathbf{x}_j}
$$

\subsection*{b)}

I am using the same notations as used in part a) (for ex: $i$ denotes rows and $j$ denotes columns). Let some values in $\mathbf{X}$ are missing. Consider an indicator matrix $\mathbf{W}$ to the optimization process, where $w_{ij}=1$ if $x_{ij}$ is available and $w_{ij}=0$ otherwise. The matrix factorization process can be modified and formulated as the following minimization,
$$
\min _{\mathbf{U}, \mathbf{V}} \mathbf{W} \odot \left\|\mathbf{X}-\mathbf{U} \mathbf{V}^{T}\right\|^{2}+\lambda\left(\left\|\mathbf{U}\right\|^{2}+\left\|\mathbf{V}\right\|^{2}\right)
$$
$$\text{(or)}$$
$$
\min _{\mathbf{U}, \mathbf{V}} \left\|\sqrt{\mathbf{W}} \odot \left(\mathbf{X}-\mathbf{U} \mathbf{V}^{T}\right)\right\|^{2}+\lambda\left(\left\|\mathbf{U}\right\|^{2}+\left\|\mathbf{V}\right\|^{2}\right)
$$
$$\text{(or)}$$
$$
\min _{\mathbf{U}, \mathbf{V}} \sum_{i, j}w_{ij}\left(x_{i j}-\mathbf{u}_{i}^{T} \mathbf{v}_{j}\right)^{2}+\lambda\left(\sum_{i}\left\|\mathbf{u}_{i}\right\|^{2}+\sum_{j}\left\|\mathbf{v}_{j}\right\|^{2}\right)
$$

where $\odot$ means the Hadamard product (element-wise product).

The above objective function minimizes the sum-of-squared-errors between real non-missing values $x_{ij}$ and reconstructed values $\hat{x}_{i j}=\mathbf{u}_{i}^{T} \mathbf{v}_{j}$. The regularization parameter $\lambda \geq 0$ is user-selected. This problem can be directly solved using gradient descent, but we will attempt a slightly different approach. To do this, we can see that for a fixed $\mathbf{V}$ we can find optimal vectors $\mathbf{u}_{i}$ by minimizing,
\begin{align*}
L_{1} &= \left\|\sqrt{\mathbf{W}^{i}}\left(\mathbf{V} \mathbf{u}_{i}-\mathbf{x}_{i}\right)\right\|^{2}+\lambda \cdot\left\|\mathbf{u}_{i}\right\|^{2}\\
      &= \left(\mathbf{V}\mathbf{u}_i-\mathbf{x}_i\right)^{T}\mathbf{W}^{i}\left(\mathbf{V}\mathbf{u}_i-\mathbf{x}_i\right) + \lambda\mathbf{u}_i^{T}\mathbf{u}_i
\end{align*}
where, $\mathbf{W}^{i}$ is a diagonal matrix whose elements in diagonal are the elements of $i^{\text{th}}$ row of matrix $\mathbf{W}$, i.e., $\mathbf{W}^{i} = \text{diag}\left(w_{i1}, w_{i2}, ..., w_{id}\right)$.

This problem can be solved in a closed form using OLS regression for every $i$. Taking the derivative and equating to $0$, we get,

\begin{align*}
	\pdv{L_1}{\mathbf{u}_i} &= \pdv{}{\mathbf{u}_i}\left(\left(\mathbf{V}\mathbf{u}_i-\mathbf{x}_i\right)^{T}\mathbf{W}^{i}\left(\mathbf{V}\mathbf{u}_i-\mathbf{x}_i\right) + \lambda\mathbf{u}_i^{T}\mathbf{u}_i\right)\\
	&= -2\mathbf{V}^{T}\mathbf{W}^{i}\left(\mathbf{x}_i-\mathbf{V}\mathbf{u}_i\right) + 2\lambda\mathbf{u}_i\\
	&= 2\left(-\mathbf{V}^{T}\mathbf{W}^{i}\mathbf{x}_i+\mathbf{V}^{T}\mathbf{W}^{i}\mathbf{V}\mathbf{u}_i+\lambda\mathbf{u}_i\right)\\
	&= 0\\
\end{align*}
$$
	\implies \boxed{\mathbf{u}_i = \left(\mathbf{V}^{T}\mathbf{W}^{i}\mathbf{V}+\lambda\mathbf{I}\right)^{-1}\mathbf{V}^{T}\mathbf{W}^{i}\mathbf{x}_i}
$$

We can equivalently express the optimization for vectors $\mathbf{v}_{j}$ and find the solution for every $j$. It will be like minimizing,
\begin{align*}
L_{2} &= \left\|\sqrt{\mathbf{W}^{j}}\left(\mathbf{U} \mathbf{v}_{j}-\mathbf{x}_{j}\right)\right\|^{2}+\lambda \cdot\left\|\mathbf{v}_{j}\right\|^{2}\\
      &= \left(\mathbf{U}\mathbf{v}_j-\mathbf{x}_j\right)^{T}\mathbf{W}^{j}\left(\mathbf{U}\mathbf{v}_j-\mathbf{x}_j\right) + \lambda\mathbf{v}_j^{T}\mathbf{v}_j
\end{align*}

where, $\mathbf{W}^{j}$ is a diagonal matrix whose elements in diagonal are the elements of $j^{\text{th}}$ column of matrix $\mathbf{W}$, i.e., $\mathbf{W}^{j} = \text{diag}\left(w_{1j}, w_{2j}, ..., w_{nj}\right)$.

This problem can be solved in a closed form using OLS regression for every $j$. Please note that $\mathbf{x}_j$ here is the column vector of matrix $\mathbf{X}$. Taking the derivative and equating to $0$, we get,

\begin{align*}
	\pdv{L_2}{\mathbf{v}_j} &= \pdv{}{\mathbf{v}_j}\left(\left(\mathbf{U}\mathbf{v}_j-\mathbf{x}_j\right)^{T}\mathbf{W}^{j}\left(\mathbf{U}\mathbf{v}_j-\mathbf{x}_j\right) + \lambda\mathbf{v}_j^{T}\mathbf{v}_j\right)\\
	&= -2\mathbf{U}^{T}\mathbf{W}^{j}\left(\mathbf{x}_j-\mathbf{U}\mathbf{v}_j\right) + 2\lambda\mathbf{v}_j\\
	&= 2\left(-\mathbf{U}^{T}\mathbf{W}^{j}\mathbf{x}_j+\mathbf{U}^{T}\mathbf{W}^{j}\mathbf{U}\mathbf{v}_j+\lambda\mathbf{v}_j\right)\\
	&= 0\\
\end{align*}
$$
	\implies \boxed{\mathbf{v}_j = \left(\mathbf{U}^{T}\mathbf{W}^{j}\mathbf{U}+\lambda\mathbf{I}\right)^{-1}\mathbf{U}^{T}\mathbf{W}^{j}\mathbf{x}_j}
$$

\subsection*{c) \&  d)}

For code, please refer to \textbf{MatrixFactorization.ipynb}.

I chose the \texttt{ml-100k} dataset\cite{movielens100} from MovieLens. This data set consists of:
\begin{itemize}
	\item 100,000 ratings (1-5) from 943 users on 1,682 movies.
	\item Each user has rated at least 20 movies.
	\item Simple demographic info for the users (age, gender, occupation, zip)
\end{itemize}

The data was collected through the MovieLens web site (movielens.umn.edu) during the seven-month period from September 19th, 1997 through April 22nd, 1998. This data has been cleaned up - users who had less than 20 ratings or did not have complete demographic information were removed from this data set. Detailed descriptions of the data file can be found at the end of this file.

The dataset \texttt{ml-100/u.data} contains $4$ columns, namely, \texttt{user\_id}, \texttt{movie\_id}, \texttt{rating} and \texttt{timestamp}. There are totally $100,000$ entries in this data. There are a total of $n = 943$ users and $d = 1,682$ movies. Since, $100000 \ne 943 \times 1682$, we must be having missing values in matrix $\textbf{X}$ with $n$ rows and $d$ columns. Our goal is to represent this matrix as product of $\textbf{U}$ and $\textbf{V}^{T}$ as per the objective function mentioned in parts a) and b). In order to take care of missing values, an indicator matrix $\textbf{W}$ is formed such that $w_{ij} = 0$ if $x_{ij}$ is missing, else $w_{ij} = 1$. Since, the input is highly sparse, I am unable to extract an exact $25 \%$ of the set to use it as testing data. If I try to extract $25 \%$ randomly, there is a chance that there can be columns with all zeroes in the training data and this implies that the training may not be so effective. Hence, I chose $100$ columns for every user from the training set where the input is dense (i.e., from a set of top $1000$ values where there are so many user ratings for the movies). If this is not working for some reason in your computer, please try re-running or reducing the value $100$. This strategy had been doubled checked with Professor during office hours and got a confirmation that this is a valid strategy. So, I have training and test data matrices of same size as input $n \times k$, but training data matrix contains $100n$ extra missing values shifted to test data matrix as per the logic mentioned above. I also made sure that training and test dataset are disjoint. Indirectly, the split is also a hyperparameter (although not a standard one). Feel free to try different values. The implementation of ALS is fairly straightforward as it is finding the vectors $\textbf{u}_i$ and $\textbf{v}_j$ as shown above.

The most important hyperparameter in the entire exercise is $k$. I varied $k$ according to the values in the list \texttt{k\_list = [5, 10, 20, 40, 80]}. $k$ represents the so called \textbf{Concepts} or \textbf{Latent Factors}. If there is knowledge the data is coming from $k$ categories, we can simply use that domain knowledge and achieve great performances. In this example of movies and ratings, $k$ can be thought of as \textbf{genres} of the movies. This hyperparameter is similar to the hyperparameter $y$ in \textbf{EM-Algorithm}. Even there if we have prior knowledge about the data that is comes from $y$ distributions, we can calculate the parameters of the actual distributions effectively. Because of this presence of latent variable, the \textbf{MatrixFactorization} can be thought of as an unsupervised learning problem. It is well known in the industry as \textbf{Collaborative Filtering}. In this problem, I have considered that the weights to be uniform (either $0$ or $1$ depending on values are missing or available). Further reasearch is actively going on on this and we can even have non-uniform weights\cite{nonuniform_mf} to obtain a beter low rank matrix factorization for real world problems.

The initialization of matrices \textbf{U} and \textbf{V} are done randomly using the function \texttt{numpy.random.random\_sample} which return values from the 'continuous uniform' distribution over the interval \texttt{[0.0, 1.0)}. The final output matrix

\begin{enumerate}
	\item $\mathbf{U} \in \mathbb{R}^{n \times k}$ is a user embedding matrix, where row $i$ is the embedding for user $i$.
	\item $\mathbf{V} \in \mathbb{R}^{d \times k}$ is an item embedding (in this case items are movies) matrix, where row $j$ is the embedding for item (movie) $j$.
\end{enumerate}

One more hyperparameter involved is regularization constant $\lambda$. It is a value chosen from the list \texttt{lmd\_list = [0.1, 2, 5, 10, 25, 50, 100]} once per iteration. Number of iterations (epochs) is another hyperparamter. It is a value chosen from the list \texttt{epochs\_list = [1, 2, 5, 10]} once per iteration and trained incrementally to avoid over-training. Refer to my code for a better understanding.

The evaluation of the predictions is done using $mean\_square\_error$, which considers only non-zero/non-missing values from the input.

After this entire experiment, the best combination of hyperparamters (with the corresponding train and test errors) obtained for my experiment are as follows:

\begin{lstlisting}[language=Python]
{'epochs': 10,
 'k': 5,
 'lambda': 5,
 'train_error': 0.6672904167634547,
 'test_error': 0.8236433053485411}
\end{lstlisting}

The whole experiment takes approximately $15$ minutes to run and get results. I highly recommend using Google Colab (setup below) in order to verify the results:

\texttt{https://colab.research.google.com/drive/1u8FoCOaL9ugfm8udb7MYYgFeOyLQPObW?usp=sharing}\\

A random seed is already set to $42$ in order to replicate the results. Make sure to change path to file in Data Cell numbered [2].\\

\subsection*{e)}

In order to see how the model performs as compared to mean, I have taken the train dataset and fill those $100n$ values transferred to test data with the mean of rest of the non zero values in the column. This strategy has been quadruple checked with Professor in office hours. Once, the mean is imputed in those removed cells of training data, I call the $get\_mse$ method and calculate the mean square error using those $100n$ values where the cells in test data has non-zero ratings. This is done to see how our model performs when compared to mean prediction. Here are the results:

\begin{lstlisting}[language=Python]
{'k': 5,
 'lambda': 5,
 'epochs': 10,
 'train_error': 0.6672904167634547,
 'test_error': 0.8236433053485411,
 'mean_error': 0.3025307958188201}
\end{lstlisting}

From the above results, it can be noticed that for $k = 5$ and $\lambda = 5$ with $10$ epochs, we get train\_error as $0.667$, test\_error as $0.824$ and $mean\_error$ as $0.303$. The train\_error and test\_error are relatively comparable and also quite less ($< 1$) in magnitude. This means that our model is robust. However, when compared to test\_error, mean\_error is quite less. Although there is a chance that our model is performing poorly compared to the basic mean prediction, I think this is a misleading conclusion because our observations is based out of training $943 \times 1582$ data points and validating against $943 \times 100$ data points. This is relatively sort of over-fitting and I see there is a chance of high variance involved in my experiment. However, I was unable to avoid it, as I want to select records such that I have at least some non zero ratings for every movie in the training data. The model however, turned out to be robust. I am anyway left with only $943 \times 100$ records in order to compute mean square error and compare mean prediction with my model predictions. If we have more testing data, it might most likely turn out that I will get a high mean\_error. Or the vice versa can also happen if my model has high variance which is unlikely as the difference between train error and test error is very less. However, one more important point to note is that, it makes more sense to think that mean predictions are not bad in this problem of user-movie rating because if $p$ persons are liking/disliking a particular movie, it is most likely that $p + 1^{th}$ also likes/dislikes it. The distribution of mean ratings for different samples of a paricular movie can be assumed to approximately follow a normal distribution with low variance and almost tending to a dirac delta spike according to weak law of large numbers\cite{lln}.

\end{solution}

\begin{problem}
Prove representational equivalence of a three-layer neural network with linear activation function in all neurons and a single-layer layer neural network with the same activation function. Assume a single-output network.
\end{problem}

\begin{solution}
\begin{figure}[H]
\includegraphics[width=16cm, keepaspectratio]{NeuralNetwork}
\caption{A three-layer neural network}
\end{figure}

The above neural network, consists of $1$ input layer, $2$ hidden layers and $1$ output layer. Here, $a^{[i]}$ represents the vector of activations present in the $i^{\text{th}}$ layer, $w^{[i]}$ represents the vector of weights for activations present in the $i^{\text{th}}$ layer. $X$ is the input feature vector of $d$-dimensions. Hidden layer 1 contains $d_1$-activations and a bias, hidden layer 2 contains $d_2$-activations and a bias. $\hat{y}$ represents the output. $a^{[i]}_{j}$ represents $j^{\text{th}}$ activation in the $i^{\text{th}}$ layer. $w^{[i]}_{jk}$ represents the weight connection $j^{\text{th}}$ activation present in $i^{\text{th}}$ layer to the $k^{\text{th}}$ activation present in the $i-1^{\text{th}}$ layer. $x_i$ represents the $i^{\text{th}}$ input data point.

Let, $d_0 = d$ for notation simplification. Let us consider a linear activation function $h(z) = c_1z +c_2$.

Applying the activation function to the activations, we can construct $d_{l-1}$ linear combinations of activations in $l-1^{\text{th}}$ layer, after which the $j_{\text{th}}$ activation in the $l^{\text{th}}$ layer would look like,
\begin{align*}
a^{[l]}_{j} &= \sum_{i = 1}^{d_{l-1}} w^{[l]}_{ji}h\left(a^{[l-1]}_{i}\right) + w^{[l]}_{j0}\\
			  &= \sum_{i = 1}^{d_{l-1}} w^{[l]}_{ji}\left(c_1a^{[l-1]}_{i}+c_2\right) + w^{[l]}_{j0}\\
			  &= \sum_{i = 1}^{d_{l-1}} c_1w^{[l]}_{ji}a^{[l-1]}_{i}+\sum_{i = 1}^{d_{l-1}}c_2 w^{[l]}_{ji} + w^{[l]}_{j0}\\
			  &= \sum_{i = 1}^{d_{l-1}} {w^{'}}^{[l]}_{ji}a^{[l-1]}_{i} + {w^{'}}^{[l]}_{j0}\\
			  &= \sum_{i = 0}^{d_{l-1}} {w^{'}}^{[l]}_{ji}a^{[l-1]}_{i}
\end{align*}

where in the above equations constants are absorbed and the weights of the new terms are represented by $w{'}$ that are linear tranformations of the original ones.

Let's try to represent activations of layer $l$ in terms of $l - 2$.
\begin{align*}
a^{[l]}_{j} &= \sum_{i = 0}^{d_{l-1}} {w^{'}}^{[l]}_{ji}a^{[l-1]}_{i}\\
			  &= \sum_{i = 0}^{d_{l-1}} {w^{'}}^{[l]}_{ji}\left(\sum_{k = 0}^{d_{l-2}} {w^{'}}^{[l-1]}_{jk}a^{[l-2]}_{k}\right)\\
			  &= \sum_{k = 0}^{d_{l-2}} {w^{''}}^{[l-1]}_{jk}a^{[l-2]}_{k}
\end{align*}

where in the above equations constants are absorbed and the weights of the new terms are represented by $w{''}$ that are linear tranformations of the ones obtained in the previous steps.

Since, the above tranformation is linear (similar to Linear Regression), we can skip hidden layer 1 and go to hidden layer 2, with similar weights and skip layer 2 and go to layer 3 (output layer) with similar weights. The final representation of the neural network from layer 0 (input layer) to layer 3 (output layer) can be represented as,

$$
a^{[3]}_{j} = \sum_{i = 0}^{d_0} {w^{'''}}^{[0]}_{ji}a^{[0]}_{i}
$$

or

\begin{equation}
\hat{y} = \sum_{i = 0}^{d} {w^{'''}}^{[0]}_{ji}x_{i}
\end{equation}

If we have a $1$-layered neural network with $1$-output and linear activation function, this can be thought similar to Linear Regression, whose general form is,

\begin{equation}
\hat{y} = \sum_{i = 0}^{d} {w}^{[0]}_{ji}x_{i}
\end{equation}

Equations $(1)$ and $(2)$ are representationally equivalent. Hence, proved.

\end{solution}

\begin{problem}
Let $A, B, C,$ and $D$ be binary input variables (features). Give decision trees to represent the following Boolean functions:

a) $(3$ points $) A \wedge \bar{B}$

b) $(3$ points $) A \vee(\bar{B} \wedge C)$

c) $(4$ points $) A \oplus \bar{B}$\\
where $\bar{A}$ is the negation of $A$ and $\oplus$ is an exclusive OR operation.
\end{problem}

\begin{solution}
We can replace 'T' with $1$ and 'F' with $0$ and still obtain the same outputs in the end.
\begin{figure}[H]
\includegraphics[width=5.5cm, keepaspectratio]{Problem6_a}
\caption{a) $A \wedge \bar{B}$}
\end{figure}
\begin{figure}[H]
\includegraphics[width=5.5cm, keepaspectratio]{Problem6_b}
\caption{b) $A \vee(\bar{B} \wedge C)$}
\end{figure}
\begin{figure}[H]
\includegraphics[width=5.5cm, keepaspectratio]{Problem6_c}
\caption{c) $A \oplus \bar{B}$}
\end{figure}

\end{solution}

\medskip

\begin{thebibliography}{9}
\bibitem{movielens100}
F. Maxwell Harper and Joseph A. Konstan. 2015. \textit{The MovieLens Datasets: History and Context. ACM Transactions on Interactive Intelligent Systems (TiiS) 5, 4, Article 19 (December 2015), 19 pages.}
\\\texttt{DOI=http://dx.doi.org/10.1145/2827872}

\bibitem{nonuniform_mf}
Xiangnan He, Jinhui Tang, Senior Member, IEEE, Xiaoyu Du, Richang Hong, Member, IEEE, Tongwei Ren, Member, IEEE, and Tat-Seng Chua. 2018. \textit{Fast Matrix Factorization with Non-Uniform Weights on Missing Data}
\\\texttt{DOI=https://arxiv.org/pdf/1811.04411.pdf}

\bibitem{lln}
Routledge, R. (2016, October 12). \textit{Law of large numbers. Encyclopedia Britannica.}
\\\texttt{URL: https://www.britannica.com/science/law-of-large-numbers}
\end{thebibliography}

\end{document}

\end{document}


\section{Initial Experiments}

We start with finding the ranking for nodes recommended in an existing network. We consider a synthetic network $G(N,m,f,h)$ with parameters $N=1000, m=2, f=0.2$ as fixed parameters, and $h$ being varied from [0.0, 1.0], step-size=0.1.

We show in figure \ref{fig-hom} the initial structure of the network along with degree distribution and degree growth for the network, which is grown considering only preferential-attachment and homophily as in equation \ref{pref-hom}. We find our results similar to \cite{karimi2018homophily}, showing similar network characteristics.

\begin{figure}[hbt!]
	\centering
	\includegraphics[scale=0.23]{images/proposal_figure_1}
	\caption{Network grown using preferential-attachment and homophily. The minority fraction has been set fixed to 0.2, number of edges added per iteration is 2. The homophily parameter ranges from 0.0(complete heterophily) to 1.0(complete homophily). Tile A shows a visualization of the network with 100 nodes, for different homophily parameters. For Tile B and C, the network contains 1000 nodes, averaged over 5 iterations.}
	\label{fig-hom}
\end{figure}

\begin{figure}
	\includegraphics[scale=0.28]{images/proposal_figure_3_1}
	\caption{Fraction of minorities recommended in the top 5 ranks for the two node groups. On the left hand side is the recommendation provided to minority nodes, while on the right-hand side are the recommendations provided to majority nodes. The 3 different ranking schemes are shown with three colors. The network considered has 1000 nodes, with a minority fraction of 0.05.}
	\label{fig-rank_1}
\end{figure}

\begin{figure}
	\includegraphics[scale=0.28]{images/proposal_figure_3_2}
	\caption{Fraction of minorities recommended in the top 5 ranks for the two node groups. On the left hand side is the recommendation provided to minority nodes, while on the right-hand side are the recommendations provided to majority nodes. The 3 different ranking schemes are shown with three colors. The network considered has 1000 nodes, with a minority fraction of 0.1.}
	\label{fig-rank_2}
\end{figure}
\begin{figure}
	\includegraphics[scale=0.28]{images/proposal_figure_3_3}
	\caption{Fraction of minorities recommended in the top 5 ranks for the two node groups. On the left hand side is the recommendation provided to minority nodes, while on the right-hand side are the recommendations provided to majority nodes. The 3 different ranking schemes are shown with three colors. The network considered has 1000 nodes, with a minority fraction of 0.2.}
	\label{fig-rank_3}
\end{figure}
\begin{figure}
	\includegraphics[scale=0.28]{images/proposal_figure_3_4}
	\caption{Fraction of minorities recommended in the top 5 ranks for the two node groups. On the left hand side is the recommendation provided to minority nodes, while on the right-hand side are the recommendations provided to majority nodes. The 3 different ranking schemes are shown with three colors. The network considered has 1000 nodes, with a minority fraction of 0.3.}
	\label{fig-rank_4}
\end{figure}
\begin{figure}
	\includegraphics[scale=0.28]{images/proposal_figure_3_5}
	\caption{Fraction of minorities recommended in the top 5 ranks for the two node groups. On the left hand side is the recommendation provided to minority nodes, while on the right-hand side are the recommendations provided to majority nodes. The 3 different ranking schemes are shown with three colors. The network considered has 1000 nodes, with a minority fraction of 0.4.}
	\label{fig-rank_5}
\end{figure}
\begin{figure}
	\includegraphics[scale=0.28]{images/proposal_figure_3_6}
	\caption{Fraction of minorities recommended in the top 5 ranks for the two node groups. On the left hand side is the recommendation provided to minority nodes, while on the right-hand side are the recommendations provided to majority nodes. The 3 different ranking schemes are shown with three colors. The network considered has 1000 nodes, with a minority fraction of 0.5.}
	\label{fig-rank_6}
\end{figure}

For the network, with varying homophily, we run our Multi-Armed Bandits approach of ranking for all the nodes as devised in \cite{radlinski2008learning}. We take the number of slots to be 5, thus training for 5 ranking spots per node in the network. The underlying MAB algorithm being used is $\epsilon$-Greedy (with $\epsilon=0.1$). We use the sample-average method for estimating the best action values, where actions are basically the choice of nodes among the network to recommend. The reward function returns 1, when the node recommended is chosen by our clicking model, and a reward value of 0 if the node recommended is not chosen. Details about the node choice model is provided in the Methods section. We train our recommender system for $10^5$ iterations. We then see at which position nodes are ranked, based on the node group asking for recommendation. We also run the TopRank algorithm \cite{lattimore2018toprank} and the Adamic-Adar Index to compare our results. Figure \ref{fig-rank_1} - Figure \ref{fig-rank_6} shows our results for different minority groups.

For the complete hetrophilic case, a sharp difference can be seen in the Adamic-Adar ranking and the Ranked-Bandits ranking. This is evident since the Adamic-Adar Index does not consider the homophily value for making link predictions, since the minority nodes have majority nodes as their neighbours always and they in turn have only minority neighbours, other minority nodes are recommended minority nodes. The Ranked-Bandit however considers the homophily parameter and hence the recommendations are more aligned to the idea of homophily.

\bigskip


\section{Abstract}
	The project is composed by two parts:
	\begin{itemize}
		\item Implementing an Integer Linear Programming model with the Cplex API
		\item Implementing a meta-heuristic solution method
	\end{itemize}
	For the second part my choice of method is a genetic algorithm, details on implementation and decisions in section 3.
	Both solution methods will be applied to the following problem:
	\blockquote{A company produces boards with holes used to build electric frames.  Boards are positioned over a machines and a drill moves over the board, stops at the desired positions and makes the holes.  Once a board is drilled, a new board is positioned and the process is iterated many times.  Given the position of the holes on the board, the company asks us to determine the hole sequence that minimizes the total drilling time, taking into account that the time needed for making an hole is the same and constant for all the holes.}
	The two different approaches are then tested using some benchmark tests from literature and compared to spot differences in results (optimality, execution time, reliability, ...)


\documentclass[conference]{IEEEtran}
\usepackage{graphicx}
\usepackage[strings]{underscore}
\usepackage{biblatex}

%Title
\title{Exploring Otsu's Method}
\author{
    \IEEEauthorblockN{Sotheanith Sok}
    \IEEEauthorblockA{
        Department of Engineering\\
        California State University of Long Beach\\
        Sotheanith.Sok@student.csulb.edu
    }
}
\date{September 9 2021}

\addbibresource {refs.bib}

\begin{document}
\maketitle

\section{Introduction}
In computer vision, image segmentation is the process of dividing an image into multiple segments and these segments tend to simplify the image which will make it easier to analyze such image for other purposes such as boundary locating, object finding, and many more. One of the simpler types of image segmentation is splitting a gray-scale image between a foreground area and a background area by picking a pixel intensity threshold that separates the two areas. Unfortunately, picking the pixel intensity threshold manually is not efficient as any given threshold can only work for a selective group of images. Thus, a more adaptive approach is necessary and Otsu's method is one of them.

Otsu's method, named after Nobuyuki Otsu, is an algorithm for automatically determining a pixel intensity threshold that separates a background area from a foreground area of any gray-scale image. This is possible under the assumption that there is a bi-modal distribution, with each peak represents a background area or a foreground area, when examining the histogram generated from pixel intensities and pixel counts of a given image \cite{wikipedia-contributors:2020}.

\begin{figure}[!htb]
    \centering
    \includegraphics[width=\linewidth]{otsu_example.png}
    \caption{Original image, binary image, and histogram used or generated by Otsu's method \cite{scipy-lectures:2020}.}
\end{figure}

\section{The Algorithm}
In order to find the pixel intensity threshold, Otsu's method will perform an exhaustive search of all pixel intensity values and select a value that maximizes the inter-class variance. For any given image, the algorithm starts by generating a histogram representing all pixel intensities and the numbers of pixels that have such intensities \cite{tay:2020}. Then, according to \cite{greensted:2010}, an inter-class variance can be represented as
\[\sigma_w^2 = W_f\sigma_f^2 + W_b\sigma_b^2\]
where:
\begin{description}
    \item[$\sigma_w^2$] is the inter-class variance.
    \item[$W_f$] is the weight of the foreground area.
    \item[$\sigma_f^2$] is the variance of the foreground area.
    \item[$W_b$] is the weight of the background area.
    \item[$\sigma_b^2$] is variance of the background area.
\end{description}
. Weights of the foreground area and the background area can be calculated with the following formula
\[W_b =  \frac{\sum_{i=0}^{t-1} P(i)}{\sum_{i=0}^{all} P(i)} \]
\[W_f =  \frac{\sum_{i=t}^{all} P(i)}{\sum_{i=0}^{all} P(i)} \]
where:
\begin{description}
    \item[$W_b$] is the weight of the background area.
    \item[$W_f$] is the weight of the foreground area.
    \item[$t$] is the pixel intensity threshold.
    \item[$P(i)$] is the pixel count at intensity $i$.
\end{description}
The remaining factors, which are variances of both the background area and the foreground area, can be calculated from the mean of each area respectively with the following formula
\[\mu_b = \frac{\sum_{i=0}^{t-1} i P(i)}{\sum_{i=0}^{t-1}P(i)}\]
\[\sigma_b^2 = \frac{\sum_{i=0}^{t-1}(i-\mu_b)^2 \times P(i)}{\sum_{i=0}^{t-1}P(i)}\]
\[\mu_f = \frac{\sum_{i=t}^{all} i P(i)}{\sum_{i=t}^{all}P(i)}\]
\[\sigma_f^2 = \frac{\sum_{i=t}^{all}(i-\mu_f)^2 \times P(i)}{\sum_{i=t}^{all}P(i)}\]
where:
\begin{description}
    \item[$\mu_b$]is the mean of the background area.
    \item[$\mu_f$]is the mean of the foreground area.
    \item[$\sigma_b^2$] is the variance of the background area.
    \item[$\sigma_f^2$] is the variance of the foreground area.
    \item[$t$] is the intensity threshold.
    \item[$P(i)$] is the pixel count at $i$ intensity.
\end{description}
The entire process is repeated for all pixel intensities and the one that produces the largest inter-class variance will be picked as the final pixel intensity threshold.

\section{Limitation}
While Otsu's method performs well on most gray-scale images, it still has some problems when processing images that do not produce the bi-modal distribution with high peaks and deep valleys \cite{kang-and-atul:2019}.
\begin{figure}[!htb]
    \centering
    \includegraphics[width=\linewidth]{small_object_image.png}
    \caption{Otsu's method performance on an image with a small foreground area \cite{altphotos:2017}.}
\end{figure}
To start with, any image that has a foreground area significantly smaller than a background area tends to not do well with Otsu's method due to the lack of pixels count in a foreground area which is often necessary for generating a high peak in the image's histogram.
\begin{figure}[!htb]
    \centering
    \includegraphics[width=\linewidth, scale=1.5]{noisy_image.png}
    \caption{Otsu's method performance on an image that contains a lot of noise \cite{qrcode:2012}.}
\end{figure}
Then, there are images with a lot of noise. This type of image also does not perform well with Otsu's method because noises in such images often prohibit the formation of bi-modal distribution in the histogram.
\begin{figure}[!htb]
    \centering
    \includegraphics[width=\linewidth]{non-uniform illumination_image.png}
    \caption{Otsu's method performance on an image with non-uniform illumination \cite{bird:2014}.}
\end{figure}
Last but not least, the Otsu's method performs poorly on images with non-uniform illumination because it uses a single global threshold to separate the background area and the foreground area. To simply put, for such an image, a section of the background area might have the same illumination as another section of the foreground area and vice versa.

\section{Final Thought}
Otsu' method is a great choice for finding a global intensity threshold for splitting a gray-scale image into a foreground area and a background area. However, due to the nature of the algorithm which relies heavily on an image's intensities having a bi-modal distribution, it should not be applied universally to all gray-scale images. To overcome the limitations of Otsu's method, preprocessing techniques such as image denoising and others should be used before applying Otsu' method onto an image.

\printbibliography

\end{document}


\graphicspath{{Pics/}}

\section{First Portion}

	\lem{}{Let the incircle and excircle (opposite to $A$) of $\triangle ABC$ meet $BC$ at $D$ and $E$ resp. Suppose $F$ is the antipode of $D$ wrt the incircle.

	\begin{enumerate}
		\item Prove that $A,F,E$ are collinear.
		\item  $M$ be the midpoint of $DE$. Prove that $MI$ meets $AD$ at it's midpoint.
	\end{enumerate}

	\figdf{.5}{basics_lem_2}{}}

	\lem{}{Let the incircle of $\triangle ABC$ meets $AB$ and $AC$ at $X$ and $Y$ resp. $BI$ and $CI$ meet $XY$ at $P$ and $Q$ respectively. Prove that $BPQC$ is cyclic. (In fact $BP\perp CP$ and $BQ\perp CQ$)
	\figdf{.5}{basics_lem_2}{}}

	\lem{}{$AD$ is an altitude of $\triangle ABC$. $E,F$ are on $AC,AB$ so that $AD,BE,CF$ are concurrent. Prove $\angle EDA=\angle FDA$.}

	\lem{}{Let $AD$ be an altitude of $\triangle ABC$ and $E\in \bigodot ABC$ so that $AE\parallel BC$. Prove that $D,G,E$ are collinear where $G$ is the centroid of $\triangle ABC$.
	\figdf{.5}{basics_lem_3}{}}

	\prob{}{}{B}{Let $O$ be the circumcenter of $\triangle ABC$ and $A',B',C'$ are reflections of $O$ on $BC,CA,AB$ resp. Prove that $AA',BB',CC'$ are concurrent.}

	\prob{}{}{B}{Let $D,E$ are on sides $AC,AB$ of $\triangle ABC$ resp. such that $BE=CD$. Let $\bigodot ABC \cap \bigodot ADE= P$. Prove that $PB=PC$.}

	\prob{}{}{B}{Let a line $PQ$ touch circle $S_1$ and $S_2$ at $P$ and $Q$ resp. Prove that the radical axis of $S_1$ and $S_2$ passes through the midpoint of $PQ$.}

	\prob{}{}{B}{Let $\omega_1,\omega_2,omega_3$ are $3$ circles. Prove that the $3$ radical axis of $\omega_1$ and $\omega_2$,$\omega_2$ and $\omega_3$,$\omega_3$ and $\omega_1$ are either concurrent or parallel. }

	\prob{}{}{B}{Two equal-radius circles $\omega_1$ and $\omega_2$ are centered at points $O_1$ and $O_2$ . A point $X$ is reflected through $O_1$ and $O_2$ to get points $A_1$ and $A_2$.
		The tangents from $A_1$ to $\omega_1$ touch $\omega_1$ at points $P_1$ and $Q_1$, and
		the tangents from $A_2$ to $\omega_2$ touch $\omega_2$ at points $P_2$ and $Q_2$.
		If $P_1Q_1$ and $P_2Q_2$ intersect at $Y$ , prove that $Y$ is equidistant from $A_1$ and $A_2$.  }

	\prob{}{}{B}{Let $BD,CE$ be the altitudes of $\triangle ABC$ and $M$ be the midpoint of $BC$. If the ray $MH$ meet $\bigodot ABC$ at point $K$, prove that $AK,BC,DE$ are concurrent. }

	\prob{}{}{B}{Two circle $\omega$ and $\Gamma$ touches one another internally at $P$ with $\omega $ inside of $\Gamma$. Let $AB$ be a chord of $\Gamma$ which touches $\omega$ at $D$. Let $PD\cap \Gamma=Q$. Prove that $QA=QB$.}

	\prob{}{}{B}{Let $AD$ be a symmedian of $\triangle ABC$ with $D$ on $\bigodot ABC$. Let $M$ be the midpoint of $AD$. Prove that $\angle BMD=\angle CMD$ and $A,M,O,D$ are cyclic where $O$ is the circumcenter of $\triangle ABC$. }

	\prob{}{}{B}{Let $ A, B $ be two fixed points and let $ P $ be varying point such that $ \frac{PA}{PB} $ is constant. Prove that the locus of $ P $ is a circle.}

	\prob{}{}{B}{Prove that $ r_1+r_2+r_3=4R+r $ ($ R, r, r_1, r_2, r_3 $ are the circumradius, inradius and three exradiuses respectively of a triangle)}

	\prob{}{}{B}{Let $ M $  be the midpoint of the altitude $ BE $ in $ \triangle ABC $ and suppose that the excircle opposite to $ B $ touches $ AC $ at $ Y $. Then $ MY $ goes through the incenter $ I $.}

	\prob{}{}{B}{Let $ ABC $ be a triangle, and draw isosceles triangles $ \triangle DBC, \triangle AEC, \triangle ABF $ external to $ \triangle ABC $ (with $ BC; CA; AB $ as their respective bases). Prove that the lines through $ A; B; C $ perpendicular to $ EF; FD; DE $, respectively, are concurrent.}

	\prob{}{}{B}{In a triangle $ ABC $ we have $ AB = AC $. A circle which is internally tangent with the circumscribed circle of the triangle is also tangent to the sides $ AB; AC $ in the points $ P $, respectively $ Q $. Prove that the midpoint of $ PQ $ is the center of the inscribed circle of the triangle $ ABC $}

	\prob{}{}{B}{\textbf{Nagel Point $ N $}: If the Excircles of $ ABC $ touch $ BC; CA; AB $ at $ D; E; F $, then the intersection point of $ AD; BE; CF $ is called the \textbf{Nagel Point} $ N $. Prove that

	\begin{enumerate}
		\item $ I; G; N $ are collinear. ($ G $ centroid, $ I $ incenter.)
		\item $ GN = 2 \cdot IG $.
		\item \textbf{Speiker center} $ S $: The incircle of the medial triangle is called the Speiker circle, and it’s center is \textbf{Speiker center} $ S $. Prove that $ S $ is the midpoint of $ IN $.
	\end{enumerate}}

\section{Second Portion}

	\prob{}{}{B}{Let $PB$ and $PC$ are tangent to $\bigodot ABC$. Let $D,E,F$ are projection of $A$ on $BC,PB,PC$ resp. Prove that $AD^2=AE\times AF$.}

	\prob{}{}{B}{Let $D$ and $E$ are on $AB$ and $AC$ s.t $DE \parallel BC$. $P$ is an arbitrary point inside $\triangle ADE$. $PB,PC\cap DE=F,G$. Let $\bigodot PDG \cap \bigodot PFE=Q$. Prove that $A,P,Q$ are collinear.  }

	\prob{}{}{B}{Let $AB$ and $CD$ be chords in a circle of center $O$ with $A , B , C , D$ distinct , and with the lines $AB$ and $CD$ meeting at a right angle at point $E$. Let also $M$ and $N$ be the midpoints of $AC$ and $BD$ respectively . If $MN \bot OE$ , prove that $AD \parallel BC$}

	\prob{}{}{B}{Circles $ \mathcal{C}_1$ and $ \mathcal{C}_2$ intersect at $ A$ and $ B$. Let $ M\in AB$. A line through $ M$ (different from $ AB$) cuts circles $ \mathcal{C}_1$ and $ \mathcal{C}_2$ at $ Z,D,E,C$ respectively such that $ D,E\in ZC$. Perpendiculars at $ B$ to the lines $ EB,ZB$ and $ AD$ respectively cut circle $ \mathcal{C}_2$ in $ F,K$ and $ N$. Prove that $ KF=NC$. }

	\prob{}{}{B}{Let $D$ be a point on side $AC$ of triangle $ABC$. Let $E$ and $F$ be points on the segments $BD$ and $BC$ respectively, such that $\angle BAE = \angle CAF$. Let $P$ and $Q$ be points on $BC$ and $BD$ respectively, such that $EP$ and $FQ$ are both parallel to $CD$. Prove that $\angle BAP = \angle CAQ$.}

	\prob{}{}{B}{In the non-isosceles triangle $ABC$ an altitude from $A$ meets side $BC$ in $D$ . Let $M$ be the midpoint of $BC$ and let $N$ be the reflection of $M$ in $D$. The circumcirle of triangle $AMN$ intersects the side $AB$ in $P\ne A$ and the side $AC$ in $Q\ne A$ . Prove that $AN,\ BQ$ and $CP$ are concurrent.}

	\prob{}{}{B}{In triangle $ABC$, the interior and exterior angle bisectors of $ \angle BAC$ intersect the line $BC$ in $D $ and $E$, respectively. Let $F$ be the second point of intersection of the line $AD$ with the circumcircle of the triangle $ ABC$. Let $O$ be the circumcenter of the triangle $ ABC $and let $D'$ be the reflection of $D$ in $O$. Prove that $ \angle D'FE =90.$}

	\prob{}{}{B}{Let $ABCD$ be a convex quadrilateral such that the line $BD$ bisects the angle $ABC.$ The circumcircle of triangle $ABC$ intersects the sides $AD$ and $CD$ in the points $P$ and $Q,$ respectively. The line through $D$ and parallel to $AC$ intersects the lines $BC$ and $BA$ at the points $R$ and $S,$ respectively. Prove that the points $P, Q, R$ and $S$ lie on a common circle.}

	\prob{}{}{B}{The incircle of triangle $ABC$ touches $BC$, $CA$, $AB$ at points $A_1$, $B_1$, $C_1$, respectively. The perpendicular from the incenter $I$ to the median from vertex $C$ meets the line $A_1B_1$ in point $K$. Prove that $CK$ is parallel to $AB$.}

	\prob{}{}{B}{Let $X$ be an arbitrary point inside the circumcircle of a triangle $ABC$. The lines $BX$ and $CX$ meet the circumcircle in points $K$ and $L$ respectively. The line $LK$ intersects $BA$ and $AC$ at points $E$ and $F$ respectively. Find the locus of points $X$ such that the circumcircles of triangles $AFK$ and $AEL$ touch.}

	\prob{}{}{B}{Let $BD$ be a bisector of triangle $ABC$. Points $I_a$, $I_c$ are the incenters of triangles $ABD$, $CBD$ respectively. The line $I_aI_c$ meets $AC$ in point $Q$. Prove that $\angle DBQ = 90^\circ$.}

	\prob{}{}{B}{Given right-angled triangle $ABC$ with hypotenuse $AB$. Let $M$ be the midpoint of $AB$ and $O$ be the center of circumcircle $\omega$ of triangle $CMB$. Line $AC$ meets $\omega$ for the second time in point $K$. Segment $KO$ meets the circumcircle of triangle $ABC$ in point $L$. Prove that segments $AL$ and $KM$ meet on the circumcircle of triangle $ACM$.}

	\prob{}{}{B}{Let $BN$ be median of triangle $ABC$. $M$ is a point on $BC$. $S$ lies on $BN$ such that $MS\parallel AB$. $P$ is a point such that $SP\perp AC$ and $BP\parallel AC$. $MP$ cuts $AB$ at $Q$. Prove that $QB=QP$.}

	\prob{}{}{B}{ Let $ABCD$ be a convex quadrilateral with $AB$ parallel to $CD$. Let $P$ and $Q$ be the midpoints of $AC$ and $BD$, respectively. Prove that if $\angle ABP=\angle CBD$, then $\angle BCQ=\angle ACD$.}

	\prob{}{}{B}{Point $P$ lies inside a triangle $ABC$. Let $D,E$ and $F$ be reflections of the point $P$ in the lines $BC,CA$ and $AB$, respectively. Prove that if the triangle $DEF$ is equilateral, then the lines $AD,BE$ and $CF$ intersect in a common point.}

	\prob{}{}{B}{Let $\triangle{ABC}$ be an acute angled triangle. The circle with diameter $AB$ intersects the sides $AC$ and $BC$ at points $E$ and $F$ respectively. The tangents drawn to the circle through $E$ and $F$ intersect at $P$. Show that $P$ lies on the altitude through the vertex $C$.}

	\prob{}{}{B}{Let $\gamma$ be circle and let $P$ be a point outside $\gamma$. Let $PA$ and $PB$ be the tangents from $P$ to $\gamma$ (where $A, B \in \gamma$). A line passing through $P$ intersects $\gamma$ at points $Q$ and $R$. Let $S$ be a point on $\gamma$ such that $BS \parallel QR$. Prove that $SA$ bisects $QR$}

	\prob{}{}{B}{Given is a convex quadrilateral $ABCD$ with $AB=CD$. Draw the triangles $ABE$ and $CDF$ outside $ABCD$ so that $\angle{ABE} = \angle{DCF}$ and $\angle{BAE}=\angle{FDC}$. Prove that the midpoints of $\overline{AD}$, $\overline{BC}$ and $\overline{EF}$ are collinear}

	\prob{}{}{B}{Let $P$ be a point out of circle $C$. Let $PA$ and $PB$ be the tangents to the circle drawn from $C$. Choose a point $K$ on $AB$ . Suppose that the circumcircle of triangle $PBK$ intersects $C$ again at $T$. Let ${P}'$ be the reflection of $P$ with respect to $A$. Prove that
		\[ \angle PBT = \angle {P}'KA \]}

	\prob{}{}{B}{Consider a circle $C_1$ and a point $O$ on it. Circle $C_2$ with center $O$, intersects $C_1$ in two points $P$ and $Q$. $C_3$ is a circle which is externally tangent to $C_2$ at $R$ and internally tangent to $C_1$ at $S$ and suppose that $RS$ passes through $Q$. Suppose $X$ and $Y$ are second intersection points of $PR$ and $OR$ with $C_1$. Prove that $QX$ is parallel with $SY$.}

	\prob{}{}{B}{In triangle $ABC$ we have $\angle A=\frac{\pi}{3}$. Construct $E$ and $F$ on continue of $AB$ and $AC$ respectively such that $BE=CF=BC$. Suppose that $EF$ meets circumcircle of $\triangle ACE$ in $K$. ($K\not \equiv E$). Prove that $K$ is on the bisector of $\angle A$}

	\prob{}{}{B}{In triangle $ABC$, $\angle A=90^{\circ}$ and $M$ is the midpoint of $BC$. Point $D$ is chosen on segment $AC$ such that $AM=AD$ and $P$ is the second meet point of the circumcircles of triangles $\Delta AMC,\Delta BDC$. Prove that the line $CP$ bisects $\angle ACB$}

	\prob{}{}{B}{Let $C_1,C_2$ be two circles such that the center of $C_1$ is on the circumference of $C_2$. Let $C_1,C_2$ intersect each other at points $M,N$. Let $A,B$ be two points on the circumference of $C_1$ such that $AB$ is the diameter of it. Let lines $AM,BN$ meet $C_2$ for the second time at $A',B'$, respectively. Prove that $A'B'=r_1$ where $r_1$ is the radius of $C_1$.}

	\prob{}{}{B}{Given a triangle $ABC$, let $P$ lie on the circumcircle of the triangle and be the midpoint of the arc $BC$ which does not contain $A$. Draw a straight line $l$ through $P$ so that $l$ is parallel to $AB$. Denote by $k$ the circle which passes through $B$, and is tangent to $l$ at the point $P$. Let $Q$ be the second point of intersection of $k$ and the line $AB$ (if there is no second point of intersection, choose $Q = B$). Prove that $AQ = AC$.}

	\prob{}{}{B}{	Let $ABCD$ be a cyclic quadrilateral in which internal angle bisectors $\angle ABC$ and $\angle ADC$ intersect on diagonal $AC$. Let $M$ be the midpoint of $AC$. Line parallel to $BC$ which passes through $D$ cuts $BM$ at $E$ and circle $ABCD$ in $F$ ($F \neq D$ ). Prove that $BCEF$ is parallelogram}

	\prob{}{}{B}{The side $BC$ of the triangle $ABC$ is extended beyond $C$ to $D$ so that $CD = BC$. The side $CA$ is extended beyond $A$ to $E$ so that $AE = 2CA$. Prove that, if $AD=BE$, then the triangle $ABC$ is right-angled}

	\prob{}{}{B}{$ABCD$ is a cyclic quadrilateral inscribed in the circle $\Gamma$ with $AB$ as diameter. Let $E$ be the intersection of the diagonals $AC$ and $BD$. The tangents to $\Gamma$ at the points $C,D$ meet at $P$. Prove that $PC=PE$}

	\prob{}{}{B}{The quadrilateral $ABCD$ is inscribed in a circle. The point $P$ lies in the interior of $ABCD$, and $\angle P AB = \angle P BC = \angle P CD = \angle P DA$. The lines $AD$ and $BC$ meet at $Q$, and the lines $AB$ and $CD$ meet at $R$. Prove that the lines $P Q$ and $P R$ form the same angle as the diagonals of $ABCD$}

	\prob{}{}{B}{Let $ABCD$ be a cyclic quadrilateral with opposite sides not parallel. Let $X$ and $Y$ be the intersections of $AB,CD$ and $AD,BC$ respectively. Let the angle bisector of $\angle AXD$ intersect $AD,BC$ at $E,F$ respectively, and let the angle bisectors of $\angle AYB$ intersect $AB,CD$ at $G,H$ respectively. Prove that $EFGH$ is a parallelogram.}

	\prob{}{}{B}{Triangle $ABC$ is given with its centroid $G$ and cicumcentre $O$ is such that $GO$ is perpendicular to $AG$. Let $A'$ be the second intersection of $AG$ with circumcircle of triangle $ABC$. Let $D$ be the intersection of lines  $CA'$ and $AB$ and $E$ the intersection of lines $BA'$ and $AC$. Prove that the circumcentre of triangle $ADE$ is on the circumcircle of triangle $ABC$}

	\prob{}{}{B}{Let $M$ be the midpoint of the side $AC$ of $ \triangle ABC$. Let $P\in AM$ and $Q\in CM$ be such that $PQ=\frac{AC}{2}$. Let $(ABQ)$ intersect with $BC$ at $X\not= B$ and $(BCP)$ intersect with $BA$ at $Y\not= B$. Prove that the quadrilateral $BXMY$ is cyclic.}

	\prob{}{}{B}{Let be given a triangle $ ABC$ and its internal angle bisector $ BD$ $ (D\in BC)$. The line $ BD$ intersects the circumcircle $ \Omega$ of triangle $ ABC$ at $ B$ and $ E$. Circle $ \omega$ with diameter $ DE$ cuts $ \Omega$ again at $ F$. Prove that $ BF$ is the symmedian line of triangle $ ABC$.}

	\prob{}{}{B}{$ \Delta ABC$ is a triangle such that $ AB \neq AC$. The incircle of $ \Delta ABC$ touches $ BC, CA, AB$ at $ D, E, F$ respectively. $ H$ is a point on the segment $ EF$ such that $ DH \bot EF$. Suppose $ AH \bot BC$, prove that $ H$ is the orthocenter of $ \Delta ABC$.}

	\prob{}{}{B}{Let $ ABC $ be a triangle and let $ P $ be a point on the angle bisector $ AD $, with $ D $ on $ BC $. Let $ E, F $ and $ G $ be the intersections of $ AP, BP $ and $ CP $ with the circumcircle of the triangle, respectively. Let $ H $ be the intersection of $ EF $ and $ AC $, and let $ I $ be the ntersection of $ EG $ and $ AB $. Determine the geometric place of the intersection of $ BH $ and $ CI $ when $ P $ varies}

	\prob{}{}{B}{Let $ D; E; F $ be the points on the sides $ BC; CA; AB $ respectively, of $ \triangle ABC $. Let $ P; Q; R $ be the second intersection of $ AD; BE; CF $ respectively, with the cricumcircle of $ \triangle ABC $.

	Show that \[\frac{AD}{PD}+\frac{BE}{QE}+\frac{CF}{RF}\geq 9\]}

	\prob{}{}{B}{Points $ D $ and $ E $ lie on sides $ AB $ and $ AC $ of triangle $ ABC $ such that $ DE \parallel BC $. Let $ P $ be an arbitrary point inside $ ABC $. The lines $ PB $ and $ PC $ intersect $ DE $ at $ F $ and $ G $, respectively. If $ O_1 $ is the circumcenter of $ PDG $ and $ O_2 $ is the circumcenter of $ PFE $, show that $ AP \parallel O_1O_2 $.}

	\prob{}{}{B}{Let $ ABC $ be a triangle. A circle passing through $ A $ and $ B $ intersects segments $ AC $ and $ BC $ at $ D $ and $ E $, respectively. Lines $ AB $ and $ DE $ intersect at $ F $, while lines $ BD $ and $ CF $ intersect at $ M $. Prove that $ MF = MC $ if and only if $ MB \cdot MD = MC^2 $}

	\prob{}{}{B}{Let $ O $ and $ I $ be the circumcenter and incenter of triangle $ ABC $, respectively. Let $ \omega A $ be the excircle of triangle $ ABC $ opposite to $ A $; let it be tangent to $ AB, AC, BC $ at $ K, M, N $, respectively. Assume that the midpoint of segment $ KM $ lies on the circumcircle of triangle $ ABC $. Prove that $ O; N; I $ are collinear.}

	\prob{}{}{B}{Let $ ABCD $ be a cyclic quadrilateral. Let $ AB \cap CD = P $ and $ AD \cap BC = Q $. Let the tangents from $ Q $ meet the circumcircle of $ ABCD $ at $ E $ and $ F $. Prove that $ P; E; F $ are collinear.}