# -*- coding: utf-8 -*-
"""predicting_covid19.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1VjqxWNcP2N4wy_NXfLt9vUToB2YXt6Ec

# Predicting Covid19 from CT Scan Images #
The patients that visit a health care centre can be diagnosed into three categries: patients who have covid19, patients who have pneumonia and healthy patients. In this project, we want to identify these patients. We use a <b> GPU runtime type</b> for efficiency.

# Packages #
First, let's run the cell below to import all the packages that we will need.
- [numpy](https://numpy.org/) is the fundamental package for scientific computing with Python.\n",
- [Keras](https://keras.io) is a deep learning API written in Python, running on top of the machine learning platform TensorFlow. It was developed with a focus on enabling fast experimentation.
- [Matplotlib](https://matplotlib.org/) is a comprehensive library for creating static, animated, and interactive visualizations in Python. It is used here for plotting images.
- The [sklearn.metrics](https://scikit-learn.org/stable/modules/classes.html#module-sklearn.metrics) module includes score functions, performance metrics and pairwise metrics and distance computations.
"""

import numpy as np
from tensorflow.keras import *
import matplotlib.pyplot as plt
from sklearn.metrics import classification_report
from keras.preprocessing.image import load_img

"""# Loading Dataset of Images #
We have a train and test dataset of the ct scan images of people's lungs in the health health center. Here we load the dataset from the drive by first copying it and the unzipping the file in the current directory.
"""

!cp "/content/drive/MyDrive/ColabNotebooks/xray.zip" .

!unzip xray.zip

"""# Reading and Preprocessing #
Here are the steps we will follow in this section:
- Reading the train and test datasets
- Convert images to grayscale
- Change the size of the images to 80x80
- Print the number of read data points and the number of the categories 
- Setting the batch size to 32 (for the following sections to use)

"""

def loadDataset(train_address="/content/Data/train", test_address="/content/Data/test"
                , target_size = (80,80), color_mode="grayscale"
                , classes= ["COVID19", "NORMAL", "PNEUMONIA"], batch_size = 32, rescale=1):
  
  dataGenerator = preprocessing.image.ImageDataGenerator(rescale=rescale, featurewise_std_normalization=True, samplewise_std_normalization=True) #will be ready to preprocess images
  classes=['COVID19', 'NORMAL', 'PNEUMONIA']
  dataTrain = dataGenerator.flow_from_directory(train_address
                                              , target_size=target_size
                                              , color_mode=color_mode
                                              , classes=classes 
                                              , batch_size=batch_size)
  
  dataTest = dataGenerator.flow_from_directory(test_address
                                              , target_size=target_size
                                              , color_mode=color_mode
                                              , classes=classes
                                              , batch_size=batch_size)
  return dataTrain, dataTest

train, test = loadDataset()

"""# Data Visualization #
In order to have an intuitive understanding of our datasets, we will do the following:
<ol>
<p> 1.Display an image of every class of patients
<p> 2.Draw a histogram of the number of patients in each class
</ol>
For the tasks above, we loop through the given dataset, count the number of data points in each category and display the images of every new category (three categories\classes in total as mentioned above).
"""

def plot_labels(dataset):
  images, labels = dataset.next()
  types = {0:"COVID19", 1:"NORMAL", 2:"PNEUMONIA"}
  visited_labels_indices = set() #set of where the 1 index is in the type/category
  for image, label in zip(images, labels):
    type_index = backend.argmax(label).numpy()
    if type_index not in visited_labels_indices:
      visited_labels_indices.add(type_index)
      plt.title(types[type_index])
      plt.imshow(image.reshape(80,80), cmap="gray")
      plt.show()

plot_labels(train)

def plot_distribution():
  train, test = loadDataset() #we have to reload the data since it might have been changes in the previous parts
  num_of_types = train.num_classes #the number of different label types
  labels_count = {} #making a dictionary of labels count
  for type_ in range(num_of_types):
    labels_count[type_] = 0
  for label in train.classes:
    labels_count[label] += 1
  types = {0:"COVID19", 1:"NORMAL", 2:"PNEUMONIA"}
  x = list(types.values())
  print(x)
  y = [labels_count[t] for t in range(num_of_types)]
  plt.bar(x, y, color='red')

plot_distribution()

"""# Defining Model Architecture #
Here we develop a neural network with the following properties:
- The network has 4 layers
- The loss function is categorical cross entropy
- The batch size is 32
- It has 10 epochs
- All the activation functions except the last one are relu 
"""

EPOCHS, BATCH_SIZE = 10, 32
def makeNN(input_shape, output_shape, activation_f="relu", optimizer=optimizers.SGD(learning_rate=0.01), loss="categorical_crossentropy"
                      , **kwargs): #make neural network
  input_ = layers.Input(shape=input_shape)
  output_ = layers.Flatten()(input_)
  output_ = layers.Dense(2048, activation=activation_f, bias_initializer='zeros', kernel_initializer='random_normal',**kwargs)(output_)
  output_ = layers.Dense(1024, activation=activation_f, bias_initializer='zeros', kernel_initializer='random_normal',**kwargs)(output_)
  output_ = layers.Dense(2048, activation=activation_f, bias_initializer='zeros', kernel_initializer='random_normal',**kwargs)(output_)
  output_ = layers.Dense(output_shape, activation="softmax", bias_initializer='zeros', kernel_initializer='random_normal',**kwargs)(output_)
  model = models.Model(inputs=input_, outputs=output_)
  model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy'])
  model.summary()
  return model

our_model = makeNN((80,80,1), train.num_classes)

"""# Traing the Model on the Test Dataset #
To train the model we need to have the labels of the test dataset. Since the labels are one-hot vectors, we will first change them to their true labels (the index of 1 in their corresponding vector).<br>
After converting the labels to integers, we can train the model, using the architechture we defined and trained above.

## Training Using ReLu Activation Function ##
As we can seen below, after 10 epochs the relu activation function results in almost perfect accuracy in the training set. The non-linearity of relu and the few number of layers, makes it efficient in our network. However, relu can not be used in networks with many layers due to the problem of vanishing gradients.<br>
<b>Advantages:</b><br>
- It is computationally effective as it involves simpler mathematical operations than sigmoid and tanh.
- Although it looks like a linear function, it adds non-linearity to the network, making it able to learn complex patterns.
- It doesn't suffer from the vanishing gradient problem.
- It is unbounded at the positive side. Hence removing the problem of gradient saturation.
- It provides sparsity to the network, which as a result lessens the space and time complexity. Sparse networks are always faster than dense networks as there are lesser things to compute in a sparse network. Sparsity results in concise models that often have better predictive power and less overfitting/noise.


<b>Disadvantages:</b><br>
- It suffers from the dying ReLU problem. ReLU is always going to discard the negative values i.e. the deactivations by making it 0. But because of this, the gradient of these units will also become 0 and by now we all know that 0 gradient means no weight updation during backpropagation. Simply speaking, the neurons which will go to this state will stop responding to the deviation of input or the error. This, as a result, hampers the ability of the model to fit the data properly.
-  It is non-differentiable at 0.

<b>Uses</b><br>

ReLU should be used in the hidden layers. As it is computationally less expensive than sigmoid and tanh, therefore it is <b>usually</b> a better choice than them. It is also to be noted that ReLU is faster than both tanh and sigmoid. Also in hidden layers, at a time only a few neurons are activated, making it efficient and easy for computation.
"""

x, y = train.next()
history = our_model.fit(train, epochs=EPOCHS)

"""<b> Getting the actual labels</b>: to be able to test our models we need to have the actual functions. To do so, we replace each label (one-hot vector) with the index of 1 in the vector.

"""

def get_labels(dataset):
  all_labels = []
  all_batches = int(np.ceil(len(dataset.classes)/dataset.batch_size))
  for batch_index in range(all_batches):
    images, labels = dataset.next()
    for label in labels:
      all_labels.append(backend.argmax(label))
  return np.array(all_labels) #since we need a numpy array for training our model

train, test = loadDataset()
predictions=our_model.predict(train)
actual_labels = get_labels(train)
print(backend.argmax(predictions).numpy().shape)
print(classification_report(actual_labels, backend.argmax(predictions).numpy()))

def plot_acc_loss(history):
  plt.plot(history.history['accuracy'])
  plt.title('model accuracy')
  plt.ylabel('accuracy')
  plt.xlabel('epoch')
  plt.legend(['accuracy'], loc='upper left')
  plt.show()
  plt.plot(history.history['loss'])
  plt.title('model loss')
  plt.ylabel('loss')
  plt.xlabel('epoch')
  plt.legend(['loss'], loc='upper left')
  plt.show()

plot_acc_loss(history)

"""## Testing Using ReLu Activation Fuction #

"""

predictions=our_model.predict(test)
actual_labels = get_labels(test)
print(classification_report(actual_labels, backend.argmax(predictions).numpy()))

"""## Training Using Tanh Activation Function ##
 Tanh is also like logistic sigmoid but better. The range of the tanh function is from (-1 to 1). The equation for tanh is $f(x) = 2/(1 + e^{-2x})-1$.<br> 
 <b>Advantages</b>

 - It has similar advantages as sigmoid but better than that because it is zero centred. The output of tanh lies between -1 and 1. Hence solving one of the issues with the sigmoid.

<b>Disadvantages</b>

- It also has the problem of vanishing gradient but the derivatives are steeper than that of the sigmoid. Hence making the gradients stronger for tanh than sigmoid.
- As it is almost similar to sigmoid, tanh is also computationally expensive.
    Similar to sigmoid, here also the gradients saturate.
<b>Uses</b><br>
tanh is generally used in the hidden layers. The output of tanh lies between -1 and +1. Hence the mean for the hidden layers stays near 0. This makes the learning of the hidden layer converge quickly.

<b>Uses</b><br>
 Tanh is generally used in the hidden layers. The output of tanh lies between -1 and +1. Hence the mean for the hidden layers stays near 0. This makes the learning of the hidden layer converge quickly.
"""

train, test = loadDataset()
tanh_model = makeNN((80,80,1), train.num_classes, activation_f='tanh')
x, y = train.next()
history = our_model.fit(train, epochs=EPOCHS)
train, test = loadDataset()
predictions=our_model.predict(train)
actual_labels = get_labels(train)
print(backend.argmax(predictions).numpy().shape)
print(classification_report(actual_labels, backend.argmax(predictions).numpy()))
plot_acc_loss(history)

"""## Testing Using Tanh Activation Fuction #

"""

predictions=our_model.predict(test)
actual_labels = get_labels(test)
print(classification_report(actual_labels, backend.argmax(predictions).numpy()))

"""## ReLu vs Tanh ##
As we can see above, Tanh activation function is working better than ReLu on these datasets. This is beacuse tanh converges to zero more quickly in the hidden layers. Moreover, ReLu is more prune to overfitting than tanh. We can solve this problem by variously overfitting methods. Also some learning rates and other hyperparameters are more suitable for some activation functions. Although, both of these functions have the gradient vanishing problem which is not obvious becasue of having a few layers.

# Normalizing and Optimizing the Neural Network #
To normalized the data we divide the pixels in the images by 255 (the range of numbers for each pixel is 0-255) and to improve the network, try adding more layers and chaning the learning rate. There are other things we could do i.e. changing the optimizer etc, that we will do in the following sections.
"""

train, test = loadDataset(rescale=1/255.0) #normalizing
EPOCHS, BATCH_SIZE = 10, 32
def makeNN(input_shape, output_shape, activation_f="relu", optimizer=optimizers.SGD(learning_rate=0.001), loss="categorical_crossentropy"): #make neural network
  input_ = layers.Input(shape=input_shape)
  output_ = layers.Flatten()(input_)
  output_ = layers.Dense(2048, activation=activation_f, bias_initializer='zeros', kernel_initializer='random_normal')(output_)
  output_ = layers.Dense(1024, activation=activation_f, bias_initializer='zeros', kernel_initializer='random_normal')(output_)
  output_ = layers.Dense(1024, activation=activation_f, bias_initializer='zeros', kernel_initializer='random_normal')(output_)
  output_ = layers.Dense(2048, activation=activation_f, bias_initializer='zeros', kernel_initializer='random_normal')(output_)
  output_ = layers.Dense(output_shape, activation="softmax", bias_initializer='zeros', kernel_initializer='random_normal')(output_)
  model = models.Model(inputs=input_, outputs=output_)
  model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy'])
  model.summary()
  return model

our_model = makeNN((80,80,1), train.num_classes, activation_f="tanh")

"""## Fitting and Testing ##"""

x, y = train.next()
history = our_model.fit(train, epochs=EPOCHS)
predictions=our_model.predict(test)
actual_labels = get_labels(test)
print(classification_report(actual_labels, backend.argmax(predictions).numpy()))

"""# Other Optimizers #
Optimizer helps us to minimize an error function(loss function)or to maximize the efficiency of production .In this section we will apply two other optimizers: momentum and Adam

## Momentum : What is it and why is it useful? ##
Momentum is not soley an optimizer, it is an improvement on Stochastic Gradient Descent. When looking at noisy time series data such as your training/validation error graphs in tensorboard, you may often notice that the raw values are often quite noisy. Quite often, you are able to see a trend in the graph. These trends often become more obvious when you add some smoothing to the raw graph values.<br>

Exponential Smoothing is one of the simplest ways to add smoothing to your data.<br>
Exponential Smoothing Formula:
$$ s_0 = x_0 s_t = ax_t + (1-\alpha)s_{t-1}, t>0, 0<= \alpha <=1$$
In the above equation, momentum specifies the amount of smoothing we want. A typical value for momentum is .9. From this equation, we can see that the value at time step t takes into account previous values from time steps (t — 1) and before. The weighting of previous time steps drops off exponentially so the most recent time step has the greatest impact.The momentum optimizer uses this same equation when optimizing a loss function. In this case, the Exponential Smoothing is called Momentum and it allows the optimizer to maintain the velocity and trajectory from previous time steps to affect the present.

## Is Momentum always a good optimizer? ##
 Unfortunately, while improvements to the momentum algorithm do exist, they all run into a certain, critical, almost inescapable lower bound. That means, we can not always apply momentum and get better results.

## Implementing Momentum Optimizer on Our Model ##
 From the models below, we can see that the method of <b> early stopping</b> applies for momentum too. When it is increased from 0.5 to 0.9 the model is improved but for momentum 0.99 the model has deteriorated.<br>
 Momentum = 0.5
"""

our_model = makeNN((80,80,1), train.num_classes, activation_f="tanh", optimizer=optimizers.SGD(learning_rate=0.001, momentum = 0.5))
x, y = train.next()
history = our_model.fit(train, epochs=EPOCHS)
predictions=our_model.predict(test)
actual_labels = get_labels(test)
print(classification_report(actual_labels, backend.argmax(predictions).numpy()))

""" Momentum = 0.9<br>
 This momentum is giving us an almost perfect model.
"""

our_model = makeNN((80,80,1), train.num_classes, activation_f="tanh", optimizer=optimizers.SGD(learning_rate=0.001, momentum = 0.9))
history = our_model.fit(train, epochs=EPOCHS)
predictions=our_model.predict(test)
actual_labels = get_labels(test)
print(classification_report(actual_labels, backend.argmax(predictions).numpy()))

""" Momentum = 0.99"""

our_model = makeNN((80,80,1), train.num_classes, activation_f="tanh", optimizer=optimizers.SGD(learning_rate=0.001, momentum = 0.99))
history = our_model.fit(train, epochs=EPOCHS)
predictions=our_model.predict(test)
actual_labels = get_labels(test)
print(classification_report(actual_labels, backend.argmax(predictions).numpy()))

"""## Adam: What is the Adam optimization algorithm? ##
Adam is an optimization algorithm that can be used instead of the classical stochastic gradient descent procedure to update network weights iterative based in training data.

Adam was presented by Diederik Kingma from OpenAI and Jimmy Ba from the University of Toronto in their 2015 ICLR paper (poster) titled “Adam: A Method for Stochastic Optimization“. I will quote liberally from their paper in this post, unless stated otherwise.

The algorithm is called Adam. It is not an acronym and is not written as “ADAM”.

    … the name Adam is derived from adaptive moment estimation.

When introducing the algorithm, the authors list the attractive benefits of using Adam on non-convex optimization problems, as follows:

-    Straightforward to implement.
-    Computationally efficient.
-    Little memory requirements.
-    Invariant to diagonal rescale of the gradients.
-    Well suited for problems that are large in terms of data and/or parameters.
-    Appropriate for non-stationary objectives.
-    Appropriate for problems with very noisy/or sparse gradients.
-    Hyper-parameters have intuitive interpretation and typically require little tuning.

### How Does Adam Work? ###

Adam is different to classical stochastic gradient descent.

Stochastic gradient descent maintains a single learning rate (termed alpha) for all weight updates and the learning rate does not change during training.

A learning rate is maintained for each network weight (parameter) and separately adapted as learning unfolds.
<br>The method computes individual adaptive learning rates for different parameters from estimates of first and second moments of the gradients.

The authors describe Adam as combining the advantages of two other extensions of stochastic gradient descent. Specifically:

Instead of adapting the parameter learning rates based on the average first moment (the mean), Adam also makes use of the average of the second moments of the gradients (the uncentered variance).

Specifically, the algorithm calculates an exponential moving average of the gradient and the squared gradient, and the parameters beta1 and beta2 control the decay rates of these moving averages.

The initial value of the moving averages and beta1 and beta2 values close to 1.0 (recommended) result in a bias of moment estimates towards zero. This bias is overcome by first calculating the biased estimates before then calculating bias-corrected estimates.

<b>Adam is Effective</b><br>

Adam is a popular algorithm in the field of deep learning because it achieves good results fast.<br>
Empirical results demonstrate that Adam works well in practice and compares favorably to other stochastic optimization methods.

<b>Adam VS SGD</b><br>
To better understand the paper’s implications, it is necessary to first look at the pros and cons of popular optimization algorithms Adam and SGD.

Gradient descent is the most common method used to optimize deep learning networks. First proposed in the 1950s, the technique can update each parameter of a model, observe how a change would affect the objective function, choose a direction that would lower the error rate, and continue iterating until the objective function converges to the minimum.

SGD is a variant of gradient descent. Instead of performing computations on the whole dataset — which is redundant and inefficient — SGD only computes on a small subset or random selection of data examples. SGD produces the same performance as regular gradient descent when the learning rate is low.

Essentially Adam is an algorithm for gradient-based optimization of stochastic objective functions. It combines the advantages of two SGD extensions — Root Mean Square Propagation (RMSProp) and Adaptive Gradient Algorithm (AdaGrad) — and computes individual adaptive learning rates for different parameters.

Despite the widespread popularity of Adam, recent research papers have noted that it can fail to converge to an optimal solution under specific settings. The paper Improving Generalization Performance by Switching from Adam to SGD demonstrates that adaptive optimization techniques such as Adam generalize poorly compared to SGD. This has prompted some researchers to explore new techniques that may improve on Adam
"""

our_model = makeNN((80,80,1), train.num_classes, activation_f="tanh", optimizer=optimizers.Adam(learning_rate=0.001,beta_1=0.9,
                                                                                                beta_2=0.999,epsilon=1e-07,
                                                                                                amsgrad=False,name="Adam"))

"""## Setting EPOCHS to 20 ##
From some epoch on, the efficiency of the model on the training set does not change which indicates the bound of the optimization scheme.
"""

EPOCHS = 20
history = our_model.fit(train, epochs=EPOCHS)
predictions=our_model.predict(test)
actual_labels = get_labels(test)
print(classification_report(actual_labels, backend.argmax(predictions).numpy()))

"""## Why are epochs necesary? ##
Generally whenever we want to optimize we use gradient descent. Gradient descent has a parameter called learning rate. In one iteration alone we can not guarantee that the gradient descent algorithm would converge to a local minima with the specified learning rate. That is the reason why we iterate again for the gradient descent to converge better.
## Do all problems need multiple epochs? ##
If the optimization is straight forward in a problem, for example if we have a linear equation to optimize, we could reach the solution with even a single pass through the dataset. That is why multiple epochs are not necessary for all problems.
## Is a large number of EPOCHS always beneficial for the problem? ##
Not necessarily. The correct answer is the number of epochs is not that significant. more important is the validation and training error. As long as these two error keeps dropping, training should continue.<br>
For instance, if the validation error starts increasing that might be an indication of overfitting.

You should set the number of epochs as high as possible and terminate the training when validation error start increasing

# Using MSE Loss Function #
Here we want to replace categorical cross entropy with the MSE loss function. MSE is the sum of squared distances between our target variable and predicted values. As we can see below, categorical cross entropy, results in a better (more accurate model) for classification.
"""

our_model = makeNN((80,80,1), train.num_classes, activation_f="tanh", optimizer='sgd', loss=losses.MeanSquaredError())
EPOCHS = 20
history = our_model.fit(train, epochs=EPOCHS)
predictions=our_model.predict(test)
actual_labels = get_labels(test)
print(classification_report(actual_labels, backend.argmax(predictions).numpy()))

"""### Why is MSE not suitable for classification? ###
Firstly, using MSE means that we assume that the underlying data has been generated from a normal distribution (a bell-shaped curve). In Bayesian terms this means we assume a Gaussian prior. While in reality, a dataset that can be classified into categories is not from a normal distribution but a Bernoulli distribution.<br>
If we use maximum likelihood estimation(MLE), assuming that the data is from a normal distribution(a wrong assumption, by the way), we get the MSE as a Cost function for optimizing our model.<br><br>
Secondly, the MSE function is non-convex for binary classification. In simple terms, if a binary classification model is trained with MSE Cost function, it is not guaranteed to minimize the Cost function. This is because MSE function expects real-valued inputs in range(-∞, ∞), while binary classification models output probabilities in range(0,1) through the sigmoid/logistic function.<br><br>
When the MSE function is passed a value that is unbounded a nice U-shaped (convex) curve is the result where there is a clear minimum point at the target value (y). On the other hand, when a bounded value from a Sigmoid function is passed to the MSE function the result is not convex; on one side the function is concave while on the other side the function convex and no clear minimum point. So, if by accident a binary classification neural network is initialized with weights which are large in magnitude such that it lands on the concave part of the MSE Cost function gradient descent will not work and consequently, weights may not update or improve very slowly.<br><br>
On a final note, MSE is a good choice for a Cost function when we are doing Linear Regression (i.e fitting a line through data for extrapolation). In the absence of any knowledge of how the data is distributed assuming normal/gaussian distribution is perfectly reasonable.

# Regularization #
One of the major aspects of training your machine learning model is avoiding overfitting. The model will have a low accuracy if it is overfitting. This happens because your model is trying too hard to capture the noise in your training dataset. By noise we mean the data points that don’t really represent the true properties of your data, but random chance. Learning such data points, makes your model more flexible, at the risk of overfitting.<br>
A modern approach to reducing generalization error is to use a larger model that may be required to use regularization during training that keeps the weights of the model small. These techniques not only reduce overfitting, but they can also lead to faster optimization of the model and better overall performance.<br>
There are two ways to approach an overfit model:

-  Reduce overfitting by training the network on more examples.
-   Reduce overfitting by changing the complexity of the network.

Below is a list of five of the most common additional regularization methods.

- 1.Activity Regularization: Penalize the model during training base on the magnitude of the activations.
- 2.Weight Constraint: Constrain the magnitude of weights to be within a range or below a limit.
- 3.Dropout: Probabilistically remove inputs during training.
- 4.Noise: Add statistical noise to inputs during training.
- 5.Early Stopping: Monitor model performance on a validation set and stop training when performance degrades.

## L2 Regularization ##
We reimplemnt our model with the L2 Regularization in Keras with a value of 0.0001. L1 regularization adds the penalty term in cost function by adding the absolute value of weight(Wj) parameters, while L2 regularization adds the squared value of weights in the cost function. This improves the accuracy, as seen below.

Loss function with L1 regularization:
$$ Loss = Error(y,\hat y) + \lambda \sum_{i=1}^{N} |w_i|$$

Loss function with L2 regularization:
$$ Loss = Error(y,\hat y) + \lambda \sum_{i=1}^{N} w_i^{2}$$
"""

our_model = makeNN((80,80,1), train.num_classes, activation_f="tanh", optimizer=optimizers.SGD(learning_rate=0.001, momentum = 0.99)
                            ,kernel_regularizer=regularizers.l2(1e-4), bias_regularizer=regularizers.l2(1e-4))
history = our_model.fit(train, epochs=EPOCHS)
predictions=our_model.predict(test)
actual_labels = get_labels(test)
print(classification_report(actual_labels, backend.argmax(predictions).numpy()))

"""## Dropout ##
Ensembles of neural networks with different model configurations are known to reduce overfitting, but require the additional computational expense of training and maintaining multiple models.<br>
Dropout is a regularization method that approximates training a large number of neural networks with different architectures in parallel.<br>

During training, some number of layer outputs are randomly ignored or “dropped out.” This has the effect of making the layer look-like and be treated-like a layer with a different number of nodes and connectivity to the prior layer. In effect, each update to a layer during training is performed with a different “view” of the configured layer.<br>
### How to Dropout? ###

Dropout is implemented per-layer in a neural network.

It can be used with most types of layers, such as dense fully connected layers, convolutional layers, and recurrent layers such as the long short-term memory network layer.

Dropout may be implemented on any or all hidden layers in the network as well as the visible or input layer. It is not used on the output layer.

A new hyperparameter is introduced that specifies the probability at which outputs of the layer are dropped out, or inversely, the probability at which outputs of the layer are retained. 

A common value is a probability of 0.5 for retaining the output of each node in a hidden layer and a value close to 1.0, such as 0.8, for retaining inputs from the visible layer.

#Applying Dropout #
Applying dropout as below, results in a better accuracy as we expected based on the explanations above.
"""

our_model.add(Dropout(0.1))
history = our_model.fit(train, epochs=EPOCHS)
predictions=our_model.predict(test)
actual_labels = get_labels(test)
print(classification_report(actual_labels, backend.argmax(predictions).numpy()))

"""# References #
- [Keras](https://keras.io/)
- [Tensorflow Quick Start for Beginners](https://www.tensorflow.org/tutorials/quickstart/beginner)
- [Gentle Introduction to the Adam Optimization Algorithm for Deep Learning](https://machinelearningmastery.com/adam-optimization-algorithm-for-deep-learning/#:~:text=Adam%20is%20an%20optimization%20algorithm,iterative%20based%20in%20training%20data.&text=The%20algorithm%20is%20called%20Adam,derived%20from%20adaptive%20moment%20estimation.)
- [Why Momentum Really Works](https://distill.pub/2017/momentum/)
- [‘Fast as Adam & Good as SGD’ ](https://medium.com/syncedreview/iclr-2019-fast-as-adam-good-as-sgd-new-optimizer-has-both-78e37e8f9a34)
- [why-use-the-momentum-optimizer-with-minimal-code](https://medium.com/analytics-vidhya/why-use-the-momentum-optimizer-with-minimal-code-example-8f5d93c33a53)
-[What is epoch and How to choose the correct number of epoch](https://medium.com/@upendravijay2what-is-epoch-and-how-to-choose-the-correct-number-of-epoch-d170656adaaf)
- [Why Using Mean Squared Error(MSE) Cost Function for Binary Classification is a Bad Idea?](https://towardsdatascience.com/why-using-mean-squared-error-mse-cost-function-for-binary-classification-is-a-bad-idea-933089e90df7)
- [Regularization in Machine Learning](https://towardsdatascience.com/regularization-in-machine-learning-76441ddcf99a)
- [How to Avoid Overfitting in Deep Learning Neural Networks](https://machinelearningmastery.com/introduction-to-regularization-to-reduce-overfitting-and-improve-generalization-error/)
- [L1 vs L2 Regularization: The intuitive difference](https://medium.com/analytics-vidhya/l1-vs-l2-regularization-which-is-better-d01068e6658c)
- [A Gentle Introduction to Dropout for Regularizing Deep Neural Networks](https://machinelearningmastery.com/dropout-for-regularizing-deep-neural-networks/)
"""