<jupyter_start><jupyter_text>Setups, Installations and Imports<jupyter_code>!nvidia-smi
%%capture
!pip install wandb
SEED = 666

import tensorflow as tf
tf.random.set_seed(SEED)

from tensorflow import keras
from tensorflow.keras.datasets import cifar10
import os
os.environ["TF_DETERMINISTIC_OPS"] = "1"

import numpy as np
np.random.seed(SEED)

import pandas as pd
import matplotlib.pyplot as plt

%matplotlib inline

from mpl_toolkits.axes_grid1 import ImageGrid
from PIL import Image
import wandb
from wandb.keras import WandbCallback

wandb.login()<jupyter_output><empty_output><jupyter_text>Download and Prepare Dataset  CIFAR-10<jupyter_code>(x_train, y_train), (x_test, y_test) = cifar10.load_data()
CLASS_NAMES = ["airplane", "automobile", "bird", "cat", "deer", "dog", "frog", "horse", "ship", "truck"]
print('Shape of x_train: ', x_train.shape)
print('Shape of y_train: ', y_train.shape)
print('Shape of x_test: ', x_test.shape)
print('Shape of y_test: ', y_test.shape)<jupyter_output>Shape of x_train:  (50000, 32, 32, 3)
Shape of y_train:  (50000, 1)
Shape of x_test:  (10000, 32, 32, 3)
Shape of y_test:  (10000, 1)<jupyter_text>Visualize<jupyter_code># Ref: https://www.tensorflow.org/tutorials/load_data/images
def show_batch(image_batch, label_batch):
  plt.figure(figsize=(10,10))
  for n in range(25):
      ax = plt.subplot(5,5,n+1)
      plt.imshow(image_batch[n])
      plt.title(CLASS_NAMES[int(label_batch[n])])
      plt.axis('off')<jupyter_output><empty_output><jupyter_text>Dataloader<jupyter_code>AUTO = tf.data.experimental.AUTOTUNE
BATCH_SIZE = 32
IMG_SHAPE = 32
CHANNELS = 3
NUM_CLASSES = 10
def preprocess_image(image, label):
  image = image/255
  image = tf.image.convert_image_dtype(image, tf.float32)
  return image, label
trainloader = tf.data.Dataset.from_tensor_slices((x_train, y_train))
testloader = tf.data.Dataset.from_tensor_slices((x_test, y_test))

trainloader = (
    trainloader
    .shuffle(1024)
    .map(preprocess_image, num_parallel_calls=AUTO)
    .batch(BATCH_SIZE)
    .prefetch(AUTO)
)

testloader = (
    testloader
    .map(preprocess_image, num_parallel_calls=AUTO)
    .batch(BATCH_SIZE)
    .prefetch(AUTO)
)
image_batch, label_batch = next(iter(trainloader))
print(image_batch.shape, label_batch.shape)

show_batch(image_batch, label_batch)<jupyter_output>(32, 32, 32, 3) (32, 1)<jupyter_text>Callbacks<jupyter_code>earlystoper = tf.keras.callbacks.EarlyStopping(
    monitor='val_loss', patience=10, verbose=0, mode='auto',
    restore_best_weights=True ## This will ensure the best instance of the model over the validation set.
)
def scheduler(epoch):
  if epoch < 7:
    return 0.001
  else:
    return 0.001 * tf.math.exp(0.1 * (7 - epoch))

lr_callback = tf.keras.callbacks.LearningRateScheduler(scheduler)<jupyter_output><empty_output><jupyter_text>Baseline  Model<jupyter_code>def Model():
  inputs = keras.layers.Input(shape=(IMG_SHAPE, IMG_SHAPE, CHANNELS))

  x = keras.layers.Conv2D(filters=32, 
                          kernel_size=(3,3),
                          strides=(1,1),
                          padding='valid',
                          activation='relu')(inputs)

  x = keras.layers.Conv2D(filters=32, 
                          kernel_size=(3,3),
                          strides=(1,1),
                          padding='valid',
                          activation='relu')(x)

  x = keras.layers.MaxPooling2D(pool_size=2)(x)

  x = keras.layers.Conv2D(filters=32, 
                          kernel_size=(3,3),
                          strides=(1,1),
                          padding='valid',
                          activation='relu')(x)

  x = keras.layers.Conv2D(filters=32, 
                          kernel_size=(3,3),
                          strides=(1,1),
                          padding='valid',
                          activation='relu')(x)

  x = keras.layers.GlobalAveragePooling2D()(x)

  x = keras.layers.Dense(128, activation='relu')(x)
  x = keras.layers.Dense(32, activation='relu')(x)
  
  outputs = keras.layers.Dense(NUM_CLASSES, activation='softmax')(x)

  return keras.models.Model(inputs=inputs, outputs=outputs)<jupyter_output><empty_output><jupyter_text>Initialize Model<jupyter_code>keras.backend.clear_session()
model = Model()
model.summary()<jupyter_output>Model: "model"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_1 (InputLayer)         [(None, 32, 32, 3)]       0         
_________________________________________________________________
conv2d (Conv2D)              (None, 30, 30, 32)        896       
_________________________________________________________________
conv2d_1 (Conv2D)            (None, 28, 28, 32)        9248      
_________________________________________________________________
max_pooling2d (MaxPooling2D) (None, 14, 14, 32)        0         
_________________________________________________________________
conv2d_2 (Conv2D)            (None, 12, 12, 32)        9248      
_________________________________________________________________
conv2d_3 (Conv2D)            (None, 10, 10, 32)        9248      
_____________________________________________________________[...]<jupyter_text>Compile<jupyter_code>model.compile('adam', 'sparse_categorical_crossentropy', metrics=['acc'])<jupyter_output><empty_output><jupyter_text>Wandb Init<jupyter_code>wandb.init(entity='authors', project='seo')<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>val_images, val_labels = next(iter(testloader))
EPOCHS = 25

_ = model.fit(trainloader,
          epochs=EPOCHS,
          validation_data=testloader,
          callbacks=[WandbCallback(data_type='image', 
                                   training_data=(val_images, val_labels), 
                                   labels=CLASS_NAMES),
                     lr_callback])<jupyter_output>Epoch 1/25
   1/1563 [..............................] - ETA: 0s - loss: 2.3030 - acc: 0.0625WARNING:tensorflow:Method (on_train_batch_end) is slow compared to the batch update (0.185473). Check your callbacks.
1563/1563 [==============================] - 19s 12ms/step - loss: 1.7858 - acc: 0.3104 - val_loss: 1.5890 - val_acc: 0.4066 - lr: 0.0010
Epoch 2/25
1563/1563 [==============================] - 18s 12ms/step - loss: 1.4486 - acc: 0.4674 - val_loss: 1.3369 - val_acc: 0.5076 - lr: 0.0010
Epoch 3/25
1563/1563 [==============================] - 18s 12ms/step - loss: 1.2733 - acc: 0.5370 - val_loss: 1.1915 - val_acc: 0.5695 - lr: 0.0010
Epoch 4/25
1563/1563 [==============================] - 18s 12ms/step - loss: 1.1637 - acc: 0.5812 - val_loss: 1.0907 - val_acc: 0.6048 - lr: 0.0010
Epoch 5/25
1563/1563 [==============================] - 18s 12ms/step - loss: 1.0787 - acc: 0.6116 - val_loss: 1.0452 - val_acc: 0.6254 - lr: 0.0010
Epoch 6/25
1563/1563 [==============================] - [...]<jupyter_text>Evaluate<jupyter_code>loss, accuracy = model.evaluate(testloader, callbacks=[WandbCallback()])
print('Test Error Rate: ', round((1-accuracy)*100, 2))
wandb.log({'Test Error Rate': round((1-accuracy)*100, 2)})<jupyter_output>313/313 [==============================] - 1s 4ms/step - loss: 0.7515 - acc: 0.7345
Test Error Rate:  26.55<jupyter_text>Batch Normalization  Model<jupyter_code>def ModelBN():
  inputs = keras.layers.Input(shape=(IMG_SHAPE, IMG_SHAPE, CHANNELS))

  x = keras.layers.Conv2D(filters=32,
                          kernel_size=(3,3),
                          strides=(1,1),
                          padding='valid')(inputs)
  x = keras.layers.BatchNormalization()(x)
  x = keras.activations.relu(x)

  x = keras.layers.Conv2D(filters=32, 
                          kernel_size=(3,3),
                          strides=(1,1),
                          padding='valid')(x)
  x = keras.layers.BatchNormalization()(x)
  x = keras.activations.relu(x)

  x = keras.layers.MaxPooling2D(pool_size=2)(x)

  x = keras.layers.Conv2D(filters=32, 
                          kernel_size=(3,3),
                          strides=(1,1),
                          padding='valid')(x)
  x = keras.layers.BatchNormalization()(x)
  x = keras.activations.relu(x)

  x = keras.layers.Conv2D(filters=32, 
                          kernel_size=(3,3),
                          strides=(1,1),
                          padding='valid')(x)
  x = keras.layers.BatchNormalization()(x)
  x = keras.activations.relu(x)

  x = keras.layers.GlobalAveragePooling2D()(x)

  x = keras.layers.Dense(128, activation='relu')(x)
  x = keras.layers.Dense(32, activation='relu')(x)
  
  outputs = keras.layers.Dense(NUM_CLASSES, activation='softmax')(x)

  return keras.models.Model(inputs=inputs, outputs=outputs)<jupyter_output><empty_output><jupyter_text>Initialize Model<jupyter_code>keras.backend.clear_session()
model = ModelBN()
model.summary()<jupyter_output>Model: "model"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_1 (InputLayer)         [(None, 32, 32, 3)]       0         
_________________________________________________________________
conv2d (Conv2D)              (None, 30, 30, 32)        896       
_________________________________________________________________
batch_normalization (BatchNo (None, 30, 30, 32)        128       
_________________________________________________________________
tf_op_layer_Relu (TensorFlow [(None, 30, 30, 32)]      0         
_________________________________________________________________
conv2d_1 (Conv2D)            (None, 28, 28, 32)        9248      
_________________________________________________________________
batch_normalization_1 (Batch (None, 28, 28, 32)        128       
_____________________________________________________________[...]<jupyter_text>Compile<jupyter_code>model.compile('adam', 'sparse_categorical_crossentropy', metrics=['acc'])<jupyter_output><empty_output><jupyter_text>Wandb Init<jupyter_code>wandb.init(entity='authors', project='seo')<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>EPOCHS = 25

_ = model.fit(trainloader,
          epochs=EPOCHS,
          validation_data=testloader,
          callbacks=[WandbCallback(data_type='image', 
                                   training_data=(val_images, val_labels), 
                                   labels=CLASS_NAMES),
                     lr_callback])<jupyter_output>Epoch 1/25
   1/1563 [..............................] - ETA: 0s - loss: 2.2980 - acc: 0.0625WARNING:tensorflow:Method (on_train_batch_end) is slow compared to the batch update (2.641788). Check your callbacks.
WARNING:tensorflow:Method (on_train_batch_end) is slow compared to the batch update (1.320914). Check your callbacks.
1563/1563 [==============================] - 20s 13ms/step - loss: 1.4229 - acc: 0.4757 - val_loss: 1.7831 - val_acc: 0.3591 - lr: 0.0010
Epoch 2/25
1563/1563 [==============================] - 20s 13ms/step - loss: 1.0872 - acc: 0.6120 - val_loss: 1.1547 - val_acc: 0.5928 - lr: 0.0010
Epoch 3/25
1563/1563 [==============================] - 20s 13ms/step - loss: 0.9603 - acc: 0.6594 - val_loss: 1.4112 - val_acc: 0.4958 - lr: 0.0010
Epoch 4/25
1563/1563 [==============================] - 20s 13ms/step - loss: 0.8833 - acc: 0.6868 - val_loss: 1.4743 - val_acc: 0.5207 - lr: 0.0010
Epoch 5/25
1563/1563 [==============================] - 20s 13ms/step - loss: 0.8274 - [...]<jupyter_text>Evaluate<jupyter_code>loss, accuracy = model.evaluate(testloader, callbacks=[WandbCallback()])
print('Test Error Rate: ', round((1-accuracy)*100, 2))
wandb.log({'Test Error Rate': round((1-accuracy)*100, 2)})<jupyter_output>313/313 [==============================] - 1s 4ms/step - loss: 0.6562 - acc: 0.7756
Test Error Rate:  22.44
StarcoderdataJupyterStructuredCleanDedup