<jupyter_start><jupyter_text>Parameter setting and import libraries<jupyter_code>from __future__ import print_function
#
import os
import glob
import re
import argparse
import pandas as pd
import numpy as np
import subprocess
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from sklearn import preprocessing
import matplotlib.pyplot as plt

import time
from time import sleep
from tqdm import tqdm # if use notebook
#from tqdm import tqdm_notebook as tqdm

from threading import Thread, Event, Timer
import queue

from PIL import Image
import cv2
import imgaug as ia
from imgaug import augmenters as iaa

import random

parser = argparse.ArgumentParser()
parser.add_argument('--gpu_id', default="4,5,6,7", type = str)
parser.add_argument('--image_dir', default="/data/seanyu/cat_dog/dataset/") # Path to where you put your data
parser.add_argument('--save_dir', default='/data/seanyu/cat_dog/model/')
parser.add_argument('--batch_size', default=48, type=int)
parser.add_argument('--do_augment', default=True, type = bool)
parser.add_argument('--epochs', default=50, type=int)
parser.add_argument('--lr', default=0.001, type=float)
parser.add_argument('--image_size', default=(200,200,3), type = int)
parser.add_argument('--n_classes', default=2, type = int)
parser.add_argument('--n_batch', default=100, type = int)
parser.add_argument('--train_ratio', default=0.99, type = float)
parser.add_argument('--use_model_ckpt', default = None, type = str)
parser.add_argument('--model_file_name', default = 'tmp_nb')
parser.add_argument('--n_threads', default = 4, type = int)
parser.add_argument('--dq_size', default = 6, type = int)
parser.add_argument('--use_pretrain', default = True, type = bool)
FLAGS = parser.parse_args([])
print(FLAGS)<jupyter_output>/home/seanyu/.conda/envs/tf18_keras/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88
  return f(*args, **kwds)
/home/seanyu/.conda/envs/tf18_keras/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88
  return f(*args, **kwds)
/home/seanyu/.conda/envs/tf18_keras/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88
  return f(*args, **kwds)
/home/seanyu/.conda/envs/tf18_keras/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88
  return f(*args, **kwds)
/home/seanyu/.conda/envs/tf18_keras/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88
  retu[...]<jupyter_text>Check path and load data<jupyter_code>os.environ['CUDA_VISIBLE_DEVICES'] = str(FLAGS.gpu_id)
import tensorflow as tf
import keras
from keras.utils.np_utils import to_categorical
import keras.backend as K
n_gpu = len(FLAGS.gpu_id.split(","))
""" 
horovod for keras setting
"""
if n_gpu > 1:
    print("Numbers of GPUs: %i" % n_gpu)
    import horovod.keras as hvd
    hvd.init()
    #config = tf.ConfigProto()
    #config.gpu_options.visible_device_list = str(hvd.local_rank())
    #K.set_session(tf.Session(config=config))


if not os.path.exists(FLAGS.save_dir):
    os.makedirs(FLAGS.save_dir)

model_dir = FLAGS.save_dir + '/model'
    
graphs_dir = FLAGS.save_dir + '/graphs'
if not os.path.exists(graphs_dir):
    os.makedirs(graphs_dir)

"""  Get data """
d_train = FLAGS.image_dir + '/train/'
d_test = FLAGS.image_dir + '/test1/'

image_train_list = glob.glob(d_train + '*.jpg')
image_test_list = glob.glob(d_test + '*.jpg')

df_train = pd.DataFrame({'img_path': image_train_list})
df_test = pd.DataFrame({'img_path': image_test_list})

df_train['cate'] = df_train.img_path.apply(os.path.basename)
df_train['cate'] = [i.split(".")[0] for i in list(df_train.cate)]
df_train.cate = df_train.cate.replace({'dog': 0, 'cat': 1})

nb_epoch = FLAGS.epochs

df_train_0, df_val_0 = train_test_split(df_train[df_train['cate'] == 0].sample(n=5000), test_size = 1-FLAGS.train_ratio)
df_train_1, df_val_1 = train_test_split(df_train[df_train['cate'] == 1].sample(n=5000), test_size = 1-FLAGS.train_ratio)

df_val = pd.concat((df_val_0, df_val_1)).reset_index(drop = True)

del df_val_0, df_val_1
def cv_load_and_resize(x, image_size, is_training = True, do_augment = False, seq = None):
    im_w, im_h, im_c = image_size
    im = cv2.imread(x)
    im = cv2.resize(im, (im_w, im_h))
    if do_augment and is_training:
        im = seq.augment_image(im)
    return im

# Read them all
x_train = np.array([cv_load_and_resize(i, image_size=FLAGS.image_size) for i in pd.concat((df_train_0, df_train_1)).img_path])
y_train = pd.concat((df_train_0, df_train_1)).cate.values
y_train = to_categorical(y_train)


x_val = np.array([cv_load_and_resize(i, image_size=FLAGS.image_size, is_training = False) for i in df_val.img_path])
y_val = df_val.cate.values
y_val = to_categorical(y_val)

print("Training set size: ", x_train.shape)
print("Validation set size: ", x_val.shape)<jupyter_output>Training set size:  (4948, 200, 200, 3)
Validation set size:  (52, 200, 200, 3)<jupyter_text>Callbacks<jupyter_code>from keras.callbacks import Callback
from keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping, TensorBoard

model_ckpt = ModelCheckpoint(filepath=FLAGS.save_dir + 'model.h5', monitor='val_loss', save_best_only=True, verbose=1)
reduce_lr = ReduceLROnPlateau(factor=0.5, monitor='val_loss', patience=4, verbose=1)
earlystop = EarlyStopping(min_delta=1e-8, monitor='val_loss', patience=10, verbose=1)
tbCallBack = TensorBoard(log_dir=FLAGS.save_dir+'/graphs', histogram_freq=0, write_graph=True, write_images=True)<jupyter_output><empty_output><jupyter_text>Build model<jupyter_code>import keras.backend as K
from keras.layers import (
    Conv2D, 
    Input, 
    Dense, 
    GlobalAveragePooling2D, 
    Activation
)
from keras.models import (
    Model, 
    load_model, 
    save_model)
from keras.optimizers import (
    SGD, 
    Adam, 
    TFOptimizer)
from keras.callbacks import (
    Callback, 
    ReduceLROnPlateau, 
    ModelCheckpoint, 
    EarlyStopping)

from keras.applications.resnet50 import ResNet50, preprocess_input
from keras.utils.generic_utils import get_custom_objects

#K.clear_session()

if FLAGS.use_pretrain:
    epoch_to_run = 25
    res_model = ResNet50(include_top=False, input_shape=FLAGS.image_size, weights=None)
    
    x = GlobalAveragePooling2D()(res_model.output)
    out = Dense(units=2, activation="softmax")(x)
    
    model = Model(inputs=[res_model.input], 
                  outputs=[out])
else:
    epoch_to_run = 50

optim = Adam(lr = FLAGS.lr)

if n_gpu > 1:
    optim = hvd.DistributedOptimizer(optim)
    callbacks = [hvd.callbacks.BroadcastGlobalVariablesCallback(0),
                ]
    if hvd.rank() == 0:
        callbacks.append(model_ckpt)
        callbacks.append(tbCallBack)
else:
    callbacks = [model_ckpt, reduce_lr, earlystop, tbCallBack]

model.compile(loss='categorical_crossentropy', metrics=['acc'], optimizer=optim)
model.summary()
history_model = model.fit(x_train, y_train, 
          batch_size=FLAGS.batch_size, 
          epochs=epoch_to_run, 
          shuffle=True, verbose=1, 
          validation_data=(x_val, y_val),
          callbacks = callbacks)<jupyter_output>Train on 4948 samples, validate on 52 samples
Epoch 1/25
4948/4948 [==============================] - 41s 8ms/step - loss: 0.8585 - acc: 0.5899 - val_loss: 8.0590 - val_acc: 0.5000

Epoch 00001: val_loss improved from inf to 8.05905, saving model to /data/seanyu/cat_dog/model/model.h5
Epoch 2/25
4948/4948 [==============================] - 35s 7ms/step - loss: 0.6827 - acc: 0.6326 - val_loss: 0.6946 - val_acc: 0.6154

Epoch 00002: val_loss improved from 8.05905 to 0.69461, saving model to /data/seanyu/cat_dog/model/model.h5
Epoch 3/25
4948/4948 [==============================] - 35s 7ms/step - loss: 0.6990 - acc: 0.6384 - val_loss: 8.0590 - val_acc: 0.5000

Epoch 00003: val_loss did not improve
Epoch 4/25
4948/4948 [==============================] - 35s 7ms/step - loss: 0.7077 - acc: 0.5855 - val_loss: 0.7316 - val_acc: 0.5385

Epoch 00004: val_loss did not improve
Epoch 5/25
4948/4948 [==============================] - 36s 7ms/step - loss: 0.7034 - acc: 0.6275 - val_loss: 0.7672 - val[...]<jupyter_text>Visualization with Grad-CAM<jupyter_code>from keras.models import load_model
import keras.backend as K
model = load_model("/data/seanyu/cat_dog/model/model.h5")
layer = 'add_16' # layer before GAP and last activation
idx = 201
sess = K.get_session()

#sess.graph.get_tensor_by_name(layer)
#model.layers
op = sess.graph.get_operations()
#tmp = [m.values() for m in op]
tmp = [m.values() for m in op if layer in m.name]
#tmp = [tmp[i][0].name for i in np.arange(len(tmp))]
tmp
#tmp.remove('')
[print(i) for i in tmp if i != '']
def grad_cam(model, image, cls_select, tf_sess, layer, alpha = 0.6):
    # Visualization with Grad-CAM
    if image.shape != 4:
        # Should be batch input, because we're going to send it into model
        image = np.expand_dims(image, axis = 0)
    H,W = image.shape[1], image.shape[2]

    #tf_sess = K.get_session()
    y_c = model.output[0, cls_select] # output tensor -> take slice of selected class
    conv_output = model.get_layer(layer).output # layer tensor
    grads = tf.gradients(y_c, conv_output)[0]

    output, grads_val = tf_sess.run([conv_output, grads], 
                                    feed_dict = {model.input.name:image})
    # --> shape: output = grads_val = 1 x 8 x 8 x 2048
    output, grads_val = output[0, :], grads_val[0, :,:,:]
    # --> shape: output = 8 x 8 x 2048,  grads_val = 8 x 8 x 2048

    weights = np.mean(grads_val, axis = (0, 1))
    cam = np.dot(output, weights) 
    # --> shape: cam = 8 x 8

    ## resize it ##
    cam = tf.image.resize_bicubic(images=cam[np.newaxis, : , :, np.newaxis], 
                             size=(H, W))
    cam = cam[0, :, :, 0]
    cam = tf.maximum(cam, 0)
    cam = cam / tf.reduce_max(cam)
    cam = tf_sess.run(cam)
    ## do image merge ##
    # smooth the image (because upsampling make image grid-like)
    
    # apply color map
    mapping = cv2.applyColorMap(np.uint8(255 * (1-cam) ), cv2.COLORMAP_JET)
    mapping = cv2.GaussianBlur(mapping, (11,11), 5)
    mapping = np.concatenate((mapping, ((mapping.max(axis=-1) - 128 )*255*alpha)[:,:,np.newaxis]), axis = -1)

    # foreground - background
    background = Image.fromarray(image[0])
    foreground = Image.fromarray(mapping.astype('uint8'))
    background.paste(foreground, (0, 0), foreground)
    
    return cam, background
layer = 'add_16' # layer before GAP and last activation
idx = 201
sess = K.get_session()
cam_out, cam_map = grad_cam(model = model, tf_sess=sess,
                            image = x_val[idx], cls_select= y_val[idx].argmax(), layer=layer)
cam_map<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup