<jupyter_start><jupyter_text>Dependencies<jupyter_code>from utillity_script_cloud_segmentation import *
from utillity_script_lr_schedulers import *
seed = 0
seed_everything(seed)
warnings.filterwarnings("ignore")
base_path = '/content/drive/My Drive/Colab Notebooks/[Kaggle] Understanding Clouds from Satellite Images/'
data_path = base_path + 'Data/'
model_base_path = base_path + 'Models/files/classification/'
train_path = data_path + 'train.csv'
hold_out_set_path = data_path + 'hold-out.csv'
train_images_dest_path = 'train_images/'<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>train = pd.read_csv(train_path)
hold_out_set = pd.read_csv(hold_out_set_path)

X_train = hold_out_set[hold_out_set['set'] == 'train']
X_val = hold_out_set[hold_out_set['set'] == 'validation']

print('Compete set samples:', len(train))
print('Train samples: ', len(X_train))
print('Validation samples: ', len(X_val))

# Preprocecss data
train['image'] = train['Image_Label'].apply(lambda x: x.split('_')[0])
label_columns=['Fish', 'Flower', 'Gravel', 'Sugar']
for label in label_columns:
  X_train[label].replace({0: 1, 1: 0}, inplace=True)
  X_val[label].replace({0: 1, 1: 0}, inplace=True)

display(X_train.head())<jupyter_output>Compete set samples: 22184
Train samples:  4420
Validation samples:  1105<jupyter_text>Model parameters<jupyter_code>BATCH_SIZE = 64
WARMUP_EPOCHS = 3
WARMUP_LEARNING_RATE = 1e-3
EPOCHS = 20
MAX_LEARNING_RATE = 10**(-1.7)
HEIGHT = 224
WIDTH = 224
CHANNELS = 3
N_CLASSES = 4
ES_PATIENCE = 8
STEP_SIZE_TRAIN = len(X_train)//BATCH_SIZE
STEP_SIZE_VALID = len(X_val)//BATCH_SIZE

model_name = '17-EfficientNetB0_%sx%s_Cosine' % (HEIGHT, WIDTH)
model_path = model_base_path + '%s.h5' % (model_name)<jupyter_output><empty_output><jupyter_text>Data generator<jupyter_code>datagen=ImageDataGenerator(rescale=1./255.,
                           vertical_flip=True,
                           horizontal_flip=True,
                           zoom_range=[1, 1.2],
                           fill_mode='constant',
                           cval=0.)

test_datagen=ImageDataGenerator(rescale=1./255.)

train_generator=datagen.flow_from_dataframe(
                        dataframe=X_train,
                        directory=train_images_dest_path,
                        x_col="image",
                        y_col=label_columns,
                        target_size=(HEIGHT, WIDTH),
                        batch_size=BATCH_SIZE,
                        class_mode="other",
                        shuffle=True,
                        seed=seed)

valid_generator=test_datagen.flow_from_dataframe(
                             dataframe=X_val,
                             directory=train_images_dest_path,
                             x_col="image",
                             y_col=label_columns,
                             target_size=(HEIGHT, WIDTH),
                             batch_size=BATCH_SIZE,
                             class_mode="other",
                             shuffle=True,
                             seed=seed)<jupyter_output>Found 4420 validated image filenames.
Found 1105 validated image filenames.<jupyter_text>Model<jupyter_code>def create_model(input_shape, N_CLASSES):
    input_tensor = Input(shape=input_shape)
    base_model = efn.EfficientNetB0(weights='imagenet', 
                                    include_top=False,
                                    input_tensor=input_tensor, 
                                    pooling='avg')

    x = base_model.output
    final_output = Dense(N_CLASSES, activation='sigmoid')(x)
    model = Model(input_tensor, final_output)
    
    return model<jupyter_output><empty_output><jupyter_text>Learning rate finder  Warmup top layers<jupyter_code>model = create_model((None, None, CHANNELS), N_CLASSES)
metric_list = ['accuracy']

for layer in model.layers[:-1]:
  layer.trainable = False

optimizer = optimizers.SGD(lr=WARMUP_LEARNING_RATE, momentum=0.9, nesterov=True)
model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=metric_list)
# model.summary()
warmup_history = model.fit_generator(generator=train_generator,
                                      steps_per_epoch=STEP_SIZE_TRAIN,
                                      validation_data=valid_generator,
                                      validation_steps=STEP_SIZE_VALID,
                                      epochs=WARMUP_EPOCHS,
                                      verbose=1).history<jupyter_output>Epoch 1/3
69/69 [==============================] - 71s 1s/step - loss: 0.7019 - acc: 0.5100 - val_loss: 0.7001 - val_acc: 0.5248
Epoch 2/3
69/69 [==============================] - 67s 969ms/step - loss: 0.6826 - acc: 0.5633 - val_loss: 0.6961 - val_acc: 0.5310
Epoch 3/3
69/69 [==============================] - 66s 964ms/step - loss: 0.6750 - acc: 0.5785 - val_loss: 0.6922 - val_acc: 0.5286<jupyter_text>Fine-tune all layers<jupyter_code>for layer in model.layers:
  layer.trainable = True

checkpoint = ModelCheckpoint(model_path, monitor='val_loss', mode='min', save_best_only=True)
es = EarlyStopping(monitor='val_loss', mode='min', patience=ES_PATIENCE, restore_best_weights=True, verbose=1)
cosine_lr = WarmUpCosineDecayScheduler(learning_rate_base=MAX_LEARNING_RATE,
                                           total_steps=(STEP_SIZE_TRAIN * EPOCHS),
                                           warmup_steps=(1 * STEP_SIZE_TRAIN),
                                           hold_base_rate_steps=0)
callback_list = [checkpoint, es, cosine_lr]

optimizer = optimizers.SGD(lr=MAX_LEARNING_RATE, momentum=0.9, nesterov=True)
model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=metric_list)
# model.summary()
history = model.fit_generator(generator=train_generator,
                              steps_per_epoch=STEP_SIZE_TRAIN,
                              validation_data=valid_generator,
                              validation_steps=STEP_SIZE_VALID,
                              callbacks=callback_list,
                              epochs=EPOCHS,
                              verbose=1).history<jupyter_output>Epoch 1/20
69/69 [==============================] - 104s 2s/step - loss: 0.6461 - acc: 0.6241 - val_loss: 0.6092 - val_acc: 0.6721
Epoch 2/20
69/69 [==============================] - 90s 1s/step - loss: 0.5595 - acc: 0.7112 - val_loss: 0.6029 - val_acc: 0.6727
Epoch 3/20
69/69 [==============================] - 91s 1s/step - loss: 0.5295 - acc: 0.7356 - val_loss: 0.5579 - val_acc: 0.7104
Epoch 4/20
69/69 [==============================] - 91s 1s/step - loss: 0.5096 - acc: 0.7464 - val_loss: 0.5500 - val_acc: 0.7109
Epoch 5/20
69/69 [==============================] - 91s 1s/step - loss: 0.4964 - acc: 0.7553 - val_loss: 0.5461 - val_acc: 0.7171
Epoch 6/20
69/69 [==============================] - 91s 1s/step - loss: 0.4811 - acc: 0.7635 - val_loss: 0.5558 - val_acc: 0.7137
Epoch 7/20
69/69 [==============================] - 90s 1s/step - loss: 0.4688 - acc: 0.7731 - val_loss: 0.5381 - val_acc: 0.7253
Epoch 8/20
69/69 [==============================] - 89s 1s/step - loss: 0.4616 - acc: 0.7[...]<jupyter_text>Model loss graph<jupyter_code>#@title
metrics_history = ['loss', 'acc']
for metric_hist in metrics_history:
  history[metric_hist] = warmup_history[metric_hist] + history[metric_hist]
  history['val_' + metric_hist] = warmup_history['val_' + metric_hist] + history['val_' + metric_hist]

plot_metrics(history, metric_list=metrics_history)<jupyter_output><empty_output><jupyter_text>Scheduler learning rates<jupyter_code>#@title
fig, ax1 = plt.subplots(1, 1, figsize=(20, 6))

ax1.plot(cosine_lr.learning_rates)
ax1.set_title('Fine-tune learning rates')

plt.xlabel('Steps')
plt.ylabel('Learning rate')
sns.despine()
plt.show()<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup