<jupyter_start><jupyter_text>Installing retinanet library<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
# a = []
# while(1):
#     a.append(1)
!git clone --recursive https://github.com/fizyr/keras-retinanet.git
!ls
!cd keras-retinanet && pip install . --user
!pip install Cython
!pip install --user git+https://github.com/cocodataset/cocoapi.git#subdirectory=PythonAPI
!cd keras-retinanet/snapshots && \
# wget https://github.com/fizyr/keras-retinanet/releases/download/0.2/resnet50_coco_best_v2.0.3.h5
# wget https://github.com/ZFTurbo/Keras-RetinaNet-for-Open-Images-Challenge-2018/releases/download/v1.3/retinanet_resnet101_500_classes_0.4986.h5
!pip install --upgrade git+https://github.com/fizyr/keras-retinanet
!pip install --upgrade git+https://github.com/broadinstitute/keras-resnet
import keras
import keras_resnet

import xml.etree.ElementTree as ET
import os
import numpy as np
import keras
import math
import tensorflow as tf
import cv2
from os import listdir, walk
from os.path import join
from keras_retinanet.bin.train import create_models
from keras_retinanet.models import backbone,load_model, convert_model
from keras_retinanet.utils.config import read_config_file, parse_anchor_parameters
from keras_retinanet.utils.visualization import draw_boxes
from sklearn.model_selection import train_test_split
from imgaug import augmenters as iaa
import keras
import keras_resnet

# tf.set_random_seed(31) # SEEDS MAKE RESULTS MORE REPRODUCABLE
np.random.seed(17)
classes = np.arange(0, 6, 1).tolist()

import pandas as pd

tr_annots = pd.read_csv('/content/drive/My Drive/RA2/Retinanet/Upload for Kaggle/training_data/retinanet train val files new/all_hand_train_03_06.csv', header = None)
tr_annots['ids'] = tr_annots[0].apply(lambda x: x.split("/")[-1].split(".")[0])
train_ids = tr_annots['ids'].unique().tolist()

import pandas as pd
tr_annots = pd.read_csv('/content/drive/My Drive/RA2/Retinanet/Upload for Kaggle/training_data/retinanet train val files new/all_hand_val_03_06.csv', header = None)
tr_annots['ids'] = tr_annots[0].apply(lambda x: x.split("/")[-1].split(".")[0])
val_ids = tr_annots['ids'].unique().tolist()

print(len(train_ids), len(val_ids))<jupyter_output>341 57<jupyter_text>Previous config files for hand models<jupyter_code># Version 3 had this config file
# with open('config.ini','w') as f:
#     f.write('[anchor_parameters]\nsizes   = 32 64 128 256 512\nstrides = 8 16 32 64 128\nratios  = 1 1.5 2 2.5 \nscales  = 1 2\n')

# #Changing the config file to increase the area of the bounding boxes and including a ratio 3 in the config file
# This gives mAP 0.9446 class loss 0.3669
# with open('config.ini','w') as f:
#     f.write('[anchor_parameters]\nsizes   = 32 64 128 256 512 1024\nstrides = 8 16 32 64 128 256\nratios  = 1 1.2 1.5 1.8 2 2.5 2.8 3\nscales  =1 1.2 1.6\n')

## Changing the config file for the retraining
# #Changing the config file to increase the area of the bounding boxes and including a ratio 3 in the config file
# with open('config.ini','w') as f:
#     f.write('[anchor_parameters]\nsizes   = 32 64 128 256 512\nstrides = 8 16 32 64 128\nratios  = 1 1.5 2 2.2\nscales  =1 2\n')<jupyter_output><empty_output><jupyter_text>Final config file for Hand models<jupyter_code># Config file for models generated on 06 june
# with open('config.ini','w') as f:
#     f.write('[anchor_parameters]\nsizes   = 32 64 128 256 512\nstrides = 8 16 32 64 128\nratios  = 0.8 1 1.3 1.5 2 2.2 2.8 3\nscales  =1 1.2 1.6\n')

# # New config file after judging from the anchors that are generated
# # Testing the config file ratios 
# with open('config.ini','w') as f:
#     f.write('[anchor_parameters]\nsizes   = 32 64 128 256 512\nstrides = 8 16 32 64 128\nratios  = 0.5 1 1.5 2 2.5 3\nscales  =1 1.2 1.6\n')<jupyter_output><empty_output><jupyter_text>Changing the ratios since the wrist joint is not being detected properly<jupyter_code>with open('config.ini','w') as f:
    f.write('[anchor_parameters]\nsizes   = 32 64 128 256 512 1024 2048\nstrides = 8 16 32 64 128 256 512\nratios  = 0.5 1 1.5 2 2.5\nscales  =1 1.5 2\n')

b = backbone('resnet50')

class args:
    batch_size = 8
    config = read_config_file('config.ini')
    random_transform = True # Image augmentation
    annotations = '/content/drive/My Drive/RA2/Retinanet/Upload for Kaggle/training_data/retinanet train val files new/all_hand_train_03_06.csv'
    val_annotations = '/content/drive/My Drive/RA2/Retinanet/Upload for Kaggle/training_data/retinanet train val files new/all_hand_val_03_06.csv'
    classes = '/content/drive/My Drive/RA2/Retinanet/Upload for Kaggle/training_data/retinanet train val files new/Hands class names.csv'
    image_min_side = 1000
    image_max_side = 1400
    no_resize = False
    dataset_type = 'csv'
    tensorboard_dir = ''
    evaluation = False
    snapshots = True
    snapshot_path = "/content/drive/My Drive/RA2/Retinanet/Upload for Kaggle/Hand models New annotations/Models v6"
    backbone = 'resnet50'
    epochs = 200
    steps = len(train_ids)//(batch_size)
    weighted_average = True
    reduce_lr_factor = 0.1
    reduce_lr_patience = 4
    compute_val_loss = True
    iou_threshold = 0.6
    nms_threshold = 0.5
    score_threshold = 0.15
    anchors = True
    resize = True
    display_name = 'Anchors'
    no_gui = False<jupyter_output><empty_output><jupyter_text>Create generators function<jupyter_code>from keras_retinanet.preprocessing.generator import Generator
# from ..utils.image import read_image_bgr
from keras_retinanet.utils.transform import random_transform_generator
from keras_retinanet.utils.image import random_visual_effect_generator

import numpy as np
from PIL import Image
from six import raise_from

import csv
import sys
import os.path
from collections import OrderedDict

import PIL.Image
import PIL.ImageOps
import numpy as np
from os import makedirs 


def exif_transpose(img):
    if not img:
        return img

    exif_orientation_tag = 274

    # Check for EXIF data (only present on some files)
    if hasattr(img, "_getexif") and isinstance(img._getexif(), dict) and exif_orientation_tag in img._getexif():
        exif_data = img._getexif()
        orientation = exif_data[exif_orientation_tag]

        # Handle EXIF Orientation
        if orientation == 1:
            # Normal image - nothing to do!
            pass
        elif orientation == 2:
            # Mirrored left to right
            img = img.transpose(PIL.Image.FLIP_LEFT_RIGHT)
        elif orientation == 3:
            # Rotated 180 degrees
            img = img.rotate(180)
        elif orientation == 4:
            # Mirrored top to bottom
            img = img.rotate(180).transpose(PIL.Image.FLIP_LEFT_RIGHT)
        elif orientation == 5:
            # Mirrored along top-left diagonal
            img = img.rotate(-90, expand=True).transpose(PIL.Image.FLIP_LEFT_RIGHT)
        elif orientation == 6:
            # Rotated 90 degrees
            img = img.rotate(-90, expand=True)
        elif orientation == 7:
            # Mirrored along top-right diagonal
            img = img.rotate(90, expand=True).transpose(PIL.Image.FLIP_LEFT_RIGHT)
        elif orientation == 8:
            # Rotated 270 degrees
            img = img.rotate(90, expand=True)

    return img


def load_image_file(file, mode='RGB'):
    # Load the image with PIL
    img = PIL.Image.open(file)

    if hasattr(PIL.ImageOps, 'exif_transpose'):
        # Very recent versions of PIL can do exit transpose internally
        img = PIL.ImageOps.exif_transpose(img)
    else:
        # Otherwise, do the exif transpose ourselves
        img = exif_transpose(img)

    img = img.convert(mode)

    return np.array(img)

def _parse(value, function, fmt):
    """
    Parse a string into a value, and format a nice ValueError if it fails.

    Returns `function(value)`.
    Any `ValueError` raised is catched and a new `ValueError` is raised
    with message `fmt.format(e)`, where `e` is the caught `ValueError`.
    """
    try:
        return function(value)
    except ValueError as e:
        raise_from(ValueError(fmt.format(e)), None)


def _read_classes(csv_reader):
    """ Parse the classes file given by csv_reader.
    """
    result = OrderedDict()
    for line, row in enumerate(csv_reader):
        line += 1

        try:
            class_name, class_id = row
        except ValueError:
            raise_from(ValueError('line {}: format should be \'class_name,class_id\''.format(line)), None)
        class_id = _parse(class_id, int, 'line {}: malformed class ID: {{}}'.format(line))

        if class_name in result:
            raise ValueError('line {}: duplicate class name: \'{}\''.format(line, class_name))
        result[class_name] = class_id
    return result


def _read_annotations(csv_reader, classes):
    """ Read annotations from the csv_reader.
    """
    result = OrderedDict()
    for line, row in enumerate(csv_reader):
        line += 1

        try:
            img_file, x1, y1, x2, y2, class_name = row[:6]
        except ValueError:
            raise_from(ValueError('line {}: format should be \'img_file,x1,y1,x2,y2,class_name\' or \'img_file,,,,,\''.format(line)), None)

        if img_file not in result:
            result[img_file] = []

        # If a row contains only an image path, it's an image without annotations.
        if (x1, y1, x2, y2, class_name) == ('', '', '', '', ''):
            continue

        x1 = _parse(x1, int, 'line {}: malformed x1: {{}}'.format(line))
        y1 = _parse(y1, int, 'line {}: malformed y1: {{}}'.format(line))
        x2 = _parse(x2, int, 'line {}: malformed x2: {{}}'.format(line))
        y2 = _parse(y2, int, 'line {}: malformed y2: {{}}'.format(line))

        # Check that the bounding box is valid.
        if x2 <= x1:
            raise ValueError('line {}: x2 ({}) must be higher than x1 ({})'.format(line, x2, x1))
        if y2 <= y1:
            raise ValueError('line {}: y2 ({}) must be higher than y1 ({})'.format(line, y2, y1))

        # check if the current class name is correctly present
        if class_name not in classes:
            raise ValueError('line {}: unknown class name: \'{}\' (classes: {})'.format(line, class_name, classes))

        result[img_file].append({'x1': x1, 'x2': x2, 'y1': y1, 'y2': y2, 'class': class_name})
    return result


def _open_for_csv(path):
    """ Open a file with flags suitable for csv.reader.

    This is different for python2 it means with mode 'rb',
    for python3 this means 'r' with "universal newlines".
    """
    if sys.version_info[0] < 3:
        return open(path, 'rb')
    else:
        return open(path, 'r', newline='')


class CSVGenerator(Generator):
    """ Generate data for a custom CSV dataset.

    See https://github.com/fizyr/keras-retinanet#csv-datasets for more information.
    """

    def __init__(
        self,
        csv_data_file,
        csv_class_file,
        base_dir=None,
        **kwargs
    ):
        """ Initialize a CSV data generator.

        Args
            csv_data_file: Path to the CSV annotations file.
            csv_class_file: Path to the CSV classes file.
            base_dir: Directory w.r.t. where the files are to be searched (defaults to the directory containing the csv_data_file).
        """
        self.image_names = []
        self.image_data  = {}
        self.base_dir    = base_dir

        # Take base_dir from annotations file if not explicitly specified.
        if self.base_dir is None:
            self.base_dir = os.path.dirname(csv_data_file)

        # parse the provided class file
        try:
            with _open_for_csv(csv_class_file) as file:
                self.classes = _read_classes(csv.reader(file, delimiter=','))
        except ValueError as e:
            raise_from(ValueError('invalid CSV class file: {}: {}'.format(csv_class_file, e)), None)

        self.labels = {}
        for key, value in self.classes.items():
            self.labels[value] = key

        # csv with img_path, x1, y1, x2, y2, class_name
        try:
            with _open_for_csv(csv_data_file) as file:
                self.image_data = _read_annotations(csv.reader(file, delimiter=','), self.classes)
        except ValueError as e:
            raise_from(ValueError('invalid CSV annotations file: {}: {}'.format(csv_data_file, e)), None)
        self.image_names = list(self.image_data.keys())

        super(CSVGenerator, self).__init__(**kwargs)

    def size(self):
        """ Size of the dataset.
        """
        return len(self.image_names)

    def num_classes(self):
        """ Number of classes in the dataset.
        """
        return max(self.classes.values()) + 1

    def has_label(self, label):
        """ Return True if label is a known label.
        """
        return label in self.labels

    def has_name(self, name):
        """ Returns True if name is a known class.
        """
        return name in self.classes

    def name_to_label(self, name):
        """ Map name to label.
        """
        return self.classes[name]

    def label_to_name(self, label):
        """ Map label to name.
        """
        return self.labels[label]

    def image_path(self, image_index):
        """ Returns the image path for image_index.
        """
        return os.path.join(self.base_dir, self.image_names[image_index])

    def image_aspect_ratio(self, image_index):
        """ Compute the aspect ratio for an image with image_index.
        """
        # PIL is fast for metadata
        image = Image.open(self.image_path(image_index))
        return float(image.width) / float(image.height)

    def load_image(self, image_index):
        """ Load an image at the image_index.
        """
        return load_image_file(self.image_path(image_index))

    def load_annotations(self, image_index):
        """ Load annotations for an image_index.
        """
        path        = self.image_names[image_index]
        annotations = {'labels': np.empty((0,)), 'bboxes': np.empty((0, 4))}

        for idx, annot in enumerate(self.image_data[path]):
            annotations['labels'] = np.concatenate((annotations['labels'], [self.name_to_label(annot['class'])]))
            annotations['bboxes'] = np.concatenate((annotations['bboxes'], [[
                float(annot['x1']),
                float(annot['y1']),
                float(annot['x2']),
                float(annot['y2']),
            ]]))

        return annotations

def mAP(y_true, y_pred):
    num_classes = y_true.shape[1]
    average_precisions = []
    relevant = K.sum(K.round(K.clip(y_true, 0, 1)))
    tp_whole = K.round(K.clip(y_true * y_pred, 0, 1))
    for index in range(num_classes):
      temp = K.sum(tp_whole[:,:index+1],axis=1)
      average_precisions.append(temp * (1/(index + 1)))
    AP = Add()(average_precisions) / relevant
    mAP = K.mean(AP,axis=0)
    return mAP

def create_callbacks(model, training_model, prediction_model, validation_generator, args):
    """ Creates the callbacks to use during training.

    Args
        model: The base model.
        training_model: The model that is used for training.
        prediction_model: The model that should be used for validation.
        validation_generator: The generator for creating validation data.
        args: parseargs args object.

    Returns:
        A list of callbacks used for training.
    """
    callbacks = []

    tensorboard_callback = None

    if args.tensorboard_dir:
        makedirs(args.tensorboard_dir)
        tensorboard_callback = keras.callbacks.TensorBoard(
            log_dir                = args.tensorboard_dir,
            histogram_freq         = 0,
            batch_size             = args.batch_size,
            write_graph            = True,
            write_grads            = False,
            write_images           = False,
            embeddings_freq        = 0,
            embeddings_layer_names = None,
            embeddings_metadata    = None
        )

    if args.evaluation and validation_generator:
        if args.dataset_type == 'coco':
            from ..callbacks.coco import CocoEval

            # use prediction model for evaluation
            evaluation = CocoEval(validation_generator, tensorboard=tensorboard_callback)
        else:
            evaluation = Evaluate(validation_generator, tensorboard=tensorboard_callback, weighted_average=args.weighted_average)
        evaluation = RedirectModel(evaluation, prediction_model)
        callbacks.append(evaluation)

    # save the model
    if args.snapshots:
        # ensure directory created first; otherwise h5py will error after epoch.
        makedirs(args.snapshot_path)
        checkpoint = keras.callbacks.ModelCheckpoint(
            os.path.join(
                args.snapshot_path,
                '{backbone}_{dataset_type}_{{epoch:02d}}.h5'.format(backbone=args.backbone, dataset_type=args.dataset_type)
            ),
            verbose=1,
            # save_best_only=True,
            # monitor="mAP",
            # mode='max'
        )
        checkpoint = RedirectModel(checkpoint, model)
        callbacks.append(checkpoint)

    callbacks.append(keras.callbacks.ReduceLROnPlateau(
        monitor    = 'loss',
        factor     = args.reduce_lr_factor,
        patience   = args.reduce_lr_patience,
        verbose    = 1,
        mode       = 'auto',
        min_delta  = 0.0001,
        cooldown   = 0,
        min_lr     = 0
    ))

    callbacks.append(keras.callbacks.EarlyStopping(
        monitor    = 'classification_loss',
        patience   = 200,
        mode       = 'max',
        min_delta  = 0.01
    ))

    if args.tensorboard_dir:
        callbacks.append(tensorboard_callback)

    return callbacks

def create_generators(args, preprocess_image):
    """ Create generators for training and validation.

    Args
        args             : parseargs object containing configuration for generators.
        preprocess_image : Function that preprocesses an image for the network.
    """
    common_args = {
        'batch_size'       : args.batch_size,
        'config'           : args.config,
        'image_min_side'   : args.image_min_side,
        'image_max_side'   : args.image_max_side,
        'no_resize'        : args.no_resize,
        'preprocess_image' : preprocess_image,
    }

    # create random transform generator for augmenting training data
    if args.random_transform:
        transform_generator = random_transform_generator(
            min_rotation=-0.1,
            max_rotation=0.1,
            min_translation=(-0.1, -0.1),
            max_translation=(0.1, 0.1),
            min_shear=-0.1,
            max_shear=0.1,
            min_scaling=(0.9, 0.9),
            max_scaling=(1.1, 1.1),
            flip_x_chance=0.5,
            flip_y_chance=0.5,
        )
        visual_effect_generator = random_visual_effect_generator(
            contrast_range=(0.9, 1.1),
            brightness_range=(-.1, .1),
            hue_range=(-0.05, 0.05),
            saturation_range=(0.95, 1.05)
        )
    else:
        transform_generator = random_transform_generator(flip_x_chance=0.5)
        visual_effect_generator = None

    if args.dataset_type == 'coco':
        # import here to prevent unnecessary dependency on cocoapi
        from ..preprocessing.coco import CocoGenerator

        train_generator = CocoGenerator(
            args.coco_path,
            'train2017',
            transform_generator=transform_generator,
            visual_effect_generator=visual_effect_generator,
            **common_args
        )

        validation_generator = CocoGenerator(
            args.coco_path,
            'val2017',
            shuffle_groups=False,
            **common_args
        )
    elif args.dataset_type == 'pascal':
        train_generator = PascalVocGenerator(
            args.pascal_path,
            'train',
            image_extension=args.image_extension,
            transform_generator=transform_generator,
            visual_effect_generator=visual_effect_generator,
            **common_args
        )

        validation_generator = PascalVocGenerator(
            args.pascal_path,
            'val',
            image_extension=args.image_extension,
            shuffle_groups=False,
            **common_args
        )
    elif args.dataset_type == 'csv':
        train_generator = CSVGenerator(
            args.annotations,
            args.classes,
            transform_generator=transform_generator,
            visual_effect_generator=visual_effect_generator,
            **common_args
        )

        if args.val_annotations:
            validation_generator = CSVGenerator(
                args.val_annotations,
                args.classes,
                shuffle_groups=False,
                **common_args
            )
        else:
            validation_generator = None
    elif args.dataset_type == 'oid':
        train_generator = OpenImagesGenerator(
            args.main_dir,
            subset='train',
            version=args.version,
            labels_filter=args.labels_filter,
            annotation_cache_dir=args.annotation_cache_dir,
            parent_label=args.parent_label,
            transform_generator=transform_generator,
            visual_effect_generator=visual_effect_generator,
            **common_args
        )

        validation_generator = OpenImagesGenerator(
            args.main_dir,
            subset='validation',
            version=args.version,
            labels_filter=args.labels_filter,
            annotation_cache_dir=args.annotation_cache_dir,
            parent_label=args.parent_label,
            shuffle_groups=False,
            **common_args
        )
    elif args.dataset_type == 'kitti':
        train_generator = KittiGenerator(
            args.kitti_path,
            subset='train',
            transform_generator=transform_generator,
            visual_effect_generator=visual_effect_generator,
            **common_args
        )

        validation_generator = KittiGenerator(
            args.kitti_path,
            subset='val',
            shuffle_groups=False,
            **common_args
        )
    else:
        raise ValueError('Invalid data type received: {}'.format(args.dataset_type))

    return train_generator, validation_generator

train_gen, valid_gen = create_generators(args, b.preprocess_image)<jupyter_output><empty_output><jupyter_text>Visualizing anchor boxes<jupyter_code># # from keras_retinanet.bin.train import create_generators
# # from keras_retinanet.models import backbone
# from keras_retinanet.utils.config import read_config_file,parse_anchor_parameters
# # # from keras_retinanet.bin.debug import run
# from google.colab.patches import cv2_imshow
# from keras_retinanet.utils.anchors import anchors_for_shape, compute_gt_annotations
# from keras_retinanet.utils.visualization import draw_annotations, draw_boxes, draw_caption


# def run(generator, args, anchor_params):
#     """ Main loop.

#     Args
#         generator: The generator to debug.
#         args: parseargs args object.
#     """
#     # display images, one at a time
#     i = 0
#     while True:
#         # load the data
#         image       = generator.load_image(i)

#         annotations = generator.load_annotations(i)
#         if len(annotations['labels']) > 0 :
#             # apply random transformations
#             if args.random_transform:
#                 image, annotations = generator.random_transform_group_entry(image, annotations)
#                 image, annotations = generator.random_visual_effect_group_entry(image, annotations)

#             # resize the image and annotations
#             if args.resize:
#                 image, image_scale = generator.resize_image(image)
#                 annotations['bboxes'] *= image_scale

#             anchors = anchors_for_shape(image.shape, anchor_params=anchor_params)
#             positive_indices, _, max_indices = compute_gt_annotations(anchors, annotations['bboxes'])

#             # draw anchors on the image
#             if args.anchors:
#                 draw_boxes(image, anchors[positive_indices], (255, 255, 0), thickness=1)

#             # draw annotations on the image
#             if args.annotations:
#                 # draw annotations in red
#                 draw_annotations(image, annotations, color=(0, 0, 255), label_to_name=generator.label_to_name)

#                 # draw regressed anchors in green to override most red annotations
#                 # result is that annotations without anchors are red, with anchors are green
#                 draw_boxes(image, annotations['bboxes'][max_indices[positive_indices], :], (0, 255, 0))

#             # display name on the image
#             if args.display_name:
#                 draw_caption(image, [0, image.shape[0]], os.path.basename(generator.image_path(i)))

#         # write to file and advance if no-gui selected
#         if args.no_gui:
#             output_path = make_output_path(args.output_dir, generator.image_path(i), flatten=args.flatten_output)
#             os.makedirs(os.path.dirname(output_path), exist_ok=True)
#             cv2.imwrite(output_path, image)
#             i += 1
#             if i == generator.size():  # have written all images
#                 break
#             else:
#                 continue

#         # if we are using the GUI, then show an image
#         cv2_imshow(image)
#         key = cv2.waitKeyEx()

#         # press right for next image and left for previous (linux or windows, doesn't work for macOS)
#         # if you run macOS, press "n" or "m" (will also work on linux and windows)

#         if key in rightkeys:
#             i = (i + 1) % generator.size()
#         if key in leftkeys:
#             i -= 1
#             if i < 0:
#                 i = generator.size() - 1

#         # press q or Esc to quit
#         if (key == ord('q')) or (key == 27):
#             return False

#     return True


# # train_gen, valid_gen = create_generators(args,b.preprocess_image)
# while run(train_gen, args, parse_anchor_parameters(args.config)):
#     pass
import imgaug as ia
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
# Define our sequence of augmentation steps that will be applied to every image.
seq = iaa.Sequential(
[      
        # sometimes(iaa.Crop(percent=(0, 0.2))),
 
        #  sometimes(iaa.Affine(
        #     scale={"x": (0.9, 1.2), "y": (0.9, 1.2)},
        #     translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)},
        #     rotate=(-45, 45),
        #     shear=(-16, 16),
        #     order=[0, 1],
        #     cval=(0, 255),
        #     mode=ia.ALL
        # )),

          # # Change brightness of images (85-115% of original value).
          # iaa.Multiply((0.90, 1.15), per_channel=0.5),

          # # # Improve or worsen the contrast of images.
          # iaa.ContrastNormalization((0.75, 1.25), per_channel=0.5),

          # Convert each image to grayscale and then overlay the
          # result with the original with random alpha. I.e. remove
          # colors with varying strengths.
          # iaa.Grayscale(alpha=(0.0, 0.25)),
#
        # Execute 1 to 9 of the following (less important) augmenters per
        # image. Don't execute all of them, as that would often be way too
        # strong.
        #
        iaa.SomeOf((1, 9),
            [

                        # Blur each image with varying strength using
                        # gaussian blur (sigma between 0 and .5),
                        # average/uniform blur (kernel size 1x1)
                        # median blur (kernel size 1x1).
                        iaa.OneOf([
                            iaa.GaussianBlur((0,1)),
                            iaa.AverageBlur(k=(2,2)),
                            iaa.MedianBlur(k=(1,1)),
                        ]),

                        # Sharpen each image, overlay the result with the original
                        # image using an alpha between 0 (no sharpening) and 1
                        # (full sharpening effect).
                        iaa.Sharpen(alpha=(0, 0.25), lightness=(0.75, 1.5)),

                        # Add gaussian noise to some images.
                        # In 50% of these cases, the noise is randomly sampled per
                        # channel and pixel.
                        # In the other 50% of all cases it is sampled once per
                        # pixel (i.e. brightness change).
                        iaa.AdditiveGaussianNoise(
                            loc=0, scale=(0.0, 0.01*255), per_channel=0.5
                        ),

                        # Either drop randomly 1 to 10% of all pixels (i.e. set
                        # them to black) or drop them on an image with 2-5% percent
                        # of the original size, leading to large dropped
                        # rectangles.
                        # iaa.OneOf([
                        #     iaa.Dropout((0.01, 0.1), per_channel=0.5),
                        #     iaa.CoarseDropout(
                        #         (0.03, 0.15), size_percent=(0.02, 0.05),
                        #         per_channel=0.2
                        #     ),
                        # ]),
             
                        
                        # Invert each image's channel with 5% probability.
                        # This sets each pixel value v to 255-v.
                        iaa.Invert(0.05, per_channel=True), # invert color channels

                        # Add a value of -5 to 5 to each pixel.
                        iaa.Add((-5, 5), per_channel=0.5),

                        # # Change brightness of images (85-115% of original value).
                        iaa.Multiply((0.85, 1.15), per_channel=0.5),

                        # # Improve or worsen the contrast of images.
                        iaa.ContrastNormalization((0.75, 1.25), per_channel=0.5),

                        # Convert each image to grayscale and then overlay the
                        # result with the original with random alpha. I.e. remove
                        # colors with varying strengths.
                        iaa.Grayscale(alpha=(0.0, 0.25)),

                        # In some images distort local areas with varying strength.
                        sometimes(iaa.PiecewiseAffine(scale=(0.001, 0.01)))
                    ],
            # do all of the above augmentations in random order
            random_order=True
        )
    ],
    # do all of the above augmentations in random order
    random_order=True
)


def augment_train_gen(train_gen, visualize=False):
    '''
    Creates a generator using another generator with applied image augmentation.
    Args
        train_gen  : keras-retinanet generator object.
        visualize  : Boolean; False will convert bounding boxes to their anchor box targets for the model.
    '''
    imgs = []
    boxes = []
    targets = []
    size = train_gen.size()
    idx = 0
    while True:
        while len(imgs) < args.batch_size:
            image       = train_gen.load_image(idx % size)
            annotations = train_gen.load_annotations(idx % size)
            image,annotations = train_gen.random_transform_group_entry(image,annotations)
            imgs.append(image)            
            boxes.append(annotations['bboxes'])
            targets.append(annotations)
            idx += 1
        if visualize:
            imgs = seq.augment_images(imgs)
            imgs = np.array(imgs)
            boxes = np.array(boxes)
            yield imgs,boxes
        else:
            imgs = seq.augment_images(imgs)
            imgs,targets = train_gen.preprocess_group(imgs,targets)
            imgs = train_gen.compute_inputs(imgs)
            targets = train_gen.compute_targets(imgs,targets)
            imgs = np.array(imgs)
            yield imgs,targets
        imgs = []
        boxes = []
        targets = []
		

model, training_model, prediction_model = create_models(
            backbone_retinanet=b.retinanet,
            num_classes=train_gen.num_classes(),
            weights=None,
            multi_gpu=False,
            freeze_backbone=True,
            lr=1e-3,
            config=args.config
        )<jupyter_output>tracking <tf.Variable 'Variable:0' shape=(15, 4) dtype=float32, numpy=
array([[-22.627417, -11.313708,  22.627417,  11.313708],
       [-33.941124, -16.970562,  33.941124,  16.970562],
       [-45.254833, -22.627417,  45.254833,  22.627417],
       [-16.      , -16.      ,  16.      ,  16.      ],
       [-24.      , -24.      ,  24.      ,  24.      ],
       [-32.      , -32.      ,  32.      ,  32.      ],
       [-13.063946, -19.595919,  13.063946,  19.595919],
       [-19.595919, -29.393877,  19.595919,  29.393877],
       [-26.127892, -39.191837,  26.127892,  39.191837],
       [-11.313708, -22.627417,  11.313708,  22.627417],
       [-16.970562, -33.941124,  16.970562,  33.941124],
       [-22.627417, -45.254833,  22.627417,  45.254833],
       [-10.119288, -25.298222,  10.119288,  25.298222],
       [-15.178933, -37.94733 ,  15.178933,  37.94733 ],
       [-20.238577, -50.596443,  20.238577,  50.596443]], dtype=float32)> anchors
tracking <tf.Variable 'Variable:0' shape=(15, 4) [...]<jupyter_text>Training with different parameters and ratios- sizes io24, 2048 , nratios  = 0.5 1 1.5 2 2.5\nscales  =1 1.5 2\n')- IOU = 0.6 and NMS back at normal, score threshold = 0.15<jupyter_code>from keras_retinanet.callbacks import RedirectModel 
callbacks = create_callbacks(
    model,
    training_model,
    prediction_model,
    valid_gen,
    args,
)
training_model.load_weights('/content/drive/My Drive/RA2/Retinanet/Upload for Kaggle/Hand models New annotations/Models v5 continuedsss/resnet50_csv_25_classloss_0.0632.h5', skip_mismatch = True, by_name = True)
training_model.fit_generator(generator=augment_train_gen(train_gen),
        steps_per_epoch = args.steps,
        epochs=args.epochs,
        verbose=1,
        callbacks = callbacks,)<jupyter_output>Epoch 1/200
29/42 [===================>..........] - ETA: 1:43 - loss: 1.0934 - regression_loss: 0.9978 - classification_loss: 0.0957<jupyter_text>Training a complete hand model currently<jupyter_code>from keras_retinanet.callbacks import RedirectModel 
callbacks = create_callbacks(
    model,
    training_model,
    prediction_model,
    valid_gen,
    args,
)
training_model.load_weights('/content/drive/My Drive/RA2/Retinanet/Upload for Kaggle/Hand models New annotations/Models v5 continued/resnet50_csv_17_class_loss_0.0862.h5', skip_mismatch = True, by_name = True)<jupyter_output><empty_output><jupyter_text>- incrased size to 1024 and reduces io and nms by 0.05- config file with open('config.ini','w') as f:    f.write('[anchor_parameters]\nsizes   = 32 64 128 256 512 1024\nstrides = 8 16 32 64 128 256\nratios  = 1.2 1.5 2 2.5 3\nscales  =1 1.5 2\n')<jupyter_code>training_model.fit_generator(generator=augment_train_gen(train_gen),
        steps_per_epoch = args.steps,
        epochs=args.epochs,
        verbose=1,
        callbacks = callbacks,)<jupyter_output>Epoch 1/200
29/42 [===================>..........] - ETA: 1:44 - loss: 0.9826 - regression_loss: 0.9180 - classification_loss: 0.0647<jupyter_text>Training continued of below logs<jupyter_code>training_model.fit_generator(generator=augment_train_gen(train_gen),
        steps_per_epoch = args.steps,
        epochs=args.epochs,
        verbose=1,
        callbacks = callbacks,)<jupyter_output>Epoch 1/200
29/42 [===================>..........] - ETA: 1:46 - loss: 1.0149 - regression_loss: 0.9326 - classification_loss: 0.0823<jupyter_text>Made few changes- nratios  = 1.2 1.3 1.5 2 2.2 2.8 3\nscales  =1 1.5 2\n')- score threshold = 0.1, iou and NMS unchanged- Added affine transformation- batch size 2- added mAP as early stopping criteria- Backbone is resnet coco<jupyter_code>training_model.fit_generator(generator=augment_train_gen(train_gen),
        steps_per_epoch = args.steps,
        epochs=args.epochs,
        verbose=1,
        callbacks = callbacks,)<jupyter_output>Epoch 1/200
29/42 [===================>..........] - ETA: 1:47 - loss: 2.8215 - regression_loss: 2.1974 - classification_loss: 0.6241<jupyter_text>Made few changes- nratios  = 0.5 1 1.5 2 2.5 3\nscales  =1 1.2 1.6\n')- score threshold = 0.1, iou and NMS unchanged- Added affine transformation- batch size 2- added mAP as early stopping criteria- Backbone is class loss 0.1714<jupyter_code>training_model.fit_generator(generator=augment_train_gen(train_gen),
        steps_per_epoch = args.steps,
        epochs=args.epochs,
        verbose=1,
        callbacks = callbacks,)<jupyter_output>Epoch 1/200
119/170 [====================>.........] - ETA: 1:58 - loss: 2.7448 - regression_loss: 2.1816 - classification_loss: 0.5632<jupyter_text>Final run for hand model- All parameters set to default- IOU  0.5- NMS = 0.5- SCORE THRESHOLD = 0.05- Image max side = 1400- Image min side = 1000- Ratios = 0.8 1 1.3 1.5 2 2.2 2.8 3- scales  =1 1.2 1.6  Restarting training from 54th epoch model<jupyter_code>training_model.fit_generator(generator=augment_train_gen(train_gen),
        steps_per_epoch = args.steps,
        epochs=args.epochs,
        verbose=1,
        callbacks=callbacks,)<jupyter_output><empty_output><jupyter_text>Training stopped due to connectivity<jupyter_code>training_model.fit_generator(generator=augment_train_gen(train_gen),
        steps_per_epoch = args.steps,
        epochs=args.epochs,
        verbose=1,
        callbacks=callbacks,)<jupyter_output><empty_output><jupyter_text>Logs for training- Changing the Iou threshold = 0.5- NMS =0.5- score threshold = 0.1- Img Max side increased to 1400- Ratios changed to 1 1.5 2 2.2<jupyter_code>training_model.fit_generator(generator=augment_train_gen(train_gen),
        steps_per_epoch = args.steps,
        epochs=args.epochs,
        verbose=1,
        callbacks=callbacks,)<jupyter_output>Epoch 1/150
85/85 [==============================] - 332s 4s/step - loss: 1.9143 - regression_loss: 1.7073 - classification_loss: 0.2070<jupyter_text>New logs for training- Chagning back the ratios and scale to normal if class loss doesnt decrease (Changed)<jupyter_code>training_model.fit_generator(generator=augment_train_gen(train_gen),
        steps_per_epoch = args.steps,
        epochs=args.epochs,
        verbose=1,
        callbacks=callbacks,)<jupyter_output>Epoch 1/150
85/85 [==============================] - 418s 5s/step - loss: 2.7223 - regression_loss: 2.2535 - classification_loss: 0.4688<jupyter_text>New logs for some changes mentioned below- Trained with best model from Version 3- Changed the Iou threshold = 0.6, NMS thresh = 0.55, score thresh = 0.1- Ratios to 1 - 3 with a gap of 0.2- Freeze backbone=False- Chnages in some augmentations<jupyter_code>training_model.fit_generator(generator=augment_train_gen(train_gen),
        steps_per_epoch = args.steps,
        epochs=args.epochs,
        verbose=1,
        callbacks=callbacks,)<jupyter_output><empty_output><jupyter_text>Logs for some changes -> - Freeze Backbone =False,  (Loaded previous best model) - Ratio increased from previous to 1, 1.2, 1.5, 2, 2.5, 3 - > 1 1.2 1.5 1.8 2 2.5 2.8 3- Image _min side changed from 1000 -> 1000 and Max side remains the same 1200 - Adding these augmentations as necessary<jupyter_code>training_model.fit_generator(generator=augment_train_gen(train_gen),
        steps_per_epoch = args.steps,
        epochs=args.epochs,
        verbose=1,
        callbacks=callbacks,)<jupyter_output>Epoch 1/150
47/47 [==============================] - 370s 8s/step - loss: 3.8632 - regression_loss: 2.5618 - classification_loss: 1.3014

Epoch 00001: saving model to /content/drive/My Drive/RA2/Retinanet/Upload for Kaggle/Hand models New annotations/resnet50_csv_01.h5<jupyter_text>Logs for -> - Free backbone = False, - Ratio increased from previous to 1, 1.2, 1.5, 2, 2.5, 3 - > 1 1.2 1.5 1.8 2 2.5 2.8 3 - Image _min side changed from 1000 -> 800 and Max side remains the same 1200<jupyter_code>training_model.fit_generator(generator=augment_train_gen(train_gen),
        steps_per_epoch = args.steps,
        epochs=args.epochs,
        verbose=1,
        callbacks=callbacks,)<jupyter_output>Epoch 1/150
47/47 [==============================] - 324s 7s/step - loss: 3.2019 - regression_loss: 2.5069 - classification_loss: 0.6950

Epoch 00001: saving model to /content/drive/My Drive/RA2/Retinanet/Upload for Kaggle/Hand models New annotations/resnet50_csv_01.h5<jupyter_text>Logs for Some changes i.e IOU = 0.6, Freeze backbone = False, N_sizes for anchors = 64, 128, 256, 1024 and scales changed to 1, 1.2, 1.6, 2<jupyter_code>training_model.fit_generator(generator=augment_train_gen(train_gen),
        steps_per_epoch = args.steps,
        epochs=args.epochs,
        verbose=1,
        callbacks=callbacks,)<jupyter_output><empty_output><jupyter_text>Right hands joint detection<jupyter_code># tf.set_random_seed(31) # SEEDS MAKE RESULTS MORE REPRODUCABLE
np.random.seed(17)
classes = np.arange(0, 11, 1).tolist()

# def convert_annotation(image_id,filename):
#     in_file = open('training_data/labels/%s.xml'%(image_id))
#     out_file = open(filename, 'a')
#     tree=ET.parse(in_file)
#     root = tree.getroot()
    
#     if root.iter('object') is not None:
#         for obj in root.iter('object'):
#             cls = obj.find('name').text
#             if cls not in classes:
#                 continue
#             cls_id = classes.index(cls)
            
#             xmlbox = obj.find('bndbox')
#             x1 = math.ceil(float(xmlbox.find('xmin').text))
#             y1 = math.ceil(float(xmlbox.find('ymin').text))
#             x2 = math.ceil(float(xmlbox.find('xmax').text))
#             y2 = math.ceil(float(xmlbox.find('ymax').text))
#             if x1 == x2 or y1 == y2:
#                 continue
                
#             out_file.write(f'training_data/images/{image_id}.jpg,{x1},{y1},{x2},{y2},{cls}\n')
#     else:
#         out_file.write(f'training_data/images/{image_id}.jpg,,,,,\n')


# _,_,image_ids = next(walk('training_data/images'))
# image_ids = [i[:-4] for i in image_ids]
# open('annotations.csv','w')
# open('val_annotations.csv','w')

# train_ids,val_ids = train_test_split(image_ids,random_state=31,test_size=0)

# for image_id in train_ids:
#     convert_annotation(image_id,'annotations.csv')
        
# for image_id in val_ids:
#     convert_annotation(image_id,'val_annotations.csv')
    
# print(len(train_ids),len(val_ids))

import pandas as pd
tr_annots = pd.read_csv('/content/drive/My Drive/RA2/Retinanet/Upload for Kaggle/training_data/Right hand train.csv', header = None)
tr_annots['ids'] = tr_annots[0].apply(lambda x: x.split("/")[-1].split(".")[0])
train_ids = tr_annots['ids'].unique().tolist()

import pandas as pd
tr_annots = pd.read_csv('/content/drive/My Drive/RA2/Retinanet/Upload for Kaggle/training_data/Right hand validation.csv', header = None)
tr_annots['ids'] = tr_annots[0].apply(lambda x: x.split("/")[-1].split(".")[0])
val_ids = tr_annots['ids'].unique().tolist()

print(len(train_ids), len(val_ids))

with open('config.ini','w') as f:
    f.write('[anchor_parameters]\nsizes   = 32 64 128 256 512\nstrides = 8 16 32 64 128\nratios  = 1 1.2 1.5 2 \nscales  = 2 3 4 \n')
	
b = backbone('resnet50')

class args:
    batch_size = 2
    config = read_config_file('config.ini')
    random_transform = True # Image augmentation
    annotations = '/content/drive/My Drive/RA2/Retinanet/Upload for Kaggle/training_data/Right hand train.csv'
    val_annotations = '/content/drive/My Drive/RA2/Retinanet/Upload for Kaggle/training_data/Right hand validation.csv'
    classes = '/content/drive/My Drive/RA2/Retinanet/Upload for Kaggle/hands_class_names.csv'
    image_min_side = 900
    image_max_side = 1200
    no_resize = False
    dataset_type = 'csv'
    tensorboard_dir = ''
    evaluation = False
    snapshots = True
    snapshot_path = "/content/drive/My Drive/RA2/Retinanet/Upload for Kaggle/saved_right_hand_models/"
    backbone = 'resnet50'
    epochs = 100
    steps = len(train_ids)//(batch_size)
    weighted_average = True
	
train_gen, valid_gen = create_generators(args, b.preprocess_image)


sometimes = lambda aug: iaa.Sometimes(0.5, aug)
# Define our sequence of augmentation steps that will be applied to every image.
seq = iaa.Sequential(
    [
        #
        # Execute 1 to 9 of the following (less important) augmenters per
        # image. Don't execute all of them, as that would often be way too
        # strong.
        #
        iaa.SomeOf((1, 9),
            [

                        # Blur each image with varying strength using
                        # gaussian blur (sigma between 0 and .5),
                        # average/uniform blur (kernel size 1x1)
                        # median blur (kernel size 1x1).
                        iaa.OneOf([
                            iaa.GaussianBlur((0,0.5)),
                            iaa.AverageBlur(k=(1)),
                            iaa.MedianBlur(k=(1)),
                        ]),

                        # Sharpen each image, overlay the result with the original
                        # image using an alpha between 0 (no sharpening) and 1
                        # (full sharpening effect).
                        iaa.Sharpen(alpha=(0, 0.25), lightness=(0.75, 1.5)),

                        # Add gaussian noise to some images.
                        # In 50% of these cases, the noise is randomly sampled per
                        # channel and pixel.
                        # In the other 50% of all cases it is sampled once per
                        # pixel (i.e. brightness change).
                        iaa.AdditiveGaussianNoise(
                            loc=0, scale=(0.0, 0.01*255), per_channel=0.5
                        ),

                        # Either drop randomly 1 to 10% of all pixels (i.e. set
                        # them to black) or drop them on an image with 2-5% percent
                        # of the original size, leading to large dropped
                        # rectangles.
                        iaa.OneOf([
                            iaa.Dropout((0.01, 0.1), per_channel=0.5),
                            iaa.CoarseDropout(
                                (0.03, 0.15), size_percent=(0.02, 0.05),
                                per_channel=0.2
                            ),
                        ]),

                        # Add a value of -5 to 5 to each pixel.
                        iaa.Add((-5, 5), per_channel=0.5),

                        # Change brightness of images (85-115% of original value).
                        iaa.Multiply((0.85, 1.15), per_channel=0.5),

                        # Improve or worsen the contrast of images.
                        iaa.ContrastNormalization((0.75, 1.25), per_channel=0.5),

                        # Convert each image to grayscale and then overlay the
                        # result with the original with random alpha. I.e. remove
                        # colors with varying strengths.
                        iaa.Grayscale(alpha=(0.0, 0.25)),

                        # In some images distort local areas with varying strength.
                        sometimes(iaa.PiecewiseAffine(scale=(0.001, 0.01)))
                    ],
            # do all of the above augmentations in random order
            random_order=True
        )
    ],
    # do all of the above augmentations in random order
    random_order=True
)


def augment_train_gen(train_gen, visualize=False):
    '''
    Creates a generator using another generator with applied image augmentation.
    Args
        train_gen  : keras-retinanet generator object.
        visualize  : Boolean; False will convert bounding boxes to their anchor box targets for the model.
    '''
    imgs = []
    boxes = []
    targets = []
    size = train_gen.size()
    idx = 0
    while True:
        while len(imgs) < args.batch_size:
            image       = train_gen.load_image(idx % size)
            annotations = train_gen.load_annotations(idx % size)
            image,annotations = train_gen.random_transform_group_entry(image,annotations)
            imgs.append(image)            
            boxes.append(annotations['bboxes'])
            targets.append(annotations)
            idx += 1
        if visualize:
            imgs = seq.augment_images(imgs)
            imgs = np.array(imgs)
            boxes = np.array(boxes)
            yield imgs,boxes
        else:
            imgs = seq.augment_images(imgs)
            imgs,targets = train_gen.preprocess_group(imgs,targets)
            imgs = train_gen.compute_inputs(imgs)
            targets = train_gen.compute_targets(imgs,targets)
            imgs = np.array(imgs)
            yield imgs,targets
        imgs = []
        boxes = []
        targets = []
		
		
# import matplotlib.pyplot as plt

# skip_batches = 5
# i = 0
# for imgs,boxes in augment_train_gen(train_gen,visualize=True):
#     if i > skip_batches:
#         fig=plt.figure(figsize=(24,96))
#         columns = 2
#         rows = 8
#         for i in range(1, columns*rows + 1):
#             draw_boxes(imgs[i], boxes[i], (0, 255, 0), thickness=1)
#             fig.add_subplot(rows, columns, i)
#             plt.imshow(cv2.cvtColor(imgs[i],cv2.COLOR_BGR2RGB))
#         plt.show()
#         break
#     else:
#         i += 1

model, training_model, prediction_model = create_models(
            backbone_retinanet=b.retinanet,
            num_classes=train_gen.num_classes(),
            weights=None,
            multi_gpu=False,
            freeze_backbone=True,
            lr=1e-3,
            config=args.config
        )

		
callbacks = create_callbacks(
    model,
    training_model,
    prediction_model,
    valid_gen,
    args,
)
training_model.load_weights('/content/drive/My Drive/RA2/Retinanet/Upload for Kaggle/resnet50_coco_best_v2.0.1.h5', skip_mismatch = True, by_name = True)

##m Final model for Right hand joints detections
training_model.fit_generator(generator=augment_train_gen(train_gen),
        steps_per_epoch = args.steps,
        epochs=args.epochs,
        verbose=1,
        callbacks=callbacks, )<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup