code:
import os
import torch
import torch.nn as nn
import numpy as np
import pickle
try:
    import cv2
except:
    from cv2 import cv2

from Config.Config import ACTNET200V13_PKL,FRAME_COUNT_PKL
from FeatureExtruct.transforms import *
from PIL import Image

class RGB_Single_Frame_Dataset(object):
    '''
    取activitynet帧数据
    '''
    def __init__(self,subset='training',modality='train'):
        # training validation testing
        self.subset = subset
        self.modality = modality
        self.init_values()

    def load_picture(self,vid,id,modality='RGB'):
        if modality == 'RGB':
            frame_path = '/mnt/md1/Dataset/ActivityNet/Frames/{}/img_{:05d}.jpg'.format(vid,id)
            img = Image.open(frame_path)
            return img

    def get_return_data(self,x):

        trans = torchvision.transforms.Compose([
            GroupScale(256),
            GroupRandomSizedCrop(224),
            Stack(roll=False),
            ToTorchFormatTensor(),
            GroupNormalize(
                mean =  [124. / 255, 117. / 255, 104. / 255],
                std = [1 / (.0167 * 255)] * 3)
        ])

        return trans(x)

    def get_return_data_for_enum(self,x):

        trans = torchvision.transforms.Compose([
            GroupScale(224),
            Stack(roll=False),
            ToTorchFormatTensor(),
            GroupNormalize(
                mean =  [124. / 255, 117. / 255, 104. / 255],
                std = [1 / (.0167 * 255)] * 3)
        ])

        return trans(x)

    def init_values(self):

        with open(ACTNET200V13_PKL,'rb') as f:
            groundtruth = pickle.load(f)['database']

        # filter by subset
        remainkeyset = []
        for key in groundtruth.keys():
            if groundtruth[key]['subset'] == self.subset:
                remainkeyset.append(key)
        origin_groundtruth = groundtruth.copy()

        self.groundtruth = dict()
        for key in remainkeyset:
            self.groundtruth[key] = origin_groundtruth[key]

        self.vids = remainkeyset

        with open(FRAME_COUNT_PKL, 'rb') as f:
            self.frames_count = pickle.load(f)

        # mod 为 0,2,4的当做训练集 1,3用来抽取特征
        self.MOD = 5
        self.TRAIN_MD = [0,2,4]
        self.EVAL_MD  = [1,3]

        if self.modality == 'test':
            self.EVAL_MD  = [0,2,4]
            self.TRAIN_MD = [1,3]

    def train_sample_from_video(self, batch=8, max_pos=6, max_neg=2, max_per_video = 2):

        # x = np.zeros((batch, 3, 224, 224))
        x = []
        y = np.ones(batch)

        # 采128张图片, 96张带类别 32章为背景
        pos_count = 0
        neg_count = 0
        count = 0

        for p, vid in enumerate(self.vids):

            item = self.groundtruth[vid]
            annos = item['annotations']
            frame_count = self.frames_count[vid]
            dur = item['duration']

            gts = []
            gt_labels = []

            for anno in annos:
                st,ed = anno['segment']
                st = max(np.ceil(st/dur*frame_count),1)
                ed = min(np.floor(ed/dur*frame_count),frame_count)
                if st > ed : continue
                gts.append([st,ed])
                gt_labels.append(anno['class'])

            def checkIn(x):
                for i in range(len(gts)):
                    if x >= gts[i][0] and x<=gts[i][1]:
                        return gt_labels[i]
                return 0

            randomidx = np.arange(1,frame_count,1)
            np.random.shuffle(randomidx)

            nd = 0
            m_count = max_per_video
            while m_count > 0 :

                if nd >= len(randomidx): break
                Id = randomidx[nd]
                nd+=1
                if Id%self.MOD not in self.TRAIN_MD:
                    continue

                lb = checkIn(Id)

                if lb!=0 and pos_count < max_pos : # postive example
                    m_count -= 1
                    img = self.load_picture(vid,Id)
                    # x[count,...] = img
                    x.append(img)
                    y[count] = lb
                    pos_count += 1
                    count += 1
                elif lb==0 and neg_count < max_neg : # neg postive
                    m_count -= 1
                    img = self.load_picture(vid,Id)
                    # x[count,...] = img
                    x.append(img)
                    y[count] = lb
                    neg_count += 1
                    count += 1

            if count == batch:
                # 位移动一下
                self.vids = self.vids[p:] + self.vids[:p]
                break

        x = self.get_return_data(x)
        # x is (8*3),224,224
        x = x.view(-1,3,224,224)
        return x,y

    def nextbatch(self):
        return self.train_sample_from_video(batch=64,max_pos=48,max_neg=16,max_per_video=8)

    def nextbatch_flow(self):
        return self.train_sample_from_video(batch=64,max_pos=32,max_neg=32,max_per_video=8)

    def enum_vid_feature(self,vid,batchsize=64):

        if os.path.exists('/mnt/md1/Dataset/ActivityNet/Frames/{}'.format(vid)) == False:
            raise FileNotFoundError

        item = self.groundtruth[vid]
        annos = item['annotations']
        frame_count = self.frames_count[vid]
        dur = item['duration']

        gts = []
        gt_labels = []

        for anno in annos:
            st,ed = anno['segment']
            st = max(np.ceil(st/dur*frame_count),1)
            ed = min(np.floor(ed/dur*frame_count),frame_count)
            if st > ed: continue
            gts.append([st,ed])
            gt_labels.append(anno['class'])

        def checkIn(x):
            for i in range(len(gts)):
                if x >= gts[i][0] and x<=gts[i][1]: return gt_labels[i]
            return 0

        imgs = []
        labels = []

        for i in range(1,frame_count):
            if i%self.MOD not in [1,3]: continue
            imgs.append(self.load_picture(vid,i))
            labels.append(checkIn(i))

            if len(labels) == batchsize:
                imgs = self.get_return_data_for_enum(imgs)
                imgs = imgs.view(-1,3,224,224)
                yield vid,imgs,labels
                imgs = []
                labels = []

        if len(labels) != 0:
            imgs = self.get_return_data_for_enum(imgs)
            imgs = imgs.view(-1,3,224,224)
            yield vid,imgs,labels

    def _enum_all_pictures(self,batchsize=8):

        for p, vid in enumerate(self.vids):

            if os.path.exists('/mnt/md1/Dataset/ActivityNet/Frames/{}'.format(vid)) == False:
                continue

            # DEBUG VAlUE
            # if vid != 'UoGoOznvKew': continue

            item = self.groundtruth[vid]
            annos = item['annotations']
            frame_count = self.frames_count[vid]
            dur = item['duration']

            gts = []
            gt_labels = []

            for anno in annos:
                st,ed = anno['segment']
                st = max(np.ceil(st/dur*frame_count),1)
                ed = min(np.floor(ed/dur*frame_count),frame_count)
                if st > ed: continue
                gts.append([st,ed])
                gt_labels.append(anno['class'])

            def checkIn(x):
                for i in range(len(gts)):
                    if x >= gts[i][0] and x<=gts[i][1]:
                        return gt_labels[i]
                return 0

            imgs = []
            labels = []

            for i in range(1,frame_count):
                if i%self.MOD not in [1,3]: continue
                imgs.append(self.load_picture(vid,i))
                labels.append(checkIn(i))

                if len(labels) == batchsize:
                    imgs = self.get_return_data_for_enum(imgs)
                    imgs = imgs.view(-1,3,224,224)
                    yield vid,imgs,labels
                    imgs = []
                    labels = []

            if len(labels) != 0:
                imgs = self.get_return_data_for_enum(imgs)
                imgs = imgs.view(-1,3,224,224)
                yield vid,imgs,labels

    def enum_all_picture(self):
        return self._enum_all_pictures(batchsize=64)

if __name__=='__main__':
    print('here')
    dataset = RGB_Single_Frame_Dataset('training')
    print('done')
    # X,Y = dataset.nextbatch()
    # import matplotlib.pyplot as plt
    # X = X.numpy()
    # plt.clf()
    # plt.imshow(X[7,0,:,:])
    # plt.show()

    gen = dataset.enum_all_picture()

    vid,imgs,labels = next(gen)
    import IPython;IPython.embed()


code:
import os
import numpy as np
import logging

import torch
import torch.nn as nn
from torch.autograd import Variable

from Tools.opts_train import pair2_parser
from Tools.TFLogger import TFLogger
from Tools.utils import AverageMeter

from Config.Config import TFLOG_ROOT_PATH

from FeatureExtruct.DPN107_RGB.RGB_SinglePicture_Dataset import RGB_Single_Frame_Dataset

args = pair2_parser.parse_args()

##########################################
# set paramtere

SAVE_DIR        = args.savedir
EX_NAME         = args.exname
EPOCHS          = args.epochs or 80
TRAIN_ITER_NUM  = 1000
VAL_ITER_NUM    = 3
LR              = args.lr or 0.1
LR_DELAY_RATE   = args.lr_delay_rate or 0.1
MOMENTUM        = args.momentum or 0.9
LR_STEPS        = args.lr_steps
PRINT_FREQ      = args.print_freq
EVAL_FREQ       = 20
RESUME_CKPT     = args.resume_ckpt
BASE_STEP       = args.basestep

###########################################
# set log
if os.path.exists(SAVE_DIR) == False:
    os.mkdir(SAVE_DIR)

logger = logging.getLogger('dpn107_rgb_201_fineturn_Test1')
logger.setLevel(logging.INFO)
fh = logging.FileHandler(os.path.join(SAVE_DIR,'run.log'))
formatter = logging.Formatter('%(asctime)s:%(message)s')
fh.setFormatter(formatter)
logger.addHandler(fh)

tflog = TFLogger(os.path.join(TFLOG_ROOT_PATH,EX_NAME))

##########################################
# prepare model

logging.info('ARG SETTING: {}'.format(args))
logging.info('PREPARE MODEL...')

# load dataset
train_dataset = RGB_Single_Frame_Dataset('training','train')
val_dataset   = RGB_Single_Frame_Dataset('validation','test')

# load model dpn107
model = torch.load('/mnt/md1/Experiments/DPN_Extruct_200_Test1/raw_dpn107_model.pkl')
model = nn.DataParallel(model).cuda()
crossloss = nn.CrossEntropyLoss().cuda()

optim = torch.optim.SGD(model.parameters(),lr=LR,momentum=0.9)

##########################################

def addjust_optime_lr(model,epochid):

    global LR,LR_DELAY_RATE,LR_STEPS

    if epochid in LR_STEPS:
        LR = LR*LR_DELAY_RATE
        logging.info('Change Learing Rate to: {}'.format(LR))
        optim = torch.optim.SGD(model.parameters(),lr=LR,momentum=0.9)
        return optim
    return None

def save_model(model,epochid):
    FILE = os.path.join(SAVE_DIR,'dpn107_RGB_fineturn_201_model_{:03d}.ckpt'.format(epochid))
    torch.save(model.state_dict(),FILE)

##########################################
# train
if RESUME_CKPT is not None:
    model.load_state_dict(torch.load(RESUME_CKPT))

logging.info('START TRAINING...')
train_step = BASE_STEP
val_step   = BASE_STEP

for ep in range(EPOCHS):

    train_loss_meta = AverageMeter()
    val_loss_meta = AverageMeter()

    logger.info('EPOCH: [{}/{}]'.format(ep,EPOCHS))

    model.train()
    for idx in range(TRAIN_ITER_NUM):

        model.zero_grad()
        inputx, labels = train_dataset.nextbatch()

        x = Variable(inputx).cuda()
        labels = Variable(torch.from_numpy(labels)).cuda().long()

        predict = model(x)

        loss = crossloss(predict,labels)
        loss.backward()
        train_loss_meta.update(loss.data[0])
        optim.step()

        train_step += 1
        tflog.scalar_summary('train_loss',loss.data[0],train_step)

        if idx%PRINT_FREQ == 0:
            logger.info('Epoch {} {}/{} current_train_loss: {} loss_avg: {avg_train_loss.avg}'.format(ep,idx, TRAIN_ITER_NUM, loss.data[0], avg_train_loss=train_loss_meta))

        if idx%EVAL_FREQ == 0 :
            # eval
            model.eval()
            for idx in range(VAL_ITER_NUM):

                inputx, labels = val_dataset.nextbatch()
                x = Variable(inputx).cuda()
                labels = Variable(torch.from_numpy(labels),volatile=True).cuda().long()

                predict = model(x)
                loss = crossloss(predict,labels)
                val_loss_meta.update(loss.data[0])

                val_step += 1
                tflog.scalar_summary('val_loss',loss.data[0],val_step)

                if idx%PRINT_FREQ == 0:
                    logger.info('Epoch {} {}/{} current_val_loss: {} loss_avg: {avg_val_loss.avg}'.format(ep, idx, VAL_ITER_NUM, loss.data[0], avg_val_loss=val_loss_meta))
            model.train()

    logger.info('Epoch {} Train avg loss: {}'.format(ep, train_loss_meta.avg))
    noptim = addjust_optime_lr(model,ep)
    if noptim is not None: optim = noptim
    logger.info('Epoch {} Val avg loss: {}'.format(ep, val_loss_meta.avg))
    # save model
    save_model(model,ep)

license: unknown


code:
import os
import numpy as np
import logging

import torch
import torch.nn as nn
from torch.autograd import Variable

from Tools.opts_train import pair2_parser
from Tools.TFLogger import TFLogger
from Tools.utils import AverageMeter

from Config.Config import TFLOG_ROOT_PATH

from FeatureExtruct.DPN107_RGB.RGB_SinglePicture_Dataset import RGB_Single_Frame_Dataset

args = pair2_parser.parse_args()

##########################################
# set paramtere

SAVE_DIR        = args.savedir
EX_NAME         = args.exname
EPOCHS          = args.epochs or 80
TRAIN_ITER_NUM  = 1000
VAL_ITER_NUM    = 3
LR              = args.lr or 0.1
LR_DELAY_RATE   = args.lr_delay_rate or 0.1
MOMENTUM        = args.momentum or 0.9
LR_STEPS        = args.lr_steps
PRINT_FREQ      = args.print_freq
EVAL_FREQ       = 20
RESUME_CKPT     = args.resume_ckpt
BASE_STEP       = args.basestep

###########################################
# set log
if os.path.exists(SAVE_DIR) == False:
    os.mkdir(SAVE_DIR)

logger = logging.getLogger('dpn107_rank_2_fineturn_Test1')
logger.setLevel(logging.INFO)
fh = logging.FileHandler(os.path.join(SAVE_DIR,'run_rank.log'))
formatter = logging.Formatter('%(asctime)s:%(message)s')
fh.setFormatter(formatter)
logger.addHandler(fh)

tflog = TFLogger(os.path.join(TFLOG_ROOT_PATH,EX_NAME))

##########################################
# prepare model

logging.info('ARG SETTING: {}'.format(args))
logging.info('PREPARE MODEL...')

# load dataset
train_dataset = RGB_Single_Frame_Dataset('training','train')
val_dataset   = RGB_Single_Frame_Dataset('validation','test')

# load model dpn107
model = torch.load('/mnt/md1/Experiments/DPN_Extruct_200_Test1/raw_dpn107_rank_model.pkl')
model = nn.DataParallel(model).cuda()
crossloss = nn.CrossEntropyLoss().cuda()

optim = torch.optim.SGD(model.parameters(),lr=LR,momentum=0.9)

##########################################

def addjust_optime_lr(model,epochid):

    global LR,LR_DELAY_RATE,LR_STEPS

    if epochid in LR_STEPS:
        LR = LR*LR_DELAY_RATE
        logging.info('Change Learing Rate to: {}'.format(LR))
        optim = torch.optim.SGD(model.parameters(),lr=LR,momentum=0.9)
        return optim
    return None

def save_model(model,epochid):
    FILE = os.path.join(SAVE_DIR,'dpn107_Rank_fineturn_2_model_{:03d}.ckpt'.format(epochid))
    torch.save(model.state_dict(),FILE)

##########################################
# train
if RESUME_CKPT is not None:
    model.load_state_dict(torch.load(RESUME_CKPT))

logging.info('START TRAINING...')
train_step = BASE_STEP
val_step   = BASE_STEP

for ep in range(EPOCHS):

    train_loss_meta = AverageMeter()
    val_loss_meta = AverageMeter()

    logger.info('EPOCH: [{}/{}]'.format(ep,EPOCHS))

    model.train()
    for idx in range(TRAIN_ITER_NUM):

        model.zero_grad()
        inputx, labels = train_dataset.nextbatch_flow()
        labels[labels>0] = 1
        x = Variable(inputx).cuda()
        labels = Variable(torch.from_numpy(labels)).cuda().long()

        predict = model(x)

        loss = crossloss(predict,labels)
        loss.backward()
        train_loss_meta.update(loss.data[0])
        optim.step()

        train_step += 1
        tflog.scalar_summary('train_loss',loss.data[0],train_step)

        if idx%PRINT_FREQ == 0:
            logger.info('Epoch {} {}/{} current_train_loss: {} loss_avg: {avg_train_loss.avg}'.format(ep,idx, TRAIN_ITER_NUM, loss.data[0], avg_train_loss=train_loss_meta))

        if idx%EVAL_FREQ == 0 :
            # eval
            model.eval()
            for idx in range(VAL_ITER_NUM):

                inputx, labels = val_dataset.nextbatch_flow()
                labels[labels>0] = 1
                x = Variable(inputx,volatile=True).cuda()
                labels = Variable(torch.from_numpy(labels),volatile=True).cuda().long()

                predict = model(x)
                loss = crossloss(predict,labels)
                val_loss_meta.update(loss.data[0])

                val_step += 1
                tflog.scalar_summary('val_loss',loss.data[0],val_step)

                if idx%PRINT_FREQ == 0:
                    logger.info('Epoch {} {}/{} current_val_loss: {} loss_avg: {avg_val_loss.avg}'.format(ep, idx, VAL_ITER_NUM, loss.data[0], avg_val_loss=val_loss_meta))

    logger.info('Epoch {} Train avg loss: {}'.format(ep, train_loss_meta.avg))
    noptim = addjust_optime_lr(model,ep)
    if noptim is not None: optim = noptim
    logger.info('Epoch {} Val avg loss: {}'.format(ep, val_loss_meta.avg))
    # save model
    save_model(model,ep)


code:
import torch
import torch.nn as nn
import pytorch_model_zoo
from torch.autograd import Variable
import numpy as np

# 准备模型DPN107 for Class

USE = 'Rank'

CKPT = '/mnt/md1/Downloads/dpn107_80e_72.40p.pth.tar'

ckpt = torch.load(CKPT)
base_model = getattr(pytorch_model_zoo, 'dpn107')()
base_model.last_layer_name = 'classifier' #attention:last layer is a conv

if USE == 'Class':
    setattr(base_model, base_model.last_layer_name, nn.Conv2d(2688,201,kernel_size=(1,1),stride=(1,1)))
elif USE == 'Rank':
    setattr(base_model, base_model.last_layer_name, nn.Conv2d(2688,2,kernel_size=(1,1),stride=(1,1)))

ckpt_keys = list(ckpt['state_dict'].keys())
ckpt_keys = list(map(lambda x: '.'.join(x.split('.')[1:]),list(filter(lambda x: x.split('.')[0]=='base_model',
                                                                      list(map(lambda x: '.'.join(x.split('.')[1:]),ckpt_keys))))))

cnt = 0
init = 0
DDD = dict()
for key in base_model.state_dict():
    cnt += 1
    if key in ckpt_keys:
        init+=1
        DDD[key] = ckpt['state_dict']['module.base_model.'+key]
    else:
        print(key)
print(init,'/',cnt)

base_model.state_dict().update(DDD)

input_size = 224
input_mean =  [124. / 255, 117. / 255, 104. / 255]
input_std = [1 / (.0167 * 255)] * 3

I = Variable(torch.from_numpy(np.random.rand(10,3,224,224))).float()
O = base_model(I)

if USE=='Class':
    torch.save(base_model.state_dict(),'/mnt/md1/Experiments/DPN_Extruct_200_Test1/dpn107_state_dict.pkl')
    torch.save(base_model,'/mnt/md1/Experiments/DPN_Extruct_200_Test1/raw_dpn107_model.pkl')
elif USE=='Rank':
    torch.save(base_model.state_dict(),'/mnt/md1/Experiments/DPN_Extruct_200_Test1/dpn107_rank_state_dict.pkl')
    torch.save(base_model,'/mnt/md1/Experiments/DPN_Extruct_200_Test1/raw_dpn107_rank_model.pkl')

base_model = torch.load('/mnt/md1/Experiments/DPN_Extruct_200_Test1/raw_dpn107_model.pkl')


code:
import os
import pickle
try:
    import cv2
except:
    from cv2 import cv2
from Config.Config import ACTNET200V13_PKL,FRAME_COUNT_PKL
from FeatureExtruct.transforms import *
from PIL import Image

class RGB_Single_Frame_Dataset(object):
    '''
    取activitynet帧数据
    '''
    def __init__(self,subset='training',modality='train'):
        # training validation testing
        self.subset = subset
        self.modality = modality
        self.init_values()

    def load_picture(self,vid,id,modality='RGB'):
        if modality == 'RGB':
            frame_path = '/mnt/md1/Dataset/ActivityNet/Frames/{}/img_{:05d}.jpg'.format(vid,id)
            img = Image.open(frame_path)
            return img

    def get_return_data(self,x):

        trans = torchvision.transforms.Compose([
            GroupScale(256),
            GroupRandomSizedCrop(224),
            Stack(roll=False),
            ToTorchFormatTensor(),
            GroupNormalize(
                mean =  [0.485, 0.456, 0.406],
                std  =  [0.229, 0.224, 0.225] )
        ])

        return trans(x)

    def get_return_data_for_enum(self,x):

        trans = torchvision.transforms.Compose([
            GroupScale(224),
            Stack(roll=False),
            ToTorchFormatTensor(),
            GroupNormalize(
                mean =  [0.485, 0.456, 0.406],
                std  =  [0.229, 0.224, 0.225] )
        ])

        return trans(x)

    def init_values(self):

        with open(ACTNET200V13_PKL,'rb') as f:
            groundtruth = pickle.load(f)['database']

        # filter by subset
        remainkeyset = []
        for key in groundtruth.keys():
            if groundtruth[key]['subset'] == self.subset:
                remainkeyset.append(key)
        origin_groundtruth = groundtruth.copy()

        self.groundtruth = dict()
        for key in remainkeyset:
            self.groundtruth[key] = origin_groundtruth[key]

        self.vids = remainkeyset

        with open(FRAME_COUNT_PKL, 'rb') as f:
            self.frames_count = pickle.load(f)

        # mod 为 0,2,4的当做训练集 1,3用来抽取特征
        self.MOD = 5
        self.TRAIN_MD = [0,2,4]
        self.EVAL_MD  = [1,3]

        if self.modality == 'test':
            self.EVAL_MD  = [0,2,4]
            self.TRAIN_MD = [1,3]

    def train_sample_from_video(self, batch=8, max_pos=6, max_neg=2, max_per_video = 2):

        # x = np.zeros((batch, 3, 224, 224))
        x = []
        y = np.ones(batch)

        # 采128张图片, 96张带类别 32章为背景
        pos_count = 0
        neg_count = 0
        count = 0

        for p, vid in enumerate(self.vids):

            item = self.groundtruth[vid]
            annos = item['annotations']
            frame_count = self.frames_count[vid]
            dur = item['duration']

            gts = []
            gt_labels = []

            for anno in annos:
                st,ed = anno['segment']
                st = max(np.ceil(st/dur*frame_count),1)
                ed = min(np.floor(ed/dur*frame_count),frame_count)
                if st > ed : continue
                gts.append([st,ed])
                gt_labels.append(anno['class'])

            def checkIn(x):
                for i in range(len(gts)):
                    if x >= gts[i][0] and x<=gts[i][1]:
                        return gt_labels[i]
                return 0

            randomidx = np.arange(1,frame_count,1)
            np.random.shuffle(randomidx)

            nd = 0
            m_count = max_per_video
            while m_count > 0 :

                if nd >= len(randomidx): break
                Id = randomidx[nd]
                nd+=1
                if Id%self.MOD not in self.TRAIN_MD:
                    continue

                lb = checkIn(Id)

                if lb!=0 and pos_count < max_pos : # postive example
                    m_count -= 1
                    img = self.load_picture(vid,Id)
                    x.append(img)
                    y[count] = lb
                    pos_count += 1
                    count += 1
                elif lb==0 and neg_count < max_neg : # neg postive
                    m_count -= 1
                    img = self.load_picture(vid,Id)
                    x.append(img)
                    y[count] = lb
                    neg_count += 1
                    count += 1

            if count == batch:
                # 位移动一下
                self.vids = self.vids[p:] + self.vids[:p]
                break

        x = self.get_return_data(x)
        # x is (8*3),224,224
        x = x.view(-1,3,224,224)
        return x,y

    def nextbatch(self):
        return self.train_sample_from_video(batch=64,max_pos=48,max_neg=16,max_per_video=8)

    def nextbatch_flow(self):
        return self.train_sample_from_video(batch=64,max_pos=32,max_neg=32,max_per_video=8)

    def enum_vid_feature(self,vid,batchsize=64):

        if os.path.exists('/mnt/md1/Dataset/ActivityNet/Frames/{}'.format(vid)) == False:
            raise FileNotFoundError

        item = self.groundtruth[vid]
        annos = item['annotations']
        frame_count = self.frames_count[vid]
        dur = item['duration']

        gts = []
        gt_labels = []

        for anno in annos:
            st,ed = anno['segment']
            st = max(np.ceil(st/dur*frame_count),1)
            ed = min(np.floor(ed/dur*frame_count),frame_count)
            if st > ed: continue
            gts.append([st,ed])
            gt_labels.append(anno['class'])

        def checkIn(x):
            for i in range(len(gts)):
                if x >= gts[i][0] and x<=gts[i][1]: return gt_labels[i]
            return 0

        imgs = []
        labels = []

        for i in range(1,frame_count):
            if i%self.MOD not in [1,3]: continue
            imgs.append(self.load_picture(vid,i))
            labels.append(checkIn(i))

            if len(labels) == batchsize:
                imgs = self.get_return_data_for_enum(imgs)
                imgs = imgs.view(-1,3,224,224)
                yield vid,imgs,labels
                imgs = []
                labels = []

        if len(labels) != 0:
            imgs = self.get_return_data_for_enum(imgs)
            imgs = imgs.view(-1,3,224,224)
            yield vid,imgs,labels

if __name__=='__main__':
    print('here')
    dataset = RGB_Single_Frame_Dataset('training')
    print('done')
    # X,Y = dataset.nextbatch()
    # import matplotlib.pyplot as plt
    # X = X.numpy()
    # plt.clf()
    # plt.imshow(X[7,0,:,:])
    # plt.show()
    import IPython;IPython.embed()


code:
import os
import numpy as np
import logging

import torch
import torch.nn as nn
from torch.autograd import Variable

from Tools.opts_train import pair2_parser
from Tools.TFLogger import TFLogger
from Tools.utils import AverageMeter,accuracy

from Config.Config import TFLOG_ROOT_PATH
from FeatureExtruct.Resnet152_RGB.RGB_SinglePicture_Dataset import RGB_Single_Frame_Dataset

args = pair2_parser.parse_args()

##########################################
# set paramtere

SAVE_DIR        = args.savedir
EX_NAME         = args.exname
EPOCHS          = args.epochs or 80
TRAIN_ITER_NUM  = 1000
VAL_ITER_NUM    = 3
LR              = args.lr or 0.1
LR_DELAY_RATE   = args.lr_delay_rate or 0.1
MOMENTUM        = args.momentum or 0.9
LR_STEPS        = args.lr_steps
PRINT_FREQ      = args.print_freq
EVAL_FREQ       = 20
RESUME_CKPT     = args.resume_ckpt
BASE_STEP       = args.basestep

###########################################
# set log
if os.path.exists(SAVE_DIR) == False:
    os.mkdir(SAVE_DIR)

logger = logging.getLogger('res152_rgb_201_fineturn_Test1')
logger.setLevel(logging.INFO)
fh = logging.FileHandler(os.path.join(SAVE_DIR,'run.log'))
formatter = logging.Formatter('%(asctime)s:%(message)s')
fh.setFormatter(formatter)
logger.addHandler(fh)

tflog = TFLogger(os.path.join(TFLOG_ROOT_PATH,EX_NAME))

##########################################
# prepare model

logging.info('ARG SETTING: {}'.format(args))
logging.info('PREPARE MODEL...')

# load dataset
train_dataset = RGB_Single_Frame_Dataset('training','train')
val_dataset   = RGB_Single_Frame_Dataset('validation','test')

# load model dpn107
model = torch.load('/mnt/md1/Model/raw_resnet152_kinitics400_rgb_model.pth')
model = nn.DataParallel(model).cuda()
crossloss = nn.CrossEntropyLoss().cuda()

optim = torch.optim.SGD(model.parameters(),lr=LR,momentum=0.9)

##########################################

def addjust_optime_lr(model,epochid):
    global LR,LR_DELAY_RATE,LR_STEPS
    if epochid in LR_STEPS:
        LR = LR*LR_DELAY_RATE
        logging.info('Change Learing Rate to: {}'.format(LR))
        optim = torch.optim.SGD(model.parameters(),lr=LR,momentum=0.9)
        return optim
    return None

def save_model(model,epochid):
    FILE = os.path.join(SAVE_DIR,'Res152_RGB_fineturn_201_model_{:03d}.ckpt'.format(epochid))
    torch.save(model.state_dict(),FILE)

##########################################
# train
if RESUME_CKPT is not None:
    model.load_state_dict(torch.load(RESUME_CKPT))

logging.info('START TRAINING...')
train_step = BASE_STEP
val_step   = BASE_STEP

for ep in range(EPOCHS):

    train_loss_meta = AverageMeter()
    val_loss_meta = AverageMeter()

    logger.info('EPOCH: [{}/{}]'.format(ep,EPOCHS))

    model.train()
    for idx in range(TRAIN_ITER_NUM):

        model.zero_grad()
        inputx, labels = train_dataset.nextbatch()

        x = Variable(inputx).cuda()
        labels = Variable(torch.from_numpy(labels)).cuda().long()

        predict = model(x)

        loss = crossloss(predict,labels)
        loss.backward()
        train_loss_meta.update(loss.data[0])
        optim.step()

        train_step += 1
        tflog.scalar_summary('train_loss',loss.data[0],train_step)

        train_acc = accuracy(predict,labels,topk=(1,5,10))
        tflog.scalar_summary('train_acc/@1',train_acc[0].data[0],train_step)
        tflog.scalar_summary('train_acc/@5',train_acc[1].data[0],train_step)
        tflog.scalar_summary('train_acc/@10',train_acc[2].data[0],train_step)

        if idx%PRINT_FREQ == 0:
            logger.info('Epoch {} {}/{} current_train_loss: {} loss_avg: {avg_train_loss.avg}'.format(ep,idx, TRAIN_ITER_NUM, loss.data[0], avg_train_loss=train_loss_meta))

        if idx%EVAL_FREQ == 0 :
            # eval
            model.eval()
            for idx in range(VAL_ITER_NUM):

                inputx, labels = val_dataset.nextbatch()
                x = Variable(inputx).cuda()
                labels = Variable(torch.from_numpy(labels),volatile=True).cuda().long()

                predict = model(x)
                loss = crossloss(predict,labels)
                val_loss_meta.update(loss.data[0])

                val_acc = accuracy(predict,labels,topk=(1,5,10))
                tflog.scalar_summary('val_acc/@1',val_acc[0].data[0],val_step)
                tflog.scalar_summary('val_acc/@5',val_acc[1].data[0],val_step)
                tflog.scalar_summary('val_acc/@10',val_acc[2].data[0],val_step)

                val_step += 1
                tflog.scalar_summary('val_loss',loss.data[0],val_step)

                if idx%PRINT_FREQ == 0:
                    logger.info('Epoch {} {}/{} current_val_loss: {} loss_avg: {avg_val_loss.avg}'.format(ep, idx, VAL_ITER_NUM, loss.data[0], avg_val_loss=val_loss_meta))

            model.train()

    logger.info('Epoch {} Train avg loss: {}'.format(ep, train_loss_meta.avg))
    noptim = addjust_optime_lr(model,ep)
    if noptim is not None: optim = noptim
    logger.info('Epoch {} Val avg loss: {}'.format(ep, val_loss_meta.avg))
    # save model
    save_model(model,ep)


code:
# resnet152 在 Kinntics 400 上面的预训练模型

# 模型地址: http://p22k53ufs.bkt.clouddn.com/trained_model/resnet152_80e_71.38p.pth.tar

# prepare_model_resnet152 从预训练中抽取并保存模型


code:
import torch
import torch.nn as nn
from torchvision.models import resnet152

ckptpath = '/mnt/md1/Downloads/model/resnet152_80e_71.38p.pth.tar'
D = torch.load(ckptpath)
model = resnet152()
D.keys()
state_dict = D['state_dict']

# 根据需要改load函数
def try_load_state_dict(self, state_dict, strict=True):
    from torch.nn.parameter import Parameter
    own_state = self.state_dict()
    for name, param in state_dict.items():
        newname = name.replace('module.base_model.','')
        if newname in own_state:
            if isinstance(param, Parameter):
                # backwards compatibility for serialized parameters
                param = param.data
            try:
                own_state[newname].copy_(param)
            except Exception:
                raise RuntimeError('While copying the parameter named {}, '
                                   'whose dimensions in the model are {} and '
                                   'whose dimensions in the checkpoint are {}.'
                                   .format(name, own_state[name].size(), param.size()))
        elif strict:
            raise KeyError('unexpected key "{}" in state_dict'
                           .format(name))
    if strict:
        missing = set(own_state.keys()) - set(state_dict.keys())
        if len(missing) > 0:
            raise KeyError('missing keys in state_dict: "{}"'.format(missing))

# 提特征
model.try_load_state_dict = try_load_state_dict
model.try_load_state_dict(model,state_dict,strict=False)

# 调整最后一层
model.fc = nn.Linear(2048,201)
# 删除多余的函数
del model.try_load_state_dict

# 检查是否正确
# model.state_dict().update(model_weight)
# model.load_state_dict()
# id(model.state_dict())
# id(model_weight)
# model.state_dict()['layer4.2.bn3.bias']
# model_weight['layer4.2.bn3.bias']
# state_dict['module.base_model.layer4.2.bn3.bias']

# save model
torch.save(model,'/mnt/md1/Model/raw_resnet152_kinitics400_rgb_model.pth')
torch.save(model.state_dict(),'/mnt/md1/Model/raw_resnet152_kinitics400_rgb_staticdict.pth')

# 模型的输入参数和训练和使用示例
import torchvision.transforms as transforms
from torch.autograd import Variable
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])

model = torch.load('/mnt/md1/Model/raw_resnet152_kinitics400_rgb_model.pth')

I = torch.randn(8,3,224,224)
I = Variable(I)
O = model(I)
O.shape

for item in model.childrens():
    print(item)


code:
#!/usr/bin/env bash

export PYTHONPATH="/mnt/md0/Qiu/Code/TCN_pytorch/:$PYTHONPATH"

EX_NAME="resnet_fineturn_1"
EX_DIR="/mnt/md1/Experiments/Resnet152_Extruct_RGB"

python3  FeatureExtruct/Resnet152_RGB/Train_Fineturn_RGB_Class.py ${EX_NAME} ${EX_DIR} --print_freq 1 --lr 0.0001 --momentum 0.9 --eval_freq 20


code:
import torchvision
import random
from PIL import Image, ImageOps
import numpy as np
import numbers
import math
import torch

class GroupRandomCrop(object):
    def __init__(self, size):
        if isinstance(size, numbers.Number):
            self.size = (int(size), int(size))
        else:
            self.size = size

    def __call__(self, img_group):

        w, h = img_group[0].size
        th, tw = self.size

        out_images = list()

        x1 = random.randint(0, w - tw)
        y1 = random.randint(0, h - th)

        for img in img_group:
            assert(img.size[0] == w and img.size[1] == h)
            if w == tw and h == th:
                out_images.append(img)
            else:
                out_images.append(img.crop((x1, y1, x1 + tw, y1 + th)))

        return out_images

class GroupCenterCrop(object):
    def __init__(self, size):
        self.worker = torchvision.transforms.CenterCrop(size)

    def __call__(self, img_group):
        return [self.worker(img) for img in img_group]

class GroupColorJitter(object):
    def __init__(self, brightness=0,contrast=0,saturation=0,hue=0):
        self.worker = torchvision.transforms.ColorJitter(brightness,contrast,saturation,hue)

    def __call__(self, img_group):
        return [self.worker(img) for img in img_group]

class GroupRandomHorizontalFlip(object):
    """Randomly horizontally flips the given PIL.Image with a probability of 0.5
    """
    def __init__(self, is_flow=False):
        self.is_flow = is_flow

    def __call__(self, img_group, is_flow=False):
        v = random.random()
        if v < 0.5:
            ret = [img.transpose(Image.FLIP_LEFT_RIGHT) for img in img_group]
            if self.is_flow:
                for i in range(0, len(ret), 2):
                    ret[i] = ImageOps.invert(ret[i])  # invert flow pixel values when flipping
            return ret
        else:
            return img_group

class GroupNormalize(object):
    def __init__(self, mean, std):
        self.mean = mean
        self.std = std

    def __call__(self, tensor):
        rep_mean = self.mean * (tensor.size()[0]//len(self.mean))
        rep_std = self.std * (tensor.size()[0]//len(self.std))

        # TODO: make efficient
        for t, m, s in zip(tensor, rep_mean, rep_std):
            t.sub_(m).div_(s)

        return tensor

class GroupScale(object):
    """ Rescales the input PIL.Image to the given 'size'.
    'size' will be the size of the smaller edge.
    For example, if height > width, then image will be
    rescaled to (size * height / width, size)
    size: size of the smaller edge
    interpolation: Default: PIL.Image.BILINEAR
    """

    def __init__(self, size, interpolation=Image.BILINEAR):
        self.worker = torchvision.transforms.Scale(size, interpolation)

    def __call__(self, img_group):
        return [self.worker(img) for img in img_group]

class GroupOverSample(object):
    def __init__(self, crop_size, scale_size=None):
        """
        :param crop_size: crop image to 224(e.g.)
        :param scale_size: scale image to shorter size 256(e.g.)
        """
        self.crop_size = crop_size if not isinstance(crop_size, int) else (crop_size, crop_size)

        if scale_size is not None:
            self.scale_worker = GroupScale(scale_size)
        else:
            self.scale_worker = None

    def __call__(self, img_group):

        if self.scale_worker is not None:
            img_group = self.scale_worker(img_group)

        image_w, image_h = img_group[0].size
        crop_w, crop_h = self.crop_size

        offsets = GroupMultiScaleCrop.fill_fix_offset(False, image_w, image_h, crop_w, crop_h)
        oversample_group = list()
        for o_w, o_h in offsets:
            normal_group = list()
            flip_group = list()
            for i, img in enumerate(img_group):
                crop = img.crop((o_w, o_h, o_w + crop_w, o_h + crop_h))
                normal_group.append(crop)
                flip_crop = crop.copy().transpose(Image.FLIP_LEFT_RIGHT)

                if img.mode == 'L' and i % 2 == 0:
                    flip_group.append(ImageOps.invert(flip_crop))
                else:
                    flip_group.append(flip_crop)

            oversample_group.extend(normal_group)
            oversample_group.extend(flip_group)
        return oversample_group

class GroupRandomResizeCrop(object):
    """
    random resize image to shorter size = [256,320] (e.g.),
    and random crop image to 224[e.g.]
    p.s.: if input size > 224, resize_range should be enlarged in equal proportion
    """
    def __init__(self, resize_range, input_size, interpolation=Image.BILINEAR):
        self.resize_range = resize_range
        self.crop_worker = GroupRandomCrop(input_size)
        self.interpolation = interpolation

    def __call__(self, img_group):
        resize_size = random.randint(self.resize_range[0],self.resize_range[1])
        resize_worker = GroupScale(resize_size)
        resized_img_group = resize_worker(img_group)
        crop_img_group = self.crop_worker(resized_img_group)

        return crop_img_group

class GroupMultiScaleCrop(object):

    def __init__(self, input_size, scales=None, max_distort=1, fix_crop=True, more_fix_crop=True):
        self.scales = scales if scales is not None else [1, .875, .75, .66]
        self.max_distort = max_distort
        self.fix_crop = fix_crop
        self.more_fix_crop = more_fix_crop
        self.input_size = input_size if not isinstance(input_size, int) else [input_size, input_size]
        self.interpolation = Image.BILINEAR

    def __call__(self, img_group):

        im_size = img_group[0].size

        crop_w, crop_h, offset_w, offset_h = self._sample_crop_size(im_size)
        crop_img_group = [img.crop((offset_w, offset_h, offset_w + crop_w, offset_h + crop_h)) for img in img_group]
        ret_img_group = [img.resize((self.input_size[0], self.input_size[1]), self.interpolation)
                         for img in crop_img_group]
        return ret_img_group

    def _sample_crop_size(self, im_size):
        image_w, image_h = im_size[0], im_size[1]

        # find a crop size
        base_size = min(image_w, image_h)
        crop_sizes = [int(base_size * x) for x in self.scales]
        crop_h = [self.input_size[1] if abs(x - self.input_size[1]) < 3 else x for x in crop_sizes]
        crop_w = [self.input_size[0] if abs(x - self.input_size[0]) < 3 else x for x in crop_sizes]

        pairs = []
        for i, h in enumerate(crop_h):
            for j, w in enumerate(crop_w):
                if abs(i - j) <= self.max_distort:
                    pairs.append((w, h))

        crop_pair = random.choice(pairs)
        if not self.fix_crop:
            w_offset = random.randint(0, image_w - crop_pair[0])
            h_offset = random.randint(0, image_h - crop_pair[1])
        else:
            w_offset, h_offset = self._sample_fix_offset(image_w, image_h, crop_pair[0], crop_pair[1])

        return crop_pair[0], crop_pair[1], w_offset, h_offset

    def _sample_fix_offset(self, image_w, image_h, crop_w, crop_h):
        offsets = self.fill_fix_offset(self.more_fix_crop, image_w, image_h, crop_w, crop_h)
        return random.choice(offsets)

    @staticmethod
    def fill_fix_offset(more_fix_crop, image_w, image_h, crop_w, crop_h):
        w_step = (image_w - crop_w) // 4
        h_step = (image_h - crop_h) // 4

        ret = list()
        ret.append((0, 0))  # upper left
        ret.append((4 * w_step, 0))  # upper right
        ret.append((0, 4 * h_step))  # lower left
        ret.append((4 * w_step, 4 * h_step))  # lower right
        ret.append((2 * w_step, 2 * h_step))  # center

        if more_fix_crop:
            ret.append((0, 2 * h_step))  # center left
            ret.append((4 * w_step, 2 * h_step))  # center right
            ret.append((2 * w_step, 4 * h_step))  # lower center
            ret.append((2 * w_step, 0 * h_step))  # upper center

            ret.append((1 * w_step, 1 * h_step))  # upper left quarter
            ret.append((3 * w_step, 1 * h_step))  # upper right quarter
            ret.append((1 * w_step, 3 * h_step))  # lower left quarter
            ret.append((3 * w_step, 3 * h_step))  # lower righ quarter

        return ret

class GroupRandomSizedCrop(object):
    """Random crop the given PIL.Image to a random size of (0.08 to 1.0) of the original size
    and and a random aspect ratio of 3/4 to 4/3 of the original aspect ratio
    This is popularly used to train the Inception networks
    size: size of the smaller edge
    interpolation: Default: PIL.Image.BILINEAR
    """
    def __init__(self, size, interpolation=Image.BILINEAR):
        self.size = size
        self.interpolation = interpolation

    def __call__(self, img_group):
        for attempt in range(10):
            area = img_group[0].size[0] * img_group[0].size[1]
            target_area = random.uniform(0.08, 1.0) * area
            aspect_ratio = random.uniform(3. / 4, 4. / 3)

            w = int(round(math.sqrt(target_area * aspect_ratio)))
            h = int(round(math.sqrt(target_area / aspect_ratio)))

            if random.random() < 0.5:
                w, h = h, w

            if w <= img_group[0].size[0] and h <= img_group[0].size[1]:
                x1 = random.randint(0, img_group[0].size[0] - w)
                y1 = random.randint(0, img_group[0].size[1] - h)
                found = True
                break
        else:
            found = False
            x1 = 0
            y1 = 0

        if found:
            out_group = list()
            for img in img_group:
                img = img.crop((x1, y1, x1 + w, y1 + h))
                assert(img.size == (w, h))
                out_group.append(img.resize((self.size, self.size), self.interpolation))
            return out_group
        else:
            # Fallback
            scale = GroupScale(self.size, interpolation=self.interpolation)
            crop = GroupRandomCrop(self.size)
            return crop(scale(img_group))

class Stack(object):

    def __init__(self, roll=False):
        self.roll = roll

    def __call__(self, img_group):
        if img_group[0].mode == 'L':
            return np.concatenate([np.expand_dims(x, 2) for x in img_group], axis=2)
        elif img_group[0].mode == 'RGB':
            if self.roll:
                return np.concatenate([np.array(x)[:, :, ::-1] for x in img_group], axis=2)
            else:
                return np.concatenate(img_group, axis=2)

class ToTorchFormatTensor(object):
    """ Converts a PIL.Image (RGB) or numpy.ndarray (H x W x C) in the range [0, 255]
    to a torch.FloatTensor of shape (C x H x W) in the range [0.0, 1.0] """
    def __init__(self, div=True):
        self.div = div

    def __call__(self, pic):
        if isinstance(pic, np.ndarray):
            # handle numpy array
            img = torch.from_numpy(pic).permute(2, 0, 1).contiguous()
        else:
            # handle PIL Image
            img = torch.ByteTensor(torch.ByteStorage.from_buffer(pic.tobytes()))
            img = img.view(pic.size[1], pic.size[0], len(pic.mode))
            # put it from HWC to CHW format
            # yikes, this transpose takes 80% of the loading time/CPU
            img = img.transpose(0, 1).transpose(0, 2).contiguous()
        return img.float().div(255) if self.div else img.float()

class IdentityTransform(object):

    def __call__(self, data):
        return data

if __name__ == "__main__":
    trans = torchvision.transforms.Compose([
        GroupScale(256),
        GroupRandomCrop(224),
        Stack(),
        ToTorchFormatTensor(),
        GroupNormalize(
            mean=[.485, .456, .406],
            std=[.229, .224, .225]
        )]
    )

    im = Image.open('../tensorflow-model-zoo.torch/lena_299.png')

    color_group = [im] * 3
    rst = trans(color_group)

    gray_group = [im.convert('L')] * 9
    gray_rst = trans(gray_group)

    trans2 = torchvision.transforms.Compose([
        GroupRandomSizedCrop(256),
        Stack(),
        ToTorchFormatTensor(),
        GroupNormalize(
            mean=[.485, .456, .406],
            std=[.229, .224, .225])
    ])
    print(trans2(color_group))


code:
import torch
import torch.nn as nn
import math
from torch.autograd import Variable

class DPN_TCN_Pair2(nn.Module):

    def __init__(self):

        super(DPN_TCN_Pair2,self).__init__()

        self.cnn1 = nn.Conv2d(in_channels=1,out_channels=1,kernel_size=(5,1))
        self.relu1 = nn.ReLU()
        self.pool1 = nn.AvgPool2d(kernel_size=(3,1),stride=(1,1))

        self.cnn2 = nn.Conv2d(in_channels=1,out_channels=1,kernel_size=(5,1))
        self.relu2 = nn.ReLU()
        self.pool2 = nn.AvgPool2d(kernel_size=(3,1),stride=(1,1))

        self.fc1 = nn.Linear(2*6*2688,500)
        self.fc2 = nn.Linear(500,2)

        self.init_weight()

    def forward(self, x1, x2):

        batchsize = x1.size(0)

        x1 = self.cnn1(x1)
        x1 = self.relu1(x1)
        x1 = self.pool1(x1)

        x2 = self.cnn2(x2)
        x2 = self.relu2(x2)
        x2 = self.pool2(x2)

        x = torch.cat([x1,x2],dim=1)

        x = x.view(batchsize,-1)
        x = self.fc1(x)
        x = self.fc2(x)

        return x

    def init_weight(self):
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))

# class DPN_TCN_Class2(nn.Module):
#
#     def __init__(self):
#         super(DPN_TCN_Class2,self).__init__()
#         self.fc1 = nn.Linear(202*202,2000)
#         self.fc2 = nn.Linear(2000,201)
#
#     def forward(self, x):
#         x = self.fc1(x)
#         x = self.fc2(x)
#         return x
#
#     def init_weight(self):
#         for m in self.modules():
#             if isinstance(m,nn.Linear) or isinstance(m,nn.Conv2d):
#                 n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
#                 m.weight.data.normal_(0, math.sqrt(2. / n))

if __name__=='__main__':

    m = DPN_TCN_Pair2()

    x = torch.randn((512,1,12,2688))
    x = Variable(x)

    o = m(x,x)

    print(o.size())


code:
import torch
import torch.nn as nn
import math
from torch.autograd import Variable

class TCN_Pair2(nn.Module):

    def __init__(self):

        super(TCN_Pair2,self).__init__()

        self.cnn1 = nn.Conv2d(in_channels=1,out_channels=1,kernel_size=(5,1))
        self.relu1 = nn.ReLU()
        self.pool1 = nn.AvgPool2d(kernel_size=(3,1),stride=(1,1))

        self.cnn2 = nn.Conv2d(in_channels=1,out_channels=1,kernel_size=(5,1))
        self.relu2 = nn.ReLU()
        self.pool2 = nn.AvgPool2d(kernel_size=(3,1),stride=(1,1))

        self.fc1 = nn.Linear(2*6*202,500)
        self.fc2 = nn.Linear(500,2)

        self.init_weight()

    def forward(self, x1, x2):

        batchsize = x1.size(0)

        x1 = self.cnn1(x1)
        x1 = self.relu1(x1)
        x1 = self.pool1(x1)

        x2 = self.cnn2(x2)
        x2 = self.relu2(x2)
        x2 = self.pool2(x2)

        x = torch.cat([x1,x2],dim=1)

        x = x.view(batchsize,-1)
        x = self.fc1(x)
        x = self.fc2(x)

        return x

    def init_weight(self):
       for m in self.modules():
           if isinstance(m, nn.Conv2d):
               n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
               m.weight.data.normal_(0, math.sqrt(2. / n))

class TCN_Class2(nn.Module):

    def __init__(self):
        super(TCN_Class2,self).__init__()
        self.fc1 = nn.Linear(202*202,2000)
        self.fc2 = nn.Linear(2000,201)

    def forward(self, x):
        x = self.fc1(x)
        x = self.fc2(x)
        return x

    def init_weight(self):
        for m in self.modules():
            if isinstance(m,nn.Linear) or isinstance(m,nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))

if __name__=='__main__':

    m = TCN_Pair2()

    x = torch.randn((512,1,12,202))
    x = Variable(x)

    o = m(x,x)

    print(o.size())


code:
import torch
import torch.nn as nn
from torch.autograd import Variable

def parse_stage_config(stage_cfg):

    if isinstance(stage_cfg, int):
        return (stage_cfg,), stage_cfg
    elif isinstance(stage_cfg, tuple) or isinstance(stage_cfg, list):
        return stage_cfg, sum(stage_cfg)
    else:
        raise ValueError("Incorrect STPP config {}".format(stage_cfg))

class StructuredTemporalPyramidPooling(torch.nn.Module):
    """
    This the STPP operator for training. Please see the ICCV paper for more details.
    """
    def __init__(self, feat_dim, standalong_classifier=False, configs=(1, (1,2), 1)):

        super(StructuredTemporalPyramidPooling, self).__init__()
        self.sc = standalong_classifier
        self.feat_dim = feat_dim

        starting_parts, starting_mult = parse_stage_config(configs[0])
        course_parts, course_mult = parse_stage_config(configs[1])
        ending_parts, ending_mult = parse_stage_config(configs[2])

        self.feat_multiplier = starting_mult + course_mult + ending_mult
        self.parts = (starting_parts, course_parts, ending_parts)
        self.norm_num = (starting_mult, course_mult, ending_mult)

    def forward(self, ft, seg_split):

        x1 = seg_split[0]
        x2 = seg_split[1]

        src = ft
        n_sample = src.size()[0]

        def get_stage_stpp(stage_ft, stage_parts, norm_num, scaling):

            stage_stpp = []
            stage_len = stage_ft.size(1)
            for n_part in stage_parts:
                ticks = torch.arange(0, stage_len + 1e-5, stage_len / n_part)
                for i in range(n_part):
                    part_ft = stage_ft[:, int(ticks[i]):int(ticks[i+1]), :].mean(dim=1) / norm_num
                    if scaling is not None:
                        part_ft = part_ft * scaling.resize(n_sample, 1)
                    stage_stpp.append(part_ft)

            return stage_stpp

        feature_parts = []
        feature_parts.extend(get_stage_stpp(src[:, :x1, :], self.parts[0], self.norm_num[0], None))  # starting
        feature_parts.extend(get_stage_stpp(src[:, x1:x2, :], self.parts[1], self.norm_num[1], None))  # course
        feature_parts.extend(get_stage_stpp(src[:, x2:, :], self.parts[2], self.norm_num[2], None))  # ending
        stpp_ft = torch.cat(feature_parts, dim=1)

        if not self.sc:
            return stpp_ft, stpp_ft
        else:
            course_ft = src[:, x1:x2, :].mean(dim=1)
            return course_ft, stpp_ft

    def activity_feat_dim(self):
        if self.sc: return self.feat_dim
        else: return self.feat_dim * self.feat_multiplier

    def completeness_feat_dim(self):
        return self.feat_dim * self.feat_multiplier

class SSN(nn.Module):
    # B x 12 x 2688

    def __init__(self,featuredim,seg_split):

        super(SSN,self).__init__()

        self.featuredim = featuredim
        self.seg_split = seg_split
        self.sptt = StructuredTemporalPyramidPooling(featuredim,configs=(1,(1,2),1))
        self.fc_cls = nn.Linear(self.sptt.activity_feat_dim(),201)
        self.fc_pair = nn.Linear(self.sptt.completeness_feat_dim(),2)

    def forward(self, x):

        corse_ft, stpp_ft = self.sptt(x,self.seg_split)
        pred_cls = self.fc_cls(corse_ft)
        pred_pai = self.fc_pair(stpp_ft)

        return pred_cls,pred_pai

class SSN_OnlyPair(nn.Module):

    def __init__(self,featuredim,seg_split):

        super(SSN_OnlyPair,self).__init__()

        self.featuredim = featuredim
        self.seg_split = seg_split
        self.sptt = StructuredTemporalPyramidPooling(featuredim,configs=(1,(1,2),1))
        self.fc_pair = nn.Linear(self.sptt.completeness_feat_dim(),2)

    def forward(self, x):
        corse_ft, stpp_ft = self.sptt(x,self.seg_split)
        pred_pai = self.fc_pair(stpp_ft)
        return pred_pai

class SSN_OnlyClass(nn.Module):

    def __init__(self,featuredim,seg_split):

        super(SSN_OnlyClass,self).__init__()

        self.featuredim = featuredim
        self.seg_split = seg_split
        self.sptt = StructuredTemporalPyramidPooling(featuredim,configs=(1,(1,2),1))
        self.fc_pair = nn.Linear(self.sptt.completeness_feat_dim(),201)

    def forward(self, x):
        corse_ft, stpp_ft = self.sptt(x,self.seg_split)
        pred_pai = self.fc_pair(stpp_ft)
        return pred_pai

if __name__=='__main__':

    x = torch.randn(8,12,202)
    x = Variable(x)
    model = SSN_OnlyClass(202,[3,9,12])
    out = model(x)
    print(out.shape)


code:
#!/usr/bin/env bash

export PYTHONPATH="/mnt/md0/Qiu/Code/TCN_pytorch/:$PYTHONPATH"

EX_DIR="/mnt/md1/Experiments/PYTSN_Test3"

python3 Train/train_class2.py ${EX_DIR} --print_freq 50 --lr 0.000005


code:
#!/usr/bin/env bash

export PYTHONPATH="/mnt/md0/Qiu/Code/TCN_pytorch/:$PYTHONPATH"

EX_NAME="PYTEST_Test4_Pair2"
EX_DIR="/mnt/md1/Experiments/PYTSN_Test4"

python3 Train/train_dpn107_pair2.py ${EX_NAME} ${EX_DIR} --print_freq 1 --lr 0.001 --momentum 0.01 --basestep 3000


code:
#!/usr/bin/env bash

export PYTHONPATH="/mnt/md0/Qiu/Code/TCN_pytorch/:$PYTHONPATH"

EX_NAME="dpn107_rgb_201_test1"
EX_DIR="/mnt/md1/Experiments/DPN_Extruct_200_Test1"

python3 FeatureExtruct/DPN107/Train_Fineturn_RGB_Class.py  ${EX_NAME} ${EX_DIR} --print_freq 1 --lr 0.0001 --momentum 0.9 --eval_freq 20


code:
#!/usr/bin/env bash

export PYTHONPATH="/mnt/md0/Qiu/Code/TCN_pytorch/:$PYTHONPATH"

EX_NAME="DPN107_201_RGB_Pari2_Test3"
EX_DIR="/mnt/md1/Experiments/DPN107_RGB_Test2"

python3 Train/train_dpn107_sptt_pair2.py ${EX_NAME} ${EX_DIR} --print_freq 1  --lr 0.1 --momentum 0.9 --eval_freq 20


code:
#!/usr/bin/env bash

export PYTHONPATH="/mnt/md0/Qiu/Code/TCN_pytorch/:$PYTHONPATH"

EX_NAME="dpn107_rgb_rank_test1"
EX_DIR="/mnt/md1/Experiments/DPN_Extruct_Rank_Test1"

python3 FeatureExtruct/DPN107/Train_Fineturn_RGB_Ranker.py  ${EX_NAME} ${EX_DIR} --print_freq 1 --lr 0.0001 --momentum 0.9 --eval_freq 20


code:
#!/usr/bin/env bash

export PYTHONPATH="/mnt/md0/Qiu/Code/TCN_pytorch/:$PYTHONPATH"

EX_NAME="PYTEST_Test5_Pair2"
EX_DIR="/mnt/md1/Experiments/PYTSN_Test5"

python3 Train/train_dpn107_sptt_pair2.py ${EX_NAME} ${EX_DIR} --print_freq 1 --lr 0.001 --momentum 0.9 --eval_freq 1


code:
#!/usr/bin/env bash

export PYTHONPATH="/mnt/md0/Qiu/Code/TCN_pytorch/:$PYTHONPATH"

EX_DIR="/mnt/md1/Experiments/PYTSN_Test3"

python3 Train/train_pair2.py ${EX_DIR}


code:
#!/usr/bin/env bash

export PYTHONPATH="/mnt/md0/Qiu/Code/TCN_pytorch/:$PYTHONPATH"

EX_NAME="PYTEST_Test7_Class2"
EX_DIR="/mnt/md1/Experiments/PYTSN_Test7"

python3 Train/train_tsnscore_sptt_class2.py ${EX_NAME} ${EX_DIR} --print_freq 1 --lr 0.1 --momentum 0.9 --eval_freq 20


code:
#!/usr/bin/env bash

export PYTHONPATH="/mnt/md0/Qiu/Code/TCN_pytorch/:$PYTHONPATH"

EX_NAME="PYTEST_Test7_Pair2"
EX_DIR="/mnt/md1/Experiments/PYTSN_Test7"

python3 Train/train_tsnscore_sptt_pair2.py ${EX_NAME} ${EX_DIR} --print_freq 1 --lr 0.01 --momentum 0.9 --eval_freq 20


code:
import pickle
import random
from Config.Config import ACTNET200V13_PKL

with open(ACTNET200V13_PKL,'rb') as f:
    data = pickle.load(f)

dataset = data['database']
vids = list(dataset.keys())

labels_count_train = dict()
labels_count_val = dict()

for vid in vids:
    item = dataset[vid]
    if item['subset'] == 'validation':
        label = item['annotations'][0]['label']
        labels_count_val[label] = labels_count_val.get(label,0)+1
        if labels_count_val[label] > 80:
            item['subset'] = 'training'
        else:
            if random.choice([True,False,True,True]):
                item['subset'] = 'training'

vids = list(dataset.keys())
a=0
b=0
c=0
for vid in vids:
    item = dataset[vid]
    if item['subset'] == 'validation':
        a+=1
    elif item['subset'] == 'training':
        b+=1
    else:
        c+=1

print(a,b,c)

with open('/tmp/moretrain_actNet200-V1-3.pkl','wb') as f:
    pickle.dump(dataset,f)


code:
import pickle
import random
from Config.Config import ACTNET200V13_PKL

with open(ACTNET200V13_PKL,'rb') as f:
    big = pickle.load(f)

small = big.copy()
big_d = big['database']
small_d = small['database']

train_num = 800
val_num   = 200

vids = list(big_d.keys())

labels_count_train = dict()
labels_count_val = dict()

for vid in vids:
    item = big_d[vid]
    if item['subset'] == 'training':
        label = item['annotations'][0]['label']
        labels_count_train[label] = labels_count_train.get(label,0)+1
        if labels_count_train[label] > 40:
            del small_d[vid]
        else:
            if random.choice([True,False,True,True]):
                del small_d[vid]
    elif item['subset'] == 'validation':
        label = item['annotations'][0]['label']
        labels_count_val[label] = labels_count_val.get(label,0)+1
        if labels_count_val[label] > 40:
            del small_d[vid]
        else:
            if random.choice([True,False,True,True]):
                del small_d[vid]
    else:
        del small_d[vid]

vids = list(small_d.keys())
a=0
b=0
for vid in vids:
    item = small_d[vid]
    if item['subset'] == 'validation':
        a+=1
    elif item['subset'] == 'training':
        b+=1

print(a,b)

with open('/mnt/md1/Experiments/TCN_Test1/PKLS/small_actNet200-V1-3.pkl','wb') as f:
    pickle.dump(small,f)


code:
import tensorflow as tf
import numpy as np
import scipy.misc
from io import BytesIO  # Python 3.x

class TFLogger(object):

    def __init__(self, log_dir):
        """Create a summary writer logging to log_dir."""
        self.writer = tf.summary.FileWriter(log_dir)

    def scalar_summary(self, tag, value, step):
        """Log a scalar variable."""
        summary = tf.Summary(value=[tf.Summary.Value(tag=tag, simple_value=value)])
        self.writer.add_summary(summary, step)

    def image_summary(self, tag, images, step):
        """Log a list of images."""

        img_summaries = []
        for i, img in enumerate(images):
            # Write the image to a string
            try:
                s = StringIO()
            except:
                s = BytesIO()
            scipy.misc.toimage(img).save(s, format="png")

            # Create an Image object
            img_sum = tf.Summary.Image(encoded_image_string=s.getvalue(),
                                       height=img.shape[0],
                                       width=img.shape[1])
            # Create a Summary value
            img_summaries.append(
                tf.Summary.Value(tag='%s/%d' % (tag, i), image=img_sum))

        # Create and write Summary
        summary = tf.Summary(value=img_summaries)
        self.writer.add_summary(summary, step)

    def histo_summary(self, tag, values, step, bins=1000):
        """Log a histogram of the tensor of values."""

        # Create a histogram using numpy
        counts, bin_edges = np.histogram(values, bins=bins)

        # Fill the fields of the histogram proto
        hist = tf.HistogramProto()
        hist.min = float(np.min(values))
        hist.max = float(np.max(values))
        hist.num = int(np.prod(values.shape))
        hist.sum = float(np.sum(values))
        hist.sum_squares = float(np.sum(values ** 2))

        # Drop the start of the first bin
        bin_edges = bin_edges[1:]

        # Add bin edges and counts
        for edge in bin_edges:
            hist.bucket_limit.append(edge)
        for c in counts:
            hist.bucket.append(c)

        # Create and write Summary
        summary = tf.Summary(value=[tf.Summary.Value(tag=tag, histo=hist)])
        self.writer.add_summary(summary, step)
        self.writer.flush()


code:
import json
import urllib.request, urllib.error, urllib.parse

import numpy as np
import pandas as pd

from Tools.utils import get_blocked_videos
from Tools.utils import interpolated_prec_rec

class ANETclassification(object):
    GROUND_TRUTH_FIELDS = ['database', 'taxonomy', 'version']
    PREDICTION_FIELDS = ['results', 'version', 'external_data']

    def __init__(self, ground_truth_filename=None, prediction_filename=None,
                 ground_truth_fields=GROUND_TRUTH_FIELDS,
                 prediction_fields=PREDICTION_FIELDS,
                 subset='validation', verbose=False, top_k=3,
                 check_status=True):
        if not ground_truth_filename:
            raise IOError('Please input a valid ground truth file.')
        if not prediction_filename:
            raise IOError('Please input a valid prediction file.')
        self.subset = subset
        self.verbose = verbose
        self.gt_fields = ground_truth_fields
        self.pred_fields = prediction_fields
        self.top_k = top_k
        self.ap = None
        self.hit_at_k = None
        self.check_status = check_status
        # Retrieve blocked videos from server.
        if self.check_status:
            self.blocked_videos = get_blocked_videos()
        else:
            self.blocked_videos = list()
        # Import ground truth and predictions.
        self.ground_truth, self.activity_index = self._import_ground_truth(
            ground_truth_filename)
        self.prediction = self._import_prediction(prediction_filename)

        if self.verbose:
            print('[INIT] Loaded annotations from {} subset.'.format(subset))
            nr_gt = len(self.ground_truth)
            print('\tNumber of ground truth instances: {}'.format(nr_gt))
            nr_pred = len(self.prediction)
            print('\tNumber of predictions: {}'.format(nr_pred))

    def _import_ground_truth(self, ground_truth_filename):
        """Reads ground truth file, checks if it is well formatted, and returns
           the ground truth instances and the activity classes.

        Parameters
        ----------
        ground_truth_filename : str
            Full path to the ground truth json file.

        Outputs
        -------
        ground_truth : df
            Data frame containing the ground truth instances.
        activity_index : dict
            Dictionary containing class index.
        """
        with open(ground_truth_filename, 'r') as fobj:
            data = json.load(fobj)
        # Checking format
        if not all([field in list(data.keys()) for field in self.gt_fields]):
            raise IOError('Please input a valid ground truth file.')

        # Initialize data frame
        activity_index, cidx = {}, 0
        video_lst, label_lst = [], []
        for videoid, v in data['database'].items():
            if self.subset != v['subset']:
                continue
            if videoid in self.blocked_videos:
                continue
            for ann in v['annotations']:
                if ann['label'] not in activity_index:
                    activity_index[ann['label']] = cidx
                    cidx += 1
                video_lst.append(videoid)
                label_lst.append(activity_index[ann['label']])
        ground_truth = pd.DataFrame({'video-id': video_lst,
                                     'label': label_lst})
        ground_truth = ground_truth.drop_duplicates().reset_index(drop=True)
        return ground_truth, activity_index

    def _import_prediction(self, prediction_filename):
        """Reads prediction file, checks if it is well formatted, and returns
           the prediction instances.

        Parameters
        ----------
        prediction_filename : str
            Full path to the prediction json file.

        Outputs
        -------
        prediction : df
            Data frame containing the prediction instances.
        """
        with open(prediction_filename, 'r') as fobj:
            data = json.load(fobj)
        # Checking format...
        if not all([field in list(data.keys()) for field in self.pred_fields]):
            raise IOError('Please input a valid prediction file.')

        # Initialize data frame
        video_lst, label_lst, score_lst = [], [], []
        for videoid, v in data['results'].items():
            if videoid in self.blocked_videos:
                continue
            for result in v:
                label = self.activity_index[result['label']]
                video_lst.append(videoid)
                label_lst.append(label)
                score_lst.append(result['score'])
        prediction = pd.DataFrame({'video-id': video_lst,
                                   'label': label_lst,
                                   'score': score_lst})
        return prediction

    def wrapper_compute_average_precision(self):
        """Computes average precision for each class in the subset.
        """
        ap = np.zeros(len(list(self.activity_index.items())))
        for activity, cidx in self.activity_index.items():
            gt_idx = self.ground_truth['label'] == cidx
            pred_idx = self.prediction['label'] == cidx
            ap[cidx] = compute_average_precision_classification(
                self.ground_truth.loc[gt_idx].reset_index(drop=True),
                self.prediction.loc[pred_idx].reset_index(drop=True))
        return ap

    def evaluate(self):
        """Evaluates a prediction file. For the detection task we measure the
        interpolated mean average precision to measure the performance of a
        method.
        """
        ap = self.wrapper_compute_average_precision()
        hit_at_k = compute_video_hit_at_k(self.ground_truth,
                                          self.prediction, top_k=self.top_k)
        avg_hit_at_k = compute_video_hit_at_k(
            self.ground_truth, self.prediction, top_k=self.top_k, avg=True)
        if self.verbose:
            print ('[RESULTS] Performance on ActivityNet untrimmed video '
                   'classification task.')
            print('\tMean Average Precision: {}'.format(ap.mean()))
            print('\tHit@{}: {}'.format(self.top_k, hit_at_k))
            print('\tAvg Hit@{}: {}'.format(self.top_k, avg_hit_at_k))
        self.ap = ap
        self.hit_at_k = hit_at_k
        self.avg_hit_at_k = avg_hit_at_k

################################################################################
# Metrics
################################################################################

def compute_average_precision_classification(ground_truth, prediction):
    """Compute average precision (classification task) between ground truth and
    predictions data frames. If multiple predictions occurs for the same
    predicted segment, only the one with highest score is matched as
    true positive. This code is greatly inspired by Pascal VOC devkit.

    Parameters
    ----------
    ground_truth : df
        Data frame containing the ground truth instances.
        Required fields: ['video-id']
    prediction : df
        Data frame containing the prediction instances.
        Required fields: ['video-id, 'score']

    Outputs
    -------
    ap : float
        Average precision score.
    """
    npos = float(len(ground_truth))
    lock_gt = np.ones(len(ground_truth)) * -1
    # Sort predictions by decreasing score order.
    sort_idx = prediction['score'].values.argsort()[::-1]
    prediction = prediction.loc[sort_idx].reset_index(drop=True)

    # Initialize true positive and false positive vectors.
    tp = np.zeros(len(prediction))
    fp = np.zeros(len(prediction))

    # Assigning true positive to truly grount truth instances.
    for idx in range(len(prediction)):
        this_pred = prediction.loc[idx]
        gt_idx = ground_truth['video-id'] == this_pred['video-id']
        # Check if there is at least one ground truth in the video associated.
        if not gt_idx.any():
            fp[idx] = 1
            continue
        this_gt = ground_truth.loc[gt_idx].reset_index()
        if lock_gt[this_gt['index']] >= 0:
            fp[idx] = 1
        else:
            tp[idx] = 1
            lock_gt[this_gt['index']] = idx

    # Computing prec-rec
    tp = np.cumsum(tp).astype(np.float)
    fp = np.cumsum(fp).astype(np.float)
    rec = tp / npos
    prec = tp / (tp + fp)
    return interpolated_prec_rec(prec, rec)

def compute_video_hit_at_k(ground_truth, prediction, top_k=3, avg=False):
    """Compute accuracy at k prediction between ground truth and
    predictions data frames. This code is greatly inspired by evaluation
    performed in Karpathy et al. CVPR14.

    Parameters
    ----------
    ground_truth : df
        Data frame containing the ground truth instances.
        Required fields: ['video-id', 'label']
    prediction : df
        Data frame containing the prediction instances.
        Required fields: ['video-id, 'label', 'score']

    Outputs
    -------
    acc : float
        Top k accuracy score.
    """
    video_ids = np.unique(ground_truth['video-id'].values)
    avg_hits_per_vid = np.zeros(video_ids.size)
    for i, vid in enumerate(video_ids):
        pred_idx = prediction['video-id'] == vid
        if not pred_idx.any():
            continue
        this_pred = prediction.loc[pred_idx].reset_index(drop=True)
        # Get top K predictions sorted by decreasing score.
        sort_idx = this_pred['score'].values.argsort()[::-1][:top_k]
        this_pred = this_pred.loc[sort_idx].reset_index(drop=True)
        # Get labels and compare against ground truth.
        pred_label = this_pred['label'].tolist()
        gt_idx = ground_truth['video-id'] == vid
        gt_label = ground_truth.loc[gt_idx]['label'].tolist()
        avg_hits_per_vid[i] = np.mean([1 if this_label in pred_label else 0
                                       for this_label in gt_label])
        if not avg:
            avg_hits_per_vid[i] = np.ceil(avg_hits_per_vid[i])
    return float(avg_hits_per_vid.mean())


code:
import json
import urllib.request, urllib.error, urllib.parse

import numpy as np
import pandas as pd

from Tools.utils import get_blocked_videos
from Tools.utils import interpolated_prec_rec
from Tools.utils import segment_iou

class ANETdetection(object):

    GROUND_TRUTH_FIELDS = ['database', 'taxonomy', 'version']
    PREDICTION_FIELDS = ['results', 'version', 'external_data']

    def __init__(self, ground_truth_filename=None, prediction_filename=None,
                 ground_truth_fields=GROUND_TRUTH_FIELDS,
                 prediction_fields=PREDICTION_FIELDS,
                 tiou_thr=0.5, subset='validation', verbose=False,
                 check_status=True):
        if not ground_truth_filename:
            raise IOError('Please input a valid ground truth file.')
        if not prediction_filename:
            raise IOError('Please input a valid prediction file.')
        self.subset = subset
        self.tiou_thr = tiou_thr
        self.verbose = verbose
        self.gt_fields = ground_truth_fields
        self.pred_fields = prediction_fields
        self.ap = None
        self.check_status = check_status
        # Retrieve blocked videos from server.
        if self.check_status:
            self.blocked_videos = get_blocked_videos()
        else:
            self.blocked_videos = list()
        # Import ground truth and predictions.
        self.ground_truth, self.activity_index = self._import_ground_truth(
            ground_truth_filename)
        self.prediction = self._import_prediction(prediction_filename)

        if self.verbose:
            print('[INIT] Loaded annotations from {} subset.'.format(subset))
            nr_gt = len(self.ground_truth)
            print('\tNumber of ground truth instances: {}'.format(nr_gt))
            nr_pred = len(self.prediction)
            print('\tNumber of predictions: {}'.format(nr_pred))
            print('\tFixed threshold for tiou score: {}'.format(self.tiou_thr))

    def _import_ground_truth(self, ground_truth_filename):
        """Reads ground truth file, checks if it is well formatted, and returns
           the ground truth instances and the activity classes.

        Parameters
        ----------
        ground_truth_filename : str
            Full path to the ground truth json file.

        Outputs
        -------
        ground_truth : df
            Data frame containing the ground truth instances.
        activity_index : dict
            Dictionary containing class index.
        """
        with open(ground_truth_filename, 'r') as fobj:
            data = json.load(fobj)
        # Checking format
        if not all([field in list(data.keys()) for field in self.gt_fields]):
            raise IOError('Please input a valid ground truth file.')

        # Read ground truth data.
        activity_index, cidx = {}, 0
        video_lst, t_start_lst, t_end_lst, label_lst = [], [], [], []
        for videoid, v in data['database'].items():
            if self.subset != v['subset']:
                continue
            if videoid in self.blocked_videos:
                continue
            for ann in v['annotations']:
                if ann['label'] not in activity_index:
                    activity_index[ann['label']] = cidx
                    cidx += 1
                video_lst.append(videoid)
                t_start_lst.append(ann['segment'][0])
                t_end_lst.append(ann['segment'][1])
                label_lst.append(activity_index[ann['label']])

        ground_truth = pd.DataFrame({'video-id': video_lst,
                                     't-start': t_start_lst,
                                     't-end': t_end_lst,
                                     'label': label_lst})
        return ground_truth, activity_index

    def _import_prediction(self, prediction_filename):
        """Reads prediction file, checks if it is well formatted, and returns
           the prediction instances.

        Parameters
        ----------
        prediction_filename : str
            Full path to the prediction json file.

        Outputs
        -------
        prediction : df
            Data frame containing the prediction instances.
        """
        with open(prediction_filename, 'r') as fobj:
            data = json.load(fobj)
        # Checking format...
        if not all([field in list(data.keys()) for field in self.pred_fields]):
            raise IOError('Please input a valid prediction file.')

        # Read predicitons.
        video_lst, t_start_lst, t_end_lst = [], [], []
        label_lst, score_lst = [], []
        for videoid, v in data['results'].items():
            if videoid in self.blocked_videos:
                continue
            for result in v:
                label = self.activity_index[result['label']]
                video_lst.append(videoid)
                t_start_lst.append(result['segment'][0])
                t_end_lst.append(result['segment'][1])
                label_lst.append(label)
                score_lst.append(result['score'])
        prediction = pd.DataFrame({'video-id': video_lst,
                                   't-start': t_start_lst,
                                   't-end': t_end_lst,
                                   'label': label_lst,
                                   'score': score_lst})
        return prediction

    def wrapper_compute_average_precision(self):
        """Computes average precision for each class in the subset.
        """
        ap = np.zeros(len(list(self.activity_index.items())))
        for activity, cidx in self.activity_index.items():
            gt_idx = self.ground_truth['label'] == cidx
            pred_idx = self.prediction['label'] == cidx
            #print np.sum(gt_idx),np.sum(pred_idx)
            ap[cidx] = compute_average_precision_detection(
                self.ground_truth.loc[gt_idx].reset_index(drop=True),
                self.prediction.loc[pred_idx].reset_index(drop=True),
                tiou_thr=self.tiou_thr)
        return ap

    def evaluate(self):
        """Evaluates a prediction file. For the detection task we measure the
        interpolated mean average precision to measure the performance of a
        method.
        """
        ap = self.wrapper_compute_average_precision()
        if self.verbose:
            print('[RESULTS] Performance on ActivityNet detection task.')
            print('\tMean Average Precision: {}'.format(ap.mean()),flush=True)
        self.ap = ap

from numba import jit
@jit
def compute_average_precision_detection(ground_truth, prediction, tiou_thr=0.5):
    """Compute average precision (detection task) between ground truth and
    predictions data frames. If multiple predictions occurs for the same
    predicted segment, only the one with highest score is matches as
    true positive. This code is greatly inspired by Pascal VOC devkit.

    Parameters
    ----------
    ground_truth : df
        Data frame containing the ground truth instances.
        Required fields: ['video-id', 't-start', 't-end']
    prediction : df
        Data frame containing the prediction instances.
        Required fields: ['video-id, 't-start', 't-end', 'score']
    tiou_thr : (float, optional)
        Temporal intersection over union threshold.

    Outputs
    -------
    ap : float
        Average precision score.
    """
    npos = float(len(ground_truth))
    lock_gt = np.ones(len(ground_truth)) * -1
    # Sort predictions by decreasing score order.
    sort_idx = prediction['score'].values.argsort()[::-1]
    prediction = prediction.loc[sort_idx].reset_index(drop=True)

    # Initialize true positive and false positive vectors.
    tp = np.zeros(len(prediction))
    fp = np.zeros(len(prediction))

    # Assigning true positive to truly grount truth instances.
    for idx in range(len(prediction)):
        this_pred = prediction.loc[idx]
        gt_idx = ground_truth['video-id'] == this_pred['video-id']
        # Check if there is at least one ground truth in the video associated.
        if not gt_idx.any():
            fp[idx] = 1
            continue
        this_gt = ground_truth.loc[gt_idx].reset_index()
        tiou_arr = segment_iou(this_pred[['t-start', 't-end']].values,
                               this_gt[['t-start', 't-end']].values)
        # We would like to retrieve the predictions with highest tiou score.
        tiou_sorted_idx = tiou_arr.argsort()[::-1]
        for jdx in tiou_sorted_idx:
            if tiou_arr[jdx] < tiou_thr:
                fp[idx] = 1
                break
            if lock_gt[this_gt.loc[jdx]['index']] >= 0:
                continue
            # Assign as true positive after the filters above.
            tp[idx] = 1
            lock_gt[this_gt.loc[jdx]['index']] = idx
            break

    # Computing prec-rec
    tp = np.cumsum(tp).astype(np.float)
    fp = np.cumsum(fp).astype(np.float)
    rec = tp / npos
    prec = tp / (tp + fp)
    return interpolated_prec_rec(prec, rec)


code:
import json
import numpy as np
import pickle as pickle

def iou(test, target):

    tt1 = np.maximum(target[0], test[0])
    tt2 = np.minimum(target[1], test[1])

    # Non-negative overlap score
    intersection = (tt2 - tt1).clip(0)
    union = ((test[1] - test[0]) +
             (target[1] - target[0]) -
             intersection)
    # Compute overlap as the ratio of the intersection
    # over union of two segments at the frame level.
    iou = intersection / union
    return iou

# support customized levels, the level means
def gen_temporal_pyramid(data, levels=[1./32, 1./16, 1./8, 1./4, 1./2, 1.], thr_ious=[.4, .45, .5, .55, .6, .7], overlap=0.5, SUBSET='validation'):
    video_id_fmt = 'v_{}'
    proposals = {}
    for vid, vitem in data.items():
        if vitem['subset'] != SUBSET:
            continue

        gt_segments = [x['segment'] for x in vitem['annotations']]
        gt_labels = [x['class'] for x in vitem['annotations']]
        duration = vitem['duration']
        split = duration*levels[0]
        anchor = np.arange(split/2, duration-split/2, split*overlap)
        proposals[vid] = []
        for p in anchor:

            one_pyramid = []
            gt_index = -1
            for i in range(len(levels)):
                split = duration*levels[i]
                # handle multiple gts case, choose the one close to the level proposoal
                ious = [iou([max(0,p-split/2), min(duration,p+split/2)], gt) for gt in gt_segments]
                gt_index = np.argmax(ious)

                # also include the nearest class label for further usage
                # start , end , iou , label
                one_pyramid.append([max(0,p-split/2), min(duration,p+split/2),
                                    ious[gt_index],
                                    gt_labels[gt_index] if ious[gt_index]>=thr_ious[i] else 0
                                    ])
                # one_pyramid.append([p-split/2, p+split/2,
                #                     ious[gt_index],
                #                     gt_labels[gt_index] if ious[gt_index]>=thr_ious[i] else 0
                #                     ])
            proposals[vid].append(one_pyramid)

    return proposals

def test_temporal_pyramid(data, levels=[1./32, 1./16, 1./8, 1./4, 1./2, 1.], overlap=0.5, SUBSET='testing'):
    video_id_fmt = 'v_{}'
    proposals = {}
    for vid, vitem in data.items():
        if vitem['subset'] != SUBSET:
            continue

        duration = vitem['duration']
        split = duration*levels[0]
        anchor = np.arange(split/2, duration-split/2, split*overlap)
        proposals[vid] = []
        for p in anchor:

            one_pyramid = []
            gt_index = -1
            for i in range(len(levels)):
                split = duration*levels[i]
                # also include the nearest class label for further usage
                one_pyramid.append([max(0,p-split/2), min(duration,p+split/2),0,0])
                # one_pyramid.append([p-split/2, p+split/2,0,0])
            proposals[vid].append(one_pyramid)

    return proposals

if __name__ == "__main__":

    ANNOTATION_FILE = 'actNet200-V1-3.pkl'
    with open(ANNOTATION_FILE,'rb') as f:
         data = pickle.load(f)['database']

    train_proposals = gen_temporal_pyramid(data, SUBSET='training')
    val_proposals = gen_temporal_pyramid(data, SUBSET='validation')
    test_proposals = test_temporal_pyramid(data)

    pickle.dump(train_proposals, open("train_proposals.pkl", "wb"))
    pickle.dump(val_proposals, open("val_proposals.pkl", "wb"))
    pickle.dump(test_proposals, open("test_proposals.pkl", "wb"))

    debug = 0


code:
import argparse

pair2_parser = argparse.ArgumentParser(description='ArgParser for pair2 train')
pair2_parser.add_argument('exname',type=str,default=None)
pair2_parser.add_argument('savedir', type=str, default=None)
pair2_parser.add_argument('--epochs', type=int, default=80)
pair2_parser.add_argument('--train_iter_num', type=int, default=1000)
pair2_parser.add_argument('--val_iter_num', type=int, default=200)
pair2_parser.add_argument('--eval_freq', type=int, default=2)
pair2_parser.add_argument('--lr', type=float, default=0.1)
pair2_parser.add_argument('--lr_steps', type=list, default=[10, 20, 40])
pair2_parser.add_argument('--lr_delay_rate', type=float, default=0.00005)
pair2_parser.add_argument('--model_arch', type=str, default=None)
pair2_parser.add_argument('--print_freq',type=int,default=50)
pair2_parser.add_argument('--resume_ckpt', type=str, default=None)
pair2_parser.add_argument('--momentum',type=float,default=0.9)
pair2_parser.add_argument('--basestep',type=int,default=0)

class2_parser = argparse.ArgumentParser(description='ArgParser for class2 train')
class2_parser.add_argument('exname',type=str,default=None)
class2_parser.add_argument('savedir', type=str, default=None)
class2_parser.add_argument('--epochs', type=int, default=80)
class2_parser.add_argument('--train_iter_num', type=int, default=1000)
class2_parser.add_argument('--val_iter_num', type=int, default=200)
class2_parser.add_argument('--eval_freq', type=int, default=2)
class2_parser.add_argument('--lr', type=float, default=0.000005)
class2_parser.add_argument('--lr_steps', type=list, default=[10, 20, 40])
class2_parser.add_argument('--lr_delay_rate', type=float, default=0.5)
class2_parser.add_argument('--model_arch', type=str, default=None)
class2_parser.add_argument('--print_freq',type=int,default=50)
class2_parser.add_argument('--resume_ckpt', type=str, default=None)
class2_parser.add_argument('--momentum',type=float,default=0.9)
class2_parser.add_argument('--basestep',type=int,default=0)


code:
import pickle as pickle
import numpy as np
import pandas as pd
import h5py
from multiprocessing import Pool
import json
import urllib.request, urllib.error, urllib.parse

API = 'http://ec2-52-11-11-89.us-west-2.compute.amazonaws.com/challenge16/api.py'

def get_blocked_videos(api=API):
    api_url = '{}?action=get_blocked'.format(api)
    req = urllib.request.Request(api_url)
    response = urllib.request.urlopen(req)
    return json.loads(response.read())

def softmax(raw_score, T=1):
    exp_s = np.exp((raw_score - raw_score.max(axis=-1)[..., None])*T)
    sum_s = exp_s.sum(axis=-1)
    return exp_s / sum_s[..., None]

def sliding_window_aggregation_func(score, spans=[1, 2, 4, 8, 16], overlap=0.2, norm=True, fps=1):
    frm_max = score.max(axis=1)
    slide_score = []

    def top_k_pool(scores, k):
        return np.sort(scores, axis=0)[-k:, :].mean(axis=0)

    for t_span in spans:
        span = t_span * fps
        step = int(np.ceil(span * (1-overlap)))
        local_agg = [frm_max[i: i+span].max(axis=0) for i in range(0, frm_max.shape[0], step)]
        k = max(15, len(local_agg)//4)
        slide_score.append(top_k_pool(np.array(local_agg), k))

    out_score = np.mean(slide_score, axis=0)

    if norm:
        return softmax(out_score)
    else:
        return out_score

def interpolated_prec_rec(prec, rec):
    """Interpolated AP - VOCdevkit from VOC 2011.
    """
    mprec = np.hstack([[0], prec, [0]])
    mrec = np.hstack([[0], rec, [1]])
    for i in range(len(mprec) - 1)[::-1]:
        mprec[i] = max(mprec[i], mprec[i + 1])
    idx = np.where(mrec[1::] != mrec[0:-1])[0] + 1
    ap = np.sum((mrec[idx] - mrec[idx - 1]) * mprec[idx])
    return ap

def segment_iou(target_segment, candidate_segments):
    tt1 = np.maximum(target_segment[0], candidate_segments[:, 0])
    tt2 = np.minimum(target_segment[1], candidate_segments[:, 1])
    # Intersection including Non-negative overlap score.
    segments_intersection = (tt2 - tt1).clip(0)
    # Segment union.
    segments_union = (candidate_segments[:, 1] - candidate_segments[:, 0]) \
      + (target_segment[1] - target_segment[0]) - segments_intersection
    # Compute overlap as the ratio of the intersection
    # over union of two segments.
    tIoU = segments_intersection.astype(float) / segments_union
    return tIoU

def batch_segment_iou(target_segments, test_segments):
    if target_segments.ndim != 2 or test_segments.ndim != 2:
        raise ValueError('Dimension of arguments is incorrect')

    m, n = target_segments.shape[0], test_segments.shape[0]
    iou = np.empty((m, n))
    for i in range(m):
        tt1 = np.maximum(target_segments[i, 0], test_segments[:, 0])
        tt2 = np.minimum(target_segments[i, 1], test_segments[:, 1])

        # Non-negative overlap score
        intersection = (tt2 - tt1).clip(0)
        union = ((test_segments[:, 1] - test_segments[:, 0]) +
                 (target_segments[i, 1] - target_segments[i, 0]) -
                 intersection)
        # Compute overlap as the ratio of the intersection
        # over union of two segments at the frame level.
        iou[i, :] = intersection / union
    return iou

def nms(proposals, scores, thresh):
    x1 = proposals[:, 0]
    x2 = proposals[:, 1]

    areas = (x2 - x1 + 1)
    order = scores.argsort()[::-1]

    keep = []
    while order.size > 0:
        i = order[0]
        keep.append(i)
        xx1 = np.maximum(x1[i], x1[order[1:]])
        xx2 = np.minimum(x2[i], x2[order[1:]])

        inter = np.maximum(0.0, xx2 - xx1 + 1)
        ovr = inter / (areas[i] + areas[order[1:]] - inter)

        inds = np.where(ovr <= thresh)[0]
        order = order[1:][inds]

    return keep

def nms_all(proposals, topK=20, nms_thor=0.45, remove_background=False):
    nms_proposals = {}
    for vid, proposal in proposals.items():
        proposal = np.array(proposal)
        if proposal.ndim>2:
            proposal = proposal.reshape(proposal.shape[0]*proposal.shape[1], proposal.shape[2])
        if remove_background:
            proposal = proposal[proposal[:,3]>0,:]
        if len(proposal.shape)!=2: proposal = np.concatenate(proposal)
        keep = nms(proposal, proposal[:, 2], nms_thor)
        nms_proposals[vid] = proposal[keep[:topK], :]
    return nms_proposals

def gen_json(proposals, id2name):
    output = {'results': {}, 'version': "VERSION 1.3", 'external_data': {'used': False, 'details': ""}}
    activity_names = id2name

    for vid, proposal_per_vid in proposals.items():
        proposal = np.asarray(proposal_per_vid)
        if len(proposal) > 0:
            if proposal.ndim>2:
                proposal = proposal.reshape((proposal.shape[0] * proposal.shape[1], proposal.shape[2]))
            output['results'][vid] = []
            for i in range(proposal.shape[0]):
                if proposal[i,3] < 0.5: continue
                try:
                    output['results'][vid].append({'segment': [float(proposal[i, 0]), float(proposal[i, 1])],
                                                   'score': float(proposal[i, 2]),
                                                   'label': activity_names[proposal[i, 3]]})
                except:
                    continue
    return output

def recall_vs_iou_thresholds(proposal_df, df, iou_threshold=np.arange(0.1, 1.1, 0.1)):
    vds = proposal_df['video-id'].unique()
    score_lst = []
    # Compute iou score
    for i, v in enumerate(vds):
        # Proposals
        idx = proposal_df['video-id'] == v
        this_df = proposal_df.loc[idx]
        proposals = np.stack((this_df['s-init'],
                              this_df['s-end']), axis=-1)

        # Sort proposals
        idx = this_df['score'].argsort()[::-1]
        proposals = proposals[idx, :]

        # Annotations
        jdx = df['video-id'] == v
        ann_df = df.loc[jdx]
        annotations = np.stack((ann_df['s-init'],
                                ann_df['s-end']), axis=-1)
        if proposals.ndim == 1:
            proposals = proposals[np.newaxis, :]
        score_lst.append(batch_segment_iou(annotations, proposals))
        if not (i+1)%500:
            print('Scored videos: {}'.format(i+1))
    matches = np.zeros((vds.shape[0], iou_threshold.shape[0]))
    pos = np.zeros(vds.shape[0])
    # Matching
    recall = np.empty(iou_threshold.shape[0])
    for cidx, this_iou in enumerate(iou_threshold):
        # Score analysis per video.
        for i, sc in enumerate(score_lst):
            pos[i] = sc.shape[0] # Positives per video.
            lmt = int(sc.shape[1])
            matches[i, cidx] = ((sc[:, :lmt] >= this_iou).sum(axis=1) > 0).sum()
        this_recall = matches[:, cidx].sum() / pos.sum()
        recall[cidx] = this_recall
    return recall

def convert(proposal):
    pdp = pd.DataFrame({'s-init': proposal['s-init'],
                          's-end': proposal['s-end'],
                          'score': proposal['score'],
                          'video-id': proposal['video-id']})
    return pdp

def get_gt(data=None, gt_file='actNet200-V1-3.pkl', SUBSET='validation'):
    if data is None:
        with open(gt_file, 'r') as fobj:
            data = pickle.load(fobj)['database']

    video_id_fmt = 'v_{}'
    video_dur = []
    gt_s_init, gt_s_end, video_id = [], [], []
    for vid, vitem in data.items():
        video_dur.append(vitem['duration'])
        if vitem['subset'] != SUBSET:
            continue
        for ann in vitem['annotations']:
            gt_s_init.append(ann['segment'][0])
            gt_s_end.append(ann['segment'][1])
            video_id.append(video_id_fmt.format(vid))
    # Creates ground truth data frame.
    ground_truth_df = pd.DataFrame({'s-init': gt_s_init,
                                    's-end': gt_s_end,
                                    'video-id': video_id})
    return ground_truth_df

def get_gt_map(gt_file='actNet200-V1-3.pkl'):
    activity_index = {}
    with open(gt_file, 'r') as fobj:
        gt = pickle.load(fobj)['actionIDs']
    for name, anno in gt.items():
        activity_index[anno['class']] = name
    return activity_index

def get_actnet_baseline(top=np.inf, SUBSET='validation', PROPOSALS_FILENAME='activitynet_v1-3_proposals.hdf5'):
    gt_file = 'actNet200-V1-3.pkl'
    with open(gt_file, 'r') as fobj:
        data = pickle.load(fobj)['database']
    intended_videos = []
    for vid, vitem in data.items():
        if vitem['subset'] == SUBSET:
            intended_videos.append('v_{}'.format(vid))

    # Reading proposals from HDF5 file.
    s_init, s_end, score, video_id = [], [], [], []
    fobj = h5py.File(PROPOSALS_FILENAME, 'r')
    for vid in list(fobj.keys()):
        if vid not in intended_videos:
            continue

        starts = fobj[vid]['segment-init'].value.tolist()
        ends = fobj[vid]['segment-end'].value.tolist()
        scs = fobj[vid]['score'].value.tolist()

        s_init.extend(starts[:min(top,len(scs))])
        s_end.extend(ends[:min(top,len(scs))])
        score.extend(scs[:min(top,len(scs))])
        video_id.extend(np.repeat(vid, min(top,len(scs))).tolist())

    fobj.close()
    proposals_df = pd.DataFrame({'s-init': s_init,
                                 's-end': s_end,
                                 'score': score,
                                 'video-id': video_id})
    print('Average number of proposals: {}'.format(proposals_df.shape[0] / float(len(intended_videos))))
    return proposals_df

class AverageMeter(object):
    """Computes and stores the average and current value"""

    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

def stats(label, pred, hist=False):
    # 统计函数
    printout = ''
    if hist==True:
        y = label.squeeze()
        ind = np.nonzero(y > 0)
        neg = np.nonzero(y == 0)
        _, ct = np.unique(ind[y.ndim-1], return_counts=True)
        printout += 'Hist ({}|{}) '.format(len(neg[y.ndim-1]), ct)

    fg = np.sum(label >= 1)
    bg = np.sum(label == 0)
    total = fg+bg
    corr = np.sum(label == pred)
    tfg = np.sum(np.logical_and(label>=1,pred>=1))
    tbg = np.sum(np.logical_and(label==0,pred==0))
    ffg = np.sum(np.logical_and(label==0,pred>=1))
    fbg = np.sum(np.logical_and(label>=1,pred==0))
    printout += 'Acc: {:.4} \n'.format(float(corr)/(fg+bg))
    printout += 'TF {:.2} FF {:.2} \nFB {:.2} TB {:.2} \n'.format(float(tfg)/total, float(ffg)/total, float(fbg)/total, float(tbg)/total, )
    return printout

def accuracy(output, target, topk=(1,)):
    """Computes the precision@k for the specified values of k"""
    maxk = max(topk)
    batch_size = target.size(0)

    _, pred = output.topk(maxk, 1, True, True)
    pred = pred.t()
    correct = pred.eq(target.view(1, -1).expand_as(pred))

    res = []
    for k in topk:
        correct_k = correct[:k].view(-1).float().sum(0)
        res.append(correct_k.mul_(100.0 / batch_size))
    return res