From d36e82f63229658f079a63661987ecb8fa37e809 Mon Sep 17 00:00:00 2001 From: coocoo90 <496792708@qq.com> Date: Mon, 29 Aug 2022 00:18:17 +0800 Subject: [PATCH 1/5] edit pretrain and eval path --- .idea/.gitignore | 3 +++ .idea/inspectionProfiles/profiles_settings.xml | 6 ++++++ .idea/misc.xml | 4 ++++ .idea/modules.xml | 8 ++++++++ .idea/tubelet-transformer.iml | 12 ++++++++++++ .idea/vcs.xml | 6 ++++++ configuration/TubeR_CSN152_AVA21.yaml | 3 ++- configuration/TubeR_CSN152_AVA22.yaml | 3 ++- configuration/TubeR_CSN50_AVA21.yaml | 3 ++- configuration/Tuber_CSN152_JHMDB.yaml | 2 +- evaluates/evaluate_ava.py | 4 ++-- utils/model_utils.py | 5 +++-- utils/video_action_recognition.py | 12 +----------- 13 files changed, 52 insertions(+), 19 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/tubelet-transformer.iml create mode 100644 .idea/vcs.xml diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..26d3352 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..d1e22ec --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..0dcd1a3 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/tubelet-transformer.iml b/.idea/tubelet-transformer.iml new file mode 100644 index 0000000..8b8c395 --- /dev/null +++ b/.idea/tubelet-transformer.iml @@ -0,0 +1,12 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/configuration/TubeR_CSN152_AVA21.yaml b/configuration/TubeR_CSN152_AVA21.yaml index 5c44df3..060d7f2 100644 --- a/configuration/TubeR_CSN152_AVA21.yaml +++ b/configuration/TubeR_CSN152_AVA21.yaml @@ -42,6 +42,7 @@ CONFIG: LABEL_PATH: '/xxx/datasets/ava_action_list_v2.1_for_activitynet_2018.pbtxt' ANNO_PATH: '/xxx/datasets/ava_{}_v21.json' DATA_PATH: '/xxx/ava/frames/{}/' + EXCLUDE_PATH: '/xxx/datasets/ava_val_excluded_timestamps_v2.1.csv' NUM_CLASSES: 80 MULTIGRID: False IMG_SIZE: 256 @@ -68,7 +69,7 @@ CONFIG: DS_RATE: 8 TEMP_LEN: 32 SAMPLE_RATE: 2 - PRETRAINED: False + PRETRAINED: True PRETRAIN_BACKBONE_DIR: "/xxx/irCSN_152_ft_kinetics_from_ig65m_f126851907.mat" PRETRAIN_TRANSFORMER_DIR: "/xxx/detr.pth" PRETRAINED_PATH: "/xxx/ADTR_CSN_152_ava_21.pth" diff --git a/configuration/TubeR_CSN152_AVA22.yaml b/configuration/TubeR_CSN152_AVA22.yaml index 9ae6f33..37df491 100644 --- a/configuration/TubeR_CSN152_AVA22.yaml +++ b/configuration/TubeR_CSN152_AVA22.yaml @@ -42,6 +42,7 @@ CONFIG: LABEL_PATH: '/xxx/datasets/ava_action_list_v2.1_for_activitynet_2018.pbtxt' ANNO_PATH: '/xxx/datasets/ava_{}_v22.json' DATA_PATH: '/xxx/ava/frames/{}/' + EXCLUDE_PATH: '/xxx/datasets/ava_val_excluded_timestamps_v2.1.csv' NUM_CLASSES: 80 MULTIGRID: False IMG_SIZE: 256 @@ -68,7 +69,7 @@ CONFIG: DS_RATE: 8 TEMP_LEN: 32 SAMPLE_RATE: 2 - PRETRAINED: False + PRETRAINED: True PRETRAIN_BACKBONE_DIR: "/xxx/irCSN_152_ft_kinetics_from_ig65m_f126851907.mat" PRETRAIN_TRANSFORMER_DIR: "/xxx/detr.pth" PRETRAINED_PATH: "/xxx/ADTR_CSN_152_decode_ava_22.pth" diff --git a/configuration/TubeR_CSN50_AVA21.yaml b/configuration/TubeR_CSN50_AVA21.yaml index 0dda70c..467b197 100644 --- a/configuration/TubeR_CSN50_AVA21.yaml +++ b/configuration/TubeR_CSN50_AVA21.yaml @@ -42,6 +42,7 @@ CONFIG: LABEL_PATH: '/xxx/datasets/ava_action_list_v2.1_for_activitynet_2018.pbtxt' ANNO_PATH: '/xxx/datasets/ava_{}_v21.json' DATA_PATH: '/xxx/ava/frames/{}/' + EXCLUDE_PATH: '/xxx/datasets/ava_val_excluded_timestamps_v2.1.csv' NUM_CLASSES: 80 MULTIGRID: False IMG_SIZE: 256 @@ -68,7 +69,7 @@ CONFIG: DS_RATE: 8 TEMP_LEN: 32 SAMPLE_RATE: 2 - PRETRAINED: False + PRETRAINED: True PRETRAIN_BACKBONE_DIR: "/xxx/irCSN_152_ft_kinetics_from_ig65m_f126851907.mat" PRETRAIN_TRANSFORMER_DIR: "/xxx/detr.pth" PRETRAINED_PATH: "/xxx/ADTR_CSN_50_decode_ava_21.pth" diff --git a/configuration/Tuber_CSN152_JHMDB.yaml b/configuration/Tuber_CSN152_JHMDB.yaml index 7c5fb54..c3cbb43 100644 --- a/configuration/Tuber_CSN152_JHMDB.yaml +++ b/configuration/Tuber_CSN152_JHMDB.yaml @@ -69,7 +69,7 @@ CONFIG: DS_RATE: 8 TEMP_LEN: 32 SAMPLE_RATE: 2 - PRETRAINED: False + PRETRAINED: True PRETRAIN_BACKBONE_DIR: "xxx" PRETRAIN_TRANSFORMER_DIR: "xxx" PRETRAINED_PATH: "/xxx/ADTR_CSN_152_jhmdb.pth" diff --git a/evaluates/evaluate_ava.py b/evaluates/evaluate_ava.py index dbb7849..075b689 100644 --- a/evaluates/evaluate_ava.py +++ b/evaluates/evaluate_ava.py @@ -27,11 +27,11 @@ class STDetectionEvaluater(object): evaluate(): run evaluation code ''' - def __init__(self, label_path, tiou_thresholds=[0.5], load_from_dataset=False, class_num=60): + def __init__(self, label_path, exclude_path=None, tiou_thresholds=[0.5], load_from_dataset=False, class_num=60): self.label_path = label_path categories, class_whitelist = read_labelmap(self.label_path) self.class_num = class_num - if class_num == 80: + if class_num == 80 and exclude_path is not None: self.exclude_keys = [] f = open("/xxx/datasets/ava_val_excluded_timestamps_v2.1.csv") while True: diff --git a/utils/model_utils.py b/utils/model_utils.py index 6887343..5fddcf1 100644 --- a/utils/model_utils.py +++ b/utils/model_utils.py @@ -57,8 +57,9 @@ def deploy_model(model, cfg, is_tuber=True): # DataParallel will divide and allocate batch_size to all available GPUs model = torch.nn.DataParallel(model).cuda() - print("loading detr") - load_detr_weights(model, cfg.CONFIG.MODEL.PRETRAIN_TRANSFORMER_DIR, cfg) + if cfg.CONFIG.MODEL.PRETRAINED: + print("loading detr") + load_detr_weights(model, cfg.CONFIG.MODEL.PRETRAIN_TRANSFORMER_DIR, cfg) return model diff --git a/utils/video_action_recognition.py b/utils/video_action_recognition.py index 065f12a..20b53d0 100644 --- a/utils/video_action_recognition.py +++ b/utils/video_action_recognition.py @@ -426,7 +426,7 @@ def validate_tuber_detection(cfg, model, criterion, postprocessors, data_loader, # aggregate files if cfg.DDP_CONFIG.GPU_WORLD_RANK == 0: # read results - evaluater = STDetectionEvaluater(cfg.CONFIG.DATA.LABEL_PATH, class_num=cfg.CONFIG.DATA.NUM_CLASSES) + evaluater = STDetectionEvaluater(cfg.CONFIG.DATA.LABEL_PATH, class_num=cfg.CONFIG.DATA.NUM_CLASSES, exclude_path=cfg.CONFIG.DATA.EXCLUDE_PATH) file_path_lst = [tmp_GT_path.format(cfg.CONFIG.LOG.BASE_PATH, cfg.CONFIG.LOG.RES_DIR, x) for x in range(cfg.DDP_CONFIG.GPU_WORLD_SIZE)] evaluater.load_GT_from_path(file_path_lst) file_path_lst = [tmp_path.format(cfg.CONFIG.LOG.BASE_PATH, cfg.CONFIG.LOG.RES_DIR, x) for x in range(cfg.DDP_CONFIG.GPU_WORLD_SIZE)] @@ -675,15 +675,5 @@ def validate_tuber_ucf_detection(cfg, model, criterion, postprocessors, data_loa writer.add_scalar('val/val_mAP_epoch', mAP[0], epoch) Map_ = mAP[0] - # evaluater = STDetectionEvaluaterSinglePerson(cfg.CONFIG.DATA.LABEL_PATH) - # file_path_lst = [tmp_GT_path.format(cfg.CONFIG.LOG.BASE_PATH, cfg.CONFIG.LOG.RES_DIR, x) for x in range(cfg.DDP_CONFIG.GPU_WORLD_SIZE)] - # evaluater.load_GT_from_path(file_path_lst) - # file_path_lst = [tmp_path.format(cfg.CONFIG.LOG.BASE_PATH, cfg.CONFIG.LOG.RES_DIR, x) for x in range(cfg.DDP_CONFIG.GPU_WORLD_SIZE)] - # evaluater.load_detection_from_path(file_path_lst) - # mAP, metrics = evaluater.evaluate() - # print(metrics) - # print_string = 'person AP: {mAP:.5f}'.format(mAP=mAP[0]) - # print(print_string) - # writer.add_scalar('val/val_person_AP_epoch', mAP[0], epoch) torch.distributed.barrier() return Map_ From 3f12332ec8de6fecfba794e3622c9f61a610c033 Mon Sep 17 00:00:00 2001 From: coocoo90 <496792708@qq.com> Date: Mon, 29 Aug 2022 00:20:27 +0800 Subject: [PATCH 2/5] remove idea --- .idea/.gitignore | 3 --- .idea/inspectionProfiles/profiles_settings.xml | 6 ------ .idea/misc.xml | 4 ---- .idea/modules.xml | 8 -------- .idea/tubelet-transformer.iml | 12 ------------ .idea/vcs.xml | 6 ------ 6 files changed, 39 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/inspectionProfiles/profiles_settings.xml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/tubelet-transformer.iml delete mode 100644 .idea/vcs.xml diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 26d3352..0000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 105ce2d..0000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index d1e22ec..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 0dcd1a3..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/tubelet-transformer.iml b/.idea/tubelet-transformer.iml deleted file mode 100644 index 8b8c395..0000000 --- a/.idea/tubelet-transformer.iml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file From 44e64a1bdf4be9f7127cb83f302b0c2c868711c8 Mon Sep 17 00:00:00 2001 From: coocoo90 <496792708@qq.com> Date: Thu, 1 Sep 2022 23:46:46 +0800 Subject: [PATCH 3/5] fix jhmdb load --- configuration/Tuber_CSN152_JHMDB.yaml | 2 +- utils/model_utils.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/configuration/Tuber_CSN152_JHMDB.yaml b/configuration/Tuber_CSN152_JHMDB.yaml index c3cbb43..dba2d52 100644 --- a/configuration/Tuber_CSN152_JHMDB.yaml +++ b/configuration/Tuber_CSN152_JHMDB.yaml @@ -51,7 +51,7 @@ CONFIG: FRAME_RATE: 2 MODEL: - SINGLE_FRAME: True + SINGLE_FRAME: False BACKBONE_NAME: CSN-152 TEMPORAL_DS_STRATEGY: decoder LAST_STRIDE: False diff --git a/utils/model_utils.py b/utils/model_utils.py index 5fddcf1..5d1eb8d 100644 --- a/utils/model_utils.py +++ b/utils/model_utils.py @@ -20,9 +20,10 @@ def load_detr_weights(model, pretrain_dir, cfg): elif k.split('.')[1] == 'query_embed': if not cfg.CONFIG.MODEL.SINGLE_FRAME: query_size = cfg.CONFIG.MODEL.QUERY_NUM * (cfg.CONFIG.MODEL.TEMP_LEN // cfg.CONFIG.MODEL.DS_RATE) + pretrained_dict.update({k: v[:query_size].repeat(cfg.CONFIG.MODEL.DS_RATE, 1)}) else: query_size = cfg.CONFIG.MODEL.QUERY_NUM - pretrained_dict.update({k: v[:query_size]}) + pretrained_dict.update({k: v[:query_size]}) pretrained_dict_ = {k: v for k, v in pretrained_dict.items() if k in model_dict} unused_dict = {k: v for k, v in pretrained_dict.items() if not k in model_dict} From d5cfa4fd588a024e0aac7411e1ee7cea8be8ade5 Mon Sep 17 00:00:00 2001 From: coocoo90 <496792708@qq.com> Date: Fri, 2 Sep 2022 00:59:38 +0800 Subject: [PATCH 4/5] fix bug --- train_tuber_jhmdb.py | 1 - 1 file changed, 1 deletion(-) diff --git a/train_tuber_jhmdb.py b/train_tuber_jhmdb.py index a8439ee..dde8109 100644 --- a/train_tuber_jhmdb.py +++ b/train_tuber_jhmdb.py @@ -78,7 +78,6 @@ def main_worker(cfg): for epoch in range(cfg.CONFIG.TRAIN.START_EPOCH, cfg.CONFIG.TRAIN.EPOCH_NUM): if cfg.DDP_CONFIG.DISTRIBUTED: train_sampler.set_epoch(epoch) - time.sleep(1000) train_tuber_detection(cfg, model, criterion, train_loader, optimizer, epoch, cfg.CONFIG.LOSS_COFS.CLIPS_MAX_NORM, lr_scheduler, writer) validate_tuber_ucf_detection(cfg, model, criterion, postprocessors, val_loader, 0, writer) From 702f714b8209e3f0e4dbf816168e775f41a77ca0 Mon Sep 17 00:00:00 2001 From: coocoo90 <496792708@qq.com> Date: Tue, 6 Sep 2022 23:20:33 +0800 Subject: [PATCH 5/5] modify sampling --- datasets/jhmdb_frame.py | 6 +++++- utils/video_action_recognition.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/datasets/jhmdb_frame.py b/datasets/jhmdb_frame.py index 8677df9..305f4ff 100644 --- a/datasets/jhmdb_frame.py +++ b/datasets/jhmdb_frame.py @@ -13,6 +13,7 @@ import datasets.video_transforms as T from utils.misc import collate_fn from glob import glob +import random # Assisting function for finding a good/bad tubelet @@ -98,7 +99,10 @@ def check_video(self, vid): def __getitem__(self, index): sample_id, frame_id = self.index_to_sample_t[index] - p_t = self.clip_len // 2 + if self.mode == 'train': + p_t = random.randint(1, self.clip_len - 2) + else: + p_t = self.clip_len // 2 target = self.load_annotation(sample_id, frame_id, index, p_t) imgs = self.loadvideo(frame_id, sample_id, target, p_t) diff --git a/utils/video_action_recognition.py b/utils/video_action_recognition.py index 20b53d0..7fb4af0 100644 --- a/utils/video_action_recognition.py +++ b/utils/video_action_recognition.py @@ -562,7 +562,7 @@ def validate_tuber_ucf_detection(cfg, model, criterion, postprocessors, data_loa buff_binary.append(output_b[..., 0]) val_label = targets[bidx]["labels"] - val_category = torch.full((len(val_label), 21), 0) + val_category = torch.full((len(val_label), cfg.CONFIG.DATA.NUM_CLASSES), 0) for vl in range(len(val_label)): label = int(val_label[vl]) val_category[vl, label] = 1