From d36e82f63229658f079a63661987ecb8fa37e809 Mon Sep 17 00:00:00 2001
From: coocoo90 <496792708@qq.com>
Date: Mon, 29 Aug 2022 00:18:17 +0800
Subject: [PATCH 1/5] edit pretrain and eval path
---
.idea/.gitignore | 3 +++
.idea/inspectionProfiles/profiles_settings.xml | 6 ++++++
.idea/misc.xml | 4 ++++
.idea/modules.xml | 8 ++++++++
.idea/tubelet-transformer.iml | 12 ++++++++++++
.idea/vcs.xml | 6 ++++++
configuration/TubeR_CSN152_AVA21.yaml | 3 ++-
configuration/TubeR_CSN152_AVA22.yaml | 3 ++-
configuration/TubeR_CSN50_AVA21.yaml | 3 ++-
configuration/Tuber_CSN152_JHMDB.yaml | 2 +-
evaluates/evaluate_ava.py | 4 ++--
utils/model_utils.py | 5 +++--
utils/video_action_recognition.py | 12 +-----------
13 files changed, 52 insertions(+), 19 deletions(-)
create mode 100644 .idea/.gitignore
create mode 100644 .idea/inspectionProfiles/profiles_settings.xml
create mode 100644 .idea/misc.xml
create mode 100644 .idea/modules.xml
create mode 100644 .idea/tubelet-transformer.iml
create mode 100644 .idea/vcs.xml
diff --git a/.idea/.gitignore b/.idea/.gitignore
new file mode 100644
index 0000000..26d3352
--- /dev/null
+++ b/.idea/.gitignore
@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml
diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml
new file mode 100644
index 0000000..105ce2d
--- /dev/null
+++ b/.idea/inspectionProfiles/profiles_settings.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
new file mode 100644
index 0000000..d1e22ec
--- /dev/null
+++ b/.idea/misc.xml
@@ -0,0 +1,4 @@
+
+
+
+
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
new file mode 100644
index 0000000..0dcd1a3
--- /dev/null
+++ b/.idea/modules.xml
@@ -0,0 +1,8 @@
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/tubelet-transformer.iml b/.idea/tubelet-transformer.iml
new file mode 100644
index 0000000..8b8c395
--- /dev/null
+++ b/.idea/tubelet-transformer.iml
@@ -0,0 +1,12 @@
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 0000000..94a25f7
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/configuration/TubeR_CSN152_AVA21.yaml b/configuration/TubeR_CSN152_AVA21.yaml
index 5c44df3..060d7f2 100644
--- a/configuration/TubeR_CSN152_AVA21.yaml
+++ b/configuration/TubeR_CSN152_AVA21.yaml
@@ -42,6 +42,7 @@ CONFIG:
LABEL_PATH: '/xxx/datasets/ava_action_list_v2.1_for_activitynet_2018.pbtxt'
ANNO_PATH: '/xxx/datasets/ava_{}_v21.json'
DATA_PATH: '/xxx/ava/frames/{}/'
+ EXCLUDE_PATH: '/xxx/datasets/ava_val_excluded_timestamps_v2.1.csv'
NUM_CLASSES: 80
MULTIGRID: False
IMG_SIZE: 256
@@ -68,7 +69,7 @@ CONFIG:
DS_RATE: 8
TEMP_LEN: 32
SAMPLE_RATE: 2
- PRETRAINED: False
+ PRETRAINED: True
PRETRAIN_BACKBONE_DIR: "/xxx/irCSN_152_ft_kinetics_from_ig65m_f126851907.mat"
PRETRAIN_TRANSFORMER_DIR: "/xxx/detr.pth"
PRETRAINED_PATH: "/xxx/ADTR_CSN_152_ava_21.pth"
diff --git a/configuration/TubeR_CSN152_AVA22.yaml b/configuration/TubeR_CSN152_AVA22.yaml
index 9ae6f33..37df491 100644
--- a/configuration/TubeR_CSN152_AVA22.yaml
+++ b/configuration/TubeR_CSN152_AVA22.yaml
@@ -42,6 +42,7 @@ CONFIG:
LABEL_PATH: '/xxx/datasets/ava_action_list_v2.1_for_activitynet_2018.pbtxt'
ANNO_PATH: '/xxx/datasets/ava_{}_v22.json'
DATA_PATH: '/xxx/ava/frames/{}/'
+ EXCLUDE_PATH: '/xxx/datasets/ava_val_excluded_timestamps_v2.1.csv'
NUM_CLASSES: 80
MULTIGRID: False
IMG_SIZE: 256
@@ -68,7 +69,7 @@ CONFIG:
DS_RATE: 8
TEMP_LEN: 32
SAMPLE_RATE: 2
- PRETRAINED: False
+ PRETRAINED: True
PRETRAIN_BACKBONE_DIR: "/xxx/irCSN_152_ft_kinetics_from_ig65m_f126851907.mat"
PRETRAIN_TRANSFORMER_DIR: "/xxx/detr.pth"
PRETRAINED_PATH: "/xxx/ADTR_CSN_152_decode_ava_22.pth"
diff --git a/configuration/TubeR_CSN50_AVA21.yaml b/configuration/TubeR_CSN50_AVA21.yaml
index 0dda70c..467b197 100644
--- a/configuration/TubeR_CSN50_AVA21.yaml
+++ b/configuration/TubeR_CSN50_AVA21.yaml
@@ -42,6 +42,7 @@ CONFIG:
LABEL_PATH: '/xxx/datasets/ava_action_list_v2.1_for_activitynet_2018.pbtxt'
ANNO_PATH: '/xxx/datasets/ava_{}_v21.json'
DATA_PATH: '/xxx/ava/frames/{}/'
+ EXCLUDE_PATH: '/xxx/datasets/ava_val_excluded_timestamps_v2.1.csv'
NUM_CLASSES: 80
MULTIGRID: False
IMG_SIZE: 256
@@ -68,7 +69,7 @@ CONFIG:
DS_RATE: 8
TEMP_LEN: 32
SAMPLE_RATE: 2
- PRETRAINED: False
+ PRETRAINED: True
PRETRAIN_BACKBONE_DIR: "/xxx/irCSN_152_ft_kinetics_from_ig65m_f126851907.mat"
PRETRAIN_TRANSFORMER_DIR: "/xxx/detr.pth"
PRETRAINED_PATH: "/xxx/ADTR_CSN_50_decode_ava_21.pth"
diff --git a/configuration/Tuber_CSN152_JHMDB.yaml b/configuration/Tuber_CSN152_JHMDB.yaml
index 7c5fb54..c3cbb43 100644
--- a/configuration/Tuber_CSN152_JHMDB.yaml
+++ b/configuration/Tuber_CSN152_JHMDB.yaml
@@ -69,7 +69,7 @@ CONFIG:
DS_RATE: 8
TEMP_LEN: 32
SAMPLE_RATE: 2
- PRETRAINED: False
+ PRETRAINED: True
PRETRAIN_BACKBONE_DIR: "xxx"
PRETRAIN_TRANSFORMER_DIR: "xxx"
PRETRAINED_PATH: "/xxx/ADTR_CSN_152_jhmdb.pth"
diff --git a/evaluates/evaluate_ava.py b/evaluates/evaluate_ava.py
index dbb7849..075b689 100644
--- a/evaluates/evaluate_ava.py
+++ b/evaluates/evaluate_ava.py
@@ -27,11 +27,11 @@ class STDetectionEvaluater(object):
evaluate(): run evaluation code
'''
- def __init__(self, label_path, tiou_thresholds=[0.5], load_from_dataset=False, class_num=60):
+ def __init__(self, label_path, exclude_path=None, tiou_thresholds=[0.5], load_from_dataset=False, class_num=60):
self.label_path = label_path
categories, class_whitelist = read_labelmap(self.label_path)
self.class_num = class_num
- if class_num == 80:
+ if class_num == 80 and exclude_path is not None:
self.exclude_keys = []
f = open("/xxx/datasets/ava_val_excluded_timestamps_v2.1.csv")
while True:
diff --git a/utils/model_utils.py b/utils/model_utils.py
index 6887343..5fddcf1 100644
--- a/utils/model_utils.py
+++ b/utils/model_utils.py
@@ -57,8 +57,9 @@ def deploy_model(model, cfg, is_tuber=True):
# DataParallel will divide and allocate batch_size to all available GPUs
model = torch.nn.DataParallel(model).cuda()
- print("loading detr")
- load_detr_weights(model, cfg.CONFIG.MODEL.PRETRAIN_TRANSFORMER_DIR, cfg)
+ if cfg.CONFIG.MODEL.PRETRAINED:
+ print("loading detr")
+ load_detr_weights(model, cfg.CONFIG.MODEL.PRETRAIN_TRANSFORMER_DIR, cfg)
return model
diff --git a/utils/video_action_recognition.py b/utils/video_action_recognition.py
index 065f12a..20b53d0 100644
--- a/utils/video_action_recognition.py
+++ b/utils/video_action_recognition.py
@@ -426,7 +426,7 @@ def validate_tuber_detection(cfg, model, criterion, postprocessors, data_loader,
# aggregate files
if cfg.DDP_CONFIG.GPU_WORLD_RANK == 0:
# read results
- evaluater = STDetectionEvaluater(cfg.CONFIG.DATA.LABEL_PATH, class_num=cfg.CONFIG.DATA.NUM_CLASSES)
+ evaluater = STDetectionEvaluater(cfg.CONFIG.DATA.LABEL_PATH, class_num=cfg.CONFIG.DATA.NUM_CLASSES, exclude_path=cfg.CONFIG.DATA.EXCLUDE_PATH)
file_path_lst = [tmp_GT_path.format(cfg.CONFIG.LOG.BASE_PATH, cfg.CONFIG.LOG.RES_DIR, x) for x in range(cfg.DDP_CONFIG.GPU_WORLD_SIZE)]
evaluater.load_GT_from_path(file_path_lst)
file_path_lst = [tmp_path.format(cfg.CONFIG.LOG.BASE_PATH, cfg.CONFIG.LOG.RES_DIR, x) for x in range(cfg.DDP_CONFIG.GPU_WORLD_SIZE)]
@@ -675,15 +675,5 @@ def validate_tuber_ucf_detection(cfg, model, criterion, postprocessors, data_loa
writer.add_scalar('val/val_mAP_epoch', mAP[0], epoch)
Map_ = mAP[0]
- # evaluater = STDetectionEvaluaterSinglePerson(cfg.CONFIG.DATA.LABEL_PATH)
- # file_path_lst = [tmp_GT_path.format(cfg.CONFIG.LOG.BASE_PATH, cfg.CONFIG.LOG.RES_DIR, x) for x in range(cfg.DDP_CONFIG.GPU_WORLD_SIZE)]
- # evaluater.load_GT_from_path(file_path_lst)
- # file_path_lst = [tmp_path.format(cfg.CONFIG.LOG.BASE_PATH, cfg.CONFIG.LOG.RES_DIR, x) for x in range(cfg.DDP_CONFIG.GPU_WORLD_SIZE)]
- # evaluater.load_detection_from_path(file_path_lst)
- # mAP, metrics = evaluater.evaluate()
- # print(metrics)
- # print_string = 'person AP: {mAP:.5f}'.format(mAP=mAP[0])
- # print(print_string)
- # writer.add_scalar('val/val_person_AP_epoch', mAP[0], epoch)
torch.distributed.barrier()
return Map_
From 3f12332ec8de6fecfba794e3622c9f61a610c033 Mon Sep 17 00:00:00 2001
From: coocoo90 <496792708@qq.com>
Date: Mon, 29 Aug 2022 00:20:27 +0800
Subject: [PATCH 2/5] remove idea
---
.idea/.gitignore | 3 ---
.idea/inspectionProfiles/profiles_settings.xml | 6 ------
.idea/misc.xml | 4 ----
.idea/modules.xml | 8 --------
.idea/tubelet-transformer.iml | 12 ------------
.idea/vcs.xml | 6 ------
6 files changed, 39 deletions(-)
delete mode 100644 .idea/.gitignore
delete mode 100644 .idea/inspectionProfiles/profiles_settings.xml
delete mode 100644 .idea/misc.xml
delete mode 100644 .idea/modules.xml
delete mode 100644 .idea/tubelet-transformer.iml
delete mode 100644 .idea/vcs.xml
diff --git a/.idea/.gitignore b/.idea/.gitignore
deleted file mode 100644
index 26d3352..0000000
--- a/.idea/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-# Default ignored files
-/shelf/
-/workspace.xml
diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml
deleted file mode 100644
index 105ce2d..0000000
--- a/.idea/inspectionProfiles/profiles_settings.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-
-
-
-
-
-
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
deleted file mode 100644
index d1e22ec..0000000
--- a/.idea/misc.xml
+++ /dev/null
@@ -1,4 +0,0 @@
-
-
-
-
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
deleted file mode 100644
index 0dcd1a3..0000000
--- a/.idea/modules.xml
+++ /dev/null
@@ -1,8 +0,0 @@
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/.idea/tubelet-transformer.iml b/.idea/tubelet-transformer.iml
deleted file mode 100644
index 8b8c395..0000000
--- a/.idea/tubelet-transformer.iml
+++ /dev/null
@@ -1,12 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
deleted file mode 100644
index 94a25f7..0000000
--- a/.idea/vcs.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-
-
-
-
-
-
\ No newline at end of file
From 44e64a1bdf4be9f7127cb83f302b0c2c868711c8 Mon Sep 17 00:00:00 2001
From: coocoo90 <496792708@qq.com>
Date: Thu, 1 Sep 2022 23:46:46 +0800
Subject: [PATCH 3/5] fix jhmdb load
---
configuration/Tuber_CSN152_JHMDB.yaml | 2 +-
utils/model_utils.py | 3 ++-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/configuration/Tuber_CSN152_JHMDB.yaml b/configuration/Tuber_CSN152_JHMDB.yaml
index c3cbb43..dba2d52 100644
--- a/configuration/Tuber_CSN152_JHMDB.yaml
+++ b/configuration/Tuber_CSN152_JHMDB.yaml
@@ -51,7 +51,7 @@ CONFIG:
FRAME_RATE: 2
MODEL:
- SINGLE_FRAME: True
+ SINGLE_FRAME: False
BACKBONE_NAME: CSN-152
TEMPORAL_DS_STRATEGY: decoder
LAST_STRIDE: False
diff --git a/utils/model_utils.py b/utils/model_utils.py
index 5fddcf1..5d1eb8d 100644
--- a/utils/model_utils.py
+++ b/utils/model_utils.py
@@ -20,9 +20,10 @@ def load_detr_weights(model, pretrain_dir, cfg):
elif k.split('.')[1] == 'query_embed':
if not cfg.CONFIG.MODEL.SINGLE_FRAME:
query_size = cfg.CONFIG.MODEL.QUERY_NUM * (cfg.CONFIG.MODEL.TEMP_LEN // cfg.CONFIG.MODEL.DS_RATE)
+ pretrained_dict.update({k: v[:query_size].repeat(cfg.CONFIG.MODEL.DS_RATE, 1)})
else:
query_size = cfg.CONFIG.MODEL.QUERY_NUM
- pretrained_dict.update({k: v[:query_size]})
+ pretrained_dict.update({k: v[:query_size]})
pretrained_dict_ = {k: v for k, v in pretrained_dict.items() if k in model_dict}
unused_dict = {k: v for k, v in pretrained_dict.items() if not k in model_dict}
From d5cfa4fd588a024e0aac7411e1ee7cea8be8ade5 Mon Sep 17 00:00:00 2001
From: coocoo90 <496792708@qq.com>
Date: Fri, 2 Sep 2022 00:59:38 +0800
Subject: [PATCH 4/5] fix bug
---
train_tuber_jhmdb.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/train_tuber_jhmdb.py b/train_tuber_jhmdb.py
index a8439ee..dde8109 100644
--- a/train_tuber_jhmdb.py
+++ b/train_tuber_jhmdb.py
@@ -78,7 +78,6 @@ def main_worker(cfg):
for epoch in range(cfg.CONFIG.TRAIN.START_EPOCH, cfg.CONFIG.TRAIN.EPOCH_NUM):
if cfg.DDP_CONFIG.DISTRIBUTED:
train_sampler.set_epoch(epoch)
- time.sleep(1000)
train_tuber_detection(cfg, model, criterion, train_loader, optimizer, epoch, cfg.CONFIG.LOSS_COFS.CLIPS_MAX_NORM, lr_scheduler, writer)
validate_tuber_ucf_detection(cfg, model, criterion, postprocessors, val_loader, 0, writer)
From 702f714b8209e3f0e4dbf816168e775f41a77ca0 Mon Sep 17 00:00:00 2001
From: coocoo90 <496792708@qq.com>
Date: Tue, 6 Sep 2022 23:20:33 +0800
Subject: [PATCH 5/5] modify sampling
---
datasets/jhmdb_frame.py | 6 +++++-
utils/video_action_recognition.py | 2 +-
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/datasets/jhmdb_frame.py b/datasets/jhmdb_frame.py
index 8677df9..305f4ff 100644
--- a/datasets/jhmdb_frame.py
+++ b/datasets/jhmdb_frame.py
@@ -13,6 +13,7 @@
import datasets.video_transforms as T
from utils.misc import collate_fn
from glob import glob
+import random
# Assisting function for finding a good/bad tubelet
@@ -98,7 +99,10 @@ def check_video(self, vid):
def __getitem__(self, index):
sample_id, frame_id = self.index_to_sample_t[index]
- p_t = self.clip_len // 2
+ if self.mode == 'train':
+ p_t = random.randint(1, self.clip_len - 2)
+ else:
+ p_t = self.clip_len // 2
target = self.load_annotation(sample_id, frame_id, index, p_t)
imgs = self.loadvideo(frame_id, sample_id, target, p_t)
diff --git a/utils/video_action_recognition.py b/utils/video_action_recognition.py
index 20b53d0..7fb4af0 100644
--- a/utils/video_action_recognition.py
+++ b/utils/video_action_recognition.py
@@ -562,7 +562,7 @@ def validate_tuber_ucf_detection(cfg, model, criterion, postprocessors, data_loa
buff_binary.append(output_b[..., 0])
val_label = targets[bidx]["labels"]
- val_category = torch.full((len(val_label), 21), 0)
+ val_category = torch.full((len(val_label), cfg.CONFIG.DATA.NUM_CLASSES), 0)
for vl in range(len(val_label)):
label = int(val_label[vl])
val_category[vl, label] = 1