done

liuhengyue · May 3, 2022 · dbd8c23 · dbd8c23
1 parent ae724df
commit dbd8c23
Show file tree

Hide file tree

Showing 18 changed files with 466 additions and 37 deletions.
diff --git a/.gitignore b/.gitignore
@@ -62,3 +62,5 @@ datasets
 # pycharm plugins
 node_modules/
 package-lock.json
+
+demo_videos
diff --git a/configs/jede/best.yaml b/configs/jede/best.yaml
@@ -0,0 +1,74 @@
+_BASE_: "../pg_rcnn/pg_rcnn_base.yaml"
+MODEL:
+#  BACKBONE:
+#    FREEZE_AT: 0
+#  RESNETS:
+#    NORM: "SyncBN"
+  WEIGHTS: "output/jede_best/train_with_all/model_final.pth"
+  ANCHOR_GENERATOR:
+    SIZES: [ [ 64 ], [ 128 ], [ 256 ], [ 512 ] ]
+  FPN:
+    IN_FEATURES: [ "res2", "res3", "res4", "res5" ]
+  RPN:
+    IN_FEATURES: [ "p3", "p4", "p5", "p6" ]
+    PRE_NMS_TOPK_TRAIN: 1000  # Per FPN level
+    POST_NMS_TOPK_TRAIN: 500
+  ROI_HEADS:
+    NAME: PGROIHeads
+    PROPOSAL_APPEND_GT: True
+    ENABLE_POSE_GUIDE: True
+    BATCH_SIZE_PER_IMAGE: 64
+  ROI_DIGIT_BOX_HEAD:
+    POOLER_RESOLUTION: 28
+  ROI_DIGIT_NECK_OUTPUT:
+    NORM: "GN"
+    MIN_OVERLAP: 0.1
+    OUTPUT_HEAD_CHANNELS: (2, 2, 2)
+    CONV_DIM: 256
+    NUM_DIGITS_CLASSIFIER_ON: 2
+  ROI_NECK_BASE_BRANCHES:
+    NORM: "GN"
+    PERSON_BRANCH:
+      NAME: "PersonROIBranch"
+      POOLER_RESOLUTION: 28
+      UP_SCALE: 1
+      DECONV_KERNEL: 1
+    KEYPOINTS_BRANCH:
+      NAME: "KptsROIBranch" # "KptsROIBranch"
+      DECONV_KERNEL: 1
+      UP_SCALE: 2
+      CONV_SPECS: [3,2,1]
+      PE: True
+  ROI_NECK_BASE:
+    BATCH_DIGIT_SIZE_PER_IMAGE: 256
+    NUM_PROPOSAL_TRAIN: 100
+DATASETS:
+  TRAIN: ("jerseynumbers_train", "keypoints_coco_2017_train")
+  TRAIN_VIDEO_IDS: [0,1,2,3,4]
+  TEST_VIDEO_IDS: [4]
+
+DATALOADER:
+  SAMPLER_TRAIN: WeightedTrainingSampler
+INPUT:
+  MAX_SIZE_TRAIN: 1589 #1333 + 256
+  MIN_SIZE_TRAIN: (640, 672, 704, 736, 768, 800, 832, 864, 896)
+#  MAX_SIZE_TEST: 1333
+#  MIN_SIZE_TEST: 800
+  MAX_SIZE_TEST: 2333
+  MIN_SIZE_TEST: 896
+  AUG:
+    HELPER_DATASET_NAME: svhn_train
+    COPY_PASTE_MIX: 10
+SOLVER:
+  IMS_PER_BATCH: 2
+  BASE_LR: 0.0002
+  STEPS: (120000,)
+  MAX_ITER: 150000
+OUTPUT_DIR: ./output/jede_best/best
+
+#python train_net.py \
+#        --num-gpus 2 \
+#        --config-file configs/jede/best_model.yaml \
+#        DATASETS.TRAIN_VIDEO_IDS [0,1,3] \
+#        DATASETS.TEST_VIDEO_IDS [2] \
+#        OUTPUT_DIR ./output/jede_best/test_2
diff --git a/configs/jede/best_train_all.yaml b/configs/jede/best_train_all.yaml
@@ -0,0 +1,68 @@
+_BASE_: "../pg_rcnn/pg_rcnn_base.yaml"
+MODEL:
+#  BACKBONE:
+#    FREEZE_AT: 0
+#  RESNETS:
+#    NORM: "SyncBN"
+  WEIGHTS: "output/pg_rcnn/aug/datasets_mix/pretrain_coco_svhn_3x/model_final.pth"
+  ANCHOR_GENERATOR:
+    SIZES: [ [ 64 ], [ 128 ], [ 256 ], [ 512 ] ]
+  FPN:
+    IN_FEATURES: [ "res2", "res3", "res4", "res5" ]
+  RPN:
+    IN_FEATURES: [ "p3", "p4", "p5", "p6" ]
+    PRE_NMS_TOPK_TRAIN: 1000  # Per FPN level
+    POST_NMS_TOPK_TRAIN: 500
+  ROI_HEADS:
+    NAME: PGROIHeads
+    PROPOSAL_APPEND_GT: True
+    ENABLE_POSE_GUIDE: True
+    BATCH_SIZE_PER_IMAGE: 64
+  ROI_DIGIT_BOX_HEAD:
+    POOLER_RESOLUTION: 28
+  ROI_DIGIT_NECK_OUTPUT:
+    NORM: "GN"
+    MIN_OVERLAP: 0.1
+    OUTPUT_HEAD_CHANNELS: (2, 2, 2)
+    CONV_DIM: 256
+    NUM_DIGITS_CLASSIFIER_ON: 2
+  ROI_NECK_BASE_BRANCHES:
+    NORM: "GN"
+    PERSON_BRANCH:
+      NAME: "PersonROIBranch"
+      UP_SCALE: 2
+      DECONV_KERNEL: 4
+    KEYPOINTS_BRANCH:
+      NAME: "KptsROIBranch" # "KptsROIBranch"
+      DECONV_KERNEL: 4
+      UP_SCALE: 2
+      PE: True
+  ROI_NECK_BASE:
+    BATCH_DIGIT_SIZE_PER_IMAGE: 64
+    NUM_PROPOSAL_TRAIN: 50
+DATASETS:
+  TRAIN_VIDEO_IDS: [0,1,2,3,4]
+  TEST_VIDEO_IDS: [4]
+INPUT:
+  MAX_SIZE_TRAIN: 1333
+  MIN_SIZE_TRAIN: (640, 672, 704, 736, 768, 800)
+#  MAX_SIZE_TEST: 1333
+#  MIN_SIZE_TEST: 800
+  MAX_SIZE_TEST: 2666
+  MIN_SIZE_TEST: 1600
+  AUG:
+    HELPER_DATASET_NAME: svhn_train
+    COPY_PASTE_MIX: 10
+SOLVER:
+  IMS_PER_BATCH: 2
+  BASE_LR: 0.0002
+  STEPS: (120000,)
+  MAX_ITER: 150000
+OUTPUT_DIR: ./output/jede_best/train_with_all
+
+#python train_net.py \
+#        --num-gpus 2 \
+#        --config-file configs/jede/best_model.yaml \
+#        DATASETS.TRAIN_VIDEO_IDS [0,1,3] \
+#        DATASETS.TEST_VIDEO_IDS [2] \
+#        OUTPUT_DIR ./output/jede_best/test_2
diff --git a/...nnels/test_0_parallel_gn_pe_pretrain_copypastemix_swapdigit_less_anchors_unfreeze_b8.yaml b/...nnels/test_0_parallel_gn_pe_pretrain_copypastemix_swapdigit_less_anchors_unfreeze_b8.yaml
@@ -1,9 +1,9 @@
 _BASE_: "../pg_rcnn_base.yaml"
 MODEL:
-  BACKBONE:
-    FREEZE_AT: 0
-  RESNETS:
-    NORM: "SyncBN"
+#  BACKBONE:
+#    FREEZE_AT: 0
+#  RESNETS:
+#    NORM: "SyncBN"
   WEIGHTS: "output/pg_rcnn/aug/datasets_mix/pretrain_coco_svhn_3x/model_final.pth"
   ANCHOR_GENERATOR:
     SIZES: [ [ 64 ], [ 128 ], [ 256 ], [ 512 ] ]
@@ -30,13 +30,20 @@ MODEL:
 DATASETS:
   TRAIN_VIDEO_IDS: [1,2,3]
   TEST_VIDEO_IDS: [0]
+#SOLVER:
+#  IMS_PER_BATCH: 8
+#  BASE_LR: 0.0008
+#  STEPS: (10000,)
+#  MAX_ITER: 12500
+#  WARMUP_FACTOR: 0.004
+#  WARMUP_ITERS: 250
 SOLVER:
-  IMS_PER_BATCH: 8
-  BASE_LR: 0.0008
-  STEPS: (10000,)
-  MAX_ITER: 12500
-  WARMUP_FACTOR: 0.004
-  WARMUP_ITERS: 250
+  IMS_PER_BATCH: 4
+  BASE_LR: 0.0004
+  STEPS: (20000,)
+  MAX_ITER: 25000
+  WARMUP_FACTOR: 0.002
+  WARMUP_ITERS: 500
 INPUT:
   AUG:
     HELPER_DATASET_NAME: svhn_train

diff --git a/configs/pg_rcnn/model_size/R101-FPN_test_0_gn.yaml b/configs/pg_rcnn/model_size/R101-FPN_test_0_gn.yaml
@@ -12,7 +12,7 @@ MODEL:
     NAME: ""
   ROI_DIGIT_NECK_OUTPUT:
     OUTPUT_HEAD_CHANNELS: (2, 2, 2)
-    NUM_DIGITS_CLASSIFIER_ON: 2
+    NUM_DIGITS_CLASSIFIER_ON: 1
 DATASETS:
   TRAIN_VIDEO_IDS: [1,2,3]
   TEST_VIDEO_IDS: [0]

diff --git a/configs/pg_rcnn/tests/baseline.yaml b/configs/pg_rcnn/tests/baseline.yaml
@@ -31,15 +31,15 @@ MODEL:
 DATASETS:
   TRAIN: ("jerseynumbers_train", ) # ("svhn_train", "keypoints_coco_2017_val_100") # jerseynumbers_train
   NUM_IMAGES: -1 # change the number of images to load
-  TRAIN_VIDEO_IDS: [0,1,2,3,4]
+  TRAIN_VIDEO_IDS: [0,1,2,3]
   TEST_VIDEO_IDS: [3]
-#INPUT:
+INPUT:
 #  MIN_SIZE_TRAIN: (256,)
 #  MAX_SIZE_TRAIN: 320
 #  MAX_SIZE_TEST: 320
-#  AUG:
-#    COPY_PASTE_MIX: 5
-#    HELPER_DATASET_NAME: svhn_train
+  AUG:
+    COPY_PASTE_MIX: 5
+    HELPER_DATASET_NAME: svhn_train
 SOLVER:
   IMS_PER_BATCH: 1
   BASE_LR: 0.001 # when IMS_PER_BATCH = 16, LR is 0.02

diff --git a/demo/demo.py b/demo/demo.py
@@ -36,7 +36,7 @@ def get_parser():
     parser = argparse.ArgumentParser(description="JEDE demo")
     parser.add_argument(
         "--config-file",
-        default="configs/jede/best_model.yaml",
+        default="configs/jede/best.yaml",
         metavar="FILE",
         help="path to config file",
     )
@@ -61,7 +61,7 @@ def get_parser():
     parser.add_argument(
         "--opts",
         help="Modify config options using the command-line 'KEY VALUE' pairs",
-        default=["MODEL.WEIGHTS", "output/jede_best/test_0/model_final.pth"],
+        default=["MODEL.WEIGHTS", "output/jede_best/best/model_final.pth"],
         nargs=argparse.REMAINDER,
     )
     return parser
@@ -98,9 +98,10 @@ def get_parser():
             )
 
             if args.output:
+                ext = ".pdf"
                 if os.path.isdir(args.output):
                     assert os.path.isdir(args.output), args.output
-                    out_filename = os.path.join(args.output, os.path.basename(path))
+                    out_filename = os.path.join(args.output, os.path.basename(path).split(".")[0] + ext)
                 else:
                     assert len(args.input) == 1, "Please specify a directory with args.output"
                     out_filename = args.output

diff --git a/demo/predictor.py b/demo/predictor.py
@@ -52,6 +52,7 @@ def run_on_image(self, image):
         image = image[:, :, ::-1]
         visualizer = JerseyNumberVisualizer(image,
                                             self.metadata,
+                                            scale=2.0,
                                             instance_mode=self.instance_mode,
                                             digit_only=self.digit_only)
         if "panoptic_seg" in predictions:

diff --git a/experiments/ablations/player_feature_size.sh b/experiments/ablations/player_feature_size.sh
@@ -0,0 +1,37 @@
+python train_net.py \
+        --num-gpus 2 \
+        --config-file configs/pg_rcnn/digit_twochannels/test_0_parallel_gn.yaml \
+        MODEL.ROI_NECK_BASE_BRANCHES.NORM "" \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.UP_SCALE 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.DECONV_KERNEL 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.UP_SCALE 2 \
+        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.DECONV_KERNEL 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.POOLER_RESOLUTION 28 \
+        MODEL.ROI_DIGIT_NECK_OUTPUT.NORM "" \
+        OUTPUT_DIR "./output/ablations/player_feature_size/28x28"
+
+
+#python train_net.py \
+#        --num-gpus 2 \
+#        --config-file configs/pg_rcnn/digit_twochannels/test_0_parallel_gn.yaml \
+#        MODEL.ROI_NECK_BASE_BRANCHES.NORM "" \
+#        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.UP_SCALE 1 \
+#        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.DECONV_KERNEL 1 \
+#        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.UP_SCALE 1 \
+#        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.DECONV_KERNEL 1 \
+#        MODEL.ROI_DIGIT_NECK_OUTPUT.NORM "" \
+#        OUTPUT_DIR "./output/ablations/player_feature_size/14x14"
+
+
+python train_net.py \
+        --num-gpus 2 \
+        --config-file configs/pg_rcnn/digit_twochannels/test_0_parallel_gn.yaml \
+        MODEL.ROI_NECK_BASE_BRANCHES.NORM "" \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.UP_SCALE 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.DECONV_KERNEL 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.UP_SCALE 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.DECONV_KERNEL 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.POOLER_RESOLUTION 56 \
+        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.CONV_SPECS 3,1,1 \
+        MODEL.ROI_DIGIT_NECK_OUTPUT.NORM "" \
+        OUTPUT_DIR "./output/ablations/player_feature_size/56x56"
diff --git a/experiments/jede_R_50_FPN_best.sh b/experiments/jede_R_50_FPN_best.sh
@@ -0,0 +1,78 @@
+python train_net.py \
+        --num-gpus 2 \
+        --config-file configs/pg_rcnn/digit_twochannels/test_0_parallel_gn_pe_pretrain_copypastemix_swapdigit_less_anchors_unfreeze_b8.yaml \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.UP_SCALE 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.DECONV_KERNEL 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.UP_SCALE 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.DECONV_KERNEL 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.POOLER_RESOLUTION 56 \
+        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.CONV_SPECS 3,1,1 \
+        DATASETS.TRAIN_VIDEO_IDS [1,2,3] \
+        DATASETS.TEST_VIDEO_IDS [0] \
+        OUTPUT_DIR "./output/jede_R_50_FPN_best/test_0"
+
+python train_net.py \
+        --num-gpus 2 \
+        --config-file configs/pg_rcnn/digit_twochannels/test_0_parallel_gn_pe_pretrain_copypastemix_swapdigit_less_anchors_unfreeze_b8.yaml \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.UP_SCALE 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.DECONV_KERNEL 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.UP_SCALE 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.DECONV_KERNEL 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.POOLER_RESOLUTION 56 \
+        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.CONV_SPECS 3,1,1 \
+        DATASETS.TRAIN_VIDEO_IDS [0,2,3] \
+        DATASETS.TEST_VIDEO_IDS [1] \
+        OUTPUT_DIR "./output/jede_R_50_FPN_best/test_1"
+
+python train_net.py \
+        --num-gpus 2 \
+        --config-file configs/pg_rcnn/digit_twochannels/test_0_parallel_gn_pe_pretrain_copypastemix_swapdigit_less_anchors_unfreeze_b8.yaml \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.UP_SCALE 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.DECONV_KERNEL 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.UP_SCALE 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.DECONV_KERNEL 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.POOLER_RESOLUTION 56 \
+        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.CONV_SPECS 3,1,1 \
+        DATASETS.TRAIN_VIDEO_IDS [0,1,3] \
+        DATASETS.TEST_VIDEO_IDS [2] \
+        OUTPUT_DIR "./output/jede_R_50_FPN_best/test_2"
+
+python train_net.py \
+        --num-gpus 2 \
+        --config-file configs/pg_rcnn/digit_twochannels/test_0_parallel_gn_pe_pretrain_copypastemix_swapdigit_less_anchors_unfreeze_b8.yaml \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.UP_SCALE 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.DECONV_KERNEL 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.UP_SCALE 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.DECONV_KERNEL 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.POOLER_RESOLUTION 56 \
+        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.CONV_SPECS 3,1,1 \
+        DATASETS.TRAIN_VIDEO_IDS [0,1,2] \
+        DATASETS.TEST_VIDEO_IDS [3] \
+        OUTPUT_DIR "./output/jede_R_50_FPN_best/test_3"
+
+python train_net.py \
+        --num-gpus 2 \
+        --config-file configs/pg_rcnn/digit_twochannels/test_0_parallel_gn_pe_pretrain_copypastemix_swapdigit_less_anchors_unfreeze_b8.yaml \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.UP_SCALE 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.DECONV_KERNEL 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.UP_SCALE 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.DECONV_KERNEL 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.POOLER_RESOLUTION 56 \
+        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.CONV_SPECS 3,1,1 \
+        DATASETS.TRAIN_VIDEO_IDS [0,1,2,3] \
+        DATASETS.TEST_VIDEO_IDS [4] \
+        OUTPUT_DIR "./output/jede_R_50_FPN_best/test_4"
+
+python train_net.py \
+        --num-gpus 2 \
+        --config-file configs/pg_rcnn/digit_twochannels/test_0_parallel_gn_pe_pretrain_copypastemix_swapdigit_less_anchors_unfreeze_b8.yaml \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.UP_SCALE 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.DECONV_KERNEL 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.UP_SCALE 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.DECONV_KERNEL 1 \
+        MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.POOLER_RESOLUTION 56 \
+        MODEL.ROI_NECK_BASE_BRANCHES.KEYPOINTS_BRANCH.CONV_SPECS 3,1,1 \
+        DATASETS.TRAIN_VIDEO_IDS [4] \
+        DATASETS.TEST_VIDEO_IDS [0,1,2,3] \
+        INPUT.AUG.COPY_PASTE_MIX 0 \
+        OUTPUT_DIR "./output/jede_R_50_FPN_best/test_5"
diff --git a/pgrcnn/config/config.py b/pgrcnn/config/config.py
@@ -6,7 +6,7 @@
 def get_cfg() -> CN:
     """
     Get a copy of the default config.
-    Then add extra fields.
+    Then add modeling fields.
 
     We
 
@@ -201,6 +201,7 @@ def add_poseguide_config(cfg):
 
     _C.MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH = CN()
     _C.MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.NAME = "PersonROIBranch"
+    _C.MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.POOLER_RESOLUTION = 14
     _C.MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.UP_SCALE = 1
     _C.MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.DECONV_KERNEL = 4
     _C.MODEL.ROI_NECK_BASE_BRANCHES.PERSON_BRANCH.CONV_DIMS = [64, 64]