[Refactor] merge the video dataset related args into config json and each dataset inside

Author: FangXinyu-0913

run.py

@@ -26,11 +26,15 @@ def build_model_from_config(cfg):
 
 def build_dataset_from_config(cfg):
     import vlmeval.dataset
+    import inspect
     config = cp.deepcopy(cfg)
     assert 'class' in config
     cls_name = config.pop('class')
     if hasattr(vlmeval.dataset, cls_name):
-        return getattr(vlmeval.dataset, cls_name)(**config)
+        cls = getattr(vlmeval.dataset, cls_name)
+        sig = inspect.signature(cls.__init__)
+        valid_params = {k: v for k, v in config.items() if k in sig.parameters}
+        return cls(**valid_params)
     else:
         raise ValueError(f'Class {cls_name} is not supported in `vlmeval.dataset`')
 
@@ -101,11 +105,6 @@ def parse_args():
     parser.add_argument('--data', type=str, nargs='+', help='Names of Datasets')
     parser.add_argument('--model', type=str, nargs='+', help='Names of Models')
     parser.add_argument('--config', type=str, help='Path to the Config Json File')
-    # Args that only apply to Video Dataset
-    parser.add_argument('--nframe', type=int, default=8)
-    parser.add_argument('--pack', action='store_true')
-    parser.add_argument('--use-subtitle', action='store_true')
-    parser.add_argument('--fps', type=float, default=-1)
     # Work Dir
     parser.add_argument('--work-dir', type=str, default='./outputs', help='select the output directory')
     # Infer + Eval or Infer Only
@@ -287,12 +286,8 @@ def main():
                         work_dir=pred_root,
                         model_name=model_name,
                         dataset=dataset,
-                        nframe=args.nframe,
-                        pack=args.pack,
                         verbose=args.verbose,
-                        subtitle=args.use_subtitle,
-                        api_nproc=args.nproc,
-                        fps=args.fps)
+                        api_nproc=args.nproc)
                 elif dataset.TYPE == 'MT':
                     model = infer_data_job_mt(
                         model,

vlmeval/dataset/longvideobench.py

@@ -94,8 +94,8 @@ class LongVideoBench(VideoBaseDataset):
 
     TYPE = 'Video-MCQ'
 
-    def __init__(self, dataset='LongVideoBench', use_subtitle=False):
-        super().__init__(dataset=dataset)
+    def __init__(self, dataset='LongVideoBench', use_subtitle=False, nframe=8, fps=-1):
+        super().__init__(dataset=dataset, nframe=nframe, fps=fps)
         self.use_subtitle = use_subtitle
         self.dataset_name = dataset
 
@@ -195,25 +195,25 @@ def concat_tar_parts(tar_parts, output_tar):
 
         return dict(data_file=data_file, root=dataset_path)
 
-    def save_video_frames(self, video_path, num_frames=8, fps=-1, video_llm=False):
+    def save_video_frames(self, video_path, video_llm=False):
 
         vid_path = osp.join(self.data_root, video_path)
         vid = decord.VideoReader(vid_path)
         video_info = {
             'fps': vid.get_avg_fps(),
             'n_frames': len(vid),
         }
-        if num_frames > 0 and fps < 0:
-            step_size = len(vid) / (num_frames + 1)
-            indices = [int(i * step_size) for i in range(1, num_frames + 1)]
-            frame_paths = self.frame_paths(video_path[:-4], num_frames)
-        elif fps > 0:
+        if self.nframe > 0 and self.fps < 0:
+            step_size = len(vid) / (self.nframe + 1)
+            indices = [int(i * step_size) for i in range(1, self.nframe + 1)]
+            frame_paths = self.frame_paths(video_path[:-4])
+        elif self.fps > 0:
             # not constrained by num_frames, get frames by fps
             total_duration = video_info['n_frames'] / video_info['fps']
-            required_frames = int(total_duration * fps)
-            step_size = video_info['fps'] / fps
+            required_frames = int(total_duration * self.fps)
+            step_size = video_info['fps'] / self.fps
             indices = [int(i * step_size) for i in range(required_frames)]
-            frame_paths = self.frame_paths_fps(video_path[:-4], len(indices), fps)
+            frame_paths = self.frame_paths_fps(video_path[:-4], len(indices))
 
         flag = np.all([osp.exists(p) for p in frame_paths])
 
@@ -226,16 +226,16 @@ def save_video_frames(self, video_path, num_frames=8, fps=-1, video_llm=False):
 
         return frame_paths, indices, video_info
 
-    def save_video_into_images(self, line, num_frames=8):
-        frame_paths, indices, video_info = self.save_video_frames(line['video_path'], num_frames)
-        return frame_paths
+    # def save_video_into_images(self, line, num_frames=8):
+    #     frame_paths, indices, video_info = self.save_video_frames(line['video_path'], num_frames)
+    #     return frame_paths
 
-    def build_prompt(self, line, num_frames, video_llm, fps):
+    def build_prompt(self, line, video_llm):
         if isinstance(line, int):
             assert line < len(self)
             line = self.data.iloc[line]
 
-        frames, indices, video_info = self.save_video_frames(line['video_path'], num_frames, fps, video_llm)
+        frames, indices, video_info = self.save_video_frames(line['video_path'], video_llm)
         fps = video_info["fps"]
 
         message = [dict(type='text', value=self.SYS)]

vlmeval/dataset/mlvu.py

@@ -152,7 +152,7 @@ def qa_template(self, data):
         answer = f"({chr(ord('A') + answer_idx)}) {answer}"
         return question, answer
 
-    def save_video_frames(self, line, num_frames=8, fps=-1):
+    def save_video_frames(self, line):
         suffix = line['video'].split('.')[-1]
         video = line['video'].replace(f'.{suffix}','')
         vid_path = osp.join(self.data_root, line['prefix'], line['video'])
@@ -161,17 +161,17 @@ def save_video_frames(self, line, num_frames=8, fps=-1):
             'fps': vid.get_avg_fps(),
             'n_frames': len(vid),
         }
-        if num_frames > 0 and fps < 0:
-            step_size = len(vid) / (num_frames + 1)
-            indices = [int(i * step_size) for i in range(1, num_frames + 1)]
-            frame_paths = self.frame_paths(video, num_frames)
-        elif fps > 0:
+        if self.nframe > 0 and self.fps < 0:
+            step_size = len(vid) / (self.nframe + 1)
+            indices = [int(i * step_size) for i in range(1, self.nframe + 1)]
+            frame_paths = self.frame_paths(video)
+        elif self.fps > 0:
             # not constrained by num_frames, get frames by fps
             total_duration = video_info['n_frames'] / video_info['fps']
-            required_frames = int(total_duration * fps)
-            step_size = video_info['fps'] / fps
+            required_frames = int(total_duration * self.fps)
+            step_size = video_info['fps'] / self.fps
             indices = [int(i * step_size) for i in range(required_frames)]
-            frame_paths = self.frame_paths_fps(video, len(indices), fps)
+            frame_paths = self.frame_paths_fps(video, len(indices))
 
         flag = np.all([osp.exists(p) for p in frame_paths])
 
@@ -184,11 +184,11 @@ def save_video_frames(self, line, num_frames=8, fps=-1):
 
         return frame_paths
 
-    def save_video_into_images(self, line, num_frames, fps):
-        frame_paths = self.save_video_frames(line, num_frames, fps)
+    def save_video_into_images(self, line):
+        frame_paths = self.save_video_frames(line)
         return frame_paths
 
-    def build_prompt(self, line, num_frames, video_llm, fps=-1):
+    def build_prompt(self, line, video_llm):
         if isinstance(line, int):
             assert line < len(self)
             line = self.data.iloc[line]
@@ -200,7 +200,7 @@ def build_prompt(self, line, num_frames, video_llm, fps=-1):
         if video_llm:
             message.append(dict(type='video', value=video_path))
         else:
-            img_frame_paths = self.save_video_into_images(line, num_frames, fps)
+            img_frame_paths = self.save_video_into_images(line)
             for im in img_frame_paths:
                 message.append(dict(type='image', value=im))
         message.append(dict(type='text', value='\nOnly give the best option.'))
@@ -355,7 +355,7 @@ def qa_template(self, data):
         answer = data['answer']
         return question, answer
 
-    def save_video_frames(self, line, num_frames=8, fps=-1):
+    def save_video_frames(self, line):
         suffix = line['video'].split('.')[-1]
         video = line['video'].replace(f'.{suffix}','')
         vid_path = osp.join(self.data_root, line['prefix'], line['video'])
@@ -364,17 +364,17 @@ def save_video_frames(self, line, num_frames=8, fps=-1):
             'fps': vid.get_avg_fps(),
             'n_frames': len(vid),
         }
-        if num_frames > 0 and fps < 0:
-            step_size = len(vid) / (num_frames + 1)
-            indices = [int(i * step_size) for i in range(1, num_frames + 1)]
-            frame_paths = self.frame_paths(video, num_frames)
-        elif fps > 0:
+        if self.nframe > 0 and self.fps < 0:
+            step_size = len(vid) / (self.nframe + 1)
+            indices = [int(i * step_size) for i in range(1, self.nframe + 1)]
+            frame_paths = self.frame_paths(video)
+        elif self.fps > 0:
             # not constrained by num_frames, get frames by fps
             total_duration = video_info['n_frames'] / video_info['fps']
-            required_frames = int(total_duration * fps)
-            step_size = video_info['fps'] / fps
+            required_frames = int(total_duration * self.fps)
+            step_size = video_info['fps'] / self.fps
             indices = [int(i * step_size) for i in range(required_frames)]
-            frame_paths = self.frame_paths_fps(video, len(indices), fps)
+            frame_paths = self.frame_paths_fps(video, len(indices))
 
         flag = np.all([osp.exists(p) for p in frame_paths])
 
@@ -387,11 +387,11 @@ def save_video_frames(self, line, num_frames=8, fps=-1):
 
         return frame_paths
 
-    def save_video_into_images(self, line, num_frames, fps):
-        frame_paths = self.save_video_frames(line, num_frames, fps)
+    def save_video_into_images(self, line):
+        frame_paths = self.save_video_frames(line)
         return frame_paths
 
-    def build_prompt(self, line, num_frames, video_llm, fps=-1):
+    def build_prompt(self, line, video_llm):
         if isinstance(line, int):
             assert line < len(self)
             line = self.data.iloc[line]
@@ -403,7 +403,7 @@ def build_prompt(self, line, num_frames, video_llm, fps=-1):
         if video_llm:
             message.append(dict(type='video', value=video_path))
         else:
-            img_frame_paths = self.save_video_into_images(line, num_frames, fps)
+            img_frame_paths = self.save_video_into_images(line)
             for im in img_frame_paths:
                 message.append(dict(type='image', value=im))
         return message

vlmeval/dataset/mmbench_video.py

@@ -59,8 +59,8 @@ class MMBenchVideo(VideoBaseDataset):
 
     TYPE = 'Video-VQA'
 
-    def __init__(self, dataset='MMBench-Video', pack=False):
-        super().__init__(dataset=dataset, pack=pack)
+    def __init__(self, dataset='MMBench-Video', pack=False, nframe=8, fps=-1):
+        super().__init__(dataset=dataset, pack=pack, nframe=nframe, fps=fps)
 
     @classmethod
     def supported_datasets(cls):
@@ -92,7 +92,7 @@ def check_integrity(pth):
 
         return dict(data_file=data_file, root=osp.join(dataset_path, 'video'))
 
-    def build_prompt_pack(self, line, num_frames, fps=-1):
+    def build_prompt_pack(self, line):
         if isinstance(line, int):
             assert line < len(self)
             video = self.videos[line]
@@ -101,7 +101,7 @@ def build_prompt_pack(self, line, num_frames, fps=-1):
         elif isinstance(line, str):
             video = line
 
-        frames = self.save_video_frames(video, num_frames, fps)
+        frames = self.save_video_frames(video)
         sub = self.data[self.data['video'] == video]
         sys_prompt = self.SYS + self.FRAMES_TMPL_PACK.format(len(frames))
         message = [dict(type='text', value=sys_prompt)]
@@ -114,7 +114,7 @@ def build_prompt_pack(self, line, num_frames, fps=-1):
         message.append(dict(type='text', value=prompt))
         return message
 
-    def build_prompt_nopack(self, line, num_frames, video_llm, fps):
+    def build_prompt_nopack(self, line, video_llm):
         if isinstance(line, int):
             assert line < len(self)
             line = self.data.iloc[line]
@@ -125,7 +125,7 @@ def build_prompt_nopack(self, line, num_frames, video_llm, fps):
             message.append(dict(type='video', value=os.path.join(self.video_path, video_idx_path)))
             return message
         else:
-            frames = self.save_video_frames(line['video'], num_frames, fps)
+            frames = self.save_video_frames(line['video'])
             sys_prompt = self.FRAMES_TMPL_NOPACK.format(len(frames))
             message = [dict(type='text', value=sys_prompt)]
             for im in frames:
@@ -134,11 +134,11 @@ def build_prompt_nopack(self, line, num_frames, video_llm, fps):
             message.append(dict(type='text', value=prompt))
         return message
 
-    def build_prompt(self, line, num_frames, video_llm, fps):
+    def build_prompt(self, line, video_llm):
         if self.pack and not video_llm:
-            return self.build_prompt_pack(line, num_frames, fps)
+            return self.build_prompt_pack(line)
         else:
-            return self.build_prompt_nopack(line, num_frames, video_llm, fps)
+            return self.build_prompt_nopack(line, video_llm)
 
     @staticmethod
     def remove_side_quote(s, syms=[',', '"', "'"]):