feat

vlmeval/dataset/__init__.py

@@ -30,6 +30,7 @@
 from .mmgenbench import MMGenBench
 
 from .miabench import MIABench
+from .cmmmu import CMMMU
 from .wildvision import WildVision
 from .mmmath import MMMath
 from .dynamath import Dynamath
@@ -128,7 +129,8 @@ def evaluate(self, eval_file, **judge_kwargs):
     MMMUDataset, OCRBench, MathVista, LLaVABench, MMVet, MTVQADataset, TableVQABench,
     MMLongBench, VCRDataset, MMDUDataset, DUDE, SlideVQA, MUIRDataset,
     GMAIMMBenchDataset, MMERealWorld, HRBenchDataset, CRPE, MathVerse, NaturalBenchDataset,
-    MIABench, OlympiadBench, WildVision, MMMath, QSpatial, Dynamath, MMGenBench, VizWiz, MMNIAH
+    MIABench, OlympiadBench, WildVision, MMMath, QSpatial, Dynamath, MMGenBench, VizWiz, MMNIAH,
+    CMMMU
 ]
 
 VIDEO_DATASET = [

vlmeval/dataset/cmmmu.py

@@ -0,0 +1,21 @@
+from .image_base import ImageBaseDataset
+
+
+class CMMMU(ImageBaseDataset):
+
+    DATASET_URL = {
+        'CMMMU_TEST': 'https://opencompass.openxlab.space/utils/VLMEval/CMMMU_TEST.tsv',
+    }
+
+    DATASET_MD5 = {
+        'CMMMU_TEST': '521afc0f3bf341e6654327792781644d',
+    }
+
+    @staticmethod
+    def evaluate(self, eval_file, **judge_kwargs):
+
+        pass
+
+    def build_prompt(self, line):
+
+        pass