diff --git a/vlmeval/dataset/utils/ccocr_evaluator/README.md b/vlmeval/dataset/utils/ccocr_evaluator/README.md index a636aa4a..99572ef5 100644 --- a/vlmeval/dataset/utils/ccocr_evaluator/README.md +++ b/vlmeval/dataset/utils/ccocr_evaluator/README.md @@ -1,12 +1,12 @@ # CC-OCR: A Comprehensive and Challenging OCR Benchmark for Evaluating Large Multimodal Models in Literacy ## Introduction - + Please refer to our [GitHub](https://github.com/AlibabaResearch/AdvancedLiterateMachinery/tree/main/Benchmarks/CC-OCR) for more information. ## Running Scripts -Once the environment is ready, execute the following script from the root directory of VLMEvalKit +Once the environment is ready, execute the following script from the root directory of VLMEvalKit to perform inference and evaluation tasks in batch. ```shell @@ -44,13 +44,13 @@ If you find our work helpful, feel free to give us a cite. ``` @misc{yang2024ccocr, - title={CC-OCR: A Comprehensive and Challenging OCR Benchmark for Evaluating Large Multimodal Models in Literacy}, + title={CC-OCR: A Comprehensive and Challenging OCR Benchmark for Evaluating Large Multimodal Models in Literacy}, author={Zhibo Yang and Jun Tang and Zhaohai Li and Pengfei Wang and Jianqiang Wan and Humen Zhong and Xuejing Liu and Mingkun Yang and Peng Wang and Shuai Bai and LianWen Jin and Junyang Lin}, year={2024}, eprint={2412.02210}, archivePrefix={arXiv}, primaryClass={cs.CV}, - url={https://arxiv.org/abs/2412.02210}, + url={https://arxiv.org/abs/2412.02210}, } ``` diff --git a/vlmeval/dataset/utils/ccocr_evaluator/__init__.py b/vlmeval/dataset/utils/ccocr_evaluator/__init__.py index cd3ff5ac..d89f6f6b 100644 --- a/vlmeval/dataset/utils/ccocr_evaluator/__init__.py +++ b/vlmeval/dataset/utils/ccocr_evaluator/__init__.py @@ -9,4 +9,4 @@ "doc_parsing": ParsingEvaluator("doc_parsing"), "multi_lan_ocr": OcrEvaluator("multi_lan_ocr"), "multi_scene_ocr": OcrEvaluator("multi_scene_ocr") -} \ No newline at end of file +} diff --git a/vlmeval/dataset/utils/ccocr_evaluator/common.py b/vlmeval/dataset/utils/ccocr_evaluator/common.py index a3e80349..6ce9bcb5 100644 --- a/vlmeval/dataset/utils/ccocr_evaluator/common.py +++ b/vlmeval/dataset/utils/ccocr_evaluator/common.py @@ -26,7 +26,7 @@ def pick_response_text(json_path): response_text = None if model_name.startswith("gpt") or model_name.startswith("o1"): - response_text = model_response.get("data", {}).get("response", {}).get("choices", [{}])[0].get("message", {}).get("content", None) + response_text = model_response.get("data", {}).get("response", {}).get("choices", [{}])[0].get("message", {}).get("content", None) # noqa: E501 elif model_name.startswith("local_"): response_text = model_response else: @@ -35,7 +35,7 @@ def pick_response_text(json_path): elif model_name.startswith("gemini"): content_list = model_response.get("candidates", [{}])[0].get("content", {}).get("parts", None) elif model_name.startswith("qwen"): - content_list = model_response.get("output", {}).get("choices", [{}])[0].get("message", {}).get("content", None) + content_list = model_response.get("output", {}).get("choices", [{}])[0].get("message", {}).get("content", None) # noqa: E501 else: raise NotImplementedError("The pick_response_text NOT implemented for model: {}".format(model_name)) @@ -115,7 +115,7 @@ def __call__(self, pdt_res_dir, gt_info, with_response_ratio=True, **kwargs): # add response_success_ratio if "summary" in eval_info and with_response_ratio: success_ratio = (len(response_info) + len(post_error_list)) / (len(gt_info) + 1e-9) - eval_info["summary"].update({"response_success_ratio": success_ratio }) + eval_info["summary"].update({"response_success_ratio": success_ratio}) return meta_info, eval_info @@ -149,9 +149,9 @@ def summary_multi_exp(exp_dir_base, dataset_list=None, is_weighted_sum=False): data_status_info = json.load(f) all_dataset_name.extend(data_status_info.keys()) dataset_list = sorted(set(all_dataset_name)) - + # summary main code - all_evaluate_info, line_index = {}, 0 + all_evaluate_info, _ = {}, 0 for exp_name in os.listdir(exp_dir_base): dir_status_path = os.path.join(exp_dir_base, exp_name, "status.json") if not os.path.exists(dir_status_path): @@ -219,4 +219,4 @@ def summary_multi_exp(exp_dir_base, dataset_list=None, is_weighted_sum=False): summary_path = summary_multi_exp(exp_base_dir, dataset_list=None, is_weighted_sum=False) print("--> info: summary saved at : {}".format(summary_path)) - print("happy coding.") \ No newline at end of file + print("happy coding.") diff --git a/vlmeval/vlm/valley/valley_eagle_chat.py b/vlmeval/vlm/valley/valley_eagle_chat.py index 4c82ccc9..374fad56 100644 --- a/vlmeval/vlm/valley/valley_eagle_chat.py +++ b/vlmeval/vlm/valley/valley_eagle_chat.py @@ -124,7 +124,7 @@ def __init__(self, from .valley_eagle.util.mm_utils import process_anyres_image from .valley_eagle import conversation as conversation_lib from .valley_eagle.util.data_util import dynamic_preprocess, preprocess - + torch_dtype = torch.float16 padding_side = 'left' use_fast = True