Skip to content

Commit

Permalink
[Fix] Fix Lint
Browse files Browse the repository at this point in the history
  • Loading branch information
kennymckormick committed Dec 31, 2024
1 parent 6e1a59a commit ae10ca4
Show file tree
Hide file tree
Showing 4 changed files with 12 additions and 12 deletions.
8 changes: 4 additions & 4 deletions vlmeval/dataset/utils/ccocr_evaluator/README.md
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
# CC-OCR: A Comprehensive and Challenging OCR Benchmark for Evaluating Large Multimodal Models in Literacy

## Introduction

Please refer to our [GitHub](https://github.com/AlibabaResearch/AdvancedLiterateMachinery/tree/main/Benchmarks/CC-OCR) for more information.

## Running Scripts

Once the environment is ready, execute the following script from the root directory of VLMEvalKit
Once the environment is ready, execute the following script from the root directory of VLMEvalKit
to perform inference and evaluation tasks in batch.

```shell
Expand Down Expand Up @@ -44,13 +44,13 @@ If you find our work helpful, feel free to give us a cite.

```
@misc{yang2024ccocr,
title={CC-OCR: A Comprehensive and Challenging OCR Benchmark for Evaluating Large Multimodal Models in Literacy},
title={CC-OCR: A Comprehensive and Challenging OCR Benchmark for Evaluating Large Multimodal Models in Literacy},
author={Zhibo Yang and Jun Tang and Zhaohai Li and Pengfei Wang and Jianqiang Wan and Humen Zhong and Xuejing Liu and Mingkun Yang and Peng Wang and Shuai Bai and LianWen Jin and Junyang Lin},
year={2024},
eprint={2412.02210},
archivePrefix={arXiv},
primaryClass={cs.CV},
url={https://arxiv.org/abs/2412.02210},
url={https://arxiv.org/abs/2412.02210},
}
```

Expand Down
2 changes: 1 addition & 1 deletion vlmeval/dataset/utils/ccocr_evaluator/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,4 @@
"doc_parsing": ParsingEvaluator("doc_parsing"),
"multi_lan_ocr": OcrEvaluator("multi_lan_ocr"),
"multi_scene_ocr": OcrEvaluator("multi_scene_ocr")
}
}
12 changes: 6 additions & 6 deletions vlmeval/dataset/utils/ccocr_evaluator/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ def pick_response_text(json_path):

response_text = None
if model_name.startswith("gpt") or model_name.startswith("o1"):
response_text = model_response.get("data", {}).get("response", {}).get("choices", [{}])[0].get("message", {}).get("content", None)
response_text = model_response.get("data", {}).get("response", {}).get("choices", [{}])[0].get("message", {}).get("content", None) # noqa: E501
elif model_name.startswith("local_"):
response_text = model_response
else:
Expand All @@ -35,7 +35,7 @@ def pick_response_text(json_path):
elif model_name.startswith("gemini"):
content_list = model_response.get("candidates", [{}])[0].get("content", {}).get("parts", None)
elif model_name.startswith("qwen"):
content_list = model_response.get("output", {}).get("choices", [{}])[0].get("message", {}).get("content", None)
content_list = model_response.get("output", {}).get("choices", [{}])[0].get("message", {}).get("content", None) # noqa: E501
else:
raise NotImplementedError("The pick_response_text NOT implemented for model: {}".format(model_name))

Expand Down Expand Up @@ -115,7 +115,7 @@ def __call__(self, pdt_res_dir, gt_info, with_response_ratio=True, **kwargs):
# add response_success_ratio
if "summary" in eval_info and with_response_ratio:
success_ratio = (len(response_info) + len(post_error_list)) / (len(gt_info) + 1e-9)
eval_info["summary"].update({"response_success_ratio": success_ratio })
eval_info["summary"].update({"response_success_ratio": success_ratio})
return meta_info, eval_info


Expand Down Expand Up @@ -149,9 +149,9 @@ def summary_multi_exp(exp_dir_base, dataset_list=None, is_weighted_sum=False):
data_status_info = json.load(f)
all_dataset_name.extend(data_status_info.keys())
dataset_list = sorted(set(all_dataset_name))

# summary main code
all_evaluate_info, line_index = {}, 0
all_evaluate_info, _ = {}, 0
for exp_name in os.listdir(exp_dir_base):
dir_status_path = os.path.join(exp_dir_base, exp_name, "status.json")
if not os.path.exists(dir_status_path):
Expand Down Expand Up @@ -219,4 +219,4 @@ def summary_multi_exp(exp_dir_base, dataset_list=None, is_weighted_sum=False):

summary_path = summary_multi_exp(exp_base_dir, dataset_list=None, is_weighted_sum=False)
print("--> info: summary saved at : {}".format(summary_path))
print("happy coding.")
print("happy coding.")
2 changes: 1 addition & 1 deletion vlmeval/vlm/valley/valley_eagle_chat.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ def __init__(self,
from .valley_eagle.util.mm_utils import process_anyres_image
from .valley_eagle import conversation as conversation_lib
from .valley_eagle.util.data_util import dynamic_preprocess, preprocess

torch_dtype = torch.float16
padding_side = 'left'
use_fast = True
Expand Down

0 comments on commit ae10ca4

Please sign in to comment.