From eec4c72981ef5606978ea62e0dca1fea3240cb7f Mon Sep 17 00:00:00 2001 From: kennymckormick Date: Tue, 19 Mar 2024 23:13:39 +0800 Subject: [PATCH] update --- Quickstart.md | 22 +++++++++++++++++++++- vlmeval/__init__.py | 2 ++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/Quickstart.md b/Quickstart.md index c6adcd51c..5bf8c2720 100644 --- a/Quickstart.md +++ b/Quickstart.md @@ -4,7 +4,9 @@ Before running the evaluation script, you need to **configure** the VLMs and set After that, you can use a single script `run.py` to inference and evaluate multiple VLMs and benchmarks at a same time. -## Step0. Installation +## Step0. Installation & Setup essential keys + +**Installation. ** ```bash git clone https://github.com/open-compass/VLMEvalKit.git @@ -12,6 +14,24 @@ cd VLMEvalKit pip install -e . ``` +**Setup Keys.** + +- To infer with API models (GPT-4v, Gemini-Pro-V, etc.) or use LLM APIs as the **judge or choice extractor**, you need to first setup API keys. You can place the required keys in `$VLMEvalKit/.env` or directly set them as the environment variable. If you choose to create a `.env` file, its content will look like: + + ```bash + # The .env file, place it under $VLMEvalKit + # Alles-apin-token, for intra-org use only + ALLES= + # API Keys of Proprietary VLMs + DASHSCOPE_API_KEY= + GOOGLE_API_KEY= + OPENAI_API_KEY= + OPENAI_API_BASE= + STEPAI_API_KEY= + ``` + +- Fill the blanks with your API keys (if necessary). Those API keys will be automatically loaded when doing the inference and evaluation. + ## Step1. Configuration **VLM Configuration**: All VLMs are configured in `vlmeval/config.py`, for some VLMs, you need to configure the code root (MiniGPT-4, PandaGPT, etc.) or the model_weight root (LLaVA-v1-7B, etc.) before conducting the evaluation. During evaluation, you should use the model name specified in `supported_VLM` in `vlmeval/config.py` to select the VLM. For MiniGPT-4 and InstructBLIP, you also need to modify the config files in `vlmeval/vlm/misc` to configure LLM path and ckpt path. diff --git a/vlmeval/__init__.py b/vlmeval/__init__.py index b8e96c287..336a61a61 100644 --- a/vlmeval/__init__.py +++ b/vlmeval/__init__.py @@ -9,3 +9,5 @@ from .utils import * from .vlm import * from .config import * + +load_env()