From ff65d7e68eb58124aef6af6fac7bce6cf89f95e4 Mon Sep 17 00:00:00 2001 From: semio Date: Sat, 11 Nov 2023 23:31:37 +0800 Subject: [PATCH] Add readme about Redis cache --- yival_experiments/README.md | 11 ++++++++++- .../custom_configuration/model_compare.py | 5 +++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/yival_experiments/README.md b/yival_experiments/README.md index f30b3fd..3ea546e 100644 --- a/yival_experiments/README.md +++ b/yival_experiments/README.md @@ -33,7 +33,7 @@ file when the source type is set to "dataset". So we need to fetch it first. ## 4. run the experiment -The full experiment configuration is [here](https://github.com/Gapminder/gapminder-ai/blob/yival/yival_experiments/latest_experiment.yaml) +The full experiment configuration is [here](https://github.com/Gapminder/gapminder-ai/blob/yival/yival_experiments/experiment_latest.yaml) To run it: @@ -46,6 +46,15 @@ This will output a pickle file in `output/experiment_name_0.pkl` which include a When the experiment is completed, Yival will start a web server to show the results. +### Use Redis for caching + +The model compare function will cache LLM call results for the +evaluator, and by default the cache is dictionary in memory. You can +also use Redis to caching, so that it won't loss the cache when Yival +exits. To do this, uncomment the line for redis cache in the top of +`custom_configuration/model_compare.py` and set the host and password +to your redis server. + ## 5. generate a result csv from output To convert the pickle to excel file and create csv files for summary report, you can run the script in output/. diff --git a/yival_experiments/custom_configuration/model_compare.py b/yival_experiments/custom_configuration/model_compare.py index 1904bf2..5e07b89 100644 --- a/yival_experiments/custom_configuration/model_compare.py +++ b/yival_experiments/custom_configuration/model_compare.py @@ -18,8 +18,9 @@ # set this to see verbose outputs litellm.set_verbose = True # enable caching in the evaluator. -# litellm.cache = litellm.Cache() -litellm.cache = litellm.Cache(type="redis", host="127.0.0.1", port=6379) +litellm.cache = litellm.Cache() +# use Redis for caching: comment the line above and uncomment the line below. +# litellm.cache = litellm.Cache(type="redis", host="127.0.0.1", port=6379) def model_compare(