add dependency optimum-habana.

docker/hpu.dockerfile

@@ -19,7 +19,7 @@ RUN pip install --upgrade pip setuptools==69.5.1
 
 # Build From Source
 RUN cd /GenAIEval && \
-    pip install -r requirements.txt && \
+    pip install -r requirements_hpu.txt && \
     python setup.py install && \
     pip list
 

evals/evaluation/lm_evaluation_harness/lm_eval/models/huggingface.py

@@ -1176,11 +1176,14 @@ def __init__(self, *args, **kwargs):
         else:
             self.static_shapes = False
 
+        # TODO
+        """
         if self.static_shapes:
             print("use hpu graphs.")
             from habana_frameworks.torch.hpu import wrap_in_hpu_graph
 
             self._model = wrap_in_hpu_graph(self._model)
+        """
 
         print("lm-eval warmup starting for Gaudi.")
         self.warm_up()

requirements_hpu.txt

@@ -0,0 +1,23 @@
+bigcode-eval@git+https://github.com/bigcode-project/bigcode-evaluation-harness.git
+click
+deepdiff
+deepeval==1.4.0
+docker
+evaluate
+flask
+jieba
+kubernetes
+langchain_community
+langchain_huggingface
+lm-eval==0.4.3
+locust
+numpy < 2.0
+optimum-habana
+prometheus_client
+pytest
+pyyaml
+ragas==v0.1.19
+requests
+rouge_score
+sseclient-py
+transformers

tests/test_lm_eval.py

@@ -13,12 +13,9 @@
 class TestLMEval(unittest.TestCase):
     def test_lm_eval(self):
         model_name_or_path = "facebook/opt-125m"
-        user_model = AutoModelForCausalLM.from_pretrained(model_name_or_path)
-        tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
         args = LMEvalParser(
             model="hf",
-            user_model=user_model,
-            tokenizer=tokenizer,
+            model_args=f"pretrained={model_name_or_path}",
             tasks="lambada_openai",
             device="cpu",
             batch_size=1,