Integrate tests to evals folder

microsoft · Apr 18, 2024 · 5205979 · 5205979
1 parent ea5d5f3
commit 5205979
Show file tree

Hide file tree

Showing 16 changed files with 52 additions and 169 deletions.
diff --git a/src/promptflow-evals/tests/evals/conftest.py b/src/promptflow-evals/tests/evals/conftest.py
@@ -170,3 +170,20 @@ def _mock_process_wrapper(*args, **kwargs):
 def _mock_create_spawned_fork_process_manager(*args, **kwargs):
     setup_recording_injection_if_enabled()
     return create_spawned_fork_process_manager(*args, **kwargs)
+
+
+@pytest.fixture
+def ml_client_config() -> dict:
+    conn_name = "azure_ai_project_scope"
+
+    with open(
+        file=CONNECTION_FILE,
+        mode="r",
+        encoding="utf-8",  # Add the encoding parameter
+    ) as f:
+        dev_connections = json.load(f)
+
+    if conn_name not in dev_connections:
+        raise ValueError(f"Connection '{conn_name}' not found in dev connections.")
+
+    return dev_connections[conn_name]["value"]
diff --git a/.../simulator/e2etests/test_adv_simulator.py → ...ests/evals/e2etests/test_adv_simulator.py b/.../simulator/e2etests/test_adv_simulator.py → ...ests/evals/e2etests/test_adv_simulator.py
diff --git a/...s/simulator/e2etests/test_qa_generator.py → ...tests/evals/e2etests/test_qa_generator.py b/...s/simulator/e2etests/test_qa_generator.py → ...tests/evals/e2etests/test_qa_generator.py
diff --git a/...test_configs/generated_qa_chat_conv.jsonl → ...ittests/data/generated_qa_chat_conv.jsonl b/...test_configs/generated_qa_chat_conv.jsonl → ...ittests/data/generated_qa_chat_conv.jsonl
diff --git a/...est_configs/generated_qa_chat_short.jsonl → ...ttests/data/generated_qa_chat_short.jsonl b/...est_configs/generated_qa_chat_short.jsonl → ...ttests/data/generated_qa_chat_short.jsonl
diff --git a/...r/test_configs/generated_qa_pf_conv.jsonl → ...unittests/data/generated_qa_pf_conv.jsonl b/...r/test_configs/generated_qa_pf_conv.jsonl → ...unittests/data/generated_qa_pf_conv.jsonl
diff --git a/.../test_configs/generated_qa_pf_short.jsonl → ...nittests/data/generated_qa_pf_short.jsonl b/.../test_configs/generated_qa_pf_short.jsonl → ...nittests/data/generated_qa_pf_short.jsonl
diff --git a/.../simulator/unittests/test_qa_simulator.py → ...ests/evals/unittests/test_qa_simulator.py b/.../simulator/unittests/test_qa_simulator.py → ...ests/evals/unittests/test_qa_simulator.py
@@ -91,7 +91,7 @@ def test_export_format(self, qa_type, structure):
         model_config = dict(api_base=API_BASE, api_key=API_KEY, deployment=DEPLOYMENT, model=MODEL)
         qa_generator = QADataGenerator(model_config=model_config)
         qas = list(zip(questions, answers))
-        filepath = os.path.join(pathlib.Path(__file__).parent.parent.resolve(), "test_configs")
+        filepath = os.path.join(pathlib.Path(__file__).parent.resolve(), "data")
         output_file = os.path.join(filepath, f"test_{qa_type.value}_{structure.value}.jsonl")
         qa_generator.export_to_file(output_file, qa_type, qas, structure)
 

diff --git a/...sts/simulator/unittests/test_simulator.py → ...s/tests/evals/unittests/test_simulator.py b/...sts/simulator/unittests/test_simulator.py → ...s/tests/evals/unittests/test_simulator.py
diff --git a/...tor/unittests/test_simulator_templates.py → ...als/unittests/test_simulator_templates.py b/...tor/unittests/test_simulator_templates.py → ...als/unittests/test_simulator_templates.py
diff --git a/src/promptflow-evals/tests/simulator/__init__.py b/src/promptflow-evals/tests/simulator/__init__.py
diff --git a/src/promptflow-evals/tests/simulator/conftest.py b/src/promptflow-evals/tests/simulator/conftest.py
diff --git a/src/promptflow-evals/tests/simulator/e2etests/__init__.py b/src/promptflow-evals/tests/simulator/e2etests/__init__.py
diff --git a/src/promptflow-recording/recordings/local/evals.node_cache.shelve.bak b/src/promptflow-recording/recordings/local/evals.node_cache.shelve.bak
@@ -1,4 +1,17 @@
-'e812113f391afbb4b12aafd0b7e93c9b4fd5633f', (0, 3992)
-'e5a1c88060db56a1f098ee4343ddca0bb97fa620', (4096, 5484)
-'34501a2950464ae7eece224e06278dde3addcfb0', (9728, 4814)
-'5cd313845b5581923f342e6fee8c7247b765e0f4', (14848, 3871)
+'e812113f391afbb4b12aafd0b7e93c9b4fd5633f', (0, 3992)
+'e5a1c88060db56a1f098ee4343ddca0bb97fa620', (4096, 5484)
+'34501a2950464ae7eece224e06278dde3addcfb0', (9728, 4814)
+'5cd313845b5581923f342e6fee8c7247b765e0f4', (14848, 3871)
+'359d733942e7ab2d1e8aaaef37646963d3c2217a', (18944, 4031)
+'150c9f5d4c033d237d8ba7c69e77329eb425777a', (23040, 2586)
+'37466a662f9a12d4285959ba159203e8f44c913f', (26112, 4361)
+'c0fd61dba210c701671e3d0de1b6a58b84931e11', (30720, 2779)
+'d2105d2b315c0b0a86ac5a246e2c5b22661e7c7f', (33792, 4329)
+'b52fcdb3e35e6a47cc67195189a63ab633b135ee', (38400, 5024)
+'458bfde1e4da8554d3113babc6dfc6cf6953933c', (43520, 5788)
+'85554f1b55073110df6738f841e7e4b880a21228', (49664, 6737)
+'fe4b4e43d8955f8edde50292bf83e3ce1a6441c1', (56832, 3120)
+'ce9dfc7543c5c7e9acc7ff9513f7ea9ba2be915b', (60416, 4649)
+'1084eea656037fc4bdab3f5a0c182b7c5b64dd22', (65536, 6570)
+'8d32fa0a507dd1dc6471601647bfa9b2f13198d3', (72192, 8518)
+'58b2a6a32d8a862e37d2f9fd0a46c53c88e888af', (80896, 9493)
diff --git a/src/promptflow-recording/recordings/local/evals.node_cache.shelve.dat b/src/promptflow-recording/recordings/local/evals.node_cache.shelve.dat
diff --git a/src/promptflow-recording/recordings/local/evals.node_cache.shelve.dir b/src/promptflow-recording/recordings/local/evals.node_cache.shelve.dir
@@ -1,4 +1,17 @@
-'e812113f391afbb4b12aafd0b7e93c9b4fd5633f', (0, 3992)
-'e5a1c88060db56a1f098ee4343ddca0bb97fa620', (4096, 5484)
-'34501a2950464ae7eece224e06278dde3addcfb0', (9728, 4814)
-'5cd313845b5581923f342e6fee8c7247b765e0f4', (14848, 3871)
+'e812113f391afbb4b12aafd0b7e93c9b4fd5633f', (0, 3992)
+'e5a1c88060db56a1f098ee4343ddca0bb97fa620', (4096, 5484)
+'34501a2950464ae7eece224e06278dde3addcfb0', (9728, 4814)
+'5cd313845b5581923f342e6fee8c7247b765e0f4', (14848, 3871)
+'359d733942e7ab2d1e8aaaef37646963d3c2217a', (18944, 4031)
+'150c9f5d4c033d237d8ba7c69e77329eb425777a', (23040, 2586)
+'37466a662f9a12d4285959ba159203e8f44c913f', (26112, 4361)
+'c0fd61dba210c701671e3d0de1b6a58b84931e11', (30720, 2779)
+'d2105d2b315c0b0a86ac5a246e2c5b22661e7c7f', (33792, 4329)
+'b52fcdb3e35e6a47cc67195189a63ab633b135ee', (38400, 5024)
+'458bfde1e4da8554d3113babc6dfc6cf6953933c', (43520, 5788)
+'85554f1b55073110df6738f841e7e4b880a21228', (49664, 6737)
+'fe4b4e43d8955f8edde50292bf83e3ce1a6441c1', (56832, 3120)
+'ce9dfc7543c5c7e9acc7ff9513f7ea9ba2be915b', (60416, 4649)
+'1084eea656037fc4bdab3f5a0c182b7c5b64dd22', (65536, 6570)
+'8d32fa0a507dd1dc6471601647bfa9b2f13198d3', (72192, 8518)
+'58b2a6a32d8a862e37d2f9fd0a46c53c88e888af', (80896, 9493)