[Internal][Executor] Fix tests to be compatible with various versions…

… of python and os (#1791)

# Description

Fix tests to be compatible with various versions of python and os.

# All Promptflow Contribution checklist:
- [x] **The pull request does not introduce [breaking changes].**
- [ ] **CHANGELOG is updated for new features, bug fixes or other
significant changes.**
- [x] **I have read the [contribution guidelines](../CONTRIBUTING.md).**
- [ ] **Create an issue and link to the pull request to get dedicated
review from promptflow team. Learn more: [suggested
workflow](../CONTRIBUTING.md#suggested-workflow).**

## General Guidelines and Best Practices
- [x] Title of the pull request is clear and informative.
- [x] There are a small number of commits, each of which have an
informative message. This means that previously merged commits do not
appear in the history of the PR. For more information on cleaning up the
commits in your PR, [see this
page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md).

### Testing Guidelines
- [x] Pull request includes test coverage for the included changes.

---------

Co-authored-by: Lina Tang <[email protected]>

src/promptflow/promptflow/executor/_async_nodes_scheduler.py

@@ -56,7 +56,7 @@ async def execute(
 
         # Semaphore should be created in the loop, otherwise it will not work.
         loop = asyncio.get_running_loop()
-        self._semaphore = asyncio.Semaphore(self._node_concurrency, loop=loop)
+        self._semaphore = asyncio.Semaphore(self._node_concurrency)
         monitor = threading.Thread(
             target=monitor_long_running_coroutine,
             args=(loop, self._task_start_time, self._task_last_log_time, self._dag_manager_completed_event),

src/promptflow/tests/executor/e2etests/test_batch_timeout.py

@@ -23,6 +23,9 @@ class TestBatchTimeout:
     def setup_method(self):
         BatchEngine.register_executor(FlowLanguage.Python, MockPythonExecutorProxy)
 
+    def teardown_method(self):
+        BatchEngine.register_executor(FlowLanguage.Python, PythonExecutorProxy)
+
     @pytest.mark.parametrize(
         "flow_folder",
         [

src/promptflow/tests/executor/e2etests/test_traces.py

@@ -182,9 +182,9 @@ def test_flow_with_trace(self, flow_file, dev_connections):
         flow_trace = api_calls[0]
         assert flow_trace["name"] == "flow"
         assert flow_trace["type"] == "Flow"
-        assert flow_trace["end_time"] - flow_trace["start_time"] == pytest.approx(1.5, abs=0.1)
+        assert flow_trace["end_time"] - flow_trace["start_time"] == pytest.approx(1.5, abs=0.3)
         assert len(flow_trace["children"]) == 1
-        assert flow_trace["system_metrics"]["duration"] == pytest.approx(1.5, abs=0.1)
+        assert flow_trace["system_metrics"]["duration"] == pytest.approx(1.5, abs=0.3)
         assert flow_trace["system_metrics"]["prompt_tokens"] == 0
         assert flow_trace["system_metrics"]["completion_tokens"] == 0
         assert flow_trace["system_metrics"]["total_tokens"] == 0
@@ -201,7 +201,7 @@ def test_flow_with_trace(self, flow_file, dev_connections):
         assert greetings_trace["output"] == {"greeting": "Hello, User 1!"}
         assert greetings_trace["error"] is None
         assert greetings_trace["children"] is not None
-        assert greetings_trace["end_time"] - greetings_trace["start_time"] == pytest.approx(1.5, abs=0.1)
+        assert greetings_trace["end_time"] - greetings_trace["start_time"] == pytest.approx(1.5, abs=0.3)
         assert len(greetings_trace["children"]) == 2
         # TODO: to verfiy the system metrics. This might need to be fixed.
         assert greetings_trace["system_metrics"] == {}
@@ -213,7 +213,7 @@ def test_flow_with_trace(self, flow_file, dev_connections):
         assert get_user_name_trace["inputs"] == {"user_id": 1}
         assert get_user_name_trace["output"] == "User 1"
         assert get_user_name_trace["error"] is None
-        assert get_user_name_trace["end_time"] - get_user_name_trace["start_time"] == pytest.approx(1.0, abs=0.1)
+        assert get_user_name_trace["end_time"] - get_user_name_trace["start_time"] == pytest.approx(1.0, abs=0.2)
         assert len(get_user_name_trace["children"]) == 1
         # TODO: to verfiy the system metrics. This might need to be fixed.
         assert get_user_name_trace["system_metrics"] == {}
@@ -225,7 +225,8 @@ def test_flow_with_trace(self, flow_file, dev_connections):
         assert is_valid_name_trace["inputs"] == {"name": "User 1"}
         assert is_valid_name_trace["output"] is True
         assert is_valid_name_trace["error"] is None
-        assert is_valid_name_trace["end_time"] - is_valid_name_trace["start_time"] == pytest.approx(0.5, abs=0.1)
+        # When running tests in MacOS, it will take longer. So we adjust abs to 0.15 and see if it needs to be extended.
+        assert is_valid_name_trace["end_time"] - is_valid_name_trace["start_time"] == pytest.approx(0.5, abs=0.15)
         assert is_valid_name_trace["children"] == []
         # TODO: to verfiy the system metrics. This might need to be fixed.
         assert is_valid_name_trace["system_metrics"] == {}
@@ -237,7 +238,8 @@ def test_flow_with_trace(self, flow_file, dev_connections):
         assert format_greeting_trace["inputs"] == {"user_name": "User 1"}
         assert format_greeting_trace["output"] == "Hello, User 1!"
         assert format_greeting_trace["error"] is None
-        assert format_greeting_trace["end_time"] - format_greeting_trace["start_time"] == pytest.approx(0.5, abs=0.1)
+        # When running tests in MacOS, it will take longer. So we adjust abs to 0.15 and see if it needs to be extended.
+        assert format_greeting_trace["end_time"] - format_greeting_trace["start_time"] == pytest.approx(0.5, abs=0.15)
         assert format_greeting_trace["children"] == []
         # TODO: to verfiy the system metrics. This might need to be fixed.
         assert format_greeting_trace["system_metrics"] == {}

src/promptflow/tests/executor/unittests/batch/test_batch_engine.py

@@ -5,13 +5,7 @@
 import pytest
 
 from promptflow._core._errors import UnexpectedError
-from promptflow.batch import (
-    AbstractExecutorProxy,
-    APIBasedExecutorProxy,
-    BatchEngine,
-    CSharpExecutorProxy,
-    PythonExecutorProxy,
-)
+from promptflow.batch import APIBasedExecutorProxy, BatchEngine, CSharpExecutorProxy, PythonExecutorProxy
 from promptflow.contracts.run_info import Status
 from promptflow.exceptions import ErrorTarget
 from promptflow.executor._errors import ConnectionNotFound
@@ -60,16 +54,11 @@ def test_register_executor(self):
         assert BatchEngine.executor_proxy_classes["python"] == PythonExecutorProxy
         assert BatchEngine.executor_proxy_classes["csharp"] == CSharpExecutorProxy
 
-        class MockPythonExecutorProxy(AbstractExecutorProxy):
-            pass
-
         class MockJSExecutorProxy(APIBasedExecutorProxy):
             pass
 
         # register new proxy
-        BatchEngine.register_executor("python", MockPythonExecutorProxy)
         BatchEngine.register_executor("js", MockJSExecutorProxy)
-        assert BatchEngine.executor_proxy_classes["python"] == MockPythonExecutorProxy
         assert BatchEngine.executor_proxy_classes["js"] == MockJSExecutorProxy
         assert len(BatchEngine.executor_proxy_classes) == 3
 

src/promptflow/tests/executor/unittests/batch/test_batch_inputs_processor.py

@@ -11,6 +11,8 @@
 from promptflow.contracts.flow import FlowInputDefinition
 from promptflow.contracts.tool import ValueType
 
+from ...utils import DATA_ROOT
+
 
 @pytest.mark.unittest
 class TestBatchInputsProcessor:
@@ -68,11 +70,12 @@ def test_resolve_data_from_input_path(self):
     @pytest.mark.parametrize(
         "data_path",
         [
-            "./tests/test_configs/datas/load_data_cases/10k.jsonl",
-            "./tests/test_configs/datas/load_data_cases/10k",
+            "10k.jsonl",
+            "10k",
         ],
     )
     def test_resolve_data_from_input_path_with_large_data(self, data_path):
+        data_path = DATA_ROOT / "load_data_cases" / data_path
         result = BatchInputsProcessor("", {})._resolve_data_from_input_path(Path(data_path))
         assert isinstance(result, list)
         assert len(result) == 10000

src/promptflow/tests/test_configs/flows/flow_with_langchain_traces/test_langchain_traces.py

@@ -1,13 +1,11 @@
 import os
 
-
 from langchain.chat_models import AzureChatOpenAI
 from langchain_core.messages import HumanMessage
 from langchain.agents.agent_types import AgentType
 from langchain.agents.initialize import initialize_agent
 from langchain.agents.load_tools import load_tools
 
-
 from promptflow import tool
 from promptflow.connections import AzureOpenAIConnection
 from promptflow.integrations.langchain import PromptFlowCallbackHandler
@@ -34,5 +32,7 @@ def test_langchain_traces(question: str, conn: AzureOpenAIConnection):
         content=question
     )
 
-    return agent.run(message)
-
+    try:
+        return agent.run(message)
+    except Exception as e:
+        return str(e)