PF Recording/Replaying part for sdk_cli #1852
GitHub Actions / SDK CLI Test Result [yigao/recording_draft](https://github.com/microsoft/promptflow/actions/workflows/promptflow-sdk-cli-test.yml?query=branch:yigao/recording_draft++)
failed
Oct 23, 2023 in 0s
8 fail, 14 skipped, 283 pass in 40m 0s
Annotations
Check warning on line 0 in tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun
github-actions / SDK CLI Test Result [yigao/recording_draft](https://github.com/microsoft/promptflow/actions/workflows/promptflow-sdk-cli-test.yml?query=branch:yigao/recording_draft++)
All 2 runs failed: test_basic_evaluation (tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun)
artifacts/Test Results (Python 3.10) (OS macos-latest)/test-results.xml [took 10m 0s]
artifacts/Test Results (Python 3.9) (OS ubuntu-latest)/test-results.xml [took 10m 0s]
Raw output
Failed: Timeout >600.0s
self = <sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun object at 0x129728af0>
pf = <promptflow.azure._pf_client.PFClient object at 0x129af5c60>
runtime = 'demo-mir'
def test_basic_evaluation(self, pf, runtime):
data_path = f"{DATAS_DIR}/webClassification3.jsonl"
run = pf.run(
flow=f"{FLOWS_DIR}/web_classification",
data=data_path,
column_mapping={"url": "${data.url}"},
variant="${summarize_text_content.variant_0}",
runtime=runtime,
)
assert isinstance(run, Run)
> run = pf.runs.stream(run=run.name)
tests/sdk_cli_azure_test/e2etests/test_run_operations.py:69:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
promptflow/_telemetry/activity.py:138: in wrapper
return f(self, *args, **kwargs)
promptflow/azure/operations/_run_operations.py:688: in stream
available_logs = self._get_log(flow_run_id=run.name)
promptflow/azure/operations/_run_operations.py:593: in _get_log
headers=self._get_headers(),
promptflow/azure/operations/_run_operations.py:205: in _get_headers
token = self._credential.get_token("https://management.azure.com/.default").token
/Users/runner/hostedtoolcache/Python/3.10.13/x64/lib/python3.10/site-packages/azure/identity/_internal/decorators.py:33: in wrapper
token = fn(*args, **kwargs)
/Users/runner/hostedtoolcache/Python/3.10.13/x64/lib/python3.10/site-packages/azure/identity/_credentials/azure_cli.py:113: in get_token
output = _run_command(command, self._process_timeout)
/Users/runner/hostedtoolcache/Python/3.10.13/x64/lib/python3.10/site-packages/azure/identity/_credentials/azure_cli.py:200: in _run_command
return subprocess.check_output(args, **kwargs)
/Users/runner/hostedtoolcache/Python/3.10.13/x64/lib/python3.10/subprocess.py:421: in check_output
return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
/Users/runner/hostedtoolcache/Python/3.10.13/x64/lib/python3.10/subprocess.py:505: in run
stdout, stderr = process.communicate(input, timeout=timeout)
/Users/runner/hostedtoolcache/Python/3.10.13/x64/lib/python3.10/subprocess.py:1154: in communicate
stdout, stderr = self._communicate(input, endtime, timeout)
/Users/runner/hostedtoolcache/Python/3.10.13/x64/lib/python3.10/subprocess.py:2021: in _communicate
ready = selector.select(timeout)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <selectors.PollSelector object at 0x1296a0d60>, timeout = 10000
def select(self, timeout=None):
# This is shared between poll() and epoll().
# epoll() has a different signature and handling of timeout parameter.
if timeout is None:
timeout = None
elif timeout <= 0:
timeout = 0
else:
# poll() has a resolution of 1 millisecond, round away from
# zero to wait *at least* timeout seconds.
timeout = math.ceil(timeout * 1e3)
ready = []
try:
> fd_event_list = self._selector.poll(timeout)
E Failed: Timeout >600.0s
/Users/runner/hostedtoolcache/Python/3.10.13/x64/lib/python3.10/selectors.py:416: Failed
Check warning on line 0 in tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun
github-actions / SDK CLI Test Result [yigao/recording_draft](https://github.com/microsoft/promptflow/actions/workflows/promptflow-sdk-cli-test.yml?query=branch:yigao/recording_draft++)
1 out of 2 runs failed: test_archive_and_restore_run (tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun)
artifacts/Test Results (Python 3.10) (OS macos-latest)/test-results.xml [took 4s]
Raw output
assert False is True
self = <sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun object at 0x12b535f30>
remote_client = <promptflow.azure._pf_client.PFClient object at 0x12bdc70a0>
def test_archive_and_restore_run(self, remote_client):
from promptflow._sdk._constants import RunHistoryKeys
run_meta_data = RunHistoryKeys.RunMetaData
hidden = RunHistoryKeys.HIDDEN
run_id = "4cf2d5e9-c78f-4ab8-a3ee-57675f92fb74"
# test archive
remote_client.runs.archive(run=run_id)
run_data = remote_client.runs._get_run_from_run_history(run_id, original_form=True)[run_meta_data]
> assert run_data[hidden] is True
E assert False is True
tests/sdk_cli_azure_test/e2etests/test_run_operations.py:273: AssertionError
Check warning on line 0 in tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun
github-actions / SDK CLI Test Result [yigao/recording_draft](https://github.com/microsoft/promptflow/actions/workflows/promptflow-sdk-cli-test.yml?query=branch:yigao/recording_draft++)
1 out of 2 runs failed: test_update_run (tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun)
artifacts/Test Results (Python 3.10) (OS macos-latest)/test-results.xml [took 5s]
Raw output
AssertionError: assert 'test_display_name_085d689d-ce2a-429f-ab55-4547babca369' == 'test_display_name_49f9b1dc-0b39-4691-9e9b-97ac69f935fc'
- test_display_name_49f9b1dc-0b39-4691-9e9b-97ac69f935fc
+ test_display_name_085d689d-ce2a-429f-ab55-4547babca369
self = <sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun object at 0x12b536020>
remote_client = <promptflow.azure._pf_client.PFClient object at 0x12bdbc7f0>
def test_update_run(self, remote_client):
run_id = "4cf2d5e9-c78f-4ab8-a3ee-57675f92fb74"
test_mark = str(uuid.uuid4())
new_display_name = f"test_display_name_{test_mark}"
new_description = f"test_description_{test_mark}"
new_tags = {"test_tag": test_mark}
run = remote_client.runs.update(
run=run_id,
display_name=new_display_name,
description=new_description,
tags=new_tags,
)
assert run.display_name == new_display_name
assert run.description == new_description
assert run.tags["test_tag"] == test_mark
# test wrong type of parameters won't raise error, just log warnings and got ignored
run = remote_client.runs.update(
run=run_id,
tags={"test_tag": {"a": 1}},
)
> assert run.display_name == new_display_name
E AssertionError: assert 'test_display_name_085d689d-ce2a-429f-ab55-4547babca369' == 'test_display_name_49f9b1dc-0b39-4691-9e9b-97ac69f935fc'
E - test_display_name_49f9b1dc-0b39-4691-9e9b-97ac69f935fc
E + test_display_name_085d689d-ce2a-429f-ab55-4547babca369
tests/sdk_cli_azure_test/e2etests/test_run_operations.py:303: AssertionError
Check warning on line 0 in tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun
github-actions / SDK CLI Test Result [yigao/recording_draft](https://github.com/microsoft/promptflow/actions/workflows/promptflow-sdk-cli-test.yml?query=branch:yigao/recording_draft++)
1 out of 2 runs failed: test_run_with_additional_includes (tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun)
artifacts/Test Results (Python 3.9) (OS ubuntu-latest)/test-results.xml [took 1m 3s]
Raw output
AssertionError: assert 'Failed' == 'Completed'
- Completed
+ Failed
self = <sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun object at 0x7f680ba74730>
remote_client = <promptflow.azure._pf_client.PFClient object at 0x7f67f9e7c520>
pf = <promptflow.azure._pf_client.PFClient object at 0x7f67f9e7c520>
runtime = 'demo-mir'
def test_run_with_additional_includes(self, remote_client, pf, runtime):
run = pf.run(
flow=f"{FLOWS_DIR}/web_classification_with_additional_include",
data=f"{DATAS_DIR}/webClassification1.jsonl",
inputs_mapping={"url": "${data.url}"},
variant="${summarize_text_content.variant_0}",
runtime=runtime,
)
run = remote_client.runs.stream(run=run.name)
> assert run.status == RunStatus.COMPLETED
E AssertionError: assert 'Failed' == 'Completed'
E - Completed
E + Failed
tests/sdk_cli_azure_test/e2etests/test_run_operations.py:316: AssertionError
Check warning on line 0 in tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun
github-actions / SDK CLI Test Result [yigao/recording_draft](https://github.com/microsoft/promptflow/actions/workflows/promptflow-sdk-cli-test.yml?query=branch:yigao/recording_draft++)
1 out of 2 runs failed: test_input_mapping_with_dict (tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun)
artifacts/Test Results (Python 3.10) (OS macos-latest)/test-results.xml [took 10m 0s]
Raw output
Failed: Timeout >600.0s
self = <sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun object at 0x121d57820>
pf = <promptflow.azure._pf_client.PFClient object at 0x1225e3610>
runtime = 'demo-mir'
def test_input_mapping_with_dict(self, pf, runtime: str):
data_path = f"{DATAS_DIR}/webClassification3.jsonl"
run = pf.run(
flow=f"{FLOWS_DIR}/flow_with_dict_input",
data=data_path,
column_mapping=dict(key={"a": 1}, extra="${data.url}"),
runtime=runtime,
)
assert '"{\\"a\\": 1}"' in run.properties["azureml.promptflow.inputs_mapping"]
> run = pf.runs.stream(run=run)
tests/sdk_cli_azure_test/e2etests/test_run_operations.py:495:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
promptflow/_telemetry/activity.py:138: in wrapper
return f(self, *args, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <promptflow.azure.operations._run_operations.RunOperations object at 0x1228cdf60>
run = <promptflow._sdk.entities._run.Run object at 0x1228cd8d0>
@monitor_operation(activity_name="pfazure.runs.stream", activity_type=ActivityType.PUBLICAPI)
def stream(self, run: Union[str, Run]) -> Run:
"""Stream the logs of a run.
:param run: The run name or run object
:type run: Union[str, ~promptflow.entities.Run]
:return: The run object
:rtype: ~promptflow.entities.Run
"""
run = self.get(run=run)
# TODO: maybe we need to make this configurable
file_handler = sys.stdout
# different from Azure ML job, flow job can run very fast, so it might not print anything;
# use below variable to track this behavior, and at least print something to the user.
try:
printed = 0
stream_count = 0
start = time.time()
while run.status in RUNNING_STATUSES or run.status == RunStatus.FINALIZING:
file_handler.flush()
stream_count += 1
# print prompt every 3 times, in case there is no log printed
if stream_count % 3 == 0:
# print prompt every 3 times
file_handler.write(f"(Run status is {run.status!r}, continue streaming...)\n")
# if the run is not started for 5 minutes, print an error message and break the loop
if run.status == RunStatus.NOT_STARTED:
current = time.time()
if current - start > 300:
file_handler.write(
f"The run {run.name!r} is in status 'NotStarted' for 5 minutes, streaming is stopped."
"Please make sure you are using the latest runtime.\n"
)
break
available_logs = self._get_log(flow_run_id=run.name)
printed = incremental_print(available_logs, printed, file_handler)
> time.sleep(10)
E Failed: Timeout >600.0s
promptflow/azure/operations/_run_operations.py:690: Failed
Check warning on line 0 in tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun
github-actions / SDK CLI Test Result [yigao/recording_draft](https://github.com/microsoft/promptflow/actions/workflows/promptflow-sdk-cli-test.yml?query=branch:yigao/recording_draft++)
All 2 runs failed: test_basic_flow_with_script_tool_with_custom_strong_type_connection (tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun)
artifacts/Test Results (Python 3.10) (OS macos-latest)/test-results.xml [took 0s]
artifacts/Test Results (Python 3.9) (OS ubuntu-latest)/test-results.xml [took 0s]
Raw output
AttributeError: 'CustomConnection' object has no attribute 'api_base'
self = <sdk_cli_test.e2etests.test_flow_run.TestFlowRun object at 0x122146140>
install_custom_tool_pkg = None
local_client = <promptflow._sdk._pf_client.PFClient object at 0x1223fe3e0>
pf = <promptflow._sdk._pf_client.PFClient object at 0x1223fe6e0>
def test_basic_flow_with_script_tool_with_custom_strong_type_connection(
self, install_custom_tool_pkg, local_client, pf
):
# Prepare custom connection
from promptflow.connections import CustomConnection
conn = CustomConnection(name="custom_connection_2", secrets={"api_key": "test"}, configs={"api_url": "test"})
local_client.connections.create_or_update(conn)
> result = pf.run(
flow=f"{FLOWS_DIR}/flow_with_script_tool_with_custom_strong_type_connection",
data=f"{FLOWS_DIR}/flow_with_script_tool_with_custom_strong_type_connection/data.jsonl",
)
tests/sdk_cli_test/e2etests/test_flow_run.py:277:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
promptflow/_sdk/_pf_client.py:128: in run
return self.runs.create_or_update(run=run, **kwargs)
promptflow/_telemetry/activity.py:138: in wrapper
return f(self, *args, **kwargs)
promptflow/_sdk/operations/_run_operations.py:95: in create_or_update
created_run = RunSubmitter(run_operations=self).submit(run=run, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:249: in submit
self._run_bulk(run=run, stream=stream, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:277: in _run_bulk
self._submit_bulk_run(flow=flow, run=run, local_storage=local_storage)
promptflow/_sdk/operations/_run_submitter.py:282: in _submit_bulk_run
connections = SubmitterHelper.resolve_connections(flow=flow)
promptflow/_sdk/operations/_run_submitter.py:216: in resolve_connections
return get_local_connections_from_executable(executable=executable, client=client)
promptflow/_sdk/_utils.py:814: in get_local_connections_from_executable
if conn.api_base == "dummy_base" and conn.api_key == "dummy_key":
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <promptflow._sdk.entities._connection.CustomConnection object at 0x122ab6290>
item = 'api_base'
def __getattr__(self, item):
# Note: This is added for compatibility with promptflow.connections custom connection usage.
if item == "secrets":
# Usually obj.secrets will not reach here
# This is added to handle copy.deepcopy loop issue
return super().__getattribute__("secrets")
if item == "configs":
# Usually obj.configs will not reach here
# This is added to handle copy.deepcopy loop issue
return super().__getattribute__("configs")
if item in self.secrets:
logger.warning("Please use connection.secrets[key] to access secrets.")
return self.secrets[item]
if item in self.configs:
logger.warning("Please use connection.configs[key] to access configs.")
return self.configs[item]
> return super().__getattribute__(item)
E AttributeError: 'CustomConnection' object has no attribute 'api_base'
promptflow/_sdk/entities/_connection.py:831: AttributeError
Check warning on line 0 in tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun
github-actions / SDK CLI Test Result [yigao/recording_draft](https://github.com/microsoft/promptflow/actions/workflows/promptflow-sdk-cli-test.yml?query=branch:yigao/recording_draft++)
All 2 runs failed: test_custom_connection_overwrite (tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun)
artifacts/Test Results (Python 3.10) (OS macos-latest)/test-results.xml [took 0s]
artifacts/Test Results (Python 3.9) (OS ubuntu-latest)/test-results.xml [took 0s]
Raw output
AttributeError: 'CustomConnection' object has no attribute 'api_base'
self = <sdk_cli_test.e2etests.test_flow_run.TestFlowRun object at 0x1293fdb70>
local_client = <promptflow._sdk._pf_client.PFClient object at 0x129a9cee0>
local_custom_connection = <promptflow._sdk.entities._connection.CustomConnection object at 0x129a9ecb0>
pf = <promptflow._sdk._pf_client.PFClient object at 0x1299df640>
def test_custom_connection_overwrite(self, local_client, local_custom_connection, pf):
> result = pf.run(
flow=f"{FLOWS_DIR}/custom_connection_flow",
data=f"{DATAS_DIR}/env_var_names.jsonl",
connections={"print_env": {"connection": "test_custom_connection"}},
)
tests/sdk_cli_test/e2etests/test_flow_run.py:233:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
promptflow/_sdk/_pf_client.py:128: in run
return self.runs.create_or_update(run=run, **kwargs)
promptflow/_telemetry/activity.py:138: in wrapper
return f(self, *args, **kwargs)
promptflow/_sdk/operations/_run_operations.py:95: in create_or_update
created_run = RunSubmitter(run_operations=self).submit(run=run, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:249: in submit
self._run_bulk(run=run, stream=stream, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:277: in _run_bulk
self._submit_bulk_run(flow=flow, run=run, local_storage=local_storage)
promptflow/_sdk/operations/_run_submitter.py:282: in _submit_bulk_run
connections = SubmitterHelper.resolve_connections(flow=flow)
promptflow/_sdk/operations/_run_submitter.py:216: in resolve_connections
return get_local_connections_from_executable(executable=executable, client=client)
promptflow/_sdk/_utils.py:814: in get_local_connections_from_executable
if conn.api_base == "dummy_base" and conn.api_key == "dummy_key":
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <promptflow._sdk.entities._connection.CustomConnection object at 0x12a23b100>
item = 'api_base'
def __getattr__(self, item):
# Note: This is added for compatibility with promptflow.connections custom connection usage.
if item == "secrets":
# Usually obj.secrets will not reach here
# This is added to handle copy.deepcopy loop issue
return super().__getattribute__("secrets")
if item == "configs":
# Usually obj.configs will not reach here
# This is added to handle copy.deepcopy loop issue
return super().__getattribute__("configs")
if item in self.secrets:
logger.warning("Please use connection.secrets[key] to access secrets.")
return self.secrets[item]
if item in self.configs:
logger.warning("Please use connection.configs[key] to access configs.")
return self.configs[item]
> return super().__getattribute__(item)
E AttributeError: 'CustomConnection' object has no attribute 'api_base'
promptflow/_sdk/entities/_connection.py:831: AttributeError
Check warning on line 0 in tests.sdk_cli_test.e2etests.test_flow_test.TestFlowTest
github-actions / SDK CLI Test Result [yigao/recording_draft](https://github.com/microsoft/promptflow/actions/workflows/promptflow-sdk-cli-test.yml?query=branch:yigao/recording_draft++)
All 2 runs failed: test_pf_test_flow_with_script_tool_with_custom_strong_type_connection (tests.sdk_cli_test.e2etests.test_flow_test.TestFlowTest)
artifacts/Test Results (Python 3.10) (OS macos-latest)/test-results.xml [took 0s]
artifacts/Test Results (Python 3.9) (OS ubuntu-latest)/test-results.xml [took 0s]
Raw output
AttributeError: 'CustomConnection' object has no attribute 'api_base'
self = <sdk_cli_test.e2etests.test_flow_test.TestFlowTest object at 0x12b627910>
def test_pf_test_flow_with_script_tool_with_custom_strong_type_connection(self):
# Prepare custom connection
from promptflow.connections import CustomConnection
conn = CustomConnection(name="custom_connection_2", secrets={"api_key": "test"}, configs={"api_url": "test"})
_client.connections.create_or_update(conn)
inputs = {"text": "Hello World!"}
flow_path = Path(f"{FLOWS_DIR}/flow_with_script_tool_with_custom_strong_type_connection").absolute()
# Test that connection would be custom strong type in flow
> result = _client.test(flow=flow_path, inputs=inputs)
tests/sdk_cli_test/e2etests/test_flow_test.py:88:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
promptflow/_sdk/_pf_client.py:234: in test
return self.flows.test(
promptflow/_telemetry/activity.py:138: in wrapper
return f(self, *args, **kwargs)
promptflow/_sdk/operations/_flow_operations.py:71: in test
result = self._test(
promptflow/_sdk/operations/_flow_operations.py:123: in _test
return submitter.flow_test(
promptflow/_sdk/operations/_test_submitter.py:151: in flow_test
connections = SubmitterHelper.resolve_connections(flow=self.flow, client=self._client)
promptflow/_sdk/operations/_run_submitter.py:216: in resolve_connections
return get_local_connections_from_executable(executable=executable, client=client)
promptflow/_sdk/_utils.py:814: in get_local_connections_from_executable
if conn.api_base == "dummy_base" and conn.api_key == "dummy_key":
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <promptflow._sdk.entities._connection.CustomConnection object at 0x12e0abbe0>
item = 'api_base'
def __getattr__(self, item):
# Note: This is added for compatibility with promptflow.connections custom connection usage.
if item == "secrets":
# Usually obj.secrets will not reach here
# This is added to handle copy.deepcopy loop issue
return super().__getattribute__("secrets")
if item == "configs":
# Usually obj.configs will not reach here
# This is added to handle copy.deepcopy loop issue
return super().__getattribute__("configs")
if item in self.secrets:
logger.warning("Please use connection.secrets[key] to access secrets.")
return self.secrets[item]
if item in self.configs:
logger.warning("Please use connection.configs[key] to access configs.")
return self.configs[item]
> return super().__getattribute__(item)
E AttributeError: 'CustomConnection' object has no attribute 'api_base'
promptflow/_sdk/entities/_connection.py:831: AttributeError
Check notice on line 0 in .github
github-actions / SDK CLI Test Result [yigao/recording_draft](https://github.com/microsoft/promptflow/actions/workflows/promptflow-sdk-cli-test.yml?query=branch:yigao/recording_draft++)
14 skipped tests found
There are 14 skipped tests, see "Raw output" for the full list of skipped tests.
Raw output
tests.sdk_cli_azure_test.e2etests.test_cli_with_azure.TestCliWithAzure ‑ test_basic_flow_with_package_tool_with_custom_strong_type_connection
tests.sdk_cli_azure_test.e2etests.test_connection_operations.TestConnectionOperations ‑ test_connection_get_create_delete
tests.sdk_cli_azure_test.e2etests.test_connection_operations.TestConnectionOperations ‑ test_custom_connection_create
tests.sdk_cli_azure_test.e2etests.test_flow_operations.TestFlow ‑ test_crud_flow
tests.sdk_cli_azure_test.e2etests.test_flow_operations.TestFlow ‑ test_flow
tests.sdk_cli_azure_test.e2etests.test_flow_operations.TestFlow ‑ test_flow_no_variants_remote_file
tests.sdk_cli_azure_test.e2etests.test_flow_operations.TestFlow ‑ test_flow_without_variants
tests.sdk_cli_azure_test.e2etests.test_flow_operations.TestFlow ‑ test_list_flows
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_automatic_runtime_creation_failure
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_run_with_symbolic
tests.sdk_cli_azure_test.unittests.test_flow_entity.TestFlow ‑ test_load_flow
tests.sdk_cli_azure_test.unittests.test_flow_entity.TestFlow ‑ test_load_flow_from_remote_storage
tests.sdk_cli_test.e2etests.test_flow_local_operations.TestFlowLocalOperations ‑ test_flow_generate_tools_meta_with_pkg_tool_with_custom_strong_type_connection
tests.sdk_cli_test.e2etests.test_flow_test.TestFlowTest ‑ test_pf_test_with_additional_includes
Check notice on line 0 in .github
github-actions / SDK CLI Test Result [yigao/recording_draft](https://github.com/microsoft/promptflow/actions/workflows/promptflow-sdk-cli-test.yml?query=branch:yigao/recording_draft++)
305 tests found
There are 305 tests, see "Raw output" for the full list of tests.
Raw output
tests.sdk_cli_azure_test.e2etests.test_arm_connection_operations.TestArmConnectionOperations ‑ test_get_connection
tests.sdk_cli_azure_test.e2etests.test_cli.TestCli ‑ test_flow_chat
tests.sdk_cli_azure_test.e2etests.test_cli.TestCli ‑ test_pf_flow_test
tests.sdk_cli_azure_test.e2etests.test_cli_with_azure.TestCliWithAzure ‑ test_basic_flow_run_bulk_without_env
tests.sdk_cli_azure_test.e2etests.test_cli_with_azure.TestCliWithAzure ‑ test_basic_flow_with_package_tool_with_custom_strong_type_connection
tests.sdk_cli_azure_test.e2etests.test_cli_with_azure.TestCliWithAzure ‑ test_run_with_remote_data
tests.sdk_cli_azure_test.e2etests.test_connection_operations.TestConnectionOperations ‑ test_connection_get_create_delete
tests.sdk_cli_azure_test.e2etests.test_connection_operations.TestConnectionOperations ‑ test_custom_connection_create
tests.sdk_cli_azure_test.e2etests.test_connection_operations.TestConnectionOperations ‑ test_get_connection
tests.sdk_cli_azure_test.e2etests.test_connection_operations.TestConnectionOperations ‑ test_list_connection_spec
tests.sdk_cli_azure_test.e2etests.test_flow_in_azure_ml.TestFlowInAzureML ‑ test_flow_as_component[parallel]
tests.sdk_cli_azure_test.e2etests.test_flow_operations.TestFlow ‑ test_crud_flow
tests.sdk_cli_azure_test.e2etests.test_flow_operations.TestFlow ‑ test_flow
tests.sdk_cli_azure_test.e2etests.test_flow_operations.TestFlow ‑ test_flow_no_variants_remote_file
tests.sdk_cli_azure_test.e2etests.test_flow_operations.TestFlow ‑ test_flow_without_variants
tests.sdk_cli_azure_test.e2etests.test_flow_operations.TestFlow ‑ test_list_flows
tests.sdk_cli_azure_test.e2etests.test_flow_serve ‑ test_serving_api
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_archive_and_restore_run
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_automatic_runtime
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_automatic_runtime_creation_failure
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_automatic_runtime_creation_user_aml_token
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_automatic_runtime_with_environment
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_basic_evaluation
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_exp_id
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_flow_id_in_submission
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_get_detail_against_partial_fail_run
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_get_invalid_run_cases
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_input_mapping_with_dict
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_list_runs
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_pf_run_with_env_var
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_run_bulk
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_run_bulk_from_yaml
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_run_bulk_not_exist
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_run_bulk_without_retry
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_run_data_not_provided
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_run_display_name_with_macro
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_run_submission_exception
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_run_with_additional_includes
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_run_with_connection_overwrite
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_run_with_env_overwrite
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_run_with_remote_data
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_run_with_symbolic
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_run_without_dump
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_show_metrics
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_show_run
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_show_run_details
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_stream_failed_run_logs
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_stream_run_logs
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_submit_run_user_aml_token
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_tools_json_ignored
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_update_run
tests.sdk_cli_azure_test.e2etests.test_run_operations.TestFlowRun ‑ test_vnext_workspace_base_url
tests.sdk_cli_azure_test.e2etests.test_telemetry.TestTelemetry ‑ test_call_from_extension
tests.sdk_cli_azure_test.e2etests.test_telemetry.TestTelemetry ‑ test_custom_event
tests.sdk_cli_azure_test.e2etests.test_telemetry.TestTelemetry ‑ test_log_for_eu_user
tests.sdk_cli_azure_test.e2etests.test_telemetry.TestTelemetry ‑ test_logging_handler
tests.sdk_cli_azure_test.unittests.test_arm_connection_build ‑ test_build_azure_openai_connection_from_rest_object
tests.sdk_cli_azure_test.unittests.test_arm_connection_build ‑ test_build_cognitive_search_connection_from_rest_object
tests.sdk_cli_azure_test.unittests.test_arm_connection_build ‑ test_build_cognitive_service_category_connection_from_rest_object
tests.sdk_cli_azure_test.unittests.test_arm_connection_build ‑ test_build_connection_missing_metadata
tests.sdk_cli_azure_test.unittests.test_arm_connection_build ‑ test_build_connection_unknown_category
tests.sdk_cli_azure_test.unittests.test_arm_connection_build ‑ test_build_custom_keys_connection_from_rest_object
tests.sdk_cli_azure_test.unittests.test_arm_connection_build ‑ test_build_default_azure_openai_connection_missing_metadata
tests.sdk_cli_azure_test.unittests.test_cli.TestAzureCli ‑ test_get_workspace_from_config
tests.sdk_cli_azure_test.unittests.test_cli.TestAzureCli ‑ test_pf_azure_version
tests.sdk_cli_azure_test.unittests.test_cli.TestAzureCli ‑ test_run_archive
tests.sdk_cli_azure_test.unittests.test_cli.TestAzureCli ‑ test_run_list_runs
tests.sdk_cli_azure_test.unittests.test_cli.TestAzureCli ‑ test_run_restore
tests.sdk_cli_azure_test.unittests.test_cli.TestAzureCli ‑ test_run_show
tests.sdk_cli_azure_test.unittests.test_cli.TestAzureCli ‑ test_run_show_details
tests.sdk_cli_azure_test.unittests.test_cli.TestAzureCli ‑ test_run_show_metrics
tests.sdk_cli_azure_test.unittests.test_cli.TestAzureCli ‑ test_run_update
tests.sdk_cli_azure_test.unittests.test_cli.TestAzureCli ‑ test_run_visualize
tests.sdk_cli_azure_test.unittests.test_flow_entity.TestFlow ‑ test_ignore_files_in_flow
tests.sdk_cli_azure_test.unittests.test_flow_entity.TestFlow ‑ test_load_flow
tests.sdk_cli_azure_test.unittests.test_flow_entity.TestFlow ‑ test_load_flow_from_remote_storage
tests.sdk_cli_azure_test.unittests.test_flow_entity.TestFlow ‑ test_load_yaml_run_with_resources
tests.sdk_cli_azure_test.unittests.test_pf_client_azure.TestPFClientAzure ‑ test_input_output_portal_url_parser
tests.sdk_cli_azure_test.unittests.test_pf_client_azure.TestPFClientAzure ‑ test_wrong_client_parameters
tests.sdk_cli_azure_test.unittests.test_run_entity.TestRun ‑ test_flow_id
tests.sdk_cli_azure_test.unittests.test_run_entity.TestRun ‑ test_input_mapping_types
tests.sdk_cli_azure_test.unittests.test_utils.TestUtils ‑ test_forbidden_new_caller
tests.sdk_cli_azure_test.unittests.test_utils.TestUtils ‑ test_url_parse
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_basic_flow_run
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_basic_flow_run_batch_and_eval
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_chat_flow_with_conditional
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_connection_create_update[azure_openai_connection.yaml-expected0-update_item0]
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_connection_create_update[custom_connection.yaml-expected1-update_item1]
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_connection_create_update[custom_strong_type_connection.yaml-expected2-update_item2]
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_connection_overwrite
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_create_with_set
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_environment_variable_overwrite
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_flow_build
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_flow_chat
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_flow_init
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_flow_init_intent_copilot
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_flow_init_with_connection_and_deployment
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_flow_test_inputs
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_flow_test_with_default_chat_history
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_flow_test_with_environment_variable
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_flow_test_with_image_input_and_output
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_flow_test_with_user_defined_chat_history
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_flow_with_exception
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_format_cli_exception
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_init_chat_flow
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_init_eval_flow
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_input_with_dict_val
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_pf_flow_test
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_pf_flow_test_debug_single_node
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_pf_flow_test_single_node
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_pf_flow_test_with_additional_includes
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_pf_flow_test_with_symbolic
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_pf_flow_with_variant
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_pf_run_no_stream_log
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_pf_run_with_stream_log
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_pf_version
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_submit_batch_variant
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_submit_run_with_yaml
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_tool_init
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_tool_list
tests.sdk_cli_test.e2etests.test_cli.TestCli ‑ test_visualize_ignore_space
tests.sdk_cli_test.e2etests.test_connection.TestConnection ‑ test_connection_get_and_update
tests.sdk_cli_test.e2etests.test_connection.TestConnection ‑ test_connection_operations
tests.sdk_cli_test.e2etests.test_connection.TestConnection ‑ test_custom_connection_get_and_update
tests.sdk_cli_test.e2etests.test_connection.TestConnection ‑ test_upsert_connection_from_file[azure_openai_connection.yaml-expected_updated_item0-expected_secret_item0]
tests.sdk_cli_test.e2etests.test_connection.TestConnection ‑ test_upsert_connection_from_file[custom_connection.yaml-expected_updated_item1-expected_secret_item1]
tests.sdk_cli_test.e2etests.test_custom_strong_type_connection.TestCustomStrongTypeConnection ‑ test_connection_get_and_update
tests.sdk_cli_test.e2etests.test_custom_strong_type_connection.TestCustomStrongTypeConnection ‑ test_connection_get_and_update_with_key
tests.sdk_cli_test.e2etests.test_custom_strong_type_connection.TestCustomStrongTypeConnection ‑ test_connection_operations
tests.sdk_cli_test.e2etests.test_custom_strong_type_connection.TestCustomStrongTypeConnection ‑ test_connection_update
tests.sdk_cli_test.e2etests.test_custom_strong_type_connection.TestCustomStrongTypeConnection ‑ test_upsert_connection_from_file[custom_strong_type_connection.yaml-expected_updated_item0-expected_secret_item0]
tests.sdk_cli_test.e2etests.test_flow_local_operations.TestFlowLocalOperations ‑ test_flow_build_as_docker
tests.sdk_cli_test.e2etests.test_flow_local_operations.TestFlowLocalOperations ‑ test_flow_build_as_docker_with_additional_includes
tests.sdk_cli_test.e2etests.test_flow_local_operations.TestFlowLocalOperations ‑ test_flow_build_as_docker_with_variant
tests.sdk_cli_test.e2etests.test_flow_local_operations.TestFlowLocalOperations ‑ test_flow_build_flow_only
tests.sdk_cli_test.e2etests.test_flow_local_operations.TestFlowLocalOperations ‑ test_flow_build_generate_flow_tools_json
tests.sdk_cli_test.e2etests.test_flow_local_operations.TestFlowLocalOperations ‑ test_flow_generate_tools_meta
tests.sdk_cli_test.e2etests.test_flow_local_operations.TestFlowLocalOperations ‑ test_flow_generate_tools_meta_with_pkg_tool_with_custom_strong_type_connection
tests.sdk_cli_test.e2etests.test_flow_local_operations.TestFlowLocalOperations ‑ test_flow_generate_tools_meta_with_script_tool_with_custom_strong_type_connection
tests.sdk_cli_test.e2etests.test_flow_local_operations.TestFlowLocalOperations ‑ test_flow_validate_generate_flow_tools_json
tests.sdk_cli_test.e2etests.test_flow_local_operations.TestFlowLocalOperations ‑ test_flow_validation_failed
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_basic_evaluation
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_basic_flow_bulk_run
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_basic_flow_with_package_tool_with_custom_strong_type_connection
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_basic_flow_with_script_tool_with_custom_strong_type_connection
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_basic_flow_with_variant
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_basic_run_bulk
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_connection_overwrite_file
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_connection_overwrite_model
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_create_run_with_tags
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_custom_connection_overwrite
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_error_message_dump
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_eval_run_not_exist
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_flow_bulk_run_with_additional_includes
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_flow_demo
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_flow_with_only_static_values
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_get_detail_against_partial_fail_run
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_get_detail_format
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_get_details
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_get_metrics_format
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_incomplete_run_visualize
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_input_mapping_source_not_found_error
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_input_mapping_with_dict
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_pf_run_with_env_overwrite
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_referenced_output_not_exist
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_resolve_connection
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_run_bulk_error
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_run_data_not_provided
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_run_display_name
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_run_dump
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_run_exist_error
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_run_list
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_run_local_storage_structure
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_run_logs
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_run_reference_failed_run
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_run_snapshot_with_flow_tools_json
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_run_with_connection
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_run_with_connection_overwrite
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_run_with_connection_overwrite_non_exist
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_run_with_env_overwrite
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_stream_incomplete_run_summary
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_stream_run_summary
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_submit_run_from_yaml
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_system_metrics_in_properties
tests.sdk_cli_test.e2etests.test_flow_run.TestFlowRun ‑ test_visualize_run
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_evaluation_flow_serving_api
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_serving_api
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_stream_llm_chat[*/*-200-application/json]
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_stream_llm_chat[-200-application/json]
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_stream_llm_chat[application/json, */*-200-application/json]
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_stream_llm_chat[application/json-200-application/json]
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_stream_llm_chat[text/event-stream, application/json-200-text/event-stream; charset=utf-8]
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_stream_llm_chat[text/event-stream-200-text/event-stream; charset=utf-8]
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_stream_llm_chat[text/html-406-application/json]
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_stream_python_nonstream_tools[*/*-200-application/json]
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_stream_python_nonstream_tools[-200-application/json]
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_stream_python_nonstream_tools[application/json, */*-200-application/json]
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_stream_python_nonstream_tools[application/json-200-application/json]
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_stream_python_nonstream_tools[text/event-stream, application/json-200-application/json]
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_stream_python_nonstream_tools[text/event-stream-406-application/json]
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_stream_python_stream_tools[*/*-200-application/json]
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_stream_python_stream_tools[-200-application/json]
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_stream_python_stream_tools[application/json, */*-200-application/json]
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_stream_python_stream_tools[application/json-200-application/json]
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_stream_python_stream_tools[text/event-stream, application/json-200-text/event-stream; charset=utf-8]
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_stream_python_stream_tools[text/event-stream-200-text/event-stream; charset=utf-8]
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_stream_python_stream_tools[text/html-406-application/json]
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_swagger
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_unknown_api
tests.sdk_cli_test.e2etests.test_flow_serve ‑ test_user_agent
tests.sdk_cli_test.e2etests.test_flow_test.TestFlowTest ‑ test_generate_tool_meta_in_additional_folder
tests.sdk_cli_test.e2etests.test_flow_test.TestFlowTest ‑ test_node_test_with_connection_input
tests.sdk_cli_test.e2etests.test_flow_test.TestFlowTest ‑ test_pf_flow_test_with_exception
tests.sdk_cli_test.e2etests.test_flow_test.TestFlowTest ‑ test_pf_flow_test_with_symbolic
tests.sdk_cli_test.e2etests.test_flow_test.TestFlowTest ‑ test_pf_flow_with_aggregation
tests.sdk_cli_test.e2etests.test_flow_test.TestFlowTest ‑ test_pf_test_flow
tests.sdk_cli_test.e2etests.test_flow_test.TestFlowTest ‑ test_pf_test_flow_with_package_tool_with_custom_connection_as_input_value
tests.sdk_cli_test.e2etests.test_flow_test.TestFlowTest ‑ test_pf_test_flow_with_package_tool_with_custom_strong_type_connection
tests.sdk_cli_test.e2etests.test_flow_test.TestFlowTest ‑ test_pf_test_flow_with_script_tool_with_custom_strong_type_connection
tests.sdk_cli_test.e2etests.test_flow_test.TestFlowTest ‑ test_pf_test_flow_with_variant
tests.sdk_cli_test.e2etests.test_flow_test.TestFlowTest ‑ test_pf_test_node
tests.sdk_cli_test.e2etests.test_flow_test.TestFlowTest ‑ test_pf_test_with_additional_includes
tests.sdk_cli_test.e2etests.test_flow_test.TestFlowTest ‑ test_pf_test_with_non_english_input
tests.sdk_cli_test.e2etests.test_flow_test.TestFlowTest ‑ test_pf_test_with_streaming_output
tests.sdk_cli_test.e2etests.test_orm.TestRunInfo ‑ test_archive
tests.sdk_cli_test.e2etests.test_orm.TestRunInfo ‑ test_get
tests.sdk_cli_test.e2etests.test_orm.TestRunInfo ‑ test_get_not_exist
tests.sdk_cli_test.e2etests.test_orm.TestRunInfo ‑ test_list_order_by_created_time_desc
tests.sdk_cli_test.e2etests.test_orm.TestRunInfo ‑ test_null_type_and_display_name
tests.sdk_cli_test.e2etests.test_orm.TestRunInfo ‑ test_restore
tests.sdk_cli_test.e2etests.test_orm.TestRunInfo ‑ test_update
tests.sdk_cli_test.unittests.test_config.TestConfig ‑ test_config_instance
tests.sdk_cli_test.unittests.test_config.TestConfig ‑ test_get_config
tests.sdk_cli_test.unittests.test_config.TestConfig ‑ test_get_or_set_installation_id
tests.sdk_cli_test.unittests.test_config.TestConfig ‑ test_set_config
tests.sdk_cli_test.unittests.test_connection.TestConnection ‑ test_connection_load_dump[azure_content_safety_connection.yaml-AzureContentSafetyConnection-init_param3-expected3]
tests.sdk_cli_test.unittests.test_connection.TestConnection ‑ test_connection_load_dump[azure_openai_connection.yaml-AzureOpenAIConnection-init_param0-expected0]
tests.sdk_cli_test.unittests.test_connection.TestConnection ‑ test_connection_load_dump[cognitive_search_connection.yaml-CognitiveSearchConnection-init_param4-expected4]
tests.sdk_cli_test.unittests.test_connection.TestConnection ‑ test_connection_load_dump[custom_connection.yaml-CustomConnection-init_param2-expected2]
tests.sdk_cli_test.unittests.test_connection.TestConnection ‑ test_connection_load_dump[form_recognizer_connection.yaml-FormRecognizerConnection-init_param6-expected6]
tests.sdk_cli_test.unittests.test_connection.TestConnection ‑ test_connection_load_dump[openai_connection.yaml-OpenAIConnection-init_param1-expected1]
tests.sdk_cli_test.unittests.test_connection.TestConnection ‑ test_connection_load_dump[qdrant_connection.yaml-QdrantConnection-init_param7-expected7]
tests.sdk_cli_test.unittests.test_connection.TestConnection ‑ test_connection_load_dump[serp_connection.yaml-SerpConnection-init_param5-expected5]
tests.sdk_cli_test.unittests.test_connection.TestConnection ‑ test_connection_load_dump[weaviate_connection.yaml-WeaviateConnection-init_param8-expected8]
tests.sdk_cli_test.unittests.test_connection.TestConnection ‑ test_connection_load_from_env
tests.sdk_cli_test.unittests.test_connection.TestConnection ‑ test_connection_load_from_env_file_bad_case
tests.sdk_cli_test.unittests.test_connection.TestConnection ‑ test_convert_to_custom_strong_type
tests.sdk_cli_test.unittests.test_connection.TestConnection ‑ test_to_execution_connection_dict
tests.sdk_cli_test.unittests.test_connection.TestConnection ‑ test_validate_and_encrypt_secrets
tests.sdk_cli_test.unittests.test_connection.TestConnection ‑ test_validate_and_interactive_get_secrets
tests.sdk_cli_test.unittests.test_flow_invoker.TestFlowInvoker ‑ test_flow_invoker_custom_connection_provider
tests.sdk_cli_test.unittests.test_flow_invoker.TestFlowInvoker ‑ test_flow_invoker_unsupported_connection_provider
tests.sdk_cli_test.unittests.test_local_storage_operations.TestLocalStorageOperations ‑ test_outputs_padding
tests.sdk_cli_test.unittests.test_orm.TestSchemaManagement ‑ test_db_without_schema_info
tests.sdk_cli_test.unittests.test_orm.TestSchemaManagement ‑ test_fixed_version
tests.sdk_cli_test.unittests.test_orm.TestSchemaManagement ‑ test_version_across_same_schema_version
tests.sdk_cli_test.unittests.test_orm.TestSchemaManagement ‑ test_version_downgrade
tests.sdk_cli_test.unittests.test_orm.TestSchemaManagement ‑ test_version_mixing
tests.sdk_cli_test.unittests.test_orm.TestSchemaManagement ‑ test_version_upgrade
tests.sdk_cli_test.unittests.test_orm.TestTransaction ‑ test_commit
tests.sdk_cli_test.unittests.test_orm.TestTransaction ‑ test_exception_during_transaction
tests.sdk_cli_test.unittests.test_orm.TestTransaction ‑ test_rollback
tests.sdk_cli_test.unittests.test_pf_client.TestPFClient ‑ test_connection_provider
tests.sdk_cli_test.unittests.test_pf_client.TestPFClient ‑ test_local_azure_connection_extract_workspace
tests.sdk_cli_test.unittests.test_pf_client.TestPFClient ‑ test_pf_client_user_agent
tests.sdk_cli_test.unittests.test_run.TestRun ‑ test_data_not_exist_validation_error
tests.sdk_cli_test.unittests.test_run.TestRun ‑ test_dot_env_resolve
tests.sdk_cli_test.unittests.test_run.TestRun ‑ test_flow_run_with_non_english_inputs
tests.sdk_cli_test.unittests.test_run.TestRun ‑ test_invalid_yaml[tests/test_configs/runs/illegal/extra_field.yaml-Unknown field]
tests.sdk_cli_test.unittests.test_run.TestRun ‑ test_invalid_yaml[tests/test_configs/runs/illegal/non_exist_data.yaml-Can't find directory or file]
tests.sdk_cli_test.unittests.test_run.TestRun ‑ test_load_run
tests.sdk_cli_test.unittests.test_run.TestRun ‑ test_overwrite_connections
tests.sdk_cli_test.unittests.test_run.TestRun ‑ test_overwrite_connections_invalid[connections0-Unsupported llm connection overwrite keys]
tests.sdk_cli_test.unittests.test_run.TestRun ‑ test_overwrite_connections_invalid[connections2-Node not_exist not found in flow]
tests.sdk_cli_test.unittests.test_run.TestRun ‑ test_overwrite_connections_invalid[connections3-Invalid connection overwrite format: 1, only dict is supported.]
tests.sdk_cli_test.unittests.test_run.TestRun ‑ test_overwrite_connections_invalid[str-Invalid connections overwrite format: str]
tests.sdk_cli_test.unittests.test_run.TestRun ‑ test_overwrite_variant
tests.sdk_cli_test.unittests.test_run.TestRun ‑ test_overwrite_variant_context
tests.sdk_cli_test.unittests.test_run.TestRun ‑ test_run_bulk_invalid_params
tests.sdk_cli_test.unittests.test_run.TestRun ‑ test_submit
tests.sdk_cli_test.unittests.test_utils.TestCLIUtils ‑ test_append_to_dict_action
tests.sdk_cli_test.unittests.test_utils.TestCLIUtils ‑ test_build_sorted_column_widths_tuple_list
tests.sdk_cli_test.unittests.test_utils.TestCLIUtils ‑ test_calculate_column_widths
tests.sdk_cli_test.unittests.test_utils.TestCLIUtils ‑ test_list_of_dict_to_nested_dict
tests.sdk_cli_test.unittests.test_utils.TestUtils ‑ test_concurrent_execution_of_refresh_connections_dir[1]
tests.sdk_cli_test.unittests.test_utils.TestUtils ‑ test_concurrent_execution_of_refresh_connections_dir[2]
tests.sdk_cli_test.unittests.test_utils.TestUtils ‑ test_concurrent_execution_of_refresh_connections_dir[4]
tests.sdk_cli_test.unittests.test_utils.TestUtils ‑ test_concurrent_execution_of_refresh_connections_dir[8]
tests.sdk_cli_test.unittests.test_utils.TestUtils ‑ test_encrypt_decrypt_value
tests.sdk_cli_test.unittests.test_utils.TestUtils ‑ test_generate_connections_dir[/Users/fake_user/anaconda3/envs/pf/bin/python3.10-e3f33eadd9be376014eb75a688930930ca83c056]
tests.sdk_cli_test.unittests.test_utils.TestUtils ‑ test_generate_connections_dir[D:\\Tools\\Anaconda3\\envs\\pf\\python.exe-a9620c3cdb7ccf3ec9f4005e5b19c12d1e1fef80]
tests.sdk_cli_test.unittests.test_utils.TestUtils ‑ test_generate_flow_tools_json
tests.sdk_cli_test.unittests.test_utils.TestUtils ‑ test_generate_flow_tools_json_expecting_fail
tests.sdk_cli_test.unittests.test_utils.TestUtils ‑ test_load_10k_data[./tests/test_configs/datas/load_data_cases/10k.jsonl]
tests.sdk_cli_test.unittests.test_utils.TestUtils ‑ test_load_10k_data[./tests/test_configs/datas/load_data_cases/10k]
tests.sdk_cli_test.unittests.test_utils.TestUtils ‑ test_load_data[./tests/test_configs/datas/load_data_cases/colors.csv]
tests.sdk_cli_test.unittests.test_utils.TestUtils ‑ test_load_data[./tests/test_configs/datas/load_data_cases/colors.json]
tests.sdk_cli_test.unittests.test_utils.TestUtils ‑ test_load_data[./tests/test_configs/datas/load_data_cases/colors.jsonl]
tests.sdk_cli_test.unittests.test_utils.TestUtils ‑ test_load_data[./tests/test_configs/datas/load_data_cases/colors.parquet]
tests.sdk_cli_test.unittests.test_utils.TestUtils ‑ test_load_data[./tests/test_configs/datas/load_data_cases/colors.tsv]
tests.sdk_cli_test.unittests.test_utils.TestUtils ‑ test_override_connection_config_with_environment_variable
tests.sdk_cli_test.unittests.test_utils.TestUtils ‑ test_refresh_connections_dir
tests.sdk_cli_test.unittests.test_utils.TestUtils ‑ test_resolve_connections_environment_variable_reference
tests.sdk_cli_test.unittests.test_utils.TestUtils ‑ test_snake_to_camel
tests.sdk_cli_test.unittests.test_utils.TestUtils ‑ test_sqlite_retry
Loading