AttributeError: 'NoneType' object has no attribute 'isoformat'
self = <sdk_cli_test.e2etests.test_cli.TestCli object at 0x7f7b480159d0>

    def test_submit_run_with_yaml(self):
        run_id = str(uuid.uuid4())
        f = io.StringIO()
        with contextlib.redirect_stdout(f):
>           run_pf_command(
                "run",
                "create",
                "--file",
                "./sample_bulk_run.yaml",
                "--name",
                run_id,
                cwd=f"{RUNS_DIR}",
            )

tests/sdk_cli_test/e2etests/test_cli.py:121: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/sdk_cli_test/e2etests/test_cli.py:45: in run_pf_command
    main()
promptflow/_cli/_pf/entry.py:97: in main
    entry(command_args)
promptflow/_cli/_pf/entry.py:79: in entry
    raise ex
promptflow/_cli/_pf/entry.py:65: in entry
    dispatch_run_commands(args)
promptflow/_cli/_pf/_run.py:391: in dispatch_run_commands
    create_run(create_func=_create_run, args=args)
promptflow/_cli/_utils.py:363: in wrapper
    raise e
promptflow/_cli/_utils.py:351: in wrapper
    return func(*args, **kwargs)
promptflow/_cli/_pf/_run.py:608: in create_run
    run = create_func(run=run, stream=stream)
promptflow/_sdk/_run_functions.py:14: in _create_run
    return client.runs.create_or_update(run=run, **kwargs)
promptflow/_telemetry/activity.py:138: in wrapper
    return f(self, *args, **kwargs)
promptflow/_sdk/operations/_run_operations.py:95: in create_or_update
    created_run = RunSubmitter(run_operations=self).submit(run=run, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:249: in submit
    self._run_bulk(run=run, stream=stream, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:277: in _run_bulk
    self._submit_bulk_run(flow=flow, run=run, local_storage=local_storage)
promptflow/_sdk/operations/_run_submitter.py:327: in _submit_bulk_run
    raise e
promptflow/_sdk/operations/_run_submitter.py:304: in _submit_bulk_run
    bulk_result = flow_executor.exec_bulk(mapped_inputs, run_id=run_id)
promptflow/executor/flow_executor.py:763: in exec_bulk
    line_results = self._exec_batch_with_threads(inputs, run_id, validate_inputs=validate_inputs)
promptflow/executor/flow_executor.py:467: in _exec_batch_with_threads
    result_list = pool.run(zip(line_number, batch_inputs))
promptflow/executor/_line_execution_process_pool.py:298: in run
    self._pool.starmap(
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:372: in starmap
    return self._map_async(func, iterable, starmapstar, chunksize).get()
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:771: in get
    raise self._value
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:125: in worker
    result = (True, func(*args, **kwds))
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:51: in starmapstar
    return list(itertools.starmap(args[0], args[1]))
promptflow/executor/_line_execution_process_pool.py:219: in _timeout_process_wrapper
    self._storage.persist_node_run(message)
promptflow/_sdk/operations/_local_storage_operations.py:386: in persist_node_run
    node_run_record = NodeRunRecord.from_run_info(run_info)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

node_run_info = RunInfo(node='summarize_text_content', flow_run_id='3879063b-8b83-4867-9a83-0cdf4a27e854', run_id='3879063b-8b83-4867-...pi_calls=None, variant_id='', cached_run_id=None, cached_flow_run_id=None, logs=None, system_metrics=None, result=None)

    @staticmethod
    def from_run_info(node_run_info: NodeRunInfo) -> "NodeRunRecord":
        return NodeRunRecord(
            NodeName=node_run_info.node,
            line_number=node_run_info.index,
            run_info=serialize(node_run_info),
            start_time=node_run_info.start_time.isoformat(),
>           end_time=node_run_info.end_time.isoformat(),
            status=node_run_info.status.value,
        )
E       AttributeError: 'NoneType' object has no attribute 'isoformat'

promptflow/_sdk/operations/_local_storage_operations.py:123: AttributeError

AttributeError: 'NoneType' object has no attribute 'isoformat'
self = <sdk_cli_test.e2etests.test_cli.TestCli object at 0x7f7b47fde4c0>
local_client = <promptflow._sdk._pf_client.PFClient object at 0x7f7b47a94610>

    def test_create_with_set(self, local_client):
        run_id = str(uuid.uuid4())
        display_name = "test_run"
        description = "test description"
        run_pf_command(
            "run",
            "create",
            "--name",
            run_id,
            "--flow",
            f"{FLOWS_DIR}/print_env_var",
            "--data",
            f"{DATAS_DIR}/env_var_names.jsonl",
            "--environment-variables",
            "API_BASE=${azure_open_ai_connection.api_base}",
            "--set",
            f"display_name={display_name}",
            "tags.key=val",
            f"description={description}",
        )
        run = local_client.runs.get(run_id)
        assert display_name in run.display_name
        assert run.tags == {"key": "val"}
        assert run.description == description
    
        run_id = str(uuid.uuid4())
>       run_pf_command(
            "run",
            "create",
            "--file",
            "./sample_bulk_run.yaml",
            "--name",
            run_id,
            "--set",
            f"display_name={display_name}",
            "tags.key=val",
            f"description={description}",
            cwd=f"{RUNS_DIR}",
        )

tests/sdk_cli_test/e2etests/test_cli.py:253: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/sdk_cli_test/e2etests/test_cli.py:45: in run_pf_command
    main()
promptflow/_cli/_pf/entry.py:97: in main
    entry(command_args)
promptflow/_cli/_pf/entry.py:79: in entry
    raise ex
promptflow/_cli/_pf/entry.py:65: in entry
    dispatch_run_commands(args)
promptflow/_cli/_pf/_run.py:391: in dispatch_run_commands
    create_run(create_func=_create_run, args=args)
promptflow/_cli/_utils.py:363: in wrapper
    raise e
promptflow/_cli/_utils.py:351: in wrapper
    return func(*args, **kwargs)
promptflow/_cli/_pf/_run.py:608: in create_run
    run = create_func(run=run, stream=stream)
promptflow/_sdk/_run_functions.py:14: in _create_run
    return client.runs.create_or_update(run=run, **kwargs)
promptflow/_telemetry/activity.py:138: in wrapper
    return f(self, *args, **kwargs)
promptflow/_sdk/operations/_run_operations.py:95: in create_or_update
    created_run = RunSubmitter(run_operations=self).submit(run=run, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:249: in submit
    self._run_bulk(run=run, stream=stream, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:277: in _run_bulk
    self._submit_bulk_run(flow=flow, run=run, local_storage=local_storage)
promptflow/_sdk/operations/_run_submitter.py:327: in _submit_bulk_run
    raise e
promptflow/_sdk/operations/_run_submitter.py:304: in _submit_bulk_run
    bulk_result = flow_executor.exec_bulk(mapped_inputs, run_id=run_id)
promptflow/executor/flow_executor.py:763: in exec_bulk
    line_results = self._exec_batch_with_threads(inputs, run_id, validate_inputs=validate_inputs)
promptflow/executor/flow_executor.py:467: in _exec_batch_with_threads
    result_list = pool.run(zip(line_number, batch_inputs))
promptflow/executor/_line_execution_process_pool.py:298: in run
    self._pool.starmap(
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:372: in starmap
    return self._map_async(func, iterable, starmapstar, chunksize).get()
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:771: in get
    raise self._value
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:125: in worker
    result = (True, func(*args, **kwds))
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:51: in starmapstar
    return list(itertools.starmap(args[0], args[1]))
promptflow/executor/_line_execution_process_pool.py:219: in _timeout_process_wrapper
    self._storage.persist_node_run(message)
promptflow/_sdk/operations/_local_storage_operations.py:386: in persist_node_run
    node_run_record = NodeRunRecord.from_run_info(run_info)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

node_run_info = RunInfo(node='summarize_text_content', flow_run_id='d6bcf151-a946-4676-aab3-9ea170bcec88', run_id='d6bcf151-a946-4676-...pi_calls=None, variant_id='', cached_run_id=None, cached_flow_run_id=None, logs=None, system_metrics=None, result=None)

    @staticmethod
    def from_run_info(node_run_info: NodeRunInfo) -> "NodeRunRecord":
        return NodeRunRecord(
            NodeName=node_run_info.node,
            line_number=node_run_info.index,
            run_info=serialize(node_run_info),
            start_time=node_run_info.start_time.isoformat(),
>           end_time=node_run_info.end_time.isoformat(),
            status=node_run_info.status.value,
        )
E       AttributeError: 'NoneType' object has no attribute 'isoformat'

promptflow/_sdk/operations/_local_storage_operations.py:123: AttributeError

BaseException: Record item not found in folder /home/runner/work/promptflow/promptflow/src/promptflow/tests/test_configs/flows/web_classification.
self = <sdk_cli_test.e2etests.test_cli.TestCli object at 0x7f7b47fde730>

    def test_pf_flow_test(self):
>       run_pf_command(
            "flow",
            "test",
            "--flow",
            f"{FLOWS_DIR}/web_classification",
            "--inputs",
            "url=https://www.youtube.com/watch?v=o5ZQyXaAv1g",
            "answer=Channel",
            "evidence=Url",
        )

tests/sdk_cli_test/e2etests/test_cli.py:271: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/sdk_cli_test/e2etests/test_cli.py:45: in run_pf_command
    main()
promptflow/_cli/_pf/entry.py:97: in main
    entry(command_args)
promptflow/_cli/_pf/entry.py:61: in entry
    dispatch_flow_commands(args)
promptflow/_cli/_pf/_flow.py:69: in dispatch_flow_commands
    test_flow(args)
promptflow/_cli/_utils.py:351: in wrapper
    return func(*args, **kwargs)
promptflow/_cli/_pf/_flow.py:404: in test_flow
    result = pf_client.flows._test(
promptflow/_sdk/operations/_flow_operations.py:123: in _test
    return submitter.flow_test(
promptflow/_sdk/operations/_test_submitter.py:163: in flow_test
    line_result = flow_executor.exec_line(inputs, index=0, allow_generator_output=allow_generator_output)
promptflow/executor/flow_executor.py:706: in exec_line
    line_result = self._exec(
promptflow/executor/flow_executor.py:878: in _exec
    output, nodes_outputs = self._traverse_nodes(inputs, context)
promptflow/executor/flow_executor.py:960: in _traverse_nodes
    nodes_outputs, bypassed_nodes = self._submit_to_scheduler(context, inputs, batch_nodes)
promptflow/executor/flow_executor.py:980: in _submit_to_scheduler
    return FlowNodesScheduler(self._tools_manager, inputs, nodes, self._node_concurrency, context).execute()
promptflow/executor/_flow_nodes_scheduler.py:56: in execute
    self._dag_manager.complete_nodes(self._collect_outputs(completed_futures))
promptflow/executor/_flow_nodes_scheduler.py:85: in _collect_outputs
    each_node_result = each_future.result()
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/concurrent/futures/_base.py:439: in result
    return self.__get_result()
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/concurrent/futures/_base.py:391: in __get_result
    raise self._exception
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/concurrent/futures/thread.py:58: in run
    result = self.fn(*self.args, **self.kwargs)
promptflow/executor/_flow_nodes_scheduler.py:114: in _exec_single_node_in_thread
    result = f(**kwargs)
promptflow/_core/tool.py:68: in new_f
    return tool_invoker.invoke_tool(func, *args, **kwargs)
promptflow/executor/_tool_invoker.py:19: in invoke_tool
    return cur_flow.invoke_tool_with_cache(f, argnames, args, kwargs)
promptflow/_core/flow_execution_context.py:126: in invoke_tool_with_cache
    result = self.invoke_tool(f, args, kwargs)
promptflow/_core/flow_execution_context.py:164: in invoke_tool
    return f(*args, **kwargs)
promptflow/_core/tool_record.py:31: in just_return
    return ToolRecord().completion(toolType, *args, **kwargs)
promptflow/_core/tool.py:68: in new_f
    return tool_invoker.invoke_tool(func, *args, **kwargs)
promptflow/executor/_tool_invoker.py:19: in invoke_tool
    return cur_flow.invoke_tool_with_cache(f, argnames, args, kwargs)
promptflow/_core/flow_execution_context.py:87: in invoke_tool_with_cache
    output = f(*args, **kwargs)  # Do nothing if we are handling another tool
promptflow/_core/tool_record.py:25: in completion
    real_item = RecordStorage.get_record(working_folder, hashDict)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

flow_directory = PosixPath('/home/runner/work/promptflow/promptflow/src/promptflow/tests/test_configs/flows/web_classification')
hashDict = OrderedDict([('prompt', 'Please summarize some keywords of this paragraph and have some details of each keywords.\nDo ...sAdvertiseDevelopersTermsPrivacyPolicy & SafetyHow YouTube worksTest new featuresNFL Sunday Ticket© 2023 Google LLC')])

    @staticmethod
    def get_record(flow_directory: Path, hashDict: OrderedDict) -> str:
        # special deal remove text_content, because it is not stable.
        if "text_content" in hashDict:
            hashDict.pop("text_content")
        hash_value: str = hashlib.sha1(str(hashDict).encode("utf-8")).hexdigest()
        path_hash: str = hashlib.sha1(str(flow_directory).encode("utf-8")).hexdigest()
        file_item: Dict[str, str] = RecordStorage.runItems.get(path_hash, None)
        if file_item is None:
            RecordStorage.load_file(flow_directory)
            file_item = RecordStorage.runItems.get(path_hash, None)
        if file_item is not None:
            item = file_item.get(hash_value, None)
            if item is not None:
                real_item = base64.b64decode(bytes(item, "utf-8")).decode()
                return real_item
            else:
>               raise BaseException(f"Record item not found in folder {flow_directory}.")
E               BaseException: Record item not found in folder /home/runner/work/promptflow/promptflow/src/promptflow/tests/test_configs/flows/web_classification.

promptflow/_utils/utils.py:76: BaseException

BaseException: Record item not found in folder /tmp/tmp8nxwt4yg.
self = <sdk_cli_test.e2etests.test_cli.TestCli object at 0x7f7b47fde9d0>
capsys = <_pytest.capture.CaptureFixture object at 0x7f7b37a71a30>

    def test_pf_flow_with_variant(self, capsys):
        with tempfile.TemporaryDirectory() as temp_dir:
            shutil.copytree((Path(FLOWS_DIR) / "web_classification").resolve().as_posix(), temp_dir, dirs_exist_ok=True)
    
            with open(Path(temp_dir) / "flow.dag.yaml", "r") as f:
                flow_dict = yaml.safe_load(f)
    
            node_name = "summarize_text_content"
            node = next(filter(lambda item: item["name"] == node_name, flow_dict["nodes"]))
            flow_dict["nodes"].remove(node)
            flow_dict["nodes"].append({"name": node_name, "use_variants": True})
            with open(Path(temp_dir) / "flow.dag.yaml", "w") as f:
                yaml.safe_dump(flow_dict, f)
    
>           run_pf_command(
                "flow",
                "test",
                "--flow",
                temp_dir,
                "--inputs",
                "url=https://www.youtube.com/watch?v=o5ZQyXaAv1g",
                "answer=Channel",
                "evidence=Url",
            )

tests/sdk_cli_test/e2etests/test_cli.py:315: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/sdk_cli_test/e2etests/test_cli.py:45: in run_pf_command
    main()
promptflow/_cli/_pf/entry.py:97: in main
    entry(command_args)
promptflow/_cli/_pf/entry.py:61: in entry
    dispatch_flow_commands(args)
promptflow/_cli/_pf/_flow.py:69: in dispatch_flow_commands
    test_flow(args)
promptflow/_cli/_utils.py:351: in wrapper
    return func(*args, **kwargs)
promptflow/_cli/_pf/_flow.py:404: in test_flow
    result = pf_client.flows._test(
promptflow/_sdk/operations/_flow_operations.py:123: in _test
    return submitter.flow_test(
promptflow/_sdk/operations/_test_submitter.py:163: in flow_test
    line_result = flow_executor.exec_line(inputs, index=0, allow_generator_output=allow_generator_output)
promptflow/executor/flow_executor.py:706: in exec_line
    line_result = self._exec(
promptflow/executor/flow_executor.py:878: in _exec
    output, nodes_outputs = self._traverse_nodes(inputs, context)
promptflow/executor/flow_executor.py:960: in _traverse_nodes
    nodes_outputs, bypassed_nodes = self._submit_to_scheduler(context, inputs, batch_nodes)
promptflow/executor/flow_executor.py:980: in _submit_to_scheduler
    return FlowNodesScheduler(self._tools_manager, inputs, nodes, self._node_concurrency, context).execute()
promptflow/executor/_flow_nodes_scheduler.py:56: in execute
    self._dag_manager.complete_nodes(self._collect_outputs(completed_futures))
promptflow/executor/_flow_nodes_scheduler.py:85: in _collect_outputs
    each_node_result = each_future.result()
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/concurrent/futures/_base.py:439: in result
    return self.__get_result()
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/concurrent/futures/_base.py:391: in __get_result
    raise self._exception
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/concurrent/futures/thread.py:58: in run
    result = self.fn(*self.args, **self.kwargs)
promptflow/executor/_flow_nodes_scheduler.py:114: in _exec_single_node_in_thread
    result = f(**kwargs)
promptflow/_core/tool.py:68: in new_f
    return tool_invoker.invoke_tool(func, *args, **kwargs)
promptflow/executor/_tool_invoker.py:19: in invoke_tool
    return cur_flow.invoke_tool_with_cache(f, argnames, args, kwargs)
promptflow/_core/flow_execution_context.py:126: in invoke_tool_with_cache
    result = self.invoke_tool(f, args, kwargs)
promptflow/_core/flow_execution_context.py:164: in invoke_tool
    return f(*args, **kwargs)
promptflow/_core/tool_record.py:31: in just_return
    return ToolRecord().completion(toolType, *args, **kwargs)
promptflow/_core/tool.py:68: in new_f
    return tool_invoker.invoke_tool(func, *args, **kwargs)
promptflow/executor/_tool_invoker.py:19: in invoke_tool
    return cur_flow.invoke_tool_with_cache(f, argnames, args, kwargs)
promptflow/_core/flow_execution_context.py:87: in invoke_tool_with_cache
    output = f(*args, **kwargs)  # Do nothing if we are handling another tool
promptflow/_core/tool_record.py:25: in completion
    real_item = RecordStorage.get_record(working_folder, hashDict)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

flow_directory = PosixPath('/tmp/tmp8nxwt4yg')
hashDict = OrderedDict([('prompt', 'Please summarize some keywords of this paragraph and have some details of each keywords.\nDo ...sAdvertiseDevelopersTermsPrivacyPolicy & SafetyHow YouTube worksTest new featuresNFL Sunday Ticket© 2023 Google LLC')])

    @staticmethod
    def get_record(flow_directory: Path, hashDict: OrderedDict) -> str:
        # special deal remove text_content, because it is not stable.
        if "text_content" in hashDict:
            hashDict.pop("text_content")
        hash_value: str = hashlib.sha1(str(hashDict).encode("utf-8")).hexdigest()
        path_hash: str = hashlib.sha1(str(flow_directory).encode("utf-8")).hexdigest()
        file_item: Dict[str, str] = RecordStorage.runItems.get(path_hash, None)
        if file_item is None:
            RecordStorage.load_file(flow_directory)
            file_item = RecordStorage.runItems.get(path_hash, None)
        if file_item is not None:
            item = file_item.get(hash_value, None)
            if item is not None:
                real_item = base64.b64decode(bytes(item, "utf-8")).decode()
                return real_item
            else:
>               raise BaseException(f"Record item not found in folder {flow_directory}.")
E               BaseException: Record item not found in folder /tmp/tmp8nxwt4yg.

promptflow/_utils/utils.py:76: BaseException

BaseException: Record item not found in folder /home/runner/work/promptflow/promptflow/src/promptflow/tests/test_configs/flows/web_classification_with_symbolic.
self = <sdk_cli_test.e2etests.test_cli.TestCli object at 0x7feede28bdc0>
prepare_symbolic_flow = PosixPath('/home/runner/work/promptflow/promptflow/src/promptflow/tests/test_configs/flows/web_classification_with_symbolic')

    def test_pf_flow_test_with_symbolic(self, prepare_symbolic_flow):
>       run_pf_command(
            "flow",
            "test",
            "--flow",
            f"{FLOWS_DIR}/web_classification_with_symbolic",
            "--inputs",
            "url=https://www.youtube.com/watch?v=o5ZQyXaAv1g",
            "answer=Channel",
            "evidence=Url",
        )

tests/sdk_cli_test/e2etests/test_cli.py:516: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/sdk_cli_test/e2etests/test_cli.py:45: in run_pf_command
    main()
promptflow/_cli/_pf/entry.py:97: in main
    entry(command_args)
promptflow/_cli/_pf/entry.py:61: in entry
    dispatch_flow_commands(args)
promptflow/_cli/_pf/_flow.py:69: in dispatch_flow_commands
    test_flow(args)
promptflow/_cli/_utils.py:351: in wrapper
    return func(*args, **kwargs)
promptflow/_cli/_pf/_flow.py:404: in test_flow
    result = pf_client.flows._test(
promptflow/_sdk/operations/_flow_operations.py:123: in _test
    return submitter.flow_test(
promptflow/_sdk/operations/_test_submitter.py:163: in flow_test
    line_result = flow_executor.exec_line(inputs, index=0, allow_generator_output=allow_generator_output)
promptflow/executor/flow_executor.py:706: in exec_line
    line_result = self._exec(
promptflow/executor/flow_executor.py:878: in _exec
    output, nodes_outputs = self._traverse_nodes(inputs, context)
promptflow/executor/flow_executor.py:960: in _traverse_nodes
    nodes_outputs, bypassed_nodes = self._submit_to_scheduler(context, inputs, batch_nodes)
promptflow/executor/flow_executor.py:980: in _submit_to_scheduler
    return FlowNodesScheduler(self._tools_manager, inputs, nodes, self._node_concurrency, context).execute()
promptflow/executor/_flow_nodes_scheduler.py:56: in execute
    self._dag_manager.complete_nodes(self._collect_outputs(completed_futures))
promptflow/executor/_flow_nodes_scheduler.py:85: in _collect_outputs
    each_node_result = each_future.result()
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/concurrent/futures/_base.py:439: in result
    return self.__get_result()
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/concurrent/futures/_base.py:391: in __get_result
    raise self._exception
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/concurrent/futures/thread.py:58: in run
    result = self.fn(*self.args, **self.kwargs)
promptflow/executor/_flow_nodes_scheduler.py:114: in _exec_single_node_in_thread
    result = f(**kwargs)
promptflow/_core/tool.py:68: in new_f
    return tool_invoker.invoke_tool(func, *args, **kwargs)
promptflow/executor/_tool_invoker.py:19: in invoke_tool
    return cur_flow.invoke_tool_with_cache(f, argnames, args, kwargs)
promptflow/_core/flow_execution_context.py:126: in invoke_tool_with_cache
    result = self.invoke_tool(f, args, kwargs)
promptflow/_core/flow_execution_context.py:164: in invoke_tool
    return f(*args, **kwargs)
promptflow/_core/tool_record.py:31: in just_return
    return ToolRecord().completion(toolType, *args, **kwargs)
promptflow/_core/tool.py:68: in new_f
    return tool_invoker.invoke_tool(func, *args, **kwargs)
promptflow/executor/_tool_invoker.py:19: in invoke_tool
    return cur_flow.invoke_tool_with_cache(f, argnames, args, kwargs)
promptflow/_core/flow_execution_context.py:87: in invoke_tool_with_cache
    output = f(*args, **kwargs)  # Do nothing if we are handling another tool
promptflow/_core/tool_record.py:25: in completion
    real_item = RecordStorage.get_record(working_folder, hashDict)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

flow_directory = PosixPath('/home/runner/work/promptflow/promptflow/src/promptflow/tests/test_configs/flows/web_classification_with_symbolic')
hashDict = OrderedDict([('prompt', 'Please summarize some keywords of this paragraph and have some details of each keywords.\nDo ...sAdvertiseDevelopersTermsPrivacyPolicy & SafetyHow YouTube worksTest new featuresNFL Sunday Ticket© 2023 Google LLC')])

    @staticmethod
    def get_record(flow_directory: Path, hashDict: OrderedDict) -> str:
        # special deal remove text_content, because it is not stable.
        if "text_content" in hashDict:
            hashDict.pop("text_content")
        hash_value: str = hashlib.sha1(str(hashDict).encode("utf-8")).hexdigest()
        path_hash: str = hashlib.sha1(str(flow_directory).encode("utf-8")).hexdigest()
        file_item: Dict[str, str] = RecordStorage.runItems.get(path_hash, None)
        if file_item is None:
            RecordStorage.load_file(flow_directory)
            file_item = RecordStorage.runItems.get(path_hash, None)
        if file_item is not None:
            item = file_item.get(hash_value, None)
            if item is not None:
                real_item = base64.b64decode(bytes(item, "utf-8")).decode()
                return real_item
            else:
>               raise BaseException(f"Record item not found in folder {flow_directory}.")
E               BaseException: Record item not found in folder /home/runner/work/promptflow/promptflow/src/promptflow/tests/test_configs/flows/web_classification_with_symbolic.

promptflow/_utils/utils.py:76: BaseException

AttributeError: 'NoneType' object has no attribute 'isoformat'
self = <sdk_cli_test.e2etests.test_flow_run.TestFlowRun object at 0x7f7b47e74a00>
azure_open_ai_connection = <promptflow._sdk.entities._connection.AzureOpenAIConnection object at 0x7f7b37f08a00>
local_client = <promptflow._sdk._pf_client.PFClient object at 0x7f7b47a94610>
pf = <promptflow._sdk._pf_client.PFClient object at 0x7f7b37f084c0>

    def test_basic_flow_with_variant(self, azure_open_ai_connection: AzureOpenAIConnection, local_client, pf) -> None:
>       result = pf.run(
            flow=f"{FLOWS_DIR}/web_classification",
            data=f"{DATAS_DIR}/webClassification1.jsonl",
            column_mapping={"url": "${data.url}"},
            variant="${summarize_text_content.variant_0}",
        )

tests/sdk_cli_test/e2etests/test_flow_run.py:89: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
promptflow/_sdk/_pf_client.py:128: in run
    return self.runs.create_or_update(run=run, **kwargs)
promptflow/_telemetry/activity.py:138: in wrapper
    return f(self, *args, **kwargs)
promptflow/_sdk/operations/_run_operations.py:95: in create_or_update
    created_run = RunSubmitter(run_operations=self).submit(run=run, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:249: in submit
    self._run_bulk(run=run, stream=stream, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:277: in _run_bulk
    self._submit_bulk_run(flow=flow, run=run, local_storage=local_storage)
promptflow/_sdk/operations/_run_submitter.py:327: in _submit_bulk_run
    raise e
promptflow/_sdk/operations/_run_submitter.py:304: in _submit_bulk_run
    bulk_result = flow_executor.exec_bulk(mapped_inputs, run_id=run_id)
promptflow/executor/flow_executor.py:763: in exec_bulk
    line_results = self._exec_batch_with_threads(inputs, run_id, validate_inputs=validate_inputs)
promptflow/executor/flow_executor.py:467: in _exec_batch_with_threads
    result_list = pool.run(zip(line_number, batch_inputs))
promptflow/executor/_line_execution_process_pool.py:298: in run
    self._pool.starmap(
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:372: in starmap
    return self._map_async(func, iterable, starmapstar, chunksize).get()
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:771: in get
    raise self._value
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:125: in worker
    result = (True, func(*args, **kwds))
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:51: in starmapstar
    return list(itertools.starmap(args[0], args[1]))
promptflow/executor/_line_execution_process_pool.py:219: in _timeout_process_wrapper
    self._storage.persist_node_run(message)
promptflow/_sdk/operations/_local_storage_operations.py:386: in persist_node_run
    node_run_record = NodeRunRecord.from_run_info(run_info)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

node_run_info = RunInfo(node='summarize_text_content', flow_run_id='web_classification_variant_0_20231023_074403_771010', run_id='web_...pi_calls=None, variant_id='', cached_run_id=None, cached_flow_run_id=None, logs=None, system_metrics=None, result=None)

    @staticmethod
    def from_run_info(node_run_info: NodeRunInfo) -> "NodeRunRecord":
        return NodeRunRecord(
            NodeName=node_run_info.node,
            line_number=node_run_info.index,
            run_info=serialize(node_run_info),
            start_time=node_run_info.start_time.isoformat(),
>           end_time=node_run_info.end_time.isoformat(),
            status=node_run_info.status.value,
        )
E       AttributeError: 'NoneType' object has no attribute 'isoformat'

promptflow/_sdk/operations/_local_storage_operations.py:123: AttributeError

AttributeError: 'NoneType' object has no attribute 'isoformat'
self = <sdk_cli_test.e2etests.test_flow_run.TestFlowRun object at 0x7f7b47d8f160>
azure_open_ai_connection = <promptflow._sdk.entities._connection.AzureOpenAIConnection object at 0x7f7b37a0dc40>
local_client = <promptflow._sdk._pf_client.PFClient object at 0x7f7b47a94610>
pf = <promptflow._sdk._pf_client.PFClient object at 0x7f7b37f084c0>

    def test_basic_evaluation(self, azure_open_ai_connection: AzureOpenAIConnection, local_client, pf):
        data_path = f"{DATAS_DIR}/webClassification3.jsonl"
    
>       result = pf.run(
            flow=f"{FLOWS_DIR}/web_classification",
            data=data_path,
            column_mapping={"url": "${data.url}"},
        )

tests/sdk_cli_test/e2etests/test_flow_run.py:149: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
promptflow/_sdk/_pf_client.py:128: in run
    return self.runs.create_or_update(run=run, **kwargs)
promptflow/_telemetry/activity.py:138: in wrapper
    return f(self, *args, **kwargs)
promptflow/_sdk/operations/_run_operations.py:95: in create_or_update
    created_run = RunSubmitter(run_operations=self).submit(run=run, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:249: in submit
    self._run_bulk(run=run, stream=stream, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:277: in _run_bulk
    self._submit_bulk_run(flow=flow, run=run, local_storage=local_storage)
promptflow/_sdk/operations/_run_submitter.py:327: in _submit_bulk_run
    raise e
promptflow/_sdk/operations/_run_submitter.py:304: in _submit_bulk_run
    bulk_result = flow_executor.exec_bulk(mapped_inputs, run_id=run_id)
promptflow/executor/flow_executor.py:763: in exec_bulk
    line_results = self._exec_batch_with_threads(inputs, run_id, validate_inputs=validate_inputs)
promptflow/executor/flow_executor.py:467: in _exec_batch_with_threads
    result_list = pool.run(zip(line_number, batch_inputs))
promptflow/executor/_line_execution_process_pool.py:298: in run
    self._pool.starmap(
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:372: in starmap
    return self._map_async(func, iterable, starmapstar, chunksize).get()
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:771: in get
    raise self._value
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:125: in worker
    result = (True, func(*args, **kwds))
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:51: in starmapstar
    return list(itertools.starmap(args[0], args[1]))
promptflow/executor/_line_execution_process_pool.py:219: in _timeout_process_wrapper
    self._storage.persist_node_run(message)
promptflow/_sdk/operations/_local_storage_operations.py:386: in persist_node_run
    node_run_record = NodeRunRecord.from_run_info(run_info)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

node_run_info = RunInfo(node='summarize_text_content', flow_run_id='web_classification_variant_0_20231023_074405_245069', run_id='web_...pi_calls=None, variant_id='', cached_run_id=None, cached_flow_run_id=None, logs=None, system_metrics=None, result=None)

    @staticmethod
    def from_run_info(node_run_info: NodeRunInfo) -> "NodeRunRecord":
        return NodeRunRecord(
            NodeName=node_run_info.node,
            line_number=node_run_info.index,
            run_info=serialize(node_run_info),
            start_time=node_run_info.start_time.isoformat(),
>           end_time=node_run_info.end_time.isoformat(),
            status=node_run_info.status.value,
        )
E       AttributeError: 'NoneType' object has no attribute 'isoformat'

promptflow/_sdk/operations/_local_storage_operations.py:123: AttributeError

AttributeError: 'NoneType' object has no attribute 'isoformat'
self = <sdk_cli_test.e2etests.test_flow_run.TestFlowRun object at 0x7feede1107f0>
azure_open_ai_connection = <promptflow._sdk.entities._connection.AzureOpenAIConnection object at 0x7feed5e06c70>
local_client = <promptflow._sdk._pf_client.PFClient object at 0x7feeddf4a4f0>
pf = <promptflow._sdk._pf_client.PFClient object at 0x7feefb884040>

    def test_basic_run_bulk(self, azure_open_ai_connection: AzureOpenAIConnection, local_client, pf):
>       result = pf.run(
            flow=f"{FLOWS_DIR}/web_classification",
            data=f"{DATAS_DIR}/webClassification1.jsonl",
            column_mapping={"url": "${data.url}"},
        )

tests/sdk_cli_test/e2etests/test_flow_run.py:71: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
promptflow/_sdk/_pf_client.py:128: in run
    return self.runs.create_or_update(run=run, **kwargs)
promptflow/_telemetry/activity.py:138: in wrapper
    return f(self, *args, **kwargs)
promptflow/_sdk/operations/_run_operations.py:95: in create_or_update
    created_run = RunSubmitter(run_operations=self).submit(run=run, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:249: in submit
    self._run_bulk(run=run, stream=stream, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:277: in _run_bulk
    self._submit_bulk_run(flow=flow, run=run, local_storage=local_storage)
promptflow/_sdk/operations/_run_submitter.py:327: in _submit_bulk_run
    raise e
promptflow/_sdk/operations/_run_submitter.py:304: in _submit_bulk_run
    bulk_result = flow_executor.exec_bulk(mapped_inputs, run_id=run_id)
promptflow/executor/flow_executor.py:763: in exec_bulk
    line_results = self._exec_batch_with_threads(inputs, run_id, validate_inputs=validate_inputs)
promptflow/executor/flow_executor.py:467: in _exec_batch_with_threads
    result_list = pool.run(zip(line_number, batch_inputs))
promptflow/executor/_line_execution_process_pool.py:298: in run
    self._pool.starmap(
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:372: in starmap
    return self._map_async(func, iterable, starmapstar, chunksize).get()
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:771: in get
    raise self._value
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:125: in worker
    result = (True, func(*args, **kwds))
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:51: in starmapstar
    return list(itertools.starmap(args[0], args[1]))
promptflow/executor/_line_execution_process_pool.py:219: in _timeout_process_wrapper
    self._storage.persist_node_run(message)
promptflow/_sdk/operations/_local_storage_operations.py:386: in persist_node_run
    node_run_record = NodeRunRecord.from_run_info(run_info)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

node_run_info = RunInfo(node='summarize_text_content', flow_run_id='web_classification_variant_0_20231023_074414_515572', run_id='web_...pi_calls=None, variant_id='', cached_run_id=None, cached_flow_run_id=None, logs=None, system_metrics=None, result=None)

    @staticmethod
    def from_run_info(node_run_info: NodeRunInfo) -> "NodeRunRecord":
        return NodeRunRecord(
            NodeName=node_run_info.node,
            line_number=node_run_info.index,
            run_info=serialize(node_run_info),
            start_time=node_run_info.start_time.isoformat(),
>           end_time=node_run_info.end_time.isoformat(),
            status=node_run_info.status.value,
        )
E       AttributeError: 'NoneType' object has no attribute 'isoformat'

promptflow/_sdk/operations/_local_storage_operations.py:123: AttributeError

AttributeError: 'NoneType' object has no attribute 'isoformat'
self = <sdk_cli_test.e2etests.test_flow_run.TestFlowRun object at 0x7feede279460>
local_client = <promptflow._sdk._pf_client.PFClient object at 0x7feeddf4a4f0>
local_aoai_connection = <promptflow._sdk.entities._connection.AzureOpenAIConnection object at 0x7feedc083ac0>
pf = <promptflow._sdk._pf_client.PFClient object at 0x7feefb884040>

    def test_run_with_connection(self, local_client, local_aoai_connection, pf):
        # remove connection file to test connection resolving
        os.environ.pop(PROMPTFLOW_CONNECTIONS)
>       result = pf.run(
            flow=f"{FLOWS_DIR}/web_classification",
            data=f"{DATAS_DIR}/webClassification1.jsonl",
            column_mapping={"url": "${data.url}"},
        )

tests/sdk_cli_test/e2etests/test_flow_run.py:209: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
promptflow/_sdk/_pf_client.py:128: in run
    return self.runs.create_or_update(run=run, **kwargs)
promptflow/_telemetry/activity.py:138: in wrapper
    return f(self, *args, **kwargs)
promptflow/_sdk/operations/_run_operations.py:95: in create_or_update
    created_run = RunSubmitter(run_operations=self).submit(run=run, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:249: in submit
    self._run_bulk(run=run, stream=stream, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:277: in _run_bulk
    self._submit_bulk_run(flow=flow, run=run, local_storage=local_storage)
promptflow/_sdk/operations/_run_submitter.py:327: in _submit_bulk_run
    raise e
promptflow/_sdk/operations/_run_submitter.py:304: in _submit_bulk_run
    bulk_result = flow_executor.exec_bulk(mapped_inputs, run_id=run_id)
promptflow/executor/flow_executor.py:763: in exec_bulk
    line_results = self._exec_batch_with_threads(inputs, run_id, validate_inputs=validate_inputs)
promptflow/executor/flow_executor.py:467: in _exec_batch_with_threads
    result_list = pool.run(zip(line_number, batch_inputs))
promptflow/executor/_line_execution_process_pool.py:298: in run
    self._pool.starmap(
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:372: in starmap
    return self._map_async(func, iterable, starmapstar, chunksize).get()
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:771: in get
    raise self._value
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:125: in worker
    result = (True, func(*args, **kwds))
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:51: in starmapstar
    return list(itertools.starmap(args[0], args[1]))
promptflow/executor/_line_execution_process_pool.py:219: in _timeout_process_wrapper
    self._storage.persist_node_run(message)
promptflow/_sdk/operations/_local_storage_operations.py:386: in persist_node_run
    node_run_record = NodeRunRecord.from_run_info(run_info)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

node_run_info = RunInfo(node='summarize_text_content', flow_run_id='web_classification_variant_0_20231023_074416_089922', run_id='web_...pi_calls=None, variant_id='', cached_run_id=None, cached_flow_run_id=None, logs=None, system_metrics=None, result=None)

    @staticmethod
    def from_run_info(node_run_info: NodeRunInfo) -> "NodeRunRecord":
        return NodeRunRecord(
            NodeName=node_run_info.node,
            line_number=node_run_info.index,
            run_info=serialize(node_run_info),
            start_time=node_run_info.start_time.isoformat(),
>           end_time=node_run_info.end_time.isoformat(),
            status=node_run_info.status.value,
        )
E       AttributeError: 'NoneType' object has no attribute 'isoformat'

promptflow/_sdk/operations/_local_storage_operations.py:123: AttributeError

AttributeError: 'CustomConnection' object has no attribute 'api_base'
self = <sdk_cli_test.e2etests.test_flow_run.TestFlowRun object at 0x7feede2ac8b0>
local_client = <promptflow._sdk._pf_client.PFClient object at 0x7feeddf4a4f0>
local_custom_connection = <promptflow._sdk.entities._connection.CustomConnection object at 0x7feed5d4be80>
pf = <promptflow._sdk._pf_client.PFClient object at 0x7feefb884040>

    def test_custom_connection_overwrite(self, local_client, local_custom_connection, pf):
>       result = pf.run(
            flow=f"{FLOWS_DIR}/custom_connection_flow",
            data=f"{DATAS_DIR}/env_var_names.jsonl",
            connections={"print_env": {"connection": "test_custom_connection"}},
        )

tests/sdk_cli_test/e2etests/test_flow_run.py:233: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
promptflow/_sdk/_pf_client.py:128: in run
    return self.runs.create_or_update(run=run, **kwargs)
promptflow/_telemetry/activity.py:138: in wrapper
    return f(self, *args, **kwargs)
promptflow/_sdk/operations/_run_operations.py:95: in create_or_update
    created_run = RunSubmitter(run_operations=self).submit(run=run, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:249: in submit
    self._run_bulk(run=run, stream=stream, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:277: in _run_bulk
    self._submit_bulk_run(flow=flow, run=run, local_storage=local_storage)
promptflow/_sdk/operations/_run_submitter.py:282: in _submit_bulk_run
    connections = SubmitterHelper.resolve_connections(flow=flow)
promptflow/_sdk/operations/_run_submitter.py:216: in resolve_connections
    return get_local_connections_from_executable(executable=executable, client=client)
promptflow/_sdk/_utils.py:814: in get_local_connections_from_executable
    if conn.api_base == "dummy_base" and conn.api_key == "dummy_key":
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <promptflow._sdk.entities._connection.CustomConnection object at 0x7feedc319130>
item = 'api_base'

    def __getattr__(self, item):
        # Note: This is added for compatibility with promptflow.connections custom connection usage.
        if item == "secrets":
            # Usually obj.secrets will not reach here
            # This is added to handle copy.deepcopy loop issue
            return super().__getattribute__("secrets")
        if item == "configs":
            # Usually obj.configs will not reach here
            # This is added to handle copy.deepcopy loop issue
            return super().__getattribute__("configs")
        if item in self.secrets:
            logger.warning("Please use connection.secrets[key] to access secrets.")
            return self.secrets[item]
        if item in self.configs:
            logger.warning("Please use connection.configs[key] to access configs.")
            return self.configs[item]
>       return super().__getattribute__(item)
E       AttributeError: 'CustomConnection' object has no attribute 'api_base'

promptflow/_sdk/entities/_connection.py:831: AttributeError

promptflow.executor._errors.ResolveToolError: Tool load failed in 'My_Second_Tool_usi3': (ConnectionNotFound) ConnectionNotFound
self = <promptflow.executor._tool_resolver.ToolResolver object at 0x7feeddbfb7c0>
node = Node(name='My_Second_Tool_usi3', tool=None, inputs={'connection': InputAssignment(value='custom_strong_type_connection...y_tool_package.tools.my_tool_2.MyTool.my_tool', path=None), type=<ToolType.PYTHON: 'python'>, skip=None, activate=None)
convert_input_types = True

    def resolve_tool_by_node(self, node: Node, convert_input_types=True) -> ResolvedTool:
        try:
            if node.source is None:
                raise UserErrorException(f"Node {node.name} does not have source defined.")
    
            if node.type is ToolType.PYTHON:
                if node.source.type == ToolSourceType.Package:
>                   return self._resolve_package_node(node, convert_input_types=convert_input_types)

promptflow/executor/_tool_resolver.py:132: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
promptflow/executor/_tool_resolver.py:292: in _resolve_package_node
    updated_node = self._convert_node_literal_input_types(updated_node, tool)
promptflow/executor/_tool_resolver.py:101: in _convert_node_literal_input_types
    updated_inputs[k].value = self._convert_to_custom_strong_type_connection_value(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <promptflow.executor._tool_resolver.ToolResolver object at 0x7feeddbfb7c0>
k = 'connection'
v = InputAssignment(value='custom_strong_type_connection', value_type=<InputValueType.LITERAL: 'Literal'>, section='', property='')
node = Node(name='My_Second_Tool_usi3', tool=None, inputs={'connection': InputAssignment(value='custom_strong_type_connection...y_tool_package.tools.my_tool_2.MyTool.my_tool', path=None), type=<ToolType.PYTHON: 'python'>, skip=None, activate=None)
tool = Tool(name='My Second Tool', type=<ToolType.PYTHON: 'python'>, inputs={'connection': InputDefinition(type=['CustomConne...2', class_name='MyTool', source=None, code=None, function='my_tool', connection_type=None, is_builtin=None, stage=None)
conn_types = ['MySecondConnection'], module = None

    def _convert_to_custom_strong_type_connection_value(
        self, k: str, v: InputAssignment, node: Node, tool: Tool, conn_types: List[str], module: types.ModuleType
    ):
        if not conn_types:
            msg = f"Input '{k}' for node '{node.name}' has invalid types: {conn_types}."
            raise NodeInputValidationError(message=msg)
        connection_value = self._connection_manager.get(v.value)
        if not connection_value:
>           raise ConnectionNotFound(f"Connection {v.value} not found for node {node.name!r} input {k!r}.")
E           promptflow.executor._errors.ConnectionNotFound: ConnectionNotFound

promptflow/executor/_tool_resolver.py:76: ConnectionNotFound

The above exception was the direct cause of the following exception:

self = <sdk_cli_test.e2etests.test_flow_run.TestFlowRun object at 0x7feede2ac4c0>
install_custom_tool_pkg = None
local_client = <promptflow._sdk._pf_client.PFClient object at 0x7feeddf4a4f0>
pf = <promptflow._sdk._pf_client.PFClient object at 0x7feefb884040>

    def test_basic_flow_with_package_tool_with_custom_strong_type_connection(
        self, install_custom_tool_pkg, local_client, pf
    ):
        # Need to reload pkg_resources to get the latest installed tools
        import importlib
    
        import pkg_resources
    
        importlib.reload(pkg_resources)
    
>       result = pf.run(
            flow=f"{FLOWS_DIR}/flow_with_package_tool_with_custom_strong_type_connection",
            data=f"{FLOWS_DIR}/flow_with_package_tool_with_custom_strong_type_connection/data.jsonl",
            connections={"My_First_Tool_00f8": {"connection": "custom_strong_type_connection"}},
        )

tests/sdk_cli_test/e2etests/test_flow_run.py:260: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
promptflow/_sdk/_pf_client.py:128: in run
    return self.runs.create_or_update(run=run, **kwargs)
promptflow/_telemetry/activity.py:138: in wrapper
    return f(self, *args, **kwargs)
promptflow/_sdk/operations/_run_operations.py:95: in create_or_update
    created_run = RunSubmitter(run_operations=self).submit(run=run, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:249: in submit
    self._run_bulk(run=run, stream=stream, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:277: in _run_bulk
    self._submit_bulk_run(flow=flow, run=run, local_storage=local_storage)
promptflow/_sdk/operations/_run_submitter.py:288: in _submit_bulk_run
    flow_executor = FlowExecutor.create(
promptflow/executor/flow_executor.py:210: in create
    resolved_tools = [tool_resolver.resolve_tool_by_node(node) for node in flow.nodes]
promptflow/executor/flow_executor.py:210: in <listcomp>
    resolved_tools = [tool_resolver.resolve_tool_by_node(node) for node in flow.nodes]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <promptflow.executor._tool_resolver.ToolResolver object at 0x7feeddbfb7c0>
node = Node(name='My_Second_Tool_usi3', tool=None, inputs={'connection': InputAssignment(value='custom_strong_type_connection...y_tool_package.tools.my_tool_2.MyTool.my_tool', path=None), type=<ToolType.PYTHON: 'python'>, skip=None, activate=None)
convert_input_types = True

    def resolve_tool_by_node(self, node: Node, convert_input_types=True) -> ResolvedTool:
        try:
            if node.source is None:
                raise UserErrorException(f"Node {node.name} does not have source defined.")
    
            if node.type is ToolType.PYTHON:
                if node.source.type == ToolSourceType.Package:
                    return self._resolve_package_node(node, convert_input_types=convert_input_types)
                elif node.source.type == ToolSourceType.Code:
                    return self._resolve_script_node(node, convert_input_types=convert_input_types)
                raise NotImplementedError(f"Tool source type {node.source.type} for python tool is not supported yet.")
            elif node.type is ToolType.PROMPT:
                return self._resolve_prompt_node(node)
            elif node.type is ToolType.LLM:
                if os.environ.get("PF_RECORDING_MODE", None) == "replay":
                    resolved_tool = self._resolve_replay_node(node, convert_input_types=convert_input_types)
                    if resolved_tool is None:
                        resolved_tool = self._resolve_llm_node(node, convert_input_types=convert_input_types)
                    return resolved_tool
                return self._resolve_llm_node(node, convert_input_types=convert_input_types)
            elif node.type is ToolType.CUSTOM_LLM:
                if node.source.type == ToolSourceType.PackageWithPrompt:
                    resolved_tool = self._resolve_package_node(node, convert_input_types=convert_input_types)
                    return self._integrate_prompt_in_package_node(node, resolved_tool)
                raise NotImplementedError(
                    f"Tool source type {node.source.type} for custom_llm tool is not supported yet."
                )
            else:
                raise NotImplementedError(f"Tool type {node.type} is not supported yet.")
        except Exception as e:
            if isinstance(e, PromptflowException) and e.target != ErrorTarget.UNKNOWN:
>               raise ResolveToolError(node_name=node.name, target=e.target, module=e.module) from e
E               promptflow.executor._errors.ResolveToolError: Tool load failed in 'My_Second_Tool_usi3': (ConnectionNotFound) ConnectionNotFound

promptflow/executor/_tool_resolver.py:156: ResolveToolError

AttributeError: 'CustomConnection' object has no attribute 'api_base'
self = <sdk_cli_test.e2etests.test_flow_run.TestFlowRun object at 0x7feede2acf70>
install_custom_tool_pkg = None
local_client = <promptflow._sdk._pf_client.PFClient object at 0x7feeddf4a4f0>
pf = <promptflow._sdk._pf_client.PFClient object at 0x7feefb884040>

    def test_basic_flow_with_script_tool_with_custom_strong_type_connection(
        self, install_custom_tool_pkg, local_client, pf
    ):
        # Prepare custom connection
        from promptflow.connections import CustomConnection
    
        conn = CustomConnection(name="custom_connection_2", secrets={"api_key": "test"}, configs={"api_url": "test"})
        local_client.connections.create_or_update(conn)
    
>       result = pf.run(
            flow=f"{FLOWS_DIR}/flow_with_script_tool_with_custom_strong_type_connection",
            data=f"{FLOWS_DIR}/flow_with_script_tool_with_custom_strong_type_connection/data.jsonl",
        )

tests/sdk_cli_test/e2etests/test_flow_run.py:277: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
promptflow/_sdk/_pf_client.py:128: in run
    return self.runs.create_or_update(run=run, **kwargs)
promptflow/_telemetry/activity.py:138: in wrapper
    return f(self, *args, **kwargs)
promptflow/_sdk/operations/_run_operations.py:95: in create_or_update
    created_run = RunSubmitter(run_operations=self).submit(run=run, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:249: in submit
    self._run_bulk(run=run, stream=stream, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:277: in _run_bulk
    self._submit_bulk_run(flow=flow, run=run, local_storage=local_storage)
promptflow/_sdk/operations/_run_submitter.py:282: in _submit_bulk_run
    connections = SubmitterHelper.resolve_connections(flow=flow)
promptflow/_sdk/operations/_run_submitter.py:216: in resolve_connections
    return get_local_connections_from_executable(executable=executable, client=client)
promptflow/_sdk/_utils.py:814: in get_local_connections_from_executable
    if conn.api_base == "dummy_base" and conn.api_key == "dummy_key":
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <promptflow._sdk.entities._connection.CustomConnection object at 0x7feedc0da850>
item = 'api_base'

    def __getattr__(self, item):
        # Note: This is added for compatibility with promptflow.connections custom connection usage.
        if item == "secrets":
            # Usually obj.secrets will not reach here
            # This is added to handle copy.deepcopy loop issue
            return super().__getattribute__("secrets")
        if item == "configs":
            # Usually obj.configs will not reach here
            # This is added to handle copy.deepcopy loop issue
            return super().__getattribute__("configs")
        if item in self.secrets:
            logger.warning("Please use connection.secrets[key] to access secrets.")
            return self.secrets[item]
        if item in self.configs:
            logger.warning("Please use connection.configs[key] to access configs.")
            return self.configs[item]
>       return super().__getattribute__(item)
E       AttributeError: 'CustomConnection' object has no attribute 'api_base'

promptflow/_sdk/entities/_connection.py:831: AttributeError

AttributeError: 'NoneType' object has no attribute 'isoformat'
self = <sdk_cli_test.e2etests.test_flow_run.TestFlowRun object at 0x7feede1104c0>
local_client = <promptflow._sdk._pf_client.PFClient object at 0x7feeddf4a4f0>
local_aoai_connection = <promptflow._sdk.entities._connection.AzureOpenAIConnection object at 0x7feedc122dc0>

    def test_connection_overwrite_file(self, local_client, local_aoai_connection):
>       run = create_yaml_run(
            source=f"{RUNS_DIR}/run_with_connections.yaml",
        )

tests/sdk_cli_test/e2etests/test_flow_run.py:345: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
promptflow/_sdk/_run_functions.py:20: in create_yaml_run
    return _create_run(run=run, **kwargs)
promptflow/_sdk/_run_functions.py:14: in _create_run
    return client.runs.create_or_update(run=run, **kwargs)
promptflow/_telemetry/activity.py:138: in wrapper
    return f(self, *args, **kwargs)
promptflow/_sdk/operations/_run_operations.py:95: in create_or_update
    created_run = RunSubmitter(run_operations=self).submit(run=run, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:249: in submit
    self._run_bulk(run=run, stream=stream, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:277: in _run_bulk
    self._submit_bulk_run(flow=flow, run=run, local_storage=local_storage)
promptflow/_sdk/operations/_run_submitter.py:327: in _submit_bulk_run
    raise e
promptflow/_sdk/operations/_run_submitter.py:304: in _submit_bulk_run
    bulk_result = flow_executor.exec_bulk(mapped_inputs, run_id=run_id)
promptflow/executor/flow_executor.py:763: in exec_bulk
    line_results = self._exec_batch_with_threads(inputs, run_id, validate_inputs=validate_inputs)
promptflow/executor/flow_executor.py:467: in _exec_batch_with_threads
    result_list = pool.run(zip(line_number, batch_inputs))
promptflow/executor/_line_execution_process_pool.py:298: in run
    self._pool.starmap(
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:372: in starmap
    return self._map_async(func, iterable, starmapstar, chunksize).get()
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:771: in get
    raise self._value
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:125: in worker
    result = (True, func(*args, **kwds))
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:51: in starmapstar
    return list(itertools.starmap(args[0], args[1]))
promptflow/executor/_line_execution_process_pool.py:219: in _timeout_process_wrapper
    self._storage.persist_node_run(message)
promptflow/_sdk/operations/_local_storage_operations.py:386: in persist_node_run
    node_run_record = NodeRunRecord.from_run_info(run_info)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

node_run_info = RunInfo(node='summarize_text_content', flow_run_id='web_classification_variant_0_20231023_074434_416244', run_id='web_...pi_calls=None, variant_id='', cached_run_id=None, cached_flow_run_id=None, logs=None, system_metrics=None, result=None)

    @staticmethod
    def from_run_info(node_run_info: NodeRunInfo) -> "NodeRunRecord":
        return NodeRunRecord(
            NodeName=node_run_info.node,
            line_number=node_run_info.index,
            run_info=serialize(node_run_info),
            start_time=node_run_info.start_time.isoformat(),
>           end_time=node_run_info.end_time.isoformat(),
            status=node_run_info.status.value,
        )
E       AttributeError: 'NoneType' object has no attribute 'isoformat'

promptflow/_sdk/operations/_local_storage_operations.py:123: AttributeError

AttributeError: 'NoneType' object has no attribute 'isoformat'
self = <sdk_cli_test.e2etests.test_flow_run.TestFlowRun object at 0x7feeddf43040>
local_client = <promptflow._sdk._pf_client.PFClient object at 0x7feeddf4a4f0>
local_aoai_connection = <promptflow._sdk.entities._connection.AzureOpenAIConnection object at 0x7feed5a965e0>

    def test_connection_overwrite_model(self, local_client, local_aoai_connection):
>       run = create_yaml_run(
            source=f"{RUNS_DIR}/run_with_connections_model.yaml",
        )

tests/sdk_cli_test/e2etests/test_flow_run.py:352: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
promptflow/_sdk/_run_functions.py:20: in create_yaml_run
    return _create_run(run=run, **kwargs)
promptflow/_sdk/_run_functions.py:14: in _create_run
    return client.runs.create_or_update(run=run, **kwargs)
promptflow/_telemetry/activity.py:138: in wrapper
    return f(self, *args, **kwargs)
promptflow/_sdk/operations/_run_operations.py:95: in create_or_update
    created_run = RunSubmitter(run_operations=self).submit(run=run, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:249: in submit
    self._run_bulk(run=run, stream=stream, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:277: in _run_bulk
    self._submit_bulk_run(flow=flow, run=run, local_storage=local_storage)
promptflow/_sdk/operations/_run_submitter.py:327: in _submit_bulk_run
    raise e
promptflow/_sdk/operations/_run_submitter.py:304: in _submit_bulk_run
    bulk_result = flow_executor.exec_bulk(mapped_inputs, run_id=run_id)
promptflow/executor/flow_executor.py:763: in exec_bulk
    line_results = self._exec_batch_with_threads(inputs, run_id, validate_inputs=validate_inputs)
promptflow/executor/flow_executor.py:467: in _exec_batch_with_threads
    result_list = pool.run(zip(line_number, batch_inputs))
promptflow/executor/_line_execution_process_pool.py:298: in run
    self._pool.starmap(
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:372: in starmap
    return self._map_async(func, iterable, starmapstar, chunksize).get()
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:771: in get
    raise self._value
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:125: in worker
    result = (True, func(*args, **kwds))
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:51: in starmapstar
    return list(itertools.starmap(args[0], args[1]))
promptflow/executor/_line_execution_process_pool.py:219: in _timeout_process_wrapper
    self._storage.persist_node_run(message)
promptflow/_sdk/operations/_local_storage_operations.py:386: in persist_node_run
    node_run_record = NodeRunRecord.from_run_info(run_info)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

node_run_info = RunInfo(node='summarize_text_content', flow_run_id='web_classification_variant_0_20231023_074436_781291', run_id='web_...pi_calls=None, variant_id='', cached_run_id=None, cached_flow_run_id=None, logs=None, system_metrics=None, result=None)

    @staticmethod
    def from_run_info(node_run_info: NodeRunInfo) -> "NodeRunRecord":
        return NodeRunRecord(
            NodeName=node_run_info.node,
            line_number=node_run_info.index,
            run_info=serialize(node_run_info),
            start_time=node_run_info.start_time.isoformat(),
>           end_time=node_run_info.end_time.isoformat(),
            status=node_run_info.status.value,
        )
E       AttributeError: 'NoneType' object has no attribute 'isoformat'

promptflow/_sdk/operations/_local_storage_operations.py:123: AttributeError

AssertionError: assert 'azure_open_ai_connection' in {}
 +  where 'azure_open_ai_connection' = <promptflow._sdk.entities._connection.AzureOpenAIConnection object at 0x7feedc06b460>.name
self = <sdk_cli_test.e2etests.test_flow_run.TestFlowRun object at 0x7feeddf43c40>
local_client = <promptflow._sdk._pf_client.PFClient object at 0x7feeddf4a4f0>
local_aoai_connection = <promptflow._sdk.entities._connection.AzureOpenAIConnection object at 0x7feedc06b460>

    def test_resolve_connection(self, local_client, local_aoai_connection):
        flow = load_flow(f"{FLOWS_DIR}/web_classification_no_variants")
        connections = SubmitterHelper.resolve_connections(flow, local_client)
>       assert local_aoai_connection.name in connections
E       AssertionError: assert 'azure_open_ai_connection' in {}
E        +  where 'azure_open_ai_connection' = <promptflow._sdk.entities._connection.AzureOpenAIConnection object at 0x7feedc06b460>.name

tests/sdk_cli_test/e2etests/test_flow_run.py:361: AssertionError

AttributeError: 'NoneType' object has no attribute 'isoformat'
self = <sdk_cli_test.e2etests.test_flow_run.TestFlowRun object at 0x7f7b47d8f520>
local_client = <promptflow._sdk._pf_client.PFClient object at 0x7f7b47a94610>
pf = <promptflow._sdk._pf_client.PFClient object at 0x7f7b37f084c0>

    def test_submit_run_from_yaml(self, local_client, pf):
        run_id = str(uuid.uuid4())
>       run = create_yaml_run(source=f"{RUNS_DIR}/sample_bulk_run.yaml", params_override=[{"name": run_id}])

tests/sdk_cli_test/e2etests/test_flow_run.py:196: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
promptflow/_sdk/_run_functions.py:20: in create_yaml_run
    return _create_run(run=run, **kwargs)
promptflow/_sdk/_run_functions.py:14: in _create_run
    return client.runs.create_or_update(run=run, **kwargs)
promptflow/_telemetry/activity.py:138: in wrapper
    return f(self, *args, **kwargs)
promptflow/_sdk/operations/_run_operations.py:95: in create_or_update
    created_run = RunSubmitter(run_operations=self).submit(run=run, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:249: in submit
    self._run_bulk(run=run, stream=stream, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:277: in _run_bulk
    self._submit_bulk_run(flow=flow, run=run, local_storage=local_storage)
promptflow/_sdk/operations/_run_submitter.py:327: in _submit_bulk_run
    raise e
promptflow/_sdk/operations/_run_submitter.py:304: in _submit_bulk_run
    bulk_result = flow_executor.exec_bulk(mapped_inputs, run_id=run_id)
promptflow/executor/flow_executor.py:763: in exec_bulk
    line_results = self._exec_batch_with_threads(inputs, run_id, validate_inputs=validate_inputs)
promptflow/executor/flow_executor.py:467: in _exec_batch_with_threads
    result_list = pool.run(zip(line_number, batch_inputs))
promptflow/executor/_line_execution_process_pool.py:298: in run
    self._pool.starmap(
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:372: in starmap
    return self._map_async(func, iterable, starmapstar, chunksize).get()
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:771: in get
    raise self._value
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:125: in worker
    result = (True, func(*args, **kwds))
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:51: in starmapstar
    return list(itertools.starmap(args[0], args[1]))
promptflow/executor/_line_execution_process_pool.py:219: in _timeout_process_wrapper
    self._storage.persist_node_run(message)
promptflow/_sdk/operations/_local_storage_operations.py:386: in persist_node_run
    node_run_record = NodeRunRecord.from_run_info(run_info)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

node_run_info = RunInfo(node='summarize_text_content', flow_run_id='15756432-6469-4692-bd12-54c62d2a7901', run_id='15756432-6469-4692-...pi_calls=None, variant_id='', cached_run_id=None, cached_flow_run_id=None, logs=None, system_metrics=None, result=None)

    @staticmethod
    def from_run_info(node_run_info: NodeRunInfo) -> "NodeRunRecord":
        return NodeRunRecord(
            NodeName=node_run_info.node,
            line_number=node_run_info.index,
            run_info=serialize(node_run_info),
            start_time=node_run_info.start_time.isoformat(),
>           end_time=node_run_info.end_time.isoformat(),
            status=node_run_info.status.value,
        )
E       AttributeError: 'NoneType' object has no attribute 'isoformat'

promptflow/_sdk/operations/_local_storage_operations.py:123: AttributeError

AttributeError: 'NoneType' object has no attribute 'isoformat'
self = <sdk_cli_test.e2etests.test_flow_run.TestFlowRun object at 0x7f7b47bb59d0>
azure_open_ai_connection = <promptflow._sdk.entities._connection.AzureOpenAIConnection object at 0x7f7b44179af0>
pf = <promptflow._sdk._pf_client.PFClient object at 0x7f7b37f084c0>

    def test_get_details(self, azure_open_ai_connection: AzureOpenAIConnection, pf) -> None:
        data_path = f"{DATAS_DIR}/webClassification3.jsonl"
>       run = pf.run(
            flow=f"{FLOWS_DIR}/web_classification",
            data=data_path,
            column_mapping={"url": "${data.url}"},
        )

tests/sdk_cli_test/e2etests/test_flow_run.py:514: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
promptflow/_sdk/_pf_client.py:128: in run
    return self.runs.create_or_update(run=run, **kwargs)
promptflow/_telemetry/activity.py:138: in wrapper
    return f(self, *args, **kwargs)
promptflow/_sdk/operations/_run_operations.py:95: in create_or_update
    created_run = RunSubmitter(run_operations=self).submit(run=run, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:249: in submit
    self._run_bulk(run=run, stream=stream, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:277: in _run_bulk
    self._submit_bulk_run(flow=flow, run=run, local_storage=local_storage)
promptflow/_sdk/operations/_run_submitter.py:327: in _submit_bulk_run
    raise e
promptflow/_sdk/operations/_run_submitter.py:304: in _submit_bulk_run
    bulk_result = flow_executor.exec_bulk(mapped_inputs, run_id=run_id)
promptflow/executor/flow_executor.py:763: in exec_bulk
    line_results = self._exec_batch_with_threads(inputs, run_id, validate_inputs=validate_inputs)
promptflow/executor/flow_executor.py:467: in _exec_batch_with_threads
    result_list = pool.run(zip(line_number, batch_inputs))
promptflow/executor/_line_execution_process_pool.py:298: in run
    self._pool.starmap(
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:372: in starmap
    return self._map_async(func, iterable, starmapstar, chunksize).get()
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:771: in get
    raise self._value
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:125: in worker
    result = (True, func(*args, **kwds))
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:51: in starmapstar
    return list(itertools.starmap(args[0], args[1]))
promptflow/executor/_line_execution_process_pool.py:219: in _timeout_process_wrapper
    self._storage.persist_node_run(message)
promptflow/_sdk/operations/_local_storage_operations.py:386: in persist_node_run
    node_run_record = NodeRunRecord.from_run_info(run_info)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

node_run_info = RunInfo(node='summarize_text_content', flow_run_id='web_classification_variant_0_20231023_074513_800332', run_id='web_...pi_calls=None, variant_id='', cached_run_id=None, cached_flow_run_id=None, logs=None, system_metrics=None, result=None)

    @staticmethod
    def from_run_info(node_run_info: NodeRunInfo) -> "NodeRunRecord":
        return NodeRunRecord(
            NodeName=node_run_info.node,
            line_number=node_run_info.index,
            run_info=serialize(node_run_info),
            start_time=node_run_info.start_time.isoformat(),
>           end_time=node_run_info.end_time.isoformat(),
            status=node_run_info.status.value,
        )
E       AttributeError: 'NoneType' object has no attribute 'isoformat'

promptflow/_sdk/operations/_local_storage_operations.py:123: AttributeError

AttributeError: 'NoneType' object has no attribute 'isoformat'
self = <sdk_cli_test.e2etests.test_flow_run.TestFlowRun object at 0x7f7b47bb5fd0>
azure_open_ai_connection = <promptflow._sdk.entities._connection.AzureOpenAIConnection object at 0x7f7b37cf1df0>
pf = <promptflow._sdk._pf_client.PFClient object at 0x7f7b37f084c0>

    def test_visualize_run(self, azure_open_ai_connection: AzureOpenAIConnection, pf) -> None:
        data_path = f"{DATAS_DIR}/webClassification3.jsonl"
>       run1 = pf.run(
            flow=f"{FLOWS_DIR}/web_classification",
            data=data_path,
            column_mapping={"url": "${data.url}"},
        )

tests/sdk_cli_test/e2etests/test_flow_run.py:531: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
promptflow/_sdk/_pf_client.py:128: in run
    return self.runs.create_or_update(run=run, **kwargs)
promptflow/_telemetry/activity.py:138: in wrapper
    return f(self, *args, **kwargs)
promptflow/_sdk/operations/_run_operations.py:95: in create_or_update
    created_run = RunSubmitter(run_operations=self).submit(run=run, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:249: in submit
    self._run_bulk(run=run, stream=stream, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:277: in _run_bulk
    self._submit_bulk_run(flow=flow, run=run, local_storage=local_storage)
promptflow/_sdk/operations/_run_submitter.py:327: in _submit_bulk_run
    raise e
promptflow/_sdk/operations/_run_submitter.py:304: in _submit_bulk_run
    bulk_result = flow_executor.exec_bulk(mapped_inputs, run_id=run_id)
promptflow/executor/flow_executor.py:763: in exec_bulk
    line_results = self._exec_batch_with_threads(inputs, run_id, validate_inputs=validate_inputs)
promptflow/executor/flow_executor.py:467: in _exec_batch_with_threads
    result_list = pool.run(zip(line_number, batch_inputs))
promptflow/executor/_line_execution_process_pool.py:298: in run
    self._pool.starmap(
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:372: in starmap
    return self._map_async(func, iterable, starmapstar, chunksize).get()
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:771: in get
    raise self._value
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:125: in worker
    result = (True, func(*args, **kwds))
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:51: in starmapstar
    return list(itertools.starmap(args[0], args[1]))
promptflow/executor/_line_execution_process_pool.py:219: in _timeout_process_wrapper
    self._storage.persist_node_run(message)
promptflow/_sdk/operations/_local_storage_operations.py:386: in persist_node_run
    node_run_record = NodeRunRecord.from_run_info(run_info)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

node_run_info = RunInfo(node='summarize_text_content', flow_run_id='web_classification_variant_0_20231023_074516_620158', run_id='web_...pi_calls=None, variant_id='', cached_run_id=None, cached_flow_run_id=None, logs=None, system_metrics=None, result=None)

    @staticmethod
    def from_run_info(node_run_info: NodeRunInfo) -> "NodeRunRecord":
        return NodeRunRecord(
            NodeName=node_run_info.node,
            line_number=node_run_info.index,
            run_info=serialize(node_run_info),
            start_time=node_run_info.start_time.isoformat(),
>           end_time=node_run_info.end_time.isoformat(),
            status=node_run_info.status.value,
        )
E       AttributeError: 'NoneType' object has no attribute 'isoformat'

promptflow/_sdk/operations/_local_storage_operations.py:123: AttributeError

ModuleNotFoundError: No module named 'azure.ai'
self = <sdk_cli_test.e2etests.test_flow_run.TestFlowRun object at 0x7f7b47cb6310>
azure_open_ai_connection = <promptflow._sdk.entities._connection.AzureOpenAIConnection object at 0x7f7b37b13730>
pf = <promptflow._sdk._pf_client.PFClient object at 0x7f7b37f084c0>
capfd = <_pytest.capture.CaptureFixture object at 0x7f7b37b13e80>

    def test_incomplete_run_visualize(
        self, azure_open_ai_connection: AzureOpenAIConnection, pf: PFClient, capfd: pytest.CaptureFixture
    ) -> None:
        failed_run = pf.run(
            flow=f"{FLOWS_DIR}/failed_flow",
            data=f"{DATAS_DIR}/webClassification1.jsonl",
            column_mapping={"text": "${data.url}"},
        )
        # "update" run status to failed since currently all run will be completed unless there's bug
        pf.runs.update(
            name=failed_run.name,
            status="Failed",
        )
    
        # patch logger.error to print, so that we can capture the error message using capfd
>       from promptflow.azure.operations import _run_operations

tests/sdk_cli_test/e2etests/test_flow_run.py:563: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
promptflow/azure/__init__.py:7: in <module>
    from ._pf_client import PFClient
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

    import os
    from os import PathLike
    from pathlib import Path
    from typing import Dict, List, Optional, Union
    
>   from azure.ai.ml import MLClient
E   ModuleNotFoundError: No module named 'azure.ai'

promptflow/azure/_pf_client.py:9: ModuleNotFoundError

AttributeError: 'NoneType' object has no attribute 'isoformat'
self = <sdk_cli_test.e2etests.test_flow_run.TestFlowRun object at 0x7feede0ef790>
local_client = <promptflow._sdk._pf_client.PFClient object at 0x7feeddf4a4f0>
pf = <promptflow._sdk._pf_client.PFClient object at 0x7feefb884040>

    def test_run_local_storage_structure(self, local_client, pf) -> None:
>       run = create_run_against_multi_line_data(pf)

tests/sdk_cli_test/e2etests/test_flow_run.py:637: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/sdk_cli_test/e2etests/test_flow_run.py:33: in create_run_against_multi_line_data
    return client.run(
promptflow/_sdk/_pf_client.py:128: in run
    return self.runs.create_or_update(run=run, **kwargs)
promptflow/_telemetry/activity.py:138: in wrapper
    return f(self, *args, **kwargs)
promptflow/_sdk/operations/_run_operations.py:95: in create_or_update
    created_run = RunSubmitter(run_operations=self).submit(run=run, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:249: in submit
    self._run_bulk(run=run, stream=stream, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:277: in _run_bulk
    self._submit_bulk_run(flow=flow, run=run, local_storage=local_storage)
promptflow/_sdk/operations/_run_submitter.py:327: in _submit_bulk_run
    raise e
promptflow/_sdk/operations/_run_submitter.py:304: in _submit_bulk_run
    bulk_result = flow_executor.exec_bulk(mapped_inputs, run_id=run_id)
promptflow/executor/flow_executor.py:763: in exec_bulk
    line_results = self._exec_batch_with_threads(inputs, run_id, validate_inputs=validate_inputs)
promptflow/executor/flow_executor.py:467: in _exec_batch_with_threads
    result_list = pool.run(zip(line_number, batch_inputs))
promptflow/executor/_line_execution_process_pool.py:298: in run
    self._pool.starmap(
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:372: in starmap
    return self._map_async(func, iterable, starmapstar, chunksize).get()
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:771: in get
    raise self._value
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:125: in worker
    result = (True, func(*args, **kwds))
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:51: in starmapstar
    return list(itertools.starmap(args[0], args[1]))
promptflow/executor/_line_execution_process_pool.py:219: in _timeout_process_wrapper
    self._storage.persist_node_run(message)
promptflow/_sdk/operations/_local_storage_operations.py:386: in persist_node_run
    node_run_record = NodeRunRecord.from_run_info(run_info)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

node_run_info = RunInfo(node='summarize_text_content', flow_run_id='web_classification_variant_0_20231023_074526_658757', run_id='web_...pi_calls=None, variant_id='', cached_run_id=None, cached_flow_run_id=None, logs=None, system_metrics=None, result=None)

    @staticmethod
    def from_run_info(node_run_info: NodeRunInfo) -> "NodeRunRecord":
        return NodeRunRecord(
            NodeName=node_run_info.node,
            line_number=node_run_info.index,
            run_info=serialize(node_run_info),
            start_time=node_run_info.start_time.isoformat(),
>           end_time=node_run_info.end_time.isoformat(),
            status=node_run_info.status.value,
        )
E       AttributeError: 'NoneType' object has no attribute 'isoformat'

promptflow/_sdk/operations/_local_storage_operations.py:123: AttributeError

AttributeError: 'NoneType' object has no attribute 'isoformat'
self = <sdk_cli_test.e2etests.test_flow_run.TestFlowRun object at 0x7feede0ef3a0>
local_client = <promptflow._sdk._pf_client.PFClient object at 0x7feeddf4a4f0>
pf = <promptflow._sdk._pf_client.PFClient object at 0x7feefb884040>

    def test_run_snapshot_with_flow_tools_json(self, local_client, pf) -> None:
>       run = create_run_against_multi_line_data(pf)

tests/sdk_cli_test/e2etests/test_flow_run.py:650: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/sdk_cli_test/e2etests/test_flow_run.py:33: in create_run_against_multi_line_data
    return client.run(
promptflow/_sdk/_pf_client.py:128: in run
    return self.runs.create_or_update(run=run, **kwargs)
promptflow/_telemetry/activity.py:138: in wrapper
    return f(self, *args, **kwargs)
promptflow/_sdk/operations/_run_operations.py:95: in create_or_update
    created_run = RunSubmitter(run_operations=self).submit(run=run, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:249: in submit
    self._run_bulk(run=run, stream=stream, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:277: in _run_bulk
    self._submit_bulk_run(flow=flow, run=run, local_storage=local_storage)
promptflow/_sdk/operations/_run_submitter.py:327: in _submit_bulk_run
    raise e
promptflow/_sdk/operations/_run_submitter.py:304: in _submit_bulk_run
    bulk_result = flow_executor.exec_bulk(mapped_inputs, run_id=run_id)
promptflow/executor/flow_executor.py:763: in exec_bulk
    line_results = self._exec_batch_with_threads(inputs, run_id, validate_inputs=validate_inputs)
promptflow/executor/flow_executor.py:467: in _exec_batch_with_threads
    result_list = pool.run(zip(line_number, batch_inputs))
promptflow/executor/_line_execution_process_pool.py:298: in run
    self._pool.starmap(
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:372: in starmap
    return self._map_async(func, iterable, starmapstar, chunksize).get()
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:771: in get
    raise self._value
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:125: in worker
    result = (True, func(*args, **kwds))
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:51: in starmapstar
    return list(itertools.starmap(args[0], args[1]))
promptflow/executor/_line_execution_process_pool.py:219: in _timeout_process_wrapper
    self._storage.persist_node_run(message)
promptflow/_sdk/operations/_local_storage_operations.py:386: in persist_node_run
    node_run_record = NodeRunRecord.from_run_info(run_info)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

node_run_info = RunInfo(node='summarize_text_content', flow_run_id='web_classification_variant_0_20231023_074529_241127', run_id='web_...pi_calls=None, variant_id='', cached_run_id=None, cached_flow_run_id=None, logs=None, system_metrics=None, result=None)

    @staticmethod
    def from_run_info(node_run_info: NodeRunInfo) -> "NodeRunRecord":
        return NodeRunRecord(
            NodeName=node_run_info.node,
            line_number=node_run_info.index,
            run_info=serialize(node_run_info),
            start_time=node_run_info.start_time.isoformat(),
>           end_time=node_run_info.end_time.isoformat(),
            status=node_run_info.status.value,
        )
E       AttributeError: 'NoneType' object has no attribute 'isoformat'

promptflow/_sdk/operations/_local_storage_operations.py:123: AttributeError

AttributeError: 'NoneType' object has no attribute 'isoformat'
self = <sdk_cli_test.e2etests.test_flow_run.TestFlowRun object at 0x7feede0ef880>
local_client = <promptflow._sdk._pf_client.PFClient object at 0x7feeddf4a4f0>
pf = <promptflow._sdk._pf_client.PFClient object at 0x7feefb884040>

    def test_get_metrics_format(self, local_client, pf) -> None:
>       run1 = create_run_against_multi_line_data(pf)

tests/sdk_cli_test/e2etests/test_flow_run.py:656: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/sdk_cli_test/e2etests/test_flow_run.py:33: in create_run_against_multi_line_data
    return client.run(
promptflow/_sdk/_pf_client.py:128: in run
    return self.runs.create_or_update(run=run, **kwargs)
promptflow/_telemetry/activity.py:138: in wrapper
    return f(self, *args, **kwargs)
promptflow/_sdk/operations/_run_operations.py:95: in create_or_update
    created_run = RunSubmitter(run_operations=self).submit(run=run, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:249: in submit
    self._run_bulk(run=run, stream=stream, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:277: in _run_bulk
    self._submit_bulk_run(flow=flow, run=run, local_storage=local_storage)
promptflow/_sdk/operations/_run_submitter.py:327: in _submit_bulk_run
    raise e
promptflow/_sdk/operations/_run_submitter.py:304: in _submit_bulk_run
    bulk_result = flow_executor.exec_bulk(mapped_inputs, run_id=run_id)
promptflow/executor/flow_executor.py:763: in exec_bulk
    line_results = self._exec_batch_with_threads(inputs, run_id, validate_inputs=validate_inputs)
promptflow/executor/flow_executor.py:467: in _exec_batch_with_threads
    result_list = pool.run(zip(line_number, batch_inputs))
promptflow/executor/_line_execution_process_pool.py:298: in run
    self._pool.starmap(
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:372: in starmap
    return self._map_async(func, iterable, starmapstar, chunksize).get()
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:771: in get
    raise self._value
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:125: in worker
    result = (True, func(*args, **kwds))
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:51: in starmapstar
    return list(itertools.starmap(args[0], args[1]))
promptflow/executor/_line_execution_process_pool.py:219: in _timeout_process_wrapper
    self._storage.persist_node_run(message)
promptflow/_sdk/operations/_local_storage_operations.py:386: in persist_node_run
    node_run_record = NodeRunRecord.from_run_info(run_info)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

node_run_info = RunInfo(node='summarize_text_content', flow_run_id='web_classification_variant_0_20231023_074532_179260', run_id='web_...pi_calls=None, variant_id='', cached_run_id=None, cached_flow_run_id=None, logs=None, system_metrics=None, result=None)

    @staticmethod
    def from_run_info(node_run_info: NodeRunInfo) -> "NodeRunRecord":
        return NodeRunRecord(
            NodeName=node_run_info.node,
            line_number=node_run_info.index,
            run_info=serialize(node_run_info),
            start_time=node_run_info.start_time.isoformat(),
>           end_time=node_run_info.end_time.isoformat(),
            status=node_run_info.status.value,
        )
E       AttributeError: 'NoneType' object has no attribute 'isoformat'

promptflow/_sdk/operations/_local_storage_operations.py:123: AttributeError

AttributeError: 'NoneType' object has no attribute 'isoformat'
self = <sdk_cli_test.e2etests.test_flow_run.TestFlowRun object at 0x7feede0ef4f0>
local_client = <promptflow._sdk._pf_client.PFClient object at 0x7feeddf4a4f0>
pf = <promptflow._sdk._pf_client.PFClient object at 0x7feefb884040>

    def test_get_detail_format(self, local_client, pf) -> None:
>       run = create_run_against_multi_line_data(pf)

tests/sdk_cli_test/e2etests/test_flow_run.py:662: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/sdk_cli_test/e2etests/test_flow_run.py:33: in create_run_against_multi_line_data
    return client.run(
promptflow/_sdk/_pf_client.py:128: in run
    return self.runs.create_or_update(run=run, **kwargs)
promptflow/_telemetry/activity.py:138: in wrapper
    return f(self, *args, **kwargs)
promptflow/_sdk/operations/_run_operations.py:95: in create_or_update
    created_run = RunSubmitter(run_operations=self).submit(run=run, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:249: in submit
    self._run_bulk(run=run, stream=stream, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:277: in _run_bulk
    self._submit_bulk_run(flow=flow, run=run, local_storage=local_storage)
promptflow/_sdk/operations/_run_submitter.py:327: in _submit_bulk_run
    raise e
promptflow/_sdk/operations/_run_submitter.py:304: in _submit_bulk_run
    bulk_result = flow_executor.exec_bulk(mapped_inputs, run_id=run_id)
promptflow/executor/flow_executor.py:763: in exec_bulk
    line_results = self._exec_batch_with_threads(inputs, run_id, validate_inputs=validate_inputs)
promptflow/executor/flow_executor.py:467: in _exec_batch_with_threads
    result_list = pool.run(zip(line_number, batch_inputs))
promptflow/executor/_line_execution_process_pool.py:298: in run
    self._pool.starmap(
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:372: in starmap
    return self._map_async(func, iterable, starmapstar, chunksize).get()
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:771: in get
    raise self._value
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:125: in worker
    result = (True, func(*args, **kwds))
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:51: in starmapstar
    return list(itertools.starmap(args[0], args[1]))
promptflow/executor/_line_execution_process_pool.py:219: in _timeout_process_wrapper
    self._storage.persist_node_run(message)
promptflow/_sdk/operations/_local_storage_operations.py:386: in persist_node_run
    node_run_record = NodeRunRecord.from_run_info(run_info)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

node_run_info = RunInfo(node='summarize_text_content', flow_run_id='web_classification_variant_0_20231023_074535_138046', run_id='web_...pi_calls=None, variant_id='', cached_run_id=None, cached_flow_run_id=None, logs=None, system_metrics=None, result=None)

    @staticmethod
    def from_run_info(node_run_info: NodeRunInfo) -> "NodeRunRecord":
        return NodeRunRecord(
            NodeName=node_run_info.node,
            line_number=node_run_info.index,
            run_info=serialize(node_run_info),
            start_time=node_run_info.start_time.isoformat(),
>           end_time=node_run_info.end_time.isoformat(),
            status=node_run_info.status.value,
        )
E       AttributeError: 'NoneType' object has no attribute 'isoformat'

promptflow/_sdk/operations/_local_storage_operations.py:123: AttributeError

AttributeError: 'NoneType' object has no attribute 'isoformat'
self = <sdk_cli_test.e2etests.test_flow_run.TestFlowRun object at 0x7f7b47bbb4f0>
pf = <promptflow._sdk._pf_client.PFClient object at 0x7f7b37f084c0>

    def test_system_metrics_in_properties(self, pf) -> None:
>       run = create_run_against_multi_line_data(pf)

tests/sdk_cli_test/e2etests/test_flow_run.py:746: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/sdk_cli_test/e2etests/test_flow_run.py:33: in create_run_against_multi_line_data
    return client.run(
promptflow/_sdk/_pf_client.py:128: in run
    return self.runs.create_or_update(run=run, **kwargs)
promptflow/_telemetry/activity.py:138: in wrapper
    return f(self, *args, **kwargs)
promptflow/_sdk/operations/_run_operations.py:95: in create_or_update
    created_run = RunSubmitter(run_operations=self).submit(run=run, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:249: in submit
    self._run_bulk(run=run, stream=stream, **kwargs)
promptflow/_sdk/operations/_run_submitter.py:277: in _run_bulk
    self._submit_bulk_run(flow=flow, run=run, local_storage=local_storage)
promptflow/_sdk/operations/_run_submitter.py:327: in _submit_bulk_run
    raise e
promptflow/_sdk/operations/_run_submitter.py:304: in _submit_bulk_run
    bulk_result = flow_executor.exec_bulk(mapped_inputs, run_id=run_id)
promptflow/executor/flow_executor.py:763: in exec_bulk
    line_results = self._exec_batch_with_threads(inputs, run_id, validate_inputs=validate_inputs)
promptflow/executor/flow_executor.py:467: in _exec_batch_with_threads
    result_list = pool.run(zip(line_number, batch_inputs))
promptflow/executor/_line_execution_process_pool.py:298: in run
    self._pool.starmap(
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:372: in starmap
    return self._map_async(func, iterable, starmapstar, chunksize).get()
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:771: in get
    raise self._value
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:125: in worker
    result = (True, func(*args, **kwds))
/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/multiprocessing/pool.py:51: in starmapstar
    return list(itertools.starmap(args[0], args[1]))
promptflow/executor/_line_execution_process_pool.py:219: in _timeout_process_wrapper
    self._storage.persist_node_run(message)
promptflow/_sdk/operations/_local_storage_operations.py:386: in persist_node_run
    node_run_record = NodeRunRecord.from_run_info(run_info)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

node_run_info = RunInfo(node='summarize_text_content', flow_run_id='web_classification_variant_0_20231023_074539_410395', run_id='web_...pi_calls=None, variant_id='', cached_run_id=None, cached_flow_run_id=None, logs=None, system_metrics=None, result=None)

    @staticmethod
    def from_run_info(node_run_info: NodeRunInfo) -> "NodeRunRecord":
        return NodeRunRecord(
            NodeName=node_run_info.node,
            line_number=node_run_info.index,
            run_info=serialize(node_run_info),
            start_time=node_run_info.start_time.isoformat(),
>           end_time=node_run_info.end_time.isoformat(),
            status=node_run_info.status.value,
        )
E       AttributeError: 'NoneType' object has no attribute 'isoformat'

promptflow/_sdk/operations/_local_storage_operations.py:123: AttributeError