Refine sdk resume test

src/promptflow/tests/sdk_cli_test/e2etests/test_cli.py

@@ -2160,6 +2160,107 @@ def get_successful_lines(output_path):
             )
             run_id = new_run_id
 
+    def test_flow_run_resume_from_token(self, capfd, local_client) -> None:
+        run_id = str(uuid.uuid4())
+        # fetch std out
+        run_pf_command(
+            "run",
+            "create",
+            "--flow",
+            f"{FLOWS_DIR}/web_classification_random_fail",
+            "--data",
+            f"{FLOWS_DIR}/web_classification_random_fail/data.jsonl",
+            "--column-mapping",
+            "url='${data.url}'",
+            "--name",
+            run_id,
+        )
+        out, _ = capfd.readouterr()
+        assert "Completed" in out
+        original_run = local_client.runs.get(name=run_id)
+
+        new_run_id = str(uuid.uuid4())
+        display_name = "test"
+        description = "new description"
+        run_pf_command(
+            "run",
+            "create",
+            "--resume-from",
+            run_id,
+            "--name",
+            new_run_id,
+            "--set",
+            f"display_name={display_name}",
+            f"description={description}",
+            "tags.A=A",
+            "tags.B=B",
+        )
+        resume_run = local_client.runs.get(name=new_run_id)
+        assert resume_run.name == new_run_id
+        assert resume_run.display_name == display_name
+        assert resume_run.description == description
+        assert resume_run.tags == {"A": "A", "B": "B"}
+        assert resume_run._resume_from == run_id
+        assert (
+            original_run.properties["system_metrics"]["total_tokens"]
+            <= resume_run.properties["system_metrics"]["total_tokens"]
+        )
+
+    def test_flow_run_resume_from_image_aggregation(self, capfd, local_client) -> None:
+        run_id = str(uuid.uuid4())
+        # fetch std out
+        run_pf_command(
+            "run",
+            "create",
+            "--flow",
+            f"{FLOWS_DIR}/eval_flow_with_image_resume_random_fail",
+            "--data",
+            f"{FLOWS_DIR}/eval_flow_with_image_resume_random_fail/data.jsonl",
+            "--column-mapping",
+            "input_image='${data.input_image}'",
+            "--name",
+            run_id,
+        )
+        out, _ = capfd.readouterr()
+        assert "Completed" in out
+        original_run = local_client.runs.get(name=run_id)
+        output_path = os.path.join(original_run.properties["output_path"], "flow_outputs", "output.jsonl")
+        with open(output_path, "r") as file:
+            original_output = [json.loads(line) for line in file]
+
+        new_run_id = str(uuid.uuid4())
+        display_name = "test"
+        description = "new description"
+        run_pf_command(
+            "run",
+            "create",
+            "--resume-from",
+            run_id,
+            "--name",
+            new_run_id,
+            "--set",
+            f"display_name={display_name}",
+            f"description={description}",
+            "tags.A=A",
+            "tags.B=B",
+        )
+        resume_run = local_client.runs.get(name=new_run_id)
+        output_path = os.path.join(resume_run.properties["output_path"], "flow_outputs", "output.jsonl")
+
+        with open(output_path, "r") as file:
+            resume_output = [json.loads(line) for line in file]
+
+        # assert original_output in resume_output
+        original_output_line_numbers = {line["line_number"] for line in original_output}
+        resume_output_line_numbers = {line["line_number"] for line in resume_output}
+        assert original_output_line_numbers.issubset(resume_output_line_numbers)
+        assert len(resume_output) >= len(original_output)
+        assert resume_run.name == new_run_id
+        assert resume_run.display_name == display_name
+        assert resume_run.description == description
+        assert resume_run.tags == {"A": "A", "B": "B"}
+        assert resume_run._resume_from == run_id
+
     def test_flow_run_exclusive_param(self, capfd) -> None:
         # fetch std out
         with pytest.raises(SystemExit):

src/promptflow/tests/test_configs/flows/eval_flow_with_image_resume_random_fail/count_image.py

@@ -0,0 +1,17 @@
+from typing import List
+from promptflow import tool
+from promptflow.contracts.multimedia import Image
+
+
+@tool
+def aggregate(images: List[Image]):
+    from promptflow import log_metric
+    image_count = 0
+    for image in images:
+        if not isinstance(image, Image):
+            print(f"Invalid image: {image}")
+        else:
+            image_count += 1
+    log_metric(key="image_count", value=image_count)
+
+    return image_count

src/promptflow/tests/test_configs/flows/eval_flow_with_image_resume_random_fail/data.jsonl

@@ -0,0 +1,9 @@
+{"input_image": {"data:image/jpg;path": "logo.gif"}}
+{"input_image": {"data:image/png;path": "logo.jpg"}}
+{"input_image": {"data:image/png;path": "logo.png"}}
+{"input_image": {"data:image/jpg;path": "logo.gif"}}
+{"input_image": {"data:image/png;path": "logo.jpg"}}
+{"input_image": {"data:image/png;path": "logo.png"}}
+{"input_image": {"data:image/jpg;path": "logo.gif"}}
+{"input_image": {"data:image/png;path": "logo.jpg"}}
+{"input_image": {"data:image/png;path": "logo.png"}}

src/promptflow/tests/test_configs/flows/eval_flow_with_image_resume_random_fail/flip_image.py

@@ -0,0 +1,18 @@
+import io
+import random
+from pathlib import Path
+from promptflow import tool
+from promptflow.contracts.multimedia import Image
+from PIL import Image as PIL_Image
+
+
+@tool
+def passthrough(input_image: Image) -> Image:
+    if random.random() < 0.5:
+        raise ValueError("Random failure")
+    image_stream = io.BytesIO(input_image)
+    pil_image = PIL_Image.open(image_stream)
+    flipped_image = pil_image.transpose(PIL_Image.FLIP_LEFT_RIGHT)
+    buffer = io.BytesIO()
+    flipped_image.save(buffer, format="PNG")
+    return Image(buffer.getvalue(), mime_type="image/png")

src/promptflow/tests/test_configs/flows/eval_flow_with_image_resume_random_fail/flow.dag.yaml

@@ -0,0 +1,25 @@
+$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Flow.schema.json
+inputs:
+  input_image:
+    type: image
+    default: https://developer.microsoft.com/_devcom/images/logo-ms-social.png
+outputs:
+  output_image:
+    type: string
+    reference: ${flip_image.output}
+nodes:
+- name: flip_image
+  type: python
+  source:
+    type: code
+    path: flip_image.py
+  inputs:
+    input_image: ${inputs.input_image}
+- name: count_image
+  type: python
+  source:
+    type: code
+    path: count_image.py
+  inputs:
+    images: ${flip_image.output}
+  aggregation: true

src/promptflow/tests/test_configs/flows/eval_flow_with_image_resume_random_fail/question_on_image.jinja2

@@ -0,0 +1,7 @@
+# system:
+As an AI assistant, your task involves interpreting images and responding to questions about the image.
+Remember to provide accurate answers based on the information present in the image.
+
+# user:
+{{question}}
+![image]({{test_image}})

src/promptflow/tests/test_configs/flows/eval_flow_with_image_resume_random_fail/requirements.txt

@@ -0,0 +1,2 @@
+promptflow
+promptflow-tools

src/promptflow/tests/test_configs/flows/web_classification/.promptflow/flow.tools.json

@@ -1,15 +1,15 @@
 {
-    "package": {},
     "code": {
         "fetch_text_content_from_url.py": {
             "type": "python",
             "inputs": {
-                "url": {
+                "fetch_url": {
                     "type": [
                         "string"
                     ]
                 }
             },
+            "source": "fetch_text_content_from_url.py",
             "function": "fetch_text_content_from_url"
         },
         "summarize_text_content.jinja2": {
@@ -21,7 +21,7 @@
                     ]
                 }
             },
-            "description": "Summarize webpage content into a short paragraph."
+            "source": "summarize_text_content.jinja2"
         },
         "summarize_text_content__variant_1.jinja2": {
             "type": "llm",
@@ -31,10 +31,12 @@
                         "string"
                     ]
                 }
-            }
+            },
+            "source": "summarize_text_content__variant_1.jinja2"
         },
         "prepare_examples.py": {
             "type": "python",
+            "source": "prepare_examples.py",
             "function": "prepare_examples"
         },
         "classify_with_llm.jinja2": {
@@ -56,7 +58,7 @@
                     ]
                 }
             },
-            "description": "Multi-class classification of a given url and text content."
+            "source": "classify_with_llm.jinja2"
         },
         "convert_to_dict.py": {
             "type": "python",
@@ -67,7 +69,9 @@
                     ]
                 }
             },
+            "source": "convert_to_dict.py",
             "function": "convert_to_dict"
         }
-    }
-}
+    },
+    "package": {}
+}

src/promptflow/tests/test_configs/flows/web_classification_random_fail/classify_with_llm.jinja2

@@ -0,0 +1,18 @@
+system:
+Your task is to classify a given url into one of the following types:
+Movie, App, Academic, Channel, Profile, PDF or None based on the text content information.
+The classification will be based on the url, the webpage text content summary, or both.
+
+user:
+Here are a few examples:
+{% for ex in examples %}
+URL: {{ex.url}}
+Text content: {{ex.text_content}}
+OUTPUT:
+{"category": "{{ex.category}}", "evidence": "{{ex.evidence}}"}
+
+{% endfor %}
+
+For a given URL : {{url}}, and text content: {{text_content}}.
+Classify above url to complete the category and indicate evidence.
+OUTPUT:

src/promptflow/tests/test_configs/flows/web_classification_random_fail/convert_to_dict.py

@@ -0,0 +1,12 @@
+import json
+
+from promptflow import tool
+
+
+@tool
+def convert_to_dict(input_str: str):
+    try:
+        return json.loads(input_str)
+    except Exception as e:
+        print("input is not valid, error: {}".format(e))
+        return {"category": "None", "evidence": "None"}

src/promptflow/tests/test_configs/flows/web_classification_random_fail/data.jsonl

@@ -0,0 +1,9 @@
+{"url": "https://www.youtube.com/watch?v=kYqRtjDBci8", "answer": "Channel", "evidence": "Both"}
+{"url": "https://arxiv.org/abs/2307.04767", "answer": "Academic", "evidence": "Both"}
+{"url": "https://play.google.com/store/apps/details?id=com.twitter.android", "answer": "App", "evidence": "Both"}
+{"url": "https://www.youtube.com/watch?v=kYqRtjDBci8", "answer": "Channel", "evidence": "Both"}
+{"url": "https://arxiv.org/abs/2307.04767", "answer": "Academic", "evidence": "Both"}
+{"url": "https://play.google.com/store/apps/details?id=com.twitter.android", "answer": "App", "evidence": "Both"}
+{"url": "https://www.youtube.com/watch?v=kYqRtjDBci8", "answer": "Channel", "evidence": "Both"}
+{"url": "https://arxiv.org/abs/2307.04767", "answer": "Academic", "evidence": "Both"}
+{"url": "https://play.google.com/store/apps/details?id=com.twitter.android", "answer": "App", "evidence": "Both"}

src/promptflow/tests/test_configs/flows/web_classification_random_fail/fetch_text_content_from_url.py

@@ -0,0 +1,30 @@
+import bs4
+import requests
+import random
+from promptflow import tool
+
+
+@tool
+def fetch_text_content_from_url(url: str):
+    if random.random() < 0.5:
+        raise ValueError("Random failure")
+    # Send a request to the URL
+    try:
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.35"
+        }
+        response = requests.get(url, headers=headers)
+        if response.status_code == 200:
+            # Parse the HTML content using BeautifulSoup
+            soup = bs4.BeautifulSoup(response.text, "html.parser")
+            soup.prettify()
+            return soup.get_text()[:2000]
+        else:
+            msg = (
+                f"Get url failed with status code {response.status_code}.\nURL: {url}\nResponse: {response.text[:100]}"
+            )
+            print(msg)
+            return "No available content"
+    except Exception as e:
+        print("Get url failed with error: {}".format(e))
+        return "No available content"