From 02aed9e6973a2017dc7a1a4a524919430226d823 Mon Sep 17 00:00:00 2001 From: Ying Chen Date: Thu, 26 Oct 2023 13:52:16 +0800 Subject: [PATCH] Add composite image CLI test and refine image output (#905) # Description Please add an informative description that covers that changes made by the pull request and link all relevant issues. # All Promptflow Contribution checklist: - [ ] **The pull request does not introduce [breaking changes].** - [ ] **CHANGELOG is updated for new features, bug fixes or other significant changes.** - [ ] **I have read the [contribution guidelines](../CONTRIBUTING.md).** - [ ] **Create an issue and link to the pull request to get dedicated review from promptflow team. Learn more: [suggested workflow](../CONTRIBUTING.md#suggested-workflow).** ## General Guidelines and Best Practices - [ ] Title of the pull request is clear and informative. - [ ] There are a small number of commits, each of which have an informative message. This means that previously merged commits do not appear in the history of the PR. For more information on cleaning up the commits in your PR, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md). ### Testing Guidelines - [ ] Pull request includes test coverage for the included changes. --------- Co-authored-by: Ying Chen <2601502859@qq.com> --- .../distribute-flow-as-executable-app.md | 84 +------- .../README.md | 82 -------- .../app.spec | 52 +++++ .../distribute-flow-as-executable-app/main.py | 183 ++++++++++++++++-- .../promptflow/_sdk/_serving/flow_invoker.py | 2 + .../_sdk/data/executable/main.py.jinja2 | 4 +- .../tests/sdk_cli_test/e2etests/test_cli.py | 12 ++ 7 files changed, 232 insertions(+), 187 deletions(-) create mode 100644 examples/tutorials/flow-deploy/distribute-flow-as-executable-app/app.spec diff --git a/docs/how-to-guides/deploy-a-flow/distribute-flow-as-executable-app.md b/docs/how-to-guides/deploy-a-flow/distribute-flow-as-executable-app.md index 74823d70136..2995b668de3 100644 --- a/docs/how-to-guides/deploy-a-flow/distribute-flow-as-executable-app.md +++ b/docs/how-to-guides/deploy-a-flow/distribute-flow-as-executable-app.md @@ -43,9 +43,6 @@ Exported files & its dependencies are located in the same folder. The structure ### A template script of the entry file PyInstaller reads a spec file or Python script written by you. It analyzes your code to discover every other module and library your script needs in order to execute. Then it collects copies of all those files, including the active Python interpreter, and puts them with your script in a single folder, or optionally in a single executable file. -::::{tab-set} -:::{tab-item} app.py -:sync: app.py We provide a Python entry script named `app.py` as the entry point for the bundled app, which enables you to serve a flow folder as an endpoint. ```python @@ -80,87 +77,7 @@ if __name__ == "__main__": st_cli.main(prog_name="streamlit") ``` -::: - -:::{tab-item} main.py -:sync: main.py -The `main.py` file will start streamlit service and be called by the entry file. - -```python -import json -import os -import streamlit as st -from pathlib import Path - -from promptflow._sdk._utils import print_yellow_warning -from promptflow._sdk._serving.flow_invoker import FlowInvoker - - -invoker = None - - -def start(): - def clear_chat() -> None: - st.session_state.messages = [] - - def show_conversation() -> None: - if "messages" not in st.session_state: - st.session_state.messages = [] - if st.session_state.messages: - for role, message in st.session_state.messages: - st.chat_message(role).write(message) - - def submit(**kwargs) -> None: - container.chat_message("user").write(json.dumps(kwargs)) - st.session_state.messages.append(("user", json.dumps(kwargs))) - response = run_flow(kwargs) - container.chat_message("assistant").write(response) - st.session_state.messages.append(("assistant", response)) - - - def run_flow(data: dict) -> dict: - global invoker - if not invoker: - flow = Path(__file__).parent / "flow" - os.chdir(flow) - invoker = FlowInvoker(flow, connection_provider="local") - result = invoker.invoke(data) - print_yellow_warning(f"Result: {result}") - return result - - - st.title("web-classification APP") - st.chat_message("assistant").write("Hello, please input following flow inputs and connection keys.") - container = st.container() - with container: - show_conversation() - - with st.form(key='input_form', clear_on_submit=True): - with open(os.path.join(os.path.dirname(__file__), "settings.json"), "r") as file: - json_data = json.load(file) - environment_variables = list(json_data.keys()) - for environment_variable in environment_variables: - secret_input = st.text_input(label=environment_variable, type="password", placeholder=f"Please input {environment_variable} here. If you input before, you can leave it blank.") - if secret_input != "": - os.environ[environment_variable] = secret_input - - url = st.text_input(label='url', placeholder='https://play.google.com/store/apps/details?id=com.twitter.android') - cols = st.columns(7) - submit_bt = cols[0].form_submit_button(label='Submit') - clear_bt = cols[1].form_submit_button(label='Clear') - - if submit_bt: - submit(url=url) - - if clear_bt: - clear_chat() - -if __name__ == "__main__": - start() -``` -::: -:::: ### A template script of the spec file The spec file tells PyInstaller how to process your script. It encodes the script names and most of the options you give to the pyinstaller command. The spec file is actually executable Python code. PyInstaller builds the app by executing the contents of the spec file. @@ -177,6 +94,7 @@ datas += collect_data_files('streamlit') datas += copy_metadata('streamlit') datas += collect_data_files('keyrings.alt', include_py_files=True) datas += copy_metadata('keyrings.alt') +datas += collect_data_files('streamlit_quill') block_cipher = None diff --git a/examples/tutorials/flow-deploy/distribute-flow-as-executable-app/README.md b/examples/tutorials/flow-deploy/distribute-flow-as-executable-app/README.md index b06fc7b8ad2..274efa28fcf 100644 --- a/examples/tutorials/flow-deploy/distribute-flow-as-executable-app/README.md +++ b/examples/tutorials/flow-deploy/distribute-flow-as-executable-app/README.md @@ -35,9 +35,6 @@ Exported files & its dependencies are located in the same folder. The structure ### A template script of the entry file PyInstaller reads a spec file or Python script written by you. It analyzes your code to discover every other module and library your script needs in order to execute. Then it collects copies of all those files, including the active Python interpreter, and puts them with your script in a single folder, or optionally in a single executable file. -::::{tab-set} -:::{tab-item} app.py -:sync: app.py We provide a Python entry script named `app.py` as the entry point for the bundled app, which enables you to serve a flow folder as an endpoint. ```python @@ -73,85 +70,6 @@ if __name__ == "__main__": ``` ::: -:::{tab-item} main.py -:sync: main.py -The `main.py` file will start Streamlit service and be called by the entry file. - -```python -import json -import os -import streamlit as st -from pathlib import Path - -from promptflow._sdk._utils import print_yellow_warning -from promptflow._sdk._serving.flow_invoker import FlowInvoker - - -invoker = None - - -def start(): - def clear_chat() -> None: - st.session_state.messages = [] - - def show_conversation() -> None: - if "messages" not in st.session_state: - st.session_state.messages = [] - if st.session_state.messages: - for role, message in st.session_state.messages: - st.chat_message(role).write(message) - - - def submit(**kwargs) -> None: - container.chat_message("user").write(json.dumps(kwargs)) - st.session_state.messages.append(("user", json.dumps(kwargs))) - response = run_flow(kwargs) - container.chat_message("assistant").write(response) - st.session_state.messages.append(("assistant", response)) - - - def run_flow(data: dict) -> dict: - global invoker - if not invoker: - flow = Path(__file__).parent / "flow" - os.chdir(flow) - invoker = FlowInvoker(flow, connection_provider="local") - result = invoker.invoke(data) - print_yellow_warning(f"Result: {result}") - return result - - - st.title("web-classification APP") - st.chat_message("assistant").write("Hello, please input following flow inputs and connection keys.") - container = st.container() - with container: - show_conversation() - - with st.form(key='input_form', clear_on_submit=True): - with open(os.path.join(os.path.dirname(__file__), "settings.json"), "r") as file: - json_data = json.load(file) - environment_variables = list(json_data.keys()) - for environment_variable in environment_variables: - secret_input = st.text_input(label=environment_variable, type="password", placeholder=f"Please input {environment_variable} here. If you input before, you can leave it blank.") - if secret_input != "": - os.environ[environment_variable] = secret_input - - url = st.text_input(label='url', placeholder='https://play.google.com/store/apps/details?id=com.twitter.android') - cols = st.columns(7) - submit_bt = cols[0].form_submit_button(label='Submit') - clear_bt = cols[1].form_submit_button(label='Clear') - - if submit_bt: - submit(url=url) - - if clear_bt: - clear_chat() - -if __name__ == "__main__": - start() -``` -::: -:::: ### A template script of the spec file The spec file tells PyInstaller how to process your script. It encodes the script names and most of the options you give to the pyinstaller command. The spec file is actually executable Python code. PyInstaller builds the app by executing the contents of the spec file. diff --git a/examples/tutorials/flow-deploy/distribute-flow-as-executable-app/app.spec b/examples/tutorials/flow-deploy/distribute-flow-as-executable-app/app.spec new file mode 100644 index 00000000000..f37ca337611 --- /dev/null +++ b/examples/tutorials/flow-deploy/distribute-flow-as-executable-app/app.spec @@ -0,0 +1,52 @@ +# -*- mode: python ; coding: utf-8 -*- +from PyInstaller.utils.hooks import collect_data_files +from PyInstaller.utils.hooks import copy_metadata + +datas = [('connections', 'connections'), ('flow', 'flow'), ('settings.json', '.'), ('main.py', '.'), ('{{streamlit_runtime_interpreter_path}}', './streamlit/runtime')] +datas += collect_data_files('streamlit') +datas += copy_metadata('streamlit') +datas += collect_data_files('keyrings.alt', include_py_files=True) +datas += copy_metadata('keyrings.alt') +datas += collect_data_files('streamlit_quill') + +block_cipher = None + + +a = Analysis( + ['app.py', 'main.py'], + pathex=[], + binaries=[], + datas=datas, + hiddenimports=['bs4'], + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False, +) +pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) + +exe = EXE( + pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + [], + name='app', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + upx_exclude=[], + runtime_tmpdir=None, + console=True, + disable_windowed_traceback=False, + argv_emulation=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None, +) \ No newline at end of file diff --git a/examples/tutorials/flow-deploy/distribute-flow-as-executable-app/main.py b/examples/tutorials/flow-deploy/distribute-flow-as-executable-app/main.py index 3989967ecef..89d4def2fea 100644 --- a/examples/tutorials/flow-deploy/distribute-flow-as-executable-app/main.py +++ b/examples/tutorials/flow-deploy/distribute-flow-as-executable-app/main.py @@ -1,10 +1,15 @@ +import base64 import json import os +import re import streamlit as st from pathlib import Path +from streamlit_quill import st_quill # noqa: F401 +from bs4 import BeautifulSoup, NavigableString, Tag from promptflow._sdk._utils import print_yellow_warning from promptflow._sdk._serving.flow_invoker import FlowInvoker +from promptflow._utils.multimedia_utils import is_multimedia_dict, MIME_PATTERN invoker = None @@ -13,46 +18,186 @@ def start(): def clear_chat() -> None: st.session_state.messages = [] + def show_image(image, key=None): + if not image.startswith("data:image"): + st.image(key + ',' + image) + else: + st.image(image) + + def json_dumps(value): + try: + return json.dumps(value) + except Exception: + return value + + def is_list_contains_rich_text(rich_text): + result = False + for item in rich_text: + if isinstance(item, list): + result |= is_list_contains_rich_text(item) + elif isinstance(item, dict): + result |= is_dict_contains_rich_text(item) + else: + if isinstance(item, str) and item.startswith("data:image"): + result = True + return result + + def is_dict_contains_rich_text(rich_text): + result = False + for rich_text_key, rich_text_value in rich_text.items(): + if isinstance(rich_text_value, list): + result |= is_list_contains_rich_text(rich_text_value) + elif isinstance(rich_text_value, dict): + result |= is_dict_contains_rich_text(rich_text_value) + elif re.match(MIME_PATTERN, rich_text_key) or ( + isinstance(rich_text_value, str) and rich_text_value.startswith("data:image")): + result = True + return result + + def render_message(role, message_items): + + def item_render_message(value, key=None): + if key and re.match(MIME_PATTERN, key): + show_image(value, key) + elif isinstance(value, str) and value.startswith("data:image"): + show_image(value) + else: + if key is None: + st.markdown(f"`{json_dumps(value)},`") + else: + st.markdown(f"`{key}: {json_dumps(value)},`") + + def list_iter_render_message(message_items): + if is_list_contains_rich_text(message_items): + st.markdown("`[ `") + for item in message_items: + if isinstance(item, list): + list_iter_render_message(item) + if isinstance(item, dict): + dict_iter_render_message(item) + else: + item_render_message(item) + st.markdown("`], `") + else: + st.markdown(f"`{json_dumps(message_items)},`") + + def dict_iter_render_message(message_items): + if is_multimedia_dict(message_items): + key = list(message_items.keys())[0] + value = message_items[key] + show_image(value, key) + elif is_dict_contains_rich_text(message_items): + st.markdown("`{ `") + for key, value in message_items.items(): + if re.match(MIME_PATTERN, key): + show_image(value, key) + else: + if isinstance(value, list): + st.markdown(f"`{key}: `") + list_iter_render_message(value) + elif isinstance(value, dict): + st.markdown(f"`{key}: `") + dict_iter_render_message(value) + else: + item_render_message(value, key) + st.markdown("`}, `") + else: + st.markdown(f"`{json_dumps(message_items)},`") + + with st.chat_message(role): + dict_iter_render_message(message_items) + def show_conversation() -> None: if "messages" not in st.session_state: st.session_state.messages = [] + st.session_state.history = [] if st.session_state.messages: - for role, message in st.session_state.messages: - st.chat_message(role).write(message) + for role, message_items in st.session_state.messages: + render_message(role, message_items) + + def get_chat_history_from_session(): + if "history" in st.session_state: + return st.session_state.history + return [] def submit(**kwargs) -> None: - container.chat_message("user").write(json.dumps(kwargs)) - st.session_state.messages.append(("user", json.dumps(kwargs))) + st.session_state.messages.append(("user", kwargs)) + session_state_history = dict() + session_state_history.update({"inputs": kwargs}) + with container: + render_message("user", kwargs) + # Force append chat history to kwargs response = run_flow(kwargs) - container.chat_message("assistant").write(response) st.session_state.messages.append(("assistant", response)) + session_state_history.update({"outputs": response}) + st.session_state.history.append(session_state_history) + with container: + render_message("assistant", response) def run_flow(data: dict) -> dict: global invoker if not invoker: flow = Path(__file__).parent / "flow" - os.chdir(flow) - invoker = FlowInvoker(flow, connection_provider="local") - result = invoker.invoke(data) - print_yellow_warning(f"Result: {result}") + dump_path = flow.parent + if flow.is_dir(): + os.chdir(flow) + else: + os.chdir(flow.parent) + invoker = FlowInvoker(flow, connection_provider="local", dump_to=dump_path) + result, result_output = invoker.invoke(data) + print_yellow_warning(f"Result: {result_output}") return result + def extract_content(node): + if isinstance(node, NavigableString): + text = node.strip() + if text: + return [text] + elif isinstance(node, Tag): + if node.name == 'img': + prefix, base64_str = node['src'].split(',', 1) + return [{prefix: base64_str}] + else: + result = [] + for child in node.contents: + result.extend(extract_content(child)) + return result + return [] + + def parse_html_content(html_content): + soup = BeautifulSoup(html_content, 'html.parser') + result = [] + for p in soup.find_all('p'): + result.extend(extract_content(p)) + return result + + def parse_image_content(image_content, image_type): + if image_content is not None: + file_contents = image_content.read() + image_content = base64.b64encode(file_contents).decode('utf-8') + prefix = f"data:{image_type};base64" + return {prefix: image_content} + st.title("web-classification APP") - st.chat_message("assistant").write("Hello, please input following flow inputs and connection keys.") + st.chat_message("assistant").write("Hello, please input following flow inputs.") container = st.container() with container: show_conversation() with st.form(key='input_form', clear_on_submit=True): - with open(os.path.join(os.path.dirname(__file__), "settings.json"), "r") as file: - json_data = json.load(file) - environment_variables = list(json_data.keys()) - for environment_variable in environment_variables: - secret_input = st.text_input(label=environment_variable, type="password", - placeholder=f"Please input {environment_variable} here. If you input before, " - f"you can leave it blank.") - if secret_input != "": - os.environ[environment_variable] = secret_input + settings_path = os.path.join(os.path.dirname(__file__), "settings.json") + if os.path.exists(settings_path): + with open(settings_path, "r") as file: + json_data = json.load(file) + environment_variables = list(json_data.keys()) + for environment_variable in environment_variables: + secret_input = st.text_input( + label=environment_variable, + type="password", + placeholder=f"Please input {environment_variable} here. If you input before, you can leave it " + f"blank.") + if secret_input != "": + os.environ[environment_variable] = secret_input url = st.text_input(label='url', placeholder='https://play.google.com/store/apps/details?id=com.twitter.android') diff --git a/src/promptflow/promptflow/_sdk/_serving/flow_invoker.py b/src/promptflow/promptflow/_sdk/_serving/flow_invoker.py index aadc7201c02..95cbe746b02 100644 --- a/src/promptflow/promptflow/_sdk/_serving/flow_invoker.py +++ b/src/promptflow/promptflow/_sdk/_serving/flow_invoker.py @@ -16,6 +16,7 @@ override_connection_config_with_environment_variable, resolve_connections_environment_variable_reference, update_environment_variables_with_connections, + print_yellow_warning, ) from promptflow._sdk.entities._connection import _Connection from promptflow._sdk.entities._flow import Flow @@ -124,4 +125,5 @@ def invoke(self, data: dict): result.output, base_dir=self._dump_to, sub_dir=Path(".promptflow/output") ) dump_flow_result(flow_folder=self._dump_to, flow_result=result, prefix=self._dump_file_prefix) + print_yellow_warning(f"Result: {result.output}") return resolved_outputs diff --git a/src/promptflow/promptflow/_sdk/data/executable/main.py.jinja2 b/src/promptflow/promptflow/_sdk/data/executable/main.py.jinja2 index 92df88619b8..d93479498c6 100644 --- a/src/promptflow/promptflow/_sdk/data/executable/main.py.jinja2 +++ b/src/promptflow/promptflow/_sdk/data/executable/main.py.jinja2 @@ -7,7 +7,6 @@ from pathlib import Path from streamlit_quill import st_quill from bs4 import BeautifulSoup, NavigableString, Tag -from promptflow._sdk._utils import print_yellow_warning from promptflow._sdk._serving.flow_invoker import FlowInvoker from promptflow._utils.multimedia_utils import is_multimedia_dict, MIME_PATTERN @@ -152,7 +151,7 @@ def start(): {{ ' ' * indent_level * 3 }}dump_path = Path('{{flow_path}}').parent {% else %} {{ ' ' * indent_level * 3 }}flow = Path(__file__).parent / "flow" -{{ ' ' * indent_level * 3 }}dump_path = None +{{ ' ' * indent_level * 3 }}dump_path = flow.parent {% endif %} if flow.is_dir(): os.chdir(flow) @@ -160,7 +159,6 @@ def start(): os.chdir(flow.parent) invoker = FlowInvoker(flow, connection_provider="local", dump_to=dump_path) result = invoker.invoke(data) - print_yellow_warning(f"Result: {result}") return result diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_cli.py b/src/promptflow/tests/sdk_cli_test/e2etests/test_cli.py index 8520b6c5643..950fae0d231 100644 --- a/src/promptflow/tests/sdk_cli_test/e2etests/test_cli.py +++ b/src/promptflow/tests/sdk_cli_test/e2etests/test_cli.py @@ -1494,6 +1494,18 @@ def test_flow_test_with_image_input_and_output(self): image_path = Path(FLOWS_DIR) / "python_tool_with_simple_image" / ".promptflow" / "intermediate" assert image_path.exists() + def test_flow_test_with_composite_image(self): + run_pf_command( + "flow", + "test", + "--flow", + f"{FLOWS_DIR}/python_tool_with_composite_image", + ) + output_path = Path(FLOWS_DIR) / "python_tool_with_composite_image" / ".promptflow" / "output" + assert output_path.exists() + image_path = Path(FLOWS_DIR) / "python_tool_with_composite_image" / ".promptflow" / "intermediate" + assert image_path.exists() + def test_run_file_with_set(self, pf) -> None: name = str(uuid.uuid4()) run_pf_command(