From 357b2c71fe8af6c7b509ed12d9f610cd7901dbd0 Mon Sep 17 00:00:00 2001 From: Ying Chen <2601502859@qq.com> Date: Sat, 9 Mar 2024 18:58:44 +0800 Subject: [PATCH] update --- .github/workflows/build_msi_installer.yml | 6 ++ .../windows/scripts/generate_dependency.py | 84 +++++++++++++++++++ ...promptflow.spec => promptflow.spec.jinja2} | 29 +++++-- 3 files changed, 111 insertions(+), 8 deletions(-) create mode 100644 scripts/installer/windows/scripts/generate_dependency.py rename scripts/installer/windows/scripts/{promptflow.spec => promptflow.spec.jinja2} (67%) diff --git a/.github/workflows/build_msi_installer.yml b/.github/workflows/build_msi_installer.yml index 5513f34ca82..f252d5d6d04 100644 --- a/.github/workflows/build_msi_installer.yml +++ b/.github/workflows/build_msi_installer.yml @@ -109,6 +109,12 @@ jobs: scriptPath: ${{ env.testWorkingDirectory }} + - name: Generate promptflow spec file to config pyinstaller + working-directory: ${{ github.workspace }}/scripts/installer/windows/scripts + run: | + python generate_dependency.py + Get-Content promptflow.spec + - name: Build Pyinstaller project working-directory: ${{ github.workspace }}/scripts/installer/windows/scripts run: | diff --git a/scripts/installer/windows/scripts/generate_dependency.py b/scripts/installer/windows/scripts/generate_dependency.py new file mode 100644 index 00000000000..a701d28febd --- /dev/null +++ b/scripts/installer/windows/scripts/generate_dependency.py @@ -0,0 +1,84 @@ +import ast +import re +import subprocess +import copy +from pathlib import Path +from promptflow._sdk._utils import render_jinja_template + +def extract_requirements(file_path): + with open(file_path, 'r') as file: + tree = ast.parse(file.read()) + + install_requires = [] + extras_requires = {} + for node in ast.walk(tree): + if isinstance(node, ast.Assign) and node.targets[0].id == 'REQUIRES': + install_requires = [elt.s for elt in node.value.elts] + elif isinstance(node, ast.Call) and getattr(node.func, 'id', None) == 'setup': + for keyword in node.keywords: + if keyword.arg == 'extras_require': + extras_requires = ast.literal_eval(keyword.value) + return install_requires, extras_requires + + +def extract_package_names(packages): + package_names = [] + for package in packages: + match = re.match(r'^([a-zA-Z0-9-_.]+)', package) + if match: + package_names.append(match.group(1)) + return package_names + + +def get_package_dependencies(package_name_list): + dependencies = [] + for package_name in package_name_list: + result = subprocess.run(['pip', 'show', package_name], stdout=subprocess.PIPE) + lines = result.stdout.decode('utf-8', errors="ignore").splitlines() + for line in lines: + if line.startswith('Requires'): + dependency = line.split(': ')[1].split(', ') + if dependency != ['']: + dependencies.extend(dependency) + break + return dependencies + + +if __name__ == '__main__': + dependencies = [] + install_requires, extras_requires = extract_requirements('../../../../src/promptflow/setup.py') + install_requires_names = extract_package_names(install_requires) + dependencies.extend(install_requires_names) + + for key in extras_requires: + extras_require_names = extract_package_names(extras_requires[key]) + dependencies.extend(extras_require_names) + direct_package_dependencies = get_package_dependencies(dependencies) + all_packages = list(set(dependencies) | set(direct_package_dependencies)) + hidden_imports = copy.deepcopy(all_packages) + meta_packages = copy.deepcopy(all_packages) + + special_packages = ["streamlit-quill", "flask-cors", "flask-restx"] + for i in range(len(hidden_imports)): + # need special handeling because it use _ to import + if hidden_imports[i] in special_packages: + hidden_imports[i] = hidden_imports[i].replace('-', '_').lower() + else: + hidden_imports[i] = hidden_imports[i].replace('-', '.').lower() + + hidden_imports.remove("azure.storage.file.share") + hidden_imports.append("azure.storage.fileshare") + hidden_imports.remove("azure.storage.file.datalake") + hidden_imports.append("azure.storage.filedatalake") + + + render_context = { + "hidden_imports": hidden_imports, + "all_packages": all_packages, + "meta_packages": meta_packages, + } + # always use unix line ending + Path("./promptflow.spec").write_bytes( + render_jinja_template("./promptflow.spec.jinja2", **render_context) + .encode("utf-8") + .replace(b"\r\n", b"\n"),) diff --git a/scripts/installer/windows/scripts/promptflow.spec b/scripts/installer/windows/scripts/promptflow.spec.jinja2 similarity index 67% rename from scripts/installer/windows/scripts/promptflow.spec rename to scripts/installer/windows/scripts/promptflow.spec.jinja2 index a1f3955be85..4c78a22f8b7 100644 --- a/scripts/installer/windows/scripts/promptflow.spec +++ b/scripts/installer/windows/scripts/promptflow.spec.jinja2 @@ -1,25 +1,38 @@ # -*- mode: python ; coding: utf-8 -*- -from PyInstaller.utils.hooks import collect_data_files -from PyInstaller.utils.hooks import copy_metadata +from PyInstaller.utils.hooks import collect_data_files, collect_all, copy_metadata datas = [('../resources/CLI_LICENSE.rtf', '.'), ('../../../../src/promptflow/NOTICE.txt', '.'), ('../../../../src/promptflow/promptflow/_sdk/data/executable/', './promptflow/_sdk/data/executable/'), ('../../../../src/promptflow-tools/promptflow/tools/', './promptflow/tools/'), ('./pf.bat', '.'), ('./pfs.bat', '.'), ('./pfazure.bat', '.'), ('./pfsvc.bat', '.')] -datas += collect_data_files('streamlit') -datas += copy_metadata('streamlit') + +all_packages = {{all_packages}} +meta_packages = {{meta_packages}} + +for package in all_packages: + datas += collect_data_files(package) + +for package in meta_packages: + datas += copy_metadata(package) + +opentelemetry_datas, opentelemetry_binaries, opentelemetry_hiddenimports = collect_all('opentelemetry') +datas += opentelemetry_datas datas += collect_data_files('streamlit_quill') datas += collect_data_files('promptflow') -datas += copy_metadata('opentelemetry-sdk') -hidden_imports = ['streamlit.runtime.scriptrunner.magic_funcs', 'win32timezone', 'promptflow', 'opentelemetry.exporter.otlp.proto.http'] +datas += copy_metadata('promptflow') +hidden_imports = ['win32timezone', 'promptflow', 'opentelemetry.context.contextvars_context', 'streamlit.runtime.scriptrunner.magic_funcs'] + {{hidden_imports}} + +hidden_imports += opentelemetry_hiddenimports +binaries = [] +binaries += opentelemetry_binaries block_cipher = None pfcli_a = Analysis( ['pfcli.py'], pathex=[], - binaries=[], + binaries=binaries, datas=datas, hiddenimports=hidden_imports, hookspath=[], @@ -62,4 +75,4 @@ coll = COLLECT( upx=True, upx_exclude=[], name='promptflow', -) +) \ No newline at end of file