Chenyin/validate msi

.cspell.json

@@ -38,8 +38,6 @@
     "**/setup.py",
     "scripts/installer/curl_install_pypi/**",
     "scripts/installer/windows/**",
-    "src/promptflow/promptflow/_sdk/_service/pfsvc.py",
-    "src/promptflow-devkit/promptflow/_sdk/_service/pfsvc.py",
     ".github/workflows/**",
     ".github/actions/**",
     ".github/pipelines/**"
@@ -94,7 +92,6 @@
     "nunit",
     "astext",
     "Likert",
-    "pfsvc",
     "geval",
     "Summ",
     "Bhavik",

.github/workflows/build_msi_installer.yml

@@ -74,7 +74,7 @@ jobs:
             if ($version -ne $run_version) {
               throw "Version input does not match the version in promptflow package. Version input: $version, version in promptflow package: $run_version"
             }
-          } 
+          }
           elseif ($env:MSI_PRIVATE_VERSION) {
             $version=$env:MSI_PRIVATE_VERSION
           }
@@ -108,6 +108,19 @@ jobs:
           setupType: promptflow_with_extra
           scriptPath: ${{ env.testWorkingDirectory }}
 
+      - name: Install wheel
+        shell: pwsh
+        run: |
+          Set-PSDebug -Trace 1
+          pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
+          pip install ${{ github.workspace }}/src/promptflow-tracing
+          pip install ${{ github.workspace }}/src/promptflow-core[executor-service]
+          pip install ${{ github.workspace }}/src/promptflow-devkit[pyarrow,executable]
+          pip install ${{ github.workspace }}/src/promptflow-azure
+          echo "Should fix this after pf-core could install this dependency"
+          pip install azureml-ai-monitoring
+          pip freeze
+
       - name: Generate promptflow spec file to config pyinstaller
         working-directory: ${{ github.workspace }}/scripts/installer/windows/scripts
         run: |
@@ -138,7 +151,7 @@ jobs:
         shell: pwsh
 
       - name: Generate portable promptflow
-        run: | 
+        run: |
           Compress-Archive -Path ${{ github.workspace }}/scripts/installer/windows/scripts/dist/promptflow -DestinationPath promptflow-${{ steps.get-version.outputs.version }}.zip
         shell: pwsh
 
@@ -177,7 +190,7 @@ jobs:
               "promptflow" = $version
             } | ConvertTo-Json -Depth 100
             $jsonContent | Out-File -FilePath latest_version.json -Encoding UTF8
-            
+
             Write-Output "Created latest_version.json with version: $version"
             az storage blob upload --account-name promptflowartifact --container-name msi-installer --file "latest_version.json" --name "latest_version.json" --overwrite
           } else {

.github/workflows/promptflow-sdk-cli-test.yml

@@ -179,4 +179,4 @@ jobs:
         osVersion: ubuntu-latest
         pythonVersion: 3.9
         coverageThreshold: 40
-        context: test/sdk_cli
+        context: test/sdk_cli

scripts/installer/curl_install_pypi/README.md

@@ -17,15 +17,12 @@ Uninstall the promptflow by directly deleting the files from the location chosen
    ```bash
    # The default install/executable location is the user's home directory ($HOME).
    rm -r $HOME/lib/promptflow
-   rm $HOME/bin/pf
-   rm $HOME/bin/pfs
-   rm $HOME/bin/pfazure
    ```
 
 2. Modify your `$HOME/.bash_profile` or `$HOME/.bashrc` file to remove the following line:
 
    ```text
-   export PATH=$PATH:$HOME/bin
+   export PATH=$PATH:$HOME/lib/promptflow/bin
    ```
 
 3. If using `bash` or `zsh`, reload your shell's command cache.

scripts/installer/curl_install_pypi/install.py

@@ -15,31 +15,15 @@
 import os
 import sys
 import platform
-import stat
 import tempfile
 import shutil
 import subprocess
-import hashlib
 
 
-PF_DISPATCH_TEMPLATE = """#!/usr/bin/env bash
-export PF_INSTALLER=Script
-{install_dir}/bin/python -m promptflow._cli._pf.entry "$@"
-"""
-
-PFAZURE_DISPATCH_TEMPLATE = """#!/usr/bin/env bash
-{install_dir}/bin/python -m promptflow.azure._cli.entry "$@"
-"""
-
-PFS_DISPATCH_TEMPLATE = """#!/usr/bin/env bash
-{install_dir}/bin/python -m promptflow._sdk._service.entry "$@"
-"""
-
 DEFAULT_INSTALL_DIR = os.path.expanduser(os.path.join('~', 'lib', 'promptflow'))
 DEFAULT_EXEC_DIR = os.path.expanduser(os.path.join('~', 'bin'))
 PF_EXECUTABLE_NAME = 'pf'
 PFAZURE_EXECUTABLE_NAME = 'pfazure'
-PFS_EXECUTABLE_NAME = 'pfs'
 
 
 USER_BASH_RC = os.path.expanduser(os.path.join('~', '.bashrc'))
@@ -96,46 +80,22 @@ def create_dir(dir):
         os.makedirs(dir)
 
 
-def is_valid_sha256sum(a_file, expected_sum):
-    sha256 = hashlib.sha256()
-    with open(a_file, 'rb') as f:
-        sha256.update(f.read())
-    computed_hash = sha256.hexdigest()
-    return expected_sum == computed_hash
-
-
 def create_virtualenv(install_dir):
     cmd = [sys.executable, '-m', 'venv', install_dir]
     exec_command(cmd)
 
 
 def install_cli(install_dir, tmp_dir):
     path_to_pip = os.path.join(install_dir, 'bin', 'pip')
-    cmd = [path_to_pip, 'install', '--cache-dir', tmp_dir, 'promptflow[azure,executable,azureml-serving]',
-           '--upgrade']
+    cmd = [path_to_pip, 'install', '--cache-dir', tmp_dir,
+           'promptflow[azure,executable,azureml-serving,executor-service]', '--upgrade']
     exec_command(cmd)
     cmd = [path_to_pip, 'install', '--cache-dir', tmp_dir, 'promptflow-tools', '--upgrade']
     exec_command(cmd)
     cmd = [path_to_pip, 'install', '--cache-dir', tmp_dir, 'keyrings.alt', '--upgrade']
     exec_command(cmd)
 
 
-def create_executable(exec_dir, install_dir):
-    create_dir(exec_dir)
-    exec_filepaths = []
-    for filename, template in [(PF_EXECUTABLE_NAME, PF_DISPATCH_TEMPLATE),
-                               (PFAZURE_EXECUTABLE_NAME, PFAZURE_DISPATCH_TEMPLATE),
-                               (PFS_EXECUTABLE_NAME, PFS_DISPATCH_TEMPLATE)]:
-        exec_filepath = os.path.join(exec_dir, filename)
-        with open(exec_filepath, 'w') as exec_file:
-            exec_file.write(template.format(install_dir=install_dir))
-        cur_stat = os.stat(exec_filepath)
-        os.chmod(exec_filepath, cur_stat.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
-        print_status("The executable is available at '{}'.".format(exec_filepath))
-        exec_filepaths.append(exec_filepath)
-    return exec_filepaths
-
-
 def get_install_dir():
     install_dir = None
     while not install_dir:
@@ -161,21 +121,6 @@ def get_install_dir():
     return install_dir
 
 
-def get_exec_dir():
-    exec_dir = None
-    while not exec_dir:
-        prompt_message = (f"In what directory would you like to place the "
-                          f"'{PFS_EXECUTABLE_NAME}/{PFS_EXECUTABLE_NAME}/{PFAZURE_EXECUTABLE_NAME}' executable?")
-        exec_dir = prompt_input_with_default(prompt_message, DEFAULT_EXEC_DIR)
-        exec_dir = os.path.realpath(os.path.expanduser(exec_dir))
-        if ' ' in exec_dir:
-            print_status("The executable directory '{}' cannot contain spaces.".format(exec_dir))
-            exec_dir = None
-    create_dir(exec_dir)
-    print_status("The executable will be in '{}'.".format(exec_dir))
-    return exec_dir
-
-
 def _backup_rc(rc_file):
     try:
         shutil.copyfile(rc_file, rc_file+'.backup')
@@ -240,28 +185,28 @@ def warn_other_azs_on_path(exec_dir, exec_filepath):
     conflicting_paths = []
     if env_path:
         for p in env_path.split(':'):
-            for file in [PF_EXECUTABLE_NAME, PFAZURE_EXECUTABLE_NAME, PFS_EXECUTABLE_NAME]:
+            for file in [PF_EXECUTABLE_NAME, PFAZURE_EXECUTABLE_NAME]:
                 p_to_pf = os.path.join(p, file)
                 if p != exec_dir and os.path.isfile(p_to_pf):
                     conflicting_paths.append(p_to_pf)
     if conflicting_paths:
         print_status()
-        print_status(f"** WARNING: Other '{PFS_EXECUTABLE_NAME}/{PFS_EXECUTABLE_NAME}/{PFAZURE_EXECUTABLE_NAME}' "
+        print_status(f"** WARNING: Other '{PF_EXECUTABLE_NAME}/{PFAZURE_EXECUTABLE_NAME}' "
                      f"executables are on your $PATH. **")
         print_status("Conflicting paths: {}".format(', '.join(conflicting_paths)))
         print_status("You can run this installation of the promptflow with '{}'.".format(exec_filepath))
 
 
-def handle_path_and_tab_completion(exec_filepath, exec_dir):
+def handle_path_and_tab_completion(exec_filepath, install_dir):
     ans_yes = prompt_y_n('Modify profile to update your $PATH now?', 'y')
     if ans_yes:
         rc_file_path = get_rc_file_path()
         if not rc_file_path:
             raise CLIInstallError('No suitable profile file found.')
         _backup_rc(rc_file_path)
-        line_to_add = "export PATH=$PATH:{}".format(exec_dir)
+        line_to_add = "export PATH=$PATH:{}".format(os.path.join(install_dir, "bin"))
         _modify_rc(rc_file_path, line_to_add)
-        warn_other_azs_on_path(exec_dir, exec_filepath)
+        warn_other_azs_on_path(install_dir, exec_filepath)
         print_status()
         print_status('** Run `exec -l $SHELL` to restart your shell. **')
         print_status()
@@ -280,59 +225,16 @@ def verify_python_version():
     print_status('Python version {}.{}.{} okay.'.format(v.major, v.minor, v.micro))
 
 
-def _native_dependencies_for_dist(verify_cmd_args, install_cmd_args, dep_list):
-    try:
-        print_status("Executing: '{} {}'".format(' '.join(verify_cmd_args), ' '.join(dep_list)))
-        subprocess.check_output(verify_cmd_args + dep_list, stderr=subprocess.STDOUT)
-        print_status('Native dependencies okay.')
-    except subprocess.CalledProcessError:
-        err_msg = 'One or more of the following native dependencies are not currently installed and may be required.\n'
-        err_msg += '"{}"'.format(' '.join(install_cmd_args + dep_list))
-        print_status(err_msg)
-        ans_yes = prompt_y_n('Missing native dependencies. Attempt to continue anyway?', 'n')
-        if not ans_yes:
-            raise CLIInstallError('Please install the native dependencies and try again.')
-
-
-def _get_linux_distro():
-    if platform.system() != 'Linux':
-        return None, None
-
-    try:
-        with open('/etc/os-release') as lines:
-            tokens = [line.strip() for line in lines]
-    except Exception:
-        return None, None
-
-    release_info = {}
-    for token in tokens:
-        if '=' in token:
-            k, v = token.split('=', 1)
-            release_info[k.lower()] = v.strip('"')
-
-    return release_info.get('name', None), release_info.get('version_id', None)
-
-
-def verify_install_dir_exec_path_conflict(install_dir, exec_dir):
-    for exec_name in [PF_EXECUTABLE_NAME, PFAZURE_EXECUTABLE_NAME, PFS_EXECUTABLE_NAME]:
-        exec_path = os.path.join(exec_dir, exec_name)
-        if install_dir == exec_path:
-            raise CLIInstallError("The executable file '{}' would clash with the install directory of '{}'. Choose "
-                                  "either a different install directory or directory to place the "
-                                  "executable.".format(exec_path, install_dir))
-
-
 def main():
     verify_python_version()
     tmp_dir = create_tmp_dir()
     install_dir = get_install_dir()
-    exec_dir = get_exec_dir()
-    verify_install_dir_exec_path_conflict(install_dir, exec_dir)
     create_virtualenv(install_dir)
     install_cli(install_dir, tmp_dir)
-    exec_filepath = create_executable(exec_dir, install_dir)
+    exec_filepath = [os.path.join(install_dir, "bin", PF_EXECUTABLE_NAME),
+                     os.path.join(install_dir, "bin", PFAZURE_EXECUTABLE_NAME)]
     try:
-        handle_path_and_tab_completion(exec_filepath, exec_dir)
+        handle_path_and_tab_completion(exec_filepath, install_dir)
     except Exception as e:
         print_status("Unable to set up PATH. ERROR: {}".format(str(e)))
     shutil.rmtree(tmp_dir)
@@ -351,10 +253,10 @@ def main():
         sys.exit(1)
 
 # SIG # Begin signature block
-# Z1F07ShfIJ7kejST2NXwW1QcFPEya4xaO2xZz6vLT847zaMzbc/PaEa1RKFlD881
-# 4J+i6Au2wtbHzOXDisyH6WeLQ3gh0X2gxFRa4EzW7Nzjcvwm4+WogiTcnPVVxlk3
-# qafM/oyVqs3695K7W5XttOiq2guv/yedsf/TW2BKSEKruFQh9IwDfIiBoi9Zv3wa
-# iuzQulRR8KyrCtjEPDV0t4WnZVB/edQea6xJZeTlMG+uLR/miBTbPhUb/VZkVjBf
-# qHBv623oLXICzoTNuaPTln9OWvL2NZpisGYvNzebKO7/Ho6AOWZNs5XOVnjs0Ax2
-# aeXvlwBzIQyfyxd25487/Q==
+# Op0/tmmNDX4QgQefj28K91e/ClVKWeYaA1w1kb5Hi8ALJZtmvyhwvxYlCRZ9eWT+
+# wFBfSvpUIzAWYxEMfVqqWy7g9AzqHGa5vE37zQ7uGkIyR0OsmO0bkauOv5FxuCWX
+# U0u9d9sir6sRTb2nrEj2O1EXAcP2xNaW77w1fcOtMX9W6ytHXx/+v5p387+/HBnQ
+# y00GvJrrcIJeRj4MboBdDdv0VgGJAAlTJp0hxO0lt5ZQUMJvi/sM4e1cPUTcx7uB
+# djCmvSZzZrJO2ZIIWiyiP2XNYWJv4A9klJWMbWKukeZOSjxYPS0pO2mTftkKoL5U
+# sOJu9RtRWIIx/kLG/Axliw==
 # SIG # End signature block

scripts/installer/windows/scripts/generate_dependency.py

@@ -1,11 +1,27 @@
 import ast
+import toml
 import re
 import subprocess
 import copy
 from pathlib import Path
 from promptflow._sdk._utils import render_jinja_template
 
 
+def get_git_base_dir():
+    return Path(
+        subprocess.run(['git', 'rev-parse', '--show-toplevel'], stdout=subprocess.PIPE)
+        .stdout.decode('utf-8').strip())
+
+
+def is_tool(name):
+    """Check whether `name` is on PATH and marked as executable."""
+
+    # from whichcraft import which
+    from shutil import which
+
+    return which(name) is not None
+
+
 def extract_requirements(file_path):
     with open(file_path, 'r') as file:
         tree = ast.parse(file.read())
@@ -34,28 +50,52 @@ def extract_package_names(packages):
 def get_package_dependencies(package_name_list):
     dependencies = []
     for package_name in package_name_list:
-        result = subprocess.run(['pip', 'show', package_name], stdout=subprocess.PIPE)
+        if (is_tool('conda')):
+            result = subprocess.run('conda activate root | pip show {}'.format(package_name),
+                                    shell=True, stdout=subprocess.PIPE)
+        else:
+            result = subprocess.run(['pip', 'show', package_name], stdout=subprocess.PIPE)
+        print("---" + package_name)
+        print(result.stdout)
         lines = result.stdout.decode('utf-8', errors="ignore").splitlines()
         for line in lines:
             if line.startswith('Requires'):
                 dependency = line.split(': ')[1].split(', ')
                 if dependency != ['']:
                     dependencies.extend(dependency)
                 break
+
+    dependencies = [dependency for dependency in dependencies if not dependency.startswith('promptflow')]
     return dependencies
 
 
 if __name__ == '__main__':
     dependencies = []
-    install_requires, extras_requires = extract_requirements('../../../../src/promptflow/setup.py')
+    install_requires, extras_requires = extract_requirements(get_git_base_dir() / 'src/promptflow/setup.py')
     install_requires_names = extract_package_names(install_requires)
     dependencies.extend(install_requires_names)
 
     for key in extras_requires:
         extras_require_names = extract_package_names(extras_requires[key])
         dependencies.extend(extras_require_names)
+    dependencies = list(set(dependencies))
     direct_package_dependencies = get_package_dependencies(dependencies)
     all_packages = list(set(dependencies) | set(direct_package_dependencies))
+
+    # get promptflow-** extra packages
+    extra_packages = []
+    for package in all_packages:
+        if package.startswith('promptflow'):
+            data = toml.load(get_git_base_dir() / "src" / package / "pyproject.toml")
+            extras = data.get("tool", {}).get("poetry", {}).get("extras", {})
+            for _, values in extras.items():
+                extra_packages.extend(values)
+    extra_packages_names = extract_package_names(extra_packages)
+    all_packages.extend(extra_packages_names)
+
+    # remove all packages starting with promptflow
+    all_packages = [package for package in all_packages if not package.startswith('promptflow')]
+
     hidden_imports = copy.deepcopy(all_packages)
     meta_packages = copy.deepcopy(all_packages)
 
@@ -79,6 +119,7 @@ def get_package_dependencies(package_name_list):
     }
     # always use unix line ending
     Path("./promptflow.spec").write_bytes(
-        render_jinja_template("./promptflow.spec.jinja2", **render_context)
+        render_jinja_template(
+            get_git_base_dir() / "scripts/installer/windows/scripts/promptflow.spec.jinja2", **render_context)
         .encode("utf-8")
         .replace(b"\r\n", b"\n"),)