diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index 2c83b94da..72b7e7dea 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -82,9 +82,6 @@ runs: - name: Install wheel package shell: bash run: pip install --quiet wheel - - name: Create wheel from setup.py - shell: bash - run: python setup.py --quiet bdist_wheel - name: Submit CPU tests to AzureML shell: bash if: contains(inputs.TEST_GROUP, 'cpu') @@ -94,7 +91,7 @@ runs: --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - --disable-warnings + --disable-warnings --sha "${GITHUB_SHA}" - name: Submit GPU tests to AzureML shell: bash if: contains(inputs.TEST_GROUP, 'gpu') @@ -104,7 +101,7 @@ runs: --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - --disable-warnings + --disable-warnings --sha "${GITHUB_SHA}" - name: Submit PySpark tests to AzureML shell: bash if: contains(inputs.TEST_GROUP, 'spark') @@ -114,7 +111,7 @@ runs: --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - --disable-warnings + --disable-warnings --sha "${GITHUB_SHA}" - name: Print test logs shell: bash run: cat ${{inputs.TEST_LOGS_PATH}} diff --git a/setup.py b/setup.py index 080dad2cb..dbf919882 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,8 @@ "jinja2>=2,<3.1", "pyyaml>=5.4.1,<6", "requests>=2.0.0,<3", - "cornac>=1.1.2,<2", + "cornac>=1.1.2,<1.15.2;python_version<='3.7'", + "cornac>=1.15.2,<2;python_version>='3.8'", # After 1.15.2, Cornac requires python 3.8 "retrying>=1.3.3", "pandera[strategies]>=0.6.5", # For generating fake datasets "scikit-surprise>=1.0.6", @@ -137,6 +138,9 @@ install_requires=install_requires, package_dir={"recommenders": "recommenders"}, python_requires=">=3.6, <3.10", - packages=find_packages(where=".", exclude=["contrib", "docs", "examples", "scenarios", "tests", "tools"]), - setup_requires=["numpy>=1.15"] + packages=find_packages( + where=".", + exclude=["contrib", "docs", "examples", "scenarios", "tests", "tools"], + ), + setup_requires=["numpy>=1.15"], ) diff --git a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py index b3a57f1ee..190089054 100644 --- a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py +++ b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py @@ -152,7 +152,7 @@ def create_run_config( add_spark_dependencies, conda_pkg_jdk, conda_pkg_python, - reco_wheel_path, + commit_sha, ): """ AzureML requires the run environment to be setup prior to submission. @@ -172,6 +172,7 @@ def create_run_config( added to the conda environment, else False add_spark_dependencies (bool) : True if PySpark packages should be added to the conda environment, else False + commit_sha (str) : the commit that triggers the workflow Return: run_azuremlcompute : AzureML run config @@ -188,32 +189,28 @@ def create_run_config( # True means the user will manually configure the environment run_azuremlcompute.environment.python.user_managed_dependencies = False - # install local version of recommenders on AzureML compute using .whl file - whl_url = run_azuremlcompute.environment.add_private_pip_wheel( - workspace=workspace, - file_path=reco_wheel_path, - exist_ok=True, - ) conda_dep = CondaDependencies() conda_dep.add_conda_package(conda_pkg_python) - conda_dep.add_pip_package(whl_url) conda_dep.add_pip_package( "pymanopt@https://github.com/pymanopt/pymanopt/archive/fb36a272cdeecb21992cfd9271eb82baafeb316d.zip" ) - # install extra dependencies + # install recommenders + reco_extras = "dev,examples" if add_gpu_dependencies and add_spark_dependencies: conda_dep.add_channel("conda-forge") conda_dep.add_conda_package(conda_pkg_jdk) - conda_dep.add_pip_package("recommenders[dev,examples,spark,gpu]") + reco_extras = reco_extras + ",spark,gpu" elif add_gpu_dependencies: - conda_dep.add_pip_package("recommenders[dev,examples,gpu]") + reco_extras = reco_extras + ",gpu" elif add_spark_dependencies: conda_dep.add_channel("conda-forge") conda_dep.add_conda_package(conda_pkg_jdk) - conda_dep.add_pip_package("recommenders[dev,examples,spark]") - else: - conda_dep.add_pip_package("recommenders[dev,examples]") + reco_extras = reco_extras + ",spark" + + conda_dep.add_pip_package( + f"recommenders[{reco_extras}]@git+https://github.com/microsoft/recommenders.git@{commit_sha}" + ) run_azuremlcompute.environment.python.conda_dependencies = conda_dep return run_azuremlcompute @@ -286,6 +283,11 @@ def create_arg_parser(): """ parser = argparse.ArgumentParser(description="Process some inputs") + parser.add_argument( + "--sha", + action="store", + help="the commit that triggers the workflow", + ) # script to run pytest parser.add_argument( "--test", @@ -448,11 +450,6 @@ def create_arg_parser(): max_nodes=args.maxnodes, ) - wheel_list = glob.glob("./dist/*.whl") - if not wheel_list: - logger.error("Wheel not found!") - logger.info("Found wheel at " + wheel_list[0]) - run_config = create_run_config( cpu_cluster=cpu_cluster, docker_proc_type=docker_proc_type, @@ -461,7 +458,7 @@ def create_arg_parser(): add_spark_dependencies=args.add_spark_dependencies, conda_pkg_jdk=args.conda_pkg_jdk, conda_pkg_python=args.conda_pkg_python, - reco_wheel_path=wheel_list[0], + commit_sha=args.sha, ) logger.info("exp: In Azure, look for experiment named {}".format(args.expname))