From 2f9595af74464822f6b8d5c23153931c9d7a2871 Mon Sep 17 00:00:00 2001 From: Wenbing Li <10278425+wenbingl@users.noreply.github.com> Date: Mon, 17 Jun 2024 16:58:11 -0700 Subject: [PATCH] Fix the pipeline breaks dues to the MSVC 19.40 and numpy 2.0 release (#747) * dd "-allow-unsupported-compiler" flags to Windows CUDA flags inspired by this PR: https://github.com/microsoft/onnxruntime/pull/21004 * switch to cmake command line * handle the issues caused by the latest MSVC release * correct the typo * correct the parameter * try one dash again * use the installed cmake * use cmake standalone installation firstly * use the standalone cmake in win32 python too * fix it more * one more try * fix the MacOS pipeline issue * fix the pip command line --- .pipelines/ci.yml | 11 +++++++---- .pyproject/cmdclass.py | 26 ++++++++++++++++---------- CMakeLists.txt | 5 +++++ build.bat | 11 ++++++++++- cmake/externals/opencv-no-rtti.patch | 13 +++++++++++++ requirements-dev.txt | 3 ++- 6 files changed, 53 insertions(+), 16 deletions(-) diff --git a/.pipelines/ci.yml b/.pipelines/ci.yml index e5290137c..8ad782daa 100644 --- a/.pipelines/ci.yml +++ b/.pipelines/ci.yml @@ -126,7 +126,7 @@ stages: - script: | python -m pip install --upgrade setuptools pip - python -m pip install numpy + python -m pip install 'numpy < 2.0.0' export OCOS_NO_OPENCV=1 export OCOS_SCB_DEBUG=1 CPU_NUMBER=8 python -m pip install -e . @@ -322,6 +322,7 @@ stages: python -m pip install --upgrade pip python -m pip install --upgrade setuptools python -m pip install --upgrade wheel + python -m pip install 'numpy < 2.0.0' python -m pip install onnxruntime==$(ort.version) displayName: Install requirements @@ -513,7 +514,7 @@ stages: - script: | python -m pip install --upgrade setuptools pip - python -m pip install numpy + python -m pip install "numpy < 2.0.0" set OCOS_NO_OPENCV=1 set OCOS_SCB_DEBUG=1 python -m pip install -v -e . @@ -570,7 +571,9 @@ stages: - script: | set CUDA_PATH=$(Agent.TempDirectory)\v11.8 - call .\build.bat -T cuda="%CUDA_PATH%" -DOCOS_ENABLE_CTEST=ON -DOCOS_USE_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=70;86^ + call .\build.bat -T cuda="%CUDA_PATH%" -DOCOS_ENABLE_CTEST=ON^ + -DCMAKE_CUDA_FLAGS_INIT=-allow-unsupported-compiler^ + -DOCOS_USE_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=70;86^ -DOCOS_ONNXRUNTIME_VERSION="$(ORT_VERSION)" -DONNXRUNTIME_PKG_DIR=.\onnxruntime-win-x64-gpu-$(ORT_VERSION) displayName: build the customop library with onnxruntime @@ -590,7 +593,7 @@ stages: - script: | set CUDA_PATH=$(Agent.TempDirectory)\v11.8 python -m pip install --upgrade setuptools pip - python -m pip install numpy coloredlogs flatbuffers packaging protobuf sympy + python -m pip install "numpy < 2.0.0" coloredlogs flatbuffers packaging protobuf sympy python -m pip install onnxruntime-gpu==$(ORT_VERSION) python -m pip install -v --config-settings "ortx-user-option=use-cuda,cuda_archs=70;86" . displayName: Build and install onnxruntime-extensions CUDA package. diff --git a/.pyproject/cmdclass.py b/.pyproject/cmdclass.py index 5b38b02c0..f56ad8c1b 100644 --- a/.pyproject/cmdclass.py +++ b/.pyproject/cmdclass.py @@ -226,6 +226,8 @@ def build_cmake(self, extension): if sys.platform == "win32": cuda_path = os.environ.get("CUDA_PATH") cmake_args += [f'-T cuda={cuda_path}'] + # TODO: temporarily add a flag for MSVC 19.40 + cmake_args += ['-DCMAKE_CUDA_FLAGS_INIT=-allow-unsupported-compiler'] f_ver = ext_fullpath.parent / "_version.py" with f_ver.open('a') as _f: _f.writelines(["\n", f"cuda = \"{cuda_ver}\"", "\n"]) @@ -235,7 +237,8 @@ def build_cmake(self, extension): else: smi = _load_nvidia_smi() if not smi: - raise RuntimeError(f"Cannot detect the CUDA archs from your machine, please specify it by yourself.") + raise RuntimeError( + "Cannot detect the CUDA archs from your machine, please specify it manually.") cmake_args += ['-DCMAKE_CUDA_ARCHITECTURES=' + smi] # CMake lets you override the generator - we need to check this. @@ -274,7 +277,6 @@ def build_cmake(self, extension): cmake_args += [ "-DCMAKE_OSX_ARCHITECTURES={}".format(";".join(archs))] - # overwrite the Python module info if the auto-detection doesn't work. # export Python3_INCLUDE_DIRS=/opt/python/cp38-cp38 # export Python3_LIBRARIES=/opt/python/cp38-cp38 @@ -292,14 +294,18 @@ def build_cmake(self, extension): '--parallel' + ('' if cpu_number is None else ' ' + cpu_number) ] cmake_exe = 'cmake' - # unlike Linux/macOS, cmake pip package on Windows fails to build some 3rd party dependencies. - # so we have to use the cmake installed from Visual Studio. - if os.environ.get(VSINSTALLDIR_NAME): - cmake_exe = os.environ[VSINSTALLDIR_NAME] + \ - 'Common7\\IDE\\CommonExtensions\\Microsoft\\CMake\\CMake\\bin\\cmake.exe' - # Add this cmake directory into PATH to make sure the child-process still find it. - os.environ['PATH'] = os.path.dirname( - cmake_exe) + os.pathsep + os.environ['PATH'] + # if sys.platform == "win32": + # # unlike Linux/macOS, cmake pip package on Windows fails to build some 3rd party dependencies. + # # so we have to use the cmake from a standalone installation or the one from Visual Studio. + # standalone_cmake = os.path.join(os.environ.get("ProgramFiles"), "\\CMake\\bin\\cmake.exe") + # if os.path.exists(standalone_cmake): + # cmake_exe = standalone_cmake + # elif os.environ.get(VSINSTALLDIR_NAME): + # cmake_exe = os.environ[VSINSTALLDIR_NAME] + \ + # 'Common7\\IDE\\CommonExtensions\\Microsoft\\CMake\\CMake\\bin\\cmake.exe' + # # Add this cmake directory into PATH to make sure the child-process still find it. + # os.environ['PATH'] = os.path.dirname( + # cmake_exe) + os.pathsep + os.environ['PATH'] self.spawn([cmake_exe, '-S', str(project_dir), '-B', str(build_temp)] + cmake_args) diff --git a/CMakeLists.txt b/CMakeLists.txt index c039f9183..3b9cb0ead 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,6 +36,11 @@ if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0") cmake_policy(SET CMP0077 NEW) endif() +# Avoid warning of Calling FetchContent_Populate(GSL) is deprecated +if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.30.0") + cmake_policy(CMP0169 OLD) +endif() + # Needed for Java set(CMAKE_C_STANDARD 99) diff --git a/build.bat b/build.bat index bdd39a1d7..1a5367a7a 100644 --- a/build.bat +++ b/build.bat @@ -1,5 +1,10 @@ @ECHO OFF SETLOCAL ENABLEDELAYEDEXPANSION + +IF NOT EXIST "%ProgramFiles%\CMake\bin\cmake.exe" GOTO :FIND_VS +set cmake_exe="%ProgramFiles%\CMake\bin\cmake.exe" + +:FIND_VS IF DEFINED VSINSTALLDIR GOTO :VSDEV_CMD set _VSFINDER=%~dp0tools\get_vsdevcmd.ps1 for /f "tokens=* USEBACKQ" %%i in ( @@ -7,6 +12,10 @@ for /f "tokens=* USEBACKQ" %%i in ( IF NOT DEFINED VSINSTALLDIR GOTO :NOT_FOUND +IF DEFINED cmake_exe GOTO :CMAKE_DEF +set cmake_exe="%VSINSTALLDIR%Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin\cmake.exe" + +:CMAKE_DEF IF "%1" == "-A" GOTO :VSDEV_CMD set GEN_PLATFORM=-A x64 @@ -16,8 +25,8 @@ IF "%VisualStudioVersion:~0,2%" == "16" GOTO :START_BUILD set GENERATOR="Visual Studio 17 2022" :START_BUILD -set cmake_exe="%VSINSTALLDIR%Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin\cmake.exe" mkdir .\out\Windows\ 2>NUL +ECHO %cmake_exe% -G %GENERATOR% %GEN_PLATFORM% %* -B out\Windows -S . %cmake_exe% -G %GENERATOR% %GEN_PLATFORM% %* -B out\Windows -S . IF %ERRORLEVEL% NEQ 0 EXIT /B %ERRORLEVEL% %cmake_exe% --build out\Windows --config RelWithDebInfo diff --git a/cmake/externals/opencv-no-rtti.patch b/cmake/externals/opencv-no-rtti.patch index 487f9296e..c94f89d5d 100644 --- a/cmake/externals/opencv-no-rtti.patch +++ b/cmake/externals/opencv-no-rtti.patch @@ -39,6 +39,19 @@ index d95e5db163..db185453df 100644 include(cmake/OpenCVCompilerOptions.cmake) ocv_cmake_hook(POST_COMPILER_OPTIONS) +diff --git a/cmake/OpenCVDetectCXXCompiler.cmake b/cmake/OpenCVDetectCXXCompiler.cmake +index 7f229cde96..92e204a5b9 100644 +--- a/cmake/OpenCVDetectCXXCompiler.cmake ++++ b/cmake/OpenCVDetectCXXCompiler.cmake +@@ -171,7 +171,7 @@ elseif(MSVC) + set(OpenCV_RUNTIME vc15) + elseif(MSVC_VERSION MATCHES "^192[0-9]$") + set(OpenCV_RUNTIME vc16) +- elseif(MSVC_VERSION MATCHES "^193[0-9]$") ++ elseif(MSVC_VERSION MATCHES "^19[34][0-9]$") + set(OpenCV_RUNTIME vc17) + else() + message(WARNING "OpenCV does not recognize MSVC_VERSION \"${MSVC_VERSION}\". Cannot set OpenCV_RUNTIME") diff --git a/modules/core/include/opencv2/core/ocl.hpp b/modules/core/include/opencv2/core/ocl.hpp index 4503fa00dd..642b0508d0 100644 --- a/modules/core/include/opencv2/core/ocl.hpp diff --git a/requirements-dev.txt b/requirements-dev.txt index 47114f13f..6d5867be1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,6 @@ pytest -onnx >= 1.9.0 +numpy < 2.0.0 +onnx >=1.9.0 protobuf < 4.0.0 # multiple versions of onnxruntime are supported, but only one can be installed at a time onnxruntime >=1.12.0