Merge pull request #27753 from jaimergp/optimize-install-times

MNT: Use micromamba to provision base environment on macOS and Windows

Author: beckermr

.azure-pipelines/azure-pipelines-osx.yml

@@ -16,9 +16,12 @@ jobs:
 
   steps:
   - script: |
-      export CI=azure
       ./.scripts/run_osx_build.sh
     displayName: Run OSX build
+    env:
+      CI: azure
+      CONDA_BLD_PATH: /Users/runner/Miniforge3/conda-bld
+      MINIFORGE_HOME: /Users/runner/Miniforge3
 
   - publish: /Users/runner/Miniforge3/conda-bld/osx-64/
     artifact: conda_pkgs_osx

.azure-pipelines/azure-pipelines-win.yml

@@ -10,27 +10,13 @@ jobs:
         CONFIG: win64
   timeoutInMinutes: 360
   steps:
-    - task: PythonScript@0
-      displayName: 'Download Miniforge'
-      inputs:
-        scriptSource: inline
-        script: |
-          import urllib.request
-          url = 'https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Windows-x86_64.exe'
-          path = r"$(Build.ArtifactStagingDirectory)/Miniforge.exe"
-          urllib.request.urlretrieve(url, path)
-    - script: |
-        start /wait "" %BUILD_ARTIFACTSTAGINGDIRECTORY%\Miniforge.exe /InstallationType=JustMe /RegisterPython=0 /S /D=D:\Miniforge
-      displayName: Install Miniforge
-    - powershell: Write-Host "##vso[task.prependpath]D:\Miniforge\Scripts"
-      displayName: Add conda to PATH
-
     - script: |
         call .scripts\run_win_build.bat
       displayName: Build recipes
       env:
         CI: azure
         CONDA_BLD_PATH: D:\bld
+        MINIFORGE_HOME: D:\Miniforge
 
     - publish: D:\\bld\\win-64\\
       artifact: conda_pkgs_win

.scripts/run_osx_build.sh

@@ -4,21 +4,32 @@ set -x
 
 source .scripts/logging_utils.sh
 
-( startgroup "Ensuring Miniforge" ) 2> /dev/null
+( startgroup "Provisioning base env with micromamba" ) 2> /dev/null
 
-MINIFORGE_URL="https://github.com/conda-forge/miniforge/releases/latest/download"
-MINIFORGE_FILE="Miniforge3-MacOSX-x86_64.sh"
-MINIFORGE_ROOT="${MINIFORGE_ROOT:-${HOME}/Miniforge3}"
+MINIFORGE_HOME=${MINIFORGE_HOME:-${HOME}/miniforge3}
+MINIFORGE_HOME=${MINIFORGE_HOME%/} # remove trailing slash
 
-if [[ -d "${MINIFORGE_ROOT}" ]]; then
-  echo "Miniforge already installed at ${MINIFORGE_ROOT}."
+if [[ -d "${MINIFORGE_HOME}" ]]; then
+  echo "Miniforge already installed at ${MINIFORGE_HOME}."
 else
-  echo "Installing Miniforge"
-  curl -L -O "${MINIFORGE_URL}/${MINIFORGE_FILE}"
-  bash $MINIFORGE_FILE -bp "${MINIFORGE_ROOT}"
+  MICROMAMBA_VERSION="1.5.10-0"
+  if [[ "$(uname -m)" == "arm64" ]]; then
+    osx_arch="osx-arm64"
+  else
+    osx_arch="osx-64"
+  fi
+  MICROMAMBA_URL="https://github.com/mamba-org/micromamba-releases/releases/download/${MICROMAMBA_VERSION}/micromamba-${osx_arch}"
+  echo "Downloading micromamba ${MICROMAMBA_VERSION}"
+  micromamba_exe="$(mktemp -d)/micromamba"
+  curl -L -o "${micromamba_exe}" "${MICROMAMBA_URL}"
+  chmod +x "${micromamba_exe}"
+  echo "Creating environment"
+  "${micromamba_exe}" create --yes --root-prefix ~/.conda --prefix "${MINIFORGE_HOME}" \
+    --channel conda-forge \
+    --file .ci_support/requirements.txt
 fi
 
-( endgroup "Ensuring Miniforge" ) 2> /dev/null
+( endgroup "Provisioning base env with micromamba" ) 2> /dev/null
 
 ( startgroup "Configuring conda" ) 2> /dev/null
 
@@ -28,12 +39,9 @@ show_channel_urls: true
 solver: libmamba
 CONDARC
 
-source "${MINIFORGE_ROOT}/etc/profile.d/conda.sh"
+source "${MINIFORGE_HOME}/etc/profile.d/conda.sh"
 conda activate base
 
-echo -e "\n\nInstalling conda-forge-ci-setup, conda-build."
-conda install --quiet --file .ci_support/requirements.txt
-
 echo -e "\n\nSetting up the condarc and mangling the compiler."
 setup_conda_rc ./ ./recipes ./.ci_support/${CONFIG}.yaml
 if [[ "${CI:-}" != "" ]]; then
@@ -54,7 +62,7 @@ source run_conda_forge_build_setup
 set -e
 
 # make sure there is a package directory so that artifact publishing works
-mkdir -p "${MINIFORGE_ROOT}/conda-bld/osx-64/" "${MINIFORGE_ROOT}/conda-bld/noarch/"
+mkdir -p "${MINIFORGE_HOME}/conda-bld/osx-64/" "${MINIFORGE_HOME}/conda-bld/noarch/"
 
 # Find the recipes from main in this PR and remove them.
 echo ""

.scripts/run_win_build.bat

@@ -3,46 +3,70 @@
 :: changes to this script, consider a proposal to conda-smithy so that other feedstocks can also
 :: benefit from the improvement.
 
-:: Note: we assume a Miniforge installation is available
-
 :: INPUTS (required environment variables)
 :: CONDA_BLD_PATH: path for the conda-build workspace
 :: CI: azure, or unset
+:: MINIFORGE_HOME: where to install the base conda environment
 
 setlocal enableextensions enabledelayedexpansion
 
+if "%MINIFORGE_HOME%"=="" set "MINIFORGE_HOME=%USERPROFILE%\Miniforge3"
+:: Remove trailing backslash, if present
+if "%MINIFORGE_HOME:~-1%"=="\" set "MINIFORGE_HOME=%MINIFORGE_HOME:~0,-1%"
+call :start_group "Provisioning base env with micromamba"
+set "MAMBA_ROOT_PREFIX=%MINIFORGE_HOME%-micromamba-%RANDOM%"
+set "MICROMAMBA_VERSION=1.5.10-0"
+set "MICROMAMBA_URL=https://github.com/mamba-org/micromamba-releases/releases/download/%MICROMAMBA_VERSION%/micromamba-win-64"
+set "MICROMAMBA_TMPDIR=%TMP%\micromamba-%RANDOM%"
+set "MICROMAMBA_EXE=%MICROMAMBA_TMPDIR%\micromamba.exe"
+
+echo Downloading micromamba %MICROMAMBA_VERSION%
+if not exist "%MICROMAMBA_TMPDIR%" mkdir "%MICROMAMBA_TMPDIR%"
+certutil -urlcache -split -f "%MICROMAMBA_URL%" "%MICROMAMBA_EXE%"
+if !errorlevel! neq 0 exit /b !errorlevel!
+
+echo Creating environment
+call "%MICROMAMBA_EXE%" create --yes --root-prefix "%MAMBA_ROOT_PREFIX%" --prefix "%MINIFORGE_HOME%" ^
+    --channel conda-forge ^
+    --file .ci_support\requirements.txt
+if !errorlevel! neq 0 exit /b !errorlevel!
+echo Moving pkgs cache from %MAMBA_ROOT_PREFIX% to %MINIFORGE_HOME%
+move /Y "%MAMBA_ROOT_PREFIX%\pkgs" "%MINIFORGE_HOME%"
+if !errorlevel! neq 0 exit /b !errorlevel!
+echo Removing %MAMBA_ROOT_PREFIX%
+del /S /Q "%MAMBA_ROOT_PREFIX%"
+del /S /Q "%MICROMAMBA_TMPDIR%"
+call :end_group
+
 call :start_group "Configuring conda"
 
 if "%CONDA_BLD_PATH%" == "" (
     set "CONDA_BLD_PATH=C:\bld"
 )
 
 :: Activate the base conda environment
-call activate base
+echo Activating "%MINIFORGE_HOME%"
+call "%MINIFORGE_HOME%\Scripts\activate"
 
+:: Set basic configuration
+echo Setting up configuration
 conda.exe config --set always_yes yes
-if errorlevel 1 exit 1
+if !errorlevel! neq 0 exit /b !errorlevel!
 conda.exe config --set channel_priority strict
-if errorlevel 1 exit 1
+if !errorlevel! neq 0 exit /b !errorlevel!
 conda.exe config --set solver libmamba
-if errorlevel 1 exit 1
-
-echo Installing dependencies
-conda.exe install --file .\.ci_support\requirements.txt
-if errorlevel 1 exit 1
+if !errorlevel! neq 0 exit /b !errorlevel!
 
-:: Set basic configuration
-echo Setting up configuration
 setup_conda_rc .\ ".\recipes" .\.ci_support\%CONFIG%.yaml
-if errorlevel 1 exit 1
+if !errorlevel! neq 0 exit /b !errorlevel!
 
 echo Run conda_forge_build_setup
 call run_conda_forge_build_setup
-if errorlevel 1 exit 1
+if !errorlevel! neq 0 exit /b !errorlevel!
 
 echo Force fetch origin/main
 git fetch --force origin main:main
-if errorlevel 1 exit 1
+if !errorlevel! neq 0 exit /b !errorlevel!
 echo Removing recipes also present in main
 cd recipes
 for /f "tokens=*" %%a in ('git ls-tree --name-only main -- .') do rmdir /s /q %%a && echo Removing recipe: %%a
@@ -54,13 +78,13 @@ if not exist "%CONDA_BLD_PATH%\noarch\" mkdir "%CONDA_BLD_PATH%\noarch\"
 
 echo Index %CONDA_BLD_PATH%
 conda.exe index "%CONDA_BLD_PATH%"
-if errorlevel 1 exit 1
+if !errorlevel! neq 0 exit /b !errorlevel!
 
 call :end_group
 
 echo Building all recipes
 python .ci_support\build_all.py --arch 64
-if errorlevel 1 exit 1
+if !errorlevel! neq 0 exit /b !errorlevel!
 
 call :start_group "Inspecting artifacts"