From 71388131b0cb847b0bd39e4a236bbbbe106d8212 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 9 Nov 2023 15:05:12 -0800 Subject: [PATCH 1/4] ADIOS2 v2.9+ Uses Blosc2 Update the compressor to use C-Blosc2. --- recipe/build-libadios.bat | 2 +- recipe/build-libadios.sh | 1 + recipe/meta.yaml | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/recipe/build-libadios.bat b/recipe/build-libadios.bat index bd5f0ad..0e6d261 100644 --- a/recipe/build-libadios.bat +++ b/recipe/build-libadios.bat @@ -21,7 +21,7 @@ cmake ^ -DBUILD_TESTING=OFF ^ -DADIOS2_USE_MPI=OFF ^ -DADIOS2_BUILD_EXAMPLES=OFF ^ - -DADIOS2_USE_Blosc=ON ^ + -DADIOS2_USE_Blosc2=ON ^ -DADIOS2_USE_BZip2=ON ^ -DADIOS2_USE_Fortran=OFF ^ -DADIOS2_USE_HDF5=ON ^ diff --git a/recipe/build-libadios.sh b/recipe/build-libadios.sh index 774f5fd..dcd7952 100644 --- a/recipe/build-libadios.sh +++ b/recipe/build-libadios.sh @@ -54,6 +54,7 @@ cmake \ -B build \ -GNinja \ -DADIOS2_BUILD_EXAMPLES=OFF \ + -DADIOS2_USE_Blosc2=ON \ -DADIOS2_HAVE_ZFP_CUDA=OFF \ -DADIOS2_INSTALL_GENERATE_CONFIG=OFF \ -DADIOS2_RUN_INSTALL_TEST=ON \ diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 6d599fc..faaa410 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -1,7 +1,7 @@ {% set name = "adios2" %} {% set version = "2.9.2" %} {% set sha256 = "78309297c82a95ee38ed3224c98b93d330128c753a43893f63bbe969320e4979" %} -{% set build = 0 %} +{% set build = 1 %} # ensure mpi is defined (needed for conda-smithy recipe-lint) {% set mpi = mpi or 'nompi' %} @@ -89,7 +89,7 @@ outputs: # TODO: Could NOT find ZeroMQ (missing: ZeroMQ_LIBRARY) on Windows - zeromq >=4.1 # [not win] - bzip2 - - blosc + - c-blosc2 - libpng - zfp 0.5 # CMake rejects zfp 1.0.0 - zlib From 40e4fd891458903cf96c8c7fa7d12eed16ffde65 Mon Sep 17 00:00:00 2001 From: "conda-forge-webservices[bot]" <91080706+conda-forge-webservices[bot]@users.noreply.github.com> Date: Thu, 9 Nov 2023 23:10:00 +0000 Subject: [PATCH 2/4] MNT: Re-rendered with conda-build 3.27.0, conda-smithy 3.28.0, and conda-forge-pinning 2023.11.09.20.47.34 --- .azure-pipelines/azure-pipelines-linux.yml | 3 +++ .azure-pipelines/azure-pipelines-osx.yml | 3 +++ .azure-pipelines/azure-pipelines-win.yml | 3 +++ .scripts/build_steps.sh | 17 +++++++++++++---- .scripts/run_docker_build.sh | 3 +++ .scripts/run_osx_build.sh | 15 +++++++++++---- .scripts/run_win_build.bat | 12 ++++++++++-- .travis.yml | 3 +++ 8 files changed, 49 insertions(+), 10 deletions(-) diff --git a/.azure-pipelines/azure-pipelines-linux.yml b/.azure-pipelines/azure-pipelines-linux.yml index 914a164..7820604 100755 --- a/.azure-pipelines/azure-pipelines-linux.yml +++ b/.azure-pipelines/azure-pipelines-linux.yml @@ -81,6 +81,9 @@ jobs: - script: | export CI=azure + export flow_run_id=azure_$(Build.BuildNumber).$(System.JobAttempt) + export remote_url=$(Build.Repository.Uri) + export sha=$(Build.SourceVersion) export GIT_BRANCH=$BUILD_SOURCEBRANCHNAME export FEEDSTOCK_NAME=$(basename ${BUILD_REPOSITORY_NAME}) if [[ "${BUILD_REASON:-}" == "PullRequest" ]]; then diff --git a/.azure-pipelines/azure-pipelines-osx.yml b/.azure-pipelines/azure-pipelines-osx.yml index 26e9b73..f17edb0 100755 --- a/.azure-pipelines/azure-pipelines-osx.yml +++ b/.azure-pipelines/azure-pipelines-osx.yml @@ -104,6 +104,9 @@ jobs: # TODO: Fast finish on azure pipelines? - script: | export CI=azure + export flow_run_id=azure_$(Build.BuildNumber).$(System.JobAttempt) + export remote_url=$(Build.Repository.Uri) + export sha=$(Build.SourceVersion) export OSX_FORCE_SDK_DOWNLOAD="1" export GIT_BRANCH=$BUILD_SOURCEBRANCHNAME export FEEDSTOCK_NAME=$(basename ${BUILD_REPOSITORY_NAME}) diff --git a/.azure-pipelines/azure-pipelines-win.yml b/.azure-pipelines/azure-pipelines-win.yml index 634ca19..4fac2f4 100755 --- a/.azure-pipelines/azure-pipelines-win.yml +++ b/.azure-pipelines/azure-pipelines-win.yml @@ -54,6 +54,9 @@ jobs: PYTHONUNBUFFERED: 1 CONFIG: $(CONFIG) CI: azure + flow_run_id: azure_$(Build.BuildNumber).$(System.JobAttempt) + remote_url: $(Build.Repository.Uri) + sha: $(Build.SourceVersion) UPLOAD_PACKAGES: $(UPLOAD_PACKAGES) UPLOAD_TEMP: $(UPLOAD_TEMP) BINSTAR_TOKEN: $(BINSTAR_TOKEN) diff --git a/.scripts/build_steps.sh b/.scripts/build_steps.sh index bd9e671..064e539 100755 --- a/.scripts/build_steps.sh +++ b/.scripts/build_steps.sh @@ -28,13 +28,15 @@ conda-build: pkgs_dirs: - ${FEEDSTOCK_ROOT}/build_artifacts/pkg_cache - /opt/conda/pkgs +solver: libmamba CONDARC +export CONDA_LIBMAMBA_SOLVER_NO_CHANNELS_FROM_INSTALLED=1 mamba install --update-specs --yes --quiet --channel conda-forge --strict-channel-priority \ - pip mamba conda-build boa conda-forge-ci-setup=3 + pip mamba conda-build conda-forge-ci-setup=4 mamba update --update-specs --yes --quiet --channel conda-forge --strict-channel-priority \ - pip mamba conda-build boa conda-forge-ci-setup=3 + pip mamba conda-build conda-forge-ci-setup=4 # set up the condarc setup_conda_rc "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" @@ -55,6 +57,12 @@ if [[ -f "${FEEDSTOCK_ROOT}/LICENSE.txt" ]]; then cp "${FEEDSTOCK_ROOT}/LICENSE.txt" "${RECIPE_ROOT}/recipe-scripts-license.txt" fi +if [[ "${sha:-}" == "" ]]; then + pushd ${FEEDSTOCK_ROOT} + sha=$(git rev-parse HEAD) + popd +fi + if [[ "${BUILD_WITH_CONDA_DEBUG:-0}" == 1 ]]; then if [[ "x${BUILD_OUTPUT_ID:-}" != "x" ]]; then EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --output-id ${BUILD_OUTPUT_ID}" @@ -66,9 +74,10 @@ if [[ "${BUILD_WITH_CONDA_DEBUG:-0}" == 1 ]]; then # Drop into an interactive shell /bin/bash else - conda mambabuild "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \ + conda build "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \ --suppress-variables ${EXTRA_CB_OPTIONS:-} \ - --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" + --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" \ + --extra-meta flow_run_id="${flow_run_id:-}" remote_url="${remote_url:-}" sha="${sha:-}" ( startgroup "Validating outputs" ) 2> /dev/null validate_recipe_outputs "${FEEDSTOCK_NAME}" diff --git a/.scripts/run_docker_build.sh b/.scripts/run_docker_build.sh index 9236239..b70ef01 100755 --- a/.scripts/run_docker_build.sh +++ b/.scripts/run_docker_build.sh @@ -91,6 +91,9 @@ docker run ${DOCKER_RUN_ARGS} \ -e CPU_COUNT \ -e BUILD_WITH_CONDA_DEBUG \ -e BUILD_OUTPUT_ID \ + -e flow_run_id \ + -e remote_url \ + -e sha \ -e BINSTAR_TOKEN \ -e FEEDSTOCK_TOKEN \ -e STAGING_BINSTAR_TOKEN \ diff --git a/.scripts/run_osx_build.sh b/.scripts/run_osx_build.sh index 870c49a..46ab346 100755 --- a/.scripts/run_osx_build.sh +++ b/.scripts/run_osx_build.sh @@ -22,11 +22,13 @@ bash $MINIFORGE_FILE -b -p ${MINIFORGE_HOME} source ${MINIFORGE_HOME}/etc/profile.d/conda.sh conda activate base +export CONDA_SOLVER="libmamba" +export CONDA_LIBMAMBA_SOLVER_NO_CHANNELS_FROM_INSTALLED=1 mamba install --update-specs --quiet --yes --channel conda-forge --strict-channel-priority \ - pip mamba conda-build boa conda-forge-ci-setup=3 + pip mamba conda-build conda-forge-ci-setup=4 mamba update --update-specs --yes --quiet --channel conda-forge --strict-channel-priority \ - pip mamba conda-build boa conda-forge-ci-setup=3 + pip mamba conda-build conda-forge-ci-setup=4 @@ -45,6 +47,10 @@ else echo -e "\n\nNot mangling homebrew as we are not running in CI" fi +if [[ "${sha:-}" == "" ]]; then + sha=$(git rev-parse HEAD) +fi + echo -e "\n\nRunning the build setup script." source run_conda_forge_build_setup @@ -75,9 +81,10 @@ else EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --no-test" fi - conda mambabuild ./recipe -m ./.ci_support/${CONFIG}.yaml \ + conda build ./recipe -m ./.ci_support/${CONFIG}.yaml \ --suppress-variables ${EXTRA_CB_OPTIONS:-} \ - --clobber-file ./.ci_support/clobber_${CONFIG}.yaml + --clobber-file ./.ci_support/clobber_${CONFIG}.yaml \ + --extra-meta flow_run_id="$flow_run_id" remote_url="$remote_url" sha="$sha" ( startgroup "Validating outputs" ) 2> /dev/null validate_recipe_outputs "${FEEDSTOCK_NAME}" diff --git a/.scripts/run_win_build.bat b/.scripts/run_win_build.bat index 07d3445..5b2b849 100755 --- a/.scripts/run_win_build.bat +++ b/.scripts/run_win_build.bat @@ -17,10 +17,14 @@ call :start_group "Configuring conda" :: Activate the base conda environment call activate base +:: Configure the solver +set "CONDA_SOLVER=libmamba" +if !errorlevel! neq 0 exit /b !errorlevel! +set "CONDA_LIBMAMBA_SOLVER_NO_CHANNELS_FROM_INSTALLED=1" :: Provision the necessary dependencies to build the recipe later echo Installing dependencies -mamba.exe install "python=3.10" pip mamba conda-build boa conda-forge-ci-setup=3 -c conda-forge --strict-channel-priority --yes +mamba.exe install "python=3.10" pip mamba conda-build conda-forge-ci-setup=4 -c conda-forge --strict-channel-priority --yes if !errorlevel! neq 0 exit /b !errorlevel! :: Set basic configuration @@ -41,11 +45,15 @@ if NOT [%HOST_PLATFORM%] == [%BUILD_PLATFORM%] ( set "EXTRA_CB_OPTIONS=%EXTRA_CB_OPTIONS% --no-test" ) +if NOT [%flow_run_id%] == [] ( + set "EXTRA_CB_OPTIONS=%EXTRA_CB_OPTIONS% --extra-meta flow_run_id=%flow_run_id% remote_url=%remote_url% sha=%sha%" +) + call :end_group :: Build the recipe echo Building recipe -conda.exe mambabuild "recipe" -m .ci_support\%CONFIG%.yaml --suppress-variables %EXTRA_CB_OPTIONS% +conda.exe build "recipe" -m .ci_support\%CONFIG%.yaml --suppress-variables %EXTRA_CB_OPTIONS% if !errorlevel! neq 0 exit /b !errorlevel! :: Prepare some environment variables for the upload step diff --git a/.travis.yml b/.travis.yml index da9f66f..4b26aff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -160,6 +160,9 @@ matrix: script: - export CI=travis - export GIT_BRANCH="$TRAVIS_BRANCH" + - export flow_run_id="travis_$TRAVIS_JOB_ID" + - export sha="$TRAVIS_COMMIT" + - export remote_url="https://github.com/$TRAVIS_REPO_SLUG" - export FEEDSTOCK_NAME=$(basename ${TRAVIS_REPO_SLUG}) - if [[ "${TRAVIS_PULL_REQUEST:-}" == "false" ]]; then export IS_PR_BUILD="False"; else export IS_PR_BUILD="True"; fi From be37378291dace587f32e23a0d54baf53b83349b Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 9 Nov 2023 16:54:43 -0800 Subject: [PATCH 3/4] [Patch] Remove FindBlosc2.cmake --- recipe/meta.yaml | 1 + recipe/rmFindBlosc2.patch | 107 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 recipe/rmFindBlosc2.patch diff --git a/recipe/meta.yaml b/recipe/meta.yaml index faaa410..1dba079 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -32,6 +32,7 @@ source: sha256: {{ sha256 }} patches: - python-exclude-from-all.patch + - rmFindBlosc2.patch build: number: {{ build }} diff --git a/recipe/rmFindBlosc2.patch b/recipe/rmFindBlosc2.patch new file mode 100644 index 0000000..cb9fb65 --- /dev/null +++ b/recipe/rmFindBlosc2.patch @@ -0,0 +1,107 @@ +From 0cb1a9104c663928e629e168100e7276923ac82b Mon Sep 17 00:00:00 2001 +From: Axel Huebl +Date: Thu, 9 Nov 2023 16:41:08 -0800 +Subject: [PATCH] Remove cmake/FindBlosc2.cmake + +--- + cmake/FindBlosc2.cmake | 91 ------------------------------------------ + 1 file changed, 91 deletions(-) + delete mode 100644 cmake/FindBlosc2.cmake + +diff --git a/cmake/FindBlosc2.cmake b/cmake/FindBlosc2.cmake +deleted file mode 100644 +index 10762297cc..0000000000 +--- a/cmake/FindBlosc2.cmake ++++ /dev/null +@@ -1,91 +0,0 @@ +-#------------------------------------------------------------------------------# +-# Distributed under the OSI-approved Apache License, Version 2.0. See +-# accompanying file Copyright.txt for details. +-#------------------------------------------------------------------------------# +-# +-# FindBLOSC22 +-# ----------- +-# +-# Try to find the BLOSC2 library +-# +-# This module defines the following variables: +-# +-# BLOSC2_FOUND - System has BLOSC2 +-# BLOSC2_INCLUDE_DIRS - The BLOSC2 include directory +-# BLOSC2_LIBRARIES - Link these to use BLOSC2 +-# BLOSC2_VERSION - Version of the BLOSC2 library to support +-# +-# and the following imported targets: +-# Blosc2::Blosc2 - The core BLOSC2 library +-# +-# You can also set the following variable to help guide the search: +-# BLOSC2_ROOT - The install prefix for BLOSC2 containing the +-# include and lib folders +-# Note: this can be set as a CMake variable or an +-# environment variable. If specified as a CMake +-# variable, it will override any setting specified +-# as an environment variable. +- +-if(NOT BLOSC2_FOUND) +- if((NOT BLOSC2_ROOT) AND (DEFINED ENV{BLOSC2_ROOT})) +- set(BLOSC2_ROOT "$ENV{BLOSC2_ROOT}") +- endif() +- if(BLOSC2_ROOT) +- set(BLOSC2_INCLUDE_OPTS HINTS ${BLOSC2_ROOT}/include NO_DEFAULT_PATHS) +- set(BLOSC2_LIBRARY_OPTS +- HINTS ${BLOSC2_ROOT}/lib ${BLOSC2_ROOT}/lib64 +- NO_DEFAULT_PATHS +- ) +- endif() +- if(WIN32) # uses a Unix-like library prefix on Windows +- set(BLOSC2_LIBRARY_OPTS +- libblosc2 ${BLOSC2_LIBRARY_OPTS} +- ) +- endif() +- +- find_path(BLOSC2_INCLUDE_DIR blosc2.h ${BLOSC2_INCLUDE_OPTS}) +- find_library(BLOSC2_LIBRARY NAMES blosc2 ${BLOSC2_LIBRARY_OPTS}) +- if(BLOSC2_INCLUDE_DIR) +- file(STRINGS ${BLOSC2_INCLUDE_DIR}/blosc2.h _ver_string +- REGEX [=[BLOSC2_VERSION_STRING +"[^"]*"]=] +- ) +- if(_ver_string MATCHES [=[BLOSC2_VERSION_STRING +"([0-9]+.[0-9]+.[0-9]+)]=]) +- set(BLOSC2_VERSION ${CMAKE_MATCH_1}) +- endif() +- endif() +- +- # Blosc2 depends on pthreads +- set(THREADS_PREFER_PTHREAD_FLAG TRUE) +- if(WIN32) +- # try to use the system library +- find_package(Threads) +- if(NOT Threads_FOUND) +- message(STATUS "Blosc2: used the internal pthread library for win32 systems.") +- set(BLOSC2_LIBRARIES) +- else() +- set(BLOSC2_LIBRARIES Threads::Threads) +- endif() +- else() +- find_package(Threads REQUIRED) +- set(BLOSC2_LIBRARIES Threads::Threads) +- endif() +- +- include(FindPackageHandleStandardArgs) +- find_package_handle_standard_args(Blosc2 +- FOUND_VAR BLOSC2_FOUND +- VERSION_VAR BLOSC2_VERSION +- REQUIRED_VARS BLOSC2_LIBRARY BLOSC2_INCLUDE_DIR +- ) +- if(BLOSC2_FOUND) +- set(BLOSC2_INCLUDE_DIRS ${BLOSC2_INCLUDE_DIR}) +- set(BLOSC2_LIBRARIES ${BLOSC2_LIBRARY}) +- if(BLOSC2_FOUND AND NOT TARGET Blosc2::Blosc2) +- add_library(Blosc2::Blosc2 UNKNOWN IMPORTED) +- set_target_properties(Blosc2::Blosc2 PROPERTIES +- IMPORTED_LOCATION "${BLOSC2_LIBRARY}" +- INTERFACE_INCLUDE_DIRECTORIES "${BLOSC2_INCLUDE_DIR}" +- INTERFACE_LINK_LIBRARIES "${BLOSC2_LIBRARIES}" +- ) +- endif() +- endif() +-endif() From 17791813d05c2a77f9c6cf09ba0c1bd5ccbf1a9e Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 10 Nov 2023 09:14:34 -0800 Subject: [PATCH 4/4] ADIOS2_Blosc2_PREFER_SHARED=ON --- recipe/build-libadios.bat | 1 + recipe/build-libadios.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/recipe/build-libadios.bat b/recipe/build-libadios.bat index 0e6d261..ee284af 100644 --- a/recipe/build-libadios.bat +++ b/recipe/build-libadios.bat @@ -21,6 +21,7 @@ cmake ^ -DBUILD_TESTING=OFF ^ -DADIOS2_USE_MPI=OFF ^ -DADIOS2_BUILD_EXAMPLES=OFF ^ + -DADIOS2_Blosc2_PREFER_SHARED=ON ^ -DADIOS2_USE_Blosc2=ON ^ -DADIOS2_USE_BZip2=ON ^ -DADIOS2_USE_Fortran=OFF ^ diff --git a/recipe/build-libadios.sh b/recipe/build-libadios.sh index dcd7952..d87793f 100644 --- a/recipe/build-libadios.sh +++ b/recipe/build-libadios.sh @@ -54,6 +54,7 @@ cmake \ -B build \ -GNinja \ -DADIOS2_BUILD_EXAMPLES=OFF \ + -DADIOS2_Blosc2_PREFER_SHARED=ON \ -DADIOS2_USE_Blosc2=ON \ -DADIOS2_HAVE_ZFP_CUDA=OFF \ -DADIOS2_INSTALL_GENERATE_CONFIG=OFF \