Skip to content

Interface ectrans with GPU backend #252

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wdeconinck wants to merge 2 commits into
base: develop
Choose a base branch
from
Open

Interface ectrans with GPU backend #252

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
721a758
Link and run tests with transi_gpu_dp
wdeconinck Oct 28, 2024
01b66a4
Improved error handling for maybe uninmplemented ectrans GPU features
wdeconinck Dec 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 26 additions & 13 deletions cmake/features/ECTRANS.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,29 +14,42 @@ if( DEFINED ATLAS_ENABLE_ECTRANS )
set( ENABLE_ECTRANS ${ATLAS_ENABLE_ECTRANS} )
endif()

macro( make_transi_alias target )
if( NOT TARGET transi )
if( CMAKE_VERSION VERSION_LESS 3.18 )
# Before CMake 3.18 it is not possible to alias a non-global imported target
# Make the import global. Warning, this may break further find_package
get_target_property( ${target}_IMPORTED ${target} IMPORTED )
if( transi_dp_IMPORTED )
set_target_properties( ${target} PROPERTIES IMPORTED_GLOBAL TRUE)
endif()
endif()
add_library( transi ALIAS ${target} )
endif()
endmacro()

set( atlas_HAVE_PACKAGE_ECTRANS 0 )
if( atlas_HAVE_ATLAS_FUNCTIONSPACE AND (ENABLE_ECTRANS OR NOT DEFINED ENABLE_ECTRANS) )
find_package( ectrans 1.1 COMPONENTS transi double QUIET )
if( TARGET transi_dp )
if( TARGET transi_dp OR TARGET transi_gpu_dp )
set( transi_FOUND TRUE )
if( NOT TARGET transi )
if( CMAKE_VERSION VERSION_LESS 3.18 )
# Before CMake 3.18 it is not possible to alias a non-global imported target
# Make the import global. Warning, this may break further find_package
get_target_property( transi_dp_IMPORTED transi_dp IMPORTED )
if( transi_dp_IMPORTED )
set_target_properties( transi_dp PROPERTIES IMPORTED_GLOBAL TRUE)
endif()
endif()
add_library( transi ALIAS transi_dp )
endif()
set( atlas_HAVE_PACKAGE_ECTRANS 1 )
else()
find_package( transi 0.8 QUIET )
endif()
endif()
ecbuild_add_option( FEATURE ECTRANS
DESCRIPTION "Support for IFS spectral transforms"
DESCRIPTION "Support for ectrans spectral transforms"
CONDITION atlas_HAVE_ATLAS_FUNCTIONSPACE AND transi_FOUND )
ecbuild_add_option( FEATURE ECTRANS_GPU DEFAULT OFF
DESCRIPTION "Support for ectrans spectral transforms"
CONDITION HAVE_ECTRANS AND TARGET transi_gpu_dp )
if( HAVE_ECTRANS_GPU AND TARGET transi_gpu_dp)
ecbuild_warn("ectrans GPU version")
make_transi_alias( transi_gpu_dp )
elseif( HAVE_ECTRANS AND TARGET transi_dp )
ecbuild_warn("ectrans CPU version")
make_transi_alias( transi_dp )
endif()

endif()
4 changes: 2 additions & 2 deletions src/tests/functionspace/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ if( HAVE_FCTEST )
LINKER_LANGUAGE Fortran
SOURCES fctest_functionspace.F90
LIBS atlas_f
ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT}
ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ATLAS_FINALISES_MPI=1
)

add_fctest( TARGET atlas_fctest_blockstructuredcolumns
Expand All @@ -28,7 +28,7 @@ if( HAVE_FCTEST )
LINKER_LANGUAGE Fortran
SOURCES fctest_blockstructuredcolumns.F90
LIBS atlas_f
ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT}
ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ATLAS_FINALISES_MPI=1
)

endif()
Expand Down
2 changes: 1 addition & 1 deletion src/tests/redistribution/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ if( HAVE_FORTRAN )
LINKER_LANGUAGE Fortran
SOURCES fctest_redistribution.F90
LIBS atlas_f
ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT}
ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ATLAS_FINALISES_MPI=1
)
endif()

Expand Down
5 changes: 3 additions & 2 deletions src/tests/trans/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
# granted to it by virtue of its status as an intergovernmental organisation nor
# does it submit to any jurisdiction.

list( APPEND ATLAS_TEST_ENVIRONMENT ATLAS_TEST_IGNORE_ECTRANS_NOT_IMPLEMENTED=1 )
if( HAVE_FCTEST )

if( atlas_HAVE_ECTRANS )
Expand All @@ -16,7 +17,7 @@ if( HAVE_FCTEST )
LIBS atlas_f
CONDITION eckit_HAVE_MPI AND ( transi_HAVE_MPI OR ectrans_HAVE_MPI ) AND MPI_SLOTS GREATER_EQUAL 4
MPI 4
ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT}
ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ATLAS_FINALISES_MPI=1
)
endif()

Expand All @@ -25,7 +26,7 @@ if( HAVE_FCTEST )
LINKER_LANGUAGE Fortran
SOURCES fctest_trans_invtrans_grad.F90
LIBS atlas_f
ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT}
ENVIRONMENT ${ATLAS_TEST_ENVIRONMENT} ATLAS_FINALISES_MPI=1
)
endif()

Expand Down
64 changes: 51 additions & 13 deletions src/tests/trans/test_trans.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
*/

#include <algorithm>
#include <cstdlib>

#include "eckit/exception/Exceptions.h"
#include "eckit/filesystem/PathName.h"
Expand Down Expand Up @@ -48,6 +49,44 @@
#endif
#endif

bool ignore_ectrans_not_implemented(std::exception& e) {
static bool ATLAS_TEST_IGNORE_ECTRANS_NOT_IMPLEMENTED = []() -> bool {
const char* env = ::getenv("ATLAS_TEST_IGNORE_ECTRANS_NOT_IMPLEMENTED");
if (env) {
return std::atoi(env);
}
return false;
}();
if (ATLAS_TEST_IGNORE_ECTRANS_NOT_IMPLEMENTED == false) {
return false;
}
std::string errstr(e.what());
for(auto& c : errstr){ c = std::tolower(c); }
std::vector<std::string> search{"trans", "not", "implemented"};
return std::all_of(search.begin() ,search.end(),[&](const std::string& s)->bool {
return errstr.find(s) != std::string::npos;
});
}

#define ECTRANS_MAYBE_NOT_IMPLEMENTED(expr) \
do { \
try { \
expr; \
} \
catch (std::exception & e) { \
if( ignore_ectrans_not_implemented(e) ) { \
atlas::Log::error() << "ERROR IGNORED: Not implemented with ectrans code path\n" \
<< "Skipping remainder of test" \
<< std::endl; \
return; \
} \
throw eckit::testing::TestException("Unexpected exception caught: "+std::string(e.what()), Here()); \
} \
catch (...) { \
throw eckit::testing::TestException("Unexpected and unknown exception caught", Here()); \
} \
} while (false)

using namespace eckit;
using atlas::grid::detail::partitioner::EqualRegionsPartitioner;
using atlas::grid::detail::partitioner::TransPartitioner;
Expand Down Expand Up @@ -166,7 +205,6 @@ CASE("test_trans_options") {
Log::info() << "trans_opts = " << opts << std::endl;
}

#ifdef TRANS_HAVE_IO
CASE("test_write_read_cache") {
Log::info() << "test_write_read_cache" << std::endl;
using namespace trans;
Expand All @@ -193,7 +231,6 @@ CASE("test_write_read_cache") {
Trans trans_cache_O24(legendre_cache_O24, Grid("O24"), 23, option::flt(false));
}
}
#endif

CASE("test_distspec") {
trans::TransIFS trans(Grid("F80"), 159);
Expand Down Expand Up @@ -518,9 +555,9 @@ CASE("test_trans_using_functionspace_StructuredColumns") {

CASE("test_trans_MIR_lonlat") {
Log::info() << "test_trans_MIR_lonlat" << std::endl;

Grid grid("L48");
trans::Trans trans(grid, 47);
trans::Trans trans;
ECTRANS_MAYBE_NOT_IMPLEMENTED((trans = trans::Trans(grid, 47)));

// global fields
std::vector<double> spf(trans.spectralCoefficients(), 0.);
Expand Down Expand Up @@ -561,7 +598,9 @@ CASE("test_trans_VorDivToUV") {
std::vector<double> field_U(nfld * nspec2);
std::vector<double> field_V(nfld * nspec2);

vordiv_to_UV.execute(nspec2, nfld, field_vor.data(), field_div.data(), field_U.data(), field_V.data());
ECTRANS_MAYBE_NOT_IMPLEMENTED(
vordiv_to_UV.execute(nspec2, nfld, field_vor.data(), field_div.data(), field_U.data(), field_V.data());
);

// TODO: do some meaningful checks
Log::info() << "Trans library" << std::endl;
Expand All @@ -580,8 +619,9 @@ CASE("test_trans_VorDivToUV") {
std::vector<double> field_U(nfld * nspec2);
std::vector<double> field_V(nfld * nspec2);

vordiv_to_UV.execute(nspec2, nfld, field_vor.data(), field_div.data(), field_U.data(), field_V.data());

EXPECT_NO_THROW(
vordiv_to_UV.execute(nspec2, nfld, field_vor.data(), field_div.data(), field_U.data(), field_V.data());
);
// TODO: do some meaningful checks
Log::info() << "Local transform" << std::endl;
Log::info() << "U: " << std::endl;
Expand All @@ -593,7 +633,6 @@ CASE("test_trans_VorDivToUV") {
}
}

#ifdef TRANS_HAVE_IO
CASE("ATLAS-256: Legendre coefficient expected unique identifiers") {
if (mpi::comm().size() == 1) {
util::Config options;
Expand Down Expand Up @@ -679,21 +718,20 @@ CASE("ATLAS-256: Legendre coefficient expected unique identifiers") {
for (auto& domain : domains) {
for (int T : spectral_T) {
for (auto name : grids) {
Log::info() << "Case name:" << name << ", T:" << T << ", domain:" << domain << ", UID:'" << *uid
<< "'" << std::endl;
//Log::info() << "Case name:" << name << ", T:" << T << ", domain:" << domain << ", UID:'" << *uid
// << "'" << std::endl;

Grid grid(name, domain);
auto test = trans::LegendreCacheCreator(grid, T, options).uid();
ATLAS_DEBUG_VAR(test);
EXPECT(test == *uid);
//ATLAS_DEBUG_VAR(test);
EXPECT_EQ(test, *uid);

uid++;
}
}
}
}
}
#endif

//-----------------------------------------------------------------------------

Expand Down
50 changes: 46 additions & 4 deletions src/tests/trans/test_transgeneral.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,45 @@
#endif
#endif

bool ignore_ectrans_not_implemented(std::exception& e) {
static bool ATLAS_TEST_IGNORE_ECTRANS_NOT_IMPLEMENTED = []() -> bool {
const char* env = ::getenv("ATLAS_TEST_IGNORE_ECTRANS_NOT_IMPLEMENTED");
if (env) {
std::cout << "ATLAS_TEST_IGNORE_ECTRANS_NOT_IMPLEMENTED=" << env << std::endl;
return std::atoi(env);
}
return false;
}();
if (ATLAS_TEST_IGNORE_ECTRANS_NOT_IMPLEMENTED == false) {
return false;
}
std::string errstr(e.what());
for(auto& c : errstr){ c = std::tolower(c); }
std::vector<std::string> search{"trans", "not", "implemented"};
return std::all_of(search.begin() ,search.end(),[&](const std::string& s)->bool {
return errstr.find(s) != std::string::npos;
});
}

#define ECTRANS_MAYBE_NOT_IMPLEMENTED(expr) \
do { \
try { \
expr; \
} \
catch (std::exception & e) { \
if( ignore_ectrans_not_implemented(e) ) { \
atlas::Log::error() << "ERROR IGNORED: Not implemented with ectrans code path\n" \
<< "Skipping remainder of test" \
<< std::endl; \
return; \
} \
throw eckit::testing::TestException("Unexpected exception caught: "+std::string(e.what()), Here()); \
} \
catch (...) { \
throw eckit::testing::TestException("Unexpected and unknown exception caught", Here()); \
} \
} while (false)

using namespace eckit;

using atlas::array::Array;
Expand Down Expand Up @@ -1578,7 +1617,8 @@ CASE("test_trans_levels") {
}
#endif


#if 1
// ECTRANS GPU VERSION DOES NOT YET SUPPORT THIS
#if ATLAS_HAVE_TRANS
#if ATLAS_HAVE_ECTRANS || defined(TRANS_HAVE_INVTRANS_ADJ)
CASE("test_2level_adjoint_test_with_powerspectrum_convolution") {
Expand Down Expand Up @@ -1652,7 +1692,7 @@ CASE("test_2level_adjoint_test_with_powerspectrum_convolution") {

// transform fields to spectral and view
if (test_name[test_type].compare("inverse") == 0) {
transIFS.invtrans_adj(gpf, spf);
ECTRANS_MAYBE_NOT_IMPLEMENTED(transIFS.invtrans_adj(gpf, spf));
} else if (test_name[test_type].compare("direct") == 0) {
transIFS.dirtrans(gpf, spf);
}
Expand Down Expand Up @@ -1692,8 +1732,9 @@ CASE("test_2level_adjoint_test_with_powerspectrum_convolution") {
if (test_name[test_type].compare("inverse") == 0) {
transIFS.invtrans(spf, gpf2);
} else if (test_name[test_type].compare("direct") == 0) {
transIFS.dirtrans_adj(spf, gpf2);
ECTRANS_MAYBE_NOT_IMPLEMENTED(transIFS.dirtrans_adj(spf, gpf2));
}

Log::info() << "adjoint test transforms " << test_name[test_type] << std::endl;


Expand Down Expand Up @@ -1791,7 +1832,7 @@ CASE("test_2level_adjoint_test_with_vortdiv") {
}
atlas::mpi::comm().allReduceInPlace(adj_value, eckit::mpi::sum());

transIFS.dirtrans_wind2vordiv_adj(spfvor, spfdiv, gpfuv2);
ECTRANS_MAYBE_NOT_IMPLEMENTED(transIFS.dirtrans_wind2vordiv_adj(spfvor, spfdiv, gpfuv2));

double adj_value2(0.0);
for (atlas::idx_t j = gridFS.j_begin(); j < gridFS.j_end(); ++j) {
Expand All @@ -1811,6 +1852,7 @@ CASE("test_2level_adjoint_test_with_vortdiv") {
}
#endif
#endif
#endif

#if 0
CASE( "test_trans_fourier_truncation" ) {
Expand Down
Loading