Merge pull request #464 from PrincetonUniversity/issue-398

Issue 398 - Implement dynamic testing

.jenkins/gnu_compiler_checks.gvy

@@ -25,7 +25,7 @@ pipeline{
                     }
                     axis{
                         name 'HostSpace'
-                        values 'SERIAL;-DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_ATOMICS_BYPASS=ON;-n 1', 'OPENMP;-DKokkos_ENABLE_OPENMP=ON;-n 10'
+                        values 'SERIAL;-DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_ATOMICS_BYPASS=ON;-n 1', 'OPENMP;-DKokkos_ENABLE_OPENMP=ON;-n 1 -c 10'
                     }
                 }
                 stages {

.jenkins/intel_compiler_checks.gvy

@@ -21,7 +21,7 @@ pipeline{
                     }
                     axis{
                         name 'HostSpace'
-                        values 'SERIAL;-DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_ATOMICS_BYPASS=ON;-n 1', 'OPENMP;-DKokkos_ENABLE_OPENMP=ON;-n 10'
+                        values 'SERIAL;-DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_ATOMICS_BYPASS=ON;-n 1', 'OPENMP;-DKokkos_ENABLE_OPENMP=ON;-n 1 -c 10'
                     }
                     axis{
                         name 'SIMD'

.jenkins/nvidia_compiler_checks.gvy

@@ -21,7 +21,7 @@ pipeline{
                     }
                     axis{
                         name 'HostSpace'
-                        values 'SERIAL;-DKokkos_ENABLE_SERIAL=ON;-n 1', 'OPENMP;-DKokkos_ENABLE_OPENMP=ON;-n 10'
+                        values 'SERIAL;-DKokkos_ENABLE_SERIAL=ON;-n 1', 'OPENMP;-DKokkos_ENABLE_OPENMP=ON; -n 1 -c 10'
                     }
                     axis{
                         name 'DeviceSpace'

CMakePresets.json

@@ -1,26 +1,43 @@
 {
   "version": 6,
   "configurePresets": [
+    {
+      "name": "release",
+      "displayName": "Default Release -- SIMD enabled",
+      "binaryDir": "build/release",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Release",
+        "BUILD_TESTS": "ON",
+        "ENABLE_SIMD": "ON",
+        "Kokkos_ARCH_NATIVE": "ON",
+        "Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION": "ON",
+        "Kokkos_ENABLE_ATOMICS_BYPASS": "ON"
+      }
+    },
     {
       "name": "debug",
-      "displayName": "Debug (Serial)",
+      "displayName": "Default Debug -- SIMD enabled",
       "binaryDir": "build/debug",
       "cacheVariables": {
         "CMAKE_BUILD_TYPE": "Debug",
         "BUILD_TESTS": "ON",
-        "ENABLE_SIMD": "OFF",
+        "ENABLE_SIMD": "ON",
         "Kokkos_ARCH_NATIVE": "ON",
         "Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION": "ON",
         "Kokkos_ENABLE_ATOMICS_BYPASS": "ON"
       }
     }
   ],
   "buildPresets": [
+    {
+      "name": "release",
+      "configurePreset": "release",
+      "targets": ["all"]
+    },
     {
       "name": "debug",
       "configurePreset": "debug",
-      "targets": ["all"],
-      "jobs": 8
+      "targets": ["all"]
     }
   ]
 }

fortran/meshfem3d/generate_databases/save_parameters.f90

@@ -5,7 +5,7 @@ subroutine save_parameters()
   use shared_parameters, only: LOCAL_PATH, MAX_STRING_LEN
 
   use generate_databases_par, only: &
-    IOUT, myrank, &
+    IOUT, myrank, sizeprocs, &
     NDIM, NSPEC_AB, NGLOB_AB, &
     NGLLX, NGLLY, NGLLZ, nfaces_surface, num_neighbors_all, NGLLSQUARE, &
     nspec2D_xmin, nspec2D_xmin, nspec2D_ymin, nspec2D_ymin, &
@@ -69,6 +69,7 @@ subroutine save_parameters()
       WRITE(IOUT) NGLLY
       WRITE(IOUT) NGLLZ
       WRITE(IOUT) NGLLSQUARE
+      WRITE(IOUT) sizeprocs
 
       ! Write test parameter
       itest = 9997

include/mesh/mesh.hpp

@@ -108,6 +108,7 @@ template <> struct mesh<specfem::dimension::type::dim3> {
   // Struct to store all the mesh parameter
   specfem::mesh::parameters<dimension> parameters;
 
+  //
   // int npgeo; ///< Total number of spectral element control nodes
   // int nspec; ///< Total number of spectral elements
   // int nproc; ///< Total number of processors

include/mesh/parameters/parameters.hpp

@@ -101,6 +101,7 @@ template <> struct parameters<specfem::dimension::type::dim3> {
   int nglly;      ///< Number of GLL points in y
   int ngllz;      ///< Number of GLL points in z
   int ngllsquare; ///< Number of GLL points in square
+  int nproc;      ///< Number of processors
 
   // Integer Parameters: Elements/Nodes
   int nspec;           ///< Number of spectral elements (SEs)

src/IO/mesh/impl/fortran/dim3/read_parameters.cpp

@@ -73,13 +73,15 @@ specfem::IO::mesh::impl::fortran::dim3::read_mesh_parameters(
   int ngly;
   int ngllz;
   int ngllsquare;
+  int nproc;
 
   // Read parameters
   specfem::IO::fortran_read_line(stream, &ndim);
   specfem::IO::fortran_read_line(stream, &ngllx);
   specfem::IO::fortran_read_line(stream, &ngly);
   specfem::IO::fortran_read_line(stream, &ngllz);
   specfem::IO::fortran_read_line(stream, &ngllsquare);
+  specfem::IO::fortran_read_line(stream, &nproc);
 
 #ifndef NDEBUG
   // Print the read parameters
@@ -88,6 +90,7 @@ specfem::IO::mesh::impl::fortran::dim3::read_mesh_parameters(
   std::cout << "ngly:................." << ngly << std::endl;
   std::cout << "ngllz:................" << ngllz << std::endl;
   std::cout << "ngllsquare:..........." << ngllsquare << std::endl;
+  std::cout << "nproc:................" << nproc << std::endl;
 #endif
 
   // Read test parameter

tests/unit-tests/CMakeLists.txt

@@ -18,6 +18,12 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/tests/unit-tests)
 
 include_directories(.)
 
+set(TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+
+# Some of the writing test need to write somewhere and we don't want that
+# to be in the source directory
+set(TEST_OUTPUT_DIR ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
+
 enable_testing()
 
 add_library(
@@ -58,6 +64,7 @@ add_executable(
   gll_tests
   gll/gll_tests.cpp
 )
+
 target_link_libraries(
   gll_tests
   gtest_main
@@ -203,6 +210,7 @@ add_executable(
   assembly/sources/sources.cpp
 )
 
+target_compile_definitions(assembly_tests PRIVATE TEST_OUTPUT_DIR=${TEST_OUTPUT_DIR})
 
 target_link_libraries(
   assembly_tests
@@ -423,22 +431,22 @@ target_link_libraries(
 # Link to gtest only if MPI is enabled
 if (NOT MPI_PARALLEL)
   include(GoogleTest)
-  gtest_discover_tests(gll_tests)
-  gtest_discover_tests(lagrange_tests)
-  gtest_discover_tests(fortranio_test)
-  gtest_discover_tests(IO_tests)
-  gtest_discover_tests(mesh_tests)
-  gtest_discover_tests(compute_partial_derivatives_tests)
-  # gtest_discover_tests(compute_elastic_tests)
-  # # gtest_discover_tests(compute_acoustic_tests)
-  # gtest_discover_tests(compute_coupled_interfaces_tests)
-  gtest_discover_tests(compute_tests)
-  gtest_discover_tests(assembly_tests)
-  gtest_discover_tests(policies)
-  gtest_discover_tests(locate_point)
-  gtest_discover_tests(interpolate_function)
-  gtest_discover_tests(rmass_inverse_tests)
-  gtest_discover_tests(displacement_newmark_tests)
+  gtest_discover_tests(gll_tests WORKING_DIRECTORY ${TEST_DIR})
+  gtest_discover_tests(lagrange_tests WORKING_DIRECTORY ${TEST_DIR})
+  gtest_discover_tests(fortranio_test WORKING_DIRECTORY ${TEST_DIR})
+  gtest_discover_tests(IO_tests WORKING_DIRECTORY ${TEST_DIR})
+  gtest_discover_tests(mesh_tests WORKING_DIRECTORY ${TEST_DIR})
+  gtest_discover_tests(compute_partial_derivatives_tests WORKING_DIRECTORY ${TEST_DIR})
+  # gtest_discover_tests(compute_elastic_tests WORKING_DIRECTORY ${TEST_DIR})
+  # # gtest_discover_tests(compute_acoustic_tests WORKING_DIRECTORY ${TEST_DIR})
+  # gtest_discover_tests(compute_coupled_interfaces_tests WORKING_DIRECTORY ${TEST_DIR})
+  gtest_discover_tests(compute_tests WORKING_DIRECTORY ${TEST_DIR})
+  gtest_discover_tests(assembly_tests WORKING_DIRECTORY ${TEST_DIR})
+  gtest_discover_tests(policies WORKING_DIRECTORY ${TEST_DIR})
+  gtest_discover_tests(locate_point WORKING_DIRECTORY ${TEST_DIR})
+  gtest_discover_tests(interpolate_function WORKING_DIRECTORY ${TEST_DIR})
+  gtest_discover_tests(rmass_inverse_tests WORKING_DIRECTORY ${TEST_DIR})
+  gtest_discover_tests(displacement_newmark_tests WORKING_DIRECTORY ${TEST_DIR})
   # gtest_discover_tests(seismogram_elastic_tests)
   # gtest_discover_tests(seismogram_acoustic_tests)
 endif(NOT MPI_PARALLEL)

tests/unit-tests/IO/sources/test_read_sources.cpp

@@ -12,8 +12,7 @@ TEST(IO_TESTS, read_sources) {
    *  This test checks whether a moment tensor source is read correctly
    */
 
-  std::string sources_file =
-      "../../../tests/unit-tests/IO/sources/data/single_moment_tensor.yml";
+  std::string sources_file = "IO/sources/data/single_moment_tensor.yml";
 
   YAML::Node databases;
   databases["sources"] = sources_file;

tests/unit-tests/algorithms/interpolate_function.cpp

@@ -16,8 +16,7 @@ inline type_real function1(const type_real x, const type_real z) {
 
 TEST(ALGORITHMS, interpolate_function) {
 
-  std::string database_file =
-      "../../../tests/unit-tests/algorithms/serial/database.bin";
+  std::string database_file = "algorithms/serial/database.bin";
 
   // Read Mesh database
   specfem::MPI::MPI *mpi = MPIEnvironment::get_mpi();

tests/unit-tests/algorithms/locate.cpp

@@ -9,8 +9,7 @@
 
 TEST(ALGORITHMS, locate_point) {
 
-  std::string database_file =
-      "../../../tests/unit-tests/algorithms/serial/database.bin";
+  std::string database_file = "algorithms/serial/database.bin";
 
   // Read Mesh database
   specfem::MPI::MPI *mpi = MPIEnvironment::get_mpi();

tests/unit-tests/assembly/compute_wavefield/compute_wavefield.cpp

@@ -100,7 +100,7 @@ void test_compute_wavefield(specfem::compute::assembly &assembly) {
 TEST_F(ASSEMBLY, compute_wavefield) {
   for (auto parameters : *this) {
     const auto Test = std::get<0>(parameters);
-    specfem::compute::assembly assembly = std::get<4>(parameters);
+    specfem::compute::assembly assembly = std::get<5>(parameters);
 
     try {
       test_compute_wavefield(assembly);

tests/unit-tests/assembly/kernels/kernels.cpp

@@ -516,7 +516,7 @@ void test_kernels(specfem::compute::assembly &assembly) {
 TEST_F(ASSEMBLY, kernels_device_functions) {
   for (auto parameters : *this) {
     const auto Test = std::get<0>(parameters);
-    specfem::compute::assembly assembly = std::get<4>(parameters);
+    specfem::compute::assembly assembly = std::get<5>(parameters);
 
     try {
       test_kernels(assembly);

tests/unit-tests/assembly/properties/properties.cpp

@@ -10,6 +10,12 @@
 #include "specfem_setup.hpp"
 #include <gtest/gtest.h>
 
+#define STRINGIFY(x) #x
+#define TOSTRING(x) STRINGIFY(x)
+#ifndef TEST_OUTPUT_DIR
+#define TEST_OUTPUT_DIR "." // Fallback in case it's not set
+#endif
+
 inline void error_message_header(std::ostringstream &message,
                                  const type_real &value, const int &mode) {
   if (mode == 0) {
@@ -647,7 +653,8 @@ void check_load_on_device(specfem::compute::properties &properties,
 
 void test_properties(
     specfem::compute::assembly &assembly,
-    const specfem::mesh::mesh<specfem::dimension::type::dim2> &mesh) {
+    const specfem::mesh::mesh<specfem::dimension::type::dim2> &mesh,
+    const std::string &suffix) {
 
   auto &properties = assembly.properties;
   auto &element_types = assembly.element_types;
@@ -662,9 +669,15 @@ void test_properties(
       WHERE(DIMENSION_TAG_DIM2) WHERE(MEDIUM_TAG_ELASTIC, MEDIUM_TAG_ACOUSTIC)
           WHERE(PROPERTY_TAG_ISOTROPIC, PROPERTY_TAG_ANISOTROPIC))
 
+  // stage 2 prepare file path
+  std::string output_dir = TOSTRING(TEST_OUTPUT_DIR);
+  boost::filesystem::path dir_path =
+      boost::filesystem::path(output_dir) / "property_io" / suffix;
+  boost::filesystem::create_directories(dir_path);
+
   // stage 2: write properties
   specfem::IO::property_writer<specfem::IO::ASCII<specfem::IO::write> > writer(
-      ".");
+      dir_path.string());
   writer.write(assembly);
 
   // stage 3: modify properties and check store_on_host and load_on_device
@@ -687,7 +700,7 @@ void test_properties(
 
   // stage 4: restore properties to initial value from disk
   specfem::IO::property_reader<specfem::IO::ASCII<specfem::IO::read> > reader(
-      ".");
+      dir_path.string());
   reader.read(assembly);
 
   // stage 5: check if properties are correctly written and read
@@ -706,16 +719,21 @@ void test_properties(
   // check_compute_to_mesh<specfem::element::medium_tag::acoustic,
   //                       specfem::element::property_tag::isotropic>(assembly,
   //  mesh);
+
+  // stage 6: remove directory
+  std::cout << "Removing directory: " << dir_path << std::endl;
+  boost::filesystem::remove_all(dir_path);
 }
 
 TEST_F(ASSEMBLY, properties) {
   for (auto parameters : *this) {
     auto Test = std::get<0>(parameters);
     auto mesh = std::get<1>(parameters);
-    auto assembly = std::get<4>(parameters);
+    auto suffix = std::get<4>(parameters);
+    auto assembly = std::get<5>(parameters);
 
     try {
-      test_properties(assembly, mesh);
+      test_properties(assembly, mesh, suffix);
 
       std::cout << "-------------------------------------------------------\n"
                 << "\033[0;32m[PASSED]\033[0m " << Test.name << "\n"
@@ -732,4 +750,11 @@ TEST_F(ASSEMBLY, properties) {
       ADD_FAILURE();
     }
   }
+
+  // Clear property_io directory
+  std::cout << "Removing directory: " << TOSTRING(TEST_OUTPUT_DIR) << std::endl;
+  std::string output_dir = TOSTRING(TEST_OUTPUT_DIR);
+  boost::filesystem::path output_dir_path =
+      boost::filesystem::path(output_dir) / "property_io";
+  boost::filesystem::remove_all(output_dir_path);
 }

tests/unit-tests/assembly/sources/sources.cpp

@@ -304,7 +304,7 @@ TEST_F(ASSEMBLY, sources) {
   for (auto parameters : *this) {
     const auto Test = std::get<0>(parameters);
     auto sources = std::get<2>(parameters);
-    specfem::compute::assembly assembly = std::get<4>(parameters);
+    specfem::compute::assembly assembly = std::get<5>(parameters);
 
     try {
       test_assembly_source_construction(sources, assembly);