Integrate the Cabana load balancer

.github/workflows/CI.yml

@@ -30,23 +30,32 @@ jobs:
           cmake -B build -DCMAKE_INSTALL_PREFIX=$HOME/kokkos -DKokkos_CXX_STANDARD=14 -DKokkos_ENABLE_${{ matrix.backend }}=ON
           cmake --build build --parallel 2
           cmake --install build
+      - name: Checkout ALL
+        run: |
+          git clone --depth 1 --branch master https://gitlab.jsc.fz-juelich.de/SLMS/loadbalancing ALL
+      - name: Build ALL
+        working-directory: ALL
+        run: |
+          cmake -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$HOME/ALL
+          cmake --build build --parallel 2
+          cmake --install build
       - name: Checkout Cabana
         uses: actions/[email protected]
         with:
-          repository: ECP-copa/Cabana
-          ref: master
+          repository: aetx/Cabana
+          ref: cajita_loadbalancer
           path: Cabana
       - name: Build Cabana
         working-directory: Cabana
         run: |
-          cmake -B build -DCMAKE_INSTALL_PREFIX=$HOME/Cabana -DCMAKE_PREFIX_PATH="$HOME/kokkos" -DCabana_REQUIRE_${{ matrix.backend }}=ON
+          cmake -B build -DCMAKE_INSTALL_PREFIX=$HOME/Cabana -DCMAKE_PREFIX_PATH="$HOME/kokkos;$HOME/ALL" -DCabana_REQUIRE_${{ matrix.backend }}=ON -DCabana_REQUIRE_ALL=ON
           cmake --build build --parallel 2
           cmake --install build
       - name: Checkout code
         uses: actions/[email protected]
       - name: Build
         run: |
-          cmake -B build -DCMAKE_PREFIX_PATH="$HOME/kokkos;$HOME/Cabana"
+          cmake -B build -DCMAKE_PREFIX_PATH="$HOME/kokkos;$HOME/Cabana;$HOME/ALL"
           cmake --build build --parallel 2
           cd build && ctest --output-on-failure
       - name: Format

.gitignore

@@ -1 +1,2 @@
-*.mp4
+*.mp4
+*.swp

CMakeLists.txt

@@ -4,6 +4,7 @@ cmake_minimum_required(VERSION 3.12)
 project(ExaMPM LANGUAGES CXX VERSION 0.1.0)
 
 include(GNUInstallDirs)
+include(CMakeDependentOption)
 
 # find dependencies
 set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH})
@@ -16,6 +17,8 @@ if( NOT Cabana_ENABLE_CAJITA )
 endif()
 find_package(Silo REQUIRED)
 
+cmake_dependent_option(ExaMPM_ENABLE_LB "Enable Cabana load balancing" ON Cabana_ENABLE_ALL OFF)
+
 # find Clang Format
 find_package( CLANG_FORMAT 10 )
 

examples/dam_break.cpp

@@ -66,7 +66,7 @@ struct ParticleInitFunc
 //---------------------------------------------------------------------------//
 void damBreak( const double cell_size, const int ppc, const int halo_size,
                const double delta_t, const double t_final, const int write_freq,
-               const std::string& device )
+               const std::string& device, const int lb_freq )
 {
     // The dam break domain is in a box on [0,1] in each dimension.
     Kokkos::Array<double, 6> global_box = { 0.0, 0.0, 0.0, 1.0, 1.0, 1.0 };
@@ -87,7 +87,8 @@ void damBreak( const double cell_size, const int ppc, const int halo_size,
     int comm_size;
     MPI_Comm_size( MPI_COMM_WORLD, &comm_size );
     std::array<int, 3> ranks_per_dim = { 1, comm_size, 1 };
-    Cajita::ManualPartitioner partitioner( ranks_per_dim );
+    std::shared_ptr<Cajita::ManualPartitioner> partitioner =
+        std::make_shared<Cajita::ManualPartitioner>( ranks_per_dim );
 
     // Material properties.
     double bulk_modulus = 1.0e5;
@@ -112,7 +113,7 @@ void damBreak( const double cell_size, const int ppc, const int halo_size,
         device, MPI_COMM_WORLD, global_box, global_num_cell, periodic,
         partitioner, halo_size, ParticleInitFunc( cell_size, ppc, density ),
         ppc, bulk_modulus, density, gamma, kappa, delta_t, gravity, bc );
-    solver->solve( t_final, write_freq );
+    solver->solve( t_final, write_freq, lb_freq );
 }
 
 //---------------------------------------------------------------------------//
@@ -143,8 +144,12 @@ int main( int argc, char* argv[] )
     // device type
     std::string device( argv[7] );
 
+    // optional lb frequency
+    int lb_freq = argc > 8 ? std::atoi( argv[8] ) : 10;
+
     // run the problem.
-    damBreak( cell_size, ppc, halo_size, delta_t, t_final, write_freq, device );
+    damBreak( cell_size, ppc, halo_size, delta_t, t_final, write_freq, device,
+              lb_freq );
 
     Kokkos::finalize();
 

examples/free_fall.cpp

@@ -65,7 +65,7 @@ struct ParticleInitFunc
 //---------------------------------------------------------------------------//
 void freeFall( const double cell_size, const int ppc, const int halo_size,
                const double delta_t, const double t_final, const int write_freq,
-               const std::string& device )
+               const std::string& device, const int lb_freq )
 {
     // The dam break domain is in a box on [0,1] in each dimension.
     Kokkos::Array<double, 6> global_box = { -0.5, -0.5, -0.5, 0.5, 0.5, 0.5 };
@@ -86,7 +86,8 @@ void freeFall( const double cell_size, const int ppc, const int halo_size,
     int comm_size;
     MPI_Comm_size( MPI_COMM_WORLD, &comm_size );
     std::array<int, 3> ranks_per_dim = { 1, comm_size, 1 };
-    Cajita::ManualPartitioner partitioner( ranks_per_dim );
+    std::shared_ptr<Cajita::ManualPartitioner> partitioner =
+        std::make_shared<Cajita::ManualPartitioner>( ranks_per_dim );
 
     // Material properties.
     double bulk_modulus = 5.0e5;
@@ -111,7 +112,7 @@ void freeFall( const double cell_size, const int ppc, const int halo_size,
         device, MPI_COMM_WORLD, global_box, global_num_cell, periodic,
         partitioner, halo_size, ParticleInitFunc( cell_size, ppc, density ),
         ppc, bulk_modulus, density, gamma, kappa, delta_t, gravity, bc );
-    solver->solve( t_final, write_freq );
+    solver->solve( t_final, write_freq, lb_freq );
 }
 
 //---------------------------------------------------------------------------//
@@ -142,8 +143,12 @@ int main( int argc, char* argv[] )
     // device type
     std::string device( argv[7] );
 
+    // optional lb frequency
+    int lb_freq = argc > 8 ? std::atoi( argv[8] ) : 10;
+
     // run the problem.
-    freeFall( cell_size, ppc, halo_size, delta_t, t_final, write_freq, device );
+    freeFall( cell_size, ppc, halo_size, delta_t, t_final, write_freq, device,
+              lb_freq );
 
     Kokkos::finalize();
 

src/CMakeLists.txt

@@ -8,6 +8,7 @@ set(HEADERS
   ExaMPM_Solver.hpp
   ExaMPM_TimeIntegrator.hpp
   ExaMPM_Types.hpp
+  ExaMPM_VTKDomainWriter.hpp
   ExaMPM_VelocityInterpolation.hpp
   )
 
@@ -17,6 +18,10 @@ set(SOURCES
 
 add_library(exampm ${SOURCES})
 
+if(ExaMPM_ENABLE_LB)
+  target_compile_definitions(exampm PUBLIC ExaMPM_ENABLE_LB)
+endif()
+
 target_link_libraries(exampm
   Cabana::cabanacore
   Cabana::Cajita

src/ExaMPM_Mesh.hpp

@@ -91,16 +91,25 @@ class Mesh
         }
 
         // Create the global mesh.
-        auto global_mesh = Cajita::createUniformGlobalMesh(
+        _global_mesh = Cajita::createUniformGlobalMesh(
             global_low_corner, global_high_corner, num_cell );
 
         // Build the global grid.
-        auto global_grid = Cajita::createGlobalGrid( comm, global_mesh,
-                                                     periodic, partitioner );
+        _global_grid = Cajita::createGlobalGrid( comm, _global_mesh, periodic,
+                                                 partitioner );
 
         // Build the local grid.
-        int halo_width = std::max( minimum_halo_cell_width, halo_cell_width );
-        _local_grid = Cajita::createLocalGrid( global_grid, halo_width );
+        _halo_width = std::max( minimum_halo_cell_width, halo_cell_width );
+        _local_grid = Cajita::createLocalGrid( _global_grid, _halo_width );
+    }
+
+    // Global grid updated and update local values
+    void newGlobalGrid(
+        std::shared_ptr<Cajita::GlobalGrid<Cajita::UniformMesh<double>>>&
+            global_grid )
+    {
+        _global_grid = global_grid;
+        _local_grid = Cajita::createLocalGrid( _global_grid, _halo_width );
     }
 
     // Get the local grid.
@@ -110,6 +119,19 @@ class Mesh
         return _local_grid;
     }
 
+    // Get the mutable global grid.
+    const std::shared_ptr<Cajita::GlobalGrid<Cajita::UniformMesh<double>>>&
+    globalGrid()
+    {
+        return _global_grid;
+    }
+
+    const std::shared_ptr<Cajita::GlobalMesh<Cajita::UniformMesh<double>>>&
+    globalMesh() const
+    {
+        return _global_mesh;
+    }
+
     // Get the cell size.
     double cellSize() const
     {
@@ -130,6 +152,12 @@ class Mesh
 
   public:
     std::shared_ptr<Cajita::LocalGrid<Cajita::UniformMesh<double>>> _local_grid;
+    std::shared_ptr<Cajita::GlobalGrid<Cajita::UniformMesh<double>>>
+        _global_grid;
+    std::shared_ptr<Cajita::GlobalMesh<Cajita::UniformMesh<double>>>
+        _global_mesh;
+
+    int _halo_width;
 
     Kokkos::Array<int, 3> _min_domain_global_node_index;
     Kokkos::Array<int, 3> _max_domain_global_node_index;

src/ExaMPM_ProblemManager.hpp

@@ -116,7 +116,12 @@ class ProblemManager
     {
         initializeParticles( exec_space, *( _mesh->localGrid() ),
                              particles_per_cell, create_functor, _particles );
+        updateMesh( mesh );
+    }
 
+    void updateMesh( const std::shared_ptr<mesh_type>& mesh )
+    {
+        _mesh = mesh;
         auto node_vector_layout =
             Cajita::createArrayLayout( _mesh->localGrid(), 3, Cajita::Node() );
         auto node_scalar_layout =
@@ -259,6 +264,9 @@ class ProblemManager
 
     void scatter( Location::Cell, Field::Density ) const
     {
+        printf( "%s: _density extent %lu %lu %lu\n", __func__,
+                _density->view().extent( 0 ), _density->view().extent( 1 ),
+                _density->view().extent( 2 ) );
         _cell_scalar_halo->scatter( execution_space(),
                                     Cajita::ScatterReduce::Sum(), *_density );
     }

src/ExaMPM_SiloParticleWriter.hpp

@@ -413,6 +413,8 @@ void writeTimeStep( const GlobalGridType& global_grid,
     // Mirror the coordinates to the host.
     auto host_coords =
         Kokkos::create_mirror_view_and_copy( Kokkos::HostSpace(), view );
+    // todo(sschulz): Handle empty domains. Without kokkos debug build this
+    // seems to work. But it should still be handled properly.
 
     // Add the point mesh.
     std::string mesh_name = "particles";