Notify instead of error when no elements on rank

src/smith/physics/mesh.cpp

@@ -24,6 +24,7 @@ Mesh::Mesh(const std::string& meshfile, const std::string& meshtag, int refine_s
 {
   auto meshtmp = mesh::refineAndDistribute(buildMeshFromFile(meshfile), refine_serial, refine_parallel, comm);
   mfem_mesh_ = &smith::StateManager::setMesh(std::move(meshtmp), mesh_tag_);
+  notifyIfRankHasNoElements();
   createDomains();
 }
 
@@ -32,6 +33,7 @@ Mesh::Mesh(mfem::Mesh&& mesh, const std::string& meshtag, int refine_serial, int
 {
   auto meshtmp = smith::mesh::refineAndDistribute(std::move(mesh), refine_serial, refine_parallel, comm);
   mfem_mesh_ = &smith::StateManager::setMesh(std::move(meshtmp), mesh_tag_);
+  notifyIfRankHasNoElements();
   createDomains();
 }
 
@@ -41,9 +43,39 @@ Mesh::Mesh(mfem::ParMesh&& mesh, const std::string& meshtag) : mesh_tag_(meshtag
   meshtmp->EnsureNodes();
   meshtmp->ExchangeFaceNbrData();
   mfem_mesh_ = &smith::StateManager::setMesh(std::move(meshtmp), mesh_tag_);
+  notifyIfRankHasNoElements();
   createDomains();
 }
 
+void Mesh::notifyIfRankHasNoElements() const
+{
+  int myRank, numRanks;
+  MPI_Comm_rank(getComm(), &myRank);
+  MPI_Comm_size(getComm(), &numRanks);
+  int rankHasNoElements = (0 == mfem_mesh_->GetNE()) ? 1 : 0;
+  ::std::vector<int> gatheredRankData((0 == myRank) ? static_cast<::std::size_t>(numRanks) : 0);
+  MPI_Gather(&rankHasNoElements, 1, MPI_INT, gatheredRankData.data(), 1, MPI_INT, 0, getComm());
+  if (0 == myRank) {
+    ::std::vector<::std::size_t> ranksWithNoElements;
+    for (::std::size_t i = 0; i < static_cast<::std::size_t>(numRanks); ++i) {
+      if (1 == gatheredRankData[i]) {
+        ranksWithNoElements.push_back(i);
+      }
+    }
+    if (0 < ranksWithNoElements.size()) {
+      ::std::ostringstream msg;
+      msg << "\nAfter refining and distributing mesh, the following " << ranksWithNoElements.size()
+          << " ranks own zero elements:\n[";
+      for (const ::std::size_t& rank : ranksWithNoElements) {
+        msg << "  " << rank;
+      }
+      msg << "]\n";
+      SLIC_INFO(msg.str());
+    }
+  }
+  ::axom::slic::flushStreams();
+}
+
 MPI_Comm Mesh::getComm() const { return mfem_mesh_->GetComm(); }
 
 void Mesh::createDomains()

src/smith/physics/mesh.hpp

@@ -129,6 +129,10 @@ class Mesh {
   smith::FiniteElementDual newShapeDisplacementDual();
 
  private:
+  /// @brief Helper function used to notify user if the size of the mesh on any local rank is 0.
+  /// This function is MPI collective and the notice is issued only on rank 0.
+  void notifyIfRankHasNoElements() const;
+
   /// @brief Sets up some initial domains: entire domain, entire boundary, and interior faces. Eventually we can read
   /// off names/blocks/attributes from the mesh and create default domains.
   void createDomains();