From 5a8aa8b9ccfc7f84a64ffb64477ba248f4f38a23 Mon Sep 17 00:00:00 2001 From: Josh Cunningham Date: Tue, 2 Jul 2024 16:48:20 -0400 Subject: [PATCH 1/2] Update NGen.cpp Remove MPI_BARRIERs --- src/NGen.cpp | 55 +++++++++++++++++++++------------------------------- 1 file changed, 22 insertions(+), 33 deletions(-) diff --git a/src/NGen.cpp b/src/NGen.cpp index 1c0cf2bf28..8b389c8bfe 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -558,57 +558,46 @@ int main(int argc, char *argv[]) { } //done time #if NGEN_WITH_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - - if (mpi_rank == 0) + MPI_Finalize(); + if (mpi_rank != 0) { - std::cout << "Finished " << manager->Simulation_Time_Object->get_total_output_times() << " timesteps." << std::endl; + return 0; } +#endif + + std::cout << "Finished " << manager->Simulation_Time_Object->get_total_output_times() << " timesteps." << std::endl; auto time_done_simulation = std::chrono::steady_clock::now(); std::chrono::duration time_elapsed_simulation = time_done_simulation - time_done_init; -#if NGEN_WITH_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - #if NGEN_WITH_ROUTING - if (mpi_rank == 0) - { // Run t-route from single process - if(manager->get_using_routing()) { - //Note: Currently, delta_time is set in the t-route yaml configuration file, and the - //number_of_timesteps is determined from the total number of nexus outputs in t-route. - //It is recommended to still pass these values to the routing_py_adapter object in - //case a future implmentation needs these two values from the ngen framework. - int number_of_timesteps = manager->Simulation_Time_Object->get_total_output_times(); - - int delta_time = manager->Simulation_Time_Object->get_output_interval_seconds(); - - router->route(number_of_timesteps, delta_time); - } + // Run t-route from single process only rank zero reaches this point + if(manager->get_using_routing()) { + //Note: Currently, delta_time is set in the t-route yaml configuration file, and the + //number_of_timesteps is determined from the total number of nexus outputs in t-route. + //It is recommended to still pass these values to the routing_py_adapter object in + //case a future implmentation needs these two values from the ngen framework. + int number_of_timesteps = manager->Simulation_Time_Object->get_total_output_times(); + + int delta_time = manager->Simulation_Time_Object->get_output_interval_seconds(); + + router->route(number_of_timesteps, delta_time); } #endif auto time_done_routing = std::chrono::steady_clock::now(); std::chrono::duration time_elapsed_routing = time_done_routing - time_done_simulation; - if (mpi_rank == 0) - { - std::cout << "NGen top-level timings:" - << "\n\tNGen::init: " << time_elapsed_init.count() - << "\n\tNGen::simulation: " << time_elapsed_simulation.count() + std::cout << "NGen top-level timings:" + << "\n\tNGen::init: " << time_elapsed_init.count() + << "\n\tNGen::simulation: " << time_elapsed_simulation.count() #if NGEN_WITH_ROUTING - << "\n\tNGen::routing: " << time_elapsed_routing.count() + << "\n\tNGen::routing: " << time_elapsed_routing.count() #endif - << std::endl; - } + << std::endl; manager->finalize(); -#if NGEN_WITH_MPI - MPI_Finalize(); -#endif return 0; } From f4025a22e063fe2b741027e21cb22aa5a52f13a1 Mon Sep 17 00:00:00 2001 From: Josh Cunningham Date: Thu, 18 Jul 2024 17:34:34 -0500 Subject: [PATCH 2/2] Manually sleep mpi threads before finalize --- src/NGen.cpp | 69 +++++++++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 31 deletions(-) diff --git a/src/NGen.cpp b/src/NGen.cpp index 8b389c8bfe..820cf16b8a 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -558,46 +558,53 @@ int main(int argc, char *argv[]) { } //done time #if NGEN_WITH_MPI - MPI_Finalize(); - if (mpi_rank != 0) - { - return 0; - } + MPI_Barrier(MPI_COMM_WORLD); + // Rank 0 continues with routing and enters barrier after #endif + if (mpi_rank == 0) { + std::cout << "Finished " << manager->Simulation_Time_Object->get_total_output_times() << " timesteps." << std::endl; - std::cout << "Finished " << manager->Simulation_Time_Object->get_total_output_times() << " timesteps." << std::endl; - - auto time_done_simulation = std::chrono::steady_clock::now(); - std::chrono::duration time_elapsed_simulation = time_done_simulation - time_done_init; + auto time_done_simulation = std::chrono::steady_clock::now(); + std::chrono::duration time_elapsed_simulation = time_done_simulation - time_done_init; #if NGEN_WITH_ROUTING - // Run t-route from single process only rank zero reaches this point - if(manager->get_using_routing()) { - //Note: Currently, delta_time is set in the t-route yaml configuration file, and the - //number_of_timesteps is determined from the total number of nexus outputs in t-route. - //It is recommended to still pass these values to the routing_py_adapter object in - //case a future implmentation needs these two values from the ngen framework. - int number_of_timesteps = manager->Simulation_Time_Object->get_total_output_times(); - - int delta_time = manager->Simulation_Time_Object->get_output_interval_seconds(); - - router->route(number_of_timesteps, delta_time); - } + // Run t-route from single process only rank zero reaches this point + if(manager->get_using_routing()) { + //Note: Currently, delta_time is set in the t-route yaml configuration file, and the + //number_of_timesteps is determined from the total number of nexus outputs in t-route. + //It is recommended to still pass these values to the routing_py_adapter object in + //case a future implmentation needs these two values from the ngen framework. + int number_of_timesteps = manager->Simulation_Time_Object->get_total_output_times(); + int delta_time = manager->Simulation_Time_Object->get_output_interval_seconds(); + router->route(number_of_timesteps, delta_time); + } #endif - auto time_done_routing = std::chrono::steady_clock::now(); - std::chrono::duration time_elapsed_routing = time_done_routing - time_done_simulation; + auto time_done_routing = std::chrono::steady_clock::now(); + std::chrono::duration time_elapsed_routing = time_done_routing - time_done_simulation; - std::cout << "NGen top-level timings:" - << "\n\tNGen::init: " << time_elapsed_init.count() - << "\n\tNGen::simulation: " << time_elapsed_simulation.count() + std::cout << "NGen top-level timings:" + << "\n\tNGen::init: " << time_elapsed_init.count() + << "\n\tNGen::simulation: " << time_elapsed_simulation.count() #if NGEN_WITH_ROUTING - << "\n\tNGen::routing: " << time_elapsed_routing.count() + << "\n\tNGen::routing: " << time_elapsed_routing.count() #endif - << std::endl; - - manager->finalize(); - + << std::endl; + } + +#if NGEN_WITH_MPI + MPI_Request barrier_request; + MPI_Ibarrier(MPI_COMM_WORLD, &barrier_request); + int flag = 0; + while (!flag) { + MPI_Test(&barrier_request, &flag, MPI_STATUS_IGNORE); + if (!flag) { + sleep(0.1); + } + } + manager->finalize(); + MPI_Finalize(); +#endif return 0; }