diff --git a/infrastructure/build/tests.mk b/infrastructure/build/tests.mk index 59599a598..fcb7be98e 100644 --- a/infrastructure/build/tests.mk +++ b/infrastructure/build/tests.mk @@ -17,7 +17,7 @@ endif ifdef MPI_TESTS UNIT_TEST_PRE_PROCESS_MACROS = USE_MPI=YES - LAUNCHER = mpiexec -n 4 + LAUNCHER = mpiexec -n 6 else UNIT_TEST_PRE_PROCESS_MACROS = NO_MPI=no_mpi # It seems that the Cray 'ftn' wrapper always builds for MPI... diff --git a/infrastructure/unit-test/mesh/partition_mod_test.pf b/infrastructure/unit-test/mesh/partition_mod_test.pf index bc698f2e7..1a40b4802 100644 --- a/infrastructure/unit-test/mesh/partition_mod_test.pf +++ b/infrastructure/unit-test/mesh/partition_mod_test.pf @@ -18,6 +18,7 @@ module partition_mod_test lfric_comm_type use partition_mod, only: partition_type, & partitioner_planar, & + partitioner_cubedsphere, & partitioner_cubedsphere_serial, & partitioner_interface use reference_element_mod, only: reference_cube_type @@ -85,7 +86,7 @@ contains !> Test partition module functionality !> - @Test(npes=[1, 4] ) + @Test(npes=[1, 3, 4] ) subroutine test_partition_BiPeriodic( this ) implicit none @@ -129,6 +130,9 @@ contains case (1) xproc = 1 yproc = 1 + case (3) + xproc = 3 + yproc = 1 case (4) xproc = 2 yproc = 2 @@ -160,11 +164,13 @@ contains local_rank, & total_ranks ) - ! Test functionality of the partition object we've just created on both 1 + ! Test functionality of the partition object we've just created on 1, 3 ! and 4 processes select case (num_processes) case (1) +! Testing over 1 (8x8) partition. All cells are owned - so no halo/ghost cells + num_cells_in_layer = partition%get_num_cells_in_layer() @assertEqual( 64, num_cells_in_layer ) @@ -195,7 +201,72 @@ contains num_cells_ghost = partition%get_num_cells_ghost() @assertEqual( 0, num_cells_ghost ) + case (3) +! Testing over 3 partitions will produce uneven partitions, PE1 and PE2 will be +! the same (3x8), but PE0 will be slightly smaller (2x8) + +! Start with quantities that will be the same for all PEs + inner_depth = partition%get_inner_depth() + @assertEqual( 2, inner_depth ) + + halo_depth = partition%get_halo_depth() + @assertEqual( 2, halo_depth ) + + num_cells_halo = partition%get_num_cells_halo(1) + @assertEqual( 16, num_cells_halo ) + +! Now check quantities that differ between PE0 and PE1/2 + select case (local_rank) + case(0) + num_cells_in_layer = partition%get_num_cells_in_layer() + @assertEqual( 48, num_cells_in_layer ) + + num_cells_inner = partition%get_num_cells_inner(1) + @assertEqual( 0, num_cells_inner ) + + last_inner_cell = partition%get_last_inner_cell(1) + @assertEqual( 0, last_inner_cell ) + + num_cells_edge = partition%get_num_cells_edge() + @assertEqual( 16, num_cells_edge ) + + last_edge_cell = partition%get_last_edge_cell() + @assertEqual( 16, last_edge_cell ) + + last_halo_cell = partition%get_last_halo_cell(1) + @assertEqual( 32, last_halo_cell ) + + num_cells_ghost = partition%get_num_cells_ghost() + @assertEqual( 16, num_cells_ghost ) + case(1, 2) + num_cells_in_layer = partition%get_num_cells_in_layer() + @assertEqual( 56, num_cells_in_layer ) + + num_cells_inner = partition%get_num_cells_inner(1) + @assertEqual( 6, num_cells_inner ) + + last_inner_cell = partition%get_last_inner_cell(1) + @assertEqual( 6, last_inner_cell ) + + num_cells_edge = partition%get_num_cells_edge() + @assertEqual( 18, num_cells_edge ) + + last_edge_cell = partition%get_last_edge_cell() + @assertEqual( 24, last_edge_cell ) + + num_cells_halo = partition%get_num_cells_halo(1) + @assertEqual( 16, num_cells_halo ) + + last_halo_cell = partition%get_last_halo_cell(1) + @assertEqual( 40, last_halo_cell ) + + num_cells_ghost = partition%get_num_cells_ghost() + @assertEqual( 8, num_cells_ghost ) + end select + case (4) +! Testing over four similar (4x4) partitions + num_cells_in_layer = partition%get_num_cells_in_layer() @assertEqual( 64, num_cells_in_layer ) @@ -230,7 +301,7 @@ contains end subroutine test_partition_BiPeriodic - @Test(npes=[1, 4] ) + @Test(npes=[1, 3, 4] ) subroutine test_partition_planar( this ) implicit none @@ -274,6 +345,9 @@ contains case (1) xproc = 1 yproc = 1 + case (3) + xproc = 3 + yproc = 1 case (4) xproc = 2 yproc = 2 @@ -306,11 +380,12 @@ contains generate_inner_halos, & local_rank, & total_ranks ) - ! Test functionality of the partition object we've just created on both 1 + ! Test functionality of the partition object we've just created on 1, 3 ! and 4 processes select case (num_processes) case (1) +! Testing over 1 (8x8) partition - no halo/ghost cells num_cells_in_layer = partition%get_num_cells_in_layer() @assertEqual( 64, num_cells_in_layer ) @@ -341,7 +416,102 @@ contains num_cells_ghost = partition%get_num_cells_ghost() @assertEqual( 0, num_cells_ghost ) + case (3) +! Testing over 3 partitions will produce uneven partitions, PE1 and PE2 will be +! the same (3x8) (with different halos), but PE0 will be slightly smaller (2x8) + +! Start with quantities that will be the same for all PEs + inner_depth = partition%get_inner_depth() + @assertEqual( 2, inner_depth ) + + halo_depth = partition%get_halo_depth() + @assertEqual( 2, halo_depth ) + +! Now check quantities that differ between the partitions + select case (local_rank) + case(0) + num_cells_in_layer = partition%get_num_cells_in_layer() + @assertEqual( 32, num_cells_in_layer ) + + num_cells_inner = partition%get_num_cells_inner(1) + @assertEqual( 0, num_cells_inner ) + + last_inner_cell = partition%get_last_inner_cell(1) + @assertEqual( 0, last_inner_cell ) + + num_cells_edge = partition%get_num_cells_edge() + @assertEqual( 16, num_cells_edge ) + + last_edge_cell = partition%get_last_edge_cell() + @assertEqual( 16, last_edge_cell ) + + num_cells_halo = partition%get_num_cells_halo(1) + @assertEqual( 8, num_cells_halo ) + + last_halo_cell = partition%get_last_halo_cell(1) + @assertEqual( 24, last_halo_cell ) + + num_cells_ghost = partition%get_num_cells_ghost() + @assertEqual( 8, num_cells_ghost ) + case(1) + num_cells_in_layer = partition%get_num_cells_in_layer() + @assertEqual( 56, num_cells_in_layer ) + + num_cells_inner = partition%get_num_cells_inner(1) + @assertEqual( 6, num_cells_inner ) + + last_inner_cell = partition%get_last_inner_cell(1) + @assertEqual( 6, last_inner_cell ) + + num_cells_edge = partition%get_num_cells_edge() + @assertEqual( 18, num_cells_edge ) + + last_edge_cell = partition%get_last_edge_cell() + @assertEqual( 24, last_edge_cell ) + + num_cells_halo = partition%get_num_cells_halo(1) + @assertEqual( 16, num_cells_halo ) + + num_cells_halo = partition%get_num_cells_halo(1) + @assertEqual( 16, num_cells_halo ) + + last_halo_cell = partition%get_last_halo_cell(1) + @assertEqual( 40, last_halo_cell ) + + num_cells_ghost = partition%get_num_cells_ghost() + @assertEqual( 8, num_cells_ghost ) + case(2) + num_cells_in_layer = partition%get_num_cells_in_layer() + @assertEqual( 40, num_cells_in_layer ) + + num_cells_inner = partition%get_num_cells_inner(1) + @assertEqual( 6, num_cells_inner ) + + last_inner_cell = partition%get_last_inner_cell(1) + @assertEqual( 6, last_inner_cell ) + + num_cells_edge = partition%get_num_cells_edge() + @assertEqual( 18, num_cells_edge ) + + last_edge_cell = partition%get_last_edge_cell() + @assertEqual( 24, last_edge_cell ) + + num_cells_halo = partition%get_num_cells_halo(1) + @assertEqual( 8, num_cells_halo ) + + num_cells_halo = partition%get_num_cells_halo(1) + @assertEqual( 8, num_cells_halo ) + + last_halo_cell = partition%get_last_halo_cell(1) + @assertEqual( 32, last_halo_cell ) + + num_cells_ghost = partition%get_num_cells_ghost() + @assertEqual( 8, num_cells_ghost ) + end select + case (4) +! Testing over four similar (4x4) partitions + num_cells_in_layer = partition%get_num_cells_in_layer() @assertEqual( 36, num_cells_in_layer ) @@ -376,7 +546,7 @@ contains end subroutine test_partition_planar - @Test(npes=[1, 4] ) + @Test(npes=[1, 3, 4] ) subroutine test_partition_trench_x( this ) implicit none @@ -420,6 +590,9 @@ contains case (1) xproc = 1 yproc = 1 + case (3) + xproc = 3 + yproc = 1 case (4) xproc = 2 yproc = 2 @@ -452,11 +625,13 @@ contains generate_inner_halos, & local_rank, & total_ranks ) - ! Test functionality of the partition object we've just created on both 1 + ! Test functionality of the partition object we've just created on 1, 3 ! and 4 processes select case (num_processes) case (1) +! Testing over 1 (8x8) partition. All cells are owned - so no halo/ghost cells + num_cells_in_layer = partition%get_num_cells_in_layer() @assertEqual( 64, num_cells_in_layer ) @@ -487,7 +662,72 @@ contains num_cells_ghost = partition%get_num_cells_ghost() @assertEqual( 0, num_cells_ghost ) + case (3) +! Testing over 3 partitions will produce uneven partitions, PE1 and PE2 will be +! the same (3x8), but PE0 will be slightly smaller (2x8) + +! Start with quantities that will be the same for all PEs + inner_depth = partition%get_inner_depth() + @assertEqual( 2, inner_depth ) + + halo_depth = partition%get_halo_depth() + @assertEqual( 2, halo_depth ) + + num_cells_halo = partition%get_num_cells_halo(1) + @assertEqual( 16, num_cells_halo ) + +! Now check quantities that differ between PE0 and PE1/2 + select case (local_rank) + case(0) + num_cells_in_layer = partition%get_num_cells_in_layer() + @assertEqual( 48, num_cells_in_layer ) + + num_cells_inner = partition%get_num_cells_inner(1) + @assertEqual( 0, num_cells_inner ) + + last_inner_cell = partition%get_last_inner_cell(1) + @assertEqual( 0, last_inner_cell ) + + num_cells_edge = partition%get_num_cells_edge() + @assertEqual( 16, num_cells_edge ) + + last_edge_cell = partition%get_last_edge_cell() + @assertEqual( 16, last_edge_cell ) + + last_halo_cell = partition%get_last_halo_cell(1) + @assertEqual( 32, last_halo_cell ) + + num_cells_ghost = partition%get_num_cells_ghost() + @assertEqual( 16, num_cells_ghost ) + case(1, 2) + num_cells_in_layer = partition%get_num_cells_in_layer() + @assertEqual( 56, num_cells_in_layer ) + + num_cells_inner = partition%get_num_cells_inner(1) + @assertEqual( 6, num_cells_inner ) + + last_inner_cell = partition%get_last_inner_cell(1) + @assertEqual( 6, last_inner_cell ) + + num_cells_edge = partition%get_num_cells_edge() + @assertEqual( 18, num_cells_edge ) + + last_edge_cell = partition%get_last_edge_cell() + @assertEqual( 24, last_edge_cell ) + + num_cells_halo = partition%get_num_cells_halo(1) + @assertEqual( 16, num_cells_halo ) + + last_halo_cell = partition%get_last_halo_cell(1) + @assertEqual( 40, last_halo_cell ) + + num_cells_ghost = partition%get_num_cells_ghost() + @assertEqual( 8, num_cells_ghost ) + end select + case (4) +! Testing over four similar (4x4) partitions + num_cells_in_layer = partition%get_num_cells_in_layer() @assertEqual( 48, num_cells_in_layer ) @@ -566,6 +806,9 @@ contains case (1) xproc = 1 yproc = 1 + case (3) + xproc = 3 + yproc = 1 case (4) xproc = 2 yproc = 2 @@ -598,11 +841,13 @@ contains generate_inner_halos, & local_rank, & total_ranks ) - ! Test functionality of the partition object we've just created on both 1 + ! Test functionality of the partition object we've just created on 1, 3 ! and 4 processes select case (num_processes) case (1) +! Testing over 1 (8x8) partition - no halo/ghost cells + num_cells_in_layer = partition%get_num_cells_in_layer() @assertEqual( 64, num_cells_in_layer ) @@ -633,7 +878,102 @@ contains num_cells_ghost = partition%get_num_cells_ghost() @assertEqual( 0, num_cells_ghost ) + case (3) +! Testing over 3 partitions will produce uneven partitions, PE1 and PE2 will be +! the same (3x8) (with different halos), but PE0 will be slightly smaller (2x8) + +! Start with quantities that will be the same for all PEs + inner_depth = partition%get_inner_depth() + @assertEqual( 2, inner_depth ) + + halo_depth = partition%get_halo_depth() + @assertEqual( 2, halo_depth ) + +! Now check quantities that differ between the partitions + select case (local_rank) + case(0) + num_cells_in_layer = partition%get_num_cells_in_layer() + @assertEqual( 32, num_cells_in_layer ) + + num_cells_inner = partition%get_num_cells_inner(1) + @assertEqual( 0, num_cells_inner ) + + last_inner_cell = partition%get_last_inner_cell(1) + @assertEqual( 0, last_inner_cell ) + + num_cells_edge = partition%get_num_cells_edge() + @assertEqual( 16, num_cells_edge ) + + last_edge_cell = partition%get_last_edge_cell() + @assertEqual( 16, last_edge_cell ) + + num_cells_halo = partition%get_num_cells_halo(1) + @assertEqual( 8, num_cells_halo ) + + last_halo_cell = partition%get_last_halo_cell(1) + @assertEqual( 24, last_halo_cell ) + + num_cells_ghost = partition%get_num_cells_ghost() + @assertEqual( 8, num_cells_ghost ) + case(1) + num_cells_in_layer = partition%get_num_cells_in_layer() + @assertEqual( 56, num_cells_in_layer ) + + num_cells_inner = partition%get_num_cells_inner(1) + @assertEqual( 6, num_cells_inner ) + + last_inner_cell = partition%get_last_inner_cell(1) + @assertEqual( 6, last_inner_cell ) + + num_cells_edge = partition%get_num_cells_edge() + @assertEqual( 18, num_cells_edge ) + + last_edge_cell = partition%get_last_edge_cell() + @assertEqual( 24, last_edge_cell ) + + num_cells_halo = partition%get_num_cells_halo(1) + @assertEqual( 16, num_cells_halo ) + + num_cells_halo = partition%get_num_cells_halo(1) + @assertEqual( 16, num_cells_halo ) + + last_halo_cell = partition%get_last_halo_cell(1) + @assertEqual( 40, last_halo_cell ) + + num_cells_ghost = partition%get_num_cells_ghost() + @assertEqual( 8, num_cells_ghost ) + case(2) + num_cells_in_layer = partition%get_num_cells_in_layer() + @assertEqual( 40, num_cells_in_layer ) + + num_cells_inner = partition%get_num_cells_inner(1) + @assertEqual( 6, num_cells_inner ) + + last_inner_cell = partition%get_last_inner_cell(1) + @assertEqual( 6, last_inner_cell ) + + num_cells_edge = partition%get_num_cells_edge() + @assertEqual( 18, num_cells_edge ) + + last_edge_cell = partition%get_last_edge_cell() + @assertEqual( 24, last_edge_cell ) + + num_cells_halo = partition%get_num_cells_halo(1) + @assertEqual( 8, num_cells_halo ) + + num_cells_halo = partition%get_num_cells_halo(1) + @assertEqual( 8, num_cells_halo ) + + last_halo_cell = partition%get_last_halo_cell(1) + @assertEqual( 32, last_halo_cell ) + + num_cells_ghost = partition%get_num_cells_ghost() + @assertEqual( 8, num_cells_ghost ) + end select + case (4) +! Testing over four similar (4x4) partitions + num_cells_in_layer = partition%get_num_cells_in_layer() @assertEqual( 48, num_cells_in_layer ) @@ -671,7 +1011,7 @@ contains !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! Test partition_cubedsphere_serial (on a single process) ! - @Test( npes=[1] ) + @Test( npes=[1,6] ) subroutine test_partition_CubedSphere( this ) implicit none @@ -714,6 +1054,7 @@ contains local_rank = this%getProcessRank() total_ranks = this%getNumProcesses() max_stencil_depth = 2 + decomposition = custom_decomposition_type(xproc, yproc) generate_inner_halos = .true. @@ -725,77 +1066,97 @@ contains call global_mesh_collection%add_new_global_mesh( global_mesh ) global_mesh_ptr => global_mesh_collection%get_global_mesh( global_mesh_id ) - partitioner_ptr => partitioner_cubedsphere_serial - decomposition = custom_decomposition_type(xproc, yproc) + ! Test functionality of the partition object we've just created on 1 + ! and 6 processes + select case (total_ranks) - partition = partition_type( global_mesh_ptr, & - partitioner_ptr, & - decomposition, & - max_stencil_depth, & - generate_inner_halos, & - local_rank, & - total_ranks ) - ! - ! Test functionality of the partition object we've just created - num_cells_in_layer = partition%get_num_cells_in_layer() - @assertEqual( 96, num_cells_in_layer ) + case (1) +! Testing over one (4x4x6) partition - no halo/ghost cells + + partitioner_ptr => partitioner_cubedsphere_serial + partition = partition_type( global_mesh_ptr, & + partitioner_ptr, & + decomposition, & + max_stencil_depth, & + generate_inner_halos, & + local_rank, & + total_ranks ) + + ! Test functionality of the single partition object we've just created + num_cells_in_layer = partition%get_num_cells_in_layer() + @assertEqual( 96, num_cells_in_layer ) - inner_depth = partition%get_inner_depth() - @assertEqual( 2, inner_depth ) + inner_depth = partition%get_inner_depth() + @assertEqual( 2, inner_depth ) - num_cells_inner = partition%get_num_cells_inner(2) - @assertEqual( 96, num_cells_inner ) + num_cells_inner = partition%get_num_cells_inner(2) + @assertEqual( 96, num_cells_inner ) - last_inner_cell = partition%get_last_inner_cell(1) - @assertEqual( 96, last_inner_cell ) + last_inner_cell = partition%get_last_inner_cell(1) + @assertEqual( 96, last_inner_cell ) - num_cells_edge = partition%get_num_cells_edge() - @assertEqual( 0, num_cells_edge ) + num_cells_edge = partition%get_num_cells_edge() + @assertEqual( 0, num_cells_edge ) - last_edge_cell = partition%get_last_edge_cell() - @assertEqual( 96, last_edge_cell ) + last_edge_cell = partition%get_last_edge_cell() + @assertEqual( 96, last_edge_cell ) + + halo_depth = partition%get_halo_depth() + @assertEqual( 2, halo_depth ) + + num_cells_halo = partition%get_num_cells_halo(1) + @assertEqual( 0, num_cells_halo ) - halo_depth = partition%get_halo_depth() - @assertEqual( 2, halo_depth ) + last_halo_cell = partition%get_last_halo_cell(1) + @assertEqual( 96, last_halo_cell ) - num_cells_halo = partition%get_num_cells_halo(1) - @assertEqual( 0, num_cells_halo ) + num_cells_ghost = partition%get_num_cells_ghost() + @assertEqual( 0, num_cells_ghost ) + case (6) +! Testing over six (4x4) partitions - one per face of the cubedsphere - last_halo_cell = partition%get_last_halo_cell(1) - @assertEqual( 96, last_halo_cell ) + partitioner_ptr => partitioner_cubedsphere + partition = partition_type( global_mesh_ptr, & + partitioner_ptr, & + decomposition, & + max_stencil_depth, & + generate_inner_halos, & + local_rank, & + total_ranks ) - num_cells_ghost = partition%get_num_cells_ghost() - @assertEqual( 0, num_cells_ghost ) + ! Test functionality of the partition objects we've just created + num_cells_in_layer = partition%get_num_cells_in_layer() + @assertEqual( 48, num_cells_in_layer ) + inner_depth = partition%get_inner_depth() + @assertEqual( 2, inner_depth ) - !> @todo Can't test a parallel cubed-sphere partition at this time, details - !> are described in ticket #1985. - ! - ! !---------------------------------------- - ! !Test partition_cubedsphere (in parallel) - ! !---------------------------------------- - ! xproc = 2 - ! yproc = 2 - ! local_rank = this%getProcessRank() - ! total_ranks = this%getNumProcesses() - ! max_stencil_depth = 2 - ! - ! filename = 'data/ugrid_quads_2d.nc' - ! global_mesh = global_mesh_type( filename ) - ! partitioner_ptr => partitioner_cubedsphere - ! - ! partition = partition_type( global_mesh, & - ! partitioner_ptr, & - ! xproc, & - ! yproc, & - ! max_stencil_depth, & - ! generate_inner_halos, & - ! local_rank, & - ! total_ranks ) - ! ! - ! ! Test functionality of the partition object we've just created - ! ... + num_cells_inner = partition%get_num_cells_inner(1) + @assertEqual( 4, num_cells_inner ) + + last_inner_cell = partition%get_last_inner_cell(1) + @assertEqual( 4, last_inner_cell ) + + num_cells_edge = partition%get_num_cells_edge() + @assertEqual( 12, num_cells_edge ) + + last_edge_cell = partition%get_last_edge_cell() + @assertEqual( 16, last_edge_cell ) + + halo_depth = partition%get_halo_depth() + @assertEqual( 2, halo_depth ) + + num_cells_halo = partition%get_num_cells_halo(1) + @assertEqual( 16, num_cells_halo ) + + last_halo_cell = partition%get_last_halo_cell(1) + @assertEqual( 32, last_halo_cell ) + + num_cells_ghost = partition%get_num_cells_ghost() + @assertEqual( 16, num_cells_ghost ) + + end select end subroutine test_partition_CubedSphere diff --git a/rose-stem/site/meto/common/default_directives.cylc b/rose-stem/site/meto/common/default_directives.cylc index f593ffab3..526a62d30 100644 --- a/rose-stem/site/meto/common/default_directives.cylc +++ b/rose-stem/site/meto/common/default_directives.cylc @@ -21,7 +21,7 @@ "tech-tests": { "tech-tests_wallclock": 15, "tech-tests_memory": [1, "GB"], - "tech-tests_cpus": 1, + "tech-tests_cpus": 6, }, }, "ex1a": { @@ -39,10 +39,10 @@ "tech-tests": { "tech-tests_wallclock": 15, "tech-tests_memory": [6, "GB"], - "tech-tests_cpus": 1, + "tech-tests_cpus": 6, }, }, }) %} -{% do LOG.debug("Finished in site/meto/common/default_directives.cylc") %} \ No newline at end of file +{% do LOG.debug("Finished in site/meto/common/default_directives.cylc") %} diff --git a/rose-stem/templates/default_directives.cylc b/rose-stem/templates/default_directives.cylc index aebc8752d..8b454cc5f 100644 --- a/rose-stem/templates/default_directives.cylc +++ b/rose-stem/templates/default_directives.cylc @@ -16,7 +16,7 @@ "mesh_cpus": 6, "tech-tests_memory": [6, "GB"], "tech-tests_wallclock": 20, - "tech-tests_cpus": 1, + "tech-tests_cpus": 6, } %} {% for item, value in defaults.items() %} @@ -38,4 +38,4 @@ {% endfor %} -{% do LOG.debug("Finished in templates/default_directives.cylc") %} \ No newline at end of file +{% do LOG.debug("Finished in templates/default_directives.cylc") %}