From 7f37899ad59ce849ef607199c11721189114fe79 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Wed, 30 Sep 2026 22:13:19 -0400 Subject: [PATCH 1/2] IB: check every patch is marked in one pass and one reduction s_check_every_patch_marked swept every local cell and ran a reduction once per global patch, and num_gbl_ibs counts every particle-cloud particle. Count markers by decoded patch id in one pass and reduce the counts in one collective. Decoding also counts the cells of a body wrapped across a periodic boundary, which the raw == gid test missed. Co-Authored-By: Claude --- src/simulation/m_ibm.fpp | 53 ++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 32 deletions(-) diff --git a/src/simulation/m_ibm.fpp b/src/simulation/m_ibm.fpp index 3b156d608..e2fd658d0 100644 --- a/src/simulation/m_ibm.fpp +++ b/src/simulation/m_ibm.fpp @@ -1795,44 +1795,33 @@ contains end subroutine s_get_neighborhood_idx - !> Abort if any immersed boundary marked no cell anywhere in the domain. - !! - !! A rank is given a patch when the patch CENTROID falls in its share of the domain, but for an STL - !! the centroid and the geometry are independent: the body is placed by model_translate, and a case - !! may legitimately leave the centroid at the origin. Ownership is then decided at a point the body - !! does not occupy, and the owning rank marks only whatever of the geometry its own subdomain happens - !! to reach. That shrinks as the decomposition is refined, so a body can erode and finally vanish -- - !! contributing no markers, no ghost points, and a force of exactly zero every step -- with nothing - !! reported. Marker generation is a pure function of geometry and grid, so a result that depends on - !! the rank count is always wrong. - !! - !! Measured on a two-body case, identical deck, only the rank count changed, with a patch at the - !! origin as a control: the control held 30191 cells at 64, 128 and 512 ranks, while a patch eight - !! chords away went 19115 -> 12667 -> 0. - !! - !! One reduction per patch at setup. It cannot see partial erosion -- that needs the marked volume, - !! which is not available here -- but it turns the total loss into an immediate, specific error. + !> Abort if any immersed boundary marks no cell anywhere. The owning rank is chosen by the patch centroid, so an STL placed with + !! model_translate while its centroid is left elsewhere is handed to ranks that do not hold it, and it vanishes silently. One + !! pass over the cells and one reduction, whatever the number of patches. impure subroutine s_check_every_patch_marked() - integer(kind=8) :: cnt_loc, cnt_glb - integer :: gid, i, j, k + real(wp), allocatable :: cnt_loc(:,:), cnt_glb(:,:) + integer :: gid, i, j, k - do gid = 1, num_gbl_ibs - cnt_loc = 0_8 - do k = 0, p - do j = 0, n - do i = 0, m - if (ib_markers%sf(i, j, k) == gid) cnt_loc = cnt_loc + 1_8 - end do + if (num_gbl_ibs == 0) return + allocate (cnt_loc(num_gbl_ibs, 1), cnt_glb(num_gbl_ibs, 1)) + cnt_loc = 0._wp + do k = 0, p + do j = 0, n + do i = 0, m + if (ib_markers%sf(i, j, k) /= 0) then + call s_decode_patch_periodicity(ib_markers%sf(i, j, k), gid) + cnt_loc(gid, 1) = cnt_loc(gid, 1) + 1._wp + end if end do end do - cnt_glb = cnt_loc -#ifdef MFC_MPI - if (num_procs > 1) call s_mpi_allreduce_integer_sum(cnt_loc, cnt_glb) -#endif - @:PROHIBIT(cnt_glb == 0_8, & - & "An immersed boundary marked no cell anywhere: its centroid decides which rank "// "owns it, so a body placed elsewhere with model_translate is handed to a rank that "// "does not hold it. Set patch_ib%x/y/z_centroid to where the body actually is.") end do + call s_mpi_allreduce_vectors_sum(cnt_loc, cnt_glb, num_gbl_ibs, 1) + @:PROHIBIT(any(cnt_glb(:, 1) == 0._wp), & + & "An immersed boundary marked no cell anywhere: its centroid decides which rank owns it, so a body placed " & + & // "elsewhere with model_translate is handed to a rank that does not hold it. Set patch_ib%x/y/z_centroid " & + & // "to where the body actually is.") + deallocate (cnt_loc, cnt_glb) end subroutine s_check_every_patch_marked From 97d7106b8e1cdd73f186a2e717afbfc03192bc86 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Wed, 30 Sep 2026 23:17:33 -0400 Subject: [PATCH 2/2] Count marked cells as integers Use 1-D integer(kind=8) counts and a new s_mpi_allreduce_integer_sum_vec (shaped like s_mpi_allreduce_min_vec, with the same non-MPI fallback) instead of real(wp) counts in a (num_gbl_ibs, 1) array, as review suggested. Co-Authored-By: Claude --- src/common/m_mpi_common.fpp | 16 ++++++++++++++++ src/simulation/m_ibm.fpp | 14 +++++++------- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/common/m_mpi_common.fpp b/src/common/m_mpi_common.fpp index 15bec952a..c54568d27 100644 --- a/src/common/m_mpi_common.fpp +++ b/src/common/m_mpi_common.fpp @@ -414,6 +414,22 @@ contains end subroutine s_mpi_allreduce_integer_sum + !> Reduce an integer(8) array to its element-wise global sum across all MPI ranks. + impure subroutine s_mpi_allreduce_integer_sum_vec(var_loc, var_glb) + + integer(kind=8), dimension(:), intent(in) :: var_loc + integer(kind=8), dimension(:), intent(out) :: var_glb + +#ifdef MFC_MPI + integer :: ierr !< Generic flag used to identify and report MPI errors + + call MPI_ALLREDUCE(var_loc, var_glb, size(var_loc), MPI_INTEGER8, MPI_SUM, MPI_COMM_WORLD, ierr) +#else + var_glb = var_loc +#endif + + end subroutine s_mpi_allreduce_integer_sum_vec + !> Reduce a local real value to its global minimum across all MPI ranks. impure subroutine s_mpi_allreduce_min(var_loc, var_glb) diff --git a/src/simulation/m_ibm.fpp b/src/simulation/m_ibm.fpp index e2fd658d0..e3dd08cc5 100644 --- a/src/simulation/m_ibm.fpp +++ b/src/simulation/m_ibm.fpp @@ -1800,24 +1800,24 @@ contains !! pass over the cells and one reduction, whatever the number of patches. impure subroutine s_check_every_patch_marked() - real(wp), allocatable :: cnt_loc(:,:), cnt_glb(:,:) - integer :: gid, i, j, k + integer(kind=8), allocatable :: cnt_loc(:), cnt_glb(:) + integer :: gid, i, j, k if (num_gbl_ibs == 0) return - allocate (cnt_loc(num_gbl_ibs, 1), cnt_glb(num_gbl_ibs, 1)) - cnt_loc = 0._wp + allocate (cnt_loc(num_gbl_ibs), cnt_glb(num_gbl_ibs)) + cnt_loc = 0_8 do k = 0, p do j = 0, n do i = 0, m if (ib_markers%sf(i, j, k) /= 0) then call s_decode_patch_periodicity(ib_markers%sf(i, j, k), gid) - cnt_loc(gid, 1) = cnt_loc(gid, 1) + 1._wp + cnt_loc(gid) = cnt_loc(gid) + 1_8 end if end do end do end do - call s_mpi_allreduce_vectors_sum(cnt_loc, cnt_glb, num_gbl_ibs, 1) - @:PROHIBIT(any(cnt_glb(:, 1) == 0._wp), & + call s_mpi_allreduce_integer_sum_vec(cnt_loc, cnt_glb) + @:PROHIBIT(any(cnt_glb == 0_8), & & "An immersed boundary marked no cell anywhere: its centroid decides which rank owns it, so a body placed " & & // "elsewhere with model_translate is handed to a rank that does not hold it. Set patch_ib%x/y/z_centroid " & & // "to where the body actually is.")