Consolidating OpenACC device-host memory transfers

This PR consolidates much of the OpenACC host and device data transfers during
the course of the dynamical execution to two subroutines mpas_atm_pre_dynamics
_h2d and mpas_atm_post_dynamics_d2h that are called before and after the call
to atm_srk3 subroutine. Due to atm_compute_solve_diagnostics also being called
once before the start of model run, we also have a pair of subroutines mpas_atm
_pre_computesolvediag_h2d and mpas_atm_post_computesolvediag_d2h to handle data
movements around the first call to atm_compute_solve_diagnostics. Any fields
copied onto the device in these subroutines are removed from explicit data
movement statements in the dynamical core.

The mesh/time-invariant fields are still copied onto the device in mpas_atm_
dynamics_init and removed from the device in mpas_atm_dynamics_finalize, with
the exception of select fields moved in mpas_atm_pre_computesolvediag_h2d and
mpas_atm_post_computesolvediag_d2h. This is a special case due to atm_compute_
solve_diagnostics being called for the first time before the call to mpas_atm_
dynamics_init

This PR also includes explicit host-device data transfers in the mpas_atm_iau,
mpas_atmphys_interface and mpas_atmphys_todynamics modules to ensure that the
physics and IAU regions, which run on CPU, use the latest values from the
dynamical core running on GPUs, and vice versa. In addition, this PR also
includes explicit data transfers around halo exchanges in the atm_srk3
subroutine.

These subroutines for data routines, and the acc update statements are an
interim solution until we have a book-keeping method in place. This PR also
introduces a couple of new timers to keep track of the cost of data transfers.

Author: abishekg7

src/core_atmosphere/dynamics/mpas_atm_boundaries.F

@@ -310,18 +310,14 @@ subroutine mpas_atm_get_bdy_tend(clock, block, vertDim, horizDim, field, delta_t
         nullify(tend)
         call mpas_pool_get_array(lbc, 'lbc_'//trim(field), tend, 1)
 
-        MPAS_ACC_TIMER_START('mpas_atm_get_bdy_tend [ACC_data_xfer]')
         if (associated(tend)) then
-            !$acc enter data copyin(tend)
         else
             call mpas_pool_get_array(lbc, 'lbc_scalars', tend_scalars, 1)
-            !$acc enter data copyin(tend_scalars)
 
             ! Ensure the integer pointed to by idx_ptr is copied to the gpu device
             call mpas_pool_get_dimension(lbc, 'index_'//trim(field), idx_ptr)
             idx = idx_ptr
         end if
-        MPAS_ACC_TIMER_STOP('mpas_atm_get_bdy_tend [ACC_data_xfer]')
 
         !$acc parallel default(present)
         if (associated(tend)) then
@@ -341,13 +337,6 @@ subroutine mpas_atm_get_bdy_tend(clock, block, vertDim, horizDim, field, delta_t
         end if
         !$acc end parallel
 
-        MPAS_ACC_TIMER_START('mpas_atm_get_bdy_tend [ACC_data_xfer]')
-        if (associated(tend)) then
-            !$acc exit data delete(tend)
-        else
-            !$acc exit data delete(tend_scalars)
-        end if
-        MPAS_ACC_TIMER_STOP('mpas_atm_get_bdy_tend [ACC_data_xfer]')
 
     end subroutine mpas_atm_get_bdy_tend
 
@@ -448,9 +437,6 @@ subroutine mpas_atm_get_bdy_state_2d(clock, block, vertDim, horizDim, field, del
         ! query the field as a scalar constituent
         !
         if (associated(tend) .and. associated(state)) then
-            MPAS_ACC_TIMER_START('mpas_atm_get_bdy_state_2d [ACC_data_xfer]')
-            !$acc enter data copyin(tend, state)
-            MPAS_ACC_TIMER_STOP('mpas_atm_get_bdy_state_2d [ACC_data_xfer]')
 
             !$acc parallel default(present)
             !$acc loop gang vector collapse(2)
@@ -461,20 +447,13 @@ subroutine mpas_atm_get_bdy_state_2d(clock, block, vertDim, horizDim, field, del
             end do
             !$acc end parallel
 
-            MPAS_ACC_TIMER_START('mpas_atm_get_bdy_state_2d [ACC_data_xfer]')
-            !$acc exit data delete(tend, state)
-            MPAS_ACC_TIMER_STOP('mpas_atm_get_bdy_state_2d [ACC_data_xfer]')
         else
             call mpas_pool_get_array(lbc, 'lbc_scalars', tend_scalars, 1)
             call mpas_pool_get_array(lbc, 'lbc_scalars', state_scalars, 2)
             call mpas_pool_get_dimension(lbc, 'index_'//trim(field), idx_ptr)
 
             idx=idx_ptr ! Avoid non-array pointer for OpenACC
 
-            MPAS_ACC_TIMER_START('mpas_atm_get_bdy_state_2d [ACC_data_xfer]')
-            !$acc enter data copyin(tend_scalars, state_scalars)
-            MPAS_ACC_TIMER_STOP('mpas_atm_get_bdy_state_2d [ACC_data_xfer]')
-
             !$acc parallel default(present)
             !$acc loop gang vector collapse(2)
             do i=1, horizDim+1
@@ -484,9 +463,6 @@ subroutine mpas_atm_get_bdy_state_2d(clock, block, vertDim, horizDim, field, del
             end do
             !$acc end parallel
 
-            MPAS_ACC_TIMER_START('mpas_atm_get_bdy_state_2d [ACC_data_xfer]')
-            !$acc exit data delete(tend_scalars, state_scalars)
-            MPAS_ACC_TIMER_STOP('mpas_atm_get_bdy_state_2d [ACC_data_xfer]')
         end if
 
     end subroutine mpas_atm_get_bdy_state_2d
@@ -567,10 +543,6 @@ subroutine mpas_atm_get_bdy_state_3d(clock, block, innerDim, vertDim, horizDim,
         call mpas_pool_get_array(lbc, 'lbc_'//trim(field), tend, 1)
         call mpas_pool_get_array(lbc, 'lbc_'//trim(field), state, 2)
 
-        MPAS_ACC_TIMER_START('mpas_atm_get_bdy_state_3d [ACC_data_xfer]')
-        !$acc enter data copyin(tend, state)
-        MPAS_ACC_TIMER_STOP('mpas_atm_get_bdy_state_3d [ACC_data_xfer]')
-
         !$acc parallel default(present)
         !$acc loop gang vector collapse(3)
         do i=1, horizDim+1
@@ -582,10 +554,6 @@ subroutine mpas_atm_get_bdy_state_3d(clock, block, innerDim, vertDim, horizDim,
         end do
         !$acc end parallel
 
-        MPAS_ACC_TIMER_START('mpas_atm_get_bdy_state_3d [ACC_data_xfer]')
-        !$acc exit data delete(tend, state)
-        MPAS_ACC_TIMER_STOP('mpas_atm_get_bdy_state_3d [ACC_data_xfer]')
-
     end subroutine mpas_atm_get_bdy_state_3d
 
 

src/core_atmosphere/dynamics/mpas_atm_iau.F

@@ -13,9 +13,20 @@ module mpas_atm_iau
  use mpas_dmpar
  use mpas_constants   
  use mpas_log, only : mpas_log_write
+ use mpas_timer
 
  !public :: atm_compute_iau_coef, atm_add_tend_anal_incr  
 
+
+ #ifdef MPAS_OPENACC
+ #define MPAS_ACC_TIMER_START(X) call mpas_timer_start(X)
+ #define MPAS_ACC_TIMER_STOP(X) call mpas_timer_stop(X)
+ #else
+ #define MPAS_ACC_TIMER_START(X)
+ #define MPAS_ACC_TIMER_STOP(X)
+ #endif
+ 
+
  contains
 
 !==================================================================================================
@@ -137,6 +148,7 @@ subroutine atm_add_tend_anal_incr (configs, structs, itimestep, dt, tend_ru, ten
       call mpas_pool_get_array(state, 'scalars', scalars, 1)
       call mpas_pool_get_array(state, 'rho_zz',   rho_zz, 2) 
       call mpas_pool_get_array(diag , 'rho_edge', rho_edge)
+      !$acc update self(theta_m, scalars, rho_zz, rho_edge)
 
       call mpas_pool_get_dimension(state, 'moist_start', moist_start)
       call mpas_pool_get_dimension(state, 'moist_end', moist_end)
@@ -149,6 +161,8 @@ subroutine atm_add_tend_anal_incr (configs, structs, itimestep, dt, tend_ru, ten
 !      call mpas_pool_get_array(tend, 'rho_zz',  tend_rho)
 !      call mpas_pool_get_array(tend, 'theta_m', tend_theta)
       call mpas_pool_get_array(tend, 'scalars_tend', tend_scalars)
+      !$acc update self(tend_scalars)
+      MPAS_ACC_TIMER_STOP('atm_srk3: physics ACC_data_xfer')
 
       call mpas_pool_get_array(tend_iau, 'theta',     theta_amb)
       call mpas_pool_get_array(tend_iau, 'rho',         rho_amb)