diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ba36b45a8..aefb4c571 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -82,15 +82,15 @@ endif() list(APPEND SRC ${IOSRC}) -if(${CMAKE_Fortran_COMPILER_ID} STREQUAL "PGI" OR - ${CMAKE_Fortran_COMPILER_ID} STREQUAL "NVHPC") - list(APPEND SRC ${CUDASRC}) - list(APPEND BACKENDSRC backend/cuda/backend.f90) -endif() - if(OMP_TGT) list(APPEND SRC ${OMPTGTSRC}) list(APPEND BACKENDSRC backend/omp/target/backend.f90) +else() + if(${CMAKE_Fortran_COMPILER_ID} STREQUAL "PGI" OR + ${CMAKE_Fortran_COMPILER_ID} STREQUAL "NVHPC") + list(APPEND SRC ${CUDASRC}) + list(APPEND BACKENDSRC backend/cuda/backend.f90) + endif() endif() if(WITH_2DECOMPFFT) @@ -121,7 +121,12 @@ target_link_libraries(xcompact PRIVATE x3d2) # if CUDA compiler if(${CMAKE_Fortran_COMPILER_ID} STREQUAL "PGI" OR ${CMAKE_Fortran_COMPILER_ID} STREQUAL "NVHPC") - set(CMAKE_Fortran_FLAGS "-cpp -cuda") + if (NOT OMP_TGT) + set(CMAKE_Fortran_FLAGS "-cpp -cuda") + target_compile_options(xcompact PRIVATE "-DCUDA") + else() + set(CMAKE_Fortran_FLAGS "-cpp") + endif() set(CMAKE_Fortran_FLAGS_DEBUG "-g -O0 -traceback -Mbounds -Mchkptr -Ktrap=fp") # Note: -Ktrap=fp traps FP exceptions process-wide. On some NVIDIA GPUs (for example RTX A4000) # it was giving a divide-by-zero inside cuFFTMp's PTX-JIT compiler during Poisson FFT plan creation @@ -130,8 +135,6 @@ if(${CMAKE_Fortran_COMPILER_ID} STREQUAL "PGI" OR target_link_options(x3d2 INTERFACE "-cuda") target_link_options(x3d2 INTERFACE "-cudalib=cufftmp") - target_compile_options(xcompact PRIVATE "-DCUDA") - elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_Fortran_COMPILER_ID} STREQUAL "LLVMFlang" OR ${CMAKE_Fortran_COMPILER_ID} STREQUAL "Flang") diff --git a/src/backend/omp/target/allocator.f90 b/src/backend/omp/target/allocator.f90 index 3b2d1979b..317a1c16c 100644 --- a/src/backend/omp/target/allocator.f90 +++ b/src/backend/omp/target/allocator.f90 @@ -33,14 +33,15 @@ module m_omptgt_allocator type, extends(field_t) :: omptgt_field_t ! A device-resident field + integer, private :: n + integer, dimension(3), private :: dims integer, private :: dev_id type(c_ptr), private :: dev_ptr = c_null_ptr - real(dp), pointer, private :: p_data_tgt(:) => null() - real(dp), pointer, contiguous :: data_tgt(:, :, :) => null() contains procedure :: fill => fill_omptgt procedure :: get_shape => get_shape_omptgt procedure :: set_shape => set_shape_omptgt + procedure :: get_dev_ptr final :: omptgt_field_destroy end type omptgt_field_t @@ -87,17 +88,15 @@ subroutine omptgt_field_init(ngrid, next, id, f) error stop "Device ID is HOST" end if - f%dev_ptr = omp_target_alloc(ngrid*c_sizeof(0.0_dp), f%dev_id) - call c_f_pointer(f%dev_ptr, f%p_data_tgt, shape=[ngrid]) + f%n = ngrid + f%dims = -1 + f%dev_ptr = omp_target_alloc(f%n*c_sizeof(0.0_dp), f%dev_id) end subroutine omptgt_field_init subroutine omptgt_field_destroy(self) type(omptgt_field_t) :: self - nullify (self%data_tgt) - nullify (self%p_data_tgt) - if (c_associated(self%dev_ptr)) then call omp_target_free(self%dev_ptr, self%dev_id) end if @@ -107,60 +106,77 @@ subroutine fill_omptgt(self, c) class(omptgt_field_t) :: self real(dp), intent(in) :: c - !call fill_omptgt_(self%p_data_tgt, c, size(self%p_data_tgt)) - call fill_omptgt_3d_(self%data_tgt, c) + call fill_omptgt_(self%dev_ptr, c, self%n) + !call fill_omptgt_3d_(self%dev_ptr, c) end subroutine fill_omptgt - subroutine fill_omptgt_(p_data_tgt, c, n) - real(dp), dimension(:), intent(inout) :: p_data_tgt + subroutine fill_omptgt_(dev_ptr, c, n) + type(c_ptr), intent(inout) :: dev_ptr real(dp), intent(in) :: c integer, intent(in) :: n + real(dp), dimension(:), pointer :: p_data_tgt integer :: i - !$omp target teams loop has_device_addr(p_data_tgt) + !$omp target teams is_device_ptr(dev_ptr) + call c_f_pointer(dev_ptr, p_data_tgt, shape=[n]) + !$omp loop do i = 1, n p_data_tgt(i) = c end do - !$omp end target teams loop + !$omp end loop + !$omp end target teams end subroutine - subroutine fill_omptgt_3d_(data_tgt, c) - real(dp), dimension(:, :, :), intent(inout) :: data_tgt - real(dp), intent(in) :: c + !!subroutine fill_omptgt_3d_(dev_ptr, c) + !! type(c_ptr), intent(inout) :: dev_ptr + !! real(dp), intent(in) :: c - integer, dimension(3) :: n - integer :: i, j, k + !! real(dp), dimension(:,:,:), pointer :: p_data_tgt + !! integer, dimension(3) :: n + !! integer :: i, j, k - n = shape(data_tgt) + !! n = shape(data_tgt) - !$omp target teams loop collapse(3) has_device_addr(data_tgt) - do k = 1, n(3) - do j = 1, n(2) - do i = 1, n(1) - data_tgt(i, j, k) = c - end do - end do - end do - !$omp end target teams loop - end subroutine + !! !$omp target teams loop collapse(3) is_device_ptr(dev_ptr) + !! call c_f_pointer(dev_ptr, p_data_tgt, shape=n) + !! do k = 1, n(3) + !! do j = 1, n(2) + !! do i = 1, n(1) + !! p_data_tgt(i, j, k) = c + !! end do + !! end do + !! end do + !! !$omp end target teams loop + !!end subroutine function get_shape_omptgt(self) result(dims) class(omptgt_field_t) :: self integer :: dims(3) + !$omp declare target - dims = shape(self%data_tgt) + dims = self%dims end function subroutine set_shape_omptgt(self, dims) class(omptgt_field_t) :: self integer, intent(in) :: dims(3) - call c_f_pointer(self%dev_ptr, self%data_tgt, shape=dims) + if (product(dims) < self%n) then + self%dims = dims + else + error stop "Trying to set shape of field greater than capacity" + end if end subroutine + type(c_ptr) function get_dev_ptr(self) result(ptr) + class(omptgt_field_t) :: self + + ptr = self%dev_ptr + end function + end module m_omptgt_allocator diff --git a/src/backend/omp/target/backend.f90 b/src/backend/omp/target/backend.f90 index fe6bd2f71..e5d3b1627 100644 --- a/src/backend/omp/target/backend.f90 +++ b/src/backend/omp/target/backend.f90 @@ -6,6 +6,8 @@ module m_omptgt_backend + use iso_c_binding, only: c_ptr, c_f_pointer + use m_common, only: dp, DIR_C, get_dirs_from_rdr use m_allocator, only: allocator_t @@ -61,7 +63,7 @@ subroutine veccopy_omptgt(self, dst, src) type is (omptgt_field_t) select type (src) type is (omptgt_field_t) - call veccopy_offload_(dst%data_tgt, src%data_tgt) + call veccopy_offload_(dst%get_dev_ptr(), dst%get_shape(), src%get_dev_ptr(), src%get_shape()) class default error stop "Called omptgt vector copy with unsupported source vector" end select @@ -71,25 +73,33 @@ subroutine veccopy_omptgt(self, dst, src) end select end subroutine - subroutine veccopy_offload_(dst, src) + subroutine veccopy_offload_(cp_dst, n_dst, cp_src, n_src) + type(c_ptr), intent(inout) :: cp_dst + type(c_ptr), intent(in) :: cp_src + integer, dimension(3), intent(in) :: n_dst, n_src - real(dp), dimension(:, :, :), intent(inout) :: dst - real(dp), dimension(:, :, :), intent(in) :: src + real(dp), dimension(:, :, :), pointer :: dst + real(dp), dimension(:, :, :), pointer :: src - integer, dimension(3) :: n integer :: i, j, k - n = shape(dst) + if (any(n_src < n_dst)) then + error stop "SRC array is smaller than destination" + end if - !$omp target teams loop collapse(3) has_device_addr(dst, src) - do k = 1, n(3) - do j = 1, n(2) - do i = 1, n(1) + !$omp target teams is_device_ptr(cp_dst, cp_src) + call c_f_pointer(cp_dst, dst, shape=n_dst) + call c_f_pointer(cp_src, src, shape=n_src) + !$omp loop collapse(3) + do k = 1, n_dst(3) + do j = 1, n_dst(2) + do i = 1, n_dst(1) dst(i, j, k) = src(i, j, k) end do end do end do - !$omp end target teams loop + !$omp end loop + !$omp end target teams end subroutine @@ -127,32 +137,31 @@ subroutine vecadd_offload(self, a, x, b, y) real(dp), intent(in) :: b type(omptgt_field_t), intent(inout) :: y + type(c_ptr) :: cp_x, cp_y + real(dp), dimension(:,:,:), pointer :: p_x, p_y + integer, dimension(3) :: dims + integer :: i, j, k dims = self%allocator%get_padded_dims(x%dir) - call vecadd_offload_(dims, a, x%data_tgt, b, y%data_tgt) - - end subroutine - - subroutine vecadd_offload_(dims, a, x, b, y) - integer, dimension(3), intent(in) :: dims - real(dp), intent(in) :: a - real(dp), dimension(:, :, :), intent(in) :: x - real(dp), intent(in) :: b - real(dp), dimension(:, :, :), intent(inout) :: y - - integer :: i, j, k + cp_x = x%get_dev_ptr() + cp_y = y%get_dev_ptr() - !$omp target teams loop collapse(3) has_device_addr(x, y) + !$omp target teams is_device_ptr(cp_x, cp_y) + call c_f_pointer(cp_x, p_x, shape=x%get_shape()) + call c_f_pointer(cp_y, p_y, shape=y%get_shape()) + !$omp loop collapse(3) do k = 1, dims(3) do j = 1, dims(2) do i = 1, dims(1) - y(i, j, k) = a*x(i, j, k) + b*y(i, j, k) + p_y(i, j, k) = a*p_x(i, j, k) + b*p_y(i, j, k) end do end do end do - !$omp end target teams loop + !$omp end loop + !$omp end target teams + end subroutine subroutine copy_data_to_f_omptgt(self, f, data) @@ -160,77 +169,68 @@ subroutine copy_data_to_f_omptgt(self, f, data) class(field_t), intent(inout) :: f real(dp), dimension(:, :, :), intent(in) :: data + type(c_ptr) :: f_dev_ptr + real(dp), dimension(:,:,:), pointer :: p_f integer, dimension(3) :: dims + integer :: i, j, k dims = self%allocator%get_padded_dims(f%dir) ! XXX: This could be improved following cuda/backend.f90:resolve_field_t() select type (f) type is (omptgt_field_t) - call copy_data_to_f_omptgt_(f%data_tgt, data, dims) + f_dev_ptr = f%get_dev_ptr() + !$omp target teams map(to:data) is_device_ptr(f_dev_ptr) + call c_f_pointer(f_dev_ptr, p_f, shape=f%get_shape()) + !$omp loop collapse(3) + do k = 1, dims(3) + do j = 1, dims(2) + do i = 1, dims(1) + p_f(i, j, k) = data(i, j, k) + end do + end do + end do + !$omp end loop + !$omp end target teams class default error stop "Unsupported" end select end subroutine copy_data_to_f_omptgt - subroutine copy_data_to_f_omptgt_(f_arr, d, dims) - real(dp), dimension(:, :, :), intent(inout) :: f_arr - real(dp), dimension(:, :, :), intent(in) :: d - integer, dimension(3), intent(in) :: dims - - integer :: i, j, k - - ! XXX: This could be improved following cuda/backend.f90:resolve_field_t() - !$omp target teams loop collapse(3) map(to:d) has_device_addr(f_arr) - do k = 1, dims(3) - do j = 1, dims(2) - do i = 1, dims(1) - f_arr(i, j, k) = d(i, j, k) - end do - end do - end do - !$omp end target teams loop - - end subroutine - subroutine copy_f_to_data_omptgt(self, data, f) class(omptgt_backend_t), intent(inout) :: self real(dp), dimension(:, :, :), intent(out) :: data class(field_t), intent(in) :: f + type(c_ptr) :: f_dev_ptr + real(dp), dimension(:,:,:), pointer :: p_f integer, dimension(3) :: dims + integer :: i, j, k dims = self%allocator%get_padded_dims(f%dir) select type (f) type is (omptgt_field_t) - call copy_f_to_data_omptgt_(data, f%data_tgt, dims) + f_dev_ptr = f%get_dev_ptr() + !$omp target teams map(from:data) is_device_ptr(f_dev_ptr) + call c_f_pointer(f_dev_ptr, p_f, shape=f%get_shape()) + !$omp loop collapse(3) + do k = 1, dims(3) + do j = 1, dims(2) + do i = 1, dims(1) + data(i, j, k) = p_f(i, j, k) + end do + end do + end do + !$omp end loop + !$omp end target teams class default error stop "Unsupported" end select end subroutine copy_f_to_data_omptgt - subroutine copy_f_to_data_omptgt_(data, f_arr, dims) - real(dp), dimension(:, :, :), intent(out) :: data - real(dp), dimension(:, :, :), intent(in) :: f_arr - integer, dimension(3), intent(in) :: dims - - integer :: i, j, k - - !$omp target teams loop collapse(3) map(from:data) has_device_addr(f_arr) - do k = 1, dims(3) - do j = 1, dims(2) - do i = 1, dims(1) - data(i, j, k) = f_arr(i, j, k) - end do - end do - end do - !$omp end target teams loop - - end subroutine - subroutine reorder_omptgt(self, u_, u, direction) class(omptgt_backend_t) :: self class(field_t), intent(inout) :: u_ @@ -248,10 +248,10 @@ subroutine reorder_omptgt(self, u_, u, direction) type is (omptgt_field_t) select type (u) type is (omptgt_field_t) - call reorder_omptgt_dd(u_%data_tgt, u%data_tgt, dims, dir_from, & + call reorder_omptgt_dd(u_, u, dims, dir_from, & dir_to, cart_padded) class default - call reorder_omptgt_dh(u_%data_tgt, u%data, dims, dir_from, dir_to, & + call reorder_omptgt_dh(u_, u%data, dims, dir_from, dir_to, & cart_padded) end select class default @@ -264,50 +264,64 @@ subroutine reorder_omptgt(self, u_, u, direction) end subroutine reorder_omptgt subroutine reorder_omptgt_dd(u_, u, dims, dir_from, dir_to, cart_padded) - real(dp), dimension(:, :, :), pointer :: u_ - real(dp), dimension(:, :, :), pointer, intent(in) :: u + type(omptgt_field_t) :: u_ + type(omptgt_field_t), intent(in) :: u integer, dimension(3), intent(in) :: dims integer, intent(in) :: dir_from, dir_to integer, dimension(3), intent(in) :: cart_padded + type(c_ptr) :: cp_u_, cp_u + real(dp), dimension(:,:,:), pointer :: p_u_, p_u integer :: i, j, k integer :: out_i, out_j, out_k - !$omp target teams loop collapse(3) private(out_i, out_j, out_k) has_device_addr(u_, u) + cp_u_ = u_%get_dev_ptr() + cp_u = u%get_dev_ptr() + !$omp target teams is_device_ptr(cp_u_, cp_u) + call c_f_pointer(cp_u_, p_u_, shape=u_%get_shape()) + call c_f_pointer(cp_u, p_u, shape=u%get_shape()) + !$omp loop collapse(3) private(out_i, out_j, out_k) do k = 1, dims(3) do j = 1, dims(2) do i = 1, dims(1) call get_index_reordering(out_i, out_j, out_k, i, j, k, & dir_from, dir_to, SZ, cart_padded) - u_(out_i, out_j, out_k) = u(i, j, k) + p_u_(out_i, out_j, out_k) = p_u(i, j, k) end do end do end do - !$omp end target teams loop + !$omp end loop + !$omp end target teams end subroutine subroutine reorder_omptgt_dh(u_, u, dims, dir_from, dir_to, cart_padded) - real(dp), dimension(:, :, :), pointer :: u_ + type(omptgt_field_t) :: u_ real(dp), dimension(:, :, :), pointer, intent(in) :: u integer, dimension(3), intent(in) :: dims integer, intent(in) :: dir_from, dir_to integer, dimension(3), intent(in) :: cart_padded + type(c_ptr) :: cp_u_ + real(dp), dimension(:,:,:), pointer :: p_u_ integer :: i, j, k integer :: out_i, out_j, out_k - !$omp target teams loop collapse(3) private(out_i, out_j, out_k) map(to:u) has_device_addr(u_) + cp_u_ = u_%get_dev_ptr() + !$omp target teams map(to:u) is_device_ptr(cp_u_) + call c_f_pointer(cp_u_, p_u_, shape=u_%get_shape()) + !$omp loop collapse(3) private(out_i, out_j, out_k) do k = 1, dims(3) do j = 1, dims(2) do i = 1, dims(1) call get_index_reordering(out_i, out_j, out_k, i, j, k, & dir_from, dir_to, SZ, cart_padded) - u_(out_i, out_j, out_k) = u(i, j, k) + p_u_(out_i, out_j, out_k) = u(i, j, k) end do end do end do - !$omp end target teams loop + !$omp end loop + !$omp end target teams end subroutine diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 53c5bdf2a..5880d344d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -49,7 +49,9 @@ function(define_test testfile np backend) target_compile_options(${test_name} PRIVATE "-fopenmp") target_compile_options(${test_name} PRIVATE "-DOMP_TGT") target_link_options(${test_name} PRIVATE "-fopenmp") - if(NOT ${CMAKE_Fortran_COMPILER_ID} STREQUAL "Cray") + if(NOT ${CMAKE_Fortran_COMPILER_ID} STREQUAL "Cray" AND + NOT ${CMAKE_Fortran_COMPILER_ID} STREQUAL "NVHPC" AND + NOT ${CMAKE_Fortran_COMPILER_ID} STREQUAL "PGI") target_compile_options(${test_name} PRIVATE "--offload-arch=${OMP_TGT_ARCH}") target_link_options(${test_name} PRIVATE "--offload-arch=${OMP_TGT_ARCH}") endif()