Uses a host buffer for meshFields to avoid GPU-CPU copy

Author: dhyan1272

src/pmpo_MPMesh.hpp

@@ -19,6 +19,10 @@ template <> const MaterialPointSlice meshFieldIndexToMPSlice < MeshF_OnSurfVeloI
 
 class MPMesh{
 
+  private:
+    
+    Kokkos::View<double**, Kokkos::HostSpace>fieldHostBuffer;
+
   public:
 
     Mesh* p_mesh;
@@ -34,7 +38,7 @@ class MPMesh{
     std::vector<std::vector<int>> ownerHaloLocalIDs;
 
     void startCommunication();
-
+    
     void communicate_and_take_halo_contributions(const Kokkos::View<double**>& meshField, int nEntities, int numEntries, int mode, int op);
     //Now Kokkos views are made 1D
     template <typename ViewType>
@@ -50,40 +54,18 @@ class MPMesh{
       MPI_Comm_rank(comm, &self);
 
       Kokkos::Timer timer;
-      auto reconVals_host = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), meshField);
+      //auto reconVals_host_old = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), meshField);
+      //Replace alocation + copy with just copy
+      if (fieldHostBuffer.extent(0) < nEntities || fieldHostBuffer.extent(1)< numEntries){
+        printf("Buffer extent before %d %d \n", fieldHostBuffer.extent(0), fieldHostBuffer.extent(1));
+        fieldHostBuffer= Kokkos::View<double**, Kokkos::HostSpace>("host buffer fields", nEntities, numEntries);
+        printf("Buffer extent after %d %d \n", fieldHostBuffer.extent(0), fieldHostBuffer.extent(1));
+      }
+      auto reconVals_host=Kokkos::subview(fieldHostBuffer, std::make_pair(0, nEntities), std::make_pair(0, numEntries));
+      Kokkos::deep_copy(reconVals_host, meshField);
       Kokkos::fence();
       pumipic::RecordTime("SD: GPU-CPU copy-" + std::to_string(self), timer.seconds());
-    
-      /*  
-      timer.reset();
-      std::vector<double> fieldData1(nEntities*numEntries);
-      pumipic::RecordTime("SD: STL alloc-" + std::to_string(self), timer.seconds());
-      timer.reset();
-      
-      for (int i=0;i<nEntities;i++)
-        for (int j=0;j<numEntries;j++)
-          fieldData1[i*numEntries+j]=reconVals_host(i,j);
-      
-      std::memcpy(fieldData1.data(), reconVals_host.data(), nEntities*numEntries*sizeof(double));
-      pumipic::RecordTime("SD: STL alloc_copy-" + std::to_string(self), timer.seconds());
-      */
-/*
-for (int i=0; i<nEntities; i++) {
-  for (int j=0; j<numEntries; j++) {
-    if (fieldData1[i*numEntries+j] != reconVals_host(i,j)) {
-      std::cout
-        << "Mismatch at "
-        << i << " " << j
-        << " : "
-        << fieldData1[i*numEntries+j]
-        << " "
-        << reconVals_host(i,j)
-        << std::endl;
-      break;
-    }
-  }
-}
-*/
+
       timer.reset();
       std::vector<std::vector<int>>    recvIDVec;
       std::vector<std::vector<double>> recvDataVec;