Use The_Comms_Arena in particle communication. (AMReX-Codes#4175)

The proposed changes:
- [ ] fix a bug or incorrect behavior in AMReX
- [ ] add new capabilities to AMReX
- [ ] changes answers in the test suite to more than roundoff level
- [ ] are likely to significantly affect the results of downstream AMReX
users
- [ ] include documentation in the code and/or rst files, if appropriate

Src/Particle/AMReX_NeighborParticles.H

@@ -467,7 +467,7 @@ protected:
     ParticleCopyPlan neighbor_copy_plan;
 
     amrex::PODVector<char, PolymorphicArenaAllocator<char> > snd_buffer;
-    amrex::Gpu::DeviceVector<char> rcv_buffer;
+    amrex::PODVector<char, PolymorphicArenaAllocator<char> > rcv_buffer;
 
     Gpu::PinnedVector<char> pinned_snd_buffer;
     Gpu::PinnedVector<char> pinned_rcv_buffer;

Src/Particle/AMReX_NeighborParticlesGPUImpl.H

@@ -255,6 +255,12 @@ updateNeighborsGPU (bool boundary_neighbors_only)
     }
 
     clearNeighbors();
+
+    if (ParallelDescriptor::UseGpuAwareMpi()) {
+        snd_buffer.setArena(The_Comms_Arena());
+        rcv_buffer.setArena(The_Comms_Arena());
+    }
+
     packBuffer(*this, neighbor_copy_op, neighbor_copy_plan, snd_buffer);
     if (ParallelDescriptor::UseGpuAwareMpi())
     {

Src/Particle/AMReX_ParticleContainerI.H

@@ -1388,8 +1388,14 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
     plan.build(*this, op, h_redistribute_int_comp,
                h_redistribute_real_comp, local);
 
+    // by default, this uses The_Arena();
     amrex::PODVector<char, PolymorphicArenaAllocator<char> > snd_buffer;
-    Gpu::DeviceVector<char> rcv_buffer;
+    amrex::PODVector<char, PolymorphicArenaAllocator<char> > rcv_buffer;
+
+    if (ParallelDescriptor::UseGpuAwareMpi()) {
+        snd_buffer.setArena(The_Comms_Arena());
+        rcv_buffer.setArena(The_Comms_Arena());
+    }
 
     packBuffer(*this, op, plan, snd_buffer);