Make HostMemoryBuffer call into the DefaultHostMemoryAllocator (#17204)

This is step 3 in a process of getting java host memory allocation to be plugable under a single allocation API. This is really only used for large memory allocations, which is what matters.

This changes the most common java host memory allocation API to call into the plugable host memory allocation API. The reason that this had to be done in multiple steps is because the Spark Plugin code was calling into the common memory allocation API, and memory allocation would end up calling itself recursively.

Step 1. Create a new API that will not be called recursively (#17197)
Step 2. Have the Java plugin use that new API instead of the old one to avoid any recursive invocations (NVIDIA/spark-rapids#11671)
Step 3. Update the common API to use the new backend (this)

There are likely to be more steps after this that involve cleaning up and removing APIs that are no longer needed.

This is marked as breaking even though it does not break any APIs, it changes the semantics enough that it feels like a breaking change.

This is blocked and should not be merged in until Step 2 is merged in, to avoid breaking the Spark plugin.

Authors:
  - Robert (Bobby) Evans (https://github.com/revans2)

Approvers:
  - Nghia Truong (https://github.com/ttnghia)
  - Alessandro Bellina (https://github.com/abellina)

URL: #17204

java/src/main/java/ai/rapids/cudf/DefaultHostMemoryAllocator.java

@@ -1,6 +1,6 @@
 /*
  *
- *  Copyright (c) 2023, NVIDIA CORPORATION.
+ *  Copyright (c) 2023-2024, NVIDIA CORPORATION.
  *
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -32,19 +32,25 @@ public static HostMemoryAllocator get() {
 
   /**
    * Sets a new default host memory allocator implementation by default.
-   * @param hostMemoryAllocator
+   * @param hostMemoryAllocator the new allocator to use.
    */
   public static void set(HostMemoryAllocator hostMemoryAllocator) {
     instance = hostMemoryAllocator;
   }
 
   @Override
   public HostMemoryBuffer allocate(long bytes, boolean preferPinned) {
-    return HostMemoryBuffer.allocate(bytes, preferPinned);
+    if (preferPinned) {
+      HostMemoryBuffer pinnedBuffer = PinnedMemoryPool.tryAllocate(bytes);
+      if (pinnedBuffer != null) {
+        return pinnedBuffer;
+      }
+    }
+    return new HostMemoryBuffer(UnsafeMemoryAccessor.allocate(bytes), bytes);
   }
 
   @Override
   public HostMemoryBuffer allocate(long bytes) {
-    return HostMemoryBuffer.allocate(bytes);
+    return allocate(bytes, HostMemoryBuffer.defaultPreferPinned);
   }
 }

java/src/main/java/ai/rapids/cudf/HostMemoryBuffer.java

@@ -41,7 +41,7 @@
  * Be aware that the off heap memory limits set by Java do not apply to these buffers.
  */
 public class HostMemoryBuffer extends MemoryBuffer {
-  private static final boolean defaultPreferPinned;
+  static final boolean defaultPreferPinned;
   private static final Logger log = LoggerFactory.getLogger(HostMemoryBuffer.class);
 
   static {
@@ -135,13 +135,7 @@ public boolean isClean() {
    * @return the newly created buffer
    */
   public static HostMemoryBuffer allocate(long bytes, boolean preferPinned) {
-    if (preferPinned) {
-      HostMemoryBuffer pinnedBuffer = PinnedMemoryPool.tryAllocate(bytes);
-      if (pinnedBuffer != null) {
-        return pinnedBuffer;
-      }
-    }
-    return new HostMemoryBuffer(UnsafeMemoryAccessor.allocate(bytes), bytes);
+    return DefaultHostMemoryAllocator.get().allocate(bytes, preferPinned);
   }
 
   /**