Use adaptive growing strategy for the edge off-heap array + handle ov…

…erflow if greater than 2Gb
bourgesl · Oct 6, 2015 · 65b8dd6 · 65b8dd6
1 parent 498fb91
commit 65b8dd6
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 7 deletions.
diff --git a/src/main/java/org/marlin/pisces/ArrayCache.java b/src/main/java/org/marlin/pisces/ArrayCache.java
@@ -40,6 +40,9 @@ public final class ArrayCache implements MarlinConst {
     final static int MIN_DIRTY_BYTE_ARRAY_SIZE = 32 * 2048; // 32px x 2048px
     final static int MAX_DIRTY_BYTE_ARRAY_SIZE;
     final static int[] DIRTY_BYTE_ARRAY_SIZES = new int[BUCKETS];
+    // large array thresholds:
+    final static long THRESHOLD_LARGE_ARRAY_SIZE;
+    final static long THRESHOLD_HUGE_ARRAY_SIZE;
     // stats
     private static int resizeInt = 0;
     private static int resizeDirtyInt = 0;
@@ -74,7 +77,10 @@ public final class ArrayCache implements MarlinConst {
         MAX_DIRTY_BYTE_ARRAY_SIZE = arraySize >> 1;
 
         // threshold to grow arrays only by (3/2) instead of 2
-        THRESHOLD_ARRAY_SIZE = Math.max(2 * 1024 * 1024, MAX_ARRAY_SIZE);
+        THRESHOLD_ARRAY_SIZE = Math.max(2 * 1024 * 1024, MAX_ARRAY_SIZE); // 2M
+
+        THRESHOLD_LARGE_ARRAY_SIZE = THRESHOLD_ARRAY_SIZE * 8; // 16M
+        THRESHOLD_HUGE_ARRAY_SIZE = THRESHOLD_LARGE_ARRAY_SIZE * 8; // 128M
 
         if (doStats || doMonitors) {
             logInfo("ArrayCache.BUCKETS        = " + BUCKETS);
@@ -90,6 +96,10 @@ public final class ArrayCache implements MarlinConst {
                     + Arrays.toString(DIRTY_BYTE_ARRAY_SIZES));
             logInfo("ArrayCache.THRESHOLD_ARRAY_SIZE = "
                     + THRESHOLD_ARRAY_SIZE);
+            logInfo("ArrayCache.THRESHOLD_LARGE_ARRAY_SIZE = "
+                    + THRESHOLD_LARGE_ARRAY_SIZE);
+            logInfo("ArrayCache.THRESHOLD_HUGE_ARRAY_SIZE = "
+                    + THRESHOLD_HUGE_ARRAY_SIZE);
         }
     }
 
@@ -158,16 +168,30 @@ public static int getNewSize(final int curSize, final int needSize) {
         final int initial = (curSize & MASK_CLR_1);
         int size;
         if (initial > THRESHOLD_ARRAY_SIZE) {
-            // use 1.5x:
-            size = (initial * 3) >> 1;
+            size = initial + (initial >> 1); // x (3/2)
         } else {
-            // use x2:
-            size = (initial) << 1;
+            size = (initial) << 1; // x2
         }
         // ensure the new size is >= needed size:
         if (size < needSize) {
-            size = needSize;
+            // align to 4096:
+            size = ((needSize >> 12) + 1) << 12;
         }
         return size;
     }
+
+    /**
+     * Return the new array size (~ x2)
+     * @param curSize current used size
+     * @return new array size
+     */
+    public static long getNewLargeSize(final long curSize) {
+        if (curSize > THRESHOLD_HUGE_ARRAY_SIZE) {
+            return curSize + (curSize >> 2L); // x (5/4)
+        }
+        if (curSize > THRESHOLD_LARGE_ARRAY_SIZE) {
+            return curSize + (curSize >> 1L); // x (3/2)
+        }
+        return curSize << 1L; // x2
+    }
 }
diff --git a/src/main/java/org/marlin/pisces/Renderer.java b/src/main/java/org/marlin/pisces/Renderer.java
@@ -441,7 +441,15 @@ private void addLine(float x1, float y1, float x2, float y2) {
             // suppose _edges.length > _SIZEOF_EDGE_BYTES
             // so doubling size is enough to add needed bytes
             // double size:
-            final long edgeNewSize = (_edges.length) << 1;
+            final long edgeNewSize = ArrayCache.getNewLargeSize(_edges.length);
+
+            if (edgeNewSize >= Integer.MAX_VALUE) {
+                // hard overflow failure - we can't even accommodate
+                // new items without overflowing
+                throw new ArrayIndexOutOfBoundsException(
+                              "edges exceeds maximum capacity !");
+            }
+
             if (doStats) {
                 RendererContext.stats.stat_rdr_edges_resizes.add(edgeNewSize);
             }
@@ -1484,6 +1492,9 @@ static final class OffHeapEdgeArray  {
          */
         void resize(final long len) {
             // TODO: handle OOME ?
+
+            System.out.println("resize: "+len);
+
             this.address = unsafe.reallocateMemory(address, len);
             this.length  = len;
             if (logUnsafeMalloc) {