add rate metric for survivor pool growth (#992)

I was troubleshooting an GC pause outlier issue caused by survivor
space growth and the associated copying, while overall allocation/tenuring
rates remained stable. Issue was obvious from GC logs, but it occurred
to me this was a gap in our existing pool rates.

spectator-ext-gc/src/main/java/com/netflix/spectator/gc/GcLogger.java

@@ -70,6 +70,11 @@ public final class GcLogger {
   private static final Counter ALLOCATION_RATE =
     Spectator.globalRegistry().counter("jvm.gc.allocationRate");
 
+  // Incremented for any positive increases in the size of the survivor memory pool
+  // before GC to after GC
+  private static final Counter SURVIVOR_RATE =
+          Spectator.globalRegistry().counter("jvm.gc.survivorRate");
+
   // Pause time due to GC event
   private static final Id PAUSE_TIME = Spectator.globalRegistry().createId("jvm.gc.pause");
 
@@ -84,6 +89,9 @@ public final class GcLogger {
   private long youngGenSizeAfter = 0L;
 
   private String youngGenPoolName = null;
+
+  private String survivorPoolName = null;
+
   private String oldGenPoolName = null;
 
   private GcNotificationListener notifListener = null;
@@ -99,13 +107,17 @@ public GcLogger() {
     }
 
     for (MemoryPoolMXBean mbean : ManagementFactory.getMemoryPoolMXBeans()) {
+      String poolName = mbean.getName();
       // For non-generational collectors the young and old gen pool names will be the
       // same
-      if (HelperFunctions.isYoungGenPool(mbean.getName())) {
-        youngGenPoolName = mbean.getName();
+      if (HelperFunctions.isYoungGenPool(poolName)) {
+        youngGenPoolName = poolName;
+      }
+      if (HelperFunctions.isSurvivorPool(poolName)) {
+        survivorPoolName = poolName;
       }
-      if (HelperFunctions.isOldGenPool(mbean.getName())) {
-        oldGenPoolName = mbean.getName();
+      if (HelperFunctions.isOldGenPool(poolName)) {
+        oldGenPoolName = poolName;
       }
     }
   }
@@ -187,6 +199,15 @@ private void updateMetrics(String name, GcInfo info) {
       }
     }
 
+    if (survivorPoolName != null) {
+      final long survivorBefore = before.get(survivorPoolName).getUsed();
+      final long survivorAfter = after.get(survivorPoolName).getUsed();
+      final long delta = survivorAfter - survivorBefore;
+      if (delta > 0L) {
+        SURVIVOR_RATE.increment(delta);
+      }
+    }
+
     if (youngGenPoolName != null) {
       final long youngBefore = before.get(youngGenPoolName).getUsed();
       final long youngAfter = after.get(youngGenPoolName).getUsed();

spectator-ext-gc/src/main/java/com/netflix/spectator/gc/HelperFunctions.java

@@ -74,6 +74,10 @@ static boolean isYoungGenPool(String name) {
         || "ZHeap".equals(name);
   }
 
+  static boolean isSurvivorPool(String name) {
+    return name.endsWith("Survivor Space");
+  }
+
   /** Compute the total usage across all pools. */
   static long getTotalUsage(Map<String, MemoryUsage> usages) {
     long sum = 0L;
@@ -101,4 +105,5 @@ static long getPromotionSize(GcInfo info) {
     long totalAfter = getTotalUsage(info.getMemoryUsageAfterGc());
     return totalAfter - totalBefore;
   }
+
 }