diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 3b2a58827f15..12f8bc6df03d 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1060,6 +1060,11 @@ public enum OperationStatusCode {
public static final float HFILE_BLOCK_CACHE_SIZE_DEFAULT = 0.4f;
+ /**
+ * Configuration key for the memory size of the block cache
+ */
+ public static final String HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY = "hfile.block.cache.memory.size";
+
/**
* Configuration key for setting the fix size of the block size, default do nothing and it should
* be explicitly set by user or only used within ClientSideRegionScanner. if it's set less than
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/StorageSize.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/StorageSize.java
new file mode 100644
index 000000000000..83ebdb58c2a6
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/StorageSize.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import static org.apache.commons.lang3.StringUtils.isBlank;
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+
+import java.util.Locale;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
+
+/**
+ * This class is adapted from the Hadoop 3.x source code to HBase as part of a backporting effort to
+ * support storage size parsing functionality in older versions of HBase.
+ *
+ * Source:
+ * Hadoop 3.1.0 StorageSize.java
+ *
+ */
+@InterfaceAudience.Private
+public class StorageSize {
+ private final StorageUnit unit;
+ private final double value;
+
+ /**
+ * Constucts a Storage Measure, which contains the value and the unit of measure.
+ * @param unit - Unit of Measure
+ * @param value - Numeric value.
+ */
+ public StorageSize(StorageUnit unit, double value) {
+ this.unit = unit;
+ this.value = value;
+ }
+
+ private static void checkState(boolean state, String errorString) {
+ if (!state) {
+ throw new IllegalStateException(errorString);
+ }
+ }
+
+ public static double getStorageSize(String value, double defaultValue, StorageUnit targetUnit) {
+ Preconditions.checkNotNull(targetUnit, "Conversion unit cannot be null.");
+
+ if (isBlank(value)) {
+ return targetUnit.getDefault(defaultValue);
+ }
+
+ final StorageSize measure = parse(value);
+ double byteValue = measure.getUnit().toBytes(measure.getValue());
+ return targetUnit.fromBytes(byteValue);
+ }
+
+ public static StorageSize parse(String value) {
+ checkState(isNotBlank(value), "value cannot be blank");
+ String sanitizedValue = value.trim().toLowerCase(Locale.ENGLISH);
+ StorageUnit parsedUnit = null;
+ for (StorageUnit unit : StorageUnit.values()) {
+ if (
+ sanitizedValue.endsWith(unit.getShortName()) || sanitizedValue.endsWith(unit.getLongName())
+ || sanitizedValue.endsWith(unit.getSuffixChar())
+ ) {
+ parsedUnit = unit;
+ break;
+ }
+ }
+
+ if (parsedUnit == null) {
+ throw new IllegalArgumentException(
+ value + " is not in expected format." + "Expected format is . e.g. 1000MB");
+ }
+
+ String suffix = "";
+ boolean found = false;
+
+ // We are trying to get the longest match first, so the order of
+ // matching is getLongName, getShortName and then getSuffixChar.
+ if (!found && sanitizedValue.endsWith(parsedUnit.getLongName())) {
+ found = true;
+ suffix = parsedUnit.getLongName();
+ }
+
+ if (!found && sanitizedValue.endsWith(parsedUnit.getShortName())) {
+ found = true;
+ suffix = parsedUnit.getShortName();
+ }
+
+ if (!found && sanitizedValue.endsWith(parsedUnit.getSuffixChar())) {
+ found = true;
+ suffix = parsedUnit.getSuffixChar();
+ }
+
+ checkState(found, "Something is wrong, we have to find a " + "match. Internal error.");
+
+ String valString = sanitizedValue.substring(0, value.length() - suffix.length());
+ return new StorageSize(parsedUnit, Double.parseDouble(valString));
+
+ }
+
+ public StorageUnit getUnit() {
+ return unit;
+ }
+
+ public double getValue() {
+ return value;
+ }
+}
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/StorageUnit.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/StorageUnit.java
new file mode 100644
index 000000000000..76403f195386
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/StorageUnit.java
@@ -0,0 +1,530 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * This class is adapted from the Hadoop 3.x source code to HBase as part of a backporting effort to
+ * support storage size parsing functionality in older versions of HBase.
+ *
+ * Source:
+ * Hadoop 3.1.0 StorageUnit.java
+ *
+ */
+@InterfaceAudience.Private
+public enum StorageUnit {
+ /*
+ * We rely on BYTES being the last to get the longest matching short names first. The short name
+ * of bytes is b, and it will match with other longer names. If we change this order, the
+ * corresponding code in StorageSize#parse needs to be changed too, since values() call returns
+ * the Enums in declared order, and we depend on it.
+ */
+
+ EB {
+ @Override
+ public double toBytes(double value) {
+ return multiply(value, EXABYTES);
+ }
+
+ @Override
+ public double toKBs(double value) {
+ return multiply(value, EXABYTES / KILOBYTES);
+ }
+
+ @Override
+ public double toMBs(double value) {
+ return multiply(value, EXABYTES / MEGABYTES);
+ }
+
+ @Override
+ public double toGBs(double value) {
+ return multiply(value, EXABYTES / GIGABYTES);
+ }
+
+ @Override
+ public double toTBs(double value) {
+ return multiply(value, EXABYTES / TERABYTES);
+ }
+
+ @Override
+ public double toPBs(double value) {
+ return multiply(value, EXABYTES / PETABYTES);
+ }
+
+ @Override
+ public double toEBs(double value) {
+ return value;
+ }
+
+ @Override
+ public String getLongName() {
+ return "exabytes";
+ }
+
+ @Override
+ public String getShortName() {
+ return "eb";
+ }
+
+ @Override
+ public String getSuffixChar() {
+ return "e";
+ }
+
+ @Override
+ public double getDefault(double value) {
+ return toEBs(value);
+ }
+
+ @Override
+ public double fromBytes(double value) {
+ return divide(value, EXABYTES);
+ }
+ },
+ PB {
+ @Override
+ public double toBytes(double value) {
+ return multiply(value, PETABYTES);
+ }
+
+ @Override
+ public double toKBs(double value) {
+ return multiply(value, PETABYTES / KILOBYTES);
+ }
+
+ @Override
+ public double toMBs(double value) {
+ return multiply(value, PETABYTES / MEGABYTES);
+ }
+
+ @Override
+ public double toGBs(double value) {
+ return multiply(value, PETABYTES / GIGABYTES);
+ }
+
+ @Override
+ public double toTBs(double value) {
+ return multiply(value, PETABYTES / TERABYTES);
+ }
+
+ @Override
+ public double toPBs(double value) {
+ return value;
+ }
+
+ @Override
+ public double toEBs(double value) {
+ return divide(value, EXABYTES / PETABYTES);
+ }
+
+ @Override
+ public String getLongName() {
+ return "petabytes";
+ }
+
+ @Override
+ public String getShortName() {
+ return "pb";
+ }
+
+ @Override
+ public String getSuffixChar() {
+ return "p";
+ }
+
+ @Override
+ public double getDefault(double value) {
+ return toPBs(value);
+ }
+
+ @Override
+ public double fromBytes(double value) {
+ return divide(value, PETABYTES);
+ }
+ },
+ TB {
+ @Override
+ public double toBytes(double value) {
+ return multiply(value, TERABYTES);
+ }
+
+ @Override
+ public double toKBs(double value) {
+ return multiply(value, TERABYTES / KILOBYTES);
+ }
+
+ @Override
+ public double toMBs(double value) {
+ return multiply(value, TERABYTES / MEGABYTES);
+ }
+
+ @Override
+ public double toGBs(double value) {
+ return multiply(value, TERABYTES / GIGABYTES);
+ }
+
+ @Override
+ public double toTBs(double value) {
+ return value;
+ }
+
+ @Override
+ public double toPBs(double value) {
+ return divide(value, PETABYTES / TERABYTES);
+ }
+
+ @Override
+ public double toEBs(double value) {
+ return divide(value, EXABYTES / TERABYTES);
+ }
+
+ @Override
+ public String getLongName() {
+ return "terabytes";
+ }
+
+ @Override
+ public String getShortName() {
+ return "tb";
+ }
+
+ @Override
+ public String getSuffixChar() {
+ return "t";
+ }
+
+ @Override
+ public double getDefault(double value) {
+ return toTBs(value);
+ }
+
+ @Override
+ public double fromBytes(double value) {
+ return divide(value, TERABYTES);
+ }
+ },
+ GB {
+ @Override
+ public double toBytes(double value) {
+ return multiply(value, GIGABYTES);
+ }
+
+ @Override
+ public double toKBs(double value) {
+ return multiply(value, GIGABYTES / KILOBYTES);
+ }
+
+ @Override
+ public double toMBs(double value) {
+ return multiply(value, GIGABYTES / MEGABYTES);
+ }
+
+ @Override
+ public double toGBs(double value) {
+ return value;
+ }
+
+ @Override
+ public double toTBs(double value) {
+ return divide(value, TERABYTES / GIGABYTES);
+ }
+
+ @Override
+ public double toPBs(double value) {
+ return divide(value, PETABYTES / GIGABYTES);
+ }
+
+ @Override
+ public double toEBs(double value) {
+ return divide(value, EXABYTES / GIGABYTES);
+ }
+
+ @Override
+ public String getLongName() {
+ return "gigabytes";
+ }
+
+ @Override
+ public String getShortName() {
+ return "gb";
+ }
+
+ @Override
+ public String getSuffixChar() {
+ return "g";
+ }
+
+ @Override
+ public double getDefault(double value) {
+ return toGBs(value);
+ }
+
+ @Override
+ public double fromBytes(double value) {
+ return divide(value, GIGABYTES);
+ }
+ },
+ MB {
+ @Override
+ public double toBytes(double value) {
+ return multiply(value, MEGABYTES);
+ }
+
+ @Override
+ public double toKBs(double value) {
+ return multiply(value, MEGABYTES / KILOBYTES);
+ }
+
+ @Override
+ public double toMBs(double value) {
+ return value;
+ }
+
+ @Override
+ public double toGBs(double value) {
+ return divide(value, GIGABYTES / MEGABYTES);
+ }
+
+ @Override
+ public double toTBs(double value) {
+ return divide(value, TERABYTES / MEGABYTES);
+ }
+
+ @Override
+ public double toPBs(double value) {
+ return divide(value, PETABYTES / MEGABYTES);
+ }
+
+ @Override
+ public double toEBs(double value) {
+ return divide(value, EXABYTES / MEGABYTES);
+ }
+
+ @Override
+ public String getLongName() {
+ return "megabytes";
+ }
+
+ @Override
+ public String getShortName() {
+ return "mb";
+ }
+
+ @Override
+ public String getSuffixChar() {
+ return "m";
+ }
+
+ @Override
+ public double fromBytes(double value) {
+ return divide(value, MEGABYTES);
+ }
+
+ @Override
+ public double getDefault(double value) {
+ return toMBs(value);
+ }
+ },
+ KB {
+ @Override
+ public double toBytes(double value) {
+ return multiply(value, KILOBYTES);
+ }
+
+ @Override
+ public double toKBs(double value) {
+ return value;
+ }
+
+ @Override
+ public double toMBs(double value) {
+ return divide(value, MEGABYTES / KILOBYTES);
+ }
+
+ @Override
+ public double toGBs(double value) {
+ return divide(value, GIGABYTES / KILOBYTES);
+ }
+
+ @Override
+ public double toTBs(double value) {
+ return divide(value, TERABYTES / KILOBYTES);
+ }
+
+ @Override
+ public double toPBs(double value) {
+ return divide(value, PETABYTES / KILOBYTES);
+ }
+
+ @Override
+ public double toEBs(double value) {
+ return divide(value, EXABYTES / KILOBYTES);
+ }
+
+ @Override
+ public String getLongName() {
+ return "kilobytes";
+ }
+
+ @Override
+ public String getShortName() {
+ return "kb";
+ }
+
+ @Override
+ public String getSuffixChar() {
+ return "k";
+ }
+
+ @Override
+ public double getDefault(double value) {
+ return toKBs(value);
+ }
+
+ @Override
+ public double fromBytes(double value) {
+ return divide(value, KILOBYTES);
+ }
+ },
+ BYTES {
+ @Override
+ public double toBytes(double value) {
+ return value;
+ }
+
+ @Override
+ public double toKBs(double value) {
+ return divide(value, KILOBYTES);
+ }
+
+ @Override
+ public double toMBs(double value) {
+ return divide(value, MEGABYTES);
+ }
+
+ @Override
+ public double toGBs(double value) {
+ return divide(value, GIGABYTES);
+ }
+
+ @Override
+ public double toTBs(double value) {
+ return divide(value, TERABYTES);
+ }
+
+ @Override
+ public double toPBs(double value) {
+ return divide(value, PETABYTES);
+ }
+
+ @Override
+ public double toEBs(double value) {
+ return divide(value, EXABYTES);
+ }
+
+ @Override
+ public String getLongName() {
+ return "bytes";
+ }
+
+ @Override
+ public String getShortName() {
+ return "b";
+ }
+
+ @Override
+ public String getSuffixChar() {
+ return "b";
+ }
+
+ @Override
+ public double getDefault(double value) {
+ return toBytes(value);
+ }
+
+ @Override
+ public double fromBytes(double value) {
+ return value;
+ }
+ };
+
+ private static final double BYTE = 1L;
+ private static final double KILOBYTES = BYTE * 1024L;
+ private static final double MEGABYTES = KILOBYTES * 1024L;
+ private static final double GIGABYTES = MEGABYTES * 1024L;
+ private static final double TERABYTES = GIGABYTES * 1024L;
+ private static final double PETABYTES = TERABYTES * 1024L;
+ private static final double EXABYTES = PETABYTES * 1024L;
+ private static final int PRECISION = 4;
+
+ /**
+ * Using BigDecimal to avoid issues with overflow and underflow.
+ * @param value - value
+ * @param divisor - divisor.
+ * @return -- returns a double that represents this value
+ */
+ private static double divide(double value, double divisor) {
+ BigDecimal val = new BigDecimal(value);
+ BigDecimal bDivisor = new BigDecimal(divisor);
+ return val.divide(bDivisor).setScale(PRECISION, RoundingMode.HALF_UP).doubleValue();
+ }
+
+ /**
+ * Using BigDecimal so we can throw if we are overflowing the Long.Max.
+ * @param first - First Num.
+ * @param second - Second Num.
+ * @return Returns a double
+ */
+ private static double multiply(double first, double second) {
+ BigDecimal firstVal = new BigDecimal(first);
+ BigDecimal secondVal = new BigDecimal(second);
+ return firstVal.multiply(secondVal).setScale(PRECISION, RoundingMode.HALF_UP).doubleValue();
+ }
+
+ public abstract double toBytes(double value);
+
+ public abstract double toKBs(double value);
+
+ public abstract double toMBs(double value);
+
+ public abstract double toGBs(double value);
+
+ public abstract double toTBs(double value);
+
+ public abstract double toPBs(double value);
+
+ public abstract double toEBs(double value);
+
+ public abstract String getLongName();
+
+ public abstract String getShortName();
+
+ public abstract String getSuffixChar();
+
+ public abstract double getDefault(double value);
+
+ public abstract double fromBytes(double value);
+
+ public String toString() {
+ return getLongName();
+ }
+}
diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml
index 4cce153e60cd..15b970c7e8d0 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -1000,6 +1000,14 @@ possible configurations would overwhelm and obscure the important.
Set to 0 to disable but it's not recommended; you need at least
enough cache to hold the storefile indices.
+
+ hfile.block.cache.memory.size
+
+ Defines the maximum heap memory allocated for the HFile block cache,
+ specified in bytes or human-readable formats like '10m' for megabytes or '10g' for gigabytes.
+ This configuration allows setting an absolute memory size instead of a percentage of the maximum heap.
+ Takes precedence over hfile.block.cache.size if both are specified.
+
hfile.block.index.cacheonwrite
false
diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestStorageSize.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestStorageSize.java
new file mode 100644
index 000000000000..386221947d59
--- /dev/null
+++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestStorageSize.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ MiscTests.class, SmallTests.class })
+public class TestStorageSize {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestStorageSize.class);
+
+ @Test
+ public void testParse() {
+ // megabytes
+ StorageSize size = StorageSize.parse("20m");
+ assertEquals(StorageUnit.MB, size.getUnit());
+ assertEquals(20.0d, size.getValue(), 0.0001);
+ size = StorageSize.parse("40mb");
+ assertEquals(StorageUnit.MB, size.getUnit());
+ assertEquals(40.0d, size.getValue(), 0.0001);
+ size = StorageSize.parse("60megabytes");
+ assertEquals(StorageUnit.MB, size.getUnit());
+ assertEquals(60.0d, size.getValue(), 0.0001);
+
+ // gigabytes
+ size = StorageSize.parse("10g");
+ assertEquals(StorageUnit.GB, size.getUnit());
+ assertEquals(10.0d, size.getValue(), 0.0001);
+ size = StorageSize.parse("30gb");
+ assertEquals(StorageUnit.GB, size.getUnit());
+ assertEquals(30.0d, size.getValue(), 0.0001);
+ size = StorageSize.parse("50gigabytes");
+ assertEquals(StorageUnit.GB, size.getUnit());
+ assertEquals(50.0d, size.getValue(), 0.0001);
+ }
+
+ @Test
+ public void testGetStorageSize() {
+ assertEquals(1024 * 1024 * 4, StorageSize.getStorageSize("4m", -1, StorageUnit.BYTES), 0.0001);
+ assertEquals(1024 * 6, StorageSize.getStorageSize("6g", -1, StorageUnit.MB), 0.0001);
+ assertEquals(-1, StorageSize.getStorageSize(null, -1, StorageUnit.BYTES), 0.0001);
+ assertEquals(-2, StorageSize.getStorageSize("", -2, StorageUnit.BYTES), 0.0001);
+ }
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java
index bf014dfb5303..15aeb2153e61 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java
@@ -22,6 +22,8 @@
import java.lang.management.MemoryUsage;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.StorageSize;
+import org.apache.hadoop.hbase.StorageUnit;
import org.apache.hadoop.hbase.regionserver.MemStoreLAB;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.yetus.audience.InterfaceAudience;
@@ -93,11 +95,16 @@ public static void checkForClusterFreeHeapMemoryLimit(Configuration conf) {
) {
throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds "
+ "the threshold required for successful cluster operation. "
- + "The combined value cannot exceed 0.8. Please check "
- + "the settings for hbase.regionserver.global.memstore.size and "
- + "hfile.block.cache.size in your configuration. "
- + "hbase.regionserver.global.memstore.size is " + globalMemstoreSize
- + " hfile.block.cache.size is " + blockCacheUpperLimit);
+ + "The combined value cannot exceed 0.8. Please check " + "the settings for "
+ + MEMSTORE_SIZE_KEY + " and either " + HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY + " or "
+ + HConstants.HFILE_BLOCK_CACHE_SIZE_KEY + " in your configuration. " + MEMSTORE_SIZE_KEY
+ + "=" + globalMemstoreSize + ", " + HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY + "="
+ + conf.get(HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY) + ", "
+ + HConstants.HFILE_BLOCK_CACHE_SIZE_KEY + "="
+ + conf.get(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY) + ". (Note: If both "
+ + HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY + " and "
+ + HConstants.HFILE_BLOCK_CACHE_SIZE_KEY + " are set, " + "the system will use "
+ + HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY + ")");
}
}
@@ -195,10 +202,30 @@ public static long getOnheapGlobalMemStoreSize(Configuration conf) {
* Retrieve configured size for on heap block cache as percentage of total heap.
*/
public static float getBlockCacheHeapPercent(final Configuration conf) {
- // L1 block cache is always on heap
- float l1CachePercent = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
+ // Check if an explicit block cache size is configured.
+ long l1CacheSizeInBytes = getBlockCacheSizeInBytes(conf);
+ if (l1CacheSizeInBytes > 0) {
+ final MemoryUsage usage = safeGetHeapMemoryUsage();
+ return usage == null ? 0 : (float) l1CacheSizeInBytes / usage.getMax();
+ }
+
+ return conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
- return l1CachePercent;
+ }
+
+ /**
+ * Retrieve an explicit block cache size in bytes in the configuration.
+ * @param conf used to read cache configs
+ * @return the number of bytes to use for LRU, negative if disabled.
+ * @throws IllegalArgumentException if HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY format is invalid
+ */
+ public static long getBlockCacheSizeInBytes(Configuration conf) {
+ final String key = HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY;
+ try {
+ return Long.parseLong(conf.get(key));
+ } catch (NumberFormatException e) {
+ return (long) StorageSize.getStorageSize(conf.get(key), -1, StorageUnit.BYTES);
+ }
}
/**
@@ -207,8 +234,7 @@ public static float getBlockCacheHeapPercent(final Configuration conf) {
* @throws IllegalArgumentException if HFILE_BLOCK_CACHE_SIZE_KEY is > 1.0
*/
public static long getOnHeapCacheSize(final Configuration conf) {
- float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
- HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
+ final float cachePercentage = getBlockCacheHeapPercent(conf);
if (cachePercentage <= 0.0001f) {
return -1;
}
@@ -216,18 +242,22 @@ public static long getOnHeapCacheSize(final Configuration conf) {
throw new IllegalArgumentException(
HConstants.HFILE_BLOCK_CACHE_SIZE_KEY + " must be between 0.0 and 1.0, and not > 1.0");
}
- long max = -1L;
+
final MemoryUsage usage = safeGetHeapMemoryUsage();
- if (usage != null) {
- max = usage.getMax();
+ if (usage == null) {
+ return -1;
}
+ final long heapMax = usage.getMax();
float onHeapCacheFixedSize =
(float) conf.getLong(HConstants.HFILE_ONHEAP_BLOCK_CACHE_FIXED_SIZE_KEY,
- HConstants.HFILE_ONHEAP_BLOCK_CACHE_FIXED_SIZE_DEFAULT) / max;
+ HConstants.HFILE_ONHEAP_BLOCK_CACHE_FIXED_SIZE_DEFAULT) / heapMax;
// Calculate the amount of heap to give the heap.
- return (onHeapCacheFixedSize > 0 && onHeapCacheFixedSize < cachePercentage)
- ? (long) (max * onHeapCacheFixedSize)
- : (long) (max * cachePercentage);
+ if (onHeapCacheFixedSize > 0 && onHeapCacheFixedSize < cachePercentage) {
+ return (long) (heapMax * onHeapCacheFixedSize);
+ } else {
+ final long cacheSizeInBytes = getBlockCacheSizeInBytes(conf);
+ return cacheSizeInBytes > 0 ? cacheSizeInBytes : (long) (heapMax * cachePercentage);
+ }
}
/**
@@ -243,5 +273,4 @@ public static long getBucketCacheSize(final Configuration conf) {
}
return (long) (bucketCacheSize * 1024 * 1024);
}
-
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java
index fe2737b0a7d2..1b28846efad0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import static org.apache.hadoop.hbase.HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY;
import static org.apache.hadoop.hbase.HConstants.HFILE_BLOCK_CACHE_SIZE_KEY;
import java.lang.management.MemoryUsage;
@@ -128,8 +129,7 @@ private ResizableBlockCache toResizableBlockCache(BlockCache blockCache) {
private boolean doInit(Configuration conf) {
boolean tuningEnabled = true;
globalMemStorePercent = MemorySizeUtil.getGlobalMemStoreHeapPercent(conf, false);
- blockCachePercent =
- conf.getFloat(HFILE_BLOCK_CACHE_SIZE_KEY, HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
+ blockCachePercent = MemorySizeUtil.getBlockCacheHeapPercent(conf);
MemorySizeUtil.checkForClusterFreeHeapMemoryLimit(conf);
// Initialize max and min range for memstore heap space
globalMemStorePercentMinRange =
@@ -160,16 +160,20 @@ private boolean doInit(Configuration conf) {
blockCachePercentMinRange = conf.getFloat(BLOCK_CACHE_SIZE_MIN_RANGE_KEY, blockCachePercent);
blockCachePercentMaxRange = conf.getFloat(BLOCK_CACHE_SIZE_MAX_RANGE_KEY, blockCachePercent);
if (blockCachePercent < blockCachePercentMinRange) {
- LOG.warn("Setting " + BLOCK_CACHE_SIZE_MIN_RANGE_KEY + " to " + blockCachePercent
- + ", same value as " + HFILE_BLOCK_CACHE_SIZE_KEY
- + " because supplied value greater than initial block cache size.");
+ LOG.warn(
+ "Setting {} to {} (lookup order: {} -> {}), "
+ + "because supplied value greater than initial block cache size.",
+ BLOCK_CACHE_SIZE_MIN_RANGE_KEY, blockCachePercent, HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY,
+ HFILE_BLOCK_CACHE_SIZE_KEY);
blockCachePercentMinRange = blockCachePercent;
conf.setFloat(BLOCK_CACHE_SIZE_MIN_RANGE_KEY, blockCachePercentMinRange);
}
if (blockCachePercent > blockCachePercentMaxRange) {
- LOG.warn("Setting " + BLOCK_CACHE_SIZE_MAX_RANGE_KEY + " to " + blockCachePercent
- + ", same value as " + HFILE_BLOCK_CACHE_SIZE_KEY
- + " because supplied value less than initial block cache size.");
+ LOG.warn(
+ "Setting {} to {} (lookup order: {} -> {}), "
+ + "because supplied value less than initial block cache size.",
+ BLOCK_CACHE_SIZE_MAX_RANGE_KEY, blockCachePercent, HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY,
+ HFILE_BLOCK_CACHE_SIZE_KEY);
blockCachePercentMaxRange = blockCachePercent;
conf.setFloat(BLOCK_CACHE_SIZE_MAX_RANGE_KEY, blockCachePercentMaxRange);
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
index 542bcda138e1..bee8ca0667de 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
@@ -394,6 +394,14 @@ public void testGetOnHeapCacheSize() {
long onHeapCacheSize = MemorySizeUtil.getOnHeapCacheSize(copyConf);
assertEquals(null, copyConf.get(HConstants.HFILE_ONHEAP_BLOCK_CACHE_FIXED_SIZE_KEY));
assertTrue(onHeapCacheSize > 0 && onHeapCacheSize != fixedSize);
+ // when HBASE_BLOCK_CACHE_MEMORY_SIZE is set in number
+ copyConf.setLong(HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY, 3 * 1024 * 1024);
+ onHeapCacheSize = MemorySizeUtil.getOnHeapCacheSize(copyConf);
+ assertEquals(3 * 1024 * 1024, onHeapCacheSize);
+ // when HBASE_BLOCK_CACHE_MEMORY_SIZE is set in human-readable format
+ copyConf.set(HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY, "2m");
+ onHeapCacheSize = MemorySizeUtil.getOnHeapCacheSize(copyConf);
+ assertEquals(2 * 1024 * 1024, onHeapCacheSize);
// when HBASE_BLOCK_CACHE_FIXED_SIZE_KEY is set, it will be a fixed size
copyConf.setLong(HConstants.HFILE_ONHEAP_BLOCK_CACHE_FIXED_SIZE_KEY, fixedSize);
onHeapCacheSize = MemorySizeUtil.getOnHeapCacheSize(copyConf);