diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index c8bb46101fb8..9858e5521eec 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -308,6 +308,16 @@ possible configurations would overwhelm and obscure the important. honor the old hbase.regionserver.global.memstore.upperLimit property if present. + + hbase.regionserver.global.memstore.memory.size + + Maximum size of all memstores in a region server before new + updates are blocked and flushes are forced, specified in bytes or human-readable formats + like '10m' for megabytes or '10g' for gigabytes. This configuration allows setting + an absolute memory size instead of a percentage of the maximum heap. Takes precedence + over hbase.regionserver.global.memstore.size if both are specified. + + hbase.regionserver.global.memstore.size.lower.limit diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java index 060c10bd63ec..8a4aaab26d10 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java @@ -37,6 +37,8 @@ public class MemorySizeUtil { public static final String MEMSTORE_SIZE_KEY = "hbase.regionserver.global.memstore.size"; + public static final String MEMSTORE_MEMORY_SIZE_KEY = + "hbase.regionserver.global.memstore.memory.size"; public static final String MEMSTORE_SIZE_OLD_KEY = "hbase.regionserver.global.memstore.upperLimit"; public static final String MEMSTORE_SIZE_LOWER_LIMIT_KEY = @@ -51,6 +53,8 @@ public class MemorySizeUtil { public static final float DEFAULT_MEMSTORE_SIZE = 0.4f; // Default lower water mark limit is 95% size of memstore size. public static final float DEFAULT_MEMSTORE_SIZE_LOWER_LIMIT = 0.95f; + private static final float MEMSTORE_HEAP_MIN_FRACTION = 0.0f; + private static final float MEMSTORE_HEAP_MAX_FRACTION = 0.8f; /** * Configuration key for the absolute amount of heap memory that must remain free for a @@ -106,9 +110,10 @@ public static void validateRegionServerHeapMemoryAllocation(Configuration conf) throw new RuntimeException(String.format( "RegionServer heap memory allocation is invalid: total memory usage exceeds 100%% " + "(memStore + blockCache + requiredFreeHeap). " - + "Check the following configuration values:%n" + " - %s = %.2f%n" + " - %s = %s%n" - + " - %s = %s%n" + " - %s = %s", - MEMSTORE_SIZE_KEY, memStoreFraction, HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY, + + "Check the following configuration values:%n" + " - %s = %s%n" + " - %s = %s%n" + + " - %s = %s%n" + " - %s = %s%n" + " - %s = %s", + MEMSTORE_MEMORY_SIZE_KEY, conf.get(MEMSTORE_MEMORY_SIZE_KEY), MEMSTORE_SIZE_KEY, + conf.get(MEMSTORE_SIZE_KEY), HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY, conf.get(HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY), HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, conf.get(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY), HBASE_REGION_SERVER_FREE_HEAP_MIN_MEMORY_SIZE_KEY, @@ -152,14 +157,36 @@ public static float getRegionServerMinFreeHeapFraction(final Configuration conf) /** * Retrieve global memstore configured size as percentage of total heap. */ - public static float getGlobalMemStoreHeapPercent(final Configuration c, + public static float getGlobalMemStoreHeapPercent(final Configuration conf, final boolean logInvalid) { + // Check if an explicit memstore size is configured. + long memStoreSizeInBytes = getMemstoreSizeInBytes(conf); + if (memStoreSizeInBytes > 0) { + final MemoryUsage usage = safeGetHeapMemoryUsage(); + if (usage != null) { + float memStoreSizeFraction = (float) memStoreSizeInBytes / usage.getMax(); + if ( + memStoreSizeFraction > MEMSTORE_HEAP_MIN_FRACTION + && memStoreSizeFraction <= MEMSTORE_HEAP_MAX_FRACTION + ) { + return memStoreSizeFraction; + } else if (logInvalid) { + LOG.warn( + "Setting global memstore memory size to {} bytes ({} of max heap {}) is outside " + + "allowed range of ({} -> {}]; using configured percentage instead.", + memStoreSizeInBytes, memStoreSizeFraction, usage.getMax(), MEMSTORE_HEAP_MIN_FRACTION, + MEMSTORE_HEAP_MAX_FRACTION); + } + } + } + float limit = - c.getFloat(MEMSTORE_SIZE_KEY, c.getFloat(MEMSTORE_SIZE_OLD_KEY, DEFAULT_MEMSTORE_SIZE)); - if (limit > 0.8f || limit <= 0.0f) { + conf.getFloat(MEMSTORE_SIZE_KEY, conf.getFloat(MEMSTORE_SIZE_OLD_KEY, DEFAULT_MEMSTORE_SIZE)); + if (limit > MEMSTORE_HEAP_MAX_FRACTION || limit <= MEMSTORE_HEAP_MIN_FRACTION) { if (logInvalid) { LOG.warn("Setting global memstore limit to default of " + DEFAULT_MEMSTORE_SIZE - + " because supplied value outside allowed range of (0 -> 0.8]"); + + " because supplied value outside allowed range of (" + MEMSTORE_HEAP_MIN_FRACTION + + " -> " + MEMSTORE_HEAP_MAX_FRACTION + "]"); } limit = DEFAULT_MEMSTORE_SIZE; } @@ -205,17 +232,17 @@ public static float getGlobalMemStoreHeapLowerMark(final Configuration conf, public static Pair getGlobalMemStoreSize(Configuration conf) { long offheapMSGlobal = conf.getLong(OFFHEAP_MEMSTORE_SIZE_KEY, 0);// Size in MBs if (offheapMSGlobal > 0) { - // Off heap memstore size has not relevance when MSLAB is turned OFF. We will go with making - // this entire size split into Chunks and pooling them in MemstoreLABPoool. We dont want to + // Off heap memstore size has no relevance when MSLAB is turned OFF. We will go with making + // this entire size split into Chunks and pooling them in MemstoreLABPool. We don't want to // create so many on demand off heap chunks. In fact when this off heap size is configured, we // will go with 100% of this size as the pool size if (MemStoreLAB.isEnabled(conf)) { - // We are in offheap Memstore use - long globalMemStoreLimit = (long) (offheapMSGlobal * 1024 * 1024); // Size in bytes + // We are in off heap memstore use + long globalMemStoreLimit = offheapMSGlobal * 1024 * 1024; // Size in bytes return new Pair<>(globalMemStoreLimit, MemoryType.NON_HEAP); } else { // Off heap max memstore size is configured with turning off MSLAB. It makes no sense. Do a - // warn log and go with on heap memstore percentage. By default it will be 40% of Xmx + // warn log and go with on heap memstore percentage. By default, it will be 40% of Xmx LOG.warn("There is no relevance of configuring '" + OFFHEAP_MEMSTORE_SIZE_KEY + "' when '" + MemStoreLAB.USEMSLAB_KEY + "' is turned off." + " Going with on heap global memstore size ('" + MEMSTORE_SIZE_KEY + "')"); @@ -301,6 +328,21 @@ public static long getOnHeapCacheSize(final Configuration conf) { } } + /** + * Retrieve an explicit memstore size in bytes in the configuration. + * @param conf used to read memstore configs + * @return the number of bytes to use for memstore, negative if not configured. + * @throws IllegalArgumentException if {@code MEMSTORE_MEMORY_SIZE_KEY} format is invalid + */ + public static long getMemstoreSizeInBytes(Configuration conf) { + try { + return Long.parseLong(conf.get(MEMSTORE_MEMORY_SIZE_KEY, "-1")); + } catch (NumberFormatException e) { + return (long) StorageSize.getStorageSize(conf.get(MEMSTORE_MEMORY_SIZE_KEY), -1, + StorageUnit.BYTES); + } + } + /** * @param conf used to read config for bucket cache size. * @return the number of bytes to use for bucket cache, negative if disabled. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java index 9c4cd5b3ca45..d90011bacd48 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java @@ -19,6 +19,8 @@ import static org.apache.hadoop.hbase.HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY; import static org.apache.hadoop.hbase.HConstants.HFILE_BLOCK_CACHE_SIZE_KEY; +import static org.apache.hadoop.hbase.io.util.MemorySizeUtil.MEMSTORE_MEMORY_SIZE_KEY; +import static org.apache.hadoop.hbase.io.util.MemorySizeUtil.MEMSTORE_SIZE_KEY; import java.lang.management.MemoryUsage; import java.util.ArrayList; @@ -120,7 +122,7 @@ public class HeapMemoryManager { private ResizableBlockCache toResizableBlockCache(BlockCache blockCache) { if (blockCache instanceof CombinedBlockCache) { - return (ResizableBlockCache) ((CombinedBlockCache) blockCache).getFirstLevelCache(); + return ((CombinedBlockCache) blockCache).getFirstLevelCache(); } else { return (ResizableBlockCache) blockCache; } @@ -137,16 +139,20 @@ private boolean doInit(Configuration conf) { globalMemStorePercentMaxRange = conf.getFloat(MEMSTORE_SIZE_MAX_RANGE_KEY, globalMemStorePercent); if (globalMemStorePercent < globalMemStorePercentMinRange) { - LOG.warn("Setting " + MEMSTORE_SIZE_MIN_RANGE_KEY + " to " + globalMemStorePercent - + ", same value as " + MemorySizeUtil.MEMSTORE_SIZE_KEY - + " because supplied value greater than initial memstore size value."); + LOG.warn( + "Setting {} to {} (lookup order: {} -> {}), " + + "because supplied value greater than initial memstore size.", + MEMSTORE_SIZE_MIN_RANGE_KEY, globalMemStorePercent, MEMSTORE_MEMORY_SIZE_KEY, + MEMSTORE_SIZE_KEY); globalMemStorePercentMinRange = globalMemStorePercent; conf.setFloat(MEMSTORE_SIZE_MIN_RANGE_KEY, globalMemStorePercentMinRange); } if (globalMemStorePercent > globalMemStorePercentMaxRange) { - LOG.warn("Setting " + MEMSTORE_SIZE_MAX_RANGE_KEY + " to " + globalMemStorePercent - + ", same value as " + MemorySizeUtil.MEMSTORE_SIZE_KEY - + " because supplied value less than initial memstore size value."); + LOG.warn( + "Setting {} to {} (lookup order: {} -> {}), " + + "because supplied value less than initial memstore size.", + MEMSTORE_SIZE_MAX_RANGE_KEY, globalMemStorePercent, MEMSTORE_MEMORY_SIZE_KEY, + MEMSTORE_SIZE_KEY); globalMemStorePercentMaxRange = globalMemStorePercent; conf.setFloat(MEMSTORE_SIZE_MAX_RANGE_KEY, globalMemStorePercentMaxRange); } @@ -376,8 +382,8 @@ private void tune() { LOG.info("Current heap configuration from HeapMemoryTuner exceeds " + "the allowed heap usage. At least " + minFreeHeapFraction + " of the heap must remain free to ensure stable RegionServer operation. " - + MemorySizeUtil.MEMSTORE_SIZE_KEY + " is " + memstoreSize + " and " - + HFILE_BLOCK_CACHE_SIZE_KEY + " is " + blockCacheSize); + + MEMSTORE_SIZE_KEY + " is " + memstoreSize + " and " + HFILE_BLOCK_CACHE_SIZE_KEY + + " is " + blockCacheSize); // NOTE: In the future, we might adjust values to not exceed limits, // but for now tuning is skipped if over threshold. } else { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/util/TestMemorySizeUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/util/TestMemorySizeUtil.java index 5f00c34dbcb0..3c7bf4b04ce3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/util/TestMemorySizeUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/util/TestMemorySizeUtil.java @@ -17,29 +17,25 @@ */ package org.apache.hadoop.hbase.io.util; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; -@Category({ MiscTests.class, SmallTests.class }) +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) public class TestMemorySizeUtil { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMemorySizeUtil.class); - private Configuration conf; - @Before + @BeforeEach public void setup() { conf = new Configuration(); } @@ -86,4 +82,37 @@ public void testGetRegionServerMinFreeHeapFraction() { minFreeHeapFraction = MemorySizeUtil.getRegionServerMinFreeHeapFraction(conf); assertEquals(0.0f, minFreeHeapFraction, 0.0f); } + + @Test + public void testGetMemstoreSizeInBytes() { + // when memstore size is not set, it should return -1 + long memstoreSizeInBytes = MemorySizeUtil.getMemstoreSizeInBytes(conf); + assertEquals(-1, memstoreSizeInBytes); + + long expectedMemstoreSizeInBytes = 123456L; + conf.setLong(MemorySizeUtil.MEMSTORE_MEMORY_SIZE_KEY, expectedMemstoreSizeInBytes); + memstoreSizeInBytes = MemorySizeUtil.getMemstoreSizeInBytes(conf); + assertEquals(expectedMemstoreSizeInBytes, memstoreSizeInBytes); + + conf.set(MemorySizeUtil.MEMSTORE_MEMORY_SIZE_KEY, "10m"); + memstoreSizeInBytes = MemorySizeUtil.getMemstoreSizeInBytes(conf); + assertEquals(10 * 1024 * 1024, memstoreSizeInBytes); + + conf.set(MemorySizeUtil.MEMSTORE_MEMORY_SIZE_KEY, "2GB"); + memstoreSizeInBytes = MemorySizeUtil.getMemstoreSizeInBytes(conf); + assertEquals(2L * 1024 * 1024 * 1024, memstoreSizeInBytes); + } + + @Test + public void testGetGlobalMemStoreHeapPercent() { + // set memstore size to a small value + conf.setLong(MemorySizeUtil.MEMSTORE_MEMORY_SIZE_KEY, 1); + conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.4f); + conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.5f); + assertEquals(HConstants.HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD, 0.2f, 0.0f); + float globalMemStoreHeapPercent = MemorySizeUtil.getGlobalMemStoreHeapPercent(conf, true); + assertTrue(globalMemStoreHeapPercent > 0.0f); + assertTrue(globalMemStoreHeapPercent < 0.4f); + MemorySizeUtil.validateRegionServerHeapMemoryAllocation(conf); + } }