From 93d39a7473983fdb02edb0129135b95e9f9341e9 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 13 Nov 2025 11:15:43 +0800 Subject: [PATCH 1/3] Added memory calculation for tablet (#636) * fx * fix * dl --- .../tsfile/utils/RamUsageEstimator.java | 76 +++++++++++++++++++ .../apache/tsfile/write/record/Tablet.java | 72 +++++++++++++++++- 2 files changed, 146 insertions(+), 2 deletions(-) diff --git a/java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java b/java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java index af7a8cff4..29b9d085e 100644 --- a/java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java +++ b/java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java @@ -27,6 +27,7 @@ import java.security.PrivilegedAction; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -35,6 +36,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -99,6 +101,11 @@ private RamUsageEstimator() {} /** Sizes of primitive classes. */ public static final Map, Integer> primitiveSizes; + public static final long LOCAL_DATE_ARRAY_SIZE; + public static final long LOCAL_DATE_SIZE; + public static final long BIT_MAP_SIZE; + public static final long SIZE_OF_ARRAYLIST; + static { Map, Integer> primitiveSizesMap = new IdentityHashMap<>(); primitiveSizesMap.put(boolean.class, 1); @@ -111,6 +118,11 @@ private RamUsageEstimator() {} primitiveSizesMap.put(long.class, Long.BYTES); primitiveSizes = Collections.unmodifiableMap(primitiveSizesMap); + + LOCAL_DATE_ARRAY_SIZE = RamUsageEstimator.shallowSizeOf(LocalDate[].class); + LOCAL_DATE_SIZE = RamUsageEstimator.shallowSizeOf(LocalDate.class); + BIT_MAP_SIZE = RamUsageEstimator.shallowSizeOfInstance(BitMap.class); + SIZE_OF_ARRAYLIST = RamUsageEstimator.shallowSizeOfInstance(ArrayList.class); } /** JVMs typically cache small longs. This tries to find out what the range is. */ @@ -654,4 +666,68 @@ public static long sizeOfDoubleArray(int length) { public static long sizeOfObjectArray(int length) { return alignObjectSize(NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_OBJECT_REF * length); } + + private static long sizeOf(final Binary binary) { + return Objects.nonNull(binary) ? binary.ramBytesUsed() : 0L; + } + + public static long sizeOf(final Binary[] binaries) { + if (binaries == null) { + return 0L; + } + + long size = 0L; + for (Binary binary : binaries) { + size += sizeOf(binary); + } + + return size + RamUsageEstimator.shallowSizeOf(binaries); + } + + public static long sizeOfStringArray(final String[] values) { + return Objects.nonNull(values) ? RamUsageEstimator.sizeOf(values) : 0L; + } + + public static long sizeOf(BitMap[] bitMaps) { + if (bitMaps == null) { + return 0L; + } + long size = + RamUsageEstimator.alignObjectSize( + NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_OBJECT_REF * bitMaps.length); + for (BitMap bitMap : bitMaps) { + size += sizeOf(bitMap); + } + return size; + } + + private static long sizeOf(final BitMap bitMap) { + if (bitMap == null) { + return 0L; + } + long size = BIT_MAP_SIZE; + + size += + RamUsageEstimator.alignObjectSize(NUM_BYTES_ARRAY_HEADER + bitMap.getByteArray().length); + return size; + } + + public static long sizeOf(final LocalDate[] input) { + if (Objects.isNull(input)) { + return 0; + } + long size = + RamUsageEstimator.alignObjectSize( + LOCAL_DATE_ARRAY_SIZE + (long) input.length * NUM_BYTES_OBJECT_REF); + for (final LocalDate date : input) { + if (Objects.nonNull(date)) { + size += LOCAL_DATE_SIZE; + } + } + return size; + } + + public static long shallowSizeOfList(final List input) { + return alignObjectSize(SIZE_OF_ARRAYLIST + (long) input.size() * NUM_BYTES_OBJECT_REF); + } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index 9bd8d88f2..a8727a5d2 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -21,11 +21,13 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Accountable; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -40,6 +42,8 @@ import java.util.Map; import java.util.Objects; +import static org.apache.tsfile.utils.RamUsageEstimator.shallowSizeOfList; + /** * A tablet data of one device, the tablet contains multiple measurements of this device that share * the same time column. @@ -50,8 +54,8 @@ * *

Notice: The tablet should not have empty cell, please use BitMap to denote null value */ -public class Tablet { - +public class Tablet implements Accountable { + private static final long TABLET_SIZE = RamUsageEstimator.shallowSizeOfInstance(Tablet.class); private static final int DEFAULT_SIZE = 1024; private static final String NOT_SUPPORT_DATATYPE = "Data type %s is not supported."; @@ -862,4 +866,68 @@ private boolean isBitMapsEqual(BitMap[] thisBitMaps, BitMap[] thatBitMaps, int c } return true; } + + @Override + public long ramBytesUsed() { + long totalSizeInBytes = + TABLET_SIZE + + RamUsageEstimator.sizeOf(insertTargetName) + + RamUsageEstimator.sizeOf(timestamps) + + shallowSizeOfList(columnCategories) + + RamUsageEstimator.sizeOf(bitMaps) + + RamUsageEstimator.shallowSizeOfList(tagColumnIndexes) + + RamUsageEstimator.sizeOfMap(measurementIndex); + + // values + final List timeSeries = schemas; + + if (timeSeries != null) { + totalSizeInBytes += RamUsageEstimator.shallowSizeOfList(timeSeries); + for (int column = 0; column < timeSeries.size(); column++) { + final IMeasurementSchema measurementSchema = timeSeries.get(column); + if (measurementSchema == null) { + continue; + } + // Measurement schema size + totalSizeInBytes += 75; + + final TSDataType tsDataType = measurementSchema.getType(); + if (tsDataType == null) { + continue; + } + + if (values == null || values.length <= column) { + continue; + } + switch (tsDataType) { + case INT64: + case TIMESTAMP: + totalSizeInBytes += RamUsageEstimator.sizeOf((long[]) values[column]); + break; + case DATE: + totalSizeInBytes += RamUsageEstimator.sizeOf((LocalDate[]) values[column]); + break; + case INT32: + totalSizeInBytes += RamUsageEstimator.sizeOf((int[]) values[column]); + break; + case DOUBLE: + totalSizeInBytes += RamUsageEstimator.sizeOf((double[]) values[column]); + break; + case FLOAT: + totalSizeInBytes += RamUsageEstimator.sizeOf((float[]) values[column]); + break; + case BOOLEAN: + totalSizeInBytes += RamUsageEstimator.sizeOf((boolean[]) values[column]); + break; + case STRING: + case TEXT: + case BLOB: + totalSizeInBytes += RamUsageEstimator.sizeOf((Binary[]) values[column]); + break; + } + } + } + + return totalSizeInBytes; + } } From 7f8794ff22b02f24933064a5efc15423654e30e1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 13 Nov 2025 14:35:29 +0800 Subject: [PATCH 2/3] fix --- .../main/java/org/apache/tsfile/write/record/Tablet.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index a8727a5d2..7bbc2e3a2 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -871,20 +871,18 @@ private boolean isBitMapsEqual(BitMap[] thisBitMaps, BitMap[] thatBitMaps, int c public long ramBytesUsed() { long totalSizeInBytes = TABLET_SIZE - + RamUsageEstimator.sizeOf(insertTargetName) + + RamUsageEstimator.sizeOf(deviceId) + RamUsageEstimator.sizeOf(timestamps) - + shallowSizeOfList(columnCategories) + RamUsageEstimator.sizeOf(bitMaps) - + RamUsageEstimator.shallowSizeOfList(tagColumnIndexes) + RamUsageEstimator.sizeOfMap(measurementIndex); // values - final List timeSeries = schemas; + final List timeSeries = schemas; if (timeSeries != null) { totalSizeInBytes += RamUsageEstimator.shallowSizeOfList(timeSeries); for (int column = 0; column < timeSeries.size(); column++) { - final IMeasurementSchema measurementSchema = timeSeries.get(column); + final MeasurementSchema measurementSchema = timeSeries.get(column); if (measurementSchema == null) { continue; } From fd1645b6c5c1aed909b94f967f81f3670460d659 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 13 Nov 2025 14:49:20 +0800 Subject: [PATCH 3/3] Update Tablet.java --- .../src/main/java/org/apache/tsfile/write/record/Tablet.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index 7bbc2e3a2..27ed30a60 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -42,8 +42,6 @@ import java.util.Map; import java.util.Objects; -import static org.apache.tsfile.utils.RamUsageEstimator.shallowSizeOfList; - /** * A tablet data of one device, the tablet contains multiple measurements of this device that share * the same time column.