Skip to content

Commit c8a7b5f

Browse files
shuwenweiJackieTien97
shuwenwei
authored and
JackieTien97
committed
correct metadata "modified" flags and related logic (#17172)
1 parent f6d9d71 commit c8a7b5f

File tree

10 files changed

+89
-210
lines changed

10 files changed

+89
-210
lines changed

‎integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterTimeSeriesTypeIT.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@
2626
import org.apache.iotdb.isession.SessionDataSet;
2727
import org.apache.iotdb.it.env.EnvFactory;
2828
import org.apache.iotdb.it.framework.IoTDBTestRunner;
29-
import org.apache.iotdb.itbase.category.TableClusterIT;
30-
import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
29+
import org.apache.iotdb.itbase.category.ClusterIT;
30+
import org.apache.iotdb.itbase.category.LocalStandaloneIT;
3131
import org.apache.iotdb.rpc.IoTDBConnectionException;
3232
import org.apache.iotdb.rpc.StatementExecutionException;
3333

@@ -87,7 +87,7 @@
8787

8888
@SuppressWarnings("ResultOfMethodCallIgnored")
8989
@RunWith(IoTDBTestRunner.class)
90-
@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
90+
@Category({LocalStandaloneIT.class, ClusterIT.class})
9191
public class IoTDBAlterTimeSeriesTypeIT {
9292

9393
private static final Logger log = LoggerFactory.getLogger(IoTDBAlterTimeSeriesTypeIT.class);

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java‎

Lines changed: 19 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@
4040
import org.apache.iotdb.db.utils.SchemaUtils;
4141

4242
import org.apache.tsfile.enums.TSDataType;
43-
import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
4443
import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata;
4544
import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata;
4645
import org.apache.tsfile.file.metadata.IChunkMetadata;
@@ -113,13 +112,11 @@ public static TimeseriesMetadata loadTimeSeriesMetadata(
113112
context.isDebug(),
114113
context);
115114
if (timeSeriesMetadata != null) {
116-
SchemaUtils.changeMetadataModified(timeSeriesMetadata, seriesPath.getSeriesType());
117115
long t2 = System.nanoTime();
118116
List<ModEntry> pathModifications =
119117
context.getPathModifications(
120118
resource, seriesPath.getDeviceId(), seriesPath.getMeasurement());
121-
timeSeriesMetadata.setModified(
122-
timeSeriesMetadata.isModified() || !pathModifications.isEmpty());
119+
timeSeriesMetadata.setModified(!pathModifications.isEmpty());
123120
timeSeriesMetadata.setChunkMetadataLoader(
124121
new DiskChunkMetadataLoader(resource, context, globalTimeFilter, pathModifications));
125122
int modificationCount = pathModifications.size();
@@ -141,13 +138,14 @@ public static TimeseriesMetadata loadTimeSeriesMetadata(
141138
timeSeriesMetadata =
142139
(TimeseriesMetadata) resource.getTimeSeriesMetadata(seriesPath, globalTimeFilter);
143140
if (timeSeriesMetadata != null) {
144-
SchemaUtils.changeMetadataModified(timeSeriesMetadata, seriesPath.getSeriesType());
145141
timeSeriesMetadata.setChunkMetadataLoader(
146142
new MemChunkMetadataLoader(resource, seriesPath, context, globalTimeFilter));
147143
}
148144
}
149145

150146
if (timeSeriesMetadata != null) {
147+
SchemaUtils.changeTimeseriesMetadataModified(
148+
timeSeriesMetadata, seriesPath.getSeriesType());
151149
if (timeSeriesMetadata.getStatistics().getStartTime()
152150
> timeSeriesMetadata.getStatistics().getEndTime()) {
153151
return null;
@@ -209,15 +207,12 @@ public static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadata(
209207
alignedTimeSeriesMetadata =
210208
loadAlignedTimeSeriesMetadataFromDisk(
211209
resource, alignedPath, context, globalTimeFilter, ignoreAllNullRows);
212-
SchemaUtils.changeAlignedMetadataModified(alignedTimeSeriesMetadata, targetDataTypeList);
213210
} else { // if the tsfile is unclosed, we just get it directly from TsFileResource
214211
loadFromMem = true;
215212
alignedTimeSeriesMetadata =
216213
(AbstractAlignedTimeSeriesMetadata)
217214
resource.getTimeSeriesMetadata(alignedPath, globalTimeFilter);
218215
if (alignedTimeSeriesMetadata != null) {
219-
SchemaUtils.changeAlignedMetadataModified(alignedTimeSeriesMetadata, targetDataTypeList);
220-
221216
alignedTimeSeriesMetadata.setChunkMetadataLoader(
222217
new MemAlignedChunkMetadataLoader(
223218
resource, alignedPath, context, globalTimeFilter, ignoreAllNullRows));
@@ -226,6 +221,8 @@ public static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadata(
226221
}
227222

228223
if (alignedTimeSeriesMetadata != null) {
224+
SchemaUtils.changeAlignedTimeseriesMetadataModified(
225+
alignedTimeSeriesMetadata, targetDataTypeList);
229226
if (alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime()
230227
> alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime()) {
231228
return null;
@@ -320,25 +317,18 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr
320317
new ArrayList<>(valueMeasurementList.size());
321318
// if all the queried aligned sensors does not exist, we will return null
322319
boolean exist = false;
323-
int i = 0;
324-
for (String valueMeasurement : valueMeasurementList) {
320+
for (String measurement : valueMeasurementList) {
325321
TimeseriesMetadata valueColumn =
326322
cache.get(
327323
filePath,
328-
new TimeSeriesMetadataCacheKey(
329-
resource.getTsFileID(), deviceId, valueMeasurement),
324+
new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, measurement),
330325
allSensors,
331326
context.ignoreNotExistsDevice()
332327
|| resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE,
333328
isDebug,
334329
context);
335330
exist = (exist || (valueColumn != null));
336-
if (valueColumn != null) {
337-
SchemaUtils.changeAlignedMetadataModified(
338-
valueColumn, alignedPath.getSchemaList().get(i).getType());
339-
}
340331
valueTimeSeriesMetadataList.add(valueColumn);
341-
i++;
342332
}
343333
if (!ignoreAllNullRows || exist) {
344334
alignedTimeSeriesMetadata =
@@ -458,55 +448,23 @@ public static List<IChunkMetadata> loadChunkMetadataList(ITimeSeriesMetadata tim
458448
* IOException will be thrown
459449
*/
460450
public static List<IPageReader> loadPageReaderList(
461-
IChunkMetadata chunkMetaData,
462-
Filter globalTimeFilter,
463-
boolean isAligned,
464-
List<TSDataType> targetDataTypeList)
451+
IChunkMetadata chunkMetaData, Filter globalTimeFilter, List<TSDataType> targetDataTypeList)
465452
throws IOException {
466453
checkArgument(chunkMetaData != null, "Can't init null chunkMeta");
467454

455+
IChunkLoader chunkLoader = chunkMetaData.getChunkLoader();
468456
IChunkReader chunkReader;
469-
boolean isModified = false;
470-
if (isAligned) {
471-
AbstractAlignedChunkMetadata alignedChunkMetadata =
472-
(AbstractAlignedChunkMetadata) chunkMetaData;
473-
for (int i = 0; i < alignedChunkMetadata.getValueChunkMetadataList().size(); i++) {
474-
if (alignedChunkMetadata.getValueChunkMetadataList().get(i) != null) {
475-
if (!SchemaUtils.isUsingSameStatistics(
476-
alignedChunkMetadata.getValueChunkMetadataList().get(i).getDataType(),
477-
targetDataTypeList.get(i))
478-
&& !SchemaUtils.canUseStatisticsAfterAlter(targetDataTypeList.get(i))) {
479-
isModified = true;
480-
alignedChunkMetadata.getValueChunkMetadataList().get(i).setModified(true);
481-
}
482-
}
483-
}
484-
IChunkLoader chunkLoader = alignedChunkMetadata.getChunkLoader();
485-
chunkReader = chunkLoader.getChunkReader(alignedChunkMetadata, globalTimeFilter);
486-
} else {
487-
if (!SchemaUtils.isUsingSameStatistics(chunkMetaData.getDataType(), targetDataTypeList.get(0))
488-
&& !SchemaUtils.canUseStatisticsAfterAlter(targetDataTypeList.get(0))) {
489-
isModified = true;
490-
chunkMetaData.setModified(true);
491-
}
492-
IChunkLoader chunkLoader = chunkMetaData.getChunkLoader();
493-
try {
494-
chunkReader = chunkLoader.getChunkReader(chunkMetaData, globalTimeFilter);
495-
} catch (ChunkTypeInconsistentException e) {
496-
// if the chunk in tsfile is a value chunk of aligned series, we should skip all data of
497-
// this chunk.
498-
return Collections.emptyList();
499-
}
457+
try {
458+
chunkReader = chunkLoader.getChunkReader(chunkMetaData, globalTimeFilter);
459+
} catch (ChunkTypeInconsistentException e) {
460+
// if the chunk in tsfile is a value chunk of aligned series but registered series is
461+
// non-aligned, we should skip all data of this chunk.
462+
return Collections.emptyList();
500463
}
501-
502-
return isModified
503-
? chunkReader.loadPageReaderList().stream()
504-
.peek(
505-
iPageReader -> {
506-
iPageReader.setModified(true);
507-
})
508-
.collect(Collectors.toList())
509-
: chunkReader.loadPageReaderList();
464+
if (chunkMetaData.isDataTypeModifiedAndCannotUseStatistics()) {
465+
chunkReader.markDataTypeModifiedAndCannotUseStatistics();
466+
}
467+
return chunkReader.loadPageReaderList();
510468
}
511469

512470
/**

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java‎

Lines changed: 2 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,6 @@
4545
import org.apache.tsfile.block.column.Column;
4646
import org.apache.tsfile.common.conf.TSFileDescriptor;
4747
import org.apache.tsfile.enums.TSDataType;
48-
import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
49-
import org.apache.tsfile.file.metadata.ChunkMetadata;
5048
import org.apache.tsfile.file.metadata.IChunkMetadata;
5149
import org.apache.tsfile.file.metadata.IDeviceID;
5250
import org.apache.tsfile.file.metadata.IMetadata;
@@ -453,15 +451,6 @@ private void initFirstChunkMetadata() throws IOException {
453451
orderUtils.getOverlapCheckTime(firstChunkMetadata.getStatistics()));
454452
unpackAllOverlappedTimeSeriesMetadataToCachedChunkMetadata(
455453
orderUtils.getOverlapCheckTime(firstChunkMetadata.getStatistics()), false);
456-
if (isAligned) {
457-
SchemaUtils.changeAlignedMetadataModified(
458-
(AbstractAlignedChunkMetadata) firstChunkMetadata,
459-
firstChunkMetadata.getDataType(),
460-
getTsDataTypeList());
461-
} else {
462-
SchemaUtils.changeMetadataModified(
463-
firstChunkMetadata, firstChunkMetadata.getDataType(), dataType);
464-
}
465454
if (firstChunkMetadata.equals(cachedChunkMetadata.peek())) {
466455
firstChunkMetadata = cachedChunkMetadata.poll();
467456
break;
@@ -489,45 +478,13 @@ private void unpackAllOverlappedTimeSeriesMetadataToCachedChunkMetadata(
489478

490479
if (init && firstChunkMetadata == null && !cachedChunkMetadata.isEmpty()) {
491480
firstChunkMetadata = cachedChunkMetadata.poll();
492-
if (isAligned) {
493-
SchemaUtils.changeAlignedMetadataModified(
494-
(AbstractAlignedChunkMetadata) firstChunkMetadata,
495-
firstChunkMetadata.getDataType(),
496-
getTsDataTypeList());
497-
} else {
498-
SchemaUtils.changeMetadataModified(
499-
firstChunkMetadata, firstChunkMetadata.getDataType(), dataType);
500-
}
501481
}
502482
}
503483

504484
protected void unpackOneTimeSeriesMetadata(ITimeSeriesMetadata timeSeriesMetadata) {
505485
List<IChunkMetadata> chunkMetadataList =
506486
FileLoaderUtils.loadChunkMetadataList(timeSeriesMetadata);
507-
chunkMetadataList.forEach(
508-
chunkMetadata -> {
509-
if (chunkMetadata instanceof AbstractAlignedChunkMetadata) {
510-
AbstractAlignedChunkMetadata alignedChunkMetadata =
511-
(AbstractAlignedChunkMetadata) chunkMetadata;
512-
for (int i = 0; i < alignedChunkMetadata.getValueChunkMetadataList().size(); i++) {
513-
if ((alignedChunkMetadata.getValueChunkMetadataList().get(i) != null)
514-
&& !SchemaUtils.isUsingSameStatistics(
515-
alignedChunkMetadata.getValueChunkMetadataList().get(i).getDataType(),
516-
getTsDataTypeList().get(i))
517-
&& !SchemaUtils.canUseStatisticsAfterAlter(getTsDataTypeList().get(i))) {
518-
alignedChunkMetadata.getValueChunkMetadataList().get(i).setModified(true);
519-
}
520-
}
521-
chunkMetadata = alignedChunkMetadata;
522-
} else if (chunkMetadata instanceof ChunkMetadata) {
523-
if (!SchemaUtils.isUsingSameStatistics(
524-
chunkMetadata.getDataType(), getTsDataTypeList().get(0))
525-
&& !SchemaUtils.canUseStatisticsAfterAlter(getTsDataTypeList().get(0))) {
526-
chunkMetadata.setModified(true);
527-
}
528-
}
529-
chunkMetadata.setSeq(timeSeriesMetadata.isSeq());
530-
});
487+
chunkMetadataList.forEach(chunkMetadata -> chunkMetadata.setSeq(timeSeriesMetadata.isSeq()));
531488

532489
cachedChunkMetadata.addAll(chunkMetadataList);
533490
}
@@ -704,7 +661,7 @@ private void unpackOneChunkMetaData(IChunkMetadata chunkMetaData) throws IOExcep
704661
}
705662
List<IPageReader> pageReaderList =
706663
FileLoaderUtils.loadPageReaderList(
707-
chunkMetaData, scanOptions.getGlobalTimeFilter(), isAligned, getTsDataTypeList());
664+
chunkMetaData, scanOptions.getGlobalTimeFilter(), getTsDataTypeList());
708665

709666
// init TsBlockBuilder for each page reader
710667
pageReaderList.forEach(p -> p.initTsBlockBuilder(getTsDataTypeList()));

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -364,7 +364,7 @@ private static void rewriteStatistics(
364364
TSDataType targetDataType = measurementMap.get(measurement);
365365
if (valueChunkMetadata.getDataType() != targetDataType) {
366366
SchemaUtils.rewriteAlignedChunkMetadataStatistics(alignedChunkMetadata, i, targetDataType);
367-
alignedChunkMetadata.setModified(true);
367+
alignedChunkMetadata.setDataTypeModifiedAndCannotUseStatistics(true);
368368
}
369369
}
370370
}
@@ -571,7 +571,7 @@ public ITimeSeriesMetadata generateTimeSeriesMetadata(
571571
return null;
572572
}
573573
chunkMetadataList.set(index, chunkMetadata);
574-
chunkMetadata.setModified(true);
574+
chunkMetadata.setDataTypeModifiedAndCannotUseStatistics(true);
575575
}
576576
if (!useFakeStatistics) {
577577
if (chunkMetadata != null && targetDataType.isCompatible(chunkMetadata.getDataType())) {
@@ -597,7 +597,7 @@ public ITimeSeriesMetadata generateTimeSeriesMetadata(
597597
return null;
598598
}
599599
memChunk.setChunkMetadata(rewritedChunkMetadata);
600-
memChunk.getChunkMetaData().setModified(true);
600+
memChunk.getChunkMetaData().setDataTypeModifiedAndCannotUseStatistics(true);
601601
}
602602
if (useFakeStatistics) {
603603
memChunk.initChunkMetaFromTVListsWithFakeStatistics();

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,13 @@ public void close() {
8585
// Do nothing because mem chunk reader will not open files
8686
}
8787

88+
@Override
89+
public void markDataTypeModifiedAndCannotUseStatistics() {
90+
for (IPageReader iPageReader : pageReaderList) {
91+
iPageReader.setModified(true);
92+
}
93+
}
94+
8895
@Override
8996
public List<IPageReader> loadPageReaderList() {
9097
return this.pageReaderList;

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ public class MemAlignedPageReader implements IPageReader {
6060
private Filter recordFilter;
6161
private PaginationController paginationController = UNLIMITED_PAGINATION_CONTROLLER;
6262

63+
// data type is modified in query and statistics cannot be used
64+
private boolean modified;
65+
6366
private TsBlockBuilder builder;
6467

6568
public MemAlignedPageReader(
@@ -214,11 +217,13 @@ public void setLimitOffset(PaginationController paginationController) {
214217

215218
@Override
216219
public boolean isModified() {
217-
return false;
220+
return modified;
218221
}
219222

220223
@Override
221-
public void setModified(boolean modified) {}
224+
public void setModified(boolean modified) {
225+
this.modified = modified;
226+
}
222227

223228
@Override
224229
public void initTsBlockBuilder(List<TSDataType> dataTypes) {

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemChunkReader.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,13 @@ public void close() {
129129
// Do nothing because mem chunk reader will not open files
130130
}
131131

132+
@Override
133+
public void markDataTypeModifiedAndCannotUseStatistics() {
134+
for (IPageReader pageReader : this.pageReaderList) {
135+
pageReader.setModified(true);
136+
}
137+
}
138+
132139
@Override
133140
public List<IPageReader> loadPageReaderList() {
134141
return this.pageReaderList;

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,9 @@ public class MemPageReader implements IPageReader {
5757

5858
private PaginationController paginationController = UNLIMITED_PAGINATION_CONTROLLER;
5959

60+
// data type is modified in query and statistics cannot be used
61+
private boolean modified;
62+
6063
public MemPageReader(
6164
Supplier<TsBlock> tsBlockSupplier,
6265
int pageIndex,
@@ -223,11 +226,13 @@ public void setLimitOffset(PaginationController paginationController) {
223226

224227
@Override
225228
public boolean isModified() {
226-
return false;
229+
return modified;
227230
}
228231

229232
@Override
230-
public void setModified(boolean modified) {}
233+
public void setModified(boolean modified) {
234+
this.modified = modified;
235+
}
231236

232237
@Override
233238
public void initTsBlockBuilder(List<TSDataType> dataTypes) {

0 commit comments

Comments
 (0)