From 96da70f1a563a585720bf3c890ed33f48161e492 Mon Sep 17 00:00:00 2001 From: zyk990424 <1534661820@qq.com> Date: Wed, 2 Mar 2022 16:19:23 +0800 Subject: [PATCH 1/4] implement MManager flush --- .../apache/iotdb/db/metadata/MManager.java | 15 +++ .../iotdb/db/metadata/MetadataConstant.java | 1 + .../metadata/mtree/service/MTreeService.java | 4 + .../mtree/store/CachedMTreeStore.java | 12 +- .../db/metadata/mtree/store/IMTreeStore.java | 2 + .../metadata/mtree/store/MemMTreeStore.java | 4 + .../iotdb/db/metadata/tag/TagLogFile.java | 4 + .../iotdb/db/metadata/tag/TagManager.java | 4 + .../metadata/template/TemplateFileReader.java | 46 +++++++ .../metadata/template/TemplateFileWriter.java | 64 ++++++++++ .../db/metadata/template/TemplateManager.java | 118 +++++++++++++++++- 11 files changed, 269 insertions(+), 5 deletions(-) create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateFileReader.java create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateFileWriter.java diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java index 9cc34667c31f8..bd9d5d8e9645d 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java @@ -292,6 +292,7 @@ public synchronized void init() { try { isRecovering = true; + templateManager.init(); tagManager.init(); mtree = new MTreeService(); mtree.init(); @@ -552,6 +553,20 @@ public void operation(PhysicalPlan plan) throws IOException, MetadataException { logger.error("Unrecognizable command {}", plan.getOperatorType()); } } + + public void flushMetadata() { + if (!config.isEnablePersistentSchema()) { + return; + } + try { + templateManager.sync(); + tagManager.sync(); + mtree.sync(); + logWriter.clear(); + } catch (MetadataException | IOException e) { + logger.error("Exception occurred while flushing MManager"); + } + } // endregion // region Interfaces for CQ diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MetadataConstant.java b/server/src/main/java/org/apache/iotdb/db/metadata/MetadataConstant.java index 3a8ec8c8a0a10..4df7640176c31 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/MetadataConstant.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/MetadataConstant.java @@ -42,6 +42,7 @@ private MetadataConstant() { MTREE_PREFIX + IoTDBConstant.FILE_NAME_SEPARATOR + MTREE_VERSION + ".snapshot.bin.tmp"; public static final String SCHEMA_FILE_DIR = "pst"; public static final String SCHEMA_FILE_SUFFIX = "pst"; + public static final String TEMPLATE_FILE = "templates.bin"; public static final PartialPath ALL_MATCH_PATTERN = new PartialPath(new String[] {"root", "**"}); diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/service/MTreeService.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/service/MTreeService.java index e2c47c459a9a4..5f65bfeb1878a 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/service/MTreeService.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/service/MTreeService.java @@ -146,6 +146,10 @@ public void init() throws MetadataException, IOException { this.root = store.getRoot(); } + public void sync() throws MetadataException, IOException { + store.sync(); + } + public void clear() { store.clear(); root = store.getRoot(); diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/CachedMTreeStore.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/CachedMTreeStore.java index cfd055ca5ce9a..b06439f7b014a 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/CachedMTreeStore.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/CachedMTreeStore.java @@ -32,6 +32,7 @@ import org.apache.iotdb.db.metadata.mtree.store.disk.cache.MemManager; import org.apache.iotdb.db.metadata.mtree.store.disk.schemafile.ISchemaFileManager; import org.apache.iotdb.db.metadata.mtree.store.disk.schemafile.SFManager; +import org.apache.iotdb.db.service.IoTDB; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -342,6 +343,11 @@ public void unPin(IMNode node) { @Override public void createSnapshot() throws IOException {} + @Override + public void sync() throws MetadataException, IOException { + flushVolatileNodes(); + } + /** clear all the data of MTreeStore in memory and disk. */ @Override public void clear() { @@ -399,7 +405,11 @@ private synchronized void registerFlushTask() { return; } hasFlushTask = true; - flushTask.submit(this::flushVolatileNodes); + flushTask.submit(this::triggerMManagerFlush); + } + + private void triggerMManagerFlush() { + IoTDB.metaManager.flushMetadata(); } /** Sync all volatile nodes to schemaFile and execute memory release after flush. */ diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/IMTreeStore.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/IMTreeStore.java index e75666e2afced..53723e48adea9 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/IMTreeStore.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/IMTreeStore.java @@ -60,6 +60,8 @@ public interface IMTreeStore { void createSnapshot() throws IOException; + void sync() throws MetadataException, IOException; + void clear(); String toString(); diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/MemMTreeStore.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/MemMTreeStore.java index 964690f3766bf..4e889f4a2c694 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/MemMTreeStore.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/MemMTreeStore.java @@ -21,6 +21,7 @@ import org.apache.iotdb.db.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory; +import org.apache.iotdb.db.exception.metadata.MetadataException; import org.apache.iotdb.db.metadata.MetadataConstant; import org.apache.iotdb.db.metadata.logfile.MLogReader; import org.apache.iotdb.db.metadata.logfile.MLogWriter; @@ -193,6 +194,9 @@ public void createSnapshot() throws IOException { } } + @Override + public void sync() throws MetadataException, IOException {} + public void serializeTo(String snapshotPath) throws IOException { try (MLogWriter mLogWriter = new MLogWriter(snapshotPath)) { root.serializeTo(mLogWriter); diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/tag/TagLogFile.java b/server/src/main/java/org/apache/iotdb/db/metadata/tag/TagLogFile.java index cae5aba91cc5e..3a0971b8189aa 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/tag/TagLogFile.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/tag/TagLogFile.java @@ -149,6 +149,10 @@ private void serializeMap(Map map, ByteBuffer byteBuffer) } } + public void sync() throws IOException { + fileChannel.force(true); + } + @Override public void close() throws IOException { fileChannel.force(true); diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/tag/TagManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/tag/TagManager.java index c597b721ad030..ed97e7dc2cb53 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/tag/TagManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/tag/TagManager.java @@ -578,6 +578,10 @@ public Pair, Map> readTagFile(long tagFileOf return tagLogFile.read(config.getTagAttributeTotalSize(), tagFileOffset); } + public void sync() throws IOException { + tagLogFile.sync(); + } + public void clear() throws IOException { this.tagIndex.clear(); if (tagLogFile != null) { diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateFileReader.java b/server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateFileReader.java new file mode 100644 index 0000000000000..e7e93d8c6e866 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateFileReader.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.template; + +import org.apache.iotdb.db.metadata.logfile.MLogReader; +import org.apache.iotdb.db.qp.physical.PhysicalPlan; + +import java.io.IOException; + +public class TemplateFileReader implements AutoCloseable { + + private MLogReader logReader; + + public TemplateFileReader(String filePath) throws IOException { + logReader = new MLogReader(filePath); + } + + public boolean hasNext() { + return logReader.hasNext(); + } + + public PhysicalPlan next() { + return logReader.next(); + } + + @Override + public void close() { + logReader.close(); + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateFileWriter.java b/server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateFileWriter.java new file mode 100644 index 0000000000000..5e29cfa74bf2a --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateFileWriter.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.template; + +import org.apache.iotdb.db.metadata.logfile.MLogWriter; +import org.apache.iotdb.db.qp.physical.sys.AppendTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.CreateTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.DropTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.PruneTemplatePlan; + +import java.io.IOException; + +public class TemplateFileWriter { + + private final MLogWriter logWriter; + + public TemplateFileWriter(String filePath) throws IOException { + logWriter = new MLogWriter(filePath); + } + + public void createSchemaTemplate(CreateTemplatePlan plan) throws IOException { + logWriter.createSchemaTemplate(plan); + } + + public void appendSchemaTemplate(AppendTemplatePlan plan) throws IOException { + logWriter.appendSchemaTemplate(plan); + } + + public void pruneSchemaTemplate(PruneTemplatePlan plan) throws IOException { + logWriter.pruneSchemaTemplate(plan); + } + + public void dropSchemaTemplate(DropTemplatePlan plan) throws IOException { + logWriter.dropSchemaTemplate(plan); + } + + public void force() throws IOException { + logWriter.force(); + } + + public void close() throws IOException { + logWriter.close(); + } + + public void clear() throws IOException { + logWriter.clear(); + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateManager.java index 45d82c14c12cf..e6f4cab21ad71 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateManager.java @@ -19,12 +19,15 @@ package org.apache.iotdb.db.metadata.template; import org.apache.iotdb.db.conf.IoTDBConstant; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.metadata.DuplicatedTemplateException; import org.apache.iotdb.db.exception.metadata.MetadataException; import org.apache.iotdb.db.exception.metadata.UndefinedTemplateException; +import org.apache.iotdb.db.metadata.MetadataConstant; import org.apache.iotdb.db.metadata.mnode.IMNode; import org.apache.iotdb.db.metadata.path.PartialPath; import org.apache.iotdb.db.metadata.utils.MetaFormatUtils; +import org.apache.iotdb.db.qp.physical.PhysicalPlan; import org.apache.iotdb.db.qp.physical.sys.AppendTemplatePlan; import org.apache.iotdb.db.qp.physical.sys.CreateTemplatePlan; import org.apache.iotdb.db.qp.physical.sys.DropTemplatePlan; @@ -34,15 +37,29 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Queue; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; public class TemplateManager { + private static final Logger logger = LoggerFactory.getLogger(TemplateManager.class); + // template name -> template - private Map templateMap = new ConcurrentHashMap<>(); + private final Map templateMap = new ConcurrentHashMap<>(); + + private Queue planBuffer = new LinkedList<>(); + private TemplateFileWriter fileWriter; + + private boolean isRecover; private static class TemplateManagerHolder { @@ -62,7 +79,58 @@ public static TemplateManager getNewInstanceForTest() { return new TemplateManager(); } - private TemplateManager() {} + private TemplateManager() { + isRecover = true; + } + + public void init() throws IOException { + String filePath = + IoTDBDescriptor.getInstance().getConfig().getSchemaDir() + + File.separator + + MetadataConstant.METADATA_LOG; + fileWriter = new TemplateFileWriter(filePath); + recoverFromTemplateFile(filePath); + isRecover = false; + } + + private void recoverFromTemplateFile(String filePath) throws IOException { + TemplateFileReader reader = new TemplateFileReader(filePath); + PhysicalPlan plan; + int idx = 0; + while (reader.hasNext()) { + try { + plan = reader.next(); + idx++; + } catch (Exception e) { + logger.error("Parse TemplateFile error at lineNumber {} because:", idx, e); + break; + } + if (plan == null) { + continue; + } + try { + switch (plan.getOperatorType()) { + case CREATE_TEMPLATE: + createSchemaTemplate((CreateTemplatePlan) plan); + break; + case APPEND_TEMPLATE: + appendSchemaTemplate((AppendTemplatePlan) plan); + break; + case PRUNE_TEMPLATE: + pruneSchemaTemplate((PruneTemplatePlan) plan); + break; + case DROP_TEMPLATE: + dropSchemaTemplate((DropTemplatePlan) plan); + break; + default: + throw new IOException( + "Template file corrupted. Read unknown plan type during recover."); + } + } catch (MetadataException | IOException e) { + logger.error("Can not operate cmd {} in TemplateFile for err:", plan.getOperatorType(), e); + } + } + } public void createSchemaTemplate(CreateTemplatePlan plan) throws MetadataException { // check schema and measurement name before create template @@ -83,10 +151,18 @@ public void createSchemaTemplate(CreateTemplatePlan plan) throws MetadataExcepti // already have template throw new MetadataException("Duplicated template name: " + plan.getName()); } + + if (!isRecover) { + planBuffer.add(plan); + } } - public void dropSchemaTemplate(DropTemplatePlan plan) { + public void dropSchemaTemplate(DropTemplatePlan plan) throws MetadataException { templateMap.remove(plan.getName()); + + if (!isRecover) { + planBuffer.add(plan); + } } public void appendSchemaTemplate(AppendTemplatePlan plan) throws MetadataException { @@ -109,6 +185,10 @@ public void appendSchemaTemplate(AppendTemplatePlan plan) throws MetadataExcepti } else { temp.addUnalignedMeasurements(measurements, dataTypes, encodings, compressionTypes); } + + if (!isRecover) { + planBuffer.add(plan); + } } else { throw new MetadataException("Template does not exists:" + plan.getName()); } @@ -121,6 +201,10 @@ public void pruneSchemaTemplate(PruneTemplatePlan plan) throws MetadataException for (int i = 0; i < plan.getPrunedMeasurements().size(); i++) { temp.deleteSeriesCascade(plan.getPrunedMeasurements().get(i)); } + + if (!isRecover) { + planBuffer.add(plan); + } } else { throw new MetadataException("Template does not exists:" + plan.getName()); } @@ -172,7 +256,33 @@ public void checkIsTemplateCompatible(Template template, IMNode node) throws Met } } - public void clear() { + public void sync() throws MetadataException, IOException { + PhysicalPlan plan; + while (!planBuffer.isEmpty()) { + plan = planBuffer.poll(); + switch (plan.getOperatorType()) { + case CREATE_TEMPLATE: + fileWriter.createSchemaTemplate((CreateTemplatePlan) plan); + break; + case APPEND_TEMPLATE: + fileWriter.appendSchemaTemplate((AppendTemplatePlan) plan); + break; + case PRUNE_TEMPLATE: + fileWriter.pruneSchemaTemplate((PruneTemplatePlan) plan); + break; + case DROP_TEMPLATE: + fileWriter.dropSchemaTemplate((DropTemplatePlan) plan); + break; + default: + break; + } + } + fileWriter.force(); + } + + public void clear() throws IOException { + planBuffer.clear(); templateMap.clear(); + fileWriter.clear(); } } From e079d7c0d510794500a2281fa9b4722f9cbd857b Mon Sep 17 00:00:00 2001 From: zyk990424 <1534661820@qq.com> Date: Thu, 3 Mar 2022 17:13:35 +0800 Subject: [PATCH 2/4] refactor TemplateFile initialization --- .../metadata/template/TemplateFileReader.java | 4 ++-- .../metadata/template/TemplateFileWriter.java | 4 ++-- .../db/metadata/template/TemplateManager.java | 22 ++++++++++--------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateFileReader.java b/server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateFileReader.java index e7e93d8c6e866..6d6fe41eab81a 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateFileReader.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateFileReader.java @@ -27,8 +27,8 @@ public class TemplateFileReader implements AutoCloseable { private MLogReader logReader; - public TemplateFileReader(String filePath) throws IOException { - logReader = new MLogReader(filePath); + public TemplateFileReader(String schemaDir, String fileName) throws IOException { + logReader = new MLogReader(schemaDir, fileName); } public boolean hasNext() { diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateFileWriter.java b/server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateFileWriter.java index 5e29cfa74bf2a..23ec0aa2f8191 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateFileWriter.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateFileWriter.java @@ -30,8 +30,8 @@ public class TemplateFileWriter { private final MLogWriter logWriter; - public TemplateFileWriter(String filePath) throws IOException { - logWriter = new MLogWriter(filePath); + public TemplateFileWriter(String schemaDir, String fileName) throws IOException { + logWriter = new MLogWriter(schemaDir, fileName); } public void createSchemaTemplate(CreateTemplatePlan plan) throws IOException { diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateManager.java index e6f4cab21ad71..1f00fcc93457f 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/template/TemplateManager.java @@ -40,7 +40,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.io.IOException; import java.util.LinkedList; import java.util.List; @@ -84,17 +83,19 @@ private TemplateManager() { } public void init() throws IOException { - String filePath = - IoTDBDescriptor.getInstance().getConfig().getSchemaDir() - + File.separator - + MetadataConstant.METADATA_LOG; - fileWriter = new TemplateFileWriter(filePath); - recoverFromTemplateFile(filePath); + fileWriter = + new TemplateFileWriter( + IoTDBDescriptor.getInstance().getConfig().getSchemaDir(), + MetadataConstant.TEMPLATE_FILE); + recoverFromTemplateFile(); isRecover = false; } - private void recoverFromTemplateFile(String filePath) throws IOException { - TemplateFileReader reader = new TemplateFileReader(filePath); + private void recoverFromTemplateFile() throws IOException { + TemplateFileReader reader = + new TemplateFileReader( + IoTDBDescriptor.getInstance().getConfig().getSchemaDir(), + MetadataConstant.TEMPLATE_FILE); PhysicalPlan plan; int idx = 0; while (reader.hasNext()) { @@ -130,6 +131,7 @@ private void recoverFromTemplateFile(String filePath) throws IOException { logger.error("Can not operate cmd {} in TemplateFile for err:", plan.getOperatorType(), e); } } + reader.close(); } public void createSchemaTemplate(CreateTemplatePlan plan) throws MetadataException { @@ -283,6 +285,6 @@ public void sync() throws MetadataException, IOException { public void clear() throws IOException { planBuffer.clear(); templateMap.clear(); - fileWriter.clear(); + fileWriter.close(); } } From 2fb742fad847fe6eaec5576b3807c1a97bcc7035 Mon Sep 17 00:00:00 2001 From: zyk990424 <1534661820@qq.com> Date: Fri, 4 Mar 2022 15:48:44 +0800 Subject: [PATCH 3/4] implement MTree singleTone and fix MTree recover --- .../apache/iotdb/db/metadata/MManager.java | 3 +-- .../metadata/mtree/service/MTreeService.java | 16 ++++++++++++++ .../mtree/store/CachedMTreeStore.java | 2 +- .../store/disk/schemafile/SFManager.java | 5 ++++- .../db/metadata/MManagerBasicTestCase.java | 22 ++++++++----------- .../db/metadata/mtree/MTreeTestCase.java | 10 +-------- 6 files changed, 32 insertions(+), 26 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java index bd9d5d8e9645d..0e3b86503e8a0 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java @@ -199,7 +199,7 @@ public class MManager { private File logFile; private MLogWriter logWriter; - private MTreeService mtree; + private MTreeService mtree = MTreeService.getInstance(); // device -> DeviceMNode private LoadingCache mNodeCache; private TagManager tagManager = TagManager.getInstance(); @@ -294,7 +294,6 @@ public synchronized void init() { templateManager.init(); tagManager.init(); - mtree = new MTreeService(); mtree.init(); int lineNumber = initFromLog(logFile); diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/service/MTreeService.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/service/MTreeService.java index 5f65bfeb1878a..73d5f9d5807ef 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/service/MTreeService.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/service/MTreeService.java @@ -132,6 +132,22 @@ public class MTreeService implements Serializable { private IMNode root; private IMTreeStore store; + // region MTree Singleton + private static class MTreeServiceHolder { + + private MTreeServiceHolder() { + // allowed to do nothing + } + + private static final MTreeService INSTANCE = new MTreeService(); + } + + /** we should not use this function in other place, but only in IoTDB class */ + public static MTreeService getInstance() { + return MTreeService.MTreeServiceHolder.INSTANCE; + } + // endregion + // region MTree initialization, clear and serialization public MTreeService() {} diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/CachedMTreeStore.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/CachedMTreeStore.java index b06439f7b014a..9a07420441880 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/CachedMTreeStore.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/CachedMTreeStore.java @@ -353,6 +353,7 @@ public void sync() throws MetadataException, IOException { public void clear() { if (flushTask != null) { flushTask.shutdown(); + while (!flushTask.isTerminated()) ; flushTask = null; } root = null; @@ -360,7 +361,6 @@ public void clear() { memManager.clear(); if (file != null) { try { - file.clear(); file.close(); } catch (MetadataException | IOException e) { logger.error(String.format("Error occurred during SchemaFile clear, %s", e.getMessage())); diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/SFManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/SFManager.java index f9d649390d219..4b7190599d42a 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/SFManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/SFManager.java @@ -82,7 +82,9 @@ protected SFManager() { @Override public IMNode init() throws MetadataException, IOException { loadSchemaFiles(); - return getUpperMTree(); + IMNode result = MockSFManager.cloneMNode(root); + SchemaFile.setNodeAddress(result, 0); + return result; } @Override @@ -388,6 +390,7 @@ private void appendStorageGroupNode(String[] nodes, long dataTTL, boolean isEnti for (int i = 1; i < nodes.length - 1; i++) { if (!cur.hasChild(nodes[i])) { cur.addChild(new InternalMNode(cur, nodes[i])); + SchemaFile.setNodeAddress(cur.getChild(nodes[i]), 0L); } cur = cur.getChild(nodes[i]); if (cur.isStorageGroup()) { diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTestCase.java b/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTestCase.java index 819b590de5f58..f96ac0ef8e0bc 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTestCase.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTestCase.java @@ -502,22 +502,18 @@ public void testRecover() { // todo fix me while recover from schemaFile manager.clear(); - MManager recoverManager = new MManager(); - recoverManager.initForMultiMManagerTest(); + manager.init(); - assertTrue(recoverManager.isStorageGroup(new PartialPath("root.laptop.d1"))); - assertFalse(recoverManager.isStorageGroup(new PartialPath("root.laptop.d2"))); - assertFalse(recoverManager.isStorageGroup(new PartialPath("root.laptop.d3"))); - assertFalse(recoverManager.isStorageGroup(new PartialPath("root.laptop"))); + assertTrue(manager.isStorageGroup(new PartialPath("root.laptop.d1"))); + assertFalse(manager.isStorageGroup(new PartialPath("root.laptop.d2"))); + assertFalse(manager.isStorageGroup(new PartialPath("root.laptop.d3"))); + assertFalse(manager.isStorageGroup(new PartialPath("root.laptop"))); // prefix with * assertEquals( devices, - recoverManager.getMatchedDevices(new PartialPath("root.**"), false).stream() + manager.getMatchedDevices(new PartialPath("root.**"), false).stream() .map(PartialPath::getFullPath) .collect(Collectors.toSet())); - - recoverManager.clear(); - manager.init(); } catch (MetadataException e) { e.printStackTrace(); fail(e.getMessage()); @@ -1896,11 +1892,11 @@ public void testTotalSeriesNumber() throws Exception { assertEquals(6, manager.getTotalSeriesNumber()); EnvironmentUtils.restartDaemon(); - assertEquals(6, manager.getTotalSeriesNumber()); + assertEquals(6, manager.getAllTimeseriesCount(new PartialPath("root.*.**"))); manager.deleteTimeseries(new PartialPath("root.laptop.d2.s1")); - assertEquals(5, manager.getTotalSeriesNumber()); + assertEquals(5, manager.getAllTimeseriesCount(new PartialPath("root.*.**"))); manager.deleteStorageGroups(Collections.singletonList(new PartialPath("root.laptop"))); - assertEquals(0, manager.getTotalSeriesNumber()); + assertEquals(0, manager.getAllTimeseriesCount(new PartialPath("root.*.**"))); } catch (MetadataException e) { e.printStackTrace(); fail(e.getMessage()); diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/mtree/MTreeTestCase.java b/server/src/test/java/org/apache/iotdb/db/metadata/mtree/MTreeTestCase.java index 9bb0ad43749e7..f34491900cf83 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/mtree/MTreeTestCase.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/mtree/MTreeTestCase.java @@ -40,7 +40,6 @@ import org.junit.Before; import org.junit.Test; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -70,14 +69,7 @@ public void tearDown() throws Exception { } private MTreeService getNewMTree() { - try { - MTreeService root = new MTreeService(); - root.init(); - return root; - } catch (MetadataException | IOException e) { - fail(); - } - return null; + return MTreeService.getInstance(); } @Test From cf868fdc1a656f66ba4b738dee7dde10b94d0299 Mon Sep 17 00:00:00 2001 From: zyk990424 <1534661820@qq.com> Date: Fri, 4 Mar 2022 15:58:23 +0800 Subject: [PATCH 4/4] fix tag recover temporarily --- .../java/org/apache/iotdb/db/metadata/MManager.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java index 0e3b86503e8a0..fc9bd455f2219 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java @@ -300,6 +300,17 @@ public synchronized void init() { logWriter = new MLogWriter(config.getSchemaDir(), MetadataConstant.METADATA_LOG); logWriter.setLogNum(lineNumber); + + // todo fix me by refactoring tag recover + for (PartialPath path : mtree.getMeasurementPaths(new PartialPath("root.**"))) { + IMeasurementMNode measurementMNode = mtree.getMeasurementMNode(path); + if (measurementMNode.getOffset() != -1) { + tagManager.recoverIndex(measurementMNode.getOffset(), measurementMNode); + } else { + mtree.unPinMNode(measurementMNode); + } + } + isRecovering = false; } catch (MetadataException | IOException e) { logger.error(