Skip to content

Commit c990aa9

Browse files
luoluoyuyuluoluoyuyu
committed
spotless
1 parent 79d8fa7 commit c990aa9

File tree

5 files changed

+326
-0
lines changed

5 files changed

+326
-0
lines changed

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotFileSet.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
2323
import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1;
2424
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
25+
import org.apache.iotdb.db.utils.ObjectTypeUtils;
2526

2627
import org.apache.tsfile.common.constant.TsFileConstant;
2728

@@ -37,6 +38,9 @@ public class SnapshotFileSet {
3738
TsFileResource.RESOURCE_SUFFIX.replace(".", ""),
3839
ModificationFileV1.FILE_SUFFIX.replace(".", ""),
3940
ModificationFile.FILE_SUFFIX.replace(".", ""),
41+
ObjectTypeUtils.OBJECT_FILE_SUFFIX.replace(".", ""),
42+
ObjectTypeUtils.OBJECT_TEMP_FILE_SUFFIX.replace(".", ""),
43+
ObjectTypeUtils.OBJECT_BACK_FILE_SUFFIX.replace(".", ""),
4044
};
4145

4246
private static final Set<String> DATA_FILE_SUFFIX_SET =

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,9 @@
2626
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
2727
import org.apache.iotdb.db.storageengine.dataregion.flush.CompressionRatio;
2828
import org.apache.iotdb.db.storageengine.rescon.disk.FolderManager;
29+
import org.apache.iotdb.db.storageengine.rescon.disk.TierManager;
2930
import org.apache.iotdb.db.storageengine.rescon.disk.strategy.DirectoryStrategyType;
31+
import org.apache.iotdb.db.utils.ObjectTypeUtils;
3032

3133
import org.apache.tsfile.external.commons.io.FileUtils;
3234
import org.slf4j.Logger;
@@ -38,6 +40,8 @@
3840
import java.nio.file.FileVisitor;
3941
import java.nio.file.Files;
4042
import java.nio.file.Path;
43+
import java.nio.file.Paths;
44+
import java.nio.file.SimpleFileVisitor;
4145
import java.nio.file.attribute.BasicFileAttributes;
4246
import java.util.ArrayList;
4347
import java.util.Arrays;
@@ -46,6 +50,7 @@
4650
import java.util.List;
4751
import java.util.Map;
4852
import java.util.Set;
53+
import java.util.concurrent.atomic.AtomicInteger;
4954

5055
public class SnapshotLoader {
5156
private Logger LOGGER = LoggerFactory.getLogger(SnapshotLoader.class);
@@ -231,6 +236,15 @@ private void deleteAllFilesInDataDirs() throws IOException {
231236
timePartitions.addAll(Arrays.asList(files));
232237
}
233238
}
239+
240+
File objectRegionDir =
241+
Paths.get(dataDirPath)
242+
.resolve(IoTDBConstant.OBJECT_FOLDER_NAME)
243+
.resolve(dataRegionId)
244+
.toFile();
245+
if (objectRegionDir.exists()) {
246+
timePartitions.add(objectRegionDir);
247+
}
234248
}
235249

236250
try {
@@ -312,6 +326,78 @@ private void createLinksFromSnapshotDirToDataDirWithoutLog(File sourceDir)
312326
createLinksFromSnapshotToSourceDir(targetSuffix, files, folderManager);
313327
}
314328
}
329+
330+
File snapshotObjectDir = new File(sourceDir, IoTDBConstant.OBJECT_FOLDER_NAME);
331+
if (snapshotObjectDir.exists()) {
332+
FolderManager objectFolderManager =
333+
new FolderManager(
334+
TierManager.getInstance().getAllObjectFileFolders(),
335+
DirectoryStrategyType.SEQUENCE_STRATEGY);
336+
linkObjectTreeFromSnapshotToObjectDirs(snapshotObjectDir, objectFolderManager);
337+
}
338+
}
339+
340+
private void linkObjectTreeFromSnapshotToObjectDirs(
341+
File sourceObjectRoot, FolderManager folderManager)
342+
throws DiskSpaceInsufficientException, IOException {
343+
Path sourceRootPath = sourceObjectRoot.toPath();
344+
// Process files during traversal to avoid loading all object file paths into memory.
345+
Files.walkFileTree(
346+
sourceRootPath,
347+
new SimpleFileVisitor<Path>() {
348+
@Override
349+
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
350+
throws IOException {
351+
if (!isObjectSnapshotCandidate(file.getFileName().toString())) {
352+
return FileVisitResult.CONTINUE;
353+
}
354+
final Path sourceFile = file;
355+
final Path targetRelPath = sourceRootPath.relativize(file);
356+
try {
357+
folderManager.getNextWithRetry(
358+
currentObjectDir -> {
359+
File targetFile =
360+
new File(currentObjectDir).toPath().resolve(targetRelPath).toFile();
361+
try {
362+
if (!targetFile.getParentFile().exists()
363+
&& !targetFile.getParentFile().mkdirs()) {
364+
throw new IOException(
365+
String.format(
366+
"Cannot create directory %s",
367+
targetFile.getParentFile().getAbsolutePath()));
368+
}
369+
try {
370+
Files.createLink(targetFile.toPath(), sourceFile);
371+
LOGGER.debug("Created hard link from {} to {}", sourceFile, targetFile);
372+
return targetFile;
373+
} catch (IOException e) {
374+
LOGGER.info(
375+
"Cannot create link from {} to {}, fallback to copy",
376+
sourceFile,
377+
targetFile);
378+
}
379+
Files.copy(sourceFile, targetFile.toPath());
380+
return targetFile;
381+
} catch (Exception e) {
382+
LOGGER.warn(
383+
"Failed to process file {} in dir {}: {}",
384+
sourceFile.getFileName(),
385+
currentObjectDir,
386+
e.getMessage(),
387+
e);
388+
throw e;
389+
}
390+
});
391+
} catch (Exception e) {
392+
throw new IOException(
393+
String.format(
394+
"Failed to process object file after retries. Source: %s",
395+
sourceFile.toAbsolutePath()),
396+
e);
397+
}
398+
return FileVisitResult.CONTINUE;
399+
}
400+
});
315401
}
316402

317403
private void createLinksFromSnapshotToSourceDir(
@@ -470,9 +556,96 @@ private int takeHardLinksFromSnapshotToDataDir(
470556
}
471557
}
472558

559+
File objectSnapshotRoot =
560+
new File(
561+
snapshotFolder.getAbsolutePath() + File.separator + IoTDBConstant.OBJECT_FOLDER_NAME);
562+
if (objectSnapshotRoot.exists()) {
563+
cnt += linkObjectSnapshotTreeToDataDir(objectSnapshotRoot, fileInfoSet);
564+
}
565+
473566
return cnt;
474567
}
475568

569+
private int linkObjectSnapshotTreeToDataDir(File objectSnapshotRoot, Set<String> fileInfoSet)
570+
throws IOException {
571+
final FolderManager folderManager;
572+
try {
573+
folderManager =
574+
new FolderManager(
575+
TierManager.getInstance().getAllObjectFileFolders(),
576+
DirectoryStrategyType.SEQUENCE_STRATEGY);
577+
} catch (DiskSpaceInsufficientException e) {
578+
throw new IOException("Failed to initialize object folder manager", e);
579+
}
580+
Path rootPath = objectSnapshotRoot.toPath();
581+
AtomicInteger cnt = new AtomicInteger(0);
582+
// Process files during traversal to avoid loading all object file paths into memory.
583+
Files.walkFileTree(
584+
rootPath,
585+
new SimpleFileVisitor<Path>() {
586+
@Override
587+
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
588+
throws IOException {
589+
if (!isObjectSnapshotCandidate(file.getFileName().toString())) {
590+
return FileVisitResult.CONTINUE;
591+
}
592+
String infoStr = getFileInfoString(file.toFile());
593+
if (!fileInfoSet.contains(infoStr)) {
594+
throw new IOException(
595+
String.format("File %s is not in the log file list", file.toAbsolutePath()));
596+
}
597+
final Path sourceFile = file;
598+
final Path targetRelPath = rootPath.relativize(file);
599+
try {
600+
folderManager.getNextWithRetry(
601+
currentObjectDir -> {
602+
File targetFile =
603+
new File(currentObjectDir).toPath().resolve(targetRelPath).toFile();
604+
try {
605+
if (!targetFile.getParentFile().exists()
606+
&& !targetFile.getParentFile().mkdirs()) {
607+
throw new IOException(
608+
String.format(
609+
"Cannot create directory %s",
610+
targetFile.getParentFile().getAbsolutePath()));
611+
}
612+
try {
613+
Files.createLink(targetFile.toPath(), sourceFile);
614+
LOGGER.debug("Created hard link from {} to {}", sourceFile, targetFile);
615+
return targetFile;
616+
} catch (IOException e) {
617+
LOGGER.info(
618+
"Cannot create link from {} to {}, fallback to copy",
619+
sourceFile,
620+
targetFile);
621+
}
622+
Files.copy(sourceFile, targetFile.toPath());
623+
return targetFile;
624+
} catch (Exception e) {
625+
LOGGER.warn(
626+
"Failed to process file {} in dir {}: {}",
627+
sourceFile.getFileName(),
628+
currentObjectDir,
629+
e.getMessage(),
630+
e);
631+
throw e;
632+
}
633+
});
634+
} catch (Exception e) {
635+
throw new IOException(
636+
String.format(
637+
"Failed to process object snapshot file after retries. Source: %s",
638+
sourceFile.toAbsolutePath()),
639+
e);
640+
}
641+
cnt.incrementAndGet();
642+
return FileVisitResult.CONTINUE;
643+
}
644+
});
645+
646+
return cnt.get();
647+
}
648+
476649
private void createLinksFromSourceToTarget(File targetDir, File[] files, Set<String> fileInfoSet)
477650
throws IOException {
478651
for (File file : files) {
@@ -492,6 +665,33 @@ private void createLinksFromSourceToTarget(File targetDir, File[] files, Set<Str
492665
}
493666

494667
private String getFileInfoString(File file) {
668+
Path filePath = file.toPath();
669+
int objectDirIndex = -1;
670+
int nameCount = filePath.getNameCount();
671+
for (int i = 0; i < nameCount; i++) {
672+
if (IoTDBConstant.OBJECT_FOLDER_NAME.equals(filePath.getName(i).toString())) {
673+
objectDirIndex = i;
674+
break;
675+
}
676+
}
677+
if (objectDirIndex >= 0 && objectDirIndex < nameCount - 1) {
678+
Path relativeToObject = filePath.subpath(objectDirIndex + 1, nameCount);
679+
String fileName = relativeToObject.getFileName().toString();
680+
Path parentPath = relativeToObject.getParent();
681+
String middlePath = "";
682+
if (parentPath != null) {
683+
List<String> pathElements = new ArrayList<>();
684+
for (Path element : parentPath) {
685+
pathElements.add(element.toString());
686+
}
687+
middlePath = String.join("/", pathElements);
688+
}
689+
return fileName
690+
+ SnapshotLogger.SPLIT_CHAR
691+
+ middlePath
692+
+ SnapshotLogger.SPLIT_CHAR
693+
+ "object";
694+
}
495695
String[] splittedStr = file.getAbsolutePath().split(File.separator.equals("\\") ? "\\\\" : "/");
496696
int length = splittedStr.length;
497697
return splittedStr[length - SnapshotLogger.FILE_NAME_OFFSET]
@@ -501,6 +701,12 @@ private String getFileInfoString(File file) {
501701
+ splittedStr[length - SnapshotLogger.SEQUENCE_OFFSET];
502702
}
503703

704+
private boolean isObjectSnapshotCandidate(String fileName) {
705+
return fileName.endsWith(ObjectTypeUtils.OBJECT_FILE_SUFFIX)
706+
|| fileName.endsWith(ObjectTypeUtils.OBJECT_TEMP_FILE_SUFFIX)
707+
|| fileName.endsWith(ObjectTypeUtils.OBJECT_BACK_FILE_SUFFIX);
708+
}
709+
504710
public List<File> getSnapshotFileInfo() throws IOException {
505711
File snapshotLogFile = getSnapshotLogFile();
506712

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLogger.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@
2424
import java.io.IOException;
2525
import java.nio.charset.StandardCharsets;
2626
import java.nio.file.Files;
27+
import java.nio.file.Path;
28+
import java.util.ArrayList;
29+
import java.util.List;
2730

2831
public class SnapshotLogger implements AutoCloseable {
2932
public static final String SNAPSHOT_LOG_NAME = "snapshot.log";
@@ -56,6 +59,26 @@ public void close() throws Exception {
5659
os.close();
5760
}
5861

62+
public void logObjectRelativePath(Path relativePathFromObjectRoot) throws IOException {
63+
String fileName = relativePathFromObjectRoot.getFileName().toString();
64+
Path parentPath = relativePathFromObjectRoot.getParent();
65+
String middlePath = "";
66+
if (parentPath != null) {
67+
List<String> pathElements = new ArrayList<>();
68+
for (Path element : parentPath) {
69+
pathElements.add(element.toString());
70+
}
71+
middlePath = String.join("/", pathElements);
72+
}
73+
os.write(fileName.getBytes(StandardCharsets.UTF_8));
74+
os.write(SPLIT_CHAR.getBytes(StandardCharsets.UTF_8));
75+
os.write(middlePath.getBytes(StandardCharsets.UTF_8));
76+
os.write(SPLIT_CHAR.getBytes(StandardCharsets.UTF_8));
77+
os.write("object".getBytes(StandardCharsets.UTF_8));
78+
os.write("\n".getBytes(StandardCharsets.UTF_8));
79+
os.flush();
80+
}
81+
5982
/**
6083
* Log the logical info for the link file, including its file name, time partition, data region
6184
* id, database name, sequence or not.

0 commit comments

Comments
 (0)