diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java index d7ee934b8f72..93c5b15d04ca 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java @@ -76,6 +76,7 @@ import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; @@ -426,6 +427,49 @@ public void testCreateKeyWithECReplicationConfig() throws Exception { createKeyWithECReplicationConfig(root, cluster.getConf()); } + @Test + void testContentSummaryErasureCodingPolicy() throws Exception { + String ratisKey = "ratis-ec-policy-key"; + String ecKey = "ec-policy-key"; + ECReplicationConfig ecConfig = new ECReplicationConfig("RS-3-2-1024k"); + Path parentDir = new Path(OZONE_URI_DELIMITER, "ec-policy-mixed-o3fs"); + Path ratisFile = new Path(parentDir, ratisKey); + Path ecFile = new Path(parentDir, ecKey); + + fs.mkdirs(parentDir); + String ratisRelKey = "ec-policy-mixed-o3fs/" + ratisKey; + String ecRelKey = "ec-policy-mixed-o3fs/" + ecKey; + TestDataUtil.createKey(ozoneBucket, ratisRelKey, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), + new byte[]{0}); + TestDataUtil.createKey(ozoneBucket, ecRelKey, ecConfig, + new byte[]{0}); + + try { + assertEquals("", + fs.getContentSummary(ROOT).getErasureCodingPolicy()); + assertEquals("Replicated", + fs.getContentSummary(ratisFile).getErasureCodingPolicy()); + assertEquals(ecConfig.getReplication(), + fs.getContentSummary(ecFile).getErasureCodingPolicy()); + assertEquals("", + fs.getContentSummary(parentDir).getErasureCodingPolicy()); + } finally { + fs.delete(parentDir, true); + } + } + + @Test + void testLsDashEDoesNotThrow() throws Exception { + FsShell shell = new FsShell(fs.getConf()); + try { + int exitCode = shell.run(new String[]{"-ls", "-R", "-e", fsRoot}); + assertEquals(0, exitCode); + } finally { + shell.close(); + } + } + @Test public void testDeleteCreatesFakeParentDir() throws Exception { deleteCreatesFakeParentDir(ROOT); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java index dadb3dc5baa0..5e7432f97dda 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java @@ -79,6 +79,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; @@ -1981,6 +1982,77 @@ void testCreateAndCheckECFileDiskUsage() throws Exception { fs.delete(filePath, true); } + @Test + void testContentSummaryErasureCodingPolicy() throws Exception { + String ratisKey = "ratis-ec-policy-key"; + String ecKey = "ec-policy-key"; + ECReplicationConfig ecConfig = new ECReplicationConfig("RS-3-2-1024k"); + Path parentDir = new Path(bucketPath, "ec-policy-mixed"); + Path ratisFile = new Path(parentDir, ratisKey); + Path ecFile = new Path(parentDir, ecKey); + + fs.mkdirs(parentDir); + OzoneBucket bucket = objectStore.getVolume(volumeName).getBucket(bucketName); + String ratisRelKey = "ec-policy-mixed/" + ratisKey; + String ecRelKey = "ec-policy-mixed/" + ecKey; + TestDataUtil.createKey(bucket, ratisRelKey, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), + new byte[]{0}); + TestDataUtil.createKey(bucket, ecRelKey, ecConfig, + new byte[]{0}); + + try { + assertEquals("", + fs.getContentSummary(new Path(OZONE_URI_DELIMITER)) + .getErasureCodingPolicy()); + assertEquals("", + fs.getContentSummary(volumePath).getErasureCodingPolicy()); + assertEquals("", + fs.getContentSummary(bucketPath).getErasureCodingPolicy()); + assertEquals("Replicated", + fs.getContentSummary(ratisFile).getErasureCodingPolicy()); + assertEquals(ecConfig.getReplication(), + fs.getContentSummary(ecFile).getErasureCodingPolicy()); + assertEquals("", + fs.getContentSummary(parentDir).getErasureCodingPolicy()); + } finally { + fs.delete(parentDir, true); + } + } + + @Test + void testContentSummaryErasureCodingPolicyOnEcBucket() throws Exception { + ECReplicationConfig ecConfig = new ECReplicationConfig("RS-3-2-1024k"); + BucketArgs ecBucketArgs = BucketArgs.newBuilder() + .setStorageType(StorageType.DISK) + .setBucketLayout(BucketLayout.LEGACY) + .setDefaultReplicationConfig(new DefaultReplicationConfig(ecConfig)) + .build(); + String ecVol = UUID.randomUUID().toString(); + String ecBuck = UUID.randomUUID().toString(); + TestDataUtil.createVolumeAndBucket(client, ecVol, ecBuck, ecBucketArgs); + + Path ecBucketPath = new Path(new Path(OZONE_URI_DELIMITER, ecVol), ecBuck); + try { + assertEquals(ecConfig.getReplication(), + fs.getContentSummary(ecBucketPath).getErasureCodingPolicy()); + } finally { + fs.delete(ecBucketPath, true); + } + } + + @Test + void testLsDashEDoesNotThrow() throws Exception { + FsShell shell = new FsShell(conf); + try { + int exitCode = shell.run(new String[]{"-ls", "-R", "-e", + OZONE_URI_DELIMITER + volumeName + OZONE_URI_DELIMITER + bucketName}); + assertEquals(0, exitCode); + } finally { + shell.close(); + } + } + @Test void testCreateAndCheckRatisFileDiskUsage() throws Exception { String key = "ratiskeytest"; diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java index f8557b61e46f..d9033054ead8 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java @@ -537,6 +537,13 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatus status, OmKeyInfo keyInfo = status.getKeyInfo(); short replication = (short) keyInfo.getReplicationConfig() .getRequiredNodes(); + boolean isEc = OzoneClientUtils.isKeyErasureCode(keyInfo); + String ecPolicy; + if (isEc) { + ecPolicy = keyInfo.getReplicationConfig().getReplication(); + } else { + ecPolicy = status.isFile() ? "Replicated" : ""; + } return new FileStatusAdapter( keyInfo.getDataSize(), keyInfo.getReplicatedSize(), @@ -553,7 +560,8 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatus status, null, getBlockLocations(status), OzoneClientUtils.isKeyEncrypted(keyInfo), - OzoneClientUtils.isKeyErasureCode(keyInfo) + isEc, + ecPolicy ); } @@ -562,6 +570,13 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatusLight status, BasicOmKeyInfo keyInfo = status.getKeyInfo(); short replication = (short) keyInfo.getReplicationConfig() .getRequiredNodes(); + boolean isEc = OzoneClientUtils.isKeyErasureCode(keyInfo); + String ecPolicy; + if (isEc) { + ecPolicy = keyInfo.getReplicationConfig().getReplication(); + } else { + ecPolicy = status.isFile() ? "Replicated" : ""; + } return new FileStatusAdapter( keyInfo.getDataSize(), keyInfo.getReplicatedSize(), @@ -578,7 +593,8 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatusLight status, null, getBlockLocations(null), keyInfo.isEncrypted(), - OzoneClientUtils.isKeyErasureCode(keyInfo) + isEc, + ecPolicy ); } diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java index 8e0f4e8a5dd2..4f0cc8b60bd8 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java @@ -51,6 +51,7 @@ import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -865,6 +866,91 @@ public BlockLocation[] getFileBlockLocations(FileStatus fileStatus, } } + @Override + public ContentSummary getContentSummary(Path f) throws IOException { + Path qualifiedPath = f.makeQualified(uri, workingDir); + String key = pathToKey(qualifiedPath); + FileStatusAdapter status; + try { + status = adapter.getFileStatus(key, uri, qualifiedPath, getUsername()); + } catch (OMException ex) { + if (ex.getResult().equals(OMException.ResultCodes.KEY_NOT_FOUND)) { + throw new FileNotFoundException("File not found. path:" + f); + } + throw ex; + } + + if (status.isFile()) { + long length = status.getLength(); + long spaceConsumed = status.getDiskConsumed(); + ContentSummary.Builder builder = new ContentSummary.Builder().length(length). + fileCount(1).directoryCount(0).spaceConsumed(spaceConsumed); + applyEcPolicy(builder, status.getErasureCodingPolicy()); + return builder.build(); + } + + long[] summary = {0, 0, 0, 1}; + for (FileStatusAdapter s : listStatusAdapter(f)) { + long length = s.getLength(); + long spaceConsumed = s.getDiskConsumed(); + ContentSummary c; + if (s.isDir()) { + c = getContentSummary(s.getPath()); + } else { + ContentSummary.Builder childBuilder = new ContentSummary.Builder().length(length). + fileCount(1).directoryCount(0).spaceConsumed(spaceConsumed); + applyEcPolicy(childBuilder, s.getErasureCodingPolicy()); + c = childBuilder.build(); + } + + summary[0] += c.getLength(); + summary[1] += c.getSpaceConsumed(); + summary[2] += c.getFileCount(); + summary[3] += c.getDirectoryCount(); + } + + ContentSummary.Builder builder = new ContentSummary.Builder().length(summary[0]). + fileCount(summary[2]).directoryCount(summary[3]). + spaceConsumed(summary[1]); + applyEcPolicy(builder, status.getErasureCodingPolicy()); + return builder.build(); + } + + /** + * Apply the erasure coding policy on the {@link ContentSummary.Builder}. + * Default implementation is a no-op so that this class can compile and run + * against Hadoop 2, where {@code ContentSummary.Builder.erasureCodingPolicy} + * does not exist. The Hadoop 3 subclass overrides this to set the policy. + */ + protected void applyEcPolicy(ContentSummary.Builder builder, String ecPolicy) { + } + + private List listStatusAdapter(Path f) throws IOException { + int numEntries = listingPageSize; + LinkedList statuses = new LinkedList<>(); + List tmpStatusList; + String startKey = ""; + int entriesAdded; + do { + tmpStatusList = adapter.listStatus(pathToKey(f), false, startKey, + numEntries, uri, workingDir, getUsername(), true); + entriesAdded = 0; + if (!tmpStatusList.isEmpty()) { + if (startKey.isEmpty() || !statuses.getLast().getPath().toString() + .equals(tmpStatusList.get(0).getPath().toString())) { + statuses.addAll(tmpStatusList); + entriesAdded += tmpStatusList.size(); + } else { + statuses.addAll(tmpStatusList.subList(1, tmpStatusList.size())); + entriesAdded += tmpStatusList.size() - 1; + } + startKey = pathToKey(statuses.getLast().getPath()); + } + } while (entriesAdded > 0); + + return statuses; + } + @Override public short getDefaultReplication() { return adapter.getDefaultReplication(); diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index c502f9096c5f..27898e7e695f 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -1035,6 +1035,13 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatus status, OmKeyInfo keyInfo = status.getKeyInfo(); short replication = (short) keyInfo.getReplicationConfig() .getRequiredNodes(); + boolean isEc = OzoneClientUtils.isKeyErasureCode(keyInfo); + String ecPolicy; + if (isEc) { + ecPolicy = keyInfo.getReplicationConfig().getReplication(); + } else { + ecPolicy = status.isFile() ? "Replicated" : ""; + } return new FileStatusAdapter( keyInfo.getDataSize(), keyInfo.getReplicatedSize(), @@ -1051,7 +1058,8 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatus status, null, getBlockLocations(status), OzoneClientUtils.isKeyEncrypted(keyInfo), - OzoneClientUtils.isKeyErasureCode(keyInfo) + isEc, + ecPolicy ); } @@ -1060,6 +1068,13 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatusLight status, BasicOmKeyInfo keyInfo = status.getKeyInfo(); short replication = (short) keyInfo.getReplicationConfig() .getRequiredNodes(); + boolean isEc = OzoneClientUtils.isKeyErasureCode(keyInfo); + String ecPolicy; + if (isEc) { + ecPolicy = keyInfo.getReplicationConfig().getReplication(); + } else { + ecPolicy = status.isFile() ? "Replicated" : ""; + } return new FileStatusAdapter( keyInfo.getDataSize(), keyInfo.getReplicatedSize(), @@ -1076,7 +1091,8 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatusLight status, null, getBlockLocations(null), keyInfo.isEncrypted(), - OzoneClientUtils.isKeyErasureCode(keyInfo) + isEc, + ecPolicy ); } @@ -1169,7 +1185,7 @@ private static FileStatusAdapter getFileStatusAdapterForVolume( return new FileStatusAdapter(0L, 0L, path, true, (short)0, 0L, ozoneVolume.getCreationTime().getEpochSecond() * 1000, 0L, FsPermission.getDirDefault().toShort(), - owner, group, null, new BlockLocation[0], false, false + owner, group, null, new BlockLocation[0], false, false, "" ); } @@ -1192,14 +1208,16 @@ private static FileStatusAdapter getFileStatusAdapterForBucket(OzoneBucket ozone UserGroupInformation ugi = UserGroupInformation.createRemoteUser(ozoneBucket.getOwner()); String owner = ugi.getShortUserName(); String group = getGroupName(ugi); + ReplicationConfig rc = ozoneBucket.getReplicationConfig(); + boolean isEc = rc != null && rc.getReplicationType() == HddsProtos.ReplicationType.EC; + String ecPolicy = isEc ? rc.getReplication() : ""; return new FileStatusAdapter(0L, 0L, path, true, (short)0, 0L, ozoneBucket.getCreationTime().getEpochSecond() * 1000, 0L, FsPermission.getDirDefault().toShort(), owner, group, null, new BlockLocation[0], !StringUtils.isEmpty(ozoneBucket.getEncryptionKeyName()), - ozoneBucket.getReplicationConfig() != null && - ozoneBucket.getReplicationConfig().getReplicationType() == - HddsProtos.ReplicationType.EC); + isEc, + ecPolicy); } /** @@ -1225,6 +1243,9 @@ private static FileStatusAdapter getFileStatusAdapterForBucketSnapshot( ozoneSnapshot.getName(), pathStr); } Path path = new Path(pathStr); + ReplicationConfig rc = ozoneBucket.getReplicationConfig(); + boolean isEc = rc != null && rc.getReplicationType() == HddsProtos.ReplicationType.EC; + String ecPolicy = isEc ? rc.getReplication() : ""; return new FileStatusAdapter( ozoneSnapshot.getReferencedSize(), ozoneSnapshot.getReferencedReplicatedSize(), @@ -1233,9 +1254,8 @@ private static FileStatusAdapter getFileStatusAdapterForBucketSnapshot( FsPermission.getDirDefault().toShort(), owner, group, null, new BlockLocation[0], !StringUtils.isEmpty(ozoneBucket.getEncryptionKeyName()), - ozoneBucket.getReplicationConfig() != null && - ozoneBucket.getReplicationConfig().getReplicationType() == - HddsProtos.ReplicationType.EC); + isEc, + ecPolicy); } /** @@ -1264,14 +1284,16 @@ private static FileStatusAdapter getFileStatusAdapterWithSnapshotIndicator( ozoneBucket.getName(), pathStr); } Path path = new Path(pathStr); + ReplicationConfig rc = ozoneBucket.getReplicationConfig(); + boolean isEc = rc != null && rc.getReplicationType() == HddsProtos.ReplicationType.EC; + String ecPolicy = isEc ? rc.getReplication() : ""; return new FileStatusAdapter(0L, 0L, path, true, (short)0, 0L, ozoneBucket.getCreationTime().getEpochSecond() * 1000, 0L, FsPermission.getDirDefault().toShort(), owner, group, null, new BlockLocation[0], !StringUtils.isEmpty(ozoneBucket.getEncryptionKeyName()), - ozoneBucket.getReplicationConfig() != null && - ozoneBucket.getReplicationConfig().getReplicationType() == - HddsProtos.ReplicationType.EC); + isEc, + ecPolicy); } /** @@ -1286,7 +1308,7 @@ private static FileStatusAdapter getFileStatusAdapterForRoot(URI uri) { return new FileStatusAdapter(0L, 0L, path, true, (short)0, 0L, System.currentTimeMillis(), 0L, FsPermission.getDirDefault().toShort(), - null, null, null, new BlockLocation[0], false, false); + null, null, null, new BlockLocation[0], false, false, ""); } @Override diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java index 67f09313d0ea..b61dca450cb7 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java @@ -1575,18 +1575,25 @@ private ContentSummary getContentSummaryInSpan(Path f) throws IOException { long length = status.getLength(); long spaceConsumed = status.getDiskConsumed(); - return new ContentSummary.Builder().length(length). - fileCount(1).directoryCount(0).spaceConsumed(spaceConsumed).build(); + ContentSummary.Builder builder = new ContentSummary.Builder().length(length). + fileCount(1).directoryCount(0).spaceConsumed(spaceConsumed); + applyEcPolicy(builder, status.getErasureCodingPolicy()); + return builder.build(); } // f is a directory long[] summary = {0, 0, 0, 1}; - int i = 0; for (FileStatusAdapter s : listStatusAdapter(f, true)) { long length = s.getLength(); long spaceConsumed = s.getDiskConsumed(); - ContentSummary c = s.isDir() ? getContentSummary(s.getPath()) : - new ContentSummary.Builder().length(length). - fileCount(1).directoryCount(0).spaceConsumed(spaceConsumed).build(); + ContentSummary c; + if (s.isDir()) { + c = getContentSummary(s.getPath()); + } else { + ContentSummary.Builder childBuilder = new ContentSummary.Builder().length(length). + fileCount(1).directoryCount(0).spaceConsumed(spaceConsumed); + applyEcPolicy(childBuilder, s.getErasureCodingPolicy()); + c = childBuilder.build(); + } summary[0] += c.getLength(); summary[1] += c.getSpaceConsumed(); @@ -1594,9 +1601,20 @@ private ContentSummary getContentSummaryInSpan(Path f) throws IOException { summary[3] += c.getDirectoryCount(); } - return new ContentSummary.Builder().length(summary[0]). + ContentSummary.Builder builder = new ContentSummary.Builder().length(summary[0]). fileCount(summary[2]).directoryCount(summary[3]). - spaceConsumed(summary[1]).build(); + spaceConsumed(summary[1]); + applyEcPolicy(builder, status.getErasureCodingPolicy()); + return builder.build(); + } + + /** + * Apply the erasure coding policy on the {@link ContentSummary.Builder}. + * Default implementation is a no-op so that this class can compile and run + * against Hadoop 2, where {@code ContentSummary.Builder.erasureCodingPolicy} + * does not exist. The Hadoop 3 subclass overrides this to set the policy. + */ + protected void applyEcPolicy(ContentSummary.Builder builder, String ecPolicy) { } @Override diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/FileStatusAdapter.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/FileStatusAdapter.java index eba24f88bb5c..232a1e31b53b 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/FileStatusAdapter.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/FileStatusAdapter.java @@ -53,6 +53,8 @@ public final class FileStatusAdapter { private final boolean isErasureCoded; + private final String erasureCodingPolicy; + @SuppressWarnings("checkstyle:ParameterNumber") public FileStatusAdapter(long length, long diskConsumed, Path path, boolean isdir, short blockReplication, long blocksize, @@ -60,6 +62,18 @@ public FileStatusAdapter(long length, long diskConsumed, Path path, String owner, String group, Path symlink, BlockLocation[] locations, boolean isEncrypted, boolean isErasureCoded) { + this(length, diskConsumed, path, isdir, blockReplication, blocksize, + modificationTime, accessTime, permission, owner, group, symlink, + locations, isEncrypted, isErasureCoded, null); + } + + @SuppressWarnings("checkstyle:ParameterNumber") + public FileStatusAdapter(long length, long diskConsumed, Path path, + boolean isdir, short blockReplication, long blocksize, + long modificationTime, long accessTime, short permission, + String owner, String group, Path symlink, + BlockLocation[] locations, boolean isEncrypted, + boolean isErasureCoded, String erasureCodingPolicy) { this.length = length; this.diskConsumed = diskConsumed; this.path = path; @@ -75,6 +89,7 @@ public FileStatusAdapter(long length, long diskConsumed, Path path, this.blockLocations = new ArrayList<>(Arrays.asList(locations)); this.isEncrypted = isEncrypted; this.isErasureCoded = isErasureCoded; + this.erasureCodingPolicy = erasureCodingPolicy; } public Path getPath() { @@ -137,6 +152,10 @@ public boolean isErasureCoded() { return isErasureCoded; } + public String getErasureCodingPolicy() { + return erasureCodingPolicy; + } + public BlockLocation[] getBlockLocations() { return blockLocations.toArray(new BlockLocation[0]); } @@ -159,6 +178,7 @@ public String toString() { .append("; group=").append(group) .append("; permission=").append(permission) .append("; isSymlink=").append(getSymlink()) + .append("; erasureCodingPolicy=").append(erasureCodingPolicy) .append('}'); return sb.toString(); diff --git a/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java index 5136c7343077..b23f0fb50877 100644 --- a/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java @@ -25,6 +25,7 @@ import java.util.List; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProviderTokenIssuer; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LeaseRecoverable; import org.apache.hadoop.fs.Path; @@ -183,4 +184,11 @@ public boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException { return setSafeModeUtil(action, isChecked); } + + @Override + protected void applyEcPolicy(ContentSummary.Builder builder, String ecPolicy) { + if (ecPolicy != null) { + builder.erasureCodingPolicy(ecPolicy); + } + } } diff --git a/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java index 6774b6588562..0031e57d31e5 100644 --- a/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java @@ -25,6 +25,7 @@ import java.util.List; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProviderTokenIssuer; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LeaseRecoverable; import org.apache.hadoop.fs.Path; @@ -188,4 +189,11 @@ public boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException { return setSafeModeUtil(action, isChecked); } + + @Override + protected void applyEcPolicy(ContentSummary.Builder builder, String ecPolicy) { + if (ecPolicy != null) { + builder.erasureCodingPolicy(ecPolicy); + } + } } diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java index 5136c7343077..b23f0fb50877 100644 --- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java +++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java @@ -25,6 +25,7 @@ import java.util.List; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProviderTokenIssuer; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LeaseRecoverable; import org.apache.hadoop.fs.Path; @@ -183,4 +184,11 @@ public boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException { return setSafeModeUtil(action, isChecked); } + + @Override + protected void applyEcPolicy(ContentSummary.Builder builder, String ecPolicy) { + if (ecPolicy != null) { + builder.erasureCodingPolicy(ecPolicy); + } + } } diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java index 74c4a30e9914..38dc72a77273 100644 --- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java @@ -25,6 +25,7 @@ import java.util.List; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProviderTokenIssuer; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LeaseRecoverable; import org.apache.hadoop.fs.Path; @@ -195,4 +196,11 @@ public boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException { return setSafeModeUtil(action, isChecked); } + + @Override + protected void applyEcPolicy(ContentSummary.Builder builder, String ecPolicy) { + if (ecPolicy != null) { + builder.erasureCodingPolicy(ecPolicy); + } + } }