From fbf88ff289ecf90eddb6c8b83967daaa3dcca216 Mon Sep 17 00:00:00 2001 From: Chi-Hsuan Huang Date: Thu, 7 May 2026 21:58:16 +0800 Subject: [PATCH 1/7] HDDS-14043. Plumb erasure coding policy through FileStatusAdapter Adds a nullable erasureCodingPolicy field to FileStatusAdapter and populates it from each key's ReplicationConfig (canonical EC scheme name when EC, "Replicated" otherwise) in BasicOzoneClientAdapterImpl and BasicRootedOzoneClientAdapterImpl. Synthetic adapters for buckets and bucket snapshots derive the policy from the bucket's own ReplicationConfig instead of hardcoding "Replicated", which previously contradicted the existing isErasureCoded flag for EC buckets. The 15-arg FileStatusAdapter constructor is preserved as a back-compat overload that delegates with a null policy. No callers read the new field yet; that change follows. --- .../fs/ozone/BasicOzoneClientAdapterImpl.java | 20 +++++++- .../BasicRootedOzoneClientAdapterImpl.java | 48 ++++++++++++++----- .../hadoop/fs/ozone/FileStatusAdapter.java | 20 ++++++++ 3 files changed, 73 insertions(+), 15 deletions(-) diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java index f8557b61e46f..d9033054ead8 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java @@ -537,6 +537,13 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatus status, OmKeyInfo keyInfo = status.getKeyInfo(); short replication = (short) keyInfo.getReplicationConfig() .getRequiredNodes(); + boolean isEc = OzoneClientUtils.isKeyErasureCode(keyInfo); + String ecPolicy; + if (isEc) { + ecPolicy = keyInfo.getReplicationConfig().getReplication(); + } else { + ecPolicy = status.isFile() ? "Replicated" : ""; + } return new FileStatusAdapter( keyInfo.getDataSize(), keyInfo.getReplicatedSize(), @@ -553,7 +560,8 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatus status, null, getBlockLocations(status), OzoneClientUtils.isKeyEncrypted(keyInfo), - OzoneClientUtils.isKeyErasureCode(keyInfo) + isEc, + ecPolicy ); } @@ -562,6 +570,13 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatusLight status, BasicOmKeyInfo keyInfo = status.getKeyInfo(); short replication = (short) keyInfo.getReplicationConfig() .getRequiredNodes(); + boolean isEc = OzoneClientUtils.isKeyErasureCode(keyInfo); + String ecPolicy; + if (isEc) { + ecPolicy = keyInfo.getReplicationConfig().getReplication(); + } else { + ecPolicy = status.isFile() ? "Replicated" : ""; + } return new FileStatusAdapter( keyInfo.getDataSize(), keyInfo.getReplicatedSize(), @@ -578,7 +593,8 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatusLight status, null, getBlockLocations(null), keyInfo.isEncrypted(), - OzoneClientUtils.isKeyErasureCode(keyInfo) + isEc, + ecPolicy ); } diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index c502f9096c5f..27898e7e695f 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -1035,6 +1035,13 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatus status, OmKeyInfo keyInfo = status.getKeyInfo(); short replication = (short) keyInfo.getReplicationConfig() .getRequiredNodes(); + boolean isEc = OzoneClientUtils.isKeyErasureCode(keyInfo); + String ecPolicy; + if (isEc) { + ecPolicy = keyInfo.getReplicationConfig().getReplication(); + } else { + ecPolicy = status.isFile() ? "Replicated" : ""; + } return new FileStatusAdapter( keyInfo.getDataSize(), keyInfo.getReplicatedSize(), @@ -1051,7 +1058,8 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatus status, null, getBlockLocations(status), OzoneClientUtils.isKeyEncrypted(keyInfo), - OzoneClientUtils.isKeyErasureCode(keyInfo) + isEc, + ecPolicy ); } @@ -1060,6 +1068,13 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatusLight status, BasicOmKeyInfo keyInfo = status.getKeyInfo(); short replication = (short) keyInfo.getReplicationConfig() .getRequiredNodes(); + boolean isEc = OzoneClientUtils.isKeyErasureCode(keyInfo); + String ecPolicy; + if (isEc) { + ecPolicy = keyInfo.getReplicationConfig().getReplication(); + } else { + ecPolicy = status.isFile() ? "Replicated" : ""; + } return new FileStatusAdapter( keyInfo.getDataSize(), keyInfo.getReplicatedSize(), @@ -1076,7 +1091,8 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatusLight status, null, getBlockLocations(null), keyInfo.isEncrypted(), - OzoneClientUtils.isKeyErasureCode(keyInfo) + isEc, + ecPolicy ); } @@ -1169,7 +1185,7 @@ private static FileStatusAdapter getFileStatusAdapterForVolume( return new FileStatusAdapter(0L, 0L, path, true, (short)0, 0L, ozoneVolume.getCreationTime().getEpochSecond() * 1000, 0L, FsPermission.getDirDefault().toShort(), - owner, group, null, new BlockLocation[0], false, false + owner, group, null, new BlockLocation[0], false, false, "" ); } @@ -1192,14 +1208,16 @@ private static FileStatusAdapter getFileStatusAdapterForBucket(OzoneBucket ozone UserGroupInformation ugi = UserGroupInformation.createRemoteUser(ozoneBucket.getOwner()); String owner = ugi.getShortUserName(); String group = getGroupName(ugi); + ReplicationConfig rc = ozoneBucket.getReplicationConfig(); + boolean isEc = rc != null && rc.getReplicationType() == HddsProtos.ReplicationType.EC; + String ecPolicy = isEc ? rc.getReplication() : ""; return new FileStatusAdapter(0L, 0L, path, true, (short)0, 0L, ozoneBucket.getCreationTime().getEpochSecond() * 1000, 0L, FsPermission.getDirDefault().toShort(), owner, group, null, new BlockLocation[0], !StringUtils.isEmpty(ozoneBucket.getEncryptionKeyName()), - ozoneBucket.getReplicationConfig() != null && - ozoneBucket.getReplicationConfig().getReplicationType() == - HddsProtos.ReplicationType.EC); + isEc, + ecPolicy); } /** @@ -1225,6 +1243,9 @@ private static FileStatusAdapter getFileStatusAdapterForBucketSnapshot( ozoneSnapshot.getName(), pathStr); } Path path = new Path(pathStr); + ReplicationConfig rc = ozoneBucket.getReplicationConfig(); + boolean isEc = rc != null && rc.getReplicationType() == HddsProtos.ReplicationType.EC; + String ecPolicy = isEc ? rc.getReplication() : ""; return new FileStatusAdapter( ozoneSnapshot.getReferencedSize(), ozoneSnapshot.getReferencedReplicatedSize(), @@ -1233,9 +1254,8 @@ private static FileStatusAdapter getFileStatusAdapterForBucketSnapshot( FsPermission.getDirDefault().toShort(), owner, group, null, new BlockLocation[0], !StringUtils.isEmpty(ozoneBucket.getEncryptionKeyName()), - ozoneBucket.getReplicationConfig() != null && - ozoneBucket.getReplicationConfig().getReplicationType() == - HddsProtos.ReplicationType.EC); + isEc, + ecPolicy); } /** @@ -1264,14 +1284,16 @@ private static FileStatusAdapter getFileStatusAdapterWithSnapshotIndicator( ozoneBucket.getName(), pathStr); } Path path = new Path(pathStr); + ReplicationConfig rc = ozoneBucket.getReplicationConfig(); + boolean isEc = rc != null && rc.getReplicationType() == HddsProtos.ReplicationType.EC; + String ecPolicy = isEc ? rc.getReplication() : ""; return new FileStatusAdapter(0L, 0L, path, true, (short)0, 0L, ozoneBucket.getCreationTime().getEpochSecond() * 1000, 0L, FsPermission.getDirDefault().toShort(), owner, group, null, new BlockLocation[0], !StringUtils.isEmpty(ozoneBucket.getEncryptionKeyName()), - ozoneBucket.getReplicationConfig() != null && - ozoneBucket.getReplicationConfig().getReplicationType() == - HddsProtos.ReplicationType.EC); + isEc, + ecPolicy); } /** @@ -1286,7 +1308,7 @@ private static FileStatusAdapter getFileStatusAdapterForRoot(URI uri) { return new FileStatusAdapter(0L, 0L, path, true, (short)0, 0L, System.currentTimeMillis(), 0L, FsPermission.getDirDefault().toShort(), - null, null, null, new BlockLocation[0], false, false); + null, null, null, new BlockLocation[0], false, false, ""); } @Override diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/FileStatusAdapter.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/FileStatusAdapter.java index eba24f88bb5c..232a1e31b53b 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/FileStatusAdapter.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/FileStatusAdapter.java @@ -53,6 +53,8 @@ public final class FileStatusAdapter { private final boolean isErasureCoded; + private final String erasureCodingPolicy; + @SuppressWarnings("checkstyle:ParameterNumber") public FileStatusAdapter(long length, long diskConsumed, Path path, boolean isdir, short blockReplication, long blocksize, @@ -60,6 +62,18 @@ public FileStatusAdapter(long length, long diskConsumed, Path path, String owner, String group, Path symlink, BlockLocation[] locations, boolean isEncrypted, boolean isErasureCoded) { + this(length, diskConsumed, path, isdir, blockReplication, blocksize, + modificationTime, accessTime, permission, owner, group, symlink, + locations, isEncrypted, isErasureCoded, null); + } + + @SuppressWarnings("checkstyle:ParameterNumber") + public FileStatusAdapter(long length, long diskConsumed, Path path, + boolean isdir, short blockReplication, long blocksize, + long modificationTime, long accessTime, short permission, + String owner, String group, Path symlink, + BlockLocation[] locations, boolean isEncrypted, + boolean isErasureCoded, String erasureCodingPolicy) { this.length = length; this.diskConsumed = diskConsumed; this.path = path; @@ -75,6 +89,7 @@ public FileStatusAdapter(long length, long diskConsumed, Path path, this.blockLocations = new ArrayList<>(Arrays.asList(locations)); this.isEncrypted = isEncrypted; this.isErasureCoded = isErasureCoded; + this.erasureCodingPolicy = erasureCodingPolicy; } public Path getPath() { @@ -137,6 +152,10 @@ public boolean isErasureCoded() { return isErasureCoded; } + public String getErasureCodingPolicy() { + return erasureCodingPolicy; + } + public BlockLocation[] getBlockLocations() { return blockLocations.toArray(new BlockLocation[0]); } @@ -159,6 +178,7 @@ public String toString() { .append("; group=").append(group) .append("; permission=").append(permission) .append("; isSymlink=").append(getSymlink()) + .append("; erasureCodingPolicy=").append(erasureCodingPolicy) .append('}'); return sb.toString(); From 75eb75a5f4b945627c1c640c97ac9bcaf474461c Mon Sep 17 00:00:00 2001 From: Chi-Hsuan Huang Date: Thu, 7 May 2026 21:58:31 +0800 Subject: [PATCH 2/7] HDDS-14043. Set erasure coding policy on ContentSummary in ofs/o3fs Hadoop's "fs -ls -e" reads ContentSummary.getErasureCodingPolicy() and throws UnsupportedOperationException when it is null. ofs and o3fs were returning a ContentSummary without the field set, producing the misleading "FileSystem ofs://om does not support Erasure Coding" error on every "-ls -e" against an Ozone cluster. BasicOzoneFileSystem and BasicRootedOzoneFileSystem now set the policy on the builder using the path's own FileStatusAdapter, matching HDFS's "policy of the nearest ancestor" semantic rather than aggregating descendants. BasicOzoneFileSystem also gains a getContentSummary override so o3fs no longer falls through to the FileSystem default (which left the field null). The Builder.erasureCodingPolicy(String) call does not exist on Hadoop 2.10.2's ContentSummary.Builder, so it is isolated behind a protected applyEcPolicy hook overridden only in the Hadoop 3 subclasses (ozonefs and ozonefs-hadoop3). ozonefs-hadoop2 inherits a no-op default and is unaffected; Hadoop 2.10.2 has no "-ls -e" flag anyway. --- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 91 +++++++++++++++++++ .../fs/ozone/BasicRootedOzoneFileSystem.java | 40 ++++++-- .../hadoop/fs/ozone/OzoneFileSystem.java | 8 ++ .../fs/ozone/RootedOzoneFileSystem.java | 8 ++ .../hadoop/fs/ozone/OzoneFileSystem.java | 8 ++ .../fs/ozone/RootedOzoneFileSystem.java | 8 ++ 6 files changed, 154 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java index 8e0f4e8a5dd2..2456bfce08c2 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java @@ -51,6 +51,7 @@ import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -865,6 +866,96 @@ public BlockLocation[] getFileBlockLocations(FileStatus fileStatus, } } + @Override + public ContentSummary getContentSummary(Path f) throws IOException { + Path qualifiedPath = f.makeQualified(uri, workingDir); + String key = pathToKey(qualifiedPath); + FileStatusAdapter status; + try { + status = adapter.getFileStatus(key, uri, qualifiedPath, getUsername()); + } catch (OMException ex) { + if (ex.getResult().equals(OMException.ResultCodes.KEY_NOT_FOUND)) { + throw new FileNotFoundException("File not found. path:" + f); + } + throw ex; + } + + if (status.isFile()) { + long length = status.getLength(); + long spaceConsumed = status.getDiskConsumed(); + ContentSummary.Builder builder = new ContentSummary.Builder().length(length). + fileCount(1).directoryCount(0).spaceConsumed(spaceConsumed); + applyEcPolicy(builder, status.getErasureCodingPolicy()); + return builder.build(); + } + + long[] summary = {0, 0, 0, 1}; + for (FileStatusAdapter s : listStatusAdapter(f)) { + long length = s.getLength(); + long spaceConsumed = s.getDiskConsumed(); + ContentSummary c; + if (s.isDir()) { + c = getContentSummary(s.getPath()); + } else { + ContentSummary.Builder childBuilder = new ContentSummary.Builder().length(length). + fileCount(1).directoryCount(0).spaceConsumed(spaceConsumed); + applyEcPolicy(childBuilder, s.getErasureCodingPolicy()); + c = childBuilder.build(); + } + + summary[0] += c.getLength(); + summary[1] += c.getSpaceConsumed(); + summary[2] += c.getFileCount(); + summary[3] += c.getDirectoryCount(); + } + + String ecPolicy = status.getErasureCodingPolicy(); + if (ecPolicy == null) { + ecPolicy = ""; + } + ContentSummary.Builder builder = new ContentSummary.Builder().length(summary[0]). + fileCount(summary[2]).directoryCount(summary[3]). + spaceConsumed(summary[1]); + applyEcPolicy(builder, ecPolicy); + return builder.build(); + } + + /** + * Apply the erasure coding policy on the {@link ContentSummary.Builder}. + * Default implementation is a no-op so that this class can compile and run + * against Hadoop 2, where {@code ContentSummary.Builder.erasureCodingPolicy} + * does not exist. The Hadoop 3 subclass overrides this to set the policy. + */ + protected void applyEcPolicy(ContentSummary.Builder builder, String ecPolicy) { + // no-op; Hadoop 3 subclass overrides + } + + private List listStatusAdapter(Path f) throws IOException { + int numEntries = listingPageSize; + LinkedList statuses = new LinkedList<>(); + List tmpStatusList; + String startKey = ""; + int entriesAdded; + do { + tmpStatusList = adapter.listStatus(pathToKey(f), false, startKey, + numEntries, uri, workingDir, getUsername(), true); + entriesAdded = 0; + if (!tmpStatusList.isEmpty()) { + if (startKey.isEmpty() || !statuses.getLast().getPath().toString() + .equals(tmpStatusList.get(0).getPath().toString())) { + statuses.addAll(tmpStatusList); + entriesAdded += tmpStatusList.size(); + } else { + statuses.addAll(tmpStatusList.subList(1, tmpStatusList.size())); + entriesAdded += tmpStatusList.size() - 1; + } + startKey = pathToKey(statuses.getLast().getPath()); + } + } while (entriesAdded > 0); + + return statuses; + } + @Override public short getDefaultReplication() { return adapter.getDefaultReplication(); diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java index 67f09313d0ea..2d54e85afb00 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java @@ -1575,18 +1575,24 @@ private ContentSummary getContentSummaryInSpan(Path f) throws IOException { long length = status.getLength(); long spaceConsumed = status.getDiskConsumed(); - return new ContentSummary.Builder().length(length). - fileCount(1).directoryCount(0).spaceConsumed(spaceConsumed).build(); + ContentSummary.Builder builder = new ContentSummary.Builder().length(length). + fileCount(1).directoryCount(0).spaceConsumed(spaceConsumed); + applyEcPolicy(builder, status.getErasureCodingPolicy()); + return builder.build(); } - // f is a directory long[] summary = {0, 0, 0, 1}; - int i = 0; for (FileStatusAdapter s : listStatusAdapter(f, true)) { long length = s.getLength(); long spaceConsumed = s.getDiskConsumed(); - ContentSummary c = s.isDir() ? getContentSummary(s.getPath()) : - new ContentSummary.Builder().length(length). - fileCount(1).directoryCount(0).spaceConsumed(spaceConsumed).build(); + ContentSummary c; + if (s.isDir()) { + c = getContentSummary(s.getPath()); + } else { + ContentSummary.Builder childBuilder = new ContentSummary.Builder().length(length). + fileCount(1).directoryCount(0).spaceConsumed(spaceConsumed); + applyEcPolicy(childBuilder, s.getErasureCodingPolicy()); + c = childBuilder.build(); + } summary[0] += c.getLength(); summary[1] += c.getSpaceConsumed(); @@ -1594,9 +1600,25 @@ private ContentSummary getContentSummaryInSpan(Path f) throws IOException { summary[3] += c.getDirectoryCount(); } - return new ContentSummary.Builder().length(summary[0]). + String ecPolicy = status.getErasureCodingPolicy(); + if (ecPolicy == null) { + ecPolicy = ""; + } + ContentSummary.Builder builder = new ContentSummary.Builder().length(summary[0]). fileCount(summary[2]).directoryCount(summary[3]). - spaceConsumed(summary[1]).build(); + spaceConsumed(summary[1]); + applyEcPolicy(builder, ecPolicy); + return builder.build(); + } + + /** + * Apply the erasure coding policy on the {@link ContentSummary.Builder}. + * Default implementation is a no-op so that this class can compile and run + * against Hadoop 2, where {@code ContentSummary.Builder.erasureCodingPolicy} + * does not exist. The Hadoop 3 subclass overrides this to set the policy. + */ + protected void applyEcPolicy(ContentSummary.Builder builder, String ecPolicy) { + // no-op; Hadoop 3 subclass overrides } @Override diff --git a/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java index 5136c7343077..b23f0fb50877 100644 --- a/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java @@ -25,6 +25,7 @@ import java.util.List; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProviderTokenIssuer; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LeaseRecoverable; import org.apache.hadoop.fs.Path; @@ -183,4 +184,11 @@ public boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException { return setSafeModeUtil(action, isChecked); } + + @Override + protected void applyEcPolicy(ContentSummary.Builder builder, String ecPolicy) { + if (ecPolicy != null) { + builder.erasureCodingPolicy(ecPolicy); + } + } } diff --git a/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java index 6774b6588562..0031e57d31e5 100644 --- a/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java @@ -25,6 +25,7 @@ import java.util.List; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProviderTokenIssuer; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LeaseRecoverable; import org.apache.hadoop.fs.Path; @@ -188,4 +189,11 @@ public boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException { return setSafeModeUtil(action, isChecked); } + + @Override + protected void applyEcPolicy(ContentSummary.Builder builder, String ecPolicy) { + if (ecPolicy != null) { + builder.erasureCodingPolicy(ecPolicy); + } + } } diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java index 5136c7343077..b23f0fb50877 100644 --- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java +++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java @@ -25,6 +25,7 @@ import java.util.List; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProviderTokenIssuer; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LeaseRecoverable; import org.apache.hadoop.fs.Path; @@ -183,4 +184,11 @@ public boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException { return setSafeModeUtil(action, isChecked); } + + @Override + protected void applyEcPolicy(ContentSummary.Builder builder, String ecPolicy) { + if (ecPolicy != null) { + builder.erasureCodingPolicy(ecPolicy); + } + } } diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java index 74c4a30e9914..38dc72a77273 100644 --- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java @@ -25,6 +25,7 @@ import java.util.List; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProviderTokenIssuer; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LeaseRecoverable; import org.apache.hadoop.fs.Path; @@ -195,4 +196,11 @@ public boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException { return setSafeModeUtil(action, isChecked); } + + @Override + protected void applyEcPolicy(ContentSummary.Builder builder, String ecPolicy) { + if (ecPolicy != null) { + builder.erasureCodingPolicy(ecPolicy); + } + } } From 874bcf05e7110a0f223b639c964b1164d41dc6ce Mon Sep 17 00:00:00 2001 From: Chi-Hsuan Huang Date: Thu, 7 May 2026 21:58:42 +0800 Subject: [PATCH 3/7] HDDS-14043. Add ls -e regression tests for ofs/o3fs Adds two tests to each of AbstractOzoneFileSystemTest and AbstractRootedOzoneFileSystemTest: - testContentSummaryErasureCodingPolicy verifies a Ratis file reports "Replicated" and an EC file reports the canonical scheme name (e.g. rs-3-2-1024k); on rooted ofs the parent directory of a mixed listing also reports the bucket's policy. - testLsDashEDoesNotThrow runs Hadoop FsShell with "-ls -e" against the bucket and asserts return code 0 - the literal regression guard for the original UnsupportedOperationException. Coverage runs through TestO3FS, TestO3FSWithFSO, TestOFS and TestOFSWithFSO. --- .../fs/ozone/AbstractOzoneFileSystemTest.java | 45 ++++++++++++++++ .../AbstractRootedOzoneFileSystemTest.java | 51 +++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java index d7ee934b8f72..f5df5a48a065 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java @@ -68,6 +68,7 @@ import java.util.TreeSet; import java.util.UUID; import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.RandomUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.fs.BlockLocation; @@ -76,6 +77,7 @@ import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; @@ -426,6 +428,49 @@ public void testCreateKeyWithECReplicationConfig() throws Exception { createKeyWithECReplicationConfig(root, cluster.getConf()); } + @Test + void testContentSummaryErasureCodingPolicy() throws Exception { + String ratisKey = "ratis-ec-policy-key"; + String ecKey = "ec-policy-key"; + ECReplicationConfig ecConfig = new ECReplicationConfig("RS-3-2-1024k"); + Path parentDir = new Path(OZONE_URI_DELIMITER, "ec-policy-mixed-o3fs"); + Path ratisFile = new Path(parentDir, ratisKey); + Path ecFile = new Path(parentDir, ecKey); + + fs.mkdirs(parentDir); + String ratisRelKey = "ec-policy-mixed-o3fs/" + ratisKey; + String ecRelKey = "ec-policy-mixed-o3fs/" + ecKey; + TestDataUtil.createKey(ozoneBucket, ratisRelKey, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), + RandomUtils.secure().randomBytes(1)); + TestDataUtil.createKey(ozoneBucket, ecRelKey, ecConfig, + RandomUtils.secure().randomBytes(1)); + + try { + assertEquals("", + fs.getContentSummary(ROOT).getErasureCodingPolicy()); + assertEquals("Replicated", + fs.getContentSummary(ratisFile).getErasureCodingPolicy()); + assertEquals(ecConfig.getReplication(), + fs.getContentSummary(ecFile).getErasureCodingPolicy()); + assertEquals("", + fs.getContentSummary(parentDir).getErasureCodingPolicy()); + } finally { + fs.delete(parentDir, true); + } + } + + @Test + void testLsDashEDoesNotThrow() throws Exception { + FsShell shell = new FsShell(fs.getConf()); + try { + int exitCode = shell.run(new String[]{"-ls", "-R", "-e", fsRoot}); + assertEquals(0, exitCode); + } finally { + shell.close(); + } + } + @Test public void testDeleteCreatesFakeParentDir() throws Exception { deleteCreatesFakeParentDir(ROOT); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java index dadb3dc5baa0..d8444e1b7bc9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java @@ -79,6 +79,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; @@ -1981,6 +1982,56 @@ void testCreateAndCheckECFileDiskUsage() throws Exception { fs.delete(filePath, true); } + @Test + void testContentSummaryErasureCodingPolicy() throws Exception { + String ratisKey = "ratis-ec-policy-key"; + String ecKey = "ec-policy-key"; + ECReplicationConfig ecConfig = new ECReplicationConfig("RS-3-2-1024k"); + Path parentDir = new Path(bucketPath, "ec-policy-mixed"); + Path ratisFile = new Path(parentDir, ratisKey); + Path ecFile = new Path(parentDir, ecKey); + + fs.mkdirs(parentDir); + OzoneBucket bucket = objectStore.getVolume(volumeName).getBucket(bucketName); + String ratisRelKey = "ec-policy-mixed/" + ratisKey; + String ecRelKey = "ec-policy-mixed/" + ecKey; + TestDataUtil.createKey(bucket, ratisRelKey, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), + RandomUtils.secure().randomBytes(1)); + TestDataUtil.createKey(bucket, ecRelKey, ecConfig, + RandomUtils.secure().randomBytes(1)); + + try { + assertEquals("", + fs.getContentSummary(new Path(OZONE_URI_DELIMITER)) + .getErasureCodingPolicy()); + assertEquals("", + fs.getContentSummary(volumePath).getErasureCodingPolicy()); + assertEquals("", + fs.getContentSummary(bucketPath).getErasureCodingPolicy()); + assertEquals("Replicated", + fs.getContentSummary(ratisFile).getErasureCodingPolicy()); + assertEquals(ecConfig.getReplication(), + fs.getContentSummary(ecFile).getErasureCodingPolicy()); + assertEquals("", + fs.getContentSummary(parentDir).getErasureCodingPolicy()); + } finally { + fs.delete(parentDir, true); + } + } + + @Test + void testLsDashEDoesNotThrow() throws Exception { + FsShell shell = new FsShell(conf); + try { + int exitCode = shell.run(new String[]{"-ls", "-R", "-e", + OZONE_URI_DELIMITER + volumeName + OZONE_URI_DELIMITER + bucketName}); + assertEquals(0, exitCode); + } finally { + shell.close(); + } + } + @Test void testCreateAndCheckRatisFileDiskUsage() throws Exception { String key = "ratiskeytest"; From 56a8e0ccd1487aebafe7cd1c0899324cc51df398 Mon Sep 17 00:00:00 2001 From: Chi-Hsuan Huang Date: Thu, 7 May 2026 22:32:45 +0800 Subject: [PATCH 4/7] HDDS-14043. Address review feedback on ofs ls -e fix - Drop redundant inline `no-op` comment in default applyEcPolicy; the Javadoc above already explains the Hadoop 2/3 split. - Drop dead `ecPolicy == null` coercion in getContentSummary; every FileStatusAdapter producer this PR touches sets a non-null string and the Hadoop 3 override already null-guards. - Replace `RandomUtils.secure().randomBytes(1)` test fillers with `new byte[]{0}`; remove now-unused import in the o3fs test. - Add `testContentSummaryErasureCodingPolicyOnEcBucket` exercising the synthetic-bucket-adapter EC branch (previously unasserted). --- .../fs/ozone/AbstractOzoneFileSystemTest.java | 5 ++--- .../AbstractRootedOzoneFileSystemTest.java | 21 +++++++++++++++++-- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 7 +------ .../fs/ozone/BasicRootedOzoneFileSystem.java | 7 +------ 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java index f5df5a48a065..93c5b15d04ca 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java @@ -68,7 +68,6 @@ import java.util.TreeSet; import java.util.UUID; import org.apache.commons.lang3.RandomStringUtils; -import org.apache.commons.lang3.RandomUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.fs.BlockLocation; @@ -442,9 +441,9 @@ void testContentSummaryErasureCodingPolicy() throws Exception { String ecRelKey = "ec-policy-mixed-o3fs/" + ecKey; TestDataUtil.createKey(ozoneBucket, ratisRelKey, RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), - RandomUtils.secure().randomBytes(1)); + new byte[]{0}); TestDataUtil.createKey(ozoneBucket, ecRelKey, ecConfig, - RandomUtils.secure().randomBytes(1)); + new byte[]{0}); try { assertEquals("", diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java index d8444e1b7bc9..c8692f4dce85 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java @@ -1997,9 +1997,9 @@ void testContentSummaryErasureCodingPolicy() throws Exception { String ecRelKey = "ec-policy-mixed/" + ecKey; TestDataUtil.createKey(bucket, ratisRelKey, RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), - RandomUtils.secure().randomBytes(1)); + new byte[]{0}); TestDataUtil.createKey(bucket, ecRelKey, ecConfig, - RandomUtils.secure().randomBytes(1)); + new byte[]{0}); try { assertEquals("", @@ -2020,6 +2020,23 @@ void testContentSummaryErasureCodingPolicy() throws Exception { } } + @Test + void testContentSummaryErasureCodingPolicyOnEcBucket() throws Exception { + ECReplicationConfig ecConfig = new ECReplicationConfig("RS-3-2-1024k"); + BucketArgs ecBucketArgs = BucketArgs.newBuilder() + .setStorageType(StorageType.DISK) + .setBucketLayout(BucketLayout.LEGACY) + .setDefaultReplicationConfig(new DefaultReplicationConfig(ecConfig)) + .build(); + String ecVol = UUID.randomUUID().toString(); + String ecBuck = UUID.randomUUID().toString(); + TestDataUtil.createVolumeAndBucket(client, ecVol, ecBuck, ecBucketArgs); + + Path ecBucketPath = new Path(new Path(OZONE_URI_DELIMITER, ecVol), ecBuck); + assertEquals(ecConfig.getReplication(), + fs.getContentSummary(ecBucketPath).getErasureCodingPolicy()); + } + @Test void testLsDashEDoesNotThrow() throws Exception { FsShell shell = new FsShell(conf); diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java index 2456bfce08c2..4f0cc8b60bd8 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java @@ -909,14 +909,10 @@ public ContentSummary getContentSummary(Path f) throws IOException { summary[3] += c.getDirectoryCount(); } - String ecPolicy = status.getErasureCodingPolicy(); - if (ecPolicy == null) { - ecPolicy = ""; - } ContentSummary.Builder builder = new ContentSummary.Builder().length(summary[0]). fileCount(summary[2]).directoryCount(summary[3]). spaceConsumed(summary[1]); - applyEcPolicy(builder, ecPolicy); + applyEcPolicy(builder, status.getErasureCodingPolicy()); return builder.build(); } @@ -927,7 +923,6 @@ public ContentSummary getContentSummary(Path f) throws IOException { * does not exist. The Hadoop 3 subclass overrides this to set the policy. */ protected void applyEcPolicy(ContentSummary.Builder builder, String ecPolicy) { - // no-op; Hadoop 3 subclass overrides } private List listStatusAdapter(Path f) throws IOException { diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java index 2d54e85afb00..017b4003d36a 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java @@ -1600,14 +1600,10 @@ private ContentSummary getContentSummaryInSpan(Path f) throws IOException { summary[3] += c.getDirectoryCount(); } - String ecPolicy = status.getErasureCodingPolicy(); - if (ecPolicy == null) { - ecPolicy = ""; - } ContentSummary.Builder builder = new ContentSummary.Builder().length(summary[0]). fileCount(summary[2]).directoryCount(summary[3]). spaceConsumed(summary[1]); - applyEcPolicy(builder, ecPolicy); + applyEcPolicy(builder, status.getErasureCodingPolicy()); return builder.build(); } @@ -1618,7 +1614,6 @@ private ContentSummary getContentSummaryInSpan(Path f) throws IOException { * does not exist. The Hadoop 3 subclass overrides this to set the policy. */ protected void applyEcPolicy(ContentSummary.Builder builder, String ecPolicy) { - // no-op; Hadoop 3 subclass overrides } @Override From 4634f83acab9d516e633e22a544fca149471b2d8 Mon Sep 17 00:00:00 2001 From: Chi-Hsuan Huang Date: Thu, 7 May 2026 22:43:49 +0800 Subject: [PATCH 5/7] HDDS-14043. Restore "f is a directory" comment in getContentSummary --- .../org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java index 017b4003d36a..b61dca450cb7 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java @@ -1580,6 +1580,7 @@ private ContentSummary getContentSummaryInSpan(Path f) throws IOException { applyEcPolicy(builder, status.getErasureCodingPolicy()); return builder.build(); } + // f is a directory long[] summary = {0, 0, 0, 1}; for (FileStatusAdapter s : listStatusAdapter(f, true)) { long length = s.getLength(); From 213969082d06c780636de1f931485b4d6925a885 Mon Sep 17 00:00:00 2001 From: Chi-Hsuan Huang Date: Thu, 7 May 2026 23:10:07 +0800 Subject: [PATCH 6/7] HDDS-14043. Clean up EC bucket in testContentSummaryErasureCodingPolicyOnEcBucket --- .../fs/ozone/AbstractRootedOzoneFileSystemTest.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java index c8692f4dce85..d8d0098cc154 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java @@ -2032,9 +2032,14 @@ void testContentSummaryErasureCodingPolicyOnEcBucket() throws Exception { String ecBuck = UUID.randomUUID().toString(); TestDataUtil.createVolumeAndBucket(client, ecVol, ecBuck, ecBucketArgs); - Path ecBucketPath = new Path(new Path(OZONE_URI_DELIMITER, ecVol), ecBuck); - assertEquals(ecConfig.getReplication(), - fs.getContentSummary(ecBucketPath).getErasureCodingPolicy()); + Path ecVolPath = new Path(OZONE_URI_DELIMITER, ecVol); + Path ecBucketPath = new Path(ecVolPath, ecBuck); + try { + assertEquals(ecConfig.getReplication(), + fs.getContentSummary(ecBucketPath).getErasureCodingPolicy()); + } finally { + fs.delete(ecVolPath, true); + } } @Test From f3d3423519c2ead83af47e49aa4126ffa7a0bff1 Mon Sep 17 00:00:00 2001 From: Chi-Hsuan Huang Date: Thu, 7 May 2026 23:56:31 +0800 Subject: [PATCH 7/7] HDDS-14043. Delete bucket, not volume, when cleaning up EC bucket test --- .../hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java index d8d0098cc154..5e7432f97dda 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java @@ -2032,13 +2032,12 @@ void testContentSummaryErasureCodingPolicyOnEcBucket() throws Exception { String ecBuck = UUID.randomUUID().toString(); TestDataUtil.createVolumeAndBucket(client, ecVol, ecBuck, ecBucketArgs); - Path ecVolPath = new Path(OZONE_URI_DELIMITER, ecVol); - Path ecBucketPath = new Path(ecVolPath, ecBuck); + Path ecBucketPath = new Path(new Path(OZONE_URI_DELIMITER, ecVol), ecBuck); try { assertEquals(ecConfig.getReplication(), fs.getContentSummary(ecBucketPath).getErasureCodingPolicy()); } finally { - fs.delete(ecVolPath, true); + fs.delete(ecBucketPath, true); } }