HDDS-14043. Fix ls -e UnsupportedOperationException on ofs/o3fs

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java

@@ -76,6 +76,7 @@
 import org.apache.hadoop.fs.FileAlreadyExistsException;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsShell;
 import org.apache.hadoop.fs.InvalidPathException;
 import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
@@ -426,6 +427,49 @@ public void testCreateKeyWithECReplicationConfig() throws Exception {
     createKeyWithECReplicationConfig(root, cluster.getConf());
   }
 
+  @Test
+  void testContentSummaryErasureCodingPolicy() throws Exception {
+    String ratisKey = "ratis-ec-policy-key";
+    String ecKey = "ec-policy-key";
+    ECReplicationConfig ecConfig = new ECReplicationConfig("RS-3-2-1024k");
+    Path parentDir = new Path(OZONE_URI_DELIMITER, "ec-policy-mixed-o3fs");
+    Path ratisFile = new Path(parentDir, ratisKey);
+    Path ecFile = new Path(parentDir, ecKey);
+
+    fs.mkdirs(parentDir);
+    String ratisRelKey = "ec-policy-mixed-o3fs/" + ratisKey;
+    String ecRelKey = "ec-policy-mixed-o3fs/" + ecKey;
+    TestDataUtil.createKey(ozoneBucket, ratisRelKey,
+        RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE),
+        new byte[]{0});
+    TestDataUtil.createKey(ozoneBucket, ecRelKey, ecConfig,
+        new byte[]{0});
+
+    try {
+      assertEquals("",
+          fs.getContentSummary(ROOT).getErasureCodingPolicy());
+      assertEquals("Replicated",
+          fs.getContentSummary(ratisFile).getErasureCodingPolicy());
+      assertEquals(ecConfig.getReplication(),
+          fs.getContentSummary(ecFile).getErasureCodingPolicy());
+      assertEquals("",
+          fs.getContentSummary(parentDir).getErasureCodingPolicy());
+    } finally {
+      fs.delete(parentDir, true);
+    }
+  }
+
+  @Test
+  void testLsDashEDoesNotThrow() throws Exception {
+    FsShell shell = new FsShell(fs.getConf());
+    try {
+      int exitCode = shell.run(new String[]{"-ls", "-R", "-e", fsRoot});
+      assertEquals(0, exitCode);
+    } finally {
+      shell.close();
+    }
+  }
+
   @Test
   public void testDeleteCreatesFakeParentDir() throws Exception {
     deleteCreatesFakeParentDir(ROOT);

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java

@@ -79,6 +79,7 @@
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsShell;
 import org.apache.hadoop.fs.InvalidPathException;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
@@ -1981,6 +1982,77 @@ void testCreateAndCheckECFileDiskUsage() throws Exception {
     fs.delete(filePath, true);
   }
 
+  @Test
+  void testContentSummaryErasureCodingPolicy() throws Exception {
+    String ratisKey = "ratis-ec-policy-key";
+    String ecKey = "ec-policy-key";
+    ECReplicationConfig ecConfig = new ECReplicationConfig("RS-3-2-1024k");
+    Path parentDir = new Path(bucketPath, "ec-policy-mixed");
+    Path ratisFile = new Path(parentDir, ratisKey);
+    Path ecFile = new Path(parentDir, ecKey);
+
+    fs.mkdirs(parentDir);
+    OzoneBucket bucket = objectStore.getVolume(volumeName).getBucket(bucketName);
+    String ratisRelKey = "ec-policy-mixed/" + ratisKey;
+    String ecRelKey = "ec-policy-mixed/" + ecKey;
+    TestDataUtil.createKey(bucket, ratisRelKey,
+        RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE),
+        new byte[]{0});
+    TestDataUtil.createKey(bucket, ecRelKey, ecConfig,
+        new byte[]{0});
+
+    try {
+      assertEquals("",
+          fs.getContentSummary(new Path(OZONE_URI_DELIMITER))
+              .getErasureCodingPolicy());
+      assertEquals("",
+          fs.getContentSummary(volumePath).getErasureCodingPolicy());
+      assertEquals("",
+          fs.getContentSummary(bucketPath).getErasureCodingPolicy());
+      assertEquals("Replicated",
+          fs.getContentSummary(ratisFile).getErasureCodingPolicy());
+      assertEquals(ecConfig.getReplication(),
+          fs.getContentSummary(ecFile).getErasureCodingPolicy());
+      assertEquals("",
+          fs.getContentSummary(parentDir).getErasureCodingPolicy());
+    } finally {
+      fs.delete(parentDir, true);
+    }
+  }
+
+  @Test
+  void testContentSummaryErasureCodingPolicyOnEcBucket() throws Exception {
+    ECReplicationConfig ecConfig = new ECReplicationConfig("RS-3-2-1024k");
+    BucketArgs ecBucketArgs = BucketArgs.newBuilder()
+        .setStorageType(StorageType.DISK)
+        .setBucketLayout(BucketLayout.LEGACY)
+        .setDefaultReplicationConfig(new DefaultReplicationConfig(ecConfig))
+        .build();
+    String ecVol = UUID.randomUUID().toString();
+    String ecBuck = UUID.randomUUID().toString();
+    TestDataUtil.createVolumeAndBucket(client, ecVol, ecBuck, ecBucketArgs);
+
+    Path ecBucketPath = new Path(new Path(OZONE_URI_DELIMITER, ecVol), ecBuck);
+    try {
+      assertEquals(ecConfig.getReplication(),
+          fs.getContentSummary(ecBucketPath).getErasureCodingPolicy());
+    } finally {
+      fs.delete(ecBucketPath, true);
+    }
+  }
+
+  @Test
+  void testLsDashEDoesNotThrow() throws Exception {
+    FsShell shell = new FsShell(conf);
+    try {
+      int exitCode = shell.run(new String[]{"-ls", "-R", "-e",
+          OZONE_URI_DELIMITER + volumeName + OZONE_URI_DELIMITER + bucketName});
+      assertEquals(0, exitCode);
+    } finally {
+      shell.close();
+    }
+  }
+
   @Test
   void testCreateAndCheckRatisFileDiskUsage() throws Exception {
     String key = "ratiskeytest";

hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java

@@ -537,6 +537,13 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatus status,
     OmKeyInfo keyInfo = status.getKeyInfo();
     short replication = (short) keyInfo.getReplicationConfig()
         .getRequiredNodes();
+    boolean isEc = OzoneClientUtils.isKeyErasureCode(keyInfo);
+    String ecPolicy;
+    if (isEc) {
+      ecPolicy = keyInfo.getReplicationConfig().getReplication();
+    } else {
+      ecPolicy = status.isFile() ? "Replicated" : "";
+    }
     return new FileStatusAdapter(
         keyInfo.getDataSize(),
         keyInfo.getReplicatedSize(),
@@ -553,7 +560,8 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatus status,
         null,
         getBlockLocations(status),
         OzoneClientUtils.isKeyEncrypted(keyInfo),
-        OzoneClientUtils.isKeyErasureCode(keyInfo)
+        isEc,
+        ecPolicy
     );
   }
 
@@ -562,6 +570,13 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatusLight status,
     BasicOmKeyInfo keyInfo = status.getKeyInfo();
     short replication = (short) keyInfo.getReplicationConfig()
         .getRequiredNodes();
+    boolean isEc = OzoneClientUtils.isKeyErasureCode(keyInfo);
+    String ecPolicy;
+    if (isEc) {
+      ecPolicy = keyInfo.getReplicationConfig().getReplication();
+    } else {
+      ecPolicy = status.isFile() ? "Replicated" : "";
+    }
     return new FileStatusAdapter(
         keyInfo.getDataSize(),
         keyInfo.getReplicatedSize(),
@@ -578,7 +593,8 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatusLight status,
         null,
         getBlockLocations(null),
         keyInfo.isEncrypted(),
-        OzoneClientUtils.isKeyErasureCode(keyInfo)
+        isEc,
+        ecPolicy
     );
   }
 

hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java

@@ -51,6 +51,7 @@
 import java.util.stream.Collectors;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -865,6 +866,91 @@ public BlockLocation[] getFileBlockLocations(FileStatus fileStatus,
     }
   }
 
+  @Override
+  public ContentSummary getContentSummary(Path f) throws IOException {
+    Path qualifiedPath = f.makeQualified(uri, workingDir);
+    String key = pathToKey(qualifiedPath);
+    FileStatusAdapter status;
+    try {
+      status = adapter.getFileStatus(key, uri, qualifiedPath, getUsername());
+    } catch (OMException ex) {
+      if (ex.getResult().equals(OMException.ResultCodes.KEY_NOT_FOUND)) {
+        throw new FileNotFoundException("File not found. path:" + f);
+      }
+      throw ex;
+    }
+
+    if (status.isFile()) {
+      long length = status.getLength();
+      long spaceConsumed = status.getDiskConsumed();
+      ContentSummary.Builder builder = new ContentSummary.Builder().length(length).
+          fileCount(1).directoryCount(0).spaceConsumed(spaceConsumed);
+      applyEcPolicy(builder, status.getErasureCodingPolicy());
+      return builder.build();
+    }
+
+    long[] summary = {0, 0, 0, 1};
+    for (FileStatusAdapter s : listStatusAdapter(f)) {
+      long length = s.getLength();
+      long spaceConsumed = s.getDiskConsumed();
+      ContentSummary c;
+      if (s.isDir()) {
+        c = getContentSummary(s.getPath());
+      } else {
+        ContentSummary.Builder childBuilder = new ContentSummary.Builder().length(length).
+            fileCount(1).directoryCount(0).spaceConsumed(spaceConsumed);
+        applyEcPolicy(childBuilder, s.getErasureCodingPolicy());
+        c = childBuilder.build();
+      }
+
+      summary[0] += c.getLength();
+      summary[1] += c.getSpaceConsumed();
+      summary[2] += c.getFileCount();
+      summary[3] += c.getDirectoryCount();
+    }
+
+    ContentSummary.Builder builder = new ContentSummary.Builder().length(summary[0]).
+        fileCount(summary[2]).directoryCount(summary[3]).
+        spaceConsumed(summary[1]);
+    applyEcPolicy(builder, status.getErasureCodingPolicy());
+    return builder.build();
+  }
+
+  /**
+   * Apply the erasure coding policy on the {@link ContentSummary.Builder}.
+   * Default implementation is a no-op so that this class can compile and run
+   * against Hadoop 2, where {@code ContentSummary.Builder.erasureCodingPolicy}
+   * does not exist. The Hadoop 3 subclass overrides this to set the policy.
+   */
+  protected void applyEcPolicy(ContentSummary.Builder builder, String ecPolicy) {
+  }
+
+  private List<FileStatusAdapter> listStatusAdapter(Path f) throws IOException {
+    int numEntries = listingPageSize;
+    LinkedList<FileStatusAdapter> statuses = new LinkedList<>();
+    List<FileStatusAdapter> tmpStatusList;
+    String startKey = "";
+    int entriesAdded;
+    do {
+      tmpStatusList = adapter.listStatus(pathToKey(f), false, startKey,
+          numEntries, uri, workingDir, getUsername(), true);
+      entriesAdded = 0;
+      if (!tmpStatusList.isEmpty()) {
+        if (startKey.isEmpty() || !statuses.getLast().getPath().toString()
+            .equals(tmpStatusList.get(0).getPath().toString())) {
+          statuses.addAll(tmpStatusList);
+          entriesAdded += tmpStatusList.size();
+        } else {
+          statuses.addAll(tmpStatusList.subList(1, tmpStatusList.size()));
+          entriesAdded += tmpStatusList.size() - 1;
+        }
+        startKey = pathToKey(statuses.getLast().getPath());
+      }
+    } while (entriesAdded > 0);
+
+    return statuses;
+  }
+
   @Override
   public short getDefaultReplication() {
     return adapter.getDefaultReplication();