HDDS-15175. Introduce StorageClass and storage policy

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneStoragePolicy.java

@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.client;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.StoragePolicyProto;
+
+/**
+ * Enum defining different storage policies using StorageTier.
+ */
+public enum OzoneStoragePolicy implements StoragePolicy {
+
+  HOT("Hot", StorageTier.SSD, StorageTier.DISK),
+  WARM("Warm", StorageTier.DISK, StorageTier.EMPTY),
+  COLD("Cold", StorageTier.ARCHIVE, StorageTier.EMPTY);
+
+  private final String name;
+  private final StorageTier creationTier;
+  private final StorageTier creationFallbackTier;
+
+  OzoneStoragePolicy(String name, StorageTier creationTier,
+      StorageTier creationFallbackTier) {
+    this.name = name;
+    this.creationTier = creationTier;
+    this.creationFallbackTier = creationFallbackTier;
+  }
+
+  @Override
+  public String getName() {
+    return name;
+  }
+
+  @Override
+  public StorageTier getCreationTier() {
+    return creationTier;
+  }
+
+  @Override
+  public StorageTier getCreationFallbackTier() {
+    return creationFallbackTier;
+  }
+
+  /**
+   * Converts the current StoragePolicyType to its protobuf representation.
+   * @return the corresponding StoragePolicyProto.
+   */
+  public StoragePolicyProto toProto() {
+    switch (this) {
+    case HOT:
+      return StoragePolicyProto.HOT;
+    case WARM:
+      return StoragePolicyProto.WARM;
+    case COLD:
+      return StoragePolicyProto.COLD;
+    default:
+      throw new IllegalArgumentException(
+          "Error: StoragePolicyType not found, type=" + this);
+    }
+  }
+
+  /**
+   * Converts a protobuf StoragePolicyProto to the corresponding StoragePolicyType.
+   * @param proto the StoragePolicyProto to convert.
+   * @return the corresponding StoragePolicyType.
+   */
+  public static OzoneStoragePolicy fromProto(StoragePolicyProto proto) {
+    if (proto == null) {
+      throw new IllegalArgumentException("StoragePolicyProto cannot be null");
+    }
+    switch (proto) {
+    case HOT:
+      return HOT;
+    case WARM:
+      return WARM;
+    case COLD:
+      return COLD;
+    default:
+      throw new IllegalArgumentException("Error: StoragePolicyProto not found, proto=" + proto);
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "OzoneStoragePolicy{"
+        + "name=" + name
+        + ", creationTier=" + creationTier
+        + ", creationFallbackTier=" + creationFallbackTier
+        + '}';
+  }
+}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StoragePolicy.java

@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.client;
+
+/**
+ * Interface for storage policies that define how to select storage tiers for data replication.
+ *
+ * <p>A storage policy specifies the preferred and fallback storage tiers for placing
+ * block replicas.
+ */
+public interface StoragePolicy {
+
+  /**
+   * Retrieves the name of the storage policy.
+   *
+   * @return a string representing the name of the storage policy.
+   */
+  String getName();
+
+  /**
+   * Retrieves the preferred storage tier used for placing data replicas.
+   *
+   * <p>This is the preferred storage tier where new data is initially stored
+   * according to the specified storage policy.
+   *
+   * @return the default {@link StorageTier} used for data placement.
+   */
+  StorageTier getCreationTier();
+
+  /**
+   * Retrieves the fallback storage tier used during the creation of new data replicas.
+   *
+   * <p>If the preferred storage tier is unavailable, this fallback tier is used to
+   * ensure that new data can still be reliably stored.
+   *
+   * @return the fallback {@link StorageTier} used for data placement.
+   */
+  StorageTier getCreationFallbackTier();
+}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StorageTier.java

@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.client;
+
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.StorageTierProto;
+
+/**
+ * Ozone specific storage tiers.
+ */
+public enum StorageTier {
+  SSD("SSD", StorageType.SSD),
+  DISK("DISK", StorageType.DISK),
+  ARCHIVE("ARCHIVE", StorageType.ARCHIVE),
+  EMPTY("EMPTY");
+
+  private final String tierName;
+  private final List<StorageType> storageTypes;
+  private final boolean uniformStorageType;
+  private static final Map<StorageTier, Map<ReplicationConfig, List<StorageType>>>
+      CACHE = new EnumMap<>(StorageTier.class);
+
+  StorageTier(String tierName) {
+    this.tierName = tierName;
+    this.storageTypes = Collections.emptyList();
+    this.uniformStorageType = true;
+  }
+
+  // Constructor for uniform storage tiers
+  StorageTier(String tierName, StorageType uniformStorageType) {
+    this.tierName = tierName;
+    this.storageTypes = Collections.singletonList(uniformStorageType);
+    this.uniformStorageType = true;
+  }
+
+  // Constructor for non-uniform storage tiers
+  StorageTier(String tierName, StorageType... storageTypes) {
+    this.tierName = tierName;
+    if (Arrays.stream(storageTypes).distinct().count() <= 1) {
+      throw new IllegalArgumentException("StorageTier '" + tierName +
+          "' requires at least two different StorageType instances." +
+          " but only " + Arrays.stream(storageTypes).distinct().count() +
+          " StorageType were provided.");
+    }
+    this.storageTypes = Arrays.asList(storageTypes);
+    this.uniformStorageType = false;
+  }
+
+  static {
+    // Precompute storage type mappings for each replication config
+    for (StorageTier tier : StorageTier.values()) {
+      Map<ReplicationConfig, List<StorageType>> tierCache = new HashMap<>();
+      List<ReplicationConfig> replicationConfigs = Arrays.asList(
+          RatisReplicationConfig.getInstance(ONE),
+          RatisReplicationConfig.getInstance(THREE),
+          StandaloneReplicationConfig.getInstance(ONE),
+          StandaloneReplicationConfig.getInstance(THREE)
+      );
+
+      for (ReplicationConfig config : replicationConfigs) {
+        tierCache.put(config, tier.computeStorageTypes(config));
+      }
+      CACHE.put(tier, tierCache);
+    }
+  }
+
+  public StorageTierProto toProto() {
+    switch (this) {
+    case SSD:
+      return StorageTierProto.SSD_TIER;
+    case DISK:
+      return StorageTierProto.DISK_TIER;
+    case ARCHIVE:
+      return StorageTierProto.ARCHIVE_TIER;
+    default:
+      throw new IllegalStateException(
+          "Illegal StorageTier: " + this);
+    }
+  }
+
+  public static StorageTier fromProto(StorageTierProto tier) {
+    switch (tier) {
+    case SSD_TIER:
+      return SSD;
+    case DISK_TIER:
+      return DISK;
+    case ARCHIVE_TIER:
+      return ARCHIVE;
+    default:
+      throw new IllegalStateException(
+          "Illegal StorageTierProto: " + tier);
+    }
+  }
+
+  public String getTierName() {
+    return tierName;
+  }
+
+  public boolean isUniformStorageType() {
+    return uniformStorageType;
+  }
+
+  /**
+   * Computes the list of StorageTypes based on replication configuration.
+   *
+   * @param replicationConfig The replication configuration.
+   * @return The list of StorageTypes for the given tier and replication configuration.
+   */
+  private List<StorageType> computeStorageTypes(
+      ReplicationConfig replicationConfig) {
+    if (isUniformStorageType()) {
+      int numberOfNodes = replicationConfig.getRequiredNodes();
+      if (storageTypes.isEmpty()) {
+        return Collections.emptyList();
+      }
+      return new ArrayList<>(Collections.nCopies(numberOfNodes, storageTypes.get(0)));
+    } else {
+      throw new UnsupportedOperationException(
+          "Unsupported not UniformStorage Storage Tier: " + replicationConfig);
+    }
+  }
+
+  /**
+   * Maps a StorageTier to its corresponding StorageType based on replication type.
+   *
+   * @param replicationConfig The replication configuration.
+   * @return The list of StorageTypes corresponding to the given tier and replication configuration.
+   * @throws IllegalArgumentException if the replication configuration is not supported.
+   */
+  public List<StorageType> getStorageTypes(
+      ReplicationConfig replicationConfig) {
+    Map<ReplicationConfig, List<StorageType>> tierCache = CACHE.get(this);
+
+    if (tierCache != null) {
+      List<StorageType> cachedStorageType = tierCache.get(replicationConfig);
+      if (cachedStorageType != null) {
+        return cachedStorageType;
+      }
+    }
+
+    throw new IllegalArgumentException("Unsupported ReplicationConfig: " +
+        replicationConfig + " for StorageTier: " + getTierName());
+  }
+
+}

hadoop-hdds/interface-client/src/main/proto/hdds.proto

@@ -326,6 +326,18 @@ enum ReplicationFactor {
     ZERO = 0; // Invalid Factor
 }
 
+enum StorageTierProto {
+    DISK_TIER = 1;
+    SSD_TIER = 2;
+    ARCHIVE_TIER = 3;
+}
+
+enum StoragePolicyProto {
+    HOT = 1;
+    WARM = 2;
+    COLD = 3;
+}
+
 message ECReplicationConfig {
     required int32 data = 1;
     required int32 parity = 2;

hadoop-ozone/common/src/main/java/org/apache/hadoop/hdds/protocol/StorageType.java

@@ -21,7 +21,13 @@
 
 /**
  * Ozone specific storage types.
+ *
+ * @deprecated Ozone buckets should not have a StorageType attribute.
+ * This class is specific to `OzoneBucket` and is planned for removal in future versions.
+ * It is recommended to use `{@link org.apache.hadoop.fs.StorageType}` instead for
+ * any storage type requirements.
  */
+@Deprecated
 public enum StorageType {
   RAM_DISK,
   SSD,