Skip to content

HDDS-15146. Remove getDatanodeInfo(DatanodeDetails) from NodeManager. #10169

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
szetszwo wants to merge 6 commits into
base: master
Choose a base branch
from
Open

HDDS-15146. Remove getDatanodeInfo(DatanodeDetails) from NodeManager. #10169

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
27faa90
HDDS-15146. Reomve getDatanodeInfo(DatanodeDetails) from NodeManager.
szetszwo Apr 30, 2026
c68ad09
Remove some queryNode methods
szetszwo May 1, 2026
96d3a51
Change getNode to return DatanodeInfo
szetszwo May 2, 2026
5836bd5
Revert "Change getNode to return DatanodeInfo"
szetszwo May 4, 2026
2ffdac3
Redo: Change getNode to return DatanodeInfo
szetszwo May 4, 2026
eed85f6
Fix findbugs
szetszwo May 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 4 additions & 15 deletions hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ default void registerSendCommandNotify(SCMCommandProto.Type type,
* @param health - The health of the node
* @return List of Datanodes that are Heartbeating SCM.
*/
List<DatanodeDetails> getNodes(
List<DatanodeInfo> getNodes(
NodeOperationalState opState, NodeState health);

/**
Expand All @@ -134,10 +134,8 @@ List<DatanodeDetails> getNodes(
int getNodeCount(
NodeOperationalState opState, NodeState health);

/**
* @return all datanodes known to SCM.
*/
List<? extends DatanodeDetails> getAllNodes();
/** @return a shadow copied list of all datanodes, sorted by {@link DatanodeID}. */
List<DatanodeInfo> getAllNodes();

/** @return the number of datanodes. */
default int getAllNodeCount() {
Expand Down Expand Up @@ -175,15 +173,6 @@ default int getAllNodeCount() {
*/
DatanodeUsageInfo getUsageInfo(DatanodeDetails dn);

/**
* Get the datanode info of a specified datanode.
*
* @param dn the usage of which we want to get
* @return DatanodeInfo of the specified datanode
*/
@Nullable
DatanodeInfo getDatanodeInfo(DatanodeDetails dn);

/**
* True if the node can accept another container of the given size.
*/
Expand Down Expand Up @@ -381,7 +370,7 @@ Map<SCMCommandProto.Type, Integer> getTotalDatanodeCommandCounts(
List<SCMCommand<?>> getCommandQueue(DatanodeID dnID);

/** @return the datanode of the given id if it exists; otherwise, return null. */
@Nullable DatanodeDetails getNode(@Nullable DatanodeID id);
@Nullable DatanodeInfo getNode(@Nullable DatanodeID id);

/**
* Given datanode address(Ipaddress or hostname), returns a list of
Expand Down
8 changes: 2 additions & 6 deletions hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -531,12 +531,8 @@ public int getVolumeFailuresNodeCount() {
return getVolumeFailuresNodes().size();
}

/**
* Returns all the nodes which have registered to NodeStateManager.
*
* @return all the managed nodes
*/
public List<DatanodeInfo> getAllNodes() {
/** @return a shadow copied list of all datanodes, sorted by {@link DatanodeID}. */
List<DatanodeInfo> getAllNodes() {
return nodeStateMap.getAllDatanodeInfos();
}

Expand Down
28 changes: 3 additions & 25 deletions hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import jakarta.annotation.Nullable;
import java.io.IOException;
import java.math.RoundingMode;
import java.net.InetAddress;
Expand Down Expand Up @@ -260,11 +259,9 @@ public List<DatanodeDetails> getNodes(NodeStatus nodeStatus) {
* @return List of Datanodes that are known to SCM in the requested states.
*/
@Override
public List<DatanodeDetails> getNodes(
public List<DatanodeInfo> getNodes(
NodeOperationalState opState, NodeState health) {
return nodeStateManager.getNodes(opState, health)
.stream()
.map(node -> (DatanodeDetails)node).collect(Collectors.toList());
return nodeStateManager.getNodes(opState, health);
}

@Override
Expand Down Expand Up @@ -1005,8 +1002,7 @@ public Map<DatanodeDetails, SCMNodeStat> getNodeStats() {
@Override
public List<DatanodeUsageInfo> getMostOrLeastUsedDatanodes(
boolean mostUsed) {
List<DatanodeDetails> healthyNodes =
getNodes(IN_SERVICE, NodeState.HEALTHY);
final List<DatanodeInfo> healthyNodes = getNodes(IN_SERVICE, NodeState.HEALTHY);

List<DatanodeUsageInfo> datanodeUsageInfoList =
new ArrayList<>(healthyNodes.size());
Expand Down Expand Up @@ -1053,24 +1049,6 @@ public DatanodeUsageInfo getUsageInfo(DatanodeDetails dn) {
return usageInfo;
}

/**
* Get the usage info of a specified datanode.
*
* @param dn the usage of which we want to get
* @return DatanodeUsageInfo of the specified datanode
*/
@Override
@Nullable
public DatanodeInfo getDatanodeInfo(DatanodeDetails dn) {
try {
return nodeStateManager.getNode(dn);
} catch (NodeNotFoundException e) {
LOG.warn("Cannot retrieve DatanodeInfo, datanode {} not found.",
dn.getUuid());
return null;
}
}

/**
* Effective space check aligned with container allocation: per-disk slot model minus
* SCM pending allocations.
Expand Down
10 changes: 3 additions & 7 deletions ...op-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/NodeStateMap.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@

package org.apache.hadoop.hdds.scm.node.states;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Function;
Expand All @@ -41,7 +41,7 @@
*/
public class NodeStateMap {
/** Map: {@link DatanodeID} -> ({@link DatanodeInfo}, {@link ContainerID}s). */
private final Map<DatanodeID, DatanodeEntry> nodeMap = new HashMap<>();
private final Map<DatanodeID, DatanodeEntry> nodeMap = new TreeMap<>();

private final ReadWriteLock lock = new ReentrantReadWriteLock();

Expand Down Expand Up @@ -166,11 +166,7 @@ public int getNodeCount() {
}
}

/**
* Returns the list of all the nodes as DatanodeInfo objects.
*
* @return list of all the node ids
*/
/** @return a shadow copied list of all datanodes, sorted by {@link DatanodeID}. */
public List<DatanodeInfo> getAllDatanodeInfos() {
lock.readLock().lock();
try {
Expand Down
81 changes: 15 additions & 66 deletions ...s/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.TreeSet;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.Stream;
Expand Down Expand Up @@ -678,9 +677,9 @@ public List<HddsProtos.Node> queryNode(
}
try {
List<HddsProtos.Node> result = new ArrayList<>();
for (DatanodeDetails node : queryNode(opState, state)) {
for (DatanodeDetails node : scm.getScmNodeManager().getNodes(opState, state)) {
NodeStatus ns = scm.getScmNodeManager().getNodeStatus(node);
DatanodeInfo datanodeInfo = scm.getScmNodeManager().getDatanodeInfo(node);
DatanodeInfo datanodeInfo = node instanceof DatanodeInfo ? (DatanodeInfo) node : null;
Copy link
Copy Markdown
Contributor

@sreejasahithi sreejasahithi May 5, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why do we need instanceof check here, we already know that getNodes returns DatanodeInfo?

HddsProtos.Node.Builder nodeBuilder = HddsProtos.Node.newBuilder()
.setNodeID(node.toProto(clientVersion))
.addNodeStates(ns.getHealth())
Expand All @@ -704,34 +703,22 @@ public List<HddsProtos.Node> queryNode(
}

@Override
public HddsProtos.Node queryNode(UUID uuid)
throws IOException {
public HddsProtos.Node queryNode(UUID uuid) {
final Map<String, String> auditMap = Maps.newHashMap();
auditMap.put("uuid", String.valueOf(uuid));
HddsProtos.Node result = null;
try {
DatanodeDetails node = scm.getScmNodeManager().getNode(DatanodeID.of(uuid));
if (node != null) {
NodeStatus ns = scm.getScmNodeManager().getNodeStatus(node);
DatanodeInfo datanodeInfo = scm.getScmNodeManager().getDatanodeInfo(node);
HddsProtos.Node.Builder nodeBuilder = HddsProtos.Node.newBuilder()
.setNodeID(node.getProtoBufMessage())
.addNodeStates(ns.getHealth())
.addNodeOperationalStates(ns.getOperationalState());

if (datanodeInfo != null) {
nodeBuilder.setTotalVolumeCount(datanodeInfo.getStorageReports().size());
nodeBuilder.setHealthyVolumeCount(datanodeInfo.getHealthyVolumeCount());
addFailedVolumes(nodeBuilder, datanodeInfo);
}
result = nodeBuilder.build();
}
} catch (NodeNotFoundException e) {
IOException ex = new IOException(
"An unexpected error occurred querying the NodeStatus", e);
AUDIT.logReadFailure(buildAuditMessageForFailure(
SCMAction.QUERY_NODE, auditMap, ex));
throw ex;
DatanodeInfo datanodeInfo = scm.getScmNodeManager().getNode(DatanodeID.of(uuid));
if (datanodeInfo != null) {
NodeStatus ns = datanodeInfo.getNodeStatus();
HddsProtos.Node.Builder nodeBuilder = HddsProtos.Node.newBuilder()
.setNodeID(datanodeInfo.getProtoBufMessage())
.addNodeStates(ns.getHealth())
.addNodeOperationalStates(ns.getOperationalState());

nodeBuilder.setTotalVolumeCount(datanodeInfo.getStorageReports().size());
nodeBuilder.setHealthyVolumeCount(datanodeInfo.getHealthyVolumeCount());
addFailedVolumes(nodeBuilder, datanodeInfo);
result = nodeBuilder.build();
}
AUDIT.logReadSuccess(buildAuditMessageForSuccess(
SCMAction.QUERY_NODE, auditMap));
Expand Down Expand Up @@ -1580,26 +1567,6 @@ public List<ContainerID> getListOfContainerIDs(
}
}

/**
* Queries a list of Node that match a set of statuses.
*
* <p>For example, if the nodeStatuses is HEALTHY and RAFT_MEMBER, then
* this call will return all
* healthy nodes which members in Raft pipeline.
*
* <p>Right now we don't support operations, so we assume it is an AND
* operation between the
* operators.
*
* @param opState - NodeOperational State
* @param state - NodeState.
* @return List of Datanodes.
*/
public List<DatanodeDetails> queryNode(
HddsProtos.NodeOperationalState opState, HddsProtos.NodeState state) {
return new ArrayList<>(queryNodeState(opState, state));
}

@VisibleForTesting
public StorageContainerManager getScm() {
return scm;
Expand All @@ -1612,24 +1579,6 @@ public boolean getSafeModeStatus() {
return scm.getScmContext().isInSafeMode();
}

/**
* Query the System for Nodes.
*
* @params opState - The node operational state
* @param nodeState - NodeState that we are interested in matching.
* @return Set of Datanodes that match the NodeState.
*/
private Set<DatanodeDetails> queryNodeState(
HddsProtos.NodeOperationalState opState, HddsProtos.NodeState nodeState) {
Set<DatanodeDetails> returnSet = new TreeSet<>();
List<DatanodeDetails> tmp = scm.getScmNodeManager()
.getNodes(opState, nodeState);
if ((tmp != null) && (!tmp.isEmpty())) {
returnSet.addAll(tmp);
}
return returnSet;
}

@Override
public AuditMessage buildAuditMessageForSuccess(
AuditAction op, Map<String, String> auditMap) {
Expand Down
Loading
Loading