From ada39e4065c23016a81f25237a796aaba8432995 Mon Sep 17 00:00:00 2001 From: sravani-revuri Date: Thu, 26 Feb 2026 18:50:39 +0530 Subject: [PATCH 1/4] HDDS-14618. Support including only specified containers in Container Balancer --- .../ContainerBalancerConfiguration.java | 33 +++++++++++++++++++ .../hadoop/hdds/scm/client/ScmClient.java | 3 +- .../StorageContainerLocationProtocol.java | 3 +- ...ocationProtocolClientSideTranslatorPB.java | 8 ++++- .../src/main/proto/ScmAdminProtocol.proto | 1 + .../src/main/proto/hdds.proto | 1 + .../ContainerBalancerSelectionCriteria.java | 14 ++++++++ ...ocationProtocolServerSideTranslatorPB.java | 7 +++- .../scm/server/SCMClientProtocolServer.java | 9 ++++- ...estContainerBalancerDatanodeNodeLimit.java | 17 ++++++++++ .../cli/ContainerBalancerStartSubcommand.java | 9 ++++- .../ContainerBalancerStatusSubcommand.java | 4 +++ .../scm/cli/ContainerOperationClient.java | 5 +-- .../TestContainerBalancerSubCommand.java | 5 +-- .../TestContainerBalancerOperations.java | 15 ++++++--- 15 files changed, 119 insertions(+), 15 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java index df7039cc6202..ec219a1ab837 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java @@ -91,6 +91,11 @@ public final class ContainerBalancerConfiguration { "to exclude from balancing. For example \"1, 4, 5\" or \"1,4,5\".") private String excludeContainers = ""; + @Config(key = "hdds.container.balancer.include.containers", type = ConfigType.STRING, defaultValue = + "", tags = {ConfigTag.BALANCER}, description = "List of container IDs " + + "to include to balancing. For example \"1, 4, 5\" or \"1,4,5\".") + private String includeContainers = ""; + @Config(key = "hdds.container.balancer.move.timeout", type = ConfigType.TIME, defaultValue = "65m", tags = {ConfigTag.BALANCER}, description = "The amount of time to allow a single container to move " + @@ -311,6 +316,17 @@ public Set getExcludeContainers() { }).collect(Collectors.toSet()); } + public Set getIncludeContainers() { + if (includeContainers.isEmpty()) { + return new HashSet<>(); + } + return Arrays.stream(includeContainers.split(",")) + .map(s -> { + s = s.trim(); + return ContainerID.valueOf(Long.parseLong(s)); + }).collect(Collectors.toSet()); + } + /** * Sets containers to exclude from balancing. * @param excludeContainers String of {@link ContainerID} to exclude. For @@ -320,6 +336,16 @@ public void setExcludeContainers(String excludeContainers) { this.excludeContainers = excludeContainers; } + /** + * Sets containers to include in balancing. When non-empty, only these + * containers will be considered for balancing. + * @param includeContainers String of {@link ContainerID} to include. For + * example, "1, 4, 5" or "1,4,5". + */ + public void setIncludeContainers(String includeContainers) { + this.includeContainers = includeContainers; + } + public Duration getMoveTimeout() { return Duration.ofMillis(moveTimeout); } @@ -422,6 +448,7 @@ public String toString() { "%-50s %s%n" + "%-50s %s%n" + "%-50s %s%n" + + "%-50s %s%n" + "%-50s %s%n", "Key", "Value", "Threshold", threshold, "Max Datanodes to Involve per Iteration(percent)", maxDatanodesPercentageToInvolvePerIteration, @@ -443,6 +470,8 @@ public String toString() { networkTopologyEnable, "Whether to Trigger Refresh Datanode Usage Info", triggerDuEnable, + "Container IDs to Include to Balancing", + includeContainers.equals("") ? "None" : includeContainers, "Container IDs to Exclude from Balancing", excludeContainers.equals("") ? "None" : excludeContainers, "Datanodes Specified to be Balanced", @@ -463,6 +492,7 @@ public ContainerBalancerConfigurationProto.Builder toProtobufBuilder() { .setSizeLeavingSourceMax(maxSizeLeavingSource) .setIterations(iterations) .setExcludeContainers(excludeContainers) + .setIncludeContainers(includeContainers) .setMoveTimeout(moveTimeout) .setBalancingIterationInterval(balancingInterval) .setIncludeDatanodes(includeNodes) @@ -497,6 +527,9 @@ static ContainerBalancerConfiguration fromProtobuf( if (proto.hasIterations()) { config.setIterations(proto.getIterations()); } + if (proto.hasIncludeContainers()) { + config.setIncludeContainers(proto.getIncludeContainers()); + } if (proto.hasExcludeContainers()) { config.setExcludeContainers(proto.getExcludeContainers()); } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java index 5ef0515b92eb..fac1467dd8f2 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java @@ -376,7 +376,8 @@ StartContainerBalancerResponseProto startContainerBalancer( Optional networkTopologyEnable, Optional includeNodes, Optional excludeNodes, - Optional excludeContainers) throws IOException; + Optional excludeContainers, + Optional includeContainers) throws IOException; /** * Stop ContainerBalancer. diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java index f0fb3378c6f8..87392fbe5cb2 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java @@ -434,7 +434,8 @@ StartContainerBalancerResponseProto startContainerBalancer( Optional networkTopologyEnable, Optional includeNodes, Optional excludeNodes, - Optional excludeContainers) throws IOException; + Optional excludeContainers, + Optional includeContainers) throws IOException; /** * Stop ContainerBalancer. diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java index 678b40ec788d..890d381010d9 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java @@ -930,7 +930,8 @@ public StartContainerBalancerResponseProto startContainerBalancer( Optional networkTopologyEnable, Optional includeNodes, Optional excludeNodes, - Optional excludeContainers) throws IOException { + Optional excludeContainers, + Optional includeContainers) throws IOException { StartContainerBalancerRequestProto.Builder builder = StartContainerBalancerRequestProto.newBuilder(); builder.setTraceID(TracingUtil.exportCurrentSpan()); @@ -1021,6 +1022,11 @@ public StartContainerBalancerResponseProto startContainerBalancer( builder.setExcludeContainers(ec); } + if (includeContainers.isPresent()) { + String ic = includeContainers.get(); + builder.setIncludeContainers(ic); + } + StartContainerBalancerRequestProto request = builder.build(); return submitRequest(Type.StartContainerBalancer, builder1 -> builder1.setStartContainerBalancerRequest(request)) diff --git a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto index d1aa4798bb66..455f048d8b5a 100644 --- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto +++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto @@ -604,6 +604,7 @@ message StartContainerBalancerRequestProto { optional string includeNodes = 14; optional string excludeNodes = 15; optional string excludeContainers = 16; + optional string includeContainers = 17; } message StartContainerBalancerResponseProto { diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto b/hadoop-hdds/interface-client/src/main/proto/hdds.proto index eb819b80a3e8..2f20fde3e093 100644 --- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto +++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto @@ -503,6 +503,7 @@ message ContainerBalancerConfigurationProto { required bool shouldRun = 18; optional int32 nextIterationIndex = 19; optional int64 moveReplicationTimeout = 20; + optional string includeContainers = 21; } message TransferLeadershipRequestProto { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerSelectionCriteria.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerSelectionCriteria.java index 2c2395af6ba1..98b1430c0dc2 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerSelectionCriteria.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerSelectionCriteria.java @@ -53,6 +53,7 @@ public class ContainerBalancerSelectionCriteria { private ContainerManager containerManager; private Map containerToSourceMap; private Set excludeContainers; + private Set includeContainers; private Set excludeContainersDueToFailure; private FindSourceStrategy findSourceStrategy; private Map> setMap; @@ -71,6 +72,7 @@ public ContainerBalancerSelectionCriteria( this.containerToSourceMap = containerToSourceMap; excludeContainersDueToFailure = new HashSet<>(); excludeContainers = balancerConfiguration.getExcludeContainers(); + includeContainers = balancerConfiguration.getIncludeContainers(); this.findSourceStrategy = findSourceStrategy; this.setMap = new HashMap<>(); } @@ -156,6 +158,10 @@ private Comparator orderContainersByUsedBytes() { public boolean shouldBeExcluded(ContainerID containerID, DatanodeDetails node, long sizeMovedAlready) { ContainerInfo container; + //if any container is excluded only those should be used + if (!includeContainers.isEmpty() && !includeContainers.contains(containerID)) { + return true; // Exclude if not in include list + } try { container = containerManager.getContainer(containerID); } catch (ContainerNotFoundException e) { @@ -227,6 +233,11 @@ public void setExcludeContainers( this.excludeContainers = excludeContainers; } + public void setIncludeContainers( + Set includeContainers) { + this.includeContainers = includeContainers; + } + public void addToExcludeDueToFailContainers(ContainerID container) { this.excludeContainersDueToFailure.add(container); } @@ -236,6 +247,9 @@ private NavigableSet getCandidateContainers(DatanodeDetails node) { new TreeSet<>(orderContainersByUsedBytes().reversed()); try { Set idSet = nodeManager.getContainers(node); + if (includeContainers != null && !includeContainers.isEmpty()) { + idSet.retainAll(includeContainers); + } if (excludeContainers != null) { idSet.removeAll(excludeContainers); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java index 6693f7be0259..a2dc06fabf4e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java @@ -1142,6 +1142,7 @@ public StartContainerBalancerResponseProto startContainerBalancer( Optional includeNodes = Optional.empty(); Optional excludeNodes = Optional.empty(); Optional excludeContainers = Optional.empty(); + Optional includeContainers = Optional.empty(); if (request.hasThreshold()) { threshold = Optional.of(request.getThreshold()); @@ -1206,12 +1207,16 @@ public StartContainerBalancerResponseProto startContainerBalancer( excludeContainers = Optional.of(request.getExcludeContainers()); } + if (request.hasIncludeContainers()) { + includeContainers = Optional.of(request.getIncludeContainers()); + } + return impl.startContainerBalancer(threshold, iterations, maxDatanodesPercentageToInvolvePerIteration, maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB, balancingInterval, moveTimeout, moveReplicationTimeout, networkTopologyEnable, includeNodes, - excludeNodes, excludeContainers); + excludeNodes, excludeContainers, includeContainers); } public StopContainerBalancerResponseProto stopContainerBalancer( diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index 80fbb062e907..f500dc82330c 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -1161,7 +1161,8 @@ public StartContainerBalancerResponseProto startContainerBalancer( Optional networkTopologyEnable, Optional includeNodes, Optional excludeNodes, - Optional excludeContainers) throws IOException { + Optional excludeContainers, + Optional includeContainers) throws IOException { Map auditMap = Maps.newHashMap(); try { getScm().checkAdminAccess(getRemoteUser(), false); @@ -1277,6 +1278,12 @@ public StartContainerBalancerResponseProto startContainerBalancer( cbc.setExcludeContainers(ec); } + if (includeContainers.isPresent()) { + String ic = includeContainers.get(); + auditMap.put("includeContainers", (ic)); + cbc.setIncludeContainers(ic); + } + ContainerBalancer containerBalancer = scm.getContainerBalancer(); containerBalancer.startBalancer(cbc); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerDatanodeNodeLimit.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerDatanodeNodeLimit.java index d5ecbb34e62c..3bf6f28c587f 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerDatanodeNodeLimit.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerDatanodeNodeLimit.java @@ -459,6 +459,23 @@ public void balancerShouldNotSelectConfiguredExcludeContainers(@Nonnull MockedSC } } + @ParameterizedTest(name = "MockedSCM #{index}: {0}") + @MethodSource("createMockedSCMs") + public void balancerShouldOnlySelectConfiguredIncludeContainers(@Nonnull MockedSCM mockedSCM) { + ContainerBalancerConfiguration config = new ContainerBalancerConfigBuilder(mockedSCM.getNodeCount()).build(); + config.setIncludeContainers("1, 4, 5"); + + ContainerBalancerTask task = mockedSCM.startBalancerTask(config); + + Set includeContainers = config.getIncludeContainers(); + assertThat(task.getContainerToSourceMap()).isNotEmpty(); + for (ContainerID container : task.getContainerToSourceMap().keySet()) { + assertThat(includeContainers) + .as("Container %s should be in the include list", container) + .contains(container); + } + } + @ParameterizedTest(name = "MockedSCM #{index}: {0}") @MethodSource("createMockedSCMs") public void checkIterationResult(@Nonnull MockedSCM mockedSCM) { diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java index 2a9925c3c26f..6311c8bf5a26 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java @@ -126,6 +126,13 @@ public class ContainerBalancerStartSubcommand extends ScmSubcommand { "(specify \"1,2,3\" for container IDs).") private Optional excludeContainers; + @Option(names = {"--include-containers"}, + description = "A list of container IDs separated by commas. " + + "The containers specified in this list are included in balancing. " + + "This configuration is empty by default " + + "(specify \"1,2,3\" for container IDs).") + private Optional includeContainers; + @Override public void execute(ScmClient scmClient) throws IOException { StartContainerBalancerResponseProto response = scmClient. @@ -134,7 +141,7 @@ public void execute(ScmClient scmClient) throws IOException { maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB, balancingInterval, moveTimeout, moveReplicationTimeout, networkTopologyEnable, includeNodes, - excludeNodes, excludeContainers); + excludeNodes, excludeContainers, includeContainers); if (response.getStart()) { System.out.println("Container Balancer started successfully."); } else { diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java index c18491c2da51..ff6834a6cb56 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java @@ -122,6 +122,7 @@ String getConfigurationPrettyString(HddsProtos.ContainerBalancerConfigurationPro "%-50s %s%n" + "%-50s %s%n" + "%-50s %s%n" + + "%-50s %s%n" + "%-50s %s%n", "Key", "Value", "Threshold", configuration.getUtilizationThreshold(), "Max Datanodes to Involve per Iteration(percent)", configuration.getDatanodesInvolvedMaxPercentagePerIteration(), @@ -143,12 +144,15 @@ String getConfigurationPrettyString(HddsProtos.ContainerBalancerConfigurationPro configuration.getMoveNetworkTopologyEnable(), "Whether to Trigger Refresh Datanode Usage Info", configuration.getTriggerDuBeforeMoveEnable(), + "Container IDs to Include to Balancing", + configuration.getIncludeContainers().isEmpty() ? "None" : configuration.getIncludeContainers(), "Container IDs to Exclude from Balancing", configuration.getExcludeContainers().isEmpty() ? "None" : configuration.getExcludeContainers(), "Datanodes Specified to be Balanced", configuration.getIncludeDatanodes().isEmpty() ? "None" : configuration.getIncludeDatanodes(), "Datanodes Excluded from Balancing", configuration.getExcludeDatanodes().isEmpty() ? "None" : configuration.getExcludeDatanodes()); + } private String getPrettyIterationStatusInfo(ContainerBalancerTaskIterationStatusInfoProto iterationStatusInfo) { diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java index 3ca49be24438..6bec77a8b7e3 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java @@ -511,13 +511,14 @@ public StartContainerBalancerResponseProto startContainerBalancer( Optional networkTopologyEnable, Optional includeNodes, Optional excludeNodes, - Optional excludeContainers) throws IOException { + Optional excludeContainers, + Optional includeContainers) throws IOException { return storageContainerLocationClient.startContainerBalancer(threshold, iterations, maxDatanodesPercentageToInvolvePerIteration, maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB, balancingInterval, moveTimeout, moveReplicationTimeout, networkTopologyEnable, includeNodes, - excludeNodes, excludeContainers); + excludeNodes, excludeContainers, includeContainers); } @Override diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java index d71b8d00be2d..eec312ebc963 100644 --- a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java +++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java @@ -80,6 +80,7 @@ class TestContainerBalancerSubCommand { "Interval between each Iteration 0min\n" + "Whether to Enable Network Topology false\n" + "Whether to Trigger Refresh Datanode Usage Info false\n" + + "Container IDs to Include to Balancing None\n" + "Container IDs to Exclude from Balancing None\n" + "Datanodes Specified to be Balanced None\n" + "Datanodes Excluded from Balancing None"; @@ -449,7 +450,7 @@ public void testContainerBalancerStartSubcommandWhenBalancerIsNotRunning() throws IOException { ScmClient scmClient = mock(ScmClient.class); when(scmClient.startContainerBalancer( - null, null, null, null, null, null, null, null, null, null, null, null, null)) + null, null, null, null, null, null, null, null, null, null, null, null, null, null)) .thenReturn( StorageContainerLocationProtocolProtos .StartContainerBalancerResponseProto.newBuilder() @@ -465,7 +466,7 @@ public void testContainerBalancerStartSubcommandWhenBalancerIsRunning() throws IOException { ScmClient scmClient = mock(ScmClient.class); when(scmClient.startContainerBalancer( - null, null, null, null, null, null, null, null, null, null, null, null, null)) + null, null, null, null, null, null, null, null, null, null, null, null, null, null)) .thenReturn(StorageContainerLocationProtocolProtos .StartContainerBalancerResponseProto.newBuilder() .setStart(false) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java index 34b990278ab2..bb8d07045383 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java @@ -94,12 +94,13 @@ public void testContainerBalancerCLIOperations() throws Exception { Optional includeNodes = Optional.of(""); Optional excludeNodes = Optional.of(""); Optional excludeContainers = Optional.of(""); + Optional includeContainers = Optional.of(""); containerBalancerClient.startContainerBalancer(threshold, iterations, maxDatanodesPercentageToInvolvePerIteration, maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB, balancingInterval, moveTimeout, moveReplicationTimeout, networkTopologyEnable, includeNodes, - excludeNodes, excludeContainers); + excludeNodes, excludeContainers, includeContainers); running = containerBalancerClient.getContainerBalancerStatus(); assertTrue(running); @@ -121,7 +122,7 @@ public void testContainerBalancerCLIOperations() throws Exception { maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB, balancingInterval, moveTimeout, moveReplicationTimeout, networkTopologyEnable, includeNodes, - excludeNodes, excludeContainers); + excludeNodes, excludeContainers, includeContainers); running = containerBalancerClient.getContainerBalancerStatus(); assertTrue(running); @@ -149,6 +150,7 @@ public void testIfCBCLIOverridesConfigs() throws Exception { Optional iterations = Optional.empty(); Optional balancingInterval = Optional.empty(); String excludedContainersList = "1,2,3"; + String includedContainersList = "4,5"; //CLI options are passed Optional threshold = Optional.of(0.1); @@ -163,12 +165,13 @@ public void testIfCBCLIOverridesConfigs() throws Exception { Optional includeNodes = Optional.of(""); Optional excludeNodes = Optional.of(""); Optional excludeContainers = Optional.of(excludedContainersList); + Optional includeContainers = Optional.of(includedContainersList); containerBalancerClient.startContainerBalancer(threshold, iterations, maxDatanodesPercentageToInvolvePerIteration, maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB, balancingInterval, moveTimeout, moveReplicationTimeout, networkTopologyEnable, includeNodes, - excludeNodes, excludeContainers); + excludeNodes, excludeContainers, includeContainers); running = containerBalancerClient.getContainerBalancerStatus(); assertTrue(running); @@ -186,8 +189,9 @@ public void testIfCBCLIOverridesConfigs() throws Exception { //then it takes the CLI option. assertEquals(100, config.getMaxDatanodesPercentageToInvolvePerIteration()); - //Verifies that the 'excludeContainers' passed via CLI overrides the default empty set + //Verifies that excluded and included containers passed via CLI overrides the default empty set assertEquals(parseContainerIDs(excludedContainersList), config.getExcludeContainers()); + assertEquals(parseContainerIDs(includedContainersList), config.getIncludeContainers()); containerBalancerClient.stopContainerBalancer(); running = containerBalancerClient.getContainerBalancerStatus(); @@ -218,12 +222,13 @@ public void testStopBalancerIdempotent() throws IOException { Optional includeNodes = Optional.of(""); Optional excludeNodes = Optional.of(""); Optional excludeContainers = Optional.of(""); + Optional includeContainers = Optional.of(""); containerBalancerClient.startContainerBalancer(threshold, iterations, maxDatanodesPercentageToInvolvePerIteration, maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB, balancingInterval, moveTimeout, moveReplicationTimeout, networkTopologyEnable, includeNodes, - excludeNodes, excludeContainers); + excludeNodes, excludeContainers, includeContainers); running = containerBalancerClient.getContainerBalancerStatus(); assertTrue(running); From 0b47b6c2612dfb3fe52771a86797172359895b92 Mon Sep 17 00:00:00 2001 From: sravani-revuri Date: Fri, 27 Feb 2026 10:21:03 +0530 Subject: [PATCH 2/4] fixed comment line and display messages --- .../container/balancer/ContainerBalancerConfiguration.java | 4 ++-- .../balancer/ContainerBalancerSelectionCriteria.java | 4 ++-- .../hdds/scm/cli/ContainerBalancerStatusSubcommand.java | 2 +- .../scm/cli/datanode/TestContainerBalancerSubCommand.java | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java index ec219a1ab837..ff5b1d00f756 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java @@ -93,7 +93,7 @@ public final class ContainerBalancerConfiguration { @Config(key = "hdds.container.balancer.include.containers", type = ConfigType.STRING, defaultValue = "", tags = {ConfigTag.BALANCER}, description = "List of container IDs " + - "to include to balancing. For example \"1, 4, 5\" or \"1,4,5\".") + "to include in balancing. For example \"1, 4, 5\" or \"1,4,5\".") private String includeContainers = ""; @Config(key = "hdds.container.balancer.move.timeout", type = ConfigType.TIME, defaultValue = "65m", @@ -470,7 +470,7 @@ public String toString() { networkTopologyEnable, "Whether to Trigger Refresh Datanode Usage Info", triggerDuEnable, - "Container IDs to Include to Balancing", + "Container IDs to Include in Balancing", includeContainers.equals("") ? "None" : includeContainers, "Container IDs to Exclude from Balancing", excludeContainers.equals("") ? "None" : excludeContainers, diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerSelectionCriteria.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerSelectionCriteria.java index ea5c6fcfb9ab..63fd08cca907 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerSelectionCriteria.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerSelectionCriteria.java @@ -160,9 +160,9 @@ private Comparator orderContainersByUsedBytes() { public boolean shouldBeExcluded(ContainerID containerID, DatanodeDetails node, long sizeMovedAlready) { ContainerInfo container; - //if any container is excluded only those should be used + //If includeContainers is specified, exclude containers not in the include list if (!includeContainers.isEmpty() && !includeContainers.contains(containerID)) { - return true; // Exclude if not in include list + return true; } try { container = containerManager.getContainer(containerID); diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java index ff6834a6cb56..6f2e9df3b02f 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java @@ -144,7 +144,7 @@ String getConfigurationPrettyString(HddsProtos.ContainerBalancerConfigurationPro configuration.getMoveNetworkTopologyEnable(), "Whether to Trigger Refresh Datanode Usage Info", configuration.getTriggerDuBeforeMoveEnable(), - "Container IDs to Include to Balancing", + "Container IDs to Include in Balancing", configuration.getIncludeContainers().isEmpty() ? "None" : configuration.getIncludeContainers(), "Container IDs to Exclude from Balancing", configuration.getExcludeContainers().isEmpty() ? "None" : configuration.getExcludeContainers(), diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java index eec312ebc963..c3afd9b844cd 100644 --- a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java +++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java @@ -80,7 +80,7 @@ class TestContainerBalancerSubCommand { "Interval between each Iteration 0min\n" + "Whether to Enable Network Topology false\n" + "Whether to Trigger Refresh Datanode Usage Info false\n" + - "Container IDs to Include to Balancing None\n" + + "Container IDs to Include in Balancing None\n" + "Container IDs to Exclude from Balancing None\n" + "Datanodes Specified to be Balanced None\n" + "Datanodes Excluded from Balancing None"; From 4be20389fc32e74aa62d8969fc924d7aebda1ed9 Mon Sep 17 00:00:00 2001 From: sravani-revuri Date: Tue, 3 Mar 2026 12:34:43 +0530 Subject: [PATCH 3/4] corrected according to comments --- .../container/balancer/ContainerBalancerConfiguration.java | 5 +++-- .../balancer/ContainerBalancerSelectionCriteria.java | 5 ----- .../hdds/scm/cli/ContainerBalancerStartSubcommand.java | 6 +++--- .../hdds/scm/cli/ContainerBalancerStatusSubcommand.java | 1 - 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java index ff5b1d00f756..8ff3b6e5e204 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java @@ -93,7 +93,8 @@ public final class ContainerBalancerConfiguration { @Config(key = "hdds.container.balancer.include.containers", type = ConfigType.STRING, defaultValue = "", tags = {ConfigTag.BALANCER}, description = "List of container IDs " + - "to include in balancing. For example \"1, 4, 5\" or \"1,4,5\".") + "to include in balancing. Only these containers will be included in balancing. " + + "For example \"1, 4, 5\" or \"1,4,5\".") private String includeContainers = ""; @Config(key = "hdds.container.balancer.move.timeout", type = ConfigType.TIME, defaultValue = "65m", @@ -471,7 +472,7 @@ public String toString() { "Whether to Trigger Refresh Datanode Usage Info", triggerDuEnable, "Container IDs to Include in Balancing", - includeContainers.equals("") ? "None" : includeContainers, + includeContainers.isEmpty() ? "None" : includeContainers, "Container IDs to Exclude from Balancing", excludeContainers.equals("") ? "None" : excludeContainers, "Datanodes Specified to be Balanced", diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerSelectionCriteria.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerSelectionCriteria.java index 63fd08cca907..52f34347c9a9 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerSelectionCriteria.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerSelectionCriteria.java @@ -260,11 +260,6 @@ public void setExcludeContainers( this.excludeContainers = excludeContainers; } - public void setIncludeContainers( - Set includeContainers) { - this.includeContainers = includeContainers; - } - public void addToExcludeDueToFailContainers(ContainerID container) { this.excludeContainersDueToFailure.add(container); } diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java index 6311c8bf5a26..0c29b0dc04f5 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java @@ -127,9 +127,9 @@ public class ContainerBalancerStartSubcommand extends ScmSubcommand { private Optional excludeContainers; @Option(names = {"--include-containers"}, - description = "A list of container IDs separated by commas. " + - "The containers specified in this list are included in balancing. " + - "This configuration is empty by default " + + description = "Only the containers specified in this list will be included in balancing." + + " If --exclude-containers is also specified, those containers will " + + "be excluded. This configuration is empty by default " + "(specify \"1,2,3\" for container IDs).") private Optional includeContainers; diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java index 6f2e9df3b02f..82e610c5e934 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java @@ -152,7 +152,6 @@ String getConfigurationPrettyString(HddsProtos.ContainerBalancerConfigurationPro configuration.getIncludeDatanodes().isEmpty() ? "None" : configuration.getIncludeDatanodes(), "Datanodes Excluded from Balancing", configuration.getExcludeDatanodes().isEmpty() ? "None" : configuration.getExcludeDatanodes()); - } private String getPrettyIterationStatusInfo(ContainerBalancerTaskIterationStatusInfoProto iterationStatusInfo) { From 87cc7dab61cf70c47c3a351ca0cbf2924851520e Mon Sep 17 00:00:00 2001 From: sravani-revuri Date: Wed, 4 Mar 2026 09:08:27 +0530 Subject: [PATCH 4/4] fixed option description --- .../hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java index 0c29b0dc04f5..cd34522d6a61 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java @@ -127,7 +127,8 @@ public class ContainerBalancerStartSubcommand extends ScmSubcommand { private Optional excludeContainers; @Option(names = {"--include-containers"}, - description = "Only the containers specified in this list will be included in balancing." + + description = "A list of container IDs separated by commas. " + + "Only the containers specified in this list will be included in balancing." + " If --exclude-containers is also specified, those containers will " + "be excluded. This configuration is empty by default " + "(specify \"1,2,3\" for container IDs).")