From ce5bd86790e407ac0fa32b2076d5c34b7e36ce1b Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Wed, 4 Mar 2026 11:28:04 +0800 Subject: [PATCH 1/7] HDDS-14764. Allow Datanode to dynamically reconfigure SCM node addresses --- .../hadoop/ozone/HddsDatanodeService.java | 13 ++++- .../conf/DatanodeReconfigurationHandler.java | 51 +++++++++++++++++++ .../TestDatanodeSCMNodesReconfiguration.java | 8 ++- 3 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeReconfigurationHandler.java diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java index b0fbfe30b93f..40ad74c6fc3b 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java @@ -62,6 +62,7 @@ import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.conf.DatanodeReconfigurationHandler; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.ReconfigurationHandler; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -299,8 +300,10 @@ public String getNamespace() { } } + scmServiceId = HddsUtils.getScmServiceId(conf); + reconfigurationHandler = - new ReconfigurationHandler("DN", conf, this::checkAdminPrivilege) + new DatanodeReconfigurationHandler("DN", scmServiceId, conf, this::checkAdminPrivilege) .register(HDDS_DATANODE_BLOCK_DELETE_THREAD_MAX, this::reconfigBlockDeleteThreadMax) .register(OZONE_BLOCK_DELETING_SERVICE_WORKERS, @@ -312,7 +315,6 @@ public String getNamespace() { .register(REPLICATION_STREAMS_LIMIT_KEY, this::reconfigReplicationStreamsLimit); - scmServiceId = HddsUtils.getScmServiceId(conf); if (scmServiceId != null) { reconfigurationHandler.register(OZONE_SCM_NODES_KEY + "." + scmServiceId, this::reconfigScmNodes); @@ -575,6 +577,13 @@ public void stop() { } } } + if (reconfigurationHandler != null) { + try { + reconfigurationHandler.close(); + } catch (IOException e) { + LOG.error("DatanodeReconfigurationHandler stop failed", e); + } + } if (datanodeStateMachine != null) { datanodeStateMachine.stopDaemon(); } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeReconfigurationHandler.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeReconfigurationHandler.java new file mode 100644 index 000000000000..7909ee2260b3 --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeReconfigurationHandler.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.conf; + +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_ADDRESS_KEY; + +import java.io.IOException; +import org.apache.hadoop.ozone.ha.ConfUtils; +import org.apache.ratis.util.function.CheckedConsumer; + +/** + * Reconfiguration handler for Datanode. Allows reconfiguration of some properties + * even if the properties are not registered in advance. + */ +public class DatanodeReconfigurationHandler extends ReconfigurationHandler { + + private final String scmServiceId; + + public DatanodeReconfigurationHandler(String name, String scmServiceId, OzoneConfiguration config, + CheckedConsumer requireAdminPrivilege) { + super(name, config, requireAdminPrivilege); + this.scmServiceId = scmServiceId; + } + + @Override + public boolean isPropertyReconfigurable(String property) { + if (scmServiceId != null && + property.startsWith(ConfUtils.addKeySuffixes(OZONE_SCM_ADDRESS_KEY, scmServiceId))) { + // Allow reconfiguration for "ozone.scm.address.." + // even if the property has not been registered. This is to allow + // SCM migration without datanode restarts. + return true; + } + return getReconfigurableProperties().contains(property); + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestDatanodeSCMNodesReconfiguration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestDatanodeSCMNodesReconfiguration.java index d2c90c21d658..6f38d89fb2b6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestDatanodeSCMNodesReconfiguration.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestDatanodeSCMNodesReconfiguration.java @@ -128,7 +128,9 @@ void testSCMMigration() throws Exception { for (StorageContainerManager scm : cluster.getStorageContainerManagers()) { String scmAddrKey = ConfUtils.addKeySuffixes( ScmConfigKeys.OZONE_SCM_ADDRESS_KEY, scmServiceId, scm.getSCMNodeId()); - datanode.getConf().set(scmAddrKey, cluster.getConf().get(scmAddrKey)); + assertTrue(datanode.getReconfigurationHandler().isPropertyReconfigurable(scmAddrKey)); + datanode.getReconfigurationHandler().reconfigureProperty(scmAddrKey, + cluster.getConf().get(scmAddrKey)); String dnPortKey = ConfUtils.addKeySuffixes( ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, scmServiceId, scm.getSCMNodeId()); datanode.getConf().set(dnPortKey, cluster.getConf().get(dnPortKey)); @@ -273,7 +275,9 @@ void testAddAndRemoveOneSCM() throws Exception { for (StorageContainerManager scm : cluster.getStorageContainerManagers()) { String scmAddrKey = ConfUtils.addKeySuffixes( ScmConfigKeys.OZONE_SCM_ADDRESS_KEY, scmServiceId, scm.getSCMNodeId()); - datanode.getConf().set(scmAddrKey, cluster.getConf().get(scmAddrKey)); + assertTrue(datanode.getReconfigurationHandler().isPropertyReconfigurable(scmAddrKey)); + datanode.getReconfigurationHandler().reconfigureProperty(scmAddrKey, + cluster.getConf().get(scmAddrKey)); String dnPortKey = ConfUtils.addKeySuffixes( ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, scmServiceId, scm.getSCMNodeId()); datanode.getConf().set(dnPortKey, cluster.getConf().get(dnPortKey)); From cf87f58c75030d71d7c5e9251a4b84717af14939 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Wed, 4 Mar 2026 16:03:02 +0800 Subject: [PATCH 2/7] Generalize to prefixProperties --- .../org/apache/hadoop/ozone/ha/ConfUtils.java | 14 +++++++- .../hadoop/ozone/HddsDatanodeService.java | 5 ++- .../conf/DatanodeReconfigurationHandler.java | 35 ++++++++++++------- 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ha/ConfUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ha/ConfUtils.java index 65d6e35c7ac7..38a9f86e2522 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ha/ConfUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ha/ConfUtils.java @@ -48,8 +48,20 @@ public static String addSuffix(String key, String suffix) { * Return configuration key of format key.suffix1.suffix2...suffixN. */ public static String addKeySuffixes(String key, String... suffixes) { + return addKeySuffixes(key, false, suffixes); + } + + /** + * Return configuration key of format key.suffix1.suffix2...suffixN with the option to + * add a trailing separator. + */ + public static String addKeySuffixes(String key, boolean withTrailingSeparator, String... suffixes) { String keySuffix = concatSuffixes(suffixes); - return addSuffix(key, keySuffix); + String suffix = addSuffix(key, keySuffix); + if (withTrailingSeparator) { + suffix = suffix.concat("."); + } + return suffix; } /** diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java index 40ad74c6fc3b..5e1b2355f0c6 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java @@ -19,6 +19,7 @@ import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name.HTTP; import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name.HTTPS; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_ADDRESS_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NODES_KEY; import static org.apache.hadoop.hdds.utils.HddsServerUtil.getRemoteUser; import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmSecurityClientWithMaxRetry; @@ -94,6 +95,7 @@ import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet; import org.apache.hadoop.ozone.container.common.volume.StorageVolume; import org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerProtocolServer; +import org.apache.hadoop.ozone.ha.ConfUtils; import org.apache.hadoop.ozone.util.OzoneNetUtils; import org.apache.hadoop.ozone.util.ShutdownHookManager; import org.apache.hadoop.security.SecurityUtil; @@ -303,7 +305,8 @@ public String getNamespace() { scmServiceId = HddsUtils.getScmServiceId(conf); reconfigurationHandler = - new DatanodeReconfigurationHandler("DN", scmServiceId, conf, this::checkAdminPrivilege) + new DatanodeReconfigurationHandler("DN", conf, this::checkAdminPrivilege) + .registerPrefix(ConfUtils.addKeySuffixes(OZONE_SCM_ADDRESS_KEY, true, scmServiceId)) .register(HDDS_DATANODE_BLOCK_DELETE_THREAD_MAX, this::reconfigBlockDeleteThreadMax) .register(OZONE_BLOCK_DELETING_SERVICE_WORKERS, diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeReconfigurationHandler.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeReconfigurationHandler.java index 7909ee2260b3..2a3ea3266473 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeReconfigurationHandler.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeReconfigurationHandler.java @@ -17,10 +17,12 @@ package org.apache.hadoop.hdds.conf; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_ADDRESS_KEY; - import java.io.IOException; -import org.apache.hadoop.ozone.ha.ConfUtils; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentSkipListSet; import org.apache.ratis.util.function.CheckedConsumer; /** @@ -29,23 +31,32 @@ */ public class DatanodeReconfigurationHandler extends ReconfigurationHandler { - private final String scmServiceId; + private final Set prefixProperties = new ConcurrentSkipListSet<>(); - public DatanodeReconfigurationHandler(String name, String scmServiceId, OzoneConfiguration config, + public DatanodeReconfigurationHandler(String name, OzoneConfiguration config, CheckedConsumer requireAdminPrivilege) { super(name, config, requireAdminPrivilege); - this.scmServiceId = scmServiceId; + } + + public DatanodeReconfigurationHandler registerPrefix(String prefixProperty) { + prefixProperties.add(prefixProperty); + return this; } @Override public boolean isPropertyReconfigurable(String property) { - if (scmServiceId != null && - property.startsWith(ConfUtils.addKeySuffixes(OZONE_SCM_ADDRESS_KEY, scmServiceId))) { - // Allow reconfiguration for "ozone.scm.address.." - // even if the property has not been registered. This is to allow - // SCM migration without datanode restarts. - return true; + for (String prefixProperty : prefixProperties) { + if (property.startsWith(prefixProperty)) { + return true; + } } return getReconfigurableProperties().contains(property); } + + @Override + public List listReconfigureProperties() throws IOException { + Set reconfigureSet = new TreeSet<>(super.listReconfigureProperties()); + reconfigureSet.addAll(prefixProperties); + return new ArrayList<>(reconfigureSet); + } } From 2d36b7c02c9dd6fee223e8ae97cee5502c036558 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Wed, 4 Mar 2026 18:11:32 +0800 Subject: [PATCH 3/7] DataReconfiguration overrides getReconfigurableProperties --- .../hdds/conf/DatanodeReconfigurationHandler.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeReconfigurationHandler.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeReconfigurationHandler.java index 2a3ea3266473..b1d15bac4623 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeReconfigurationHandler.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeReconfigurationHandler.java @@ -17,8 +17,11 @@ package org.apache.hadoop.hdds.conf; +import static java.util.Collections.unmodifiableSet; + import java.io.IOException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.TreeSet; @@ -43,6 +46,13 @@ public DatanodeReconfigurationHandler registerPrefix(String prefixProperty) { return this; } + @Override + public Set getReconfigurableProperties() { + Set properties = new HashSet<>(prefixProperties); + properties.addAll(super.getReconfigurableProperties()); + return unmodifiableSet(properties); + } + @Override public boolean isPropertyReconfigurable(String property) { for (String prefixProperty : prefixProperties) { From 154f7bd2347a8ecf9b4515abffe0e11381f06d8a Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Wed, 4 Mar 2026 18:12:01 +0800 Subject: [PATCH 4/7] Only allow reconfiguration if scmServiceId is not null --- .../java/org/apache/hadoop/ozone/HddsDatanodeService.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java index 5e1b2355f0c6..e09b3ff3f68a 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java @@ -302,11 +302,8 @@ public String getNamespace() { } } - scmServiceId = HddsUtils.getScmServiceId(conf); - reconfigurationHandler = new DatanodeReconfigurationHandler("DN", conf, this::checkAdminPrivilege) - .registerPrefix(ConfUtils.addKeySuffixes(OZONE_SCM_ADDRESS_KEY, true, scmServiceId)) .register(HDDS_DATANODE_BLOCK_DELETE_THREAD_MAX, this::reconfigBlockDeleteThreadMax) .register(OZONE_BLOCK_DELETING_SERVICE_WORKERS, @@ -318,7 +315,11 @@ public String getNamespace() { .register(REPLICATION_STREAMS_LIMIT_KEY, this::reconfigReplicationStreamsLimit); + scmServiceId = HddsUtils.getScmServiceId(conf); + if (scmServiceId != null) { + ((DatanodeReconfigurationHandler) reconfigurationHandler) + .registerPrefix(ConfUtils.addKeySuffixes(OZONE_SCM_ADDRESS_KEY, true, scmServiceId)); reconfigurationHandler.register(OZONE_SCM_NODES_KEY + "." + scmServiceId, this::reconfigScmNodes); } From 62c21bc25094e372c7019bedabce820bf30b668c Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Wed, 4 Mar 2026 18:12:28 +0800 Subject: [PATCH 5/7] Defer closing reconfigurationhandler for future patch --- .../java/org/apache/hadoop/ozone/HddsDatanodeService.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java index e09b3ff3f68a..7d88b3a5c0bd 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java @@ -581,13 +581,6 @@ public void stop() { } } } - if (reconfigurationHandler != null) { - try { - reconfigurationHandler.close(); - } catch (IOException e) { - LOG.error("DatanodeReconfigurationHandler stop failed", e); - } - } if (datanodeStateMachine != null) { datanodeStateMachine.stopDaemon(); } From 2f0704b3f8747eb416b3fd0327e500aaf3372eff Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Wed, 4 Mar 2026 18:19:37 +0800 Subject: [PATCH 6/7] Add trailing separator in registerPrefix --- .../java/org/apache/hadoop/ozone/ha/ConfUtils.java | 14 +------------- .../apache/hadoop/ozone/HddsDatanodeService.java | 5 ++--- .../hdds/conf/DatanodeReconfigurationHandler.java | 4 +++- 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ha/ConfUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ha/ConfUtils.java index 38a9f86e2522..65d6e35c7ac7 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ha/ConfUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ha/ConfUtils.java @@ -48,20 +48,8 @@ public static String addSuffix(String key, String suffix) { * Return configuration key of format key.suffix1.suffix2...suffixN. */ public static String addKeySuffixes(String key, String... suffixes) { - return addKeySuffixes(key, false, suffixes); - } - - /** - * Return configuration key of format key.suffix1.suffix2...suffixN with the option to - * add a trailing separator. - */ - public static String addKeySuffixes(String key, boolean withTrailingSeparator, String... suffixes) { String keySuffix = concatSuffixes(suffixes); - String suffix = addSuffix(key, keySuffix); - if (withTrailingSeparator) { - suffix = suffix.concat("."); - } - return suffix; + return addSuffix(key, keySuffix); } /** diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java index 7d88b3a5c0bd..4d39af4d2123 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java @@ -319,9 +319,8 @@ public String getNamespace() { if (scmServiceId != null) { ((DatanodeReconfigurationHandler) reconfigurationHandler) - .registerPrefix(ConfUtils.addKeySuffixes(OZONE_SCM_ADDRESS_KEY, true, scmServiceId)); - reconfigurationHandler.register(OZONE_SCM_NODES_KEY + "." + scmServiceId, - this::reconfigScmNodes); + .registerPrefix(ConfUtils.addKeySuffixes(OZONE_SCM_ADDRESS_KEY, scmServiceId)) + .register(OZONE_SCM_NODES_KEY + "." + scmServiceId, this::reconfigScmNodes); } reconfigurationHandler.setReconfigurationCompleteCallback(reconfigurationHandler.defaultLoggingCallback()); diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeReconfigurationHandler.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeReconfigurationHandler.java index b1d15bac4623..fa5446fba1e9 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeReconfigurationHandler.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeReconfigurationHandler.java @@ -42,7 +42,9 @@ public DatanodeReconfigurationHandler(String name, OzoneConfiguration config, } public DatanodeReconfigurationHandler registerPrefix(String prefixProperty) { - prefixProperties.add(prefixProperty); + prefixProperties.add( + prefixProperty.endsWith(".") ? prefixProperty : prefixProperty.concat(".") + ); return this; } From de6176e586ee475ffae69c9957502652a9cdaabd Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Thu, 5 Mar 2026 09:52:14 +0800 Subject: [PATCH 7/7] No need to introduce new class --- .../hadoop/ozone/HddsDatanodeService.java | 5 +- .../conf/DatanodeReconfigurationHandler.java | 74 ------------------- .../hdds/conf/ReconfigurationHandler.java | 28 ++++++- 3 files changed, 28 insertions(+), 79 deletions(-) delete mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeReconfigurationHandler.java diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java index 4d39af4d2123..190c542809ff 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java @@ -63,7 +63,6 @@ import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.conf.ConfigurationSource; -import org.apache.hadoop.hdds.conf.DatanodeReconfigurationHandler; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.ReconfigurationHandler; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -303,7 +302,7 @@ public String getNamespace() { } reconfigurationHandler = - new DatanodeReconfigurationHandler("DN", conf, this::checkAdminPrivilege) + new ReconfigurationHandler("DN", conf, this::checkAdminPrivilege) .register(HDDS_DATANODE_BLOCK_DELETE_THREAD_MAX, this::reconfigBlockDeleteThreadMax) .register(OZONE_BLOCK_DELETING_SERVICE_WORKERS, @@ -318,7 +317,7 @@ public String getNamespace() { scmServiceId = HddsUtils.getScmServiceId(conf); if (scmServiceId != null) { - ((DatanodeReconfigurationHandler) reconfigurationHandler) + reconfigurationHandler .registerPrefix(ConfUtils.addKeySuffixes(OZONE_SCM_ADDRESS_KEY, scmServiceId)) .register(OZONE_SCM_NODES_KEY + "." + scmServiceId, this::reconfigScmNodes); } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeReconfigurationHandler.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeReconfigurationHandler.java deleted file mode 100644 index fa5446fba1e9..000000000000 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeReconfigurationHandler.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdds.conf; - -import static java.util.Collections.unmodifiableSet; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.TreeSet; -import java.util.concurrent.ConcurrentSkipListSet; -import org.apache.ratis.util.function.CheckedConsumer; - -/** - * Reconfiguration handler for Datanode. Allows reconfiguration of some properties - * even if the properties are not registered in advance. - */ -public class DatanodeReconfigurationHandler extends ReconfigurationHandler { - - private final Set prefixProperties = new ConcurrentSkipListSet<>(); - - public DatanodeReconfigurationHandler(String name, OzoneConfiguration config, - CheckedConsumer requireAdminPrivilege) { - super(name, config, requireAdminPrivilege); - } - - public DatanodeReconfigurationHandler registerPrefix(String prefixProperty) { - prefixProperties.add( - prefixProperty.endsWith(".") ? prefixProperty : prefixProperty.concat(".") - ); - return this; - } - - @Override - public Set getReconfigurableProperties() { - Set properties = new HashSet<>(prefixProperties); - properties.addAll(super.getReconfigurableProperties()); - return unmodifiableSet(properties); - } - - @Override - public boolean isPropertyReconfigurable(String property) { - for (String prefixProperty : prefixProperties) { - if (property.startsWith(prefixProperty)) { - return true; - } - } - return getReconfigurableProperties().contains(property); - } - - @Override - public List listReconfigureProperties() throws IOException { - Set reconfigureSet = new TreeSet<>(super.listReconfigureProperties()); - reconfigureSet.addAll(prefixProperties); - return new ArrayList<>(reconfigureSet); - } -} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/ReconfigurationHandler.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/ReconfigurationHandler.java index 979525f7a1a7..32eb67c0797f 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/ReconfigurationHandler.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/ReconfigurationHandler.java @@ -23,11 +23,13 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentSkipListSet; import java.util.function.BiConsumer; import java.util.function.UnaryOperator; import org.apache.hadoop.conf.Configuration; @@ -51,6 +53,7 @@ public class ReconfigurationHandler extends ReconfigurableBase private final CheckedConsumer requireAdminPrivilege; private final Map> properties = new ConcurrentHashMap<>(); + private final Set prefixProperties = new ConcurrentSkipListSet<>(); private final List completeCallbacks = new ArrayList<>(); private BiConsumer reconfigurationStatusListener; @@ -122,6 +125,13 @@ public ReconfigurationHandler register(ReconfigurableConfig config) { return this; } + public ReconfigurationHandler registerPrefix(String prefixProperty) { + prefixProperties.add( + prefixProperty.endsWith(".") ? prefixProperty : prefixProperty.concat(".") + ); + return this; + } + @Override protected Configuration getNewConf() { return new OzoneConfiguration(); @@ -129,7 +139,19 @@ protected Configuration getNewConf() { @Override public Set getReconfigurableProperties() { - return unmodifiableSet(properties.keySet()); + Set reconfigureProperties = new HashSet<>(properties.keySet()); + reconfigureProperties.addAll(prefixProperties); + return unmodifiableSet(reconfigureProperties); + } + + @Override + public boolean isPropertyReconfigurable(String property) { + for (String prefixProperty : prefixProperties) { + if (property.startsWith(prefixProperty)) { + return true; + } + } + return properties.containsKey(property); } @Override @@ -164,7 +186,9 @@ public ReconfigurationTaskStatus getReconfigureStatus() throws IOException { @Override public List listReconfigureProperties() throws IOException { requireAdminPrivilege.accept("listReconfigurableProperties"); - return new ArrayList<>(new TreeSet<>(getReconfigurableProperties())); + Set reconfigureProperties = new TreeSet<>(getReconfigurableProperties()); + reconfigureProperties.addAll(prefixProperties); + return new ArrayList<>(reconfigureProperties); } @Override