From 44c93582f91905ea481ae6ab103313eaf30a1bc5 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Thu, 5 Feb 2026 20:08:45 +0000 Subject: [PATCH 1/4] HDDS-14570. SCM should finalize all versions with one Ratis request --- .../ozone/upgrade/BasicUpgradeFinalizer.java | 7 +++++ .../DefaultUpgradeFinalizationExecutor.java | 4 +-- .../upgrade/FinalizationStateManager.java | 2 +- .../upgrade/FinalizationStateManagerImpl.java | 6 ++-- .../server/upgrade/SCMUpgradeFinalizer.java | 28 ++++++++++++++----- .../hdds/scm/upgrade/TestScmFinalization.java | 2 +- 6 files changed, 35 insertions(+), 14 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/BasicUpgradeFinalizer.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/BasicUpgradeFinalizer.java index 4d75e88cdf81..fda69ff872f0 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/BasicUpgradeFinalizer.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/BasicUpgradeFinalizer.java @@ -246,6 +246,13 @@ private static boolean isFinalized(Status status) { public abstract void finalizeLayoutFeature(LayoutFeature lf, T context) throws UpgradeException; + public void finalizeLayoutFeatures(Iterable features, T context) + throws UpgradeException { + for (LayoutFeature lf : features) { + finalizeLayoutFeature(lf, context); + } + } + protected void finalizeLayoutFeature(LayoutFeature lf, Optional action, Storage storage) throws UpgradeException { diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/DefaultUpgradeFinalizationExecutor.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/DefaultUpgradeFinalizationExecutor.java index dad1ebd0d7e1..fe689ae1517d 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/DefaultUpgradeFinalizationExecutor.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/DefaultUpgradeFinalizationExecutor.java @@ -66,8 +66,6 @@ public void execute(T component, BasicUpgradeFinalizer finalizer) protected void finalizeFeatures(T component, BasicUpgradeFinalizer finalizer, Iterable lfs) throws UpgradeException { - for (LayoutFeature lf: lfs) { - finalizer.finalizeLayoutFeature(lf, component); - } + finalizer.finalizeLayoutFeatures(lfs, component); } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManager.java index 6d9a61eb9b4d..5e2cf59144e1 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManager.java @@ -33,7 +33,7 @@ public interface FinalizationStateManager { void removeFinalizingMark() throws IOException; @Replicate - void finalizeLayoutFeature(Integer layoutVersion) + void finalizeLayoutFeatures(Integer startLayoutVersion, Integer endLayoutVersion) throws IOException; /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java index 4dde4089f0a6..fc58c37d855e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java @@ -99,8 +99,10 @@ public void addFinalizingMark() throws IOException { } @Override - public void finalizeLayoutFeature(Integer layoutVersion) throws IOException { - finalizeLayoutFeatureLocal(layoutVersion); + public void finalizeLayoutFeatures(Integer startLayoutVersion, Integer endLayoutersion) throws IOException { + for (int version = startLayoutVersion; version <= endLayoutersion; version++) { + finalizeLayoutFeatureLocal(version); + } } /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java index 553d1d09b19e..7c3f57bba94a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java @@ -70,15 +70,29 @@ public void preFinalizeUpgrade(SCMUpgradeFinalizationContext context) @Override public void finalizeLayoutFeature(LayoutFeature lf, - SCMUpgradeFinalizationContext context) throws UpgradeException { - // Run upgrade actions, update VERSION file, and update layout version in - // DB. + SCMUpgradeFinalizationContext context) throws UpgradeException { + throw new UnsupportedOperationException("FinalizeLayoutFeature is not supported in this implemementation. " + + "Please use finalizeLayoutFeatures instead."); + } + + @Override + public void finalizeLayoutFeatures(Iterable features, SCMUpgradeFinalizationContext context) + throws UpgradeException { + int firstLv = -1; + int lastLv = -1; + for (LayoutFeature lf : features) { + if (firstLv == -1) { + firstLv = lf.layoutVersion(); + } + lastLv = lf.layoutVersion(); + } try { - context.getFinalizationStateManager() - .finalizeLayoutFeature(lf.layoutVersion()); + if (firstLv > -1) { + // If there are no feature to finalize we just skip this and the finalize operation is a noop. + context.getFinalizationStateManager().finalizeLayoutFeatures(firstLv, lastLv); + } } catch (IOException ex) { - throw new UpgradeException(ex, - UpgradeException.ResultCodes.LAYOUT_FEATURE_FINALIZATION_FAILED); + throw new UpgradeException(ex, UpgradeException.ResultCodes.LAYOUT_FEATURE_FINALIZATION_FAILED); } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmFinalization.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmFinalization.java index a6bc7e4acaa6..d0e972479abd 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmFinalization.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmFinalization.java @@ -123,7 +123,7 @@ public void testUpgradeStateToCheckpointMapping() throws Exception { for (HDDSLayoutFeature feature: HDDSLayoutFeature.values()) { // Cannot finalize initial version since we are already there. if (!feature.equals(HDDSLayoutFeature.INITIAL_VERSION)) { - stateManager.finalizeLayoutFeature(feature.layoutVersion()); + stateManager.finalizeLayoutFeatures(feature.layoutVersion(), feature.layoutVersion()); if (versionManager.needsFinalization()) { assertCurrentCheckpoint(scmContext, stateManager, FinalizationCheckpoint.FINALIZATION_STARTED); From c501d9e4d6f82928ceab07a83bcc939b852a416c Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Fri, 6 Feb 2026 16:26:41 +0000 Subject: [PATCH 2/4] Refactor test to finalize all versions in a single call --- .../hdds/scm/upgrade/TestScmFinalization.java | 33 +++++-------------- 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmFinalization.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmFinalization.java index d0e972479abd..c8e9e84dd264 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmFinalization.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmFinalization.java @@ -114,33 +114,18 @@ public void testUpgradeStateToCheckpointMapping() throws Exception { .build(); stateManager.setUpgradeContext(context); - assertCurrentCheckpoint(scmContext, stateManager, - FinalizationCheckpoint.FINALIZATION_REQUIRED); + assertCurrentCheckpoint(scmContext, stateManager, FinalizationCheckpoint.FINALIZATION_REQUIRED); stateManager.addFinalizingMark(); - assertCurrentCheckpoint(scmContext, stateManager, - FinalizationCheckpoint.FINALIZATION_STARTED); - - for (HDDSLayoutFeature feature: HDDSLayoutFeature.values()) { - // Cannot finalize initial version since we are already there. - if (!feature.equals(HDDSLayoutFeature.INITIAL_VERSION)) { - stateManager.finalizeLayoutFeatures(feature.layoutVersion(), feature.layoutVersion()); - if (versionManager.needsFinalization()) { - assertCurrentCheckpoint(scmContext, stateManager, - FinalizationCheckpoint.FINALIZATION_STARTED); - } else { - assertCurrentCheckpoint(scmContext, stateManager, - FinalizationCheckpoint.MLV_EQUALS_SLV); - } - } - } - // Make sure we reached this checkpoint if we finished finalizing all - // layout features. - assertCurrentCheckpoint(scmContext, stateManager, - FinalizationCheckpoint.MLV_EQUALS_SLV); + assertCurrentCheckpoint(scmContext, stateManager, FinalizationCheckpoint.FINALIZATION_STARTED); + HDDSLayoutFeature[] finalizationFeatures = HDDSLayoutFeature.values(); + HDDSLayoutFeature finalVersion = finalizationFeatures[finalizationFeatures.length - 1]; + assertCurrentCheckpoint(scmContext, stateManager, FinalizationCheckpoint.FINALIZATION_STARTED); + stateManager.finalizeLayoutFeatures( + HDDSLayoutFeature.INITIAL_VERSION.layoutVersion(), finalVersion.layoutVersion()); + assertCurrentCheckpoint(scmContext, stateManager, FinalizationCheckpoint.MLV_EQUALS_SLV); stateManager.removeFinalizingMark(); - assertCurrentCheckpoint(scmContext, stateManager, - FinalizationCheckpoint.FINALIZATION_COMPLETE); + assertCurrentCheckpoint(scmContext, stateManager, FinalizationCheckpoint.FINALIZATION_COMPLETE); } private void assertCurrentCheckpoint(SCMContext context, From 25991d7586acf45090c9f175ac21ffcb09a634c0 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Thu, 5 Mar 2026 12:00:01 +0000 Subject: [PATCH 3/4] Skip finalize if the feature is already finalized --- .../upgrade/FinalizationStateManagerImpl.java | 16 +++++++++++----- .../scm/server/upgrade/SCMUpgradeFinalizer.java | 3 +-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java index fc58c37d855e..1802c5f43917 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java @@ -99,8 +99,8 @@ public void addFinalizingMark() throws IOException { } @Override - public void finalizeLayoutFeatures(Integer startLayoutVersion, Integer endLayoutersion) throws IOException { - for (int version = startLayoutVersion; version <= endLayoutersion; version++) { + public void finalizeLayoutFeatures(Integer startLayoutVersion, Integer endLayoutVersion) throws IOException { + for (int version = startLayoutVersion; version <= endLayoutVersion; version++) { finalizeLayoutFeatureLocal(version); } } @@ -117,9 +117,15 @@ private void finalizeLayoutFeatureLocal(Integer layoutVersion) // version. This is updated in the replicated finalization steps. // Layout version will be written to the DB as well so followers can // finalize from a snapshot. - HDDSLayoutFeature feature = - (HDDSLayoutFeature)versionManager.getFeature(layoutVersion); - upgradeFinalizer.replicatedFinalizationSteps(feature, upgradeContext); + if (versionManager.getMetadataLayoutVersion() >= layoutVersion) { + LOG.warn("Attempting to finalize layout feature for layout version {}, but " + + "current metadata layout version is {}. Skipping finalization for this layout version.", + layoutVersion, versionManager.getMetadataLayoutVersion()); + } else { + HDDSLayoutFeature feature = + (HDDSLayoutFeature) versionManager.getFeature(layoutVersion); + upgradeFinalizer.replicatedFinalizationSteps(feature, upgradeContext); + } } finally { checkpointLock.writeLock().unlock(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java index 7c3f57bba94a..f3a0b9ad2c63 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java @@ -98,8 +98,7 @@ public void finalizeLayoutFeatures(Iterable features, SCMUpgradeF /** * Run on each SCM (leader and follower) when a layout feature is being - * finalized to run its finalization actions, update the VERSION file, and - * move the state of its in memory datanodes to healthy readonly. + * finalized to run its finalization actions, update the VERSION file. * * @param lf The layout feature that is being finalized. * @param context Supplier of objects needed to run the steps. From b1a344e99caa91d5121b62b0ae163c64b9c8d35b Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Fri, 6 Mar 2026 12:42:42 +0000 Subject: [PATCH 4/4] Change from and to version to just target version --- .../scm/server/upgrade/FinalizationStateManager.java | 2 +- .../server/upgrade/FinalizationStateManagerImpl.java | 5 +++-- .../hdds/scm/server/upgrade/SCMUpgradeFinalizer.java | 10 ++++------ .../hadoop/hdds/scm/upgrade/TestScmFinalization.java | 3 +-- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManager.java index 5e2cf59144e1..84cb6389ea68 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManager.java @@ -33,7 +33,7 @@ public interface FinalizationStateManager { void removeFinalizingMark() throws IOException; @Replicate - void finalizeLayoutFeatures(Integer startLayoutVersion, Integer endLayoutVersion) + void finalizeLayoutFeatures(Integer toLayoutVersion) throws IOException; /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java index 1802c5f43917..5ffa8062a5e3 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java @@ -99,8 +99,9 @@ public void addFinalizingMark() throws IOException { } @Override - public void finalizeLayoutFeatures(Integer startLayoutVersion, Integer endLayoutVersion) throws IOException { - for (int version = startLayoutVersion; version <= endLayoutVersion; version++) { + public void finalizeLayoutFeatures(Integer toVersion) throws IOException { + int startLayoutVersion = versionManager.getMetadataLayoutVersion() + 1; + for (int version = startLayoutVersion; version <= toVersion; version++) { finalizeLayoutFeatureLocal(version); } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java index f3a0b9ad2c63..773ade68a2fa 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java @@ -78,18 +78,16 @@ public void finalizeLayoutFeature(LayoutFeature lf, @Override public void finalizeLayoutFeatures(Iterable features, SCMUpgradeFinalizationContext context) throws UpgradeException { - int firstLv = -1; int lastLv = -1; for (LayoutFeature lf : features) { - if (firstLv == -1) { - firstLv = lf.layoutVersion(); - } lastLv = lf.layoutVersion(); } try { - if (firstLv > -1) { + if (lastLv > -1) { // If there are no feature to finalize we just skip this and the finalize operation is a noop. - context.getFinalizationStateManager().finalizeLayoutFeatures(firstLv, lastLv); + context.getFinalizationStateManager().finalizeLayoutFeatures(lastLv); + } else { + LOG.info("No layout features to finalize."); } } catch (IOException ex) { throw new UpgradeException(ex, UpgradeException.ResultCodes.LAYOUT_FEATURE_FINALIZATION_FAILED); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmFinalization.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmFinalization.java index c8e9e84dd264..9364b9ff37c4 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmFinalization.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmFinalization.java @@ -121,8 +121,7 @@ public void testUpgradeStateToCheckpointMapping() throws Exception { HDDSLayoutFeature[] finalizationFeatures = HDDSLayoutFeature.values(); HDDSLayoutFeature finalVersion = finalizationFeatures[finalizationFeatures.length - 1]; assertCurrentCheckpoint(scmContext, stateManager, FinalizationCheckpoint.FINALIZATION_STARTED); - stateManager.finalizeLayoutFeatures( - HDDSLayoutFeature.INITIAL_VERSION.layoutVersion(), finalVersion.layoutVersion()); + stateManager.finalizeLayoutFeatures(finalVersion.layoutVersion()); assertCurrentCheckpoint(scmContext, stateManager, FinalizationCheckpoint.MLV_EQUALS_SLV); stateManager.removeFinalizingMark(); assertCurrentCheckpoint(scmContext, stateManager, FinalizationCheckpoint.FINALIZATION_COMPLETE);