From 2b04aa1bb2ed2ec1431eefe024e951b4c9b0347a Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Tue, 5 May 2026 17:55:19 +0800 Subject: [PATCH 1/2] HDDS-15174. Add tests for Ozone Iceberg statistics file copy plan. --- .../TestRewriteTablePathOzoneAction.java | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/hadoop-ozone/iceberg/src/test/java/org/apache/hadoop/ozone/iceberg/TestRewriteTablePathOzoneAction.java b/hadoop-ozone/iceberg/src/test/java/org/apache/hadoop/ozone/iceberg/TestRewriteTablePathOzoneAction.java index 8678fe52301e..7f557d4c5345 100644 --- a/hadoop-ozone/iceberg/src/test/java/org/apache/hadoop/ozone/iceberg/TestRewriteTablePathOzoneAction.java +++ b/hadoop-ozone/iceberg/src/test/java/org/apache/hadoop/ozone/iceberg/TestRewriteTablePathOzoneAction.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.iceberg; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.BufferedReader; @@ -33,12 +34,14 @@ import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.FileFormat; +import org.apache.iceberg.GenericStatisticsFile; import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.RewriteTablePathUtil; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.StaticTableOperations; +import org.apache.iceberg.StatisticsFile; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableMetadata.MetadataLogEntry; @@ -188,6 +191,51 @@ void tablePathRewriteForStartAndEndVersionProvided() throws Exception { assertAllInternalPathsRewritten(csvPairs, targetPrefix); } + @Test + void statsFileCopyPlanReturnsEmptySetForEmptyStats() { + Set> copyPlan = + RewriteTablePathOzoneUtils.statsFileCopyPlan(List.of(), List.of()); + + assertTrue(copyPlan.isEmpty()); + } + + @Test + void statsFileCopyPlanRejectsMismatchedStatsCount() { + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> RewriteTablePathOzoneUtils.statsFileCopyPlan( + List.of(statisticsFile("before-1.puffin", 100)), + List.of())); + + assertEquals("Before and after path rewrite, statistic files count should be same", + exception.getMessage()); + } + + @Test + void statsFileCopyPlanRejectsMismatchedStatsFileSize() { + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> RewriteTablePathOzoneUtils.statsFileCopyPlan( + List.of(statisticsFile("before-1.puffin", 100)), + List.of(statisticsFile("after-1.puffin", 200)))); + + assertEquals("Before and after path rewrite, statistic files size should be same", + exception.getMessage()); + } + + @Test + void statsFileCopyPlanReturnsBeforeToAfterPathPairs() { + Set> copyPlan = RewriteTablePathOzoneUtils.statsFileCopyPlan( + List.of( + statisticsFile("before-1.puffin", 100), + statisticsFile("before-2.puffin", 200)), + List.of( + statisticsFile("after-1.puffin", 100), + statisticsFile("after-2.puffin", 200))); + + assertEquals(Set.of( + Pair.of("before-1.puffin", "after-1.puffin"), + Pair.of("before-2.puffin", "after-2.puffin")), copyPlan); + } + /** * For every staged metadata JSON file in the CSV, parses the file and asserts that: * - The table location starts with target @@ -276,4 +324,8 @@ private DataFile dummyDataFile(String dataPath) { .withFormat(FileFormat.PARQUET) .build(); } + + private static StatisticsFile statisticsFile(String path, long fileSizeInBytes) { + return new GenericStatisticsFile(1L, path, fileSizeInBytes, 0L, List.of()); + } } From 6bafb5ca6bb3d7156b590a1f589957ae19c07183 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 6 May 2026 10:52:28 +0800 Subject: [PATCH 2/2] HDDS-15174. Add tests for Ozone Iceberg statistics file copy plan. --- .../TestRewriteTablePathOzoneAction.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/iceberg/src/test/java/org/apache/hadoop/ozone/iceberg/TestRewriteTablePathOzoneAction.java b/hadoop-ozone/iceberg/src/test/java/org/apache/hadoop/ozone/iceberg/TestRewriteTablePathOzoneAction.java index 7f557d4c5345..fb657c981982 100644 --- a/hadoop-ozone/iceberg/src/test/java/org/apache/hadoop/ozone/iceberg/TestRewriteTablePathOzoneAction.java +++ b/hadoop-ozone/iceberg/src/test/java/org/apache/hadoop/ozone/iceberg/TestRewriteTablePathOzoneAction.java @@ -203,7 +203,7 @@ void statsFileCopyPlanReturnsEmptySetForEmptyStats() { void statsFileCopyPlanRejectsMismatchedStatsCount() { IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> RewriteTablePathOzoneUtils.statsFileCopyPlan( - List.of(statisticsFile("before-1.puffin", 100)), + List.of(statisticsFile("before-1.stats", 100)), List.of())); assertEquals("Before and after path rewrite, statistic files count should be same", @@ -214,8 +214,8 @@ void statsFileCopyPlanRejectsMismatchedStatsCount() { void statsFileCopyPlanRejectsMismatchedStatsFileSize() { IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> RewriteTablePathOzoneUtils.statsFileCopyPlan( - List.of(statisticsFile("before-1.puffin", 100)), - List.of(statisticsFile("after-1.puffin", 200)))); + List.of(statisticsFile("before-1.stats", 100)), + List.of(statisticsFile("after-1.stats", 200)))); assertEquals("Before and after path rewrite, statistic files size should be same", exception.getMessage()); @@ -225,15 +225,15 @@ void statsFileCopyPlanRejectsMismatchedStatsFileSize() { void statsFileCopyPlanReturnsBeforeToAfterPathPairs() { Set> copyPlan = RewriteTablePathOzoneUtils.statsFileCopyPlan( List.of( - statisticsFile("before-1.puffin", 100), - statisticsFile("before-2.puffin", 200)), + statisticsFile("before-1.stats", 100), + statisticsFile("before-2.stats", 200)), List.of( - statisticsFile("after-1.puffin", 100), - statisticsFile("after-2.puffin", 200))); + statisticsFile("after-1.stats", 100), + statisticsFile("after-2.stats", 200))); assertEquals(Set.of( - Pair.of("before-1.puffin", "after-1.puffin"), - Pair.of("before-2.puffin", "after-2.puffin")), copyPlan); + Pair.of("before-1.stats", "after-1.stats"), + Pair.of("before-2.stats", "after-2.stats")), copyPlan); } /**