Skip to content

Commit 1e4e991

Browse files
committed
Refactor mixed-format operation rules for Iceberg 1.10.x compatibility
1 parent 186ad77 commit 1e4e991

3 files changed

Lines changed: 17 additions & 5 deletions

File tree

amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@
4949
<dependency>
5050
<groupId>org.apache.avro</groupId>
5151
<artifactId>avro</artifactId>
52-
<version>1.12.1</version>
52+
<version>1.12.0</version>
5353
</dependency>
5454

5555
<dependency>

amoro-format-mixed/amoro-mixed-spark/v3.4/amoro-mixed-spark-3.4/src/main/scala/org/apache/amoro/spark/MixedFormatSparkExtensions.scala

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,17 @@ class MixedFormatSparkExtensions extends (SparkSessionExtensions => Unit) {
4343

4444
extensions.injectPostHocResolutionRule(spark => RewriteMixedFormatCommand(spark))
4545

46+
// mixed-format row-level operation rewrite rules
47+
// These must be resolution rules (not optimizer rules) so they run BEFORE Iceberg 1.10.x's
48+
// RewriteUpdateTableForRowLineage and RewriteMergeIntoTableForRowLineage rules. Those Iceberg
49+
// rules do pattern matching on the table and throw scala.MatchError for non-SparkTable types
50+
// (i.e., MixedSparkTable).
51+
extensions.injectResolutionRule { spark => RewriteUpdateMixedFormatTable(spark) }
52+
extensions.injectResolutionRule { spark => RewriteDeleteFromMixedFormatTable(spark) }
53+
4654
// mixed-format optimizer rules
4755
extensions.injectPostHocResolutionRule { spark => QueryWithConstraintCheck(spark) }
4856
extensions.injectOptimizerRule { spark => RewriteAppendMixedFormatTable(spark) }
49-
extensions.injectOptimizerRule { spark => RewriteDeleteFromMixedFormatTable(spark) }
50-
extensions.injectOptimizerRule { spark => RewriteUpdateMixedFormatTable(spark) }
5157

5258
// planner extensions
5359
extensions.injectPlannerStrategy { spark => MixedFormatExtendedDataSourceV2Strategy(spark) }

amoro-format-mixed/amoro-mixed-spark/v3.5/amoro-mixed-spark-3.5/src/main/scala/org/apache/amoro/spark/MixedFormatSparkExtensions.scala

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,17 @@ class MixedFormatSparkExtensions extends (SparkSessionExtensions => Unit) {
4343

4444
extensions.injectPostHocResolutionRule(spark => RewriteMixedFormatCommand(spark))
4545

46+
// mixed-format row-level operation rewrite rules
47+
// These must be resolution rules (not optimizer rules) so they run BEFORE Iceberg 1.10.x's
48+
// RewriteUpdateTableForRowLineage and RewriteMergeIntoTableForRowLineage rules. Those Iceberg
49+
// rules do pattern matching on the table and throw scala.MatchError for non-SparkTable types
50+
// (i.e., MixedSparkTable).
51+
extensions.injectResolutionRule { spark => RewriteUpdateMixedFormatTable(spark) }
52+
extensions.injectResolutionRule { spark => RewriteDeleteFromMixedFormatTable(spark) }
53+
4654
// mixed-format optimizer rules
4755
extensions.injectPostHocResolutionRule { spark => QueryWithConstraintCheck(spark) }
4856
extensions.injectOptimizerRule { spark => RewriteAppendMixedFormatTable(spark) }
49-
extensions.injectOptimizerRule { spark => RewriteDeleteFromMixedFormatTable(spark) }
50-
extensions.injectOptimizerRule { spark => RewriteUpdateMixedFormatTable(spark) }
5157

5258
// planner extensions
5359
extensions.injectPlannerStrategy { spark => MixedFormatExtendedDataSourceV2Strategy(spark) }

0 commit comments

Comments
 (0)