diff --git a/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala b/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala index 6216dd8747f..142aeb64921 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala @@ -21,42 +21,30 @@ import org.apache.gluten.execution._ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.aggregate._ -import org.apache.spark.sql.catalyst.plans.physical.ClusteredDistribution import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.trees.TreePattern.EXCHANGE import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.exchange.ShuffleExchangeLike import org.apache.spark.sql.types.{DataType, DoubleType, FloatType} +import scala.collection.mutable + /** * To transform regular aggregation to intermediate aggregation that internally enables * optimizations such as flushing and abandoning. */ case class FlushableHashAggregateRule(session: SparkSession) extends Rule[SparkPlan] { - import FlushableHashAggregateRule._ override def apply(plan: SparkPlan): SparkPlan = { if (!VeloxConfig.get.enableVeloxFlushablePartialAggregation) { return plan } + val protectedAggs = collectProtectedOneDistinctPartialMergeAggs(plan) plan.transformUpWithPruning(_.containsPattern(EXCHANGE)) { case s: ShuffleExchangeLike => // If an exchange follows a hash aggregate in which all functions are in partial mode, // then it's safe to convert the hash aggregate to flushable hash aggregate. val out = s.withNewChildren( - List( - replaceEligibleAggregates(s.child) { - agg => - FlushableHashAggregateExecTransformer( - agg.requiredChildDistributionExpressions, - agg.groupingExpressions, - agg.aggregateExpressions, - agg.aggregateAttributes, - agg.initialInputBufferOffset, - agg.resultExpressions, - agg.child - ) - } - ) + List(replaceEligibleAggregates(s.child, protectedAggs)) ) out } @@ -85,19 +73,35 @@ case class FlushableHashAggregateRule(session: SparkSession) extends Rule[SparkP /** * Walks the plan downward, applying func to each RegularHashAggregateExecTransformer or * SortHashAggregateExecTransformer that is eligible for flushable conversion. An aggregate is - * eligible when all expressions are Partial/PartialMerge, input is not already partitioned by the - * grouping keys, and no aggregate function disallows flushing. + * eligible when all expressions are Partial/PartialMerge, it is not the protected PartialMerge + * aggregate directly below a distinct-partial aggregate, and no aggregate function disallows + * flushing. */ - private def replaceEligibleAggregates(plan: SparkPlan)( - func: HashAggregateExecTransformer => SparkPlan): SparkPlan = { + private def replaceEligibleAggregates( + plan: SparkPlan, + protectedAggs: mutable.Map[Int, HashAggregateExecTransformer]): SparkPlan = { + def toFlushableAgg(agg: HashAggregateExecTransformer): FlushableHashAggregateExecTransformer = { + FlushableHashAggregateExecTransformer( + agg.requiredChildDistributionExpressions, + agg.groupingExpressions, + agg.aggregateExpressions, + agg.aggregateAttributes, + agg.initialInputBufferOffset, + agg.resultExpressions, + agg.child + ) + } + def transformDown: SparkPlan => SparkPlan = { case agg: RegularHashAggregateExecTransformer if !agg.aggregateExpressions.forall(p => p.mode == Partial || p.mode == PartialMerge) => // Not an intermediate agg. Skip. agg case agg: RegularHashAggregateExecTransformer - if isAggInputAlreadyDistributedWithAggKeys(agg) => - // Data already grouped by aggregate keys. Skip. + if protectedAggs.contains(agg.id) => + // This is the PartialMerge aggregate directly below a distinct-partial aggregate in + // Spark's one-distinct pipeline. Keep it non-flushable so the distinct step continues to + // see globally de-duplicated (grouping + distinct) keys. agg case agg: RegularHashAggregateExecTransformer if aggregatesNotSupportFlush(agg.aggregateExpressions) => @@ -105,13 +109,13 @@ case class FlushableHashAggregateRule(session: SparkSession) extends Rule[SparkP agg case agg: RegularHashAggregateExecTransformer => // All guards passed; replace with the flushable variant. - func(agg) + toFlushableAgg(agg) case agg: SortHashAggregateExecTransformer if !agg.aggregateExpressions.forall(p => p.mode == Partial || p.mode == PartialMerge) => // Not an intermediate agg. Skip. agg - case agg: SortHashAggregateExecTransformer if isAggInputAlreadyDistributedWithAggKeys(agg) => - // Data already grouped by aggregate keys. Skip. + case agg: SortHashAggregateExecTransformer if protectedAggs.contains(agg.id) => + // See the RegularHashAggregateExecTransformer branch above. agg case agg: SortHashAggregateExecTransformer if aggregatesNotSupportFlush(agg.aggregateExpressions) => @@ -119,8 +123,11 @@ case class FlushableHashAggregateRule(session: SparkSession) extends Rule[SparkP agg case agg: SortHashAggregateExecTransformer => // All guards passed; replace with the flushable variant. - func(agg) - case p if !canPropagate(p) => p + toFlushableAgg(agg) + case exchange: ShuffleExchangeLike => + // Stop at the next exchange. This rule is applied from an exchange boundary and should not + // continue rewriting into a different shuffle region. + exchange case other => other.withNewChildren(other.children.map(transformDown)) } @@ -128,36 +135,57 @@ case class FlushableHashAggregateRule(session: SparkSession) extends Rule[SparkP out } - private def canPropagate(plan: SparkPlan): Boolean = plan match { - case _: ProjectExecTransformer => true - case _: VeloxResizeBatchesExec => true - case _ => false - } -} - -object FlushableHashAggregateRule { - /** - * If child output already partitioned by aggregation keys (this function returns true), we - * usually avoid the optimization converting to flushable aggregation. + * Collect the PartialMerge aggregates that must stay regular in Spark's one-distinct aggregation + * pipeline. + * + * Example plan shape: + * + * RegularHashAggregateExecTransformer [k] [count(distinct v)] // finalAggregate +- + * RegularHashAggregateExecTransformer [k] [count(distinct v)] // partialDistinctAggregate +- + * RegularHashAggregateExecTransformer [k, v] [count(...)] // partialMergeAggregate +- + * ColumnarExchange hashpartitioning(k, v, 200) +- RegularHashAggregateExecTransformer [k, v] + * [count(...)] // partialAggregate +- ... * - * For example, if input is hash-partitioned by keys (a, b) and aggregate node requests "group by - * a, b, c", then the aggregate should NOT flush as the grouping set (a, b, c) will be created - * only on a single partition among the whole cluster. Spark's planner may use this information to - * perform optimizations like doing "partial_count(a, b, c)" directly on the output data. + * We walk every aggregate node and, when we encounter the `partialDistinctAggregate`, we record + * its child `partialMergeAggregate` as protected. + * + * That `partialMergeAggregate` must stay regular. It is the step that materializes the + * de-duplicated `(k, v)` stream consumed by the distinct-partial aggregate above it. If it + * flushes, duplicate `(k, v)` keys may be reintroduced within one partition and the distinct + * aggregation pipeline would no longer see the shape Spark planned for. */ - private def isAggInputAlreadyDistributedWithAggKeys( - agg: HashAggregateExecTransformer): Boolean = { - if (agg.groupingExpressions.isEmpty) { - // Empty grouping set () should not be satisfied by any partitioning patterns. - // E.g., - // (a, b) satisfies (a, b, c) - // (a, b) satisfies (a, b) - // (a, b) doesn't satisfy (a) - // (a, b) doesn't satisfy () - return false + private def collectProtectedOneDistinctPartialMergeAggs( + plan: SparkPlan): mutable.Map[Int, HashAggregateExecTransformer] = { + val protectedAggs = mutable.HashMap.empty[Int, HashAggregateExecTransformer] + plan.foreach { + case agg: HashAggregateExecTransformer => + findProtectedPartialMergeAgg(agg).foreach { + protectedAgg => protectedAggs.put(protectedAgg.id, protectedAgg) + } + case _ => + } + protectedAggs + } + + /** If this aggregate is the distinct-partial stage, return its child PartialMerge aggregate. */ + private def findProtectedPartialMergeAgg( + distinctPartialAgg: HashAggregateExecTransformer): Option[HashAggregateExecTransformer] = { + if ( + !distinctPartialAgg.aggregateExpressions.exists( + expr => expr.isDistinct && expr.mode == Partial) + ) { + return None } - val distribution = ClusteredDistribution(agg.groupingExpressions) - agg.child.outputPartitioning.satisfies(distribution) + + for { + partialMergeAgg <- asAggregate(distinctPartialAgg.child) + if partialMergeAgg.aggregateExpressions.forall(_.mode == PartialMerge) + } yield partialMergeAgg + } + + private def asAggregate(plan: SparkPlan): Option[HashAggregateExecTransformer] = plan match { + case agg: HashAggregateExecTransformer => Some(agg) + case _ => None } } diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala index 9f1c8d5cc13..98b46255ff2 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala @@ -1258,20 +1258,26 @@ class VeloxAggregateFunctionsFlushSuite extends VeloxAggregateFunctionsSuite { } } - test("flushable aggregate rule - agg input already distributed by keys") { + test("flushable aggregate rule - count distinct keeps partial merge regular") { withSQLConf( SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", SQLConf.FILES_MAX_PARTITION_BYTES.key -> "1k") { - runQueryAndCompare( - "select * from (select distinct l_orderkey,l_partkey from lineitem) a" + - " inner join (select l_orderkey from lineitem limit 10) b" + - " on a.l_orderkey = b.l_orderkey limit 10") { + runQueryAndCompare("select count(distinct l_partkey) from lineitem group by l_orderkey") { df => val executedPlan = getExecutedPlan(df) + val regularAggCount = executedPlan.count { + plan => plan.isInstanceOf[RegularHashAggregateExecTransformer] + } + val flushableAggCount = executedPlan.count { + plan => plan.isInstanceOf[FlushableHashAggregateExecTransformer] + } assert( - executedPlan.exists(plan => plan.isInstanceOf[RegularHashAggregateExecTransformer])) + regularAggCount == 2, + s"expected 2 regular hash aggregates in one-distinct pipeline, got $regularAggCount") assert( - executedPlan.exists(plan => plan.isInstanceOf[FlushableHashAggregateExecTransformer])) + flushableAggCount == 2, + s"expected 2 flushable hash aggregates in one-distinct pipeline, got" + + s" $flushableAggCount") } } } diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14a.sf100/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14a.sf100/explain.txt index 91e065185c1..e2d3402ee23 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14a.sf100/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14a.sf100/explain.txt @@ -39,7 +39,7 @@ VeloxColumnarToRow (187) : : : : +- ColumnarExchange (58) : : : : +- VeloxResizeBatches (57) : : : : +- ^ ProjectExecTransformer (55) - : : : : +- ^ RegularHashAggregateExecTransformer (54) + : : : : +- ^ FlushableHashAggregateExecTransformer (54) : : : : +- ^ InputIteratorTransformer (53) : : : : +- ColumnarExchange (51) : : : : +- VeloxResizeBatches (50) @@ -373,7 +373,7 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] (53) InputIteratorTransformer Input [3]: [brand_id#27, class_id#28, category_id#29] -(54) RegularHashAggregateExecTransformer +(54) FlushableHashAggregateExecTransformer Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14a.sf100/simplified.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14a.sf100/simplified.txt index 9fe87fb5a04..ba3b8bc9e0f 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14a.sf100/simplified.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14a.sf100/simplified.txt @@ -95,7 +95,7 @@ VeloxColumnarToRow VeloxResizeBatches WholeStageCodegenTransformer (22) ProjectExecTransformer [brand_id,class_id,category_id] - RegularHashAggregateExecTransformer [brand_id,class_id,category_id] + FlushableHashAggregateExecTransformer [brand_id,class_id,category_id] InputIteratorTransformer InputAdapter ColumnarExchange [brand_id,class_id,category_id] #7 diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14b.sf100/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14b.sf100/explain.txt index c2761a85a07..c49e2b293cd 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14b.sf100/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14b.sf100/explain.txt @@ -30,7 +30,7 @@ VeloxColumnarToRow (150) : : : : +- ColumnarExchange (58) : : : : +- VeloxResizeBatches (57) : : : : +- ^ ProjectExecTransformer (55) - : : : : +- ^ RegularHashAggregateExecTransformer (54) + : : : : +- ^ FlushableHashAggregateExecTransformer (54) : : : : +- ^ InputIteratorTransformer (53) : : : : +- ColumnarExchange (51) : : : : +- VeloxResizeBatches (50) @@ -345,7 +345,7 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] (53) InputIteratorTransformer Input [3]: [brand_id#27, class_id#28, category_id#29] -(54) RegularHashAggregateExecTransformer +(54) FlushableHashAggregateExecTransformer Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14b.sf100/simplified.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14b.sf100/simplified.txt index 670b1d15ce2..8de7ae6341b 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14b.sf100/simplified.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14b.sf100/simplified.txt @@ -90,7 +90,7 @@ VeloxColumnarToRow VeloxResizeBatches WholeStageCodegenTransformer (24) ProjectExecTransformer [brand_id,class_id,category_id] - RegularHashAggregateExecTransformer [brand_id,class_id,category_id] + FlushableHashAggregateExecTransformer [brand_id,class_id,category_id] InputIteratorTransformer InputAdapter ColumnarExchange [brand_id,class_id,category_id] #6 diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q38.sf100/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q38.sf100/explain.txt index 7f48666f41d..9dc843bff3a 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q38.sf100/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q38.sf100/explain.txt @@ -8,7 +8,7 @@ VeloxColumnarToRow (104) : : +- ColumnarExchange (34) : : +- VeloxResizeBatches (33) : : +- ^ ProjectExecTransformer (31) - : : +- ^ RegularHashAggregateExecTransformer (30) + : : +- ^ FlushableHashAggregateExecTransformer (30) : : +- ^ InputIteratorTransformer (29) : : +- ColumnarExchange (27) : : +- VeloxResizeBatches (26) @@ -35,7 +35,7 @@ VeloxColumnarToRow (104) : +- ColumnarExchange (65) : +- VeloxResizeBatches (64) : +- ^ ProjectExecTransformer (62) - : +- ^ RegularHashAggregateExecTransformer (61) + : +- ^ FlushableHashAggregateExecTransformer (61) : +- ^ InputIteratorTransformer (60) : +- ColumnarExchange (58) : +- VeloxResizeBatches (57) @@ -58,7 +58,7 @@ VeloxColumnarToRow (104) +- ColumnarExchange (97) +- VeloxResizeBatches (96) +- ^ ProjectExecTransformer (94) - +- ^ RegularHashAggregateExecTransformer (93) + +- ^ FlushableHashAggregateExecTransformer (93) +- ^ InputIteratorTransformer (92) +- ColumnarExchange (90) +- VeloxResizeBatches (89) @@ -200,7 +200,7 @@ Input [3]: [c_last_name#9, c_first_name#8, d_date#5] (29) InputIteratorTransformer Input [3]: [c_last_name#9, c_first_name#8, d_date#5] -(30) RegularHashAggregateExecTransformer +(30) FlushableHashAggregateExecTransformer Input [3]: [c_last_name#9, c_first_name#8, d_date#5] Keys [3]: [c_last_name#9, c_first_name#8, d_date#5] Functions: [] @@ -326,7 +326,7 @@ Input [3]: [c_last_name#20, c_first_name#19, d_date#16] (60) InputIteratorTransformer Input [3]: [c_last_name#20, c_first_name#19, d_date#16] -(61) RegularHashAggregateExecTransformer +(61) FlushableHashAggregateExecTransformer Input [3]: [c_last_name#20, c_first_name#19, d_date#16] Keys [3]: [c_last_name#20, c_first_name#19, d_date#16] Functions: [] @@ -458,7 +458,7 @@ Input [3]: [c_last_name#30, c_first_name#29, d_date#26] (92) InputIteratorTransformer Input [3]: [c_last_name#30, c_first_name#29, d_date#26] -(93) RegularHashAggregateExecTransformer +(93) FlushableHashAggregateExecTransformer Input [3]: [c_last_name#30, c_first_name#29, d_date#26] Keys [3]: [c_last_name#30, c_first_name#29, d_date#26] Functions: [] diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q38.sf100/simplified.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q38.sf100/simplified.txt index 54c142449b1..d8b5fd2b904 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q38.sf100/simplified.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q38.sf100/simplified.txt @@ -10,7 +10,7 @@ VeloxColumnarToRow VeloxResizeBatches WholeStageCodegenTransformer (6) ProjectExecTransformer [c_last_name,c_first_name,d_date] - RegularHashAggregateExecTransformer [c_last_name,c_first_name,d_date] + FlushableHashAggregateExecTransformer [c_last_name,c_first_name,d_date] InputIteratorTransformer InputAdapter ColumnarExchange [c_last_name,c_first_name,d_date] #2 @@ -52,7 +52,7 @@ VeloxColumnarToRow VeloxResizeBatches WholeStageCodegenTransformer (12) ProjectExecTransformer [c_last_name,c_first_name,d_date] - RegularHashAggregateExecTransformer [c_last_name,c_first_name,d_date] + FlushableHashAggregateExecTransformer [c_last_name,c_first_name,d_date] InputIteratorTransformer InputAdapter ColumnarExchange [c_last_name,c_first_name,d_date] #7 @@ -84,7 +84,7 @@ VeloxColumnarToRow VeloxResizeBatches WholeStageCodegenTransformer (18) ProjectExecTransformer [c_last_name,c_first_name,d_date] - RegularHashAggregateExecTransformer [c_last_name,c_first_name,d_date] + FlushableHashAggregateExecTransformer [c_last_name,c_first_name,d_date] InputIteratorTransformer InputAdapter ColumnarExchange [c_last_name,c_first_name,d_date] #10 diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q8.sf100/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q8.sf100/explain.txt index 1e5a88f5693..057d344bc68 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q8.sf100/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q8.sf100/explain.txt @@ -29,7 +29,7 @@ VeloxColumnarToRow (85) +- ColumnarExchange (69) +- VeloxResizeBatches (68) +- ^ ProjectExecTransformer (66) - +- ^ RegularHashAggregateExecTransformer (65) + +- ^ FlushableHashAggregateExecTransformer (65) +- ^ InputIteratorTransformer (64) +- ColumnarExchange (62) +- VeloxResizeBatches (61) @@ -337,7 +337,7 @@ Input [1]: [ca_zip#11] (64) InputIteratorTransformer Input [1]: [ca_zip#11] -(65) RegularHashAggregateExecTransformer +(65) FlushableHashAggregateExecTransformer Input [1]: [ca_zip#11] Keys [1]: [ca_zip#11] Functions: [] diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q8.sf100/simplified.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q8.sf100/simplified.txt index 77da4dfe6e1..f24a2989dd3 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q8.sf100/simplified.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q8.sf100/simplified.txt @@ -44,7 +44,7 @@ VeloxColumnarToRow VeloxResizeBatches WholeStageCodegenTransformer (10) ProjectExecTransformer [ca_zip] - RegularHashAggregateExecTransformer [ca_zip] + FlushableHashAggregateExecTransformer [ca_zip] InputIteratorTransformer InputAdapter ColumnarExchange [ca_zip] #6 diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q87.sf100/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q87.sf100/explain.txt index 4fc5497a3c5..9573f717fa9 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q87.sf100/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q87.sf100/explain.txt @@ -8,7 +8,7 @@ VeloxColumnarToRow (104) : : +- ColumnarExchange (34) : : +- VeloxResizeBatches (33) : : +- ^ ProjectExecTransformer (31) - : : +- ^ RegularHashAggregateExecTransformer (30) + : : +- ^ FlushableHashAggregateExecTransformer (30) : : +- ^ InputIteratorTransformer (29) : : +- ColumnarExchange (27) : : +- VeloxResizeBatches (26) @@ -35,7 +35,7 @@ VeloxColumnarToRow (104) : +- ColumnarExchange (65) : +- VeloxResizeBatches (64) : +- ^ ProjectExecTransformer (62) - : +- ^ RegularHashAggregateExecTransformer (61) + : +- ^ FlushableHashAggregateExecTransformer (61) : +- ^ InputIteratorTransformer (60) : +- ColumnarExchange (58) : +- VeloxResizeBatches (57) @@ -58,7 +58,7 @@ VeloxColumnarToRow (104) +- ColumnarExchange (97) +- VeloxResizeBatches (96) +- ^ ProjectExecTransformer (94) - +- ^ RegularHashAggregateExecTransformer (93) + +- ^ FlushableHashAggregateExecTransformer (93) +- ^ InputIteratorTransformer (92) +- ColumnarExchange (90) +- VeloxResizeBatches (89) @@ -200,7 +200,7 @@ Input [3]: [c_last_name#9, c_first_name#8, d_date#5] (29) InputIteratorTransformer Input [3]: [c_last_name#9, c_first_name#8, d_date#5] -(30) RegularHashAggregateExecTransformer +(30) FlushableHashAggregateExecTransformer Input [3]: [c_last_name#9, c_first_name#8, d_date#5] Keys [3]: [c_last_name#9, c_first_name#8, d_date#5] Functions: [] @@ -326,7 +326,7 @@ Input [3]: [c_last_name#20, c_first_name#19, d_date#16] (60) InputIteratorTransformer Input [3]: [c_last_name#20, c_first_name#19, d_date#16] -(61) RegularHashAggregateExecTransformer +(61) FlushableHashAggregateExecTransformer Input [3]: [c_last_name#20, c_first_name#19, d_date#16] Keys [3]: [c_last_name#20, c_first_name#19, d_date#16] Functions: [] @@ -458,7 +458,7 @@ Input [3]: [c_last_name#30, c_first_name#29, d_date#26] (92) InputIteratorTransformer Input [3]: [c_last_name#30, c_first_name#29, d_date#26] -(93) RegularHashAggregateExecTransformer +(93) FlushableHashAggregateExecTransformer Input [3]: [c_last_name#30, c_first_name#29, d_date#26] Keys [3]: [c_last_name#30, c_first_name#29, d_date#26] Functions: [] diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q87.sf100/simplified.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q87.sf100/simplified.txt index 54c142449b1..d8b5fd2b904 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q87.sf100/simplified.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q87.sf100/simplified.txt @@ -10,7 +10,7 @@ VeloxColumnarToRow VeloxResizeBatches WholeStageCodegenTransformer (6) ProjectExecTransformer [c_last_name,c_first_name,d_date] - RegularHashAggregateExecTransformer [c_last_name,c_first_name,d_date] + FlushableHashAggregateExecTransformer [c_last_name,c_first_name,d_date] InputIteratorTransformer InputAdapter ColumnarExchange [c_last_name,c_first_name,d_date] #2 @@ -52,7 +52,7 @@ VeloxColumnarToRow VeloxResizeBatches WholeStageCodegenTransformer (12) ProjectExecTransformer [c_last_name,c_first_name,d_date] - RegularHashAggregateExecTransformer [c_last_name,c_first_name,d_date] + FlushableHashAggregateExecTransformer [c_last_name,c_first_name,d_date] InputIteratorTransformer InputAdapter ColumnarExchange [c_last_name,c_first_name,d_date] #7 @@ -84,7 +84,7 @@ VeloxColumnarToRow VeloxResizeBatches WholeStageCodegenTransformer (18) ProjectExecTransformer [c_last_name,c_first_name,d_date] - RegularHashAggregateExecTransformer [c_last_name,c_first_name,d_date] + FlushableHashAggregateExecTransformer [c_last_name,c_first_name,d_date] InputIteratorTransformer InputAdapter ColumnarExchange [c_last_name,c_first_name,d_date] #10 diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14.sf100/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14.sf100/explain.txt index b56686f4da2..14307ccf270 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14.sf100/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14.sf100/explain.txt @@ -30,7 +30,7 @@ VeloxColumnarToRow (150) : : : : +- ColumnarExchange (58) : : : : +- VeloxResizeBatches (57) : : : : +- ^ ProjectExecTransformer (55) - : : : : +- ^ RegularHashAggregateExecTransformer (54) + : : : : +- ^ FlushableHashAggregateExecTransformer (54) : : : : +- ^ InputIteratorTransformer (53) : : : : +- ColumnarExchange (51) : : : : +- VeloxResizeBatches (50) @@ -345,7 +345,7 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] (53) InputIteratorTransformer Input [3]: [brand_id#27, class_id#28, category_id#29] -(54) RegularHashAggregateExecTransformer +(54) FlushableHashAggregateExecTransformer Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14.sf100/simplified.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14.sf100/simplified.txt index 670b1d15ce2..8de7ae6341b 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14.sf100/simplified.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14.sf100/simplified.txt @@ -90,7 +90,7 @@ VeloxColumnarToRow VeloxResizeBatches WholeStageCodegenTransformer (24) ProjectExecTransformer [brand_id,class_id,category_id] - RegularHashAggregateExecTransformer [brand_id,class_id,category_id] + FlushableHashAggregateExecTransformer [brand_id,class_id,category_id] InputIteratorTransformer InputAdapter ColumnarExchange [brand_id,class_id,category_id] #6 diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14a.sf100/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14a.sf100/explain.txt index e014c7566fc..9895bfc7359 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14a.sf100/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14a.sf100/explain.txt @@ -45,7 +45,7 @@ VeloxColumnarToRow (250) : : : : : +- ColumnarExchange (58) : : : : : +- VeloxResizeBatches (57) : : : : : +- ^ ProjectExecTransformer (55) - : : : : : +- ^ RegularHashAggregateExecTransformer (54) + : : : : : +- ^ FlushableHashAggregateExecTransformer (54) : : : : : +- ^ InputIteratorTransformer (53) : : : : : +- ColumnarExchange (51) : : : : : +- VeloxResizeBatches (50) @@ -418,7 +418,7 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] (53) InputIteratorTransformer Input [3]: [brand_id#27, class_id#28, category_id#29] -(54) RegularHashAggregateExecTransformer +(54) FlushableHashAggregateExecTransformer Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14a.sf100/simplified.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14a.sf100/simplified.txt index c063978c3b9..2592299a1a1 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14a.sf100/simplified.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14a.sf100/simplified.txt @@ -110,7 +110,7 @@ VeloxColumnarToRow VeloxResizeBatches WholeStageCodegenTransformer (22) ProjectExecTransformer [brand_id,class_id,category_id] - RegularHashAggregateExecTransformer [brand_id,class_id,category_id] + FlushableHashAggregateExecTransformer [brand_id,class_id,category_id] InputIteratorTransformer InputAdapter ColumnarExchange [brand_id,class_id,category_id] #8 diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a.sf100/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a.sf100/explain.txt index 2a5c7113d0b..d248f624126 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a.sf100/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a.sf100/explain.txt @@ -9,7 +9,7 @@ VeloxColumnarToRow (81) +- VeloxResizeBatches (72) +- ^ ProjectExecTransformer (70) +- ^ ProjectExecTransformer (69) - +- ^ RegularHashAggregateExecTransformer (68) + +- ^ FlushableHashAggregateExecTransformer (68) +- ^ InputIteratorTransformer (67) +- ColumnarExchange (65) +- VeloxResizeBatches (64) @@ -348,7 +348,7 @@ Input [6]: [gross_margin#22, i_category#12, i_class#11, t_category#23, t_class#2 (67) InputIteratorTransformer Input [6]: [gross_margin#22, i_category#12, i_class#11, t_category#23, t_class#24, lochierarchy#25] -(68) RegularHashAggregateExecTransformer +(68) FlushableHashAggregateExecTransformer Input [6]: [gross_margin#22, i_category#12, i_class#11, t_category#23, t_class#24, lochierarchy#25] Keys [6]: [gross_margin#22, i_category#12, i_class#11, t_category#23, t_class#24, lochierarchy#25] Functions: [] diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a.sf100/simplified.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a.sf100/simplified.txt index 0129209b62e..e643c62aeed 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a.sf100/simplified.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a.sf100/simplified.txt @@ -11,7 +11,7 @@ VeloxColumnarToRow WholeStageCodegenTransformer (21) ProjectExecTransformer [lochierarchy,_w0,gross_margin,i_category,i_class] ProjectExecTransformer [gross_margin,i_category,i_class,lochierarchy,t_class] - RegularHashAggregateExecTransformer [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] + FlushableHashAggregateExecTransformer [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] InputIteratorTransformer InputAdapter ColumnarExchange [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] #2 diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a/explain.txt index 9fad6bd36fc..de6e3285a98 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a/explain.txt @@ -9,7 +9,7 @@ VeloxColumnarToRow (81) +- VeloxResizeBatches (72) +- ^ ProjectExecTransformer (70) +- ^ ProjectExecTransformer (69) - +- ^ RegularHashAggregateExecTransformer (68) + +- ^ FlushableHashAggregateExecTransformer (68) +- ^ InputIteratorTransformer (67) +- ColumnarExchange (65) +- VeloxResizeBatches (64) @@ -348,7 +348,7 @@ Input [6]: [gross_margin#22, i_category#10, i_class#9, t_category#23, t_class#24 (67) InputIteratorTransformer Input [6]: [gross_margin#22, i_category#10, i_class#9, t_category#23, t_class#24, lochierarchy#25] -(68) RegularHashAggregateExecTransformer +(68) FlushableHashAggregateExecTransformer Input [6]: [gross_margin#22, i_category#10, i_class#9, t_category#23, t_class#24, lochierarchy#25] Keys [6]: [gross_margin#22, i_category#10, i_class#9, t_category#23, t_class#24, lochierarchy#25] Functions: [] diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a/simplified.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a/simplified.txt index 1429b04fd98..d5cc8efaac4 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a/simplified.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a/simplified.txt @@ -11,7 +11,7 @@ VeloxColumnarToRow WholeStageCodegenTransformer (21) ProjectExecTransformer [lochierarchy,_w0,gross_margin,i_category,i_class] ProjectExecTransformer [gross_margin,i_category,i_class,lochierarchy,t_class] - RegularHashAggregateExecTransformer [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] + FlushableHashAggregateExecTransformer [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] InputIteratorTransformer InputAdapter ColumnarExchange [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] #2 diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a.sf100/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a.sf100/explain.txt index 8b62712d862..8ef5a372350 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a.sf100/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a.sf100/explain.txt @@ -9,7 +9,7 @@ VeloxColumnarToRow (106) +- VeloxResizeBatches (97) +- ^ ProjectExecTransformer (95) +- ^ ProjectExecTransformer (94) - +- ^ RegularHashAggregateExecTransformer (93) + +- ^ FlushableHashAggregateExecTransformer (93) +- ^ InputIteratorTransformer (92) +- ColumnarExchange (90) +- VeloxResizeBatches (89) @@ -475,7 +475,7 @@ Input [6]: [total_sum#27, s_state#8, s_county#7, g_state#28, g_county#29, lochie (92) InputIteratorTransformer Input [6]: [total_sum#27, s_state#8, s_county#7, g_state#28, g_county#29, lochierarchy#30] -(93) RegularHashAggregateExecTransformer +(93) FlushableHashAggregateExecTransformer Input [6]: [total_sum#27, s_state#8, s_county#7, g_state#28, g_county#29, lochierarchy#30] Keys [6]: [total_sum#27, s_state#8, s_county#7, g_state#28, g_county#29, lochierarchy#30] Functions: [] diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a.sf100/simplified.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a.sf100/simplified.txt index 81dd34e12ba..08237488257 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a.sf100/simplified.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a.sf100/simplified.txt @@ -11,7 +11,7 @@ VeloxColumnarToRow WholeStageCodegenTransformer (33) ProjectExecTransformer [lochierarchy,_w0,total_sum,s_state,s_county] ProjectExecTransformer [total_sum,s_state,s_county,lochierarchy,g_county] - RegularHashAggregateExecTransformer [total_sum,s_state,s_county,g_state,g_county,lochierarchy] + FlushableHashAggregateExecTransformer [total_sum,s_state,s_county,g_state,g_county,lochierarchy] InputIteratorTransformer InputAdapter ColumnarExchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2 diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a/explain.txt index 148593a5123..823a0332a72 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a/explain.txt @@ -9,7 +9,7 @@ VeloxColumnarToRow (106) +- VeloxResizeBatches (97) +- ^ ProjectExecTransformer (95) +- ^ ProjectExecTransformer (94) - +- ^ RegularHashAggregateExecTransformer (93) + +- ^ FlushableHashAggregateExecTransformer (93) +- ^ InputIteratorTransformer (92) +- ColumnarExchange (90) +- VeloxResizeBatches (89) @@ -475,7 +475,7 @@ Input [6]: [total_sum#27, s_state#8, s_county#7, g_state#28, g_county#29, lochie (92) InputIteratorTransformer Input [6]: [total_sum#27, s_state#8, s_county#7, g_state#28, g_county#29, lochierarchy#30] -(93) RegularHashAggregateExecTransformer +(93) FlushableHashAggregateExecTransformer Input [6]: [total_sum#27, s_state#8, s_county#7, g_state#28, g_county#29, lochierarchy#30] Keys [6]: [total_sum#27, s_state#8, s_county#7, g_state#28, g_county#29, lochierarchy#30] Functions: [] diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a/simplified.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a/simplified.txt index 07d935f23b4..f4b2f2aa2c5 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a/simplified.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a/simplified.txt @@ -11,7 +11,7 @@ VeloxColumnarToRow WholeStageCodegenTransformer (33) ProjectExecTransformer [lochierarchy,_w0,total_sum,s_state,s_county] ProjectExecTransformer [total_sum,s_state,s_county,lochierarchy,g_county] - RegularHashAggregateExecTransformer [total_sum,s_state,s_county,g_state,g_county,lochierarchy] + FlushableHashAggregateExecTransformer [total_sum,s_state,s_county,g_state,g_county,lochierarchy] InputIteratorTransformer InputAdapter ColumnarExchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2 diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a.sf100/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a.sf100/explain.txt index 24ec6ab60ba..eaa13679e1b 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a.sf100/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a.sf100/explain.txt @@ -9,7 +9,7 @@ VeloxColumnarToRow (72) +- VeloxResizeBatches (63) +- ^ ProjectExecTransformer (61) +- ^ ProjectExecTransformer (60) - +- ^ RegularHashAggregateExecTransformer (59) + +- ^ FlushableHashAggregateExecTransformer (59) +- ^ InputIteratorTransformer (58) +- ColumnarExchange (56) +- VeloxResizeBatches (55) @@ -302,7 +302,7 @@ Input [6]: [total_sum#14, i_category#8, i_class#7, g_category#15, g_class#16, lo (58) InputIteratorTransformer Input [6]: [total_sum#14, i_category#8, i_class#7, g_category#15, g_class#16, lochierarchy#17] -(59) RegularHashAggregateExecTransformer +(59) FlushableHashAggregateExecTransformer Input [6]: [total_sum#14, i_category#8, i_class#7, g_category#15, g_class#16, lochierarchy#17] Keys [6]: [total_sum#14, i_category#8, i_class#7, g_category#15, g_class#16, lochierarchy#17] Functions: [] diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a.sf100/simplified.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a.sf100/simplified.txt index 2acb03c77a1..92c4138a099 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a.sf100/simplified.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a.sf100/simplified.txt @@ -11,7 +11,7 @@ VeloxColumnarToRow WholeStageCodegenTransformer (18) ProjectExecTransformer [lochierarchy,_w0,total_sum,i_category,i_class] ProjectExecTransformer [total_sum,i_category,i_class,lochierarchy,g_class] - RegularHashAggregateExecTransformer [total_sum,i_category,i_class,g_category,g_class,lochierarchy] + FlushableHashAggregateExecTransformer [total_sum,i_category,i_class,g_category,g_class,lochierarchy] InputIteratorTransformer InputAdapter ColumnarExchange [total_sum,i_category,i_class,g_category,g_class,lochierarchy] #2 diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a/explain.txt index 24ec6ab60ba..eaa13679e1b 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a/explain.txt @@ -9,7 +9,7 @@ VeloxColumnarToRow (72) +- VeloxResizeBatches (63) +- ^ ProjectExecTransformer (61) +- ^ ProjectExecTransformer (60) - +- ^ RegularHashAggregateExecTransformer (59) + +- ^ FlushableHashAggregateExecTransformer (59) +- ^ InputIteratorTransformer (58) +- ColumnarExchange (56) +- VeloxResizeBatches (55) @@ -302,7 +302,7 @@ Input [6]: [total_sum#14, i_category#8, i_class#7, g_category#15, g_class#16, lo (58) InputIteratorTransformer Input [6]: [total_sum#14, i_category#8, i_class#7, g_category#15, g_class#16, lochierarchy#17] -(59) RegularHashAggregateExecTransformer +(59) FlushableHashAggregateExecTransformer Input [6]: [total_sum#14, i_category#8, i_class#7, g_category#15, g_class#16, lochierarchy#17] Keys [6]: [total_sum#14, i_category#8, i_class#7, g_category#15, g_class#16, lochierarchy#17] Functions: [] diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a/simplified.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a/simplified.txt index 2acb03c77a1..92c4138a099 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a/simplified.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a/simplified.txt @@ -11,7 +11,7 @@ VeloxColumnarToRow WholeStageCodegenTransformer (18) ProjectExecTransformer [lochierarchy,_w0,total_sum,i_category,i_class] ProjectExecTransformer [total_sum,i_category,i_class,lochierarchy,g_class] - RegularHashAggregateExecTransformer [total_sum,i_category,i_class,g_category,g_class,lochierarchy] + FlushableHashAggregateExecTransformer [total_sum,i_category,i_class,g_category,g_class,lochierarchy] InputIteratorTransformer InputAdapter ColumnarExchange [total_sum,i_category,i_class,g_category,g_class,lochierarchy] #2 diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14a.sf100/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14a.sf100/explain.txt index 16e504bc530..623a054d45b 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14a.sf100/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14a.sf100/explain.txt @@ -39,7 +39,7 @@ VeloxColumnarToRow (187) : : : : +- ColumnarExchange (58) : : : : +- VeloxResizeBatches (57) : : : : +- ^ ProjectExecTransformer (55) - : : : : +- ^ RegularHashAggregateExecTransformer (54) + : : : : +- ^ FlushableHashAggregateExecTransformer (54) : : : : +- ^ InputIteratorTransformer (53) : : : : +- ColumnarExchange (51) : : : : +- VeloxResizeBatches (50) @@ -373,7 +373,7 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] (53) InputIteratorTransformer Input [3]: [brand_id#27, class_id#28, category_id#29] -(54) RegularHashAggregateExecTransformer +(54) FlushableHashAggregateExecTransformer Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14a.sf100/simplified.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14a.sf100/simplified.txt index f84f9ea6feb..e94b37c1e46 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14a.sf100/simplified.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14a.sf100/simplified.txt @@ -95,7 +95,7 @@ VeloxColumnarToRow VeloxResizeBatches WholeStageCodegenTransformer (22) ProjectExecTransformer [brand_id,class_id,category_id] - RegularHashAggregateExecTransformer [brand_id,class_id,category_id] + FlushableHashAggregateExecTransformer [brand_id,class_id,category_id] InputIteratorTransformer InputAdapter ColumnarExchange [brand_id,class_id,category_id] #7 diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14b.sf100/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14b.sf100/explain.txt index c2761a85a07..c49e2b293cd 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14b.sf100/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14b.sf100/explain.txt @@ -30,7 +30,7 @@ VeloxColumnarToRow (150) : : : : +- ColumnarExchange (58) : : : : +- VeloxResizeBatches (57) : : : : +- ^ ProjectExecTransformer (55) - : : : : +- ^ RegularHashAggregateExecTransformer (54) + : : : : +- ^ FlushableHashAggregateExecTransformer (54) : : : : +- ^ InputIteratorTransformer (53) : : : : +- ColumnarExchange (51) : : : : +- VeloxResizeBatches (50) @@ -345,7 +345,7 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] (53) InputIteratorTransformer Input [3]: [brand_id#27, class_id#28, category_id#29] -(54) RegularHashAggregateExecTransformer +(54) FlushableHashAggregateExecTransformer Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14b.sf100/simplified.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14b.sf100/simplified.txt index 670b1d15ce2..8de7ae6341b 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14b.sf100/simplified.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q14b.sf100/simplified.txt @@ -90,7 +90,7 @@ VeloxColumnarToRow VeloxResizeBatches WholeStageCodegenTransformer (24) ProjectExecTransformer [brand_id,class_id,category_id] - RegularHashAggregateExecTransformer [brand_id,class_id,category_id] + FlushableHashAggregateExecTransformer [brand_id,class_id,category_id] InputIteratorTransformer InputAdapter ColumnarExchange [brand_id,class_id,category_id] #6 diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q38.sf100/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q38.sf100/explain.txt index 7f48666f41d..9dc843bff3a 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q38.sf100/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q38.sf100/explain.txt @@ -8,7 +8,7 @@ VeloxColumnarToRow (104) : : +- ColumnarExchange (34) : : +- VeloxResizeBatches (33) : : +- ^ ProjectExecTransformer (31) - : : +- ^ RegularHashAggregateExecTransformer (30) + : : +- ^ FlushableHashAggregateExecTransformer (30) : : +- ^ InputIteratorTransformer (29) : : +- ColumnarExchange (27) : : +- VeloxResizeBatches (26) @@ -35,7 +35,7 @@ VeloxColumnarToRow (104) : +- ColumnarExchange (65) : +- VeloxResizeBatches (64) : +- ^ ProjectExecTransformer (62) - : +- ^ RegularHashAggregateExecTransformer (61) + : +- ^ FlushableHashAggregateExecTransformer (61) : +- ^ InputIteratorTransformer (60) : +- ColumnarExchange (58) : +- VeloxResizeBatches (57) @@ -58,7 +58,7 @@ VeloxColumnarToRow (104) +- ColumnarExchange (97) +- VeloxResizeBatches (96) +- ^ ProjectExecTransformer (94) - +- ^ RegularHashAggregateExecTransformer (93) + +- ^ FlushableHashAggregateExecTransformer (93) +- ^ InputIteratorTransformer (92) +- ColumnarExchange (90) +- VeloxResizeBatches (89) @@ -200,7 +200,7 @@ Input [3]: [c_last_name#9, c_first_name#8, d_date#5] (29) InputIteratorTransformer Input [3]: [c_last_name#9, c_first_name#8, d_date#5] -(30) RegularHashAggregateExecTransformer +(30) FlushableHashAggregateExecTransformer Input [3]: [c_last_name#9, c_first_name#8, d_date#5] Keys [3]: [c_last_name#9, c_first_name#8, d_date#5] Functions: [] @@ -326,7 +326,7 @@ Input [3]: [c_last_name#20, c_first_name#19, d_date#16] (60) InputIteratorTransformer Input [3]: [c_last_name#20, c_first_name#19, d_date#16] -(61) RegularHashAggregateExecTransformer +(61) FlushableHashAggregateExecTransformer Input [3]: [c_last_name#20, c_first_name#19, d_date#16] Keys [3]: [c_last_name#20, c_first_name#19, d_date#16] Functions: [] @@ -458,7 +458,7 @@ Input [3]: [c_last_name#30, c_first_name#29, d_date#26] (92) InputIteratorTransformer Input [3]: [c_last_name#30, c_first_name#29, d_date#26] -(93) RegularHashAggregateExecTransformer +(93) FlushableHashAggregateExecTransformer Input [3]: [c_last_name#30, c_first_name#29, d_date#26] Keys [3]: [c_last_name#30, c_first_name#29, d_date#26] Functions: [] diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q38.sf100/simplified.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q38.sf100/simplified.txt index 54c142449b1..d8b5fd2b904 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q38.sf100/simplified.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q38.sf100/simplified.txt @@ -10,7 +10,7 @@ VeloxColumnarToRow VeloxResizeBatches WholeStageCodegenTransformer (6) ProjectExecTransformer [c_last_name,c_first_name,d_date] - RegularHashAggregateExecTransformer [c_last_name,c_first_name,d_date] + FlushableHashAggregateExecTransformer [c_last_name,c_first_name,d_date] InputIteratorTransformer InputAdapter ColumnarExchange [c_last_name,c_first_name,d_date] #2 @@ -52,7 +52,7 @@ VeloxColumnarToRow VeloxResizeBatches WholeStageCodegenTransformer (12) ProjectExecTransformer [c_last_name,c_first_name,d_date] - RegularHashAggregateExecTransformer [c_last_name,c_first_name,d_date] + FlushableHashAggregateExecTransformer [c_last_name,c_first_name,d_date] InputIteratorTransformer InputAdapter ColumnarExchange [c_last_name,c_first_name,d_date] #7 @@ -84,7 +84,7 @@ VeloxColumnarToRow VeloxResizeBatches WholeStageCodegenTransformer (18) ProjectExecTransformer [c_last_name,c_first_name,d_date] - RegularHashAggregateExecTransformer [c_last_name,c_first_name,d_date] + FlushableHashAggregateExecTransformer [c_last_name,c_first_name,d_date] InputIteratorTransformer InputAdapter ColumnarExchange [c_last_name,c_first_name,d_date] #10 diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q8.sf100/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q8.sf100/explain.txt index 1e5a88f5693..057d344bc68 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q8.sf100/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q8.sf100/explain.txt @@ -29,7 +29,7 @@ VeloxColumnarToRow (85) +- ColumnarExchange (69) +- VeloxResizeBatches (68) +- ^ ProjectExecTransformer (66) - +- ^ RegularHashAggregateExecTransformer (65) + +- ^ FlushableHashAggregateExecTransformer (65) +- ^ InputIteratorTransformer (64) +- ColumnarExchange (62) +- VeloxResizeBatches (61) @@ -337,7 +337,7 @@ Input [1]: [ca_zip#11] (64) InputIteratorTransformer Input [1]: [ca_zip#11] -(65) RegularHashAggregateExecTransformer +(65) FlushableHashAggregateExecTransformer Input [1]: [ca_zip#11] Keys [1]: [ca_zip#11] Functions: [] diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q8.sf100/simplified.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q8.sf100/simplified.txt index 77da4dfe6e1..f24a2989dd3 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q8.sf100/simplified.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q8.sf100/simplified.txt @@ -44,7 +44,7 @@ VeloxColumnarToRow VeloxResizeBatches WholeStageCodegenTransformer (10) ProjectExecTransformer [ca_zip] - RegularHashAggregateExecTransformer [ca_zip] + FlushableHashAggregateExecTransformer [ca_zip] InputIteratorTransformer InputAdapter ColumnarExchange [ca_zip] #6 diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q87.sf100/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q87.sf100/explain.txt index 4fc5497a3c5..9573f717fa9 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q87.sf100/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q87.sf100/explain.txt @@ -8,7 +8,7 @@ VeloxColumnarToRow (104) : : +- ColumnarExchange (34) : : +- VeloxResizeBatches (33) : : +- ^ ProjectExecTransformer (31) - : : +- ^ RegularHashAggregateExecTransformer (30) + : : +- ^ FlushableHashAggregateExecTransformer (30) : : +- ^ InputIteratorTransformer (29) : : +- ColumnarExchange (27) : : +- VeloxResizeBatches (26) @@ -35,7 +35,7 @@ VeloxColumnarToRow (104) : +- ColumnarExchange (65) : +- VeloxResizeBatches (64) : +- ^ ProjectExecTransformer (62) - : +- ^ RegularHashAggregateExecTransformer (61) + : +- ^ FlushableHashAggregateExecTransformer (61) : +- ^ InputIteratorTransformer (60) : +- ColumnarExchange (58) : +- VeloxResizeBatches (57) @@ -58,7 +58,7 @@ VeloxColumnarToRow (104) +- ColumnarExchange (97) +- VeloxResizeBatches (96) +- ^ ProjectExecTransformer (94) - +- ^ RegularHashAggregateExecTransformer (93) + +- ^ FlushableHashAggregateExecTransformer (93) +- ^ InputIteratorTransformer (92) +- ColumnarExchange (90) +- VeloxResizeBatches (89) @@ -200,7 +200,7 @@ Input [3]: [c_last_name#9, c_first_name#8, d_date#5] (29) InputIteratorTransformer Input [3]: [c_last_name#9, c_first_name#8, d_date#5] -(30) RegularHashAggregateExecTransformer +(30) FlushableHashAggregateExecTransformer Input [3]: [c_last_name#9, c_first_name#8, d_date#5] Keys [3]: [c_last_name#9, c_first_name#8, d_date#5] Functions: [] @@ -326,7 +326,7 @@ Input [3]: [c_last_name#20, c_first_name#19, d_date#16] (60) InputIteratorTransformer Input [3]: [c_last_name#20, c_first_name#19, d_date#16] -(61) RegularHashAggregateExecTransformer +(61) FlushableHashAggregateExecTransformer Input [3]: [c_last_name#20, c_first_name#19, d_date#16] Keys [3]: [c_last_name#20, c_first_name#19, d_date#16] Functions: [] @@ -458,7 +458,7 @@ Input [3]: [c_last_name#30, c_first_name#29, d_date#26] (92) InputIteratorTransformer Input [3]: [c_last_name#30, c_first_name#29, d_date#26] -(93) RegularHashAggregateExecTransformer +(93) FlushableHashAggregateExecTransformer Input [3]: [c_last_name#30, c_first_name#29, d_date#26] Keys [3]: [c_last_name#30, c_first_name#29, d_date#26] Functions: [] diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q87.sf100/simplified.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q87.sf100/simplified.txt index 54c142449b1..d8b5fd2b904 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q87.sf100/simplified.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q87.sf100/simplified.txt @@ -10,7 +10,7 @@ VeloxColumnarToRow VeloxResizeBatches WholeStageCodegenTransformer (6) ProjectExecTransformer [c_last_name,c_first_name,d_date] - RegularHashAggregateExecTransformer [c_last_name,c_first_name,d_date] + FlushableHashAggregateExecTransformer [c_last_name,c_first_name,d_date] InputIteratorTransformer InputAdapter ColumnarExchange [c_last_name,c_first_name,d_date] #2 @@ -52,7 +52,7 @@ VeloxColumnarToRow VeloxResizeBatches WholeStageCodegenTransformer (12) ProjectExecTransformer [c_last_name,c_first_name,d_date] - RegularHashAggregateExecTransformer [c_last_name,c_first_name,d_date] + FlushableHashAggregateExecTransformer [c_last_name,c_first_name,d_date] InputIteratorTransformer InputAdapter ColumnarExchange [c_last_name,c_first_name,d_date] #7 @@ -84,7 +84,7 @@ VeloxColumnarToRow VeloxResizeBatches WholeStageCodegenTransformer (18) ProjectExecTransformer [c_last_name,c_first_name,d_date] - RegularHashAggregateExecTransformer [c_last_name,c_first_name,d_date] + FlushableHashAggregateExecTransformer [c_last_name,c_first_name,d_date] InputIteratorTransformer InputAdapter ColumnarExchange [c_last_name,c_first_name,d_date] #10 diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14.sf100/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14.sf100/explain.txt index b56686f4da2..14307ccf270 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14.sf100/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14.sf100/explain.txt @@ -30,7 +30,7 @@ VeloxColumnarToRow (150) : : : : +- ColumnarExchange (58) : : : : +- VeloxResizeBatches (57) : : : : +- ^ ProjectExecTransformer (55) - : : : : +- ^ RegularHashAggregateExecTransformer (54) + : : : : +- ^ FlushableHashAggregateExecTransformer (54) : : : : +- ^ InputIteratorTransformer (53) : : : : +- ColumnarExchange (51) : : : : +- VeloxResizeBatches (50) @@ -345,7 +345,7 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] (53) InputIteratorTransformer Input [3]: [brand_id#27, class_id#28, category_id#29] -(54) RegularHashAggregateExecTransformer +(54) FlushableHashAggregateExecTransformer Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14.sf100/simplified.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14.sf100/simplified.txt index 670b1d15ce2..8de7ae6341b 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14.sf100/simplified.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14.sf100/simplified.txt @@ -90,7 +90,7 @@ VeloxColumnarToRow VeloxResizeBatches WholeStageCodegenTransformer (24) ProjectExecTransformer [brand_id,class_id,category_id] - RegularHashAggregateExecTransformer [brand_id,class_id,category_id] + FlushableHashAggregateExecTransformer [brand_id,class_id,category_id] InputIteratorTransformer InputAdapter ColumnarExchange [brand_id,class_id,category_id] #6 diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14a.sf100/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14a.sf100/explain.txt index b98f1c93a8d..19a274494ec 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14a.sf100/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14a.sf100/explain.txt @@ -40,7 +40,7 @@ VeloxColumnarToRow (327) : : : : : +- ColumnarExchange (58) : : : : : +- VeloxResizeBatches (57) : : : : : +- ^ ProjectExecTransformer (55) - : : : : : +- ^ RegularHashAggregateExecTransformer (54) + : : : : : +- ^ FlushableHashAggregateExecTransformer (54) : : : : : +- ^ InputIteratorTransformer (53) : : : : : +- ColumnarExchange (51) : : : : : +- VeloxResizeBatches (50) @@ -473,7 +473,7 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] (53) InputIteratorTransformer Input [3]: [brand_id#27, class_id#28, category_id#29] -(54) RegularHashAggregateExecTransformer +(54) FlushableHashAggregateExecTransformer Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14a.sf100/simplified.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14a.sf100/simplified.txt index 655299facf8..6b4558ed20c 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14a.sf100/simplified.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q14a.sf100/simplified.txt @@ -103,7 +103,7 @@ VeloxColumnarToRow VeloxResizeBatches WholeStageCodegenTransformer (22) ProjectExecTransformer [brand_id,class_id,category_id] - RegularHashAggregateExecTransformer [brand_id,class_id,category_id] + FlushableHashAggregateExecTransformer [brand_id,class_id,category_id] InputIteratorTransformer InputAdapter ColumnarExchange [brand_id,class_id,category_id] #7 diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a.sf100/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a.sf100/explain.txt index 2a5c7113d0b..d248f624126 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a.sf100/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a.sf100/explain.txt @@ -9,7 +9,7 @@ VeloxColumnarToRow (81) +- VeloxResizeBatches (72) +- ^ ProjectExecTransformer (70) +- ^ ProjectExecTransformer (69) - +- ^ RegularHashAggregateExecTransformer (68) + +- ^ FlushableHashAggregateExecTransformer (68) +- ^ InputIteratorTransformer (67) +- ColumnarExchange (65) +- VeloxResizeBatches (64) @@ -348,7 +348,7 @@ Input [6]: [gross_margin#22, i_category#12, i_class#11, t_category#23, t_class#2 (67) InputIteratorTransformer Input [6]: [gross_margin#22, i_category#12, i_class#11, t_category#23, t_class#24, lochierarchy#25] -(68) RegularHashAggregateExecTransformer +(68) FlushableHashAggregateExecTransformer Input [6]: [gross_margin#22, i_category#12, i_class#11, t_category#23, t_class#24, lochierarchy#25] Keys [6]: [gross_margin#22, i_category#12, i_class#11, t_category#23, t_class#24, lochierarchy#25] Functions: [] diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a.sf100/simplified.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a.sf100/simplified.txt index 0129209b62e..e643c62aeed 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a.sf100/simplified.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a.sf100/simplified.txt @@ -11,7 +11,7 @@ VeloxColumnarToRow WholeStageCodegenTransformer (21) ProjectExecTransformer [lochierarchy,_w0,gross_margin,i_category,i_class] ProjectExecTransformer [gross_margin,i_category,i_class,lochierarchy,t_class] - RegularHashAggregateExecTransformer [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] + FlushableHashAggregateExecTransformer [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] InputIteratorTransformer InputAdapter ColumnarExchange [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] #2 diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a/explain.txt index 9fad6bd36fc..de6e3285a98 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a/explain.txt @@ -9,7 +9,7 @@ VeloxColumnarToRow (81) +- VeloxResizeBatches (72) +- ^ ProjectExecTransformer (70) +- ^ ProjectExecTransformer (69) - +- ^ RegularHashAggregateExecTransformer (68) + +- ^ FlushableHashAggregateExecTransformer (68) +- ^ InputIteratorTransformer (67) +- ColumnarExchange (65) +- VeloxResizeBatches (64) @@ -348,7 +348,7 @@ Input [6]: [gross_margin#22, i_category#10, i_class#9, t_category#23, t_class#24 (67) InputIteratorTransformer Input [6]: [gross_margin#22, i_category#10, i_class#9, t_category#23, t_class#24, lochierarchy#25] -(68) RegularHashAggregateExecTransformer +(68) FlushableHashAggregateExecTransformer Input [6]: [gross_margin#22, i_category#10, i_class#9, t_category#23, t_class#24, lochierarchy#25] Keys [6]: [gross_margin#22, i_category#10, i_class#9, t_category#23, t_class#24, lochierarchy#25] Functions: [] diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a/simplified.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a/simplified.txt index 1429b04fd98..d5cc8efaac4 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a/simplified.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q36a/simplified.txt @@ -11,7 +11,7 @@ VeloxColumnarToRow WholeStageCodegenTransformer (21) ProjectExecTransformer [lochierarchy,_w0,gross_margin,i_category,i_class] ProjectExecTransformer [gross_margin,i_category,i_class,lochierarchy,t_class] - RegularHashAggregateExecTransformer [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] + FlushableHashAggregateExecTransformer [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] InputIteratorTransformer InputAdapter ColumnarExchange [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] #2 diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a.sf100/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a.sf100/explain.txt index 8b62712d862..8ef5a372350 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a.sf100/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a.sf100/explain.txt @@ -9,7 +9,7 @@ VeloxColumnarToRow (106) +- VeloxResizeBatches (97) +- ^ ProjectExecTransformer (95) +- ^ ProjectExecTransformer (94) - +- ^ RegularHashAggregateExecTransformer (93) + +- ^ FlushableHashAggregateExecTransformer (93) +- ^ InputIteratorTransformer (92) +- ColumnarExchange (90) +- VeloxResizeBatches (89) @@ -475,7 +475,7 @@ Input [6]: [total_sum#27, s_state#8, s_county#7, g_state#28, g_county#29, lochie (92) InputIteratorTransformer Input [6]: [total_sum#27, s_state#8, s_county#7, g_state#28, g_county#29, lochierarchy#30] -(93) RegularHashAggregateExecTransformer +(93) FlushableHashAggregateExecTransformer Input [6]: [total_sum#27, s_state#8, s_county#7, g_state#28, g_county#29, lochierarchy#30] Keys [6]: [total_sum#27, s_state#8, s_county#7, g_state#28, g_county#29, lochierarchy#30] Functions: [] diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a.sf100/simplified.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a.sf100/simplified.txt index 81dd34e12ba..08237488257 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a.sf100/simplified.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a.sf100/simplified.txt @@ -11,7 +11,7 @@ VeloxColumnarToRow WholeStageCodegenTransformer (33) ProjectExecTransformer [lochierarchy,_w0,total_sum,s_state,s_county] ProjectExecTransformer [total_sum,s_state,s_county,lochierarchy,g_county] - RegularHashAggregateExecTransformer [total_sum,s_state,s_county,g_state,g_county,lochierarchy] + FlushableHashAggregateExecTransformer [total_sum,s_state,s_county,g_state,g_county,lochierarchy] InputIteratorTransformer InputAdapter ColumnarExchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2 diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a/explain.txt index 148593a5123..823a0332a72 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a/explain.txt @@ -9,7 +9,7 @@ VeloxColumnarToRow (106) +- VeloxResizeBatches (97) +- ^ ProjectExecTransformer (95) +- ^ ProjectExecTransformer (94) - +- ^ RegularHashAggregateExecTransformer (93) + +- ^ FlushableHashAggregateExecTransformer (93) +- ^ InputIteratorTransformer (92) +- ColumnarExchange (90) +- VeloxResizeBatches (89) @@ -475,7 +475,7 @@ Input [6]: [total_sum#27, s_state#8, s_county#7, g_state#28, g_county#29, lochie (92) InputIteratorTransformer Input [6]: [total_sum#27, s_state#8, s_county#7, g_state#28, g_county#29, lochierarchy#30] -(93) RegularHashAggregateExecTransformer +(93) FlushableHashAggregateExecTransformer Input [6]: [total_sum#27, s_state#8, s_county#7, g_state#28, g_county#29, lochierarchy#30] Keys [6]: [total_sum#27, s_state#8, s_county#7, g_state#28, g_county#29, lochierarchy#30] Functions: [] diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a/simplified.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a/simplified.txt index 07d935f23b4..f4b2f2aa2c5 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a/simplified.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q70a/simplified.txt @@ -11,7 +11,7 @@ VeloxColumnarToRow WholeStageCodegenTransformer (33) ProjectExecTransformer [lochierarchy,_w0,total_sum,s_state,s_county] ProjectExecTransformer [total_sum,s_state,s_county,lochierarchy,g_county] - RegularHashAggregateExecTransformer [total_sum,s_state,s_county,g_state,g_county,lochierarchy] + FlushableHashAggregateExecTransformer [total_sum,s_state,s_county,g_state,g_county,lochierarchy] InputIteratorTransformer InputAdapter ColumnarExchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2 diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a.sf100/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a.sf100/explain.txt index 24ec6ab60ba..eaa13679e1b 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a.sf100/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a.sf100/explain.txt @@ -9,7 +9,7 @@ VeloxColumnarToRow (72) +- VeloxResizeBatches (63) +- ^ ProjectExecTransformer (61) +- ^ ProjectExecTransformer (60) - +- ^ RegularHashAggregateExecTransformer (59) + +- ^ FlushableHashAggregateExecTransformer (59) +- ^ InputIteratorTransformer (58) +- ColumnarExchange (56) +- VeloxResizeBatches (55) @@ -302,7 +302,7 @@ Input [6]: [total_sum#14, i_category#8, i_class#7, g_category#15, g_class#16, lo (58) InputIteratorTransformer Input [6]: [total_sum#14, i_category#8, i_class#7, g_category#15, g_class#16, lochierarchy#17] -(59) RegularHashAggregateExecTransformer +(59) FlushableHashAggregateExecTransformer Input [6]: [total_sum#14, i_category#8, i_class#7, g_category#15, g_class#16, lochierarchy#17] Keys [6]: [total_sum#14, i_category#8, i_class#7, g_category#15, g_class#16, lochierarchy#17] Functions: [] diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a.sf100/simplified.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a.sf100/simplified.txt index 2acb03c77a1..92c4138a099 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a.sf100/simplified.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a.sf100/simplified.txt @@ -11,7 +11,7 @@ VeloxColumnarToRow WholeStageCodegenTransformer (18) ProjectExecTransformer [lochierarchy,_w0,total_sum,i_category,i_class] ProjectExecTransformer [total_sum,i_category,i_class,lochierarchy,g_class] - RegularHashAggregateExecTransformer [total_sum,i_category,i_class,g_category,g_class,lochierarchy] + FlushableHashAggregateExecTransformer [total_sum,i_category,i_class,g_category,g_class,lochierarchy] InputIteratorTransformer InputAdapter ColumnarExchange [total_sum,i_category,i_class,g_category,g_class,lochierarchy] #2 diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a/explain.txt index 24ec6ab60ba..eaa13679e1b 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a/explain.txt @@ -9,7 +9,7 @@ VeloxColumnarToRow (72) +- VeloxResizeBatches (63) +- ^ ProjectExecTransformer (61) +- ^ ProjectExecTransformer (60) - +- ^ RegularHashAggregateExecTransformer (59) + +- ^ FlushableHashAggregateExecTransformer (59) +- ^ InputIteratorTransformer (58) +- ColumnarExchange (56) +- VeloxResizeBatches (55) @@ -302,7 +302,7 @@ Input [6]: [total_sum#14, i_category#8, i_class#7, g_category#15, g_class#16, lo (58) InputIteratorTransformer Input [6]: [total_sum#14, i_category#8, i_class#7, g_category#15, g_class#16, lochierarchy#17] -(59) RegularHashAggregateExecTransformer +(59) FlushableHashAggregateExecTransformer Input [6]: [total_sum#14, i_category#8, i_class#7, g_category#15, g_class#16, lochierarchy#17] Keys [6]: [total_sum#14, i_category#8, i_class#7, g_category#15, g_class#16, lochierarchy#17] Functions: [] diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a/simplified.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a/simplified.txt index 2acb03c77a1..92c4138a099 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a/simplified.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v2_7/q86a/simplified.txt @@ -11,7 +11,7 @@ VeloxColumnarToRow WholeStageCodegenTransformer (18) ProjectExecTransformer [lochierarchy,_w0,total_sum,i_category,i_class] ProjectExecTransformer [total_sum,i_category,i_class,lochierarchy,g_class] - RegularHashAggregateExecTransformer [total_sum,i_category,i_class,g_category,g_class,lochierarchy] + FlushableHashAggregateExecTransformer [total_sum,i_category,i_class,g_category,g_class,lochierarchy] InputIteratorTransformer InputAdapter ColumnarExchange [total_sum,i_category,i_class,g_category,g_class,lochierarchy] #2 diff --git a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/Parameterized.java b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/Parameterized.java index 80a5cb8ce12..77f26c3f0d8 100644 --- a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/Parameterized.java +++ b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/Parameterized.java @@ -68,8 +68,8 @@ public class Parameterized implements Callable { private String[] excludedDims = new String[0]; private static final Pattern dimPattern1 = - Pattern.compile("([\\w-]+):([^,:]+)((?:,[^=,]+=[^=,]+)*)"); - private static final Pattern dimPattern2 = Pattern.compile("([^,:]+)((?:,[^=,]+=[^=,]+)*)"); + Pattern.compile("([\\w-]+):([^,:=]+)((?:,[^=,]+=[^=,]+)*)"); + private static final Pattern dimPattern2 = Pattern.compile("([^,:=]+)((?:,[^=,]+=[^=,]+)*)"); private static final Pattern excludedDimsPattern = Pattern.compile("[\\w-]+:[^,:]+(?:,[\\w-]+:[^,:]+)*");