From 4a9d538dd08350e7c6e2e24b30cecffef027cc5d Mon Sep 17 00:00:00 2001 From: Mariam-Almesfer Date: Sun, 10 May 2026 13:37:12 +0300 Subject: [PATCH] Enable hour(timestamp_ntz) native execution in Velox backend --- .../DateFunctionsValidateSuite.scala | 7 ++----- .../columnar/validator/Validators.scala | 19 +++++++++++++++++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala index e5e25c88b6f..47b3e5b692e 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala @@ -599,12 +599,9 @@ class DateFunctionsValidateSuite extends FunctionsValidateSuite { checkGlutenPlan[BatchScanExecTransformer] } - // Ensures the fallback of unsupported function works. + // hour(timestamp_ntz) runs natively; output is int (no NTZ propagation). runQueryAndCompare("select hour(ts) from view") { - df => - assert(collect(df.queryExecution.executedPlan) { - case p if p.isInstanceOf[ProjectExec] => p - }.nonEmpty) + checkGlutenPlan[ProjectExecTransformer] } } } diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala index 3218e045f54..1290527a5b3 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala @@ -262,9 +262,24 @@ object Validators { case p if HiveTableScanExecTransformer.isHiveTableScan(p) => true case _ => false } - val hasNTZ = plan.output.exists(a => containsNTZ(a.dataType)) || + def ntzOnlyFromFileScans(p: SparkPlan): Boolean = { + if (!p.output.exists(a => containsNTZ(a.dataType))) { + true + } else { + p match { + case _: BatchScanExec => true + case _: FileSourceScanExec => true + case q if HiveTableScanExecTransformer.isHiveTableScan(q) => true + case _ if p.children.isEmpty => false + case _ => p.children.forall(ntzOnlyFromFileScans) + } + } + } + val outputHasNTZ = plan.output.exists(a => containsNTZ(a.dataType)) + val writeChildHasNTZ = plan.isInstanceOf[WriteFilesExec] && plan.children.exists(_.output.exists(a => containsNTZ(a.dataType))) - if (isScan || !hasNTZ) { + val childNTZFromFileScans = plan.children.forall(ntzOnlyFromFileScans) + if (isScan || (!outputHasNTZ && !writeChildHasNTZ && childNTZFromFileScans)) { return pass() } }