From cd529cfd7e1faa06eb1239c1ced13998f4221e8e Mon Sep 17 00:00:00 2001 From: Max Gekk Date: Fri, 19 Jun 2026 17:05:00 +0200 Subject: [PATCH 1/2] [SPARK-54203][SQL] Support TimeType in RowToColumnConverter ### What changes were proposed in this pull request? Add a `TimeType` branch (via the abstract `AnyTimeType`) to `RowToColumnConverter.getConverterForType` in `Columnar.scala`, routing TIME columns through the long-backed `LongConverter` path. TIME is stored as nanos-of-day `Long`, and the on/off-heap column vectors already reserve long storage for it, so no vector changes are required. ### Why are the changes needed? Previously `getConverterForType` had no `TimeType` case, so a TIME column fell through to `unsupportedDataTypeError`, blocking row-to-column conversion paths (RowToColumnar transitions, vectorized execution, etc.) for TIME. ### Does this PR introduce any user-facing change? No. It enables an existing code path for the TIME data type. ### How was this patch tested? Added round-trip and null-handling tests for `TimeType` to `RowToColumnConverterSuite`. Ran `build/sbt 'sql/testOnly *RowToColumnConverterSuite'` (all 13 tests pass) and scalastyle on the sql module (0 errors). --- .../apache/spark/sql/execution/Columnar.scala | 3 ++- .../execution/RowToColumnConverterSuite.scala | 24 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/Columnar.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/Columnar.scala index 81fc5d2c1f736..35f62a98ba0f3 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/Columnar.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/Columnar.scala @@ -265,7 +265,8 @@ private object RowToColumnConverter { case ShortType => ShortConverter case IntegerType | DateType | _: YearMonthIntervalType => IntConverter case FloatType => FloatConverter - case LongType | TimestampType | TimestampNTZType | _: DayTimeIntervalType => LongConverter + case LongType | TimestampType | TimestampNTZType | _: DayTimeIntervalType | _: AnyTimeType => + LongConverter case DoubleType => DoubleConverter case StringType => StringConverter case _: GeographyType | _: GeometryType => BinaryViewConverter diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/RowToColumnConverterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/RowToColumnConverterSuite.scala index 96d7fb6fbd094..d1d8e89de3912 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/RowToColumnConverterSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/RowToColumnConverterSuite.scala @@ -184,6 +184,30 @@ class RowToColumnConverterSuite extends SparkFunSuite { assert(vectors.head.getTimestampLTZNanos(2) === TimestampNanosVal.fromParts(200L, 1.toShort)) } + test("TimeType column roundtrip") { + Seq(0, 3, 6).foreach { precision => + val schema = StructType(Seq(StructField("t", TimeType(precision)))) + val values = Seq(0L, 12L * 60 * 60 * 1000 * 1000 * 1000, 86399999999999L) + val rows = values.map(v => InternalRow(v)) + val vectors = convertRows(rows, schema) + values.zipWithIndex.foreach { case (v, i) => + assert(vectors.head.getLong(i) === v) + } + } + } + + test("TimeType column with nulls") { + val schema = StructType(Seq(StructField("t", TimeType(6), nullable = true))) + val rows = Seq( + InternalRow(0L), + InternalRow(null), + InternalRow(86399999999999L)) + val vectors = convertRows(rows, schema) + assert(vectors.head.getLong(0) === 0L) + assert(vectors.head.isNullAt(1)) + assert(vectors.head.getLong(2) === 86399999999999L) + } + test("multiple columns") { val schema = StructType( Seq(StructField("s", ShortType), StructField("i", IntegerType), StructField("l", LongType))) From b33694c970bd16f7788eef47a568ecc9c0de259b Mon Sep 17 00:00:00 2001 From: Max Gekk Date: Fri, 19 Jun 2026 18:18:24 +0200 Subject: [PATCH 2/2] [SPARK-54203][SQL] Match concrete TimeType in RowToColumnConverter dispatch Use `_: TimeType` instead of the abstract `_: AnyTimeType` in the long-backed branch of `getConverterForType`, matching every peer long-backed dispatch site and keeping a future non-long-backed time subtype on the explicit `unsupportedDataTypeError` path. Co-authored-by: Isaac --- .../main/scala/org/apache/spark/sql/execution/Columnar.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/Columnar.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/Columnar.scala index 35f62a98ba0f3..5c98e36853273 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/Columnar.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/Columnar.scala @@ -265,7 +265,7 @@ private object RowToColumnConverter { case ShortType => ShortConverter case IntegerType | DateType | _: YearMonthIntervalType => IntConverter case FloatType => FloatConverter - case LongType | TimestampType | TimestampNTZType | _: DayTimeIntervalType | _: AnyTimeType => + case LongType | TimestampType | TimestampNTZType | _: DayTimeIntervalType | _: TimeType => LongConverter case DoubleType => DoubleConverter case StringType => StringConverter