apparently we can already create encoders from UDTs out of the box instead of just serializing them. Added tests and updated examples

Author: Jolanrensen

examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/MLlib.kt

@@ -61,7 +61,7 @@ private fun KSparkSession.correlation() {
         Vectors.dense(4.0, 5.0, 0.0, 3.0),
         Vectors.dense(6.0, 7.0, 0.0, 8.0),
         Vectors.sparse(4, intArrayOf(0, 3), doubleArrayOf(9.0, 1.0))
-    ).map(::tupleOf)
+    )
 
     val df = data.toDF("features")
 

examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/UdtRegistration.kt

@@ -83,8 +83,12 @@ fun main() = withSpark {
         City("Amsterdam", 1),
         City("Breda", 2),
         City("Oosterhout", 3),
-    ).map(::tupleOf)
+    )
 
-    val ds = items.toDS()
+    val ds = items.map(::tupleOf).toDS()
     ds.showDS()
+
+    // Unlike in Scala, you can also directly encode UDT registered types to a Dataset!
+    val ds2 = items.toDS()
+    ds2.showDS()
 }

kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt

@@ -154,6 +154,8 @@ private fun isSupportedByKotlinClassEncoder(cls: KClass<*>): Boolean =
         cls.isSubclassOf(Iterable::class) -> true
         cls.isSubclassOf(Product::class) -> true
         cls.java.isArray -> true
+        cls.hasAnnotation<SQLUserDefinedType>() -> true
+        UDTRegistration.exists(cls.jvmName) -> true
         else -> false
     }
 

kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UdtTest.kt

@@ -73,6 +73,19 @@ class UdtTest : ShouldSpec({
 
                 ds.collectAsList().single() shouldBe input
             }
+
+            should("Be able to create encoder from UDT too") {
+
+                val input = listOf(
+                    City("Amsterdam", 1),
+                    City("Breda", 2),
+                    City("Oosterhout", 3),
+                )
+
+                val ds = input.toDS()
+
+                ds.collectAsList() shouldBe input
+            }
         }
     }
 })