Merge branch 'main' into spark-3.3.2

Author: devongleeson

README.md

@@ -119,7 +119,7 @@ To it, simply add
 to the top of your notebook. This will get the latest version of the API, together with the latest version of Spark.
 To define a certain version of Spark or the API itself, simply add it like this:
 ```jupyterpython
-%use spark(spark=3.3.1, scala=2.13, v=1.2.2)
+%use spark(spark=3.3.1, scala=2.13, v=1.2.3)
 ```
 
 Inside the notebook a Spark session will be initiated automatically. This can be accessed via the `spark` value.

buildSrc/src/main/kotlin/Versions.kt

@@ -1,16 +1,16 @@
 object Versions {
-    const val project = "1.2.3"
+    const val project = "1.2.4-SNAPSHOT"
     const val groupID = "org.jetbrains.kotlinx.spark"
     const val kotlin = "1.8.0"
     const val jvmTarget = "8"
-    const val jupyterJvmTarget = "11"
+    const val jupyterJvmTarget = "8"
 
     inline val spark get() = System.getProperty("spark") as String
     inline val scala get() = System.getProperty("scala") as String
     inline val sparkMinor get() = spark.substringBeforeLast('.')
     inline val scalaCompat get() = scala.substringBeforeLast('.')
 
-    const val jupyter = "0.11.0-210"
+    const val jupyter = "0.12.0-32-1"
     const val kotest = "5.5.4"
     const val kotestTestContainers = "1.3.3"
     const val dokka = "1.7.10"

core/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala

@@ -82,15 +82,15 @@ object CatalystTypeConverters {
     final def toCatalyst(@Nullable maybeScalaValue: Any): CatalystType = {
       if (maybeScalaValue == null) {
         null.asInstanceOf[CatalystType]
-      } else if (maybeScalaValue.isInstanceOf[Option[ScalaInputType]]) {
-        val opt = maybeScalaValue.asInstanceOf[Option[ScalaInputType]]
-        if (opt.isDefined) {
-          toCatalystImpl(opt.get)
-        } else {
-          null.asInstanceOf[CatalystType]
-        }
-      } else {
-        toCatalystImpl(maybeScalaValue.asInstanceOf[ScalaInputType])
+      } else maybeScalaValue match {
+        case opt: Option[ScalaInputType] =>
+          if (opt.isDefined) {
+            toCatalystImpl(opt.get)
+          } else {
+            null.asInstanceOf[CatalystType]
+          }
+        case _ =>
+          toCatalystImpl(maybeScalaValue.asInstanceOf[ScalaInputType])
       }
     }
 
@@ -429,10 +429,11 @@ object CatalystTypeConverters {
       // a measurable performance impact. Note that this optimization will be unnecessary if we
       // use code generation to construct Scala Row -> Catalyst Row converters.
       def convert(maybeScalaValue: Any): Any = {
-        if (maybeScalaValue.isInstanceOf[Option[Any]]) {
-          maybeScalaValue.asInstanceOf[Option[Any]].orNull
-        } else {
-          maybeScalaValue
+        maybeScalaValue match {
+          case option: Option[Any] =>
+            option.orNull
+          case _ =>
+            maybeScalaValue
         }
       }
 

jupyter/src/test/kotlin/org/jetbrains/kotlinx/spark/api/jupyter/JupyterTests.kt

@@ -59,6 +59,7 @@ class JupyterTests : ShouldSpec({
                 librariesScanner.addLibrariesFromClassLoader(
                     classLoader = currentClassLoader,
                     host = this,
+                    notebook = notebook,
                     integrationTypeNameRules = listOf(
                         PatternNameAcceptanceRule(
                             acceptsFlag = false,
@@ -341,6 +342,7 @@ class JupyterStreamingTests : ShouldSpec({
                 librariesScanner.addLibrariesFromClassLoader(
                     classLoader = currentClassLoader,
                     host = this,
+                    notebook = notebook,
                     integrationTypeNameRules = listOf(
                         PatternNameAcceptanceRule(
                             acceptsFlag = false,