Merge branch 'main' into release

Author: asm0dey

.github/workflows/build.yml

@@ -59,7 +59,7 @@ jobs:
             -Pspark=${{ matrix.spark }}
             -Pscala=${{ matrix.scala }}
             clean
-            build
+            test
             --scan
 
   #  qodana:

.github/workflows/publish_dev_version.yml

@@ -57,17 +57,25 @@ jobs:
       - name: Validate Gradle wrapper
         uses: gradle/wrapper-validation-action@v1
 
-      - name: Deploy to GH Packages with Gradle
+      - name: Setup Gradle
         uses: gradle/gradle-build-action@v2
+
+      - name: Set Swap Space
+        uses: pierotofy/set-swap-space@master
         with:
-          arguments: |
-            -Pspark=${{ matrix.spark }}
-            -Pscala=${{ matrix.scala }}
-            -PskipScalaTuplesInKotlin=${{ !(matrix.spark == '3.0.0' || matrix.scala == '2.13.8' && matrix.spark == '3.2.0') }}
-            clean
-            publishMavenPublicationToGitHubPackagesRepository
-            --scan
+          swap-size-gb: 12
+
+      - name: Deploy to GH Packages with Gradle
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: >
+          ./gradlew 
+          -Pspark=${{ matrix.spark }} 
+          -Pscala=${{ matrix.scala }} 
+          -PskipScalaTuplesInKotlin=${{ !(matrix.spark == '3.0.0' || matrix.scala == '2.13.8' && matrix.spark == '3.2.0') }} 
+          clean 
+          publishMavenPublicationToGitHubPackagesRepository 
+          --scan
+        
 
 

.github/workflows/publish_release_version.yml

@@ -0,0 +1,88 @@
+name: Deploy release version to Maven Central
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  build-and-deploy-mvn-central:
+    strategy:
+      matrix:
+        scala: [ "2.12.15", "2.13.8" ]
+        spark: [ "3.3.0", "3.2.1", "3.2.0", "3.1.3", "3.1.2", "3.1.1", "3.1.0", "3.0.3", "3.0.2", "3.0.1", "3.0.0" ]
+        exclude:
+          - scala: "2.13.8"
+            spark: "3.1.3"
+          - scala: "2.13.8"
+            spark: "3.1.2"
+          - scala: "2.13.8"
+            spark: "3.1.1"
+          - scala: "2.13.8"
+            spark: "3.1.0"
+          - scala: "2.13.8"
+            spark: "3.0.3"
+          - scala: "2.13.8"
+            spark: "3.0.2"
+          - scala: "2.13.8"
+            spark: "3.0.1"
+          - scala: "2.13.8"
+            spark: "3.0.0"
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v3
+        with:
+          distribution: adopt
+          java-version: 11
+          check-latest: true
+          server-id: ossrh
+          server-username: ORG_GRADLE_PROJECT_mavenCentralUsername
+          server-password: ORG_GRADLE_PROJECT_mavenCentralPassword
+          gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }}
+          gpg-passphrase: ORG_GRADLE_PROJECT_signingInMemoryKeyPassword
+
+      - name: Cache Gradle packages
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+            ~/.gradle/jdks
+          key: ${{ runner.os }}-gradle-spark-${{ matrix.spark }}-${{ matrix.scala }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+
+      - name: Validate Gradle wrapper
+        uses: gradle/wrapper-validation-action@v1
+
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
+
+      - name: Set Swap Space
+        uses: pierotofy/set-swap-space@master
+        with:
+          swap-size-gb: 12
+
+
+      - name: Deploy to Maven Central with Gradle
+        env:
+          ORG_GRADLE_PROJECT_mavenCentralUsername: ${{ secrets.OSSRH_USERNAME }}
+          ORG_GRADLE_PROJECT_mavenCentralPassword: ${{ secrets.OSSRH_TOKEN }}
+          ORG_GRADLE_PROJECT_signingInMemoryKeyPassword: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
+        run: >
+          ./gradlew 
+          -Pspark=${{ matrix.spark }} 
+          -Pscala=${{ matrix.scala }} 
+          -PskipScalaTuplesInKotlin=${{ !(matrix.spark == '3.0.0' || matrix.scala == '2.13.8' && matrix.spark == '3.2.0') }} 
+          clean 
+          publishMavenPublicationToMavenCentralRepository
+          --scan
+        
+
+

README.md

@@ -2,7 +2,7 @@
 
 [![Kotlin Stable](https://kotl.in/badges/stable.svg)](https://kotlinlang.org/docs/components-stability.html)
 [![JetBrains official project](http://jb.gg/badges/official.svg)](https://confluence.jetbrains.com/display/ALL/JetBrains+on+GitHub)
-[![Maven Central](https://img.shields.io/maven-central/v/org.jetbrains.kotlinx.spark/kotlin-spark-api-parent-3.2.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:org.jetbrains.kotlinx.spark%20AND%20v:1.1.0)
+[![Maven Central](https://img.shields.io/maven-central/v/org.jetbrains.kotlinx.spark/kotlin-spark-api_3.3.0_2.13.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:"org.jetbrains.kotlinx.spark"%20AND%20a:"kotlin-spark-api_3.3.0_2.13")
 [![Join the chat at https://gitter.im/JetBrains/kotlin-spark-api](https://badges.gitter.im/JetBrains/kotlin-spark-api.svg)](https://gitter.im/JetBrains/kotlin-spark-api?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 Your next API to work with  [Apache Spark](https://spark.apache.org/). 
@@ -62,9 +62,12 @@ We have opened a Spark Project Improvement Proposal: [Kotlin support for Apache
 
 The list of Kotlin for Apache Spark releases is available [here](https://github.com/JetBrains/kotlin-spark-api/releases/).
 The Kotlin for Spark artifacts adhere to the following convention:
-`[Apache Spark version]_[Scala core version]:[Kotlin for Apache Spark API version]` 
+`[name]_[Apache Spark version]_[Scala core version]:[Kotlin for Apache Spark API version]` 
 
-[![Maven Central](https://img.shields.io/maven-central/v/org.jetbrains.kotlinx.spark/kotlin-spark-api-parent-3.2.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:"org.jetbrains.kotlinx.spark"%20AND%20a:"kotlin-spark-api-3.2")
+The only exception to this is `scala-tuples-in-kotlin_[Scala core version]:[Kotlin for Apache Spark API version]`, which is 
+independent of Spark.
+
+[![Maven Central](https://img.shields.io/maven-central/v/org.jetbrains.kotlinx.spark/kotlin-spark-api_3.3.0_2.13.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:"org.jetbrains.kotlinx.spark"%20AND%20a:"kotlin-spark-api_3.3.0_2.13")
 
 ## How to configure Kotlin for Apache Spark in your project
 
@@ -360,7 +363,7 @@ Of the [main](https://github.com/Kotlin/kotlin-spark-api/tree/main) branch, deve
 [GitHub Packages](https://github.com/orgs/Kotlin/packages?tab=packages&q=kotlin-spark-api). This way, new features can be
 tested quickly without having to wait for a full release. 
 
-For full releases, the `release` branch is updated.
+For full releases, the [release](https://github.com/Kotlin/kotlin-spark-api/tree/release) branch is updated.
 
 ## Code of Conduct
 This project and the corresponding community is governed by the [JetBrains Open Source and Community Code of Conduct](https://confluence.jetbrains.com/display/ALL/JetBrains+Open+Source+and+Community+Code+of+Conduct). Please make sure you read it. 

build.gradle.kts

@@ -15,6 +15,7 @@ plugins {
     mavenPublish version Versions.mavenPublish
     dokka version Versions.dokka
     idea
+    kotlin version Versions.kotlin apply false
 }
 
 group = Versions.groupID
@@ -58,6 +59,12 @@ allprojects {
             // ORG_GRADLE_PROJECT_mavenCentralUsername and
             // ORG_GRADLE_PROJECT_mavenCentralPassword.
 
+            // also ORG_GRADLE_PROJECT_signingInMemoryKey=exported_ascii_armored_key
+            // # optional
+            // ORG_GRADLE_PROJECT_signingInMemoryKeyId=24875D73
+            // # if key was created with a password
+            // ORG_GRADLE_PROJECT_signingInMemoryKeyPassword=secret
+
             signAllPublications()
             pom {
                 name.set("Kotlin Spark API")
@@ -101,12 +108,11 @@ allprojects {
                 }
 
                 scm {
-                    connection.set("scm:git:https://github.com/JetBrains/kotlin-spark-api.git")
-                    url.set("https://github.com/JetBrains/kotlin-spark-api")
+                    connection.set("scm:git:https://github.com/Kotlin/kotlin-spark-api.git")
+                    url.set("https://github.com/Kotlin/kotlin-spark-api")
                     tag.set("HEAD")
                 }
             }
-
         }
     }
 }

buildSrc/src/main/kotlin/Plugins.kt

@@ -4,7 +4,7 @@ import org.gradle.plugin.use.PluginDependenciesSpec
 
 
 inline val PluginDependenciesSpec.kotlin
-    get() = kotlin("jvm") version Versions.kotlin
+    get() = kotlin("jvm")
 
 inline val PluginDependenciesSpec.dokka
     get() = id("org.jetbrains.dokka")
@@ -15,7 +15,6 @@ inline val PluginDependenciesSpec.license
 inline val PluginDependenciesSpec.jcp
     get() = id("com.igormaznitsa.jcp")
 
-
 inline val DependencyHandlerScope.jcp
     get() = "com.igormaznitsa:jcp:${Versions.jcp}"
 

buildSrc/src/main/kotlin/Versions.kt

@@ -2,20 +2,20 @@ object Versions {
     const val project = "1.1.1"
     const val groupID = "org.jetbrains.kotlinx.spark"
     const val kotlin = "1.7.10"
+    const val jvmTarget = "11"
 
     inline val spark get() = System.getProperty("spark") as String
     inline val scala get() = System.getProperty("scala") as String
     inline val sparkMinor get() = spark.substringBeforeLast('.')
     inline val scalaCompat get() = scala.substringBeforeLast('.')
 
-    const val jupyter = "0.11.0-95"
+    const val jupyter = "0.11.0-134"
     const val kotest = "5.3.2"
     const val kotestTestContainers = "1.3.3"
-    const val dokka = "1.7.0"
+    const val dokka = "1.7.10"
     const val jcp = "7.0.5"
     const val mavenPublish = "0.20.0"
     const val atrium = "0.17.0"
-    const val kotestExtensionAllure = "1.1.0"
     const val licenseGradlePluginVersion = "0.15.0"
     const val kafkaStreamsTestUtils = "3.1.0"
     const val hadoop = "3.3.1"
@@ -29,6 +29,7 @@ object Versions {
             "scalaCompat" to scalaCompat,
             "spark" to spark,
             "sparkMinor" to sparkMinor,
+            "version" to project,
         )
 
 }

core/build.gradle.kts

@@ -1,8 +1,7 @@
-@file:Suppress("UnstableApiUsage")
+@file:Suppress("UnstableApiUsage", "NOTHING_TO_INLINE")
 
 import com.igormaznitsa.jcp.gradle.JcpTask
 import com.vanniktech.maven.publish.JavaLibrary
-import com.vanniktech.maven.publish.JavadocJar
 import com.vanniktech.maven.publish.JavadocJar.Javadoc
 
 plugins {
@@ -48,6 +47,7 @@ tasks.compileScala {
     outputs.upToDateWhen {
         preprocessMain.outcomingFiles.files.isEmpty()
     }
+
     doFirst {
         scala {
             sourceSets {

core/src/main/scala/org/jetbrains/kotlinx/spark/extensions/KSparkExtensions.scala

@@ -21,9 +21,8 @@ package org.jetbrains.kotlinx.spark.extensions
 
 import org.apache.spark.SparkContext
 import org.apache.spark.sql._
-
 import java.util
-import scala.collection.JavaConverters
+import scala.reflect.ClassTag
 
 object KSparkExtensions {
 
@@ -39,7 +38,13 @@ object KSparkExtensions {
 
   def lit(literal: Any): Column = functions.lit(literal)
 
-  def collectAsList[T](ds: Dataset[T]): util.List[T] = JavaConverters.seqAsJavaList(ds.collect())
+  def collectAsList[T](ds: Dataset[T]): util.List[T] = {
+    //#if scalaCompat >= 2.13
+    scala.jdk.javaapi.CollectionConverters.asJava(ds.collect())
+    //#else
+    //$scala.collection.JavaConverters.seqAsJavaList(ds.collect())
+    //#endif
+  }
 
 
   def debugCodegen(df: Dataset[_]): Unit = {
@@ -53,4 +58,17 @@ object KSparkExtensions {
   }
 
   def sparkContext(s: SparkSession): SparkContext = s.sparkContext
+
+  /**
+   * Produces a ClassTag[T], which is actually just a casted ClassTag[AnyRef].
+   *
+   * This method is used to keep ClassTags out of the external Java API, as the Java compiler
+   * cannot produce them automatically. While this ClassTag-faking does please the compiler,
+   * it can cause problems at runtime if the Scala API relies on ClassTags for correctness.
+   *
+   * Often, though, a ClassTag[AnyRef] will not lead to incorrect behavior, just worse performance
+   * or security issues. For instance, an Array[AnyRef] can hold any type T, but may lose primitive
+   * specialization.
+   */
+  def fakeClassTag[T]: ClassTag[T] = ClassTag.AnyRef.asInstanceOf[ClassTag[T]]
 }

examples/build.gradle.kts

@@ -1,3 +1,5 @@
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+
 plugins {
     kotlin
     idea
@@ -28,3 +30,9 @@ dependencies {
 
     }
 }
+
+tasks.withType<KotlinCompile> {
+    kotlinOptions {
+        jvmTarget = Versions.jvmTarget
+    }
+}