Skip to content

Commit 52398d6

Browse files
JolanrensenJolanrensen
committed
split off jupyter into a separate module
1 parent 2b6aca4 commit 52398d6

File tree

11 files changed

+163
-49
lines changed

11 files changed

+163
-49
lines changed

‎jupyter/pom.xml‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
6+
<modelVersion>4.0.0</modelVersion>
7+
8+
<name>Kotlin Spark API: Jupyter integration for Spark 3.2+ (Scala 2.12)</name>
9+
<artifactId>kotlin-spark-api-jupyter-3.2</artifactId>
10+
<description>Jupyter integration</description>
11+
<parent>
12+
<groupId>org.jetbrains.kotlinx.spark</groupId>
13+
<artifactId>kotlin-spark-api-parent_2.12</artifactId>
14+
<version>1.0.4-SNAPSHOT</version>
15+
<relativePath>../pom_2.12.xml</relativePath>
16+
</parent>
17+
<packaging>jar</packaging>
18+
19+
20+
<properties>
21+
<maven.compiler.source>11</maven.compiler.source>
22+
<maven.compiler.target>11</maven.compiler.target>
23+
</properties>
24+
25+
<repositories>
26+
<repository>
27+
<id>kotlinx-html</id>
28+
<name>kotlinx-html</name>
29+
<url>https://maven.pkg.jetbrains.space/public/p/kotlinx-html/maven</url>
30+
</repository>
31+
<repository>
32+
<id>kotlin</id>
33+
<name>kotlin</name>
34+
<url>https://maven.pkg.jetbrains.space/kotlin/p/kotlin/dev</url>
35+
</repository>
36+
</repositories>
37+
38+
<dependencies>
39+
<dependency>
40+
<groupId>org.jetbrains.kotlinx.spark</groupId>
41+
<artifactId>kotlin-spark-api-3.2</artifactId>
42+
<version>${project.version}</version>
43+
</dependency>
44+
<dependency>
45+
<groupId>org.jetbrains.kotlinx</groupId>
46+
<artifactId>kotlinx-html-jvm</artifactId>
47+
<version>${kotlinx.html.version}</version>
48+
</dependency>
49+
<dependency> <!-- Needed for Jupyter -->
50+
<groupId>org.apache.spark</groupId>
51+
<artifactId>spark-repl_${scala.compat.version}</artifactId>
52+
<version>${spark3.version}</version>
53+
</dependency>
54+
<dependency>
55+
<groupId>org.jetbrains.kotlinx</groupId>
56+
<artifactId>kotlin-jupyter-api</artifactId>
57+
<version>${kotlin-jupyter-api.version}</version>
58+
</dependency>
59+
60+
<!-- Testing -->
61+
<dependency>
62+
<groupId>io.kotest</groupId>
63+
<artifactId>kotest-runner-junit5-jvm</artifactId>
64+
<version>${kotest.version}</version>
65+
<scope>test</scope>
66+
</dependency>
67+
<dependency>
68+
<groupId>io.kotest.extensions</groupId>
69+
<artifactId>kotest-extensions-allure</artifactId>
70+
<version>${kotest-extensions-allure.version}</version>
71+
<scope>test</scope>
72+
</dependency>
73+
<dependency>
74+
<groupId>org.jetbrains.kotlinx</groupId>
75+
<artifactId>kotlin-jupyter-test-kit</artifactId>
76+
<version>${kotlin-jupyter-api.version}</version>
77+
<scope>test</scope>
78+
</dependency>
79+
</dependencies>
80+
81+
<build>
82+
<sourceDirectory>src/main/kotlin</sourceDirectory>
83+
<testSourceDirectory>src/test/kotlin</testSourceDirectory>
84+
<directory>target/${scala.compat.version}</directory>
85+
<plugins>
86+
<plugin>
87+
<groupId>org.jetbrains.kotlin</groupId>
88+
<artifactId>kotlin-maven-plugin</artifactId>
89+
<executions>
90+
<execution>
91+
<id>compile</id>
92+
<goals>
93+
<goal>compile</goal>
94+
</goals>
95+
</execution>
96+
<execution>
97+
<id>test-compile</id>
98+
<goals>
99+
<goal>test-compile</goal>
100+
</goals>
101+
</execution>
102+
</executions>
103+
</plugin>
104+
<plugin>
105+
<groupId>org.apache.maven.plugins</groupId>
106+
<artifactId>maven-assembly-plugin</artifactId>
107+
<version>${maven-assembly-plugin.version}</version>
108+
<configuration>
109+
<descriptorRefs>
110+
<descriptorRef>jar-with-dependencies</descriptorRef>
111+
</descriptorRefs>
112+
<archive>
113+
<manifest>
114+
<mainClass>org.jetbrains.spark.api.examples.WordCountKt</mainClass>
115+
</manifest>
116+
</archive>
117+
</configuration>
118+
</plugin>
119+
<plugin>
120+
<groupId>org.apache.maven.plugins</groupId>
121+
<artifactId>maven-site-plugin</artifactId>
122+
<configuration>
123+
<skip>true</skip>
124+
</configuration>
125+
</plugin>
126+
<plugin>
127+
<groupId>org.apache.maven.plugins</groupId>
128+
<artifactId>maven-deploy-plugin</artifactId>
129+
<configuration>
130+
<skip>false</skip>
131+
</configuration>
132+
</plugin>
133+
<plugin>
134+
<groupId>org.sonatype.plugins</groupId>
135+
<artifactId>nexus-staging-maven-plugin</artifactId>
136+
<configuration>
137+
<skipNexusStagingDeployMojo>false</skipNexusStagingDeployMojo>
138+
</configuration>
139+
</plugin>
140+
<plugin>
141+
<groupId>org.apache.maven.plugins</groupId>
142+
<artifactId>maven-compiler-plugin</artifactId>
143+
<version>${maven-compiler-plugin.version}</version>
144+
<configuration>
145+
<source>9</source>
146+
<target>9</target>
147+
</configuration>
148+
</plugin>
149+
</plugins>
150+
</build>
151+
</project>

kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/jupyter/HtmlRendering.kt renamed to jupyter/src/main/kotlin/org/jetbrains/kotlinx/spark/api/jupyter/HtmlRendering.kt

File renamed without changes.

kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/jupyter/Integration.kt renamed to jupyter/src/main/kotlin/org/jetbrains/kotlinx/spark/api/jupyter/Integration.kt

File renamed without changes.

kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/jupyter/SparkIntegration.kt renamed to jupyter/src/main/kotlin/org/jetbrains/kotlinx/spark/api/jupyter/SparkIntegration.kt

File renamed without changes.

kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/jupyter/SparkStreamingIntegration.kt renamed to jupyter/src/main/kotlin/org/jetbrains/kotlinx/spark/api/jupyter/SparkStreamingIntegration.kt

File renamed without changes.

kotlin-spark-api/3.2/src/main/resources/META-INF/kotlin-jupyter-libraries/libraries.json renamed to jupyter/src/main/resources/META-INF/kotlin-jupyter-libraries/libraries.json

File renamed without changes.

kotlin-spark-api/3.2/src/main/resources/table.css renamed to jupyter/src/main/resources/table.css

File renamed without changes.

kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/JupyterTests.kt renamed to jupyter/src/test/kotlin/org/jetbrains/kotlinx/spark/api/jupyter/JupyterTests.kt

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
* limitations under the License.
1818
* =LICENSEEND=
1919
*/
20-
package org.jetbrains.kotlinx.spark.api
20+
package org.jetbrains.kotlinx.spark.api.jupyter
2121

2222
import io.kotest.assertions.throwables.shouldThrowAny
2323
import io.kotest.core.spec.style.ShouldSpec
@@ -40,9 +40,10 @@ import org.jetbrains.kotlinx.jupyter.libraries.EmptyResolutionInfoProvider
4040
import org.jetbrains.kotlinx.jupyter.repl.EvalResultEx
4141
import org.jetbrains.kotlinx.jupyter.testkit.ReplProvider
4242
import org.jetbrains.kotlinx.jupyter.util.PatternNameAcceptanceRule
43-
import org.jetbrains.kotlinx.spark.api.tuples.X
44-
import org.jetbrains.kotlinx.spark.api.tuples.component1
45-
import org.jetbrains.kotlinx.spark.api.tuples.component2
43+
import org.jetbrains.kotlinx.spark.api.tuples.*
44+
import org.jetbrains.kotlinx.spark.api.*
45+
import scala.Tuple2
46+
import java.io.Serializable
4647
import java.util.*
4748
import kotlin.script.experimental.jvm.util.classpathFromClassloader
4849

@@ -151,7 +152,7 @@ class JupyterTests : ShouldSpec({
151152
html shouldContain "4, 5, 6"
152153
}
153154

154-
xshould("not render JavaRDDs with custom class") {
155+
should("render JavaRDDs with custom class") {
155156

156157
@Language("kts")
157158
val klass = exec("""
@@ -173,7 +174,7 @@ class JupyterTests : ShouldSpec({
173174
rdd
174175
""".trimIndent()
175176
)
176-
html shouldContain "Cannot render this RDD of this class."
177+
html shouldContain "Test(longFirstName=aaaaaaaa..."
177178
}
178179

179180
should("render JavaPairRDDs") {
@@ -326,4 +327,6 @@ private fun ReplForJupyter.execHtml(code: Code): String {
326327
val html = res["text/html"]
327328
html.shouldNotBeNull()
328329
return html
329-
}
330+
}
331+
332+
class Counter(@Volatile var value: Int) : Serializable

‎kotlin-spark-api/3.2/pom_2.12.xml‎

Lines changed: 0 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -14,18 +14,6 @@
1414
</parent>
1515
<packaging>jar</packaging>
1616

17-
<repositories>
18-
<repository>
19-
<id>kotlinx-html</id>
20-
<name>kotlinx-html</name>
21-
<url>https://maven.pkg.jetbrains.space/public/p/kotlinx-html/maven</url>
22-
</repository>
23-
<repository>
24-
<id>kotlin</id>
25-
<name>kotlin</name>
26-
<url>https://maven.pkg.jetbrains.space/kotlin/p/kotlin/dev</url>
27-
</repository>
28-
</repositories>
2917

3018
<dependencies>
3119
<dependency>
@@ -49,25 +37,11 @@
4937
<artifactId>spark-streaming-kafka-0-10_${scala.compat.version}</artifactId>
5038
<version>${spark3.version}</version>
5139
</dependency>
52-
<dependency>
53-
<groupId>org.jetbrains.kotlinx</groupId>
54-
<artifactId>kotlinx-html-jvm</artifactId>
55-
<version>${kotlinx.html.version}</version>
56-
</dependency>
57-
<dependency>
58-
<groupId>org.jetbrains.kotlinx</groupId>
59-
<artifactId>kotlin-jupyter-api</artifactId>
60-
</dependency>
6140
<dependency>
6241
<groupId>org.apache.spark</groupId>
6342
<artifactId>spark-sql_${scala.compat.version}</artifactId>
6443
<version>${spark3.version}</version>
6544
</dependency>
66-
<dependency> <!-- Needed for Jupyter -->
67-
<groupId>org.apache.spark</groupId>
68-
<artifactId>spark-repl_${scala.compat.version}</artifactId>
69-
<version>${spark3.version}</version>
70-
</dependency>
7145
<dependency>
7246
<groupId>org.apache.spark</groupId>
7347
<artifactId>spark-streaming_${scala.compat.version}</artifactId>
@@ -124,11 +98,6 @@
12498
<version>3.1.0</version>
12599
<scope>test</scope>
126100
</dependency>
127-
<dependency>
128-
<groupId>org.jetbrains.kotlinx</groupId>
129-
<artifactId>kotlin-jupyter-test-kit</artifactId>
130-
<scope>test</scope>
131-
</dependency>
132101
</dependencies>
133102

134103
<build>

‎pom.xml‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
<embedded-kafka.version>3.1.0</embedded-kafka.version>
1919
<spark3.version>3.2.1</spark3.version>
2020
<kotlin-jupyter-api.version>0.11.0-83</kotlin-jupyter-api.version>
21-
<kotlinx.html.version>0.7.3</kotlinx.html.version>
21+
<kotlinx.html.version>0.7.5</kotlinx.html.version>
2222
<hadoop.version>3.3.1</hadoop.version>
2323

2424
<!-- Plugin versions -->
@@ -56,16 +56,6 @@
5656
<artifactId>kotlin-reflect</artifactId>
5757
<version>${kotlin.version}</version>
5858
</dependency>
59-
<dependency>
60-
<groupId>org.jetbrains.kotlinx</groupId>
61-
<artifactId>kotlin-jupyter-api</artifactId>
62-
<version>${kotlin-jupyter-api.version}</version>
63-
</dependency>
64-
<dependency>
65-
<groupId>org.jetbrains.kotlinx</groupId>
66-
<artifactId>kotlin-jupyter-test-kit</artifactId>
67-
<version>${kotlin-jupyter-api.version}</version>
68-
</dependency>
6959
</dependencies>
7060
</dependencyManagement>
7161

0 commit comments

Comments
 (0)