Skip to content

Commit c1b2344

Browse files
nikolay-egorovnikolay-egorov
committed
Add special type classes in shared-compiler
1 parent 5aebd3d commit c1b2344

File tree

6 files changed

+93
-37
lines changed

6 files changed

+93
-37
lines changed

‎jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/compiler/util/serializedCompiledScript.kt‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,48 @@ data class SerializedCompiledScriptsData(
2020
}
2121
}
2222

23+
@Serializable
24+
data class SerializableTypeInfo(val type: Type = Type.Custom, val isPrimitive: Boolean = false, val fullType: String = "") {
25+
companion object {
26+
val ignoreSet = setOf("int", "double", "boolean", "char", "float", "byte", "string", "entry")
27+
28+
val propertyNamesForNullFilter = setOf("data", "size")
29+
30+
fun makeFromSerializedVariablesState(type: String?, isContainer: Boolean?): SerializableTypeInfo {
31+
val fullType = type.orEmpty()
32+
val enumType = fullType.toTypeEnum()
33+
val isPrimitive = !(
34+
if (fullType != "Entry") (isContainer ?: false)
35+
else true
36+
)
37+
38+
return SerializableTypeInfo(enumType, isPrimitive, fullType)
39+
}
40+
}
41+
}
42+
43+
@Serializable
44+
enum class Type {
45+
Map,
46+
Entry,
47+
Array,
48+
List,
49+
Custom
50+
}
51+
52+
fun String.toTypeEnum(): Type {
53+
return when (this) {
54+
"Map" -> Type.Map
55+
"Entry" -> Type.Entry
56+
"Array" -> Type.Array
57+
"List" -> Type.List
58+
else -> Type.Custom
59+
}
60+
}
61+
2362
@Serializable
2463
data class SerializedVariablesState(
25-
val type: String = "",
64+
val type: SerializableTypeInfo = SerializableTypeInfo(),
2665
val value: String? = null,
2766
val isContainer: Boolean = false,
2867
val stateId: String = ""

‎src/main/kotlin/org/jetbrains/kotlinx/jupyter/apiImpl.kt‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -133,8 +133,9 @@ class NotebookImpl(
133133

134134
private val history = arrayListOf<CodeCellImpl>()
135135
private var mainCellCreated = false
136-
private val unchangedVariables: MutableSet<String> = mutableSetOf()
136+
private val _unchangedVariables: MutableSet<String> = mutableSetOf()
137137

138+
val unchangedVariables: Set<String> get() = _unchangedVariables
138139
val displays = DisplayContainerImpl()
139140

140141
override fun getAllDisplays(): List<DisplayResultWithCell> {
@@ -153,16 +154,14 @@ class NotebookImpl(
153154
fun updateVariablesState(evaluator: InternalEvaluator) {
154155
variablesState += evaluator.variablesHolder
155156
currentCellVariables = evaluator.cellVariables
156-
unchangedVariables.clear()
157-
unchangedVariables.addAll(evaluator.getUnchangedVariables())
157+
_unchangedVariables.clear()
158+
_unchangedVariables.addAll(evaluator.getUnchangedVariables())
158159
}
159160

160161
fun updateVariablesState(varsStateUpdate: Map<String, VariableState>) {
161162
variablesState += varsStateUpdate
162163
}
163164

164-
fun unchangedVariables(): Set<String> = unchangedVariables
165-
166165
fun variablesReportAsHTML(): String {
167166
return generateHTMLVarsReport(variablesState)
168167
}

‎src/main/kotlin/org/jetbrains/kotlinx/jupyter/protocol.kt‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,11 @@ package org.jetbrains.kotlinx.jupyter
22

33
import ch.qos.logback.classic.Level
44
import kotlinx.serialization.json.Json
5+
import kotlinx.serialization.json.JsonElement
6+
import kotlinx.serialization.json.JsonNull
57
import kotlinx.serialization.json.JsonObject
68
import kotlinx.serialization.json.encodeToJsonElement
9+
import kotlinx.serialization.json.jsonObject
710
import org.jetbrains.annotations.TestOnly
811
import org.jetbrains.kotlinx.jupyter.LoggingManagement.disableLogging
912
import org.jetbrains.kotlinx.jupyter.LoggingManagement.mainLoggerLevel
@@ -82,7 +85,6 @@ class OkResponseWithMessage(
8285
)
8386
)
8487
}
85-
8688
socket.send(
8789
makeReplyMessage(
8890
requestMsg,
@@ -92,7 +94,7 @@ class OkResponseWithMessage(
9294
"engine" to Json.encodeToJsonElement(requestMsg.data.header?.session),
9395
"status" to Json.encodeToJsonElement("ok"),
9496
"started" to Json.encodeToJsonElement(startedTime),
95-
"eval_metadata" to Json.encodeToJsonElement(metadata),
97+
"eval_metadata" to Json.encodeToJsonElement(metadata.convertToNullIfEmpty()),
9698
),
9799
content = ExecuteReply(
98100
MessageStatus.OK,
@@ -317,7 +319,7 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup
317319
if (data.isEmpty()) return sendWrapped(msg, makeReplyMessage(msg, MessageType.SERIALIZATION_REPLY))
318320
log.debug("Message data: $data")
319321
val messageContent = getVariablesDescriptorsFromJson(data)
320-
GlobalScope.launch(Dispatchers.Default) {
322+
connection.launchJob {
321323
repl.serializeVariables(
322324
messageContent.topLevelDescriptorName,
323325
messageContent.descriptorsState,
@@ -343,7 +345,7 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup
343345
}
344346
}
345347
is SerializationRequest -> {
346-
GlobalScope.launch(Dispatchers.Default) {
348+
connection.launchJob {
347349
if (content.topLevelDescriptorName.isNotEmpty()) {
348350
repl.serializeVariables(content.topLevelDescriptorName, content.descriptorsState, commID = content.commId, content.pathToDescriptor) { result ->
349351
sendWrapped(msg, makeReplyMessage(msg, MessageType.SERIALIZATION_REPLY, content = result))
@@ -545,3 +547,8 @@ fun JupyterConnection.evalWithIO(repl: ReplForJupyter, srcMessage: Message, body
545547
KernelStreams.setStreams(false, out, err)
546548
}
547549
}
550+
551+
fun EvaluatedSnippetMetadata?.convertToNullIfEmpty(): JsonElement? {
552+
val jsonNode = Json.encodeToJsonElement(this)
553+
return if (jsonNode is JsonNull || jsonNode?.jsonObject.isEmpty()) null else jsonNode
554+
}

‎src/main/kotlin/org/jetbrains/kotlinx/jupyter/repl.kt‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -433,9 +433,10 @@ class ReplForJupyterImpl(
433433
val newImports: List<String>
434434
val oldDeclarations: MutableMap<String, Int> = mutableMapOf()
435435
oldDeclarations.putAll(internalEvaluator.getVariablesDeclarationInfo())
436+
val jupyterId = evalData.jupyterId
436437
val result = try {
437-
log.debug("Current cell id: ${evalData.jupyterId}")
438-
executor.execute(evalData.code, evalData.displayHandler, currentCellId = evalData.jupyterId - 1) { internalId, codeToExecute ->
438+
log.debug("Current cell id: $jupyterId")
439+
executor.execute(evalData.code, evalData.displayHandler, currentCellId = jupyterId - 1) { internalId, codeToExecute ->
439440
if (evalData.storeHistory) {
440441
cell = notebook.addCell(internalId, codeToExecute, EvalData(evalData))
441442
}
@@ -464,7 +465,7 @@ class ReplForJupyterImpl(
464465
// printVars()
465466
// printUsagesInfo(jupyterId, cellVariables[jupyterId - 1])
466467
val variablesCells: Map<String, Int> = notebook.variablesState.mapValues { internalEvaluator.findVariableCell(it.key) }
467-
val serializedData = variablesSerializer.serializeVariables(jupyterId - 1, notebook.variablesState, oldDeclarations, variablesCells, notebook.unchangedVariables())
468+
val serializedData = variablesSerializer.serializeVariables(jupyterId - 1, notebook.variablesState, oldDeclarations, variablesCells, notebook.unchangedVariables)
468469

469470
GlobalScope.launch(Dispatchers.Default) {
470471
variablesSerializer.tryValidateCache(jupyterId - 1, notebook.cellVariables)

‎src/main/kotlin/org/jetbrains/kotlinx/jupyter/serializationUtils.kt‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import kotlinx.serialization.json.Json
55
import kotlinx.serialization.json.JsonObject
66
import kotlinx.serialization.json.decodeFromJsonElement
77
import org.jetbrains.kotlinx.jupyter.api.VariableState
8+
import org.jetbrains.kotlinx.jupyter.compiler.util.SerializableTypeInfo
89
import org.jetbrains.kotlinx.jupyter.compiler.util.SerializedVariablesState
910
import java.lang.reflect.Field
1011
import kotlin.contracts.ExperimentalContracts
@@ -32,14 +33,14 @@ enum class PropertiesType {
3233
}
3334

3435
@Serializable
35-
data class SerializedCommMessageContent(
36+
data class VariablesStateCommMessageContent(
3637
val topLevelDescriptorName: String,
3738
val descriptorsState: Map<String, SerializedVariablesState>,
3839
val pathToDescriptor: List<String> = emptyList()
3940
)
4041

41-
fun getVariablesDescriptorsFromJson(json: JsonObject): SerializedCommMessageContent {
42-
return Json.decodeFromJsonElement<SerializedCommMessageContent>(json)
42+
fun getVariablesDescriptorsFromJson(json: JsonObject): VariablesStateCommMessageContent {
43+
return Json.decodeFromJsonElement<VariablesStateCommMessageContent>(json)
4344
}
4445

4546
class ProcessedSerializedVarsState(
@@ -216,7 +217,12 @@ class VariablesSerializer(
216217
} else {
217218
""
218219
}
219-
val serializedVersion = SerializedVariablesState(simpleTypeName, stringedValue, true, varID)
220+
val serializedVersion = SerializedVariablesState(
221+
SerializableTypeInfo.makeFromSerializedVariablesState(simpleTypeName, true),
222+
stringedValue,
223+
true,
224+
varID
225+
)
220226
val descriptors = serializedVersion.fieldDescriptor
221227

222228
// only for set case
@@ -700,7 +706,12 @@ class VariablesSerializer(
700706
""
701707
}
702708

703-
val serializedVariablesState = SerializedVariablesState(type, getProperString(value), isContainer, finalID)
709+
val serializedVariablesState = SerializedVariablesState(
710+
SerializableTypeInfo.makeFromSerializedVariablesState(simpleTypeName, isContainer),
711+
getProperString(value),
712+
isContainer,
713+
finalID
714+
)
704715

705716
return ProcessedSerializedVarsState(serializedVariablesState, membersProperties?.toTypedArray())
706717
}

‎src/test/kotlin/org/jetbrains/kotlinx/jupyter/test/repl/ReplTests.kt‎

Lines changed: 18 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -810,14 +810,13 @@ class ReplVarsTest : AbstractSingleReplTest() {
810810
""".trimIndent(),
811811
jupyterId = 1
812812
)
813-
var state = repl.notebook.unchangedVariables()
814-
val res = eval(
813+
eval(
815814
"""
816815
l += 11111
817816
""".trimIndent(),
818817
jupyterId = 2
819818
).metadata.evaluatedVariablesState
820-
state = repl.notebook.unchangedVariables()
819+
val state: Set<String> = repl.notebook.unchangedVariables
821820
assertEquals(1, state.size)
822821
assertTrue(state.contains("m"))
823822
}
@@ -887,7 +886,7 @@ class ReplVarsTest : AbstractSingleReplTest() {
887886
""".trimIndent(),
888887
jupyterId = 1
889888
)
890-
var state = repl.notebook.unchangedVariables()
889+
var state = repl.notebook.unchangedVariables
891890
assertEquals(3, state.size)
892891

893892
eval(
@@ -898,7 +897,7 @@ class ReplVarsTest : AbstractSingleReplTest() {
898897
""".trimIndent(),
899898
jupyterId = 2
900899
)
901-
state = repl.notebook.unchangedVariables()
900+
state = repl.notebook.unchangedVariables
902901
assertEquals(0, state.size)
903902

904903
eval(
@@ -907,7 +906,7 @@ class ReplVarsTest : AbstractSingleReplTest() {
907906
""".trimIndent(),
908907
jupyterId = 3
909908
)
910-
state = repl.notebook.unchangedVariables()
909+
state = repl.notebook.unchangedVariables
911910
assertEquals(1, state.size)
912911
}
913912
}
@@ -943,7 +942,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
943942
assertEquals(listOf(1, 2, 3, 4).toString().substring(1, actualContainer.value!!.length + 1), actualContainer.value)
944943

945944
val serializer = repl.variablesSerializer
946-
val newData = serializer.doIncrementalSerialization(0, "x", "data", actualContainer)
945+
serializer.doIncrementalSerialization(0, "x", "data", actualContainer)
947946
}
948947

949948
@Test
@@ -959,7 +958,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
959958
assertEquals(2, varsData.size)
960959
assertTrue(varsData.containsKey("x"))
961960
assertTrue(varsData.containsKey("f"))
962-
var unchangedVariables = repl.notebook.unchangedVariables()
961+
var unchangedVariables = repl.notebook.unchangedVariables
963962
assertTrue(unchangedVariables.isNotEmpty())
964963

965964
eval(
@@ -968,7 +967,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
968967
""".trimIndent(),
969968
jupyterId = 1
970969
)
971-
unchangedVariables = repl.notebook.unchangedVariables()
970+
unchangedVariables = repl.notebook.unchangedVariables
972971
assertTrue(unchangedVariables.contains("x"))
973972
assertTrue(unchangedVariables.contains("f"))
974973
}
@@ -1032,7 +1031,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
10321031

10331032
val serializer = repl.variablesSerializer
10341033

1035-
val newData = serializer.doIncrementalSerialization(0, "c", "i", descriptor["i"]!!)
1034+
serializer.doIncrementalSerialization(0, "c", "i", descriptor["i"]!!)
10361035
}
10371036

10381037
@Test
@@ -1321,7 +1320,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
13211320
""".trimIndent(),
13221321
jupyterId = 1
13231322
)
1324-
val state = repl.notebook.unchangedVariables()
1323+
val state = repl.notebook.unchangedVariables
13251324
val setOfCell = setOf("x", "f", "z")
13261325
assertTrue(state.isNotEmpty())
13271326
assertEquals(setOfCell, state)
@@ -1348,7 +1347,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
13481347
""".trimIndent(),
13491348
jupyterId = 1
13501349
)
1351-
var state = repl.notebook.unchangedVariables()
1350+
var state = repl.notebook.unchangedVariables
13521351
val setOfCell = setOf("x", "f", "z")
13531352
assertTrue(state.isNotEmpty())
13541353
assertEquals(setOfCell, state)
@@ -1372,9 +1371,9 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
13721371
""".trimIndent(),
13731372
jupyterId = 3
13741373
)
1375-
state = repl.notebook.unchangedVariables()
1376-
// assertTrue(state.isNotEmpty())
1377-
// assertEquals(state, setOfPrevCell)
1374+
state = repl.notebook.unchangedVariables
1375+
assertTrue(state.isEmpty())
1376+
// assertEquals(state, setOfPrevCell)
13781377

13791378
eval(
13801379
"""
@@ -1384,20 +1383,20 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
13841383
""".trimIndent(),
13851384
jupyterId = 4
13861385
)
1387-
state = repl.notebook.unchangedVariables()
1386+
state = repl.notebook.unchangedVariables
13881387
assertTrue(state.isEmpty())
13891388
}
13901389

13911390
@Test
13921391
fun testSerializationClearInfo() {
1393-
var res = eval(
1392+
eval(
13941393
"""
13951394
val x = listOf(1, 2, 3, 4)
13961395
""".trimIndent(),
13971396
jupyterId = 1
13981397
).metadata.evaluatedVariablesState
1399-
var state = repl.notebook.unchangedVariables()
1400-
res = eval(
1398+
repl.notebook.unchangedVariables
1399+
eval(
14011400
"""
14021401
val x = listOf(1, 2, 3, 4)
14031402
""".trimIndent(),

0 commit comments

Comments
 (0)