Consider changes in objects via immutable refs

Author: nikolay-egorov

jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/compiler/util/serializedCompiledScript.kt

@@ -25,10 +25,29 @@ data class SerializedVariablesState(
     val type: String = "",
     val value: String? = null,
     val isContainer: Boolean = false,
-    val ID: String = ""
+    val stateId: String = ""
 ) {
     // todo: not null
     val fieldDescriptor: MutableMap<String, SerializedVariablesState?> = mutableMapOf()
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (javaClass != other?.javaClass) return false
+
+        other as SerializedVariablesState
+
+        if (type != other.type) return false
+        if (value != other.value) return false
+        if (isContainer != other.isContainer) return false
+
+        return true
+    }
+
+    override fun hashCode(): Int {
+        var result = type.hashCode()
+        result = 31 * result + (value?.hashCode() ?: 0)
+        result = 31 * result + isContainer.hashCode()
+        return result
+    }
 }
 
 @Serializable

src/main/kotlin/org/jetbrains/kotlinx/jupyter/repl.kt

@@ -461,7 +461,8 @@ class ReplForJupyterImpl(
             notebook.updateVariablesState(internalEvaluator)
             // printVars()
             // printUsagesInfo(jupyterId, cellVariables[jupyterId - 1])
-            val serializedData = variablesSerializer.serializeVariables(jupyterId - 1, notebook.variablesState, notebook.unchangedVariables())
+            val variablesCells: Map<String, Int> = notebook.variablesState.mapValues { internalEvaluator.findVariableCell(it.key) }
+            val serializedData = variablesSerializer.serializeVariables(jupyterId - 1, notebook.variablesState, variablesCells, notebook.unchangedVariables())
 
             GlobalScope.launch(Dispatchers.Default) {
                 variablesSerializer.tryValidateCache(jupyterId - 1, notebook.cellVariables)

src/main/kotlin/org/jetbrains/kotlinx/jupyter/serializationUtils.kt

@@ -358,7 +358,7 @@ class VariablesSerializer(
         clearOldData(currentCellId, cellVariables)
     }
 
-    fun serializeVariables(cellId: Int, variablesState: Map<String, VariableState>, unchangedVariables: Set<String>): Map<String, SerializedVariablesState> {
+    fun serializeVariables(cellId: Int, variablesState: Map<String, VariableState>, variablesCells: Map<String, Int>, unchangedVariables: Set<String>): Map<String, SerializedVariablesState> {
         fun removeNonExistingEntries() {
             val toRemoveSet = mutableSetOf<String>()
             serializedVariablesCache.forEach { (name, _) ->
@@ -382,17 +382,22 @@ class VariablesSerializer(
             val wasRedeclared = !unchangedVariables.contains(it)
             if (wasRedeclared) {
                 removedFromSightVariables.remove(it)
-            }
-            (unchangedVariables.contains(it) || serializedVariablesCache[it]?.value != variablesState[it]?.stringValue) &&
+            } /*
+            (unchangedVariables.contains(it) || serializedVariablesCache[it]?.value != variablesState[it]?.value?.getOrNull().toString()) &&
+                !removedFromSightVariables.contains(it)*/
+            (unchangedVariables.contains(it)) &&
                 !removedFromSightVariables.contains(it)
         }
         log.debug("Variables state as is: $variablesState")
         log.debug("Serializing variables after filter: $neededEntries")
-        log.debug("Unchanged variables: $unchangedVariables")
+        log.debug("Unchanged variables: ${unchangedVariables - neededEntries.keys}")
 
         // remove previous data
         computedDescriptorsPerCell[cellId]?.instancesPerState?.clear()
-        val serializedData = neededEntries.mapValues { serializeVariableState(cellId, it.key, it.value) }
+        val serializedData = neededEntries.mapValues {
+            val actualCell = variablesCells[it.key] ?: cellId
+            serializeVariableState(actualCell, it.key, it.value)
+        }
 
         serializedVariablesCache.putAll(serializedData)
         removeNonExistingEntries()
@@ -730,7 +735,7 @@ class VariablesSerializer(
                 ""
             }
 
-        val serializedVariablesState = SerializedVariablesState(type, getProperString(value), isContainer, ID = finalID)
+        val serializedVariablesState = SerializedVariablesState(type, getProperString(value), isContainer, finalID)
 
         return ProcessedSerializedVarsState(serializedVariablesState, membersProperties?.toTypedArray())
     }

src/test/kotlin/org/jetbrains/kotlinx/jupyter/test/repl/ReplTests.kt

@@ -747,7 +747,7 @@ class ReplVarsTest : AbstractSingleReplTest() {
 
     @Test
     fun testRecursiveVarsState() {
-        eval(
+        val res = eval(
             """
             val l = mutableListOf<Any>()
             l.add(listOf(l))
@@ -757,7 +757,7 @@ class ReplVarsTest : AbstractSingleReplTest() {
             val z = setOf(1, 2, 4)
             """.trimIndent(),
             jupyterId = 1
-        )
+        ).metadata
         val state = repl.notebook.variablesState
         assertTrue(state.contains("l"))
         assertTrue(state.contains("m"))
@@ -766,6 +766,54 @@ class ReplVarsTest : AbstractSingleReplTest() {
         assertEquals("ArrayList: recursive structure", state["l"]!!.stringValue)
         assertTrue(state["m"]!!.stringValue!!.contains(" recursive structure"))
         assertEquals("[1, 2, 4]", state["z"]!!.stringValue)
+
+        val serializer = repl.variablesSerializer
+        val descriptor = res.evaluatedVariablesState["l"]!!.fieldDescriptor
+        val innerList = descriptor["elementData"]!!.fieldDescriptor["data"]
+        val newData = serializer.doIncrementalSerialization(0, "data", innerList!!)
+        assertEquals(2, newData.fieldDescriptor.size)
+    }
+
+    @Test
+    fun testUnchangedVars() {
+        eval(
+            """
+            var l = 11111
+            val m = "abc"
+            """.trimIndent(),
+            jupyterId = 1
+        )
+        var state = repl.notebook.unchangedVariables()
+        val res = eval(
+            """
+            l += 11111
+            """.trimIndent(),
+            jupyterId = 2
+        ).metadata.evaluatedVariablesState
+        state = repl.notebook.unchangedVariables()
+        assertEquals(1, state.size)
+        assertTrue(state.contains("m"))
+    }
+
+    @Test
+    fun testMutableList() {
+        eval(
+            """
+            val l = mutableListOf(1, 2, 3, 4)
+            """.trimIndent(),
+            jupyterId = 1
+        )
+        val serializer = repl.variablesSerializer
+        val res = eval(
+            """
+            l.add(5)
+            """.trimIndent(),
+            jupyterId = 2
+        ).metadata.evaluatedVariablesState
+        val innerList = res["l"]!!.fieldDescriptor["elementData"]!!.fieldDescriptor["data"]
+        val newData = serializer.doIncrementalSerialization(0, "data", innerList!!)
+        assertTrue(newData.isContainer)
+        assertTrue(newData.fieldDescriptor.size > 4)
     }
 
     @Test