Skip to content

Commit 21f06bb

Browse files
tochilinaktochilinak
authored
Some fixes in Python fuzzer logic (#1922)
* FakeNode as empty object; fuzzing for abstract classes * Better handling of default arguments * Random type before each 6th iteration of BaselineAlgorithm * Fixed subtype fuzzing * small fixes
1 parent 06b90e9 commit 21f06bb

File tree

21 files changed

+230
-75
lines changed

21 files changed

+230
-75
lines changed

‎utbot-python/samples/easy_samples/field.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
class NoTestsProblem:
2+
def __init__(self):
3+
self.board = []
4+
5+
def set_position(self, row, col, symbol):
6+
self.board[row][col] = symbol
7+
return symbol
8+
9+
def start(self):
10+
self.set_position(1, 2, "O")

‎utbot-python/samples/easy_samples/general.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
import heapq
33
import typing
44
from socket import socket
5-
from typing import List, Dict, Set, Optional
5+
from typing import List, Dict, Set, Optional, AbstractSet
66
from dataclasses import dataclass
77
import logging
88
import datetime
@@ -94,7 +94,7 @@ def empty():
9494
return 1
9595

9696

97-
def id_(x):
97+
def id_(x: AbstractSet[int]):
9898
return x
9999

100100

‎utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import org.utbot.fuzzing.utils.Trie
1111
import org.utbot.python.evaluation.*
1212
import org.utbot.python.evaluation.serialiation.MemoryDump
1313
import org.utbot.python.evaluation.serialiation.toPythonTree
14+
import org.utbot.python.framework.api.python.PythonTree
1415
import org.utbot.python.framework.api.python.PythonTreeModel
1516
import org.utbot.python.framework.api.python.PythonTreeWrapper
1617
import org.utbot.python.fuzzing.*
@@ -259,6 +260,12 @@ class PythonEngine(
259260
return@PythonFuzzing PythonFeedback(control = Control.STOP)
260261
}
261262

263+
if (arguments.any { PythonTree.containsFakeNode(it.tree) }) {
264+
logger.debug("FakeNode in Python model")
265+
emit(InvalidExecution(UtError("Bad input object", Throwable())))
266+
return@PythonFuzzing PythonFeedback(control = Control.CONTINUE)
267+
}
268+
262269
val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) })
263270
val mem = cache.get(pair)
264271
if (mem != null) {
@@ -278,7 +285,7 @@ class PythonEngine(
278285
return@PythonFuzzing result.fuzzingPlatformFeedback
279286
}.fuzz(pmd)
280287
} catch (_: Exception) { // e.g. NoSeedValueException
281-
logger.info { "Cannot fuzz values for types: $parameters" }
288+
logger.info { "Cannot fuzz values for types: ${parameters.map { it.pythonTypeRepresentation() }}" }
282289
}
283290
}
284291
manager.disconnect()

‎utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt‎

Lines changed: 53 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import org.utbot.python.utils.TimeoutMode
2929
import java.io.File
3030

3131
private val logger = KotlinLogging.logger {}
32+
private const val RANDOM_TYPE_FREQUENCY = 6
3233

3334
class PythonTestCaseGenerator(
3435
private val withMinimization: Boolean = true,
@@ -117,28 +118,28 @@ class PythonTestCaseGenerator(
117118
}.take(maxSubstitutions)
118119
}
119120

120-
fun generate(method: PythonMethod, until: Long): PythonTestSet {
121-
storageForMypyMessages.clear()
121+
private fun methodHandler(
122+
method: PythonMethod,
123+
typeStorage: PythonTypeStorage,
124+
coveredLines: MutableSet<Int>,
125+
errors: MutableList<UtError>,
126+
executions: MutableList<UtExecution>,
127+
initMissingLines: Set<Int>?,
128+
until: Long,
129+
additionalVars: String = ""
130+
): Set<Int>? { // returns missing lines
122131
val limitManager = TestGenerationLimitManager(
123132
ExecutionWithTimeoutMode,
124133
until,
125134
)
126-
127-
val typeStorage = PythonTypeStorage.get(mypyStorage)
135+
var missingLines = initMissingLines
128136

129137
val (hintCollector, constantCollector) = constructCollectors(mypyStorage, typeStorage, method)
130138
val constants = constantCollector.result.map { (type, value) ->
131139
logger.debug("Collected constant: ${type.pythonTypeRepresentation()}: $value")
132140
PythonFuzzedConcreteValue(type, value)
133141
}
134142

135-
val executions = mutableListOf<UtExecution>()
136-
val errors = mutableListOf<UtError>()
137-
var missingLines: Set<Int>? = null
138-
val coveredLines = mutableSetOf<Int>()
139-
var generated = 0
140-
141-
logger.info("Start test generation for ${method.name}")
142143
substituteTypeParameters(method, typeStorage).forEach { newMethod ->
143144
inferAnnotations(
144145
newMethod,
@@ -148,6 +149,7 @@ class PythonTestCaseGenerator(
148149
mypyReportLine,
149150
mypyConfigFile,
150151
limitManager,
152+
additionalVars
151153
) { functionType ->
152154
val args = (functionType as FunctionType).arguments
153155

@@ -168,7 +170,6 @@ class PythonTestCaseGenerator(
168170
val fuzzerCancellation = { isCancelled() || limitManager.isCancelled() }
169171

170172
engine.fuzzing(args, fuzzerCancellation, until).collect {
171-
generated += 1
172173
when (it) {
173174
is ValidExecution -> {
174175
executions += it.utFuzzedExecution
@@ -196,6 +197,42 @@ class PythonTestCaseGenerator(
196197
feedback
197198
}
198199
}
200+
return missingLines
201+
}
202+
203+
fun generate(method: PythonMethod, until: Long): PythonTestSet {
204+
storageForMypyMessages.clear()
205+
206+
val typeStorage = PythonTypeStorage.get(mypyStorage)
207+
208+
val executions = mutableListOf<UtExecution>()
209+
val errors = mutableListOf<UtError>()
210+
val coveredLines = mutableSetOf<Int>()
211+
212+
logger.info("Start test generation for ${method.name}")
213+
val meta = method.definition.type.pythonDescription() as PythonCallableTypeDescription
214+
val argKinds = meta.argumentKinds
215+
if (argKinds.any { it != PythonCallableTypeDescription.ArgKind.ARG_POS }) {
216+
val now = System.currentTimeMillis()
217+
val firstUntil = (until - now) / 2 + now
218+
val originalDef = method.definition
219+
val shortType = meta.removeNonPositionalArgs(originalDef.type)
220+
val shortMeta = PythonFuncItemDescription(
221+
originalDef.meta.name,
222+
originalDef.meta.args.take(shortType.arguments.size)
223+
)
224+
val additionalVars = originalDef.meta.args
225+
.drop(shortType.arguments.size)
226+
.joinToString(separator="\n", prefix="\n") { arg ->
227+
"${arg.name}: ${pythonAnyType.pythonTypeRepresentation()}" // TODO: better types
228+
}
229+
method.definition = PythonFunctionDefinition(shortMeta, shortType)
230+
val missingLines = methodHandler(method, typeStorage, coveredLines, errors, executions, null, firstUntil, additionalVars)
231+
method.definition = originalDef
232+
methodHandler(method, typeStorage, coveredLines, errors, executions, missingLines, until)
233+
} else {
234+
methodHandler(method, typeStorage, coveredLines, errors, executions, null, until)
235+
}
199236

200237
logger.info("Collect all test executions for ${method.name}")
201238
val (successfulExecutions, failedExecutions) = executions.partition { it.result is UtExecutionSuccess }
@@ -234,6 +271,7 @@ class PythonTestCaseGenerator(
234271
report: List<MypyReportLine>,
235272
mypyConfigFile: File,
236273
limitManager: TestGenerationLimitManager,
274+
additionalVars: String,
237275
annotationHandler: suspend (Type) -> InferredTypeFeedback,
238276
) {
239277
val namesInModule = mypyStorage.names
@@ -257,7 +295,9 @@ class PythonTestCaseGenerator(
257295
getOffsetLine(sourceFileContent, method.ast.beginOffset),
258296
getOffsetLine(sourceFileContent, method.ast.endOffset)
259297
),
260-
mypyConfigFile
298+
mypyConfigFile,
299+
additionalVars,
300+
randomTypeFrequency = RANDOM_TYPE_FREQUENCY
261301
)
262302

263303
runBlocking breaking@{

‎utbot-python/src/main/kotlin/org/utbot/python/code/CodeGen.kt‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,8 @@ object PythonCodeGenerator {
4545
methodAnnotations: Map<String, Type>,
4646
directoriesForSysPath: Set<String>,
4747
moduleToImport: String,
48-
namesInModule: Collection<String>
48+
namesInModule: Collection<String>,
49+
additionalVars: String
4950
): String {
5051
val context = UtContext(this::class.java.classLoader)
5152
withUtContext(context) {
@@ -60,7 +61,8 @@ object PythonCodeGenerator {
6061
methodAnnotations,
6162
directoriesForSysPath,
6263
moduleToImport,
63-
namesInModule
64+
namesInModule,
65+
additionalVars
6466
)
6567
}
6668
}

‎utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,19 @@ object PythonTree {
2626
return tree.children.any { isRecursiveObjectDFS(it, visited) }
2727
}
2828

29+
fun containsFakeNode(tree: PythonTreeNode): Boolean {
30+
return containsFakeNodeDFS(tree, mutableSetOf())
31+
}
32+
33+
private fun containsFakeNodeDFS(tree: PythonTreeNode, visited: MutableSet<PythonTreeNode>): Boolean {
34+
if (visited.contains(tree))
35+
return false
36+
if (tree is FakeNode)
37+
return true
38+
visited.add(tree)
39+
return tree.children.any { containsFakeNodeDFS(it, visited) }
40+
}
41+
2942
open class PythonTreeNode(
3043
val id: Long,
3144
val type: PythonClassId,
@@ -69,6 +82,8 @@ object PythonTree {
6982
1 + children.fold(0) { acc, child -> acc + child.diversity() }
7083
}
7184

85+
object FakeNode: PythonTreeNode(0L, PythonClassId(""))
86+
7287
class PrimitiveNode(
7388
id: Long,
7489
type: PythonClassId,

‎utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/PythonCodeGenerator.kt‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,8 @@ class PythonCodeGenerator(
211211
methodAnnotations: Map<String, Type>,
212212
directoriesForSysPath: Set<String>,
213213
moduleToImport: String,
214-
namesInModule: Collection<String>
214+
namesInModule: Collection<String>,
215+
additionalVars: String
215216
): String {
216217
val cgRendererContext = CgRendererContext.fromCgContext(context)
217218
val printer = CgPrinterImpl()
@@ -243,6 +244,8 @@ class PythonCodeGenerator(
243244
val mypyCheckCode = listOf(
244245
renderer.toString(),
245246
"",
247+
additionalVars,
248+
"",
246249
functionName,
247250
) + method.codeAsString.split("\n").map { " $it" }
248251
return mypyCheckCode.joinToString("\n")

‎utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt‎

Lines changed: 9 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,10 @@ import org.utbot.fuzzing.utils.Trie
1616
import org.utbot.python.framework.api.python.PythonTree
1717
import org.utbot.python.fuzzing.provider.*
1818
import org.utbot.python.fuzzing.provider.utils.isAny
19-
import org.utbot.python.newtyping.PythonProtocolDescription
20-
import org.utbot.python.newtyping.PythonSubtypeChecker
21-
import org.utbot.python.newtyping.PythonTypeStorage
19+
import org.utbot.python.fuzzing.provider.utils.isConcreteType
20+
import org.utbot.python.newtyping.*
21+
import org.utbot.python.newtyping.general.DefaultSubstitutionProvider
2222
import org.utbot.python.newtyping.general.Type
23-
import org.utbot.python.newtyping.pythonTypeRepresentation
2423

2524
private val logger = KotlinLogging.logger {}
2625

@@ -59,7 +58,7 @@ class PythonFuzzedValue(
5958
val summary: String? = null,
6059
)
6160

62-
fun pythonDefaultValueProviders() = listOf(
61+
fun pythonDefaultValueProviders(typeStorage: PythonTypeStorage) = listOf(
6362
NoneValueProvider,
6463
BoolValueProvider,
6564
IntValueProvider,
@@ -76,36 +75,22 @@ fun pythonDefaultValueProviders() = listOf(
7675
BytearrayValueProvider,
7776
ReduceValueProvider,
7877
ConstantValueProvider,
79-
TypeAliasValueProvider
78+
TypeAliasValueProvider,
79+
SubtypeValueProvider(typeStorage)
8080
)
8181

8282
class PythonFuzzing(
8383
private val pythonTypeStorage: PythonTypeStorage,
8484
val execute: suspend (description: PythonMethodDescription, values: List<PythonFuzzedValue>) -> PythonFeedback,
8585
) : Fuzzing<Type, PythonFuzzedValue, PythonMethodDescription, PythonFeedback> {
8686

87-
private fun generateDefault(description: PythonMethodDescription, type: Type): Sequence<Seed<Type, PythonFuzzedValue>> {
88-
var providers = emptyList<Seed<Type, PythonFuzzedValue>>().asSequence()
89-
pythonDefaultValueProviders().asSequence().forEach { provider ->
87+
private fun generateDefault(description: PythonMethodDescription, type: Type)= sequence<Seed<Type, PythonFuzzedValue>> {
88+
pythonDefaultValueProviders(pythonTypeStorage).asSequence().forEach { provider ->
9089
if (provider.accept(type)) {
9190
logger.debug { "Provider ${provider.javaClass.simpleName} accepts type ${type.pythonTypeRepresentation()}" }
92-
providers += provider.generate(description, type)
91+
yieldAll(provider.generate(description, type))
9392
}
9493
}
95-
return providers
96-
}
97-
98-
private fun generateSubtype(description: PythonMethodDescription, type: Type): Sequence<Seed<Type, PythonFuzzedValue>> {
99-
var providers = emptyList<Seed<Type, PythonFuzzedValue>>().asSequence()
100-
if (type.meta is PythonProtocolDescription) {
101-
val subtypes = pythonTypeStorage.allTypes.filter {
102-
PythonSubtypeChecker.checkIfRightIsSubtypeOfLeft(type, it, pythonTypeStorage)
103-
}
104-
subtypes.forEach {
105-
providers += generateDefault(description, it)
106-
}
107-
}
108-
return providers
10994
}
11095

11196
override fun generate(description: PythonMethodDescription, type: Type): Sequence<Seed<Type, PythonFuzzedValue>> {
@@ -115,7 +100,6 @@ class PythonFuzzing(
115100
logger.debug("Any does not have provider")
116101
} else {
117102
providers += generateDefault(description, type)
118-
providers += generateSubtype(description, type)
119103
}
120104

121105
return providers

‎utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import org.utbot.python.framework.api.python.PythonTree
88
import org.utbot.python.framework.api.python.util.toPythonRepr
99
import org.utbot.python.fuzzing.PythonFuzzedValue
1010
import org.utbot.python.fuzzing.PythonMethodDescription
11+
import org.utbot.python.fuzzing.provider.utils.isConcreteType
1112
import org.utbot.python.newtyping.*
1213
import org.utbot.python.newtyping.general.FunctionType
1314
import org.utbot.python.newtyping.general.Type
@@ -30,7 +31,7 @@ object ReduceValueProvider : ValueProvider<Type, PythonFuzzedValue, PythonMethod
3031
override fun accept(type: Type): Boolean {
3132
val hasSupportedType =
3233
!unsupportedTypes.contains(type.pythonTypeName())
33-
return hasSupportedType && type.meta is PythonConcreteCompositeTypeDescription // && (hasInit || hasNew)
34+
return hasSupportedType && isConcreteType(type) // && (hasInit || hasNew)
3435
}
3536

3637
override fun generate(description: PythonMethodDescription, type: Type) = sequence {
@@ -100,12 +101,7 @@ object ReduceValueProvider : ValueProvider<Type, PythonFuzzedValue, PythonMethod
100101
)
101102
},
102103
modify = modifications,
103-
empty = Routine.Empty {
104-
PythonFuzzedValue(
105-
PythonTree.fromObject(),
106-
"%var% = ${type.pythonTypeRepresentation()}"
107-
)
108-
}
104+
empty = Routine.Empty { PythonFuzzedValue(PythonTree.FakeNode) }
109105
)
110106
}
111107

‎utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/SubtypeValueProvider.kt‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
package org.utbot.python.fuzzing.provider
2+
3+
import org.utbot.fuzzing.Routine
4+
import org.utbot.fuzzing.Seed
5+
import org.utbot.fuzzing.ValueProvider
6+
import org.utbot.python.framework.api.python.PythonTree
7+
import org.utbot.python.fuzzing.PythonFuzzedValue
8+
import org.utbot.python.fuzzing.PythonMethodDescription
9+
import org.utbot.python.fuzzing.provider.utils.isConcreteType
10+
import org.utbot.python.newtyping.*
11+
import org.utbot.python.newtyping.PythonSubtypeChecker.Companion.checkIfRightIsSubtypeOfLeft
12+
import org.utbot.python.newtyping.general.DefaultSubstitutionProvider
13+
import org.utbot.python.newtyping.general.Type
14+
15+
class SubtypeValueProvider(
16+
private val typeStorage: PythonTypeStorage
17+
) : ValueProvider<Type, PythonFuzzedValue, PythonMethodDescription> {
18+
override fun accept(type: Type): Boolean {
19+
return type.meta is PythonProtocolDescription ||
20+
((type.meta as? PythonConcreteCompositeTypeDescription)?.isAbstract == true)
21+
}
22+
23+
private val concreteTypes = typeStorage.allTypes.filter {
24+
isConcreteType(it) && it.pythonDescription().name.name.first() != '_' // Don't substitute private classes
25+
}.map {
26+
DefaultSubstitutionProvider.substituteAll(it, it.parameters.map { pythonAnyType })
27+
}
28+
29+
override fun generate(description: PythonMethodDescription, type: Type) = sequence {
30+
val subtypes = concreteTypes.filter { checkIfRightIsSubtypeOfLeft(type, it, typeStorage) }
31+
subtypes.forEach { subtype ->
32+
yield(
33+
Seed.Recursive(
34+
construct = Routine.Create(listOf(subtype)) { v -> v.first() },
35+
empty = Routine.Empty { PythonFuzzedValue(PythonTree.FakeNode) }
36+
))
37+
}
38+
}
39+
}

0 commit comments

Comments
 (0)