diff --git a/.gitignore b/.gitignore index 68ddfe43..7c5c0585 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ /CMakeSettings.json /cmake-build-* **/__pycache__ -/benchmarks/pending/__debug__* \ No newline at end of file +/benchmarks/pending/__debug__* +/benchmarks/fast/_.c \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 9e075b3b..db2dca1f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ -project(fizz) - cmake_minimum_required(VERSION 3.20 FATAL_ERROR) +project(fizzer) + macro(append_compiler_flags FLAGS) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FLAGS}") endmacro() @@ -70,12 +70,13 @@ find_package(Threads REQUIRED) # find and add Boost message("Searching for Boost library ...") -find_package(Boost 1.69 REQUIRED COMPONENTS filesystem) +set(Boost_USE_STATIC_LIBS ON) +find_package(Boost REQUIRED COMPONENTS filesystem) message("Boost STATUS: Includes ${Boost_INCLUDE_DIRS} - Libs ${Boost_LIBRARIES} -") + Libs ${Boost_LIBRARIES}") include_directories(${Boost_INCLUDE_DIRS}) +set(BOOST_LIST_OF_LIBRARIES_TO_LINK_WITH "${Boost_LIBRARIES}") set(LLVM_EXPORT_SYMBOLS_FOR_PLUGINS "yes") find_package(LLVM REQUIRED CONFIG) @@ -92,8 +93,7 @@ message("LLVM STATUS: Definitions ${LLVM_DEFINITIONS} Includes ${LLVM_INCLUDE_DIRS} Libraries ${LLVM_LIBRARY_DIRS} - Targets ${LLVM_TARGETS_TO_BUILD}" -) + Targets ${LLVM_TARGETS_TO_BUILD}") if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) @@ -113,7 +113,9 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/src") # Add project specific sources add_subdirectory(./src) -install(DIRECTORY ./benchmarks DESTINATION .) -install(FILES ./LICENSE.txt DESTINATION .) +if(FIZZ_BUILD_LIBS_32_BIT STREQUAL "No") + install(DIRECTORY ./benchmarks DESTINATION .) + install(FILES ./LICENSE.txt DESTINATION .) +endif() message("Generating build files ...") diff --git a/README.md b/README.md index c0c832d4..6a169a36 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,11 @@ start with the **age** project: tasks, e.g., building benchmarks and killing non-terminating clients. You only need to copy the file from the `setup` folder to the folder `.vscode` folder. +- (Optional) If you also want to analyze 32-bit programs, then you must also + build 32-bit version of Fizzer's libraries. That is done automatically via + Fizzer's `build.sh` script. However, 32-bit version of C++ standard library + must be available in the C++ compiler. On Linux (Ubuntu) you can install + this library using: `sudo apt install g++-multilib` - (optional) **SmartGit** Git GUI client: https://www.syntevo.com/smartgit/ ## Downloading **SBT-Fizzer** diff --git a/benchmarks/benman.py b/benchmarks/benman.py index 3da6a36b..43ba4d83 100755 --- a/benchmarks/benman.py +++ b/benchmarks/benman.py @@ -106,99 +106,69 @@ def _execute_and_check_output(self, cmdline : str, desired_output : str, work_di self._execute(cmdline, os.path.dirname(desired_output) if work_dir is None else work_dir) ASSUMPTION(os.path.isfile(desired_output), "_execute_and_check_output(): the output is missing: " + desired_output) - def _check_outcomes(self, config : dict, outcomes : dict): - checked_properties_and_comparators = { - "termination_type": "EQ", - "termination_reason": "EQ", - "num_executions": "LE", - "num_covered_branchings": "GE", - "covered_branchings": None, - "num_generated_tests": "GE", - "num_crashes": "GE", - "num_boundary_violations": "LE" - } - - def is_valid(obtained, expected, op : str) -> bool: - if op == "EQ": return obtained == expected - if op == "NE": return obtained != expected - if op == "LT": return obtained < expected - if op == "LE": return obtained <= expected - if op == "GT": return obtained > expected - if op == "GE": return obtained >= expected - raise Exception("Invalid comparison operator '" + op + "'.") - - for property, expected_value in config["results"].items(): - ASSUMPTION( - property in checked_properties_and_comparators, - "Unsupported key '" + property + "' in the 'results' section of benchmark's config JSON file." - ) - ASSUMPTION( - property in outcomes, - "The valid key '" + property + "' was not found in the 'outcomes' JSON file." - ) - if type(expected_value) in [int, float, str]: - if not is_valid(outcomes[property], expected_value, checked_properties_and_comparators[property]): + @staticmethod + def _add_error_message(text: str, errors: list, properties: list): + errors.append(("In " + "/".join(properties) + ": " if len(properties) > 0 else "") + text) + + @staticmethod + def _epsilon_for_property(properties): + if len(properties) == 0: return None + if properties[-1] == "num_executions": return 5.0 + return None + + @staticmethod + def _check_outcomes(obtained, expected, errors: list, properties = []) -> bool: + if type(expected) is dict: + if type(obtained) is not dict: + Benchmark._add_error_message("Mismatch in JSON structure. Expected dictionary.", errors, properties) + return False + result = True + for key in expected: + if key not in obtained: + Benchmark._add_error_message("Missing property: " + key, errors, properties) + return False + r = Benchmark._check_outcomes(obtained[key], expected[key], errors, properties + [key]) + result = result and r + return result + elif type(expected) is list: + if type(obtained) is not list: + Benchmark._add_error_message("Mismatch in JSON structure. Expected list.", errors, properties) + return False + if len(obtained) != len(expected): + Benchmark._add_error_message("Different list size.", errors, properties) + return False + result = True + for i in range(min(len(obtained), len(expected))): + r = Benchmark._check_outcomes(obtained[i], expected[i], errors, properties) + result = result and r + return result + elif type(expected) in [int, float]: + if type(obtained) not in [int, float]: + Benchmark._add_error_message("Mismatch in JSON structure. Expected int or float.", errors, properties) + return False + epsilon = Benchmark._epsilon_for_property(properties) + if epsilon is None: + if obtained != expected: + Benchmark._add_error_message("Expected " + str(expected) + ", obtained " + str(obtained), errors, properties) return False else: - ASSUMPTION(property == "covered_branchings", "Only 'covered_branchings' can be a 'list' property to check.") - ASSUMPTION(len(expected_value) % 2 == 0, "Expected covered branchings list must have even number of elements.") - ASSUMPTION(len(outcomes[property]) % 2 == 0, "Obtained covered branchings list must have even number of elements.") - def get_branchings(seq : list) -> set: - result = set() - if len(seq) > 0: - for i in range(0, len(seq)-1, 2): - result.add((seq[i], seq[i+1])) - return result - expected_branchings = get_branchings(expected_value) - obtained_branchings = get_branchings(outcomes[property]) - for x in expected_branchings: - if x not in obtained_branchings: - return False - return True - - def _ok_stats_message(self, config : dict, outcomes : dict) -> str: - max_num_execution = config["results"]["num_executions"] - try: - num_execution = outcomes["num_executions"] - except Exception as e: - return "Unknown executions count" - percentage = 100.0 * num_execution / max_num_execution - return "#" + ("%.2f" % (percentage - 100)) + "%" if max_num_execution >= 100 and percentage < 90 else "" - - def _fail_stats_message(self, config : dict, outcomes : dict) -> str: - expected_termination_type = config["results"]["termination_type"] - try: - termination_type = outcomes["termination_type"] - except Exception as e: - return "Unknown termination type" - if termination_type != expected_termination_type: - return termination_type - - result = "" - - expected_termination_reason = config["results"]["termination_reason"] - try: - termination_reason = outcomes["termination_reason"] - except Exception as e: - return "Unknown termination reason" - if termination_reason != expected_termination_reason: - result += termination_reason - - max_num_execution = config["results"]["num_executions"] - try: - num_execution = outcomes["num_executions"] - except Exception as e: - return result + ("" if len(result) == 0 else ", ") + "unknown executions count" - if len(result) == 0 and num_execution > max_num_execution: - percentage = 100.0 * num_execution / max_num_execution - result += ("" if len(result) == 0 else ", ") + "#" + ("+%.2f" % (percentage - 100)) + "%" - - return result - - def _embrace_stats_message(self, msg : str) -> str: - if len(msg) == 0: - return msg - return "[" + msg + "]" + percentage = (100.0 * obtained) / expected if expected > 0 else 100.0 * obtained + 100.0 + error = percentage - 100.0 + if abs(error) > epsilon: + Benchmark._add_error_message("Expected " + str(expected) + ", obtained " + str(obtained) + " [error: " + ("%.2f" % error) + "%]", errors, properties) + return False + return True + elif type(expected) is str: + if type(obtained) is not str: + Benchmark._add_error_message("Mismatch in JSON structure. Expected string.", errors, properties) + return False + if obtained != expected: + Benchmark._add_error_message("Expected " + expected + ", obtained " + obtained, errors, properties) + return False + return True + else: + Benchmark._add_error_message("Unexpected JSON content [type: " + str(type(expected)) + "].", errors, properties) + return False def build(self, benchmarks_root_dir : str, output_root_dir : str) -> None: self.log("===") @@ -216,7 +186,7 @@ def build(self, benchmarks_root_dir : str, output_root_dir : str) -> None: "--skip_fuzzing", "--input_file", self.src_file, "--output_dir", self.work_dir, - "--silent_build", + "--silent_mode", "--save_mapping" ] + (["--m32"] if "m32" in self.config["args"] and self.config["args"]["m32"] is True else []), self.fuzz_target_file, @@ -263,19 +233,20 @@ def fuzz(self, benchmarks_root_dir : str, output_root_dir : str) -> bool: output_dir ) + errors = [] try: outcomes_pathname = os.path.join(output_dir, self.name + "_outcomes.json") with open(outcomes_pathname, "rb") as fp: outcomes = json.load(fp) - if self._check_outcomes(self.config, outcomes) is True: - stats_msg = self._embrace_stats_message(self._ok_stats_message(self.config, outcomes)) - self.log("The outcomes are as expected => the test has PASSED. [Details: " + stats_msg + "]", "ok " + stats_msg + "\n") + if self._check_outcomes(outcomes, self.config["results"], errors) is True: + ASSUMPTION(len(errors) == 0) + self.log("The outcomes are as expected => the test has PASSED.", "ok\n") return True except Exception as e: self.log("FAILURE due to an EXCEPTION: " + str(e), "EXCEPTION[" + str(e) + "]\n") return False - stats_msg = self._embrace_stats_message(self._fail_stats_message(self.config, outcomes)) - self.log("The outcomes are NOT as expected => the test has FAILED. Details: " + stats_msg, "FAILED " + stats_msg + "\n") + error_messages = "\n " + "\n ".join(errors) + self.log("The outcomes are NOT as expected => the test has FAILED. Details:" + error_messages, "FAILED " + error_messages + "\n") return False def clear(self, benchmarks_root_dir : str, output_root_dir : str) -> None: @@ -328,7 +299,7 @@ def search_for_benchmarks(folder : str) -> list: pass return benchmarks - kinds = ["fast", "medium", "slow", "pending"] + kinds = ["fast", "iid_testing", "testcomp-selection-selection", "medium", "slow", "pending"] benchmarks = [] if name == "all": for kind in kinds: diff --git a/benchmarks/fast/array-1.json b/benchmarks/fast/array-1.json index 1a633e84..b4a54506 100644 --- a/benchmarks/fast/array-1.json +++ b/benchmarks/fast/array-1.json @@ -7,11 +7,12 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 2,0, 3,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } diff --git a/benchmarks/fast/array1_pattern.json b/benchmarks/fast/array1_pattern.json index a0c5b2f3..ae2ab2c2 100644 --- a/benchmarks/fast/array1_pattern.json +++ b/benchmarks/fast/array1_pattern.json @@ -7,11 +7,12 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -22,8 +23,17 @@ 1,2654435899, 1,2654435962, 3,0, 4,0, 5,0, 6,0, 7,0, 8,0 ], - "num_generated_tests": 7, - "num_crashes": 0, - "num_boundary_violations": 2 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 1, + "num_boundary_violations": 2 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 1, + "num_boundary_violations": 0 + } + } } } diff --git a/benchmarks/fast/big_issue.json b/benchmarks/fast/big_issue.json index 4d72ff91..05f6fddb 100644 --- a/benchmarks/fast/big_issue.json +++ b/benchmarks/fast/big_issue.json @@ -11,18 +11,28 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "FUZZING_STRATEGY_DEPLETED", - "num_executions": 105, + "num_executions": 103, "num_covered_branchings": 2, "covered_branchings": [ 2,0, 4,0 ], - "num_generated_tests": 3, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } diff --git a/benchmarks/fast/bool_flag_one_and_two.json b/benchmarks/fast/bool_flag_one_and_two.json index d05b81ff..1aabbfd3 100644 --- a/benchmarks/fast/bool_flag_one_and_two.json +++ b/benchmarks/fast/bool_flag_one_and_two.json @@ -7,23 +7,38 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 165, + "num_executions": 163, "num_covered_branchings": 6, "covered_branchings": [ 1,0, 2,0, 3,0, 4,0, 5,0, 6,0 ], - "num_generated_tests": 4, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/c_string_count_chars.json b/benchmarks/fast/c_string_count_chars.json index 8497ded3..465ad190 100644 --- a/benchmarks/fast/c_string_count_chars.json +++ b/benchmarks/fast/c_string_count_chars.json @@ -7,23 +7,43 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 2230, + "num_executions": 2709, "num_covered_branchings": 7, "covered_branchings": [ 1,0, 2,0, 3,0, 4,0, 5,0, 6,0, 7,0 ], - "num_generated_tests": 6, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/c_string_parse_two_ints.json b/benchmarks/fast/c_string_parse_two_ints.json index f7a5e48d..e29efd96 100644 --- a/benchmarks/fast/c_string_parse_two_ints.json +++ b/benchmarks/fast/c_string_parse_two_ints.json @@ -7,16 +7,17 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 3790, + "num_executions": 4785, "num_covered_branchings": 35, "covered_branchings": [ 1,0, 2,0, 3,0, 4,0, @@ -29,8 +30,27 @@ 18,1013904498, 19,1013904435, 19,1013904498, 20,1013904435, 20,1013904498, 21,1013904435, 21,1013904498 ], - "num_generated_tests": 30, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 7, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 12, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 8, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/call_bool_arg.json b/benchmarks/fast/call_bool_arg.json index 383178d7..67a6b777 100644 --- a/benchmarks/fast/call_bool_arg.json +++ b/benchmarks/fast/call_bool_arg.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 35, + "num_executions": 2, "num_covered_branchings": 2, "covered_branchings": [ 1,2654435832, 2,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 1, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } diff --git a/benchmarks/fast/ex2-alloca.json b/benchmarks/fast/ex2-alloca.json index 195511d6..3a44931e 100644 --- a/benchmarks/fast/ex2-alloca.json +++ b/benchmarks/fast/ex2-alloca.json @@ -7,23 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", - "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 420, - "num_covered_branchings": 6, + "termination_reason": "FUZZING_STRATEGY_DEPLETED", + "num_executions": 4940, + "num_covered_branchings": 2, "covered_branchings": [ - 1,2654435832, 2,2654435832, 3,2654435832, 4,2654435832, - 5,2654435832, 6,2654435832 + 1,2654435832, 2,2654435832 ], - "num_generated_tests": 4, - "num_crashes": 0, - "num_boundary_violations": 1 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } diff --git a/benchmarks/fast/float_if_deg_to_rad.json b/benchmarks/fast/float_if_deg_to_rad.json index 23bca22a..b1c6a207 100644 --- a/benchmarks/fast/float_if_deg_to_rad.json +++ b/benchmarks/fast/float_if_deg_to_rad.json @@ -7,22 +7,37 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 55, + "num_executions": 56, "num_covered_branchings": 2, "covered_branchings": [ 1,0, 2,0 ], - "num_generated_tests": 3, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/float_if_deg_to_rad_inline.json b/benchmarks/fast/float_if_deg_to_rad_inline.json index 6e602d9b..7031d001 100644 --- a/benchmarks/fast/float_if_deg_to_rad_inline.json +++ b/benchmarks/fast/float_if_deg_to_rad_inline.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 30, + "num_executions": 40, "num_covered_branchings": 2, "covered_branchings": [ 1,0, 2,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } } + } } \ No newline at end of file diff --git a/benchmarks/fast/float_if_parabola.json b/benchmarks/fast/float_if_parabola.json index bfdf91ae..8c63cb3a 100644 --- a/benchmarks/fast/float_if_parabola.json +++ b/benchmarks/fast/float_if_parabola.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 115, + "num_executions": 27, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/float_if_parabola2.json b/benchmarks/fast/float_if_parabola2.json index e6dca3c1..a69ccf3d 100644 --- a/benchmarks/fast/float_if_parabola2.json +++ b/benchmarks/fast/float_if_parabola2.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 125, + "num_executions": 121, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/float_if_x_eq_c.json b/benchmarks/fast/float_if_x_eq_c.json index 50fec118..a5a6d382 100644 --- a/benchmarks/fast/float_if_x_eq_c.json +++ b/benchmarks/fast/float_if_x_eq_c.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 50, + "num_executions": 49, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } } + } } \ No newline at end of file diff --git a/benchmarks/fast/float_if_x_eq_cos_x.json b/benchmarks/fast/float_if_x_eq_cos_x.json index 36676479..ef220aea 100644 --- a/benchmarks/fast/float_if_x_eq_cos_x.json +++ b/benchmarks/fast/float_if_x_eq_cos_x.json @@ -7,11 +7,12 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/float_if_x_lt_c.json b/benchmarks/fast/float_if_x_lt_c.json index 5f88c5e7..fe9e2ad9 100644 --- a/benchmarks/fast/float_if_x_lt_c.json +++ b/benchmarks/fast/float_if_x_lt_c.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 30, + "num_executions": 34, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/float_if_xy_level_ring.json b/benchmarks/fast/float_if_xy_level_ring.json index 4581623f..b85c86c7 100644 --- a/benchmarks/fast/float_if_xy_level_ring.json +++ b/benchmarks/fast/float_if_xy_level_ring.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", - "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 150, - "num_covered_branchings": 2, + "termination_reason": "FUZZING_STRATEGY_DEPLETED", + "num_executions": 169, + "num_covered_branchings": 1, "covered_branchings": [ - 1,0, 2,0 + 1,0 ], - "num_generated_tests": 3, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/infinite01.json b/benchmarks/fast/infinite01.json index cff0a4e1..feb3387c 100644 --- a/benchmarks/fast/infinite01.json +++ b/benchmarks/fast/infinite01.json @@ -7,22 +7,33 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "EXECUTIONS_BUDGET_DEPLETED", "num_executions": 500, - "num_covered_branchings": 4, + "num_covered_branchings": 5, "covered_branchings": [ - 1,0, 2,0, 4,0, 7,0 + 1,0, 2,0, 3,0, 4,0, + 7,0 ], - "num_generated_tests": 3, - "num_crashes": 0, - "num_boundary_violations": 5 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 4 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 1 + } + } } } diff --git a/benchmarks/fast/int16_equal.json b/benchmarks/fast/int16_equal.json index 5bb41420..7fdea7d2 100644 --- a/benchmarks/fast/int16_equal.json +++ b/benchmarks/fast/int16_equal.json @@ -7,22 +7,37 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 95, + "num_executions": 92, "num_covered_branchings": 4, "covered_branchings": [ 1,0, 2,0, 3,0, 4,0 ], - "num_generated_tests": 5, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int16_if_parabola.json b/benchmarks/fast/int16_if_parabola.json index 368fc6fe..34b97396 100644 --- a/benchmarks/fast/int16_if_parabola.json +++ b/benchmarks/fast/int16_if_parabola.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 60, + "num_executions": 7, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int16_if_x_equal_c.json b/benchmarks/fast/int16_if_x_equal_c.json index ce2633b4..8a006888 100644 --- a/benchmarks/fast/int16_if_x_equal_c.json +++ b/benchmarks/fast/int16_if_x_equal_c.json @@ -7,11 +7,12 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } } + } } \ No newline at end of file diff --git a/benchmarks/fast/int16_if_x_equal_y_c.json b/benchmarks/fast/int16_if_x_equal_y_c.json index 10e59ccc..6ecd1e2c 100644 --- a/benchmarks/fast/int16_if_x_equal_y_c.json +++ b/benchmarks/fast/int16_if_x_equal_y_c.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 45, + "num_executions": 44, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int16_if_x_ge_c.json b/benchmarks/fast/int16_if_x_ge_c.json index ffb36aee..41d77310 100644 --- a/benchmarks/fast/int16_if_x_ge_c.json +++ b/benchmarks/fast/int16_if_x_ge_c.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 20, + "num_executions": 11, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int16_if_x_lt_c.json b/benchmarks/fast/int16_if_x_lt_c.json index a97fe99c..5326c55d 100644 --- a/benchmarks/fast/int16_if_x_lt_c.json +++ b/benchmarks/fast/int16_if_x_lt_c.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 20, + "num_executions": 11, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int16_if_x_xor_a_eq_b.json b/benchmarks/fast/int16_if_x_xor_a_eq_b.json index 08e034e4..dfd1c9a8 100644 --- a/benchmarks/fast/int16_if_x_xor_a_eq_b.json +++ b/benchmarks/fast/int16_if_x_xor_a_eq_b.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 100, + "num_executions": 98, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int16_less.json b/benchmarks/fast/int16_less.json index 770a8e9b..b32267a4 100644 --- a/benchmarks/fast/int16_less.json +++ b/benchmarks/fast/int16_less.json @@ -7,22 +7,37 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 135, + "num_executions": 70, "num_covered_branchings": 4, "covered_branchings": [ 1,0, 2,0, 3,0, 4,0 ], - "num_generated_tests": 5, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int32_ackermann.json b/benchmarks/fast/int32_ackermann.json index ae2f9a25..0bee70c9 100644 --- a/benchmarks/fast/int32_ackermann.json +++ b/benchmarks/fast/int32_ackermann.json @@ -7,11 +7,12 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -27,8 +28,17 @@ 8,2027809127, 8,2654435832, 8,3668340393, 8,3668340399, 8,3668340520, 8,3668340524 ], - "num_generated_tests": 12, - "num_crashes": 0, - "num_boundary_violations": 2 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 11, + "num_crashes": 0, + "num_boundary_violations": 1 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int32_if_parabola.json b/benchmarks/fast/int32_if_parabola.json index 98c4a671..34b97396 100644 --- a/benchmarks/fast/int32_if_parabola.json +++ b/benchmarks/fast/int32_if_parabola.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 80, + "num_executions": 7, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int32_if_x_equal_c.json b/benchmarks/fast/int32_if_x_equal_c.json index de5498da..3df9ae0e 100644 --- a/benchmarks/fast/int32_if_x_equal_c.json +++ b/benchmarks/fast/int32_if_x_equal_c.json @@ -7,11 +7,12 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int32_if_x_equal_y_c.json b/benchmarks/fast/int32_if_x_equal_y_c.json index 43697fa4..ee9470f9 100644 --- a/benchmarks/fast/int32_if_x_equal_y_c.json +++ b/benchmarks/fast/int32_if_x_equal_y_c.json @@ -7,11 +7,12 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int32_if_x_ge_c.json b/benchmarks/fast/int32_if_x_ge_c.json index 9487834a..97576d1d 100644 --- a/benchmarks/fast/int32_if_x_ge_c.json +++ b/benchmarks/fast/int32_if_x_ge_c.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 50, + "num_executions": 27, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int32_if_x_lt_c.json b/benchmarks/fast/int32_if_x_lt_c.json index 9487834a..97576d1d 100644 --- a/benchmarks/fast/int32_if_x_lt_c.json +++ b/benchmarks/fast/int32_if_x_lt_c.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 50, + "num_executions": 27, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int32_if_x_xor_a_eq_b.json b/benchmarks/fast/int32_if_x_xor_a_eq_b.json index f20148fe..ae47b400 100644 --- a/benchmarks/fast/int32_if_x_xor_a_eq_b.json +++ b/benchmarks/fast/int32_if_x_xor_a_eq_b.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 1040, + "num_executions": 1038, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int32_logical_or_two_vars.json b/benchmarks/fast/int32_logical_or_two_vars.json index d2f1c2f4..ba0ed659 100644 --- a/benchmarks/fast/int32_logical_or_two_vars.json +++ b/benchmarks/fast/int32_logical_or_two_vars.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 150, + "num_executions": 58, "num_covered_branchings": 4, "covered_branchings": [ 1,0, 2,0, 3,0, 4,0 ], - "num_generated_tests": 5, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int8_if_hash_x_y_z_eq_c.json b/benchmarks/fast/int8_if_hash_x_y_z_eq_c.json index 8f4a36de..ebdf2eac 100644 --- a/benchmarks/fast/int8_if_hash_x_y_z_eq_c.json +++ b/benchmarks/fast/int8_if_hash_x_y_z_eq_c.json @@ -7,21 +7,26 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "FUZZING_STRATEGY_DEPLETED", - "num_executions": 15000, + "num_executions": 14947, "num_covered_branchings": 0, "covered_branchings": [ ], - "num_generated_tests": 1, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int8_if_x_equal_c.json b/benchmarks/fast/int8_if_x_equal_c.json index a97fe99c..94840880 100644 --- a/benchmarks/fast/int8_if_x_equal_c.json +++ b/benchmarks/fast/int8_if_x_equal_c.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 20, + "num_executions": 17, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int8_if_x_equal_y_c.json b/benchmarks/fast/int8_if_x_equal_y_c.json index e8ae8f20..715d8092 100644 --- a/benchmarks/fast/int8_if_x_equal_y_c.json +++ b/benchmarks/fast/int8_if_x_equal_y_c.json @@ -7,11 +7,12 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int8_if_x_ge_c.json b/benchmarks/fast/int8_if_x_ge_c.json index a97fe99c..5326c55d 100644 --- a/benchmarks/fast/int8_if_x_ge_c.json +++ b/benchmarks/fast/int8_if_x_ge_c.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 20, + "num_executions": 11, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int8_if_x_lt_c.json b/benchmarks/fast/int8_if_x_lt_c.json index a97fe99c..5326c55d 100644 --- a/benchmarks/fast/int8_if_x_lt_c.json +++ b/benchmarks/fast/int8_if_x_lt_c.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 20, + "num_executions": 11, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int8_if_x_xor_a_eq_b.json b/benchmarks/fast/int8_if_x_xor_a_eq_b.json index 249c0126..2191a190 100644 --- a/benchmarks/fast/int8_if_x_xor_a_eq_b.json +++ b/benchmarks/fast/int8_if_x_xor_a_eq_b.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 30, + "num_executions": 24, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/log_and.json b/benchmarks/fast/log_and.json index fd8fdb82..fd3c1c77 100644 --- a/benchmarks/fast/log_and.json +++ b/benchmarks/fast/log_and.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 40, + "num_executions": 53, "num_covered_branchings": 3, "covered_branchings": [ 1,0, 2,0, 3,0 ], - "num_generated_tests": 3, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } } + } } diff --git a/benchmarks/fast/log_cond.json b/benchmarks/fast/log_cond.json index 1efa132f..e5a5c2a8 100644 --- a/benchmarks/fast/log_cond.json +++ b/benchmarks/fast/log_cond.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 55, + "num_executions": 59, "num_covered_branchings": 4, "covered_branchings": [ 1,0, 2,0, 3,0, 4,0 ], - "num_generated_tests": 4, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } diff --git a/benchmarks/fast/machine32bit.json b/benchmarks/fast/machine32bit.json index d91181ed..aa0e825e 100644 --- a/benchmarks/fast/machine32bit.json +++ b/benchmarks/fast/machine32bit.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -22,8 +22,17 @@ "covered_branchings": [ 4,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } diff --git a/benchmarks/fast/matrix-2.json b/benchmarks/fast/matrix-2.json index 462e2e2a..173fd678 100644 --- a/benchmarks/fast/matrix-2.json +++ b/benchmarks/fast/matrix-2.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 2,0, 3,0, 4,0, 5,0 ], - "num_generated_tests": 5, - "num_crashes": 1, - "num_boundary_violations": 4 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 1, + "num_boundary_violations": 4 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } diff --git a/benchmarks/fast/mul_two_int16.json b/benchmarks/fast/mul_two_int16.json index bf49c77b..95b7b0bf 100644 --- a/benchmarks/fast/mul_two_int16.json +++ b/benchmarks/fast/mul_two_int16.json @@ -7,22 +7,26 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", - "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 70, - "num_covered_branchings": 1, + "termination_reason": "FUZZING_STRATEGY_DEPLETED", + "num_executions": 37, + "num_covered_branchings": 0, "covered_branchings": [ - 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/nested_ifs.json b/benchmarks/fast/nested_ifs.json index ab082396..839bcd1b 100644 --- a/benchmarks/fast/nested_ifs.json +++ b/benchmarks/fast/nested_ifs.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 185, + "num_executions": 183, "num_covered_branchings": 4, "covered_branchings": [ 1,0, 2,0, 3,0, 4,0 ], - "num_generated_tests": 5, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/std_redef_malloc_free.json b/benchmarks/fast/std_redef_malloc_free.json index c8769626..45719e0c 100644 --- a/benchmarks/fast/std_redef_malloc_free.json +++ b/benchmarks/fast/std_redef_malloc_free.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 40, + "num_executions": 10, "num_covered_branchings": 2, "covered_branchings": [ 1,0, 2,0 ], - "num_generated_tests": 3, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/switch.json b/benchmarks/fast/switch.json index 3169bb2b..f99800bb 100644 --- a/benchmarks/fast/switch.json +++ b/benchmarks/fast/switch.json @@ -7,23 +7,38 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 100, + "num_executions": 77, "num_covered_branchings": 5, "covered_branchings": [ 1,0, 2,0, 3,0, 4,0, 5,0 ], - "num_generated_tests": 6, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/uint16_if_parabola.json b/benchmarks/fast/uint16_if_parabola.json index dd5df545..34b97396 100644 --- a/benchmarks/fast/uint16_if_parabola.json +++ b/benchmarks/fast/uint16_if_parabola.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 30, + "num_executions": 7, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } } + } } \ No newline at end of file diff --git a/benchmarks/fast/uint32_if_parabola.json b/benchmarks/fast/uint32_if_parabola.json index fa0b9389..34b97396 100644 --- a/benchmarks/fast/uint32_if_parabola.json +++ b/benchmarks/fast/uint32_if_parabola.json @@ -7,22 +7,32 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 45, + "num_executions": 7, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } } + } } \ No newline at end of file diff --git a/benchmarks/iid_testing/_.c b/benchmarks/iid_testing/_.c new file mode 100644 index 00000000..b70849fc --- /dev/null +++ b/benchmarks/iid_testing/_.c @@ -0,0 +1,36 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + k += s[ i ]; + ++i; + } + + if ( k == 500 ) // ID: 6 + return 1; + + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_01_char_countX_gt15.c b/benchmarks/iid_testing/aigen_01_char_countX_gt15.c new file mode 100644 index 00000000..de254423 --- /dev/null +++ b/benchmarks/iid_testing/aigen_01_char_countX_gt15.c @@ -0,0 +1,39 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 100 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; // Invalid size + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop + int count = 0; + for ( short i = 0; i < size; ++i ) { + // 3. Modify internal state based on data/condition (linear update) + if ( data[ i ] == 'X' ) { + count++; // Linear change (+1) + } + } + + // 4. Final condition based on internal variable + // 5. Condition check is linear + if ( count > 15 ) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_01_char_countX_gt15.json b/benchmarks/iid_testing/aigen_01_char_countX_gt15.json new file mode 100644 index 00000000..6df0b390 --- /dev/null +++ b/benchmarks/iid_testing/aigen_01_char_countX_gt15.json @@ -0,0 +1,59 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3880, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_02_char_balancePN_eq15.c b/benchmarks/iid_testing/aigen_02_char_balancePN_eq15.c new file mode 100644 index 00000000..8707a6e7 --- /dev/null +++ b/benchmarks/iid_testing/aigen_02_char_balancePN_eq15.c @@ -0,0 +1,41 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 50 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop + int balance = 0; + for ( short i = 0; i < size; ++i ) { + // 3. Modify internal state based on data/condition (linear update) + if ( data[ i ] == 'P' ) { + balance++; // Linear change (+1) + } else if ( data[ i ] == 'N' ) { + balance--; // Linear change (-1) + } + } + + // 4. Final condition based on internal variable + // 5. Condition check is linear + if ( balance == 15 ) { + return 1; // Balanced + } else { + return 0; // Not balanced + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_02_char_balancePN_eq15.json b/benchmarks/iid_testing/aigen_02_char_balancePN_eq15.json new file mode 100644 index 00000000..8f3766bc --- /dev/null +++ b/benchmarks/iid_testing/aigen_02_char_balancePN_eq15.json @@ -0,0 +1,61 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3211, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_03_char_sum_AB_val_eq25.c b/benchmarks/iid_testing/aigen_03_char_sum_AB_val_eq25.c new file mode 100644 index 00000000..24c324ef --- /dev/null +++ b/benchmarks/iid_testing/aigen_03_char_sum_AB_val_eq25.c @@ -0,0 +1,44 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 50 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop + int value = 0; + short idx = 0; + while ( idx < size ) { + // 3. Modify internal state based on data/condition + // 6. Change is static (+2 or +1) + if ( data[ idx ] == 'A' ) { + value += 2; // Static change + } else if ( data[ idx ] == 'B' ) { + value += 1; // Static change + } + idx++; + } + + // 4. Final condition based on internal variable + // 5. Condition check is linear + if ( value == 25 ) { + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_03_char_sum_AB_val_eq25.json b/benchmarks/iid_testing/aigen_03_char_sum_AB_val_eq25.json new file mode 100644 index 00000000..87aadfe3 --- /dev/null +++ b/benchmarks/iid_testing/aigen_03_char_sum_AB_val_eq25.json @@ -0,0 +1,61 @@ +{ + "args": { + "max_executions": 20000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 9524, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_04_char_nonvowel_upper_decr_target.c b/benchmarks/iid_testing/aigen_04_char_nonvowel_upper_decr_target.c new file mode 100644 index 00000000..311203ca --- /dev/null +++ b/benchmarks/iid_testing/aigen_04_char_nonvowel_upper_decr_target.c @@ -0,0 +1,45 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 50 +#define START_COUNT 100 // Start high and decrement + +int main() +{ + char data[ MAX_SIZE ]; + short size; + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop + int non_vowel_upper_count = START_COUNT; + for ( short i = 0; i < size; ++i ) { + // 3. Modify internal state based on data/condition (nested) + // 6. Change is static (-1) + if ( data[ i ] >= 'A' && data[ i ] <= 'Z' ) { // Is uppercase? + if ( data[ i ] != 'A' && data[ i ] != 'E' && data[ i ] != 'I' && data[ i ] != 'O' && + data[ i ] != 'U' ) { // Is not a vowel? + non_vowel_upper_count--; // Static change + } + } + } + + // 4. Final condition based on internal variable + // 5. Condition check is linear (< START_COUNT - 5) + // Checks if at least 6 non-vowel uppercase letters were found + if ( non_vowel_upper_count < START_COUNT - 5 ) { + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_04_char_nonvowel_upper_decr_target.json b/benchmarks/iid_testing/aigen_04_char_nonvowel_upper_decr_target.json new file mode 100644 index 00000000..4e054f12 --- /dev/null +++ b/benchmarks/iid_testing/aigen_04_char_nonvowel_upper_decr_target.json @@ -0,0 +1,71 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1799, + "num_covered_branchings": 12, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0, + 12, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 6, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_05_char_statemachine_XY_resetR_count_gt3.c b/benchmarks/iid_testing/aigen_05_char_statemachine_XY_resetR_count_gt3.c new file mode 100644 index 00000000..68163642 --- /dev/null +++ b/benchmarks/iid_testing/aigen_05_char_statemachine_XY_resetR_count_gt3.c @@ -0,0 +1,47 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 150 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + int state = 0; // State variable (e.g., 0=Start, 1=SeenX, 2=SeenY) + int state2_entries = 0; // Counts entries into state 2 + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop + for ( short i = 0; i < size; ++i ) { + // 3. Modify internal state variables based on data/condition + // 6. State change is static (assignment), counter change is static (+1) + if ( state == 0 && data[ i ] == 'X' ) { + state = 1; // Static change (assignment) + } else if ( state == 1 && data[ i ] == 'Y' ) { + state = 2; // Static change (assignment) + state2_entries++; // Static change (+1) + } else if ( data[ i ] == 'R' ) { // Reset condition + state = 0; // Static change (assignment) + } + // Stay in current state otherwise, or remain in state 2 + } + + // 4. Final condition based on internal variable + // 5. Condition check is linear + if ( state2_entries > 3 ) { + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_05_char_statemachine_XY_resetR_count_gt3.json b/benchmarks/iid_testing/aigen_05_char_statemachine_XY_resetR_count_gt3.json new file mode 100644 index 00000000..09563861 --- /dev/null +++ b/benchmarks/iid_testing/aigen_05_char_statemachine_XY_resetR_count_gt3.json @@ -0,0 +1,67 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 6945, + "num_covered_branchings": 10, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_06_char_bounded_counter_UD_eq5.c b/benchmarks/iid_testing/aigen_06_char_bounded_counter_UD_eq5.c new file mode 100644 index 00000000..7b564b79 --- /dev/null +++ b/benchmarks/iid_testing/aigen_06_char_bounded_counter_UD_eq5.c @@ -0,0 +1,48 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 100 +#define UPPER_BOUND 10 +#define LOWER_BOUND -10 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + int bounded_counter = 0; // State variable with bounds + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop + for ( short i = 0; i < size; ++i ) { + // 3. Modify internal state based on data/condition + // 6. Changes are static (+1 or -1), but bounded + if ( data[ i ] == 'U' ) { + if ( bounded_counter < UPPER_BOUND ) { + bounded_counter++; // Static change + } + } else if ( data[ i ] == 'D' ) { + if ( bounded_counter > LOWER_BOUND ) { + bounded_counter--; // Static change + } + } + } + + // 4. Final condition based on internal variable + // 5. Condition check is linear + if ( bounded_counter == 5 ) { // Check final value against a specific target + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_06_char_bounded_counter_UD_eq5.json b/benchmarks/iid_testing/aigen_06_char_bounded_counter_UD_eq5.json new file mode 100644 index 00000000..da22a0d4 --- /dev/null +++ b/benchmarks/iid_testing/aigen_06_char_bounded_counter_UD_eq5.json @@ -0,0 +1,65 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 9907, + "num_covered_branchings": 9, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_07_char_two_arrays_matchST_gt5.c b/benchmarks/iid_testing/aigen_07_char_two_arrays_matchST_gt5.c new file mode 100644 index 00000000..4c3cb3f5 --- /dev/null +++ b/benchmarks/iid_testing/aigen_07_char_two_arrays_matchST_gt5.c @@ -0,0 +1,42 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 50 + +int main() +{ + char data1[ MAX_SIZE ]; + char data2[ MAX_SIZE ]; // Second array + short size; + int match_count = 0; // State variable + + // 1. Fill data structures in loops + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data1[ i ] = __VERIFIER_nondet_char(); + data2[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop iterating through data + for ( short i = 0; i < size; ++i ) { + // 3. Modify internal state based on data from both arrays + // 6. Change is static (+1) + if ( data1[ i ] == 'S' && data2[ i ] == 'T' ) { + match_count++; // Static change + } + } + + // 4. Final condition based on internal variable + // 5. Condition check is linear + if ( match_count > 5 ) { // Check if the S/T pair occurred more than once + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_07_char_two_arrays_matchST_gt5.json b/benchmarks/iid_testing/aigen_07_char_two_arrays_matchST_gt5.json new file mode 100644 index 00000000..473e872d --- /dev/null +++ b/benchmarks/iid_testing/aigen_07_char_two_arrays_matchST_gt5.json @@ -0,0 +1,61 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3107, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_08_char_ptr_countP_eq5.c b/benchmarks/iid_testing/aigen_08_char_ptr_countP_eq5.c new file mode 100644 index 00000000..e3dd7377 --- /dev/null +++ b/benchmarks/iid_testing/aigen_08_char_ptr_countP_eq5.c @@ -0,0 +1,44 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 60 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + int p_count = 0; // State variable + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop using pointer arithmetic + char* ptr = data; + char* end_ptr = data + size; // Pointer to one past the last element + + while ( ptr < end_ptr ) { + // 3. Modify internal state based on data accessed via pointer + // 6. Change is static (+1) + if ( *ptr == 'P' ) { + p_count++; // Static change + } + ptr++; // Move pointer to the next element + } + + // 4. Final condition based on internal variable + // 5. Condition check is linear + if ( p_count == 5 ) { // Check if 'P' was never found + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_08_char_ptr_countP_eq5.json b/benchmarks/iid_testing/aigen_08_char_ptr_countP_eq5.json new file mode 100644 index 00000000..46a3863e --- /dev/null +++ b/benchmarks/iid_testing/aigen_08_char_ptr_countP_eq5.json @@ -0,0 +1,59 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 923, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_09_char_odd_ascii_ratio_gt_half.c b/benchmarks/iid_testing/aigen_09_char_odd_ascii_ratio_gt_half.c new file mode 100644 index 00000000..eb1f23d7 --- /dev/null +++ b/benchmarks/iid_testing/aigen_09_char_odd_ascii_ratio_gt_half.c @@ -0,0 +1,40 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 90 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + int odd_count = 0; // State variable + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop + for ( short i = 0; i < size; ++i ) { + // 3. Modify internal state based on bitwise check of data + // 6. Change is static (+1) + if ( ( data[ i ] & 1 ) != 0 ) { // Check if the LSB is 1 (ASCII value is odd) + odd_count++; // Static change + } + } + + // 4. Final condition based on internal variable + // 5. Condition check is linear + if ( odd_count * 2 > size ) { // Check if more than half the chars had odd ASCII values + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_09_char_odd_ascii_ratio_gt_half.json b/benchmarks/iid_testing/aigen_09_char_odd_ascii_ratio_gt_half.json new file mode 100644 index 00000000..5f667ed5 --- /dev/null +++ b/benchmarks/iid_testing/aigen_09_char_odd_ascii_ratio_gt_half.json @@ -0,0 +1,49 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 536, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_10_char_two_loops_countA_eq10_or_countB_eq10.c b/benchmarks/iid_testing/aigen_10_char_two_loops_countA_eq10_or_countB_eq10.c new file mode 100644 index 00000000..468f7e7a --- /dev/null +++ b/benchmarks/iid_testing/aigen_10_char_two_loops_countA_eq10_or_countB_eq10.c @@ -0,0 +1,59 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 100 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + int countA = 0; // State variable for first loop + int countB = 0; // State variable for second loop + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in sequence of loops + // Loop 1 + for ( short i = 0; i < size; ++i ) { + // 3. Modify state variable (countA) + // 5. Change is constant (+1) + if ( data[ i ] == 'A' ) { + countA++; // Constant change + } + } + // Loop 2 + for ( short i = 0; i < size; ++i ) { + // 3. Modify state variable (countB) + // 5. Change is constant (+1) + if ( data[ i ] == 'B' ) { + countB++; // Constant change + } + } // Both loops always complete + + // 4. Final condition based on multiple internal variables + // if ( countA == countB && countA > 10 ) { // Check if counts are equal and non-zero + // return 1; + // } else { + // return 0; + // } + + if ( countA == 10 ) { // Check if counts are equal and non-zero + return 1; + } + + if ( countB == 10 ) { + return 1; + } + + return 0; +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_10_char_two_loops_countA_eq10_or_countB_eq10.json b/benchmarks/iid_testing/aigen_10_char_two_loops_countA_eq10_or_countB_eq10.json new file mode 100644 index 00000000..8288bd4f --- /dev/null +++ b/benchmarks/iid_testing/aigen_10_char_two_loops_countA_eq10_or_countB_eq10.json @@ -0,0 +1,65 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3652, + "num_covered_branchings": 9, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_11_char_continueS_countF_ge10.c b/benchmarks/iid_testing/aigen_11_char_continueS_countF_ge10.c new file mode 100644 index 00000000..b1fcbf5a --- /dev/null +++ b/benchmarks/iid_testing/aigen_11_char_continueS_countF_ge10.c @@ -0,0 +1,43 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 90 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + int f_count = 0; // State variable + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop using 'continue' + for ( short i = 0; i < size; ++i ) { + if ( data[ i ] == 'S' ) { // Skip character 'S' + continue; // Jumps to the next iteration + } + // 3. Modify internal state if not skipped + // 5. Change is constant (+1) + if ( data[ i ] == 'F' ) { + f_count++; // Constant change + } + // Potentially other processing for non-'S' characters here + } // Loop always completes (though iterations might be skipped) + + // 4. Final condition based on internal variable + if ( f_count >= 10 ) { // Check if 'F' was found enough times + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_11_char_continueS_countF_ge10.json b/benchmarks/iid_testing/aigen_11_char_continueS_countF_ge10.json new file mode 100644 index 00000000..e586b99e --- /dev/null +++ b/benchmarks/iid_testing/aigen_11_char_continueS_countF_ge10.json @@ -0,0 +1,61 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 2252, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_12_int_array_pos_gt20_or_neg_gt20.c b/benchmarks/iid_testing/aigen_12_int_array_pos_gt20_or_neg_gt20.c new file mode 100644 index 00000000..f49a5187 --- /dev/null +++ b/benchmarks/iid_testing/aigen_12_int_array_pos_gt20_or_neg_gt20.c @@ -0,0 +1,53 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern int __VERIFIER_nondet_int(); // Use int for values + +#define MAX_SIZE 60 + +int main() +{ + int data[ MAX_SIZE ]; + short size; + int pos_count = 0; // State variable 1 + int neg_count = 0; // State variable 2 + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_int(); // Fill with non-det integers + } + + // 2. Core logic in loop + for ( short i = 0; i < size; ++i ) { + // 3. Modify internal state variables based on data + // 5. Changes are constant (+1) + if ( data[ i ] > 0 ) { + pos_count++; // Constant change + } else if ( data[ i ] < 0 ) { + neg_count++; // Constant change + } + // Ignore zeros + } // Loop always completes + + // 4. Multiple final conditions based on internal variables + // if ( pos_count > neg_count + 2 ) { // Check relation 1 + // return 1; // More positives by a margin + // } else if ( neg_count > pos_count + 2 ) { // Check relation 2 + // return 2; // More negatives by a margin + // } else { + // return 0; // Neither condition met (counts are close or equal) + // } + + if ( pos_count > 20 ) { // Check relation 1 + return 1; // More positives by a margin + } else if ( neg_count > 20 ) { // Check relation 2 + return 2; // More negatives by a margin + } else { + return 0; // Neither condition met (counts are close or equal) + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_12_int_array_pos_gt20_or_neg_gt20.json b/benchmarks/iid_testing/aigen_12_int_array_pos_gt20_or_neg_gt20.json new file mode 100644 index 00000000..6fd503e2 --- /dev/null +++ b/benchmarks/iid_testing/aigen_12_int_array_pos_gt20_or_neg_gt20.json @@ -0,0 +1,53 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 600, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 48669, + "num_covered_branchings": 8, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 6, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_13_char_strlen_gt40.c b/benchmarks/iid_testing/aigen_13_char_strlen_gt40.c new file mode 100644 index 00000000..f4a7dff6 --- /dev/null +++ b/benchmarks/iid_testing/aigen_13_char_strlen_gt40.c @@ -0,0 +1,43 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 50 // Note: actual string length can be less + +int main() +{ + char data[ MAX_SIZE ]; + short max_len; + int len = 0; // State variable + + // 1. Fill data structure (making it a C string) + max_len = __VERIFIER_nondet_short(); + if ( max_len <= 0 || max_len >= MAX_SIZE ) { // Need space for null terminator + return -1; + } + // Fill up to max_len - 1 + for ( short i = 0; i < max_len; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + data[ max_len ] = '\0'; // Ensure null termination + + // 2. Core logic in loop to find length (simulates strlen) + // 6. Uses break based on C string convention + for ( int i = 0;; ++i ) { // Loop bound by buffer size + if ( data[ i ] == '\0' ) { + break; // Found null terminator + } + // 3. Modify internal state variable + // 5. Change is constant (+1) + len++; // Constant change + } // Loop breaks on null terminator + + // 4. Final condition based on internal variable (calculated length) + if ( len > 40 ) { + return 1; // Length is greater than 10 + } else { + return 0; // Length is 10 or less + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_13_char_strlen_gt40.json b/benchmarks/iid_testing/aigen_13_char_strlen_gt40.json new file mode 100644 index 00000000..ae81b51d --- /dev/null +++ b/benchmarks/iid_testing/aigen_13_char_strlen_gt40.json @@ -0,0 +1,47 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 600, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 21189, + "num_covered_branchings": 5, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_14_struct_statusA_ge10.c b/benchmarks/iid_testing/aigen_14_struct_statusA_ge10.c new file mode 100644 index 00000000..33486456 --- /dev/null +++ b/benchmarks/iid_testing/aigen_14_struct_statusA_ge10.c @@ -0,0 +1,45 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +// Simple struct definition +typedef struct { + char status; // e.g., 'A'ctive, 'I'nactive +} Item; + +#define MAX_ITEMS 40 + +int main() +{ + Item inventory[ MAX_ITEMS ]; // Array of structs + short num_items; + int active_count = 0; // State variable + + // 1. Fill data structure (array of structs) in an initial loop + num_items = __VERIFIER_nondet_short(); + if ( num_items <= 0 || num_items > MAX_ITEMS ) { + return -1; + } + for ( short i = 0; i < num_items; ++i ) { + inventory[ i ].status = __VERIFIER_nondet_char(); // Assign status + } + + // 2. Core logic in loop iterating through the array of structs + for ( short i = 0; i < num_items; ++i ) { + // 3. Modify internal state based on struct member + // 5. Change is constant (+1) + if ( inventory[ i ].status == 'A' ) { + active_count++; // Constant change + } + } // Loop always completes + + // 4. Final condition based on internal variable + if ( active_count >= 10 ) { // Check if at least half are active + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_14_struct_statusA_ge10.json b/benchmarks/iid_testing/aigen_14_struct_statusA_ge10.json new file mode 100644 index 00000000..02c1726d --- /dev/null +++ b/benchmarks/iid_testing/aigen_14_struct_statusA_ge10.json @@ -0,0 +1,59 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 600, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1910, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_15_char_countX_eq10.c b/benchmarks/iid_testing/aigen_15_char_countX_eq10.c new file mode 100644 index 00000000..58b24266 --- /dev/null +++ b/benchmarks/iid_testing/aigen_15_char_countX_eq10.c @@ -0,0 +1,42 @@ +/* File: gen_case_01_char_count_eq_10.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int count = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + // 3. Modify internal state based on data/condition + if (data[i] == 'X') { + count++; // 6. Static change (+1) + } + } + // 4. Final condition based on internal variable(s) + if (count == 10) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_15_char_countX_eq10.json b/benchmarks/iid_testing/aigen_15_char_countX_eq10.json new file mode 100644 index 00000000..d519fa9b --- /dev/null +++ b/benchmarks/iid_testing/aigen_15_char_countX_eq10.json @@ -0,0 +1,53 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1810, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_16_char_countY_gt15.c b/benchmarks/iid_testing/aigen_16_char_countY_gt15.c new file mode 100644 index 00000000..757121c5 --- /dev/null +++ b/benchmarks/iid_testing/aigen_16_char_countY_gt15.c @@ -0,0 +1,41 @@ +/* File: gen_case_02_char_count_gt_15.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int y_count = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'Y') { + y_count++; + } + } + // 4. Final condition based on internal variable(s) + if (y_count > 15) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_16_char_countY_gt15.json b/benchmarks/iid_testing/aigen_16_char_countY_gt15.json new file mode 100644 index 00000000..493e7ac3 --- /dev/null +++ b/benchmarks/iid_testing/aigen_16_char_countY_gt15.json @@ -0,0 +1,53 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3544, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_17_char_balanceAB_eq13.c b/benchmarks/iid_testing/aigen_17_char_balanceAB_eq13.c new file mode 100644 index 00000000..ba943632 --- /dev/null +++ b/benchmarks/iid_testing/aigen_17_char_balanceAB_eq13.c @@ -0,0 +1,43 @@ +/* File: gen_case_04_char_balance_AB_eq_7.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + int balance = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; // Invalid size + } + + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for ( short i = 0; i < size; ++i ) { + if ( data[ i ] == 'A' ) { + balance++; + } else if ( data[ i ] == 'B' ) { + balance--; + } + } + // 4. Final condition based on internal variable(s) + if ( balance == 13 ) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_17_char_balanceAB_eq13.json b/benchmarks/iid_testing/aigen_17_char_balanceAB_eq13.json new file mode 100644 index 00000000..a2c59a81 --- /dev/null +++ b/benchmarks/iid_testing/aigen_17_char_balanceAB_eq13.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 2593, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_18_char_digitrange678_eq3.c b/benchmarks/iid_testing/aigen_18_char_digitrange678_eq3.c new file mode 100644 index 00000000..2543d78a --- /dev/null +++ b/benchmarks/iid_testing/aigen_18_char_digitrange678_eq3.c @@ -0,0 +1,41 @@ +/* File: gen_case_05_char_range_58_eq_3.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 80 + +int main() +{ + char data[MAX_SIZE]; + short size; + int special_count = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] > '5' && data[i] < '9') { // Check if digit is 6, 7, or 8 + special_count++; // 6. Static change (+1) + } + } + // 4. Final condition based on internal variable(s) + if (special_count == 3) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_18_char_digitrange678_eq3.json b/benchmarks/iid_testing/aigen_18_char_digitrange678_eq3.json new file mode 100644 index 00000000..86072272 --- /dev/null +++ b/benchmarks/iid_testing/aigen_18_char_digitrange678_eq3.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1561, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_19_char_continue_S_count_F_ge_10.c b/benchmarks/iid_testing/aigen_19_char_continue_S_count_F_ge_10.c new file mode 100644 index 00000000..9a17b34f --- /dev/null +++ b/benchmarks/iid_testing/aigen_19_char_continue_S_count_F_ge_10.c @@ -0,0 +1,44 @@ +/* File: gen_case_07_char_continue_S_count_F_ge_10.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 90 + +int main() +{ + char data[MAX_SIZE]; + short size; + int f_count = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'S') { + continue; // 3. Control flow modification + } + if (data[i] == 'F') { + f_count++; // 6. Static change + } + } + // 4. Final condition based on internal variable(s) + if (f_count >= 10) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_19_char_continue_S_count_F_ge_10.json b/benchmarks/iid_testing/aigen_19_char_continue_S_count_F_ge_10.json new file mode 100644 index 00000000..d81f1cfe --- /dev/null +++ b/benchmarks/iid_testing/aigen_19_char_continue_S_count_F_ge_10.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 2252, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_20_char_break_Q_count_P_eq_12.c b/benchmarks/iid_testing/aigen_20_char_break_Q_count_P_eq_12.c new file mode 100644 index 00000000..3462fd67 --- /dev/null +++ b/benchmarks/iid_testing/aigen_20_char_break_Q_count_P_eq_12.c @@ -0,0 +1,44 @@ +/* File: gen_case_08_char_break_Q_count_P_eq_12.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int p_count = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'Q') { + break; // 3. Control flow modification + } + if (data[i] == 'P') { + p_count++; + } + } + // 4. Final condition based on internal variable(s) + if (p_count == 12) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_20_char_break_Q_count_P_eq_12.json b/benchmarks/iid_testing/aigen_20_char_break_Q_count_P_eq_12.json new file mode 100644 index 00000000..892ee80a --- /dev/null +++ b/benchmarks/iid_testing/aigen_20_char_break_Q_count_P_eq_12.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 2243, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_21_char_multi_state_A_gt_B_plus_5.c b/benchmarks/iid_testing/aigen_21_char_multi_state_A_gt_B_plus_5.c new file mode 100644 index 00000000..919f1aba --- /dev/null +++ b/benchmarks/iid_testing/aigen_21_char_multi_state_A_gt_B_plus_5.c @@ -0,0 +1,44 @@ +/* File: gen_case_12_char_multi_state_A_gt_B_plus_5.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 100 + +int main() +{ + char data[MAX_SIZE]; + short size; + int countA = 0; + int countB = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'A') { + countA++; + } else if (data[i] == 'B') { + countB++; + } + } + // 4. Final condition based on internal variable(s) + if (countA > countB + 5) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_21_char_multi_state_A_gt_B_plus_5.json b/benchmarks/iid_testing/aigen_21_char_multi_state_A_gt_B_plus_5.json new file mode 100644 index 00000000..fbe5458f --- /dev/null +++ b/benchmarks/iid_testing/aigen_21_char_multi_state_A_gt_B_plus_5.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1396, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_22_char_countXYZ_sum_eq10.c b/benchmarks/iid_testing/aigen_22_char_countXYZ_sum_eq10.c new file mode 100644 index 00000000..4ca83f09 --- /dev/null +++ b/benchmarks/iid_testing/aigen_22_char_countXYZ_sum_eq10.c @@ -0,0 +1,47 @@ +/* File: gen_case_13_char_multi_state_XYZ_combo_eq_10.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int countX = 0; + int countY = 0; + int countZ = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'X') { + countX++; + } else if (data[i] == 'Y') { + countY++; + } else if (data[i] == 'Z') { + countZ++; + } + } + // 4. Final condition based on internal variable(s) + if ((countX + countY - countZ) == 10) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_22_char_countXYZ_sum_eq10.json b/benchmarks/iid_testing/aigen_22_char_countXYZ_sum_eq10.json new file mode 100644 index 00000000..e3074c95 --- /dev/null +++ b/benchmarks/iid_testing/aigen_22_char_countXYZ_sum_eq10.json @@ -0,0 +1,57 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1822, + "num_covered_branchings": 8, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_23_char_two_loops_A_or_B_eq_10.c b/benchmarks/iid_testing/aigen_23_char_two_loops_A_or_B_eq_10.c new file mode 100644 index 00000000..e0b73938 --- /dev/null +++ b/benchmarks/iid_testing/aigen_23_char_two_loops_A_or_B_eq_10.c @@ -0,0 +1,49 @@ +/* File: gen_case_15_char_two_loops_A_or_B_eq_10.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 100 + +int main() +{ + char data[MAX_SIZE]; + short size; + int countA = 0; + int countB = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + // Loop 1 + for (short i = 0; i < size; ++i) { + if (data[i] == 'A') { + countA++; + } + } + // Loop 2 + for (short i = 0; i < size; ++i) { + if (data[i] == 'B') { + countB++; + } + } + // 4. Final condition based on internal variable(s) + if (countA == 10 || countB == 10) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_23_char_two_loops_A_or_B_eq_10.json b/benchmarks/iid_testing/aigen_23_char_two_loops_A_or_B_eq_10.json new file mode 100644 index 00000000..333c3700 --- /dev/null +++ b/benchmarks/iid_testing/aigen_23_char_two_loops_A_or_B_eq_10.json @@ -0,0 +1,59 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3652, + "num_covered_branchings": 9, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_24_char_two_loops_dependent_ABCD_eq_7.c b/benchmarks/iid_testing/aigen_24_char_two_loops_dependent_ABCD_eq_7.c new file mode 100644 index 00000000..e28424fb --- /dev/null +++ b/benchmarks/iid_testing/aigen_24_char_two_loops_dependent_ABCD_eq_7.c @@ -0,0 +1,46 @@ +/* File: gen_case_16_char_two_loops_dependent_ABCD_eq_7.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int k = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + // Loop 1 + for (short i = 0; i < size; ++i) { + if (data[i] == 'A') ++k; + if (data[i] == 'B') --k; + } + // Loop 2 (depends on k from loop 1) + for (short i = 0; i < size; ++i) { + if (data[i] == 'C') ++k; + if (data[i] == 'D') --k; + } + // 4. Final condition based on internal variable(s) + if (k == 7) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_24_char_two_loops_dependent_ABCD_eq_7.json b/benchmarks/iid_testing/aigen_24_char_two_loops_dependent_ABCD_eq_7.json new file mode 100644 index 00000000..73888b8c --- /dev/null +++ b/benchmarks/iid_testing/aigen_24_char_two_loops_dependent_ABCD_eq_7.json @@ -0,0 +1,61 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1299, + "num_covered_branchings": 10, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_25_struct_item_status_A_ge_10.c b/benchmarks/iid_testing/aigen_25_struct_item_status_A_ge_10.c new file mode 100644 index 00000000..52f69c7a --- /dev/null +++ b/benchmarks/iid_testing/aigen_25_struct_item_status_A_ge_10.c @@ -0,0 +1,49 @@ +/* File: gen_case_24_struct_item_status_A_ge_10.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +typedef struct { + int id; + char status; // e.g., 'A'ctive, 'I'nactive +} Item; + + +#define MAX_SIZE 50 + +int main() +{ + Item inventory[MAX_SIZE]; + short num_items; + int active_count = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + num_items = __VERIFIER_nondet_short(); + if (num_items <= 0 || num_items > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < num_items; ++i) { + inventory[i].id = __VERIFIER_nondet_int(); + inventory[i].status = __VERIFIER_nondet_char(); + + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < num_items; ++i) { + if (inventory[i].status == 'A') { + active_count++; // Constant change + } + } + // 4. Final condition based on internal variable(s) + if (active_count >= 10) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_25_struct_item_status_A_ge_10.json b/benchmarks/iid_testing/aigen_25_struct_item_status_A_ge_10.json new file mode 100644 index 00000000..46f9ae80 --- /dev/null +++ b/benchmarks/iid_testing/aigen_25_struct_item_status_A_ge_10.json @@ -0,0 +1,53 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 6834, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_26_struct_dataval_balance_PN_eq_20.c b/benchmarks/iid_testing/aigen_26_struct_dataval_balance_PN_eq_20.c new file mode 100644 index 00000000..88d56f51 --- /dev/null +++ b/benchmarks/iid_testing/aigen_26_struct_dataval_balance_PN_eq_20.c @@ -0,0 +1,51 @@ +/* File: gen_case_26_struct_dataval_balance_PN_eq_20.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +typedef struct { + char code; // 'P', 'N' + int value; // irrelevant for this logic, just for struct complexity +} DataVal; + + +#define MAX_SIZE 50 + +int main() +{ + DataVal items[MAX_SIZE]; + short num_items; + int balance = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + num_items = __VERIFIER_nondet_short(); + if (num_items <= 0 || num_items > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < num_items; ++i) { + items[i].code = __VERIFIER_nondet_char(); + items[i].value = __VERIFIER_nondet_int(); + + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < num_items; ++i) { + if (items[i].code == 'P') { + balance++; + } else if (items[i].code == 'N') { + balance--; + } + } + // 4. Final condition based on internal variable(s) + if (balance == 20) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_26_struct_dataval_balance_PN_eq_20.json b/benchmarks/iid_testing/aigen_26_struct_dataval_balance_PN_eq_20.json new file mode 100644 index 00000000..7dd23d1c --- /dev/null +++ b/benchmarks/iid_testing/aigen_26_struct_dataval_balance_PN_eq_20.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 20230, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_27_char_count_X_nested_check_eq_8_11.c b/benchmarks/iid_testing/aigen_27_char_count_X_nested_check_eq_8_11.c new file mode 100644 index 00000000..c38370ca --- /dev/null +++ b/benchmarks/iid_testing/aigen_27_char_count_X_nested_check_eq_8_11.c @@ -0,0 +1,41 @@ +/* File: gen_case_28_char_count_X_nested_check_eq_8_11.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int k = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'X') { + k++; + } + } + // 4. Final condition based on internal variable(s) + if (k == 8 || k == 11) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_27_char_count_X_nested_check_eq_8_11.json b/benchmarks/iid_testing/aigen_27_char_count_X_nested_check_eq_8_11.json new file mode 100644 index 00000000..3fb5b44f --- /dev/null +++ b/benchmarks/iid_testing/aigen_27_char_count_X_nested_check_eq_8_11.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 2702, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_28_char_countA_nested_eq11.c b/benchmarks/iid_testing/aigen_28_char_countA_nested_eq11.c new file mode 100644 index 00000000..3666d1b5 --- /dev/null +++ b/benchmarks/iid_testing/aigen_28_char_countA_nested_eq11.c @@ -0,0 +1,42 @@ +/* File: gen_case_29_char_count_A_deep_nested_check.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int k = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + int i = 0; + while(i < size) { + if (data[i] == 'A') k++; + i++; + } + + // 4. Final condition based on internal variable(s) + if (k >= 6 && k > 8 && k == 11) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_28_char_countA_nested_eq11.json b/benchmarks/iid_testing/aigen_28_char_countA_nested_eq11.json new file mode 100644 index 00000000..94e96484 --- /dev/null +++ b/benchmarks/iid_testing/aigen_28_char_countA_nested_eq11.json @@ -0,0 +1,57 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 2145, + "num_covered_branchings": 8, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_29_char_countA_mul3_eq30.c b/benchmarks/iid_testing/aigen_29_char_countA_mul3_eq30.c new file mode 100644 index 00000000..fad86294 --- /dev/null +++ b/benchmarks/iid_testing/aigen_29_char_countA_mul3_eq30.c @@ -0,0 +1,39 @@ +/* File: gen_case_30_char_count_A_mult_check_eq_30.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int k = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'A') k++; + } + // 4. Final condition based on internal variable(s) + if (3 * k == 30) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_29_char_countA_mul3_eq30.json b/benchmarks/iid_testing/aigen_29_char_countA_mul3_eq30.json new file mode 100644 index 00000000..d519fa9b --- /dev/null +++ b/benchmarks/iid_testing/aigen_29_char_countA_mul3_eq30.json @@ -0,0 +1,53 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1810, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_30_char_state_dependent_inc_gt_50.c b/benchmarks/iid_testing/aigen_30_char_state_dependent_inc_gt_50.c new file mode 100644 index 00000000..3f0c10c4 --- /dev/null +++ b/benchmarks/iid_testing/aigen_30_char_state_dependent_inc_gt_50.c @@ -0,0 +1,45 @@ +/* File: gen_case_31_char_state_dependent_inc_gt_50.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 80 + +int main() +{ + char data[MAX_SIZE]; + short size; + int counter = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'X') { + if (counter % 2 == 0) { + counter += 1; // Static change +1 + } else { + counter += 2; // Static change +2 + } + } + } + // 4. Final condition based on internal variable(s) + if (counter > 50) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_30_char_state_dependent_inc_gt_50.json b/benchmarks/iid_testing/aigen_30_char_state_dependent_inc_gt_50.json new file mode 100644 index 00000000..51f24c49 --- /dev/null +++ b/benchmarks/iid_testing/aigen_30_char_state_dependent_inc_gt_50.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 11414, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_31_two_char_arrays_match_ST_gt_5.c b/benchmarks/iid_testing/aigen_31_two_char_arrays_match_ST_gt_5.c new file mode 100644 index 00000000..ad5e13b6 --- /dev/null +++ b/benchmarks/iid_testing/aigen_31_two_char_arrays_match_ST_gt_5.c @@ -0,0 +1,41 @@ +/* File: gen_case_32_two_char_arrays_match_ST_gt_5.c */ +#include + +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data1[MAX_SIZE]; + char data2[MAX_SIZE]; // Second array + short size; + int match_count = 0; // State variable + + // 1. Fill data structures in loops + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; + } + for (short i = 0; i < size; ++i) { + data1[i] = __VERIFIER_nondet_char(); + data2[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop iterating through data + for (short i = 0; i < size; ++i) { + // 3. Modify internal state based on data from both arrays + if (data1[i] == 'S' && data2[i] == 'T') { + match_count++; // 6. Static change (+1) + } + } + + // 4. Final condition based on internal variable + if (match_count > 5) { // 5. Linear check + return 1; + } else { + return 0; + } +} diff --git a/benchmarks/iid_testing/aigen_31_two_char_arrays_match_ST_gt_5.json b/benchmarks/iid_testing/aigen_31_two_char_arrays_match_ST_gt_5.json new file mode 100644 index 00000000..c0e68500 --- /dev/null +++ b/benchmarks/iid_testing/aigen_31_two_char_arrays_match_ST_gt_5.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3107, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_32_char_state_machine_XYZ_count2_gt_3.c b/benchmarks/iid_testing/aigen_32_char_state_machine_XYZ_count2_gt_3.c new file mode 100644 index 00000000..8a2235c4 --- /dev/null +++ b/benchmarks/iid_testing/aigen_32_char_state_machine_XYZ_count2_gt_3.c @@ -0,0 +1,47 @@ +/* File: gen_case_33_char_state_machine_XYZ_count2_gt_3.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 150 + +int main() +{ + char data[MAX_SIZE]; + short size; + int state = 0; + int state2_entries = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (state == 0 && data[i] == 'X') { + state = 1; + } else if (state == 1 && data[i] == 'Y') { + state = 2; + state2_entries++; // Static change +1 + } else if (data[i] == 'R') { // Reset condition + state = 0; + } + } + // 4. Final condition based on internal variable(s) + if (state2_entries > 3) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_32_char_state_machine_XYZ_count2_gt_3.json b/benchmarks/iid_testing/aigen_32_char_state_machine_XYZ_count2_gt_3.json new file mode 100644 index 00000000..8b25e86e --- /dev/null +++ b/benchmarks/iid_testing/aigen_32_char_state_machine_XYZ_count2_gt_3.json @@ -0,0 +1,61 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 6945, + "num_covered_branchings": 10, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_33_char_input_cycle_nested_A_eq_10.c b/benchmarks/iid_testing/aigen_33_char_input_cycle_nested_A_eq_10.c new file mode 100644 index 00000000..8f356607 --- /dev/null +++ b/benchmarks/iid_testing/aigen_33_char_input_cycle_nested_A_eq_10.c @@ -0,0 +1,54 @@ +/* File: gen_case_39_char_input_cycle_nested_A_eq_10.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int k = 0; + short loop_count_1 = 0; + short loop_count_2 = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + loop_count_1 = __VERIFIER_nondet_short(); + loop_count_2 = __VERIFIER_nondet_short(); + + if (loop_count_1 < 0 || loop_count_1 > 10) loop_count_1 = 3; // Bound loops + if (loop_count_2 < 0 || loop_count_2 > 10) loop_count_2 = 4; // Bound loops + + for (short index_1 = 0; index_1 < loop_count_1; ++index_1) { + for (short index_2 = 0; index_2 < loop_count_2; ++index_2) { + int i = 0; + while (i < size) { // Assuming non-null terminated for simplicity here + if (data[i] == 'A') ++k; + i++; + } + } + } + + // 4. Final condition based on internal variable(s) + if (k == 10) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_33_char_input_cycle_nested_A_eq_10.json b/benchmarks/iid_testing/aigen_33_char_input_cycle_nested_A_eq_10.json new file mode 100644 index 00000000..19710352 --- /dev/null +++ b/benchmarks/iid_testing/aigen_33_char_input_cycle_nested_A_eq_10.json @@ -0,0 +1,65 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 31124, + "num_covered_branchings": 12, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0, + 12, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 8, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_34_char_count_K_lt_10.c b/benchmarks/iid_testing/aigen_34_char_count_K_lt_10.c new file mode 100644 index 00000000..7ed18e53 --- /dev/null +++ b/benchmarks/iid_testing/aigen_34_char_count_K_lt_10.c @@ -0,0 +1,41 @@ +/* File: gen_case_40_char_count_K_lt_10.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 100 + +int main() +{ + char data[MAX_SIZE]; + short size; + int count = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'K') { + count++; + } + } + // 4. Final condition based on internal variable(s) + if (count < 10) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_34_char_count_K_lt_10.json b/benchmarks/iid_testing/aigen_34_char_count_K_lt_10.json new file mode 100644 index 00000000..e9f7da97 --- /dev/null +++ b/benchmarks/iid_testing/aigen_34_char_count_K_lt_10.json @@ -0,0 +1,53 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 2246, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_35_char_balance_Uinc2_Rdec1_gt10.c b/benchmarks/iid_testing/aigen_35_char_balance_Uinc2_Rdec1_gt10.c new file mode 100644 index 00000000..74153b12 --- /dev/null +++ b/benchmarks/iid_testing/aigen_35_char_balance_Uinc2_Rdec1_gt10.c @@ -0,0 +1,43 @@ +/* File: gen_case_42_char_balance_U2_R1_gt_10.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 100 + +int main() +{ + char data[MAX_SIZE]; + short size; + int balance = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'U') { + balance += 2; + } else if (data[i] == 'R') { + balance -= 1; + } + } + // 4. Final condition based on internal variable(s) + if (balance > 10) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_35_char_balance_Uinc2_Rdec1_gt10.json b/benchmarks/iid_testing/aigen_35_char_balance_Uinc2_Rdec1_gt10.json new file mode 100644 index 00000000..fbe5458f --- /dev/null +++ b/benchmarks/iid_testing/aigen_35_char_balance_Uinc2_Rdec1_gt10.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1396, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_36_char_count_H_eq_12.c b/benchmarks/iid_testing/aigen_36_char_count_H_eq_12.c new file mode 100644 index 00000000..6874fbba --- /dev/null +++ b/benchmarks/iid_testing/aigen_36_char_count_H_eq_12.c @@ -0,0 +1,41 @@ +/* File: gen_case_44_char_count_H_eq_12.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int count = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'H') { + count++; + } + } + // 4. Final condition based on internal variable(s) + if (count == 12) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_36_char_count_H_eq_12.json b/benchmarks/iid_testing/aigen_36_char_count_H_eq_12.json new file mode 100644 index 00000000..0df483ec --- /dev/null +++ b/benchmarks/iid_testing/aigen_36_char_count_H_eq_12.json @@ -0,0 +1,53 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 2308, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_37_char_balance_Z1_A1_gt_15.c b/benchmarks/iid_testing/aigen_37_char_balance_Z1_A1_gt_15.c new file mode 100644 index 00000000..13232609 --- /dev/null +++ b/benchmarks/iid_testing/aigen_37_char_balance_Z1_A1_gt_15.c @@ -0,0 +1,43 @@ +/* File: gen_case_49_char_balance_Z1_A1_gt_15.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 80 + +int main() +{ + char data[MAX_SIZE]; + short size; + int balance = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'Z') { + balance += 1; + } else if (data[i] == 'A') { + balance -= 1; + } + } + // 4. Final condition based on internal variable(s) + if (balance > 15) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_37_char_balance_Z1_A1_gt_15.json b/benchmarks/iid_testing/aigen_37_char_balance_Z1_A1_gt_15.json new file mode 100644 index 00000000..45057f6e --- /dev/null +++ b/benchmarks/iid_testing/aigen_37_char_balance_Z1_A1_gt_15.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3897, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_38_char_count_R_gt_12.c b/benchmarks/iid_testing/aigen_38_char_count_R_gt_12.c new file mode 100644 index 00000000..516a913c --- /dev/null +++ b/benchmarks/iid_testing/aigen_38_char_count_R_gt_12.c @@ -0,0 +1,41 @@ +/* File: gen_case_50_char_count_R_gt_12.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int count = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'R') { + count++; + } + } + // 4. Final condition based on internal variable(s) + if (count > 12) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_38_char_count_R_gt_12.json b/benchmarks/iid_testing/aigen_38_char_count_R_gt_12.json new file mode 100644 index 00000000..f100021d --- /dev/null +++ b/benchmarks/iid_testing/aigen_38_char_count_R_gt_12.json @@ -0,0 +1,53 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 2587, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_39_char_multi_state_R_gt_P_plus_2.c b/benchmarks/iid_testing/aigen_39_char_multi_state_R_gt_P_plus_2.c new file mode 100644 index 00000000..08218c7a --- /dev/null +++ b/benchmarks/iid_testing/aigen_39_char_multi_state_R_gt_P_plus_2.c @@ -0,0 +1,44 @@ +/* File: gen_case_51_char_multi_state_R_gt_P_plus_2.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 80 + +int main() +{ + char data[MAX_SIZE]; + short size; + int countR = 0; + int countP = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'R') { + countR++; + } else if (data[i] == 'P') { + countP++; + } + } + // 4. Final condition based on internal variable(s) + if (countR > countP + 2) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_39_char_multi_state_R_gt_P_plus_2.json b/benchmarks/iid_testing/aigen_39_char_multi_state_R_gt_P_plus_2.json new file mode 100644 index 00000000..30ef0fcf --- /dev/null +++ b/benchmarks/iid_testing/aigen_39_char_multi_state_R_gt_P_plus_2.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1050, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_40_char_two_loops_balancePQ_ge7.c b/benchmarks/iid_testing/aigen_40_char_two_loops_balancePQ_ge7.c new file mode 100644 index 00000000..5378e594 --- /dev/null +++ b/benchmarks/iid_testing/aigen_40_char_two_loops_balancePQ_ge7.c @@ -0,0 +1,44 @@ +/* File: gen_case_53_char_two_loops_dep_PQ_ge_7.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int k = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + // Loop 1 + for (short i = 0; i < size; ++i) { + if (data[i] == 'P') ++k; + } + // Loop 2 + for (short i = 0; i < size; ++i) { + if (data[i] == 'Q') --k; + } + // 4. Final condition based on internal variable(s) + if (k >= 7) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_40_char_two_loops_balancePQ_ge7.json b/benchmarks/iid_testing/aigen_40_char_two_loops_balancePQ_ge7.json new file mode 100644 index 00000000..acd1cf89 --- /dev/null +++ b/benchmarks/iid_testing/aigen_40_char_two_loops_balancePQ_ge7.json @@ -0,0 +1,57 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1357, + "num_covered_branchings": 8, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_41_char_count_S_ge_5.c b/benchmarks/iid_testing/aigen_41_char_count_S_ge_5.c new file mode 100644 index 00000000..4dbc52fe --- /dev/null +++ b/benchmarks/iid_testing/aigen_41_char_count_S_ge_5.c @@ -0,0 +1,41 @@ +/* File: gen_case_54_char_count_S_ge_5.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 80 + +int main() +{ + char data[MAX_SIZE]; + short size; + int count = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'S') { + count++; + } + } + // 4. Final condition based on internal variable(s) + if (count >= 5) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/aigen_41_char_count_S_ge_5.json b/benchmarks/iid_testing/aigen_41_char_count_S_ge_5.json new file mode 100644 index 00000000..1b3a9fdf --- /dev/null +++ b/benchmarks/iid_testing/aigen_41_char_count_S_ge_5.json @@ -0,0 +1,53 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1312, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_balanceAB_eq10.c b/benchmarks/iid_testing/char_array_balanceAB_eq10.c new file mode 100644 index 00000000..4fe1d4a0 --- /dev/null +++ b/benchmarks/iid_testing/char_array_balanceAB_eq10.c @@ -0,0 +1,37 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + if ( s[ i ] == 'B' ) // ID: 7 + --k; + ++i; + } + if ( k == 10 ) // ID: 8 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/char_array_balanceAB_eq10.json b/benchmarks/iid_testing/char_array_balanceAB_eq10.json new file mode 100644 index 00000000..b44807f3 --- /dev/null +++ b/benchmarks/iid_testing/char_array_balanceAB_eq10.json @@ -0,0 +1,62 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3007, + "num_covered_branchings": 8, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_balance_A2_B1_eq5.c b/benchmarks/iid_testing/char_array_balance_A2_B1_eq5.c new file mode 100644 index 00000000..d63fb0b8 --- /dev/null +++ b/benchmarks/iid_testing/char_array_balance_A2_B1_eq5.c @@ -0,0 +1,37 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + k += 2; + if ( s[ i ] == 'B' ) // ID: 7 + --k; + ++i; + } + if ( k == 5 ) // ID: 8 + return 1; + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_balance_A2_B1_eq5.json b/benchmarks/iid_testing/char_array_balance_A2_B1_eq5.json new file mode 100644 index 00000000..dca556b5 --- /dev/null +++ b/benchmarks/iid_testing/char_array_balance_A2_B1_eq5.json @@ -0,0 +1,62 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3908, + "num_covered_branchings": 8, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_balance_many_chars_eq10.c b/benchmarks/iid_testing/char_array_balance_many_chars_eq10.c new file mode 100644 index 00000000..bcca4ed8 --- /dev/null +++ b/benchmarks/iid_testing/char_array_balance_many_chars_eq10.c @@ -0,0 +1,53 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + if ( s[ i ] == 'B' ) // ID: 7 + --k; + if ( s[ i ] == 'C' ) // ID: 8 + ++k; + if ( s[ i ] == 'D' ) // ID: 9 + --k; + if ( s[ i ] == 'E' ) // ID: 10 + ++k; + if ( s[ i ] == 'F' ) // ID: 11 + --k; + if ( s[ i ] == 'G' ) // ID: 12 + ++k; + if ( s[ i ] == 'H' ) // ID: 13 + --k; + if ( s[ i ] == 'I' ) // ID: 14 + ++k; + if ( s[ i ] == 'J' ) // ID: 15 + --k; + ++i; + } + if ( k == 10 ) // ID: 16 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/char_array_balance_many_chars_eq10.json b/benchmarks/iid_testing/char_array_balance_many_chars_eq10.json new file mode 100644 index 00000000..658096c1 --- /dev/null +++ b/benchmarks/iid_testing/char_array_balance_many_chars_eq10.json @@ -0,0 +1,78 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3061, + "num_covered_branchings": 16, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0, + 12, + 0, + 13, + 0, + 14, + 0, + 15, + 0, + 16, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 10, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_conditional_AB_balance_eq10.c b/benchmarks/iid_testing/char_array_conditional_AB_balance_eq10.c new file mode 100644 index 00000000..982004eb --- /dev/null +++ b/benchmarks/iid_testing/char_array_conditional_AB_balance_eq10.c @@ -0,0 +1,40 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + if ( s[ i ] % 2 == 0 ) // ID: 7 + { + if ( s[ i ] == 'B' ) // ID: 8 + --k; + } + ++i; + } + if ( k == 10 ) // ID: 9 + return 1; + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_conditional_AB_balance_eq10.json b/benchmarks/iid_testing/char_array_conditional_AB_balance_eq10.json new file mode 100644 index 00000000..541e5b8d --- /dev/null +++ b/benchmarks/iid_testing/char_array_conditional_AB_balance_eq10.json @@ -0,0 +1,64 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3226, + "num_covered_branchings": 9, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 5, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_conditional_balanceAB_eq10.c b/benchmarks/iid_testing/char_array_conditional_balanceAB_eq10.c new file mode 100644 index 00000000..991c55a2 --- /dev/null +++ b/benchmarks/iid_testing/char_array_conditional_balanceAB_eq10.c @@ -0,0 +1,43 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + + if ( s[ 0 ] > 20 ) // ID: 5 + { + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 6 + break; + if ( s[ i ] == 'A' ) // ID: 7 + ++k; + if ( s[ i ] == 'B' ) // ID: 8 + --k; + ++i; + } + } + + if ( k == 10 ) // ID: 9 + return 1; + + return 0; + } +} diff --git a/benchmarks/iid_testing/char_array_conditional_balanceAB_eq10.json b/benchmarks/iid_testing/char_array_conditional_balanceAB_eq10.json new file mode 100644 index 00000000..fd6e7b10 --- /dev/null +++ b/benchmarks/iid_testing/char_array_conditional_balanceAB_eq10.json @@ -0,0 +1,59 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3000, + "num_covered_branchings": 9, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 6, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_countA_deep_nested_complex.c b/benchmarks/iid_testing/char_array_countA_deep_nested_complex.c new file mode 100644 index 00000000..de13fdd7 --- /dev/null +++ b/benchmarks/iid_testing/char_array_countA_deep_nested_complex.c @@ -0,0 +1,52 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + ++i; + } + + if ( k >= 6 ) // ID: 7 + { + if ( k > 8 ) // ID: 8 + { + if ( 12 < k ) // ID: 9 + { + if ( k == 13 ) // ID: 10 + return 1; + } + + if ( k >= 16 ) // ID: 11 + { + if ( k == 30 ) // ID: 12 + return 1; + } + } + } + + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_countA_deep_nested_complex.json b/benchmarks/iid_testing/char_array_countA_deep_nested_complex.json new file mode 100644 index 00000000..26037999 --- /dev/null +++ b/benchmarks/iid_testing/char_array_countA_deep_nested_complex.json @@ -0,0 +1,70 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 40502, + "num_covered_branchings": 12, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0, + 12, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 6, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_countA_eq10.c b/benchmarks/iid_testing/char_array_countA_eq10.c new file mode 100644 index 00000000..aa117a8c --- /dev/null +++ b/benchmarks/iid_testing/char_array_countA_eq10.c @@ -0,0 +1,35 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + ++i; + } + if ( k == 10 ) // ID: 7 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/char_array_countA_eq10.json b/benchmarks/iid_testing/char_array_countA_eq10.json new file mode 100644 index 00000000..c106e488 --- /dev/null +++ b/benchmarks/iid_testing/char_array_countA_eq10.json @@ -0,0 +1,61 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3001, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_countA_eq10_OR_countB_eq10_v2.c b/benchmarks/iid_testing/char_array_countA_eq10_OR_countB_eq10_v2.c new file mode 100644 index 00000000..3eecf0dc --- /dev/null +++ b/benchmarks/iid_testing/char_array_countA_eq10_OR_countB_eq10_v2.c @@ -0,0 +1,48 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + ++i; + } + if ( k == 10 ) // ID: 7 + return 1; + + i = 0; + int l = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 8 + break; + if ( s[ i ] == 'B' ) // ID: 9 + ++l; + ++i; + } + if ( l == 10 ) // ID: 10 + return 1; + + return 0; + } +} diff --git a/benchmarks/iid_testing/char_array_countA_eq10_OR_countB_eq10_v2.json b/benchmarks/iid_testing/char_array_countA_eq10_OR_countB_eq10_v2.json new file mode 100644 index 00000000..d92f6ffa --- /dev/null +++ b/benchmarks/iid_testing/char_array_countA_eq10_OR_countB_eq10_v2.json @@ -0,0 +1,66 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 5489, + "num_covered_branchings": 10, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_countA_eq8_or_10_or_11.c b/benchmarks/iid_testing/char_array_countA_eq8_or_10_or_11.c new file mode 100644 index 00000000..2fe177aa --- /dev/null +++ b/benchmarks/iid_testing/char_array_countA_eq8_or_10_or_11.c @@ -0,0 +1,42 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + ++i; + } + if ( k == 8 ) // ID: 7 + return 1; + + if ( k == 10 ) // ID: 8 + return 1; + + if ( k == 11 ) // ID: 9 + return 1; + + return 0; + } +} diff --git a/benchmarks/iid_testing/char_array_countA_eq8_or_10_or_11.json b/benchmarks/iid_testing/char_array_countA_eq8_or_10_or_11.json new file mode 100644 index 00000000..e0eaf0e4 --- /dev/null +++ b/benchmarks/iid_testing/char_array_countA_eq8_or_10_or_11.json @@ -0,0 +1,64 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 6913, + "num_covered_branchings": 9, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_countA_gt10.c b/benchmarks/iid_testing/char_array_countA_gt10.c new file mode 100644 index 00000000..d196bddb --- /dev/null +++ b/benchmarks/iid_testing/char_array_countA_gt10.c @@ -0,0 +1,35 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + ++i; + } + if ( 10 < k ) // ID: 7 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/char_array_countA_gt10.json b/benchmarks/iid_testing/char_array_countA_gt10.json new file mode 100644 index 00000000..b18e29d2 --- /dev/null +++ b/benchmarks/iid_testing/char_array_countA_gt10.json @@ -0,0 +1,61 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3764, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_countA_mul4_eq20.c b/benchmarks/iid_testing/char_array_countA_mul4_eq20.c new file mode 100644 index 00000000..2f9fdccf --- /dev/null +++ b/benchmarks/iid_testing/char_array_countA_mul4_eq20.c @@ -0,0 +1,35 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + ++i; + } + if ( 4 * k == 20 ) // ID: 7 + return 1; + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_countA_mul4_eq20.json b/benchmarks/iid_testing/char_array_countA_mul4_eq20.json new file mode 100644 index 00000000..9db8041b --- /dev/null +++ b/benchmarks/iid_testing/char_array_countA_mul4_eq20.json @@ -0,0 +1,60 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1473, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_countA_nested_check_eq11.c b/benchmarks/iid_testing/char_array_countA_nested_check_eq11.c new file mode 100644 index 00000000..cc420c35 --- /dev/null +++ b/benchmarks/iid_testing/char_array_countA_nested_check_eq11.c @@ -0,0 +1,43 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + ++i; + } + + // if ( k % 2 == 0 ) // ID: 7 + if ( k >= 6 ) // ID: 7 + { + if ( k > 8 ) // ID: 8 + { + if ( k == 11 ) // ID: 9 + return 1; + } + } + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_countA_nested_check_eq11.json b/benchmarks/iid_testing/char_array_countA_nested_check_eq11.json new file mode 100644 index 00000000..3cbff683 --- /dev/null +++ b/benchmarks/iid_testing/char_array_countA_nested_check_eq11.json @@ -0,0 +1,64 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 7492, + "num_covered_branchings": 9, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_countA_vs_countB_plus5.c b/benchmarks/iid_testing/char_array_countA_vs_countB_plus5.c new file mode 100644 index 00000000..b8b85d56 --- /dev/null +++ b/benchmarks/iid_testing/char_array_countA_vs_countB_plus5.c @@ -0,0 +1,36 @@ +#include + +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 100 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + int countA = 0; + int countB = 0; + + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + for ( short i = 0; i < size; ++i ) { + if ( data[ i ] == 'A' ) { + countA++; + } else if ( data[ i ] == 'B' ) { + countB++; + } + } + + if ( countA > countB + 5 ) { + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_countA_vs_countB_plus5.json b/benchmarks/iid_testing/char_array_countA_vs_countB_plus5.json new file mode 100644 index 00000000..ad9eb90b --- /dev/null +++ b/benchmarks/iid_testing/char_array_countA_vs_countB_plus5.json @@ -0,0 +1,56 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1396, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_doublenested_countA_eq10.c b/benchmarks/iid_testing/char_array_doublenested_countA_eq10.c new file mode 100644 index 00000000..d12a2229 --- /dev/null +++ b/benchmarks/iid_testing/char_array_doublenested_countA_eq10.c @@ -0,0 +1,54 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + short loop_count_1; + loop_count_1 = __VERIFIER_nondet_short(); + short loop_count_2; + loop_count_2 = __VERIFIER_nondet_short(); + + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + if ( loop_count_1 > 20 ) // ID: 5 + return -1; + + if ( loop_count_2 > 20 ) // ID: 6 + return -1; + + int i = 0, k = 0; + + for ( short index_1 = 0; index_1 < loop_count_1; ++index_1 ) { // ID: 7 + for ( short index_2 = 0; index_2 < loop_count_2; ++index_2 ) { // ID: 8 + i = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 9 + break; + if ( s[ i ] == 'A' ) // ID: 10 + ++k; + ++i; + } + } + } + + if ( k == 10 ) // ID: 11 + return 1; + + return 0; + } +} diff --git a/benchmarks/iid_testing/char_array_doublenested_countA_eq10.json b/benchmarks/iid_testing/char_array_doublenested_countA_eq10.json new file mode 100644 index 00000000..6981c0b7 --- /dev/null +++ b/benchmarks/iid_testing/char_array_doublenested_countA_eq10.json @@ -0,0 +1,63 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3013, + "num_covered_branchings": 11, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 8, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_dummyload_countA_eq10.c b/benchmarks/iid_testing/char_array_dummyload_countA_eq10.c new file mode 100644 index 00000000..09d0aa3d --- /dev/null +++ b/benchmarks/iid_testing/char_array_dummyload_countA_eq10.c @@ -0,0 +1,41 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + char tmp; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) { // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + tmp = __VERIFIER_nondet_char(); + tmp = __VERIFIER_nondet_char(); + tmp = __VERIFIER_nondet_char(); + tmp = __VERIFIER_nondet_char(); + } + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + ++i; + } + if ( k == 10 ) // ID: 7 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/char_array_dummyload_countA_eq10.json b/benchmarks/iid_testing/char_array_dummyload_countA_eq10.json new file mode 100644 index 00000000..cc7cbb2f --- /dev/null +++ b/benchmarks/iid_testing/char_array_dummyload_countA_eq10.json @@ -0,0 +1,61 @@ +{ + "args": { + "max_executions": 20000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 13558, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_ifelse_balance_eq5.c b/benchmarks/iid_testing/char_array_ifelse_balance_eq5.c new file mode 100644 index 00000000..d1a570b8 --- /dev/null +++ b/benchmarks/iid_testing/char_array_ifelse_balance_eq5.c @@ -0,0 +1,40 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + { + ++k; + } else { + --k; + } + + ++i; + } + if ( k == 5 ) // ID: 7 + return 1; + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_ifelse_balance_eq5.json b/benchmarks/iid_testing/char_array_ifelse_balance_eq5.json new file mode 100644 index 00000000..9db8041b --- /dev/null +++ b/benchmarks/iid_testing/char_array_ifelse_balance_eq5.json @@ -0,0 +1,60 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1473, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_indexed_charA_check.c b/benchmarks/iid_testing/char_array_indexed_charA_check.c new file mode 100644 index 00000000..093a68a0 --- /dev/null +++ b/benchmarks/iid_testing/char_array_indexed_charA_check.c @@ -0,0 +1,55 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + if ( s[ 0 ] == 'A' ) // ID: 5 + return 1; + + if ( s[ 1 ] == 'A' ) // ID: 6 + return 1; + + if ( s[ 2 ] == 'A' ) // ID: 7 + return 1; + + if ( s[ 3 ] == 'A' ) // ID: 8 + return 1; + + if ( s[ 4 ] == 'A' ) // ID: 9 + return 1; + + if ( s[ 5 ] == 'A' ) // ID: 10 + return 1; + + if ( s[ 6 ] == 'A' ) // ID: 11 + return 1; + + if ( s[ 7 ] == 'A' ) // ID: 12 + return 1; + + if ( s[ 8 ] == 'A' ) // ID: 13 + return 1; + + if ( s[ 9 ] == 'A' ) // ID: 14 + return 1; + + return 0; + } +} diff --git a/benchmarks/iid_testing/char_array_indexed_charA_check.json b/benchmarks/iid_testing/char_array_indexed_charA_check.json new file mode 100644 index 00000000..57e11217 --- /dev/null +++ b/benchmarks/iid_testing/char_array_indexed_charA_check.json @@ -0,0 +1,69 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 410, + "num_covered_branchings": 14, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0, + 12, + 0, + 13, + 0, + 14, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 10, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_kAB_lXY_k_eq10.c b/benchmarks/iid_testing/char_array_kAB_lXY_k_eq10.c new file mode 100644 index 00000000..e98f1023 --- /dev/null +++ b/benchmarks/iid_testing/char_array_kAB_lXY_k_eq10.c @@ -0,0 +1,48 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + if ( s[ i ] == 'B' ) // ID: 7 + --k; + ++i; + } + i = 0; + int l = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 8 + break; + if ( s[ i ] == 'X' ) // ID: 9 + ++l; + if ( s[ i ] == 'Y' ) // ID: 10 + --l; + ++i; + } + if ( k == 10 ) // ID: 11 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/char_array_kAB_lXY_k_eq10.json b/benchmarks/iid_testing/char_array_kAB_lXY_k_eq10.json new file mode 100644 index 00000000..9d65986d --- /dev/null +++ b/benchmarks/iid_testing/char_array_kAB_lXY_k_eq10.json @@ -0,0 +1,68 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3019, + "num_covered_branchings": 11, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_kAB_lXY_k_eq10_v2.c b/benchmarks/iid_testing/char_array_kAB_lXY_k_eq10_v2.c new file mode 100644 index 00000000..33c40aff --- /dev/null +++ b/benchmarks/iid_testing/char_array_kAB_lXY_k_eq10_v2.c @@ -0,0 +1,50 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + if ( s[ i ] == 'B' ) // ID: 7 + --k; + ++i; + } + + i = 0; + int l = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 8 + break; + if ( s[ i ] == 'X' ) // ID: 9 + ++l; + if ( s[ i ] == 'Y' ) // ID: 10 + --l; + ++i; + } + + if ( k == 10 ) // ID: 11 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/char_array_kAB_lXY_k_eq10_v2.json b/benchmarks/iid_testing/char_array_kAB_lXY_k_eq10_v2.json new file mode 100644 index 00000000..9d65986d --- /dev/null +++ b/benchmarks/iid_testing/char_array_kAB_lXY_k_eq10_v2.json @@ -0,0 +1,68 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3019, + "num_covered_branchings": 11, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_load2by2_countA_eq10.c b/benchmarks/iid_testing/char_array_load2by2_countA_eq10.c new file mode 100644 index 00000000..740c61a6 --- /dev/null +++ b/benchmarks/iid_testing/char_array_load2by2_countA_eq10.c @@ -0,0 +1,40 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n % 2 == 1 ) // ID: 3 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 4 + return -1; + for ( short i = 0; i < n / 2; ++i ) // ID: 5 + { + s[ 2 * i ] = __VERIFIER_nondet_char(); + s[ 2 * i + 1 ] = __VERIFIER_nondet_char(); + } + if ( s[ n - 1 ] != '\0' ) // ID: 6 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + ++i; + } + if ( k == 10 ) // ID: 7 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/char_array_load2by2_countA_eq10.json b/benchmarks/iid_testing/char_array_load2by2_countA_eq10.json new file mode 100644 index 00000000..7c9094b3 --- /dev/null +++ b/benchmarks/iid_testing/char_array_load2by2_countA_eq10.json @@ -0,0 +1,62 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 5316, + "num_covered_branchings": 8, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 5, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_multi_predicate_loops.c b/benchmarks/iid_testing/char_array_multi_predicate_loops.c new file mode 100644 index 00000000..bed5237b --- /dev/null +++ b/benchmarks/iid_testing/char_array_multi_predicate_loops.c @@ -0,0 +1,85 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + ++i; + } + if ( k == 10 ) // ID: 7 + return 1; + + i = 0; + k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 8 + break; + if ( s[ i ] == 'B' ) // ID: 9 + ++k; + ++i; + } + if ( k > 10 ) // ID: 10 + return 1; + + i = 0; + k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 11 + break; + if ( s[ i ] == 'C' ) // ID: 12 + ++k; + ++i; + } + if ( k >= 10 ) // ID: 13 + return 1; + + i = 0; + k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 14 + break; + if ( s[ i ] == 'D' ) // ID: 15 + ++k; + ++i; + } + // if ( 10 > k ) // ID: 16 + if ( 10 < k ) // ID: 16 + return 1; + + i = 0; + k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 17 + break; + if ( s[ i ] == 'E' ) // ID: 18 + ++k; + ++i; + } + if ( 10 <= k ) // ID: 19 + return 1; + + return 0; + } +} diff --git a/benchmarks/iid_testing/char_array_multi_predicate_loops.json b/benchmarks/iid_testing/char_array_multi_predicate_loops.json new file mode 100644 index 00000000..ffe39afc --- /dev/null +++ b/benchmarks/iid_testing/char_array_multi_predicate_loops.json @@ -0,0 +1,84 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 14204, + "num_covered_branchings": 19, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0, + 12, + 0, + 13, + 0, + 14, + 0, + 15, + 0, + 16, + 0, + 17, + 0, + 18, + 0, + 19, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 5, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 5, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_multibreak_countA_eq10.c b/benchmarks/iid_testing/char_array_multibreak_countA_eq10.c new file mode 100644 index 00000000..730b17f3 --- /dev/null +++ b/benchmarks/iid_testing/char_array_multibreak_countA_eq10.c @@ -0,0 +1,39 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] > 'R' ) // ID: 6 + break; + if ( s[ i ] == 'F' ) // ID: 7 + break; + if ( s[ i ] == 'A' ) // ID: 8 + ++k; + ++i; + } + if ( k == 10 ) // ID: 9 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/char_array_multibreak_countA_eq10.json b/benchmarks/iid_testing/char_array_multibreak_countA_eq10.json new file mode 100644 index 00000000..dc336b80 --- /dev/null +++ b/benchmarks/iid_testing/char_array_multibreak_countA_eq10.json @@ -0,0 +1,64 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3008, + "num_covered_branchings": 9, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 5, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_multinested_AB_balance_eq10.c b/benchmarks/iid_testing/char_array_multinested_AB_balance_eq10.c new file mode 100644 index 00000000..27261d3b --- /dev/null +++ b/benchmarks/iid_testing/char_array_multinested_AB_balance_eq10.c @@ -0,0 +1,63 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + short loop_count_1; + loop_count_1 = __VERIFIER_nondet_short(); + short loop_count_2; + loop_count_2 = __VERIFIER_nondet_short(); + + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + if ( loop_count_1 > 20 ) // ID: 5 + return -1; + + if ( loop_count_2 > 20 ) // ID: 6 + return -1; + + int i = 0, k = 0; + + for ( short index_1 = 0; index_1 < loop_count_1; ++index_1 ) { // ID: 7 + for ( short index_2 = 0; index_2 < loop_count_2; ++index_2 ) { // ID: 8 + i = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 9 + break; + if ( s[ i ] == 'A' ) // ID: 10 + ++k; + ++i; + } + } + + i = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 11 + break; + if ( s[ i ] == 'B' ) // ID: 12 + --k; + ++i; + } + } + + if ( k == 10 ) // ID: 13 + return 1; + + return 0; + } +} diff --git a/benchmarks/iid_testing/char_array_multinested_AB_balance_eq10.json b/benchmarks/iid_testing/char_array_multinested_AB_balance_eq10.json new file mode 100644 index 00000000..0554eca6 --- /dev/null +++ b/benchmarks/iid_testing/char_array_multinested_AB_balance_eq10.json @@ -0,0 +1,67 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 2365, + "num_covered_branchings": 13, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0, + 12, + 0, + 13, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 9, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_nested_loop_balanceAB_eq10.c b/benchmarks/iid_testing/char_array_nested_loop_balanceAB_eq10.c new file mode 100644 index 00000000..ef0f1e72 --- /dev/null +++ b/benchmarks/iid_testing/char_array_nested_loop_balanceAB_eq10.c @@ -0,0 +1,48 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + short loop_count; + loop_count = __VERIFIER_nondet_short(); + if ( loop_count > 20 ) // ID: 5 + return -1; + + int i = 0, k = 0; + + for ( short index = 0; index < loop_count; ++index ) { // ID: 6 + i = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 7 + break; + if ( s[ i ] == 'A' ) // ID: 8 + ++k; + if ( s[ i ] == 'B' ) // ID: 9 + --k; + ++i; + } + } + + if ( k == 10 ) // ID: 10 + return 1; + + return 0; + } +} diff --git a/benchmarks/iid_testing/char_array_nested_loop_balanceAB_eq10.json b/benchmarks/iid_testing/char_array_nested_loop_balanceAB_eq10.json new file mode 100644 index 00000000..f67e8fdc --- /dev/null +++ b/benchmarks/iid_testing/char_array_nested_loop_balanceAB_eq10.json @@ -0,0 +1,61 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1110, + "num_covered_branchings": 10, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 6, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_nested_loop_countA_eq10.c b/benchmarks/iid_testing/char_array_nested_loop_countA_eq10.c new file mode 100644 index 00000000..3763aa5a --- /dev/null +++ b/benchmarks/iid_testing/char_array_nested_loop_countA_eq10.c @@ -0,0 +1,47 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + short loop_count; + loop_count = __VERIFIER_nondet_short(); + + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + if ( loop_count > 20 ) // ID: 5 + return -1; + + int i = 0, k = 0; + + for ( short index = 0; index < loop_count; ++index ) { // ID: 6 + i = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 7 + break; + if ( s[ i ] == 'A' ) // ID: 8 + ++k; + ++i; + } + } + + if ( k == 10 ) // ID: 9 + return 1; + + return 0; + } +} diff --git a/benchmarks/iid_testing/char_array_nested_loop_countA_eq10.json b/benchmarks/iid_testing/char_array_nested_loop_countA_eq10.json new file mode 100644 index 00000000..bd182c8f --- /dev/null +++ b/benchmarks/iid_testing/char_array_nested_loop_countA_eq10.json @@ -0,0 +1,59 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1117, + "num_covered_branchings": 9, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 6, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_offset20_countA_eq10.c b/benchmarks/iid_testing/char_array_offset20_countA_eq10.c new file mode 100644 index 00000000..8e0d0935 --- /dev/null +++ b/benchmarks/iid_testing/char_array_offset20_countA_eq10.c @@ -0,0 +1,39 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + for ( short i = 0; i < 50; ++i ) // ID: 1 + s[ i ] = '\0'; + + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 2 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 3 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 4 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 5 + return -1; + } + { + int i = 20, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 6 + break; + if ( s[ i ] == 'A' ) // ID: 7 + ++k; + ++i; + } + if ( k == 10 ) // ID: 8 + return 1; + + return 0; + } +} diff --git a/benchmarks/iid_testing/char_array_offset20_countA_eq10.json b/benchmarks/iid_testing/char_array_offset20_countA_eq10.json new file mode 100644 index 00000000..c1f5c1ff --- /dev/null +++ b/benchmarks/iid_testing/char_array_offset20_countA_eq10.json @@ -0,0 +1,60 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 8652, + "num_covered_branchings": 7, + "covered_branchings": [ + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_strlen_eq19.c b/benchmarks/iid_testing/char_array_strlen_eq19.c new file mode 100644 index 00000000..94b5c0cc --- /dev/null +++ b/benchmarks/iid_testing/char_array_strlen_eq19.c @@ -0,0 +1,36 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 7 + --k; + ++k; + ++i; + } + if ( k == 19 ) // ID: 8 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/char_array_strlen_eq19.json b/benchmarks/iid_testing/char_array_strlen_eq19.json new file mode 100644 index 00000000..5cd1e344 --- /dev/null +++ b/benchmarks/iid_testing/char_array_strlen_eq19.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 6789, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 5, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_sum_chars_eq500.c b/benchmarks/iid_testing/char_array_sum_chars_eq500.c new file mode 100644 index 00000000..b70849fc --- /dev/null +++ b/benchmarks/iid_testing/char_array_sum_chars_eq500.c @@ -0,0 +1,36 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + k += s[ i ]; + ++i; + } + + if ( k == 500 ) // ID: 6 + return 1; + + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_sum_chars_eq500.json b/benchmarks/iid_testing/char_array_sum_chars_eq500.json new file mode 100644 index 00000000..fa1ac97a --- /dev/null +++ b/benchmarks/iid_testing/char_array_sum_chars_eq500.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 20000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 72926, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "num_uncovered_branchings": 0, + "uncovered_branchings": [], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_two_pass_balanceABCD_eq7.c b/benchmarks/iid_testing/char_array_two_pass_balanceABCD_eq7.c new file mode 100644 index 00000000..4bd33d7f --- /dev/null +++ b/benchmarks/iid_testing/char_array_two_pass_balanceABCD_eq7.c @@ -0,0 +1,49 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + if ( s[ i ] == 'B' ) // ID: 7 + --k; + ++i; + } + + i = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 8 + break; + if ( s[ i ] == 'C' ) // ID: 9 + ++k; + if ( s[ i ] == 'D' ) // ID: 10 + --k; + ++i; + } + + if ( k == 7 ) // ID: 11 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/char_array_two_pass_balanceABCD_eq7.json b/benchmarks/iid_testing/char_array_two_pass_balanceABCD_eq7.json new file mode 100644 index 00000000..c7c8e9a9 --- /dev/null +++ b/benchmarks/iid_testing/char_array_two_pass_balanceABCD_eq7.json @@ -0,0 +1,68 @@ +{ + "args": { + "max_executions": 15000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 2085, + "num_covered_branchings": 11, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_two_pass_balanceAB_eq10.c b/benchmarks/iid_testing/char_array_two_pass_balanceAB_eq10.c new file mode 100644 index 00000000..1aeb4025 --- /dev/null +++ b/benchmarks/iid_testing/char_array_two_pass_balanceAB_eq10.c @@ -0,0 +1,49 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + if ( s[ i ] == 'B' ) // ID: 7 + --k; + ++i; + } + + i = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 8 + break; + if ( s[ i ] == 'A' ) // ID: 9 + ++k; + if ( s[ i ] == 'B' ) // ID: 10 + --k; + ++i; + } + + if ( k == 10 ) // ID: 11 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/char_array_two_pass_balanceAB_eq10.json b/benchmarks/iid_testing/char_array_two_pass_balanceAB_eq10.json new file mode 100644 index 00000000..745179d8 --- /dev/null +++ b/benchmarks/iid_testing/char_array_two_pass_balanceAB_eq10.json @@ -0,0 +1,68 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1479, + "num_covered_branchings": 11, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_two_arrays_matchST_gt10.c b/benchmarks/iid_testing/char_two_arrays_matchST_gt10.c new file mode 100644 index 00000000..6ac68521 --- /dev/null +++ b/benchmarks/iid_testing/char_two_arrays_matchST_gt10.c @@ -0,0 +1,36 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 50 + +int main() +{ + char data1[ MAX_SIZE ]; + char data2[ MAX_SIZE ]; + short size; + int match_count = 0; + + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data1[ i ] = __VERIFIER_nondet_char(); + data2[ i ] = __VERIFIER_nondet_char(); + } + + for ( short i = 0; i < size; ++i ) { + if ( data1[ i ] == 'S' && data2[ i ] == 'T' ) { + match_count++; + } + } + + if ( match_count > 10 ) { + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_two_arrays_matchST_gt10.json b/benchmarks/iid_testing/char_two_arrays_matchST_gt10.json new file mode 100644 index 00000000..6ffe0f77 --- /dev/null +++ b/benchmarks/iid_testing/char_two_arrays_matchST_gt10.json @@ -0,0 +1,62 @@ +{ + "args": { + "max_executions": 20000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 6897, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "num_uncovered_branchings": 0, + "uncovered_branchings": [], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/custom_struct.c b/benchmarks/iid_testing/custom_struct.c new file mode 100644 index 00000000..11676e9c --- /dev/null +++ b/benchmarks/iid_testing/custom_struct.c @@ -0,0 +1,40 @@ +#include + +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +typedef struct { + int id; + char status; +} Item; + +#define MAX_ITEMS 40 + +int main() +{ + Item inventory[ MAX_ITEMS ]; + short num_items; + int active_count = 0; + + num_items = __VERIFIER_nondet_short(); + if ( num_items < 0 || num_items > MAX_ITEMS ) { + return -1; + } + for ( short i = 0; i < num_items; ++i ) { + inventory[ i ].id = __VERIFIER_nondet_int(); + inventory[ i ].status = __VERIFIER_nondet_char(); + } + + for ( short i = 0; i < num_items; ++i ) { + if ( inventory[ i ].status == 'A' ) { + active_count++; + } + } + + if ( active_count >= 10 ) { + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/custom_struct.json b/benchmarks/iid_testing/custom_struct.json new file mode 100644 index 00000000..6da49d82 --- /dev/null +++ b/benchmarks/iid_testing/custom_struct.json @@ -0,0 +1,58 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 6508, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/digit_range.c b/benchmarks/iid_testing/digit_range.c new file mode 100644 index 00000000..cac65be4 --- /dev/null +++ b/benchmarks/iid_testing/digit_range.c @@ -0,0 +1,33 @@ +#include + +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 80 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + int special_count = 0; + for ( short i = 0; i < size; ++i ) { + if ( data[ i ] > '5' && data[ i ] < '9' ) { + special_count++; + } + } + + if ( special_count == 3 ) { + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/digit_range.json b/benchmarks/iid_testing/digit_range.json new file mode 100644 index 00000000..abdff7aa --- /dev/null +++ b/benchmarks/iid_testing/digit_range.json @@ -0,0 +1,60 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1561, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/greater_in_increase.c b/benchmarks/iid_testing/greater_in_increase.c new file mode 100644 index 00000000..3d172d6d --- /dev/null +++ b/benchmarks/iid_testing/greater_in_increase.c @@ -0,0 +1,41 @@ +/* File: gen_case_18_int_count_pos_gt_20.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + int data[MAX_SIZE]; + short size; + int pos_count = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_int(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] > 0) { + pos_count++; + } + } + // 4. Final condition based on internal variable(s) + if (pos_count > 20) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/greater_in_increase.json b/benchmarks/iid_testing/greater_in_increase.json new file mode 100644 index 00000000..33664200 --- /dev/null +++ b/benchmarks/iid_testing/greater_in_increase.json @@ -0,0 +1,48 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 25029, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/linked_list.c b/benchmarks/iid_testing/linked_list.c new file mode 100644 index 00000000..a0d59dac --- /dev/null +++ b/benchmarks/iid_testing/linked_list.c @@ -0,0 +1,55 @@ +#include +#include // For NULL potentially, though not strictly needed here + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern int __VERIFIER_nondet_int(); + +// Simple Node structure for linked list +typedef struct Node { + int value; + struct Node* next; +} Node; + +#define MAX_NODES 50 + +int main() +{ + Node nodes[ MAX_NODES ]; + Node* head = NULL; + Node* current = NULL; + short size; + int count_gt_10 = 0; + + size = __VERIFIER_nondet_short(); + if ( size < 0 || size > MAX_NODES ) { + return -1; + } + + for ( short i = 0; i < size; ++i ) { + nodes[ i ].value = __VERIFIER_nondet_int(); + nodes[ i ].next = NULL; + if ( head == NULL ) { + head = &nodes[ i ]; + current = head; + } else { + current->next = &nodes[ i ]; + current = current->next; + } + } + + current = head; + while ( current != NULL ) { + if ( current->value == 10 ) { + count_gt_10++; + } + + current = current->next; + } + + if ( count_gt_10 >= 10 ) { + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/linked_list.json b/benchmarks/iid_testing/linked_list.json new file mode 100644 index 00000000..af0667ee --- /dev/null +++ b/benchmarks/iid_testing/linked_list.json @@ -0,0 +1,61 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 5088, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/nested_modulo.c b/benchmarks/iid_testing/nested_modulo.c new file mode 100644 index 00000000..02494ed1 --- /dev/null +++ b/benchmarks/iid_testing/nested_modulo.c @@ -0,0 +1,37 @@ +#include + +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 80 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + int counter = 0; + + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + for ( short i = 0; i < size; ++i ) { + if ( data[ i ] == 'X' ) { + if ( counter % 2 == 0 ) { + counter += 1; + } else { + counter += 2; + } + } + } + + if ( counter > 50 ) { + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/nested_modulo.json b/benchmarks/iid_testing/nested_modulo.json new file mode 100644 index 00000000..5921b69a --- /dev/null +++ b/benchmarks/iid_testing/nested_modulo.json @@ -0,0 +1,61 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 11414, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/pointer.c b/benchmarks/iid_testing/pointer.c new file mode 100644 index 00000000..ce76348f --- /dev/null +++ b/benchmarks/iid_testing/pointer.c @@ -0,0 +1,40 @@ +#include +#include + +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 60 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + int p_count = 0; + + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + + char* ptr = data; + char* end_ptr = data + size; + + while ( ptr < end_ptr ) { + if ( *ptr == 'P' ) { + p_count++; + } + ptr++; + } + if ( p_count == 5 ) { + return 1; + } else { + return 0; + } +} diff --git a/benchmarks/iid_testing/pointer.json b/benchmarks/iid_testing/pointer.json new file mode 100644 index 00000000..bdeead36 --- /dev/null +++ b/benchmarks/iid_testing/pointer.json @@ -0,0 +1,53 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 923, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/{ReachSafety-BitVectors}--{bitvector-loops}{verisec_sendmail_tTflag_arr_one_loop}.c b/benchmarks/iid_testing/{ReachSafety-BitVectors}--{bitvector-loops}{verisec_sendmail_tTflag_arr_one_loop}.c new file mode 100644 index 00000000..be8b8ed5 --- /dev/null +++ b/benchmarks/iid_testing/{ReachSafety-BitVectors}--{bitvector-loops}{verisec_sendmail_tTflag_arr_one_loop}.c @@ -0,0 +1,47 @@ +extern void abort( void ); +extern void __assert_fail( const char*, const char*, unsigned int, const char* ) + __attribute__( ( __nothrow__, __leaf__ ) ) __attribute__( ( __noreturn__ ) ); +void reach_error() { __assert_fail( "0", "verisec_sendmail_tTflag_arr_one_loop.c", 3, "reach_error" ); } + +extern char __VERIFIER_nondet_char(); + +// void __VERIFIER_assert( int cond ) +// { +// if ( !( cond ) ) { +// ERROR: { +// reach_error(); +// abort(); +// } +// } +// return; +// } + +int main( void ) +{ + char in[ 11 ]; // = "3277192070"; + char* s; + unsigned char c; + int i, j; + int idx_in; + for ( i = 0; i < 11; i++ ) + in[ i ] = __VERIFIER_nondet_char(); + in[ 10 ] = 0; + idx_in = 0; + s = in; + i = 0; + c = in[ idx_in ]; + while ( ( '0' <= c ) && ( c <= '9' ) ) { + j = c - '0'; + i = i * 10U + j; + idx_in++; + c = in[ idx_in ]; + } + /* BAD */ + // + // __VERIFIER_assert( i >= 0 ); + + if ( idx_in != 10 ) + return 1; + + return 0; +} diff --git a/benchmarks/iid_testing/{ReachSafety-BitVectors}--{bitvector-loops}{verisec_sendmail_tTflag_arr_one_loop}.json b/benchmarks/iid_testing/{ReachSafety-BitVectors}--{bitvector-loops}{verisec_sendmail_tTflag_arr_one_loop}.json new file mode 100644 index 00000000..99521c89 --- /dev/null +++ b/benchmarks/iid_testing/{ReachSafety-BitVectors}--{bitvector-loops}{verisec_sendmail_tTflag_arr_one_loop}.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 2296, + "num_covered_branchings": 4, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/{ReachSafety-Loops}--{nla-digbench}{ps4}.c b/benchmarks/iid_testing/{ReachSafety-Loops}--{nla-digbench}{ps4}.c new file mode 100644 index 00000000..9c60e2f3 --- /dev/null +++ b/benchmarks/iid_testing/{ReachSafety-Loops}--{nla-digbench}{ps4}.c @@ -0,0 +1,55 @@ +extern void abort( void ); +extern void __assert_fail( const char*, const char*, unsigned int, const char* ) + __attribute__( ( __nothrow__, __leaf__ ) ) __attribute__( ( __noreturn__ ) ); +void reach_error() { __assert_fail( "0", "ps4.c", 3, "reach_error" ); } +extern int __VERIFIER_nondet_int( void ); +extern void abort( void ); +// void assume_abort_if_not(int cond) +// { +// if (!cond) +// { +// abort(); +// } +// } +// void __VERIFIER_assert(int cond) +// { +// if (!(cond)) +// { +// ERROR: +// { +// reach_error(); +// } +// } +// return; +// } + +int main() +{ + int k, y, x, c; + k = __VERIFIER_nondet_int(); + + y = 0; + x = 0; + c = 0; + + while ( 1 ) { + // __VERIFIER_assert(4 * x - y * y * y * y - 2 * y * y * y - y * y == 0); + + if ( !( c < k ) ) + break; + + c = c + 1; + y = y + 1; + x = y * y * y + x; + } + // __VERIFIER_assert(k * y - (y * y) == 0); + // __VERIFIER_assert(4 * x - y * y * y * y - 2 * y * y * y - y * y == 0); + + if ( c == 14 ) + return 0; + + if ( y == 28 ) + return 0; + + return 0; +} diff --git a/benchmarks/iid_testing/{ReachSafety-Loops}--{nla-digbench}{ps4}.json b/benchmarks/iid_testing/{ReachSafety-Loops}--{nla-digbench}{ps4}.json new file mode 100644 index 00000000..c1eac294 --- /dev/null +++ b/benchmarks/iid_testing/{ReachSafety-Loops}--{nla-digbench}{ps4}.json @@ -0,0 +1,48 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 118, + "num_covered_branchings": 3, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 1 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4423b9b7..922c109e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -20,13 +20,15 @@ add_subdirectory(./iomodels) message(" iomodels") add_subdirectory(./connection) message(" connection") - + if(FIZZ_BUILD_ALSO_TOOLS STREQUAL "Yes") - include_directories( - "${PROJECT_SOURCE_DIR}/src/fuzzing/include" - "${PROJECT_SOURCE_DIR}/src/tools" - ) - add_subdirectory(./fuzzing) - message(" fuzzing") - add_subdirectory(./tools) + if(FIZZ_BUILD_LIBS_32_BIT STREQUAL "No") + include_directories( + "${PROJECT_SOURCE_DIR}/src/fuzzing/include" + "${PROJECT_SOURCE_DIR}/src/tools" + ) + add_subdirectory(./fuzzing) + message(" fuzzing") + add_subdirectory(./tools) + endif() endif() diff --git a/src/connection/CMakeLists.txt b/src/connection/CMakeLists.txt index 7ca16635..7b062ad3 100644 --- a/src/connection/CMakeLists.txt +++ b/src/connection/CMakeLists.txt @@ -7,6 +7,8 @@ add_library(${THIS_TARGET_NAME} ./include/connection/client.hpp ./src/client.cpp + ./include/connection/medium.hpp + ./include/connection/shared_memory.hpp ./src/shared_memory.cpp @@ -30,6 +32,8 @@ add_library(${THIS_TARGET_NAME} ./src/benchmark_executor.cpp ) +target_compile_options(${THIS_TARGET_NAME} PUBLIC "-Wno-deprecated") + set_target_properties(${THIS_TARGET_NAME} PROPERTIES DEBUG_OUTPUT_NAME "${THIS_TARGET_NAME}_${CMAKE_SYSTEM_NAME}_Debug" RELEASE_OUTPUT_NAME "${THIS_TARGET_NAME}_${CMAKE_SYSTEM_NAME}_Release" diff --git a/src/connection/include/connection/medium.hpp b/src/connection/include/connection/medium.hpp new file mode 100644 index 00000000..4195d284 --- /dev/null +++ b/src/connection/include/connection/medium.hpp @@ -0,0 +1,23 @@ +#ifndef CONNECTION_MEDIUM_HPP_INCLUDED +# define CONNECTION_MEDIUM_HPP_INCLUDED + +# include + +namespace connection { + + +struct medium +{ + virtual ~medium() {} + virtual void clear() {} + virtual bool can_accept_bytes(std::size_t n) const { return true; } + virtual bool can_deliver_bytes(std::size_t n) const { return true; } + virtual void accept_bytes(const void* src, std::size_t n) {} + virtual void deliver_bytes(void* dest, std::size_t n) {} + virtual void set_termination(instrumentation::target_termination termination) {} +}; + + +} + +#endif \ No newline at end of file diff --git a/src/connection/include/connection/message.hpp b/src/connection/include/connection/message.hpp index 14033aff..51b1e2a5 100644 --- a/src/connection/include/connection/message.hpp +++ b/src/connection/include/connection/message.hpp @@ -1,6 +1,7 @@ #ifndef CONNECTION_MESSAGE_HPP_INCLUDED # define CONNECTION_MESSAGE_HPP_INCLUDED +# include # include # include @@ -17,16 +18,18 @@ friend struct message; }; -struct message +struct message : public medium { + message() : medium() {} + natural_32_bit size(); - void clear(); + void clear() override; bool empty(); - bool can_accept_bytes(size_t n) const; - bool can_deliver_bytes(size_t n) const; - void accept_bytes(const void* src, size_t n); - void deliver_bytes(void* dest, size_t n); + bool can_accept_bytes(size_t n) const override; + bool can_deliver_bytes(size_t n) const override; + void accept_bytes(const void* src, size_t n) override; + void deliver_bytes(void* dest, size_t n) override; bool exhausted() const; @@ -49,9 +52,9 @@ struct message message& operator>>(std::string& dest); - message_header header; + message_header header{}; private: - vecu8 bytes; + vecu8 bytes{}; natural_32_bit cursor = 0; friend struct connection; diff --git a/src/connection/include/connection/shared_memory.hpp b/src/connection/include/connection/shared_memory.hpp index 7cab5e20..c59320d5 100644 --- a/src/connection/include/connection/shared_memory.hpp +++ b/src/connection/include/connection/shared_memory.hpp @@ -1,6 +1,7 @@ #ifndef CONNECTION_SHARED_MEMORY_HPP_INCLUDED # define CONNECTION_SHARED_MEMORY_HPP_INCLUDED +# include # include # include # include @@ -13,29 +14,32 @@ namespace connection { -class shared_memory { +class shared_memory : public medium { inline static const char* segment_name = "SBT-Fizzer_Shared_Memory"; - boost::interprocess::shared_memory_object shm; - boost::interprocess::mapped_region region; + boost::interprocess::shared_memory_object shm{}; + boost::interprocess::mapped_region region{}; natural_32_bit cursor = 0; natural_8_bit* memory = nullptr; natural_32_bit* saved = nullptr; public: + + shared_memory() : medium() {} + natural_32_bit get_size() const; void set_size(natural_32_bit bytes); - void clear(); + void clear() override; void open_or_create(); void map_region(); static void remove(); - bool can_accept_bytes(size_t n) const; - bool can_deliver_bytes(size_t n) const; + bool can_accept_bytes(size_t n) const override; + bool can_deliver_bytes(size_t n) const override; - void accept_bytes(const void* src, size_t n); - void deliver_bytes(void* dest, size_t n); + void accept_bytes(const void* src, size_t n) override; + void deliver_bytes(void* dest, size_t n) override; void accept_bytes(message& src); void deliver_bytes(message& dest); @@ -45,7 +49,7 @@ class shared_memory { /*Interprets the first two bytes as termination type*/ std::optional get_termination() const; /*Overwrites the first two bytes to set termination type*/ - void set_termination(instrumentation::target_termination termination); + void set_termination(instrumentation::target_termination termination) override; template::value, int>::type = 0> shared_memory& operator<<(const T& src) diff --git a/src/connection/src/target_executor.cpp b/src/connection/src/target_executor.cpp index 20d3b42a..727cf01b 100644 --- a/src/connection/src/target_executor.cpp +++ b/src/connection/src/target_executor.cpp @@ -36,7 +36,7 @@ void target_executor::init_shared_memory(std::size_t size) { void target_executor::execute_target() { using namespace std::chrono_literals; - bp::child target = bp::child(target_invocation, bp::std_out > bp::null); + bp::child target = bp::child(target_invocation, bp::std_out > bp::null, bp::std_err > bp::null); if (!wait_for_wrapper(target, std::chrono::milliseconds(timeout_ms))) { target.terminate(); get_shared_memory().set_termination(target_termination::timeout); diff --git a/src/fuzzing/CMakeLists.txt b/src/fuzzing/CMakeLists.txt index 00b2778c..8245ef50 100644 --- a/src/fuzzing/CMakeLists.txt +++ b/src/fuzzing/CMakeLists.txt @@ -48,8 +48,10 @@ add_library(${THIS_TARGET_NAME} ./include/fuzzing/execution_record.hpp ./include/fuzzing/execution_record_writer.hpp ./src/execution_record_writer.cpp - ) + ./include/fuzzing/iid_vector_analysis.hpp + ./src/iid_vector_analysis.cpp +) set_target_properties(${THIS_TARGET_NAME} PROPERTIES DEBUG_OUTPUT_NAME "${THIS_TARGET_NAME}_${CMAKE_SYSTEM_NAME}_Debug" RELEASE_OUTPUT_NAME "${THIS_TARGET_NAME}_${CMAKE_SYSTEM_NAME}_Release" diff --git a/src/fuzzing/include/fuzzing/analysis_outcomes.hpp b/src/fuzzing/include/fuzzing/analysis_outcomes.hpp index 96e96e2a..f2c3f6f5 100644 --- a/src/fuzzing/include/fuzzing/analysis_outcomes.hpp +++ b/src/fuzzing/include/fuzzing/analysis_outcomes.hpp @@ -21,23 +21,30 @@ struct analysis_outcomes CLIENT_COMMUNICATION_ERROR, UNCLASSIFIED_ERROR }; + + struct output_statistics + { + natural_32_bit num_generated_tests{ 0U }; + natural_32_bit num_crashes{ 0U }; + natural_32_bit num_boundary_violations{ 0U }; + }; + TERMINATION_TYPE termination_type{ TERMINATION_TYPE::NORMAL }; fuzzer::TERMINATION_REASON termination_reason{ // Valid only if 'termination_type == NORMAL'. fuzzer::TERMINATION_REASON::ALL_REACHABLE_BRANCHINGS_COVERED }; std::string error_message{}; // Valid only if 'termination_type != NORMAL'. natural_32_bit num_executions{ 0U }; - long num_elapsed_seconds{ 0 }; + float_64_bit num_elapsed_seconds{ 0.0 }; std::vector covered_branchings{}; std::vector uncovered_branchings{}; sensitivity_analysis::performance_statistics sensitivity_statistics{}; typed_minimization_analysis::performance_statistics typed_minimization_statistics{}; minimization_analysis::performance_statistics minimization_statistics{}; bitshare_analysis::performance_statistics bitshare_statistics{}; - fuzzer::performance_statistics statistics{}; - natural_32_bit num_generated_tests{ 0U }; - natural_32_bit num_crashes{ 0U }; - natural_32_bit num_boundary_violations{ 0U }; + fuzzer::performance_statistics fuzzer_statistics{}; + std::unordered_map output_statistics{}; + iid_vector_analysis_statistics iid_vector_analysis_statistics{}; }; diff --git a/src/fuzzing/include/fuzzing/branching_node.hpp b/src/fuzzing/include/fuzzing/branching_node.hpp index e1a2a2f7..6b5ea9eb 100644 --- a/src/fuzzing/include/fuzzing/branching_node.hpp +++ b/src/fuzzing/include/fuzzing/branching_node.hpp @@ -43,7 +43,8 @@ struct branching_node branching_function_value_type best_coverage_value_, branching_function_value_type best_summary_value_, natural_32_bit execution_number, - bool xor_like_branching_function_ + bool xor_like_branching_function_, + BRANCHING_PREDICATE branching_predicate_ ) : id{ id_ } , trace_index{ trace_index_ } @@ -70,6 +71,7 @@ struct branching_node , sensitive_stdin_bits{} , xor_like_branching_function{ xor_like_branching_function_ } + , branching_predicate{ branching_predicate_ } , closed{ false } @@ -108,6 +110,8 @@ struct branching_node bool is_closed() const { return closed; } void set_closed(bool const state = true) { closed = state; } + int get_depth() const; + guid_type guid() const { return guid__; } location_id id; @@ -136,6 +140,8 @@ struct branching_node bool xor_like_branching_function; + BRANCHING_PREDICATE branching_predicate; + bool closed; trace_index_type max_successors_trace_index; diff --git a/src/fuzzing/include/fuzzing/dump.hpp b/src/fuzzing/include/fuzzing/dump.hpp index 10ec4631..1ad46119 100644 --- a/src/fuzzing/include/fuzzing/dump.hpp +++ b/src/fuzzing/include/fuzzing/dump.hpp @@ -30,6 +30,7 @@ void save_fuzzing_configuration( termination_info const& terminator ); +void save_iid_vector_analysis(std::ostream& ostr, analysis_outcomes const& results); void print_analysis_outcomes(std::ostream& ostr, analysis_outcomes const& results); void log_analysis_outcomes(analysis_outcomes const& results); void save_analysis_outcomes( diff --git a/src/fuzzing/include/fuzzing/execution_record.hpp b/src/fuzzing/include/fuzzing/execution_record.hpp index c663f555..aaad9290 100644 --- a/src/fuzzing/include/fuzzing/execution_record.hpp +++ b/src/fuzzing/include/fuzzing/execution_record.hpp @@ -19,7 +19,8 @@ struct execution_record static execution_flags constexpr MEDIUM_OVERFLOW = 1 << 4; static execution_flags constexpr EMPTY_STARTUP_TRACE = 1 << 5; - execution_flags flags { 0 }; + execution_flags flags { 0 }; + std::string analysis_name {}; vecu8 stdin_bytes {}; input_types_vector stdin_types {}; execution_path path {}; diff --git a/src/fuzzing/include/fuzzing/fuzzer.hpp b/src/fuzzing/include/fuzzing/fuzzer.hpp index 3db126f7..2fa79dd5 100644 --- a/src/fuzzing/include/fuzzing/fuzzer.hpp +++ b/src/fuzzing/include/fuzzing/fuzzer.hpp @@ -4,6 +4,7 @@ # include # include # include +# include # include # include # include @@ -17,6 +18,8 @@ # include # include # include +# include +# include namespace fuzzing { @@ -62,11 +65,11 @@ struct fuzzer final termination_info const& get_termination_info() const { return termination_props; } - long num_remaining_driver_executions() const { return (long)termination_props.max_executions - (long)num_driver_executions; } - long num_remaining_seconds() const { return (long)termination_props.max_seconds - get_elapsed_seconds(); } + natural_32_bit num_remaining_driver_executions() const { return termination_props.max_executions - get_performed_driver_executions(); } + float_64_bit num_remaining_seconds() const { return (float_64_bit)termination_props.max_seconds - get_elapsed_seconds(); } natural_32_bit get_performed_driver_executions() const { return num_driver_executions; } - long get_elapsed_seconds() const { return (long)std::chrono::duration_cast(time_point_current - time_point_start).count(); } + float_64_bit get_elapsed_seconds() const { return std::chrono::duration(time_point_current - time_point_start).count(); } std::unordered_set const& get_covered_branchings() const { return covered_branchings; } std::unordered_set const& get_uncovered_branchings() const { return uncovered_branchings; } @@ -74,13 +77,14 @@ struct fuzzer final bool can_make_progress() const { return state != FINISHED; } bool round_begin(TERMINATION_REASON& termination_reason); - execution_record::execution_flags round_end(); + std::pair round_end(); sensitivity_analysis::performance_statistics const& get_sensitivity_statistics() const { return sensitivity.get_statistics(); } typed_minimization_analysis::performance_statistics const& get_typed_minimization_statistics() const { return typed_minimization.get_statistics(); } minimization_analysis::performance_statistics const& get_minimization_statistics() const { return minimization.get_statistics(); } bitshare_analysis::performance_statistics const& get_bitshare_statistics() const { return bitshare.get_statistics(); } performance_statistics const& get_fuzzer_statistics() const { return statistics; } + iid_vector_analysis_statistics get_iid_vector_analysis_statistics() const { return iid_dependences.get_stats(); } private: @@ -194,13 +198,15 @@ struct fuzzer final using histogram_of_false_direction_probabilities = std::unordered_map; using probability_generators_for_locations = std::unordered_map >; +public: struct loop_boundary_props { branching_node* entry; branching_node* exit; // branching_node* successor; }; - + +private: struct iid_pivot_props { std::vector loop_boundaries; @@ -217,18 +223,21 @@ struct fuzzer final mutable random_generator_for_natural_32_bit generator_for_pivot_selection; }; + static std::string const& get_analysis_name_from_state(STATE state); + static void update_close_flags_from(branching_node* node); static std::vector const& get_input_width_classes(); static std::unordered_set const& get_input_width_classes_set(); static natural_32_bit get_input_width_class(natural_32_bit num_input_bytes); static natural_32_bit get_input_width_class_index(natural_32_bit num_input_bytes); - +public: static void detect_loops_along_path_to_node( branching_node* const end_node, std::unordered_map >& loop_heads_to_bodies, std::vector* loops ); +private: static void compute_loop_boundaries( std::vector const& loops, std::vector& loop_boundaries @@ -248,6 +257,11 @@ struct fuzzer final histogram_of_false_direction_probabilities& histogram ); + static branching_node* select_start_node_for_monte_carlo_search_with_vector( + generated_path const& path, + std::vector const& loop_boundaries, + branching_node* fallback_node + ); static branching_node* select_start_node_for_monte_carlo_search( std::vector const& loop_boundaries, random_generator_for_natural_32_bit& random_generator, @@ -268,7 +282,8 @@ struct fuzzer final branching_node* root, histogram_of_false_direction_probabilities const& histogram, probability_generators_for_locations const& generators, - probability_generator_random_uniform& location_miss_generator + probability_generator_random_uniform& location_miss_generator, + generated_path& path ); static std::pair monte_carlo_backward_search( branching_node* const start_node, @@ -283,14 +298,21 @@ struct fuzzer final probability_generators_for_locations const& generators, probability_generator_random_uniform& location_miss_generator ); + static branching_node* monte_carlo_step_with_path( + branching_node* const pivot, + histogram_of_false_direction_probabilities const& histogram, + probability_generators_for_locations const& generators, + probability_generator_random_uniform& location_miss_generator, + generated_path& path + ); - void generate_next_input(vecb& stdin_bits); + bool generate_next_input(vecb& stdin_bits, TERMINATION_REASON& termination_reason); execution_record::execution_flags process_execution_results(); void do_cleanup(); void collect_iid_pivots_from_sensitivity_results(); void select_next_state(); - branching_node* select_iid_coverage_target() const; + branching_node* select_iid_coverage_target(); void remove_leaf_branching_node(branching_node* node); bool apply_coverage_failures_with_hope(); @@ -310,8 +332,9 @@ struct fuzzer final primary_coverage_target_branchings primary_coverage_targets; std::unordered_map iid_pivots; + iid_dependencies iid_dependences; - std::unordered_set coverage_failures_with_hope; // EXPLAIN + std::unordered_set coverage_failures_with_hope; STATE state; sensitivity_analysis sensitivity; diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp new file mode 100644 index 00000000..a1ad3b86 --- /dev/null +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -0,0 +1,493 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + + +namespace fuzzing +{ +template < typename T > +struct mean_counter { + T mean; + int count; + + void add( T value ); +}; + +struct node_id_with_direction { + location_id::id_type node_id; + bool branching_direction; + + auto operator<=>( node_id_with_direction const& other ) const; + bool operator==( node_id_with_direction const& other ) const = default; + friend std::ostream& operator<<( std::ostream& os, const node_id_with_direction& nav ) + { + return os << nav.node_id << " " << ( nav.branching_direction ? "True" : "False" ); + } +}; + +enum generation_state { + STATE_NOT_COVERED, + STATE_GENERATING_ARTIFICIAL_DATA, + STATE_GENERATION_MORE, + STATE_COVERED, + STATE_GENERATION_DATA_FOR_NEXT_NODE, + + STATE_COVERED_BY_OTHER +}; + +enum failed_generation_method { + METHOD_NONE, + METHOD_GENERATE_FROM_OTHER_NODE, + METHOD_GENERATE_ARTIFICIAL_DATA, + METHOD_DO_NOT_GENERATE +}; + +struct iid_node_generations_stats { + int method_calls = 0; + int generation_starts = 0; + int successful_generations = 0; + int successful_generations_artificial_data = 0; + int failed_generations = 0; + + int generated_for_other_node_count = 0; + int generate_artificial_data_count = 0; + + // Not saved values, because they are cleared multiple times through out the run + int failed_generations_in_row = 0; + + int generated_after_covered_max = 0; + int generated_after_covered = 0; + + int generated_for_other_node = 0; + int generated_for_other_node_max = 0; + + int generate_artificial_data = 0; + int generate_artificial_data_max = 0; + + int do_not_generate_counter = 0; + + generation_state state = generation_state::STATE_NOT_COVERED; + + failed_generation_method last_failed_method = METHOD_NONE; +}; + +struct loaded_bits_props { + mean_counter< float > average_bits_read; + natural_32_bit minimal_bit_offset = std::numeric_limits< natural_32_bit >::max(); + mean_counter< float > average_bits_used; +}; + +struct loop_head_properties { + int count; +}; + +struct loop_properties { + std::optional< node_id_with_direction > chosen_loop_head; + std::map< node_id_with_direction, loop_head_properties > heads; + std::set< node_id_with_direction > bodies; + + bool is_loading_loop; + std::set< node_id_with_direction > nodes_dependent_by_loading; + mean_counter< float > loaded_bits_per_loop; + std::map< location_id::id_type, loaded_bits_props > bits_read_by_node; + + bool is_same( const std::unordered_set< location_id::id_type >& other_ids ) const; + const std::unordered_set< location_id::id_type >& get_all_ids() const; + const std::unordered_set< location_id::id_type >& get_loop_head_ids() const; + location_id::id_type get_smallest_loop_head_id() const; + std::optional< location_id::id_type > get_smallest_body_id() const; + location_id::id_type get_smallest_id() const; + void set_chosen_loop_head(); + void update_stored_ids(); + +private: + std::unordered_set< location_id::id_type > all_ids = {}; + std::unordered_set< location_id::id_type > loop_head_ids = {}; +}; + +struct loop_dependencies { + std::vector< loop_properties > loops; + + loop_properties& get_props( const std::unordered_set< location_id::id_type >& ids, + location_id::id_type loop_head_id ); + void merge_properties(); + const loop_properties& get_props_by_loop_head_id( location_id::id_type loop_head_id ) const; + loop_properties& get_props_by_loop_head_id( location_id::id_type loop_head_id ); + std::set< location_id::id_type > get_loop_heads( bool include_loading_loops ) const; + bool compute_node_subsets_for_computation( std::set< node_id_with_direction >& computation_subset, + const std::unordered_set< location_id::id_type >& matrix_ids ) const; + void print_dependencies() const; +}; + +struct iid_vector_analysis_statistics_per_node { + iid_node_generations_stats generation_stats; + std::pair< std::size_t, std::size_t > vector_dimensions; + std::vector< location_id::id_type > node_ids; + bool matrix_generated; +}; + +struct iid_vector_analysis_statistics { + std::map< location_id, iid_vector_analysis_statistics_per_node > iid_nodes_stats; + std::vector< location_id::id_type > ignored_node_ids; + std::vector< location_id > covered_node_ids; + loop_dependencies loop_to_properties; + int processed_nodes; + int dependencies_computed; +}; + + +struct node_counts { + int left_count; + int right_count; + + int get_max_count() const; + int get_total_count() const { return left_count + right_count; } +}; + + +struct node_props_in_path { + node_props_in_path( node_counts computed_counts, bool is_loop_head, bool loop_head_end_direction ) + : computed_counts( computed_counts ) + , taken_counts( { 0, 0 } ) + , is_loop_head( is_loop_head ) + , loop_head_end_direction( loop_head_end_direction ) + {} + + bool get_desired_direction() const; + bool can_go_direction( bool direction ) const; + void go_direction( bool direction ); + bool can_take_next_direction() const; + + float_32_bit get_false_direction_probability() const; + + friend std::ostream& operator<<( std::ostream& os, const node_props_in_path& eq ) + { + os << "L-" << eq.computed_counts.left_count << " R-" << eq.computed_counts.right_count; + if ( eq.is_loop_head ) { + os << " " << ( eq.loop_head_end_direction ? "R" : "L" ); + } + + return os; + } + +private: + node_counts computed_counts; + node_counts taken_counts; + bool is_loop_head; + bool loop_head_end_direction; + + bool get_preferred_direction_loop_head() const; +}; + + +struct generated_path { + generated_path( std::map< location_id::id_type, node_props_in_path > path ) + : path( std::move( path ) ) + , iid_node_id( std::nullopt ) + {} + + generated_path() = default; + + bool contains( location_id::id_type id ) const; + std::map< location_id::id_type, node_props_in_path > get_path() const; + node_props_in_path& get_props( location_id::id_type id ) { return path.at( id ); } + std::optional< location_id > get_iid_node_id() const { return iid_node_id; } + void set_iid_node_id( location_id iid_node_id ) { this->iid_node_id = iid_node_id; } + bool empty() const { return path.empty(); } + + friend std::ostream& operator<<( std::ostream& os, const generated_path& eq ) + { + for ( const auto& [ id, props ] : eq.path ) { + os << id << ": " << props << std::endl; + } + + return os; + } + +private: + std::map< location_id::id_type, node_props_in_path > path; + std::optional< location_id > iid_node_id; +}; + +struct equation { + equation( std::vector< int > values, double best_value ) + : values( std::move( values ) ) + , best_value( best_value ) + {} + + std::vector< int > values; + double best_value; + + equation operator+( const equation& other ) const; + equation operator+( int scalar ) const; + equation operator-( const equation& other ) const; + equation operator*( int scalar ) const; + equation operator*( double scalar ) const; + equation operator/( const equation& other ) const; + auto operator<=>( const equation& other ) const = default; + bool operator==( const equation& other ) const = default; + + equation add_to_positive( int value ) const; + equation add_to_values( const equation& other ) const; + int simplify_by_gcd(); + int get_vector_size() const; + int get_one_way_branching_count() const; + int get_biggest_value() const; + bool is_any_negative() const; + bool same_values() const; + bool is_linear_dependent( const equation& other ) const; + + + friend std::ostream& operator<<( std::ostream& os, const equation& eq ) + { + for ( int i = 0; i < eq.values.size(); ++i ) { + os << ( i ? " " : "" ) << eq.values[ i ]; + } + return os << " -> " << eq.best_value; + } +}; +} // namespace fuzzing + +namespace std +{ +template <> +struct hash< fuzzing::node_id_with_direction > { + std::size_t operator()( const fuzzing::node_id_with_direction& key ) const noexcept + { + std::size_t h1 = std::hash< location_id::id_type >{}( key.node_id ); + std::size_t h2 = std::hash< bool >{}( key.branching_direction ); + return h1 ^ ( h2 << 1 ); + } +}; + +template <> +struct hash< fuzzing::equation > { + std::size_t operator()( const fuzzing::equation& eq ) const noexcept + { + std::size_t h1 = std::hash< double >{}( eq.best_value ); + for ( int val : eq.values ) { + h1 ^= std::hash< int >{}( val ) + 0x9e3779b9 + ( h1 << 6 ) + ( h1 >> 2 ); + } + return h1; + } +}; +} // namespace std + + +namespace fuzzing +{ +struct loaded_bits_counter { + natural_32_bit min; + natural_32_bit max; + int loop_count; + + std::set< std::pair< natural_32_bit, natural_32_bit > > loaded_intervals; +}; + +struct FloatComparator { + bool operator()( const float& a, const float& b ) const + { + const float epsilon = 1e-6f; + return std::abs( a - b ) > epsilon && std::abs( a ) < std::abs( b ); + } +}; + + +using path_id_direction_count = std::vector< location_id::id_type >; +using loop_head_to_loaded_bits_counter = std::unordered_map< location_id::id_type, loaded_bits_counter >; +using loop_endings = std::unordered_map< location_id::id_type, bool >; +using loop_head_to_bodies_t = std::unordered_map< location_id, std::unordered_set< location_id > >; +using nodes_to_counts = std::map< location_id::id_type, node_counts >; + +struct equation_matrix { + equation_matrix get_submatrix( std::set< node_id_with_direction > const& subset ) const; + void process_node( branching_node* end_node, + bool compute_matrix, + const path_id_direction_count& directions_in_path, + bool add_columns, + std::size_t max_directions_in_path_index ); + void start_compute_matrix(); + bool contains( node_id_with_direction const& node ) const; + std::pair< std::size_t, std::size_t > get_dimensions() const; + const std::unordered_map< equation, int >& compute_vectors_with_hits(); + std::vector< equation >& get_matrix(); + std::optional< equation > get_new_subset_counts_from_vectors( const std::vector< equation >& vector, + const iid_node_generations_stats& state ); + int get_desired_vector_direction() const; + float get_biggest_branching_value() const; + const std::unordered_set< location_id::id_type >& get_node_ids() const; + bool empty() const { return matrix.empty(); } + + void print_matrix(); + + BRANCHING_PREDICATE get_branching_predicate() const; + int get_number_of_different_branchings() const { return branching_values.size(); } + + std::map< double, int, FloatComparator > branching_values; + + std::vector< std::map< int, int > > vector_counts_histogram; + +private: + void add_path( branching_node* end_node, + const path_id_direction_count& directions_in_path, + bool add_columns, + std::size_t max_directions_in_path_index ); + + std::vector< equation > matrix; + std::unordered_set< equation > unique_rows; + std::vector< branching_node* > all_paths; + std::vector< node_id_with_direction > nodes; + std::unordered_set< location_id::id_type > node_ids; + + std::unordered_set< equation > vectors; + std::unordered_map< equation, int > vectors_with_hits; + int computed_vectors = 0; +}; + +struct iid_node_dependence_props { + generated_path generate_probabilities( const loop_dependencies& loop_to_properties ); + void process_path_effective( branching_node* end_node, + const path_id_direction_count& directions_in_path, + std::size_t max_directions_in_path_index ); + iid_node_generations_stats& get_generations_stats() { return stats; } + const equation_matrix& get_matrix() const { return matrix; } + const iid_node_generations_stats& get_generations_stats() const { return stats; } + + bool is_covered() const; + bool should_generate_new( const loop_dependencies& loop_to_properties ) const; + failed_generation_method determine_recovery_strategy(); + bool too_much_failed_in_row(); + void set_as_generating_for_other_node( int minimal_max_generation_for_other_node ); + void set_as_generating_artificial_data( int minimal_max_generation_artificial_data ); + bool is_equal_branching_predicate() const; + bool is_matrix_generated() const { return matrix_generated; } + + void print_stats( bool only_state = false ) const; + + +private: + std::optional< std::vector< equation > > generate_vectors_if_not_enough_data( equation_matrix& submatrix ); + std::optional< std::vector< equation > > get_best_vectors( equation_matrix& submatrix, int number_of_vectors ); + generated_path return_empty_path(); + generated_path return_path( const generated_path& path ); + void compute_node_counts_for_nested_loops( nodes_to_counts& path_counts, + std::map< location_id::id_type, int >& child_loop_counts, + location_id::id_type loop_head_id, + int minimum_count, + const loop_dependencies& loop_to_properties, + bool use_random = false ); + int compute_loading_loop_interation( nodes_to_counts& path_counts, + location_id::id_type id, + const std::set< location_id::id_type >& loop_heads, + const loop_properties& props, + const loop_dependencies& loop_to_properties ); + void compute_node_counts_for_loading_loops( nodes_to_counts& path_counts, + const equation& path, + const std::set< location_id::id_type >& loop_heads, + const loop_dependencies& loop_to_properties ); + void compute_node_counts_for_loops( nodes_to_counts& path_counts, + const equation& path, + const std::set< location_id::id_type >& loop_heads, + const loop_dependencies& loop_to_properties, + const std::unordered_set< location_id::id_type >& subset_ids ); + nodes_to_counts compute_node_counts( const equation& path, + const std::set< node_id_with_direction >& all_leafs, + const loop_dependencies& loop_to_properties, + const std::unordered_set< location_id::id_type >& subset_ids ); + std::vector< equation > compute_best_vectors( const std::unordered_map< equation, int >& vectors_with_hits, + int number_of_vectors, + int desired_direction, + float biggest_branching_value ); + std::unordered_map< equation, int > + get_linear_dependent_vector( const std::unordered_map< equation, int >& vectors_with_hits, + equation& best_vector ); + generated_path generate_path_from_node_counts( const nodes_to_counts& path_counts, + const loop_dependencies& loop_to_properties ); + + equation_matrix matrix; + iid_node_generations_stats stats; + + std::set< node_id_with_direction > computation_subset; + equation_matrix computation_submatrix; + + bool matrix_generated = false; +}; + +struct iid_dependencies { + void update_ignored_nodes( sensitivity_analysis& sensitivity ); + void process_node_dependence( branching_node* node ); + void process_node( branching_node* end_node ); + void remove_node_dependence( location_id id ); + void remove_all_covered( const std::unordered_set< location_id >& covered_branchings ); + iid_node_dependence_props& get_props( location_id id ); + std::vector< location_id > get_iid_nodes(); + std::optional< location_id > get_next_iid_node(); + void compute_dependencies(); + + generated_path generate_probabilities(); + + iid_vector_analysis_statistics get_stats() const; + +private: + loop_endings get_loop_heads_ending( branching_node* end_node, loop_head_to_bodies_t& loop_heads_to_bodies ); + void compute_dependencies_by_loading( branching_node* end_node, + const loop_head_to_bodies_t& loop_heads_to_bodies, + const loop_endings& loop_heads_ending ); + void compute_dependencies_by_loops( const loop_head_to_bodies_t& loop_heads_to_bodies, + const loop_endings& loop_heads_ending ); + void compute_loading_loops( branching_node* end_node, + const loop_head_to_bodies_t& loop_heads_to_bodies, + loop_head_to_loaded_bits_counter& loading_loops, + const loop_endings& loop_heads_ending ); + void compute_paths( branching_node* end_node ); + std::vector< node_id_with_direction > get_path( branching_node* node ); + bool is_tracked( location_id id ) const; + + std::map< location_id, iid_node_dependence_props > node_id_to_equation_map; + std::unordered_set< location_id::id_type > ignored_node_ids; + std::unordered_set< location_id > covered_node_ids; + loop_dependencies loop_to_properties; + int processed_nodes = 0; + int dependencies_computed = 0; + + std::vector< branching_node* > end_nodes; + +public: + inline static std::size_t biggest_node_id = 0; + + // Set configurations + inline static int minimal_max_generation_after_covered = 10; + inline static int minimal_max_generation_for_other_node = 10; + inline static bool generate_for_bad_nodes = true; + inline static int minimal_max_generation_artificial_data = 5; + + // Configurations + inline static int biggest_value_in_difference_vector = 4; + inline static bool generate_more_data_after_coverage = true; + inline static int max_failed_generations_in_row = 3; + inline static int maximal_number_of_branching_values = 30; + inline static int maximal_number_of_equations_in_matrix = 500; + inline static int maximal_number_of_equations_with_same_branching_value = 100; + inline static float percentage_to_add_to_path = 0.5; + inline static bool random_direction_in_path = false; + inline static bool random_nested_loop_counts = false; + inline static bool random_node_selection = true; + + inline static bool verbose = false; + // inline static bool verbose = true; +}; + + +std::pair< path_id_direction_count, std::size_t > get_directions_in_path( branching_node* node ); +bool should_generate_more_data( const generation_state& state ); +} // namespace fuzzing diff --git a/src/fuzzing/include/fuzzing/minimization_analysis.hpp b/src/fuzzing/include/fuzzing/minimization_analysis.hpp index 0f6c619f..f0718adb 100644 --- a/src/fuzzing/include/fuzzing/minimization_analysis.hpp +++ b/src/fuzzing/include/fuzzing/minimization_analysis.hpp @@ -62,7 +62,7 @@ struct minimization_analysis , descent{} , computed_input_stdin{} , hashes_of_generated_bits{} - , random_generator{} + , random_generator{ 1U } , stopped_early{ false } , statistics{} {} diff --git a/src/fuzzing/include/fuzzing/optimizer.hpp b/src/fuzzing/include/fuzzing/optimizer.hpp index 0d8fcc1b..2da95559 100644 --- a/src/fuzzing/include/fuzzing/optimizer.hpp +++ b/src/fuzzing/include/fuzzing/optimizer.hpp @@ -36,14 +36,14 @@ struct optimizer final struct performance_statistics { natural_32_bit num_executions{ 0 }; - natural_32_bit num_seconds{ 0 }; + float_64_bit num_seconds{ 0.0 }; natural_32_bit num_extended_tests{ 0 }; }; optimizer(configuration const& cfg); - natural_32_bit num_remaining_seconds() const { return (natural_32_bit)config.max_seconds - get_elapsed_seconds(); } - natural_32_bit get_elapsed_seconds() const { return (natural_32_bit)std::chrono::duration_cast(time_point_current - time_point_start).count(); } + float_64_bit num_remaining_seconds() const { return (float_64_bit)config.max_seconds - get_elapsed_seconds(); } + float_64_bit get_elapsed_seconds() const { return std::chrono::duration(time_point_current - time_point_start).count(); } optimization_outcomes run( std::vector const& inputs_leading_to_boundary_violation, diff --git a/src/fuzzing/include/fuzzing/progress_recorder.hpp b/src/fuzzing/include/fuzzing/progress_recorder.hpp index b9bc9aba..1b3e8899 100644 --- a/src/fuzzing/include/fuzzing/progress_recorder.hpp +++ b/src/fuzzing/include/fuzzing/progress_recorder.hpp @@ -94,7 +94,7 @@ struct progress_recorder PRIORITY_STEP_DISTANCE_TO_WIDTH = 5, PRIORITY_STEP_STDIN_SIZE = 6, PRIORITY_STEP_TRACE_INDEX = 7, - PRIORITY_STEP_SUCCESOR_TRACE_INDEX = 8, + PRIORITY_STEP_SUCCESSOR_TRACE_INDEX = 8, BEST_LOOP_HEAD = 9, BEST_SENSITIVE = 10, @@ -111,7 +111,10 @@ struct progress_recorder MONTE_CARLO_STEP_BACKWARD = 18, BEST_MONTE_CARLO_BACKWARD = 19, - STARTUP = 20, + DEPENDENCY_STEP = 20, + DEPENDENCY_END = 21, + + STARTUP = 22, }; void flush_node_choosing_data(); diff --git a/src/fuzzing/src/branching_node.cpp b/src/fuzzing/src/branching_node.cpp index 128456ce..d79d773a 100644 --- a/src/fuzzing/src/branching_node.cpp +++ b/src/fuzzing/src/branching_node.cpp @@ -3,6 +3,13 @@ namespace fuzzing { +int fuzzing::branching_node::get_depth() const +{ + int depth = 0; + for ( branching_node const* node = this; node != nullptr; node = node->predecessor ) + ++depth; + return depth; +} branching_node::guid_type branching_node::get_fresh_guid__() { diff --git a/src/fuzzing/src/dump.cpp b/src/fuzzing/src/dump.cpp index 3694ce67..cb9dbdda 100644 --- a/src/fuzzing/src/dump.cpp +++ b/src/fuzzing/src/dump.cpp @@ -33,7 +33,7 @@ void print_fuzzing_configuration( << shift << "\"max_exec_megabytes\": " << ioconfig.max_exec_megabytes << ",\n" << shift << "\"stdin_model\": \"" << ioconfig.stdin_model_name << "\",\n" << shift << "\"stdout_model\": \"" << ioconfig.stdout_model_name << "\"\n" - << "}\n" + << "}" ; } @@ -113,9 +113,9 @@ void print_analysis_outcomes(std::ostream& ostr, analysis_outcomes const& res ostr << shift << "\"error_message\": \"" << results.error_message << "\",\n"; std::vector warnings; - if (results.statistics.leaf_nodes_created != results.statistics.leaf_nodes_destroyed) + if (results.fuzzer_statistics.leaf_nodes_created != results.fuzzer_statistics.leaf_nodes_destroyed) warnings.push_back("The number of created and destroyed leaf nodes differ."); - if (results.statistics.nodes_created != results.statistics.nodes_destroyed) + if (results.fuzzer_statistics.nodes_created != results.fuzzer_statistics.nodes_destroyed) warnings.push_back("The number of created and destroyed nodes differ => Memory leak!"); if (results.sensitivity_statistics.start_calls != results.sensitivity_statistics.stop_calls_regular + results.sensitivity_statistics.stop_calls_early) warnings.push_back("The number of starts does not match to the number of stops in the sensitivity analysis."); @@ -177,22 +177,22 @@ void print_analysis_outcomes(std::ostream& ostr, analysis_outcomes const& res << shift << shift << "\"num_deletions\": " << results.bitshare_statistics.num_deletions << "\n" << shift << "},\n" << shift << "\"fuzzer\": {\n" - << shift << shift << "\"leaf_nodes_created\": " << results.statistics.leaf_nodes_created << ",\n" - << shift << shift << "\"leaf_nodes_destroyed\": " << results.statistics.leaf_nodes_destroyed << ",\n" - << shift << shift << "\"nodes_created\": " << results.statistics.nodes_created << ",\n" - << shift << shift << "\"nodes_destroyed\": " << results.statistics.nodes_destroyed << ",\n" - << shift << shift << "\"max_leaf_nodes\": " << results.statistics.max_leaf_nodes << ",\n" - << shift << shift << "\"max_input_width\": " << results.statistics.max_input_width << ",\n" - << shift << shift << "\"longest_branch\": " << results.statistics.longest_branch << ",\n" - << shift << shift << "\"traces_to_crash\": " << results.statistics.traces_to_crash << ",\n" - << shift << shift << "\"traces_to_boundary_violation\": " << results.statistics.traces_to_boundary_violation << ",\n" - << shift << shift << "\"traces_to_medium_overflow\": " << results.statistics.traces_to_medium_overflow << ",\n" - << shift << shift << "\"strategy_primary_loop_head\": " << results.statistics.strategy_primary_loop_head << ",\n" - << shift << shift << "\"strategy_primary_sensitive\": " << results.statistics.strategy_primary_sensitive << ",\n" - << shift << shift << "\"strategy_primary_untouched\": " << results.statistics.strategy_primary_untouched << ",\n" - << shift << shift << "\"strategy_primary_iid_twins\": " << results.statistics.strategy_primary_iid_twins << ",\n" - << shift << shift << "\"strategy_monte_carlo\": " << results.statistics.strategy_monte_carlo << ",\n" - << shift << shift << "\"coverage_failure_resets\": " << results.statistics.coverage_failure_resets << "\n" + << shift << shift << "\"leaf_nodes_created\": " << results.fuzzer_statistics.leaf_nodes_created << ",\n" + << shift << shift << "\"leaf_nodes_destroyed\": " << results.fuzzer_statistics.leaf_nodes_destroyed << ",\n" + << shift << shift << "\"nodes_created\": " << results.fuzzer_statistics.nodes_created << ",\n" + << shift << shift << "\"nodes_destroyed\": " << results.fuzzer_statistics.nodes_destroyed << ",\n" + << shift << shift << "\"max_leaf_nodes\": " << results.fuzzer_statistics.max_leaf_nodes << ",\n" + << shift << shift << "\"max_input_width\": " << results.fuzzer_statistics.max_input_width << ",\n" + << shift << shift << "\"longest_branch\": " << results.fuzzer_statistics.longest_branch << ",\n" + << shift << shift << "\"traces_to_crash\": " << results.fuzzer_statistics.traces_to_crash << ",\n" + << shift << shift << "\"traces_to_boundary_violation\": " << results.fuzzer_statistics.traces_to_boundary_violation << ",\n" + << shift << shift << "\"traces_to_medium_overflow\": " << results.fuzzer_statistics.traces_to_medium_overflow << ",\n" + << shift << shift << "\"strategy_primary_loop_head\": " << results.fuzzer_statistics.strategy_primary_loop_head << ",\n" + << shift << shift << "\"strategy_primary_sensitive\": " << results.fuzzer_statistics.strategy_primary_sensitive << ",\n" + << shift << shift << "\"strategy_primary_untouched\": " << results.fuzzer_statistics.strategy_primary_untouched << ",\n" + << shift << shift << "\"strategy_primary_iid_twins\": " << results.fuzzer_statistics.strategy_primary_iid_twins << ",\n" + << shift << shift << "\"strategy_monte_carlo\": " << results.fuzzer_statistics.strategy_monte_carlo << ",\n" + << shift << shift << "\"coverage_failure_resets\": " << results.fuzzer_statistics.coverage_failure_resets << "\n" << shift << "},\n" ; @@ -221,13 +221,188 @@ void print_analysis_outcomes(std::ostream& ostr, analysis_outcomes const& res } ostr << '\n' << shift << "],\n"; - ostr << shift << "\"num_generated_tests\": " << results.num_generated_tests << ",\n"; - ostr << shift << "\"num_crashes\": " << results.num_crashes << ",\n"; - ostr << shift << "\"num_boundary_violations\": " << results.num_boundary_violations << "\n"; + ostr << shift << "\"output_statistics\": {\n"; + for (auto it = results.output_statistics.begin(); it != results.output_statistics.end(); ++it) + { + ostr << shift << shift << '\"' << it->first << "\": {\n"; + ostr << shift << shift << shift << "\"num_generated_tests\": " << it->second.num_generated_tests << ",\n"; + ostr << shift << shift << shift << "\"num_crashes\": " << it->second.num_crashes << ",\n"; + ostr << shift << shift << shift << "\"num_boundary_violations\": " << it->second.num_boundary_violations << "\n"; + ostr << shift << shift << '}' << (std::next(it) != results.output_statistics.end() ? "," : "") << '\n'; + } + ostr << shift << "}\n"; - ostr << "}\n"; + ostr << "}"; } +void save_iid_vector_analysis( std::ostream& ostr, analysis_outcomes const& result ) +{ + bool print_dependencies = true; + + std::string const shift = " "; + auto indent = [ & ]( int level ) -> std::string { + std::string result; + for ( int i = 0; i < level; ++i ) { + result += shift; + } + return result; + }; + + fuzzing::iid_node_generations_stats total_stats; + for ( const auto& [ _ ,node_stat ] : result.iid_vector_analysis_statistics.iid_nodes_stats ) { + total_stats.method_calls += node_stat.generation_stats.method_calls; + total_stats.generation_starts += node_stat.generation_stats.generation_starts; + total_stats.successful_generations += node_stat.generation_stats.successful_generations; + total_stats.successful_generations_artificial_data += node_stat.generation_stats.successful_generations_artificial_data; + total_stats.failed_generations += node_stat.generation_stats.failed_generations; + total_stats.generate_artificial_data_count += node_stat.generation_stats.generate_artificial_data_count; + total_stats.generated_for_other_node_count += node_stat.generation_stats.generated_for_other_node_count; + } + + ostr << "{\n"; + + ostr << indent( 1 ) << "\"Total statistics\": {\n"; + ostr << indent( 2 ) << "\"Processed nodes\": " << result.iid_vector_analysis_statistics.processed_nodes << ",\n"; + ostr << indent( 2 ) << "\"Dependencies computed\": " << result.iid_vector_analysis_statistics.dependencies_computed << ",\n"; + ostr << indent( 2 ) << "\"Method calls\": " << total_stats.method_calls << ",\n"; + ostr << indent( 2 ) << "\"Generation starts\": " << total_stats.generation_starts << ",\n"; + ostr << indent( 2 ) << "\"Successful generations\": " << total_stats.successful_generations << ",\n"; + ostr << indent( 2 ) << "\"Successful generations of artificial data\": " << total_stats.successful_generations_artificial_data << ",\n"; + ostr << indent( 2 ) << "\"Failed generations\": " << total_stats.failed_generations << ",\n"; + ostr << indent( 2 ) << "\"Artificial generations\": " << total_stats.generate_artificial_data_count << ",\n"; + ostr << indent( 2 ) << "\"For other node generations\": " << total_stats.generated_for_other_node_count << "\n"; + ostr << indent( 1 ) << "},\n"; + + ostr << indent( 1 ) << "\"Dependencies\": [\n"; + for ( auto loop_it = result.iid_vector_analysis_statistics.loop_to_properties.loops.begin(); + loop_it != result.iid_vector_analysis_statistics.loop_to_properties.loops.end(); + ++loop_it ) { + + ostr << indent( 2 ) << "{\n"; + ostr << indent( 3 ) << "\"Loop heads\": [\n"; + for ( auto head_it = loop_it->heads.begin(); head_it != loop_it->heads.end(); ++head_it ) { + ostr << indent( 4 ) << std::dec << "\"" << head_it->first << "\"" + << ( std::next( head_it ) != loop_it->heads.end() ? "," : "" ) << '\n'; + } + ostr << indent( 3 ) << "],\n"; + + ostr << indent( 3 ) << "\"Loop bodies\": [\n"; + for ( auto body_it = loop_it->bodies.begin(); body_it != loop_it->bodies.end(); ++body_it ) { + ostr << indent( 4 ) << std::dec << "\"" << *body_it << "\"" + << ( std::next( body_it ) != loop_it->bodies.end() ? "," : "" ) << '\n'; + } + ostr << indent( 3 ) << "],\n"; + + ostr << indent( 3 ) + << "\"Is loading loop\": " << ( loop_it->is_loading_loop ? "true," : "false" ) << "\n"; + + if ( !loop_it->is_loading_loop ) { + ostr << indent( 2 ) << "}" + << ( std::next( loop_it ) != result.iid_vector_analysis_statistics.loop_to_properties.loops.end() ? "," : "" ) + << '\n'; + continue; + } + + ostr << indent( 3 ) << "\"Loaded bits per loop\": " << loop_it->loaded_bits_per_loop.mean << ",\n"; + ostr << indent( 3 ) << "\"Dependent nodes\": [\n"; + for ( auto body_it = loop_it->bits_read_by_node.begin(); + body_it != loop_it->bits_read_by_node.end(); + ++body_it ) { + ostr << indent( 4 ) << "{\n"; + ostr << indent( 5 ) << "\"Node\": " << std::dec << body_it->first << ",\n"; + ostr << indent( 5 ) << "\"Bits read\": " << body_it->second.average_bits_read.mean << ",\n"; + ostr << indent( 5 ) << "\"Minimal bit offset\": " << body_it->second.minimal_bit_offset + << '\n'; + ostr << indent( 4 ) << "}" + << ( std::next( body_it ) != loop_it->bits_read_by_node.end() ? "," : "" ) << '\n'; + } + ostr << indent( 3 ) << "]\n"; + + ostr << indent( 2 ) << "}" + << ( std::next( loop_it ) != result.iid_vector_analysis_statistics.loop_to_properties.loops.end() ? "," : "" ) << '\n'; + } + + ostr << indent( 1 ) << "],\n"; + + ostr << indent( 1 ) << "\"Ignored nodes\": [\n"; + for ( std::size_t i = 0, n = result.iid_vector_analysis_statistics.ignored_node_ids.size(); i < n; ++i ) { + ostr << indent( 2 ) << std::dec << result.iid_vector_analysis_statistics.ignored_node_ids.at( i ); + if ( i + 1 < n ) + ostr << ','; + ostr << '\n'; + } + ostr << indent( 1 ) << "],\n"; + + ostr << indent( 1 ) << "\"Covered nodes\": [\n"; + for ( std::size_t i = 0, n = result.iid_vector_analysis_statistics.covered_node_ids.size(); i < n; ++i ) { + ostr << indent( 2 ) << std::dec << "\"" << result.iid_vector_analysis_statistics.covered_node_ids.at( i ) <<"\""; + if ( i + 1 < n ) + ostr << ','; + ostr << '\n'; + } + ostr << indent( 1 ) << "],\n"; + + ostr << indent( 1 ) << "\"Analyzed nodes\": {\n"; + + for ( auto it = result.iid_vector_analysis_statistics.iid_nodes_stats.begin(); + it != result.iid_vector_analysis_statistics.iid_nodes_stats.end(); + ++it ) { + ostr << indent( 2 ) << std::dec << "\"" << it->first << "\"" << ": {\n"; + + ostr << indent( 3 ) << "\"Generation state\": \""; + switch ( it->second.generation_stats.state ) { + case generation_state::STATE_NOT_COVERED: ostr << "STATE_NOT_COVERED"; break; + case generation_state::STATE_GENERATING_ARTIFICIAL_DATA: + ostr << "STATE_GENERATING_ARTIFICIAL_DATA"; + break; + case generation_state::STATE_GENERATION_MORE: ostr << "STATE_GENERATION_MORE"; break; + case generation_state::STATE_COVERED: ostr << "STATE_COVERED"; break; + case generation_state::STATE_GENERATION_DATA_FOR_NEXT_NODE: + ostr << "STATE_GENERATION_DATA_FOR_NEXT_NODE"; + break; + case generation_state::STATE_COVERED_BY_OTHER: ostr << "STATE_COVERED_BY_OTHER"; break; + default: ostr << "UNKNOWN"; break; + } + ostr << "\",\n"; + + ostr << indent( 3 ) << "\"Method calls\": " << it->second.generation_stats.method_calls << ",\n"; + ostr << indent( 3 ) << "\"Generation starts\": " << it->second.generation_stats.generation_starts << ",\n"; + ostr << indent( 3 ) + << "\"Successful generations\": " << it->second.generation_stats.successful_generations << ",\n"; + ostr << indent( 3 ) << "\"Successful generations artificial data\": " + << it->second.generation_stats.successful_generations_artificial_data << ",\n"; + ostr << indent( 3 ) << "\"Failed generations\": " << it->second.generation_stats.failed_generations + << ",\n"; + ostr << indent( 3 ) + << "\"Artificial generations\": " << it->second.generation_stats.generate_artificial_data_count + << ",\n"; + ostr << indent( 3 ) + << "\"For other node generations\": " << it->second.generation_stats.generated_for_other_node_count + << ",\n"; + ostr << indent( 3 ) << "\" Is matrix generated\": " << ( it->second.matrix_generated ? "true" : "false" ) + << ",\n"; + ostr << indent( 3 ) << "\"Matrix dimension\": {\n"; + ostr << indent( 4 ) << "\"Rows\": " << it->second.vector_dimensions.first << ",\n"; + ostr << indent( 4 ) << "\"Columns\": " << it->second.vector_dimensions.second << "\n"; + ostr << indent( 3 ) << "},\n"; + ostr << indent( 3 ) << "\"Possible nodes\": [\n"; + for ( std::size_t i = 0, n = it->second.node_ids.size(); i < n; ++i ) { + ostr << indent( 4 ) << std::dec << it->second.node_ids.at( i ); + if ( i + 1 < n ) + ostr << ','; + ostr << '\n'; + } + ostr << indent( 3 ) << "]\n"; + + ostr << indent( 2 ) << "}" + << ( std::next( it ) != result.iid_vector_analysis_statistics.iid_nodes_stats.end() ? "," : "" ) + << '\n'; + } + + ostr << indent( 1 ) << "}\n"; + + ostr << "}"; +} void log_analysis_outcomes(analysis_outcomes const& results) { @@ -236,7 +411,6 @@ void log_analysis_outcomes(analysis_outcomes const& results) LOG(LSL_INFO, sstr.str()); } - void save_analysis_outcomes( std::filesystem::path const& output_dir, std::string const& benchmark, @@ -246,6 +420,12 @@ void save_analysis_outcomes( std::filesystem::path const test_file_path = output_dir / (benchmark + "_outcomes.json"); std::ofstream ostr(test_file_path.c_str(), std::ios::binary); print_analysis_outcomes(ostr, results); + + if ( true ) { + std::filesystem::path const iid_vector_analysis_path = output_dir / (benchmark + "_iid_vector_analysis.json"); + std::ofstream iid_ostr( iid_vector_analysis_path.c_str(), std::ios::binary ); + save_iid_vector_analysis( iid_ostr, results ); + } } @@ -256,7 +436,7 @@ void print_optimization_configuration(std::ostream& ostr, optimizer::configura << shift << "\"max_seconds\": " << config.max_seconds << ",\n" << shift << "\"max_trace_length\": " << config.max_trace_length << ",\n" << shift << "\"max_stdin_bytes\": " << config.max_stdin_bytes << "\n" - << "}\n" + << "}" ; } @@ -354,7 +534,7 @@ void print_optimization_outcomes(std::ostream& ostr, optimization_outcomes con } ostr << '\n' << shift << "]\n"; - ostr << "}\n"; + ostr << "}"; } diff --git a/src/fuzzing/src/dump_native.cpp b/src/fuzzing/src/dump_native.cpp index a129a6d7..49b1f362 100644 --- a/src/fuzzing/src/dump_native.cpp +++ b/src/fuzzing/src/dump_native.cpp @@ -29,6 +29,7 @@ void save_native_test(std::ostream& ostr, execution_record const& record) << shift << "\"boundary_violation\": " << ((record.flags & execution_record::BOUNDARY_CONDITION_VIOLATION) != 0) << ",\n" << shift << "\"medium_overflow\": " << ((record.flags & execution_record::MEDIUM_OVERFLOW) != 0) << ",\n" << shift << "\"empty_startup_trace\": " << ((record.flags & execution_record::EMPTY_STARTUP_TRACE) != 0) << ",\n" + << shift << "\"analysis_name\": \"" << record.analysis_name << "\",\n" ; ostr << shift << "\"num_bytes\": " << record.stdin_bytes.size() << ",\n" @@ -48,7 +49,9 @@ void save_native_test(std::ostream& ostr, execution_record const& record) { if (i % 8U == 0U) ostr << '\n' << shift << shift; ostr << std::dec << '"' << to_string(record.stdin_types.at(i)) << "\","; + ostr << '"'; save_value(ostr, record.stdin_types.at(i), &chunk_values.at(i)); + ostr << '"'; if (i + 1 < n) ostr << ',' << shift; } diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 02ba7ce1..ee8664a1 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -7,6 +7,10 @@ #include #include #include +#include + +constexpr bool use_vector_analysis = false; +// constexpr bool use_vector_analysis = true; namespace fuzzing { @@ -273,11 +277,11 @@ branching_node* fuzzer::primary_coverage_target_branchings::get_best( if (node->max_successors_trace_index > other.node->max_successors_trace_index) { - recorder().on_node_chosen(node, fuzzing::progress_recorder::PRIORITY_STEP_SUCCESOR_TRACE_INDEX); + recorder().on_node_chosen(node, fuzzing::progress_recorder::PRIORITY_STEP_SUCCESSOR_TRACE_INDEX); return true; } - return false; - // return node->max_successors_trace_index > other.node->max_successors_trace_index; + // return false; + return node->max_successors_trace_index > other.node->max_successors_trace_index; } private: branching_node* node; @@ -354,6 +358,20 @@ float_32_bit fuzzer::probability_generator_all_then_all::next() } +std::string const& fuzzer::get_analysis_name_from_state(STATE state) +{ + static std::unordered_map const map { + { STARTUP, "STARTUP" }, + { SENSITIVITY, "sensitivity_analysis" }, + { TYPED_MINIMIZATION, "typed_minimization_analysis" }, + { MINIMIZATION, "minimization_analysis" }, + { BITSHARE, "bitshare_analysis" }, + { FINISHED, "FINISHED" }, + }; + return map.at(state); +} + + void fuzzer::update_close_flags_from(branching_node* const node) { if (node->is_closed() || node->is_open_branching()) @@ -415,6 +433,7 @@ void fuzzer::detect_loops_along_path_to_node( std::vector* const loops ) { + TMPROF_BLOCK(); struct loop_exit_props { branching_node* exit; @@ -615,6 +634,26 @@ void fuzzer::compute_histogram_of_false_direction_probabilities( } } +branching_node* fuzzer::select_start_node_for_monte_carlo_search_with_vector( + const generated_path& path, + std::vector< branching_node* > const& loop_boundaries, + branching_node* fallback_node ) +{ + std::vector< branching_node* > sorted_loop_boundaries = loop_boundaries; + std::sort( sorted_loop_boundaries.begin(), + sorted_loop_boundaries.end(), + []( branching_node* left, branching_node* right ) { + return left->get_location_id().id < right->get_location_id().id; + } ); + + for ( const auto& node : sorted_loop_boundaries ) { + if ( path.contains( node->get_location_id().id ) && !node->is_closed() ) { + return node; + } + } + + return fallback_node; +} branching_node* fuzzer::select_start_node_for_monte_carlo_search( std::vector const& loop_boundaries, @@ -693,17 +732,23 @@ branching_node* fuzzer::monte_carlo_search( branching_node* const start_node, histogram_of_false_direction_probabilities const& histogram, probability_generators_for_locations const& generators, - probability_generator_random_uniform& location_miss_generator + probability_generator_random_uniform& location_miss_generator, + generated_path& path ) { TMPROF_BLOCK(); ASSUMPTION(start_node != nullptr && !start_node->is_closed()); - + branching_node* pivot = start_node; + branching_node* successor; while (true) { - branching_node* const successor{ monte_carlo_step(pivot, histogram, generators, location_miss_generator) }; + if ( use_vector_analysis ) { + successor = monte_carlo_step_with_path( pivot, histogram, generators, location_miss_generator, path ); + } else { + successor = monte_carlo_step( pivot, histogram, generators, location_miss_generator ); + } if (successor == nullptr) break; pivot = successor; @@ -746,6 +791,64 @@ std::pair fuzzer::monte_carlo_backward_search( return { pivot->predecessor, !pivot->predecessor->successor_direction(pivot) }; } +branching_node* +fuzzing::fuzzer::monte_carlo_step_with_path( branching_node* const pivot, + histogram_of_false_direction_probabilities const& histogram, + probability_generators_for_locations const& generators, + probability_generator_random_uniform& location_miss_generator, + generated_path& path ) +{ + INVARIANT( pivot != nullptr && !pivot->is_closed() ); + + branching_node* successor = nullptr; + + branching_node* left = pivot->successor( false ).pointer; + branching_node* right = pivot->successor( true ).pointer; + + bool const can_go_left = left != nullptr && !left->is_closed(); + bool const can_go_right = right != nullptr && !right->is_closed(); + + bool desired_direction; + { + float_32_bit false_direction_probability; + { + auto const it = histogram.find( pivot->get_location_id().id ); + false_direction_probability = it != histogram.end() ? it->second : 0.5f; + } + float_32_bit probability; + { + auto const it = generators.find( pivot->get_location_id().id ); + probability = it != generators.end() ? it->second->next() : location_miss_generator.next(); + } + desired_direction = probability <= false_direction_probability ? false : true; + } + + location_id::id_type pivot_id = pivot->get_location_id().id; + if ( path.contains( pivot_id ) ) { + node_props_in_path& props = path.get_props( pivot_id ); + if ( props.can_take_next_direction() ) { + desired_direction = props.get_desired_direction(); + // std::cout << "Desired direction: " << desired_direction << ", for node id:" << pivot_id << std::endl; + } + } + + bool const can_go_desired_direction = ( desired_direction == false && can_go_left ) || + ( desired_direction == true && can_go_right ); + + if ( path.contains( pivot_id ) && can_go_desired_direction ) { + node_props_in_path& props = path.get_props( pivot_id ); + if ( props.can_take_next_direction() ) { + props.go_direction( desired_direction ); + } + } + + if ( can_go_desired_direction ) + successor = desired_direction == false ? left : right; + else if ( !pivot->is_open_branching() ) + successor = can_go_left ? left : right; + + return successor; +} branching_node* fuzzer::monte_carlo_step( branching_node* const pivot, @@ -813,6 +916,7 @@ fuzzer::fuzzer(termination_info const& info) &statistics } , iid_pivots{} + , iid_dependences{} , coverage_failures_with_hope{} @@ -824,9 +928,9 @@ fuzzer::fuzzer(termination_info const& info) , max_input_width{ 0U } - , generator_for_iid_location_selection{} - , generator_for_iid_approach_selection{} - , generator_for_generator_selection{} + , generator_for_iid_location_selection{ 1U } + , generator_for_iid_approach_selection{ 1U } + , generator_for_generator_selection{ 1U } , statistics{} {} @@ -862,35 +966,12 @@ bool fuzzer::round_begin(TERMINATION_REASON& termination_reason) { TMPROF_BLOCK(); - if (get_performed_driver_executions() > 0U) - { - if (uncovered_branchings.empty()) - { - terminate(); - termination_reason = TERMINATION_REASON::ALL_REACHABLE_BRANCHINGS_COVERED; - return false; - } - } - - if (num_remaining_seconds() <= 0L) - { - terminate(); - termination_reason = TERMINATION_REASON::TIME_BUDGET_DEPLETED; - return false; - } - - if (num_remaining_driver_executions() <= 0L) - { - terminate(); - termination_reason = TERMINATION_REASON::EXECUTIONS_BUDGET_DEPLETED; - return false; - } - iomodels::iomanager::instance().get_stdin()->clear(); iomodels::iomanager::instance().get_stdout()->clear(); vecb stdin_bits; - generate_next_input(stdin_bits); + if (!generate_next_input(stdin_bits, termination_reason)) + return false; if (!can_make_progress()) { terminate(); @@ -907,55 +988,79 @@ bool fuzzer::round_begin(TERMINATION_REASON& termination_reason) } -execution_record::execution_flags fuzzer::round_end() +std::pair fuzzer::round_end() { TMPROF_BLOCK(); execution_record::execution_flags const flags = process_execution_results(); - time_point_current = std::chrono::steady_clock::now(); ++num_driver_executions; - return flags; + return { flags, get_analysis_name_from_state(state) }; } -void fuzzer::generate_next_input(vecb& stdin_bits) +bool fuzzer::generate_next_input(vecb& stdin_bits, TERMINATION_REASON& termination_reason) { TMPROF_BLOCK(); while (true) { + if (get_performed_driver_executions() > 0U) + { + if (uncovered_branchings.empty()) + { + terminate(); + termination_reason = TERMINATION_REASON::ALL_REACHABLE_BRANCHINGS_COVERED; + return false; + } + } + + time_point_current = std::chrono::steady_clock::now(); + if (num_remaining_seconds() <= 0.0) + { + terminate(); + termination_reason = TERMINATION_REASON::TIME_BUDGET_DEPLETED; + return false; + } + + if (num_remaining_driver_executions() <= 0U) + { + terminate(); + termination_reason = TERMINATION_REASON::EXECUTIONS_BUDGET_DEPLETED; + return false; + } + switch (state) { case STARTUP: if (get_performed_driver_executions() == 0U) - return; // Fizzer input is empty + return true; break; case SENSITIVITY: if (sensitivity.generate_next_input(stdin_bits)) - return; + return true; break; case TYPED_MINIMIZATION: if (typed_minimization.generate_next_input(stdin_bits)) - return; + return true; break; case MINIMIZATION: if (minimization.generate_next_input(stdin_bits)) - return; + return true; break; case BITSHARE: if (bitshare.generate_next_input(stdin_bits)) - return; + return true; break; case FINISHED: if (!apply_coverage_failures_with_hope()) - return; + return true; break; default: { UNREACHABLE(); break; } @@ -971,7 +1076,6 @@ void fuzzer::generate_next_input(vecb& stdin_bits) UNREACHABLE(); } - execution_record::execution_flags fuzzer::process_execution_results() { TMPROF_BLOCK(); @@ -1005,7 +1109,8 @@ execution_record::execution_flags fuzzer::process_execution_results() std::numeric_limits::max(), std::numeric_limits::max(), num_driver_executions, - trace->front().xor_like_branching_function + trace->front().xor_like_branching_function, + trace->front().predicate ); construction_props.diverging_node = entry_branching; @@ -1019,6 +1124,10 @@ execution_record::execution_flags fuzzer::process_execution_results() for (; true; ++trace_index) { branching_coverage_info const& info = trace->at(trace_index); + + if ( use_vector_analysis ) { + iid_dependencies::biggest_node_id = std::max( iid_dependencies::biggest_node_id, std::size_t( info.id.id ) ); + } INVARIANT(construction_props.leaf->id == info.id); @@ -1046,6 +1155,32 @@ execution_record::execution_flags fuzzer::process_execution_results() } } + // Here we try to remove bad float (INF, NaN) from 'info.value'. + // It would be better, if fuzzer and analyses could deal with bad floats, but that is complicated. + if (!std::isfinite(info.value) || std::isnan(info.value)) + { + branching_function_value_type& value_ref{ const_cast(info.value) }; + switch (info.predicate) + { + case BP_EQUAL: + value_ref = info.direction ? 0.0 : std::numeric_limits::max(); + break; + case BP_UNEQUAL: + value_ref = info.direction ? std::numeric_limits::max() : 0.0; + break; + case BP_LESS_EQUAL: + case BP_LESS: + value_ref = (info.direction ? -1.0 : 1.0) * std::numeric_limits::max(); + break; + break; + case BP_GREATER: + case BP_GREATER_EQUAL: + value_ref = (info.direction ? 1.0 : -1.0) * std::numeric_limits::max(); + break; + default: UNREACHABLE(); break; + } + } + summary_value += info.value * info.value; bool const value_ok = std::isfinite(summary_value); if (construction_props.leaf->best_stdin == nullptr || (value_ok && construction_props.leaf->best_summary_value > summary_value)) @@ -1074,9 +1209,7 @@ execution_record::execution_flags fuzzer::process_execution_results() node->set_closed(false); branching_coverage_info const& succ_info = trace->at(trace_index + 1); - construction_props.leaf->set_successor(info.direction, { - branching_node::successor_pointer::VISITED, - new branching_node( + auto new_node = new branching_node( succ_info.id, trace_index + 1, succ_info.num_input_bytes, @@ -1087,8 +1220,13 @@ execution_record::execution_flags fuzzer::process_execution_results() succ_info.value, succ_info.value * succ_info.value, num_driver_executions, - succ_info.xor_like_branching_function - ) + succ_info.xor_like_branching_function, + succ_info.predicate + ); + + construction_props.leaf->set_successor(info.direction, { + branching_node::successor_pointer::VISITED, + new_node }); ++statistics.nodes_created; @@ -1110,6 +1248,9 @@ execution_record::execution_flags fuzzer::process_execution_results() construction_props.leaf->successor(trace->back().direction).pointer }); + if ( use_vector_analysis ) + iid_dependences.process_node( construction_props.leaf ); + if (construction_props.diverging_node != nullptr) { auto const it_and_state = leaf_branchings.insert(construction_props.leaf); @@ -1256,6 +1397,8 @@ void fuzzer::do_cleanup() update_close_flags_from(node); break; } + if ( use_vector_analysis ) + iid_dependences.update_ignored_nodes( sensitivity ); collect_iid_pivots_from_sensitivity_results(); break; case BITSHARE: @@ -1289,7 +1432,12 @@ void fuzzer::do_cleanup() for (auto it = iid_pivots.begin(); it != iid_pivots.end(); ) if (covered_branchings.contains(it->first)) + { + if ( use_vector_analysis ) + iid_dependences.remove_node_dependence( it->first ); + it = iid_pivots.erase(it); + } else ++it; @@ -1298,6 +1446,9 @@ void fuzzer::do_cleanup() it = coverage_failures_with_hope.erase(it); else ++it; + + if ( use_vector_analysis ) + iid_dependences.remove_all_covered( covered_branchings ); } @@ -1463,6 +1614,13 @@ void fuzzer::select_next_state() if (winner == nullptr && !entry_branching->is_closed()) winner = select_iid_coverage_target(); + + // else if (winner != nullptr && get_random_natural_32_bit_in_range(1, 10, generator_for_iid_approach_selection) == 10) + // { + // winner = select_iid_coverage_target(); + // } + + if (winner == nullptr) { state = FINISHED; @@ -1516,17 +1674,34 @@ void fuzzer::select_next_state() } -branching_node* fuzzer::select_iid_coverage_target() const +branching_node* fuzzer::select_iid_coverage_target() { TMPROF_BLOCK(); if (iid_pivots.empty() || entry_branching->is_closed()) return nullptr; - auto const it_loc = std::next( - iid_pivots.begin(), - get_random_natural_32_bit_in_range(0, (natural_32_bit)iid_pivots.size() - 1, generator_for_iid_location_selection) - ); + if ( use_vector_analysis ) { + iid_dependences.compute_dependencies(); + } + + generated_path path; + if ( use_vector_analysis ) { + path = iid_dependences.generate_probabilities(); + } + + auto it_loc = std::next( + iid_pivots.begin(), + get_random_natural_32_bit_in_range(0, (natural_32_bit)iid_pivots.size() - 1, generator_for_iid_location_selection) + ); + + if ( use_vector_analysis && path.get_iid_node_id().has_value() ) { + auto it_loc_new = iid_pivots.find( path.get_iid_node_id().value() ); + if ( it_loc_new != iid_pivots.end() ) { + it_loc = it_loc_new; + } + } + auto const it_pivot = select_best_iid_pivot( it_loc->second.pivots, max_input_width, @@ -1545,6 +1720,12 @@ branching_node* fuzzer::select_iid_coverage_target() const histogram_of_hit_counts_per_direction::hit_counts_map hit_counts; it_pivot->second.histogram_ptr->merge(hit_counts); + if ( use_vector_analysis ) { + for ( const auto& path_props : path.get_path() ) { + histogram[path_props.first] = path_props.second.get_false_direction_probability(); + } + } + probability_generators_for_locations generators; auto const random_uniform_generator = compute_probability_generators_for_locations( histogram, @@ -1559,34 +1740,42 @@ branching_node* fuzzer::select_iid_coverage_target() const if (false) // original code: if (get_random_natural_32_bit_in_range(1, 100, generator_for_iid_approach_selection) <= 50) // Currently diabled, because it performs worse for some yet unknown reason. { - auto const node_and_direction = monte_carlo_backward_search( - it_pivot->first, - entry_branching, - histogram, - generators, - *random_uniform_generator - ); - branching_node* const successor = node_and_direction.first->successor(node_and_direction.second).pointer; - if (successor != nullptr) - winner = monte_carlo_search(successor, histogram, generators, *random_uniform_generator); - else if (!node_and_direction.first->is_open_branching()) - winner = monte_carlo_search(node_and_direction.first, histogram, generators, *random_uniform_generator); - else - winner = node_and_direction.first; - - recorder().on_strategy_turn_monte_carlo_backward(winner); + // auto const node_and_direction = monte_carlo_backward_search( + // it_pivot->first, + // entry_branching, + // histogram, + // generators, + // *random_uniform_generator + // ); + // branching_node* const successor = node_and_direction.first->successor(node_and_direction.second).pointer; + // if (successor != nullptr) + // winner = monte_carlo_search(successor, histogram, generators, *random_uniform_generator); + // else if (!node_and_direction.first->is_open_branching()) + // winner = monte_carlo_search(node_and_direction.first, histogram, generators, *random_uniform_generator); + // else + // winner = node_and_direction.first; + + // recorder().on_strategy_turn_monte_carlo_backward(winner); } else { - branching_node* const start_node = select_start_node_for_monte_carlo_search( + branching_node* start_node = select_start_node_for_monte_carlo_search( it_pivot->second.loop_boundaries, it_pivot->second.generator_for_start_node_selection, 0.75f, entry_branching ); + if ( use_vector_analysis && !path.get_path().empty() ) { + start_node = select_start_node_for_monte_carlo_search_with_vector( + path, + it_pivot->second.loop_boundaries, + entry_branching + ); + } + recorder().on_node_chosen(start_node, fuzzing::progress_recorder::START_MONTE_CARLO); - winner = monte_carlo_search(start_node, histogram, generators, *random_uniform_generator); + winner = monte_carlo_search(start_node, histogram, generators, *random_uniform_generator, path); ++statistics.strategy_monte_carlo; recorder().on_strategy_turn_monte_carlo(winner); diff --git a/src/fuzzing/src/fuzzing_loop.cpp b/src/fuzzing/src/fuzzing_loop.cpp index e0c0b91c..22028894 100644 --- a/src/fuzzing/src/fuzzing_loop.cpp +++ b/src/fuzzing/src/fuzzing_loop.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace fuzzing { @@ -55,7 +56,7 @@ analysis_outcomes run( } execution_record record; - record.flags = f.round_end(); + std::tie(record.flags, record.analysis_name) = f.round_end(); if ((record.flags & (execution_record::BRANCH_DISCOVERED | execution_record::BRANCH_COVERED | @@ -63,18 +64,18 @@ analysis_outcomes run( { local::fill_record(record); save_execution_record(record); - ++results.num_generated_tests; + ++results.output_statistics[record.analysis_name].num_generated_tests; if ((record.flags & execution_record::EXECUTION_CRASHES) != 0) { hashes_of_crashes.insert(compute_hash(record.path)); - ++results.num_crashes; + ++results.output_statistics[record.analysis_name].num_crashes; } else if ((record.flags & execution_record::BOUNDARY_CONDITION_VIOLATION) != 0) { exit_locations_of_boundary_violations.insert(record.path.back().first.id); collector_of_boundary_violations(record); - ++results.num_boundary_violations; + ++results.output_statistics[record.analysis_name].num_boundary_violations; } } else if ((record.flags & execution_record::EXECUTION_CRASHES) != 0) @@ -83,8 +84,8 @@ analysis_outcomes run( if (hashes_of_crashes.insert(compute_hash(record.path)).second) { save_execution_record(record); - ++results.num_generated_tests; - ++results.num_crashes; + ++results.output_statistics[record.analysis_name].num_generated_tests; + ++results.output_statistics[record.analysis_name].num_crashes; } } else if ((record.flags & execution_record::BOUNDARY_CONDITION_VIOLATION) != 0) @@ -93,7 +94,7 @@ analysis_outcomes run( if (exit_locations_of_boundary_violations.insert(record.path.back().first.id).second) { collector_of_boundary_violations(record); - ++results.num_boundary_violations; + ++results.output_statistics[record.analysis_name].num_boundary_violations; } } } @@ -124,7 +125,8 @@ analysis_outcomes run( results.typed_minimization_statistics = f.get_typed_minimization_statistics(); results.minimization_statistics = f.get_minimization_statistics(); results.bitshare_statistics = f.get_bitshare_statistics(); - results.statistics = f.get_fuzzer_statistics(); + results.fuzzer_statistics = f.get_fuzzer_statistics(); + results.iid_vector_analysis_statistics = f.get_iid_vector_analysis_statistics(); return results; } diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp new file mode 100644 index 00000000..a565cf7e --- /dev/null +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -0,0 +1,2387 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fuzzing +{ + +// mean_counter +// ------------------------------------------------------------------------------------------------ +template < typename T > +inline void fuzzing::mean_counter< T >::add( T value ) +{ + mean = ( mean * count + value ) / ( count + 1 ); + count++; +} + +// node_counts +// ------------------------------------------------------------------------------------------------ +int node_counts::get_max_count() const { return std::max( left_count, right_count ); } + +// node_props_in_path +// ------------------------------------------------------------------------------------------------ +bool node_props_in_path::get_desired_direction() const +{ + INVARIANT( computed_counts.left_count + computed_counts.right_count > 0 ); + + if ( is_loop_head ) { + return get_preferred_direction_loop_head(); + } + + bool can_go_left = taken_counts.left_count < computed_counts.left_count; + bool can_go_right = taken_counts.right_count < computed_counts.right_count; + + INVARIANT( can_go_left || can_go_right ); + + if ( can_go_left && can_go_right && iid_dependencies::random_direction_in_path ) { + return rand() % 2 == 0; + } + + if ( can_go_left ) { + return false; + } + + if ( can_go_right ) { + return true; + } + + throw std::runtime_error( "No direction to go" ); +} + +// ------------------------------------------------------------------------------------------------ +bool node_props_in_path::can_go_direction( bool direction ) const +{ + if ( direction ) { + return taken_counts.right_count < computed_counts.right_count; + } else { + return taken_counts.left_count < computed_counts.left_count; + } +} + +// ------------------------------------------------------------------------------------------------ +void node_props_in_path::go_direction( bool direction ) +{ + if ( direction ) { + taken_counts.right_count++; + } else { + taken_counts.left_count++; + } + + // Reset the counts if the loop ended + if ( is_loop_head && direction == loop_head_end_direction ) { + taken_counts = { 0, 0 }; + } +} + +// ------------------------------------------------------------------------------------------------ +bool node_props_in_path::can_take_next_direction() const +{ + return taken_counts.left_count < computed_counts.left_count || + taken_counts.right_count < computed_counts.right_count; +} + +// ------------------------------------------------------------------------------------------------ +float_32_bit node_props_in_path::get_false_direction_probability() const +{ + INVARIANT( computed_counts.left_count + computed_counts.right_count > 0 ); + + return float_32_bit( computed_counts.left_count ) / + ( computed_counts.left_count + computed_counts.right_count ); +} + +// ------------------------------------------------------------------------------------------------ +bool node_props_in_path::get_preferred_direction_loop_head() const +{ + auto is_depleted = []( int computed, int taken ) { return computed == taken; }; + + if ( !loop_head_end_direction ) { + return !is_depleted( computed_counts.right_count, taken_counts.right_count ); + } else { + return is_depleted( computed_counts.left_count, taken_counts.left_count ); + } +} + +// generated_path +// ------------------------------------------------------------------------------------------------ +bool generated_path::contains( location_id::id_type id ) const { return path.contains( id ); } + +// ------------------------------------------------------------------------------------------------ +std::map< location_id::id_type, node_props_in_path > generated_path::get_path() const { return path; } + +// equation +// ------------------------------------------------------------------------------------------------ +equation equation::operator+( const equation& other ) const +{ + INVARIANT( values.size() == other.values.size() ); + + std::vector< int > new_values; + for ( int i = 0; i < values.size(); ++i ) { + new_values.push_back( values[ i ] + other.values[ i ] ); + } + + return { new_values, best_value + other.best_value }; +} + +// ------------------------------------------------------------------------------------------------ +equation equation::operator+( int scalar ) const +{ + std::vector< int > new_values; + for ( int i = 0; i < values.size(); ++i ) { + new_values.push_back( values[ i ] + scalar ); + } + + return { new_values, best_value }; +} + +// ------------------------------------------------------------------------------------------------ +equation equation::operator-( const equation& other ) const +{ + INVARIANT( values.size() == other.values.size() ); + + std::vector< int > new_values; + for ( int i = 0; i < values.size(); ++i ) { + new_values.push_back( values[ i ] - other.values[ i ] ); + } + + return { new_values, best_value - other.best_value }; +} + +// ------------------------------------------------------------------------------------------------ +equation equation::operator*( int scalar ) const +{ + std::vector< int > new_values; + for ( int i = 0; i < values.size(); ++i ) { + new_values.push_back( values[ i ] * scalar ); + } + + return { new_values, best_value * scalar }; +} + +// ------------------------------------------------------------------------------------------------ +equation equation::operator*( double scalar ) const +{ + std::vector< int > new_values; + for ( int i = 0; i < values.size(); ++i ) { + new_values.push_back( values[ i ] * scalar ); + } + + return { new_values, best_value * scalar }; +} + +// ------------------------------------------------------------------------------------------------ +equation equation::operator/( const equation& other ) const +{ + INVARIANT( values.size() == other.values.size() ); + + std::vector< int > new_values; + for ( int i = 0; i < values.size(); ++i ) { + if ( other.values[ i ] == 0 ) { + new_values.push_back( 0 ); + } else { + new_values.push_back( values[ i ] / other.values[ i ] ); + } + } + + + return { new_values, best_value / other.best_value }; +} + +// ------------------------------------------------------------------------------------------------ +equation equation::add_to_positive( int value ) const +{ + std::vector< int > new_values; + for ( int i = 0; i < values.size(); ++i ) { + if ( values[ i ] != 0 ) { + new_values.push_back( values[ i ] + value ); + } else { + new_values.push_back( values[ i ] ); + } + } + + return { new_values, best_value }; +} + +// ------------------------------------------------------------------------------------------------ +equation fuzzing::equation::add_to_values( const equation& other ) const +{ + INVARIANT( values.size() == other.values.size() ); + + std::vector< int > new_values; + for ( int i = 0; i < values.size(); ++i ) { + new_values.push_back( values[ i ] + other.values[ i ] ); + } + + return { new_values, best_value }; +} + +// ------------------------------------------------------------------------------------------------ +int fuzzing::equation::simplify_by_gcd() +{ + if ( values.empty() ) { + return 1; + } + + int gcd = std::abs( best_value ); + for ( int i = 0; i < values.size(); ++i ) { + gcd = std::gcd( gcd, values[ i ] ); + } + + if ( gcd == 0 || std::abs( gcd ) == 1 ) { + return 1; + } + + for ( int i = 0; i < values.size(); ++i ) { + values[ i ] /= gcd; + } + + best_value /= gcd; + + return gcd; +} + + +// ------------------------------------------------------------------------------------------------ +int equation::get_vector_size() const +{ + return std::accumulate( values.begin(), values.end(), 0, []( int sum, int val ) { return sum + val; } ); +} + +// ------------------------------------------------------------------------------------------------ +int equation::get_one_way_branching_count() const +{ + return std::count_if( values.begin(), values.end(), []( int val ) { return val == 0; } ); +} + +// ------------------------------------------------------------------------------------------------ +int equation::get_biggest_value() const { return *std::max_element( values.begin(), values.end() ); } + +// ------------------------------------------------------------------------------------------------ +const std::unordered_set< location_id::id_type >& fuzzing::equation_matrix::get_node_ids() const +{ + return node_ids; +} + +// ------------------------------------------------------------------------------------------------ +bool equation::is_any_negative() const +{ + return std::any_of( values.begin(), values.end(), []( int val ) { return val < 0; } ); +} + +// ------------------------------------------------------------------------------------------------ +bool equation::same_values() const +{ + for ( int i = 0; i < values.size(); ++i ) { + if ( values[ i ] != best_value && values[ i ] != 0 ) { + return false; + } + } + + return true; +} + +// ------------------------------------------------------------------------------------------------ +bool equation::is_linear_dependent( const equation& other ) const +{ + INVARIANT( values.size() == other.values.size() ); + + double ratio = std::numeric_limits< double >::quiet_NaN(); + for ( int i = 0; i < values.size(); ++i ) { + if ( values[ i ] == 0 && other.values[ i ] == 0 ) { + continue; + } + + if ( values[ i ] == 0 || other.values[ i ] == 0 ) { + return false; + } + + double current_ratio = double( values[ i ] ) / other.values[ i ]; + if ( std::isnan( ratio ) ) { + ratio = current_ratio; + } else if ( std::abs( ratio - current_ratio ) > 1e-9 ) { + return false; + } + } + + if ( best_value == 0 && other.best_value == 0 ) { + return true; + } + + if ( best_value == 0 || other.best_value == 0 ) { + return false; + } + + if ( std::isnan( ratio ) ) { + return ( best_value - other.best_value ) < 1e-7; + } + + return std::abs( best_value / other.best_value - ratio ) < 1e-7; +} + +// node_id_with_direction +// ------------------------------------------------------------------------------------------------ +auto node_id_with_direction::operator<=>( node_id_with_direction const& other ) const +{ + if ( auto const cmp = node_id <=> other.node_id; cmp != 0 ) + return cmp; + + return branching_direction <=> other.branching_direction; +} + +// loop_properties +// ------------------------------------------------------------------------------------------------ +bool fuzzing::loop_properties::is_same( const std::unordered_set< location_id::id_type >& other_ids ) const +{ + TMPROF_BLOCK(); + return all_ids == other_ids; +} + +// ------------------------------------------------------------------------------------------------ +const std::unordered_set< location_id::id_type >& fuzzing::loop_properties::get_all_ids() const +{ + TMPROF_BLOCK(); + return all_ids; +} + +// ------------------------------------------------------------------------------------------------ +const std::unordered_set< location_id::id_type >& fuzzing::loop_properties::get_loop_head_ids() const +{ + TMPROF_BLOCK(); + return loop_head_ids; +} + +// ------------------------------------------------------------------------------------------------ +location_id::id_type fuzzing::loop_properties::get_smallest_loop_head_id() const +{ + TMPROF_BLOCK(); + + location_id::id_type smallest_id = *loop_head_ids.begin(); + + for ( const auto& id : loop_head_ids ) { + if ( id < smallest_id ) { + smallest_id = id; + } + } + + return smallest_id; +} + +// ------------------------------------------------------------------------------------------------ +std::optional< location_id::id_type > fuzzing::loop_properties::get_smallest_body_id() const +{ + TMPROF_BLOCK(); + + std::optional< location_id::id_type > smallest_id; + + for ( const auto& body : bodies ) { + if ( !smallest_id.has_value() || body.node_id < smallest_id.value() ) { + smallest_id = body.node_id; + } + } + + return smallest_id; +} + +// ------------------------------------------------------------------------------------------------ +location_id::id_type fuzzing::loop_properties::get_smallest_id() const +{ + TMPROF_BLOCK(); + location_id::id_type smallest_id = *all_ids.begin(); + + for ( const auto& id : all_ids ) { + if ( id < smallest_id ) { + smallest_id = id; + } + } + + return smallest_id; +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::loop_properties::update_stored_ids() +{ + TMPROF_BLOCK(); + + for ( const auto& [ head, props ] : heads ) { + all_ids.insert( head.node_id ); + } + + for ( const auto& body : bodies ) { + all_ids.insert( body.node_id ); + } + + if ( loop_head_ids.size() == heads.size() ) { + return; + } + + for ( const auto& [ head, props ] : heads ) { + loop_head_ids.insert( head.node_id ); + } +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::loop_properties::set_chosen_loop_head() +{ + TMPROF_BLOCK(); + + for ( const auto& [ head, props ] : heads ) { + if ( !chosen_loop_head.has_value() ) { + chosen_loop_head = head; + } + + if ( props.count > heads.at( *chosen_loop_head ).count ) { + chosen_loop_head = head; + } + } +} + +// loop_dependencies +// ------------------------------------------------------------------------------------------------ +loop_properties& fuzzing::loop_dependencies::get_props( const std::unordered_set< location_id::id_type >& ids, + location_id::id_type loop_head_id ) +{ + TMPROF_BLOCK(); + + for ( loop_properties& loop : loops ) { + for ( const auto& [ head, props ] : loop.heads ) { + if ( head.node_id == loop_head_id ) { + return loop; + } + } + + if ( loop.is_same( ids ) ) { + return loop; + } + } + + loops.emplace_back(); + return loops.back(); +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::loop_dependencies::merge_properties() +{ + TMPROF_BLOCK(); + + for ( loop_properties& loop : loops ) { + loop.update_stored_ids(); + } + + std::unordered_map< location_id::id_type, loop_properties > merged_loops; + + for ( const auto& loop : loops ) { + location_id::id_type smallest_id = loop.get_smallest_id(); + + auto [ it, inserted ] = merged_loops.try_emplace( smallest_id, std::move( loop ) ); + + if ( !inserted ) { + for ( const auto& [ head, props ] : loop.heads ) { + it->second.heads[ head ].count += props.count; + } + it->second.bodies.insert( loop.bodies.begin(), loop.bodies.end() ); + } + } + + for ( auto& [ id, loop ] : merged_loops ) { + const std::unordered_set< location_id::id_type >& head_ids = loop.get_loop_head_ids(); + + for ( auto body_it = loop.bodies.begin(); body_it != loop.bodies.end(); ) { + if ( head_ids.contains( body_it->node_id ) ) { + body_it = loop.bodies.erase( body_it ); + } else { + ++body_it; + } + } + } + + loops.clear(); + loops.reserve( merged_loops.size() ); + + for ( auto& [ id, loop ] : merged_loops ) { + loop.update_stored_ids(); + loops.push_back( std::move( loop ) ); + } +} + +// ------------------------------------------------------------------------------------------------ +const loop_properties& fuzzing::loop_dependencies::get_props_by_loop_head_id( location_id::id_type loop_head_id ) const +{ + for ( const loop_properties& loop : loops ) { + for ( const auto& [ head, props ] : loop.heads ) { + if ( head.node_id == loop_head_id ) { + return loop; + } + } + } + + throw std::runtime_error( "Loop head not found: " + std::to_string( loop_head_id ) ); +} + +// ------------------------------------------------------------------------------------------------ +loop_properties& fuzzing::loop_dependencies::get_props_by_loop_head_id( location_id::id_type loop_head_id ) +{ + return const_cast< loop_properties& >( + static_cast< const fuzzing::loop_dependencies& >( *this ).get_props_by_loop_head_id( loop_head_id ) ); +} + +// ------------------------------------------------------------------------------------------------ +std::set< location_id::id_type > fuzzing::loop_dependencies::get_loop_heads( bool include_loading_loops ) const +{ + std::set< location_id::id_type > loop_heads; + for ( const auto& props : loops ) { + if ( props.is_loading_loop && !include_loading_loops ) { + continue; + } + + for ( const auto& [ head, _ ] : props.heads ) { + loop_heads.insert( head.node_id ); + } + } + + return loop_heads; +} + +// ------------------------------------------------------------------------------------------------ +bool fuzzing::loop_dependencies::compute_node_subsets_for_computation( + std::set< node_id_with_direction >& computation_subset, + const std::unordered_set< location_id::id_type >& matrix_ids ) const +{ + TMPROF_BLOCK(); + + bool changed = false; + + for ( const auto& loop : loops ) { + if ( loop.is_loading_loop || !matrix_ids.contains( loop.get_smallest_loop_head_id() ) ) { + continue; + } + + for ( const auto& body : loop.bodies ) { + auto [ it, inserted ] = computation_subset.insert( body ); + changed |= inserted; + } + + for ( const auto& [ head, _ ] : loop.heads ) { + auto [ it, inserted ] = computation_subset.insert( { head.node_id, !head.branching_direction } ); + changed |= inserted; + + auto it_head = computation_subset.find( head ); + if ( it_head != computation_subset.end() ) { + computation_subset.erase( it_head ); + changed = true; + } + } + } + + return changed; +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::loop_dependencies::print_dependencies() const +{ + bool print_dependencies_by_loops = true; + bool print_dependencies_by_loading = true; + + if ( !print_dependencies_by_loops && !print_dependencies_by_loading ) { + return; + } + + std::cout << "# Dependencies:" << std::endl; + if ( print_dependencies_by_loops ) { + std::cout << "## Dependencies by loops:" << std::endl; + for ( const auto& loop : loops ) { + if ( loop.is_loading_loop ) { + continue; + } + + std::cout << "Loop heads:" << std::endl; + for ( const auto& [ head, head_props ] : loop.heads ) { + std::cout << "- " << head << " (" << head_props.count << ")" << std::endl; + } + + std::cout << "Loop bodies:" << std::endl; + for ( const auto& body : loop.bodies ) { + std::cout << "- " << body << std::endl; + } + + if ( loop.is_loading_loop ) { + std::cout << "Loading loop" << std::endl; + std::cout << "Average bits per loop: " << loop.loaded_bits_per_loop.mean << std::endl; + for ( const auto& [ body, body_props ] : loop.bits_read_by_node ) { + std::cout << "- " << body << ", Bits: " << body_props.average_bits_read.mean + << ", offset: " << body_props.minimal_bit_offset << std::endl; + } + } + } + } + + if ( print_dependencies_by_loading ) { + std::cout << "## Dependencies by loading:" << std::endl; + for ( const auto& loop : loops ) { + if ( !loop.is_loading_loop ) { + continue; + } + + std::cout << "Loop heads:" << std::endl; + for ( const auto& [ head, head_props ] : loop.heads ) { + std::cout << "- " << head << " (" << head_props.count << ")" << std::endl; + } + + std::cout << "Loop bodies:" << std::endl; + for ( const auto& body : loop.bodies ) { + std::cout << "- " << body << std::endl; + } + + std::cout << "Dependent nodes:" << std::endl; + for ( const auto& [ body, body_props ] : loop.bits_read_by_node ) { + std::cout << "- " << body << ", Bits: " << body_props.average_bits_read.mean + << ", offset: " << body_props.minimal_bit_offset + << " bits used: " << body_props.average_bits_used.mean << std::endl; + } + + std::cout << "Loaded bits per loop: " << loop.loaded_bits_per_loop.mean << std::endl; + } + } +} + +// equation_matrix +// ------------------------------------------------------------------------------------------------ +equation_matrix equation_matrix::get_submatrix( std::set< node_id_with_direction > const& subset ) const +{ + TMPROF_BLOCK(); + + equation_matrix result; + result.nodes = std::vector< node_id_with_direction >( subset.begin(), subset.end() ); + + for ( const node_id_with_direction& nav : result.nodes ) { + result.node_ids.insert( nav.node_id ); + } + + for ( int i = 0; i < matrix.size(); ++i ) { + const equation& row = matrix[ i ]; + + std::vector< int > new_row_values; + for ( const node_id_with_direction& nav : subset ) { + auto it = std::find( nodes.begin(), nodes.end(), nav ); + if ( it != nodes.end() ) { + new_row_values.push_back( row.values[ std::distance( nodes.begin(), it ) ] ); + } else { + new_row_values.push_back( 0 ); + } + } + + equation new_row = { new_row_values, row.best_value }; + + auto [ it, inserted ] = result.unique_rows.insert( new_row ); + if ( inserted ) { + result.matrix.push_back( new_row ); + result.all_paths.push_back( all_paths[ i ] ); + result.branching_values[ row.best_value ]++; + } + } + + return result; +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::equation_matrix::process_node( branching_node* end_node, + bool compute_matrix, + const path_id_direction_count& directions_in_path, + bool add_columns, + std::size_t max_directions_in_path_index ) +{ + TMPROF_BLOCK(); + + if ( unique_rows.size() >= iid_dependencies::maximal_number_of_equations_in_matrix ) { + return; + } + + auto it = branching_values.find( end_node->best_coverage_value ); + + if ( branching_values.size() >= iid_dependencies::maximal_number_of_branching_values && it == branching_values.end() ) { + return; + } + + if ( it != branching_values.end() && + it->second >= iid_dependencies::maximal_number_of_equations_with_same_branching_value ) { + return; + } + + all_paths.push_back( end_node ); + + if ( !compute_matrix ) { + return; + } + + add_path( end_node, directions_in_path, add_columns, max_directions_in_path_index ); +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::equation_matrix::start_compute_matrix() +{ + TMPROF_BLOCK(); + + for ( branching_node* node : all_paths ) { + if ( unique_rows.size() >= iid_dependencies::maximal_number_of_equations_in_matrix ) { + return; + } + + auto [ directions_in_path, max_directions_in_path_index ] = get_directions_in_path( node ); + add_path( node, directions_in_path, true, max_directions_in_path_index ); + } +} + +// ------------------------------------------------------------------------------------------------ +bool equation_matrix::contains( node_id_with_direction const& node ) const +{ + TMPROF_BLOCK(); + + auto it = std::find( nodes.begin(), nodes.end(), node ); + return it != nodes.end(); +} + +// ------------------------------------------------------------------------------------------------ +std::pair< std::size_t, std::size_t > equation_matrix::get_dimensions() const +{ + return { unique_rows.size(), nodes.size() }; +} + +// ------------------------------------------------------------------------------------------------ +const std::unordered_map< equation, int >& equation_matrix::compute_vectors_with_hits() +{ + TMPROF_BLOCK(); + + if ( computed_vectors == matrix.size() ) { + return vectors_with_hits; + } + + for ( int i = computed_vectors; i < matrix.size(); ++i ) { + for ( int j = 0; j < matrix.size(); ++j ) { + if ( i == j ) + continue; + + equation difference = matrix[ i ] - matrix[ j ]; + + if ( difference.is_any_negative() || difference.best_value == 0 || + difference.get_biggest_value() > iid_dependencies::biggest_value_in_difference_vector ) + continue; + + vectors.insert( difference ); + } + + computed_vectors++; + } + + vectors_with_hits.clear(); + + for ( const auto& vector : vectors ) { + for ( const auto& row : matrix ) { + equation new_possible_equation = row + vector; + if ( std::find( unique_rows.begin(), unique_rows.end(), new_possible_equation ) != unique_rows.end() ) { + equation vector_copy = vector; + int gcd = vector_copy.simplify_by_gcd(); + vectors_with_hits[ vector_copy ] += gcd; + } + } + } + + return vectors_with_hits; +} + +// ------------------------------------------------------------------------------------------------ +std::vector< equation >& equation_matrix::get_matrix() { return matrix; } + +// ------------------------------------------------------------------------------------------------ +int equation_matrix::get_desired_vector_direction() const +{ + TMPROF_BLOCK(); + + auto is_positive = []( const equation& eq ) { return eq.best_value > 0; }; + auto is_negative = []( const equation& eq ) { return eq.best_value < 0; }; + + if ( std::all_of( matrix.begin(), matrix.end(), is_positive ) ) { + return -1; + } else if ( std::all_of( matrix.begin(), matrix.end(), is_negative ) ) { + return 1; + } else { + return 0; + } +} + +// ------------------------------------------------------------------------------------------------ +float equation_matrix::get_biggest_branching_value() const +{ + TMPROF_BLOCK(); + + float biggest_value = 0.0f; + + for ( const equation& row : matrix ) { + if ( std::abs( row.best_value ) > biggest_value ) { + biggest_value = std::abs( row.best_value ); + } + } + + return biggest_value; +} + +// ------------------------------------------------------------------------------------------------ +std::optional< equation > +equation_matrix::get_new_subset_counts_from_vectors( const std::vector< equation >& vectors, + const iid_node_generations_stats& stats ) +{ + TMPROF_BLOCK(); + + INVARIANT( !vectors.empty() ); + INVARIANT( vectors[ 0 ].values.size() == nodes.size() ); + + bool generate_more_data = should_generate_more_data( stats.state ); + + std::vector< equation > paths; + + bool is_equal_sign = get_branching_predicate() == BRANCHING_PREDICATE::BP_EQUAL; + + for ( const auto& vector : vectors ) { + for ( const auto& row : matrix ) { + double counts = std::abs( row.best_value ) / std::abs( vector.best_value ); + int rounded_counts = static_cast< int >( std::round( counts ) ); + + if ( std::abs( counts - double( rounded_counts ) ) > 1e-8 && !generate_more_data ) { + continue; + } + + equation new_path = vector * rounded_counts + row; + + if ( !is_equal_sign ) { + new_path = new_path.add_to_values( vector ); + if ( generate_more_data ) { + double best_value = new_path.best_value; + float procents_to_add = 1 + iid_dependencies::percentage_to_add_to_path; + + if ( stats.state == generation_state::STATE_GENERATING_ARTIFICIAL_DATA ) { + // new_path = new_path.add_to_positive( stats.generate_artificial_data_count ); + // new_path = new_path + vector * stats.generate_artificial_data_count; + procents_to_add += stats.generate_artificial_data_count / 3.0; + } + + new_path = new_path * procents_to_add; + new_path.best_value = best_value; + } + } + + if ( !new_path.is_any_negative() ) { + paths.push_back( new_path ); + } + } + } + + if ( paths.empty() ) { + return std::nullopt; + } + + auto compare_equations = []( const equation& a, const equation& b ) { + if ( std::abs( a.best_value ) != std::abs( b.best_value ) ) { + return std::abs( a.best_value ) < std::abs( b.best_value ); + } + if ( a.get_vector_size() != b.get_vector_size() ) { + return a.get_vector_size() < b.get_vector_size(); + } + return a.get_one_way_branching_count() > b.get_one_way_branching_count(); + }; + + auto min_it = std::min_element( paths.begin(), paths.end(), compare_equations ); + + INVARIANT( min_it != paths.end() ); + return *min_it; +} + +// ------------------------------------------------------------------------------------------------ +void equation_matrix::print_matrix() +{ + std::cout << "# Matrix:" << std::endl; + for ( const node_id_with_direction& nav : nodes ) { + std::cout << nav << " "; + } + std::cout << std::endl; + for ( size_t i = 0; i < matrix.size(); ++i ) { + for ( size_t j = 0; j < matrix[ i ].values.size(); ++j ) { + std::cout << ( j ? " " : "" ) << matrix[ i ].values[ j ]; + } + std::cout << " -> | " << matrix[ i ].best_value << std::endl; + } +} + + +// ------------------------------------------------------------------------------------------------ +BRANCHING_PREDICATE equation_matrix::get_branching_predicate() const +{ + ASSUMPTION( all_paths.size() > 0 ); + return all_paths[ 0 ]->branching_predicate; +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::equation_matrix::add_path( branching_node* end_node, + const path_id_direction_count& directions_in_path, + bool add_columns, + std::size_t max_directions_in_path_index ) +{ + auto branching_values_it = branching_values.find( end_node->best_coverage_value ); + + if ( branching_values.size() >= iid_dependencies::maximal_number_of_branching_values && + branching_values_it == branching_values.end() ) { + return; + } + + if ( branching_values_it != branching_values.end() && + branching_values_it->second >= iid_dependencies::maximal_number_of_equations_with_same_branching_value ) { + return; + } + + TMPROF_BLOCK(); + + if ( add_columns ) { + int added_nodes = 0; + INVARIANT( max_directions_in_path_index < directions_in_path.size() ); + + for ( size_t i = 0; i < max_directions_in_path_index + 1; ++i ) { + if ( directions_in_path[ i ] == 0 ) { + continue; + } + + int id = i / 2; + bool direction = i % 2; + + auto [ it, inserted ] = node_ids.insert( id ); + if ( !inserted ) { + continue; + } + + for ( bool direction : { true, false } ) { + nodes.push_back( { location_id::id_type( id ), direction } ); + added_nodes++; + } + } + + for ( int i = 0; i < added_nodes; ++i ) { + for ( int j = 0; j < matrix.size(); ++j ) { + matrix[ j ].values.push_back( 0 ); + } + } + } + + std::vector< int > values_in_path( nodes.size() ); + for ( size_t i = 0; i < nodes.size(); ++i ) { + int index = 2 * nodes[ i ].node_id + ( nodes[ i ].branching_direction ? 1 : 0 ); + values_in_path[ i ] = directions_in_path[ index ]; + } + + equation row = { values_in_path, end_node->best_coverage_value }; + + auto [ it, inserted ] = unique_rows.insert( row ); + if ( inserted ) { + matrix.push_back( row ); + branching_values[ end_node->best_coverage_value ]++; + } +} + +// iid_node_dependence_props +// ------------------------------------------------------------------------------------------------ +generated_path iid_node_dependence_props::generate_probabilities( const loop_dependencies& loop_to_properties ) +{ + TMPROF_BLOCK(); + + stats.method_calls++; + + if ( !matrix_generated ) { + matrix_generated = true; + matrix.start_compute_matrix(); + } + + if ( loop_to_properties.loops.empty() ) { + if ( iid_dependencies::verbose ) + std::cout << "No loops" << std::endl; + + return {}; + } + + bool subset_changed = loop_to_properties.compute_node_subsets_for_computation( computation_subset, + matrix.get_node_ids() ); + + if ( computation_subset.empty() ) { + if ( iid_dependencies::verbose ) + std::cout << "No nodes to compute" << std::endl; + + return {}; + } + + std::unordered_set< location_id::id_type > subset_ids; + for ( const auto& nav : computation_subset ) { + subset_ids.insert( nav.node_id ); + } + + stats.generation_starts++; + + if ( subset_changed ) { + computation_submatrix = matrix.get_submatrix( computation_subset ); + } + + if ( computation_submatrix.empty() ) { + if ( iid_dependencies::verbose ) + std::cout << "Empty submatrix" << std::endl; + + return return_empty_path(); + } + + if ( iid_dependencies::verbose ) { + print_stats( false ); + loop_to_properties.print_dependencies(); + computation_submatrix.print_matrix(); + } + + std::optional< std::vector< equation > > best_vectors; + + if ( stats.state == generation_state::STATE_GENERATING_ARTIFICIAL_DATA ) { + best_vectors = generate_vectors_if_not_enough_data( computation_submatrix ); + } else { + best_vectors = get_best_vectors( computation_submatrix, 1 ); + } + + if ( !best_vectors.has_value() ) { + if ( iid_dependencies::verbose ) + std::cout << "No vectors" << std::endl; + + return return_empty_path(); + } + + std::optional< equation > new_subset_counts = + computation_submatrix.get_new_subset_counts_from_vectors( *best_vectors, stats ); + + if ( !new_subset_counts.has_value() ) { + if ( iid_dependencies::verbose ) + std::cout << "No new subset counts" << std::endl; + + return return_empty_path(); + } + + if ( iid_dependencies::verbose ) { + std::cout << "New subset counts: " << std::endl; + for (size_t i = 0; i < new_subset_counts->values.size(); ++i) { + std::cout << new_subset_counts->values[i] << " "; + } + std::cout << "-> Best Value: " << new_subset_counts->best_value << std::endl; + } + + nodes_to_counts node_counts = + compute_node_counts( *new_subset_counts, computation_subset, loop_to_properties, subset_ids ); + generated_path path = generate_path_from_node_counts( node_counts, loop_to_properties ); + + if ( iid_dependencies::verbose ) { + std::cout << "Generated path: " << std::endl << path << std::endl; + } + + return return_path( path ); +} + + +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_node_dependence_props::process_path_effective( branching_node* end_node, + const path_id_direction_count& directions_in_path, + std::size_t max_directions_in_path_index ) +{ + TMPROF_BLOCK(); + matrix.process_node( end_node, matrix_generated, directions_in_path, true, max_directions_in_path_index ); + + if ( !computation_submatrix.empty() ) { + computation_submatrix.process_node( end_node, true, directions_in_path, false, max_directions_in_path_index ); + } +} + +// ------------------------------------------------------------------------------------------------ +bool iid_node_dependence_props::is_covered() const +{ + return stats.state == generation_state::STATE_COVERED || + stats.state == generation_state::STATE_COVERED_BY_OTHER; +} + +// ------------------------------------------------------------------------------------------------ +bool iid_node_dependence_props::should_generate_new( const loop_dependencies& loop_to_properties ) const +{ + if ( stats.state == generation_state::STATE_COVERED || + stats.state == generation_state::STATE_COVERED_BY_OTHER ) { + return false; + } + + if ( !matrix_generated ) { + return true; + } + + if ( stats.state == generation_state::STATE_GENERATING_ARTIFICIAL_DATA || + stats.state == generation_state::STATE_GENERATION_DATA_FOR_NEXT_NODE || + stats.state == generation_state::STATE_GENERATION_MORE ) { + return true; + } + + if ( matrix.get_number_of_different_branchings() <= 1 ) { + return false; + } + + std::set< fuzzing::node_id_with_direction > computation_subset; + loop_to_properties.compute_node_subsets_for_computation( computation_subset, matrix.get_node_ids() ); + + if ( !computation_subset.empty() ) { + return true; + } + + return false; +} + +// ------------------------------------------------------------------------------------------------ +failed_generation_method fuzzing::iid_node_dependence_props::determine_recovery_strategy() +{ + switch ( stats.last_failed_method ) { + case failed_generation_method::METHOD_NONE: + return failed_generation_method::METHOD_GENERATE_ARTIFICIAL_DATA; + case failed_generation_method::METHOD_GENERATE_ARTIFICIAL_DATA: + return failed_generation_method::METHOD_GENERATE_FROM_OTHER_NODE; + case failed_generation_method::METHOD_GENERATE_FROM_OTHER_NODE: + return failed_generation_method::METHOD_DO_NOT_GENERATE; + case failed_generation_method::METHOD_DO_NOT_GENERATE: + return failed_generation_method::METHOD_GENERATE_ARTIFICIAL_DATA; + } +} + +// ------------------------------------------------------------------------------------------------ +bool iid_node_dependence_props::too_much_failed_in_row() +{ + if ( stats.state != generation_state::STATE_NOT_COVERED ) { + return false; + } + + if ( stats.failed_generations_in_row > iid_dependencies::max_failed_generations_in_row ) { + return true; + } + + if ( stats.do_not_generate_counter > 3 ) { + return true; + } + + return false; +} + +// ------------------------------------------------------------------------------------------------ +void iid_node_dependence_props::set_as_generating_for_other_node( int minimal_max_generation_for_other_node ) +{ + if ( stats.state != generation_state::STATE_COVERED ) { + return; + } + + stats.state = generation_state::STATE_GENERATION_DATA_FOR_NEXT_NODE; + stats.generated_for_other_node_max = minimal_max_generation_for_other_node; + stats.generated_for_other_node = 0; + stats.generated_for_other_node_count++; +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_node_dependence_props::set_as_generating_artificial_data( int minimal_max_generation_artificial_data ) +{ + stats.state = generation_state::STATE_GENERATING_ARTIFICIAL_DATA; + stats.generate_artificial_data_max = minimal_max_generation_artificial_data; + stats.generate_artificial_data = 0; + stats.generate_artificial_data_count++; +} + +// ------------------------------------------------------------------------------------------------ +bool iid_node_dependence_props::is_equal_branching_predicate() const +{ + return matrix.get_branching_predicate() == BRANCHING_PREDICATE::BP_EQUAL; +} + +// ------------------------------------------------------------------------------------------------ +void iid_node_dependence_props::print_stats( bool only_state ) const +{ + switch ( stats.state ) { + case generation_state::STATE_NOT_COVERED: + std::cout << "Status: STATE_NOT_COVERED" << std::endl; + if ( !only_state ) { + std::cout << "Failed generations/Total generations: " << stats.failed_generations << "/" + << stats.generation_starts << std::endl; + std::cout << "Failed generations in row: " << stats.failed_generations_in_row << std::endl; + } + break; + case generation_state::STATE_GENERATION_MORE: + std::cout << "Status STATE_GENERATION_MORE" << std::endl; + if ( !only_state ) { + std::cout << "Generated after covered: " << stats.generated_after_covered << "/" + << stats.generated_after_covered_max << std::endl; + } + break; + case generation_state::STATE_COVERED: std::cout << "STATE_COVERED" << std::endl; break; + case generation_state::STATE_GENERATION_DATA_FOR_NEXT_NODE: { + std::cout << "Status: STATE_GENERATION_DATA_FOR_NEXT_NODE" << std::endl; + if ( !only_state ) + std::cout << "Generated for other node: " << stats.generated_for_other_node << "/" + << stats.generated_for_other_node_max << std::endl; + } break; + case generation_state::STATE_GENERATING_ARTIFICIAL_DATA: + std::cout << "Status: STATE_GENERATING_ARTIFICIAL_DATA" << std::endl; + if ( !only_state ) { + std::cout << "Generated artificial data: " << stats.generate_artificial_data << "/" + << stats.generate_artificial_data_max << std::endl; + } + break; + case generation_state::STATE_COVERED_BY_OTHER: + std::cout << "Status: STATE_COVERED_BY_OTHER" << std::endl; + break; + } +} + +// ------------------------------------------------------------------------------------------------ +std::optional< std::vector< equation > > +fuzzing::iid_node_dependence_props::generate_vectors_if_not_enough_data( equation_matrix& submatrix ) +{ + TMPROF_BLOCK(); + + std::vector< equation > best_vectors; + int desired_direction = submatrix.get_desired_vector_direction(); + const std::unordered_map< equation, int >& vectors = submatrix.compute_vectors_with_hits(); + + auto add_to_best_vectors = [ & ]( std::vector< int > values ) { + if ( desired_direction == 0 ) { + best_vectors.emplace_back( values, 1 ); + best_vectors.emplace_back( values, -1 ); + } else { + best_vectors.emplace_back( values, desired_direction ); + } + }; + + if ( vectors.empty() ) { + std::vector< int > values( submatrix.get_dimensions().second, 1 ); + add_to_best_vectors( values ); + } else { + for ( auto& [ vector, hits ] : vectors ) { + equation modified_vector = vector; + + for ( auto& value : modified_vector.values ) { + if ( value != 0 ) { + value = 1; + } + } + + add_to_best_vectors( modified_vector.values ); + } + } + + if ( best_vectors.empty() ) { + return std::nullopt; + } + + return best_vectors; +} + +// ------------------------------------------------------------------------------------------------ +std::optional< std::vector< equation > > +fuzzing::iid_node_dependence_props::get_best_vectors( equation_matrix& submatrix, int number_of_vectors ) +{ + TMPROF_BLOCK(); + + const std::unordered_map< equation, int >& vectors = submatrix.compute_vectors_with_hits(); + if ( vectors.empty() ) { + return std::nullopt; + } + + if ( iid_dependencies::verbose ) { + for ( const auto& [ vector, hits ] : vectors ) { + std::cout << "Vector: "; + for ( const auto& value : vector.values ) { + std::cout << value << " "; + } + std::cout << "-> Best Value: " << vector.best_value << " (" << hits << ")" << std::endl; + } + } + + + int desired_vector_direction = submatrix.get_desired_vector_direction(); + float biggest_branching_value = submatrix.get_biggest_branching_value(); + std::vector< equation > best_vectors = + compute_best_vectors( vectors, number_of_vectors, desired_vector_direction, biggest_branching_value ); + + if ( best_vectors.empty() ) { + return std::nullopt; + } + + + if ( iid_dependencies::verbose ) { + for ( const auto& vector : best_vectors ) { + std::cout << "Vector: "; + for ( const auto& value : vector.values ) { + std::cout << value << " "; + } + std::cout << "-> Best Value: " << vector.best_value << std::endl; + } + } + + return best_vectors; +} + +// ------------------------------------------------------------------------------------------------ +generated_path iid_node_dependence_props::return_empty_path() +{ + stats.failed_generations++; + stats.failed_generations_in_row++; + return generated_path(); +} + +// ------------------------------------------------------------------------------------------------ +generated_path iid_node_dependence_props::return_path( const generated_path& path ) +{ + stats.failed_generations_in_row = 0; + if ( stats.state == generation_state::STATE_GENERATING_ARTIFICIAL_DATA ) { + stats.successful_generations_artificial_data++; + } else { + stats.successful_generations++; + } + + if ( stats.state == generation_state::STATE_GENERATION_MORE ) { + stats.generated_after_covered++; + + if ( stats.generated_after_covered > stats.generated_after_covered_max ) { + stats.state = generation_state::STATE_COVERED; + } + } + + if ( stats.state == generation_state::STATE_GENERATION_DATA_FOR_NEXT_NODE ) { + stats.generated_for_other_node++; + + if ( stats.generated_for_other_node > stats.generated_for_other_node_max ) { + stats.state = generation_state::STATE_COVERED; + stats.generated_for_other_node = 0; + } + } + + if ( stats.state == generation_state::STATE_GENERATING_ARTIFICIAL_DATA ) { + stats.generate_artificial_data++; + + if ( stats.generate_artificial_data > stats.generate_artificial_data_max ) { + stats.state = generation_state::STATE_NOT_COVERED; + stats.generate_artificial_data = 0; + } + } + + return path; +} + +// ------------------------------------------------------------------------------------------------ +void iid_node_dependence_props::compute_node_counts_for_nested_loops( + nodes_to_counts& path_counts, + std::map< location_id::id_type, int >& child_loop_counts, + location_id::id_type loop_head_id, + int minimum_count, + const loop_dependencies& loop_to_properties, + bool use_random ) +{ + TMPROF_BLOCK(); + + INVARIANT( !child_loop_counts.empty() ); + + int max_child_count = + std::max_element( child_loop_counts.begin(), child_loop_counts.end(), []( const auto& a, const auto& b ) { + return a.second < b.second; + } )->second; + + std::set< int > possible_counts; + + for ( int i = minimum_count; i <= max_child_count; ++i ) { + bool is_good = true; + for ( const auto& [ node_id, count ] : child_loop_counts ) { + if ( count % i != 0 ) { + is_good = false; + break; + } + } + + if ( is_good ) { + possible_counts.insert( i ); + } + } + + if ( possible_counts.empty() ) { + path_counts[ loop_head_id ] = { 1, 1 }; + return; + } + + int highest_count = use_random ? *std::next( possible_counts.begin(), rand() % possible_counts.size() ) : + *possible_counts.rbegin(); + + for ( auto& [ node_id, count ] : child_loop_counts ) { + const loop_properties& props = loop_to_properties.get_props_by_loop_head_id( node_id ); + + for ( auto& [ head, _ ] : props.heads ) { + auto& [ left_count, right_count ] = path_counts[ head.node_id ]; + + if ( head.branching_direction ) { + left_count = count / highest_count; + } else { + right_count = count / highest_count; + } + } + } + + + const loop_properties& props = loop_to_properties.get_props_by_loop_head_id( loop_head_id ); + for ( auto& [ head, _ ] : props.heads ) { + if ( head.branching_direction ) { + path_counts[ head.node_id ] = { highest_count, 1 }; + } else { + path_counts[ head.node_id ] = { 1, highest_count }; + } + } +} + +// ------------------------------------------------------------------------------------------------ +int fuzzing::iid_node_dependence_props::compute_loading_loop_interation( + nodes_to_counts& path_counts, + location_id::id_type id, + const std::set< location_id::id_type >& loop_heads, + const loop_properties& props, + const loop_dependencies& loop_to_properties ) +{ + TMPROF_BLOCK(); + + float average_bits_read = props.bits_read_by_node.at( id ).average_bits_read.mean; + natural_32_bit offset = props.bits_read_by_node.at( id ).minimal_bit_offset; + float average_bits_used = props.bits_read_by_node.at( id ).average_bits_used.mean; + + float loaded_per_loop = std::min( props.loaded_bits_per_loop.mean, average_bits_used ); + loaded_per_loop = std::max( loaded_per_loop, 8.0f ); + + if ( offset == natural_32_bit( props.loaded_bits_per_loop.mean - average_bits_used ) ) + offset = 0; + + bool is_loop_head = true; + if ( !loop_heads.contains( id ) ) { + for ( const auto& loop : loop_to_properties.loops ) { + for ( const auto& body : loop.bodies ) { + if ( body.node_id == id ) { + id = ( *loop.chosen_loop_head ).node_id; + is_loop_head = false; + } + } + } + } + + int total_count = is_loop_head ? path_counts[ id ].get_total_count() : path_counts[ id ].get_max_count(); + float bits_needed = average_bits_read * total_count + offset; + + return std::ceil( bits_needed / loaded_per_loop ); +} + +// ------------------------------------------------------------------------------------------------ +void iid_node_dependence_props::compute_node_counts_for_loading_loops( nodes_to_counts& path_counts, + const equation& path, + const std::set< location_id::id_type >& loop_heads, + const loop_dependencies& loop_to_properties ) +{ + TMPROF_BLOCK(); + + for ( const auto& loop_props : std::ranges::views::reverse( loop_to_properties.loops ) ) { + if ( !loop_props.is_loading_loop ) { + continue; + } + + int loop_count = 1; + std::map< location_id::id_type, int > child_loop_counts; + + for ( const auto& body : loop_props.nodes_dependent_by_loading ) { + if ( !path_counts.contains( body.node_id ) ) { + continue; + } + + int minimal_count = compute_loading_loop_interation( + path_counts, body.node_id, loop_heads, loop_props, loop_to_properties ); + loop_count = std::max( loop_count, minimal_count ); + } + + for ( const auto& body : loop_props.bodies ) { + if ( loop_heads.contains( body.node_id ) ) { + const auto& inner_loop_props = loop_to_properties.get_props_by_loop_head_id( body.node_id ); + if ( inner_loop_props.is_loading_loop ) { + loop_count = 1; + } + } + } + + for ( const auto& body : loop_props.bodies ) { + auto& [ left_count, right_count ] = path_counts[ body.node_id ]; + + if ( loop_heads.contains( body.node_id ) ) { + child_loop_counts[ body.node_id ] = std::max( left_count, right_count ); + } else { + loop_count = std::max( loop_count, left_count + right_count ); + } + } + + if ( child_loop_counts.empty() ) { + for ( const auto& [ head, _ ] : loop_props.heads ) { + int end_count = head == ( *loop_props.chosen_loop_head ) ? 1 : 0; + + if ( head.branching_direction ) { + path_counts[ head.node_id ] = { loop_count, end_count }; + } else { + path_counts[ head.node_id ] = { end_count, loop_count }; + } + } + } else { + compute_node_counts_for_nested_loops( path_counts, + child_loop_counts, + ( *loop_props.chosen_loop_head ).node_id, + loop_count, + loop_to_properties, + iid_dependencies::random_nested_loop_counts ); + } + } +} + +// ------------------------------------------------------------------------------------------------ +void iid_node_dependence_props::compute_node_counts_for_loops( nodes_to_counts& path_counts, + const equation& path, + const std::set< location_id::id_type >& loop_heads, + const loop_dependencies& loop_to_properties, + const std::unordered_set< location_id::id_type >& subset_ids ) +{ + TMPROF_BLOCK(); + + for ( const auto& props : std::ranges::views::reverse( loop_to_properties.loops ) ) { + if ( props.bodies.empty() || props.is_loading_loop || + !subset_ids.contains( props.get_smallest_loop_head_id() ) ) { + continue; + } + + std::map< location_id::id_type, int > child_loop_counts; + int non_loop_child_max_count = 1; + + for ( const auto& body : props.bodies ) { + auto& [ left_count, right_count ] = path_counts[ body.node_id ]; + + if ( loop_heads.contains( body.node_id ) ) { + child_loop_counts[ body.node_id ] = std::max( left_count, right_count ); + } else { + non_loop_child_max_count = std::max( non_loop_child_max_count, left_count + right_count ); + } + } + + if ( child_loop_counts.empty() ) { + continue; + } + + compute_node_counts_for_nested_loops( path_counts, + child_loop_counts, + ( *props.chosen_loop_head ).node_id, + non_loop_child_max_count, + loop_to_properties, + iid_dependencies::random_nested_loop_counts ); + } +} + +// ------------------------------------------------------------------------------------------------ +nodes_to_counts +iid_node_dependence_props::compute_node_counts( const equation& path, + std::set< node_id_with_direction > const& computation_subset, + const loop_dependencies& loop_to_properties, + const std::unordered_set< location_id::id_type >& subset_ids ) +{ + TMPROF_BLOCK(); + + nodes_to_counts path_counts; + + auto leafs = std::vector< node_id_with_direction >( computation_subset.begin(), computation_subset.end() ); + INVARIANT( leafs.size() == path.values.size() ); + + for ( int i = 0; i < leafs.size(); ++i ) { + auto& [ left_count, right_count ] = path_counts[ leafs[ i ].node_id ]; + if ( leafs[ i ].branching_direction ) { + right_count = path.values[ i ]; + } else { + left_count = path.values[ i ]; + } + } + + for ( const auto& loop_props : std::ranges::views::reverse( loop_to_properties.loops ) ) { + if ( !subset_ids.contains( loop_props.get_smallest_loop_head_id() ) ) { + continue; + } + + int loop_count = 0; + + for ( const auto& body : loop_props.bodies ) { + loop_count = std::max( loop_count, path_counts[ body.node_id ].get_total_count() ); + } + + for ( const auto& head : loop_props.heads ) { + loop_count = std::max( loop_count, path_counts[ head.first.node_id ].get_total_count() ); + } + + if ( loop_count == 0 ) { + continue; + } + + for ( const auto& [ head, _ ] : loop_props.heads ) { + int end_count = head == ( *loop_props.chosen_loop_head ) ? 1 : 0; + + if ( head.branching_direction ) { + path_counts[ head.node_id ] = { loop_count, end_count }; + } else { + path_counts[ head.node_id ] = { end_count, loop_count }; + } + } + } + + std::set< location_id::id_type > loop_heads = loop_to_properties.get_loop_heads( false ); + compute_node_counts_for_loops( path_counts, path, loop_heads, loop_to_properties, subset_ids ); + loop_heads = loop_to_properties.get_loop_heads( true ); + compute_node_counts_for_loading_loops( path_counts, path, loop_heads, loop_to_properties ); + + return path_counts; +} + +// ------------------------------------------------------------------------------------------------ +std::vector< equation > +iid_node_dependence_props::compute_best_vectors( const std::unordered_map< equation, int >& vectors_with_hits, + int number_of_vectors, + int desired_direction, + float biggest_branching_value ) +{ + TMPROF_BLOCK(); + + if ( vectors_with_hits.empty() ) { + throw std::invalid_argument( "Input map is empty." ); + } + + std::unordered_map< equation, int > filtered_vectors_with_hits; + if ( desired_direction < 0 ) { + std::copy_if( vectors_with_hits.begin(), + vectors_with_hits.end(), + std::inserter( filtered_vectors_with_hits, filtered_vectors_with_hits.end() ), + []( const auto& pair ) { return pair.first.best_value < 0; } ); + } else if ( desired_direction > 0 ) { + std::copy_if( vectors_with_hits.begin(), + vectors_with_hits.end(), + std::inserter( filtered_vectors_with_hits, filtered_vectors_with_hits.end() ), + []( const auto& pair ) { return pair.first.best_value > 0; } ); + } else { + filtered_vectors_with_hits = vectors_with_hits; + } + + biggest_branching_value = std::abs( biggest_branching_value ); + std::erase_if( filtered_vectors_with_hits, [ biggest_branching_value ]( const auto& pair ) { + return std::abs( pair.first.best_value ) > biggest_branching_value; + } ); + + if ( filtered_vectors_with_hits.empty() ) { + return {}; + } + + std::vector< std::pair< equation, int > > sorted_vectors( filtered_vectors_with_hits.begin(), + filtered_vectors_with_hits.end() ); + + std::sort( sorted_vectors.begin(), sorted_vectors.end(), []( const auto& a, const auto& b ) { + if ( a.second == b.second ) { + return std::abs( a.first.best_value ) > std::abs( b.first.best_value ); + } + return a.second > b.second; + } ); + + if ( sorted_vectors.size() > 5 ) { + sorted_vectors.erase( sorted_vectors.begin() + 5, sorted_vectors.end() ); + } + + std::sort( sorted_vectors.begin(), sorted_vectors.end(), []( const auto& a, const auto& b ) { + if ( a.first.get_vector_size() == b.first.get_vector_size() ) + return a.second > b.second; + + return a.first.get_vector_size() < b.first.get_vector_size(); + } ); + + std::vector< equation > best_vectors; + best_vectors.push_back( sorted_vectors[ 0 ].first ); + + return best_vectors; +} + +// ------------------------------------------------------------------------------------------------ +std::unordered_map< equation, int > iid_node_dependence_props::get_linear_dependent_vector( + const std::unordered_map< equation, int >& vectors_with_hits, + equation& best_vector ) +{ + std::unordered_map< equation, int > dependent_vectors_with_hits; + + for ( const auto& [ vector, hits ] : vectors_with_hits ) { + if ( best_vector.is_linear_dependent( vector ) ) { + dependent_vectors_with_hits[ vector ] = hits; + } + } + + INVARIANT( !dependent_vectors_with_hits.empty() ); + return dependent_vectors_with_hits; +} + +// ------------------------------------------------------------------------------------------------ +generated_path iid_node_dependence_props::generate_path_from_node_counts( const nodes_to_counts& path_counts, + const loop_dependencies& loop_to_properties ) +{ + TMPROF_BLOCK(); + + std::map< location_id::id_type, node_props_in_path > path; + for ( const auto& [ id, counts ] : path_counts ) { + if ( counts.left_count == 0 && counts.right_count == 0 ) { + continue; + } + + bool loop_head_end_direction = false; + bool is_loop_head = false; + + for ( const auto& loop : loop_to_properties.loops ) { + for ( const auto& [ head, _ ] : loop.heads ) { + if ( head.node_id == id ) { + is_loop_head = true; + loop_head_end_direction = head.branching_direction; + } + } + } + + node_props_in_path props = { counts, is_loop_head, loop_head_end_direction }; + path.emplace( id, props ); + } + + return generated_path( path ); +} + +// iid_dependencies +// ------------------------------------------------------------------------------------------------ +void iid_dependencies::update_ignored_nodes( sensitivity_analysis& sensitivity ) +{ + for ( branching_node* node : sensitivity.get_changed_nodes() ) { + if ( node->is_did_branching() ) { + location_id::id_type location_id = node->get_location_id().id; + if ( ignored_node_ids.insert( location_id ).second ) { + node_id_to_equation_map.erase( location_id ); + } + } + } +} + +// ------------------------------------------------------------------------------------------------ +void iid_dependencies::process_node( branching_node* end_node ) +{ + TMPROF_BLOCK(); + + end_nodes.push_back( end_node ); +} + +// ------------------------------------------------------------------------------------------------ +void iid_dependencies::remove_node_dependence( location_id id ) +{ + auto it = node_id_to_equation_map.find( id ); + if ( it != node_id_to_equation_map.end() ) { + iid_node_generations_stats& stats = it->second.get_generations_stats(); + stats.state = generation_state::STATE_COVERED; + covered_node_ids.insert( id ); + + if ( iid_dependencies::generate_more_data_after_coverage && !it->second.is_equal_branching_predicate() ) { + stats.state = generation_state::STATE_GENERATION_MORE; + int max_generation_after_covered = std::max( iid_dependencies::minimal_max_generation_after_covered, + stats.successful_generations / 2 ); + stats.generated_after_covered_max = max_generation_after_covered; + } + } +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_dependencies::remove_all_covered( const std::unordered_set< location_id >& covered_branchings ) +{ + std::set< location_id > covered_ids; + for ( const auto& branching : covered_branchings ) { + covered_ids.insert( branching.id ); + } + + for ( auto it = node_id_to_equation_map.begin(); it != node_id_to_equation_map.end(); ) { + if ( covered_ids.contains( it->first ) && + it->second.get_generations_stats().state == generation_state::STATE_NOT_COVERED ) { + covered_node_ids.insert( it->first ); + iid_node_generations_stats& stats = it->second.get_generations_stats(); + stats.state = generation_state::STATE_COVERED_BY_OTHER; + } + + ++it; + } +} + +// ------------------------------------------------------------------------------------------------ +iid_node_dependence_props& iid_dependencies::get_props( location_id id ) +{ + return node_id_to_equation_map.at( id ); +} + +// ------------------------------------------------------------------------------------------------ +std::vector< location_id > iid_dependencies::get_iid_nodes() +{ + std::vector< location_id > result; + for ( const auto& [ key, _ ] : node_id_to_equation_map ) { + result.push_back( key ); + } + + std::sort( result.begin(), result.end() ); + return result; +} + +// ------------------------------------------------------------------------------------------------ +std::optional< location_id > iid_dependencies::get_next_iid_node() +{ + TMPROF_BLOCK(); + + std::vector< location_id > all_non_covered; + std::vector< location_id > possible_nodes; + + for ( auto it = node_id_to_equation_map.begin(); it != node_id_to_equation_map.end(); ++it ) { + iid_node_dependence_props& props = it->second; + + auto& stats = props.get_generations_stats(); + + if ( !props.is_covered() ) + all_non_covered.push_back( it->first ); + + if ( stats.last_failed_method == failed_generation_method::METHOD_DO_NOT_GENERATE ) { + stats.do_not_generate_counter++; + } + + if ( props.too_much_failed_in_row() ) { + stats.failed_generations_in_row = 0; + stats.do_not_generate_counter = 0; + + failed_generation_method recovery_method = props.determine_recovery_strategy(); + auto prev_it = std::prev( it ); + + if ( recovery_method == failed_generation_method::METHOD_GENERATE_ARTIFICIAL_DATA ) { + props.set_as_generating_artificial_data( iid_dependencies::minimal_max_generation_artificial_data ); + stats.last_failed_method = failed_generation_method::METHOD_GENERATE_ARTIFICIAL_DATA; + } else if ( prev_it != node_id_to_equation_map.end() && + recovery_method == failed_generation_method::METHOD_GENERATE_FROM_OTHER_NODE ) { + iid_node_dependence_props& prev_props = prev_it->second; + prev_props.set_as_generating_for_other_node( + iid_dependencies::minimal_max_generation_for_other_node ); + possible_nodes.push_back( prev_it->first ); + stats.last_failed_method = failed_generation_method::METHOD_GENERATE_FROM_OTHER_NODE; + } else { + stats.last_failed_method = failed_generation_method::METHOD_DO_NOT_GENERATE; + } + } + + if ( props.should_generate_new( loop_to_properties ) ) { + possible_nodes.push_back( it->first ); + } + } + + if ( possible_nodes.empty() ) { + if ( verbose ) + std::cout << "No more nodes to generate." << std::endl; + + if ( generate_for_bad_nodes && !all_non_covered.empty() ) { + return random_node_selection ? all_non_covered[ rand() % all_non_covered.size() ] : + all_non_covered[ 0 ]; + } + + return std::nullopt; + } + + if ( verbose ) { + std::cout << "Possible nodes to generate: "; + for ( const auto& node : possible_nodes ) { + std::cout << node << " "; + } + std::cout << std::endl; + } + + return iid_dependencies::random_node_selection ? possible_nodes[ rand() % possible_nodes.size() ] : + possible_nodes[ 0 ]; +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_dependencies::compute_dependencies() +{ + TMPROF_BLOCK(); + + dependencies_computed++; + + for ( branching_node* end_node : end_nodes ) { + auto it = node_id_to_equation_map.find( end_node->get_location_id() ); + if ( it != node_id_to_equation_map.end() && + ( it->second.get_generations_stats().state == generation_state::STATE_COVERED || + it->second.get_generations_stats().state == generation_state::STATE_COVERED_BY_OTHER ) ) { + continue; + } + + loop_head_to_bodies_t loop_heads_to_bodies; + loop_endings loop_heads_ending = get_loop_heads_ending( end_node, loop_heads_to_bodies ); + + compute_dependencies_by_loops( loop_heads_to_bodies, loop_heads_ending ); + compute_dependencies_by_loading( end_node, loop_heads_to_bodies, loop_heads_ending ); + + for ( auto& loop : loop_to_properties.loops ) { + loop.set_chosen_loop_head(); + } + + compute_paths( end_node ); + } + + end_nodes.clear(); +} + +// ------------------------------------------------------------------------------------------------ +generated_path fuzzing::iid_dependencies::generate_probabilities() +{ + TMPROF_BLOCK(); + std::optional< location_id > id = get_next_iid_node(); + if ( !id.has_value() ) { + if ( verbose ) + std::cout << "No more nodes to generate probabilities for." << std::endl; + + return {}; + } + + if ( verbose ) { + std::cout << "Generating probabilities for node: " << *id << std::endl; + } + + iid_node_dependence_props& props = get_props( id.value() ); + generated_path path = props.generate_probabilities( loop_to_properties ); + + if ( !path.empty() ) { + path.set_iid_node_id( id.value() ); + } + + return path; +} + +// ------------------------------------------------------------------------------------------------ +iid_vector_analysis_statistics fuzzing::iid_dependencies::get_stats() const +{ + iid_vector_analysis_statistics stats; + + for ( const auto& [ id, props ] : node_id_to_equation_map ) { + iid_vector_analysis_statistics_per_node node_stats; + node_stats.generation_stats = props.get_generations_stats(); + node_stats.node_ids = std::vector< location_id::id_type >( props.get_matrix().get_node_ids().begin(), + props.get_matrix().get_node_ids().end() ); + std::sort( node_stats.node_ids.begin(), node_stats.node_ids.end() ); + node_stats.vector_dimensions = props.get_matrix().get_dimensions(); + node_stats.matrix_generated = props.is_matrix_generated(); + stats.iid_nodes_stats[ id ] = node_stats; + } + + stats.ignored_node_ids = std::vector< location_id::id_type >( ignored_node_ids.begin(), + ignored_node_ids.end() ); + std::sort( stats.ignored_node_ids.begin(), stats.ignored_node_ids.end() ); + + stats.covered_node_ids = std::vector< location_id >( covered_node_ids.begin(), covered_node_ids.end() ); + std::sort( stats.covered_node_ids.begin(), stats.covered_node_ids.end() ); + + stats.loop_to_properties = loop_to_properties; + stats.processed_nodes = processed_nodes; + stats.dependencies_computed = dependencies_computed; + return stats; +} + +// ------------------------------------------------------------------------------------------------ +loop_endings fuzzing::iid_dependencies::get_loop_heads_ending( branching_node* end_node, + loop_head_to_bodies_t& loop_heads_to_bodies ) +{ + TMPROF_BLOCK(); + + std::vector< fuzzer::loop_boundary_props > loops; + fuzzer::detect_loops_along_path_to_node( end_node, loop_heads_to_bodies, &loops ); + + loop_endings loop_heads_ending; + + auto is_outside_loop = [ & ]( branching_node* successor, + location_id loop_head_id, + const std::unordered_set< location_id >& loop_bodies ) { + if ( successor == nullptr ) { + return false; + } + + if ( successor->get_location_id() == loop_head_id ) { + return false; + } + + return !loop_bodies.contains( successor->get_location_id().id ); + }; + + for ( const auto& loop : loops ) { + location_id loop_head_id = loop.exit->get_location_id(); + + const auto& loop_bodies = loop_heads_to_bodies.at( loop_head_id ); + + branching_node* left_successor = loop.exit->successor( false ).pointer; + branching_node* right_successor = loop.exit->successor( true ).pointer; + + if ( is_outside_loop( left_successor, loop_head_id, loop_bodies ) ) { + loop_heads_ending[ loop.exit->get_location_id().id ] = false; + } else if ( is_outside_loop( right_successor, loop_head_id, loop_bodies ) ) { + loop_heads_ending[ loop.exit->get_location_id().id ] = true; + } + } + + return loop_heads_ending; +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_dependencies::compute_dependencies_by_loading( branching_node* end_node, + const loop_head_to_bodies_t& loop_heads_to_bodies, + const loop_endings& loop_heads_ending ) +{ + TMPROF_BLOCK(); + + loop_head_to_loaded_bits_counter loading_loops; + compute_loading_loops( end_node, loop_heads_to_bodies, loading_loops, loop_heads_ending ); + + struct dependent_body_props { + natural_32_bit min = std::numeric_limits< natural_32_bit >::max(); + natural_32_bit max = std::numeric_limits< natural_32_bit >::min(); + std::vector< natural_32_bit > sensitive_stdin_bit_counts; + std::unordered_set< stdin_bit_index > sensitivity_bits = {}; + }; + + std::unordered_map< location_id::id_type, std::unordered_map< location_id::id_type, dependent_body_props > > loop_head_to_props; + std::unordered_map< location_id::id_type, dependent_body_props > node_id_to_props; + + branching_node* node = end_node; + + while ( node != nullptr ) { + const auto node_id = node->get_location_id().id; + const auto& sensitive_bits = node->sensitive_stdin_bits; + if ( sensitive_bits.empty() ) { + node = node->predecessor; + continue; + } + + auto& props = node_id_to_props[ node_id ]; + auto min_it = std::min_element( sensitive_bits.begin(), sensitive_bits.end() ); + auto max_it = std::max_element( sensitive_bits.begin(), sensitive_bits.end() ); + natural_32_bit node_min = ( min_it != sensitive_bits.end() ) ? + *min_it : + std::numeric_limits< natural_32_bit >::max(); + natural_32_bit node_max = ( max_it != sensitive_bits.end() ) ? + *max_it : + std::numeric_limits< natural_32_bit >::min(); + + props.min = std::min( props.min, node_min ); + props.max = std::max( props.max, node_max ); + props.sensitive_stdin_bit_counts.push_back( sensitive_bits.size() ); + props.sensitivity_bits.insert( sensitive_bits.begin(), sensitive_bits.end() ); + + node = node->predecessor; + } + + for ( const auto& [ node_id, props ] : node_id_to_props ) { + const auto& [ node_min, node_max, sensitive_counts, sensitive_bits ] = props; + + for ( const auto& [ loop_head, loop_props ] : loading_loops ) { + if ( !loop_heads_ending.contains( loop_head ) ) { + continue; + } + + if ( node_max >= loop_props.min && node_min < loop_props.max ) { + auto& loop_body_props = loop_head_to_props[ loop_head ][ node_id ]; + loop_body_props.min = std::min( loop_body_props.min, node_min ); + loop_body_props.max = std::max( loop_body_props.max, node_max ); + loop_body_props.sensitive_stdin_bit_counts.insert( loop_body_props.sensitive_stdin_bit_counts.end(), + sensitive_counts.begin(), + sensitive_counts.end() ); + loop_body_props.sensitivity_bits.insert( sensitive_bits.begin(), sensitive_bits.end() ); + } + } + } + + for ( const auto& [ loop_head_id, body ] : loop_head_to_props ) { + const auto& loading_props = loading_loops.at( loop_head_id ); + natural_32_bit loaded_bits = loading_props.max - loading_props.min; + + if ( loading_props.loop_count == 0 || loaded_bits == 0 ) { + continue; + } + + double per_loop = double( loaded_bits ) / double( loading_props.loop_count ); + + loop_properties& dependencies = loop_to_properties.get_props_by_loop_head_id( loop_head_id ); + dependencies.is_loading_loop = true; + dependencies.loaded_bits_per_loop.add( per_loop ); + + bool loop_head_end_direction = loop_heads_ending.at( loop_head_id ); + + for ( const auto& [ body_id, props ] : body ) { + auto& body_props = dependencies.bits_read_by_node[ body_id ]; + + for ( const auto& [ start, end ] : loading_props.loaded_intervals ) { + int count = 0; + for ( int i = start; i < end; ++i ) { + if ( props.sensitivity_bits.contains( i ) ) { + count++; + } + } + + if ( count == 0 ) { + continue; + } + + body_props.average_bits_used.add( count ); + } + + natural_32_bit minimal_offset = props.min - loading_props.min; + if ( props.min < loading_props.min ) { + minimal_offset = 0; + } + + INVARIANT( minimal_offset >= 0 ); + body_props.minimal_bit_offset = std::min( body_props.minimal_bit_offset, minimal_offset ); + + for ( const auto& count : props.sensitive_stdin_bit_counts ) { + body_props.average_bits_read.add( count ); + } + + for ( bool direction : { true, false } ) { + node_id_with_direction node_id_direction = { body_id, direction }; + dependencies.nodes_dependent_by_loading.insert( node_id_direction ); + } + } + } +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_dependencies::compute_dependencies_by_loops( const loop_head_to_bodies_t& loop_heads_to_bodies, + const loop_endings& loop_heads_ending ) +{ + TMPROF_BLOCK(); + + bool changed = false; + + for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { + location_id::id_type loop_head_id = loop_head.id; + + auto loop_heads_ending_it = loop_heads_ending.find( loop_head_id ); + if ( loop_heads_ending_it == loop_heads_ending.end() ) { + continue; + } + bool loop_head_end_direction = loop_heads_ending_it->second; + + node_id_with_direction loop_head_direction = { loop_head_id, loop_head_end_direction }; + + std::unordered_set< location_id::id_type > all_ids; + + all_ids.reserve( loop_bodies.size() + 1 ); + all_ids.insert( loop_head_id ); + + std::vector< location_id::id_type > temp_ids; + temp_ids.reserve( loop_bodies.size() ); + std::transform( loop_bodies.begin(), + loop_bodies.end(), + std::back_inserter( temp_ids ), + []( const auto& loop_body ) { return loop_body.id; } ); + + all_ids.insert( temp_ids.begin(), temp_ids.end() ); + + loop_properties& props = loop_to_properties.get_props( all_ids, loop_head_id ); + + auto [ props_heads_it, inserted ] = props.heads.insert( { loop_head_direction, {} } ); + props_heads_it->second.count++; + changed |= inserted; + + const std::unordered_set< location_id::id_type >& all_props_ids = props.get_all_ids(); + + for ( const auto& body : loop_bodies ) { + if ( all_props_ids.contains( body.id ) || loop_head.context_hash != body.context_hash ) { + continue; + } + + for ( bool direction : { true, false } ) { + auto [ it, inserted ] = props.bodies.emplace( body.id, direction ); + changed |= inserted; + } + } + } + + if ( changed ) { + loop_to_properties.merge_properties(); + + std::sort( loop_to_properties.loops.begin(), + loop_to_properties.loops.end(), + []( const auto& a, const auto& b ) { + return a.get_smallest_loop_head_id() < b.get_smallest_loop_head_id(); + } ); + } +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_dependencies::compute_loading_loops( branching_node* end_node, + const loop_head_to_bodies_t& loop_heads_to_bodies, + loop_head_to_loaded_bits_counter& loading_loops, + const loop_endings& loop_heads_ending ) +{ + TMPROF_BLOCK(); + + for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { + loading_loops[ loop_head.id ] = { + std::numeric_limits< natural_32_bit >::max(), std::numeric_limits< natural_32_bit >::min(), 0, {} + }; + } + + branching_node* node = end_node; + while ( node != nullptr ) { + location_id node_id = node->get_location_id(); + + if ( auto it = loop_heads_to_bodies.find( node_id ); it != loop_heads_to_bodies.end() ) { + auto& props = loading_loops[ node_id.id ]; + + natural_32_bit bits_count = node->get_num_stdin_bits(); + + if ( props.min != std::numeric_limits< natural_32_bit >::max() ) + props.loaded_intervals.insert( { bits_count, props.min } ); + + props.min = std::min( props.min, bits_count ); + props.max = std::max( props.max, bits_count ); + props.loop_count++; + } + + if ( node->predecessor != nullptr ) { + branching_node* predecessor = node->predecessor; + const auto predecessor_id = predecessor->get_location_id().id; + + if ( auto it = loop_heads_ending.find( predecessor_id ); it != loop_heads_ending.end() ) { + bool node_direction = predecessor->successor( true ).pointer == node; + if ( it->second == node_direction ) + loading_loops[ predecessor_id ].loop_count--; + } + } + + node = node->predecessor; + } + + // Remove all loops that did not load any data inside + for ( auto it = loading_loops.begin(); it != loading_loops.end(); ) { + if ( it->second.min == it->second.max ) { + it = loading_loops.erase( it ); + } else { + ++it; + } + } +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_dependencies::compute_paths( branching_node* end_node ) +{ + TMPROF_BLOCK(); + + INVARIANT( end_node != nullptr ); + + branching_node* current_node = end_node; + while ( current_node->predecessor != nullptr ) { + current_node = current_node->predecessor; + } + + std::vector< node_id_with_direction > full_path = get_path( end_node ); + path_id_direction_count directions_in_path( iid_dependencies::biggest_node_id * 2 + 2 ); + std::size_t max_directions_in_path_index = 0; + + for ( auto it = full_path.rbegin(); it != full_path.rend(); ++it ) { + processed_nodes++; + const auto& path_node = *it; + + std::size_t index = 2 * path_node.node_id + ( path_node.branching_direction ? 1 : 0 ); + directions_in_path[ index ]++; + + max_directions_in_path_index = std::max( max_directions_in_path_index, index ); + + current_node = current_node->successor( path_node.branching_direction ).pointer; + location_id current_node_id = current_node->get_location_id(); + + if ( !is_tracked( current_node_id ) ) { + continue; + } + + iid_node_dependence_props& props = node_id_to_equation_map[ current_node_id ]; + props.process_path_effective( current_node, directions_in_path, max_directions_in_path_index ); + } +} + +// ------------------------------------------------------------------------------------------------ +std::vector< node_id_with_direction > fuzzing::iid_dependencies::get_path( branching_node* end_node ) +{ + TMPROF_BLOCK(); + + std::vector< node_id_with_direction > result; + + bool iid_seen = false; + branching_node* current = end_node; + + while ( current != nullptr ) { + branching_node* predecessor = current->predecessor; + iid_seen = iid_seen || is_tracked( current->get_location_id() ); + + if ( iid_seen && predecessor != nullptr ) { + node_id_with_direction nav = { predecessor->get_location_id().id, + predecessor->successor_direction( current ) }; + result.push_back( nav ); + } + + current = predecessor; + } + + return result; +} + +// ------------------------------------------------------------------------------------------------ +bool fuzzing::iid_dependencies::is_tracked( location_id id ) const +{ + TMPROF_BLOCK(); + + return !ignored_node_ids.contains( id.id ) && !covered_node_ids.contains( id ); +} + +// non member functions +// ------------------------------------------------------------------------------------------------ +std::pair< path_id_direction_count, std::size_t > get_directions_in_path( branching_node* node ) +{ + TMPROF_BLOCK(); + + path_id_direction_count result( iid_dependencies::biggest_node_id * 2 + 2 ); + std::size_t max_directions_in_path_index = 0; + + branching_node* current = node; + while ( current != nullptr ) { + branching_node* predecessor = current->predecessor; + if ( predecessor != nullptr ) { + int id = predecessor->get_location_id().id; + bool direction = predecessor->successor_direction( current ); + std::size_t index = 2 * id + ( direction ? 1 : 0 ); + + result[ index ]++; + max_directions_in_path_index = std::max( max_directions_in_path_index, index ); + } + + current = predecessor; + } + + return { result, max_directions_in_path_index }; +} + +// ------------------------------------------------------------------------------------------------ +bool should_generate_more_data( const generation_state& state ) +{ + return state == generation_state::STATE_GENERATION_MORE || + state == generation_state::STATE_GENERATION_DATA_FOR_NEXT_NODE || + state == generation_state::STATE_GENERATING_ARTIFICIAL_DATA; +} + +} // namespace fuzzing diff --git a/src/fuzzing/src/optimizer.cpp b/src/fuzzing/src/optimizer.cpp index 4c4a5d83..20469f7f 100644 --- a/src/fuzzing/src/optimizer.cpp +++ b/src/fuzzing/src/optimizer.cpp @@ -165,11 +165,11 @@ optimization_outcomes optimizer::run( std::sort(outcomes.uncovered_branchings.begin(),outcomes.uncovered_branchings.end()); } - outcomes.statistics = statistics; - time_point_current = std::chrono::steady_clock::now(); statistics.num_seconds = get_elapsed_seconds(); + outcomes.statistics = statistics; + return outcomes; } diff --git a/src/fuzzing/src/progress_recorder.cpp b/src/fuzzing/src/progress_recorder.cpp index 4d25b55e..ecedca0b 100644 --- a/src/fuzzing/src/progress_recorder.cpp +++ b/src/fuzzing/src/progress_recorder.cpp @@ -943,7 +943,7 @@ void progress_recorder::node_chossing_data::save() const case SELECTION_REASON::PRIORITY_STEP_DISTANCE_TO_WIDTH: ostr << "PRIORITY_STEP_DISTANCE_TO_WIDTH"; break; case SELECTION_REASON::PRIORITY_STEP_STDIN_SIZE: ostr << "PRIORITY_STEP_STDIN_SIZE"; break; case SELECTION_REASON::PRIORITY_STEP_TRACE_INDEX: ostr << "PRIORITY_STEP_TRACE_INDEX"; break; - case SELECTION_REASON::PRIORITY_STEP_SUCCESOR_TRACE_INDEX: ostr << "PRIORITY_STEP_SUCCESOR_TRACE_INDEX"; break; + case SELECTION_REASON::PRIORITY_STEP_SUCCESSOR_TRACE_INDEX: ostr << "PRIORITY_STEP_SUCCESSOR_TRACE_INDEX"; break; case SELECTION_REASON::BEST_LOOP_HEAD: ostr << "BEST_LOOP_HEAD"; break; case SELECTION_REASON::BEST_SENSITIVE: ostr << "BEST_SENSITIVE"; break; case SELECTION_REASON::BEST_UNTOUCHED: ostr << "BEST_UNTOUCHED"; break; @@ -955,6 +955,8 @@ void progress_recorder::node_chossing_data::save() const case SELECTION_REASON::START_MONTE_CARLO_BACKWARD: ostr << "START_MONTE_CARLO_BACKWARD"; break; case SELECTION_REASON::MONTE_CARLO_STEP_BACKWARD: ostr << "MONTE_CARLO_STEP_BACKWARD"; break; case SELECTION_REASON::BEST_MONTE_CARLO_BACKWARD: ostr << "BEST_MONTE_CARLO_BACKWARD"; break; + case SELECTION_REASON::DEPENDENCY_STEP: ostr << "DEPENDENCY_STEP"; break; + case SELECTION_REASON::DEPENDENCY_END: ostr << "DEPENDENCY_END"; break; case SELECTION_REASON::STARTUP: ostr << "STARTUP"; break; default: UNREACHABLE(); break; } diff --git a/src/fuzzing/src/sensitivity_analysis.cpp b/src/fuzzing/src/sensitivity_analysis.cpp index b1ae24df..f6fc8a20 100644 --- a/src/fuzzing/src/sensitivity_analysis.cpp +++ b/src/fuzzing/src/sensitivity_analysis.cpp @@ -313,9 +313,9 @@ void sensitivity_analysis::process_execution_results(execution_trace_pointer co INVARIANT(info_orig.id == info_curr.id && info_orig.id == n->id); if (info_orig.value != info_curr.value) - for (stdin_bit_index i = probed_bit_start_index; i != probed_bit_end_index; ++i) + for (stdin_bit_index j = probed_bit_start_index, j_end = std::min(probed_bit_end_index, n->get_num_stdin_bits()); j < j_end; ++j) { - auto const it_and_state = n->sensitive_stdin_bits.insert(i); + auto const it_and_state = n->sensitive_stdin_bits.insert(j); if (it_and_state.second) changed_nodes.insert(n); } diff --git a/src/fuzzing/src/typed_minimization_analysis.cpp b/src/fuzzing/src/typed_minimization_analysis.cpp index 78445797..1b3d5811 100644 --- a/src/fuzzing/src/typed_minimization_analysis.cpp +++ b/src/fuzzing/src/typed_minimization_analysis.cpp @@ -43,8 +43,8 @@ typed_minimization_analysis::typed_minimization_analysis() , hashes_of_generated_bits{} , num_fast_and_genuine_executions{ 0U } , stopped_early{ false } - , random_generator32{} - , random_generator64{} + , random_generator32{ 1U } + , random_generator64{ 1U } , statistics{} {} diff --git a/src/instrumentation/include/instrumentation/fuzz_target.hpp b/src/instrumentation/include/instrumentation/fuzz_target.hpp index b95c703a..7e33babd 100644 --- a/src/instrumentation/include/instrumentation/fuzz_target.hpp +++ b/src/instrumentation/include/instrumentation/fuzz_target.hpp @@ -30,7 +30,13 @@ class fuzz_target { fuzz_target(); - void process_condition(location_id id, bool direction, branching_function_value_type value, bool xor_like_branching_function); + void process_condition( + location_id::id_type id_type, + bool direction, + branching_function_value_type value, + bool xor_like_branching_function, + natural_8_bit predicate + ); void process_br_instr(location_id id, bool covered_branch); void process_call_begin(natural_32_bit const id); diff --git a/src/instrumentation/include/instrumentation/instrumentation_types.hpp b/src/instrumentation/include/instrumentation/instrumentation_types.hpp index 3c29138e..3b0ff120 100644 --- a/src/instrumentation/include/instrumentation/instrumentation_types.hpp +++ b/src/instrumentation/include/instrumentation/instrumentation_types.hpp @@ -36,6 +36,20 @@ std::ostream& operator<<(std::ostream& ostr, location_id id); using branching_function_value_type = float_64_bit; + +enum BRANCHING_PREDICATE : natural_8_bit +{ + BP_EQUAL = 0, + BP_UNEQUAL = 1, + BP_LESS = 2, + BP_LESS_EQUAL = 3, + BP_GREATER = 4, + BP_GREATER_EQUAL = 5 +}; + +BRANCHING_PREDICATE opposite_predicate(BRANCHING_PREDICATE predicate); + + struct branching_coverage_info { explicit branching_coverage_info(location_id const id_); @@ -46,6 +60,7 @@ struct branching_coverage_info branching_function_value_type value; natural_32_bit idx_to_br_instr; bool xor_like_branching_function; + BRANCHING_PREDICATE predicate; natural_32_bit num_input_bytes; }; diff --git a/src/instrumentation/src/fuzz_target.cpp b/src/instrumentation/src/fuzz_target.cpp index b70f18b5..c6232c79 100644 --- a/src/instrumentation/src/fuzz_target.cpp +++ b/src/instrumentation/src/fuzz_target.cpp @@ -22,7 +22,14 @@ fuzz_target::fuzz_target(): } -void fuzz_target::process_condition(location_id id, bool direction, branching_function_value_type value, bool xor_like_branching_function) { +void fuzz_target::process_condition( + location_id::id_type const id_type, + bool const direction, + branching_function_value_type value, + bool const xor_like_branching_function, + natural_8_bit const predicate + ) +{ if (stdin_model->num_bytes_read() == 0) return; @@ -36,13 +43,9 @@ void fuzz_target::process_condition(location_id id, bool direction, branching_fu exit(0); } - if (std::isnan(value)) { - value = std::numeric_limits::max(); - } - - id.context_hash = context_hashes.back(); + location_id const id{ id_type, context_hashes.back() }; natural_32_bit idx_to_br_instr = br_instr_trace_length; - shared_memory << data_record_id::condition << id << direction << value << idx_to_br_instr << xor_like_branching_function; + shared_memory << data_record_id::condition << id << direction << value << idx_to_br_instr << xor_like_branching_function << predicate; ++trace_length; } diff --git a/src/instrumentation/src/instrumentation.cpp b/src/instrumentation/src/instrumentation.cpp index 87dfb06a..67626b91 100644 --- a/src/instrumentation/src/instrumentation.cpp +++ b/src/instrumentation/src/instrumentation.cpp @@ -12,9 +12,10 @@ void __sbt_fizzer_process_condition( location_id::id_type const id, bool const direction, branching_function_value_type const value, - bool const xor_like_branching_function + bool const xor_like_branching_function, + natural_8_bit const predicate ) { - sbt_fizzer_target->process_condition(id, direction, value, xor_like_branching_function); + sbt_fizzer_target->process_condition(id, direction, value, xor_like_branching_function, predicate); } void __sbt_fizzer_process_br_instr(location_id::id_type const id, bool const direction) { diff --git a/src/instrumentation/src/instrumentation_types.cpp b/src/instrumentation/src/instrumentation_types.cpp index 8423d557..9dce76ea 100644 --- a/src/instrumentation/src/instrumentation_types.cpp +++ b/src/instrumentation/src/instrumentation_types.cpp @@ -6,6 +6,21 @@ namespace instrumentation { +BRANCHING_PREDICATE opposite_predicate(BRANCHING_PREDICATE predicate) +{ + switch (predicate) + { + case BP_EQUAL: return BP_UNEQUAL; + case BP_UNEQUAL: return BP_EQUAL; + case BP_LESS: return BP_GREATER_EQUAL; + case BP_LESS_EQUAL: return BP_GREATER; + case BP_GREATER: return BP_LESS_EQUAL; + case BP_GREATER_EQUAL: return BP_LESS; + default: UNREACHABLE(); + } +} + + branching_coverage_info::branching_coverage_info(location_id const id_) : id{id_} , direction{} diff --git a/src/iomodels/include/iomodels/stdin_base.hpp b/src/iomodels/include/iomodels/stdin_base.hpp index 3a526fc7..9371a1fe 100644 --- a/src/iomodels/include/iomodels/stdin_base.hpp +++ b/src/iomodels/include/iomodels/stdin_base.hpp @@ -26,7 +26,8 @@ struct stdin_base virtual bool load_record(connection::message& src) = 0; virtual bool load_record(connection::shared_memory& src) = 0; virtual size_t min_flattened_size() const = 0; - virtual void read(natural_8_bit* ptr, type_of_input_bits type, connection::shared_memory& dest) = 0; + void read(natural_8_bit* ptr, type_of_input_bits type, connection::medium& dest) { if (!read_bytes(ptr, type, dest)) exit(0); } + virtual bool read_bytes(natural_8_bit* ptr, type_of_input_bits type, connection::medium& dest) = 0; virtual vecu8 const& get_bytes() const = 0; virtual input_types_vector const& get_types() const = 0; diff --git a/src/iomodels/include/iomodels/stdin_replay_bytes_then_repeat_byte.hpp b/src/iomodels/include/iomodels/stdin_replay_bytes_then_repeat_byte.hpp index b48546c9..7e224d10 100644 --- a/src/iomodels/include/iomodels/stdin_replay_bytes_then_repeat_byte.hpp +++ b/src/iomodels/include/iomodels/stdin_replay_bytes_then_repeat_byte.hpp @@ -19,7 +19,7 @@ struct stdin_replay_bytes_then_repeat_byte : public stdin_base bool load_record(connection::message& src) override; bool load_record(connection::shared_memory& src) override; std::size_t min_flattened_size() const override; - void read(natural_8_bit* ptr, type_of_input_bits type, connection::shared_memory& dest) override; + bool read_bytes(natural_8_bit* ptr, type_of_input_bits type, connection::medium& dest) override; vecu8 const& get_bytes() const override { return bytes; } input_types_vector const& get_types() const override { return types; } diff --git a/src/iomodels/src/iomanager.cpp b/src/iomodels/src/iomanager.cpp index 863cad73..2fc3f591 100644 --- a/src/iomodels/src/iomanager.cpp +++ b/src/iomodels/src/iomanager.cpp @@ -62,6 +62,7 @@ bool iomanager::load_trace_record(Medium& src) { src >> info.value; src >> info.idx_to_br_instr; src >> uchr; info.xor_like_branching_function = uchr != 0U; + src >> uchr; info.predicate = (BRANCHING_PREDICATE)uchr; info.num_input_bytes = (natural_32_bit)get_stdin()->get_bytes().size(); trace.push_back(info); return true; diff --git a/src/iomodels/src/stdin_replay_bytes_then_repeat_byte.cpp b/src/iomodels/src/stdin_replay_bytes_then_repeat_byte.cpp index ef486ecd..191a8ec2 100644 --- a/src/iomodels/src/stdin_replay_bytes_then_repeat_byte.cpp +++ b/src/iomodels/src/stdin_replay_bytes_then_repeat_byte.cpp @@ -119,28 +119,33 @@ std::size_t stdin_replay_bytes_then_repeat_byte::min_flattened_size() const { } - -void stdin_replay_bytes_then_repeat_byte::read(natural_8_bit* ptr, - type_of_input_bits const type, - shared_memory& dest) +bool stdin_replay_bytes_then_repeat_byte::read_bytes( + natural_8_bit* ptr, type_of_input_bits const type, medium& dest + ) { natural_8_bit const count = num_bytes(type); if (cursor + count > max_bytes()) { dest.set_termination(target_termination::boundary_condition_violation); - exit(0); + return false; } if (!dest.can_accept_bytes(count + 2)) { dest.set_termination(target_termination::medium_overflow); - exit(0); + return false; } if (cursor + count > bytes.size()) { bytes.resize(cursor + count, repeat_byte); } memcpy(ptr, bytes.data() + cursor, count); - dest << data_record_id::stdin_bytes << to_id(type); + { + auto const rec_type{ data_record_id::stdin_bytes }; + dest.accept_bytes(&rec_type, sizeof(rec_type)); + auto const type_id{ to_id(type) }; + dest.accept_bytes(&type_id, sizeof(type_id)); + } dest.accept_bytes(bytes.data() + cursor, count); cursor += count; types.push_back(type); + return true; } diff --git a/src/tools/client/CMakeLists.txt b/src/tools/client/CMakeLists.txt index f4809630..86630d43 100644 --- a/src/tools/client/CMakeLists.txt +++ b/src/tools/client/CMakeLists.txt @@ -17,7 +17,7 @@ target_link_libraries(${THIS_TARGET_NAME} instrumentation iomodels utility - ${Boost_LIBRARIES} + ${BOOST_LIST_OF_LIBRARIES_TO_LINK_WITH} Threads::Threads ) diff --git a/src/tools/instrumenter/llvm_instrumenter.cpp b/src/tools/instrumenter/llvm_instrumenter.cpp index dcf8b43c..53d624ae 100644 --- a/src/tools/instrumenter/llvm_instrumenter.cpp +++ b/src/tools/instrumenter/llvm_instrumenter.cpp @@ -5,6 +5,7 @@ # pragma warning(disable:4624) // LLVM: warning C4624: 'llvm::detail::copy_construction_triviality_helper': destructor was implicitly defined # pragma warning(disable:4146) // LLVM: warning C4146: unary minus operator applied to unsigned type, result still unsigned #endif +#include #include #include #include @@ -33,7 +34,7 @@ bool llvm_instrumenter::doInitialization(Module *M) { processCondFunc = module->getOrInsertFunction("__sbt_fizzer_process_condition", VoidTy, - Int32Ty, Int1Ty, DoubleTy, Int1Ty); + Int32Ty, Int1Ty, DoubleTy, Int1Ty, Int8Ty); processCondBrFunc = module->getOrInsertFunction("__sbt_fizzer_process_br_instr", VoidTy, @@ -74,52 +75,45 @@ void llvm_instrumenter::printErrCond(Value *cond) { Value *llvm_instrumenter::instrumentIcmp(Value *lhs, Value *rhs, CmpInst *cmpInst, IRBuilder<> &builder) { - - // pointer comparison -> consider the distance to be 1 - if (lhs->getType()->isPointerTy()) { - return ConstantFP::get(DoubleTy, 1); - } - - bool isUnsigned = cmpInst->isUnsigned(); - - if ((!lhs->getType()->isIntegerTy() || ((llvm::IntegerType const*)lhs->getType())->getBitWidth() < 64) && - (!rhs->getType()->isIntegerTy() || ((llvm::IntegerType const*)rhs->getType())->getBitWidth() < 64) ) + if (lhs->getType()->isPointerTy() && rhs->getType()->isPointerTy()) { - // if the value was extended we can't overflow meaning we don't need to - // cast to a higher type - if (!(dyn_cast(lhs) || dyn_cast(lhs) || - dyn_cast(rhs) || dyn_cast(rhs))) { - - // extend based on the signedness - if (isUnsigned) { - lhs = - builder.CreateZExt(lhs, lhs->getType()->getExtendedType()); - rhs = - builder.CreateZExt(rhs, rhs->getType()->getExtendedType()); - } - else { - lhs = - builder.CreateSExt(lhs, lhs->getType()->getExtendedType()); - rhs = - builder.CreateSExt(rhs, rhs->getType()->getExtendedType()); - } + llvm::ConstantPointerNull* const val = llvm::ConstantPointerNull::get(llvm::cast(lhs->getType())); + if (val == lhs || val == rhs) + { + Value* value = cmpInst->getPredicate() == llvm::CmpInst::Predicate::ICMP_EQ ? builder.CreateXor(cmpInst, 1) : cmpInst; + return builder.CreateUIToFP(builder.CreateZExt(value, Int32Ty), DoubleTy); } } - Value *distance = builder.CreateSub(lhs, rhs); + if (lhs->getType()->isPointerTy()) + lhs = builder.CreatePtrToInt(lhs, Int64Ty); + if (rhs->getType()->isPointerTy()) + rhs = builder.CreatePtrToInt(rhs, Int64Ty); - if (isUnsigned) { - return builder.CreateUIToFP(distance, DoubleTy); + if (cmpInst->isUnsigned()) + { + if (lhs->getType()->isIntegerTy()) + lhs = builder.CreateUIToFP(lhs, DoubleTy); + if (rhs->getType()->isIntegerTy()) + rhs = builder.CreateUIToFP(rhs, DoubleTy); } - return builder.CreateSIToFP(distance, DoubleTy); + else + { + if (lhs->getType()->isIntegerTy()) + lhs = builder.CreateSIToFP(lhs, DoubleTy); + if (rhs->getType()->isIntegerTy()) + rhs = builder.CreateSIToFP(rhs, DoubleTy); + } + + return instrumentFcmp(lhs, rhs, nullptr, builder); } -Value *llvm_instrumenter::instrumentFcmp(Value *lhs, Value *rhs, CmpInst *cmpInst, +Value *llvm_instrumenter::instrumentFcmp(Value *lhs, Value *rhs, CmpInst *, IRBuilder<> &builder) { - if (lhs->getType()->isFloatTy()) { + if (lhs->getType()->isFloatTy()) lhs = builder.CreateFPExt(lhs, DoubleTy); + if (rhs->getType()->isFloatTy()) rhs = builder.CreateFPExt(rhs, DoubleTy); - } Value *distance = builder.CreateFSub(lhs, rhs); @@ -147,15 +141,65 @@ bool llvm_instrumenter::instrumentCond(Instruction *inst, bool const xor_like_br } IRBuilder<> builder(inst->getNextNode()); + // We emulate the 'enum BRANCHING_PREDICATE' (see instrumentation_types.hpp) as follows: + natural_8_bit constexpr BP_EQUAL{ 0 }; + natural_8_bit constexpr BP_UNEQUAL{ 1 }; + natural_8_bit constexpr BP_LESS{ 2 }; + natural_8_bit constexpr BP_LESS_EQUAL{ 3 }; + natural_8_bit constexpr BP_GREATER{ 4 }; + natural_8_bit constexpr BP_GREATER_EQUAL{ 5 }; + natural_8_bit predicate{ BP_UNEQUAL }; + Value *distance; if (auto *cmpInst = dyn_cast(inst)) { distance = instrumentCmp(cmpInst, builder); + switch (cmpInst->getPredicate()) + { + case llvm::CmpInst::Predicate::FCMP_OEQ: + case llvm::CmpInst::Predicate::FCMP_UEQ: + case llvm::CmpInst::Predicate::ICMP_EQ: + predicate = BP_EQUAL; + break; + case llvm::CmpInst::Predicate::FCMP_OGT: + case llvm::CmpInst::Predicate::FCMP_UGT: + case llvm::CmpInst::Predicate::ICMP_UGT: + case llvm::CmpInst::Predicate::ICMP_SGT: + predicate = BP_GREATER; + break; + case llvm::CmpInst::Predicate::FCMP_OGE: + case llvm::CmpInst::Predicate::FCMP_UGE: + case llvm::CmpInst::Predicate::ICMP_UGE: + case llvm::CmpInst::Predicate::ICMP_SGE: + predicate = BP_GREATER_EQUAL; + break; + case llvm::CmpInst::Predicate::FCMP_OLT: + case llvm::CmpInst::Predicate::FCMP_ULT: + case llvm::CmpInst::Predicate::ICMP_ULT: + case llvm::CmpInst::Predicate::ICMP_SLT: + predicate = BP_LESS; + break; + case llvm::CmpInst::Predicate::FCMP_OLE: + case llvm::CmpInst::Predicate::FCMP_ULE: + case llvm::CmpInst::Predicate::ICMP_ULE: + case llvm::CmpInst::Predicate::ICMP_SLE: + predicate = BP_LESS_EQUAL; + break; + case llvm::CmpInst::Predicate::FCMP_ONE: + case llvm::CmpInst::Predicate::FCMP_UNE: + case llvm::CmpInst::Predicate::ICMP_NE: + predicate = BP_UNEQUAL; + break; + + case llvm::CmpInst::Predicate::FCMP_UNO: // TODO: "is_nan" - what to do with that? + default: + break; + } // truncating a number to i1, happens for example with bool in C } else if (dyn_cast(inst)) { - distance = ConstantFP::get(DoubleTy, 1); + distance = builder.CreateUIToFP(builder.CreateZExt(inst, Int32Ty), DoubleTy); // i1 as a return from a call to a function } else if (dyn_cast(inst)) { - distance = ConstantFP::get(DoubleTy, 1); + distance = builder.CreateUIToFP(builder.CreateZExt(inst, Int32Ty), DoubleTy); } else { return false; } @@ -163,8 +207,13 @@ bool llvm_instrumenter::instrumentCond(Instruction *inst, bool const xor_like_br Value *location = ConstantInt::get(Int32Ty, ++condCounter); Value *cond = inst; - builder.CreateCall(processCondFunc, - {location, cond, distance, ConstantInt::get(Int1Ty, xor_like_branching_function ? 1 : 0) }); + builder.CreateCall(processCondFunc, { + location, + cond, + distance, + ConstantInt::get(Int1Ty, xor_like_branching_function ? 1 : 0), + ConstantInt::get(Int8Ty, predicate) + }); return true; } diff --git a/src/tools/instrumenter/llvm_instrumenter.hpp b/src/tools/instrumenter/llvm_instrumenter.hpp index 1a812daa..9778a381 100644 --- a/src/tools/instrumenter/llvm_instrumenter.hpp +++ b/src/tools/instrumenter/llvm_instrumenter.hpp @@ -13,10 +13,10 @@ # endif # include # include +# include # include # include # include -# include # include # include # include diff --git a/src/tools/runner/sbt-fizzer.py b/src/tools/runner/sbt-fizzer.py old mode 100644 new mode 100755 index d8a6f628..d7c9436e --- a/src/tools/runner/sbt-fizzer.py +++ b/src/tools/runner/sbt-fizzer.py @@ -33,22 +33,23 @@ def benchmark_target_name(input_file): return benchmark_name(input_file) + "_sbt-fizzer_target" -def build(self_dir, input_file, output_dir, options, use_m32, silent_build): +def build(self_dir, input_file, output_dir, options, use_m32, silent_mode): ll_file = os.path.join(output_dir, benchmark_ll_name(input_file)) - if silent_build is False: print("Compiling...", end='', flush=True) + if silent_mode is False: print("\"build_times\": {", flush=True) + if silent_mode is False: print(" \"Compiling[C->LLVM]\": ", end='', flush=True) t0 = time.time() if _execute( [ "clang" ] + (["-m32"] if use_m32 is True else []) + [ "-O0", "-g", "-S", "-emit-llvm", "-Wno-everything", "-fbracket-depth=1024", input_file, "-o", ll_file], None).returncode: - raise Exception("Compilation has failed: " + input_file) + raise Exception("Compilation[C->LLVM] has failed: " + input_file) t1 = time.time() - if silent_build is False: print("Done[%ds]" % int(round(t1 - t0)), flush=True) + if silent_mode is False: print("%.2f," % (t1 - t0), flush=True) instrumented_ll_file = os.path.join(output_dir, benchmark_instrumented_ll_name(input_file)) - if silent_build is False: print("Instrumenting...", end='', flush=True) + if silent_mode is False: print(" \"Instrumenting\": ", end='', flush=True) t0 = time.time() if _execute( [ os.path.join(self_dir, "tools", "@FIZZER_INSTRUMENTER_FILE@") ] + @@ -57,7 +58,7 @@ def build(self_dir, input_file, output_dir, options, use_m32, silent_build): None).returncode: raise Exception("Instrumentation has failed: " + ll_file) t1 = time.time() - if silent_build is False: print("Done[%ds]" % int(round(t1 - t0)), flush=True) + if silent_mode is False: print("%.2f," % (t1 - t0), flush=True) fuzz_target_libraries = list(map( # type: ignore lambda lib_name: os.path.join(self_dir, "lib32" if use_m32 is True else "lib", lib_name).replace("\\", "/"), @@ -65,7 +66,7 @@ def build(self_dir, input_file, output_dir, options, use_m32, silent_build): )) target_file = os.path.join(output_dir, benchmark_target_name(input_file)) - if silent_build is False: print("Linking...", end='', flush=True) + if silent_mode is False: print(" \"Linking\": ", end='', flush=True) t0 = time.time() if _execute( [ "clang++" ] + @@ -75,11 +76,51 @@ def build(self_dir, input_file, output_dir, options, use_m32, silent_build): fuzz_target_libraries + [ "-o", target_file ], None).returncode: - raise Exception("Compilation has failed: " + input_file) + raise Exception("Linking has failed: " + input_file) t1 = time.time() - if silent_build is False: print("Done[%ds]" % int(round(t1 - t0)), flush=True) + if silent_mode is False: print("%.2f" % (t1 - t0), flush=True) + if silent_mode is False: print("},", flush=True) +def adjust_timeouts(options, start_time, silent_mode): + time_taken = time.time() - start_time + if time_taken < 0.1: + return + + def find_option_value_and_index(option): + try: idx = options.index(option) + except Exception: return None, None + if idx >= len(options): + return None + idx += 1 + try: return int(options[idx]), idx + except: return None, None + + def reduce_option_value(name, value, idx, total_time, suffix=""): + if total_time > time_taken: + percentage = 1.0 - time_taken / total_time + else: + percentage = 0.0 + new_value = int(value * percentage) + if silent_mode is False: print(" \"" + name + "\": [ " + str(value) + ", " + str(new_value) + " ]" + suffix, flush=True) + options[idx] = str(new_value) + + if silent_mode is False: print("\"adjusting_timeouts\": {", flush=True) + if silent_mode is False: print(" \"time_already_taken\": %.2f," % time_taken, flush=True) + + fuzz_value, fuzz_idx = find_option_value_and_index("--max_seconds") + opt_value, opt_idx = find_option_value_and_index("--optimizer_max_seconds") + + if fuzz_value is not None and opt_value is not None: + reduce_option_value("--max_seconds", fuzz_value, fuzz_idx, fuzz_value + opt_value, ",") + reduce_option_value("--optimizer_max_seconds", opt_value, opt_idx, fuzz_value + opt_value) + elif fuzz_value is not None: + reduce_option_value("--max_seconds", fuzz_value, fuzz_idx, fuzz_value) + elif opt_value is not None: + reduce_option_value("--optimizer_max_seconds", opt_value, opt_idx, opt_value) + + if silent_mode is False: print("},", flush=True) + def generate_testcomp_metadata_xml(input_file, output_dir, use_m32): test_suite_dir = os.path.join(output_dir, "test-suite") os.makedirs(test_suite_dir, exist_ok=True) @@ -107,36 +148,6 @@ def fuzz(self_dir, input_file, output_dir, options, start_time, silent_mode): if not os.path.isfile(target): raise Exception("Cannot find the fuzzing target file: " + target) - time_taken = time.time() - start_time - if time_taken > 0.5: - def find_option_value_and_index(option): - try: idx = options.index(option) - except ...: return None, None - if idx >= len(options): - return None - idx += 1 - try: return int(options[idx]), idx - except: return None, None - - def reduce_option_value(name, value, idx, total_time): - if total_time > time_taken: - percentage = 1.0 - time_taken / total_time - else: - percentage = 0.0 - new_value = int(round(value * percentage)) - if silent_mode is False: print("Adjusting '" + name + "': " + str(value) + " -> " + str(new_value), flush=True) - options[idx] = str(new_value) - - fuzz_value, fuzz_idx = find_option_value_and_index("--max_seconds") - opt_value, opt_idx = find_option_value_and_index("--optimizer_max_seconds") - - if fuzz_value is not None and opt_value is not None: - reduce_option_value("--max_seconds", fuzz_value, fuzz_idx, fuzz_value + opt_value) - reduce_option_value("--optimizer_max_seconds", opt_value, opt_idx, fuzz_value + opt_value) - elif fuzz_value is not None: - reduce_option_value("--max_seconds", fuzz_value, fuzz_idx, fuzz_value) - elif opt_value is not None: - reduce_option_value("--optimizer_max_seconds", opt_value, opt_idx, opt_value) if _execute( [ os.path.join(self_dir, "tools", "@SERVER_FILE@"), @@ -160,8 +171,7 @@ def help(self_dir): print(" instead of shared memory. This option is introduced so that") print(" you do not have to use options 'path_to_target' and") print(" 'path_to_client' listed below.") - print("silent_build When specified, no messages about the building process") - print(" of the passed source C file will be printed.") + print("silent_mode When specified, no messages will be printed.") print("m32 When specified, the source C file will be compiled for") print(" 32-bit machine (cpu). Otherwise, 64-bit machine is assumed.") print("\nNext follows a listing of options of tools called from this script. When they are") @@ -192,7 +202,6 @@ def main(): skip_building = False skip_fuzzing = False silent_mode = False - silent_build = False copy_source_file = False generate_testcomp_metadata = False use_m32 = False @@ -209,7 +218,6 @@ def main(): return if arg == "--silent_mode": - silent_build = True silent_mode = True elif arg == "--progress_recording": copy_source_file = True @@ -232,8 +240,6 @@ def main(): skip_building = True elif arg == "--skip_fuzzing": skip_fuzzing = True - elif arg == "--silent_build": - silent_build = True elif arg in [ "--save_mapping", "--br_too" ]: options_instument.append(arg) elif arg == "--m32": @@ -242,9 +248,6 @@ def main(): options.append(arg) i += 1 - if input_file is None: - raise Exception("Cannot find the input file.") - if clear_output_dir is True and os.path.isdir(output_dir): shutil.rmtree(output_dir) if copy_source_file is True: @@ -254,19 +257,27 @@ def main(): old_cwd = os.getcwd() os.chdir(output_dir) try: + if input_file is None: + raise Exception("Cannot find the input file.") + if silent_mode is False: print("### starting fizzer's pipeline ###\n{", flush=True) if skip_building is False: - build(self_dir, input_file, output_dir, options_instument, use_m32, silent_build) + build(self_dir, input_file, output_dir, options_instument, use_m32, silent_mode) + adjust_timeouts(options, start_time, silent_mode) if skip_fuzzing is False: if generate_testcomp_metadata is True: generate_testcomp_metadata_xml(input_file, output_dir, use_m32) fuzz(self_dir, input_file, output_dir, options, start_time, silent_mode) + if silent_mode is False: print(",", flush=True) + if silent_mode is False: print("\"exit_code\": 0,", flush=True) except Exception as e: os.chdir(old_cwd) - print("Stopped[%ds]" % int(round(time.time() - start_time)), flush=True) + if silent_mode is False: print("\"error_message\": \"" + str(e) + "\"", flush=True) + if silent_mode is False: print("\"exit_code\": 1,", flush=True) raise e - - if silent_mode is False and ((skip_building is False and silent_build is False) or skip_fuzzing is False): - print("Done[%ds]" % int(round(time.time() - start_time)), flush=True) + finally: + if silent_mode is False: + print("\"total_time\": %.2f" % (time.time() - start_time), flush=True) + print("}", flush=True) if __name__ == "__main__": @@ -275,5 +286,4 @@ def main(): main() except Exception as e: exit_code = 1 - print("ERROR: " + str(e), flush=True) exit(exit_code) diff --git a/src/tools/server/CMakeLists.txt b/src/tools/server/CMakeLists.txt index 3253073f..6da40dbe 100644 --- a/src/tools/server/CMakeLists.txt +++ b/src/tools/server/CMakeLists.txt @@ -21,7 +21,7 @@ target_link_libraries(${THIS_TARGET_NAME} fuzzing # Duplicating because of Mingw (otherwise linking errors) utility Threads::Threads - ${Boost_LIBRARIES} + ${BOOST_LIST_OF_LIBRARIES_TO_LINK_WITH} ) if (LIBRT) diff --git a/src/tools/server/run.cpp b/src/tools/server/run.cpp index e5b2ceca..7433ab61 100644 --- a/src/tools/server/run.cpp +++ b/src/tools/server/run.cpp @@ -18,6 +18,8 @@ void run(int argc, char* argv[]) { + std::chrono::system_clock::time_point const start_time_point = std::chrono::system_clock::now(); + if (get_program_options()->has("list_stdin_models")) { for (auto const& name_and_constructor : iomodels::get_stdin_models_map()) @@ -60,8 +62,17 @@ void run(int argc, char* argv[]) if (get_program_options()->has("clear_output_dir")) { for (const auto& entry : std::filesystem::directory_iterator(output_dir)) - if (entry.is_regular_file() && entry.path().extension() == ".json") - std::filesystem::remove(entry); + if (entry.is_regular_file()) + { + auto const name{ entry.path().filename().string() }; + for (auto const& suffix : { + "_config.json", "_outcomes.json", + "_LOG.html", "_TMPROF.html", + "0.json", "1.json", "2.json", "3.json", "4.json", "5.json", "6.json", "7.json", "8.json", "9.json", + }) + if (name.ends_with(suffix)) + std::filesystem::remove(entry); + } if (std::filesystem::is_directory(output_dir / "test-suite")) for (const auto& entry : std::filesystem::directory_iterator(output_dir / "test-suite")) std::filesystem::remove(entry); @@ -105,7 +116,7 @@ void run(int argc, char* argv[]) } } - fuzzing::termination_info const terminator{ + fuzzing::termination_info terminator{ .max_executions = (natural_32_bit)std::max(0, std::stoi(get_program_options()->value("max_executions"))), .max_seconds = (natural_32_bit)std::max(0, std::stoi(get_program_options()->value("max_seconds"))) }; @@ -121,7 +132,7 @@ void run(int argc, char* argv[]) .stdout_model_name = get_program_options()->value("stdout_model") }); - fuzzing::optimizer::configuration const optimizer_config{ + fuzzing::optimizer::configuration optimizer_config{ .max_seconds = (natural_32_bit)std::max(0, std::stoi(get_program_options()->value("optimizer_max_seconds"))), .max_trace_length = (natural_32_bit)std::max(0, std::stoi(get_program_options()->value("optimizer_max_trace_length"))), .max_stdin_bytes = (iomodels::stdin_base::byte_count_type)std::max(0, std::stoi(get_program_options()->value("optimizer_max_stdin_bytes"))) @@ -154,7 +165,7 @@ void run(int argc, char* argv[]) if (get_program_options()->has("path_to_client")) { if (!get_program_options()->has("silent_mode")) - std::cout << "Communication type: network" << std::endl; + std::cout << "\"communication_type\": \"network\"," << std::endl; benchmark_executor = std::make_shared( get_program_options()->value("path_to_client"), @@ -165,13 +176,29 @@ void run(int argc, char* argv[]) else { if (!get_program_options()->has("silent_mode")) - std::cout << "Communication type: shared memory" << std::endl; + std::cout << "\"communication_type\": \"shared_memory\"," << std::endl; benchmark_executor = std::make_shared( get_program_options()->value("path_to_target") ); } + auto const startup_time = std::chrono::duration(std::chrono::system_clock::now() - start_time_point).count(); + + { + float_64_bit const total_time{ std::max((float_64_bit)(terminator.max_seconds + optimizer_config.max_seconds), 1.0) }; + float_64_bit const remaining_time{ std::max(total_time - startup_time, 0.0) }; + terminator.max_seconds = (natural_32_bit)(remaining_time * (terminator.max_seconds / total_time)); + optimizer_config.max_seconds = (natural_32_bit)(remaining_time * (optimizer_config.max_seconds / total_time)); + + if (!get_program_options()->has("silent_mode")) + std::cout << "\"fuzzing_startup\": {" << std::endl + << " \"time\": " << startup_time << ',' << std::endl + << " \"--max_seconds\": " << terminator.max_seconds << ',' << std::endl + << " \"--optimizer_max_seconds\": " << optimizer_config.max_seconds << std::endl + << "}," << std::endl; + } + fuzzing::execution_record_writer execution_record_writer{ output_dir, target_name, @@ -181,13 +208,14 @@ void run(int argc, char* argv[]) if (!get_program_options()->has("silent_mode")) { - std::cout << "Configuration for fuzzing:" << std::endl; + std::cout << "\"fuzzing_configuration\": "; fuzzing::print_fuzzing_configuration( std::cout, target_name, iomodels::iomanager::instance().get_config(), terminator ); + std::cout << ',' << std::endl; } fuzzing::log_fuzzing_configuration( target_name, @@ -201,9 +229,6 @@ void run(int argc, char* argv[]) terminator ); - if (!get_program_options()->has("silent_mode")) - std::cout << "Fuzzing was started..." << std::endl; - std::vector inputs_leading_to_boundary_violation; fuzzing::analysis_outcomes const results = fuzzing::run( *benchmark_executor, @@ -217,8 +242,10 @@ void run(int argc, char* argv[]) if (!get_program_options()->has("silent_mode")) { - std::cout << "Fuzzing was stopped. Details:" << std::endl; + std::cout << "\"fuzzing_results\": "; fuzzing::print_analysis_outcomes(std::cout, results); + std::cout << ", \"IID_analysis_info\": "; + fuzzing::save_iid_vector_analysis(std::cout, results); } fuzzing::log_analysis_outcomes(results); fuzzing::save_analysis_outcomes(output_dir, target_name, results); @@ -229,15 +256,14 @@ void run(int argc, char* argv[]) { if (!get_program_options()->has("silent_mode")) { - std::cout << "Configuration for test suite optimization:" << std::endl; + std::cout << ',' << std::endl + << "\"optimization_configuration\": "; fuzzing::print_optimization_configuration(std::cout, optimizer_config); + std::cout << ',' << std::endl; } fuzzing::log_optimization_configuration(optimizer_config); fuzzing::save_optimization_configuration(output_dir, target_name, optimizer_config); - if (!get_program_options()->has("silent_mode")) - std::cout << "Optimization was started..." << std::endl; - fuzzing::optimizer opt{ optimizer_config }; { @@ -258,7 +284,7 @@ void run(int argc, char* argv[]) if (!get_program_options()->has("silent_mode")) { - std::cout << "Optimization was stopped. Details:" << std::endl; + std::cout << "\"optimization_results\": "; fuzzing::print_optimization_outcomes(std::cout, opt_results); } fuzzing::log_optimization_outcomes(opt_results); diff --git a/src/utility/CMakeLists.txt b/src/utility/CMakeLists.txt index 328cebd5..1620e9ab 100644 --- a/src/utility/CMakeLists.txt +++ b/src/utility/CMakeLists.txt @@ -1,56 +1,11 @@ set(THIS_TARGET_NAME utility) -add_library(${THIS_TARGET_NAME} - ./include/utility/assumptions.hpp - - ./include/utility/basic_numeric_types.hpp - - ./src/bits_reference.cpp - ./include/utility/bits_reference.hpp - - ./src/bit_count.cpp - ./include/utility/bit_count.hpp - - ./include/utility/config.hpp - - ./include/utility/development.hpp - - ./include/utility/endian.hpp - - ./src/fail_message.cpp - ./include/utility/fail_message.hpp - - ./include/utility/invariants.hpp - - ./src/log.cpp - ./include/utility/log.hpp +file(GLOB "${THIS_TARGET_NAME}_HPP" "./include/${THIS_TARGET_NAME}/*.hpp") +file(GLOB "${THIS_TARGET_NAME}_CPP" "./src/*.cpp") - ./src/timestamp.cpp - ./include/utility/timestamp.hpp - - ./src/timeprof.cpp - ./include/utility/timeprof.hpp - - ./src/checked_number_operations.cpp - ./include/utility/checked_number_operations.hpp - - ./src/random.cpp - ./include/utility/random.hpp - - ./include/utility/typefn_if_then_else.hpp - - ./include/utility/msgstream.hpp - - ./include/utility/dynamic_linking.hpp - - ./include/utility/hash_combine.hpp - ./include/utility/std_pair_hash.hpp - - ./include/utility/program_options_base.hpp - ./src/program_options_base.cpp - - include/utility/math.hpp - src/math.cpp +add_library(${THIS_TARGET_NAME} + "${${THIS_TARGET_NAME}_HPP}" + "${${THIS_TARGET_NAME}_CPP}" ) set_target_properties(${THIS_TARGET_NAME} PROPERTIES diff --git a/src/utility/src/program_options_base.cpp b/src/utility/src/program_options_base.cpp index 77c1fa9a..f4c316c4 100644 --- a/src/utility/src/program_options_base.cpp +++ b/src/utility/src/program_options_base.cpp @@ -227,6 +227,4 @@ program_options_default::program_options_default(int argc, char* argv[]) { add_option("help", "Produces this help message.", "0"); add_option("version", "Prints the version string.", "0"); - add_option("data", "A root directory under which program's data are stored.", "1"); - add_value("data", "../data"); }